diff --git a/.claude/commands/apple-container.md b/.claude/commands/apple-container.md new file mode 100644 index 00000000000..ca6876ad530 --- /dev/null +++ b/.claude/commands/apple-container.md @@ -0,0 +1,46 @@ +--- +allowed-tools: Bash(container *), Bash(cargo *), Read, Grep, Glob +--- + +# Run Tests in Linux Container (Apple `container` CLI) + +Run RustPython tests inside a Linux container using Apple's `container` CLI. +**NEVER use Docker, Podman, or any other container runtime.** Only use the `container` command. + +## Arguments +- `$ARGUMENTS`: Test command to run (e.g., `test_io`, `test_codecs -v`, `test_io -v -m "test_errors"`) + +## Prerequisites + +The `container` CLI is installed via `brew install container`. +The dev image `rustpython-dev` is already built. + +## Steps + +1. **Check if the container is already running** + ```shell + container list 2>/dev/null | grep rustpython-test + ``` + +2. **Start the container if not running** + ```shell + container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source=/Users/al03219714/Projects/RustPython3,target=/workspace \ + -w /workspace rustpython-dev sleep infinity + ``` + +3. **Run the test inside the container** + ```shell + container exec rustpython-test sh -c "cargo run --release -- -m test $ARGUMENTS" + ``` + +4. **Report results** + - Show test summary (pass/fail counts, expected failures, unexpected successes) + - Highlight any new failures compared to macOS results if available + - Do NOT stop or remove the container after testing (keep it for reuse) + +## Notes +- The workspace is bind-mounted, so local code changes are immediately available +- Use `container exec rustpython-test sh -c "..."` for any command inside the container +- To rebuild after code changes, run: `container exec rustpython-test sh -c "cargo build --release"` +- To stop the container when done: `container rm -f rustpython-test` diff --git a/.claude/commands/investigate-test-failure.md b/.claude/commands/investigate-test-failure.md new file mode 100644 index 00000000000..e9d6b2d2d2c --- /dev/null +++ b/.claude/commands/investigate-test-failure.md @@ -0,0 +1,49 @@ +--- +allowed-tools: Bash(python3:*), Bash(cargo run:*), Read, Grep, Glob, Bash(git add:*), Bash(git commit:*), Bash(cargo fmt:*), Bash(git diff:*), Task +--- + +# Investigate Test Failure + +Investigate why a specific test is failing and determine if it can be fixed or needs an issue. + +## Arguments +- `$ARGUMENTS`: Failed test identifier (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + +## Steps + +1. **Analyze failure cause** + - Read the test code + - Analyze failure message/traceback + - Check related RustPython code + +2. **Verify behavior in CPython** + - Run the test with `python3 -m unittest` to confirm expected behavior + - Document the expected output + +3. **Determine fix feasibility** + - **Simple fix** (import issues, small logic bugs): Fix code → Run `cargo fmt --all` → Pre-commit review → Commit + - **Complex fix** (major unimplemented features): Collect issue info and report to user + + **Pre-commit review process**: + - Run `git diff` to see the changes + - Use Task tool with `general-purpose` subagent to review: + - Compare implementation against cpython/ source code + - Verify the fix aligns with CPython behavior + - Check for any missed edge cases + - Proceed to commit only after review passes + +4. **For complex issues - Collect issue information** + Following `.github/ISSUE_TEMPLATE/report-incompatibility.md` format: + + - **Feature**: Description of missing/broken Python feature + - **Minimal reproduction code**: Smallest code that reproduces the issue + - **CPython behavior**: Result when running with python3 + - **RustPython behavior**: Result when running with cargo run + - **Python Documentation link**: Link to relevant CPython docs + + Report collected information to the user. Issue creation is done only upon user request. + + Example issue creation command: + ``` + gh issue create --template report-incompatibility.md --title "..." --body "..." + ``` diff --git a/.claude/commands/upgrade-pylib-next.md b/.claude/commands/upgrade-pylib-next.md new file mode 100644 index 00000000000..712b79433b3 --- /dev/null +++ b/.claude/commands/upgrade-pylib-next.md @@ -0,0 +1,33 @@ +--- +allowed-tools: Skill(upgrade-pylib), Bash(gh pr list:*) +--- + +# Upgrade Next Python Library + +Find the next Python library module ready for upgrade and run `/upgrade-pylib` for it. + +## Current TODO Status + +!`cargo run --release -- scripts/update_lib todo 2>/dev/null` + +## Open Upgrade PRs + +!`gh pr list --search "Update in:title" --json number,title --template '{{range .}}#{{.number}} {{.title}}{{"\n"}}{{end}}'` + +## Instructions + +From the TODO list above, find modules matching these patterns (in priority order): + +1. `[ ] [no deps]` - Modules with no dependencies (can be upgraded immediately) +2. `[ ] [0/n]` - Modules where all dependencies are already upgraded (e.g., `[0/3]`, `[0/5]`) + +These patterns indicate modules that are ready to upgrade without blocking dependencies. + +**Important**: Skip any modules that already have an open PR in the "Open Upgrade PRs" list above. + +**After identifying a suitable module**, run: +``` +/upgrade-pylib +``` + +If no modules match these criteria, inform the user that all eligible modules have dependencies that need to be upgraded first. diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md new file mode 100644 index 00000000000..d54305d2616 --- /dev/null +++ b/.claude/commands/upgrade-pylib.md @@ -0,0 +1,157 @@ +--- +allowed-tools: Bash(git add:*), Bash(git commit:*), Bash(python3 scripts/update_lib quick:*), Bash(python3 scripts/update_lib auto-mark:*) +--- + +# Upgrade Python Library from CPython + +Upgrade a Python standard library module from CPython to RustPython. + +## Arguments +- `$ARGUMENTS`: Library name to upgrade (e.g., `inspect`, `asyncio`, `json`) + +## Important: Report Tool Issues First + +If during the upgrade process you encounter any of the following issues with `scripts/update_lib`: +- A feature that should be automated but isn't supported +- A bug or unexpected behavior in the tool +- Missing functionality that would make the upgrade easier + +**STOP the upgrade and report the issue first.** Describe: +1. What you were trying to do + - Library name + - The full command executed (e.g. python scripts/update_lib quick cpython/Lib/$ARGUMENTS.py) +2. What went wrong or what's missing +3. Expected vs actual behavior + +This helps improve the tooling for future upgrades. + +## Steps + +1. **Run quick upgrade with update_lib** + - Run: `python3 scripts/update_lib quick $ARGUMENTS` (module name) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS.py` (library file path) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS/` (library directory path) + - This will: + - Copy library files (delete existing `Lib/$ARGUMENTS.py` or `Lib/$ARGUMENTS/`, then copy from `cpython/Lib/`) + - Patch test files preserving existing RustPython markers + - Run tests and auto-mark new test failures (not regressions) + - Remove `@unittest.expectedFailure` from tests that now pass + - Create a git commit with the changes + - **Handle warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed and markers couldn't be transferred automatically. These need to be manually restored in step 2 or added in step 3. + +2. **Review git diff and restore RUSTPYTHON-specific changes** + - Run `git diff Lib/test/test_$ARGUMENTS` to review all changes + - **Only restore changes that have explicit `RUSTPYTHON` comments**. Look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Comments marking RustPython-specific code modifications + - `# TODO: RUSTPYTHON` - Comments marking tests that need work + - Code changes with inline `# ... RUSTPYTHON` comments + - **Do NOT restore other diff changes** - these are likely upstream CPython changes, not RustPython-specific modifications + - When restoring, preserve the original context and formatting + +3. **Investigate test failures with subagent** + - First, get dependent tests using the deps command: + ``` + cargo run --release -- scripts/update_lib deps $ARGUMENTS + ``` + - Look for the line `- [ ] $ARGUMENTS: test_xxx test_yyy ...` to get the direct dependent tests + - Run those tests to collect failures: + ``` + cargo run --release -- -m test test_xxx test_yyy ... 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For example, if deps output shows `- [ ] linecache: test_bdb test_inspect test_linecache test_traceback test_zipimport`, run: + ``` + cargo run --release -- -m test test_bdb test_inspect test_linecache test_traceback test_zipimport 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For each failure, use the Task tool with `general-purpose` subagent to investigate: + - Subagent should follow the `/investigate-test-failure` skill workflow + - Pass the failed test identifier as the argument (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + - If subagent can fix the issue easily: fix and commit + - If complex issue: subagent collects issue info and reports back (issue creation on user request only) + - Using subagent prevents context pollution in the main conversation + +4. **Mark remaining test failures with auto-mark** + - Run: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS.py --mark-failure` + - Or for directory: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS/ --mark-failure` + - This will: + - Run tests and mark ALL failing tests with `@unittest.expectedFailure` + - Remove `@unittest.expectedFailure` from tests that now pass + - **Note**: The `--mark-failure` flag marks all failures including regressions. Review the changes before committing. + +5. **Handle panics manually** + - If any tests cause panics/crashes (not just assertion failures), they need `@unittest.skip` instead: + ```python + @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") + def test_crashes(self): + ... + ``` + - auto-mark cannot detect panics automatically - check the test output for crash messages + +6. **Handle class-specific failures** + - If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, move the marker to the specific subclass: + ```python + # Base class - no marker here + class TestFoo: + def test_something(self): + ... + + class TestPyFoo(TestFoo, PyTest): pass + + class TestCFoo(TestFoo, CTest): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_something(self): + return super().test_something() + ``` + +7. **Commit the test fixes** + - Run: `git add -u && git commit -m "Mark failing tests"` + - This creates a separate commit for the test markers added in steps 2-6 + +## Example Usage +``` +# Using module names (recommended) +/upgrade-pylib inspect +/upgrade-pylib json +/upgrade-pylib asyncio + +# Using library paths (alternative) +/upgrade-pylib cpython/Lib/inspect.py +/upgrade-pylib cpython/Lib/json/ +``` + +## Example: Restoring RUSTPYTHON changes + +When git diff shows removed RUSTPYTHON-specific code like: +```diff +-# XXX RUSTPYTHON: we don't import _json as fresh since... +-cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) ++cjson = import_helper.import_fresh_module('json', fresh=['_json']) +``` + +You should restore the RustPython version: +```python +# XXX RUSTPYTHON: we don't import _json as fresh since... +cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) +``` + +## Notes +- The cpython/ directory should contain the CPython source that we're syncing from +- `scripts/update_lib` package handles patching and auto-marking: + - `quick` - Combined patch + auto-mark (recommended) + - `migrate` - Only migrate (patch), no test running + - `auto-mark` - Only run tests and mark failures + - `copy-lib` - Copy library files (not tests) +- The patching: + - Transfers `@unittest.expectedFailure` and `@unittest.skip` decorators with `TODO: RUSTPYTHON` markers + - Adds `import unittest # XXX: RUSTPYTHON` if needed for the decorators + - **Limitation**: If a class was restructured (e.g., method overrides removed), update_lib will warn and skip those markers +- The smart auto-mark: + - Marks NEW test failures automatically (tests that didn't exist before) + - Does NOT mark regressions (existing tests that now fail) - these are warnings + - Removes `@unittest.expectedFailure` from tests that now pass +- The script does NOT preserve all RustPython-specific changes - you must review `git diff` and restore them +- Common RustPython markers to look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Inline comments for code modifications + - `# TODO: RUSTPYTHON` - Test skip/failure markers + - Any code with `RUSTPYTHON` in comments that was removed in the diff +- **Important**: Not all changes in the git diff need to be restored. Only restore changes that have explicit `RUSTPYTHON` comments. Other changes are upstream CPython updates. diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 26921e04080..f428c42e5f6 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -1,68 +1,212 @@ +ADDOP +aftersign +argdefs argtypes asdl asname +attro augassign badcert badsyntax +baseinfo basetype +binop +bltin boolop +BUFMAX +BUILDSTDLIB bxor +byteswap cached_tsver cadata cafile +calldepth +callinfo +callproc +capath +carg cellarg cellvar cellvars +cfield +CLASSDEREF +classdict cmpop +codedepth +constevaluator +CODEUNIT +CONIN +CONOUT +CONVFUNC +convparam +copyslot +cpucount +defaultdict denom +dictbytype +DICTFLAG dictoffset +distpoint +dynload elts +eofs +evalloop excepthandler +exceptiontable +fblock +fblocks +fdescr +ffi_argtypes +fielddesc +fieldlist fileutils finalbody +finalizers +flowgraph formatfloat freevar freevars fromlist +getdict +getfunc +getiter +getsets +getslice +globalgetvar +HASARRAY +HASBITFIELD +HASPOINTER +HASSTRUCT +HASUNION heaptype +hexdigit HIGHRES +IFUNC IMMUTABLETYPE +INCREF +inlinedepth +inplace +ismine +ISPOINTER +iteminfo Itertool +keeped +kwnames kwonlyarg kwonlyargs lasti +libffi linearise +lineiterator +linetable +loadfast +localsplus +Lshift +lsprof +MAXBLOCKS maxdepth +metavars +miscompiles mult +multibytecodec +nameobj +nameop +nconsts +newargs +newfree +NEWLOCALS +newsemlockobject +nfrees nkwargs +nkwelts +Nondescriptor +noninteger +nops noraise +nseen +NSIGNALS numer +opname +opnames orelse +outparam +outparm +paramfunc +parg pathconfig patma +peepholer +phcount +platstdlib posonlyarg posonlyargs prec preinitialized +pybuilddir +pycore +pydecimal +Pyfunc +pylifecycle +pymain +pyrepl +PYTHONTRACEMALLOC +pythonw PYTHREAD_NAME +releasebuffer +repr +resinfo +Rshift SA_ONSTACK +saveall +scls +setdict +setfunc +SETREF +setresult +setslice +SLOTDEFINED +SMALLBUF SOABI +SSLEOF stackdepth +staticbase +stginfo +storefast stringlib structseq +subkwargs subparams +subscr +sval swappedbytes +templatelib +testconsole ticketer +tmptype tok_oldval tvars +typeobject +typeparam +Typeparam +typeparams +typeslots unaryop +uncollectable +Unhandle unparse unparser VARKEYWORDS varkwarg +venvlauncher +venvlaunchert +venvw +venvwlauncher +venvwlaunchert wbits weakreflist +weakrefobject webpki +winconsoleio withitem withs xstat diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 58a0e816087..2ce5d246d72 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -1,10 +1,13 @@ abiflags abstractmethods +addcompare aenter aexit aiter +altzone anext anextawaitable +annotationlib appendleft argcount arrayiterator @@ -23,6 +26,7 @@ breakpointhook cformat chunksize classcell +classmethods closefd closesocket codepoint @@ -31,6 +35,8 @@ codesize contextvar cpython cratio +ctype +ctypes dealloc debugbuild decompressor @@ -66,12 +72,15 @@ fnctl frombytes fromhex fromunicode +frozensets fset fspath fstring fstrings ftruncate genexpr +genexpressions +getargs getattro getcodesize getdefaultencoding @@ -81,14 +90,17 @@ getformat getframe getframemodulename getnewargs +getopt getpip getrandom getrecursionlimit getrefcount getsizeof getswitchinterval +getweakref getweakrefcount getweakrefs +getweakrefs getwindowsversion gmtoff groupdict @@ -101,8 +113,12 @@ idxs impls indexgroup infj +inittab +Inittab instancecheck instanceof +interpchannels +interpqueues irepeat isabstractmethod isbytes @@ -127,6 +143,7 @@ listcomp longrange lvalue mappingproxy +markupbase maskpri maxdigits MAXGROUPS @@ -142,17 +159,20 @@ mformat mro mros multiarch +mymodule namereplace nanj nbytes ncallbacks ndigits ndim +needsfree nldecoder nlocals NOARGS nonbytes Nonprintable +onceregistry origname ospath pendingcr @@ -167,7 +187,11 @@ profilefunc pycache pycodecs pycs +pydatetime pyexpat +pyio +pymain +PYTHONAPI PYTHONBREAKPOINT PYTHONDEBUG PYTHONDONTWRITEBYTECODE @@ -176,6 +200,8 @@ PYTHONHASHSEED PYTHONHOME PYTHONINSPECT PYTHONINTMAXSTRDIGITS +PYTHONIOENCODING +PYTHONNODEBUGRANGES PYTHONNOUSERSITE PYTHONOPTIMIZE PYTHONPATH @@ -197,6 +223,7 @@ readbuffer reconstructor refcnt releaselevel +reraised reverseitemiterator reverseiterator reversekeyiterator @@ -213,10 +240,13 @@ scproxy seennl setattro setcomp +setprofileallthreads setrecursionlimit setswitchinterval +settraceallthreads showwarnmsg signum +sitebuiltins slotnames STACKLESS stacklevel @@ -225,14 +255,17 @@ startpos subclassable subclasscheck subclasshook +subclassing suboffset suboffsets SUBPATTERN +subpatterns sumprod surrogateescape surrogatepass sysconf sysconfigdata +sysdict sysvars teedata thisclass @@ -259,6 +292,7 @@ warnopts weaklist weakproxy weakrefs +weakrefset winver withdata xmlcharrefreplace diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index ff2013e81a7..c3ebd61833a 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -28,6 +28,7 @@ hexf hexversion idents illumos +ilog indexmap insta keccak @@ -86,4 +87,7 @@ wasmer wasmtime widestring winapi +winresource winsock +bitvec +Bitvec diff --git a/.cspell.json b/.cspell.json index 9f88a74f96d..0a93ac35cd5 100644 --- a/.cspell.json +++ b/.cspell.json @@ -69,15 +69,18 @@ "GetSet", "groupref", "internable", + "interps", "jitted", "jitting", "lossily", "makeunicodedata", + "microbenchmark", + "microbenchmarks", "miri", "notrace", + "oparg", "openat", "pyarg", - "pyarg", "pyargs", "pyast", "PyAttr", @@ -107,8 +110,10 @@ "pystruct", "pystructseq", "pytrace", + "pytype", "reducelib", "richcompare", + "rustix", "RustPython", "significand", "struc", @@ -116,14 +121,17 @@ "sysmodule", "tracebacks", "typealiases", - "unconstructible", + "typevartuples", "unhashable", "uninit", "unraisable", "unresizable", + "varint", "wasi", "zelf", // unix + "posixshmem", + "shm", "CLOEXEC", "codeset", "endgrent", @@ -131,12 +139,17 @@ "getrusage", "nanosleep", "sigaction", + "sighandler", "WRLCK", // win32 "birthtime", "IFEXEC", // "stat" - "FIRMLINK" + "FIRMLINK", + // CPython internal names + "sysdict", + "settraceallthreads", + "setprofileallthreads" ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 339cdb69bbf..cdd54a47d5b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye +FROM rust:bullseye # Install clang RUN apt-get update \ diff --git a/.gitattributes b/.gitattributes index d1dd182a9b0..d076a34f977 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,5 +4,67 @@ Cargo.lock linguist-generated vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF -*.pck binary crates/rustpython_doc_db/src/*.inc.rs linguist-generated=true + +# Binary data types +*.aif binary +*.aifc binary +*.aiff binary +*.au binary +*.bmp binary +*.exe binary +*.icns binary +*.gif binary +*.ico binary +*.jpg binary +*.pck binary +*.pdf binary +*.png binary +*.psd binary +*.tar binary +*.wav binary +*.whl binary +*.zip binary + +# Text files that should not be subject to eol conversion +[attr]noeol -text + +Lib/test/cjkencodings/* noeol +Lib/test/tokenizedata/coding20731.py noeol +Lib/test/decimaltestdata/*.decTest noeol +Lib/test/test_email/data/*.txt noeol +Lib/test/xmltestdata/* noeol + +# Shell scripts should have LF even on Windows because of Cygwin +Lib/venv/scripts/common/activate text eol=lf +Lib/venv/scripts/posix/* text eol=lf + +# CRLF files +[attr]dos text eol=crlf + +# Language aware diff headers +# https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more +# https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81 +*.css diff=css +*.html diff=html +*.py diff=python +*.md diff=markdown + +# Generated files +# https://github.com/github/linguist/blob/master/docs/overrides.md +# +# To always hide generated files in local diffs, mark them as binary: +# $ git config diff.generated.binary true +# +[attr]generated linguist-generated=true diff=generated + +Lib/_opcode_metadata.py generated +Lib/keyword.py generated +Lib/idlelib/help.html generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated +Lib/test/levenshtein_examples.json generated +Lib/test/test_stable_abi_ctypes.py generated +Lib/token.py generated + +.github/workflows/*.lock.yml linguist-generated=true merge=ours \ No newline at end of file diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 00000000000..ad986cbd051 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw/actions/setup@v0.43.22": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.43.22", + "sha": "fe858c3e14589bf396594a0b106e634d9065823e" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b3b7b446e4a..54cc31cec43 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,131 @@ +# cspell:ignore manyhow tinyvec zeroize version: 2 updates: - package-ecosystem: cargo directory: / schedule: interval: weekly + groups: + cranelift: + patterns: + - "cranelift*" + criterion: + patterns: + - "criterion*" + futures: + patterns: + - "futures*" + get-size2: + patterns: + - "get-size*2" + iana-time-zone: + patterns: + - "iana-time-zone*" + jiff: + patterns: + - "jiff*" + lexical: + patterns: + - "lexical*" + libffi: + patterns: + - "libffi*" + malachite: + patterns: + - "malachite*" + manyhow: + patterns: + - "manyhow*" + num: + patterns: + - "num-bigint" + - "num-complex" + - "num-integer" + - "num-iter" + - "num-rational" + - "num-traits" + num_enum: + patterns: + - "num_enum*" + openssl: + patterns: + - "openssl*" + parking_lot: + patterns: + - "parking_lot*" + phf: + patterns: + - "phf*" + plotters: + patterns: + - "plotters*" + portable-atomic: + patterns: + - "portable-atomic*" + pyo3: + patterns: + - "pyo3*" + quote-use: + patterns: + - "quote-use*" + rayon: + patterns: + - "rayon*" + regex: + patterns: + - "regex*" + result-like: + patterns: + - "result-like*" + security-framework: + patterns: + - "security-framework*" + serde: + patterns: + - "serde" + - "serde_core" + - "serde_derive" + system-configuration: + patterns: + - "system-configuration*" + thiserror: + patterns: + - "thiserror*" + time: + patterns: + - "time*" + tinyvec: + patterns: + - "tinyvec*" + tls_codec: + patterns: + - "tls_codec*" + toml: + patterns: + - "toml*" + wasm-bindgen: + patterns: + - "wasm-bindgen*" + wasmtime: + patterns: + - "wasmtime*" + webpki-root: + patterns: + - "webpki-root*" + windows: + patterns: + - "windows*" + zerocopy: + patterns: + - "zerocopy*" + zeroize: + patterns: + - "zeroize*" ignore: # TODO: Remove when we use ruff from crates.io # for some reason dependabot only updates the Cargo.lock file when dealing # with git dependencies. i.e. not updating the version in Cargo.toml - - dependency-name: "ruff_*" + - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / schedule: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8539db1a27e..7f0f7b8af2d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,8 +16,13 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls - CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env + CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env + # Crates excluded from workspace builds: + # - rustpython_wasm: requires wasm target + # - rustpython-compiler-source: deprecated + # - rustpython-venvlauncher: Windows-only + WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher # Skip additional tests on Windows. They are checked on Linux and MacOS. # test_glob: many failing tests # test_pathlib: panic by surrogate chars @@ -80,6 +85,7 @@ env: test_math test_operator test_ordered_dict + test_pep646_syntax test_pow test_raise test_richcmp @@ -94,14 +100,22 @@ env: test_subclassinit test_super test_syntax + test_tstring test_tuple test_types test_unary test_unpack + test_unpack_ex test_weakref test_yield_from + ENV_POLLUTING_TESTS_COMMON: >- + ENV_POLLUTING_TESTS_LINUX: >- + ENV_POLLUTING_TESTS_MACOS: >- + ENV_POLLUTING_TESTS_WINDOWS: >- # Python version targeted by the CI. - PYTHON_VERSION: "3.13.1" + PYTHON_VERSION: "3.14.3" + X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD + X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include jobs: rust_tests: @@ -109,57 +123,80 @@ jobs: env: RUST_BACKTRACE: full name: Run rust tests - runs-on: ${{ matrix.os }} - timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 35 }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [macos-latest, ubuntu-latest, windows-2025] fail-fast: false steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable with: components: clippy - uses: Swatinem/rust-cache@v2 - # Only for OpenSSL builds - # - name: Set up the Windows environment - # shell: bash - # run: | - # git config --system core.longpaths true - # cargo install --target-dir=target -v cargo-vcpkg - # cargo vcpkg -v build - # if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool if: runner.os == 'macOS' - name: run clippy - run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm --exclude rustpython-compiler-source -- -Dwarnings + run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings - name: run rust tests - run: cargo test --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --verbose --features threading ${{ env.CARGO_ARGS }} + run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }} if: runner.os != 'macOS' - name: run rust tests - run: cargo test --workspace --exclude rustpython_wasm --exclude rustpython-jit --exclude rustpython-compiler-source --verbose --features threading ${{ env.CARGO_ARGS }} + run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }} if: runner.os == 'macOS' - name: check compilation without threading run: cargo check ${{ env.CARGO_ARGS }} + - name: check compilation without host_env (sandbox mode) + run: | + cargo check -p rustpython-vm --no-default-features --features compiler + cargo check -p rustpython-stdlib --no-default-features --features compiler + cargo build --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib + if: runner.os == 'Linux' + + - name: sandbox smoke test + run: | + target/debug/rustpython extra_tests/snippets/sandbox_smoke.py + target/debug/rustpython extra_tests/snippets/stdlib_re.py + if: runner.os == 'Linux' + + - name: Test openssl build + run: cargo build --no-default-features --features ssl-openssl + if: runner.os == 'Linux' + + # - name: Install tk-dev for tkinter build + # run: sudo apt-get update && sudo apt-get install -y tk-dev + # if: runner.os == 'Linux' + + # - name: Test tkinter build + # run: cargo build --features tkinter + # if: runner.os == 'Linux' + - name: Test example projects - run: + run: | cargo run --manifest-path example_projects/barebone/Cargo.toml cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml if: runner.os == 'Linux' - - name: prepare AppleSilicon build + - name: run update_lib tests + run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v + env: + PYTHONPATH: scripts + if: runner.os == 'Linux' + + - name: prepare Intel MacOS build uses: dtolnay/rust-toolchain@stable with: - target: aarch64-apple-darwin + target: x86_64-apple-darwin if: runner.os == 'macOS' - - name: Check compilation for Apple Silicon - run: cargo check --target aarch64-apple-darwin + - name: Check compilation for Intel MacOS + run: cargo check --target x86_64-apple-darwin if: runner.os == 'macOS' - name: prepare iOS build uses: dtolnay/rust-toolchain@stable @@ -176,7 +213,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable with: target: i686-unknown-linux-gnu @@ -190,8 +227,19 @@ jobs: with: target: aarch64-linux-android + - name: Setup Android NDK + id: setup-ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: r27 + add-to-path: true + - name: Check compilation for android run: cargo check --target aarch64-linux-android ${{ env.CARGO_ARGS_NO_SSL }} + env: + CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang + AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar + CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - uses: dtolnay/rust-toolchain@stable with: @@ -223,6 +271,13 @@ jobs: - name: Check compilation for freeBSD run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} + - uses: dtolnay/rust-toolchain@stable + with: + target: wasm32-wasip2 + + - name: Check compilation for wasip2 + run: cargo check --target wasm32-wasip2 ${{ env.CARGO_ARGS_NO_SSL }} + # - name: Prepare repository for redox compilation # run: bash scripts/redox/uncomment-cargo.sh # - name: Check compilation for Redox @@ -236,26 +291,18 @@ jobs: env: RUST_BACKTRACE: full name: Run snippets and cpython tests - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [macos-latest, ubuntu-latest, windows-2025] fail-fast: false steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - # Only for OpenSSL builds - # - name: Set up the Windows environment - # shell: bash - # run: | - # git config --system core.longpaths true - # cargo install cargo-vcpkg - # cargo vcpkg build - # if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool openssl@3 if: runner.os == 'macOS' @@ -265,7 +312,7 @@ jobs: - name: build rustpython run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit if: runner.os != 'macOS' - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: run snippets @@ -274,26 +321,116 @@ jobs: - if: runner.os == 'Linux' name: run cpython platform-independent tests - run: - target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 35 + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 45 - if: runner.os == 'Linux' name: run cpython platform-dependent tests (Linux) - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 35 + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 45 - if: runner.os == 'macOS' name: run cpython platform-dependent tests (MacOS) - run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 35 + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 45 - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) - run: - target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} timeout-minutes: 45 + - if: runner.os == 'Linux' + name: run cpython tests to check if env polluters have stopped polluting (Common/Linux) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_LINUX }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_LINUX in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + + - if: runner.os == 'macOS' + name: run cpython tests to check if env polluters have stopped polluting (Common/macOS) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_MACOS }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_MACOS in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + + - if: runner.os == 'Windows' + name: run cpython tests to check if env polluters have stopped polluting (Common/windows) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_WINDOWS }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_WINDOWS in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + - if: runner.os != 'Windows' name: check that --install-pip succeeds run: | @@ -309,27 +446,48 @@ jobs: run: | target/release/rustpython -m venv testvenv testvenv/bin/rustpython -m pip install wheel - - name: Check whats_left is not broken - run: python -I whats_left.py + - if: runner.os != 'macOS' + name: Check whats_left is not broken + shell: bash + run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading,jit" + - if: runner.os == 'macOS' # TODO fix jit on macOS + name: Check whats_left is not broken (macOS) + shell: bash + run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading" # no jit on macOS for now lint: - name: Check Rust code with clippy + name: Lint Rust & Python code runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 + - uses: actions/setup-python@v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Check for redundant test patches + run: python scripts/check_redundant_patches.py + - uses: dtolnay/rust-toolchain@stable with: - components: clippy + components: clippy + - name: run clippy on wasm run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings - name: Ensure docs generate no warnings - run: cargo doc + run: cargo doc --locked + + - name: Ensure Lib/_opcode_metadata is updated + run: | + python scripts/generate_opcode_metadata.py + if [ -n "$(git status --porcelain)" ]; then + exit 1 + fi - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 + uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 with: - version: "0.14.9" + version: "0.14.11" args: "--version" - run: ruff check --diff @@ -348,7 +506,7 @@ jobs: - name: spell checker uses: streetsidesoftware/cspell-action@v8 with: - files: '**/*.rs' + files: "**/*.rs" incremental_files_only: true miri: @@ -357,23 +515,23 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: - NIGHTLY_CHANNEL: nightly + NIGHTLY_CHANNEL: nightly-2026-02-11 # https://github.com/rust-lang/miri/issues/4855 steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ env.NIGHTLY_CHANNEL }} - components: miri + toolchain: ${{ env.NIGHTLY_CHANNEL }} + components: miri - uses: Swatinem/rust-cache@v2 - name: Run tests under miri run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test env: - # miri-ignore-leaks because the type-object circular reference means that there will always be - # a memory leak, at least until we have proper cyclic gc - MIRIFLAGS: '-Zmiri-ignore-leaks' + # miri-ignore-leaks because the type-object circular reference means that there will always be + # a memory leak, at least until we have proper cyclic gc + MIRIFLAGS: "-Zmiri-ignore-leaks" wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} @@ -381,7 +539,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -392,7 +550,7 @@ jobs: wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt @@ -444,7 +602,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml index d1a457c73e6..3f3402270ea 100644 --- a/.github/workflows/comment-commands.yml +++ b/.github/workflows/comment-commands.yml @@ -7,7 +7,7 @@ on: jobs: issue_assign: if: (!github.event.issue.pull_request) && github.event.comment.body == 'take' - runs-on: ubuntu-latest + runs-on: ubuntu-slim concurrency: group: ${{ github.actor }}-issue-assign diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 59d664e0ea1..f451984fb53 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -5,12 +5,15 @@ on: push: paths: - .github/workflows/cron-ci.yaml + pull_request: + paths: + - .github/workflows/cron-ci.yaml name: Periodic checks/tasks env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit - PYTHON_VERSION: "3.13.1" + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit + PYTHON_VERSION: "3.14.3" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. @@ -21,15 +24,15 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true @@ -39,6 +42,7 @@ jobs: - name: Prepare code coverage data run: cargo llvm-cov report --lcov --output-path='codecov.lcov' - name: Upload to Codecov + if: ${{ github.event_name != 'pull_request' }} uses: codecov/codecov-action@v5 with: file: ./codecov.lcov @@ -49,7 +53,7 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -58,6 +62,7 @@ jobs: env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: upload tests data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} GITHUB_ACTOR: ${{ github.actor }} @@ -80,20 +85,21 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: build rustpython run: cargo build --release --verbose - name: Collect what is left data run: | - chmod +x ./whats_left.py - ./whats_left.py --features "ssl,sqlite" > whats_left.temp + chmod +x ./scripts/whats_left.py + ./scripts/whats_left.py --features "ssl,sqlite" > whats_left.temp env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: Upload data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} GITHUB_ACTOR: ${{ github.actor }} @@ -137,11 +143,11 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.1.0 + - uses: actions/setup-python@v6.2.0 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - run: cargo install cargo-criterion - name: build benchmarks run: cargo build --release --benches @@ -162,6 +168,7 @@ jobs: mv reports/* . rmdir reports - name: upload benchmark data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} run: | @@ -173,7 +180,11 @@ jobs: cd website rm -rf ./assets/criterion cp -r ../target/criterion ./assets/criterion - git add ./assets/criterion + printf '{\n "generated_at": "%s",\n "rustpython_commit": "%s",\n "rustpython_ref": "%s"\n}\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "${{ github.sha }}" \ + "${{ github.ref_name }}" > ./_data/criterion-metadata.json + git add ./assets/criterion ./_data/criterion-metadata.json if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then git push fi diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml new file mode 100644 index 00000000000..e4e7bd4ee45 --- /dev/null +++ b/.github/workflows/lib-deps-check.yaml @@ -0,0 +1,123 @@ +name: Lib Dependencies Check + +on: + pull_request_target: + types: [opened, synchronize, reopened] + paths: + - "Lib/**" + +concurrency: + group: lib-deps-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.14.3" + +jobs: + check_deps: + permissions: + pull-requests: write + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout base branch + uses: actions/checkout@v6.0.2 + with: + # Use base branch for scripts (security: don't run PR code with elevated permissions) + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 0 + + - name: Fetch PR head + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + + - name: Checkout PR Lib files + run: | + # Checkout only Lib/ directory from PR head for accurate comparison + git checkout ${{ github.event.pull_request.head.sha }} -- Lib/ + + - name: Checkout CPython + run: | + git clone --depth 1 --branch "v${{ env.PYTHON_VERSION }}" https://github.com/python/cpython.git cpython + + - name: Get changed Lib files + id: changed-files + run: | + # Get the list of changed files under Lib/ + changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py' | head -50) + echo "Changed files:" + echo "$changed" + + # Extract unique module names + modules="" + for file in $changed; do + if [[ "$file" == Lib/test/* ]]; then + # Test files: Lib/test/test_pydoc.py -> test_pydoc, Lib/test/test_pydoc/foo.py -> test_pydoc + module=$(echo "$file" | sed -E 's|^Lib/test/||; s|\.py$||; s|/.*||') + # Skip non-test files in test/ (e.g., support.py, __init__.py) + if [[ ! "$module" == test_* ]]; then + continue + fi + else + # Lib files: Lib/foo.py -> foo, Lib/foo/__init__.py -> foo + module=$(echo "$file" | sed -E 's|^Lib/||; s|/__init__\.py$||; s|\.py$||; s|/.*||') + fi + if [[ -n "$module" && ! " $modules " =~ " $module " ]]; then + modules="$modules $module" + fi + done + + modules=$(echo "$modules" | xargs) # trim whitespace + echo "Detected modules: $modules" + echo "modules=$modules" >> $GITHUB_OUTPUT + + - name: Setup Python + if: steps.changed-files.outputs.modules != '' + uses: actions/setup-python@v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Run deps check + if: steps.changed-files.outputs.modules != '' + id: deps-check + run: | + # Run deps for all modules at once + python scripts/update_lib deps ${{ steps.changed-files.outputs.modules }} --depth 2 > /tmp/deps_output.txt 2>&1 || true + + # Read output for GitHub Actions + echo "deps_output<> $GITHUB_OUTPUT + cat /tmp/deps_output.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Check if there's any meaningful output + if [ -s /tmp/deps_output.txt ]; then + echo "has_output=true" >> $GITHUB_OUTPUT + else + echo "has_output=false" >> $GITHUB_OUTPUT + fi + + - name: Post comment + if: steps.deps-check.outputs.has_output == 'true' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: lib-deps-check + number: ${{ github.event.pull_request.number }} + recreate: true + message: | + ## 📦 Library Dependencies + + The following Lib/ modules were modified. Here are their dependencies: + + ${{ steps.deps-check.outputs.deps_output }} + + **Legend:** + - `[+]` path exists in CPython + - `[x]` up-to-date, `[ ]` outdated + + - name: Remove comment if no Lib changes + if: steps.changed-files.outputs.modules == '' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: lib-deps-check + number: ${{ github.event.pull_request.number }} + delete: true diff --git a/.github/workflows/pr-auto-commit.yaml b/.github/workflows/pr-auto-commit.yaml index 0cbd2bfefb0..e27cfe2ce16 100644 --- a/.github/workflows/pr-auto-commit.yaml +++ b/.github/workflows/pr-auto-commit.yaml @@ -14,7 +14,6 @@ concurrency: jobs: auto_format: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} permissions: contents: write pull-requests: write @@ -22,7 +21,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout PR branch - uses: actions/checkout@v6.0.1 + uses: actions/checkout@v6.0.2 with: ref: ${{ github.event.pull_request.head.sha }} repository: ${{ github.event.pull_request.head.repo.full_name }} @@ -34,41 +33,77 @@ jobs: with: components: rustfmt + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + echo "" > /tmp/committed_commands.txt + - name: Run cargo fmt run: | echo "Running cargo fmt --all on PR #${{ github.event.pull_request.number }}" cargo fmt --all + if [ -n "$(git status --porcelain)" ]; then + git add -u + git commit -m "Auto-format: cargo fmt --all" + echo "- \`cargo fmt --all\`" >> /tmp/committed_commands.txt + fi - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 + uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 with: - version: "0.14.9" + version: "0.14.11" args: "--version" - - run: ruff format - - run: ruff check --select I --fix + - name: Run ruff format + run: | + ruff format + if [ -n "$(git status --porcelain)" ]; then + git add -u + git commit -m "Auto-format: ruff format" + echo "- \`ruff format\`" >> /tmp/committed_commands.txt + fi - - name: Configure git + - name: Run ruff check import sorting run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + ruff check --select I --fix + if [ -n "$(git status --porcelain)" ]; then + git add -u + git commit -m "Auto-format: ruff check --select I --fix" + echo "- \`ruff check --select I --fix\`" >> /tmp/committed_commands.txt + fi + + - name: Run generate_opcode_metadata.py + run: | + python scripts/generate_opcode_metadata.py + if [ -n "$(git status --porcelain)" ]; then + git add -u + git commit -m "Auto-generate: generate_opcode_metadata.py" + echo "- \`python scripts/generate_opcode_metadata.py\`" >> /tmp/committed_commands.txt + fi - name: Check for changes id: check-changes run: | - if [ -n "$(git status --porcelain)" ]; then + if [ "$(git rev-parse HEAD)" != "${{ github.event.pull_request.head.sha }}" ]; then echo "has_changes=true" >> $GITHUB_OUTPUT else echo "has_changes=false" >> $GITHUB_OUTPUT fi - - name: Commit and push formatting changes + - name: Push formatting changes if: steps.check-changes.outputs.has_changes == 'true' run: | - git add -u - git commit -m "Auto-format: cargo fmt --all" git push origin HEAD:${{ github.event.pull_request.head.ref }} + - name: Read committed commands + id: committed-commands + if: steps.check-changes.outputs.has_changes == 'true' + run: | + echo "list<> $GITHUB_OUTPUT + cat /tmp/committed_commands.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Comment on PR if: steps.check-changes.outputs.has_changes == 'true' uses: marocchino/sticky-pull-request-comment@v2 @@ -77,7 +112,8 @@ jobs: message: | **Code has been automatically formatted** - The code in this PR has been formatted using `cargo fmt --all`. + The code in this PR has been formatted using: + ${{ steps.committed-commands.outputs.list }} Please pull the latest changes before pushing again: ```bash git pull origin ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3efff295c37..9a6d0ad9838 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,8 @@ permissions: env: CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl + X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD + X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include jobs: build: @@ -42,28 +44,21 @@ jobs: target: aarch64-apple-darwin # - runner: macos-latest # target: x86_64-apple-darwin - - runner: windows-latest + - runner: windows-2025 target: x86_64-pc-windows-msvc -# - runner: windows-latest +# - runner: windows-2025 # target: i686-pc-windows-msvc -# - runner: windows-latest +# - runner: windows-2025 # target: aarch64-pc-windows-msvc fail-fast: false steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - uses: cargo-bins/cargo-binstall@main - name: Set up Environment shell: bash run: rustup target add ${{ matrix.platform.target }} - - name: Set up Windows Environment - shell: bash - run: | - git config --global core.longpaths true - cargo install --target-dir=target -v cargo-vcpkg - cargo vcpkg -v build - if: runner.os == 'Windows' - name: Set up MacOS Environment run: brew install autoconf automake libtool if: runner.os == 'macOS' @@ -93,7 +88,7 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.1 + - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1 @@ -144,6 +139,8 @@ jobs: if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} needs: [build, build-wasm] steps: + - uses: actions/checkout@v6.0.2 + - name: Download Binary Artifacts uses: actions/download-artifact@v7.0.0 with: @@ -151,6 +148,10 @@ jobs: pattern: rustpython-* merge-multiple: true + - name: Create Lib Archive + run: | + zip -r bin/rustpython-lib.zip Lib/ + - name: List Binaries run: | ls -lah bin/ @@ -174,6 +175,7 @@ jobs: --repo="$GITHUB_REPOSITORY" \ --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \ --target="$tag" \ + --notes "⚠️ **Important**: To run RustPython, you must download both the binary for your platform AND the \`rustpython-lib.zip\` archive. Extract the Lib directory from the archive to the same location as the binary, or set the \`RUSTPYTHONPATH\` environment variable to point to the Lib directory." \ --generate-notes \ $PRERELEASE_ARG \ bin/rustpython-release-* diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index c580e7d0eaf..27fd13cd212 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -2,6 +2,7 @@ name: Update doc DB permissions: contents: write + pull-requests: write on: workflow_dispatch: @@ -9,11 +10,11 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.13.9" - ref: - description: Branch to commit to (leave empty for current branch) + default: "3.14.3" + base-ref: + description: Base branch to create the update branch from type: string - default: "" + default: "main" defaults: run: @@ -29,13 +30,13 @@ jobs: - windows-latest - macos-latest steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 with: persist-credentials: false sparse-checkout: | crates/doc - - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ inputs.python-version }} @@ -54,11 +55,14 @@ jobs: runs-on: ubuntu-latest needs: generate steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ inputs.base-ref }} token: ${{ secrets.AUTO_COMMIT_PAT }} + - name: Create update branch + run: git switch -c update-doc-${{ inputs.python-version }} + - name: Download generated doc DBs uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -96,13 +100,18 @@ jobs: retention-days: 7 overwrite: true - - name: Commit and push (non-main branches only) - if: github.ref != 'refs/heads/main' && inputs.ref != 'main' + - name: Commit, push and create PR + env: + GH_TOKEN: ${{ secrets.AUTO_COMMIT_PAT }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if [ -n "$(git status --porcelain)" ]; then git add crates/doc/src/data.inc.rs git commit -m "Update doc DB for CPython ${{ inputs.python-version }}" - git push + git push -u origin HEAD + gh pr create \ + --base ${{ inputs.base-ref }} \ + --title "Update doc DB for CPython ${{ inputs.python-version }}" \ + --body "Auto-generated by update-doc-db workflow." fi diff --git a/.github/workflows/update-libs-status.yaml b/.github/workflows/update-libs-status.yaml new file mode 100644 index 00000000000..d3430dcd14b --- /dev/null +++ b/.github/workflows/update-libs-status.yaml @@ -0,0 +1,90 @@ +name: Updated libs status + +on: + push: + branches: + - main + paths: + - "Lib/**" + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + PYTHON_VERSION: "v3.14.3" + ISSUE_ID: "6839" + +jobs: + update-issue: + runs-on: ubuntu-latest + steps: + - name: Clone RustPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: rustpython + persist-credentials: "false" + sparse-checkout: |- + Lib + scripts/update_lib + + + - name: Clone CPython ${{ env.PYTHON_VERSION }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: python/cpython + path: cpython + ref: ${{ env.PYTHON_VERSION }} + persist-credentials: "false" + sparse-checkout: | + Lib + + - name: Get current date + id: current_date + run: | + now=$(date -u +"%Y-%m-%d %H:%M:%S") + echo "date=$now" >> "$GITHUB_OUTPUT" + + - name: Write body prefix + run: | + cat > body.txt < + + ## Summary + + Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ env.PYTHON_VERSION }}\`. + + Previous versions' issues as reference + - 3.13: #5529 + + + + ## Details + + ${{ steps.current_date.outputs.date }} (UTC) + \`\`\`shell + $ python3 scripts/update_lib todo --done + \`\`\` + EOF + + - name: Run todo + run: python3 rustpython/scripts/update_lib todo --cpython cpython --lib rustpython/Lib --done >> body.txt + + - name: Update GH issue + run: gh issue edit ${{ env.ISSUE_ID }} --body-file ../body.txt + env: + GH_TOKEN: ${{ github.token }} + working-directory: rustpython + + diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml new file mode 100644 index 00000000000..4d8bd37a005 --- /dev/null +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -0,0 +1,1093 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.43.22). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Pick an out-of-sync Python library from the todo list and upgrade it +# by running `scripts/update_lib quick`, then open a pull request. +# +# frontmatter-hash: 3129480d6628afe028911bb8b31b6bb3b5eb251e395f00ed0677922cd21727cb + +name: "Upgrade Python Library" +"on": + workflow_dispatch: + inputs: + name: + description: Module name to upgrade (leave empty to auto-pick) + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Upgrade Python Library" + +env: + ISSUE_ID: "6839" + PYTHON_VERSION: v3.14.3 + +# Cache configuration from frontmatter was processed and added to the main job steps + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + with: + destination: /opt/gh-aw/actions + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "upgrade-pylib.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + model: ${{ steps.generate_aw_info.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + # Cache configuration from frontmatter processed below + - name: Cache (cpython-lib-${{ env.PYTHON_VERSION }}) + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: | + cpython-lib- + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Generate agentic run info + id: generate_aw_info + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const fs = require('fs'); + + const awInfo = { + engine_id: "copilot", + engine_name: "GitHub Copilot CLI", + model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", + version: "", + agent_version: "0.0.409", + cli_version: "v0.43.22", + workflow_name: "Upgrade Python Library", + experimental: false, + supports_tools_allowlist: true, + supports_http_transport: true, + run_id: context.runId, + run_number: context.runNumber, + run_attempt: process.env.GITHUB_RUN_ATTEMPT, + repository: context.repo.owner + '/' + context.repo.repo, + ref: context.ref, + sha: context.sha, + actor: context.actor, + event_name: context.eventName, + staged: false, + allowed_domains: ["defaults","rust","python"], + firewall_enabled: true, + awf_version: "v0.16.4", + awmg_version: "", + steps: { + firewall: "squid" + }, + created_at: new Date().toISOString() + }; + + // Write to /tmp/gh-aw directory to avoid inclusion in PR + const tmpPath = '/tmp/gh-aw/aw_info.json'; + fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); + console.log('Generated aw_info.json at:', tmpPath); + console.log(JSON.stringify(awInfo, null, 2)); + + // Set model as output for reuse in other steps/jobs + core.setOutput('model', awInfo.model); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.16.4 + - name: Determine automatic lockdown mode for GitHub MCP server + id: determine-automatic-lockdown + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.16.4 ghcr.io/github/gh-aw-firewall/squid:0.16.4 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_pull_request":{"expires":30},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub pull request to propose code changes. Use this after making file edits to submit them for review and merging. The PR will be created from the current branch with your committed changes. For code review comments on an existing PR, use create_pull_request_review_comment instead. CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"Update \". Labels [pylib-sync] will be automatically added.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed PR description in Markdown. Include what changes were made, why, testing notes, and any breaking changes. Do NOT repeat the title as a heading.", + "type": "string" + }, + "branch": { + "description": "Source branch name containing the changes. If omitted, uses the current working branch.", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the PR (e.g., 'enhancement', 'bugfix'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "description": "Concise PR title describing the changes. Follow repository conventions (e.g., conventional commits). The title appears as the main heading.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_pull_request" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_pull_request": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "branch": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "env": { + "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Generate workflow overview + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); + await generateWorkflowOverview(core); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GitHub API Access Instructions + + The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. + + + To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. + + Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). + + **IMPORTANT - temporary_id format rules:** + - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) + - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i + - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) + - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 + - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) + - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 + - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate + + Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. + + Discover available tools from the safeoutputs MCP server. + + **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. + + **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. + + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + {{#runtime-import .github/workflows/upgrade-pylib.md}} + GH_AW_PROMPT_EOF + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_ENV_ISSUE_ID: process.env.GH_AW_ENV_ISSUE_ID, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_INPUTS_NAME: process.env.GH_AW_GITHUB_EVENT_INPUTS_NAME, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 45 + run: | + set -o pipefail + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.16.4 --skip-pull \ + -- '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' \ + 2>&1 | tee /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + /tmp/gh-aw/aw.patch + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: 1 + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Handle Create Pull Request Error + id: handle_create_pr_error + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_create_pr_error.cjs'); + await main(); + - name: Update reaction comment with completion status + id: conclusion + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_COMMENT_REPO: ${{ needs.activation.outputs.comment_repo }} + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.result }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/notify_comment_error.cjs'); + await main(); + + detection: + needs: agent + if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + runs-on: ubuntu-latest + permissions: {} + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + timeout-minutes: 10 + outputs: + success: ${{ steps.parse_results.outputs.success }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + with: + destination: /opt/gh-aw/actions + - name: Download agent artifacts + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: agent-artifacts + path: /tmp/gh-aw/threat-detection/ + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: agent-output + path: /tmp/gh-aw/threat-detection/ + - name: Echo agent output types + env: + AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + run: | + echo "Agent output-types: $AGENT_OUTPUT_TYPES" + - name: Setup threat detection + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "Upgrade Python Library" + WORKFLOW_DESCRIPTION: "Pick an out-of-sync Python library from the todo list and upgrade it\nby running `scripts/update_lib quick`, then open a pull request." + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" + mkdir -p /tmp/ + mkdir -p /tmp/gh-aw/ + mkdir -p /tmp/gh-aw/agent/ + mkdir -p /tmp/gh-aw/sandbox/agent/logs/ + copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_results + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + + safe_outputs: + needs: + - activation + - agent + - detection + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + runs-on: ubuntu-slim + permissions: + contents: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + outputs: + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Download patch artifact + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: agent-artifacts + path: /tmp/gh-aw/ + - name: Checkout repository + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + token: ${{ github.token }} + persist-credentials: false + fetch-depth: 1 + - name: Configure Git credentials + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GIT_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GIT_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_pull_request\":{\"base_branch\":\"${{ github.ref_name }}\",\"draft\":false,\"expires\":30,\"labels\":[\"pylib-sync\"],\"max\":1,\"max_patch_size\":1024,\"title_prefix\":\"Update \"},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + diff --git a/.github/workflows/upgrade-pylib.md b/.github/workflows/upgrade-pylib.md new file mode 100644 index 00000000000..c2e82d94828 --- /dev/null +++ b/.github/workflows/upgrade-pylib.md @@ -0,0 +1,134 @@ +--- +description: | + Pick an out-of-sync Python library from the todo list and upgrade it + by running `scripts/update_lib quick`, then open a pull request. + +on: + workflow_dispatch: + inputs: + name: + description: "Module name to upgrade (leave empty to auto-pick)" + required: false + type: string + +timeout-minutes: 45 + +permissions: + contents: read + issues: read + pull-requests: read + +network: + allowed: + - defaults + - rust + - python + +engine: copilot + +runtimes: + python: + version: "3.12" + +tools: + bash: + - ":*" + edit: + github: + toolsets: [repos, issues, pull_requests] + read-only: true + +safe-outputs: + create-pull-request: + title-prefix: "Update " + labels: [pylib-sync] + draft: false + expires: 30 + +cache: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: + - cpython-lib- + +env: + PYTHON_VERSION: "v3.14.3" + ISSUE_ID: "6839" +--- + +# Upgrade Python Library + +You are an automated maintenance agent for RustPython, a Python 3 interpreter written in Rust. Your task is to upgrade one out-of-sync Python standard library module from CPython. + +## Step 1: Set up the environment + +The CPython source may already be cached. Check if the `cpython` directory exists and has the correct version: + +```bash +if [ -d "cpython/Lib" ]; then + echo "CPython cache hit, skipping clone" +else + git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +fi +``` + +## Step 2: Determine module name + +Run this script to determine the module name: + +```bash +MODULE_NAME="${{ github.event.inputs.name }}" +if [ -z "$MODULE_NAME" ]; then + echo "No module specified, running todo to find one..." + python3 scripts/update_lib todo + echo "Pick one module from the list above that is marked [ ], has no unmet deps, and has a small Δ number." + echo "Do NOT pick: opcode, datetime, random, hashlib, tokenize, pdb, _pyrepl, concurrent, asyncio, multiprocessing, ctypes, idlelib, tkinter, shutil, tarfile, email, unittest" +else + echo "Module specified by user: $MODULE_NAME" +fi +``` + +If the script printed "Module specified by user: ...", use that exact name. If it printed the todo list, pick one suitable module from it. + +## Step 3: Run the upgrade + +Run the quick upgrade command. This will copy the library from CPython, migrate test files preserving RustPython markers, auto-mark test failures, and create a git commit: + +```bash +python3 scripts/update_lib quick +``` + +This takes a while because it builds RustPython (`cargo build --release`) and runs tests to determine which ones pass or fail. + +If the command fails, report the error and stop. Do not try to fix Rust code or modify test files manually. + +## Step 4: Verify the result + +After the script succeeds, check what changed: + +```bash +git log -1 --stat +git diff HEAD~1 --stat +``` + +Make sure the commit was created with the correct message format: `Update from `. + +## Step 5: Create the pull request + +Create a pull request. Reference issue #${{ env.ISSUE_ID }} in the body but do **NOT** use keywords that auto-close issues (Fix, Close, Resolve). + +Use this format for the PR body: + +``` +## Summary + +Upgrade `` from CPython $PYTHON_VERSION. + +Part of #$ISSUE_ID + +## Changes + +- Updated `Lib/` from CPython +- Migrated test files preserving RustPython markers +- Auto-marked test failures with `@expectedFailure` +``` diff --git a/.gitignore b/.gitignore index fea93ace80a..c03ae1a997e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* !Lib/test/data/README +cpython/ + diff --git a/.vscode/launch.json b/.vscode/launch.json index fa6f96c5fd8..6e00e14aff2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,18 +16,6 @@ }, "cwd": "${workspaceFolder}" }, - { - "type": "lldb", - "request": "launch", - "name": "Debug executable 'rustpython' without SSL", - "preLaunchTask": "Build RustPython Debug without SSL", - "program": "target/debug/rustpython", - "args": [], - "env": { - "RUST_BACKTRACE": "1" - }, - "cwd": "${workspaceFolder}" - }, { "type": "lldb", "request": "launch", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 18a3d6010d1..50a52aaf8be 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,28 +1,12 @@ { "version": "2.0.0", "tasks": [ - { - "label": "Build RustPython Debug without SSL", - "type": "shell", - "command": "cargo", - "args": [ - "build", - ], - "problemMatcher": [ - "$rustc", - ], - "group": { - "kind": "build", - "isDefault": true, - }, - }, { "label": "Build RustPython Debug", "type": "shell", "command": "cargo", "args": [ "build", - "--features=ssl" ], "problemMatcher": [ "$rustc", diff --git a/.github/copilot-instructions.md b/AGENTS.md similarity index 71% rename from .github/copilot-instructions.md rename to AGENTS.md index 1db02dd17a9..b407328cffb 100644 --- a/.github/copilot-instructions.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This document provides guidelines for working with GitHub Copilot when contribut ## Project Overview -RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide: +RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0+ compatibility. The project aims to provide: - A complete Python-3 environment entirely in Rust (not CPython bindings) - A clean implementation without compatibility hacks @@ -30,6 +30,14 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0 - `jit/` - Experimental JIT compiler implementation - `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically) +## AI Agent Rules + +**CRITICAL: Git Operations** +- NEVER create pull requests directly without explicit user permission +- NEVER push commits to remote without explicit user permission +- Always ask the user before performing any git operations that affect the remote repository +- Commits can be created locally when requested, but pushing and PR creation require explicit approval + ## Important Development Notes ### Running Python Code @@ -73,17 +81,28 @@ The `Lib/` directory contains Python standard library files copied from the CPyt - `unittest.skip("TODO: RustPython ")` - `unittest.expectedFailure` with `# TODO: RUSTPYTHON ` comment +### Clean Build + +When you modify bytecode instructions, a full clean is required: + +```bash +rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r +``` + ### Testing ```bash # Run Rust unit tests -cargo test --workspace --exclude rustpython_wasm +cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher + +# Run Python snippets tests (debug mode recommended for faster compilation) +cargo run -- extra_tests/snippets/builtin_bytes.py -# Run Python snippets tests +# Run all Python snippets tests with pytest cd extra_tests pytest -v -# Run the Python test module +# Run the Python test module (release mode recommended for better performance) cargo run --release -- -m test ${TEST_MODULE} cargo run --release -- -m test test_unicode # to test test_unicode.py @@ -91,9 +110,11 @@ cargo run --release -- -m test test_unicode # to test test_unicode.py cargo run --release -- -m test test_unicode -k test_unicode_escape ``` +**Note**: For `extra_tests/snippets` tests, use debug mode (`cargo run`) as compilation is faster. For `unittest` (`-m test`), use release mode (`cargo run --release`) for better runtime performance. + ### Determining What to Implement -Run `./whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities. +Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities. ## Coding Guidelines @@ -104,6 +125,36 @@ Run `./whats_left.py` to get a list of unimplemented methods, which is helpful w - Follow Rust best practices for error handling and memory management - Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust +#### Comments + +- Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code. +- Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value. + +#### Avoid Duplicate Code in Branches + +When branches differ only in a value but share common logic, extract the differing value first, then call the common logic once. + +**Bad:** +```rust +let result = if condition { + let msg = format!("message A: {x}"); + some_function(msg, shared_arg) +} else { + let msg = format!("message B"); + some_function(msg, shared_arg) +}; +``` + +**Good:** +```rust +let msg = if condition { + format!("message A: {x}") +} else { + format!("message B") +}; +let result = some_function(msg, shared_arg); +``` + ### Python Code - **IMPORTANT**: In most cases, Python code should not be edited. Bug fixes should be made through Rust code modifications only @@ -176,6 +227,16 @@ cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdli cargo run --features jit ``` +### Linux Build and Debug on macOS + +See the "Testing on Linux from macOS" section in [DEVELOPMENT.md](DEVELOPMENT.md#testing-on-linux-from-macos). + +### Building venvlauncher (Windows) + +See DEVELOPMENT.md "CPython Version Upgrade Checklist" section. + +**IMPORTANT**: All 4 venvlauncher binaries use the same source code. Do NOT add multiple `[[bin]]` entries to Cargo.toml. Build once and copy with different names. + ## Test Code Modification Rules **CRITICAL: Test code modification restrictions** diff --git a/Cargo.lock b/Cargo.lock index 2873f3a529f..5b13350cb56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -166,7 +175,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -240,9 +249,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-fips-sys" -version = "0.13.10" +version = "0.13.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57900537c00a0565a35b63c4c281b372edfc9744b072fd4a3b414350a8f5ed48" +checksum = "df6ea8e07e2df15b9f09f2ac5ee2977369b06d116f0c4eb5fa4ad443b73c7f53" dependencies = [ "bindgen 0.72.1", "cc", @@ -254,9 +263,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.1" +version = "1.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -266,9 +275,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.34.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" dependencies = [ "cc", "cmake", @@ -284,9 +293,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bindgen" @@ -294,7 +303,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -314,7 +323,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -336,9 +345,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "blake2" @@ -380,9 +389,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" dependencies = [ "allocator-api2", ] @@ -395,9 +404,9 @@ checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2" @@ -443,9 +452,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.47" +version = "1.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" dependencies = [ "find-msvc-tools", "jobserver", @@ -482,9 +491,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", "js-sys", @@ -543,18 +552,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstyle", "clap_lex", @@ -562,9 +571,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" [[package]] name = "clipboard-win" @@ -577,9 +586,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" dependencies = [ "cc", ] @@ -691,9 +700,9 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971376deb1edf5e9c0ac77ef00479d740ce7a60e6181adb0648afe1dc7b8f4" +checksum = "9aaf0bd3cb9d164f355ecaa41e57de67ada7d5f3f451c8d29376bf6612059036" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -702,42 +711,42 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30054f4aef4d614d37f27d5b77e36e165f0b27a71563be348e7c9fcfac41eed8" +checksum = "0377b13bf002a0774fcccac4f1102a10f04893d24060cf4b7350c87e4cbb647c" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beab56413879d4f515e08bcf118b1cb85f294129bb117057f573d37bfbb925a" +checksum = "cfa027979140d023b25bf7509fb7ede3a54c3d3871fb5ead4673c4b633f671a2" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d054747549a69b264d5299c8ca1b0dd45dc6bd0ee43f1edfcc42a8b12952c7a" +checksum = "618e4da87d9179a70b3c2f664451ca8898987aa6eb9f487d16988588b5d8cc40" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b92d481b77a7dc9d07c96e24a16f29e0c9c27d042828fdf7e49e54ee9819bf" +checksum = "db53764b5dad233b37b8f5dc54d3caa9900c54579195e00f17ea21f03f71aaa7" [[package]] name = "cranelift-codegen" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eeccfc043d599b0ef1806942707fc51cdd1c3965c343956dc975a55d82a920f" +checksum = "4ae927f1d8c0abddaa863acd201471d56e7fc6c3925104f4861ed4dc3e28b421" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -761,9 +770,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1174cdb9d9d43b2bdaa612a07ed82af13db9b95526bc2c286c2aec4689bcc038" +checksum = "d3fcf1e3e6757834bd2584f4cbff023fcc198e9279dcb5d684b4bb27a9b19f54" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -773,33 +782,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d572be73fae802eb115f45e7e67a9ed16acb4ee683b67c4086768786545419a" +checksum = "205dcb9e6ccf9d368b7466be675ff6ee54a63e36da6fe20e72d45169cf6fd254" [[package]] name = "cranelift-control" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1587465cc84c5cc793b44add928771945f3132bbf6b3621ee9473c631a87156" +checksum = "108eca9fcfe86026054f931eceaf57b722c1b97464bf8265323a9b5877238817" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063b83448b1343e79282c3c7cbda7ed5f0816f0b763a4c15f7cecb0a17d87ea6" +checksum = "a0d96496910065d3165f84ff8e1e393916f4c086f88ac8e1b407678bc78735aa" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4461c2d2ca48bc72883f5f5c3129d9aefac832df1db824af9db8db3efee109" +checksum = "e303983ad7e23c850f24d9c41fc3cb346e1b930f066d3966545e4c98dac5c9fb" dependencies = [ "cranelift-codegen", "log", @@ -809,15 +818,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd811b25e18f14810d09c504e06098acc1d9dbfa24879bf0d6b6fb44415fc66" +checksum = "24b0cf8d867d891245836cac7abafb0a5b0ea040a019d720702b3b8bcba40bfa" [[package]] name = "cranelift-jit" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01527663ba63c10509d7c87fd1f8495d21170ba35bf714f57271495689d8fde5" +checksum = "dcf1e35da6eca2448395f483eb172ce71dd7842f7dc96f44bb8923beafe43c6d" dependencies = [ "anyhow", "cranelift-codegen", @@ -830,14 +839,14 @@ dependencies = [ "region", "target-lexicon", "wasmtime-internal-jit-icache-coherence", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "cranelift-module" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72328edb49aeafb1655818c91c476623970cb7b8a89ffbdadd82ce7d13dedc1d" +checksum = "792ba2a54100e34f8a36e3e329a5207cafd1f0918a031d34695db73c163fdcc7" dependencies = [ "anyhow", "cranelift-codegen", @@ -846,9 +855,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2417046989d8d6367a55bbab2e406a9195d176f4779be4aa484d645887217d37" +checksum = "e24b641e315443e27807b69c440fe766737d7e718c68beb665a2d69259c77bf3" dependencies = [ "cranelift-codegen", "libc", @@ -857,9 +866,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.126.1" +version = "0.128.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d039de901c8d928222b8128e1b9a9ab27b82a7445cb749a871c75d9cb25c57d" +checksum = "a4e378a54e7168a689486d67ee1f818b7e5356e54ae51a1d7a53f4f13f7f8b7a" [[package]] name = "crc32fast" @@ -872,9 +881,9 @@ dependencies = [ [[package]] name = "criterion" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ "alloca", "anes", @@ -897,9 +906,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools 0.13.0", @@ -957,9 +966,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "der" @@ -1181,14 +1190,14 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "flagset" @@ -1234,13 +1243,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ - "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs 0.6.0", ] [[package]] @@ -1270,6 +1278,31 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1282,9 +1315,9 @@ dependencies = [ [[package]] name = "get-size-derive2" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff47daa61505c85af126e9dd64af6a342a33dc0cccfe1be74ceadc7d352e6efd" +checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" dependencies = [ "attribute-derive", "quote", @@ -1293,13 +1326,14 @@ dependencies = [ [[package]] name = "get-size2" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac7bb8710e1f09672102be7ddf39f764d8440ae74a9f4e30aaa4820dcdffa4af" +checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" dependencies = [ "compact_str", "get-size-derive2", "hashbrown 0.16.1", + "ordermap", "smallvec", ] @@ -1324,9 +1358,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", @@ -1458,23 +1492,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", ] -[[package]] -name = "indoc" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" -dependencies = [ - "rustversion", -] - [[package]] name = "inout" version = "0.1.4" @@ -1487,13 +1512,14 @@ dependencies = [ [[package]] name = "insta" -version = "1.44.3" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c943d4415edd8153251b6f197de5eb1640e56d84e8d9159bea190421c73698" +checksum = "248b42847813a1550dafd15296fd9748c651d0c32194559dbc05d804d54b21e8" dependencies = [ "console", "once_cell", "similar", + "tempfile", ] [[package]] @@ -1540,15 +1566,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" dependencies = [ "jiff-static", "log", @@ -1559,9 +1585,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" dependencies = [ "proc-macro2", "quote", @@ -1602,9 +1628,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -1612,12 +1638,12 @@ dependencies = [ [[package]] name = "junction" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8" +checksum = "642883fdc81cf2da15ee8183fa1d2c7da452414dd41541a0f3e1428069345447" dependencies = [ "scopeguard", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1680,15 +1706,15 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libffi" -version = "4.1.2" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0feebbe0ccd382a2790f78d380540500d7b78ed7a3498b68fcfbc1593749a94" +checksum = "0498fe5655f857803e156523e644dcdcdc3b3c7edda42ea2afdae2e09b2db87b" dependencies = [ "libc", "libffi-sys", @@ -1696,9 +1722,9 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "3.3.3" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c6c6e17136d4bc439d43a2f3c6ccf0731cccc016d897473a29791d3c2160c3" +checksum = "71d4f1d4ce15091955144350b75db16a96d4a63728500122706fb4d29a26afbb" dependencies = [ "cc", ] @@ -1725,25 +1751,25 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "libc", ] [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" dependencies = [ "cc", "pkg-config", @@ -1752,11 +1778,11 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" dependencies = [ - "zlib-rs", + "zlib-rs 0.5.5", ] [[package]] @@ -1821,9 +1847,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c91cb6071ed9ac48669d3c79bd2792db596c7e542dbadd217b385bb359f42d" +checksum = "a8b6f86fdbb1eb9955946be91775239dfcb0acdb1a51bb07d5fc9b8c854f5ccd" dependencies = [ "hashbrown 0.16.1", "itertools 0.14.0", @@ -1833,9 +1859,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff3af5010102f29f2ef4ee6f7b1c5b3f08a6c261b5164e01c41cf43772b6f90" +checksum = "67fcd6e504ffc67db2b3c6d5e90e08054646e2b04f42115a5460bf1c1e37d3bc" dependencies = [ "malachite-base", "malachite-nz", @@ -1846,9 +1872,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9ecf4dd76246fd622de4811097966106aa43f9cd7cc36cb85e774fe84c8adc" +checksum = "0197a2f5cfee19d59178e282985c6ca79a9233e26a2adcf40acb693896aa09f6" dependencies = [ "itertools 0.14.0", "libm", @@ -1858,9 +1884,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7bc9d9adf5b0a7999d84f761c809bec3dc46fe983e4de547725d2b7730462a0" +checksum = "be2add95162aede090c48f0ee51bea7d328847ce3180aa44588111f846cc116b" dependencies = [ "itertools 0.14.0", "malachite-base", @@ -1914,9 +1940,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -1954,11 +1980,11 @@ dependencies = [ [[package]] name = "mt19937" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" +checksum = "56bc7ea7924ea1a79a9e817d0483e39295424cf2b1276cf2b968f9a6c9b63b54" dependencies = [ - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -1976,7 +2002,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -1989,7 +2015,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -2027,9 +2053,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -2080,6 +2106,15 @@ dependencies = [ "syn", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.8.1" @@ -2113,7 +2148,7 @@ version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -2135,9 +2170,9 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-src" @@ -2167,6 +2202,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" +[[package]] +name = "ordermap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa78c92071bbd3628c22b1a964f7e0eb201dc1456555db072beb1662ecd6715" +dependencies = [ + "indexmap", +] + [[package]] name = "page_size" version = "0.6.0" @@ -2315,6 +2359,18 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs5" version = "0.7.1" @@ -2389,9 +2445,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" [[package]] name = "portable-atomic-util" @@ -2440,53 +2496,65 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "pymath" -version = "0.0.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b66ab66a8610ce209d8b36cd0fecc3a15c494f715e0cb26f0586057f293abc9" +checksum = "bbfb6723b732fc7f0b29a0ee7150c7f70f947bf467b8c3e82530b13589a78b4c" dependencies = [ "libc", + "libm", + "malachite-bigint", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "pyo3" -version = "0.27.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "14c738662e2181be11cb82487628404254902bb3225d8e9e99c31f3ef82a405c" dependencies = [ - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "f9ca0864a7dd3c133a7f3f020cbff2e12e88420da854c35540fd20ce2d60e435" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "9dfc1956b709823164763a34cc42bbfd26b8730afa77809a3df8b94a3ae3b059" dependencies = [ "libc", "pyo3-build-config", @@ -2494,9 +2562,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "29dc660ad948bae134d579661d08033fbb1918f4529c3bbe3257a68f2009ddf2" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2506,9 +2574,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "e78cd6c6d718acfcedf26c3d21fe0f053624368b0d44298c55d7138fde9331f7" dependencies = [ "heck", "proc-macro2", @@ -2519,9 +2587,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -2591,7 +2659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -2611,7 +2679,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -2620,14 +2688,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] @@ -2664,7 +2732,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] @@ -2673,16 +2741,16 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", "thiserror 1.0.69", ] [[package]] name = "regalloc2" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e249c660440317032a71ddac302f25f1d5dff387667bcc3978d1f77aa31ac34" +checksum = "08effbc1fa53aaebff69521a5c05640523fab037b34a4a2c109506bc938246fa" dependencies = [ "allocator-api2", "bumpalo", @@ -2762,7 +2830,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted 0.9.0", "windows-sys 0.52.0", @@ -2771,28 +2839,27 @@ dependencies = [ [[package]] name = "ruff_python_ast" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +source = "git+https://github.com/astral-sh/ruff.git?rev=a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d#a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" dependencies = [ "aho-corasick", - "bitflags 2.10.0", + "bitflags 2.11.0", "compact_str", "get-size2", "is-macro", - "itertools 0.14.0", "memchr", "ruff_python_trivia", "ruff_source_file", "ruff_text_size", "rustc-hash", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] name = "ruff_python_parser" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +source = "git+https://github.com/astral-sh/ruff.git?rev=a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d#a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "bstr", "compact_str", "get-size2", @@ -2810,7 +2877,7 @@ dependencies = [ [[package]] name = "ruff_python_trivia" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +source = "git+https://github.com/astral-sh/ruff.git?rev=a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d#a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" dependencies = [ "itertools 0.14.0", "ruff_source_file", @@ -2821,7 +2888,7 @@ dependencies = [ [[package]] name = "ruff_source_file" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +source = "git+https://github.com/astral-sh/ruff.git?rev=a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d#a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" dependencies = [ "memchr", "ruff_text_size", @@ -2830,7 +2897,7 @@ dependencies = [ [[package]] name = "ruff_text_size" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +source = "git+https://github.com/astral-sh/ruff.git?rev=a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d#a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" dependencies = [ "get-size2", ] @@ -2852,11 +2919,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -2865,9 +2932,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "aws-lc-rs", "once_cell", @@ -2879,9 +2946,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2900,9 +2967,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "zeroize", ] @@ -2936,9 +3003,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "aws-lc-rs", "ring", @@ -2974,7 +3041,7 @@ name = "rustpython-codegen" version = "0.4.0" dependencies = [ "ahash", - "bitflags 2.10.0", + "bitflags 2.11.0", "indexmap", "insta", "itertools 0.14.0", @@ -2989,7 +3056,7 @@ dependencies = [ "rustpython-compiler-core", "rustpython-literal", "rustpython-wtf8", - "thiserror 2.0.17", + "thiserror 2.0.18", "unicode_names2 2.0.0", ] @@ -2998,7 +3065,7 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "getrandom 0.3.4", "itertools 0.14.0", @@ -3010,7 +3077,6 @@ dependencies = [ "nix 0.30.1", "num-complex", "num-traits", - "once_cell", "parking_lot", "radium", "rustpython-literal", @@ -3031,14 +3097,14 @@ dependencies = [ "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "itertools 0.14.0", "lz4_flex", "malachite-bigint", @@ -3098,7 +3164,8 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror 2.0.17", + "rustpython-wtf8", + "thiserror 2.0.18", ] [[package]] @@ -3127,7 +3194,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "criterion", "num_enum", "optional", @@ -3154,10 +3221,12 @@ dependencies = [ "digest", "dns-lookup", "dyn-clone", + "flame", "flate2", "foreign-types-shared", "gethostname", "hex", + "hmac", "indexmap", "itertools 0.14.0", "libc", @@ -3182,11 +3251,12 @@ dependencies = [ "page_size", "parking_lot", "paste", + "pbkdf2", "pem-rfc7468 1.0.0", "phf 0.13.1", "pkcs8", "pymath", - "rand_core 0.9.3", + "rand_core 0.9.5", "rustix", "rustls", "rustls-native-certs", @@ -3224,13 +3294,20 @@ dependencies = [ "xz2", ] +[[package]] +name = "rustpython-venvlauncher" +version = "0.4.0" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "rustpython-vm" version = "0.4.0" dependencies = [ "ahash", "ascii", - "bitflags 2.10.0", + "bitflags 2.11.0", "bstr", "caseless", "cfg-if", @@ -3261,10 +3338,10 @@ dependencies = [ "num-traits", "num_cpus", "num_enum", - "once_cell", "optional", "parking_lot", "paste", + "psm", "result-like", "ruff_python_ast", "ruff_python_parser", @@ -3281,11 +3358,11 @@ dependencies = [ "rustyline", "scoped-tls", "scopeguard", - "serde", + "serde_core", "static_assertions", "strum", "strum_macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "thread_local", "timsort", "uname", @@ -3320,8 +3397,8 @@ dependencies = [ "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", - "serde", "serde-wasm-bindgen", + "serde_core", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -3339,7 +3416,7 @@ version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "clipboard-win", "fd-lock", @@ -3357,15 +3434,15 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "safe_arch" -version = "0.9.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629516c85c29fe757770fa03f2074cf1eac43d44c02a3de9fc2ef7b0e207dfdd" +checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" dependencies = [ "bytemuck", ] @@ -3426,7 +3503,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3486,22 +3563,22 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" dependencies = [ "serde_core", ] @@ -3574,9 +3651,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "similar" @@ -3586,9 +3663,15 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" @@ -3598,9 +3681,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", "windows-sys 0.60.2", @@ -3654,9 +3737,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.111" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -3691,7 +3774,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3708,9 +3791,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" [[package]] name = "tcl-sys" @@ -3721,6 +3804,19 @@ dependencies = [ "shared-build", ] +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "termios" version = "0.3.3" @@ -3747,11 +3843,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -3767,9 +3863,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -3798,30 +3894,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -3890,9 +3986,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.8" +version = "0.9.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" +checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" dependencies = [ "indexmap", "serde_core", @@ -3905,27 +4001,27 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.3" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "twox-hash" @@ -4140,12 +4236,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "untrusted" version = "0.7.1" @@ -4166,9 +4256,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ "atomic", "js-sys", @@ -4205,18 +4295,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", @@ -4227,11 +4317,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -4240,9 +4331,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4250,9 +4341,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ "bumpalo", "proc-macro2", @@ -4263,39 +4354,39 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "39.0.1" +version = "41.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ccd36e25390258ce6720add639ffe5a7d81a5c904350aa08f5bbc60433d22" +checksum = "bada5ca1cc47df7d14100e2254e187c2486b426df813cea2dd2553a7469f7674" dependencies = [ "anyhow", "cfg-if", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "wasmtime-internal-math" -version = "39.0.1" +version = "41.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1b856e1bbf0230ab560ba4204e944b141971adc4e6cdf3feb6979c1a7b7953" +checksum = "cf6f615d528eda9adc6eefb062135f831b5215c348f4c3ec3e143690c730605b" dependencies = [ "libm", ] [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -4303,18 +4394,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" +checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -4332,9 +4423,9 @@ dependencies = [ [[package]] name = "wide" -version = "0.8.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ca908d26e4786149c48efcf6c0ea09ab0e06d1fe3c17dc1b4b0f1ca4a7e788" +checksum = "ac11b009ebeae802ed758530b6496784ebfee7a87b9abfbcaf3bbe25b814eb25" dependencies = [ "bytemuck", "safe_arch", @@ -4675,9 +4766,9 @@ checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" [[package]] name = "winresource" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b021990998587d4438bb672b5c5f034cbc927f51b45e3807ab7323645ef4899" +checksum = "e287ced0f21cd11f4035fe946fd3af145f068d1acb708afd248100f89ec7432d" dependencies = [ "toml", "version_check", @@ -4691,9 +4782,9 @@ checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" [[package]] name = "x509-cert" @@ -4711,9 +4802,9 @@ dependencies = [ [[package]] name = "x509-parser" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3e137310115a65136898d2079f003ce33331a6c4b0d51f1531d1be082b6425" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ "asn1-rs", "data-encoding", @@ -4722,15 +4813,15 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] [[package]] name = "xml" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df5825faced2427b2da74d9100f1e2e93c533fff063506a81ede1cf517b2e7e" +checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" [[package]] name = "xz2" @@ -4743,18 +4834,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.30" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" +checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.30" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" +checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" dependencies = [ "proc-macro2", "quote", @@ -4772,9 +4863,9 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", @@ -4783,6 +4874,18 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zlib-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" + +[[package]] +name = "zmij" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" diff --git a/Cargo.toml b/Cargo.toml index f68e6e68157..6356eef8c0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,12 +10,13 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls"] +default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"] +host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdio = ["rustpython-vm/stdio"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] -flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] +flame-it = ["rustpython-vm/flame-it", "rustpython-stdlib/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] @@ -53,7 +54,8 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.27", features = ["auto-initialize"] } +pyo3 = { version = "0.28", features = ["auto-initialize"] } +rustpython-stdlib = { workspace = true } [[bench]] name = "execution" @@ -91,16 +93,6 @@ lto = "thin" # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END -# Used only on Windows to build the vcpkg dependencies -[package.metadata.vcpkg] -git = "https://github.com/microsoft/vcpkg" -# The revision of the vcpkg repository to use -# https://github.com/microsoft/vcpkg/tags -rev = "2025.09.17" - -[package.metadata.vcpkg.target] -x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] } - [package.metadata.packager] product-name = "RustPython" identifier = "com.rustpython.rustpython" @@ -126,12 +118,13 @@ members = [ ".", "crates/*", ] +exclude = ["pymath"] [workspace.package] version = "0.4.0" authors = ["RustPython Team"] edition = "2024" -rust-version = "1.89.0" +rust-version = "1.93.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" @@ -151,20 +144,21 @@ rustpython-sre_engine = { path = "crates/sre_engine", version = "0.4.0" } rustpython-wtf8 = { path = "crates/wtf8", version = "0.4.0" } rustpython-doc = { path = "crates/doc", version = "0.4.0" } -# Ruff tag 0.14.1 is based on commit 2bffef59665ce7d2630dfd72ee99846663660db8 +# Ruff tag 0.15.1 is based on commit a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d # at the time of this capture. We use the commit hash to ensure reproducible builds. -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "a2f11d239f91cf8daedb0764ec15fcfe29c5ae6d" } phf = { version = "0.13.1", default-features = false, features = ["macros"]} ahash = "0.8.12" ascii = "1.1" -bitflags = "2.9.4" +bitflags = "2.11.0" bstr = "1" +bytes = "1.11.1" cfg-if = "1.0" -chrono = { version = "0.4.42", default-features = false, features = ["clock", "oldtime", "std"] } +chrono = { version = "0.4.43", default-features = false, features = ["clock", "oldtime", "std"] } constant_time_eq = "0.4" criterion = { version = "0.8", features = ["html_reports"] } crossbeam-utils = "0.8.21" @@ -172,36 +166,35 @@ flame = "0.2.2" getrandom = { version = "0.3", features = ["std"] } glob = "0.3" hex = "0.4.3" -indexmap = { version = "2.11.3", features = ["std"] } -insta = "1.44" +indexmap = { version = "2.13.0", features = ["std"] } +insta = "1.46" itertools = "0.14.0" is-macro = "0.3.7" -junction = "1.3.0" -libc = "0.2.178" -libffi = "4.1" +junction = "1.4.1" +libc = "0.2.180" +libffi = "5" log = "0.4.29" nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.8" -malachite-q = "0.8" -malachite-base = "0.8" -memchr = "2.7.4" +malachite-bigint = "0.9.1" +malachite-q = "0.9.1" +malachite-base = "0.9.1" +memchr = "2.8.0" num-complex = "0.4.6" num-integer = "0.1.46" num-traits = "0.2" num_enum = { version = "0.7", default-features = false } optional = "0.5" -once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" -proc-macro2 = "1.0.93" -pymath = "0.0.2" -quote = "1.0.38" +proc-macro2 = "1.0.105" +pymath = { version = "0.1.5", features = ["mul_add", "malachite-bigint", "complex"] } +quote = "1.0.44" radium = "1.1.1" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } -rustix = { version = "1.0", features = ["event"] } +rustix = { version = "1.1", features = ["event"] } rustyline = "17.0.1" -serde = { version = "1.0.225", default-features = false } +serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] } schannel = "0.1.28" scoped-tls = "1" scopeguard = "1" @@ -232,6 +225,9 @@ unsafe_op_in_unsafe_fn = "deny" elided_lifetimes_in_paths = "warn" [workspace.lints.clippy] +alloc_instead_of_core = "warn" +std_instead_of_alloc = "warn" +std_instead_of_core = "warn" perf = "warn" style = "warn" complexity = "warn" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aa7d99eef33..7573f0f2640 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,13 +19,13 @@ The contents of the Development Guide include: RustPython requires the following: -- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023) +- Rust latest stable version (e.g 1.92.0 as of Jan 7 2026) - To check Rust version: `rustc --version` - If you have `rustup` on your system, enter to update to the latest stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.13 or higher +- CPython version 3.14 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as @@ -65,7 +65,7 @@ $ pytest -v Rust unit tests can be run with `cargo`: ```shell -$ cargo test --workspace --exclude rustpython_wasm +$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher ``` Python unit tests can be run by compiling RustPython and running the test module: @@ -95,6 +95,41 @@ To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely: $ cargo run --release -- -m test test_cmath -v ``` +### Testing on Linux from macOS + +You can test RustPython on Linux from macOS using Apple's `container` CLI. + +**Setup (one-time):** + +```shell +# Install container CLI +$ brew install container + +# Disable Rosetta requirement for arm64-only builds +$ defaults write com.apple.container.defaults build.rosetta -bool false + +# Build the development image +$ container build --arch arm64 -t rustpython-dev -f .devcontainer/Dockerfile . +``` + +**Running tests:** + +```shell +# Start a persistent container in background (8GB memory, 4 CPUs for compilation) +$ container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source=$(pwd),target=/workspace \ + -w /workspace rustpython-dev sleep infinity + +# Run tests inside the container +$ container exec rustpython-test sh -c "cargo run --release -- -m test test_ensurepip" + +# Run any command +$ container exec rustpython-test sh -c "cargo test --workspace" + +# Stop and remove the container when done +$ container rm -f rustpython-test +``` + ## Profiling To profile RustPython, build it in `release` mode with the `flame-it` feature. @@ -118,19 +153,19 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i Understanding a new codebase takes time. Here's a brief view of the repository's structure: -- `compiler/src`: python compilation to bytecode - - `core/src`: python bytecode representation in rust structures - - `parser/src`: python lexing, parsing and ast -- `derive/src`: Rust language extensions and macros specific to rustpython +- `crates/compiler/src`: python compilation to bytecode + - `crates/compiler-core/src`: python bytecode representation in rust structures +- `crates/derive/src` and `crates/derive-impl/src`: Rust language extensions and macros specific to rustpython - `Lib`: Carefully selected / copied files from CPython sourcecode. This is the python side of the standard library. - `test`: CPython test suite -- `vm/src`: python virtual machine +- `crates/vm/src`: python virtual machine - `builtins`: Builtin functions and types - `stdlib`: Standard library parts implemented in rust. - `src`: using the other subcrates to bring rustpython to life. -- `wasm`: Binary crate and resources for WebAssembly build -- `extra_tests`: extra integration test snippets as a supplement to `Lib/test` +- `crates/wasm`: Binary crate and resources for WebAssembly build +- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`. + Add new RustPython-only regression tests here; do not place new tests under `Lib/test`. ## Understanding Internals @@ -140,9 +175,9 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`). The top-level `rustpython` binary depends on several lower-level crates including: -- `rustpython-parser` (implementation in `compiler/parser/src`) -- `rustpython-compiler` (implementation in `compiler/src`) -- `rustpython-vm` (implementation in `vm/src`) +- `ruff_python_parser` and `ruff_python_ast` (external dependencies from the Ruff project) +- `rustpython-compiler` (implementation in `crates/compiler/src`) +- `rustpython-vm` (implementation in `crates/vm/src`) Together, these crates provide the functions of a programming language and enable a line of code to go through a series of steps: @@ -153,31 +188,26 @@ enable a line of code to go through a series of steps: - compile the AST into bytecode - execute the bytecode in the virtual machine (VM). -### rustpython-parser +### Parser and AST -This crate contains the lexer and parser to convert a line of code to -an Abstract Syntax Tree (AST): +RustPython uses the Ruff project's parser and AST implementation: -- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens -- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses - the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax - tree are Rust structs and enums. - - The Parser relies on `LALRPOP`, a Rust parser generator framework. The - LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`. - - More information on parsers and a tutorial can be found in the - [LALRPOP book](https://lalrpop.github.io/lalrpop/). -- AST: `compiler/ast/` implements in Rust the Python types and expressions - represented by the AST nodes. +- Parser: `ruff_python_parser` is used to convert Python source code into tokens + and parse them into an Abstract Syntax Tree (AST) +- AST: `ruff_python_ast` provides the Rust types and expressions represented by + the AST nodes +- These are external dependencies maintained by the Ruff project +- For more information, visit the [Ruff GitHub repository](https://github.com/astral-sh/ruff) ### rustpython-compiler The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax Tree) to bytecode. The implementation of the compiler is found in the -`compiler/src` directory. The compiler implements Python's symbol table, +`crates/compiler/src` directory. The compiler implements Python's symbol table, ast->bytecode compiler, and bytecode optimizer in Rust. -Implementation of bytecode structure in Rust is found in the `compiler/core/src` -directory. `compiler/core/src/bytecode.rs` contains the representation of +Implementation of bytecode structure in Rust is found in the `crates/compiler-core/src` +directory. `crates/compiler-core/src/bytecode.rs` contains the representation of instructions and operations in Rust. Further information about Python's bytecode instructions can be found in the [Python documentation](https://docs.python.org/3/library/dis.html#bytecodes). @@ -185,14 +215,14 @@ bytecode instructions can be found in the ### rustpython-vm The `rustpython-vm` crate has the important job of running the virtual machine that -executes Python's instructions. The `vm/src` directory contains code to +executes Python's instructions. The `crates/vm/src` directory contains code to implement the read and evaluation loop that fetches and dispatches instructions. This directory also contains the implementation of the -Python Standard Library modules in Rust (`vm/src/stdlib`). In Python -everything can be represented as an object. The `vm/src/builtins` directory holds +Python Standard Library modules in Rust (`crates/vm/src/stdlib`). In Python +everything can be represented as an object. The `crates/vm/src/builtins` directory holds the Rust code used to represent different Python objects and their methods. The core implementation of what a Python object is can be found in -`vm/src/object/core.rs`. +`crates/vm/src/object/core.rs`. ### Code generation diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py new file mode 100644 index 00000000000..1c8741b5a55 --- /dev/null +++ b/Lib/_ast_unparse.py @@ -0,0 +1,1161 @@ +# This module contains ``ast.unparse()``, defined here +# to improve the import time for the ``ast`` module. +import sys +from _ast import * +from ast import NodeVisitor +from contextlib import contextmanager, nullcontext +from enum import IntEnum, auto, _simple_enum + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + +@_simple_enum(IntEnum) +class _Precedence: + """Precedence table that originated from python grammar.""" + + NAMED_EXPR = auto() # := + TUPLE = auto() # , + YIELD = auto() # 'yield', 'yield from' + TEST = auto() # 'if'-'else', 'lambda' + OR = auto() # 'or' + AND = auto() # 'and' + NOT = auto() # 'not' + CMP = auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = auto() + BOR = EXPR # '|' + BXOR = auto() # '^' + BAND = auto() # '&' + SHIFT = auto() # '<<', '>>' + ARITH = auto() # '+', '-' + TERM = auto() # '*', '@', '/', '%', '//' + FACTOR = auto() # unary '+', '-', '~' + POWER = auto() # '**' + AWAIT = auto() # 'await' + ATOM = auto() + + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + +class Unparser(NodeVisitor): + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded.""" + + def __init__(self): + self._source = [] + self._precedences = {} + self._type_ignores = {} + self._indent = 0 + self._in_try_star = False + self._in_interactive = False + + def interleave(self, inter, f, seq): + """Call f on each item in seq, calling inter() in between.""" + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + + def items_view(self, traverser, items): + """Traverse and separate the given *items* with a comma and append it to + the buffer. If *items* is a single item sequence, a trailing comma + will be added.""" + if len(items) == 1: + traverser(items[0]) + self.write(",") + else: + self.interleave(lambda: self.write(", "), traverser, items) + + def maybe_newline(self): + """Adds a newline if it isn't the start of generated source""" + if self._source: + self.write("\n") + + def maybe_semicolon(self): + """Adds a "; " delimiter if it isn't the start of generated source""" + if self._source: + self.write("; ") + + def fill(self, text="", *, allow_semicolon=True): + """Indent a piece of text and append it, according to the current + indentation level, or only delineate with semicolon if applicable""" + if self._in_interactive and not self._indent and allow_semicolon: + self.maybe_semicolon() + self.write(text) + else: + self.maybe_newline() + self.write(" " * self._indent + text) + + def write(self, *text): + """Add new source parts""" + self._source.extend(text) + + @contextmanager + def buffered(self, buffer = None): + if buffer is None: + buffer = [] + + original_source = self._source + self._source = buffer + yield buffer + self._source = original_source + + @contextmanager + def block(self, *, extra = None): + """A context manager for preparing the source for blocks. It adds + the character':', increases the indentation on enter and decreases + the indentation on exit. If *extra* is given, it will be directly + appended after the colon character. + """ + self.write(":") + if extra: + self.write(extra) + self._indent += 1 + yield + self._indent -= 1 + + @contextmanager + def delimit(self, start, end): + """A context manager for preparing the source for expressions. It adds + *start* to the buffer and enters, after exit it adds *end*.""" + + self.write(start) + yield + self.write(end) + + def delimit_if(self, start, end, condition): + if condition: + return self.delimit(start, end) + else: + return nullcontext() + + def require_parens(self, precedence, node): + """Shortcut to adding precedence related parens""" + return self.delimit_if("(", ")", self.get_precedence(node) > precedence) + + def get_precedence(self, node): + return self._precedences.get(node, _Precedence.TEST) + + def set_precedence(self, precedence, *nodes): + for node in nodes: + self._precedences[node] = precedence + + def get_raw_docstring(self, node): + """If a docstring node is found in the body of the *node* parameter, + return that docstring node, None otherwise. + + Logic mirrored from ``_PyAST_GetDocString``.""" + if not isinstance( + node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) + ) or len(node.body) < 1: + return None + node = node.body[0] + if not isinstance(node, Expr): + return None + node = node.value + if isinstance(node, Constant) and isinstance(node.value, str): + return node + + def get_type_comment(self, node): + comment = self._type_ignores.get(node.lineno) or node.type_comment + if comment is not None: + return f" # type: {comment}" + + def traverse(self, node): + if isinstance(node, list): + for item in node: + self.traverse(item) + else: + super().visit(node) + + # Note: as visit() resets the output text, do NOT rely on + # NodeVisitor.generic_visit to handle any nodes (as it calls back in to + # the subclass visit() method, which resets self._source to an empty list) + def visit(self, node): + """Outputs a source code string that, if converted back to an ast + (using ast.parse) will generate an AST equivalent to *node*""" + self._source = [] + self.traverse(node) + return "".join(self._source) + + def _write_docstring_and_traverse_body(self, node): + if (docstring := self.get_raw_docstring(node)): + self._write_docstring(docstring) + self.traverse(node.body[1:]) + else: + self.traverse(node.body) + + def visit_Module(self, node): + self._type_ignores = { + ignore.lineno: f"ignore{ignore.tag}" + for ignore in node.type_ignores + } + try: + self._write_docstring_and_traverse_body(node) + finally: + self._type_ignores.clear() + + def visit_Interactive(self, node): + self._in_interactive = True + try: + self._write_docstring_and_traverse_body(node) + finally: + self._in_interactive = False + + def visit_FunctionType(self, node): + with self.delimit("(", ")"): + self.interleave( + lambda: self.write(", "), self.traverse, node.argtypes + ) + + self.write(" -> ") + self.traverse(node.returns) + + def visit_Expr(self, node): + self.fill() + self.set_precedence(_Precedence.YIELD, node.value) + self.traverse(node.value) + + def visit_NamedExpr(self, node): + with self.require_parens(_Precedence.NAMED_EXPR, node): + self.set_precedence(_Precedence.ATOM, node.target, node.value) + self.traverse(node.target) + self.write(" := ") + self.traverse(node.value) + + def visit_Import(self, node): + self.fill("import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_ImportFrom(self, node): + self.fill("from ") + self.write("." * (node.level or 0)) + if node.module: + self.write(node.module) + self.write(" import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_Assign(self, node): + self.fill() + for target in node.targets: + self.set_precedence(_Precedence.TUPLE, target) + self.traverse(target) + self.write(" = ") + self.traverse(node.value) + if type_comment := self.get_type_comment(node): + self.write(type_comment) + + def visit_AugAssign(self, node): + self.fill() + self.traverse(node.target) + self.write(" " + self.binop[node.op.__class__.__name__] + "= ") + self.traverse(node.value) + + def visit_AnnAssign(self, node): + self.fill() + with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): + self.traverse(node.target) + self.write(": ") + self.traverse(node.annotation) + if node.value: + self.write(" = ") + self.traverse(node.value) + + def visit_Return(self, node): + self.fill("return") + if node.value: + self.write(" ") + self.traverse(node.value) + + def visit_Pass(self, node): + self.fill("pass") + + def visit_Break(self, node): + self.fill("break") + + def visit_Continue(self, node): + self.fill("continue") + + def visit_Delete(self, node): + self.fill("del ") + self.interleave(lambda: self.write(", "), self.traverse, node.targets) + + def visit_Assert(self, node): + self.fill("assert ") + self.traverse(node.test) + if node.msg: + self.write(", ") + self.traverse(node.msg) + + def visit_Global(self, node): + self.fill("global ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Nonlocal(self, node): + self.fill("nonlocal ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Await(self, node): + with self.require_parens(_Precedence.AWAIT, node): + self.write("await") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Yield(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_YieldFrom(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield from ") + if not node.value: + raise ValueError("Node can't be used without a value attribute.") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Raise(self, node): + self.fill("raise") + if not node.exc: + if node.cause: + raise ValueError(f"Node can't use cause without an exception.") + return + self.write(" ") + self.traverse(node.exc) + if node.cause: + self.write(" from ") + self.traverse(node.cause) + + def do_visit_try(self, node): + self.fill("try", allow_semicolon=False) + with self.block(): + self.traverse(node.body) + for ex in node.handlers: + self.traverse(ex) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + if node.finalbody: + self.fill("finally", allow_semicolon=False) + with self.block(): + self.traverse(node.finalbody) + + def visit_Try(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = False + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_TryStar(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = True + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_ExceptHandler(self, node): + self.fill("except*" if self._in_try_star else "except", allow_semicolon=False) + if node.type: + self.write(" ") + self.traverse(node.type) + if node.name: + self.write(" as ") + self.write(node.name) + with self.block(): + self.traverse(node.body) + + def visit_ClassDef(self, node): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + self.fill("class " + node.name, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit_if("(", ")", condition = node.bases or node.keywords): + comma = False + for e in node.bases: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + with self.block(): + self._write_docstring_and_traverse_body(node) + + def visit_FunctionDef(self, node): + self._function_helper(node, "def") + + def visit_AsyncFunctionDef(self, node): + self._function_helper(node, "async def") + + def _function_helper(self, node, fill_suffix): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + def_str = fill_suffix + " " + node.name + self.fill(def_str, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit("(", ")"): + self.traverse(node.args) + if node.returns: + self.write(" -> ") + self.traverse(node.returns) + with self.block(extra=self.get_type_comment(node)): + self._write_docstring_and_traverse_body(node) + + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, type_params) + + def visit_TypeVar(self, node): + self.write(node.name) + if node.bound: + self.write(": ") + self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeVarTuple(self, node): + self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_ParamSpec(self, node): + self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeAlias(self, node): + self.fill("type ") + self.traverse(node.name) + self._type_params_helper(node.type_params) + self.write(" = ") + self.traverse(node.value) + + def visit_For(self, node): + self._for_helper("for ", node) + + def visit_AsyncFor(self, node): + self._for_helper("async for ", node) + + def _for_helper(self, fill, node): + self.fill(fill, allow_semicolon=False) + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.traverse(node.iter) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_If(self, node): + self.fill("if ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # collapse nested ifs into equivalent elifs. + while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): + node = node.orelse[0] + self.fill("elif ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # final else + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_While(self, node): + self.fill("while ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_With(self, node): + self.fill("with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def visit_AsyncWith(self, node): + self.fill("async with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def _str_literal_helper( + self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False + ): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + + def _ftstring_helper(self, parts): + new_parts = [] + quote_types = list(_ALL_QUOTES) + fallback_to_repr = False + for value, is_constant in parts: + if is_constant: + value, new_quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + if set(new_quote_types).isdisjoint(quote_types): + fallback_to_repr = True + break + quote_types = new_quote_types + else: + if "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] + assert quote_types + + new_quote_types = [q for q in quote_types if q not in value] + if new_quote_types: + quote_types = new_quote_types + new_parts.append(value) + + if fallback_to_repr: + # If we weren't able to find a quote type that works for all parts + # of the JoinedStr, fallback to using repr and triple single quotes. + quote_types = ["'''"] + new_parts.clear() + for value, is_constant in parts: + if is_constant: + value = repr('"' + value) # force repr to use single quotes + expected_prefix = "'\"" + assert value.startswith(expected_prefix), repr(value) + value = value[len(expected_prefix):-1] + new_parts.append(value) + + value = "".join(new_parts) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") + + def _write_ftstring(self, values, prefix): + self.write(prefix) + fstring_parts = [] + for value in values: + with self.buffered() as buffer: + self._write_ftstring_inner(value) + fstring_parts.append( + ("".join(buffer), isinstance(value, Constant)) + ) + self._ftstring_helper(fstring_parts) + + def visit_JoinedStr(self, node): + self._write_ftstring(node.values, "f") + + def visit_TemplateStr(self, node): + self._write_ftstring(node.values, "t") + + def _write_ftstring_inner(self, node, is_format_spec=False): + if isinstance(node, JoinedStr): + # for both the f-string itself, and format_spec + for value in node.values: + self._write_ftstring_inner(value, is_format_spec=is_format_spec) + elif isinstance(node, Constant) and isinstance(node.value, str): + value = node.value.replace("{", "{{").replace("}", "}}") + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') + value = value.replace("\n", "\\n") + self.write(value) + elif isinstance(node, FormattedValue): + self.visit_FormattedValue(node) + elif isinstance(node, Interpolation): + self.visit_Interpolation(node) + else: + raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") + + def _unparse_interpolation_value(self, inner): + unparser = type(self)() + unparser.set_precedence(_Precedence.TEST.next(), inner) + return unparser.visit(inner) + + def _write_interpolation(self, node, use_str_attr=False): + with self.delimit("{", "}"): + if use_str_attr: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) + if expr.startswith("{"): + # Separate pair of opening brackets as "{ {" + self.write(" ") + self.write(expr) + if node.conversion != -1: + self.write(f"!{chr(node.conversion)}") + if node.format_spec: + self.write(":") + self._write_ftstring_inner(node.format_spec, is_format_spec=True) + + def visit_FormattedValue(self, node): + self._write_interpolation(node) + + def visit_Interpolation(self, node): + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) + + def visit_Name(self, node): + self.write(node.id) + + def _write_docstring(self, node): + self.fill(allow_semicolon=False) + if node.kind == "u": + self.write("u") + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) + + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write( + repr(value) + .replace("inf", _INFSTR) + .replace("nan", f"({_INFSTR}-{_INFSTR})") + ) + else: + self.write(repr(value)) + + def visit_Constant(self, node): + value = node.value + if isinstance(value, tuple): + with self.delimit("(", ")"): + self.items_view(self._write_constant, value) + elif value is ...: + self.write("...") + else: + if node.kind == "u": + self.write("u") + self._write_constant(node.value) + + def visit_List(self, node): + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + + def visit_ListComp(self, node): + with self.delimit("[", "]"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_GeneratorExp(self, node): + with self.delimit("(", ")"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_SetComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_DictComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + for gen in node.generators: + self.traverse(gen) + + def visit_comprehension(self, node): + if node.is_async: + self.write(" async for ") + else: + self.write(" for ") + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) + self.traverse(node.iter) + for if_clause in node.ifs: + self.write(" if ") + self.traverse(if_clause) + + def visit_IfExp(self, node): + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.TEST.next(), node.body, node.test) + self.traverse(node.body) + self.write(" if ") + self.traverse(node.test) + self.write(" else ") + self.set_precedence(_Precedence.TEST, node.orelse) + self.traverse(node.orelse) + + def visit_Set(self, node): + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write('{*()}') + + def visit_Dict(self, node): + def write_key_value_pair(k, v): + self.traverse(k) + self.write(": ") + self.traverse(v) + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.set_precedence(_Precedence.EXPR, v) + self.traverse(v) + else: + write_key_value_pair(k, v) + + with self.delimit("{", "}"): + self.interleave( + lambda: self.write(", "), write_item, zip(node.keys, node.values) + ) + + def visit_Tuple(self, node): + with self.delimit_if( + "(", + ")", + len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE + ): + self.items_view(self.traverse, node.elts) + + unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} + unop_precedence = { + "not": _Precedence.NOT, + "~": _Precedence.FACTOR, + "+": _Precedence.FACTOR, + "-": _Precedence.FACTOR, + } + + def visit_UnaryOp(self, node): + operator = self.unop[node.op.__class__.__name__] + operator_precedence = self.unop_precedence[operator] + with self.require_parens(operator_precedence, node): + self.write(operator) + # factor prefixes (+, -, ~) shouldn't be separated + # from the value they belong, (e.g: +1 instead of + 1) + if operator_precedence is not _Precedence.FACTOR: + self.write(" ") + self.set_precedence(operator_precedence, node.operand) + self.traverse(node.operand) + + binop = { + "Add": "+", + "Sub": "-", + "Mult": "*", + "MatMult": "@", + "Div": "/", + "Mod": "%", + "LShift": "<<", + "RShift": ">>", + "BitOr": "|", + "BitXor": "^", + "BitAnd": "&", + "FloorDiv": "//", + "Pow": "**", + } + + binop_precedence = { + "+": _Precedence.ARITH, + "-": _Precedence.ARITH, + "*": _Precedence.TERM, + "@": _Precedence.TERM, + "/": _Precedence.TERM, + "%": _Precedence.TERM, + "<<": _Precedence.SHIFT, + ">>": _Precedence.SHIFT, + "|": _Precedence.BOR, + "^": _Precedence.BXOR, + "&": _Precedence.BAND, + "//": _Precedence.TERM, + "**": _Precedence.POWER, + } + + binop_rassoc = frozenset(("**",)) + def visit_BinOp(self, node): + operator = self.binop[node.op.__class__.__name__] + operator_precedence = self.binop_precedence[operator] + with self.require_parens(operator_precedence, node): + if operator in self.binop_rassoc: + left_precedence = operator_precedence.next() + right_precedence = operator_precedence + else: + left_precedence = operator_precedence + right_precedence = operator_precedence.next() + + self.set_precedence(left_precedence, node.left) + self.traverse(node.left) + self.write(f" {operator} ") + self.set_precedence(right_precedence, node.right) + self.traverse(node.right) + + cmpops = { + "Eq": "==", + "NotEq": "!=", + "Lt": "<", + "LtE": "<=", + "Gt": ">", + "GtE": ">=", + "Is": "is", + "IsNot": "is not", + "In": "in", + "NotIn": "not in", + } + + def visit_Compare(self, node): + with self.require_parens(_Precedence.CMP, node): + self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) + self.traverse(node.left) + for o, e in zip(node.ops, node.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.traverse(e) + + boolops = {"And": "and", "Or": "or"} + boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} + + def visit_BoolOp(self, node): + operator = self.boolops[node.op.__class__.__name__] + operator_precedence = self.boolop_precedence[operator] + + def increasing_level_traverse(node): + nonlocal operator_precedence + operator_precedence = operator_precedence.next() + self.set_precedence(operator_precedence, node) + self.traverse(node) + + with self.require_parens(operator_precedence, node): + s = f" {operator} " + self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) + + def visit_Attribute(self, node): + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + # Special case: 3.__abs__() is a syntax error, so if node.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(node.value, Constant) and isinstance(node.value.value, int): + self.write(" ") + self.write(".") + self.write(node.attr) + + def visit_Call(self, node): + self.set_precedence(_Precedence.ATOM, node.func) + self.traverse(node.func) + with self.delimit("(", ")"): + comma = False + for e in node.args: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + def visit_Subscript(self, node): + def is_non_empty_tuple(slice_value): + return ( + isinstance(slice_value, Tuple) + and slice_value.elts + ) + + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + with self.delimit("[", "]"): + if is_non_empty_tuple(node.slice): + # parentheses can be omitted if the tuple isn't empty + self.items_view(self.traverse, node.slice.elts) + else: + self.traverse(node.slice) + + def visit_Starred(self, node): + self.write("*") + self.set_precedence(_Precedence.EXPR, node.value) + self.traverse(node.value) + + def visit_Ellipsis(self, node): + self.write("...") + + def visit_Slice(self, node): + if node.lower: + self.traverse(node.lower) + self.write(":") + if node.upper: + self.traverse(node.upper) + if node.step: + self.write(":") + self.traverse(node.step) + + def visit_Match(self, node): + self.fill("match ", allow_semicolon=False) + self.traverse(node.subject) + with self.block(): + for case in node.cases: + self.traverse(case) + + def visit_arg(self, node): + self.write(node.arg) + if node.annotation: + self.write(": ") + self.traverse(node.annotation) + + def visit_arguments(self, node): + first = True + # normal arguments + all_args = node.posonlyargs + node.args + defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements + if first: + first = False + else: + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + if index == len(node.posonlyargs): + self.write(", /") + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if node.vararg or node.kwonlyargs: + if first: + first = False + else: + self.write(", ") + self.write("*") + if node.vararg: + self.write(node.vararg.arg) + if node.vararg.annotation: + self.write(": ") + self.traverse(node.vararg.annotation) + + # keyword-only arguments + if node.kwonlyargs: + for a, d in zip(node.kwonlyargs, node.kw_defaults): + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + + # kwargs + if node.kwarg: + if first: + first = False + else: + self.write(", ") + self.write("**" + node.kwarg.arg) + if node.kwarg.annotation: + self.write(": ") + self.traverse(node.kwarg.annotation) + + def visit_keyword(self, node): + if node.arg is None: + self.write("**") + else: + self.write(node.arg) + self.write("=") + self.traverse(node.value) + + def visit_Lambda(self, node): + with self.require_parens(_Precedence.TEST, node): + self.write("lambda") + with self.buffered() as buffer: + self.traverse(node.args) + if buffer: + self.write(" ", *buffer) + self.write(": ") + self.set_precedence(_Precedence.TEST, node.body) + self.traverse(node.body) + + def visit_alias(self, node): + self.write(node.name) + if node.asname: + self.write(" as " + node.asname) + + def visit_withitem(self, node): + self.traverse(node.context_expr) + if node.optional_vars: + self.write(" as ") + self.traverse(node.optional_vars) + + def visit_match_case(self, node): + self.fill("case ", allow_semicolon=False) + self.traverse(node.pattern) + if node.guard: + self.write(" if ") + self.traverse(node.guard) + with self.block(): + self.traverse(node.body) + + def visit_MatchValue(self, node): + self.traverse(node.value) + + def visit_MatchSingleton(self, node): + self._write_constant(node.value) + + def visit_MatchSequence(self, node): + with self.delimit("[", "]"): + self.interleave( + lambda: self.write(", "), self.traverse, node.patterns + ) + + def visit_MatchStar(self, node): + name = node.name + if name is None: + name = "_" + self.write(f"*{name}") + + def visit_MatchMapping(self, node): + def write_key_pattern_pair(pair): + k, p = pair + self.traverse(k) + self.write(": ") + self.traverse(p) + + with self.delimit("{", "}"): + keys = node.keys + self.interleave( + lambda: self.write(", "), + write_key_pattern_pair, + zip(keys, node.patterns, strict=True), + ) + rest = node.rest + if rest is not None: + if keys: + self.write(", ") + self.write(f"**{rest}") + + def visit_MatchClass(self, node): + self.set_precedence(_Precedence.ATOM, node.cls) + self.traverse(node.cls) + with self.delimit("(", ")"): + patterns = node.patterns + self.interleave( + lambda: self.write(", "), self.traverse, patterns + ) + attrs = node.kwd_attrs + if attrs: + def write_attr_pattern(pair): + attr, pattern = pair + self.write(f"{attr}=") + self.traverse(pattern) + + if patterns: + self.write(", ") + self.interleave( + lambda: self.write(", "), + write_attr_pattern, + zip(attrs, node.kwd_patterns, strict=True), + ) + + def visit_MatchAs(self, node): + name = node.name + pattern = node.pattern + if name is None: + self.write("_") + elif pattern is None: + self.write(node.name) + else: + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.BOR, node.pattern) + self.traverse(node.pattern) + self.write(f" as {node.name}") + + def visit_MatchOr(self, node): + with self.require_parens(_Precedence.BOR, node): + self.set_precedence(_Precedence.BOR.next(), *node.patterns) + self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index e02fc227384..241d40d5740 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import type_repr return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([type_repr(a) for a in self.__args__[:-1]])}], ' + f'{type_repr(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -512,10 +513,6 @@ def __getitem__(self, item): new_args = (t_args, t_result) return _CallableGenericAlias(Callable, tuple(new_args)) - # TODO: RUSTPYTHON; patch for common call - def __or__(self, other): - super().__or__(other) - def _is_param_expr(obj): """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or ``_ConcatenateGenericAlias`` from typing.py @@ -528,23 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - (Keep this roughly in sync with the typing version.) - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): @@ -1077,6 +1057,7 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) @@ -1183,4 +1164,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 9eb6f0933b8..d6673f6692f 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,13 +1,16 @@ -from __future__ import annotations -import io import os import sys +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field, Field + COLORIZE = True + # types if False: - from typing import IO + from typing import IO, Self, ClassVar + _theme: Theme class ANSIColors: @@ -17,11 +20,13 @@ class ANSIColors: BLUE = "\x1b[34m" CYAN = "\x1b[36m" GREEN = "\x1b[32m" + GREY = "\x1b[90m" MAGENTA = "\x1b[35m" RED = "\x1b[31m" WHITE = "\x1b[37m" # more like LIGHT GRAY YELLOW = "\x1b[33m" + BOLD = "\x1b[1m" BOLD_BLACK = "\x1b[1;30m" # DARK GRAY BOLD_BLUE = "\x1b[1;34m" BOLD_CYAN = "\x1b[1;36m" @@ -60,13 +65,196 @@ class ANSIColors: INTENSE_BACKGROUND_YELLOW = "\x1b[103m" +ColorCodes = set() NoColors = ANSIColors() -for attr in dir(NoColors): +for attr, code in ANSIColors.__dict__.items(): if not attr.startswith("__"): + ColorCodes.add(code) setattr(NoColors, attr, "") +# +# Experimental theming support (see gh-133346) +# + +# - Create a theme by copying an existing `Theme` with one or more sections +# replaced, using `default_theme.copy_with()`; +# - create a theme section by copying an existing `ThemeSection` with one or +# more colors replaced, using for example `default_theme.syntax.copy_with()`; +# - create a theme from scratch by instantiating a `Theme` data class with +# the required sections (which are also dataclass instances). +# +# Then call `_colorize.set_theme(your_theme)` to set it. +# +# Put your theme configuration in $PYTHONSTARTUP for the interactive shell, +# or sitecustomize.py in your virtual environment or Python installation for +# other uses. Your applications can call `_colorize.set_theme()` too. +# +# Note that thanks to the dataclasses providing default values for all fields, +# creating a new theme or theme section from scratch is possible without +# specifying all keys. +# +# For example, here's a theme that makes punctuation and operators less prominent: +# +# try: +# from _colorize import set_theme, default_theme, Syntax, ANSIColors +# except ImportError: +# pass +# else: +# theme_with_dim_operators = default_theme.copy_with( +# syntax=Syntax(op=ANSIColors.INTENSE_BLACK), +# ) +# set_theme(theme_with_dim_operators) +# del set_theme, default_theme, Syntax, ANSIColors, theme_with_dim_operators +# +# Guarding the import ensures that your .pythonstartup file will still work in +# Python 3.13 and older. Deleting the variables ensures they don't remain in your +# interactive shell's global scope. + +class ThemeSection(Mapping[str, str]): + """A mixin/base class for theme sections. + + It enables dictionary access to a section, as well as implements convenience + methods. + """ + + # The two types below are just that: types to inform the type checker that the + # mixin will work in context of those fields existing + __dataclass_fields__: ClassVar[dict[str, Field[str]]] + _name_to_value: Callable[[str], str] + + def __post_init__(self) -> None: + name_to_value = {} + for color_name in self.__dataclass_fields__: + name_to_value[color_name] = getattr(self, color_name) + super().__setattr__('_name_to_value', name_to_value.__getitem__) + + def copy_with(self, **kwargs: str) -> Self: + color_state: dict[str, str] = {} + for color_name in self.__dataclass_fields__: + color_state[color_name] = getattr(self, color_name) + color_state.update(kwargs) + return type(self)(**color_state) + + @classmethod + def no_colors(cls) -> Self: + color_state: dict[str, str] = {} + for color_name in cls.__dataclass_fields__: + color_state[color_name] = "" + return cls(**color_state) + + def __getitem__(self, key: str) -> str: + return self._name_to_value(key) + + def __len__(self) -> int: + return len(self.__dataclass_fields__) + + def __iter__(self) -> Iterator[str]: + return iter(self.__dataclass_fields__) + + +@dataclass(frozen=True, kw_only=True) +class Argparse(ThemeSection): + usage: str = ANSIColors.BOLD_BLUE + prog: str = ANSIColors.BOLD_MAGENTA + prog_extra: str = ANSIColors.MAGENTA + heading: str = ANSIColors.BOLD_BLUE + summary_long_option: str = ANSIColors.CYAN + summary_short_option: str = ANSIColors.GREEN + summary_label: str = ANSIColors.YELLOW + summary_action: str = ANSIColors.GREEN + long_option: str = ANSIColors.BOLD_CYAN + short_option: str = ANSIColors.BOLD_GREEN + label: str = ANSIColors.BOLD_YELLOW + action: str = ANSIColors.BOLD_GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Syntax(ThemeSection): + prompt: str = ANSIColors.BOLD_MAGENTA + keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE + builtin: str = ANSIColors.CYAN + comment: str = ANSIColors.RED + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + op: str = ANSIColors.RESET + definition: str = ANSIColors.BOLD + soft_keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Traceback(ThemeSection): + type: str = ANSIColors.BOLD_MAGENTA + message: str = ANSIColors.MAGENTA + filename: str = ANSIColors.MAGENTA + line_no: str = ANSIColors.MAGENTA + frame: str = ANSIColors.MAGENTA + error_highlight: str = ANSIColors.BOLD_RED + error_range: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Unittest(ThemeSection): + passed: str = ANSIColors.GREEN + warn: str = ANSIColors.YELLOW + fail: str = ANSIColors.RED + fail_info: str = ANSIColors.BOLD_RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Theme: + """A suite of themes for all sections of Python. + + When adding a new one, remember to also modify `copy_with` and `no_colors` + below. + """ + argparse: Argparse = field(default_factory=Argparse) + syntax: Syntax = field(default_factory=Syntax) + traceback: Traceback = field(default_factory=Traceback) + unittest: Unittest = field(default_factory=Unittest) + + def copy_with( + self, + *, + argparse: Argparse | None = None, + syntax: Syntax | None = None, + traceback: Traceback | None = None, + unittest: Unittest | None = None, + ) -> Self: + """Return a new Theme based on this instance with some sections replaced. + + Themes are immutable to protect against accidental modifications that + could lead to invalid terminal states. + """ + return type(self)( + argparse=argparse or self.argparse, + syntax=syntax or self.syntax, + traceback=traceback or self.traceback, + unittest=unittest or self.unittest, + ) + + @classmethod + def no_colors(cls) -> Self: + """Return a new Theme where colors in all sections are empty strings. + + This allows writing user code as if colors are always used. The color + fields will be ANSI color code strings when colorization is desired + and possible, and empty strings otherwise. + """ + return cls( + argparse=Argparse.no_colors(), + syntax=Syntax.no_colors(), + traceback=Traceback.no_colors(), + unittest=Unittest.no_colors(), + ) + + def get_colors( colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None ) -> ANSIColors: @@ -76,22 +264,37 @@ def get_colors( return NoColors +def decolor(text: str) -> str: + """Remove ANSI color codes from a string.""" + for code in ColorCodes: + text = text.replace(code, "") + return text + + def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -108,5 +311,45 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() + + +default_theme = Theme() +theme_no_color = default_theme.no_colors() + + +def get_theme( + *, + tty_file: IO[str] | IO[bytes] | None = None, + force_color: bool = False, + force_no_color: bool = False, +) -> Theme: + """Returns the currently set theme, potentially in a zero-color variant. + + In cases where colorizing is not possible (see `can_colorize`), the returned + theme contains all empty strings in all color definitions. + See `Theme.no_colors()` for more information. + + It is recommended not to cache the result of this function for extended + periods of time because the user might influence theme selection by + the interactive shell, a debugger, or application-specific code. The + environment (including environment variable state and console configuration + on Windows) can also change in the course of the application life cycle. + """ + if force_color or (not force_no_color and + can_colorize(file=tty_file)): + return _theme + return theme_no_color + + +def set_theme(t: Theme) -> None: + global _theme + + if not isinstance(t, Theme): + raise ValueError(f"Expected Theme object, found {t}") + + _theme = t + + +set_theme(default_theme) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 17b9010278f..60793c391ae 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -22,7 +22,6 @@ 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', @@ -257,3 +256,4 @@ for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError') +del excname diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py index 424b0b3be5e..0630d4e59fa 100644 --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -11,15 +11,35 @@ import _dummy_thread as _thread """ + # Exports only things specified by thread documentation; # skipping obsolete synonyms allocate(), start_new(), exit_thread(). -__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', - 'interrupt_main', 'LockType', 'RLock', - '_count'] +__all__ = [ + "error", + "start_new_thread", + "exit", + "get_ident", + "allocate_lock", + "interrupt_main", + "LockType", + "RLock", + "_count", + "start_joinable_thread", + "daemon_threads_allowed", + "_shutdown", + "_make_thread_handle", + "_ThreadHandle", + "_get_main_thread_ident", + "_is_main_interpreter", + "_local", +] # A dummy value TIMEOUT_MAX = 2**31 +# Main thread ident for dummy implementation +_MAIN_THREAD_IDENT = -1 + # NOTE: this module can be imported early in the extension building process, # and so top level imports of other modules should be avoided. Instead, all # imports are done when needed on a function-by-function basis. Since threads @@ -27,6 +47,7 @@ error = RuntimeError + def start_new_thread(function, args, kwargs={}): """Dummy implementation of _thread.start_new_thread(). @@ -52,6 +73,7 @@ def start_new_thread(function, args, kwargs={}): pass except: import traceback + traceback.print_exc() _main = True global _interrupt @@ -59,10 +81,58 @@ def start_new_thread(function, args, kwargs={}): _interrupt = False raise KeyboardInterrupt + +def start_joinable_thread(function, handle=None, daemon=True): + """Dummy implementation of _thread.start_joinable_thread(). + + In dummy thread, we just run the function synchronously. + """ + if handle is None: + handle = _ThreadHandle() + try: + function() + except SystemExit: + pass + except: + import traceback + + traceback.print_exc() + handle._set_done() + return handle + + +def daemon_threads_allowed(): + """Dummy implementation of _thread.daemon_threads_allowed().""" + return True + + +def _shutdown(): + """Dummy implementation of _thread._shutdown().""" + pass + + +def _make_thread_handle(ident): + """Dummy implementation of _thread._make_thread_handle().""" + handle = _ThreadHandle() + handle._ident = ident + return handle + + +def _get_main_thread_ident(): + """Dummy implementation of _thread._get_main_thread_ident().""" + return _MAIN_THREAD_IDENT + + +def _is_main_interpreter(): + """Dummy implementation of _thread._is_main_interpreter().""" + return True + + def exit(): """Dummy implementation of _thread.exit().""" raise SystemExit + def get_ident(): """Dummy implementation of _thread.get_ident(). @@ -70,26 +140,31 @@ def get_ident(): available, it is safe to assume that the current process is the only thread. Thus a constant can be safely returned. """ - return -1 + return _MAIN_THREAD_IDENT + def allocate_lock(): """Dummy implementation of _thread.allocate_lock().""" return LockType() + def stack_size(size=None): """Dummy implementation of _thread.stack_size().""" if size is not None: raise error("setting thread stack size not supported") return 0 + def _set_sentinel(): """Dummy implementation of _thread._set_sentinel().""" return LockType() + def _count(): """Dummy implementation of _thread._count().""" return 0 + class LockType(object): """Class implementing dummy implementation of _thread.LockType. @@ -125,6 +200,7 @@ def acquire(self, waitflag=None, timeout=-1): else: if timeout > 0: import time + time.sleep(timeout) return False @@ -153,14 +229,41 @@ def __repr__(self): "locked" if self.locked_status else "unlocked", self.__class__.__module__, self.__class__.__qualname__, - hex(id(self)) + hex(id(self)), ) + +class _ThreadHandle: + """Dummy implementation of _thread._ThreadHandle.""" + + def __init__(self): + self._ident = _MAIN_THREAD_IDENT + self._done = False + + @property + def ident(self): + return self._ident + + def _set_done(self): + self._done = True + + def is_done(self): + return self._done + + def join(self, timeout=None): + # In dummy thread, thread is always done + return + + def __repr__(self): + return f"<_ThreadHandle ident={self._ident}>" + + # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" _main = True + def interrupt_main(): """Set _interrupt flag to True to have start_new_thread raise KeyboardInterrupt upon exiting.""" @@ -170,6 +273,7 @@ def interrupt_main(): global _interrupt _interrupt = True + class RLock: def __init__(self): self.locked_count = 0 @@ -190,7 +294,7 @@ def release(self): return True def locked(self): - return self.locked_status != 0 + return self.locked_count != 0 def __repr__(self): return "<%s %s.%s object owner=%s count=%s at %s>" % ( @@ -199,5 +303,36 @@ def __repr__(self): self.__class__.__qualname__, get_ident() if self.locked_count else 0, self.locked_count, - hex(id(self)) + hex(id(self)), ) + + +class _local: + """Dummy implementation of _thread._local (thread-local storage).""" + + def __init__(self): + object.__setattr__(self, "_local__impl", {}) + + def __getattribute__(self, name): + if name.startswith("_local__"): + return object.__getattribute__(self, name) + impl = object.__getattribute__(self, "_local__impl") + try: + return impl[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + if name.startswith("_local__"): + return object.__setattr__(self, name, value) + impl = object.__getattribute__(self, "_local__impl") + impl[name] = value + + def __delattr__(self, name): + if name.startswith("_local__"): + return object.__delattr__(self, name) + impl = object.__getattribute__(self, "_local__impl") + try: + del impl[name] + except KeyError: + raise AttributeError(name) diff --git a/Lib/_markupbase.py b/Lib/_markupbase.py index 3ad7e279960..614f0cd16dd 100644 --- a/Lib/_markupbase.py +++ b/Lib/_markupbase.py @@ -13,7 +13,7 @@ _markedsectionclose = re.compile(r']\s*]\s*>') # An analysis of the MS-Word extensions is available at -# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf +# http://web.archive.org/web/20060321153828/http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf _msmarkedsectionclose = re.compile(r']\s*>') diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b3d7b8103e8..bb55ee423cf 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -1,343 +1,252 @@ -# This file is generated by Tools/cases_generator/py_metadata_generator.py -# from: -# Python/bytecodes.c +# This file is generated by scripts/generate_opcode_metadata.py +# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). # Do not edit! -_specializations = { - "RESUME": [ - "RESUME_CHECK", - ], - "TO_BOOL": [ - "TO_BOOL_ALWAYS_TRUE", - "TO_BOOL_BOOL", - "TO_BOOL_INT", - "TO_BOOL_LIST", - "TO_BOOL_NONE", - "TO_BOOL_STR", - ], - "BINARY_OP": [ - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_ADD_INT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_STR_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "SEND": [ - "SEND_GEN", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_TWO_TUPLE", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_LIST", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_CLASS", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_METHOD_WITH_VALUES", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "CONTAINS_OP": [ - "CONTAINS_OP_SET", - "CONTAINS_OP_DICT", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "CALL": [ - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_PY_EXACT_ARGS", - "CALL_TYPE_1", - "CALL_STR_1", - "CALL_TUPLE_1", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_O", - "CALL_BUILTIN_FAST", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_LEN", - "CALL_ISINSTANCE", - "CALL_LIST_APPEND", - "CALL_METHOD_DESCRIPTOR_O", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_NOARGS", - "CALL_METHOD_DESCRIPTOR_FAST", - "CALL_ALLOC_AND_ENTER_INIT", - "CALL_PY_GENERAL", - "CALL_BOUND_METHOD_GENERAL", - "CALL_NON_PY_GENERAL", - ], -} -_specialized_opmap = { - 'BINARY_OP_ADD_FLOAT': 150, - 'BINARY_OP_ADD_INT': 151, - 'BINARY_OP_ADD_UNICODE': 152, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 153, - 'BINARY_OP_MULTIPLY_INT': 154, - 'BINARY_OP_SUBTRACT_FLOAT': 155, - 'BINARY_OP_SUBTRACT_INT': 156, - 'BINARY_SUBSCR_DICT': 157, - 'BINARY_SUBSCR_GETITEM': 158, - 'BINARY_SUBSCR_LIST_INT': 159, - 'BINARY_SUBSCR_STR_INT': 160, - 'BINARY_SUBSCR_TUPLE_INT': 161, - 'CALL_ALLOC_AND_ENTER_INIT': 162, - 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BOUND_METHOD_GENERAL': 164, - 'CALL_BUILTIN_CLASS': 165, - 'CALL_BUILTIN_FAST': 166, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, - 'CALL_BUILTIN_O': 168, - 'CALL_ISINSTANCE': 169, - 'CALL_LEN': 170, - 'CALL_LIST_APPEND': 171, - 'CALL_METHOD_DESCRIPTOR_FAST': 172, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, - 'CALL_METHOD_DESCRIPTOR_O': 175, - 'CALL_NON_PY_GENERAL': 176, - 'CALL_PY_EXACT_ARGS': 177, - 'CALL_PY_GENERAL': 178, - 'CALL_STR_1': 179, - 'CALL_TUPLE_1': 180, - 'CALL_TYPE_1': 181, - 'COMPARE_OP_FLOAT': 182, - 'COMPARE_OP_INT': 183, - 'COMPARE_OP_STR': 184, - 'CONTAINS_OP_DICT': 185, - 'CONTAINS_OP_SET': 186, - 'FOR_ITER_GEN': 187, - 'FOR_ITER_LIST': 188, - 'FOR_ITER_RANGE': 189, - 'FOR_ITER_TUPLE': 190, - 'LOAD_ATTR_CLASS': 191, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, - 'LOAD_ATTR_INSTANCE_VALUE': 193, - 'LOAD_ATTR_METHOD_LAZY_DICT': 194, - 'LOAD_ATTR_METHOD_NO_DICT': 195, - 'LOAD_ATTR_METHOD_WITH_VALUES': 196, - 'LOAD_ATTR_MODULE': 197, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, - 'LOAD_ATTR_PROPERTY': 200, - 'LOAD_ATTR_SLOT': 201, - 'LOAD_ATTR_WITH_HINT': 202, - 'LOAD_GLOBAL_BUILTIN': 203, - 'LOAD_GLOBAL_MODULE': 204, - 'LOAD_SUPER_ATTR_ATTR': 205, - 'LOAD_SUPER_ATTR_METHOD': 206, - 'RESUME_CHECK': 207, - 'SEND_GEN': 208, - 'STORE_ATTR_INSTANCE_VALUE': 209, - 'STORE_ATTR_SLOT': 210, - 'STORE_ATTR_WITH_HINT': 211, - 'STORE_SUBSCR_DICT': 212, - 'STORE_SUBSCR_LIST_INT': 213, - 'TO_BOOL_ALWAYS_TRUE': 214, - 'TO_BOOL_BOOL': 215, - 'TO_BOOL_INT': 216, - 'TO_BOOL_LIST': 217, - 'TO_BOOL_NONE': 218, - 'TO_BOOL_STR': 219, - 'UNPACK_SEQUENCE_LIST': 220, - 'UNPACK_SEQUENCE_TUPLE': 221, - 'UNPACK_SEQUENCE_TWO_TUPLE': 222, -} +_specializations = {} + +_specialized_opmap = {} opmap = { 'CACHE': 0, + 'BINARY_SLICE': 1, + 'BUILD_TEMPLATE': 2, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'CALL_FUNCTION_EX': 4, + 'CHECK_EG_MATCH': 5, + 'CHECK_EXC_MATCH': 6, + 'CLEANUP_THROW': 7, + 'DELETE_SUBSCR': 8, + 'END_FOR': 9, + 'END_SEND': 10, + 'EXIT_INIT_CHECK': 11, + 'FORMAT_SIMPLE': 12, + 'FORMAT_WITH_SPEC': 13, + 'GET_AITER': 14, + 'GET_ANEXT': 15, + 'GET_ITER': 16, 'RESERVED': 17, - 'RESUME': 149, + 'GET_LEN': 18, + 'GET_YIELD_FROM_ITER': 19, + 'INTERPRETER_EXIT': 20, + 'LOAD_BUILD_CLASS': 21, + 'LOAD_LOCALS': 22, + 'MAKE_FUNCTION': 23, + 'MATCH_KEYS': 24, + 'MATCH_MAPPING': 25, + 'MATCH_SEQUENCE': 26, + 'NOP': 27, + 'NOT_TAKEN': 28, + 'POP_EXCEPT': 29, + 'POP_ITER': 30, + 'POP_TOP': 31, + 'PUSH_EXC_INFO': 32, + 'PUSH_NULL': 33, + 'RETURN_GENERATOR': 34, + 'RETURN_VALUE': 35, + 'SETUP_ANNOTATIONS': 36, + 'STORE_SLICE': 37, + 'STORE_SUBSCR': 38, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_INTERPOLATION': 45, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'END_ASYNC_FOR': 68, + 'EXTENDED_ARG': 69, + 'FOR_ITER': 70, + 'GET_AWAITABLE': 71, + 'IMPORT_FROM': 72, + 'IMPORT_NAME': 73, + 'IS_OP': 74, + 'JUMP_BACKWARD': 75, + 'JUMP_BACKWARD_NO_INTERRUPT': 76, + 'JUMP_FORWARD': 77, + 'LIST_APPEND': 78, + 'LIST_EXTEND': 79, + 'LOAD_ATTR': 80, + 'LOAD_COMMON_CONSTANT': 81, + 'LOAD_CONST': 82, + 'LOAD_DEREF': 83, + 'LOAD_FAST': 84, + 'LOAD_FAST_AND_CLEAR': 85, + 'LOAD_FAST_BORROW': 86, + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, + 'LOAD_FAST_CHECK': 88, + 'LOAD_FAST_LOAD_FAST': 89, + 'LOAD_FROM_DICT_OR_DEREF': 90, + 'LOAD_FROM_DICT_OR_GLOBALS': 91, + 'LOAD_GLOBAL': 92, + 'LOAD_NAME': 93, + 'LOAD_SMALL_INT': 94, + 'LOAD_SPECIAL': 95, + 'LOAD_SUPER_ATTR': 96, + 'MAKE_CELL': 97, + 'MAP_ADD': 98, + 'MATCH_CLASS': 99, + 'POP_JUMP_IF_FALSE': 100, + 'POP_JUMP_IF_NONE': 101, + 'POP_JUMP_IF_NOT_NONE': 102, + 'POP_JUMP_IF_TRUE': 103, + 'RAISE_VARARGS': 104, + 'RERAISE': 105, + 'SEND': 106, + 'SET_ADD': 107, + 'SET_FUNCTION_ATTRIBUTE': 108, + 'SET_UPDATE': 109, + 'STORE_ATTR': 110, + 'STORE_DEREF': 111, + 'STORE_FAST': 112, + 'STORE_FAST_LOAD_FAST': 113, + 'STORE_FAST_STORE_FAST': 114, + 'STORE_GLOBAL': 115, + 'STORE_NAME': 116, + 'SWAP': 117, + 'UNPACK_EX': 118, + 'UNPACK_SEQUENCE': 119, + 'YIELD_VALUE': 120, + 'RESUME': 128, + 'BINARY_OP_ADD_FLOAT': 129, + 'BINARY_OP_ADD_INT': 130, + 'BINARY_OP_ADD_UNICODE': 131, + 'BINARY_OP_EXTEND': 132, + 'BINARY_OP_MULTIPLY_FLOAT': 133, + 'BINARY_OP_MULTIPLY_INT': 134, + 'BINARY_OP_SUBSCR_DICT': 135, + 'BINARY_OP_SUBSCR_GETITEM': 136, + 'BINARY_OP_SUBSCR_LIST_INT': 137, + 'BINARY_OP_SUBSCR_LIST_SLICE': 138, + 'BINARY_OP_SUBSCR_STR_INT': 139, + 'BINARY_OP_SUBSCR_TUPLE_INT': 140, + 'BINARY_OP_SUBTRACT_FLOAT': 141, + 'BINARY_OP_SUBTRACT_INT': 142, + 'CALL_ALLOC_AND_ENTER_INIT': 143, + 'CALL_BOUND_METHOD_EXACT_ARGS': 144, + 'CALL_BOUND_METHOD_GENERAL': 145, + 'CALL_BUILTIN_CLASS': 146, + 'CALL_BUILTIN_FAST': 147, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, + 'CALL_BUILTIN_O': 149, + 'CALL_ISINSTANCE': 150, + 'CALL_KW_BOUND_METHOD': 151, + 'CALL_KW_NON_PY': 152, + 'CALL_KW_PY': 153, + 'CALL_LEN': 154, + 'CALL_LIST_APPEND': 155, + 'CALL_METHOD_DESCRIPTOR_FAST': 156, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, + 'CALL_METHOD_DESCRIPTOR_O': 159, + 'CALL_NON_PY_GENERAL': 160, + 'CALL_PY_EXACT_ARGS': 161, + 'CALL_PY_GENERAL': 162, + 'CALL_STR_1': 163, + 'CALL_TUPLE_1': 164, + 'CALL_TYPE_1': 165, + 'COMPARE_OP_FLOAT': 166, + 'COMPARE_OP_INT': 167, + 'COMPARE_OP_STR': 168, + 'CONTAINS_OP_DICT': 169, + 'CONTAINS_OP_SET': 170, + 'FOR_ITER_GEN': 171, + 'FOR_ITER_LIST': 172, + 'FOR_ITER_RANGE': 173, + 'FOR_ITER_TUPLE': 174, + 'JUMP_BACKWARD_JIT': 175, + 'JUMP_BACKWARD_NO_JIT': 176, + 'LOAD_ATTR_CLASS': 177, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, + 'LOAD_ATTR_INSTANCE_VALUE': 180, + 'LOAD_ATTR_METHOD_LAZY_DICT': 181, + 'LOAD_ATTR_METHOD_NO_DICT': 182, + 'LOAD_ATTR_METHOD_WITH_VALUES': 183, + 'LOAD_ATTR_MODULE': 184, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, + 'LOAD_ATTR_PROPERTY': 187, + 'LOAD_ATTR_SLOT': 188, + 'LOAD_ATTR_WITH_HINT': 189, + 'LOAD_CONST_IMMORTAL': 190, + 'LOAD_CONST_MORTAL': 191, + 'LOAD_GLOBAL_BUILTIN': 192, + 'LOAD_GLOBAL_MODULE': 193, + 'LOAD_SUPER_ATTR_ATTR': 194, + 'LOAD_SUPER_ATTR_METHOD': 195, + 'RESUME_CHECK': 196, + 'SEND_GEN': 197, + 'STORE_ATTR_INSTANCE_VALUE': 198, + 'STORE_ATTR_SLOT': 199, + 'STORE_ATTR_WITH_HINT': 200, + 'STORE_SUBSCR_DICT': 201, + 'STORE_SUBSCR_LIST_INT': 202, + 'TO_BOOL_ALWAYS_TRUE': 203, + 'TO_BOOL_BOOL': 204, + 'TO_BOOL_INT': 205, + 'TO_BOOL_LIST': 206, + 'TO_BOOL_NONE': 207, + 'TO_BOOL_STR': 208, + 'UNPACK_SEQUENCE_LIST': 209, + 'UNPACK_SEQUENCE_TUPLE': 210, + 'UNPACK_SEQUENCE_TWO_TUPLE': 211, + 'INSTRUMENTED_END_FOR': 234, + 'INSTRUMENTED_POP_ITER': 235, + 'INSTRUMENTED_END_SEND': 236, + 'INSTRUMENTED_FOR_ITER': 237, + 'INSTRUMENTED_INSTRUCTION': 238, + 'INSTRUMENTED_JUMP_FORWARD': 239, + 'INSTRUMENTED_NOT_TAKEN': 240, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, + 'INSTRUMENTED_RESUME': 245, + 'INSTRUMENTED_RETURN_VALUE': 246, + 'INSTRUMENTED_YIELD_VALUE': 247, + 'INSTRUMENTED_END_ASYNC_FOR': 248, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, + 'INSTRUMENTED_CALL': 250, + 'INSTRUMENTED_CALL_KW': 251, + 'INSTRUMENTED_CALL_FUNCTION_EX': 252, + 'INSTRUMENTED_JUMP_BACKWARD': 253, 'INSTRUMENTED_LINE': 254, - 'BEFORE_ASYNC_WITH': 1, - 'BEFORE_WITH': 2, - 'BINARY_SLICE': 4, - 'BINARY_SUBSCR': 5, - 'CHECK_EG_MATCH': 6, - 'CHECK_EXC_MATCH': 7, - 'CLEANUP_THROW': 8, - 'DELETE_SUBSCR': 9, - 'END_ASYNC_FOR': 10, - 'END_FOR': 11, - 'END_SEND': 12, - 'EXIT_INIT_CHECK': 13, - 'FORMAT_SIMPLE': 14, - 'FORMAT_WITH_SPEC': 15, - 'GET_AITER': 16, - 'GET_ANEXT': 18, - 'GET_ITER': 19, - 'GET_LEN': 20, - 'GET_YIELD_FROM_ITER': 21, - 'INTERPRETER_EXIT': 22, - 'LOAD_ASSERTION_ERROR': 23, - 'LOAD_BUILD_CLASS': 24, - 'LOAD_LOCALS': 25, - 'MAKE_FUNCTION': 26, - 'MATCH_KEYS': 27, - 'MATCH_MAPPING': 28, - 'MATCH_SEQUENCE': 29, - 'NOP': 30, - 'POP_EXCEPT': 31, - 'POP_TOP': 32, - 'PUSH_EXC_INFO': 33, - 'PUSH_NULL': 34, - 'RETURN_GENERATOR': 35, - 'RETURN_VALUE': 36, - 'SETUP_ANNOTATIONS': 37, - 'STORE_SLICE': 38, - 'STORE_SUBSCR': 39, - 'TO_BOOL': 40, - 'UNARY_INVERT': 41, - 'UNARY_NEGATIVE': 42, - 'UNARY_NOT': 43, - 'WITH_EXCEPT_START': 44, - 'BINARY_OP': 45, - 'BUILD_CONST_KEY_MAP': 46, - 'BUILD_LIST': 47, - 'BUILD_MAP': 48, - 'BUILD_SET': 49, - 'BUILD_SLICE': 50, - 'BUILD_STRING': 51, - 'BUILD_TUPLE': 52, - 'CALL': 53, - 'CALL_FUNCTION_EX': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'ENTER_EXECUTOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_CONST': 83, - 'LOAD_DEREF': 84, - 'LOAD_FAST': 85, - 'LOAD_FAST_AND_CLEAR': 86, - 'LOAD_FAST_CHECK': 87, - 'LOAD_FAST_LOAD_FAST': 88, - 'LOAD_FROM_DICT_OR_DEREF': 89, - 'LOAD_FROM_DICT_OR_GLOBALS': 90, - 'LOAD_GLOBAL': 91, - 'LOAD_NAME': 92, - 'LOAD_SUPER_ATTR': 93, - 'MAKE_CELL': 94, - 'MAP_ADD': 95, - 'MATCH_CLASS': 96, - 'POP_JUMP_IF_FALSE': 97, - 'POP_JUMP_IF_NONE': 98, - 'POP_JUMP_IF_NOT_NONE': 99, - 'POP_JUMP_IF_TRUE': 100, - 'RAISE_VARARGS': 101, - 'RERAISE': 102, - 'RETURN_CONST': 103, - 'SEND': 104, - 'SET_ADD': 105, - 'SET_FUNCTION_ATTRIBUTE': 106, - 'SET_UPDATE': 107, - 'STORE_ATTR': 108, - 'STORE_DEREF': 109, - 'STORE_FAST': 110, - 'STORE_FAST_LOAD_FAST': 111, - 'STORE_FAST_STORE_FAST': 112, - 'STORE_GLOBAL': 113, - 'STORE_NAME': 114, - 'SWAP': 115, - 'UNPACK_EX': 116, - 'UNPACK_SEQUENCE': 117, - 'YIELD_VALUE': 118, - 'INSTRUMENTED_RESUME': 236, - 'INSTRUMENTED_END_FOR': 237, - 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_RETURN_VALUE': 239, - 'INSTRUMENTED_RETURN_CONST': 240, - 'INSTRUMENTED_YIELD_VALUE': 241, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, - 'INSTRUMENTED_FOR_ITER': 243, - 'INSTRUMENTED_CALL': 244, - 'INSTRUMENTED_CALL_KW': 245, - 'INSTRUMENTED_CALL_FUNCTION_EX': 246, - 'INSTRUMENTED_INSTRUCTION': 247, - 'INSTRUMENTED_JUMP_FORWARD': 248, - 'INSTRUMENTED_JUMP_BACKWARD': 249, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, - 'JUMP': 256, - 'JUMP_NO_INTERRUPT': 257, - 'LOAD_CLOSURE': 258, - 'LOAD_METHOD': 259, - 'LOAD_SUPER_METHOD': 260, - 'LOAD_ZERO_SUPER_ATTR': 261, - 'LOAD_ZERO_SUPER_METHOD': 262, - 'POP_BLOCK': 263, - 'SETUP_CLEANUP': 264, - 'SETUP_FINALLY': 265, - 'SETUP_WITH': 266, - 'STORE_FAST_MAYBE_NULL': 267, + 'ENTER_EXECUTOR': 255, + 'ANNOTATIONS_PLACEHOLDER': 256, + 'JUMP': 257, + 'JUMP_IF_FALSE': 258, + 'JUMP_IF_TRUE': 259, + 'JUMP_NO_INTERRUPT': 260, + 'LOAD_CLOSURE': 261, + 'POP_BLOCK': 262, + 'SETUP_CLEANUP': 263, + 'SETUP_FINALLY': 264, + 'SETUP_WITH': 265, + 'STORE_FAST_MAYBE_NULL': 266, } +# CPython 3.13 compatible: opcodes < 44 have no argument HAVE_ARGUMENT = 44 MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py new file mode 100644 index 00000000000..55f8c069591 --- /dev/null +++ b/Lib/_py_warnings.py @@ -0,0 +1,869 @@ +"""Python part of the warnings subsystem.""" + +import sys +import _contextvars +import _thread + + +__all__ = ["warn", "warn_explicit", "showwarning", + "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings", "deprecated"] + + +# Normally '_wm' is sys.modules['warnings'] but for unit tests it can be +# a different module. User code is allowed to reassign global attributes +# of the 'warnings' module, commonly 'filters' or 'showwarning'. So we +# need to lookup these global attributes dynamically on the '_wm' object, +# rather than binding them earlier. The code in this module consistently uses +# '_wm.' rather than using the globals of this module. If the +# '_warnings' C extension is in use, some globals are replaced by functions +# and variables defined in that extension. +_wm = None + + +def _set_module(module): + global _wm + _wm = module + + +# filters contains a sequence of filter 5-tuples +# The components of the 5-tuple are: +# - an action: error, ignore, always, all, default, module, or once +# - a compiled regex that must match the warning message +# - a class representing the warning category +# - a compiled regex that must match the module that is being warned +# - a line number for the line being warning, or 0 to mean any line +# If either if the compiled regexs are None, match anything. +filters = [] + + +defaultaction = "default" +onceregistry = {} +_lock = _thread.RLock() +_filters_version = 1 + + +# If true, catch_warnings() will use a context var to hold the modified +# filters list. Otherwise, catch_warnings() will operate on the 'filters' +# global of the warnings module. +_use_context = sys.flags.context_aware_warnings + + +class _Context: + def __init__(self, filters): + self._filters = filters + self.log = None # if set to a list, logging is enabled + + def copy(self): + context = _Context(self._filters[:]) + if self.log is not None: + context.log = self.log + return context + + def _record_warning(self, msg): + self.log.append(msg) + + +class _GlobalContext(_Context): + def __init__(self): + self.log = None + + @property + def _filters(self): + # Since there is quite a lot of code that assigns to + # warnings.filters, this needs to return the current value of + # the module global. + try: + return _wm.filters + except AttributeError: + # 'filters' global was deleted. Do we need to actually handle this case? + return [] + + +_global_context = _GlobalContext() + + +_warnings_context = _contextvars.ContextVar('warnings_context') + + +def _get_context(): + if not _use_context: + return _global_context + try: + return _wm._warnings_context.get() + except LookupError: + return _global_context + + +def _set_context(context): + assert _use_context + _wm._warnings_context.set(context) + + +def _new_context(): + assert _use_context + old_context = _wm._get_context() + new_context = old_context.copy() + _wm._set_context(new_context) + return old_context, new_context + + +def _get_filters(): + """Return the current list of filters. This is a non-public API used by + module functions and by the unit tests.""" + return _wm._get_context()._filters + + +def _filters_mutated_lock_held(): + _wm._filters_version += 1 + + +def showwarning(message, category, filename, lineno, file=None, line=None): + """Hook to write a warning to a file; replace if you like.""" + msg = _wm.WarningMessage(message, category, filename, lineno, file, line) + _wm._showwarnmsg_impl(msg) + + +def formatwarning(message, category, filename, lineno, line=None): + """Function to format a warning the standard way.""" + msg = _wm.WarningMessage(message, category, filename, lineno, None, line) + return _wm._formatwarnmsg_impl(msg) + + +def _showwarnmsg_impl(msg): + context = _wm._get_context() + if context.log is not None: + context._record_warning(msg) + return + file = msg.file + if file is None: + file = sys.stderr + if file is None: + # sys.stderr is None when run with pythonw.exe: + # warnings get lost + return + text = _wm._formatwarnmsg(msg) + try: + file.write(text) + except OSError: + # the file (probably stderr) is invalid - this warning gets lost. + pass + + +def _formatwarnmsg_impl(msg): + category = msg.category.__name__ + s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" + + if msg.line is None: + try: + import linecache + line = linecache.getline(msg.filename, msg.lineno) + except Exception: + # When a warning is logged during Python shutdown, linecache + # and the import machinery don't work anymore + line = None + linecache = None + else: + line = msg.line + if line: + line = line.strip() + s += " %s\n" % line + + if msg.source is not None: + try: + import tracemalloc + # Logging a warning should not raise a new exception: + # catch Exception, not only ImportError and RecursionError. + except Exception: + # don't suggest to enable tracemalloc if it's not available + suggest_tracemalloc = False + tb = None + else: + try: + suggest_tracemalloc = not tracemalloc.is_tracing() + tb = tracemalloc.get_object_traceback(msg.source) + except Exception: + # When a warning is logged during Python shutdown, tracemalloc + # and the import machinery don't work anymore + suggest_tracemalloc = False + tb = None + + if tb is not None: + s += 'Object allocated at (most recent call last):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + + try: + if linecache is not None: + line = linecache.getline(frame.filename, frame.lineno) + else: + line = None + except Exception: + line = None + if line: + line = line.strip() + s += ' %s\n' % line + elif suggest_tracemalloc: + s += (f'{category}: Enable tracemalloc to get the object ' + f'allocation traceback\n') + return s + + +# Keep a reference to check if the function was replaced +_showwarning_orig = showwarning + + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + try: + sw = _wm.showwarning + except AttributeError: + pass + else: + if sw is not _showwarning_orig: + # warnings.showwarning() was replaced + if not callable(sw): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + sw(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + _wm._showwarnmsg_impl(msg) + + +# Keep a reference to check if the function was replaced +_formatwarning_orig = formatwarning + + +def _formatwarnmsg(msg): + """Function to format a warning the standard way.""" + try: + fw = _wm.formatwarning + except AttributeError: + pass + else: + if fw is not _formatwarning_orig: + # warnings.formatwarning() was replaced + return fw(msg.message, msg.category, + msg.filename, msg.lineno, msg.line) + return _wm._formatwarnmsg_impl(msg) + + +def filterwarnings(action, message="", category=Warning, module="", lineno=0, + append=False): + """Insert an entry into the list of warnings filters (at the front). + + 'action' -- one of "error", "ignore", "always", "all", "default", "module", + or "once" + 'message' -- a regex that the warning message must match + 'category' -- a class that the warning must be a subclass of + 'module' -- a regex that the module name must match + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(message, str): + raise TypeError("message must be a string") + if not isinstance(category, type) or not issubclass(category, Warning): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _wm._add_filter(action, message, category, module, lineno, append=append) + + +def simplefilter(action, category=Warning, lineno=0, append=False): + """Insert a simple entry into the list of warnings filters (at the front). + + A simple filter matches all modules and messages. + 'action' -- one of "error", "ignore", "always", "all", "default", "module", + or "once" + 'category' -- a class that the warning must be a subclass of + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") + _wm._add_filter(action, None, category, None, lineno, append=append) + + +def _filters_mutated(): + # Even though this function is not part of the public API, it's used by + # a fair amount of user code. + with _wm._lock: + _wm._filters_mutated_lock_held() + + +def _add_filter(*item, append): + with _wm._lock: + filters = _wm._get_filters() + if not append: + # Remove possible duplicate filters, so new one will be placed + # in correct place. If append=True and duplicate exists, do nothing. + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + else: + if item not in filters: + filters.append(item) + _wm._filters_mutated_lock_held() + + +def resetwarnings(): + """Clear the list of warning filters, so that no filters are active.""" + with _wm._lock: + del _wm._get_filters()[:] + _wm._filters_mutated_lock_held() + + +class _OptionError(Exception): + """Exception used by option processing helpers.""" + pass + + +# Helper to process -W options passed via sys.warnoptions +def _processoptions(args): + for arg in args: + try: + _wm._setoption(arg) + except _wm._OptionError as msg: + print("Invalid -W option ignored:", msg, file=sys.stderr) + + +# Helper for _processoptions() +def _setoption(arg): + parts = arg.split(':') + if len(parts) > 5: + raise _wm._OptionError("too many fields (max 5): %r" % (arg,)) + while len(parts) < 5: + parts.append('') + action, message, category, module, lineno = [s.strip() + for s in parts] + action = _wm._getaction(action) + category = _wm._getcategory(category) + if message or module: + import re + if message: + message = re.escape(message) + if module: + module = re.escape(module) + r'\z' + if lineno: + try: + lineno = int(lineno) + if lineno < 0: + raise ValueError + except (ValueError, OverflowError): + raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None + else: + lineno = 0 + _wm.filterwarnings(action, message, category, module, lineno) + + +# Helper for _setoption() +def _getaction(action): + if not action: + return "default" + for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'): + if a.startswith(action): + return a + raise _wm._OptionError("invalid action: %r" % (action,)) + + +# Helper for _setoption() +def _getcategory(category): + if not category: + return Warning + if '.' not in category: + import builtins as m + klass = category + else: + module, _, klass = category.rpartition('.') + try: + m = __import__(module, None, None, [klass]) + except ImportError: + raise _wm._OptionError("invalid module name: %r" % (module,)) from None + try: + cat = getattr(m, klass) + except AttributeError: + raise _wm._OptionError("unknown warning category: %r" % (category,)) from None + if not issubclass(cat, Warning): + raise _wm._OptionError("invalid warning category: %r" % (category,)) + return cat + + +def _is_internal_filename(filename): + return 'importlib' in filename and '_bootstrap' in filename + + +def _is_filename_to_skip(filename, skip_file_prefixes): + return any(filename.startswith(prefix) for prefix in skip_file_prefixes) + + +def _is_internal_frame(frame): + """Signal whether the frame is an internal CPython implementation detail.""" + return _is_internal_filename(frame.f_code.co_filename) + + +def _next_external_frame(frame, skip_file_prefixes): + """Find the next frame that doesn't involve Python or user internals.""" + frame = frame.f_back + while frame is not None and ( + _is_internal_filename(filename := frame.f_code.co_filename) or + _is_filename_to_skip(filename, skip_file_prefixes)): + frame = frame.f_back + return frame + + +# Code typically replaced by _warnings +def warn(message, category=None, stacklevel=1, source=None, + *, skip_file_prefixes=()): + """Issue a warning, or maybe ignore it or raise an exception.""" + # Check if message is already a Warning object + if isinstance(message, Warning): + category = message.__class__ + # Check category argument + if category is None: + category = UserWarning + if not (isinstance(category, type) and issubclass(category, Warning)): + raise TypeError("category must be a Warning subclass, " + "not '{:s}'".format(type(category).__name__)) + if not isinstance(skip_file_prefixes, tuple): + # The C version demands a tuple for implementation performance. + raise TypeError('skip_file_prefixes must be a tuple of strs.') + if skip_file_prefixes: + stacklevel = max(2, stacklevel) + # Get context information + try: + if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): + # If frame is too small to care or if the warning originated in + # internal code, then do not try to hide any frames. + frame = sys._getframe(stacklevel) + else: + frame = sys._getframe(1) + # Look for one frame less since the above line starts us off. + for x in range(stacklevel-1): + frame = _next_external_frame(frame, skip_file_prefixes) + if frame is None: + raise ValueError + except ValueError: + globals = sys.__dict__ + filename = "" + lineno = 0 + else: + globals = frame.f_globals + filename = frame.f_code.co_filename + lineno = frame.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + registry = globals.setdefault("__warningregistry__", {}) + _wm.warn_explicit( + message, + category, + filename, + lineno, + module, + registry, + globals, + source=source, + ) + + +def warn_explicit(message, category, filename, lineno, + module=None, registry=None, module_globals=None, + source=None): + lineno = int(lineno) + if module is None: + module = filename or "" + if module[-3:].lower() == ".py": + module = module[:-3] # XXX What about leading pathname? + if isinstance(message, Warning): + text = str(message) + category = message.__class__ + else: + text = message + message = category(message) + key = (text, category, lineno) + with _wm._lock: + if registry is None: + registry = {} + if registry.get('version', 0) != _wm._filters_version: + registry.clear() + registry['version'] = _wm._filters_version + # Quick test for common case + if registry.get(key): + return + # Search the filters + for item in _wm._get_filters(): + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = _wm.defaultaction + # Early exit actions + if action == "ignore": + return + + if action == "error": + raise message + # Other actions + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if _wm.onceregistry.get(oncekey): + return + _wm.onceregistry[oncekey] = 1 + elif action in {"always", "all"}: + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + # Unrecognized actions are errors + raise RuntimeError( + "Unrecognized action (%r) in warnings.filters:\n %s" % + (action, item)) + + # Prime the linecache for formatting, in case the + # "file" is actually in a zipfile or something. + import linecache + linecache.getlines(filename, module_globals) + + # Print message and context + msg = _wm.WarningMessage(message, category, filename, lineno, source=source) + _wm._showwarnmsg(msg) + + +class WarningMessage(object): + + _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", + "line", "source") + + def __init__(self, message, category, filename, lineno, file=None, + line=None, source=None): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line + self.source = source + self._category_name = category.__name__ if category else None + + def __str__(self): + return ("{message : %r, category : %r, filename : %r, lineno : %s, " + "line : %r}" % (self.message, self._category_name, + self.filename, self.lineno, self.line)) + + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + + +class catch_warnings(object): + + """A context manager that copies and restores the warnings filter upon + exiting the context. + + The 'record' argument specifies whether warnings should be captured by a + custom implementation of warnings.showwarning() and be appended to a list + returned by the context manager. Otherwise None is returned by the context + manager. The objects appended to the list are arguments whose attributes + mirror the arguments to showwarning(). + + The 'module' argument is to specify an alternative module to the module + named 'warnings' and imported under that name. This argument is only useful + when testing the warnings module itself. + + If the 'action' argument is not None, the remaining arguments are passed + to warnings.simplefilter() as if it were called immediately on entering the + context. + """ + + def __init__(self, *, record=False, module=None, + action=None, category=Warning, lineno=0, append=False): + """Specify whether to record warnings and if an alternative module + should be used other than sys.modules['warnings']. + + """ + self._record = record + self._module = sys.modules['warnings'] if module is None else module + self._entered = False + if action is None: + self._filter = None + else: + self._filter = (action, category, lineno, append) + + def __repr__(self): + args = [] + if self._record: + args.append("record=True") + if self._module is not sys.modules['warnings']: + args.append("module=%r" % self._module) + name = type(self).__name__ + return "%s(%s)" % (name, ", ".join(args)) + + def __enter__(self): + if self._entered: + raise RuntimeError("Cannot enter %r twice" % self) + self._entered = True + with _wm._lock: + if _use_context: + self._saved_context, context = self._module._new_context() + else: + context = None + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._showwarning = self._module.showwarning + self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._module._filters_mutated_lock_held() + if self._record: + if _use_context: + context.log = log = [] + else: + log = [] + self._module._showwarnmsg_impl = log.append + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig + else: + log = None + if self._filter is not None: + self._module.simplefilter(*self._filter) + return log + + def __exit__(self, *exc_info): + if not self._entered: + raise RuntimeError("Cannot exit %r without entering first" % self) + with _wm._lock: + if _use_context: + self._module._warnings_context.set(self._saved_context) + else: + self._module.filters = self._filters + self._module.showwarning = self._showwarning + self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module._filters_mutated_lock_held() + + +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, /, *args, **kwargs): + if cls is arg: + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + if "__init_subclass__" in arg.__dict__: + # __init_subclass__ is directly present on the decorated class. + # Synthesize a wrapper that calls this method directly. + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python). + # Otherwise, it likely means it's a builtin such as + # object's implementation of __init_subclass__. + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + else: + def __init_subclass__(cls, *args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return super(arg, cls).__init_subclass__(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + import inspect + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + +_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" + + +def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): + """Warn that *name* is deprecated or should be removed. + + RuntimeError is raised if *remove* specifies a major/minor tuple older than + the current Python version or the same version but past the alpha. + + The *message* argument is formatted with *name* and *remove* as a Python + version tuple (e.g. (3, 11)). + + """ + remove_formatted = f"{remove[0]}.{remove[1]}" + if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): + msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha" + raise RuntimeError(msg) + else: + msg = message.format(name=name, remove=remove_formatted) + _wm.warn(msg, DeprecationWarning, stacklevel=3) + + +# Private utility function called by _PyErr_WarnUnawaitedCoroutine +def _warn_unawaited_coroutine(coro): + msg_lines = [ + f"coroutine '{coro.__qualname__}' was never awaited\n" + ] + if coro.cr_origin is not None: + import linecache, traceback + def extract(): + for filename, lineno, funcname in reversed(coro.cr_origin): + line = linecache.getline(filename, lineno) + yield (filename, lineno, funcname, line) + msg_lines.append("Coroutine created at (most recent call last)\n") + msg_lines += traceback.format_list(list(extract())) + msg = "".join(msg_lines).rstrip("\n") + # Passing source= here means that if the user happens to have tracemalloc + # enabled and tracking where the coroutine was created, the warning will + # contain that traceback. This does mean that if they have *both* + # coroutine origin tracking *and* tracemalloc enabled, they'll get two + # partially-redundant tracebacks. If we wanted to be clever we could + # probably detect this case and avoid it, but for now we don't bother. + _wm.warn( + msg, category=RuntimeWarning, stacklevel=2, source=coro + ) + + +def _setup_defaults(): + # Several warning categories are ignored by default in regular builds + if hasattr(sys, 'gettotalrefcount'): + return + _wm.filterwarnings("default", category=DeprecationWarning, module="__main__", append=1) + _wm.simplefilter("ignore", category=DeprecationWarning, append=1) + _wm.simplefilter("ignore", category=PendingDeprecationWarning, append=1) + _wm.simplefilter("ignore", category=ImportWarning, append=1) + _wm.simplefilter("ignore", category=ResourceWarning, append=1) diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index d0efa9ad6bb..98dec3c614d 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -22,10 +22,10 @@ The builtin Unicode codecs use the following interface: - _encode(Unicode_object[,errors='strict']) -> + _encode(Unicode_object[,errors='strict']) -> (string object, bytes consumed) - _decode(char_buffer_obj[,errors='strict']) -> + _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) _encode() interfaces also accept non-Unicode object as @@ -44,47 +44,76 @@ From PyPy v1.0.0 """ -#from unicodecodec import * - -__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode', - 'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode', - 'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode', - 'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode', - 'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build', - 'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode', - 'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode', - 'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode', - 'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',] +# from unicodecodec import * + +__all__ = [ + "register", + "lookup", + "lookup_error", + "register_error", + "encode", + "decode", + "latin_1_encode", + "mbcs_decode", + "readbuffer_encode", + "escape_encode", + "utf_8_decode", + "raw_unicode_escape_decode", + "utf_7_decode", + "unicode_escape_encode", + "latin_1_decode", + "utf_16_decode", + "unicode_escape_decode", + "ascii_decode", + "charmap_encode", + "charmap_build", + "unicode_internal_encode", + "unicode_internal_decode", + "utf_16_ex_decode", + "escape_decode", + "charmap_decode", + "utf_7_encode", + "mbcs_encode", + "ascii_encode", + "utf_16_encode", + "raw_unicode_escape_encode", + "utf_8_encode", + "utf_16_le_encode", + "utf_16_be_encode", + "utf_16_le_decode", + "utf_16_be_decode", + "utf_32_ex_decode", +] import sys import warnings from _codecs import * -def latin_1_encode( obj, errors='strict'): - """None - """ +def latin_1_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeLatin1(obj, len(obj), errors) res = bytes(res) return res, len(obj) + + # XXX MBCS codec might involve ctypes ? def mbcs_decode(): - """None - """ + """None""" pass -def readbuffer_encode( obj, errors='strict'): - """None - """ + +def readbuffer_encode(obj, errors="strict"): + """None""" if isinstance(obj, str): res = obj.encode() else: res = bytes(obj) return res, len(obj) -def escape_encode( obj, errors='strict'): - """None - """ + +def escape_encode(obj, errors="strict"): + """None""" if not isinstance(obj, bytes): raise TypeError("must be bytes") s = repr(obj).encode() @@ -93,85 +122,88 @@ def escape_encode( obj, errors='strict'): v = v.replace(b"'", b"\\'").replace(b'\\"', b'"') return v, len(obj) -def raw_unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) -def utf_7_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeUTF7(data, len(data), errors) - res = ''.join(res) - return res, len(data) +def raw_unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def unicode_escape_encode( obj, errors='strict'): - """None - """ + +def utf_7_decode(data, errors="strict", final=False): + """None""" + res, consumed = PyUnicode_DecodeUTF7(data, len(data), errors, final) + res = "".join(res) + return res, consumed + + +def unicode_escape_encode(obj, errors="strict"): + """None""" res = unicodeescape_string(obj, len(obj), 0) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def latin_1_decode( data, errors='strict'): - """None - """ + +def latin_1_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_16_decode( data, errors='strict', final=False): - """None - """ + +def utf_16_decode(data, errors="strict", final=False): + """None""" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "native", final + ) + res = "".join(res) return res, consumed -def unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) +def unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def ascii_decode( data, errors='strict'): - """None - """ + +def ascii_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeASCII(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def charmap_encode(obj, errors='strict', mapping='latin-1'): - """None - """ + +def charmap_encode(obj, errors="strict", mapping="latin-1"): + """None""" res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors) res = bytes(res) return res, len(obj) + def charmap_build(s): return {ord(c): i for i, c in enumerate(s)} + if sys.maxunicode == 65535: unicode_bytes = 2 else: unicode_bytes = 4 -def unicode_internal_encode( obj, errors='strict'): - """None - """ + +def unicode_internal_encode(obj, errors="strict"): + """None""" if type(obj) == str: p = bytearray() t = [ord(x) for x in obj] for i in t: b = bytearray() for j in range(unicode_bytes): - b.append(i%256) + b.append(i % 256) i >>= 8 if sys.byteorder == "big": b.reverse() @@ -179,12 +211,12 @@ def unicode_internal_encode( obj, errors='strict'): res = bytes(p) return res, len(res) else: - res = "You can do better than this" # XXX make this right + res = "You can do better than this" # XXX make this right return res, len(res) -def unicode_internal_decode( unistr, errors='strict'): - """None - """ + +def unicode_internal_decode(unistr, errors="strict"): + """None""" if type(unistr) == str: return unistr, len(unistr) else: @@ -198,165 +230,418 @@ def unicode_internal_decode( unistr, errors='strict'): start = 0 stop = unicode_bytes step = 1 - while i < len(unistr)-unicode_bytes+1: + while i < len(unistr) - unicode_bytes + 1: t = 0 h = 0 for j in range(start, stop, step): - t += ord(unistr[i+j])<<(h*8) + t += ord(unistr[i + j]) << (h * 8) h += 1 i += unicode_bytes p += chr(t) - res = ''.join(p) + res = "".join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ + +def utf_16_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" if byteorder == 0: - bm = 'native' + bm = "native" elif byteorder == -1: - bm = 'little' + bm = "little" else: - bm = 'big' + bm = "big" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, bm, final + ) + res = "".join(res) return res, consumed, byteorder -# XXX needs error messages when the input is invalid -def escape_decode(data, errors='strict'): - """None - """ + +def utf_32_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" + if byteorder == 0: + if len(data) < 4: + if final and len(data): + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + return "", 0, 0 + if data[0:4] == b"\xff\xfe\x00\x00": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + return "".join(res), consumed + 4, -1 + if data[0:4] == b"\x00\x00\xfe\xff": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + return "".join(res), consumed + 4, 1 + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + + if byteorder == -1: + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + return "".join(res), consumed, -1 + + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + return "".join(res), consumed, 1 + + +def _is_hex_digit(b): + return ( + 0x30 <= b <= 0x39 # 0-9 + or 0x41 <= b <= 0x46 # A-F + or 0x61 <= b <= 0x66 + ) # a-f + + +def escape_decode(data, errors="strict"): + if isinstance(data, str): + data = data.encode("latin-1") l = len(data) i = 0 res = bytearray() while i < l: - - if data[i] == '\\': + if data[i] == 0x5C: # '\\' i += 1 if i >= l: raise ValueError("Trailing \\ in string") - else: - if data[i] == '\\': - res += b'\\' - elif data[i] == 'n': - res += b'\n' - elif data[i] == 't': - res += b'\t' - elif data[i] == 'r': - res += b'\r' - elif data[i] == 'b': - res += b'\b' - elif data[i] == '\'': - res += b'\'' - elif data[i] == '\"': - res += b'\"' - elif data[i] == 'f': - res += b'\f' - elif data[i] == 'a': - res += b'\a' - elif data[i] == 'v': - res += b'\v' - elif '0' <= data[i] <= '9': - # emulate a strange wrap-around behavior of CPython: - # \400 is the same as \000 because 0400 == 256 - octal = data[i:i+3] - res.append(int(octal, 8) & 0xFF) - i += 2 - elif data[i] == 'x': - hexa = data[i+1:i+3] - res.append(int(hexa, 16)) + ch = data[i] + if ch == 0x5C: + res.append(0x5C) # \\ + elif ch == 0x27: + res.append(0x27) # \' + elif ch == 0x22: + res.append(0x22) # \" + elif ch == 0x61: + res.append(0x07) # \a + elif ch == 0x62: + res.append(0x08) # \b + elif ch == 0x66: + res.append(0x0C) # \f + elif ch == 0x6E: + res.append(0x0A) # \n + elif ch == 0x72: + res.append(0x0D) # \r + elif ch == 0x74: + res.append(0x09) # \t + elif ch == 0x76: + res.append(0x0B) # \v + elif ch == 0x0A: + pass # \ continuation + elif 0x30 <= ch <= 0x37: # \0-\7 octal + val = ch - 0x30 + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + res.append(val & 0xFF) + elif ch == 0x78: # \x hex + hex_count = 0 + for j in range(1, 3): + if i + j < l and _is_hex_digit(data[i + j]): + hex_count += 1 + else: + break + if hex_count < 2: + if errors == "strict": + raise ValueError("invalid \\x escape at position %d" % (i - 1)) + elif errors == "replace": + res.append(0x3F) # '?' + i += hex_count + else: + res.append(int(bytes(data[i + 1 : i + 3]), 16)) i += 2 + else: + import warnings + + warnings.warn( + '"\\%c" is an invalid escape sequence' % ch + if 0x20 <= ch < 0x7F + else '"\\x%02x" is an invalid escape sequence' % ch, + DeprecationWarning, + stacklevel=2, + ) + res.append(0x5C) + res.append(ch) else: res.append(data[i]) i += 1 - res = bytes(res) - return res, len(res) + return bytes(res), l + -def charmap_decode( data, errors='strict', mapping=None): - """None - """ +def charmap_decode(data, errors="strict", mapping=None): + """None""" res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_7_encode( obj, errors='strict'): - """None - """ +def utf_7_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def mbcs_encode( obj, errors='strict'): - """None - """ + +def mbcs_encode(obj, errors="strict"): + """None""" pass + + ## return (PyUnicode_EncodeMBCS( -## (obj), +## (obj), ## len(obj), ## errors), ## len(obj)) - -def ascii_encode( obj, errors='strict'): - """None - """ + +def ascii_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeASCII(obj, len(obj), errors) res = bytes(res) return res, len(obj) -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') + +def utf_16_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "native") res = bytes(res) return res, len(obj) -def raw_unicode_escape_encode( obj, errors='strict'): - """None - """ + +def raw_unicode_escape_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj)) res = bytes(res) return res, len(obj) -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') + +def utf_16_le_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "little") res = bytes(res) return res, len(obj) -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') + +def utf_16_be_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "big") res = bytes(res) return res, len(obj) -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = ''.join(res) + +def utf_16_le_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) return res, consumed -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = ''.join(res) + +def utf_16_be_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) return res, consumed +def STORECHAR32(ch, byteorder): + """Store a 32-bit character as 4 bytes in the specified byte order.""" + b0 = ch & 0xFF + b1 = (ch >> 8) & 0xFF + b2 = (ch >> 16) & 0xFF + b3 = (ch >> 24) & 0xFF + if byteorder == "little": + return [b0, b1, b2, b3] + else: # big-endian + return [b3, b2, b1, b0] + + +def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): + """Encode a Unicode string to UTF-32.""" + p = [] + bom = sys.byteorder + + if byteorder == "native": + bom = sys.byteorder + # Add BOM for native encoding + p += STORECHAR32(0xFEFF, bom) + + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" + + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR32(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + p += STORECHAR32(ord(c), bom) + else: + p += STORECHAR32(ch, bom) + pos += 1 + + return p + + +def utf_32_encode(obj, errors="strict"): + """UTF-32 encoding with BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "native") + return bytes(encoded), len(obj) + + +def utf_32_le_encode(obj, errors="strict"): + """UTF-32 little-endian encoding without BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "little") + return bytes(encoded), len(obj) + + +def utf_32_be_encode(obj, errors="strict"): + """UTF-32 big-endian encoding without BOM.""" + res = PyUnicode_EncodeUTF32(obj, len(obj), errors, "big") + res = bytes(res) + return res, len(obj) + + +def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=0): + """Decode UTF-32 encoded bytes to Unicode string.""" + if size == 0: + return [], 0, 0 + + result = [] + pos = 0 + aligned_size = (size // 4) * 4 + + while pos + 3 < aligned_size: + if byteorder == "little": + ch = ( + data[pos] + | (data[pos + 1] << 8) + | (data[pos + 2] << 16) + | (data[pos + 3] << 24) + ) + else: # big-endian + ch = ( + (data[pos] << 24) + | (data[pos + 1] << 16) + | (data[pos + 2] << 8) + | data[pos + 3] + ) + + # Validate code point + if ch > 0x10FFFF: + if errors == "strict": + raise UnicodeDecodeError( + "utf-32", + bytes(data), + pos, + pos + 4, + "codepoint not in range(0x110000)", + ) + elif errors == "replace": + result.append("\ufffd") + # 'ignore' - skip this character + pos += 4 + elif 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + result.append(chr(ch)) + pos += 4 + else: + msg = "code point in surrogate code point range(0xd800, 0xe000)" + res, pos = unicode_call_errorhandler( + errors, "utf-32", msg, data, pos, pos + 4, True + ) + result.append(res) + else: + result.append(chr(ch)) + pos += 4 + + # Handle trailing incomplete bytes + if pos < size: + if final: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "truncated data", data, pos, size, True + ) + if res: + result.append(res) + + return result, pos, 0 + + +def utf_32_decode(data, errors="strict", final=0): + """UTF-32 decoding with BOM detection.""" + if len(data) >= 4: + # Check for BOM + if data[0:4] == b"\xff\xfe\x00\x00": + # UTF-32 LE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + res = "".join(res) + return res, consumed + 4 + elif data[0:4] == b"\x00\x00\xfe\xff": + # UTF-32 BE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + res = "".join(res) + return res, consumed + 4 + + # Default to little-endian if no BOM + byteorder = "little" if sys.byteorder == "little" else "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, byteorder, final + ) + res = "".join(res) + return res, consumed + + +def utf_32_le_decode(data, errors="strict", final=0): + """UTF-32 little-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) + return res, consumed + + +def utf_32_be_decode(data, errors="strict", final=0): + """UTF-32 big-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) + return res, consumed # ---------------------------------------------------------------------- @@ -364,9 +649,9 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ##import sys ##""" Python implementation of CPythons builtin unicode codecs. ## -## Generally the functions in this module take a list of characters an returns +## Generally the functions in this module take a list of characters an returns ## a list of characters. -## +## ## For use in the PyPy project""" @@ -376,283 +661,496 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ## 1 - special ## 2 - whitespace (optional) ## 3 - RFC2152 Set O (optional) - + utf7_special = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 3, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 3, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 1, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 1, + 1, ] -unicode_latin1 = [None]*256 +unicode_latin1 = [None] * 256 def SPECIAL(c, encodeO, encodeWS): c = ord(c) - return (c>127 or utf7_special[c] == 1) or \ - (encodeWS and (utf7_special[(c)] == 2)) or \ - (encodeO and (utf7_special[(c)] == 3)) + return ( + (c > 127 or utf7_special[c] == 1) + or (encodeWS and (utf7_special[(c)] == 2)) + or (encodeO and (utf7_special[(c)] == 3)) + ) + + def B64(n): - return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]]) + return bytes( + [ + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[ + (n) & 0x3F + ] + ] + ) + + def B64CHAR(c): - return (c.isalnum() or (c) == b'+' or (c) == b'/') + return c.isalnum() or (c) == b"+" or (c) == b"/" + + def UB64(c): - if (c) == b'+' : - return 62 - elif (c) == b'/': - return 63 - elif (c) >= b'a': - return ord(c) - 71 - elif (c) >= b'A': - return ord(c) - 65 - else: + if (c) == b"+": + return 62 + elif (c) == b"/": + return 63 + elif (c) >= b"a": + return ord(c) - 71 + elif (c) >= b"A": + return ord(c) - 65 + else: return ord(c) + 4 -def ENCODE( ch, bits) : + +def ENCODE(ch, bits): out = [] - while (bits >= 6): - out += B64(ch >> (bits-6)) - bits -= 6 + while bits >= 6: + out += B64(ch >> (bits - 6)) + bits -= 6 return out, bits -def PyUnicode_DecodeUTF7(s, size, errors): - starts = s - errmsg = "" - inShift = 0 - bitsleft = 0 - charsleft = 0 - surrogate = 0 - p = [] - errorHandler = None - exc = None +def _IS_BASE64(ch): + return ( + (ord("A") <= ch <= ord("Z")) + or (ord("a") <= ch <= ord("z")) + or (ord("0") <= ch <= ord("9")) + or ch == ord("+") + or ch == ord("/") + ) - if (size == 0): - return '' + +def _FROM_BASE64(ch): + if ch == ord("+"): + return 62 + if ch == ord("/"): + return 63 + if ch >= ord("a"): + return ch - 71 + if ch >= ord("A"): + return ch - 65 + if ch >= ord("0"): + return ch - ord("0") + 52 + return -1 + + +def _DECODE_DIRECT(ch): + return ch <= 127 and ch != ord("+") + + +def PyUnicode_DecodeUTF7(s, size, errors, final=False): + if size == 0: + return [], 0 + + p = [] + inShift = False + base64bits = 0 + base64buffer = 0 + surrogate = 0 + startinpos = 0 + shiftOutStart = 0 i = 0 + while i < size: - - ch = bytes([s[i]]) - if (inShift): - if ((ch == b'-') or not B64CHAR(ch)): - inShift = 0 + ch = s[i] + if inShift: + if _IS_BASE64(ch): + base64buffer = (base64buffer << 6) | _FROM_BASE64(ch) + base64bits += 6 i += 1 - - while (bitsleft >= 16): - outCh = ((charsleft) >> (bitsleft-16)) & 0xffff - bitsleft -= 16 - - if (surrogate): - ## We have already generated an error for the high surrogate - ## so let's not bother seeing if the low surrogate is correct or not - surrogate = 0 - elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF): - ## This is a surrogate pair. Unfortunately we can't represent - ## it in a 16-bit character - surrogate = 1 - msg = "code pairs are not supported" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - p.append(out) - bitsleft = 0 - break + if base64bits >= 16: + outCh = (base64buffer >> (base64bits - 16)) & 0xFFFF + base64bits -= 16 + base64buffer &= (1 << base64bits) - 1 + if surrogate: + if 0xDC00 <= outCh <= 0xDFFF: + ch2 = ( + 0x10000 + + ((surrogate - 0xD800) << 10) + + (outCh - 0xDC00) + ) + p.append(chr(ch2)) + surrogate = 0 + continue + else: + p.append(chr(surrogate)) + surrogate = 0 + if 0xD800 <= outCh <= 0xDBFF: + surrogate = outCh else: - p.append(chr(outCh )) - #p += out - if (bitsleft >= 6): -## /* The shift sequence has a partial character in it. If -## bitsleft < 6 then we could just classify it as padding -## but that is not the case here */ - msg = "partial character in shift sequence" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - -## /* According to RFC2152 the remaining bits should be zero. We -## choose to signal an error/insert a replacement character -## here so indicate the potential of a misencoded character. */ - -## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */ -## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))): -## raise UnicodeDecodeError, "non-zero padding bits in shift sequence" - if (ch == b'-') : - if ((i < size) and (s[i] == '-')) : - p += '-' - inShift = 1 - - elif SPECIAL(ch, 0, 0) : - raise UnicodeDecodeError("unexpected special character") - - else: - p.append(chr(ord(ch))) + p.append(chr(outCh)) else: - charsleft = (charsleft << 6) | UB64(ch) - bitsleft += 6 - i += 1 -## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate); - elif ( ch == b'+' ): + inShift = False + if base64bits > 0: + if base64bits >= 6: + i += 1 + errmsg = "partial character in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + else: + if base64buffer != 0: + i += 1 + errmsg = "non-zero padding bits in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + if surrogate and _DECODE_DIRECT(ch): + p.append(chr(surrogate)) + surrogate = 0 + if ch == ord("-"): + i += 1 + elif ch == ord("+"): startinpos = i i += 1 - if (i= 6 or (base64bits > 0 and base64buffer != 0): + errmsg = "unterminated shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, size + ) + p.append(out) + + return p, size + + +def _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + c = ord(ch) if isinstance(ch, str) else ch + if c > 127: + return False + if utf7_special[c] == 0: + return True + if utf7_special[c] == 2: + return not encodeWhiteSpace + if utf7_special[c] == 3: + return not encodeSetO + return False - if (inShift) : - #XXX This aint right - endinpos = size - raise UnicodeDecodeError("unterminated shift sequence") - - return p def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors): - -# /* It might be possible to tighten this worst case */ inShift = False - i = 0 - bitsleft = 0 - charsleft = 0 + base64bits = 0 + base64buffer = 0 out = [] - for ch in s: - if (not inShift) : - if (ch == '+'): - out.append(b'+-') - elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - charsleft = ord(ch) - bitsleft = 16 - out.append(b'+') - p, bitsleft = ENCODE( charsleft, bitsleft) - out.append(p) - inShift = bitsleft > 0 + + for i, ch in enumerate(s): + ch_ord = ord(ch) + if inShift: + if _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + # shifting out + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + base64buffer = 0 + base64bits = 0 + inShift = False + if B64CHAR(ch) or ch == "-": + out.append(b"-") + out.append(bytes([ch_ord])) else: - out.append(bytes([ord(ch)])) + # encode character in base64 + if ch_ord >= 0x10000: + # split into surrogate pair + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 else: - if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - out.append(B64((charsleft) << (6-bitsleft))) - charsleft = 0 - bitsleft = 0 -## /* Characters not in the BASE64 set implicitly unshift the sequence -## so no '-' is required, except if the character is itself a '-' */ - if (B64CHAR(ch) or ch == '-'): - out.append(b'-') - inShift = False - out.append(bytes([ord(ch)])) + if ch == "+": + out.append(b"+-") + elif _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + out.append(bytes([ch_ord])) else: - bitsleft += 16 - charsleft = (((charsleft) << 16) | ord(ch)) - p, bitsleft = ENCODE(charsleft, bitsleft) - out.append(p) -## /* If the next character is special then we dont' need to terminate -## the shift sequence. If the next character is not a BASE64 character -## or '-' then the shift sequence will be terminated implicitly and we -## don't have to insert a '-'. */ - - if (bitsleft == 0): - if (i + 1 < size): - ch2 = s[i+1] - - if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)): - pass - elif (B64CHAR(ch2) or ch2 == '-'): - out.append(b'-') - inShift = False - else: + out.append(b"+") + inShift = True + # encode character in base64 + if ch_ord >= 0x10000: + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + + if base64bits == 0: + if i + 1 < size: + ch2 = s[i + 1] + if _ENCODE_DIRECT(ch2, encodeSetO, encodeWhiteSpace): + if B64CHAR(ch2) or ch2 == "-": + out.append(b"-") inShift = False else: - out.append(b'-') + out.append(b"-") inShift = False - i += 1 - - if (bitsleft): - out.append(B64(charsleft << (6-bitsleft) ) ) - out.append(b'-') + + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + if inShift: + out.append(b"-") return out -unicode_empty = '' -def unicodeescape_string(s, size, quotes): +unicode_empty = "" + +def unicodeescape_string(s, size, quotes): p = [] - if (quotes) : - if (s.find('\'') != -1 and s.find('"') == -1): + if quotes: + if s.find("'") != -1 and s.find('"') == -1: p.append(b'"') else: - p.append(b'\'') + p.append(b"'") pos = 0 - while (pos < size): + while pos < size: ch = s[pos] - #/* Escape quotes */ - if (quotes and (ch == p[1] or ch == '\\')): - p.append(b'\\%c' % ord(ch)) + # /* Escape quotes */ + if quotes and (ch == p[1] or ch == "\\"): + p.append(b"\\%c" % ord(ch)) pos += 1 continue -#ifdef Py_UNICODE_WIDE - #/* Map 21-bit characters to '\U00xxxxxx' */ - elif (ord(ch) >= 0x10000): - p.append(b'\\U%08x' % ord(ch)) + # ifdef Py_UNICODE_WIDE + # /* Map 21-bit characters to '\U00xxxxxx' */ + elif ord(ch) >= 0x10000: + p.append(b"\\U%08x" % ord(ch)) pos += 1 - continue -#endif - #/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ - elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00): + continue + # endif + # /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ + elif ord(ch) >= 0xD800 and ord(ch) < 0xDC00: pos += 1 ch2 = s[pos] - - if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF): + + if ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF: ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000 - p.append(b'\\U%08x' % ucs) + p.append(b"\\U%08x" % ucs) pos += 1 continue - - #/* Fall through: isolated surrogates are copied as-is */ + + # /* Fall through: isolated surrogates are copied as-is */ pos -= 1 - - #/* Map 16-bit characters to '\uxxxx' */ - if (ord(ch) >= 256): - p.append(b'\\u%04x' % ord(ch)) - - #/* Map special whitespace to '\t', \n', '\r' */ - elif (ch == '\t'): - p.append(b'\\t') - - elif (ch == '\n'): - p.append(b'\\n') - - elif (ch == '\r'): - p.append(b'\\r') - - elif (ch == '\\'): - p.append(b'\\\\') - - #/* Map non-printable US ASCII to '\xhh' */ - elif (ch < ' ' or ch >= chr(0x7F)) : - p.append(b'\\x%02x' % ord(ch)) - #/* Copy everything else as-is */ + + # /* Map 16-bit characters to '\uxxxx' */ + if ord(ch) >= 256: + p.append(b"\\u%04x" % ord(ch)) + + # /* Map special whitespace to '\t', \n', '\r' */ + elif ch == "\t": + p.append(b"\\t") + + elif ch == "\n": + p.append(b"\\n") + + elif ch == "\r": + p.append(b"\\r") + + elif ch == "\\": + p.append(b"\\\\") + + # /* Map non-printable US ASCII to '\xhh' */ + elif ch < " " or ch >= chr(0x7F): + p.append(b"\\x%02x" % ord(ch)) + # /* Copy everything else as-is */ else: p.append(bytes([ord(ch)])) pos += 1 - if (quotes): + if quotes: p.append(p[0]) return p -def PyUnicode_DecodeASCII(s, size, errors): -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : +def PyUnicode_DecodeASCII(s, size, errors): + # /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if size == 1 and ord(s) < 128: return [chr(ord(s))] - if (size == 0): - return [''] #unicode('') + if size == 0: + return [""] # unicode('') p = [] pos = 0 while pos < len(s): @@ -661,54 +1159,50 @@ def PyUnicode_DecodeASCII(s, size, errors): p += chr(c) pos += 1 else: - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) + errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1 + ) p += res[0] pos = res[1] return p -def PyUnicode_EncodeASCII(p, size, errors): +def PyUnicode_EncodeASCII(p, size, errors): return unicode_encode_ucs1(p, size, errors, 128) -def PyUnicode_AsASCIIString(unistr): +def PyUnicode_AsASCIIString(unistr): if not type(unistr) == str: raise TypeError - return PyUnicode_EncodeASCII(str(unistr), - len(str), - None) + return PyUnicode_EncodeASCII(unistr, len(unistr), None) -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - bo = 0 #/* assume native ordering by default */ +def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=True): + bo = 0 # /* assume native ordering by default */ consumed = 0 errmsg = "" - if sys.byteorder == 'little': + if sys.byteorder == "little": ihi = 1 ilo = 0 else: ihi = 0 ilo = 1 - - #/* Unpack UTF-16 encoded data */ + # /* Unpack UTF-16 encoded data */ -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ + ## /* Check for BOM marks (U+FEFF) in the input and adjust current + ## byte order setting accordingly. In native mode, the leading BOM + ## mark is skipped, in all other modes, it is copied to the output + ## stream as-is (giving a ZWNBSP character). */ q = 0 p = [] - if byteorder == 'native': - if (size >= 2): + if byteorder == "native": + if size >= 2: bom = (s[ihi] << 8) | s[ilo] -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): + # ifdef BYTEORDER_IS_LITTLE_ENDIAN + if sys.byteorder == "little": + if bom == 0xFEFF: q += 2 bo = -1 elif bom == 0xFFFE: @@ -721,118 +1215,143 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru elif bom == 0xFFFE: q += 2 bo = -1 - elif byteorder == 'little': + elif byteorder == "little": bo = -1 else: bo = 1 - - if (size == 0): - return [''], 0, bo - - if (bo == -1): - #/* force LE */ + + if size == 0: + return [""], 0, bo + + if bo == -1: + # /* force LE */ ihi = 1 ilo = 0 - elif (bo == 1): - #/* force BE */ + elif bo == 1: + # /* force BE */ ihi = 0 ilo = 1 - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): + while q < len(s): + # /* remaining bytes at the end? (size should be even) */ + if len(s) - q < 2: if not final: break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (s[q+ihi] << 8) | s[q+ilo] - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): + res, q = unicode_call_errorhandler( + errors, "utf-16", "truncated data", s, q, len(s), True + ) + p.append(res) + break + + ch = (s[q + ihi] << 8) | s[q + ilo] + + if ch < 0xD800 or ch > 0xDFFF: p.append(chr(ch)) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += [chr(ch), chr(ch2)] + continue + + # /* UTF-16 code pair: high surrogate */ + if 0xD800 <= ch <= 0xDBFF: + if q + 4 <= len(s): + ch2 = (s[q + 2 + ihi] << 8) | s[q + 2 + ilo] + if 0xDC00 <= ch2 <= 0xDFFF: + # Valid surrogate pair - always assemble + p.append(chr((((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000)) + q += 4 + continue else: - p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) - #endif + # High surrogate followed by non-low-surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True + ) + p.append(res) + else: + # High surrogate at end of data + if not final: + break + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "unexpected end of data", s, q, len(s), True + ) + p.append(res) + else: + # Low surrogate without preceding high surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal encoding", s, q, q + 2, True + ) + p.append(res) - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - return p, q, bo + # moved out of local scope, especially because it didn't # have any nested variables. + def STORECHAR(CH, byteorder): - hi = (CH >> 8) & 0xff - lo = CH & 0xff - if byteorder == 'little': + hi = (CH >> 8) & 0xFF + lo = CH & 0xFF + if byteorder == "little": return [lo, hi] else: return [hi, lo] -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): -# /* Offsets from p for storing byte pairs in the right order. */ +def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"): + # /* Offsets from p for storing byte pairs in the right order. */ - p = [] bom = sys.byteorder - if (byteorder == 'native'): - + if byteorder == "native": bom = sys.byteorder p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" - - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) - - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + cp = ord(c) + cp2 = 0 + if cp >= 0x10000: + cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF) + cp = 0xD800 | ((cp - 0x10000) >> 10) + p += STORECHAR(cp, bom) + if cp2: + p += STORECHAR(cp2, bom) + else: + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF) + ch = 0xD800 | ((ch - 0x10000) >> 10) + p += STORECHAR(ch, bom) + if ch2: + p += STORECHAR(ch2, bom) + pos += 1 return p @@ -840,119 +1359,149 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): def PyUnicode_DecodeMBCS(s, size, errors): pass + def PyUnicode_EncodeMBCS(p, size, errors): pass -def unicode_call_errorhandler(errors, encoding, - reason, input, startinpos, endinpos, decode=True): - + +def unicode_call_errorhandler( + errors, encoding, reason, input, startinpos, endinpos, decode=True +): errorHandler = lookup_error(errors) if decode: - exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeDecodeError( + encoding, input, startinpos, endinpos, reason + ) else: - exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeEncodeError( + encoding, input, startinpos, endinpos, reason + ) res = errorHandler(exceptionObject) - if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int): + if ( + isinstance(res, tuple) + and isinstance(res[0], (str, bytes)) + and isinstance(res[1], int) + ): newpos = res[1] - if (newpos < 0): + if newpos < 0: newpos = len(input) + newpos if newpos < 0 or newpos > len(input): - raise IndexError( "position %d from error handler out of bounds" % newpos) + raise IndexError("position %d from error handler out of bounds" % newpos) return res[0], newpos else: - raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) + raise TypeError( + "encoding error handler must return (unicode, int) tuple, not %s" + % repr(res) + ) + + +# /* --- Latin-1 Codec ------------------------------------------------------ */ -#/* --- Latin-1 Codec ------------------------------------------------------ */ def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] + # /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ + ## if (size == 1): + ## return [PyUnicode_FromUnicode(s, 1)] pos = 0 p = [] - while (pos < size): + while pos < size: p += chr(s[pos]) pos += 1 return p + def unicode_encode_ucs1(p, size, errors, limit): - if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return [] res = bytearray() pos = 0 while pos < len(p): - #for ch in p: + # for ch in p: ch = p[pos] - + if ord(ch) < limit: res.append(ord(ch)) pos += 1 else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 + # /* startpos for collecting unencodable chars */ + collstart = pos + collend = pos + 1 while collend < len(p) and ord(p[collend]) >= limit: collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += x[0].encode() + x = unicode_call_errorhandler( + errors, encoding, reason, p, collstart, collend, False + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += replacement + else: + res += replacement.encode() pos = x[1] - + return res + def PyUnicode_EncodeLatin1(p, size, errors): res = unicode_encode_ucs1(p, size, errors, 256) return res -hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)] + +hexdigits = [ord(hex(i)[-1]) for i in range(16)] + [ + ord(hex(i)[-1].upper()) for i in range(10, 16) +] + def hex_number_end(s, pos, digits): target_end = pos + digits - while pos < target_end and pos < len(s) and s[pos] in hexdigits: + while pos < target_end and pos < len(s) and s[pos] in hexdigits: pos += 1 return pos + def hexescape(s, pos, digits, message, errors): ch = 0 p = [] number_end = hex_number_end(s, pos, digits) if number_end - pos != digits: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, number_end + ) p.append(x[0]) pos = x[1] else: - ch = int(s[pos:pos+digits], 16) - #/* when we get here, ch is a 32-bit unicode character */ + ch = int(s[pos : pos + digits], 16) + # /* when we get here, ch is a 32-bit unicode character */ if ch <= sys.maxunicode: p.append(chr(ch)) pos += digits - elif (ch <= 0x10ffff): + elif ch <= 0x10FFFF: ch -= 0x10000 p.append(chr(0xD800 + (ch >> 10))) - p.append(chr(0xDC00 + (ch & 0x03FF))) + p.append(chr(0xDC00 + (ch & 0x03FF))) pos += digits else: message = "illegal Unicode character" - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, - pos+digits) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, pos + digits + ) p.append(x[0]) pos = x[1] res = p return res, pos + def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 - if (size == 0): - return '' - if isinstance(s, str): s = s.encode() @@ -960,131 +1509,168 @@ def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): p = [] pos = 0 - while (pos < size): -## /* Non-escape characters are interpreted as Unicode ordinals */ - if (chr(s[pos]) != '\\') : + while pos < size: + ## /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): p.append(chr(s[pos])) pos += 1 continue -## /* \ - Escapes */ - else: - pos += 1 - if pos >= len(s): - errmessage = "\\ at end of string" - unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size) - ch = chr(s[pos]) - pos += 1 - ## /* \x escapes */ - if ch == '\n': pass - elif ch == '\\': p += '\\' - elif ch == '\'': p += '\'' - elif ch == '\"': p += '\"' - elif ch == 'b' : p += '\b' - elif ch == 'f' : p += '\014' #/* FF */ - elif ch == 't' : p += '\t' - elif ch == 'n' : p += '\n' - elif ch == 'r' : p += '\r' - elif ch == 'v' : p += '\013' #break; /* VT */ - elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */ - elif '0' <= ch <= '7': - x = ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - p.append(chr(x)) - ## /* hex escapes */ - ## /* \xXX */ - elif ch == 'x': + ## /* \ - Escapes */ + escape_start = pos + pos += 1 + if pos >= size: + if not final: + pos = escape_start + break + errmessage = "\\ at end of string" + unicode_call_errorhandler( + errors, "unicodeescape", errmessage, s, pos - 1, size + ) + break + ch = chr(s[pos]) + pos += 1 + ## /* \x escapes */ + if ch == "\n": + pass + elif ch == "\\": + p += "\\" + elif ch == "'": + p += "'" + elif ch == '"': + p += '"' + elif ch == "b": + p += "\b" + elif ch == "f": + p += "\014" # /* FF */ + elif ch == "t": + p += "\t" + elif ch == "n": + p += "\n" + elif ch == "r": + p += "\r" + elif ch == "v": + p += "\013" # break; /* VT */ + elif ch == "a": + p += "\007" # break; /* BEL, not classic C */ + elif "0" <= ch <= "7": + x = ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + p.append(chr(x)) + ## /* hex escapes */ + ## /* \xXX */ + elif ch in ("x", "u", "U"): + if ch == "x": digits = 2 message = "truncated \\xXX escape" - x = hexescape(s, pos, digits, message, errors) - p += x[0] - pos = x[1] - - # /* \uXXXX */ - elif ch == 'u': + elif ch == "u": digits = 4 message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + number_end = hex_number_end(s, pos, digits) + if number_end - pos != digits: + if not final: + pos = escape_start + break x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] - - # /* \UXXXXXXXX */ - elif ch == 'U': - digits = 8 - message = "truncated \\UXXXXXXXX escape" + else: x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] -## /* \N{name} */ - elif ch == 'N': - message = "malformed \\N character escape" - # pos += 1 - look = pos - try: - import unicodedata - except ImportError: - message = "\\N escapes not supported (can't load unicodedata module)" - unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, size) - if look < size and chr(s[look]) == '{': - #/* look for the closing brace */ - while (look < size and chr(s[look]) != '}'): - look += 1 - if (look > pos+1 and look < size and chr(s[look]) == '}'): - #/* found a name. look it up in the unicode database */ - message = "unknown Unicode character name" - st = s[pos+1:look] - try: - chr_codec = unicodedata.lookup("%s" % st) - except LookupError as e: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = chr_codec, look + 1 - p.append(x[0]) - pos = x[1] - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) + ## /* \N{name} */ + elif ch == "N": + message = "malformed \\N character escape" + look = pos + try: + import unicodedata + except ImportError: + message = "\\N escapes not supported (can't load unicodedata module)" + unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, size + ) + continue + if look < size and chr(s[look]) == "{": + # /* look for the closing brace */ + while look < size and chr(s[look]) != "}": + look += 1 + if look > pos + 1 and look < size and chr(s[look]) == "}": + # /* found a name. look it up in the unicode database */ + message = "unknown Unicode character name" + st = s[pos + 1 : look] + try: + chr_codec = unicodedata.lookup("%s" % st) + except LookupError as e: + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + else: + x = chr_codec, look + 1 + p.append(x[0]) + pos = x[1] + else: + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] else: - if not found_invalid_escape: - found_invalid_escape = True - warnings.warn("invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2) - p.append('\\') - p.append(ch) - return p + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] + else: + if not found_invalid_escape: + found_invalid_escape = True + warnings.warn( + "invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2 + ) + p.append("\\") + p.append(ch) + return p, pos + def PyUnicode_EncodeRawUnicodeEscape(s, size): - - if (size == 0): - return b'' + if size == 0: + return b"" p = bytearray() for ch in s: -# /* Map 32-bit characters to '\Uxxxxxxxx' */ - if (ord(ch) >= 0x10000): - p += b'\\U%08x' % ord(ch) - elif (ord(ch) >= 256) : -# /* Map 16-bit characters to '\uxxxx' */ - p += b'\\u%04x' % (ord(ch)) -# /* Copy everything else as-is */ + # /* Map 32-bit characters to '\Uxxxxxxxx' */ + if ord(ch) >= 0x10000: + p += b"\\U%08x" % ord(ch) + elif ord(ch) >= 256: + # /* Map 16-bit characters to '\uxxxx' */ + p += b"\\u%04x" % (ord(ch)) + # /* Copy everything else as-is */ else: p.append(ord(ch)) - - #p += '\0' + + # p += '\0' return p -def charmapencode_output(c, mapping): +def charmapencode_output(c, mapping): rep = mapping[c] - if isinstance(rep, int) or isinstance(rep, int): + if isinstance(rep, int): if rep < 256: return [rep] else: @@ -1098,144 +1684,156 @@ def charmapencode_output(c, mapping): else: raise TypeError("character mapping must return integer, None or str") -def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'): -## /* the following variable is used for caching string comparisons -## * -1=not initialized, 0=unknown, 1=strict, 2=replace, -## * 3=ignore, 4=xmlcharrefreplace */ +def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"): + ## /* the following variable is used for caching string comparisons + ## * -1=not initialized, 0=unknown, 1=strict, 2=replace, + ## * 3=ignore, 4=xmlcharrefreplace */ -# /* Default to Latin-1 */ - if mapping == 'latin-1': + # /* Default to Latin-1 */ + if mapping == "latin-1": return PyUnicode_EncodeLatin1(p, size, errors) - if (size == 0): - return b'' + if size == 0: + return b"" inpos = 0 res = [] - while (inpos", p, inpos, inpos+1, False) - try: - for y in x[0]: - res += charmapencode_output(ord(y), mapping) - except KeyError: - raise UnicodeEncodeError("charmap", p, inpos, inpos+1, - "character maps to ") + x = unicode_call_errorhandler( + errors, + "charmap", + "character maps to ", + p, + inpos, + inpos + 1, + False, + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += list(replacement) + else: + try: + for y in replacement: + res += charmapencode_output(ord(y), mapping) + except KeyError: + raise UnicodeEncodeError( + "charmap", p, inpos, inpos + 1, "character maps to " + ) inpos += 1 return res -def PyUnicode_DecodeCharmap(s, size, mapping, errors): -## /* Default to Latin-1 */ - if (mapping == None): +def PyUnicode_DecodeCharmap(s, size, mapping, errors): + ## /* Default to Latin-1 */ + if mapping == None: return PyUnicode_DecodeLatin1(s, size, errors) - if (size == 0): - return '' + if size == 0: + return "" p = [] inpos = 0 - while (inpos< len(s)): - - #/* Get mapping (char ordinal -> integer, Unicode char or None) */ + while inpos < len(s): + # /* Get mapping (char ordinal -> integer, Unicode char or None) */ ch = s[inpos] try: x = mapping[ch] if isinstance(x, int): - if x < 65536: + if x == 0xFFFE: + raise KeyError + if 0 <= x <= 0x10FFFF: p += chr(x) else: - raise TypeError("character mapping must be in range(65536)") + raise TypeError( + "character mapping must be in range(0x%x)" % (0x110000,) + ) elif isinstance(x, str): + if len(x) == 1 and x == "\ufffe": + raise KeyError p += x - elif not x: + elif x is None: raise KeyError else: raise TypeError - except KeyError: - x = unicode_call_errorhandler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) + except (KeyError, IndexError): + x = unicode_call_errorhandler( + errors, "charmap", "character maps to ", s, inpos, inpos + 1 + ) p += x[0] inpos += 1 return p -def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): - if (size == 0): - return '' +def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 if isinstance(s, str): s = s.encode() pos = 0 p = [] - while (pos < len(s)): - ch = chr(s[pos]) - #/* Non-escape characters are interpreted as Unicode ordinals */ - if (ch != '\\'): - p.append(ch) + while pos < len(s): + # /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): + p.append(chr(s[pos])) pos += 1 - continue + continue startinpos = pos -## /* \u-escapes are only interpreted iff the number of leading -## backslashes is odd */ + p_len_before = len(p) + ## /* \u-escapes are only interpreted iff the number of leading + ## backslashes is odd */ bs = pos while pos < size: - if (s[pos] != ord('\\')): + if s[pos] != ord("\\"): break p.append(chr(s[pos])) pos += 1 - - if (pos >= size): + + if pos >= size: + if not final: + del p[p_len_before:] + pos = startinpos break - if (((pos - bs) & 1) == 0 or - (s[pos] != ord('u') and s[pos] != ord('U'))) : + if ((pos - bs) & 1) == 0 or (s[pos] != ord("u") and s[pos] != ord("U")): p.append(chr(s[pos])) pos += 1 continue - + p.pop(-1) - if s[pos] == ord('u'): - count = 4 - else: - count = 8 + count = 4 if s[pos] == ord("u") else 8 pos += 1 - #/* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ + # /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ number_end = hex_number_end(s, pos, count) if number_end - pos != count: + if not final: + del p[p_len_before:] + pos = startinpos + break res = unicode_call_errorhandler( - errors, "rawunicodeescape", "truncated \\uXXXX", - s, pos-2, number_end) + errors, "rawunicodeescape", "truncated \\uXXXX", s, pos - 2, number_end + ) p.append(res[0]) pos = res[1] else: - x = int(s[pos:pos+count], 16) - #ifndef Py_UNICODE_WIDE - if sys.maxunicode > 0xffff: - if (x > sys.maxunicode): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - else: - p.append(chr(x)) - pos += count + x = int(s[pos : pos + count], 16) + if x > sys.maxunicode: + res = unicode_call_errorhandler( + errors, + "rawunicodeescape", + "\\Uxxxxxxxx out of range", + s, + pos - 2, + pos + count, + ) + pos = res[1] + p.append(res[0]) else: - if (x > 0x10000): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - - #endif - else: - p.append(chr(x)) - pos += count + p.append(chr(x)) + pos += count - return p + return p, pos diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index cd0ea900bfb..38e1f764f00 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -402,6 +402,8 @@ def _parse_hh_mm_ss_ff(tstr): raise ValueError("Invalid microsecond component") else: pos += 1 + if not all(map(_is_ascii_digit, tstr[pos:])): + raise ValueError("Non-digit values in fraction") len_remainder = len_str - pos @@ -413,9 +415,6 @@ def _parse_hh_mm_ss_ff(tstr): time_comps[3] = int(tstr[pos:(pos+to_parse)]) if to_parse < 6: time_comps[3] *= _FRACTION_CORRECTION[to_parse-1] - if (len_remainder > to_parse - and not all(map(_is_ascii_digit, tstr[(pos+to_parse):]))): - raise ValueError("Non-digit values in unparsed fraction") return time_comps @@ -556,10 +555,6 @@ def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -970,6 +965,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @@ -1059,8 +1056,8 @@ def isoformat(self): This is 'YYYY-MM-DD'. References: - - http://www.w3.org/TR/NOTE-datetime - - http://www.cl.cam.ac.uk/~mgk25/iso-time.html + - https://www.w3.org/TR/NOTE-datetime + - https://www.cl.cam.ac.uk/~mgk25/iso-time.html """ return "%04d-%02d-%02d" % (self._year, self._month, self._day) @@ -1108,35 +1105,38 @@ def replace(self, year=None, month=None, day=None): day = self._day return type(self)(year, month, day) + __replace__ = replace + # Comparisons of date objects with other. def __eq__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) + assert not isinstance(other, datetime) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) @@ -1191,7 +1191,7 @@ def isocalendar(self): The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from - http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm (used with permission) """ year = self._year @@ -1633,6 +1633,8 @@ def replace(self, hour=None, minute=None, second=None, microsecond=None, fold = self._fold return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + # Pickle support. def _getstate(self, protocol=3): @@ -1680,7 +1682,7 @@ class datetime(date): The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. """ - __slots__ = date.__slots__ + time.__slots__ + __slots__ = time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): @@ -1979,6 +1981,8 @@ def replace(self, year=None, month=None, day=None, hour=None, return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() @@ -2040,7 +2044,7 @@ def isoformat(self, sep='T', timespec='auto'): By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. @@ -2131,42 +2135,32 @@ def dst(self): def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) @@ -2311,7 +2305,6 @@ def __reduce__(self): def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 - # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above @@ -2341,6 +2334,9 @@ def __new__(cls, offset, name=_Omitted): "timedelta(hours=24).") return cls._create(offset, name) + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ff80180a79e..97a629fe92c 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -38,10 +38,10 @@ 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', # Functions for manipulating contexts - 'setcontext', 'getcontext', 'localcontext', + 'setcontext', 'getcontext', 'localcontext', 'IEEEContext', # Limits for the C version for compatibility - 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', 'IEEE_CONTEXT_MAX_BITS', # C version: compile time choice that enables the thread local context (deprecated, now always true) 'HAVE_THREADS', @@ -83,10 +83,12 @@ MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 MIN_EMIN = -999999999999999999 + IEEE_CONTEXT_MAX_BITS = 512 else: MAX_PREC = 425000000 MAX_EMAX = 425000000 MIN_EMIN = -425000000 + IEEE_CONTEXT_MAX_BITS = 256 MIN_ETINY = MIN_EMIN - (MAX_PREC-1) @@ -417,6 +419,27 @@ def sin(x): return ctx_manager +def IEEEContext(bits, /): + """ + Return a context object initialized to the proper values for one of the + IEEE interchange formats. The argument must be a multiple of 32 and less + than IEEE_CONTEXT_MAX_BITS. + """ + if bits <= 0 or bits > IEEE_CONTEXT_MAX_BITS or bits % 32: + raise ValueError("argument must be a multiple of 32, " + f"with a maximum of {IEEE_CONTEXT_MAX_BITS}") + + ctx = Context() + ctx.prec = 9 * (bits//32) - 2 + ctx.Emax = 3 * (1 << (bits//16 + 3)) + ctx.Emin = 1 - ctx.Emax + ctx.rounding = ROUND_HALF_EVEN + ctx.clamp = 1 + ctx.traps = dict.fromkeys(_signals, False) + + return ctx + + ##### Decimal class ####################################################### # Do not subclass Decimal from numbers.Real and do not register it as such @@ -582,6 +605,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. @@ -2425,12 +2463,12 @@ def __pow__(self, other, modulo=None, context=None): return ans - def __rpow__(self, other, context=None): + def __rpow__(self, other, modulo=None, context=None): """Swaps self/other and returns __pow__.""" other = _convert_other(other) if other is NotImplemented: return other - return other.__pow__(self, context=context) + return other.__pow__(self, modulo, context=context) def normalize(self, context=None): """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" @@ -3302,7 +3340,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3319,14 +3360,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3343,7 +3390,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -6058,7 +6108,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P\d*) # with (possibly empty) diagnostic info. ) # \s* - \Z + \z """, re.VERBOSE | re.IGNORECASE).match _all_zeros = re.compile('0*$').match @@ -6082,11 +6132,15 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? -(?P,)? -(?:\.(?P0|(?!0)\d+))? +(?P\d+)? +(?P[,_])? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? -\Z +\z """, re.VERBOSE|re.DOTALL) del re @@ -6177,6 +6231,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6296,6 +6353,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 2629ed9e009..116ce4f37ec 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,15 +16,16 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : valid_seek_flags.add(os.SEEK_HOLE) valid_seek_flags.add(os.SEEK_DATA) -# open() uses st_blksize whenever we can -DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes +# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) +# when the device block size is available. +DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs # defined in io.py. We don't use real inheritance though, because we don't want @@ -33,11 +34,8 @@ # Rebind for compatibility BlockingIOError = BlockingIOError -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) # Does open() check its 'errors' argument? -_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE +_CHECK_ERRORS = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) def text_encoding(encoding, stacklevel=2): @@ -126,10 +124,10 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, the size of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is chosen using a heuristic trying to determine the underlying device's - "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. - On many systems, the buffer will typically be 4096 or 8192 bytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) + when the device block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -241,18 +239,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE) if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -416,18 +407,12 @@ def __del__(self): if closed: return - if _IOBASE_EMITS_UNRAISABLE: - self.close() - else: - # The try/except block is in case this is called at program - # exit time, when it's possible that globals have already been - # deleted, and then the close() call might fail. Since - # there's nothing we can do about such failures and they annoy - # the end users, we suppress the traceback. - try: - self.close() - except: - pass + if dealloc_warn := getattr(self, "_dealloc_warn", None): + dealloc_warn(self) + + # If close() fails, the caller logs the exception with + # sys.unraisablehook. close() must be called at the end at __del__(). + self.close() ### Inquiries ### @@ -632,16 +617,15 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] return bytes(b) def readall(self): """Read until EOF, using multiple read() call.""" res = bytearray() - while True: - data = self.read(DEFAULT_BUFFER_SIZE) - if not data: - break + while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: return bytes(res) @@ -666,8 +650,6 @@ def write(self, b): self._unsupported("write") io.RawIOBase.register(RawIOBase) -from _io import FileIO -RawIOBase.register(FileIO) class BufferedIOBase(IOBase): @@ -874,6 +856,10 @@ def __repr__(self): else: return "<{}.{} name={!r}>".format(modname, clsname, name) + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.raw, "_dealloc_warn", None): + dealloc_warn(source) + ### Lower-level APIs ### def fileno(self): @@ -949,22 +935,22 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 - pos = self._pos - if pos > len(self._buffer): - # Inserts null bytes between the current end of the file - # and the new write position. - padding = b'\x00' * (pos - len(self._buffer)) - self._buffer += padding - self._buffer[pos:pos + n] = b - self._pos += n + if n == 0: + return 0 + + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n return n def seek(self, pos, whence=0): @@ -1478,6 +1464,17 @@ def write(self, b): return BufferedWriter.write(self, b) +def _new_buffersize(bytes_read): + # Parallels _io/fileio.c new_buffersize + if bytes_read > 65536: + addend = bytes_read >> 3 + else: + addend = 256 + bytes_read + if addend < DEFAULT_BUFFER_SIZE: + addend = DEFAULT_BUFFER_SIZE + return bytes_read + addend + + class FileIO(RawIOBase): _fd = -1 _created = False @@ -1502,6 +1499,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1511,6 +1509,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') @@ -1569,24 +1572,22 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: - raise OSError('Negative file descriptor') + # bpo-27066: Raise a ValueError for bad value. + raise ValueError(f'opener returned {fd}') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1603,17 +1604,17 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise self._fd = fd - def __del__(self): + def _dealloc_warn(self, source): if self._fd >= 0 and self._closefd and not self.closed: import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, + warnings.warn(f'unclosed file {source!r}', ResourceWarning, stacklevel=2, source=self) - self.close() def __getstate__(self): raise TypeError(f"cannot pickle {self.__class__.__name__!r} object") @@ -1632,6 +1633,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1643,7 +1655,13 @@ def _checkWritable(self, msg=None): def read(self, size=None): """Read at most size bytes, returned as bytes. - Only makes one system call, so less data may be returned than requested + If size is less than 0, read all bytes in the file making + multiple read calls. See ``FileIO.readall``. + + Attempts to make only one system call, retrying only per + PEP 475 (EINTR). This means less data may be returned than + requested. + In non-blocking mode, returns None if no data is available. Return an empty bytes object at EOF. """ @@ -1659,45 +1677,57 @@ def read(self, size=None): def readall(self): """Read all data from the file, returned as bytes. - In non-blocking mode, returns as much as is immediately available, - or None if no data is available. Return an empty bytes object at EOF. + Reads until either there is an error or read() returns size 0 + (indicates EOF). If the file is already at EOF, returns an + empty bytes object. + + In non-blocking mode, returns as much data as could be read + before EAGAIN. If no data is available (EAGAIN is returned + before bytes are read) returns None. """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 - result = bytearray() - while True: - if len(result) >= bufsize: - bufsize = len(result) - bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) - n = bufsize - len(result) - try: - chunk = os.read(self._fd, n) - except BlockingIOError: - if result: - break + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass + + result = bytearray(bufsize) + bytes_read = 0 + try: + while n := os.readinto(self._fd, memoryview(result)[bytes_read:]): + bytes_read += n + if bytes_read >= len(result): + result.resize(_new_buffersize(bytes_read)) + except BlockingIOError: + if not bytes_read: return None - if not chunk: # reached the end of the file - break - result += chunk + assert len(result) - bytes_read >= 1, \ + "os.readinto buffer size 0 will result in erroneous EOF / returns 0" + result.resize(bytes_read) return bytes(result) - def readinto(self, b): + def readinto(self, buffer): """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n + self._checkClosed() + self._checkReadable() + try: + return os.readinto(self._fd, buffer) + except BlockingIOError: + return None def write(self, b): """Write bytes b to file, return number written. @@ -1747,6 +1777,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): @@ -1756,8 +1787,9 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: - if self._closefd: + if self._closefd and self._fd >= 0: os.close(self._fd) finally: super().close() @@ -1794,6 +1826,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" @@ -2018,8 +2065,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None, raise ValueError("invalid encoding: %r" % encoding) if not codecs.lookup(encoding)._is_text_encoding: - msg = ("%r is not a text encoding; " - "use codecs.open() to handle arbitrary codecs") + msg = "%r is not a text encoding" raise LookupError(msg % encoding) if errors is None: @@ -2527,9 +2573,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None @@ -2649,6 +2698,10 @@ def readline(self, size=None): def newlines(self): return self._decoder.newlines if self._decoder else None + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None): + dealloc_warn(source) + class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 4970eb3fa67..be1acd17ce3 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -45,10 +45,16 @@ # # and `mycache[lo]` replaces `base**lo` in the inner function. # -# While this does give minor speedups (a few percent at best), the primary -# intent is to simplify the functions using this, by eliminating the need for -# them to craft their own ad-hoc caching schemes. -def compute_powers(w, base, more_than, show=False): +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): seen = set() need = set() ws = {w} @@ -58,40 +64,70 @@ def compute_powers(w, base, more_than, show=False): continue seen.add(w) lo = w >> 1 - # only _need_ lo here; some other path may, or may not, need hi - need.add(lo) - ws.add(lo) - if w & 1: - ws.add(lo + 1) + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra d = {} - if not need: - return d - it = iter(sorted(need)) - first = next(it) - if show: - print("pow at", first) - d[first] = base ** first - for this in it: - if this - 1 in d: + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: if show: - print("* base at", this) - d[this] = d[this - 1] * base # cheap - else: - lo = this >> 1 - hi = this - lo - assert lo in d + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. if show: - print("square at", this) - # Multiplying a bigint by itself (same object!) is about twice - # as fast in CPython. - sq = d[lo] * d[lo] + print("square", end="") + result = d[lo] * d[lo] # same object if hi != lo: - assert hi == lo + 1 if show: - print(" and * base") - sq *= base - d[this] = sq + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] return d _unbounded_dec_context = decimal.getcontext().copy() @@ -211,6 +247,145 @@ def inner(a, b): return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -219,7 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -352,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -361,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 489eec714e0..d1c7fcaeec9 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -83,21 +43,15 @@ def __reduce__(self): return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -108,18 +62,12 @@ def pop(self): return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -136,8 +84,6 @@ def difference(self, other): def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -151,8 +97,6 @@ def intersection(self, other): def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -184,8 +128,6 @@ def symmetric_difference(self, other): def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000..a5788cdbfae --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,1143 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_from_class_namespace", + "get_annotations", + "annotations_to_string", + "type_repr", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + # These are always set to None here but may be non-None if a ForwardRef + # is created through __class__ assignment on a _Stringifier object. + self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). + self.__cell__ = None + self.__extra_names__ = None + # These are initially None but serve as a cache and may be set to a non-None + # value later. + self.__code__ = None + self.__ast_node__ = None + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + match format: + case Format.STRING: + return self.__forward_arg__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) + if isinstance(self.__cell__, types.CellType): + try: + return self.__cell__.cell_contents + except ValueError: + pass + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: + for param in type_params: + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + + if self.__extra_names__: + locals.update(self.__extra_names__) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + return locals[arg] + elif arg in globals: + return globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + elif is_forwardref_format: + return self + else: + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) + else: + code = self.__forward_code__ + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. + new_locals = _StringifierDict( + {**builtins.__dict__, **globals, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, + format=format, + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify(self.__cell__) + return result + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + try: + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + # Use "is" here because we use id() for this in __hash__ + # because dictionaries are not hashable. + and self.__globals__ is other.__globals__ + and self.__forward_is_class__ == other.__forward_is_class__ + and self.__cell__ == other.__cell__ + and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) + ) + + def __hash__(self): + return hash(( + self.__forward_arg__, + self.__forward_module__, + id(self.__globals__), # dictionaries are not hashable, so hash by identity + self.__forward_is_class__, + tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, + )) + + def __or__(self, other): + return types.UnionType[self, other] + + def __ror__(self, other): + return types.UnionType[other, self] + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_Template = type(t"") + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + extra_names=None, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__extra_names__ = extra_names + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): + return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + + return ast.Slice( + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names + else: + return self.__convert_to_ast(other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names or None, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + other = ast.Tuple(elts) + else: + other, extra_names = self.__convert_to_ast_getitem(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(self.__get_ast(), op, rhs), extra_names + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(new_other, op, self.__get_ast()), extra_names + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[rhs], + ), + extra_names, + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + +class _StringifierDict(dict): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + self.next_id = 1 + self.format = format + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + def transmogrify(self, cell_dict): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict + + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + + globals = _StringifierDict({}, format=format) + is_class = isinstance(owner, type) + closure, _ = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + if _is_evaluate: + return _stringify_single(annos) + return { + key: _stringify_single(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict( + namespace, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) + except Exception: + pass + else: + globals.transmogrify(cell_dict) + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + globals.transmogrify(cell_dict) + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + if not annotate.__closure__: + return None, None + new_closure = [] + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure), cell_dict + + +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) + else: + return repr(anno) + + +def get_annotate_from_class_namespace(obj): + """Retrieve the annotate function from a class namespace dictionary. + + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. + """ + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, module, or other object with + __annotate__ or __annotations__ attributes. + Passing any other object raises TypeError. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the Format enum or its integer equivalent. + For the VALUE format, the __annotations__ is tried first; if it + does not exist, the __annotate__ function is called. The + FORWARDREF format uses __annotations__ if it exists and can be + evaluated, and otherwise falls back to calling the __annotate__ function. + The SOURCE format tries __annotate__ first, and falls back to + using __annotations__, stringified using annotations_to_string(). + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we first look at __annotations__ + ann = _get_dunder_annotations(obj) + + # If it's not there, try __annotate__ instead + if ann is None: + ann = _get_and_call_annotate(obj, format) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except Exception: + pass + else: + if ann is not None: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is None: + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + ann = _get_dunder_annotations(obj) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return dict(ann) + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + if ann is not None: + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if ann is None: + if isinstance(obj, type) or callable(obj): + return {} + raise TypeError(f"{obj!r} does not have annotations") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) + for key, value in ann.items() + } + return return_value + + +def type_repr(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is intended as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) + if value is ...: + return "..." + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format. + + Always returns a fresh a dictionary. + """ + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } + + +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + +def _get_and_call_annotate(obj, format): + """Get the __annotate__ function and call it. + + May not return a fresh dictionary. + """ + annotate = getattr(obj, "__annotate__", None) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return ann + return None + + +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def _get_dunder_annotations(obj): + """Return the annotations for an object, checking that it is a dictionary. + + Does not return a fresh dictionary. + """ + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return ann diff --git a/Lib/argparse.py b/Lib/argparse.py index bd088ea0e66..1d7d34f9924 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -159,18 +161,21 @@ class HelpFormatter(object): provided by the class are considered an implementation detail. """ - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - + def __init__( + self, + prog, + indent_increment=2, + max_help_position=24, + width=None, + color=True, + ): # default setting for width if width is None: import shutil width = shutil.get_terminal_size().columns width -= 2 + self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -187,6 +192,16 @@ def __init__(self, self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + def _set_color(self, color): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = lambda text: text + # =============================== # Section and indentation methods # =============================== @@ -225,7 +240,11 @@ def format_help(self): if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent heading_text = _('%(heading)s:') % dict(heading=self.heading) - heading = '%*s%s\n' % (current_indent, '', heading_text) + t = self.formatter._theme + heading = ( + f'{" " * current_indent}' + f'{t.heading}{heading_text}{t.reset}\n' + ) else: heading = '' @@ -262,7 +281,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -296,42 +315,39 @@ def _join_parts(self, part_strings): if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): + t = self._theme + if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: - usage = usage % dict(prog=self._prog) + usage = ( + t.prog_extra + + usage + % {"prog": f"{t.prog}{self._prog}{t.reset}{t.prog_extra}"} + + t.reset + ) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) + usage = f"{t.prog}{self._prog}{t.reset}" # if optionals and positionals are available, calculate usage elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: + if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -343,12 +359,13 @@ def get_lines(parts, indent, prefix=None): else: line_len = indent_length - 1 for part in parts: - if line_len + 1 + len(part) > text_width and line: + part_len = len(self._decolor(part)) + if line_len + 1 + part_len > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = indent_length - 1 line.append(part) - line_len += len(part) + 1 + line_len += part_len + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: @@ -356,8 +373,9 @@ def get_lines(parts, indent, prefix=None): return lines # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) + prog_len = len(self._decolor(prog)) + if len(prefix) + prog_len <= 0.75 * text_width: + indent = ' ' * (len(prefix) + prog_len + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) lines.extend(get_lines(pos_parts, indent)) @@ -380,97 +398,120 @@ def get_lines(parts, indent, prefix=None): # join lines into usage usage = '\n'.join(lines) + usage = usage.removeprefix(prog) + usage = f"{t.prog}{prog}{t.reset}{usage}" + # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) + return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) + def _is_long_option(self, string): + return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') - - if all(action.help is SUPPRESS for action in group._group_actions): - continue + """Get usage parts with split index for optionals/positionals. - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] + t = self._theme + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -486,6 +527,7 @@ def _format_action(self, action): help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) + action_header_no_color = self._decolor(action_header) # no help; start on same line and add a final newline if not action.help: @@ -493,9 +535,15 @@ def _format_action(self, action): action_header = '%*s%s\n' % tup # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header + elif len(action_header_no_color) <= action_width: + # calculate widths without color codes + action_header_color = action_header + tup = self._current_indent, '', action_width, action_header_no_color action_header = '%*s%-*s ' % tup + # swap in the colored header + action_header = action_header.replace( + action_header_no_color, action_header_color + ) indent_first = 0 # long action name; start on the next line @@ -528,23 +576,42 @@ def _format_action(self, action): return self._join_parts(parts) def _format_action_invocation(self, action): + t = self._theme + if not action.option_strings: default = self._get_default_metavar_for_positional(action) - return ' '.join(self._metavar_formatter(action, default)(1)) + return ( + t.action + + ' '.join(self._metavar_formatter(action, default)(1)) + + t.reset + ) else: + def color_option_strings(strings): + parts = [] + for s in strings: + if self._is_long_option(s): + parts.append(f"{t.long_option}{s}{t.reset}") + else: + parts.append(f"{t.short_option}{s}{t.reset}") + return parts + # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - return ', '.join(action.option_strings) + option_strings = color_option_strings(action.option_strings) + return ', '.join(option_strings) # if the Optional takes a value, format is: # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - return ', '.join(action.option_strings) + ' ' + args_string + option_strings = color_option_strings(action.option_strings) + args_string = ( + f"{t.label}{self._format_args(action, default)}{t.reset}" + ) + return ', '.join(option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -590,16 +657,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return self._get_help_string(action) % params + return help_string % params def _iter_indented_subactions(self, action): try: @@ -669,11 +739,14 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') return help @@ -844,22 +917,16 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -867,38 +934,19 @@ def __init__(self, _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) @@ -1180,6 +1228,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1195,23 +1244,30 @@ def add_parser(self, name, *, deprecated=False, **kwargs): if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + # set color + if kwargs.get('color') is None: + kwargs['color'] = self._color + aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1276,7 +1332,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1293,6 +1349,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1396,7 +1458,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1449,7 +1511,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1465,27 +1528,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1529,8 +1599,8 @@ def _add_container_actions(self, container): if group.title in title_group_map: # This branch could happen if a derived class added # groups with duplicated titles in __init__ - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1571,13 +1641,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required nargs = kwargs.get('nargs') - if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]: + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1590,11 +1662,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1610,8 +1680,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1627,8 +1697,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1667,10 +1737,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1702,14 +1788,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1720,7 +1799,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1730,14 +1809,29 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') + +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' class ArgumentParser(_AttributeHolder, _ActionsContainer): @@ -1760,6 +1854,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + - color - Allow color output in help messages (default: ``False``) """ def __init__(self, @@ -1775,19 +1872,18 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): - + exit_on_error=True, + *, + suggest_on_error=False, + color=True, + ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1795,6 +1891,11 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error + self.color = color + + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1844,7 +1945,7 @@ def _get_kwargs(self): def add_subparsers(self, **kwargs): if self._subparsers is not None: - raise ArgumentError(None, _('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1859,15 +1960,19 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') + formatter.add_usage(None, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + action._color = self.color + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -2498,7 +2603,6 @@ def _get_values(self, action, arg_strings): value = action.default if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2506,11 +2610,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2543,8 +2644,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2568,14 +2668,27 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) choices = action.choices - if choices is not None: - if isinstance(choices, str): - choices = iter(choices) - if value not in choices: - args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods @@ -2611,7 +2724,16 @@ def format_help(self): return formatter.format_help() def _get_formatter(self): - return self.formatter_class(prog=self.prog) + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color) + return formatter + + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter # ===================== # Help-printing methods diff --git a/Lib/ast.py b/Lib/ast.py index 37b20206b8a..2f11683ecf7 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -20,11 +20,7 @@ :copyright: Copyright 2008 by Armin Ronacher. :license: Python License. """ -import sys -import re from _ast import * -from contextlib import contextmanager, nullcontext -from enum import IntEnum, auto, _simple_enum def parse(source, filename='', mode='exec', *, @@ -319,12 +315,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -395,6 +397,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -431,27 +515,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -511,151 +574,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -696,1147 +614,21 @@ class Param(expr_context): """Deprecated AST node class. Unused in Python 3.""" -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -@_simple_enum(IntEnum) -class _Precedence: - """Precedence table that originated from python grammar.""" - - NAMED_EXPR = auto() # := - TUPLE = auto() # , - YIELD = auto() # 'yield', 'yield from' - TEST = auto() # 'if'-'else', 'lambda' - OR = auto() # 'or' - AND = auto() # 'and' - NOT = auto() # 'not' - CMP = auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = auto() - BOR = EXPR # '|' - BXOR = auto() # '^' - BAND = auto() # '&' - SHIFT = auto() # '<<', '>>' - ARITH = auto() # '+', '-' - TERM = auto() # '*', '@', '/', '%', '//' - FACTOR = auto() # unary '+', '-', '~' - POWER = auto() # '**' - AWAIT = auto() # 'await' - ATOM = auto() - - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - - -_SINGLE_QUOTES = ("'", '"') -_MULTI_QUOTES = ('"""', "'''") -_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) - -class _Unparser(NodeVisitor): - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded.""" - - def __init__(self): - self._source = [] - self._precedences = {} - self._type_ignores = {} - self._indent = 0 - self._in_try_star = False - - def interleave(self, inter, f, seq): - """Call f on each item in seq, calling inter() in between.""" - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - - def items_view(self, traverser, items): - """Traverse and separate the given *items* with a comma and append it to - the buffer. If *items* is a single item sequence, a trailing comma - will be added.""" - if len(items) == 1: - traverser(items[0]) - self.write(",") - else: - self.interleave(lambda: self.write(", "), traverser, items) - - def maybe_newline(self): - """Adds a newline if it isn't the start of generated source""" - if self._source: - self.write("\n") - - def fill(self, text=""): - """Indent a piece of text and append it, according to the current - indentation level""" - self.maybe_newline() - self.write(" " * self._indent + text) - - def write(self, *text): - """Add new source parts""" - self._source.extend(text) - - @contextmanager - def buffered(self, buffer = None): - if buffer is None: - buffer = [] - - original_source = self._source - self._source = buffer - yield buffer - self._source = original_source - - @contextmanager - def block(self, *, extra = None): - """A context manager for preparing the source for blocks. It adds - the character':', increases the indentation on enter and decreases - the indentation on exit. If *extra* is given, it will be directly - appended after the colon character. - """ - self.write(":") - if extra: - self.write(extra) - self._indent += 1 - yield - self._indent -= 1 - - @contextmanager - def delimit(self, start, end): - """A context manager for preparing the source for expressions. It adds - *start* to the buffer and enters, after exit it adds *end*.""" - - self.write(start) - yield - self.write(end) - - def delimit_if(self, start, end, condition): - if condition: - return self.delimit(start, end) - else: - return nullcontext() - - def require_parens(self, precedence, node): - """Shortcut to adding precedence related parens""" - return self.delimit_if("(", ")", self.get_precedence(node) > precedence) - - def get_precedence(self, node): - return self._precedences.get(node, _Precedence.TEST) - - def set_precedence(self, precedence, *nodes): - for node in nodes: - self._precedences[node] = precedence - - def get_raw_docstring(self, node): - """If a docstring node is found in the body of the *node* parameter, - return that docstring node, None otherwise. - - Logic mirrored from ``_PyAST_GetDocString``.""" - if not isinstance( - node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) - ) or len(node.body) < 1: - return None - node = node.body[0] - if not isinstance(node, Expr): - return None - node = node.value - if isinstance(node, Constant) and isinstance(node.value, str): - return node - - def get_type_comment(self, node): - comment = self._type_ignores.get(node.lineno) or node.type_comment - if comment is not None: - return f" # type: {comment}" - - def traverse(self, node): - if isinstance(node, list): - for item in node: - self.traverse(item) - else: - super().visit(node) - - # Note: as visit() resets the output text, do NOT rely on - # NodeVisitor.generic_visit to handle any nodes (as it calls back in to - # the subclass visit() method, which resets self._source to an empty list) - def visit(self, node): - """Outputs a source code string that, if converted back to an ast - (using ast.parse) will generate an AST equivalent to *node*""" - self._source = [] - self.traverse(node) - return "".join(self._source) - - def _write_docstring_and_traverse_body(self, node): - if (docstring := self.get_raw_docstring(node)): - self._write_docstring(docstring) - self.traverse(node.body[1:]) - else: - self.traverse(node.body) - - def visit_Module(self, node): - self._type_ignores = { - ignore.lineno: f"ignore{ignore.tag}" - for ignore in node.type_ignores - } - self._write_docstring_and_traverse_body(node) - self._type_ignores.clear() - - def visit_FunctionType(self, node): - with self.delimit("(", ")"): - self.interleave( - lambda: self.write(", "), self.traverse, node.argtypes - ) - - self.write(" -> ") - self.traverse(node.returns) - - def visit_Expr(self, node): - self.fill() - self.set_precedence(_Precedence.YIELD, node.value) - self.traverse(node.value) - - def visit_NamedExpr(self, node): - with self.require_parens(_Precedence.NAMED_EXPR, node): - self.set_precedence(_Precedence.ATOM, node.target, node.value) - self.traverse(node.target) - self.write(" := ") - self.traverse(node.value) - - def visit_Import(self, node): - self.fill("import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_ImportFrom(self, node): - self.fill("from ") - self.write("." * (node.level or 0)) - if node.module: - self.write(node.module) - self.write(" import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_Assign(self, node): - self.fill() - for target in node.targets: - self.set_precedence(_Precedence.TUPLE, target) - self.traverse(target) - self.write(" = ") - self.traverse(node.value) - if type_comment := self.get_type_comment(node): - self.write(type_comment) - - def visit_AugAssign(self, node): - self.fill() - self.traverse(node.target) - self.write(" " + self.binop[node.op.__class__.__name__] + "= ") - self.traverse(node.value) - - def visit_AnnAssign(self, node): - self.fill() - with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): - self.traverse(node.target) - self.write(": ") - self.traverse(node.annotation) - if node.value: - self.write(" = ") - self.traverse(node.value) - - def visit_Return(self, node): - self.fill("return") - if node.value: - self.write(" ") - self.traverse(node.value) - - def visit_Pass(self, node): - self.fill("pass") - - def visit_Break(self, node): - self.fill("break") - - def visit_Continue(self, node): - self.fill("continue") - - def visit_Delete(self, node): - self.fill("del ") - self.interleave(lambda: self.write(", "), self.traverse, node.targets) - - def visit_Assert(self, node): - self.fill("assert ") - self.traverse(node.test) - if node.msg: - self.write(", ") - self.traverse(node.msg) - - def visit_Global(self, node): - self.fill("global ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Nonlocal(self, node): - self.fill("nonlocal ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Await(self, node): - with self.require_parens(_Precedence.AWAIT, node): - self.write("await") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Yield(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_YieldFrom(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield from ") - if not node.value: - raise ValueError("Node can't be used without a value attribute.") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Raise(self, node): - self.fill("raise") - if not node.exc: - if node.cause: - raise ValueError(f"Node can't use cause without an exception.") - return - self.write(" ") - self.traverse(node.exc) - if node.cause: - self.write(" from ") - self.traverse(node.cause) - - def do_visit_try(self, node): - self.fill("try") - with self.block(): - self.traverse(node.body) - for ex in node.handlers: - self.traverse(ex) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - if node.finalbody: - self.fill("finally") - with self.block(): - self.traverse(node.finalbody) - - def visit_Try(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = False - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_TryStar(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = True - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_ExceptHandler(self, node): - self.fill("except*" if self._in_try_star else "except") - if node.type: - self.write(" ") - self.traverse(node.type) - if node.name: - self.write(" as ") - self.write(node.name) - with self.block(): - self.traverse(node.body) - - def visit_ClassDef(self, node): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - self.fill("class " + node.name) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit_if("(", ")", condition = node.bases or node.keywords): - comma = False - for e in node.bases: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - with self.block(): - self._write_docstring_and_traverse_body(node) - - def visit_FunctionDef(self, node): - self._function_helper(node, "def") - - def visit_AsyncFunctionDef(self, node): - self._function_helper(node, "async def") - - def _function_helper(self, node, fill_suffix): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - def_str = fill_suffix + " " + node.name - self.fill(def_str) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit("(", ")"): - self.traverse(node.args) - if node.returns: - self.write(" -> ") - self.traverse(node.returns) - with self.block(extra=self.get_type_comment(node)): - self._write_docstring_and_traverse_body(node) - - def _type_params_helper(self, type_params): - if type_params is not None and len(type_params) > 0: - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, type_params) - - def visit_TypeVar(self, node): - self.write(node.name) - if node.bound: - self.write(": ") - self.traverse(node.bound) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_TypeVarTuple(self, node): - self.write("*" + node.name) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_ParamSpec(self, node): - self.write("**" + node.name) - if node.default_value: - self.write(" = ") - self.traverse(node.default_value) - - def visit_TypeAlias(self, node): - self.fill("type ") - self.traverse(node.name) - self._type_params_helper(node.type_params) - self.write(" = ") - self.traverse(node.value) - - def visit_For(self, node): - self._for_helper("for ", node) - - def visit_AsyncFor(self, node): - self._for_helper("async for ", node) - - def _for_helper(self, fill, node): - self.fill(fill) - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.traverse(node.iter) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_If(self, node): - self.fill("if ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # collapse nested ifs into equivalent elifs. - while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): - node = node.orelse[0] - self.fill("elif ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # final else - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_While(self, node): - self.fill("while ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_With(self, node): - self.fill("with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def visit_AsyncWith(self, node): - self.fill("async with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def _str_literal_helper( - self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False - ): - """Helper for writing string literals, minimizing escapes. - Returns the tuple (string literal to write, possible quote types). - """ - def escape_char(c): - # \n and \t are non-printable, but we only escape them if - # escape_special_whitespace is True - if not escape_special_whitespace and c in "\n\t": - return c - # Always escape backslashes and other non-printable characters - if c == "\\" or not c.isprintable(): - return c.encode("unicode_escape").decode("ascii") - return c - - escaped_string = "".join(map(escape_char, string)) - possible_quotes = quote_types - if "\n" in escaped_string: - possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] - possible_quotes = [q for q in possible_quotes if q not in escaped_string] - if not possible_quotes: - # If there aren't any possible_quotes, fallback to using repr - # on the original string. Try to use a quote from quote_types, - # e.g., so that we use triple quotes for docstrings. - string = repr(string) - quote = next((q for q in quote_types if string[0] in q), string[0]) - return string[1:-1], [quote] - if escaped_string: - # Sort so that we prefer '''"''' over """\"""" - possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) - # If we're using triple quotes and we'd need to escape a final - # quote, escape it - if possible_quotes[0][0] == escaped_string[-1]: - assert len(possible_quotes[0]) == 3 - escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] - return escaped_string, possible_quotes - - def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): - """Write string literal value with a best effort attempt to avoid backslashes.""" - string, quote_types = self._str_literal_helper(string, quote_types=quote_types) - quote_type = quote_types[0] - self.write(f"{quote_type}{string}{quote_type}") - - def visit_JoinedStr(self, node): - self.write("f") - - fstring_parts = [] - for value in node.values: - with self.buffered() as buffer: - self._write_fstring_inner(value) - fstring_parts.append( - ("".join(buffer), isinstance(value, Constant)) - ) - - new_fstring_parts = [] - quote_types = list(_ALL_QUOTES) - fallback_to_repr = False - for value, is_constant in fstring_parts: - if is_constant: - value, new_quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=True, - ) - if set(new_quote_types).isdisjoint(quote_types): - fallback_to_repr = True - break - quote_types = new_quote_types - else: - if "\n" in value: - quote_types = [q for q in quote_types if q in _MULTI_QUOTES] - assert quote_types - - new_quote_types = [q for q in quote_types if q not in value] - if new_quote_types: - quote_types = new_quote_types - new_fstring_parts.append(value) - - if fallback_to_repr: - # If we weren't able to find a quote type that works for all parts - # of the JoinedStr, fallback to using repr and triple single quotes. - quote_types = ["'''"] - new_fstring_parts.clear() - for value, is_constant in fstring_parts: - if is_constant: - value = repr('"' + value) # force repr to use single quotes - expected_prefix = "'\"" - assert value.startswith(expected_prefix), repr(value) - value = value[len(expected_prefix):-1] - new_fstring_parts.append(value) - - value = "".join(new_fstring_parts) - quote_type = quote_types[0] - self.write(f"{quote_type}{value}{quote_type}") - - def _write_fstring_inner(self, node, is_format_spec=False): - if isinstance(node, JoinedStr): - # for both the f-string itself, and format_spec - for value in node.values: - self._write_fstring_inner(value, is_format_spec=is_format_spec) - elif isinstance(node, Constant) and isinstance(node.value, str): - value = node.value.replace("{", "{{").replace("}", "}}") - - if is_format_spec: - value = value.replace("\\", "\\\\") - value = value.replace("'", "\\'") - value = value.replace('"', '\\"') - value = value.replace("\n", "\\n") - self.write(value) - elif isinstance(node, FormattedValue): - self.visit_FormattedValue(node) - else: - raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") - - def visit_FormattedValue(self, node): - def unparse_inner(inner): - unparser = type(self)() - unparser.set_precedence(_Precedence.TEST.next(), inner) - return unparser.visit(inner) - - with self.delimit("{", "}"): - expr = unparse_inner(node.value) - if expr.startswith("{"): - # Separate pair of opening brackets as "{ {" - self.write(" ") - self.write(expr) - if node.conversion != -1: - self.write(f"!{chr(node.conversion)}") - if node.format_spec: - self.write(":") - self._write_fstring_inner(node.format_spec, is_format_spec=True) - - def visit_Name(self, node): - self.write(node.id) - - def _write_docstring(self, node): - self.fill() - if node.kind == "u": - self.write("u") - self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - # Substitute overflowing decimal literal for AST infinities, - # and inf - inf for NaNs. - self.write( - repr(value) - .replace("inf", _INFSTR) - .replace("nan", f"({_INFSTR}-{_INFSTR})") - ) - else: - self.write(repr(value)) - - def visit_Constant(self, node): - value = node.value - if isinstance(value, tuple): - with self.delimit("(", ")"): - self.items_view(self._write_constant, value) - elif value is ...: - self.write("...") - else: - if node.kind == "u": - self.write("u") - self._write_constant(node.value) - - def visit_List(self, node): - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - - def visit_ListComp(self, node): - with self.delimit("[", "]"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_GeneratorExp(self, node): - with self.delimit("(", ")"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_SetComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_DictComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) - for gen in node.generators: - self.traverse(gen) - - def visit_comprehension(self, node): - if node.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) - self.traverse(node.iter) - for if_clause in node.ifs: - self.write(" if ") - self.traverse(if_clause) - - def visit_IfExp(self, node): - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.TEST.next(), node.body, node.test) - self.traverse(node.body) - self.write(" if ") - self.traverse(node.test) - self.write(" else ") - self.set_precedence(_Precedence.TEST, node.orelse) - self.traverse(node.orelse) - - def visit_Set(self, node): - if node.elts: - with self.delimit("{", "}"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - else: - # `{}` would be interpreted as a dictionary literal, and - # `set` might be shadowed. Thus: - self.write('{*()}') - - def visit_Dict(self, node): - def write_key_value_pair(k, v): - self.traverse(k) - self.write(": ") - self.traverse(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.set_precedence(_Precedence.EXPR, v) - self.traverse(v) - else: - write_key_value_pair(k, v) - - with self.delimit("{", "}"): - self.interleave( - lambda: self.write(", "), write_item, zip(node.keys, node.values) - ) - - def visit_Tuple(self, node): - with self.delimit_if( - "(", - ")", - len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE - ): - self.items_view(self.traverse, node.elts) - - unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} - unop_precedence = { - "not": _Precedence.NOT, - "~": _Precedence.FACTOR, - "+": _Precedence.FACTOR, - "-": _Precedence.FACTOR, - } - - def visit_UnaryOp(self, node): - operator = self.unop[node.op.__class__.__name__] - operator_precedence = self.unop_precedence[operator] - with self.require_parens(operator_precedence, node): - self.write(operator) - # factor prefixes (+, -, ~) shouldn't be separated - # from the value they belong, (e.g: +1 instead of + 1) - if operator_precedence is not _Precedence.FACTOR: - self.write(" ") - self.set_precedence(operator_precedence, node.operand) - self.traverse(node.operand) - - binop = { - "Add": "+", - "Sub": "-", - "Mult": "*", - "MatMult": "@", - "Div": "/", - "Mod": "%", - "LShift": "<<", - "RShift": ">>", - "BitOr": "|", - "BitXor": "^", - "BitAnd": "&", - "FloorDiv": "//", - "Pow": "**", - } - - binop_precedence = { - "+": _Precedence.ARITH, - "-": _Precedence.ARITH, - "*": _Precedence.TERM, - "@": _Precedence.TERM, - "/": _Precedence.TERM, - "%": _Precedence.TERM, - "<<": _Precedence.SHIFT, - ">>": _Precedence.SHIFT, - "|": _Precedence.BOR, - "^": _Precedence.BXOR, - "&": _Precedence.BAND, - "//": _Precedence.TERM, - "**": _Precedence.POWER, - } - - binop_rassoc = frozenset(("**",)) - def visit_BinOp(self, node): - operator = self.binop[node.op.__class__.__name__] - operator_precedence = self.binop_precedence[operator] - with self.require_parens(operator_precedence, node): - if operator in self.binop_rassoc: - left_precedence = operator_precedence.next() - right_precedence = operator_precedence - else: - left_precedence = operator_precedence - right_precedence = operator_precedence.next() - - self.set_precedence(left_precedence, node.left) - self.traverse(node.left) - self.write(f" {operator} ") - self.set_precedence(right_precedence, node.right) - self.traverse(node.right) - - cmpops = { - "Eq": "==", - "NotEq": "!=", - "Lt": "<", - "LtE": "<=", - "Gt": ">", - "GtE": ">=", - "Is": "is", - "IsNot": "is not", - "In": "in", - "NotIn": "not in", - } - - def visit_Compare(self, node): - with self.require_parens(_Precedence.CMP, node): - self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) - self.traverse(node.left) - for o, e in zip(node.ops, node.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.traverse(e) - - boolops = {"And": "and", "Or": "or"} - boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} - - def visit_BoolOp(self, node): - operator = self.boolops[node.op.__class__.__name__] - operator_precedence = self.boolop_precedence[operator] - - def increasing_level_traverse(node): - nonlocal operator_precedence - operator_precedence = operator_precedence.next() - self.set_precedence(operator_precedence, node) - self.traverse(node) - - with self.require_parens(operator_precedence, node): - s = f" {operator} " - self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) - - def visit_Attribute(self, node): - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - # Special case: 3.__abs__() is a syntax error, so if node.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(node.value, Constant) and isinstance(node.value.value, int): - self.write(" ") - self.write(".") - self.write(node.attr) - - def visit_Call(self, node): - self.set_precedence(_Precedence.ATOM, node.func) - self.traverse(node.func) - with self.delimit("(", ")"): - comma = False - for e in node.args: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - def visit_Subscript(self, node): - def is_non_empty_tuple(slice_value): - return ( - isinstance(slice_value, Tuple) - and slice_value.elts - ) - - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - with self.delimit("[", "]"): - if is_non_empty_tuple(node.slice): - # parentheses can be omitted if the tuple isn't empty - self.items_view(self.traverse, node.slice.elts) - else: - self.traverse(node.slice) - - def visit_Starred(self, node): - self.write("*") - self.set_precedence(_Precedence.EXPR, node.value) - self.traverse(node.value) - - def visit_Ellipsis(self, node): - self.write("...") - - def visit_Slice(self, node): - if node.lower: - self.traverse(node.lower) - self.write(":") - if node.upper: - self.traverse(node.upper) - if node.step: - self.write(":") - self.traverse(node.step) - - def visit_Match(self, node): - self.fill("match ") - self.traverse(node.subject) - with self.block(): - for case in node.cases: - self.traverse(case) - - def visit_arg(self, node): - self.write(node.arg) - if node.annotation: - self.write(": ") - self.traverse(node.annotation) - - def visit_arguments(self, node): - first = True - # normal arguments - all_args = node.posonlyargs + node.args - defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults - for index, elements in enumerate(zip(all_args, defaults), 1): - a, d = elements - if first: - first = False - else: - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - if index == len(node.posonlyargs): - self.write(", /") - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if node.vararg or node.kwonlyargs: - if first: - first = False - else: - self.write(", ") - self.write("*") - if node.vararg: - self.write(node.vararg.arg) - if node.vararg.annotation: - self.write(": ") - self.traverse(node.vararg.annotation) - - # keyword-only arguments - if node.kwonlyargs: - for a, d in zip(node.kwonlyargs, node.kw_defaults): - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - - # kwargs - if node.kwarg: - if first: - first = False - else: - self.write(", ") - self.write("**" + node.kwarg.arg) - if node.kwarg.annotation: - self.write(": ") - self.traverse(node.kwarg.annotation) - - def visit_keyword(self, node): - if node.arg is None: - self.write("**") - else: - self.write(node.arg) - self.write("=") - self.traverse(node.value) - - def visit_Lambda(self, node): - with self.require_parens(_Precedence.TEST, node): - self.write("lambda") - with self.buffered() as buffer: - self.traverse(node.args) - if buffer: - self.write(" ", *buffer) - self.write(": ") - self.set_precedence(_Precedence.TEST, node.body) - self.traverse(node.body) - - def visit_alias(self, node): - self.write(node.name) - if node.asname: - self.write(" as " + node.asname) - - def visit_withitem(self, node): - self.traverse(node.context_expr) - if node.optional_vars: - self.write(" as ") - self.traverse(node.optional_vars) - - def visit_match_case(self, node): - self.fill("case ") - self.traverse(node.pattern) - if node.guard: - self.write(" if ") - self.traverse(node.guard) - with self.block(): - self.traverse(node.body) - - def visit_MatchValue(self, node): - self.traverse(node.value) - - def visit_MatchSingleton(self, node): - self._write_constant(node.value) - - def visit_MatchSequence(self, node): - with self.delimit("[", "]"): - self.interleave( - lambda: self.write(", "), self.traverse, node.patterns - ) - - def visit_MatchStar(self, node): - name = node.name - if name is None: - name = "_" - self.write(f"*{name}") - - def visit_MatchMapping(self, node): - def write_key_pattern_pair(pair): - k, p = pair - self.traverse(k) - self.write(": ") - self.traverse(p) - - with self.delimit("{", "}"): - keys = node.keys - self.interleave( - lambda: self.write(", "), - write_key_pattern_pair, - zip(keys, node.patterns, strict=True), - ) - rest = node.rest - if rest is not None: - if keys: - self.write(", ") - self.write(f"**{rest}") - - def visit_MatchClass(self, node): - self.set_precedence(_Precedence.ATOM, node.cls) - self.traverse(node.cls) - with self.delimit("(", ")"): - patterns = node.patterns - self.interleave( - lambda: self.write(", "), self.traverse, patterns - ) - attrs = node.kwd_attrs - if attrs: - def write_attr_pattern(pair): - attr, pattern = pair - self.write(f"{attr}=") - self.traverse(pattern) - - if patterns: - self.write(", ") - self.interleave( - lambda: self.write(", "), - write_attr_pattern, - zip(attrs, node.kwd_patterns, strict=True), - ) - - def visit_MatchAs(self, node): - name = node.name - pattern = node.pattern - if name is None: - self.write("_") - elif pattern is None: - self.write(node.name) - else: - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.BOR, node.pattern) - self.traverse(node.pattern) - self.write(f" as {node.name}") - - def visit_MatchOr(self, node): - with self.require_parens(_Precedence.BOR, node): - self.set_precedence(_Precedence.BOR.next(), *node.patterns) - self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) - def unparse(ast_obj): - unparser = _Unparser() + global _Unparser + try: + unparser = _Unparser() + except NameError: + from _ast_unparse import Unparser as _Unparser + unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - -def main(): +def main(args=None): import argparse + import sys - parser = argparse.ArgumentParser(prog='python -m ast') + parser = argparse.ArgumentParser(color=True) parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', @@ -1849,7 +641,16 @@ def main(): 'column offsets') parser.add_argument('-i', '--indent', type=int, default=3, help='indentation of nodes (number of spaces)') - args = parser.parse_args() + parser.add_argument('--feature-version', + type=str, default=None, metavar='VERSION', + help='Python version in the format 3.x ' + '(for example, 3.10)') + parser.add_argument('-O', '--optimize', + type=int, default=-1, metavar='LEVEL', + help='optimization level for parser (default -1)') + parser.add_argument('--show-empty', default=False, action='store_true', + help='show empty lists and fields in dump output') + args = parser.parse_args(args) if args.infile == '-': name = '' @@ -1858,8 +659,22 @@ def main(): name = args.infile with open(args.infile, 'rb') as infile: source = infile.read() - tree = parse(source, name, args.mode, type_comments=args.no_type_comments) - print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) + + # Process feature_version + feature_version = None + if args.feature_version: + try: + major, minor = map(int, args.feature_version.split('.', 1)) + except ValueError: + parser.error('Invalid format for --feature-version; ' + 'expected format 3.x (for example, 3.10)') + + feature_version = (major, minor) + + tree = parse(source, name, args.mode, type_comments=args.no_type_comments, + feature_version=feature_version, optimize=args.optimize) + print(dump(tree, include_attributes=args.include_attributes, + indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': main() diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb..32a5dbae03a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,6 +10,7 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * @@ -27,6 +28,7 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + @@ -45,3 +47,28 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 18bb87a5bc4..e07dd52a2a5 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,41 +1,53 @@ +import argparse import ast import asyncio -import code +import asyncio.tools import concurrent.futures +import contextvars import inspect +import os +import site import sys import threading import types import warnings +from _colorize import get_theme +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop + self.context = contextvars.copy_context() def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -47,39 +59,72 @@ def callback(): return try: - repl_future = self.loop.create_task(coro) + repl_future = self.loop.create_task(coro, context=self.context) futures._chain_future(repl_future, future) except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback) + self.loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: - self.write("\nKeyboardInterrupt\n") + if keyboard_interrupted: + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() - + return self.STATEMENT_FAILED class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if CAN_USE_PYREPL: + theme = get_theme().syntax + ps1 = f"{theme.prompt}{ps1}{theme.reset}" + console.write(f"{ps1}import asyncio\n") + + if CAN_USE_PYREPL: + from _pyrepl.simple_interact import ( + run_multiline_interactive_console, + ) + try: + run_multiline_interactive_console(console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 + else: + console.interact(banner="", exitmsg="") finally: warnings.filterwarnings( 'ignore', @@ -88,8 +133,56 @@ def run(self): loop.call_soon_threadsafe(loop.stop) + def interrupt(self) -> None: + if not CAN_USE_PYREPL: + return + + from _pyrepl.simple_interact import _get_reader + r = _get_reader() + if r.threading_hook is not None: + r.threading_hook.add("") # type: ignore + if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + color=True, + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + + sys.audit("cpython.run_stdin") + + if os.getenv('PYTHON_BASIC_REPL'): + CAN_USE_PYREPL = False + else: + from _pyrepl.main import CAN_USE_PYREPL + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -102,14 +195,31 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA except ImportError: - pass + readline = None - repl_thread = REPLThread() + interactive_hook = getattr(sys, "__interactivehook__", None) + + if interactive_hook is not None: + sys.audit("cpython.run_interactivehook", interactive_hook) + interactive_hook() + + if interactive_hook is site.register_readline: + # Fix the completer function to use the interactive console locals + try: + import rlcompleter + except: + pass + else: + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) + + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -117,9 +227,13 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True + repl_thread.interrupt() continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 29eff0499cb..8cbb71f7085 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,7 +17,6 @@ import collections.abc import concurrent.futures import errno -import functools import heapq import itertools import os @@ -279,7 +278,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +293,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +348,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' @@ -422,6 +430,8 @@ def __init__(self): self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None self.set_debug(coroutines._is_debug_mode()) + # The preserved state of async generator hooks. + self._old_agen_hooks = None # In debug mode, if the execution of a callback or a step of a task # exceed this duration in seconds, the slow callback/task is logged. self.slow_callback_duration = 0.1 @@ -448,26 +458,24 @@ def create_future(self): """Create a Future object attached to the loop.""" return futures.Future(loop=self) - def create_task(self, coro, *, name=None, context=None): - """Schedule a coroutine object. + def create_task(self, coro, **kwargs): + """Schedule or begin executing a coroutine object. Return a task object. """ self._check_closed() - if self._task_factory is None: - task = tasks.Task(coro, loop=self, name=name, context=context) - if task._source_traceback: - del task._source_traceback[-1] - else: - if context is None: - # Use legacy API if context is not needed - task = self._task_factory(self, coro) - else: - task = self._task_factory(self, coro, context=context) - - tasks._set_task_name(task, name) + if self._task_factory is not None: + return self._task_factory(self, coro, **kwargs) - return task + task = tasks.Task(coro, loop=self, **kwargs) + if task._source_traceback: + del task._source_traceback[-1] + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). @@ -475,9 +483,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object. The callable - must return a Future. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the active + event loop, 'coro' will be a coroutine object, and **kwargs will be + arbitrary keyword arguments that should be passed on to Task. + The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -624,29 +633,52 @@ def _check_running(self): raise RuntimeError( 'Cannot run the event loop while another loop is running') - def run_forever(self): - """Run until stop() is called.""" + def _run_forever_setup(self): + """Prepare the run loop to process events. + + This method exists so that custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop setup logic. + """ self._check_closed() self._check_running() self._set_coroutine_origin_tracking(self._debug) - old_agen_hooks = sys.get_asyncgen_hooks() - try: - self._thread_id = threading.get_ident() - sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, - finalizer=self._asyncgen_finalizer_hook) + self._old_agen_hooks = sys.get_asyncgen_hooks() + self._thread_id = threading.get_ident() + sys.set_asyncgen_hooks( + firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook + ) + + events._set_running_loop(self) + + def _run_forever_cleanup(self): + """Clean up after an event loop finishes the looping over events. + + This method exists so that custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop cleanup logic. + """ + self._stopping = False + self._thread_id = None + events._set_running_loop(None) + self._set_coroutine_origin_tracking(False) + # Restore any pre-existing async generator hooks. + if self._old_agen_hooks is not None: + sys.set_asyncgen_hooks(*self._old_agen_hooks) + self._old_agen_hooks = None - events._set_running_loop(self) + def run_forever(self): + """Run until stop() is called.""" + self._run_forever_setup() + try: while True: self._run_once() if self._stopping: break finally: - self._stopping = False - self._thread_id = None - events._set_running_loop(None) - self._set_coroutine_origin_tracking(False) - sys.set_asyncgen_hooks(*old_agen_hooks) + self._run_forever_cleanup() def run_until_complete(self, future): """Run until the Future is done. @@ -803,7 +835,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -840,7 +872,10 @@ def call_soon_threadsafe(self, callback, *args, context=None): self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args, context) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -981,39 +1016,43 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): family, type_, proto, _, address = addr_info sock = None try: - sock = socket.socket(family=family, type=type_, proto=proto) - sock.setblocking(False) - if local_addr_infos is not None: - for lfamily, _, _, _, laddr in local_addr_infos: - # skip local addresses of different family - if lfamily != family: - continue - try: - sock.bind(laddr) - break - except OSError as exc: - msg = ( - f'error while attempting to bind on ' - f'address {laddr!r}: ' - f'{exc.strerror.lower()}' - ) - exc = OSError(exc.errno, msg) - my_exceptions.append(exc) - else: # all bind attempts failed - if my_exceptions: - raise my_exceptions.pop() - else: - raise OSError(f"no matching local address with {family=} found") - await self.sock_connect(sock, address) - return sock - except OSError as exc: - my_exceptions.append(exc) - if sock is not None: - sock.close() - raise + try: + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: {str(exc).lower()}' + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await self.sock_connect(sock, address) + return sock + except OSError as exc: + my_exceptions.append(exc) + raise except: if sock is not None: - sock.close() + try: + sock.close() + except OSError: + # An error when closing a newly created socket is + # not important, but it can overwrite more important + # non-OSError error. So ignore it. + pass raise finally: exceptions = my_exceptions = None @@ -1107,11 +1146,18 @@ async def create_connection( except OSError: continue else: # using happy eyeballs - sock, _, _ = await staggered.staggered_race( - (functools.partial(self._connect_sock, - exceptions, addrinfo, laddr_infos) - for addrinfo in infos), - happy_eyeballs_delay, loop=self) + sock = (await staggered.staggered_race( + ( + # can't use functools.partial as it keeps a reference + # to exceptions + lambda addrinfo=addrinfo: self._connect_sock( + exceptions, addrinfo, laddr_infos + ) + for addrinfo in infos + ), + happy_eyeballs_delay, + loop=self, + ))[0] # can't use sock, _, _ as it keeks a reference to exceptions if sock is None: exceptions = [exc for sub in exceptions for exc in sub] @@ -1120,7 +1166,7 @@ async def create_connection( raise ExceptionGroup("create_connection failed", exceptions) if len(exceptions) == 1: raise exceptions[0] - else: + elif exceptions: # If they all have the same str(), raise one. model = str(exceptions[0]) if all(str(exc) == model for exc in exceptions): @@ -1129,6 +1175,9 @@ async def create_connection( # the various error messages. raise OSError('Multiple exceptions: {}'.format( ', '.join(str(exc) for exc in exceptions))) + else: + # No exceptions were collected, raise a timeout error + raise TimeoutError('create_connection failed') finally: exceptions = None @@ -1254,8 +1303,8 @@ async def _sendfile_fallback(self, transp, file, offset, count): read = await self.run_in_executor(None, file.readinto, view) if not read: return total_sent # EOF - await proto.drain() transp.write(view[:read]) + await proto.drain() total_sent += read finally: if total_sent > 0 and hasattr(file, 'seek'): @@ -1474,6 +1523,7 @@ async def create_server( ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -1545,8 +1595,13 @@ async def create_server( if reuse_address: sock.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and af in (socket.AF_INET, socket.AF_INET6): _set_reuseport(sock) + if keep_alive: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. @@ -1561,7 +1616,7 @@ async def create_server( except OSError as err: msg = ('error while attempting ' 'to bind on address %r: %s' - % (sa, err.strerror.lower())) + % (sa, str(err).lower())) if err.errno == errno.EADDRNOTAVAIL: # Assume the family is not enabled (bpo-30945) sockets.pop() @@ -1619,8 +1674,7 @@ async def connect_accepted_socket( raise ValueError( 'ssl_shutdown_timeout is only meaningful with ssl') - if sock is not None: - _check_ssl_socket(sock) + _check_ssl_socket(sock) transport, protocol = await self._create_connection_transport( sock, protocol_factory, ssl, '', server_side=True, @@ -1833,6 +1887,8 @@ def call_exception_handler(self, context): - 'protocol' (optional): Protocol instance; - 'transport' (optional): Transport instance; - 'socket' (optional): Socket instance; + - 'source_traceback' (optional): Traceback of the source; + - 'handle_traceback' (optional): Traceback of the handle; - 'asyncgen' (optional): Asynchronous generator that caused the exception. @@ -1943,8 +1999,11 @@ def _run_once(self): timeout = 0 elif self._scheduled: # Compute the desired timeout. - when = self._scheduled[0]._when - timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) + timeout = self._scheduled[0]._when - self.time() + if timeout > MAXIMUM_SELECT_TIMEOUT: + timeout = MAXIMUM_SELECT_TIMEOUT + elif timeout < 0: + timeout = 0 event_list = self._selector.select(timeout) self._process_events(event_list) diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 4c9b0dd5653..321a4e5d5d1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -1,6 +1,9 @@ import collections import subprocess import warnings +import os +import signal +import sys from . import protocols from . import transports @@ -23,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -101,7 +105,12 @@ def close(self): for proto in self._pipes.values(): if proto is None: continue - proto.pipe.close() + # See gh-114177 + # skip closing the pipe if loop is already closed + # this can happen e.g. when loop is closed immediately after + # process is killed + if self._loop and not self._loop.is_closed(): + proto.pipe.close() if (self._proc is not None and # has the child process finished? @@ -115,7 +124,8 @@ def close(self): try: self._proc.kill() - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # the process may have already exited or may be running setuid pass # Don't clear the _proc reference yet: _post_init() may still run @@ -141,17 +151,31 @@ def _check_proc(self): if self._proc is None: raise ProcessLookupError() - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) + if sys.platform == 'win32': + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + else: + def send_signal(self, signal): + self._check_proc() + try: + os.kill(self._proc.pid, signal) + except ProcessLookupError: + pass - def terminate(self): - self._check_proc() - self._proc.terminate() + def terminate(self): + self.send_signal(signal.SIGTERM) - def kill(self): - self._check_proc() - self._proc.kill() + def kill(self): + self.send_signal(signal.SIGKILL) async def _connect_pipes(self, waiter): try: @@ -190,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -233,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index ab4f30eb51b..a51319cb72a 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,7 +18,16 @@ def _is_debug_mode(): def iscoroutinefunction(func): + import warnings """Return True if func is a decorated coroutine function.""" + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + + +def _iscoroutinefunction(func): return (inspect.iscoroutinefunction(func) or getattr(func, '_is_coroutine', None) is _is_coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 016852880ca..a7fb55982ab 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -22,6 +26,7 @@ import subprocess import sys import threading +import warnings from . import format_helpers @@ -54,7 +59,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -90,7 +96,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, @@ -102,6 +109,34 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" @@ -173,6 +208,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError @@ -282,7 +325,7 @@ def create_future(self): # Method scheduling a coroutine object: create a task. - def create_task(self, coro, *, name=None, context=None): + def create_task(self, coro, **kwargs): raise NotImplementedError # Methods for interacting with threads. @@ -320,6 +363,7 @@ async def create_server( *, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -358,6 +402,9 @@ async def create_server( they all set this flag when being created. This option is not supported on Windows. + keep_alive set to True keeps connections active by enabling the + periodic transmission of messages. + ssl_handshake_timeout is the time in seconds that an SSL server will wait for completion of the SSL handshake before aborting the connection. Default is 60s. @@ -615,7 +662,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -638,18 +685,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -666,7 +702,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() @@ -676,28 +711,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - stacklevel = 2 - try: - f = sys._getframe(1) - except AttributeError: - pass - else: - # Move up the call stack so that the warning is attached - # to the line outside asyncio itself. - while f: - module = f.f_globals.get('__name__') - if not (module == 'asyncio' or module.startswith('asyncio.')): - break - f = f.f_back - stacklevel += 1 - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -706,7 +719,6 @@ def get_event_loop(self): def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True if loop is not None and not isinstance(loop, AbstractEventLoop): raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop @@ -776,26 +788,35 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() -def set_event_loop_policy(policy): +def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy +def set_event_loop_policy(policy): + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) + _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. @@ -810,28 +831,17 @@ def get_event_loop(): current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + _get_event_loop_policy().set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) + return _get_event_loop_policy().new_event_loop() # Alias pure-Python implementations for testing purposes. @@ -861,7 +871,7 @@ def set_child_watcher(watcher): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9..93737b7708a 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 97fc4e3fcb6..d1df6707302 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -43,7 +44,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,12 +61,15 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between - # `await Future()` or`yield from Future()` (correct) vs. + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + __asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -116,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -138,9 +147,6 @@ def _make_cancelled_error(self): exc = exceptions.CancelledError() else: exc = exceptions.CancelledError(self._cancel_message) - exc.__context__ = self._cancelled_exc - # Remove the reference since we don't need this anymore. - self._cancelled_exc = None return exc def cancel(self, msg=None): @@ -194,8 +200,7 @@ def result(self): the future is done and has an exception set, this exception is raised. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Result is not ready.') self.__log_traceback = False @@ -212,8 +217,7 @@ def exception(self): InvalidStateError. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Exception is not set.') self.__log_traceback = False @@ -272,9 +276,13 @@ def set_exception(self, exception): raise exceptions.InvalidStateError(f'{self._state}: {self!r}') if isinstance(exception, type): exception = exception() - if type(exception) is StopIteration: - raise TypeError("StopIteration interacts badly with generators " - "and cannot be raised into a Future") + if isinstance(exception, StopIteration): + new_exc = RuntimeError("StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future") + new_exc.__cause__ = exception + new_exc.__context__ = exception + exception = new_exc self._exception = exception self._exception_tb = exception.__traceback__ self._state = _FINISHED @@ -318,11 +326,9 @@ def _set_result_unless_cancelled(fut, result): def _convert_future_exc(exc): exc_class = type(exc) if exc_class is concurrent.futures.CancelledError: - return exceptions.CancelledError(*exc.args) - elif exc_class is concurrent.futures.TimeoutError: - return exceptions.TimeoutError(*exc.args) + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) elif exc_class is concurrent.futures.InvalidStateError: - return exceptions.InvalidStateError(*exc.args) + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) else: return exc @@ -419,6 +425,49 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter, /): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that most of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter, /): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implementation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) + + +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -426,3 +475,7 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py new file mode 100644 index 00000000000..b5bfeb1630a --- /dev/null +++ b/Lib/asyncio/graph.py @@ -0,0 +1,276 @@ +"""Introspection utils for tasks call graphs.""" + +import dataclasses +import io +import sys +import types + +from . import events +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_graph', + 'format_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'FutureCallGraph', +) + +# Sadly, we can't re-use the traceback module's datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call graph. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + + +@dataclasses.dataclass(frozen=True, slots=True) +class FrameCallGraphEntry: + frame: types.FrameType + + +@dataclasses.dataclass(frozen=True, slots=True) +class FutureCallGraph: + future: futures.Future + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] + + +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + coro = None + if get_coro := getattr(future, 'get_coro', None): + coro = get_coro() if limit != 0 else None + + st: list[FrameCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.ag_await + else: + break + + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] + st.reverse() + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) + + +def capture_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> FutureCallGraph | None: + """Capture the async call graph for the current task or the provided Future. + + The graph is represented with three data structures: + + * FutureCallGraph(future, call_stack, awaited_by) + + Where 'future' is an instance of asyncio.Future or asyncio.Task. + + 'call_stack' is a tuple of FrameGraphEntry objects. + + 'awaited_by' is a tuple of FutureCallGraph objects. + + * FrameCallGraphEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + Receives an optional 'future' argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only 'depth' argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. + """ + + loop = events._get_running_loop() + + if future is not None: + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(loop=loop): + return _build_graph_for_future(future, limit=limit) + # else: future is the current task, move on. + else: + if loop is None: + raise RuntimeError( + 'capture_call_graph() is called outside of a running ' + 'event loop and no *future* to introspect was provided') + future = tasks.current_task(loop=loop) + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + call_stack: list[FrameCallGraphEntry] = [] + + f = sys._getframe(depth) if limit != 0 else None + try: + while f is not None: + is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) + + if is_async: + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + + f = f.f_back + finally: + del f + + awaited_by = [] + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] + + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) + + +def format_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> str: + """Return the async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ + + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' + ) + else: + add_line( + f'* Future(id={id(st.future):#x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: + f = ste.frame + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' + ) + else: + c = f.f_generator + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, buf, level + 1) + + graph = capture_call_graph(future, depth=depth + 1, limit=limit) + if graph is None: + return "" + + buf: list[str] = [] + try: + render_level(graph, buf, 0) + finally: + # 'graph' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del graph + return '\n'.join(buf) + +def print_call_graph( + future: futures.Future | None = None, + /, + *, + file: io.Writer[str] | None = None, + depth: int = 1, + limit: int | None = None, +) -> None: + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index ce5d8d5bfb2..fa3a94764b5 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'await'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. Locks also support the asynchronous context management protocol. 'async with lock' statement should be used. @@ -95,6 +93,8 @@ async def acquire(self): This method blocks until the lock is unlocked, then sets it to locked and returns True. """ + # Implement fair scheduling, where thread always waits + # its turn. Jumping the queue if all are cancelled is an optimization. if (not self._locked and (self._waiters is None or all(w.cancelled() for w in self._waiters))): self._locked = True @@ -105,19 +105,22 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: + # Currently the only exception designed be able to occur here. + + # Ensure the lock invariant: If lock is not claimed (or about + # to be claimed by us) and there is a Task in waiters, + # ensure that the Task at the head will run. if not self._locked: self._wake_up_first() raise + # assert self._locked is False self._locked = True return True @@ -125,7 +128,7 @@ def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -139,7 +142,7 @@ def release(self): raise RuntimeError('Lock is not acquired.') def _wake_up_first(self): - """Wake up the first waiter if it isn't done.""" + """Ensure that the first waiter will wake up.""" if not self._waiters: return try: @@ -147,9 +150,7 @@ def _wake_up_first(self): except StopIteration: return - # .done() necessarily means that a waiter will wake up later on and - # either take the lock, or, if it was cancelled and lock wasn't - # taken already, will hit this again and wake up a new waiter. + # .done() means that the waiter is already set to wake up. if not fut.done(): fut.set_result(True) @@ -179,8 +180,8 @@ def is_set(self): return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -191,7 +192,7 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False @@ -200,7 +201,7 @@ async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: @@ -219,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ @@ -247,45 +248,64 @@ def __repr__(self): async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._get_loop().create_future() - self._waiters.append(fut) try: - await fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must reacquire lock even if wait is cancelled - cancelled = False - while True: + self._waiters.append(fut) try: - await self.acquire() - break - except exceptions.CancelledError: - cancelled = True + await fut + return True + finally: + self._waiters.remove(fut) - if cancelled: - raise exceptions.CancelledError + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() @@ -295,20 +315,22 @@ async def wait_for(self, predicate): return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -319,9 +341,9 @@ def notify(self, n=1): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) @@ -357,6 +379,7 @@ def __repr__(self): def locked(self): """Returns True if semaphore cannot be acquired immediately.""" + # Due to state, or FIFO rules (must allow others to run first). return self._value == 0 or ( any(not w.cancelled() for w in (self._waiters or ()))) @@ -365,11 +388,12 @@ async def acquire(self): If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ if not self.locked(): + # Maintain FIFO, wait for others to start even if _value > 0. self._value -= 1 return True @@ -378,29 +402,34 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: - if not fut.cancelled(): + # Currently the only exception designed be able to occur here. + if fut.done() and not fut.cancelled(): + # Our Future was successfully set to True via _wake_up_next(), + # but we are not about to successfully acquire(). Therefore we + # must undo the bookkeeping already done and attempt to wake + # up someone else. self._value += 1 - self._wake_up_next() raise - if self._value > 0: - self._wake_up_next() + finally: + # New waiters may have arrived but had to wait due to FIFO. + # Wake up as many as are allowed. + while self._value > 0: + if not self._wake_up_next(): + break # There was no-one to wake up. return True def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() @@ -408,13 +437,15 @@ def release(self): def _wake_up_next(self): """Wake up the first waiter that isn't done.""" if not self._waiters: - return + return False for fut in self._waiters: if not fut.done(): self._value -= 1 fut.set_result(True) - return + # `fut` is now `done()` and not `cancelled()`. + return True + return False class BoundedSemaphore(Semaphore): @@ -454,7 +485,7 @@ class Barrier(mixins._LoopBoundMixin): def __init__(self, parties): """Create a barrier, initialised to 'parties' tasks.""" if parties < 1: - raise ValueError('parties must be > 0') + raise ValueError('parties must be >= 1') self._cond = Condition() # notify all tasks when state changes diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 1e2a730cf36..f404273c3ae 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True @@ -460,6 +460,8 @@ def _pipe_closed(self, fut): class _ProactorDatagramTransport(_ProactorBasePipeTransport, transports.DatagramTransport): max_size = 256 * 1024 + _header_size = 8 + def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address @@ -487,9 +489,6 @@ def sendto(self, data, addr=None): raise TypeError('data argument must be bytes-like object (%r)', type(data)) - if not data: - return - if self._address is not None and addr not in (None, self._address): raise ValueError( f'Invalid address: must be None or {self._address}') @@ -502,7 +501,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size if self._write_fut is None: # No current write operations are active, kick one off @@ -529,7 +528,7 @@ def _loop_writing(self, fut=None): return data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size if self._address is not None: self._write_fut = self._loop._proactor.send(self._sock, data) @@ -724,6 +723,8 @@ async def sock_sendto(self, sock, data, address): return await self._proactor.sendto(sock, data, 0, address) async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") return await self._proactor.connect(sock, address) async def sock_accept(self, sock): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df56..084fccaaff2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,11 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +53,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -81,6 +94,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -112,8 +127,12 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raises QueueShutDown if the queue has been shut down. """ while self.full(): + if self._is_shutdown: + raise QueueShutDown putter = self._get_loop().create_future() self._putters.append(putter) try: @@ -125,7 +144,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take @@ -138,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -150,8 +173,13 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): + if self._is_shutdown and self.empty(): + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -163,7 +191,7 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take @@ -176,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -214,6 +247,36 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 1b89236599a..ba37e003a65 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -84,10 +85,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +93,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() @@ -168,6 +177,7 @@ def run(main, *, debug=None, loop_factory=None): running in the same thread. If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 790711f8340..ff7e16df3c6 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,16 +173,20 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, @@ -265,22 +269,17 @@ def _ensure_fd_no_transport(self, fd): except (AttributeError, TypeError, ValueError): # This code matches selectors._fileobj_to_fd function. raise ValueError(f"Invalid file object: {fd!r}") from None - try: - transport = self._transports[fileno] - except KeyError: - pass - else: - if not transport.is_closing(): - raise RuntimeError( - f'File descriptor {fd!r} is used by transport ' - f'{transport!r}') + transport = self._transports.get(fileno) + if transport and not transport.is_closing(): + raise RuntimeError( + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_READ, (handle, None)) else: @@ -294,30 +293,27 @@ def _add_reader(self, fd, callback, *args): def _remove_reader(self, fd): if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + mask &= ~selectors.EVENT_READ + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - mask &= ~selectors.EVENT_READ - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (None, writer)) + self._selector.modify(fd, mask, (None, writer)) - if reader is not None: - reader.cancel() - return True - else: - return False + if reader is not None: + reader.cancel() + return True + else: + return False def _add_writer(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_WRITE, (None, handle)) else: @@ -332,24 +328,22 @@ def _remove_writer(self, fd): """Remove a writer callback.""" if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + # Remove both writer and connector. + mask &= ~selectors.EVENT_WRITE + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - # Remove both writer and connector. - mask &= ~selectors.EVENT_WRITE - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (reader, None)) + self._selector.modify(fd, mask, (reader, None)) - if writer is not None: - writer.cancel() - return True - else: - return False + if writer is not None: + writer.cancel() + return True + else: + return False def add_reader(self, fd, callback, *args): """Add a reader callback.""" @@ -801,7 +795,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -878,6 +872,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -916,7 +912,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): @@ -1054,8 +1050,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: @@ -1178,20 +1174,31 @@ def writelines(self, list_of_data): raise RuntimeError('unable to writelines; sendfile is in progress') if not list_of_data: return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + self._buffer.extend([memoryview(data) for data in list_of_data]) self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: self._loop._add_writer(self._sock_fd, self._write_ready) + self._maybe_pause_protocol() def can_write_eof(self): return True def _call_connection_lost(self, exc): - super()._call_connection_lost(exc) - if self._empty_waiter is not None: - self._empty_waiter.set_exception( - ConnectionError("Connection is closed by peer")) + try: + super()._call_connection_lost(exc) + finally: + self._write_ready = None + if self._empty_waiter is not None: + self._empty_waiter.set_exception( + ConnectionError("Connection is closed by peer")) def _make_empty_waiter(self): if self._empty_waiter is not None: @@ -1206,13 +1213,13 @@ def _reset_empty_waiter(self): def close(self): self._read_ready_cb = None - self._write_ready = None super().close() class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): @@ -1251,8 +1258,6 @@ def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError(f'data argument must be a bytes-like object, ' f'not {type(data).__name__!r}') - if not data: - return if self._address: if addr not in (None, self._address): @@ -1288,13 +1293,13 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size try: if self._extra['peername']: self._sock.send(data) @@ -1302,7 +1307,7 @@ def _sendto_ready(self): self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index e51669a2ab2..74c5f0d5ca0 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -101,7 +101,7 @@ def get_protocol(self): return self._ssl_protocol._app_protocol def is_closing(self): - return self._closed + return self._closed or self._ssl_protocol._is_transport_closing() def close(self): """Close the transport. @@ -379,6 +379,9 @@ def _get_app_transport(self): self._app_transport_created = True return self._app_transport + def _is_transport_closing(self): + return self._transport is not None and self._transport.is_closing() + def connection_made(self, transport): """Called when the low-level connection is made. @@ -542,7 +545,7 @@ def _start_handshake(self): # start handshake timeout count down self._handshake_timeout_handle = \ self._loop.call_later(self._ssl_handshake_timeout, - lambda: self._check_handshake_timeout()) + self._check_handshake_timeout) self._do_handshake() @@ -623,7 +626,7 @@ def _start_shutdown(self): self._set_state(SSLProtocolState.FLUSHING) self._shutdown_timeout_handle = self._loop.call_later( self._ssl_shutdown_timeout, - lambda: self._check_shutdown_timeout() + self._check_shutdown_timeout ) self._do_flush() @@ -762,7 +765,7 @@ def _do_read__buffered(self): else: break else: - self._loop.call_soon(lambda: self._do_read()) + self._loop.call_soon(self._do_read) except SSLAgainErrors: pass if offset > 0: diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 451a53a16f3..2ad65d8648e 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -3,24 +3,15 @@ __all__ = 'staggered_race', import contextlib -import typing from . import events from . import exceptions as exceptions_mod from . import locks from . import tasks +from . import futures -async def staggered_race( - coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]], - delay: typing.Optional[float], - *, - loop: events.AbstractEventLoop = None, -) -> typing.Tuple[ - typing.Any, - typing.Optional[int], - typing.List[typing.Optional[Exception]] -]: +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is @@ -73,14 +64,38 @@ async def staggered_race( """ # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() + parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None + unhandled_exceptions = [] exceptions = [] - running_tasks = [] + running_tasks = set() + on_completed_fut = None + + def task_done(task): + running_tasks.discard(task) + futures.future_discard_from_awaited_by(task, parent_task) + if ( + on_completed_fut is not None + and not on_completed_fut.done() + and not running_tasks + ): + on_completed_fut.set_result(None) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + unhandled_exceptions.append(exc) - async def run_one_coro( - previous_failed: typing.Optional[locks.Event]) -> None: + async def run_one_coro(ok_to_start, previous_failed) -> None: + # in eager tasks this waits for the calling task to append this task + # to running_tasks, in regular tasks this wait is a no-op that does + # not yield a future. See gh-124309. + await ok_to_start.wait() # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -96,9 +111,14 @@ async def run_one_coro( return # Start task that will run the next coroutine this_failed = locks.Event() - next_task = loop.create_task(run_one_coro(this_failed)) - running_tasks.append(next_task) - assert len(running_tasks) == this_index + 2 + next_ok_to_start = locks.Event() + next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + futures.future_add_to_awaited_by(next_task, parent_task) + running_tasks.add(next_task) + next_task.add_done_callback(task_done) + # next_task has been appended to running_tasks so next_task is ok to + # start. + next_ok_to_start.set() # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -123,27 +143,37 @@ async def run_one_coro( # up as done() == True, cancelled() == False, exception() == # asyncio.CancelledError. This behavior is specified in # https://bugs.python.org/issue30048 - for i, t in enumerate(running_tasks): - if i != this_index: + current_task = tasks.current_task(loop) + for t in running_tasks: + if t is not current_task: t.cancel() - first_task = loop.create_task(run_one_coro(None)) - running_tasks.append(first_task) + propagate_cancellation_error = None try: - # Wait for a growing list of tasks to all finish: poor man's version of - # curio's TaskGroup or trio's nursery - done_count = 0 - while done_count != len(running_tasks): - done, _ = await tasks.wait(running_tasks) - done_count = len(done) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) + futures.future_add_to_awaited_by(first_task, parent_task) + running_tasks.add(first_task) + first_task.add_done_callback(task_done) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() + propagate_cancellation_error = None + # Make sure no tasks are left running if we leave this function + while running_tasks: + on_completed_fut = loop.create_future() + try: + await on_completed_fut + except exceptions_mod.CancelledError as ex: + propagate_cancellation_error = ex + for task in running_tasks: + task.cancel(*ex.args) + on_completed_fut = None + if __debug__ and unhandled_exceptions: # If run_one_coro raises an unhandled exception, it's probably a # programming error, and I want to see it. - if __debug__: - for d in done: - if d.done() and not d.cancelled() and d.exception(): - raise d.exception() + raise ExceptionGroup("staggered race failed", unhandled_exceptions) + if propagate_cancellation_error is not None: + raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - # Make sure no tasks are left running if we leave this function - for t in running_tasks: - t.cancel() + del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index f310aa2f367..64aac4cc50d 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,11 +236,8 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -405,9 +399,9 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) - def __del__(self): + def __del__(self, warnings=warnings): if not self._transport.is_closing(): if self._loop.is_closed(): warnings.warn("loop is closed", ResourceWarning) @@ -596,20 +590,34 @@ async def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, tuple): + # Makes sure shortest matches wins + separator = sorted(separator, key=len) + else: + separator = [separator] + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -622,26 +630,35 @@ async def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', @@ -650,7 +667,7 @@ async def readuntil(self, separator=b'\n'): # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() @@ -659,12 +676,12 @@ async def readuntil(self, separator=b'\n'): # _wait_for_data() will resume reading if stream was paused. await self._wait_for_data('readuntil') - if isep > self._limit: + if match_start > self._limit: raise exceptions.LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index d264e51f1fd..00e8f6d5d1a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -66,6 +67,20 @@ async def __aenter__(self): return self async def __aexit__(self, et, exc, tb): + tb = None + try: + return await self._aexit(et, exc) + finally: + # Exceptions are heavy objects that can have object + # cycles (bad for GC); let's not keep a reference to + # a bunch of them. It would be nicer to use a try/finally + # in __aexit__ directly but that introduced some diff noise + self._parent_task = None + self._errors = None + self._base_error = None + exc = None + + async def _aexit(self, et, exc): self._exiting = True if (exc is not None and @@ -73,14 +88,10 @@ async def __aexit__(self, et, exc, tb): self._base_error is None): self._base_error = exc - propagate_cancellation_error = \ - exc if et is exceptions.CancelledError else None - if self._parent_cancel_requested: - # If this flag is set we *must* call uncancel(). - if self._parent_task.uncancel() == 0: - # If there are no pending cancellations left, - # don't propagate CancelledError. - propagate_cancellation_error = None + if et is not None and issubclass(et, exceptions.CancelledError): + propagate_cancellation_error = exc + else: + propagate_cancellation_error = None if et is not None: if not self._aborting: @@ -126,51 +137,78 @@ async def __aexit__(self, et, exc, tb): assert not self._tasks if self._base_error is not None: - raise self._base_error + try: + raise self._base_error + finally: + exc = None + + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error and not self._errors: - raise propagate_cancellation_error - - if et is not None and et is not exceptions.CancelledError: + try: + if propagate_cancellation_error is not None and not self._errors: + try: + raise propagate_cancellation_error + finally: + exc = None + finally: + propagate_cancellation_error = None + + if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) if self._errors: - # Exceptions are heavy objects that can have object - # cycles (bad for GC); let's not keep a reference to - # a bunch of them. + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() try: - me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors) - raise me from None + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: - self._errors = None + exc = None - def create_task(self, coro, *, name=None, context=None): + + def create_task(self, coro, **kwargs): """Create a new task in this group and return it. Similar to `asyncio.create_task`. """ if not self._entered: + coro.close() raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is finished") if self._aborting: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") - if context is None: - task = self._loop.create_task(coro) - else: - task = self._loop.create_task(coro, context=context) - tasks._set_task_name(task, name) - # optimization: Immediately call the done callback if the task is + task = self._loop.create_task(coro, **kwargs) + + futures.future_add_to_awaited_by(task, self._parent_task) + + # Always schedule the done callback even if the task is # already done (e.g. if the coro was able to complete eagerly), - # and skip scheduling a done callback - if task.done(): - self._on_task_done(task) - else: - self._tasks.add(task) - task.add_done_callback(self._on_task_done) - return task + # otherwise if the task completes with an exception then it will cancel + # the current task too early. gh-128550, gh-128588 + self._tasks.add(task) + task.add_done_callback(self._on_task_done) + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task # Since Python 3.8 Tasks propagate all exceptions correctly, # except for KeyboardInterrupt and SystemExit which are @@ -190,6 +228,8 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + futures.future_discard_from_awaited_by(task, self._parent_task) + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 0b22e28d8e0..fbd5c39a7c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -15,8 +15,8 @@ import functools import inspect import itertools +import math import types -import warnings import weakref from types import GenericAlias @@ -25,6 +25,7 @@ from . import events from . import exceptions from . import futures +from . import queues from . import timeouts # Helper to generate new task names @@ -47,37 +48,9 @@ def all_tasks(loop=None): # capturing the set of eager tasks first, so if an eager task "graduates" # to a regular task in another thread, we don't risk missing it. eager_tasks = list(_eager_tasks) - # Looping over the WeakSet isn't safe as it can be updated from another - # thread, therefore we cast it to list prior to filtering. The list cast - # itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). - # See issues 34970 and 36607 for details. - scheduled_tasks = None - i = 0 - while True: - try: - scheduled_tasks = list(_scheduled_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in itertools.chain(scheduled_tasks, eager_tasks) - if futures._get_loop(t) is loop and not t.done()} - -def _set_task_name(task, name): - if name is not None: - try: - set_name = task.set_name - except AttributeError: - warnings.warn("Task.set_name() was added in Python 3.8, " - "the method support will be mandatory for third-party " - "task implementations since 3.13.", - DeprecationWarning, stacklevel=3) - else: - set_name(name) + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) + if futures._get_loop(t) is loop and not t.done()} class Task(futures._PyFuture): # Inherit Python Task implementation @@ -137,7 +110,7 @@ def __init__(self, coro, *, loop=None, name=None, context=None, self.__eager_start() else: self._loop.call_soon(self.__step, context=self._context) - _register_task(self) + _py_register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -267,42 +240,44 @@ def uncancel(self): """ if self._num_cancels_requested > 0: self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False return self._num_cancels_requested def __eager_start(self): - prev_task = _swap_current_task(self._loop, self) + prev_task = _py_swap_current_task(self._loop, self) try: - _register_eager_task(self) + _py_register_eager_task(self) try: self._context.run(self.__step_run_and_handle_result, None) finally: - _unregister_eager_task(self) + _py_unregister_eager_task(self) finally: try: - curtask = _swap_current_task(self._loop, prev_task) + curtask = _py_swap_current_task(self._loop, prev_task) assert curtask is self finally: if self.done(): self._coro = None self = None # Needed to break cycles when an exception occurs. else: - _register_task(self) + _py_register_task(self) def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() self._must_cancel = False self._fut_waiter = None - _enter_task(self._loop, self) + _py_enter_task(self._loop, self) try: self.__step_run_and_handle_result(exc) finally: - _leave_task(self._loop, self) + _py_leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. def __step_run_and_handle_result(self, exc): @@ -347,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -381,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -389,7 +366,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -409,20 +386,13 @@ def __wakeup(self, future): Task = _CTask = _asyncio.Task -def create_task(coro, *, name=None, context=None): +def create_task(coro, **kwargs): """Schedule the execution of a coroutine object in a spawn task. Return a Task object. """ loop = events.get_running_loop() - if context is None: - # Use legacy API if context is not needed - task = loop.create_task(coro) - else: - task = loop.create_task(coro, context=context) - - _set_task_name(task, name) - return task + return loop.create_task(coro, **kwargs) # wait() and as_completed() similar to those in PEP 3148. @@ -437,8 +407,6 @@ async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): The fs iterable must not be empty. - Coroutines will be wrapped in Tasks. - Returns two sets of Future: (done, pending). Usage: @@ -530,6 +498,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -542,9 +511,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -580,62 +551,125 @@ async def _cancel_and_wait(fut): fut.remove_done_callback(cb) -# This is *not* a @coroutine! It is just an iterator (yielding Futures). +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. + + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. + """ + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None + + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} + for f in todo: + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f + + def as_completed(fs, *, timeout=None): - """Return an iterator whose values are coroutines. + """Create an iterator of awaitables or their results in completion order. - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. - This differs from PEP 3148; the proper way to use this is: + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: - for f in as_completed(fs): - result = await f # The 'await' may raise. - # Use result. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - If a timeout is specified, the 'await' will raise - TimeoutError when the timeout occurs before all Futures are done. + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect - Note: The futures 'f' are not necessarily members of fs. - """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") - from .queues import Queue # Import here to avoid circular import problem. - done = Queue() + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - loop = events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - timeout_handle = None + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect - async def _wait_for_one(): - f = await done.get() - if f is None: - # Dummy value from _on_timeout(). - raise exceptions.TimeoutError - return f.result() # May raise f.exception(). + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + return _AsCompletedIterator(fs, timeout) @types.coroutine @@ -656,6 +690,9 @@ async def sleep(delay, result=None): await __sleep0() return result + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") + loop = events.get_running_loop() future = loop.create_future() h = loop.call_later(delay, @@ -764,10 +801,19 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + futures.future_discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -824,7 +870,6 @@ def _done_callback(fut): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: @@ -837,12 +882,13 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(_done_callback) else: @@ -862,6 +908,25 @@ def _done_callback(fut): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -902,11 +967,16 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() + if loop is not None and (cur_task := current_task(loop)) is not None: + futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None + + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -918,10 +988,16 @@ def _inner_done_callback(inner): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) @@ -970,9 +1046,9 @@ def create_eager_task_factory(custom_task_constructor): used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. """ - def factory(loop, coro, *, name=None, context=None): + def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( - coro, loop=loop, name=name, context=context, eager_start=True) + coro, loop=loop, eager_start=eager_start, **kwargs) return factory @@ -1044,14 +1120,13 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, - _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1063,3 +1138,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 30042abb3ad..09342dc7c13 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -1,7 +1,6 @@ import enum from types import TracebackType -from typing import final, Optional, Type from . import events from . import exceptions @@ -23,14 +22,13 @@ class _State(enum.Enum): EXITED = "finished" -@final class Timeout: """Asynchronous context manager for cancelling overdue coroutines. Use `timeout()` or `timeout_at()` rather than instantiating this class directly. """ - def __init__(self, when: Optional[float]) -> None: + def __init__(self, when: float | None) -> None: """Schedule a timeout that will trigger at a given loop time. - If `when` is `None`, the timeout will never trigger. @@ -39,15 +37,15 @@ def __init__(self, when: Optional[float]) -> None: """ self._state = _State.CREATED - self._timeout_handler: Optional[events.TimerHandle] = None - self._task: Optional[tasks.Task] = None + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None self._when = when - def when(self) -> Optional[float]: + def when(self) -> float | None: """Return the current deadline.""" return self._when - def reschedule(self, when: Optional[float]) -> None: + def reschedule(self, when: float | None) -> None: """Reschedule the timeout.""" if self._state is not _State.ENTERED: if self._state is _State.CREATED: @@ -96,10 +94,10 @@ async def __aenter__(self) -> "Timeout": async def __aexit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> Optional[bool]: + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: assert self._state in (_State.ENTERED, _State.EXPIRING) if self._timeout_handler is not None: @@ -109,10 +107,16 @@ async def __aexit__( if self._state is _State.EXPIRING: self._state = _State.EXPIRED - if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + if self._task.uncancel() <= self._cancelling and exc_type is not None: # Since there are no new cancel requests, we're # handling this. - raise TimeoutError from exc_val + if issubclass(exc_type, exceptions.CancelledError): + raise TimeoutError from exc_val + elif exc_val is not None: + self._insert_timeout_error(exc_val) + if isinstance(exc_val, ExceptionGroup): + for exc in exc_val.exceptions: + self._insert_timeout_error(exc) elif self._state is _State.ENTERED: self._state = _State.EXITED @@ -125,8 +129,18 @@ def _on_timeout(self) -> None: # drop the reference early self._timeout_handler = None + @staticmethod + def _insert_timeout_error(exc_val: BaseException) -> None: + while exc_val.__context__ is not None: + if isinstance(exc_val.__context__, exceptions.CancelledError): + te = TimeoutError() + te.__context__ = te.__cause__ = exc_val.__context__ + exc_val.__context__ = te + break + exc_val = exc_val.__context__ -def timeout(delay: Optional[float]) -> Timeout: + +def timeout(delay: float | None) -> Timeout: """Timeout async context manager. Useful in cases when you want to apply timeout logic around block @@ -146,7 +160,7 @@ def timeout(delay: Optional[float]) -> Timeout: return Timeout(loop.time() + delay if delay is not None else None) -def timeout_at(when: Optional[float]) -> Timeout: +def timeout_at(when: float | None) -> Timeout: """Schedule the timeout at absolute time. Like timeout() but argument gives absolute time in the same clock system diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000..f39e11fdd51 --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,276 @@ +"""Tools to analyze tasks running in asyncio programs.""" + +from collections import defaultdict, namedtuple +from itertools import count +from enum import Enum +import sys +from _remote_debugging import RemoteUnwinder, FrameInfo + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + + + +# ─── indexing helpers ─────────────────────────────────────────── +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return elem + + +def _index(result): + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks + + +def _build_tree(id2name, awaits, task_stacks): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) + + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key + return node_key + + # Build task dependency tree with coroutine frames + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + + return id2label, children + + +def _roots(id2label, children): + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g + + +def _find_cycles(graph): + """ + Depth-first search for back-edges. + + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = defaultdict(lambda: WHITE) + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): + """ + Build a list of strings for pretty-print an async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. + """ + id2name, awaits, task_stacks = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) + labels, children = _build_tree(id2name, awaits, task_stacks) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + return [render(root) for root in _roots(labels, children)] + + +def build_task_table(result): + id2name, _, _ = _index(result) + table = [] + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) + + return table + +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles - cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + +def _get_awaited_by_tasks(pid: int) -> list: + try: + return get_all_awaited_by(pid) + except RuntimeError as e: + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}") + sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() + + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" + ) + print("-" * 180) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = build_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 30fd41d49af..34c7ad44ffd 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -181,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f2e920ada46..1c1458127db 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,10 +28,7 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', - 'DefaultEventLoopPolicy', + 'EventLoop', ) @@ -63,6 +60,11 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} + self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -92,7 +94,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -195,33 +197,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -283,7 +274,7 @@ async def create_unix_server( sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - start_serving=True): + start_serving=True, cleanup_socket=True): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') @@ -339,6 +330,15 @@ async def create_unix_server( raise ValueError( f'A UNIX Domain Stream Socket was expected, got {sock!r}') + if cleanup_socket: + path = sock.getsockname() + # Check for abstract socket. `str` and `bytes` paths are supported. + if path[0] not in (0, '\x00'): + try: + self._unix_server_sockets[sock] = os.stat(path).st_ino + except FileNotFoundError: + pass + sock.setblocking(False) server = base_events.Server(self, [sock], protocol_factory, ssl, backlog, ssl_handshake_timeout, @@ -393,6 +393,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fut.set_result(total_sent) return + # On 32-bit architectures truncate to 1GiB to avoid OverflowError + blocksize = min(blocksize, sys.maxsize//2 + 1) + try: sent = os.sendfile(fd, fileno, offset, blocksize) except (BlockingIOError, InterruptedError): @@ -456,6 +459,27 @@ def cb(fut): self.remove_writer(fd) fut.add_done_callback(cb) + def _stop_serving(self, sock): + # Is this a unix socket that needs cleanup? + if sock in self._unix_server_sockets: + path = sock.getsockname() + else: + path = None + + super()._stop_serving(sock) + + if path is not None: + prev_ino = self._unix_server_sockets[sock] + del self._unix_server_sockets[sock] + try: + if os.stat(path).st_ino == prev_ino: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as err: + logger.error('Unable to clean up listening UNIX socket ' + '%r: %r', path, err) + class _UnixReadPipeTransport(transports.ReadTransport): @@ -830,93 +854,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -928,21 +866,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -967,386 +890,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1363,18 +907,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1392,15 +924,6 @@ def add_child_handler(self, pid, callback, *args): self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1439,62 +962,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index cb613451a58..5f75b17d8ca 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -315,24 +315,25 @@ def __init__(self, proactor=None): proactor = IocpProactor() super().__init__(proactor) - def run_forever(self): - try: - assert self._self_reading_future is None - self.call_soon(self._loop_self_reading) - super().run_forever() - finally: - if self._self_reading_future is not None: - ov = self._self_reading_future._ov - self._self_reading_future.cancel() - # self_reading_future always uses IOCP, so even though it's - # been cancelled, we need to make sure that the IOCP message - # is received so that the kernel is not holding on to the - # memory, possibly causing memory corruption later. Only - # unregister it if IO is complete in all respects. Otherwise - # we need another _poll() later to complete the IO. - if ov is not None and not ov.pending: - self._proactor._unregister(ov) - self._self_reading_future = None + def _run_forever_setup(self): + assert self._self_reading_future is None + self.call_soon(self._loop_self_reading) + super()._run_forever_setup() + + def _run_forever_cleanup(self): + super()._run_forever_cleanup() + if self._self_reading_future is not None: + ov = self._self_reading_future._ov + self._self_reading_future.cancel() + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: + self._proactor._unregister(ov) + self._self_reading_future = None async def create_pipe_connection(self, protocol_factory, address): f = self._proactor.connect_pipe(address) @@ -890,12 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy +EventLoop = ProactorEventLoop diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index 5a7e790a193..f95132a4274 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,12 +1,9 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import re import struct import binascii @@ -286,7 +283,7 @@ def b16decode(s, casefold=False): s = _bytes_from_decode_data(s) if casefold: s = s.upper() - if re.search(b'[^0-9A-F]', s): + if s.translate(None, delete=b'0123456789ABCDEF'): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) @@ -465,9 +462,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 @@ -604,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index 0f3eec653ba..f256b56daaa 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -32,7 +33,12 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self._load_breaks() @@ -60,6 +66,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -84,24 +96,28 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -110,7 +126,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) if self.quitting: raise BdbQuit return self.trace_dispatch @@ -157,6 +178,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -186,6 +212,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -272,7 +309,22 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -285,6 +337,21 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -299,19 +366,17 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) + + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" @@ -328,11 +393,15 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -349,6 +418,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. @@ -387,6 +459,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return 'Line %s:%d does not exist' % (filename, lineno) self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back return None def _load_breaks(self): diff --git a/Lib/bz2.py b/Lib/bz2.py index 2420cd01906..eb58f4da596 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -10,9 +10,9 @@ __author__ = "Nadeem Vawda " from builtins import open as _builtin_open +from compression._common import _streams import io import os -import _compression from _bz2 import BZ2Compressor, BZ2Decompressor @@ -23,7 +23,7 @@ _MODE_WRITE = 3 -class BZ2File(_compression.BaseStream): +class BZ2File(_streams.BaseStream): """A file object providing transparent bzip2 (de)compression. @@ -88,7 +88,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: - raw = _compression.DecompressReader(self._fp, + raw = _streams.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: @@ -248,7 +248,7 @@ def writelines(self, seq): Line separators are not added between the written byte strings. """ - return _compression.BaseStream.writelines(self, seq) + return _streams.BaseStream.writelines(self, seq) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. diff --git a/Lib/calendar.py b/Lib/calendar.py index 8c1c646da46..18f76d52ff8 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -428,6 +428,7 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): @@ -646,6 +647,117 @@ def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): return super().formatmonthname(theyear, themonth, withyear) + +class _CLIDemoCalendar(TextCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + highlight_day = self.highlight_day.day + else: + highlight_day = None + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month in months + ): + month_pos = months.index(self.highlight_day.month) + else: + month_pos = None + + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for k, cal in enumerate(row): + if j >= len(cal): + weeks.append('') + else: + day = ( + self.highlight_day.day if k == month_pos else None + ) + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + +class _CLIDemoLocaleCalendar(LocaleTextCalendar, _CLIDemoCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + # Support for old module level interface c = TextCalendar() @@ -698,7 +810,7 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( @@ -765,6 +877,7 @@ def main(args=None): sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": if options.month: @@ -781,23 +894,23 @@ def main(args=None): optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: - cal = LocaleTextCalendar(locale=locale) + cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) else: - cal = TextCalendar() + cal = _CLIDemoCalendar(highlight_day=today) cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if options.month is not None: + else: _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + result = cal.formatyear(today.year, **optdict) elif options.month is None: result = cal.formatyear(options.year, **optdict) else: diff --git a/Lib/cmd.py b/Lib/cmd.py index a37d16cd7bd..51495fb3216 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,31 +23,34 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ -import inspect, string, sys +import sys __all__ = ["Cmd"] PROMPT = '(Cmd) ' -IDENTCHARS = string.ascii_letters + string.digits + '_' +IDENTCHARS = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' + 'abcdefghijklmnopqrstuvwxyz' + '0123456789' + '_') class Cmd: """A simple framework for writing line-oriented command interpreters. @@ -270,7 +273,7 @@ def complete(self, text, state): endidx = readline.get_endidx() - stripped if begidx>0: cmd, args, foo = self.parseline(line) - if cmd == '': + if not cmd: compfunc = self.completedefault else: try: @@ -303,9 +306,11 @@ def do_help(self, arg): try: func = getattr(self, 'help_' + arg) except AttributeError: + from inspect import cleandoc + try: doc=getattr(self, 'do_' + arg).__doc__ - doc = inspect.cleandoc(doc) + doc = cleandoc(doc) if doc: self.stdout.write("%s\n"%str(doc)) return diff --git a/Lib/code.py b/Lib/code.py index 2bd5fa3e795..b134886dc26 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -24,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -63,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -93,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -105,29 +106,14 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() - sys.last_exc = value - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_exc = sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and issubclass(typ, SyntaxError): + value.filename = filename + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) + finally: + typ = value = tb = None def showtraceback(self): """Display the exception that just occurred. @@ -137,19 +123,46 @@ def showtraceback(self): The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb - sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, "") finally: - last_tb = ei = None + typ = value = tb = None + + def _showtraceback(self, typ, value, tb, source): + sys.last_type = typ + sys.last_traceback = tb + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and value.lineno is not None + and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value + if sys.excepthook is sys.__excepthook__: + self._excepthook(typ, value, tb) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) + + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -169,7 +182,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename=""): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -181,6 +194,7 @@ def __init__(self, locals=None, filename=""): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -205,12 +219,17 @@ def interact(self, banner=None, exitmsg=None): """ try: sys.ps1 + delete_ps1_after = False except AttributeError: sys.ps1 = ">>> " + delete_ps1_after = True try: - sys.ps2 + _ps2 = sys.ps2 + delete_ps2_after = False except AttributeError: sys.ps2 = "... " + delete_ps2_after = True + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' if banner is None: self.write("Python %s on %s\n%s\n(%s)\n" % @@ -219,29 +238,72 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) - - def push(self, line): + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if delete_ps1_after: + del sys.ps1 + + if delete_ps2_after: + del sys.ps2 + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) + + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -257,7 +319,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -276,8 +340,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -290,14 +368,15 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) @@ -306,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e4f4e1b5c02..e4a8010aba9 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -884,7 +884,6 @@ def __reduce_ex__(self, proto): ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): - """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -912,8 +911,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): .encoding which allows querying the used encoding. This attribute is only available if an encoding was specified as parameter. - """ + import warnings + warnings.warn("codecs.open() is deprecated. Use open() instead.", + DeprecationWarning, stacklevel=2) + if encoding is not None and \ 'b' not in mode: # Force opening of the file in binary mode @@ -1109,24 +1111,15 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index adf000ba29f..8cac00442d9 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -47,7 +47,7 @@ PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 -def _maybe_compile(compiler, source, filename, symbol): +def _maybe_compile(compiler, source, filename, symbol, flags): # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() @@ -61,10 +61,10 @@ def _maybe_compile(compiler, source, filename, symbol): with warnings.catch_warnings(): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source, filename, symbol) + compiler(source, filename, symbol, flags=flags) except SyntaxError: # Let other compile() errors propagate. try: - compiler(source + "\n", filename, symbol) + compiler(source + "\n", filename, symbol, flags=flags) return None except _IncompleteInputError as e: return None @@ -74,14 +74,13 @@ def _maybe_compile(compiler, source, filename, symbol): return compiler(source, filename, symbol, incomplete_input=False) -def _compile(source, filename, symbol, incomplete_input=True): - flags = 0 +def _compile(source, filename, symbol, incomplete_input=True, *, flags=0): if incomplete_input: flags |= PyCF_ALLOW_INCOMPLETE_INPUT flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) -def compile_command(source, filename="", symbol="single"): +def compile_command(source, filename="", symbol="single", flags=0): r"""Compile a command and determine whether it is incomplete. Arguments: @@ -100,7 +99,7 @@ def compile_command(source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(_compile, source, filename, symbol) + return _maybe_compile(_compile, source, filename, symbol, flags) class Compile: """Instances of this class behave much like the built-in compile @@ -152,4 +151,4 @@ def __call__(self, source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(self.compiler, source, filename, symbol) + return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index f7348ee918d..3d3bbd7a39a 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -29,6 +29,9 @@ import _collections_abc import sys as _sys +_sys.modules['collections.abc'] = _collections_abc +abc = _collections_abc + from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -46,19 +49,19 @@ _collections_abc.MutableSequence.register(deque) try: - from _collections import _deque_iterator + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 except ImportError: pass try: from _collections import defaultdict except ImportError: - # FIXME: try to implement defaultdict in collections.rs rather than in Python - # I (coolreader18) couldn't figure out some class stuff with __new__ and - # __init__ and __missing__ and subclassing built-in types from Rust, so I went - # with this instead. + # TODO: RUSTPYTHON - implement defaultdict in Rust from ._defaultdict import defaultdict +heapq = None # Lazily imported + ################################################################################ ### OrderedDict @@ -461,7 +464,7 @@ def _make(cls, iterable): def _replace(self, /, **kwds): result = self._make(_map(kwds.pop, field_names, self)) if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') return result _replace.__doc__ = (f'Return a new {typename} object replacing specified ' @@ -499,6 +502,7 @@ def __getnewargs__(self): '_field_defaults': field_defaults, '__new__': __new__, '_make': _make, + '__replace__': _replace, '_replace': _replace, '__repr__': __repr__, '_asdict': _asdict, @@ -592,7 +596,7 @@ class Counter(dict): # References: # http://en.wikipedia.org/wiki/Multiset # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html - # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm + # http://www.java2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 @@ -632,7 +636,10 @@ def most_common(self, n=None): return sorted(self.items(), key=_itemgetter(1), reverse=True) # Lazy import to speedup Python startup time - import heapq + global heapq + if heapq is None: + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): @@ -642,7 +649,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) @@ -683,7 +691,7 @@ def update(self, iterable=None, /, **kwds): ''' # The regular dict.update() operation makes no sense here because the - # replace behavior results in the some of original untouched counts + # replace behavior results in some of the original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs @@ -1018,7 +1026,7 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible @@ -1030,7 +1038,10 @@ def __iter__(self): return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) @@ -1040,9 +1051,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' @@ -1485,6 +1496,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1553,6 +1566,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py deleted file mode 100644 index 86ca8b8a841..00000000000 --- a/Lib/collections/abc.py +++ /dev/null @@ -1,3 +0,0 @@ -from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias diff --git a/Lib/compression/__init__.py b/Lib/compression/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/compression/_common/__init__.py b/Lib/compression/_common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/_compression.py b/Lib/compression/_common/_streams.py similarity index 98% rename from Lib/_compression.py rename to Lib/compression/_common/_streams.py index e8b70aa0a3e..9f367d4e304 100644 --- a/Lib/_compression.py +++ b/Lib/compression/_common/_streams.py @@ -1,4 +1,4 @@ -"""Internal classes used by the gzip, lzma and bz2 modules""" +"""Internal classes used by compression modules""" import io import sys diff --git a/Lib/compression/bz2.py b/Lib/compression/bz2.py new file mode 100644 index 00000000000..16815d6cd20 --- /dev/null +++ b/Lib/compression/bz2.py @@ -0,0 +1,5 @@ +import bz2 +__doc__ = bz2.__doc__ +del bz2 + +from bz2 import * diff --git a/Lib/compression/gzip.py b/Lib/compression/gzip.py new file mode 100644 index 00000000000..552f48f948a --- /dev/null +++ b/Lib/compression/gzip.py @@ -0,0 +1,5 @@ +import gzip +__doc__ = gzip.__doc__ +del gzip + +from gzip import * diff --git a/Lib/compression/lzma.py b/Lib/compression/lzma.py new file mode 100644 index 00000000000..b4bc7ccb1db --- /dev/null +++ b/Lib/compression/lzma.py @@ -0,0 +1,5 @@ +import lzma +__doc__ = lzma.__doc__ +del lzma + +from lzma import * diff --git a/Lib/compression/zlib.py b/Lib/compression/zlib.py new file mode 100644 index 00000000000..3aa7e2db90e --- /dev/null +++ b/Lib/compression/zlib.py @@ -0,0 +1,5 @@ +import zlib +__doc__ = zlib.__doc__ +del zlib + +from zlib import * diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py new file mode 100644 index 00000000000..84b25914b0a --- /dev/null +++ b/Lib/compression/zstd/__init__.py @@ -0,0 +1,242 @@ +"""Python bindings to the Zstandard (zstd) compression library (RFC-8878).""" + +__all__ = ( + # compression.zstd + 'COMPRESSION_LEVEL_DEFAULT', + 'compress', + 'CompressionParameter', + 'decompress', + 'DecompressionParameter', + 'finalize_dict', + 'get_frame_info', + 'Strategy', + 'train_dict', + + # compression.zstd._zstdfile + 'open', + 'ZstdFile', + + # _zstd + 'get_frame_size', + 'zstd_version', + 'zstd_version_info', + 'ZstdCompressor', + 'ZstdDecompressor', + 'ZstdDict', + 'ZstdError', +) + +import _zstd +import enum +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, + get_frame_size, zstd_version) +from compression.zstd._zstdfile import ZstdFile, open, _nbytes + +# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) +zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), + _zstd.zstd_version_number % 100) +"""Version number of the runtime zstd library as a tuple of integers.""" + +COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT +"""The default compression level for Zstandard, currently '3'.""" + + +class FrameInfo: + """Information about a Zstandard frame.""" + + __slots__ = 'decompressed_size', 'dictionary_id' + + def __init__(self, decompressed_size, dictionary_id): + super().__setattr__('decompressed_size', decompressed_size) + super().__setattr__('dictionary_id', dictionary_id) + + def __repr__(self): + return (f'FrameInfo(decompressed_size={self.decompressed_size}, ' + f'dictionary_id={self.dictionary_id})') + + def __setattr__(self, name, _): + raise AttributeError(f"can't set attribute {name!r}") + + +def get_frame_info(frame_buffer): + """Get Zstandard frame information from a frame header. + + *frame_buffer* is a bytes-like object. It should start from the beginning + of a frame, and needs to include at least the frame header (6 to 18 bytes). + + The returned FrameInfo object has two attributes. + 'decompressed_size' is the size in bytes of the data in the frame when + decompressed, or None when the decompressed size is unknown. + 'dictionary_id' is an int in the range (0, 2**32). The special value 0 + means that the dictionary ID was not recorded in the frame header, + the frame may or may not need a dictionary to be decoded, + and the ID of such a dictionary is not specified. + """ + return FrameInfo(*_zstd.get_frame_info(frame_buffer)) + + +def train_dict(samples, dict_size): + """Return a ZstdDict representing a trained Zstandard dictionary. + + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. + + *dict_size* is the dictionary's maximum size, in bytes. + """ + if not isinstance(dict_size, int): + ds_cls = type(dict_size).__qualname__ + raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.') + + samples = tuple(samples) + chunks = b''.join(samples) + chunk_sizes = tuple(_nbytes(sample) for sample in samples) + if not chunks: + raise ValueError("samples contained no data; can't train dictionary.") + dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size) + return ZstdDict(dict_content) + + +def finalize_dict(zstd_dict, /, samples, dict_size, level): + """Return a ZstdDict representing a finalized Zstandard dictionary. + + Given a custom content as a basis for dictionary, and a set of samples, + finalize *zstd_dict* by adding headers and statistics according to the + Zstandard dictionary format. + + You may compose an effective dictionary content by hand, which is used as + basis dictionary, and use some samples to finalize a dictionary. The basis + dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + + *samples* is an iterable of samples, where a sample is a bytes-like object + representing a file. + *dict_size* is the dictionary's maximum size, in bytes. + *level* is the expected compression level. The statistics for each + compression level differ, so tuning the dictionary to the compression level + can provide improvements. + """ + + if not isinstance(zstd_dict, ZstdDict): + raise TypeError('zstd_dict argument should be a ZstdDict object.') + if not isinstance(dict_size, int): + raise TypeError('dict_size argument should be an int object.') + if not isinstance(level, int): + raise TypeError('level argument should be an int object.') + + samples = tuple(samples) + chunks = b''.join(samples) + chunk_sizes = tuple(_nbytes(sample) for sample in samples) + if not chunks: + raise ValueError("The samples are empty content, can't finalize the " + "dictionary.") + dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) + return ZstdDict(dict_content) + + +def compress(data, level=None, options=None, zstd_dict=None): + """Return Zstandard compressed *data* as bytes. + + *level* is an int specifying the compression level to use, defaulting to + COMPRESSION_LEVEL_DEFAULT ('3'). + *options* is a dict object that contains advanced compression + parameters. See CompressionParameter for more on options. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See + the function train_dict for how to train a ZstdDict on sample data. + + For incremental compression, use a ZstdCompressor instead. + """ + comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) + return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) + + +def decompress(data, zstd_dict=None, options=None): + """Decompress one or more frames of Zstandard compressed *data*. + + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See + the function train_dict for how to train a ZstdDict on sample data. + *options* is a dict object that contains advanced compression + parameters. See DecompressionParameter for more on options. + + For incremental decompression, use a ZstdDecompressor instead. + """ + results = [] + while True: + decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) + results.append(decomp.decompress(data)) + if not decomp.eof: + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') + data = decomp.unused_data + if not data: + break + return b''.join(results) + + +class CompressionParameter(enum.IntEnum): + """Compression parameters.""" + + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy + + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog + + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag + + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog + + def bounds(self): + """Return the (lower, upper) int bounds of a compression parameter. + + Both the lower and upper bounds are inclusive. + """ + return _zstd.get_param_bounds(self.value, is_compress=True) + + +class DecompressionParameter(enum.IntEnum): + """Decompression parameters.""" + + window_log_max = _zstd.ZSTD_d_windowLogMax + + def bounds(self): + """Return the (lower, upper) int bounds of a decompression parameter. + + Both the lower and upper bounds are inclusive. + """ + return _zstd.get_param_bounds(self.value, is_compress=False) + + +class Strategy(enum.IntEnum): + """Compression strategies, listed from fastest to strongest. + + Note that new strategies might be added in the future. + Only the order (from fast to strong) is guaranteed, + the numeric value might change. + """ + + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 + + +# Check validity of the CompressionParameter & DecompressionParameter types +_zstd.set_parameter_types(CompressionParameter, DecompressionParameter) diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py new file mode 100644 index 00000000000..d709f5efc65 --- /dev/null +++ b/Lib/compression/zstd/_zstdfile.py @@ -0,0 +1,345 @@ +import io +from os import PathLike +from _zstd import ZstdCompressor, ZstdDecompressor, ZSTD_DStreamOutSize +from compression._common import _streams + +__all__ = ('ZstdFile', 'open') + +_MODE_CLOSED = 0 +_MODE_READ = 1 +_MODE_WRITE = 2 + + +def _nbytes(dat, /): + if isinstance(dat, (bytes, bytearray)): + return len(dat) + with memoryview(dat) as mv: + return mv.nbytes + + +class ZstdFile(_streams.BaseStream): + """A file-like object providing transparent Zstandard (de)compression. + + A ZstdFile can act as a wrapper for an existing file object, or refer + directly to a named file on disk. + + ZstdFile provides a *binary* file interface. Data is read and returned as + bytes, and may only be written to objects that support the Buffer Protocol. + """ + + FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK + FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME + + def __init__(self, file, /, mode='r', *, + level=None, options=None, zstd_dict=None): + """Open a Zstandard compressed file in binary mode. + + *file* can be either an file-like object, or a file name to open. + + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for + creating exclusively, or 'a' for appending. These can equivalently be + given as 'rb', 'wb', 'xb' and 'ab' respectively. + + *level* is an optional int specifying the compression level to use, + or COMPRESSION_LEVEL_DEFAULT if not given. + + *options* is an optional dict for advanced compression parameters. + See CompressionParameter and DecompressionParameter for the possible + options. + + *zstd_dict* is an optional ZstdDict object, a pre-trained Zstandard + dictionary. See train_dict() to train ZstdDict on sample data. + """ + self._fp = None + self._close_fp = False + self._mode = _MODE_CLOSED + self._buffer = None + + if not isinstance(mode, str): + raise ValueError('mode must be a str') + if options is not None and not isinstance(options, dict): + raise TypeError('options must be a dict or None') + mode = mode.removesuffix('b') # handle rb, wb, xb, ab + if mode == 'r': + if level is not None: + raise TypeError('level is illegal in read mode') + self._mode = _MODE_READ + elif mode in {'w', 'a', 'x'}: + if level is not None and not isinstance(level, int): + raise TypeError('level must be int or None') + self._mode = _MODE_WRITE + self._compressor = ZstdCompressor(level=level, options=options, + zstd_dict=zstd_dict) + self._pos = 0 + else: + raise ValueError(f'Invalid mode: {mode!r}') + + if isinstance(file, (str, bytes, PathLike)): + self._fp = io.open(file, f'{mode}b') + self._close_fp = True + elif ((mode == 'r' and hasattr(file, 'read')) + or (mode != 'r' and hasattr(file, 'write'))): + self._fp = file + else: + raise TypeError('file must be a file-like object ' + 'or a str, bytes, or PathLike object') + + if self._mode == _MODE_READ: + raw = _streams.DecompressReader( + self._fp, + ZstdDecompressor, + zstd_dict=zstd_dict, + options=options, + ) + self._buffer = io.BufferedReader(raw) + + def close(self): + """Flush and close the file. + + May be called multiple times. Once the file has been closed, + any other operation on it will raise ValueError. + """ + if self._fp is None: + return + try: + if self._mode == _MODE_READ: + if getattr(self, '_buffer', None): + self._buffer.close() + self._buffer = None + elif self._mode == _MODE_WRITE: + self.flush(self.FLUSH_FRAME) + self._compressor = None + finally: + self._mode = _MODE_CLOSED + try: + if self._close_fp: + self._fp.close() + finally: + self._fp = None + self._close_fp = False + + def write(self, data, /): + """Write a bytes-like object *data* to the file. + + Returns the number of uncompressed bytes written, which is + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until .flush() + or .close() is called. + """ + self._check_can_write() + + length = _nbytes(data) + + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += length + return length + + def flush(self, mode=FLUSH_BLOCK): + """Flush remaining data to the underlying stream. + + The mode argument can be FLUSH_BLOCK or FLUSH_FRAME. Abuse of this + method will reduce compression ratio, use it only when necessary. + + If the program is interrupted afterwards, all data can be recovered. + To ensure saving to disk, also need to use os.fsync(fd). + + This method does nothing in reading mode. + """ + if self._mode == _MODE_READ: + return + self._check_not_closed() + if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}: + raise ValueError('Invalid mode argument, expected either ' + 'ZstdFile.FLUSH_FRAME or ' + 'ZstdFile.FLUSH_BLOCK') + if self._compressor.last_mode == mode: + return + # Flush zstd block/frame, and write. + data = self._compressor.flush(mode) + self._fp.write(data) + if hasattr(self._fp, 'flush'): + self._fp.flush() + + def read(self, size=-1): + """Read up to size uncompressed bytes from the file. + + If size is negative or omitted, read until EOF is reached. + Returns b'' if the file is already at EOF. + """ + if size is None: + size = -1 + self._check_can_read() + return self._buffer.read(size) + + def read1(self, size=-1): + """Read up to size uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns b'' if the file is at EOF. + """ + self._check_can_read() + if size < 0: + # Note this should *not* be io.DEFAULT_BUFFER_SIZE. + # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing + # a full block is read. + size = ZSTD_DStreamOutSize + return self._buffer.read1(size) + + def readinto(self, b): + """Read bytes into b. + + Returns the number of bytes read (0 for EOF). + """ + self._check_can_read() + return self._buffer.readinto(b) + + def readinto1(self, b): + """Read bytes into b, while trying to avoid making multiple reads + from the underlying stream. + + Returns the number of bytes read (0 for EOF). + """ + self._check_can_read() + return self._buffer.readinto1(b) + + def readline(self, size=-1): + """Read a line of uncompressed bytes from the file. + + The terminating newline (if present) is retained. If size is + non-negative, no more than size bytes will be read (in which + case the line may be incomplete). Returns b'' if already at EOF. + """ + self._check_can_read() + return self._buffer.readline(size) + + def seek(self, offset, whence=io.SEEK_SET): + """Change the file position. + + The new position is specified by offset, relative to the + position indicated by whence. Possible values for whence are: + + 0: start of stream (default): offset must not be negative + 1: current stream position + 2: end of stream; offset must not be positive + + Returns the new file position. + + Note that seeking is emulated, so depending on the arguments, + this operation may be extremely slow. + """ + self._check_can_read() + + # BufferedReader.seek() checks seekable + return self._buffer.seek(offset, whence) + + def peek(self, size=-1): + """Return buffered data without advancing the file position. + + Always returns at least one byte of data, unless at EOF. + The exact number of bytes returned is unspecified. + """ + # Relies on the undocumented fact that BufferedReader.peek() always + # returns at least one byte (except at EOF) + self._check_can_read() + return self._buffer.peek(size) + + def __next__(self): + if ret := self._buffer.readline(): + return ret + raise StopIteration + + def tell(self): + """Return the current file position.""" + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + elif self._mode == _MODE_WRITE: + return self._pos + + def fileno(self): + """Return the file descriptor for the underlying file.""" + self._check_not_closed() + return self._fp.fileno() + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' + + @property + def closed(self): + """True if this file is closed.""" + return self._mode == _MODE_CLOSED + + def seekable(self): + """Return whether the file supports seeking.""" + return self.readable() and self._buffer.seekable() + + def readable(self): + """Return whether the file was opened for reading.""" + self._check_not_closed() + return self._mode == _MODE_READ + + def writable(self): + """Return whether the file was opened for writing.""" + self._check_not_closed() + return self._mode == _MODE_WRITE + + +def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, + encoding=None, errors=None, newline=None): + """Open a Zstandard compressed file in binary or text mode. + + file can be either a file name (given as a str, bytes, or PathLike object), + in which case the named file is opened, or it can be an existing file object + to read from or write to. + + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', + 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + + The level, options, and zstd_dict parameters specify the settings the same + as ZstdFile. + + When using read mode (decompression), the options parameter is a dict + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one of + level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both modes, + zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. + + For binary mode, this function is equivalent to the ZstdFile constructor: + ZstdFile(filename, mode, ...). In this case, the encoding, errors and + newline parameters must not be provided. + + For text mode, an ZstdFile object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error handling + behavior, and line ending(s). + """ + + text_mode = 't' in mode + mode = mode.replace('t', '') + + if text_mode: + if 'b' in mode: + raise ValueError(f'Invalid mode: {mode!r}') + else: + if encoding is not None: + raise ValueError('Argument "encoding" not supported in binary mode') + if errors is not None: + raise ValueError('Argument "errors" not supported in binary mode') + if newline is not None: + raise ValueError('Argument "newline" not supported in binary mode') + + binary_file = ZstdFile(file, mode, level=level, options=options, + zstd_dict=zstd_dict) + + if text_mode: + return io.TextIOWrapper(binary_file, encoding, errors, newline) + else: + return binary_file diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index d746aeac50a..72de617a5b6 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,6 +23,7 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', @@ -50,4 +51,4 @@ def __getattr__(name): ThreadPoolExecutor = te return te - raise AttributeError(f"module {__name__} has no attribute {name}") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index cf119ac6437..7d69a5baead 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -50,9 +50,7 @@ class CancelledError(Error): """The Future was cancelled.""" pass -class TimeoutError(Error): - """The operation exceeded the given deadline.""" - pass +TimeoutError = TimeoutError # make local alias for the standard exception class InvalidStateError(Error): """The operation is not allowed in this state.""" @@ -284,7 +282,7 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): A named 2-tuple of sets. The first set, named 'done', contains the futures that completed (is finished or cancelled) before the wait completed. The second set, named 'not_done', contains uncompleted - futures. Duplicate futures given to *fs* are removed and will be + futures. Duplicate futures given to *fs* are removed and will be returned only once. """ fs = set(fs) @@ -312,6 +310,18 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): done.update(waiter.finished_futures) return DoneAndNotDoneFutures(done, fs - done) + +def _result_or_cancel(fut, timeout=None): + try: + try: + return fut.result(timeout) + finally: + fut.cancel() + finally: + # Break a reference cycle with the exception in self._exception + del fut + + class Future(object): """Represents the result of an asynchronous computation.""" @@ -386,7 +396,7 @@ def done(self): return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED] def __get_result(self): - if self._exception: + if self._exception is not None: try: raise self._exception finally: @@ -606,9 +616,9 @@ def result_iterator(): while fs: # Careful not to keep a reference to the popped future if timeout is None: - yield fs.pop().result() + yield _result_or_cancel(fs.pop()) else: - yield fs.pop().result(end_time - time.monotonic()) + yield _result_or_cancel(fs.pop(), end_time - time.monotonic()) finally: for future in fs: future.cancel() diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 57941e485d8..0dee8303ba2 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -49,6 +49,8 @@ from concurrent.futures import _base import queue import multiprocessing as mp +# This import is required to load the multiprocessing.connection submodule +# so that it can be accessed later as `mp.connection` import multiprocessing.connection from multiprocessing.queues import Queue import threading @@ -56,7 +58,7 @@ from functools import partial import itertools import sys -import traceback +from traceback import format_exception _threads_wakeups = weakref.WeakKeyDictionary() @@ -66,22 +68,31 @@ class _ThreadWakeup: def __init__(self): self._closed = False + self._lock = threading.Lock() self._reader, self._writer = mp.Pipe(duplex=False) def close(self): - if not self._closed: - self._closed = True - self._writer.close() - self._reader.close() + # Please note that we do not take the self._lock when + # calling clear() (to avoid deadlocking) so this method can + # only be called safely from the same thread as all calls to + # clear() even if you hold the lock. Otherwise we + # might try to read from the closed pipe. + with self._lock: + if not self._closed: + self._closed = True + self._writer.close() + self._reader.close() def wakeup(self): - if not self._closed: - self._writer.send_bytes(b"") + with self._lock: + if not self._closed: + self._writer.send_bytes(b"") def clear(self): - if not self._closed: - while self._reader.poll(): - self._reader.recv_bytes() + if self._closed: + raise RuntimeError('operation on closed _ThreadWakeup') + while self._reader.poll(): + self._reader.recv_bytes() def _python_exit(): @@ -123,8 +134,7 @@ def __str__(self): class _ExceptionWithTraceback: def __init__(self, exc, tb): - tb = traceback.format_exception(type(exc), exc, tb) - tb = ''.join(tb) + tb = ''.join(format_exception(type(exc), exc, tb)) self.exc = exc # Traceback object needs to be garbage-collected as its frames # contain references to all the objects in the exception scope @@ -145,10 +155,11 @@ def __init__(self, future, fn, args, kwargs): self.kwargs = kwargs class _ResultItem(object): - def __init__(self, work_id, exception=None, result=None): + def __init__(self, work_id, exception=None, result=None, exit_pid=None): self.work_id = work_id self.exception = exception self.result = result + self.exit_pid = exit_pid class _CallItem(object): def __init__(self, work_id, fn, args, kwargs): @@ -160,20 +171,17 @@ def __init__(self, work_id, fn, args, kwargs): class _SafeQueue(Queue): """Safe Queue set exception to the future object linked to a job""" - def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock, - thread_wakeup): + def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup): self.pending_work_items = pending_work_items - self.shutdown_lock = shutdown_lock self.thread_wakeup = thread_wakeup super().__init__(max_size, ctx=ctx) def _on_queue_feeder_error(self, e, obj): if isinstance(obj, _CallItem): - tb = traceback.format_exception(type(e), e, e.__traceback__) + tb = format_exception(type(e), e, e.__traceback__) e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) work_item = self.pending_work_items.pop(obj.work_id, None) - with self.shutdown_lock: - self.thread_wakeup.wakeup() + self.thread_wakeup.wakeup() # work_item can be None if another process terminated. In this # case, the executor_manager_thread fails all work_items # with BrokenProcessPool @@ -183,16 +191,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -205,17 +203,19 @@ def _process_chunk(fn, chunk): return [fn(*args) for args in chunk] -def _sendback_result(result_queue, work_id, result=None, exception=None): +def _sendback_result(result_queue, work_id, result=None, exception=None, + exit_pid=None): """Safely send back the given result or exception""" try: result_queue.put(_ResultItem(work_id, result=result, - exception=exception)) + exception=exception, exit_pid=exit_pid)) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(work_id, exception=exc)) + result_queue.put(_ResultItem(work_id, exception=exc, + exit_pid=exit_pid)) -def _process_worker(call_queue, result_queue, initializer, initargs): +def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. @@ -236,25 +236,38 @@ def _process_worker(call_queue, result_queue, initializer, initargs): # The parent will notice that the process stopped and # mark the pool broken return + num_tasks = 0 + exit_pid = None while True: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(os.getpid()) return + + if max_tasks is not None: + num_tasks += 1 + if num_tasks >= max_tasks: + exit_pid = os.getpid() + try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - _sendback_result(result_queue, call_item.work_id, exception=exc) + _sendback_result(result_queue, call_item.work_id, exception=exc, + exit_pid=exit_pid) else: - _sendback_result(result_queue, call_item.work_id, result=r) + _sendback_result(result_queue, call_item.work_id, result=r, + exit_pid=exit_pid) del r # Liberate the resource as soon as possible, to avoid holding onto # open files or shared memory that is not needed anymore del call_item + if exit_pid is not None: + return + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -284,11 +297,10 @@ def __init__(self, executor): # if there is no pending work item. def weakref_cb(_, thread_wakeup=self.thread_wakeup, - shutdown_lock=self.shutdown_lock): - mp.util.debug('Executor collected: triggering callback for' + mp_util_debug=mp.util.debug): + mp_util_debug('Executor collected: triggering callback for' ' QueueManager wakeup') - with shutdown_lock: - thread_wakeup.wakeup() + thread_wakeup.wakeup() self.executor_reference = weakref.ref(executor, weakref_cb) @@ -305,6 +317,10 @@ def weakref_cb(_, # A queue.Queue of work ids e.g. Queue([5, 6, ...]). self.work_ids_queue = executor._work_ids + # Maximum number of tasks a worker process can execute before + # exiting safely + self.max_tasks_per_child = executor._max_tasks_per_child + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -315,7 +331,14 @@ def run(self): # Main loop for the executor manager thread. while True: - self.add_call_item_to_queue() + # gh-109047: During Python finalization, self.call_queue.put() + # creation of a thread can fail with RuntimeError. + try: + self.add_call_item_to_queue() + except BaseException as exc: + cause = format_exception(exc) + self.terminate_broken(cause) + return result_item, is_broken, cause = self.wait_result_broken_or_wakeup() @@ -324,19 +347,32 @@ def run(self): return if result_item is not None: self.process_result_item(result_item) + + process_exited = result_item.exit_pid is not None + if process_exited: + p = self.processes.pop(result_item.exit_pid) + p.join() + # Delete reference to result_item to avoid keeping references # while waiting on new results. del result_item - # attempt to increment idle process count - executor = self.executor_reference() - if executor is not None: - executor._idle_worker_semaphore.release() - del executor + if executor := self.executor_reference(): + if process_exited: + with self.shutdown_lock: + executor._adjust_process_count() + else: + executor._idle_worker_semaphore.release() + del executor if self.is_shutting_down(): self.flag_executor_shutting_down() + # When only canceled futures remain in pending_work_items, our + # next call to wait_result_broken_or_wakeup would hang forever. + # This makes sure we have some running futures or none at all. + self.add_call_item_to_queue() + # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not self.pending_work_items: @@ -386,14 +422,13 @@ def wait_result_broken_or_wakeup(self): try: result_item = result_reader.recv() is_broken = False - except BaseException as e: - cause = traceback.format_exception(type(e), e, e.__traceback__) + except BaseException as exc: + cause = format_exception(exc) elif wakeup_reader in ready: is_broken = False - with self.shutdown_lock: - self.thread_wakeup.clear() + self.thread_wakeup.clear() return result_item, is_broken, cause @@ -401,24 +436,14 @@ def process_result_item(self, result_item): # Process the received a result_item. This can be either the PID of a # worker that exited gracefully or a _ResultItem - if isinstance(result_item, int): - # Clean shutdown of a worker using its PID - # (avoids marking the executor broken) - assert self.is_shutting_down() - p = self.processes.pop(result_item) - p.join() - if not self.processes: - self.join_executor_internals() - return - else: - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(result_item.work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - if result_item.exception: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception is not None: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) def is_shutting_down(self): # Check whether we should start shutting down the executor. @@ -430,7 +455,7 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def terminate_broken(self, cause): + def _terminate_broken(self, cause): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that # lead the executor into becoming broken. @@ -455,7 +480,14 @@ def terminate_broken(self, cause): # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - work_item.future.set_exception(bpe) + try: + work_item.future.set_exception(bpe) + except _base.InvalidStateError: + # set_exception() fails if the future is cancelled: ignore it. + # Trying to check if the future is cancelled before calling + # set_exception() would leave a race condition if the future is + # cancelled between the check and set_exception(). + pass # Delete references to object. See issue16284 del work_item self.pending_work_items.clear() @@ -465,8 +497,14 @@ def terminate_broken(self, cause): for p in self.processes.values(): p.terminate() + self.call_queue._terminate_broken() + # clean up resources - self.join_executor_internals() + self._join_executor_internals(broken=True) + + def terminate_broken(self, cause): + with self.shutdown_lock: + self._terminate_broken(cause) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -509,15 +547,24 @@ def shutdown_workers(self): break def join_executor_internals(self): - self.shutdown_workers() + with self.shutdown_lock: + self._join_executor_internals() + + def _join_executor_internals(self, broken=False): + # If broken, call_queue was closed and so can no longer be used. + if not broken: + self.shutdown_workers() + # Release the queue's resources as soon as possible. self.call_queue.close() self.call_queue.join_thread() - with self.shutdown_lock: - self.thread_wakeup.close() + self.thread_wakeup.close() + # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in self.processes.values(): + if broken: + p.terminate() p.join() def get_n_children_alive(self): @@ -582,22 +629,29 @@ class BrokenProcessPool(_base.BrokenExecutor): class ProcessPoolExecutor(_base.Executor): def __init__(self, max_workers=None, mp_context=None, - initializer=None, initargs=()): + initializer=None, initargs=(), *, max_tasks_per_child=None): """Initializes a new ProcessPoolExecutor instance. Args: max_workers: The maximum number of processes that can be used to execute the given calls. If None or not given then as many worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers. This + mp_context: A multiprocessing context to launch the workers created + using the multiprocessing.get_context('start method') API. This object should provide SimpleQueue, Queue and Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. + max_tasks_per_child: The maximum number of tasks a worker process + can complete before it will exit and be replaced with a fresh + worker process. The default of None means worker process will + live as long as the executor. Requires a non-'fork' mp_context + start method. When given, we default to using 'spawn' if no + mp_context is supplied. """ _check_system_limits() if max_workers is None: - self._max_workers = os.cpu_count() or 1 + self._max_workers = os.process_cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) @@ -612,7 +666,10 @@ def __init__(self, max_workers=None, mp_context=None, self._max_workers = max_workers if mp_context is None: - mp_context = mp.get_context() + if max_tasks_per_child is not None: + mp_context = mp.get_context("spawn") + else: + mp_context = mp.get_context() self._mp_context = mp_context # https://github.com/python/cpython/issues/90622 @@ -624,6 +681,18 @@ def __init__(self, max_workers=None, mp_context=None, self._initializer = initializer self._initargs = initargs + if max_tasks_per_child is not None: + if not isinstance(max_tasks_per_child, int): + raise TypeError("max_tasks_per_child must be an integer") + elif max_tasks_per_child <= 0: + raise ValueError("max_tasks_per_child must be >= 1") + if self._mp_context.get_start_method(allow_none=False) == "fork": + # https://github.com/python/cpython/issues/90622 + raise ValueError("max_tasks_per_child is incompatible with" + " the 'fork' multiprocessing start method;" + " supply a different mp_context.") + self._max_tasks_per_child = max_tasks_per_child + # Management thread self._executor_manager_thread = None @@ -646,7 +715,9 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup. + # Care must be taken to only call clear and close from the + # executor_manager_thread, since _ThreadWakeup.clear() is not protected + # by a lock. self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor @@ -657,7 +728,6 @@ def __init__(self, max_workers=None, mp_context=None, self._call_queue = _SafeQueue( max_size=queue_size, ctx=self._mp_context, pending_work_items=self._pending_work_items, - shutdown_lock=self._shutdown_lock, thread_wakeup=self._executor_manager_thread_wakeup) # Killed worker processes can produce spurious "broken pipe" # tracebacks in the queue's own worker thread. But we detect killed @@ -677,6 +747,11 @@ def _start_executor_manager_thread(self): self._executor_manager_thread_wakeup def _adjust_process_count(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return @@ -705,7 +780,8 @@ def _spawn_process(self): args=(self._call_queue, self._result_queue, self._initializer, - self._initargs)) + self._initargs, + self._max_tasks_per_child)) p.start() self._processes[p.pid] = p @@ -759,7 +835,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 493861d314d..9021dde48ef 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -37,14 +37,14 @@ def _python_exit(): threading._register_atexit(_python_exit) # At fork, reinitialize the `_global_shutdown_lock` lock in the child process -# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet -if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'): +if hasattr(os, 'register_at_fork'): os.register_at_fork(before=_global_shutdown_lock.acquire, after_in_child=_global_shutdown_lock._at_fork_reinit, after_in_parent=_global_shutdown_lock.release) + os.register_at_fork(after_in_child=_threads_queues.clear) -class _WorkItem(object): +class _WorkItem: def __init__(self, future, fn, args, kwargs): self.future = future self.fn = fn @@ -79,17 +79,20 @@ def _worker(executor_reference, work_queue, initializer, initargs): return try: while True: - work_item = work_queue.get(block=True) - if work_item is not None: - work_item.run() - # Delete references to object. See issue16284 - del work_item - - # attempt to increment idle count + try: + work_item = work_queue.get_nowait() + except queue.Empty: + # attempt to increment idle count if queue is empty executor = executor_reference() if executor is not None: executor._idle_semaphore.release() del executor + work_item = work_queue.get(block=True) + + if work_item is not None: + work_item.run() + # Delete references to object. See GH-60488 + del work_item continue executor = executor_reference() @@ -137,10 +140,10 @@ def __init__(self, max_workers=None, thread_name_prefix='', # * CPU bound task which releases GIL # * I/O bound task (which releases GIL, of course) # - # We use cpu_count + 4 for both types of tasks. + # We use process_cpu_count + 4 for both types of tasks. # But we limit it to 32 to avoid consuming surprisingly large resource # on many core machine. - max_workers = min(32, (os.cpu_count() or 1) + 4) + max_workers = min(32, (os.process_cpu_count() or 1) + 4) if max_workers <= 0: raise ValueError("max_workers must be greater than 0") diff --git a/Lib/configparser.py b/Lib/configparser.py index 05b86acb919..d435a5c2fe0 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -154,14 +154,13 @@ import os import re import sys -import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "MultilineContinuationError", - "ConfigParser", "RawConfigParser", + "MultilineContinuationError", "UnnamedSectionDisabledError", + "InvalidWriteError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") @@ -362,11 +361,27 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + class _UnnamedSection: def __repr__(self): return "" +class InvalidWriteError(Error): + """Raised when attempting to write data that the parser would read back differently. + ex: writing a key which begins with the section header pattern would read back as a + new section """ + + def __init__(self, msg=''): + Error.__init__(self, msg) + UNNAMED_SECTION = _UnnamedSection() @@ -556,35 +571,36 @@ def __init__(self): class _Line(str): + __slots__ = 'clean', 'has_comments' def __new__(cls, val, *args, **kwargs): return super().__new__(cls, val) - def __init__(self, val, prefixes): - self.prefixes = prefixes - - @functools.cached_property - def clean(self): - return self._strip_full() and self._strip_inline() + def __init__(self, val, comments): + trimmed = val.strip() + self.clean = comments.strip(trimmed) + self.has_comments = trimmed != self.clean - @property - def has_comments(self): - return self.strip() != self.clean - def _strip_inline(self): - """ - Search for the earliest prefix at the beginning of the line or following a space. - """ - matcher = re.compile( - '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) - # match nothing if no prefixes - or '(?!)' +class _CommentSpec: + def __init__(self, full_prefixes, inline_prefixes): + full_patterns = ( + # prefix at the beginning of a line + fr'^({re.escape(prefix)}).*' + for prefix in full_prefixes ) - match = matcher.search(self) - return self[:match.start() if match else None].strip() + inline_patterns = ( + # prefix at the beginning of the line or following a space + fr'(^|\s)({re.escape(prefix)}.*)' + for prefix in inline_prefixes + ) + self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) + + def strip(self, text): + return self.pattern.sub('', text).rstrip() - def _strip_full(self): - return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + def wrap(self, text): + return _Line(text, self) class RawConfigParser(MutableMapping): @@ -653,10 +669,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._prefixes = types.SimpleNamespace( - full=tuple(comment_prefixes or ()), - inline=tuple(inline_comment_prefixes or ()), - ) + self._comments = _CommentSpec(comment_prefixes or (), inline_comment_prefixes or ()) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -694,6 +707,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -777,7 +794,8 @@ def read_dict(self, dictionary, source=''): """ elements_added = set() for section, keys in dictionary.items(): - section = str(section) + if section is not UNNAMED_SECTION: + section = str(section) try: self.add_section(section) except (DuplicateSectionError, ValueError): @@ -949,7 +967,7 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) - if UNNAMED_SECTION in self._sections: + if UNNAMED_SECTION in self._sections and self._sections[UNNAMED_SECTION]: self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) for section in self._sections: @@ -959,10 +977,11 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified 'fp'.""" if not unnamed: fp.write("[{}]\n".format(section_name)) for key, value in section_items: + self._validate_key_contents(key) value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: @@ -1047,7 +1066,6 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - try: ParsingError._raise_all(self._read_inner(fp, fpname)) finally: @@ -1056,8 +1074,7 @@ def _read(self, fp, fpname): def _read_inner(self, fp, fpname): st = _ReadState() - Line = functools.partial(_Line, prefixes=self._prefixes) - for st.lineno, line in enumerate(map(Line, fp), start=1): + for st.lineno, line in enumerate(map(self._comments.wrap, fp), start=1): if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no @@ -1200,21 +1217,32 @@ def _convert_to_boolean(self, value): raise ValueError('Not a boolean: %s' % value) return self.BOOLEAN_STATES[value.lower()] + def _validate_key_contents(self, key): + """Raises an InvalidWriteError for any keys containing + delimiters or that begins with the section header pattern""" + if re.match(self.SECTCRE, key): + raise InvalidWriteError( + f"Cannot write key {key}; begins with section pattern") + for delim in self._delimiters: + if delim in key: + raise InvalidWriteError( + f"Cannot write key {key}; contains delimiter {delim}") + def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/contextvars.py b/Lib/contextvars.py index d78c80dfe6f..14514f185e0 100644 --- a/Lib/contextvars.py +++ b/Lib/contextvars.py @@ -1,4 +1,8 @@ +import _collections_abc from _contextvars import Context, ContextVar, Token, copy_context __all__ = ('Context', 'ContextVar', 'Token', 'copy_context') + + +_collections_abc.Mapping.register(Context) diff --git a/Lib/copy.py b/Lib/copy.py index 2a4606246aa..c64fc076179 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -67,13 +67,15 @@ def copy(x): cls = type(x) - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) + if cls in _copy_atomic_types: + return x + if cls in _copy_builtin_containers: + return cls.copy(x) + if issubclass(cls, type): # treat it as a regular class: - return _copy_immutable(x) + return x copier = getattr(cls, "__copy__", None) if copier is not None: @@ -98,23 +100,12 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_dispatch = d = {} - -def _copy_immutable(x): - return x -for t in (types.NoneType, int, float, bool, complex, str, tuple, +_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref): - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -del d, t + weakref.ref, super} +_copy_builtin_containers = {list, dict, set, bytearray} def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -122,6 +113,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -130,14 +126,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -168,26 +162,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Lib/copyreg.py b/Lib/copyreg.py index 578392409b4..a5e8add4a55 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -31,11 +31,16 @@ def pickle_complex(c): pickle(complex, pickle_complex, complex) def pickle_union(obj): - import functools, operator - return functools.reduce, (operator.or_, obj.__args__) + import typing, operator + return operator.getitem, (typing.Union, obj.__args__) pickle(type(int | str), pickle_union) +def pickle_super(obj): + return super, (obj.__thisclass__, obj.__self__) + +pickle(super, pickle_super) + # Support for pickling new-style objects def _reconstructor(cls, base, state): diff --git a/Lib/csv.py b/Lib/csv.py index cd202659873..0a627ba7a51 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -63,7 +63,6 @@ class excel: written as two quotes """ -import re import types from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ @@ -281,6 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): If there is no quotechar the delimiter can't be determined this way. """ + import re matches = [] for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 3599e13ed28..04ec0270148 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -1,6 +1,8 @@ """create and manipulate C data types in Python""" -import os as _os, sys as _sys +import os as _os +import sys as _sys +import sysconfig as _sysconfig import types as _types __version__ = "1.1.0" @@ -12,6 +14,7 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import ArgumentError from _ctypes import SIZEOF_TIME_T +from _ctypes import CField from struct import calcsize as _calcsize @@ -19,7 +22,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -36,9 +39,6 @@ FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR -# TODO: RUSTPYTHON remove this -from _ctypes import _non_existing_function - # WINOLEAPI -> HRESULT # WINOLEAPI_(type) # @@ -110,7 +110,7 @@ class CFunctionType(_CFuncPtr): return CFunctionType if _os.name == "nt": - from _ctypes import LoadLibrary as _dlopen + from _ctypes import LoadLibrary as _LoadLibrary from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL _win_functype_cache = {} @@ -164,6 +164,7 @@ def __repr__(self): return super().__repr__() except ValueError: return "%s()" % type(self).__name__ + __class_getitem__ = classmethod(_types.GenericAlias) _check_size(py_object, "P") class c_short(_SimpleCData): @@ -208,6 +209,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "D" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "F" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "G" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -255,7 +268,72 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" -from _ctypes import POINTER, pointer, _pointer_type_cache +def POINTER(cls): + """Create and return a new ctypes pointer type. + + Pointer types are cached and reused internally, + so calling this function repeatedly is cheap. + """ + if cls is None: + return c_void_p + try: + return cls.__pointer_type__ + except AttributeError: + pass + if isinstance(cls, str): + # handle old-style incomplete types (see test_ctypes.test_incomplete) + import warnings + warnings._deprecated("ctypes.POINTER with string", remove=(3, 19)) + try: + return _pointer_type_cache_fallback[cls] + except KeyError: + result = type(f'LP_{cls}', (_Pointer,), {}) + _pointer_type_cache_fallback[cls] = result + return result + + # create pointer type and set __pointer_type__ for cls + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + +def pointer(obj): + """Create a new pointer instance, pointing to 'obj'. + + The returned object is of the type POINTER(type(obj)). Note that if you + just want to pass a pointer to an object to a foreign function call, you + should use byref(obj) which is much faster. + """ + typ = POINTER(type(obj)) + return typ(obj) + +class _PointerTypeCache: + def __setitem__(self, cls, pointer_type): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + cls.__pointer_type__ = pointer_type + except AttributeError: + _pointer_type_cache_fallback[cls] = pointer_type + + def __getitem__(self, cls): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback[cls] + + def get(self, cls, default=None): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback.get(cls, default) + + def __contains__(self, cls): + return hasattr(cls, '__pointer_type__') + +_pointer_type_cache_fallback = {} +_pointer_type_cache = _PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" @@ -266,7 +344,7 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): - _pointer_type_cache.clear() + _pointer_type_cache_fallback.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() @@ -274,7 +352,6 @@ def _reset_cache(): POINTER(c_wchar).from_param = c_wchar_p.from_param # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param - _pointer_type_cache[None] = c_void_p def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array @@ -305,17 +382,11 @@ def create_unicode_buffer(init, size=None): raise TypeError(init) -# XXX Deprecated def SetPointerType(pointer, cls): - if _pointer_type_cache.get(cls, None) is not None: - raise RuntimeError("This type already exists in the cache") - if id(pointer) not in _pointer_type_cache: - raise RuntimeError("What's this???") + import warnings + warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) pointer.set_type(cls) - _pointer_type_cache[cls] = pointer - del _pointer_type_cache[id(pointer)] -# XXX Deprecated def ARRAY(typ, len): return typ * len @@ -347,52 +418,59 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None): + class _FuncPtr(_CFuncPtr): + _flags_ = self._func_flags_ + _restype_ = self._func_restype_ + if use_errno: + _flags_ |= _FUNCFLAG_USE_ERRNO + if use_last_error: + _flags_ |= _FUNCFLAG_USE_LASTERROR + + self._FuncPtr = _FuncPtr if name: name = _os.fspath(name) + self._handle = self._load_library(name, mode, handle, winmode) + + if _os.name == "nt": + def _load_library(self, name, mode, handle, winmode): + if winmode is None: + import nt as _nt + winmode = _nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS + # WINAPI LoadLibrary searches for a DLL if the given name + # is not fully qualified with an explicit drive. For POSIX + # compatibility, and because the DLL search path no longer + # contains the working directory, begin by fully resolving + # any name that contains a path separator. + if name is not None and ('/' in name or '\\' in name): + name = _nt._getfullpathname(name) + winmode |= _nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + self._name = name + if handle is not None: + return handle + return _LoadLibrary(self._name, winmode) + + else: + def _load_library(self, name, mode, handle, winmode): # If the filename that has been provided is an iOS/tvOS/watchOS # .fwork file, dereference the location to the true origin of the # binary. - if name.endswith(".fwork"): + if name and name.endswith(".fwork"): with open(name) as f: name = _os.path.join( _os.path.dirname(_sys.executable), f.read().strip() ) - - self._name = name - flags = self._func_flags_ - if use_errno: - flags |= _FUNCFLAG_USE_ERRNO - if use_last_error: - flags |= _FUNCFLAG_USE_LASTERROR - if _sys.platform.startswith("aix"): - """When the name contains ".a(" and ends with ")", - e.g., "libFOO.a(libFOO.so)" - this is taken to be an - archive(member) syntax for dlopen(), and the mode is adjusted. - Otherwise, name is presented to dlopen() as a file argument. - """ - if name and name.endswith(")") and ".a(" in name: - mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) - if _os.name == "nt": - if winmode is not None: - mode = winmode - else: - import nt - mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS - if '/' in name or '\\' in name: - self._name = nt._getfullpathname(self._name) - mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR - - class _FuncPtr(_CFuncPtr): - _flags_ = flags - _restype_ = self._func_restype_ - self._FuncPtr = _FuncPtr - - if handle is None: - self._handle = _dlopen(self._name, mode) - else: - self._handle = handle + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW + self._name = name + return _dlopen(name, mode) def __repr__(self): return "<%s '%s', handle %x at %#x>" % \ @@ -480,10 +558,9 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "android": - pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) +elif _sys.platform in ["android", "cygwin"]: + # These are Unix-like platforms which use a dynamically-linked libpython. + pythonapi = PyDLL(_sysconfig.get_config_var("LDLIBRARY")) else: pythonapi = PyDLL(None) @@ -515,6 +592,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -540,6 +618,14 @@ def string_at(ptr, size=-1): Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index 34dee64b1a6..6382dd22b8a 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -1,5 +1,5 @@ import sys -from ctypes import * +from ctypes import Array, Structure, Union _array_type = type(Array) @@ -15,8 +15,8 @@ def _other_endian(typ): # if typ is array if isinstance(typ, _array_type): return _other_endian(typ._type_) * typ._length_ - # if typ is structure - if issubclass(typ, Structure): + # if typ is structure or union + if issubclass(typ, (Structure, Union)): return typ raise TypeError("This type does not support other endian: %s" % typ) @@ -37,7 +37,7 @@ class _swapped_union_meta(_swapped_meta, type(Union)): pass ################################################################ # Note: The Structure metaclass checks for the *presence* (not the -# value!) of a _swapped_bytes_ attribute to determine the bit order in +# value!) of a _swappedbytes_ attribute to determine the bit order in # structures containing bit fields. if sys.byteorder == "little": diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000..2048ccb6a1c --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,330 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + pack = getattr(cls, '_pack_', None) + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32': + gcc_layout = False + elif pack: + if is_struct: + base_type_name = 'Structure' + else: + base_type_name = 'Union' + warnings._deprecated( + '_pack_ without _layout_', + f"Due to '_pack_', the '{cls.__name__}' {base_type_name} will " + + "use memory layout compatible with MSVC (Windows). " + + "If this is intended, set _layout_ to 'ms'. " + + "The implicit default is deprecated and slated to become " + + "an error in Python {remove}.", + remove=(3, 19), + ) + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align = 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + bit_offset = next_bit_offset - 8 * offset + assert bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + bit_offset = last_field_bit_size + next_bit_offset + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + if is_bitfield and big_endian: + # On big-endian architectures, bit fields are also laid out + # starting with the big end. + bit_offset = type_bit_size - bit_size - bit_offset + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + f"field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + byte_size=type_size, + byte_offset=offset, + bit_size=bit_size if is_bitfield else None, + bit_offset=bit_offset if is_bitfield else None, + index=i, + + # Do not use CField outside ctypes, yet. + # The constructor is internal API and may change without warning. + _internal_use=True, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py deleted file mode 100644 index 6e496fa5a52..00000000000 --- a/Lib/ctypes/test/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import unittest -from test import support -from test.support import import_helper - - -# skip tests if _ctypes was not built -ctypes = import_helper.import_module('ctypes') -ctypes_symbols = dir(ctypes) - -def need_symbol(name): - return unittest.skipUnless(name in ctypes_symbols, - '{!r} is required'.format(name)) - -def load_tests(*args): - return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/ctypes/test/__main__.py b/Lib/ctypes/test/__main__.py deleted file mode 100644 index 362a9ec8cff..00000000000 --- a/Lib/ctypes/test/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from ctypes.test import load_tests -import unittest - -unittest.main() diff --git a/Lib/ctypes/test/test_bitfields.py b/Lib/ctypes/test/test_bitfields.py deleted file mode 100644 index 66acd62e685..00000000000 --- a/Lib/ctypes/test/test_bitfields.py +++ /dev/null @@ -1,297 +0,0 @@ -from ctypes import * -from ctypes.test import need_symbol -from test import support -import unittest -import os - -import _ctypes_test - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - -func = CDLL(_ctypes_test.__file__).unpack_bitfields -func.argtypes = POINTER(BITS), c_char - -##for n in "ABCDEFGHIMNOPQRS": -## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset - -class C_Test(unittest.TestCase): - - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) - - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) - def test_shorts(self): - b = BITS() - name = "M" - if func(byref(b), name.encode('ascii')) == 999: - self.skipTest("Compiler does not support signed short bitfields") - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) - -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types - -class BitFieldTest(unittest.TestCase): - - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - self.assertEqual(sizeof(X), sizeof(c_longlong)) - x = X() - x.a, x.b, x.c = -1, 7, -1 - self.assertEqual((x.a, x.b, x.c), (-1, 7, -1)) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - self.assertEqual(sizeof(X), sizeof(c_longlong)) - x = X() - self.assertEqual((x.a, x.b, x.c), (0, 0, 0)) - x.a, x.b, x.c = 7, 7, 7 - self.assertEqual((x.a, x.b, x.c), (1, 7, 1)) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) - - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) - - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p')) - - result = self.fail_fields(("a", c_void_p, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p')) - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int')) - - result = self.fail_fields(("a", c_char, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) - - @need_symbol('c_wchar') - def test_c_wchar(self): - result = self.fail_fields(("a", c_wchar, 1)) - self.assertEqual(result, - (TypeError, 'bit fields not allowed for type c_wchar')) - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - self.assertEqual(sizeof(X), sizeof(c_short)) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - self.assertEqual(sizeof(X), sizeof(c_short)*3) - self.assertEqual(X.a.offset, 0) - self.assertEqual(X.a1.offset, sizeof(c_short)) - self.assertEqual(X.b.offset, sizeof(c_short)*2) - self.assertEqual(X.c.offset, sizeof(c_short)*2) - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - self.assertEqual(sizeof(X), sizeof(c_short)*3) - self.assertEqual(X.a.offset, sizeof(c_short)*0) - self.assertEqual(X.b.offset, sizeof(c_short)*1) - self.assertEqual(X.c.offset, sizeof(c_short)*2) - - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name == "nt": - self.assertEqual(sizeof(X), sizeof(c_int)*2) - else: - self.assertEqual(sizeof(X), sizeof(c_int)) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - self.assertEqual(sizeof(X), sizeof(c_byte)) - - def test_mixed_4(self): - class X(Structure): - _fields_ = [("a", c_short, 4), - ("b", c_short, 4), - ("c", c_int, 24), - ("d", c_short, 4), - ("e", c_short, 4), - ("f", c_int, 24)] - # MSVC does NOT combine c_short and c_int into one field, GCC - # does (unless GCC is run with '-mms-bitfields' which - # produces code compatible with MSVC). - if os.name == "nt": - self.assertEqual(sizeof(X), sizeof(c_int) * 4) - else: - self.assertEqual(sizeof(X), sizeof(c_int) * 2) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - @need_symbol('c_uint32') - def test_uint32(self): - class X(Structure): - _fields_ = [("a", c_uint32, 32)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFDCBA987 - self.assertEqual(x.a, 0xFDCBA987) - - @need_symbol('c_uint64') - def test_uint64(self): - class X(Structure): - _fields_ = [("a", c_uint64, 64)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFEDCBA9876543211 - self.assertEqual(x.a, 0xFEDCBA9876543211) - - @need_symbol('c_uint32') - def test_uint32_swap_little_endian(self): - # Issue #23319 - class Little(LittleEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Little.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xef\xcd\xab\x21') - - @need_symbol('c_uint32') - def test_uint32_swap_big_endian(self): - # Issue #23319 - class Big(BigEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Big.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xab\xcd\xef\x12') - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_delattr.py b/Lib/ctypes/test/test_delattr.py deleted file mode 100644 index 0f4d58691b5..00000000000 --- a/Lib/ctypes/test/test_delattr.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -from ctypes import * - -class X(Structure): - _fields_ = [("foo", c_int)] - -class TestCase(unittest.TestCase): - def test_simple(self): - self.assertRaises(TypeError, - delattr, c_int(42), "value") - - def test_chararray(self): - self.assertRaises(TypeError, - delattr, (c_char * 5)(), "value") - - def test_struct(self): - self.assertRaises(TypeError, - delattr, X(), "foo") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_incomplete.py b/Lib/ctypes/test/test_incomplete.py deleted file mode 100644 index 00c430ef53c..00000000000 --- a/Lib/ctypes/test/test_incomplete.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from ctypes import * - -################################################################ -# -# The incomplete pointer example from the tutorial -# - -class MyTestCase(unittest.TestCase): - - def test_incomplete_example(self): - lpcell = POINTER("cell") - class cell(Structure): - _fields_ = [("name", c_char_p), - ("next", lpcell)] - - SetPointerType(lpcell, cell) - - c1 = cell() - c1.name = b"foo" - c2 = cell() - c2.name = b"bar" - - c1.next = pointer(c2) - c2.next = pointer(c1) - - p = c1 - - result = [] - for i in range(8): - result.append(p.name) - p = p.next[0] - self.assertEqual(result, [b"foo", b"bar"] * 4) - - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[cell] - -################################################################ - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_libc.py b/Lib/ctypes/test/test_libc.py deleted file mode 100644 index 56285b5ff81..00000000000 --- a/Lib/ctypes/test/test_libc.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from ctypes import * -import _ctypes_test - -lib = CDLL(_ctypes_test.__file__) - -def three_way_cmp(x, y): - """Return -1 if x < y, 0 if x == y and 1 if x > y""" - return (x > y) - (x < y) - -class LibTest(unittest.TestCase): - def test_sqrt(self): - lib.my_sqrt.argtypes = c_double, - lib.my_sqrt.restype = c_double - self.assertEqual(lib.my_sqrt(4.0), 2.0) - import math - self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) - - def test_qsort(self): - comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) - lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc - lib.my_qsort.restype = None - - def sort(a, b): - return three_way_cmp(a[0], b[0]) - - chars = create_string_buffer(b"spam, spam, and spam") - lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) - self.assertEqual(chars.raw, b" ,,aaaadmmmnpppsss\x00") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_pointers.py b/Lib/ctypes/test/test_pointers.py deleted file mode 100644 index e97515879f1..00000000000 --- a/Lib/ctypes/test/test_pointers.py +++ /dev/null @@ -1,223 +0,0 @@ -import unittest, sys - -from ctypes import * -import _ctypes_test - -ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, - c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] -python_types = [int, int, int, int, int, int, - int, int, int, int, float, float] - -class PointersTestCase(unittest.TestCase): - - def test_pointer_crash(self): - - class A(POINTER(c_ulong)): - pass - - POINTER(c_ulong)(c_ulong(22)) - # Pointer can't set contents: has no _type_ - self.assertRaises(TypeError, A, c_ulong(33)) - - def test_pass_pointers(self): - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_p_p - if sizeof(c_longlong) == sizeof(c_void_p): - func.restype = c_longlong - else: - func.restype = c_long - - i = c_int(12345678) -## func.argtypes = (POINTER(c_int),) - address = func(byref(i)) - self.assertEqual(c_int.from_address(address).value, 12345678) - - func.restype = POINTER(c_int) - res = func(pointer(i)) - self.assertEqual(res.contents.value, 12345678) - self.assertEqual(res[0], 12345678) - - def test_change_pointers(self): - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_p_p - - i = c_int(87654) - func.restype = POINTER(c_int) - func.argtypes = (POINTER(c_int),) - - res = func(pointer(i)) - self.assertEqual(res[0], 87654) - self.assertEqual(res.contents.value, 87654) - - # C code: *res = 54345 - res[0] = 54345 - self.assertEqual(i.value, 54345) - - # C code: - # int x = 12321; - # res = &x - x = c_int(12321) - res.contents = x - self.assertEqual(i.value, 54345) - - x.value = -99 - self.assertEqual(res.contents.value, -99) - - def test_callbacks_with_pointers(self): - # a function type receiving a pointer - PROTOTYPE = CFUNCTYPE(c_int, POINTER(c_int)) - - self.result = [] - - def func(arg): - for i in range(10): -## print arg[i], - self.result.append(arg[i]) -## print - return 0 - callback = PROTOTYPE(func) - - dll = CDLL(_ctypes_test.__file__) - # This function expects a function pointer, - # and calls this with an integer pointer as parameter. - # The int pointer points to a table containing the numbers 1..10 - doit = dll._testfunc_callback_with_pointer - -## i = c_int(42) -## callback(byref(i)) -## self.assertEqual(i.value, 84) - - doit(callback) -## print self.result - doit(callback) -## print self.result - - def test_basics(self): - from operator import delitem - for ct, pt in zip(ctype_types, python_types): - i = ct(42) - p = pointer(i) -## print type(p.contents), ct - self.assertIs(type(p.contents), ct) - # p.contents is the same as p[0] -## print p.contents -## self.assertEqual(p.contents, 42) -## self.assertEqual(p[0], 42) - - self.assertRaises(TypeError, delitem, p, 0) - - def test_from_address(self): - from array import array - a = array('i', [100, 200, 300, 400, 500]) - addr = a.buffer_info()[0] - - p = POINTER(POINTER(c_int)) -## print dir(p) -## print p.from_address -## print p.from_address(addr)[0][0] - - def test_other(self): - class Table(Structure): - _fields_ = [("a", c_int), - ("b", c_int), - ("c", c_int)] - - pt = pointer(Table(1, 2, 3)) - - self.assertEqual(pt.contents.a, 1) - self.assertEqual(pt.contents.b, 2) - self.assertEqual(pt.contents.c, 3) - - pt.contents.c = 33 - - from ctypes import _pointer_type_cache - del _pointer_type_cache[Table] - - def test_basic(self): - p = pointer(c_int(42)) - # Although a pointer can be indexed, it has no length - self.assertRaises(TypeError, len, p) - self.assertEqual(p[0], 42) - self.assertEqual(p[0:1], [42]) - self.assertEqual(p.contents.value, 42) - - def test_charpp(self): - """Test that a character pointer-to-pointer is correctly passed""" - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_c_p_p - func.restype = c_char_p - argv = (c_char_p * 2)() - argc = c_int( 2 ) - argv[0] = b'hello' - argv[1] = b'world' - result = func( byref(argc), argv ) - self.assertEqual(result, b'world') - - def test_bug_1467852(self): - # http://sourceforge.net/tracker/?func=detail&atid=532154&aid=1467852&group_id=71702 - x = c_int(5) - dummy = [] - for i in range(32000): - dummy.append(c_int(i)) - y = c_int(6) - p = pointer(x) - pp = pointer(p) - q = pointer(y) - pp[0] = q # <== - self.assertEqual(p[0], 6) - def test_c_void_p(self): - # http://sourceforge.net/tracker/?func=detail&aid=1518190&group_id=5470&atid=105470 - if sizeof(c_void_p) == 4: - self.assertEqual(c_void_p(0xFFFFFFFF).value, - c_void_p(-1).value) - self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFF).value, - c_void_p(-1).value) - elif sizeof(c_void_p) == 8: - self.assertEqual(c_void_p(0xFFFFFFFF).value, - 0xFFFFFFFF) - self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFF).value, - c_void_p(-1).value) - self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFFFFFFFFFF).value, - c_void_p(-1).value) - - self.assertRaises(TypeError, c_void_p, 3.14) # make sure floats are NOT accepted - self.assertRaises(TypeError, c_void_p, object()) # nor other objects - - def test_pointers_bool(self): - # NULL pointers have a boolean False value, non-NULL pointers True. - self.assertEqual(bool(POINTER(c_int)()), False) - self.assertEqual(bool(pointer(c_int())), True) - - self.assertEqual(bool(CFUNCTYPE(None)(0)), False) - self.assertEqual(bool(CFUNCTYPE(None)(42)), True) - - # COM methods are boolean True: - if sys.platform == "win32": - mth = WINFUNCTYPE(None)(42, "name", (), None) - self.assertEqual(bool(mth), True) - - def test_pointer_type_name(self): - LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) - self.assertTrue(POINTER(LargeNamedType)) - - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[LargeNamedType] - - def test_pointer_type_str_name(self): - large_string = 'T' * 2 ** 25 - P = POINTER(large_string) - self.assertTrue(P) - - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[id(P)] - - def test_abstract(self): - from ctypes import _Pointer - - self.assertRaises(TypeError, _Pointer.set_type, 42) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_simplesubclasses.py b/Lib/ctypes/test/test_simplesubclasses.py deleted file mode 100644 index 3da2794a794..00000000000 --- a/Lib/ctypes/test/test_simplesubclasses.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -from ctypes import * - -class MyInt(c_int): - def __eq__(self, other): - if type(other) != MyInt: - return NotImplementedError - return self.value == other.value - -class Test(unittest.TestCase): - - def test_compare(self): - self.assertEqual(MyInt(3), MyInt(3)) - self.assertNotEqual(MyInt(42), MyInt(43)) - - def test_ignore_retval(self): - # Test if the return value of a callback is ignored - # if restype is None - proto = CFUNCTYPE(None) - def func(): - return (1, "abc", None) - - cb = proto(func) - self.assertEqual(None, cb()) - - - def test_int_callback(self): - args = [] - def func(arg): - args.append(arg) - return arg - - cb = CFUNCTYPE(None, MyInt)(func) - - self.assertEqual(None, cb(42)) - self.assertEqual(type(args[-1]), MyInt) - - cb = CFUNCTYPE(c_int, c_int)(func) - - self.assertEqual(42, cb(42)) - self.assertEqual(type(args[-1]), int) - - def test_int_struct(self): - class X(Structure): - _fields_ = [("x", MyInt)] - - self.assertEqual(X().x, MyInt()) - - s = X() - s.x = MyInt(42) - - self.assertEqual(s.x, MyInt(42)) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_struct_fields.py b/Lib/ctypes/test/test_struct_fields.py deleted file mode 100644 index e444f5e1f77..00000000000 --- a/Lib/ctypes/test/test_struct_fields.py +++ /dev/null @@ -1,97 +0,0 @@ -import unittest -from ctypes import * - -class StructFieldsTestCase(unittest.TestCase): - # Structure/Union classes must get 'finalized' sooner or - # later, when one of these things happen: - # - # 1. _fields_ is set. - # 2. An instance is created. - # 3. The type is used as field of another Structure/Union. - # 4. The type is subclassed - # - # When they are finalized, assigning _fields_ is no longer allowed. - - def test_1_A(self): - class X(Structure): - pass - self.assertEqual(sizeof(X), 0) # not finalized - X._fields_ = [] # finalized - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - - def test_1_B(self): - class X(Structure): - _fields_ = [] # finalized - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - - def test_2(self): - class X(Structure): - pass - X() - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - - def test_3(self): - class X(Structure): - pass - class Y(Structure): - _fields_ = [("x", X)] # finalizes X - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - - def test_4(self): - class X(Structure): - pass - class Y(X): - pass - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - Y._fields_ = [] - self.assertRaises(AttributeError, setattr, X, "_fields_", []) - - def test_5(self): - class X(Structure): - _fields_ = (("char", c_char * 5),) - - x = X(b'#' * 5) - x.char = b'a\0b\0' - self.assertEqual(bytes(x), b'a\x00###') - - def test_6(self): - class X(Structure): - _fields_ = [("x", c_int)] - CField = type(X.x) - self.assertRaises(TypeError, CField) - - def test_gh99275(self): - class BrokenStructure(Structure): - def __init_subclass__(cls, **kwargs): - cls._fields_ = [] # This line will fail, `stgdict` is not ready - - with self.assertRaisesRegex(TypeError, - 'ctypes state is not initialized'): - class Subclass(BrokenStructure): ... - - # __set__ and __get__ should raise a TypeError in case their self - # argument is not a ctype instance. - def test___set__(self): - class MyCStruct(Structure): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCStruct.field.__set__, 'wrong type self', 42) - - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__set__, 'wrong type self', 42) - - def test___get__(self): - class MyCStruct(Structure): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCStruct.field.__get__, 'wrong type self', 42) - - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__get__, 'wrong type self', 42) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 0c2510e1619..378f12167c6 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,66 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": + # Listing loaded DLLs on Windows relies on the following APIs: + # https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-enumprocessmodules + # https://learn.microsoft.com/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulefilenamew + import ctypes + from ctypes import wintypes + + _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + _get_current_process = _kernel32["GetCurrentProcess"] + _get_current_process.restype = wintypes.HANDLE + + _k32_get_module_file_name = _kernel32["GetModuleFileNameW"] + _k32_get_module_file_name.restype = wintypes.DWORD + _k32_get_module_file_name.argtypes = ( + wintypes.HMODULE, + wintypes.LPWSTR, + wintypes.DWORD, + ) + + _psapi = ctypes.WinDLL('psapi', use_last_error=True) + _enum_process_modules = _psapi["EnumProcessModules"] + _enum_process_modules.restype = wintypes.BOOL + _enum_process_modules.argtypes = ( + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + wintypes.LPDWORD, + ) + + def _get_module_filename(module: wintypes.HMODULE): + name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS + if _k32_get_module_file_name(module, name, len(name)): + return name.value + return None + + + def _get_module_handles(): + process = _get_current_process() + space_needed = wintypes.DWORD() + n = 1024 + while True: + modules = (wintypes.HMODULE * n)() + if not _enum_process_modules(process, + modules, + ctypes.sizeof(modules), + ctypes.byref(space_needed)): + err = ctypes.get_last_error() + msg = ctypes.FormatError(err).strip() + raise ctypes.WinError(err, f"EnumProcessModules failed: {msg}") + n = space_needed.value // ctypes.sizeof(wintypes.HMODULE) + if n <= len(modules): + return modules[:n] + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + modules = _get_module_handles() + libraries = [name for h in modules + if (name := _get_module_filename(h)) is not None] + return libraries + +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -80,6 +139,22 @@ def find_library(name): continue return None + # Listing loaded libraries on Apple systems relies on the following API: + # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dyld.3.html + import ctypes + + _libc = ctypes.CDLL(find_library("c")) + _dyld_get_image_name = _libc["_dyld_get_image_name"] + _dyld_get_image_name.restype = ctypes.c_char_p + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + num_images = _libc._dyld_image_count() + libraries = [os.fsdecode(name) for i in range(num_images) + if (name := _dyld_get_image_name(i)) is not None] + + return libraries + elif sys.platform.startswith("aix"): # AIX has two styles of storing shared libraries # GNU auto_tools refer to these as svr4 and aix @@ -89,6 +164,34 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + +elif sys.platform == "emscripten": + def _is_wasm(filename): + # Return True if the given file is an WASM module + wasm_header = b"\x00asm" + with open(filename, 'br') as thefile: + return thefile.read(4) == wasm_header + + def find_library(name): + candidates = [f"lib{name}.so", f"lib{name}.wasm"] + paths = os.environ.get("LD_LIBRARY_PATH", "") + for libdir in paths.split(":"): + for name in candidates: + libfile = os.path.join(libdir, name) + + if os.path.isfile(libfile) and _is_wasm(libfile): + return libfile + + return None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -96,8 +199,11 @@ def find_library(name): def _is_elf(filename): "Return True if the given file is an ELF file" elf_header = b'\x7fELF' - with open(filename, 'br') as thefile: - return thefile.read(4) == elf_header + try: + with open(filename, 'br') as thefile: + return thefile.read(4) == elf_header + except FileNotFoundError: + return False def _findLib_gcc(name): # Run GCC's linker with the -t (aka --trace) option and examine the @@ -329,6 +435,55 @@ def find_library(name): return _findSoname_ldconfig(name) or \ _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) + +# Listing loaded libraries on other systems will try to use +# functions common to Linux and a few other Unix-like systems. +# See the following for several platforms' documentation of the same API: +# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html +# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr +# https://man.openbsd.org/dl_iterate_phdr +# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html +if (os.name == "posix" and + sys.platform not in {"darwin", "ios", "tvos", "watchos"}): + import ctypes + if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): + + class _dl_phdr_info(ctypes.Structure): + _fields_ = [ + ("dlpi_addr", ctypes.c_void_p), + ("dlpi_name", ctypes.c_char_p), + ("dlpi_phdr", ctypes.c_void_p), + ("dlpi_phnum", ctypes.c_ushort), + ] + + _dl_phdr_callback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.POINTER(_dl_phdr_info), + ctypes.c_size_t, + ctypes.POINTER(ctypes.py_object), + ) + + @_dl_phdr_callback + def _info_callback(info, _size, data): + libraries = data.contents.value + name = os.fsdecode(info.contents.dlpi_name) + libraries.append(name) + return 0 + + _dl_iterate_phdr = _libc["dl_iterate_phdr"] + _dl_iterate_phdr.argtypes = [ + _dl_phdr_callback, + ctypes.POINTER(ctypes.py_object), + ] + _dl_iterate_phdr.restype = ctypes.c_int + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + libraries = [] + _dl_iterate_phdr(_info_callback, + ctypes.byref(ctypes.py_object(libraries))) + return libraries + ################################################################ # test code @@ -372,5 +527,12 @@ def test(): print(cdll.LoadLibrary("libcrypt.so")) print(find_library("crypt")) + try: + dllist + except NameError: + print('dllist() not available') + else: + print(dllist()) + if __name__ == "__main__": test() diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438a..4beba0d1951 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7883ce78e57..c8dbb247745 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -5,6 +5,7 @@ import inspect import keyword import itertools +import annotationlib import abc from reprlib import recursive_repr @@ -243,6 +244,10 @@ def __repr__(self): property, }) +# Any marker is used in `make_dataclass` to mark unannotated fields as `Any` +# without importing `typing` module. +_ANY_MARKER = object() + class InitVar: __slots__ = ('type', ) @@ -282,11 +287,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -299,6 +305,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -314,6 +321,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -381,7 +389,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -393,7 +401,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -401,7 +409,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -433,9 +441,11 @@ def __init__(self, globals): self.locals = {} self.overwrite_errors = {} self.unconditional_adds = {} + self.method_annotations = {} def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, - overwrite_error=False, unconditional_add=False, decorator=None): + overwrite_error=False, unconditional_add=False, decorator=None, + annotation_fields=None): if locals is not None: self.locals.update(locals) @@ -456,16 +466,14 @@ def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, self.names.append(name) - if return_type is not MISSING: - self.locals[f'__dataclass_{name}_return_type__'] = return_type - return_annotation = f'->__dataclass_{name}_return_type__' - else: - return_annotation = '' + if annotation_fields is not None: + self.method_annotations[name] = (annotation_fields, return_type) + args = ','.join(args) body = '\n'.join(body) # Compute the text of the entire function, add it to the text we're generating. - self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}):\n{body}') def add_fns_to_class(self, cls): # The source to all of the functions we're generating. @@ -501,6 +509,15 @@ def add_fns_to_class(self, cls): # Now that we've generated the functions, assign them into cls. for name, fn in zip(self.names, fns): fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): setattr(cls, name, fn) else: @@ -516,6 +533,49 @@ def add_fns_to_class(self, cls): raise TypeError(error_msg) +def _make_annotate_function(__class__, method_name, annotation_fields, return_type): + # Create an __annotate__ function for a dataclass + # Try to return annotations in the same format as they would be + # from a regular __init__ function + + def __annotate__(format, /): + Format = annotationlib.Format + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + cls_annotations = {} + for base in reversed(__class__.__mro__): + cls_annotations.update( + annotationlib.get_annotations(base, format=format) + ) + + new_annotations = {} + for k in annotation_fields: + # gh-142214: The annotation may be missing in unusual dynamic cases. + # If so, just skip it. + try: + new_annotations[k] = cls_annotations[k] + except KeyError: + pass + + if return_type is not MISSING: + if format == Format.STRING: + new_annotations["return"] = annotationlib.type_repr(return_type) + else: + new_annotations["return"] = return_type + + return new_annotations + + case _: + raise NotImplementedError(format) + + # This is a flag for _add_slots to know it needs to regenerate this method + # In order to remove references to the original class when it is replaced + __annotate__.__generated_by_dataclasses__ = True + __annotate__.__qualname__ = f"{__class__.__qualname__}.{method_name}.__annotate__" + + return __annotate__ + + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple @@ -604,7 +664,7 @@ def _init_param(f): elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=__dataclass_HAS_DEFAULT_FACTORY__' - return f'{f.name}:__dataclass_type_{f.name}__{default}' + return f'{f.name}{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, @@ -627,11 +687,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, - **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - } - } + annotation_fields = [f.name for f in fields if f.init] + + locals = {'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object} body_lines = [] for f in fields: @@ -655,14 +714,15 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if kw_only_fields: # Add the keyword-only args. Because the * can only be added if # there's at least one keyword-only arg, there needs to be a test here - # (instead of just concatenting the lists together). + # (instead of just concatenating the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] func_builder.add_fn('__init__', [self_name] + _init_params, body_lines, locals=locals, - return_type=None) + return_type=None, + annotation_fields=annotation_fields) def _frozen_get_del_attr(cls, fields, func_builder): @@ -689,11 +749,8 @@ def _frozen_get_del_attr(cls, fields, func_builder): def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): @@ -981,7 +1038,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = inspect.get_annotations(cls) + cls_annotations = annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where @@ -1161,7 +1219,10 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, try: # In some cases fetching a signature is not possible. # But, we surely should not fail in this case. - text_sig = str(inspect.signature(cls)).replace(' -> None', '') + text_sig = str(inspect.signature( + cls, + annotation_format=annotationlib.Format.FORWARDREF, + )).replace(' -> None', '') except (TypeError, ValueError): text_sig = '' cls.__doc__ = (cls.__name__ + text_sig) @@ -1175,7 +1236,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1219,14 +1280,65 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") -def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots[slot] = doc + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: raise TypeError(f'{cls.__name__} already specifies __slots__') + # gh-102069: Remove existing __weakref__ descriptor. + # gh-135228: Make sure the original class can be garbage collected. + sys._clear_type_descriptors(cls) + # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) @@ -1234,17 +1346,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1252,26 +1356,59 @@ def _add_slots(cls, is_frozen, weakref_slot): # available in _MARKER. cls_dict.pop(field_name, None) - # Remove __dict__ itself. - cls_dict.pop('__dict__', None) - - # Clear existing `__weakref__` descriptor, it belongs to a previous type: - cls_dict.pop('__weakref__', None) # gh-102069 - # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann - return cls + # Fix the class reference in the __annotate__ method + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, @@ -1490,7 +1627,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1528,7 +1665,7 @@ class C(Base): for item in fields: if isinstance(item, str): name = item - tp = 'typing.Any' + tp = _ANY_MARKER elif len(item) == 2: name, tp, = item elif len(item) == 3: @@ -1547,15 +1684,49 @@ class C(Base): seen.add(name) annotations[name] = tp + # We initially block the VALUE format, because inside dataclass() we'll + # call get_annotations(), which will try the VALUE format first. If we don't + # block, that means we'd always end up eagerly importing typing here, which + # is what we're trying to avoid. + value_blocked = True + + def annotate_method(format): + def get_any(): + match format: + case annotationlib.Format.STRING: + return 'typing.Any' + case annotationlib.Format.FORWARDREF: + typing = sys.modules.get("typing") + if typing is None: + return annotationlib.ForwardRef("Any", module="typing") + else: + return typing.Any + case annotationlib.Format.VALUE: + if value_blocked: + raise NotImplementedError + from typing import Any + return Any + case _: + raise NotImplementedError + annos = { + ann: get_any() if t is _ANY_MARKER else t + for ann, t in annotations.items() + } + if format == annotationlib.Format.STRING: + return annotationlib.annotations_to_string(annos) + else: + return annos + # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): ns.update(namespace) ns.update(defaults) - ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For now, set annotations including the _ANY_MARKER. + cls.__annotate__ = annotate_method # For pickling to work, the __module__ variable needs to be set to the frame # where the dataclass is created. @@ -1570,11 +1741,14 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, kw_only=kw_only, slots=slots, - weakref_slot=weakref_slot) + # Apply the normal provided decorator. + cls = decorator(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) + # Now that the class is ready, allow the VALUE format. + value_blocked = False + return cls def replace(obj, /, **changes): diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index f65da521af4..4fdbc54e74c 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -109,17 +109,18 @@ def whichdb(filename): """ # Check for ndbm first -- this has a .pag and a .dir file + filename = os.fsencode(filename) try: - f = io.open(filename + ".pag", "rb") + f = io.open(filename + b".pag", "rb") f.close() - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") f.close() return "dbm.ndbm" except OSError: # some dbm emulations based on Berkeley DB generate a .db file # some do not, but they should be caught by the bsd checks try: - f = io.open(filename + ".db", "rb") + f = io.open(filename + b".db", "rb") f.close() # guarantee we can actually open the file using dbm # kind of overkill, but since we are dealing with emulations @@ -134,12 +135,12 @@ def whichdb(filename): # Check for dumbdbm next -- this has a .dir and a .dat file try: # First check for presence of files - os.stat(filename + ".dat") - size = os.stat(filename + ".dir").st_size + os.stat(filename + b".dat") + size = os.stat(filename + b".dir").st_size # dumbdbm files with no keys are empty if size == 0: return "dbm.dumb" - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") try: if f.read(1) in (b"'", b'"'): return "dbm.dumb" @@ -163,6 +164,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 864ad371ec9..def120ffc37 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -46,6 +46,7 @@ class _Database(collections.abc.MutableMapping): _io = _io # for _commit() def __init__(self, filebasename, mode, flag='c'): + filebasename = self._os.fsencode(filebasename) self._mode = mode self._readonly = (flag == 'r') @@ -54,14 +55,14 @@ def __init__(self, filebasename, mode, flag='c'): # where key is the string key, pos is the offset into the dat # file of the associated value's first byte, and siz is the number # of bytes in the associated value. - self._dirfile = filebasename + '.dir' + self._dirfile = filebasename + b'.dir' # The data file is a binary file pointed into by the directory # file, and holds the values associated with keys. Each value # begins at a _BLOCKSIZE-aligned byte offset, and is a raw # binary 8-bit string value. - self._datfile = filebasename + '.dat' - self._bakfile = filebasename + '.bak' + self._datfile = filebasename + b'.dat' + self._bakfile = filebasename + b'.bak' # The index is an in-memory dict, mirroring the directory file. self._index = None # maps keys to (pos, siz) pairs @@ -97,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -133,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/dbm/gnu.py b/Lib/dbm/gnu.py new file mode 100644 index 00000000000..b07a1defffd --- /dev/null +++ b/Lib/dbm/gnu.py @@ -0,0 +1,3 @@ +"""Provide the _gdbm module as a dbm submodule.""" + +from _gdbm import * diff --git a/Lib/dbm/ndbm.py b/Lib/dbm/ndbm.py new file mode 100644 index 00000000000..23056a29ef2 --- /dev/null +++ b/Lib/dbm/ndbm.py @@ -0,0 +1,3 @@ +"""Provide the _dbm module as a dbm submodule.""" + +from _dbm import * diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 00000000000..d0eed54e0f8 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,144 @@ +import os +import sqlite3 +from pathlib import Path +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE IF NOT EXISTS Dict ( + key BLOB UNIQUE NOT NULL, + value BLOB NOT NULL + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" +ITER_KEYS = "SELECT key FROM Dict" + + +class error(OSError): + pass + + +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + +def _normalize_uri(path): + path = Path(path) + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri + + +class _Database(MutableMapping): + + def __init__(self, path, /, *, flag, mode): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + + path = os.fsdecode(path) + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) + case "n": + flag = "rwc" + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") + + # We use the URI format when opening the database. + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" + if flag == "ro": + # Add immutable=1 to allow read-only SQLite access even if wal/shm missing + uri += "&immutable=1" + + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) + + if flag != "ro": + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") + + if flag == "rwc": + self._execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) + try: + return closing(self._cx.execute(*args, **kwargs)) + except sqlite3.Error as exc: + raise error(str(exc)) + + def __len__(self): + with self._execute(GET_SIZE) as cu: + row = cu.fetchone() + return row[0] + + def __getitem__(self, key): + with self._execute(LOOKUP_KEY, (key,)) as cu: + row = cu.fetchone() + if not row: + raise KeyError(key) + return row[0] + + def __setitem__(self, key, value): + self._execute(STORE_KV, (key, value)) + + def __delitem__(self, key): + with self._execute(DELETE_KEY, (key,)) as cu: + if not cu.rowcount: + raise KeyError(key) + + def __iter__(self): + try: + with self._execute(ITER_KEYS) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + if self._cx: + self._cx.close() + self._cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def open(filename, /, flag="r", mode=0o666): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. + """ + return _Database(filename, flag=flag, mode=mode) diff --git a/Lib/decimal.py b/Lib/decimal.py index ee3147f5dde..530bdfb3895 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -100,8 +100,8 @@ try: from _decimal import * - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: import _pydecimal import sys diff --git a/Lib/difflib.py b/Lib/difflib.py index 33e7e6c165a..ac1ba4a6e4e 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -78,8 +78,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 3)) - 0.866 + >>> print(round(s.ratio(), 2)) + 0.87 >>> If you're only interested in where the sequences match, @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1040,11 +1038,9 @@ def _qformat(self, aline, bline, atags, btags): # remaining is that perhaps it was really the case that " volatile" # was inserted after "private". I can live with that . -import re - -def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): +def IS_LINE_JUNK(line, pat=None): r""" - Return True for ignorable line: iff `line` is blank or contains a single '#'. + Return True for ignorable line: if `line` is blank or contains a single '#'. Examples: @@ -1056,6 +1052,11 @@ def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): False """ + if pat is None: + # Default: match '#' or the empty string + return line.strip() in '#' + # Previous versions used the undocumented parameter 'pat' as a + # match function. Retain this behaviour for compatibility. return pat(line) is not None def IS_CHARACTER_JUNK(ch, ws=" \t"): @@ -1266,6 +1267,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) @@ -1628,13 +1635,22 @@ def _line_pair_iterator(): """ _styles = """ + :root {color-scheme: light dark} table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} - .diff_add {background-color:#aaffaa} + .diff_add {background-color:palegreen} .diff_chg {background-color:#ffff77} - .diff_sub {background-color:#ffaaaa}""" + .diff_sub {background-color:#ffaaaa} + + @media (prefers-color-scheme: dark) { + .diff_header {background-color:#666} + .diff_next {background-color:#393939} + .diff_add {background-color:darkgreen} + .diff_chg {background-color:#847415} + .diff_sub {background-color:darkred} + }""" _table_template = """ '). \ replace('\t',' ') -del re def restore(delta, which): r""" @@ -2047,10 +2062,3 @@ def restore(delta, which): for line in delta: if line[:2] in prefixes: yield line[2:] - -def _test(): - import doctest, difflib - return doctest.testmod(difflib) - -if __name__ == "__main__": - _test() diff --git a/Lib/dis.py b/Lib/dis.py index 05de51ce49c..d6d2c1386dd 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,15 +11,16 @@ _cache_format, _inline_cache_entries, _nb_ops, + _common_constants, _intrinsic_1_descs, _intrinsic_2_descs, + _special_method_names, _specializations, _specialized_opmap, ) from _opcode import get_executor - __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all @@ -31,12 +32,11 @@ CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] -FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] -LOAD_CONST = opmap['LOAD_CONST'] -RETURN_CONST = opmap['RETURN_CONST'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] +LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] BINARY_OP = opmap['BINARY_OP'] JUMP_BACKWARD = opmap['JUMP_BACKWARD'] FOR_ITER = opmap['FOR_ITER'] @@ -45,9 +45,15 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] +LOAD_SPECIAL = opmap['LOAD_SPECIAL'] LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] +LOAD_FAST_BORROW_LOAD_FAST_BORROW = opmap['LOAD_FAST_BORROW_LOAD_FAST_BORROW'] STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] +IS_OP = opmap['IS_OP'] +CONTAINS_OP = opmap['CONTAINS_OP'] +END_ASYNC_FOR = opmap['END_ASYNC_FOR'] CACHE = opmap["CACHE"] @@ -77,7 +83,7 @@ def _try_compile(source, name): return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -88,7 +94,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, """ if x is None: distb(file=file, show_caches=show_caches, adaptive=adaptive, - show_offsets=show_offsets) + show_offsets=show_offsets, show_positions=show_positions) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -109,12 +115,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) elif isinstance(x, (bytes, bytearray)): # Raw bytecode labels_map = _make_labels_map(x) label_width = 4 + len(str(len(labels_map))) @@ -125,12 +131,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, arg_resolver = ArgResolver(labels_map=labels_map) _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -141,22 +147,24 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to # turn the co_flags field into a human readable list. COMPILER_FLAG_NAMES = { - 1: "OPTIMIZED", - 2: "NEWLOCALS", - 4: "VARARGS", - 8: "VARKEYWORDS", - 16: "NESTED", - 32: "GENERATOR", - 64: "NOFREE", - 128: "COROUTINE", - 256: "ITERABLE_COROUTINE", - 512: "ASYNC_GENERATOR", + 1: "OPTIMIZED", + 2: "NEWLOCALS", + 4: "VARARGS", + 8: "VARKEYWORDS", + 16: "NESTED", + 32: "GENERATOR", + 64: "NOFREE", + 128: "COROUTINE", + 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", + 0x4000000: "HAS_DOCSTRING", + 0x8000000: "METHOD", } def pretty_flags(flags): @@ -370,6 +378,14 @@ class Instruction(_Instruction): entries (if any) """ + @staticmethod + def make( + opname, arg, argval, argrepr, offset, start_offset, starts_line, + line_number, label=None, positions=None, cache_info=None + ): + return Instruction(opname, _all_opmap[opname], arg, argval, argrepr, offset, + start_offset, starts_line, line_number, label, positions, cache_info) + @property def oparg(self): """Alias for Instruction.arg.""" @@ -424,21 +440,25 @@ def __str__(self): class Formatter: def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, - line_offset=0, show_caches=False): + line_offset=0, show_caches=False, *, show_positions=False): """Create a Formatter *file* where to write the output - *lineno_width* sets the width of the line number field (0 omits it) + *lineno_width* sets the width of the source location field (0 omits it). + Should be large enough for a line number or full positions (depending + on the value of *show_positions*). *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field *show_caches* is a boolean indicating whether to display cache lines - + *show_positions* is a boolean indicating whether full positions should + be reported instead of only the line numbers. """ self.file = file self.lineno_width = lineno_width self.offset_width = offset_width self.label_width = label_width self.show_caches = show_caches + self.show_positions = show_positions def print_instruction(self, instr, mark_as_current=False): self.print_instruction_line(instr, mark_as_current) @@ -471,15 +491,27 @@ def print_instruction_line(self, instr, mark_as_current): print(file=self.file) fields = [] - # Column: Source code line number + # Column: Source code locations information if lineno_width: - if instr.starts_line: - lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" - lineno_fmt = lineno_fmt % lineno_width - lineno = _NO_LINENO if instr.line_number is None else instr.line_number - fields.append(lineno_fmt % lineno) + if self.show_positions: + # reporting positions instead of just line numbers + if instr_positions := instr.positions: + if all(p is None for p in instr_positions): + positions_str = _NO_LINENO + else: + ps = tuple('?' if p is None else p for p in instr_positions) + positions_str = f"{ps[0]}:{ps[2]}-{ps[1]}:{ps[3]}" + fields.append(f'{positions_str:{lineno_width}}') + else: + fields.append(' ' * lineno_width) else: - fields.append(' ' * lineno_width) + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) # Column: Label if instr.label is not None: lbl = f"L{instr.label}:" @@ -575,8 +607,9 @@ def get_argval_argrepr(self, op, arg, offset): argval = self.offset_from_jump_arg(op, arg, offset) lbl = self.get_label_for_offset(argval) assert lbl is not None - argrepr = f"to L{lbl}" - elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + preposition = "from" if deop == END_ASYNC_FOR else "to" + argrepr = f"{preposition} L{lbl}" + elif deop in (LOAD_FAST_LOAD_FAST, LOAD_FAST_BORROW_LOAD_FAST_BORROW, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): arg1 = arg >> 4 arg2 = arg & 15 val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) @@ -602,6 +635,18 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: argrepr = _intrinsic_2_descs[arg] + elif deop == LOAD_COMMON_CONSTANT: + obj = _common_constants[arg] + if isinstance(obj, type): + argrepr = obj.__name__ + else: + argrepr = repr(obj) + elif deop == LOAD_SPECIAL: + argrepr = _special_method_names[arg] + elif deop == IS_OP: + argrepr = 'is not' if argval else 'is' + elif deop == CONTAINS_OP: + argrepr = 'not in' if argval else 'in' return argval, argrepr def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): @@ -641,8 +686,10 @@ def _get_const_value(op, arg, co_consts): Otherwise (if it is a LOAD_CONST and co_consts is not provided) returns the dis.UNKNOWN sentinel. """ - assert op in hasconst + assert op in hasconst or op == LOAD_SMALL_INT + if op == LOAD_SMALL_INT: + return arg argval = UNKNOWN if co_consts is not None: argval = co_consts[arg] @@ -701,7 +748,8 @@ def _parse_exception_table(code): def _is_backward_jump(op): return opname[op] in ('JUMP_BACKWARD', - 'JUMP_BACKWARD_NO_INTERRUPT') + 'JUMP_BACKWARD_NO_INTERRUPT', + 'END_ASYNC_FOR') # Not really a jump, but it has a "target" def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, original_code=None, arg_resolver=None): @@ -745,8 +793,10 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N if caches: cache_info = [] + cache_offset = offset for name, size in _cache_format[opname[deop]].items(): - data = code[offset + 2: offset + 2 + 2 * size] + data = code[cache_offset + 2: cache_offset + 2 + 2 * size] + cache_offset += size * 2 cache_info.append((name, size, data)) else: cache_info = None @@ -758,17 +808,22 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) + if show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(linestarts) labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=file, - lineno_width=_get_lineno_width(linestarts), + lineno_width=lineno_width, offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, label_width=label_width, - show_caches=show_caches) + show_caches=show_caches, + show_positions=show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, varname_from_oparg=co._varname_from_oparg, @@ -777,8 +832,8 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, exception_entries=exception_entries, co_positions=co.co_positions(), original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -788,7 +843,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( x, file=file, depth=depth, show_caches=show_caches, - adaptive=adaptive, show_offsets=show_offsets + adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions ) @@ -821,6 +876,22 @@ def _get_lineno_width(linestarts): lineno_width = len(_NO_LINENO) return lineno_width +def _get_positions_width(code): + # Positions are formatted as 'LINE:COL-ENDLINE:ENDCOL ' (note trailing space). + # A missing component appears as '?', and when all components are None, we + # render '_NO_LINENO'. thus the minimum width is 1 + len(_NO_LINENO). + # + # If all values are missing, positions are not printed (i.e. positions_width = 0). + has_value = False + values_width = 0 + for positions in code.co_positions(): + has_value |= any(isinstance(p, int) for p in positions) + width = sum(1 if p is None else len(str(p)) for p in positions) + values_width = max(width, values_width) + if has_value: + # 3 = number of separators in a normal format + return 1 + max(len(_NO_LINENO), 3 + values_width) + return 0 def _disassemble_bytes(code, lasti=-1, linestarts=None, *, line_offset=0, exception_entries=(), @@ -938,7 +1009,8 @@ def _find_imports(co): if op == IMPORT_NAME and i >= 2: from_op = opargs[i-1] level_op = opargs[i-2] - if (from_op[0] in hasconst and level_op[0] in hasconst): + if (from_op[0] in hasconst and + (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)): level = _get_const_value(level_op[0], level_op[1], consts) fromlist = _get_const_value(from_op[0], from_op[1], consts) yield (names[oparg], level, fromlist) @@ -967,7 +1039,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -982,6 +1054,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.show_caches = show_caches self.adaptive = adaptive self.show_offsets = show_offsets + self.show_positions = show_positions def __iter__(self): co = self.codeobj @@ -1025,16 +1098,19 @@ def dis(self): with io.StringIO() as output: code = _get_code_array(co, self.adaptive) offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 - - + if self.show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(self._linestarts) labels_map = _make_labels_map(co.co_code, self.exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=output, - lineno_width=_get_lineno_width(self._linestarts), + lineno_width=lineno_width, offset_width=offset_width, label_width=label_width, line_offset=self._line_offset, - show_caches=self.show_caches) + show_caches=self.show_caches, + show_positions=self.show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, @@ -1052,21 +1128,30 @@ def dis(self): return output.getvalue() -from _dis import * - - -# Disassembling a file by following cpython Lib/dis.py -def _test(): - """Simple test program to disassemble a file.""" +def main(args=None): import argparse - parser = argparse.ArgumentParser() - parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') - args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") - dis(code) + parser = argparse.ArgumentParser(color=True) + parser.add_argument('-C', '--show-caches', action='store_true', + help='show inline caches') + parser.add_argument('-O', '--show-offsets', action='store_true', + help='show instruction offsets') + parser.add_argument('-P', '--show-positions', action='store_true', + help='show instruction positions') + parser.add_argument('-S', '--specialized', action='store_true', + help='show specialized bytecode') + parser.add_argument('infile', nargs='?', default='-') + args = parser.parse_args(args=args) + if args.infile == '-': + name = '' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + code = compile(source, name, "exec") + dis(code, show_caches=args.show_caches, adaptive=args.specialized, + show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": - _test() + main() diff --git a/Lib/distutils/README b/Lib/distutils/README deleted file mode 100644 index 408a203b85d..00000000000 --- a/Lib/distutils/README +++ /dev/null @@ -1,13 +0,0 @@ -This directory contains the Distutils package. - -There's a full documentation available at: - - http://docs.python.org/distutils/ - -The Distutils-SIG web page is also a good starting point: - - http://www.python.org/sigs/distutils-sig/ - -WARNING : Distutils must remain compatible with 2.3 - -$Id$ diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py deleted file mode 100644 index d823d040a1c..00000000000 --- a/Lib/distutils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""distutils - -The main package for the Python Module Distribution Utilities. Normally -used from a setup script as - - from distutils.core import setup - - setup (...) -""" - -import sys - -__version__ = sys.version[:sys.version.index(' ')] diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py deleted file mode 100644 index 30b3b473985..00000000000 --- a/Lib/distutils/_msvccompiler.py +++ /dev/null @@ -1,574 +0,0 @@ -"""distutils._msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for Microsoft Visual Studio 2015. - -The module is compatible with VS 2015 and later. You can find legacy support -for older versions in distutils.msvc9compiler and distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS 2005 and VS 2008 by Christian Heimes -# ported to VS 2015 by Steve Dower - -import os -import shutil -import stat -import subprocess -import winreg - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_lib_options -from distutils import log -from distutils.util import get_platform - -from itertools import count - -def _find_vc2015(): - try: - key = winreg.OpenKeyEx( - winreg.HKEY_LOCAL_MACHINE, - r"Software\Microsoft\VisualStudio\SxS\VC7", - access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY - ) - except OSError: - log.debug("Visual C++ is not registered") - return None, None - - best_version = 0 - best_dir = None - with key: - for i in count(): - try: - v, vc_dir, vt = winreg.EnumValue(key, i) - except OSError: - break - if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): - try: - version = int(float(v)) - except (ValueError, TypeError): - continue - if version >= 14 and version > best_version: - best_version, best_dir = version, vc_dir - return best_version, best_dir - -def _find_vc2017(): - import _distutils_findvs - import threading - - best_version = 0, # tuple for full version comparisons - best_dir = None - - # We need to call findall() on its own thread because it will - # initialize COM. - all_packages = [] - def _getall(): - all_packages.extend(_distutils_findvs.findall()) - t = threading.Thread(target=_getall) - t.start() - t.join() - - for name, version_str, path, packages in all_packages: - if 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' in packages: - vc_dir = os.path.join(path, 'VC', 'Auxiliary', 'Build') - if not os.path.isdir(vc_dir): - continue - try: - version = tuple(int(i) for i in version_str.split('.')) - except (ValueError, TypeError): - continue - if version > best_version: - best_version, best_dir = version, vc_dir - try: - best_version = best_version[0] - except IndexError: - best_version = None - return best_version, best_dir - -def _find_vcvarsall(plat_spec): - best_version, best_dir = _find_vc2017() - vcruntime = None - vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86' - if best_version: - vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**", - "Microsoft.VC141.CRT", "vcruntime140.dll") - try: - import glob - vcruntime = glob.glob(vcredist, recursive=True)[-1] - except (ImportError, OSError, LookupError): - vcruntime = None - - if not best_version: - best_version, best_dir = _find_vc2015() - if best_version: - vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat, - "Microsoft.VC140.CRT", "vcruntime140.dll") - - if not best_version: - log.debug("No suitable Visual C++ version found") - return None, None - - vcvarsall = os.path.join(best_dir, "vcvarsall.bat") - if not os.path.isfile(vcvarsall): - log.debug("%s cannot be found", vcvarsall) - return None, None - - if not vcruntime or not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime - -def _get_vc_env(plat_spec): - if os.getenv("DISTUTILS_USE_SDK"): - return { - key.lower(): value - for key, value in os.environ.items() - } - - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) - if not vcvarsall: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - - try: - out = subprocess.check_output( - 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), - stderr=subprocess.STDOUT, - ).decode('utf-16le', errors='replace') - except subprocess.CalledProcessError as exc: - log.error(exc.output) - raise DistutilsPlatformError("Error executing {}" - .format(exc.cmd)) - - env = { - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } - - if vcruntime: - env['py_vcruntime_redist'] = vcruntime - return env - -def _find_exe(exe, paths=None): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - if not paths: - paths = os.getenv('path').split(os.pathsep) - for p in paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - return exe - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Always cross-compile from x86 to work with the -# lighter-weight MSVC installs that do not include native 64-bit tools. -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'x86_amd64', -} - -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - if plat_name not in PLAT_TO_VCVARS: - raise DistutilsPlatformError("--plat-name must be one of {}" - .format(tuple(PLAT_TO_VCVARS))) - - # Get the vcvarsall.bat spec for the requested platform. - plat_spec = PLAT_TO_VCVARS[plat_name] - - vc_env = _get_vc_env(plat_spec) - if not vc_env: - raise DistutilsPlatformError("Unable to find a compatible " - "Visual Studio installation.") - - self._paths = vc_env.get('path', '') - paths = self._paths.split(os.pathsep) - self.cc = _find_exe("cl.exe", paths) - self.linker = _find_exe("link.exe", paths) - self.lib = _find_exe("lib.exe", paths) - self.rc = _find_exe("rc.exe", paths) # resource compiler - self.mc = _find_exe("mc.exe", paths) # message compiler - self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') - - for dir in vc_env.get('include', '').split(os.pathsep): - if dir: - self.add_include_dir(dir.rstrip(os.sep)) - - for dir in vc_env.get('lib', '').split(os.pathsep): - if dir: - self.add_library_dir(dir.rstrip(os.sep)) - - self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. - self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' - ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') - - self.compile_options_debug = [ - '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' - ] - - ldflags = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG' - ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) - - ldflags_debug = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' - ] - - self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] - self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] - self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_static = [*ldflags] - self.ldflags_static_debug = [*ldflags_debug] - - self._ldflags = { - (CCompiler.EXECUTABLE, None): self.ldflags_exe, - (CCompiler.EXECUTABLE, False): self.ldflags_exe, - (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug, - (CCompiler.SHARED_OBJECT, None): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, False): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug, - (CCompiler.SHARED_LIBRARY, None): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, False): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug, - } - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - ext_map = { - **{ext: self.obj_extension for ext in self.src_extensions}, - **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions}, - } - - output_dir = output_dir or '' - - def make_out_path(p): - base, ext = os.path.splitext(p) - if strip_dir: - base = os.path.basename(base) - else: - _, base = os.path.splitdrive(base) - if base.startswith((os.path.sep, os.path.altsep)): - base = base[1:] - try: - # XXX: This may produce absurdly long paths. We should check - # the length of the result and trim base until we fit within - # 260 characters. - return os.path.join(output_dir, base + ext_map[ext]) - except LookupError: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError("Don't know how to compile {}".format(p)) - - return list(map(make_out_path, source_filenames)) - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - - add_cpp_opts = False - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - add_cpp_opts = True - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt, input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src]) - base, _ = os.path.splitext(os.path.basename (src)) - rc_file = os.path.join(rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc, "/fo" + obj, rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile {} to {}" - .format(src, obj)) - - args = [self.cc] + compile_opts + pp_opts - if add_cpp_opts: - args.append('/EHsc') - args.append(input_opt) - args.append("/Fo" + obj) - args.extend(extra_postargs) - - try: - self.spawn(args) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args)) - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - if runtime_library_dirs: - self.warn("I don't know what to do with 'runtime_library_dirs': " - + str(runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ldflags = self._ldflags[target_desc, debug] - - export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])] - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - output_dir = os.path.dirname(os.path.abspath(output_filename)) - self.mkpath(output_dir) - try: - log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) - self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - - def spawn(self, cmd): - old_path = os.getenv('path') - try: - os.environ['path'] = self._paths - return super().spawn(cmd) - finally: - os.environ['path'] = old_path - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC") - - def library_option(self, lib): - return self.library_filename(lib) - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.isfile(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py deleted file mode 100644 index b002dc3b845..00000000000 --- a/Lib/distutils/archive_util.py +++ /dev/null @@ -1,256 +0,0 @@ -"""distutils.archive_util - -Utility functions for creating archive files (tarballs, zip files, -that sort of thing).""" - -import os -from warnings import warn -import sys - -try: - import zipfile -except ImportError: - zipfile = None - - -from distutils.errors import DistutilsExecError -from distutils.spawn import spawn -from distutils.dir_util import mkpath -from distutils import log - -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - -def _get_gid(name): - """Returns a gid, given a group name.""" - if getgrnam is None or name is None: - return None - try: - result = getgrnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def _get_uid(name): - """Returns an uid, given a user name.""" - if getpwnam is None or name is None: - return None - try: - result = getpwnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None): - """Create a (possibly compressed) tar file from all the files under - 'base_dir'. - - 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or - None. ("compress" will be deprecated in Python 3.2) - - 'owner' and 'group' can be used to define an owner and a group for the - archive that is being built. If not provided, the current owner and group - will be used. - - The output tar file will be named 'base_dir' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z"). - - Returns the output filename. - """ - tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '', - 'compress': ''} - compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz', - 'compress': '.Z'} - - # flags for compression program, each element of list will be an argument - if compress is not None and compress not in compress_ext.keys(): - raise ValueError( - "bad value for 'compress': must be None, 'gzip', 'bzip2', " - "'xz' or 'compress'") - - archive_name = base_name + '.tar' - if compress != 'compress': - archive_name += compress_ext.get(compress, '') - - mkpath(os.path.dirname(archive_name), dry_run=dry_run) - - # creating the tarball - import tarfile # late import so Python build itself doesn't break - - log.info('Creating tar archive') - - uid = _get_uid(owner) - gid = _get_gid(group) - - def _set_uid_gid(tarinfo): - if gid is not None: - tarinfo.gid = gid - tarinfo.gname = group - if uid is not None: - tarinfo.uid = uid - tarinfo.uname = owner - return tarinfo - - if not dry_run: - tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) - try: - tar.add(base_dir, filter=_set_uid_gid) - finally: - tar.close() - - # compression using `compress` - if compress == 'compress': - warn("'compress' will be deprecated.", PendingDeprecationWarning) - # the option varies depending on the platform - compressed_name = archive_name + compress_ext[compress] - if sys.platform == 'win32': - cmd = [compress, archive_name, compressed_name] - else: - cmd = [compress, '-f', archive_name] - spawn(cmd, dry_run=dry_run) - return compressed_name - - return archive_name - -def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): - """Create a zip file from all the files under 'base_dir'. - - The output zip file will be named 'base_name' + ".zip". Uses either the - "zipfile" Python module (if available) or the InfoZIP "zip" utility - (if installed and found on the default search path). If neither tool is - available, raises DistutilsExecError. Returns the name of the output zip - file. - """ - zip_filename = base_name + ".zip" - mkpath(os.path.dirname(zip_filename), dry_run=dry_run) - - # If zipfile module is not available, try spawning an external - # 'zip' command. - if zipfile is None: - if verbose: - zipoptions = "-r" - else: - zipoptions = "-rq" - - try: - spawn(["zip", zipoptions, zip_filename, base_dir], - dry_run=dry_run) - except DistutilsExecError: - # XXX really should distinguish between "couldn't find - # external 'zip' command" and "zip failed". - raise DistutilsExecError(("unable to create zip file '%s': " - "could neither import the 'zipfile' module nor " - "find a standalone zip utility") % zip_filename) - - else: - log.info("creating '%s' and adding '%s' to it", - zip_filename, base_dir) - - if not dry_run: - try: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_DEFLATED) - except RuntimeError: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_STORED) - - if base_dir != os.curdir: - path = os.path.normpath(os.path.join(base_dir, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for dirpath, dirnames, filenames in os.walk(base_dir): - for name in dirnames: - path = os.path.normpath(os.path.join(dirpath, name, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) - if os.path.isfile(path): - zip.write(path, path) - log.info("adding '%s'", path) - zip.close() - - return zip_filename - -ARCHIVE_FORMATS = { - 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), - 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), - 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"), - 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"), - 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"), - 'zip': (make_zipfile, [],"ZIP file") - } - -def check_archive_formats(formats): - """Returns the first format from the 'format' list that is unknown. - - If all formats are known, returns None - """ - for format in formats: - if format not in ARCHIVE_FORMATS: - return format - return None - -def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, - dry_run=0, owner=None, group=None): - """Create an archive file (eg. zip or tar). - - 'base_name' is the name of the file to create, minus any format-specific - extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "xztar", or "ztar". - - 'root_dir' is a directory that will be the root directory of the - archive; ie. we typically chdir into 'root_dir' before creating the - archive. 'base_dir' is the directory where we start archiving from; - ie. 'base_dir' will be the common prefix of all files and - directories in the archive. 'root_dir' and 'base_dir' both default - to the current directory. Returns the name of the archive file. - - 'owner' and 'group' are used when creating a tar archive. By default, - uses the current owner and group. - """ - save_cwd = os.getcwd() - if root_dir is not None: - log.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run} - - try: - format_info = ARCHIVE_FORMATS[format] - except KeyError: - raise ValueError("unknown archive format '%s'" % format) - - func = format_info[0] - for arg, val in format_info[1]: - kwargs[arg] = val - - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group - - try: - filename = func(base_name, base_dir, **kwargs) - finally: - if root_dir is not None: - log.debug("changing back to '%s'", save_cwd) - os.chdir(save_cwd) - - return filename diff --git a/Lib/distutils/bcppcompiler.py b/Lib/distutils/bcppcompiler.py deleted file mode 100644 index 9f4c432d90e..00000000000 --- a/Lib/distutils/bcppcompiler.py +++ /dev/null @@ -1,393 +0,0 @@ -"""distutils.bcppcompiler - -Contains BorlandCCompiler, an implementation of the abstract CCompiler class -for the Borland C++ compiler. -""" - -# This implementation by Lyle Johnson, based on the original msvccompiler.py -# module and using the directions originally published by Gordon Williams. - -# XXX looks like there's a LOT of overlap between these two classes: -# someone should sit down and factor out the common code as -# WindowsCCompiler! --GPW - - -import os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError, UnknownFileError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.file_util import write_file -from distutils.dep_util import newer -from distutils import log - -class BCPPCompiler(CCompiler) : - """Concrete class that implements an interface to the Borland C/C++ - compiler, as defined by the CCompiler abstract class. - """ - - compiler_type = 'bcpp' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = _c_extensions + _cpp_extensions - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__ (self, - verbose=0, - dry_run=0, - force=0): - - CCompiler.__init__ (self, verbose, dry_run, force) - - # These executables are assumed to all be in the path. - # Borland doesn't seem to use any special registry settings to - # indicate their installation locations. - - self.cc = "bcc32.exe" - self.linker = "ilink32.exe" - self.lib = "tlib.exe" - - self.preprocess_options = None - self.compile_options = ['/tWM', '/O2', '/q', '/g0'] - self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] - - self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_static = [] - self.ldflags_exe = ['/Gn', '/q', '/x'] - self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] - - - # -- Worker methods ------------------------------------------------ - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - compile_opts = extra_preargs or [] - compile_opts.append ('-c') - if debug: - compile_opts.extend (self.compile_options_debug) - else: - compile_opts.extend (self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - # XXX why do the normpath here? - src = os.path.normpath(src) - obj = os.path.normpath(obj) - # XXX _setup_compile() did a mkpath() too but before the normpath. - # Is it possible to skip the normpath? - self.mkpath(os.path.dirname(obj)) - - if ext == '.res': - # This is already a binary file -- skip it. - continue # the 'for' loop - if ext == '.rc': - # This needs to be compiled to a .res file -- do it now. - try: - self.spawn (["brcc32", "-fo", obj, src]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue # the 'for' loop - - # The next two are both for the real compiler. - if ext in self._c_extensions: - input_opt = "" - elif ext in self._cpp_extensions: - input_opt = "-P" - else: - # Unknown file type -- no extra options. The compiler - # will probably fail, but let it just in case this is a - # file the compiler recognizes even if we don't. - input_opt = "" - - output_opt = "-o" + obj - - # Compiler command line syntax is: "bcc32 [options] file(s)". - # Note that the source file names must appear at the end of - # the command line. - try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs + [src]) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - # compile () - - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) - - if self._need_link (objects, output_filename): - lib_args = [output_filename, '/u'] + objects - if debug: - pass # XXX what goes here? - try: - self.spawn ([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # create_static_lib () - - - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - # XXX this ignores 'build_temp'! should follow the lead of - # msvccompiler.py - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) - - if runtime_library_dirs: - log.warn("I don't know what to do with 'runtime_library_dirs': %s", - str(runtime_library_dirs)) - - if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): - - # Figure out linker args based on type of target. - if target_desc == CCompiler.EXECUTABLE: - startup_obj = 'c0w32' - if debug: - ld_args = self.ldflags_exe_debug[:] - else: - ld_args = self.ldflags_exe[:] - else: - startup_obj = 'c0d32' - if debug: - ld_args = self.ldflags_shared_debug[:] - else: - ld_args = self.ldflags_shared[:] - - - # Create a temporary exports file for use by the linker - if export_symbols is None: - def_file = '' - else: - head, tail = os.path.split (output_filename) - modname, ext = os.path.splitext (tail) - temp_dir = os.path.dirname(objects[0]) # preserve tree structure - def_file = os.path.join (temp_dir, '%s.def' % modname) - contents = ['EXPORTS'] - for sym in (export_symbols or []): - contents.append(' %s=_%s' % (sym, sym)) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # Borland C++ has problems with '/' in paths - objects2 = map(os.path.normpath, objects) - # split objects in .obj and .res files - # Borland C++ needs them at different positions in the command line - objects = [startup_obj] - resources = [] - for file in objects2: - (base, ext) = os.path.splitext(os.path.normcase(file)) - if ext == '.res': - resources.append(file) - else: - objects.append(file) - - - for l in library_dirs: - ld_args.append("/L%s" % os.path.normpath(l)) - ld_args.append("/L.") # we sometimes use relative paths - - # list of object files - ld_args.extend(objects) - - # XXX the command-line syntax for Borland C++ is a bit wonky; - # certain filenames are jammed together in one big string, but - # comma-delimited. This doesn't mesh too well with the - # Unix-centric attitude (with a DOS/Windows quoting hack) of - # 'spawn()', so constructing the argument list is a bit - # awkward. Note that doing the obvious thing and jamming all - # the filenames and commas into one argument would be wrong, - # because 'spawn()' would quote any filenames with spaces in - # them. Arghghh!. Apparently it works fine as coded... - - # name of dll/exe file - ld_args.extend([',',output_filename]) - # no map file and start libraries - ld_args.append(',,') - - for lib in libraries: - # see if we find it and if there is a bcpp specific lib - # (xxx_bcpp.lib) - libfile = self.find_library_file(library_dirs, lib, debug) - if libfile is None: - ld_args.append(lib) - # probably a BCPP internal library -- don't warn - else: - # full name which prefers bcpp_xxx.lib over xxx.lib - ld_args.append(libfile) - - # some default libraries - ld_args.append ('import32') - ld_args.append ('cw32mt') - - # def file for export symbols - ld_args.extend([',',def_file]) - # add resource files - ld_args.append(',') - ld_args.extend(resources) - - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath (os.path.dirname (output_filename)) - try: - self.spawn ([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # link () - - # -- Miscellaneous methods ----------------------------------------- - - - def find_library_file (self, dirs, lib, debug=0): - # List of effective library names to try, in order of preference: - # xxx_bcpp.lib is better than xxx.lib - # and xxx_d.lib is better than xxx.lib if debug is set - # - # The "_bcpp" suffix is to handle a Python installation for people - # with multiple compilers (primarily Distutils hackers, I suspect - # ;-). The idea is they'd have one static library for each - # compiler they care about, since (almost?) every Windows compiler - # seems to have a different format for static libraries. - if debug: - dlib = (lib + "_d") - try_names = (dlib + "_bcpp", lib + "_bcpp", dlib, lib) - else: - try_names = (lib + "_bcpp", lib) - - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # overwrite the one from CCompiler to support rc and res-files - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - (base, ext) = os.path.splitext (os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext == '.res': - # these can go unchanged - obj_names.append (os.path.join (output_dir, base + ext)) - elif ext == '.rc': - # these need to be compiled to .res-files - obj_names.append (os.path.join (output_dir, base + '.res')) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - # object_filenames () - - def preprocess (self, - source, - output_file=None, - macros=None, - include_dirs=None, - extra_preargs=None, - extra_postargs=None): - - (_, macros, include_dirs) = \ - self._fix_compile_args(None, macros, include_dirs) - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = ['cpp32.exe'] + pp_opts - if output_file is not None: - pp_args.append('-o' + output_file) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or the - # source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - print(msg) - raise CompileError(msg) - - # preprocess() diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py deleted file mode 100644 index b71d1d39bcd..00000000000 --- a/Lib/distutils/ccompiler.py +++ /dev/null @@ -1,1115 +0,0 @@ -"""distutils.ccompiler - -Contains CCompiler, an abstract base class that defines the interface -for the Distutils compiler abstraction model.""" - -import sys, os, re -from distutils.errors import * -from distutils.spawn import spawn -from distutils.file_util import move_file -from distutils.dir_util import mkpath -from distutils.dep_util import newer_pairwise, newer_group -from distutils.util import split_quoted, execute -from distutils import log - -class CCompiler: - """Abstract base class to define the interface that must be implemented - by real compiler classes. Also has some utility methods used by - several compiler classes. - - The basic idea behind a compiler abstraction class is that each - instance can be used for all the compile/link steps in building a - single project. Thus, attributes common to all of those compile and - link steps -- include directories, macros to define, libraries to link - against, etc. -- are attributes of the compiler instance. To allow for - variability in how individual files are treated, most of those - attributes may be varied on a per-compilation or per-link basis. - """ - - # 'compiler_type' is a class attribute that identifies this class. It - # keeps code that wants to know what kind of compiler it's dealing with - # from having to import all possible compiler classes just to do an - # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' - # should really, really be one of the keys of the 'compiler_class' - # dictionary (see below -- used by the 'new_compiler()' factory - # function) -- authors of new compiler interface classes are - # responsible for updating 'compiler_class'! - compiler_type = None - - # XXX things not handled by this compiler abstraction model: - # * client can't provide additional options for a compiler, - # e.g. warning, optimization, debugging flags. Perhaps this - # should be the domain of concrete compiler abstraction classes - # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base - # class should have methods for the common ones. - # * can't completely override the include or library searchg - # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". - # I'm not sure how widely supported this is even by Unix - # compilers, much less on other platforms. And I'm even less - # sure how useful it is; maybe for cross-compiling, but - # support for that is a ways off. (And anyways, cross - # compilers probably have a dedicated binary with the - # right paths compiled in. I hope.) - # * can't do really freaky things with the library list/library - # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against - # different versions of libfoo.a in different locations. I - # think this is useless without the ability to null out the - # library search path anyways. - - - # Subclasses that rely on the standard filename generation methods - # implemented below should override these; see the comment near - # those methods ('object_filenames()' et. al.) for details: - src_extensions = None # list of strings - obj_extension = None # string - static_lib_extension = None - shared_lib_extension = None # string - static_lib_format = None # format string - shared_lib_format = None # prob. same as static_lib_format - exe_extension = None # string - - # Default language settings. language_map is used to detect a source - # file or Extension target language, checking source filenames. - # language_order is used to detect the language precedence, when deciding - # what language to use when mixing source types. For example, if some - # extension has two files with ".c" extension, and one with ".cpp", it - # is still linked as c++. - language_map = {".c" : "c", - ".cc" : "c++", - ".cpp" : "c++", - ".cxx" : "c++", - ".m" : "objc", - } - language_order = ["c++", "objc", "c"] - - def __init__(self, verbose=0, dry_run=0, force=0): - self.dry_run = dry_run - self.force = force - self.verbose = verbose - - # 'output_dir': a common output directory for object, library, - # shared object, and shared library files - self.output_dir = None - - # 'macros': a list of macro definitions (or undefinitions). A - # macro definition is a 2-tuple (name, value), where the value is - # either a string or None (no explicit value). A macro - # undefinition is a 1-tuple (name,). - self.macros = [] - - # 'include_dirs': a list of directories to search for include files - self.include_dirs = [] - - # 'libraries': a list of libraries to include in any link - # (library names, not filenames: eg. "foo" not "libfoo.a") - self.libraries = [] - - # 'library_dirs': a list of directories to search for libraries - self.library_dirs = [] - - # 'runtime_library_dirs': a list of directories to search for - # shared libraries/objects at runtime - self.runtime_library_dirs = [] - - # 'objects': a list of object files (or similar, such as explicitly - # named library files) to include on any link - self.objects = [] - - for key in self.executables.keys(): - self.set_executable(key, self.executables[key]) - - def set_executables(self, **kwargs): - """Define the executables (and options for them) that will be run - to perform the various stages of compilation. The exact set of - executables that may be specified here depends on the compiler - class (via the 'executables' class attribute), but most will have: - compiler the C/C++ compiler - linker_so linker used to create shared objects and libraries - linker_exe linker used to create binary executables - archiver static library creator - - On platforms with a command-line (Unix, DOS/Windows), each of these - is a string that will be split into executable name and (optional) - list of arguments. (Splitting the string is done similarly to how - Unix shells operate: words are delimited by spaces, but quotes and - backslashes can override this. See - 'distutils.util.split_quoted()'.) - """ - - # Note that some CCompiler implementation classes will define class - # attributes 'cpp', 'cc', etc. with hard-coded executable names; - # this is appropriate when a compiler class is for exactly one - # compiler/OS combination (eg. MSVCCompiler). Other compiler - # classes (UnixCCompiler, in particular) are driven by information - # discovered at run-time, since there are many different ways to do - # basically the same things with Unix C compilers. - - for key in kwargs: - if key not in self.executables: - raise ValueError("unknown executable '%s' for class %s" % - (key, self.__class__.__name__)) - self.set_executable(key, kwargs[key]) - - def set_executable(self, key, value): - if isinstance(value, str): - setattr(self, key, split_quoted(value)) - else: - setattr(self, key, value) - - def _find_macro(self, name): - i = 0 - for defn in self.macros: - if defn[0] == name: - return i - i += 1 - return None - - def _check_macro_definitions(self, definitions): - """Ensures that every element of 'definitions' is a valid macro - definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do - nothing if all definitions are OK, raise TypeError otherwise. - """ - for defn in definitions: - if not (isinstance(defn, tuple) and - (len(defn) in (1, 2) and - (isinstance (defn[1], str) or defn[1] is None)) and - isinstance (defn[0], str)): - raise TypeError(("invalid macro definition '%s': " % defn) + \ - "must be tuple (string,), (string, string), or " + \ - "(string, None)") - - - # -- Bookkeeping methods ------------------------------------------- - - def define_macro(self, name, value=None): - """Define a preprocessor macro for all compilations driven by this - compiler object. The optional parameter 'value' should be a - string; if it is not supplied, then the macro will be defined - without an explicit value and the exact outcome depends on the - compiler used (XXX true? does ANSI say anything about this?) - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - self.macros.append((name, value)) - - def undefine_macro(self, name): - """Undefine a preprocessor macro for all compilations driven by - this compiler object. If the same macro is defined by - 'define_macro()' and undefined by 'undefine_macro()' the last call - takes precedence (including multiple redefinitions or - undefinitions). If the macro is redefined/undefined on a - per-compilation basis (ie. in the call to 'compile()'), then that - takes precedence. - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - undefn = (name,) - self.macros.append(undefn) - - def add_include_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - header files. The compiler is instructed to search directories in - the order in which they are supplied by successive calls to - 'add_include_dir()'. - """ - self.include_dirs.append(dir) - - def set_include_dirs(self, dirs): - """Set the list of directories that will be searched to 'dirs' (a - list of strings). Overrides any preceding calls to - 'add_include_dir()'; subsequence calls to 'add_include_dir()' add - to the list passed to 'set_include_dirs()'. This does not affect - any list of standard include directories that the compiler may - search by default. - """ - self.include_dirs = dirs[:] - - def add_library(self, libname): - """Add 'libname' to the list of libraries that will be included in - all links driven by this compiler object. Note that 'libname' - should *not* be the name of a file containing a library, but the - name of the library itself: the actual filename will be inferred by - the linker, the compiler, or the compiler class (depending on the - platform). - - The linker will be instructed to link against libraries in the - order they were supplied to 'add_library()' and/or - 'set_libraries()'. It is perfectly valid to duplicate library - names; the linker will be instructed to link against libraries as - many times as they are mentioned. - """ - self.libraries.append(libname) - - def set_libraries(self, libnames): - """Set the list of libraries to be included in all links driven by - this compiler object to 'libnames' (a list of strings). This does - not affect any standard system libraries that the linker may - include by default. - """ - self.libraries = libnames[:] - - def add_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - libraries specified to 'add_library()' and 'set_libraries()'. The - linker will be instructed to search for libraries in the order they - are supplied to 'add_library_dir()' and/or 'set_library_dirs()'. - """ - self.library_dirs.append(dir) - - def set_library_dirs(self, dirs): - """Set the list of library search directories to 'dirs' (a list of - strings). This does not affect any standard library search path - that the linker may search by default. - """ - self.library_dirs = dirs[:] - - def add_runtime_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - shared libraries at runtime. - """ - self.runtime_library_dirs.append(dir) - - def set_runtime_library_dirs(self, dirs): - """Set the list of directories to search for shared libraries at - runtime to 'dirs' (a list of strings). This does not affect any - standard search path that the runtime linker may search by - default. - """ - self.runtime_library_dirs = dirs[:] - - def add_link_object(self, object): - """Add 'object' to the list of object files (or analogues, such as - explicitly named library files or the output of "resource - compilers") to be included in every link driven by this compiler - object. - """ - self.objects.append(object) - - def set_link_objects(self, objects): - """Set the list of object files (or analogues) to be included in - every link to 'objects'. This does not affect any standard object - files that the linker may include by default (such as system - libraries). - """ - self.objects = objects[:] - - - # -- Private utility methods -------------------------------------- - # (here for the convenience of subclasses) - - # Helper method to prep compiler in subclass compile() methods - - def _setup_compile(self, outdir, macros, incdirs, sources, depends, - extra): - """Process arguments and decide which source files to compile.""" - if outdir is None: - outdir = self.output_dir - elif not isinstance(outdir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if incdirs is None: - incdirs = self.include_dirs - elif isinstance(incdirs, (list, tuple)): - incdirs = list(incdirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - if extra is None: - extra = [] - - # Get the list of expected output (object) files - objects = self.object_filenames(sources, strip_dir=0, - output_dir=outdir) - assert len(objects) == len(sources) - - pp_opts = gen_preprocess_options(macros, incdirs) - - build = {} - for i in range(len(sources)): - src = sources[i] - obj = objects[i] - ext = os.path.splitext(src)[1] - self.mkpath(os.path.dirname(obj)) - build[obj] = (src, ext) - - return macros, objects, extra, pp_opts, build - - def _get_cc_args(self, pp_opts, debug, before): - # works for unixccompiler, cygwinccompiler - cc_args = pp_opts + ['-c'] - if debug: - cc_args[:0] = ['-g'] - if before: - cc_args[:0] = before - return cc_args - - def _fix_compile_args(self, output_dir, macros, include_dirs): - """Typecheck and fix-up some of the arguments to the 'compile()' - method, and return fixed-up values. Specifically: if 'output_dir' - is None, replaces it with 'self.output_dir'; ensures that 'macros' - is a list, and augments it with 'self.macros'; ensures that - 'include_dirs' is a list, and augments it with 'self.include_dirs'. - Guarantees that the returned values are of the correct type, - i.e. for 'output_dir' either string or None, and for 'macros' and - 'include_dirs' either list or None. - """ - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if include_dirs is None: - include_dirs = self.include_dirs - elif isinstance(include_dirs, (list, tuple)): - include_dirs = list(include_dirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - return output_dir, macros, include_dirs - - def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. - - Determine the list of object files corresponding to 'sources', - and figure out which ones really need to be recompiled. - Return a list of all object files and a dictionary telling - which source files can be skipped. - """ - # Get the list of expected output (object) files - objects = self.object_filenames(sources, output_dir=output_dir) - assert len(objects) == len(sources) - - # Return an empty dict for the "which source files can be skipped" - # return value to preserve API compatibility. - return objects, {} - - def _fix_object_args(self, objects, output_dir): - """Typecheck and fix up some arguments supplied to various methods. - Specifically: ensure that 'objects' is a list; if output_dir is - None, replace with self.output_dir. Return fixed versions of - 'objects' and 'output_dir'. - """ - if not isinstance(objects, (list, tuple)): - raise TypeError("'objects' must be a list or tuple of strings") - objects = list(objects) - - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - return (objects, output_dir) - - def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): - """Typecheck and fix up some of the arguments supplied to the - 'link_*' methods. Specifically: ensure that all arguments are - lists, and augment them with their permanent versions - (eg. 'self.libraries' augments 'libraries'). Return a tuple with - fixed versions of all arguments. - """ - if libraries is None: - libraries = self.libraries - elif isinstance(libraries, (list, tuple)): - libraries = list (libraries) + (self.libraries or []) - else: - raise TypeError( - "'libraries' (if supplied) must be a list of strings") - - if library_dirs is None: - library_dirs = self.library_dirs - elif isinstance(library_dirs, (list, tuple)): - library_dirs = list (library_dirs) + (self.library_dirs or []) - else: - raise TypeError( - "'library_dirs' (if supplied) must be a list of strings") - - if runtime_library_dirs is None: - runtime_library_dirs = self.runtime_library_dirs - elif isinstance(runtime_library_dirs, (list, tuple)): - runtime_library_dirs = (list(runtime_library_dirs) + - (self.runtime_library_dirs or [])) - else: - raise TypeError("'runtime_library_dirs' (if supplied) " - "must be a list of strings") - - return (libraries, library_dirs, runtime_library_dirs) - - def _need_link(self, objects, output_file): - """Return true if we need to relink the files listed in 'objects' - to recreate 'output_file'. - """ - if self.force: - return True - else: - if self.dry_run: - newer = newer_group (objects, output_file, missing='newer') - else: - newer = newer_group (objects, output_file) - return newer - - def detect_language(self, sources): - """Detect the language of a given file, or list of files. Uses - language_map, and language_order to do the job. - """ - if not isinstance(sources, list): - sources = [sources] - lang = None - index = len(self.language_order) - for source in sources: - base, ext = os.path.splitext(source) - extlang = self.language_map.get(ext) - try: - extindex = self.language_order.index(extlang) - if extindex < index: - lang = extlang - index = extindex - except ValueError: - pass - return lang - - - # -- Worker methods ------------------------------------------------ - # (must be implemented by subclasses) - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - """Preprocess a single C/C++ source file, named in 'source'. - Output will be written to file named 'output_file', or stdout if - 'output_file' not supplied. 'macros' is a list of macro - definitions as for 'compile()', which will augment the macros set - with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a - list of directory names that will be added to the default list. - - Raises PreprocessError on failure. - """ - pass - - def compile(self, sources, output_dir=None, macros=None, - include_dirs=None, debug=0, extra_preargs=None, - extra_postargs=None, depends=None): - """Compile one or more source files. - - 'sources' must be a list of filenames, most likely C/C++ - files, but in reality anything that can be handled by a - particular compiler and compiler class (eg. MSVCCompiler can - handle resource files in 'sources'). Return a list of object - filenames, one per source filename in 'sources'. Depending on - the implementation, not all source files will necessarily be - compiled, but all corresponding object filenames will be - returned. - - If 'output_dir' is given, object files will be put under it, while - retaining their original path component. That is, "foo/bar.c" - normally compiles to "foo/bar.o" (for a Unix implementation); if - 'output_dir' is "build", then it would compile to - "build/foo/bar.o". - - 'macros', if given, must be a list of macro definitions. A macro - definition is either a (name, value) 2-tuple or a (name,) 1-tuple. - The former defines a macro; if the value is None, the macro is - defined without an explicit value. The 1-tuple case undefines a - macro. Later definitions/redefinitions/ undefinitions take - precedence. - - 'include_dirs', if given, must be a list of strings, the - directories to add to the default include file search path for this - compilation only. - - 'debug' is a boolean; if true, the compiler will be instructed to - output debug symbols in (or alongside) the object file(s). - - 'extra_preargs' and 'extra_postargs' are implementation- dependent. - On platforms that have the notion of a command-line (e.g. Unix, - DOS/Windows), they are most likely lists of strings: extra - command-line arguments to prepand/append to the compiler command - line. On other platforms, consult the implementation class - documentation. In any event, they are intended as an escape hatch - for those occasions when the abstract compiler framework doesn't - cut the mustard. - - 'depends', if given, is a list of filenames that all targets - depend on. If a source file is older than any file in - depends, then the source file will be recompiled. This - supports dependency tracking, but only at a coarse - granularity. - - Raises CompileError on failure. - """ - # A concrete compiler class can either override this method - # entirely or implement _compile(). - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # Return *all* object filenames, not just the ones we just built. - return objects - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compile 'src' to product 'obj'.""" - # A concrete compiler class that does not override compile() - # should implement _compile(). - pass - - def create_static_lib(self, objects, output_libname, output_dir=None, - debug=0, target_lang=None): - """Link a bunch of stuff together to create a static library file. - The "bunch of stuff" consists of the list of object files supplied - as 'objects', the extra object files supplied to - 'add_link_object()' and/or 'set_link_objects()', the libraries - supplied to 'add_library()' and/or 'set_libraries()', and the - libraries supplied as 'libraries' (if any). - - 'output_libname' should be a library name, not a filename; the - filename will be inferred from the library name. 'output_dir' is - the directory where the library file will be put. - - 'debug' is a boolean; if true, debugging information will be - included in the library (note that on most platforms, it is the - compile step where this matters: the 'debug' flag is included here - just for consistency). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LibError on failure. - """ - pass - - - # values for target_desc parameter in link() - SHARED_OBJECT = "shared_object" - SHARED_LIBRARY = "shared_library" - EXECUTABLE = "executable" - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - """Link a bunch of stuff together to create an executable or - shared library file. - - The "bunch of stuff" consists of the list of object files supplied - as 'objects'. 'output_filename' should be a filename. If - 'output_dir' is supplied, 'output_filename' is relative to it - (i.e. 'output_filename' can provide directory components if - needed). - - 'libraries' is a list of libraries to link against. These are - library names, not filenames, since they're translated into - filenames in a platform-specific way (eg. "foo" becomes "libfoo.a" - on Unix and "foo.lib" on DOS/Windows). However, they can include a - directory component, which means the linker will look in that - specific directory rather than searching all the normal locations. - - 'library_dirs', if supplied, should be a list of directories to - search for libraries that were specified as bare library names - (ie. no directory component). These are on top of the system - default and those supplied to 'add_library_dir()' and/or - 'set_library_dirs()'. 'runtime_library_dirs' is a list of - directories that will be embedded into the shared library and used - to search for other shared libraries that *it* depends on at - run-time. (This may only be relevant on Unix.) - - 'export_symbols' is a list of symbols that the shared library will - export. (This appears to be relevant only on Windows.) - - 'debug' is as for 'compile()' and 'create_static_lib()', with the - slight distinction that it actually matters on most platforms (as - opposed to 'create_static_lib()', which includes a 'debug' flag - mostly for form's sake). - - 'extra_preargs' and 'extra_postargs' are as for 'compile()' (except - of course that they supply command-line arguments for the - particular linker being used). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LinkError on failure. - """ - raise NotImplementedError - - - # Old 'link_*()' methods, rewritten to use the new 'link()' method. - - def link_shared_lib(self, - objects, - output_libname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_LIBRARY, objects, - self.library_filename(output_libname, lib_type='shared'), - output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_shared_object(self, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_OBJECT, objects, - output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_executable(self, - objects, - output_progname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - target_lang=None): - self.link(CCompiler.EXECUTABLE, objects, - self.executable_filename(output_progname), output_dir, - libraries, library_dirs, runtime_library_dirs, None, - debug, extra_preargs, extra_postargs, None, target_lang) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function; there is - # no appropriate default implementation so subclasses should - # implement all of these. - - def library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for libraries. - """ - raise NotImplementedError - - def runtime_library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for runtime libraries. - """ - raise NotImplementedError - - def library_option(self, lib): - """Return the compiler option to add 'lib' to the list of libraries - linked into the shared library or executable. - """ - raise NotImplementedError - - def has_function(self, funcname, includes=None, include_dirs=None, - libraries=None, library_dirs=None): - """Return a boolean indicating whether funcname is supported on - the current platform. The optional arguments can be used to - augment the compilation environment. - """ - # this can't be included at module scope because it tries to - # import math which might not be available at that point - maybe - # the necessary logic should just be inlined? - import tempfile - if includes is None: - includes = [] - if include_dirs is None: - include_dirs = [] - if libraries is None: - libraries = [] - if library_dirs is None: - library_dirs = [] - fd, fname = tempfile.mkstemp(".c", funcname, text=True) - f = os.fdopen(fd, "w") - try: - for incl in includes: - f.write("""#include "%s"\n""" % incl) - f.write("""\ -main (int argc, char **argv) { - %s(); -} -""" % funcname) - finally: - f.close() - try: - objects = self.compile([fname], include_dirs=include_dirs) - except CompileError: - return False - - try: - self.link_executable(objects, "a.out", - libraries=libraries, - library_dirs=library_dirs) - except (LinkError, TypeError): - return False - return True - - def find_library_file (self, dirs, lib, debug=0): - """Search the specified list of directories for a static or shared - library file 'lib' and return the full path to that file. If - 'debug' true, look for a debugging version (if that makes sense on - the current platform). Return None if 'lib' wasn't found in any of - the specified directories. - """ - raise NotImplementedError - - # -- Filename generation methods ----------------------------------- - - # The default implementation of the filename generating methods are - # prejudiced towards the Unix/DOS/Windows view of the world: - # * object files are named by replacing the source file extension - # (eg. .c/.cpp -> .o/.obj) - # * library files (shared or static) are named by plugging the - # library name and extension into a format string, eg. - # "lib%s.%s" % (lib_name, ".a") for Unix static libraries - # * executables are named by appending an extension (possibly - # empty) to the program name: eg. progname + ".exe" for - # Windows - # - # To reduce redundant code, these methods expect to find - # several attributes in the current object (presumably defined - # as class attributes): - # * src_extensions - - # list of C/C++ source file extensions, eg. ['.c', '.cpp'] - # * obj_extension - - # object file extension, eg. '.o' or '.obj' - # * static_lib_extension - - # extension for static library files, eg. '.a' or '.lib' - # * shared_lib_extension - - # extension for shared library/object files, eg. '.so', '.dll' - # * static_lib_format - - # format string for generating static library filenames, - # eg. 'lib%s.%s' or '%s.%s' - # * shared_lib_format - # format string for generating shared library filenames - # (probably same as static_lib_format, since the extension - # is one of the intended parameters to the format string) - # * exe_extension - - # extension for executable files, eg. '' or '.exe' - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - base, ext = os.path.splitext(src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - raise UnknownFileError( - "unknown file type '%s' (from '%s')" % (ext, src_name)) - if strip_dir: - base = os.path.basename(base) - obj_names.append(os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - - def shared_object_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + self.shared_lib_extension) - - def executable_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + (self.exe_extension or '')) - - def library_filename(self, libname, lib_type='static', # or 'shared' - strip_dir=0, output_dir=''): - assert output_dir is not None - if lib_type not in ("static", "shared", "dylib", "xcode_stub"): - raise ValueError( - "'lib_type' must be \"static\", \"shared\", \"dylib\", or \"xcode_stub\"") - fmt = getattr(self, lib_type + "_lib_format") - ext = getattr(self, lib_type + "_lib_extension") - - dir, base = os.path.split(libname) - filename = fmt % (base, ext) - if strip_dir: - dir = '' - - return os.path.join(output_dir, dir, filename) - - - # -- Utility methods ----------------------------------------------- - - def announce(self, msg, level=1): - log.debug(msg) - - def debug_print(self, msg): - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - def warn(self, msg): - sys.stderr.write("warning: %s\n" % msg) - - def execute(self, func, args, msg=None, level=1): - execute(func, args, msg, self.dry_run) - - def spawn(self, cmd): - spawn(cmd, dry_run=self.dry_run) - - def move_file(self, src, dst): - return move_file(src, dst, dry_run=self.dry_run) - - def mkpath (self, name, mode=0o777): - mkpath(name, mode, dry_run=self.dry_run) - - -# Map a sys.platform/os.name ('posix', 'nt') to the default compiler -# type for that platform. Keys are interpreted as re match -# patterns. Order is important; platform mappings are preferred over -# OS names. -_default_compilers = ( - - # Platform string mappings - - # on a cygwin built python we can use gcc like an ordinary UNIXish - # compiler - ('cygwin.*', 'unix'), - - # OS name mappings - ('posix', 'unix'), - ('nt', 'msvc'), - - ) - -def get_default_compiler(osname=None, platform=None): - """Determine the default compiler to use for the given platform. - - osname should be one of the standard Python OS names (i.e. the - ones returned by os.name) and platform the common value - returned by sys.platform for the platform in question. - - The default values are os.name and sys.platform in case the - parameters are not given. - """ - if osname is None: - osname = os.name - if platform is None: - platform = sys.platform - for pattern, compiler in _default_compilers: - if re.match(pattern, platform) is not None or \ - re.match(pattern, osname) is not None: - return compiler - # Default to Unix compiler - return 'unix' - -# Map compiler types to (module_name, class_name) pairs -- ie. where to -# find the code that implements an interface to this compiler. (The module -# is assumed to be in the 'distutils' package.) -compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', - "standard UNIX-style compiler"), - 'msvc': ('_msvccompiler', 'MSVCCompiler', - "Microsoft Visual C++"), - 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', - "Cygwin port of GNU C Compiler for Win32"), - 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', - "Mingw32 port of GNU C Compiler for Win32"), - 'bcpp': ('bcppcompiler', 'BCPPCompiler', - "Borland C++ Compiler"), - } - -def show_compilers(): - """Print list of available compilers (used by the "--help-compiler" - options to "build", "build_ext", "build_clib"). - """ - # XXX this "knows" that the compiler option it's describing is - # "--compiler", which just happens to be the case for the three - # commands that use it. - from distutils.fancy_getopt import FancyGetopt - compilers = [] - for compiler in compiler_class.keys(): - compilers.append(("compiler="+compiler, None, - compiler_class[compiler][2])) - compilers.sort() - pretty_printer = FancyGetopt(compilers) - pretty_printer.print_help("List of available compilers:") - - -def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): - """Generate an instance of some CCompiler subclass for the supplied - platform/compiler combination. 'plat' defaults to 'os.name' - (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler - for that platform. Currently only 'posix' and 'nt' are supported, and - the default compilers are "traditional Unix interface" (UnixCCompiler - class) and Visual C++ (MSVCCompiler class). Note that it's perfectly - possible to ask for a Unix compiler object under Windows, and a - Microsoft compiler object under Unix -- if you supply a value for - 'compiler', 'plat' is ignored. - """ - if plat is None: - plat = os.name - - try: - if compiler is None: - compiler = get_default_compiler(plat) - - (module_name, class_name, long_description) = compiler_class[compiler] - except KeyError: - msg = "don't know how to compile C/C++ code on platform '%s'" % plat - if compiler is not None: - msg = msg + " with '%s' compiler" % compiler - raise DistutilsPlatformError(msg) - - try: - module_name = "distutils." + module_name - __import__ (module_name) - module = sys.modules[module_name] - klass = vars(module)[class_name] - except ImportError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to load module '%s'" % \ - module_name) - except KeyError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to find class '%s' " - "in module '%s'" % (class_name, module_name)) - - # XXX The None is necessary to preserve backwards compatibility - # with classes that expect verbose to be the first positional - # argument. - return klass(None, dry_run, force) - - -def gen_preprocess_options(macros, include_dirs): - """Generate C pre-processor options (-D, -U, -I) as used by at least - two types of compilers: the typical Unix compiler and Visual C++. - 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) - means undefine (-U) macro 'name', and (name,value) means define (-D) - macro 'name' to 'value'. 'include_dirs' is just a list of directory - names to be added to the header file search path (-I). Returns a list - of command-line options suitable for either Unix compilers or Visual - C++. - """ - # XXX it would be nice (mainly aesthetic, and so we don't generate - # stupid-looking command lines) to go over 'macros' and eliminate - # redundant definitions/undefinitions (ie. ensure that only the - # latest mention of a particular macro winds up on the command - # line). I don't think it's essential, though, since most (all?) - # Unix C compilers only pay attention to the latest -D or -U - # mention of a macro on their command line. Similar situation for - # 'include_dirs'. I'm punting on both for now. Anyways, weeding out - # redundancies like this should probably be the province of - # CCompiler, since the data structures used are inherited from it - # and therefore common to all CCompiler classes. - pp_opts = [] - for macro in macros: - if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): - raise TypeError( - "bad macro definition '%s': " - "each element of 'macros' list must be a 1- or 2-tuple" - % macro) - - if len(macro) == 1: # undefine this macro - pp_opts.append("-U%s" % macro[0]) - elif len(macro) == 2: - if macro[1] is None: # define with no explicit value - pp_opts.append("-D%s" % macro[0]) - else: - # XXX *don't* need to be clever about quoting the - # macro value here, because we're going to avoid the - # shell at all costs when we spawn the command! - pp_opts.append("-D%s=%s" % macro) - - for dir in include_dirs: - pp_opts.append("-I%s" % dir) - return pp_opts - - -def gen_lib_options (compiler, library_dirs, runtime_library_dirs, libraries): - """Generate linker options for searching library directories and - linking with specific libraries. 'libraries' and 'library_dirs' are, - respectively, lists of library names (not filenames!) and search - directories. Returns a list of command-line options suitable for use - with some compiler (depending on the two format strings passed in). - """ - lib_opts = [] - - for dir in library_dirs: - lib_opts.append(compiler.library_dir_option(dir)) - - for dir in runtime_library_dirs: - opt = compiler.runtime_library_dir_option(dir) - if isinstance(opt, list): - lib_opts = lib_opts + opt - else: - lib_opts.append(opt) - - # XXX it's important that we *not* remove redundant library mentions! - # sometimes you really do have to say "-lfoo -lbar -lfoo" in order to - # resolve all symbols. I just hope we never have to say "-lfoo obj.o - # -lbar" to get things to work -- that's certainly a possibility, but a - # pretty nasty way to arrange your C code. - - for lib in libraries: - (lib_dir, lib_name) = os.path.split(lib) - if lib_dir: - lib_file = compiler.find_library_file([lib_dir], lib_name) - if lib_file: - lib_opts.append(lib_file) - else: - compiler.warn("no library file corresponding to " - "'%s' found (skipping)" % lib) - else: - lib_opts.append(compiler.library_option (lib)) - return lib_opts diff --git a/Lib/distutils/cmd.py b/Lib/distutils/cmd.py deleted file mode 100644 index 939f7959457..00000000000 --- a/Lib/distutils/cmd.py +++ /dev/null @@ -1,434 +0,0 @@ -"""distutils.cmd - -Provides the Command class, the base class for the command classes -in the distutils.command package. -""" - -import sys, os, re -from distutils.errors import DistutilsOptionError -from distutils import util, dir_util, file_util, archive_util, dep_util -from distutils import log - -class Command: - """Abstract base class for defining command classes, the "worker bees" - of the Distutils. A useful analogy for command classes is to think of - them as subroutines with local variables called "options". The options - are "declared" in 'initialize_options()' and "defined" (given their - final values, aka "finalized") in 'finalize_options()', both of which - must be defined by every command class. The distinction between the - two is necessary because option values might come from the outside - world (command line, config file, ...), and any options dependent on - other options must be computed *after* these outside influences have - been processed -- hence 'finalize_options()'. The "body" of the - subroutine, where it does all its work based on the values of its - options, is the 'run()' method, which must also be implemented by every - command class. - """ - - # 'sub_commands' formalizes the notion of a "family" of commands, - # eg. "install" as the parent with sub-commands "install_lib", - # "install_headers", etc. The parent of a family of commands - # defines 'sub_commands' as a class attribute; it's a list of - # (command_name : string, predicate : unbound_method | string | None) - # tuples, where 'predicate' is a method of the parent command that - # determines whether the corresponding command is applicable in the - # current situation. (Eg. we "install_headers" is only applicable if - # we have any C header files to install.) If 'predicate' is None, - # that command is always applicable. - # - # 'sub_commands' is usually defined at the *end* of a class, because - # predicates can be unbound methods, so they must already have been - # defined. The canonical example is the "install" command. - sub_commands = [] - - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, dist): - """Create and initialize a new Command object. Most importantly, - invokes the 'initialize_options()' method, which is the real - initializer and depends on the actual command being - instantiated. - """ - # late import because of mutual dependence between these classes - from distutils.dist import Distribution - - if not isinstance(dist, Distribution): - raise TypeError("dist must be a Distribution instance") - if self.__class__ is Command: - raise RuntimeError("Command is an abstract class") - - self.distribution = dist - self.initialize_options() - - # Per-command versions of the global flags, so that the user can - # customize Distutils' behaviour command-by-command and let some - # commands fall back on the Distribution's behaviour. None means - # "not defined, check self.distribution's copy", while 0 or 1 mean - # false and true (duh). Note that this means figuring out the real - # value of each flag is a touch complicated -- hence "self._dry_run" - # will be handled by __getattr__, below. - # XXX This needs to be fixed. - self._dry_run = None - - # verbose is largely ignored, but needs to be set for - # backwards compatibility (I think)? - self.verbose = dist.verbose - - # Some commands define a 'self.force' option to ignore file - # timestamps, but methods defined *here* assume that - # 'self.force' exists for all commands. So define it here - # just to be safe. - self.force = None - - # The 'help' flag is just used for command-line parsing, so - # none of that complicated bureaucracy is needed. - self.help = 0 - - # 'finalized' records whether or not 'finalize_options()' has been - # called. 'finalize_options()' itself should not pay attention to - # this flag: it is the business of 'ensure_finalized()', which - # always calls 'finalize_options()', to respect/update it. - self.finalized = 0 - - # XXX A more explicit way to customize dry_run would be better. - def __getattr__(self, attr): - if attr == 'dry_run': - myval = getattr(self, "_" + attr) - if myval is None: - return getattr(self.distribution, attr) - else: - return myval - else: - raise AttributeError(attr) - - def ensure_finalized(self): - if not self.finalized: - self.finalize_options() - self.finalized = 1 - - # Subclasses must define: - # initialize_options() - # provide default values for all options; may be customized by - # setup script, by options from config file(s), or by command-line - # options - # finalize_options() - # decide on the final values for all options; this is called - # after all possible intervention from the outside world - # (command-line, option file, etc.) has been processed - # run() - # run the command: do whatever it is we're here to do, - # controlled by the command's various option values - - def initialize_options(self): - """Set default values for all the options that this command - supports. Note that these defaults may be overridden by other - commands, by the setup script, by config files, or by the - command-line. Thus, this is not the place to code dependencies - between options; generally, 'initialize_options()' implementations - are just a bunch of "self.foo = None" assignments. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def finalize_options(self): - """Set final values for all the options that this command supports. - This is always called as late as possible, ie. after any option - assignments from the command-line or from other commands have been - done. Thus, this is the place to code option dependencies: if - 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as - long as 'foo' still has the same value it was assigned in - 'initialize_options()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - - def dump_options(self, header=None, indent=""): - from distutils.fancy_getopt import longopt_xlate - if header is None: - header = "command options for '%s':" % self.get_command_name() - self.announce(indent + header, level=log.INFO) - indent = indent + " " - for (option, _, _) in self.user_options: - option = option.translate(longopt_xlate) - if option[-1] == "=": - option = option[:-1] - value = getattr(self, option) - self.announce(indent + "%s = %s" % (option, value), - level=log.INFO) - - def run(self): - """A command's raison d'etre: carry out the action it exists to - perform, controlled by the options initialized in - 'initialize_options()', customized by other commands, the setup - script, the command-line, and config files, and finalized in - 'finalize_options()'. All terminal output and filesystem - interaction should be done by 'run()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def announce(self, msg, level=1): - """If the current verbosity level is of greater than or equal to - 'level' print 'msg' to stdout. - """ - log.log(level, msg) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - sys.stdout.flush() - - - # -- Option validation methods ------------------------------------- - # (these are very handy in writing the 'finalize_options()' method) - # - # NB. the general philosophy here is to ensure that a particular option - # value meets certain type and value constraints. If not, we try to - # force it into conformance (eg. if we expect a list but have a string, - # split the string on comma and/or whitespace). If we can't force the - # option into conformance, raise DistutilsOptionError. Thus, command - # classes need do nothing more than (eg.) - # self.ensure_string_list('foo') - # and they can be guaranteed that thereafter, self.foo will be - # a list of strings. - - def _ensure_stringlike(self, option, what, default=None): - val = getattr(self, option) - if val is None: - setattr(self, option, default) - return default - elif not isinstance(val, str): - raise DistutilsOptionError("'%s' must be a %s (got `%s`)" - % (option, what, val)) - return val - - def ensure_string(self, option, default=None): - """Ensure that 'option' is a string; if not defined, set it to - 'default'. - """ - self._ensure_stringlike(option, "string", default) - - def ensure_string_list(self, option): - r"""Ensure that 'option' is a list of strings. If 'option' is - currently a string, we split it either on /,\s*/ or /\s+/, so - "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become - ["foo", "bar", "baz"]. - """ - val = getattr(self, option) - if val is None: - return - elif isinstance(val, str): - setattr(self, option, re.split(r',\s*|\s+', val)) - else: - if isinstance(val, list): - ok = all(isinstance(v, str) for v in val) - else: - ok = False - if not ok: - raise DistutilsOptionError( - "'%s' must be a list of strings (got %r)" - % (option, val)) - - def _ensure_tested_string(self, option, tester, what, error_fmt, - default=None): - val = self._ensure_stringlike(option, what, default) - if val is not None and not tester(val): - raise DistutilsOptionError(("error in '%s' option: " + error_fmt) - % (option, val)) - - def ensure_filename(self, option): - """Ensure that 'option' is the name of an existing file.""" - self._ensure_tested_string(option, os.path.isfile, - "filename", - "'%s' does not exist or is not a file") - - def ensure_dirname(self, option): - self._ensure_tested_string(option, os.path.isdir, - "directory name", - "'%s' does not exist or is not a directory") - - - # -- Convenience methods for commands ------------------------------ - - def get_command_name(self): - if hasattr(self, 'command_name'): - return self.command_name - else: - return self.__class__.__name__ - - def set_undefined_options(self, src_cmd, *option_pairs): - """Set the values of any "undefined" options from corresponding - option values in some other command object. "Undefined" here means - "is None", which is the convention used to indicate that an option - has not been changed between 'initialize_options()' and - 'finalize_options()'. Usually called from 'finalize_options()' for - options that depend on some other command rather than another - option of the same command. 'src_cmd' is the other command from - which option values will be taken (a command object will be created - for it if necessary); the remaining arguments are - '(src_option,dst_option)' tuples which mean "take the value of - 'src_option' in the 'src_cmd' command object, and copy it to - 'dst_option' in the current command object". - """ - # Option_pairs: list of (src_option, dst_option) tuples - src_cmd_obj = self.distribution.get_command_obj(src_cmd) - src_cmd_obj.ensure_finalized() - for (src_option, dst_option) in option_pairs: - if getattr(self, dst_option) is None: - setattr(self, dst_option, getattr(src_cmd_obj, src_option)) - - def get_finalized_command(self, command, create=1): - """Wrapper around Distribution's 'get_command_obj()' method: find - (create if necessary and 'create' is true) the command object for - 'command', call its 'ensure_finalized()' method, and return the - finalized command object. - """ - cmd_obj = self.distribution.get_command_obj(command, create) - cmd_obj.ensure_finalized() - return cmd_obj - - # XXX rename to 'get_reinitialized_command()'? (should do the - # same in dist.py, if so) - def reinitialize_command(self, command, reinit_subcommands=0): - return self.distribution.reinitialize_command(command, - reinit_subcommands) - - def run_command(self, command): - """Run some other command: uses the 'run_command()' method of - Distribution, which creates and finalizes the command object if - necessary and then invokes its 'run()' method. - """ - self.distribution.run_command(command) - - def get_sub_commands(self): - """Determine the sub-commands that are relevant in the current - distribution (ie., that need to be run). This is based on the - 'sub_commands' class attribute: each tuple in that list may include - a method that we call to determine if the subcommand needs to be - run for the current distribution. Return a list of command names. - """ - commands = [] - for (cmd_name, method) in self.sub_commands: - if method is None or method(self): - commands.append(cmd_name) - return commands - - - # -- External world manipulation ----------------------------------- - - def warn(self, msg): - log.warn("warning: %s: %s\n", self.get_command_name(), msg) - - def execute(self, func, args, msg=None, level=1): - util.execute(func, args, msg, dry_run=self.dry_run) - - def mkpath(self, name, mode=0o777): - dir_util.mkpath(name, mode, dry_run=self.dry_run) - - def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, - link=None, level=1): - """Copy a file respecting verbose, dry-run and force flags. (The - former two default to whatever is in the Distribution object, and - the latter defaults to false for commands that don't define it.)""" - return file_util.copy_file(infile, outfile, preserve_mode, - preserve_times, not self.force, link, - dry_run=self.dry_run) - - def copy_tree(self, infile, outfile, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, level=1): - """Copy an entire directory tree respecting verbose, dry-run, - and force flags. - """ - return dir_util.copy_tree(infile, outfile, preserve_mode, - preserve_times, preserve_symlinks, - not self.force, dry_run=self.dry_run) - - def move_file (self, src, dst, level=1): - """Move a file respecting dry-run flag.""" - return file_util.move_file(src, dst, dry_run=self.dry_run) - - def spawn(self, cmd, search_path=1, level=1): - """Spawn an external command respecting dry-run flag.""" - from distutils.spawn import spawn - spawn(cmd, search_path, dry_run=self.dry_run) - - def make_archive(self, base_name, format, root_dir=None, base_dir=None, - owner=None, group=None): - return archive_util.make_archive(base_name, format, root_dir, base_dir, - dry_run=self.dry_run, - owner=owner, group=group) - - def make_file(self, infiles, outfile, func, args, - exec_msg=None, skip_msg=None, level=1): - """Special case of 'execute()' for operations that process one or - more input files and generate one output file. Works just like - 'execute()', except the operation is skipped and a different - message printed if 'outfile' already exists and is newer than all - files listed in 'infiles'. If the command defined 'self.force', - and it is true, then the command is unconditionally run -- does no - timestamp checks. - """ - if skip_msg is None: - skip_msg = "skipping %s (inputs unchanged)" % outfile - - # Allow 'infiles' to be a single string - if isinstance(infiles, str): - infiles = (infiles,) - elif not isinstance(infiles, (list, tuple)): - raise TypeError( - "'infiles' must be a string, or a list or tuple of strings") - - if exec_msg is None: - exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles)) - - # If 'outfile' must be regenerated (either because it doesn't - # exist, is out-of-date, or the 'force' flag is true) then - # perform the action that presumably regenerates it - if self.force or dep_util.newer_group(infiles, outfile): - self.execute(func, args, exec_msg, level) - # Otherwise, print the "skip" message - else: - log.debug(skip_msg) - -# XXX 'install_misc' class not currently used -- it was the base class for -# both 'install_scripts' and 'install_data', but they outgrew it. It might -# still be useful for 'install_headers', though, so I'm keeping it around -# for the time being. - -class install_misc(Command): - """Common base class for installing some files in a subdirectory. - Currently used by install_data and install_scripts. - """ - - user_options = [('install-dir=', 'd', "directory to install the files to")] - - def initialize_options (self): - self.install_dir = None - self.outfiles = [] - - def _install_dir_from(self, dirname): - self.set_undefined_options('install', (dirname, 'install_dir')) - - def _copy_files(self, filelist): - self.outfiles = [] - if not filelist: - return - self.mkpath(self.install_dir) - for f in filelist: - self.copy_file(f, self.install_dir) - self.outfiles.append(os.path.join(self.install_dir, f)) - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/__init__.py b/Lib/distutils/command/__init__.py deleted file mode 100644 index 481eea9fd4b..00000000000 --- a/Lib/distutils/command/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""distutils.command - -Package containing implementation of all the standard Distutils -commands.""" - -__all__ = ['build', - 'build_py', - 'build_ext', - 'build_clib', - 'build_scripts', - 'clean', - 'install', - 'install_lib', - 'install_headers', - 'install_scripts', - 'install_data', - 'sdist', - 'register', - 'bdist', - 'bdist_dumb', - 'bdist_rpm', - 'bdist_wininst', - 'check', - 'upload', - # These two are reserved for future use: - #'bdist_sdux', - #'bdist_pkgtool', - # Note: - # bdist_packager is not included because it only provides - # an abstract base class - ] diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py deleted file mode 100644 index 014871d280e..00000000000 --- a/Lib/distutils/command/bdist.py +++ /dev/null @@ -1,143 +0,0 @@ -"""distutils.command.bdist - -Implements the Distutils 'bdist' command (create a built [binary] -distribution).""" - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.util import get_platform - - -def show_formats(): - """Print list of available formats (arguments to "--format" option). - """ - from distutils.fancy_getopt import FancyGetopt - formats = [] - for format in bdist.format_commands: - formats.append(("formats=" + format, None, - bdist.format_command[format][1])) - pretty_printer = FancyGetopt(formats) - pretty_printer.print_help("List of available distribution formats:") - - -class bdist(Command): - - description = "create a built (binary) distribution" - - user_options = [('bdist-base=', 'b', - "temporary directory for creating built distributions"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('formats=', None, - "formats for distribution (comma-separated list)"), - ('dist-dir=', 'd', - "directory to put final built distributions in " - "[default: dist]"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['skip-build'] - - help_options = [ - ('help-formats', None, - "lists available distribution formats", show_formats), - ] - - # The following commands do not take a format option from bdist - no_format_option = ('bdist_rpm',) - - # This won't do in reality: will need to distinguish RPM-ish Linux, - # Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS. - default_format = {'posix': 'gztar', - 'nt': 'zip'} - - # Establish the preferred order (for the --help-formats option). - format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'wininst', 'zip', 'msi'] - - # And the real information. - format_command = {'rpm': ('bdist_rpm', "RPM distribution"), - 'gztar': ('bdist_dumb', "gzip'ed tar file"), - 'bztar': ('bdist_dumb', "bzip2'ed tar file"), - 'xztar': ('bdist_dumb', "xz'ed tar file"), - 'ztar': ('bdist_dumb', "compressed tar file"), - 'tar': ('bdist_dumb', "tar file"), - 'wininst': ('bdist_wininst', - "Windows executable installer"), - 'zip': ('bdist_dumb', "ZIP file"), - 'msi': ('bdist_msi', "Microsoft Installer") - } - - - def initialize_options(self): - self.bdist_base = None - self.plat_name = None - self.formats = None - self.dist_dir = None - self.skip_build = 0 - self.group = None - self.owner = None - - def finalize_options(self): - # have to finalize 'plat_name' before 'bdist_base' - if self.plat_name is None: - if self.skip_build: - self.plat_name = get_platform() - else: - self.plat_name = self.get_finalized_command('build').plat_name - - # 'bdist_base' -- parent of per-built-distribution-format - # temporary directories (eg. we'll probably have - # "build/bdist./dumb", "build/bdist./rpm", etc.) - if self.bdist_base is None: - build_base = self.get_finalized_command('build').build_base - self.bdist_base = os.path.join(build_base, - 'bdist.' + self.plat_name) - - self.ensure_string_list('formats') - if self.formats is None: - try: - self.formats = [self.default_format[os.name]] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create built distributions " - "on platform %s" % os.name) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # Figure out which sub-commands we need to run. - commands = [] - for format in self.formats: - try: - commands.append(self.format_command[format][0]) - except KeyError: - raise DistutilsOptionError("invalid format '%s'" % format) - - # Reinitialize and run each command. - for i in range(len(self.formats)): - cmd_name = commands[i] - sub_cmd = self.reinitialize_command(cmd_name) - if cmd_name not in self.no_format_option: - sub_cmd.format = self.formats[i] - - # passing the owner and group names for tar archiving - if cmd_name == 'bdist_dumb': - sub_cmd.owner = self.owner - sub_cmd.group = self.group - - # If we're going to need to run this command again, tell it to - # keep its temporary files around so subsequent runs go faster. - if cmd_name in commands[i+1:]: - sub_cmd.keep_temp = 1 - self.run_command(cmd_name) diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py deleted file mode 100644 index f0d6b5b8cd8..00000000000 --- a/Lib/distutils/command/bdist_dumb.py +++ /dev/null @@ -1,123 +0,0 @@ -"""distutils.command.bdist_dumb - -Implements the Distutils 'bdist_dumb' command (create a "dumb" built -distribution -- i.e., just an archive to be unpacked under $prefix or -$exec_prefix).""" - -import os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import remove_tree, ensure_relative -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_dumb(Command): - - description = "create a \"dumb\" built distribution" - - user_options = [('bdist-dir=', 'd', - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('format=', 'f', - "archive format to create (tar, gztar, bztar, xztar, " - "ztar, zip)"), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('relative', None, - "build the archive using relative paths " - "(default: false)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['keep-temp', 'skip-build', 'relative'] - - default_format = { 'posix': 'gztar', - 'nt': 'zip' } - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.format = None - self.keep_temp = 0 - self.dist_dir = None - self.skip_build = None - self.relative = 0 - self.owner = None - self.group = None - - def finalize_options(self): - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'dumb') - - if self.format is None: - try: - self.format = self.default_format[os.name] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create dumb built distributions " - "on platform %s" % os.name) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ('skip_build', 'skip_build')) - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - log.info("installing to %s", self.bdist_dir) - self.run_command('install') - - # And make an archive relative to the root of the - # pseudo-installation tree. - archive_basename = "%s.%s" % (self.distribution.get_fullname(), - self.plat_name) - - pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) - if not self.relative: - archive_root = self.bdist_dir - else: - if (self.distribution.has_ext_modules() and - (install.install_base != install.install_platbase)): - raise DistutilsPlatformError( - "can't make a dumb built distribution where " - "base and platbase are different (%s, %s)" - % (repr(install.install_base), - repr(install.install_platbase))) - else: - archive_root = os.path.join(self.bdist_dir, - ensure_relative(install.install_base)) - - # Make the archive - filename = self.make_archive(pseudoinstall_root, - self.format, root_dir=archive_root, - owner=self.owner, group=self.group) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_dumb', pyversion, - filename)) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py deleted file mode 100644 index 80104c372d9..00000000000 --- a/Lib/distutils/command/bdist_msi.py +++ /dev/null @@ -1,741 +0,0 @@ -# Copyright (C) 2005, 2006 Martin von Löwis -# Licensed to PSF under a Contributor Agreement. -# The bdist_wininst command proper -# based on bdist_wininst -""" -Implements the bdist_msi command. -""" - -import sys, os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils.sysconfig import get_python_version -from distutils.version import StrictVersion -from distutils.errors import DistutilsOptionError -from distutils.util import get_platform -from distutils import log -import msilib -from msilib import schema, sequence, text -from msilib import Directory, Feature, Dialog, add_data - -class PyDialog(Dialog): - """Dialog class with a fixed layout: controls at the top, then a ruler, - then a list of buttons: back, next, cancel. Optionally a bitmap at the - left.""" - def __init__(self, *args, **kw): - """Dialog(database, name, x, y, w, h, attributes, title, first, - default, cancel, bitmap=true)""" - Dialog.__init__(self, *args) - ruler = self.h - 36 - bmwidth = 152*ruler/328 - #if kw.get("bitmap", True): - # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") - self.line("BottomLine", 0, ruler, self.w, 0) - - def title(self, title): - "Set the title text of the dialog at the top." - # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, - # text, in VerdanaBold10 - self.text("Title", 15, 10, 320, 60, 0x30003, - r"{\VerdanaBold10}%s" % title) - - def back(self, title, next, name = "Back", active = 1): - """Add a back button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) - - def cancel(self, title, next, name = "Cancel", active = 1): - """Add a cancel button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) - - def next(self, title, next, name = "Next", active = 1): - """Add a Next button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) - - def xbutton(self, name, title, next, xpos): - """Add a button with a given title, the tab-next button, - its name in the Control table, giving its x position; the - y-position is aligned with the other buttons. - - Return the button, so that events can be associated""" - return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) - -class bdist_msi(Command): - - description = "create a Microsoft Installer (.msi) binary distribution" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', - '2.5', '2.6', '2.7', '2.8', '2.9', - '3.0', '3.1', '3.2', '3.3', '3.4', - '3.5', '3.6', '3.7', '3.8', '3.9'] - other_version = 'X' - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.versions = None - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'msi') - - short_version = get_python_version() - if (not self.target_version) and self.distribution.has_ext_modules(): - self.target_version = short_version - - if self.target_version: - self.versions = [self.target_version] - if not self.skip_build and self.distribution.has_ext_modules()\ - and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" - " option must be specified" % (short_version,)) - else: - self.versions = list(self.all_versions) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.pre_install_script: - raise DistutilsOptionError( - "the pre-install-script feature is not yet implemented") - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - self.install_script_key = None - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.prefix = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - self.mkpath(self.dist_dir) - fullname = self.distribution.get_fullname() - installer_name = self.get_installer_filename(fullname) - installer_name = os.path.abspath(installer_name) - if os.path.exists(installer_name): os.unlink(installer_name) - - metadata = self.distribution.metadata - author = metadata.author - if not author: - author = metadata.maintainer - if not author: - author = "UNKNOWN" - version = metadata.get_version() - # ProductVersion must be strictly numeric - # XXX need to deal with prerelease versions - sversion = "%d.%d.%d" % StrictVersion(version).version - # Prefix ProductName with Python x.y, so that - # it sorts together with the other Python packages - # in Add-Remove-Programs (APR) - fullname = self.distribution.get_fullname() - if self.target_version: - product_name = "Python %s %s" % (self.target_version, fullname) - else: - product_name = "Python %s" % (fullname) - self.db = msilib.init_database(installer_name, schema, - product_name, msilib.gen_uuid(), - sversion, author) - msilib.add_tables(self.db, sequence) - props = [('DistVersion', version)] - email = metadata.author_email or metadata.maintainer_email - if email: - props.append(("ARPCONTACT", email)) - if metadata.url: - props.append(("ARPURLINFOABOUT", metadata.url)) - if props: - add_data(self.db, 'Property', props) - - self.add_find_python() - self.add_files() - self.add_scripts() - self.add_ui() - self.db.Commit() - - if hasattr(self.distribution, 'dist_files'): - tup = 'bdist_msi', self.target_version or 'any', fullname - self.distribution.dist_files.append(tup) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def add_files(self): - db = self.db - cab = msilib.CAB("distfiles") - rootdir = os.path.abspath(self.bdist_dir) - - root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") - f = Feature(db, "Python", "Python", "Everything", - 0, 1, directory="TARGETDIR") - - items = [(f, root, '')] - for version in self.versions + [self.other_version]: - target = "TARGETDIR" + version - name = default = "Python" + version - desc = "Everything" - if version is self.other_version: - title = "Python from another location" - level = 2 - else: - title = "Python %s from registry" % version - level = 1 - f = Feature(db, name, title, desc, 1, level, directory=target) - dir = Directory(db, cab, root, rootdir, target, default) - items.append((f, dir, version)) - db.Commit() - - seen = {} - for feature, dir, version in items: - todo = [dir] - while todo: - dir = todo.pop() - for file in os.listdir(dir.absolute): - afile = os.path.join(dir.absolute, file) - if os.path.isdir(afile): - short = "%s|%s" % (dir.make_short(file), file) - default = file + version - newdir = Directory(db, cab, dir, file, default, short) - todo.append(newdir) - else: - if not dir.component: - dir.start_component(dir.logical, feature, 0) - if afile not in seen: - key = seen[afile] = dir.add_file(file) - if file==self.install_script: - if self.install_script_key: - raise DistutilsOptionError( - "Multiple files with name %s" % file) - self.install_script_key = '[#%s]' % key - else: - key = seen[afile] - add_data(self.db, "DuplicateFile", - [(key + version, dir.component, key, None, dir.logical)]) - db.Commit() - cab.commit(db) - - def add_find_python(self): - """Adds code to the installer to compute the location of Python. - - Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the - registry for each version of Python. - - Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, - else from PYTHON.MACHINE.X.Y. - - Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" - - start = 402 - for ver in self.versions: - install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver - machine_reg = "python.machine." + ver - user_reg = "python.user." + ver - machine_prop = "PYTHON.MACHINE." + ver - user_prop = "PYTHON.USER." + ver - machine_action = "PythonFromMachine" + ver - user_action = "PythonFromUser" + ver - exe_action = "PythonExe" + ver - target_dir_prop = "TARGETDIR" + ver - exe_prop = "PYTHON" + ver - if msilib.Win64: - # type: msidbLocatorTypeRawValue + msidbLocatorType64bit - Type = 2+16 - else: - Type = 2 - add_data(self.db, "RegLocator", - [(machine_reg, 2, install_path, None, Type), - (user_reg, 1, install_path, None, Type)]) - add_data(self.db, "AppSearch", - [(machine_prop, machine_reg), - (user_prop, user_reg)]) - add_data(self.db, "CustomAction", - [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), - (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), - (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), - ]) - add_data(self.db, "InstallExecuteSequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "InstallUISequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "Condition", - [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) - start += 4 - assert start < 500 - - def add_scripts(self): - if self.install_script: - start = 6800 - for ver in self.versions + [self.other_version]: - install_action = "install_script." + ver - exe_prop = "PYTHON" + ver - add_data(self.db, "CustomAction", - [(install_action, 50, exe_prop, self.install_script_key)]) - add_data(self.db, "InstallExecuteSequence", - [(install_action, "&Python%s=3" % ver, start)]) - start += 1 - # XXX pre-install scripts are currently refused in finalize_options() - # but if this feature is completed, it will also need to add - # entries for each version as the above code does - if self.pre_install_script: - scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") - f = open(scriptfn, "w") - # The batch file will be executed with [PYTHON], so that %1 - # is the path to the Python interpreter; %0 will be the path - # of the batch file. - # rem =""" - # %1 %0 - # exit - # """ - # - f.write('rem ="""\n%1 %0\nexit\n"""\n') - f.write(open(self.pre_install_script).read()) - f.close() - add_data(self.db, "Binary", - [("PreInstall", msilib.Binary(scriptfn)) - ]) - add_data(self.db, "CustomAction", - [("PreInstall", 2, "PreInstall", None) - ]) - add_data(self.db, "InstallExecuteSequence", - [("PreInstall", "NOT Installed", 450)]) - - - def add_ui(self): - db = self.db - x = y = 50 - w = 370 - h = 300 - title = "[ProductName] Setup" - - # see "Dialog Style Bits" - modal = 3 # visible | modal - modeless = 1 # visible - track_disk_space = 32 - - # UI customization properties - add_data(db, "Property", - # See "DefaultUIFont Property" - [("DefaultUIFont", "DlgFont8"), - # See "ErrorDialog Style Bit" - ("ErrorDialog", "ErrorDlg"), - ("Progress1", "Install"), # modified in maintenance type dlg - ("Progress2", "installs"), - ("MaintenanceForm_Action", "Repair"), - # possible values: ALL, JUSTME - ("WhichUsers", "ALL") - ]) - - # Fonts, see "TextStyle Table" - add_data(db, "TextStyle", - [("DlgFont8", "Tahoma", 9, None, 0), - ("DlgFontBold8", "Tahoma", 8, None, 1), #bold - ("VerdanaBold10", "Verdana", 10, None, 1), - ("VerdanaRed9", "Verdana", 9, 255, 0), - ]) - - # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" - # Numbers indicate sequence; see sequence.py for how these action integrate - add_data(db, "InstallUISequence", - [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), - ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), - # In the user interface, assume all-users installation if privileged. - ("SelectFeaturesDlg", "Not Installed", 1230), - # XXX no support for resume installations yet - #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), - ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), - ("ProgressDlg", None, 1280)]) - - add_data(db, 'ActionText', text.ActionText) - add_data(db, 'UIText', text.UIText) - ##################################################################### - # Standard dialogs: FatalError, UserExit, ExitDialog - fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - fatal.title("[ProductName] Installer ended prematurely") - fatal.back("< Back", "Finish", active = 0) - fatal.cancel("Cancel", "Back", active = 0) - fatal.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") - fatal.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c=fatal.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - user_exit.title("[ProductName] Installer was interrupted") - user_exit.back("< Back", "Finish", active = 0) - user_exit.cancel("Cancel", "Back", active = 0) - user_exit.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup was interrupted. Your system has not been modified. " - "To install this program at a later time, please run the installation again.") - user_exit.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = user_exit.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - exit_dialog.title("Completing the [ProductName] Installer") - exit_dialog.back("< Back", "Finish", active = 0) - exit_dialog.cancel("Cancel", "Back", active = 0) - exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = exit_dialog.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Return") - - ##################################################################### - # Required dialog: FilesInUse, ErrorDlg - inuse = PyDialog(db, "FilesInUse", - x, y, w, h, - 19, # KeepModeless|Modal|Visible - title, - "Retry", "Retry", "Retry", bitmap=False) - inuse.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Files in Use") - inuse.text("Description", 20, 23, 280, 20, 0x30003, - "Some files that need to be updated are currently in use.") - inuse.text("Text", 20, 55, 330, 50, 3, - "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") - inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", - None, None, None) - c=inuse.back("Exit", "Ignore", name="Exit") - c.event("EndDialog", "Exit") - c=inuse.next("Ignore", "Retry", name="Ignore") - c.event("EndDialog", "Ignore") - c=inuse.cancel("Retry", "Exit", name="Retry") - c.event("EndDialog","Retry") - - # See "Error Dialog". See "ICE20" for the required names of the controls. - error = Dialog(db, "ErrorDlg", - 50, 10, 330, 101, - 65543, # Error|Minimize|Modal|Visible - title, - "ErrorText", None, None) - error.text("ErrorText", 50,9,280,48,3, "") - #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) - error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") - error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") - error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") - error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") - error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") - error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") - error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") - - ##################################################################### - # Global "Query Cancel" dialog - cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, - "No", "No", "No") - cancel.text("Text", 48, 15, 194, 30, 3, - "Are you sure you want to cancel [ProductName] installation?") - #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, - # "py.ico", None, None) - c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") - c.event("EndDialog", "Exit") - - c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") - c.event("EndDialog", "Return") - - ##################################################################### - # Global "Wait for costing" dialog - costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, - "Return", "Return", "Return") - costing.text("Text", 48, 15, 194, 30, 3, - "Please wait while the installer finishes determining your disk space requirements.") - c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) - c.event("EndDialog", "Exit") - - ##################################################################### - # Preparation dialog: no user input except cancellation - prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel") - prep.text("Description", 15, 70, 320, 40, 0x30003, - "Please wait while the Installer prepares to guide you through the installation.") - prep.title("Welcome to the [ProductName] Installer") - c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") - c.mapping("ActionText", "Text") - c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) - c.mapping("ActionData", "Text") - prep.back("Back", None, active=0) - prep.next("Next", None, active=0) - c=prep.cancel("Cancel", None) - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Feature (Python directory) selection - seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - seldlg.title("Select Python Installations") - - seldlg.text("Hint", 15, 30, 300, 20, 3, - "Select the Python locations where %s should be installed." - % self.distribution.get_fullname()) - - seldlg.back("< Back", None, active=0) - c = seldlg.next("Next >", "Cancel") - order = 1 - c.event("[TARGETDIR]", "[SourceDir]", ordering=order) - for version in self.versions + [self.other_version]: - order += 1 - c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, - "FEATURE_SELECTED AND &Python%s=3" % version, - ordering=order) - c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) - c.event("EndDialog", "Return", ordering=order + 2) - c = seldlg.cancel("Cancel", "Features") - c.event("SpawnDialog", "CancelDlg") - - c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, - "FEATURE", None, "PathEdit", None) - c.event("[FEATURE_SELECTED]", "1") - ver = self.other_version - install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver - dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver - - c = seldlg.text("Other", 15, 200, 300, 15, 3, - "Provide an alternate Python location") - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, - "TARGETDIR" + ver, None, "Next", None) - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - ##################################################################### - # Disk cost - cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, - "OK", "OK", "OK", bitmap=False) - cost.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Disk Space Requirements") - cost.text("Description", 20, 20, 280, 20, 0x30003, - "The disk space required for the installation of the selected features.") - cost.text("Text", 20, 53, 330, 60, 3, - "The highlighted volumes (if any) do not have enough disk space " - "available for the currently selected features. You can either " - "remove some files from the highlighted volumes, or choose to " - "install less features onto local drive(s), or select different " - "destination drive(s).") - cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, - None, "{120}{70}{70}{70}{70}", None, None) - cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") - - ##################################################################### - # WhichUsers Dialog. Only available on NT, and for privileged users. - # This must be run before FindRelatedProducts, because that will - # take into account whether the previous installation was per-user - # or per-machine. We currently don't support going back to this - # dialog after "Next" was selected; to support this, we would need to - # find how to reset the ALLUSERS property, and how to re-run - # FindRelatedProducts. - # On Windows9x, the ALLUSERS property is ignored on the command line - # and in the Property table, but installer fails according to the documentation - # if a dialog attempts to set ALLUSERS. - whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, - "AdminInstall", "Next", "Cancel") - whichusers.title("Select whether to install [ProductName] for all users of this computer.") - # A radio group with two options: allusers, justme - g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, - "WhichUsers", "", "Next") - g.add("ALL", 0, 5, 150, 20, "Install for all users") - g.add("JUSTME", 0, 25, 150, 20, "Install just for me") - - whichusers.back("Back", None, active=0) - - c = whichusers.next("Next >", "Cancel") - c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) - c.event("EndDialog", "Return", ordering = 2) - - c = whichusers.cancel("Cancel", "AdminInstall") - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Installation Progress dialog (modeless) - progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel", bitmap=False) - progress.text("Title", 20, 15, 200, 15, 0x30003, - r"{\DlgFontBold8}[Progress1] [ProductName]") - progress.text("Text", 35, 65, 300, 30, 3, - "Please wait while the Installer [Progress2] [ProductName]. " - "This may take several minutes.") - progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") - - c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") - c.mapping("ActionText", "Text") - - #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) - #c.mapping("ActionData", "Text") - - c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, - None, "Progress done", None, None) - c.mapping("SetProgress", "Progress") - - progress.back("< Back", "Next", active=False) - progress.next("Next >", "Cancel", active=False) - progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") - - ################################################################### - # Maintenance type: repair/uninstall - maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - maint.title("Welcome to the [ProductName] Setup Wizard") - maint.text("BodyText", 15, 63, 330, 42, 3, - "Select whether you want to repair or remove [ProductName].") - g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, - "MaintenanceForm_Action", "", "Next") - #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") - g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") - g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") - - maint.back("< Back", None, active=False) - c=maint.next("Finish", "Cancel") - # Change installation: Change progress dialog to "Change", then ask - # for feature selection - #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) - #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) - - # Reinstall: Change progress dialog to "Repair", then invoke reinstall - # Also set list of reinstalled features to "ALL" - c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) - c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) - c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) - c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) - - # Uninstall: Change progress to "Remove", then invoke uninstall - # Also set list of removed features to "ALL" - c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) - c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) - c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) - c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) - - # Close dialog when maintenance action scheduled - c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) - #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) - - maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") - - def get_installer_filename(self, fullname): - # Factored out to allow overriding in subclasses - if self.target_version: - base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, - self.target_version) - else: - base_name = "%s.%s.msi" % (fullname, self.plat_name) - installer_name = os.path.join(self.dist_dir, base_name) - return installer_name diff --git a/Lib/distutils/command/bdist_rpm.py b/Lib/distutils/command/bdist_rpm.py deleted file mode 100644 index 02f10dd89d9..00000000000 --- a/Lib/distutils/command/bdist_rpm.py +++ /dev/null @@ -1,582 +0,0 @@ -"""distutils.command.bdist_rpm - -Implements the Distutils 'bdist_rpm' command (create RPM source and binary -distributions).""" - -import subprocess, sys, os -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.util import get_platform -from distutils.file_util import write_file -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_rpm(Command): - - description = "create an RPM distribution" - - user_options = [ - ('bdist-base=', None, - "base directory for creating built distributions"), - ('rpm-base=', None, - "base directory for creating RPMs (defaults to \"rpm\" under " - "--bdist-base; must be specified for RPM 2)"), - ('dist-dir=', 'd', - "directory to put final RPM files in " - "(and .spec files if --spec-only)"), - ('python=', None, - "path to Python interpreter to hard-code in the .spec file " - "(default: \"python\")"), - ('fix-python', None, - "hard-code the exact path to the current Python interpreter in " - "the .spec file"), - ('spec-only', None, - "only regenerate spec file"), - ('source-only', None, - "only generate source RPM"), - ('binary-only', None, - "only generate binary RPM"), - ('use-bzip2', None, - "use bzip2 instead of gzip to create source distribution"), - - # More meta-data: too RPM-specific to put in the setup script, - # but needs to go in the .spec file -- so we make these options - # to "bdist_rpm". The idea is that packagers would put this - # info in setup.cfg, although they are of course free to - # supply it on the command line. - ('distribution-name=', None, - "name of the (Linux) distribution to which this " - "RPM applies (*not* the name of the module distribution!)"), - ('group=', None, - "package classification [default: \"Development/Libraries\"]"), - ('release=', None, - "RPM release number"), - ('serial=', None, - "RPM serial number"), - ('vendor=', None, - "RPM \"vendor\" (eg. \"Joe Blow \") " - "[default: maintainer or author from setup script]"), - ('packager=', None, - "RPM packager (eg. \"Jane Doe \") " - "[default: vendor]"), - ('doc-files=', None, - "list of documentation files (space or comma-separated)"), - ('changelog=', None, - "RPM changelog"), - ('icon=', None, - "name of icon file"), - ('provides=', None, - "capabilities provided by this package"), - ('requires=', None, - "capabilities required by this package"), - ('conflicts=', None, - "capabilities which conflict with this package"), - ('build-requires=', None, - "capabilities required to build this package"), - ('obsoletes=', None, - "capabilities made obsolete by this package"), - ('no-autoreq', None, - "do not automatically calculate dependencies"), - - # Actions to take when building RPM - ('keep-temp', 'k', - "don't clean up RPM build directory"), - ('no-keep-temp', None, - "clean up RPM build directory [default]"), - ('use-rpm-opt-flags', None, - "compile with RPM_OPT_FLAGS when building from source RPM"), - ('no-rpm-opt-flags', None, - "do not pass any RPM CFLAGS to compiler"), - ('rpm3-mode', None, - "RPM 3 compatibility mode (default)"), - ('rpm2-mode', None, - "RPM 2 compatibility mode"), - - # Add the hooks necessary for specifying custom scripts - ('prep-script=', None, - "Specify a script for the PREP phase of RPM building"), - ('build-script=', None, - "Specify a script for the BUILD phase of RPM building"), - - ('pre-install=', None, - "Specify a script for the pre-INSTALL phase of RPM building"), - ('install-script=', None, - "Specify a script for the INSTALL phase of RPM building"), - ('post-install=', None, - "Specify a script for the post-INSTALL phase of RPM building"), - - ('pre-uninstall=', None, - "Specify a script for the pre-UNINSTALL phase of RPM building"), - ('post-uninstall=', None, - "Specify a script for the post-UNINSTALL phase of RPM building"), - - ('clean-script=', None, - "Specify a script for the CLEAN phase of RPM building"), - - ('verify-script=', None, - "Specify a script for the VERIFY phase of the RPM build"), - - # Allow a packager to explicitly force an architecture - ('force-arch=', None, - "Force an architecture onto the RPM build process"), - - ('quiet', 'q', - "Run the INSTALL phase of RPM building in quiet mode"), - ] - - boolean_options = ['keep-temp', 'use-rpm-opt-flags', 'rpm3-mode', - 'no-autoreq', 'quiet'] - - negative_opt = {'no-keep-temp': 'keep-temp', - 'no-rpm-opt-flags': 'use-rpm-opt-flags', - 'rpm2-mode': 'rpm3-mode'} - - - def initialize_options(self): - self.bdist_base = None - self.rpm_base = None - self.dist_dir = None - self.python = None - self.fix_python = None - self.spec_only = None - self.binary_only = None - self.source_only = None - self.use_bzip2 = None - - self.distribution_name = None - self.group = None - self.release = None - self.serial = None - self.vendor = None - self.packager = None - self.doc_files = None - self.changelog = None - self.icon = None - - self.prep_script = None - self.build_script = None - self.install_script = None - self.clean_script = None - self.verify_script = None - self.pre_install = None - self.post_install = None - self.pre_uninstall = None - self.post_uninstall = None - self.prep = None - self.provides = None - self.requires = None - self.conflicts = None - self.build_requires = None - self.obsoletes = None - - self.keep_temp = 0 - self.use_rpm_opt_flags = 1 - self.rpm3_mode = 1 - self.no_autoreq = 0 - - self.force_arch = None - self.quiet = 0 - - def finalize_options(self): - self.set_undefined_options('bdist', ('bdist_base', 'bdist_base')) - if self.rpm_base is None: - if not self.rpm3_mode: - raise DistutilsOptionError( - "you must specify --rpm-base in RPM 2 mode") - self.rpm_base = os.path.join(self.bdist_base, "rpm") - - if self.python is None: - if self.fix_python: - self.python = sys.executable - else: - self.python = "python3" - elif self.fix_python: - raise DistutilsOptionError( - "--python and --fix-python are mutually exclusive options") - - if os.name != 'posix': - raise DistutilsPlatformError("don't know how to create RPM " - "distributions on platform %s" % os.name) - if self.binary_only and self.source_only: - raise DistutilsOptionError( - "cannot supply both '--source-only' and '--binary-only'") - - # don't pass CFLAGS to pure python distributions - if not self.distribution.has_ext_modules(): - self.use_rpm_opt_flags = 0 - - self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) - self.finalize_package_data() - - def finalize_package_data(self): - self.ensure_string('group', "Development/Libraries") - self.ensure_string('vendor', - "%s <%s>" % (self.distribution.get_contact(), - self.distribution.get_contact_email())) - self.ensure_string('packager') - self.ensure_string_list('doc_files') - if isinstance(self.doc_files, list): - for readme in ('README', 'README.txt'): - if os.path.exists(readme) and readme not in self.doc_files: - self.doc_files.append(readme) - - self.ensure_string('release', "1") - self.ensure_string('serial') # should it be an int? - - self.ensure_string('distribution_name') - - self.ensure_string('changelog') - # Format changelog correctly - self.changelog = self._format_changelog(self.changelog) - - self.ensure_filename('icon') - - self.ensure_filename('prep_script') - self.ensure_filename('build_script') - self.ensure_filename('install_script') - self.ensure_filename('clean_script') - self.ensure_filename('verify_script') - self.ensure_filename('pre_install') - self.ensure_filename('post_install') - self.ensure_filename('pre_uninstall') - self.ensure_filename('post_uninstall') - - # XXX don't forget we punted on summaries and descriptions -- they - # should be handled here eventually! - - # Now *this* is some meta-data that belongs in the setup script... - self.ensure_string_list('provides') - self.ensure_string_list('requires') - self.ensure_string_list('conflicts') - self.ensure_string_list('build_requires') - self.ensure_string_list('obsoletes') - - self.ensure_string('force_arch') - - def run(self): - if DEBUG: - print("before _get_package_data():") - print("vendor =", self.vendor) - print("packager =", self.packager) - print("doc_files =", self.doc_files) - print("changelog =", self.changelog) - - # make directories - if self.spec_only: - spec_dir = self.dist_dir - self.mkpath(spec_dir) - else: - rpm_dir = {} - for d in ('SOURCES', 'SPECS', 'BUILD', 'RPMS', 'SRPMS'): - rpm_dir[d] = os.path.join(self.rpm_base, d) - self.mkpath(rpm_dir[d]) - spec_dir = rpm_dir['SPECS'] - - # Spec file goes into 'dist_dir' if '--spec-only specified', - # build/rpm. otherwise. - spec_path = os.path.join(spec_dir, - "%s.spec" % self.distribution.get_name()) - self.execute(write_file, - (spec_path, - self._make_spec_file()), - "writing '%s'" % spec_path) - - if self.spec_only: # stop if requested - return - - # Make a source distribution and copy to SOURCES directory with - # optional icon. - saved_dist_files = self.distribution.dist_files[:] - sdist = self.reinitialize_command('sdist') - if self.use_bzip2: - sdist.formats = ['bztar'] - else: - sdist.formats = ['gztar'] - self.run_command('sdist') - self.distribution.dist_files = saved_dist_files - - source = sdist.get_archive_files()[0] - source_dir = rpm_dir['SOURCES'] - self.copy_file(source, source_dir) - - if self.icon: - if os.path.exists(self.icon): - self.copy_file(self.icon, source_dir) - else: - raise DistutilsFileError( - "icon file '%s' does not exist" % self.icon) - - # build package - log.info("building RPMs") - rpm_cmd = ['rpm'] - if os.path.exists('/usr/bin/rpmbuild') or \ - os.path.exists('/bin/rpmbuild'): - rpm_cmd = ['rpmbuild'] - - if self.source_only: # what kind of RPMs? - rpm_cmd.append('-bs') - elif self.binary_only: - rpm_cmd.append('-bb') - else: - rpm_cmd.append('-ba') - rpm_cmd.extend(['--define', '__python %s' % self.python]) - if self.rpm3_mode: - rpm_cmd.extend(['--define', - '_topdir %s' % os.path.abspath(self.rpm_base)]) - if not self.keep_temp: - rpm_cmd.append('--clean') - - if self.quiet: - rpm_cmd.append('--quiet') - - rpm_cmd.append(spec_path) - # Determine the binary rpm names that should be built out of this spec - # file - # Note that some of these may not be really built (if the file - # list is empty) - nvr_string = "%{name}-%{version}-%{release}" - src_rpm = nvr_string + ".src.rpm" - non_src_rpm = "%{arch}/" + nvr_string + ".%{arch}.rpm" - q_cmd = r"rpm -q --qf '%s %s\n' --specfile '%s'" % ( - src_rpm, non_src_rpm, spec_path) - - out = os.popen(q_cmd) - try: - binary_rpms = [] - source_rpm = None - while True: - line = out.readline() - if not line: - break - l = line.strip().split() - assert(len(l) == 2) - binary_rpms.append(l[1]) - # The source rpm is named after the first entry in the spec file - if source_rpm is None: - source_rpm = l[0] - - status = out.close() - if status: - raise DistutilsExecError("Failed to execute: %s" % repr(q_cmd)) - - finally: - out.close() - - self.spawn(rpm_cmd) - - if not self.dry_run: - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - - if not self.binary_only: - srpm = os.path.join(rpm_dir['SRPMS'], source_rpm) - assert(os.path.exists(srpm)) - self.move_file(srpm, self.dist_dir) - filename = os.path.join(self.dist_dir, source_rpm) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - if not self.source_only: - for rpm in binary_rpms: - rpm = os.path.join(rpm_dir['RPMS'], rpm) - if os.path.exists(rpm): - self.move_file(rpm, self.dist_dir) - filename = os.path.join(self.dist_dir, - os.path.basename(rpm)) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - def _dist_path(self, path): - return os.path.join(self.dist_dir, os.path.basename(path)) - - def _make_spec_file(self): - """Generate the text of an RPM spec file and return it as a - list of strings (one per line). - """ - # definitions and headers - spec_file = [ - '%define name ' + self.distribution.get_name(), - '%define version ' + self.distribution.get_version().replace('-','_'), - '%define unmangled_version ' + self.distribution.get_version(), - '%define release ' + self.release.replace('-','_'), - '', - 'Summary: ' + self.distribution.get_description(), - ] - - # Workaround for #14443 which affects some RPM based systems such as - # RHEL6 (and probably derivatives) - vendor_hook = subprocess.getoutput('rpm --eval %{__os_install_post}') - # Generate a potential replacement value for __os_install_post (whilst - # normalizing the whitespace to simplify the test for whether the - # invocation of brp-python-bytecompile passes in __python): - vendor_hook = '\n'.join([' %s \\' % line.strip() - for line in vendor_hook.splitlines()]) - problem = "brp-python-bytecompile \\\n" - fixed = "brp-python-bytecompile %{__python} \\\n" - fixed_hook = vendor_hook.replace(problem, fixed) - if fixed_hook != vendor_hook: - spec_file.append('# Workaround for http://bugs.python.org/issue14443') - spec_file.append('%define __os_install_post ' + fixed_hook + '\n') - - # put locale summaries into spec file - # XXX not supported for now (hard to put a dictionary - # in a config file -- arg!) - #for locale in self.summaries.keys(): - # spec_file.append('Summary(%s): %s' % (locale, - # self.summaries[locale])) - - spec_file.extend([ - 'Name: %{name}', - 'Version: %{version}', - 'Release: %{release}',]) - - # XXX yuck! this filename is available from the "sdist" command, - # but only after it has run: and we create the spec file before - # running "sdist", in case of --spec-only. - if self.use_bzip2: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.bz2') - else: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.gz') - - spec_file.extend([ - 'License: ' + self.distribution.get_license(), - 'Group: ' + self.group, - 'BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot', - 'Prefix: %{_prefix}', ]) - - if not self.force_arch: - # noarch if no extension modules - if not self.distribution.has_ext_modules(): - spec_file.append('BuildArch: noarch') - else: - spec_file.append( 'BuildArch: %s' % self.force_arch ) - - for field in ('Vendor', - 'Packager', - 'Provides', - 'Requires', - 'Conflicts', - 'Obsoletes', - ): - val = getattr(self, field.lower()) - if isinstance(val, list): - spec_file.append('%s: %s' % (field, ' '.join(val))) - elif val is not None: - spec_file.append('%s: %s' % (field, val)) - - - if self.distribution.get_url() != 'UNKNOWN': - spec_file.append('Url: ' + self.distribution.get_url()) - - if self.distribution_name: - spec_file.append('Distribution: ' + self.distribution_name) - - if self.build_requires: - spec_file.append('BuildRequires: ' + - ' '.join(self.build_requires)) - - if self.icon: - spec_file.append('Icon: ' + os.path.basename(self.icon)) - - if self.no_autoreq: - spec_file.append('AutoReq: 0') - - spec_file.extend([ - '', - '%description', - self.distribution.get_long_description() - ]) - - # put locale descriptions into spec file - # XXX again, suppressed because config file syntax doesn't - # easily support this ;-( - #for locale in self.descriptions.keys(): - # spec_file.extend([ - # '', - # '%description -l ' + locale, - # self.descriptions[locale], - # ]) - - # rpm scripts - # figure out default build script - def_setup_call = "%s %s" % (self.python,os.path.basename(sys.argv[0])) - def_build = "%s build" % def_setup_call - if self.use_rpm_opt_flags: - def_build = 'env CFLAGS="$RPM_OPT_FLAGS" ' + def_build - - # insert contents of files - - # XXX this is kind of misleading: user-supplied options are files - # that we open and interpolate into the spec file, but the defaults - # are just text that we drop in as-is. Hmmm. - - install_cmd = ('%s install -O1 --root=$RPM_BUILD_ROOT ' - '--record=INSTALLED_FILES') % def_setup_call - - script_options = [ - ('prep', 'prep_script', "%setup -n %{name}-%{unmangled_version}"), - ('build', 'build_script', def_build), - ('install', 'install_script', install_cmd), - ('clean', 'clean_script', "rm -rf $RPM_BUILD_ROOT"), - ('verifyscript', 'verify_script', None), - ('pre', 'pre_install', None), - ('post', 'post_install', None), - ('preun', 'pre_uninstall', None), - ('postun', 'post_uninstall', None), - ] - - for (rpm_opt, attr, default) in script_options: - # Insert contents of file referred to, if no file is referred to - # use 'default' as contents of script - val = getattr(self, attr) - if val or default: - spec_file.extend([ - '', - '%' + rpm_opt,]) - if val: - spec_file.extend(open(val, 'r').read().split('\n')) - else: - spec_file.append(default) - - - # files section - spec_file.extend([ - '', - '%files -f INSTALLED_FILES', - '%defattr(-,root,root)', - ]) - - if self.doc_files: - spec_file.append('%doc ' + ' '.join(self.doc_files)) - - if self.changelog: - spec_file.extend([ - '', - '%changelog',]) - spec_file.extend(self.changelog) - - return spec_file - - def _format_changelog(self, changelog): - """Format the changelog correctly and convert it to a list of strings - """ - if not changelog: - return changelog - new_changelog = [] - for line in changelog.strip().split('\n'): - line = line.strip() - if line[0] == '*': - new_changelog.extend(['', line]) - elif line[0] == '-': - new_changelog.append(line) - else: - new_changelog.append(' ' + line) - - # strip trailing newline inserted by first changelog entry - if not new_changelog[0]: - del new_changelog[0] - - return new_changelog diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py deleted file mode 100644 index 1db47f9b983..00000000000 --- a/Lib/distutils/command/bdist_wininst.py +++ /dev/null @@ -1,367 +0,0 @@ -"""distutils.command.bdist_wininst - -Implements the Distutils 'bdist_wininst' command: create a windows installer -exe-program.""" - -import sys, os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import create_tree, remove_tree -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_wininst(Command): - - description = "create an executable installer for MS Windows" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('bitmap=', 'b', - "bitmap to use for the installer instead of python-powered logo"), - ('title=', 't', - "title to display on the installer background instead of default"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ('user-access-control=', None, - "specify Vista's UAC handling - 'none'/default=no " - "handling, 'auto'=use UAC if target Python installed for " - "all users, 'force'=always use UAC"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.bitmap = None - self.title = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.user_access_control = None - - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - if self.skip_build and self.plat_name: - # If build is skipped and plat_name is overridden, bdist will - # not see the correct 'plat_name' - so set that up manually. - bdist = self.distribution.get_command_obj('bdist') - bdist.plat_name = self.plat_name - # next the command will be initialized using that name - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'wininst') - - if not self.target_version: - self.target_version = "" - - if not self.skip_build and self.distribution.has_ext_modules(): - short_version = get_python_version() - if self.target_version and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" \ - " option must be specified" % (short_version,)) - self.target_version = short_version - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - - def run(self): - if (sys.platform != "win32" and - (self.distribution.has_ext_modules() or - self.distribution.has_c_libraries())): - raise DistutilsPlatformError \ - ("distribution contains extensions and/or C libraries; " - "must be compiled on a Windows 32 platform") - - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - install.plat_name = self.plat_name - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - # Use a custom scheme for the zip-file, because we have to decide - # at installation time which scheme to use. - for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): - value = key.upper() - if key == 'headers': - value = value + '/Include/$dist_name' - setattr(install, - 'install_' + key, - value) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - # And make an archive relative to the root of the - # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() - fullname = self.distribution.get_fullname() - arcname = self.make_archive(archive_basename, "zip", - root_dir=self.bdist_dir) - # create an exe containing the zip-file - self.create_exe(arcname, fullname, self.bitmap) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_wininst', pyversion, - self.get_installer_filename(fullname))) - # remove the zip-file again - log.debug("removing temporary file '%s'", arcname) - os.remove(arcname) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def get_inidata(self): - # Return data describing the installation. - lines = [] - metadata = self.distribution.metadata - - # Write the [metadata] section. - lines.append("[metadata]") - - # 'info' will be displayed in the installer's dialog box, - # describing the items to be installed. - info = (metadata.long_description or '') + '\n' - - # Escape newline characters - def escape(s): - return s.replace("\n", "\\n") - - for name in ["author", "author_email", "description", "maintainer", - "maintainer_email", "name", "url", "version"]: - data = getattr(metadata, name, "") - if data: - info = info + ("\n %s: %s" % \ - (name.capitalize(), escape(data))) - lines.append("%s=%s" % (name, escape(data))) - - # The [setup] section contains entries controlling - # the installer runtime. - lines.append("\n[Setup]") - if self.install_script: - lines.append("install_script=%s" % self.install_script) - lines.append("info=%s" % escape(info)) - lines.append("target_compile=%d" % (not self.no_target_compile)) - lines.append("target_optimize=%d" % (not self.no_target_optimize)) - if self.target_version: - lines.append("target_version=%s" % self.target_version) - if self.user_access_control: - lines.append("user_access_control=%s" % self.user_access_control) - - title = self.title or self.distribution.get_fullname() - lines.append("title=%s" % escape(title)) - import time - import distutils - build_info = "Built %s with distutils-%s" % \ - (time.ctime(time.time()), distutils.__version__) - lines.append("build_info=%s" % build_info) - return "\n".join(lines) - - def create_exe(self, arcname, fullname, bitmap=None): - import struct - - self.mkpath(self.dist_dir) - - cfgdata = self.get_inidata() - - installer_name = self.get_installer_filename(fullname) - self.announce("creating %s" % installer_name) - - if bitmap: - bitmapdata = open(bitmap, "rb").read() - bitmaplen = len(bitmapdata) - else: - bitmaplen = 0 - - file = open(installer_name, "wb") - file.write(self.get_exe_bytes()) - if bitmap: - file.write(bitmapdata) - - # Convert cfgdata from unicode to ascii, mbcs encoded - if isinstance(cfgdata, str): - cfgdata = cfgdata.encode("mbcs") - - # Append the pre-install script - cfgdata = cfgdata + b"\0" - if self.pre_install_script: - # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin-1" simply avoids any possible - # failures. - with open(self.pre_install_script, "r", - encoding="latin-1") as script: - script_data = script.read().encode("latin-1") - cfgdata = cfgdata + script_data + b"\n\0" - else: - # empty pre-install script - cfgdata = cfgdata + b"\0" - file.write(cfgdata) - - # The 'magic number' 0x1234567B is used to make sure that the - # binary layout of 'cfgdata' is what the wininst.exe binary - # expects. If the layout changes, increment that number, make - # the corresponding changes to the wininst.exe sources, and - # recompile them. - header = struct.pack("' under the base build directory. We only use one of - # them for a given distribution, though -- - if self.build_purelib is None: - self.build_purelib = os.path.join(self.build_base, 'lib') - if self.build_platlib is None: - self.build_platlib = os.path.join(self.build_base, - 'lib' + plat_specifier) - - # 'build_lib' is the actual directory that we will use for this - # particular module distribution -- if user didn't supply it, pick - # one of 'build_purelib' or 'build_platlib'. - if self.build_lib is None: - if self.distribution.ext_modules: - self.build_lib = self.build_platlib - else: - self.build_lib = self.build_purelib - - # 'build_temp' -- temporary directory for compiler turds, - # "build/temp." - if self.build_temp is None: - self.build_temp = os.path.join(self.build_base, - 'temp' + plat_specifier) - if self.build_scripts is None: - self.build_scripts = os.path.join(self.build_base, - 'scripts-%d.%d' % sys.version_info[:2]) - - if self.executable is None: - self.executable = os.path.normpath(sys.executable) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - # Run all relevant sub-commands. This will be some subset of: - # - build_py - pure Python modules - # - build_clib - standalone C libraries - # - build_ext - Python extensions - # - build_scripts - (Python) scripts - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - - # -- Predicates for the sub-command list --------------------------- - - def has_pure_modules(self): - return self.distribution.has_pure_modules() - - def has_c_libraries(self): - return self.distribution.has_c_libraries() - - def has_ext_modules(self): - return self.distribution.has_ext_modules() - - def has_scripts(self): - return self.distribution.has_scripts() - - - sub_commands = [('build_py', has_pure_modules), - ('build_clib', has_c_libraries), - ('build_ext', has_ext_modules), - ('build_scripts', has_scripts), - ] diff --git a/Lib/distutils/command/build_clib.py b/Lib/distutils/command/build_clib.py deleted file mode 100644 index 3e20ef23cd8..00000000000 --- a/Lib/distutils/command/build_clib.py +++ /dev/null @@ -1,209 +0,0 @@ -"""distutils.command.build_clib - -Implements the Distutils 'build_clib' command, to build a C/C++ library -that is included in the module distribution and needed by an extension -module.""" - - -# XXX this module has *lots* of code ripped-off quite transparently from -# build_ext.py -- not surprisingly really, as the work required to build -# a static library from a collection of C source files is not really all -# that different from what's required to build a shared object file from -# a collection of C source files. Nevertheless, I haven't done the -# necessary refactoring to account for the overlap in code between the -# two modules, mainly because a number of subtle details changed in the -# cut 'n paste. Sigh. - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler -from distutils import log - -def show_compilers(): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_clib(Command): - - description = "build C/C++ libraries used by Python extensions" - - user_options = [ - ('build-clib=', 'b', - "directory to build C/C++ libraries to"), - ('build-temp=', 't', - "directory to put temporary build by-products"), - ('debug', 'g', - "compile with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ] - - boolean_options = ['debug', 'force'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.build_clib = None - self.build_temp = None - - # List of libraries to build - self.libraries = None - - # Compilation options for all libraries - self.include_dirs = None - self.define = None - self.undef = None - self.debug = None - self.force = 0 - self.compiler = None - - - def finalize_options(self): - # This might be confusing: both build-clib and build-temp default - # to build-temp as defined by the "build" command. This is because - # I think that C libraries are really just temporary build - # by-products, at least from the point of view of building Python - # extensions -- but I want to keep my options open. - self.set_undefined_options('build', - ('build_temp', 'build_clib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) - - self.libraries = self.distribution.libraries - if self.libraries: - self.check_library_list(self.libraries) - - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # XXX same as for build_ext -- what about 'self.define' and - # 'self.undef' ? - - - def run(self): - if not self.libraries: - return - - # Yech -- this is cut 'n pasted from build_ext.py! - from distutils.ccompiler import new_compiler - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name,value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - - self.build_libraries(self.libraries) - - - def check_library_list(self, libraries): - """Ensure that the list of libraries is valid. - - `library` is presumably provided as a command option 'libraries'. - This method checks that it is a list of 2-tuples, where the tuples - are (library_name, build_info_dict). - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(libraries, list): - raise DistutilsSetupError( - "'libraries' option must be a list of tuples") - - for lib in libraries: - if not isinstance(lib, tuple) and len(lib) != 2: - raise DistutilsSetupError( - "each element of 'libraries' must a 2-tuple") - - name, build_info = lib - - if not isinstance(name, str): - raise DistutilsSetupError( - "first element of each tuple in 'libraries' " - "must be a string (the library name)") - - if '/' in name or (os.sep != '/' and os.sep in name): - raise DistutilsSetupError("bad library name '%s': " - "may not contain directory separators" % lib[0]) - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'libraries' " - "must be a dictionary (build info)") - - - def get_library_names(self): - # Assume the library list is valid -- 'check_library_list()' is - # called from 'finalize_options()', so it should be! - if not self.libraries: - return None - - lib_names = [] - for (lib_name, build_info) in self.libraries: - lib_names.append(lib_name) - return lib_names - - - def get_source_files(self): - self.check_library_list(self.libraries) - filenames = [] - for (lib_name, build_info) in self.libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - - filenames.extend(sources) - return filenames - - - def build_libraries(self, libraries): - for (lib_name, build_info) in libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - sources = list(sources) - - log.info("building '%s' library", lib_name) - - # First, compile the source code to object files in the library - # directory. (This should probably change to putting object - # files in a temporary build directory.) - macros = build_info.get('macros') - include_dirs = build_info.get('include_dirs') - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug) - - # Now "link" the object files together into a static library. - # (On Unix at least, this isn't really linking -- it just - # builds an archive. Whatever.) - self.compiler.create_static_lib(objects, lib_name, - output_dir=self.build_clib, - debug=self.debug) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py deleted file mode 100644 index acf2fc5484a..00000000000 --- a/Lib/distutils/command/build_ext.py +++ /dev/null @@ -1,755 +0,0 @@ -"""distutils.command.build_ext - -Implements the Distutils 'build_ext' command, for building extension -modules (currently limited to C extensions, should accommodate C++ -extensions ASAP).""" - -import contextlib -import os -import re -import sys -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler, get_python_version -from distutils.sysconfig import get_config_h_filename -from distutils.dep_util import newer_group -from distutils.extension import Extension -from distutils.util import get_platform -from distutils import log - -from site import USER_BASE - -# An extension name is just a dot-separated list of Python NAMEs (ie. -# the same as a fully-qualified module name). -extension_name_re = re.compile \ - (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') - - -def show_compilers (): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_ext(Command): - - description = "build C/C++ extensions (compile/link to build directory)" - - # XXX thoughts on how to deal with complex command-line options like - # these, i.e. how to make it so fancy_getopt can suck them off the - # command line and make it look like setup.py defined the appropriate - # lists of tuples of what-have-you. - # - each command needs a callback to process its command-line options - # - Command.__init__() needs access to its share of the whole - # command line (must ultimately come from - # Distribution.parse_command_line()) - # - it then calls the current command class' option-parsing - # callback to deal with weird options like -D, which have to - # parse the option text and churn out some custom data - # structure - # - that data structure (in this case, a list of 2-tuples) - # will then be present in the command object by the time - # we get to finalize_options() (i.e. the constructor - # takes care of both command-line and client options - # in between initialize_options() and finalize_options()) - - sep_by = " (separated by '%s')" % os.pathsep - user_options = [ - ('build-lib=', 'b', - "directory for compiled extension modules"), - ('build-temp=', 't', - "directory for temporary files (build by-products)"), - ('plat-name=', 'p', - "platform name to cross-compile for, if supported " - "(default: %s)" % get_platform()), - ('inplace', 'i', - "ignore build-lib and put compiled extensions into the source " + - "directory alongside your pure Python modules"), - ('include-dirs=', 'I', - "list of directories to search for header files" + sep_by), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries" + sep_by), - ('rpath=', 'R', - "directories to search for shared C libraries at runtime"), - ('link-objects=', 'O', - "extra explicit link objects to include in the link"), - ('debug', 'g', - "compile/link with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ('parallel=', 'j', - "number of parallel build jobs"), - ('swig-cpp', None, - "make SWIG create C++ files (default is C)"), - ('swig-opts=', None, - "list of SWIG command line options"), - ('swig=', None, - "path to the SWIG executable"), - ('user', None, - "add user include, library and rpath") - ] - - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.extensions = None - self.build_lib = None - self.plat_name = None - self.build_temp = None - self.inplace = 0 - self.package = None - - self.include_dirs = None - self.define = None - self.undef = None - self.libraries = None - self.library_dirs = None - self.rpath = None - self.link_objects = None - self.debug = None - self.force = None - self.compiler = None - self.swig = None - self.swig_cpp = None - self.swig_opts = None - self.user = None - self.parallel = None - - def finalize_options(self): - from distutils import sysconfig - - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('parallel', 'parallel'), - ('plat_name', 'plat_name'), - ) - - if self.package is None: - self.package = self.distribution.ext_package - - self.extensions = self.distribution.ext_modules - - # Make sure Python's include directories (for Python.h, pyconfig.h, - # etc.) are in the include search path. - py_include = sysconfig.get_python_inc() - plat_py_include = sysconfig.get_python_inc(plat_specific=1) - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # If in a virtualenv, add its include directory - # Issue 16116 - if sys.exec_prefix != sys.base_exec_prefix: - self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) - - # Put the Python "system" include dir at the end, so that - # any local include dirs take precedence. - self.include_dirs.append(py_include) - if plat_py_include != py_include: - self.include_dirs.append(plat_py_include) - - self.ensure_string_list('libraries') - self.ensure_string_list('link_objects') - - # Life is easier if we're not forever checking for None, so - # simplify these options to empty lists if unset - if self.libraries is None: - self.libraries = [] - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - if self.rpath is None: - self.rpath = [] - elif isinstance(self.rpath, str): - self.rpath = self.rpath.split(os.pathsep) - - # for extensions under windows use different directories - # for Release and Debug builds. - # also Python's library directory must be appended to library_dirs - if os.name == 'nt': - # the 'libs' directory is for binary installs - we assume that - # must be the *native* platform. But we don't really support - # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) - if sys.base_exec_prefix != sys.prefix: # Issue 16116 - self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs')) - if self.debug: - self.build_temp = os.path.join(self.build_temp, "Debug") - else: - self.build_temp = os.path.join(self.build_temp, "Release") - - # Append the source distribution include and library directories, - # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.dirname(get_config_h_filename())) - _sys_home = getattr(sys, '_home', None) - if _sys_home: - self.library_dirs.append(_sys_home) - - # Use the .lib files for the correct architecture - if self.plat_name == 'win32': - suffix = 'win32' - else: - # win-amd64 or win-ia64 - suffix = self.plat_name[4:] - new_lib = os.path.join(sys.exec_prefix, 'PCbuild') - if suffix: - new_lib = os.path.join(new_lib, suffix) - self.library_dirs.append(new_lib) - - # for extensions under Cygwin and AtheOS Python's library directory must be - # appended to library_dirs - if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): - # building third party extensions - self.library_dirs.append(os.path.join(sys.prefix, "lib", - "python" + get_python_version(), - "config")) - else: - # building python standard extensions - self.library_dirs.append('.') - - # For building extensions with a shared Python library, - # Python's library directory must be appended to library_dirs - # See Issues: #1600860, #4366 - if False and (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if not sysconfig.python_build: - # building third party extensions - self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) - else: - # building python standard extensions - self.library_dirs.append('.') - - # The argument parsing will result in self.define being a string, but - # it has to be a list of 2-tuples. All the preprocessor symbols - # specified by the 'define' option will be set to '1'. Multiple - # symbols can be separated with commas. - - if self.define: - defines = self.define.split(',') - self.define = [(symbol, '1') for symbol in defines] - - # The option for macros to undefine is also a string from the - # option parsing, but has to be a list. Multiple symbols can also - # be separated with commas here. - if self.undef: - self.undef = self.undef.split(',') - - if self.swig_opts is None: - self.swig_opts = [] - else: - self.swig_opts = self.swig_opts.split(' ') - - # Finally add the user include and library directories if requested - if self.user: - user_include = os.path.join(USER_BASE, "include") - user_lib = os.path.join(USER_BASE, "lib") - if os.path.isdir(user_include): - self.include_dirs.append(user_include) - if os.path.isdir(user_lib): - self.library_dirs.append(user_lib) - self.rpath.append(user_lib) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - from distutils.ccompiler import new_compiler - - # 'self.extensions', as supplied by setup.py, is a list of - # Extension instances. See the documentation for Extension (in - # distutils.extension) for details. - # - # For backwards compatibility with Distutils 0.8.2 and earlier, we - # also allow the 'extensions' list to be a list of tuples: - # (ext_name, build_info) - # where build_info is a dictionary containing everything that - # Extension instances do except the name, with a few things being - # differently named. We convert these 2-tuples to Extension - # instances as needed. - - if not self.extensions: - return - - # If we were asked to build any C/C++ libraries, make sure that the - # directory where we put them is in the library search path for - # linking extensions. - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.libraries.extend(build_clib.get_library_names() or []) - self.library_dirs.append(build_clib.build_clib) - - # Setup the CCompiler object that we'll use to do all the - # compiling and linking - self.compiler = new_compiler(compiler=self.compiler, - verbose=self.verbose, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - # If we are cross-compiling, init the compiler now (if we are not - # cross-compiling, init would not hurt, but people may rely on - # late initialization of compiler even if they shouldn't...) - if os.name == 'nt' and self.plat_name != get_platform(): - self.compiler.initialize(self.plat_name) - - # And make sure that any compile/link-related options (which might - # come from the command-line or from the setup script) are set in - # that CCompiler object -- that way, they automatically apply to - # all compiling and linking done here. - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name, value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - if self.libraries is not None: - self.compiler.set_libraries(self.libraries) - if self.library_dirs is not None: - self.compiler.set_library_dirs(self.library_dirs) - if self.rpath is not None: - self.compiler.set_runtime_library_dirs(self.rpath) - if self.link_objects is not None: - self.compiler.set_link_objects(self.link_objects) - - # Now actually compile and link everything. - self.build_extensions() - - def check_extensions_list(self, extensions): - """Ensure that the list of extensions (presumably provided as a - command option 'extensions') is valid, i.e. it is a list of - Extension objects. We also support the old-style list of 2-tuples, - where the tuples are (ext_name, build_info), which are converted to - Extension instances here. - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(extensions, list): - raise DistutilsSetupError( - "'ext_modules' option must be a list of Extension instances") - - for i, ext in enumerate(extensions): - if isinstance(ext, Extension): - continue # OK! (assume type-checking done - # by Extension constructor) - - if not isinstance(ext, tuple) or len(ext) != 2: - raise DistutilsSetupError( - "each element of 'ext_modules' option must be an " - "Extension instance or 2-tuple") - - ext_name, build_info = ext - - log.warn("old-style (ext_name, build_info) tuple found in " - "ext_modules for extension '%s' " - "-- please convert to Extension instance", ext_name) - - if not (isinstance(ext_name, str) and - extension_name_re.match(ext_name)): - raise DistutilsSetupError( - "first element of each tuple in 'ext_modules' " - "must be the extension name (a string)") - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'ext_modules' " - "must be a dictionary (build info)") - - # OK, the (ext_name, build_info) dict is type-safe: convert it - # to an Extension instance. - ext = Extension(ext_name, build_info['sources']) - - # Easy stuff: one-to-one mapping from dict elements to - # instance attributes. - for key in ('include_dirs', 'library_dirs', 'libraries', - 'extra_objects', 'extra_compile_args', - 'extra_link_args'): - val = build_info.get(key) - if val is not None: - setattr(ext, key, val) - - # Medium-easy stuff: same syntax/semantics, different names. - ext.runtime_library_dirs = build_info.get('rpath') - if 'def_file' in build_info: - log.warn("'def_file' element of build info dict " - "no longer supported") - - # Non-trivial stuff: 'macros' split into 'define_macros' - # and 'undef_macros'. - macros = build_info.get('macros') - if macros: - ext.define_macros = [] - ext.undef_macros = [] - for macro in macros: - if not (isinstance(macro, tuple) and len(macro) in (1, 2)): - raise DistutilsSetupError( - "'macros' element of build info dict " - "must be 1- or 2-tuple") - if len(macro) == 1: - ext.undef_macros.append(macro[0]) - elif len(macro) == 2: - ext.define_macros.append(macro) - - extensions[i] = ext - - def get_source_files(self): - self.check_extensions_list(self.extensions) - filenames = [] - - # Wouldn't it be neat if we knew the names of header files too... - for ext in self.extensions: - filenames.extend(ext.sources) - return filenames - - def get_outputs(self): - # Sanity check the 'extensions' list -- can't assume this is being - # done in the same run as a 'build_extensions()' call (in fact, we - # can probably assume that it *isn't*!). - self.check_extensions_list(self.extensions) - - # And build the list of output (built) filenames. Note that this - # ignores the 'inplace' flag, and assumes everything goes in the - # "build" tree. - outputs = [] - for ext in self.extensions: - outputs.append(self.get_ext_fullpath(ext.name)) - return outputs - - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - if self.parallel: - self._build_extensions_parallel() - else: - self._build_extensions_serial() - - def _build_extensions_parallel(self): - workers = self.parallel - if self.parallel is True: - workers = os.cpu_count() # may return None - try: - from concurrent.futures import ThreadPoolExecutor - except ImportError: - workers = None - - if workers is None: - self._build_extensions_serial() - return - - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [executor.submit(self.build_extension, ext) - for ext in self.extensions] - for ext, fut in zip(self.extensions, futures): - with self._filter_build_errors(ext): - fut.result() - - def _build_extensions_serial(self): - for ext in self.extensions: - with self._filter_build_errors(ext): - self.build_extension(ext) - - @contextlib.contextmanager - def _filter_build_errors(self, ext): - try: - yield - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) - - def build_extension(self, ext): - sources = ext.sources - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'ext_modules' option (extension '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % ext.name) - sources = list(sources) - - ext_path = self.get_ext_fullpath(ext.name) - depends = sources + ext.depends - if not (self.force or newer_group(depends, ext_path, 'newer')): - log.debug("skipping '%s' extension (up-to-date)", ext.name) - return - else: - log.info("building '%s' extension", ext.name) - - # First, scan the sources for SWIG definition files (.i), run - # SWIG on 'em to create .c files, and modify the sources list - # accordingly. - sources = self.swig_sources(sources, ext) - - # Next, compile the source code to object files. - - # XXX not honouring 'define_macros' or 'undef_macros' -- the - # CCompiler API needs to change to accommodate this, and I - # want to do one thing at a time! - - # Two possible sources for extra compiler arguments: - # - 'extra_compile_args' in Extension object - # - CFLAGS environment variable (not particularly - # elegant, but people seem to expect it and I - # guess it's useful) - # The environment variable should take precedence, and - # any sensible compiler will give precedence to later - # command line args. Hence we combine them in order: - extra_args = ext.extra_compile_args or [] - - macros = ext.define_macros[:] - for undef in ext.undef_macros: - macros.append((undef,)) - - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=ext.include_dirs, - debug=self.debug, - extra_postargs=extra_args, - depends=ext.depends) - - # XXX outdated variable, kept here in case third-part code - # needs it. - self._built_objects = objects[:] - - # Now link the object files together into a "shared object" -- - # of course, first we have to figure out all the other things - # that go into the mix. - if ext.extra_objects: - objects.extend(ext.extra_objects) - extra_args = ext.extra_link_args or [] - - # Detect target language, if not provided - language = ext.language or self.compiler.detect_language(sources) - - self.compiler.link_shared_object( - objects, ext_path, - libraries=self.get_libraries(ext), - library_dirs=ext.library_dirs, - runtime_library_dirs=ext.runtime_library_dirs, - extra_postargs=extra_args, - export_symbols=self.get_export_symbols(ext), - debug=self.debug, - build_temp=self.build_temp, - target_lang=language) - - def swig_sources(self, sources, extension): - """Walk the list of source files in 'sources', looking for SWIG - interface (.i) files. Run SWIG on all that are found, and - return a modified 'sources' list with SWIG source files replaced - by the generated C (or C++) files. - """ - new_sources = [] - swig_sources = [] - swig_targets = {} - - # XXX this drops generated C/C++ files into the source tree, which - # is fine for developers who want to distribute the generated - # source -- but there should be an option to put SWIG output in - # the temp dir. - - if self.swig_cpp: - log.warn("--swig-cpp is deprecated - use --swig-opts=-c++") - - if self.swig_cpp or ('-c++' in self.swig_opts) or \ - ('-c++' in extension.swig_opts): - target_ext = '.cpp' - else: - target_ext = '.c' - - for source in sources: - (base, ext) = os.path.splitext(source) - if ext == ".i": # SWIG interface file - new_sources.append(base + '_wrap' + target_ext) - swig_sources.append(source) - swig_targets[source] = new_sources[-1] - else: - new_sources.append(source) - - if not swig_sources: - return new_sources - - swig = self.swig or self.find_swig() - swig_cmd = [swig, "-python"] - swig_cmd.extend(self.swig_opts) - if self.swig_cpp: - swig_cmd.append("-c++") - - # Do not override commandline arguments - if not self.swig_opts: - for o in extension.swig_opts: - swig_cmd.append(o) - - for source in swig_sources: - target = swig_targets[source] - log.info("swigging %s to %s", source, target) - self.spawn(swig_cmd + ["-o", target, source]) - - return new_sources - - def find_swig(self): - """Return the name of the SWIG executable. On Unix, this is - just "swig" -- it should be in the PATH. Tries a bit harder on - Windows. - """ - if os.name == "posix": - return "swig" - elif os.name == "nt": - # Look for SWIG in its standard installation directory on - # Windows (or so I presume!). If we find it there, great; - # if not, act like Unix and assume it's in the PATH. - for vers in ("1.3", "1.2", "1.1"): - fn = os.path.join("c:\\swig%s" % vers, "swig.exe") - if os.path.isfile(fn): - return fn - else: - return "swig.exe" - else: - raise DistutilsPlatformError( - "I don't know how to find (much less run) SWIG " - "on platform '%s'" % os.name) - - # -- Name generators ----------------------------------------------- - # (extension names, filenames, whatever) - def get_ext_fullpath(self, ext_name): - """Returns the path of the filename for a given extension. - - The file is located in `build_lib` or directly in the package - (inplace option). - """ - fullname = self.get_ext_fullname(ext_name) - modpath = fullname.split('.') - filename = self.get_ext_filename(modpath[-1]) - - if not self.inplace: - # no further work needed - # returning : - # build_dir/package/path/filename - filename = os.path.join(*modpath[:-1]+[filename]) - return os.path.join(self.build_lib, filename) - - # the inplace option requires to find the package directory - # using the build_py command for that - package = '.'.join(modpath[0:-1]) - build_py = self.get_finalized_command('build_py') - package_dir = os.path.abspath(build_py.get_package_dir(package)) - - # returning - # package_dir/filename - return os.path.join(package_dir, filename) - - def get_ext_fullname(self, ext_name): - """Returns the fullname of a given extension name. - - Adds the `package.` prefix""" - if self.package is None: - return ext_name - else: - return self.package + '.' + ext_name - - def get_ext_filename(self, ext_name): - r"""Convert the name of an extension (eg. "foo.bar") into the name - of the file from which it will be loaded (eg. "foo/bar.so", or - "foo\bar.pyd"). - """ - from distutils.sysconfig import get_config_var - ext_path = ext_name.split('.') - ext_suffix = get_config_var('EXT_SUFFIX') - return os.path.join(*ext_path) + ext_suffix - - def get_export_symbols(self, ext): - """Return the list of symbols that a shared extension has to - export. This either uses 'ext.export_symbols' or, if it's not - provided, "PyInit_" + module_name. Only relevant on Windows, where - the .pyd file (DLL) must export the module "PyInit_" function. - """ - initfunc_name = "PyInit_" + ext.name.split('.')[-1] - if initfunc_name not in ext.export_symbols: - ext.export_symbols.append(initfunc_name) - return ext.export_symbols - - def get_libraries(self, ext): - """Return the list of libraries to link against when building a - shared extension. On most platforms, this is just 'ext.libraries'; - on Windows, we add the Python library (eg. python20.dll). - """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. - if sys.platform == "win32": - from distutils._msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries - elif sys.platform[:6] == "cygwin": - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - elif sys.platform[:6] == "atheos": - from distutils import sysconfig - - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # Get SHLIBS from Makefile - extra = [] - for lib in sysconfig.get_config_var('SHLIBS').split(): - if lib.startswith('-l'): - extra.append(lib[2:]) - else: - extra.append(lib) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib, "m"] + extra - elif sys.platform == 'darwin': - # Don't use the default code below - return ext.libraries - elif sys.platform[:3] == 'aix': - # Don't use the default code below - return ext.libraries - else: - from distutils import sysconfig - if False and sysconfig.get_config_var('Py_ENABLE_SHARED'): - pythonlib = 'python{}.{}{}'.format( - sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff, - sysconfig.get_config_var('ABIFLAGS')) - return ext.libraries + [pythonlib] - else: - return ext.libraries diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py deleted file mode 100644 index cf0ca57c320..00000000000 --- a/Lib/distutils/command/build_py.py +++ /dev/null @@ -1,416 +0,0 @@ -"""distutils.command.build_py - -Implements the Distutils 'build_py' command.""" - -import os -import importlib.util -import sys -from glob import glob - -from distutils.core import Command -from distutils.errors import * -from distutils.util import convert_path, Mixin2to3 -from distutils import log - -class build_py (Command): - - description = "\"build\" pure Python modules (copy to build directory)" - - user_options = [ - ('build-lib=', 'd', "directory to \"build\" (copy) to"), - ('compile', 'c', "compile .py to .pyc"), - ('no-compile', None, "don't compile .py files [default]"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('force', 'f', "forcibly build everything (ignore file timestamps)"), - ] - - boolean_options = ['compile', 'force'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - self.build_lib = None - self.py_modules = None - self.package = None - self.package_data = None - self.package_dir = None - self.compile = 0 - self.optimize = 0 - self.force = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('force', 'force')) - - # Get the distribution options that are aliases for build_py - # options -- list of packages and list of modules. - self.packages = self.distribution.packages - self.py_modules = self.distribution.py_modules - self.package_data = self.distribution.package_data - self.package_dir = {} - if self.distribution.package_dir: - for name, path in self.distribution.package_dir.items(): - self.package_dir[name] = convert_path(path) - self.data_files = self.get_data_files() - - # Ick, copied straight from install_lib.py (fancy_getopt needs a - # type system! Hell, *everything* needs a type system!!!) - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - assert 0 <= self.optimize <= 2 - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # XXX copy_file by default preserves atime and mtime. IMHO this is - # the right thing to do, but perhaps it should be an option -- in - # particular, a site administrator might want installed files to - # reflect the time of installation rather than the last - # modification time before the installed release. - - # XXX copy_file by default preserves mode, which appears to be the - # wrong thing to do: if a file is read-only in the working - # directory, we want it to be installed read/write so that the next - # installation of the same module distribution can overwrite it - # without problems. (This might be a Unix-specific issue.) Thus - # we turn off 'preserve_mode' when copying to the build directory, - # since the build directory is supposed to be exactly what the - # installation will look like (ie. we preserve mode when - # installing). - - # Two options control which modules will be installed: 'packages' - # and 'py_modules'. The former lets us work with whole packages, not - # specifying individual modules at all; the latter is for - # specifying modules one-at-a-time. - - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def get_data_files(self): - """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir)+1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - - def find_data_files(self, package, src_dir): - """Return filenames for package's data files in 'src_dir'""" - globs = (self.package_data.get('', []) - + self.package_data.get(package, [])) - files = [] - for pattern in globs: - # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) - # Files that match more than one pattern are only added once - files.extend([fn for fn in filelist if fn not in files - and os.path.isfile(fn)]) - return files - - def build_package_data(self): - """Copy data files into build directory""" - lastdir = None - for package, src_dir, build_dir, filenames in self.data_files: - for filename in filenames: - target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) - self.copy_file(os.path.join(src_dir, filename), target, - preserve_mode=False) - - def get_package_dir(self, package): - """Return the directory, relative to the top of the source - distribution, where package 'package' should be found - (at least according to the 'package_dir' option, if any).""" - path = package.split('.') - - if not self.package_dir: - if path: - return os.path.join(*path) - else: - return '' - else: - tail = [] - while path: - try: - pdir = self.package_dir['.'.join(path)] - except KeyError: - tail.insert(0, path[-1]) - del path[-1] - else: - tail.insert(0, pdir) - return os.path.join(*tail) - else: - # Oops, got all the way through 'path' without finding a - # match in package_dir. If package_dir defines a directory - # for the root (nameless) package, then fallback on it; - # otherwise, we might as well have not consulted - # package_dir at all, as we just use the directory implied - # by 'tail' (which should be the same as the original value - # of 'path' at this point). - pdir = self.package_dir.get('') - if pdir is not None: - tail.insert(0, pdir) - - if tail: - return os.path.join(*tail) - else: - return '' - - def check_package(self, package, package_dir): - # Empty dir name means current directory, which we can probably - # assume exists. Also, os.path.exists and isdir don't know about - # my "empty string means current dir" convention, so we have to - # circumvent them. - if package_dir != "": - if not os.path.exists(package_dir): - raise DistutilsFileError( - "package directory '%s' does not exist" % package_dir) - if not os.path.isdir(package_dir): - raise DistutilsFileError( - "supposed package directory '%s' exists, " - "but is not a directory" % package_dir) - - # Require __init__.py for all but the "root package" - if package: - init_py = os.path.join(package_dir, "__init__.py") - if os.path.isfile(init_py): - return init_py - else: - log.warn(("package init file '%s' not found " + - "(or not a regular file)"), init_py) - - # Either not in a package at all (__init__.py not expected), or - # __init__.py doesn't exist -- so don't return the filename. - return None - - def check_module(self, module, module_file): - if not os.path.isfile(module_file): - log.warn("file %s (for module %s) not found", module_file, module) - return False - else: - return True - - def find_package_modules(self, package, package_dir): - self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) - modules = [] - setup_script = os.path.abspath(self.distribution.script_name) - - for f in module_files: - abs_f = os.path.abspath(f) - if abs_f != setup_script: - module = os.path.splitext(os.path.basename(f))[0] - modules.append((package, module, f)) - else: - self.debug_print("excluding %s" % setup_script) - return modules - - def find_modules(self): - """Finds individually-specified Python modules, ie. those listed by - module name in 'self.py_modules'. Returns a list of tuples (package, - module_base, filename): 'package' is a tuple of the path through - package-space to the module; 'module_base' is the bare (no - packages, no dots) module name, and 'filename' is the path to the - ".py" file (relative to the distribution root) that implements the - module. - """ - # Map package names to tuples of useful info about the package: - # (package_dir, checked) - # package_dir - the directory where we'll find source files for - # this package - # checked - true if we have checked that the package directory - # is valid (exists, contains __init__.py, ... ?) - packages = {} - - # List of (package, module, filename) tuples to return - modules = [] - - # We treat modules-in-packages almost the same as toplevel modules, - # just the "package" for a toplevel is empty (either an empty - # string or empty list, depending on context). Differences: - # - don't check for __init__.py in directory for empty package - for module in self.py_modules: - path = module.split('.') - package = '.'.join(path[0:-1]) - module_base = path[-1] - - try: - (package_dir, checked) = packages[package] - except KeyError: - package_dir = self.get_package_dir(package) - checked = 0 - - if not checked: - init_py = self.check_package(package, package_dir) - packages[package] = (package_dir, 1) - if init_py: - modules.append((package, "__init__", init_py)) - - # XXX perhaps we should also check for just .pyc files - # (so greedy closed-source bastards can distribute Python - # modules too) - module_file = os.path.join(package_dir, module_base + ".py") - if not self.check_module(module, module_file): - continue - - modules.append((package, module_base, module_file)) - - return modules - - def find_all_modules(self): - """Compute the list of all modules that will be built, whether - they are specified one-module-at-a-time ('self.py_modules') or - by whole packages ('self.packages'). Return a list of tuples - (package, module, module_file), just like 'find_modules()' and - 'find_package_modules()' do.""" - modules = [] - if self.py_modules: - modules.extend(self.find_modules()) - if self.packages: - for package in self.packages: - package_dir = self.get_package_dir(package) - m = self.find_package_modules(package, package_dir) - modules.extend(m) - return modules - - def get_source_files(self): - return [module[-1] for module in self.find_all_modules()] - - def get_module_outfile(self, build_dir, package, module): - outfile_path = [build_dir] + list(package) + [module + ".py"] - return os.path.join(*outfile_path) - - def get_outputs(self, include_bytecode=1): - modules = self.find_all_modules() - outputs = [] - for (package, module, module_file) in modules: - package = package.split('.') - filename = self.get_module_outfile(self.build_lib, package, module) - outputs.append(filename) - if include_bytecode: - if self.compile: - outputs.append(importlib.util.cache_from_source( - filename, optimization='')) - if self.optimize > 0: - outputs.append(importlib.util.cache_from_source( - filename, optimization=self.optimize)) - - outputs += [ - os.path.join(build_dir, filename) - for package, src_dir, build_dir, filenames in self.data_files - for filename in filenames - ] - - return outputs - - def build_module(self, module, module_file, package): - if isinstance(package, str): - package = package.split('.') - elif not isinstance(package, (list, tuple)): - raise TypeError( - "'package' must be a string (dot-separated), list, or tuple") - - # Now put the module source file into the "build" area -- this is - # easy, we just copy it somewhere under self.build_lib (the build - # directory for Python source). - outfile = self.get_module_outfile(self.build_lib, package, module) - dir = os.path.dirname(outfile) - self.mkpath(dir) - return self.copy_file(module_file, outfile, preserve_mode=0) - - def build_modules(self): - modules = self.find_modules() - for (package, module, module_file) in modules: - # Now "build" the module -- ie. copy the source file to - # self.build_lib (the build directory for Python source). - # (Actually, it gets copied to the directory for this package - # under self.build_lib.) - self.build_module(module, module_file, package) - - def build_packages(self): - for package in self.packages: - # Get list of (package, module, module_file) tuples based on - # scanning the package directory. 'package' is only included - # in the tuple so that 'find_modules()' and - # 'find_package_tuples()' have a consistent interface; it's - # ignored here (apart from a sanity check). Also, 'module' is - # the *unqualified* module name (ie. no dots, no package -- we - # already know its package!), and 'module_file' is the path to - # the .py file, relative to the current directory - # (ie. including 'package_dir'). - package_dir = self.get_package_dir(package) - modules = self.find_package_modules(package, package_dir) - - # Now loop over the modules we found, "building" each one (just - # copy it to self.build_lib). - for (package_, module, module_file) in modules: - assert package == package_ - self.build_module(module, module_file, package) - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - prefix = self.build_lib - if prefix[-1] != os.sep: - prefix = prefix + os.sep - - # XXX this code is essentially the same as the 'byte_compile() - # method of the "install_lib" command, except for the determination - # of the 'prefix' string. Hmmm. - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=prefix, dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=prefix, dry_run=self.dry_run) - -class build_py_2to3(build_py, Mixin2to3): - def run(self): - self.updated_files = [] - - # Base class code - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - # 2to3 - self.run_2to3(self.updated_files) - - # Remaining base class code - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def build_module(self, module, module_file, package): - res = build_py.build_module(self, module, module_file, package) - if res[1]: - # file was copied - self.updated_files.append(res[0]) - return res diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py deleted file mode 100644 index ccc70e64650..00000000000 --- a/Lib/distutils/command/build_scripts.py +++ /dev/null @@ -1,160 +0,0 @@ -"""distutils.command.build_scripts - -Implements the Distutils 'build_scripts' command.""" - -import os, re -from stat import ST_MODE -from distutils import sysconfig -from distutils.core import Command -from distutils.dep_util import newer -from distutils.util import convert_path, Mixin2to3 -from distutils import log -import tokenize - -# check if Python is called on the first line with this expression -first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') - -class build_scripts(Command): - - description = "\"build\" scripts (copy and fixup #! line)" - - user_options = [ - ('build-dir=', 'd', "directory to \"build\" (copy) to"), - ('force', 'f', "forcibly build everything (ignore file timestamps"), - ('executable=', 'e', "specify final destination interpreter path"), - ] - - boolean_options = ['force'] - - - def initialize_options(self): - self.build_dir = None - self.scripts = None - self.force = None - self.executable = None - self.outfiles = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_scripts', 'build_dir'), - ('force', 'force'), - ('executable', 'executable')) - self.scripts = self.distribution.scripts - - def get_source_files(self): - return self.scripts - - def run(self): - if not self.scripts: - return - self.copy_scripts() - - - def copy_scripts(self): - r"""Copy each script listed in 'self.scripts'; if it's marked as a - Python script in the Unix way (first line matches 'first_line_re', - ie. starts with "\#!" and contains "python"), then adjust the first - line to refer to the current Python interpreter as we copy. - """ - self.mkpath(self.build_dir) - outfiles = [] - updated_files = [] - for script in self.scripts: - adjust = False - script = convert_path(script) - outfile = os.path.join(self.build_dir, os.path.basename(script)) - outfiles.append(outfile) - - if not self.force and not newer(script, outfile): - log.debug("not copying %s (up-to-date)", script) - continue - - # Always open the file, but ignore failures in dry-run mode -- - # that way, we'll get accurate feedback if we can read the - # script. - try: - f = open(script, "rb") - except OSError: - if not self.dry_run: - raise - f = None - else: - encoding, lines = tokenize.detect_encoding(f.readline) - f.seek(0) - first_line = f.readline() - if not first_line: - self.warn("%s is an empty file (skipping)" % script) - continue - - match = first_line_re.match(first_line) - if match: - adjust = True - post_interp = match.group(1) or b'' - - if adjust: - log.info("copying and adjusting %s -> %s", script, - self.build_dir) - updated_files.append(outfile) - if not self.dry_run: - if not sysconfig.python_build: - executable = self.executable - else: - executable = os.path.join( - sysconfig.get_config_var("BINDIR"), - "python%s%s" % (sysconfig.get_config_var("VERSION"), - sysconfig.get_config_var("EXE"))) - executable = os.fsencode(executable) - shebang = b"#!" + executable + post_interp + b"\n" - # Python parser starts to read a script using UTF-8 until - # it gets a #coding:xxx cookie. The shebang has to be the - # first line of a file, the #coding:xxx cookie cannot be - # written before. So the shebang has to be decodable from - # UTF-8. - try: - shebang.decode('utf-8') - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from utf-8".format(shebang)) - # If the script is encoded to a custom encoding (use a - # #coding:xxx cookie), the shebang has to be decodable from - # the script encoding too. - try: - shebang.decode(encoding) - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from the script encoding ({})" - .format(shebang, encoding)) - with open(outfile, "wb") as outf: - outf.write(shebang) - outf.writelines(f.readlines()) - if f: - f.close() - else: - if f: - f.close() - updated_files.append(outfile) - self.copy_file(script, outfile) - - if os.name == 'posix': - for file in outfiles: - if self.dry_run: - log.info("changing mode of %s", file) - else: - oldmode = os.stat(file)[ST_MODE] & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - if newmode != oldmode: - log.info("changing mode of %s from %o to %o", - file, oldmode, newmode) - os.chmod(file, newmode) - # XXX should we modify self.outfiles? - return outfiles, updated_files - -class build_scripts_2to3(build_scripts, Mixin2to3): - - def copy_scripts(self): - outfiles, updated_files = build_scripts.copy_scripts(self) - if not self.dry_run: - self.run_2to3(updated_files) - return outfiles, updated_files diff --git a/Lib/distutils/command/check.py b/Lib/distutils/command/check.py deleted file mode 100644 index 7ebe707cff4..00000000000 --- a/Lib/distutils/command/check.py +++ /dev/null @@ -1,145 +0,0 @@ -"""distutils.command.check - -Implements the Distutils 'check' command. -""" -from distutils.core import Command -from distutils.errors import DistutilsSetupError - -try: - # docutils is installed - from docutils.utils import Reporter - from docutils.parsers.rst import Parser - from docutils import frontend - from docutils import nodes - from io import StringIO - - class SilentReporter(Reporter): - - def __init__(self, source, report_level, halt_level, stream=None, - debug=0, encoding='ascii', error_handler='replace'): - self.messages = [] - Reporter.__init__(self, source, report_level, halt_level, stream, - debug, encoding, error_handler) - - def system_message(self, level, message, *children, **kwargs): - self.messages.append((level, message, children, kwargs)) - return nodes.system_message(message, level=level, - type=self.levels[level], - *children, **kwargs) - - HAS_DOCUTILS = True -except Exception: - # Catch all exceptions because exceptions besides ImportError probably - # indicate that docutils is not ported to Py3k. - HAS_DOCUTILS = False - -class check(Command): - """This command checks the meta-data of the package. - """ - description = ("perform some checks on the package") - user_options = [('metadata', 'm', 'Verify meta-data'), - ('restructuredtext', 'r', - ('Checks if long string meta-data syntax ' - 'are reStructuredText-compliant')), - ('strict', 's', - 'Will exit with an error if a check fails')] - - boolean_options = ['metadata', 'restructuredtext', 'strict'] - - def initialize_options(self): - """Sets default values for options.""" - self.restructuredtext = 0 - self.metadata = 1 - self.strict = 0 - self._warnings = 0 - - def finalize_options(self): - pass - - def warn(self, msg): - """Counts the number of warnings that occurs.""" - self._warnings += 1 - return Command.warn(self, msg) - - def run(self): - """Runs the command.""" - # perform the various tests - if self.metadata: - self.check_metadata() - if self.restructuredtext: - if HAS_DOCUTILS: - self.check_restructuredtext() - elif self.strict: - raise DistutilsSetupError('The docutils package is needed.') - - # let's raise an error in strict mode, if we have at least - # one warning - if self.strict and self._warnings > 0: - raise DistutilsSetupError('Please correct your package.') - - def check_metadata(self): - """Ensures that all required elements of meta-data are supplied. - - name, version, URL, (author and author_email) or - (maintainer and maintainer_email)). - - Warns if any are missing. - """ - metadata = self.distribution.metadata - - missing = [] - for attr in ('name', 'version', 'url'): - if not (hasattr(metadata, attr) and getattr(metadata, attr)): - missing.append(attr) - - if missing: - self.warn("missing required meta-data: %s" % ', '.join(missing)) - if metadata.author: - if not metadata.author_email: - self.warn("missing meta-data: if 'author' supplied, " + - "'author_email' must be supplied too") - elif metadata.maintainer: - if not metadata.maintainer_email: - self.warn("missing meta-data: if 'maintainer' supplied, " + - "'maintainer_email' must be supplied too") - else: - self.warn("missing meta-data: either (author and author_email) " + - "or (maintainer and maintainer_email) " + - "must be supplied") - - def check_restructuredtext(self): - """Checks if the long string fields are reST-compliant.""" - data = self.distribution.get_long_description() - for warning in self._check_rst_data(data): - line = warning[-1].get('line') - if line is None: - warning = warning[1] - else: - warning = '%s (line %s)' % (warning[1], line) - self.warn(warning) - - def _check_rst_data(self, data): - """Returns warnings when the provided data doesn't compile.""" - source_path = StringIO() - parser = Parser() - settings = frontend.OptionParser(components=(Parser,)).get_default_values() - settings.tab_width = 4 - settings.pep_references = None - settings.rfc_references = None - reporter = SilentReporter(source_path, - settings.report_level, - settings.halt_level, - stream=settings.warning_stream, - debug=settings.debug, - encoding=settings.error_encoding, - error_handler=settings.error_encoding_error_handler) - - document = nodes.document(settings, reporter, source=source_path) - document.note_source(source_path, -1) - try: - parser.parse(data, document) - except AttributeError as e: - reporter.messages.append( - (-1, 'Could not finish the parsing: %s.' % e, '', {})) - - return reporter.messages diff --git a/Lib/distutils/command/clean.py b/Lib/distutils/command/clean.py deleted file mode 100644 index 0cb27016621..00000000000 --- a/Lib/distutils/command/clean.py +++ /dev/null @@ -1,76 +0,0 @@ -"""distutils.command.clean - -Implements the Distutils 'clean' command.""" - -# contributed by Bastian Kleineidam , added 2000-03-18 - -import os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils import log - -class clean(Command): - - description = "clean up temporary files from 'build' command" - user_options = [ - ('build-base=', 'b', - "base build directory (default: 'build.build-base')"), - ('build-lib=', None, - "build directory for all modules (default: 'build.build-lib')"), - ('build-temp=', 't', - "temporary build directory (default: 'build.build-temp')"), - ('build-scripts=', None, - "build directory for scripts (default: 'build.build-scripts')"), - ('bdist-base=', None, - "temporary directory for built distributions"), - ('all', 'a', - "remove all build output, not just temporary by-products") - ] - - boolean_options = ['all'] - - def initialize_options(self): - self.build_base = None - self.build_lib = None - self.build_temp = None - self.build_scripts = None - self.bdist_base = None - self.all = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib'), - ('build_scripts', 'build_scripts'), - ('build_temp', 'build_temp')) - self.set_undefined_options('bdist', - ('bdist_base', 'bdist_base')) - - def run(self): - # remove the build/temp. directory (unless it's already - # gone) - if os.path.exists(self.build_temp): - remove_tree(self.build_temp, dry_run=self.dry_run) - else: - log.debug("'%s' does not exist -- can't clean it", - self.build_temp) - - if self.all: - # remove build directories - for directory in (self.build_lib, - self.bdist_base, - self.build_scripts): - if os.path.exists(directory): - remove_tree(directory, dry_run=self.dry_run) - else: - log.warn("'%s' does not exist -- can't clean it", - directory) - - # just for the heck of it, try to remove the base build directory: - # we might have emptied it right now, but if not we don't care - if not self.dry_run: - try: - os.rmdir(self.build_base) - log.info("removing '%s'", self.build_base) - except OSError: - pass diff --git a/Lib/distutils/command/command_template b/Lib/distutils/command/command_template deleted file mode 100644 index 6106819db84..00000000000 --- a/Lib/distutils/command/command_template +++ /dev/null @@ -1,33 +0,0 @@ -"""distutils.command.x - -Implements the Distutils 'x' command. -""" - -# created 2000/mm/dd, John Doe - -__revision__ = "$Id$" - -from distutils.core import Command - - -class x(Command): - - # Brief (40-50 characters) description of the command - description = "" - - # List of option tuples: long name, short name (None if no short - # name), and help string. - user_options = [('', '', - ""), - ] - - def initialize_options(self): - self. = None - self. = None - self. = None - - def finalize_options(self): - if self.x is None: - self.x = - - def run(self): diff --git a/Lib/distutils/command/config.py b/Lib/distutils/command/config.py deleted file mode 100644 index 4ae153d1943..00000000000 --- a/Lib/distutils/command/config.py +++ /dev/null @@ -1,347 +0,0 @@ -"""distutils.command.config - -Implements the Distutils 'config' command, a (mostly) empty command class -that exists mainly to be sub-classed by specific module distributions and -applications. The idea is that while every "config" command is different, -at least they're all named the same, and users always see "config" in the -list of standard commands. Also, this is a good place to put common -configure-like tasks: "try to compile this C code", or "figure out where -this header file lives". -""" - -import os, re - -from distutils.core import Command -from distutils.errors import DistutilsExecError -from distutils.sysconfig import customize_compiler -from distutils import log - -LANG_EXT = {"c": ".c", "c++": ".cxx"} - -class config(Command): - - description = "prepare to build" - - user_options = [ - ('compiler=', None, - "specify the compiler type"), - ('cc=', None, - "specify the compiler executable"), - ('include-dirs=', 'I', - "list of directories to search for header files"), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries"), - - ('noisy', None, - "show every action (compile, link, run, ...) taken"), - ('dump-source', None, - "dump generated source files before attempting to compile them"), - ] - - - # The three standard command methods: since the "config" command - # does nothing by default, these are empty. - - def initialize_options(self): - self.compiler = None - self.cc = None - self.include_dirs = None - self.libraries = None - self.library_dirs = None - - # maximal output for now - self.noisy = 1 - self.dump_source = 1 - - # list of temporary files generated along-the-way that we have - # to clean at some point - self.temp_files = [] - - def finalize_options(self): - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - elif isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - if self.libraries is None: - self.libraries = [] - elif isinstance(self.libraries, str): - self.libraries = [self.libraries] - - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - def run(self): - pass - - # Utility methods for actual "config" commands. The interfaces are - # loosely based on Autoconf macros of similar names. Sub-classes - # may use these freely. - - def _check_compiler(self): - """Check that 'self.compiler' really is a CCompiler object; - if not, make it one. - """ - # We do this late, and only on-demand, because this is an expensive - # import. - from distutils.ccompiler import CCompiler, new_compiler - if not isinstance(self.compiler, CCompiler): - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, force=1) - customize_compiler(self.compiler) - if self.include_dirs: - self.compiler.set_include_dirs(self.include_dirs) - if self.libraries: - self.compiler.set_libraries(self.libraries) - if self.library_dirs: - self.compiler.set_library_dirs(self.library_dirs) - - def _gen_temp_sourcefile(self, body, headers, lang): - filename = "_configtest" + LANG_EXT[lang] - file = open(filename, "w") - if headers: - for header in headers: - file.write("#include <%s>\n" % header) - file.write("\n") - file.write(body) - if body[-1] != "\n": - file.write("\n") - file.close() - return filename - - def _preprocess(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - out = "_configtest.i" - self.temp_files.extend([src, out]) - self.compiler.preprocess(src, out, include_dirs=include_dirs) - return (src, out) - - def _compile(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - if self.dump_source: - dump_file(src, "compiling '%s':" % src) - (obj,) = self.compiler.object_filenames([src]) - self.temp_files.extend([src, obj]) - self.compiler.compile([src], include_dirs=include_dirs) - return (src, obj) - - def _link(self, body, headers, include_dirs, libraries, library_dirs, - lang): - (src, obj) = self._compile(body, headers, include_dirs, lang) - prog = os.path.splitext(os.path.basename(src))[0] - self.compiler.link_executable([obj], prog, - libraries=libraries, - library_dirs=library_dirs, - target_lang=lang) - - if self.compiler.exe_extension is not None: - prog = prog + self.compiler.exe_extension - self.temp_files.append(prog) - - return (src, obj, prog) - - def _clean(self, *filenames): - if not filenames: - filenames = self.temp_files - self.temp_files = [] - log.info("removing: %s", ' '.join(filenames)) - for filename in filenames: - try: - os.remove(filename) - except OSError: - pass - - - # XXX these ignore the dry-run flag: what to do, what to do? even if - # you want a dry-run build, you still need some sort of configuration - # info. My inclination is to make it up to the real config command to - # consult 'dry_run', and assume a default (minimal) configuration if - # true. The problem with trying to do it here is that you'd have to - # return either true or false from all the 'try' methods, neither of - # which is correct. - - # XXX need access to the header search path and maybe default macros. - - def try_cpp(self, body=None, headers=None, include_dirs=None, lang="c"): - """Construct a source file from 'body' (a string containing lines - of C/C++ code) and 'headers' (a list of header files to include) - and run it through the preprocessor. Return true if the - preprocessor succeeded, false if there were any errors. - ('body' probably isn't of much use, but what the heck.) - """ - from distutils.ccompiler import CompileError - self._check_compiler() - ok = True - try: - self._preprocess(body, headers, include_dirs, lang) - except CompileError: - ok = False - - self._clean() - return ok - - def search_cpp(self, pattern, body=None, headers=None, include_dirs=None, - lang="c"): - """Construct a source file (just like 'try_cpp()'), run it through - the preprocessor, and return true if any line of the output matches - 'pattern'. 'pattern' should either be a compiled regex object or a - string containing a regex. If both 'body' and 'headers' are None, - preprocesses an empty file -- which can be useful to determine the - symbols the preprocessor and compiler set by default. - """ - self._check_compiler() - src, out = self._preprocess(body, headers, include_dirs, lang) - - if isinstance(pattern, str): - pattern = re.compile(pattern) - - file = open(out) - match = False - while True: - line = file.readline() - if line == '': - break - if pattern.search(line): - match = True - break - - file.close() - self._clean() - return match - - def try_compile(self, body, headers=None, include_dirs=None, lang="c"): - """Try to compile a source file built from 'body' and 'headers'. - Return true on success, false otherwise. - """ - from distutils.ccompiler import CompileError - self._check_compiler() - try: - self._compile(body, headers, include_dirs, lang) - ok = True - except CompileError: - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_link(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile and link a source file, built from 'body' and - 'headers', to executable form. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - ok = True - except (CompileError, LinkError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_run(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile, link to an executable, and run a program - built from 'body' and 'headers'. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - src, obj, exe = self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - self.spawn([exe]) - ok = True - except (CompileError, LinkError, DistutilsExecError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - - # -- High-level methods -------------------------------------------- - # (these are the ones that are actually likely to be useful - # when implementing a real-world config command!) - - def check_func(self, func, headers=None, include_dirs=None, - libraries=None, library_dirs=None, decl=0, call=0): - """Determine if function 'func' is available by constructing a - source file that refers to 'func', and compiles and links it. - If everything succeeds, returns true; otherwise returns false. - - The constructed source file starts out by including the header - files listed in 'headers'. If 'decl' is true, it then declares - 'func' (as "int func()"); you probably shouldn't supply 'headers' - and set 'decl' true in the same call, or you might get errors about - a conflicting declarations for 'func'. Finally, the constructed - 'main()' function either references 'func' or (if 'call' is true) - calls it. 'libraries' and 'library_dirs' are used when - linking. - """ - self._check_compiler() - body = [] - if decl: - body.append("int %s ();" % func) - body.append("int main () {") - if call: - body.append(" %s();" % func) - else: - body.append(" %s;" % func) - body.append("}") - body = "\n".join(body) + "\n" - - return self.try_link(body, headers, include_dirs, - libraries, library_dirs) - - def check_lib(self, library, library_dirs=None, headers=None, - include_dirs=None, other_libraries=[]): - """Determine if 'library' is available to be linked against, - without actually checking that any particular symbols are provided - by it. 'headers' will be used in constructing the source file to - be compiled, but the only effect of this is to check if all the - header files listed are available. Any libraries listed in - 'other_libraries' will be included in the link, in case 'library' - has symbols that depend on other libraries. - """ - self._check_compiler() - return self.try_link("int main (void) { }", headers, include_dirs, - [library] + other_libraries, library_dirs) - - def check_header(self, header, include_dirs=None, library_dirs=None, - lang="c"): - """Determine if the system header file named by 'header_file' - exists and can be found by the preprocessor; return true if so, - false otherwise. - """ - return self.try_cpp(body="/* No body */", headers=[header], - include_dirs=include_dirs) - - -def dump_file(filename, head=None): - """Dumps a file content into log.info. - - If head is not None, will be dumped before the file content. - """ - if head is None: - log.info('%s', filename) - else: - log.info(head) - file = open(filename) - try: - log.info(file.read()) - finally: - file.close() diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py deleted file mode 100644 index fd3357ea78e..00000000000 --- a/Lib/distutils/command/install.py +++ /dev/null @@ -1,705 +0,0 @@ -"""distutils.command.install - -Implements the Distutils 'install' command.""" - -import sys -import os - -from distutils import log -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.sysconfig import get_config_vars -from distutils.errors import DistutilsPlatformError -from distutils.file_util import write_file -from distutils.util import convert_path, subst_vars, change_root -from distutils.util import get_platform -from distutils.errors import DistutilsOptionError - -from site import USER_BASE -from site import USER_SITE -HAS_USER_SITE = True - -WINDOWS_SCHEME = { - 'purelib': '$base/Lib/site-packages', - 'platlib': '$base/Lib/site-packages', - 'headers': '$base/Include/$dist_name', - 'scripts': '$base/Scripts', - 'data' : '$base', -} - -INSTALL_SCHEMES = { - 'unix_prefix': { - 'purelib': '$base/lib/python$py_version_short/site-packages', - 'platlib': '$platbase/lib/python$py_version_short/site-packages', - 'headers': '$base/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_local': { - 'purelib': '$base/local/lib/python$py_version_short/dist-packages', - 'platlib': '$platbase/local/lib/python$py_version_short/dist-packages', - 'headers': '$base/local/include/python$py_version_short/$dist_name', - 'scripts': '$base/local/bin', - 'data' : '$base/local', - }, - 'deb_system': { - 'purelib': '$base/lib/python3/dist-packages', - 'platlib': '$platbase/lib/python3/dist-packages', - 'headers': '$base/include/python$py_version_short/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_home': { - 'purelib': '$base/lib/python', - 'platlib': '$base/lib/python', - 'headers': '$base/include/python/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'nt': WINDOWS_SCHEME, - } - -# user site schemes -if HAS_USER_SITE: - INSTALL_SCHEMES['nt_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': '$userbase/Python$py_version_nodot/Include/$dist_name', - 'scripts': '$userbase/Python$py_version_nodot/Scripts', - 'data' : '$userbase', - } - - INSTALL_SCHEMES['unix_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': - '$userbase/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$userbase/bin', - 'data' : '$userbase', - } - -# XXX RUSTPYTHON: replace python with rustpython in all these paths -for group in INSTALL_SCHEMES.values(): - for key in group.keys(): - group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython") - -# The keys to an installation scheme; if any new types of files are to be -# installed, be sure to add an entry to every installation scheme above, -# and to SCHEME_KEYS here. -SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') - - -class install(Command): - - description = "install everything from build directory" - - user_options = [ - # Select installation scheme and set base director(y|ies) - ('prefix=', None, - "installation prefix"), - ('exec-prefix=', None, - "(Unix only) prefix for platform-specific files"), - ('home=', None, - "(Unix only) home directory to install under"), - - # Or, just set the base director(y|ies) - ('install-base=', None, - "base installation directory (instead of --prefix or --home)"), - ('install-platbase=', None, - "base installation directory for platform-specific files " + - "(instead of --exec-prefix or --home)"), - ('root=', None, - "install everything relative to this alternate root directory"), - - # Or, explicitly set the installation scheme - ('install-purelib=', None, - "installation directory for pure Python module distributions"), - ('install-platlib=', None, - "installation directory for non-pure module distributions"), - ('install-lib=', None, - "installation directory for all module distributions " + - "(overrides --install-purelib and --install-platlib)"), - - ('install-headers=', None, - "installation directory for C/C++ headers"), - ('install-scripts=', None, - "installation directory for Python scripts"), - ('install-data=', None, - "installation directory for data files"), - - # Byte-compilation options -- see install_lib.py for details, as - # these are duplicated from there (but only install_lib does - # anything with them). - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - - # Miscellaneous control options - ('force', 'f', - "force installation (overwrite any existing files)"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - - # Where to install documentation (eventually!) - #('doc-format=', None, "format of documentation to generate"), - #('install-man=', None, "directory for Unix man pages"), - #('install-html=', None, "directory for HTML documentation"), - #('install-info=', None, "directory for GNU info files"), - - ('record=', None, - "filename in which to record list of installed files"), - - ('install-layout=', None, - "installation layout to choose (known values: deb, unix)"), - ] - - boolean_options = ['compile', 'force', 'skip-build'] - - if HAS_USER_SITE: - user_options.append(('user', None, - "install in user site-package '%s'" % USER_SITE)) - boolean_options.append('user') - - negative_opt = {'no-compile' : 'compile'} - - - def initialize_options(self): - """Initializes options.""" - # High-level options: these select both an installation base - # and scheme. - self.prefix = None - self.exec_prefix = None - self.home = None - self.user = 0 - self.prefix_option = None - - # These select only the installation base; it's up to the user to - # specify the installation scheme (currently, that means supplying - # the --install-{platlib,purelib,scripts,data} options). - self.install_base = None - self.install_platbase = None - self.root = None - - # These options are the actual installation directories; if not - # supplied by the user, they are filled in using the installation - # scheme implied by prefix/exec-prefix/home and the contents of - # that installation scheme. - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers - self.install_lib = None # set to either purelib or platlib - self.install_scripts = None - self.install_data = None - self.install_userbase = USER_BASE - self.install_usersite = USER_SITE - - # enable custom installation, known values: deb - self.install_layout = None - self.multiarch = None - - self.compile = None - self.optimize = None - - # Deprecated - # These two are for putting non-packagized distributions into their - # own directory and creating a .pth file if it makes sense. - # 'extra_path' comes from the setup file; 'install_path_file' can - # be turned off if it makes no sense to install a .pth file. (But - # better to install it uselessly than to guess wrong and not - # install it when it's necessary and would be used!) Currently, - # 'install_path_file' is always true unless some outsider meddles - # with it. - self.extra_path = None - self.install_path_file = 1 - - # 'force' forces installation, even if target files are not - # out-of-date. 'skip_build' skips running the "build" command, - # handy if you know it's not necessary. 'warn_dir' (which is *not* - # a user option, it's just there so the bdist_* commands can turn - # it off) determines whether we warn about installing to a - # directory not in sys.path. - self.force = 0 - self.skip_build = 0 - self.warn_dir = 1 - - # These are only here as a conduit from the 'build' command to the - # 'install_*' commands that do the real work. ('build_base' isn't - # actually used anywhere, but it might be useful in future.) They - # are not user options, because if the user told the install - # command where the build directory is, that wouldn't affect the - # build command. - self.build_base = None - self.build_lib = None - - # Not defined yet because we don't know anything about - # documentation yet. - #self.install_man = None - #self.install_html = None - #self.install_info = None - - self.record = None - - - # -- Option finalizing methods ------------------------------------- - # (This is rather more involved than for most commands, - # because this is where the policy for installing third- - # party Python modules on various platforms given a wide - # array of user input is decided. Yes, it's quite complex!) - - def finalize_options(self): - """Finalizes options.""" - # This method (and its pliant slaves, like 'finalize_unix()', - # 'finalize_other()', and 'select_scheme()') is where the default - # installation directories for modules, extension modules, and - # anything else we care to install from a Python module - # distribution. Thus, this code makes a pretty important policy - # statement about how third-party stuff is added to a Python - # installation! Note that the actual work of installation is done - # by the relatively simple 'install_*' commands; they just take - # their orders from the installation directory options determined - # here. - - # Check for errors/inconsistencies in the options; first, stuff - # that's wrong on any platform. - - if ((self.prefix or self.exec_prefix or self.home) and - (self.install_base or self.install_platbase)): - raise DistutilsOptionError( - "must supply either prefix/exec-prefix/home or " + - "install-base/install-platbase -- not both") - - if self.home and (self.prefix or self.exec_prefix): - raise DistutilsOptionError( - "must supply either home or prefix/exec-prefix -- not both") - - if self.user and (self.prefix or self.exec_prefix or self.home or - self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with prefix, " - "exec_prefix/home, or install_(plat)base") - - # Next, stuff that's wrong (or dubious) only on certain platforms. - if os.name != "posix": - if self.exec_prefix: - self.warn("exec-prefix option ignored on this platform") - self.exec_prefix = None - - # Now the interesting logic -- so interesting that we farm it out - # to other methods. The goal of these methods is to set the final - # values for the install_{lib,scripts,data,...} options, using as - # input a heady brew of prefix, exec_prefix, home, install_base, - # install_platbase, user-supplied versions of - # install_{purelib,platlib,lib,scripts,data,...}, and the - # INSTALL_SCHEME dictionary above. Phew! - - self.dump_dirs("pre-finalize_{unix,other}") - - if os.name == 'posix': - self.finalize_unix() - else: - self.finalize_other() - - self.dump_dirs("post-finalize_{unix,other}()") - - # Expand configuration variables, tilde, etc. in self.install_base - # and self.install_platbase -- that way, we can use $base or - # $platbase in the other installation directories and not worry - # about needing recursive variable expansion (shudder). - - py_version = sys.version.split()[0] - (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix') - try: - abiflags = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - abiflags = '' - self.config_vars = {'dist_name': self.distribution.get_name(), - 'dist_version': self.distribution.get_version(), - 'dist_fullname': self.distribution.get_fullname(), - 'py_version': py_version, - 'py_version_short': '%d.%d' % sys.version_info[:2], - 'py_version_nodot': '%d%d' % sys.version_info[:2], - 'sys_prefix': prefix, - 'prefix': prefix, - 'sys_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, - 'abiflags': abiflags, - } - - if HAS_USER_SITE: - self.config_vars['userbase'] = self.install_userbase - self.config_vars['usersite'] = self.install_usersite - - self.expand_basedirs() - - self.dump_dirs("post-expand_basedirs()") - - # Now define config vars for the base directories so we can expand - # everything else. - self.config_vars['base'] = self.install_base - self.config_vars['platbase'] = self.install_platbase - - if DEBUG: - from pprint import pprint - print("config vars:") - pprint(self.config_vars) - - # Expand "~" and configuration variables in the installation - # directories. - self.expand_dirs() - - self.dump_dirs("post-expand_dirs()") - - # Create directories in the home dir: - if self.user: - self.create_home_path() - - # Pick the actual directory to install all modules to: either - # install_purelib or install_platlib, depending on whether this - # module distribution is pure or not. Of course, if the user - # already specified install_lib, use their selection. - if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure - self.install_lib = self.install_platlib - else: - self.install_lib = self.install_purelib - - - # Convert directories from Unix /-separated syntax to the local - # convention. - self.convert_paths('lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers', - 'userbase', 'usersite') - - # Deprecated - # Well, we're not actually fully completely finalized yet: we still - # have to deal with 'extra_path', which is the hack for allowing - # non-packagized module distributions (hello, Numerical Python!) to - # get their own directories. - self.handle_extra_path() - self.install_libbase = self.install_lib # needed for .pth file - self.install_lib = os.path.join(self.install_lib, self.extra_dirs) - - # If a new root directory was supplied, make all the installation - # dirs relative to it. - if self.root is not None: - self.change_roots('libbase', 'lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers') - - self.dump_dirs("after prepending root") - - # Find out the build directories, ie. where to install from. - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib')) - - # Punt on doc directories for now -- after all, we're punting on - # documentation completely! - - def dump_dirs(self, msg): - """Dumps the list of user options.""" - if not DEBUG: - return - from distutils.fancy_getopt import longopt_xlate - log.debug(msg + ":") - for opt in self.user_options: - opt_name = opt[0] - if opt_name[-1] == "=": - opt_name = opt_name[0:-1] - if opt_name in self.negative_opt: - opt_name = self.negative_opt[opt_name] - opt_name = opt_name.translate(longopt_xlate) - val = not getattr(self, opt_name) - else: - opt_name = opt_name.translate(longopt_xlate) - val = getattr(self, opt_name) - log.debug(" %s: %s", opt_name, val) - - def finalize_unix(self): - """Finalizes options for posix platforms.""" - if self.install_base is not None or self.install_platbase is not None: - if ((self.install_lib is None and - self.install_purelib is None and - self.install_platlib is None) or - self.install_headers is None or - self.install_scripts is None or - self.install_data is None): - raise DistutilsOptionError( - "install-base or install-platbase supplied, but " - "installation scheme is incomplete") - return - - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme("unix_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - self.prefix_option = self.prefix - if self.prefix is None: - if self.exec_prefix is not None: - raise DistutilsOptionError( - "must not supply exec-prefix without prefix") - - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) - - else: - if self.exec_prefix is None: - self.exec_prefix = self.prefix - - self.install_base = self.prefix - self.install_platbase = self.exec_prefix - if self.install_layout: - if self.install_layout.lower() in ['deb']: - import sysconfig - self.multiarch = sysconfig.get_config_var('MULTIARCH') - self.select_scheme("deb_system") - elif self.install_layout.lower() in ['unix']: - self.select_scheme("unix_prefix") - else: - raise DistutilsOptionError( - "unknown value for --install-layout") - elif ((self.prefix_option and - os.path.normpath(self.prefix) != '/usr/local') - or sys.base_prefix != sys.prefix - or 'PYTHONUSERBASE' in os.environ - or 'VIRTUAL_ENV' in os.environ - or 'real_prefix' in sys.__dict__): - self.select_scheme("unix_prefix") - else: - if os.path.normpath(self.prefix) == '/usr/local': - self.prefix = self.exec_prefix = '/usr' - self.install_base = self.install_platbase = '/usr' - self.select_scheme("unix_local") - - def finalize_other(self): - """Finalizes options for non-posix platforms""" - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme(os.name + "_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - if self.prefix is None: - self.prefix = os.path.normpath(sys.prefix) - - self.install_base = self.install_platbase = self.prefix - try: - self.select_scheme(os.name) - except KeyError: - raise DistutilsPlatformError( - "I don't know how to install stuff on '%s'" % os.name) - - def select_scheme(self, name): - """Sets the install directories by applying the install schemes.""" - # it's the caller's problem if they supply a bad name! - scheme = INSTALL_SCHEMES[name] - for key in SCHEME_KEYS: - attrname = 'install_' + key - if getattr(self, attrname) is None: - setattr(self, attrname, scheme[key]) - - def _expand_attrs(self, attrs): - for attr in attrs: - val = getattr(self, attr) - if val is not None: - if os.name == 'posix' or os.name == 'nt': - val = os.path.expanduser(val) - val = subst_vars(val, self.config_vars) - setattr(self, attr, val) - - def expand_basedirs(self): - """Calls `os.path.expanduser` on install_base, install_platbase and - root.""" - self._expand_attrs(['install_base', 'install_platbase', 'root']) - - def expand_dirs(self): - """Calls `os.path.expanduser` on install dirs.""" - self._expand_attrs(['install_purelib', 'install_platlib', - 'install_lib', 'install_headers', - 'install_scripts', 'install_data',]) - - def convert_paths(self, *names): - """Call `convert_path` over `names`.""" - for name in names: - attr = "install_" + name - setattr(self, attr, convert_path(getattr(self, attr))) - - def handle_extra_path(self): - """Set `path_file` and `extra_dirs` using `extra_path`.""" - if self.extra_path is None: - self.extra_path = self.distribution.extra_path - - if self.extra_path is not None: - log.warn( - "Distribution option extra_path is deprecated. " - "See issue27919 for details." - ) - if isinstance(self.extra_path, str): - self.extra_path = self.extra_path.split(',') - - if len(self.extra_path) == 1: - path_file = extra_dirs = self.extra_path[0] - elif len(self.extra_path) == 2: - path_file, extra_dirs = self.extra_path - else: - raise DistutilsOptionError( - "'extra_path' option must be a list, tuple, or " - "comma-separated string with 1 or 2 elements") - - # convert to local form in case Unix notation used (as it - # should be in setup scripts) - extra_dirs = convert_path(extra_dirs) - else: - path_file = None - extra_dirs = '' - - # XXX should we warn if path_file and not extra_dirs? (in which - # case the path file would be harmless but pointless) - self.path_file = path_file - self.extra_dirs = extra_dirs - - def change_roots(self, *names): - """Change the install directories pointed by name using root.""" - for name in names: - attr = "install_" + name - setattr(self, attr, change_root(self.root, getattr(self, attr))) - - def create_home_path(self): - """Create directories under ~.""" - if not self.user: - return - home = convert_path(os.path.expanduser("~")) - for name, path in self.config_vars.items(): - if path.startswith(home) and not os.path.isdir(path): - self.debug_print("os.makedirs('%s', 0o700)" % path) - os.makedirs(path, 0o700) - - # -- Command execution methods ------------------------------------- - - def run(self): - """Runs the command.""" - # Obviously have to build before we can install - if not self.skip_build: - self.run_command('build') - # If we built for any other platform, we can't install. - build_plat = self.distribution.get_command_obj('build').plat_name - # check warn_dir - it is a clue that the 'install' is happening - # internally, and not to sys.path, so we don't check the platform - # matches what we are running. - if self.warn_dir and build_plat != get_platform(): - raise DistutilsPlatformError("Can't install when " - "cross-compiling") - - # Run all sub-commands (at least those that need to be run) - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.path_file: - self.create_path_file() - - # write list of installed files, if requested. - if self.record: - outputs = self.get_outputs() - if self.root: # strip any package prefix - root_len = len(self.root) - for counter in range(len(outputs)): - outputs[counter] = outputs[counter][root_len:] - self.execute(write_file, - (self.record, outputs), - "writing list of installed files to '%s'" % - self.record) - - sys_path = map(os.path.normpath, sys.path) - sys_path = map(os.path.normcase, sys_path) - install_lib = os.path.normcase(os.path.normpath(self.install_lib)) - if (self.warn_dir and - not (self.path_file and self.install_path_file) and - install_lib not in sys_path): - log.debug(("modules installed to '%s', which is not in " - "Python's module search path (sys.path) -- " - "you'll have to change the search path yourself"), - self.install_lib) - - def create_path_file(self): - """Creates the .pth file""" - filename = os.path.join(self.install_libbase, - self.path_file + ".pth") - if self.install_path_file: - self.execute(write_file, - (filename, [self.extra_dirs]), - "creating %s" % filename) - else: - self.warn("path file '%s' not created" % filename) - - - # -- Reporting methods --------------------------------------------- - - def get_outputs(self): - """Assembles the outputs of all the sub-commands.""" - outputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - # Add the contents of cmd.get_outputs(), ensuring - # that outputs doesn't contain duplicate entries - for filename in cmd.get_outputs(): - if filename not in outputs: - outputs.append(filename) - - if self.path_file and self.install_path_file: - outputs.append(os.path.join(self.install_libbase, - self.path_file + ".pth")) - - return outputs - - def get_inputs(self): - """Returns the inputs of all the sub-commands""" - # XXX gee, this looks familiar ;-( - inputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - inputs.extend(cmd.get_inputs()) - - return inputs - - # -- Predicates for sub-command list ------------------------------- - - def has_lib(self): - """Returns true if the current distribution has any Python - modules to install.""" - return (self.distribution.has_pure_modules() or - self.distribution.has_ext_modules()) - - def has_headers(self): - """Returns true if the current distribution has any headers to - install.""" - return self.distribution.has_headers() - - def has_scripts(self): - """Returns true if the current distribution has any scripts to. - install.""" - return self.distribution.has_scripts() - - def has_data(self): - """Returns true if the current distribution has any data to. - install.""" - return self.distribution.has_data_files() - - # 'sub_commands': a list of commands this command might have to run to - # get its work done. See cmd.py for more info. - sub_commands = [('install_lib', has_lib), - ('install_headers', has_headers), - ('install_scripts', has_scripts), - ('install_data', has_data), - ('install_egg_info', lambda self:True), - ] diff --git a/Lib/distutils/command/install_data.py b/Lib/distutils/command/install_data.py deleted file mode 100644 index 947cd76a99e..00000000000 --- a/Lib/distutils/command/install_data.py +++ /dev/null @@ -1,79 +0,0 @@ -"""distutils.command.install_data - -Implements the Distutils 'install_data' command, for installing -platform-independent data files.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils.util import change_root, convert_path - -class install_data(Command): - - description = "install data files" - - user_options = [ - ('install-dir=', 'd', - "base directory for installing data files " - "(default: installation base dir)"), - ('root=', None, - "install everything relative to this alternate root directory"), - ('force', 'f', "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.outfiles = [] - self.root = None - self.force = 0 - self.data_files = self.distribution.data_files - self.warn_dir = 1 - - def finalize_options(self): - self.set_undefined_options('install', - ('install_data', 'install_dir'), - ('root', 'root'), - ('force', 'force'), - ) - - def run(self): - self.mkpath(self.install_dir) - for f in self.data_files: - if isinstance(f, str): - # it's a simple file, so copy it - f = convert_path(f) - if self.warn_dir: - self.warn("setup script did not provide a directory for " - "'%s' -- installing right in '%s'" % - (f, self.install_dir)) - (out, _) = self.copy_file(f, self.install_dir) - self.outfiles.append(out) - else: - # it's a tuple with path to install to and a list of files - dir = convert_path(f[0]) - if not os.path.isabs(dir): - dir = os.path.join(self.install_dir, dir) - elif self.root: - dir = change_root(self.root, dir) - self.mkpath(dir) - - if f[1] == []: - # If there are no files listed, the user must be - # trying to create an empty directory, so add the - # directory to the list of output files. - self.outfiles.append(dir) - else: - # Copy files, adding them to the list of output files. - for data in f[1]: - data = convert_path(data) - (out, _) = self.copy_file(data, dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.data_files or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py deleted file mode 100644 index 0a71b610005..00000000000 --- a/Lib/distutils/command/install_egg_info.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.command.install_egg_info - -Implements the Distutils 'install_egg_info' command, for installing -a package's PKG-INFO metadata.""" - - -from distutils.cmd import Command -from distutils import log, dir_util -import os, sys, re - -class install_egg_info(Command): - """Install an .egg-info file for the package""" - - description = "Install package's PKG-INFO metadata as an .egg-info file" - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('install-layout', None, "custom installation layout"), - ] - - def initialize_options(self): - self.install_dir = None - self.install_layout = None - self.prefix_option = None - - def finalize_options(self): - self.set_undefined_options('install_lib',('install_dir','install_dir')) - self.set_undefined_options('install',('install_layout','install_layout')) - self.set_undefined_options('install',('prefix_option','prefix_option')) - if self.install_layout: - if not self.install_layout.lower() in ['deb', 'unix']: - raise DistutilsOptionError( - "unknown value for --install-layout") - no_pyver = (self.install_layout.lower() == 'deb') - elif self.prefix_option: - no_pyver = False - else: - no_pyver = True - if no_pyver: - basename = "%s-%s.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())) - ) - else: - basename = "%s-%s-py%d.%d.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())), - *sys.version_info[:2] - ) - self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] - - def run(self): - target = self.target - if os.path.isdir(target) and not os.path.islink(target): - dir_util.remove_tree(target, dry_run=self.dry_run) - elif os.path.exists(target): - self.execute(os.unlink,(self.target,),"Removing "+target) - elif not os.path.isdir(self.install_dir): - self.execute(os.makedirs, (self.install_dir,), - "Creating "+self.install_dir) - log.info("Writing %s", target) - if not self.dry_run: - with open(target, 'w', encoding='UTF-8') as f: - self.distribution.metadata.write_pkg_file(f) - - def get_outputs(self): - return self.outputs - - -# The following routines are taken from setuptools' pkg_resources module and -# can be replaced by importing them from pkg_resources once it is included -# in the stdlib. - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') diff --git a/Lib/distutils/command/install_headers.py b/Lib/distutils/command/install_headers.py deleted file mode 100644 index 9bb0b18dc0d..00000000000 --- a/Lib/distutils/command/install_headers.py +++ /dev/null @@ -1,47 +0,0 @@ -"""distutils.command.install_headers - -Implements the Distutils 'install_headers' command, to install C/C++ header -files to the Python include directory.""" - -from distutils.core import Command - - -# XXX force is never used -class install_headers(Command): - - description = "install C/C++ header files" - - user_options = [('install-dir=', 'd', - "directory to install header files to"), - ('force', 'f', - "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.outfiles = [] - - def finalize_options(self): - self.set_undefined_options('install', - ('install_headers', 'install_dir'), - ('force', 'force')) - - - def run(self): - headers = self.distribution.headers - if not headers: - return - - self.mkpath(self.install_dir) - for header in headers: - (out, _) = self.copy_file(header, self.install_dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.distribution.headers or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py deleted file mode 100644 index eef63626ff7..00000000000 --- a/Lib/distutils/command/install_lib.py +++ /dev/null @@ -1,221 +0,0 @@ -"""distutils.command.install_lib - -Implements the Distutils 'install_lib' command -(install all Python modules).""" - -import os -import importlib.util -import sys - -from distutils.core import Command -from distutils.errors import DistutilsOptionError - - -# Extension for Python source files. -PYTHON_SOURCE_EXTENSION = ".py" - -class install_lib(Command): - - description = "install all Python modules (extensions and pure Python)" - - # The byte-compilation options are a tad confusing. Here are the - # possible scenarios: - # 1) no compilation at all (--no-compile --no-optimize) - # 2) compile .pyc only (--compile --no-optimize; default) - # 3) compile .pyc and "opt-1" .pyc (--compile --optimize) - # 4) compile "opt-1" .pyc only (--no-compile --optimize) - # 5) compile .pyc and "opt-2" .pyc (--compile --optimize-more) - # 6) compile "opt-2" .pyc only (--no-compile --optimize-more) - # - # The UI for this is two options, 'compile' and 'optimize'. - # 'compile' is strictly boolean, and only decides whether to - # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and - # decides both whether to generate .pyc files and what level of - # optimization to use. - - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'compile', 'skip-build'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - # let the 'install' command dictate our installation directory - self.install_dir = None - self.build_dir = None - self.force = 0 - self.compile = None - self.optimize = None - self.skip_build = None - self.multiarch = None # if we should rename the extensions - - def finalize_options(self): - # Get all the information we need to install pure Python modules - # from the umbrella 'install' command -- build (source) directory, - # install (target) directory, and whether to compile .py files. - self.set_undefined_options('install', - ('build_lib', 'build_dir'), - ('install_lib', 'install_dir'), - ('force', 'force'), - ('compile', 'compile'), - ('optimize', 'optimize'), - ('skip_build', 'skip_build'), - ('multiarch', 'multiarch'), - ) - - if self.compile is None: - self.compile = True - if self.optimize is None: - self.optimize = False - - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - if self.optimize not in (0, 1, 2): - raise AssertionError - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # Make sure we have built everything we need first - self.build() - - # Install everything: simply dump the entire contents of the build - # directory to the installation directory (that's the beauty of - # having a build directory!) - outfiles = self.install() - - # (Optionally) compile .py to .pyc - if outfiles is not None and self.distribution.has_pure_modules(): - self.byte_compile(outfiles) - - # -- Top-level worker functions ------------------------------------ - # (called from 'run()') - - def build(self): - if not self.skip_build: - if self.distribution.has_pure_modules(): - self.run_command('build_py') - if self.distribution.has_ext_modules(): - self.run_command('build_ext') - - def install(self): - if os.path.isdir(self.build_dir): - import distutils.dir_util - distutils.dir_util._multiarch = self.multiarch - outfiles = self.copy_tree(self.build_dir, self.install_dir) - else: - self.warn("'%s' does not exist -- no Python modules to install" % - self.build_dir) - return - return outfiles - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - - # Get the "--root" directory supplied to the "install" command, - # and use it as a prefix to strip off the purported filename - # encoded in bytecode files. This is far from complete, but it - # should at least generate usable bytecode in RPM distributions. - install_root = self.get_finalized_command('install').root - - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=install_root, - dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=install_root, - verbose=self.verbose, dry_run=self.dry_run) - - - # -- Utility methods ----------------------------------------------- - - def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir): - if not has_any: - return [] - - build_cmd = self.get_finalized_command(build_cmd) - build_files = build_cmd.get_outputs() - build_dir = getattr(build_cmd, cmd_option) - - prefix_len = len(build_dir) + len(os.sep) - outputs = [] - for file in build_files: - outputs.append(os.path.join(output_dir, file[prefix_len:])) - - return outputs - - def _bytecode_filenames(self, py_filenames): - bytecode_files = [] - for py_file in py_filenames: - # Since build_py handles package data installation, the - # list of outputs can contain more than just .py files. - # Make sure we only report bytecode for the .py files. - ext = os.path.splitext(os.path.normcase(py_file))[1] - if ext != PYTHON_SOURCE_EXTENSION: - continue - if self.compile: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization='')) - if self.optimize > 0: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization=self.optimize)) - - return bytecode_files - - - # -- External interface -------------------------------------------- - # (called by outsiders) - - def get_outputs(self): - """Return the list of files that would be installed if this command - were actually run. Not affected by the "dry-run" flag or whether - modules have actually been built yet. - """ - pure_outputs = \ - self._mutate_outputs(self.distribution.has_pure_modules(), - 'build_py', 'build_lib', - self.install_dir) - if self.compile: - bytecode_outputs = self._bytecode_filenames(pure_outputs) - else: - bytecode_outputs = [] - - ext_outputs = \ - self._mutate_outputs(self.distribution.has_ext_modules(), - 'build_ext', 'build_lib', - self.install_dir) - - return pure_outputs + bytecode_outputs + ext_outputs - - def get_inputs(self): - """Get the list of files that are input to this command, ie. the - files that get installed as they are named in the build tree. - The files in this list correspond one-to-one to the output - filenames returned by 'get_outputs()'. - """ - inputs = [] - - if self.distribution.has_pure_modules(): - build_py = self.get_finalized_command('build_py') - inputs.extend(build_py.get_outputs()) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - inputs.extend(build_ext.get_outputs()) - - return inputs diff --git a/Lib/distutils/command/install_scripts.py b/Lib/distutils/command/install_scripts.py deleted file mode 100644 index 31a1130ee54..00000000000 --- a/Lib/distutils/command/install_scripts.py +++ /dev/null @@ -1,60 +0,0 @@ -"""distutils.command.install_scripts - -Implements the Distutils 'install_scripts' command, for installing -Python scripts.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils import log -from stat import ST_MODE - - -class install_scripts(Command): - - description = "install scripts (Python or otherwise)" - - user_options = [ - ('install-dir=', 'd', "directory to install scripts to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'skip-build'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.build_dir = None - self.skip_build = None - - def finalize_options(self): - self.set_undefined_options('build', ('build_scripts', 'build_dir')) - self.set_undefined_options('install', - ('install_scripts', 'install_dir'), - ('force', 'force'), - ('skip_build', 'skip_build'), - ) - - def run(self): - if not self.skip_build: - self.run_command('build_scripts') - self.outfiles = self.copy_tree(self.build_dir, self.install_dir) - if os.name == 'posix': - # Set the executable bits (owner, group, and world) on - # all the scripts we just installed. - for file in self.get_outputs(): - if self.dry_run: - log.info("changing mode of %s", file) - else: - mode = ((os.stat(file)[ST_MODE]) | 0o555) & 0o7777 - log.info("changing mode of %s to %o", file, mode) - os.chmod(file, mode) - - def get_inputs(self): - return self.distribution.scripts or [] - - def get_outputs(self): - return self.outfiles or [] diff --git a/Lib/distutils/command/register.py b/Lib/distutils/command/register.py deleted file mode 100644 index 0fac94e9e54..00000000000 --- a/Lib/distutils/command/register.py +++ /dev/null @@ -1,304 +0,0 @@ -"""distutils.command.register - -Implements the Distutils 'register' command (register with the repository). -""" - -# created 2002/10/21, Richard Jones - -import getpass -import io -import urllib.parse, urllib.request -from warnings import warn - -from distutils.core import PyPIRCCommand -from distutils.errors import * -from distutils import log - -class register(PyPIRCCommand): - - description = ("register the distribution with the Python package index") - user_options = PyPIRCCommand.user_options + [ - ('list-classifiers', None, - 'list the valid Trove classifiers'), - ('strict', None , - 'Will stop the registering if the meta-data are not fully compliant') - ] - boolean_options = PyPIRCCommand.boolean_options + [ - 'verify', 'list-classifiers', 'strict'] - - sub_commands = [('check', lambda self: True)] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.list_classifiers = 0 - self.strict = 0 - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - # setting options for the `check` subcommand - check_options = {'strict': ('register', self.strict), - 'restructuredtext': ('register', 1)} - self.distribution.command_options['check'] = check_options - - def run(self): - self.finalize_options() - self._set_config() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.dry_run: - self.verify_metadata() - elif self.list_classifiers: - self.classifiers() - else: - self.send_metadata() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.register.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.strict = self.strict - check.restructuredtext = 1 - check.run() - - def _set_config(self): - ''' Reads the configuration file and set attributes. - ''' - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - self.has_config = True - else: - if self.repository not in ('pypi', self.DEFAULT_REPOSITORY): - raise ValueError('%s not found in .pypirc' % self.repository) - if self.repository == 'pypi': - self.repository = self.DEFAULT_REPOSITORY - self.has_config = False - - def classifiers(self): - ''' Fetch the list of classifiers from the server. - ''' - url = self.repository+'?:action=list_classifiers' - response = urllib.request.urlopen(url) - log.info(self._read_pypi_response(response)) - - def verify_metadata(self): - ''' Send the metadata to the package index server to be checked. - ''' - # send the info to the server and report the result - (code, result) = self.post_to_server(self.build_post_data('verify')) - log.info('Server response (%s): %s', code, result) - - def send_metadata(self): - ''' Send the metadata to the package index server. - - Well, do the following: - 1. figure who the user is, and then - 2. send the data as a Basic auth'ed POST. - - First we try to read the username/password from $HOME/.pypirc, - which is a ConfigParser-formatted file with a section - [distutils] containing username and password entries (both - in clear text). Eg: - - [distutils] - index-servers = - pypi - - [pypi] - username: fred - password: sekrit - - Otherwise, to figure who the user is, we offer the user three - choices: - - 1. use existing login, - 2. register as a new user, or - 3. set the password to a random string and email the user. - - ''' - # see if we can short-cut and get the username/password from the - # config - if self.has_config: - choice = '1' - username = self.username - password = self.password - else: - choice = 'x' - username = password = '' - - # get the user's login info - choices = '1 2 3 4'.split() - while choice not in choices: - self.announce('''\ -We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit -Your selection [default 1]: ''', log.INFO) - choice = input() - if not choice: - choice = '1' - elif choice not in choices: - print('Please choose one of the four options!') - - if choice == '1': - # get the username and password - while not username: - username = input('Username: ') - while not password: - password = getpass.getpass('Password: ') - - # set up the authentication - auth = urllib.request.HTTPPasswordMgr() - host = urllib.parse.urlparse(self.repository)[1] - auth.add_password(self.realm, host, username, password) - # send the info to the server and report the result - code, result = self.post_to_server(self.build_post_data('submit'), - auth) - self.announce('Server response (%s): %s' % (code, result), - log.INFO) - - # possibly save the login - if code == 200: - if self.has_config: - # sharing the password in the distribution instance - # so the upload command can reuse it - self.distribution.password = password - else: - self.announce(('I can store your PyPI login so future ' - 'submissions will be faster.'), log.INFO) - self.announce('(the login will be stored in %s)' % \ - self._get_rc_file(), log.INFO) - choice = 'X' - while choice.lower() not in 'yn': - choice = input('Save your login (y/N)?') - if not choice: - choice = 'n' - if choice.lower() == 'y': - self._store_pypirc(username, password) - - elif choice == '2': - data = {':action': 'user'} - data['name'] = data['password'] = data['email'] = '' - data['confirm'] = None - while not data['name']: - data['name'] = input('Username: ') - while data['password'] != data['confirm']: - while not data['password']: - data['password'] = getpass.getpass('Password: ') - while not data['confirm']: - data['confirm'] = getpass.getpass(' Confirm: ') - if data['password'] != data['confirm']: - data['password'] = '' - data['confirm'] = None - print("Password and confirm don't match!") - while not data['email']: - data['email'] = input(' EMail: ') - code, result = self.post_to_server(data) - if code != 200: - log.info('Server response (%s): %s', code, result) - else: - log.info('You will receive an email shortly.') - log.info(('Follow the instructions in it to ' - 'complete registration.')) - elif choice == '3': - data = {':action': 'password_reset'} - data['email'] = '' - while not data['email']: - data['email'] = input('Your email address: ') - code, result = self.post_to_server(data) - log.info('Server response (%s): %s', code, result) - - def build_post_data(self, action): - # figure the data to send - the metadata plus some additional - # information used by the package server - meta = self.distribution.metadata - data = { - ':action': action, - 'metadata_version' : '1.0', - 'name': meta.get_name(), - 'version': meta.get_version(), - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - if data['provides'] or data['requires'] or data['obsoletes']: - data['metadata_version'] = '1.1' - return data - - def post_to_server(self, data, auth=None): - ''' Post a query to the server, and return a string response. - ''' - if 'name' in data: - self.announce('Registering %s to %s' % (data['name'], - self.repository), - log.INFO) - # Build up the MIME payload for the urllib2 POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' - body = io.StringIO() - for key, value in data.items(): - # handle multiple entries for the same name - if type(value) not in (type([]), type( () )): - value = [value] - for value in value: - value = str(value) - body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) - body.write("\n\n") - body.write(value) - if value and value[-1] == '\r': - body.write('\n') # write an extra newline (lurve Macs) - body.write(end_boundary) - body.write("\n") - body = body.getvalue().encode("utf-8") - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary, - 'Content-length': str(len(body)) - } - req = urllib.request.Request(self.repository, body, headers) - - # handle HTTP and include the Basic Auth handler - opener = urllib.request.build_opener( - urllib.request.HTTPBasicAuthHandler(password_mgr=auth) - ) - data = '' - try: - result = opener.open(req) - except urllib.error.HTTPError as e: - if self.show_response: - data = e.fp.read() - result = e.code, e.msg - except urllib.error.URLError as e: - result = 500, str(e) - else: - if self.show_response: - data = self._read_pypi_response(result) - result = 200, 'OK' - if self.show_response: - msg = '\n'.join(('-' * 75, data, '-' * 75)) - self.announce(msg, log.INFO) - return result diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py deleted file mode 100644 index 4fd1d4715de..00000000000 --- a/Lib/distutils/command/sdist.py +++ /dev/null @@ -1,456 +0,0 @@ -"""distutils.command.sdist - -Implements the Distutils 'sdist' command (create a source distribution).""" - -import os -import sys -from types import * -from glob import glob -from warnings import warn - -from distutils.core import Command -from distutils import dir_util, dep_util, file_util, archive_util -from distutils.text_file import TextFile -from distutils.errors import * -from distutils.filelist import FileList -from distutils import log -from distutils.util import convert_path - -def show_formats(): - """Print all possible values for the 'formats' option (used by - the "--help-formats" command-line option). - """ - from distutils.fancy_getopt import FancyGetopt - from distutils.archive_util import ARCHIVE_FORMATS - formats = [] - for format in ARCHIVE_FORMATS.keys(): - formats.append(("formats=" + format, None, - ARCHIVE_FORMATS[format][2])) - formats.sort() - FancyGetopt(formats).print_help( - "List of available source distribution formats:") - -class sdist(Command): - - description = "create a source distribution (tarball, zip file, etc.)" - - def checking_metadata(self): - """Callable used for the check sub-command. - - Placed here so user_options can view it""" - return self.metadata_check - - user_options = [ - ('template=', 't', - "name of manifest template file [default: MANIFEST.in]"), - ('manifest=', 'm', - "name of manifest file [default: MANIFEST]"), - ('use-defaults', None, - "include the default file set in the manifest " - "[default; disable with --no-defaults]"), - ('no-defaults', None, - "don't include the default file set"), - ('prune', None, - "specifically exclude files/directories that should not be " - "distributed (build tree, RCS/CVS dirs, etc.) " - "[default; disable with --no-prune]"), - ('no-prune', None, - "don't automatically exclude anything"), - ('manifest-only', 'o', - "just regenerate the manifest and then stop " - "(implies --force-manifest)"), - ('force-manifest', 'f', - "forcibly regenerate the manifest and carry on as usual. " - "Deprecated: now the manifest is always regenerated."), - ('formats=', None, - "formats for source distribution (comma-separated list)"), - ('keep-temp', 'k', - "keep the distribution tree around after creating " + - "archive file(s)"), - ('dist-dir=', 'd', - "directory to put the source distribution archive(s) in " - "[default: dist]"), - ('metadata-check', None, - "Ensure that all required elements of meta-data " - "are supplied. Warn if any missing. [default]"), - ('owner=', 'u', - "Owner name used when creating a tar file [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file [default: current group]"), - ] - - boolean_options = ['use-defaults', 'prune', - 'manifest-only', 'force-manifest', - 'keep-temp', 'metadata-check'] - - help_options = [ - ('help-formats', None, - "list available distribution formats", show_formats), - ] - - negative_opt = {'no-defaults': 'use-defaults', - 'no-prune': 'prune' } - - sub_commands = [('check', checking_metadata)] - - def initialize_options(self): - # 'template' and 'manifest' are, respectively, the names of - # the manifest template and manifest file. - self.template = None - self.manifest = None - - # 'use_defaults': if true, we will include the default file set - # in the manifest - self.use_defaults = 1 - self.prune = 1 - - self.manifest_only = 0 - self.force_manifest = 0 - - self.formats = ['gztar'] - self.keep_temp = 0 - self.dist_dir = None - - self.archive_files = None - self.metadata_check = 1 - self.owner = None - self.group = None - - def finalize_options(self): - if self.manifest is None: - self.manifest = "MANIFEST" - if self.template is None: - self.template = "MANIFEST.in" - - self.ensure_string_list('formats') - - bad_format = archive_util.check_archive_formats(self.formats) - if bad_format: - raise DistutilsOptionError( - "unknown archive format '%s'" % bad_format) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # 'filelist' contains the list of files that will make up the - # manifest - self.filelist = FileList() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - # Do whatever it takes to get the list of files to process - # (process the manifest template, read an existing manifest, - # whatever). File list is accumulated in 'self.filelist'. - self.get_file_list() - - # If user just wanted us to regenerate the manifest, stop now. - if self.manifest_only: - return - - # Otherwise, go ahead and create the source distribution tarball, - # or zipfile, or whatever. - self.make_distribution() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.sdist.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.run() - - def get_file_list(self): - """Figure out the list of files to include in the source - distribution, and put it in 'self.filelist'. This might involve - reading the manifest template (and writing the manifest), or just - reading the manifest, or just using the default file set -- it all - depends on the user's options. - """ - # new behavior when using a template: - # the file list is recalculated every time because - # even if MANIFEST.in or setup.py are not changed - # the user might have added some files in the tree that - # need to be included. - # - # This makes --force the default and only behavior with templates. - template_exists = os.path.isfile(self.template) - if not template_exists and self._manifest_is_not_generated(): - self.read_manifest() - self.filelist.sort() - self.filelist.remove_duplicates() - return - - if not template_exists: - self.warn(("manifest template '%s' does not exist " + - "(using default file list)") % - self.template) - self.filelist.findall() - - if self.use_defaults: - self.add_defaults() - - if template_exists: - self.read_template() - - if self.prune: - self.prune_file_list() - - self.filelist.sort() - self.filelist.remove_duplicates() - self.write_manifest() - - def add_defaults(self): - """Add all the default files to self.filelist: - - README or README.txt - - setup.py - - test/test*.py - - all pure Python modules mentioned in setup script - - all files pointed by package_data (build_py) - - all files defined in data_files. - - all files defined as scripts. - - all C sources listed as part of extensions or C libraries - in the setup script (doesn't catch C headers!) - Warns if (README or README.txt) or setup.py are missing; everything - else is optional. - """ - standards = [('README', 'README.txt'), self.distribution.script_name] - for fn in standards: - if isinstance(fn, tuple): - alts = fn - got_it = False - for fn in alts: - if os.path.exists(fn): - got_it = True - self.filelist.append(fn) - break - - if not got_it: - self.warn("standard file not found: should have one of " + - ', '.join(alts)) - else: - if os.path.exists(fn): - self.filelist.append(fn) - else: - self.warn("standard file '%s' not found" % fn) - - optional = ['test/test*.py', 'setup.cfg'] - for pattern in optional: - files = filter(os.path.isfile, glob(pattern)) - self.filelist.extend(files) - - # build_py is used to get: - # - python modules - # - files defined in package_data - build_py = self.get_finalized_command('build_py') - - # getting python files - if self.distribution.has_pure_modules(): - self.filelist.extend(build_py.get_source_files()) - - # getting package_data files - # (computed in build_py.data_files by build_py.finalize_options) - for pkg, src_dir, build_dir, filenames in build_py.data_files: - for filename in filenames: - self.filelist.append(os.path.join(src_dir, filename)) - - # getting distribution.data_files - if self.distribution.has_data_files(): - for item in self.distribution.data_files: - if isinstance(item, str): # plain file - item = convert_path(item) - if os.path.isfile(item): - self.filelist.append(item) - else: # a (dirname, filenames) tuple - dirname, filenames = item - for f in filenames: - f = convert_path(f) - if os.path.isfile(f): - self.filelist.append(f) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - self.filelist.extend(build_ext.get_source_files()) - - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.filelist.extend(build_clib.get_source_files()) - - if self.distribution.has_scripts(): - build_scripts = self.get_finalized_command('build_scripts') - self.filelist.extend(build_scripts.get_source_files()) - - def read_template(self): - """Read and parse manifest template file named by self.template. - - (usually "MANIFEST.in") The parsing and processing is done by - 'self.filelist', which updates itself accordingly. - """ - log.info("reading manifest template '%s'", self.template) - template = TextFile(self.template, strip_comments=1, skip_blanks=1, - join_lines=1, lstrip_ws=1, rstrip_ws=1, - collapse_join=1) - - try: - while True: - line = template.readline() - if line is None: # end of file - break - - try: - self.filelist.process_template_line(line) - # the call above can raise a DistutilsTemplateError for - # malformed lines, or a ValueError from the lower-level - # convert_path function - except (DistutilsTemplateError, ValueError) as msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) - finally: - template.close() - - def prune_file_list(self): - """Prune off branches that might slip into the file list as created - by 'read_template()', but really don't belong there: - * the build tree (typically "build") - * the release tree itself (only an issue if we ran "sdist" - previously with --keep-temp, or it aborted) - * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories - """ - build = self.get_finalized_command('build') - base_dir = self.distribution.get_fullname() - - self.filelist.exclude_pattern(None, prefix=build.build_base) - self.filelist.exclude_pattern(None, prefix=base_dir) - - if sys.platform == 'win32': - seps = r'/|\\' - else: - seps = '/' - - vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', - '_darcs'] - vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) - self.filelist.exclude_pattern(vcs_ptrn, is_regex=1) - - def write_manifest(self): - """Write the file list in 'self.filelist' (presumably as filled in - by 'add_defaults()' and 'read_template()') to the manifest file - named by 'self.manifest'. - """ - if self._manifest_is_not_generated(): - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return - - content = self.filelist.files[:] - content.insert(0, '# file GENERATED by distutils, do NOT edit') - self.execute(file_util.write_file, (self.manifest, content), - "writing manifest file '%s'" % self.manifest) - - def _manifest_is_not_generated(self): - # check for special comment used in 3.1.3 and higher - if not os.path.isfile(self.manifest): - return False - - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - return first_line != '# file GENERATED by distutils, do NOT edit\n' - - def read_manifest(self): - """Read the manifest file (named by 'self.manifest') and use it to - fill in 'self.filelist', the list of files to include in the source - distribution. - """ - log.info("reading manifest file '%s'", self.manifest) - manifest = open(self.manifest) - for line in manifest: - # ignore comments and blank lines - line = line.strip() - if line.startswith('#') or not line: - continue - self.filelist.append(line) - manifest.close() - - def make_release_tree(self, base_dir, files): - """Create the directory tree that will become the source - distribution archive. All directories implied by the filenames in - 'files' are created under 'base_dir', and then we hard link or copy - (if hard linking is unavailable) those files into place. - Essentially, this duplicates the developer's source tree, but in a - directory named after the distribution, containing only the files - to be distributed. - """ - # Create all the directories under 'base_dir' necessary to - # put 'files' there; the 'mkpath()' is just so we don't die - # if the manifest happens to be empty. - self.mkpath(base_dir) - dir_util.create_tree(base_dir, files, dry_run=self.dry_run) - - # And walk over the list of files, either making a hard link (if - # os.link exists) to each one that doesn't already exist in its - # corresponding location under 'base_dir', or copying each file - # that's out-of-date in 'base_dir'. (Usually, all files will be - # out-of-date, because by default we blow away 'base_dir' when - # we're done making the distribution archives.) - - if hasattr(os, 'link'): # can make hard links on this system - link = 'hard' - msg = "making hard links in %s..." % base_dir - else: # nope, have to copy - link = None - msg = "copying files to %s..." % base_dir - - if not files: - log.warn("no files to distribute -- empty manifest?") - else: - log.info(msg) - for file in files: - if not os.path.isfile(file): - log.warn("'%s' not a regular file -- skipping", file) - else: - dest = os.path.join(base_dir, file) - self.copy_file(file, dest, link=link) - - self.distribution.metadata.write_pkg_info(base_dir) - - def make_distribution(self): - """Create the source distribution(s). First, we create the release - tree with 'make_release_tree()'; then, we create all required - archive files (according to 'self.formats') from the release tree. - Finally, we clean up by blowing away the release tree (unless - 'self.keep_temp' is true). The list of archive files created is - stored so it can be retrieved later by 'get_archive_files()'. - """ - # Don't warn about missing meta-data here -- should be (and is!) - # done elsewhere. - base_dir = self.distribution.get_fullname() - base_name = os.path.join(self.dist_dir, base_dir) - - self.make_release_tree(base_dir, self.filelist.files) - archive_files = [] # remember names of files we create - # tar archive must be created last to avoid overwrite and remove - if 'tar' in self.formats: - self.formats.append(self.formats.pop(self.formats.index('tar'))) - - for fmt in self.formats: - file = self.make_archive(base_name, fmt, base_dir=base_dir, - owner=self.owner, group=self.group) - archive_files.append(file) - self.distribution.dist_files.append(('sdist', '', file)) - - self.archive_files = archive_files - - if not self.keep_temp: - dir_util.remove_tree(base_dir, dry_run=self.dry_run) - - def get_archive_files(self): - """Return the list of archive files created when the command - was run, or None if the command hasn't run yet. - """ - return self.archive_files diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py deleted file mode 100644 index 32dda359bad..00000000000 --- a/Lib/distutils/command/upload.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -distutils.command.upload - -Implements the Distutils 'upload' subcommand (upload package to a package -index). -""" - -import os -import io -import platform -import hashlib -from base64 import standard_b64encode -from urllib.request import urlopen, Request, HTTPError -from urllib.parse import urlparse -from distutils.errors import DistutilsError, DistutilsOptionError -from distutils.core import PyPIRCCommand -from distutils.spawn import spawn -from distutils import log - -class upload(PyPIRCCommand): - - description = "upload binary package to PyPI" - - user_options = PyPIRCCommand.user_options + [ - ('sign', 's', - 'sign files to upload using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), - ] - - boolean_options = PyPIRCCommand.boolean_options + ['sign'] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.username = '' - self.password = '' - self.show_response = 0 - self.sign = False - self.identity = None - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - if self.identity and not self.sign: - raise DistutilsOptionError( - "Must use --sign for --identity to have meaning" - ) - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - - # getting the password from the distribution - # if previously set by the register command - if not self.password and self.distribution.password: - self.password = self.distribution.password - - def run(self): - if not self.distribution.dist_files: - msg = ("Must create and upload files in one command " - "(e.g. setup.py sdist upload)") - raise DistutilsOptionError(msg) - for command, pyversion, filename in self.distribution.dist_files: - self.upload_file(command, pyversion, filename) - - def upload_file(self, command, pyversion, filename): - # Makes sure the repository URL is compliant - schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) - if params or query or fragments: - raise AssertionError("Incompatible url %s" % self.repository) - - if schema not in ('http', 'https'): - raise AssertionError("unsupported schema " + schema) - - # Sign if requested - if self.sign: - gpg_args = ["gpg", "--detach-sign", "-a", filename] - if self.identity: - gpg_args[2:2] = ["--local-user", self.identity] - spawn(gpg_args, - dry_run=self.dry_run) - - # Fill in the data - send all the meta-data in case we need to - # register a new release - f = open(filename,'rb') - try: - content = f.read() - finally: - f.close() - meta = self.distribution.metadata - data = { - # action - ':action': 'file_upload', - 'protocol_version': '1', - - # identify release - 'name': meta.get_name(), - 'version': meta.get_version(), - - # file content - 'content': (os.path.basename(filename),content), - 'filetype': command, - 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), - - # additional meta-data - 'metadata_version': '1.0', - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - comment = '' - if command == 'bdist_rpm': - dist, version, id = platform.dist() - if dist: - comment = 'built for %s %s' % (dist, version) - elif command == 'bdist_dumb': - comment = 'built for %s' % platform.platform(terse=1) - data['comment'] = comment - - if self.sign: - data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc", "rb").read()) - - # set up the authentication - user_pass = (self.username + ":" + self.password).encode('ascii') - # The exact encoding of the authentication string is debated. - # Anyway PyPI only accepts ascii for both username or password. - auth = "Basic " + standard_b64encode(user_pass).decode('ascii') - - # Build up the MIME payload for the POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = b'\r\n--' + boundary.encode('ascii') - end_boundary = sep_boundary + b'--\r\n' - body = io.BytesIO() - for key, value in data.items(): - title = '\r\nContent-Disposition: form-data; name="%s"' % key - # handle multiple entries for the same name - if not isinstance(value, list): - value = [value] - for value in value: - if type(value) is tuple: - title += '; filename="%s"' % value[0] - value = value[1] - else: - value = str(value).encode('utf-8') - body.write(sep_boundary) - body.write(title.encode('utf-8')) - body.write(b"\r\n\r\n") - body.write(value) - body.write(end_boundary) - body = body.getvalue() - - msg = "Submitting %s to %s" % (filename, self.repository) - self.announce(msg, log.INFO) - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s' % boundary, - 'Content-length': str(len(body)), - 'Authorization': auth, - } - - request = Request(self.repository, data=body, - headers=headers) - # send the data - try: - result = urlopen(request) - status = result.getcode() - reason = result.msg - except HTTPError as e: - status = e.code - reason = e.msg - except OSError as e: - self.announce(str(e), log.ERROR) - raise - - if status == 200: - self.announce('Server response (%s): %s' % (status, reason), - log.INFO) - if self.show_response: - text = self._read_pypi_response(result) - msg = '\n'.join(('-' * 75, text, '-' * 75)) - self.announce(msg, log.INFO) - else: - msg = 'Upload failed (%s): %s' % (status, reason) - self.announce(msg, log.ERROR) - raise DistutilsError(msg) diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py deleted file mode 100644 index bf8d8dd2f5a..00000000000 --- a/Lib/distutils/config.py +++ /dev/null @@ -1,131 +0,0 @@ -"""distutils.pypirc - -Provides the PyPIRCCommand class, the base class for the command classes -that uses .pypirc in the distutils.command package. -""" -import os -from configparser import RawConfigParser - -from distutils.cmd import Command - -DEFAULT_PYPIRC = """\ -[distutils] -index-servers = - pypi - -[pypi] -username:%s -password:%s -""" - -class PyPIRCCommand(Command): - """Base command that knows how to handle the .pypirc file - """ - DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/' - DEFAULT_REALM = 'pypi' - repository = None - realm = None - - user_options = [ - ('repository=', 'r', - "url of repository [default: %s]" % \ - DEFAULT_REPOSITORY), - ('show-response', None, - 'display full response text from server')] - - boolean_options = ['show-response'] - - def _get_rc_file(self): - """Returns rc file path.""" - return os.path.join(os.path.expanduser('~'), '.pypirc') - - def _store_pypirc(self, username, password): - """Creates a default .pypirc file.""" - rc = self._get_rc_file() - with os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: - f.write(DEFAULT_PYPIRC % (username, password)) - - def _read_pypirc(self): - """Reads the .pypirc file.""" - rc = self._get_rc_file() - if os.path.exists(rc): - self.announce('Using PyPI login from %s' % rc) - repository = self.repository or self.DEFAULT_REPOSITORY - realm = self.realm or self.DEFAULT_REALM - - config = RawConfigParser() - config.read(rc) - sections = config.sections() - if 'distutils' in sections: - # let's get the list of servers - index_servers = config.get('distutils', 'index-servers') - _servers = [server.strip() for server in - index_servers.split('\n') - if server.strip() != ''] - if _servers == []: - # nothing set, let's try to get the default pypi - if 'pypi' in sections: - _servers = ['pypi'] - else: - # the file is not properly defined, returning - # an empty dict - return {} - for server in _servers: - current = {'server': server} - current['username'] = config.get(server, 'username') - - # optional params - for key, default in (('repository', - self.DEFAULT_REPOSITORY), - ('realm', self.DEFAULT_REALM), - ('password', None)): - if config.has_option(server, key): - current[key] = config.get(server, key) - else: - current[key] = default - - # work around people having "repository" for the "pypi" - # section of their config set to the HTTP (rather than - # HTTPS) URL - if (server == 'pypi' and - repository in (self.DEFAULT_REPOSITORY, 'pypi')): - current['repository'] = self.DEFAULT_REPOSITORY - return current - - if (current['server'] == repository or - current['repository'] == repository): - return current - elif 'server-login' in sections: - # old format - server = 'server-login' - if config.has_option(server, 'repository'): - repository = config.get(server, 'repository') - else: - repository = self.DEFAULT_REPOSITORY - return {'username': config.get(server, 'username'), - 'password': config.get(server, 'password'), - 'repository': repository, - 'server': server, - 'realm': self.DEFAULT_REALM} - - return {} - - def _read_pypi_response(self, response): - """Read and decode a PyPI HTTP response.""" - import cgi - content_type = response.getheader('content-type', 'text/plain') - encoding = cgi.parse_header(content_type)[1].get('charset', 'ascii') - return response.read().decode(encoding) - - def initialize_options(self): - """Initialize options.""" - self.repository = None - self.realm = None - self.show_response = 0 - - def finalize_options(self): - """Finalizes options.""" - if self.repository is None: - self.repository = self.DEFAULT_REPOSITORY - if self.realm is None: - self.realm = self.DEFAULT_REALM diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py deleted file mode 100644 index d603d4a45a7..00000000000 --- a/Lib/distutils/core.py +++ /dev/null @@ -1,234 +0,0 @@ -"""distutils.core - -The only module that needs to be imported to use the Distutils; provides -the 'setup' function (which is to be called from the setup script). Also -indirectly provides the Distribution and Command classes, although they are -really defined in distutils.dist and distutils.cmd. -""" - -import os -import sys - -from distutils.debug import DEBUG -from distutils.errors import * - -# Mainly import these so setup scripts can "from distutils.core import" them. -from distutils.dist import Distribution -from distutils.cmd import Command -from distutils.config import PyPIRCCommand -from distutils.extension import Extension - -# This is a barebones help message generated displayed when the user -# runs the setup script with no arguments at all. More useful help -# is generated with various --help options: global help, list commands, -# and per-command help. -USAGE = """\ -usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] - or: %(script)s --help [cmd1 cmd2 ...] - or: %(script)s --help-commands - or: %(script)s cmd --help -""" - -def gen_usage (script_name): - script = os.path.basename(script_name) - return USAGE % vars() - - -# Some mild magic to control the behaviour of 'setup()' from 'run_setup()'. -_setup_stop_after = None -_setup_distribution = None - -# Legal keyword arguments for the setup() function -setup_keywords = ('distclass', 'script_name', 'script_args', 'options', - 'name', 'version', 'author', 'author_email', - 'maintainer', 'maintainer_email', 'url', 'license', - 'description', 'long_description', 'keywords', - 'platforms', 'classifiers', 'download_url', - 'requires', 'provides', 'obsoletes', - ) - -# Legal keyword arguments for the Extension constructor -extension_keywords = ('name', 'sources', 'include_dirs', - 'define_macros', 'undef_macros', - 'library_dirs', 'libraries', 'runtime_library_dirs', - 'extra_objects', 'extra_compile_args', 'extra_link_args', - 'swig_opts', 'export_symbols', 'depends', 'language') - -def setup (**attrs): - """The gateway to the Distutils: do everything your setup script needs - to do, in a highly flexible and user-driven way. Briefly: create a - Distribution instance; find and parse config files; parse the command - line; run each Distutils command found there, customized by the options - supplied to 'setup()' (as keyword arguments), in config files, and on - the command line. - - The Distribution instance might be an instance of a class supplied via - the 'distclass' keyword argument to 'setup'; if no such class is - supplied, then the Distribution class (in dist.py) is instantiated. - All other arguments to 'setup' (except for 'cmdclass') are used to set - attributes of the Distribution instance. - - The 'cmdclass' argument, if supplied, is a dictionary mapping command - names to command classes. Each command encountered on the command line - will be turned into a command class, which is in turn instantiated; any - class found in 'cmdclass' is used in place of the default, which is - (for command 'foo_bar') class 'foo_bar' in module - 'distutils.command.foo_bar'. The command class must provide a - 'user_options' attribute which is a list of option specifiers for - 'distutils.fancy_getopt'. Any command-line options between the current - and the next command are used to set attributes of the current command - object. - - When the entire command-line has been successfully parsed, calls the - 'run()' method on each command object in turn. This method will be - driven entirely by the Distribution object (which each command object - has a reference to, thanks to its constructor), and the - command-specific options that became attributes of each command - object. - """ - - global _setup_stop_after, _setup_distribution - - # Determine the distribution class -- either caller-supplied or - # our Distribution (see below). - klass = attrs.get('distclass') - if klass: - del attrs['distclass'] - else: - klass = Distribution - - if 'script_name' not in attrs: - attrs['script_name'] = os.path.basename(sys.argv[0]) - if 'script_args' not in attrs: - attrs['script_args'] = sys.argv[1:] - - # Create the Distribution instance, using the remaining arguments - # (ie. everything except distclass) to initialize it - try: - _setup_distribution = dist = klass(attrs) - except DistutilsSetupError as msg: - if 'name' not in attrs: - raise SystemExit("error in setup command: %s" % msg) - else: - raise SystemExit("error in %s setup command: %s" % \ - (attrs['name'], msg)) - - if _setup_stop_after == "init": - return dist - - # Find and parse the config file(s): they will override options from - # the setup script, but be overridden by the command line. - dist.parse_config_files() - - if DEBUG: - print("options (after parsing config files):") - dist.dump_option_dicts() - - if _setup_stop_after == "config": - return dist - - # Parse the command line and override config files; any - # command-line errors are the end user's fault, so turn them into - # SystemExit to suppress tracebacks. - try: - ok = dist.parse_command_line() - except DistutilsArgError as msg: - raise SystemExit(gen_usage(dist.script_name) + "\nerror: %s" % msg) - - if DEBUG: - print("options (after parsing command line):") - dist.dump_option_dicts() - - if _setup_stop_after == "commandline": - return dist - - # And finally, run all the commands found on the command line. - if ok: - try: - dist.run_commands() - except KeyboardInterrupt: - raise SystemExit("interrupted") - except OSError as exc: - if DEBUG: - sys.stderr.write("error: %s\n" % (exc,)) - raise - else: - raise SystemExit("error: %s" % (exc,)) - - except (DistutilsError, - CCompilerError) as msg: - if DEBUG: - raise - else: - raise SystemExit("error: " + str(msg)) - - return dist - -# setup () - - -def run_setup (script_name, script_args=None, stop_after="run"): - """Run a setup script in a somewhat controlled environment, and - return the Distribution instance that drives things. This is useful - if you need to find out the distribution meta-data (passed as - keyword args from 'script' to 'setup()', or the contents of the - config files or command-line. - - 'script_name' is a file that will be read and run with 'exec()'; - 'sys.argv[0]' will be replaced with 'script' for the duration of the - call. 'script_args' is a list of strings; if supplied, - 'sys.argv[1:]' will be replaced by 'script_args' for the duration of - the call. - - 'stop_after' tells 'setup()' when to stop processing; possible - values: - init - stop after the Distribution instance has been created and - populated with the keyword arguments to 'setup()' - config - stop after config files have been parsed (and their data - stored in the Distribution instance) - commandline - stop after the command-line ('sys.argv[1:]' or 'script_args') - have been parsed (and the data stored in the Distribution) - run [default] - stop after all commands have been run (the same as if 'setup()' - had been called in the usual way - - Returns the Distribution instance, which provides all information - used to drive the Distutils. - """ - if stop_after not in ('init', 'config', 'commandline', 'run'): - raise ValueError("invalid value for 'stop_after': %r" % (stop_after,)) - - global _setup_stop_after, _setup_distribution - _setup_stop_after = stop_after - - save_argv = sys.argv.copy() - g = {'__file__': script_name} - try: - try: - sys.argv[0] = script_name - if script_args is not None: - sys.argv[1:] = script_args - with open(script_name, 'rb') as f: - exec(f.read(), g) - finally: - sys.argv = save_argv - _setup_stop_after = None - except SystemExit: - # Hmm, should we do something if exiting with a non-zero code - # (ie. error)? - pass - - if _setup_distribution is None: - raise RuntimeError(("'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?") % \ - script_name) - - # I wonder if the setup script's namespace -- g and l -- would be of - # any interest to callers? - #print "_setup_distribution:", _setup_distribution - return _setup_distribution - -# run_setup () diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py deleted file mode 100644 index 1c369903477..00000000000 --- a/Lib/distutils/cygwinccompiler.py +++ /dev/null @@ -1,405 +0,0 @@ -"""distutils.cygwinccompiler - -Provides the CygwinCCompiler class, a subclass of UnixCCompiler that -handles the Cygwin port of the GNU C compiler to Windows. It also contains -the Mingw32CCompiler class which handles the mingw32 port of GCC (same as -cygwin in no-cygwin mode). -""" - -# problems: -# -# * if you use a msvc compiled python version (1.5.2) -# 1. you have to insert a __GNUC__ section in its config.h -# 2. you have to generate an import library for its dll -# - create a def-file for python??.dll -# - create an import library using -# dlltool --dllname python15.dll --def python15.def \ -# --output-lib libpython15.a -# -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# -# * We put export_symbols in a def-file, and don't use -# --export-all-symbols because it doesn't worked reliable in some -# tested configurations. And because other windows compilers also -# need their symbols specified this no serious problem. -# -# tested configurations: -# -# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works -# (after patching python's config.h and for C++ some other include files) -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works -# (ld doesn't support -shared, so we use dllwrap) -# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now -# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 -# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html -# - using gcc -mdll instead dllwrap doesn't work without -static because -# it tries to link against dlls instead their import libraries. (If -# it finds the dll first.) -# By specifying -static we force ld to link against the import libraries, -# this is windows standard and there are normally not the necessary symbols -# in the dlls. -# *** only the version of June 2000 shows these problems -# * cygwin gcc 3.2/ld 2.13.90 works -# (ld supports -shared) -# * mingw gcc 3.2/ld 2.13 works -# (ld supports -shared) - -import os -import sys -import copy -from subprocess import Popen, PIPE, check_output -import re - -from distutils.ccompiler import gen_preprocess_options, gen_lib_options -from distutils.unixccompiler import UnixCCompiler -from distutils.file_util import write_file -from distutils.errors import (DistutilsExecError, CCompilerError, - CompileError, UnknownFileError) -from distutils import log -from distutils.version import LooseVersion -from distutils.spawn import find_executable - -def get_msvcr(): - """Include the appropriate MSVC runtime library if Python was built - with MSVC 7.0 or later. - """ - msc_pos = sys.version.find('MSC v.') - if msc_pos != -1: - msc_ver = sys.version[msc_pos+6:msc_pos+10] - if msc_ver == '1300': - # MSVC 7.0 - return ['msvcr70'] - elif msc_ver == '1310': - # MSVC 7.1 - return ['msvcr71'] - elif msc_ver == '1400': - # VS2005 / MSVC 8.0 - return ['msvcr80'] - elif msc_ver == '1500': - # VS2008 / MSVC 9.0 - return ['msvcr90'] - elif msc_ver == '1600': - # VS2010 / MSVC 10.0 - return ['msvcr100'] - else: - raise ValueError("Unknown MS Compiler version %s " % msc_ver) - - -class CygwinCCompiler(UnixCCompiler): - """ Handles the Cygwin port of the GNU C compiler to Windows. - """ - compiler_type = 'cygwin' - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".dll" - static_lib_format = "lib%s%s" - shared_lib_format = "%s%s" - exe_extension = ".exe" - - def __init__(self, verbose=0, dry_run=0, force=0): - - UnixCCompiler.__init__(self, verbose, dry_run, force) - - status, details = check_config_h() - self.debug_print("Python's GCC status: %s (details: %s)" % - (status, details)) - if status is not CONFIG_H_OK: - self.warn( - "Python's pyconfig.h doesn't seem to support your compiler. " - "Reason: %s. " - "Compiling may fail because of undefined preprocessor macros." - % details) - - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', - linker_so=('%s -mcygwin %s' % - (self.linker_dll, shared_option))) - - # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": - # cygwin shouldn't need msvcrt, but without the dlls will crash - # (gcc version 2.91.57) -- perhaps something about initialization - self.dll_libraries=["msvcrt"] - self.warn( - "Consider upgrading to a newer version of gcc") - else: - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compiles the source by spawning GCC and windres if needed.""" - if ext == '.rc' or ext == '.res': - # gcc needs '.res' and '.rc' compiled to object files !!! - try: - self.spawn(["windres", "-i", src, "-o", obj]) - except DistutilsExecError as msg: - raise CompileError(msg) - else: # for other files use the C-compiler - try: - self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def link(self, target_desc, objects, output_filename, output_dir=None, - libraries=None, library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - """Link the objects.""" - # use separate copies, so we can modify the lists - extra_preargs = copy.copy(extra_preargs or []) - libraries = copy.copy(libraries or []) - objects = copy.copy(objects or []) - - # Additional libraries - libraries.extend(self.dll_libraries) - - # handle export symbols by creating a def-file - # with executables this only works with gcc/ld as linker - if ((export_symbols is not None) and - (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - # (The linker doesn't do anything if output is up-to-date. - # So it would probably better to check if we really need this, - # but for this we had to insert some unchanged parts of - # UnixCCompiler, and this is not what we want.) - - # we want to put some files in the same directory as the - # object files are, build_temp doesn't help much - # where are the object files - temp_dir = os.path.dirname(objects[0]) - # name of dll to give the helper files the same base name - (dll_name, dll_extension) = os.path.splitext( - os.path.basename(output_filename)) - - # generate the filenames for these files - def_file = os.path.join(temp_dir, dll_name + ".def") - lib_file = os.path.join(temp_dir, 'lib' + dll_name + ".a") - - # Generate .def file - contents = [ - "LIBRARY %s" % os.path.basename(output_filename), - "EXPORTS"] - for sym in export_symbols: - contents.append(sym) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # next add options for def-file and to creating import libraries - - # dllwrap uses different options than gcc/ld - if self.linker_dll == "dllwrap": - extra_preargs.extend(["--output-lib", lib_file]) - # for dllwrap we have to use a special option - extra_preargs.extend(["--def", def_file]) - # we use gcc/ld here and can be sure ld is >= 2.9.10 - else: - # doesn't work: bfd_close build\...\libfoo.a: Invalid operation - #extra_preargs.extend(["-Wl,--out-implib,%s" % lib_file]) - # for gcc/ld the def-file is specified as any object files - objects.append(def_file) - - #end: if ((export_symbols is not None) and - # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - - # who wants symbols and a many times larger output file - # should explicitly switch the debug mode on - # otherwise we let dllwrap/ld strip the output file - # (On my machine: 10KB < stripped_file < ??100KB - # unstripped_file = stripped_file + XXX KB - # ( XXX=254 for a typical python extension)) - if not debug: - extra_preargs.append("-s") - - UnixCCompiler.link(self, target_desc, objects, output_filename, - output_dir, libraries, library_dirs, - runtime_library_dirs, - None, # export_symbols, we do this in our def-file - debug, extra_preargs, extra_postargs, build_temp, - target_lang) - - # -- Miscellaneous methods ----------------------------------------- - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - """Adds supports for rc and res files.""" - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - base, ext = os.path.splitext(os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext in ('.res', '.rc'): - # these need to be compiled to object files - obj_names.append (os.path.join(output_dir, - base + ext + self.obj_extension)) - else: - obj_names.append (os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - -# the same as cygwin plus some additional parameters -class Mingw32CCompiler(CygwinCCompiler): - """ Handles the Mingw32 port of the GNU C compiler to Windows. - """ - compiler_type = 'mingw32' - - def __init__(self, verbose=0, dry_run=0, force=0): - - CygwinCCompiler.__init__ (self, verbose, dry_run, force) - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # A real mingw32 doesn't need to specify a different entry point, - # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": - entry_point = '--entry _DllMain@12' - else: - entry_point = '' - - if is_cygwingcc(): - raise CCompilerError( - 'Cygwin gcc cannot be used with --compiler=mingw32') - - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', - linker_so='%s %s %s' - % (self.linker_dll, shared_option, - entry_point)) - # Maybe we should also append -mthreads, but then the finished - # dlls need another dll (mingwm10.dll see Mingw32 docs) - # (-mthreads: Support thread-safe exception handling on `Mingw32') - - # no additional libraries needed - self.dll_libraries=[] - - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - -# Because these compilers aren't configured in Python's pyconfig.h file by -# default, we should at least warn the user if he is using an unmodified -# version. - -CONFIG_H_OK = "ok" -CONFIG_H_NOTOK = "not ok" -CONFIG_H_UNCERTAIN = "uncertain" - -def check_config_h(): - """Check if the current Python installation appears amenable to building - extensions with GCC. - - Returns a tuple (status, details), where 'status' is one of the following - constants: - - - CONFIG_H_OK: all is well, go ahead and compile - - CONFIG_H_NOTOK: doesn't look good - - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h - - 'details' is a human-readable string explaining the situation. - - Note there are two ways to conclude "OK": either 'sys.version' contains - the string "GCC" (implying that this Python was built with GCC), or the - installed "pyconfig.h" contains the string "__GNUC__". - """ - - # XXX since this function also checks sys.version, it's not strictly a - # "pyconfig.h" check -- should probably be renamed... - - from distutils import sysconfig - - # if sys.version contains GCC then python was compiled with GCC, and the - # pyconfig.h file should be OK - if "GCC" in sys.version: - return CONFIG_H_OK, "sys.version mentions 'GCC'" - - # let's see if __GNUC__ is mentioned in python.h - fn = sysconfig.get_config_h_filename() - try: - config_h = open(fn) - try: - if "__GNUC__" in config_h.read(): - return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn - else: - return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn - finally: - config_h.close() - except OSError as exc: - return (CONFIG_H_UNCERTAIN, - "couldn't read '%s': %s" % (fn, exc.strerror)) - -RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') - -def _find_exe_version(cmd): - """Find the version of an executable by running `cmd` in the shell. - - If the command is not found, or the output does not match - `RE_VERSION`, returns None. - """ - executable = cmd.split()[0] - if find_executable(executable) is None: - return None - out = Popen(cmd, shell=True, stdout=PIPE).stdout - try: - out_string = out.read() - finally: - out.close() - result = RE_VERSION.search(out_string) - if result is None: - return None - # LooseVersion works with strings - # so we need to decode our bytes - return LooseVersion(result.group(1).decode()) - -def get_versions(): - """ Try to find out the versions of gcc, ld and dllwrap. - - If not possible it returns None for it. - """ - commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] - return tuple([_find_exe_version(cmd) for cmd in commands]) - -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) - return out_string.strip().endswith(b'cygwin') diff --git a/Lib/distutils/debug.py b/Lib/distutils/debug.py deleted file mode 100644 index daf1660f0d8..00000000000 --- a/Lib/distutils/debug.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -# If DISTUTILS_DEBUG is anything other than the empty string, we run in -# debug mode. -DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Lib/distutils/dep_util.py b/Lib/distutils/dep_util.py deleted file mode 100644 index d74f5e4e92f..00000000000 --- a/Lib/distutils/dep_util.py +++ /dev/null @@ -1,92 +0,0 @@ -"""distutils.dep_util - -Utility functions for simple, timestamp-based dependency of files -and groups of files; also, function based entirely on such -timestamp dependency analysis.""" - -import os -from distutils.errors import DistutilsFileError - - -def newer (source, target): - """Return true if 'source' exists and is more recently modified than - 'target', or if 'source' exists and 'target' doesn't. Return false if - both exist and 'target' is the same age or younger than 'source'. - Raise DistutilsFileError if 'source' does not exist. - """ - if not os.path.exists(source): - raise DistutilsFileError("file '%s' does not exist" % - os.path.abspath(source)) - if not os.path.exists(target): - return 1 - - from stat import ST_MTIME - mtime1 = os.stat(source)[ST_MTIME] - mtime2 = os.stat(target)[ST_MTIME] - - return mtime1 > mtime2 - -# newer () - - -def newer_pairwise (sources, targets): - """Walk two filename lists in parallel, testing if each source is newer - than its corresponding target. Return a pair of lists (sources, - targets) where source is newer than target, according to the semantics - of 'newer()'. - """ - if len(sources) != len(targets): - raise ValueError("'sources' and 'targets' must be same length") - - # build a pair of lists (sources, targets) where source is newer - n_sources = [] - n_targets = [] - for i in range(len(sources)): - if newer(sources[i], targets[i]): - n_sources.append(sources[i]) - n_targets.append(targets[i]) - - return (n_sources, n_targets) - -# newer_pairwise () - - -def newer_group (sources, target, missing='error'): - """Return true if 'target' is out-of-date with respect to any file - listed in 'sources'. In other words, if 'target' exists and is newer - than every file in 'sources', return false; otherwise return true. - 'missing' controls what we do when a source file is missing; the - default ("error") is to blow up with an OSError from inside 'stat()'; - if it is "ignore", we silently drop any missing source files; if it is - "newer", any missing source files make us assume that 'target' is - out-of-date (this is handy in "dry-run" mode: it'll make you pretend to - carry out commands that wouldn't work because inputs are missing, but - that doesn't matter because you're not actually going to run the - commands). - """ - # If the target doesn't even exist, then it's definitely out-of-date. - if not os.path.exists(target): - return 1 - - # Otherwise we have to find out the hard way: if *any* source file - # is more recent than 'target', then 'target' is out-of-date and - # we can immediately return true. If we fall through to the end - # of the loop, then 'target' is up-to-date and we return false. - from stat import ST_MTIME - target_mtime = os.stat(target)[ST_MTIME] - for source in sources: - if not os.path.exists(source): - if missing == 'error': # blow up when we stat() the file - pass - elif missing == 'ignore': # missing source dropped from - continue # target's dependency list - elif missing == 'newer': # missing source means target is - return 1 # out-of-date - - source_mtime = os.stat(source)[ST_MTIME] - if source_mtime > target_mtime: - return 1 - else: - return 0 - -# newer_group () diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py deleted file mode 100644 index df4d751c942..00000000000 --- a/Lib/distutils/dir_util.py +++ /dev/null @@ -1,223 +0,0 @@ -"""distutils.dir_util - -Utility functions for manipulating directories and directory trees.""" - -import os -import errno -from distutils.errors import DistutilsFileError, DistutilsInternalError -from distutils import log - -# cache for by mkpath() -- in addition to cheapening redundant calls, -# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode -_path_created = {} - -# I don't use os.makedirs because a) it's new to Python 1.5.2, and -# b) it blows up if the directory already exists (I want to silently -# succeed in that case). -def mkpath(name, mode=0o777, verbose=1, dry_run=0): - """Create a directory and any missing ancestor directories. - - If the directory already exists (or if 'name' is the empty string, which - means the current directory, which of course exists), then do nothing. - Raise DistutilsFileError if unable to create some directory along the way - (eg. some sub-path exists, but is a file rather than a directory). - If 'verbose' is true, print a one-line summary of each mkdir to stdout. - Return the list of directories actually created. - """ - - global _path_created - - # Detect a common bug -- name is None - if not isinstance(name, str): - raise DistutilsInternalError( - "mkpath: 'name' must be a string (got %r)" % (name,)) - - # XXX what's the better way to handle verbosity? print as we create - # each directory in the path (the current behaviour), or only announce - # the creation of the whole path? (quite easy to do the latter since - # we're not using a recursive algorithm) - - name = os.path.normpath(name) - created_dirs = [] - if os.path.isdir(name) or name == '': - return created_dirs - if _path_created.get(os.path.abspath(name)): - return created_dirs - - (head, tail) = os.path.split(name) - tails = [tail] # stack of lone dirs to create - - while head and tail and not os.path.isdir(head): - (head, tail) = os.path.split(head) - tails.insert(0, tail) # push next higher dir onto stack - - # now 'head' contains the deepest directory that already exists - # (that is, the child of 'head' in 'name' is the highest directory - # that does *not* exist) - for d in tails: - #print "head = %s, d = %s: " % (head, d), - head = os.path.join(head, d) - abs_head = os.path.abspath(head) - - if _path_created.get(abs_head): - continue - - if verbose >= 1: - log.info("creating %s", head) - - if not dry_run: - try: - os.mkdir(head, mode) - except OSError as exc: - if not (exc.errno == errno.EEXIST and os.path.isdir(head)): - raise DistutilsFileError( - "could not create '%s': %s" % (head, exc.args[-1])) - created_dirs.append(head) - - _path_created[abs_head] = 1 - return created_dirs - -def create_tree(base_dir, files, mode=0o777, verbose=1, dry_run=0): - """Create all the empty directories under 'base_dir' needed to put 'files' - there. - - 'base_dir' is just the name of a directory which doesn't necessarily - exist yet; 'files' is a list of filenames to be interpreted relative to - 'base_dir'. 'base_dir' + the directory portion of every file in 'files' - will be created if it doesn't already exist. 'mode', 'verbose' and - 'dry_run' flags are as for 'mkpath()'. - """ - # First get the list of directories to create - need_dir = set() - for file in files: - need_dir.add(os.path.join(base_dir, os.path.dirname(file))) - - # Now create them - for dir in sorted(need_dir): - mkpath(dir, mode, verbose=verbose, dry_run=dry_run) - -import sysconfig -_multiarch = None - -def copy_tree(src, dst, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, update=0, verbose=1, dry_run=0): - """Copy an entire directory tree 'src' to a new location 'dst'. - - Both 'src' and 'dst' must be directory names. If 'src' is not a - directory, raise DistutilsFileError. If 'dst' does not exist, it is - created with 'mkpath()'. The end result of the copy is that every - file in 'src' is copied to 'dst', and directories under 'src' are - recursively copied to 'dst'. Return the list of files that were - copied or might have been copied, using their output name. The - return value is unaffected by 'update' or 'dry_run': it is simply - the list of all files under 'src', with the names changed to be - under 'dst'. - - 'preserve_mode' and 'preserve_times' are the same as for - 'copy_file'; note that they only apply to regular files, not to - directories. If 'preserve_symlinks' is true, symlinks will be - copied as symlinks (on platforms that support them!); otherwise - (the default), the destination of the symlink will be copied. - 'update' and 'verbose' are the same as for 'copy_file'. - """ - from distutils.file_util import copy_file - - if not dry_run and not os.path.isdir(src): - raise DistutilsFileError( - "cannot copy tree '%s': not a directory" % src) - try: - names = os.listdir(src) - except OSError as e: - if dry_run: - names = [] - else: - raise DistutilsFileError( - "error listing files in '%s': %s" % (src, e.strerror)) - - ext_suffix = sysconfig.get_config_var ('EXT_SUFFIX') - _multiarch = sysconfig.get_config_var ('MULTIARCH') - if ext_suffix.endswith(_multiarch + ext_suffix[-3:]): - new_suffix = None - else: - new_suffix = "%s-%s%s" % (ext_suffix[:-3], _multiarch, ext_suffix[-3:]) - - if not dry_run: - mkpath(dst, verbose=verbose) - - outputs = [] - - for n in names: - src_name = os.path.join(src, n) - dst_name = os.path.join(dst, n) - if new_suffix and _multiarch and n.endswith(ext_suffix) and not n.endswith(new_suffix): - dst_name = os.path.join(dst, n.replace(ext_suffix, new_suffix)) - log.info("renaming extension %s -> %s", n, n.replace(ext_suffix, new_suffix)) - - if n.startswith('.nfs'): - # skip NFS rename files - continue - - if preserve_symlinks and os.path.islink(src_name): - link_dest = os.readlink(src_name) - if verbose >= 1: - log.info("linking %s -> %s", dst_name, link_dest) - if not dry_run: - os.symlink(link_dest, dst_name) - outputs.append(dst_name) - - elif os.path.isdir(src_name): - outputs.extend( - copy_tree(src_name, dst_name, preserve_mode, - preserve_times, preserve_symlinks, update, - verbose=verbose, dry_run=dry_run)) - else: - copy_file(src_name, dst_name, preserve_mode, - preserve_times, update, verbose=verbose, - dry_run=dry_run) - outputs.append(dst_name) - - return outputs - -def _build_cmdtuple(path, cmdtuples): - """Helper for remove_tree().""" - for f in os.listdir(path): - real_f = os.path.join(path,f) - if os.path.isdir(real_f) and not os.path.islink(real_f): - _build_cmdtuple(real_f, cmdtuples) - else: - cmdtuples.append((os.remove, real_f)) - cmdtuples.append((os.rmdir, path)) - -def remove_tree(directory, verbose=1, dry_run=0): - """Recursively remove an entire directory tree. - - Any errors are ignored (apart from being reported to stdout if 'verbose' - is true). - """ - global _path_created - - if verbose >= 1: - log.info("removing '%s' (and everything under it)", directory) - if dry_run: - return - cmdtuples = [] - _build_cmdtuple(directory, cmdtuples) - for cmd in cmdtuples: - try: - cmd[0](cmd[1]) - # remove dir from cache if it's already there - abspath = os.path.abspath(cmd[1]) - if abspath in _path_created: - del _path_created[abspath] - except OSError as exc: - log.warn("error removing %s: %s", directory, exc) - -def ensure_relative(path): - """Take the full path 'path', and make it a relative path. - - This is useful to make 'path' the second argument to os.path.join(). - """ - drive, path = os.path.splitdrive(path) - if path[0:1] == os.sep: - path = drive + path[1:] - return path diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py deleted file mode 100644 index 62a24516cfa..00000000000 --- a/Lib/distutils/dist.py +++ /dev/null @@ -1,1236 +0,0 @@ -"""distutils.dist - -Provides the Distribution class, which represents the module distribution -being built/installed/distributed. -""" - -import sys -import os -import re -from email import message_from_file - -try: - import warnings -except ImportError: - warnings = None - -from distutils.errors import * -from distutils.fancy_getopt import FancyGetopt, translate_longopt -from distutils.util import check_environ, strtobool, rfc822_escape -from distutils import log -from distutils.debug import DEBUG - -# Regex to define acceptable Distutils command names. This is not *quite* -# the same as a Python NAME -- I don't allow leading underscores. The fact -# that they're very similar is no coincidence; the default naming scheme is -# to look for a Python module named after the command. -command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') - - -class Distribution: - """The core of the Distutils. Most of the work hiding behind 'setup' - is really done within a Distribution instance, which farms the work out - to the Distutils commands specified on the command line. - - Setup scripts will almost never instantiate Distribution directly, - unless the 'setup()' function is totally inadequate to their needs. - However, it is conceivable that a setup script might wish to subclass - Distribution for some specialized purpose, and then pass the subclass - to 'setup()' as the 'distclass' keyword argument. If so, it is - necessary to respect the expectations that 'setup' has of Distribution. - See the code for 'setup()', in core.py, for details. - """ - - # 'global_options' describes the command-line options that may be - # supplied to the setup script prior to any actual commands. - # Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of - # these global options. This list should be kept to a bare minimum, - # since every global option is also valid as a command option -- and we - # don't want to pollute the commands with too many options that they - # have minimal control over. - # The fourth entry for verbose means that it can be repeated. - global_options = [ - ('verbose', 'v', "run verbosely (default)", 1), - ('quiet', 'q', "run quietly (turns verbosity off)"), - ('dry-run', 'n', "don't actually do anything"), - ('help', 'h', "show detailed help message"), - ('no-user-cfg', None, - 'ignore pydistutils.cfg in your home directory'), - ] - - # 'common_usage' is a short (2-3 line) string describing the common - # usage of the setup script. - common_usage = """\ -Common commands: (see '--help-commands' for more) - - setup.py build will build the package underneath 'build/' - setup.py install will install the package -""" - - # options that are not propagated to the commands - display_options = [ - ('help-commands', None, - "list all available commands"), - ('name', None, - "print package name"), - ('version', 'V', - "print package version"), - ('fullname', None, - "print -"), - ('author', None, - "print the author's name"), - ('author-email', None, - "print the author's email address"), - ('maintainer', None, - "print the maintainer's name"), - ('maintainer-email', None, - "print the maintainer's email address"), - ('contact', None, - "print the maintainer's name if known, else the author's"), - ('contact-email', None, - "print the maintainer's email address if known, else the author's"), - ('url', None, - "print the URL for this package"), - ('license', None, - "print the license of the package"), - ('licence', None, - "alias for --license"), - ('description', None, - "print the package description"), - ('long-description', None, - "print the long package description"), - ('platforms', None, - "print the list of platforms"), - ('classifiers', None, - "print the list of classifiers"), - ('keywords', None, - "print the list of keywords"), - ('provides', None, - "print the list of packages/modules provided"), - ('requires', None, - "print the list of packages/modules required"), - ('obsoletes', None, - "print the list of packages/modules made obsolete") - ] - display_option_names = [translate_longopt(x[0]) for x in display_options] - - # negative options are options that exclude other options - negative_opt = {'quiet': 'verbose'} - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, attrs=None): - """Construct a new Distribution instance: initialize all the - attributes of a Distribution, and then use 'attrs' (a dictionary - mapping attribute names to values) to assign some of those - attributes their "real" values. (Any attributes not mentioned in - 'attrs' will be assigned to some null value: 0, None, an empty list - or dictionary, etc.) Most importantly, initialize the - 'command_obj' attribute to the empty dictionary; this will be - filled in with real command objects by 'parse_command_line()'. - """ - - # Default values for our command-line options - self.verbose = 1 - self.dry_run = 0 - self.help = 0 - for attr in self.display_option_names: - setattr(self, attr, 0) - - # Store the distribution meta-data (name, version, author, and so - # forth) in a separate object -- we're getting to have enough - # information here (and enough command-line options) that it's - # worth it. Also delegate 'get_XXX()' methods to the 'metadata' - # object in a sneaky and underhanded (but efficient!) way. - self.metadata = DistributionMetadata() - for basename in self.metadata._METHOD_BASENAMES: - method_name = "get_" + basename - setattr(self, method_name, getattr(self.metadata, method_name)) - - # 'cmdclass' maps command names to class objects, so we - # can 1) quickly figure out which class to instantiate when - # we need to create a new command object, and 2) have a way - # for the setup script to override command classes - self.cmdclass = {} - - # 'command_packages' is a list of packages in which commands - # are searched for. The factory for command 'foo' is expected - # to be named 'foo' in the module 'foo' in one of the packages - # named here. This list is searched from the left; an error - # is raised if no named package provides the command being - # searched for. (Always access using get_command_packages().) - self.command_packages = None - - # 'script_name' and 'script_args' are usually set to sys.argv[0] - # and sys.argv[1:], but they can be overridden when the caller is - # not necessarily a setup script run from the command-line. - self.script_name = None - self.script_args = None - - # 'command_options' is where we store command options between - # parsing them (from config files, the command-line, etc.) and when - # they are actually needed -- ie. when the command in question is - # instantiated. It is a dictionary of dictionaries of 2-tuples: - # command_options = { command_name : { option : (source, value) } } - self.command_options = {} - - # 'dist_files' is the list of (command, pyversion, file) that - # have been created by any dist commands run so far. This is - # filled regardless of whether the run is dry or not. pyversion - # gives sysconfig.get_python_version() if the dist file is - # specific to a Python version, 'any' if it is good for all - # Python versions on the target platform, and '' for a source - # file. pyversion should not be used to specify minimum or - # maximum required Python versions; use the metainfo for that - # instead. - self.dist_files = [] - - # These options are really the business of various commands, rather - # than of the Distribution itself. We provide aliases for them in - # Distribution as a convenience to the developer. - self.packages = None - self.package_data = {} - self.package_dir = None - self.py_modules = None - self.libraries = None - self.headers = None - self.ext_modules = None - self.ext_package = None - self.include_dirs = None - self.extra_path = None - self.scripts = None - self.data_files = None - self.password = '' - - # And now initialize bookkeeping stuff that can't be supplied by - # the caller at all. 'command_obj' maps command names to - # Command instances -- that's how we enforce that every command - # class is a singleton. - self.command_obj = {} - - # 'have_run' maps command names to boolean values; it keeps track - # of whether we have actually run a particular command, to make it - # cheap to "run" a command whenever we think we might need to -- if - # it's already been done, no need for expensive filesystem - # operations, we just check the 'have_run' dictionary and carry on. - # It's only safe to query 'have_run' for a command class that has - # been instantiated -- a false value will be inserted when the - # command object is created, and replaced with a true value when - # the command is successfully run. Thus it's probably best to use - # '.get()' rather than a straight lookup. - self.have_run = {} - - # Now we'll use the attrs dictionary (ultimately, keyword args from - # the setup script) to possibly override any or all of these - # distribution options. - - if attrs: - # Pull out the set of command options and work on them - # specifically. Note that this order guarantees that aliased - # command options will override any supplied redundantly - # through the general options dictionary. - options = attrs.get('options') - if options is not None: - del attrs['options'] - for (command, cmd_options) in options.items(): - opt_dict = self.get_option_dict(command) - for (opt, val) in cmd_options.items(): - opt_dict[opt] = ("setup script", val) - - if 'licence' in attrs: - attrs['license'] = attrs['licence'] - del attrs['licence'] - msg = "'licence' distribution option is deprecated; use 'license'" - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # Now work on the rest of the attributes. Any attribute that's - # not already defined is invalid! - for (key, val) in attrs.items(): - if hasattr(self.metadata, "set_" + key): - getattr(self.metadata, "set_" + key)(val) - elif hasattr(self.metadata, key): - setattr(self.metadata, key, val) - elif hasattr(self, key): - setattr(self, key, val) - else: - msg = "Unknown distribution option: %s" % repr(key) - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # no-user-cfg is handled before other command line args - # because other args override the config files, and this - # one is needed before we can load the config files. - # If attrs['script_args'] wasn't passed, assume false. - # - # This also make sure we just look at the global options - self.want_user_cfg = True - - if self.script_args is not None: - for arg in self.script_args: - if not arg.startswith('-'): - break - if arg == '--no-user-cfg': - self.want_user_cfg = False - break - - self.finalize_options() - - def get_option_dict(self, command): - """Get the option dictionary for a given command. If that - command's option dictionary hasn't been created yet, then create it - and return the new dictionary; otherwise, return the existing - option dictionary. - """ - dict = self.command_options.get(command) - if dict is None: - dict = self.command_options[command] = {} - return dict - - def dump_option_dicts(self, header=None, commands=None, indent=""): - from pprint import pformat - - if commands is None: # dump all command option dicts - commands = sorted(self.command_options.keys()) - - if header is not None: - self.announce(indent + header) - indent = indent + " " - - if not commands: - self.announce(indent + "no commands known yet") - return - - for cmd_name in commands: - opt_dict = self.command_options.get(cmd_name) - if opt_dict is None: - self.announce(indent + - "no option dict for '%s' command" % cmd_name) - else: - self.announce(indent + - "option dict for '%s' command:" % cmd_name) - out = pformat(opt_dict) - for line in out.split('\n'): - self.announce(indent + " " + line) - - # -- Config file finding/parsing methods --------------------------- - - def find_config_files(self): - """Find as many configuration files as should be processed for this - platform, and return a list of filenames in the order in which they - should be parsed. The filenames returned are guaranteed to exist - (modulo nasty race conditions). - - There are three possible config files: distutils.cfg in the - Distutils installation directory (ie. where the top-level - Distutils __inst__.py file lives), a file in the user's home - directory named .pydistutils.cfg on Unix and pydistutils.cfg - on Windows/Mac; and setup.cfg in the current directory. - - The file in the user's home directory can be disabled with the - --no-user-cfg option. - """ - files = [] - check_environ() - - # Where to look for the system-wide Distutils config file - sys_dir = os.path.dirname(sys.modules['distutils'].__file__) - - # Look for the system config file - sys_file = os.path.join(sys_dir, "distutils.cfg") - if os.path.isfile(sys_file): - files.append(sys_file) - - # What to call the per-user config file - if os.name == 'posix': - user_filename = ".pydistutils.cfg" - else: - user_filename = "pydistutils.cfg" - - # And look for the user config file - if self.want_user_cfg: - user_file = os.path.join(os.path.expanduser('~'), user_filename) - if os.path.isfile(user_file): - files.append(user_file) - - # All platforms support local setup.cfg - local_file = "setup.cfg" - if os.path.isfile(local_file): - files.append(local_file) - - if DEBUG: - self.announce("using config files: %s" % ', '.join(files)) - - return files - - def parse_config_files(self, filenames=None): - from configparser import ConfigParser - - # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] - - ignore_options = frozenset(ignore_options) - - if filenames is None: - filenames = self.find_config_files() - - if DEBUG: - self.announce("Distribution.parse_config_files():") - - parser = ConfigParser() - for filename in filenames: - if DEBUG: - self.announce(" reading %s" % filename) - parser.read(filename) - for section in parser.sections(): - options = parser.options(section) - opt_dict = self.get_option_dict(section) - - for opt in options: - if opt != '__name__' and opt not in ignore_options: - val = parser.get(section,opt) - opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) - - # Make the ConfigParser forget everything (so we retain - # the original filenames that options come from) - parser.__init__() - - # If there was a "global" section in the config file, use it - # to set Distribution options. - - if 'global' in self.command_options: - for (opt, (src, val)) in self.command_options['global'].items(): - alias = self.negative_opt.get(opt) - try: - if alias: - setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! - setattr(self, opt, strtobool(val)) - else: - setattr(self, opt, val) - except ValueError as msg: - raise DistutilsOptionError(msg) - - # -- Command-line parsing methods ---------------------------------- - - def parse_command_line(self): - """Parse the setup script's command line, taken from the - 'script_args' instance attribute (which defaults to 'sys.argv[1:]' - -- see 'setup()' in core.py). This list is first processed for - "global options" -- options that set attributes of the Distribution - instance. Then, it is alternately scanned for Distutils commands - and options for that command. Each new command terminates the - options for the previous command. The allowed options for a - command are determined by the 'user_options' attribute of the - command class -- thus, we have to be able to load command classes - in order to parse the command line. Any error in that 'options' - attribute raises DistutilsGetoptError; any error on the - command-line raises DistutilsArgError. If no Distutils commands - were found on the command line, raises DistutilsArgError. Return - true if command-line was successfully parsed and we should carry - on with executing commands; false if no errors but we shouldn't - execute commands (currently, this only happens if user asks for - help). - """ - # - # We now have enough information to show the Macintosh dialog - # that allows the user to interactively specify the "command line". - # - toplevel_options = self._get_toplevel_options() - - # We have to parse the command line a bit at a time -- global - # options, then the first command, then its options, and so on -- - # because each command will be handled by a different class, and - # the options that are valid for a particular class aren't known - # until we have loaded the command class, which doesn't happen - # until we know what the command is. - - self.commands = [] - parser = FancyGetopt(toplevel_options + self.display_options) - parser.set_negative_aliases(self.negative_opt) - parser.set_aliases({'licence': 'license'}) - args = parser.getopt(args=self.script_args, object=self) - option_order = parser.get_option_order() - log.set_verbosity(self.verbose) - - # for display options we return immediately - if self.handle_display_options(option_order): - return - while args: - args = self._parse_command_opts(parser, args) - if args is None: # user asked for help (and got it) - return - - # Handle the cases of --help as a "global" option, ie. - # "setup.py --help" and "setup.py --help command ...". For the - # former, we show global options (--verbose, --dry-run, etc.) - # and display-only options (--name, --version, etc.); for the - # latter, we omit the display-only options and show help for - # each command listed on the command line. - if self.help: - self._show_help(parser, - display_options=len(self.commands) == 0, - commands=self.commands) - return - - # Oops, no commands found -- an end-user error - if not self.commands: - raise DistutilsArgError("no commands supplied") - - # All is well: return true - return True - - def _get_toplevel_options(self): - """Return the non-display options recognized at the top level. - - This includes options that are recognized *only* at the top - level as well as options recognized for commands. - """ - return self.global_options + [ - ("command-packages=", None, - "list of packages that provide distutils commands"), - ] - - def _parse_command_opts(self, parser, args): - """Parse the command-line options for a single command. - 'parser' must be a FancyGetopt instance; 'args' must be the list - of arguments, starting with the current command (whose options - we are about to parse). Returns a new version of 'args' with - the next command at the front of the list; will be the empty - list if there are no more commands on the command line. Returns - None if the user asked for help on this command. - """ - # late import because of mutual dependence between these modules - from distutils.cmd import Command - - # Pull the current command from the head of the command line - command = args[0] - if not command_re.match(command): - raise SystemExit("invalid command name '%s'" % command) - self.commands.append(command) - - # Dig up the command class that implements this command, so we - # 1) know that it's a valid command, and 2) know which options - # it takes. - try: - cmd_class = self.get_command_class(command) - except DistutilsModuleError as msg: - raise DistutilsArgError(msg) - - # Require that the command class be derived from Command -- want - # to be sure that the basic "command" interface is implemented. - if not issubclass(cmd_class, Command): - raise DistutilsClassError( - "command class %s must subclass Command" % cmd_class) - - # Also make sure that the command object provides a list of its - # known options. - if not (hasattr(cmd_class, 'user_options') and - isinstance(cmd_class.user_options, list)): - msg = ("command class %s must provide " - "'user_options' attribute (a list of tuples)") - raise DistutilsClassError(msg % cmd_class) - - # If the command class has a list of negative alias options, - # merge it in with the global negative aliases. - negative_opt = self.negative_opt - if hasattr(cmd_class, 'negative_opt'): - negative_opt = negative_opt.copy() - negative_opt.update(cmd_class.negative_opt) - - # Check for help_options in command class. They have a different - # format (tuple of four) so we need to preprocess them here. - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_options = fix_help_options(cmd_class.help_options) - else: - help_options = [] - - # All commands support the global options too, just by adding - # in 'global_options'. - parser.set_option_table(self.global_options + - cmd_class.user_options + - help_options) - parser.set_negative_aliases(negative_opt) - (args, opts) = parser.getopt(args[1:]) - if hasattr(opts, 'help') and opts.help: - self._show_help(parser, display_options=0, commands=[cmd_class]) - return - - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_option_found=0 - for (help_option, short, desc, func) in cmd_class.help_options: - if hasattr(opts, parser.get_attr_name(help_option)): - help_option_found=1 - if callable(func): - func() - else: - raise DistutilsClassError( - "invalid help function %r for help option '%s': " - "must be a callable object (function, etc.)" - % (func, help_option)) - - if help_option_found: - return - - # Put the options from the command-line into their official - # holding pen, the 'command_options' dictionary. - opt_dict = self.get_option_dict(command) - for (name, value) in vars(opts).items(): - opt_dict[name] = ("command line", value) - - return args - - def finalize_options(self): - """Set final values for all the options on the Distribution - instance, analogous to the .finalize_options() method of Command - objects. - """ - for attr in ('keywords', 'platforms'): - value = getattr(self.metadata, attr) - if value is None: - continue - if isinstance(value, str): - value = [elm.strip() for elm in value.split(',')] - setattr(self.metadata, attr, value) - - def _show_help(self, parser, global_options=1, display_options=1, - commands=[]): - """Show help for the setup script command-line in the form of - several lists of command-line options. 'parser' should be a - FancyGetopt instance; do not expect it to be returned in the - same state, as its option table will be reset to make it - generate the correct help text. - - If 'global_options' is true, lists the global options: - --verbose, --dry-run, etc. If 'display_options' is true, lists - the "display-only" options: --name, --version, etc. Finally, - lists per-command help for every command name or command class - in 'commands'. - """ - # late import because of mutual dependence between these modules - from distutils.core import gen_usage - from distutils.cmd import Command - - if global_options: - if display_options: - options = self._get_toplevel_options() - else: - options = self.global_options - parser.set_option_table(options) - parser.print_help(self.common_usage + "\nGlobal options:") - print('') - - if display_options: - parser.set_option_table(self.display_options) - parser.print_help( - "Information display options (just display " + - "information, ignore any commands)") - print('') - - for command in self.commands: - if isinstance(command, type) and issubclass(command, Command): - klass = command - else: - klass = self.get_command_class(command) - if (hasattr(klass, 'help_options') and - isinstance(klass.help_options, list)): - parser.set_option_table(klass.user_options + - fix_help_options(klass.help_options)) - else: - parser.set_option_table(klass.user_options) - parser.print_help("Options for '%s' command:" % klass.__name__) - print('') - - print(gen_usage(self.script_name)) - - def handle_display_options(self, option_order): - """If there were any non-global "display-only" options - (--help-commands or the metadata display options) on the command - line, display the requested info and return true; else return - false. - """ - from distutils.core import gen_usage - - # User just wants a list of commands -- we'll print it out and stop - # processing now (ie. if they ran "setup --help-commands foo bar", - # we ignore "foo bar"). - if self.help_commands: - self.print_commands() - print('') - print(gen_usage(self.script_name)) - return 1 - - # If user supplied any of the "display metadata" options, then - # display that metadata in the order in which the user supplied the - # metadata options. - any_display_options = 0 - is_display_option = {} - for option in self.display_options: - is_display_option[option[0]] = 1 - - for (opt, val) in option_order: - if val and is_display_option.get(opt): - opt = translate_longopt(opt) - value = getattr(self.metadata, "get_"+opt)() - if opt in ['keywords', 'platforms']: - print(','.join(value)) - elif opt in ('classifiers', 'provides', 'requires', - 'obsoletes'): - print('\n'.join(value)) - else: - print(value) - any_display_options = 1 - - return any_display_options - - def print_command_list(self, commands, header, max_length): - """Print a subset of the list of all commands -- used by - 'print_commands()'. - """ - print(header + ":") - - for cmd in commands: - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - - print(" %-*s %s" % (max_length, cmd, description)) - - def print_commands(self): - """Print out a help message listing all available commands with a - description of each. The list is divided into "standard commands" - (listed in distutils.command.__all__) and "extra commands" - (mentioned in self.cmdclass, but not a standard command). The - descriptions come from the command class attribute - 'description'. - """ - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - max_length = 0 - for cmd in (std_commands + extra_commands): - if len(cmd) > max_length: - max_length = len(cmd) - - self.print_command_list(std_commands, - "Standard commands", - max_length) - if extra_commands: - print() - self.print_command_list(extra_commands, - "Extra commands", - max_length) - - def get_command_list(self): - """Get a list of (command, description) tuples. - The list is divided into "standard commands" (listed in - distutils.command.__all__) and "extra commands" (mentioned in - self.cmdclass, but not a standard command). The descriptions come - from the command class attribute 'description'. - """ - # Currently this is only used on Mac OS, for the Mac-only GUI - # Distutils interface (by Jack Jansen) - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - rv = [] - for cmd in (std_commands + extra_commands): - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - rv.append((cmd, description)) - return rv - - # -- Command class/object methods ---------------------------------- - - def get_command_packages(self): - """Return a list of packages from which commands are loaded.""" - pkgs = self.command_packages - if not isinstance(pkgs, list): - if pkgs is None: - pkgs = '' - pkgs = [pkg.strip() for pkg in pkgs.split(',') if pkg != ''] - if "distutils.command" not in pkgs: - pkgs.insert(0, "distutils.command") - self.command_packages = pkgs - return pkgs - - def get_command_class(self, command): - """Return the class that implements the Distutils command named by - 'command'. First we check the 'cmdclass' dictionary; if the - command is mentioned there, we fetch the class object from the - dictionary and return it. Otherwise we load the command module - ("distutils.command." + command) and fetch the command class from - the module. The loaded class is also stored in 'cmdclass' - to speed future calls to 'get_command_class()'. - - Raises DistutilsModuleError if the expected module could not be - found, or if that module does not define the expected class. - """ - klass = self.cmdclass.get(command) - if klass: - return klass - - for pkgname in self.get_command_packages(): - module_name = "%s.%s" % (pkgname, command) - klass_name = command - - try: - __import__(module_name) - module = sys.modules[module_name] - except ImportError: - continue - - try: - klass = getattr(module, klass_name) - except AttributeError: - raise DistutilsModuleError( - "invalid command '%s' (no class '%s' in module '%s')" - % (command, klass_name, module_name)) - - self.cmdclass[command] = klass - return klass - - raise DistutilsModuleError("invalid command '%s'" % command) - - def get_command_obj(self, command, create=1): - """Return the command object for 'command'. Normally this object - is cached on a previous call to 'get_command_obj()'; if no command - object for 'command' is in the cache, then we either create and - return it (if 'create' is true) or return None. - """ - cmd_obj = self.command_obj.get(command) - if not cmd_obj and create: - if DEBUG: - self.announce("Distribution.get_command_obj(): " - "creating '%s' command object" % command) - - klass = self.get_command_class(command) - cmd_obj = self.command_obj[command] = klass(self) - self.have_run[command] = 0 - - # Set any options that were supplied in config files - # or on the command line. (NB. support for error - # reporting is lame here: any errors aren't reported - # until 'finalize_options()' is called, which means - # we won't report the source of the error.) - options = self.command_options.get(command) - if options: - self._set_command_options(cmd_obj, options) - - return cmd_obj - - def _set_command_options(self, command_obj, option_dict=None): - """Set the options for 'command_obj' from 'option_dict'. Basically - this means copying elements of a dictionary ('option_dict') to - attributes of an instance ('command'). - - 'command_obj' must be a Command instance. If 'option_dict' is not - supplied, uses the standard option dictionary for this command - (from 'self.command_options'). - """ - command_name = command_obj.get_command_name() - if option_dict is None: - option_dict = self.get_option_dict(command_name) - - if DEBUG: - self.announce(" setting options for '%s' command:" % command_name) - for (option, (source, value)) in option_dict.items(): - if DEBUG: - self.announce(" %s = %s (from %s)" % (option, value, - source)) - try: - bool_opts = [translate_longopt(o) - for o in command_obj.boolean_options] - except AttributeError: - bool_opts = [] - try: - neg_opt = command_obj.negative_opt - except AttributeError: - neg_opt = {} - - try: - is_string = isinstance(value, str) - if option in neg_opt and is_string: - setattr(command_obj, neg_opt[option], not strtobool(value)) - elif option in bool_opts and is_string: - setattr(command_obj, option, strtobool(value)) - elif hasattr(command_obj, option): - setattr(command_obj, option, value) - else: - raise DistutilsOptionError( - "error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) - except ValueError as msg: - raise DistutilsOptionError(msg) - - def reinitialize_command(self, command, reinit_subcommands=0): - """Reinitializes a command to the state it was in when first - returned by 'get_command_obj()': ie., initialized but not yet - finalized. This provides the opportunity to sneak option - values in programmatically, overriding or supplementing - user-supplied values from the config files and command line. - You'll have to re-finalize the command object (by calling - 'finalize_options()' or 'ensure_finalized()') before using it for - real. - - 'command' should be a command name (string) or command object. If - 'reinit_subcommands' is true, also reinitializes the command's - sub-commands, as declared by the 'sub_commands' class attribute (if - it has one). See the "install" command for an example. Only - reinitializes the sub-commands that actually matter, ie. those - whose test predicates return true. - - Returns the reinitialized command object. - """ - from distutils.cmd import Command - if not isinstance(command, Command): - command_name = command - command = self.get_command_obj(command_name) - else: - command_name = command.get_command_name() - - if not command.finalized: - return command - command.initialize_options() - command.finalized = 0 - self.have_run[command_name] = 0 - self._set_command_options(command) - - if reinit_subcommands: - for sub in command.get_sub_commands(): - self.reinitialize_command(sub, reinit_subcommands) - - return command - - # -- Methods that operate on the Distribution ---------------------- - - def announce(self, msg, level=log.INFO): - log.log(level, msg) - - def run_commands(self): - """Run each command that was seen on the setup script command line. - Uses the list of commands found and cache of command objects - created by 'get_command_obj()'. - """ - for cmd in self.commands: - self.run_command(cmd) - - # -- Methods that operate on its Commands -------------------------- - - def run_command(self, command): - """Do whatever it takes to run a command (including nothing at all, - if the command has already been run). Specifically: if we have - already created and run the command named by 'command', return - silently without doing anything. If the command named by 'command' - doesn't even have a command object yet, create one. Then invoke - 'run()' on that command object (or an existing one). - """ - # Already been here, done that? then return silently. - if self.have_run.get(command): - return - - log.info("running %s", command) - cmd_obj = self.get_command_obj(command) - cmd_obj.ensure_finalized() - cmd_obj.run() - self.have_run[command] = 1 - - # -- Distribution query methods ------------------------------------ - - def has_pure_modules(self): - return len(self.packages or self.py_modules or []) > 0 - - def has_ext_modules(self): - return self.ext_modules and len(self.ext_modules) > 0 - - def has_c_libraries(self): - return self.libraries and len(self.libraries) > 0 - - def has_modules(self): - return self.has_pure_modules() or self.has_ext_modules() - - def has_headers(self): - return self.headers and len(self.headers) > 0 - - def has_scripts(self): - return self.scripts and len(self.scripts) > 0 - - def has_data_files(self): - return self.data_files and len(self.data_files) > 0 - - def is_pure(self): - return (self.has_pure_modules() and - not self.has_ext_modules() and - not self.has_c_libraries()) - - # -- Metadata query methods ---------------------------------------- - - # If you're looking for 'get_name()', 'get_version()', and so forth, - # they are defined in a sneaky way: the constructor binds self.get_XXX - # to self.metadata.get_XXX. The actual code is in the - # DistributionMetadata class, below. - -class DistributionMetadata: - """Dummy class to hold the distribution meta-data: name, version, - author, and so forth. - """ - - _METHOD_BASENAMES = ("name", "version", "author", "author_email", - "maintainer", "maintainer_email", "url", - "license", "description", "long_description", - "keywords", "platforms", "fullname", "contact", - "contact_email", "classifiers", "download_url", - # PEP 314 - "provides", "requires", "obsoletes", - ) - - def __init__(self, path=None): - if path is not None: - self.read_pkg_file(open(path)) - else: - self.name = None - self.version = None - self.author = None - self.author_email = None - self.maintainer = None - self.maintainer_email = None - self.url = None - self.license = None - self.description = None - self.long_description = None - self.keywords = None - self.platforms = None - self.classifiers = None - self.download_url = None - # PEP 314 - self.provides = None - self.requires = None - self.obsoletes = None - - def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) - - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None - return value - - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values - - metadata_version = msg['metadata-version'] - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') - # we are filling author only. - self.author = _read_field('author') - self.maintainer = None - self.author_email = _read_field('author-email') - self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') - - if 'download-url' in msg: - self.download_url = _read_field('download-url') - else: - self.download_url = None - - self.long_description = _read_field('description') - self.description = _read_field('summary') - - if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') - - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') - - # PEP 314 - these fields only exist in 1.1 - if metadata_version == '1.1': - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') - else: - self.requires = None - self.provides = None - self.obsoletes = None - - def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) - - def write_pkg_file(self, file): - """Write the PKG-INFO format data to a file object. - """ - version = '1.0' - if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - version = '1.1' - - file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) - if self.download_url: - file.write('Download-URL: %s\n' % self.download_url) - - long_desc = rfc822_escape(self.get_long_description()) - file.write('Description: %s\n' % long_desc) - - keywords = ','.join(self.get_keywords()) - if keywords: - file.write('Keywords: %s\n' % keywords) - - self._write_list(file, 'Platform', self.get_platforms()) - self._write_list(file, 'Classifier', self.get_classifiers()) - - # PEP 314 - self._write_list(file, 'Requires', self.get_requires()) - self._write_list(file, 'Provides', self.get_provides()) - self._write_list(file, 'Obsoletes', self.get_obsoletes()) - - def _write_list(self, file, name, values): - for value in values: - file.write('%s: %s\n' % (name, value)) - - # -- Metadata query methods ---------------------------------------- - - def get_name(self): - return self.name or "UNKNOWN" - - def get_version(self): - return self.version or "0.0.0" - - def get_fullname(self): - return "%s-%s" % (self.get_name(), self.get_version()) - - def get_author(self): - return self.author or "UNKNOWN" - - def get_author_email(self): - return self.author_email or "UNKNOWN" - - def get_maintainer(self): - return self.maintainer or "UNKNOWN" - - def get_maintainer_email(self): - return self.maintainer_email or "UNKNOWN" - - def get_contact(self): - return self.maintainer or self.author or "UNKNOWN" - - def get_contact_email(self): - return self.maintainer_email or self.author_email or "UNKNOWN" - - def get_url(self): - return self.url or "UNKNOWN" - - def get_license(self): - return self.license or "UNKNOWN" - get_licence = get_license - - def get_description(self): - return self.description or "UNKNOWN" - - def get_long_description(self): - return self.long_description or "UNKNOWN" - - def get_keywords(self): - return self.keywords or [] - - def get_platforms(self): - return self.platforms or ["UNKNOWN"] - - def get_classifiers(self): - return self.classifiers or [] - - def get_download_url(self): - return self.download_url or "UNKNOWN" - - # PEP 314 - def get_requires(self): - return self.requires or [] - - def set_requires(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.requires = value - - def get_provides(self): - return self.provides or [] - - def set_provides(self, value): - value = [v.strip() for v in value] - for v in value: - import distutils.versionpredicate - distutils.versionpredicate.split_provision(v) - self.provides = value - - def get_obsoletes(self): - return self.obsoletes or [] - - def set_obsoletes(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.obsoletes = value - -def fix_help_options(options): - """Convert a 4-tuple 'help_options' list as found in various command - classes to the 3-tuple form required by FancyGetopt. - """ - new_options = [] - for help_tuple in options: - new_options.append(help_tuple[0:3]) - return new_options diff --git a/Lib/distutils/errors.py b/Lib/distutils/errors.py deleted file mode 100644 index 8b93059e19f..00000000000 --- a/Lib/distutils/errors.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.errors - -Provides exceptions used by the Distutils modules. Note that Distutils -modules may raise standard exceptions; in particular, SystemExit is -usually raised for errors that are obviously the end-user's fault -(eg. bad command-line arguments). - -This module is safe to use in "from ... import *" mode; it only exports -symbols whose names start with "Distutils" and end with "Error".""" - -class DistutilsError (Exception): - """The root of all Distutils evil.""" - pass - -class DistutilsModuleError (DistutilsError): - """Unable to load an expected module, or to find an expected class - within some module (in particular, command modules and classes).""" - pass - -class DistutilsClassError (DistutilsError): - """Some command class (or possibly distribution class, if anyone - feels a need to subclass Distribution) is found not to be holding - up its end of the bargain, ie. implementing some part of the - "command "interface.""" - pass - -class DistutilsGetoptError (DistutilsError): - """The option table provided to 'fancy_getopt()' is bogus.""" - pass - -class DistutilsArgError (DistutilsError): - """Raised by fancy_getopt in response to getopt.error -- ie. an - error in the command line usage.""" - pass - -class DistutilsFileError (DistutilsError): - """Any problems in the filesystem: expected file not found, etc. - Typically this is for problems that we detect before OSError - could be raised.""" - pass - -class DistutilsOptionError (DistutilsError): - """Syntactic/semantic errors in command options, such as use of - mutually conflicting options, or inconsistent options, - badly-spelled values, etc. No distinction is made between option - values originating in the setup script, the command line, config - files, or what-have-you -- but if we *know* something originated in - the setup script, we'll raise DistutilsSetupError instead.""" - pass - -class DistutilsSetupError (DistutilsError): - """For errors that can be definitely blamed on the setup script, - such as invalid keyword arguments to 'setup()'.""" - pass - -class DistutilsPlatformError (DistutilsError): - """We don't know how to do something on the current platform (but - we do know how to do it on some platform) -- eg. trying to compile - C files on a platform not supported by a CCompiler subclass.""" - pass - -class DistutilsExecError (DistutilsError): - """Any problems executing an external program (such as the C - compiler, when compiling C files).""" - pass - -class DistutilsInternalError (DistutilsError): - """Internal inconsistencies or impossibilities (obviously, this - should never be seen if the code is working!).""" - pass - -class DistutilsTemplateError (DistutilsError): - """Syntax error in a file list template.""" - -class DistutilsByteCompileError(DistutilsError): - """Byte compile error.""" - -# Exception classes used by the CCompiler implementation classes -class CCompilerError (Exception): - """Some compile/link operation failed.""" - -class PreprocessError (CCompilerError): - """Failure to preprocess one or more C/C++ files.""" - -class CompileError (CCompilerError): - """Failure to compile one or more C/C++ source files.""" - -class LibError (CCompilerError): - """Failure to create a static library from one or more C/C++ object - files.""" - -class LinkError (CCompilerError): - """Failure to link one or more C/C++ object files into an executable - or shared library file.""" - -class UnknownFileError (CCompilerError): - """Attempt to process an unknown file type.""" diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py deleted file mode 100644 index c507da360aa..00000000000 --- a/Lib/distutils/extension.py +++ /dev/null @@ -1,240 +0,0 @@ -"""distutils.extension - -Provides the Extension class, used to describe C/C++ extension -modules in setup scripts.""" - -import os -import warnings - -# This class is really only used by the "build_ext" command, so it might -# make sense to put it in distutils.command.build_ext. However, that -# module is already big enough, and I want to make this class a bit more -# complex to simplify some common cases ("foo" module in "foo.c") and do -# better error-checking ("foo.c" actually exists). -# -# Also, putting this in build_ext.py means every setup script would have to -# import that large-ish module (indirectly, through distutils.core) in -# order to do anything. - -class Extension: - """Just a collection of attributes that describes an extension - module and everything needed to build it (hopefully in a portable - way, but there are hooks that let you be as unportable as you need). - - Instance attributes: - name : string - the full name of the extension, including any packages -- ie. - *not* a filename or pathname, but Python dotted name - sources : [string] - list of source filenames, relative to the distribution root - (where the setup script lives), in Unix form (slash-separated) - for portability. Source files may be C, C++, SWIG (.i), - platform-specific resource files, or whatever else is recognized - by the "build_ext" command as source for a Python extension. - include_dirs : [string] - list of directories to search for C/C++ header files (in Unix - form for portability) - define_macros : [(name : string, value : string|None)] - list of macros to define; each macro is defined using a 2-tuple, - where 'value' is either the string to define it to or None to - define it without a particular value (equivalent of "#define - FOO" in source or -DFOO on Unix C compiler command line) - undef_macros : [string] - list of macros to undefine explicitly - library_dirs : [string] - list of directories to search for C/C++ libraries at link time - libraries : [string] - list of library names (not filenames or paths) to link against - runtime_library_dirs : [string] - list of directories to search for C/C++ libraries at run time - (for shared extensions, this is when the extension is loaded) - extra_objects : [string] - list of extra files to link with (eg. object files not implied - by 'sources', static library that must be explicitly specified, - binary resource files, etc.) - extra_compile_args : [string] - any extra platform- and compiler-specific information to use - when compiling the source files in 'sources'. For platforms and - compilers where "command line" makes sense, this is typically a - list of command-line arguments, but for other platforms it could - be anything. - extra_link_args : [string] - any extra platform- and compiler-specific information to use - when linking object files together to create the extension (or - to create a new static Python interpreter). Similar - interpretation as for 'extra_compile_args'. - export_symbols : [string] - list of symbols to be exported from a shared extension. Not - used on all platforms, and not generally necessary for Python - extensions, which typically export exactly one symbol: "init" + - extension_name. - swig_opts : [string] - any extra options to pass to SWIG if a source file has the .i - extension. - depends : [string] - list of files that the extension depends on - language : string - extension language (i.e. "c", "c++", "objc"). Will be detected - from the source extensions if not provided. - optional : boolean - specifies that a build failure in the extension should not abort the - build process, but simply not install the failing extension. - """ - - # When adding arguments to this constructor, be sure to update - # setup_keywords in core.py. - def __init__(self, name, sources, - include_dirs=None, - define_macros=None, - undef_macros=None, - library_dirs=None, - libraries=None, - runtime_library_dirs=None, - extra_objects=None, - extra_compile_args=None, - extra_link_args=None, - export_symbols=None, - swig_opts = None, - depends=None, - language=None, - optional=None, - **kw # To catch unknown keywords - ): - if not isinstance(name, str): - raise AssertionError("'name' must be a string") - if not (isinstance(sources, list) and - all(isinstance(v, str) for v in sources)): - raise AssertionError("'sources' must be a list of strings") - - self.name = name - self.sources = sources - self.include_dirs = include_dirs or [] - self.define_macros = define_macros or [] - self.undef_macros = undef_macros or [] - self.library_dirs = library_dirs or [] - self.libraries = libraries or [] - self.runtime_library_dirs = runtime_library_dirs or [] - self.extra_objects = extra_objects or [] - self.extra_compile_args = extra_compile_args or [] - self.extra_link_args = extra_link_args or [] - self.export_symbols = export_symbols or [] - self.swig_opts = swig_opts or [] - self.depends = depends or [] - self.language = language - self.optional = optional - - # If there are unknown keyword options, warn about them - if len(kw) > 0: - options = [repr(option) for option in kw] - options = ', '.join(sorted(options)) - msg = "Unknown Extension options: %s" % options - warnings.warn(msg) - - def __repr__(self): - return '<%s.%s(%r) at %#x>' % ( - self.__class__.__module__, - self.__class__.__qualname__, - self.name, - id(self)) - - -def read_setup_file(filename): - """Reads a Setup file and returns Extension instances.""" - from distutils.sysconfig import (parse_makefile, expand_makefile_vars, - _variable_rx) - - from distutils.text_file import TextFile - from distutils.util import split_quoted - - # First pass over the file to gather "VAR = VALUE" assignments. - vars = parse_makefile(filename) - - # Second pass to gobble up the real content: lines of the form - # ... [ ...] [ ...] [ ...] - file = TextFile(filename, - strip_comments=1, skip_blanks=1, join_lines=1, - lstrip_ws=1, rstrip_ws=1) - try: - extensions = [] - - while True: - line = file.readline() - if line is None: # eof - break - if _variable_rx.match(line): # VAR=VALUE, handled in first pass - continue - - if line[0] == line[-1] == "*": - file.warn("'%s' lines not handled yet" % line) - continue - - line = expand_makefile_vars(line, vars) - words = split_quoted(line) - - # NB. this parses a slightly different syntax than the old - # makesetup script: here, there must be exactly one extension per - # line, and it must be the first word of the line. I have no idea - # why the old syntax supported multiple extensions per line, as - # they all wind up being the same. - - module = words[0] - ext = Extension(module, []) - append_next_word = None - - for word in words[1:]: - if append_next_word is not None: - append_next_word.append(word) - append_next_word = None - continue - - suffix = os.path.splitext(word)[1] - switch = word[0:2] ; value = word[2:] - - if suffix in (".c", ".cc", ".cpp", ".cxx", ".c++", ".m", ".mm"): - # hmm, should we do something about C vs. C++ sources? - # or leave it up to the CCompiler implementation to - # worry about? - ext.sources.append(word) - elif switch == "-I": - ext.include_dirs.append(value) - elif switch == "-D": - equals = value.find("=") - if equals == -1: # bare "-DFOO" -- no value - ext.define_macros.append((value, None)) - else: # "-DFOO=blah" - ext.define_macros.append((value[0:equals], - value[equals+2:])) - elif switch == "-U": - ext.undef_macros.append(value) - elif switch == "-C": # only here 'cause makesetup has it! - ext.extra_compile_args.append(word) - elif switch == "-l": - ext.libraries.append(value) - elif switch == "-L": - ext.library_dirs.append(value) - elif switch == "-R": - ext.runtime_library_dirs.append(value) - elif word == "-rpath": - append_next_word = ext.runtime_library_dirs - elif word == "-Xlinker": - append_next_word = ext.extra_link_args - elif word == "-Xcompiler": - append_next_word = ext.extra_compile_args - elif switch == "-u": - ext.extra_link_args.append(word) - if not value: - append_next_word = ext.extra_link_args - elif suffix in (".a", ".so", ".sl", ".o", ".dylib"): - # NB. a really faithful emulation of makesetup would - # append a .o file to extra_objects only if it - # had a slash in it; otherwise, it would s/.o/.c/ - # and append it to sources. Hmmmm. - ext.extra_objects.append(word) - else: - file.warn("unrecognized argument '%s'" % word) - - extensions.append(ext) - finally: - file.close() - - return extensions diff --git a/Lib/distutils/fancy_getopt.py b/Lib/distutils/fancy_getopt.py deleted file mode 100644 index 7d170dd2773..00000000000 --- a/Lib/distutils/fancy_getopt.py +++ /dev/null @@ -1,457 +0,0 @@ -"""distutils.fancy_getopt - -Wrapper around the standard getopt module that provides the following -additional features: - * short and long options are tied together - * options have help strings, so fancy_getopt could potentially - create a complete usage summary - * options set attributes of a passed-in object -""" - -import sys, string, re -import getopt -from distutils.errors import * - -# Much like command_re in distutils.core, this is close to but not quite -# the same as a Python NAME -- except, in the spirit of most GNU -# utilities, we use '-' in place of '_'. (The spirit of LISP lives on!) -# The similarities to NAME are again not a coincidence... -longopt_pat = r'[a-zA-Z](?:[a-zA-Z0-9-]*)' -longopt_re = re.compile(r'^%s$' % longopt_pat) - -# For recognizing "negative alias" options, eg. "quiet=!verbose" -neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) - -# This is used to translate long options to legitimate Python identifiers -# (for use as attributes of some object). -longopt_xlate = str.maketrans('-', '_') - -class FancyGetopt: - """Wrapper around the standard 'getopt()' module that provides some - handy extra functionality: - * short and long options are tied together - * options have help strings, and help text can be assembled - from them - * options set attributes of a passed-in object - * boolean options can have "negative aliases" -- eg. if - --quiet is the "negative alias" of --verbose, then "--quiet" - on the command line sets 'verbose' to false - """ - - def __init__(self, option_table=None): - # The option table is (currently) a list of tuples. The - # tuples may have 3 or four values: - # (long_option, short_option, help_string [, repeatable]) - # if an option takes an argument, its long_option should have '=' - # appended; short_option should just be a single character, no ':' - # in any case. If a long_option doesn't have a corresponding - # short_option, short_option should be None. All option tuples - # must have long options. - self.option_table = option_table - - # 'option_index' maps long option names to entries in the option - # table (ie. those 3-tuples). - self.option_index = {} - if self.option_table: - self._build_index() - - # 'alias' records (duh) alias options; {'foo': 'bar'} means - # --foo is an alias for --bar - self.alias = {} - - # 'negative_alias' keeps track of options that are the boolean - # opposite of some other option - self.negative_alias = {} - - # These keep track of the information in the option table. We - # don't actually populate these structures until we're ready to - # parse the command-line, since the 'option_table' passed in here - # isn't necessarily the final word. - self.short_opts = [] - self.long_opts = [] - self.short2long = {} - self.attr_name = {} - self.takes_arg = {} - - # And 'option_order' is filled up in 'getopt()'; it records the - # original order of options (and their values) on the command-line, - # but expands short options, converts aliases, etc. - self.option_order = [] - - def _build_index(self): - self.option_index.clear() - for option in self.option_table: - self.option_index[option[0]] = option - - def set_option_table(self, option_table): - self.option_table = option_table - self._build_index() - - def add_option(self, long_option, short_option=None, help_string=None): - if long_option in self.option_index: - raise DistutilsGetoptError( - "option conflict: already an option '%s'" % long_option) - else: - option = (long_option, short_option, help_string) - self.option_table.append(option) - self.option_index[long_option] = option - - def has_option(self, long_option): - """Return true if the option table for this parser has an - option with long name 'long_option'.""" - return long_option in self.option_index - - def get_attr_name(self, long_option): - """Translate long option name 'long_option' to the form it - has as an attribute of some object: ie., translate hyphens - to underscores.""" - return long_option.translate(longopt_xlate) - - def _check_alias_dict(self, aliases, what): - assert isinstance(aliases, dict) - for (alias, opt) in aliases.items(): - if alias not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "option '%s' not defined") % (what, alias, alias)) - if opt not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "aliased option '%s' not defined") % (what, alias, opt)) - - def set_aliases(self, alias): - """Set the aliases for this option parser.""" - self._check_alias_dict(alias, "alias") - self.alias = alias - - def set_negative_aliases(self, negative_alias): - """Set the negative aliases for this option parser. - 'negative_alias' should be a dictionary mapping option names to - option names, both the key and value must already be defined - in the option table.""" - self._check_alias_dict(negative_alias, "negative alias") - self.negative_alias = negative_alias - - def _grok_option_table(self): - """Populate the various data structures that keep tabs on the - option table. Called by 'getopt()' before it can do anything - worthwhile. - """ - self.long_opts = [] - self.short_opts = [] - self.short2long.clear() - self.repeat = {} - - for option in self.option_table: - if len(option) == 3: - long, short, help = option - repeat = 0 - elif len(option) == 4: - long, short, help, repeat = option - else: - # the option table is part of the code, so simply - # assert that it is correct - raise ValueError("invalid option tuple: %r" % (option,)) - - # Type- and value-check the option names - if not isinstance(long, str) or len(long) < 2: - raise DistutilsGetoptError(("invalid long option '%s': " - "must be a string of length >= 2") % long) - - if (not ((short is None) or - (isinstance(short, str) and len(short) == 1))): - raise DistutilsGetoptError("invalid short option '%s': " - "must a single character or None" % short) - - self.repeat[long] = repeat - self.long_opts.append(long) - - if long[-1] == '=': # option takes an argument? - if short: short = short + ':' - long = long[0:-1] - self.takes_arg[long] = 1 - else: - # Is option is a "negative alias" for some other option (eg. - # "quiet" == "!verbose")? - alias_to = self.negative_alias.get(long) - if alias_to is not None: - if self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid negative alias '%s': " - "aliased option '%s' takes a value" - % (long, alias_to)) - - self.long_opts[-1] = long # XXX redundant?! - self.takes_arg[long] = 0 - - # If this is an alias option, make sure its "takes arg" flag is - # the same as the option it's aliased to. - alias_to = self.alias.get(long) - if alias_to is not None: - if self.takes_arg[long] != self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid alias '%s': inconsistent with " - "aliased option '%s' (one of them takes a value, " - "the other doesn't" - % (long, alias_to)) - - # Now enforce some bondage on the long option name, so we can - # later translate it to an attribute name on some object. Have - # to do this a bit late to make sure we've removed any trailing - # '='. - if not longopt_re.match(long): - raise DistutilsGetoptError( - "invalid long option name '%s' " - "(must be letters, numbers, hyphens only" % long) - - self.attr_name[long] = self.get_attr_name(long) - if short: - self.short_opts.append(short) - self.short2long[short[0]] = long - - def getopt(self, args=None, object=None): - """Parse command-line options in args. Store as attributes on object. - - If 'args' is None or not supplied, uses 'sys.argv[1:]'. If - 'object' is None or not supplied, creates a new OptionDummy - object, stores option values there, and returns a tuple (args, - object). If 'object' is supplied, it is modified in place and - 'getopt()' just returns 'args'; in both cases, the returned - 'args' is a modified copy of the passed-in 'args' list, which - is left untouched. - """ - if args is None: - args = sys.argv[1:] - if object is None: - object = OptionDummy() - created_object = True - else: - created_object = False - - self._grok_option_table() - - short_opts = ' '.join(self.short_opts) - try: - opts, args = getopt.getopt(args, short_opts, self.long_opts) - except getopt.error as msg: - raise DistutilsArgError(msg) - - for opt, val in opts: - if len(opt) == 2 and opt[0] == '-': # it's a short option - opt = self.short2long[opt[1]] - else: - assert len(opt) > 2 and opt[:2] == '--' - opt = opt[2:] - - alias = self.alias.get(opt) - if alias: - opt = alias - - if not self.takes_arg[opt]: # boolean option? - assert val == '', "boolean option can't have value" - alias = self.negative_alias.get(opt) - if alias: - opt = alias - val = 0 - else: - val = 1 - - attr = self.attr_name[opt] - # The only repeating option at the moment is 'verbose'. - # It has a negative option -q quiet, which should set verbose = 0. - if val and self.repeat.get(attr) is not None: - val = getattr(object, attr, 0) + 1 - setattr(object, attr, val) - self.option_order.append((opt, val)) - - # for opts - if created_object: - return args, object - else: - return args - - def get_option_order(self): - """Returns the list of (option, value) tuples processed by the - previous run of 'getopt()'. Raises RuntimeError if - 'getopt()' hasn't been called yet. - """ - if self.option_order is None: - raise RuntimeError("'getopt()' hasn't been called yet") - else: - return self.option_order - - def generate_help(self, header=None): - """Generate help text (a list of strings, one per suggested line of - output) from the option table for this FancyGetopt object. - """ - # Blithely assume the option table is good: probably wouldn't call - # 'generate_help()' unless you've already called 'getopt()'. - - # First pass: determine maximum length of long option names - max_opt = 0 - for option in self.option_table: - long = option[0] - short = option[1] - l = len(long) - if long[-1] == '=': - l = l - 1 - if short is not None: - l = l + 5 # " (-x)" where short == 'x' - if l > max_opt: - max_opt = l - - opt_width = max_opt + 2 + 2 + 2 # room for indent + dashes + gutter - - # Typical help block looks like this: - # --foo controls foonabulation - # Help block for longest option looks like this: - # --flimflam set the flim-flam level - # and with wrapped text: - # --flimflam set the flim-flam level (must be between - # 0 and 100, except on Tuesdays) - # Options with short names will have the short name shown (but - # it doesn't contribute to max_opt): - # --foo (-f) controls foonabulation - # If adding the short option would make the left column too wide, - # we push the explanation off to the next line - # --flimflam (-l) - # set the flim-flam level - # Important parameters: - # - 2 spaces before option block start lines - # - 2 dashes for each long option name - # - min. 2 spaces between option and explanation (gutter) - # - 5 characters (incl. space) for short option name - - # Now generate lines of help text. (If 80 columns were good enough - # for Jesus, then 78 columns are good enough for me!) - line_width = 78 - text_width = line_width - opt_width - big_indent = ' ' * opt_width - if header: - lines = [header] - else: - lines = ['Option summary:'] - - for option in self.option_table: - long, short, help = option[:3] - text = wrap_text(help, text_width) - if long[-1] == '=': - long = long[0:-1] - - # Case 1: no short option at all (makes life easy) - if short is None: - if text: - lines.append(" --%-*s %s" % (max_opt, long, text[0])) - else: - lines.append(" --%-*s " % (max_opt, long)) - - # Case 2: we have a short option, so we have to include it - # just after the long option - else: - opt_names = "%s (-%s)" % (long, short) - if text: - lines.append(" --%-*s %s" % - (max_opt, opt_names, text[0])) - else: - lines.append(" --%-*s" % opt_names) - - for l in text[1:]: - lines.append(big_indent + l) - return lines - - def print_help(self, header=None, file=None): - if file is None: - file = sys.stdout - for line in self.generate_help(header): - file.write(line + "\n") - - -def fancy_getopt(options, negative_opt, object, args): - parser = FancyGetopt(options) - parser.set_negative_aliases(negative_opt) - return parser.getopt(args, object) - - -WS_TRANS = {ord(_wschar) : ' ' for _wschar in string.whitespace} - -def wrap_text(text, width): - """wrap_text(text : string, width : int) -> [string] - - Split 'text' into multiple lines of no more than 'width' characters - each, and return the list of strings that results. - """ - if text is None: - return [] - if len(text) <= width: - return [text] - - text = text.expandtabs() - text = text.translate(WS_TRANS) - chunks = re.split(r'( +|-+)', text) - chunks = [ch for ch in chunks if ch] # ' - ' results in empty strings - lines = [] - - while chunks: - cur_line = [] # list of chunks (to-be-joined) - cur_len = 0 # length of current line - - while chunks: - l = len(chunks[0]) - if cur_len + l <= width: # can squeeze (at least) this chunk in - cur_line.append(chunks[0]) - del chunks[0] - cur_len = cur_len + l - else: # this line is full - # drop last chunk if all space - if cur_line and cur_line[-1][0] == ' ': - del cur_line[-1] - break - - if chunks: # any chunks left to process? - # if the current line is still empty, then we had a single - # chunk that's too big too fit on a line -- so we break - # down and break it up at the line width - if cur_len == 0: - cur_line.append(chunks[0][0:width]) - chunks[0] = chunks[0][width:] - - # all-whitespace chunks at the end of a line can be discarded - # (and we know from the re.split above that if a chunk has - # *any* whitespace, it is *all* whitespace) - if chunks[0][0] == ' ': - del chunks[0] - - # and store this line in the list-of-all-lines -- as a single - # string, of course! - lines.append(''.join(cur_line)) - - return lines - - -def translate_longopt(opt): - """Convert a long option name to a valid Python identifier by - changing "-" to "_". - """ - return opt.translate(longopt_xlate) - - -class OptionDummy: - """Dummy class just used as a place to hold command-line option - values as instance attributes.""" - - def __init__(self, options=[]): - """Create a new OptionDummy instance. The attributes listed in - 'options' will be initialized to None.""" - for opt in options: - setattr(self, opt, None) - - -if __name__ == "__main__": - text = """\ -Tra-la-la, supercalifragilisticexpialidocious. -How *do* you spell that odd word, anyways? -(Someone ask Mary -- she'll know [or she'll -say, "How should I know?"].)""" - - for w in (10, 20, 30, 40): - print("width: %d" % w) - print("\n".join(wrap_text(text, w))) - print() diff --git a/Lib/distutils/file_util.py b/Lib/distutils/file_util.py deleted file mode 100644 index b3fee35a6cc..00000000000 --- a/Lib/distutils/file_util.py +++ /dev/null @@ -1,238 +0,0 @@ -"""distutils.file_util - -Utility functions for operating on single files. -""" - -import os -from distutils.errors import DistutilsFileError -from distutils import log - -# for generating verbose output in 'copy_file()' -_copy_action = { None: 'copying', - 'hard': 'hard linking', - 'sym': 'symbolically linking' } - - -def _copy_file_contents(src, dst, buffer_size=16*1024): - """Copy the file 'src' to 'dst'; both must be filenames. Any error - opening either file, reading from 'src', or writing to 'dst', raises - DistutilsFileError. Data is read/written in chunks of 'buffer_size' - bytes (default 16k). No attempt is made to handle anything apart from - regular files. - """ - # Stolen from shutil module in the standard library, but with - # custom error-handling added. - fsrc = None - fdst = None - try: - try: - fsrc = open(src, 'rb') - except OSError as e: - raise DistutilsFileError("could not open '%s': %s" % (src, e.strerror)) - - if os.path.exists(dst): - try: - os.unlink(dst) - except OSError as e: - raise DistutilsFileError( - "could not delete '%s': %s" % (dst, e.strerror)) - - try: - fdst = open(dst, 'wb') - except OSError as e: - raise DistutilsFileError( - "could not create '%s': %s" % (dst, e.strerror)) - - while True: - try: - buf = fsrc.read(buffer_size) - except OSError as e: - raise DistutilsFileError( - "could not read from '%s': %s" % (src, e.strerror)) - - if not buf: - break - - try: - fdst.write(buf) - except OSError as e: - raise DistutilsFileError( - "could not write to '%s': %s" % (dst, e.strerror)) - finally: - if fdst: - fdst.close() - if fsrc: - fsrc.close() - -def copy_file(src, dst, preserve_mode=1, preserve_times=1, update=0, - link=None, verbose=1, dry_run=0): - """Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src' is - copied there with the same name; otherwise, it must be a filename. (If - the file exists, it will be ruthlessly clobbered.) If 'preserve_mode' - is true (the default), the file's mode (type and permission bits, or - whatever is analogous on the current platform) is copied. If - 'preserve_times' is true (the default), the last-modified and - last-access times are copied as well. If 'update' is true, 'src' will - only be copied if 'dst' does not exist, or if 'dst' does exist but is - older than 'src'. - - 'link' allows you to make hard links (os.link) or symbolic links - (os.symlink) instead of copying: set it to "hard" or "sym"; if it is - None (the default), files are copied. Don't set 'link' on systems that - don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. If hardlink fails, falls back to - _copy_file_contents(). - - Under Mac OS, uses the native file copy function in macostools; on - other systems, uses '_copy_file_contents()' to copy file contents. - - Return a tuple (dest_name, copied): 'dest_name' is the actual name of - the output file, and 'copied' is true if the file was copied (or would - have been copied, if 'dry_run' true). - """ - # XXX if the destination file already exists, we clobber it if - # copying, but blow up if linking. Hmmm. And I don't know what - # macostools.copyfile() does. Should definitely be consistent, and - # should probably blow up if destination exists and we would be - # changing it (ie. it's not already a hard/soft link to src OR - # (not update) and (src newer than dst). - - from distutils.dep_util import newer - from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE - - if not os.path.isfile(src): - raise DistutilsFileError( - "can't copy '%s': doesn't exist or not a regular file" % src) - - if os.path.isdir(dst): - dir = dst - dst = os.path.join(dst, os.path.basename(src)) - else: - dir = os.path.dirname(dst) - - if update and not newer(src, dst): - if verbose >= 1: - log.debug("not copying %s (output up-to-date)", src) - return (dst, 0) - - try: - action = _copy_action[link] - except KeyError: - raise ValueError("invalid value '%s' for 'link' argument" % link) - - if verbose >= 1: - if os.path.basename(dst) == os.path.basename(src): - log.info("%s %s -> %s", action, src, dir) - else: - log.info("%s %s -> %s", action, src, dst) - - if dry_run: - return (dst, 1) - - # If linking (hard or symbolic), use the appropriate system call - # (Unix only, of course, but that's the caller's responsibility) - elif link == 'hard': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - try: - os.link(src, dst) - return (dst, 1) - except OSError: - # If hard linking fails, fall back on copying file - # (some special filesystems don't support hard linking - # even under Unix, see issue #8876). - pass - elif link == 'sym': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.symlink(src, dst) - return (dst, 1) - - # Otherwise (non-Mac, not linking), copy the file contents and - # (optionally) copy the times and mode. - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) - - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) - - return (dst, 1) - - -# XXX I suspect this is Unix-specific -- need porting help! -def move_file (src, dst, - verbose=1, - dry_run=0): - - """Move a file 'src' to 'dst'. If 'dst' is a directory, the file will - be moved into it with the same name; otherwise, 'src' is just renamed - to 'dst'. Return the new full name of the file. - - Handles cross-device moves on Unix using 'copy_file()'. What about - other systems??? - """ - from os.path import exists, isfile, isdir, basename, dirname - import errno - - if verbose >= 1: - log.info("moving %s -> %s", src, dst) - - if dry_run: - return dst - - if not isfile(src): - raise DistutilsFileError("can't move '%s': not a regular file" % src) - - if isdir(dst): - dst = os.path.join(dst, basename(src)) - elif exists(dst): - raise DistutilsFileError( - "can't move '%s': destination '%s' already exists" % - (src, dst)) - - if not isdir(dirname(dst)): - raise DistutilsFileError( - "can't move '%s': destination '%s' not a valid path" % - (src, dst)) - - copy_it = False - try: - os.rename(src, dst) - except OSError as e: - (num, msg) = e.args - if num == errno.EXDEV: - copy_it = True - else: - raise DistutilsFileError( - "couldn't move '%s' to '%s': %s" % (src, dst, msg)) - - if copy_it: - copy_file(src, dst, verbose=verbose) - try: - os.unlink(src) - except OSError as e: - (num, msg) = e.args - try: - os.unlink(dst) - except OSError: - pass - raise DistutilsFileError( - "couldn't move '%s' to '%s' by copy/delete: " - "delete '%s' failed: %s" - % (src, dst, src, msg)) - return dst - - -def write_file (filename, contents): - """Create a file with the specified name and write 'contents' (a - sequence of strings without line terminators) to it. - """ - f = open(filename, "w") - try: - for line in contents: - f.write(line + "\n") - finally: - f.close() diff --git a/Lib/distutils/filelist.py b/Lib/distutils/filelist.py deleted file mode 100644 index c92d5fdba39..00000000000 --- a/Lib/distutils/filelist.py +++ /dev/null @@ -1,327 +0,0 @@ -"""distutils.filelist - -Provides the FileList class, used for poking about the filesystem -and building lists of files. -""" - -import os, re -import fnmatch -import functools -from distutils.util import convert_path -from distutils.errors import DistutilsTemplateError, DistutilsInternalError -from distutils import log - -class FileList: - """A list of files built by on exploring the filesystem and filtered by - applying various patterns to what we find there. - - Instance attributes: - dir - directory from which files will be taken -- only used if - 'allfiles' not supplied to constructor - files - list of filenames currently being built/filtered/manipulated - allfiles - complete list of files under consideration (ie. without any - filtering applied) - """ - - def __init__(self, warn=None, debug_print=None): - # ignore argument to FileList, but keep them for backwards - # compatibility - self.allfiles = None - self.files = [] - - def set_allfiles(self, allfiles): - self.allfiles = allfiles - - def findall(self, dir=os.curdir): - self.allfiles = findall(dir) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - # -- List-like methods --------------------------------------------- - - def append(self, item): - self.files.append(item) - - def extend(self, items): - self.files.extend(items) - - def sort(self): - # Not a strict lexical sort! - sortable_files = sorted(map(os.path.split, self.files)) - self.files = [] - for sort_tuple in sortable_files: - self.files.append(os.path.join(*sort_tuple)) - - - # -- Other miscellaneous utility methods --------------------------- - - def remove_duplicates(self): - # Assumes list has been sorted! - for i in range(len(self.files) - 1, 0, -1): - if self.files[i] == self.files[i - 1]: - del self.files[i] - - - # -- "File template" methods --------------------------------------- - - def _parse_template_line(self, line): - words = line.split() - action = words[0] - - patterns = dir = dir_pattern = None - - if action in ('include', 'exclude', - 'global-include', 'global-exclude'): - if len(words) < 2: - raise DistutilsTemplateError( - "'%s' expects ..." % action) - patterns = [convert_path(w) for w in words[1:]] - elif action in ('recursive-include', 'recursive-exclude'): - if len(words) < 3: - raise DistutilsTemplateError( - "'%s' expects ..." % action) - dir = convert_path(words[1]) - patterns = [convert_path(w) for w in words[2:]] - elif action in ('graft', 'prune'): - if len(words) != 2: - raise DistutilsTemplateError( - "'%s' expects a single " % action) - dir_pattern = convert_path(words[1]) - else: - raise DistutilsTemplateError("unknown action '%s'" % action) - - return (action, patterns, dir, dir_pattern) - - def process_template_line(self, line): - # Parse the line: split it up, make sure the right number of words - # is there, and return the relevant words. 'action' is always - # defined: it's the first word of the line. Which of the other - # three are defined depends on the action; it'll be either - # patterns, (dir and patterns), or (dir_pattern). - (action, patterns, dir, dir_pattern) = self._parse_template_line(line) - - # OK, now we know that the action is valid and we have the - # right number of words on the line for that action -- so we - # can proceed with minimal error-checking. - if action == 'include': - self.debug_print("include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=1): - log.warn("warning: no files found matching '%s'", - pattern) - - elif action == 'exclude': - self.debug_print("exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=1): - log.warn(("warning: no previously-included files " - "found matching '%s'"), pattern) - - elif action == 'global-include': - self.debug_print("global-include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=0): - log.warn(("warning: no files found matching '%s' " - "anywhere in distribution"), pattern) - - elif action == 'global-exclude': - self.debug_print("global-exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=0): - log.warn(("warning: no previously-included files matching " - "'%s' found anywhere in distribution"), - pattern) - - elif action == 'recursive-include': - self.debug_print("recursive-include %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.include_pattern(pattern, prefix=dir): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) - - elif action == 'recursive-exclude': - self.debug_print("recursive-exclude %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.exclude_pattern(pattern, prefix=dir): - log.warn(("warning: no previously-included files matching " - "'%s' found under directory '%s'"), - pattern, dir) - - elif action == 'graft': - self.debug_print("graft " + dir_pattern) - if not self.include_pattern(None, prefix=dir_pattern): - log.warn("warning: no directories found matching '%s'", - dir_pattern) - - elif action == 'prune': - self.debug_print("prune " + dir_pattern) - if not self.exclude_pattern(None, prefix=dir_pattern): - log.warn(("no previously-included directories found " - "matching '%s'"), dir_pattern) - else: - raise DistutilsInternalError( - "this cannot happen: invalid action '%s'" % action) - - - # -- Filtering/selection methods ----------------------------------- - - def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): - """Select strings (presumably filenames) from 'self.files' that - match 'pattern', a Unix-style wildcard (glob) pattern. Patterns - are not quite the same as implemented by the 'fnmatch' module: '*' - and '?' match non-special characters, where "special" is platform- - dependent: slash on Unix; colon, slash, and backslash on - DOS/Windows; and colon on Mac OS. - - If 'anchor' is true (the default), then the pattern match is more - stringent: "*.py" will match "foo.py" but not "foo/bar.py". If - 'anchor' is false, both of these will match. - - If 'prefix' is supplied, then only filenames starting with 'prefix' - (itself a pattern) and ending with 'pattern', with anything in between - them, will match. 'anchor' is ignored in this case. - - If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and - 'pattern' is assumed to be either a string containing a regex or a - regex object -- no translation is done, the regex is just compiled - and used as-is. - - Selected strings will be added to self.files. - - Return True if files are found, False otherwise. - """ - # XXX docstring lying about what the special chars are? - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("include_pattern: applying regex r'%s'" % - pattern_re.pattern) - - # delayed loading of allfiles list - if self.allfiles is None: - self.findall() - - for name in self.allfiles: - if pattern_re.search(name): - self.debug_print(" adding " + name) - self.files.append(name) - files_found = True - return files_found - - - def exclude_pattern (self, pattern, - anchor=1, prefix=None, is_regex=0): - """Remove strings (presumably filenames) from 'files' that match - 'pattern'. Other parameters are the same as for - 'include_pattern()', above. - The list 'self.files' is modified in place. - Return True if files are found, False otherwise. - """ - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("exclude_pattern: applying regex r'%s'" % - pattern_re.pattern) - for i in range(len(self.files)-1, -1, -1): - if pattern_re.search(self.files[i]): - self.debug_print(" removing " + self.files[i]) - del self.files[i] - files_found = True - return files_found - - -# ---------------------------------------------------------------------- -# Utility functions - -def _find_all_simple(path): - """ - Find all files under 'path' - """ - results = ( - os.path.join(base, file) - for base, dirs, files in os.walk(path, followlinks=True) - for file in files - ) - return filter(os.path.isfile, results) - - -def findall(dir=os.curdir): - """ - Find all files under 'dir' and return the list of full filenames. - Unless dir is '.', return full filenames with dir prepended. - """ - files = _find_all_simple(dir) - if dir == os.curdir: - make_rel = functools.partial(os.path.relpath, start=dir) - files = map(make_rel, files) - return list(files) - - -def glob_to_re(pattern): - """Translate a shell-like glob pattern to a regular expression; return - a string containing the regex. Differs from 'fnmatch.translate()' in - that '*' does not match "special characters" (which are - platform-specific). - """ - pattern_re = fnmatch.translate(pattern) - - # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which - # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, - # and by extension they shouldn't match such "special characters" under - # any OS. So change all non-escaped dots in the RE to match any - # character except the special characters (currently: just os.sep). - sep = os.sep - if os.sep == '\\': - # we're using a regex to manipulate a regex, so we need - # to escape the backslash twice - sep = r'\\\\' - escaped = r'\1[^%s]' % sep - pattern_re = re.sub(r'((?= self.threshold: - if args: - msg = msg % args - if level in (WARN, ERROR, FATAL): - stream = sys.stderr - else: - stream = sys.stdout - try: - stream.write('%s\n' % msg) - except UnicodeEncodeError: - # emulate backslashreplace error handler - encoding = stream.encoding - msg = msg.encode(encoding, "backslashreplace").decode(encoding) - stream.write('%s\n' % msg) - stream.flush() - - def log(self, level, msg, *args): - self._log(level, msg, args) - - def debug(self, msg, *args): - self._log(DEBUG, msg, args) - - def info(self, msg, *args): - self._log(INFO, msg, args) - - def warn(self, msg, *args): - self._log(WARN, msg, args) - - def error(self, msg, *args): - self._log(ERROR, msg, args) - - def fatal(self, msg, *args): - self._log(FATAL, msg, args) - -_global_log = Log() -log = _global_log.log -debug = _global_log.debug -info = _global_log.info -warn = _global_log.warn -error = _global_log.error -fatal = _global_log.fatal - -def set_threshold(level): - # return the old threshold for use from tests - old = _global_log.threshold - _global_log.threshold = level - return old - -def set_verbosity(v): - if v <= 0: - set_threshold(WARN) - elif v == 1: - set_threshold(INFO) - elif v >= 2: - set_threshold(DEBUG) diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py deleted file mode 100644 index 21191276227..00000000000 --- a/Lib/distutils/msvc9compiler.py +++ /dev/null @@ -1,791 +0,0 @@ -"""distutils.msvc9compiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio 2008. - -The module is compatible with VS 2005 and VS 2008. You can find legacy support -for older versions of VS in distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS2005 and VS 2008 by Christian Heimes - -import os -import subprocess -import sys -import re - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_preprocess_options, \ - gen_lib_options -from distutils import log -from distutils.util import get_platform - -import winreg - -RegOpenKeyEx = winreg.OpenKeyEx -RegEnumKey = winreg.EnumKey -RegEnumValue = winreg.EnumValue -RegError = winreg.error - -HKEYS = (winreg.HKEY_USERS, - winreg.HKEY_CURRENT_USER, - winreg.HKEY_LOCAL_MACHINE, - winreg.HKEY_CLASSES_ROOT) - -NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) -if NATIVE_WIN64: - # Visual C++ is a 32-bit application, so we need to look in - # the corresponding registry branch, if we're running a - # 64-bit Python on Win64 - VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" -else: - VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Microsoft\.NETFramework" - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is -# the param to cross-compile on x86 targeting amd64.) -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'amd64', - 'win-ia64' : 'ia64', -} - -class Reg: - """Helper class to read values from the registry - """ - - def get_value(cls, path, key): - for base in HKEYS: - d = cls.read_values(base, path) - if d and key in d: - return d[key] - raise KeyError(key) - get_value = classmethod(get_value) - - def read_keys(cls, base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - read_keys = classmethod(read_keys) - - def read_values(cls, base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) - i += 1 - return d - read_values = classmethod(read_values) - - def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - convert_mbcs = staticmethod(convert_mbcs) - -class MacroExpander: - - def __init__(self, version): - self.macros = {} - self.vsbase = VS_BASE % version - self.load_macros(version) - - def set_macro(self, macro, path, key): - self.macros["$(%s)" % macro] = Reg.get_value(path, key) - - def load_macros(self, version): - self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") - self.set_macro("FrameworkDir", NET_BASE, "installroot") - try: - if version >= 8.0: - self.set_macro("FrameworkSDKDir", NET_BASE, - "sdkinstallrootv2.0") - else: - raise KeyError("sdkinstallrootv2.0") - except KeyError: - raise DistutilsPlatformError( - """Python was built with Visual Studio 2008; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2008 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - if version >= 9.0: - self.set_macro("FrameworkVersion", self.vsbase, "clr version") - self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") - else: - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = Reg.get_value(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - -def removeDuplicates(variable): - """Remove duplicate values of an environment variable. - """ - oldList = variable.split(os.pathsep) - newList = [] - for i in oldList: - if i not in newList: - newList.append(i) - newVariable = os.pathsep.join(newList) - return newVariable - -def find_vcvarsall(version): - """Find the vcvarsall.bat file - - At first it tries to find the productdir of VS 2008 in the registry. If - that fails it falls back to the VS90COMNTOOLS env var. - """ - vsbase = VS_BASE % version - try: - productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, - "productdir") - except KeyError: - log.debug("Unable to find productdir in registry") - productdir = None - - if not productdir or not os.path.isdir(productdir): - toolskey = "VS%0.f0COMNTOOLS" % version - toolsdir = os.environ.get(toolskey, None) - - if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) - if not os.path.isdir(productdir): - log.debug("%s is not a valid directory" % productdir) - return None - else: - log.debug("Env var %s is not set or invalid" % toolskey) - if not productdir: - log.debug("No productdir found") - return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): - return vcvarsall - log.debug("Unable to find vcvarsall.bat") - return None - -def query_vcvarsall(version, arch="x86"): - """Launch vcvarsall.bat and read the settings from its environment - """ - vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) - result = {} - - if vcvarsall is None: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) - popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - stdout, stderr = popen.communicate() - if popen.wait() != 0: - raise DistutilsPlatformError(stderr.decode("mbcs")) - - stdout = stdout.decode("mbcs") - for line in stdout.split("\n"): - line = Reg.convert_mbcs(line) - if '=' not in line: - continue - line = line.strip() - key, value = line.split('=', 1) - key = key.lower() - if key in interesting: - if value.endswith(os.pathsep): - value = value[:-1] - result[key] = removeDuplicates(value) - - finally: - popen.stdout.close() - popen.stderr.close() - - if len(result) != len(interesting): - raise ValueError(str(list(result.keys()))) - - return result - -# More globals -VERSION = get_build_version() -if VERSION < 8.0: - raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) -# MACROS = MacroExpander(VERSION) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = VERSION - self.__root = r"Software\Microsoft\VisualStudio" - # self.__macros = MACROS - self.__paths = [] - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.__arch = None # deprecated name - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - ok_plats = 'win32', 'win-amd64', 'win-ia64' - if plat_name not in ok_plats: - raise DistutilsPlatformError("--plat-name must be one of %s" % - (ok_plats,)) - - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; - # to cross compile, you use 'x86_amd64'. - # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross - # compile use 'x86' (ie, it runs the x86 compiler directly) - # No idea how itanium handles this, if at all. - if plat_name == get_platform() or plat_name == 'win32': - # native build or cross-compile to win32 - plat_spec = PLAT_TO_VCVARS[plat_name] - else: - # cross compile from win32 -> some 64bit - plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ - PLAT_TO_VCVARS[plat_name] - - vc_env = query_vcvarsall(VERSION, plat_spec) - - self.__paths = vc_env['path'].split(os.pathsep) - os.environ['lib'] = vc_env['lib'] - os.environ['include'] = vc_env['include'] - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - #self.set_path_env_var('lib') - #self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "x86": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - self.manifest_setup_ldargs(output_filename, build_temp, ld_args) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - # embed the manifest - # XXX - this is somewhat fragile - if mt.exe fails, distutils - # will still consider the DLL up-to-date, but it will not have a - # manifest. Maybe we should link to a temp file? OTOH, that - # implies a build environment error that shouldn't go undetected. - mfinfo = self.manifest_get_embed_info(target_desc, ld_args) - if mfinfo is not None: - mffilename, mfid = mfinfo - out_arg = '-outputresource:%s;%s' % (output_filename, mfid) - try: - self.spawn(['mt.exe', '-nologo', '-manifest', - mffilename, out_arg]) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): - # If we need a manifest at all, an embedded manifest is recommended. - # See MSDN article titled - # "How to: Embed a Manifest Inside a C/C++ Application" - # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) - # Ask the linker to generate the manifest in the temp dir, so - # we can check it, and possibly embed it, later. - temp_manifest = os.path.join( - build_temp, - os.path.basename(output_filename) + ".manifest") - ld_args.append('/MANIFESTFILE:' + temp_manifest) - - def manifest_get_embed_info(self, target_desc, ld_args): - # If a manifest should be embedded, return a tuple of - # (manifest_filename, resource_id). Returns None if no manifest - # should be embedded. See http://bugs.python.org/issue7833 for why - # we want to avoid any manifest for extension modules if we can) - for arg in ld_args: - if arg.startswith("/MANIFESTFILE:"): - temp_manifest = arg.split(":", 1)[1] - break - else: - # no /MANIFESTFILE so nothing to do. - return None - if target_desc == CCompiler.EXECUTABLE: - # by default, executables always get the manifest with the - # CRT referenced. - mfid = 1 - else: - # Extension modules try and avoid any manifest if possible. - mfid = 2 - temp_manifest = self._remove_visual_c_ref(temp_manifest) - if temp_manifest is None: - return None - return temp_manifest, mfid - - def _remove_visual_c_ref(self, manifest_file): - try: - # Remove references to the Visual C runtime, so they will - # fall through to the Visual C dependency of Python.exe. - # This way, when installed for a restricted user (e.g. - # runtimes are not in WinSxS folder, but in Python's own - # folder), the runtimes do not need to be in every folder - # with .pyd's. - # Returns either the filename of the modified manifest or - # None if no manifest should be embedded. - manifest_f = open(manifest_file) - try: - manifest_buf = manifest_f.read() - finally: - manifest_f.close() - pattern = re.compile( - r"""|)""", - re.DOTALL) - manifest_buf = re.sub(pattern, "", manifest_buf) - pattern = r"\s*" - manifest_buf = re.sub(pattern, "", manifest_buf) - # Now see if any other assemblies are referenced - if not, we - # don't want a manifest embedded. - pattern = re.compile( - r"""|)""", re.DOTALL) - if re.search(pattern, manifest_buf) is None: - return None - - manifest_f = open(manifest_file, 'w') - try: - manifest_f.write(manifest_buf) - return manifest_file - finally: - manifest_f.close() - except OSError: - pass - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe diff --git a/Lib/distutils/msvccompiler.py b/Lib/distutils/msvccompiler.py deleted file mode 100644 index 1048cd41593..00000000000 --- a/Lib/distutils/msvccompiler.py +++ /dev/null @@ -1,643 +0,0 @@ -"""distutils.msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) - -import sys, os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils import log - -_can_read_reg = False -try: - import winreg - - _can_read_reg = True - hkey_mod = winreg - - RegOpenKeyEx = winreg.OpenKeyEx - RegEnumKey = winreg.EnumKey - RegEnumValue = winreg.EnumValue - RegError = winreg.error - -except ImportError: - try: - import win32api - import win32con - _can_read_reg = True - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) - -def read_keys(base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - -def read_values(base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i += 1 - return d - -def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - -class MacroExpander: - def __init__(self, version): - self.macros = {} - self.load_macros(version) - - def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break - - def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") - try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") - else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError as exc: # - raise DistutilsPlatformError( - """Python was built with Visual Studio 2003; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return "Intel" - j = sys.version.find(")", i) - return sys.version[i+len(prefix):j] - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - - self.initialized = False - - def initialize(self): - self.__paths = [] - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - self.__paths = self.get_msvc_paths("path") - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - os.path.dirname(objects[0]), - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return self.__macros.sub(d[path]).split(";") - else: - return d[path].split(";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = ';'.join(p) - - -if get_build_version() >= 8.0: - log.debug("Importing new compiler from distutils.msvc9compiler") - OldMSVCCompiler = MSVCCompiler - from distutils.msvc9compiler import MSVCCompiler - # get_build_architecture not really relevant now we support cross-compile - from distutils.msvc9compiler import MacroExpander diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py deleted file mode 100644 index 53876880932..00000000000 --- a/Lib/distutils/spawn.py +++ /dev/null @@ -1,192 +0,0 @@ -"""distutils.spawn - -Provides the 'spawn()' function, a front-end to various platform- -specific functions for launching another program in a sub-process. -Also provides the 'find_executable()' to search the path for a given -executable name. -""" - -import sys -import os - -from distutils.errors import DistutilsPlatformError, DistutilsExecError -from distutils.debug import DEBUG -from distutils import log - -def spawn(cmd, search_path=1, verbose=0, dry_run=0): - """Run another program, specified as a command list 'cmd', in a new process. - - 'cmd' is just the argument list for the new process, ie. - cmd[0] is the program to run and cmd[1:] are the rest of its arguments. - There is no way to run a program with a name different from that of its - executable. - - If 'search_path' is true (the default), the system's executable - search path will be used to find the program; otherwise, cmd[0] - must be the exact path to the executable. If 'dry_run' is true, - the command will not actually be run. - - Raise DistutilsExecError if running the program fails in any way; just - return on success. - """ - # cmd is documented as a list, but just in case some code passes a tuple - # in, protect our %-formatting code against horrible death - cmd = list(cmd) - if os.name == 'posix': - _spawn_posix(cmd, search_path, dry_run=dry_run) - elif os.name == 'nt': - _spawn_nt(cmd, search_path, dry_run=dry_run) - else: - raise DistutilsPlatformError( - "don't know how to spawn programs on platform '%s'" % os.name) - -def _nt_quote_args(args): - """Quote command-line arguments for DOS/Windows conventions. - - Just wraps every argument which contains blanks in double quotes, and - returns a new argument list. - """ - # XXX this doesn't seem very robust to me -- but if the Windows guys - # say it'll work, I guess I'll have to accept it. (What if an arg - # contains quotes? What other magic characters, other than spaces, - # have to be escaped? Is there an escaping mechanism other than - # quoting?) - for i, arg in enumerate(args): - if ' ' in arg: - args[i] = '"%s"' % arg - return args - -def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0): - executable = cmd[0] - cmd = _nt_quote_args(cmd) - if search_path: - # either we find one or it stays the same - executable = find_executable(executable) or executable - log.info(' '.join([executable] + cmd[1:])) - if not dry_run: - # spawn for NT requires a full path to the .exe - try: - rc = os.spawnv(os.P_WAIT, executable, cmd) - except OSError as exc: - # this seems to happen when the command isn't found - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if rc != 0: - # and this reflects the command running but failing - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" % (cmd, rc)) - -if sys.platform == 'darwin': - from distutils import sysconfig - _cfg_target = None - _cfg_target_split = None - -def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0): - log.info(' '.join(cmd)) - if dry_run: - return - executable = cmd[0] - exec_fn = search_path and os.execvp or os.execv - env = None - if sys.platform == 'darwin': - global _cfg_target, _cfg_target_split - if _cfg_target is None: - _cfg_target = sysconfig.get_config_var( - 'MACOSX_DEPLOYMENT_TARGET') or '' - if _cfg_target: - _cfg_target_split = [int(x) for x in _cfg_target.split('.')] - if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' - % (cur_target, _cfg_target)) - raise DistutilsPlatformError(my_msg) - env = dict(os.environ, - MACOSX_DEPLOYMENT_TARGET=cur_target) - exec_fn = search_path and os.execvpe or os.execve - pid = os.fork() - if pid == 0: # in the child - try: - if env is None: - exec_fn(executable, cmd) - else: - exec_fn(executable, cmd, env) - except OSError as e: - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r: %s\n" - % (cmd, e.strerror)) - os._exit(1) - - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r for unknown reasons" % cmd) - os._exit(1) - else: # in the parent - # Loop until the child either exits or is terminated by a signal - # (ie. keep waiting if it's merely stopped) - while True: - try: - pid, status = os.waitpid(pid, 0) - except OSError as exc: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if os.WIFSIGNALED(status): - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r terminated by signal %d" - % (cmd, os.WTERMSIG(status))) - elif os.WIFEXITED(status): - exit_status = os.WEXITSTATUS(status) - if exit_status == 0: - return # hey, it succeeded! - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" - % (cmd, exit_status)) - elif os.WIFSTOPPED(status): - continue - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "unknown error executing %r: termination status %d" - % (cmd, status)) - -def find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - """ - if path is None: - path = os.environ.get('PATH', os.defpath) - - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - - if (sys.platform == 'win32') and (ext != '.exe'): - executable = executable + '.exe' - - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: - return executable diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py deleted file mode 100644 index 3a5984f5c01..00000000000 --- a/Lib/distutils/sysconfig.py +++ /dev/null @@ -1,556 +0,0 @@ -"""Provide access to Python's configuration information. The specific -configuration variables available depend heavily on the platform and -configuration. The values may be retrieved using -get_config_var(name), and the list of variables is available via -get_config_vars().keys(). Additional convenience functions are also -available. - -Written by: Fred L. Drake, Jr. -Email: -""" - -import _imp -import os -import re -import sys - -from .errors import DistutilsPlatformError - -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - -# Path to the base directory of the project. On Windows the binary may -# live in project/PCbuild/win32 or project/PCbuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - if sys.executable: - project_base = os.path.dirname(os.path.abspath(sys.executable)) - else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - project_base = os.getcwd() - - -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(PREFIX, "PCbuild"))): - return PREFIX - return d - project_base = _fix_pcbuild(project_base) - _sys_home = _fix_pcbuild(_sys_home) - -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) - -python_build = _python_build() - - -# Calculate the build qualifier flags if they are defined. Adding the flags -# to the include and lib directories only makes sense for an installation, not -# an in-source build. -build_flags = '' -try: - if not python_build: - build_flags = sys.abiflags -except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass - -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. - """ - return '%d.%d' % sys.version_info[:2] - - -def get_python_inc(plat_specific=0, prefix=None): - """Return the directory containing installed Python header files. - - If 'plat_specific' is false (the default), this is the path to the - non-platform-specific header files, i.e. Python.h and so on; - otherwise, this is the path to platform-specific header files - (namely pyconfig.h). - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if os.name == "posix": - if python_build: - # Assume the executable is in the build directory. The - # pyconfig.h file should be in the same directory. Since - # the build directory may not be the source directory, we - # must use "srcdir" from the makefile to find the "Include" - # directory. - if plat_specific: - return _sys_home or project_base - else: - incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) - python_dir = 'python' + get_python_version() + build_flags - return os.path.join(prefix, "include", python_dir) - elif os.name == "nt": - if python_build: - # Include both the include and PC dir to ensure we can find - # pyconfig.h - return (os.path.join(prefix, "include") + os.path.pathsep + - os.path.join(prefix, "PC")) - return os.path.join(prefix, "include") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its C header files " - "on platform '%s'" % os.name) - - -def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - """Return the directory containing the Python library (standard or - site additions). - - If 'plat_specific' is true, return the directory containing - platform-specific modules, i.e. any module from a non-pure-Python - module distribution; otherwise, return the platform-shared library - directory. If 'standard_lib' is true, return the directory - containing standard Python library modules; otherwise, return the - directory for site-specific modules. - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - if standard_lib: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - else: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - if plat_specific or standard_lib: - # Platform-specific modules (any module from a non-pure-Python - # module distribution) or standard Python library modules. - libdir = sys.platlibdir - else: - # Pure Python - libdir = "lib" - libpython = os.path.join(prefix, libdir, - # XXX RUSTPYTHON: changed from python->rustpython - "rustpython" + get_python_version()) - if standard_lib: - return libpython - else: - return os.path.join(libpython, "site-packages") - elif os.name == "nt": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - return os.path.join(prefix, "Lib", "site-packages") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its library " - "on platform '%s'" % os.name) - - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - - (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - - if 'CC' in os.environ: - newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - ldshared = newcc + ldshared[len(cc):] - cc = newcc - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = cflags + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - if 'AR' in os.environ: - ar = os.environ['AR'] - if 'ARFLAGS' in os.environ: - archiver = ar + ' ' + os.environ['ARFLAGS'] - else: - archiver = ar + ' ' + ar_flags - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc, - archiver=archiver) - - compiler.shared_lib_extension = shlib_suffix - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") - else: - inc_dir = _sys_home or project_base - else: - inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(_sys_home or project_base, "Makefile") - lib_dir = get_python_lib(plat_specific=0, standard_lib=1) - config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch - return os.path.join(lib_dir, config_file, 'Makefile') - - -def parse_config_h(fp, g=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g - - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") - -def parse_makefile(fn, g=None): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - from distutils.text_file import TextFile - fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape") - - if g is None: - g = {} - done = {} - notdone = {} - - while True: - line = fp.readline() - if line is None: # eof - break - m = _variable_rx.match(line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - # do variable interpolation here - while notdone: - for name in list(notdone): - value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) - if m: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if name.startswith('PY_') and name[3:] in renamed_variables: - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - else: - done[n] = item = "" - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - del notdone[name] - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - else: - # bogus variable reference; just drop it since we can't deal - del notdone[name] - - fp.close() - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - g.update(done) - return g - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - global _config_vars - _config_vars = {} - _config_vars.update(build_time_vars) - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() - else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - - # Convert srcdir into an absolute path if it appears necessary. - # Normally it is relative to the build directory. However, during - # testing, for example, we might be running a non-installed python - # from a different directory. - if python_build and os.name == "posix": - base = project_base - if (not os.path.isabs(_config_vars['srcdir']) and - base != os.getcwd()): - # srcdir is relative and we are not in the same directory - # as the executable. Assume executable is in the build - # directory and make srcdir absolute. - srcdir = os.path.join(base, _config_vars['srcdir']) - _config_vars['srcdir'] = os.path.normpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals - else: - return _config_vars - -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) - """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) - return get_config_vars().get(name) diff --git a/Lib/distutils/text_file.py b/Lib/distutils/text_file.py deleted file mode 100644 index 93abad38f43..00000000000 --- a/Lib/distutils/text_file.py +++ /dev/null @@ -1,286 +0,0 @@ -"""text_file - -provides the TextFile class, which gives an interface to text files -that (optionally) takes care of stripping comments, ignoring blank -lines, and joining lines with backslashes.""" - -import sys, io - - -class TextFile: - """Provides a file-like object that takes care of all the things you - commonly want to do when processing a text file that has some - line-by-line syntax: strip comments (as long as "#" is your - comment character), skip blank lines, join adjacent lines by - escaping the newline (ie. backslash at end of line), strip - leading and/or trailing whitespace. All of these are optional - and independently controllable. - - Provides a 'warn()' method so you can generate warning messages that - report physical line number, even if the logical line in question - spans multiple physical lines. Also provides 'unreadline()' for - implementing line-at-a-time lookahead. - - Constructor is called as: - - TextFile (filename=None, file=None, **options) - - It bombs (RuntimeError) if both 'filename' and 'file' are None; - 'filename' should be a string, and 'file' a file object (or - something that provides 'readline()' and 'close()' methods). It is - recommended that you supply at least 'filename', so that TextFile - can include it in warning messages. If 'file' is not supplied, - TextFile creates its own using 'io.open()'. - - The options are all boolean, and affect the value returned by - 'readline()': - strip_comments [default: true] - strip from "#" to end-of-line, as well as any whitespace - leading up to the "#" -- unless it is escaped by a backslash - lstrip_ws [default: false] - strip leading whitespace from each line before returning it - rstrip_ws [default: true] - strip trailing whitespace (including line terminator!) from - each line before returning it - skip_blanks [default: true} - skip lines that are empty *after* stripping comments and - whitespace. (If both lstrip_ws and rstrip_ws are false, - then some lines may consist of solely whitespace: these will - *not* be skipped, even if 'skip_blanks' is true.) - join_lines [default: false] - if a backslash is the last non-newline character on a line - after stripping comments and whitespace, join the following line - to it to form one "logical line"; if N consecutive lines end - with a backslash, then N+1 physical lines will be joined to - form one logical line. - collapse_join [default: false] - strip leading whitespace from lines that are joined to their - predecessor; only matters if (join_lines and not lstrip_ws) - errors [default: 'strict'] - error handler used to decode the file content - - Note that since 'rstrip_ws' can strip the trailing newline, the - semantics of 'readline()' must differ from those of the builtin file - object's 'readline()' method! In particular, 'readline()' returns - None for end-of-file: an empty string might just be a blank line (or - an all-whitespace line), if 'rstrip_ws' is true but 'skip_blanks' is - not.""" - - default_options = { 'strip_comments': 1, - 'skip_blanks': 1, - 'lstrip_ws': 0, - 'rstrip_ws': 1, - 'join_lines': 0, - 'collapse_join': 0, - 'errors': 'strict', - } - - def __init__(self, filename=None, file=None, **options): - """Construct a new TextFile object. At least one of 'filename' - (a string) and 'file' (a file-like object) must be supplied. - They keyword argument options are described above and affect - the values returned by 'readline()'.""" - if filename is None and file is None: - raise RuntimeError("you must supply either or both of 'filename' and 'file'") - - # set values for all options -- either from client option hash - # or fallback to default_options - for opt in self.default_options.keys(): - if opt in options: - setattr(self, opt, options[opt]) - else: - setattr(self, opt, self.default_options[opt]) - - # sanity check client option hash - for opt in options.keys(): - if opt not in self.default_options: - raise KeyError("invalid TextFile option '%s'" % opt) - - if file is None: - self.open(filename) - else: - self.filename = filename - self.file = file - self.current_line = 0 # assuming that file is at BOF! - - # 'linebuf' is a stack of lines that will be emptied before we - # actually read from the file; it's only populated by an - # 'unreadline()' operation - self.linebuf = [] - - def open(self, filename): - """Open a new file named 'filename'. This overrides both the - 'filename' and 'file' arguments to the constructor.""" - self.filename = filename - self.file = io.open(self.filename, 'r', errors=self.errors) - self.current_line = 0 - - def close(self): - """Close the current file and forget everything we know about it - (filename, current line number).""" - file = self.file - self.file = None - self.filename = None - self.current_line = None - file.close() - - def gen_error(self, msg, line=None): - outmsg = [] - if line is None: - line = self.current_line - outmsg.append(self.filename + ", ") - if isinstance(line, (list, tuple)): - outmsg.append("lines %d-%d: " % tuple(line)) - else: - outmsg.append("line %d: " % line) - outmsg.append(str(msg)) - return "".join(outmsg) - - def error(self, msg, line=None): - raise ValueError("error: " + self.gen_error(msg, line)) - - def warn(self, msg, line=None): - """Print (to stderr) a warning message tied to the current logical - line in the current file. If the current logical line in the - file spans multiple physical lines, the warning refers to the - whole range, eg. "lines 3-5". If 'line' supplied, it overrides - the current line number; it may be a list or tuple to indicate a - range of physical lines, or an integer for a single physical - line.""" - sys.stderr.write("warning: " + self.gen_error(msg, line) + "\n") - - def readline(self): - """Read and return a single logical line from the current file (or - from an internal buffer if lines have previously been "unread" - with 'unreadline()'). If the 'join_lines' option is true, this - may involve reading multiple physical lines concatenated into a - single string. Updates the current line number, so calling - 'warn()' after 'readline()' emits a warning about the physical - line(s) just read. Returns None on end-of-file, since the empty - string can occur if 'rstrip_ws' is true but 'strip_blanks' is - not.""" - # If any "unread" lines waiting in 'linebuf', return the top - # one. (We don't actually buffer read-ahead data -- lines only - # get put in 'linebuf' if the client explicitly does an - # 'unreadline()'. - if self.linebuf: - line = self.linebuf[-1] - del self.linebuf[-1] - return line - - buildup_line = '' - - while True: - # read the line, make it None if EOF - line = self.file.readline() - if line == '': - line = None - - if self.strip_comments and line: - - # Look for the first "#" in the line. If none, never - # mind. If we find one and it's the first character, or - # is not preceded by "\", then it starts a comment -- - # strip the comment, strip whitespace before it, and - # carry on. Otherwise, it's just an escaped "#", so - # unescape it (and any other escaped "#"'s that might be - # lurking in there) and otherwise leave the line alone. - - pos = line.find("#") - if pos == -1: # no "#" -- no comments - pass - - # It's definitely a comment -- either "#" is the first - # character, or it's elsewhere and unescaped. - elif pos == 0 or line[pos-1] != "\\": - # Have to preserve the trailing newline, because it's - # the job of a later step (rstrip_ws) to remove it -- - # and if rstrip_ws is false, we'd better preserve it! - # (NB. this means that if the final line is all comment - # and has no trailing newline, we will think that it's - # EOF; I think that's OK.) - eol = (line[-1] == '\n') and '\n' or '' - line = line[0:pos] + eol - - # If all that's left is whitespace, then skip line - # *now*, before we try to join it to 'buildup_line' -- - # that way constructs like - # hello \\ - # # comment that should be ignored - # there - # result in "hello there". - if line.strip() == "": - continue - else: # it's an escaped "#" - line = line.replace("\\#", "#") - - # did previous line end with a backslash? then accumulate - if self.join_lines and buildup_line: - # oops: end of file - if line is None: - self.warn("continuation line immediately precedes " - "end-of-file") - return buildup_line - - if self.collapse_join: - line = line.lstrip() - line = buildup_line + line - - # careful: pay attention to line number when incrementing it - if isinstance(self.current_line, list): - self.current_line[1] = self.current_line[1] + 1 - else: - self.current_line = [self.current_line, - self.current_line + 1] - # just an ordinary line, read it as usual - else: - if line is None: # eof - return None - - # still have to be careful about incrementing the line number! - if isinstance(self.current_line, list): - self.current_line = self.current_line[1] + 1 - else: - self.current_line = self.current_line + 1 - - # strip whitespace however the client wants (leading and - # trailing, or one or the other, or neither) - if self.lstrip_ws and self.rstrip_ws: - line = line.strip() - elif self.lstrip_ws: - line = line.lstrip() - elif self.rstrip_ws: - line = line.rstrip() - - # blank line (whether we rstrip'ed or not)? skip to next line - # if appropriate - if (line == '' or line == '\n') and self.skip_blanks: - continue - - if self.join_lines: - if line[-1] == '\\': - buildup_line = line[:-1] - continue - - if line[-2:] == '\\\n': - buildup_line = line[0:-2] + '\n' - continue - - # well, I guess there's some actual content there: return it - return line - - def readlines(self): - """Read and return the list of all logical lines remaining in the - current file.""" - lines = [] - while True: - line = self.readline() - if line is None: - return lines - lines.append(line) - - def unreadline(self, line): - """Push 'line' (a string) onto an internal buffer that will be - checked by future 'readline()' calls. Handy for implementing - a parser with line-at-a-time lookahead.""" - self.linebuf.append(line) diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py deleted file mode 100644 index 4524417e668..00000000000 --- a/Lib/distutils/unixccompiler.py +++ /dev/null @@ -1,333 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - try: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - # filter out standard library paths, which are not explicitely needed - # for linking - system_libdirs = ['/lib', '/lib64', '/usr/lib', '/usr/lib64'] - multiarch = sysconfig.get_config_var("MULTIARCH") - if multiarch: - system_libdirs.extend(['/lib/%s' % multiarch, '/usr/lib/%s' % multiarch]) - library_dirs = [dir for dir in library_dirs - if not dir in system_libdirs] - runtime_library_dirs = [dir for dir in runtime_library_dirs - if not dir in system_libdirs] - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - if target_lang == "c++" and self.compiler_cxx: - # skip over environment variable settings if /usr/bin/env - # is used to set up the linker's environment. - # This is needed on OSX. Note: this assumes that the - # normal and C++ compiler have the same environment - # settings. - i = 0 - if os.path.basename(linker[0]) == "env": - i = 1 - while '=' in linker[i]: - i += 1 - linker[i] = self.compiler_cxx[i] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - elif sys.platform[:7] == "irix646" or sys.platform[:6] == "osf1V5": - return ["-rpath", dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s+(\S+)', cflags) - if m is None: - sysroot = '/' - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py deleted file mode 100644 index fdcf6fabae2..00000000000 --- a/Lib/distutils/util.py +++ /dev/null @@ -1,557 +0,0 @@ -"""distutils.util - -Miscellaneous utility functions -- anything that doesn't fit into -one of the other *util.py modules. -""" - -import os -import re -import importlib.util -import string -import sys -from distutils.errors import DistutilsPlatformError -from distutils.dep_util import newer -from distutils.spawn import spawn -from distutils import log -from distutils.errors import DistutilsByteCompileError - -def get_platform (): - """Return a string that identifies the current platform. This is used - mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name - and version and the architecture (as supplied by 'os.uname()'), - although the exact information included depends on the OS; eg. for IRIX - the architecture isn't particularly important (IRIX only runs on SGI - hardware), but for Linux the kernel version isn't particularly - important. - - Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - irix-5.3 - irix64-6.2 - - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win-ia64 (64bit Windows on Itanium) - win32 (all others - specifically, sys.platform is returned) - - For other non-POSIX platforms, currently just returns 'sys.platform'. - """ - if os.name == 'nt': - # sniff sys.version for architecture. - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return sys.platform - j = sys.version.find(")", i) - look = sys.version[i+len(prefix):j].lower() - if look == 'amd64': - return 'win-amd64' - if look == 'itanium': - return 'win-ia64' - return sys.platform - - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - if os.name != "posix" or not hasattr(os, 'uname'): - # XXX what about the architecture? NT is Intel or Alpha, - # Mac OS is M68k or PPC, etc. - return sys.platform - - # Try to distinguish various flavours of Unix - - (osname, host, release, version, machine) = os.uname() - - # Convert the OS name to lowercase, remove '/' characters - # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return "%s-%s" % (osname, machine) - elif osname[:5] == "sunos": - if release[0] >= "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = "%d.%s" % (int(release[0]) - 3, release[2:]) - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += ".%s" % bitness[sys.maxsize] - # fall through to standard osname-release-machine representation - elif osname[:4] == "irix": # could be "irix64"! - return "%s-%s" % (osname, release) - elif osname[:3] == "aix": - return "%s-%s.%s" % (osname, version, release) - elif osname[:6] == "cygwin": - osname = "cygwin" - rel_re = re.compile (r'[\d.]+', re.ASCII) - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - import _osx_support, distutils.sysconfig - osname, release, machine = _osx_support.get_platform_osx( - distutils.sysconfig.get_config_vars(), - osname, release, machine) - - return "%s-%s-%s" % (osname, release, machine) - -# get_platform () - - -def convert_path (pathname): - """Return 'pathname' as a name that will work on the native filesystem, - i.e. split it on '/' and put it back together again using the current - directory separator. Needed because filenames in the setup script are - always supplied in Unix style, and have to be converted to the local - convention before we can actually use them in the filesystem. Raises - ValueError on non-Unix-ish systems if 'pathname' either starts or - ends with a slash. - """ - if os.sep == '/': - return pathname - if not pathname: - return pathname - if pathname[0] == '/': - raise ValueError("path '%s' cannot be absolute" % pathname) - if pathname[-1] == '/': - raise ValueError("path '%s' cannot end with '/'" % pathname) - - paths = pathname.split('/') - while '.' in paths: - paths.remove('.') - if not paths: - return os.curdir - return os.path.join(*paths) - -# convert_path () - - -def change_root (new_root, pathname): - """Return 'pathname' with 'new_root' prepended. If 'pathname' is - relative, this is equivalent to "os.path.join(new_root,pathname)". - Otherwise, it requires making 'pathname' relative and then joining the - two, which is tricky on DOS/Windows and Mac OS. - """ - if os.name == 'posix': - if not os.path.isabs(pathname): - return os.path.join(new_root, pathname) - else: - return os.path.join(new_root, pathname[1:]) - - elif os.name == 'nt': - (drive, path) = os.path.splitdrive(pathname) - if path[0] == '\\': - path = path[1:] - return os.path.join(new_root, path) - - else: - raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) - - -_environ_checked = 0 -def check_environ (): - """Ensure that 'os.environ' has all the environment variables we - guarantee that users can use in config files, command-line options, - etc. Currently this includes: - HOME - user's home directory (Unix only) - PLAT - description of the current platform, including hardware - and OS (see 'get_platform()') - """ - global _environ_checked - if _environ_checked: - return - - if os.name == 'posix' and 'HOME' not in os.environ: - import pwd - os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] - - if 'PLAT' not in os.environ: - os.environ['PLAT'] = get_platform() - - _environ_checked = 1 - - -def subst_vars (s, local_vars): - """Perform shell/Perl-style variable substitution on 'string'. Every - occurrence of '$' followed by a name is considered a variable, and - variable is substituted by the value found in the 'local_vars' - dictionary, or in 'os.environ' if it's not in 'local_vars'. - 'os.environ' is first checked/augmented to guarantee that it contains - certain values: see 'check_environ()'. Raise ValueError for any - variables not found in either 'local_vars' or 'os.environ'. - """ - check_environ() - def _subst (match, local_vars=local_vars): - var_name = match.group(1) - if var_name in local_vars: - return str(local_vars[var_name]) - else: - return os.environ[var_name] - - try: - return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) - except KeyError as var: - raise ValueError("invalid variable '$%s'" % var) - -# subst_vars () - - -def grok_environment_error (exc, prefix="error: "): - # Function kept for backward compatibility. - # Used to try clever things with EnvironmentErrors, - # but nowadays str(exception) produces good messages. - return prefix + str(exc) - - -# Needed by 'split_quoted()' -_wordchars_re = _squote_re = _dquote_re = None -def _init_regex(): - global _wordchars_re, _squote_re, _dquote_re - _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) - _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") - _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') - -def split_quoted (s): - """Split a string up according to Unix shell-like rules for quotes and - backslashes. In short: words are delimited by spaces, as long as those - spaces are not escaped by a backslash, or inside a quoted string. - Single and double quotes are equivalent, and the quote characters can - be backslash-escaped. The backslash is stripped from any two-character - escape sequence, leaving only the escaped character. The quote - characters are stripped from any quoted string. Returns a list of - words. - """ - - # This is a nice algorithm for splitting up a single string, since it - # doesn't require character-by-character examination. It was a little - # bit of a brain-bender to get it working right, though... - if _wordchars_re is None: _init_regex() - - s = s.strip() - words = [] - pos = 0 - - while s: - m = _wordchars_re.match(s, pos) - end = m.end() - if end == len(s): - words.append(s[:end]) - break - - if s[end] in string.whitespace: # unescaped, unquoted whitespace: now - words.append(s[:end]) # we definitely have a word delimiter - s = s[end:].lstrip() - pos = 0 - - elif s[end] == '\\': # preserve whatever is being escaped; - # will become part of the current word - s = s[:end] + s[end+1:] - pos = end+1 - - else: - if s[end] == "'": # slurp singly-quoted string - m = _squote_re.match(s, end) - elif s[end] == '"': # slurp doubly-quoted string - m = _dquote_re.match(s, end) - else: - raise RuntimeError("this can't happen (bad char '%c')" % s[end]) - - if m is None: - raise ValueError("bad string (mismatched %s quotes?)" % s[end]) - - (beg, end) = m.span() - s = s[:beg] + s[beg+1:end-1] + s[end:] - pos = m.end() - 2 - - if pos >= len(s): - words.append(s) - break - - return words - -# split_quoted () - - -def execute (func, args, msg=None, verbose=0, dry_run=0): - """Perform some action that affects the outside world (eg. by - writing to the filesystem). Such actions are special because they - are disabled by the 'dry_run' flag. This method takes care of all - that bureaucracy for you; all you have to do is supply the - function to call and an argument tuple for it (to embody the - "external action" being performed), and an optional message to - print. - """ - if msg is None: - msg = "%s%r" % (func.__name__, args) - if msg[-2:] == ',)': # correct for singleton tuple - msg = msg[0:-2] + ')' - - log.info(msg) - if not dry_run: - func(*args) - - -def strtobool (val): - """Convert a string representation of truth to true (1) or false (0). - - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - """ - val = val.lower() - if val in ('y', 'yes', 't', 'true', 'on', '1'): - return 1 - elif val in ('n', 'no', 'f', 'false', 'off', '0'): - return 0 - else: - raise ValueError("invalid truth value %r" % (val,)) - - -def byte_compile (py_files, - optimize=0, force=0, - prefix=None, base_dir=None, - verbose=1, dry_run=0, - direct=None): - """Byte-compile a collection of Python source files to .pyc - files in a __pycache__ subdirectory. 'py_files' is a list - of files to compile; any files that don't end in ".py" are silently - skipped. 'optimize' must be one of the following: - 0 - don't optimize - 1 - normal optimization (like "python -O") - 2 - extra optimization (like "python -OO") - If 'force' is true, all files are recompiled regardless of - timestamps. - - The source filename encoded in each bytecode file defaults to the - filenames listed in 'py_files'; you can modify these with 'prefix' and - 'basedir'. 'prefix' is a string that will be stripped off of each - source filename, and 'base_dir' is a directory name that will be - prepended (after 'prefix' is stripped). You can supply either or both - (or neither) of 'prefix' and 'base_dir', as you wish. - - If 'dry_run' is true, doesn't actually do anything that would - affect the filesystem. - - Byte-compilation is either done directly in this interpreter process - with the standard py_compile module, or indirectly by writing a - temporary script and executing it. Normally, you should let - 'byte_compile()' figure out to use direct compilation or not (see - the source for details). The 'direct' flag is used by the script - generated in indirect mode; unless you know what you're doing, leave - it set to None. - """ - - # Late import to fix a bootstrap issue: _posixsubprocess is built by - # setup.py, but setup.py uses distutils. - import subprocess - - # nothing is done if sys.dont_write_bytecode is True - if sys.dont_write_bytecode: - raise DistutilsByteCompileError('byte-compiling is disabled.') - - # First, if the caller didn't force us into direct or indirect mode, - # figure out which mode we should be in. We take a conservative - # approach: choose direct mode *only* if the current interpreter is - # in debug mode and optimize is 0. If we're not in debug mode (-O - # or -OO), we don't know which level of optimization this - # interpreter is running with, so we can't do direct - # byte-compilation and be certain that it's the right thing. Thus, - # always compile indirectly if the current interpreter is in either - # optimize mode, or if either optimization level was requested by - # the caller. - if direct is None: - direct = (__debug__ and optimize == 0) - - # "Indirect" byte-compilation: write a temporary script and then - # run it with the appropriate flags. - if not direct: - try: - from tempfile import mkstemp - (script_fd, script_name) = mkstemp(".py") - except ImportError: - from tempfile import mktemp - (script_fd, script_name) = None, mktemp(".py") - log.info("writing byte-compilation script '%s'", script_name) - if not dry_run: - if script_fd is not None: - script = os.fdopen(script_fd, "w") - else: - script = open(script_name, "w") - - script.write("""\ -from distutils.util import byte_compile -files = [ -""") - - # XXX would be nice to write absolute filenames, just for - # safety's sake (script should be more robust in the face of - # chdir'ing before running it). But this requires abspath'ing - # 'prefix' as well, and that breaks the hack in build_lib's - # 'byte_compile()' method that carefully tacks on a trailing - # slash (os.sep really) to make sure the prefix here is "just - # right". This whole prefix business is rather delicate -- the - # problem is that it's really a directory, but I'm treating it - # as a dumb string, so trailing slashes and so forth matter. - - #py_files = map(os.path.abspath, py_files) - #if prefix: - # prefix = os.path.abspath(prefix) - - script.write(",\n".join(map(repr, py_files)) + "]\n") - script.write(""" -byte_compile(files, optimize=%r, force=%r, - prefix=%r, base_dir=%r, - verbose=%r, dry_run=0, - direct=1) -""" % (optimize, force, prefix, base_dir, verbose)) - - script.close() - - cmd = [sys.executable] - cmd.extend(subprocess._optim_args_from_interpreter_flags()) - cmd.append(script_name) - spawn(cmd, dry_run=dry_run) - execute(os.remove, (script_name,), "removing %s" % script_name, - dry_run=dry_run) - - # "Direct" byte-compilation: use the py_compile module to compile - # right here, right now. Note that the script generated in indirect - # mode simply calls 'byte_compile()' in direct mode, a weird sort of - # cross-process recursion. Hey, it works! - else: - from py_compile import compile - - for file in py_files: - if file[-3:] != ".py": - # This lets us be lazy and not filter filenames in - # the "install_lib" command. - continue - - # Terminology from the py_compile module: - # cfile - byte-compiled file - # dfile - purported source filename (same as 'file' by default) - if optimize >= 0: - opt = '' if optimize == 0 else optimize - cfile = importlib.util.cache_from_source( - file, optimization=opt) - else: - cfile = importlib.util.cache_from_source(file) - dfile = file - if prefix: - if file[:len(prefix)] != prefix: - raise ValueError("invalid prefix: filename %r doesn't start with %r" - % (file, prefix)) - dfile = dfile[len(prefix):] - if base_dir: - dfile = os.path.join(base_dir, dfile) - - cfile_base = os.path.basename(cfile) - if direct: - if force or newer(file, cfile): - log.info("byte-compiling %s to %s", file, cfile_base) - if not dry_run: - compile(file, cfile, dfile) - else: - log.debug("skipping byte-compilation of %s to %s", - file, cfile_base) - -# byte_compile () - -def rfc822_escape (header): - """Return a version of the string escaped for inclusion in an - RFC-822 header, by ensuring there are 8 spaces space after each newline. - """ - lines = header.split('\n') - sep = '\n' + 8 * ' ' - return sep.join(lines) - -# 2to3 support - -def run_2to3(files, fixer_names=None, options=None, explicit=None): - """Invoke 2to3 on a list of Python files. - The files should all come from the build area, as the - modification is done in-place. To reduce the build time, - only files modified since the last invocation of this - function should be passed in the files argument.""" - - if not files: - return - - # Make this class local, to delay import of 2to3 - from lib2to3.refactor import RefactoringTool, get_fixers_from_package - class DistutilsRefactoringTool(RefactoringTool): - def log_error(self, msg, *args, **kw): - log.error(msg, *args) - - def log_message(self, msg, *args): - log.info(msg, *args) - - def log_debug(self, msg, *args): - log.debug(msg, *args) - - if fixer_names is None: - fixer_names = get_fixers_from_package('lib2to3.fixes') - r = DistutilsRefactoringTool(fixer_names, options=options) - r.refactor(files, write=True) - -def copydir_run_2to3(src, dest, template=None, fixer_names=None, - options=None, explicit=None): - """Recursively copy a directory, only copying new and changed files, - running run_2to3 over all newly copied Python modules afterward. - - If you give a template string, it's parsed like a MANIFEST.in. - """ - from distutils.dir_util import mkpath - from distutils.file_util import copy_file - from distutils.filelist import FileList - filelist = FileList() - curdir = os.getcwd() - os.chdir(src) - try: - filelist.findall() - finally: - os.chdir(curdir) - filelist.files[:] = filelist.allfiles - if template: - for line in template.splitlines(): - line = line.strip() - if not line: continue - filelist.process_template_line(line) - copied = [] - for filename in filelist.files: - outname = os.path.join(dest, filename) - mkpath(os.path.dirname(outname)) - res = copy_file(os.path.join(src, filename), outname, update=1) - if res[1]: copied.append(outname) - run_2to3([fn for fn in copied if fn.lower().endswith('.py')], - fixer_names=fixer_names, options=options, explicit=explicit) - return copied - -class Mixin2to3: - '''Mixin class for commands that run 2to3. - To configure 2to3, setup scripts may either change - the class variables, or inherit from individual commands - to override how 2to3 is invoked.''' - - # provide list of fixers to run; - # defaults to all from lib2to3.fixers - fixer_names = None - - # options dictionary - options = None - - # list of fixers to invoke even though they are marked as explicit - explicit = None - - def run_2to3(self, files): - return run_2to3(files, self.fixer_names, self.options, self.explicit) diff --git a/Lib/distutils/version.py b/Lib/distutils/version.py deleted file mode 100644 index af14cc13481..00000000000 --- a/Lib/distutils/version.py +++ /dev/null @@ -1,343 +0,0 @@ -# -# distutils/version.py -# -# Implements multiple version numbering conventions for the -# Python Module Distribution Utilities. -# -# $Id$ -# - -"""Provides classes to represent module version numbers (one class for -each style of version numbering). There are currently two such classes -implemented: StrictVersion and LooseVersion. - -Every version number class implements the following interface: - * the 'parse' method takes a string and parses it to some internal - representation; if the string is an invalid version number, - 'parse' raises a ValueError exception - * the class constructor takes an optional string argument which, - if supplied, is passed to 'parse' - * __str__ reconstructs the string that was passed to 'parse' (or - an equivalent string -- ie. one that will generate an equivalent - version number instance) - * __repr__ generates Python code to recreate the version number instance - * _cmp compares the current instance with either another instance - of the same class or a string (which will be parsed to an instance - of the same class, thus must follow the same rules) -""" - -import re - -class Version: - """Abstract base class for version numbering classes. Just provides - constructor (__init__) and reproducer (__repr__), because those - seem to be the same for all version numbering classes; and route - rich comparisons to _cmp. - """ - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - def __repr__ (self): - return "%s ('%s')" % (self.__class__.__name__, str(self)) - - def __eq__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c == 0 - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c < 0 - - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c <= 0 - - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c > 0 - - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c >= 0 - - -# Interface for version-number classes -- must be implemented -# by the following classes (the concrete ones -- Version should -# be treated as an abstract class). -# __init__ (string) - create and take same action as 'parse' -# (string parameter is optional) -# parse (string) - convert a string representation to whatever -# internal representation is appropriate for -# this style of version numbering -# __str__ (self) - convert back to a string; should be very similar -# (if not identical to) the string supplied to parse -# __repr__ (self) - generate Python code to recreate -# the instance -# _cmp (self, other) - compare two version numbers ('other' may -# be an unparsed version string, or another -# instance of your version class) - - -class StrictVersion (Version): - - """Version numbering for anal retentives and software idealists. - Implements the standard interface for version number classes as - described above. A version number consists of two or three - dot-separated numeric components, with an optional "pre-release" tag - on the end. The pre-release tag consists of the letter 'a' or 'b' - followed by a number. If the numeric components of two version - numbers are equal, then one with a pre-release tag will always - be deemed earlier (lesser) than one without. - - The following are valid version numbers (shown in the order that - would be obtained by sorting according to the supplied cmp function): - - 0.4 0.4.0 (these two are equivalent) - 0.4.1 - 0.5a1 - 0.5b3 - 0.5 - 0.9.6 - 1.0 - 1.0.4a3 - 1.0.4b1 - 1.0.4 - - The following are examples of invalid version numbers: - - 1 - 2.7.2.2 - 1.3.a4 - 1.3pl1 - 1.3c4 - - The rationale for this version numbering system will be explained - in the distutils documentation. - """ - - version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', - re.VERBOSE | re.ASCII) - - - def parse (self, vstring): - match = self.version_re.match(vstring) - if not match: - raise ValueError("invalid version number '%s'" % vstring) - - (major, minor, patch, prerelease, prerelease_num) = \ - match.group(1, 2, 4, 5, 6) - - if patch: - self.version = tuple(map(int, [major, minor, patch])) - else: - self.version = tuple(map(int, [major, minor])) + (0,) - - if prerelease: - self.prerelease = (prerelease[0], int(prerelease_num)) - else: - self.prerelease = None - - - def __str__ (self): - - if self.version[2] == 0: - vstring = '.'.join(map(str, self.version[0:2])) - else: - vstring = '.'.join(map(str, self.version)) - - if self.prerelease: - vstring = vstring + self.prerelease[0] + str(self.prerelease[1]) - - return vstring - - - def _cmp (self, other): - if isinstance(other, str): - other = StrictVersion(other) - - if self.version != other.version: - # numeric versions don't match - # prerelease stuff doesn't matter - if self.version < other.version: - return -1 - else: - return 1 - - # have to compare prerelease - # case 1: neither has prerelease; they're equal - # case 2: self has prerelease, other doesn't; other is greater - # case 3: self doesn't have prerelease, other does: self is greater - # case 4: both have prerelease: must compare them! - - if (not self.prerelease and not other.prerelease): - return 0 - elif (self.prerelease and not other.prerelease): - return -1 - elif (not self.prerelease and other.prerelease): - return 1 - elif (self.prerelease and other.prerelease): - if self.prerelease == other.prerelease: - return 0 - elif self.prerelease < other.prerelease: - return -1 - else: - return 1 - else: - assert False, "never get here" - -# end class StrictVersion - - -# The rules according to Greg Stein: -# 1) a version number has 1 or more numbers separated by a period or by -# sequences of letters. If only periods, then these are compared -# left-to-right to determine an ordering. -# 2) sequences of letters are part of the tuple for comparison and are -# compared lexicographically -# 3) recognize the numeric components may have leading zeroes -# -# The LooseVersion class below implements these rules: a version number -# string is split up into a tuple of integer and string components, and -# comparison is a simple tuple comparison. This means that version -# numbers behave in a predictable and obvious way, but a way that might -# not necessarily be how people *want* version numbers to behave. There -# wouldn't be a problem if people could stick to purely numeric version -# numbers: just split on period and compare the numbers as tuples. -# However, people insist on putting letters into their version numbers; -# the most common purpose seems to be: -# - indicating a "pre-release" version -# ('alpha', 'beta', 'a', 'b', 'pre', 'p') -# - indicating a post-release patch ('p', 'pl', 'patch') -# but of course this can't cover all version number schemes, and there's -# no way to know what a programmer means without asking him. -# -# The problem is what to do with letters (and other non-numeric -# characters) in a version number. The current implementation does the -# obvious and predictable thing: keep them as strings and compare -# lexically within a tuple comparison. This has the desired effect if -# an appended letter sequence implies something "post-release": -# eg. "0.99" < "0.99pl14" < "1.0", and "5.001" < "5.001m" < "5.002". -# -# However, if letters in a version number imply a pre-release version, -# the "obvious" thing isn't correct. Eg. you would expect that -# "1.5.1" < "1.5.2a2" < "1.5.2", but under the tuple/lexical comparison -# implemented here, this just isn't so. -# -# Two possible solutions come to mind. The first is to tie the -# comparison algorithm to a particular set of semantic rules, as has -# been done in the StrictVersion class above. This works great as long -# as everyone can go along with bondage and discipline. Hopefully a -# (large) subset of Python module programmers will agree that the -# particular flavour of bondage and discipline provided by StrictVersion -# provides enough benefit to be worth using, and will submit their -# version numbering scheme to its domination. The free-thinking -# anarchists in the lot will never give in, though, and something needs -# to be done to accommodate them. -# -# Perhaps a "moderately strict" version class could be implemented that -# lets almost anything slide (syntactically), and makes some heuristic -# assumptions about non-digits in version number strings. This could -# sink into special-case-hell, though; if I was as talented and -# idiosyncratic as Larry Wall, I'd go ahead and implement a class that -# somehow knows that "1.2.1" < "1.2.2a2" < "1.2.2" < "1.2.2pl3", and is -# just as happy dealing with things like "2g6" and "1.13++". I don't -# think I'm smart enough to do it right though. -# -# In any case, I've coded the test suite for this module (see -# ../test/test_version.py) specifically to fail on things like comparing -# "1.2a2" and "1.2". That's not because the *code* is doing anything -# wrong, it's because the simple, obvious design doesn't match my -# complicated, hairy expectations for real-world version numbers. It -# would be a snap to fix the test suite to say, "Yep, LooseVersion does -# the Right Thing" (ie. the code matches the conception). But I'd rather -# have a conception that matches common notions about version numbers. - -class LooseVersion (Version): - - """Version numbering for anarchists and software realists. - Implements the standard interface for version number classes as - described above. A version number consists of a series of numbers, - separated by either periods or strings of letters. When comparing - version numbers, the numeric components will be compared - numerically, and the alphabetic components lexically. The following - are all valid version numbers, in no particular order: - - 1.5.1 - 1.5.2b2 - 161 - 3.10a - 8.02 - 3.4j - 1996.07.12 - 3.2.pl0 - 3.1.1.6 - 2g6 - 11g - 0.960923 - 2.2beta29 - 1.13++ - 5.5.kw - 2.0b1pl0 - - In fact, there is no such thing as an invalid version number under - this scheme; the rules for comparison are simple and predictable, - but may not always give the results you want (for some definition - of "want"). - """ - - component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - - def parse (self, vstring): - # I've given up on thinking I can reconstruct the version string - # from the parsed tuple -- so I just store the string here for - # use by __str__ - self.vstring = vstring - components = [x for x in self.component_re.split(vstring) - if x and x != '.'] - for i, obj in enumerate(components): - try: - components[i] = int(obj) - except ValueError: - pass - - self.version = components - - - def __str__ (self): - return self.vstring - - - def __repr__ (self): - return "LooseVersion ('%s')" % str(self) - - - def _cmp (self, other): - if isinstance(other, str): - other = LooseVersion(other) - - if self.version == other.version: - return 0 - if self.version < other.version: - return -1 - if self.version > other.version: - return 1 - - -# end class LooseVersion diff --git a/Lib/distutils/versionpredicate.py b/Lib/distutils/versionpredicate.py deleted file mode 100644 index 062c98f2489..00000000000 --- a/Lib/distutils/versionpredicate.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Module for parsing and testing package version predicate strings. -""" -import re -import distutils.version -import operator - - -re_validPackage = re.compile(r"(?i)^\s*([a-z_]\w*(?:\.[a-z_]\w*)*)(.*)", - re.ASCII) -# (package) (rest) - -re_paren = re.compile(r"^\s*\((.*)\)\s*$") # (list) inside of parentheses -re_splitComparison = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") -# (comp) (version) - - -def splitUp(pred): - """Parse a single version comparison. - - Return (comparison string, StrictVersion) - """ - res = re_splitComparison.match(pred) - if not res: - raise ValueError("bad package restriction syntax: %r" % pred) - comp, verStr = res.groups() - return (comp, distutils.version.StrictVersion(verStr)) - -compmap = {"<": operator.lt, "<=": operator.le, "==": operator.eq, - ">": operator.gt, ">=": operator.ge, "!=": operator.ne} - -class VersionPredicate: - """Parse and test package version predicates. - - >>> v = VersionPredicate('pyepat.abc (>1.0, <3333.3a1, !=1555.1b3)') - - The `name` attribute provides the full dotted name that is given:: - - >>> v.name - 'pyepat.abc' - - The str() of a `VersionPredicate` provides a normalized - human-readable version of the expression:: - - >>> print(v) - pyepat.abc (> 1.0, < 3333.3a1, != 1555.1b3) - - The `satisfied_by()` method can be used to determine with a given - version number is included in the set described by the version - restrictions:: - - >>> v.satisfied_by('1.1') - True - >>> v.satisfied_by('1.4') - True - >>> v.satisfied_by('1.0') - False - >>> v.satisfied_by('4444.4') - False - >>> v.satisfied_by('1555.1b3') - False - - `VersionPredicate` is flexible in accepting extra whitespace:: - - >>> v = VersionPredicate(' pat( == 0.1 ) ') - >>> v.name - 'pat' - >>> v.satisfied_by('0.1') - True - >>> v.satisfied_by('0.2') - False - - If any version numbers passed in do not conform to the - restrictions of `StrictVersion`, a `ValueError` is raised:: - - >>> v = VersionPredicate('p1.p2.p3.p4(>=1.0, <=1.3a1, !=1.2zb3)') - Traceback (most recent call last): - ... - ValueError: invalid version number '1.2zb3' - - It the module or package name given does not conform to what's - allowed as a legal module or package name, `ValueError` is - raised:: - - >>> v = VersionPredicate('foo-bar') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: '-bar' - - >>> v = VersionPredicate('foo bar (12.21)') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: 'bar (12.21)' - - """ - - def __init__(self, versionPredicateStr): - """Parse a version predicate string. - """ - # Fields: - # name: package name - # pred: list of (comparison string, StrictVersion) - - versionPredicateStr = versionPredicateStr.strip() - if not versionPredicateStr: - raise ValueError("empty package restriction") - match = re_validPackage.match(versionPredicateStr) - if not match: - raise ValueError("bad package name in %r" % versionPredicateStr) - self.name, paren = match.groups() - paren = paren.strip() - if paren: - match = re_paren.match(paren) - if not match: - raise ValueError("expected parenthesized list: %r" % paren) - str = match.groups()[0] - self.pred = [splitUp(aPred) for aPred in str.split(",")] - if not self.pred: - raise ValueError("empty parenthesized list in %r" - % versionPredicateStr) - else: - self.pred = [] - - def __str__(self): - if self.pred: - seq = [cond + " " + str(ver) for cond, ver in self.pred] - return self.name + " (" + ", ".join(seq) + ")" - else: - return self.name - - def satisfied_by(self, version): - """True if version is compatible with all the predicates in self. - The parameter version must be acceptable to the StrictVersion - constructor. It may be either a string or StrictVersion. - """ - for cond, ver in self.pred: - if not compmap[cond](version, ver): - return False - return True - - -_provision_rx = None - -def split_provision(value): - """Return the name and optional version number of a provision. - - The version number, if given, will be returned as a `StrictVersion` - instance, otherwise it will be `None`. - - >>> split_provision('mypkg') - ('mypkg', None) - >>> split_provision(' mypkg( 1.2 ) ') - ('mypkg', StrictVersion ('1.2')) - """ - global _provision_rx - if _provision_rx is None: - _provision_rx = re.compile( - r"([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(?:\s*\(\s*([^)\s]+)\s*\))?$", - re.ASCII) - value = value.strip() - m = _provision_rx.match(value) - if not m: - raise ValueError("illegal provides specification: %r" % value) - ver = m.group(2) or None - if ver: - ver = distutils.version.StrictVersion(ver) - return m.group(1), ver diff --git a/Lib/doctest.py b/Lib/doctest.py index 387f71b184a..a66888d8fc9 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -104,8 +105,28 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize + + +__unittest = True + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + # Leave the repr() unchanged for backward compatibility + # if skipped is zero + return super().__repr__() -TestResults = namedtuple('TestResults', 'failed attempted') # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -207,7 +228,13 @@ def _normalize_module(module, depth=2): elif isinstance(module, str): return __import__(module, globals(), locals(), ["*"]) elif module is None: - return sys.modules[sys._getframe(depth).f_globals['__name__']] + try: + try: + return sys.modules[sys._getframemodulename(depth)] + except AttributeError: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + except KeyError: + pass else: raise TypeError("Expected a module, string, or None") @@ -229,7 +256,6 @@ def _load_testfile(filename, package, module_relative, encoding): file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -366,11 +392,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test @@ -570,9 +596,11 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented - return ((self.name, self.filename, self.lineno, id(self)) + self_lno = self.lineno if self.lineno is not None else -1 + other_lno = other.lineno if other.lineno is not None else -1 + return ((self.name, self.filename, self_lno, id(self)) < - (other.name, other.filename, other.lineno, id(other))) + (other.name, other.filename, other_lno, id(other))) ###################################################################### ## 3. DocTestParser @@ -957,7 +985,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): @@ -1104,7 +1133,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i @@ -1112,13 +1141,24 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, property): + obj = obj.fget + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): - lineno = getattr(obj, 'co_firstlineno', None)-1 + lineno = obj.co_firstlineno - 1 # Find the line number where the docstring starts. Assume # that it's the first line that begins with a quote mark. @@ -1144,8 +1184,10 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults instance. + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1158,27 +1200,29 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults + instance: >>> runner.summarize(verbose=1) 4 items passed all tests: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: + The aggregated number of tried examples and failed examples is also + available via the `tries`, `failures` and `skips` attributes: >>> runner.tries 7 >>> runner.failures 0 + >>> runner.skips + 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a @@ -1188,13 +1232,15 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1227,7 +1273,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skips = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1272,7 +1319,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1296,13 +1346,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults + instance. The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failed, attempted, skipped examples. + failures = attempted = skips = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1314,6 +1362,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1331,10 +1380,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skips += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1351,11 +1400,11 @@ def __run(self, test, compileflags, out): exec(compile(example.source, filename, "single", compileflags, True), test.globs) self.debugger.set_continue() # ==== Example Finished ==== - exception = None + exc_info = None except KeyboardInterrupt: raise - except: - exception = sys.exc_info() + except BaseException as exc: + exc_info = type(exc), exc, exc.__traceback__.tb_next self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output @@ -1364,15 +1413,32 @@ def __run(self, test, compileflags, out): # If the example executed without raising any exceptions, # verify its output. - if exception is None: + if exc_info is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exc_info[:2]) + if issubclass(exc_info[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exception_line_prefixes = ( + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", + ) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(exception_line_prefixes) + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: - got += _exception_traceback(exception) + got += _exception_traceback(exc_info) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1401,7 +1467,7 @@ def __run(self, test, compileflags, out): elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exception) + exc_info) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -1412,19 +1478,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skips) + return TestResults(failures, attempted, skipped=skips) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skips): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skips2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skips + skips2) + self.failures += failures + self.tries += tries + self.skips += skips __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1493,7 +1562,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda *args, **kwargs: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1502,6 +1575,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1513,9 +1590,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults instance. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1523,66 +1598,98 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + + notests, passed, failed = [], [], [] + total_tries = total_failures = total_skips = 0 + + for name, (failures, tries, skips) in self._stats.items(): + assert failures <= tries + total_tries += tries + total_failures += failures + total_skips += skips + + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") + if passed: - print(len(passed), "items passed all tests:") - passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(len(failed), "items had failures:") - failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): + print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + + if total_failures: + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" + if total_skips: + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" + print(f"{msg}.") elif verbose: - print("Test passed.") - return TestResults(totalf, totalt) + print(f"{bold_green}Test passed.{reset}") + + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skips) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skips2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skips = skips + skips2 + d[name] = (failures, tries, skips) + + +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -1887,8 +1994,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. @@ -1978,7 +2085,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2101,7 +2209,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): @@ -2176,13 +2285,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) @@ -2213,13 +2322,14 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: - raise self.failureException(self.format_failure(new.getvalue())) + if results.failed: + raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) def format_failure(self, err): test = self._dt_test @@ -2629,7 +2739,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) @@ -2765,7 +2875,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner") + parser = argparse.ArgumentParser(description="doctest runner", color=True) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', diff --git a/Lib/dummy_threading.py b/Lib/dummy_threading.py index 1bb7eee338a..662f3b89a9a 100644 --- a/Lib/dummy_threading.py +++ b/Lib/dummy_threading.py @@ -6,6 +6,7 @@ regardless of whether ``_thread`` was available which is not desired. """ + from sys import modules as sys_modules import _dummy_thread @@ -19,35 +20,38 @@ # Could have checked if ``_thread`` was not in sys.modules and gone # a different route, but decided to mirror technique used with # ``threading`` below. - if '_thread' in sys_modules: - held_thread = sys_modules['_thread'] + if "_thread" in sys_modules: + held_thread = sys_modules["_thread"] holding_thread = True # Must have some module named ``_thread`` that implements its API # in order to initially import ``threading``. - sys_modules['_thread'] = sys_modules['_dummy_thread'] + sys_modules["_thread"] = sys_modules["_dummy_thread"] - if 'threading' in sys_modules: + if "threading" in sys_modules: # If ``threading`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. - held_threading = sys_modules['threading'] + held_threading = sys_modules["threading"] holding_threading = True - del sys_modules['threading'] + del sys_modules["threading"] - if '_threading_local' in sys_modules: + if "_threading_local" in sys_modules: # If ``_threading_local`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. - held__threading_local = sys_modules['_threading_local'] + held__threading_local = sys_modules["_threading_local"] holding__threading_local = True - del sys_modules['_threading_local'] + del sys_modules["_threading_local"] import threading + # Need a copy of the code kept somewhere... - sys_modules['_dummy_threading'] = sys_modules['threading'] - del sys_modules['threading'] - sys_modules['_dummy__threading_local'] = sys_modules['_threading_local'] - del sys_modules['_threading_local'] + sys_modules["_dummy_threading"] = sys_modules["threading"] + del sys_modules["threading"] + # _threading_local may not be imported if _thread._local is available + if "_threading_local" in sys_modules: + sys_modules["_dummy__threading_local"] = sys_modules["_threading_local"] + del sys_modules["_threading_local"] from _dummy_threading import * from _dummy_threading import __all__ @@ -55,23 +59,23 @@ # Put back ``threading`` if we overwrote earlier if holding_threading: - sys_modules['threading'] = held_threading + sys_modules["threading"] = held_threading del held_threading del holding_threading # Put back ``_threading_local`` if we overwrote earlier if holding__threading_local: - sys_modules['_threading_local'] = held__threading_local + sys_modules["_threading_local"] = held__threading_local del held__threading_local del holding__threading_local # Put back ``thread`` if we overwrote, else del the entry we made if holding_thread: - sys_modules['_thread'] = held_thread + sys_modules["_thread"] = held_thread del held_thread else: - del sys_modules['_thread'] + del sys_modules["_thread"] del holding_thread del _dummy_thread diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index ec2215a5e5f..91243378dc0 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -95,8 +95,16 @@ NLSET = {'\n', '\r'} SPECIALSNL = SPECIALS | NLSET + +def make_quoted_pairs(value): + """Escape dquote and backslash for use within a quoted-string.""" + return str(value).replace('\\', '\\\\').replace('"', '\\"') + + def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' + escaped = make_quoted_pairs(value) + return f'"{escaped}"' + # Match a RFC 2047 word, looks like =?utf-8?q?someword?= rfc2047_matcher = re.compile(r''' @@ -1012,6 +1020,8 @@ def _get_ptext_to_endchars(value, endchars): a flag that is True iff there were any quoted printables decoded. """ + if not value: + return '', '', False fragment, *remainder = _wsp_splitter(value, 1) vchars = [] escape = False @@ -1045,7 +1055,7 @@ def get_fws(value): fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') return fws, newvalue -def get_encoded_word(value): +def get_encoded_word(value, terminal_type='vtext'): """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" """ @@ -1084,7 +1094,7 @@ def get_encoded_word(value): ew.append(token) continue chars, *remainder = _wsp_splitter(text, 1) - vtext = ValueTerminal(chars, 'vtext') + vtext = ValueTerminal(chars, terminal_type) _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) @@ -1126,7 +1136,7 @@ def get_unstructured(value): valid_ew = True if value.startswith('=?'): try: - token, value = get_encoded_word(value) + token, value = get_encoded_word(value, 'utext') except _InvalidEwError: valid_ew = False except errors.HeaderParseError: @@ -1155,7 +1165,7 @@ def get_unstructured(value): # the parser to go in an infinite loop. if valid_ew and rfc2047_matcher.search(tok): tok, *remainder = value.partition('=?') - vtext = ValueTerminal(tok, 'vtext') + vtext = ValueTerminal(tok, 'utext') _validate_xtext(vtext) unstructured.append(vtext) value = ''.join(remainder) @@ -1565,7 +1575,7 @@ def get_dtext(value): def _check_for_early_dl_end(value, domain_literal): if value: return False - domain_literal.append(errors.InvalidHeaderDefect( + domain_literal.defects.append(errors.InvalidHeaderDefect( "end of input inside domain-literal")) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) return True @@ -1584,9 +1594,9 @@ def get_domain_literal(value): raise errors.HeaderParseError("expected '[' at start of domain-literal " "but found '{}'".format(value)) value = value[1:] + domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value - domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) @@ -2805,7 +2815,7 @@ def _refold_parse_tree(parse_tree, *, policy): continue tstr = str(part) if not want_encoding: - if part.token_type == 'ptext': + if part.token_type in ('ptext', 'vtext'): # Encode if tstr contains special characters. want_encoding = not SPECIALSNL.isdisjoint(tstr) else: @@ -2905,6 +2915,15 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. newparts = list(part) + if part.token_type == 'bare-quoted-string': + # To fold a quoted string we need to create a list of terminal + # tokens that will render the leading and trailing quotes + # and use quoted pairs in the value as appropriate. + newparts = ( + [ValueTerminal('"', 'ptext')] + + [ValueTerminal(make_quoted_pairs(p), 'ptext') + for p in newparts] + + [ValueTerminal('"', 'ptext')]) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 0f1bf8e4253..565af0cf361 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -146,8 +146,9 @@ def _parsedate_tz(data): return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for + # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) already + # mandated a 4-digit yy, and RFC 5322 (which obsoletes RFC 2822) continues + # this requirement. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). @@ -233,9 +234,11 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. + # Note that RFC 2822 section 4.1 introduced '.' as obs-phrase to handle + # existing practice (periods in display names), even though it was not + # allowed in RFC 822. RFC 5322 section 4.1 (which obsoletes RFC 2822) + # continues this requirement. We must recognize obsolete syntax, so + # allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9f0d743090..0d486c90a9c 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -370,7 +370,7 @@ def _fold(self, name, value, sanitize): h = value if h is not None: # The Header class interprets a value of None for maxlinelen as the - # default value of 78, as recommended by RFC 2822. + # default value of 78, as recommended by RFC 5322 section 2.1.1. maxlinelen = 0 if self.max_line_length is not None: maxlinelen = self.max_line_length diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b4f5830bead..11d1536db27 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -2,6 +2,7 @@ import email.charset import email.message import email.errors +import sys from email import quoprimime class ContentManager: @@ -142,13 +143,15 @@ def _encode_base64(data, max_line_length): def _encode_text(string, charset, cte, policy): + # If max_line_length is 0 or None, there is no limit. + maxlen = policy.max_line_length or sys.maxsize lines = string.encode(charset).splitlines() linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte is None: # Use heuristics to decide on the "best" encoding. - if max((len(x) for x in lines), default=0) <= policy.max_line_length: + if max(map(len, lines), default=0) <= maxlen: try: return '7bit', normal_body(lines).decode('ascii') except UnicodeDecodeError: @@ -156,8 +159,7 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' if policy.cte_type == '8bit': return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) - sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), - policy.max_line_length) + sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), maxlen) sniff_base64 = binascii.b2a_base64(sniff) # This is a little unfair to qp; it includes lineseps, base64 doesn't. if len(sniff_qp) > len(sniff_base64): @@ -172,9 +174,9 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' data = normal_body(lines).decode('ascii', 'surrogateescape') elif cte == 'quoted-printable': data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), - policy.max_line_length) + maxlen) elif cte == 'base64': - data = _encode_base64(embedded_body(lines), policy.max_line_length) + data = _encode_base64(embedded_body(lines), maxlen) else: raise ValueError("Unknown content transfer encoding {}".format(cte)) return cte, data diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 06d6b4a3afc..bc773f38030 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -32,7 +32,7 @@ NLCRE_bol = re.compile(r'(\r\n|\r|\n)') NLCRE_eol = re.compile(r'(\r\n|\r|\n)\Z') NLCRE_crack = re.compile(r'(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character +# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' @@ -294,7 +294,7 @@ def _parsegen(self): return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. + # another RFC 5322 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 47b9df8f4e6..ce94f5c56fe 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -50,7 +50,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. + by RFC 5322 section 2.1.1. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, diff --git a/Lib/email/header.py b/Lib/email/header.py index 984851a7d9a..a0aadb97ca6 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -59,16 +59,22 @@ def decode_header(header): """Decode a message header value without converting charset. - Returns a list of (string, charset) pairs containing each of the decoded - parts of the header. Charset is None for non-encoded parts of the header, - otherwise a lower-case string containing the name of the character set - specified in the encoded string. + For historical reasons, this function may return either: + + 1. A list of length 1 containing a pair (str, None). + 2. A list of (bytes, charset) pairs containing each of the decoded + parts of the header. Charset is None for non-encoded parts of the header, + otherwise a lower-case string containing the name of the character set + specified in the encoded string. header may be a string that may or may not contain RFC2047 encoded words, or it may be a Header object. An email.errors.HeaderParseError may be raised when certain decoding error occurs (e.g. a base64 decoding exception). + + This function exists for backwards compatibility only. For new code, we + recommend using email.headerregistry.HeaderRegistry instead. """ # If it is a Header object, we can just return the encoded chunks. if hasattr(header, '_chunks'): @@ -161,6 +167,9 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, This function takes one of those sequence of pairs and returns a Header instance. Optional maxlinelen, header_name, and continuation_ws are as in the Header constructor. + + This function exists for backwards compatibility only, and is not + recommended for use in new code. """ h = Header(maxlinelen=maxlinelen, header_name=header_name, continuation_ws=continuation_ws) diff --git a/Lib/email/message.py b/Lib/email/message.py index 46bb8c21942..80f01d66a33 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -74,19 +74,25 @@ def _parseparam(s): # RDM This might be a Header, so for now stringify it. s = ';' + str(s) plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) + start = 0 + while s.find(';', start) == start: + start += 1 + end = s.find(';', start) + ind, diff = start, 0 + while end > 0: + diff += s.count('"', ind, end) - s.count('\\"', ind, end) + if diff % 2 == 0: + break + end, ind = ind, s.find(';', end + 1) if end < 0: end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() + i = s.find('=', start, end) + if i == -1: + f = s[start:end] + else: + f = s[start:i].rstrip().lower() + '=' + s[i+1:end].lstrip() plist.append(f.strip()) - s = s[end:] + start = end return plist @@ -135,7 +141,7 @@ def _decode_uu(encoded): class Message: """Basic message object. - A message object is defined as something that has a bunch of RFC 2822 + A message object is defined as something that has a bunch of RFC 5322 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message @@ -286,8 +292,12 @@ def get_payload(self, i=None, decode=False): if i is not None and not isinstance(self._payload, list): raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload - # cte might be a Header, so for now stringify it. - cte = str(self.get('content-transfer-encoding', '')).lower() + cte = self.get('content-transfer-encoding', '') + if hasattr(cte, 'cte'): + cte = cte.cte + else: + # cte might be a Header, so for now stringify it. + cte = str(cte).strip().lower() # payload may be bytes here. if not decode: if isinstance(payload, str) and utils._has_surrogates(payload): @@ -309,6 +319,8 @@ def get_payload(self, i=None, decode=False): # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': @@ -560,7 +572,7 @@ def add_header(self, _name, _value, **_params): msg.add_header('content-disposition', 'attachment', filename='bud.gif') msg.add_header('content-disposition', 'attachment', - filename=('utf-8', '', Fußballer.ppt')) + filename=('utf-8', '', 'Fußballer.ppt')) msg.add_header('content-disposition', 'attachment', filename='Fußballer.ppt')) """ diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 06d99b17f2f..e3003118ce1 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -2,7 +2,7 @@ # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org -"""A parser of RFC 2822 and MIME email messages.""" +"""A parser of RFC 5322 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] @@ -15,14 +15,14 @@ class Parser: def __init__(self, _class=None, *, policy=compat32): - """Parser of RFC 2822 and MIME email messages. + """Parser of RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + The string must be formatted as a block of RFC 5322 headers and header + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -75,14 +75,14 @@ def parsestr(self, text, headersonly=True): class BytesParser: def __init__(self, *args, **kw): - """Parser of binary RFC 2822 and MIME email messages. + """Parser of binary RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + The input must be formatted as a block of RFC 5322 headers and header + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/utils.py b/Lib/email/utils.py index e42674fa4f3..e4d35f06abc 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -417,8 +417,14 @@ def decode_params(params): for name, continuations in rfc2231_params.items(): value = [] extended = False - # Sort by number - continuations.sort() + # Sort by number, treating None as 0 if there is no 0, + # and ignore it if there is already a 0. + has_zero = any(x[0] == 0 for x in continuations) + if has_zero: + continuations = [x for x in continuations if x[0] is not None] + else: + continuations = [(x[0] or 0, x[1], x[2]) for x in continuations] + continuations.sort(key=lambda x: x[0]) # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a *, then the entire string, after diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index f9075b8f0d9..298177eb800 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -156,19 +156,22 @@ def search_function(encoding): codecs.register(search_function) if sys.platform == 'win32': - # bpo-671666, bpo-46668: If Python does not implement a codec for current - # Windows ANSI code page, use the "mbcs" codec instead: - # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP. - # Python does not support custom code pages. - def _alias_mbcs(encoding): + from ._win_cp_codecs import create_win32_code_page_codec + + def win32_code_page_search_function(encoding): + encoding = encoding.lower() + if not encoding.startswith('cp'): + return None try: - import _winapi - ansi_code_page = "cp%s" % _winapi.GetACP() - if encoding == ansi_code_page: - import encodings.mbcs - return encodings.mbcs.getregentry() - except ImportError: - # Imports may fail while we are shutting down - pass + cp = int(encoding[2:]) + except ValueError: + return None + # Test if the code page is supported + try: + codecs.code_page_encode(cp, 'x') + except (OverflowError, OSError): + return None + + return create_win32_code_page_codec(cp) - codecs.register(_alias_mbcs) + codecs.register(win32_code_page_search_function) diff --git a/Lib/encodings/_win_cp_codecs.py b/Lib/encodings/_win_cp_codecs.py new file mode 100644 index 00000000000..4f8eb886794 --- /dev/null +++ b/Lib/encodings/_win_cp_codecs.py @@ -0,0 +1,36 @@ +import codecs + +def create_win32_code_page_codec(cp): + from codecs import code_page_encode, code_page_decode + + def encode(input, errors='strict'): + return code_page_encode(cp, input, errors) + + def decode(input, errors='strict'): + return code_page_decode(cp, input, errors, True) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return code_page_encode(cp, input, self.errors)[0] + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return code_page_encode(cp, input, errors) + + class StreamReader(codecs.StreamReader): + def decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + return codecs.CodecInfo( + name=f'cp{cp}', + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index 6a5ca046b5e..4ecb6b6e297 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -204,6 +204,11 @@ 'csibm869' : 'cp869', 'ibm869' : 'cp869', + # cp874 codec + '874' : 'cp874', + 'ms874' : 'cp874', + 'windows_874' : 'cp874', + # cp932 codec '932' : 'cp932', 'ms932' : 'cp932', @@ -241,6 +246,7 @@ 'ks_c_5601_1987' : 'euc_kr', 'ksx1001' : 'euc_kr', 'ks_x_1001' : 'euc_kr', + 'cseuckr' : 'euc_kr', # gb18030 codec 'gb18030_2000' : 'gb18030', @@ -399,6 +405,8 @@ 'iso_8859_8' : 'iso8859_8', 'iso_8859_8_1988' : 'iso8859_8', 'iso_ir_138' : 'iso8859_8', + 'iso_8859_8_i' : 'iso8859_8', + 'iso_8859_8_e' : 'iso8859_8', # iso8859_9 codec 'csisolatin5' : 'iso8859_9', diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index ab6d32478e4..21bbfad0fe6 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.2" +_PIP_VERSION = "25.3" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser(color=True) parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl similarity index 78% rename from Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl index 4db7e720718..755e1aa0c3d 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d..4183c28a809 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/enum.py b/Lib/enum.py index 7cffb71863c..b4551da1c17 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,12 +1,10 @@ import sys import builtins as bltns from types import MappingProxyType, DynamicClassAttribute -from operator import or_ as _or_ -from functools import reduce __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', @@ -63,8 +61,8 @@ def _is_sunder(name): return ( len(name) > 2 and name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' + name[1] != '_' and + name[-2] != '_' ) def _is_internal_class(cls_name, obj): @@ -83,7 +81,6 @@ def _is_private(cls_name, name): if ( len(name) > pat_len and name.startswith(pattern) - and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -132,7 +129,7 @@ def show_flag_values(value): def bin(num, max_bits=None): """ Like built-in bin(), except negative values are represented in - twos-compliment, and the leading bit always indicates sign + twos-complement, and the leading bit always indicates sign (0=positive, 1=negative). >>> bin(10) @@ -141,6 +138,7 @@ def bin(num, max_bits=None): '0b1 0101' """ + num = num.__index__() ceiling = 2 ** (num).bit_length() if num >= 0: s = bltns.bin(num + ceiling).replace('1', '0', 1) @@ -153,18 +151,10 @@ def bin(num, max_bits=None): digits = (sign[-1] * max_bits + digits)[-max_bits:] return "%s %s" % (sign, digits) -def _dedent(text): - """ - Like textwrap.dedent. Rewritten because we cannot import textwrap. - """ - lines = text.split('\n') - blanks = 0 - for i, ch in enumerate(lines[0]): - if ch != ' ': - break - for j, l in enumerate(lines): - lines[j] = l[i:] - return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() class _auto_null: def __repr__(self): @@ -206,7 +196,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor @@ -283,9 +273,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -313,72 +304,40 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -389,23 +348,19 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) - if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + if self._cls_name is not None and _is_private(self._cls_name, key): + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', - ): + '_add_alias_', '_add_value_alias_', + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -441,10 +396,9 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -457,10 +411,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -475,12 +430,21 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singly + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -491,6 +455,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -502,8 +468,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: @@ -564,7 +529,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} - classdict['_unhashable_values_'] = [] + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -579,15 +546,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: - exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e - if exc is not None: - raise exc - # + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) + except Exception as e: + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) # @@ -706,7 +674,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -735,18 +703,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -760,10 +728,20 @@ def __contains__(cls, value): `value` is in `cls` if: 1) `value` is a member of `cls`, or 2) `value` is the value of one of the `cls`'s members. + 3) `value` is a pseudo-member (flags) """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + if issubclass(cls, Flag): + try: + result = cls._missing_(value) + return isinstance(result, cls) + except ValueError: + pass + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1059,7 +1037,70 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1102,13 +1143,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -1125,12 +1159,17 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: + return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1165,8 +1204,33 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) @staticmethod def _generate_next_value_(name, start, count, last_values): @@ -1181,28 +1245,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in reversed(last_values): - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): @@ -1217,14 +1266,13 @@ def __str__(self): def __dir__(self): """ - Returns all members and all public methods + Returns public methods and other interesting attributes. """ - if self.__class__._member_type_ is object: - interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) - else: + interesting = set() + if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): - if name[0] != '_': + if name[0] != '_' and name not in self._member_map_: interesting.add(name) for cls in self.__class__.mro(): for name, obj in cls.__dict__.items(): @@ -1237,7 +1285,7 @@ def __dir__(self): else: # in case it was added by `dir(self)` interesting.discard(name) - else: + elif name not in self._member_map_: interesting.add(name) names = sorted( set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) @@ -1525,37 +1573,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) @@ -1622,7 +1683,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1665,7 +1726,7 @@ def _simple_enum(etype=Enum, *, boundary=None, use_args=None): Class decorator that converts a normal class into an :class:`Enum`. No safety checks are done, and some advanced behavior (such as :func:`__init_subclass__`) is not available. Enum creation can be faster - using :func:`simple_enum`. + using :func:`_simple_enum`. >>> from enum import Enum, _simple_enum >>> @_simple_enum(Enum) @@ -1696,7 +1757,9 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_hashable_values_'] = hashable_values = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1743,35 +1806,42 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member + hashable_values.append(value) if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ member_names.append(name) @@ -1793,37 +1863,53 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ @@ -1880,7 +1966,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) @@ -1908,7 +1994,8 @@ def __call__(self, enumeration): missed = [v for v in values if v not in member_values] if missed: missing_names.append(name) - missing_value |= reduce(_or_, missed) + for val in missed: + missing_value |= val if missing_names: if len(missing_names) == 1: alias = 'alias %s is missing' % missing_names[0] @@ -1941,8 +2028,7 @@ def _test_simple_enum(checked_enum, simple_enum): ... RED = auto() ... GREEN = auto() ... BLUE = auto() - ... # TODO: RUSTPYTHON - >>> _test_simple_enum(CheckedColor, Color) # doctest: +SKIP + >>> _test_simple_enum(CheckedColor, Color) If differences are found, a :exc:`TypeError` is raised. """ @@ -1957,7 +2043,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 30bd900fa80..c5b8d854d77 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -121,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -132,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -160,12 +164,14 @@ def phase2(self): # Distinguish files, directories, funnies ok = True try: a_stat = os.stat(a_path) - except OSError: + except (OSError, ValueError): + # See https://github.com/python/cpython/issues/122400 + # for the rationale for protecting against ValueError. # print('Can\'t stat', a_path, ':', why.args[1]) ok = False try: b_stat = os.stat(b_path) - except OSError: + except (OSError, ValueError): # print('Can\'t stat', b_path, ':', why.args[1]) ok = False @@ -184,7 +190,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -196,7 +202,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() @@ -280,12 +287,12 @@ def cmpfiles(a, b, common, shallow=True): # Return: # 0 for equal # 1 for different -# 2 for funny cases (can't stat, etc.) +# 2 for funny cases (can't stat, NUL bytes, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except (OSError, ValueError): return 2 diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 73acb1fe8d4..10e1c936688 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -9,12 +9,15 @@ The function translate(PATTERN) returns a regular expression corresponding to PATTERN. (It does not compile it.) """ + +import functools +import itertools import os import posixpath import re -import functools -__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] +__all__ = ["filter", "filterfalse", "fnmatch", "fnmatchcase", "translate"] + def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -35,6 +38,7 @@ def fnmatch(name, pat): pat = os.path.normcase(pat) return fnmatchcase(name, pat) + @functools.lru_cache(maxsize=32768, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): @@ -45,6 +49,7 @@ def _compile_pattern(pat): res = translate(pat) return re.compile(res).match + def filter(names, pat): """Construct a list from those elements of the iterable NAMES that match PAT.""" result = [] @@ -61,6 +66,22 @@ def filter(names, pat): result.append(name) return result + +def filterfalse(names, pat): + """Construct a list from those elements of the iterable NAMES that do not match PAT.""" + pat = os.path.normcase(pat) + match = _compile_pattern(pat) + if os.path is posixpath: + # normcase on posix is NOP. Optimize it away from the loop. + return list(itertools.filterfalse(match, names)) + + result = [] + for name in names: + if match(os.path.normcase(name)) is None: + result.append(name) + return result + + def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. @@ -77,24 +98,32 @@ def translate(pat): There is no way to quote meta-characters. """ - STAR = object() - parts = _translate(pat, STAR, '.') - return _join_translated_parts(parts, STAR) + parts, star_indices = _translate(pat, '*', '.') + return _join_translated_parts(parts, star_indices) + +_re_setops_sub = re.compile(r'([&~|])').sub +_re_escape = functools.lru_cache(maxsize=512)(re.escape) -def _translate(pat, STAR, QUESTION_MARK): + +def _translate(pat, star, question_mark): res = [] add = res.append + star_indices = [] + i, n = 0, len(pat) while i < n: c = pat[i] i = i+1 if c == '*': + # store the position of the wildcard + star_indices.append(len(res)) + add(star) # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) + while i < n and pat[i] == '*': + i += 1 elif c == '?': - add(QUESTION_MARK) + add(question_mark) elif c == '[': j = i if j < n and pat[j] == '!': @@ -133,8 +162,6 @@ def _translate(pat, STAR, QUESTION_MARK): # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 if not stuff: # Empty range: never match. @@ -143,50 +170,40 @@ def _translate(pat, STAR, QUESTION_MARK): # Negated empty range: match any character. add('.') else: + # Escape set operations (&&, ~~ and ||). + stuff = _re_setops_sub(r'\\\1', stuff) if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff add(f'[{stuff}]') else: - add(re.escape(c)) - assert i == n - return res - - -def _join_translated_parts(inp, STAR): - # Deal with STARs. - res = [] - add = res.append - i, n = 0, len(inp) - # Fixed pieces at the start? - while i < n and inp[i] is not STAR: - add(inp[i]) - i += 1 - # Now deal with STAR fixed STAR fixed ... - # For an interior `STAR fixed` pairing, we want to do a minimal - # .*? match followed by `fixed`, with no possibility of backtracking. - # Atomic groups ("(?>...)") allow us to spell that directly. - # Note: people rely on the undocumented ability to join multiple - # translate() results together via "|" to build large regexps matching - # "one of many" shell patterns. - while i < n: - assert inp[i] is STAR - i += 1 - if i == n: - add(".*") - break - assert inp[i] is not STAR - fixed = [] - while i < n and inp[i] is not STAR: - fixed.append(inp[i]) - i += 1 - fixed = "".join(fixed) - if i == n: - add(".*") - add(fixed) - else: - add(f"(?>.*?{fixed})") + add(_re_escape(c)) assert i == n - res = "".join(res) - return fr'(?s:{res})\Z' + return res, star_indices + + +def _join_translated_parts(parts, star_indices): + if not star_indices: + return fr'(?s:{"".join(parts)})\z' + iter_star_indices = iter(star_indices) + j = next(iter_star_indices) + buffer = parts[:j] # fixed pieces at the start + append, extend = buffer.append, buffer.extend + i = j + 1 + for j in iter_star_indices: + # Now deal with STAR fixed STAR fixed ... + # For an interior `STAR fixed` pairing, we want to do a minimal + # .*? match followed by `fixed`, with no possibility of backtracking. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. + append('(?>.*?') + extend(parts[i:j]) + append(')') + i = j + 1 + append('.*') + extend(parts[i:]) + res = ''.join(buffer) + return fr'(?s:{res})\z' diff --git a/Lib/fractions.py b/Lib/fractions.py index 88b418fe383..a497ee19935 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,6 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal import functools import math import numbers @@ -65,7 +64,7 @@ def _hash_algorithm(numerator, denominator): (?:\.(?P\d*|\d+(_\d+)*))? # an optional fractional part (?:E(?P[-+]?\d+(_\d+)*))? # and optional exponent ) - \s*\Z # and optional whitespace to finish + \s*\z # and optional whitespace to finish """, re.VERBOSE | re.IGNORECASE) @@ -139,6 +138,23 @@ def _round_to_figures(n, d, figures): return sign, significand, exponent +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + # Pattern for matching float-style format specifications; # supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. _FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" @@ -152,9 +168,13 @@ def _round_to_figures(n, d, figures): # A '0' that's *not* followed by another digit is parsed as a minimum width # rather than a zeropad flag. (?P0(?=[0-9]))? - (?P0|[1-9][0-9]*)? + (?P[0-9]+)? (?P[,_])? - (?:\.(?P0|[1-9][0-9]*))? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch @@ -227,7 +247,9 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -261,8 +283,8 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -288,6 +310,28 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. @@ -414,27 +458,42 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def __format__(self, format_spec, /): - """Format this fraction according to the given format specification.""" - - # Backwards compatiblility with existing formatting. - if not format_spec: - return str(self) + def _format_general(self, match): + """Helper method for __format__. + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ # Validate and parse the format specifier. - match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec) - if match is None: - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}" - ) - elif match["align"] is not None and match["zeropad"] is not None: - # Avoid the temptation to guess. - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}; " - "can't use explicit alignment when zero-padding" - ) + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" fill = match["fill"] or " " align = match["align"] or ">" pos_sign = "" if match["sign"] == "-" else match["sign"] @@ -444,11 +503,15 @@ def __format__(self, format_spec, /): minimumwidth = int(match["minimumwidth"] or "0") thousands_sep = match["thousands_sep"] precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" presentation_type = match["presentation_type"] trim_zeros = presentation_type in "gG" and not alternate_form trim_point = not alternate_form exponent_indicator = "E" if presentation_type in "EFG" else "e" + if align == '=' and fill == '0': + zeropad = True + # Round to get the digits we need, figure out where to place the point, # and decide whether to use scientific notation. 'point_pos' is the # relative to the _end_ of the digit string: that is, it's the number @@ -497,6 +560,9 @@ def __format__(self, format_spec, /): if trim_zeros: frac_part = frac_part.rstrip("0") separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) trailing = separator + frac_part + suffix # Do zero padding if required. @@ -530,7 +596,25 @@ def __format__(self, format_spec, /): else: # align == "=" return sign + padding + body - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -617,8 +701,8 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) + elif handle_complex and isinstance(b, complex): + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -630,8 +714,8 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + elif handle_complex and isinstance(a, numbers.Complex): + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -781,7 +865,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -789,16 +873,16 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -806,6 +890,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator @@ -825,11 +911,15 @@ def __pow__(a, b): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented - def __rpow__(b, a): + def __rpow__(b, a, modulo=None): """a ** b""" + if modulo is not None: + return NotImplemented if b._denominator == 1 and b._numerator >= 0: # If a is an int, keep it that way if possible. return a ** b._numerator diff --git a/Lib/ftplib.py b/Lib/ftplib.py index a56e0c30857..50771e8c17c 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. @@ -900,11 +900,17 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): def test(): '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + Usage: ftplib [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... - -d dir - -l list - -p password + Options: + -d increase debugging level + -r[file] set alternate ~/.netrc file + + Commands: + -l[dir] list directory + -d[dir] change the current directory + -p toggle passive and active mode + file retrieve the file and write it to stdout ''' if len(sys.argv) < 2: @@ -930,15 +936,14 @@ def test(): netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") + print("Could not open account file -- using anonymous login.", + file=sys.stderr) else: try: userid, acct, passwd = netrcobj.authenticators(host) - except KeyError: + except (KeyError, TypeError): # no account for host - sys.stderr.write( - "No account -- using anonymous login.") + print("No account -- using anonymous login.", file=sys.stderr) ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': @@ -951,7 +956,9 @@ def test(): ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) + sys.stdout.buffer.write, 1024) + sys.stdout.buffer.flush() + sys.stdout.flush() ftp.quit() diff --git a/Lib/functools.py b/Lib/functools.py index 4c1175b815d..df4660eef3f 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,23 +6,22 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -# Avoid importing types, so we can speedup import time -GenericAlias = type(list[int]) - ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ @@ -31,7 +30,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__', '__type_params__') + '__annotate__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -237,7 +236,7 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial], /) -> value + reduce(function, iterable, /[, initial]) -> value Apply a function of two arguments cumulatively to the items of an iterable, from left to right. @@ -265,63 +264,138 @@ def reduce(function, sequence, initial=_initial_missing): return value -try: - from _functools import reduce -except ImportError: - pass - ################################################################################ ### partial() argument application ################################################################################ -# Purely functional, no descriptor behaviour -class partial: - """New function with partial application of the given arguments - and keywords. + +class _PlaceholderType: + """The type of the Placeholder singleton. + + Used as a placeholder for partial arguments. """ + __instance = None + __slots__ = () - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") - def __new__(cls, func, /, *args, **keywords): + def __new__(cls): + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance + + def __repr__(self): + return 'Placeholder' + + def __reduce__(self): + return 'Placeholder' + +Placeholder = _PlaceholderType() + +def _partial_prepare_merger(args): + if not args: + return 0, None + nargs = len(args) + order = [] + j = nargs + for i, a in enumerate(args): + if a is Placeholder: + order.append(j) + j += 1 + else: + order.append(i) + phcount = j - nargs + merger = itemgetter(*order) if phcount else None + return phcount, merger + +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") + else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + for value in keywords.values(): + if value is Placeholder: + raise TypeError("Placeholder cannot be passed as a keyword argument") + if isinstance(func, base_cls): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" - if isinstance(func, partial): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ - self = super(partial, cls).__new__(cls) + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", + "__dict__", "__weakref__") - self.func = func - self.args = args - self.keywords = keywords - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) - - @recursive_repr() - def __repr__(self): - cls = type(self) - qualname = cls.__qualname__ - module = cls.__module__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + return self.func(*pto_args, *args, **keywords) def __get__(self, obj, objtype=None): if obj is None: return self - import warnings - warnings.warn('functools.partial will be a method descriptor in ' - 'future Python versions; wrap it in staticmethod() ' - 'if you want to preserve the old behavior', - FutureWarning, 2) - return self + return MethodType(self, obj) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -338,6 +412,10 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -350,56 +428,43 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self._phcount = phcount + self._merger = merger __class_getitem__ = classmethod(GenericAlias) try: - from _functools import partial + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - cls = type(self) - module = cls.__module__ - qualname = cls.__qualname__ - args = [repr(self.func)] - args.extend(map(repr, self.args)) - args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + __new__ = _partial_new + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partialmethod' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.__partialmethod__ = self return _method @@ -407,7 +472,7 @@ def _method(cls_or_self, /, *args, **keywords): def __get__(self, obj, cls=None): get = getattr(self.func, "__get__", None) result = None - if get is not None and not isinstance(self.func, partial): + if get is not None: new_func = get(obj, cls) if new_func is not self.func: # Assume __get__ returning something new indicates the @@ -454,22 +519,6 @@ def _unwrap_partialmethod(func): _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) -class _HashedSeq(list): - """ This class guarantees that hash() will be called no more than once - per element. This is important because the lru_cache() will hash - the key multiple times on a cache miss. - - """ - - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - def _make_key(args, kwds, typed, kwd_mark = (object(),), fasttypes = {int, str}, @@ -499,7 +548,7 @@ def _make_key(args, kwds, typed, key += tuple(type(v) for v in kwds.values()) elif len(key) == 1 and type(key[0]) in fasttypes: return key[0] - return _HashedSeq(key) + return key def lru_cache(maxsize=128, typed=False): """Least-recently-used cache decorator. @@ -835,7 +884,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -864,16 +913,11 @@ def dispatch(cls): dispatch_cache[cls] = impl return impl - def _is_union_type(cls): - from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} - def _is_valid_dispatch_type(cls): if isinstance(cls, type): return True - from typing import get_args - return (_is_union_type(cls) and - all(isinstance(arg, type) for arg in get_args(cls))) + return (isinstance(cls, UnionType) and + all(isinstance(arg, type) for arg in cls.__args__)) def register(cls, func=None): """generic_func.register(cls, func) -> func @@ -891,8 +935,8 @@ def register(cls, func=None): f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type." ) - ann = getattr(cls, '__annotations__', {}) - if not ann: + ann = getattr(cls, '__annotate__', None) + if ann is None: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " @@ -902,23 +946,27 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints - argname, cls = next(iter(get_type_hints(func).items())) + from annotationlib import Format, ForwardRef + argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): - if _is_union_type(cls): + if isinstance(cls, UnionType): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." ) + elif isinstance(cls, ForwardRef): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is an unresolved forward reference." + ) else: raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} is not a class." ) - if _is_union_type(cls): - from typing import get_args - - for arg in get_args(cls): + if isinstance(cls, UnionType): + for arg in cls.__args__: registry[arg] = func else: registry[cls] = func @@ -937,7 +985,7 @@ def wrapper(*args, **kw): registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper @@ -947,8 +995,7 @@ def wrapper(*args, **kw): class singledispatchmethod: """Single-dispatch generic method descriptor. - Supports wrapping existing descriptors and handles non-descriptor - callables as instance methods. + Supports wrapping existing descriptors. """ def __init__(self, func): @@ -966,24 +1013,77 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): - dispatch = self.dispatcher.dispatch - funcname = getattr(self.func, '__name__', 'singledispatchmethod method') - def _method(*args, **kwargs): - if not args: - raise TypeError(f'{funcname} requires at least ' - '1 positional argument') - return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) - - _method.__isabstractmethod__ = self.__isabstractmethod__ - _method.register = self.register - update_wrapper(_method, self.func) - - return _method + return _singledispatchmethod_get(self, obj, cls) @property def __isabstractmethod__(self): return getattr(self.func, '__isabstractmethod__', False) + def __repr__(self): + try: + name = self.func.__qualname__ + except AttributeError: + try: + name = self.func.__name__ + except AttributeError: + name = '?' + return f'' + +class _singledispatchmethod_get: + def __init__(self, unbound, obj, cls): + self._unbound = unbound + self._dispatch = unbound.dispatcher.dispatch + self._obj = obj + self._cls = cls + # Set instance attributes which cannot be handled in __getattr__() + # because they conflict with type descriptors. + func = unbound.func + try: + self.__module__ = func.__module__ + except AttributeError: + pass + try: + self.__doc__ = func.__doc__ + except AttributeError: + pass + + def __repr__(self): + try: + name = self.__qualname__ + except AttributeError: + try: + name = self.__name__ + except AttributeError: + name = '?' + if self._obj is not None: + return f'' + else: + return f'' + + def __call__(self, /, *args, **kwargs): + if not args: + funcname = getattr(self._unbound.func, '__name__', + 'singledispatchmethod method') + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + + def __getattr__(self, name): + # Resolve these attributes lazily to speed up creation of + # the _singledispatchmethod_get instance. + if name not in {'__name__', '__qualname__', '__isabstractmethod__', + '__annotations__', '__type_params__'}: + raise AttributeError + return getattr(self._unbound.func, name) + + @property + def __wrapped__(self): + return self._unbound.func + + @property + def register(self): + return self._unbound.register + ################################################################################ ### cached_property() - property result cached as instance attribute @@ -1035,3 +1135,31 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) + +def _warn_python_reduce_kwargs(py_reduce): + @wraps(py_reduce) + def wrapper(*args, **kwargs): + if 'function' in kwargs or 'sequence' in kwargs: + import os + import warnings + warnings.warn( + 'Calling functools.reduce with keyword arguments ' + '"function" or "sequence" ' + 'is deprecated in Python 3.14 and will be ' + 'forbidden in Python 3.16.', + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return py_reduce(*args, **kwargs) + return wrapper + +reduce = _warn_python_reduce_kwargs(reduce) +del _warn_python_reduce_kwargs + +# The import of the C accelerated version of reduce() has been moved +# here due to gh-121676. In Python 3.16, _warn_python_reduce_kwargs() +# should be removed and the import block should be moved back right +# after the definition of reduce(). +try: + from _functools import reduce +except ImportError: + pass diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d7..25f3e2439b3 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -24,21 +24,14 @@ # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' @@ -61,12 +54,14 @@ def getopt(args, shortopts, longopts = []): running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If + colon and options that accept an optional argument followed by + two colons (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign - ('='). + ('='). Options which accept an optional argument should be + followed by an equal sign and question mark ('=?'). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments @@ -105,7 +100,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. @@ -118,8 +113,13 @@ def gnu_getopt(args, shortopts, longopts = []): else: longopts = list(longopts) + return_in_order = False + if shortopts.startswith('-'): + shortopts = shortopts[1:] + all_options_first = False + return_in_order = True # Allow options after non-option arguments? - if shortopts.startswith('+'): + elif shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): @@ -133,8 +133,14 @@ def gnu_getopt(args, shortopts, longopts = []): break if args[0][:2] == '--': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: @@ -156,7 +162,7 @@ def do_longs(opts, opt, longopts, args): has_arg, opt = long_has_args(opt, longopts) if has_arg: - if optarg is None: + if optarg is None and has_arg != '?': if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] @@ -177,13 +183,19 @@ def long_has_args(opt, longopts): return False, opt elif opt + '=' in possibilities: return True, opt - # No exact match, so better be unique. + elif opt + '=?' in possibilities: + return '?', opt + # Possibilities must be unique to be accepted if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) + raise GetoptError( + _("option --%s not a unique prefix; possible options: %s") + % (opt, ", ".join(possibilities)), + opt, + ) assert len(possibilities) == 1 unique_match = possibilities[0] + if unique_match.endswith('=?'): + return '?', unique_match[:-2] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] @@ -192,8 +204,9 @@ def long_has_args(opt, longopts): def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': + has_arg = short_has_arg(opt, shortopts) + if has_arg: + if optstring == '' and has_arg != '?': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) @@ -207,7 +220,11 @@ def do_shorts(opts, optstring, shortopts, args): def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) + if not shortopts.startswith(':', i+1): + return False + if shortopts.startswith('::', i+1): + return '?' + return True raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': diff --git a/Lib/getpass.py b/Lib/getpass.py index bd0097ced94..3d9bb1f0d14 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -1,6 +1,7 @@ """Utilities to get a password and/or the current user name. -getpass(prompt[, stream]) - Prompt for a password, with echo turned off. +getpass(prompt[, stream[, echo_char]]) - Prompt for a password, with echo +turned off and optional keyboard feedback. getuser() - Get the user name from the environment or password database. GetPassWarning - This UserWarning is issued when getpass() cannot prevent @@ -25,13 +26,15 @@ class GetPassWarning(UserWarning): pass -def unix_getpass(prompt='Password: ', stream=None): +def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for a password, with echo turned off. Args: prompt: Written on stream to ask for the input. Default: 'Password: ' stream: A writable file object to display the prompt. Defaults to the tty. If no tty is available defaults to sys.stderr. + echo_char: A single ASCII character to mask input (e.g., '*'). + If None, input is hidden. Returns: The seKr3t input. Raises: @@ -40,6 +43,8 @@ def unix_getpass(prompt='Password: ', stream=None): Always restores terminal settings before returning. """ + _check_echo_char(echo_char) + passwd = None with contextlib.ExitStack() as stack: try: @@ -68,12 +73,16 @@ def unix_getpass(prompt='Password: ', stream=None): old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' + if echo_char: + new[3] &= ~termios.ICANON tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) - passwd = _raw_input(prompt, stream, input=input) + passwd = _raw_input(prompt, stream, input=input, + echo_char=echo_char) + finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() # issue7208 @@ -93,10 +102,11 @@ def unix_getpass(prompt='Password: ', stream=None): return passwd -def win_getpass(prompt='Password: ', stream=None): +def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for password with echo off, using Windows getwch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) + _check_echo_char(echo_char) for c in prompt: msvcrt.putwch(c) @@ -108,25 +118,48 @@ def win_getpass(prompt='Password: ', stream=None): if c == '\003': raise KeyboardInterrupt if c == '\b': + if echo_char and pw: + msvcrt.putwch('\b') + msvcrt.putwch(' ') + msvcrt.putwch('\b') pw = pw[:-1] else: pw = pw + c + if echo_char: + msvcrt.putwch(echo_char) msvcrt.putwch('\r') msvcrt.putwch('\n') return pw -def fallback_getpass(prompt='Password: ', stream=None): +def fallback_getpass(prompt='Password: ', stream=None, *, echo_char=None): + _check_echo_char(echo_char) import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) - return _raw_input(prompt, stream) + return _raw_input(prompt, stream, echo_char=echo_char) + +def _check_echo_char(echo_char): + # Single-character ASCII excluding control characters + if echo_char is None: + return + if not isinstance(echo_char, str): + raise TypeError("'echo_char' must be a str or None, not " + f"{type(echo_char).__name__}") + if not ( + len(echo_char) == 1 + and echo_char.isprintable() + and echo_char.isascii() + ): + raise ValueError("'echo_char' must be a single printable ASCII " + f"character, got: {echo_char!r}") -def _raw_input(prompt="", stream=None, input=None): + +def _raw_input(prompt="", stream=None, input=None, echo_char=None): # This doesn't save the string in the GNU readline history. if not stream: stream = sys.stderr @@ -143,6 +176,8 @@ def _raw_input(prompt="", stream=None, input=None): stream.write(prompt) stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. + if echo_char: + return _readline_with_echo_char(stream, input, echo_char) line = input.readline() if not line: raise EOFError @@ -151,6 +186,35 @@ def _raw_input(prompt="", stream=None, input=None): return line +def _readline_with_echo_char(stream, input, echo_char): + passwd = "" + eof_pressed = False + while True: + char = input.read(1) + if char == '\n' or char == '\r': + break + elif char == '\x03': + raise KeyboardInterrupt + elif char == '\x7f' or char == '\b': + if passwd: + stream.write("\b \b") + stream.flush() + passwd = passwd[:-1] + elif char == '\x04': + if eof_pressed: + break + else: + eof_pressed = True + elif char == '\x00': + continue + else: + passwd += char + stream.write(echo_char) + stream.flush() + eof_pressed = False + return passwd + + def getuser(): """Get the username from the environment or password database. diff --git a/Lib/gettext.py b/Lib/gettext.py index 62cff81b7b3..6c11ab2b1eb 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -41,14 +41,10 @@ # to do binary searches and lazy initializations. Or you might want to use # the undocumented double-hash algorithm for .mo files with hash tables, but # you'll need to study the GNU gettext code to do this. -# -# - Support Solaris .mo file formats. Unfortunately, we've been unable to -# find this format documented anywhere. import operator import os -import re import sys @@ -70,22 +66,26 @@ # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y -_token_pattern = re.compile(r""" - (?P[ \t]+) | # spaces and horizontal tabs - (?P[0-9]+\b) | # decimal integer - (?Pn\b) | # only n is allowed - (?P[()]) | - (?P[-*/%+?:]|[>, - # <=, >=, ==, !=, &&, ||, - # ? : - # unary and bitwise ops - # not allowed - (?P\w+|.) # invalid token - """, re.VERBOSE|re.DOTALL) - +_token_pattern = None def _tokenize(plural): - for mo in re.finditer(_token_pattern, plural): + global _token_pattern + if _token_pattern is None: + import re + _token_pattern = re.compile(r""" + (?P[ \t]+) | # spaces and horizontal tabs + (?P[0-9]+\b) | # decimal integer + (?Pn\b) | # only n is allowed + (?P[()]) | + (?P[-*/%+?:]|[>, + # <=, >=, ==, !=, &&, ||, + # ? : + # unary and bitwise ops + # not allowed + (?P\w+|.) # invalid token + """, re.VERBOSE|re.DOTALL) + + for mo in _token_pattern.finditer(plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue @@ -648,7 +648,7 @@ def npgettext(context, msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/glob.py b/Lib/glob.py index c506e0e2157..f1a87c82fc5 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -22,6 +22,9 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns by default. + The order of the returned list is undefined. Sort it if you need a + particular order. + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden directories. @@ -40,6 +43,9 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns. + The order of the returned paths is undefined. Sort them if you need a + particular order. + If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ @@ -312,24 +318,24 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): if part: if not include_hidden and part[0] in '*?': results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0]) if idx < last_part_idx: results.append(any_sep) res = ''.join(results) - return fr'(?s:{res})\Z' + return fr'(?s:{res})\z' @functools.lru_cache(maxsize=512) -def _compile_pattern(pat, sep, case_sensitive, recursive=True): +def _compile_pattern(pat, seps, case_sensitive, recursive=True): """Compile given glob pattern to a re.Pattern object (observing case sensitivity).""" flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + regex = translate(pat, recursive=recursive, include_hidden=True, seps=seps) return re.compile(regex, flags=flags).match -class _Globber: - """Class providing shell-style pattern matching and globbing. +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. """ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): @@ -338,34 +344,31 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): self.case_pedantic = case_pedantic self.recursive = recursive - # Low-level methods - - lstat = operator.methodcaller('lstat') - add_slash = operator.methodcaller('joinpath', '') + # Abstract methods @staticmethod - def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). + def lexists(path): + """Implements os.path.lexists(). """ - return contextlib.nullcontext(path.iterdir()) + raise NotImplementedError @staticmethod - def concat_path(path, text): - """Appends text to the given path. + def scandir(path): + """Like os.scandir(), but generates (entry, name, path) tuples. """ - return path.with_segments(path._raw_path + text) + raise NotImplementedError @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). + def concat_path(path, text): + """Implements path concatenation. """ - return entry + raise NotImplementedError # High-level methods - def compile(self, pat): - return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + def compile(self, pat, altsep=None): + seps = (self.sep, altsep) if altsep else self.sep + return _compile_pattern(pat, seps, self.case_sensitive, self.recursive) def selector(self, parts): """Returns a function that selects from a given path, walking and @@ -387,10 +390,12 @@ def selector(self, parts): def special_selector(self, part, parts): """Returns a function that selects special children of the given path. """ + if parts: + part += self.sep select_next = self.selector(parts) def select_special(path, exists=False): - path = self.concat_path(self.add_slash(path), part) + path = self.concat_path(path, part) return select_next(path, exists) return select_special @@ -400,14 +405,16 @@ def literal_selector(self, part, parts): # Optimization: consume and join any subsequent literal parts here, # rather than leaving them for the next selector. This reduces the - # number of string concatenation operations and calls to add_slash(). + # number of string concatenation operations. while parts and magic_check.search(parts[-1]) is None: part += self.sep + parts.pop() + if parts: + part += self.sep select_next = self.selector(parts) def select_literal(path, exists=False): - path = self.concat_path(self.add_slash(path), part) + path = self.concat_path(path, part) return select_next(path, exists=False) return select_literal @@ -423,23 +430,19 @@ def wildcard_selector(self, part, parts): def select_wildcard(path, exists=False): try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) + entries = self.scandir(path) except OSError: pass else: - for entry in entries: - if match is None or match(entry.name): + for entry, entry_name, entry_path in entries: + if match is None or match(entry_name): if dir_only: try: if not entry.is_dir(): continue except OSError: continue - entry_path = self.parse_entry(entry) - if dir_only: + entry_path = self.concat_path(entry_path, self.sep) yield from select_next(entry_path, exists=True) else: yield entry_path @@ -469,7 +472,6 @@ def recursive_selector(self, part, parts): select_next = self.selector(parts) def select_recursive(path, exists=False): - path = self.add_slash(path) match_pos = len(str(path)) if match is None or match(str(path), match_pos): yield from select_next(path, exists) @@ -480,14 +482,11 @@ def select_recursive(path, exists=False): def select_recursive_step(stack, match_pos): path = stack.pop() try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) + entries = self.scandir(path) except OSError: pass else: - for entry in entries: + for entry, _entry_name, entry_path in entries: is_dir = False try: if entry.is_dir(follow_symlinks=follow_symlinks): @@ -496,8 +495,10 @@ def select_recursive_step(stack, match_pos): pass if is_dir or not dir_only: - entry_path = self.parse_entry(entry) - if match is None or match(str(entry_path), match_pos): + entry_path_str = str(entry_path) + if dir_only: + entry_path = self.concat_path(entry_path, self.sep) + if match is None or match(entry_path_str, match_pos): if dir_only: yield from select_next(entry_path, exists=True) else: @@ -516,30 +517,37 @@ def select_exists(self, path, exists=False): # Optimization: this path is already known to exist, e.g. because # it was returned from os.scandir(), so we skip calling lstat(). yield path - else: - try: - self.lstat(path) - yield path - except OSError: - pass + elif self.lexists(path): + yield path -class _StringGlobber(_Globber): - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) concat_path = operator.add - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' + @staticmethod + def scandir(path): + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with os.scandir(path) as scandir_it: + entries = list(scandir_it) + return ((entry, entry.name, entry.path) for entry in entries) + + +class _PathGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for pathlib paths. + """ + + @staticmethod + def lexists(path): + return path.info.exists(follow_symlinks=False) + + @staticmethod + def scandir(path): + return ((child.info, child.name, child) for child in path.iterdir()) + + @staticmethod + def concat_path(path, text): + return path.with_segments(str(path) + text) diff --git a/Lib/graphlib.py b/Lib/graphlib.py index 9512865a8e5..7961c9c5cac 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -90,20 +90,24 @@ def prepare(self): still be used to obtain as many nodes as possible until cycles block more progress. After a call to this function, the graph cannot be modified and therefore no more nodes can be added using "add". + + Raise ValueError if nodes have already been passed out of the sorter. + """ - if self._ready_nodes is not None: - raise ValueError("cannot prepare() more than once") + if self._npassedout > 0: + raise ValueError("cannot prepare() after starting sort") - self._ready_nodes = [ - i.node for i in self._node2info.values() if i.npredecessors == 0 - ] + if self._ready_nodes is None: + self._ready_nodes = [ + i.node for i in self._node2info.values() if i.npredecessors == 0 + ] # ready_nodes is set before we look for cycles on purpose: # if the user wants to catch the CycleError, that's fine, # they can continue using the instance to grab as many # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: - raise CycleError(f"nodes are in a cycle", cycle) + raise CycleError("nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. diff --git a/Lib/gzip.py b/Lib/gzip.py index a550c20a7a0..c00f51858de 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -5,7 +5,6 @@ # based on Andrew Kuchling's minigzip.py distributed with the zlib module -import _compression import builtins import io import os @@ -14,6 +13,7 @@ import time import weakref import zlib +from compression._common import _streams __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"] @@ -144,7 +144,7 @@ def writable(self): return True -class GzipFile(_compression.BaseStream): +class GzipFile(_streams.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -193,6 +193,11 @@ def __init__(self, filename=None, mode=None, """ + # Ensure attributes exist at __del__ + self.mode = None + self.fileobj = None + self._buffer = None + if mode and ('t' in mode or 'U' in mode): raise ValueError("Invalid mode: {!r}".format(mode)) if mode and 'b' not in mode: @@ -332,11 +337,15 @@ def _write_raw(self, data): return length - def read(self, size=-1): - self._check_not_closed() + def _check_read(self, caller): if self.mode != READ: import errno - raise OSError(errno.EBADF, "read() on write-only GzipFile object") + msg = f"{caller}() on write-only GzipFile object" + raise OSError(errno.EBADF, msg) + + def read(self, size=-1): + self._check_not_closed() + self._check_read("read") return self._buffer.read(size) def read1(self, size=-1): @@ -344,19 +353,25 @@ def read1(self, size=-1): Reads up to a buffer's worth of data if size is negative.""" self._check_not_closed() - if self.mode != READ: - import errno - raise OSError(errno.EBADF, "read1() on write-only GzipFile object") + self._check_read("read1") if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) + def readinto(self, b): + self._check_not_closed() + self._check_read("readinto") + return self._buffer.readinto(b) + + def readinto1(self, b): + self._check_not_closed() + self._check_read("readinto1") + return self._buffer.readinto1(b) + def peek(self, n): self._check_not_closed() - if self.mode != READ: - import errno - raise OSError(errno.EBADF, "peek() on write-only GzipFile object") + self._check_read("peek") return self._buffer.peek(n) @property @@ -365,7 +380,9 @@ def closed(self): def close(self): fileobj = self.fileobj - if fileobj is None or self._buffer.closed: + if fileobj is None: + return + if self._buffer is None or self._buffer.closed: return try: if self.mode == WRITE: @@ -445,6 +462,13 @@ def readline(self, size=-1): self._check_not_closed() return self._buffer.readline(size) + def __del__(self): + if self.mode == WRITE and not self.closed: + import warnings + warnings.warn("unclosed GzipFile", + ResourceWarning, source=self, stacklevel=2) + + super().__del__() def _read_exact(fp, n): '''Read exactly *n* bytes from `fp` @@ -499,7 +523,7 @@ def _read_gzip_header(fp): return last_mtime -class _GzipReader(_compression.DecompressReader): +class _GzipReader(_streams.DecompressReader): def __init__(self, fp): super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor, wbits=-zlib.MAX_WBITS) @@ -597,12 +621,12 @@ def _rewind(self): self._new_member = True -def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None): +def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0): """Compress data in one shot and return the compressed string. compresslevel sets the compression level in range of 0-9. - mtime can be used to set the modification time. The modification time is - set to the current time by default. + mtime can be used to set the modification time. + The modification time is set to 0 by default, for reproducibility. """ # Wbits=31 automatically includes a gzip header and trailer. gzip_data = zlib.compress(data, level=compresslevel, wbits=31) @@ -643,7 +667,9 @@ def main(): from argparse import ArgumentParser parser = ArgumentParser(description= "A simple command line interface for the gzip module: act like gzip, " - "but do not delete the input file.") + "but do not delete the input file.", + color=True, + ) group = parser.add_mutually_exclusive_group() group.add_argument('--fast', action='store_true', help='compress faster') group.add_argument('--best', action='store_true', help='compress better') diff --git a/Lib/hashlib.py b/Lib/hashlib.py index e5f81d754ac..0e9bd98aa1f 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -33,7 +33,7 @@ - hexdigest(): Like digest() except the digest is returned as a string of double length, containing only hexadecimal digits. - copy(): Return a copy (clone) of the hash object. This can be used to - efficiently compute the digests of datas that share a common + efficiently compute the digests of data that share a common initial substring. For example, to obtain the digest of the byte string 'Nobody inspects the @@ -65,7 +65,7 @@ algorithms_available = set(__always_supported) __all__ = __always_supported + ('new', 'algorithms_guaranteed', - 'algorithms_available', 'pbkdf2_hmac', 'file_digest') + 'algorithms_available', 'file_digest') __builtin_constructor_cache = {} @@ -92,13 +92,13 @@ def __get_builtin_constructor(name): import _md5 cache['MD5'] = cache['md5'] = _md5.md5 elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}: - import _sha256 - cache['SHA224'] = cache['sha224'] = _sha256.sha224 - cache['SHA256'] = cache['sha256'] = _sha256.sha256 + import _sha2 + cache['SHA224'] = cache['sha224'] = _sha2.sha224 + cache['SHA256'] = cache['sha256'] = _sha2.sha256 elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}: - import _sha512 - cache['SHA384'] = cache['sha384'] = _sha512.sha384 - cache['SHA512'] = cache['sha512'] = _sha512.sha512 + import _sha2 + cache['SHA384'] = cache['sha384'] = _sha2.sha384 + cache['SHA512'] = cache['sha512'] = _sha2.sha512 elif name in {'blake2b', 'blake2s'}: import _blake2 cache['blake2b'] = _blake2.blake2b @@ -141,38 +141,37 @@ def __get_openssl_constructor(name): return __get_builtin_constructor(name) -def __py_new(name, data=b'', **kwargs): +def __py_new(name, *args, **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) -def __hash_new(name, data=b'', **kwargs): +def __hash_new(name, *args, **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ if name in __block_openssl_constructor: # Prefer our builtin blake2 implementation. - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) try: - return _hashlib.new(name, data, **kwargs) + return _hashlib.new(name, *args, **kwargs) except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. - return __get_builtin_constructor(name)(data) + return __get_builtin_constructor(name)(*args, **kwargs) try: import _hashlib new = __hash_new __get_hash = __get_openssl_constructor - # TODO: RUSTPYTHON set in _hashlib instance PyFrozenSet algorithms_available - '''algorithms_available = algorithms_available.union( - _hashlib.openssl_md_meth_names)''' + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: _hashlib = None new = __py_new @@ -181,76 +180,14 @@ def __hash_new(name, data=b'', **kwargs): try: # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA from _hashlib import pbkdf2_hmac + __all__ += ('pbkdf2_hmac',) except ImportError: - from warnings import warn as _warn - _trans_5C = bytes((x ^ 0x5C) for x in range(256)) - _trans_36 = bytes((x ^ 0x36) for x in range(256)) - - def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): - """Password based key derivation function 2 (PKCS #5 v2.0) - - This Python implementations based on the hmac module about as fast - as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster - for long passwords. - """ - _warn( - "Python implementation of pbkdf2_hmac() is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - if not isinstance(hash_name, str): - raise TypeError(hash_name) - - if not isinstance(password, (bytes, bytearray)): - password = bytes(memoryview(password)) - if not isinstance(salt, (bytes, bytearray)): - salt = bytes(memoryview(salt)) - - # Fast inline HMAC implementation - inner = new(hash_name) - outer = new(hash_name) - blocksize = getattr(inner, 'block_size', 64) - if len(password) > blocksize: - password = new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - dkey = b'' - loop = 1 - from_bytes = int.from_bytes - while len(dkey) < dklen: - prev = prf(salt + loop.to_bytes(4)) - # endianness doesn't matter here as long to / from use the same - rkey = from_bytes(prev) - for i in range(iterations - 1): - prev = prf(prev) - # rkey = rkey ^ prev - rkey ^= from_bytes(prev) - loop += 1 - dkey += rkey.to_bytes(inner.digest_size) - - return dkey[:dklen] + pass + try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt + from _hashlib import scrypt # noqa: F401 except ImportError: pass @@ -294,6 +231,8 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): view = memoryview(buf) while True: size = fileobj.readinto(buf) + if size is None: + raise BlockingIOError("I/O operation would block.") if size == 0: break # EOF digestobj.update(view[:size]) diff --git a/Lib/heapq.py b/Lib/heapq.py index 2fd9d1ff4bf..17f62dd2d58 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -42,7 +42,7 @@ property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory -representation for a tournament. The numbers below are `k', not a[k]: +representation for a tournament. The numbers below are 'k', not a[k]: 0 @@ -55,7 +55,7 @@ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications @@ -78,7 +78,7 @@ not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled -time. When an event schedule other events for execution, they are +time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). @@ -91,14 +91,14 @@ Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted -sequences, which size is usually related to the amount of CPU memory), +sequences, whose size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way -to that. If, using all the memory available to hold a tournament, you -replace and percolate items that happen to fit the current run, you'll -produce runs which are twice the size of the memory for random input, -and much better for input fuzzily ordered. +to achieve that. If, using all the memory available to hold a +tournament, you replace and percolate items that happen to fit the +current run, you'll produce runs which are twice the size of the +memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over @@ -110,7 +110,7 @@ effective! In a word, heaps are useful memory structures to know. I use them in -a few applications, and I think it is good to keep a `heap' module +a few applications, and I think it is good to keep a 'heap' module around. :-) -------------------- @@ -126,8 +126,9 @@ From all times, sorting has always been a Great Art! :-) """ -__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', - 'nlargest', 'nsmallest', 'heappushpop'] +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'heappushpop', + 'heappush_max', 'heappop_max', 'heapify_max', 'heapreplace_max', + 'heappushpop_max', 'nlargest', 'nsmallest', 'merge'] def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" @@ -178,7 +179,7 @@ def heapify(x): for i in reversed(range(n//2)): _siftup(x, i) -def _heappop_max(heap): +def heappop_max(heap): """Maxheap version of a heappop.""" lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: @@ -188,19 +189,32 @@ def _heappop_max(heap): return returnitem return lastelt -def _heapreplace_max(heap, item): +def heapreplace_max(heap, item): """Maxheap version of a heappop followed by a heappush.""" returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item _siftup_max(heap, 0) return returnitem -def _heapify_max(x): +def heappush_max(heap, item): + """Maxheap version of a heappush.""" + heap.append(item) + _siftdown_max(heap, 0, len(heap)-1) + +def heappushpop_max(heap, item): + """Maxheap fast version of a heappush followed by a heappop.""" + if heap and item < heap[0]: + item, heap[0] = heap[0], item + _siftup_max(heap, 0) + return item + +def heapify_max(x): """Transform list into a maxheap, in-place, in O(len(x)) time.""" n = len(x) for i in reversed(range(n//2)): _siftup_max(x, i) + # 'heap' is a heap at all indices >= startpos, except possibly for pos. pos # is the index of a leaf with a possibly out-of-order value. Restore the # heap invariant. @@ -335,9 +349,9 @@ def merge(*iterables, key=None, reverse=False): h_append = h.append if reverse: - _heapify = _heapify_max - _heappop = _heappop_max - _heapreplace = _heapreplace_max + _heapify = heapify_max + _heappop = heappop_max + _heapreplace = heapreplace_max direction = -1 else: _heapify = heapify @@ -490,10 +504,10 @@ def nsmallest(n, iterable, key=None): result = [(elem, i) for i, elem in zip(range(n), it)] if not result: return result - _heapify_max(result) + heapify_max(result) top = result[0][0] order = n - _heapreplace = _heapreplace_max + _heapreplace = heapreplace_max for elem in it: if elem < top: _heapreplace(result, (elem, order)) @@ -507,10 +521,10 @@ def nsmallest(n, iterable, key=None): result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] if not result: return result - _heapify_max(result) + heapify_max(result) top = result[0][0] order = n - _heapreplace = _heapreplace_max + _heapreplace = heapreplace_max for elem in it: k = key(elem) if k < top: @@ -583,19 +597,13 @@ def nlargest(n, iterable, key=None): from _heapq import * except ImportError: pass -try: - from _heapq import _heapreplace_max -except ImportError: - pass -try: - from _heapq import _heapify_max -except ImportError: - pass -try: - from _heapq import _heappop_max -except ImportError: - pass +# For backwards compatibility +_heappop_max = heappop_max +_heapreplace_max = heapreplace_max +_heappush_max = heappush_max +_heappushpop_max = heappushpop_max +_heapify_max = heapify_max if __name__ == "__main__": diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db95..2d6016cda11 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -3,7 +3,6 @@ Implements the HMAC algorithm as described by RFC 2104. """ -import warnings as _warnings try: import _hashlib as _hashopenssl except ImportError: @@ -14,7 +13,10 @@ compare_digest = _hashopenssl.compare_digest _functype = type(_hashopenssl.openssl_sha256) # builtin type -import hashlib as _hashlib +try: + import _hmac +except ImportError: + _hmac = None trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -24,11 +26,27 @@ digest_size = None +def _get_digest_constructor(digest_like): + if callable(digest_like): + return digest_like + if isinstance(digest_like, str): + def digest_wrapper(d=b''): + import hashlib + return hashlib.new(digest_like, d) + else: + def digest_wrapper(d=b''): + return digest_like.new(d) + return digest_wrapper + + class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. This supports the API for Cryptographic Hash Functions (PEP 247). """ + + # Note: self.blocksize is the default blocksize; self.block_size + # is effective block size as well as the public API attribute. blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( @@ -50,31 +68,47 @@ def __init__(self, key, msg=None, digestmod=''): """ if not isinstance(key, (bytes, bytearray)): - raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) + raise TypeError(f"key: expected bytes or bytearray, " + f"but got {type(key).__name__!r}") if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") + self.__init(key, msg, digestmod) + + def __init(self, key, msg, digestmod): if _hashopenssl and isinstance(digestmod, (str, _functype)): try: - self._init_hmac(key, msg, digestmod) - except _hashopenssl.UnsupportedDigestmodError: - self._init_old(key, msg, digestmod) - else: - self._init_old(key, msg, digestmod) + self._init_openssl_hmac(key, msg, digestmod) + return + except _hashopenssl.UnsupportedDigestmodError: # pragma: no cover + pass + if _hmac and isinstance(digestmod, str): + try: + self._init_builtin_hmac(key, msg, digestmod) + return + except _hmac.UnknownHashError: # pragma: no cover + pass + self._init_old(key, msg, digestmod) - def _init_hmac(self, key, msg, digestmod): + def _init_openssl_hmac(self, key, msg, digestmod): self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + _init_hmac = _init_openssl_hmac # for backward compatibility (if any) + + def _init_builtin_hmac(self, key, msg, digestmod): + self._hmac = _hmac.new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined self.digest_size = self._hmac.digest_size self.block_size = self._hmac.block_size def _init_old(self, key, msg, digestmod): - if callable(digestmod): - digest_cons = digestmod - elif isinstance(digestmod, str): - digest_cons = lambda d=b'': _hashlib.new(digestmod, d) - else: - digest_cons = lambda d=b'': digestmod.new(d) + import warnings + + digest_cons = _get_digest_constructor(digestmod) self._hmac = None self._outer = digest_cons() @@ -84,21 +118,19 @@ def _init_old(self, key, msg, digestmod): if hasattr(self._inner, 'block_size'): blocksize = self._inner.block_size if blocksize < 16: - _warnings.warn('block_size of %d seems too small; using our ' - 'default of %d.' % (blocksize, self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn(f"block_size of {blocksize} seems too small; " + f"using our default of {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover else: - _warnings.warn('No block_size attribute on given digest object; ' - 'Assuming %d.' % (self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn("No block_size attribute on given digest object; " + f"Assuming {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover if len(key) > blocksize: key = digest_cons(key).digest() - # self.blocksize is the default blocksize. self.block_size is - # effective block size as well as the public API attribute. self.block_size = blocksize key = key.ljust(blocksize, b'\0') @@ -127,6 +159,7 @@ def copy(self): # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) other.digest_size = self.digest_size + other.block_size = self.block_size if self._hmac: other._hmac = self._hmac.copy() other._inner = other._outer = None @@ -164,6 +197,7 @@ def hexdigest(self): h = self._current() return h.hexdigest() + def new(key, msg=None, digestmod=''): """Create a new hashing object and return it. @@ -193,25 +227,41 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if _hashopenssl is not None and isinstance(digest, (str, _functype)): + if _hashopenssl and isinstance(digest, (str, _functype)): try: return _hashopenssl.hmac_digest(key, msg, digest) + except OverflowError: + # OpenSSL's HMAC limits the size of the key to INT_MAX. + # Instead of falling back to HACL* implementation which + # may still not be supported due to a too large key, we + # directly switch to the pure Python fallback instead + # even if we could have used streaming HMAC for small keys + # but large messages. + return _compute_digest_fallback(key, msg, digest) except _hashopenssl.UnsupportedDigestmodError: pass - if callable(digest): - digest_cons = digest - elif isinstance(digest, str): - digest_cons = lambda d=b'': _hashlib.new(digest, d) - else: - digest_cons = lambda d=b'': digest.new(d) + if _hmac and isinstance(digest, str): + try: + return _hmac.compute_digest(key, msg, digest) + except (OverflowError, _hmac.UnknownHashError): + # HACL* HMAC limits the size of the key to UINT32_MAX + # so we fallback to the pure Python implementation even + # if streaming HMAC may have been used for small keys + # and large messages. + pass + + return _compute_digest_fallback(key, msg, digest) + +def _compute_digest_fallback(key, msg, digest): + digest_cons = _get_digest_constructor(digest) inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) if len(key) > blocksize: key = digest_cons(key).digest() - key = key + b'\x00' * (blocksize - len(key)) + key = key.ljust(blocksize, b'\0') inner.update(key.translate(trans_36)) outer.update(key.translate(trans_5C)) inner.update(msg) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 5d7050dad23..80fb8c3f929 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -24,6 +24,7 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') @@ -127,17 +128,25 @@ class HTMLParser(_markupbase.ParserBase): argument. """ - CDATA_CONTENT_ELEMENTS = ("script", "style") + # See the HTML5 specs section "13.4 Parsing HTML fragments". + # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments + # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode + CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes") RCDATA_CONTENT_ELEMENTS = ("textarea", "title") - def __init__(self, *, convert_charrefs=True): + def __init__(self, *, convert_charrefs=True, scripting=False): """Initialize and reset this instance. - If convert_charrefs is True (the default), all character references + If convert_charrefs is true (the default), all character references are automatically converted to the corresponding Unicode characters. + + If *scripting* is false (the default), the content of the + ``noscript`` element is parsed normally; if it's true, + it's returned as is without being parsed. """ super().__init__() self.convert_charrefs = convert_charrefs + self.scripting = scripting self.reset() def reset(self): @@ -172,7 +181,9 @@ def get_starttag_text(self): def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() self._escapable = escapable - if escapable and not self.convert_charrefs: + if self.cdata_elem == 'plaintext': + self.interesting = re.compile(r'\z') + elif escapable and not self.convert_charrefs: self.interesting = re.compile(r'&|])' % self.cdata_elem, re.IGNORECASE|re.ASCII) else: @@ -294,10 +305,20 @@ def goahead(self, end): k = k - 1 i = self.updatepos(i, k) continue + match = incomplete_charref.match(rawdata, i) + if match: + if end: + self.handle_charref(rawdata[i+2:]) + i = self.updatepos(i, n) + break + # incomplete + break + elif i + 3 < n: # larger than "&#x" + # not the end of the buffer, and can't be confused + # with some other construct + self.handle_data("&#") + i = self.updatepos(i, i + 2) else: - if ";" in rawdata[i:]: # bail by consuming &# - self.handle_data(rawdata[i:i+2]) - i = self.updatepos(i, i+2) break elif startswith('&', i): match = entityref.match(rawdata, i) @@ -311,15 +332,13 @@ def goahead(self, end): continue match = incomplete.match(rawdata, i) if match: - # match.group() will contain at least 2 chars - if end and match.group() == rawdata[i:]: - k = match.end() - if k <= i: - k = n - i = self.updatepos(i, i + 1) + if end: + self.handle_entityref(rawdata[i+1:]) + i = self.updatepos(i, n) + break # incomplete break - elif (i + 1) < n: + elif i + 1 < n: # not the end of the buffer, and can't be confused # with some other construct self.handle_data("&") @@ -444,8 +463,10 @@ def parse_starttag(self, i): self.handle_startendtag(tag, attrs) else: self.handle_starttag(tag, attrs) - if tag in self.CDATA_CONTENT_ELEMENTS: - self.set_cdata_mode(tag) + if (tag in self.CDATA_CONTENT_ELEMENTS or + (self.scripting and tag == "noscript") or + tag == "plaintext"): + self.set_cdata_mode(tag, escapable=False) elif tag in self.RCDATA_CONTENT_ELEMENTS: self.set_cdata_mode(tag, escapable=True) return endpos diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index bf8d7d68868..17a47b180e5 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -1,14 +1,15 @@ -from enum import IntEnum +from enum import StrEnum, IntEnum, _simple_enum -__all__ = ['HTTPStatus'] +__all__ = ['HTTPStatus', 'HTTPMethod'] -class HTTPStatus(IntEnum): +@_simple_enum(IntEnum) +class HTTPStatus: """HTTP status codes and reason phrases Status codes from the following RFCs are all observed: - * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 9110: HTTP Semantics, obsoletes 7231, which obsoleted 2616 * RFC 6585: Additional HTTP Status Codes * RFC 3229: Delta encoding in HTTP * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 @@ -25,11 +26,30 @@ class HTTPStatus(IntEnum): def __new__(cls, value, phrase, description=''): obj = int.__new__(cls, value) obj._value_ = value - obj.phrase = phrase obj.description = description return obj + @property + def is_informational(self): + return 100 <= self <= 199 + + @property + def is_success(self): + return 200 <= self <= 299 + + @property + def is_redirection(self): + return 300 <= self <= 399 + + @property + def is_client_error(self): + return 400 <= self <= 499 + + @property + def is_server_error(self): + return 500 <= self <= 599 + # informational CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', @@ -94,22 +114,25 @@ def __new__(cls, value, phrase, description=''): 'Client must specify Content-Length') PRECONDITION_FAILED = (412, 'Precondition Failed', 'Precondition in headers is false') - REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large', - 'Entity is too large') - REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long', + CONTENT_TOO_LARGE = (413, 'Content Too Large', + 'Content is too large') + REQUEST_ENTITY_TOO_LARGE = CONTENT_TOO_LARGE + URI_TOO_LONG = (414, 'URI Too Long', 'URI is too long') + REQUEST_URI_TOO_LONG = URI_TOO_LONG UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', 'Entity body in unsupported format') - REQUESTED_RANGE_NOT_SATISFIABLE = (416, - 'Requested Range Not Satisfiable', + RANGE_NOT_SATISFIABLE = (416, 'Range Not Satisfiable', 'Cannot satisfy request range') + REQUESTED_RANGE_NOT_SATISFIABLE = RANGE_NOT_SATISFIABLE EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', 'Server refuses to brew coffee because it is a teapot.') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity' + UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT LOCKED = 423, 'Locked' FAILED_DEPENDENCY = 424, 'Failed Dependency' TOO_EARLY = 425, 'Too Early' @@ -148,3 +171,32 @@ def __new__(cls, value, phrase, description=''): NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') + + +@_simple_enum(StrEnum) +class HTTPMethod: + """HTTP methods and descriptions + + Methods from the following RFCs are all observed: + + * RFC 9110: HTTP Semantics, obsoletes 7231, which obsoleted 2616 + * RFC 5789: PATCH Method for HTTP + """ + def __new__(cls, value, description): + obj = str.__new__(cls, value) + obj._value_ = value + obj.description = description + return obj + + def __repr__(self): + return "<%s.%s>" % (self.__class__.__name__, self._name_) + + CONNECT = 'CONNECT', 'Establish a connection to the server.' + DELETE = 'DELETE', 'Remove the target.' + GET = 'GET', 'Retrieve the target.' + HEAD = 'HEAD', 'Same as GET, but only retrieve the status line and header section.' + OPTIONS = 'OPTIONS', 'Describe the communication options for the target.' + PATCH = 'PATCH', 'Apply partial modifications to a target.' + POST = 'POST', 'Perform target-specific processing with the request payload.' + PUT = 'PUT', 'Replace the target with the request payload.' + TRACE = 'TRACE', 'Perform a message loop-back test along the path to the target.' diff --git a/Lib/http/client.py b/Lib/http/client.py index a6ab135b2c3..dd5f4136e9e 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -111,6 +111,11 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -172,6 +177,13 @@ def _encode(data, name='data'): "if you want to send it encoded in UTF-8." % (name.title(), data[err.start:err.end], name)) from None +def _strip_ipv6_iface(enc_name: bytes) -> bytes: + """Remove interface scope from IPv6 address.""" + enc_name, percent, _ = enc_name.partition(b"%") + if percent: + assert enc_name.startswith(b'['), enc_name + enc_name += b']' + return enc_name class HTTPMessage(email.message.Message): # XXX The only usage of this method is in @@ -221,8 +233,9 @@ def _read_headers(fp): break return headers -def parse_headers(fp, _class=HTTPMessage): - """Parses only RFC2822 headers from a file pointer. +def _parse_header_lines(header_lines, _class=HTTPMessage): + """ + Parses only RFC 5322 headers from header lines. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes @@ -231,10 +244,15 @@ def parse_headers(fp, _class=HTTPMessage): to parse. """ - headers = _read_headers(fp) - hstring = b''.join(headers).decode('iso-8859-1') + hstring = b''.join(header_lines).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) +def parse_headers(fp, _class=HTTPMessage): + """Parses only RFC 5322 headers from a file pointer.""" + + headers = _read_headers(fp) + return _parse_header_lines(headers, _class) + class HTTPResponse(io.BufferedIOBase): @@ -448,6 +466,7 @@ def isclosed(self): return self.fp is None def read(self, amt=None): + """Read and return the response body, or up to the next amt bytes.""" if self.fp is None: return b"" @@ -458,7 +477,7 @@ def read(self, amt=None): if self.chunked: return self._read_chunked(amt) - if amt is not None: + if amt is not None and amt >= 0: if self.length is not None and amt > self.length: # clip the read to the "end of response" amt = self.length @@ -576,13 +595,11 @@ def _get_chunk_left(self): def _read_chunked(self, amt=None): assert self.chunked != _UNKNOWN + if amt is not None and amt < 0: + amt = None value = [] try: - while True: - chunk_left = self._get_chunk_left() - if chunk_left is None: - break - + while (chunk_left := self._get_chunk_left()) is not None: if amt is not None and amt <= chunk_left: value.append(self._safe_read(amt)) self.chunk_left = chunk_left - amt @@ -593,8 +610,8 @@ def _read_chunked(self, amt=None): amt -= chunk_left self.chunk_left = 0 return b''.join(value) - except IncompleteRead: - raise IncompleteRead(b''.join(value)) + except IncompleteRead as exc: + raise IncompleteRead(b''.join(value)) from exc def _readinto_chunked(self, b): assert self.chunked != _UNKNOWN @@ -627,10 +644,25 @@ def _safe_read(self, amt): reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - data = self.fp.read(amt) - if len(data) < amt: - raise IncompleteRead(data, amt-len(data)) - return data + cursize = min(amt, _MIN_READ_BUF_SIZE) + data = self.fp.read(cursize) + if len(data) >= amt: + return data + if len(data) < cursize: + raise IncompleteRead(data, amt - len(data)) + + data = io.BytesIO(data) + data.seek(0, 2) + while True: + # This is a geometric increase in read size (never more than + # doubling out the current length of data per loop iteration). + delta = min(cursize, amt - cursize) + data.write(self.fp.read(delta)) + if data.tell() >= amt: + return data.getvalue() + cursize += delta + if data.tell() < cursize: + raise IncompleteRead(data.getvalue(), amt - data.tell()) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" @@ -655,6 +687,8 @@ def read1(self, n=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def peek(self, n=-1): @@ -679,6 +713,8 @@ def readline(self, limit=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def _read1_chunked(self, n): @@ -786,6 +822,20 @@ def getcode(self): ''' return self.status + +def _create_https_context(http_version): + # Function also used by urllib.request to be able to set the check_hostname + # attribute on a context object. + context = ssl._create_default_https_context() + # send ALPN extension to indicate HTTP/1.1 protocol + if http_version == 11: + context.set_alpn_protocols(['http/1.1']) + # enable PHA for TLS 1.3 connections if available + if context.post_handshake_auth is not None: + context.post_handshake_auth = True + return context + + class HTTPConnection: _http_vsn = 11 @@ -847,6 +897,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} + self._raw_proxy_headers = None (self.host, self.port) = self._get_hostport(host, port) @@ -859,9 +910,9 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, def set_tunnel(self, host, port=None, headers=None): """Set up host and port for HTTP CONNECT tunnelling. - In a connection that uses HTTP CONNECT tunneling, the host passed to the - constructor is used as a proxy server that relays all communication to - the endpoint passed to `set_tunnel`. This done by sending an HTTP + In a connection that uses HTTP CONNECT tunnelling, the host passed to + the constructor is used as a proxy server that relays all communication + to the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. This method must be called before the HTTP connection has been @@ -869,6 +920,13 @@ def set_tunnel(self, host, port=None, headers=None): The headers argument should be a mapping of extra HTTP headers to send with the CONNECT request. + + As HTTP/1.1 is used for HTTP CONNECT tunnelling request, as per the RFC + (https://tools.ietf.org/html/rfc7231#section-4.3.6), a HTTP Host: + header must be provided, matching the authority-form of the request + target provided as the destination for the CONNECT request. If a + HTTP Host: header is not provided via the headers argument, one + is generated and transmitted automatically. """ if self.sock: @@ -876,10 +934,15 @@ def set_tunnel(self, host, port=None, headers=None): self._tunnel_host, self._tunnel_port = self._get_hostport(host, port) if headers: - self._tunnel_headers = headers + self._tunnel_headers = headers.copy() else: self._tunnel_headers.clear() + if not any(header.lower() == "host" for header in self._tunnel_headers): + encoded_host = self._tunnel_host.encode("idna").decode("ascii") + self._tunnel_headers["Host"] = "%s:%d" % ( + encoded_host, self._tunnel_port) + def _get_hostport(self, host, port): if port is None: i = host.rfind(':') @@ -895,17 +958,24 @@ def _get_hostport(self, host, port): host = host[:i] else: port = self.default_port - if host and host[0] == '[' and host[-1] == ']': - host = host[1:-1] + if host and host[0] == '[' and host[-1] == ']': + host = host[1:-1] return (host, port) def set_debuglevel(self, level): self.debuglevel = level + def _wrap_ipv6(self, ip): + if b':' in ip and ip[0] != b'['[0]: + return b"[" + ip + b"]" + return ip + def _tunnel(self): - connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( - self._tunnel_host.encode("ascii"), self._tunnel_port) + connect = b"CONNECT %s:%d %s\r\n" % ( + self._wrap_ipv6(self._tunnel_host.encode("idna")), + self._tunnel_port, + self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): headers.append(f"{header}: {value}\r\n".encode("latin-1")) @@ -917,23 +987,35 @@ def _tunnel(self): del headers response = self.response_class(self.sock, method=self._method) - (version, code, message) = response._read_status() + try: + (version, code, message) = response._read_status() - if code != http.HTTPStatus.OK: - self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break + self._raw_proxy_headers = _read_headers(response.fp) if self.debuglevel > 0: - print('header:', line.decode()) + for header in self._raw_proxy_headers: + print('header:', header.decode()) + + if code != http.HTTPStatus.OK: + self.close() + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + + finally: + response.close() + + def get_proxy_response_headers(self): + """ + Returns a dictionary with the headers of the response + received from the proxy server to the CONNECT request + sent to set the tunnel. + + If the CONNECT request was not sent, the method returns None. + """ + return ( + _parse_header_lines(self._raw_proxy_headers) + if self._raw_proxy_headers is not None + else None + ) def connect(self): """Connect to the host and port specified in __init__.""" @@ -942,7 +1024,7 @@ def connect(self): (self.host,self.port), self.timeout, self.source_address) # Might fail in OSs that don't implement TCP_NODELAY try: - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except OSError as e: if e.errno != errno.ENOPROTOOPT: raise @@ -980,14 +1062,11 @@ def send(self, data): print("send:", repr(data)) if hasattr(data, "read") : if self.debuglevel > 0: - print("sendIng a read()able") + print("sending a readable") encode = self._is_textIO(data) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") - while 1: - datablock = data.read(self.blocksize) - if not datablock: - break + while datablock := data.read(self.blocksize): if encode: datablock = datablock.encode("iso-8859-1") sys.audit("http.client.send", self, datablock) @@ -1013,14 +1092,11 @@ def _output(self, s): def _read_readable(self, readable): if self.debuglevel > 0: - print("sendIng a read()able") + print("reading a readable") encode = self._is_textIO(readable) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") - while True: - datablock = readable.read(self.blocksize) - if not datablock: - break + while datablock := readable.read(self.blocksize): if encode: datablock = datablock.encode("iso-8859-1") yield datablock @@ -1157,7 +1233,7 @@ def putrequest(self, method, url, skip_host=False, netloc_enc = netloc.encode("ascii") except UnicodeEncodeError: netloc_enc = netloc.encode("idna") - self.putheader('Host', netloc_enc) + self.putheader('Host', _strip_ipv6_iface(netloc_enc)) else: if self._tunnel_host: host = self._tunnel_host @@ -1173,9 +1249,9 @@ def putrequest(self, method, url, skip_host=False, # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - - if host.find(':') >= 0: - host_enc = b'[' + host_enc + b']' + host_enc = self._wrap_ipv6(host_enc) + if ":" in host: + host_enc = _strip_ipv6_iface(host_enc) if port == self.default_port: self.putheader('Host', host_enc) @@ -1400,46 +1476,15 @@ class HTTPSConnection(HTTPConnection): default_port = HTTPS_PORT - # XXX Should key_file and cert_file be deprecated in favour of context? - - def __init__(self, host, port=None, key_file=None, cert_file=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, *, context=None, - check_hostname=None, blocksize=8192): + def __init__(self, host, port=None, + *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, context=None, blocksize=8192): super(HTTPSConnection, self).__init__(host, port, timeout, source_address, blocksize=blocksize) - if (key_file is not None or cert_file is not None or - check_hostname is not None): - import warnings - warnings.warn("key_file, cert_file and check_hostname are " - "deprecated, use a custom context instead.", - DeprecationWarning, 2) - self.key_file = key_file - self.cert_file = cert_file if context is None: - context = ssl._create_default_https_context() - # send ALPN extension to indicate HTTP/1.1 protocol - if self._http_vsn == 11: - context.set_alpn_protocols(['http/1.1']) - # enable PHA for TLS 1.3 connections if available - if context.post_handshake_auth is not None: - context.post_handshake_auth = True - will_verify = context.verify_mode != ssl.CERT_NONE - if check_hostname is None: - check_hostname = context.check_hostname - if check_hostname and not will_verify: - raise ValueError("check_hostname needs a SSL context with " - "either CERT_OPTIONAL or CERT_REQUIRED") - if key_file or cert_file: - context.load_cert_chain(cert_file, key_file) - # cert and key file means the user wants to authenticate. - # enable TLS 1.3 PHA implicitly even for custom contexts. - if context.post_handshake_auth is not None: - context.post_handshake_auth = True + context = _create_https_context(self._http_vsn) self._context = context - if check_hostname is not None: - self._context.check_hostname = check_hostname def connect(self): "Connect to a host on a given (SSL) port." diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 685f6a0b976..9a2f0fb851c 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -34,10 +34,7 @@ import re import time import urllib.parse, urllib.request -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading +import threading as _threading import http.client # only for the default HTTP port from calendar import timegm @@ -92,8 +89,7 @@ def _timegm(tt): DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] -MONTHS_LOWER = [] -for month in MONTHS: MONTHS_LOWER.append(month.lower()) +MONTHS_LOWER = [month.lower() for month in MONTHS] def time2isoz(t=None): """Return a string representing time in seconds since epoch, t. @@ -108,9 +104,9 @@ def time2isoz(t=None): """ if t is None: - dt = datetime.datetime.utcnow() + dt = datetime.datetime.now(tz=datetime.UTC) else: - dt = datetime.datetime.utcfromtimestamp(t) + dt = datetime.datetime.fromtimestamp(t, tz=datetime.UTC) return "%04d-%02d-%02d %02d:%02d:%02dZ" % ( dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) @@ -126,9 +122,9 @@ def time2netscape(t=None): """ if t is None: - dt = datetime.datetime.utcnow() + dt = datetime.datetime.now(tz=datetime.UTC) else: - dt = datetime.datetime.utcfromtimestamp(t) + dt = datetime.datetime.fromtimestamp(t, tz=datetime.UTC) return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % ( DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1], dt.year, dt.hour, dt.minute, dt.second) @@ -434,6 +430,7 @@ def split_header_words(header_values): if pairs: result.append(pairs) return result +HEADER_JOIN_TOKEN_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])") def join_header_words(lists): """Do the inverse (almost) of the conversion done by split_header_words. @@ -441,10 +438,10 @@ def join_header_words(lists): Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. - >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) - 'text/plain; charset="iso-8859-1"' - >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) - 'text/plain, charset="iso-8859-1"' + >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]]) + 'text/plain; charset="iso-8859/1"' + >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]]) + 'text/plain, charset="iso-8859/1"' """ headers = [] @@ -452,7 +449,7 @@ def join_header_words(lists): attr = [] for k, v in pairs: if v is not None: - if not re.search(r"^\w+$", v): + if not HEADER_JOIN_TOKEN_RE.fullmatch(v): v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \ v = '"%s"' % v k = "%s=%s" % (k, v) @@ -644,7 +641,7 @@ def eff_request_host(request): """ erhn = req_host = request_host(request) - if req_host.find(".") == -1 and not IPV4_RE.search(req_host): + if "." not in req_host: erhn = req_host + ".local" return req_host, erhn @@ -1047,12 +1044,13 @@ def set_ok_domain(self, cookie, request): else: undotted_domain = domain embedded_dots = (undotted_domain.find(".") >= 0) - if not embedded_dots and domain != ".local": + if not embedded_dots and not erhn.endswith(".local"): _debug(" non-local domain %s contains no embedded dot", domain) return False if cookie.version == 0: - if (not erhn.endswith(domain) and + if (not (erhn.endswith(domain) or + erhn.endswith(f"{undotted_domain}.local")) and (not erhn.startswith(".") and not ("."+erhn).endswith(domain))): _debug(" effective request-host %s (even with added " @@ -1227,14 +1225,9 @@ def path_return_ok(self, path, request): _debug(" %s does not path-match %s", req_path, path) return False -def vals_sorted_by_key(adict): - keys = sorted(adict.keys()) - return map(adict.get, keys) - def deepvalues(mapping): - """Iterates over nested mapping, depth-first, in sorted order by key.""" - values = vals_sorted_by_key(mapping) - for obj in values: + """Iterates over nested mapping, depth-first""" + for obj in list(mapping.values()): mapping = False try: obj.items @@ -1898,7 +1891,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with open(filename, "w") as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. @@ -1923,9 +1919,7 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): "comment", "commenturl") try: - while 1: - line = f.readline() - if line == "": break + while (line := f.readline()) != "": if not line.startswith(header): continue line = line[len(header):].strip() @@ -1993,7 +1987,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. lynx uses this file format, too. + `cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if @@ -2025,12 +2019,9 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): filename) try: - while 1: - line = f.readline() + while (line := f.readline()) != "": rest = {} - if line == "": break - # httponly is a cookie flag as defined in rfc6265 # when encoded in a netscape cookie file, # the line is prepended with "#HttpOnly_" @@ -2094,7 +2085,10 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with open(filename, "w") as f: + with os.fdopen( + os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), + 'w', + ) as f: f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 35ac2dc6ae2..57791c6ab08 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -184,8 +184,13 @@ def _quote(str): return '"' + str.translate(_Translator) + '"' -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") +_unquote_sub = re.compile(r'\\(?:([0-3][0-7][0-7])|(.))').sub + +def _unquote_replace(m): + if m[1]: + return chr(int(m[1], 8)) + else: + return m[2] def _unquote(str): # If there aren't any doublequotes, @@ -205,36 +210,13 @@ def _unquote(str): # \012 --> \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] @@ -442,9 +424,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' diff --git a/Lib/http/server.py b/Lib/http/server.py index 58abadf7377..0ec479003a4 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -2,18 +2,18 @@ Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, -and CGIHTTPRequestHandler for CGI scripts. +and (deprecated) CGIHTTPRequestHandler for CGI scripts. -It does, however, optionally implement HTTP/1.1 persistent connections, -as of version 0.3. +It does, however, optionally implement HTTP/1.1 persistent connections. Notes on CGIHTTPRequestHandler ------------------------------ -This class implements GET and POST requests to cgi-bin scripts. +This class is deprecated. It implements GET and POST requests to cgi-bin scripts. -If the os.fork() function is not present (e.g. on Windows), -subprocess.Popen() is used as a fallback, with slightly altered semantics. +If the os.fork() function is not present (Windows), subprocess.Popen() is used, +with slightly altered but never documented semantics. Use from a threaded +process is likely to trigger a warning at os.fork() time. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -109,11 +110,10 @@ # Default error message template DEFAULT_ERROR_MESSAGE = """\ - - + + - + Error response @@ -127,6 +127,10 @@ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + class HTTPServer(socketserver.TCPServer): allow_reuse_address = 1 # Seems to make sense in testing environment @@ -275,6 +279,7 @@ def parse_request(self): error response has already been sent back. """ + is_http_0_9 = False self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True @@ -300,6 +305,10 @@ def parse_request(self): # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError + if any(not component.isdigit() for component in version_number): + raise ValueError("non digit in http version") + if any(len(component) > 10 for component in version_number): + raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( @@ -328,8 +337,21 @@ def parse_request(self): HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) return False + is_http_0_9 = True self.command, self.path = command, path + # gh-87389: The purpose of replacing '//' with '/' is to protect + # against open redirect attacks possibly triggered if the path starts + # with '//' because http clients treat //path as an absolute URI + # without scheme (similar to http://path) rather than a path. + if self.path.startswith('//'): + self.path = '/' + self.path.lstrip('/') # Reduce to a single / + + # For HTTP/0.9, headers are not expected at all. + if is_http_0_9: + self.headers = {} + return True + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, @@ -556,6 +578,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -571,12 +598,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" @@ -637,6 +668,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """ server_version = "SimpleHTTP/" + __version__ + index_pages = ("index.html", "index.htm") extensions_map = _encodings_map_default = { '.gz': 'application/gzip', '.Z': 'application/octet-stream', @@ -680,7 +712,7 @@ def send_head(self): f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): + if not parts.path.endswith(('/', '%2f', '%2F')): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', @@ -690,9 +722,9 @@ def send_head(self): self.send_header("Content-Length", "0") self.end_headers() return None - for index in "index.html", "index.htm": + for index in self.index_pages: index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: @@ -702,7 +734,7 @@ def send_head(self): # The test for this was added in test_httpserver.py # However, some OS platforms accept a trailingSlash as a filename # See discussion on python-dev and Issue34711 regarding - # parseing and rejection of filenames with a trailing slash + # parsing and rejection of filenames with a trailing slash if path.endswith("/"): self.send_error(HTTPStatus.NOT_FOUND, "File not found") return None @@ -770,21 +802,23 @@ def list_directory(self, path): return None list.sort(key=lambda a: a.lower()) r = [] + displaypath = self.path + displaypath = displaypath.split('#', 1)[0] + displaypath = displaypath.split('?', 1)[0] try: - displaypath = urllib.parse.unquote(self.path, + displaypath = urllib.parse.unquote(displaypath, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(path) + displaypath = urllib.parse.unquote(displaypath) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() - title = 'Directory listing for %s' % displaypath - r.append('') - r.append('\n') - r.append('' % enc) - r.append('%s\n' % title) - r.append('\n

%s

' % title) + title = f'Directory listing for {displaypath}' + r.append('') + r.append('') + r.append('') + r.append(f'') + r.append(f'{title}\n') + r.append(f'\n

{title}

') r.append('
\n
    ') for name in list: fullname = os.path.join(path, name) @@ -820,14 +854,14 @@ def translate_path(self, path): """ # abandon query parameters - path = path.split('?',1)[0] - path = path.split('#',1)[0] + path = path.split('#', 1)[0] + path = path.split('?', 1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) + trailing_slash = path.endswith('/') path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) @@ -877,7 +911,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' @@ -966,6 +1000,12 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): """ + def __init__(self, *args, **kwargs): + import warnings + warnings._deprecated("http.server.CGIHTTPRequestHandler", + remove=(3, 15)) + super().__init__(*args, **kwargs) + # Determine platform specifics have_fork = hasattr(os, 'fork') @@ -1078,7 +1118,7 @@ def run_cgi(self): "CGI script is not executable (%r)" % scriptname) return - # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html + # Reference: https://www6.uniovi.es/~antonio/ncsa_httpd/cgi/env.html # XXX Much of the following could be prepared ahead of time! env = copy.deepcopy(os.environ) env['SERVER_SOFTWARE'] = self.version_string() @@ -1198,7 +1238,18 @@ def run_cgi(self): env = env ) if self.command.lower() == "post" and nbytes > 0: - data = self.rfile.read(nbytes) + cursize = 0 + data = self.rfile.read(min(nbytes, _MIN_READ_BUF_SIZE)) + while len(data) < nbytes and len(data) != cursize: + cursize = len(data) + # This is a geometric increase in read size (never more + # than doubling out the current length of data per loop + # iteration). + delta = min(cursize, nbytes - cursize) + try: + data += self.rfile.read(delta) + except TimeoutError: + break else: data = None # throw away additional data [see bug #427345] @@ -1258,15 +1309,19 @@ def test(HandlerClass=BaseHTTPRequestHandler, parser = argparse.ArgumentParser() parser.add_argument('--cgi', action='store_true', help='run as CGI server') - parser.add_argument('--bind', '-b', metavar='ADDRESS', - help='specify alternate bind address ' + parser.add_argument('-b', '--bind', metavar='ADDRESS', + help='bind to this address ' '(default: all interfaces)') - parser.add_argument('--directory', '-d', default=os.getcwd(), - help='specify alternate directory ' + parser.add_argument('-d', '--directory', default=os.getcwd(), + help='serve this directory ' '(default: current directory)') - parser.add_argument('port', action='store', default=8000, type=int, - nargs='?', - help='specify alternate port (default: 8000)') + parser.add_argument('-p', '--protocol', metavar='VERSION', + default='HTTP/1.0', + help='conform to this HTTP version ' + '(default: %(default)s)') + parser.add_argument('port', default=8000, type=int, nargs='?', + help='bind to this port ' + '(default: %(default)s)') args = parser.parse_args() if args.cgi: handler_class = CGIHTTPRequestHandler @@ -1292,4 +1347,5 @@ def finish_request(self, request, client_address): ServerClass=DualStackServer, port=args.port, bind=args.bind, + protocol=args.protocol, ) diff --git a/Lib/imaplib.py b/Lib/imaplib.py new file mode 100644 index 00000000000..cbe129b3e7c --- /dev/null +++ b/Lib/imaplib.py @@ -0,0 +1,1967 @@ +"""IMAP4 client. + +Based on RFC 2060. + +Public class: IMAP4 +Public variable: Debug +Public functions: Internaldate2tuple + Int2AP + ParseFlags + Time2Internaldate +""" + +# Author: Piers Lauder December 1997. +# +# Authentication code contributed by Donn Cave June 1998. +# String method conversion by ESR, February 2001. +# GET/SETACL contributed by Anthony Baxter April 2001. +# IMAP4_SSL contributed by Tino Lange March 2002. +# GET/SETQUOTA contributed by Andreas Zeidler June 2002. +# PROXYAUTH contributed by Rick Holbert November 2002. +# GET/SETANNOTATION contributed by Tomas Lindroos June 2005. +# IDLE contributed by Forest August 2024. + +__version__ = "2.60" + +import binascii, errno, random, re, socket, subprocess, sys, time, calendar +from datetime import datetime, timezone, timedelta +from io import DEFAULT_BUFFER_SIZE + +try: + import ssl + HAVE_SSL = True +except ImportError: + HAVE_SSL = False + +__all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple", + "Int2AP", "ParseFlags", "Time2Internaldate"] + +# Globals + +CRLF = b'\r\n' +Debug = 0 +IMAP4_PORT = 143 +IMAP4_SSL_PORT = 993 +AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + +# Maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) +# don't specify a line length. RFC 2683 suggests limiting client +# command lines to 1000 octets and that servers should be prepared +# to accept command lines up to 8000 octets, so we used to use 10K here. +# In the modern world (eg: gmail) the response to, for example, a +# search command can be quite large, so we now use 1M. +_MAXLINE = 1000000 + + +# Commands + +Commands = { + # name valid states + 'APPEND': ('AUTH', 'SELECTED'), + 'AUTHENTICATE': ('NONAUTH',), + 'CAPABILITY': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'CHECK': ('SELECTED',), + 'CLOSE': ('SELECTED',), + 'COPY': ('SELECTED',), + 'CREATE': ('AUTH', 'SELECTED'), + 'DELETE': ('AUTH', 'SELECTED'), + 'DELETEACL': ('AUTH', 'SELECTED'), + 'ENABLE': ('AUTH', ), + 'EXAMINE': ('AUTH', 'SELECTED'), + 'EXPUNGE': ('SELECTED',), + 'FETCH': ('SELECTED',), + 'GETACL': ('AUTH', 'SELECTED'), + 'GETANNOTATION':('AUTH', 'SELECTED'), + 'GETQUOTA': ('AUTH', 'SELECTED'), + 'GETQUOTAROOT': ('AUTH', 'SELECTED'), + 'IDLE': ('AUTH', 'SELECTED'), + 'MYRIGHTS': ('AUTH', 'SELECTED'), + 'LIST': ('AUTH', 'SELECTED'), + 'LOGIN': ('NONAUTH',), + 'LOGOUT': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'LSUB': ('AUTH', 'SELECTED'), + 'MOVE': ('SELECTED',), + 'NAMESPACE': ('AUTH', 'SELECTED'), + 'NOOP': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'PARTIAL': ('SELECTED',), # NB: obsolete + 'PROXYAUTH': ('AUTH',), + 'RENAME': ('AUTH', 'SELECTED'), + 'SEARCH': ('SELECTED',), + 'SELECT': ('AUTH', 'SELECTED'), + 'SETACL': ('AUTH', 'SELECTED'), + 'SETANNOTATION':('AUTH', 'SELECTED'), + 'SETQUOTA': ('AUTH', 'SELECTED'), + 'SORT': ('SELECTED',), + 'STARTTLS': ('NONAUTH',), + 'STATUS': ('AUTH', 'SELECTED'), + 'STORE': ('SELECTED',), + 'SUBSCRIBE': ('AUTH', 'SELECTED'), + 'THREAD': ('SELECTED',), + 'UID': ('SELECTED',), + 'UNSUBSCRIBE': ('AUTH', 'SELECTED'), + 'UNSELECT': ('SELECTED',), + } + +# Patterns to match server responses + +Continuation = re.compile(br'\+( (?P.*))?') +Flags = re.compile(br'.*FLAGS \((?P[^\)]*)\)') +InternalDate = re.compile(br'.*INTERNALDATE "' + br'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + br' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' + br' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' + br'"') +# Literal is no longer used; kept for backward compatibility. +Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) +MapCRLF = re.compile(br'\r\n|\r|\n') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') +Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') +# Untagged_status is no longer used; kept for backward compatibility +Untagged_status = re.compile( + br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?', re.ASCII) +# We compile these in _mode_xxx. +_Literal = br'.*{(?P\d+)}$' +_Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' + + + +class IMAP4: + + r"""IMAP4 client class. + + Instantiate with: IMAP4([host[, port[, timeout=None]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port). + timeout - socket timeout (default: None) + If timeout is not given or is None, + the global default socket timeout is used + + All IMAP4rev1 commands are supported by methods of the same + name (in lowercase). + + All arguments to commands are converted to strings, except for + AUTHENTICATE, and the last argument to APPEND which is passed as + an IMAP4 literal. If necessary (the string contains any + non-printing characters or white-space and isn't enclosed with + either parentheses or double quotes) each string is quoted. + However, the 'password' argument to the LOGIN command is always + quoted. If you want to avoid having an argument string quoted + (eg: the 'flags' argument to STORE) then enclose the string in + parentheses (eg: "(\Deleted)"). + + Each command returns a tuple: (type, [data, ...]) where 'type' + is usually 'OK' or 'NO', and 'data' is either the text from the + tagged response, or untagged results from command. Each 'data' + is either a string, or a tuple. If a tuple, then the first part + is the header of the response, and the second part contains + the data (ie: 'literal' value). + + Errors raise the exception class .error(""). + IMAP4 server errors raise .abort(""), + which is a sub-class of 'error'. Mailbox status changes + from READ-WRITE to READ-ONLY raise the exception class + .readonly(""), which is a sub-class of 'abort'. + + "error" exceptions imply a program error. + "abort" exceptions imply the connection should be reset, and + the command re-tried. + "readonly" exceptions imply the command should be re-tried. + + Note: to use this module, you must read the RFCs pertaining to the + IMAP4 protocol, as the semantics of the arguments to each IMAP4 + command are left to the invoker, not to mention the results. Also, + most IMAP servers implement a sub-set of the commands available here. + """ + + class error(Exception): pass # Logical errors - debug required + class abort(error): pass # Service errors - close and retry + class readonly(abort): pass # Mailbox status changed to READ-ONLY + class _responsetimeout(TimeoutError): pass # No response during IDLE + + def __init__(self, host='', port=IMAP4_PORT, timeout=None): + self.debug = Debug + self.state = 'LOGOUT' + self.literal = None # A literal argument to a command + self.tagged_commands = {} # Tagged commands awaiting response + self.untagged_responses = {} # {typ: [data, ...], ...} + self.continuation_response = '' # Last continuation response + self._idle_responses = [] # Response queue for idle iteration + self._idle_capture = False # Whether to queue responses for idle + self.is_readonly = False # READ-ONLY desired state + self.tagnum = 0 + self._tls_established = False + self._mode_ascii() + self._readbuf = [] + + # Open socket to server. + + self.open(host, port, timeout) + + try: + self._connect() + except Exception: + try: + self.shutdown() + except OSError: + pass + raise + + def _mode_ascii(self): + self.utf8_enabled = False + self._encoding = 'ascii' + self.Literal = re.compile(_Literal, re.ASCII) + self.Untagged_status = re.compile(_Untagged_status, re.ASCII) + + + def _mode_utf8(self): + self.utf8_enabled = True + self._encoding = 'utf-8' + self.Literal = re.compile(_Literal) + self.Untagged_status = re.compile(_Untagged_status) + + + def _connect(self): + # Create unique tag for this session, + # and compile tagged response matcher. + + self.tagpre = Int2AP(random.randint(4096, 65535)) + self.tagre = re.compile(br'(?P' + + self.tagpre + + br'\d+) (?P[A-Z]+) (?P.*)', re.ASCII) + + # Get server welcome message, + # request and store CAPABILITY response. + + if __debug__: + self._cmd_log_len = 10 + self._cmd_log_idx = 0 + self._cmd_log = {} # Last '_cmd_log_len' interactions + if self.debug >= 1: + self._mesg('imaplib version %s' % __version__) + self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) + + self.welcome = self._get_response() + if 'PREAUTH' in self.untagged_responses: + self.state = 'AUTH' + elif 'OK' in self.untagged_responses: + self.state = 'NONAUTH' + else: + raise self.error(self.welcome) + + self._get_capabilities() + if __debug__: + if self.debug >= 3: + self._mesg('CAPABILITIES: %r' % (self.capabilities,)) + + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + return + + raise self.error('server not IMAP4 compliant') + + + def __getattr__(self, attr): + # Allow UPPERCASE variants of IMAP4 command methods. + if attr in Commands: + return getattr(self, attr.lower()) + raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + + def __enter__(self): + return self + + def __exit__(self, *args): + if self.state == "LOGOUT": + return + + try: + self.logout() + except OSError: + pass + + + # Overridable methods + + + def _create_socket(self, timeout): + # Default value of IMAP4.host is '', but socket.getaddrinfo() + # (which is used by socket.create_connection()) expects None + # as a default value for host. + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + host = None if not self.host else self.host + sys.audit("imaplib.open", self, self.host, self.port) + address = (host, self.port) + if timeout is not None: + return socket.create_connection(address, timeout) + return socket.create_connection(address) + + def open(self, host='', port=IMAP4_PORT, timeout=None): + """Setup connection to remote server on "host:port" + (default: localhost:standard IMAP4 port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = host + self.port = port + self.sock = self._create_socket(timeout) + self._file = self.sock.makefile('rb') + + + @property + def file(self): + # The old 'file' attribute is no longer used now that we do our own + # read() and readline() buffering, with which it conflicts. + # As an undocumented interface, it should never have been accessed by + # external code, and therefore does not warrant deprecation. + # Nevertheless, we provide this property for now, to avoid suddenly + # breaking any code in the wild that might have been using it in a + # harmless way. + import warnings + warnings.warn( + 'IMAP4.file is unsupported, can cause errors, and may be removed.', + RuntimeWarning, + stacklevel=2) + return self._file + + + def read(self, size): + """Read 'size' bytes from remote.""" + # We need buffered read() to continue working after socket timeouts, + # since we use them during IDLE. Unfortunately, the standard library's + # SocketIO implementation makes this impossible, by setting a permanent + # error condition instead of letting the caller decide how to handle a + # timeout. We therefore implement our own buffered read(). + # https://github.com/python/cpython/issues/51571 + # + # Reading in chunks instead of delegating to a single + # BufferedReader.read() call also means we avoid its preallocation + # of an unreasonably large memory block if a malicious server claims + # it will send a huge literal without actually sending one. + # https://github.com/python/cpython/issues/119511 + + parts = [] + + while size > 0: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + if len(buf) >= size: + parts.append(buf[:size]) + self._readbuf = [buf[size:]] + self._readbuf[len(parts):] + break + parts.append(buf) + size -= len(buf) + + return b''.join(parts) + + + def readline(self): + """Read line from remote.""" + # The comment in read() explains why we implement our own readline(). + + LF = b'\n' + parts = [] + length = 0 + + while length < _MAXLINE: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + pos = buf.find(LF) + if pos != -1: + pos += 1 + parts.append(buf[:pos]) + self._readbuf = [buf[pos:]] + self._readbuf[len(parts):] + break + parts.append(buf) + length += len(buf) + + line = b''.join(parts) + if len(line) > _MAXLINE: + raise self.error("got more than %d bytes" % _MAXLINE) + return line + + + def send(self, data): + """Send data to remote.""" + sys.audit("imaplib.send", self, data) + self.sock.sendall(data) + + + def shutdown(self): + """Close I/O established in "open".""" + self._file.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): + raise + finally: + self.sock.close() + + + def socket(self): + """Return socket instance used to connect to IMAP4 server. + + socket = .socket() + """ + return self.sock + + + + # Utility methods + + + def recent(self): + """Return most recent 'RECENT' responses if any exist, + else prompt server for an update using the 'NOOP' command. + + (typ, [data]) = .recent() + + 'data' is None if no new messages, + else list of RECENT responses, most recent last. + """ + name = 'RECENT' + typ, dat = self._untagged_response('OK', [None], name) + if dat[-1]: + return typ, dat + typ, dat = self.noop() # Prod server for response + return self._untagged_response(typ, dat, name) + + + def response(self, code): + """Return data for response 'code' if received, or None. + + Old value for response 'code' is cleared. + + (code, [data]) = .response(code) + """ + return self._untagged_response(code, [None], code.upper()) + + + + # IMAP4 commands + + + def append(self, mailbox, flags, date_time, message): + """Append message to named mailbox. + + (typ, [data]) = .append(mailbox, flags, date_time, message) + + All args except 'message' can be None. + """ + name = 'APPEND' + if not mailbox: + mailbox = 'INBOX' + if flags: + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags + else: + flags = None + if date_time: + date_time = Time2Internaldate(date_time) + else: + date_time = None + literal = MapCRLF.sub(CRLF, message) + self.literal = literal + return self._simple_command(name, mailbox, flags, date_time) + + + def authenticate(self, mechanism, authobject): + """Authenticate command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - it must appear in .capabilities in the + form AUTH=. + + 'authobject' must be a callable object: + + data = authobject(response) + + It will be called to process server continuation responses; the + response argument it is passed will be a bytes. It should return bytes + data that will be base64 encoded and sent to the server. It should + return None if the client abort response '*' should be sent instead. + """ + mech = mechanism.upper() + # XXX: shouldn't this code be removed, not commented out? + #cap = 'AUTH=%s' % mech + #if not cap in self.capabilities: # Let the server decide! + # raise self.error("Server doesn't allow %s authentication." % mech) + self.literal = _Authenticator(authobject).process + typ, dat = self._simple_command('AUTHENTICATE', mech) + if typ != 'OK': + raise self.error(dat[-1].decode('utf-8', 'replace')) + self.state = 'AUTH' + return typ, dat + + + def capability(self): + """(typ, [data]) = .capability() + Fetch capabilities list from server.""" + + name = 'CAPABILITY' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def check(self): + """Checkpoint mailbox on server. + + (typ, [data]) = .check() + """ + return self._simple_command('CHECK') + + + def close(self): + """Close currently selected mailbox. + + Deleted messages are removed from writable mailbox. + This is the recommended command before 'LOGOUT'. + + (typ, [data]) = .close() + """ + try: + typ, dat = self._simple_command('CLOSE') + finally: + self.state = 'AUTH' + return typ, dat + + + def copy(self, message_set, new_mailbox): + """Copy 'message_set' messages onto end of 'new_mailbox'. + + (typ, [data]) = .copy(message_set, new_mailbox) + """ + return self._simple_command('COPY', message_set, new_mailbox) + + + def create(self, mailbox): + """Create new mailbox. + + (typ, [data]) = .create(mailbox) + """ + return self._simple_command('CREATE', mailbox) + + + def delete(self, mailbox): + """Delete old mailbox. + + (typ, [data]) = .delete(mailbox) + """ + return self._simple_command('DELETE', mailbox) + + def deleteacl(self, mailbox, who): + """Delete the ACLs (remove any rights) set for who on mailbox. + + (typ, [data]) = .deleteacl(mailbox, who) + """ + return self._simple_command('DELETEACL', mailbox, who) + + def enable(self, capability): + """Send an RFC5161 enable string to the server. + + (typ, [data]) = .enable(capability) + """ + if 'ENABLE' not in self.capabilities: + raise IMAP4.error("Server does not support ENABLE") + typ, data = self._simple_command('ENABLE', capability) + if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): + self._mode_utf8() + return typ, data + + def expunge(self): + """Permanently remove deleted items from selected mailbox. + + Generates 'EXPUNGE' response for each deleted message. + + (typ, [data]) = .expunge() + + 'data' is list of 'EXPUNGE'd message numbers in order received. + """ + name = 'EXPUNGE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def fetch(self, message_set, message_parts): + """Fetch (parts of) messages. + + (typ, [data, ...]) = .fetch(message_set, message_parts) + + 'message_parts' should be a string of selected parts + enclosed in parentheses, eg: "(UID BODY[TEXT])". + + 'data' are tuples of message part envelope and data. + """ + name = 'FETCH' + typ, dat = self._simple_command(name, message_set, message_parts) + return self._untagged_response(typ, dat, name) + + + def getacl(self, mailbox): + """Get the ACLs for a mailbox. + + (typ, [data]) = .getacl(mailbox) + """ + typ, dat = self._simple_command('GETACL', mailbox) + return self._untagged_response(typ, dat, 'ACL') + + + def getannotation(self, mailbox, entry, attribute): + """(typ, [data]) = .getannotation(mailbox, entry, attribute) + Retrieve ANNOTATIONs.""" + + typ, dat = self._simple_command('GETANNOTATION', mailbox, entry, attribute) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def getquota(self, root): + """Get the quota root's resource usage and limits. + + Part of the IMAP4 QUOTA extension defined in rfc2087. + + (typ, [data]) = .getquota(root) + """ + typ, dat = self._simple_command('GETQUOTA', root) + return self._untagged_response(typ, dat, 'QUOTA') + + + def getquotaroot(self, mailbox): + """Get the list of quota roots for the named mailbox. + + (typ, [[QUOTAROOT responses...], [QUOTA responses]]) = .getquotaroot(mailbox) + """ + typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, quota = self._untagged_response(typ, dat, 'QUOTA') + typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') + return typ, [quotaroot, quota] + + + def idle(self, duration=None): + """Return an iterable IDLE context manager producing untagged responses. + If the argument is not None, limit iteration to 'duration' seconds. + + with M.idle(duration=29 * 60) as idler: + for typ, data in idler: + print(typ, data) + + Note: 'duration' requires a socket connection (not IMAP4_stream). + """ + return Idler(self, duration) + + + def list(self, directory='""', pattern='*'): + """List mailbox names in directory matching pattern. + + (typ, [data]) = .list(directory='""', pattern='*') + + 'data' is list of LIST responses. + """ + name = 'LIST' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + + def login(self, user, password): + """Identify client using plaintext password. + + (typ, [data]) = .login(user, password) + + NB: 'password' will be quoted. + """ + typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + if typ != 'OK': + raise self.error(dat[-1]) + self.state = 'AUTH' + return typ, dat + + + def login_cram_md5(self, user, password): + """ Force use of CRAM-MD5 authentication. + + (typ, [data]) = .login_cram_md5(user, password) + """ + self.user, self.password = user, password + return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH) + + + def _CRAM_MD5_AUTH(self, challenge): + """ Authobject to use with CRAM-MD5 authentication. """ + import hmac + + if isinstance(self.password, str): + password = self.password.encode('utf-8') + else: + password = self.password + + try: + authcode = hmac.HMAC(password, challenge, 'md5') + except ValueError: # HMAC-MD5 is not available + raise self.error("CRAM-MD5 authentication is not supported") + return f"{self.user} {authcode.hexdigest()}" + + + def logout(self): + """Shutdown connection to server. + + (typ, [data]) = .logout() + + Returns server 'BYE' response. + """ + self.state = 'LOGOUT' + typ, dat = self._simple_command('LOGOUT') + self.shutdown() + return typ, dat + + + def lsub(self, directory='""', pattern='*'): + """List 'subscribed' mailbox names in directory matching pattern. + + (typ, [data, ...]) = .lsub(directory='""', pattern='*') + + 'data' are tuples of message part envelope and data. + """ + name = 'LSUB' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + def myrights(self, mailbox): + """Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). + + (typ, [data]) = .myrights(mailbox) + """ + typ,dat = self._simple_command('MYRIGHTS', mailbox) + return self._untagged_response(typ, dat, 'MYRIGHTS') + + def namespace(self): + """ Returns IMAP namespaces ala rfc2342 + + (typ, [data, ...]) = .namespace() + """ + name = 'NAMESPACE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def noop(self): + """Send NOOP command. + + (typ, [data]) = .noop() + """ + if __debug__: + if self.debug >= 3: + self._dump_ur(self.untagged_responses) + return self._simple_command('NOOP') + + + def partial(self, message_num, message_part, start, length): + """Fetch truncated part of a message. + + (typ, [data, ...]) = .partial(message_num, message_part, start, length) + + 'data' is tuple of message part envelope and data. + """ + name = 'PARTIAL' + typ, dat = self._simple_command(name, message_num, message_part, start, length) + return self._untagged_response(typ, dat, 'FETCH') + + + def proxyauth(self, user): + """Assume authentication as "user". + + Allows an authorised administrator to proxy into any user's + mailbox. + + (typ, [data]) = .proxyauth(user) + """ + + name = 'PROXYAUTH' + return self._simple_command('PROXYAUTH', user) + + + def rename(self, oldmailbox, newmailbox): + """Rename old mailbox name to new. + + (typ, [data]) = .rename(oldmailbox, newmailbox) + """ + return self._simple_command('RENAME', oldmailbox, newmailbox) + + + def search(self, charset, *criteria): + """Search mailbox for matching messages. + + (typ, [data]) = .search(charset, criterion, ...) + + 'data' is space separated list of matching message numbers. + If UTF8 is enabled, charset MUST be None. + """ + name = 'SEARCH' + if charset: + if self.utf8_enabled: + raise IMAP4.error("Non-None charset not valid in UTF8 mode") + typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria) + else: + typ, dat = self._simple_command(name, *criteria) + return self._untagged_response(typ, dat, name) + + + def select(self, mailbox='INBOX', readonly=False): + """Select a mailbox. + + Flush all untagged responses. + + (typ, [data]) = .select(mailbox='INBOX', readonly=False) + + 'data' is count of messages in mailbox ('EXISTS' response). + + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via .response('FLAGS') etc. + """ + self.untagged_responses = {} # Flush old responses. + self.is_readonly = readonly + if readonly: + name = 'EXAMINE' + else: + name = 'SELECT' + typ, dat = self._simple_command(name, mailbox) + if typ != 'OK': + self.state = 'AUTH' # Might have been 'SELECTED' + return typ, dat + self.state = 'SELECTED' + if 'READ-ONLY' in self.untagged_responses \ + and not readonly: + if __debug__: + if self.debug >= 1: + self._dump_ur(self.untagged_responses) + raise self.readonly('%s is not writable' % mailbox) + return typ, self.untagged_responses.get('EXISTS', [None]) + + + def setacl(self, mailbox, who, what): + """Set a mailbox acl. + + (typ, [data]) = .setacl(mailbox, who, what) + """ + return self._simple_command('SETACL', mailbox, who, what) + + + def setannotation(self, *args): + """(typ, [data]) = .setannotation(mailbox[, entry, attribute]+) + Set ANNOTATIONs.""" + + typ, dat = self._simple_command('SETANNOTATION', *args) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def setquota(self, root, limits): + """Set the quota root's resource limits. + + (typ, [data]) = .setquota(root, limits) + """ + typ, dat = self._simple_command('SETQUOTA', root, limits) + return self._untagged_response(typ, dat, 'QUOTA') + + + def sort(self, sort_criteria, charset, *search_criteria): + """IMAP4rev1 extension SORT command. + + (typ, [data]) = .sort(sort_criteria, charset, search_criteria, ...) + """ + name = 'SORT' + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unimplemented extension command: %s' % name) + if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): + sort_criteria = '(%s)' % sort_criteria + typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def starttls(self, ssl_context=None): + name = 'STARTTLS' + if not HAVE_SSL: + raise self.error('SSL support missing') + if self._tls_established: + raise self.abort('TLS session already established') + if name not in self.capabilities: + raise self.abort('TLS not supported by server') + # Generate a default SSL context if none was passed. + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + typ, dat = self._simple_command(name) + if typ == 'OK': + self.sock = ssl_context.wrap_socket(self.sock, + server_hostname=self.host) + self._file = self.sock.makefile('rb') + self._tls_established = True + self._get_capabilities() + else: + raise self.error("Couldn't establish TLS session") + return self._untagged_response(typ, dat, name) + + + def status(self, mailbox, names): + """Request named status conditions for mailbox. + + (typ, [data]) = .status(mailbox, names) + """ + name = 'STATUS' + #if self.PROTOCOL_VERSION == 'IMAP4': # Let the server decide! + # raise self.error('%s unimplemented in IMAP4 (obtain IMAP4rev1 server, or re-code)' % name) + typ, dat = self._simple_command(name, mailbox, names) + return self._untagged_response(typ, dat, name) + + + def store(self, message_set, command, flags): + """Alters flag dispositions for messages in mailbox. + + (typ, [data]) = .store(message_set, command, flags) + """ + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags # Avoid quoting the flags + typ, dat = self._simple_command('STORE', message_set, command, flags) + return self._untagged_response(typ, dat, 'FETCH') + + + def subscribe(self, mailbox): + """Subscribe to new mailbox. + + (typ, [data]) = .subscribe(mailbox) + """ + return self._simple_command('SUBSCRIBE', mailbox) + + + def thread(self, threading_algorithm, charset, *search_criteria): + """IMAPrev1 extension THREAD command. + + (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) + """ + name = 'THREAD' + typ, dat = self._simple_command(name, threading_algorithm, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def uid(self, command, *args): + """Execute "command arg ..." with messages identified by UID, + rather than message number. + + (typ, [data]) = .uid(command, arg1, arg2, ...) + + Returns response appropriate to 'command'. + """ + command = command.upper() + if not command in Commands: + raise self.error("Unknown IMAP4 UID command: %s" % command) + if self.state not in Commands[command]: + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (command, self.state, + ', '.join(Commands[command]))) + name = 'UID' + typ, dat = self._simple_command(name, command, *args) + if command in ('SEARCH', 'SORT', 'THREAD'): + name = command + else: + name = 'FETCH' + return self._untagged_response(typ, dat, name) + + + def unsubscribe(self, mailbox): + """Unsubscribe from old mailbox. + + (typ, [data]) = .unsubscribe(mailbox) + """ + return self._simple_command('UNSUBSCRIBE', mailbox) + + + def unselect(self): + """Free server's resources associated with the selected mailbox + and returns the server to the authenticated state. + This command performs the same actions as CLOSE, except + that no messages are permanently removed from the currently + selected mailbox. + + (typ, [data]) = .unselect() + """ + try: + typ, data = self._simple_command('UNSELECT') + finally: + self.state = 'AUTH' + return typ, data + + + def xatom(self, name, *args): + """Allow simple extension commands + notified by server in CAPABILITY response. + + Assumes command is legal in current state. + + (typ, [data]) = .xatom(name, arg, ...) + + Returns response appropriate to extension command 'name'. + """ + name = name.upper() + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unknown extension command: %s' % name) + if not name in Commands: + Commands[name] = (self.state,) + return self._simple_command(name, *args) + + + + # Private methods + + + def _append_untagged(self, typ, dat): + if dat is None: + dat = b'' + + # During idle, queue untagged responses for delivery via iteration + if self._idle_capture: + # Responses containing literal strings are passed to us one data + # fragment at a time, while others arrive in a single call. + if (not self._idle_responses or + isinstance(self._idle_responses[-1][1][-1], bytes)): + # We are not continuing a fragmented response; start a new one + self._idle_responses.append((typ, [dat])) + else: + # We are continuing a fragmented response; append the fragment + response = self._idle_responses[-1] + assert response[0] == typ + response[1].append(dat) + if __debug__ and self.debug >= 5: + self._mesg(f'idle: queue untagged {typ} {dat!r}') + return + + ur = self.untagged_responses + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] %s += ["%r"]' % + (typ, len(ur.get(typ,'')), dat)) + if typ in ur: + ur[typ].append(dat) + else: + ur[typ] = [dat] + + + def _check_bye(self): + bye = self.untagged_responses.get('BYE') + if bye: + raise self.abort(bye[-1].decode(self._encoding, 'replace')) + + + def _command(self, name, *args): + + if self.state not in Commands[name]: + self.literal = None + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (name, self.state, + ', '.join(Commands[name]))) + + for typ in ('OK', 'NO', 'BAD'): + if typ in self.untagged_responses: + del self.untagged_responses[typ] + + if 'READ-ONLY' in self.untagged_responses \ + and not self.is_readonly: + raise self.readonly('mailbox status changed to READ-ONLY') + + tag = self._new_tag() + name = bytes(name, self._encoding) + data = tag + b' ' + name + for arg in args: + if arg is None: continue + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + data = data + b' ' + arg + + literal = self.literal + if literal is not None: + self.literal = None + if type(literal) is type(self._command): + literator = literal + else: + literator = None + if self.utf8_enabled: + data = data + bytes(' UTF8 (~{%s}' % len(literal), self._encoding) + literal = literal + b')' + else: + data = data + bytes(' {%s}' % len(literal), self._encoding) + + if __debug__: + if self.debug >= 4: + self._mesg('> %r' % data) + else: + self._log('> %r' % data) + + try: + self.send(data + CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if literal is None: + return tag + + while 1: + # Wait for continuation response + + while self._get_response(): + if self.tagged_commands[tag]: # BAD/NO? + return tag + + # Send literal + + if literator: + literal = literator(self.continuation_response) + + if __debug__: + if self.debug >= 4: + self._mesg('write literal size %s' % len(literal)) + + try: + self.send(literal) + self.send(CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if not literator: + break + + return tag + + + def _command_complete(self, name, tag): + logout = (name == 'LOGOUT') + # BYE is expected after LOGOUT + if not logout: + self._check_bye() + try: + typ, data = self._get_tagged_response(tag, expect_bye=logout) + except self.abort as val: + raise self.abort('command: %s => %s' % (name, val)) + except self.error as val: + raise self.error('command: %s => %s' % (name, val)) + if not logout: + self._check_bye() + if typ == 'BAD': + raise self.error('%s command error: %s %s' % (name, typ, data)) + return typ, data + + + def _get_capabilities(self): + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + dat = str(dat[-1], self._encoding) + dat = dat.upper() + self.capabilities = tuple(dat.split()) + + + def _get_response(self, start_timeout=False): + + # Read response and store. + # + # Returns None for continuation responses, + # otherwise first response line received. + # + # If start_timeout is given, temporarily uses it as a socket + # timeout while waiting for the start of a response, raising + # _responsetimeout if one doesn't arrive. (Used by Idler.) + + if start_timeout is not False and self.sock: + assert start_timeout is None or start_timeout > 0 + saved_timeout = self.sock.gettimeout() + self.sock.settimeout(start_timeout) + try: + resp = self._get_line() + except TimeoutError as err: + raise self._responsetimeout from err + finally: + self.sock.settimeout(saved_timeout) + else: + resp = self._get_line() + + # Command completion response? + + if self._match(self.tagre, resp): + tag = self.mo.group('tag') + if not tag in self.tagged_commands: + raise self.abort('unexpected tagged response: %r' % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + self.tagged_commands[tag] = (typ, [dat]) + else: + dat2 = None + + # '*' (untagged) responses? + + if not self._match(Untagged_response, resp): + if self._match(self.Untagged_status, resp): + dat2 = self.mo.group('data2') + + if self.mo is None: + # Only other possibility is '+' (continuation) response... + + if self._match(Continuation, resp): + self.continuation_response = self.mo.group('data') + return None # NB: indicates continuation + + raise self.abort("unexpected response: %r" % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + if dat is None: dat = b'' # Null untagged response + if dat2: dat = dat + b' ' + dat2 + + # Is there a literal to come? + + while self._match(self.Literal, dat): + + # Read literal direct from connection. + + size = int(self.mo.group('size')) + if __debug__: + if self.debug >= 4: + self._mesg('read literal size %s' % size) + data = self.read(size) + + # Store response with literal as tuple + + self._append_untagged(typ, (dat, data)) + + # Read trailer - possibly containing another literal + + dat = self._get_line() + + self._append_untagged(typ, dat) + + # Bracketed response information? + + if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat): + typ = self.mo.group('type') + typ = str(typ, self._encoding) + self._append_untagged(typ, self.mo.group('data')) + + if __debug__: + if self.debug >= 1 and typ in ('NO', 'BAD', 'BYE'): + self._mesg('%s response: %r' % (typ, dat)) + + return resp + + + def _get_tagged_response(self, tag, expect_bye=False): + + while 1: + result = self.tagged_commands[tag] + if result is not None: + del self.tagged_commands[tag] + return result + + if expect_bye: + typ = 'BYE' + bye = self.untagged_responses.pop(typ, None) + if bye is not None: + # Server replies to the "LOGOUT" command with "BYE" + return (typ, bye) + + # If we've seen a BYE at this point, the socket will be + # closed, so report the BYE now. + self._check_bye() + + # Some have reported "unexpected response" exceptions. + # Note that ignoring them here causes loops. + # Instead, send me details of the unexpected response and + # I'll update the code in '_get_response()'. + + try: + self._get_response() + except self.abort as val: + if __debug__: + if self.debug >= 1: + self.print_log() + raise + + + def _get_line(self): + + line = self.readline() + if not line: + raise self.abort('socket error: EOF') + + # Protocol mandates all lines terminated by CRLF + if not line.endswith(b'\r\n'): + raise self.abort('socket error: unterminated line: %r' % line) + + line = line[:-2] + if __debug__: + if self.debug >= 4: + self._mesg('< %r' % line) + else: + self._log('< %r' % line) + return line + + + def _match(self, cre, s): + + # Run compiled regular expression match method on 's'. + # Save result, return success. + + self.mo = cre.match(s) + if __debug__: + if self.mo is not None and self.debug >= 5: + self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) + return self.mo is not None + + + def _new_tag(self): + + tag = self.tagpre + bytes(str(self.tagnum), self._encoding) + self.tagnum = self.tagnum + 1 + self.tagged_commands[tag] = None + return tag + + + def _quote(self, arg): + + arg = arg.replace('\\', '\\\\') + arg = arg.replace('"', '\\"') + + return '"' + arg + '"' + + + def _simple_command(self, name, *args): + + return self._command_complete(name, self._command(name, *args)) + + + def _untagged_response(self, typ, dat, name): + if typ == 'NO': + return typ, dat + if not name in self.untagged_responses: + return typ, [None] + data = self.untagged_responses.pop(name) + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] => %s' % (name, data)) + return typ, data + + + if __debug__: + + def _mesg(self, s, secs=None): + if secs is None: + secs = time.time() + tm = time.strftime('%M:%S', time.localtime(secs)) + sys.stderr.write(' %s.%02d %s\n' % (tm, (secs*100)%100, s)) + sys.stderr.flush() + + def _dump_ur(self, untagged_resp_dict): + if not untagged_resp_dict: + return + items = (f'{key}: {value!r}' + for key, value in untagged_resp_dict.items()) + self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) + + def _log(self, line): + # Keep log of last '_cmd_log_len' interactions for debugging. + self._cmd_log[self._cmd_log_idx] = (line, time.time()) + self._cmd_log_idx += 1 + if self._cmd_log_idx >= self._cmd_log_len: + self._cmd_log_idx = 0 + + def print_log(self): + self._mesg('last %d IMAP4 interactions:' % len(self._cmd_log)) + i, n = self._cmd_log_idx, self._cmd_log_len + while n: + try: + self._mesg(*self._cmd_log[i]) + except: + pass + i += 1 + if i >= self._cmd_log_len: + i = 0 + n -= 1 + + +class Idler: + """Iterable IDLE context manager: start IDLE & produce untagged responses. + + An object of this type is returned by the IMAP4.idle() method. + + Note: The name and structure of this class are subject to change. + """ + + def __init__(self, imap, duration=None): + if 'IDLE' not in imap.capabilities: + raise imap.error("Server does not support IMAP4 IDLE") + if duration is not None and not imap.sock: + # IMAP4_stream pipes don't support timeouts + raise imap.error('duration requires a socket connection') + self._duration = duration + self._deadline = None + self._imap = imap + self._tag = None + self._saved_state = None + + def __enter__(self): + imap = self._imap + assert not imap._idle_responses + assert not imap._idle_capture + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle start duration={self._duration}') + + # Start capturing untagged responses before sending IDLE, + # so we can deliver via iteration any that arrive while + # the IDLE command continuation request is still pending. + imap._idle_capture = True + + try: + self._tag = imap._command('IDLE') + # As with any command, the server is allowed to send us unrelated, + # untagged responses before acting on IDLE. These lines will be + # returned by _get_response(). When the server is ready, it will + # send an IDLE continuation request, indicated by _get_response() + # returning None. We therefore process responses in a loop until + # this occurs. + while resp := imap._get_response(): + if imap.tagged_commands[self._tag]: + typ, data = imap.tagged_commands.pop(self._tag) + if typ == 'NO': + raise imap.error(f'idle denied: {data}') + raise imap.abort(f'unexpected status response: {resp}') + + if __debug__ and imap.debug >= 4: + prompt = imap.continuation_response + imap._mesg(f'idle continuation prompt: {prompt}') + except BaseException: + imap._idle_capture = False + raise + + if self._duration is not None: + self._deadline = time.monotonic() + self._duration + + self._saved_state = imap.state + imap.state = 'IDLING' + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + imap = self._imap + + if __debug__ and imap.debug >= 4: + imap._mesg('idle done') + imap.state = self._saved_state + + # Stop intercepting untagged responses before sending DONE, + # since we can no longer deliver them via iteration. + imap._idle_capture = False + + # If we captured untagged responses while the IDLE command + # continuation request was still pending, but the user did not + # iterate over them before exiting IDLE, we must put them + # someplace where the user can retrieve them. The only + # sensible place for this is the untagged_responses dict, + # despite its unfortunate inability to preserve the relative + # order of different response types. + if leftovers := len(imap._idle_responses): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle quit with {leftovers} leftover responses') + while imap._idle_responses: + typ, data = imap._idle_responses.pop(0) + # Append one fragment at a time, just as _get_response() does + for datum in data: + imap._append_untagged(typ, datum) + + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + except OSError: + if not exc_type: + raise + + return False # Do not suppress context body exceptions + + def __iter__(self): + return self + + def _pop(self, timeout, default=('', None)): + # Get the next response, or a default value on timeout. + # The timeout arg can be an int or float, or None for no timeout. + # Timeouts require a socket connection (not IMAP4_stream). + # This method ignores self._duration. + + # Historical Note: + # The timeout was originally implemented using select() after + # checking for the presence of already-buffered data. + # That allowed timeouts on pipe connetions like IMAP4_stream. + # However, it seemed possible that SSL data arriving without any + # IMAP data afterward could cause select() to indicate available + # application data when there was none, leading to a read() call + # that would block with no timeout. It was unclear under what + # conditions this would happen in practice. Our implementation was + # changed to use socket timeouts instead of select(), just to be + # safe. + + imap = self._imap + if imap.state != 'IDLING': + raise imap.error('_pop() only works during IDLE') + + if imap._idle_responses: + # Response is ready to return to the user + resp = imap._idle_responses.pop(0) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) de-queued {resp[0]}') + return resp + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) reading') + + if timeout is not None: + if timeout <= 0: + return default + timeout = float(timeout) # Required by socket.settimeout() + + try: + imap._get_response(timeout) # Reads line, calls _append_untagged() + except IMAP4._responsetimeout: + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) done') + return default + + resp = imap._idle_responses.pop(0) + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) read {resp[0]}') + return resp + + def __next__(self): + imap = self._imap + + if self._duration is None: + timeout = None + else: + timeout = self._deadline - time.monotonic() + typ, data = self._pop(timeout) + + if not typ: + if __debug__ and imap.debug >= 4: + imap._mesg('idle iterator exhausted') + raise StopIteration + + return typ, data + + def burst(self, interval=0.1): + """Yield a burst of responses no more than 'interval' seconds apart. + + with M.idle() as idler: + # get a response and any others following by < 0.1 seconds + batch = list(idler.burst()) + print(f'processing {len(batch)} responses...') + print(batch) + + Note: This generator requires a socket connection (not IMAP4_stream). + """ + if not self._imap.sock: + raise self._imap.error('burst() requires a socket connection') + + try: + yield next(self) + except StopIteration: + return + + while response := self._pop(interval, None): + yield response + + +if HAVE_SSL: + + class IMAP4_SSL(IMAP4): + + """IMAP4 client class over SSL connection + + Instantiate with: IMAP4_SSL([host[, port[, ssl_context[, timeout=None]]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + ssl_context - a SSLContext object that contains your certificate chain + and private key (default: None) + timeout - socket timeout (default: None) If timeout is not given or is None, + the global default socket timeout is used + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, host='', port=IMAP4_SSL_PORT, + *, ssl_context=None, timeout=None): + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + self.ssl_context = ssl_context + IMAP4.__init__(self, host, port, timeout) + + def _create_socket(self, timeout): + sock = IMAP4._create_socket(self, timeout) + return self.ssl_context.wrap_socket(sock, + server_hostname=self.host) + + def open(self, host='', port=IMAP4_SSL_PORT, timeout=None): + """Setup connection to remote server on "host:port". + (default: localhost:standard IMAP4 SSL port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + IMAP4.open(self, host, port, timeout) + + __all__.append("IMAP4_SSL") + + +class IMAP4_stream(IMAP4): + + """IMAP4 client class over a stream + + Instantiate with: IMAP4_stream(command) + + "command" - a string that can be passed to subprocess.Popen() + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, command): + self.command = command + IMAP4.__init__(self) + + + def open(self, host=None, port=None, timeout=None): + """Setup a stream connection. + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = None # For compatibility with parent class + self.port = None + self.sock = None + self._file = None + self.process = subprocess.Popen(self.command, + bufsize=DEFAULT_BUFFER_SIZE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + shell=True, close_fds=True) + self.writefile = self.process.stdin + self.readfile = self.process.stdout + + def read(self, size): + """Read 'size' bytes from remote.""" + return self.readfile.read(size) + + + def readline(self): + """Read line from remote.""" + return self.readfile.readline() + + + def send(self, data): + """Send data to remote.""" + self.writefile.write(data) + self.writefile.flush() + + + def shutdown(self): + """Close I/O established in "open".""" + self.readfile.close() + self.writefile.close() + self.process.wait() + + + +class _Authenticator: + + """Private class to provide en/decoding + for base64-based authentication conversation. + """ + + def __init__(self, mechinst): + self.mech = mechinst # Callable object to provide/process data + + def process(self, data): + ret = self.mech(self.decode(data)) + if ret is None: + return b'*' # Abort conversation + return self.encode(ret) + + def encode(self, inp): + # + # Invoke binascii.b2a_base64 iteratively with + # short even length buffers, strip the trailing + # line feed from the result and append. "Even" + # means a number that factors to both 6 and 8, + # so when it gets to the end of the 8-bit input + # there's no partial 6-bit output. + # + oup = b'' + if isinstance(inp, str): + inp = inp.encode('utf-8') + while inp: + if len(inp) > 48: + t = inp[:48] + inp = inp[48:] + else: + t = inp + inp = b'' + e = binascii.b2a_base64(t) + if e: + oup = oup + e[:-1] + return oup + + def decode(self, inp): + if not inp: + return b'' + return binascii.a2b_base64(inp) + +Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') +Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} + +def Internaldate2tuple(resp): + """Parse an IMAP4 INTERNALDATE string. + + Return corresponding local time. The return value is a + time.struct_time tuple or None if the string has wrong format. + """ + + mo = InternalDate.match(resp) + if not mo: + return None + + mon = Mon2num[mo.group('mon')] + zonen = mo.group('zonen') + + day = int(mo.group('day')) + year = int(mo.group('year')) + hour = int(mo.group('hour')) + min = int(mo.group('min')) + sec = int(mo.group('sec')) + zoneh = int(mo.group('zoneh')) + zonem = int(mo.group('zonem')) + + # INTERNALDATE timezone must be subtracted to get UT + + zone = (zoneh*60 + zonem)*60 + if zonen == b'-': + zone = -zone + + tt = (year, mon, day, hour, min, sec, -1, -1, -1) + utc = calendar.timegm(tt) - zone + + return time.localtime(utc) + + + +def Int2AP(num): + + """Convert integer to A-P string representation.""" + + val = b''; AP = b'ABCDEFGHIJKLMNOP' + num = int(abs(num)) + while num: + num, mod = divmod(num, 16) + val = AP[mod:mod+1] + val + return val + + + +def ParseFlags(resp): + + """Convert IMAP4 flags response to python tuple.""" + + mo = Flags.match(resp) + if not mo: + return () + + return tuple(mo.group('flags').split()) + + +def Time2Internaldate(date_time): + + """Convert date_time to IMAP4 INTERNALDATE representation. + + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time, an instance of time.struct_time (as + returned by time.localtime()), an aware datetime instance or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. + """ + if isinstance(date_time, (int, float)): + dt = datetime.fromtimestamp(date_time, + timezone.utc).astimezone() + elif isinstance(date_time, tuple): + try: + gmtoff = date_time.tm_gmtoff + except AttributeError: + if time.daylight: + dst = date_time[8] + if dst == -1: + dst = time.localtime(time.mktime(date_time))[8] + gmtoff = -(time.timezone, time.altzone)[dst] + else: + gmtoff = -time.timezone + delta = timedelta(seconds=gmtoff) + dt = datetime(*date_time[:6], tzinfo=timezone(delta)) + elif isinstance(date_time, datetime): + if date_time.tzinfo is None: + raise ValueError("date_time must be aware") + dt = date_time + elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): + return date_time # Assume in correct format + else: + raise ValueError("date_time not of a known type") + fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month]) + return dt.strftime(fmt) + + + +if __name__ == '__main__': + + # To test: invoke either as 'python imaplib.py [IMAP4_server_hostname]' + # or 'python imaplib.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' + # to test the IMAP4_stream class + + import getopt, getpass + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'd:s:') + except getopt.error as val: + optlist, args = (), () + + stream_command = None + for opt,val in optlist: + if opt == '-d': + Debug = int(val) + elif opt == '-s': + stream_command = val + if not args: args = (stream_command,) + + if not args: args = ('',) + + host = args[0] + + USER = getpass.getuser() + PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) + + test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)sdata...%(lf)s' % {'user':USER, 'lf':'\n'} + test_seq1 = ( + ('login', (USER, PASSWD)), + ('create', ('/tmp/xxx 1',)), + ('rename', ('/tmp/xxx 1', '/tmp/yyy')), + ('CREATE', ('/tmp/yyz 2',)), + ('append', ('/tmp/yyz 2', None, None, test_mesg)), + ('list', ('/tmp', 'yy*')), + ('select', ('/tmp/yyz 2',)), + ('search', (None, 'SUBJECT', 'test')), + ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), + ('store', ('1', 'FLAGS', r'(\Deleted)')), + ('namespace', ()), + ('expunge', ()), + ('recent', ()), + ('close', ()), + ) + + test_seq2 = ( + ('select', ()), + ('response',('UIDVALIDITY',)), + ('uid', ('SEARCH', 'ALL')), + ('response', ('EXISTS',)), + ('append', (None, None, None, test_mesg)), + ('recent', ()), + ('logout', ()), + ) + + def run(cmd, args): + M._mesg('%s %s' % (cmd, args)) + typ, dat = getattr(M, cmd)(*args) + M._mesg('%s => %s %s' % (cmd, typ, dat)) + if typ == 'NO': raise dat[0] + return dat + + try: + if stream_command: + M = IMAP4_stream(stream_command) + else: + M = IMAP4(host) + if M.state == 'AUTH': + test_seq1 = test_seq1[1:] # Login not needed + M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + M._mesg('CAPABILITIES = %r' % (M.capabilities,)) + + for cmd,args in test_seq1: + run(cmd, args) + + for ml in run('list', ('/tmp/', 'yy%')): + mo = re.match(r'.*"([^"]+)"$', ml) + if mo: path = mo.group(1) + else: path = ml.split()[-1] + run('delete', (path,)) + + for cmd,args in test_seq2: + dat = run(cmd, args) + + if (cmd,args) != ('uid', ('SEARCH', 'ALL')): + continue + + uid = dat[-1].split() + if not uid: continue + run('uid', ('FETCH', '%s' % uid[-1], + '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) + + print('\nAll tests OK.') + + except: + print('\nTests failed.') + + if not Debug: + print(''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0]) + + raise diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 707c081cb2c..a7d57561ead 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -54,8 +54,6 @@ # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import warnings - # Public API ######################################################### @@ -105,7 +103,7 @@ def reload(module): try: name = module.__name__ except AttributeError: - raise TypeError("reload() argument must be a module") + raise TypeError("reload() argument must be a module") from None if sys.modules.get(name) is not module: raise ImportError(f"module {name} not in sys.modules", name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 093a0b82456..499da1e04ef 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to @@ -382,6 +382,9 @@ def release(self): self.waiters.pop() self.wakeup.release() + def locked(self): + return bool(self.count) + def __repr__(self): return f'_ModuleLock({self.name!r}) at {id(self)}' @@ -490,8 +493,7 @@ def _call_with_frames_removed(f, *args, **kwds): def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - # XXX RUSTPYTHON: hasattr check because we might be bootstrapping and we wouldn't have stderr yet - if sys.flags.verbose >= verbosity and hasattr(sys, "stderr"): + if sys.flags.verbose >= verbosity: if not message.startswith(('#', 'import ')): message = '# ' + message print(message.format(*args), file=sys.stderr) @@ -527,7 +529,7 @@ def _load_module_shim(self, fullname): """ msg = ("the load_module() method is deprecated and slated for removal in " - "Python 3.12; use exec_module() instead") + "Python 3.15; use exec_module() instead") _warnings.warn(msg, DeprecationWarning) spec = spec_from_loader(fullname, self) if fullname in sys.modules: @@ -825,10 +827,16 @@ def _module_repr_from_spec(spec): """Return the repr to use for the module.""" name = '?' if spec.name is None else spec.name if spec.origin is None: - if spec.loader is None: + loader = spec.loader + if loader is None: return f'' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'' else: - return f'' + return f'' else: if spec.has_location: return f'' @@ -1129,7 +1137,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. @@ -1236,10 +1244,12 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") + # gh-130094: Copy sys.meta_path so that we have a consistent view of the + # list while iterating over it. + meta_path = list(meta_path) if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) @@ -1294,7 +1304,6 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1304,8 +1313,9 @@ def _find_and_load_unlocked(name, import_): if parent not in sys.modules: _call_with_frames_removed(import_, parent) # Crazy side-effects! - if name in sys.modules: - return sys.modules[name] + module = sys.modules.get(name) + if module is not None: + return module parent_module = sys.modules[parent] try: path = parent_module.__path__ @@ -1313,6 +1323,12 @@ def _find_and_load_unlocked(name, import_): msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ + if getattr(parent_spec, '_initializing', False): + _call_with_frames_removed(import_, parent) + # Crazy side-effects (again)! + module = sys.modules.get(name) + if module is not None: + return module child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 73ac4405cb5..95ce14b2c39 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -81,6 +81,11 @@ def _pack_uint32(x): return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') +def _unpack_uint64(data): + """Convert 8 bytes in little-endian to an integer.""" + assert len(data) == 8 + return int.from_bytes(data, 'little') + def _unpack_uint32(data): """Convert 4 bytes in little-endian to an integer.""" assert len(data) == 4 @@ -203,7 +208,7 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: + with _io.open(fd, 'wb') as file: file.write(data) _os.replace(path_tmp, path) except OSError: @@ -216,254 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) - -# Python 3.13 will start with 3550 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (3531).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -535,7 +293,8 @@ def cache_from_source(path, debug_override=None, *, optimization=None): # Strip initial drive from a Windows path. We know we have an absolute # path here, so the second part of the check rules out a POSIX path that # happens to contain a colon at the second character. - if head[1] == ':' and head[0] not in path_separators: + # Slicing avoids issues with an empty (or short) `head`. + if head[1:2] == ':' and head[0:1] not in path_separators: head = head[2:] # Strip initial path separator from `head` to complete the conversion @@ -954,6 +713,12 @@ def _search_registry(cls, fullname): @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None @@ -1102,7 +867,7 @@ def get_code(self, fullname): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1131,7 +896,7 @@ def get_code(self, fullname): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1437,7 +1202,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. @@ -1449,6 +1214,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" @@ -1473,7 +1241,7 @@ def _path_importer_cache(cls, path): if path == '': try: path = _os.getcwd() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Don't cache the failure as the cwd can easily change to # a valid directory later on. return None @@ -1690,6 +1458,52 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + try: + module.__file__ = path + except AttributeError: + # Not important enough to report. + # (The error is also ignored in _bootstrap._init_module_attrs or + # import_run_extension in import.c) + pass + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1722,10 +1536,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + extension_loaders = [] + if hasattr(_imp, 'create_dynamic'): + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c..1e47495f65f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -80,10 +64,13 @@ def invalidate_caches(self): class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod @@ -180,7 +167,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): @@ -211,6 +202,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b..63d726445c3 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,18 +3,48 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 82e0ce1b281..8ce62dd864f 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,9 +1,13 @@ +from __future__ import annotations + import os import re import abc -import csv import sys +import json import email +import types +import inspect import pathlib import zipfile import operator @@ -12,11 +16,9 @@ import functools import itertools import posixpath -import contextlib import collections -import inspect -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -26,8 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import List, Mapping, Optional, cast - +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -48,11 +49,11 @@ class PackageNotFoundError(ModuleNotFoundError): """The package was not found.""" - def __str__(self): + def __str__(self) -> str: return f"No package metadata was found for {self.name}" @property - def name(self): + def name(self) -> str: # type: ignore[override] (name,) = self.args return name @@ -118,38 +119,11 @@ def read(text, filter_=None): yield Pair(name, value) @staticmethod - def valid(line): + def valid(line: str): return line and not line.startswith('#') -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=2, - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): +class EntryPoint: """An entry point as defined by Python packaging conventions. See `the packaging docs on entry points @@ -191,34 +165,37 @@ class EntryPoint(DeprecatedTuple): value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None - def __init__(self, name, value, group): + def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @property - def module(self): + def module(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('module') @property - def attr(self): + def attr(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('attr') @property - def extras(self): + def extras(self) -> List[str]: match = self.pattern.match(self.value) + assert match is not None return re.findall(r'\w+', match.group('extras') or '') def _for(self, dist): @@ -266,7 +243,7 @@ def __repr__(self): f'group={self.group!r})' ) - def __hash__(self): + def __hash__(self) -> int: return hash(self._key()) @@ -277,7 +254,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name): # -> EntryPoint: + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] """ Get the EntryPoint in self matching name. """ @@ -286,7 +263,14 @@ def __getitem__(self, name): # -> EntryPoint: except StopIteration: raise KeyError(name) - def select(self, **params): + def __repr__(self): + """ + Repr with classname and tuple constructor to + signal that we deviate from regular tuple behavior. + """ + return '%s(%r)' % (self.__class__.__name__, tuple(self)) + + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -294,14 +278,14 @@ def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self): + def names(self) -> Set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self): + def groups(self) -> Set[str]: """ Return the set of all groups of all entry points. """ @@ -322,28 +306,31 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() + hash: Optional[FileHash] + size: int + dist: Distribution - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() + def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] + return self.locate().read_text(encoding=encoding) - def locate(self): + def read_binary(self) -> bytes: + return self.locate().read_bytes() + + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) class FileHash: - def __init__(self, spec): + def __init__(self, spec: str) -> None: self.mode, _, self.value = spec.partition('=') - def __repr__(self): + def __repr__(self) -> str: return f'' class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -363,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str): + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -394,21 +404,23 @@ def from_name(cls, name: str): if not name: raise ValueError("A distribution name is required.") try: - return next(cls.discover(name=name)) + return next(iter(cls.discover(name=name))) except StopIteration: raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs): + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -417,8 +429,8 @@ def discover(cls, **kwargs): ) @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -427,7 +439,7 @@ def at(path): @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -438,8 +450,15 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') @@ -452,7 +471,7 @@ def metadata(self) -> _meta.PackageMetadata: return _adapters.Message(email.message_from_string(text)) @property - def name(self): + def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] @@ -462,16 +481,22 @@ def _normalized_name(self): return Prepared.normalize(self.name) @property - def version(self): + def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" return self.metadata['Version'] @property - def entry_points(self): + def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self): + def files(self) -> Optional[List[PackagePath]]: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -480,6 +505,10 @@ def files(self): (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -491,6 +520,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none @@ -507,7 +540,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -534,7 +567,7 @@ def _read_files_egginfo_installed(self): paths = ( (subdir / name) .resolve() - .relative_to(self.locate_file('').resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) .as_posix() for name in text.splitlines() ) @@ -556,7 +589,7 @@ def _read_files_egginfo_sources(self): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self): + def requires(self) -> Optional[List[str]]: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -607,10 +640,23 @@ def url_req_space(req): space = url_req_space(section.value) yield section.value + space + quoted_marker(section.name) + @property + def origin(self): + return self._load_json('direct_url.json') + + def _load_json(self, filename): + return pass_none(json.loads)( + self.read_text(filename), + object_hook=lambda data: types.SimpleNamespace(**data), + ) + class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -623,6 +669,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -635,7 +692,7 @@ def __init__(self, **kwargs): vars(self).update(kwargs) @property - def path(self): + def path(self) -> List[str]: """ The sequence of directory path that a distribution finder should search. @@ -646,7 +703,7 @@ def path(self): return vars(self).get('path', sys.path) @abc.abstractmethod - def find_distributions(self, context=Context()): + def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ Find distributions. @@ -658,11 +715,18 @@ def find_distributions(self, context=Context()): class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -704,7 +768,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -725,7 +801,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -741,13 +820,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -775,7 +869,9 @@ def __bool__(self): class MetadataPathFinder(DistributionFinder): @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): + def find_distributions( + cls, context=DistributionFinder.Context() + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -795,19 +891,20 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) - def invalidate_caches(cls): + @classmethod + def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() class PathDistribution(Distribution): - def __init__(self, path: SimplePath): + def __init__(self, path: SimplePath) -> None: """Construct a distribution. :param path: SimplePath indicating the metadata directory. """ self._path = path - def read_text(self, filename): + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -817,9 +914,11 @@ def read_text(self, filename): ): return self._path.joinpath(filename).read_text(encoding='utf-8') + return None + read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property @@ -852,7 +951,7 @@ def _name_from_stem(stem): return name -def distribution(distribution_name): +def distribution(distribution_name: str) -> Distribution: """Get the ``Distribution`` instance for the named package. :param distribution_name: The name of the distribution package as a string. @@ -861,7 +960,7 @@ def distribution(distribution_name): return Distribution.from_name(distribution_name) -def distributions(**kwargs): +def distributions(**kwargs) -> Iterable[Distribution]: """Get all ``Distribution`` instances in the current environment. :return: An iterable of ``Distribution`` instances. @@ -869,7 +968,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> _meta.PackageMetadata: +def metadata(distribution_name: str) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -878,7 +977,7 @@ def metadata(distribution_name) -> _meta.PackageMetadata: return Distribution.from_name(distribution_name).metadata -def version(distribution_name): +def version(distribution_name: str) -> str: """Get the version string for the named package. :param distribution_name: The name of the distribution package to query. @@ -912,7 +1011,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name): +def files(distribution_name: str) -> Optional[List[PackagePath]]: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -921,11 +1020,11 @@ def files(distribution_name): return distribution(distribution_name).files -def requires(distribution_name): +def requires(distribution_name: str) -> Optional[List[str]]: """ Return a list of requirements for the named package. - :return: An iterator of requirements, suitable for + :return: An iterable of requirements, suitable for packaging.requirement.Requirement. """ return distribution(distribution_name).requires @@ -952,13 +1051,42 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() +def _topmost(name: PackagePath) -> Optional[str]: + """ + Return the top-most parent as long as there is a parent. + """ + top, *rest = name.parts + return top if rest else None + + +def _get_toplevel_name(name: PackagePath) -> str: + """ + Infer a possibly importable module name from a name presumed on + sys.path. + + >>> _get_toplevel_name(PackagePath('foo.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pyc')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo/__init__.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pth')) + 'foo.pth' + >>> _get_toplevel_name(PackagePath('foo.dist-info')) + 'foo.dist-info' + """ + return _topmost(name) or ( + # python/typeshed#10328 + inspect.getmodulename(name) # type: ignore + or str(name) + ) + + def _top_level_inferred(dist): - opt_names = { - f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) - for f in always_iterable(dist.files) - } + opt_names = set(map(_get_toplevel_name, always_iterable(dist.files))) - @pass_none def importable_name(name): return '.' not in name diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 6aed69a3085..59116880895 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -53,7 +53,7 @@ def __iter__(self): def __getitem__(self, item): """ Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. + missing key is supplied. Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index c9a7ef906a8..1927d0f624d 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/importlib/metadata/diagnose.py b/Lib/importlib/metadata/diagnose.py new file mode 100644 index 00000000000..e405471ac4d --- /dev/null +++ b/Lib/importlib/metadata/diagnose.py @@ -0,0 +1,21 @@ +import sys + +from . import Distribution + + +def inspect(path): + print("Inspecting", path) + dists = list(Distribution.discover(path=[path])) + if not dists: + return + print("Found", len(dists), "packages:", end=' ') + print(', '.join(dist.name for dist in dists)) + + +def run(): + for path in sys.path: + inspect(path) + + +if __name__ == '__main__': + run() diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 34e3a9950cc..723c9f9eb33 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -1,20 +1,27 @@ -"""Read resources contained within a package.""" +""" +Read resources contained within a package. + +This codebase is shared between importlib.resources in the stdlib +and importlib_resources in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" from ._common import ( as_file, files, Package, + Anchor, ) -from ._legacy import ( +from ._functional import ( contents, + is_resource, open_binary, - read_binary, open_text, - read_text, - is_resource, path, - Resource, + read_binary, + read_text, ) from .abc import ResourceReader @@ -22,11 +29,11 @@ __all__ = [ 'Package', - 'Resource', + 'Anchor', 'ResourceReader', 'as_file', - 'contents', 'files', + 'contents', 'is_resource', 'open_binary', 'open_text', diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index b402e05116e..4e9014c45a0 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -27,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() @@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr] if reader is None: return None - return reader(spec.name) # type: ignore + return reader(spec.name) # type: ignore[union-attr] @functools.singledispatch @@ -77,12 +77,12 @@ def resolve(cand: Optional[Anchor]) -> types.ModuleType: return cast(types.ModuleType, cand) -@resolve.register(str) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: str) -> types.ModuleType: return importlib.import_module(cand) -@resolve.register(type(None)) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: None) -> types.ModuleType: return resolve(_infer_caller().f_globals['__name__']) @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame @@ -109,6 +110,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() @@ -179,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000..f59416f2dd6 --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,81 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbe..70fc7e2b9c0 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,8 +1,14 @@ +from __future__ import annotations + import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile +from collections.abc import Iterator from . import abc @@ -31,8 +37,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): @@ -62,7 +70,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +138,40 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path))) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable | None: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + + path_str might also be a sentinel used by editable packages to + trigger other behaviors (see python/importlib_resources#311). + In that case, return None. + """ + dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return next(dirs, None) + + @classmethod + def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]: + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str: str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +183,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84..2e75299b13a 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -77,7 +77,7 @@ class ResourceHandle(Traversable): def __init__(self, parent: ResourceContainer, name: str): self.parent = parent - self.name = name # type: ignore + self.name = name # type: ignore[misc] def is_file(self): return True @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f4d6e823315..2b564e9b52e 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -135,7 +134,7 @@ class _incompatible_extension_module_restrictions: may not be imported in a subinterpreter. That implies modules that do not implement multi-phase init or that explicitly of out. - Likewise for modules import in a subinterpeter with its own GIL + Likewise for modules import in a subinterpreter with its own GIL when the extension does not support a per-interpreter GIL. This implies the module does not have a Py_mod_multiple_interpreters slot set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. @@ -145,7 +144,7 @@ class _incompatible_extension_module_restrictions: You can get the same effect as this function by implementing the basic interface of multi-phase init (PEP 489) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). """ def __init__(self, *, disable_check): @@ -171,36 +170,57 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + __class__ = loader_state['__class__'] + + # Reentrant calls from the same thread must be allowed to proceed without + # triggering the load again. + # exec_module() and self-referential imports are the primary ways this can + # happen, but in any case we must return something to avoid deadlock. + if loader_state['is_loading']: + return __class__.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = __class__.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) + return getattr(self, attr) def __delattr__(self, attr): @@ -235,6 +255,9 @@ def create_module(self, spec): def exec_module(self, module): """Make the module load lazily.""" + # Threading is only needed for lazy loading, and importlib.util can + # be pulled in at interpreter startup, so defer until needed. + import threading module.__spec__.loader = self.loader module.__loader__ = self.loader # Don't need to worry about deep-copying as trying to set an attribute @@ -244,5 +267,13 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index 5a814f97b5b..3cee85f39a6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -6,9 +6,9 @@ Here are some of the useful functions provided by this module: - ismodule(), isclass(), ismethod(), isfunction(), isgeneratorfunction(), - isgenerator(), istraceback(), isframe(), iscode(), isbuiltin(), - isroutine() - check object types + ismodule(), isclass(), ismethod(), ispackage(), isfunction(), + isgeneratorfunction(), isgenerator(), istraceback(), isframe(), + iscode(), isbuiltin(), isroutine() - check object types getmembers() - get members of an object that satisfy a given condition getfile(), getsourcefile(), getsource() - find an object's source code @@ -24,8 +24,6 @@ stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable - - get_annotations() - safely compute an object's annotations """ # This module is in the public domain. No warranties. @@ -58,6 +56,8 @@ "CO_OPTIMIZED", "CO_VARARGS", "CO_VARKEYWORDS", + "CO_HAS_DOCSTRING", + "CO_METHOD", "ClassFoundException", "ClosureVars", "EndOfBlock", @@ -130,6 +130,7 @@ "ismethoddescriptor", "ismethodwrapper", "ismodule", + "ispackage", "isroutine", "istraceback", "markcoroutinefunction", @@ -142,6 +143,8 @@ import abc +from annotationlib import Format, ForwardRef +from annotationlib import get_annotations # re-exported import ast import dis import collections.abc @@ -173,127 +176,6 @@ TPFLAGS_IS_ABSTRACT = 1 << 20 -def get_annotations(obj, *, globals=None, locals=None, eval_str=False): - """Compute the annotations dict for an object. - - obj may be a callable, class, or module. - Passing in an object of any other type raises TypeError. - - Returns a dict. get_annotations() returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If eval_str is true, values of type str will - be un-stringized using eval(). This is intended - for use with stringized annotations - ("from __future__ import annotations"). - * If obj doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using getattr() and dict.get() for safety. - * Always, always, always returns a freshly-created dict. - - eval_str controls whether or not values of type str are replaced - with the result of calling eval() on those values: - - * If eval_str is true, eval() is called on values of type str. - * If eval_str is false (the default), values of type str are unchanged. - - globals and locals are passed in to eval(); see the documentation - for eval() for more information. If either globals or locals is - None, this function may replace that value with a context-specific - default, contingent on type(obj): - - * If obj is a module, globals defaults to obj.__dict__. - * If obj is a class, globals defaults to - sys.modules[obj.__module__].__dict__ and locals - defaults to the obj class namespace. - * If obj is a callable, globals defaults to obj.__globals__, - although if obj is a wrapped function (using - functools.update_wrapper()) it is first unwrapped. - """ - if isinstance(obj, type): - # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None - - obj_globals = None - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, '__dict__', None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__dict__') - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__globals__', None) - obj_locals = None - unwrap = obj - else: - raise TypeError(f"{obj!r} is not a module, class, or callable.") - - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - - if not ann: - return {} - - if not eval_str: - return dict(ann) - - if unwrap is not None: - while True: - if hasattr(unwrap, '__wrapped__'): - unwrap = unwrap.__wrapped__ - continue - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals or {} - - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - if type_params := getattr(obj, "__type_params__", ()): - locals = {param.__name__: param for param in type_params} | locals - - return_value = {key: - value if not isinstance(value, str) else eval(value, globals, locals) - for key, value in ann.items() } - return return_value - - # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module.""" @@ -307,6 +189,10 @@ def ismethod(object): """Return true if the object is an instance method.""" return isinstance(object, types.MethodType) +def ispackage(object): + """Return true if the object is a package.""" + return ismodule(object) and hasattr(object, "__path__") + def ismethoddescriptor(object): """Return true if the object is a method descriptor. @@ -325,11 +211,6 @@ def ismethoddescriptor(object): if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False - if isinstance(object, functools.partial): - # Lie for children. The addition of partial.__get__ - # doesn't currently change the partial objects behaviour, - # not counting a warning about future changes. - return False tp = type(object) return (hasattr(tp, "__get__") and not hasattr(tp, "__set__") @@ -389,11 +270,16 @@ def isfunction(object): Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined + __qualname__ qualified name of this function + __module__ name of the module the function was defined in or None __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations - __kwdefaults__ dict of keyword only parameters with defaults""" + __kwdefaults__ dict of keyword only parameters with defaults + __dict__ namespace which is supporting arbitrary function attributes + __closure__ a tuple of cells or None + __type_params__ tuple of type parameters""" return isinstance(object, types.FunctionType) def _has_code_flag(f, flag): @@ -458,17 +344,19 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container - close raises a new GeneratorExit exception inside the - generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise - next return the next item from the container - send resumes the generator and "sends" a value that becomes + gi_suspended set to 1 when the generator is suspended at a yield point, 0 otherwise + gi_yieldfrom object being iterated by yield from or None + + __iter__() defined to support iteration over container + close() raises a new GeneratorExit exception inside the + generator to terminate the iteration + send() resumes the generator and "sends" a value that becomes the result of the current yield-expression - throw used to raise an exception inside the generator""" + throw() used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): @@ -503,7 +391,11 @@ def isframe(object): f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame - f_trace tracing function for this frame, or None""" + f_trace tracing function for this frame, or None + f_trace_lines is a tracing event triggered for each source line? + f_trace_opcodes are per-opcode events being requested? + + clear() used to clear all references to local variables""" return isinstance(object, types.FrameType) def iscode(object): @@ -520,6 +412,7 @@ def iscode(object): co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg | 16=nested | 32=generator | 64=nofree | 128=coroutine | 256=iterable_coroutine | 512=async_generator + | 0x4000000=has_docstring co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) @@ -528,7 +421,12 @@ def iscode(object): co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_varnames tuple of names of arguments and local variables + co_qualname fully qualified function name + + co_lines() returns an iterator that yields successive bytecode ranges + co_positions() returns an iterator of source code positions for each bytecode instruction + replace() returns a copy of the code object with a new values""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -550,7 +448,8 @@ def isroutine(object): or isfunction(object) or ismethod(object) or ismethoddescriptor(object) - or ismethodwrapper(object)) + or ismethodwrapper(object) + or isinstance(object, functools._singledispatchmethod_get)) def isabstract(object): """Return true if the object is an abstract base class (ABC).""" @@ -961,8 +860,7 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] - all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] if any(filename.endswith(s) for s in all_bytecode_suffixes): filename = (os.path.splitext(filename)[0] + importlib.machinery.SOURCE_SUFFIXES[0]) @@ -1438,7 +1336,9 @@ def getargvalues(frame): args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) -def formatannotation(annotation, base_module=None): +def formatannotation(annotation, base_module=None, *, quote_annotation_strings=True): + if not quote_annotation_strings and isinstance(annotation, str): + return annotation if getattr(annotation, '__module__', None) == 'typing': def repl(match): text = match.group() @@ -1450,6 +1350,8 @@ def repl(match): if annotation.__module__ in ('builtins', base_module): return annotation.__qualname__ return annotation.__module__+'.'+annotation.__qualname__ + if isinstance(annotation, ForwardRef): + return annotation.__forward_arg__ return repr(annotation) def formatannotationrelativeto(object): @@ -2067,7 +1969,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -2089,7 +1996,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # also change kind to positional only + # and remove default + if arg_value is functools.Placeholder: + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -2167,13 +2084,11 @@ def _signature_is_functionlike(obj): code = getattr(obj, '__code__', None) defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here - annotations = getattr(obj, '__annotations__', None) return (isinstance(code, types.CodeType) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and - (kwdefaults is None or isinstance(kwdefaults, dict)) and - (isinstance(annotations, (dict)) or annotations is None) ) + (kwdefaults is None or isinstance(kwdefaults, dict))) def _signature_strip_non_python_syntax(signature): @@ -2390,7 +2305,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): def _signature_from_function(cls, func, skip_bound_arg=True, - globals=None, locals=None, eval_str=False): + globals=None, locals=None, eval_str=False, + *, annotation_format=Format.VALUE): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2416,7 +2332,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str, + format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2499,7 +2416,8 @@ def _signature_from_callable(obj, *, globals=None, locals=None, eval_str=False, - sigcls): + sigcls, + annotation_format=Format.VALUE): """Private helper function to get signature for arbitrary callable objects. @@ -2511,7 +2429,8 @@ def _signature_from_callable(obj, *, globals=globals, locals=locals, sigcls=sigcls, - eval_str=eval_str) + eval_str=eval_str, + annotation_format=annotation_format) if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) @@ -2544,18 +2463,10 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: - # since __text_signature__ is not writable on classes, __signature__ - # may contain text (or be a callable that returns text); - # if so, convert it - o_sig = sig - if not isinstance(sig, (Signature, str)) and callable(sig): - sig = sig() - if isinstance(sig, str): - sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(o_sig)) + 'attribute'.format(sig)) return sig try: @@ -2583,6 +2494,11 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) @@ -2595,7 +2511,8 @@ def _signature_from_callable(obj, *, # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, skip_bound_arg=skip_bound_arg, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, @@ -2848,13 +2765,17 @@ def replace(self, *, name=_void, kind=_void, return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): + return self._format() + + def _format(self, *, quote_annotation_strings=True): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: - formatted = '{}: {}'.format(formatted, - formatannotation(self._annotation)) + annotation = formatannotation(self._annotation, + quote_annotation_strings=quote_annotation_strings) + formatted = '{}: {}'.format(formatted, annotation) if self._default is not _empty: if self._annotation is not _empty: @@ -3061,11 +2982,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params = OrderedDict() top_kind = _POSITIONAL_ONLY seen_default = False + seen_var_parameters = set() for param in parameters: kind = param.kind name = param.name + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + if kind in seen_var_parameters: + msg = f'more than one {kind.description} parameter' + raise ValueError(msg) + + seen_var_parameters.add(kind) + if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' @@ -3102,11 +3031,13 @@ def __init__(self, parameters=None, *, return_annotation=_empty, @classmethod def from_callable(cls, obj, *, - follow_wrapped=True, globals=None, locals=None, eval_str=False): + follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, follow_wrapper_chains=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) @property def parameters(self): @@ -3324,19 +3255,24 @@ def __repr__(self): def __str__(self): return self.format() - def format(self, *, max_width=None): + def format(self, *, max_width=None, quote_annotation_strings=True): """Create a string representation of the Signature object. If *max_width* integer is passed, signature will try to fit into the *max_width*. If signature is longer than *max_width*, all parameters will be on separate lines. + + If *quote_annotation_strings* is False, annotations + in the signature are displayed without opening and closing quotation + marks. This is useful when the signature was created with the + STRING format or when ``from __future__ import annotations`` was used. """ result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): - formatted = str(param) + formatted = param._format(quote_annotation_strings=quote_annotation_strings) kind = param.kind @@ -3373,16 +3309,19 @@ def format(self, *, max_width=None): rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) + anno = formatannotation(self.return_annotation, + quote_annotation_strings=quote_annotation_strings) rendered += ' -> {}'.format(anno) return rendered -def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) class BufferFlags(enum.IntFlag): @@ -3412,7 +3351,7 @@ def _main(): import argparse import importlib - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument( 'object', help="The object to be analysed. " diff --git a/Lib/io.py b/Lib/io.py index f0e2fa15d5a..63ffadb1d38 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -46,21 +46,20 @@ "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", - "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder", + "Reader", "Writer"] import _io import abc +from _collections_abc import _check_methods from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, text_encoding, TextIOWrapper) -# Pretend this exception was created here. -UnsupportedOperation.__module__ = "io" - # for seek() SEEK_SET = 0 SEEK_CUR = 1 @@ -97,3 +96,55 @@ class TextIOBase(_io._TextIOBase, IOBase): pass else: RawIOBase.register(_WindowsConsoleIO) + +# +# Static Typing Support +# + +GenericAlias = type(list[int]) + + +class Reader(metaclass=abc.ABCMeta): + """Protocol for simple I/O reader instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def read(self, size=..., /): + """Read data from the input stream and return it. + + If *size* is specified, at most *size* items (bytes/characters) will be + read. + """ + + @classmethod + def __subclasshook__(cls, C): + if cls is Reader: + return _check_methods(C, "read") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Writer(metaclass=abc.ABCMeta): + """Protocol for simple I/O writer instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def write(self, data, /): + """Write *data* to the output stream and return the number of items written.""" + + @classmethod + def __subclasshook__(cls, C): + if cls is Writer: + return _check_methods(C, "write") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 67e45450fc1..ca732e4f2e8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -239,7 +239,7 @@ def summarize_address_range(first, last): else: raise ValueError('unknown IP version') - ip_bits = first._max_prefixlen + ip_bits = first.max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: @@ -326,12 +326,12 @@ def collapse_addresses(addresses): # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: + elif ip._prefixlen == ip.max_prefixlen: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: @@ -339,7 +339,7 @@ def collapse_addresses(addresses): except AttributeError: ips.append(ip.network_address) else: - if nets and nets[-1]._version != ip._version: + if nets and nets[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) @@ -407,26 +407,21 @@ def reverse_pointer(self): """ return self._reverse_pointer() - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) + raise AddressValueError(msg % (address, self.version)) if address > self._ALL_ONES: msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) + raise AddressValueError(msg % (address, self.max_prefixlen, + self.version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) + expected_len, self.version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): @@ -455,12 +450,12 @@ def _prefix_from_ip_int(cls, ip_int): ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes + cls.max_prefixlen) + prefixlen = cls.max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 + byteslen = cls.max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) @@ -492,7 +487,7 @@ def _prefix_from_prefix_string(cls, prefixlen_str): prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @@ -542,7 +537,7 @@ def _split_addr_prefix(cls, address): """ # a packed address or integer if isinstance(address, (bytes, int)): - return address, cls._max_prefixlen + return address, cls.max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation @@ -552,7 +547,7 @@ def _split_addr_prefix(cls, address): # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address - return address[0], cls._max_prefixlen + return address[0], cls.max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @@ -577,14 +572,14 @@ def __int__(self): def __eq__(self, other): try: return (self._ip == other._ip - and self._version == other._version) + and self.version == other.version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: @@ -613,7 +608,7 @@ def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): - return (self._version, self) + return (self.version, self) def __reduce__(self): return self.__class__, (self._ip,) @@ -649,15 +644,15 @@ def __format__(self, fmt): # Set some defaults if fmt_base == 'n': - if self._version == 4: + if self.version == 4: fmt_base = 'b' # Binary is default for ipv4 else: fmt_base = 'x' # Hex is default for ipv6 if fmt_base == 'b': - padlen = self._max_prefixlen + padlen = self.max_prefixlen else: - padlen = self._max_prefixlen // 4 + padlen = self.max_prefixlen // 4 if grouping: padlen += padlen // 4 - 1 @@ -716,7 +711,7 @@ def __getitem__(self, n): def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: @@ -727,7 +722,7 @@ def __lt__(self, other): def __eq__(self, other): try: - return (self._version == other._version and + return (self.version == other.version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: @@ -738,7 +733,7 @@ def __hash__(self): def __contains__(self, other): # always false if one is v4 and the other is v6. - if self._version != other._version: + if self.version != other.version: return False # dealing with another network. if isinstance(other, _BaseNetwork): @@ -829,7 +824,7 @@ def address_exclude(self, other): ValueError: If other is not completely contained by self. """ - if not self._version == other._version: + if not self.version == other.version: raise TypeError("%s and %s are not of the same version" % ( self, other)) @@ -901,10 +896,10 @@ def compare_networks(self, other): """ # does this need to raise a ValueError? - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same type' % ( self, other)) - # self._version == other._version below here: + # self.version == other.version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: @@ -924,7 +919,7 @@ def _get_networks_key(self): and list.sort(). """ - return (self._version, self.network_address, self.netmask) + return (self.version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. @@ -952,7 +947,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): number means a larger network) """ - if self._prefixlen == self._max_prefixlen: + if self._prefixlen == self.max_prefixlen: yield self return @@ -967,7 +962,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if new_prefixlen > self._max_prefixlen: + if new_prefixlen > self.max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1036,7 +1031,7 @@ def is_multicast(self): def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. - if a._version != b._version: + if a.version != b.version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) @@ -1146,11 +1141,11 @@ class _BaseV4: """ __slots__ = () - _version = 4 + version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2**IPV4LENGTH) - 1 - _max_prefixlen = IPV4LENGTH + max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @@ -1170,7 +1165,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: @@ -1268,15 +1263,6 @@ def _reverse_pointer(self): reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" @@ -1556,10 +1542,10 @@ def __init__(self, address, strict=True): self.network_address = IPv4Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == (self._max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + elif self._prefixlen == (self.max_prefixlen): + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -1628,11 +1614,11 @@ class _BaseV6: """ __slots__ = () - _version = 6 + version = 6 _ALL_ONES = (2**IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH + max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). @@ -1650,7 +1636,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) @@ -1921,15 +1907,6 @@ def _split_scope_id(ip_str): raise AddressValueError('Invalid IPv6 address: "%r"' % ip_str) return addr, scope_id - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" @@ -2356,10 +2333,10 @@ def __init__(self, address, strict=True): self.network_address = IPv6Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == self._max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + elif self._prefixlen == self.max_prefixlen: + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index ed2c74771ea..9eaa4f3fbc1 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -86,13 +86,13 @@ '[2.0, 1.0]' -Using json.tool from the shell to validate and pretty-print:: +Using json from the shell to validate and pretty-print:: - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{ 1.2:3.4}' | python -m json.tool + $ echo '{ 1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' @@ -128,8 +128,9 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can - contain non-ASCII characters if they appear in strings contained in - ``obj``. Otherwise, all such characters are escaped in JSON strings. + contain non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in JSON + strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -145,10 +146,11 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -189,9 +191,10 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is false, then the return value can contain non-ASCII - characters if they appear in strings contained in ``obj``. Otherwise, all - such characters are escaped in JSON strings. + If ``ensure_ascii`` is false, then the return value can contain + non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in + JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -207,10 +210,11 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -281,11 +285,12 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -306,11 +311,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/__main__.py b/Lib/json/__main__.py new file mode 100644 index 00000000000..1808eaddb62 --- /dev/null +++ b/Lib/json/__main__.py @@ -0,0 +1,20 @@ +"""Command-line tool to validate and pretty-print JSON + +Usage:: + + $ echo '{"json":"obj"}' | python -m json + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m json + Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + +""" +import json.tool + + +if __name__ == '__main__': + try: + json.tool.main() + except BrokenPipeError as exc: + raise SystemExit(exc.errno) diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 9e6ca981d76..db87724a897 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -311,10 +311,10 @@ def __init__(self, *, object_hook=None, parse_float=None, place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). - ``object_pairs_hook``, if specified will be called with the result of - every JSON object decoded with an ordered list of pairs. The return - value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. + ``object_pairs_hook``, if specified will be called with the result + of every JSON object decoded with an ordered list of pairs. The + return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 08ef39d1592..5cf6d64f3ea 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -111,9 +111,10 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, encoding of keys that are not str, int, float, bool or None. If skipkeys is True, such items are simply skipped. - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming non-ASCII characters escaped. If - ensure_ascii is false, the output can contain non-ASCII characters. + If ensure_ascii is true, the output is guaranteed to be str objects + with all incoming non-ASCII and non-printable characters escaped. + If ensure_ascii is false, the output can contain non-ASCII and + non-printable characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -134,14 +135,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be an (item_separator, + key_separator) tuple. The default is (', ', ': ') if *indent* is + ``None`` and (',', ': ') otherwise. To get the most compact JSON + representation, you should specify (',', ':') to eliminate + whitespace. If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. + that can't otherwise be serialized. It should return a JSON + encodable version of the object or raise a ``TypeError``. """ @@ -293,37 +295,40 @@ def _iterencode_list(lst, _current_indent_level): else: newline_indent = None separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: + for i, value in enumerate(lst): + if i: buf = separator - if isinstance(value, str): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, int): - # Subclasses of int/float may override __repr__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) - elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __repr__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + _intstr(value) + elif isinstance(value, float): + # see comment above for int + yield buf + _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(lst).__name__} item {i}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -383,28 +388,34 @@ def _iterencode_dict(dct, _current_indent_level): yield item_separator yield _encoder(key) yield _key_separator - if isinstance(value, str): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) - elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield _intstr(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + raise if not first and newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -437,8 +448,14 @@ def _iterencode(o, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) + newobj = _default(o) + try: + yield from _iterencode(newobj, _current_indent_level) + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(o).__name__} object') + raise if markers is not None: del markers[markerid] return _iterencode diff --git a/Lib/json/tool.py b/Lib/json/tool.py index fdfc3372bcc..1967817add8 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -1,25 +1,50 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) +"""Command-line tool to validate and pretty-print JSON +See `json.__main__` for a usage example (invocation as +`python -m json.tool` is supported for backwards compatibility). """ import argparse import json +import re import sys +from _colorize import get_theme, can_colorize + + +# The string we are colorizing is valid JSON, +# so we can use a looser but simpler regex to match +# the various parts, most notably strings and numbers, +# where the regex given by the spec is much more complex. +_color_pattern = re.compile(r''' + (?P"(\\.|[^"\\])*")(?=:) | + (?P"(\\.|[^"\\])*") | + (?PNaN|-?Infinity|[0-9\-+.Ee]+) | + (?Ptrue|false) | + (?Pnull) +''', re.VERBOSE) + +_group_to_theme_color = { + "key": "definition", + "string": "string", + "number": "number", + "boolean": "keyword", + "null": "keyword", +} + + +def _colorize_json(json_str, theme): + def _replace_match_callback(match): + for group, color in _group_to_theme_color.items(): + if m := match.group(group): + return f"{theme[color]}{m}{theme.reset}" + return match.group() + + return re.sub(_color_pattern, _replace_match_callback, json_str) def main(): - prog = 'python -m json.tool' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed', default='-') @@ -75,9 +100,16 @@ def main(): else: outfile = open(options.outfile, 'w', encoding='utf-8') with outfile: - for obj in objs: - json.dump(obj, outfile, **dump_args) - outfile.write('\n') + if can_colorize(file=outfile): + t = get_theme(tty_file=outfile).syntax + for obj in objs: + json_str = json.dumps(obj, **dump_args) + outfile.write(_colorize_json(json_str, t)) + outfile.write('\n') + else: + for obj in objs: + json.dump(obj, outfile, **dump_args) + outfile.write('\n') except ValueError as e: raise SystemExit(e) @@ -86,4 +118,4 @@ def main(): try: main() except BrokenPipeError as exc: - sys.exit(exc.errno) + raise SystemExit(exc.errno) diff --git a/Lib/linecache.py b/Lib/linecache.py index dc02de19eb6..ef3b2d9136b 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -33,10 +33,9 @@ def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" - if filename in cache: - entry = cache[filename] - if len(entry) != 1: - return cache[filename][2] + entry = cache.get(filename, None) + if entry is not None and len(entry) != 1: + return entry[2] try: return updatecache(filename, module_globals) @@ -56,13 +55,22 @@ def _make_key(code): def _getlines_from_code(code): code_id = _make_key(code) - if code_id in _interactive_cache: - entry = _interactive_cache[code_id] - if len(entry) != 1: - return _interactive_cache[code_id][2] + entry = _interactive_cache.get(code_id, None) + if entry is not None and len(entry) != 1: + return entry[2] return [] +def _source_unavailable(filename): + """Return True if the source code is unavailable for such file name.""" + return ( + not filename + or (filename.startswith('<') + and filename.endswith('>') + and not filename.startswith('')): + entry = cache.pop(filename, None) + if _source_unavailable(filename): return [] - fullname = filename + if filename.startswith('')): - return False + return None # Try for a __loader__, if available if module_globals and '__name__' in module_globals: spec = module_globals.get('__spec__') @@ -213,9 +236,10 @@ def lazycache(filename, module_globals): if name and get_source: def get_lines(name=name, *args, **kwargs): return get_source(name, *args, **kwargs) - cache[filename] = (get_lines,) - return True - return False + return (get_lines,) + return None + + def _register_code(code, string, name): entry = (len(string), @@ -228,4 +252,5 @@ def _register_code(code, string, name): for const in code.co_consts: if isinstance(const, type(code)): stack.append(const) - _interactive_cache[_make_key(code)] = entry + key = _make_key(code) + _interactive_cache[key] = entry diff --git a/Lib/lzma.py b/Lib/lzma.py index c1e3d33deb6..316066d024e 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -24,9 +24,9 @@ import builtins import io import os +from compression._common import _streams from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties -import _compression +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 # Value 0 no longer used @@ -35,7 +35,7 @@ _MODE_WRITE = 3 -class LZMAFile(_compression.BaseStream): +class LZMAFile(_streams.BaseStream): """A file object providing transparent LZMA (de)compression. @@ -127,7 +127,7 @@ def __init__(self, filename=None, mode="r", *, raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: - raw = _compression.DecompressReader(self._fp, LZMADecompressor, + raw = _streams.DecompressReader(self._fp, LZMADecompressor, trailing_error=LZMAError, format=format, filters=filters) self._buffer = io.BufferedReader(raw) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 70da07ed2e9..364af6bb010 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() @@ -540,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -698,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: @@ -778,10 +834,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1089,10 +1146,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1146,7 +1217,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1169,7 +1244,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): @@ -1956,10 +2031,7 @@ def readlines(self, sizehint=None): def __iter__(self): """Iterate over lines.""" - while True: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def tell(self): @@ -2111,11 +2183,7 @@ def _unlock_file(f): def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) - try: - return open(path, 'rb+') - finally: - os.close(fd) + return open(path, 'xb+') def _create_temporary(path): """Create a temp file based on path and open for reading and writing.""" diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 954bb0a7453..7d0f4c1fd40 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -23,10 +23,11 @@ read_mime_types(file) -- parse one file, return a dictionary or None """ -import os -import sys -import posixpath -import urllib.parse +try: + from _winapi import _mimetypes_read_windows_registry +except ImportError: + _mimetypes_read_windows_registry = None + try: import winreg as _winreg except ImportError: @@ -34,7 +35,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -88,7 +89,21 @@ def add_type(self, type, ext, strict=True): If strict is true, information will be added to list of standard types, else to the list of non-standard types. + + Valid extensions are empty or start with a '.'. """ + if ext and not ext.startswith('.'): + from warnings import _deprecated + + _deprecated( + "Undotted extensions", + "Using undotted extensions is deprecated and " + "will raise a ValueError in Python {remove}", + remove=(3, 16), + ) + + if not type: + return self.types_map[strict][ext] = type exts = self.types_map_inv[strict].setdefault(type, []) if ext not in exts: @@ -110,11 +125,21 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # Lazy import to improve module import time + import os + import urllib.parse + + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) - scheme, url = urllib.parse._splittype(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -134,26 +159,43 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) - while ext in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext]) + + # Lazy import to improve module import time + import posixpath + + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + # Lazy import to improve module import time + import os + + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) + while (ext_lower := ext.lower()) in self.suffix_map: + base, ext = splitext(base + self.suffix_map[ext_lower]) + # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None + ext = ext.lower() types_map = self.types_map[True] if ext in types_map: return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding elif strict: return None, encoding types_map = self.types_map[False] if ext in types_map: return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding else: return None, encoding @@ -163,13 +205,13 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() - extensions = self.types_map_inv[True].get(type, []) + extensions = list(self.types_map_inv[True].get(type, [])) if not strict: for ext in self.types_map_inv[False].get(type, []): if ext not in extensions: @@ -182,11 +224,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -213,10 +255,7 @@ def readfp(self, fp, strict=True): list of standard types, else to the list of non-standard types. """ - while 1: - line = fp.readline() - if not line: - break + while line := fp.readline(): words = line.split() for i in range(len(words)): if words[i][0] == '#': @@ -237,10 +276,21 @@ def read_windows_registry(self, strict=True): types. """ - # Windows only - if not _winreg: + if not _mimetypes_read_windows_registry and not _winreg: return + add_type = self.add_type + if strict: + add_type = lambda type, ext: self.add_type(type, ext, True) + + # Accelerated function if it is available + if _mimetypes_read_windows_registry: + _mimetypes_read_windows_registry(add_type) + elif _winreg: + self._read_windows_registry(add_type) + + @classmethod + def _read_windows_registry(cls, add_type): def enum_types(mimedb): i = 0 while True: @@ -265,7 +315,7 @@ def enum_types(mimedb): subkey, 'Content Type') if datatype != _winreg.REG_SZ: continue - self.add_type(mimetype, subkeyname, strict) + add_type(mimetype, subkeyname) except OSError: continue @@ -284,7 +334,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -292,17 +342,27 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -315,10 +375,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -349,8 +409,8 @@ def init(files=None): if files is None or _db is None: db = MimeTypes() - if _winreg: - db.read_windows_registry() + # Quick return if not supported + db.read_windows_registry() if files is None: files = knownfiles @@ -359,6 +419,9 @@ def init(files=None): else: db = _db + # Lazy import to improve module import time + import os + for file in files: if os.path.isfile(file): db.read(file) @@ -401,23 +464,28 @@ def _default_mime_types(): '.Z': 'compress', '.bz2': 'bzip2', '.xz': 'xz', + '.br': 'br', } # Before adding new types, make sure they are either registered with IANA, - # at http://www.iana.org/assignments/media-types + # at https://www.iana.org/assignments/media-types/media-types.xhtml # or extensions, i.e. using the x- prefix # If you add to these, please keep them sorted by mime type. # Make sure the entry with the preferred file extension for a particular mime type # appears before any others of the same mimetype. types_map = _types_map_default = { - '.js' : 'application/javascript', - '.mjs' : 'application/javascript', + '.js' : 'text/javascript', + '.mjs' : 'text/javascript', + '.epub' : 'application/epub+zip', + '.gz' : 'application/gzip', '.json' : 'application/json', '.webmanifest': 'application/manifest+json', '.doc' : 'application/msword', '.dot' : 'application/msword', '.wiz' : 'application/msword', + '.nq' : 'application/n-quads', + '.nt' : 'application/n-triples', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', @@ -426,24 +494,37 @@ def _default_mime_types(): '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', '.oda' : 'application/oda', + '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.ai' : 'application/postscript', '.eps' : 'application/postscript', + '.trig' : 'application/trig', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', + '.eot' : 'application/vnd.ms-fontobject', '.ppt' : 'application/vnd.ms-powerpoint', '.pot' : 'application/vnd.ms-powerpoint', '.ppa' : 'application/vnd.ms-powerpoint', '.pps' : 'application/vnd.ms-powerpoint', '.pwz' : 'application/vnd.ms-powerpoint', + '.odg' : 'application/vnd.oasis.opendocument.graphics', + '.odp' : 'application/vnd.oasis.opendocument.presentation', + '.ods' : 'application/vnd.oasis.opendocument.spreadsheet', + '.odt' : 'application/vnd.oasis.opendocument.text', + '.pptx' : 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.xlsx' : 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx' : 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.rar' : 'application/vnd.rar', '.wasm' : 'application/wasm', + '.7z' : 'application/x-7z-compressed', '.bcpio' : 'application/x-bcpio', '.cpio' : 'application/x-cpio', '.csh' : 'application/x-csh', + '.deb' : 'application/x-debian-package', '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', @@ -453,10 +534,12 @@ def _default_mime_types(): '.cdf' : 'application/x-netcdf', '.nc' : 'application/x-netcdf', '.p12' : 'application/x-pkcs12', + '.php' : 'application/x-httpd-php', '.pfx' : 'application/x-pkcs12', '.ram' : 'application/x-pn-realaudio', '.pyc' : 'application/x-python-code', '.pyo' : 'application/x-python-code', + '.rpm' : 'application/x-rpm', '.sh' : 'application/x-sh', '.shar' : 'application/x-shar', '.swf' : 'application/x-shockwave-flash', @@ -479,29 +562,61 @@ def _default_mime_types(): '.rdf' : 'application/xml', '.wsdl' : 'application/xml', '.xpdl' : 'application/xml', + '.yaml' : 'application/yaml', + '.yml' : 'application/yaml', '.zip' : 'application/zip', + '.3gp' : 'audio/3gpp', + '.3gpp' : 'audio/3gpp', + '.3g2' : 'audio/3gpp2', + '.3gpp2' : 'audio/3gpp2', + '.aac' : 'audio/aac', + '.adts' : 'audio/aac', + '.loas' : 'audio/aac', + '.ass' : 'audio/aac', '.au' : 'audio/basic', '.snd' : 'audio/basic', + '.flac' : 'audio/flac', + '.mka' : 'audio/matroska', + '.m4a' : 'audio/mp4', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', + '.ogg' : 'audio/ogg', + '.opus' : 'audio/opus', '.aif' : 'audio/x-aiff', '.aifc' : 'audio/x-aiff', '.aiff' : 'audio/x-aiff', '.ra' : 'audio/x-pn-realaudio', - '.wav' : 'audio/x-wav', + '.wav' : 'audio/vnd.wave', + '.otf' : 'font/otf', + '.ttf' : 'font/ttf', + '.weba' : 'audio/webm', + '.woff' : 'font/woff', + '.woff2' : 'font/woff2', + '.avif' : 'image/avif', '.bmp' : 'image/bmp', + '.emf' : 'image/emf', + '.fits' : 'image/fits', + '.g3' : 'image/g3fax', '.gif' : 'image/gif', '.ief' : 'image/ief', + '.jp2' : 'image/jp2', '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', + '.jpm' : 'image/jpm', + '.jpx' : 'image/jpx', + '.heic' : 'image/heic', + '.heif' : 'image/heif', '.png' : 'image/png', '.svg' : 'image/svg+xml', + '.t38' : 'image/t38', '.tiff' : 'image/tiff', '.tif' : 'image/tiff', + '.tfx' : 'image/tiff-fx', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', + '.wmf' : 'image/wmf', '.ras' : 'image/x-cmu-raster', - '.bmp' : 'image/x-ms-bmp', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', '.pgm' : 'image/x-portable-graymap', @@ -514,34 +629,49 @@ def _default_mime_types(): '.mht' : 'message/rfc822', '.mhtml' : 'message/rfc822', '.nws' : 'message/rfc822', + '.gltf' : 'model/gltf+json', + '.glb' : 'model/gltf-binary', + '.stl' : 'model/stl', '.css' : 'text/css', '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', + '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', '.c' : 'text/plain', '.h' : 'text/plain', '.ksh' : 'text/plain', '.pl' : 'text/plain', + '.srt' : 'text/plain', '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', + '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', '.vcf' : 'text/x-vcard', '.xml' : 'text/xml', + '.mkv' : 'video/matroska', + '.mk3d' : 'video/matroska-3d', '.mp4' : 'video/mp4', '.mpeg' : 'video/mpeg', '.m1v' : 'video/mpeg', '.mpa' : 'video/mpeg', '.mpe' : 'video/mpeg', '.mpg' : 'video/mpeg', + '.ogv' : 'video/ogg', '.mov' : 'video/quicktime', '.qt' : 'video/quicktime', '.webm' : 'video/webm', - '.avi' : 'video/x-msvideo', + '.avi' : 'video/vnd.avi', + '.m4v' : 'video/x-m4v', + '.wmv' : 'video/x-ms-wmv', '.movie' : 'video/x-sgi-movie', } @@ -551,6 +681,7 @@ def _default_mime_types(): # Please sort these too common_types = _common_types_default = { '.rtf' : 'application/rtf', + '.apk' : 'application/vnd.android.package-archive', '.midi': 'audio/midi', '.mid' : 'audio/midi', '.jpg' : 'image/jpg', @@ -564,51 +695,53 @@ def _default_mime_types(): _default_mime_types() -def _main(): - import getopt - - USAGE = """\ -Usage: mimetypes.py [options] type - -Options: - --help / -h -- print this message and exit - --lenient / -l -- additionally search of some common, but non-standard - types. - --extension / -e -- guess extension instead of type - -More than one type argument may be given. -""" - - def usage(code, msg=''): - print(USAGE) - if msg: print(msg) - sys.exit(code) - - try: - opts, args = getopt.getopt(sys.argv[1:], 'hle', - ['help', 'lenient', 'extension']) - except getopt.error as msg: - usage(1, msg) - - strict = 1 - extension = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-l', '--lenient'): - strict = 0 - elif opt in ('-e', '--extension'): - extension = 1 - for gtype in args: - if extension: - guess = guess_extension(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print(guess) - else: - guess, encoding = guess_type(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print('type:', guess, 'encoding:', encoding) +def _parse_args(args): + from argparse import ArgumentParser + + parser = ArgumentParser( + description='map filename extensions to MIME types', color=True + ) + parser.add_argument( + '-e', '--extension', + action='store_true', + help='guess extension instead of type' + ) + parser.add_argument( + '-l', '--lenient', + action='store_true', + help='additionally search for common but non-standard types' + ) + parser.add_argument('type', nargs='+', help='a type to search') + args = parser.parse_args(args) + return args, parser.format_help() + + +def _main(args=None): + """Run the mimetypes command-line interface and return a text to print.""" + args, help_text = _parse_args(args) + + results = [] + if args.extension: + for gtype in args.type: + guess = guess_extension(gtype, not args.lenient) + if guess: + results.append(str(guess)) + else: + results.append(f"error: unknown type {gtype}") + return results + else: + for gtype in args.type: + guess, encoding = guess_type(gtype, not args.lenient) + if guess: + results.append(f"type: {guess} encoding: {encoding}") + else: + results.append(f"error: media type unknown for {gtype}") + return results if __name__ == '__main__': - _main() + import sys + + results = _main() + print("\n".join(results)) + sys.exit(any(result.startswith("error: ") for result in results)) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 8caddd204d7..abd88adf76e 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -74,7 +74,7 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 05633ac21a2..22e3bbcf21b 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,11 +15,15 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys import threading import warnings +from collections import deque + +import json from . import spawn from . import util @@ -66,6 +70,14 @@ def __init__(self): self._fd = None self._pid = None self._exitcode = None + self._reentrant_messages = deque() + + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = True def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -102,7 +114,7 @@ def _stop_locked( # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. if self._lock._recursion_count() > 1: - return self._reentrant_call_error() + raise self._reentrant_call_error() if self._fd is None: # not running return @@ -113,7 +125,12 @@ def _stop_locked( close(self._fd) self._fd = None - _, status = waitpid(self._pid, 0) + try: + _, status = waitpid(self._pid, 0) + except ChildProcessError: + self._pid = None + self._exitcode = None + return self._pid = None @@ -132,76 +149,119 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' + return self._ensure_running_and_write() + + def _teardown_dead_process(self): + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + self._exitcode = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + def _launch(self): + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [ + exe, + *util._args_from_interpreter_flags(), + '-c', + f'from multiprocessing.resource_tracker import main;main({r})', + ] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + prev_sigmask = None + try: + if _HAVE_SIGMASK: + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _make_probe_message(self): + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: # The code below is certainly not reentrant-safe, so bail out - return self._reentrant_call_error() + if msg is None: + raise self._reentrant_call_error() + return self._reentrant_messages.append(msg) + if self._fd is not None: # resource tracker was launched before, is it still running? - if self._check_alive(): - # => still alive - return - # => dead, launch it again - os.close(self._fd) - - # Clean-up to avoid dangling processes. + if msg is None: + to_send = self._make_probe_message() + else: + to_send = msg try: - # _pid can be None if this process is a child from another - # python process, which has started the resource_tracker. - if self._pid is not None: - os.waitpid(self._pid, 0) - except ChildProcessError: - # The resource_tracker has already been terminated. - pass - self._fd = None - self._pid = None - self._exitcode = None + self._write(to_send) + except OSError: + self._teardown_dead_process() + self._launch() - warnings.warn('resource_tracker: process died unexpectedly, ' - 'relaunching. Some resources might leak.') + msg = None # message was sent in probe + else: + self._launch() - fds_to_pass = [] - try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.resource_tracker import main;main(%d)' - r, w = os.pipe() + while True: try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - # bpo-33613: Register a signal mask that will block the signals. - # This signal mask will be inherited by the child that is going - # to be spawned and will protect the child from a race condition - # that can make the child die before it registers signal handlers - # for SIGINT and SIGTERM. The mask is unregistered after spawning - # the child. - prev_sigmask = None - try: - if _HAVE_SIGMASK: - prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = util.spawnv_passfds(exe, args, fds_to_pass) - finally: - if prev_sigmask is not None: - signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) - except: - os.close(w) - raise - else: - self._fd = w - self._pid = pid - finally: - os.close(r) + reentrant_msg = self._reentrant_messages.popleft() + except IndexError: + break + self._write(reentrant_msg) + if msg is not None: + self._write(msg) def _check_alive(self): '''Check that the pipe has not been closed by sending a probe.''' try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -215,27 +275,42 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype): - try: - self.ensure_running() - except ReentrantCallError: - # The code below might or might not work, depending on whether - # the resource tracker was already running and still alive. - # Better warn the user. - # (XXX is warnings.warn itself reentrant-safe? :-) - warnings.warn( - f"ResourceTracker called reentrantly for resource cleanup, " - f"which is unsupported. " - f"The {rtype} object {name!r} might leak.") - msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + def _write(self, msg): nbytes = os.write(self._fd, msg) - assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( - nbytes, len(msg)) + assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" + def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') + + self._ensure_running_and_write(msg) _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running @@ -244,6 +319,30 @@ def _send(self, cmd, name, rtype): getfd = _resource_tracker.getfd +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -266,7 +365,7 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 75dde02d88c..4c8425064fe 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -34,6 +34,7 @@ DEBUG = 10 INFO = 20 SUBWARNING = 25 +WARNING = 30 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' @@ -53,6 +54,10 @@ def info(msg, *args): if _logger: _logger.log(INFO, msg, *args, stacklevel=2) +def _warn(msg, *args): + if _logger: + _logger.log(WARNING, msg, *args, stacklevel=2) + def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args, stacklevel=2) @@ -121,6 +126,23 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. +# +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html + +if sys.platform == 'linux': + _SUN_PATH_MAX = 108 +elif sys.platform.startswith(('openbsd', 'freebsd')): + _SUN_PATH_MAX = 104 +else: + # On Windows platforms, we do not create AF_UNIX sockets. + _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -130,12 +152,69 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None +def _get_base_temp_dir(tempfile): + """Get a temporary directory where socket files will be created. + + To prevent additional imports, pass a pre-imported 'tempfile' module. + """ + if os.name == 'nt': + return None + # Most of the time, the default temporary directory is /tmp. Thus, + # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do + # not have a path length exceeding SUN_PATH_MAX. + # + # If users specify their own temporary directory, we may be unable + # to create those files. Therefore, we fall back to the system-wide + # temporary directory /tmp, assumed to exist on POSIX systems. + # + # See https://github.com/python/cpython/issues/132124. + base_tempdir = tempfile.gettempdir() + # Files created in a temporary directory are suffixed by a string + # generated by tempfile._RandomNameSequence, which, by design, + # is 8 characters long. + # + # Thus, the socket file path length (without NULL terminator) will be: + # + # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') + sun_path_len = len(base_tempdir) + 14 + 14 + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: + return base_tempdir + # Fallback to the default system-wide temporary directory. + # This ignores user-defined environment variables. + # + # On POSIX systems, /tmp MUST be writable by any application [1]. + # We however emit a warning if this is not the case to prevent + # obscure errors later in the execution. + # + # On some legacy systems, /var/tmp and /usr/tmp can be present + # and will be used instead. + # + # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html + dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] + try: + base_system_tempdir = tempfile._get_default_tempdir(dirlist) + except FileNotFoundError: + _warn("Process-wide temporary directory %s will not be usable for " + "creating socket files and no usable system-wide temporary " + "directory was found in %s", base_tempdir, dirlist) + # At this point, the system-wide temporary directory is not usable + # but we may assume that the user-defined one is, even if we will + # not be able to write socket files out there. + return base_tempdir + _warn("Ignoring user-defined temporary directory: %s", base_tempdir) + # at most max(map(len, dirlist)) + 14 + 14 = 36 characters + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX + return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile - tempdir = tempfile.mkdtemp(prefix='pymp-') + base_tempdir = _get_base_temp_dir(tempfile) + tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer # can be called late during Python shutdown @@ -438,15 +517,13 @@ def _flush_std_streams(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - import subprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, -1, None, None, None, -1, None, - subprocess._USE_VFORK) + False, False, -1, None, None, None, -1, None) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 9cdc16480f9..01f060e70be 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -400,17 +400,23 @@ def expanduser(path): # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. +_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" +_varsub = None +_varsubb = None + def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path - import string - varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') - quote = b'\'' + if not _varsubb: + import re + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb percent = b'%' brace = b'{' rbrace = b'}' @@ -419,94 +425,44 @@ def expandvars(path): else: if '$' not in path and '%' not in path: return path - import string - varchars = string.ascii_letters + string.digits + '_-' - quote = '\'' + if not _varsub: + import re + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ - res = path[:0] - index = 0 - pathlen = len(path) - while index < pathlen: - c = path[index:index+1] - if c == quote: # no expansion within single quotes - path = path[index + 1:] - pathlen = len(path) - try: - index = path.index(c) - res += c + path[:index + 1] - except ValueError: - res += c + path - index = pathlen - 1 - elif c == percent: # variable or '%' - if path[index + 1:index + 2] == percent: - res += c - index += 1 - else: - path = path[index+1:] - pathlen = len(path) - try: - index = path.index(percent) - except ValueError: - res += percent + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent - res += value - elif c == dollar: # variable or '$$' - if path[index + 1:index + 2] == dollar: - res += c - index += 1 - elif path[index + 1:index + 2] == brace: - path = path[index+2:] - pathlen = len(path) - try: - index = path.index(rbrace) - except ValueError: - res += dollar + brace + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value - else: - var = path[:0] - index += 1 - c = path[index:index + 1] - while c and c in varchars: - var += c - index += 1 - c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var - res += value - if c: - index -= 1 + + def repl(m): + lastindex = m.lastindex + if lastindex is None: + return m[0] + name = m[lastindex] + if lastindex == 1: + if name == percent: + return name + if not name.endswith(percent): + return m[0] + name = name[:-1] else: - res += c - index += 1 - return res + if name == dollar: + return name + if name.startswith(brace): + if not name.endswith(rbrace): + return m[0] + name = name[1:-1] + + try: + if environ is None: + return os.fsencode(os.environ[os.fsdecode(name)]) + else: + return environ[name] + except KeyError: + return m[0] + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff589..57c7858dff0 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -3,7 +3,15 @@ This module only exists to provide OS-specific code for urllib.requests, thus do not use directly. """ -# Testing is done through test_urllib. +# Testing is done through test_nturl2path. + +import warnings + + +warnings._deprecated( + __name__, + message=f"{warnings._DEPRECATED_MSG}; use 'urllib.request' instead", + remove=(3, 19)) def url2pathname(url): """OS-specific conversion from a relative URL of the 'file' scheme @@ -14,33 +22,25 @@ def url2pathname(url): # ///C:/foo/bar/spam.foo # become # C:\foo\bar\spam.foo - import string, urllib.parse - # Windows itself uses ":" even in URLs. - url = url.replace(':', '|') - if not '|' in url: - # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) - url = url[2:] - components = url.split('/') - # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) - comp = url.split('|') - if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: - error = 'Bad URL: ' + url - raise OSError(error) - drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path + import urllib.parse + if url[:3] == '///': + # URL has an empty authority section, so the path begins on the third + # character. + url = url[2:] + elif url[:12] == '//localhost/': + # Skip past 'localhost' authority. + url = url[11:] + if url[:3] == '///': + # Skip past extra slash before UNC drive in URL path. + url = url[1:] + else: + if url[:1] == '/' and url[2:3] in (':', '|'): + # Skip past extra slash before DOS drive in URL path. + url = url[1:] + if url[1:2] == '|': + # Older URLs use a pipe after a drive letter + url = url[:1] + ':' + url[2:] + return urllib.parse.unquote(url.replace('/', '\\')) def pathname2url(p): """OS-specific conversion from a file system path to a relative URL @@ -49,33 +49,26 @@ def pathname2url(p): # C:\foo\bar\spam.foo # becomes # ///C:/foo/bar/spam.foo + import ntpath import urllib.parse # First, clean up some special forms. We are going to sacrifice # the additional information anyway - if p[:4] == '\\\\?\\': + p = p.replace('\\', '/') + if p[:4] == '//?/': p = p[4:] - if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] - elif p[1:2] != ':': - raise OSError('Bad path: ' + p) - if not ':' in p: - # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) - comp = p.split(':', maxsplit=2) - if len(comp) != 2 or len(comp[0]) > 1: - error = 'Bad path: ' + p - raise OSError(error) + if p[:4].upper() == 'UNC/': + p = '//' + p[4:] + drive, root, tail = ntpath.splitroot(p) + if drive: + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = f'///{drive}' + drive = urllib.parse.quote(drive, safe='/:') + elif root: + # Add explicitly empty authority to path beginning with one slash. + root = f'//{root}' - drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path + tail = urllib.parse.quote(tail) + return drive + root + tail diff --git a/Lib/opcode.py b/Lib/opcode.py index 5735686fa7f..0e9520b6832 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -9,16 +9,18 @@ "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] +import builtins import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, - HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] -for op, i in opmap.items(): - opname[i] = op +for m in (opmap, _specialized_opmap): + for op, i in m.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') @@ -36,6 +38,9 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_special_method_names = _opcode.get_special_method_names() +_common_constants = [builtins.AssertionError, builtins.NotImplementedError, + builtins.tuple, builtins.all, builtins.any] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] @@ -49,6 +54,7 @@ }, "BINARY_OP": { "counter": 1, + "descr": 4, }, "UNPACK_SEQUENCE": { "counter": 1, @@ -59,9 +65,6 @@ "CONTAINS_OP": { "counter": 1, }, - "BINARY_SUBSCR": { - "counter": 1, - }, "FOR_ITER": { "counter": 1, }, @@ -83,6 +86,10 @@ "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Lib/operator.py b/Lib/operator.py index 02ccdaa13dd..1b765522f85 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -14,8 +14,8 @@ 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'is_', 'is_none', 'is_not', 'is_not_none', 'isub', 'itemgetter', 'itruediv', + 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor'] @@ -66,6 +66,14 @@ def is_not(a, b): "Same as a is not b." return a is not b +def is_none(a): + "Same as a is None." + return a is None + +def is_not_none(a): + "Same as a is not None." + return a is not None + # Mathematical/Bitwise Operations *********************************************# def abs(a): @@ -415,7 +423,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe..38cf16d21ef 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -74,7 +74,8 @@ """ import sys, os -import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +87,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): @@ -263,6 +251,7 @@ def _format_text(self, text): Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ + import textwrap text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, @@ -319,6 +308,7 @@ def format_option(self, option): indent_first = 0 result.append(opts) if option.help: + import textwrap help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) diff --git a/Lib/os.py b/Lib/os.py index b4c9f84c36d..ac03b416390 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -10,7 +10,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') + - os.linesep is the line separator in text files ('\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -366,61 +374,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - if followlinks is _walk_symlinks_as_files: - is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() - else: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs @@ -774,7 +766,7 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -804,9 +796,24 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping + + +if _exists("_create_environ"): + def reload_environ(): + data = _create_environ() + if name == 'nt': + encodekey = environ.encodekey + data = {encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + env_data = environ._data + env_data.clear() + env_data.update(data) + __all__.append("reload_environ") def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. diff --git a/Lib/pathlib.py b/Lib/pathlib.py deleted file mode 100644 index bd5a096f9e3..00000000000 --- a/Lib/pathlib.py +++ /dev/null @@ -1,1435 +0,0 @@ -"""Object-oriented filesystem paths. - -This module provides classes to represent abstract paths and concrete -paths with operations that have semantics appropriate for different -operating systems. -""" - -import fnmatch -import functools -import io -import ntpath -import os -import posixpath -import re -import sys -import warnings -from _collections_abc import Sequence -from errno import ENOENT, ENOTDIR, EBADF, ELOOP -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from urllib.parse import quote_from_bytes as urlquote_from_bytes - - -__all__ = [ - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - -# -# Internals -# - -# Reference for Windows paths can be found at -# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . -_WIN_RESERVED_NAMES = frozenset( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | - {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} -) - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - - -@functools.cache -def _is_case_sensitive(flavour): - return flavour.normcase('Aa') == 'Aa' - -# -# Globbing helpers -# - - -# fnmatch.translate() returns a regular expression that includes a prefix and -# a suffix, which enable matching newlines and ensure the end of the string is -# matched, respectively. These features are undesirable for our implementation -# of PurePatch.match(), which represents path separators as newlines and joins -# pattern segments together. As a workaround, we define a slice object that -# can remove the prefix and suffix from any translate() result. See the -# _compile_pattern_lines() function for more details. -_FNMATCH_PREFIX, _FNMATCH_SUFFIX = fnmatch.translate('_').split('_') -_FNMATCH_SLICE = slice(len(_FNMATCH_PREFIX), -len(_FNMATCH_SUFFIX)) -_SWAP_SEP_AND_NEWLINE = { - '/': str.maketrans({'/': '\n', '\n': '/'}), - '\\': str.maketrans({'\\': '\n', '\n': '\\'}), -} - - -@functools.lru_cache() -def _make_selector(pattern_parts, flavour, case_sensitive): - pat = pattern_parts[0] - if not pat: - return _TerminatingSelector() - if pat == '**': - child_parts_idx = 1 - while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': - child_parts_idx += 1 - child_parts = pattern_parts[child_parts_idx:] - if '**' in child_parts: - cls = _DoubleRecursiveWildcardSelector - else: - cls = _RecursiveWildcardSelector - else: - child_parts = pattern_parts[1:] - if pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - else: - cls = _WildcardSelector - return cls(pat, child_parts, flavour, case_sensitive) - - -@functools.lru_cache(maxsize=256) -def _compile_pattern(pat, case_sensitive): - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - return re.compile(fnmatch.translate(pat), flags).match - - -@functools.lru_cache() -def _compile_pattern_lines(pattern_lines, case_sensitive): - """Compile the given pattern lines to an `re.Pattern` object. - - The *pattern_lines* argument is a glob-style pattern (e.g. '*/*.py') with - its path separators and newlines swapped (e.g. '*\n*.py`). By using - newlines to separate path components, and not setting `re.DOTALL`, we - ensure that the `*` wildcard cannot match path separators. - - The returned `re.Pattern` object may have its `match()` method called to - match a complete pattern, or `search()` to match from the right. The - argument supplied to these methods must also have its path separators and - newlines swapped. - """ - - # Match the start of the path, or just after a path separator - parts = ['^'] - for part in pattern_lines.splitlines(keepends=True): - if part == '*\n': - part = r'.+\n' - elif part == '*': - part = r'.+' - else: - # Any other component: pass to fnmatch.translate(). We slice off - # the common prefix and suffix added by translate() to ensure that - # re.DOTALL is not set, and the end of the string not matched, - # respectively. With DOTALL not set, '*' wildcards will not match - # path separators, because the '.' characters in the pattern will - # not match newlines. - part = fnmatch.translate(part)[_FNMATCH_SLICE] - parts.append(part) - # Match the end of the path, always. - parts.append(r'\Z') - flags = re.MULTILINE - if not case_sensitive: - flags |= re.IGNORECASE - return re.compile(''.join(parts), flags=flags) - - -class _Selector: - """A selector matches a specific glob pattern part against the children - of a given path.""" - - def __init__(self, child_parts, flavour, case_sensitive): - self.child_parts = child_parts - if child_parts: - self.successor = _make_selector(child_parts, flavour, case_sensitive) - self.dironly = True - else: - self.successor = _TerminatingSelector() - self.dironly = False - - def select_from(self, parent_path): - """Iterate over all child paths of `parent_path` matched by this - selector. This can contain parent_path itself.""" - path_cls = type(parent_path) - scandir = path_cls._scandir - if not parent_path.is_dir(): - return iter([]) - return self._select_from(parent_path, scandir) - - -class _TerminatingSelector: - - def _select_from(self, parent_path, scandir): - yield parent_path - - -class _ParentSelector(_Selector): - - def __init__(self, name, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - - def _select_from(self, parent_path, scandir): - path = parent_path._make_child_relpath('..') - for p in self.successor._select_from(path, scandir): - yield p - - -class _WildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - if case_sensitive is None: - # TODO: evaluate case-sensitivity of each directory in _select_from() - case_sensitive = _is_case_sensitive(flavour) - self.match = _compile_pattern(pat, case_sensitive) - - def _select_from(self, parent_path, scandir): - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with scandir(parent_path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if self.dironly: - try: - if not entry.is_dir(): - continue - except OSError: - continue - name = entry.name - if self.match(name): - path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, scandir): - yield p - - -class _RecursiveWildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - - def _iterate_directories(self, parent_path): - yield parent_path - for dirpath, dirnames, _ in parent_path.walk(): - for dirname in dirnames: - yield dirpath._make_child_relpath(dirname) - - def _select_from(self, parent_path, scandir): - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path): - for p in successor_select(starting_point, scandir): - yield p - - -class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): - """ - Like _RecursiveWildcardSelector, but also de-duplicates results from - successive selectors. This is necessary if the pattern contains - multiple non-adjacent '**' segments. - """ - - def _select_from(self, parent_path, scandir): - yielded = set() - try: - for p in super()._select_from(parent_path, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() - - -# -# Public API -# - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -class PurePath(object): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_lines_cached` slot stores the string path with path separators - # and newlines swapped. This is used to implement `match()`. - '_lines_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - _flavour = os.path - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg._flavour is ntpath and self._flavour is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - self._raw_paths = paths - - def with_segments(self, *pathsegments): - """Construct a new path object from any number of path-like objects. - Subclasses may override this method to customize how new path objects - are created from methods like `iterdir()`. - """ - return type(self)(*pathsegments) - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls._flavour.sep - altsep = cls._flavour.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls._flavour.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - def _load_parts(self): - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self._flavour.join(*paths) - drv, root, tail = self._parse_path(path) - self._drv = drv - self._root = root - self._tail_cached = tail - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls._flavour.sep.join(tail) - elif tail and cls._flavour.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls._flavour.sep.join(tail) - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - def __fspath__(self): - return str(self) - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - f = self._flavour - return str(self).replace(f.sep, '/') - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def as_uri(self): - """Return the path as a 'file' URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - return prefix + urlquote_from_bytes(os.fsencode(path)) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _is_case_sensitive(self._flavour): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self._flavour.sep) - return self._parts_normcase_cached - - @property - def _lines(self): - # Path with separators and newlines swapped, for pattern matching. - try: - return self._lines_cached - except AttributeError: - path_str = str(self) - if path_str == '.': - self._lines_cached = '' - else: - trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] - self._lines_cached = path_str.translate(trans) - return self._lines_cached - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self._flavour is other._flavour - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __lt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._load_parts() - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._load_parts() - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._load_parts() - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - anchor = self.drive + self.root - return anchor - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - @property - def suffix(self): - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' - - @property - def suffixes(self): - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] - - @property - def stem(self): - """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name - - def with_name(self, name): - """Return a new path with the file name changed.""" - if not self.name: - raise ValueError("%r has an empty name" % (self,)) - f = self._flavour - if not name or f.sep in name or (f.altsep and f.altsep in name) or name == '.': - raise ValueError("Invalid name %r" % (name)) - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def with_stem(self, stem): - """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) - - def with_suffix(self, suffix): - """Return a new path with the file suffix changed. If the path - has no suffix, add given suffix. If the given suffix is an empty - string, remove the suffix from the path. - """ - f = self._flavour - if f.sep in suffix or f.altsep and f.altsep in suffix: - raise ValueError("Invalid suffix %r" % (suffix,)) - if suffix and not suffix.startswith('.') or suffix == '.': - raise ValueError("Invalid suffix %r" % (suffix)) - name = self.name - if not name: - raise ValueError("%r has an empty name" % (self,)) - old_suffix = self.suffix - if not old_suffix: - name = name + suffix - else: - name = name[:-len(old_suffix)] + suffix - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, - remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - for step, path in enumerate([other] + list(other.parents)): - if self.is_relative_to(path): - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self.with_segments(*parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", - msg, remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - return other == self or other in self.parents - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.joinpath(key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self._flavour is ntpath: - # ntpath.isabs() is defective - see GH-44626. - return bool(self.drive and self.root) - elif self._flavour is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - else: - return self._flavour.isabs(str(self)) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - if self._flavour is posixpath or not self._tail: - return False - - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if self.drive.startswith('\\\\'): - # UNC paths are never reserved. - return False - name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in _WIN_RESERVED_NAMES - - def match(self, path_pattern, *, case_sensitive=None): - """ - Return True if this path matches the given pattern. - """ - if not isinstance(path_pattern, PurePath): - path_pattern = self.with_segments(path_pattern) - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self._flavour) - pattern = _compile_pattern_lines(path_pattern._lines, case_sensitive) - if path_pattern.drive or path_pattern.root: - return pattern.match(self._lines) is not None - elif path_pattern._tail: - return pattern.search(self._lines) is not None - else: - raise ValueError("empty pattern") - - -# Can't subclass os.PathLike from PurePath and keep the constructor -# optimizations in PurePath.__slots__. -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = ntpath - __slots__ = () - - -# Filesystem-accessing classes - - -class Path(PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return self._flavour.samestat(st, other_st) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - encoding = io.text_encoding(encoding) - with self.open(mode='r', encoding=encoding, errors=errors) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - encoding = io.text_encoding(encoding) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - for name in os.listdir(self): - yield self._make_child_relpath(name) - - def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) - - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def glob(self, pattern, *, case_sensitive=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - - def rglob(self, pattern, *, case_sensitive=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). - """ - return cls("~").expanduser() - - def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. - - Use resolve() to get the canonical path to a file. - """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) - else: - cwd = os.getcwd() - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - if not self.root and not self._tail: - result = self.with_segments(cwd) - result._str = cwd - return result - return self.with_segments(cwd, self) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - - try: - s = self._flavour.realpath(self, strict=strict) - except OSError as e: - check_eloop(e) - raise - p = self.with_segments(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p - - def owner(self): - """ - Return the login name of the file owner. - """ - try: - import pwd - return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") - - def group(self): - """ - Return the group name of the file gid. - """ - - try: - import grp - return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") - - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - if not hasattr(os, "readlink"): - raise NotImplementedError("os.readlink() not available on this system") - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - if not hasattr(os, "symlink"): - raise NotImplementedError("os.symlink() not available on this system") - os.symlink(target, self, target_is_directory) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - if not hasattr(os, "link"): - raise NotImplementedError("os.link() not available on this system") - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = self._flavour.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise NotImplementedError( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise NotImplementedError( - f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000..0d763d1f0dc --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,1307 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +import io +import ntpath +import operator +import os +import posixpath +import sys +from errno import * +from glob import _StringGlobber, _no_recurse_symlinks +from itertools import chain +from stat import S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from pathlib._os import ( + PathInfo, DirEntryInfo, + ensure_different_files, ensure_distinct_paths, + copyfile2, copyfileobj, magic_open, copy_info, +) + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath: + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + self._raw_paths = paths + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + return drv, root, [x for x in rel.split(sep) if x and x != '.'] + + @classmethod + def _parse_pattern(cls, pattern): + """Parse a glob pattern to a list of parts. This is much like + _parse_path, except: + + - Rather than normalizing and returning the drive and root, we raise + NotImplementedError if either are present. + - If the path has no real parts, we raise ValueError. + - If the path ends in a slash, then a final empty part is added. + """ + drv, root, rel = cls.parser.splitroot(pattern) + if root or drv: + raise NotImplementedError("Non-relative patterns are unsupported") + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + rel = rel.replace(altsep, sep) + parts = [x for x in rel.split(sep) if x and x != '.'] + if not parts: + raise ValueError(f"Unacceptable pattern: {str(pattern)!r}") + elif rel.endswith(sep): + # GH-65238: preserve trailing slash in glob patterns. + parts.append('') + return parts + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + return self.parser.join(*paths) + else: + return '' + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings._deprecated("pathlib.PurePath.is_reserved", msg, remove=(3, 15)) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + import warnings + msg = ("pathlib.PurePath.as_uri() is deprecated and scheduled " + "for removal in Python 3.19. Use pathlib.Path.as_uri().") + warnings._deprecated("pathlib.PurePath.as_uri", msg, remove=(3, 19)) + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not hasattr(pattern, 'with_segments'): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not hasattr(path_pattern, 'with_segments'): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = _StringGlobber(self.parser.sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = ('_info',) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + @property + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + try: + return self._info + except AttributeError: + self._info = PathInfo(self) + return self._info + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def _from_dir_entry(self, dir_entry, path_str): + path = self.with_segments(path_str) + path._str = path_str + path._info = DirEntryInfo(dir_entry) + return path + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + entries = list(scandir_it) + if root_dir == '.': + return (self._from_dir_entry(e, e.name) for e in entries) + else: + return (self._from_dir_entry(e, e.path) for e in entries) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + parts = self._parse_pattern(pattern) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) + root = str(self) + paths = select(self.parser.join(root, '')) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + pattern = self.parser.join('**', pattern) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.copy(target, **kwargs) + + def _copy_from(self, source, follow_symlinks=True, preserve_metadata=False): + """ + Recursively copy the given path to this path. + """ + if not follow_symlinks and source.info.is_symlink(): + self._copy_from_symlink(source, preserve_metadata) + elif source.info.is_dir(): + children = source.iterdir() + os.mkdir(self) + for child in children: + self.joinpath(child.name)._copy_from( + child, follow_symlinks, preserve_metadata) + if preserve_metadata: + copy_info(source.info, self) + else: + self._copy_from_file(source, preserve_metadata) + + def _copy_from_file(self, source, preserve_metadata=False): + ensure_different_files(source, self) + with magic_open(source, 'rb') as source_f: + with open(self, 'wb') as target_f: + copyfileobj(source_f, target_f) + if preserve_metadata: + copy_info(source.info, self) + + if copyfile2: + # Use fast OS routine for local file copying where available. + _copy_from_file_fallback = _copy_from_file + def _copy_from_file(self, source, preserve_metadata=False): + try: + source = os.fspath(source) + except TypeError: + pass + else: + copyfile2(source, str(self)) + return + self._copy_from_file_fallback(source, preserve_metadata) + + if os.name == 'nt': + # If a directory-symlink is copied *before* its target, then + # os.symlink() incorrectly creates a file-symlink on Windows. Avoid + # this by passing *target_is_dir* to os.symlink() on Windows. + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self, source.info.is_dir()) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + else: + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + # Use os.replace() if the target is os.PathLike and on the same FS. + try: + target = self.with_segments(target) + except TypeError: + pass + else: + ensure_different_files(self, target) + try: + os.replace(self, target) + except OSError as err: + if err.errno != EXDEV: + raise + else: + return target.joinpath() # Empty join to ensure fresh metadata. + # Fall back to copy+delete. + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative paths can't be expressed as file URIs") + from urllib.request import pathname2url + return pathname2url(str(self), add_scheme=True) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + from urllib.error import URLError + from urllib.request import url2pathname + try: + path = cls(url2pathname(uri, require_scheme=True)) + except URLError as exc: + raise ValueError(exc.reason) from None + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000..58e137f2a92 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,12 @@ +""" +This module exists so that pathlib objects pickled under Python 3.13 can be +unpickled in 3.14+. +""" + +from pathlib import * + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", +] diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000..039836941dd --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,530 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +from io import TextIOWrapper, text_encoding +from stat import S_ISDIR, S_ISREG, S_ISLNK, S_IMODE +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def _get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def _ficlone(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + _ficlone = None + + +if posix and hasattr(posix, '_fcopyfile'): + def _fcopyfile(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +else: + _fcopyfile = None + + +if hasattr(os, 'copy_file_range'): + def _copy_file_range(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +else: + _copy_file_range = None + + +if hasattr(os, 'sendfile'): + def _sendfile(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + _sendfile = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile2(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile2 = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if _ficlone: + try: + _ficlone(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if _fcopyfile: + try: + _fcopyfile(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EINVAL, ENOTSUP): + raise err + if _copy_file_range: + try: + _copy_file_range(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (ETXTBSY, EXDEV): + raise err + if _sendfile: + try: + _sendfile(source_fd, target_fd) + return + except OSError as err: + if err.errno != ENOTSOCK: + raise err + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) + + +def magic_open(path, mode='r', buffering=-1, encoding=None, errors=None, + newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + text = 'b' not in mode + if text: + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + try: + return open(path, mode, buffering, encoding, errors, newline) + except TypeError: + pass + cls = type(path) + mode = ''.join(sorted(c for c in mode if c not in 'bt')) + if text: + try: + attr = getattr(cls, f'__open_{mode}__') + except AttributeError: + pass + else: + return attr(path, buffering, encoding, errors, newline) + elif encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + elif errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + elif newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + + try: + attr = getattr(cls, f'__open_{mode}b__') + except AttributeError: + pass + else: + stream = attr(path, buffering) + if text: + stream = TextIOWrapper(stream, encoding, errors, newline) + return stream + + raise TypeError(f"{cls.__name__} can't be opened with mode {mode!r}") + + +def ensure_distinct_paths(source, target): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == target: + err = OSError(EINVAL, "Source and target are the same path") + elif source in target.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def ensure_different_files(source, target): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + source_file_id = source.info._file_id + target_file_id = target.info._file_id + except AttributeError: + if source != target: + return + else: + try: + if source_file_id() != target_file_id(): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def copy_info(info, target, follow_symlinks=True): + """Copy metadata from the given PathInfo to the given local path.""" + copy_times_ns = ( + hasattr(info, '_access_time_ns') and + hasattr(info, '_mod_time_ns') and + (follow_symlinks or os.utime in os.supports_follow_symlinks)) + if copy_times_ns: + t0 = info._access_time_ns(follow_symlinks=follow_symlinks) + t1 = info._mod_time_ns(follow_symlinks=follow_symlinks) + os.utime(target, ns=(t0, t1), follow_symlinks=follow_symlinks) + + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + copy_xattrs = ( + hasattr(info, '_xattrs') and + hasattr(os, 'setxattr') and + (follow_symlinks or os.setxattr in os.supports_follow_symlinks)) + if copy_xattrs: + xattrs = info._xattrs(follow_symlinks=follow_symlinks) + for attr, value in xattrs: + try: + os.setxattr(target, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + + copy_posix_permissions = ( + hasattr(info, '_posix_permissions') and + (follow_symlinks or os.chmod in os.supports_follow_symlinks)) + if copy_posix_permissions: + posix_permissions = info._posix_permissions(follow_symlinks=follow_symlinks) + try: + os.chmod(target, posix_permissions, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + + copy_bsd_flags = ( + hasattr(info, '_bsd_flags') and + hasattr(os, 'chflags') and + (follow_symlinks or os.chflags in os.supports_follow_symlinks)) + if copy_bsd_flags: + bsd_flags = info._bsd_flags(follow_symlinks=follow_symlinks) + try: + os.chflags(target, bsd_flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + +class _PathInfoBase: + __slots__ = ('_path', '_stat_result', '_lstat_result') + + def __init__(self, path): + self._path = str(path) + + def __repr__(self): + path_type = "WindowsPath" if os.name == "nt" else "PosixPath" + return f"<{path_type}.info>" + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + """Return the status as an os.stat_result, or None if stat() fails and + ignore_errors is true.""" + if follow_symlinks: + try: + result = self._stat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._stat_result = os.stat(self._path) + except (OSError, ValueError): + self._stat_result = None + if not ignore_errors: + raise + return self._stat_result + else: + try: + result = self._lstat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._lstat_result = os.lstat(self._path) + except (OSError, ValueError): + self._lstat_result = None + if not ignore_errors: + raise + return self._lstat_result + + def _posix_permissions(self, *, follow_symlinks=True): + """Return the POSIX file permissions.""" + return S_IMODE(self._stat(follow_symlinks=follow_symlinks).st_mode) + + def _file_id(self, *, follow_symlinks=True): + """Returns the identifier of the file.""" + st = self._stat(follow_symlinks=follow_symlinks) + return st.st_dev, st.st_ino + + def _access_time_ns(self, *, follow_symlinks=True): + """Return the access time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_atime_ns + + def _mod_time_ns(self, *, follow_symlinks=True): + """Return the modify time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_mtime_ns + + if hasattr(os.stat_result, 'st_flags'): + def _bsd_flags(self, *, follow_symlinks=True): + """Return the flags.""" + return self._stat(follow_symlinks=follow_symlinks).st_flags + + if hasattr(os, 'listxattr'): + def _xattrs(self, *, follow_symlinks=True): + """Return the xattrs as a list of (attr, value) pairs, or an empty + list if extended attributes aren't supported.""" + try: + return [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return [] + + +class _WindowsPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for Windows paths. Don't try to construct it yourself.""" + __slots__ = ('_exists', '_is_dir', '_is_file', '_is_symlink') + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks and self.is_symlink(): + return True + try: + return self._exists + except AttributeError: + if os.path.exists(self._path): + self._exists = True + return True + else: + self._exists = self._is_dir = self._is_file = False + return False + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_dir + except AttributeError: + if os.path.isdir(self._path): + self._is_dir = self._exists = True + return True + else: + self._is_dir = False + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_file + except AttributeError: + if os.path.isfile(self._path): + self._is_file = self._exists = True + return True + else: + self._is_file = False + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._is_symlink + except AttributeError: + self._is_symlink = os.path.islink(self._path) + return self._is_symlink + + +class _PosixPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for POSIX paths. Don't try to construct it yourself.""" + __slots__ = () + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISDIR(st.st_mode) + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISREG(st.st_mode) + + def is_symlink(self): + """Whether this path is a symbolic link.""" + st = self._stat(follow_symlinks=False, ignore_errors=True) + if st is None: + return False + return S_ISLNK(st.st_mode) + + +PathInfo = _WindowsPathInfo if os.name == 'nt' else _PosixPathInfo + + +class DirEntryInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information by querying a wrapped os.DirEntry object. Don't try to + construct it yourself.""" + __slots__ = ('_entry',) + + def __init__(self, entry): + super().__init__(entry.path) + self._entry = entry + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + try: + return self._entry.stat(follow_symlinks=follow_symlinks) + except OSError: + if not ignore_errors: + raise + return None + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks: + return True + return self._stat(ignore_errors=True) is not None + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + try: + return self._entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + try: + return self._entry.is_file(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._entry.is_symlink() + except OSError: + return False diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py new file mode 100644 index 00000000000..d8f5c34a1a7 --- /dev/null +++ b/Lib/pathlib/types.py @@ -0,0 +1,430 @@ +""" +Protocols for supporting classes in pathlib. +""" + +# This module also provides abstract base classes for rich path objects. +# These ABCs are a *private* part of the Python standard library, but they're +# made available as a PyPI package called "pathlib-abc". It's possible they'll +# become an official part of the standard library in future. +# +# Three ABCs are provided -- _JoinablePath, _ReadablePath and _WritablePath + + +from abc import ABC, abstractmethod +from glob import _PathGlobber +from io import text_encoding +from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj +from pathlib import PurePath, Path +from typing import Optional, Protocol, runtime_checkable + + +def _explode_path(path, split): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + +@runtime_checkable +class _PathParser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide JoinablePath functionality. Each JoinablePath + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + altsep: Optional[str] + def split(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... + + +@runtime_checkable +class PathInfo(Protocol): + """Protocol for path info objects, which support querying the file type. + Methods may return cached results. + """ + def exists(self, *, follow_symlinks: bool = True) -> bool: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def is_symlink(self) -> bool: ... + + +class _JoinablePath(ABC): + """Abstract base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its implementation PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. + """ + __slots__ = () + + @property + @abstractmethod + def parser(self): + """Implementation of pathlib._types.Parser used for low-level path + parsing and manipulation. + """ + raise NotImplementedError + + @abstractmethod + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + raise NotImplementedError + + @abstractmethod + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + raise NotImplementedError + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return _explode_path(str(self), self.parser.split)[0] + + @property + def name(self): + """The final path component, if any.""" + return self.parser.split(str(self))[1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + return self.parser.splitext(self.name)[1] + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + return self.parser.splitext(self.name)[0] + + def with_name(self, name): + """Return a new path with the file name changed.""" + split = self.parser.split + if split(name)[0]: + raise ValueError(f"Invalid name {name!r}") + path = str(self) + path = path.removesuffix(split(path)[1]) + name + return self.with_segments(path) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + anchor, parts = _explode_path(str(self), self.parser.split) + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(str(self), *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(str(self), key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, str(self)) + except TypeError: + return NotImplemented + + @property + def parent(self): + """The logical parent of the path.""" + path = str(self) + parent = self.parser.split(path)[0] + if path != parent: + return self.with_segments(parent) + return self + + @property + def parents(self): + """A sequence of this path's logical parents.""" + split = self.parser.split + path = str(self) + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) + + def full_match(self, pattern): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern, altsep=self.parser.altsep) + return match(str(self)) is not None + + +class _ReadablePath(_JoinablePath): + """Abstract base class for readable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement readable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @property + @abstractmethod + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + raise NotImplementedError + + @abstractmethod + def __open_rb__(self, buffering=-1): + """ + Open the file pointed to by this path for reading in binary mode and + return a file object, like open(mode='rb'). + """ + raise NotImplementedError + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with magic_open(self, mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + with magic_open(self, mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + @abstractmethod + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise NotImplementedError + + def glob(self, pattern, *, recurse_symlinks=True): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + anchor, parts = _explode_path(pattern, self.parser.split) + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + elif not parts: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + elif not recurse_symlinks: + raise NotImplementedError("recurse_symlinks=False is unsupported") + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + select = globber.selector(parts) + return select(self.joinpath('')) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + if child.info.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] + + @abstractmethod + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + raise NotImplementedError + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + return self.copy(target_dir / name, **kwargs) + + +class _WritablePath(_JoinablePath): + """Abstract base class for writable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement writable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @abstractmethod + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + raise NotImplementedError + + @abstractmethod + def mkdir(self): + """ + Create a new directory at this given path. + """ + raise NotImplementedError + + @abstractmethod + def __open_wb__(self, buffering=-1): + """ + Open the file pointed to by this path for writing in binary mode and + return a file object, like open(mode='wb'). + """ + raise NotImplementedError + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with magic_open(self, mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def _copy_from(self, source, follow_symlinks=True): + """ + Recursively copy the given path to this path. + """ + stack = [(source, self)] + while stack: + src, dst = stack.pop() + if not follow_symlinks and src.info.is_symlink(): + dst.symlink_to(str(src.readlink()), src.info.is_dir()) + elif src.info.is_dir(): + children = src.iterdir() + dst.mkdir() + for child in children: + stack.append((child, dst.joinpath(child.name))) + else: + ensure_different_files(src, dst) + with magic_open(src, 'rb') as source_f: + with magic_open(dst, 'wb') as target_f: + copyfileobj(source_f, target_f) + + +_JoinablePath.register(PurePath) +_ReadablePath.register(Path) +_WritablePath.register(Path) diff --git a/Lib/pdb.py b/Lib/pdb.py index bf503f1e73e..ec6cf06e58b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -185,6 +185,15 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + def set_trace(self, frame=None, *, commands=None): + if frame is None: + frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt diff --git a/Lib/pickle.py b/Lib/pickle.py index 550f8675f2c..beaefae0479 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,12 +26,11 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize from struct import pack, unpack -import re import io import codecs import _compat_pickle @@ -51,7 +50,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +67,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -188,7 +187,7 @@ def __init__(self, value): NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) +__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) class _Framer: @@ -313,38 +312,46 @@ def load_frame(self, frame_size): # Tools used for pickling. -def _getattribute(obj, name): - top = obj - for subpath in name.split('.'): - if subpath == '': - raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, top)) - try: - parent = obj - obj = getattr(obj, subpath) - except AttributeError: - raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, top)) from None - return obj, parent +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + obj = getattr(obj, subpath) + return obj def whichmodule(obj, name): """Find the module an object belong to.""" + dotted_path = name.split('.') module_name = getattr(obj, '__module__', None) - if module_name is not None: - return module_name - # Protect the iteration by using a list copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in sys.modules.copy().items(): - if (module_name == '__main__' - or module_name == '__mp_main__' # bpo-42406 - or module is None): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except AttributeError: - pass - return '__main__' + if '' in dotted_path: + raise PicklingError(f"Can't pickle local object {obj!r}") + if module_name is None: + # Protect the iteration by using a list copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in sys.modules.copy().items(): + if (module_name == '__main__' + or module_name == '__mp_main__' # bpo-42406 + or module is None): + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + + try: + __import__(module_name, level=0) + module = sys.modules[module_name] + except (ImportError, ValueError, KeyError) as exc: + raise PicklingError(f"Can't pickle {obj!r}: {exc!s}") + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError(f"Can't pickle {obj!r}: " + f"it's not found as {module_name}.{name}") + + raise PicklingError( + f"Can't pickle {obj!r}: it's not the same object as {module_name}.{name}") def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. @@ -396,6 +403,13 @@ def decode_long(data): """ return int.from_bytes(data, byteorder='little', signed=True) +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module in (None, 'builtins', '__main__'): + return cls.__qualname__ + return f'{module}.{cls.__qualname__}' + _NoValue = object() @@ -409,7 +423,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -579,26 +593,29 @@ def save(self, obj, save_persistent_id=True): if reduce is not _NoValue: rv = reduce() else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + raise PicklingError(f"Can't pickle {_T(t)} object") # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return - # Assert that reduce() returned a tuple - if not isinstance(rv, tuple): - raise PicklingError("%s must return string or tuple" % reduce) - - # Assert that it returned an appropriately sized tuple - l = len(rv) - if not (2 <= l <= 6): - raise PicklingError("Tuple returned by %s must have " - "two to six elements" % reduce) - - # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + try: + # Assert that reduce() returned a tuple + if not isinstance(rv, tuple): + raise PicklingError(f'__reduce__ must return a string or tuple, not {_T(rv)}') + + # Assert that it returned an appropriately sized tuple + l = len(rv) + if not (2 <= l <= 6): + raise PicklingError("tuple returned by __reduce__ " + "must contain 2 through 6 elements") + + # Save the reduce() output and finally memoize the object + self.save_reduce(obj=obj, *rv) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} object') + raise def persistent_id(self, obj): # This exists so a subclass can override it @@ -620,10 +637,12 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, *, obj=None): # This API is called by some subclasses - if not isinstance(args, tuple): - raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): - raise PicklingError("func from save_reduce() must be callable") + raise PicklingError(f"first item of the tuple returned by __reduce__ " + f"must be callable, not {_T(func)}") + if not isinstance(args, tuple): + raise PicklingError(f"second item of the tuple returned by __reduce__ " + f"must be a tuple, not {_T(args)}") save = self.save write = self.write @@ -632,19 +651,30 @@ def save_reduce(self, func, args, state=None, listitems=None, if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): - raise PicklingError("args[0] from {} args has no __new__" - .format(func_name)) + raise PicklingError("first argument to __newobj_ex__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError("args[0] from {} args has the wrong class" - .format(func_name)) + raise PicklingError(f"first argument to __newobj_ex__() " + f"must be {obj.__class__!r}, not {cls!r}") if self.proto >= 4: - save(cls) - save(args) - save(kwargs) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + save(kwargs) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, *args, **kwargs) - save(func) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": @@ -676,18 +706,33 @@ def save_reduce(self, func, args, state=None, listitems=None, # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): - raise PicklingError( - "args[0] from __newobj__ args has no __new__") + raise PicklingError("first argument to __newobj__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError( - "args[0] from __newobj__ args has the wrong class") + raise PicklingError(f"first argument to __newobj__() " + f"must be {obj.__class__!r}, not {cls!r}") args = args[1:] - save(cls) - save(args) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ) else: - save(func) - save(args) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor arguments') + raise write(REDUCE) if obj is not None: @@ -705,23 +750,35 @@ def save_reduce(self, func, args, state=None, listitems=None, # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + self._batch_appends(listitems, obj) if dictitems is not None: - self._batch_setitems(dictitems) + self._batch_setitems(dictitems, obj) if state is not None: if state_setter is None: - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(BUILD) else: # If a state_setter is specified, call it instead of load_build # to update obj's with its previous state. # First, push state_setter and its tuple of expected arguments # (obj, state) onto the stack. - save(state_setter) + try: + save(state_setter) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state setter') + raise save(obj) # simple BINGET opcode as obj is already memoized. - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(TUPLE2) # Trigger a state_setter(obj, state) function call. write(REDUCE) @@ -901,8 +958,12 @@ def save_tuple(self, obj): save = self.save memo = self.memo if n <= 3 and self.proto >= 2: - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -916,8 +977,12 @@ def save_tuple(self, obj): # has more than 3 elements. write = self.write write(MARK) - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -947,38 +1012,47 @@ def save_list(self, obj): self.write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + self._batch_appends(obj, obj) dispatch[list] = save_list _BATCHSIZE = 1000 - def _batch_appends(self, items): + def _batch_appends(self, items, obj): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: - for x in items: - save(x) + for i, x in enumerate(items): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPEND) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + start = 0 + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for x in tmp: - save(x) + for i, x in enumerate(batch, start): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPENDS) - elif n: - save(tmp[0]) + else: + try: + save(batch[0]) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {start}') + raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return + start += batch_len def save_dict(self, obj): if self.bin: @@ -987,11 +1061,11 @@ def save_dict(self, obj): self.write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + self._batch_setitems(obj.items(), obj) dispatch[dict] = save_dict - def _batch_setitems(self, items): + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write @@ -999,28 +1073,34 @@ def _batch_setitems(self, items): if not self.bin: for k, v in items: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1033,17 +1113,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: for item in batch: save(item) - write(ADDITEMS) - if n < self._BATCHSIZE: - return + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): @@ -1055,8 +1133,12 @@ def save_frozenset(self, obj): return write(MARK) - for item in obj: - save(item) + try: + for item in obj: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise if id(obj) in self.memo: # If the object is already in the memo, this means it is @@ -1075,24 +1157,10 @@ def save_global(self, obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) - if name is None: - name = obj.__name__ + if name is None: + name = obj.__name__ module_name = whichmodule(obj, name) - try: - __import__(module_name, level=0) - module = sys.modules[module_name] - obj2, parent = _getattribute(module, name) - except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module_name, name)) from None - else: - if obj2 is not obj: - raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % - (obj, module_name, name)) - if self.proto >= 2: code = _extension_registry.get((module_name, name), _NoValue) if code is not _NoValue: @@ -1109,10 +1177,7 @@ def save_global(self, obj, name=None): else: write(EXT4 + pack("= 3. + if self.proto >= 4: self.save(module_name) self.save(name) @@ -1144,8 +1209,7 @@ def save_global(self, obj, name=None): def _save_toplevel_by_name(self, module_name, name): if self.proto >= 3: # Non-ASCII identifiers are supported only with protocols >= 3. - self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + encoding = "utf-8" else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1154,13 +1218,19 @@ def _save_toplevel_by_name(self, module_name, name): module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] - try: - self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') - except UnicodeEncodeError: - raise PicklingError( - "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module_name, name, self.proto)) from None + encoding = "ascii" + try: + self.write(GLOBAL + bytes(module_name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle module identifier {module_name!r} using " + f"pickle protocol {self.proto}") + try: + self.write(bytes(name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle global identifier {name!r} using " + f"pickle protocol {self.proto}") def save_type(self, obj): if obj is type(None): @@ -1316,7 +1386,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1336,7 +1406,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): @@ -1620,8 +1690,13 @@ def find_class(self, module, name): elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - if self.proto >= 4: - return _getattribute(sys.modules[module], name)[0] + if self.proto >= 4 and '.' in name: + dotted_path = name.split('.') + try: + return _getattribute(sys.modules[module], dotted_path) + except AttributeError: + raise AttributeError( + f"Can't resolve path {name!r} on module {module!r}") else: return getattr(sys.modules[module], name) @@ -1831,36 +1906,26 @@ def _loads(s, /, *, fix_imports=True, encoding="ASCII", errors="strict", Pickler, Unpickler = _Pickler, _Unpickler dump, dumps, load, loads = _dump, _dumps, _load, _loads -# Doctest -def _test(): - import doctest - return doctest.testmod() -if __name__ == "__main__": +def _main(args=None): import argparse + import pprint parser = argparse.ArgumentParser( - description='display contents of the pickle files') + description='display contents of the pickle files', + color=True, + ) parser.add_argument( 'pickle_file', - nargs='*', help='the pickle file') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') - parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') - args = parser.parse_args() - if args.test: - _test() - else: - if not args.pickle_file: - parser.print_help() + nargs='+', help='the pickle file') + args = parser.parse_args(args) + for fn in args.pickle_file: + if fn == '-': + obj = load(sys.stdin.buffer) else: - import pprint - for fn in args.pickle_file: - if fn == '-': - obj = load(sys.stdin.buffer) - else: - with open(fn, 'rb') as f: - obj = load(f) - pprint.pprint(obj) + with open(fn, 'rb') as f: + obj = load(f) + pprint.pprint(obj) + + +if __name__ == "__main__": + _main() diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 33a51492ea9..254b6c7fcc9 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -348,7 +348,7 @@ def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): - raise ValueError("strinq quote %r not found at both " + raise ValueError("string quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" @@ -2842,17 +2838,16 @@ def __init__(self, value): 'disassembler_memo_test': _memo_test, } -def _test(): - import doctest - return doctest.testmod() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( - description='disassemble one or more pickle files') + description='disassemble one or more pickle files', + color=True, + ) parser.add_argument( 'pickle_file', - nargs='*', help='the pickle file') + nargs='+', help='the pickle file') parser.add_argument( '-o', '--output', help='the file where the output should be written') @@ -2869,36 +2864,24 @@ def _test(): '-p', '--preamble', default="==> {name} <==", help='if more than one pickle file is specified, print this before' ' each disassembly') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') - parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') args = parser.parse_args() - if args.test: - _test() + annotate = 30 if args.annotate else 0 + memo = {} if args.memo else None + if args.output is None: + output = sys.stdout else: - if not args.pickle_file: - parser.print_help() - else: - annotate = 30 if args.annotate else 0 - memo = {} if args.memo else None - if args.output is None: - output = sys.stdout + output = open(args.output, 'w') + try: + for arg in args.pickle_file: + if len(args.pickle_file) > 1: + name = '' if arg == '-' else arg + preamble = args.preamble.format(name=name) + output.write(preamble + '\n') + if arg == '-': + dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) else: - output = open(args.output, 'w') - try: - for arg in args.pickle_file: - if len(args.pickle_file) > 1: - name = '' if arg == '-' else arg - preamble = args.preamble.format(name=name) - output.write(preamble + '\n') - if arg == '-': - dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) - else: - with open(arg, 'rb') as f: - dis(f, output, memo, args.indentlevel, annotate) - finally: - if output is not sys.stdout: - output.close() + with open(arg, 'rb') as f: + dis(f, output, memo, args.indentlevel, annotate) + finally: + if output is not sys.stdout: + output.close() diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index a4c474006ba..dccbec52aa7 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -14,7 +14,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', + 'read_code', 'extend_path', 'ModuleInfo', ] @@ -23,20 +23,6 @@ ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' -def _get_spec(finder, name): - """Return the finder-specific module spec.""" - # Works with legacy finders. - try: - find_spec = finder.find_spec - except AttributeError: - loader = finder.find_module(name) - if loader is None: - return None - return importlib.util.spec_from_loader(name, loader) - else: - return find_spec(name) - - def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files @@ -184,6 +170,7 @@ def _iter_file_finder_modules(importer, prefix=''): iter_importer_modules.register( importlib.machinery.FileFinder, _iter_file_finder_modules) + try: import zipimport from zipimport import zipimporter @@ -231,6 +218,7 @@ def get_importer(path_item): The cache (or part of it) can be cleared manually if a rescan of sys.path_hooks is necessary. """ + path_item = os.fsdecode(path_item) try: importer = sys.path_importer_cache[path_item] except KeyError: @@ -282,6 +270,10 @@ def get_loader(module_or_name): If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. """ + warnings._deprecated("pkgutil.get_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] if module_or_name is None: @@ -306,6 +298,10 @@ def find_loader(fullname): importlib.util.find_spec that converts most failures to ImportError and only returns the loader rather than the full spec """ + warnings._deprecated("pkgutil.find_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) @@ -328,10 +324,10 @@ def extend_path(path, name): from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - This will add to the package's __path__ all subdirectories of - directories on sys.path named after the package. This is useful - if one wants to distribute different parts of a single logical - package as multiple directories. + For each directory on sys.path that has a subdirectory that + matches the package name, add the subdirectory to the package's + __path__. This is useful if one wants to distribute different + parts of a single logical package as multiple directories. It also looks for *.pkg files beginning where * matches the name argument. This feature is similar to *.pth files (see site.py), diff --git a/Lib/platform.py b/Lib/platform.py index 58b66078e1a..1a533688a94 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -10,7 +10,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -118,6 +119,10 @@ import sys import functools import itertools +try: + import _wmi +except ImportError: + _wmi = None ### Globals & Constants @@ -136,11 +141,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +157,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +190,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +206,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +253,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +298,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -310,44 +317,62 @@ def _syscmd_ver(system='', release='', version='', version = _norm_version(version) return system, release, version -_WIN32_CLIENT_RELEASES = { - (5, 0): "2000", - (5, 1): "XP", - # Strictly, 5.2 client is XP 64-bit, but platform.py historically - # has always called it 2003 Server - (5, 2): "2003Server", - (5, None): "post2003", - - (6, 0): "Vista", - (6, 1): "7", - (6, 2): "8", - (6, 3): "8.1", - (6, None): "post8.1", - - (10, 0): "10", - (10, None): "post10", -} - -# Server release name lookup will default to client names if necessary -_WIN32_SERVER_RELEASES = { - (5, 2): "2003Server", - (6, 0): "2008Server", - (6, 1): "2008ServerR2", - (6, 2): "2012Server", - (6, 3): "2012ServerR2", - (6, None): "post2012ServerR2", -} +def _wmi_query(table, *keys): + global _wmi + if not _wmi: + raise OSError("not supported") + table = { + "OS": "Win32_OperatingSystem", + "CPU": "Win32_Processor", + }[table] + try: + data = _wmi.exec_query("SELECT {} FROM {}".format( + ",".join(keys), + table, + )).split("\0") + except OSError: + _wmi = None + raise OSError("not supported") + split_data = (i.partition("=") for i in data) + dict_data = {i[0]: i[2] for i in split_data} + return (dict_data[k] for k in keys) + + +_WIN32_CLIENT_RELEASES = [ + ((10, 1, 0), "post11"), + ((10, 0, 22000), "11"), + ((6, 4, 0), "10"), + ((6, 3, 0), "8.1"), + ((6, 2, 0), "8"), + ((6, 1, 0), "7"), + ((6, 0, 0), "Vista"), + ((5, 2, 3790), "XP64"), + ((5, 2, 0), "XPMedia"), + ((5, 1, 0), "XP"), + ((5, 0, 0), "2000"), +] + +_WIN32_SERVER_RELEASES = [ + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), + ((10, 0, 20348), "2022Server"), + ((10, 0, 17763), "2019Server"), + ((6, 4, 0), "2016Server"), + ((6, 3, 0), "2012ServerR2"), + ((6, 2, 0), "2012Server"), + ((6, 1, 0), "2008ServerR2"), + ((6, 0, 0), "2008Server"), + ((5, 2, 0), "2003Server"), + ((5, 0, 0), "2000Server"), +] def win32_is_iot(): return win32_edition() in ('IoTUAP', 'NanoServer', 'WindowsCoreHeadless', 'IoTEdgeOS') def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -360,22 +385,40 @@ def win32_edition(): return None -def win32_ver(release='', version='', csd='', ptype=''): +def _win32_ver(version, csd, ptype): + # Try using WMI first, as this is the canonical source of data + try: + (version, product_type, ptype, spmajor, spminor) = _wmi_query( + 'OS', + 'Version', + 'ProductType', + 'BuildType', + 'ServicePackMajorVersion', + 'ServicePackMinorVersion', + ) + is_client = (int(product_type) == 1) + if spminor and spminor != '0': + csd = f'SP{spmajor}.{spminor}' + else: + csd = f'SP{spmajor}' + return version, csd, ptype, is_client + except OSError: + pass + + # Fall back to a combination of sys.getwindowsversion and "ver" try: from sys import getwindowsversion except ImportError: - return release, version, csd, ptype + return version, csd, ptype, True winver = getwindowsversion() + is_client = (getattr(winver, 'product_type', 1) == 1) try: - major, minor, build = map(int, _syscmd_ver()[2].split('.')) + version = _syscmd_ver()[2] + major, minor, build = map(int, version.split('.')) except ValueError: major, minor, build = winver.platform_version or winver[:3] - version = '{0}.{1}.{2}'.format(major, minor, build) - - release = (_WIN32_CLIENT_RELEASES.get((major, minor)) or - _WIN32_CLIENT_RELEASES.get((major, None)) or - release) + version = '{0}.{1}.{2}'.format(major, minor, build) # getwindowsversion() reflect the compatibility mode Python is # running under, and so the service pack value is only going to be @@ -387,17 +430,8 @@ def win32_ver(release='', version='', csd='', ptype=''): if csd[:13] == 'Service Pack ': csd = 'SP' + csd[13:] - # VER_NT_SERVER = 3 - if getattr(winver, 'product_type', None) == 3: - release = (_WIN32_SERVER_RELEASES.get((major, minor)) or - _WIN32_SERVER_RELEASES.get((major, None)) or - release) - try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -408,6 +442,18 @@ def win32_ver(release='', version='', csd='', ptype=''): except OSError: pass + return version, csd, ptype, is_client + +def win32_ver(release='', version='', csd='', ptype=''): + is_client = False + + version, csd, ptype, is_client = _win32_ver(version, csd, ptype) + + if version: + intversion = tuple(map(int, version.split('.'))) + releases = _WIN32_CLIENT_RELEASES if is_client else _WIN32_SERVER_RELEASES + release = next((r for v, r in releases if v <= intversion), release) + return release, version, csd, ptype @@ -452,8 +498,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -475,6 +545,8 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: import java.lang @@ -496,6 +568,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -562,12 +675,12 @@ def _platform(*args): platform = platform.replace('unknown', '') # Fold '--'s and remove trailing '-' - while 1: + while True: cleaned = platform.replace('--', '-') if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -608,7 +721,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -702,6 +815,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: @@ -726,6 +841,21 @@ def _get_machine_win32(): # http://www.geocities.com/rick_lively/MANUALS/ENV/MSWIN/PROCESSI.HTM # WOW64 processes mask the native architecture + try: + [arch, *_] = _wmi_query('CPU', 'Architecture') + except OSError: + pass + else: + try: + arch = ['x86', 'MIPS', 'Alpha', 'PowerPC', None, + 'ARM', 'ia64', None, None, + 'AMD64', None, None, 'ARM64', + ][int(arch)] + except (ValueError, IndexError): + pass + else: + if arch: + return arch return ( os.environ.get('PROCESSOR_ARCHITEW6432', '') or os.environ.get('PROCESSOR_ARCHITECTURE', '') @@ -739,7 +869,12 @@ def get(cls): return func() or '' def get_win32(): - return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + try: + manufacturer, caption = _wmi_query('CPU', 'Manufacturer', 'Caption') + except OSError: + return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + else: + return f'{caption}, {manufacturer}' def get_OpenVMS(): try: @@ -750,6 +885,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -904,6 +1047,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -971,32 +1123,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_ironpython_sys_version_parser = re.compile( - r'IronPython\s*' - r'([\d\.]+)' - r'(?: \(([\d\.]+)\))?' - r' on (.NET [\d\.]+)', re.ASCII) - -# IronPython covering 2.6 and 2.7 -_ironpython26_sys_version_parser = re.compile( - r'([\d.]+)\s*' - r'\(IronPython\s*' - r'[\d.]+\s*' - r'\(([\d.]+)\) on ([\w.]+ [\d.]+(?: \(\d+-bit\))?)\)' -) - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1028,28 +1154,16 @@ def _sys_version(sys_version=None): if result is not None: return result - # Parse it - if 'IronPython' in sys_version: - # IronPython - name = 'IronPython' - if sys_version.startswith('IronPython'): - match = _ironpython_sys_version_parser.match(sys_version) - else: - match = _ironpython26_sys_version_parser.match(sys_version) - - if match is None: - raise ValueError( - 'failed to parse IronPython sys.version: %s' % - repr(sys_version)) - - version, alt_version, compiler = match.groups() - buildno = '' - builddate = '' - - elif sys.platform.startswith('java'): + if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1061,8 +1175,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1071,7 +1190,14 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1080,11 +1206,10 @@ def _sys_version(sys_version=None): match.groups() # XXX: RUSTPYTHON support - if "rustc" in sys_version: + if "RustPython" in sys_version: name = "RustPython" else: name = 'CPython' - if builddate is None: builddate = '' elif buildtime: @@ -1115,7 +1240,6 @@ def python_implementation(): Currently, the following implementations are identified: 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), 'Jython' (Java implementation of Python), 'PyPy' (Python implementation of Python). @@ -1190,7 +1314,7 @@ def python_compiler(): _platform_cache = {} -def platform(aliased=0, terse=0): +def platform(aliased=False, terse=False): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). @@ -1222,11 +1346,14 @@ def platform(aliased=0, terse=0): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms @@ -1236,7 +1363,7 @@ def platform(aliased=0, terse=0): else: platform = _platform(system, release, version, csd) - elif system in ('Linux',): + elif system == 'Linux': # check for libc vs. glibc libcname, libcversion = libc_ver() platform = _platform(system, release, machine, processor, @@ -1267,13 +1394,6 @@ def platform(aliased=0, terse=0): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1288,10 +1408,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2eeebe4c9a4..655c51eea3d 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,6 +21,9 @@ Generate Plist example: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -28,22 +31,28 @@ anInt = 728, aDict = dict( anotherString = "", - aUnicodeValue = "M\xe4ssig, Ma\xdf", + aThirdString = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parse Plist example: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b''' + + foo + bar + + ''' + pl = plistlib.loads(plist) + print(pl["foo"]) """ __all__ = [ "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID" @@ -64,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -131,7 +143,7 @@ def _decode_base64(s): _dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -140,10 +152,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -152,7 +168,7 @@ def _date_to_string(d): def _escape(text): m = _controlCharPat.search(text) if m is not None: - raise ValueError("strings can't contains control characters; " + raise ValueError("strings can't contain control characters; " "use bytes instead") text = text.replace("\r\n", "\n") # convert DOS line endings text = text.replace("\r", "\n") # convert Mac line endings @@ -162,11 +178,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -178,8 +195,8 @@ def parse(self, fileobj): return self.root def handle_entity_decl(self, entity_name, is_parameter_entity, value, base, system_id, public_id, notation_name): - # Reject plist files with entity declarations to avoid XML vulnerabilies in expat. - # Regular plist files don't contain those declerations, and Apple's plutil tool does not + # Reject plist files with entity declarations to avoid XML vulnerabilities in expat. + # Regular plist files don't contain those declarations, and Apple's plutil tool does not # accept them either. raise InvalidFileException("XML entity declarations are not supported in plist files") @@ -199,7 +216,7 @@ def handle_data(self, data): def add_object(self, value): if self.current_key is not None: - if not isinstance(self.stack[-1], type({})): + if not isinstance(self.stack[-1], dict): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1][self.current_key] = value @@ -208,7 +225,7 @@ def add_object(self, value): # this is the root object self.root = value else: - if not isinstance(self.stack[-1], type([])): + if not isinstance(self.stack[-1], list): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1].append(value) @@ -232,7 +249,7 @@ def end_dict(self): self.stack.pop() def end_key(self): - if self.current_key or not isinstance(self.stack[-1], type({})): + if self.current_key or not isinstance(self.stack[-1], dict): raise ValueError("unexpected key at line %d" % self.parser.CurrentLineNumber) self.current_key = self.get_data() @@ -268,7 +285,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -312,13 +330,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -351,7 +370,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -452,8 +472,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -490,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -547,27 +580,24 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID @@ -579,7 +609,8 @@ def _read_object(self, ref): obj_refs = self._read_refs(s) result = [] self._objects[ref] = result - result.extend(self._read_object(x) for x in obj_refs) + for x in obj_refs: + result.append(self._read_object(x)) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -620,10 +651,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -769,7 +801,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -853,7 +890,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -871,32 +908,41 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ + if isinstance(value, str): + if fmt == FMT_BINARY: + raise TypeError("value must be bytes-like object when fmt is " + "FMT_BINARY") + value = value.encode() fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 80561ae7e52..ad86cc06c01 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -284,42 +284,41 @@ def expanduser(path): # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -327,13 +326,11 @@ def expandvars(path): else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. @@ -410,6 +407,8 @@ def realpath(filename, *, strict=False): else: ignored_error = OSError + lstat = os.lstat + readlink = os.readlink maxlinks = None # The stack of unresolved path parts. When popped, a special value of None @@ -432,6 +431,10 @@ def realpath(filename, *, strict=False): # the same links. seen = {} + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 + while part_count: name = rest.pop() if name is None: @@ -451,14 +454,22 @@ def realpath(filename, *, strict=False): else: newpath = path + sep + name try: - st_mode = os.lstat(newpath).st_mode + st_mode = lstat(newpath).st_mode if not stat.S_ISLNK(st_mode): if strict and part_count and not stat.S_ISDIR(st_mode): raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), newpath) path = newpath continue - if newpath in seen: + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: # Already seen this path path = seen[newpath] if path is not None: @@ -466,11 +477,11 @@ def realpath(filename, *, strict=False): continue # The symlink is not resolved, so we must have a symlink loop. if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) path = newpath continue - target = os.readlink(newpath) + target = readlink(newpath) except ignored_error: pass else: diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db34..dc0953cec67 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -179,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -197,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -291,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -632,9 +647,11 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) diff --git a/Lib/pty.py b/Lib/pty.py index 8d8ce40df54..1d97994abef 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -40,6 +40,9 @@ def master_open(): Open a pty master and return the fd, and the filename of the slave end. Deprecated, use openpty() instead.""" + import warnings + warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + try: master_fd, slave_fd = os.openpty() except (AttributeError, OSError): @@ -69,6 +72,9 @@ def slave_open(tty_name): opened filedescriptor. Deprecated, use openpty() instead.""" + import warnings + warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + result = os.open(tty_name, os.O_RDWR) try: from fcntl import ioctl, I_PUSH @@ -101,32 +107,14 @@ def fork(): master_fd, slave_fd = openpty() pid = os.fork() if pid == CHILD: - # Establish a new session. - os.setsid() os.close(master_fd) - - # Slave becomes stdin/stdout/stderr of child. - os.dup2(slave_fd, STDIN_FILENO) - os.dup2(slave_fd, STDOUT_FILENO) - os.dup2(slave_fd, STDERR_FILENO) - if slave_fd > STDERR_FILENO: - os.close(slave_fd) - - # Explicitly open the tty to make it become a controlling tty. - tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) - os.close(tmp_fd) + os.login_tty(slave_fd) else: os.close(slave_fd) # Parent and child process. return pid, master_fd -def _writen(fd, data): - """Write all the data to a descriptor.""" - while data: - n = os.write(fd, data) - data = data[n:] - def _read(fd): """Default read function.""" return os.read(fd, 1024) @@ -136,9 +124,42 @@ def _copy(master_fd, master_read=_read, stdin_read=_read): Copies pty master -> standard output (master_read) standard input -> pty master (stdin_read)""" - fds = [master_fd, STDIN_FILENO] - while fds: - rfds, _wfds, _xfds = select(fds, [], []) + if os.get_blocking(master_fd): + # If we write more than tty/ndisc is willing to buffer, we may block + # indefinitely. So we set master_fd to non-blocking temporarily during + # the copy operation. + os.set_blocking(master_fd, False) + try: + _copy(master_fd, master_read=master_read, stdin_read=stdin_read) + finally: + # restore blocking mode for backwards compatibility + os.set_blocking(master_fd, True) + return + high_waterlevel = 4096 + stdin_avail = master_fd != STDIN_FILENO + stdout_avail = master_fd != STDOUT_FILENO + i_buf = b'' + o_buf = b'' + while 1: + rfds = [] + wfds = [] + if stdin_avail and len(i_buf) < high_waterlevel: + rfds.append(STDIN_FILENO) + if stdout_avail and len(o_buf) < high_waterlevel: + rfds.append(master_fd) + if stdout_avail and len(o_buf) > 0: + wfds.append(STDOUT_FILENO) + if len(i_buf) > 0: + wfds.append(master_fd) + + rfds, wfds, _xfds = select(rfds, wfds, []) + + if STDOUT_FILENO in wfds: + try: + n = os.write(STDOUT_FILENO, o_buf) + o_buf = o_buf[n:] + except OSError: + stdout_avail = False if master_fd in rfds: # Some OSes signal EOF by returning an empty byte string, @@ -150,19 +171,22 @@ def _copy(master_fd, master_read=_read, stdin_read=_read): if not data: # Reached EOF. return # Assume the child process has exited and is # unreachable, so we clean up. - else: - os.write(STDOUT_FILENO, data) + o_buf += data + + if master_fd in wfds: + n = os.write(master_fd, i_buf) + i_buf = i_buf[n:] - if STDIN_FILENO in rfds: + if stdin_avail and STDIN_FILENO in rfds: data = stdin_read(STDIN_FILENO) if not data: - fds.remove(STDIN_FILENO) + stdin_avail = False else: - _writen(master_fd, data) + i_buf += data def spawn(argv, master_read=_read, stdin_read=_read): """Create a spawned process.""" - if type(argv) == type(''): + if isinstance(argv, str): argv = (argv,) sys.audit('pty.spawn', argv) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 388614e51b1..43d8ec90ffb 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -177,7 +177,7 @@ def main(): import argparse description = 'A simple command-line interface for py_compile module.' - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument( '-q', '--quiet', action='store_true', diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b521a550472..1f8a6ef3d7c 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -16,12 +15,15 @@ class or function within a module or module in a package. If the Run "pydoc -k " to search for a keyword in the synopsis lines of all available modules. +Run "pydoc -n " to start an HTTP server with the given +hostname (default: localhost) on the local machine. + Run "pydoc -p " to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and -open a Web browser to interactively browse documentation. The -p option -can be used with the -b option to explicitly specify the server port. +open a web browser to interactively browse documentation. Combine with +the -n and -p options to control the hostname and port used. Run "pydoc -w " to write out the HTML documentation for a module to a file named ".html". @@ -51,6 +53,8 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast +import __future__ import builtins import importlib._bootstrap import importlib._bootstrap_external @@ -63,14 +67,32 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig +import textwrap import time import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -86,9 +108,100 @@ def pathdirs(): normdirs.append(normdir) return dirs +def _findclass(func): + cls = sys.modules.get(func.__module__) + if cls is None: + return None + for name in func.__qualname__.split('.')[:-1]: + cls = getattr(cls, name) + if not inspect.isclass(cls): + return None + return cls + +def _finddoc(obj): + if inspect.ismethod(obj): + name = obj.__func__.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + getattr(getattr(self, name, None), '__func__') is obj.__func__): + # classmethod + cls = self + else: + cls = self.__class__ + elif inspect.isfunction(obj): + name = obj.__name__ + cls = _findclass(obj) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.isbuiltin(obj): + name = obj.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + self.__qualname__ + '.' + name == obj.__qualname__): + # classmethod + cls = self + else: + cls = self.__class__ + # Should be tested before isdatadescriptor(). + elif isinstance(obj, property): + name = obj.__name__ + cls = _findclass(obj.fget) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): + name = obj.__name__ + cls = obj.__objclass__ + if getattr(cls, name) is not obj: + return None + if inspect.ismemberdescriptor(obj): + slots = getattr(cls, '__slots__', None) + if isinstance(slots, dict) and name in slots: + return slots[name] + else: + return None + for base in cls.__mro__: + try: + doc = _getowndoc(getattr(base, name)) + except AttributeError: + continue + if doc is not None: + return doc + return None + +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def _getdoc(object): + """Get the documentation string for an object. + + All tabs are expanded to spaces. To clean up docstrings that are + indented to line up with blocks of code, any whitespace than can be + uniformly removed from the second line onwards is removed.""" + doc = _getowndoc(object) + if doc is None: + try: + doc = _finddoc(object) + except (AttributeError, TypeError): + return None + if not isinstance(doc, str): + return None + return inspect.cleandoc(doc) + def getdoc(object): """Get the doc string or comments for an object.""" - result = inspect.getdoc(object) or inspect.getcomments(object) + result = _getdoc(object) or inspect.getcomments(object) return result and re.sub('^ *\n', '', result.rstrip()) or '' def splitdoc(doc): @@ -100,6 +213,27 @@ def splitdoc(doc): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) +def _getargspec(object): + try: + signature = inspect.signature(object, annotation_format=Format.STRING) + if signature: + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width, quote_annotation_strings=False) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ @@ -107,6 +241,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname and object.__module__ is not None: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -134,12 +281,6 @@ def stripid(text): # The behaviour of %p is implementation-dependent in terms of case. return _re_stripid.sub(r'\1', text) -def _is_some_method(obj): - return (inspect.isfunction(obj) or - inspect.ismethod(obj) or - inspect.isbuiltin(obj) or - inspect.ismethoddescriptor(obj)) - def _is_bound_method(fn): """ Returns True if fn is a bound method, regardless of whether @@ -155,7 +296,7 @@ def _is_bound_method(fn): def allmethods(cl): methods = {} - for key, value in inspect.getmembers(cl, _is_some_method): + for key, value in inspect.getmembers(cl, inspect.isroutine): methods[key] = 1 for base in cl.__bases__: methods.update(allmethods(base)) # all your base are belong to us @@ -180,6 +321,8 @@ def _split_list(s, predicate): no.append(x) return yes, no +_future_feature_names = set(__future__.all_feature_names) + def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. @@ -187,13 +330,19 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__', + '__annotate_func__', '__annotations_cache__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore if name.startswith('_') and hasattr(obj, '_fields'): return True + # Ignore __future__ imports. + if obj is not __future__ and name in _future_feature_names: + if isinstance(getattr(obj, name, None), __future__._Feature): + return False if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -201,11 +350,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' + if isinstance(value, property) and value.fset is None: + kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -225,6 +378,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): @@ -232,21 +387,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" @@ -290,8 +453,17 @@ def synopsis(filename, cache={}): class ErrorDuringImport(Exception): """Errors that occurred while trying to import something to document it.""" def __init__(self, filename, exc_info): + if not isinstance(exc_info, tuple): + assert isinstance(exc_info, BaseException) + self.exc = type(exc_info) + self.value = exc_info + self.tb = exc_info.__traceback__ + else: + warnings.warn("A tuple value for exc_info is deprecated, use an exception instance", + DeprecationWarning) + + self.exc, self.value, self.tb = exc_info self.filename = filename - self.exc, self.value, self.tb = exc_info def __str__(self): exc = self.exc.__name__ @@ -312,8 +484,8 @@ def importfile(path): spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) - except: - raise ErrorDuringImport(path, sys.exc_info()) + except BaseException as err: + raise ErrorDuringImport(path, err) def safeimport(path, forceload=0, cache={}): """Import a module; handle errors; return None if the module isn't found. @@ -340,25 +512,21 @@ def safeimport(path, forceload=0, cache={}): # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] - module = __import__(path) - except: + module = importlib.import_module(path) + except BaseException as err: # Did the error occur before or after the module was found? - (exc, value, tb) = info = sys.exc_info() if path in sys.modules: # An error occurred while executing the imported module. - raise ErrorDuringImport(sys.modules[path].__file__, info) - elif exc is SyntaxError: + raise ErrorDuringImport(sys.modules[path].__file__, err) + elif type(err) is SyntaxError: # A SyntaxError occurred before we could execute the module. - raise ErrorDuringImport(value.filename, info) - elif issubclass(exc, ImportError) and value.name == path: + raise ErrorDuringImport(err.filename, err) + elif isinstance(err, ImportError) and err.name == path: # No such module in the path. return None else: # Some other error occurred during the importing process. - raise ErrorDuringImport(path, sys.exc_info()) - for part in path.split('.')[1:]: - try: module = getattr(module, part) - except AttributeError: return None + raise ErrorDuringImport(path, err) return module # ---------------------------------------------------- formatter base class @@ -376,15 +544,13 @@ def document(self, object, name=None, *args): # identifies something in a way that pydoc itself has issues handling; # think 'super' and how it is a descriptor (which raises the exception # by lacking a __name__ attribute) and an instance. - if inspect.isgetsetdescriptor(object): return self.docdata(*args) - if inspect.ismemberdescriptor(object): return self.docdata(*args) try: if inspect.ismodule(object): return self.docmodule(*args) if inspect.isclass(object): return self.docclass(*args) if inspect.isroutine(object): return self.docroutine(*args) except AttributeError: pass - if isinstance(object, property): return self.docproperty(*args) + if inspect.isdatadescriptor(object): return self.docdata(*args) return self.docother(*args) def fail(self, object, name=None, *args): @@ -395,9 +561,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: @@ -409,16 +573,26 @@ def getdocloc(self, object, basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and - (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', + (object.__name__ in ('errno', 'exceptions', 'gc', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): - if docloc.startswith(("http://", "https://")): - docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): + + try: + from pydoc_data import module_docs + except ImportError: + module_docs = None + + if module_docs and object.__name__ in module_docs.module_docs: + doc_name = module_docs.module_docs[object.__name__] + if docloc.startswith(("http://", "https://")): + docloc = "{}/{}".format(docloc.rstrip("/"), doc_name) + else: + docloc = os.path.join(docloc, doc_name) else: - docloc = os.path.join(docloc, object.__name__.lower() + ".html") + docloc = None else: docloc = None return docloc @@ -454,7 +628,7 @@ def repr_string(self, x, level): # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', - r'\1', + r'\1', self.escape(testrepr)) repr_str = repr_string @@ -479,49 +653,48 @@ class HTMLDoc(Doc): def page(self, title, contents): """Format an HTML page.""" return '''\ - -Python: %s - - + + + + +Python: %s + %s ''' % (title, contents) - def heading(self, title, fgcol, bgcol, extras=''): + def heading(self, title, extras=''): """Format a page heading.""" return ''' -
- -
 
- 
%s
%s
- ''' % (bgcol, fgcol, title, fgcol, extras or ' ') - - def section(self, title, fgcol, bgcol, contents, width=6, + + + +
 
%s
%s
+ ''' % (title, extras or ' ') + + def section(self, title, cls, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: - marginalia = '' + ' ' * width + '' + marginalia = '' + ' ' * width + '' result = '''

- - - - ''' % (bgcol, fgcol, title) +
 
-%s
+ + + ''' % (cls, title) if prelude: result = result + ''' - - -''' % (bgcol, marginalia, prelude, gap) + + +''' % (cls, marginalia, cls, prelude, gap) else: result = result + ''' -''' % (bgcol, marginalia, gap) +''' % (cls, marginalia, gap) - return result + '\n
 
%s
%s%s
%s
%s%s
%s
%s%s
%s%s%s
' % contents + return result + '\n%s' % contents def bigsection(self, title, *args): """Format a section with a big heading.""" - title = '%s' % title + title = '%s' % title return self.section(title, *args) def preformat(self, text): @@ -530,19 +703,19 @@ def preformat(self, text): return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '
\n') - def multicolumn(self, list, format, cols=4): + def multicolumn(self, list, format): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)//cols - for col in range(cols): - result = result + '' % (100//cols) + rows = (len(list) + 3) // 4 + for col in range(4): + result = result + '' for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '
\n' result = result + '' - return '%s
' % result + return '%s
' % result - def grey(self, text): return '%s' % text + def grey(self, text): return '%s' % text def namelink(self, name, *dicts): """Make a link for an identifier, given name-to-URL mappings.""" @@ -559,6 +732,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '%s' % (object.__name__, object.__name__) @@ -588,13 +780,11 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): escape = escape or self.escape results = [] here = 0 - pattern = re.compile(r'\b((http|ftp)://\S+[\w/]|' + pattern = re.compile(r'\b((http|https|ftp)://\S+[\w/]|' r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?(\w+))') - while True: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) @@ -603,10 +793,10 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): url = escape(all).replace('"', '"') results.append('%s' % (url, url)) elif rfc: - url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('%s' % (url, escape(all))) elif pep: - url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://peps.python.org/pep-%04d/' % int(pep) results.append('%s' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' @@ -629,17 +819,17 @@ def formattree(self, tree, modname, parent=None): """Produce HTML for a class tree as given by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry - result = result + '

' + result = result + '
' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' - result = result + '\n
' - elif type(entry) is type([]): + result = result + '\n' + elif isinstance(entry, list): result = result + '
\n%s
\n' % self.formattree( entry, modname, c) return '
\n%s
\n' % result @@ -655,10 +845,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): links = [] for i in range(len(parts)-1): links.append( - '%s' % + '%s' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) - head = '%s' % linkedname + head = '%s' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) @@ -680,9 +870,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): docloc = '
Module Reference' % locals() else: docloc = '' - result = self.heading( - head, '#ffffff', '#7799ee', - 'index
' + filelink + docloc) + result = self.heading(head, 'index
' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) @@ -704,9 +892,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -717,7 +906,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) - doc = doc and '%s' % doc + doc = doc and '%s' % doc result = result + '

%s

\n' % doc if hasattr(object, '__path__'): @@ -727,12 +916,12 @@ def docmodule(self, object, name=None, mod=None, *ignored): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( - 'Package Contents', '#ffffff', '#aa55cc', contents) + 'Package Contents', 'pkg-content', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( - 'Modules', '#ffffff', '#aa55cc', contents) + 'Modules', 'pkg-content', contents) if classes: classlist = [value for (key, value) in classes] @@ -741,27 +930,25 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) + 'Classes', 'index', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) + 'Functions', 'functions', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( - 'Data', '#ffffff', '#55aa55', '
\n'.join(contents)) + 'Data', 'data', '
\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) - result = result + self.bigsection( - 'Author', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Author', 'author', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) - result = result + self.bigsection( - 'Credits', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Credits', 'credits', contents) return result @@ -806,10 +993,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -819,7 +1006,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -829,16 +1016,13 @@ def spilldata(msg, attrs, predicate): push(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) - if callable(value) or inspect.isdatadescriptor(value): - doc = getattr(value, "__doc__", None) - else: - doc = None - if doc is None: + doc = getdoc(value) + if not doc: push('
%s
\n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) - doc = '
%s' % doc + doc = '
%s' % doc push('
%s%s
\n' % (base, doc)) push('\n') return attrs @@ -870,7 +1054,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -889,6 +1073,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill('Static methods %s' % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors('Data descriptors %s' % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata('Data and other attributes %s' % tag, attrs, @@ -909,100 +1095,129 @@ def spilldata(msg, attrs, predicate): for base in bases: parents.append(self.classlink(base, object.__module__)) title = title + '(%s)' % ', '.join(parents) - doc = self.markup(getdoc(object), self.preformat, funcs, classes, mdict) - doc = doc and '%s
 
' % doc - return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) + decl = '' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' + + doc = getdoc(object) + if decl: + doc = decl + (doc or '') + doc = self.markup(doc, self.preformat, funcs, classes, mdict) + doc = doc and '%s
 
' % doc + + return self.section(title, 'title', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = '%s' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '%s' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '%s = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = '%s lambda ' % name - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' - decl = title + self.escape(argspec) + (note and self.grey( - '%s' % note)) + decl = asyncqualifier + title + self.escape(argspec) + (note and + self.grey('%s' % note)) if skipdocs: return '
%s
\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) - doc = doc and '
%s
' % doc + doc = doc and '
%s
' % doc return '
%s
%s
\n' % (decl, doc) - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce html documentation for a data descriptor.""" results = [] push = results.append if name: push('
%s
\n' % name) - if value.__doc__ is not None: - doc = self.markup(getdoc(value), self.preformat) - push('
%s
\n' % doc) + doc = self.markup(getdoc(object), self.preformat) + if doc: + push('
%s
\n' % doc) push('
\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata def docother(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a data object.""" lhs = name and '%s = ' % name or '' return lhs + self.repr(object) - def docdata(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - def index(self, dir, shadowed=None): """Generate an HTML index for a directory of modules.""" modpkgs = [] @@ -1016,7 +1231,7 @@ def index(self, dir, shadowed=None): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) - return self.bigsection(dir, '#ffffff', '#ee77aa', contents) + return self.bigsection(dir, 'index', contents) # -------------------------------------------- text documentation generator @@ -1067,8 +1282,7 @@ def bold(self, text): def indent(self, text, prefix=' '): """Indent text by prepending a given prefix to each line.""" if not text: return '' - lines = [prefix + line for line in text.split('\n')] - if lines: lines[-1] = lines[-1].rstrip() + lines = [(prefix + line).rstrip() for line in text.split('\n')] return '\n'.join(lines) def section(self, title, contents): @@ -1082,19 +1296,19 @@ def formattree(self, tree, modname, parent=None, prefix=''): """Render in text a class tree as returned by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry result = result + prefix + classname(c, modname) if bases and bases != (parent,): parents = (classname(c, modname) for c in bases) result = result + '(%s)' % ', '.join(parents) result = result + '\n' - elif type(entry) is type([]): + elif isinstance(entry, list): result = result + self.formattree( entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1123,9 +1337,10 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1212,10 +1427,17 @@ def makename(c, m=object.__module__): parents = map(makename, bases) title = title + '(%s)' % ', '.join(parents) - doc = getdoc(object) - contents = doc and [doc + '\n'] or [] + contents = [] push = contents.append + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') + + doc = getdoc(object) + if doc: + push(doc + '\n') + # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: @@ -1224,6 +1446,25 @@ def makename(c, m=object.__module__): push(' ' + makename(base)) push('') + # List the built-in subclasses, if any: + subclasses = sorted( + (str(cls.__name__) for cls in type.__subclasses__(object) + if (not cls.__name__.startswith("_") and + getattr(cls, '__module__', '') == "builtins")), + key=str.lower + ) + no_of_subclasses = len(subclasses) + MAX_SUBCLASSES_TO_DISPLAY = 4 + if subclasses: + push("Built-in subclasses:") + for subclassname in subclasses[:MAX_SUBCLASSES_TO_DISPLAY]: + push(' ' + subclassname) + if no_of_subclasses > MAX_SUBCLASSES_TO_DISPLAY: + push(' ... and ' + + str(no_of_subclasses - MAX_SUBCLASSES_TO_DISPLAY) + + ' other subclasses') + push('') + # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): @@ -1245,10 +1486,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1257,7 +1498,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -1266,10 +1507,7 @@ def spilldata(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - if callable(value) or inspect.isdatadescriptor(value): - doc = getdoc(value) - else: - doc = None + doc = getdoc(value) try: obj = getattr(object, name) except AttributeError: @@ -1289,7 +1527,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -1307,6 +1545,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill("Static methods %s:\n" % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s:\n" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, @@ -1324,48 +1564,73 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = self.bold(name) + ' lambda ' - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' - decl = title + argspec + note + decl = asyncqualifier + title + argspec + note if skipdocs: return decl + '\n' @@ -1373,28 +1638,24 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce text documentation for a data descriptor.""" results = [] push = results.append if name: push(self.bold(name)) push('\n') - doc = getdoc(value) or '' + doc = getdoc(object) or '' if doc: push(self.indent(doc)) push('\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata - def docdata(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: @@ -1402,8 +1663,10 @@ def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=No chop = maxlen - len(line) if chop < 0: repr = repr[:chop] + '...' line = (name and self.bold(name) + ' = ' or '') + repr - if doc is not None: - line += '\n' + self.indent(str(doc)) + if not doc: + doc = getdoc(object) + if doc: + line += '\n' + self.indent(str(doc)) + '\n' return line class _PlainTextDoc(TextDoc): @@ -1413,136 +1676,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) - try: - with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - filename = tempfile.mktemp() - with open(filename, 'w', errors='backslashreplace') as file: - file.write(text) - try: - os.system(cmd + ' "' + filename + '"') - finally: - os.unlink(filename) - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1569,6 +1707,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): @@ -1626,36 +1771,49 @@ def render_doc(thing, title='Python Library Documentation: %s', forceload=0, if not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or - inspect.isgetsetdescriptor(object) or - inspect.ismemberdescriptor(object) or - isinstance(object, property)): + inspect.isdatadescriptor(object) or + _getdoc(object)): # If the passed object is a piece of data or an instance, # document its available methods instead of its value. - object = type(object) - desc += ' object' + if hasattr(object, '__origin__'): + object = object.__origin__ + else: + object = type(object) + desc += ' object' return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, - output=None): + output=None, is_cli=False): """Display text documentation, given an object or a path to an object.""" - try: - if output is None: - pager(render_doc(thing, title, forceload)) - else: - output.write(render_doc(thing, title, forceload, plaintext)) - except (ImportError, ErrorDuringImport) as value: - print(value) + if output is None: + try: + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') + except ImportError as exc: + if is_cli: + raise + print(exc) + else: + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" - try: - object, name = resolve(thing, forceload) - page = html.page(describe(object), html.document(object, name)) - with open(name + '.html', 'w', encoding='utf-8') as file: - file.write(page) - print('wrote', name + '.html') - except (ImportError, ErrorDuringImport) as value: - print(value) + object, name = resolve(thing, forceload) + page = html.page(describe(object), html.document(object, name)) + with open(name + '.html', 'w', encoding='utf-8') as file: + file.write(page) + print('wrote', name + '.html') def writedocs(dir, pkgpath='', done=None): """Write out HTML documentation for all modules in a directory tree.""" @@ -1664,6 +1822,37 @@ def writedocs(dir, pkgpath='', done=None): writedoc(modname) return + +def _introdoc(): + import textwrap + ver = '%d.%d' % sys.version_info[:2] + if os.environ.get('PYTHON_BASIC_REPL'): + pyrepl_keys = '' + else: + # Additional help for keyboard shortcuts if enhanced REPL is used. + pyrepl_keys = ''' + You can use the following keyboard shortcuts at the main interpreter prompt. + F1: enter interactive help, F2: enter history browsing mode, F3: enter paste + mode (press again to exit). + ''' + return textwrap.dedent(f'''\ + Welcome to Python {ver}'s help utility! If this is your first time using + Python, you should definitely check out the tutorial at + https://docs.python.org/{ver}/tutorial/. + + Enter the name of any module, keyword, or topic to get help on writing + Python programs and using Python modules. To get a list of available + modules, keywords, symbols, or topics, enter "modules", "keywords", + "symbols", or "topics". + {pyrepl_keys} + Each module also comes with a one-line summary of what it does; to list + the modules whose name or summary contain a given string such as "spam", + enter "modules spam". + + To quit this help utility and return to the interpreter, + enter "q", "quit" or "exit". + ''') + class Helper: # These dictionaries map a topic name to either an alias, or a tuple @@ -1672,7 +1861,7 @@ class Helper: # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the - # list of needed labels in Doc/tools/pyspecific.py and + # list of needed labels in Doc/tools/extensions/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. @@ -1684,6 +1873,8 @@ class Helper: 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), + 'async': ('async', ''), + 'await': ('await', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), @@ -1735,6 +1926,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1749,6 +1941,7 @@ class Helper: if topic not in topics: topics = topics + ' ' + topic symbols[symbol] = topics + del topic, symbols_, symbol, topics topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' @@ -1827,6 +2020,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', @@ -1841,8 +2035,13 @@ def __init__(self, input=None, output=None): self._input = input self._output = output - input = property(lambda self: self._input or sys.stdin) - output = property(lambda self: self._output or sys.stdout) + @property + def input(self): + return self._input or sys.stdin + + @property + def output(self): + return self._output or sys.stdout def __repr__(self): if inspect.stack()[1][3] == '?': @@ -1854,7 +2053,10 @@ def __repr__(self): _GoInteractive = object() def __call__(self, request=_GoInteractive): if request is not self._GoInteractive: - self.help(request) + try: + self.help(request) + except ImportError as err: + self.output.write(f'{err}\n') else: self.intro() self.interact() @@ -1870,17 +2072,18 @@ def interact(self): while True: try: request = self.getline('help> ') - if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() + if not request: + continue # back to the prompt # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -1895,8 +2098,8 @@ def getline(self, prompt): self.output.flush() return self.input.readline() - def help(self, request): - if type(request) is type(''): + def help(self, request, is_cli=False): + if isinstance(request, str): request = request.strip() if request == 'keywords': self.listkeywords() elif request == 'symbols': self.listsymbols() @@ -1907,34 +2110,20 @@ def help(self, request): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:') + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) - elif request: doc(request, 'Help on %s:', output=self._output) - else: doc(str, 'Help on %s:', output=self._output) + elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + else: doc(str, 'Help on %s:', output=self._output, is_cli=is_cli) elif isinstance(request, Helper): self() - else: doc(request, 'Help on %s:', output=self._output) + else: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) self.output.write('\n') def intro(self): - self.output.write(''' -Welcome to Python {0}'s help utility! - -If this is your first time using Python, you should definitely check out -the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. - -Enter the name of any module, keyword, or topic to get help on writing -Python programs and using Python modules. To quit this help utility and -return to the interpreter, just type "quit". - -To get a list of available modules, keywords, symbols, or topics, type -"modules", "keywords", "symbols", or "topics". Each module also comes -with a one-line summary of what it does; to list the modules whose name -or summary contain a given string such as "spam", type "modules spam". -'''.format('%d.%d' % sys.version_info[:2])) + self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -1966,7 +2155,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: @@ -1981,7 +2170,7 @@ def showtopic(self, topic, more_xrefs=''): if not target: self.output.write('no documentation found for %s\n' % repr(topic)) return - if type(target) is type(''): + if isinstance(target, str): return self.showtopic(target, more_xrefs) label, xrefs = target @@ -1998,7 +2187,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2090,7 +2283,7 @@ def run(self, callback, key=None, completer=None, onerror=None): callback(None, modname, '') else: try: - spec = pkgutil._get_spec(importer, modname) + spec = importer.find_spec(modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue @@ -2135,13 +2328,13 @@ def onerror(modname): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------- enhanced Web browser interface +# --------------------------------------- enhanced web browser interface -def _start_server(urlhandler, port): +def _start_server(urlhandler, hostname, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be - browsed dynamically and interactively with a Web browser. Example use: + browsed dynamically and interactively with a web browser. Example use: >>> import time >>> import pydoc @@ -2177,14 +2370,14 @@ def _start_server(urlhandler, port): Let the server do its thing. We just need to monitor its status. Use time.sleep so the loop doesn't hog the CPU. - >>> starttime = time.time() + >>> starttime = time.monotonic() >>> timeout = 1 #seconds This is a short timeout for testing purposes. >>> while serverthread.serving: ... time.sleep(.01) - ... if serverthread.serving and time.time() - starttime > timeout: + ... if serverthread.serving and time.monotonic() - starttime > timeout: ... serverthread.stop() ... break @@ -2222,8 +2415,8 @@ def log_message(self, *args): class DocServer(http.server.HTTPServer): - def __init__(self, port, callback): - self.host = 'localhost' + def __init__(self, host, port, callback): + self.host = host self.address = (self.host, port) self.callback = callback self.base.__init__(self, self.address, self.handler) @@ -2243,12 +2436,14 @@ def server_activate(self): class ServerThread(threading.Thread): - def __init__(self, urlhandler, port): + def __init__(self, urlhandler, host, port): self.urlhandler = urlhandler + self.host = host self.port = int(port) threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2257,11 +2452,11 @@ def run(self): DocServer.handler = DocHandler DocHandler.MessageClass = email.message.Message DocHandler.urlhandler = staticmethod(self.urlhandler) - docsvr = DocServer(self.port, self.ready) + docsvr = DocServer(self.host, self.port, self.ready) self.docserver = docsvr docsvr.serve_until_quit() - except Exception as e: - self.error = e + except Exception as err: + self.error = err def ready(self, server): self.serving = True @@ -2279,11 +2474,11 @@ def stop(self): self.serving = False self.url = None - thread = ServerThread(urlhandler, port) + thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread @@ -2306,15 +2501,14 @@ def page(self, title, contents): '' % css_path) return '''\ - -Pydoc: %s - -%s%s
%s
+ + + + +Pydoc: %s +%s%s
%s
''' % (title, css_link, html_navbar(), contents) - def filelink(self, url, path): - return '%s' % (url, path) - html = _HTMLDoc() @@ -2352,22 +2546,21 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'Index of Modules', - '#ffffff', '#7799ee') + 'Index of Modules' + ) names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '

' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] + 'Built-in Modules', 'index', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( - '

pydoc by Ka-Ping Yee' - '<ping@lfw.org>') + '

pydoc by Ka-Ping Yee' + '<ping@lfw.org>

') return 'Index of Modules', ''.join(contents) def html_search(key): @@ -2392,27 +2585,14 @@ def bltinlink(name): results = [] heading = html.heading( - 'Search Results', - '#ffffff', '#7799ee') + 'Search Results', + ) for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( - 'key = %s' % key, '#ffffff', '#ee77aa', '
'.join(results)) + 'key = %s' % key, 'index', '
'.join(results)) return 'Search Results', contents - def html_getfile(path): - """Get and display a source file listing safely.""" - path = urllib.parse.unquote(path) - with tokenize.open(path) as fp: - lines = html.escape(fp.read()) - body = '
%s
' % lines - heading = html.heading( - 'File Listing', - '#ffffff', '#7799ee') - contents = heading + html.bigsection( - 'File: %s' % path, '#ffffff', '#ee77aa', body) - return 'getfile %s' % path, contents - def html_topics(): """Index of topic texts available.""" @@ -2420,20 +2600,20 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Topics', '#ffffff', '#ee77aa', contents) + 'Topics', 'index', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.keywords.keys()) def bltinlink(name): @@ -2441,7 +2621,7 @@ def bltinlink(name): contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Keywords', '#ffffff', '#ee77aa', contents) + 'Keywords', 'index', contents) return 'Keywords', contents def html_topicpage(topic): @@ -2454,10 +2634,10 @@ def html_topicpage(topic): else: title = 'TOPIC' heading = html.heading( - '%s' % title, - '#ffffff', '#7799ee') + '%s' % title, + ) contents = '
%s
' % html.markup(contents) - contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) + contents = html.bigsection(topic , 'index', contents) if xrefs: xrefs = sorted(xrefs.split()) @@ -2465,8 +2645,7 @@ def bltinlink(name): return '%s' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) - xrefs = html.section('Related help topics: ', - '#ffffff', '#ee77aa', xrefs) + xrefs = html.section('Related help topics: ', 'index', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) @@ -2480,12 +2659,11 @@ def html_getobj(url): def html_error(url, exc): heading = html.heading( - 'Error', - '#ffffff', '#7799ee') + 'Error', + ) contents = '
'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) - contents = heading + html.bigsection(url, '#ffffff', '#bb0000', - contents) + contents = heading + html.bigsection(url, 'error', contents) return "Error - %s" % url, contents def get_html_page(url): @@ -2504,8 +2682,6 @@ def get_html_page(url): op, _, url = url.partition('=') if op == "search?key": title, content = html_search(url) - elif op == "getfile?key": - title, content = html_getfile(url) elif op == "topic?key": # try topics first, then objects. try: @@ -2543,14 +2719,14 @@ def get_html_page(url): raise TypeError('unknown content type %r for url %s' % (content_type, url)) -def browse(port=0, *, open_browser=True): - """Start the enhanced pydoc Web server and open a Web browser. +def browse(port=0, *, open_browser=True, hostname='localhost'): + """Start the enhanced pydoc web server and open a web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. """ import webbrowser - serverthread = _start_server(_url_handler, port) + serverthread = _start_server(_url_handler, hostname, port) if serverthread.error: print(serverthread.error) return @@ -2583,25 +2759,58 @@ def browse(port=0, *, open_browser=True): def ispath(x): return isinstance(x, str) and x.find(os.sep) >= 0 +def _get_revised_path(given_path, argv0): + """Ensures current directory is on returned path, and argv0 directory is not + + Exception: argv0 dir is left alone if it's also pydoc's directory. + + Returns a new path entry list, or None if no adjustment is needed. + """ + # Scripts may get the current directory in their path by default if they're + # run with the -m switch, or directly from the current directory. + # The interactive prompt also allows imports from the current directory. + + # Accordingly, if the current directory is already present, don't make + # any changes to the given_path + if '' in given_path or os.curdir in given_path or os.getcwd() in given_path: + return None + + # Otherwise, add the current directory to the given path, and remove the + # script directory (as long as the latter isn't also pydoc's directory. + stdlib_dir = os.path.dirname(__file__) + script_dir = os.path.dirname(argv0) + revised_path = given_path.copy() + if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir): + revised_path.remove(script_dir) + revised_path.insert(0, os.getcwd()) + return revised_path + + +# Note: the tests only cover _get_revised_path, not _adjust_cli_path itself +def _adjust_cli_sys_path(): + """Ensures current directory is on sys.path, and __main__ directory is not. + + Exception: __main__ dir is left alone if it's also pydoc's directory. + """ + revised_path = _get_revised_path(sys.path, sys.argv[0]) + if revised_path is not None: + sys.path[:] = revised_path + + def cli(): """Command-line interface (looks at sys.argv to decide what to do).""" import getopt class BadUsage(Exception): pass - # Scripts don't get the current directory in their path by default - # unless they are run with the '-m' switch - if '' not in sys.path: - scriptdir = os.path.dirname(sys.argv[0]) - if scriptdir in sys.path: - sys.path.remove(scriptdir) - sys.path.insert(0, '.') + _adjust_cli_sys_path() try: - opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') + opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w') writing = False start_server = False open_browser = False - port = None + port = 0 + hostname = 'localhost' for opt, val in opts: if opt == '-b': start_server = True @@ -2614,18 +2823,19 @@ class BadUsage(Exception): pass port = val if opt == '-w': writing = True + if opt == '-n': + start_server = True + hostname = val if start_server: - if port is None: - port = 0 - browse(port, open_browser=open_browser) + browse(port, hostname=hostname, open_browser=open_browser) return if not args: raise BadUsage for arg in args: if ispath(arg) and not os.path.exists(arg): print('file %r does not exist' % arg) - break + sys.exit(1) try: if ispath(arg) and os.path.isfile(arg): arg = importfile(arg) @@ -2635,9 +2845,10 @@ class BadUsage(Exception): pass else: writedoc(arg) else: - help.help(arg) - except ErrorDuringImport as value: + help.help(arg, is_cli=True) + except (ImportError, ErrorDuringImport) as value: print(value) + sys.exit(1) except (getopt.error, BadUsage): cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] @@ -2654,14 +2865,17 @@ class BadUsage(Exception): pass {cmd} -k Search for a keyword in the synopsis lines of all available modules. +{cmd} -n + Start an HTTP server with the given hostname (default: localhost). + {cmd} -p Start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. {cmd} -b - Start an HTTP server on an arbitrary unused port and open a Web browser - to interactively browse documentation. The -p option can be used with - the -b option to explicitly specify the server port. + Start an HTTP server on an arbitrary unused port and open a web browser + to interactively browse documentation. This option can be used in + combination with -n and/or -p. {cmd} -w ... Write out the HTML documentation for a module to a file in the current diff --git a/Lib/pydoc_data/_pydoc.css b/Lib/pydoc_data/_pydoc.css index f036ef37a5a..a6aa2e4c1a0 100644 --- a/Lib/pydoc_data/_pydoc.css +++ b/Lib/pydoc_data/_pydoc.css @@ -4,3 +4,109 @@ Contents of this file are subject to change without notice. */ + +body { + background-color: #f0f0f8; +} + +table.heading tr { + background-color: #7799ee; +} + +.decor { + color: #ffffff; +} + +.title-decor { + background-color: #ffc8d8; + color: #000000; +} + +.pkg-content-decor { + background-color: #aa55cc; +} + +.index-decor { + background-color: #ee77aa; +} + +.functions-decor { + background-color: #eeaa77; +} + +.data-decor { + background-color: #55aa55; +} + +.author-decor { + background-color: #7799ee; +} + +.credits-decor { + background-color: #7799ee; +} + +.error-decor { + background-color: #bb0000; +} + +.grey { + color: #909090; +} + +.white { + color: #ffffff; +} + +.repr { + color: #c040c0; +} + +table.heading tr td.title { + vertical-align: bottom; +} + +table.heading tr td.extra { + vertical-align: bottom; + text-align: right; +} + +.heading-text { + font-family: helvetica, arial; +} + +.bigsection { + font-size: larger; +} + +.title { + font-size: x-large; +} + +.code { + font-family: monospace; +} + +table { + width: 100%; + border-spacing : 0; + border-collapse : collapse; + border: 0; +} + +td { + padding: 2; +} + +td.section-title { + vertical-align: bottom; +} + +td.multicolumn { + width: 25%; + vertical-align: bottom; +} + +td.singlecolumn { + width: 100%; +} diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py new file mode 100644 index 00000000000..2a6ede3aa14 --- /dev/null +++ b/Lib/pydoc_data/module_docs.py @@ -0,0 +1,320 @@ +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 +# as part of the release process. + +module_docs = { + '__future__': '__future__#module-__future__', + '__main__': '__main__#module-__main__', + '_thread': '_thread#module-_thread', + '_tkinter': 'tkinter#module-_tkinter', + 'abc': 'abc#module-abc', + 'aifc': 'aifc#module-aifc', + 'annotationlib': 'annotationlib#module-annotationlib', + 'argparse': 'argparse#module-argparse', + 'array': 'array#module-array', + 'ast': 'ast#module-ast', + 'asynchat': 'asynchat#module-asynchat', + 'asyncio': 'asyncio#module-asyncio', + 'asyncore': 'asyncore#module-asyncore', + 'atexit': 'atexit#module-atexit', + 'audioop': 'audioop#module-audioop', + 'base64': 'base64#module-base64', + 'bdb': 'bdb#module-bdb', + 'binascii': 'binascii#module-binascii', + 'bisect': 'bisect#module-bisect', + 'builtins': 'builtins#module-builtins', + 'bz2': 'bz2#module-bz2', + 'cProfile': 'profile#module-cProfile', + 'calendar': 'calendar#module-calendar', + 'cgi': 'cgi#module-cgi', + 'cgitb': 'cgitb#module-cgitb', + 'chunk': 'chunk#module-chunk', + 'cmath': 'cmath#module-cmath', + 'cmd': 'cmd#module-cmd', + 'code': 'code#module-code', + 'codecs': 'codecs#module-codecs', + 'codeop': 'codeop#module-codeop', + 'collections': 'collections#module-collections', + 'collections.abc': 'collections.abc#module-collections.abc', + 'colorsys': 'colorsys#module-colorsys', + 'compileall': 'compileall#module-compileall', + 'compression': 'compression#module-compression', + 'compression.zstd': 'compression.zstd#module-compression.zstd', + 'concurrent.futures': 'concurrent.futures#module-concurrent.futures', + 'concurrent.interpreters': 'concurrent.interpreters#module-concurrent.interpreters', + 'configparser': 'configparser#module-configparser', + 'contextlib': 'contextlib#module-contextlib', + 'contextvars': 'contextvars#module-contextvars', + 'copy': 'copy#module-copy', + 'copyreg': 'copyreg#module-copyreg', + 'crypt': 'crypt#module-crypt', + 'csv': 'csv#module-csv', + 'ctypes': 'ctypes#module-ctypes', + 'curses': 'curses#module-curses', + 'curses.ascii': 'curses.ascii#module-curses.ascii', + 'curses.panel': 'curses.panel#module-curses.panel', + 'curses.textpad': 'curses#module-curses.textpad', + 'dataclasses': 'dataclasses#module-dataclasses', + 'datetime': 'datetime#module-datetime', + 'dbm': 'dbm#module-dbm', + 'dbm.dumb': 'dbm#module-dbm.dumb', + 'dbm.gnu': 'dbm#module-dbm.gnu', + 'dbm.ndbm': 'dbm#module-dbm.ndbm', + 'dbm.sqlite3': 'dbm#module-dbm.sqlite3', + 'decimal': 'decimal#module-decimal', + 'difflib': 'difflib#module-difflib', + 'dis': 'dis#module-dis', + 'distutils': 'distutils#module-distutils', + 'doctest': 'doctest#module-doctest', + 'email': 'email#module-email', + 'email.charset': 'email.charset#module-email.charset', + 'email.contentmanager': 'email.contentmanager#module-email.contentmanager', + 'email.encoders': 'email.encoders#module-email.encoders', + 'email.errors': 'email.errors#module-email.errors', + 'email.generator': 'email.generator#module-email.generator', + 'email.header': 'email.header#module-email.header', + 'email.headerregistry': 'email.headerregistry#module-email.headerregistry', + 'email.iterators': 'email.iterators#module-email.iterators', + 'email.message': 'email.message#module-email.message', + 'email.mime': 'email.mime#module-email.mime', + 'email.mime.application': 'email.mime#module-email.mime.application', + 'email.mime.audio': 'email.mime#module-email.mime.audio', + 'email.mime.base': 'email.mime#module-email.mime.base', + 'email.mime.image': 'email.mime#module-email.mime.image', + 'email.mime.message': 'email.mime#module-email.mime.message', + 'email.mime.multipart': 'email.mime#module-email.mime.multipart', + 'email.mime.nonmultipart': 'email.mime#module-email.mime.nonmultipart', + 'email.mime.text': 'email.mime#module-email.mime.text', + 'email.parser': 'email.parser#module-email.parser', + 'email.policy': 'email.policy#module-email.policy', + 'email.utils': 'email.utils#module-email.utils', + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + 'encodings.mbcs': 'codecs#module-encodings.mbcs', + 'encodings.utf_8_sig': 'codecs#module-encodings.utf_8_sig', + 'ensurepip': 'ensurepip#module-ensurepip', + 'enum': 'enum#module-enum', + 'errno': 'errno#module-errno', + 'faulthandler': 'faulthandler#module-faulthandler', + 'fcntl': 'fcntl#module-fcntl', + 'filecmp': 'filecmp#module-filecmp', + 'fileinput': 'fileinput#module-fileinput', + 'fnmatch': 'fnmatch#module-fnmatch', + 'fractions': 'fractions#module-fractions', + 'ftplib': 'ftplib#module-ftplib', + 'functools': 'functools#module-functools', + 'gc': 'gc#module-gc', + 'getopt': 'getopt#module-getopt', + 'getpass': 'getpass#module-getpass', + 'gettext': 'gettext#module-gettext', + 'glob': 'glob#module-glob', + 'graphlib': 'graphlib#module-graphlib', + 'grp': 'grp#module-grp', + 'gzip': 'gzip#module-gzip', + 'hashlib': 'hashlib#module-hashlib', + 'heapq': 'heapq#module-heapq', + 'hmac': 'hmac#module-hmac', + 'html': 'html#module-html', + 'html.entities': 'html.entities#module-html.entities', + 'html.parser': 'html.parser#module-html.parser', + 'http': 'http#module-http', + 'http.client': 'http.client#module-http.client', + 'http.cookiejar': 'http.cookiejar#module-http.cookiejar', + 'http.cookies': 'http.cookies#module-http.cookies', + 'http.server': 'http.server#module-http.server', + 'idlelib': 'idle#module-idlelib', + 'imaplib': 'imaplib#module-imaplib', + 'imghdr': 'imghdr#module-imghdr', + 'imp': 'imp#module-imp', + 'importlib': 'importlib#module-importlib', + 'importlib.abc': 'importlib#module-importlib.abc', + 'importlib.machinery': 'importlib#module-importlib.machinery', + 'importlib.metadata': 'importlib.metadata#module-importlib.metadata', + 'importlib.resources': 'importlib.resources#module-importlib.resources', + 'importlib.resources.abc': 'importlib.resources.abc#module-importlib.resources.abc', + 'importlib.util': 'importlib#module-importlib.util', + 'inspect': 'inspect#module-inspect', + 'io': 'io#module-io', + 'ipaddress': 'ipaddress#module-ipaddress', + 'itertools': 'itertools#module-itertools', + 'json': 'json#module-json', + 'json.tool': 'json#module-json.tool', + 'keyword': 'keyword#module-keyword', + 'linecache': 'linecache#module-linecache', + 'locale': 'locale#module-locale', + 'logging': 'logging#module-logging', + 'logging.config': 'logging.config#module-logging.config', + 'logging.handlers': 'logging.handlers#module-logging.handlers', + 'lzma': 'lzma#module-lzma', + 'mailbox': 'mailbox#module-mailbox', + 'mailcap': 'mailcap#module-mailcap', + 'marshal': 'marshal#module-marshal', + 'math': 'math#module-math', + 'mimetypes': 'mimetypes#module-mimetypes', + 'mmap': 'mmap#module-mmap', + 'modulefinder': 'modulefinder#module-modulefinder', + 'msilib': 'msilib#module-msilib', + 'msvcrt': 'msvcrt#module-msvcrt', + 'multiprocessing': 'multiprocessing#module-multiprocessing', + 'multiprocessing.connection': 'multiprocessing#module-multiprocessing.connection', + 'multiprocessing.dummy': 'multiprocessing#module-multiprocessing.dummy', + 'multiprocessing.managers': 'multiprocessing#module-multiprocessing.managers', + 'multiprocessing.pool': 'multiprocessing#module-multiprocessing.pool', + 'multiprocessing.shared_memory': 'multiprocessing.shared_memory#module-multiprocessing.shared_memory', + 'multiprocessing.sharedctypes': 'multiprocessing#module-multiprocessing.sharedctypes', + 'netrc': 'netrc#module-netrc', + 'nis': 'nis#module-nis', + 'nntplib': 'nntplib#module-nntplib', + 'numbers': 'numbers#module-numbers', + 'operator': 'operator#module-operator', + 'optparse': 'optparse#module-optparse', + 'os': 'os#module-os', + 'os.path': 'os.path#module-os.path', + 'ossaudiodev': 'ossaudiodev#module-ossaudiodev', + 'pathlib': 'pathlib#module-pathlib', + 'pathlib.types': 'pathlib#module-pathlib.types', + 'pdb': 'pdb#module-pdb', + 'pickle': 'pickle#module-pickle', + 'pickletools': 'pickletools#module-pickletools', + 'pipes': 'pipes#module-pipes', + 'pkgutil': 'pkgutil#module-pkgutil', + 'platform': 'platform#module-platform', + 'plistlib': 'plistlib#module-plistlib', + 'poplib': 'poplib#module-poplib', + 'posix': 'posix#module-posix', + 'pprint': 'pprint#module-pprint', + 'profile': 'profile#module-profile', + 'pstats': 'profile#module-pstats', + 'pty': 'pty#module-pty', + 'pwd': 'pwd#module-pwd', + 'py_compile': 'py_compile#module-py_compile', + 'pyclbr': 'pyclbr#module-pyclbr', + 'pydoc': 'pydoc#module-pydoc', + 'queue': 'queue#module-queue', + 'quopri': 'quopri#module-quopri', + 'random': 'random#module-random', + 're': 're#module-re', + 'readline': 'readline#module-readline', + 'reprlib': 'reprlib#module-reprlib', + 'resource': 'resource#module-resource', + 'rlcompleter': 'rlcompleter#module-rlcompleter', + 'runpy': 'runpy#module-runpy', + 'sched': 'sched#module-sched', + 'secrets': 'secrets#module-secrets', + 'select': 'select#module-select', + 'selectors': 'selectors#module-selectors', + 'shelve': 'shelve#module-shelve', + 'shlex': 'shlex#module-shlex', + 'shutil': 'shutil#module-shutil', + 'signal': 'signal#module-signal', + 'site': 'site#module-site', + 'sitecustomize': 'site#module-sitecustomize', + 'smtpd': 'smtpd#module-smtpd', + 'smtplib': 'smtplib#module-smtplib', + 'sndhdr': 'sndhdr#module-sndhdr', + 'socket': 'socket#module-socket', + 'socketserver': 'socketserver#module-socketserver', + 'spwd': 'spwd#module-spwd', + 'sqlite3': 'sqlite3#module-sqlite3', + 'ssl': 'ssl#module-ssl', + 'stat': 'stat#module-stat', + 'statistics': 'statistics#module-statistics', + 'string': 'string#module-string', + 'string.templatelib': 'string.templatelib#module-string.templatelib', + 'stringprep': 'stringprep#module-stringprep', + 'struct': 'struct#module-struct', + 'subprocess': 'subprocess#module-subprocess', + 'sunau': 'sunau#module-sunau', + 'symtable': 'symtable#module-symtable', + 'sys': 'sys#module-sys', + 'sys.monitoring': 'sys.monitoring#module-sys.monitoring', + 'sysconfig': 'sysconfig#module-sysconfig', + 'syslog': 'syslog#module-syslog', + 'tabnanny': 'tabnanny#module-tabnanny', + 'tarfile': 'tarfile#module-tarfile', + 'telnetlib': 'telnetlib#module-telnetlib', + 'tempfile': 'tempfile#module-tempfile', + 'termios': 'termios#module-termios', + 'test': 'test#module-test', + 'test.regrtest': 'test#module-test.regrtest', + 'test.support': 'test#module-test.support', + 'test.support.bytecode_helper': 'test#module-test.support.bytecode_helper', + 'test.support.import_helper': 'test#module-test.support.import_helper', + 'test.support.os_helper': 'test#module-test.support.os_helper', + 'test.support.script_helper': 'test#module-test.support.script_helper', + 'test.support.socket_helper': 'test#module-test.support.socket_helper', + 'test.support.threading_helper': 'test#module-test.support.threading_helper', + 'test.support.warnings_helper': 'test#module-test.support.warnings_helper', + 'textwrap': 'textwrap#module-textwrap', + 'threading': 'threading#module-threading', + 'time': 'time#module-time', + 'timeit': 'timeit#module-timeit', + 'tkinter': 'tkinter#module-tkinter', + 'tkinter.colorchooser': 'tkinter.colorchooser#module-tkinter.colorchooser', + 'tkinter.commondialog': 'dialog#module-tkinter.commondialog', + 'tkinter.dnd': 'tkinter.dnd#module-tkinter.dnd', + 'tkinter.filedialog': 'dialog#module-tkinter.filedialog', + 'tkinter.font': 'tkinter.font#module-tkinter.font', + 'tkinter.messagebox': 'tkinter.messagebox#module-tkinter.messagebox', + 'tkinter.scrolledtext': 'tkinter.scrolledtext#module-tkinter.scrolledtext', + 'tkinter.simpledialog': 'dialog#module-tkinter.simpledialog', + 'tkinter.ttk': 'tkinter.ttk#module-tkinter.ttk', + 'token': 'token#module-token', + 'tokenize': 'tokenize#module-tokenize', + 'tomllib': 'tomllib#module-tomllib', + 'trace': 'trace#module-trace', + 'traceback': 'traceback#module-traceback', + 'tracemalloc': 'tracemalloc#module-tracemalloc', + 'tty': 'tty#module-tty', + 'turtle': 'turtle#module-turtle', + 'turtledemo': 'turtle#module-turtledemo', + 'types': 'types#module-types', + 'typing': 'typing#module-typing', + 'unicodedata': 'unicodedata#module-unicodedata', + 'unittest': 'unittest#module-unittest', + 'unittest.mock': 'unittest.mock#module-unittest.mock', + 'urllib': 'urllib#module-urllib', + 'urllib.error': 'urllib.error#module-urllib.error', + 'urllib.parse': 'urllib.parse#module-urllib.parse', + 'urllib.request': 'urllib.request#module-urllib.request', + 'urllib.response': 'urllib.request#module-urllib.response', + 'urllib.robotparser': 'urllib.robotparser#module-urllib.robotparser', + 'usercustomize': 'site#module-usercustomize', + 'uu': 'uu#module-uu', + 'uuid': 'uuid#module-uuid', + 'venv': 'venv#module-venv', + 'warnings': 'warnings#module-warnings', + 'wave': 'wave#module-wave', + 'weakref': 'weakref#module-weakref', + 'webbrowser': 'webbrowser#module-webbrowser', + 'winreg': 'winreg#module-winreg', + 'winsound': 'winsound#module-winsound', + 'wsgiref': 'wsgiref#module-wsgiref', + 'wsgiref.handlers': 'wsgiref#module-wsgiref.handlers', + 'wsgiref.headers': 'wsgiref#module-wsgiref.headers', + 'wsgiref.simple_server': 'wsgiref#module-wsgiref.simple_server', + 'wsgiref.types': 'wsgiref#module-wsgiref.types', + 'wsgiref.util': 'wsgiref#module-wsgiref.util', + 'wsgiref.validate': 'wsgiref#module-wsgiref.validate', + 'xdrlib': 'xdrlib#module-xdrlib', + 'xml': 'xml#module-xml', + 'xml.dom': 'xml.dom#module-xml.dom', + 'xml.dom.minidom': 'xml.dom.minidom#module-xml.dom.minidom', + 'xml.dom.pulldom': 'xml.dom.pulldom#module-xml.dom.pulldom', + 'xml.etree.ElementInclude': 'xml.etree.elementtree#module-xml.etree.ElementInclude', + 'xml.etree.ElementTree': 'xml.etree.elementtree#module-xml.etree.ElementTree', + 'xml.parsers.expat': 'pyexpat#module-xml.parsers.expat', + 'xml.parsers.expat.errors': 'pyexpat#module-xml.parsers.expat.errors', + 'xml.parsers.expat.model': 'pyexpat#module-xml.parsers.expat.model', + 'xml.sax': 'xml.sax#module-xml.sax', + 'xml.sax.handler': 'xml.sax.handler#module-xml.sax.handler', + 'xml.sax.saxutils': 'xml.sax.utils#module-xml.sax.saxutils', + 'xml.sax.xmlreader': 'xml.sax.reader#module-xml.sax.xmlreader', + 'xmlrpc': 'xmlrpc#module-xmlrpc', + 'xmlrpc.client': 'xmlrpc.client#module-xmlrpc.client', + 'xmlrpc.server': 'xmlrpc.server#module-xmlrpc.server', + 'zipapp': 'zipapp#module-zipapp', + 'zipfile': 'zipfile#module-zipfile', + 'zipimport': 'zipimport#module-zipimport', + 'zlib': 'zlib#module-zlib', + 'zoneinfo': 'zoneinfo#module-zoneinfo', +} diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e4c63058087..4e31cf08bb5 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,13062 +1,14272 @@ -# -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sun Dec 23 16:24:21 2018 -topics = {'assert': 'The "assert" statement\n' - '**********************\n' - '\n' - 'Assert statements are a convenient way to insert debugging ' - 'assertions\n' - 'into a program:\n' - '\n' - ' assert_stmt ::= "assert" expression ["," expression]\n' - '\n' - 'The simple form, "assert expression", is equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression: raise AssertionError\n' - '\n' - 'The extended form, "assert expression1, expression2", is ' - 'equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression1: raise AssertionError(expression2)\n' - '\n' - 'These equivalences assume that "__debug__" and "AssertionError" ' - 'refer\n' - 'to the built-in variables with those names. In the current\n' - 'implementation, the built-in variable "__debug__" is "True" under\n' - 'normal circumstances, "False" when optimization is requested ' - '(command\n' - 'line option "-O"). The current code generator emits no code for ' - 'an\n' - 'assert statement when optimization is requested at compile time. ' - 'Note\n' - 'that it is unnecessary to include the source code for the ' - 'expression\n' - 'that failed in the error message; it will be displayed as part of ' - 'the\n' - 'stack trace.\n' - '\n' - 'Assignments to "__debug__" are illegal. The value for the ' - 'built-in\n' - 'variable is determined when the interpreter starts.\n', - 'assignment': 'Assignment statements\n' - '*********************\n' - '\n' - 'Assignment statements are used to (re)bind names to values and ' - 'to\n' - 'modify attributes or items of mutable objects:\n' - '\n' - ' assignment_stmt ::= (target_list "=")+ (starred_expression ' - '| yield_expression)\n' - ' target_list ::= target ("," target)* [","]\n' - ' target ::= identifier\n' - ' | "(" [target_list] ")"\n' - ' | "[" [target_list] "]"\n' - ' | attributeref\n' - ' | subscription\n' - ' | slicing\n' - ' | "*" target\n' - '\n' - '(See section Primaries for the syntax definitions for ' - '*attributeref*,\n' - '*subscription*, and *slicing*.)\n' - '\n' - 'An assignment statement evaluates the expression list ' - '(remember that\n' - 'this can be a single expression or a comma-separated list, the ' - 'latter\n' - 'yielding a tuple) and assigns the single resulting object to ' - 'each of\n' - 'the target lists, from left to right.\n' - '\n' - 'Assignment is defined recursively depending on the form of the ' - 'target\n' - '(list). When a target is part of a mutable object (an ' - 'attribute\n' - 'reference, subscription or slicing), the mutable object must\n' - 'ultimately perform the assignment and decide about its ' - 'validity, and\n' - 'may raise an exception if the assignment is unacceptable. The ' - 'rules\n' - 'observed by various types and the exceptions raised are given ' - 'with the\n' - 'definition of the object types (see section The standard type\n' - 'hierarchy).\n' - '\n' - 'Assignment of an object to a target list, optionally enclosed ' - 'in\n' - 'parentheses or square brackets, is recursively defined as ' - 'follows.\n' - '\n' - '* If the target list is a single target with no trailing ' - 'comma,\n' - ' optionally in parentheses, the object is assigned to that ' - 'target.\n' - '\n' - '* Else: The object must be an iterable with the same number of ' - 'items\n' - ' as there are targets in the target list, and the items are ' - 'assigned,\n' - ' from left to right, to the corresponding targets.\n' - '\n' - ' * If the target list contains one target prefixed with an\n' - ' asterisk, called a “starred” target: The object must be ' - 'an\n' - ' iterable with at least as many items as there are targets ' - 'in the\n' - ' target list, minus one. The first items of the iterable ' - 'are\n' - ' assigned, from left to right, to the targets before the ' - 'starred\n' - ' target. The final items of the iterable are assigned to ' - 'the\n' - ' targets after the starred target. A list of the remaining ' - 'items\n' - ' in the iterable is then assigned to the starred target ' - '(the list\n' - ' can be empty).\n' - '\n' - ' * Else: The object must be an iterable with the same number ' - 'of\n' - ' items as there are targets in the target list, and the ' - 'items are\n' - ' assigned, from left to right, to the corresponding ' - 'targets.\n' - '\n' - 'Assignment of an object to a single target is recursively ' - 'defined as\n' - 'follows.\n' - '\n' - '* If the target is an identifier (name):\n' - '\n' - ' * If the name does not occur in a "global" or "nonlocal" ' - 'statement\n' - ' in the current code block: the name is bound to the object ' - 'in the\n' - ' current local namespace.\n' - '\n' - ' * Otherwise: the name is bound to the object in the global\n' - ' namespace or the outer namespace determined by ' - '"nonlocal",\n' - ' respectively.\n' - '\n' - ' The name is rebound if it was already bound. This may cause ' - 'the\n' - ' reference count for the object previously bound to the name ' - 'to reach\n' - ' zero, causing the object to be deallocated and its ' - 'destructor (if it\n' - ' has one) to be called.\n' - '\n' - '* If the target is an attribute reference: The primary ' - 'expression in\n' - ' the reference is evaluated. It should yield an object with\n' - ' assignable attributes; if this is not the case, "TypeError" ' - 'is\n' - ' raised. That object is then asked to assign the assigned ' - 'object to\n' - ' the given attribute; if it cannot perform the assignment, it ' - 'raises\n' - ' an exception (usually but not necessarily ' - '"AttributeError").\n' - '\n' - ' Note: If the object is a class instance and the attribute ' - 'reference\n' - ' occurs on both sides of the assignment operator, the RHS ' - 'expression,\n' - ' "a.x" can access either an instance attribute or (if no ' - 'instance\n' - ' attribute exists) a class attribute. The LHS target "a.x" ' - 'is always\n' - ' set as an instance attribute, creating it if necessary. ' - 'Thus, the\n' - ' two occurrences of "a.x" do not necessarily refer to the ' - 'same\n' - ' attribute: if the RHS expression refers to a class ' - 'attribute, the\n' - ' LHS creates a new instance attribute as the target of the\n' - ' assignment:\n' - '\n' - ' class Cls:\n' - ' x = 3 # class variable\n' - ' inst = Cls()\n' - ' inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x ' - 'as 3\n' - '\n' - ' This description does not necessarily apply to descriptor\n' - ' attributes, such as properties created with "property()".\n' - '\n' - '* If the target is a subscription: The primary expression in ' - 'the\n' - ' reference is evaluated. It should yield either a mutable ' - 'sequence\n' - ' object (such as a list) or a mapping object (such as a ' - 'dictionary).\n' - ' Next, the subscript expression is evaluated.\n' - '\n' - ' If the primary is a mutable sequence object (such as a ' - 'list), the\n' - ' subscript must yield an integer. If it is negative, the ' - 'sequence’s\n' - ' length is added to it. The resulting value must be a ' - 'nonnegative\n' - ' integer less than the sequence’s length, and the sequence is ' - 'asked\n' - ' to assign the assigned object to its item with that index. ' - 'If the\n' - ' index is out of range, "IndexError" is raised (assignment to ' - 'a\n' - ' subscripted sequence cannot add new items to a list).\n' - '\n' - ' If the primary is a mapping object (such as a dictionary), ' - 'the\n' - ' subscript must have a type compatible with the mapping’s key ' - 'type,\n' - ' and the mapping is then asked to create a key/datum pair ' - 'which maps\n' - ' the subscript to the assigned object. This can either ' - 'replace an\n' - ' existing key/value pair with the same key value, or insert a ' - 'new\n' - ' key/value pair (if no key with the same value existed).\n' - '\n' - ' For user-defined objects, the "__setitem__()" method is ' - 'called with\n' - ' appropriate arguments.\n' - '\n' - '* If the target is a slicing: The primary expression in the\n' - ' reference is evaluated. It should yield a mutable sequence ' - 'object\n' - ' (such as a list). The assigned object should be a sequence ' - 'object\n' - ' of the same type. Next, the lower and upper bound ' - 'expressions are\n' - ' evaluated, insofar they are present; defaults are zero and ' - 'the\n' - ' sequence’s length. The bounds should evaluate to integers. ' - 'If\n' - ' either bound is negative, the sequence’s length is added to ' - 'it. The\n' - ' resulting bounds are clipped to lie between zero and the ' - 'sequence’s\n' - ' length, inclusive. Finally, the sequence object is asked to ' - 'replace\n' - ' the slice with the items of the assigned sequence. The ' - 'length of\n' - ' the slice may be different from the length of the assigned ' - 'sequence,\n' - ' thus changing the length of the target sequence, if the ' - 'target\n' - ' sequence allows it.\n' - '\n' - '**CPython implementation detail:** In the current ' - 'implementation, the\n' - 'syntax for targets is taken to be the same as for expressions, ' - 'and\n' - 'invalid syntax is rejected during the code generation phase, ' - 'causing\n' - 'less detailed error messages.\n' - '\n' - 'Although the definition of assignment implies that overlaps ' - 'between\n' - 'the left-hand side and the right-hand side are ‘simultaneous’ ' - '(for\n' - 'example "a, b = b, a" swaps two variables), overlaps *within* ' - 'the\n' - 'collection of assigned-to variables occur left-to-right, ' - 'sometimes\n' - 'resulting in confusion. For instance, the following program ' - 'prints\n' - '"[0, 2]":\n' - '\n' - ' x = [0, 1]\n' - ' i = 0\n' - ' i, x[i] = 1, 2 # i is updated, then x[i] is ' - 'updated\n' - ' print(x)\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3132** - Extended Iterable Unpacking\n' - ' The specification for the "*target" feature.\n' - '\n' - '\n' - 'Augmented assignment statements\n' - '===============================\n' - '\n' - 'Augmented assignment is the combination, in a single ' - 'statement, of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal ' - 'effect. In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the ' - 'old object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to ' - '"a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n' - '\n' - '\n' - 'Annotated assignment statements\n' - '===============================\n' - '\n' - 'Annotation assignment is the combination, in a single ' - 'statement, of a\n' - 'variable or attribute annotation and an optional assignment ' - 'statement:\n' - '\n' - ' annotated_assignment_stmt ::= augtarget ":" expression ["=" ' - 'expression]\n' - '\n' - 'The difference from normal Assignment statements is that only ' - 'single\n' - 'target and only single right hand side value is allowed.\n' - '\n' - 'For simple names as assignment targets, if in class or module ' - 'scope,\n' - 'the annotations are evaluated and stored in a special class or ' - 'module\n' - 'attribute "__annotations__" that is a dictionary mapping from ' - 'variable\n' - 'names (mangled if private) to evaluated annotations. This ' - 'attribute is\n' - 'writable and is automatically created at the start of class or ' - 'module\n' - 'body execution, if annotations are found statically.\n' - '\n' - 'For expressions as assignment targets, the annotations are ' - 'evaluated\n' - 'if in class or module scope, but not stored.\n' - '\n' - 'If a name is annotated in a function scope, then this name is ' - 'local\n' - 'for that scope. Annotations are never evaluated and stored in ' - 'function\n' - 'scopes.\n' - '\n' - 'If the right hand side is present, an annotated assignment ' - 'performs\n' - 'the actual assignment before evaluating annotations (where\n' - 'applicable). If the right hand side is not present for an ' - 'expression\n' - 'target, then the interpreter evaluates the target except for ' - 'the last\n' - '"__setitem__()" or "__setattr__()" call.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 526** - Syntax for Variable Annotations\n' - ' The proposal that added syntax for annotating the types ' - 'of\n' - ' variables (including class variables and instance ' - 'variables),\n' - ' instead of expressing them through comments.\n' - '\n' - ' **PEP 484** - Type hints\n' - ' The proposal that added the "typing" module to provide a ' - 'standard\n' - ' syntax for type annotations that can be used in static ' - 'analysis\n' - ' tools and IDEs.\n', - 'atom-identifiers': 'Identifiers (Names)\n' - '*******************\n' - '\n' - 'An identifier occurring as an atom is a name. See ' - 'section Identifiers\n' - 'and keywords for lexical definition and section Naming ' - 'and binding for\n' - 'documentation of naming and binding.\n' - '\n' - 'When the name is bound to an object, evaluation of the ' - 'atom yields\n' - 'that object. When a name is not bound, an attempt to ' - 'evaluate it\n' - 'raises a "NameError" exception.\n' - '\n' - '**Private name mangling:** When an identifier that ' - 'textually occurs in\n' - 'a class definition begins with two or more underscore ' - 'characters and\n' - 'does not end in two or more underscores, it is ' - 'considered a *private\n' - 'name* of that class. Private names are transformed to a ' - 'longer form\n' - 'before code is generated for them. The transformation ' - 'inserts the\n' - 'class name, with leading underscores removed and a ' - 'single underscore\n' - 'inserted, in front of the name. For example, the ' - 'identifier "__spam"\n' - 'occurring in a class named "Ham" will be transformed to ' - '"_Ham__spam".\n' - 'This transformation is independent of the syntactical ' - 'context in which\n' - 'the identifier is used. If the transformed name is ' - 'extremely long\n' - '(longer than 255 characters), implementation defined ' - 'truncation may\n' - 'happen. If the class name consists only of underscores, ' - 'no\n' - 'transformation is done.\n', - 'atom-literals': 'Literals\n' - '********\n' - '\n' - 'Python supports string and bytes literals and various ' - 'numeric\n' - 'literals:\n' - '\n' - ' literal ::= stringliteral | bytesliteral\n' - ' | integer | floatnumber | imagnumber\n' - '\n' - 'Evaluation of a literal yields an object of the given type ' - '(string,\n' - 'bytes, integer, floating point number, complex number) with ' - 'the given\n' - 'value. The value may be approximated in the case of ' - 'floating point\n' - 'and imaginary (complex) literals. See section Literals for ' - 'details.\n' - '\n' - 'All literals correspond to immutable data types, and hence ' - 'the\n' - 'object’s identity is less important than its value. ' - 'Multiple\n' - 'evaluations of literals with the same value (either the ' - 'same\n' - 'occurrence in the program text or a different occurrence) ' - 'may obtain\n' - 'the same object or a different object with the same ' - 'value.\n', - 'attribute-access': 'Customizing attribute access\n' - '****************************\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with ' - 'an\n' - ' "AttributeError" (either "__getattribute__()" raises ' - 'an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or ' - '"__get__()" of a *name*\n' - ' property raises "AttributeError"). This method ' - 'should either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the ' - 'normal mechanism,\n' - ' "__getattr__()" is not called. (This is an ' - 'intentional asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise ' - '"__getattr__()" would have\n' - ' no way to access other attributes of the instance. ' - 'Note that at\n' - ' least for instance variables, you can fake total ' - 'control by not\n' - ' inserting any values in the instance attribute ' - 'dictionary (but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to ' - 'actually get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute ' - 'accesses for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless ' - '"__getattribute__()" either\n' - ' calls it explicitly or raises an "AttributeError". ' - 'This method\n' - ' should return the (computed) attribute value or raise ' - 'an\n' - ' "AttributeError" exception. In order to avoid ' - 'infinite recursion in\n' - ' this method, its implementation should always call ' - 'the base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note: This method may still be bypassed when looking ' - 'up special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. ' - 'This is called\n' - ' instead of the normal mechanism (i.e. store the value ' - 'in the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, ' - 'for example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion ' - 'instead of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. A ' - 'sequence must be\n' - ' returned. "dir()" converts the returned sequence to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '===================================\n' - '\n' - 'For a more fine grained customization of the module ' - 'behavior (setting\n' - 'attributes, properties, etc.), one can set the ' - '"__class__" attribute\n' - 'of a module object to a subclass of "types.ModuleType". ' - 'For example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' setattr(self, attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note: Setting module "__class__" only affects lookups ' - 'made using the\n' - ' attribute access syntax – directly accessing the ' - 'module globals\n' - ' (whether by code within the module, or via a reference ' - 'to the\n' - ' module’s globals dictionary) is unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is ' - 'now writable.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '========================\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the ' - 'owner’s class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). *owner* is always the owner class, while ' - '*instance* is the\n' - ' instance that the attribute was accessed through, or ' - '"None" when\n' - ' the attribute is accessed through the *owner*. This ' - 'method should\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* ' - 'of the owner\n' - ' class to a new value, *value*.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance ' - '*instance* of the\n' - ' owner class.\n' - '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is ' - 'created. The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' New in version 3.6.\n' - '\n' - 'The attribute "__objclass__" is interpreted by the ' - '"inspect" module as\n' - 'specifying the class where this object was defined ' - '(setting this\n' - 'appropriately can assist in runtime introspection of ' - 'dynamic class\n' - 'attributes). For callables, it may indicate that an ' - 'instance of the\n' - 'given type (or a subclass) is expected or required as ' - 'the first\n' - 'positional argument (for example, CPython sets this ' - 'attribute for\n' - 'unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '====================\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been ' - 'overridden by methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", ' - 'and\n' - '"__delete__()". If any of those methods are defined for ' - 'an object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, ' - 'set, or delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the ' - 'base classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining ' - 'one of the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs ' - 'in the\n' - 'precedence chain depends on which descriptor methods ' - 'were defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a ' - 'binding, "a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is ' - 'transformed into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' If "a" is an instance of "super", then the binding ' - '"super(B,\n' - ' obj).m()" searches "obj.__class__.__mro__" for the ' - 'base class "A"\n' - ' immediately preceding "B" and then invokes the ' - 'descriptor with the\n' - ' call: "A.__dict__[\'m\'].__get__(obj, ' - 'obj.__class__)".\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on the which descriptor methods are defined. A ' - 'descriptor can define\n' - 'any combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' - 'does not define "__get__()", then accessing the ' - 'attribute will return\n' - 'the descriptor object itself unless there is a value in ' - 'the object’s\n' - 'instance dictionary. If the descriptor defines ' - '"__set__()" and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors ' - 'define both\n' - '"__get__()" and "__set__()", while non-data descriptors ' - 'have just the\n' - '"__get__()" method. Data descriptors with "__set__()" ' - 'and "__get__()"\n' - 'defined always override a redefinition in an instance ' - 'dictionary. In\n' - 'contrast, non-data descriptors can be overridden by ' - 'instances.\n' - '\n' - 'Python methods (including "staticmethod()" and ' - '"classmethod()") are\n' - 'implemented as non-data descriptors. Accordingly, ' - 'instances can\n' - 'redefine and override methods. This allows individual ' - 'instances to\n' - 'acquire behaviors that differ from other instances of ' - 'the same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '=========\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of *__dict__* and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available ' - 'in a parent.)\n' - '\n' - 'The space saved over using *__dict__* can be ' - 'significant.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, ' - 'iterable, or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and ' - 'prevents the\n' - ' automatic creation of *__dict__* and *__weakref__* ' - 'for each\n' - ' instance.\n' - '\n' - '\n' - 'Notes on using *__slots__*\n' - '--------------------------\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will ' - 'always be\n' - ' accessible.\n' - '\n' - '* Without a *__dict__* variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then ' - 'add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then ' - 'add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors (Implementing Descriptors) for each ' - 'variable name. As a\n' - ' result, class attributes cannot be used to set default ' - 'values for\n' - ' instance variables defined by *__slots__*; otherwise, ' - 'the class\n' - ' attribute would overwrite the descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited ' - 'to the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses ' - 'will get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' - '\n' - '* If a class defines a slot also defined in a base ' - 'class, the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In ' - 'the future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", ' - '"bytes" and "tuple".\n' - '\n' - '* Any non-string iterable may be assigned to ' - '*__slots__*. Mappings\n' - ' may also be used; however, in the future, special ' - 'meaning may be\n' - ' assigned to the values corresponding to each key.\n' - '\n' - '* *__class__* assignment works only if both classes have ' - 'the same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent ' - 'classes can be\n' - ' used, but only one parent is allowed to have ' - 'attributes created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n', - 'attribute-references': 'Attribute references\n' - '********************\n' - '\n' - 'An attribute reference is a primary followed by a ' - 'period and a name:\n' - '\n' - ' attributeref ::= primary "." identifier\n' - '\n' - 'The primary must evaluate to an object of a type ' - 'that supports\n' - 'attribute references, which most objects do. This ' - 'object is then\n' - 'asked to produce the attribute whose name is the ' - 'identifier. This\n' - 'production can be customized by overriding the ' - '"__getattr__()" method.\n' - 'If this attribute is not available, the exception ' - '"AttributeError" is\n' - 'raised. Otherwise, the type and value of the object ' - 'produced is\n' - 'determined by the object. Multiple evaluations of ' - 'the same attribute\n' - 'reference may yield different objects.\n', - 'augassign': 'Augmented assignment statements\n' - '*******************************\n' - '\n' - 'Augmented assignment is the combination, in a single statement, ' - 'of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal effect. ' - 'In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the old ' - 'object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to "a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n', - 'binary': 'Binary arithmetic operations\n' - '****************************\n' - '\n' - 'The binary arithmetic operations have the conventional priority\n' - 'levels. Note that some of these operations also apply to certain ' - 'non-\n' - 'numeric types. Apart from the power operator, there are only two\n' - 'levels, one for multiplicative operators and one for additive\n' - 'operators:\n' - '\n' - ' m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n' - ' m_expr "//" u_expr | m_expr "/" u_expr |\n' - ' m_expr "%" u_expr\n' - ' a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n' - '\n' - 'The "*" (multiplication) operator yields the product of its ' - 'arguments.\n' - 'The arguments must either both be numbers, or one argument must be ' - 'an\n' - 'integer and the other must be a sequence. In the former case, the\n' - 'numbers are converted to a common type and then multiplied ' - 'together.\n' - 'In the latter case, sequence repetition is performed; a negative\n' - 'repetition factor yields an empty sequence.\n' - '\n' - 'The "@" (at) operator is intended to be used for matrix\n' - 'multiplication. No builtin Python types implement this operator.\n' - '\n' - 'New in version 3.5.\n' - '\n' - 'The "/" (division) and "//" (floor division) operators yield the\n' - 'quotient of their arguments. The numeric arguments are first\n' - 'converted to a common type. Division of integers yields a float, ' - 'while\n' - 'floor division of integers results in an integer; the result is ' - 'that\n' - 'of mathematical division with the ‘floor’ function applied to the\n' - 'result. Division by zero raises the "ZeroDivisionError" ' - 'exception.\n' - '\n' - 'The "%" (modulo) operator yields the remainder from the division ' - 'of\n' - 'the first argument by the second. The numeric arguments are ' - 'first\n' - 'converted to a common type. A zero right argument raises the\n' - '"ZeroDivisionError" exception. The arguments may be floating ' - 'point\n' - 'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals ' - '"4*0.7 +\n' - '0.34".) The modulo operator always yields a result with the same ' - 'sign\n' - 'as its second operand (or zero); the absolute value of the result ' - 'is\n' - 'strictly smaller than the absolute value of the second operand ' - '[1].\n' - '\n' - 'The floor division and modulo operators are connected by the ' - 'following\n' - 'identity: "x == (x//y)*y + (x%y)". Floor division and modulo are ' - 'also\n' - 'connected with the built-in function "divmod()": "divmod(x, y) ==\n' - '(x//y, x%y)". [2].\n' - '\n' - 'In addition to performing the modulo operation on numbers, the ' - '"%"\n' - 'operator is also overloaded by string objects to perform ' - 'old-style\n' - 'string formatting (also known as interpolation). The syntax for\n' - 'string formatting is described in the Python Library Reference,\n' - 'section printf-style String Formatting.\n' - '\n' - 'The floor division operator, the modulo operator, and the ' - '"divmod()"\n' - 'function are not defined for complex numbers. Instead, convert to ' - 'a\n' - 'floating point number using the "abs()" function if appropriate.\n' - '\n' - 'The "+" (addition) operator yields the sum of its arguments. The\n' - 'arguments must either both be numbers or both be sequences of the ' - 'same\n' - 'type. In the former case, the numbers are converted to a common ' - 'type\n' - 'and then added together. In the latter case, the sequences are\n' - 'concatenated.\n' - '\n' - 'The "-" (subtraction) operator yields the difference of its ' - 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n', - 'bitwise': 'Binary bitwise operations\n' - '*************************\n' - '\n' - 'Each of the three bitwise operations has a different priority ' - 'level:\n' - '\n' - ' and_expr ::= shift_expr | and_expr "&" shift_expr\n' - ' xor_expr ::= and_expr | xor_expr "^" and_expr\n' - ' or_expr ::= xor_expr | or_expr "|" xor_expr\n' - '\n' - 'The "&" operator yields the bitwise AND of its arguments, which ' - 'must\n' - 'be integers.\n' - '\n' - 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers.\n' - '\n' - 'The "|" operator yields the bitwise (inclusive) OR of its ' - 'arguments,\n' - 'which must be integers.\n', - 'bltin-code-objects': 'Code Objects\n' - '************\n' - '\n' - 'Code objects are used by the implementation to ' - 'represent “pseudo-\n' - 'compiled” executable Python code such as a function ' - 'body. They differ\n' - 'from function objects because they don’t contain a ' - 'reference to their\n' - 'global execution environment. Code objects are ' - 'returned by the built-\n' - 'in "compile()" function and can be extracted from ' - 'function objects\n' - 'through their "__code__" attribute. See also the ' - '"code" module.\n' - '\n' - 'A code object can be executed or evaluated by passing ' - 'it (instead of a\n' - 'source string) to the "exec()" or "eval()" built-in ' - 'functions.\n' - '\n' - 'See The standard type hierarchy for more ' - 'information.\n', - 'bltin-ellipsis-object': 'The Ellipsis Object\n' - '*******************\n' - '\n' - 'This object is commonly used by slicing (see ' - 'Slicings). It supports\n' - 'no special operations. There is exactly one ' - 'ellipsis object, named\n' - '"Ellipsis" (a built-in name). "type(Ellipsis)()" ' - 'produces the\n' - '"Ellipsis" singleton.\n' - '\n' - 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': 'The Null Object\n' - '***************\n' - '\n' - 'This object is returned by functions that don’t ' - 'explicitly return a\n' - 'value. It supports no special operations. There is ' - 'exactly one null\n' - 'object, named "None" (a built-in name). "type(None)()" ' - 'produces the\n' - 'same singleton.\n' - '\n' - 'It is written as "None".\n', - 'bltin-type-objects': 'Type Objects\n' - '************\n' - '\n' - 'Type objects represent the various object types. An ' - 'object’s type is\n' - 'accessed by the built-in function "type()". There are ' - 'no special\n' - 'operations on types. The standard module "types" ' - 'defines names for\n' - 'all standard built-in types.\n' - '\n' - 'Types are written like this: "".\n', - 'booleans': 'Boolean operations\n' - '******************\n' - '\n' - ' or_test ::= and_test | or_test "or" and_test\n' - ' and_test ::= not_test | and_test "and" not_test\n' - ' not_test ::= comparison | "not" not_test\n' - '\n' - 'In the context of Boolean operations, and also when expressions ' - 'are\n' - 'used by control flow statements, the following values are ' - 'interpreted\n' - 'as false: "False", "None", numeric zero of all types, and empty\n' - 'strings and containers (including strings, tuples, lists,\n' - 'dictionaries, sets and frozensets). All other values are ' - 'interpreted\n' - 'as true. User-defined objects can customize their truth value ' - 'by\n' - 'providing a "__bool__()" method.\n' - '\n' - 'The operator "not" yields "True" if its argument is false, ' - '"False"\n' - 'otherwise.\n' - '\n' - 'The expression "x and y" first evaluates *x*; if *x* is false, ' - 'its\n' - 'value is returned; otherwise, *y* is evaluated and the resulting ' - 'value\n' - 'is returned.\n' - '\n' - 'The expression "x or y" first evaluates *x*; if *x* is true, its ' - 'value\n' - 'is returned; otherwise, *y* is evaluated and the resulting value ' - 'is\n' - 'returned.\n' - '\n' - 'Note that neither "and" nor "or" restrict the value and type ' - 'they\n' - 'return to "False" and "True", but rather return the last ' - 'evaluated\n' - 'argument. This is sometimes useful, e.g., if "s" is a string ' - 'that\n' - 'should be replaced by a default value if it is empty, the ' - 'expression\n' - '"s or \'foo\'" yields the desired value. Because "not" has to ' - 'create a\n' - 'new value, it returns a boolean value regardless of the type of ' - 'its\n' - 'argument (for example, "not \'foo\'" produces "False" rather ' - 'than "\'\'".)\n', - 'break': 'The "break" statement\n' - '*********************\n' - '\n' - ' break_stmt ::= "break"\n' - '\n' - '"break" may only occur syntactically nested in a "for" or "while"\n' - 'loop, but not nested in a function or class definition within that\n' - 'loop.\n' - '\n' - 'It terminates the nearest enclosing loop, skipping the optional ' - '"else"\n' - 'clause if the loop has one.\n' - '\n' - 'If a "for" loop is terminated by "break", the loop control target\n' - 'keeps its current value.\n' - '\n' - 'When "break" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'loop.\n', - 'callable-types': 'Emulating callable objects\n' - '**************************\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' - ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': 'Calls\n' - '*****\n' - '\n' - 'A call calls a callable object (e.g., a *function*) with a ' - 'possibly\n' - 'empty series of *arguments*:\n' - '\n' - ' call ::= primary "(" [argument_list [","] | ' - 'comprehension] ")"\n' - ' argument_list ::= positional_arguments ["," ' - 'starred_and_keywords]\n' - ' ["," keywords_arguments]\n' - ' | starred_and_keywords ["," ' - 'keywords_arguments]\n' - ' | keywords_arguments\n' - ' positional_arguments ::= ["*"] expression ("," ["*"] ' - 'expression)*\n' - ' starred_and_keywords ::= ("*" expression | keyword_item)\n' - ' ("," "*" expression | "," ' - 'keyword_item)*\n' - ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "," "**" ' - 'expression)*\n' - ' keyword_item ::= identifier "=" expression\n' - '\n' - 'An optional trailing comma may be present after the positional and\n' - 'keyword arguments but does not affect the semantics.\n' - '\n' - 'The primary must evaluate to a callable object (user-defined\n' - 'functions, built-in functions, methods of built-in objects, class\n' - 'objects, methods of class instances, and all objects having a\n' - '"__call__()" method are callable). All argument expressions are\n' - 'evaluated before the call is attempted. Please refer to section\n' - 'Function definitions for the syntax of formal *parameter* lists.\n' - '\n' - 'If keyword arguments are present, they are first converted to\n' - 'positional arguments, as follows. First, a list of unfilled slots ' - 'is\n' - 'created for the formal parameters. If there are N positional\n' - 'arguments, they are placed in the first N slots. Next, for each\n' - 'keyword argument, the identifier is used to determine the\n' - 'corresponding slot (if the identifier is the same as the first ' - 'formal\n' - 'parameter name, the first slot is used, and so on). If the slot ' - 'is\n' - 'already filled, a "TypeError" exception is raised. Otherwise, the\n' - 'value of the argument is placed in the slot, filling it (even if ' - 'the\n' - 'expression is "None", it fills the slot). When all arguments have\n' - 'been processed, the slots that are still unfilled are filled with ' - 'the\n' - 'corresponding default value from the function definition. ' - '(Default\n' - 'values are calculated, once, when the function is defined; thus, a\n' - 'mutable object such as a list or dictionary used as default value ' - 'will\n' - 'be shared by all calls that don’t specify an argument value for ' - 'the\n' - 'corresponding slot; this should usually be avoided.) If there are ' - 'any\n' - 'unfilled slots for which no default value is specified, a ' - '"TypeError"\n' - 'exception is raised. Otherwise, the list of filled slots is used ' - 'as\n' - 'the argument list for the call.\n' - '\n' - '**CPython implementation detail:** An implementation may provide\n' - 'built-in functions whose positional parameters do not have names, ' - 'even\n' - 'if they are ‘named’ for the purpose of documentation, and which\n' - 'therefore cannot be supplied by keyword. In CPython, this is the ' - 'case\n' - 'for functions implemented in C that use "PyArg_ParseTuple()" to ' - 'parse\n' - 'their arguments.\n' - '\n' - 'If there are more positional arguments than there are formal ' - 'parameter\n' - 'slots, a "TypeError" exception is raised, unless a formal ' - 'parameter\n' - 'using the syntax "*identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a tuple containing the excess positional ' - 'arguments\n' - '(or an empty tuple if there were no excess positional arguments).\n' - '\n' - 'If any keyword argument does not correspond to a formal parameter\n' - 'name, a "TypeError" exception is raised, unless a formal parameter\n' - 'using the syntax "**identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a dictionary containing the excess keyword\n' - 'arguments (using the keywords as keys and the argument values as\n' - 'corresponding values), or a (new) empty dictionary if there were ' - 'no\n' - 'excess keyword arguments.\n' - '\n' - 'If the syntax "*expression" appears in the function call, ' - '"expression"\n' - 'must evaluate to an *iterable*. Elements from these iterables are\n' - 'treated as if they were additional positional arguments. For the ' - 'call\n' - '"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, ' - '*yM*,\n' - 'this is equivalent to a call with M+4 positional arguments *x1*, ' - '*x2*,\n' - '*y1*, …, *yM*, *x3*, *x4*.\n' - '\n' - 'A consequence of this is that although the "*expression" syntax ' - 'may\n' - 'appear *after* explicit keyword arguments, it is processed ' - '*before*\n' - 'the keyword arguments (and any "**expression" arguments – see ' - 'below).\n' - 'So:\n' - '\n' - ' >>> def f(a, b):\n' - ' ... print(a, b)\n' - ' ...\n' - ' >>> f(b=1, *(2,))\n' - ' 2 1\n' - ' >>> f(a=1, *(2,))\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: f() got multiple values for keyword argument 'a'\n" - ' >>> f(1, *(2,))\n' - ' 1 2\n' - '\n' - 'It is unusual for both keyword arguments and the "*expression" ' - 'syntax\n' - 'to be used in the same call, so in practice this confusion does ' - 'not\n' - 'arise.\n' - '\n' - 'If the syntax "**expression" appears in the function call,\n' - '"expression" must evaluate to a *mapping*, the contents of which ' - 'are\n' - 'treated as additional keyword arguments. If a keyword is already\n' - 'present (as an explicit keyword argument, or from another ' - 'unpacking),\n' - 'a "TypeError" exception is raised.\n' - '\n' - 'Formal parameters using the syntax "*identifier" or "**identifier"\n' - 'cannot be used as positional argument slots or as keyword argument\n' - 'names.\n' - '\n' - 'Changed in version 3.5: Function calls accept any number of "*" ' - 'and\n' - '"**" unpackings, positional arguments may follow iterable ' - 'unpackings\n' - '("*"), and keyword arguments may follow dictionary unpackings ' - '("**").\n' - 'Originally proposed by **PEP 448**.\n' - '\n' - 'A call always returns some value, possibly "None", unless it raises ' - 'an\n' - 'exception. How this value is computed depends on the type of the\n' - 'callable object.\n' - '\n' - 'If it is—\n' - '\n' - 'a user-defined function:\n' - ' The code block for the function is executed, passing it the\n' - ' argument list. The first thing the code block will do is bind ' - 'the\n' - ' formal parameters to the arguments; this is described in ' - 'section\n' - ' Function definitions. When the code block executes a "return"\n' - ' statement, this specifies the return value of the function ' - 'call.\n' - '\n' - 'a built-in function or method:\n' - ' The result is up to the interpreter; see Built-in Functions for ' - 'the\n' - ' descriptions of built-in functions and methods.\n' - '\n' - 'a class object:\n' - ' A new instance of that class is returned.\n' - '\n' - 'a class instance method:\n' - ' The corresponding user-defined function is called, with an ' - 'argument\n' - ' list that is one longer than the argument list of the call: the\n' - ' instance becomes the first argument.\n' - '\n' - 'a class instance:\n' - ' The class must define a "__call__()" method; the effect is then ' - 'the\n' - ' same as if that method was called.\n', - 'class': 'Class definitions\n' - '*****************\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [inheritance] ":" ' - 'suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame (see\n' - 'Naming and binding), using a newly created local namespace and the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes execution, ' - 'its\n' - 'execution frame is discarded but its local namespace is saved. [3] ' - 'A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same as ' - 'for\n' - 'function decorators. The result is then bound to the class name.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n', - 'comparisons': 'Comparisons\n' - '***********\n' - '\n' - 'Unlike C, all comparison operations in Python have the same ' - 'priority,\n' - 'which is lower than that of any arithmetic, shifting or ' - 'bitwise\n' - 'operation. Also unlike C, expressions like "a < b < c" have ' - 'the\n' - 'interpretation that is conventional in mathematics:\n' - '\n' - ' comparison ::= or_expr (comp_operator or_expr)*\n' - ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' - ' | "is" ["not"] | ["not"] "in"\n' - '\n' - 'Comparisons yield boolean values: "True" or "False".\n' - '\n' - 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' - 'is\n' - 'equivalent to "x < y and y <= z", except that "y" is ' - 'evaluated only\n' - 'once (but in both cases "z" is not evaluated at all when "x < ' - 'y" is\n' - 'found to be false).\n' - '\n' - 'Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and ' - '*op1*,\n' - '*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ' - '... y\n' - 'opN z" is equivalent to "a op1 b and b op2 c and ... y opN ' - 'z", except\n' - 'that each expression is evaluated at most once.\n' - '\n' - 'Note that "a op1 b op2 c" doesn’t imply any kind of ' - 'comparison between\n' - '*a* and *c*, so that, e.g., "x < y > z" is perfectly legal ' - '(though\n' - 'perhaps not pretty).\n' - '\n' - '\n' - 'Value comparisons\n' - '=================\n' - '\n' - 'The operators "<", ">", "==", ">=", "<=", and "!=" compare ' - 'the values\n' - 'of two objects. The objects do not need to have the same ' - 'type.\n' - '\n' - 'Chapter Objects, values and types states that objects have a ' - 'value (in\n' - 'addition to type and identity). The value of an object is a ' - 'rather\n' - 'abstract notion in Python: For example, there is no canonical ' - 'access\n' - 'method for an object’s value. Also, there is no requirement ' - 'that the\n' - 'value of an object should be constructed in a particular way, ' - 'e.g.\n' - 'comprised of all its data attributes. Comparison operators ' - 'implement a\n' - 'particular notion of what the value of an object is. One can ' - 'think of\n' - 'them as defining the value of an object indirectly, by means ' - 'of their\n' - 'comparison implementation.\n' - '\n' - 'Because all types are (direct or indirect) subtypes of ' - '"object", they\n' - 'inherit the default comparison behavior from "object". Types ' - 'can\n' - 'customize their comparison behavior by implementing *rich ' - 'comparison\n' - 'methods* like "__lt__()", described in Basic customization.\n' - '\n' - 'The default behavior for equality comparison ("==" and "!=") ' - 'is based\n' - 'on the identity of the objects. Hence, equality comparison ' - 'of\n' - 'instances with the same identity results in equality, and ' - 'equality\n' - 'comparison of instances with different identities results in\n' - 'inequality. A motivation for this default behavior is the ' - 'desire that\n' - 'all objects should be reflexive (i.e. "x is y" implies "x == ' - 'y").\n' - '\n' - 'A default order comparison ("<", ">", "<=", and ">=") is not ' - 'provided;\n' - 'an attempt raises "TypeError". A motivation for this default ' - 'behavior\n' - 'is the lack of a similar invariant as for equality.\n' - '\n' - 'The behavior of the default equality comparison, that ' - 'instances with\n' - 'different identities are always unequal, may be in contrast ' - 'to what\n' - 'types will need that have a sensible definition of object ' - 'value and\n' - 'value-based equality. Such types will need to customize ' - 'their\n' - 'comparison behavior, and in fact, a number of built-in types ' - 'have done\n' - 'that.\n' - '\n' - 'The following list describes the comparison behavior of the ' - 'most\n' - 'important built-in types.\n' - '\n' - '* Numbers of built-in numeric types (Numeric Types — int, ' - 'float,\n' - ' complex) and of the standard library types ' - '"fractions.Fraction" and\n' - ' "decimal.Decimal" can be compared within and across their ' - 'types,\n' - ' with the restriction that complex numbers do not support ' - 'order\n' - ' comparison. Within the limits of the types involved, they ' - 'compare\n' - ' mathematically (algorithmically) correct without loss of ' - 'precision.\n' - '\n' - ' The not-a-number values "float(\'NaN\')" and ' - '"Decimal(\'NaN\')" are\n' - ' special. They are identical to themselves ("x is x" is ' - 'true) but\n' - ' are not equal to themselves ("x == x" is false). ' - 'Additionally,\n' - ' comparing any number to a not-a-number value will return ' - '"False".\n' - ' For example, both "3 < float(\'NaN\')" and "float(\'NaN\') ' - '< 3" will\n' - ' return "False".\n' - '\n' - '* Binary sequences (instances of "bytes" or "bytearray") can ' - 'be\n' - ' compared within and across their types. They compare\n' - ' lexicographically using the numeric values of their ' - 'elements.\n' - '\n' - '* Strings (instances of "str") compare lexicographically ' - 'using the\n' - ' numerical Unicode code points (the result of the built-in ' - 'function\n' - ' "ord()") of their characters. [3]\n' - '\n' - ' Strings and binary sequences cannot be directly compared.\n' - '\n' - '* Sequences (instances of "tuple", "list", or "range") can ' - 'be\n' - ' compared only within each of their types, with the ' - 'restriction that\n' - ' ranges do not support order comparison. Equality ' - 'comparison across\n' - ' these types results in inequality, and ordering comparison ' - 'across\n' - ' these types raises "TypeError".\n' - '\n' - ' Sequences compare lexicographically using comparison of\n' - ' corresponding elements, whereby reflexivity of the elements ' - 'is\n' - ' enforced.\n' - '\n' - ' In enforcing reflexivity of elements, the comparison of ' - 'collections\n' - ' assumes that for a collection element "x", "x == x" is ' - 'always true.\n' - ' Based on that assumption, element identity is compared ' - 'first, and\n' - ' element comparison is performed only for distinct ' - 'elements. This\n' - ' approach yields the same result as a strict element ' - 'comparison\n' - ' would, if the compared elements are reflexive. For ' - 'non-reflexive\n' - ' elements, the result is different than for strict element\n' - ' comparison, and may be surprising: The non-reflexive ' - 'not-a-number\n' - ' values for example result in the following comparison ' - 'behavior when\n' - ' used in a list:\n' - '\n' - " >>> nan = float('NaN')\n" - ' >>> nan is nan\n' - ' True\n' - ' >>> nan == nan\n' - ' False <-- the defined non-reflexive ' - 'behavior of NaN\n' - ' >>> [nan] == [nan]\n' - ' True <-- list enforces reflexivity and ' - 'tests identity first\n' - '\n' - ' Lexicographical comparison between built-in collections ' - 'works as\n' - ' follows:\n' - '\n' - ' * For two collections to compare equal, they must be of the ' - 'same\n' - ' type, have the same length, and each pair of ' - 'corresponding\n' - ' elements must compare equal (for example, "[1,2] == ' - '(1,2)" is\n' - ' false because the type is not the same).\n' - '\n' - ' * Collections that support order comparison are ordered the ' - 'same\n' - ' as their first unequal elements (for example, "[1,2,x] <= ' - '[1,2,y]"\n' - ' has the same value as "x <= y"). If a corresponding ' - 'element does\n' - ' not exist, the shorter collection is ordered first (for ' - 'example,\n' - ' "[1,2] < [1,2,3]" is true).\n' - '\n' - '* Mappings (instances of "dict") compare equal if and only if ' - 'they\n' - ' have equal *(key, value)* pairs. Equality comparison of the ' - 'keys and\n' - ' values enforces reflexivity.\n' - '\n' - ' Order comparisons ("<", ">", "<=", and ">=") raise ' - '"TypeError".\n' - '\n' - '* Sets (instances of "set" or "frozenset") can be compared ' - 'within\n' - ' and across their types.\n' - '\n' - ' They define order comparison operators to mean subset and ' - 'superset\n' - ' tests. Those relations do not define total orderings (for ' - 'example,\n' - ' the two sets "{1,2}" and "{2,3}" are not equal, nor subsets ' - 'of one\n' - ' another, nor supersets of one another). Accordingly, sets ' - 'are not\n' - ' appropriate arguments for functions which depend on total ' - 'ordering\n' - ' (for example, "min()", "max()", and "sorted()" produce ' - 'undefined\n' - ' results given a list of sets as inputs).\n' - '\n' - ' Comparison of sets enforces reflexivity of its elements.\n' - '\n' - '* Most other built-in types have no comparison methods ' - 'implemented,\n' - ' so they inherit the default comparison behavior.\n' - '\n' - 'User-defined classes that customize their comparison behavior ' - 'should\n' - 'follow some consistency rules, if possible:\n' - '\n' - '* Equality comparison should be reflexive. In other words, ' - 'identical\n' - ' objects should compare equal:\n' - '\n' - ' "x is y" implies "x == y"\n' - '\n' - '* Comparison should be symmetric. In other words, the ' - 'following\n' - ' expressions should have the same result:\n' - '\n' - ' "x == y" and "y == x"\n' - '\n' - ' "x != y" and "y != x"\n' - '\n' - ' "x < y" and "y > x"\n' - '\n' - ' "x <= y" and "y >= x"\n' - '\n' - '* Comparison should be transitive. The following ' - '(non-exhaustive)\n' - ' examples illustrate that:\n' - '\n' - ' "x > y and y > z" implies "x > z"\n' - '\n' - ' "x < y and y <= z" implies "x < z"\n' - '\n' - '* Inverse comparison should result in the boolean negation. ' - 'In other\n' - ' words, the following expressions should have the same ' - 'result:\n' - '\n' - ' "x == y" and "not x != y"\n' - '\n' - ' "x < y" and "not x >= y" (for total ordering)\n' - '\n' - ' "x > y" and "not x <= y" (for total ordering)\n' - '\n' - ' The last two expressions apply to totally ordered ' - 'collections (e.g.\n' - ' to sequences, but not to sets or mappings). See also the\n' - ' "total_ordering()" decorator.\n' - '\n' - '* The "hash()" result should be consistent with equality. ' - 'Objects\n' - ' that are equal should either have the same hash value, or ' - 'be marked\n' - ' as unhashable.\n' - '\n' - 'Python does not enforce these consistency rules. In fact, ' - 'the\n' - 'not-a-number values are an example for not following these ' - 'rules.\n' - '\n' - '\n' - 'Membership test operations\n' - '==========================\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in ' - 's"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" ' - 'otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which ' - '"in" tests\n' - 'whether the dictionary has a given key. For container types ' - 'such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e ' - 'for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and ' - 'only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != ' - '-1".\n' - 'Empty strings are always considered to be a substring of any ' - 'other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" ' - 'method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true ' - 'value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" ' - 'but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z" ' - 'with "x ==\n' - 'z" is produced while iterating over "y". If an exception is ' - 'raised\n' - 'during the iteration, it is as if "in" raised that ' - 'exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class ' - 'defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a ' - 'non-\n' - 'negative integer index *i* such that "x == y[i]", and all ' - 'lower\n' - 'integer indices do not raise "IndexError" exception. (If any ' - 'other\n' - 'exception is raised, it is as if "in" raised that ' - 'exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse true ' - 'value of\n' - '"in".\n' - '\n' - '\n' - 'Identity comparisons\n' - '====================\n' - '\n' - 'The operators "is" and "is not" test for object identity: "x ' - 'is y" is\n' - 'true if and only if *x* and *y* are the same object. Object ' - 'identity\n' - 'is determined using the "id()" function. "x is not y" yields ' - 'the\n' - 'inverse truth value. [4]\n', - 'compound': 'Compound statements\n' - '*******************\n' - '\n' - 'Compound statements contain (groups of) other statements; they ' - 'affect\n' - 'or control the execution of those other statements in some way. ' - 'In\n' - 'general, compound statements span multiple lines, although in ' - 'simple\n' - 'incarnations a whole compound statement may be contained in one ' - 'line.\n' - '\n' - 'The "if", "while" and "for" statements implement traditional ' - 'control\n' - 'flow constructs. "try" specifies exception handlers and/or ' - 'cleanup\n' - 'code for a group of statements, while the "with" statement ' - 'allows the\n' - 'execution of initialization and finalization code around a block ' - 'of\n' - 'code. Function and class definitions are also syntactically ' - 'compound\n' - 'statements.\n' - '\n' - 'A compound statement consists of one or more ‘clauses.’ A ' - 'clause\n' - 'consists of a header and a ‘suite.’ The clause headers of a\n' - 'particular compound statement are all at the same indentation ' - 'level.\n' - 'Each clause header begins with a uniquely identifying keyword ' - 'and ends\n' - 'with a colon. A suite is a group of statements controlled by a\n' - 'clause. A suite can be one or more semicolon-separated simple\n' - 'statements on the same line as the header, following the ' - 'header’s\n' - 'colon, or it can be one or more indented statements on ' - 'subsequent\n' - 'lines. Only the latter form of a suite can contain nested ' - 'compound\n' - 'statements; the following is illegal, mostly because it wouldn’t ' - 'be\n' - 'clear to which "if" clause a following "else" clause would ' - 'belong:\n' - '\n' - ' if test1: if test2: print(x)\n' - '\n' - 'Also note that the semicolon binds tighter than the colon in ' - 'this\n' - 'context, so that in the following example, either all or none of ' - 'the\n' - '"print()" calls are executed:\n' - '\n' - ' if x < y < z: print(x); print(y); print(z)\n' - '\n' - 'Summarizing:\n' - '\n' - ' compound_stmt ::= if_stmt\n' - ' | while_stmt\n' - ' | for_stmt\n' - ' | try_stmt\n' - ' | with_stmt\n' - ' | funcdef\n' - ' | classdef\n' - ' | async_with_stmt\n' - ' | async_for_stmt\n' - ' | async_funcdef\n' - ' suite ::= stmt_list NEWLINE | NEWLINE INDENT ' - 'statement+ DEDENT\n' - ' statement ::= stmt_list NEWLINE | compound_stmt\n' - ' stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n' - '\n' - 'Note that statements always end in a "NEWLINE" possibly followed ' - 'by a\n' - '"DEDENT". Also note that optional continuation clauses always ' - 'begin\n' - 'with a keyword that cannot start a statement, thus there are no\n' - 'ambiguities (the ‘dangling "else"’ problem is solved in Python ' - 'by\n' - 'requiring nested "if" statements to be indented).\n' - '\n' - 'The formatting of the grammar rules in the following sections ' - 'places\n' - 'each clause on a separate line for clarity.\n' - '\n' - '\n' - 'The "if" statement\n' - '==================\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the ' - 'expressions one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is ' - 'executed\n' - '(and no other part of the "if" statement is executed or ' - 'evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, ' - 'if\n' - 'present, is executed.\n' - '\n' - '\n' - 'The "while" statement\n' - '=====================\n' - '\n' - 'The "while" statement is used for repeated execution as long as ' - 'an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, ' - 'executes the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n' - '\n' - '\n' - 'The "for" statement\n' - '===================\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" expression_list ":" ' - 'suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The expression list is evaluated once; it should yield an ' - 'iterable\n' - 'object. An iterator is created for the result of the\n' - '"expression_list". The suite is then executed once for each ' - 'item\n' - 'provided by the iterator, in the order returned by the ' - 'iterator. Each\n' - 'item in turn is assigned to the target list using the standard ' - 'rules\n' - 'for assignments (see Assignment statements), and then the suite ' - 'is\n' - 'executed. When the items are exhausted (which is immediately ' - 'when the\n' - 'sequence is empty or an iterator raises a "StopIteration" ' - 'exception),\n' - 'the suite in the "else" clause, if present, is executed, and the ' - 'loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no ' - 'next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables(s) in the target ' - 'list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with ' - 'the next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is ' - 'finished,\n' - 'but if the sequence is empty, they will not have been assigned ' - 'to at\n' - 'all by the loop. Hint: the built-in function "range()" returns ' - 'an\n' - 'iterator of integers suitable to emulate the effect of Pascal’s ' - '"for i\n' - ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, ' - '2]".\n' - '\n' - 'Note: There is a subtlety when the sequence is being modified by ' - 'the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). ' - 'An\n' - ' internal counter is used to keep track of which item is used ' - 'next,\n' - ' and this is incremented on each iteration. When this counter ' - 'has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item ' - 'from the\n' - ' sequence, the next item will be skipped (since it gets the ' - 'index of\n' - ' the current item which has already been treated). Likewise, ' - 'if the\n' - ' suite inserts an item in the sequence before the current item, ' - 'the\n' - ' current item will be treated again the next time through the ' - 'loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n' - '\n' - '\n' - 'The "try" statement\n' - '===================\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup ' - 'code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. ' - 'When no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search ' - 'for an\n' - 'exception handler is started. This search inspects the except ' - 'clauses\n' - 'in turn until one is found that matches the exception. An ' - 'expression-\n' - 'less except clause, if present, must be last; it matches any\n' - 'exception. For an except clause with an expression, that ' - 'expression\n' - 'is evaluated, and the clause matches the exception if the ' - 'resulting\n' - 'object is “compatible” with the exception. An object is ' - 'compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object or a tuple containing an item compatible with the ' - 'exception.\n' - '\n' - 'If no except clause matches the exception, the search for an ' - 'exception\n' - 'handler continues in the surrounding code and on the invocation ' - 'stack.\n' - '[1]\n' - '\n' - 'If the evaluation of an expression in the header of an except ' - 'clause\n' - 'raises an exception, the original search for a handler is ' - 'canceled and\n' - 'a search starts for the new exception in the surrounding code ' - 'and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching except clause is found, the exception is ' - 'assigned to\n' - 'the target specified after the "as" keyword in that except ' - 'clause, if\n' - 'present, and the except clause’s suite is executed. All except\n' - 'clauses must have an executable block. When the end of this ' - 'block is\n' - 'reached, execution continues normally after the entire try ' - 'statement.\n' - '(This means that if two nested handlers exist for the same ' - 'exception,\n' - 'and the exception occurs in the try clause of the inner handler, ' - 'the\n' - 'outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is ' - 'cleared\n' - 'at the end of the except clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to ' - 'be\n' - 'able to refer to it after the except clause. Exceptions are ' - 'cleared\n' - 'because with the traceback attached to them, they form a ' - 'reference\n' - 'cycle with the stack frame, keeping all locals in that frame ' - 'alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an except clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed ' - 'via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' - 'of the\n' - 'exception class, the exception instance and a traceback object ' - '(see\n' - 'section The standard type hierarchy) identifying the point in ' - 'the\n' - 'program where the exception occurred. "sys.exc_info()" values ' - 'are\n' - 'restored to their previous values (before the call) when ' - 'returning\n' - 'from a function that handled an exception.\n' - '\n' - 'The optional "else" clause is executed if the control flow ' - 'leaves the\n' - '"try" suite, no exception was raised, and no "return", ' - '"continue", or\n' - '"break" statement was executed. Exceptions in the "else" clause ' - 'are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. ' - 'If an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is ' - 'executed. If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the ' - 'saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return" or "break" statement, the saved ' - 'exception\n' - 'is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program ' - 'during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in ' - 'the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause ' - 'is\n' - 'also executed ‘on the way out.’ A "continue" statement is ' - 'illegal in\n' - 'the "finally" clause. (The reason is a problem with the current\n' - 'implementation — this restriction may be lifted in the future).\n' - '\n' - 'The return value of a function is determined by the last ' - '"return"\n' - 'statement executed. Since the "finally" clause always executes, ' - 'a\n' - '"return" statement executed in the "finally" clause will always ' - 'be the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n' - '\n' - '\n' - 'The "with" statement\n' - '====================\n' - '\n' - 'The "with" statement is used to wrap the execution of a block ' - 'with\n' - 'methods defined by a context manager (see section With ' - 'Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds ' - 'as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the ' - '"with_item")\n' - ' is evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' - '\n' - ' Note: The "with" statement guarantees that if the ' - '"__enter__()"\n' - ' method returns without an error, then "__exit__()" will ' - 'always be\n' - ' called. Thus, if an error occurs during the assignment to ' - 'the\n' - ' target list, it will be treated the same as an error ' - 'occurring\n' - ' within the suite would be. See step 6 below.\n' - '\n' - '5. The suite is executed.\n' - '\n' - '6. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, ' - 'and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, ' - 'and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an ' - 'exception, the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'With more than one item, the context managers are processed as ' - 'if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' suite\n' - '\n' - 'is equivalent to\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' suite\n' - '\n' - 'Changed in version 3.1: Support for multiple context ' - 'expressions.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n' - '\n' - '\n' - 'Function definitions\n' - '====================\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" dotted_name ["(" ' - '[argument_list [","]] ")"] NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' - ' parameter_list ::= defparameter ("," defparameter)* ' - '["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [2]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from position arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' - '\n' - 'Parameters may have annotations of the form “": expression"” ' - 'following\n' - 'the parameter name. Any parameter may have an annotation even ' - 'those\n' - 'of the form "*identifier" or "**identifier". Functions may ' - 'have\n' - '“return” annotation of the form “"-> expression"” after the ' - 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n' - '\n' - '\n' - 'Class definitions\n' - '=================\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [inheritance] ' - '":" suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame ' - '(see\n' - 'Naming and binding), using a newly created local namespace and ' - 'the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes ' - 'execution, its\n' - 'execution frame is discarded but its local namespace is saved. ' - '[3] A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original ' - 'local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same ' - 'as for\n' - 'function decorators. The result is then bound to the class ' - 'name.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class ' - 'and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n' - '\n' - '\n' - 'Coroutines\n' - '==========\n' - '\n' - 'New in version 3.5.\n' - '\n' - '\n' - 'Coroutine function definition\n' - '-----------------------------\n' - '\n' - ' async_funcdef ::= [decorators] "async" "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - '\n' - 'Execution of Python coroutines can be suspended and resumed at ' - 'many\n' - 'points (see *coroutine*). In the body of a coroutine, any ' - '"await" and\n' - '"async" identifiers become reserved keywords; "await" ' - 'expressions,\n' - '"async for" and "async with" can only be used in coroutine ' - 'bodies.\n' - '\n' - 'Functions defined with "async def" syntax are always coroutine\n' - 'functions, even if they do not contain "await" or "async" ' - 'keywords.\n' - '\n' - 'It is a "SyntaxError" to use "yield from" expressions in "async ' - 'def"\n' - 'coroutines.\n' - '\n' - 'An example of a coroutine function:\n' - '\n' - ' async def func(param1, param2):\n' - ' do_stuff()\n' - ' await some_coroutine()\n' - '\n' - '\n' - 'The "async for" statement\n' - '-------------------------\n' - '\n' - ' async_for_stmt ::= "async" for_stmt\n' - '\n' - 'An *asynchronous iterable* is able to call asynchronous code in ' - 'its\n' - '*iter* implementation, and *asynchronous iterator* can call\n' - 'asynchronous code in its *next* method.\n' - '\n' - 'The "async for" statement allows convenient iteration over\n' - 'asynchronous iterators.\n' - '\n' - 'The following code:\n' - '\n' - ' async for TARGET in ITER:\n' - ' BLOCK\n' - ' else:\n' - ' BLOCK2\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' iter = (ITER)\n' - ' iter = type(iter).__aiter__(iter)\n' - ' running = True\n' - ' while running:\n' - ' try:\n' - ' TARGET = await type(iter).__anext__(iter)\n' - ' except StopAsyncIteration:\n' - ' running = False\n' - ' else:\n' - ' BLOCK\n' - ' else:\n' - ' BLOCK2\n' - '\n' - 'See also "__aiter__()" and "__anext__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use "async for" statement outside of ' - 'an\n' - '"async def" function.\n' - '\n' - '\n' - 'The "async with" statement\n' - '--------------------------\n' - '\n' - ' async_with_stmt ::= "async" with_stmt\n' - '\n' - 'An *asynchronous context manager* is a *context manager* that is ' - 'able\n' - 'to suspend execution in its *enter* and *exit* methods.\n' - '\n' - 'The following code:\n' - '\n' - ' async with EXPR as VAR:\n' - ' BLOCK\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' mgr = (EXPR)\n' - ' aexit = type(mgr).__aexit__\n' - ' aenter = type(mgr).__aenter__(mgr)\n' - '\n' - ' VAR = await aenter\n' - ' try:\n' - ' BLOCK\n' - ' except:\n' - ' if not await aexit(mgr, *sys.exc_info()):\n' - ' raise\n' - ' else:\n' - ' await aexit(mgr, None, None, None)\n' - '\n' - 'See also "__aenter__()" and "__aexit__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use "async with" statement outside of ' - 'an\n' - '"async def" function.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 492** - Coroutines with async and await syntax\n' - ' The proposal that made coroutines a proper standalone ' - 'concept in\n' - ' Python, and added supporting syntax.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] The exception is propagated to the invocation stack unless\n' - ' there is a "finally" clause which happens to raise another\n' - ' exception. That new exception causes the old one to be ' - 'lost.\n' - '\n' - '[2] A string literal appearing as the first statement in the\n' - ' function body is transformed into the function’s "__doc__"\n' - ' attribute and therefore the function’s *docstring*.\n' - '\n' - '[3] A string literal appearing as the first statement in the ' - 'class\n' - ' body is transformed into the namespace’s "__doc__" item and\n' - ' therefore the class’s *docstring*.\n', - 'context-managers': 'With Statement Context Managers\n' - '*******************************\n' - '\n' - 'A *context manager* is an object that defines the ' - 'runtime context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context ' - 'managers are normally\n' - 'invoked using the "with" statement (described in section ' - 'The with\n' - 'statement), but can also be used by directly invoking ' - 'their methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if ' - 'any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), ' - 'it should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n', - 'continue': 'The "continue" statement\n' - '************************\n' - '\n' - ' continue_stmt ::= "continue"\n' - '\n' - '"continue" may only occur syntactically nested in a "for" or ' - '"while"\n' - 'loop, but not nested in a function or class definition or ' - '"finally"\n' - 'clause within that loop. It continues with the next cycle of ' - 'the\n' - 'nearest enclosing loop.\n' - '\n' - 'When "continue" passes control out of a "try" statement with a\n' - '"finally" clause, that "finally" clause is executed before ' - 'really\n' - 'starting the next loop cycle.\n', - 'conversions': 'Arithmetic conversions\n' - '**********************\n' - '\n' - 'When a description of an arithmetic operator below uses the ' - 'phrase\n' - '“the numeric arguments are converted to a common type,” this ' - 'means\n' - 'that the operator implementation for built-in types works as ' - 'follows:\n' - '\n' - '* If either argument is a complex number, the other is ' - 'converted to\n' - ' complex;\n' - '\n' - '* otherwise, if either argument is a floating point number, ' - 'the\n' - ' other is converted to floating point;\n' - '\n' - '* otherwise, both must be integers and no conversion is ' - 'necessary.\n' - '\n' - 'Some additional rules apply for certain operators (e.g., a ' - 'string as a\n' - 'left argument to the ‘%’ operator). Extensions must define ' - 'their own\n' - 'conversion behavior.\n', - 'customization': 'Basic customization\n' - '*******************\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those ' - 'passed to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly-created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" returns an instance of *cls*, then the ' - 'new\n' - ' instance’s "__init__()" method will be invoked like\n' - ' "__init__(self[, ...])", where *self* is the new ' - 'instance and the\n' - ' remaining arguments are the same as were passed to ' - '"__new__()".\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, ' - 'then the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by ' - '"__init__()"; doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s ' - '"__del__()" method,\n' - ' if any, must explicitly call it to ensure proper ' - 'deletion of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a ' - 'new reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected ' - 'and deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning: Due to the precarious circumstances under ' - 'which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. ' - 'The return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation ' - 'of an\n' - ' object. The "format_spec" argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the "format_spec" argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of ' - '"object" itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are ' - 'returned for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "__ne__()" delegates to "__eq__()" and ' - 'inverts the\n' - ' result unless it is "NotImplemented". There are no ' - 'other implied\n' - ' relationships among the comparison operators, for ' - 'example, the\n' - ' truth of "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, collections.Hashable)" ' - 'call.\n' - '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' - ' Although they remain constant within an individual ' - 'Python\n' - ' process, they are not predictable between repeated ' - 'invocations of\n' - ' Python.This is intended to provide protection against ' - 'a denial-\n' - ' of-service caused by carefully-chosen inputs that ' - 'exploit the\n' - ' worst case performance of a dict insertion, O(n^2) ' - 'complexity.\n' - ' See ' - 'http://www.ocert.org/advisories/ocert-2011-003.html for\n' - ' details.Changing hash values affects the iteration ' - 'order of\n' - ' dicts, sets and other mappings. Python has never made ' - 'guarantees\n' - ' about this ordering (and it typically varies between ' - '32-bit and\n' - ' 64-bit builds).See also "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and ' - 'the object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n', - 'debugger': '"pdb" — The Python Debugger\n' - '***************************\n' - '\n' - '**Source code:** Lib/pdb.py\n' - '\n' - '======================================================================\n' - '\n' - 'The module "pdb" defines an interactive source code debugger ' - 'for\n' - 'Python programs. It supports setting (conditional) breakpoints ' - 'and\n' - 'single stepping at the source line level, inspection of stack ' - 'frames,\n' - 'source code listing, and evaluation of arbitrary Python code in ' - 'the\n' - 'context of any stack frame. It also supports post-mortem ' - 'debugging\n' - 'and can be called under program control.\n' - '\n' - 'The debugger is extensible – it is actually defined as the ' - 'class\n' - '"Pdb". This is currently undocumented but easily understood by ' - 'reading\n' - 'the source. The extension interface uses the modules "bdb" and ' - '"cmd".\n' - '\n' - 'The debugger’s prompt is "(Pdb)". Typical usage to run a program ' - 'under\n' - 'control of the debugger is:\n' - '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - " >>> pdb.run('mymodule.test()')\n" - ' > (0)?()\n' - ' (Pdb) continue\n' - ' > (1)?()\n' - ' (Pdb) continue\n' - " NameError: 'spam'\n" - ' > (1)?()\n' - ' (Pdb)\n' - '\n' - 'Changed in version 3.3: Tab-completion via the "readline" module ' - 'is\n' - 'available for commands and command arguments, e.g. the current ' - 'global\n' - 'and local names are offered as arguments of the "p" command.\n' - '\n' - '"pdb.py" can also be invoked as a script to debug other ' - 'scripts. For\n' - 'example:\n' - '\n' - ' python3 -m pdb myscript.py\n' - '\n' - 'When invoked as a script, pdb will automatically enter ' - 'post-mortem\n' - 'debugging if the program being debugged exits abnormally. After ' - 'post-\n' - 'mortem debugging (or after normal exit of the program), pdb ' - 'will\n' - 'restart the program. Automatic restarting preserves pdb’s state ' - '(such\n' - 'as breakpoints) and in most cases is more useful than quitting ' - 'the\n' - 'debugger upon program’s exit.\n' - '\n' - 'New in version 3.2: "pdb.py" now accepts a "-c" option that ' - 'executes\n' - 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' - '\n' - 'The typical usage to break into the debugger from a running ' - 'program is\n' - 'to insert\n' - '\n' - ' import pdb; pdb.set_trace()\n' - '\n' - 'at the location you want to break into the debugger. You can ' - 'then\n' - 'step through the code following this statement, and continue ' - 'running\n' - 'without the debugger using the "continue" command.\n' - '\n' - 'The typical usage to inspect a crashed program is:\n' - '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - ' >>> mymodule.test()\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - ' File "./mymodule.py", line 4, in test\n' - ' test2()\n' - ' File "./mymodule.py", line 3, in test2\n' - ' print(spam)\n' - ' NameError: spam\n' - ' >>> pdb.pm()\n' - ' > ./mymodule.py(3)test2()\n' - ' -> print(spam)\n' - ' (Pdb)\n' - '\n' - 'The module defines the following functions; each enters the ' - 'debugger\n' - 'in a slightly different way:\n' - '\n' - 'pdb.run(statement, globals=None, locals=None)\n' - '\n' - ' Execute the *statement* (given as a string or a code object) ' - 'under\n' - ' debugger control. The debugger prompt appears before any ' - 'code is\n' - ' executed; you can set breakpoints and type "continue", or you ' - 'can\n' - ' step through the statement using "step" or "next" (all these\n' - ' commands are explained below). The optional *globals* and ' - '*locals*\n' - ' arguments specify the environment in which the code is ' - 'executed; by\n' - ' default the dictionary of the module "__main__" is used. ' - '(See the\n' - ' explanation of the built-in "exec()" or "eval()" functions.)\n' - '\n' - 'pdb.runeval(expression, globals=None, locals=None)\n' - '\n' - ' Evaluate the *expression* (given as a string or a code ' - 'object)\n' - ' under debugger control. When "runeval()" returns, it returns ' - 'the\n' - ' value of the expression. Otherwise this function is similar ' - 'to\n' - ' "run()".\n' - '\n' - 'pdb.runcall(function, *args, **kwds)\n' - '\n' - ' Call the *function* (a function or method object, not a ' - 'string)\n' - ' with the given arguments. When "runcall()" returns, it ' - 'returns\n' - ' whatever the function call returned. The debugger prompt ' - 'appears\n' - ' as soon as the function is entered.\n' - '\n' - 'pdb.set_trace()\n' - '\n' - ' Enter the debugger at the calling stack frame. This is ' - 'useful to\n' - ' hard-code a breakpoint at a given point in a program, even if ' - 'the\n' - ' code is not otherwise being debugged (e.g. when an assertion\n' - ' fails).\n' - '\n' - 'pdb.post_mortem(traceback=None)\n' - '\n' - ' Enter post-mortem debugging of the given *traceback* object. ' - 'If no\n' - ' *traceback* is given, it uses the one of the exception that ' - 'is\n' - ' currently being handled (an exception must be being handled ' - 'if the\n' - ' default is to be used).\n' - '\n' - 'pdb.pm()\n' - '\n' - ' Enter post-mortem debugging of the traceback found in\n' - ' "sys.last_traceback".\n' - '\n' - 'The "run*" functions and "set_trace()" are aliases for ' - 'instantiating\n' - 'the "Pdb" class and calling the method of the same name. If you ' - 'want\n' - 'to access further features, you have to do this yourself:\n' - '\n' - "class pdb.Pdb(completekey='tab', stdin=None, stdout=None, " - 'skip=None, nosigint=False, readrc=True)\n' - '\n' - ' "Pdb" is the debugger class.\n' - '\n' - ' The *completekey*, *stdin* and *stdout* arguments are passed ' - 'to the\n' - ' underlying "cmd.Cmd" class; see the description there.\n' - '\n' - ' The *skip* argument, if given, must be an iterable of ' - 'glob-style\n' - ' module name patterns. The debugger will not step into frames ' - 'that\n' - ' originate in a module that matches one of these patterns. ' - '[1]\n' - '\n' - ' By default, Pdb sets a handler for the SIGINT signal (which ' - 'is sent\n' - ' when the user presses "Ctrl-C" on the console) when you give ' - 'a\n' - ' "continue" command. This allows you to break into the ' - 'debugger\n' - ' again by pressing "Ctrl-C". If you want Pdb not to touch ' - 'the\n' - ' SIGINT handler, set *nosigint* to true.\n' - '\n' - ' The *readrc* argument defaults to true and controls whether ' - 'Pdb\n' - ' will load .pdbrc files from the filesystem.\n' - '\n' - ' Example call to enable tracing with *skip*:\n' - '\n' - " import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n" - '\n' - ' New in version 3.1: The *skip* argument.\n' - '\n' - ' New in version 3.2: The *nosigint* argument. Previously, a ' - 'SIGINT\n' - ' handler was never set by Pdb.\n' - '\n' - ' Changed in version 3.6: The *readrc* argument.\n' - '\n' - ' run(statement, globals=None, locals=None)\n' - ' runeval(expression, globals=None, locals=None)\n' - ' runcall(function, *args, **kwds)\n' - ' set_trace()\n' - '\n' - ' See the documentation for the functions explained above.\n' - '\n' - '\n' - 'Debugger Commands\n' - '=================\n' - '\n' - 'The commands recognized by the debugger are listed below. Most\n' - 'commands can be abbreviated to one or two letters as indicated; ' - 'e.g.\n' - '"h(elp)" means that either "h" or "help" can be used to enter ' - 'the help\n' - 'command (but not "he" or "hel", nor "H" or "Help" or "HELP").\n' - 'Arguments to commands must be separated by whitespace (spaces ' - 'or\n' - 'tabs). Optional arguments are enclosed in square brackets ' - '("[]") in\n' - 'the command syntax; the square brackets must not be typed.\n' - 'Alternatives in the command syntax are separated by a vertical ' - 'bar\n' - '("|").\n' - '\n' - 'Entering a blank line repeats the last command entered. ' - 'Exception: if\n' - 'the last command was a "list" command, the next 11 lines are ' - 'listed.\n' - '\n' - 'Commands that the debugger doesn’t recognize are assumed to be ' - 'Python\n' - 'statements and are executed in the context of the program being\n' - 'debugged. Python statements can also be prefixed with an ' - 'exclamation\n' - 'point ("!"). This is a powerful way to inspect the program ' - 'being\n' - 'debugged; it is even possible to change a variable or call a ' - 'function.\n' - 'When an exception occurs in such a statement, the exception name ' - 'is\n' - 'printed but the debugger’s state is not changed.\n' - '\n' - 'The debugger supports aliases. Aliases can have parameters ' - 'which\n' - 'allows one a certain level of adaptability to the context under\n' - 'examination.\n' - '\n' - 'Multiple commands may be entered on a single line, separated by ' - '";;".\n' - '(A single ";" is not used as it is the separator for multiple ' - 'commands\n' - 'in a line that is passed to the Python parser.) No intelligence ' - 'is\n' - 'applied to separating the commands; the input is split at the ' - 'first\n' - '";;" pair, even if it is in the middle of a quoted string.\n' - '\n' - 'If a file ".pdbrc" exists in the user’s home directory or in ' - 'the\n' - 'current directory, it is read in and executed as if it had been ' - 'typed\n' - 'at the debugger prompt. This is particularly useful for ' - 'aliases. If\n' - 'both files exist, the one in the home directory is read first ' - 'and\n' - 'aliases defined there can be overridden by the local file.\n' - '\n' - 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' - 'continue debugging, such as "continue" or "next". Previously, ' - 'these\n' - 'commands had no effect.\n' - '\n' - 'h(elp) [command]\n' - '\n' - ' Without argument, print the list of available commands. With ' - 'a\n' - ' *command* as argument, print help about that command. "help ' - 'pdb"\n' - ' displays the full documentation (the docstring of the "pdb"\n' - ' module). Since the *command* argument must be an identifier, ' - '"help\n' - ' exec" must be entered to get help on the "!" command.\n' - '\n' - 'w(here)\n' - '\n' - ' Print a stack trace, with the most recent frame at the ' - 'bottom. An\n' - ' arrow indicates the current frame, which determines the ' - 'context of\n' - ' most commands.\n' - '\n' - 'd(own) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels down in ' - 'the\n' - ' stack trace (to a newer frame).\n' - '\n' - 'u(p) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels up in the ' - 'stack\n' - ' trace (to an older frame).\n' - '\n' - 'b(reak) [([filename:]lineno | function) [, condition]]\n' - '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' - ' With a *function* argument, set a break at the first ' - 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' - '\n' - ' If a second argument is present, it is an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored.\n' - '\n' - ' Without argument, list all breaks, including for each ' - 'breakpoint,\n' - ' the number of times that breakpoint has been hit, the ' - 'current\n' - ' ignore count, and the associated condition if any.\n' - '\n' - 'tbreak [([filename:]lineno | function) [, condition]]\n' - '\n' - ' Temporary breakpoint, which is removed automatically when it ' - 'is\n' - ' first hit. The arguments are the same as for "break".\n' - '\n' - 'cl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n' - '\n' - ' With a *filename:lineno* argument, clear all the breakpoints ' - 'at\n' - ' this line. With a space separated list of breakpoint numbers, ' - 'clear\n' - ' those breakpoints. Without argument, clear all breaks (but ' - 'first\n' - ' ask confirmation).\n' - '\n' - 'disable [bpnumber [bpnumber ...]]\n' - '\n' - ' Disable the breakpoints given as a space separated list of\n' - ' breakpoint numbers. Disabling a breakpoint means it cannot ' - 'cause\n' - ' the program to stop execution, but unlike clearing a ' - 'breakpoint, it\n' - ' remains in the list of breakpoints and can be (re-)enabled.\n' - '\n' - 'enable [bpnumber [bpnumber ...]]\n' - '\n' - ' Enable the breakpoints specified.\n' - '\n' - 'ignore bpnumber [count]\n' - '\n' - ' Set the ignore count for the given breakpoint number. If ' - 'count is\n' - ' omitted, the ignore count is set to 0. A breakpoint becomes ' - 'active\n' - ' when the ignore count is zero. When non-zero, the count is\n' - ' decremented each time the breakpoint is reached and the ' - 'breakpoint\n' - ' is not disabled and any associated condition evaluates to ' - 'true.\n' - '\n' - 'condition bpnumber [condition]\n' - '\n' - ' Set a new *condition* for the breakpoint, an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored. If ' - '*condition*\n' - ' is absent, any existing condition is removed; i.e., the ' - 'breakpoint\n' - ' is made unconditional.\n' - '\n' - 'commands [bpnumber]\n' - '\n' - ' Specify a list of commands for breakpoint number *bpnumber*. ' - 'The\n' - ' commands themselves appear on the following lines. Type a ' - 'line\n' - ' containing just "end" to terminate the commands. An example:\n' - '\n' - ' (Pdb) commands 1\n' - ' (com) p some_variable\n' - ' (com) end\n' - ' (Pdb)\n' - '\n' - ' To remove all commands from a breakpoint, type commands and ' - 'follow\n' - ' it immediately with "end"; that is, give no commands.\n' - '\n' - ' With no *bpnumber* argument, commands refers to the last ' - 'breakpoint\n' - ' set.\n' - '\n' - ' You can use breakpoint commands to start your program up ' - 'again.\n' - ' Simply use the continue command, or step, or any other ' - 'command that\n' - ' resumes execution.\n' - '\n' - ' Specifying any command resuming execution (currently ' - 'continue,\n' - ' step, next, return, jump, quit and their abbreviations) ' - 'terminates\n' - ' the command list (as if that command was immediately followed ' - 'by\n' - ' end). This is because any time you resume execution (even ' - 'with a\n' - ' simple next or step), you may encounter another ' - 'breakpoint—which\n' - ' could have its own command list, leading to ambiguities about ' - 'which\n' - ' list to execute.\n' - '\n' - ' If you use the ‘silent’ command in the command list, the ' - 'usual\n' - ' message about stopping at a breakpoint is not printed. This ' - 'may be\n' - ' desirable for breakpoints that are to print a specific ' - 'message and\n' - ' then continue. If none of the other commands print anything, ' - 'you\n' - ' see no sign that the breakpoint was reached.\n' - '\n' - 's(tep)\n' - '\n' - ' Execute the current line, stop at the first possible ' - 'occasion\n' - ' (either in a function that is called or on the next line in ' - 'the\n' - ' current function).\n' - '\n' - 'n(ext)\n' - '\n' - ' Continue execution until the next line in the current ' - 'function is\n' - ' reached or it returns. (The difference between "next" and ' - '"step"\n' - ' is that "step" stops inside a called function, while "next"\n' - ' executes called functions at (nearly) full speed, only ' - 'stopping at\n' - ' the next line in the current function.)\n' - '\n' - 'unt(il) [lineno]\n' - '\n' - ' Without argument, continue execution until the line with a ' - 'number\n' - ' greater than the current one is reached.\n' - '\n' - ' With a line number, continue execution until a line with a ' - 'number\n' - ' greater or equal to that is reached. In both cases, also ' - 'stop when\n' - ' the current frame returns.\n' - '\n' - ' Changed in version 3.2: Allow giving an explicit line ' - 'number.\n' - '\n' - 'r(eturn)\n' - '\n' - ' Continue execution until the current function returns.\n' - '\n' - 'c(ont(inue))\n' - '\n' - ' Continue execution, only stop when a breakpoint is ' - 'encountered.\n' - '\n' - 'j(ump) lineno\n' - '\n' - ' Set the next line that will be executed. Only available in ' - 'the\n' - ' bottom-most frame. This lets you jump back and execute code ' - 'again,\n' - ' or jump forward to skip code that you don’t want to run.\n' - '\n' - ' It should be noted that not all jumps are allowed – for ' - 'instance it\n' - ' is not possible to jump into the middle of a "for" loop or ' - 'out of a\n' - ' "finally" clause.\n' - '\n' - 'l(ist) [first[, last]]\n' - '\n' - ' List source code for the current file. Without arguments, ' - 'list 11\n' - ' lines around the current line or continue the previous ' - 'listing.\n' - ' With "." as argument, list 11 lines around the current line. ' - 'With\n' - ' one argument, list 11 lines around at that line. With two\n' - ' arguments, list the given range; if the second argument is ' - 'less\n' - ' than the first, it is interpreted as a count.\n' - '\n' - ' The current line in the current frame is indicated by "->". ' - 'If an\n' - ' exception is being debugged, the line where the exception ' - 'was\n' - ' originally raised or propagated is indicated by ">>", if it ' - 'differs\n' - ' from the current line.\n' - '\n' - ' New in version 3.2: The ">>" marker.\n' - '\n' - 'll | longlist\n' - '\n' - ' List all source code for the current function or frame.\n' - ' Interesting lines are marked as for "list".\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'a(rgs)\n' - '\n' - ' Print the argument list of the current function.\n' - '\n' - 'p expression\n' - '\n' - ' Evaluate the *expression* in the current context and print ' - 'its\n' - ' value.\n' - '\n' - ' Note: "print()" can also be used, but is not a debugger ' - 'command —\n' - ' this executes the Python "print()" function.\n' - '\n' - 'pp expression\n' - '\n' - ' Like the "p" command, except the value of the expression is ' - 'pretty-\n' - ' printed using the "pprint" module.\n' - '\n' - 'whatis expression\n' - '\n' - ' Print the type of the *expression*.\n' - '\n' - 'source expression\n' - '\n' - ' Try to get source code for the given object and display it.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'display [expression]\n' - '\n' - ' Display the value of the expression if it changed, each time\n' - ' execution stops in the current frame.\n' - '\n' - ' Without expression, list all display expressions for the ' - 'current\n' - ' frame.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'undisplay [expression]\n' - '\n' - ' Do not display the expression any more in the current frame.\n' - ' Without expression, clear all display expressions for the ' - 'current\n' - ' frame.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'interact\n' - '\n' - ' Start an interactive interpreter (using the "code" module) ' - 'whose\n' - ' global namespace contains all the (global and local) names ' - 'found in\n' - ' the current scope.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'alias [name [command]]\n' - '\n' - ' Create an alias called *name* that executes *command*. The ' - 'command\n' - ' must *not* be enclosed in quotes. Replaceable parameters can ' - 'be\n' - ' indicated by "%1", "%2", and so on, while "%*" is replaced by ' - 'all\n' - ' the parameters. If no command is given, the current alias ' - 'for\n' - ' *name* is shown. If no arguments are given, all aliases are ' - 'listed.\n' - '\n' - ' Aliases may be nested and can contain anything that can be ' - 'legally\n' - ' typed at the pdb prompt. Note that internal pdb commands ' - '*can* be\n' - ' overridden by aliases. Such a command is then hidden until ' - 'the\n' - ' alias is removed. Aliasing is recursively applied to the ' - 'first\n' - ' word of the command line; all other words in the line are ' - 'left\n' - ' alone.\n' - '\n' - ' As an example, here are two useful aliases (especially when ' - 'placed\n' - ' in the ".pdbrc" file):\n' - '\n' - ' # Print instance variables (usage "pi classInst")\n' - ' alias pi for k in %1.__dict__.keys(): ' - 'print("%1.",k,"=",%1.__dict__[k])\n' - ' # Print instance variables in self\n' - ' alias ps pi self\n' - '\n' - 'unalias name\n' - '\n' - ' Delete the specified alias.\n' - '\n' - '! statement\n' - '\n' - ' Execute the (one-line) *statement* in the context of the ' - 'current\n' - ' stack frame. The exclamation point can be omitted unless the ' - 'first\n' - ' word of the statement resembles a debugger command. To set ' - 'a\n' - ' global variable, you can prefix the assignment command with ' - 'a\n' - ' "global" statement on the same line, e.g.:\n' - '\n' - " (Pdb) global list_options; list_options = ['-l']\n" - ' (Pdb)\n' - '\n' - 'run [args ...]\n' - 'restart [args ...]\n' - '\n' - ' Restart the debugged Python program. If an argument is ' - 'supplied,\n' - ' it is split with "shlex" and the result is used as the new\n' - ' "sys.argv". History, breakpoints, actions and debugger ' - 'options are\n' - ' preserved. "restart" is an alias for "run".\n' - '\n' - 'q(uit)\n' - '\n' - ' Quit from the debugger. The program being executed is ' - 'aborted.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] Whether a frame is considered to originate in a certain ' - 'module\n' - ' is determined by the "__name__" in the frame globals.\n', - 'del': 'The "del" statement\n' - '*******************\n' - '\n' - ' del_stmt ::= "del" target_list\n' - '\n' - 'Deletion is recursively defined very similar to the way assignment ' - 'is\n' - 'defined. Rather than spelling it out in full details, here are some\n' - 'hints.\n' - '\n' - 'Deletion of a target list recursively deletes each target, from left\n' - 'to right.\n' - '\n' - 'Deletion of a name removes the binding of that name from the local ' - 'or\n' - 'global namespace, depending on whether the name occurs in a "global"\n' - 'statement in the same code block. If the name is unbound, a\n' - '"NameError" exception will be raised.\n' - '\n' - 'Deletion of attribute references, subscriptions and slicings is ' - 'passed\n' - 'to the primary object involved; deletion of a slicing is in general\n' - 'equivalent to assignment of an empty slice of the right type (but ' - 'even\n' - 'this is determined by the sliced object).\n' - '\n' - 'Changed in version 3.2: Previously it was illegal to delete a name\n' - 'from the local namespace if it occurs as a free variable in a nested\n' - 'block.\n', - 'dict': 'Dictionary displays\n' - '*******************\n' - '\n' - 'A dictionary display is a possibly empty series of key/datum pairs\n' - 'enclosed in curly braces:\n' - '\n' - ' dict_display ::= "{" [key_datum_list | dict_comprehension] ' - '"}"\n' - ' key_datum_list ::= key_datum ("," key_datum)* [","]\n' - ' key_datum ::= expression ":" expression | "**" or_expr\n' - ' dict_comprehension ::= expression ":" expression comp_for\n' - '\n' - 'A dictionary display yields a new dictionary object.\n' - '\n' - 'If a comma-separated sequence of key/datum pairs is given, they are\n' - 'evaluated from left to right to define the entries of the ' - 'dictionary:\n' - 'each key object is used as a key into the dictionary to store the\n' - 'corresponding datum. This means that you can specify the same key\n' - 'multiple times in the key/datum list, and the final dictionary’s ' - 'value\n' - 'for that key will be the last one given.\n' - '\n' - 'A double asterisk "**" denotes *dictionary unpacking*. Its operand\n' - 'must be a *mapping*. Each mapping item is added to the new\n' - 'dictionary. Later values replace values already set by earlier\n' - 'key/datum pairs and earlier dictionary unpackings.\n' - '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'A dict comprehension, in contrast to list and set comprehensions,\n' - 'needs two expressions separated with a colon followed by the usual\n' - '“for” and “if” clauses. When the comprehension is run, the ' - 'resulting\n' - 'key and value elements are inserted in the new dictionary in the ' - 'order\n' - 'they are produced.\n' - '\n' - 'Restrictions on the types of the key values are listed earlier in\n' - 'section The standard type hierarchy. (To summarize, the key type\n' - 'should be *hashable*, which excludes all mutable objects.) Clashes\n' - 'between duplicate keys are not detected; the last datum (textually\n' - 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': 'Interaction with dynamic features\n' - '*********************************\n' - '\n' - 'Name resolution of free variables occurs at runtime, not ' - 'at compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access ' - 'to the full\n' - 'environment for resolving names. Names may be resolved ' - 'in the local\n' - 'and global namespaces of the caller. Free variables are ' - 'not resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional ' - 'arguments to\n' - 'override the global and local namespace. If only one ' - 'namespace is\n' - 'specified, it is used for both.\n', - 'else': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'exceptions': 'Exceptions\n' - '**********\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other ' - 'exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at ' - 'an outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece ' - 'of code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack backtrace, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a base class thereof. The instance ' - 'can be\n' - 'received by the handler and can carry additional information ' - 'about the\n' - 'exceptional condition.\n' - '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', - 'execmodel': 'Execution model\n' - '***************\n' - '\n' - '\n' - 'Structure of a program\n' - '======================\n' - '\n' - 'A Python program is constructed from code blocks. A *block* is ' - 'a piece\n' - 'of Python program text that is executed as a unit. The ' - 'following are\n' - 'blocks: a module, a function body, and a class definition. ' - 'Each\n' - 'command typed interactively is a block. A script file (a file ' - 'given\n' - 'as standard input to the interpreter or specified as a command ' - 'line\n' - 'argument to the interpreter) is a code block. A script command ' - '(a\n' - 'command specified on the interpreter command line with the ' - '"-c"\n' - 'option) is a code block. The string argument passed to the ' - 'built-in\n' - 'functions "eval()" and "exec()" is a code block.\n' - '\n' - 'A code block is executed in an *execution frame*. A frame ' - 'contains\n' - 'some administrative information (used for debugging) and ' - 'determines\n' - 'where and how execution continues after the code block’s ' - 'execution has\n' - 'completed.\n' - '\n' - '\n' - 'Naming and binding\n' - '==================\n' - '\n' - '\n' - 'Binding of names\n' - '----------------\n' - '\n' - '*Names* refer to objects. Names are introduced by name ' - 'binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, ' - 'or after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. ' - 'This form\n' - 'may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered ' - 'bound for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block ' - 'defined by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound ' - 'at the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution ' - 'rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '-------------------\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If ' - 'a local\n' - 'variable is defined in a block, its scope includes that block. ' - 'If the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point ' - 'where the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code ' - 'block, all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used ' - 'within a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur ' - 'anywhere\n' - 'within a code block. The local variables of a code block can ' - 'be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of ' - 'the name\n' - 'specified in the statement refer to the binding of that name in ' - 'the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the ' - 'module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the name is not found there, the builtins namespace is ' - 'searched. The\n' - '"global" statement must precede all uses of the name.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope.\n' - '\n' - 'The namespace for a module is automatically created the first ' - 'time a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition ' - 'is an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute ' - 'dictionary of\n' - 'the class. The scope of names defined in a class block is ' - 'limited to\n' - 'the class block; it does not extend to the code blocks of ' - 'methods –\n' - 'this includes comprehensions and generator expressions since ' - 'they are\n' - 'implemented using a function scope. This means that the ' - 'following\n' - 'will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '---------------------------------\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the ' - 'latter case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when ' - 'in any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '---------------------------------\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n' - '\n' - '\n' - 'Exceptions\n' - '==========\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at an ' - 'outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece of ' - 'code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack backtrace, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a base class thereof. The instance ' - 'can be\n' - 'received by the handler and can carry additional information ' - 'about the\n' - 'exceptional condition.\n' - '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', - 'exprlists': 'Expression lists\n' - '****************\n' - '\n' - ' expression_list ::= expression ("," expression)* [","]\n' - ' starred_list ::= starred_item ("," starred_item)* ' - '[","]\n' - ' starred_expression ::= expression | (starred_item ",")* ' - '[starred_item]\n' - ' starred_item ::= expression | "*" or_expr\n' - '\n' - 'Except when part of a list or set display, an expression list\n' - 'containing at least one comma yields a tuple. The length of ' - 'the tuple\n' - 'is the number of expressions in the list. The expressions are\n' - 'evaluated from left to right.\n' - '\n' - 'An asterisk "*" denotes *iterable unpacking*. Its operand must ' - 'be an\n' - '*iterable*. The iterable is expanded into a sequence of items, ' - 'which\n' - 'are included in the new tuple, list, or set, at the site of ' - 'the\n' - 'unpacking.\n' - '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'The trailing comma is required only to create a single tuple ' - '(a.k.a. a\n' - '*singleton*); it is optional in all other cases. A single ' - 'expression\n' - 'without a trailing comma doesn’t create a tuple, but rather ' - 'yields the\n' - 'value of that expression. (To create an empty tuple, use an ' - 'empty pair\n' - 'of parentheses: "()".)\n', - 'floating': 'Floating point literals\n' - '***********************\n' - '\n' - 'Floating point literals are described by the following lexical\n' - 'definitions:\n' - '\n' - ' floatnumber ::= pointfloat | exponentfloat\n' - ' pointfloat ::= [digitpart] fraction | digitpart "."\n' - ' exponentfloat ::= (digitpart | pointfloat) exponent\n' - ' digitpart ::= digit (["_"] digit)*\n' - ' fraction ::= "." digitpart\n' - ' exponent ::= ("e" | "E") ["+" | "-"] digitpart\n' - '\n' - 'Note that the integer and exponent parts are always interpreted ' - 'using\n' - 'radix 10. For example, "077e010" is legal, and denotes the same ' - 'number\n' - 'as "77e10". The allowed range of floating point literals is\n' - 'implementation-dependent. As in integer literals, underscores ' - 'are\n' - 'supported for digit grouping.\n' - '\n' - 'Some examples of floating point literals:\n' - '\n' - ' 3.14 10. .001 1e100 3.14e-10 0e0 ' - '3.14_15_93\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'for': 'The "for" statement\n' - '*******************\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" expression_list ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The expression list is evaluated once; it should yield an iterable\n' - 'object. An iterator is created for the result of the\n' - '"expression_list". The suite is then executed once for each item\n' - 'provided by the iterator, in the order returned by the iterator. ' - 'Each\n' - 'item in turn is assigned to the target list using the standard rules\n' - 'for assignments (see Assignment statements), and then the suite is\n' - 'executed. When the items are exhausted (which is immediately when ' - 'the\n' - 'sequence is empty or an iterator raises a "StopIteration" ' - 'exception),\n' - 'the suite in the "else" clause, if present, is executed, and the ' - 'loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the loop\n' - 'without executing the "else" clause’s suite. A "continue" statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables(s) in the target ' - 'list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with the ' - 'next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is finished,\n' - 'but if the sequence is empty, they will not have been assigned to at\n' - 'all by the loop. Hint: the built-in function "range()" returns an\n' - 'iterator of integers suitable to emulate the effect of Pascal’s "for ' - 'i\n' - ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n' - '\n' - 'Note: There is a subtlety when the sequence is being modified by the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). An\n' - ' internal counter is used to keep track of which item is used next,\n' - ' and this is incremented on each iteration. When this counter has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item from ' - 'the\n' - ' sequence, the next item will be skipped (since it gets the index ' - 'of\n' - ' the current item which has already been treated). Likewise, if ' - 'the\n' - ' suite inserts an item in the sequence before the current item, the\n' - ' current item will be treated again the next time through the loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n', - 'formatstrings': 'Format String Syntax\n' - '********************\n' - '\n' - 'The "str.format()" method and the "Formatter" class share ' - 'the same\n' - 'syntax for format strings (although in the case of ' - '"Formatter",\n' - 'subclasses can define their own format string syntax). The ' - 'syntax is\n' - 'related to that of formatted string literals, but there ' - 'are\n' - 'differences.\n' - '\n' - 'Format strings contain “replacement fields” surrounded by ' - 'curly braces\n' - '"{}". Anything that is not contained in braces is ' - 'considered literal\n' - 'text, which is copied unchanged to the output. If you need ' - 'to include\n' - 'a brace character in the literal text, it can be escaped by ' - 'doubling:\n' - '"{{" and "}}".\n' - '\n' - 'The grammar for a replacement field is as follows:\n' - '\n' - ' replacement_field ::= "{" [field_name] ["!" ' - 'conversion] [":" format_spec] "}"\n' - ' field_name ::= arg_name ("." attribute_name | ' - '"[" element_index "]")*\n' - ' arg_name ::= [identifier | digit+]\n' - ' attribute_name ::= identifier\n' - ' element_index ::= digit+ | index_string\n' - ' index_string ::= +\n' - ' conversion ::= "r" | "s" | "a"\n' - ' format_spec ::= \n' - '\n' - 'In less formal terms, the replacement field can start with ' - 'a\n' - '*field_name* that specifies the object whose value is to be ' - 'formatted\n' - 'and inserted into the output instead of the replacement ' - 'field. The\n' - '*field_name* is optionally followed by a *conversion* ' - 'field, which is\n' - 'preceded by an exclamation point "\'!\'", and a ' - '*format_spec*, which is\n' - 'preceded by a colon "\':\'". These specify a non-default ' - 'format for the\n' - 'replacement value.\n' - '\n' - 'See also the Format Specification Mini-Language section.\n' - '\n' - 'The *field_name* itself begins with an *arg_name* that is ' - 'either a\n' - 'number or a keyword. If it’s a number, it refers to a ' - 'positional\n' - 'argument, and if it’s a keyword, it refers to a named ' - 'keyword\n' - 'argument. If the numerical arg_names in a format string ' - 'are 0, 1, 2,\n' - '… in sequence, they can all be omitted (not just some) and ' - 'the numbers\n' - '0, 1, 2, … will be automatically inserted in that order. ' - 'Because\n' - '*arg_name* is not quote-delimited, it is not possible to ' - 'specify\n' - 'arbitrary dictionary keys (e.g., the strings "\'10\'" or ' - '"\':-]\'") within\n' - 'a format string. The *arg_name* can be followed by any ' - 'number of index\n' - 'or attribute expressions. An expression of the form ' - '"\'.name\'" selects\n' - 'the named attribute using "getattr()", while an expression ' - 'of the form\n' - '"\'[index]\'" does an index lookup using "__getitem__()".\n' - '\n' - 'Changed in version 3.1: The positional argument specifiers ' - 'can be\n' - 'omitted for "str.format()", so "\'{} {}\'.format(a, b)" is ' - 'equivalent to\n' - '"\'{0} {1}\'.format(a, b)".\n' - '\n' - 'Changed in version 3.4: The positional argument specifiers ' - 'can be\n' - 'omitted for "Formatter".\n' - '\n' - 'Some simple format string examples:\n' - '\n' - ' "First, thou shalt count to {0}" # References first ' - 'positional argument\n' - ' "Bring me a {}" # Implicitly ' - 'references the first positional argument\n' - ' "From {} to {}" # Same as "From {0} to ' - '{1}"\n' - ' "My quest is {name}" # References keyword ' - "argument 'name'\n" - ' "Weight in tons {0.weight}" # \'weight\' attribute ' - 'of first positional arg\n' - ' "Units destroyed: {players[0]}" # First element of ' - "keyword argument 'players'.\n" - '\n' - 'The *conversion* field causes a type coercion before ' - 'formatting.\n' - 'Normally, the job of formatting a value is done by the ' - '"__format__()"\n' - 'method of the value itself. However, in some cases it is ' - 'desirable to\n' - 'force a type to be formatted as a string, overriding its ' - 'own\n' - 'definition of formatting. By converting the value to a ' - 'string before\n' - 'calling "__format__()", the normal formatting logic is ' - 'bypassed.\n' - '\n' - 'Three conversion flags are currently supported: "\'!s\'" ' - 'which calls\n' - '"str()" on the value, "\'!r\'" which calls "repr()" and ' - '"\'!a\'" which\n' - 'calls "ascii()".\n' - '\n' - 'Some examples:\n' - '\n' - ' "Harold\'s a clever {0!s}" # Calls str() on the ' - 'argument first\n' - ' "Bring out the holy {name!r}" # Calls repr() on the ' - 'argument first\n' - ' "More {!a}" # Calls ascii() on the ' - 'argument first\n' - '\n' - 'The *format_spec* field contains a specification of how the ' - 'value\n' - 'should be presented, including such details as field width, ' - 'alignment,\n' - 'padding, decimal precision and so on. Each value type can ' - 'define its\n' - 'own “formatting mini-language” or interpretation of the ' - '*format_spec*.\n' - '\n' - 'Most built-in types support a common formatting ' - 'mini-language, which\n' - 'is described in the next section.\n' - '\n' - 'A *format_spec* field can also include nested replacement ' - 'fields\n' - 'within it. These nested replacement fields may contain a ' - 'field name,\n' - 'conversion flag and format specification, but deeper ' - 'nesting is not\n' - 'allowed. The replacement fields within the format_spec ' - 'are\n' - 'substituted before the *format_spec* string is interpreted. ' - 'This\n' - 'allows the formatting of a value to be dynamically ' - 'specified.\n' - '\n' - 'See the Format examples section for some examples.\n' - '\n' - '\n' - 'Format Specification Mini-Language\n' - '==================================\n' - '\n' - '“Format specifications” are used within replacement fields ' - 'contained\n' - 'within a format string to define how individual values are ' - 'presented\n' - '(see Format String Syntax and Formatted string literals). ' - 'They can\n' - 'also be passed directly to the built-in "format()" ' - 'function. Each\n' - 'formattable type may define how the format specification is ' - 'to be\n' - 'interpreted.\n' - '\n' - 'Most built-in types implement the following options for ' - 'format\n' - 'specifications, although some of the formatting options are ' - 'only\n' - 'supported by the numeric types.\n' - '\n' - 'A general convention is that an empty format string ("""") ' - 'produces\n' - 'the same result as if you had called "str()" on the value. ' - 'A non-empty\n' - 'format string typically modifies the result.\n' - '\n' - 'The general form of a *standard format specifier* is:\n' - '\n' - ' format_spec ::= ' - '[[fill]align][sign][#][0][width][grouping_option][.precision][type]\n' - ' fill ::= \n' - ' align ::= "<" | ">" | "=" | "^"\n' - ' sign ::= "+" | "-" | " "\n' - ' width ::= digit+\n' - ' grouping_option ::= "_" | ","\n' - ' precision ::= digit+\n' - ' type ::= "b" | "c" | "d" | "e" | "E" | "f" | ' - '"F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n' - '\n' - 'If a valid *align* value is specified, it can be preceded ' - 'by a *fill*\n' - 'character that can be any character and defaults to a space ' - 'if\n' - 'omitted. It is not possible to use a literal curly brace ' - '(“"{"” or\n' - '“"}"”) as the *fill* character in a formatted string ' - 'literal or when\n' - 'using the "str.format()" method. However, it is possible ' - 'to insert a\n' - 'curly brace with a nested replacement field. This ' - 'limitation doesn’t\n' - 'affect the "format()" function.\n' - '\n' - 'The meaning of the various alignment options is as ' - 'follows:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Option | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'<\'" | Forces the field to be left-aligned ' - 'within the available |\n' - ' | | space (this is the default for most ' - 'objects). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'>\'" | Forces the field to be right-aligned ' - 'within the available |\n' - ' | | space (this is the default for ' - 'numbers). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'=\'" | Forces the padding to be placed after ' - 'the sign (if any) |\n' - ' | | but before the digits. This is used for ' - 'printing fields |\n' - ' | | in the form ‘+000000120’. This alignment ' - 'option is only |\n' - ' | | valid for numeric types. It becomes the ' - 'default when ‘0’ |\n' - ' | | immediately precedes the field ' - 'width. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'^\'" | Forces the field to be centered within ' - 'the available |\n' - ' | | ' - 'space. ' - '|\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'Note that unless a minimum field width is defined, the ' - 'field width\n' - 'will always be the same size as the data to fill it, so ' - 'that the\n' - 'alignment option has no meaning in this case.\n' - '\n' - 'The *sign* option is only valid for number types, and can ' - 'be one of\n' - 'the following:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Option | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'+\'" | indicates that a sign should be used for ' - 'both positive as |\n' - ' | | well as negative ' - 'numbers. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'-\'" | indicates that a sign should be used ' - 'only for negative |\n' - ' | | numbers (this is the default ' - 'behavior). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | space | indicates that a leading space should be ' - 'used on positive |\n' - ' | | numbers, and a minus sign on negative ' - 'numbers. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The "\'#\'" option causes the “alternate form” to be used ' - 'for the\n' - 'conversion. The alternate form is defined differently for ' - 'different\n' - 'types. This option is only valid for integer, float, ' - 'complex and\n' - 'Decimal types. For integers, when binary, octal, or ' - 'hexadecimal output\n' - 'is used, this option adds the prefix respective "\'0b\'", ' - '"\'0o\'", or\n' - '"\'0x\'" to the output value. For floats, complex and ' - 'Decimal the\n' - 'alternate form causes the result of the conversion to ' - 'always contain a\n' - 'decimal-point character, even if no digits follow it. ' - 'Normally, a\n' - 'decimal-point character appears in the result of these ' - 'conversions\n' - 'only if a digit follows it. In addition, for "\'g\'" and ' - '"\'G\'"\n' - 'conversions, trailing zeros are not removed from the ' - 'result.\n' - '\n' - 'The "\',\'" option signals the use of a comma for a ' - 'thousands separator.\n' - 'For a locale aware separator, use the "\'n\'" integer ' - 'presentation type\n' - 'instead.\n' - '\n' - 'Changed in version 3.1: Added the "\',\'" option (see also ' - '**PEP 378**).\n' - '\n' - 'The "\'_\'" option signals the use of an underscore for a ' - 'thousands\n' - 'separator for floating point presentation types and for ' - 'integer\n' - 'presentation type "\'d\'". For integer presentation types ' - '"\'b\'", "\'o\'",\n' - '"\'x\'", and "\'X\'", underscores will be inserted every 4 ' - 'digits. For\n' - 'other presentation types, specifying this option is an ' - 'error.\n' - '\n' - 'Changed in version 3.6: Added the "\'_\'" option (see also ' - '**PEP 515**).\n' - '\n' - '*width* is a decimal integer defining the minimum field ' - 'width. If not\n' - 'specified, then the field width will be determined by the ' - 'content.\n' - '\n' - 'When no explicit alignment is given, preceding the *width* ' - 'field by a\n' - 'zero ("\'0\'") character enables sign-aware zero-padding ' - 'for numeric\n' - 'types. This is equivalent to a *fill* character of "\'0\'" ' - 'with an\n' - '*alignment* type of "\'=\'".\n' - '\n' - 'The *precision* is a decimal number indicating how many ' - 'digits should\n' - 'be displayed after the decimal point for a floating point ' - 'value\n' - 'formatted with "\'f\'" and "\'F\'", or before and after the ' - 'decimal point\n' - 'for a floating point value formatted with "\'g\'" or ' - '"\'G\'". For non-\n' - 'number types the field indicates the maximum field size - ' - 'in other\n' - 'words, how many characters will be used from the field ' - 'content. The\n' - '*precision* is not allowed for integer values.\n' - '\n' - 'Finally, the *type* determines how the data should be ' - 'presented.\n' - '\n' - 'The available string presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'s\'" | String format. This is the default type ' - 'for strings and |\n' - ' | | may be ' - 'omitted. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'s\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The available integer presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'b\'" | Binary format. Outputs the number in ' - 'base 2. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'c\'" | Character. Converts the integer to the ' - 'corresponding |\n' - ' | | unicode character before ' - 'printing. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'d\'" | Decimal Integer. Outputs the number in ' - 'base 10. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'o\'" | Octal format. Outputs the number in base ' - '8. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'x\'" | Hex format. Outputs the number in base ' - '16, using lower- |\n' - ' | | case letters for the digits above ' - '9. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'X\'" | Hex format. Outputs the number in base ' - '16, using upper- |\n' - ' | | case letters for the digits above ' - '9. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'d\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'d\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'In addition to the above presentation types, integers can ' - 'be formatted\n' - 'with the floating point presentation types listed below ' - '(except "\'n\'"\n' - 'and "None"). When doing so, "float()" is used to convert ' - 'the integer\n' - 'to a floating point number before formatting.\n' - '\n' - 'The available presentation types for floating point and ' - 'decimal values\n' - 'are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'e\'" | Exponent notation. Prints the number in ' - 'scientific |\n' - ' | | notation using the letter ‘e’ to indicate ' - 'the exponent. |\n' - ' | | The default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'E\'" | Exponent notation. Same as "\'e\'" ' - 'except it uses an upper |\n' - ' | | case ‘E’ as the separator ' - 'character. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'f\'" | Fixed-point notation. Displays the ' - 'number as a fixed-point |\n' - ' | | number. The default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'F\'" | Fixed-point notation. Same as "\'f\'", ' - 'but converts "nan" to |\n' - ' | | "NAN" and "inf" to ' - '"INF". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'g\'" | General format. For a given precision ' - '"p >= 1", this |\n' - ' | | rounds the number to "p" significant ' - 'digits and then |\n' - ' | | formats the result in either fixed-point ' - 'format or in |\n' - ' | | scientific notation, depending on its ' - 'magnitude. The |\n' - ' | | precise rules are as follows: suppose that ' - 'the result |\n' - ' | | formatted with presentation type "\'e\'" ' - 'and precision "p-1" |\n' - ' | | would have exponent "exp". Then if "-4 <= ' - 'exp < p", the |\n' - ' | | number is formatted with presentation type ' - '"\'f\'" and |\n' - ' | | precision "p-1-exp". Otherwise, the ' - 'number is formatted |\n' - ' | | with presentation type "\'e\'" and ' - 'precision "p-1". In both |\n' - ' | | cases insignificant trailing zeros are ' - 'removed from the |\n' - ' | | significand, and the decimal point is also ' - 'removed if |\n' - ' | | there are no remaining digits following ' - 'it. Positive and |\n' - ' | | negative infinity, positive and negative ' - 'zero, and nans, |\n' - ' | | are formatted as "inf", "-inf", "0", "-0" ' - 'and "nan" |\n' - ' | | respectively, regardless of the ' - 'precision. A precision of |\n' - ' | | "0" is treated as equivalent to a ' - 'precision of "1". The |\n' - ' | | default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'G\'" | General format. Same as "\'g\'" except ' - 'switches to "\'E\'" if |\n' - ' | | the number gets too large. The ' - 'representations of infinity |\n' - ' | | and NaN are uppercased, ' - 'too. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'g\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'%\'" | Percentage. Multiplies the number by 100 ' - 'and displays in |\n' - ' | | fixed ("\'f\'") format, followed by a ' - 'percent sign. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | Similar to "\'g\'", except that ' - 'fixed-point notation, when |\n' - ' | | used, has at least one digit past the ' - 'decimal point. The |\n' - ' | | default precision is as high as needed to ' - 'represent the |\n' - ' | | particular value. The overall effect is to ' - 'match the |\n' - ' | | output of "str()" as altered by the other ' - 'format |\n' - ' | | ' - 'modifiers. ' - '|\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - '\n' - 'Format examples\n' - '===============\n' - '\n' - 'This section contains examples of the "str.format()" syntax ' - 'and\n' - 'comparison with the old "%"-formatting.\n' - '\n' - 'In most of the cases the syntax is similar to the old ' - '"%"-formatting,\n' - 'with the addition of the "{}" and with ":" used instead of ' - '"%". For\n' - 'example, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n' - '\n' - 'The new format syntax also supports new and different ' - 'options, shown\n' - 'in the following examples.\n' - '\n' - 'Accessing arguments by position:\n' - '\n' - " >>> '{0}, {1}, {2}'.format('a', 'b', 'c')\n" - " 'a, b, c'\n" - " >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only\n" - " 'a, b, c'\n" - " >>> '{2}, {1}, {0}'.format('a', 'b', 'c')\n" - " 'c, b, a'\n" - " >>> '{2}, {1}, {0}'.format(*'abc') # unpacking " - 'argument sequence\n' - " 'c, b, a'\n" - " >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' " - 'indices can be repeated\n' - " 'abracadabra'\n" - '\n' - 'Accessing arguments by name:\n' - '\n' - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(latitude='37.24N', " - "longitude='-115.81W')\n" - " 'Coordinates: 37.24N, -115.81W'\n" - " >>> coord = {'latitude': '37.24N', 'longitude': " - "'-115.81W'}\n" - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(**coord)\n" - " 'Coordinates: 37.24N, -115.81W'\n" - '\n' - 'Accessing arguments’ attributes:\n' - '\n' - ' >>> c = 3-5j\n' - " >>> ('The complex number {0} is formed from the real " - "part {0.real} '\n" - " ... 'and the imaginary part {0.imag}.').format(c)\n" - " 'The complex number (3-5j) is formed from the real part " - "3.0 and the imaginary part -5.0.'\n" - ' >>> class Point:\n' - ' ... def __init__(self, x, y):\n' - ' ... self.x, self.y = x, y\n' - ' ... def __str__(self):\n' - " ... return 'Point({self.x}, " - "{self.y})'.format(self=self)\n" - ' ...\n' - ' >>> str(Point(4, 2))\n' - " 'Point(4, 2)'\n" - '\n' - 'Accessing arguments’ items:\n' - '\n' - ' >>> coord = (3, 5)\n' - " >>> 'X: {0[0]}; Y: {0[1]}'.format(coord)\n" - " 'X: 3; Y: 5'\n" - '\n' - 'Replacing "%s" and "%r":\n' - '\n' - ' >>> "repr() shows quotes: {!r}; str() doesn\'t: ' - '{!s}".format(\'test1\', \'test2\')\n' - ' "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n' - '\n' - 'Aligning the text and specifying a width:\n' - '\n' - " >>> '{:<30}'.format('left aligned')\n" - " 'left aligned '\n" - " >>> '{:>30}'.format('right aligned')\n" - " ' right aligned'\n" - " >>> '{:^30}'.format('centered')\n" - " ' centered '\n" - " >>> '{:*^30}'.format('centered') # use '*' as a fill " - 'char\n' - " '***********centered***********'\n" - '\n' - 'Replacing "%+f", "%-f", and "% f" and specifying a sign:\n' - '\n' - " >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it " - 'always\n' - " '+3.140000; -3.140000'\n" - " >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space " - 'for positive numbers\n' - " ' 3.140000; -3.140000'\n" - " >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the " - "minus -- same as '{:f}; {:f}'\n" - " '3.140000; -3.140000'\n" - '\n' - 'Replacing "%x" and "%o" and converting the value to ' - 'different bases:\n' - '\n' - ' >>> # format also supports binary numbers\n' - ' >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: ' - '{0:b}".format(42)\n' - " 'int: 42; hex: 2a; oct: 52; bin: 101010'\n" - ' >>> # with 0x, 0o, or 0b as prefix:\n' - ' >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: ' - '{0:#b}".format(42)\n' - " 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010'\n" - '\n' - 'Using the comma as a thousands separator:\n' - '\n' - " >>> '{:,}'.format(1234567890)\n" - " '1,234,567,890'\n" - '\n' - 'Expressing a percentage:\n' - '\n' - ' >>> points = 19\n' - ' >>> total = 22\n' - " >>> 'Correct answers: {:.2%}'.format(points/total)\n" - " 'Correct answers: 86.36%'\n" - '\n' - 'Using type-specific formatting:\n' - '\n' - ' >>> import datetime\n' - ' >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n' - " >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)\n" - " '2010-07-04 12:15:58'\n" - '\n' - 'Nesting arguments and more complex examples:\n' - '\n' - " >>> for align, text in zip('<^>', ['left', 'center', " - "'right']):\n" - " ... '{0:{fill}{align}16}'.format(text, fill=align, " - 'align=align)\n' - ' ...\n' - " 'left<<<<<<<<<<<<'\n" - " '^^^^^center^^^^^'\n" - " '>>>>>>>>>>>right'\n" - ' >>>\n' - ' >>> octets = [192, 168, 0, 1]\n' - " >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets)\n" - " 'C0A80001'\n" - ' >>> int(_, 16)\n' - ' 3232235521\n' - ' >>>\n' - ' >>> width = 5\n' - ' >>> for num in range(5,12): \n' - " ... for base in 'dXob':\n" - " ... print('{0:{width}{base}}'.format(num, " - "base=base, width=width), end=' ')\n" - ' ... print()\n' - ' ...\n' - ' 5 5 5 101\n' - ' 6 6 6 110\n' - ' 7 7 7 111\n' - ' 8 8 10 1000\n' - ' 9 9 11 1001\n' - ' 10 A 12 1010\n' - ' 11 B 13 1011\n', - 'function': 'Function definitions\n' - '********************\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" dotted_name ["(" ' - '[argument_list [","]] ")"] NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' - ' parameter_list ::= defparameter ("," defparameter)* ' - '["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [2]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from position arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' - '\n' - 'Parameters may have annotations of the form “": expression"” ' - 'following\n' - 'the parameter name. Any parameter may have an annotation even ' - 'those\n' - 'of the form "*identifier" or "**identifier". Functions may ' - 'have\n' - '“return” annotation of the form “"-> expression"” after the ' - 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n', - 'global': 'The "global" statement\n' - '**********************\n' - '\n' - ' global_stmt ::= "global" identifier ("," identifier)*\n' - '\n' - 'The "global" statement is a declaration which holds for the ' - 'entire\n' - 'current code block. It means that the listed identifiers are to ' - 'be\n' - 'interpreted as globals. It would be impossible to assign to a ' - 'global\n' - 'variable without "global", although free variables may refer to\n' - 'globals without being declared global.\n' - '\n' - 'Names listed in a "global" statement must not be used in the same ' - 'code\n' - 'block textually preceding that "global" statement.\n' - '\n' - 'Names listed in a "global" statement must not be defined as ' - 'formal\n' - 'parameters or in a "for" loop control target, "class" definition,\n' - 'function definition, "import" statement, or variable annotation.\n' - '\n' - '**CPython implementation detail:** The current implementation does ' - 'not\n' - 'enforce some of these restrictions, but programs should not abuse ' - 'this\n' - 'freedom, as future implementations may enforce them or silently ' - 'change\n' - 'the meaning of the program.\n' - '\n' - '**Programmer’s note:** "global" is a directive to the parser. It\n' - 'applies only to code parsed at the same time as the "global"\n' - 'statement. In particular, a "global" statement contained in a ' - 'string\n' - 'or code object supplied to the built-in "exec()" function does ' - 'not\n' - 'affect the code block *containing* the function call, and code\n' - 'contained in such a string is unaffected by "global" statements in ' - 'the\n' - 'code containing the function call. The same applies to the ' - '"eval()"\n' - 'and "compile()" functions.\n', - 'id-classes': 'Reserved classes of identifiers\n' - '*******************************\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' - 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' - '\n' - ' Note: The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for the\n' - ' "gettext" module for more information on this ' - 'convention.\n' - '\n' - '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'identifiers': 'Identifiers and keywords\n' - '************************\n' - '\n' - 'Identifiers (also referred to as *names*) are described by ' - 'the\n' - 'following lexical definitions.\n' - '\n' - 'The syntax of identifiers in Python is based on the Unicode ' - 'standard\n' - 'annex UAX-31, with elaboration and changes as defined below; ' - 'see also\n' - '**PEP 3131** for further details.\n' - '\n' - 'Within the ASCII range (U+0001..U+007F), the valid characters ' - 'for\n' - 'identifiers are the same as in Python 2.x: the uppercase and ' - 'lowercase\n' - 'letters "A" through "Z", the underscore "_" and, except for ' - 'the first\n' - 'character, the digits "0" through "9".\n' - '\n' - 'Python 3.0 introduces additional characters from outside the ' - 'ASCII\n' - 'range (see **PEP 3131**). For these characters, the ' - 'classification\n' - 'uses the version of the Unicode Character Database as ' - 'included in the\n' - '"unicodedata" module.\n' - '\n' - 'Identifiers are unlimited in length. Case is significant.\n' - '\n' - ' identifier ::= xid_start xid_continue*\n' - ' id_start ::= \n' - ' id_continue ::= \n' - ' xid_start ::= \n' - ' xid_continue ::= \n' - '\n' - 'The Unicode category codes mentioned above stand for:\n' - '\n' - '* *Lu* - uppercase letters\n' - '\n' - '* *Ll* - lowercase letters\n' - '\n' - '* *Lt* - titlecase letters\n' - '\n' - '* *Lm* - modifier letters\n' - '\n' - '* *Lo* - other letters\n' - '\n' - '* *Nl* - letter numbers\n' - '\n' - '* *Mn* - nonspacing marks\n' - '\n' - '* *Mc* - spacing combining marks\n' - '\n' - '* *Nd* - decimal numbers\n' - '\n' - '* *Pc* - connector punctuations\n' - '\n' - '* *Other_ID_Start* - explicit list of characters in ' - 'PropList.txt to\n' - ' support backwards compatibility\n' - '\n' - '* *Other_ID_Continue* - likewise\n' - '\n' - 'All identifiers are converted into the normal form NFKC while ' - 'parsing;\n' - 'comparison of identifiers is based on NFKC.\n' - '\n' - 'A non-normative HTML file listing all valid identifier ' - 'characters for\n' - 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' - 'potsdam.de/home/loewis/table-3131.html.\n' - '\n' - '\n' - 'Keywords\n' - '========\n' - '\n' - 'The following identifiers are used as reserved words, or ' - '*keywords* of\n' - 'the language, and cannot be used as ordinary identifiers. ' - 'They must\n' - 'be spelled exactly as written here:\n' - '\n' - ' False class finally is return\n' - ' None continue for lambda try\n' - ' True def from nonlocal while\n' - ' and del global not with\n' - ' as elif if or yield\n' - ' assert else import pass\n' - ' break except in raise\n' - '\n' - '\n' - 'Reserved classes of identifiers\n' - '===============================\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' - 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' - '\n' - ' Note: The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for ' - 'the\n' - ' "gettext" module for more information on this ' - 'convention.\n' - '\n' - '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'if': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'imaginary': 'Imaginary literals\n' - '******************\n' - '\n' - 'Imaginary literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' imagnumber ::= (floatnumber | digitpart) ("j" | "J")\n' - '\n' - 'An imaginary literal yields a complex number with a real part ' - 'of 0.0.\n' - 'Complex numbers are represented as a pair of floating point ' - 'numbers\n' - 'and have the same restrictions on their range. To create a ' - 'complex\n' - 'number with a nonzero real part, add a floating point number to ' - 'it,\n' - 'e.g., "(3+4j)". Some examples of imaginary literals:\n' - '\n' - ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' - '3.14_15_93j\n', - 'import': 'The "import" statement\n' - '**********************\n' - '\n' - ' import_stmt ::= "import" module ["as" identifier] ("," ' - 'module ["as" identifier])*\n' - ' | "from" relative_module "import" identifier ' - '["as" identifier]\n' - ' ("," identifier ["as" identifier])*\n' - ' | "from" relative_module "import" "(" ' - 'identifier ["as" identifier]\n' - ' ("," identifier ["as" identifier])* [","] ")"\n' - ' | "from" module "import" "*"\n' - ' module ::= (identifier ".")* identifier\n' - ' relative_module ::= "."* module | "."+\n' - '\n' - 'The basic import statement (no "from" clause) is executed in two\n' - 'steps:\n' - '\n' - '1. find a module, loading and initializing it if necessary\n' - '\n' - '2. define a name or names in the local namespace for the scope\n' - ' where the "import" statement occurs.\n' - '\n' - 'When the statement contains multiple clauses (separated by commas) ' - 'the\n' - 'two steps are carried out separately for each clause, just as ' - 'though\n' - 'the clauses had been separated out into individual import ' - 'statements.\n' - '\n' - 'The details of the first step, finding and loading modules are\n' - 'described in greater detail in the section on the import system, ' - 'which\n' - 'also describes the various types of packages and modules that can ' - 'be\n' - 'imported, as well as all the hooks that can be used to customize ' - 'the\n' - 'import system. Note that failures in this step may indicate ' - 'either\n' - 'that the module could not be located, *or* that an error occurred\n' - 'while initializing the module, which includes execution of the\n' - 'module’s code.\n' - '\n' - 'If the requested module is retrieved successfully, it will be ' - 'made\n' - 'available in the local namespace in one of three ways:\n' - '\n' - '* If the module name is followed by "as", then the name following\n' - ' "as" is bound directly to the imported module.\n' - '\n' - '* If no other name is specified, and the module being imported is ' - 'a\n' - ' top level module, the module’s name is bound in the local ' - 'namespace\n' - ' as a reference to the imported module\n' - '\n' - '* If the module being imported is *not* a top level module, then ' - 'the\n' - ' name of the top level package that contains the module is bound ' - 'in\n' - ' the local namespace as a reference to the top level package. ' - 'The\n' - ' imported module must be accessed using its full qualified name\n' - ' rather than directly\n' - '\n' - 'The "from" form uses a slightly more complex process:\n' - '\n' - '1. find the module specified in the "from" clause, loading and\n' - ' initializing it if necessary;\n' - '\n' - '2. for each of the identifiers specified in the "import" clauses:\n' - '\n' - ' 1. check if the imported module has an attribute by that name\n' - '\n' - ' 2. if not, attempt to import a submodule with that name and ' - 'then\n' - ' check the imported module again for that attribute\n' - '\n' - ' 3. if the attribute is not found, "ImportError" is raised.\n' - '\n' - ' 4. otherwise, a reference to that value is stored in the local\n' - ' namespace, using the name in the "as" clause if it is ' - 'present,\n' - ' otherwise using the attribute name\n' - '\n' - 'Examples:\n' - '\n' - ' import foo # foo imported and bound locally\n' - ' import foo.bar.baz # foo.bar.baz imported, foo bound ' - 'locally\n' - ' import foo.bar.baz as fbb # foo.bar.baz imported and bound as ' - 'fbb\n' - ' from foo.bar import baz # foo.bar.baz imported and bound as ' - 'baz\n' - ' from foo import attr # foo imported and foo.attr bound as ' - 'attr\n' - '\n' - 'If the list of identifiers is replaced by a star ("\'*\'"), all ' - 'public\n' - 'names defined in the module are bound in the local namespace for ' - 'the\n' - 'scope where the "import" statement occurs.\n' - '\n' - 'The *public names* defined by a module are determined by checking ' - 'the\n' - 'module’s namespace for a variable named "__all__"; if defined, it ' - 'must\n' - 'be a sequence of strings which are names defined or imported by ' - 'that\n' - 'module. The names given in "__all__" are all considered public ' - 'and\n' - 'are required to exist. If "__all__" is not defined, the set of ' - 'public\n' - 'names includes all names found in the module’s namespace which do ' - 'not\n' - 'begin with an underscore character ("\'_\'"). "__all__" should ' - 'contain\n' - 'the entire public API. It is intended to avoid accidentally ' - 'exporting\n' - 'items that are not part of the API (such as library modules which ' - 'were\n' - 'imported and used within the module).\n' - '\n' - 'The wild card form of import — "from module import *" — is only\n' - 'allowed at the module level. Attempting to use it in class or\n' - 'function definitions will raise a "SyntaxError".\n' - '\n' - 'When specifying what module to import you do not have to specify ' - 'the\n' - 'absolute name of the module. When a module or package is ' - 'contained\n' - 'within another package it is possible to make a relative import ' - 'within\n' - 'the same top package without having to mention the package name. ' - 'By\n' - 'using leading dots in the specified module or package after "from" ' - 'you\n' - 'can specify how high to traverse up the current package hierarchy\n' - 'without specifying exact names. One leading dot means the current\n' - 'package where the module making the import exists. Two dots means ' - 'up\n' - 'one package level. Three dots is up two levels, etc. So if you ' - 'execute\n' - '"from . import mod" from a module in the "pkg" package then you ' - 'will\n' - 'end up importing "pkg.mod". If you execute "from ..subpkg2 import ' - 'mod"\n' - 'from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\n' - 'specification for relative imports is contained within **PEP ' - '328**.\n' - '\n' - '"importlib.import_module()" is provided to support applications ' - 'that\n' - 'determine dynamically the modules to be loaded.\n' - '\n' - '\n' - 'Future statements\n' - '=================\n' - '\n' - 'A *future statement* is a directive to the compiler that a ' - 'particular\n' - 'module should be compiled using syntax or semantics that will be\n' - 'available in a specified future release of Python where the ' - 'feature\n' - 'becomes standard.\n' - '\n' - 'The future statement is intended to ease migration to future ' - 'versions\n' - 'of Python that introduce incompatible changes to the language. ' - 'It\n' - 'allows use of the new features on a per-module basis before the\n' - 'release in which the feature becomes standard.\n' - '\n' - ' future_stmt ::= "from" "__future__" "import" feature ["as" ' - 'identifier]\n' - ' ("," feature ["as" identifier])*\n' - ' | "from" "__future__" "import" "(" feature ' - '["as" identifier]\n' - ' ("," feature ["as" identifier])* [","] ")"\n' - ' feature ::= identifier\n' - '\n' - 'A future statement must appear near the top of the module. The ' - 'only\n' - 'lines that can appear before a future statement are:\n' - '\n' - '* the module docstring (if any),\n' - '\n' - '* comments,\n' - '\n' - '* blank lines, and\n' - '\n' - '* other future statements.\n' - '\n' - 'The features recognized by Python 3.0 are "absolute_import",\n' - '"division", "generators", "unicode_literals", "print_function",\n' - '"nested_scopes" and "with_statement". They are all redundant ' - 'because\n' - 'they are always enabled, and only kept for backwards ' - 'compatibility.\n' - '\n' - 'A future statement is recognized and treated specially at compile\n' - 'time: Changes to the semantics of core constructs are often\n' - 'implemented by generating different code. It may even be the ' - 'case\n' - 'that a new feature introduces new incompatible syntax (such as a ' - 'new\n' - 'reserved word), in which case the compiler may need to parse the\n' - 'module differently. Such decisions cannot be pushed off until\n' - 'runtime.\n' - '\n' - 'For any given release, the compiler knows which feature names ' - 'have\n' - 'been defined, and raises a compile-time error if a future ' - 'statement\n' - 'contains a feature not known to it.\n' - '\n' - 'The direct runtime semantics are the same as for any import ' - 'statement:\n' - 'there is a standard module "__future__", described later, and it ' - 'will\n' - 'be imported in the usual way at the time the future statement is\n' - 'executed.\n' - '\n' - 'The interesting runtime semantics depend on the specific feature\n' - 'enabled by the future statement.\n' - '\n' - 'Note that there is nothing special about the statement:\n' - '\n' - ' import __future__ [as name]\n' - '\n' - 'That is not a future statement; it’s an ordinary import statement ' - 'with\n' - 'no special semantics or syntax restrictions.\n' - '\n' - 'Code compiled by calls to the built-in functions "exec()" and\n' - '"compile()" that occur in a module "M" containing a future ' - 'statement\n' - 'will, by default, use the new syntax or semantics associated with ' - 'the\n' - 'future statement. This can be controlled by optional arguments ' - 'to\n' - '"compile()" — see the documentation of that function for details.\n' - '\n' - 'A future statement typed at an interactive interpreter prompt ' - 'will\n' - 'take effect for the rest of the interpreter session. If an\n' - 'interpreter is started with the "-i" option, is passed a script ' - 'name\n' - 'to execute, and the script includes a future statement, it will be ' - 'in\n' - 'effect in the interactive session started after the script is\n' - 'executed.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 236** - Back to the __future__\n' - ' The original proposal for the __future__ mechanism.\n', - 'in': 'Membership test operations\n' - '**************************\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which "in" ' - 'tests\n' - 'whether the dictionary has a given key. For container types such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' - 'Empty strings are always considered to be a substring of any other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n' - 'z" is produced while iterating over "y". If an exception is raised\n' - 'during the iteration, it is as if "in" raised that exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' - 'negative integer index *i* such that "x == y[i]", and all lower\n' - 'integer indices do not raise "IndexError" exception. (If any other\n' - 'exception is raised, it is as if "in" raised that exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse true value of\n' - '"in".\n', - 'integers': 'Integer literals\n' - '****************\n' - '\n' - 'Integer literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' integer ::= decinteger | bininteger | octinteger | ' - 'hexinteger\n' - ' decinteger ::= nonzerodigit (["_"] digit)* | "0"+ (["_"] ' - '"0")*\n' - ' bininteger ::= "0" ("b" | "B") (["_"] bindigit)+\n' - ' octinteger ::= "0" ("o" | "O") (["_"] octdigit)+\n' - ' hexinteger ::= "0" ("x" | "X") (["_"] hexdigit)+\n' - ' nonzerodigit ::= "1"..."9"\n' - ' digit ::= "0"..."9"\n' - ' bindigit ::= "0" | "1"\n' - ' octdigit ::= "0"..."7"\n' - ' hexdigit ::= digit | "a"..."f" | "A"..."F"\n' - '\n' - 'There is no limit for the length of integer literals apart from ' - 'what\n' - 'can be stored in available memory.\n' - '\n' - 'Underscores are ignored for determining the numeric value of ' - 'the\n' - 'literal. They can be used to group digits for enhanced ' - 'readability.\n' - 'One underscore can occur between digits, and after base ' - 'specifiers\n' - 'like "0x".\n' - '\n' - 'Note that leading zeros in a non-zero decimal number are not ' - 'allowed.\n' - 'This is for disambiguation with C-style octal literals, which ' - 'Python\n' - 'used before version 3.0.\n' - '\n' - 'Some examples of integer literals:\n' - '\n' - ' 7 2147483647 0o177 0b100110111\n' - ' 3 79228162514264337593543950336 0o377 0xdeadbeef\n' - ' 100_000_000_000 0b_1110_0101\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'lambda': 'Lambdas\n' - '*******\n' - '\n' - ' lambda_expr ::= "lambda" [parameter_list] ":" ' - 'expression\n' - ' lambda_expr_nocond ::= "lambda" [parameter_list] ":" ' - 'expression_nocond\n' - '\n' - 'Lambda expressions (sometimes called lambda forms) are used to ' - 'create\n' - 'anonymous functions. The expression "lambda parameters: ' - 'expression"\n' - 'yields a function object. The unnamed object behaves like a ' - 'function\n' - 'object defined with:\n' - '\n' - ' def (parameters):\n' - ' return expression\n' - '\n' - 'See section Function definitions for the syntax of parameter ' - 'lists.\n' - 'Note that functions created with lambda expressions cannot ' - 'contain\n' - 'statements or annotations.\n', - 'lists': 'List displays\n' - '*************\n' - '\n' - 'A list display is a possibly empty series of expressions enclosed ' - 'in\n' - 'square brackets:\n' - '\n' - ' list_display ::= "[" [starred_list | comprehension] "]"\n' - '\n' - 'A list display yields a new list object, the contents being ' - 'specified\n' - 'by either a list of expressions or a comprehension. When a comma-\n' - 'separated list of expressions is supplied, its elements are ' - 'evaluated\n' - 'from left to right and placed into the list object in that order.\n' - 'When a comprehension is supplied, the list is constructed from the\n' - 'elements resulting from the comprehension.\n', - 'naming': 'Naming and binding\n' - '******************\n' - '\n' - '\n' - 'Binding of names\n' - '================\n' - '\n' - '*Names* refer to objects. Names are introduced by name binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, or ' - 'after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. This ' - 'form\n' - 'may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered bound ' - 'for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block defined ' - 'by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound at ' - 'the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '===================\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If a ' - 'local\n' - 'variable is defined in a block, its scope includes that block. If ' - 'the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point where ' - 'the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code block, ' - 'all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used within ' - 'a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur anywhere\n' - 'within a code block. The local variables of a code block can be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of the ' - 'name\n' - 'specified in the statement refer to the binding of that name in ' - 'the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the name is not found there, the builtins namespace is searched. ' - 'The\n' - '"global" statement must precede all uses of the name.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope.\n' - '\n' - 'The namespace for a module is automatically created the first time ' - 'a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition is ' - 'an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute dictionary ' - 'of\n' - 'the class. The scope of names defined in a class block is limited ' - 'to\n' - 'the class block; it does not extend to the code blocks of methods ' - '–\n' - 'this includes comprehensions and generator expressions since they ' - 'are\n' - 'implemented using a function scope. This means that the ' - 'following\n' - 'will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '=================================\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the latter ' - 'case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when in ' - 'any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '=================================\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global namespace. ' - '[1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n', - 'nonlocal': 'The "nonlocal" statement\n' - '************************\n' - '\n' - ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' - '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', - 'numbers': 'Numeric literals\n' - '****************\n' - '\n' - 'There are three types of numeric literals: integers, floating ' - 'point\n' - 'numbers, and imaginary numbers. There are no complex literals\n' - '(complex numbers can be formed by adding a real number and an\n' - 'imaginary number).\n' - '\n' - 'Note that numeric literals do not include a sign; a phrase like ' - '"-1"\n' - 'is actually an expression composed of the unary operator ‘"-"‘ ' - 'and the\n' - 'literal "1".\n', - 'numeric-types': 'Emulating numeric types\n' - '***********************\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For ' - 'instance, to\n' - ' evaluate the expression "x + y", where *x* is an ' - 'instance of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be ' - 'defined to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the ' - 'reflected method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in ' - 'unexpected errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ' - '("-", "+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' - ' "__index__()" is defined "__int__()" should also be ' - 'defined, and\n' - ' both should return the same value.\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' If "__int__()" is not defined then the built-in function ' - '"int()"\n' - ' falls back to "__trunc__()".\n', - 'objects': 'Objects, values and types\n' - '*************************\n' - '\n' - '*Objects* are Python’s abstraction for data. All data in a ' - 'Python\n' - 'program is represented by objects or by relations between ' - 'objects. (In\n' - 'a sense, and in conformance to Von Neumann’s model of a “stored\n' - 'program computer,” code is also represented by objects.)\n' - '\n' - 'Every object has an identity, a type and a value. An object’s\n' - '*identity* never changes once it has been created; you may think ' - 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' - 'the\n' - 'identity of two objects; the "id()" function returns an integer\n' - 'representing its identity.\n' - '\n' - '**CPython implementation detail:** For CPython, "id(x)" is the ' - 'memory\n' - 'address where "x" is stored.\n' - '\n' - 'An object’s type determines the operations that the object ' - 'supports\n' - '(e.g., “does it have a length?”) and also defines the possible ' - 'values\n' - 'for objects of that type. The "type()" function returns an ' - 'object’s\n' - 'type (which is an object itself). Like its identity, an ' - 'object’s\n' - '*type* is also unchangeable. [1]\n' - '\n' - 'The *value* of some objects can change. Objects whose value can\n' - 'change are said to be *mutable*; objects whose value is ' - 'unchangeable\n' - 'once they are created are called *immutable*. (The value of an\n' - 'immutable container object that contains a reference to a ' - 'mutable\n' - 'object can change when the latter’s value is changed; however ' - 'the\n' - 'container is still considered immutable, because the collection ' - 'of\n' - 'objects it contains cannot be changed. So, immutability is not\n' - 'strictly the same as having an unchangeable value, it is more ' - 'subtle.)\n' - 'An object’s mutability is determined by its type; for instance,\n' - 'numbers, strings and tuples are immutable, while dictionaries ' - 'and\n' - 'lists are mutable.\n' - '\n' - 'Objects are never explicitly destroyed; however, when they ' - 'become\n' - 'unreachable they may be garbage-collected. An implementation is\n' - 'allowed to postpone garbage collection or omit it altogether — it ' - 'is a\n' - 'matter of implementation quality how garbage collection is\n' - 'implemented, as long as no objects are collected that are still\n' - 'reachable.\n' - '\n' - '**CPython implementation detail:** CPython currently uses a ' - 'reference-\n' - 'counting scheme with (optional) delayed detection of cyclically ' - 'linked\n' - 'garbage, which collects most objects as soon as they become\n' - 'unreachable, but is not guaranteed to collect garbage containing\n' - 'circular references. See the documentation of the "gc" module ' - 'for\n' - 'information on controlling the collection of cyclic garbage. ' - 'Other\n' - 'implementations act differently and CPython may change. Do not ' - 'depend\n' - 'on immediate finalization of objects when they become unreachable ' - '(so\n' - 'you should always close files explicitly).\n' - '\n' - 'Note that the use of the implementation’s tracing or debugging\n' - 'facilities may keep objects alive that would normally be ' - 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' - 'statement\n' - 'may keep objects alive.\n' - '\n' - 'Some objects contain references to “external” resources such as ' - 'open\n' - 'files or windows. It is understood that these resources are ' - 'freed\n' - 'when the object is garbage-collected, but since garbage ' - 'collection is\n' - 'not guaranteed to happen, such objects also provide an explicit ' - 'way to\n' - 'release the external resource, usually a "close()" method. ' - 'Programs\n' - 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' - '\n' - 'Some objects contain references to other objects; these are ' - 'called\n' - '*containers*. Examples of containers are tuples, lists and\n' - 'dictionaries. The references are part of a container’s value. ' - 'In\n' - 'most cases, when we talk about the value of a container, we imply ' - 'the\n' - 'values, not the identities of the contained objects; however, ' - 'when we\n' - 'talk about the mutability of a container, only the identities of ' - 'the\n' - 'immediately contained objects are implied. So, if an immutable\n' - 'container (like a tuple) contains a reference to a mutable ' - 'object, its\n' - 'value changes if that mutable object is changed.\n' - '\n' - 'Types affect almost all aspects of object behavior. Even the\n' - 'importance of object identity is affected in some sense: for ' - 'immutable\n' - 'types, operations that compute new values may actually return a\n' - 'reference to any existing object with the same type and value, ' - 'while\n' - 'for mutable objects this is not allowed. E.g., after "a = 1; b = ' - '1",\n' - '"a" and "b" may or may not refer to the same object with the ' - 'value\n' - 'one, depending on the implementation, but after "c = []; d = []", ' - '"c"\n' - 'and "d" are guaranteed to refer to two different, unique, newly\n' - 'created empty lists. (Note that "c = d = []" assigns the same ' - 'object\n' - 'to both "c" and "d".)\n', - 'operator-summary': 'Operator precedence\n' - '*******************\n' - '\n' - 'The following table summarizes the operator precedence ' - 'in Python, from\n' - 'lowest precedence (least binding) to highest precedence ' - '(most\n' - 'binding). Operators in the same box have the same ' - 'precedence. Unless\n' - 'the syntax is explicitly given, operators are binary. ' - 'Operators in\n' - 'the same box group left to right (except for ' - 'exponentiation, which\n' - 'groups from right to left).\n' - '\n' - 'Note that comparisons, membership tests, and identity ' - 'tests, all have\n' - 'the same precedence and have a left-to-right chaining ' - 'feature as\n' - 'described in the Comparisons section.\n' - '\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| Operator | ' - 'Description |\n' - '+=================================================+=======================================+\n' - '| "lambda" | ' - 'Lambda expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "if" – "else" | ' - 'Conditional expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "or" | ' - 'Boolean OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "and" | ' - 'Boolean AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "not" "x" | ' - 'Boolean NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "in", "not in", "is", "is not", "<", "<=", ">", | ' - 'Comparisons, including membership |\n' - '| ">=", "!=", "==" | ' - 'tests and identity tests |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "|" | ' - 'Bitwise OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "^" | ' - 'Bitwise XOR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "&" | ' - 'Bitwise AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "<<", ">>" | ' - 'Shifts |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+", "-" | ' - 'Addition and subtraction |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "*", "@", "/", "//", "%" | ' - 'Multiplication, matrix |\n' - '| | ' - 'multiplication, division, floor |\n' - '| | ' - 'division, remainder [5] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+x", "-x", "~x" | ' - 'Positive, negative, bitwise NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "**" | ' - 'Exponentiation [6] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "await" "x" | ' - 'Await expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "x[index]", "x[index:index]", | ' - 'Subscription, slicing, call, |\n' - '| "x(arguments...)", "x.attribute" | ' - 'attribute reference |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "(expressions...)", "[expressions...]", "{key: | ' - 'Binding or tuple display, list |\n' - '| value...}", "{expressions...}" | ' - 'display, dictionary display, set |\n' - '| | ' - 'display |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] While "abs(x%y) < abs(y)" is true mathematically, ' - 'for floats\n' - ' it may not be true numerically due to roundoff. For ' - 'example, and\n' - ' assuming a platform on which a Python float is an ' - 'IEEE 754 double-\n' - ' precision number, in order that "-1e-100 % 1e100" ' - 'have the same\n' - ' sign as "1e100", the computed result is "-1e-100 + ' - '1e100", which\n' - ' is numerically exactly equal to "1e100". The ' - 'function\n' - ' "math.fmod()" returns a result whose sign matches ' - 'the sign of the\n' - ' first argument instead, and so returns "-1e-100" in ' - 'this case.\n' - ' Which approach is more appropriate depends on the ' - 'application.\n' - '\n' - '[2] If x is very close to an exact integer multiple of ' - 'y, it’s\n' - ' possible for "x//y" to be one larger than ' - '"(x-x%y)//y" due to\n' - ' rounding. In such cases, Python returns the latter ' - 'result, in\n' - ' order to preserve that "divmod(x,y)[0] * y + x % y" ' - 'be very close\n' - ' to "x".\n' - '\n' - '[3] The Unicode standard distinguishes between *code ' - 'points* (e.g.\n' - ' U+0041) and *abstract characters* (e.g. “LATIN ' - 'CAPITAL LETTER A”).\n' - ' While most abstract characters in Unicode are only ' - 'represented\n' - ' using one code point, there is a number of abstract ' - 'characters\n' - ' that can in addition be represented using a sequence ' - 'of more than\n' - ' one code point. For example, the abstract character ' - '“LATIN\n' - ' CAPITAL LETTER C WITH CEDILLA” can be represented as ' - 'a single\n' - ' *precomposed character* at code position U+00C7, or ' - 'as a sequence\n' - ' of a *base character* at code position U+0043 (LATIN ' - 'CAPITAL\n' - ' LETTER C), followed by a *combining character* at ' - 'code position\n' - ' U+0327 (COMBINING CEDILLA).\n' - '\n' - ' The comparison operators on strings compare at the ' - 'level of\n' - ' Unicode code points. This may be counter-intuitive ' - 'to humans. For\n' - ' example, ""\\u00C7" == "\\u0043\\u0327"" is "False", ' - 'even though both\n' - ' strings represent the same abstract character “LATIN ' - 'CAPITAL\n' - ' LETTER C WITH CEDILLA”.\n' - '\n' - ' To compare strings at the level of abstract ' - 'characters (that is,\n' - ' in a way intuitive to humans), use ' - '"unicodedata.normalize()".\n' - '\n' - '[4] Due to automatic garbage-collection, free lists, and ' - 'the\n' - ' dynamic nature of descriptors, you may notice ' - 'seemingly unusual\n' - ' behaviour in certain uses of the "is" operator, like ' - 'those\n' - ' involving comparisons between instance methods, or ' - 'constants.\n' - ' Check their documentation for more info.\n' - '\n' - '[5] The "%" operator is also used for string formatting; ' - 'the same\n' - ' precedence applies.\n' - '\n' - '[6] The power operator "**" binds less tightly than an ' - 'arithmetic\n' - ' or bitwise unary operator on its right, that is, ' - '"2**-1" is "0.5".\n', - 'pass': 'The "pass" statement\n' - '********************\n' - '\n' - ' pass_stmt ::= "pass"\n' - '\n' - '"pass" is a null operation — when it is executed, nothing happens. ' - 'It\n' - 'is useful as a placeholder when a statement is required ' - 'syntactically,\n' - 'but no code needs to be executed, for example:\n' - '\n' - ' def f(arg): pass # a function that does nothing (yet)\n' - '\n' - ' class C: pass # a class with no methods (yet)\n', - 'power': 'The power operator\n' - '******************\n' - '\n' - 'The power operator binds more tightly than unary operators on its\n' - 'left; it binds less tightly than unary operators on its right. ' - 'The\n' - 'syntax is:\n' - '\n' - ' power ::= (await_expr | primary) ["**" u_expr]\n' - '\n' - 'Thus, in an unparenthesized sequence of power and unary operators, ' - 'the\n' - 'operators are evaluated from right to left (this does not ' - 'constrain\n' - 'the evaluation order for the operands): "-1**2" results in "-1".\n' - '\n' - 'The power operator has the same semantics as the built-in "pow()"\n' - 'function, when called with two arguments: it yields its left ' - 'argument\n' - 'raised to the power of its right argument. The numeric arguments ' - 'are\n' - 'first converted to a common type, and the result is of that type.\n' - '\n' - 'For int operands, the result has the same type as the operands ' - 'unless\n' - 'the second argument is negative; in that case, all arguments are\n' - 'converted to float and a float result is delivered. For example,\n' - '"10**2" returns "100", but "10**-2" returns "0.01".\n' - '\n' - 'Raising "0.0" to a negative power results in a ' - '"ZeroDivisionError".\n' - 'Raising a negative number to a fractional power results in a ' - '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': 'The "raise" statement\n' - '*********************\n' - '\n' - ' raise_stmt ::= "raise" [expression ["from" expression]]\n' - '\n' - 'If no expressions are present, "raise" re-raises the last ' - 'exception\n' - 'that was active in the current scope. If no exception is active ' - 'in\n' - 'the current scope, a "RuntimeError" exception is raised indicating\n' - 'that this is an error.\n' - '\n' - 'Otherwise, "raise" evaluates the first expression as the exception\n' - 'object. It must be either a subclass or an instance of\n' - '"BaseException". If it is a class, the exception instance will be\n' - 'obtained when needed by instantiating the class with no arguments.\n' - '\n' - 'The *type* of the exception is the exception instance’s class, the\n' - '*value* is the instance itself.\n' - '\n' - 'A traceback object is normally created automatically when an ' - 'exception\n' - 'is raised and attached to it as the "__traceback__" attribute, ' - 'which\n' - 'is writable. You can create an exception and set your own traceback ' - 'in\n' - 'one step using the "with_traceback()" exception method (which ' - 'returns\n' - 'the same exception instance, with its traceback set to its ' - 'argument),\n' - 'like so:\n' - '\n' - ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' - '\n' - 'The "from" clause is used for exception chaining: if given, the ' - 'second\n' - '*expression* must be another exception class or instance, which ' - 'will\n' - 'then be attached to the raised exception as the "__cause__" ' - 'attribute\n' - '(which is writable). If the raised exception is not handled, both\n' - 'exceptions will be printed:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except Exception as exc:\n' - ' ... raise RuntimeError("Something bad happened") from exc\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' The above exception was the direct cause of the following ' - 'exception:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'A similar mechanism works implicitly if an exception is raised ' - 'inside\n' - 'an exception handler or a "finally" clause: the previous exception ' - 'is\n' - 'then attached as the new exception’s "__context__" attribute:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened")\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' During handling of the above exception, another exception ' - 'occurred:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Exception chaining can be explicitly suppressed by specifying ' - '"None"\n' - 'in the "from" clause:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened") from None\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information about handling exceptions is in ' - 'section\n' - 'The try statement.\n' - '\n' - 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' - 'automatic display of the exception context.\n', - 'return': 'The "return" statement\n' - '**********************\n' - '\n' - ' return_stmt ::= "return" [expression_list]\n' - '\n' - '"return" may only occur syntactically nested in a function ' - 'definition,\n' - 'not within a nested class definition.\n' - '\n' - 'If an expression list is present, it is evaluated, else "None" is\n' - 'substituted.\n' - '\n' - '"return" leaves the current function call with the expression list ' - '(or\n' - '"None") as return value.\n' - '\n' - 'When "return" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'function.\n' - '\n' - 'In a generator function, the "return" statement indicates that ' - 'the\n' - 'generator is done and will cause "StopIteration" to be raised. ' - 'The\n' - 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n' - '\n' - 'In an asynchronous generator function, an empty "return" ' - 'statement\n' - 'indicates that the asynchronous generator is done and will cause\n' - '"StopAsyncIteration" to be raised. A non-empty "return" statement ' - 'is\n' - 'a syntax error in an asynchronous generator function.\n', - 'sequence-types': 'Emulating container types\n' - '*************************\n' - '\n' - 'The following methods can be defined to implement ' - 'container objects.\n' - 'Containers usually are sequences (such as lists or tuples) ' - 'or mappings\n' - '(like dictionaries), but can represent other containers as ' - 'well. The\n' - 'first set of methods is used either to emulate a sequence ' - 'or to\n' - 'emulate a mapping; the difference is that for a sequence, ' - 'the\n' - 'allowable keys should be the integers *k* for which "0 <= ' - 'k < N" where\n' - '*N* is the length of the sequence, or slice objects, which ' - 'define a\n' - 'range of items. It is also recommended that mappings ' - 'provide the\n' - 'methods "keys()", "values()", "items()", "get()", ' - '"clear()",\n' - '"setdefault()", "pop()", "popitem()", "copy()", and ' - '"update()"\n' - 'behaving similar to those for Python’s standard dictionary ' - 'objects.\n' - 'The "collections" module provides a "MutableMapping" ' - 'abstract base\n' - 'class to help create those methods from a base set of ' - '"__getitem__()",\n' - '"__setitem__()", "__delitem__()", and "keys()". Mutable ' - 'sequences\n' - 'should provide methods "append()", "count()", "index()", ' - '"extend()",\n' - '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' - 'like Python\n' - 'standard list objects. Finally, sequence types should ' - 'implement\n' - 'addition (meaning concatenation) and multiplication ' - '(meaning\n' - 'repetition) by defining the methods "__add__()", ' - '"__radd__()",\n' - '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' - 'described\n' - 'below; they should not define other numerical operators. ' - 'It is\n' - 'recommended that both mappings and sequences implement ' - 'the\n' - '"__contains__()" method to allow efficient use of the "in" ' - 'operator;\n' - 'for mappings, "in" should search the mapping’s keys; for ' - 'sequences, it\n' - 'should search through the values. It is further ' - 'recommended that both\n' - 'mappings and sequences implement the "__iter__()" method ' - 'to allow\n' - 'efficient iteration through the container; for mappings, ' - '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the ' - 'length is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may ' - 'raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater ' - 'or less than\n' - ' the actual length). The length must be an integer ">=" ' - '0. This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - 'sequence types,\n' - ' the accepted keys should be integers and slice ' - 'objects. Note that\n' - ' the special interpretation of negative indexes (if the ' - 'class wishes\n' - ' to emulate a sequence type) is up to the ' - '"__getitem__()" method. If\n' - ' *key* is of an inappropriate type, "TypeError" may be ' - 'raised; if of\n' - ' a value outside the set of indexes for the sequence ' - '(after any\n' - ' special interpretation of negative values), ' - '"IndexError" should be\n' - ' raised. For mapping types, if *key* is missing (not in ' - 'the\n' - ' container), "KeyError" should be raised.\n' - '\n' - ' Note: "for" loops expect that an "IndexError" will be ' - 'raised for\n' - ' illegal indexes to allow proper detection of the end ' - 'of the\n' - ' sequence.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same ' - 'note as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or ' - 'if new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences ' - 'if elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement ' - '"self[key]" for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an iterator is required for ' - 'a container.\n' - ' This method should return a new iterator object that ' - 'can iterate\n' - ' over all the objects in the container. For mappings, ' - 'it should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; ' - 'they are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in ' - 'reverse order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the ' - 'sequence protocol\n' - ' should only provide "__reversed__()" if they can ' - 'provide an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a sequence. However, ' - 'container\n' - 'objects can supply the following special method with a ' - 'more efficient\n' - 'implementation, which also does not require the object be ' - 'a sequence.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this ' - 'section in the\n' - ' language reference.\n', - 'shifting': 'Shifting operations\n' - '*******************\n' - '\n' - 'The shifting operations have lower priority than the arithmetic\n' - 'operations:\n' - '\n' - ' shift_expr ::= a_expr | shift_expr ("<<" | ">>") a_expr\n' - '\n' - 'These operators accept integers as arguments. They shift the ' - 'first\n' - 'argument to the left or right by the number of bits given by ' - 'the\n' - 'second argument.\n' - '\n' - 'A right shift by *n* bits is defined as floor division by ' - '"pow(2,n)".\n' - 'A left shift by *n* bits is defined as multiplication with ' - '"pow(2,n)".\n' - '\n' - 'Note: In the current implementation, the right-hand operand is\n' - ' required to be at most "sys.maxsize". If the right-hand ' - 'operand is\n' - ' larger than "sys.maxsize" an "OverflowError" exception is ' - 'raised.\n', - 'slicings': 'Slicings\n' - '********\n' - '\n' - 'A slicing selects a range of items in a sequence object (e.g., ' - 'a\n' - 'string, tuple or list). Slicings may be used as expressions or ' - 'as\n' - 'targets in assignment or "del" statements. The syntax for a ' - 'slicing:\n' - '\n' - ' slicing ::= primary "[" slice_list "]"\n' - ' slice_list ::= slice_item ("," slice_item)* [","]\n' - ' slice_item ::= expression | proper_slice\n' - ' proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" ' - '[stride] ]\n' - ' lower_bound ::= expression\n' - ' upper_bound ::= expression\n' - ' stride ::= expression\n' - '\n' - 'There is ambiguity in the formal syntax here: anything that ' - 'looks like\n' - 'an expression list also looks like a slice list, so any ' - 'subscription\n' - 'can be interpreted as a slicing. Rather than further ' - 'complicating the\n' - 'syntax, this is disambiguated by defining that in this case the\n' - 'interpretation as a subscription takes priority over the\n' - 'interpretation as a slicing (this is the case if the slice list\n' - 'contains no proper slice).\n' - '\n' - 'The semantics for a slicing are as follows. The primary is ' - 'indexed\n' - '(using the same "__getitem__()" method as normal subscription) ' - 'with a\n' - 'key that is constructed from the slice list, as follows. If the ' - 'slice\n' - 'list contains at least one comma, the key is a tuple containing ' - 'the\n' - 'conversion of the slice items; otherwise, the conversion of the ' - 'lone\n' - 'slice item is the key. The conversion of a slice item that is ' - 'an\n' - 'expression is that expression. The conversion of a proper slice ' - 'is a\n' - 'slice object (see section The standard type hierarchy) whose ' - '"start",\n' - '"stop" and "step" attributes are the values of the expressions ' - 'given\n' - 'as lower bound, upper bound and stride, respectively, ' - 'substituting\n' - '"None" for missing expressions.\n', - 'specialattrs': 'Special Attributes\n' - '******************\n' - '\n' - 'The implementation adds a few special read-only attributes ' - 'to several\n' - 'object types, where they are relevant. Some of these are ' - 'not reported\n' - 'by the "dir()" built-in function.\n' - '\n' - 'object.__dict__\n' - '\n' - ' A dictionary or other mapping object used to store an ' - 'object’s\n' - ' (writable) attributes.\n' - '\n' - 'instance.__class__\n' - '\n' - ' The class to which a class instance belongs.\n' - '\n' - 'class.__bases__\n' - '\n' - ' The tuple of base classes of a class object.\n' - '\n' - 'definition.__name__\n' - '\n' - ' The name of the class, function, method, descriptor, or ' - 'generator\n' - ' instance.\n' - '\n' - 'definition.__qualname__\n' - '\n' - ' The *qualified name* of the class, function, method, ' - 'descriptor, or\n' - ' generator instance.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'class.__mro__\n' - '\n' - ' This attribute is a tuple of classes that are considered ' - 'when\n' - ' looking for base classes during method resolution.\n' - '\n' - 'class.mro()\n' - '\n' - ' This method can be overridden by a metaclass to customize ' - 'the\n' - ' method resolution order for its instances. It is called ' - 'at class\n' - ' instantiation, and its result is stored in "__mro__".\n' - '\n' - 'class.__subclasses__()\n' - '\n' - ' Each class keeps a list of weak references to its ' - 'immediate\n' - ' subclasses. This method returns a list of all those ' - 'references\n' - ' still alive. Example:\n' - '\n' - ' >>> int.__subclasses__()\n' - " []\n" - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] Additional information on these special methods may be ' - 'found\n' - ' in the Python Reference Manual (Basic customization).\n' - '\n' - '[2] As a consequence, the list "[1, 2]" is considered equal ' - 'to\n' - ' "[1.0, 2.0]", and similarly for tuples.\n' - '\n' - '[3] They must have since the parser can’t tell the type of ' - 'the\n' - ' operands.\n' - '\n' - '[4] Cased characters are those with general category ' - 'property\n' - ' being one of “Lu” (Letter, uppercase), “Ll” (Letter, ' - 'lowercase),\n' - ' or “Lt” (Letter, titlecase).\n' - '\n' - '[5] To format only a tuple you should therefore provide a\n' - ' singleton tuple whose only element is the tuple to be ' - 'formatted.\n', - 'specialnames': 'Special method names\n' - '********************\n' - '\n' - 'A class can implement certain operations that are invoked by ' - 'special\n' - 'syntax (such as arithmetic operations or subscripting and ' - 'slicing) by\n' - 'defining methods with special names. This is Python’s ' - 'approach to\n' - '*operator overloading*, allowing classes to define their own ' - 'behavior\n' - 'with respect to language operators. For instance, if a ' - 'class defines\n' - 'a method named "__getitem__()", and "x" is an instance of ' - 'this class,\n' - 'then "x[i]" is roughly equivalent to "type(x).__getitem__(x, ' - 'i)".\n' - 'Except where mentioned, attempts to execute an operation ' - 'raise an\n' - 'exception when no appropriate method is defined (typically\n' - '"AttributeError" or "TypeError").\n' - '\n' - 'Setting a special method to "None" indicates that the ' - 'corresponding\n' - 'operation is not available. For example, if a class sets ' - '"__iter__()"\n' - 'to "None", the class is not iterable, so calling "iter()" on ' - 'its\n' - 'instances will raise a "TypeError" (without falling back to\n' - '"__getitem__()"). [2]\n' - '\n' - 'When implementing a class that emulates any built-in type, ' - 'it is\n' - 'important that the emulation only be implemented to the ' - 'degree that it\n' - 'makes sense for the object being modelled. For example, ' - 'some\n' - 'sequences may work well with retrieval of individual ' - 'elements, but\n' - 'extracting a slice may not make sense. (One example of this ' - 'is the\n' - '"NodeList" interface in the W3C’s Document Object Model.)\n' - '\n' - '\n' - 'Basic customization\n' - '===================\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those passed ' - 'to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly-created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" returns an instance of *cls*, then the ' - 'new\n' - ' instance’s "__init__()" method will be invoked like\n' - ' "__init__(self[, ...])", where *self* is the new instance ' - 'and the\n' - ' remaining arguments are the same as were passed to ' - '"__new__()".\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, then ' - 'the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by "__init__()"; ' - 'doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s "__del__()" ' - 'method,\n' - ' if any, must explicitly call it to ensure proper deletion ' - 'of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a new ' - 'reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected and ' - 'deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning: Due to the precarious circumstances under which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. The ' - 'return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation of ' - 'an\n' - ' object. The "format_spec" argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the "format_spec" argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of "object" ' - 'itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are returned ' - 'for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "__ne__()" delegates to "__eq__()" and ' - 'inverts the\n' - ' result unless it is "NotImplemented". There are no other ' - 'implied\n' - ' relationships among the comparison operators, for ' - 'example, the\n' - ' truth of "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, collections.Hashable)" ' - 'call.\n' - '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' - ' Although they remain constant within an individual ' - 'Python\n' - ' process, they are not predictable between repeated ' - 'invocations of\n' - ' Python.This is intended to provide protection against a ' - 'denial-\n' - ' of-service caused by carefully-chosen inputs that ' - 'exploit the\n' - ' worst case performance of a dict insertion, O(n^2) ' - 'complexity.\n' - ' See http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' - ' details.Changing hash values affects the iteration ' - 'order of\n' - ' dicts, sets and other mappings. Python has never made ' - 'guarantees\n' - ' about this ordering (and it typically varies between ' - '32-bit and\n' - ' 64-bit builds).See also "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and the ' - 'object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n' - '\n' - '\n' - 'Customizing attribute access\n' - '============================\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with an\n' - ' "AttributeError" (either "__getattribute__()" raises an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or "__get__()" of ' - 'a *name*\n' - ' property raises "AttributeError"). This method should ' - 'either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the normal ' - 'mechanism,\n' - ' "__getattr__()" is not called. (This is an intentional ' - 'asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise "__getattr__()" ' - 'would have\n' - ' no way to access other attributes of the instance. Note ' - 'that at\n' - ' least for instance variables, you can fake total control ' - 'by not\n' - ' inserting any values in the instance attribute dictionary ' - '(but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to actually ' - 'get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute accesses ' - 'for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless "__getattribute__()" ' - 'either\n' - ' calls it explicitly or raises an "AttributeError". This ' - 'method\n' - ' should return the (computed) attribute value or raise an\n' - ' "AttributeError" exception. In order to avoid infinite ' - 'recursion in\n' - ' this method, its implementation should always call the ' - 'base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note: This method may still be bypassed when looking up ' - 'special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. This ' - 'is called\n' - ' instead of the normal mechanism (i.e. store the value in ' - 'the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, for ' - 'example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion instead ' - 'of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. A sequence ' - 'must be\n' - ' returned. "dir()" converts the returned sequence to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '-----------------------------------\n' - '\n' - 'For a more fine grained customization of the module behavior ' - '(setting\n' - 'attributes, properties, etc.), one can set the "__class__" ' - 'attribute\n' - 'of a module object to a subclass of "types.ModuleType". For ' - 'example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' setattr(self, attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note: Setting module "__class__" only affects lookups made ' - 'using the\n' - ' attribute access syntax – directly accessing the module ' - 'globals\n' - ' (whether by code within the module, or via a reference to ' - 'the\n' - ' module’s globals dictionary) is unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is now ' - 'writable.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '------------------------\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the owner’s ' - 'class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). *owner* is always the owner class, while ' - '*instance* is the\n' - ' instance that the attribute was accessed through, or ' - '"None" when\n' - ' the attribute is accessed through the *owner*. This ' - 'method should\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* of ' - 'the owner\n' - ' class to a new value, *value*.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance *instance* ' - 'of the\n' - ' owner class.\n' - '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is created. ' - 'The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' New in version 3.6.\n' - '\n' - 'The attribute "__objclass__" is interpreted by the "inspect" ' - 'module as\n' - 'specifying the class where this object was defined (setting ' - 'this\n' - 'appropriately can assist in runtime introspection of dynamic ' - 'class\n' - 'attributes). For callables, it may indicate that an instance ' - 'of the\n' - 'given type (or a subclass) is expected or required as the ' - 'first\n' - 'positional argument (for example, CPython sets this ' - 'attribute for\n' - 'unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '--------------------\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been overridden by ' - 'methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", and\n' - '"__delete__()". If any of those methods are defined for an ' - 'object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, set, or ' - 'delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the base ' - 'classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining one of ' - 'the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs in ' - 'the\n' - 'precedence chain depends on which descriptor methods were ' - 'defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a binding, ' - '"a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is transformed ' - 'into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' If "a" is an instance of "super", then the binding ' - '"super(B,\n' - ' obj).m()" searches "obj.__class__.__mro__" for the base ' - 'class "A"\n' - ' immediately preceding "B" and then invokes the descriptor ' - 'with the\n' - ' call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on the which descriptor methods are defined. A descriptor ' - 'can define\n' - 'any combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' - 'does not define "__get__()", then accessing the attribute ' - 'will return\n' - 'the descriptor object itself unless there is a value in the ' - 'object’s\n' - 'instance dictionary. If the descriptor defines "__set__()" ' - 'and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors define ' - 'both\n' - '"__get__()" and "__set__()", while non-data descriptors have ' - 'just the\n' - '"__get__()" method. Data descriptors with "__set__()" and ' - '"__get__()"\n' - 'defined always override a redefinition in an instance ' - 'dictionary. In\n' - 'contrast, non-data descriptors can be overridden by ' - 'instances.\n' - '\n' - 'Python methods (including "staticmethod()" and ' - '"classmethod()") are\n' - 'implemented as non-data descriptors. Accordingly, instances ' - 'can\n' - 'redefine and override methods. This allows individual ' - 'instances to\n' - 'acquire behaviors that differ from other instances of the ' - 'same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '---------\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of *__dict__* and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available in a ' - 'parent.)\n' - '\n' - 'The space saved over using *__dict__* can be significant.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, iterable, ' - 'or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and prevents ' - 'the\n' - ' automatic creation of *__dict__* and *__weakref__* for ' - 'each\n' - ' instance.\n' - '\n' - '\n' - 'Notes on using *__slots__*\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will always ' - 'be\n' - ' accessible.\n' - '\n' - '* Without a *__dict__* variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors (Implementing Descriptors) for each variable ' - 'name. As a\n' - ' result, class attributes cannot be used to set default ' - 'values for\n' - ' instance variables defined by *__slots__*; otherwise, the ' - 'class\n' - ' attribute would overwrite the descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited to ' - 'the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses will ' - 'get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' - '\n' - '* If a class defines a slot also defined in a base class, ' - 'the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In the ' - 'future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", "bytes" ' - 'and "tuple".\n' - '\n' - '* Any non-string iterable may be assigned to *__slots__*. ' - 'Mappings\n' - ' may also be used; however, in the future, special meaning ' - 'may be\n' - ' assigned to the values corresponding to each key.\n' - '\n' - '* *__class__* assignment works only if both classes have the ' - 'same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent classes ' - 'can be\n' - ' used, but only one parent is allowed to have attributes ' - 'created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n' - '\n' - '\n' - 'Customizing class creation\n' - '==========================\n' - '\n' - 'Whenever a class inherits from another class, ' - '*__init_subclass__* is\n' - 'called on that class. This way, it is possible to write ' - 'classes which\n' - 'change the behavior of subclasses. This is closely related ' - 'to class\n' - 'decorators, but where class decorators only affect the ' - 'specific class\n' - 'they’re applied to, "__init_subclass__" solely applies to ' - 'future\n' - 'subclasses of the class defining the method.\n' - '\n' - 'classmethod object.__init_subclass__(cls)\n' - '\n' - ' This method is called whenever the containing class is ' - 'subclassed.\n' - ' *cls* is then the new subclass. If defined as a normal ' - 'instance\n' - ' method, this method is implicitly converted to a class ' - 'method.\n' - '\n' - ' Keyword arguments which are given to a new class are ' - 'passed to the\n' - ' parent’s class "__init_subclass__". For compatibility ' - 'with other\n' - ' classes using "__init_subclass__", one should take out ' - 'the needed\n' - ' keyword arguments and pass the others over to the base ' - 'class, as\n' - ' in:\n' - '\n' - ' class Philosopher:\n' - ' def __init_subclass__(cls, default_name, ' - '**kwargs):\n' - ' super().__init_subclass__(**kwargs)\n' - ' cls.default_name = default_name\n' - '\n' - ' class AustralianPhilosopher(Philosopher, ' - 'default_name="Bruce"):\n' - ' pass\n' - '\n' - ' The default implementation "object.__init_subclass__" ' - 'does nothing,\n' - ' but raises an error if it is called with any arguments.\n' - '\n' - ' Note: The metaclass hint "metaclass" is consumed by the ' - 'rest of\n' - ' the type machinery, and is never passed to ' - '"__init_subclass__"\n' - ' implementations. The actual metaclass (rather than the ' - 'explicit\n' - ' hint) can be accessed as "type(cls)".\n' - '\n' - ' New in version 3.6.\n' - '\n' - '\n' - 'Metaclasses\n' - '-----------\n' - '\n' - 'By default, classes are constructed using "type()". The ' - 'class body is\n' - 'executed in a new namespace and the class name is bound ' - 'locally to the\n' - 'result of "type(name, bases, namespace)".\n' - '\n' - 'The class creation process can be customized by passing the\n' - '"metaclass" keyword argument in the class definition line, ' - 'or by\n' - 'inheriting from an existing class that included such an ' - 'argument. In\n' - 'the following example, both "MyClass" and "MySubclass" are ' - 'instances\n' - 'of "Meta":\n' - '\n' - ' class Meta(type):\n' - ' pass\n' - '\n' - ' class MyClass(metaclass=Meta):\n' - ' pass\n' - '\n' - ' class MySubclass(MyClass):\n' - ' pass\n' - '\n' - 'Any other keyword arguments that are specified in the class ' - 'definition\n' - 'are passed through to all metaclass operations described ' - 'below.\n' - '\n' - 'When a class definition is executed, the following steps ' - 'occur:\n' - '\n' - '* the appropriate metaclass is determined\n' - '\n' - '* the class namespace is prepared\n' - '\n' - '* the class body is executed\n' - '\n' - '* the class object is created\n' - '\n' - '\n' - 'Determining the appropriate metaclass\n' - '-------------------------------------\n' - '\n' - 'The appropriate metaclass for a class definition is ' - 'determined as\n' - 'follows:\n' - '\n' - '* if no bases and no explicit metaclass are given, then ' - '"type()" is\n' - ' used\n' - '\n' - '* if an explicit metaclass is given and it is *not* an ' - 'instance of\n' - ' "type()", then it is used directly as the metaclass\n' - '\n' - '* if an instance of "type()" is given as the explicit ' - 'metaclass, or\n' - ' bases are defined, then the most derived metaclass is ' - 'used\n' - '\n' - 'The most derived metaclass is selected from the explicitly ' - 'specified\n' - 'metaclass (if any) and the metaclasses (i.e. "type(cls)") of ' - 'all\n' - 'specified base classes. The most derived metaclass is one ' - 'which is a\n' - 'subtype of *all* of these candidate metaclasses. If none of ' - 'the\n' - 'candidate metaclasses meets that criterion, then the class ' - 'definition\n' - 'will fail with "TypeError".\n' - '\n' - '\n' - 'Preparing the class namespace\n' - '-----------------------------\n' - '\n' - 'Once the appropriate metaclass has been identified, then the ' - 'class\n' - 'namespace is prepared. If the metaclass has a "__prepare__" ' - 'attribute,\n' - 'it is called as "namespace = metaclass.__prepare__(name, ' - 'bases,\n' - '**kwds)" (where the additional keyword arguments, if any, ' - 'come from\n' - 'the class definition).\n' - '\n' - 'If the metaclass has no "__prepare__" attribute, then the ' - 'class\n' - 'namespace is initialised as an empty ordered mapping.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' Introduced the "__prepare__" namespace hook\n' - '\n' - '\n' - 'Executing the class body\n' - '------------------------\n' - '\n' - 'The class body is executed (approximately) as "exec(body, ' - 'globals(),\n' - 'namespace)". The key difference from a normal call to ' - '"exec()" is that\n' - 'lexical scoping allows the class body (including any ' - 'methods) to\n' - 'reference names from the current and outer scopes when the ' - 'class\n' - 'definition occurs inside a function.\n' - '\n' - 'However, even when the class definition occurs inside the ' - 'function,\n' - 'methods defined inside the class still cannot see names ' - 'defined at the\n' - 'class scope. Class variables must be accessed through the ' - 'first\n' - 'parameter of instance or class methods, or through the ' - 'implicit\n' - 'lexically scoped "__class__" reference described in the next ' - 'section.\n' - '\n' - '\n' - 'Creating the class object\n' - '-------------------------\n' - '\n' - 'Once the class namespace has been populated by executing the ' - 'class\n' - 'body, the class object is created by calling ' - '"metaclass(name, bases,\n' - 'namespace, **kwds)" (the additional keywords passed here are ' - 'the same\n' - 'as those passed to "__prepare__").\n' - '\n' - 'This class object is the one that will be referenced by the ' - 'zero-\n' - 'argument form of "super()". "__class__" is an implicit ' - 'closure\n' - 'reference created by the compiler if any methods in a class ' - 'body refer\n' - 'to either "__class__" or "super". This allows the zero ' - 'argument form\n' - 'of "super()" to correctly identify the class being defined ' - 'based on\n' - 'lexical scoping, while the class or instance that was used ' - 'to make the\n' - 'current call is identified based on the first argument ' - 'passed to the\n' - 'method.\n' - '\n' - '**CPython implementation detail:** In CPython 3.6 and later, ' - 'the\n' - '"__class__" cell is passed to the metaclass as a ' - '"__classcell__" entry\n' - 'in the class namespace. If present, this must be propagated ' - 'up to the\n' - '"type.__new__" call in order for the class to be ' - 'initialised\n' - 'correctly. Failing to do so will result in a ' - '"DeprecationWarning" in\n' - 'Python 3.6, and a "RuntimeError" in Python 3.8.\n' - '\n' - 'When using the default metaclass "type", or any metaclass ' - 'that\n' - 'ultimately calls "type.__new__", the following additional\n' - 'customisation steps are invoked after creating the class ' - 'object:\n' - '\n' - '* first, "type.__new__" collects all of the descriptors in ' - 'the class\n' - ' namespace that define a "__set_name__()" method;\n' - '\n' - '* second, all of these "__set_name__" methods are called ' - 'with the\n' - ' class being defined and the assigned name of that ' - 'particular\n' - ' descriptor; and\n' - '\n' - '* finally, the "__init_subclass__()" hook is called on the ' - 'immediate\n' - ' parent of the new class in its method resolution order.\n' - '\n' - 'After the class object is created, it is passed to the ' - 'class\n' - 'decorators included in the class definition (if any) and the ' - 'resulting\n' - 'object is bound in the local namespace as the defined ' - 'class.\n' - '\n' - 'When a new class is created by "type.__new__", the object ' - 'provided as\n' - 'the namespace parameter is copied to a new ordered mapping ' - 'and the\n' - 'original object is discarded. The new copy is wrapped in a ' - 'read-only\n' - 'proxy, which becomes the "__dict__" attribute of the class ' - 'object.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3135** - New super\n' - ' Describes the implicit "__class__" closure reference\n' - '\n' - '\n' - 'Uses for metaclasses\n' - '--------------------\n' - '\n' - 'The potential uses for metaclasses are boundless. Some ideas ' - 'that have\n' - 'been explored include enum, logging, interface checking, ' - 'automatic\n' - 'delegation, automatic property creation, proxies, ' - 'frameworks, and\n' - 'automatic resource locking/synchronization.\n' - '\n' - '\n' - 'Customizing instance and subclass checks\n' - '========================================\n' - '\n' - 'The following methods are used to override the default ' - 'behavior of the\n' - '"isinstance()" and "issubclass()" built-in functions.\n' - '\n' - 'In particular, the metaclass "abc.ABCMeta" implements these ' - 'methods in\n' - 'order to allow the addition of Abstract Base Classes (ABCs) ' - 'as\n' - '“virtual base classes” to any class or type (including ' - 'built-in\n' - 'types), including other ABCs.\n' - '\n' - 'class.__instancecheck__(self, instance)\n' - '\n' - ' Return true if *instance* should be considered a (direct ' - 'or\n' - ' indirect) instance of *class*. If defined, called to ' - 'implement\n' - ' "isinstance(instance, class)".\n' - '\n' - 'class.__subclasscheck__(self, subclass)\n' - '\n' - ' Return true if *subclass* should be considered a (direct ' - 'or\n' - ' indirect) subclass of *class*. If defined, called to ' - 'implement\n' - ' "issubclass(subclass, class)".\n' - '\n' - 'Note that these methods are looked up on the type ' - '(metaclass) of a\n' - 'class. They cannot be defined as class methods in the ' - 'actual class.\n' - 'This is consistent with the lookup of special methods that ' - 'are called\n' - 'on instances, only in this case the instance is itself a ' - 'class.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3119** - Introducing Abstract Base Classes\n' - ' Includes the specification for customizing ' - '"isinstance()" and\n' - ' "issubclass()" behavior through "__instancecheck__()" ' - 'and\n' - ' "__subclasscheck__()", with motivation for this ' - 'functionality in\n' - ' the context of adding Abstract Base Classes (see the ' - '"abc"\n' - ' module) to the language.\n' - '\n' - '\n' - 'Emulating callable objects\n' - '==========================\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' - ' "x.__call__(arg1, arg2, ...)".\n' - '\n' - '\n' - 'Emulating container types\n' - '=========================\n' - '\n' - 'The following methods can be defined to implement container ' - 'objects.\n' - 'Containers usually are sequences (such as lists or tuples) ' - 'or mappings\n' - '(like dictionaries), but can represent other containers as ' - 'well. The\n' - 'first set of methods is used either to emulate a sequence or ' - 'to\n' - 'emulate a mapping; the difference is that for a sequence, ' - 'the\n' - 'allowable keys should be the integers *k* for which "0 <= k ' - '< N" where\n' - '*N* is the length of the sequence, or slice objects, which ' - 'define a\n' - 'range of items. It is also recommended that mappings ' - 'provide the\n' - 'methods "keys()", "values()", "items()", "get()", ' - '"clear()",\n' - '"setdefault()", "pop()", "popitem()", "copy()", and ' - '"update()"\n' - 'behaving similar to those for Python’s standard dictionary ' - 'objects.\n' - 'The "collections" module provides a "MutableMapping" ' - 'abstract base\n' - 'class to help create those methods from a base set of ' - '"__getitem__()",\n' - '"__setitem__()", "__delitem__()", and "keys()". Mutable ' - 'sequences\n' - 'should provide methods "append()", "count()", "index()", ' - '"extend()",\n' - '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' - 'like Python\n' - 'standard list objects. Finally, sequence types should ' - 'implement\n' - 'addition (meaning concatenation) and multiplication ' - '(meaning\n' - 'repetition) by defining the methods "__add__()", ' - '"__radd__()",\n' - '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' - 'described\n' - 'below; they should not define other numerical operators. It ' - 'is\n' - 'recommended that both mappings and sequences implement the\n' - '"__contains__()" method to allow efficient use of the "in" ' - 'operator;\n' - 'for mappings, "in" should search the mapping’s keys; for ' - 'sequences, it\n' - 'should search through the values. It is further recommended ' - 'that both\n' - 'mappings and sequences implement the "__iter__()" method to ' - 'allow\n' - 'efficient iteration through the container; for mappings, ' - '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the length ' - 'is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater or ' - 'less than\n' - ' the actual length). The length must be an integer ">=" 0. ' - 'This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - 'sequence types,\n' - ' the accepted keys should be integers and slice objects. ' - 'Note that\n' - ' the special interpretation of negative indexes (if the ' - 'class wishes\n' - ' to emulate a sequence type) is up to the "__getitem__()" ' - 'method. If\n' - ' *key* is of an inappropriate type, "TypeError" may be ' - 'raised; if of\n' - ' a value outside the set of indexes for the sequence ' - '(after any\n' - ' special interpretation of negative values), "IndexError" ' - 'should be\n' - ' raised. For mapping types, if *key* is missing (not in ' - 'the\n' - ' container), "KeyError" should be raised.\n' - '\n' - ' Note: "for" loops expect that an "IndexError" will be ' - 'raised for\n' - ' illegal indexes to allow proper detection of the end of ' - 'the\n' - ' sequence.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or if ' - 'new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences if ' - 'elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement "self[key]" ' - 'for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an iterator is required for a ' - 'container.\n' - ' This method should return a new iterator object that can ' - 'iterate\n' - ' over all the objects in the container. For mappings, it ' - 'should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; they ' - 'are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in reverse ' - 'order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the sequence ' - 'protocol\n' - ' should only provide "__reversed__()" if they can provide ' - 'an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a sequence. However, ' - 'container\n' - 'objects can supply the following special method with a more ' - 'efficient\n' - 'implementation, which also does not require the object be a ' - 'sequence.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this section ' - 'in the\n' - ' language reference.\n' - '\n' - '\n' - 'Emulating numeric types\n' - '=======================\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, ' - 'to\n' - ' evaluate the expression "x + y", where *x* is an instance ' - 'of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be defined ' - 'to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the reflected ' - 'method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in unexpected ' - 'errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ("-", ' - '"+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' - ' "__index__()" is defined "__int__()" should also be ' - 'defined, and\n' - ' both should return the same value.\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' If "__int__()" is not defined then the built-in function ' - '"int()"\n' - ' falls back to "__trunc__()".\n' - '\n' - '\n' - 'With Statement Context Managers\n' - '===============================\n' - '\n' - 'A *context manager* is an object that defines the runtime ' - 'context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context managers ' - 'are normally\n' - 'invoked using the "with" statement (described in section The ' - 'with\n' - 'statement), but can also be used by directly invoking their ' - 'methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), it ' - 'should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n' - '\n' - '\n' - 'Special method lookup\n' - '=====================\n' - '\n' - 'For custom classes, implicit invocations of special methods ' - 'are only\n' - 'guaranteed to work correctly if defined on an object’s type, ' - 'not in\n' - 'the object’s instance dictionary. That behaviour is the ' - 'reason why\n' - 'the following code raises an exception:\n' - '\n' - ' >>> class C:\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__ = lambda: 5\n' - ' >>> len(c)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: object of type 'C' has no len()\n" - '\n' - 'The rationale behind this behaviour lies with a number of ' - 'special\n' - 'methods such as "__hash__()" and "__repr__()" that are ' - 'implemented by\n' - 'all objects, including type objects. If the implicit lookup ' - 'of these\n' - 'methods used the conventional lookup process, they would ' - 'fail when\n' - 'invoked on the type object itself:\n' - '\n' - ' >>> 1 .__hash__() == hash(1)\n' - ' True\n' - ' >>> int.__hash__() == hash(int)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: descriptor '__hash__' of 'int' object needs an " - 'argument\n' - '\n' - 'Incorrectly attempting to invoke an unbound method of a ' - 'class in this\n' - 'way is sometimes referred to as ‘metaclass confusion’, and ' - 'is avoided\n' - 'by bypassing the instance when looking up special methods:\n' - '\n' - ' >>> type(1).__hash__(1) == hash(1)\n' - ' True\n' - ' >>> type(int).__hash__(int) == hash(int)\n' - ' True\n' - '\n' - 'In addition to bypassing any instance attributes in the ' - 'interest of\n' - 'correctness, implicit special method lookup generally also ' - 'bypasses\n' - 'the "__getattribute__()" method even of the object’s ' - 'metaclass:\n' - '\n' - ' >>> class Meta(type):\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Metaclass getattribute invoked")\n' - ' ... return type.__getattribute__(*args)\n' - ' ...\n' - ' >>> class C(object, metaclass=Meta):\n' - ' ... def __len__(self):\n' - ' ... return 10\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Class getattribute invoked")\n' - ' ... return object.__getattribute__(*args)\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__() # Explicit lookup via ' - 'instance\n' - ' Class getattribute invoked\n' - ' 10\n' - ' >>> type(c).__len__(c) # Explicit lookup via ' - 'type\n' - ' Metaclass getattribute invoked\n' - ' 10\n' - ' >>> len(c) # Implicit lookup\n' - ' 10\n' - '\n' - 'Bypassing the "__getattribute__()" machinery in this fashion ' - 'provides\n' - 'significant scope for speed optimisations within the ' - 'interpreter, at\n' - 'the cost of some flexibility in the handling of special ' - 'methods (the\n' - 'special method *must* be set on the class object itself in ' - 'order to be\n' - 'consistently invoked by the interpreter).\n', - 'string-methods': 'String Methods\n' - '**************\n' - '\n' - 'Strings implement all of the common sequence operations, ' - 'along with\n' - 'the additional methods described below.\n' - '\n' - 'Strings also support two styles of string formatting, one ' - 'providing a\n' - 'large degree of flexibility and customization (see ' - '"str.format()",\n' - 'Format String Syntax and Custom String Formatting) and the ' - 'other based\n' - 'on C "printf" style formatting that handles a narrower ' - 'range of types\n' - 'and is slightly harder to use correctly, but is often ' - 'faster for the\n' - 'cases it can handle (printf-style String Formatting).\n' - '\n' - 'The Text Processing Services section of the standard ' - 'library covers a\n' - 'number of other modules that provide various text related ' - 'utilities\n' - '(including regular expression support in the "re" ' - 'module).\n' - '\n' - 'str.capitalize()\n' - '\n' - ' Return a copy of the string with its first character ' - 'capitalized\n' - ' and the rest lowercased.\n' - '\n' - 'str.casefold()\n' - '\n' - ' Return a casefolded copy of the string. Casefolded ' - 'strings may be\n' - ' used for caseless matching.\n' - '\n' - ' Casefolding is similar to lowercasing but more ' - 'aggressive because\n' - ' it is intended to remove all case distinctions in a ' - 'string. For\n' - ' example, the German lowercase letter "\'ß\'" is ' - 'equivalent to ""ss"".\n' - ' Since it is already lowercase, "lower()" would do ' - 'nothing to "\'ß\'";\n' - ' "casefold()" converts it to ""ss"".\n' - '\n' - ' The casefolding algorithm is described in section 3.13 ' - 'of the\n' - ' Unicode Standard.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'str.center(width[, fillchar])\n' - '\n' - ' Return centered in a string of length *width*. Padding ' - 'is done\n' - ' using the specified *fillchar* (default is an ASCII ' - 'space). The\n' - ' original string is returned if *width* is less than or ' - 'equal to\n' - ' "len(s)".\n' - '\n' - 'str.count(sub[, start[, end]])\n' - '\n' - ' Return the number of non-overlapping occurrences of ' - 'substring *sub*\n' - ' in the range [*start*, *end*]. Optional arguments ' - '*start* and\n' - ' *end* are interpreted as in slice notation.\n' - '\n' - 'str.encode(encoding="utf-8", errors="strict")\n' - '\n' - ' Return an encoded version of the string as a bytes ' - 'object. Default\n' - ' encoding is "\'utf-8\'". *errors* may be given to set a ' - 'different\n' - ' error handling scheme. The default for *errors* is ' - '"\'strict\'",\n' - ' meaning that encoding errors raise a "UnicodeError". ' - 'Other possible\n' - ' values are "\'ignore\'", "\'replace\'", ' - '"\'xmlcharrefreplace\'",\n' - ' "\'backslashreplace\'" and any other name registered ' - 'via\n' - ' "codecs.register_error()", see section Error Handlers. ' - 'For a list\n' - ' of possible encodings, see section Standard Encodings.\n' - '\n' - ' Changed in version 3.1: Support for keyword arguments ' - 'added.\n' - '\n' - 'str.endswith(suffix[, start[, end]])\n' - '\n' - ' Return "True" if the string ends with the specified ' - '*suffix*,\n' - ' otherwise return "False". *suffix* can also be a tuple ' - 'of suffixes\n' - ' to look for. With optional *start*, test beginning at ' - 'that\n' - ' position. With optional *end*, stop comparing at that ' - 'position.\n' - '\n' - 'str.expandtabs(tabsize=8)\n' - '\n' - ' Return a copy of the string where all tab characters ' - 'are replaced\n' - ' by one or more spaces, depending on the current column ' - 'and the\n' - ' given tab size. Tab positions occur every *tabsize* ' - 'characters\n' - ' (default is 8, giving tab positions at columns 0, 8, 16 ' - 'and so on).\n' - ' To expand the string, the current column is set to zero ' - 'and the\n' - ' string is examined character by character. If the ' - 'character is a\n' - ' tab ("\\t"), one or more space characters are inserted ' - 'in the result\n' - ' until the current column is equal to the next tab ' - 'position. (The\n' - ' tab character itself is not copied.) If the character ' - 'is a newline\n' - ' ("\\n") or return ("\\r"), it is copied and the current ' - 'column is\n' - ' reset to zero. Any other character is copied unchanged ' - 'and the\n' - ' current column is incremented by one regardless of how ' - 'the\n' - ' character is represented when printed.\n' - '\n' - " >>> '01\\t012\\t0123\\t01234'.expandtabs()\n" - " '01 012 0123 01234'\n" - " >>> '01\\t012\\t0123\\t01234'.expandtabs(4)\n" - " '01 012 0123 01234'\n" - '\n' - 'str.find(sub[, start[, end]])\n' - '\n' - ' Return the lowest index in the string where substring ' - '*sub* is\n' - ' found within the slice "s[start:end]". Optional ' - 'arguments *start*\n' - ' and *end* are interpreted as in slice notation. Return ' - '"-1" if\n' - ' *sub* is not found.\n' - '\n' - ' Note: The "find()" method should be used only if you ' - 'need to know\n' - ' the position of *sub*. To check if *sub* is a ' - 'substring or not,\n' - ' use the "in" operator:\n' - '\n' - " >>> 'Py' in 'Python'\n" - ' True\n' - '\n' - 'str.format(*args, **kwargs)\n' - '\n' - ' Perform a string formatting operation. The string on ' - 'which this\n' - ' method is called can contain literal text or ' - 'replacement fields\n' - ' delimited by braces "{}". Each replacement field ' - 'contains either\n' - ' the numeric index of a positional argument, or the name ' - 'of a\n' - ' keyword argument. Returns a copy of the string where ' - 'each\n' - ' replacement field is replaced with the string value of ' - 'the\n' - ' corresponding argument.\n' - '\n' - ' >>> "The sum of 1 + 2 is {0}".format(1+2)\n' - " 'The sum of 1 + 2 is 3'\n" - '\n' - ' See Format String Syntax for a description of the ' - 'various\n' - ' formatting options that can be specified in format ' - 'strings.\n' - '\n' - ' Note: When formatting a number ("int", "float", ' - '"complex",\n' - ' "decimal.Decimal" and subclasses) with the "n" type ' - '(ex:\n' - ' "\'{:n}\'.format(1234)"), the function temporarily ' - 'sets the\n' - ' "LC_CTYPE" locale to the "LC_NUMERIC" locale to ' - 'decode\n' - ' "decimal_point" and "thousands_sep" fields of ' - '"localeconv()" if\n' - ' they are non-ASCII or longer than 1 byte, and the ' - '"LC_NUMERIC"\n' - ' locale is different than the "LC_CTYPE" locale. This ' - 'temporary\n' - ' change affects other threads.\n' - '\n' - ' Changed in version 3.6.5: When formatting a number with ' - 'the "n"\n' - ' type, the function sets temporarily the "LC_CTYPE" ' - 'locale to the\n' - ' "LC_NUMERIC" locale in some cases.\n' - '\n' - 'str.format_map(mapping)\n' - '\n' - ' Similar to "str.format(**mapping)", except that ' - '"mapping" is used\n' - ' directly and not copied to a "dict". This is useful if ' - 'for example\n' - ' "mapping" is a dict subclass:\n' - '\n' - ' >>> class Default(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return key\n' - ' ...\n' - " >>> '{name} was born in " - "{country}'.format_map(Default(name='Guido'))\n" - " 'Guido was born in country'\n" - '\n' - ' New in version 3.2.\n' - '\n' - 'str.index(sub[, start[, end]])\n' - '\n' - ' Like "find()", but raise "ValueError" when the ' - 'substring is not\n' - ' found.\n' - '\n' - 'str.isalnum()\n' - '\n' - ' Return true if all characters in the string are ' - 'alphanumeric and\n' - ' there is at least one character, false otherwise. A ' - 'character "c"\n' - ' is alphanumeric if one of the following returns ' - '"True":\n' - ' "c.isalpha()", "c.isdecimal()", "c.isdigit()", or ' - '"c.isnumeric()".\n' - '\n' - 'str.isalpha()\n' - '\n' - ' Return true if all characters in the string are ' - 'alphabetic and\n' - ' there is at least one character, false otherwise. ' - 'Alphabetic\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Letter”, i.e., those with general category ' - 'property\n' - ' being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note ' - 'that this is\n' - ' different from the “Alphabetic” property defined in the ' - 'Unicode\n' - ' Standard.\n' - '\n' - 'str.isdecimal()\n' - '\n' - ' Return true if all characters in the string are decimal ' - 'characters\n' - ' and there is at least one character, false otherwise. ' - 'Decimal\n' - ' characters are those that can be used to form numbers ' - 'in base 10,\n' - ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' - 'decimal character\n' - ' is a character in the Unicode General Category “Nd”.\n' - '\n' - 'str.isdigit()\n' - '\n' - ' Return true if all characters in the string are digits ' - 'and there is\n' - ' at least one character, false otherwise. Digits ' - 'include decimal\n' - ' characters and digits that need special handling, such ' - 'as the\n' - ' compatibility superscript digits. This covers digits ' - 'which cannot\n' - ' be used to form numbers in base 10, like the Kharosthi ' - 'numbers.\n' - ' Formally, a digit is a character that has the property ' - 'value\n' - ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' - '\n' - 'str.isidentifier()\n' - '\n' - ' Return true if the string is a valid identifier ' - 'according to the\n' - ' language definition, section Identifiers and keywords.\n' - '\n' - ' Use "keyword.iskeyword()" to test for reserved ' - 'identifiers such as\n' - ' "def" and "class".\n' - '\n' - 'str.islower()\n' - '\n' - ' Return true if all cased characters [4] in the string ' - 'are lowercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' - '\n' - 'str.isnumeric()\n' - '\n' - ' Return true if all characters in the string are numeric ' - 'characters,\n' - ' and there is at least one character, false otherwise. ' - 'Numeric\n' - ' characters include digit characters, and all characters ' - 'that have\n' - ' the Unicode numeric value property, e.g. U+2155, VULGAR ' - 'FRACTION\n' - ' ONE FIFTH. Formally, numeric characters are those with ' - 'the\n' - ' property value Numeric_Type=Digit, Numeric_Type=Decimal ' - 'or\n' - ' Numeric_Type=Numeric.\n' - '\n' - 'str.isprintable()\n' - '\n' - ' Return true if all characters in the string are ' - 'printable or the\n' - ' string is empty, false otherwise. Nonprintable ' - 'characters are\n' - ' those characters defined in the Unicode character ' - 'database as\n' - ' “Other” or “Separator”, excepting the ASCII space ' - '(0x20) which is\n' - ' considered printable. (Note that printable characters ' - 'in this\n' - ' context are those which should not be escaped when ' - '"repr()" is\n' - ' invoked on a string. It has no bearing on the handling ' - 'of strings\n' - ' written to "sys.stdout" or "sys.stderr".)\n' - '\n' - 'str.isspace()\n' - '\n' - ' Return true if there are only whitespace characters in ' - 'the string\n' - ' and there is at least one character, false otherwise. ' - 'Whitespace\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Other” or “Separator” and those with ' - 'bidirectional\n' - ' property being one of “WS”, “B”, or “S”.\n' - '\n' - 'str.istitle()\n' - '\n' - ' Return true if the string is a titlecased string and ' - 'there is at\n' - ' least one character, for example uppercase characters ' - 'may only\n' - ' follow uncased characters and lowercase characters only ' - 'cased ones.\n' - ' Return false otherwise.\n' - '\n' - 'str.isupper()\n' - '\n' - ' Return true if all cased characters [4] in the string ' - 'are uppercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' - '\n' - 'str.join(iterable)\n' - '\n' - ' Return a string which is the concatenation of the ' - 'strings in\n' - ' *iterable*. A "TypeError" will be raised if there are ' - 'any non-\n' - ' string values in *iterable*, including "bytes" ' - 'objects. The\n' - ' separator between elements is the string providing this ' - 'method.\n' - '\n' - 'str.ljust(width[, fillchar])\n' - '\n' - ' Return the string left justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.lower()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to lowercase.\n' - '\n' - ' The lowercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' - '\n' - 'str.lstrip([chars])\n' - '\n' - ' Return a copy of the string with leading characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'prefix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.lstrip()\n" - " 'spacious '\n" - " >>> 'www.example.com'.lstrip('cmowz.')\n" - " 'example.com'\n" - '\n' - 'static str.maketrans(x[, y[, z]])\n' - '\n' - ' This static method returns a translation table usable ' - 'for\n' - ' "str.translate()".\n' - '\n' - ' If there is only one argument, it must be a dictionary ' - 'mapping\n' - ' Unicode ordinals (integers) or characters (strings of ' - 'length 1) to\n' - ' Unicode ordinals, strings (of arbitrary lengths) or ' - '"None".\n' - ' Character keys will then be converted to ordinals.\n' - '\n' - ' If there are two arguments, they must be strings of ' - 'equal length,\n' - ' and in the resulting dictionary, each character in x ' - 'will be mapped\n' - ' to the character at the same position in y. If there ' - 'is a third\n' - ' argument, it must be a string, whose characters will be ' - 'mapped to\n' - ' "None" in the result.\n' - '\n' - 'str.partition(sep)\n' - '\n' - ' Split the string at the first occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing the string itself, ' - 'followed by\n' - ' two empty strings.\n' - '\n' - 'str.replace(old, new[, count])\n' - '\n' - ' Return a copy of the string with all occurrences of ' - 'substring *old*\n' - ' replaced by *new*. If the optional argument *count* is ' - 'given, only\n' - ' the first *count* occurrences are replaced.\n' - '\n' - 'str.rfind(sub[, start[, end]])\n' - '\n' - ' Return the highest index in the string where substring ' - '*sub* is\n' - ' found, such that *sub* is contained within ' - '"s[start:end]".\n' - ' Optional arguments *start* and *end* are interpreted as ' - 'in slice\n' - ' notation. Return "-1" on failure.\n' - '\n' - 'str.rindex(sub[, start[, end]])\n' - '\n' - ' Like "rfind()" but raises "ValueError" when the ' - 'substring *sub* is\n' - ' not found.\n' - '\n' - 'str.rjust(width[, fillchar])\n' - '\n' - ' Return the string right justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.rpartition(sep)\n' - '\n' - ' Split the string at the last occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing two empty strings, ' - 'followed by\n' - ' the string itself.\n' - '\n' - 'str.rsplit(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit* splits\n' - ' are done, the *rightmost* ones. If *sep* is not ' - 'specified or\n' - ' "None", any whitespace string is a separator. Except ' - 'for splitting\n' - ' from the right, "rsplit()" behaves like "split()" which ' - 'is\n' - ' described in detail below.\n' - '\n' - 'str.rstrip([chars])\n' - '\n' - ' Return a copy of the string with trailing characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'suffix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.rstrip()\n" - " ' spacious'\n" - " >>> 'mississippi'.rstrip('ipz')\n" - " 'mississ'\n" - '\n' - 'str.split(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit*\n' - ' splits are done (thus, the list will have at most ' - '"maxsplit+1"\n' - ' elements). If *maxsplit* is not specified or "-1", ' - 'then there is\n' - ' no limit on the number of splits (all possible splits ' - 'are made).\n' - '\n' - ' If *sep* is given, consecutive delimiters are not ' - 'grouped together\n' - ' and are deemed to delimit empty strings (for example,\n' - ' "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', ' - '\'2\']"). The *sep* argument\n' - ' may consist of multiple characters (for example,\n' - ' "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', ' - '\'3\']"). Splitting an\n' - ' empty string with a specified separator returns ' - '"[\'\']".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1,2,3'.split(',')\n" - " ['1', '2', '3']\n" - " >>> '1,2,3'.split(',', maxsplit=1)\n" - " ['1', '2,3']\n" - " >>> '1,2,,3,'.split(',')\n" - " ['1', '2', '', '3', '']\n" - '\n' - ' If *sep* is not specified or is "None", a different ' - 'splitting\n' - ' algorithm is applied: runs of consecutive whitespace ' - 'are regarded\n' - ' as a single separator, and the result will contain no ' - 'empty strings\n' - ' at the start or end if the string has leading or ' - 'trailing\n' - ' whitespace. Consequently, splitting an empty string or ' - 'a string\n' - ' consisting of just whitespace with a "None" separator ' - 'returns "[]".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1 2 3'.split()\n" - " ['1', '2', '3']\n" - " >>> '1 2 3'.split(maxsplit=1)\n" - " ['1', '2 3']\n" - " >>> ' 1 2 3 '.split()\n" - " ['1', '2', '3']\n" - '\n' - 'str.splitlines([keepends])\n' - '\n' - ' Return a list of the lines in the string, breaking at ' - 'line\n' - ' boundaries. Line breaks are not included in the ' - 'resulting list\n' - ' unless *keepends* is given and true.\n' - '\n' - ' This method splits on the following line boundaries. ' - 'In\n' - ' particular, the boundaries are a superset of *universal ' - 'newlines*.\n' - '\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | Representation | ' - 'Description |\n' - ' ' - '+=========================+===============================+\n' - ' | "\\n" | Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r" | Carriage ' - 'Return |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r\\n" | Carriage Return + Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\v" or "\\x0b" | Line ' - 'Tabulation |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\f" or "\\x0c" | Form ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1c" | File ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1d" | Group ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1e" | Record ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x85" | Next Line (C1 Control ' - 'Code) |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2028" | Line ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2029" | Paragraph ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - '\n' - ' Changed in version 3.2: "\\v" and "\\f" added to list ' - 'of line\n' - ' boundaries.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'ab c\\n\\nde fg\\rkl\\r\\n'.splitlines()\n" - " ['ab c', '', 'de fg', 'kl']\n" - " >>> 'ab c\\n\\nde " - "fg\\rkl\\r\\n'.splitlines(keepends=True)\n" - " ['ab c\\n', '\\n', 'de fg\\r', 'kl\\r\\n']\n" - '\n' - ' Unlike "split()" when a delimiter string *sep* is ' - 'given, this\n' - ' method returns an empty list for the empty string, and ' - 'a terminal\n' - ' line break does not result in an extra line:\n' - '\n' - ' >>> "".splitlines()\n' - ' []\n' - ' >>> "One line\\n".splitlines()\n' - " ['One line']\n" - '\n' - ' For comparison, "split(\'\\n\')" gives:\n' - '\n' - " >>> ''.split('\\n')\n" - " ['']\n" - " >>> 'Two lines\\n'.split('\\n')\n" - " ['Two lines', '']\n" - '\n' - 'str.startswith(prefix[, start[, end]])\n' - '\n' - ' Return "True" if string starts with the *prefix*, ' - 'otherwise return\n' - ' "False". *prefix* can also be a tuple of prefixes to ' - 'look for.\n' - ' With optional *start*, test string beginning at that ' - 'position.\n' - ' With optional *end*, stop comparing string at that ' - 'position.\n' - '\n' - 'str.strip([chars])\n' - '\n' - ' Return a copy of the string with the leading and ' - 'trailing\n' - ' characters removed. The *chars* argument is a string ' - 'specifying the\n' - ' set of characters to be removed. If omitted or "None", ' - 'the *chars*\n' - ' argument defaults to removing whitespace. The *chars* ' - 'argument is\n' - ' not a prefix or suffix; rather, all combinations of its ' - 'values are\n' - ' stripped:\n' - '\n' - " >>> ' spacious '.strip()\n" - " 'spacious'\n" - " >>> 'www.example.com'.strip('cmowz.')\n" - " 'example'\n" - '\n' - ' The outermost leading and trailing *chars* argument ' - 'values are\n' - ' stripped from the string. Characters are removed from ' - 'the leading\n' - ' end until reaching a string character that is not ' - 'contained in the\n' - ' set of characters in *chars*. A similar action takes ' - 'place on the\n' - ' trailing end. For example:\n' - '\n' - " >>> comment_string = '#....... Section 3.2.1 Issue " - "#32 .......'\n" - " >>> comment_string.strip('.#! ')\n" - " 'Section 3.2.1 Issue #32'\n" - '\n' - 'str.swapcase()\n' - '\n' - ' Return a copy of the string with uppercase characters ' - 'converted to\n' - ' lowercase and vice versa. Note that it is not ' - 'necessarily true that\n' - ' "s.swapcase().swapcase() == s".\n' - '\n' - 'str.title()\n' - '\n' - ' Return a titlecased version of the string where words ' - 'start with an\n' - ' uppercase character and the remaining characters are ' - 'lowercase.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'Hello world'.title()\n" - " 'Hello World'\n" - '\n' - ' The algorithm uses a simple language-independent ' - 'definition of a\n' - ' word as groups of consecutive letters. The definition ' - 'works in\n' - ' many contexts but it means that apostrophes in ' - 'contractions and\n' - ' possessives form word boundaries, which may not be the ' - 'desired\n' - ' result:\n' - '\n' - ' >>> "they\'re bill\'s friends from the UK".title()\n' - ' "They\'Re Bill\'S Friends From The Uk"\n' - '\n' - ' A workaround for apostrophes can be constructed using ' - 'regular\n' - ' expressions:\n' - '\n' - ' >>> import re\n' - ' >>> def titlecase(s):\n' - ' ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n' - ' ... lambda mo: ' - 'mo.group(0)[0].upper() +\n' - ' ... ' - 'mo.group(0)[1:].lower(),\n' - ' ... s)\n' - ' ...\n' - ' >>> titlecase("they\'re bill\'s friends.")\n' - ' "They\'re Bill\'s Friends."\n' - '\n' - 'str.translate(table)\n' - '\n' - ' Return a copy of the string in which each character has ' - 'been mapped\n' - ' through the given translation table. The table must be ' - 'an object\n' - ' that implements indexing via "__getitem__()", typically ' - 'a *mapping*\n' - ' or *sequence*. When indexed by a Unicode ordinal (an ' - 'integer), the\n' - ' table object can do any of the following: return a ' - 'Unicode ordinal\n' - ' or a string, to map the character to one or more other ' - 'characters;\n' - ' return "None", to delete the character from the return ' - 'string; or\n' - ' raise a "LookupError" exception, to map the character ' - 'to itself.\n' - '\n' - ' You can use "str.maketrans()" to create a translation ' - 'map from\n' - ' character-to-character mappings in different formats.\n' - '\n' - ' See also the "codecs" module for a more flexible ' - 'approach to custom\n' - ' character mappings.\n' - '\n' - 'str.upper()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to uppercase. Note that ' - '"s.upper().isupper()" might be\n' - ' "False" if "s" contains uncased characters or if the ' - 'Unicode\n' - ' category of the resulting character(s) is not “Lu” ' - '(Letter,\n' - ' uppercase), but e.g. “Lt” (Letter, titlecase).\n' - '\n' - ' The uppercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' - '\n' - 'str.zfill(width)\n' - '\n' - ' Return a copy of the string left filled with ASCII ' - '"\'0\'" digits to\n' - ' make a string of length *width*. A leading sign prefix\n' - ' ("\'+\'"/"\'-\'") is handled by inserting the padding ' - '*after* the sign\n' - ' character rather than before. The original string is ' - 'returned if\n' - ' *width* is less than or equal to "len(s)".\n' - '\n' - ' For example:\n' - '\n' - ' >>> "42".zfill(5)\n' - " '00042'\n" - ' >>> "-42".zfill(5)\n' - " '-0042'\n", - 'strings': 'String and Bytes literals\n' - '*************************\n' - '\n' - 'String literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' stringliteral ::= [stringprefix](shortstring | longstring)\n' - ' stringprefix ::= "r" | "u" | "R" | "U" | "f" | "F"\n' - ' | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | ' - '"Rf" | "RF"\n' - ' shortstring ::= "\'" shortstringitem* "\'" | \'"\' ' - 'shortstringitem* \'"\'\n' - ' longstring ::= "\'\'\'" longstringitem* "\'\'\'" | ' - '\'"""\' longstringitem* \'"""\'\n' - ' shortstringitem ::= shortstringchar | stringescapeseq\n' - ' longstringitem ::= longstringchar | stringescapeseq\n' - ' shortstringchar ::= \n' - ' longstringchar ::= \n' - ' stringescapeseq ::= "\\" \n' - '\n' - ' bytesliteral ::= bytesprefix(shortbytes | longbytes)\n' - ' bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | ' - '"rb" | "rB" | "Rb" | "RB"\n' - ' shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' ' - 'shortbytesitem* \'"\'\n' - ' longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' ' - 'longbytesitem* \'"""\'\n' - ' shortbytesitem ::= shortbyteschar | bytesescapeseq\n' - ' longbytesitem ::= longbyteschar | bytesescapeseq\n' - ' shortbyteschar ::= \n' - ' longbyteschar ::= \n' - ' bytesescapeseq ::= "\\" \n' - '\n' - 'One syntactic restriction not indicated by these productions is ' - 'that\n' - 'whitespace is not allowed between the "stringprefix" or ' - '"bytesprefix"\n' - 'and the rest of the literal. The source character set is defined ' - 'by\n' - 'the encoding declaration; it is UTF-8 if no encoding declaration ' - 'is\n' - 'given in the source file; see section Encoding declarations.\n' - '\n' - 'In plain English: Both types of literals can be enclosed in ' - 'matching\n' - 'single quotes ("\'") or double quotes ("""). They can also be ' - 'enclosed\n' - 'in matching groups of three single or double quotes (these are\n' - 'generally referred to as *triple-quoted strings*). The ' - 'backslash\n' - '("\\") character is used to escape characters that otherwise have ' - 'a\n' - 'special meaning, such as newline, backslash itself, or the quote\n' - 'character.\n' - '\n' - 'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they ' - 'produce\n' - 'an instance of the "bytes" type instead of the "str" type. They ' - 'may\n' - 'only contain ASCII characters; bytes with a numeric value of 128 ' - 'or\n' - 'greater must be expressed with escapes.\n' - '\n' - 'Both string and bytes literals may optionally be prefixed with a\n' - 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' - 'and treat\n' - 'backslashes as literal characters. As a result, in string ' - 'literals,\n' - '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated ' - 'specially.\n' - 'Given that Python 2.x’s raw unicode literals behave differently ' - 'than\n' - 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' - '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' - 'added as a synonym of "\'br\'".\n' - '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' - '("u\'value\'") was reintroduced to simplify the maintenance of ' - 'dual\n' - 'Python 2.x and 3.x codebases. See **PEP 414** for more ' - 'information.\n' - '\n' - 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' - '*formatted\n' - 'string literal*; see Formatted string literals. The "\'f\'" may ' - 'be\n' - 'combined with "\'r\'", but not with "\'b\'" or "\'u\'", therefore ' - 'raw\n' - 'formatted strings are possible, but formatted bytes literals are ' - 'not.\n' - '\n' - 'In triple-quoted literals, unescaped newlines and quotes are ' - 'allowed\n' - '(and are retained), except that three unescaped quotes in a row\n' - 'terminate the literal. (A “quote” is the character used to open ' - 'the\n' - 'literal, i.e. either "\'" or """.)\n' - '\n' - 'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences ' - 'in string\n' - 'and bytes literals are interpreted according to rules similar to ' - 'those\n' - 'used by Standard C. The recognized escape sequences are:\n' - '\n' - '+-------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | Notes ' - '|\n' - '+===================+===================================+=========+\n' - '| "\\newline" | Backslash and newline ignored ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\\\" | Backslash ("\\") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\\'" | Single quote ("\'") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\"" | Double quote (""") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\a" | ASCII Bell (BEL) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\b" | ASCII Backspace (BS) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\f" | ASCII Formfeed (FF) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\n" | ASCII Linefeed (LF) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\r" | ASCII Carriage Return (CR) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\t" | ASCII Horizontal Tab (TAB) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\v" | ASCII Vertical Tab (VT) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\ooo" | Character with octal value *ooo* | ' - '(1,3) |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\xhh" | Character with hex value *hh* | ' - '(2,3) |\n' - '+-------------------+-----------------------------------+---------+\n' - '\n' - 'Escape sequences only recognized in string literals are:\n' - '\n' - '+-------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | Notes ' - '|\n' - '+===================+===================================+=========+\n' - '| "\\N{name}" | Character named *name* in the | ' - '(4) |\n' - '| | Unicode database | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\uxxxx" | Character with 16-bit hex value | ' - '(5) |\n' - '| | *xxxx* | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\Uxxxxxxxx" | Character with 32-bit hex value | ' - '(6) |\n' - '| | *xxxxxxxx* | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '\n' - 'Notes:\n' - '\n' - '1. As in Standard C, up to three octal digits are accepted.\n' - '\n' - '2. Unlike in Standard C, exactly two hex digits are required.\n' - '\n' - '3. In a bytes literal, hexadecimal and octal escapes denote the\n' - ' byte with the given value. In a string literal, these escapes\n' - ' denote a Unicode character with the given value.\n' - '\n' - '4. Changed in version 3.3: Support for name aliases [1] has been\n' - ' added.\n' - '\n' - '5. Exactly four hex digits are required.\n' - '\n' - '6. Any Unicode character can be encoded this way. Exactly eight\n' - ' hex digits are required.\n' - '\n' - 'Unlike Standard C, all unrecognized escape sequences are left in ' - 'the\n' - 'string unchanged, i.e., *the backslash is left in the result*. ' - '(This\n' - 'behavior is useful when debugging: if an escape sequence is ' - 'mistyped,\n' - 'the resulting output is more easily recognized as broken.) It is ' - 'also\n' - 'important to note that the escape sequences only recognized in ' - 'string\n' - 'literals fall into the category of unrecognized escapes for ' - 'bytes\n' - 'literals.\n' - '\n' - ' Changed in version 3.6: Unrecognized escape sequences produce ' - 'a\n' - ' DeprecationWarning. In some future version of Python they ' - 'will be\n' - ' a SyntaxError.\n' - '\n' - 'Even in a raw literal, quotes can be escaped with a backslash, ' - 'but the\n' - 'backslash remains in the result; for example, "r"\\""" is a ' - 'valid\n' - 'string literal consisting of two characters: a backslash and a ' - 'double\n' - 'quote; "r"\\"" is not a valid string literal (even a raw string ' - 'cannot\n' - 'end in an odd number of backslashes). Specifically, *a raw ' - 'literal\n' - 'cannot end in a single backslash* (since the backslash would ' - 'escape\n' - 'the following quote character). Note also that a single ' - 'backslash\n' - 'followed by a newline is interpreted as those two characters as ' - 'part\n' - 'of the literal, *not* as a line continuation.\n', - 'subscriptions': 'Subscriptions\n' - '*************\n' - '\n' - 'A subscription selects an item of a sequence (string, tuple ' - 'or list)\n' - 'or mapping (dictionary) object:\n' - '\n' - ' subscription ::= primary "[" expression_list "]"\n' - '\n' - 'The primary must evaluate to an object that supports ' - 'subscription\n' - '(lists or dictionaries for example). User-defined objects ' - 'can support\n' - 'subscription by defining a "__getitem__()" method.\n' - '\n' - 'For built-in objects, there are two types of objects that ' - 'support\n' - 'subscription:\n' - '\n' - 'If the primary is a mapping, the expression list must ' - 'evaluate to an\n' - 'object whose value is one of the keys of the mapping, and ' - 'the\n' - 'subscription selects the value in the mapping that ' - 'corresponds to that\n' - 'key. (The expression list is a tuple except if it has ' - 'exactly one\n' - 'item.)\n' - '\n' - 'If the primary is a sequence, the expression list must ' - 'evaluate to an\n' - 'integer or a slice (as discussed in the following ' - 'section).\n' - '\n' - 'The formal syntax makes no special provision for negative ' - 'indices in\n' - 'sequences; however, built-in sequences all provide a ' - '"__getitem__()"\n' - 'method that interprets negative indices by adding the ' - 'length of the\n' - 'sequence to the index (so that "x[-1]" selects the last ' - 'item of "x").\n' - 'The resulting value must be a nonnegative integer less than ' - 'the number\n' - 'of items in the sequence, and the subscription selects the ' - 'item whose\n' - 'index is that value (counting from zero). Since the support ' - 'for\n' - 'negative indices and slicing occurs in the object’s ' - '"__getitem__()"\n' - 'method, subclasses overriding this method will need to ' - 'explicitly add\n' - 'that support.\n' - '\n' - 'A string’s items are characters. A character is not a ' - 'separate data\n' - 'type but a string of exactly one character.\n', - 'truth': 'Truth Value Testing\n' - '*******************\n' - '\n' - 'Any object can be tested for truth value, for use in an "if" or\n' - '"while" condition or as operand of the Boolean operations below.\n' - '\n' - 'By default, an object is considered true unless its class defines\n' - 'either a "__bool__()" method that returns "False" or a "__len__()"\n' - 'method that returns zero, when called with the object. [1] Here ' - 'are\n' - 'most of the built-in objects considered false:\n' - '\n' - '* constants defined to be false: "None" and "False".\n' - '\n' - '* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)",\n' - ' "Fraction(0, 1)"\n' - '\n' - '* empty sequences and collections: "\'\'", "()", "[]", "{}", ' - '"set()",\n' - ' "range(0)"\n' - '\n' - 'Operations and built-in functions that have a Boolean result ' - 'always\n' - 'return "0" or "False" for false and "1" or "True" for true, unless\n' - 'otherwise stated. (Important exception: the Boolean operations ' - '"or"\n' - 'and "and" always return one of their operands.)\n', - 'try': 'The "try" statement\n' - '*******************\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. When ' - 'no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search for ' - 'an\n' - 'exception handler is started. This search inspects the except ' - 'clauses\n' - 'in turn until one is found that matches the exception. An ' - 'expression-\n' - 'less except clause, if present, must be last; it matches any\n' - 'exception. For an except clause with an expression, that expression\n' - 'is evaluated, and the clause matches the exception if the resulting\n' - 'object is “compatible” with the exception. An object is compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object or a tuple containing an item compatible with the exception.\n' - '\n' - 'If no except clause matches the exception, the search for an ' - 'exception\n' - 'handler continues in the surrounding code and on the invocation ' - 'stack.\n' - '[1]\n' - '\n' - 'If the evaluation of an expression in the header of an except clause\n' - 'raises an exception, the original search for a handler is canceled ' - 'and\n' - 'a search starts for the new exception in the surrounding code and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching except clause is found, the exception is assigned to\n' - 'the target specified after the "as" keyword in that except clause, ' - 'if\n' - 'present, and the except clause’s suite is executed. All except\n' - 'clauses must have an executable block. When the end of this block ' - 'is\n' - 'reached, execution continues normally after the entire try ' - 'statement.\n' - '(This means that if two nested handlers exist for the same ' - 'exception,\n' - 'and the exception occurs in the try clause of the inner handler, the\n' - 'outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is cleared\n' - 'at the end of the except clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to be\n' - 'able to refer to it after the except clause. Exceptions are cleared\n' - 'because with the traceback attached to them, they form a reference\n' - 'cycle with the stack frame, keeping all locals in that frame alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an except clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' - 'the\n' - 'exception class, the exception instance and a traceback object (see\n' - 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. "sys.exc_info()" values are\n' - 'restored to their previous values (before the call) when returning\n' - 'from a function that handled an exception.\n' - '\n' - 'The optional "else" clause is executed if the control flow leaves ' - 'the\n' - '"try" suite, no exception was raised, and no "return", "continue", ' - 'or\n' - '"break" statement was executed. Exceptions in the "else" clause are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. If ' - 'an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is executed. ' - 'If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return" or "break" statement, the saved exception\n' - 'is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause is\n' - 'also executed ‘on the way out.’ A "continue" statement is illegal in\n' - 'the "finally" clause. (The reason is a problem with the current\n' - 'implementation — this restriction may be lifted in the future).\n' - '\n' - 'The return value of a function is determined by the last "return"\n' - 'statement executed. Since the "finally" clause always executes, a\n' - '"return" statement executed in the "finally" clause will always be ' - 'the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n', - 'types': 'The standard type hierarchy\n' - '***************************\n' - '\n' - 'Below is a list of the types that are built into Python. ' - 'Extension\n' - 'modules (written in C, Java, or other languages, depending on the\n' - 'implementation) can define additional types. Future versions of\n' - 'Python may add types to the type hierarchy (e.g., rational ' - 'numbers,\n' - 'efficiently stored arrays of integers, etc.), although such ' - 'additions\n' - 'will often be provided via the standard library instead.\n' - '\n' - 'Some of the type descriptions below contain a paragraph listing\n' - '‘special attributes.’ These are attributes that provide access to ' - 'the\n' - 'implementation and are not intended for general use. Their ' - 'definition\n' - 'may change in the future.\n' - '\n' - 'None\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the built-in name "None". ' - 'It\n' - ' is used to signify the absence of a value in many situations, ' - 'e.g.,\n' - ' it is returned from functions that don’t explicitly return\n' - ' anything. Its truth value is false.\n' - '\n' - 'NotImplemented\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the built-in name\n' - ' "NotImplemented". Numeric methods and rich comparison methods\n' - ' should return this value if they do not implement the operation ' - 'for\n' - ' the operands provided. (The interpreter will then try the\n' - ' reflected operation, or some other fallback, depending on the\n' - ' operator.) Its truth value is true.\n' - '\n' - ' See Implementing the arithmetic operations for more details.\n' - '\n' - 'Ellipsis\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the literal "..." or the\n' - ' built-in name "Ellipsis". Its truth value is true.\n' - '\n' - '"numbers.Number"\n' - ' These are created by numeric literals and returned as results ' - 'by\n' - ' arithmetic operators and arithmetic built-in functions. ' - 'Numeric\n' - ' objects are immutable; once created their value never changes.\n' - ' Python numbers are of course strongly related to mathematical\n' - ' numbers, but subject to the limitations of numerical ' - 'representation\n' - ' in computers.\n' - '\n' - ' Python distinguishes between integers, floating point numbers, ' - 'and\n' - ' complex numbers:\n' - '\n' - ' "numbers.Integral"\n' - ' These represent elements from the mathematical set of ' - 'integers\n' - ' (positive and negative).\n' - '\n' - ' There are two types of integers:\n' - '\n' - ' Integers ("int")\n' - '\n' - ' These represent numbers in an unlimited range, subject to\n' - ' available (virtual) memory only. For the purpose of ' - 'shift\n' - ' and mask operations, a binary representation is assumed, ' - 'and\n' - ' negative numbers are represented in a variant of 2’s\n' - ' complement which gives the illusion of an infinite string ' - 'of\n' - ' sign bits extending to the left.\n' - '\n' - ' Booleans ("bool")\n' - ' These represent the truth values False and True. The two\n' - ' objects representing the values "False" and "True" are ' - 'the\n' - ' only Boolean objects. The Boolean type is a subtype of ' - 'the\n' - ' integer type, and Boolean values behave like the values 0 ' - 'and\n' - ' 1, respectively, in almost all contexts, the exception ' - 'being\n' - ' that when converted to a string, the strings ""False"" or\n' - ' ""True"" are returned, respectively.\n' - '\n' - ' The rules for integer representation are intended to give ' - 'the\n' - ' most meaningful interpretation of shift and mask operations\n' - ' involving negative integers.\n' - '\n' - ' "numbers.Real" ("float")\n' - ' These represent machine-level double precision floating ' - 'point\n' - ' numbers. You are at the mercy of the underlying machine\n' - ' architecture (and C or Java implementation) for the accepted\n' - ' range and handling of overflow. Python does not support ' - 'single-\n' - ' precision floating point numbers; the savings in processor ' - 'and\n' - ' memory usage that are usually the reason for using these are\n' - ' dwarfed by the overhead of using objects in Python, so there ' - 'is\n' - ' no reason to complicate the language with two kinds of ' - 'floating\n' - ' point numbers.\n' - '\n' - ' "numbers.Complex" ("complex")\n' - ' These represent complex numbers as a pair of machine-level\n' - ' double precision floating point numbers. The same caveats ' - 'apply\n' - ' as for floating point numbers. The real and imaginary parts ' - 'of a\n' - ' complex number "z" can be retrieved through the read-only\n' - ' attributes "z.real" and "z.imag".\n' - '\n' - 'Sequences\n' - ' These represent finite ordered sets indexed by non-negative\n' - ' numbers. The built-in function "len()" returns the number of ' - 'items\n' - ' of a sequence. When the length of a sequence is *n*, the index ' - 'set\n' - ' contains the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* ' - 'is\n' - ' selected by "a[i]".\n' - '\n' - ' Sequences also support slicing: "a[i:j]" selects all items with\n' - ' index *k* such that *i* "<=" *k* "<" *j*. When used as an\n' - ' expression, a slice is a sequence of the same type. This ' - 'implies\n' - ' that the index set is renumbered so that it starts at 0.\n' - '\n' - ' Some sequences also support “extended slicing” with a third ' - '“step”\n' - ' parameter: "a[i:j:k]" selects all items of *a* with index *x* ' - 'where\n' - ' "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n' - '\n' - ' Sequences are distinguished according to their mutability:\n' - '\n' - ' Immutable sequences\n' - ' An object of an immutable sequence type cannot change once it ' - 'is\n' - ' created. (If the object contains references to other ' - 'objects,\n' - ' these other objects may be mutable and may be changed; ' - 'however,\n' - ' the collection of objects directly referenced by an ' - 'immutable\n' - ' object cannot change.)\n' - '\n' - ' The following types are immutable sequences:\n' - '\n' - ' Strings\n' - ' A string is a sequence of values that represent Unicode ' - 'code\n' - ' points. All the code points in the range "U+0000 - ' - 'U+10FFFF"\n' - ' can be represented in a string. Python doesn’t have a ' - '"char"\n' - ' type; instead, every code point in the string is ' - 'represented\n' - ' as a string object with length "1". The built-in ' - 'function\n' - ' "ord()" converts a code point from its string form to an\n' - ' integer in the range "0 - 10FFFF"; "chr()" converts an\n' - ' integer in the range "0 - 10FFFF" to the corresponding ' - 'length\n' - ' "1" string object. "str.encode()" can be used to convert ' - 'a\n' - ' "str" to "bytes" using the given text encoding, and\n' - ' "bytes.decode()" can be used to achieve the opposite.\n' - '\n' - ' Tuples\n' - ' The items of a tuple are arbitrary Python objects. Tuples ' - 'of\n' - ' two or more items are formed by comma-separated lists of\n' - ' expressions. A tuple of one item (a ‘singleton’) can be\n' - ' formed by affixing a comma to an expression (an expression ' - 'by\n' - ' itself does not create a tuple, since parentheses must be\n' - ' usable for grouping of expressions). An empty tuple can ' - 'be\n' - ' formed by an empty pair of parentheses.\n' - '\n' - ' Bytes\n' - ' A bytes object is an immutable array. The items are ' - '8-bit\n' - ' bytes, represented by integers in the range 0 <= x < 256.\n' - ' Bytes literals (like "b\'abc\'") and the built-in ' - '"bytes()"\n' - ' constructor can be used to create bytes objects. Also, ' - 'bytes\n' - ' objects can be decoded to strings via the "decode()" ' - 'method.\n' - '\n' - ' Mutable sequences\n' - ' Mutable sequences can be changed after they are created. ' - 'The\n' - ' subscription and slicing notations can be used as the target ' - 'of\n' - ' assignment and "del" (delete) statements.\n' - '\n' - ' There are currently two intrinsic mutable sequence types:\n' - '\n' - ' Lists\n' - ' The items of a list are arbitrary Python objects. Lists ' - 'are\n' - ' formed by placing a comma-separated list of expressions ' - 'in\n' - ' square brackets. (Note that there are no special cases ' - 'needed\n' - ' to form lists of length 0 or 1.)\n' - '\n' - ' Byte Arrays\n' - ' A bytearray object is a mutable array. They are created ' - 'by\n' - ' the built-in "bytearray()" constructor. Aside from being\n' - ' mutable (and hence unhashable), byte arrays otherwise ' - 'provide\n' - ' the same interface and functionality as immutable "bytes"\n' - ' objects.\n' - '\n' - ' The extension module "array" provides an additional example ' - 'of a\n' - ' mutable sequence type, as does the "collections" module.\n' - '\n' - 'Set types\n' - ' These represent unordered, finite sets of unique, immutable\n' - ' objects. As such, they cannot be indexed by any subscript. ' - 'However,\n' - ' they can be iterated over, and the built-in function "len()"\n' - ' returns the number of items in a set. Common uses for sets are ' - 'fast\n' - ' membership testing, removing duplicates from a sequence, and\n' - ' computing mathematical operations such as intersection, union,\n' - ' difference, and symmetric difference.\n' - '\n' - ' For set elements, the same immutability rules apply as for\n' - ' dictionary keys. Note that numeric types obey the normal rules ' - 'for\n' - ' numeric comparison: if two numbers compare equal (e.g., "1" and\n' - ' "1.0"), only one of them can be contained in a set.\n' - '\n' - ' There are currently two intrinsic set types:\n' - '\n' - ' Sets\n' - ' These represent a mutable set. They are created by the ' - 'built-in\n' - ' "set()" constructor and can be modified afterwards by ' - 'several\n' - ' methods, such as "add()".\n' - '\n' - ' Frozen sets\n' - ' These represent an immutable set. They are created by the\n' - ' built-in "frozenset()" constructor. As a frozenset is ' - 'immutable\n' - ' and *hashable*, it can be used again as an element of ' - 'another\n' - ' set, or as a dictionary key.\n' - '\n' - 'Mappings\n' - ' These represent finite sets of objects indexed by arbitrary ' - 'index\n' - ' sets. The subscript notation "a[k]" selects the item indexed by ' - '"k"\n' - ' from the mapping "a"; this can be used in expressions and as ' - 'the\n' - ' target of assignments or "del" statements. The built-in ' - 'function\n' - ' "len()" returns the number of items in a mapping.\n' - '\n' - ' There is currently a single intrinsic mapping type:\n' - '\n' - ' Dictionaries\n' - ' These represent finite sets of objects indexed by nearly\n' - ' arbitrary values. The only types of values not acceptable ' - 'as\n' - ' keys are values containing lists or dictionaries or other\n' - ' mutable types that are compared by value rather than by ' - 'object\n' - ' identity, the reason being that the efficient implementation ' - 'of\n' - ' dictionaries requires a key’s hash value to remain constant.\n' - ' Numeric types used for keys obey the normal rules for ' - 'numeric\n' - ' comparison: if two numbers compare equal (e.g., "1" and ' - '"1.0")\n' - ' then they can be used interchangeably to index the same\n' - ' dictionary entry.\n' - '\n' - ' Dictionaries are mutable; they can be created by the "{...}"\n' - ' notation (see section Dictionary displays).\n' - '\n' - ' The extension modules "dbm.ndbm" and "dbm.gnu" provide\n' - ' additional examples of mapping types, as does the ' - '"collections"\n' - ' module.\n' - '\n' - 'Callable types\n' - ' These are the types to which the function call operation (see\n' - ' section Calls) can be applied:\n' - '\n' - ' User-defined functions\n' - ' A user-defined function object is created by a function\n' - ' definition (see section Function definitions). It should be\n' - ' called with an argument list containing the same number of ' - 'items\n' - ' as the function’s formal parameter list.\n' - '\n' - ' Special attributes:\n' - '\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | Attribute | Meaning ' - '| |\n' - ' ' - '+===========================+=================================+=============+\n' - ' | "__doc__" | The function’s documentation ' - '| Writable |\n' - ' | | string, or "None" if ' - '| |\n' - ' | | unavailable; not inherited by ' - '| |\n' - ' | | subclasses ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__name__" | The function’s name ' - '| Writable |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__qualname__" | The function’s *qualified name* ' - '| Writable |\n' - ' | | New in version 3.3. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__module__" | The name of the module the ' - '| Writable |\n' - ' | | function was defined in, or ' - '| |\n' - ' | | "None" if unavailable. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__defaults__" | A tuple containing default ' - '| Writable |\n' - ' | | argument values for those ' - '| |\n' - ' | | arguments that have defaults, ' - '| |\n' - ' | | or "None" if no arguments have ' - '| |\n' - ' | | a default value ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__code__" | The code object representing ' - '| Writable |\n' - ' | | the compiled function body. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__globals__" | A reference to the dictionary ' - '| Read-only |\n' - ' | | that holds the function’s ' - '| |\n' - ' | | global variables — the global ' - '| |\n' - ' | | namespace of the module in ' - '| |\n' - ' | | which the function was defined. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__dict__" | The namespace supporting ' - '| Writable |\n' - ' | | arbitrary function attributes. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__closure__" | "None" or a tuple of cells that ' - '| Read-only |\n' - ' | | contain bindings for the ' - '| |\n' - ' | | function’s free variables. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__annotations__" | A dict containing annotations ' - '| Writable |\n' - ' | | of parameters. The keys of the ' - '| |\n' - ' | | dict are the parameter names, ' - '| |\n' - ' | | and "\'return\'" for the ' - 'return | |\n' - ' | | annotation, if provided. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__kwdefaults__" | A dict containing defaults for ' - '| Writable |\n' - ' | | keyword-only parameters. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - '\n' - ' Most of the attributes labelled “Writable” check the type of ' - 'the\n' - ' assigned value.\n' - '\n' - ' Function objects also support getting and setting arbitrary\n' - ' attributes, which can be used, for example, to attach ' - 'metadata\n' - ' to functions. Regular attribute dot-notation is used to get ' - 'and\n' - ' set such attributes. *Note that the current implementation ' - 'only\n' - ' supports function attributes on user-defined functions. ' - 'Function\n' - ' attributes on built-in functions may be supported in the\n' - ' future.*\n' - '\n' - ' Additional information about a function’s definition can be\n' - ' retrieved from its code object; see the description of ' - 'internal\n' - ' types below.\n' - '\n' - ' Instance methods\n' - ' An instance method object combines a class, a class instance ' - 'and\n' - ' any callable object (normally a user-defined function).\n' - '\n' - ' Special read-only attributes: "__self__" is the class ' - 'instance\n' - ' object, "__func__" is the function object; "__doc__" is the\n' - ' method’s documentation (same as "__func__.__doc__"); ' - '"__name__"\n' - ' is the method name (same as "__func__.__name__"); ' - '"__module__"\n' - ' is the name of the module the method was defined in, or ' - '"None"\n' - ' if unavailable.\n' - '\n' - ' Methods also support accessing (but not setting) the ' - 'arbitrary\n' - ' function attributes on the underlying function object.\n' - '\n' - ' User-defined method objects may be created when getting an\n' - ' attribute of a class (perhaps via an instance of that class), ' - 'if\n' - ' that attribute is a user-defined function object or a class\n' - ' method object.\n' - '\n' - ' When an instance method object is created by retrieving a ' - 'user-\n' - ' defined function object from a class via one of its ' - 'instances,\n' - ' its "__self__" attribute is the instance, and the method ' - 'object\n' - ' is said to be bound. The new method’s "__func__" attribute ' - 'is\n' - ' the original function object.\n' - '\n' - ' When a user-defined method object is created by retrieving\n' - ' another method object from a class or instance, the behaviour ' - 'is\n' - ' the same as for a function object, except that the ' - '"__func__"\n' - ' attribute of the new instance is not the original method ' - 'object\n' - ' but its "__func__" attribute.\n' - '\n' - ' When an instance method object is created by retrieving a ' - 'class\n' - ' method object from a class or instance, its "__self__" ' - 'attribute\n' - ' is the class itself, and its "__func__" attribute is the\n' - ' function object underlying the class method.\n' - '\n' - ' When an instance method object is called, the underlying\n' - ' function ("__func__") is called, inserting the class ' - 'instance\n' - ' ("__self__") in front of the argument list. For instance, ' - 'when\n' - ' "C" is a class which contains a definition for a function ' - '"f()",\n' - ' and "x" is an instance of "C", calling "x.f(1)" is equivalent ' - 'to\n' - ' calling "C.f(x, 1)".\n' - '\n' - ' When an instance method object is derived from a class ' - 'method\n' - ' object, the “class instance” stored in "__self__" will ' - 'actually\n' - ' be the class itself, so that calling either "x.f(1)" or ' - '"C.f(1)"\n' - ' is equivalent to calling "f(C,1)" where "f" is the ' - 'underlying\n' - ' function.\n' - '\n' - ' Note that the transformation from function object to ' - 'instance\n' - ' method object happens each time the attribute is retrieved ' - 'from\n' - ' the instance. In some cases, a fruitful optimization is to\n' - ' assign the attribute to a local variable and call that local\n' - ' variable. Also notice that this transformation only happens ' - 'for\n' - ' user-defined functions; other callable objects (and all non-\n' - ' callable objects) are retrieved without transformation. It ' - 'is\n' - ' also important to note that user-defined functions which are\n' - ' attributes of a class instance are not converted to bound\n' - ' methods; this *only* happens when the function is an ' - 'attribute\n' - ' of the class.\n' - '\n' - ' Generator functions\n' - ' A function or method which uses the "yield" statement (see\n' - ' section The yield statement) is called a *generator ' - 'function*.\n' - ' Such a function, when called, always returns an iterator ' - 'object\n' - ' which can be used to execute the body of the function: ' - 'calling\n' - ' the iterator’s "iterator.__next__()" method will cause the\n' - ' function to execute until it provides a value using the ' - '"yield"\n' - ' statement. When the function executes a "return" statement ' - 'or\n' - ' falls off the end, a "StopIteration" exception is raised and ' - 'the\n' - ' iterator will have reached the end of the set of values to ' - 'be\n' - ' returned.\n' - '\n' - ' Coroutine functions\n' - ' A function or method which is defined using "async def" is\n' - ' called a *coroutine function*. Such a function, when ' - 'called,\n' - ' returns a *coroutine* object. It may contain "await"\n' - ' expressions, as well as "async with" and "async for" ' - 'statements.\n' - ' See also the Coroutine Objects section.\n' - '\n' - ' Asynchronous generator functions\n' - ' A function or method which is defined using "async def" and\n' - ' which uses the "yield" statement is called a *asynchronous\n' - ' generator function*. Such a function, when called, returns ' - 'an\n' - ' asynchronous iterator object which can be used in an "async ' - 'for"\n' - ' statement to execute the body of the function.\n' - '\n' - ' Calling the asynchronous iterator’s "aiterator.__anext__()"\n' - ' method will return an *awaitable* which when awaited will\n' - ' execute until it provides a value using the "yield" ' - 'expression.\n' - ' When the function executes an empty "return" statement or ' - 'falls\n' - ' off the end, a "StopAsyncIteration" exception is raised and ' - 'the\n' - ' asynchronous iterator will have reached the end of the set ' - 'of\n' - ' values to be yielded.\n' - '\n' - ' Built-in functions\n' - ' A built-in function object is a wrapper around a C function.\n' - ' Examples of built-in functions are "len()" and "math.sin()"\n' - ' ("math" is a standard built-in module). The number and type ' - 'of\n' - ' the arguments are determined by the C function. Special ' - 'read-\n' - ' only attributes: "__doc__" is the function’s documentation\n' - ' string, or "None" if unavailable; "__name__" is the ' - 'function’s\n' - ' name; "__self__" is set to "None" (but see the next item);\n' - ' "__module__" is the name of the module the function was ' - 'defined\n' - ' in or "None" if unavailable.\n' - '\n' - ' Built-in methods\n' - ' This is really a different disguise of a built-in function, ' - 'this\n' - ' time containing an object passed to the C function as an\n' - ' implicit extra argument. An example of a built-in method is\n' - ' "alist.append()", assuming *alist* is a list object. In this\n' - ' case, the special read-only attribute "__self__" is set to ' - 'the\n' - ' object denoted by *alist*.\n' - '\n' - ' Classes\n' - ' Classes are callable. These objects normally act as ' - 'factories\n' - ' for new instances of themselves, but variations are possible ' - 'for\n' - ' class types that override "__new__()". The arguments of the\n' - ' call are passed to "__new__()" and, in the typical case, to\n' - ' "__init__()" to initialize the new instance.\n' - '\n' - ' Class Instances\n' - ' Instances of arbitrary classes can be made callable by ' - 'defining\n' - ' a "__call__()" method in their class.\n' - '\n' - 'Modules\n' - ' Modules are a basic organizational unit of Python code, and are\n' - ' created by the import system as invoked either by the "import"\n' - ' statement (see "import"), or by calling functions such as\n' - ' "importlib.import_module()" and built-in "__import__()". A ' - 'module\n' - ' object has a namespace implemented by a dictionary object (this ' - 'is\n' - ' the dictionary referenced by the "__globals__" attribute of\n' - ' functions defined in the module). Attribute references are\n' - ' translated to lookups in this dictionary, e.g., "m.x" is ' - 'equivalent\n' - ' to "m.__dict__["x"]". A module object does not contain the code\n' - ' object used to initialize the module (since it isn’t needed ' - 'once\n' - ' the initialization is done).\n' - '\n' - ' Attribute assignment updates the module’s namespace dictionary,\n' - ' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' - '\n' - ' Predefined (writable) attributes: "__name__" is the module’s ' - 'name;\n' - ' "__doc__" is the module’s documentation string, or "None" if\n' - ' unavailable; "__annotations__" (optional) is a dictionary\n' - ' containing *variable annotations* collected during module body\n' - ' execution; "__file__" is the pathname of the file from which ' - 'the\n' - ' module was loaded, if it was loaded from a file. The "__file__"\n' - ' attribute may be missing for certain types of modules, such as ' - 'C\n' - ' modules that are statically linked into the interpreter; for\n' - ' extension modules loaded dynamically from a shared library, it ' - 'is\n' - ' the pathname of the shared library file.\n' - '\n' - ' Special read-only attribute: "__dict__" is the module’s ' - 'namespace\n' - ' as a dictionary object.\n' - '\n' - ' **CPython implementation detail:** Because of the way CPython\n' - ' clears module dictionaries, the module dictionary will be ' - 'cleared\n' - ' when the module falls out of scope even if the dictionary still ' - 'has\n' - ' live references. To avoid this, copy the dictionary or keep ' - 'the\n' - ' module around while using its dictionary directly.\n' - '\n' - 'Custom classes\n' - ' Custom class types are typically created by class definitions ' - '(see\n' - ' section Class definitions). A class has a namespace implemented ' - 'by\n' - ' a dictionary object. Class attribute references are translated ' - 'to\n' - ' lookups in this dictionary, e.g., "C.x" is translated to\n' - ' "C.__dict__["x"]" (although there are a number of hooks which ' - 'allow\n' - ' for other means of locating attributes). When the attribute name ' - 'is\n' - ' not found there, the attribute search continues in the base\n' - ' classes. This search of the base classes uses the C3 method\n' - ' resolution order which behaves correctly even in the presence ' - 'of\n' - ' ‘diamond’ inheritance structures where there are multiple\n' - ' inheritance paths leading back to a common ancestor. Additional\n' - ' details on the C3 MRO used by Python can be found in the\n' - ' documentation accompanying the 2.3 release at\n' - ' https://www.python.org/download/releases/2.3/mro/.\n' - '\n' - ' When a class attribute reference (for class "C", say) would ' - 'yield a\n' - ' class method object, it is transformed into an instance method\n' - ' object whose "__self__" attribute is "C". When it would yield ' - 'a\n' - ' static method object, it is transformed into the object wrapped ' - 'by\n' - ' the static method object. See section Implementing Descriptors ' - 'for\n' - ' another way in which attributes retrieved from a class may ' - 'differ\n' - ' from those actually contained in its "__dict__".\n' - '\n' - ' Class attribute assignments update the class’s dictionary, ' - 'never\n' - ' the dictionary of a base class.\n' - '\n' - ' A class object can be called (see above) to yield a class ' - 'instance\n' - ' (see below).\n' - '\n' - ' Special attributes: "__name__" is the class name; "__module__" ' - 'is\n' - ' the module name in which the class was defined; "__dict__" is ' - 'the\n' - ' dictionary containing the class’s namespace; "__bases__" is a ' - 'tuple\n' - ' containing the base classes, in the order of their occurrence ' - 'in\n' - ' the base class list; "__doc__" is the class’s documentation ' - 'string,\n' - ' or "None" if undefined; "__annotations__" (optional) is a\n' - ' dictionary containing *variable annotations* collected during ' - 'class\n' - ' body execution.\n' - '\n' - 'Class instances\n' - ' A class instance is created by calling a class object (see ' - 'above).\n' - ' A class instance has a namespace implemented as a dictionary ' - 'which\n' - ' is the first place in which attribute references are searched.\n' - ' When an attribute is not found there, and the instance’s class ' - 'has\n' - ' an attribute by that name, the search continues with the class\n' - ' attributes. If a class attribute is found that is a ' - 'user-defined\n' - ' function object, it is transformed into an instance method ' - 'object\n' - ' whose "__self__" attribute is the instance. Static method and\n' - ' class method objects are also transformed; see above under\n' - ' “Classes”. See section Implementing Descriptors for another way ' - 'in\n' - ' which attributes of a class retrieved via its instances may ' - 'differ\n' - ' from the objects actually stored in the class’s "__dict__". If ' - 'no\n' - ' class attribute is found, and the object’s class has a\n' - ' "__getattr__()" method, that is called to satisfy the lookup.\n' - '\n' - ' Attribute assignments and deletions update the instance’s\n' - ' dictionary, never a class’s dictionary. If the class has a\n' - ' "__setattr__()" or "__delattr__()" method, this is called ' - 'instead\n' - ' of updating the instance dictionary directly.\n' - '\n' - ' Class instances can pretend to be numbers, sequences, or ' - 'mappings\n' - ' if they have methods with certain special names. See section\n' - ' Special method names.\n' - '\n' - ' Special attributes: "__dict__" is the attribute dictionary;\n' - ' "__class__" is the instance’s class.\n' - '\n' - 'I/O objects (also known as file objects)\n' - ' A *file object* represents an open file. Various shortcuts are\n' - ' available to create file objects: the "open()" built-in ' - 'function,\n' - ' and also "os.popen()", "os.fdopen()", and the "makefile()" ' - 'method\n' - ' of socket objects (and perhaps by other functions or methods\n' - ' provided by extension modules).\n' - '\n' - ' The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n' - ' initialized to file objects corresponding to the interpreter’s\n' - ' standard input, output and error streams; they are all open in ' - 'text\n' - ' mode and therefore follow the interface defined by the\n' - ' "io.TextIOBase" abstract class.\n' - '\n' - 'Internal types\n' - ' A few types used internally by the interpreter are exposed to ' - 'the\n' - ' user. Their definitions may change with future versions of the\n' - ' interpreter, but they are mentioned here for completeness.\n' - '\n' - ' Code objects\n' - ' Code objects represent *byte-compiled* executable Python ' - 'code,\n' - ' or *bytecode*. The difference between a code object and a\n' - ' function object is that the function object contains an ' - 'explicit\n' - ' reference to the function’s globals (the module in which it ' - 'was\n' - ' defined), while a code object contains no context; also the\n' - ' default argument values are stored in the function object, ' - 'not\n' - ' in the code object (because they represent values calculated ' - 'at\n' - ' run-time). Unlike function objects, code objects are ' - 'immutable\n' - ' and contain no references (directly or indirectly) to ' - 'mutable\n' - ' objects.\n' - '\n' - ' Special read-only attributes: "co_name" gives the function ' - 'name;\n' - ' "co_argcount" is the number of positional arguments ' - '(including\n' - ' arguments with default values); "co_nlocals" is the number ' - 'of\n' - ' local variables used by the function (including arguments);\n' - ' "co_varnames" is a tuple containing the names of the local\n' - ' variables (starting with the argument names); "co_cellvars" ' - 'is a\n' - ' tuple containing the names of local variables that are\n' - ' referenced by nested functions; "co_freevars" is a tuple\n' - ' containing the names of free variables; "co_code" is a ' - 'string\n' - ' representing the sequence of bytecode instructions; ' - '"co_consts"\n' - ' is a tuple containing the literals used by the bytecode;\n' - ' "co_names" is a tuple containing the names used by the ' - 'bytecode;\n' - ' "co_filename" is the filename from which the code was ' - 'compiled;\n' - ' "co_firstlineno" is the first line number of the function;\n' - ' "co_lnotab" is a string encoding the mapping from bytecode\n' - ' offsets to line numbers (for details see the source code of ' - 'the\n' - ' interpreter); "co_stacksize" is the required stack size\n' - ' (including local variables); "co_flags" is an integer ' - 'encoding a\n' - ' number of flags for the interpreter.\n' - '\n' - ' The following flag bits are defined for "co_flags": bit ' - '"0x04"\n' - ' is set if the function uses the "*arguments" syntax to accept ' - 'an\n' - ' arbitrary number of positional arguments; bit "0x08" is set ' - 'if\n' - ' the function uses the "**keywords" syntax to accept ' - 'arbitrary\n' - ' keyword arguments; bit "0x20" is set if the function is a\n' - ' generator.\n' - '\n' - ' Future feature declarations ("from __future__ import ' - 'division")\n' - ' also use bits in "co_flags" to indicate whether a code ' - 'object\n' - ' was compiled with a particular feature enabled: bit "0x2000" ' - 'is\n' - ' set if the function was compiled with future division ' - 'enabled;\n' - ' bits "0x10" and "0x1000" were used in earlier versions of\n' - ' Python.\n' - '\n' - ' Other bits in "co_flags" are reserved for internal use.\n' - '\n' - ' If a code object represents a function, the first item in\n' - ' "co_consts" is the documentation string of the function, or\n' - ' "None" if undefined.\n' - '\n' - ' Frame objects\n' - ' Frame objects represent execution frames. They may occur in\n' - ' traceback objects (see below).\n' - '\n' - ' Special read-only attributes: "f_back" is to the previous ' - 'stack\n' - ' frame (towards the caller), or "None" if this is the bottom\n' - ' stack frame; "f_code" is the code object being executed in ' - 'this\n' - ' frame; "f_locals" is the dictionary used to look up local\n' - ' variables; "f_globals" is used for global variables;\n' - ' "f_builtins" is used for built-in (intrinsic) names; ' - '"f_lasti"\n' - ' gives the precise instruction (this is an index into the\n' - ' bytecode string of the code object).\n' - '\n' - ' Special writable attributes: "f_trace", if not "None", is a\n' - ' function called at the start of each source code line (this ' - 'is\n' - ' used by the debugger); "f_lineno" is the current line number ' - 'of\n' - ' the frame — writing to this from within a trace function ' - 'jumps\n' - ' to the given line (only for the bottom-most frame). A ' - 'debugger\n' - ' can implement a Jump command (aka Set Next Statement) by ' - 'writing\n' - ' to f_lineno.\n' - '\n' - ' Frame objects support one method:\n' - '\n' - ' frame.clear()\n' - '\n' - ' This method clears all references to local variables held ' - 'by\n' - ' the frame. Also, if the frame belonged to a generator, ' - 'the\n' - ' generator is finalized. This helps break reference ' - 'cycles\n' - ' involving frame objects (for example when catching an\n' - ' exception and storing its traceback for later use).\n' - '\n' - ' "RuntimeError" is raised if the frame is currently ' - 'executing.\n' - '\n' - ' New in version 3.4.\n' - '\n' - ' Traceback objects\n' - ' Traceback objects represent a stack trace of an exception. ' - 'A\n' - ' traceback object is created when an exception occurs. When ' - 'the\n' - ' search for an exception handler unwinds the execution stack, ' - 'at\n' - ' each unwound level a traceback object is inserted in front ' - 'of\n' - ' the current traceback. When an exception handler is ' - 'entered,\n' - ' the stack trace is made available to the program. (See ' - 'section\n' - ' The try statement.) It is accessible as the third item of ' - 'the\n' - ' tuple returned by "sys.exc_info()". When the program contains ' - 'no\n' - ' suitable handler, the stack trace is written (nicely ' - 'formatted)\n' - ' to the standard error stream; if the interpreter is ' - 'interactive,\n' - ' it is also made available to the user as ' - '"sys.last_traceback".\n' - '\n' - ' Special read-only attributes: "tb_next" is the next level in ' - 'the\n' - ' stack trace (towards the frame where the exception occurred), ' - 'or\n' - ' "None" if there is no next level; "tb_frame" points to the\n' - ' execution frame of the current level; "tb_lineno" gives the ' - 'line\n' - ' number where the exception occurred; "tb_lasti" indicates ' - 'the\n' - ' precise instruction. The line number and last instruction ' - 'in\n' - ' the traceback may differ from the line number of its frame\n' - ' object if the exception occurred in a "try" statement with ' - 'no\n' - ' matching except clause or with a finally clause.\n' - '\n' - ' Slice objects\n' - ' Slice objects are used to represent slices for ' - '"__getitem__()"\n' - ' methods. They are also created by the built-in "slice()"\n' - ' function.\n' - '\n' - ' Special read-only attributes: "start" is the lower bound; ' - '"stop"\n' - ' is the upper bound; "step" is the step value; each is "None" ' - 'if\n' - ' omitted. These attributes can have any type.\n' - '\n' - ' Slice objects support one method:\n' - '\n' - ' slice.indices(self, length)\n' - '\n' - ' This method takes a single integer argument *length* and\n' - ' computes information about the slice that the slice ' - 'object\n' - ' would describe if applied to a sequence of *length* ' - 'items.\n' - ' It returns a tuple of three integers; respectively these ' - 'are\n' - ' the *start* and *stop* indices and the *step* or stride\n' - ' length of the slice. Missing or out-of-bounds indices are\n' - ' handled in a manner consistent with regular slices.\n' - '\n' - ' Static method objects\n' - ' Static method objects provide a way of defeating the\n' - ' transformation of function objects to method objects ' - 'described\n' - ' above. A static method object is a wrapper around any other\n' - ' object, usually a user-defined method object. When a static\n' - ' method object is retrieved from a class or a class instance, ' - 'the\n' - ' object actually returned is the wrapped object, which is not\n' - ' subject to any further transformation. Static method objects ' - 'are\n' - ' not themselves callable, although the objects they wrap ' - 'usually\n' - ' are. Static method objects are created by the built-in\n' - ' "staticmethod()" constructor.\n' - '\n' - ' Class method objects\n' - ' A class method object, like a static method object, is a ' - 'wrapper\n' - ' around another object that alters the way in which that ' - 'object\n' - ' is retrieved from classes and class instances. The behaviour ' - 'of\n' - ' class method objects upon such retrieval is described above,\n' - ' under “User-defined methods”. Class method objects are ' - 'created\n' - ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': 'Functions\n' - '*********\n' - '\n' - 'Function objects are created by function definitions. The ' - 'only\n' - 'operation on a function object is to call it: ' - '"func(argument-list)".\n' - '\n' - 'There are really two flavors of function objects: built-in ' - 'functions\n' - 'and user-defined functions. Both support the same ' - 'operation (to call\n' - 'the function), but the implementation is different, hence ' - 'the\n' - 'different object types.\n' - '\n' - 'See Function definitions for more information.\n', - 'typesmapping': 'Mapping Types — "dict"\n' - '**********************\n' - '\n' - 'A *mapping* object maps *hashable* values to arbitrary ' - 'objects.\n' - 'Mappings are mutable objects. There is currently only one ' - 'standard\n' - 'mapping type, the *dictionary*. (For other containers see ' - 'the built-\n' - 'in "list", "set", and "tuple" classes, and the "collections" ' - 'module.)\n' - '\n' - 'A dictionary’s keys are *almost* arbitrary values. Values ' - 'that are\n' - 'not *hashable*, that is, values containing lists, ' - 'dictionaries or\n' - 'other mutable types (that are compared by value rather than ' - 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' - '\n' - 'Dictionaries can be created by placing a comma-separated ' - 'list of "key:\n' - 'value" pairs within braces, for example: "{\'jack\': 4098, ' - "'sjoerd':\n" - '4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the ' - '"dict"\n' - 'constructor.\n' - '\n' - 'class dict(**kwarg)\n' - 'class dict(mapping, **kwarg)\n' - 'class dict(iterable, **kwarg)\n' - '\n' - ' Return a new dictionary initialized from an optional ' - 'positional\n' - ' argument and a possibly empty set of keyword arguments.\n' - '\n' - ' If no positional argument is given, an empty dictionary ' - 'is created.\n' - ' If a positional argument is given and it is a mapping ' - 'object, a\n' - ' dictionary is created with the same key-value pairs as ' - 'the mapping\n' - ' object. Otherwise, the positional argument must be an ' - '*iterable*\n' - ' object. Each item in the iterable must itself be an ' - 'iterable with\n' - ' exactly two objects. The first object of each item ' - 'becomes a key\n' - ' in the new dictionary, and the second object the ' - 'corresponding\n' - ' value. If a key occurs more than once, the last value ' - 'for that key\n' - ' becomes the corresponding value in the new dictionary.\n' - '\n' - ' If keyword arguments are given, the keyword arguments and ' - 'their\n' - ' values are added to the dictionary created from the ' - 'positional\n' - ' argument. If a key being added is already present, the ' - 'value from\n' - ' the keyword argument replaces the value from the ' - 'positional\n' - ' argument.\n' - '\n' - ' To illustrate, the following examples all return a ' - 'dictionary equal\n' - ' to "{"one": 1, "two": 2, "three": 3}":\n' - '\n' - ' >>> a = dict(one=1, two=2, three=3)\n' - " >>> b = {'one': 1, 'two': 2, 'three': 3}\n" - " >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3]))\n" - " >>> d = dict([('two', 2), ('one', 1), ('three', 3)])\n" - " >>> e = dict({'three': 3, 'one': 1, 'two': 2})\n" - ' >>> a == b == c == d == e\n' - ' True\n' - '\n' - ' Providing keyword arguments as in the first example only ' - 'works for\n' - ' keys that are valid Python identifiers. Otherwise, any ' - 'valid keys\n' - ' can be used.\n' - '\n' - ' These are the operations that dictionaries support (and ' - 'therefore,\n' - ' custom mapping types should support too):\n' - '\n' - ' len(d)\n' - '\n' - ' Return the number of items in the dictionary *d*.\n' - '\n' - ' d[key]\n' - '\n' - ' Return the item of *d* with key *key*. Raises a ' - '"KeyError" if\n' - ' *key* is not in the map.\n' - '\n' - ' If a subclass of dict defines a method "__missing__()" ' - 'and *key*\n' - ' is not present, the "d[key]" operation calls that ' - 'method with\n' - ' the key *key* as argument. The "d[key]" operation ' - 'then returns\n' - ' or raises whatever is returned or raised by the\n' - ' "__missing__(key)" call. No other operations or ' - 'methods invoke\n' - ' "__missing__()". If "__missing__()" is not defined, ' - '"KeyError"\n' - ' is raised. "__missing__()" must be a method; it cannot ' - 'be an\n' - ' instance variable:\n' - '\n' - ' >>> class Counter(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return 0\n' - ' >>> c = Counter()\n' - " >>> c['red']\n" - ' 0\n' - " >>> c['red'] += 1\n" - " >>> c['red']\n" - ' 1\n' - '\n' - ' The example above shows part of the implementation of\n' - ' "collections.Counter". A different "__missing__" ' - 'method is used\n' - ' by "collections.defaultdict".\n' - '\n' - ' d[key] = value\n' - '\n' - ' Set "d[key]" to *value*.\n' - '\n' - ' del d[key]\n' - '\n' - ' Remove "d[key]" from *d*. Raises a "KeyError" if ' - '*key* is not\n' - ' in the map.\n' - '\n' - ' key in d\n' - '\n' - ' Return "True" if *d* has a key *key*, else "False".\n' - '\n' - ' key not in d\n' - '\n' - ' Equivalent to "not key in d".\n' - '\n' - ' iter(d)\n' - '\n' - ' Return an iterator over the keys of the dictionary. ' - 'This is a\n' - ' shortcut for "iter(d.keys())".\n' - '\n' - ' clear()\n' - '\n' - ' Remove all items from the dictionary.\n' - '\n' - ' copy()\n' - '\n' - ' Return a shallow copy of the dictionary.\n' - '\n' - ' classmethod fromkeys(seq[, value])\n' - '\n' - ' Create a new dictionary with keys from *seq* and ' - 'values set to\n' - ' *value*.\n' - '\n' - ' "fromkeys()" is a class method that returns a new ' - 'dictionary.\n' - ' *value* defaults to "None".\n' - '\n' - ' get(key[, default])\n' - '\n' - ' Return the value for *key* if *key* is in the ' - 'dictionary, else\n' - ' *default*. If *default* is not given, it defaults to ' - '"None", so\n' - ' that this method never raises a "KeyError".\n' - '\n' - ' items()\n' - '\n' - ' Return a new view of the dictionary’s items ("(key, ' - 'value)"\n' - ' pairs). See the documentation of view objects.\n' - '\n' - ' keys()\n' - '\n' - ' Return a new view of the dictionary’s keys. See the\n' - ' documentation of view objects.\n' - '\n' - ' pop(key[, default])\n' - '\n' - ' If *key* is in the dictionary, remove it and return ' - 'its value,\n' - ' else return *default*. If *default* is not given and ' - '*key* is\n' - ' not in the dictionary, a "KeyError" is raised.\n' - '\n' - ' popitem()\n' - '\n' - ' Remove and return an arbitrary "(key, value)" pair ' - 'from the\n' - ' dictionary.\n' - '\n' - ' "popitem()" is useful to destructively iterate over a\n' - ' dictionary, as often used in set algorithms. If the ' - 'dictionary\n' - ' is empty, calling "popitem()" raises a "KeyError".\n' - '\n' - ' setdefault(key[, default])\n' - '\n' - ' If *key* is in the dictionary, return its value. If ' - 'not, insert\n' - ' *key* with a value of *default* and return *default*. ' - '*default*\n' - ' defaults to "None".\n' - '\n' - ' update([other])\n' - '\n' - ' Update the dictionary with the key/value pairs from ' - '*other*,\n' - ' overwriting existing keys. Return "None".\n' - '\n' - ' "update()" accepts either another dictionary object or ' - 'an\n' - ' iterable of key/value pairs (as tuples or other ' - 'iterables of\n' - ' length two). If keyword arguments are specified, the ' - 'dictionary\n' - ' is then updated with those key/value pairs: ' - '"d.update(red=1,\n' - ' blue=2)".\n' - '\n' - ' values()\n' - '\n' - ' Return a new view of the dictionary’s values. See ' - 'the\n' - ' documentation of view objects.\n' - '\n' - ' Dictionaries compare equal if and only if they have the ' - 'same "(key,\n' - ' value)" pairs. Order comparisons (‘<’, ‘<=’, ‘>=’, ‘>’) ' - 'raise\n' - ' "TypeError".\n' - '\n' - 'See also: "types.MappingProxyType" can be used to create a ' - 'read-only\n' - ' view of a "dict".\n' - '\n' - '\n' - 'Dictionary view objects\n' - '=======================\n' - '\n' - 'The objects returned by "dict.keys()", "dict.values()" and\n' - '"dict.items()" are *view objects*. They provide a dynamic ' - 'view on the\n' - 'dictionary’s entries, which means that when the dictionary ' - 'changes,\n' - 'the view reflects these changes.\n' - '\n' - 'Dictionary views can be iterated over to yield their ' - 'respective data,\n' - 'and support membership tests:\n' - '\n' - 'len(dictview)\n' - '\n' - ' Return the number of entries in the dictionary.\n' - '\n' - 'iter(dictview)\n' - '\n' - ' Return an iterator over the keys, values or items ' - '(represented as\n' - ' tuples of "(key, value)") in the dictionary.\n' - '\n' - ' Keys and values are iterated over in an arbitrary order ' - 'which is\n' - ' non-random, varies across Python implementations, and ' - 'depends on\n' - ' the dictionary’s history of insertions and deletions. If ' - 'keys,\n' - ' values and items views are iterated over with no ' - 'intervening\n' - ' modifications to the dictionary, the order of items will ' - 'directly\n' - ' correspond. This allows the creation of "(value, key)" ' - 'pairs using\n' - ' "zip()": "pairs = zip(d.values(), d.keys())". Another ' - 'way to\n' - ' create the same list is "pairs = [(v, k) for (k, v) in ' - 'd.items()]".\n' - '\n' - ' Iterating views while adding or deleting entries in the ' - 'dictionary\n' - ' may raise a "RuntimeError" or fail to iterate over all ' - 'entries.\n' - '\n' - 'x in dictview\n' - '\n' - ' Return "True" if *x* is in the underlying dictionary’s ' - 'keys, values\n' - ' or items (in the latter case, *x* should be a "(key, ' - 'value)"\n' - ' tuple).\n' - '\n' - 'Keys views are set-like since their entries are unique and ' - 'hashable.\n' - 'If all values are hashable, so that "(key, value)" pairs are ' - 'unique\n' - 'and hashable, then the items view is also set-like. (Values ' - 'views are\n' - 'not treated as set-like since the entries are generally not ' - 'unique.)\n' - 'For set-like views, all of the operations defined for the ' - 'abstract\n' - 'base class "collections.abc.Set" are available (for example, ' - '"==",\n' - '"<", or "^").\n' - '\n' - 'An example of dictionary view usage:\n' - '\n' - " >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, " - "'spam': 500}\n" - ' >>> keys = dishes.keys()\n' - ' >>> values = dishes.values()\n' - '\n' - ' >>> # iteration\n' - ' >>> n = 0\n' - ' >>> for val in values:\n' - ' ... n += val\n' - ' >>> print(n)\n' - ' 504\n' - '\n' - ' >>> # keys and values are iterated over in the same ' - 'order\n' - ' >>> list(keys)\n' - " ['eggs', 'bacon', 'sausage', 'spam']\n" - ' >>> list(values)\n' - ' [2, 1, 1, 500]\n' - '\n' - ' >>> # view objects are dynamic and reflect dict changes\n' - " >>> del dishes['eggs']\n" - " >>> del dishes['sausage']\n" - ' >>> list(keys)\n' - " ['spam', 'bacon']\n" - '\n' - ' >>> # set operations\n' - " >>> keys & {'eggs', 'bacon', 'salad'}\n" - " {'bacon'}\n" - " >>> keys ^ {'sausage', 'juice'}\n" - " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': 'Methods\n' - '*******\n' - '\n' - 'Methods are functions that are called using the attribute ' - 'notation.\n' - 'There are two flavors: built-in methods (such as "append()" ' - 'on lists)\n' - 'and class instance methods. Built-in methods are described ' - 'with the\n' - 'types that support them.\n' - '\n' - 'If you access a method (a function defined in a class ' - 'namespace)\n' - 'through an instance, you get a special object: a *bound ' - 'method* (also\n' - 'called *instance method*) object. When called, it will add ' - 'the "self"\n' - 'argument to the argument list. Bound methods have two ' - 'special read-\n' - 'only attributes: "m.__self__" is the object on which the ' - 'method\n' - 'operates, and "m.__func__" is the function implementing the ' - 'method.\n' - 'Calling "m(arg-1, arg-2, ..., arg-n)" is completely ' - 'equivalent to\n' - 'calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n' - '\n' - 'Like function objects, bound method objects support getting ' - 'arbitrary\n' - 'attributes. However, since method attributes are actually ' - 'stored on\n' - 'the underlying function object ("meth.__func__"), setting ' - 'method\n' - 'attributes on bound methods is disallowed. Attempting to ' - 'set an\n' - 'attribute on a method results in an "AttributeError" being ' - 'raised. In\n' - 'order to set a method attribute, you need to explicitly set ' - 'it on the\n' - 'underlying function object:\n' - '\n' - ' >>> class C:\n' - ' ... def method(self):\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - " >>> c.method.whoami = 'my name is method' # can't set on " - 'the method\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " AttributeError: 'method' object has no attribute " - "'whoami'\n" - " >>> c.method.__func__.whoami = 'my name is method'\n" - ' >>> c.method.whoami\n' - " 'my name is method'\n" - '\n' - 'See The standard type hierarchy for more information.\n', - 'typesmodules': 'Modules\n' - '*******\n' - '\n' - 'The only special operation on a module is attribute access: ' - '"m.name",\n' - 'where *m* is a module and *name* accesses a name defined in ' - '*m*’s\n' - 'symbol table. Module attributes can be assigned to. (Note ' - 'that the\n' - '"import" statement is not, strictly speaking, an operation ' - 'on a module\n' - 'object; "import foo" does not require a module object named ' - '*foo* to\n' - 'exist, rather it requires an (external) *definition* for a ' - 'module\n' - 'named *foo* somewhere.)\n' - '\n' - 'A special attribute of every module is "__dict__". This is ' - 'the\n' - 'dictionary containing the module’s symbol table. Modifying ' - 'this\n' - 'dictionary will actually change the module’s symbol table, ' - 'but direct\n' - 'assignment to the "__dict__" attribute is not possible (you ' - 'can write\n' - '"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but ' - 'you can’t\n' - 'write "m.__dict__ = {}"). Modifying "__dict__" directly is ' - 'not\n' - 'recommended.\n' - '\n' - 'Modules built into the interpreter are written like this: ' - '"". If loaded from a file, they are ' - 'written as\n' - '"".\n', - 'typesseq': 'Sequence Types — "list", "tuple", "range"\n' - '*****************************************\n' - '\n' - 'There are three basic sequence types: lists, tuples, and range\n' - 'objects. Additional sequence types tailored for processing of ' - 'binary\n' - 'data and text strings are described in dedicated sections.\n' - '\n' - '\n' - 'Common Sequence Operations\n' - '==========================\n' - '\n' - 'The operations in the following table are supported by most ' - 'sequence\n' - 'types, both mutable and immutable. The ' - '"collections.abc.Sequence" ABC\n' - 'is provided to make it easier to correctly implement these ' - 'operations\n' - 'on custom sequence types.\n' - '\n' - 'This table lists the sequence operations sorted in ascending ' - 'priority.\n' - 'In the table, *s* and *t* are sequences of the same type, *n*, ' - '*i*,\n' - '*j* and *k* are integers and *x* is an arbitrary object that ' - 'meets any\n' - 'type and value restrictions imposed by *s*.\n' - '\n' - 'The "in" and "not in" operations have the same priorities as ' - 'the\n' - 'comparison operations. The "+" (concatenation) and "*" ' - '(repetition)\n' - 'operations have the same priority as the corresponding numeric\n' - 'operations. [3]\n' - '\n' - '+----------------------------+----------------------------------+------------+\n' - '| Operation | Result ' - '| Notes |\n' - '+============================+==================================+============+\n' - '| "x in s" | "True" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "False" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "x not in s" | "False" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "True" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s + t" | the concatenation of *s* and *t* ' - '| (6)(7) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s * n" or "n * s" | equivalent to adding *s* to ' - '| (2)(7) |\n' - '| | itself *n* times ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i]" | *i*th item of *s*, origin 0 ' - '| (3) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j]" | slice of *s* from *i* to *j* ' - '| (3)(4) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j:k]" | slice of *s* from *i* to *j* ' - '| (3)(5) |\n' - '| | with step *k* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "len(s)" | length of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "min(s)" | smallest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "max(s)" | largest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.index(x[, i[, j]])" | index of the first occurrence of ' - '| (8) |\n' - '| | *x* in *s* (at or after index ' - '| |\n' - '| | *i* and before index *j*) ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.count(x)" | total number of occurrences of ' - '| |\n' - '| | *x* in *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '\n' - 'Sequences of the same type also support comparisons. In ' - 'particular,\n' - 'tuples and lists are compared lexicographically by comparing\n' - 'corresponding elements. This means that to compare equal, every\n' - 'element must compare equal and the two sequences must be of the ' - 'same\n' - 'type and have the same length. (For full details see ' - 'Comparisons in\n' - 'the language reference.)\n' - '\n' - 'Notes:\n' - '\n' - '1. While the "in" and "not in" operations are used only for ' - 'simple\n' - ' containment testing in the general case, some specialised ' - 'sequences\n' - ' (such as "str", "bytes" and "bytearray") also use them for\n' - ' subsequence testing:\n' - '\n' - ' >>> "gg" in "eggs"\n' - ' True\n' - '\n' - '2. Values of *n* less than "0" are treated as "0" (which yields ' - 'an\n' - ' empty sequence of the same type as *s*). Note that items in ' - 'the\n' - ' sequence *s* are not copied; they are referenced multiple ' - 'times.\n' - ' This often haunts new Python programmers; consider:\n' - '\n' - ' >>> lists = [[]] * 3\n' - ' >>> lists\n' - ' [[], [], []]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists\n' - ' [[3], [3], [3]]\n' - '\n' - ' What has happened is that "[[]]" is a one-element list ' - 'containing\n' - ' an empty list, so all three elements of "[[]] * 3" are ' - 'references\n' - ' to this single empty list. Modifying any of the elements of\n' - ' "lists" modifies this single list. You can create a list of\n' - ' different lists this way:\n' - '\n' - ' >>> lists = [[] for i in range(3)]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists[1].append(5)\n' - ' >>> lists[2].append(7)\n' - ' >>> lists\n' - ' [[3], [5], [7]]\n' - '\n' - ' Further explanation is available in the FAQ entry How do I ' - 'create a\n' - ' multidimensional list?.\n' - '\n' - '3. If *i* or *j* is negative, the index is relative to the end ' - 'of\n' - ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' - 'But\n' - ' note that "-0" is still "0".\n' - '\n' - '4. The slice of *s* from *i* to *j* is defined as the sequence ' - 'of\n' - ' items with index *k* such that "i <= k < j". If *i* or *j* ' - 'is\n' - ' greater than "len(s)", use "len(s)". If *i* is omitted or ' - '"None",\n' - ' use "0". If *j* is omitted or "None", use "len(s)". If *i* ' - 'is\n' - ' greater than or equal to *j*, the slice is empty.\n' - '\n' - '5. The slice of *s* from *i* to *j* with step *k* is defined as ' - 'the\n' - ' sequence of items with index "x = i + n*k" such that "0 <= n ' - '<\n' - ' (j-i)/k". In other words, the indices are "i", "i+k", ' - '"i+2*k",\n' - ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). When *k* is positive, *i* and *j* are ' - 'reduced to\n' - ' "len(s)" if they are greater. When *k* is negative, *i* and ' - '*j* are\n' - ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' - 'are\n' - ' omitted or "None", they become “end” values (which end ' - 'depends on\n' - ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' - '"None", it\n' - ' is treated like "1".\n' - '\n' - '6. Concatenating immutable sequences always results in a new\n' - ' object. This means that building up a sequence by repeated\n' - ' concatenation will have a quadratic runtime cost in the ' - 'total\n' - ' sequence length. To get a linear runtime cost, you must ' - 'switch to\n' - ' one of the alternatives below:\n' - '\n' - ' * if concatenating "str" objects, you can build a list and ' - 'use\n' - ' "str.join()" at the end or else write to an "io.StringIO"\n' - ' instance and retrieve its value when complete\n' - '\n' - ' * if concatenating "bytes" objects, you can similarly use\n' - ' "bytes.join()" or "io.BytesIO", or you can do in-place\n' - ' concatenation with a "bytearray" object. "bytearray" ' - 'objects are\n' - ' mutable and have an efficient overallocation mechanism\n' - '\n' - ' * if concatenating "tuple" objects, extend a "list" instead\n' - '\n' - ' * for other types, investigate the relevant class ' - 'documentation\n' - '\n' - '7. Some sequence types (such as "range") only support item\n' - ' sequences that follow specific patterns, and hence don’t ' - 'support\n' - ' sequence concatenation or repetition.\n' - '\n' - '8. "index" raises "ValueError" when *x* is not found in *s*. ' - 'Not\n' - ' all implementations support passing the additional arguments ' - '*i*\n' - ' and *j*. These arguments allow efficient searching of ' - 'subsections\n' - ' of the sequence. Passing the extra arguments is roughly ' - 'equivalent\n' - ' to using "s[i:j].index(x)", only without copying any data and ' - 'with\n' - ' the returned index being relative to the start of the ' - 'sequence\n' - ' rather than the start of the slice.\n' - '\n' - '\n' - 'Immutable Sequence Types\n' - '========================\n' - '\n' - 'The only operation that immutable sequence types generally ' - 'implement\n' - 'that is not also implemented by mutable sequence types is ' - 'support for\n' - 'the "hash()" built-in.\n' - '\n' - 'This support allows immutable sequences, such as "tuple" ' - 'instances, to\n' - 'be used as "dict" keys and stored in "set" and "frozenset" ' - 'instances.\n' - '\n' - 'Attempting to hash an immutable sequence that contains ' - 'unhashable\n' - 'values will result in "TypeError".\n' - '\n' - '\n' - 'Mutable Sequence Types\n' - '======================\n' - '\n' - 'The operations in the following table are defined on mutable ' - 'sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is provided to ' - 'make\n' - 'it easier to correctly implement these operations on custom ' - 'sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence type, *t* ' - 'is any\n' - 'iterable object and *x* is an arbitrary object that meets any ' - 'type and\n' - 'value restrictions imposed by *s* (for example, "bytearray" ' - 'only\n' - 'accepts integers that meet the value restriction "0 <= x <= ' - '255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes |\n' - '+================================+==================================+=======================+\n' - '| "s[i] = x" | item *i* of *s* is replaced ' - 'by | |\n' - '| | ' - '*x* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* to *j* ' - 'is | |\n' - '| | replaced by the contents of ' - 'the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of "s[i:j:k]" ' - 'are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the end of ' - 'the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items from *s* ' - '(same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow copy of ' - '*s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the contents ' - 'of | |\n' - '| | *t* (for the most part the ' - 'same | |\n' - '| | as "s[len(s):len(s)] = ' - 't") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* at ' - 'the | |\n' - '| | index given by *i* (same ' - 'as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at *i* ' - 'and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item from ' - '*s* | (3) |\n' - '| | where "s[i] == ' - 'x" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of *s* ' - 'in | (4) |\n' - '| | ' - 'place | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that by ' - 'default\n' - ' the last item is removed and returned.\n' - '\n' - '3. "remove" raises "ValueError" when *x* is not found in *s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place for\n' - ' economy of space when reversing a large sequence. To remind ' - 'users\n' - ' that it operates by side effect, it does not return the ' - 'reversed\n' - ' sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency with the\n' - ' interfaces of mutable containers that don’t support slicing\n' - ' operations (such as "dict" and "set")\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object implementing\n' - ' "__index__()". Zero and negative values of *n* clear the ' - 'sequence.\n' - ' Items in the sequence are not copied; they are referenced ' - 'multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n' - '\n' - '\n' - 'Lists\n' - '=====\n' - '\n' - 'Lists are mutable sequences, typically used to store collections ' - 'of\n' - 'homogeneous items (where the precise degree of similarity will ' - 'vary by\n' - 'application).\n' - '\n' - 'class list([iterable])\n' - '\n' - ' Lists may be constructed in several ways:\n' - '\n' - ' * Using a pair of square brackets to denote the empty list: ' - '"[]"\n' - '\n' - ' * Using square brackets, separating items with commas: ' - '"[a]",\n' - ' "[a, b, c]"\n' - '\n' - ' * Using a list comprehension: "[x for x in iterable]"\n' - '\n' - ' * Using the type constructor: "list()" or "list(iterable)"\n' - '\n' - ' The constructor builds a list whose items are the same and in ' - 'the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a list, a copy is made and\n' - ' returned, similar to "iterable[:]". For example, ' - '"list(\'abc\')"\n' - ' returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" ' - 'returns "[1, 2,\n' - ' 3]". If no argument is given, the constructor creates a new ' - 'empty\n' - ' list, "[]".\n' - '\n' - ' Many other operations also produce lists, including the ' - '"sorted()"\n' - ' built-in.\n' - '\n' - ' Lists implement all of the common and mutable sequence ' - 'operations.\n' - ' Lists also provide the following additional method:\n' - '\n' - ' sort(*, key=None, reverse=False)\n' - '\n' - ' This method sorts the list in place, using only "<" ' - 'comparisons\n' - ' between items. Exceptions are not suppressed - if any ' - 'comparison\n' - ' operations fail, the entire sort operation will fail (and ' - 'the\n' - ' list will likely be left in a partially modified state).\n' - '\n' - ' "sort()" accepts two arguments that can only be passed by\n' - ' keyword (keyword-only arguments):\n' - '\n' - ' *key* specifies a function of one argument that is used ' - 'to\n' - ' extract a comparison key from each list element (for ' - 'example,\n' - ' "key=str.lower"). The key corresponding to each item in ' - 'the list\n' - ' is calculated once and then used for the entire sorting ' - 'process.\n' - ' The default value of "None" means that list items are ' - 'sorted\n' - ' directly without calculating a separate key value.\n' - '\n' - ' The "functools.cmp_to_key()" utility is available to ' - 'convert a\n' - ' 2.x style *cmp* function to a *key* function.\n' - '\n' - ' *reverse* is a boolean value. If set to "True", then the ' - 'list\n' - ' elements are sorted as if each comparison were reversed.\n' - '\n' - ' This method modifies the sequence in place for economy of ' - 'space\n' - ' when sorting a large sequence. To remind users that it ' - 'operates\n' - ' by side effect, it does not return the sorted sequence ' - '(use\n' - ' "sorted()" to explicitly request a new sorted list ' - 'instance).\n' - '\n' - ' The "sort()" method is guaranteed to be stable. A sort ' - 'is\n' - ' stable if it guarantees not to change the relative order ' - 'of\n' - ' elements that compare equal — this is helpful for sorting ' - 'in\n' - ' multiple passes (for example, sort by department, then by ' - 'salary\n' - ' grade).\n' - '\n' - ' **CPython implementation detail:** While a list is being ' - 'sorted,\n' - ' the effect of attempting to mutate, or even inspect, the ' - 'list is\n' - ' undefined. The C implementation of Python makes the list ' - 'appear\n' - ' empty for the duration, and raises "ValueError" if it can ' - 'detect\n' - ' that the list has been mutated during a sort.\n' - '\n' - '\n' - 'Tuples\n' - '======\n' - '\n' - 'Tuples are immutable sequences, typically used to store ' - 'collections of\n' - 'heterogeneous data (such as the 2-tuples produced by the ' - '"enumerate()"\n' - 'built-in). Tuples are also used for cases where an immutable ' - 'sequence\n' - 'of homogeneous data is needed (such as allowing storage in a ' - '"set" or\n' - '"dict" instance).\n' - '\n' - 'class tuple([iterable])\n' - '\n' - ' Tuples may be constructed in a number of ways:\n' - '\n' - ' * Using a pair of parentheses to denote the empty tuple: ' - '"()"\n' - '\n' - ' * Using a trailing comma for a singleton tuple: "a," or ' - '"(a,)"\n' - '\n' - ' * Separating items with commas: "a, b, c" or "(a, b, c)"\n' - '\n' - ' * Using the "tuple()" built-in: "tuple()" or ' - '"tuple(iterable)"\n' - '\n' - ' The constructor builds a tuple whose items are the same and ' - 'in the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a tuple, it is returned\n' - ' unchanged. For example, "tuple(\'abc\')" returns "(\'a\', ' - '\'b\', \'c\')"\n' - ' and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument ' - 'is\n' - ' given, the constructor creates a new empty tuple, "()".\n' - '\n' - ' Note that it is actually the comma which makes a tuple, not ' - 'the\n' - ' parentheses. The parentheses are optional, except in the ' - 'empty\n' - ' tuple case, or when they are needed to avoid syntactic ' - 'ambiguity.\n' - ' For example, "f(a, b, c)" is a function call with three ' - 'arguments,\n' - ' while "f((a, b, c))" is a function call with a 3-tuple as the ' - 'sole\n' - ' argument.\n' - '\n' - ' Tuples implement all of the common sequence operations.\n' - '\n' - 'For heterogeneous collections of data where access by name is ' - 'clearer\n' - 'than access by index, "collections.namedtuple()" may be a more\n' - 'appropriate choice than a simple tuple object.\n' - '\n' - '\n' - 'Ranges\n' - '======\n' - '\n' - 'The "range" type represents an immutable sequence of numbers and ' - 'is\n' - 'commonly used for looping a specific number of times in "for" ' - 'loops.\n' - '\n' - 'class range(stop)\n' - 'class range(start, stop[, step])\n' - '\n' - ' The arguments to the range constructor must be integers ' - '(either\n' - ' built-in "int" or any object that implements the "__index__"\n' - ' special method). If the *step* argument is omitted, it ' - 'defaults to\n' - ' "1". If the *start* argument is omitted, it defaults to "0". ' - 'If\n' - ' *step* is zero, "ValueError" is raised.\n' - '\n' - ' For a positive *step*, the contents of a range "r" are ' - 'determined\n' - ' by the formula "r[i] = start + step*i" where "i >= 0" and ' - '"r[i] <\n' - ' stop".\n' - '\n' - ' For a negative *step*, the contents of the range are still\n' - ' determined by the formula "r[i] = start + step*i", but the\n' - ' constraints are "i >= 0" and "r[i] > stop".\n' - '\n' - ' A range object will be empty if "r[0]" does not meet the ' - 'value\n' - ' constraint. Ranges do support negative indices, but these ' - 'are\n' - ' interpreted as indexing from the end of the sequence ' - 'determined by\n' - ' the positive indices.\n' - '\n' - ' Ranges containing absolute values larger than "sys.maxsize" ' - 'are\n' - ' permitted but some features (such as "len()") may raise\n' - ' "OverflowError".\n' - '\n' - ' Range examples:\n' - '\n' - ' >>> list(range(10))\n' - ' [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n' - ' >>> list(range(1, 11))\n' - ' [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n' - ' >>> list(range(0, 30, 5))\n' - ' [0, 5, 10, 15, 20, 25]\n' - ' >>> list(range(0, 10, 3))\n' - ' [0, 3, 6, 9]\n' - ' >>> list(range(0, -10, -1))\n' - ' [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n' - ' >>> list(range(0))\n' - ' []\n' - ' >>> list(range(1, 0))\n' - ' []\n' - '\n' - ' Ranges implement all of the common sequence operations ' - 'except\n' - ' concatenation and repetition (due to the fact that range ' - 'objects\n' - ' can only represent sequences that follow a strict pattern ' - 'and\n' - ' repetition and concatenation will usually violate that ' - 'pattern).\n' - '\n' - ' start\n' - '\n' - ' The value of the *start* parameter (or "0" if the ' - 'parameter was\n' - ' not supplied)\n' - '\n' - ' stop\n' - '\n' - ' The value of the *stop* parameter\n' - '\n' - ' step\n' - '\n' - ' The value of the *step* parameter (or "1" if the parameter ' - 'was\n' - ' not supplied)\n' - '\n' - 'The advantage of the "range" type over a regular "list" or ' - '"tuple" is\n' - 'that a "range" object will always take the same (small) amount ' - 'of\n' - 'memory, no matter the size of the range it represents (as it ' - 'only\n' - 'stores the "start", "stop" and "step" values, calculating ' - 'individual\n' - 'items and subranges as needed).\n' - '\n' - 'Range objects implement the "collections.abc.Sequence" ABC, and\n' - 'provide features such as containment tests, element index ' - 'lookup,\n' - 'slicing and support for negative indices (see Sequence Types — ' - 'list,\n' - 'tuple, range):\n' - '\n' - '>>> r = range(0, 20, 2)\n' - '>>> r\n' - 'range(0, 20, 2)\n' - '>>> 11 in r\n' - 'False\n' - '>>> 10 in r\n' - 'True\n' - '>>> r.index(10)\n' - '5\n' - '>>> r[5]\n' - '10\n' - '>>> r[:5]\n' - 'range(0, 10, 2)\n' - '>>> r[-1]\n' - '18\n' - '\n' - 'Testing range objects for equality with "==" and "!=" compares ' - 'them as\n' - 'sequences. That is, two range objects are considered equal if ' - 'they\n' - 'represent the same sequence of values. (Note that two range ' - 'objects\n' - 'that compare equal might have different "start", "stop" and ' - '"step"\n' - 'attributes, for example "range(0) == range(2, 1, 3)" or ' - '"range(0, 3,\n' - '2) == range(0, 4, 2)".)\n' - '\n' - 'Changed in version 3.2: Implement the Sequence ABC. Support ' - 'slicing\n' - 'and negative indices. Test "int" objects for membership in ' - 'constant\n' - 'time instead of iterating through all items.\n' - '\n' - 'Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range ' - 'objects\n' - 'based on the sequence of values they define (instead of ' - 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' - '\n' - 'See also:\n' - '\n' - ' * The linspace recipe shows how to implement a lazy version ' - 'of\n' - ' range suitable for floating point applications.\n', - 'typesseq-mutable': 'Mutable Sequence Types\n' - '**********************\n' - '\n' - 'The operations in the following table are defined on ' - 'mutable sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is ' - 'provided to make\n' - 'it easier to correctly implement these operations on ' - 'custom sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence ' - 'type, *t* is any\n' - 'iterable object and *x* is an arbitrary object that ' - 'meets any type and\n' - 'value restrictions imposed by *s* (for example, ' - '"bytearray" only\n' - 'accepts integers that meet the value restriction "0 <= x ' - '<= 255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes ' - '|\n' - '+================================+==================================+=======================+\n' - '| "s[i] = x" | item *i* of *s* is ' - 'replaced by | |\n' - '| | ' - '*x* | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* ' - 'to *j* is | |\n' - '| | replaced by the ' - 'contents of the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of ' - '"s[i:j:k]" are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the ' - 'end of the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items ' - 'from *s* (same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow ' - 'copy of *s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the ' - 'contents of | |\n' - '| | *t* (for the most ' - 'part the same | |\n' - '| | as "s[len(s):len(s)] ' - '= t") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* ' - 'at the | |\n' - '| | index given by *i* ' - '(same as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at ' - '*i* and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item ' - 'from *s* | (3) |\n' - '| | where "s[i] == ' - 'x" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of ' - '*s* in | (4) |\n' - '| | ' - 'place | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is ' - 'replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that ' - 'by default\n' - ' the last item is removed and returned.\n' - '\n' - '3. "remove" raises "ValueError" when *x* is not found in ' - '*s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place ' - 'for\n' - ' economy of space when reversing a large sequence. To ' - 'remind users\n' - ' that it operates by side effect, it does not return ' - 'the reversed\n' - ' sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency ' - 'with the\n' - ' interfaces of mutable containers that don’t support ' - 'slicing\n' - ' operations (such as "dict" and "set")\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object ' - 'implementing\n' - ' "__index__()". Zero and negative values of *n* clear ' - 'the sequence.\n' - ' Items in the sequence are not copied; they are ' - 'referenced multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n', - 'unary': 'Unary arithmetic and bitwise operations\n' - '***************************************\n' - '\n' - 'All unary arithmetic and bitwise operations have the same ' - 'priority:\n' - '\n' - ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' - '\n' - 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument.\n' - '\n' - 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged.\n' - '\n' - 'The unary "~" (invert) operator yields the bitwise inversion of ' - 'its\n' - 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers.\n' - '\n' - 'In all three cases, if the argument does not have the proper type, ' - 'a\n' - '"TypeError" exception is raised.\n', - 'while': 'The "while" statement\n' - '*********************\n' - '\n' - 'The "while" statement is used for repeated execution as long as an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, executes ' - 'the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n', - 'with': 'The "with" statement\n' - '********************\n' - '\n' - 'The "with" statement is used to wrap the execution of a block with\n' - 'methods defined by a context manager (see section With Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the "with_item")\n' - ' is evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' - '\n' - ' Note: The "with" statement guarantees that if the "__enter__()"\n' - ' method returns without an error, then "__exit__()" will always ' - 'be\n' - ' called. Thus, if an error occurs during the assignment to the\n' - ' target list, it will be treated the same as an error occurring\n' - ' within the suite would be. See step 6 below.\n' - '\n' - '5. The suite is executed.\n' - '\n' - '6. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an exception, ' - 'the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'With more than one item, the context managers are processed as if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' suite\n' - '\n' - 'is equivalent to\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' suite\n' - '\n' - 'Changed in version 3.1: Support for multiple context expressions.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n', - 'yield': 'The "yield" statement\n' - '*********************\n' - '\n' - ' yield_stmt ::= yield_expression\n' - '\n' - 'A "yield" statement is semantically equivalent to a yield ' - 'expression.\n' - 'The yield statement can be used to omit the parentheses that would\n' - 'otherwise be required in the equivalent yield expression ' - 'statement.\n' - 'For example, the yield statements\n' - '\n' - ' yield \n' - ' yield from \n' - '\n' - 'are equivalent to the yield expression statements\n' - '\n' - ' (yield )\n' - ' (yield from )\n' - '\n' - 'Yield expressions and statements are only used when defining a\n' - '*generator* function, and are only used in the body of the ' - 'generator\n' - 'function. Using yield in a function definition is sufficient to ' - 'cause\n' - 'that definition to create a generator function instead of a normal\n' - 'function.\n' - '\n' - 'For full details of "yield" semantics, refer to the Yield ' - 'expressions\n' - 'section.\n'} +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 +# as part of the release process. + +topics = { + 'assert': r'''The "assert" statement +********************** + +Assert statements are a convenient way to insert debugging assertions +into a program: + + assert_stmt: "assert" expression ["," expression] + +The simple form, "assert expression", is equivalent to + + if __debug__: + if not expression: raise AssertionError + +The extended form, "assert expression1, expression2", is equivalent to + + if __debug__: + if not expression1: raise AssertionError(expression2) + +These equivalences assume that "__debug__" and "AssertionError" refer +to the built-in variables with those names. In the current +implementation, the built-in variable "__debug__" is "True" under +normal circumstances, "False" when optimization is requested (command +line option "-O"). The current code generator emits no code for an +"assert" statement when optimization is requested at compile time. +Note that it is unnecessary to include the source code for the +expression that failed in the error message; it will be displayed as +part of the stack trace. + +Assignments to "__debug__" are illegal. The value for the built-in +variable is determined when the interpreter starts. +''', + 'assignment': r'''Assignment statements +********************* + +Assignment statements are used to (re)bind names to values and to +modify attributes or items of mutable objects: + + assignment_stmt: (target_list "=")+ (starred_expression | yield_expression) + target_list: target ("," target)* [","] + target: identifier + | "(" [target_list] ")" + | "[" [target_list] "]" + | attributeref + | subscription + | slicing + | "*" target + +(See section Primaries for the syntax definitions for *attributeref*, +*subscription*, and *slicing*.) + +An assignment statement evaluates the expression list (remember that +this can be a single expression or a comma-separated list, the latter +yielding a tuple) and assigns the single resulting object to each of +the target lists, from left to right. + +Assignment is defined recursively depending on the form of the target +(list). When a target is part of a mutable object (an attribute +reference, subscription or slicing), the mutable object must +ultimately perform the assignment and decide about its validity, and +may raise an exception if the assignment is unacceptable. The rules +observed by various types and the exceptions raised are given with the +definition of the object types (see section The standard type +hierarchy). + +Assignment of an object to a target list, optionally enclosed in +parentheses or square brackets, is recursively defined as follows. + +* If the target list is a single target with no trailing comma, + optionally in parentheses, the object is assigned to that target. + +* Else: + + * If the target list contains one target prefixed with an asterisk, + called a “starred” target: The object must be an iterable with at + least as many items as there are targets in the target list, minus + one. The first items of the iterable are assigned, from left to + right, to the targets before the starred target. The final items + of the iterable are assigned to the targets after the starred + target. A list of the remaining items in the iterable is then + assigned to the starred target (the list can be empty). + + * Else: The object must be an iterable with the same number of items + as there are targets in the target list, and the items are + assigned, from left to right, to the corresponding targets. + +Assignment of an object to a single target is recursively defined as +follows. + +* If the target is an identifier (name): + + * If the name does not occur in a "global" or "nonlocal" statement + in the current code block: the name is bound to the object in the + current local namespace. + + * Otherwise: the name is bound to the object in the global namespace + or the outer namespace determined by "nonlocal", respectively. + + The name is rebound if it was already bound. This may cause the + reference count for the object previously bound to the name to reach + zero, causing the object to be deallocated and its destructor (if it + has one) to be called. + +* If the target is an attribute reference: The primary expression in + the reference is evaluated. It should yield an object with + assignable attributes; if this is not the case, "TypeError" is + raised. That object is then asked to assign the assigned object to + the given attribute; if it cannot perform the assignment, it raises + an exception (usually but not necessarily "AttributeError"). + + Note: If the object is a class instance and the attribute reference + occurs on both sides of the assignment operator, the right-hand side + expression, "a.x" can access either an instance attribute or (if no + instance attribute exists) a class attribute. The left-hand side + target "a.x" is always set as an instance attribute, creating it if + necessary. Thus, the two occurrences of "a.x" do not necessarily + refer to the same attribute: if the right-hand side expression + refers to a class attribute, the left-hand side creates a new + instance attribute as the target of the assignment: + + class Cls: + x = 3 # class variable + inst = Cls() + inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 + + This description does not necessarily apply to descriptor + attributes, such as properties created with "property()". + +* If the target is a subscription: The primary expression in the + reference is evaluated. It should yield either a mutable sequence + object (such as a list) or a mapping object (such as a dictionary). + Next, the subscript expression is evaluated. + + If the primary is a mutable sequence object (such as a list), the + subscript must yield an integer. If it is negative, the sequence’s + length is added to it. The resulting value must be a nonnegative + integer less than the sequence’s length, and the sequence is asked + to assign the assigned object to its item with that index. If the + index is out of range, "IndexError" is raised (assignment to a + subscripted sequence cannot add new items to a list). + + If the primary is a mapping object (such as a dictionary), the + subscript must have a type compatible with the mapping’s key type, + and the mapping is then asked to create a key/value pair which maps + the subscript to the assigned object. This can either replace an + existing key/value pair with the same key value, or insert a new + key/value pair (if no key with the same value existed). + + For user-defined objects, the "__setitem__()" method is called with + appropriate arguments. + +* If the target is a slicing: The primary expression in the reference + is evaluated. It should yield a mutable sequence object (such as a + list). The assigned object should be a sequence object of the same + type. Next, the lower and upper bound expressions are evaluated, + insofar they are present; defaults are zero and the sequence’s + length. The bounds should evaluate to integers. If either bound is + negative, the sequence’s length is added to it. The resulting + bounds are clipped to lie between zero and the sequence’s length, + inclusive. Finally, the sequence object is asked to replace the + slice with the items of the assigned sequence. The length of the + slice may be different from the length of the assigned sequence, + thus changing the length of the target sequence, if the target + sequence allows it. + +**CPython implementation detail:** In the current implementation, the +syntax for targets is taken to be the same as for expressions, and +invalid syntax is rejected during the code generation phase, causing +less detailed error messages. + +Although the definition of assignment implies that overlaps between +the left-hand side and the right-hand side are ‘simultaneous’ (for +example "a, b = b, a" swaps two variables), overlaps *within* the +collection of assigned-to variables occur left-to-right, sometimes +resulting in confusion. For instance, the following program prints +"[0, 2]": + + x = [0, 1] + i = 0 + i, x[i] = 1, 2 # i is updated, then x[i] is updated + print(x) + +See also: + + **PEP 3132** - Extended Iterable Unpacking + The specification for the "*target" feature. + + +Augmented assignment statements +=============================== + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) + augtarget: identifier | attributeref | subscription | slicing + augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. + + +Annotated assignment statements +=============================== + +*Annotation* assignment is the combination, in a single statement, of +a variable or attribute annotation and an optional assignment +statement: + + annotated_assignment_stmt: augtarget ":" expression + ["=" (starred_expression | yield_expression)] + +The difference from normal Assignment statements is that only a single +target is allowed. + +The assignment target is considered “simple” if it consists of a +single name that is not enclosed in parentheses. For simple assignment +targets, if in class or module scope, the annotations are gathered in +a lazily evaluated annotation scope. The annotations can be evaluated +using the "__annotations__" attribute of a class or module, or using +the facilities in the "annotationlib" module. + +If the assignment target is not simple (an attribute, subscript node, +or parenthesized name), the annotation is never evaluated. + +If a name is annotated in a function scope, then this name is local +for that scope. Annotations are never evaluated and stored in function +scopes. + +If the right hand side is present, an annotated assignment performs +the actual assignment as if there was no annotation present. If the +right hand side is not present for an expression target, then the +interpreter evaluates the target except for the last "__setitem__()" +or "__setattr__()" call. + +See also: + + **PEP 526** - Syntax for Variable Annotations + The proposal that added syntax for annotating the types of + variables (including class variables and instance variables), + instead of expressing them through comments. + + **PEP 484** - Type hints + The proposal that added the "typing" module to provide a standard + syntax for type annotations that can be used in static analysis + tools and IDEs. + +Changed in version 3.8: Now annotated assignments allow the same +expressions in the right hand side as regular assignments. Previously, +some expressions (like un-parenthesized tuple expressions) caused a +syntax error. + +Changed in version 3.14: Annotations are now lazily evaluated in a +separate annotation scope. If the assignment target is not simple, +annotations are never evaluated. +''', + 'assignment-expressions': r'''Assignment expressions +********************** + + assignment_expression: [identifier ":="] expression + +An assignment expression (sometimes also called a “named expression” +or “walrus”) assigns an "expression" to an "identifier", while also +returning the value of the "expression". + +One common use case is when handling matched regular expressions: + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + + while chunk := file.read(9000): + process(chunk) + +Assignment expressions must be surrounded by parentheses when used as +expression statements and when used as sub-expressions in slicing, +conditional, lambda, keyword-argument, and comprehension-if +expressions and in "assert", "with", and "assignment" statements. In +all other places where they can be used, parentheses are not required, +including in "if" and "while" statements. + +Added in version 3.8: See **PEP 572** for more details about +assignment expressions. +''', + 'async': r'''Coroutines +********** + +Added in version 3.5. + + +Coroutine function definition +============================= + + async_funcdef: [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +========================= + + async_for_stmt: "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +========================== + + async_with_stmt: "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. +''', + 'atom-identifiers': r'''Identifiers (Names) +******************* + +An identifier occurring as an atom is a name. See section Names +(identifiers and keywords) for lexical definition and section Naming +and binding for documentation of naming and binding. + +When the name is bound to an object, evaluation of the atom yields +that object. When a name is not bound, an attempt to evaluate it +raises a "NameError" exception. + + +Private name mangling +===================== + +When an identifier that textually occurs in a class definition begins +with two or more underscore characters and does not end in two or more +underscores, it is considered a *private name* of that class. + +See also: The class specifications. + +More precisely, private names are transformed to a longer form before +code is generated for them. If the transformed name is longer than +255 characters, implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which +the identifier is used but only the following private identifiers are +mangled: + +* Any name used as the name of a variable that is assigned or read or + any name of an attribute being accessed. + + The "__name__" attribute of nested functions, classes, and type + aliases is however not mangled. + +* The name of imported modules, e.g., "__spam" in "import __spam". If + the module is part of a package (i.e., its name contains a dot), the + name is *not* mangled, e.g., the "__foo" in "import __foo.bar" is + not mangled. + +* The name of an imported member, e.g., "__f" in "from spam import + __f". + +The transformation rule is defined as follows: + +* The class name, with leading underscores removed and a single + leading underscore inserted, is inserted in front of the identifier, + e.g., the identifier "__spam" occurring in a class named "Foo", + "_Foo" or "__Foo" is transformed to "_Foo__spam". + +* If the class name consists only of underscores, the transformation + is the identity, e.g., the identifier "__spam" occurring in a class + named "_" or "__" is left as is. +''', + 'atom-literals': r'''Literals +******** + +Python supports string and bytes literals and various numeric +literals: + + literal: strings | NUMBER + +Evaluation of a literal yields an object of the given type (string, +bytes, integer, floating-point number, complex number) with the given +value. The value may be approximated in the case of floating-point +and imaginary (complex) literals. See section Literals for details. +See section String literal concatenation for details on "strings". + +All literals correspond to immutable data types, and hence the +object’s identity is less important than its value. Multiple +evaluations of literals with the same value (either the same +occurrence in the program text or a different occurrence) may obtain +the same object or a different object with the same value. + + +String literal concatenation +============================ + +Multiple adjacent string or bytes literals (delimited by whitespace), +possibly using different quoting conventions, are allowed, and their +meaning is the same as their concatenation: + + >>> "hello" 'world' + "helloworld" + +Formally: + + strings: ( STRING | fstring)+ | tstring+ + +This feature is defined at the syntactical level, so it only works +with literals. To concatenate string expressions at run time, the ‘+’ +operator may be used: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted +strings, and formatted string literals. For example: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes needed, +to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. Also, template string literals +may only be combined with other template string literals: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) +''', + 'attribute-access': r'''Customizing attribute access +**************************** + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +=================================== + +module.__getattr__() +module.__dir__() + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +module.__class__ + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +======================== + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +==================== + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +========= + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. +''', + 'attribute-references': r'''Attribute references +******************** + +An attribute reference is a primary followed by a period and a name: + + attributeref: primary "." identifier + +The primary must evaluate to an object of a type that supports +attribute references, which most objects do. This object is then +asked to produce the attribute whose name is the identifier. The type +and value produced is determined by the object. Multiple evaluations +of the same attribute reference may yield different objects. + +This production can be customized by overriding the +"__getattribute__()" method or the "__getattr__()" method. The +"__getattribute__()" method is called first and either returns a value +or raises "AttributeError" if the attribute is not available. + +If an "AttributeError" is raised and the object has a "__getattr__()" +method, that method is called as a fallback. +''', + 'augassign': r'''Augmented assignment statements +******************************* + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) + augtarget: identifier | attributeref | subscription | slicing + augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. +''', + 'await': r'''Await expression +**************** + +Suspend the execution of *coroutine* on an *awaitable* object. Can +only be used inside a *coroutine function*. + + await_expr: "await" primary + +Added in version 3.5. +''', + 'binary': r'''Binary arithmetic operations +**************************** + +The binary arithmetic operations have the conventional priority +levels. Note that some of these operations also apply to certain non- +numeric types. Apart from the power operator, there are only two +levels, one for multiplicative operators and one for additive +operators: + + m_expr: u_expr | m_expr "*" u_expr | m_expr "@" m_expr | + m_expr "//" u_expr | m_expr "/" u_expr | + m_expr "%" u_expr + a_expr: m_expr | a_expr "+" m_expr | a_expr "-" m_expr + +The "*" (multiplication) operator yields the product of its arguments. +The arguments must either both be numbers, or one argument must be an +integer and the other must be a sequence. In the former case, the +numbers are converted to a common real type and then multiplied +together. In the latter case, sequence repetition is performed; a +negative repetition factor yields an empty sequence. + +This operation can be customized using the special "__mul__()" and +"__rmul__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "@" (at) operator is intended to be used for matrix +multiplication. No builtin Python types implement this operator. + +This operation can be customized using the special "__matmul__()" and +"__rmatmul__()" methods. + +Added in version 3.5. + +The "/" (division) and "//" (floor division) operators yield the +quotient of their arguments. The numeric arguments are first +converted to a common type. Division of integers yields a float, while +floor division of integers results in an integer; the result is that +of mathematical division with the ‘floor’ function applied to the +result. Division by zero raises the "ZeroDivisionError" exception. + +The division operation can be customized using the special +"__truediv__()" and "__rtruediv__()" methods. The floor division +operation can be customized using the special "__floordiv__()" and +"__rfloordiv__()" methods. + +The "%" (modulo) operator yields the remainder from the division of +the first argument by the second. The numeric arguments are first +converted to a common type. A zero right argument raises the +"ZeroDivisionError" exception. The arguments may be floating-point +numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 + +0.34".) The modulo operator always yields a result with the same sign +as its second operand (or zero); the absolute value of the result is +strictly smaller than the absolute value of the second operand [1]. + +The floor division and modulo operators are connected by the following +identity: "x == (x//y)*y + (x%y)". Floor division and modulo are also +connected with the built-in function "divmod()": "divmod(x, y) == +(x//y, x%y)". [2]. + +In addition to performing the modulo operation on numbers, the "%" +operator is also overloaded by string objects to perform old-style +string formatting (also known as interpolation). The syntax for +string formatting is described in the Python Library Reference, +section printf-style String Formatting. + +The *modulo* operation can be customized using the special "__mod__()" +and "__rmod__()" methods. + +The floor division operator, the modulo operator, and the "divmod()" +function are not defined for complex numbers. Instead, convert to a +floating-point number using the "abs()" function if appropriate. + +The "+" (addition) operator yields the sum of its arguments. The +arguments must either both be numbers or both be sequences of the same +type. In the former case, the numbers are converted to a common real +type and then added together. In the latter case, the sequences are +concatenated. + +This operation can be customized using the special "__add__()" and +"__radd__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "-" (subtraction) operator yields the difference of its arguments. +The numeric arguments are first converted to a common real type. + +This operation can be customized using the special "__sub__()" and +"__rsub__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. +''', + 'bitwise': r'''Binary bitwise operations +************************* + +Each of the three bitwise operations has a different priority level: + + and_expr: shift_expr | and_expr "&" shift_expr + xor_expr: and_expr | xor_expr "^" and_expr + or_expr: xor_expr | or_expr "|" xor_expr + +The "&" operator yields the bitwise AND of its arguments, which must +be integers or one of them must be a custom object overriding +"__and__()" or "__rand__()" special methods. + +The "^" operator yields the bitwise XOR (exclusive OR) of its +arguments, which must be integers or one of them must be a custom +object overriding "__xor__()" or "__rxor__()" special methods. + +The "|" operator yields the bitwise (inclusive) OR of its arguments, +which must be integers or one of them must be a custom object +overriding "__or__()" or "__ror__()" special methods. +''', + 'bltin-code-objects': r'''Code Objects +************ + +Code objects are used by the implementation to represent “pseudo- +compiled” executable Python code such as a function body. They differ +from function objects because they don’t contain a reference to their +global execution environment. Code objects are returned by the built- +in "compile()" function and can be extracted from function objects +through their "__code__" attribute. See also the "code" module. + +Accessing "__code__" raises an auditing event "object.__getattr__" +with arguments "obj" and ""__code__"". + +A code object can be executed or evaluated by passing it (instead of a +source string) to the "exec()" or "eval()" built-in functions. + +See The standard type hierarchy for more information. +''', + 'bltin-ellipsis-object': r'''The Ellipsis Object +******************* + +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. + +It is written as "Ellipsis" or "...". + +In typical use, "..." as the "Ellipsis" object appears in a few +different places, for instance: + +* In type annotations, such as callable arguments or tuple elements. + +* As the body of a function instead of a pass statement. + +* In third-party libraries, such as Numpy’s slicing and striding. + +Python also uses three dots in ways that are not "Ellipsis" objects, +for instance: + +* Doctest’s "ELLIPSIS", as a pattern for missing content. + +* The default Python prompt of the *interactive* shell when partial + input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional +English usage to mean omitted content, even in code examples that also +use them as the "Ellipsis". +''', + 'bltin-null-object': r'''The Null Object +*************** + +This object is returned by functions that don’t explicitly return a +value. It supports no special operations. There is exactly one null +object, named "None" (a built-in name). "type(None)()" produces the +same singleton. + +It is written as "None". +''', + 'bltin-type-objects': r'''Type Objects +************ + +Type objects represent the various object types. An object’s type is +accessed by the built-in function "type()". There are no special +operations on types. The standard module "types" defines names for +all standard built-in types. + +Types are written like this: "". +''', + 'booleans': r'''Boolean operations +****************** + + or_test: and_test | or_test "or" and_test + and_test: not_test | and_test "and" not_test + not_test: comparison | "not" not_test + +In the context of Boolean operations, and also when expressions are +used by control flow statements, the following values are interpreted +as false: "False", "None", numeric zero of all types, and empty +strings and containers (including strings, tuples, lists, +dictionaries, sets and frozensets). All other values are interpreted +as true. User-defined objects can customize their truth value by +providing a "__bool__()" method. + +The operator "not" yields "True" if its argument is false, "False" +otherwise. + +The expression "x and y" first evaluates *x*; if *x* is false, its +value is returned; otherwise, *y* is evaluated and the resulting value +is returned. + +The expression "x or y" first evaluates *x*; if *x* is true, its value +is returned; otherwise, *y* is evaluated and the resulting value is +returned. + +Note that neither "and" nor "or" restrict the value and type they +return to "False" and "True", but rather return the last evaluated +argument. This is sometimes useful, e.g., if "s" is a string that +should be replaced by a default value if it is empty, the expression +"s or 'foo'" yields the desired value. Because "not" has to create a +new value, it returns a boolean value regardless of the type of its +argument (for example, "not 'foo'" produces "False" rather than "''".) +''', + 'break': r'''The "break" statement +********************* + + break_stmt: "break" + +"break" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. + +It terminates the nearest enclosing loop, skipping the optional "else" +clause if the loop has one. + +If a "for" loop is terminated by "break", the loop control target +keeps its current value. + +When "break" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +loop. +''', + 'callable-types': r'''Emulating callable objects +************************** + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. +''', + 'calls': r'''Calls +***** + +A call calls a callable object (e.g., a *function*) with a possibly +empty series of *arguments*: + + call: primary "(" [argument_list [","] | comprehension] ")" + argument_list: positional_arguments ["," starred_and_keywords] + ["," keywords_arguments] + | starred_and_keywords ["," keywords_arguments] + | keywords_arguments + positional_arguments: positional_item ("," positional_item)* + positional_item: assignment_expression | "*" expression + starred_and_keywords: ("*" expression | keyword_item) + ("," "*" expression | "," keyword_item)* + keywords_arguments: (keyword_item | "**" expression) + ("," keyword_item | "," "**" expression)* + keyword_item: identifier "=" expression + +An optional trailing comma may be present after the positional and +keyword arguments but does not affect the semantics. + +The primary must evaluate to a callable object (user-defined +functions, built-in functions, methods of built-in objects, class +objects, methods of class instances, and all objects having a +"__call__()" method are callable). All argument expressions are +evaluated before the call is attempted. Please refer to section +Function definitions for the syntax of formal *parameter* lists. + +If keyword arguments are present, they are first converted to +positional arguments, as follows. First, a list of unfilled slots is +created for the formal parameters. If there are N positional +arguments, they are placed in the first N slots. Next, for each +keyword argument, the identifier is used to determine the +corresponding slot (if the identifier is the same as the first formal +parameter name, the first slot is used, and so on). If the slot is +already filled, a "TypeError" exception is raised. Otherwise, the +argument is placed in the slot, filling it (even if the expression is +"None", it fills the slot). When all arguments have been processed, +the slots that are still unfilled are filled with the corresponding +default value from the function definition. (Default values are +calculated, once, when the function is defined; thus, a mutable object +such as a list or dictionary used as default value will be shared by +all calls that don’t specify an argument value for the corresponding +slot; this should usually be avoided.) If there are any unfilled +slots for which no default value is specified, a "TypeError" exception +is raised. Otherwise, the list of filled slots is used as the +argument list for the call. + +**CPython implementation detail:** An implementation may provide +built-in functions whose positional parameters do not have names, even +if they are ‘named’ for the purpose of documentation, and which +therefore cannot be supplied by keyword. In CPython, this is the case +for functions implemented in C that use "PyArg_ParseTuple()" to parse +their arguments. + +If there are more positional arguments than there are formal parameter +slots, a "TypeError" exception is raised, unless a formal parameter +using the syntax "*identifier" is present; in this case, that formal +parameter receives a tuple containing the excess positional arguments +(or an empty tuple if there were no excess positional arguments). + +If any keyword argument does not correspond to a formal parameter +name, a "TypeError" exception is raised, unless a formal parameter +using the syntax "**identifier" is present; in this case, that formal +parameter receives a dictionary containing the excess keyword +arguments (using the keywords as keys and the argument values as +corresponding values), or a (new) empty dictionary if there were no +excess keyword arguments. + +If the syntax "*expression" appears in the function call, "expression" +must evaluate to an *iterable*. Elements from these iterables are +treated as if they were additional positional arguments. For the call +"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, *yM*, +this is equivalent to a call with M+4 positional arguments *x1*, *x2*, +*y1*, …, *yM*, *x3*, *x4*. + +A consequence of this is that although the "*expression" syntax may +appear *after* explicit keyword arguments, it is processed *before* +the keyword arguments (and any "**expression" arguments – see below). +So: + + >>> def f(a, b): + ... print(a, b) + ... + >>> f(b=1, *(2,)) + 2 1 + >>> f(a=1, *(2,)) + Traceback (most recent call last): + File "", line 1, in + TypeError: f() got multiple values for keyword argument 'a' + >>> f(1, *(2,)) + 1 2 + +It is unusual for both keyword arguments and the "*expression" syntax +to be used in the same call, so in practice this confusion does not +often arise. + +If the syntax "**expression" appears in the function call, +"expression" must evaluate to a *mapping*, the contents of which are +treated as additional keyword arguments. If a parameter matching a key +has already been given a value (by an explicit keyword argument, or +from another unpacking), a "TypeError" exception is raised. + +When "**expression" is used, each key in this mapping must be a +string. Each value from the mapping is assigned to the first formal +parameter eligible for keyword assignment whose name is equal to the +key. A key need not be a Python identifier (e.g. ""max-temp °F"" is +acceptable, although it will not match any formal parameter that could +be declared). If there is no match to a formal parameter the key-value +pair is collected by the "**" parameter, if there is one, or if there +is not, a "TypeError" exception is raised. + +Formal parameters using the syntax "*identifier" or "**identifier" +cannot be used as positional argument slots or as keyword argument +names. + +Changed in version 3.5: Function calls accept any number of "*" and +"**" unpackings, positional arguments may follow iterable unpackings +("*"), and keyword arguments may follow dictionary unpackings ("**"). +Originally proposed by **PEP 448**. + +A call always returns some value, possibly "None", unless it raises an +exception. How this value is computed depends on the type of the +callable object. + +If it is— + +a user-defined function: + The code block for the function is executed, passing it the + argument list. The first thing the code block will do is bind the + formal parameters to the arguments; this is described in section + Function definitions. When the code block executes a "return" + statement, this specifies the return value of the function call. + If execution reaches the end of the code block without executing a + "return" statement, the return value is "None". + +a built-in function or method: + The result is up to the interpreter; see Built-in Functions for the + descriptions of built-in functions and methods. + +a class object: + A new instance of that class is returned. + +a class instance method: + The corresponding user-defined function is called, with an argument + list that is one longer than the argument list of the call: the + instance becomes the first argument. + +a class instance: + The class must define a "__call__()" method; the effect is then the + same as if that method was called. +''', + 'class': r'''Class definitions +***************** + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef: [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance: "(" [argument_list] ")" + classname: identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. +''', + 'comparisons': r'''Comparisons +*********** + +Unlike C, all comparison operations in Python have the same priority, +which is lower than that of any arithmetic, shifting or bitwise +operation. Also unlike C, expressions like "a < b < c" have the +interpretation that is conventional in mathematics: + + comparison: or_expr (comp_operator or_expr)* + comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" + | "is" ["not"] | ["not"] "in" + +Comparisons yield boolean values: "True" or "False". Custom *rich +comparison methods* may return non-boolean values. In this case Python +will call "bool()" on such value in boolean contexts. + +Comparisons can be chained arbitrarily, e.g., "x < y <= z" is +equivalent to "x < y and y <= z", except that "y" is evaluated only +once (but in both cases "z" is not evaluated at all when "x < y" is +found to be false). + +Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and *op1*, +*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ... y +opN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except +that each expression is evaluated at most once. + +Note that "a op1 b op2 c" doesn’t imply any kind of comparison between +*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though +perhaps not pretty). + + +Value comparisons +================= + +The operators "<", ">", "==", ">=", "<=", and "!=" compare the values +of two objects. The objects do not need to have the same type. + +Chapter Objects, values and types states that objects have a value (in +addition to type and identity). The value of an object is a rather +abstract notion in Python: For example, there is no canonical access +method for an object’s value. Also, there is no requirement that the +value of an object should be constructed in a particular way, e.g. +comprised of all its data attributes. Comparison operators implement a +particular notion of what the value of an object is. One can think of +them as defining the value of an object indirectly, by means of their +comparison implementation. + +Because all types are (direct or indirect) subtypes of "object", they +inherit the default comparison behavior from "object". Types can +customize their comparison behavior by implementing *rich comparison +methods* like "__lt__()", described in Basic customization. + +The default behavior for equality comparison ("==" and "!=") is based +on the identity of the objects. Hence, equality comparison of +instances with the same identity results in equality, and equality +comparison of instances with different identities results in +inequality. A motivation for this default behavior is the desire that +all objects should be reflexive (i.e. "x is y" implies "x == y"). + +A default order comparison ("<", ">", "<=", and ">=") is not provided; +an attempt raises "TypeError". A motivation for this default behavior +is the lack of a similar invariant as for equality. + +The behavior of the default equality comparison, that instances with +different identities are always unequal, may be in contrast to what +types will need that have a sensible definition of object value and +value-based equality. Such types will need to customize their +comparison behavior, and in fact, a number of built-in types have done +that. + +The following list describes the comparison behavior of the most +important built-in types. + +* Numbers of built-in numeric types (Numeric Types — int, float, + complex) and of the standard library types "fractions.Fraction" and + "decimal.Decimal" can be compared within and across their types, + with the restriction that complex numbers do not support order + comparison. Within the limits of the types involved, they compare + mathematically (algorithmically) correct without loss of precision. + + The not-a-number values "float('NaN')" and "decimal.Decimal('NaN')" + are special. Any ordered comparison of a number to a not-a-number + value is false. A counter-intuitive implication is that not-a-number + values are not equal to themselves. For example, if "x = + float('NaN')", "3 < x", "x < 3" and "x == x" are all false, while "x + != x" is true. This behavior is compliant with IEEE 754. + +* "None" and "NotImplemented" are singletons. **PEP 8** advises that + comparisons for singletons should always be done with "is" or "is + not", never the equality operators. + +* Binary sequences (instances of "bytes" or "bytearray") can be + compared within and across their types. They compare + lexicographically using the numeric values of their elements. + +* Strings (instances of "str") compare lexicographically using the + numerical Unicode code points (the result of the built-in function + "ord()") of their characters. [3] + + Strings and binary sequences cannot be directly compared. + +* Sequences (instances of "tuple", "list", or "range") can be compared + only within each of their types, with the restriction that ranges do + not support order comparison. Equality comparison across these + types results in inequality, and ordering comparison across these + types raises "TypeError". + + Sequences compare lexicographically using comparison of + corresponding elements. The built-in containers typically assume + identical objects are equal to themselves. That lets them bypass + equality tests for identical objects to improve performance and to + maintain their internal invariants. + + Lexicographical comparison between built-in collections works as + follows: + + * For two collections to compare equal, they must be of the same + type, have the same length, and each pair of corresponding + elements must compare equal (for example, "[1,2] == (1,2)" is + false because the type is not the same). + + * Collections that support order comparison are ordered the same as + their first unequal elements (for example, "[1,2,x] <= [1,2,y]" + has the same value as "x <= y"). If a corresponding element does + not exist, the shorter collection is ordered first (for example, + "[1,2] < [1,2,3]" is true). + +* Mappings (instances of "dict") compare equal if and only if they + have equal "(key, value)" pairs. Equality comparison of the keys and + values enforces reflexivity. + + Order comparisons ("<", ">", "<=", and ">=") raise "TypeError". + +* Sets (instances of "set" or "frozenset") can be compared within and + across their types. + + They define order comparison operators to mean subset and superset + tests. Those relations do not define total orderings (for example, + the two sets "{1,2}" and "{2,3}" are not equal, nor subsets of one + another, nor supersets of one another). Accordingly, sets are not + appropriate arguments for functions which depend on total ordering + (for example, "min()", "max()", and "sorted()" produce undefined + results given a list of sets as inputs). + + Comparison of sets enforces reflexivity of its elements. + +* Most other built-in types have no comparison methods implemented, so + they inherit the default comparison behavior. + +User-defined classes that customize their comparison behavior should +follow some consistency rules, if possible: + +* Equality comparison should be reflexive. In other words, identical + objects should compare equal: + + "x is y" implies "x == y" + +* Comparison should be symmetric. In other words, the following + expressions should have the same result: + + "x == y" and "y == x" + + "x != y" and "y != x" + + "x < y" and "y > x" + + "x <= y" and "y >= x" + +* Comparison should be transitive. The following (non-exhaustive) + examples illustrate that: + + "x > y and y > z" implies "x > z" + + "x < y and y <= z" implies "x < z" + +* Inverse comparison should result in the boolean negation. In other + words, the following expressions should have the same result: + + "x == y" and "not x != y" + + "x < y" and "not x >= y" (for total ordering) + + "x > y" and "not x <= y" (for total ordering) + + The last two expressions apply to totally ordered collections (e.g. + to sequences, but not to sets or mappings). See also the + "total_ordering()" decorator. + +* The "hash()" result should be consistent with equality. Objects that + are equal should either have the same hash value, or be marked as + unhashable. + +Python does not enforce these consistency rules. In fact, the +not-a-number values are an example for not following these rules. + + +Membership test operations +========================== + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". + + +Identity comparisons +==================== + +The operators "is" and "is not" test for an object’s identity: "x is +y" is true if and only if *x* and *y* are the same object. An +Object’s identity is determined using the "id()" function. "x is not +y" yields the inverse truth value. [4] +''', + 'compound': r'''Compound statements +******************* + +Compound statements contain (groups of) other statements; they affect +or control the execution of those other statements in some way. In +general, compound statements span multiple lines, although in simple +incarnations a whole compound statement may be contained in one line. + +The "if", "while" and "for" statements implement traditional control +flow constructs. "try" specifies exception handlers and/or cleanup +code for a group of statements, while the "with" statement allows the +execution of initialization and finalization code around a block of +code. Function and class definitions are also syntactically compound +statements. + +A compound statement consists of one or more ‘clauses.’ A clause +consists of a header and a ‘suite.’ The clause headers of a +particular compound statement are all at the same indentation level. +Each clause header begins with a uniquely identifying keyword and ends +with a colon. A suite is a group of statements controlled by a +clause. A suite can be one or more semicolon-separated simple +statements on the same line as the header, following the header’s +colon, or it can be one or more indented statements on subsequent +lines. Only the latter form of a suite can contain nested compound +statements; the following is illegal, mostly because it wouldn’t be +clear to which "if" clause a following "else" clause would belong: + + if test1: if test2: print(x) + +Also note that the semicolon binds tighter than the colon in this +context, so that in the following example, either all or none of the +"print()" calls are executed: + + if x < y < z: print(x); print(y); print(z) + +Summarizing: + + compound_stmt: if_stmt + | while_stmt + | for_stmt + | try_stmt + | with_stmt + | match_stmt + | funcdef + | classdef + | async_with_stmt + | async_for_stmt + | async_funcdef + suite: stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT + statement: stmt_list NEWLINE | compound_stmt + stmt_list: simple_stmt (";" simple_stmt)* [";"] + +Note that statements always end in a "NEWLINE" possibly followed by a +"DEDENT". Also note that optional continuation clauses always begin +with a keyword that cannot start a statement, thus there are no +ambiguities (the ‘dangling "else"’ problem is solved in Python by +requiring nested "if" statements to be indented). + +The formatting of the grammar rules in the following sections places +each clause on a separate line for clarity. + + +The "if" statement +================== + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. + + +The "while" statement +===================== + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt: "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. + + +The "for" statement +=================== + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt: "for" target_list "in" starred_expression_list ":" suite + ["else" ":" suite] + +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. + + +The "try" statement +=================== + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt: try1_stmt | try2_stmt | try3_stmt + try1_stmt: "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt: "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt: "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + +Changed in version 3.14: Support for optionally dropping grouping +parentheses when using multiple exception types. See **PEP 758**. + + +"except" clause +--------------- + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. +Parentheses can be dropped if multiple exception types are provided +and the "as" clause is not used. The raised exception matches an +"except" clause whose expression evaluates to the class or a *non- +virtual base class* of the exception object, or to a tuple that +contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +---------------- + +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError(),)) + +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +------------- + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +---------------- + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded. For example, this function returns 42. + + def f(): + try: + 1/0 + finally: + return 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed. The following function returns ‘finally’. + + def foo(): + try: + return 'try' + finally: + return 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. + +Changed in version 3.14: The compiler emits a "SyntaxWarning" when a +"return", "break" or "continue" appears in a "finally" block (see +**PEP 765**). + + +The "with" statement +==================== + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt: "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents: with_item ("," with_item)* + with_item: expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +The "match" statement +===================== + +Added in version 3.10. + +The match statement is used for pattern matching. Syntax: + + match_stmt: 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT + subject_expr: `!star_named_expression` "," `!star_named_expressions`? + | `!named_expression` + case_block: 'case' patterns [guard] ":" `!block` + +Note: + + This section uses single quotes to denote soft keywords. + +Pattern matching takes a pattern as input (following "case") and a +subject value (following "match"). The pattern (which may contain +subpatterns) is matched against the subject value. The outcomes are: + +* A match success or failure (also termed a pattern success or + failure). + +* Possible binding of matched values to a name. The prerequisites for + this are further discussed below. + +The "match" and "case" keywords are soft keywords. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Overview +-------- + +Here’s an overview of the logical flow of a match statement: + +1. The subject expression "subject_expr" is evaluated and a resulting + subject value obtained. If the subject expression contains a comma, + a tuple is constructed using the standard rules. + +2. Each pattern in a "case_block" is attempted to match with the + subject value. The specific rules for success or failure are + described below. The match attempt can also bind some or all of the + standalone names within the pattern. The precise pattern binding + rules vary per pattern type and are specified below. **Name + bindings made during a successful pattern match outlive the + executed block and can be used after the match statement**. + + Note: + + During failed pattern matches, some subpatterns may succeed. Do + not rely on bindings being made for a failed match. Conversely, + do not rely on variables remaining unchanged after a failed + match. The exact behavior is dependent on implementation and may + vary. This is an intentional decision made to allow different + implementations to add optimizations. + +3. If the pattern succeeds, the corresponding guard (if present) is + evaluated. In this case all name bindings are guaranteed to have + happened. + + * If the guard evaluates as true or is missing, the "block" inside + "case_block" is executed. + + * Otherwise, the next "case_block" is attempted as described above. + + * If there are no further case blocks, the match statement is + completed. + +Note: + + Users should generally never rely on a pattern being evaluated. + Depending on implementation, the interpreter may cache values or use + other optimizations which skip repeated evaluations. + +A sample match statement: + + >>> flag = False + >>> match (100, 200): + ... case (100, 300): # Mismatch: 200 != 300 + ... print('Case 1') + ... case (100, 200) if flag: # Successful match, but guard fails + ... print('Case 2') + ... case (100, y): # Matches and binds y to 200 + ... print(f'Case 3, y: {y}') + ... case _: # Pattern not attempted + ... print('Case 4, I match anything!') + ... + Case 3, y: 200 + +In this case, "if flag" is a guard. Read more about that in the next +section. + + +Guards +------ + + guard: "if" `!named_expression` + +A "guard" (which is part of the "case") must succeed for code inside +the "case" block to execute. It takes the form: "if" followed by an +expression. + +The logical flow of a "case" block with a "guard" follows: + +1. Check that the pattern in the "case" block succeeded. If the + pattern failed, the "guard" is not evaluated and the next "case" + block is checked. + +2. If the pattern succeeded, evaluate the "guard". + + * If the "guard" condition evaluates as true, the case block is + selected. + + * If the "guard" condition evaluates as false, the case block is + not selected. + + * If the "guard" raises an exception during evaluation, the + exception bubbles up. + +Guards are allowed to have side effects as they are expressions. +Guard evaluation must proceed from the first to the last case block, +one at a time, skipping case blocks whose pattern(s) don’t all +succeed. (I.e., guard evaluation must happen in order.) Guard +evaluation must stop once a case block is selected. + + +Irrefutable Case Blocks +----------------------- + +An irrefutable case block is a match-all case block. A match +statement may have at most one irrefutable case block, and it must be +last. + +A case block is considered irrefutable if it has no guard and its +pattern is irrefutable. A pattern is considered irrefutable if we can +prove from its syntax alone that it will always succeed. Only the +following patterns are irrefutable: + +* AS Patterns whose left-hand side is irrefutable + +* OR Patterns containing at least one irrefutable pattern + +* Capture Patterns + +* Wildcard Patterns + +* parenthesized irrefutable patterns + + +Patterns +-------- + +Note: + + This section uses grammar notations beyond standard EBNF: + + * the notation "SEP.RULE+" is shorthand for "RULE (SEP RULE)*" + + * the notation "!RULE" is shorthand for a negative lookahead + assertion + +The top-level syntax for "patterns" is: + + patterns: open_sequence_pattern | pattern + pattern: as_pattern | or_pattern + closed_pattern: | literal_pattern + | capture_pattern + | wildcard_pattern + | value_pattern + | group_pattern + | sequence_pattern + | mapping_pattern + | class_pattern + +The descriptions below will include a description “in simple terms” of +what a pattern does for illustration purposes (credits to Raymond +Hettinger for a document that inspired most of the descriptions). Note +that these descriptions are purely for illustration purposes and **may +not** reflect the underlying implementation. Furthermore, they do not +cover all valid forms. + + +OR Patterns +~~~~~~~~~~~ + +An OR pattern is two or more patterns separated by vertical bars "|". +Syntax: + + or_pattern: "|".closed_pattern+ + +Only the final subpattern may be irrefutable, and each subpattern must +bind the same set of names to avoid ambiguity. + +An OR pattern matches each of its subpatterns in turn to the subject +value, until one succeeds. The OR pattern is then considered +successful. Otherwise, if none of the subpatterns succeed, the OR +pattern fails. + +In simple terms, "P1 | P2 | ..." will try to match "P1", if it fails +it will try to match "P2", succeeding immediately if any succeeds, +failing otherwise. + + +AS Patterns +~~~~~~~~~~~ + +An AS pattern matches an OR pattern on the left of the "as" keyword +against a subject. Syntax: + + as_pattern: or_pattern "as" capture_pattern + +If the OR pattern fails, the AS pattern fails. Otherwise, the AS +pattern binds the subject to the name on the right of the as keyword +and succeeds. "capture_pattern" cannot be a "_". + +In simple terms "P as NAME" will match with "P", and on success it +will set "NAME = ". + + +Literal Patterns +~~~~~~~~~~~~~~~~ + +A literal pattern corresponds to most literals in Python. Syntax: + + literal_pattern: signed_number + | signed_number "+" NUMBER + | signed_number "-" NUMBER + | strings + | "None" + | "True" + | "False" + signed_number: ["-"] NUMBER + +The rule "strings" and the token "NUMBER" are defined in the standard +Python grammar. Triple-quoted strings are supported. Raw strings and +byte strings are supported. f-strings and t-strings are not +supported. + +The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" +are for expressing complex numbers; they require a real number on the +left and an imaginary number on the right. E.g. "3 + 4j". + +In simple terms, "LITERAL" will succeed only if " == +LITERAL". For the singletons "None", "True" and "False", the "is" +operator is used. + + +Capture Patterns +~~~~~~~~~~~~~~~~ + +A capture pattern binds the subject value to a name. Syntax: + + capture_pattern: !'_' NAME + +A single underscore "_" is not a capture pattern (this is what "!'_'" +expresses). It is instead treated as a "wildcard_pattern". + +In a given pattern, a given name can only be bound once. E.g. "case +x, x: ..." is invalid while "case [x] | x: ..." is allowed. + +Capture patterns always succeed. The binding follows scoping rules +established by the assignment expression operator in **PEP 572**; the +name becomes a local variable in the closest containing function scope +unless there’s an applicable "global" or "nonlocal" statement. + +In simple terms "NAME" will always succeed and it will set "NAME = +". + + +Wildcard Patterns +~~~~~~~~~~~~~~~~~ + +A wildcard pattern always succeeds (matches anything) and binds no +name. Syntax: + + wildcard_pattern: '_' + +"_" is a soft keyword within any pattern, but only within patterns. +It is an identifier, as usual, even within "match" subject +expressions, "guard"s, and "case" blocks. + +In simple terms, "_" will always succeed. + + +Value Patterns +~~~~~~~~~~~~~~ + +A value pattern represents a named value in Python. Syntax: + + value_pattern: attr + attr: name_or_attr "." NAME + name_or_attr: attr | NAME + +The dotted name in the pattern is looked up using standard Python name +resolution rules. The pattern succeeds if the value found compares +equal to the subject value (using the "==" equality operator). + +In simple terms "NAME1.NAME2" will succeed only if " == +NAME1.NAME2" + +Note: + + If the same value occurs multiple times in the same match statement, + the interpreter may cache the first value found and reuse it rather + than repeat the same lookup. This cache is strictly tied to a given + execution of a given match statement. + + +Group Patterns +~~~~~~~~~~~~~~ + +A group pattern allows users to add parentheses around patterns to +emphasize the intended grouping. Otherwise, it has no additional +syntax. Syntax: + + group_pattern: "(" pattern ")" + +In simple terms "(P)" has the same effect as "P". + + +Sequence Patterns +~~~~~~~~~~~~~~~~~ + +A sequence pattern contains several subpatterns to be matched against +sequence elements. The syntax is similar to the unpacking of a list or +tuple. + + sequence_pattern: "[" [maybe_sequence_pattern] "]" + | "(" [open_sequence_pattern] ")" + open_sequence_pattern: maybe_star_pattern "," [maybe_sequence_pattern] + maybe_sequence_pattern: ",".maybe_star_pattern+ ","? + maybe_star_pattern: star_pattern | pattern + star_pattern: "*" (capture_pattern | wildcard_pattern) + +There is no difference if parentheses or square brackets are used for +sequence patterns (i.e. "(...)" vs "[...]" ). + +Note: + + A single pattern enclosed in parentheses without a trailing comma + (e.g. "(3 | 4)") is a group pattern. While a single pattern enclosed + in square brackets (e.g. "[3 | 4]") is still a sequence pattern. + +At most one star subpattern may be in a sequence pattern. The star +subpattern may occur in any position. If no star subpattern is +present, the sequence pattern is a fixed-length sequence pattern; +otherwise it is a variable-length sequence pattern. + +The following is the logical flow for matching a sequence pattern +against a subject value: + +1. If the subject value is not a sequence [2], the sequence pattern + fails. + +2. If the subject value is an instance of "str", "bytes" or + "bytearray" the sequence pattern fails. + +3. The subsequent steps depend on whether the sequence pattern is + fixed or variable-length. + + If the sequence pattern is fixed-length: + + 1. If the length of the subject sequence is not equal to the number + of subpatterns, the sequence pattern fails + + 2. Subpatterns in the sequence pattern are matched to their + corresponding items in the subject sequence from left to right. + Matching stops as soon as a subpattern fails. If all + subpatterns succeed in matching their corresponding item, the + sequence pattern succeeds. + + Otherwise, if the sequence pattern is variable-length: + + 1. If the length of the subject sequence is less than the number of + non-star subpatterns, the sequence pattern fails. + + 2. The leading non-star subpatterns are matched to their + corresponding items as for fixed-length sequences. + + 3. If the previous step succeeds, the star subpattern matches a + list formed of the remaining subject items, excluding the + remaining items corresponding to non-star subpatterns following + the star subpattern. + + 4. Remaining non-star subpatterns are matched to their + corresponding subject items, as for a fixed-length sequence. + + Note: + + The length of the subject sequence is obtained via "len()" (i.e. + via the "__len__()" protocol). This length may be cached by the + interpreter in a similar manner as value patterns. + +In simple terms "[P1, P2, P3," … ", P]" matches only if all the +following happens: + +* check "" is a sequence + +* "len(subject) == " + +* "P1" matches "[0]" (note that this match can also bind + names) + +* "P2" matches "[1]" (note that this match can also bind + names) + +* … and so on for the corresponding pattern/element. + + +Mapping Patterns +~~~~~~~~~~~~~~~~ + +A mapping pattern contains one or more key-value patterns. The syntax +is similar to the construction of a dictionary. Syntax: + + mapping_pattern: "{" [items_pattern] "}" + items_pattern: ",".key_value_pattern+ ","? + key_value_pattern: (literal_pattern | value_pattern) ":" pattern + | double_star_pattern + double_star_pattern: "**" capture_pattern + +At most one double star pattern may be in a mapping pattern. The +double star pattern must be the last subpattern in the mapping +pattern. + +Duplicate keys in mapping patterns are disallowed. Duplicate literal +keys will raise a "SyntaxError". Two keys that otherwise have the same +value will raise a "ValueError" at runtime. + +The following is the logical flow for matching a mapping pattern +against a subject value: + +1. If the subject value is not a mapping [3],the mapping pattern + fails. + +2. If every key given in the mapping pattern is present in the subject + mapping, and the pattern for each key matches the corresponding + item of the subject mapping, the mapping pattern succeeds. + +3. If duplicate keys are detected in the mapping pattern, the pattern + is considered invalid. A "SyntaxError" is raised for duplicate + literal values; or a "ValueError" for named keys of the same value. + +Note: + + Key-value pairs are matched using the two-argument form of the + mapping subject’s "get()" method. Matched key-value pairs must + already be present in the mapping, and not created on-the-fly via + "__missing__()" or "__getitem__()". + +In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all the +following happens: + +* check "" is a mapping + +* "KEY1 in " + +* "P1" matches "[KEY1]" + +* … and so on for the corresponding KEY/pattern pair. + + +Class Patterns +~~~~~~~~~~~~~~ + +A class pattern represents a class and its positional and keyword +arguments (if any). Syntax: + + class_pattern: name_or_attr "(" [pattern_arguments ","?] ")" + pattern_arguments: positional_patterns ["," keyword_patterns] + | keyword_patterns + positional_patterns: ",".pattern+ + keyword_patterns: ",".keyword_pattern+ + keyword_pattern: NAME "=" pattern + +The same keyword should not be repeated in class patterns. + +The following is the logical flow for matching a class pattern against +a subject value: + +1. If "name_or_attr" is not an instance of the builtin "type" , raise + "TypeError". + +2. If the subject value is not an instance of "name_or_attr" (tested + via "isinstance()"), the class pattern fails. + +3. If no pattern arguments are present, the pattern succeeds. + Otherwise, the subsequent steps depend on whether keyword or + positional argument patterns are present. + + For a number of built-in types (specified below), a single + positional subpattern is accepted which will match the entire + subject; for these types keyword patterns also work as for other + types. + + If only keyword patterns are present, they are processed as + follows, one by one: + + 1. The keyword is looked up as an attribute on the subject. + + * If this raises an exception other than "AttributeError", the + exception bubbles up. + + * If this raises "AttributeError", the class pattern has failed. + + * Else, the subpattern associated with the keyword pattern is + matched against the subject’s attribute value. If this fails, + the class pattern fails; if this succeeds, the match proceeds + to the next keyword. + + 2. If all keyword patterns succeed, the class pattern succeeds. + + If any positional patterns are present, they are converted to + keyword patterns using the "__match_args__" attribute on the class + "name_or_attr" before matching: + + 1. The equivalent of "getattr(cls, "__match_args__", ())" is + called. + + * If this raises an exception, the exception bubbles up. + + * If the returned value is not a tuple, the conversion fails and + "TypeError" is raised. + + * If there are more positional patterns than + "len(cls.__match_args__)", "TypeError" is raised. + + * Otherwise, positional pattern "i" is converted to a keyword + pattern using "__match_args__[i]" as the keyword. + "__match_args__[i]" must be a string; if not "TypeError" is + raised. + + * If there are duplicate keywords, "TypeError" is raised. + + See also: + + Customizing positional arguments in class pattern matching + + 2. Once all positional patterns have been converted to keyword + patterns, the match proceeds as if there were only keyword + patterns. + + For the following built-in types the handling of positional + subpatterns is different: + + * "bool" + + * "bytearray" + + * "bytes" + + * "dict" + + * "float" + + * "frozenset" + + * "int" + + * "list" + + * "set" + + * "str" + + * "tuple" + + These classes accept a single positional argument, and the pattern + there is matched against the whole object rather than an attribute. + For example "int(0|1)" matches the value "0", but not the value + "0.0". + +In simple terms "CLS(P1, attr=P2)" matches only if the following +happens: + +* "isinstance(, CLS)" + +* convert "P1" to a keyword pattern using "CLS.__match_args__" + +* For each keyword argument "attr=P2": + + * "hasattr(, "attr")" + + * "P2" matches ".attr" + +* … and so on for the corresponding keyword argument/pattern pair. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Function definitions +==================== + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef: [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators: decorator+ + decorator: "@" assignment_expression NEWLINE + parameter_list: defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] + | parameter_star_kwargs + parameter_star_kwargs: "**" parameter [","] + parameter: identifier [":" expression] + star_parameter: identifier [":" ["*"] expression] + defparameter: parameter ["=" expression] + funcname: identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. + + +Class definitions +================= + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef: [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance: "(" [argument_list] ")" + classname: identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. + + +Coroutines +========== + +Added in version 3.5. + + +Coroutine function definition +----------------------------- + + async_funcdef: [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +------------------------- + + async_for_stmt: "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +-------------------------- + + async_with_stmt: "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. + + +Type parameter lists +==================== + +Added in version 3.12. + +Changed in version 3.13: Support for default values was added (see +**PEP 696**). + + type_params: "[" type_param ("," type_param)* "]" + type_param: typevar | typevartuple | paramspec + typevar: identifier (":" expression)? ("=" expression)? + typevartuple: "*" identifier ("=" expression)? + paramspec: "**" identifier ("=" expression)? + +Functions (including coroutines), classes and type aliases may contain +a type parameter list: + + def max[T](args: list[T]) -> T: + ... + + async def amax[T](args: list[T]) -> T: + ... + + class Bag[T]: + def __iter__(self) -> Iterator[T]: + ... + + def add(self, arg: T) -> None: + ... + + type ListOrSet[T] = list[T] | set[T] + +Semantically, this indicates that the function, class, or type alias +is generic over a type variable. This information is primarily used by +static type checkers, and at runtime, generic objects behave much like +their non-generic counterparts. + +Type parameters are declared in square brackets ("[]") immediately +after the name of the function, class, or type alias. The type +parameters are accessible within the scope of the generic object, but +not elsewhere. Thus, after a declaration "def func[T](): pass", the +name "T" is not available in the module scope. Below, the semantics of +generic objects are described with more precision. The scope of type +parameters is modeled with a special function (technically, an +annotation scope) that wraps the creation of the generic object. + +Generic functions, classes, and type aliases have a "__type_params__" +attribute listing their type parameters. + +Type parameters come in three kinds: + +* "typing.TypeVar", introduced by a plain name (e.g., "T"). + Semantically, this represents a single type to a type checker. + +* "typing.TypeVarTuple", introduced by a name prefixed with a single + asterisk (e.g., "*Ts"). Semantically, this stands for a tuple of any + number of types. + +* "typing.ParamSpec", introduced by a name prefixed with two asterisks + (e.g., "**P"). Semantically, this stands for the parameters of a + callable. + +"typing.TypeVar" declarations can define *bounds* and *constraints* +with a colon (":") followed by an expression. A single expression +after the colon indicates a bound (e.g. "T: int"). Semantically, this +means that the "typing.TypeVar" can only represent types that are a +subtype of this bound. A parenthesized tuple of expressions after the +colon indicates a set of constraints (e.g. "T: (str, bytes)"). Each +member of the tuple should be a type (again, this is not enforced at +runtime). Constrained type variables can only take on one of the types +in the list of constraints. + +For "typing.TypeVar"s declared using the type parameter list syntax, +the bound and constraints are not evaluated when the generic object is +created, but only when the value is explicitly accessed through the +attributes "__bound__" and "__constraints__". To accomplish this, the +bounds or constraints are evaluated in a separate annotation scope. + +"typing.TypeVarTuple"s and "typing.ParamSpec"s cannot have bounds or +constraints. + +All three flavors of type parameters can also have a *default value*, +which is used when the type parameter is not explicitly provided. This +is added by appending a single equals sign ("=") followed by an +expression. Like the bounds and constraints of type variables, the +default value is not evaluated when the object is created, but only +when the type parameter’s "__default__" attribute is accessed. To this +end, the default value is evaluated in a separate annotation scope. If +no default value is specified for a type parameter, the "__default__" +attribute is set to the special sentinel object "typing.NoDefault". + +The following example indicates the full set of allowed type parameter +declarations: + + def overly_generic[ + SimpleTypeVar, + TypeVarWithDefault = int, + TypeVarWithBound: int, + TypeVarWithConstraints: (str, bytes), + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), + ]( + a: SimpleTypeVar, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, + ): ... + + +Generic functions +----------------- + +Generic functions are declared as follows: + + def func[T](arg: T): ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_func(): + T = typing.TypeVar("T") + def func(arg: T): ... + func.__type_params__ = (T,) + return func + func = TYPE_PARAMS_OF_func() + +Here "annotation-def" indicates an annotation scope, which is not +actually bound to any name at runtime. (One other liberty is taken in +the translation: the syntax does not go through attribute access on +the "typing" module, but creates an instance of "typing.TypeVar" +directly.) + +The annotations of generic functions are evaluated within the +annotation scope used for declaring the type parameters, but the +function’s defaults and decorators are not. + +The following example illustrates the scoping rules for these cases, +as well as for additional flavors of type parameters: + + @decorator + def func[T: int, *Ts, **P](*args: *Ts, arg: Callable[P, T] = some_default): + ... + +Except for the lazy evaluation of the "TypeVar" bound, this is +equivalent to: + + DEFAULT_OF_arg = some_default + + annotation-def TYPE_PARAMS_OF_func(): + + annotation-def BOUND_OF_T(): + return int + # In reality, BOUND_OF_T() is evaluated only on demand. + T = typing.TypeVar("T", bound=BOUND_OF_T()) + + Ts = typing.TypeVarTuple("Ts") + P = typing.ParamSpec("P") + + def func(*args: *Ts, arg: Callable[P, T] = DEFAULT_OF_arg): + ... + + func.__type_params__ = (T, Ts, P) + return func + func = decorator(TYPE_PARAMS_OF_func()) + +The capitalized names like "DEFAULT_OF_arg" are not actually bound at +runtime. + + +Generic classes +--------------- + +Generic classes are declared as follows: + + class Bag[T]: ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(typing.Generic[T]): + __type_params__ = (T,) + ... + return Bag + Bag = TYPE_PARAMS_OF_Bag() + +Here again "annotation-def" (not a real keyword) indicates an +annotation scope, and the name "TYPE_PARAMS_OF_Bag" is not actually +bound at runtime. + +Generic classes implicitly inherit from "typing.Generic". The base +classes and keyword arguments of generic classes are evaluated within +the type scope for the type parameters, and decorators are evaluated +outside that scope. This is illustrated by this example: + + @decorator + class Bag(Base[T], arg=T): ... + +This is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(Base[T], typing.Generic[T], arg=T): + __type_params__ = (T,) + ... + return Bag + Bag = decorator(TYPE_PARAMS_OF_Bag()) + + +Generic type aliases +-------------------- + +The "type" statement can also be used to create a generic type alias: + + type ListOrSet[T] = list[T] | set[T] + +Except for the lazy evaluation of the value, this is equivalent to: + + annotation-def TYPE_PARAMS_OF_ListOrSet(): + T = typing.TypeVar("T") + + annotation-def VALUE_OF_ListOrSet(): + return list[T] | set[T] + # In reality, the value is lazily evaluated + return typing.TypeAliasType("ListOrSet", VALUE_OF_ListOrSet(), type_params=(T,)) + ListOrSet = TYPE_PARAMS_OF_ListOrSet() + +Here, "annotation-def" (not a real keyword) indicates an annotation +scope. The capitalized names like "TYPE_PARAMS_OF_ListOrSet" are not +actually bound at runtime. + + +Annotations +=========== + +Changed in version 3.14: Annotations are now lazily evaluated by +default. + +Variables and function parameters may carry *annotations*, created by +adding a colon after the name, followed by an expression: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow: + + def f() -> annotation: ... + +Annotations are conventionally used for *type hints*, but this is not +enforced by the language, and in general annotations may contain +arbitrary expressions. The presence of annotations does not change the +runtime semantics of the code, except if some mechanism is used that +introspects and uses the annotations (such as "dataclasses" or +"functools.singledispatch()"). + +By default, annotations are lazily evaluated in an annotation scope. +This means that they are not evaluated when the code containing the +annotation is evaluated. Instead, the interpreter saves information +that can be used to evaluate the annotation later if requested. The +"annotationlib" module provides tools for evaluating annotations. + +If the future statement "from __future__ import annotations" is +present, all annotations are instead stored as strings: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + +This future statement will be deprecated and removed in a future +version of Python, but not before Python 3.13 reaches its end of life +(see **PEP 749**). When it is used, introspection tools like +"annotationlib.get_annotations()" and "typing.get_type_hints()" are +less likely to be able to resolve annotations at runtime. + +-[ Footnotes ]- + +[1] The exception is propagated to the invocation stack unless there + is a "finally" clause which happens to raise another exception. + That new exception causes the old one to be lost. + +[2] In pattern matching, a sequence is defined as one of the + following: + + * a class that inherits from "collections.abc.Sequence" + + * a Python class that has been registered as + "collections.abc.Sequence" + + * a builtin class that has its (CPython) "Py_TPFLAGS_SEQUENCE" bit + set + + * a class that inherits from any of the above + + The following standard library classes are sequences: + + * "array.array" + + * "collections.deque" + + * "list" + + * "memoryview" + + * "range" + + * "tuple" + + Note: + + Subject values of type "str", "bytes", and "bytearray" do not + match sequence patterns. + +[3] In pattern matching, a mapping is defined as one of the following: + + * a class that inherits from "collections.abc.Mapping" + + * a Python class that has been registered as + "collections.abc.Mapping" + + * a builtin class that has its (CPython) "Py_TPFLAGS_MAPPING" bit + set + + * a class that inherits from any of the above + + The standard library classes "dict" and "types.MappingProxyType" + are mappings. + +[4] A string literal appearing as the first statement in the function + body is transformed into the function’s "__doc__" attribute and + therefore the function’s *docstring*. + +[5] A string literal appearing as the first statement in the class + body is transformed into the namespace’s "__doc__" item and + therefore the class’s *docstring*. +''', + 'context-managers': r'''With Statement Context Managers +******************************* + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'continue': r'''The "continue" statement +************************ + + continue_stmt: "continue" + +"continue" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. It continues with the next cycle of the nearest enclosing loop. + +When "continue" passes control out of a "try" statement with a +"finally" clause, that "finally" clause is executed before really +starting the next loop cycle. +''', + 'conversions': r'''Arithmetic conversions +********************** + +When a description of an arithmetic operator below uses the phrase +“the numeric arguments are converted to a common real type”, this +means that the operator implementation for built-in types works as +follows: + +* If both arguments are complex numbers, no conversion is performed; + +* if either argument is a complex or a floating-point number, the + other is converted to a floating-point number; + +* otherwise, both must be integers and no conversion is necessary. + +Some additional rules apply for certain operators (e.g., a string as a +left argument to the ‘%’ operator). Extensions must define their own +conversion behavior. +''', + 'customization': r'''Basic customization +******************* + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. +''', + 'debugger': r'''"pdb" — The Python Debugger +*************************** + +**Source code:** Lib/pdb.py + +====================================================================== + +The module "pdb" defines an interactive source code debugger for +Python programs. It supports setting (conditional) breakpoints and +single stepping at the source line level, inspection of stack frames, +source code listing, and evaluation of arbitrary Python code in the +context of any stack frame. It also supports post-mortem debugging +and can be called under program control. + +The debugger is extensible – it is actually defined as the class +"Pdb". This is currently undocumented but easily understood by reading +the source. The extension interface uses the modules "bdb" and "cmd". + +See also: + + Module "faulthandler" + Used to dump Python tracebacks explicitly, on a fault, after a + timeout, or on a user signal. + + Module "traceback" + Standard interface to extract, format and print stack traces of + Python programs. + +The typical usage to break into the debugger is to insert: + + import pdb; pdb.set_trace() + +Or: + + breakpoint() + +at the location you want to break into the debugger, and then run the +program. You can then step through the code following this statement, +and continue running without the debugger using the "continue" +command. + +Changed in version 3.7: The built-in "breakpoint()", when called with +defaults, can be used instead of "import pdb; pdb.set_trace()". + + def double(x): + breakpoint() + return x * 2 + val = 3 + print(f"{val} * 2 is {double(val)}") + +The debugger’s prompt is "(Pdb)", which is the indicator that you are +in debug mode: + + > ...(2)double() + -> breakpoint() + (Pdb) p x + 3 + (Pdb) continue + 3 * 2 is 6 + +Changed in version 3.3: Tab-completion via the "readline" module is +available for commands and command arguments, e.g. the current global +and local names are offered as arguments of the "p" command. + + +Command-line interface +====================== + +You can also invoke "pdb" from the command line to debug other +scripts. For example: + + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] + +When invoked as a module, pdb will automatically enter post-mortem +debugging if the program being debugged exits abnormally. After post- +mortem debugging (or after normal exit of the program), pdb will +restart the program. Automatic restarting preserves pdb’s state (such +as breakpoints) and in most cases is more useful than quitting the +debugger upon program’s exit. + +-c, --command + + To execute commands as if given in a ".pdbrc" file; see Debugger + commands. + + Changed in version 3.2: Added the "-c" option. + +-m + + To execute modules similar to the way "python -m" does. As with a + script, the debugger will pause execution just before the first + line of the module. + + Changed in version 3.7: Added the "-m" option. + +-p, --pid + + Attach to the process with the specified PID. + + Added in version 3.14. + +To attach to a running Python process for remote debugging, use the +"-p" or "--pid" option with the target process’s PID: + + python -m pdb -p 1234 + +Note: + + Attaching to a process that is blocked in a system call or waiting + for I/O will only work once the next bytecode instruction is + executed or when the process receives a signal. + +Typical usage to execute a statement under control of the debugger is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + >>> pdb.run("f(2)") + > (1)() + (Pdb) continue + 0.5 + >>> + +The typical usage to inspect a crashed program is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + ... + >>> f(0) + Traceback (most recent call last): + File "", line 1, in + File "", line 2, in f + ZeroDivisionError: division by zero + >>> pdb.pm() + > (2)f() + (Pdb) p x + 0 + (Pdb) + +Changed in version 3.13: The implementation of **PEP 667** means that +name assignments made via "pdb" will immediately affect the active +scope, even when running inside an *optimized scope*. + +The module defines the following functions; each enters the debugger +in a slightly different way: + +pdb.run(statement, globals=None, locals=None) + + Execute the *statement* (given as a string or a code object) under + debugger control. The debugger prompt appears before any code is + executed; you can set breakpoints and type "continue", or you can + step through the statement using "step" or "next" (all these + commands are explained below). The optional *globals* and *locals* + arguments specify the environment in which the code is executed; by + default the dictionary of the module "__main__" is used. (See the + explanation of the built-in "exec()" or "eval()" functions.) + +pdb.runeval(expression, globals=None, locals=None) + + Evaluate the *expression* (given as a string or a code object) + under debugger control. When "runeval()" returns, it returns the + value of the *expression*. Otherwise this function is similar to + "run()". + +pdb.runcall(function, *args, **kwds) + + Call the *function* (a function or method object, not a string) + with the given arguments. When "runcall()" returns, it returns + whatever the function call returned. The debugger prompt appears + as soon as the function is entered. + +pdb.set_trace(*, header=None, commands=None) + + Enter the debugger at the calling stack frame. This is useful to + hard-code a breakpoint at a given point in a program, even if the + code is not otherwise being debugged (e.g. when an assertion + fails). If given, *header* is printed to the console just before + debugging begins. The *commands* argument, if given, is a list of + commands to execute when the debugger starts. + + Changed in version 3.7: The keyword-only argument *header*. + + Changed in version 3.13: "set_trace()" will enter the debugger + immediately, rather than on the next line of code to be executed. + + Added in version 3.14: The *commands* argument. + +awaitable pdb.set_trace_async(*, header=None, commands=None) + + async version of "set_trace()". This function should be used inside + an async function with "await". + + async def f(): + await pdb.set_trace_async() + + "await" statements are supported if the debugger is invoked by this + function. + + Added in version 3.14. + +pdb.post_mortem(t=None) + + Enter post-mortem debugging of the given exception or traceback + object. If no value is given, it uses the exception that is + currently being handled, or raises "ValueError" if there isn’t one. + + Changed in version 3.13: Support for exception objects was added. + +pdb.pm() + + Enter post-mortem debugging of the exception found in + "sys.last_exc". + +pdb.set_default_backend(backend) + + There are two supported backends for pdb: "'settrace'" and + "'monitoring'". See "bdb.Bdb" for details. The user can set the + default backend to use if none is specified when instantiating + "Pdb". If no backend is specified, the default is "'settrace'". + + Note: + + "breakpoint()" and "set_trace()" will not be affected by this + function. They always use "'monitoring'" backend. + + Added in version 3.14. + +pdb.get_default_backend() + + Returns the default backend for pdb. + + Added in version 3.14. + +The "run*" functions and "set_trace()" are aliases for instantiating +the "Pdb" class and calling the method of the same name. If you want +to access further features, you have to do this yourself: + +class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True, mode=None, backend=None, colorize=False) + + "Pdb" is the debugger class. + + The *completekey*, *stdin* and *stdout* arguments are passed to the + underlying "cmd.Cmd" class; see the description there. + + The *skip* argument, if given, must be an iterable of glob-style + module name patterns. The debugger will not step into frames that + originate in a module that matches one of these patterns. [1] + + By default, Pdb sets a handler for the SIGINT signal (which is sent + when the user presses "Ctrl"-"C" on the console) when you give a + "continue" command. This allows you to break into the debugger + again by pressing "Ctrl"-"C". If you want Pdb not to touch the + SIGINT handler, set *nosigint* to true. + + The *readrc* argument defaults to true and controls whether Pdb + will load .pdbrc files from the filesystem. + + The *mode* argument specifies how the debugger was invoked. It + impacts the workings of some debugger commands. Valid values are + "'inline'" (used by the breakpoint() builtin), "'cli'" (used by the + command line invocation) or "None" (for backwards compatible + behaviour, as before the *mode* argument was added). + + The *backend* argument specifies the backend to use for the + debugger. If "None" is passed, the default backend will be used. + See "set_default_backend()". Otherwise the supported backends are + "'settrace'" and "'monitoring'". + + The *colorize* argument, if set to "True", will enable colorized + output in the debugger, if color is supported. This will highlight + source code displayed in pdb. + + Example call to enable tracing with *skip*: + + import pdb; pdb.Pdb(skip=['django.*']).set_trace() + + Raises an auditing event "pdb.Pdb" with no arguments. + + Changed in version 3.1: Added the *skip* parameter. + + Changed in version 3.2: Added the *nosigint* parameter. Previously, + a SIGINT handler was never set by Pdb. + + Changed in version 3.6: The *readrc* argument. + + Added in version 3.14: Added the *mode* argument. + + Added in version 3.14: Added the *backend* argument. + + Added in version 3.14: Added the *colorize* argument. + + Changed in version 3.14: Inline breakpoints like "breakpoint()" or + "pdb.set_trace()" will always stop the program at calling frame, + ignoring the *skip* pattern (if any). + + run(statement, globals=None, locals=None) + runeval(expression, globals=None, locals=None) + runcall(function, *args, **kwds) + set_trace() + + See the documentation for the functions explained above. + + +Debugger commands +================= + +The commands recognized by the debugger are listed below. Most +commands can be abbreviated to one or two letters as indicated; e.g. +"h(elp)" means that either "h" or "help" can be used to enter the help +command (but not "he" or "hel", nor "H" or "Help" or "HELP"). +Arguments to commands must be separated by whitespace (spaces or +tabs). Optional arguments are enclosed in square brackets ("[]") in +the command syntax; the square brackets must not be typed. +Alternatives in the command syntax are separated by a vertical bar +("|"). + +Entering a blank line repeats the last command entered. Exception: if +the last command was a "list" command, the next 11 lines are listed. + +Commands that the debugger doesn’t recognize are assumed to be Python +statements and are executed in the context of the program being +debugged. Python statements can also be prefixed with an exclamation +point ("!"). This is a powerful way to inspect the program being +debugged; it is even possible to change a variable or call a function. +When an exception occurs in such a statement, the exception name is +printed but the debugger’s state is not changed. + +Changed in version 3.13: Expressions/Statements whose prefix is a pdb +command are now correctly identified and executed. + +The debugger supports aliases. Aliases can have parameters which +allows one a certain level of adaptability to the context under +examination. + +Multiple commands may be entered on a single line, separated by ";;". +(A single ";" is not used as it is the separator for multiple commands +in a line that is passed to the Python parser.) No intelligence is +applied to separating the commands; the input is split at the first +";;" pair, even if it is in the middle of a quoted string. A +workaround for strings with double semicolons is to use implicit +string concatenation "';'';'" or "";"";"". + +To set a temporary global variable, use a *convenience variable*. A +*convenience variable* is a variable whose name starts with "$". For +example, "$foo = 1" sets a global variable "$foo" which you can use in +the debugger session. The *convenience variables* are cleared when +the program resumes execution so it’s less likely to interfere with +your program compared to using normal variables like "foo = 1". + +There are four preset *convenience variables*: + +* "$_frame": the current frame you are debugging + +* "$_retval": the return value if the frame is returning + +* "$_exception": the exception if the frame is raising an exception + +* "$_asynctask": the asyncio task if pdb stops in an async function + +Added in version 3.12: Added the *convenience variable* feature. + +Added in version 3.14: Added the "$_asynctask" convenience variable. + +If a file ".pdbrc" exists in the user’s home directory or in the +current directory, it is read with "'utf-8'" encoding and executed as +if it had been typed at the debugger prompt, with the exception that +empty lines and lines starting with "#" are ignored. This is +particularly useful for aliases. If both files exist, the one in the +home directory is read first and aliases defined there can be +overridden by the local file. + +Changed in version 3.2: ".pdbrc" can now contain commands that +continue debugging, such as "continue" or "next". Previously, these +commands had no effect. + +Changed in version 3.11: ".pdbrc" is now read with "'utf-8'" encoding. +Previously, it was read with the system locale encoding. + +h(elp) [command] + + Without argument, print the list of available commands. With a + *command* as argument, print help about that command. "help pdb" + displays the full documentation (the docstring of the "pdb" + module). Since the *command* argument must be an identifier, "help + exec" must be entered to get help on the "!" command. + +w(here) [count] + + Print a stack trace, with the most recent frame at the bottom. if + *count* is 0, print the current frame entry. If *count* is + negative, print the least recent - *count* frames. If *count* is + positive, print the most recent *count* frames. An arrow (">") + indicates the current frame, which determines the context of most + commands. + + Changed in version 3.14: *count* argument is added. + +d(own) [count] + + Move the current frame *count* (default one) levels down in the + stack trace (to a newer frame). + +u(p) [count] + + Move the current frame *count* (default one) levels up in the stack + trace (to an older frame). + +b(reak) [([filename:]lineno | function) [, condition]] + + With a *lineno* argument, set a break at line *lineno* in the + current file. The line number may be prefixed with a *filename* and + a colon, to specify a breakpoint in another file (possibly one that + hasn’t been loaded yet). The file is searched on "sys.path". + Acceptable forms of *filename* are "/abspath/to/file.py", + "relpath/file.py", "module" and "package.module". + + With a *function* argument, set a break at the first executable + statement within that function. *function* can be any expression + that evaluates to a function in the current namespace. + + If a second argument is present, it is an expression which must + evaluate to true before the breakpoint is honored. + + Without argument, list all breaks, including for each breakpoint, + the number of times that breakpoint has been hit, the current + ignore count, and the associated condition if any. + + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + +tbreak [([filename:]lineno | function) [, condition]] + + Temporary breakpoint, which is removed automatically when it is + first hit. The arguments are the same as for "break". + +cl(ear) [filename:lineno | bpnumber ...] + + With a *filename:lineno* argument, clear all the breakpoints at + this line. With a space separated list of breakpoint numbers, clear + those breakpoints. Without argument, clear all breaks (but first + ask confirmation). + +disable bpnumber [bpnumber ...] + + Disable the breakpoints given as a space separated list of + breakpoint numbers. Disabling a breakpoint means it cannot cause + the program to stop execution, but unlike clearing a breakpoint, it + remains in the list of breakpoints and can be (re-)enabled. + +enable bpnumber [bpnumber ...] + + Enable the breakpoints specified. + +ignore bpnumber [count] + + Set the ignore count for the given breakpoint number. If *count* + is omitted, the ignore count is set to 0. A breakpoint becomes + active when the ignore count is zero. When non-zero, the *count* + is decremented each time the breakpoint is reached and the + breakpoint is not disabled and any associated condition evaluates + to true. + +condition bpnumber [condition] + + Set a new *condition* for the breakpoint, an expression which must + evaluate to true before the breakpoint is honored. If *condition* + is absent, any existing condition is removed; i.e., the breakpoint + is made unconditional. + +commands [bpnumber] + + Specify a list of commands for breakpoint number *bpnumber*. The + commands themselves appear on the following lines. Type a line + containing just "end" to terminate the commands. An example: + + (Pdb) commands 1 + (com) p some_variable + (com) end + (Pdb) + + To remove all commands from a breakpoint, type "commands" and + follow it immediately with "end"; that is, give no commands. + + With no *bpnumber* argument, "commands" refers to the last + breakpoint set. + + You can use breakpoint commands to start your program up again. + Simply use the "continue" command, or "step", or any other command + that resumes execution. + + Specifying any command resuming execution (currently "continue", + "step", "next", "return", "until", "jump", "quit" and their + abbreviations) terminates the command list (as if that command was + immediately followed by end). This is because any time you resume + execution (even with a simple next or step), you may encounter + another breakpoint—which could have its own command list, leading + to ambiguities about which list to execute. + + If the list of commands contains the "silent" command, or a command + that resumes execution, then the breakpoint message containing + information about the frame is not displayed. + + Changed in version 3.14: Frame information will not be displayed if + a command that resumes execution is present in the command list. + +s(tep) + + Execute the current line, stop at the first possible occasion + (either in a function that is called or on the next line in the + current function). + +n(ext) + + Continue execution until the next line in the current function is + reached or it returns. (The difference between "next" and "step" + is that "step" stops inside a called function, while "next" + executes called functions at (nearly) full speed, only stopping at + the next line in the current function.) + +unt(il) [lineno] + + Without argument, continue execution until the line with a number + greater than the current one is reached. + + With *lineno*, continue execution until a line with a number + greater or equal to *lineno* is reached. In both cases, also stop + when the current frame returns. + + Changed in version 3.2: Allow giving an explicit line number. + +r(eturn) + + Continue execution until the current function returns. + +c(ont(inue)) + + Continue execution, only stop when a breakpoint is encountered. + +j(ump) lineno + + Set the next line that will be executed. Only available in the + bottom-most frame. This lets you jump back and execute code again, + or jump forward to skip code that you don’t want to run. + + It should be noted that not all jumps are allowed – for instance it + is not possible to jump into the middle of a "for" loop or out of a + "finally" clause. + +l(ist) [first[, last]] + + List source code for the current file. Without arguments, list 11 + lines around the current line or continue the previous listing. + With "." as argument, list 11 lines around the current line. With + one argument, list 11 lines around at that line. With two + arguments, list the given range; if the second argument is less + than the first, it is interpreted as a count. + + The current line in the current frame is indicated by "->". If an + exception is being debugged, the line where the exception was + originally raised or propagated is indicated by ">>", if it differs + from the current line. + + Changed in version 3.2: Added the ">>" marker. + +ll | longlist + + List all source code for the current function or frame. + Interesting lines are marked as for "list". + + Added in version 3.2. + +a(rgs) + + Print the arguments of the current function and their current + values. + +p expression + + Evaluate *expression* in the current context and print its value. + + Note: + + "print()" can also be used, but is not a debugger command — this + executes the Python "print()" function. + +pp expression + + Like the "p" command, except the value of *expression* is pretty- + printed using the "pprint" module. + +whatis expression + + Print the type of *expression*. + +source expression + + Try to get source code of *expression* and display it. + + Added in version 3.2. + +display [expression] + + Display the value of *expression* if it changed, each time + execution stops in the current frame. + + Without *expression*, list all display expressions for the current + frame. + + Note: + + Display evaluates *expression* and compares to the result of the + previous evaluation of *expression*, so when the result is + mutable, display may not be able to pick up the changes. + + Example: + + lst = [] + breakpoint() + pass + lst.append(1) + print(lst) + + Display won’t realize "lst" has been changed because the result of + evaluation is modified in place by "lst.append(1)" before being + compared: + + > example.py(3)() + -> pass + (Pdb) display lst + display lst: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + (Pdb) + + You can do some tricks with copy mechanism to make it work: + + > example.py(3)() + -> pass + (Pdb) display lst[:] + display lst[:]: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + display lst[:]: [1] [old: []] + (Pdb) + + Added in version 3.2. + +undisplay [expression] + + Do not display *expression* anymore in the current frame. Without + *expression*, clear all display expressions for the current frame. + + Added in version 3.2. + +interact + + Start an interactive interpreter (using the "code" module) in a new + global namespace initialised from the local and global namespaces + for the current scope. Use "exit()" or "quit()" to exit the + interpreter and return to the debugger. + + Note: + + As "interact" creates a new dedicated namespace for code + execution, assignments to variables will not affect the original + namespaces. However, modifications to any referenced mutable + objects will be reflected in the original namespaces as usual. + + Added in version 3.2. + + Changed in version 3.13: "exit()" and "quit()" can be used to exit + the "interact" command. + + Changed in version 3.13: "interact" directs its output to the + debugger’s output channel rather than "sys.stderr". + +alias [name [command]] + + Create an alias called *name* that executes *command*. The + *command* must *not* be enclosed in quotes. Replaceable parameters + can be indicated by "%1", "%2", … and "%9", while "%*" is replaced + by all the parameters. If *command* is omitted, the current alias + for *name* is shown. If no arguments are given, all aliases are + listed. + + Aliases may be nested and can contain anything that can be legally + typed at the pdb prompt. Note that internal pdb commands *can* be + overridden by aliases. Such a command is then hidden until the + alias is removed. Aliasing is recursively applied to the first + word of the command line; all other words in the line are left + alone. + + As an example, here are two useful aliases (especially when placed + in the ".pdbrc" file): + + # Print instance variables (usage "pi classInst") + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + # Print instance variables in self + alias ps pi self + +unalias name + + Delete the specified alias *name*. + +! statement + + Execute the (one-line) *statement* in the context of the current + stack frame. The exclamation point can be omitted unless the first + word of the statement resembles a debugger command, e.g.: + + (Pdb) ! n=42 + (Pdb) + + To set a global variable, you can prefix the assignment command + with a "global" statement on the same line, e.g.: + + (Pdb) global list_options; list_options = ['-l'] + (Pdb) + +run [args ...] +restart [args ...] + + Restart the debugged Python program. If *args* is supplied, it is + split with "shlex" and the result is used as the new "sys.argv". + History, breakpoints, actions and debugger options are preserved. + "restart" is an alias for "run". + + Changed in version 3.14: "run" and "restart" commands are disabled + when the debugger is invoked in "'inline'" mode. + +q(uit) + + Quit from the debugger. The program being executed is aborted. An + end-of-file input is equivalent to "quit". + + A confirmation prompt will be shown if the debugger is invoked in + "'inline'" mode. Either "y", "Y", "" or "EOF" will confirm + the quit. + + Changed in version 3.14: A confirmation prompt will be shown if the + debugger is invoked in "'inline'" mode. After the confirmation, the + debugger will call "sys.exit()" immediately, instead of raising + "bdb.BdbQuit" in the next trace event. + +debug code + + Enter a recursive debugger that steps through *code* (which is an + arbitrary expression or statement to be executed in the current + environment). + +retval + + Print the return value for the last return of the current function. + +exceptions [excnumber] + + List or jump between chained exceptions. + + When using "pdb.pm()" or "Pdb.post_mortem(...)" with a chained + exception instead of a traceback, it allows the user to move + between the chained exceptions using "exceptions" command to list + exceptions, and "exceptions " to switch to that exception. + + Example: + + def out(): + try: + middle() + except Exception as e: + raise ValueError("reraise middle() error") from e + + def middle(): + try: + return inner(0) + except Exception as e: + raise ValueError("Middle fail") + + def inner(x): + 1 / x + + out() + + calling "pdb.pm()" will allow to move between exceptions: + + > example.py(5)out() + -> raise ValueError("reraise middle() error") from e + + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + 1 ValueError('Middle fail') + > 2 ValueError('reraise middle() error') + + (Pdb) exceptions 0 + > example.py(16)inner() + -> 1 / x + + (Pdb) up + > example.py(10)middle() + -> return inner(0) + + Added in version 3.13. + +-[ Footnotes ]- + +[1] Whether a frame is considered to originate in a certain module is + determined by the "__name__" in the frame globals. +''', + 'del': r'''The "del" statement +******************* + + del_stmt: "del" target_list + +Deletion is recursively defined very similar to the way assignment is +defined. Rather than spelling it out in full details, here are some +hints. + +Deletion of a target list recursively deletes each target, from left +to right. + +Deletion of a name removes the binding of that name from the local or +global namespace, depending on whether the name occurs in a "global" +statement in the same code block. Trying to delete an unbound name +raises a "NameError" exception. + +Deletion of attribute references, subscriptions and slicings is passed +to the primary object involved; deletion of a slicing is in general +equivalent to assignment of an empty slice of the right type (but even +this is determined by the sliced object). + +Changed in version 3.2: Previously it was illegal to delete a name +from the local namespace if it occurs as a free variable in a nested +block. +''', + 'dict': r'''Dictionary displays +******************* + +A dictionary display is a possibly empty series of dict items +(key/value pairs) enclosed in curly braces: + + dict_display: "{" [dict_item_list | dict_comprehension] "}" + dict_item_list: dict_item ("," dict_item)* [","] + dict_item: expression ":" expression | "**" or_expr + dict_comprehension: expression ":" expression comp_for + +A dictionary display yields a new dictionary object. + +If a comma-separated sequence of dict items is given, they are +evaluated from left to right to define the entries of the dictionary: +each key object is used as a key into the dictionary to store the +corresponding value. This means that you can specify the same key +multiple times in the dict item list, and the final dictionary’s value +for that key will be the last one given. + +A double asterisk "**" denotes *dictionary unpacking*. Its operand +must be a *mapping*. Each mapping item is added to the new +dictionary. Later values replace values already set by earlier dict +items and earlier dictionary unpackings. + +Added in version 3.5: Unpacking into dictionary displays, originally +proposed by **PEP 448**. + +A dict comprehension, in contrast to list and set comprehensions, +needs two expressions separated with a colon followed by the usual +“for” and “if” clauses. When the comprehension is run, the resulting +key and value elements are inserted in the new dictionary in the order +they are produced. + +Restrictions on the types of the key values are listed earlier in +section The standard type hierarchy. (To summarize, the key type +should be *hashable*, which excludes all mutable objects.) Clashes +between duplicate keys are not detected; the last value (textually +rightmost in the display) stored for a given key value prevails. + +Changed in version 3.8: Prior to Python 3.8, in dict comprehensions, +the evaluation order of key and value was not well-defined. In +CPython, the value was evaluated before the key. Starting with 3.8, +the key is evaluated before the value, as proposed by **PEP 572**. +''', + 'dynamic-features': r'''Interaction with dynamic features +********************************* + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'else': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'exceptions': r'''Exceptions +********** + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. +''', + 'execmodel': r'''Execution model +*************** + + +Structure of a program +====================== + +A Python program is constructed from code blocks. A *block* is a piece +of Python program text that is executed as a unit. The following are +blocks: a module, a function body, and a class definition. Each +command typed interactively is a block. A script file (a file given +as standard input to the interpreter or specified as a command line +argument to the interpreter) is a code block. A script command (a +command specified on the interpreter command line with the "-c" +option) is a code block. A module run as a top level script (as module +"__main__") from the command line using a "-m" argument is also a code +block. The string argument passed to the built-in functions "eval()" +and "exec()" is a code block. + +A code block is executed in an *execution frame*. A frame contains +some administrative information (used for debugging) and determines +where and how execution continues after the code block’s execution has +completed. + + +Naming and binding +================== + + +Binding of names +---------------- + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +------------------- + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +----------------- + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +--------------- + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +--------------------------------- + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +--------------------------------- + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. + + +Exceptions +========== + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. + + +Runtime Components +================== + + +General Computing Model +----------------------- + +Python’s execution model does not operate in a vacuum. It runs on a +host machine and through that host’s runtime environment, including +its operating system (OS), if there is one. When a program runs, the +conceptual layers of how it runs on the host look something like this: + + **host machine** + **process** (global resources) + **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process’ +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same is +true for hosts.) The host manages the process’ access to its assigned +resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program’s machine +code, running relative to the resources assigned to the program’s +process. It’s strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process’ resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time (“concurrently”) or physically (“in +parallel”). Either way, the threads effectively run at a non- +synchronized rate. + +Note: + + That non-synchronized rate means none of the process’ memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other’s use of some + shared data. All this is true for both Python programs and the + Python runtime.The cost of this broad, unstructured requirement is + the tradeoff for the kind of raw concurrency that threads provide. + The alternative to the required discipline generally means dealing + with non-deterministic bugs and data corruption. + + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + **host machine** + **process** (global resources) + Python global runtime (*state*) + Python interpreter (*state*) + **thread** (runs Python bytecode and “C-API”) + Python thread *state* + +At the conceptual level: when a Python program starts, it looks +exactly like that diagram, with one of each. The runtime may grow to +include multiple interpreters, and each interpreter may grow to +include multiple thread states. + +Note: + + A Python implementation won’t necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the "threading" module. + +Note: + + The initial interpreter is typically called the “main” interpreter. + Some Python implementations, like CPython, assign special roles to + the main interpreter.Likewise, the host thread where the runtime was + initialized is known as the “main” thread. It may be different from + the process’ initial thread, though they are often the same. In + some cases “main thread” may be even more specific and refer to the + initial thread state. A Python runtime might assign specific + responsibilities to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an “interpreter” is conceptually what we would normally +think of as the (full-featured) “Python runtime”. When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +Note: + + The term “interpreter” here is not the same as the “bytecode + interpreter”, which is what regularly runs in threads, executing + compiled Python code.In an ideal world, “Python runtime” would refer + to what we currently call “interpreter”. However, it’s been called + “interpreter” at least since introduced in 1997 (CPython:a027efa5b). + +Each interpreter completely encapsulates all of the non-process- +global, non-thread-specific state needed for the Python runtime to +work. Notably, the interpreter’s state persists between uses. It +includes fundamental data like "sys.modules". The runtime ensures +multiple threads using the same interpreter will safely share it +between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own "sys.modules". + +For thread-specific runtime state, each interpreter has a set of +thread states, which it manages, in the same way the global runtime +contains a set of interpreters. It can have thread states for as many +host threads as it needs. It may even have multiple thread states for +the same host thread, though that isn’t as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread’s Python +call stack. It may include other thread-specific resources. + +Note: + + The term “Python thread” can sometimes refer to a thread state, but + normally it means a thread created using the "threading" module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether +for different interpreters or even the same interpreter. However, for +any given host thread, only one of the thread states tied to it can be +used by the thread at a time. + +Thread states are isolated and independent from one another and don’t +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +"threading" module (on platforms and Python implementations that +support threads). Additional processes can be created using the "os", +"subprocess", and "multiprocessing" modules. Interpreters can be +created and used with the "interpreters" module. Coroutines (async) +can be run using "asyncio" in each interpreter, typically only in a +single thread (often the main thread). + +-[ Footnotes ]- + +[1] This limitation occurs because the code that is executed by these + operations is not available at the time the module is compiled. +''', + 'exprlists': r'''Expression lists +**************** + + starred_expression: "*" or_expr | expression + flexible_expression: assignment_expression | starred_expression + flexible_expression_list: flexible_expression ("," flexible_expression)* [","] + starred_expression_list: starred_expression ("," starred_expression)* [","] + expression_list: expression ("," expression)* [","] + yield_list: expression_list | starred_expression "," [starred_expression_list] + +Except when part of a list or set display, an expression list +containing at least one comma yields a tuple. The length of the tuple +is the number of expressions in the list. The expressions are +evaluated from left to right. + +An asterisk "*" denotes *iterable unpacking*. Its operand must be an +*iterable*. The iterable is expanded into a sequence of items, which +are included in the new tuple, list, or set, at the site of the +unpacking. + +Added in version 3.5: Iterable unpacking in expression lists, +originally proposed by **PEP 448**. + +Added in version 3.11: Any item in an expression list may be starred. +See **PEP 646**. + +A trailing comma is required only to create a one-item tuple, such as +"1,"; it is optional in all other cases. A single expression without a +trailing comma doesn’t create a tuple, but rather yields the value of +that expression. (To create an empty tuple, use an empty pair of +parentheses: "()".) +''', + 'floating': r'''Floating-point literals +*********************** + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'for': r'''The "for" statement +******************* + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt: "for" target_list "in" starred_expression_list ":" suite + ["else" ":" suite] + +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. +''', + 'formatstrings': r'''Format String Syntax +******************** + +The "str.format()" method and the "Formatter" class share the same +syntax for format strings (although in the case of "Formatter", +subclasses can define their own format string syntax). The syntax is +related to that of formatted string literals and template string +literals, but it is less sophisticated and, in particular, does not +support arbitrary expressions in interpolations. + +Format strings contain “replacement fields” surrounded by curly braces +"{}". Anything that is not contained in braces is considered literal +text, which is copied unchanged to the output. If you need to include +a brace character in the literal text, it can be escaped by doubling: +"{{" and "}}". + +The grammar for a replacement field is as follows: + + replacement_field: "{" [field_name] ["!" conversion] [":" format_spec] "}" + field_name: arg_name ("." attribute_name | "[" element_index "]")* + arg_name: [identifier | digit+] + attribute_name: identifier + element_index: digit+ | index_string + index_string: + + conversion: "r" | "s" | "a" + format_spec: format-spec:format_spec + +In less formal terms, the replacement field can start with a +*field_name* that specifies the object whose value is to be formatted +and inserted into the output instead of the replacement field. The +*field_name* is optionally followed by a *conversion* field, which is +preceded by an exclamation point "'!'", and a *format_spec*, which is +preceded by a colon "':'". These specify a non-default format for the +replacement value. + +See also the Format Specification Mini-Language section. + +The *field_name* itself begins with an *arg_name* that is either a +number or a keyword. If it’s a number, it refers to a positional +argument, and if it’s a keyword, it refers to a named keyword +argument. An *arg_name* is treated as a number if a call to +"str.isdecimal()" on the string would return true. If the numerical +arg_names in a format string are 0, 1, 2, … in sequence, they can all +be omitted (not just some) and the numbers 0, 1, 2, … will be +automatically inserted in that order. Because *arg_name* is not quote- +delimited, it is not possible to specify arbitrary dictionary keys +(e.g., the strings "'10'" or "':-]'") within a format string. The +*arg_name* can be followed by any number of index or attribute +expressions. An expression of the form "'.name'" selects the named +attribute using "getattr()", while an expression of the form +"'[index]'" does an index lookup using "__getitem__()". + +Changed in version 3.1: The positional argument specifiers can be +omitted for "str.format()", so "'{} {}'.format(a, b)" is equivalent to +"'{0} {1}'.format(a, b)". + +Changed in version 3.4: The positional argument specifiers can be +omitted for "Formatter". + +Some simple format string examples: + + "First, thou shalt count to {0}" # References first positional argument + "Bring me a {}" # Implicitly references the first positional argument + "From {} to {}" # Same as "From {0} to {1}" + "My quest is {name}" # References keyword argument 'name' + "Weight in tons {0.weight}" # 'weight' attribute of first positional arg + "Units destroyed: {players[0]}" # First element of keyword argument 'players'. + +The *conversion* field causes a type coercion before formatting. +Normally, the job of formatting a value is done by the "__format__()" +method of the value itself. However, in some cases it is desirable to +force a type to be formatted as a string, overriding its own +definition of formatting. By converting the value to a string before +calling "__format__()", the normal formatting logic is bypassed. + +Three conversion flags are currently supported: "'!s'" which calls +"str()" on the value, "'!r'" which calls "repr()" and "'!a'" which +calls "ascii()". + +Some examples: + + "Harold's a clever {0!s}" # Calls str() on the argument first + "Bring out the holy {name!r}" # Calls repr() on the argument first + "More {!a}" # Calls ascii() on the argument first + +The *format_spec* field contains a specification of how the value +should be presented, including such details as field width, alignment, +padding, decimal precision and so on. Each value type can define its +own “formatting mini-language” or interpretation of the *format_spec*. + +Most built-in types support a common formatting mini-language, which +is described in the next section. + +A *format_spec* field can also include nested replacement fields +within it. These nested replacement fields may contain a field name, +conversion flag and format specification, but deeper nesting is not +allowed. The replacement fields within the format_spec are +substituted before the *format_spec* string is interpreted. This +allows the formatting of a value to be dynamically specified. + +See the Format examples section for some examples. + + +Format Specification Mini-Language +================================== + +“Format specifications” are used within replacement fields contained +within a format string to define how individual values are presented +(see Format String Syntax, f-strings, and t-strings). They can also be +passed directly to the built-in "format()" function. Each formattable +type may define how the format specification is to be interpreted. + +Most built-in types implement the following options for format +specifications, although some of the formatting options are only +supported by the numeric types. + +A general convention is that an empty format specification produces +the same result as if you had called "str()" on the value. A non-empty +format specification typically modifies the result. + +The general form of a *standard format specifier* is: + + format_spec: [options][width_and_precision][type] + options: [[fill]align][sign]["z"]["#"]["0"] + fill: + align: "<" | ">" | "=" | "^" + sign: "+" | "-" | " " + width_and_precision: [width_with_grouping][precision_with_grouping] + width_with_grouping: [width][grouping] + precision_with_grouping: "." [precision][grouping] | "." grouping + width: digit+ + precision: digit+ + grouping: "," | "_" + type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" + | "G" | "n" | "o" | "s" | "x" | "X" | "%" + +If a valid *align* value is specified, it can be preceded by a *fill* +character that can be any character and defaults to a space if +omitted. It is not possible to use a literal curly brace (”"{"” or +“"}"”) as the *fill* character in a formatted string literal or when +using the "str.format()" method. However, it is possible to insert a +curly brace with a nested replacement field. This limitation doesn’t +affect the "format()" function. + +The meaning of the various alignment options is as follows: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'<'" | Forces the field to be left-aligned within the available | +| | space (this is the default for most objects). | ++-----------+------------------------------------------------------------+ +| "'>'" | Forces the field to be right-aligned within the available | +| | space (this is the default for numbers). | ++-----------+------------------------------------------------------------+ +| "'='" | Forces the padding to be placed after the sign (if any) | +| | but before the digits. This is used for printing fields | +| | in the form ‘+000000120’. This alignment option is only | +| | valid for numeric types, excluding "complex". It becomes | +| | the default for numbers when ‘0’ immediately precedes the | +| | field width. | ++-----------+------------------------------------------------------------+ +| "'^'" | Forces the field to be centered within the available | +| | space. | ++-----------+------------------------------------------------------------+ + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the +alignment option has no meaning in this case. + +The *sign* option is only valid for number types, and can be one of +the following: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'+'" | Indicates that a sign should be used for both positive as | +| | well as negative numbers. | ++-----------+------------------------------------------------------------+ +| "'-'" | Indicates that a sign should be used only for negative | +| | numbers (this is the default behavior). | ++-----------+------------------------------------------------------------+ +| space | Indicates that a leading space should be used on positive | +| | numbers, and a minus sign on negative numbers. | ++-----------+------------------------------------------------------------+ + +The "'z'" option coerces negative zero floating-point values to +positive zero after rounding to the format precision. This option is +only valid for floating-point presentation types. + +Changed in version 3.11: Added the "'z'" option (see also **PEP +682**). + +The "'#'" option causes the “alternate form” to be used for the +conversion. The alternate form is defined differently for different +types. This option is only valid for integer, float and complex +types. For integers, when binary, octal, or hexadecimal output is +used, this option adds the respective prefix "'0b'", "'0o'", "'0x'", +or "'0X'" to the output value. For float and complex the alternate +form causes the result of the conversion to always contain a decimal- +point character, even if no digits follow it. Normally, a decimal- +point character appears in the result of these conversions only if a +digit follows it. In addition, for "'g'" and "'G'" conversions, +trailing zeros are not removed from the result. + +The *width* is a decimal integer defining the minimum total field +width, including any prefixes, separators, and other formatting +characters. If not specified, then the field width will be determined +by the content. + +When no explicit alignment is given, preceding the *width* field by a +zero ("'0'") character enables sign-aware zero-padding for numeric +types, excluding "complex". This is equivalent to a *fill* character +of "'0'" with an *alignment* type of "'='". + +Changed in version 3.10: Preceding the *width* field by "'0'" no +longer affects the default alignment for strings. + +The *precision* is a decimal integer indicating how many digits should +be displayed after the decimal point for presentation types "'f'" and +"'F'", or before and after the decimal point for presentation types +"'g'" or "'G'". For string presentation types the field indicates the +maximum field size - in other words, how many characters will be used +from the field content. The *precision* is not allowed for integer +presentation types. + +The *grouping* option after *width* and *precision* fields specifies a +digit group separator for the integral and fractional parts of a +number respectively. It can be one of the following: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "','" | Inserts a comma every 3 digits for integer presentation | +| | type "'d'" and floating-point presentation types, | +| | excluding "'n'". For other presentation types, this option | +| | is not supported. | ++-----------+------------------------------------------------------------+ +| "'_'" | Inserts an underscore every 3 digits for integer | +| | presentation type "'d'" and floating-point presentation | +| | types, excluding "'n'". For integer presentation types | +| | "'b'", "'o'", "'x'", and "'X'", underscores are inserted | +| | every 4 digits. For other presentation types, this option | +| | is not supported. | ++-----------+------------------------------------------------------------+ + +For a locale aware separator, use the "'n'" presentation type instead. + +Changed in version 3.1: Added the "','" option (see also **PEP 378**). + +Changed in version 3.6: Added the "'_'" option (see also **PEP 515**). + +Changed in version 3.14: Support the *grouping* option for the +fractional part. + +Finally, the *type* determines how the data should be presented. + +The available string presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'s'" | String format. This is the default type for strings and | + | | may be omitted. | + +-----------+------------------------------------------------------------+ + | None | The same as "'s'". | + +-----------+------------------------------------------------------------+ + +The available integer presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'b'" | Binary format. Outputs the number in base 2. | + +-----------+------------------------------------------------------------+ + | "'c'" | Character. Converts the integer to the corresponding | + | | unicode character before printing. | + +-----------+------------------------------------------------------------+ + | "'d'" | Decimal Integer. Outputs the number in base 10. | + +-----------+------------------------------------------------------------+ + | "'o'" | Octal format. Outputs the number in base 8. | + +-----------+------------------------------------------------------------+ + | "'x'" | Hex format. Outputs the number in base 16, using lower- | + | | case letters for the digits above 9. | + +-----------+------------------------------------------------------------+ + | "'X'" | Hex format. Outputs the number in base 16, using upper- | + | | case letters for the digits above 9. In case "'#'" is | + | | specified, the prefix "'0x'" will be upper-cased to "'0X'" | + | | as well. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'d'", except that it uses the | + | | current locale setting to insert the appropriate digit | + | | group separators. | + +-----------+------------------------------------------------------------+ + | None | The same as "'d'". | + +-----------+------------------------------------------------------------+ + +In addition to the above presentation types, integers can be formatted +with the floating-point presentation types listed below (except "'n'" +and "None"). When doing so, "float()" is used to convert the integer +to a floating-point number before formatting. + +The available presentation types for "float" and "Decimal" values are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'e'" | Scientific notation. For a given precision "p", formats | + | | the number in scientific notation with the letter ‘e’ | + | | separating the coefficient from the exponent. The | + | | coefficient has one digit before and "p" digits after the | + | | decimal point, for a total of "p + 1" significant digits. | + | | With no precision given, uses a precision of "6" digits | + | | after the decimal point for "float", and shows all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. For | + | | "float", the exponent always contains at least two digits, | + | | and is zero if the value is zero. | + +-----------+------------------------------------------------------------+ + | "'E'" | Scientific notation. Same as "'e'" except it uses an upper | + | | case ‘E’ as the separator character. | + +-----------+------------------------------------------------------------+ + | "'f'" | Fixed-point notation. For a given precision "p", formats | + | | the number as a decimal number with exactly "p" digits | + | | following the decimal point. With no precision given, uses | + | | a precision of "6" digits after the decimal point for | + | | "float", and uses a precision large enough to show all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. | + +-----------+------------------------------------------------------------+ + | "'F'" | Fixed-point notation. Same as "'f'", but converts "nan" to | + | | "NAN" and "inf" to "INF". | + +-----------+------------------------------------------------------------+ + | "'g'" | General format. For a given precision "p >= 1", this | + | | rounds the number to "p" significant digits and then | + | | formats the result in either fixed-point format or in | + | | scientific notation, depending on its magnitude. A | + | | precision of "0" is treated as equivalent to a precision | + | | of "1". The precise rules are as follows: suppose that | + | | the result formatted with presentation type "'e'" and | + | | precision "p-1" would have exponent "exp". Then, if "m <= | + | | exp < p", where "m" is -4 for floats and -6 for | + | | "Decimals", the number is formatted with presentation type | + | | "'f'" and precision "p-1-exp". Otherwise, the number is | + | | formatted with presentation type "'e'" and precision | + | | "p-1". In both cases insignificant trailing zeros are | + | | removed from the significand, and the decimal point is | + | | also removed if there are no remaining digits following | + | | it, unless the "'#'" option is used. With no precision | + | | given, uses a precision of "6" significant digits for | + | | "float". For "Decimal", the coefficient of the result is | + | | formed from the coefficient digits of the value; | + | | scientific notation is used for values smaller than "1e-6" | + | | in absolute value and values where the place value of the | + | | least significant digit is larger than 1, and fixed-point | + | | notation is used otherwise. Positive and negative | + | | infinity, positive and negative zero, and nans, are | + | | formatted as "inf", "-inf", "0", "-0" and "nan" | + | | respectively, regardless of the precision. | + +-----------+------------------------------------------------------------+ + | "'G'" | General format. Same as "'g'" except switches to "'E'" if | + | | the number gets too large. The representations of infinity | + | | and NaN are uppercased, too. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'g'", except that it uses the | + | | current locale setting to insert the appropriate digit | + | | group separators for the integral part of a number. | + +-----------+------------------------------------------------------------+ + | "'%'" | Percentage. Multiplies the number by 100 and displays in | + | | fixed ("'f'") format, followed by a percent sign. | + +-----------+------------------------------------------------------------+ + | None | For "float" this is like the "'g'" type, except that when | + | | fixed- point notation is used to format the result, it | + | | always includes at least one digit past the decimal point, | + | | and switches to the scientific notation when "exp >= p - | + | | 1". When the precision is not specified, the latter will | + | | be as large as needed to represent the given value | + | | faithfully. For "Decimal", this is the same as either | + | | "'g'" or "'G'" depending on the value of | + | | "context.capitals" for the current decimal context. The | + | | overall effect is to match the output of "str()" as | + | | altered by the other format modifiers. | + +-----------+------------------------------------------------------------+ + +The result should be correctly rounded to a given precision "p" of +digits after the decimal point. The rounding mode for "float" matches +that of the "round()" builtin. For "Decimal", the rounding mode of +the current context will be used. + +The available presentation types for "complex" are the same as those +for "float" ("'%'" is not allowed). Both the real and imaginary +components of a complex number are formatted as floating-point +numbers, according to the specified presentation type. They are +separated by the mandatory sign of the imaginary part, the latter +being terminated by a "j" suffix. If the presentation type is +missing, the result will match the output of "str()" (complex numbers +with a non-zero real part are also surrounded by parentheses), +possibly altered by other format modifiers. + + +Format examples +=============== + +This section contains examples of the "str.format()" syntax and +comparison with the old "%"-formatting. + +In most of the cases the syntax is similar to the old "%"-formatting, +with the addition of the "{}" and with ":" used instead of "%". For +example, "'%03.2f'" can be translated to "'{:03.2f}'". + +The new format syntax also supports new and different options, shown +in the following examples. + +Accessing arguments by position: + + >>> '{0}, {1}, {2}'.format('a', 'b', 'c') + 'a, b, c' + >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only + 'a, b, c' + >>> '{2}, {1}, {0}'.format('a', 'b', 'c') + 'c, b, a' + >>> '{2}, {1}, {0}'.format(*'abc') # unpacking argument sequence + 'c, b, a' + >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' indices can be repeated + 'abracadabra' + +Accessing arguments by name: + + >>> 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') + 'Coordinates: 37.24N, -115.81W' + >>> coord = {'latitude': '37.24N', 'longitude': '-115.81W'} + >>> 'Coordinates: {latitude}, {longitude}'.format(**coord) + 'Coordinates: 37.24N, -115.81W' + +Accessing arguments’ attributes: + + >>> c = 3-5j + >>> ('The complex number {0} is formed from the real part {0.real} ' + ... 'and the imaginary part {0.imag}.').format(c) + 'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.' + >>> class Point: + ... def __init__(self, x, y): + ... self.x, self.y = x, y + ... def __str__(self): + ... return 'Point({self.x}, {self.y})'.format(self=self) + ... + >>> str(Point(4, 2)) + 'Point(4, 2)' + +Accessing arguments’ items: + + >>> coord = (3, 5) + >>> 'X: {0[0]}; Y: {0[1]}'.format(coord) + 'X: 3; Y: 5' + +Replacing "%s" and "%r": + + >>> "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') + "repr() shows quotes: 'test1'; str() doesn't: test2" + +Aligning the text and specifying a width: + + >>> '{:<30}'.format('left aligned') + 'left aligned ' + >>> '{:>30}'.format('right aligned') + ' right aligned' + >>> '{:^30}'.format('centered') + ' centered ' + >>> '{:*^30}'.format('centered') # use '*' as a fill char + '***********centered***********' + +Replacing "%+f", "%-f", and "% f" and specifying a sign: + + >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it always + '+3.140000; -3.140000' + >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space for positive numbers + ' 3.140000; -3.140000' + >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the minus -- same as '{:f}; {:f}' + '3.140000; -3.140000' + +Replacing "%x" and "%o" and converting the value to different bases: + + >>> # format also supports binary numbers + >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42) + 'int: 42; hex: 2a; oct: 52; bin: 101010' + >>> # with 0x, 0o, or 0b as prefix: + >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42) + 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' + +Using the comma or the underscore as a digit group separator: + + >>> '{:,}'.format(1234567890) + '1,234,567,890' + >>> '{:_}'.format(1234567890) + '1_234_567_890' + >>> '{:_b}'.format(1234567890) + '100_1001_1001_0110_0000_0010_1101_0010' + >>> '{:_x}'.format(1234567890) + '4996_02d2' + >>> '{:_}'.format(123456789.123456789) + '123_456_789.12345679' + >>> '{:.,}'.format(123456789.123456789) + '123456789.123,456,79' + >>> '{:,._}'.format(123456789.123456789) + '123,456,789.123_456_79' + +Expressing a percentage: + + >>> points = 19 + >>> total = 22 + >>> 'Correct answers: {:.2%}'.format(points/total) + 'Correct answers: 86.36%' + +Using type-specific formatting: + + >>> import datetime + >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) + '2010-07-04 12:15:58' + +Nesting arguments and more complex examples: + + >>> for align, text in zip('<^>', ['left', 'center', 'right']): + ... '{0:{fill}{align}16}'.format(text, fill=align, align=align) + ... + 'left<<<<<<<<<<<<' + '^^^^^center^^^^^' + '>>>>>>>>>>>right' + >>> + >>> octets = [192, 168, 0, 1] + >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets) + 'C0A80001' + >>> int(_, 16) + 3232235521 + >>> + >>> width = 5 + >>> for num in range(5,12): + ... for base in 'dXob': + ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') + ... print() + ... + 5 5 5 101 + 6 6 6 110 + 7 7 7 111 + 8 8 10 1000 + 9 9 11 1001 + 10 A 12 1010 + 11 B 13 1011 +''', + 'function': r'''Function definitions +******************** + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef: [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators: decorator+ + decorator: "@" assignment_expression NEWLINE + parameter_list: defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] + | parameter_star_kwargs + parameter_star_kwargs: "**" parameter [","] + parameter: identifier [":" expression] + star_parameter: identifier [":" ["*"] expression] + defparameter: parameter ["=" expression] + funcname: identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. +''', + 'global': r'''The "global" statement +********************** + + global_stmt: "global" identifier ("," identifier)* + +The "global" statement causes the listed identifiers to be interpreted +as globals. It would be impossible to assign to a global variable +without "global", although free variables may refer to globals without +being declared global. + +The "global" statement applies to the entire current scope (module, +function body or class definition). A "SyntaxError" is raised if a +variable is used or assigned to prior to its global declaration in the +scope. + +At the module level, all variables are global, so a "global" statement +has no effect. However, variables must still not be used or assigned +to prior to their "global" declaration. This requirement is relaxed in +the interactive prompt (*REPL*). + +**Programmer’s note:** "global" is a directive to the parser. It +applies only to code parsed at the same time as the "global" +statement. In particular, a "global" statement contained in a string +or code object supplied to the built-in "exec()" function does not +affect the code block *containing* the function call, and code +contained in such a string is unaffected by "global" statements in the +code containing the function call. The same applies to the "eval()" +and "compile()" functions. +''', + 'id-classes': r'''Reserved classes of identifiers +******************************* + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). +''', + 'identifiers': r'''Names (identifiers and keywords) +******************************** + +"NAME" tokens represent *identifiers*, *keywords*, and *soft +keywords*. + +Names are composed of the following characters: + +* uppercase and lowercase letters ("A-Z" and "a-z"), + +* the underscore ("_"), + +* digits ("0" through "9"), which cannot appear as the first + character, and + +* non-ASCII characters. Valid names may only contain “letter-like” and + “digit-like” characters; see Non-ASCII characters in names for + details. + +Names must contain at least one character, but have no upper length +limit. Case is significant. + +Formally, names are described by the following lexical definitions: + + NAME: name_start name_continue* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: + +Note that not all names matched by this grammar are valid; see Non- +ASCII characters in names for details. + + +Keywords +======== + +The following names are used as reserved words, or *keywords* of the +language, and cannot be used as ordinary identifiers. They must be +spelled exactly as written here: + + False await else import pass + None break except in raise + True class finally is return + and continue for lambda try + as def from nonlocal while + assert del global not with + async elif if or yield + + +Soft Keywords +============= + +Added in version 3.10. + +Some names are only reserved under specific contexts. These are known +as *soft keywords*: + +* "match", "case", and "_", when used in the "match" statement. + +* "type", when used in the "type" statement. + +These syntactically act as keywords in their specific contexts, but +this distinction is done at the parser level, not when tokenizing. + +As soft keywords, their use in the grammar is possible while still +preserving compatibility with existing code that uses these names as +identifier names. + +Changed in version 3.12: "type" is now a soft keyword. + + +Reserved classes of identifiers +=============================== + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). + + +Non-ASCII characters in names +============================= + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained above. For +example, "ř_1", "蛇", or "साँप" are valid names, but "r〰2", "€", or +"🐍" are not. + +This section explains the exact rules. + +All names are converted into the normalization form NFKC while +parsing. This means that, for example, some typographic variants of +characters are converted to their “basic” form. For example, +"fiⁿₐˡᵢᶻₐᵗᵢᵒₙ" normalizes to "finalization", so Python treats them as +the same name: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +Note: + + Normalization is done at the lexical level only. Run-time functions + that take names as *strings* generally do not normalize their + arguments. For example, the variable defined above is accessible at + run time in the "globals()" dictionary as + "globals()["finalization"]" but not "globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]". + +Similarly to how ASCII-only names must contain only letters, digits +and the underscore, and cannot start with a digit, a valid name must +start with a character in the “letter-like” set "xid_start", and the +remaining characters must be in the “letter- and digit-like” set +"xid_continue". + +These sets based on the *XID_Start* and *XID_Continue* sets as defined +by the Unicode standard annex UAX-31. Python’s "xid_start" +additionally includes the underscore ("_"). Note that Python does not +necessarily conform to UAX-31. + +A non-normative listing of characters in the *XID_Start* and +*XID_Continue* sets as defined by Unicode is available in the +DerivedCoreProperties.txt file in the Unicode Character Database. For +reference, the construction rules for the "xid_*" sets are given +below. + +The set "id_start" is defined as the union of: + +* Unicode category "" - uppercase letters (includes "A" to "Z") + +* Unicode category "" - lowercase letters (includes "a" to "z") + +* Unicode category "" - titlecase letters + +* Unicode category "" - modifier letters + +* Unicode category "" - other letters + +* Unicode category "" - letter numbers + +* {""_""} - the underscore + +* "" - an explicit set of characters in PropList.txt + to support backwards compatibility + +The set "xid_start" then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +"id_start id_continue*". + +The set "id_continue" is defined as the union of: + +* "id_start" (see above) + +* Unicode category "" - decimal numbers (includes "0" to "9") + +* Unicode category "" - connector punctuations + +* Unicode category "" - nonspacing marks + +* Unicode category "" - spacing combining marks + +* "" - another explicit set of characters in + PropList.txt to support backwards compatibility + +Again, "xid_continue" closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database +as included in the "unicodedata" module. + +See also: + + * **PEP 3131** – Supporting Non-ASCII Identifiers + + * **PEP 672** – Unicode-related Security Considerations for Python +''', + 'if': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'imaginary': r'''Imaginary literals +****************** + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") +''', + 'import': r'''The "import" statement +********************** + + import_stmt: "import" module ["as" identifier] ("," module ["as" identifier])* + | "from" relative_module "import" identifier ["as" identifier] + ("," identifier ["as" identifier])* + | "from" relative_module "import" "(" identifier ["as" identifier] + ("," identifier ["as" identifier])* [","] ")" + | "from" relative_module "import" "*" + module: (identifier ".")* identifier + relative_module: "."* module | "."+ + +The basic import statement (no "from" clause) is executed in two +steps: + +1. find a module, loading and initializing it if necessary + +2. define a name or names in the local namespace for the scope where + the "import" statement occurs. + +When the statement contains multiple clauses (separated by commas) the +two steps are carried out separately for each clause, just as though +the clauses had been separated out into individual import statements. + +The details of the first step, finding and loading modules, are +described in greater detail in the section on the import system, which +also describes the various types of packages and modules that can be +imported, as well as all the hooks that can be used to customize the +import system. Note that failures in this step may indicate either +that the module could not be located, *or* that an error occurred +while initializing the module, which includes execution of the +module’s code. + +If the requested module is retrieved successfully, it will be made +available in the local namespace in one of three ways: + +* If the module name is followed by "as", then the name following "as" + is bound directly to the imported module. + +* If no other name is specified, and the module being imported is a + top level module, the module’s name is bound in the local namespace + as a reference to the imported module + +* If the module being imported is *not* a top level module, then the + name of the top level package that contains the module is bound in + the local namespace as a reference to the top level package. The + imported module must be accessed using its full qualified name + rather than directly + +The "from" form uses a slightly more complex process: + +1. find the module specified in the "from" clause, loading and + initializing it if necessary; + +2. for each of the identifiers specified in the "import" clauses: + + 1. check if the imported module has an attribute by that name + + 2. if not, attempt to import a submodule with that name and then + check the imported module again for that attribute + + 3. if the attribute is not found, "ImportError" is raised. + + 4. otherwise, a reference to that value is stored in the local + namespace, using the name in the "as" clause if it is present, + otherwise using the attribute name + +Examples: + + import foo # foo imported and bound locally + import foo.bar.baz # foo, foo.bar, and foo.bar.baz imported, foo bound locally + import foo.bar.baz as fbb # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as fbb + from foo.bar import baz # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as baz + from foo import attr # foo imported and foo.attr bound as attr + +If the list of identifiers is replaced by a star ("'*'"), all public +names defined in the module are bound in the local namespace for the +scope where the "import" statement occurs. + +The *public names* defined by a module are determined by checking the +module’s namespace for a variable named "__all__"; if defined, it must +be a sequence of strings which are names defined or imported by that +module. The names given in "__all__" are all considered public and +are required to exist. If "__all__" is not defined, the set of public +names includes all names found in the module’s namespace which do not +begin with an underscore character ("'_'"). "__all__" should contain +the entire public API. It is intended to avoid accidentally exporting +items that are not part of the API (such as library modules which were +imported and used within the module). + +The wild card form of import — "from module import *" — is only +allowed at the module level. Attempting to use it in class or +function definitions will raise a "SyntaxError". + +When specifying what module to import you do not have to specify the +absolute name of the module. When a module or package is contained +within another package it is possible to make a relative import within +the same top package without having to mention the package name. By +using leading dots in the specified module or package after "from" you +can specify how high to traverse up the current package hierarchy +without specifying exact names. One leading dot means the current +package where the module making the import exists. Two dots means up +one package level. Three dots is up two levels, etc. So if you execute +"from . import mod" from a module in the "pkg" package then you will +end up importing "pkg.mod". If you execute "from ..subpkg2 import mod" +from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The +specification for relative imports is contained in the Package +Relative Imports section. + +"importlib.import_module()" is provided to support applications that +determine dynamically the modules to be loaded. + +Raises an auditing event "import" with arguments "module", "filename", +"sys.path", "sys.meta_path", "sys.path_hooks". + + +Future statements +================= + +A *future statement* is a directive to the compiler that a particular +module should be compiled using syntax or semantics that will be +available in a specified future release of Python where the feature +becomes standard. + +The future statement is intended to ease migration to future versions +of Python that introduce incompatible changes to the language. It +allows use of the new features on a per-module basis before the +release in which the feature becomes standard. + + future_stmt: "from" "__future__" "import" feature ["as" identifier] + ("," feature ["as" identifier])* + | "from" "__future__" "import" "(" feature ["as" identifier] + ("," feature ["as" identifier])* [","] ")" + feature: identifier + +A future statement must appear near the top of the module. The only +lines that can appear before a future statement are: + +* the module docstring (if any), + +* comments, + +* blank lines, and + +* other future statements. + +The only feature that requires using the future statement is +"annotations" (see **PEP 563**). + +All historical features enabled by the future statement are still +recognized by Python 3. The list includes "absolute_import", +"division", "generators", "generator_stop", "unicode_literals", +"print_function", "nested_scopes" and "with_statement". They are all +redundant because they are always enabled, and only kept for backwards +compatibility. + +A future statement is recognized and treated specially at compile +time: Changes to the semantics of core constructs are often +implemented by generating different code. It may even be the case +that a new feature introduces new incompatible syntax (such as a new +reserved word), in which case the compiler may need to parse the +module differently. Such decisions cannot be pushed off until +runtime. + +For any given release, the compiler knows which feature names have +been defined, and raises a compile-time error if a future statement +contains a feature not known to it. + +The direct runtime semantics are the same as for any import statement: +there is a standard module "__future__", described later, and it will +be imported in the usual way at the time the future statement is +executed. + +The interesting runtime semantics depend on the specific feature +enabled by the future statement. + +Note that there is nothing special about the statement: + + import __future__ [as name] + +That is not a future statement; it’s an ordinary import statement with +no special semantics or syntax restrictions. + +Code compiled by calls to the built-in functions "exec()" and +"compile()" that occur in a module "M" containing a future statement +will, by default, use the new syntax or semantics associated with the +future statement. This can be controlled by optional arguments to +"compile()" — see the documentation of that function for details. + +A future statement typed at an interactive interpreter prompt will +take effect for the rest of the interpreter session. If an +interpreter is started with the "-i" option, is passed a script name +to execute, and the script includes a future statement, it will be in +effect in the interactive session started after the script is +executed. + +See also: + + **PEP 236** - Back to the __future__ + The original proposal for the __future__ mechanism. +''', + 'in': r'''Membership test operations +************************** + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". +''', + 'integers': r'''Integer literals +**************** + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'lambda': r'''Lambdas +******* + + lambda_expr: "lambda" [parameter_list] ":" expression + +Lambda expressions (sometimes called lambda forms) are used to create +anonymous functions. The expression "lambda parameters: expression" +yields a function object. The unnamed object behaves like a function +object defined with: + + def (parameters): + return expression + +See section Function definitions for the syntax of parameter lists. +Note that functions created with lambda expressions cannot contain +statements or annotations. +''', + 'lists': r'''List displays +************* + +A list display is a possibly empty series of expressions enclosed in +square brackets: + + list_display: "[" [flexible_expression_list | comprehension] "]" + +A list display yields a new list object, the contents being specified +by either a list of expressions or a comprehension. When a comma- +separated list of expressions is supplied, its elements are evaluated +from left to right and placed into the list object in that order. +When a comprehension is supplied, the list is constructed from the +elements resulting from the comprehension. +''', + 'naming': r'''Naming and binding +****************** + + +Binding of names +================ + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +=================== + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +================= + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +=============== + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +================================= + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +================================= + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'nonlocal': r'''The "nonlocal" statement +************************ + + nonlocal_stmt: "nonlocal" identifier ("," identifier)* + +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The "nonlocal" statement causes the +listed identifiers to refer to names previously bound in nonlocal +scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a "SyntaxError" is raised. + +The "nonlocal" statement applies to the entire scope of a function or +class body. A "SyntaxError" is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. + +See also: + + **PEP 3104** - Access to Names in Outer Scopes + The specification for the "nonlocal" statement. + +**Programmer’s note:** "nonlocal" is a directive to the parser and +applies only to code parsed along with it. See the note for the +"global" statement. +''', + 'numbers': r'''Numeric literals +**************** + +"NUMBER" tokens represent numeric literals, of which there are three +types: integers, floating-point numbers, and imaginary numbers. + + NUMBER: integer | floatnumber | imagnumber + +The numeric value of a numeric literal is the same as if it were +passed as a string to the "int", "float" or "complex" class +constructor, respectively. Note that not all valid inputs for those +constructors are also valid literals. + +Numeric literals do not include a sign; a phrase like "-1" is actually +an expression composed of the unary operator ‘"-"’ and the literal +"1". + + +Integer literals +================ + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Floating-point literals +======================= + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Imaginary literals +================== + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") +''', + 'numeric-types': r'''Emulating numeric types +*********************** + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the three-argument version of the + built-in "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that "__rpow__()" should be defined to accept an optional + third argument if the three-argument version of the built-in + "pow()" function is to be supported. + + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. +''', + 'objects': r'''Objects, values and types +************************* + +*Objects* are Python’s abstraction for data. All data in a Python +program is represented by objects or by relations between objects. +Even code is represented by objects. + +Every object has an identity, a type and a value. An object’s +*identity* never changes once it has been created; you may think of it +as the object’s address in memory. The "is" operator compares the +identity of two objects; the "id()" function returns an integer +representing its identity. + +**CPython implementation detail:** For CPython, "id(x)" is the memory +address where "x" is stored. + +An object’s type determines the operations that the object supports +(e.g., “does it have a length?”) and also defines the possible values +for objects of that type. The "type()" function returns an object’s +type (which is an object itself). Like its identity, an object’s +*type* is also unchangeable. [1] + +The *value* of some objects can change. Objects whose value can +change are said to be *mutable*; objects whose value is unchangeable +once they are created are called *immutable*. (The value of an +immutable container object that contains a reference to a mutable +object can change when the latter’s value is changed; however the +container is still considered immutable, because the collection of +objects it contains cannot be changed. So, immutability is not +strictly the same as having an unchangeable value, it is more subtle.) +An object’s mutability is determined by its type; for instance, +numbers, strings and tuples are immutable, while dictionaries and +lists are mutable. + +Objects are never explicitly destroyed; however, when they become +unreachable they may be garbage-collected. An implementation is +allowed to postpone garbage collection or omit it altogether — it is a +matter of implementation quality how garbage collection is +implemented, as long as no objects are collected that are still +reachable. + +**CPython implementation detail:** CPython currently uses a reference- +counting scheme with (optional) delayed detection of cyclically linked +garbage, which collects most objects as soon as they become +unreachable, but is not guaranteed to collect garbage containing +circular references. See the documentation of the "gc" module for +information on controlling the collection of cyclic garbage. Other +implementations act differently and CPython may change. Do not depend +on immediate finalization of objects when they become unreachable (so +you should always close files explicitly). + +Note that the use of the implementation’s tracing or debugging +facilities may keep objects alive that would normally be collectable. +Also note that catching an exception with a "try"…"except" statement +may keep objects alive. + +Some objects contain references to “external” resources such as open +files or windows. It is understood that these resources are freed +when the object is garbage-collected, but since garbage collection is +not guaranteed to happen, such objects also provide an explicit way to +release the external resource, usually a "close()" method. Programs +are strongly recommended to explicitly close such objects. The +"try"…"finally" statement and the "with" statement provide convenient +ways to do this. + +Some objects contain references to other objects; these are called +*containers*. Examples of containers are tuples, lists and +dictionaries. The references are part of a container’s value. In +most cases, when we talk about the value of a container, we imply the +values, not the identities of the contained objects; however, when we +talk about the mutability of a container, only the identities of the +immediately contained objects are implied. So, if an immutable +container (like a tuple) contains a reference to a mutable object, its +value changes if that mutable object is changed. + +Types affect almost all aspects of object behavior. Even the +importance of object identity is affected in some sense: for immutable +types, operations that compute new values may actually return a +reference to any existing object with the same type and value, while +for mutable objects this is not allowed. For example, after "a = 1; b += 1", *a* and *b* may or may not refer to the same object with the +value one, depending on the implementation. This is because "int" is +an immutable type, so the reference to "1" can be reused. This +behaviour depends on the implementation used, so should not be relied +upon, but is something to be aware of when making use of object +identity tests. However, after "c = []; d = []", *c* and *d* are +guaranteed to refer to two different, unique, newly created empty +lists. (Note that "e = f = []" assigns the *same* object to both *e* +and *f*.) +''', + 'operator-summary': r'''Operator precedence +******************* + +The following table summarizes the operator precedence in Python, from +highest precedence (most binding) to lowest precedence (least +binding). Operators in the same box have the same precedence. Unless +the syntax is explicitly given, operators are binary. Operators in +the same box group left to right (except for exponentiation and +conditional expressions, which group from right to left). + +Note that comparisons, membership tests, and identity tests, all have +the same precedence and have a left-to-right chaining feature as +described in the Comparisons section. + ++-------------------------------------------------+---------------------------------------+ +| Operator | Description | +|=================================================|=======================================| +| "(expressions...)", "[expressions...]", "{key: | Binding or parenthesized expression, | +| value...}", "{expressions...}" | list display, dictionary display, set | +| | display | ++-------------------------------------------------+---------------------------------------+ +| "x[index]", "x[index:index]", | Subscription, slicing, call, | +| "x(arguments...)", "x.attribute" | attribute reference | ++-------------------------------------------------+---------------------------------------+ +| "await x" | Await expression | ++-------------------------------------------------+---------------------------------------+ +| "**" | Exponentiation [5] | ++-------------------------------------------------+---------------------------------------+ +| "+x", "-x", "~x" | Positive, negative, bitwise NOT | ++-------------------------------------------------+---------------------------------------+ +| "*", "@", "/", "//", "%" | Multiplication, matrix | +| | multiplication, division, floor | +| | division, remainder [6] | ++-------------------------------------------------+---------------------------------------+ +| "+", "-" | Addition and subtraction | ++-------------------------------------------------+---------------------------------------+ +| "<<", ">>" | Shifts | ++-------------------------------------------------+---------------------------------------+ +| "&" | Bitwise AND | ++-------------------------------------------------+---------------------------------------+ +| "^" | Bitwise XOR | ++-------------------------------------------------+---------------------------------------+ +| "|" | Bitwise OR | ++-------------------------------------------------+---------------------------------------+ +| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership | +| ">=", "!=", "==" | tests and identity tests | ++-------------------------------------------------+---------------------------------------+ +| "not x" | Boolean NOT | ++-------------------------------------------------+---------------------------------------+ +| "and" | Boolean AND | ++-------------------------------------------------+---------------------------------------+ +| "or" | Boolean OR | ++-------------------------------------------------+---------------------------------------+ +| "if" – "else" | Conditional expression | ++-------------------------------------------------+---------------------------------------+ +| "lambda" | Lambda expression | ++-------------------------------------------------+---------------------------------------+ +| ":=" | Assignment expression | ++-------------------------------------------------+---------------------------------------+ + +-[ Footnotes ]- + +[1] While "abs(x%y) < abs(y)" is true mathematically, for floats it + may not be true numerically due to roundoff. For example, and + assuming a platform on which a Python float is an IEEE 754 double- + precision number, in order that "-1e-100 % 1e100" have the same + sign as "1e100", the computed result is "-1e-100 + 1e100", which + is numerically exactly equal to "1e100". The function + "math.fmod()" returns a result whose sign matches the sign of the + first argument instead, and so returns "-1e-100" in this case. + Which approach is more appropriate depends on the application. + +[2] If x is very close to an exact integer multiple of y, it’s + possible for "x//y" to be one larger than "(x-x%y)//y" due to + rounding. In such cases, Python returns the latter result, in + order to preserve that "divmod(x,y)[0] * y + x % y" be very close + to "x". + +[3] The Unicode standard distinguishes between *code points* (e.g. + U+0041) and *abstract characters* (e.g. “LATIN CAPITAL LETTER A”). + While most abstract characters in Unicode are only represented + using one code point, there is a number of abstract characters + that can in addition be represented using a sequence of more than + one code point. For example, the abstract character “LATIN + CAPITAL LETTER C WITH CEDILLA” can be represented as a single + *precomposed character* at code position U+00C7, or as a sequence + of a *base character* at code position U+0043 (LATIN CAPITAL + LETTER C), followed by a *combining character* at code position + U+0327 (COMBINING CEDILLA). + + The comparison operators on strings compare at the level of + Unicode code points. This may be counter-intuitive to humans. For + example, ""\u00C7" == "\u0043\u0327"" is "False", even though both + strings represent the same abstract character “LATIN CAPITAL + LETTER C WITH CEDILLA”. + + To compare strings at the level of abstract characters (that is, + in a way intuitive to humans), use "unicodedata.normalize()". + +[4] Due to automatic garbage-collection, free lists, and the dynamic + nature of descriptors, you may notice seemingly unusual behaviour + in certain uses of the "is" operator, like those involving + comparisons between instance methods, or constants. Check their + documentation for more info. + +[5] The power operator "**" binds less tightly than an arithmetic or + bitwise unary operator on its right, that is, "2**-1" is "0.5". + +[6] The "%" operator is also used for string formatting; the same + precedence applies. +''', + 'pass': r'''The "pass" statement +******************** + + pass_stmt: "pass" + +"pass" is a null operation — when it is executed, nothing happens. It +is useful as a placeholder when a statement is required syntactically, +but no code needs to be executed, for example: + + def f(arg): pass # a function that does nothing (yet) + + class C: pass # a class with no methods (yet) +''', + 'power': r'''The power operator +****************** + +The power operator binds more tightly than unary operators on its +left; it binds less tightly than unary operators on its right. The +syntax is: + + power: (await_expr | primary) ["**" u_expr] + +Thus, in an unparenthesized sequence of power and unary operators, the +operators are evaluated from right to left (this does not constrain +the evaluation order for the operands): "-1**2" results in "-1". + +The power operator has the same semantics as the built-in "pow()" +function, when called with two arguments: it yields its left argument +raised to the power of its right argument. The numeric arguments are +first converted to a common type, and the result is of that type. + +For int operands, the result has the same type as the operands unless +the second argument is negative; in that case, all arguments are +converted to float and a float result is delivered. For example, +"10**2" returns "100", but "10**-2" returns "0.01". + +Raising "0.0" to a negative power results in a "ZeroDivisionError". +Raising a negative number to a fractional power results in a "complex" +number. (In earlier versions it raised a "ValueError".) + +This operation can be customized using the special "__pow__()" and +"__rpow__()" methods. +''', + 'raise': r'''The "raise" statement +********************* + + raise_stmt: "raise" [expression ["from" expression]] + +If no expressions are present, "raise" re-raises the exception that is +currently being handled, which is also known as the *active +exception*. If there isn’t currently an active exception, a +"RuntimeError" exception is raised indicating that this is an error. + +Otherwise, "raise" evaluates the first expression as the exception +object. It must be either a subclass or an instance of +"BaseException". If it is a class, the exception instance will be +obtained when needed by instantiating the class with no arguments. + +The *type* of the exception is the exception instance’s class, the +*value* is the instance itself. + +A traceback object is normally created automatically when an exception +is raised and attached to it as the "__traceback__" attribute. You can +create an exception and set your own traceback in one step using the +"with_traceback()" exception method (which returns the same exception +instance, with its traceback set to its argument), like so: + + raise Exception("foo occurred").with_traceback(tracebackobj) + +The "from" clause is used for exception chaining: if given, the second +*expression* must be another exception class or instance. If the +second expression is an exception instance, it will be attached to the +raised exception as the "__cause__" attribute (which is writable). If +the expression is an exception class, the class will be instantiated +and the resulting exception instance will be attached to the raised +exception as the "__cause__" attribute. If the raised exception is not +handled, both exceptions will be printed: + + >>> try: + ... print(1 / 0) + ... except Exception as exc: + ... raise RuntimeError("Something bad happened") from exc + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") from exc + RuntimeError: Something bad happened + +A similar mechanism works implicitly if a new exception is raised when +an exception is already being handled. An exception may be handled +when an "except" or "finally" clause, or a "with" statement, is used. +The previous exception is then attached as the new exception’s +"__context__" attribute: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") + RuntimeError: Something bad happened + +Exception chaining can be explicitly suppressed by specifying "None" +in the "from" clause: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + +Additional information on exceptions can be found in section +Exceptions, and information about handling exceptions is in section +The try statement. + +Changed in version 3.3: "None" is now permitted as "Y" in "raise X +from Y".Added the "__suppress_context__" attribute to suppress +automatic display of the exception context. + +Changed in version 3.11: If the traceback of the active exception is +modified in an "except" clause, a subsequent "raise" statement re- +raises the exception with the modified traceback. Previously, the +exception was re-raised with the traceback it had when it was caught. +''', + 'return': r'''The "return" statement +********************** + + return_stmt: "return" [expression_list] + +"return" may only occur syntactically nested in a function definition, +not within a nested class definition. + +If an expression list is present, it is evaluated, else "None" is +substituted. + +"return" leaves the current function call with the expression list (or +"None") as return value. + +When "return" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +function. + +In a generator function, the "return" statement indicates that the +generator is done and will cause "StopIteration" to be raised. The +returned value (if any) is used as an argument to construct +"StopIteration" and becomes the "StopIteration.value" attribute. + +In an asynchronous generator function, an empty "return" statement +indicates that the asynchronous generator is done and will cause +"StopAsyncIteration" to be raised. A non-empty "return" statement is +a syntax error in an asynchronous generator function. +''', + 'sequence-types': r'''Emulating container types +************************* + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. +''', + 'shifting': r'''Shifting operations +******************* + +The shifting operations have lower priority than the arithmetic +operations: + + shift_expr: a_expr | shift_expr ("<<" | ">>") a_expr + +These operators accept integers as arguments. They shift the first +argument to the left or right by the number of bits given by the +second argument. + +The left shift operation can be customized using the special +"__lshift__()" and "__rlshift__()" methods. The right shift operation +can be customized using the special "__rshift__()" and "__rrshift__()" +methods. + +A right shift by *n* bits is defined as floor division by "pow(2,n)". +A left shift by *n* bits is defined as multiplication with "pow(2,n)". +''', + 'slicings': r'''Slicings +******** + +A slicing selects a range of items in a sequence object (e.g., a +string, tuple or list). Slicings may be used as expressions or as +targets in assignment or "del" statements. The syntax for a slicing: + + slicing: primary "[" slice_list "]" + slice_list: slice_item ("," slice_item)* [","] + slice_item: expression | proper_slice + proper_slice: [lower_bound] ":" [upper_bound] [ ":" [stride] ] + lower_bound: expression + upper_bound: expression + stride: expression + +There is ambiguity in the formal syntax here: anything that looks like +an expression list also looks like a slice list, so any subscription +can be interpreted as a slicing. Rather than further complicating the +syntax, this is disambiguated by defining that in this case the +interpretation as a subscription takes priority over the +interpretation as a slicing (this is the case if the slice list +contains no proper slice). + +The semantics for a slicing are as follows. The primary is indexed +(using the same "__getitem__()" method as normal subscription) with a +key that is constructed from the slice list, as follows. If the slice +list contains at least one comma, the key is a tuple containing the +conversion of the slice items; otherwise, the conversion of the lone +slice item is the key. The conversion of a slice item that is an +expression is that expression. The conversion of a proper slice is a +slice object (see section The standard type hierarchy) whose "start", +"stop" and "step" attributes are the values of the expressions given +as lower bound, upper bound and stride, respectively, substituting +"None" for missing expressions. +''', + 'specialattrs': r'''Special Attributes +****************** + +The implementation adds a few special read-only attributes to several +object types, where they are relevant. Some of these are not reported +by the "dir()" built-in function. + +definition.__name__ + + The name of the class, function, method, descriptor, or generator + instance. + +definition.__qualname__ + + The *qualified name* of the class, function, method, descriptor, or + generator instance. + + Added in version 3.3. + +definition.__module__ + + The name of the module in which a class or function was defined. + +definition.__doc__ + + The documentation string of a class or function, or "None" if + undefined. + +definition.__type_params__ + + The type parameters of generic classes, functions, and type + aliases. For classes and functions that are not generic, this will + be an empty tuple. + + Added in version 3.12. +''', + 'specialnames': r'''Special method names +******************** + +A class can implement certain operations that are invoked by special +syntax (such as arithmetic operations or subscripting and slicing) by +defining methods with special names. This is Python’s approach to +*operator overloading*, allowing classes to define their own behavior +with respect to language operators. For instance, if a class defines +a method named "__getitem__()", and "x" is an instance of this class, +then "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)". +Except where mentioned, attempts to execute an operation raise an +exception when no appropriate method is defined (typically +"AttributeError" or "TypeError"). + +Setting a special method to "None" indicates that the corresponding +operation is not available. For example, if a class sets "__iter__()" +to "None", the class is not iterable, so calling "iter()" on its +instances will raise a "TypeError" (without falling back to +"__getitem__()"). [2] + +When implementing a class that emulates any built-in type, it is +important that the emulation only be implemented to the degree that it +makes sense for the object being modelled. For example, some +sequences may work well with retrieval of individual elements, but +extracting a slice may not make sense. (One example of this is the +NodeList interface in the W3C’s Document Object Model.) + + +Basic customization +=================== + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. + + +Customizing attribute access +============================ + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +----------------------------------- + +module.__getattr__() +module.__dir__() + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +module.__class__ + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +------------------------ + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +-------------------- + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +--------- + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. + + +Customizing class creation +========================== + +Whenever a class inherits from another class, "__init_subclass__()" is +called on the parent class. This way, it is possible to write classes +which change the behavior of subclasses. This is closely related to +class decorators, but where class decorators only affect the specific +class they’re applied to, "__init_subclass__" solely applies to future +subclasses of the class defining the method. + +classmethod object.__init_subclass__(cls) + + This method is called whenever the containing class is subclassed. + *cls* is then the new subclass. If defined as a normal instance + method, this method is implicitly converted to a class method. + + Keyword arguments which are given to a new class are passed to the + parent class’s "__init_subclass__". For compatibility with other + classes using "__init_subclass__", one should take out the needed + keyword arguments and pass the others over to the base class, as + in: + + class Philosopher: + def __init_subclass__(cls, /, default_name, **kwargs): + super().__init_subclass__(**kwargs) + cls.default_name = default_name + + class AustralianPhilosopher(Philosopher, default_name="Bruce"): + pass + + The default implementation "object.__init_subclass__" does nothing, + but raises an error if it is called with any arguments. + + Note: + + The metaclass hint "metaclass" is consumed by the rest of the + type machinery, and is never passed to "__init_subclass__" + implementations. The actual metaclass (rather than the explicit + hint) can be accessed as "type(cls)". + + Added in version 3.6. + +When a class is created, "type.__new__()" scans the class variables +and makes callbacks to those with a "__set_name__()" hook. + +object.__set_name__(self, owner, name) + + Automatically called at the time the owning class *owner* is + created. The object has been assigned to *name* in that class: + + class A: + x = C() # Automatically calls: x.__set_name__(A, 'x') + + If the class variable is assigned after the class is created, + "__set_name__()" will not be called automatically. If needed, + "__set_name__()" can be called directly: + + class A: + pass + + c = C() + A.x = c # The hook is not called + c.__set_name__(A, 'x') # Manually invoke the hook + + See Creating the class object for more details. + + Added in version 3.6. + + +Metaclasses +----------- + +By default, classes are constructed using "type()". The class body is +executed in a new namespace and the class name is bound locally to the +result of "type(name, bases, namespace)". + +The class creation process can be customized by passing the +"metaclass" keyword argument in the class definition line, or by +inheriting from an existing class that included such an argument. In +the following example, both "MyClass" and "MySubclass" are instances +of "Meta": + + class Meta(type): + pass + + class MyClass(metaclass=Meta): + pass + + class MySubclass(MyClass): + pass + +Any other keyword arguments that are specified in the class definition +are passed through to all metaclass operations described below. + +When a class definition is executed, the following steps occur: + +* MRO entries are resolved; + +* the appropriate metaclass is determined; + +* the class namespace is prepared; + +* the class body is executed; + +* the class object is created. + + +Resolving MRO entries +--------------------- + +object.__mro_entries__(self, bases) + + If a base that appears in a class definition is not an instance of + "type", then an "__mro_entries__()" method is searched on the base. + If an "__mro_entries__()" method is found, the base is substituted + with the result of a call to "__mro_entries__()" when creating the + class. The method is called with the original bases tuple passed to + the *bases* parameter, and must return a tuple of classes that will + be used instead of the base. The returned tuple may be empty: in + these cases, the original base is ignored. + +See also: + + "types.resolve_bases()" + Dynamically resolve bases that are not instances of "type". + + "types.get_original_bases()" + Retrieve a class’s “original bases” prior to modifications by + "__mro_entries__()". + + **PEP 560** + Core support for typing module and generic types. + + +Determining the appropriate metaclass +------------------------------------- + +The appropriate metaclass for a class definition is determined as +follows: + +* if no bases and no explicit metaclass are given, then "type()" is + used; + +* if an explicit metaclass is given and it is *not* an instance of + "type()", then it is used directly as the metaclass; + +* if an instance of "type()" is given as the explicit metaclass, or + bases are defined, then the most derived metaclass is used. + +The most derived metaclass is selected from the explicitly specified +metaclass (if any) and the metaclasses (i.e. "type(cls)") of all +specified base classes. The most derived metaclass is one which is a +subtype of *all* of these candidate metaclasses. If none of the +candidate metaclasses meets that criterion, then the class definition +will fail with "TypeError". + + +Preparing the class namespace +----------------------------- + +Once the appropriate metaclass has been identified, then the class +namespace is prepared. If the metaclass has a "__prepare__" attribute, +it is called as "namespace = metaclass.__prepare__(name, bases, +**kwds)" (where the additional keyword arguments, if any, come from +the class definition). The "__prepare__" method should be implemented +as a "classmethod". The namespace returned by "__prepare__" is passed +in to "__new__", but when the final class object is created the +namespace is copied into a new "dict". + +If the metaclass has no "__prepare__" attribute, then the class +namespace is initialised as an empty ordered mapping. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + Introduced the "__prepare__" namespace hook + + +Executing the class body +------------------------ + +The class body is executed (approximately) as "exec(body, globals(), +namespace)". The key difference from a normal call to "exec()" is that +lexical scoping allows the class body (including any methods) to +reference names from the current and outer scopes when the class +definition occurs inside a function. + +However, even when the class definition occurs inside the function, +methods defined inside the class still cannot see names defined at the +class scope. Class variables must be accessed through the first +parameter of instance or class methods, or through the implicit +lexically scoped "__class__" reference described in the next section. + + +Creating the class object +------------------------- + +Once the class namespace has been populated by executing the class +body, the class object is created by calling "metaclass(name, bases, +namespace, **kwds)" (the additional keywords passed here are the same +as those passed to "__prepare__"). + +This class object is the one that will be referenced by the zero- +argument form of "super()". "__class__" is an implicit closure +reference created by the compiler if any methods in a class body refer +to either "__class__" or "super". This allows the zero argument form +of "super()" to correctly identify the class being defined based on +lexical scoping, while the class or instance that was used to make the +current call is identified based on the first argument passed to the +method. + +**CPython implementation detail:** In CPython 3.6 and later, the +"__class__" cell is passed to the metaclass as a "__classcell__" entry +in the class namespace. If present, this must be propagated up to the +"type.__new__" call in order for the class to be initialised +correctly. Failing to do so will result in a "RuntimeError" in Python +3.8. + +When using the default metaclass "type", or any metaclass that +ultimately calls "type.__new__", the following additional +customization steps are invoked after creating the class object: + +1. The "type.__new__" method collects all of the attributes in the + class namespace that define a "__set_name__()" method; + +2. Those "__set_name__" methods are called with the class being + defined and the assigned name of that particular attribute; + +3. The "__init_subclass__()" hook is called on the immediate parent of + the new class in its method resolution order. + +After the class object is created, it is passed to the class +decorators included in the class definition (if any) and the resulting +object is bound in the local namespace as the defined class. + +When a new class is created by "type.__new__", the object provided as +the namespace parameter is copied to a new ordered mapping and the +original object is discarded. The new copy is wrapped in a read-only +proxy, which becomes the "__dict__" attribute of the class object. + +See also: + + **PEP 3135** - New super + Describes the implicit "__class__" closure reference + + +Uses for metaclasses +-------------------- + +The potential uses for metaclasses are boundless. Some ideas that have +been explored include enum, logging, interface checking, automatic +delegation, automatic property creation, proxies, frameworks, and +automatic resource locking/synchronization. + + +Customizing instance and subclass checks +======================================== + +The following methods are used to override the default behavior of the +"isinstance()" and "issubclass()" built-in functions. + +In particular, the metaclass "abc.ABCMeta" implements these methods in +order to allow the addition of Abstract Base Classes (ABCs) as +“virtual base classes” to any class or type (including built-in +types), including other ABCs. + +type.__instancecheck__(self, instance) + + Return true if *instance* should be considered a (direct or + indirect) instance of *class*. If defined, called to implement + "isinstance(instance, class)". + +type.__subclasscheck__(self, subclass) + + Return true if *subclass* should be considered a (direct or + indirect) subclass of *class*. If defined, called to implement + "issubclass(subclass, class)". + +Note that these methods are looked up on the type (metaclass) of a +class. They cannot be defined as class methods in the actual class. +This is consistent with the lookup of special methods that are called +on instances, only in this case the instance is itself a class. + +See also: + + **PEP 3119** - Introducing Abstract Base Classes + Includes the specification for customizing "isinstance()" and + "issubclass()" behavior through "__instancecheck__()" and + "__subclasscheck__()", with motivation for this functionality in + the context of adding Abstract Base Classes (see the "abc" + module) to the language. + + +Emulating generic types +======================= + +When using *type annotations*, it is often useful to *parameterize* a +*generic type* using Python’s square-brackets notation. For example, +the annotation "list[int]" might be used to signify a "list" in which +all the elements are of type "int". + +See also: + + **PEP 484** - Type Hints + Introducing Python’s framework for type annotations + + Generic Alias Types + Documentation for objects representing parameterized generic + classes + + Generics, user-defined generics and "typing.Generic" + Documentation on how to implement generic classes that can be + parameterized at runtime and understood by static type-checkers. + +A class can *generally* only be parameterized if it defines the +special class method "__class_getitem__()". + +classmethod object.__class_getitem__(cls, key) + + Return an object representing the specialization of a generic class + by type arguments found in *key*. + + When defined on a class, "__class_getitem__()" is automatically a + class method. As such, there is no need for it to be decorated with + "@classmethod" when it is defined. + + +The purpose of *__class_getitem__* +---------------------------------- + +The purpose of "__class_getitem__()" is to allow runtime +parameterization of standard-library generic classes in order to more +easily apply *type hints* to these classes. + +To implement custom generic classes that can be parameterized at +runtime and understood by static type-checkers, users should either +inherit from a standard library class that already implements +"__class_getitem__()", or inherit from "typing.Generic", which has its +own implementation of "__class_getitem__()". + +Custom implementations of "__class_getitem__()" on classes defined +outside of the standard library may not be understood by third-party +type-checkers such as mypy. Using "__class_getitem__()" on any class +for purposes other than type hinting is discouraged. + + +*__class_getitem__* versus *__getitem__* +---------------------------------------- + +Usually, the subscription of an object using square brackets will call +the "__getitem__()" instance method defined on the object’s class. +However, if the object being subscribed is itself a class, the class +method "__class_getitem__()" may be called instead. +"__class_getitem__()" should return a GenericAlias object if it is +properly defined. + +Presented with the *expression* "obj[x]", the Python interpreter +follows something like the following process to decide whether +"__getitem__()" or "__class_getitem__()" should be called: + + from inspect import isclass + + def subscribe(obj, x): + """Return the result of the expression 'obj[x]'""" + + class_of_obj = type(obj) + + # If the class of obj defines __getitem__, + # call class_of_obj.__getitem__(obj, x) + if hasattr(class_of_obj, '__getitem__'): + return class_of_obj.__getitem__(obj, x) + + # Else, if obj is a class and defines __class_getitem__, + # call obj.__class_getitem__(x) + elif isclass(obj) and hasattr(obj, '__class_getitem__'): + return obj.__class_getitem__(x) + + # Else, raise an exception + else: + raise TypeError( + f"'{class_of_obj.__name__}' object is not subscriptable" + ) + +In Python, all classes are themselves instances of other classes. The +class of a class is known as that class’s *metaclass*, and most +classes have the "type" class as their metaclass. "type" does not +define "__getitem__()", meaning that expressions such as "list[int]", +"dict[str, float]" and "tuple[str, bytes]" all result in +"__class_getitem__()" being called: + + >>> # list has class "type" as its metaclass, like most classes: + >>> type(list) + + >>> type(dict) == type(list) == type(tuple) == type(str) == type(bytes) + True + >>> # "list[int]" calls "list.__class_getitem__(int)" + >>> list[int] + list[int] + >>> # list.__class_getitem__ returns a GenericAlias object: + >>> type(list[int]) + + +However, if a class has a custom metaclass that defines +"__getitem__()", subscribing the class may result in different +behaviour. An example of this can be found in the "enum" module: + + >>> from enum import Enum + >>> class Menu(Enum): + ... """A breakfast menu""" + ... SPAM = 'spam' + ... BACON = 'bacon' + ... + >>> # Enum classes have a custom metaclass: + >>> type(Menu) + + >>> # EnumMeta defines __getitem__, + >>> # so __class_getitem__ is not called, + >>> # and the result is not a GenericAlias object: + >>> Menu['SPAM'] + + >>> type(Menu['SPAM']) + + +See also: + + **PEP 560** - Core Support for typing module and generic types + Introducing "__class_getitem__()", and outlining when a + subscription results in "__class_getitem__()" being called + instead of "__getitem__()" + + +Emulating callable objects +========================== + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. + + +Emulating container types +========================= + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. + + +Emulating numeric types +======================= + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the three-argument version of the + built-in "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that "__rpow__()" should be defined to accept an optional + third argument if the three-argument version of the built-in + "pow()" function is to be supported. + + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. + + +With Statement Context Managers +=============================== + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +Customizing positional arguments in class pattern matching +========================================================== + +When using a class name in a pattern, positional arguments in the +pattern are not allowed by default, i.e. "case MyClass(x, y)" is +typically invalid without special support in "MyClass". To be able to +use that kind of pattern, the class needs to define a *__match_args__* +attribute. + +object.__match_args__ + + This class variable can be assigned a tuple of strings. When this + class is used in a class pattern with positional arguments, each + positional argument will be converted into a keyword argument, + using the corresponding value in *__match_args__* as the keyword. + The absence of this attribute is equivalent to setting it to "()". + +For example, if "MyClass.__match_args__" is "("left", "center", +"right")" that means that "case MyClass(x, y)" is equivalent to "case +MyClass(left=x, center=y)". Note that the number of arguments in the +pattern must be smaller than or equal to the number of elements in +*__match_args__*; if it is larger, the pattern match attempt will +raise a "TypeError". + +Added in version 3.10. + +See also: + + **PEP 634** - Structural Pattern Matching + The specification for the Python "match" statement. + + +Emulating buffer types +====================== + +The buffer protocol provides a way for Python objects to expose +efficient access to a low-level memory array. This protocol is +implemented by builtin types such as "bytes" and "memoryview", and +third-party libraries may define additional buffer types. + +While buffer types are usually implemented in C, it is also possible +to implement the protocol in Python. + +object.__buffer__(self, flags) + + Called when a buffer is requested from *self* (for example, by the + "memoryview" constructor). The *flags* argument is an integer + representing the kind of buffer requested, affecting for example + whether the returned buffer is read-only or writable. + "inspect.BufferFlags" provides a convenient way to interpret the + flags. The method must return a "memoryview" object. + +object.__release_buffer__(self, buffer) + + Called when a buffer is no longer needed. The *buffer* argument is + a "memoryview" object that was previously returned by + "__buffer__()". The method must release any resources associated + with the buffer. This method should return "None". Buffer objects + that do not need to perform any cleanup are not required to + implement this method. + +Added in version 3.12. + +See also: + + **PEP 688** - Making the buffer protocol accessible in Python + Introduces the Python "__buffer__" and "__release_buffer__" + methods. + + "collections.abc.Buffer" + ABC for buffer types. + + +Annotations +=========== + +Functions, classes, and modules may contain *annotations*, which are a +way to associate information (usually *type hints*) with a symbol. + +object.__annotations__ + + This attribute contains the annotations for an object. It is lazily + evaluated, so accessing the attribute may execute arbitrary code + and raise exceptions. If evaluation is successful, the attribute is + set to a dictionary mapping from variable names to annotations. + + Changed in version 3.14: Annotations are now lazily evaluated. + +object.__annotate__(format) + + An *annotate function*. Returns a new dictionary object mapping + attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations + values should be provided. It must be a member of the + "annotationlib.Format" enum, or an integer with a value + corresponding to a member of the enum. + + If an annotate function doesn’t support the requested format, it + must raise "NotImplementedError". Annotate functions must always + support "VALUE" format; they must not raise "NotImplementedError()" + when called with this format. + + When called with "VALUE" format, an annotate function may raise + "NameError"; it must not raise "NameError" when called requesting + any other format. + + If an object does not have any annotations, "__annotate__" should + preferably be set to "None" (it can’t be deleted), rather than set + to a function that returns an empty dict. + + Added in version 3.14. + +See also: + + **PEP 649** — Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the "__annotate__" + function. + + +Special method lookup +===================== + +For custom classes, implicit invocations of special methods are only +guaranteed to work correctly if defined on an object’s type, not in +the object’s instance dictionary. That behaviour is the reason why +the following code raises an exception: + + >>> class C: + ... pass + ... + >>> c = C() + >>> c.__len__ = lambda: 5 + >>> len(c) + Traceback (most recent call last): + File "", line 1, in + TypeError: object of type 'C' has no len() + +The rationale behind this behaviour lies with a number of special +methods such as "__hash__()" and "__repr__()" that are implemented by +all objects, including type objects. If the implicit lookup of these +methods used the conventional lookup process, they would fail when +invoked on the type object itself: + + >>> 1 .__hash__() == hash(1) + True + >>> int.__hash__() == hash(int) + Traceback (most recent call last): + File "", line 1, in + TypeError: descriptor '__hash__' of 'int' object needs an argument + +Incorrectly attempting to invoke an unbound method of a class in this +way is sometimes referred to as ‘metaclass confusion’, and is avoided +by bypassing the instance when looking up special methods: + + >>> type(1).__hash__(1) == hash(1) + True + >>> type(int).__hash__(int) == hash(int) + True + +In addition to bypassing any instance attributes in the interest of +correctness, implicit special method lookup generally also bypasses +the "__getattribute__()" method even of the object’s metaclass: + + >>> class Meta(type): + ... def __getattribute__(*args): + ... print("Metaclass getattribute invoked") + ... return type.__getattribute__(*args) + ... + >>> class C(object, metaclass=Meta): + ... def __len__(self): + ... return 10 + ... def __getattribute__(*args): + ... print("Class getattribute invoked") + ... return object.__getattribute__(*args) + ... + >>> c = C() + >>> c.__len__() # Explicit lookup via instance + Class getattribute invoked + 10 + >>> type(c).__len__(c) # Explicit lookup via type + Metaclass getattribute invoked + 10 + >>> len(c) # Implicit lookup + 10 + +Bypassing the "__getattribute__()" machinery in this fashion provides +significant scope for speed optimisations within the interpreter, at +the cost of some flexibility in the handling of special methods (the +special method *must* be set on the class object itself in order to be +consistently invoked by the interpreter). +''', + 'string-methods': r'''String Methods +************** + +Strings implement all of the common sequence operations, along with +the additional methods described below. + +Strings also support two styles of string formatting, one providing a +large degree of flexibility and customization (see "str.format()", +Format String Syntax and Custom String Formatting) and the other based +on C "printf" style formatting that handles a narrower range of types +and is slightly harder to use correctly, but is often faster for the +cases it can handle (printf-style String Formatting). + +The Text Processing Services section of the standard library covers a +number of other modules that provide various text related utilities +(including regular expression support in the "re" module). + +str.capitalize() + + Return a copy of the string with its first character capitalized + and the rest lowercased. + + Changed in version 3.8: The first character is now put into + titlecase rather than uppercase. This means that characters like + digraphs will only have their first letter capitalized, instead of + the full character. + +str.casefold() + + Return a casefolded copy of the string. Casefolded strings may be + used for caseless matching. + + Casefolding is similar to lowercasing but more aggressive because + it is intended to remove all case distinctions in a string. For + example, the German lowercase letter "'ß'" is equivalent to ""ss"". + Since it is already lowercase, "lower()" would do nothing to "'ß'"; + "casefold()" converts it to ""ss"". For example: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' + + The casefolding algorithm is described in section 3.13 ‘Default + Case Folding’ of the Unicode Standard. + + Added in version 3.3. + +str.center(width, fillchar=' ', /) + + Return centered in a string of length *width*. Padding is done + using the specified *fillchar* (default is an ASCII space). The + original string is returned if *width* is less than or equal to + "len(s)". For example: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' + +str.count(sub[, start[, end]]) + + Return the number of non-overlapping occurrences of substring *sub* + in the range [*start*, *end*]. Optional arguments *start* and + *end* are interpreted as in slice notation. + + If *sub* is empty, returns the number of empty strings between + characters which is the length of the string plus one. For example: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 + +str.encode(encoding='utf-8', errors='strict') + + Return the string encoded to "bytes". + + *encoding* defaults to "'utf-8'"; see Standard Encodings for + possible values. + + *errors* controls how encoding errors are handled. If "'strict'" + (the default), a "UnicodeError" exception is raised. Other possible + values are "'ignore'", "'replace'", "'xmlcharrefreplace'", + "'backslashreplace'" and any other name registered via + "codecs.register_error()". See Error Handlers for details. + + For performance reasons, the value of *errors* is not checked for + validity unless an encoding error actually occurs, Python + Development Mode is enabled or a debug build is used. For example: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' + + Changed in version 3.1: Added support for keyword arguments. + + Changed in version 3.9: The value of the *errors* argument is now + checked in Python Development Mode and in debug mode. + +str.endswith(suffix[, start[, end]]) + + Return "True" if the string ends with the specified *suffix*, + otherwise return "False". *suffix* can also be a tuple of suffixes + to look for. With optional *start*, test beginning at that + position. With optional *end*, stop comparing at that position. + Using *start* and *end* is equivalent to + "str[start:end].endswith(suffix)". For example: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also "startswith()" and "removesuffix()". + +str.expandtabs(tabsize=8) + + Return a copy of the string where all tab characters are replaced + by one or more spaces, depending on the current column and the + given tab size. Tab positions occur every *tabsize* characters + (default is 8, giving tab positions at columns 0, 8, 16 and so on). + To expand the string, the current column is set to zero and the + string is examined character by character. If the character is a + tab ("\t"), one or more space characters are inserted in the result + until the current column is equal to the next tab position. (The + tab character itself is not copied.) If the character is a newline + ("\n") or return ("\r"), it is copied and the current column is + reset to zero. Any other character is copied unchanged and the + current column is incremented by one regardless of how the + character is represented when printed. For example: + + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 + +str.find(sub[, start[, end]]) + + Return the lowest index in the string where substring *sub* is + found within the slice "s[start:end]". Optional arguments *start* + and *end* are interpreted as in slice notation. Return "-1" if + *sub* is not found. For example: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also "rfind()" and "index()". + + Note: + + The "find()" method should be used only if you need to know the + position of *sub*. To check if *sub* is a substring or not, use + the "in" operator: + + >>> 'Py' in 'Python' + True + +str.format(*args, **kwargs) + + Perform a string formatting operation. The string on which this + method is called can contain literal text or replacement fields + delimited by braces "{}". Each replacement field contains either + the numeric index of a positional argument, or the name of a + keyword argument. Returns a copy of the string where each + replacement field is replaced with the string value of the + corresponding argument. For example: + + >>> "The sum of 1 + 2 is {0}".format(1+2) + 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' + + See Format String Syntax for a description of the various + formatting options that can be specified in format strings. + + Note: + + When formatting a number ("int", "float", "complex", + "decimal.Decimal" and subclasses) with the "n" type (ex: + "'{:n}'.format(1234)"), the function temporarily sets the + "LC_CTYPE" locale to the "LC_NUMERIC" locale to decode + "decimal_point" and "thousands_sep" fields of "localeconv()" if + they are non-ASCII or longer than 1 byte, and the "LC_NUMERIC" + locale is different than the "LC_CTYPE" locale. This temporary + change affects other threads. + + Changed in version 3.7: When formatting a number with the "n" type, + the function sets temporarily the "LC_CTYPE" locale to the + "LC_NUMERIC" locale in some cases. + +str.format_map(mapping, /) + + Similar to "str.format(**mapping)", except that "mapping" is used + directly and not copied to a "dict". This is useful if for example + "mapping" is a dict subclass: + + >>> class Default(dict): + ... def __missing__(self, key): + ... return key + ... + >>> '{name} was born in {country}'.format_map(Default(name='Guido')) + 'Guido was born in country' + + Added in version 3.2. + +str.index(sub[, start[, end]]) + + Like "find()", but raise "ValueError" when the substring is not + found. For example: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "rindex()". + +str.isalnum() + + Return "True" if all characters in the string are alphanumeric and + there is at least one character, "False" otherwise. A character + "c" is alphanumeric if one of the following returns "True": + "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()". + +str.isalpha() + + Return "True" if all characters in the string are alphabetic and + there is at least one character, "False" otherwise. Alphabetic + characters are those characters defined in the Unicode character + database as “Letter”, i.e., those with general category property + being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note that this is + different from the Alphabetic property defined in the section 4.10 + ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. For + example: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See Unicode Properties. + +str.isascii() + + Return "True" if the string is empty or all characters in the + string are ASCII, "False" otherwise. ASCII characters have code + points in the range U+0000-U+007F. For example: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False + + Added in version 3.7. + +str.isdecimal() + + Return "True" if all characters in the string are decimal + characters and there is at least one character, "False" otherwise. + Decimal characters are those that can be used to form numbers in + base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a + decimal character is a character in the Unicode General Category + “Nd”. For example: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False + +str.isdigit() + + Return "True" if all characters in the string are digits and there + is at least one character, "False" otherwise. Digits include + decimal characters and digits that need special handling, such as + the compatibility superscript digits. This covers digits which + cannot be used to form numbers in base 10, like the Kharosthi + numbers. Formally, a digit is a character that has the property + value Numeric_Type=Digit or Numeric_Type=Decimal. + +str.isidentifier() + + Return "True" if the string is a valid identifier according to the + language definition, section Names (identifiers and keywords). + + "keyword.iskeyword()" can be used to test whether string "s" is a + reserved identifier, such as "def" and "class". + + Example: + + >>> from keyword import iskeyword + + >>> 'hello'.isidentifier(), iskeyword('hello') + (True, False) + >>> 'def'.isidentifier(), iskeyword('def') + (True, True) + +str.islower() + + Return "True" if all cased characters [4] in the string are + lowercase and there is at least one cased character, "False" + otherwise. + +str.isnumeric() + + Return "True" if all characters in the string are numeric + characters, and there is at least one character, "False" otherwise. + Numeric characters include digit characters, and all characters + that have the Unicode numeric value property, e.g. U+2155, VULGAR + FRACTION ONE FIFTH. Formally, numeric characters are those with + the property value Numeric_Type=Digit, Numeric_Type=Decimal or + Numeric_Type=Numeric. For example: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isdigit()". Numeric characters are a + superset of decimal numbers. + +str.isprintable() + + Return "True" if all characters in the string are printable, + "False" if it contains at least one non-printable character. + + Here “printable” means the character is suitable for "repr()" to + use in its output; “non-printable” means that "repr()" on built-in + types will hex-escape the character. It has no bearing on the + handling of strings written to "sys.stdout" or "sys.stderr". + + The printable characters are those which in the Unicode character + database (see "unicodedata") have a general category in group + Letter, Mark, Number, Punctuation, or Symbol (L, M, N, P, or S); + plus the ASCII space 0x20. Nonprintable characters are those in + group Separator or Other (Z or C), except the ASCII space. + + For example: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + +str.isspace() + + Return "True" if there are only whitespace characters in the string + and there is at least one character, "False" otherwise. + + A character is *whitespace* if in the Unicode character database + (see "unicodedata"), either its general category is "Zs" + (“Separator, space”), or its bidirectional class is one of "WS", + "B", or "S". + +str.istitle() + + Return "True" if the string is a titlecased string and there is at + least one character, for example uppercase characters may only + follow uncased characters and lowercase characters only cased ones. + Return "False" otherwise. + + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + +str.isupper() + + Return "True" if all cased characters [4] in the string are + uppercase and there is at least one cased character, "False" + otherwise. + + >>> 'BANANA'.isupper() + True + >>> 'banana'.isupper() + False + >>> 'baNana'.isupper() + False + >>> ' '.isupper() + False + +str.join(iterable, /) + + Return a string which is the concatenation of the strings in + *iterable*. A "TypeError" will be raised if there are any non- + string values in *iterable*, including "bytes" objects. The + separator between elements is the string providing this method. For + example: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also "split()". + +str.ljust(width, fillchar=' ', /) + + Return the string left justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + + For example: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also "rjust()". + +str.lower() + + Return a copy of the string with all the cased characters [4] + converted to lowercase. For example: + + >>> 'Lower Method Example'.lower() + 'lower method example' + + The lowercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.lstrip(chars=None, /) + + Return a copy of the string with leading characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a prefix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.lstrip() + 'spacious ' + >>> 'www.example.com'.lstrip('cmowz.') + 'example.com' + + See "str.removeprefix()" for a method that will remove a single + prefix string rather than all of a set of characters. For example: + + >>> 'Arthur: three!'.lstrip('Arthur: ') + 'ee!' + >>> 'Arthur: three!'.removeprefix('Arthur: ') + 'three!' + +static str.maketrans(dict, /) +static str.maketrans(from, to, remove='', /) + + This static method returns a translation table usable for + "str.translate()". + + If there is only one argument, it must be a dictionary mapping + Unicode ordinals (integers) or characters (strings of length 1) to + Unicode ordinals, strings (of arbitrary lengths) or "None". + Character keys will then be converted to ordinals. + + If there are two arguments, they must be strings of equal length, + and in the resulting dictionary, each character in *from* will be + mapped to the character at the same position in *to*. If there is + a third argument, it must be a string, whose characters will be + mapped to "None" in the result. + +str.partition(sep, /) + + Split the string at the first occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing the string itself, followed by + two empty strings. + +str.removeprefix(prefix, /) + + If the string starts with the *prefix* string, return + "string[len(prefix):]". Otherwise, return a copy of the original + string: + + >>> 'TestHook'.removeprefix('Test') + 'Hook' + >>> 'BaseTestCase'.removeprefix('Test') + 'BaseTestCase' + + Added in version 3.9. + + See also "removesuffix()" and "startswith()". + +str.removesuffix(suffix, /) + + If the string ends with the *suffix* string and that *suffix* is + not empty, return "string[:-len(suffix)]". Otherwise, return a copy + of the original string: + + >>> 'MiscTests'.removesuffix('Tests') + 'Misc' + >>> 'TmpDirMixin'.removesuffix('Tests') + 'TmpDirMixin' + + Added in version 3.9. + + See also "removeprefix()" and "endswith()". + +str.replace(old, new, /, count=-1) + + Return a copy of the string with all occurrences of substring *old* + replaced by *new*. If *count* is given, only the first *count* + occurrences are replaced. If *count* is not specified or "-1", then + all occurrences are replaced. For example: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' + + Changed in version 3.13: *count* is now supported as a keyword + argument. + +str.rfind(sub[, start[, end]]) + + Return the highest index in the string where substring *sub* is + found, such that *sub* is contained within "s[start:end]". + Optional arguments *start* and *end* are interpreted as in slice + notation. Return "-1" on failure. For example: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also "find()" and "rindex()". + +str.rindex(sub[, start[, end]]) + + Like "rfind()" but raises "ValueError" when the substring *sub* is + not found. For example: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "index()" and "find()". + +str.rjust(width, fillchar=' ', /) + + Return the string right justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + +str.rpartition(sep, /) + + Split the string at the last occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing two empty strings, followed by + the string itself. + + For example: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also "partition()". + +str.rsplit(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* splits + are done, the *rightmost* ones. If *sep* is not specified or + "None", any whitespace string is a separator. Except for splitting + from the right, "rsplit()" behaves like "split()" which is + described in detail below. + +str.rstrip(chars=None, /) + + Return a copy of the string with trailing characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a suffix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.rstrip() + ' spacious' + >>> 'mississippi'.rstrip('ipz') + 'mississ' + + See "str.removesuffix()" for a method that will remove a single + suffix string rather than all of a set of characters. For example: + + >>> 'Monty Python'.rstrip(' Python') + 'M' + >>> 'Monty Python'.removesuffix(' Python') + 'Monty' + +str.split(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* + splits are done (thus, the list will have at most "maxsplit+1" + elements). If *maxsplit* is not specified or "-1", then there is + no limit on the number of splits (all possible splits are made). + + If *sep* is given, consecutive delimiters are not grouped together + and are deemed to delimit empty strings (for example, + "'1,,2'.split(',')" returns "['1', '', '2']"). The *sep* argument + may consist of multiple characters as a single delimiter (to split + with multiple delimiters, use "re.split()"). Splitting an empty + string with a specified separator returns "['']". + + For example: + + >>> '1,2,3'.split(',') + ['1', '2', '3'] + >>> '1,2,3'.split(',', maxsplit=1) + ['1', '2,3'] + >>> '1,2,,3,'.split(',') + ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] + + If *sep* is not specified or is "None", a different splitting + algorithm is applied: runs of consecutive whitespace are regarded + as a single separator, and the result will contain no empty strings + at the start or end if the string has leading or trailing + whitespace. Consequently, splitting an empty string or a string + consisting of just whitespace with a "None" separator returns "[]". + + For example: + + >>> '1 2 3'.split() + ['1', '2', '3'] + >>> '1 2 3'.split(maxsplit=1) + ['1', '2 3'] + >>> ' 1 2 3 '.split() + ['1', '2', '3'] + + If *sep* is not specified or is "None" and *maxsplit* is "0", only + leading runs of consecutive whitespace are considered. + + For example: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + + See also "join()". + +str.splitlines(keepends=False) + + Return a list of the lines in the string, breaking at line + boundaries. Line breaks are not included in the resulting list + unless *keepends* is given and true. + + This method splits on the following line boundaries. In + particular, the boundaries are a superset of *universal newlines*. + + +-------------------------+-------------------------------+ + | Representation | Description | + |=========================|===============================| + | "\n" | Line Feed | + +-------------------------+-------------------------------+ + | "\r" | Carriage Return | + +-------------------------+-------------------------------+ + | "\r\n" | Carriage Return + Line Feed | + +-------------------------+-------------------------------+ + | "\v" or "\x0b" | Line Tabulation | + +-------------------------+-------------------------------+ + | "\f" or "\x0c" | Form Feed | + +-------------------------+-------------------------------+ + | "\x1c" | File Separator | + +-------------------------+-------------------------------+ + | "\x1d" | Group Separator | + +-------------------------+-------------------------------+ + | "\x1e" | Record Separator | + +-------------------------+-------------------------------+ + | "\x85" | Next Line (C1 Control Code) | + +-------------------------+-------------------------------+ + | "\u2028" | Line Separator | + +-------------------------+-------------------------------+ + | "\u2029" | Paragraph Separator | + +-------------------------+-------------------------------+ + + Changed in version 3.2: "\v" and "\f" added to list of line + boundaries. + + For example: + + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() + ['ab c', '', 'de fg', 'kl'] + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) + ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] + + Unlike "split()" when a delimiter string *sep* is given, this + method returns an empty list for the empty string, and a terminal + line break does not result in an extra line: + + >>> "".splitlines() + [] + >>> "One line\n".splitlines() + ['One line'] + + For comparison, "split('\n')" gives: + + >>> ''.split('\n') + [''] + >>> 'Two lines\n'.split('\n') + ['Two lines', ''] + +str.startswith(prefix[, start[, end]]) + + Return "True" if string starts with the *prefix*, otherwise return + "False". *prefix* can also be a tuple of prefixes to look for. + With optional *start*, test string beginning at that position. + With optional *end*, stop comparing string at that position. + +str.strip(chars=None, /) + + Return a copy of the string with the leading and trailing + characters removed. The *chars* argument is a string specifying the + set of characters to be removed. If omitted or "None", the *chars* + argument defaults to removing whitespace. The *chars* argument is + not a prefix or suffix; rather, all combinations of its values are + stripped: + + >>> ' spacious '.strip() + 'spacious' + >>> 'www.example.com'.strip('cmowz.') + 'example' + + The outermost leading and trailing *chars* argument values are + stripped from the string. Characters are removed from the leading + end until reaching a string character that is not contained in the + set of characters in *chars*. A similar action takes place on the + trailing end. For example: + + >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' + >>> comment_string.strip('.#! ') + 'Section 3.2.1 Issue #32' + +str.swapcase() + + Return a copy of the string with uppercase characters converted to + lowercase and vice versa. Note that it is not necessarily true that + "s.swapcase().swapcase() == s". + +str.title() + + Return a titlecased version of the string where words start with an + uppercase character and the remaining characters are lowercase. + + For example: + + >>> 'Hello world'.title() + 'Hello World' + + The algorithm uses a simple language-independent definition of a + word as groups of consecutive letters. The definition works in + many contexts but it means that apostrophes in contractions and + possessives form word boundaries, which may not be the desired + result: + + >>> "they're bill's friends from the UK".title() + "They'Re Bill'S Friends From The Uk" + + The "string.capwords()" function does not have this problem, as it + splits words on spaces only. + + Alternatively, a workaround for apostrophes can be constructed + using regular expressions: + + >>> import re + >>> def titlecase(s): + ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", + ... lambda mo: mo.group(0).capitalize(), + ... s) + ... + >>> titlecase("they're bill's friends.") + "They're Bill's Friends." + + See also "istitle()". + +str.translate(table, /) + + Return a copy of the string in which each character has been mapped + through the given translation table. The table must be an object + that implements indexing via "__getitem__()", typically a *mapping* + or *sequence*. When indexed by a Unicode ordinal (an integer), the + table object can do any of the following: return a Unicode ordinal + or a string, to map the character to one or more other characters; + return "None", to delete the character from the return string; or + raise a "LookupError" exception, to map the character to itself. + + You can use "str.maketrans()" to create a translation map from + character-to-character mappings in different formats. + + See also the "codecs" module for a more flexible approach to custom + character mappings. + +str.upper() + + Return a copy of the string with all the cased characters [4] + converted to uppercase. Note that "s.upper().isupper()" might be + "False" if "s" contains uncased characters or if the Unicode + category of the resulting character(s) is not “Lu” (Letter, + uppercase), but e.g. “Lt” (Letter, titlecase). + + The uppercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.zfill(width, /) + + Return a copy of the string left filled with ASCII "'0'" digits to + make a string of length *width*. A leading sign prefix + ("'+'"/"'-'") is handled by inserting the padding *after* the sign + character rather than before. The original string is returned if + *width* is less than or equal to "len(s)". + + For example: + + >>> "42".zfill(5) + '00042' + >>> "-42".zfill(5) + '-0042' +''', + 'strings': '''String and Bytes literals +************************* + +String literals are text enclosed in single quotes ("'") or double +quotes ("""). For example: + + "spam" + 'eggs' + +The quote used to start the literal also terminates it, so a string +literal can only contain the other quote (except with escape +sequences, see below). For example: + + 'Say "Hello", please.' + "Don't do that!" + +Except for this limitation, the choice of quote character ("'" or """) +does not affect how the literal is parsed. + +Inside a string literal, the backslash ("\\") character introduces an +*escape sequence*, which has special meaning depending on the +character after the backslash. For example, "\\"" denotes the double +quote character, and does *not* end the string: + + >>> print("Say \\"Hello\\" to everyone!") + Say "Hello" to everyone! + +See escape sequences below for a full list of such sequences, and more +details. + + +Triple-quoted strings +===================== + +Strings can also be enclosed in matching groups of three single or +double quotes. These are generally referred to as *triple-quoted +strings*: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the +literal, if they are of the same kind ("'" or """) used at the start: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained: + + \'\'\'This triple-quoted string + continues on the next line.\'\'\' + + +String prefixes +=============== + +String literals can have an optional *prefix* that influences how the +content of the literal is parsed, for example: + + b"data" + f'{result=}' + +The allowed prefixes are: + +* "b": Bytes literal + +* "r": Raw string + +* "f": Formatted string literal (“f-string”) + +* "t": Template string literal (“t-string”) + +* "u": No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, ‘"B"’ works the same as +‘"b"’). The ‘"r"’ prefix can be combined with ‘"f"’, ‘"t"’ or ‘"b"’, +so ‘"fr"’, ‘"rf"’, ‘"tr"’, ‘"rt"’, ‘"br"’, and ‘"rb"’ are also valid +prefixes. + +Added in version 3.3: The "'rb'" prefix of raw bytes literals has been +added as a synonym of "'br'".Support for the unicode legacy literal +("u'value'") was reintroduced to simplify the maintenance of dual +Python 2.x and 3.x codebases. See **PEP 414** for more information. + + +Formal grammar +============== + +String literals, except “f-strings” and “t-strings”, are described by +the following lexical definitions. + +These definitions use negative lookaheads ("!") to indicate that an +ending quote ends the literal. + + STRING: [stringprefix] (stringcontent) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "\'\'\'" ( !"\'\'\'" longstringitem)* "\'\'\'" + | '"""' ( !'"""' longstringitem)* '"""' + | "'" ( !"'" stringitem)* "'" + | '"' ( !'"' stringitem)* '"' + stringitem: stringchar | stringescapeseq + stringchar: + longstringitem: stringitem | newline + stringescapeseq: "\\" + +Note that as in all lexical definitions, whitespace is significant. In +particular, the prefix (if any) must be immediately followed by the +starting quote. + + +Escape sequences +================ + +Unless an ‘"r"’ or ‘"R"’ prefix is present, escape sequences in string +and bytes literals are interpreted according to rules similar to those +used by Standard C. The recognized escape sequences are: + ++----------------------------------------------------+----------------------------------------------------+ +| Escape Sequence | Meaning | +|====================================================|====================================================| +| "\\" | Ignored end of line | ++----------------------------------------------------+----------------------------------------------------+ +| "\\\\" | Backslash | ++----------------------------------------------------+----------------------------------------------------+ +| "\\'" | Single quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\"" | Double quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\a" | ASCII Bell (BEL) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\b" | ASCII Backspace (BS) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\f" | ASCII Formfeed (FF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\n" | ASCII Linefeed (LF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\r" | ASCII Carriage Return (CR) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\t" | ASCII Horizontal Tab (TAB) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\v" | ASCII Vertical Tab (VT) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\*ooo*" | Octal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\x*hh*" | Hexadecimal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\N{*name*}" | Named Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\u*xxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\U*xxxxxxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ + + +Ignored end of line +------------------- + +A backslash can be added at the end of a line to ignore the newline: + + >>> 'This string will not include \\ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' + +The same result can be achieved using triple-quoted strings, or +parentheses and string literal concatenation. + + +Escaped characters +------------------ + +To include a backslash in a non-raw Python string literal, it must be +doubled. The "\\\\" escape sequence denotes a single backslash +character: + + >>> print('C:\\\\Program Files') + C:\\Program Files + +Similarly, the "\\'" and "\\"" sequences denote the single and double +quote character, respectively: + + >>> print('\\' and \\"') + ' and " + + +Octal character +--------------- + +The sequence "\\*ooo*" denotes a *character* with the octal (base 8) +value *ooo*: + + >>> '\\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + +Changed in version 3.11: Octal escapes with value larger than "0o377" +(255) produce a "DeprecationWarning". + +Changed in version 3.12: Octal escapes with value larger than "0o377" +(255) produce a "SyntaxWarning". In a future Python version they will +raise a "SyntaxError". + + +Hexadecimal character +--------------------- + +The sequence "\\x*hh*" denotes a *character* with the hex (base 16) +value *hh*: + + >>> '\\x50' + 'P' + +Unlike in Standard C, exactly two hex digits are required. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + + +Named Unicode character +----------------------- + +The sequence "\\N{*name*}" denotes a Unicode character with the given +*name*: + + >>> '\\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\\N{SNAKE}' + '🐍' + +This sequence cannot appear in bytes literals. + +Changed in version 3.3: Support for name aliases has been added. + + +Hexadecimal Unicode characters +------------------------------ + +These sequences "\\u*xxxx*" and "\\U*xxxxxxxx*" denote the Unicode +character with the given hex (base 16) value. Exactly four digits are +required for "\\u"; exactly eight digits are required for "\\U". The +latter can encode any Unicode character. + + >>> '\\u1234' + 'ሴ' + >>> '\\U0001f40d' + '🐍' + +These sequences cannot appear in bytes literals. + + +Unrecognized escape sequences +----------------------------- + +Unlike in Standard C, all unrecognized escape sequences are left in +the string unchanged, that is, *the backslash is left in the result*: + + >>> print('\\q') + \\q + >>> list('\\q') + ['\\\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in +string literals ("\\N...", "\\u...", "\\U...") fall into the category of +unrecognized escapes. + +Changed in version 3.6: Unrecognized escape sequences produce a +"DeprecationWarning". + +Changed in version 3.12: Unrecognized escape sequences produce a +"SyntaxWarning". In a future Python version they will raise a +"SyntaxError". + + +Bytes literals +============== + +*Bytes literals* are always prefixed with ‘"b"’ or ‘"B"’; they produce +an instance of the "bytes" type instead of the "str" type. They may +only contain ASCII characters; bytes with a numeric value of 128 or +greater must be expressed with escape sequences (typically Hexadecimal +character or Octal character): + + >>> b'\\x89PNG\\r\\n\\x1a\\n' + b'\\x89PNG\\r\\n\\x1a\\n' + >>> list(b'\\x89PNG\\r\\n\\x1a\\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence +(typically "\\0" or "\\x00"). + + +Raw string literals +=================== + +Both string and bytes literals may optionally be prefixed with a +letter ‘"r"’ or ‘"R"’; such constructs are called *raw string +literals* and *raw bytes literals* respectively and treat backslashes +as literal characters. As a result, in raw string literals, escape +sequences are not treated specially: + + >>> r'\\d{4}-\\d{2}-\\d{2}' + '\\\\d{4}-\\\\d{2}-\\\\d{2}' + +Even in a raw literal, quotes can be escaped with a backslash, but the +backslash remains in the result; for example, "r"\\""" is a valid +string literal consisting of two characters: a backslash and a double +quote; "r"\\"" is not a valid string literal (even a raw string cannot +end in an odd number of backslashes). Specifically, *a raw literal +cannot end in a single backslash* (since the backslash would escape +the following quote character). Note also that a single backslash +followed by a newline is interpreted as those two characters as part +of the literal, *not* as a line continuation. + + +f-strings +========= + +Added in version 3.6. + +Changed in version 3.7: The "await" and "async for" can be used in +expressions within f-strings. + +Changed in version 3.8: Added the debug specifier ("=") + +Changed in version 3.12: Many restrictions on expressions within +f-strings have been removed. Notably, nested strings, comments, and +backslashes are now permitted. + +A *formatted string literal* or *f-string* is a string literal that is +prefixed with ‘"f"’ or ‘"F"’. Unlike other string literals, f-strings +do not have a constant value. They may contain *replacement fields* +delimited by curly braces "{}". Replacement fields contain expressions +which are evaluated at run time. For example: + + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' + +Any doubled curly braces ("{{" or "}}") outside replacement fields are +replaced with the corresponding single curly brace: + + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in +ordinary string literals. This means that escape sequences are decoded +(except when a literal is also marked as a raw string), and newlines +are possible in triple-quoted f-strings: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\\Users\\{name}") + C:\\Users\\Galahad + >>> print(f\'\'\'Three shall be the number of the counting + ... and the number of the counting shall be three.\'\'\') + Three shall be the number of the counting + and the number of the counting shall be three. + +Expressions in formatted string literals are treated like regular +Python expressions. Each expression is evaluated in the context where +the formatted string literal appears, in order from left to right. An +empty expression is not allowed, and both "lambda" and assignment +expressions ":=" must be surrounded by explicit parentheses: + + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' + +Reusing the outer f-string quoting type inside a replacement field is +permitted: + + >>> a = dict(x=2) + >>> f"abc {a["x"]} def" + 'abc 2 def' + +Backslashes are also allowed in replacement fields and are evaluated +the same way as in any other context: + + >>> a = ["a", "b", "c"] + >>> print(f"List a contains:\\n{"\\n".join(a)}") + List a contains: + a + b + c + +It is possible to nest f-strings: + + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' + +Portable Python programs should not use more than 5 levels of nesting. + +**CPython implementation detail:** CPython does not limit nesting of +f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. Everything that +comes after a "#" inside a replacement field is a comment (even +closing braces and quotes). This means that replacement fields with +comments must be closed in a different line: + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* – an equal sign ("="), optionally surrounded by + whitespace on one or both sides; + +* a *conversion specifier* – "!s", "!r" or "!a"; and/or + +* a *format specifier* prefixed with a colon (":"). + +See the Standard Library section on f-strings for details on how these +fields are evaluated. + +As that section explains, *format specifiers* are passed as the second +argument to the "format()" function to format a replacement field +value. For example, they can be used to specify a field width and +padding characters using the Format Specification Mini-Language: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and format +specifiers: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested +replacement fields. + +Formatted string literals cannot be used as *docstrings*, even if they +do not include expressions: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> print(foo.__doc__) + None + +See also: + + * **PEP 498** – Literal String Interpolation + + * **PEP 701** – Syntactic formalization of f-strings + + * "str.format()", which uses a related format string mechanism. + + +t-strings +========= + +Added in version 3.14. + +A *template string literal* or *t-string* is a string literal that is +prefixed with ‘"t"’ or ‘"T"’. These strings follow the same syntax +rules as formatted string literals. For differences in evaluation +rules, see the Standard Library section on t-strings + + +Formal grammar for f-strings +============================ + +F-strings are handled partly by the *lexical analyzer*, which produces +the tokens "FSTRING_START", "FSTRING_MIDDLE" and "FSTRING_END", and +partly by the parser, which handles expressions in the replacement +field. The exact way the work is split is a CPython implementation +detail. + +Correspondingly, the f-string grammar is a mix of lexical and +syntactic definitions. + +Whitespace is significant in these situations: + +* There may be no whitespace in "FSTRING_START" (between the prefix + and quote). + +* Whitespace in "FSTRING_MIDDLE" is part of the literal string + contents. + +* In "fstring_replacement_field", if "f_debug_specifier" is present, + all whitespace after the opening brace until the + "f_debug_specifier", as well as whitespace immediately following + "f_debug_specifier", is retained as part of the expression. + + **CPython implementation detail:** The expression is not handled in + the tokenization phase; it is retrieved from the source code using + locations of the "{" token and the token after "=". + +The "FSTRING_MIDDLE" definition uses negative lookaheads ("!") to +indicate special characters (backslash, newline, "{", "}") and +sequences ("f_quote"). + + fstring: FSTRING_START fstring_middle* FSTRING_END + + FSTRING_START: fstringprefix ("'" | '"' | "\'\'\'" | '"""') + FSTRING_END: f_quote + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | fstring_replacement_field + | FSTRING_MIDDLE + FSTRING_MIDDLE: + | (!"\\" !newline !'{' !'}' !f_quote) source_character + | stringescapeseq + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' f_expression [f_debug_specifier] [fstring_conversion] + [fstring_full_format_spec] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' fstring_format_spec* + fstring_format_spec: + | FSTRING_MIDDLE + | fstring_replacement_field + f_expression: + | ','.(conditional_expression | "*" or_expr)+ [","] + | yield_expression + +Note: + + In the above grammar snippet, the "f_quote" and "FSTRING_MIDDLE" + rules are context-sensitive – they depend on the contents of + "FSTRING_START" of the nearest enclosing "fstring".Constructing a + more traditional formal grammar from this template is left as an + exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with +*t* instead of *f* at the beginning of rule and token names and in the +prefix. + + tstring: TSTRING_START tstring_middle* TSTRING_END + + +''', + 'subscriptions': r'''Subscriptions +************* + +The subscription of an instance of a container class will generally +select an element from the container. The subscription of a *generic +class* will generally return a GenericAlias object. + + subscription: primary "[" flexible_expression_list "]" + +When an object is subscripted, the interpreter will evaluate the +primary and the expression list. + +The primary must evaluate to an object that supports subscription. An +object may support subscription through defining one or both of +"__getitem__()" and "__class_getitem__()". When the primary is +subscripted, the evaluated result of the expression list will be +passed to one of these methods. For more details on when +"__class_getitem__" is called instead of "__getitem__", see +__class_getitem__ versus __getitem__. + +If the expression list contains at least one comma, or if any of the +expressions are starred, the expression list will evaluate to a +"tuple" containing the items of the expression list. Otherwise, the +expression list will evaluate to the value of the list’s sole member. + +Changed in version 3.11: Expressions in an expression list may be +starred. See **PEP 646**. + +For built-in objects, there are two types of objects that support +subscription via "__getitem__()": + +1. Mappings. If the primary is a *mapping*, the expression list must + evaluate to an object whose value is one of the keys of the + mapping, and the subscription selects the value in the mapping that + corresponds to that key. An example of a builtin mapping class is + the "dict" class. + +2. Sequences. If the primary is a *sequence*, the expression list must + evaluate to an "int" or a "slice" (as discussed in the following + section). Examples of builtin sequence classes include the "str", + "list" and "tuple" classes. + +The formal syntax makes no special provision for negative indices in +*sequences*. However, built-in sequences all provide a "__getitem__()" +method that interprets negative indices by adding the length of the +sequence to the index so that, for example, "x[-1]" selects the last +item of "x". The resulting value must be a nonnegative integer less +than the number of items in the sequence, and the subscription selects +the item whose index is that value (counting from zero). Since the +support for negative indices and slicing occurs in the object’s +"__getitem__()" method, subclasses overriding this method will need to +explicitly add that support. + +A "string" is a special kind of sequence whose items are *characters*. +A character is not a separate data type but a string of exactly one +character. +''', + 'truth': r'''Truth Value Testing +******************* + +Any object can be tested for truth value, for use in an "if" or +"while" condition or as operand of the Boolean operations below. + +By default, an object is considered true unless its class defines +either a "__bool__()" method that returns "False" or a "__len__()" +method that returns zero, when called with the object. [1] If one of +the methods raises an exception when called, the exception is +propagated and the object does not have a truth value (for example, +"NotImplemented"). Here are most of the built-in objects considered +false: + +* constants defined to be false: "None" and "False" + +* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)", + "Fraction(0, 1)" + +* empty sequences and collections: "''", "()", "[]", "{}", "set()", + "range(0)" + +Operations and built-in functions that have a Boolean result always +return "0" or "False" for false and "1" or "True" for true, unless +otherwise stated. (Important exception: the Boolean operations "or" +and "and" always return one of their operands.) +''', + 'try': r'''The "try" statement +******************* + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt: try1_stmt | try2_stmt | try3_stmt + try1_stmt: "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt: "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt: "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + +Changed in version 3.14: Support for optionally dropping grouping +parentheses when using multiple exception types. See **PEP 758**. + + +"except" clause +=============== + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. +Parentheses can be dropped if multiple exception types are provided +and the "as" clause is not used. The raised exception matches an +"except" clause whose expression evaluates to the class or a *non- +virtual base class* of the exception object, or to a tuple that +contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +================ + +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError(),)) + +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +============= + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +================ + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded. For example, this function returns 42. + + def f(): + try: + 1/0 + finally: + return 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed. The following function returns ‘finally’. + + def foo(): + try: + return 'try' + finally: + return 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. + +Changed in version 3.14: The compiler emits a "SyntaxWarning" when a +"return", "break" or "continue" appears in a "finally" block (see +**PEP 765**). +''', + 'types': r'''The standard type hierarchy +*************************** + +Below is a list of the types that are built into Python. Extension +modules (written in C, Java, or other languages, depending on the +implementation) can define additional types. Future versions of +Python may add types to the type hierarchy (e.g., rational numbers, +efficiently stored arrays of integers, etc.), although such additions +will often be provided via the standard library instead. + +Some of the type descriptions below contain a paragraph listing +‘special attributes.’ These are attributes that provide access to the +implementation and are not intended for general use. Their definition +may change in the future. + + +None +==== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name "None". It is +used to signify the absence of a value in many situations, e.g., it is +returned from functions that don’t explicitly return anything. Its +truth value is false. + + +NotImplemented +============== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name +"NotImplemented". Numeric methods and rich comparison methods should +return this value if they do not implement the operation for the +operands provided. (The interpreter will then try the reflected +operation, or some other fallback, depending on the operator.) It +should not be evaluated in a boolean context. + +See Implementing the arithmetic operations for more details. + +Changed in version 3.9: Evaluating "NotImplemented" in a boolean +context was deprecated. + +Changed in version 3.14: Evaluating "NotImplemented" in a boolean +context now raises a "TypeError". It previously evaluated to "True" +and emitted a "DeprecationWarning" since Python 3.9. + + +Ellipsis +======== + +This type has a single value. There is a single object with this +value. This object is accessed through the literal "..." or the built- +in name "Ellipsis". Its truth value is true. + + +"numbers.Number" +================ + +These are created by numeric literals and returned as results by +arithmetic operators and arithmetic built-in functions. Numeric +objects are immutable; once created their value never changes. Python +numbers are of course strongly related to mathematical numbers, but +subject to the limitations of numerical representation in computers. + +The string representations of the numeric classes, computed by +"__repr__()" and "__str__()", have the following properties: + +* They are valid numeric literals which, when passed to their class + constructor, produce an object having the value of the original + numeric. + +* The representation is in base 10, when possible. + +* Leading zeros, possibly excepting a single zero before a decimal + point, are not shown. + +* Trailing zeros, possibly excepting a single zero after a decimal + point, are not shown. + +* A sign is shown only when the number is negative. + +Python distinguishes between integers, floating-point numbers, and +complex numbers: + + +"numbers.Integral" +------------------ + +These represent elements from the mathematical set of integers +(positive and negative). + +Note: + + The rules for integer representation are intended to give the most + meaningful interpretation of shift and mask operations involving + negative integers. + +There are two types of integers: + +Integers ("int") + These represent numbers in an unlimited range, subject to available + (virtual) memory only. For the purpose of shift and mask + operations, a binary representation is assumed, and negative + numbers are represented in a variant of 2’s complement which gives + the illusion of an infinite string of sign bits extending to the + left. + +Booleans ("bool") + These represent the truth values False and True. The two objects + representing the values "False" and "True" are the only Boolean + objects. The Boolean type is a subtype of the integer type, and + Boolean values behave like the values 0 and 1, respectively, in + almost all contexts, the exception being that when converted to a + string, the strings ""False"" or ""True"" are returned, + respectively. + + +"numbers.Real" ("float") +------------------------ + +These represent machine-level double precision floating-point numbers. +You are at the mercy of the underlying machine architecture (and C or +Java implementation) for the accepted range and handling of overflow. +Python does not support single-precision floating-point numbers; the +savings in processor and memory usage that are usually the reason for +using these are dwarfed by the overhead of using objects in Python, so +there is no reason to complicate the language with two kinds of +floating-point numbers. + + +"numbers.Complex" ("complex") +----------------------------- + +These represent complex numbers as a pair of machine-level double +precision floating-point numbers. The same caveats apply as for +floating-point numbers. The real and imaginary parts of a complex +number "z" can be retrieved through the read-only attributes "z.real" +and "z.imag". + + +Sequences +========= + +These represent finite ordered sets indexed by non-negative numbers. +The built-in function "len()" returns the number of items of a +sequence. When the length of a sequence is *n*, the index set contains +the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected by +"a[i]". Some sequences, including built-in sequences, interpret +negative subscripts by adding the sequence length. For example, +"a[-2]" equals "a[n-2]", the second to last item of sequence a with +length "n". + +Sequences also support slicing: "a[i:j]" selects all items with index +*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a +slice is a sequence of the same type. The comment above about negative +indexes also applies to negative slice positions. + +Some sequences also support “extended slicing” with a third “step” +parameter: "a[i:j:k]" selects all items of *a* with index *x* where "x += i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*. + +Sequences are distinguished according to their mutability: + + +Immutable sequences +------------------- + +An object of an immutable sequence type cannot change once it is +created. (If the object contains references to other objects, these +other objects may be mutable and may be changed; however, the +collection of objects directly referenced by an immutable object +cannot change.) + +The following types are immutable sequences: + +Strings + A string is a sequence of values that represent Unicode code + points. All the code points in the range "U+0000 - U+10FFFF" can be + represented in a string. Python doesn’t have a char type; instead, + every code point in the string is represented as a string object + with length "1". The built-in function "ord()" converts a code + point from its string form to an integer in the range "0 - 10FFFF"; + "chr()" converts an integer in the range "0 - 10FFFF" to the + corresponding length "1" string object. "str.encode()" can be used + to convert a "str" to "bytes" using the given text encoding, and + "bytes.decode()" can be used to achieve the opposite. + +Tuples + The items of a tuple are arbitrary Python objects. Tuples of two or + more items are formed by comma-separated lists of expressions. A + tuple of one item (a ‘singleton’) can be formed by affixing a comma + to an expression (an expression by itself does not create a tuple, + since parentheses must be usable for grouping of expressions). An + empty tuple can be formed by an empty pair of parentheses. + +Bytes + A bytes object is an immutable array. The items are 8-bit bytes, + represented by integers in the range 0 <= x < 256. Bytes literals + (like "b'abc'") and the built-in "bytes()" constructor can be used + to create bytes objects. Also, bytes objects can be decoded to + strings via the "decode()" method. + + +Mutable sequences +----------------- + +Mutable sequences can be changed after they are created. The +subscription and slicing notations can be used as the target of +assignment and "del" (delete) statements. + +Note: + + The "collections" and "array" module provide additional examples of + mutable sequence types. + +There are currently two intrinsic mutable sequence types: + +Lists + The items of a list are arbitrary Python objects. Lists are formed + by placing a comma-separated list of expressions in square + brackets. (Note that there are no special cases needed to form + lists of length 0 or 1.) + +Byte Arrays + A bytearray object is a mutable array. They are created by the + built-in "bytearray()" constructor. Aside from being mutable (and + hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable "bytes" objects. + + +Set types +========= + +These represent unordered, finite sets of unique, immutable objects. +As such, they cannot be indexed by any subscript. However, they can be +iterated over, and the built-in function "len()" returns the number of +items in a set. Common uses for sets are fast membership testing, +removing duplicates from a sequence, and computing mathematical +operations such as intersection, union, difference, and symmetric +difference. + +For set elements, the same immutability rules apply as for dictionary +keys. Note that numeric types obey the normal rules for numeric +comparison: if two numbers compare equal (e.g., "1" and "1.0"), only +one of them can be contained in a set. + +There are currently two intrinsic set types: + +Sets + These represent a mutable set. They are created by the built-in + "set()" constructor and can be modified afterwards by several + methods, such as "add()". + +Frozen sets + These represent an immutable set. They are created by the built-in + "frozenset()" constructor. As a frozenset is immutable and + *hashable*, it can be used again as an element of another set, or + as a dictionary key. + + +Mappings +======== + +These represent finite sets of objects indexed by arbitrary index +sets. The subscript notation "a[k]" selects the item indexed by "k" +from the mapping "a"; this can be used in expressions and as the +target of assignments or "del" statements. The built-in function +"len()" returns the number of items in a mapping. + +There is currently a single intrinsic mapping type: + + +Dictionaries +------------ + +These represent finite sets of objects indexed by nearly arbitrary +values. The only types of values not acceptable as keys are values +containing lists or dictionaries or other mutable types that are +compared by value rather than by object identity, the reason being +that the efficient implementation of dictionaries requires a key’s +hash value to remain constant. Numeric types used for keys obey the +normal rules for numeric comparison: if two numbers compare equal +(e.g., "1" and "1.0") then they can be used interchangeably to index +the same dictionary entry. + +Dictionaries preserve insertion order, meaning that keys will be +produced in the same order they were added sequentially over the +dictionary. Replacing an existing key does not change the order, +however removing a key and re-inserting it will add it to the end +instead of keeping its old place. + +Dictionaries are mutable; they can be created by the "{}" notation +(see section Dictionary displays). + +The extension modules "dbm.ndbm" and "dbm.gnu" provide additional +examples of mapping types, as does the "collections" module. + +Changed in version 3.7: Dictionaries did not preserve insertion order +in versions of Python before 3.6. In CPython 3.6, insertion order was +preserved, but it was considered an implementation detail at that time +rather than a language guarantee. + + +Callable types +============== + +These are the types to which the function call operation (see section +Calls) can be applied: + + +User-defined functions +---------------------- + +A user-defined function object is created by a function definition +(see section Function definitions). It should be called with an +argument list containing the same number of items as the function’s +formal parameter list. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__builtins__ | A reference to the "dictionary" that holds the | +| | function’s builtins namespace. Added in version | +| | 3.10. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__globals__ | A reference to the "dictionary" that holds the | +| | function’s global variables – the global namespace | +| | of the module in which the function was defined. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__closure__ | "None" or a "tuple" of cells that contain bindings | +| | for the names specified in the "co_freevars" | +| | attribute of the function’s "code object". A cell | +| | object has the attribute "cell_contents". This can | +| | be used to get the value of the cell, as well as | +| | set the value. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most of these attributes check the type of the assigned value: + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__doc__ | The function’s documentation string, or "None" if | +| | unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__name__ | The function’s name. See also: "__name__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__qualname__ | The function’s *qualified name*. See also: | +| | "__qualname__ attributes". Added in version 3.3. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__module__ | The name of the module the function was defined | +| | in, or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__defaults__ | A "tuple" containing default *parameter* values | +| | for those parameters that have defaults, or "None" | +| | if no parameters have a default value. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__code__ | The code object representing the compiled function | +| | body. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__dict__ | The namespace supporting arbitrary function | +| | attributes. See also: "__dict__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotations__ | A "dictionary" containing annotations of | +| | *parameters*. The keys of the dictionary are the | +| | parameter names, and "'return'" for the return | +| | annotation, if provided. See also: | +| | "object.__annotations__". Changed in version | +| | 3.14: Annotations are now lazily evaluated. See | +| | **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotate__ | The *annotate function* for this function, or | +| | "None" if the function has no annotations. See | +| | "object.__annotate__". Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__kwdefaults__ | A "dictionary" containing defaults for keyword- | +| | only *parameters*. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__type_params__ | A "tuple" containing the type parameters of a | +| | generic function. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ + +Function objects also support getting and setting arbitrary +attributes, which can be used, for example, to attach metadata to +functions. Regular attribute dot-notation is used to get and set such +attributes. + +**CPython implementation detail:** CPython’s current implementation +only supports function attributes on user-defined functions. Function +attributes on built-in functions may be supported in the future. + +Additional information about a function’s definition can be retrieved +from its code object (accessible via the "__code__" attribute). + + +Instance methods +---------------- + +An instance method object combines a class, a class instance and any +callable object (normally a user-defined function). + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| method.__self__ | Refers to the class instance object to which the | +| | method is bound | ++----------------------------------------------------+----------------------------------------------------+ +| method.__func__ | Refers to the original function object | ++----------------------------------------------------+----------------------------------------------------+ +| method.__doc__ | The method’s documentation (same as | +| | "method.__func__.__doc__"). A "string" if the | +| | original function had a docstring, else "None". | ++----------------------------------------------------+----------------------------------------------------+ +| method.__name__ | The name of the method (same as | +| | "method.__func__.__name__") | ++----------------------------------------------------+----------------------------------------------------+ +| method.__module__ | The name of the module the method was defined in, | +| | or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ + +Methods also support accessing (but not setting) the arbitrary +function attributes on the underlying function object. + +User-defined method objects may be created when getting an attribute +of a class (perhaps via an instance of that class), if that attribute +is a user-defined function object or a "classmethod" object. + +When an instance method object is created by retrieving a user-defined +function object from a class via one of its instances, its "__self__" +attribute is the instance, and the method object is said to be +*bound*. The new method’s "__func__" attribute is the original +function object. + +When an instance method object is created by retrieving a +"classmethod" object from a class or instance, its "__self__" +attribute is the class itself, and its "__func__" attribute is the +function object underlying the class method. + +When an instance method object is called, the underlying function +("__func__") is called, inserting the class instance ("__self__") in +front of the argument list. For instance, when "C" is a class which +contains a definition for a function "f()", and "x" is an instance of +"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)". + +When an instance method object is derived from a "classmethod" object, +the “class instance” stored in "__self__" will actually be the class +itself, so that calling either "x.f(1)" or "C.f(1)" is equivalent to +calling "f(C,1)" where "f" is the underlying function. + +It is important to note that user-defined functions which are +attributes of a class instance are not converted to bound methods; +this *only* happens when the function is an attribute of the class. + + +Generator functions +------------------- + +A function or method which uses the "yield" statement (see section The +yield statement) is called a *generator function*. Such a function, +when called, always returns an *iterator* object which can be used to +execute the body of the function: calling the iterator’s +"iterator.__next__()" method will cause the function to execute until +it provides a value using the "yield" statement. When the function +executes a "return" statement or falls off the end, a "StopIteration" +exception is raised and the iterator will have reached the end of the +set of values to be returned. + + +Coroutine functions +------------------- + +A function or method which is defined using "async def" is called a +*coroutine function*. Such a function, when called, returns a +*coroutine* object. It may contain "await" expressions, as well as +"async with" and "async for" statements. See also the Coroutine +Objects section. + + +Asynchronous generator functions +-------------------------------- + +A function or method which is defined using "async def" and which uses +the "yield" statement is called a *asynchronous generator function*. +Such a function, when called, returns an *asynchronous iterator* +object which can be used in an "async for" statement to execute the +body of the function. + +Calling the asynchronous iterator’s "aiterator.__anext__" method will +return an *awaitable* which when awaited will execute until it +provides a value using the "yield" expression. When the function +executes an empty "return" statement or falls off the end, a +"StopAsyncIteration" exception is raised and the asynchronous iterator +will have reached the end of the set of values to be yielded. + + +Built-in functions +------------------ + +A built-in function object is a wrapper around a C function. Examples +of built-in functions are "len()" and "math.sin()" ("math" is a +standard built-in module). The number and type of the arguments are +determined by the C function. Special read-only attributes: + +* "__doc__" is the function’s documentation string, or "None" if + unavailable. See "function.__doc__". + +* "__name__" is the function’s name. See "function.__name__". + +* "__self__" is set to "None" (but see the next item). + +* "__module__" is the name of the module the function was defined in + or "None" if unavailable. See "function.__module__". + + +Built-in methods +---------------- + +This is really a different disguise of a built-in function, this time +containing an object passed to the C function as an implicit extra +argument. An example of a built-in method is "alist.append()", +assuming *alist* is a list object. In this case, the special read-only +attribute "__self__" is set to the object denoted by *alist*. (The +attribute has the same semantics as it does with "other instance +methods".) + + +Classes +------- + +Classes are callable. These objects normally act as factories for new +instances of themselves, but variations are possible for class types +that override "__new__()". The arguments of the call are passed to +"__new__()" and, in the typical case, to "__init__()" to initialize +the new instance. + + +Class Instances +--------------- + +Instances of arbitrary classes can be made callable by defining a +"__call__()" method in their class. + + +Modules +======= + +Modules are a basic organizational unit of Python code, and are +created by the import system as invoked either by the "import" +statement, or by calling functions such as "importlib.import_module()" +and built-in "__import__()". A module object has a namespace +implemented by a "dictionary" object (this is the dictionary +referenced by the "__globals__" attribute of functions defined in the +module). Attribute references are translated to lookups in this +dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A module +object does not contain the code object used to initialize the module +(since it isn’t needed once the initialization is done). + +Attribute assignment updates the module’s namespace dictionary, e.g., +"m.x = 1" is equivalent to "m.__dict__["x"] = 1". + + +Import-related attributes on module objects +------------------------------------------- + +Module objects have the following attributes that relate to the import +system. When a module is created using the machinery associated with +the import system, these attributes are filled in based on the +module’s *spec*, before the *loader* executes and loads the module. + +To create a module dynamically rather than using the import system, +it’s recommended to use "importlib.util.module_from_spec()", which +will set the various import-controlled attributes to appropriate +values. It’s also possible to use the "types.ModuleType" constructor +to create modules directly, but this technique is more error-prone, as +most attributes must be manually set on the module object after it has +been created when using this approach. + +Caution: + + With the exception of "__name__", it is **strongly** recommended + that you rely on "__spec__" and its attributes instead of any of the + other individual attributes listed in this subsection. Note that + updating an attribute on "__spec__" will not update the + corresponding attribute on the module itself: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ""__main__"". + + This attribute must be set to the fully qualified name of the + module. It is expected to match the value of + "module.__spec__.name". + +module.__spec__ + + A record of the module’s import-system-related state. + + Set to the "module spec" that was used when importing the module. + See Module specs for more details. + + Added in version 3.4. + +module.__package__ + + The *package* a module belongs to. + + If the module is top-level (that is, not a part of any specific + package) then the attribute should be set to "''" (the empty + string). Otherwise, it should be set to the name of the module’s + package (which can be equal to "module.__name__" if the module + itself is a package). See **PEP 366** for further details. + + This attribute is used instead of "__name__" to calculate explicit + relative imports for main modules. It defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor; use "importlib.util.module_from_spec()" instead to + ensure the attribute is set to a "str". + + It is **strongly** recommended that you use + "module.__spec__.parent" instead of "module.__package__". + "__package__" is now only used as a fallback if "__spec__.parent" + is not set, and this fallback path is deprecated. + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Changed in version 3.6: The value of "__package__" is expected to + be the same as "__spec__.parent". "__package__" is now only used as + a fallback during import resolution if "__spec__.parent" is not + defined. + + Changed in version 3.10: "ImportWarning" is raised if an import + resolution falls back to "__package__" instead of + "__spec__.parent". + + Changed in version 3.12: Raise "DeprecationWarning" instead of + "ImportWarning" when falling back to "__package__" during import + resolution. + + Deprecated since version 3.13, will be removed in version 3.15: + "__package__" will cease to be set or taken into consideration by + the import system or standard library. + +module.__loader__ + + The *loader* object that the import machinery used to load the + module. + + This attribute is mostly useful for introspection, but can be used + for additional loader-specific functionality, for example getting + data associated with a loader. + + "__loader__" defaults to "None" for modules created dynamically + using the "types.ModuleType" constructor; use + "importlib.util.module_from_spec()" instead to ensure the attribute + is set to a *loader* object. + + It is **strongly** recommended that you use + "module.__spec__.loader" instead of "module.__loader__". + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Deprecated since version 3.12, will be removed in version 3.16: + Setting "__loader__" on a module while failing to set + "__spec__.loader" is deprecated. In Python 3.16, "__loader__" will + cease to be set or taken into consideration by the import system or + the standard library. + +module.__path__ + + A (possibly empty) *sequence* of strings enumerating the locations + where the package’s submodules will be found. Non-package modules + should not have a "__path__" attribute. See __path__ attributes on + modules for more details. + + It is **strongly** recommended that you use + "module.__spec__.submodule_search_locations" instead of + "module.__path__". + +module.__file__ + +module.__cached__ + + "__file__" and "__cached__" are both optional attributes that may + or may not be set. Both attributes should be a "str" when they are + available. + + "__file__" indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared + library file for extension modules loaded dynamically from a shared + library. It might be missing for certain types of modules, such as + C modules that are statically linked into the interpreter, and the + import system may opt to leave it unset if it has no semantic + meaning (for example, a module loaded from a database). + + If "__file__" is set then the "__cached__" attribute might also be + set, which is the path to any compiled version of the code (for + example, a byte-compiled file). The file does not need to exist to + set this attribute; the path can simply point to where the compiled + file *would* exist (see **PEP 3147**). + + Note that "__cached__" may be set even if "__file__" is not set. + However, that scenario is quite atypical. Ultimately, the *loader* + is what makes use of the module spec provided by the *finder* (from + which "__file__" and "__cached__" are derived). So if a loader can + load from a cached module but otherwise does not load from a file, + that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + "module.__spec__.cached" instead of "module.__cached__". + + Deprecated since version 3.13, will be removed in version 3.15: + Setting "__cached__" on a module while failing to set + "__spec__.cached" is deprecated. In Python 3.15, "__cached__" will + cease to be set or taken into consideration by the import system or + standard library. + + +Other writable attributes on module objects +------------------------------------------- + +As well as the import-related attributes listed above, module objects +also have the following writable attributes: + +module.__doc__ + + The module’s documentation string, or "None" if unavailable. See + also: "__doc__ attributes". + +module.__annotations__ + + A dictionary containing *variable annotations* collected during + module body execution. For best practices on working with + "__annotations__", see "annotationlib". + + Changed in version 3.14: Annotations are now lazily evaluated. See + **PEP 649**. + +module.__annotate__ + + The *annotate function* for this module, or "None" if the module + has no annotations. See also: "__annotate__" attributes. + + Added in version 3.14. + + +Module dictionaries +------------------- + +Module objects also have the following special read-only attribute: + +module.__dict__ + + The module’s namespace as a dictionary object. Uniquely among the + attributes listed here, "__dict__" cannot be accessed as a global + variable from within a module; it can only be accessed as an + attribute on module objects. + + **CPython implementation detail:** Because of the way CPython + clears module dictionaries, the module dictionary will be cleared + when the module falls out of scope even if the dictionary still has + live references. To avoid this, copy the dictionary or keep the + module around while using its dictionary directly. + + +Custom classes +============== + +Custom class types are typically created by class definitions (see +section Class definitions). A class has a namespace implemented by a +dictionary object. Class attribute references are translated to +lookups in this dictionary, e.g., "C.x" is translated to +"C.__dict__["x"]" (although there are a number of hooks which allow +for other means of locating attributes). When the attribute name is +not found there, the attribute search continues in the base classes. +This search of the base classes uses the C3 method resolution order +which behaves correctly even in the presence of ‘diamond’ inheritance +structures where there are multiple inheritance paths leading back to +a common ancestor. Additional details on the C3 MRO used by Python can +be found at The Python 2.3 Method Resolution Order. + +When a class attribute reference (for class "C", say) would yield a +class method object, it is transformed into an instance method object +whose "__self__" attribute is "C". When it would yield a +"staticmethod" object, it is transformed into the object wrapped by +the static method object. See section Implementing Descriptors for +another way in which attributes retrieved from a class may differ from +those actually contained in its "__dict__". + +Class attribute assignments update the class’s dictionary, never the +dictionary of a base class. + +A class object can be called (see above) to yield a class instance +(see below). + + +Special attributes +------------------ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| type.__name__ | The class’s name. See also: "__name__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__qualname__ | The class’s *qualified name*. See also: | +| | "__qualname__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__module__ | The name of the module in which the class was | +| | defined. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__dict__ | A "mapping proxy" providing a read-only view of | +| | the class’s namespace. See also: "__dict__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__bases__ | A "tuple" containing the class’s bases. In most | +| | cases, for a class defined as "class X(A, B, C)", | +| | "X.__bases__" will be exactly equal to "(A, B, | +| | C)". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__doc__ | The class’s documentation string, or "None" if | +| | undefined. Not inherited by subclasses. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotations__ | A dictionary containing *variable annotations* | +| | collected during class body execution. See also: | +| | "__annotations__ attributes". For best practices | +| | on working with "__annotations__", please see | +| | "annotationlib". Use | +| | "annotationlib.get_annotations()" instead of | +| | accessing this attribute directly. Warning: | +| | Accessing the "__annotations__" attribute directly | +| | on a class object may return annotations for the | +| | wrong class, specifically in certain cases where | +| | the class, its base class, or a metaclass is | +| | defined under "from __future__ import | +| | annotations". See **749** for details.This | +| | attribute does not exist on certain builtin | +| | classes. On user-defined classes without | +| | "__annotations__", it is an empty dictionary. | +| | Changed in version 3.14: Annotations are now | +| | lazily evaluated. See **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotate__() | The *annotate function* for this class, or "None" | +| | if the class has no annotations. See also: | +| | "__annotate__ attributes". Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__type_params__ | A "tuple" containing the type parameters of a | +| | generic class. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__static_attributes__ | A "tuple" containing names of attributes of this | +| | class which are assigned through "self.X" from any | +| | function in its body. Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__firstlineno__ | The line number of the first line of the class | +| | definition, including decorators. Setting the | +| | "__module__" attribute removes the | +| | "__firstlineno__" item from the type’s dictionary. | +| | Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__mro__ | The "tuple" of classes that are considered when | +| | looking for base classes during method resolution. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special methods +--------------- + +In addition to the special attributes described above, all Python +classes also have the following two methods available: + +type.mro() + + This method can be overridden by a metaclass to customize the + method resolution order for its instances. It is called at class + instantiation, and its result is stored in "__mro__". + +type.__subclasses__() + + Each class keeps a list of weak references to its immediate + subclasses. This method returns a list of all those references + still alive. The list is in definition order. Example: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] + + +Class instances +=============== + +A class instance is created by calling a class object (see above). A +class instance has a namespace implemented as a dictionary which is +the first place in which attribute references are searched. When an +attribute is not found there, and the instance’s class has an +attribute by that name, the search continues with the class +attributes. If a class attribute is found that is a user-defined +function object, it is transformed into an instance method object +whose "__self__" attribute is the instance. Static method and class +method objects are also transformed; see above under “Classes”. See +section Implementing Descriptors for another way in which attributes +of a class retrieved via its instances may differ from the objects +actually stored in the class’s "__dict__". If no class attribute is +found, and the object’s class has a "__getattr__()" method, that is +called to satisfy the lookup. + +Attribute assignments and deletions update the instance’s dictionary, +never a class’s dictionary. If the class has a "__setattr__()" or +"__delattr__()" method, this is called instead of updating the +instance dictionary directly. + +Class instances can pretend to be numbers, sequences, or mappings if +they have methods with certain special names. See section Special +method names. + + +Special attributes +------------------ + +object.__class__ + + The class to which a class instance belongs. + +object.__dict__ + + A dictionary or other mapping object used to store an object’s + (writable) attributes. Not all instances have a "__dict__" + attribute; see the section on __slots__ for more details. + + +I/O objects (also known as file objects) +======================================== + +A *file object* represents an open file. Various shortcuts are +available to create file objects: the "open()" built-in function, and +also "os.popen()", "os.fdopen()", and the "makefile()" method of +socket objects (and perhaps by other functions or methods provided by +extension modules). + +The objects "sys.stdin", "sys.stdout" and "sys.stderr" are initialized +to file objects corresponding to the interpreter’s standard input, +output and error streams; they are all open in text mode and therefore +follow the interface defined by the "io.TextIOBase" abstract class. + + +Internal types +============== + +A few types used internally by the interpreter are exposed to the +user. Their definitions may change with future versions of the +interpreter, but they are mentioned here for completeness. + + +Code objects +------------ + +Code objects represent *byte-compiled* executable Python code, or +*bytecode*. The difference between a code object and a function object +is that the function object contains an explicit reference to the +function’s globals (the module in which it was defined), while a code +object contains no context; also the default argument values are +stored in the function object, not in the code object (because they +represent values calculated at run-time). Unlike function objects, +code objects are immutable and contain no references (directly or +indirectly) to mutable objects. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_name | The function name | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_qualname | The fully qualified function name Added in | +| | version 3.11. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_argcount | The total number of positional *parameters* | +| | (including positional-only parameters and | +| | parameters with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_posonlyargcount | The number of positional-only *parameters* | +| | (including arguments with default values) that the | +| | function has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_kwonlyargcount | The number of keyword-only *parameters* (including | +| | arguments with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_nlocals | The number of local variables used by the function | +| | (including parameters) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_varnames | A "tuple" containing the names of the local | +| | variables in the function (starting with the | +| | parameter names) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_cellvars | A "tuple" containing the names of local variables | +| | that are referenced from at least one *nested | +| | scope* inside the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_freevars | A "tuple" containing the names of *free (closure) | +| | variables* that a *nested scope* references in an | +| | outer scope. See also "function.__closure__". | +| | Note: references to global and builtin names are | +| | *not* included. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_code | A string representing the sequence of *bytecode* | +| | instructions in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_consts | A "tuple" containing the literals used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_names | A "tuple" containing the names used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_filename | The name of the file from which the code was | +| | compiled | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_firstlineno | The line number of the first line of the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_lnotab | A string encoding the mapping from *bytecode* | +| | offsets to line numbers. For details, see the | +| | source code of the interpreter. Deprecated since | +| | version 3.12: This attribute of code objects is | +| | deprecated, and may be removed in Python 3.15. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_stacksize | The required stack size of the code object | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_flags | An "integer" encoding a number of flags for the | +| | interpreter. | ++----------------------------------------------------+----------------------------------------------------+ + +The following flag bits are defined for "co_flags": bit "0x04" is set +if the function uses the "*arguments" syntax to accept an arbitrary +number of positional arguments; bit "0x08" is set if the function uses +the "**keywords" syntax to accept arbitrary keyword arguments; bit +"0x20" is set if the function is a generator. See Code Objects Bit +Flags for details on the semantics of each flags that might be +present. + +Future feature declarations (for example, "from __future__ import +division") also use bits in "co_flags" to indicate whether a code +object was compiled with a particular feature enabled. See +"compiler_flag". + +Other bits in "co_flags" are reserved for internal use. + +If a code object represents a function and has a docstring, the +"CO_HAS_DOCSTRING" bit is set in "co_flags" and the first item in +"co_consts" is the docstring of the function. + + +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ + +codeobject.co_positions() + + Returns an iterable over the source code positions of each + *bytecode* instruction in the code object. + + The iterator returns "tuple"s containing the "(start_line, + end_line, start_column, end_column)". The *i-th* tuple corresponds + to the position of the source code that compiled to the *i-th* code + unit. Column information is 0-indexed utf-8 byte offsets on the + given source line. + + This positional information can be missing. A non-exhaustive lists + of cases where this may happen: + + * Running the interpreter with "-X" "no_debug_ranges". + + * Loading a pyc file compiled while using "-X" "no_debug_ranges". + + * Position tuples corresponding to artificial instructions. + + * Line and column numbers that can’t be represented due to + implementation specific limitations. + + When this occurs, some or all of the tuple elements can be "None". + + Added in version 3.11. + + Note: + + This feature requires storing column positions in code objects + which may result in a small increase of disk usage of compiled + Python files or interpreter memory usage. To avoid storing the + extra information and/or deactivate printing the extra traceback + information, the "-X" "no_debug_ranges" command line flag or the + "PYTHONNODEBUGRANGES" environment variable can be used. + +codeobject.co_lines() + + Returns an iterator that yields information about successive ranges + of *bytecode*s. Each item yielded is a "(start, end, lineno)" + "tuple": + + * "start" (an "int") represents the offset (inclusive) of the start + of the *bytecode* range + + * "end" (an "int") represents the offset (exclusive) of the end of + the *bytecode* range + + * "lineno" is an "int" representing the line number of the + *bytecode* range, or "None" if the bytecodes in the given range + have no line number + + The items yielded will have the following properties: + + * The first range yielded will have a "start" of 0. + + * The "(start, end)" ranges will be non-decreasing and consecutive. + That is, for any pair of "tuple"s, the "start" of the second will + be equal to the "end" of the first. + + * No range will be backwards: "end >= start" for all triples. + + * The last "tuple" yielded will have "end" equal to the size of the + *bytecode*. + + Zero-width ranges, where "start == end", are allowed. Zero-width + ranges are used for lines that are present in the source code, but + have been eliminated by the *bytecode* compiler. + + Added in version 3.10. + + See also: + + **PEP 626** - Precise line numbers for debugging and other tools. + The PEP that introduced the "co_lines()" method. + +codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified + fields. + + Code objects are also supported by the generic function + "copy.replace()". + + Added in version 3.8. + + +Frame objects +------------- + +Frame objects represent execution frames. They may occur in traceback +objects, and are also passed to registered trace functions. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_back | Points to the previous stack frame (towards the | +| | caller), or "None" if this is the bottom stack | +| | frame | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_code | The code object being executed in this frame. | +| | Accessing this attribute raises an auditing event | +| | "object.__getattr__" with arguments "obj" and | +| | ""f_code"". | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_locals | The mapping used by the frame to look up local | +| | variables. If the frame refers to an *optimized | +| | scope*, this may return a write-through proxy | +| | object. Changed in version 3.13: Return a proxy | +| | for optimized scopes. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_globals | The dictionary used by the frame to look up global | +| | variables | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_builtins | The dictionary used by the frame to look up built- | +| | in (intrinsic) names | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lasti | The “precise instruction” of the frame object | +| | (this is an index into the *bytecode* string of | +| | the code object) | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_generator | The *generator* or *coroutine* object that owns | +| | this frame, or "None" if the frame is a normal | +| | function. Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace | If not "None", this is a function called for | +| | various events during code execution (this is used | +| | by debuggers). Normally an event is triggered for | +| | each new source line (see "f_trace_lines"). | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_lines | Set this attribute to "False" to disable | +| | triggering a tracing event for each source line. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_opcodes | Set this attribute to "True" to allow per-opcode | +| | events to be requested. Note that this may lead to | +| | undefined interpreter behaviour if exceptions | +| | raised by the trace function escape to the | +| | function being traced. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lineno | The current line number of the frame – writing to | +| | this from within a trace function jumps to the | +| | given line (only for the bottom-most frame). A | +| | debugger can implement a Jump command (aka Set | +| | Next Statement) by writing to this attribute. | ++----------------------------------------------------+----------------------------------------------------+ + + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ + +Frame objects support one method: + +frame.clear() + + This method clears all references to local variables held by the + frame. Also, if the frame belonged to a *generator*, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an exception and storing its + traceback for later use). + + "RuntimeError" is raised if the frame is currently executing or + suspended. + + Added in version 3.4. + + Changed in version 3.13: Attempting to clear a suspended frame + raises "RuntimeError" (as has always been the case for executing + frames). + + +Traceback objects +----------------- + +Traceback objects represent the stack trace of an exception. A +traceback object is implicitly created when an exception occurs, and +may also be explicitly created by calling "types.TracebackType". + +Changed in version 3.7: Traceback objects can now be explicitly +instantiated from Python code. + +For implicitly created tracebacks, when the search for an exception +handler unwinds the execution stack, at each unwound level a traceback +object is inserted in front of the current traceback. When an +exception handler is entered, the stack trace is made available to the +program. (See section The try statement.) It is accessible as the +third item of the tuple returned by "sys.exc_info()", and as the +"__traceback__" attribute of the caught exception. + +When the program contains no suitable handler, the stack trace is +written (nicely formatted) to the standard error stream; if the +interpreter is interactive, it is also made available to the user as +"sys.last_traceback". + +For explicitly created tracebacks, it is up to the creator of the +traceback to determine how the "tb_next" attributes should be linked +to form a full stack trace. + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_frame | Points to the execution frame of the current | +| | level. Accessing this attribute raises an | +| | auditing event "object.__getattr__" with arguments | +| | "obj" and ""tb_frame"". | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lineno | Gives the line number where the exception occurred | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lasti | Indicates the “precise instruction”. | ++----------------------------------------------------+----------------------------------------------------+ + +The line number and last instruction in the traceback may differ from +the line number of its frame object if the exception occurred in a +"try" statement with no matching except clause or with a "finally" +clause. + +traceback.tb_next + + The special writable attribute "tb_next" is the next level in the + stack trace (towards the frame where the exception occurred), or + "None" if there is no next level. + + Changed in version 3.7: This attribute is now writable + + +Slice objects +------------- + +Slice objects are used to represent slices for "__getitem__()" +methods. They are also created by the built-in "slice()" function. + +Special read-only attributes: "start" is the lower bound; "stop" is +the upper bound; "step" is the step value; each is "None" if omitted. +These attributes can have any type. + +Slice objects support one method: + +slice.indices(self, length) + + This method takes a single integer argument *length* and computes + information about the slice that the slice object would describe if + applied to a sequence of *length* items. It returns a tuple of + three integers; respectively these are the *start* and *stop* + indices and the *step* or stride length of the slice. Missing or + out-of-bounds indices are handled in a manner consistent with + regular slices. + + +Static method objects +--------------------- + +Static method objects provide a way of defeating the transformation of +function objects to method objects described above. A static method +object is a wrapper around any other object, usually a user-defined +method object. When a static method object is retrieved from a class +or a class instance, the object actually returned is the wrapped +object, which is not subject to any further transformation. Static +method objects are also callable. Static method objects are created by +the built-in "staticmethod()" constructor. + + +Class method objects +-------------------- + +A class method object, like a static method object, is a wrapper +around another object that alters the way in which that object is +retrieved from classes and class instances. The behaviour of class +method objects upon such retrieval is described above, under “instance +methods”. Class method objects are created by the built-in +"classmethod()" constructor. +''', + 'typesfunctions': r'''Functions +********* + +Function objects are created by function definitions. The only +operation on a function object is to call it: "func(argument-list)". + +There are really two flavors of function objects: built-in functions +and user-defined functions. Both support the same operation (to call +the function), but the implementation is different, hence the +different object types. + +See Function definitions for more information. +''', + 'typesmapping': r'''Mapping Types — "dict" +********************** + +A *mapping* object maps *hashable* values to arbitrary objects. +Mappings are mutable objects. There is currently only one standard +mapping type, the *dictionary*. (For other containers see the built- +in "list", "set", and "tuple" classes, and the "collections" module.) + +A dictionary’s keys are *almost* arbitrary values. Values that are +not *hashable*, that is, values containing lists, dictionaries or +other mutable types (that are compared by value rather than by object +identity) may not be used as keys. Values that compare equal (such as +"1", "1.0", and "True") can be used interchangeably to index the same +dictionary entry. + +class dict(**kwargs) +class dict(mapping, /, **kwargs) +class dict(iterable, /, **kwargs) + + Return a new dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + Dictionaries can be created by several means: + + * Use a comma-separated list of "key: value" pairs within braces: + "{'jack': 4098, 'sjoerd': 4127}" or "{4098: 'jack', 4127: + 'sjoerd'}" + + * Use a dict comprehension: "{}", "{x: x ** 2 for x in range(10)}" + + * Use the type constructor: "dict()", "dict([('foo', 100), ('bar', + 200)])", "dict(foo=100, bar=200)" + + If no positional argument is given, an empty dictionary is created. + If a positional argument is given and it defines a "keys()" method, + a dictionary is created by calling "__getitem__()" on the argument + with each returned key from the method. Otherwise, the positional + argument must be an *iterable* object. Each item in the iterable + must itself be an iterable with exactly two elements. The first + element of each item becomes a key in the new dictionary, and the + second element the corresponding value. If a key occurs more than + once, the last value for that key becomes the corresponding value + in the new dictionary. + + If keyword arguments are given, the keyword arguments and their + values are added to the dictionary created from the positional + argument. If a key being added is already present, the value from + the keyword argument replaces the value from the positional + argument. + + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary + creation and equality, the following examples all return a + dictionary equal to "{"one": 1, "two": 2, "three": 3}": + + >>> a = dict(one=1, two=2, three=3) + >>> b = {'one': 1, 'two': 2, 'three': 3} + >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) + >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) + >>> e = dict({'three': 3, 'one': 1, 'two': 2}) + >>> f = dict({'one': 1, 'three': 3}, two=2) + >>> a == b == c == d == e == f + True + + Providing keyword arguments as in the first example only works for + keys that are valid Python identifiers. Otherwise, any valid keys + can be used. + + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + + These are the operations that dictionaries support (and therefore, + custom mapping types should support too): + + list(d) + + Return a list of all the keys used in the dictionary *d*. + + len(d) + + Return the number of items in the dictionary *d*. + + d[key] + + Return the item of *d* with key *key*. Raises a "KeyError" if + *key* is not in the map. + + If a subclass of dict defines a method "__missing__()" and *key* + is not present, the "d[key]" operation calls that method with + the key *key* as argument. The "d[key]" operation then returns + or raises whatever is returned or raised by the + "__missing__(key)" call. No other operations or methods invoke + "__missing__()". If "__missing__()" is not defined, "KeyError" + is raised. "__missing__()" must be a method; it cannot be an + instance variable: + + >>> class Counter(dict): + ... def __missing__(self, key): + ... return 0 + ... + >>> c = Counter() + >>> c['red'] + 0 + >>> c['red'] += 1 + >>> c['red'] + 1 + + The example above shows part of the implementation of + "collections.Counter". A different "__missing__()" method is + used by "collections.defaultdict". + + d[key] = value + + Set "d[key]" to *value*. + + del d[key] + + Remove "d[key]" from *d*. Raises a "KeyError" if *key* is not + in the map. + + key in d + + Return "True" if *d* has a key *key*, else "False". + + key not in d + + Equivalent to "not key in d". + + iter(d) + + Return an iterator over the keys of the dictionary. This is a + shortcut for "iter(d.keys())". + + clear() + + Remove all items from the dictionary. + + copy() + + Return a shallow copy of the dictionary. + + classmethod fromkeys(iterable, value=None, /) + + Create a new dictionary with keys from *iterable* and values set + to *value*. + + "fromkeys()" is a class method that returns a new dictionary. + *value* defaults to "None". All of the values refer to just a + single instance, so it generally doesn’t make sense for *value* + to be a mutable object such as an empty list. To get distinct + values, use a dict comprehension instead. + + get(key, default=None, /) + + Return the value for *key* if *key* is in the dictionary, else + *default*. If *default* is not given, it defaults to "None", so + that this method never raises a "KeyError". + + items() + + Return a new view of the dictionary’s items ("(key, value)" + pairs). See the documentation of view objects. + + keys() + + Return a new view of the dictionary’s keys. See the + documentation of view objects. + + pop(key, /) + pop(key, default, /) + + If *key* is in the dictionary, remove it and return its value, + else return *default*. If *default* is not given and *key* is + not in the dictionary, a "KeyError" is raised. + + popitem() + + Remove and return a "(key, value)" pair from the dictionary. + Pairs are returned in LIFO (last-in, first-out) order. + + "popitem()" is useful to destructively iterate over a + dictionary, as often used in set algorithms. If the dictionary + is empty, calling "popitem()" raises a "KeyError". + + Changed in version 3.7: LIFO order is now guaranteed. In prior + versions, "popitem()" would return an arbitrary key/value pair. + + reversed(d) + + Return a reverse iterator over the keys of the dictionary. This + is a shortcut for "reversed(d.keys())". + + Added in version 3.8. + + setdefault(key, default=None, /) + + If *key* is in the dictionary, return its value. If not, insert + *key* with a value of *default* and return *default*. *default* + defaults to "None". + + update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) + + Update the dictionary with the key/value pairs from *mapping* or + *iterable* and *kwargs*, overwriting existing keys. Return + "None". + + "update()" accepts either another object with a "keys()" method + (in which case "__getitem__()" is called with every key returned + from the method) or an iterable of key/value pairs (as tuples or + other iterables of length two). If keyword arguments are + specified, the dictionary is then updated with those key/value + pairs: "d.update(red=1, blue=2)". + + values() + + Return a new view of the dictionary’s values. See the + documentation of view objects. + + An equality comparison between one "dict.values()" view and + another will always return "False". This also applies when + comparing "dict.values()" to itself: + + >>> d = {'a': 1} + >>> d.values() == d.values() + False + + d | other + + Create a new dictionary with the merged keys and values of *d* + and *other*, which must both be dictionaries. The values of + *other* take priority when *d* and *other* share keys. + + Added in version 3.9. + + d |= other + + Update the dictionary *d* with keys and values from *other*, + which may be either a *mapping* or an *iterable* of key/value + pairs. The values of *other* take priority when *d* and *other* + share keys. + + Added in version 3.9. + + Dictionaries and dictionary views are reversible. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(reversed(d)) + ['four', 'three', 'two', 'one'] + >>> list(reversed(d.values())) + [4, 3, 2, 1] + >>> list(reversed(d.items())) + [('four', 4), ('three', 3), ('two', 2), ('one', 1)] + + Changed in version 3.8: Dictionaries are now reversible. + +See also: + + "types.MappingProxyType" can be used to create a read-only view of a + "dict". + + +Dictionary view objects +======================= + +The objects returned by "dict.keys()", "dict.values()" and +"dict.items()" are *view objects*. They provide a dynamic view on the +dictionary’s entries, which means that when the dictionary changes, +the view reflects these changes. + +Dictionary views can be iterated over to yield their respective data, +and support membership tests: + +len(dictview) + + Return the number of entries in the dictionary. + +iter(dictview) + + Return an iterator over the keys, values or items (represented as + tuples of "(key, value)") in the dictionary. + + Keys and values are iterated over in insertion order. This allows + the creation of "(value, key)" pairs using "zip()": "pairs = + zip(d.values(), d.keys())". Another way to create the same list is + "pairs = [(v, k) for (k, v) in d.items()]". + + Iterating views while adding or deleting entries in the dictionary + may raise a "RuntimeError" or fail to iterate over all entries. + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. + +x in dictview + + Return "True" if *x* is in the underlying dictionary’s keys, values + or items (in the latter case, *x* should be a "(key, value)" + tuple). + +reversed(dictview) + + Return a reverse iterator over the keys, values or items of the + dictionary. The view will be iterated in reverse order of the + insertion. + + Changed in version 3.8: Dictionary views are now reversible. + +dictview.mapping + + Return a "types.MappingProxyType" that wraps the original + dictionary to which the view refers. + + Added in version 3.10. + +Keys views are set-like since their entries are unique and *hashable*. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. If all values in an items view +are hashable as well, then the items view can interoperate with other +sets. (Values views are not treated as set-like since the entries are +generally not unique.) For set-like views, all of the operations +defined for the abstract base class "collections.abc.Set" are +available (for example, "==", "<", or "^"). While using set +operators, set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. + +An example of dictionary view usage: + + >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} + >>> keys = dishes.keys() + >>> values = dishes.values() + + >>> # iteration + >>> n = 0 + >>> for val in values: + ... n += val + ... + >>> print(n) + 504 + + >>> # keys and values are iterated over in the same order (insertion order) + >>> list(keys) + ['eggs', 'sausage', 'bacon', 'spam'] + >>> list(values) + [2, 1, 1, 500] + + >>> # view objects are dynamic and reflect dict changes + >>> del dishes['eggs'] + >>> del dishes['sausage'] + >>> list(keys) + ['bacon', 'spam'] + + >>> # set operations + >>> keys & {'eggs', 'bacon', 'salad'} + {'bacon'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True + + >>> # get back a read-only proxy for the original dictionary + >>> values.mapping + mappingproxy({'bacon': 1, 'spam': 500}) + >>> values.mapping['spam'] + 500 +''', + 'typesmethods': r'''Methods +******* + +Methods are functions that are called using the attribute notation. +There are two flavors: built-in methods (such as "append()" on lists) +and class instance method. Built-in methods are described with the +types that support them. + +If you access a method (a function defined in a class namespace) +through an instance, you get a special object: a *bound method* (also +called instance method) object. When called, it will add the "self" +argument to the argument list. Bound methods have two special read- +only attributes: "m.__self__" is the object on which the method +operates, and "m.__func__" is the function implementing the method. +Calling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to +calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)". + +Like function objects, bound method objects support getting arbitrary +attributes. However, since method attributes are actually stored on +the underlying function object ("method.__func__"), setting method +attributes on bound methods is disallowed. Attempting to set an +attribute on a method results in an "AttributeError" being raised. In +order to set a method attribute, you need to explicitly set it on the +underlying function object: + + >>> class C: + ... def method(self): + ... pass + ... + >>> c = C() + >>> c.method.whoami = 'my name is method' # can't set on the method + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'method' object has no attribute 'whoami' + >>> c.method.__func__.whoami = 'my name is method' + >>> c.method.whoami + 'my name is method' + +See Instance methods for more information. +''', + 'typesmodules': r'''Modules +******* + +The only special operation on a module is attribute access: "m.name", +where *m* is a module and *name* accesses a name defined in *m*’s +symbol table. Module attributes can be assigned to. (Note that the +"import" statement is not, strictly speaking, an operation on a module +object; "import foo" does not require a module object named *foo* to +exist, rather it requires an (external) *definition* for a module +named *foo* somewhere.) + +A special attribute of every module is "__dict__". This is the +dictionary containing the module’s symbol table. Modifying this +dictionary will actually change the module’s symbol table, but direct +assignment to the "__dict__" attribute is not possible (you can write +"m.__dict__['a'] = 1", which defines "m.a" to be "1", but you can’t +write "m.__dict__ = {}"). Modifying "__dict__" directly is not +recommended. + +Modules built into the interpreter are written like this: "". If loaded from a file, they are written as +"". +''', + 'typesseq': r'''Sequence Types — "list", "tuple", "range" +***************************************** + +There are three basic sequence types: lists, tuples, and range +objects. Additional sequence types tailored for processing of binary +data and text strings are described in dedicated sections. + + +Common Sequence Operations +========================== + +The operations in the following table are supported by most sequence +types, both mutable and immutable. The "collections.abc.Sequence" ABC +is provided to make it easier to correctly implement these operations +on custom sequence types. + +This table lists the sequence operations sorted in ascending priority. +In the table, *s* and *t* are sequences of the same type, *n*, *i*, +*j* and *k* are integers and *x* is an arbitrary object that meets any +type and value restrictions imposed by *s*. + +The "in" and "not in" operations have the same priorities as the +comparison operations. The "+" (concatenation) and "*" (repetition) +operations have the same priority as the corresponding numeric +operations. [3] + ++----------------------------+----------------------------------+------------+ +| Operation | Result | Notes | +|============================|==================================|============| +| "x in s" | "True" if an item of *s* is | (1) | +| | equal to *x*, else "False" | | ++----------------------------+----------------------------------+------------+ +| "x not in s" | "False" if an item of *s* is | (1) | +| | equal to *x*, else "True" | | ++----------------------------+----------------------------------+------------+ +| "s + t" | the concatenation of *s* and *t* | (6)(7) | ++----------------------------+----------------------------------+------------+ +| "s * n" or "n * s" | equivalent to adding *s* to | (2)(7) | +| | itself *n* times | | ++----------------------------+----------------------------------+------------+ +| "s[i]" | *i*th item of *s*, origin 0 | (3)(8) | ++----------------------------+----------------------------------+------------+ +| "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) | ++----------------------------+----------------------------------+------------+ +| "s[i:j:k]" | slice of *s* from *i* to *j* | (3)(5) | +| | with step *k* | | ++----------------------------+----------------------------------+------------+ +| "len(s)" | length of *s* | | ++----------------------------+----------------------------------+------------+ +| "min(s)" | smallest item of *s* | | ++----------------------------+----------------------------------+------------+ +| "max(s)" | largest item of *s* | | ++----------------------------+----------------------------------+------------+ + +Sequences of the same type also support comparisons. In particular, +tuples and lists are compared lexicographically by comparing +corresponding elements. This means that to compare equal, every +element must compare equal and the two sequences must be of the same +type and have the same length. (For full details see Comparisons in +the language reference.) + +Forward and reversed iterators over mutable sequences access values +using an index. That index will continue to march forward (or +backward) even if the underlying sequence is mutated. The iterator +terminates only when an "IndexError" or a "StopIteration" is +encountered (or when the index drops below zero). + +Notes: + +1. While the "in" and "not in" operations are used only for simple + containment testing in the general case, some specialised sequences + (such as "str", "bytes" and "bytearray") also use them for + subsequence testing: + + >>> "gg" in "eggs" + True + +2. Values of *n* less than "0" are treated as "0" (which yields an + empty sequence of the same type as *s*). Note that items in the + sequence *s* are not copied; they are referenced multiple times. + This often haunts new Python programmers; consider: + + >>> lists = [[]] * 3 + >>> lists + [[], [], []] + >>> lists[0].append(3) + >>> lists + [[3], [3], [3]] + + What has happened is that "[[]]" is a one-element list containing + an empty list, so all three elements of "[[]] * 3" are references + to this single empty list. Modifying any of the elements of + "lists" modifies this single list. You can create a list of + different lists this way: + + >>> lists = [[] for i in range(3)] + >>> lists[0].append(3) + >>> lists[1].append(5) + >>> lists[2].append(7) + >>> lists + [[3], [5], [7]] + + Further explanation is available in the FAQ entry How do I create a + multidimensional list?. + +3. If *i* or *j* is negative, the index is relative to the end of + sequence *s*: "len(s) + i" or "len(s) + j" is substituted. But + note that "-0" is still "0". + +4. The slice of *s* from *i* to *j* is defined as the sequence of + items with index *k* such that "i <= k < j". + + * If *i* is omitted or "None", use "0". + + * If *j* is omitted or "None", use "len(s)". + + * If *i* or *j* is less than "-len(s)", use "0". + + * If *i* or *j* is greater than "len(s)", use "len(s)". + + * If *i* is greater than or equal to *j*, the slice is empty. + +5. The slice of *s* from *i* to *j* with step *k* is defined as the + sequence of items with index "x = i + n*k" such that "0 <= n < + (j-i)/k". In other words, the indices are "i", "i+k", "i+2*k", + "i+3*k" and so on, stopping when *j* is reached (but never + including *j*). When *k* is positive, *i* and *j* are reduced to + "len(s)" if they are greater. When *k* is negative, *i* and *j* are + reduced to "len(s) - 1" if they are greater. If *i* or *j* are + omitted or "None", they become “end” values (which end depends on + the sign of *k*). Note, *k* cannot be zero. If *k* is "None", it + is treated like "1". + +6. Concatenating immutable sequences always results in a new object. + This means that building up a sequence by repeated concatenation + will have a quadratic runtime cost in the total sequence length. + To get a linear runtime cost, you must switch to one of the + alternatives below: + + * if concatenating "str" objects, you can build a list and use + "str.join()" at the end or else write to an "io.StringIO" + instance and retrieve its value when complete + + * if concatenating "bytes" objects, you can similarly use + "bytes.join()" or "io.BytesIO", or you can do in-place + concatenation with a "bytearray" object. "bytearray" objects are + mutable and have an efficient overallocation mechanism + + * if concatenating "tuple" objects, extend a "list" instead + + * for other types, investigate the relevant class documentation + +7. Some sequence types (such as "range") only support item sequences + that follow specific patterns, and hence don’t support sequence + concatenation or repetition. + +8. An "IndexError" is raised if *i* is outside the sequence range. + +-[ Sequence Methods ]- + +Sequence types also support the following methods: + +sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +sequence.index(value[, start[, stop]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises "ValueError" if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching of + subsections of the sequence, beginning at *start* and ending at + *stop*. This is roughly equivalent to "start + + sequence[start:stop].index(value)", only without copying any data. + + Caution: + + Not all sequence types support passing the *start* and *stop* + arguments. + + +Immutable Sequence Types +======================== + +The only operation that immutable sequence types generally implement +that is not also implemented by mutable sequence types is support for +the "hash()" built-in. + +This support allows immutable sequences, such as "tuple" instances, to +be used as "dict" keys and stored in "set" and "frozenset" instances. + +Attempting to hash an immutable sequence that contains unhashable +values will result in "TypeError". + + +Mutable Sequence Types +====================== + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (2) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. + +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". + + +Lists +===== + +Lists are mutable sequences, typically used to store collections of +homogeneous items (where the precise degree of similarity will vary by +application). + +class list(iterable=(), /) + + Lists may be constructed in several ways: + + * Using a pair of square brackets to denote the empty list: "[]" + + * Using square brackets, separating items with commas: "[a]", "[a, + b, c]" + + * Using a list comprehension: "[x for x in iterable]" + + * Using the type constructor: "list()" or "list(iterable)" + + The constructor builds a list whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a list, a copy is made and + returned, similar to "iterable[:]". For example, "list('abc')" + returns "['a', 'b', 'c']" and "list( (1, 2, 3) )" returns "[1, 2, + 3]". If no argument is given, the constructor creates a new empty + list, "[]". + + Many other operations also produce lists, including the "sorted()" + built-in. + + Lists implement all of the common and mutable sequence operations. + Lists also provide the following additional method: + + sort(*, key=None, reverse=False) + + This method sorts the list in place, using only "<" comparisons + between items. Exceptions are not suppressed - if any comparison + operations fail, the entire sort operation will fail (and the + list will likely be left in a partially modified state). + + "sort()" accepts two arguments that can only be passed by + keyword (keyword-only arguments): + + *key* specifies a function of one argument that is used to + extract a comparison key from each list element (for example, + "key=str.lower"). The key corresponding to each item in the list + is calculated once and then used for the entire sorting process. + The default value of "None" means that list items are sorted + directly without calculating a separate key value. + + The "functools.cmp_to_key()" utility is available to convert a + 2.x style *cmp* function to a *key* function. + + *reverse* is a boolean value. If set to "True", then the list + elements are sorted as if each comparison were reversed. + + This method modifies the sequence in place for economy of space + when sorting a large sequence. To remind users that it operates + by side effect, it does not return the sorted sequence (use + "sorted()" to explicitly request a new sorted list instance). + + The "sort()" method is guaranteed to be stable. A sort is + stable if it guarantees not to change the relative order of + elements that compare equal — this is helpful for sorting in + multiple passes (for example, sort by department, then by salary + grade). + + For sorting examples and a brief sorting tutorial, see Sorting + Techniques. + + **CPython implementation detail:** While a list is being sorted, + the effect of attempting to mutate, or even inspect, the list is + undefined. The C implementation of Python makes the list appear + empty for the duration, and raises "ValueError" if it can detect + that the list has been mutated during a sort. + +Thread safety: Reading a single element from a "list" is *atomic*: + + lst[i] # list.__getitem__ + +The following methods traverse the list and use *atomic* reads of each +item to perform their function. That means that they may return +results affected by concurrent modifications: + + item in lst + lst.index(item) + lst.count(item) + +All of the above methods/operations are also lock-free. They do not +block concurrent modifications. Other operations that hold a lock will +not block these from observing intermediate states.All other +operations from here on block using the per-object lock.Writing a +single item via "lst[i] = x" is safe to call from multiple threads and +will not corrupt the list.The following operations return new objects +and appear *atomic* to other threads: + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +Methods that only operate on a single elements with no shifting +required are *atomic*: + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The "clear()" method is also *atomic*. Other threads cannot observe +elements being removed.The "sort()" method is not *atomic*. Other +threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort.The following +operations may allow lock-free operations to observe intermediate +states since they modify multiple elements in place: + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The "remove()" method may allow concurrent modifications since element +comparison may execute arbitrary Python code (via +"__eq__()")."extend()" is safe to call from multiple threads. +However, its guarantees depend on the iterable passed to it. If it is +a "list", a "tuple", a "set", a "frozenset", a "dict" or a dictionary +view object (but not their subclasses), the "extend" operation is safe +from concurrent modifications to the iterable. Otherwise, an iterator +is created which can be concurrently modified by another thread. The +same applies to inplace concatenation of a list with other iterables +when using "lst += iterable".Similarly, assigning to a list slice with +"lst[i:j] = iterable" is safe to call from multiple threads, but +"iterable" is only locked when it is also a "list" (but not its +subclasses).Operations that involve multiple accesses, as well as +iteration, are never atomic. For example: + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing "list" instances across +threads. See Python support for free threading for more information. + + +Tuples +====== + +Tuples are immutable sequences, typically used to store collections of +heterogeneous data (such as the 2-tuples produced by the "enumerate()" +built-in). Tuples are also used for cases where an immutable sequence +of homogeneous data is needed (such as allowing storage in a "set" or +"dict" instance). + +class tuple(iterable=(), /) + + Tuples may be constructed in a number of ways: + + * Using a pair of parentheses to denote the empty tuple: "()" + + * Using a trailing comma for a singleton tuple: "a," or "(a,)" + + * Separating items with commas: "a, b, c" or "(a, b, c)" + + * Using the "tuple()" built-in: "tuple()" or "tuple(iterable)" + + The constructor builds a tuple whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a tuple, it is returned + unchanged. For example, "tuple('abc')" returns "('a', 'b', 'c')" + and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument is + given, the constructor creates a new empty tuple, "()". + + Note that it is actually the comma which makes a tuple, not the + parentheses. The parentheses are optional, except in the empty + tuple case, or when they are needed to avoid syntactic ambiguity. + For example, "f(a, b, c)" is a function call with three arguments, + while "f((a, b, c))" is a function call with a 3-tuple as the sole + argument. + + Tuples implement all of the common sequence operations. + +For heterogeneous collections of data where access by name is clearer +than access by index, "collections.namedtuple()" may be a more +appropriate choice than a simple tuple object. + + +Ranges +====== + +The "range" type represents an immutable sequence of numbers and is +commonly used for looping a specific number of times in "for" loops. + +class range(stop, /) +class range(start, stop, step=1, /) + + The arguments to the range constructor must be integers (either + built-in "int" or any object that implements the "__index__()" + special method). If the *step* argument is omitted, it defaults to + "1". If the *start* argument is omitted, it defaults to "0". If + *step* is zero, "ValueError" is raised. + + For a positive *step*, the contents of a range "r" are determined + by the formula "r[i] = start + step*i" where "i >= 0" and "r[i] < + stop". + + For a negative *step*, the contents of the range are still + determined by the formula "r[i] = start + step*i", but the + constraints are "i >= 0" and "r[i] > stop". + + A range object will be empty if "r[0]" does not meet the value + constraint. Ranges do support negative indices, but these are + interpreted as indexing from the end of the sequence determined by + the positive indices. + + Ranges containing absolute values larger than "sys.maxsize" are + permitted but some features (such as "len()") may raise + "OverflowError". + + Range examples: + + >>> list(range(10)) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(range(1, 11)) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + >>> list(range(0, 30, 5)) + [0, 5, 10, 15, 20, 25] + >>> list(range(0, 10, 3)) + [0, 3, 6, 9] + >>> list(range(0, -10, -1)) + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] + >>> list(range(0)) + [] + >>> list(range(1, 0)) + [] + + Ranges implement all of the common sequence operations except + concatenation and repetition (due to the fact that range objects + can only represent sequences that follow a strict pattern and + repetition and concatenation will usually violate that pattern). + + start + + The value of the *start* parameter (or "0" if the parameter was + not supplied) + + stop + + The value of the *stop* parameter + + step + + The value of the *step* parameter (or "1" if the parameter was + not supplied) + +The advantage of the "range" type over a regular "list" or "tuple" is +that a "range" object will always take the same (small) amount of +memory, no matter the size of the range it represents (as it only +stores the "start", "stop" and "step" values, calculating individual +items and subranges as needed). + +Range objects implement the "collections.abc.Sequence" ABC, and +provide features such as containment tests, element index lookup, +slicing and support for negative indices (see Sequence Types — list, +tuple, range): + +>>> r = range(0, 20, 2) +>>> r +range(0, 20, 2) +>>> 11 in r +False +>>> 10 in r +True +>>> r.index(10) +5 +>>> r[5] +10 +>>> r[:5] +range(0, 10, 2) +>>> r[-1] +18 + +Testing range objects for equality with "==" and "!=" compares them as +sequences. That is, two range objects are considered equal if they +represent the same sequence of values. (Note that two range objects +that compare equal might have different "start", "stop" and "step" +attributes, for example "range(0) == range(2, 1, 3)" or "range(0, 3, +2) == range(0, 4, 2)".) + +Changed in version 3.2: Implement the Sequence ABC. Support slicing +and negative indices. Test "int" objects for membership in constant +time instead of iterating through all items. + +Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range objects +based on the sequence of values they define (instead of comparing +based on object identity).Added the "start", "stop" and "step" +attributes. + +See also: + + * The linspace recipe shows how to implement a lazy version of range + suitable for floating-point applications. +''', + 'typesseq-mutable': r'''Mutable Sequence Types +********************** + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (2) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. + +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". +''', + 'unary': r'''Unary arithmetic and bitwise operations +*************************************** + +All unary arithmetic and bitwise operations have the same priority: + + u_expr: power | "-" u_expr | "+" u_expr | "~" u_expr + +The unary "-" (minus) operator yields the negation of its numeric +argument; the operation can be overridden with the "__neg__()" special +method. + +The unary "+" (plus) operator yields its numeric argument unchanged; +the operation can be overridden with the "__pos__()" special method. + +The unary "~" (invert) operator yields the bitwise inversion of its +integer argument. The bitwise inversion of "x" is defined as +"-(x+1)". It only applies to integral numbers or to custom objects +that override the "__invert__()" special method. + +In all three cases, if the argument does not have the proper type, a +"TypeError" exception is raised. +''', + 'while': r'''The "while" statement +********************* + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt: "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. +''', + 'with': r'''The "with" statement +******************** + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt: "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents: with_item ("," with_item)* + with_item: expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'yield': r'''The "yield" statement +********************* + + yield_stmt: yield_expression + +A "yield" statement is semantically equivalent to a yield expression. +The "yield" statement can be used to omit the parentheses that would +otherwise be required in the equivalent yield expression statement. +For example, the yield statements + + yield + yield from + +are equivalent to the yield expression statements + + (yield ) + (yield from ) + +Yield expressions and statements are only used when defining a +*generator* function, and are only used in the body of the generator +function. Using "yield" in a function definition is sufficient to +cause that definition to create a generator function instead of a +normal function. + +For full details of "yield" semantics, refer to the Yield expressions +section. +''', +} diff --git a/Lib/queue.py b/Lib/queue.py index 25beb46e30d..c0b35987654 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -80,9 +80,6 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -239,9 +236,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index f36cf7b3951..129fd2f5c7c --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index 428d1b0d5fb..af2808a77da 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -61,7 +61,7 @@ resulting RE will match the second character. \number Matches the contents of the group of the same number. \A Matches only at the start of the string. - \Z Matches only at the end of the string. + \z Matches only at the end of the string. \b Matches the empty string, but only at the start or end of a word. \B Matches the empty string, but not at the start or end of a word. \d Matches any decimal digit; equivalent to the set [0-9] in @@ -117,7 +117,8 @@ U UNICODE For compatibility only. Ignored for string patterns (it is the default), and forbidden for bytes patterns. -This module also defines an exception 'error'. +This module also defines exception 'PatternError', aliased to 'error' for +backward compatibility. """ @@ -133,7 +134,7 @@ "findall", "finditer", "compile", "purge", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", - "UNICODE", "NOFLAG", "RegexFlag", + "UNICODE", "NOFLAG", "RegexFlag", "PatternError" ] __version__ = "2.2.1" @@ -155,7 +156,7 @@ class RegexFlag: _numeric_repr_ = hex # sre exception -error = _compiler.error +PatternError = error = _compiler.PatternError # -------------------------------------------------------------------- # public interface diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index 06507d08bee..fed2d84fc01 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,4 +1,4 @@ -# Auto-generated by Tools/scripts/generate_re_casefix.py. +# Auto-generated by Tools/build/generate_re_casefix.py. # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 861bbdb130a..20dd561d1c1 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -28,6 +28,8 @@ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), } +_CHARSET_ALL = [(NEGATE, None)] + def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=_parser.TYPE_FLAGS): if add_flags & TYPE_FLAGS: @@ -84,25 +86,28 @@ def _compile(code, pattern, flags): code[skip] = _len(code) - skip elif op is IN: charset, hascased = _optimize_charset(av, iscased, tolower, fixes) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - emit(IN_LOC_IGNORE) - elif not hascased: - emit(IN) - elif not fixes: # ascii - emit(IN_IGNORE) + if not charset: + emit(FAILURE) + elif charset == _CHARSET_ALL: + emit(ANY_ALL) else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - _compile_charset(charset, flags, code) - code[skip] = _len(code) - skip + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + emit(IN_LOC_IGNORE) + elif not hascased: + emit(IN) + elif not fixes: # ascii + emit(IN_IGNORE) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + _compile_charset(charset, flags, code) + code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(ANY_ALL) else: emit(ANY) elif op in REPEATING_CODES: - if flags & SRE_FLAG_TEMPLATE: - raise error("internal: unsupported template operator %r" % (op,)) if _simple(av[2]): emit(REPEATING_CODES[op][2]) skip = _len(code); emit(0) @@ -152,7 +157,7 @@ def _compile(code, pattern, flags): if lo > MAXCODE: raise error("looks too much behind") if lo != hi: - raise error("look-behind requires fixed-width pattern") + raise PatternError("look-behind requires fixed-width pattern") emit(lo) # look behind _compile(code, av[1], flags) emit(SUCCESS) @@ -211,7 +216,7 @@ def _compile(code, pattern, flags): else: code[skipyes] = _len(code) - skipyes + 1 else: - raise error("internal: unsupported operand type %r" % (op,)) + raise PatternError(f"internal: unsupported operand type {op!r}") def _compile_charset(charset, flags, code): # compile charset subprogram @@ -237,7 +242,7 @@ def _compile_charset(charset, flags, code): else: emit(av) else: - raise error("internal: unsupported set operator %r" % (op,)) + raise PatternError(f"internal: unsupported set operator {op!r}") emit(FAILURE) def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): @@ -250,11 +255,11 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): while True: try: if op is LITERAL: - if fixup: - lo = fixup(av) - charmap[lo] = 1 - if fixes and lo in fixes: - for k in fixes[lo]: + if fixup: # IGNORECASE and not LOCALE + av = fixup(av) + charmap[av] = 1 + if fixes and av in fixes: + for k in fixes[av]: charmap[k] = 1 if not hascased and iscased(av): hascased = True @@ -262,7 +267,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) - if fixup: + if fixup: # IGNORECASE and not LOCALE if fixes: for i in map(fixup, r): charmap[i] = 1 @@ -279,6 +284,10 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[i] = 1 elif op is NEGATE: out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False else: tail.append((op, av)) except IndexError: @@ -289,8 +298,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Character set contains non-BMP character codes. # For range, all BMP characters in the range are already # proceeded. - if fixup: - hascased = True + if fixup: # IGNORECASE and not LOCALE # For now, IN_UNI_IGNORE+LITERAL and # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP # characters, because two characters (at least one of @@ -301,7 +309,13 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Also, both c.lower() and c.lower().upper() are single # characters for every non-BMP character. if op is RANGE: - op = RANGE_UNI_IGNORE + if fixes: # not ASCII + op = RANGE_UNI_IGNORE + hascased = True + else: + assert op is LITERAL + if not hascased and iscased(av): + hascased = True tail.append((op, av)) break @@ -521,13 +535,18 @@ def _compile_info(code, pattern, flags): # look for a literal prefix prefix = [] prefix_skip = 0 - charset = [] # not used + charset = None # not used if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): # look for literal prefix prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) # if no prefix, look for charset prefix if not prefix: charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -562,8 +581,6 @@ def _compile_info(code, pattern, flags): # generate overlap table code.extend(_generate_overlap_table(prefix)) elif charset: - charset, hascased = _optimize_charset(charset) - assert not hascased _compile_charset(charset, flags, code) code[skip] = len(code) - skip @@ -763,4 +780,3 @@ def compile(p, flags=0): p.state.groups-1, groupindex, tuple(indexgroup) ) - diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 92494e385cf..d6f32302d37 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -13,14 +13,14 @@ # update when constants are added or removed -MAGIC = 20221023 +MAGIC = 20230612 -from _sre import MAXREPEAT, MAXGROUPS +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 # SRE standard exception (access as sre.error) # should this really be here? -class error(Exception): +class PatternError(Exception): """Exception raised for invalid regular expressions. Attributes: @@ -53,6 +53,9 @@ def __init__(self, msg, pattern=None, pos=None): super().__init__(msg) +# Backward compatibility after renaming in 3.13 +error = PatternError + class _NamedIntConstant(int): def __new__(cls, value, name): self = super(_NamedIntConstant, cls).__new__(cls, value) @@ -203,8 +206,9 @@ def _makecodes(*names): CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } +CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2])) + # flags -SRE_FLAG_TEMPLATE = 1 # template mode (unknown purpose, deprecated) SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_MULTILINE = 8 # treat target as multiline string @@ -218,4 +222,3 @@ def _makecodes(*names): SRE_INFO_PREFIX = 1 # has prefix SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) SRE_INFO_CHARSET = 4 # pattern starts with character from given set -RE_INFO_CHARSET = 4 # pattern starts with character from given set diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index 4a492b79e84..35ab7ede2a7 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -49,7 +49,8 @@ r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), - r"\Z": (AT, AT_END_STRING), # end of string + r"\z": (AT, AT_END_STRING), # end of string + r"\Z": (AT, AT_END_STRING), # end of string (obsolete) } FLAGS = { @@ -61,12 +62,11 @@ "x": SRE_FLAG_VERBOSE, # extensions "a": SRE_FLAG_ASCII, - "t": SRE_FLAG_TEMPLATE, "u": SRE_FLAG_UNICODE, } TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE -GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE +GLOBAL_FLAGS = SRE_FLAG_DEBUG # Maximal value returned by SubPattern.getwidth(). # Must be larger than MAXREPEAT, MAXCODE and sys.maxsize. @@ -781,8 +781,10 @@ def _parse(source, state, verbose, nested, first=False): source.tell() - start) if char == "=": subpatternappend((ASSERT, (dir, p))) - else: + elif p: subpatternappend((ASSERT_NOT, (dir, p))) + else: + subpatternappend((FAILURE, ())) continue elif char == "(": @@ -806,14 +808,6 @@ def _parse(source, state, verbose, nested, first=False): state.grouprefpos[condgroup] = ( source.tell() - len(condname) - 1 ) - if not (condname.isdecimal() and condname.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(condname) if source.istext else ascii(condname), - source.tell() - len(condname) - 1), - DeprecationWarning, stacklevel=nested + 6 - ) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): @@ -1037,14 +1031,6 @@ def addgroup(index, pos): if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) - if not (name.isdecimal() and name.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(name) if s.istext else ascii(name), - s.tell() - len(name) - 1), - DeprecationWarning, stacklevel=5 - ) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 19dbe3a07eb..ab18247682b 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -28,7 +28,7 @@ def wrapper(self): wrapper.__doc__ = getattr(user_function, '__doc__') wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') - wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__annotate__ = getattr(user_function, '__annotate__', None) wrapper.__type_params__ = getattr(user_function, '__type_params__', ()) wrapper.__wrapped__ = user_function return wrapper @@ -181,7 +181,22 @@ def repr_str(self, x, level): return s def repr_int(self, x, level): - s = builtins.repr(x) # XXX Hope this isn't too slow... + try: + s = builtins.repr(x) + except ValueError as exc: + assert 'sys.set_int_max_str_digits()' in str(exc) + # Those imports must be deferred due to Python's build system + # where the reprlib module is imported before the math module. + import math, sys + # Integers with more than sys.get_int_max_str_digits() digits + # are rendered differently as their repr() raises a ValueError. + # See https://github.com/python/cpython/issues/135487. + k = 1 + int(math.log10(abs(x))) + # Note: math.log10(abs(x)) may be overestimated or underestimated, + # but for simplicity, we do not compute the exact number of digits. + max_digits = sys.get_int_max_str_digits() + return (f'<{x.__class__.__name__} instance with roughly {k} ' + f'digits (limit at {max_digits}) at 0x{id(x):x}>') if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) diff --git a/Lib/shelve.py b/Lib/shelve.py index 5d443a0fa8d..50584716e9e 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,7 +56,7 @@ the persistent dictionary on disk, if feasible). """ -from pickle import Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler from io import BytesIO import collections.abc @@ -85,7 +85,7 @@ def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8"): self.dict = dict if protocol is None: - protocol = 3 + protocol = DEFAULT_PROTOCOL self._protocol = protocol self.writeback = writeback self.cache = {} @@ -226,6 +226,13 @@ def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + def clear(self): + """Remove all items from the shelf.""" + # Call through to the clear method on dbm-backed shelves. + # see https://github.com/python/cpython/issues/107089 + self.cache.clear() + self.dict.clear() + def open(filename, flag='c', protocol=None, writeback=False): """Open a persistent dictionary for reading and writing. diff --git a/Lib/shlex.py b/Lib/shlex.py index f4821616b62..5959f52dd12 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -7,11 +7,7 @@ # iterator interface by Gustavo Niemeyer, April 2003. # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. -import os -import re import sys -from collections import deque - from io import StringIO __all__ = ["shlex", "split", "quote", "join"] @@ -20,6 +16,8 @@ class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): + from collections import deque # deferred import for performance + if isinstance(instream, str): instream = StringIO(instream) if instream is not None: @@ -278,6 +276,7 @@ def read_token(self): def sourcehook(self, newfile): "Hook called on a filename to be sourced." + import os.path if newfile[0] == '"': newfile = newfile[1:-1] # This implements cpp-like semantics for relative-path inclusion. @@ -318,13 +317,20 @@ def join(split_command): return ' '.join(quote(arg) for arg in split_command) -_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search - def quote(s): """Return a shell-escaped version of the string *s*.""" if not s: return "''" - if _find_unsafe(s) is None: + + if not isinstance(s, str): + raise TypeError(f"expected string object, got {type(s).__name__!r}") + + # Use bytes.translate() for performance + safe_chars = (b'%+,-./0123456789:=@' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' + b'abcdefghijklmnopqrstuvwxyz') + # No quoting is needed if `s` is an ASCII string consisting only of `safe_chars` + if s.isascii() and not s.encode().translate(None, delete=safe_chars): return s # use single quotes, and put single quotes into double quotes diff --git a/Lib/shutil.py b/Lib/shutil.py index 6803ee3ce6e..8d8fe145567 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -10,7 +10,6 @@ import fnmatch import collections import errno -import warnings try: import zlib @@ -33,6 +32,13 @@ except ImportError: _LZMA_SUPPORTED = False +try: + from compression import zstd + del zstd + _ZSTD_SUPPORTED = True +except ImportError: + _ZSTD_SUPPORTED = False + _WINDOWS = os.name == 'nt' posix = nt = None if os.name == 'posix': @@ -45,10 +51,12 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android", "sunos"))) +_USE_CP_COPY_FILE_RANGE = hasattr(os, "copy_file_range") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -56,7 +64,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -74,8 +82,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -109,10 +115,70 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): else: raise err from None +def _determine_linux_fastcopy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + +def _fastcopy_copy_file_range(fsrc, fdst): + """Copy data from one regular mmap-like fd to another by using + a high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side copy. + + This should work on Linux >= 4.5 only. + """ + try: + infd = fsrc.fileno() + outfd = fdst.fileno() + except Exception as err: + raise _GiveupOnFastCopy(err) # not a regular file + + blocksize = _determine_linux_fastcopy_blocksize(infd) + offset = 0 + while True: + try: + n_copied = os.copy_file_range(infd, outfd, blocksize, offset_dst=offset) + except OSError as err: + # ...in oder to have a more informative exception. + err.filename = fsrc.name + err.filename2 = fdst.name + + if err.errno == errno.ENOSPC: # filesystem is full + raise err from None + + # Give up on first call and if no data was copied. + if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0: + raise _GiveupOnFastCopy(err) + + raise err + else: + if n_copied == 0: + # If no bytes have been copied yet, copy_file_range + # might silently fail. + # https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 + if offset == 0: + raise _GiveupOnFastCopy() + break + offset += n_copied + def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -130,39 +196,24 @@ def _fastcopy_sendfile(fsrc, fdst): except Exception as err: raise _GiveupOnFastCopy(err) # not a regular file - # Hopefully the whole file will be copied in a single call. - # sendfile() is called in a loop 'till EOF is reached (0 return) - # so a bufsize smaller or bigger than the actual file size - # should not make any difference, also in case the file content - # changes while being copied. - try: - blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB - except OSError: - blocksize = 2 ** 27 # 128MiB - # On 32-bit architectures truncate to 1GiB to avoid OverflowError, - # see bpo-38319. - if sys.maxsize < 2 ** 32: - blocksize = min(blocksize, 2 ** 30) - + blocksize = _determine_linux_fastcopy_blocksize(infd) offset = 0 while True: try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOTSOCK: + if err.errno == errno.ENOTSOCK: # sendfile() on this platform (probably Linux < 2.6.33) # does not support copies between regular files (only # sockets). _USE_CP_SENDFILE = False raise _GiveupOnFastCopy(err) - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOSPC: # filesystem is full + if err.errno == errno.ENOSPC: # filesystem is full raise err from None # Give up on first call and if no data was copied. @@ -269,13 +320,21 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux - elif _USE_CP_SENDFILE: - try: - _fastcopy_sendfile(fsrc, fdst) - return dst - except _GiveupOnFastCopy: - pass + # Linux / Android / Solaris + elif _USE_CP_SENDFILE or _USE_CP_COPY_FILE_RANGE: + # reflink may be implicit in copy_file_range. + if _USE_CP_COPY_FILE_RANGE: + try: + _fastcopy_copy_file_range(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + if _USE_CP_SENDFILE: + try: + _fastcopy_sendfile(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass # Windows, see: # https://github.com/python/cpython/pull/7160#discussion_r195405230 elif _WINDOWS and file_size > 0: @@ -304,16 +363,17 @@ def copymode(src, dst, *, follow_symlinks=True): sys.audit("shutil.copymode", src, dst) if not follow_symlinks and _islink(src) and os.path.islink(dst): - if os.name == 'nt': - stat_func, chmod_func = os.lstat, os.chmod - elif hasattr(os, 'lchmod'): + if hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod else: return else: + stat_func = _stat if os.name == 'nt' and os.path.islink(dst): - dst = os.path.realpath(dst, strict=True) - stat_func, chmod_func = _stat, os.chmod + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -388,16 +448,8 @@ def lookup(name): # We must copy extended attributes before the file is (potentially) # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. _copyxattr(src, dst, follow_symlinks=follow) - _chmod = lookup("chmod") - if os.name == 'nt': - if follow: - if os.path.islink(dst): - dst = os.path.realpath(dst, strict=True) - else: - def _chmod(*args, **kwargs): - os.chmod(*args) try: - _chmod(dst, mode, follow_symlinks=follow) + lookup("chmod")(dst, mode, follow_symlinks=follow) except NotImplementedError: # if we got a NotImplementedError, it's because # * follow_symlinks=False, @@ -565,7 +617,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -605,123 +657,155 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - def _rmtree_islink(path): - try: - st = os.lstat(path) - return (stat.S_ISLNK(st.st_mode) or - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False + def _rmtree_islink(st): + return (stat.S_ISLNK(st.st_mode) or + (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT + and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) else: - def _rmtree_islink(path): - return os.path.islink(path) + def _rmtree_islink(st): + return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onexc): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) + st = os.lstat(path) except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") + except OSError as err: + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, fullname, err) + os.rmdir(fullname) + except FileNotFoundError: continue - _rmtree_unsafe(fullname, onexc) - else: + except OSError as err: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) try: os.rmdir(path) + except FileNotFoundError: + pass except OSError as err: onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] try: - with os.scandir(topfd) as scandir_it: - entries = list(scandir_it) - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue try: - dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) - dirfd_closed = False + os.close(fd) except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - except OSError as err: - # close() should not be retried after an error. - dirfd_closed = True - onexc(os.close, fullname, err) - dirfd_closed = True - try: - os.rmdir(entry.name, dir_fd=topfd) - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - try: - os.close(dirfd) - except OSError as err: - onexc(os.close, fullname, err) + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name + try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. + with os.scandir(topfd) as scandir_it: + entries = list(scandir_it) + for entry in entries: + fullname = os.path.join(path, entry.name) + try: + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue + except FileNotFoundError: + continue + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. @@ -765,61 +849,7 @@ def onexc(*args): exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except Exception as err: - onexc(os.lstat, path, err) - return - try: - fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) - fd_closed = False - except Exception as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - except OSError as err: - # close() should not be retried after an error. - fd_closed = True - onexc(os.close, path, err) - fd_closed = True - try: - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - finally: - if not fd_closed: - try: - os.close(fd) - except OSError as err: - onexc(os.close, path, err) - else: - if dir_fd is not None: - raise NotImplementedError("dir_fd unavailable on this platform") - try: - if _rmtree_islink(path): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - # can't continue even if onexc hook returns - return - return _rmtree_unsafe(path, onexc) + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform @@ -964,14 +994,14 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", or None. + 'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", or ".xz"). + the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst"). Returns the output filename. """ @@ -983,6 +1013,8 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, tar_compression = 'bz2' elif _LZMA_SUPPORTED and compress == 'xz': tar_compression = 'xz' + elif _ZSTD_SUPPORTED and compress == 'zst': + tar_compression = 'zst' else: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) @@ -1111,6 +1143,10 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _ARCHIVE_FORMATS['zstdtar'] = (_make_tarball, [('compress', 'zst')], + "zstd'ed tar-file") + def get_archive_formats(): """Returns a list of supported formats for archiving and unarchiving. @@ -1151,7 +1187,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", or "xztar". Or any other registered format. + "bztar", "xztar", or "zstdtar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -1301,7 +1337,7 @@ def _unpack_zipfile(filename, extract_dir): zip.close() def _unpack_tarfile(filename, extract_dir, *, filter=None): - """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` + """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: @@ -1336,6 +1372,10 @@ def _unpack_tarfile(filename, extract_dir, *, filter=None): _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _UNPACK_FORMATS['zstdtar'] = (['.tar.zst', '.tzst'], _unpack_tarfile, [], + "zstd'ed tar-file") + def _find_unpack_format(filename): for name, info in _UNPACK_FORMATS.items(): for extension in info[0]: @@ -1352,7 +1392,7 @@ def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", - or "xztar". Or any other registered format. If not provided, + "xztar", or "zstdtar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. @@ -1428,11 +1468,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1458,7 +1505,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. @@ -1575,21 +1623,21 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if sys.platform == "win32": # PATHEXT is necessary to check on Windows. pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT - pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] + pathext = pathext_source.split(os.pathsep) + pathext = [ext.rstrip('.') for ext in pathext if ext] if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - files = ([cmd] + [cmd + ext for ext in pathext]) + files = [cmd + ext for ext in pathext] - # gh-109590. If we are looking for an executable, we need to look - # for a PATHEXT match. The first cmd is the direct match - # (e.g. python.exe instead of python) - # Check that direct match first if and only if the extension is in PATHEXT - # Otherwise check it last - suffix = os.path.splitext(files[0])[1].upper() - if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): - files.append(files.pop(0)) + # If X_OK in mode, simulate the cmd.exe behavior: look at direct + # match if and only if the extension is in PATHEXT. + # If X_OK not in mode, simulate the first result of where.exe: + # always look at direct match before a PATHEXT match. + normcmd = cmd.upper() + if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext): + files.insert(0, cmd) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. @@ -1598,10 +1646,22 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/site.py b/Lib/site.py index 2983ca71544..5305d67b3b8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,7 +73,7 @@ import os import builtins import _sitebuiltins -import io +import _io as io import stat import errno @@ -95,6 +95,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -444,9 +450,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software - Foundation, and a cast of thousands for supporting Python - development. See www.python.org for more information.""") +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -574,7 +580,7 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass @@ -626,17 +632,17 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths @@ -646,7 +652,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -666,7 +672,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100644 new mode 100755 index 912233d8176..9b81bcfbc41 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -171,7 +171,7 @@ def quotedata(data): internet CRLF end-of-line. """ return re.sub(r'(?m)^\.', '..', - re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) + re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) def _quote_periods(bindata): return re.sub(br'(?m)^\.', b'..', bindata) @@ -179,6 +179,16 @@ def _quote_periods(bindata): def _fix_eols(data): return re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data) + +try: + hmac.digest(b'', b'', 'md5') +# except ValueError: +except (ValueError, AttributeError): # TODO: RUSTPYTHON + _have_cram_md5_support = False +else: + _have_cram_md5_support = True + + try: import ssl except ImportError: @@ -475,7 +485,7 @@ def ehlo(self, name=''): if auth_match: # This doesn't remove duplicates, but that's no problem self.esmtp_features["auth"] = self.esmtp_features.get("auth", "") \ - + " " + auth_match.groups(0)[0] + + " " + auth_match.groups(0)[0] continue # RFC 1869 requires a space between ehlo keyword and parameters. @@ -488,7 +498,7 @@ def ehlo(self, name=''): params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ - + " " + params + + " " + params else: self.esmtp_features[feature] = params return (code, msg) @@ -542,7 +552,7 @@ def mail(self, sender, options=()): raise SMTPNotSupportedError( 'SMTPUTF8 not supported by server') optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) + self.putcmd("mail", "from:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() def rcpt(self, recip, options=()): @@ -550,7 +560,7 @@ def rcpt(self, recip, options=()): optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) + self.putcmd("rcpt", "to:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() def data(self, msg): @@ -667,8 +677,11 @@ def auth_cram_md5(self, challenge=None): # CRAM-MD5 does not support initial-response. if challenge is None: return None - return self.user + " " + hmac.HMAC( - self.password.encode('ascii'), challenge, 'md5').hexdigest() + if not _have_cram_md5_support: + raise SMTPException("CRAM-MD5 is not supported") + password = self.password.encode('ascii') + authcode = hmac.HMAC(password, challenge, 'md5') + return f"{self.user} {authcode.hexdigest()}" def auth_plain(self, challenge=None): """ Authobject to use with PLAIN authentication. Requires self.user and @@ -720,8 +733,10 @@ def login(self, user, password, *, initial_response_ok=True): advertised_authlist = self.esmtp_features["auth"].split() # Authentication methods we can handle in our preferred order: - preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] - + if _have_cram_md5_support: + preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] + else: + preferred_auths = ['PLAIN', 'LOGIN'] # We try the supported authentications in our preferred order, if # the server supports them. authlist = [auth for auth in preferred_auths @@ -905,7 +920,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -919,7 +934,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -938,10 +953,10 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] - if (header_prefix + 'Sender') in msg - else msg[header_prefix + 'From']) + if (header_prefix + 'Sender') in msg + else msg[header_prefix + 'From']) from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], diff --git a/Lib/socket.py b/Lib/socket.py index 42ee1307732..727b0e75f03 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -110,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -306,7 +309,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: @@ -347,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -548,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -569,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, @@ -591,16 +598,65 @@ def fromshare(info): return socket(0, 0, 0, info) __all__.append("fromshare") -if hasattr(_socket, "socketpair"): +# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +# This is used if _socket doesn't natively provide socketpair. It's +# always defined so that it can be patched in for testing purposes. +def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen() + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() - def socketpair(family=None, type=SOCK_STREAM, proto=0): - """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise - Create a pair of socket objects from the sockets returned by the platform - socketpair() function. - The arguments are the same as for socket() except the default family is - AF_UNIX if defined on the platform; otherwise, the default is AF_INET. - """ + return (ssock, csock) + +if hasattr(_socket, "socketpair"): + def socketpair(family=None, type=SOCK_STREAM, proto=0): if family is None: try: family = AF_UNIX @@ -612,44 +668,7 @@ def socketpair(family=None, type=SOCK_STREAM, proto=0): return a, b else: - - # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. - def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): - if family == AF_INET: - host = _LOCALHOST - elif family == AF_INET6: - host = _LOCALHOST_V6 - else: - raise ValueError("Only AF_INET and AF_INET6 socket address families " - "are supported") - if type != SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with - # setblocking(False) that prevents us from having to create a thread. - lsock = socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen() - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) + socketpair = _fallback_socketpair __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) @@ -702,16 +721,15 @@ def readinto(self, b): self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - if e.errno in _blocking_errnos: - return None - raise + try: + return self._sock.recv_into(b) + except timeout: + self._timeout_occurred = True + raise + except error as e: + if e.errno in _blocking_errnos: + return None + raise def write(self, b): """Write the given bytes or bytearray object *b* to the socket @@ -919,7 +937,9 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # Fail later on bind(), for platforms which may not # support this option. pass - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and family in (AF_INET, AF_INET6): sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1) if has_ipv6 and family == AF_INET6: if dualstack_ipv6: diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 2905e3eac36..35b2723de3b 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -127,10 +127,7 @@ class will essentially render the service "deaf" while one request is import selectors import os import sys -try: - import threading -except ImportError: - import dummy_threading as threading +import threading from io import BufferedIOBase from time import monotonic as time @@ -144,6 +141,8 @@ class will essentially render the service "deaf" while one request is __all__.extend(["UnixStreamServer","UnixDatagramServer", "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) + if hasattr(os, "fork"): + __all__.extend(["ForkingUnixStreamServer", "ForkingUnixDatagramServer"]) # poll/select have the advantage of not requiring any extra file descriptor, # contrarily to epoll/kqueue (also, they require a single syscall). @@ -190,6 +189,7 @@ class BaseServer: - address_family - socket_type - allow_reuse_address + - allow_reuse_port Instance variables: @@ -294,8 +294,7 @@ def handle_request(self): selector.register(self, selectors.EVENT_READ) while True: - ready = selector.select(timeout) - if ready: + if selector.select(timeout): return self._handle_request_noblock() else: if timeout is not None: @@ -428,6 +427,7 @@ class TCPServer(BaseServer): - socket_type - request_queue_size (only for stream sockets) - allow_reuse_address + - allow_reuse_port Instance variables: @@ -445,6 +445,8 @@ class TCPServer(BaseServer): allow_reuse_address = False + allow_reuse_port = False + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): """Constructor. May be extended, do not override.""" BaseServer.__init__(self, server_address, RequestHandlerClass) @@ -464,8 +466,15 @@ def server_bind(self): May be overridden. """ - if self.allow_reuse_address: + if self.allow_reuse_address and hasattr(socket, "SO_REUSEADDR"): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if ( + self.allow_reuse_port and hasattr(socket, "SO_REUSEPORT") + and self.address_family in (socket.AF_INET, socket.AF_INET6) + ): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() @@ -522,6 +531,8 @@ class UDPServer(TCPServer): allow_reuse_address = False + allow_reuse_port = False + socket_type = socket.SOCK_DGRAM max_packet_size = 8192 @@ -723,6 +734,11 @@ class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass + if hasattr(os, "fork"): + class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): pass + + class ForkingUnixDatagramServer(ForkingMixIn, UnixDatagramServer): pass + class BaseRequestHandler: """Base class for request handler classes. diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b9..ed727fae609 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -22,7 +22,7 @@ """ The sqlite3 extension module provides a DB-API 2.0 (PEP 249) compliant -interface to the SQLite library, and requires SQLite 3.7.15 or newer. +interface to the SQLite library, and requires SQLite 3.15.2 or newer. To use the module, start by creating a database Connection object: @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index f8a5cca24e5..4ccf292ddf2 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -46,26 +46,34 @@ def runsource(self, source, filename="", symbol="single"): """Override runsource, the core of the InteractiveConsole REPL. Return True if more input is needed; buffering is done automatically. - Return False is input is a complete statement ready for execution. + Return False if input is a complete statement ready for execution. """ - match source: - case ".version": - print(f"{sqlite3.sqlite_version}") - case ".help": - print("Enter SQL code and press enter.") - case ".quit": - sys.exit(0) - case _: - if not sqlite3.complete_statement(source): - return True - execute(self._cur, source) + if not source or source.isspace(): + return False + if source[0] == ".": + match source[1:].strip(): + case "version": + print(f"{sqlite3.sqlite_version}") + case "help": + print("Enter SQL code and press enter.") + case "quit": + sys.exit(0) + case "": + pass + case _ as unknown: + self.write("Error: unknown command or invalid arguments:" + f' "{unknown}".\n') + else: + if not sqlite3.complete_statement(source): + return True + execute(self._cur, source) return False -def main(): +def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", - prog="python -m sqlite3", + color=True, ) parser.add_argument( "filename", type=str, default=":memory:", nargs="?", @@ -86,7 +94,7 @@ def main(): version=f"SQLite version {sqlite3.sqlite_version}", help="Print underlying SQLite library version", ) - args = parser.parse_args() + args = parser.parse_args(*args) if args.filename == ":memory:": db_name = "a transient in-memory database" @@ -94,12 +102,16 @@ def main(): db_name = repr(args.filename) # Prepare REPL banner and prompts. + if sys.platform == "win32" and "idlelib.run" not in sys.modules: + eofkey = "CTRL-Z" + else: + eofkey = "CTRL-D" banner = dedent(f""" sqlite3 shell, running on SQLite version {sqlite3.sqlite_version} Connected to {db_name} Each command will be run using execute() on the cursor. - Type ".help" for more information; type ".quit" or CTRL-D to quit. + Type ".help" for more information; type ".quit" or {eofkey} to quit. """).strip() sys.ps1 = "sqlite> " sys.ps2 = " ... " @@ -112,9 +124,16 @@ def main(): else: # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) + try: + import readline # noqa: F401 + except ImportError: + pass console.interact(banner, exitmsg="") finally: con.close() + sys.exit(0) + -main() +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c..0315760516e 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 07b9da10b92..57e6a3b4f1e 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -7,7 +7,15 @@ # future enhancements, you should normally quote any identifier that # is an English language word, even if you do not have to." -def _iterdump(connection): +def _quote_name(name): + return '"{0}"'.format(name.replace('"', '""')) + + +def _quote_value(value): + return "'{0}'".format(value.replace("'", "''")) + + +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -16,64 +24,87 @@ def _iterdump(connection): directly but instead called from the Connection method, iterdump(). """ + writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. + # Disable foreign key constraints, if there is any foreign key violation. + violations = cu.execute("PRAGMA foreign_key_check").fetchall() + if violations: + yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': - rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() + rows = cu.execute('SELECT * FROM "sqlite_sequence";') sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] sqlite_sequence += [ - f'INSERT INTO "sqlite_sequence" VALUES(\'{row[0]}\',{row[1]})' - for row in rows + f'INSERT INTO "sqlite_sequence" VALUES({_quote_value(table_name)},{seq_value})' + for table_name, seq_value in rows.fetchall() ] continue elif table_name == 'sqlite_stat1': yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'): continue - # NOTE: Virtual table support not implemented - #elif sql.startswith('CREATE VIRTUAL TABLE'): - # qtable = table_name.replace("'", "''") - # yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"\ - # "VALUES('table','{0}','{0}',0,'{1}');".format( - # qtable, - # sql.replace("''"))) + elif sql.startswith('CREATE VIRTUAL TABLE'): + if not writeable_schema: + writeable_schema = True + yield('PRAGMA writable_schema=ON;') + yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table',{0},{0},0,{1});".format( + _quote_value(table_name), + _quote_value(sql), + )) else: yield('{0};'.format(sql)) # Build the insert statement for each row of the current table - table_name_ident = table_name.replace('"', '""') - res = cu.execute('PRAGMA table_info("{0}")'.format(table_name_ident)) + table_name_ident = _quote_name(table_name) + res = cu.execute(f'PRAGMA table_info({table_name_ident})') column_names = [str(table_info[1]) for table_info in res.fetchall()] - q = """SELECT 'INSERT INTO "{0}" VALUES({1})' FROM "{0}";""".format( + q = "SELECT 'INSERT INTO {0} VALUES('{1}')' FROM {0};".format( table_name_ident, - ",".join("""'||quote("{0}")||'""".format(col.replace('"', '""')) for col in column_names)) + "','".join( + "||quote({0})||".format(_quote_name(col)) for col in column_names + ) + ) query_res = cu.execute(q) for row in query_res: yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) + if writeable_schema: + yield('PRAGMA writable_schema=OFF;') + # gh-79009: Yield statements concerning the sqlite_sequence table at the # end of the transaction. for row in sqlite_sequence: diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index 8543e2bc8c0..fa09d044292 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -5,81 +5,3 @@ from re import _constants as _ globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) - -if __name__ == "__main__": - def dump(f, d, typ, int_t, prefix): - items = sorted(d) - f.write(f"""\ -#[derive(num_enum::TryFromPrimitive, Debug)] -#[repr({int_t})] -#[allow(non_camel_case_types, clippy::upper_case_acronyms)] -pub enum {typ} {{ -""") - for item in items: - name = str(item).removeprefix(prefix) - val = int(item) - f.write(f" {name} = {val},\n") - f.write("""\ -} -""") - import sys - if len(sys.argv) > 1: - constants_file = sys.argv[1] - else: - import os - constants_file = os.path.join(os.path.dirname(__file__), "../../sre-engine/src/constants.rs") - with open(constants_file, "w") as f: - f.write("""\ -/* - * Secret Labs' Regular Expression Engine - * - * regular expression matching engine - * - * NOTE: This file is generated by sre_constants.py. If you need - * to change anything in here, edit sre_constants.py and run it. - * - * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. - * - * See the _sre.c file for information on usage and redistribution. - */ - -""") - - f.write("use bitflags::bitflags;\n\n"); - - f.write("pub const SRE_MAGIC: usize = %d;\n" % MAGIC) - - dump(f, OPCODES, "SreOpcode", "u32", "") - dump(f, ATCODES, "SreAtCode", "u32", "AT_") - dump(f, CHCODES, "SreCatCode", "u32", "CATEGORY_") - - def bitflags(typ, int_t, prefix, flags): - f.write(f"""\ -bitflags! {{ - pub struct {typ}: {int_t} {{ -""") - for name in flags: - val = globals()[prefix + name] - f.write(f" const {name} = {val};\n") - f.write("""\ - } -} -""") - - bitflags("SreFlag", "u16", "SRE_FLAG_", [ - "TEMPLATE", - "IGNORECASE", - "LOCALE", - "MULTILINE", - "DOTALL", - "UNICODE", - "VERBOSE", - "DEBUG", - "ASCII", - ]) - - bitflags("SreInfo", "u32", "SRE_INFO_", [ - "PREFIX", "LITERAL", "CHARSET", - ]) - - print("done") diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc94..81f694329bf 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -166,9 +166,14 @@ def filemode(mode): perm = [] for index, table in enumerate(_filemode_table): for bit, char in table: - if mode & bit == bit: - perm.append(char) - break + if index == 0: + if S_IFMT(mode) == bit: + perm.append(char) + break + else: + if mode & bit == bit: + perm.append(char) + break else: if index == 0: # Unknown filetype diff --git a/Lib/statistics.py b/Lib/statistics.py index ad4a94219cf..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -138,7 +138,7 @@ from decimal import Decimal from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter @@ -147,447 +147,149 @@ _SQRT2 = sqrt(2.0) _random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. - Ties are averaged so that equal values receive the same rank: +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - The operation is idempotent: + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60]) + 48.0 - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: + If ``data`` is empty, or any element is less than zero, + ``harmonic_mean`` will raise ``StatisticsError``. - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - If ``data`` is empty, or any element is less than zero, - ``harmonic_mean`` will raise ``StatisticsError``. """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -599,6 +301,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -608,16 +311,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -654,6 +360,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -797,6 +506,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -805,347 +515,48 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -def kde(data, h, kernel='normal', *, cumulative=False): - """Kernel Density Estimation: Create a continuous probability density - function or cumulative distribution function from discrete samples. +## Measures of spread ###################################################### - The basic idea is to smooth the data using a kernel function - to help draw inferences about a population from a sample. +def variance(data, xbar=None): + """Return the sample variance of data. - The degree of smoothing is controlled by the scaling parameter h - which is called the bandwidth. Smaller values emphasize local - features while larger values give smoother results. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. - The kernel determines the relative weights of the sample data - points. Generally, the choice of kernel shape does not matter - as much as the more influential bandwidth smoothing parameter. + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. - Kernels that give some weight to every sample point: + Examples: - normal (gauss) - logistic - sigmoid + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Kernels that only give weight to sample points within - the bandwidth: + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - rectangular (uniform) - triangular - parabolic (epanechnikov) - quartic (biweight) - triweight - cosine + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - If *cumulative* is true, will return a cumulative distribution function. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. - A StatisticsError will be raised if the data sequence is empty. + Decimals and Fractions are supported: - Example - ------- + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') - Given a sample of six data points, construct a continuous - function that estimates the underlying probability density: + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde(sample, h=1.5) - - Compute the area under the curve: - - >>> area = sum(f_hat(x) for x in range(-20, 20)) - >>> round(area, 4) - 1.0 - - Plot the estimated probability density function at - evenly spaced points from -6 to 10: - - >>> for x in range(-6, 11): - ... density = f_hat(x) - ... plot = ' ' * int(density * 400) + 'x' - ... print(f'{x:2}: {density:.3f} {plot}') - ... - -6: 0.002 x - -5: 0.009 x - -4: 0.031 x - -3: 0.070 x - -2: 0.111 x - -1: 0.125 x - 0: 0.110 x - 1: 0.086 x - 2: 0.068 x - 3: 0.059 x - 4: 0.066 x - 5: 0.082 x - 6: 0.082 x - 7: 0.058 x - 8: 0.028 x - 9: 0.009 x - 10: 0.002 x - - Estimate P(4.5 < X <= 7.5), the probability that a new sample value - will be between 4.5 and 7.5: - - >>> cdf = kde(sample, h=1.5, cumulative=True) - >>> round(cdf(7.5) - cdf(4.5), 2) - 0.22 - - References - ---------- - - Kernel density estimation and its application: - https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - - Kernel functions in common use: - https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - - Interactive graphical demonstration and exploration: - https://demonstrations.wolfram.com/KernelDensityEstimation/ - - Kernel estimation of cumulative distribution function of a random variable with bounded support - https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - - """ - - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - match kernel: - - case 'normal' | 'gauss': - sqrt2pi = sqrt(2 * pi) - sqrt2 = sqrt(2) - K = lambda t: exp(-1/2 * t * t) / sqrt2pi - W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) - support = None - - case 'logistic': - # 1.0 / (exp(t) + 2.0 + exp(-t)) - K = lambda t: 1/2 / (1.0 + cosh(t)) - W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) - support = None - - case 'sigmoid': - # (2/pi) / (exp(t) + exp(-t)) - c1 = 1 / pi - c2 = 2 / pi - K = lambda t: c1 / cosh(t) - W = lambda t: c2 * atan(exp(t)) - support = None - - case 'rectangular' | 'uniform': - K = lambda t: 1/2 - W = lambda t: 1/2 * t + 1/2 - support = 1.0 - - case 'triangular': - K = lambda t: 1.0 - abs(t) - W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 - support = 1.0 - - case 'parabolic' | 'epanechnikov': - K = lambda t: 3/4 * (1.0 - t * t) - W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 - support = 1.0 - - case 'quartic' | 'biweight': - K = lambda t: 15/16 * (1.0 - t * t) ** 2 - W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 - support = 1.0 - - case 'triweight': - K = lambda t: 35/32 * (1.0 - t * t) ** 3 - W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 - support = 1.0 - - case 'cosine': - c1 = pi / 4 - c2 = pi / 2 - K = lambda t: c1 * cos(c2 * t) - W = lambda t: 1/2 * sin(c2 * t) + 1/2 - support = 1.0 - - case _: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - if support is None: - - def pdf(x): - n = len(data) - return sum(K((x - x_i) / h) for x_i in data) / (n * h) - - def cdf(x): - n = len(data) - return sum(W((x - x_i) / h) for x_i in data) / n - - else: - - sample = sorted(data) - bandwidth = h * support - - def pdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - - def cdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum((W((x - x_i) / h) for x_i in supported), i) / n - - if cumulative: - cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' - return cdf - - else: - pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' - return pdf - - -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. - -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". - -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. - -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." - -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. - -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. - - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html + # http://mathworld.wolfram.com/SampleVariance.html - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. - - Decimals and Fractions are supported: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - """ T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') @@ -1187,6 +598,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -1206,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -1224,51 +642,17 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1287,6 +671,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1320,7 +705,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1328,18 +716,22 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) + else: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1387,381 +779,317 @@ def linear_regression(x, y, /, *, proportional=False): LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] # List because used three times below y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### - +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + neg_sqrt2 = -sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * erfc(t / neg_sqrt2) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support -def _normal_dist_inv_cdf(p, mu, sigma): - # There is no closed-form solution to the inverse CDF for the normal - # distribution, so we use a rational approximation instead: - # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the - # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 - # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. - q = p - 0.5 - if fabs(q) <= 0.425: - r = 0.180625 - q * q - # Hash sum: 55.88319_28806_14901_4439 - num = (((((((2.50908_09287_30122_6727e+3 * r + - 3.34305_75583_58812_8105e+4) * r + - 6.72657_70927_00870_0853e+4) * r + - 4.59219_53931_54987_1457e+4) * r + - 1.37316_93765_50946_1125e+4) * r + - 1.97159_09503_06551_4427e+3) * r + - 1.33141_66789_17843_7745e+2) * r + - 3.38713_28727_96366_6080e+0) * q - den = (((((((5.22649_52788_52854_5610e+3 * r + - 2.87290_85735_72194_2674e+4) * r + - 3.93078_95800_09271_0610e+4) * r + - 2.12137_94301_58659_5867e+4) * r + - 5.39419_60214_24751_1077e+3) * r + - 6.87187_00749_20579_0830e+2) * r + - 4.23133_30701_60091_1252e+1) * r + - 1.0) - x = num / den - return mu + (x * sigma) - r = p if q <= 0.0 else 1.0 - p - r = sqrt(-log(r)) - if r <= 5.0: - r = r - 1.6 - # Hash sum: 49.33206_50330_16102_89036 - num = (((((((7.74545_01427_83414_07640e-4 * r + - 2.27238_44989_26918_45833e-2) * r + - 2.41780_72517_74506_11770e-1) * r + - 1.27045_82524_52368_38258e+0) * r + - 3.64784_83247_63204_60504e+0) * r + - 5.76949_72214_60691_40550e+0) * r + - 4.63033_78461_56545_29590e+0) * r + - 1.42343_71107_49683_57734e+0) - den = (((((((1.05075_00716_44416_84324e-9 * r + - 5.47593_80849_95344_94600e-4) * r + - 1.51986_66563_61645_71966e-2) * r + - 1.48103_97642_74800_74590e-1) * r + - 6.89767_33498_51000_04550e-1) * r + - 1.67638_48301_83803_84940e+0) * r + - 2.05319_16266_37758_82187e+0) * r + - 1.0) - else: - r = r - 5.0 - # Hash sum: 47.52583_31754_92896_71629 - num = (((((((2.01033_43992_92288_13265e-7 * r + - 2.71155_55687_43487_57815e-5) * r + - 1.24266_09473_88078_43860e-3) * r + - 2.65321_89526_57612_30930e-2) * r + - 2.96560_57182_85048_91230e-1) * r + - 1.78482_65399_17291_33580e+0) * r + - 5.46378_49111_64114_36990e+0) * r + - 6.65790_46435_01103_77720e+0) - den = (((((((2.04426_31033_89939_78564e-15 * r + - 1.42151_17583_16445_88870e-7) * r + - 1.84631_83175_10054_68180e-5) * r + - 7.86869_13114_56132_59100e-4) * r + - 1.48753_61290_85061_48525e-2) * r + - 1.36929_88092_27358_05310e-1) * r + - 5.99832_20655_58879_37690e-1) * r + - 1.0) - x = num / den - if q < 0.0: - x = -x - return mu + (x * sigma) +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv +def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign -# If available, use C implementation -try: - from _statistics import _normal_dist_inv_cdf -except ImportError: - pass +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support +def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) + return x * sign -class NormalDist: - "Normal distribution of a random variable" - # https://en.wikipedia.org/wiki/Normal_distribution - # https://en.wikipedia.org/wiki/Variance#Properties +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel - __slots__ = { - '_mu': 'Arithmetic mean of a normal distribution', - '_sigma': 'Standard deviation of a normal distribution', - } - def __init__(self, mu=0.0, sigma=1.0): - "NormalDist where mu is the mean and sigma is the standard deviation." - if sigma < 0.0: - raise StatisticsError('sigma must be non-negative') - self._mu = float(mu) - self._sigma = float(sigma) +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. - @classmethod - def from_samples(cls, data): - "Make a normal distribution instance from sample data." - return cls(*_mean_stdev(data)) + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. - def samples(self, n, *, seed=None): - "Generate *n* samples for a given mean and standard deviation." - rnd = random.random if seed is None else random.Random(seed).random - inv_cdf = _normal_dist_inv_cdf - mu = self._mu - sigma = self._sigma - return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. - def pdf(self, x): - "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma * self._sigma - if not variance: - raise StatisticsError('pdf() not defined when sigma is zero') - diff = x - self._mu - return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. - def cdf(self, x): - "Cumulative distribution function. P(X <= x)" - if not self._sigma: - raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) + Kernels that give some weight to every sample point: - def inv_cdf(self, p): - """Inverse cumulative distribution function. x : P(X <= x) = p + normal (gauss) + logistic + sigmoid - Finds the value of the random variable such that the probability of - the variable being less than or equal to that value equals the given - probability. + Kernels that only give weight to sample points within + the bandwidth: - This function is also called the percent point function or quantile - function. - """ - if p <= 0.0 or p >= 1.0: - raise StatisticsError('p must be in the range 0.0 < p < 1.0') - return _normal_dist_inv_cdf(p, self._mu, self._sigma) + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine - def quantiles(self, n=4): - """Divide into *n* continuous intervals with equal probability. + If *cumulative* is true, will return a cumulative distribution function. - Returns a list of (n - 1) cut points separating the intervals. + A StatisticsError will be raised if the data sequence is empty. - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution in to 100 equal sized groups. - """ - return [self.inv_cdf(i / n) for i in range(1, n)] + Example + ------- - def overlap(self, other): - """Compute the overlapping coefficient (OVL) between two normal distributions. + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving the overlapping area in - the two underlying probability density functions. + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) - >>> N1 = NormalDist(2.4, 1.6) - >>> N2 = NormalDist(3.2, 2.0) - >>> N1.overlap(N2) - 0.8035050657330205 - """ - # See: "The overlapping coefficient as a measure of agreement between - # probability distributions and point estimation of the overlap of two - # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr - # http://dx.doi.org/10.1080/03610928908830127 - if not isinstance(other, NormalDist): - raise TypeError('Expected another NormalDist instance') - X, Y = self, other - if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity - X, Y = Y, X - X_var, Y_var = X.variance, Y.variance - if not X_var or not Y_var: - raise StatisticsError('overlap() not defined when sigma is zero') - dv = Y_var - X_var - dm = fabs(Y._mu - X._mu) - if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) - a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) - x1 = (a + b) / dv - x2 = (a - b) / dv - return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + Compute the area under the curve: - def zscore(self, x): - """Compute the Standard Score. (x - mean) / stdev + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 - Describes *x* in terms of the number of standard deviations - above or below the mean of the normal distribution. - """ - # https://www.statisticshowto.com/probability-and-statistics/z-score/ - if not self._sigma: - raise StatisticsError('zscore() not defined when sigma is zero') - return (x - self._mu) / self._sigma + Plot the estimated probability density function at + evenly spaced points from -6 to 10: - @property - def mean(self): - "Arithmetic mean of the normal distribution." - return self._mu - - @property - def median(self): - "Return the median of the normal distribution" - return self._mu - - @property - def mode(self): - """Return the mode of the normal distribution - - The mode is the value x where which the probability density - function (pdf) takes its maximum value. - """ - return self._mu - - @property - def stdev(self): - "Standard deviation of the normal distribution." - return self._sigma - - @property - def variance(self): - "Square of the standard deviation." - return self._sigma * self._sigma - - def __add__(x1, x2): - """Add a constant or another NormalDist instance. - - If *other* is a constant, translate mu by the constant, - leaving sigma unchanged. - - If *other* is a NormalDist, add both the means and the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu + x2, x1._sigma) - - def __sub__(x1, x2): - """Subtract a constant or another NormalDist instance. - - If *other* is a constant, translate by the constant mu, - leaving sigma unchanged. + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x - If *other* is a NormalDist, subtract the means and add the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu - x2, x1._sigma) + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: - def __mul__(x1, x2): - """Multiply both mu and sigma by a constant. + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + References + ---------- - def __truediv__(x1, x2): - """Divide both mu and sigma by a constant. + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - def __pos__(x1): - "Return a copy of the instance." - return NormalDist(x1._mu, x1._sigma) + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ - def __neg__(x1): - "Negates mu while keeping sigma the same." - return NormalDist(-x1._mu, x1._sigma) + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - __radd__ = __add__ + """ - def __rsub__(x1, x2): - "Subtract a NormalDist from a constant or another NormalDist." - return -(x1 - x2) + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') - __rmul__ = __mul__ + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') - def __eq__(x1, x2): - "Two NormalDist objects are equal if their mu and sigma are both equal." - if not isinstance(x2, NormalDist): - return NotImplemented - return x1._mu == x2._mu and x1._sigma == x2._sigma + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - def __hash__(self): - "NormalDist objects hash equal if their mu and sigma are both equal." - return hash((self._mu, self._sigma)) + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] - def __repr__(self): - return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + if support is None: - def __getstate__(self): - return self._mu, self._sigma + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) - def __setstate__(self, state): - self._mu, self._sigma = state + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + else: -## kde_random() ############################################################## + sample = sorted(data) + bandwidth = h * support -def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): - def f_inv(y): - "Return x such that f(x) ≈ y within the specified tolerance." - x = f_inv_estimate(y) - while abs(diff := f(x) - y) > tolerance: - x -= diff / f_prime(x) - return x - return f_inv + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) -def _quartic_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: - x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) - return x * sign + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n -_quartic_invcdf = _newton_raphson( - f_inv_estimate = _quartic_invcdf_estimate, - f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, - f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf -def _triweight_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.3400218741872791 - 1.0 - return x * sign + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf -_triweight_invcdf = _newton_raphson( - f_inv_estimate = _triweight_invcdf_estimate, - f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, - f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) - -_kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'quartic': _quartic_invcdf, - 'triweight': _triweight_invcdf, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), - 'cosine': lambda p: 2 * asin(2*p - 1) / pi, -} -_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] -_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] -_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] -_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] def kde_random(data, h, kernel='normal', *, seed=None): """Return a function that makes a random selection from the estimated @@ -1791,17 +1119,761 @@ def kde_random(data, h, kernel='normal', *, seed=None): if h <= 0.0: raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - kernel_invcdf = _kernel_invcdfs.get(kernel) - if kernel_invcdf is None: + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] prng = _random.Random(seed) random = prng.random choice = prng.choice def rand(): - return choice(data) + h * kernel_invcdf(random()) + return choice(data) + h * invcdf(random()) rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### + +class NormalDist: + "Normal distribution of a random variable" + # https://en.wikipedia.org/wiki/Normal_distribution + # https://en.wikipedia.org/wiki/Variance#Properties + + __slots__ = { + '_mu': 'Arithmetic mean of a normal distribution', + '_sigma': 'Standard deviation of a normal distribution', + } + + def __init__(self, mu=0.0, sigma=1.0): + "NormalDist where mu is the mean and sigma is the standard deviation." + if sigma < 0.0: + raise StatisticsError('sigma must be non-negative') + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + "Make a normal distribution instance from sample data." + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + "Generate *n* samples for a given mean and standard deviation." + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + + def pdf(self, x): + "Probability density function. P(x <= X < x+dx) / dx" + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError('pdf() not defined when sigma is zero') + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + + def cdf(self, x): + "Cumulative distribution function. P(X <= x)" + if not self._sigma: + raise StatisticsError('cdf() not defined when sigma is zero') + return 0.5 * erfc((self._mu - x) / (self._sigma * _SQRT2)) + + def inv_cdf(self, p): + """Inverse cumulative distribution function. x : P(X <= x) = p + + Finds the value of the random variable such that the probability of + the variable being less than or equal to that value equals the given + probability. + + This function is also called the percent point function or quantile + function. + """ + if p <= 0.0 or p >= 1.0: + raise StatisticsError('p must be in the range 0.0 < p < 1.0') + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + """Divide into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution in to 100 equal sized groups. + """ + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + """Compute the overlapping coefficient (OVL) between two normal distributions. + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving the overlapping area in + the two underlying probability density functions. + + >>> N1 = NormalDist(2.4, 1.6) + >>> N2 = NormalDist(3.2, 2.0) + >>> N1.overlap(N2) + 0.8035050657330205 + """ + # See: "The overlapping coefficient as a measure of agreement between + # probability distributions and point estimation of the overlap of two + # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr + # http://dx.doi.org/10.1080/03610928908830127 + if not isinstance(other, NormalDist): + raise TypeError('Expected another NormalDist instance') + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity + X, Y = Y, X + X_var, Y_var = X.variance, Y.variance + if not X_var or not Y_var: + raise StatisticsError('overlap() not defined when sigma is zero') + dv = Y_var - X_var + dm = fabs(Y._mu - X._mu) + if not dv: + return erfc(dm / (2.0 * X._sigma * _SQRT2)) + a = X._mu * Y_var - Y._mu * X_var + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) + x1 = (a + b) / dv + x2 = (a - b) / dv + return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + + def zscore(self, x): + """Compute the Standard Score. (x - mean) / stdev + + Describes *x* in terms of the number of standard deviations + above or below the mean of the normal distribution. + """ + # https://www.statisticshowto.com/probability-and-statistics/z-score/ + if not self._sigma: + raise StatisticsError('zscore() not defined when sigma is zero') + return (x - self._mu) / self._sigma + + @property + def mean(self): + "Arithmetic mean of the normal distribution." + return self._mu + + @property + def median(self): + "Return the median of the normal distribution" + return self._mu + + @property + def mode(self): + """Return the mode of the normal distribution + + The mode is the value x where which the probability density + function (pdf) takes its maximum value. + """ + return self._mu + + @property + def stdev(self): + "Standard deviation of the normal distribution." + return self._sigma + + @property + def variance(self): + "Square of the standard deviation." + return self._sigma * self._sigma + + def __add__(x1, x2): + """Add a constant or another NormalDist instance. + + If *other* is a constant, translate mu by the constant, + leaving sigma unchanged. + + If *other* is a NormalDist, add both the means and the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + """Subtract a constant or another NormalDist instance. + + If *other* is a constant, translate by the constant mu, + leaving sigma unchanged. + + If *other* is a NormalDist, subtract the means and add the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + """Multiply both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + + def __truediv__(x1, x2): + """Divide both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + + def __pos__(x1): + "Return a copy of the instance." + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + "Negates mu while keeping sigma the same." + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + "Subtract a NormalDist from a constant or another NormalDist." + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + "Two NormalDist objects are equal if their mu and sigma are both equal." + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + "NormalDist objects hash equal if their mu and sigma are both equal." + return hash((self._mu, self._sigma)) + + def __repr__(self): + return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + + if issubclass(T, int) and value.denominator != 1: + T = float + + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) + + +def _normal_dist_inv_cdf(p, mu, sigma): + # There is no closed-form solution to the inverse CDF for the normal + # distribution, so we use a rational approximation instead: + # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the + # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 + # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. + q = p - 0.5 + + if fabs(q) <= 0.425: + r = 0.180625 - q * q + # Hash sum: 55.88319_28806_14901_4439 + num = (((((((2.50908_09287_30122_6727e+3 * r + + 3.34305_75583_58812_8105e+4) * r + + 6.72657_70927_00870_0853e+4) * r + + 4.59219_53931_54987_1457e+4) * r + + 1.37316_93765_50946_1125e+4) * r + + 1.97159_09503_06551_4427e+3) * r + + 1.33141_66789_17843_7745e+2) * r + + 3.38713_28727_96366_6080e+0) * q + den = (((((((5.22649_52788_52854_5610e+3 * r + + 2.87290_85735_72194_2674e+4) * r + + 3.93078_95800_09271_0610e+4) * r + + 2.12137_94301_58659_5867e+4) * r + + 5.39419_60214_24751_1077e+3) * r + + 6.87187_00749_20579_0830e+2) * r + + 4.23133_30701_60091_1252e+1) * r + + 1.0) + x = num / den + return mu + (x * sigma) + + r = p if q <= 0.0 else 1.0 - p + r = sqrt(-log(r)) + if r <= 5.0: + r = r - 1.6 + # Hash sum: 49.33206_50330_16102_89036 + num = (((((((7.74545_01427_83414_07640e-4 * r + + 2.27238_44989_26918_45833e-2) * r + + 2.41780_72517_74506_11770e-1) * r + + 1.27045_82524_52368_38258e+0) * r + + 3.64784_83247_63204_60504e+0) * r + + 5.76949_72214_60691_40550e+0) * r + + 4.63033_78461_56545_29590e+0) * r + + 1.42343_71107_49683_57734e+0) + den = (((((((1.05075_00716_44416_84324e-9 * r + + 5.47593_80849_95344_94600e-4) * r + + 1.51986_66563_61645_71966e-2) * r + + 1.48103_97642_74800_74590e-1) * r + + 6.89767_33498_51000_04550e-1) * r + + 1.67638_48301_83803_84940e+0) * r + + 2.05319_16266_37758_82187e+0) * r + + 1.0) + else: + r = r - 5.0 + # Hash sum: 47.52583_31754_92896_71629 + num = (((((((2.01033_43992_92288_13265e-7 * r + + 2.71155_55687_43487_57815e-5) * r + + 1.24266_09473_88078_43860e-3) * r + + 2.65321_89526_57612_30930e-2) * r + + 2.96560_57182_85048_91230e-1) * r + + 1.78482_65399_17291_33580e+0) * r + + 5.46378_49111_64114_36990e+0) * r + + 6.65790_46435_01103_77720e+0) + den = (((((((2.04426_31033_89939_78564e-15 * r + + 1.42151_17583_16445_88870e-7) * r + + 1.84631_83175_10054_68180e-5) * r + + 7.86869_13114_56132_59100e-4) * r + + 1.48753_61290_85061_48525e-2) * r + + 1.36929_88092_27358_05310e-1) * r + + 5.99832_20655_58879_37690e-1) * r + + 1.0) + + x = num / den + if q < 0.0: + x = -x + + return mu + (x * sigma) + + +# If available, use C implementation +try: + from _statistics import _normal_dist_inv_cdf +except ImportError: + pass diff --git a/Lib/string.py b/Lib/string/__init__.py similarity index 87% rename from Lib/string.py rename to Lib/string/__init__.py index 2eab6d4f595..eab5067c9b1 100644 --- a/Lib/string.py +++ b/Lib/string/__init__.py @@ -49,11 +49,18 @@ def capwords(s, sep=None): #################################################################### -import re as _re -from collections import ChainMap as _ChainMap - _sentinel_dict = {} + +class _TemplatePattern: + # This descriptor is overwritten in ``Template._compile_pattern()``. + def __get__(self, instance, cls=None): + if cls is None: + return self + return cls._compile_pattern() +_TemplatePattern = _TemplatePattern() + + class Template: """A string class for supporting $-substitutions.""" @@ -64,14 +71,21 @@ class Template: # See https://bugs.python.org/issue31672 idpattern = r'(?a:[_a-z][_a-z0-9]*)' braceidpattern = None - flags = _re.IGNORECASE + flags = None # default: re.IGNORECASE + + pattern = _TemplatePattern # use a descriptor to compile the pattern def __init_subclass__(cls): super().__init_subclass__() - if 'pattern' in cls.__dict__: - pattern = cls.pattern - else: - delim = _re.escape(cls.delimiter) + cls._compile_pattern() + + @classmethod + def _compile_pattern(cls): + import re # deferred import, for performance + + pattern = cls.__dict__.get('pattern', _TemplatePattern) + if pattern is _TemplatePattern: + delim = re.escape(cls.delimiter) id = cls.idpattern bid = cls.braceidpattern or cls.idpattern pattern = fr""" @@ -82,7 +96,10 @@ def __init_subclass__(cls): (?P) # Other ill-formed delimiter exprs ) """ - cls.pattern = _re.compile(pattern, cls.flags | _re.VERBOSE) + if cls.flags is None: + cls.flags = re.IGNORECASE + pat = cls.pattern = re.compile(pattern, cls.flags | re.VERBOSE) + return pat def __init__(self, template): self.template = template @@ -105,7 +122,8 @@ def substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): # Check the most common path first. @@ -124,7 +142,8 @@ def safe_substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') @@ -170,10 +189,6 @@ def get_identifiers(self): self.pattern) return ids -# Initialize Template.pattern. __init_subclass__() is automatically called -# only for subclasses, not for the Template class itself. -Template.__init_subclass__() - ######################################################################## # the Formatter class @@ -212,19 +227,20 @@ def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, # this is some markup, find the object and do # the formatting - # handle arg indexing when empty field_names are given. - if field_name == '': + # handle arg indexing when empty field first parts are given. + field_first, _ = _string.formatter_field_name_split(field_name) + if field_first == '': if auto_arg_index is False: raise ValueError('cannot switch from manual field ' 'specification to automatic field ' 'numbering') - field_name = str(auto_arg_index) + field_name = str(auto_arg_index) + field_name auto_arg_index += 1 - elif field_name.isdigit(): + elif isinstance(field_first, int): if auto_arg_index: - raise ValueError('cannot switch from manual field ' - 'specification to automatic field ' - 'numbering') + raise ValueError('cannot switch from automatic field ' + 'numbering to manual field ' + 'specification') # disable auto arg incrementing, if it gets # used later on, then an exception will be raised auto_arg_index = False diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py new file mode 100644 index 00000000000..8164872432a --- /dev/null +++ b/Lib/string/templatelib.py @@ -0,0 +1,33 @@ +"""Support for template string literals (t-strings).""" + +t = t"{0}" +Template = type(t) +Interpolation = type(t.interpolations[0]) +del t + +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + +def _template_unpickle(*args): + import itertools + + if len(args) != 2: + raise ValueError('Template expects tuple of length 2 to unpickle') + + strings, interpolations = args + parts = [] + for string, interpolation in itertools.zip_longest(strings, interpolations): + if string is not None: + parts.append(string) + if interpolation is not None: + parts.append(interpolation) + return Template(*parts) diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636..ff98e8c4cb3 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 1d17ae3608a..6911cd8e859 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -74,15 +74,16 @@ else: _mswindows = True -# wasm32-emscripten and wasm32-wasi do not support processes -_can_fork_exec = sys.platform not in {"emscripten", "wasi"} +# some platforms do not support subprocesses +_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"} if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +94,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", @@ -103,18 +105,22 @@ if _can_fork_exec: from _posixsubprocess import fork_exec as _fork_exec # used in methods that are called by __del__ - _waitpid = os.waitpid - _waitstatus_to_exitcode = os.waitstatus_to_exitcode - _WIFSTOPPED = os.WIFSTOPPED - _WSTOPSIG = os.WSTOPSIG - _WNOHANG = os.WNOHANG + class _del_safe: + waitpid = os.waitpid + waitstatus_to_exitcode = os.waitstatus_to_exitcode + WIFSTOPPED = os.WIFSTOPPED + WSTOPSIG = os.WSTOPSIG + WNOHANG = os.WNOHANG + ECHILD = errno.ECHILD else: - _fork_exec = None - _waitpid = None - _waitstatus_to_exitcode = None - _WIFSTOPPED = None - _WSTOPSIG = None - _WNOHANG = None + class _del_safe: + waitpid = None + waitstatus_to_exitcode = None + WIFSTOPPED = None + WSTOPSIG = None + WNOHANG = None + ECHILD = errno.ECHILD + import select import selectors @@ -346,7 +352,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -380,7 +386,7 @@ def _text_encoding(): def call(*popenargs, timeout=None, **kwargs): """Run command with arguments. Wait for command to complete or - timeout, then return the returncode attribute. + for timeout seconds, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: @@ -517,8 +523,8 @@ def run(*popenargs, in the returncode attribute, and output & stderr attributes if those streams were captured. - If timeout is given, and the process takes too long, a TimeoutExpired - exception will be raised. + If timeout (seconds) is given and the process takes too long, + a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument @@ -709,6 +715,9 @@ def _use_posix_spawn(): # os.posix_spawn() is not available return False + if ((_env := os.environ.get('_PYTHON_SUBPROCESS_USE_POSIX_SPAWN')) in ('0', '1')): + return bool(int(_env)) + if sys.platform in ('darwin', 'sunos5'): # posix_spawn() is a syscall on both macOS and Solaris, # and properly reports errors @@ -743,7 +752,7 @@ def _use_posix_spawn(): # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() -_USE_VFORK = True +_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') class Popen: @@ -834,6 +843,9 @@ def __init__(self, args, bufsize=-1, executable=None, if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if stdout is STDOUT: + raise ValueError("STDOUT can only be used for stderr") + if pipesize is None: pipesize = -1 # Restore default if not isinstance(pipesize, int): @@ -1112,10 +1124,9 @@ def __exit__(self, exc_type, value, traceback): except TimeoutExpired: pass self._sigint_wait_secs = 0 # Note that this has been done. - return # resume the KeyboardInterrupt - - # Wait for the process to terminate, to avoid zombies. - self.wait() + else: + # Wait for the process to terminate, to avoid zombies. + self.wait() def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): if not self._child_created: @@ -1224,8 +1235,11 @@ def communicate(self, input=None, timeout=None): finally: self._communication_started = True - - sts = self.wait(timeout=self._remaining_time(endtime)) + try: + sts = self.wait(timeout=self._remaining_time(endtime)) + except TimeoutExpired as exc: + exc.timeout = timeout + raise return (stdout, stderr) @@ -1600,6 +1614,10 @@ def _readerthread(self, fh, buffer): fh.close() + def _writerthread(self, input): + self._stdin_write(input) + + def _communicate(self, input, endtime, orig_timeout): # Start reader threads feeding into a list hanging off of this # object, unless they've already been started. @@ -1618,8 +1636,23 @@ def _communicate(self, input, endtime, orig_timeout): self.stderr_thread.daemon = True self.stderr_thread.start() - if self.stdin: - self._stdin_write(input) + # Start writer thread to send input to stdin, unless already + # started. The thread writes input and closes stdin when done, + # or continues in the background on timeout. + if self.stdin and not hasattr(self, "_stdin_thread"): + self._stdin_thread = \ + threading.Thread(target=self._writerthread, + args=(input,)) + self._stdin_thread.daemon = True + self._stdin_thread.start() + + # Wait for the writer thread, or time out. If we time out, the + # thread remains writing and the fd left open in case the user + # calls communicate again. + if hasattr(self, "_stdin_thread"): + self._stdin_thread.join(self._remaining_time(endtime)) + if self._stdin_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user @@ -1749,14 +1782,11 @@ def _get_handles(self, stdin, stdout, stderr): errread, errwrite) - def _posix_spawn(self, args, executable, env, restore_signals, + def _posix_spawn(self, args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program using os.posix_spawn().""" - if env is None: - env = os.environ - kwargs = {} if restore_signals: # See _Py_RestoreSignals() in Python/pylifecycle.c @@ -1778,6 +1808,10 @@ def _posix_spawn(self, args, executable, env, restore_signals, ): if fd != -1: file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) + + if close_fds: + file_actions.append((os.POSIX_SPAWN_CLOSEFROM, 3)) + if file_actions: kwargs['file_actions'] = file_actions @@ -1825,7 +1859,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None - and not close_fds + and (not close_fds or _HAVE_POSIX_SPAWN_CLOSEFROM) and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) @@ -1837,7 +1871,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, and gids is None and uid is None and umask < 0): - self._posix_spawn(args, executable, env, restore_signals, + self._posix_spawn(args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -1891,7 +1925,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, - preexec_fn, _USE_VFORK) + preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what @@ -1958,20 +1992,16 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, raise child_exception_type(err_msg) - def _handle_exitstatus(self, sts, - _waitstatus_to_exitcode=_waitstatus_to_exitcode, - _WIFSTOPPED=_WIFSTOPPED, - _WSTOPSIG=_WSTOPSIG): + def _handle_exitstatus(self, sts, _del_safe=_del_safe): """All callers to this function MUST hold self._waitpid_lock.""" # This method is called (indirectly) by __del__, so it cannot # refer to anything outside of its local scope. - if _WIFSTOPPED(sts): - self.returncode = -_WSTOPSIG(sts) + if _del_safe.WIFSTOPPED(sts): + self.returncode = -_del_safe.WSTOPSIG(sts) else: - self.returncode = _waitstatus_to_exitcode(sts) + self.returncode = _del_safe.waitstatus_to_exitcode(sts) - def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, - _WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD): + def _internal_poll(self, _deadstate=None, _del_safe=_del_safe): """Check if child process has terminated. Returns returncode attribute. @@ -1987,13 +2017,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, try: if self.returncode is not None: return self.returncode # Another thread waited. - pid, sts = _waitpid(self.pid, _WNOHANG) + pid, sts = _del_safe.waitpid(self.pid, _del_safe.WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) except OSError as e: if _deadstate is not None: self.returncode = _deadstate - elif e.errno == _ECHILD: + elif e.errno == _del_safe.ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we @@ -2067,6 +2097,10 @@ def _communicate(self, input, endtime, orig_timeout): self.stdin.flush() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. + except ValueError: + # ignore ValueError: I/O operation on closed file. + if not self.stdin.closed: + raise if not input: try: self.stdin.close() @@ -2092,10 +2126,13 @@ def _communicate(self, input, endtime, orig_timeout): self._save_input(input) if self._input: - input_view = memoryview(self._input) + if not isinstance(self._input, memoryview): + input_view = memoryview(self._input) + else: + input_view = self._input.cast("b") # byte input required with _PopenSelector() as selector: - if self.stdin and input: + if self.stdin and not self.stdin.closed and self._input: selector.register(self.stdin, selectors.EVENT_WRITE) if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) @@ -2128,7 +2165,7 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() else: - if self._input_offset >= len(self._input): + if self._input_offset >= len(input_view): selector.unregister(key.fileobj) key.fileobj.close() elif key.fileobj in (self.stdout, self.stderr): @@ -2137,8 +2174,11 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() self._fileobj2output[key.fileobj].append(data) - - self.wait(timeout=self._remaining_time(endtime)) + try: + self.wait(timeout=self._remaining_time(endtime)) + except TimeoutExpired as exc: + exc.timeout = orig_timeout + raise # All data exchanged. Translate lists into strings. if stdout is not None: diff --git a/Lib/symtable.py b/Lib/symtable.py index 672ec0ce1ff..7a30e1ac4ca 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,9 +1,16 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref from enum import StrEnum @@ -165,6 +172,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -190,7 +201,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -199,7 +210,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -214,7 +225,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -226,6 +237,12 @@ class Class(SymbolTable): def get_methods(self): """Return a tuple of methods declared in the class. """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + if self.__methods is None: d = {} @@ -268,7 +285,7 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope @@ -293,13 +310,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ @@ -332,6 +354,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -342,6 +369,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index d06c4c221e9..c0097351b26 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 3aceb3f70fd..8036e93cd6d 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -46,7 +46,6 @@ import sys as _sys import types as _types import weakref as _weakref - import _thread _allocate_lock = _thread.allocate_lock @@ -181,7 +180,7 @@ def _candidate_tempdir_list(): return dirlist -def _get_default_tempdir(): +def _get_default_tempdir(dirlist=None): """Calculate the default directory to use for temporary files. This routine should be called exactly once. @@ -191,7 +190,8 @@ def _get_default_tempdir(): service, the name of the test file must be randomized.""" namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() + if dirlist is None: + dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: @@ -204,8 +204,7 @@ def _get_default_tempdir(): fd = _os.open(filename, _bin_openflags, 0o600) try: try: - with _io.open(fd, 'wb', closefd=False) as fp: - fp.write(b'blat') + _os.write(fd, b'blat') finally: _os.close(fd) finally: @@ -245,6 +244,7 @@ def _get_candidate_names(): def _mkstemp_inner(dir, pre, suf, flags, output_type): """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" + dir = _os.path.abspath(dir) names = _get_candidate_names() if output_type is bytes: names = map(_os.fsencode, names) @@ -265,11 +265,27 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): continue else: raise - return (fd, _os.path.abspath(file)) + return fd, file raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") +def _dont_follow_symlinks(func, path, *args): + # Pass follow_symlinks=False, unless not supported on this platform. + if func in _os.supports_follow_symlinks: + func(path, *args, follow_symlinks=False) + elif not _os.path.islink(path): + func(path, *args) + +def _resetperms(path): + try: + chflags = _os.chflags + except AttributeError: + pass + else: + _dont_follow_symlinks(chflags, path, 0) + _dont_follow_symlinks(_os.chmod, path, 0o700) + # User visible interfaces. @@ -377,7 +393,7 @@ def mkdtemp(suffix=None, prefix=None, dir=None): continue else: raise - return file + return _os.path.abspath(file) raise FileExistsError(_errno.EEXIST, "No usable temporary directory name found") @@ -419,42 +435,42 @@ class _TemporaryFileCloser: underlying file object, without adding a __del__ method to the temporary file.""" - file = None # Set here since __del__ checks it + cleanup_called = False close_called = False - def __init__(self, file, name, delete=True): + def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name self.delete = delete + self.delete_on_close = delete_on_close - # NT provides delete-on-close as a primitive, so we don't need - # the wrapper to do anything special. We still use it so that - # file.name is useful (i.e. not "(fdopen)") with NamedTemporaryFile. - if _os.name != 'nt': - # Cache the unlinker so we don't get spurious errors at - # shutdown when the module-level "os" is None'd out. Note - # that this must be referenced as self.unlink, because the - # name TemporaryFileWrapper may also get None'd out before - # __del__ is called. - - def close(self, unlink=_os.unlink): - if not self.close_called and self.file is not None: - self.close_called = True - try: + def cleanup(self, windows=(_os.name == 'nt'), unlink=_os.unlink): + if not self.cleanup_called: + self.cleanup_called = True + try: + if not self.close_called: + self.close_called = True self.file.close() - finally: - if self.delete: + finally: + # Windows provides delete-on-close as a primitive, in which + # case the file was deleted by self.file.close(). + if self.delete and not (windows and self.delete_on_close): + try: unlink(self.name) + except FileNotFoundError: + pass - # Need to ensure the file is deleted on __del__ - def __del__(self): - self.close() - - else: - def close(self): - if not self.close_called: - self.close_called = True + def close(self): + if not self.close_called: + self.close_called = True + try: self.file.close() + finally: + if self.delete and self.delete_on_close: + self.cleanup() + + def __del__(self): + self.cleanup() class _TemporaryFileWrapper: @@ -465,11 +481,11 @@ class _TemporaryFileWrapper: remove the file when it is no longer needed. """ - def __init__(self, file, name, delete=True): + def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name - self.delete = delete - self._closer = _TemporaryFileCloser(file, name, delete) + self._closer = _TemporaryFileCloser(file, name, delete, + delete_on_close) def __getattr__(self, name): # Attribute lookups are delegated to the underlying file @@ -500,7 +516,7 @@ def __enter__(self): # deleted when used in a with statement def __exit__(self, exc, value, tb): result = self.file.__exit__(exc, value, tb) - self.close() + self._closer.cleanup() return result def close(self): @@ -519,10 +535,10 @@ def __iter__(self): for line in self.file: yield line - def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, - dir=None, delete=True, *, errors=None): + dir=None, delete=True, *, errors=None, + delete_on_close=True): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. @@ -530,13 +546,20 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) - 'delete' -- whether the file is deleted on close (default True). + 'delete' -- whether the file is automatically deleted (default True). + 'delete_on_close' -- if 'delete', whether the file is deleted on close + (default True) or otherwise either on context manager exit + (if context manager was used) or on object finalization. . 'errors' -- the errors argument to io.open (default None) The file is created as mkstemp() would do it. Returns an object with a file-like interface; the name of the file is accessible as its 'name' attribute. The file will be automatically deleted when it is closed unless the 'delete' argument is set to False. + + On POSIX, NamedTemporaryFiles cannot be automatically deleted if + the creating process is terminated abruptly with a SIGKILL signal. + Windows can delete the file even in this case. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) @@ -545,21 +568,33 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. - if _os.name == 'nt' and delete: + if _os.name == 'nt' and delete and delete_on_close: flags |= _os.O_TEMPORARY if "b" not in mode: encoding = _io.text_encoding(encoding) - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + name = None + def opener(*args): + nonlocal name + fd, name = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + return fd try: - file = _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, errors=errors) - - return _TemporaryFileWrapper(file, name, delete) - except BaseException: - _os.unlink(name) - _os.close(fd) + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, errors=errors, + opener=opener) + try: + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = name + return _TemporaryFileWrapper(file, name, delete, delete_on_close) + except: + file.close() + raise + except: + if name is not None and not ( + _os.name == 'nt' and delete and delete_on_close): + _os.unlink(name) raise if _os.name != 'posix' or _sys.platform == 'cygwin': @@ -598,9 +633,20 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None, flags = _bin_openflags if _O_TMPFILE_WORKS: - try: + fd = None + def opener(*args): + nonlocal fd flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT fd = _os.open(dir, flags2, 0o600) + return fd + try: + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, + errors=errors, opener=opener) + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = fd + return file except IsADirectoryError: # Linux kernel older than 3.11 ignores the O_TMPFILE flag: # O_TMPFILE is read as O_DIRECTORY. Trying to open a directory @@ -617,26 +663,27 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None, # fails with NotADirectoryError, because O_TMPFILE is read as # O_DIRECTORY. pass - else: - try: - return _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, - errors=errors) - except: - _os.close(fd) - raise # Fallback to _mkstemp_inner(). - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) - try: - _os.unlink(name) - return _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, errors=errors) - except: - _os.close(fd) - raise + fd = None + def opener(*args): + nonlocal fd + fd, name = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + try: + _os.unlink(name) + except BaseException as e: + _os.close(fd) + raise + return fd + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, errors=errors, + opener=opener) + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = fd + return file -class SpooledTemporaryFile: +class SpooledTemporaryFile(_io.IOBase): """Temporary file wrapper, specialized to switch from BytesIO or StringIO to a real file when it exceeds a certain size or when a fileno is needed. @@ -701,6 +748,16 @@ def __exit__(self, exc, value, tb): def __iter__(self): return self._file.__iter__() + def __del__(self): + if not self.closed: + _warnings.warn( + "Unclosed file {!r}".format(self), + ResourceWarning, + stacklevel=2, + source=self + ) + self.close() + def close(self): self._file.close() @@ -744,15 +801,30 @@ def name(self): def newlines(self): return self._file.newlines + def readable(self): + return self._file.readable() + def read(self, *args): return self._file.read(*args) + def read1(self, *args): + return self._file.read1(*args) + + def readinto(self, b): + return self._file.readinto(b) + + def readinto1(self, b): + return self._file.readinto1(b) + def readline(self, *args): return self._file.readline(*args) def readlines(self, *args): return self._file.readlines(*args) + def seekable(self): + return self._file.seekable() + def seek(self, *args): return self._file.seek(*args) @@ -761,11 +833,14 @@ def tell(self): def truncate(self, size=None): if size is None: - self._file.truncate() + return self._file.truncate() else: if size > self._max_size: self.rollover() - self._file.truncate(size) + return self._file.truncate(size) + + def writable(self): + return self._file.writable() def write(self, s): file = self._file @@ -774,10 +849,17 @@ def write(self, s): return rv def writelines(self, iterable): - file = self._file - rv = file.writelines(iterable) - self._check(file) - return rv + if self._max_size == 0 or self._rolled: + return self._file.writelines(iterable) + + it = iter(iterable) + for line in it: + self.write(line) + if self._rolled: + return self._file.writelines(it) + + def detach(self): + return self._file.detach() class TemporaryDirectory: @@ -789,53 +871,74 @@ class TemporaryDirectory: ... Upon exiting the context, the directory and everything contained - in it are removed. + in it are removed (unless delete=False is passed or an exception + is raised during cleanup and ignore_cleanup_errors is not True). + + Optional Arguments: + suffix - A str suffix for the directory name. (see mkdtemp) + prefix - A str prefix for the directory name. (see mkdtemp) + dir - A directory to create this temp dir in. (see mkdtemp) + ignore_cleanup_errors - False; ignore exceptions during cleanup? + delete - True; whether the directory is automatically deleted. """ def __init__(self, suffix=None, prefix=None, dir=None, - ignore_cleanup_errors=False): + ignore_cleanup_errors=False, *, delete=True): self.name = mkdtemp(suffix, prefix, dir) self._ignore_cleanup_errors = ignore_cleanup_errors + self._delete = delete self._finalizer = _weakref.finalize( self, self._cleanup, self.name, warn_message="Implicitly cleaning up {!r}".format(self), - ignore_errors=self._ignore_cleanup_errors) + ignore_errors=self._ignore_cleanup_errors, delete=self._delete) @classmethod - def _rmtree(cls, name, ignore_errors=False): - def onerror(func, path, exc_info): - if issubclass(exc_info[0], PermissionError): - def resetperms(path): - try: - _os.chflags(path, 0) - except AttributeError: - pass - _os.chmod(path, 0o700) + def _rmtree(cls, name, ignore_errors=False, repeated=False): + def onexc(func, path, exc): + if isinstance(exc, PermissionError): + if repeated and path == name: + if ignore_errors: + return + raise try: if path != name: - resetperms(_os.path.dirname(path)) - resetperms(path) + _resetperms(_os.path.dirname(path)) + _resetperms(path) try: _os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError): + except IsADirectoryError: cls._rmtree(path, ignore_errors=ignore_errors) + except PermissionError: + # The PermissionError handler was originally added for + # FreeBSD in directories, but it seems that it is raised + # on Windows too. + # bpo-43153: Calling _rmtree again may + # raise NotADirectoryError and mask the PermissionError. + # So we must re-raise the current PermissionError if + # path is not a directory. + if not _os.path.isdir(path) or _os.path.isjunction(path): + if ignore_errors: + return + raise + cls._rmtree(path, ignore_errors=ignore_errors, + repeated=(path == name)) except FileNotFoundError: pass - elif issubclass(exc_info[0], FileNotFoundError): + elif isinstance(exc, FileNotFoundError): pass else: if not ignore_errors: raise - _shutil.rmtree(name, onerror=onerror) + _shutil.rmtree(name, onexc=onexc) @classmethod - def _cleanup(cls, name, warn_message, ignore_errors=False): - cls._rmtree(name, ignore_errors=ignore_errors) - _warnings.warn(warn_message, ResourceWarning) + def _cleanup(cls, name, warn_message, ignore_errors=False, delete=True): + if delete: + cls._rmtree(name, ignore_errors=ignore_errors) + _warnings.warn(warn_message, ResourceWarning) def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.name) @@ -844,7 +947,8 @@ def __enter__(self): return self.name def __exit__(self, exc, value, tb): - self.cleanup() + if self._delete: + self.cleanup() def cleanup(self): if self._finalizer.detach() or _os.path.exists(self.name): diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 19a6b2b8904..82b50ad2c6e 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,2 @@ -from test.libregrtest import main -main() +from test.libregrtest.main import main +main(_add_python_opts=True) diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py new file mode 100644 index 00000000000..c8f04e9625c --- /dev/null +++ b/Lib/test/_test_eintr.py @@ -0,0 +1,550 @@ +""" +This test suite exercises some system calls subject to interruption with EINTR, +to check that it is actually handled transparently. +It is intended to be run by the main test suite within a child process, to +ensure there is no background thread running (so that signals are delivered to +the correct thread). +Signals are generated in-process using setitimer(ITIMER_REAL), which allows +sub-second periodicity (contrarily to signal()). +""" + +import contextlib +import faulthandler +import fcntl +import os +import platform +import select +import signal +import socket +import subprocess +import sys +import textwrap +import time +import unittest + +from test import support +from test.support import os_helper +from test.support import socket_helper + + +# gh-109592: Tolerate a difference of 20 ms when comparing timings +# (clock resolution) +CLOCK_RES = 0.020 + + +@contextlib.contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class EINTRBaseTest(unittest.TestCase): + """ Base class for EINTR tests. """ + + # delay for initial signal delivery + signal_delay = 0.1 + # signal delivery periodicity + signal_period = 0.1 + # default sleep time for tests - should obviously have: + # sleep_time > signal_period + sleep_time = 0.2 + + def sighandler(self, signum, frame): + self.signals += 1 + + def setUp(self): + self.signals = 0 + self.orig_handler = signal.signal(signal.SIGALRM, self.sighandler) + signal.setitimer(signal.ITIMER_REAL, self.signal_delay, + self.signal_period) + + # Use faulthandler as watchdog to debug when a test hangs + # (timeout of 10 minutes) + faulthandler.dump_traceback_later(10 * 60, exit=True, + file=sys.__stderr__) + + @staticmethod + def stop_alarm(): + signal.setitimer(signal.ITIMER_REAL, 0, 0) + + def tearDown(self): + self.stop_alarm() + signal.signal(signal.SIGALRM, self.orig_handler) + faulthandler.cancel_dump_traceback_later() + + def subprocess(self, *args, **kw): + cmd_args = (sys.executable, '-c') + args + return subprocess.Popen(cmd_args, **kw) + + def check_elapsed_time(self, elapsed): + self.assertGreaterEqual(elapsed, self.sleep_time - CLOCK_RES) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class OSEINTRTest(EINTRBaseTest): + """ EINTR tests for the os module. """ + + def new_sleep_process(self): + code = 'import time; time.sleep(%r)' % self.sleep_time + return self.subprocess(code) + + def _test_wait_multiple(self, wait_func): + num = 3 + processes = [self.new_sleep_process() for _ in range(num)] + for _ in range(num): + wait_func() + # Call the Popen method to avoid a ResourceWarning + for proc in processes: + proc.wait() + + def test_wait(self): + self._test_wait_multiple(os.wait) + + @unittest.skipUnless(hasattr(os, 'wait3'), 'requires wait3()') + def test_wait3(self): + self._test_wait_multiple(lambda: os.wait3(0)) + + def _test_wait_single(self, wait_func): + proc = self.new_sleep_process() + wait_func(proc.pid) + # Call the Popen method to avoid a ResourceWarning + proc.wait() + + def test_waitpid(self): + self._test_wait_single(lambda pid: os.waitpid(pid, 0)) + + @unittest.skipUnless(hasattr(os, 'wait4'), 'requires wait4()') + def test_wait4(self): + self._test_wait_single(lambda pid: os.wait4(pid, 0)) + + def test_read(self): + rd, wr = os.pipe() + self.addCleanup(os.close, rd) + # wr closed explicitly by parent + + # the payload below are smaller than PIPE_BUF, hence the writes will be + # atomic + datas = [b"hello", b"world", b"spam"] + + code = '\n'.join(( + 'import os, sys, time', + '', + 'wr = int(sys.argv[1])', + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'for data in datas:', + ' # let the parent block on read()', + ' time.sleep(sleep_time)', + ' os.write(wr, data)', + )) + + proc = self.subprocess(code, str(wr), pass_fds=[wr]) + with kill_on_error(proc): + os.close(wr) + for data in datas: + self.assertEqual(data, os.read(rd, len(data))) + self.assertEqual(proc.wait(), 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call + def test_write(self): + rd, wr = os.pipe() + self.addCleanup(os.close, wr) + # rd closed explicitly by parent + + # we must write enough data for the write() to block + data = b"x" * support.PIPE_MAX_SIZE + + code = '\n'.join(( + 'import io, os, sys, time', + '', + 'rd = int(sys.argv[1])', + 'sleep_time = %r' % self.sleep_time, + 'data = b"x" * %s' % support.PIPE_MAX_SIZE, + 'data_len = len(data)', + '', + '# let the parent block on write()', + 'time.sleep(sleep_time)', + '', + 'read_data = io.BytesIO()', + 'while len(read_data.getvalue()) < data_len:', + ' chunk = os.read(rd, 2 * data_len)', + ' read_data.write(chunk)', + '', + 'value = read_data.getvalue()', + 'if value != data:', + ' raise Exception("read error: %s vs %s bytes"', + ' % (len(value), data_len))', + )) + + proc = self.subprocess(code, str(rd), pass_fds=[rd]) + with kill_on_error(proc): + os.close(rd) + written = 0 + while written < len(data): + written += os.write(wr, memoryview(data)[written:]) + self.assertEqual(proc.wait(), 0) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class SocketEINTRTest(EINTRBaseTest): + """ EINTR tests for the socket module. """ + + @unittest.skipUnless(hasattr(socket, 'socketpair'), 'needs socketpair()') + def _test_recv(self, recv_func): + rd, wr = socket.socketpair() + self.addCleanup(rd.close) + # wr closed explicitly by parent + + # single-byte payload guard us against partial recv + datas = [b"x", b"y", b"z"] + + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(wr.family), + 'sock_type = %s' % int(wr.type), + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'wr = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with wr:', + ' for data in datas:', + ' # let the parent block on recv()', + ' time.sleep(sleep_time)', + ' wr.sendall(data)', + )) + + fd = wr.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with kill_on_error(proc): + wr.close() + for data in datas: + self.assertEqual(data, recv_func(rd, len(data))) + self.assertEqual(proc.wait(), 0) + + def test_recv(self): + self._test_recv(socket.socket.recv) + + @unittest.skipUnless(hasattr(socket.socket, 'recvmsg'), 'needs recvmsg()') + def test_recvmsg(self): + self._test_recv(lambda sock, data: sock.recvmsg(data)[0]) + + def _test_send(self, send_func): + rd, wr = socket.socketpair() + self.addCleanup(wr.close) + # rd closed explicitly by parent + + # we must send enough data for the send() to block + data = b"xyz" * (support.SOCK_MAX_SIZE // 3) + + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(rd.family), + 'sock_type = %s' % int(rd.type), + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + 'data_len = len(data)', + '', + 'rd = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with rd:', + ' # let the parent block on send()', + ' time.sleep(sleep_time)', + '', + ' received_data = bytearray(data_len)', + ' n = 0', + ' while n < data_len:', + ' n += rd.recv_into(memoryview(received_data)[n:])', + '', + 'if received_data != data:', + ' raise Exception("recv error: %s vs %s bytes"', + ' % (len(received_data), data_len))', + )) + + fd = rd.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with kill_on_error(proc): + rd.close() + written = 0 + while written < len(data): + sent = send_func(wr, memoryview(data)[written:]) + # sendall() returns None + written += len(data) if sent is None else sent + self.assertEqual(proc.wait(), 0) + + def test_send(self): + self._test_send(socket.socket.send) + + def test_sendall(self): + self._test_send(socket.socket.sendall) + + @unittest.skipUnless(hasattr(socket.socket, 'sendmsg'), 'needs sendmsg()') + def test_sendmsg(self): + self._test_send(lambda sock, data: sock.sendmsg([data])) + + def test_accept(self): + sock = socket.create_server((socket_helper.HOST, 0)) + self.addCleanup(sock.close) + port = sock.getsockname()[1] + + code = '\n'.join(( + 'import socket, time', + '', + 'host = %r' % socket_helper.HOST, + 'port = %s' % port, + 'sleep_time = %r' % self.sleep_time, + '', + '# let parent block on accept()', + 'time.sleep(sleep_time)', + 'with socket.create_connection((host, port)):', + ' time.sleep(sleep_time)', + )) + + proc = self.subprocess(code) + with kill_on_error(proc): + client_sock, _ = sock.accept() + client_sock.close() + self.assertEqual(proc.wait(), 0) + + # Issue #25122: There is a race condition in the FreeBSD kernel on + # handling signals in the FIFO device. Skip the test until the bug is + # fixed in the kernel. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 + @support.requires_freebsd_version(10, 3) + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') + def _test_open(self, do_open_close_reader, do_open_close_writer): + filename = os_helper.TESTFN + + # Use a fifo: until the child opens it for reading, the parent will + # block when trying to open it for writing. + os_helper.unlink(filename) + try: + os.mkfifo(filename) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + self.addCleanup(os_helper.unlink, filename) + + code = '\n'.join(( + 'import os, time', + '', + 'path = %a' % filename, + 'sleep_time = %r' % self.sleep_time, + '', + '# let the parent block', + 'time.sleep(sleep_time)', + '', + do_open_close_reader, + )) + + proc = self.subprocess(code) + with kill_on_error(proc): + do_open_close_writer(filename) + self.assertEqual(proc.wait(), 0) + + def python_open(self, path): + fp = open(path, 'w') + fp.close() + + @unittest.skipIf(sys.platform == "darwin", + "hangs under macOS; see bpo-25234, bpo-35363") + def test_open(self): + self._test_open("fp = open(path, 'r')\nfp.close()", + self.python_open) + + def os_open(self, path): + fd = os.open(path, os.O_WRONLY) + os.close(fd) + + @unittest.skipIf(sys.platform == "darwin", + "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") + def test_os_open(self): + self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", + self.os_open) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class TimeEINTRTest(EINTRBaseTest): + """ EINTR tests for the time module. """ + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sleep(self): + t0 = time.monotonic() + time.sleep(self.sleep_time) + self.stop_alarm() + dt = time.monotonic() - t0 + self.check_elapsed_time(dt) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +# bpo-30320: Need pthread_sigmask() to block the signal, otherwise the test +# is vulnerable to a race condition between the child and the parent processes. +@unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +class SignalEINTRTest(EINTRBaseTest): + """ EINTR tests for the signal module. """ + + def check_sigwait(self, wait_func): + signum = signal.SIGUSR1 + pid = os.getpid() + + old_handler = signal.signal(signum, lambda *args: None) + self.addCleanup(signal.signal, signum, old_handler) + + code = '\n'.join(( + 'import os, time', + 'pid = %s' % os.getpid(), + 'signum = %s' % int(signum), + 'sleep_time = %r' % self.sleep_time, + 'time.sleep(sleep_time)', + 'os.kill(pid, signum)', + )) + + old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) + + proc = self.subprocess(code) + with kill_on_error(proc): + wait_func(signum) + + self.assertEqual(proc.wait(), 0) + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + def wait_func(signum): + signal.sigwaitinfo([signum]) + + self.check_sigwait(wait_func) + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigwaitinfo()') + def test_sigtimedwait(self): + def wait_func(signum): + signal.sigtimedwait([signum], 120.0) + + self.check_sigwait(wait_func) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class SelectEINTRTest(EINTRBaseTest): + """ EINTR tests for the select module. """ + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_select(self): + t0 = time.monotonic() + select.select([], [], [], self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skip('TODO: RUSTPYTHON timed out at the 10 minute mark') + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") + @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') + def test_poll(self): + poller = select.poll() + + t0 = time.monotonic() + poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'epoll'), 'need select.epoll') + def test_epoll(self): + poller = select.epoll() + self.addCleanup(poller.close) + + t0 = time.monotonic() + poller.poll(self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'kqueue'), 'need select.kqueue') + def test_kqueue(self): + kqueue = select.kqueue() + self.addCleanup(kqueue.close) + + t0 = time.monotonic() + kqueue.control(None, 1, self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'devpoll'), 'need select.devpoll') + def test_devpoll(self): + poller = select.devpoll() + self.addCleanup(poller.close) + + t0 = time.monotonic() + poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + +class FCNTLEINTRTest(EINTRBaseTest): + def _lock(self, lock_func, lock_name): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) + with kill_on_error(proc): + with open(os_helper.TESTFN, 'wb') as f: + # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready + start_time = time.monotonic() + os.write(wr2, b"go") + + # the child locked the file just a moment ago for 'sleep_time' seconds + # that means that the lock below will block for 'sleep_time' minus some + # potential context switch delay + lock_func(f, fcntl.LOCK_EX) + dt = time.monotonic() - start_time + self.stop_alarm() + self.check_elapsed_time(dt) + proc.wait() + + @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call + # Issue 35633: See https://bugs.python.org/issue35633#msg333662 + # skip test rather than accept PermissionError from all platforms + @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + def test_lockf(self): + self._lock(fcntl.lockf, "lockf") + + @unittest.expectedFailure # TODO: RUSTPYTHON InterruptedError: [Errno 4] Interrupted system call + def test_flock(self): + self._lock(fcntl.flock, "flock") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c22ce769c48..d05ed68144b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -38,7 +38,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper - +from test.support import subTests # Skip tests if _multiprocessing wasn't built. _multiprocessing = import_helper.import_module('_multiprocessing') @@ -1109,7 +1109,7 @@ def test_put(self): @classmethod def _test_get(cls, queue, child_can_start, parent_can_continue): child_can_start.wait() - #queue.put(1) + queue.put(1) queue.put(2) queue.put(3) queue.put(4) @@ -1133,15 +1133,16 @@ def test_get(self): child_can_start.set() parent_can_continue.wait() - time.sleep(DELTA) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if not queue_empty(queue): + break self.assertEqual(queue_empty(queue), False) - # Hangs unexpectedly, remove for now - #self.assertEqual(queue.get(), 1) + self.assertEqual(queue.get_nowait(), 1) self.assertEqual(queue.get(True, None), 2) self.assertEqual(queue.get(True), 3) self.assertEqual(queue.get(timeout=1), 4) - self.assertEqual(queue.get_nowait(), 5) + self.assertEqual(queue.get(), 5) self.assertEqual(queue_empty(queue), True) @@ -1458,6 +1459,7 @@ def _acquire_release(lock, timeout, l=None, n=1): for _ in range(n): lock.release() + @unittest.skip("TODO: RUSTPYTHON; flaky timeout") def test_repr_rlock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -2970,6 +2972,8 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.monotonic() - t_start, 0.9) + # TODO: RUSTPYTHON - reference counting differences + @unittest.skip("TODO: RUSTPYTHON") def test_release_task_refs(self): # Issue #29861: task arguments and results should not be kept # alive after we are done with them. @@ -3882,6 +3886,8 @@ def _remote(cls, conn): conn.close() + # TODO: RUSTPYTHON - hangs + @unittest.skip("TODO: RUSTPYTHON") def test_pickling(self): families = self.connection.families @@ -4103,6 +4109,8 @@ def _double(cls, x, y, z, foo, arr, string): for i in range(len(arr)): arr[i] *= 2 + # TODO: RUSTPYTHON - ctypes Structure shared memory not working + @unittest.expectedFailure def test_sharedctypes(self, lock=False): x = Value('i', 7, lock=lock) y = Value(c_double, 1.0/3.0, lock=lock) @@ -4126,6 +4134,8 @@ def test_sharedctypes(self, lock=False): self.assertAlmostEqual(arr[i], i*2) self.assertEqual(string.value, latin('hellohello')) + # TODO: RUSTPYTHON - calls test_sharedctypes which fails + @unittest.expectedFailure def test_synchronize(self): self.test_sharedctypes(lock=True) @@ -4140,6 +4150,19 @@ def test_copy(self): self.assertEqual(bar.z, 2 ** 33) +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") @hashlib_helper.requires_hashdigest('sha256') class _TestSharedMemory(BaseTestCase): @@ -4392,6 +4415,7 @@ def test_shared_memory_across_processes(self): sms.close() + @unittest.skip("TODO: RUSTPYTHON; flaky") @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") def test_shared_memory_SharedMemoryServer_ignores_sigint(self): # bpo-36368: protect SharedMemoryManager server process from @@ -4416,7 +4440,9 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() + @unittest.skip("TODO: RUSTPYTHON: sem_unlink cleanup race causes spurious stderr output") @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. @@ -4667,6 +4693,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted @@ -4694,6 +4721,7 @@ def test_shared_memory_untracking(self): mem.close() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=True, it must cause the memory to be deleted when @@ -4787,6 +4815,8 @@ def test_finalize(self): result = [obj for obj in iter(conn.recv, 'STOP')] self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) + # TODO: RUSTPYTHON - gc.get_threshold() and gc.set_threshold() not implemented + @unittest.expectedFailure @support.requires_resource('cpu') def test_thread_safety(self): # bpo-24484: _run_finalizers() should be thread-safe @@ -5414,6 +5444,8 @@ def run_in_child(cls, start_method): flags = (tuple(sys.flags), grandchild_flags) print(json.dumps(flags)) + # TODO: RUSTPYTHON - SyntaxError in subprocess after fork + @unittest.expectedFailure def test_flags(self): import json # start child process using unusual flags @@ -6457,28 +6489,13 @@ def test_std_streams_flushed_after_preload(self): if multiprocessing.get_start_method() != "forkserver": self.skipTest("forkserver specific test") - # Create a test module in the temporary directory on the child's path - # TODO: This can all be simplified once gh-126631 is fixed and we can - # use __main__ instead of a module. - dirname = os.path.join(self._temp_dir, 'preloaded_module') - init_name = os.path.join(dirname, '__init__.py') - os.mkdir(dirname) - with open(init_name, "w") as f: - cmd = '''if 1: - import sys - print('stderr', end='', file=sys.stderr) - print('stdout', end='', file=sys.stdout) - ''' - f.write(cmd) - name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') - env = {'PYTHONPATH': self._temp_dir} - _, out, err = test.support.script_helper.assert_python_ok(name, **env) + _, out, err = test.support.script_helper.assert_python_ok(name) # Check stderr first, as it is more likely to be useful to see in the # event of a failure. - self.assertEqual(err.decode().rstrip(), 'stderr') - self.assertEqual(out.decode().rstrip(), 'stdout') + self.assertEqual(err.decode().rstrip(), '__main____mp_main__') + self.assertEqual(out.decode().rstrip(), '__main____mp_main__') class MiscTestCase(unittest.TestCase): @@ -6574,13 +6591,15 @@ def tearDownClass(cls): # cycles. Trigger a garbage collection to break these cycles. test.support.gc_collect() - processes = set(multiprocessing.process._dangling) - set(cls.dangling[0]) + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in cls.dangling[0] if p.is_alive()} if processes: test.support.environment_altered = True support.print_warning(f'Dangling processes: {processes}') processes = None - threads = set(threading._dangling) - set(cls.dangling[1]) + # TODO: RUSTPYTHON: Filter out stopped threads since gc.collect() is a no-op + threads = {t for t in threading._dangling if t.is_alive()} - {t for t in cls.dangling[1] if t.is_alive()} if threads: test.support.environment_altered = True support.print_warning(f'Dangling threads: {threads}') @@ -6770,14 +6789,16 @@ def tearDownModule(): multiprocessing.set_start_method(old_start_method[0], force=True) # pause a bit so we don't get warning about dangling threads/processes - processes = set(multiprocessing.process._dangling) - set(dangling[0]) + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in dangling[0] if p.is_alive()} if processes: need_sleep = True test.support.environment_altered = True support.print_warning(f'Dangling processes: {processes}') processes = None - threads = set(threading._dangling) - set(dangling[1]) + # TODO: RUSTPYTHON: Filter out stopped threads since gc.collect() is a no-op + threads = {t for t in threading._dangling if t.is_alive()} - {t for t in dangling[1] if t.is_alive()} if threads: need_sleep = True test.support.environment_altered = True @@ -6804,3 +6825,52 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) + + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + from multiprocessing import resource_tracker + import time + + resource_tracker._resource_tracker._use_simple_format = %s + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """ % use_simple_format) + + rc, out, err = script_helper.assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 00000000000..24745941b08 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,177 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import swap_attr +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/audiodata/pluck-alaw.aifc b/Lib/test/audiodata/pluck-alaw.aifc deleted file mode 100644 index 3b7fbd2af75..00000000000 Binary files a/Lib/test/audiodata/pluck-alaw.aifc and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm16.aiff b/Lib/test/audiodata/pluck-pcm16.aiff deleted file mode 100644 index 6c8c40d1409..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm16.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm16.au b/Lib/test/audiodata/pluck-pcm16.au deleted file mode 100644 index 398f07f0719..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm16.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm24.aiff b/Lib/test/audiodata/pluck-pcm24.aiff deleted file mode 100644 index 8eba145a44d..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm24.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm24.au b/Lib/test/audiodata/pluck-pcm24.au deleted file mode 100644 index 0bb230418a3..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm24.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm32.aiff b/Lib/test/audiodata/pluck-pcm32.aiff deleted file mode 100644 index 46ac0373f6a..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm32.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm32.au b/Lib/test/audiodata/pluck-pcm32.au deleted file mode 100644 index 92ee5965e40..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm32.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm8.aiff b/Lib/test/audiodata/pluck-pcm8.aiff deleted file mode 100644 index 5de4f3b2d87..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm8.aiff and /dev/null differ diff --git a/Lib/test/audiodata/pluck-pcm8.au b/Lib/test/audiodata/pluck-pcm8.au deleted file mode 100644 index b7172c8f234..00000000000 Binary files a/Lib/test/audiodata/pluck-pcm8.au and /dev/null differ diff --git a/Lib/test/audiodata/pluck-ulaw.aifc b/Lib/test/audiodata/pluck-ulaw.aifc deleted file mode 100644 index 3085cf097fb..00000000000 Binary files a/Lib/test/audiodata/pluck-ulaw.aifc and /dev/null differ diff --git a/Lib/test/audiodata/pluck-ulaw.au b/Lib/test/audiodata/pluck-ulaw.au deleted file mode 100644 index 11103535c6b..00000000000 Binary files a/Lib/test/audiodata/pluck-ulaw.au and /dev/null differ diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py new file mode 100644 index 00000000000..b5a1fab404c --- /dev/null +++ b/Lib/test/autotest.py @@ -0,0 +1,5 @@ +# This should be equivalent to running regrtest.py from the cmdline. +# It can be especially handy if you're in an interactive shell, e.g., +# from test import autotest. +from test.libregrtest.main import main +main() diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000..84d2ca29953 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/cjkencodings/big5-utf8.txt b/Lib/test/cjkencodings/big5-utf8.txt new file mode 100644 index 00000000000..a0a534a964d --- /dev/null +++ b/Lib/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/big5.txt b/Lib/test/cjkencodings/big5.txt new file mode 100644 index 00000000000..f4424959e91 --- /dev/null +++ b/Lib/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +pb Python ϥάJ C library? +@bTާֳtoi, }oδճn骺t׬Oe +D. [ֶ}oδժt, ڭ̫K`ƱQΤ@Ǥw}on +library, æ@ fast prototyping programming language i +Ѩϥ. ثe\\hh library OH C g, Python O@ +fast prototyping programming language. Gڭ̧ƱNJ +C library Python ҤդξX. 䤤̥Dn]Oڭ̩ +nQתDNO: + diff --git a/Lib/test/cjkencodings/big5hkscs-utf8.txt b/Lib/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 index 00000000000..f744ce9ae08 --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/Lib/test/cjkencodings/big5hkscs.txt b/Lib/test/cjkencodings/big5hkscs.txt new file mode 100644 index 00000000000..81c42b3503d --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +E\sڍ +fb diff --git a/Lib/test/cjkencodings/cp949-utf8.txt b/Lib/test/cjkencodings/cp949-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/cp949.txt b/Lib/test/cjkencodings/cp949.txt new file mode 100644 index 00000000000..16549aa5e49 --- /dev/null +++ b/Lib/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +c氢 ݶ + +!! Вp ިR ѵ . . +䬿Ѵ . . . . ʫR ! ! !. + ٤_  O h O +j ʫR . . . . ֚f ѱ ސtƒO  +;R ! ! 䬿 ʫɱ ߾ ɱŴ 䬴ɵR ۾֊ +޷ ǴR  Ĩ!! ҡ* + diff --git a/Lib/test/cjkencodings/euc_jisx0213-utf8.txt b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 index 00000000000..9a56a2e18bd --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ ト゚ トキ喝塀 𡚴𪎌 麀齁𩛰 diff --git a/Lib/test/cjkencodings/euc_jisx0213.txt b/Lib/test/cjkencodings/euc_jisx0213.txt new file mode 100644 index 00000000000..51e9268ca98 --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + +Τ ȥ ԏ diff --git a/Lib/test/cjkencodings/euc_jp-utf8.txt b/Lib/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/euc_jp.txt b/Lib/test/cjkencodings/euc_jp.txt new file mode 100644 index 00000000000..9da6b5d83da --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + diff --git a/Lib/test/cjkencodings/euc_kr-utf8.txt b/Lib/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 index 00000000000..16c37412b6a --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/Lib/test/cjkencodings/euc_kr.txt b/Lib/test/cjkencodings/euc_kr.txt new file mode 100644 index 00000000000..f68dd350289 --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ + ̽(Python) , α׷ Դϴ. ̽ +ȿ ȿ üα׷ +մϴ. ̽ () Ÿ, ׸ +ȯ ̽ ũð о߿ κ ÷ +ø̼ ִ ̻ ݴϴ. + +ù: ƶ ԤФԤԤФԾ~ ԤҤŭ! ԤѤݾ ԤȤϴ. ԤΤ. ׷ ԤѤ. diff --git a/Lib/test/cjkencodings/gb18030-utf8.txt b/Lib/test/cjkencodings/gb18030-utf8.txt new file mode 100644 index 00000000000..2060d2593eb --- /dev/null +++ b/Lib/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/Lib/test/cjkencodings/gb18030.txt b/Lib/test/cjkencodings/gb18030.txt new file mode 100644 index 00000000000..5d1f6dca232 --- /dev/null +++ b/Lib/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: +51332131 760463 858635 3195 0930 435755 5509899304 292599. + diff --git a/Lib/test/cjkencodings/gb2312-utf8.txt b/Lib/test/cjkencodings/gb2312-utf8.txt new file mode 100644 index 00000000000..efb7d8f95cd --- /dev/null +++ b/Lib/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 + diff --git a/Lib/test/cjkencodings/gb2312.txt b/Lib/test/cjkencodings/gb2312.txt new file mode 100644 index 00000000000..1536ac10b9e --- /dev/null +++ b/Lib/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + diff --git a/Lib/test/cjkencodings/gbk-utf8.txt b/Lib/test/cjkencodings/gbk-utf8.txt new file mode 100644 index 00000000000..75bbd31ec5a --- /dev/null +++ b/Lib/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python(派森)语言是一种功能强大而完善的通用型计算机程序设计语言, +已经具有十多年的发展历史,成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点,适合完成各种高层任务,几乎可以在所有的操作系统中 +运行。这种语言简单而强大,适合各种人士学习使用。目前,基于这 +种语言的相关技术正在飞速的发展,用户数量急剧扩大,相关的资源非常多。 +如何在 Python 中使用既有的 C library? + 在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/gbk.txt b/Lib/test/cjkencodings/gbk.txt new file mode 100644 index 00000000000..8788f8a2dc4 --- /dev/null +++ b/Lib/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: + diff --git a/Lib/test/cjkencodings/hz-utf8.txt b/Lib/test/cjkencodings/hz-utf8.txt new file mode 100644 index 00000000000..7c11735c1f1 --- /dev/null +++ b/Lib/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲,勿施於人。Bye. diff --git a/Lib/test/cjkencodings/hz.txt b/Lib/test/cjkencodings/hz.txt new file mode 100644 index 00000000000..f882d463447 --- /dev/null +++ b/Lib/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/Lib/test/cjkencodings/iso2022_jp-utf8.txt b/Lib/test/cjkencodings/iso2022_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/iso2022_jp.txt b/Lib/test/cjkencodings/iso2022_jp.txt new file mode 100644 index 00000000000..fc398d64ad2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp.txt @@ -0,0 +1,7 @@ +Python $B$N3+H/$O!"(B1990 $BG/$4$m$+$i3+;O$5$l$F$$$^$9!#(B +$B3+H/e$NL\E*$K$O$"$^$jE,$7$F$$$^$;$s$G$7$?!#(B +$B$3$N$?$a!"(BGuido $B$O$h$j$E$1$^$7$?!#(B +$B$3$N$h$&$JGX7J$+$i@8$^$l$?(B Python $B$N8@8l@_7W$O!"!V%7%s%W%k!W$G!V=,F@$,MF0W!W$H$$$&L\I8$K=EE@$,CV$+$l$F$$$^$9!#(B +$BB?$/$N%9%/%j%W%H7O8@8l$G$O%f!<%6$NL\@h$NMxJX@-$rM%@h$7$F?'!9$J5!G=$r8@8lMWAG$H$7$Fl9g$,B?$$$N$G$9$,!"(BPython $B$G$O$=$&$$$C$?>.:Y9)$,DI2C$5$l$k$3$H$O$"$^$j$"$j$^$;$s!#(B +$B8@8l<+BN$N5!G=$O:G>.8B$K2!$5$(!"I,MW$J5!G=$O3HD%%b%8%e!<%k$H$7$FDI2C$9$k!"$H$$$&$N$,(B Python $B$N%]%j%7!<$G$9!#(B + diff --git a/Lib/test/cjkencodings/iso2022_kr-utf8.txt b/Lib/test/cjkencodings/iso2022_kr-utf8.txt new file mode 100644 index 00000000000..d5c9d6eeeb2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓩~ 큼! 금없이 전니다. 그런거 다. diff --git a/Lib/test/cjkencodings/iso2022_kr.txt b/Lib/test/cjkencodings/iso2022_kr.txt new file mode 100644 index 00000000000..2cece21c5dd --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr.txt @@ -0,0 +1,7 @@ +$)C!] FD@L=c(Python)@: 9h?l1b =10m, 0-7BGQ GA7N1W7!9V >p>n@T4O4Y. FD@L=c@: +H?@2@{@N 0mF(iPd:)GQ 9.9}0z 5?@{ E8@LGN, 1W8.0m @NEMGA8.FC +H/0f@: FD@L=c@; =:E)83FC0z ?)7/ :P>_?!<-?M 4k:N:P@G GC7'F{?!<-@G :|8% +>VGC8.DI@Lp>n7N 885i>nA]4O4Y. + +!YC90!3!: 3/>F6s >1~ E-! 1]>x@L @|4O4Y. 1W710E 4Y. diff --git a/Lib/test/cjkencodings/johab-utf8.txt b/Lib/test/cjkencodings/johab-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/johab.txt b/Lib/test/cjkencodings/johab.txt new file mode 100644 index 00000000000..067781b785a --- /dev/null +++ b/Lib/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +wba \ũa + +s!! gÚ zٯٯٯ w ѕ . . + as_datetime) - self.assertRaises(TypeError, lambda: as_datetime > as_date) - self.assertRaises(TypeError, lambda: as_date >= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime >= as_date) - - # Nevertheless, comparison should work with the base-class (date) - # projection if use of a date method is forced. - self.assertEqual(as_date.__eq__(as_datetime), True) - different_day = (as_date.day + 1) % 20 + 1 - as_different = as_datetime.replace(day= different_day) - self.assertEqual(as_date.__eq__(as_different), False) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. - date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) - self.assertEqual(as_date, date_sc) - self.assertEqual(date_sc, as_date) - # Ditto for datetimes. - datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, - as_date.day, 0, 0, 0) - self.assertEqual(as_datetime, datetime_sc) - self.assertEqual(datetime_sc, as_datetime) + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): @@ -6679,6 +6848,126 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + # TODO: RUSTPYTHON + # AssertionError: Process return code is 1 + @unittest.expectedFailure + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 3427b51b60a..8b137891791 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,5 +1 @@ -# We import importlib *ASAP* in order to test #15386 -import importlib -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES -from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0a97c8c19b1..e7a12e4d0b6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,8 +1,9 @@ import argparse -import os +import os.path +import shlex import sys -from test import support -from test.support import os_helper +from test.support import os_helper, Py_DEBUG +from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter USAGE = """\ @@ -27,8 +28,10 @@ Additional option details: -r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. +int seed value for the randomizer. The randseed value will be used +to set seeds for all random usages in tests +(including randomizing the tests order if -r is set). +By default we always set random seed, but do not randomize test order. -s On the first invocation of regrtest using -s, the first test file found or the first test file given on the command line is run, and the name of @@ -41,11 +44,19 @@ doing memory analysis on the Python interpreter, which process tends to consume too many resources to run the full regression test non-stop. --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). +-S is used to resume running tests after an interrupted run. It will +maintain the order a standard run (i.e. it assumes -r is not used). This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. +reason and you want to resume the run from where you left off rather +than starting from the beginning. Note: this is different from --prioritize. + +--prioritize is used to influence the order of selected tests, such that +the tests listed as an argument are executed first. This is especially +useful when combined with -j and -r to pin the longest-running tests +to start at the beginning of a test run. Pass --prioritize=test_a,test_b +to make test_a run first, followed by test_b, and then the other tests. +If test_a wasn't selected for execution by regular means, --prioritize will +not make it execute. -f reads the names of tests from the file given as f's argument, one or more test names per line. Whitespace is ignored. Blank lines and @@ -84,36 +95,40 @@ The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: - all - Enable all special resources. + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) - none - Disable all special resources (this is the default). + curses - Tests that use curses and will modify the terminal's + state and output modes. - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2 GiB of disk space temporarily. - curses - Tests that use curses and will modify the terminal's - state and output modes. + extralargefile - Like 'largefile', but even larger (and slower). - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2 GiB of disk space temporarily. + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. + decimal - Test the decimal module against a large suite that + verifies compliance with standards. - decimal - Test the decimal module against a large suite that - verifies compliance with standards. + cpu - Used for certain CPU-heavy tests. - cpu - Used for certain CPU-heavy tests. + walltime - Long running but not CPU-bound tests. - subprocess Run all tests for the subprocess module. + subprocess Run all tests for the subprocess module. - urlfetch - It is okay to download files required on testing. + urlfetch - It is okay to download files required on testing. - gui - Run tests that require a running GUI. + gui - Run tests that require a running GUI. - tzdata - Run tests that require timezone data. + tzdata - Run tests that require timezone data. To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the @@ -128,17 +143,53 @@ """ -ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') +class Namespace(argparse.Namespace): + def __init__(self, **kwargs) -> None: + self.ci = False + self.testdir = None + self.verbose = 0 + self.quiet = False + self.exclude = False + self.cleanup = False + self.wait = False + self.list_cases = False + self.list_tests = False + self.single = False + self.randomize = False + self.fromfile = None + self.fail_env_changed = False + self.use_resources: list[str] = [] + self.trace = False + self.coverdir = 'coverage' + self.runleaks = False + self.huntrleaks: tuple[int, int, str] | None = None + self.rerun = False + self.verbose3 = False + self.print_slow = False + self.random_seed = None + self.use_mp = None + self.parallel_threads = None + self.forever = False + self.header = False + self.failfast = False + self.match_tests: TestFilter = [] + self.pgo = False + self.pgo_extended = False + self.tsan = False + self.tsan_parallel = False + self.worker_json = None + self.start = None + self.timeout = None + self.memlimit = None + self.threshold = None + self.fail_rerun = False + self.tempdir = None + self._add_python_opts = True + self.xmlpath = None + self.single_process = False + + super().__init__(**kwargs) -# Other resources excluded from --use=all: -# -# - extralagefile (ex: test_zipfile64): really too slow to be enabled -# "by default" -# - tzdata: while needed to validate fully test_datetime, it makes -# test_datetime too slow (15-20 min on some buildbots) and so is disabled by -# default (see bpo-30822). -RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') class _ArgParser(argparse.ArgumentParser): @@ -146,6 +197,20 @@ def error(self, message): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -155,6 +220,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -164,23 +230,35 @@ def _create_parser(): # We add help explicitly to control what argument group it renders under. group.add_argument('-h', '--help', action='help', help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, + group.add_argument('--fast-ci', action='store_true', + help='Fast Continuous Integration (CI) mode used by ' + 'GitHub Actions') + group.add_argument('--slow-ci', action='store_true', + help='Slow Continuous Integration (CI) mode used by ' + 'buildbot workers') + group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' 'more than TIMEOUT seconds; disabled if TIMEOUT ' 'is negative or equals to zero') group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' 'to be attached') - group.add_argument('--worker-args', metavar='ARGS') group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + + help='resume an interrupted run at the following test.' + more_details) + group.add_argument('-p', '--python', metavar='PYTHON', + help='Command to run Python test subprocesses with.') + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a global random seed') group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', + group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') + group.add_argument('--verbose2', action='store_true', dest='rerun', + help='deprecated alias to --rerun') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -193,10 +271,13 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') + group.add_argument('--no-randomize', dest='no_randomize', action='store_true', + help='do not randomize test execution order, even if ' + 'it would be implied by another option') + group.add_argument('--prioritize', metavar='TEST1,TEST2,...', + action='append', type=priority_list, + help='select these tests first, even if the order is' + ' randomized.' + more_details) group.add_argument('-f', '--fromfile', metavar='FILE', help='read names of tests to run from a file.' + more_details) @@ -206,12 +287,21 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') + group.add_argument('-i', '--ignore', metavar='PAT', + dest='match_tests', action=FilterAction, const=False, + help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') + group.add_argument('--ignorefile', metavar='FILENAME', + dest='match_tests', + action=FromFileFilterAction, const=False, + help='similar to --matchfile but it receives patterns ' + 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -227,9 +317,6 @@ def _create_parser(): '(instead of the Python stdlib test suite)') group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_const', const=2, - default=1, - help='deprecated alias to --fail-env-changed') group.add_argument('-L', '--runleaks', action='store_true', help='run the leaks(1) command just before exit.' + more_details) @@ -240,6 +327,16 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') + group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS', + type=int, + help='run copies of each test in PARALLEL_THREADS at ' + 'once') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -257,7 +354,7 @@ def _create_parser(): help='suppress error message boxes on Windows') group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' - 'error happens') + 'error happens; imply --failfast') group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") @@ -265,16 +362,33 @@ def _create_parser(): help='only write the name of test cases that will be run' ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', - help='enable Profile Guided Optimization training') + help='enable Profile Guided Optimization (PGO) training') + group.add_argument('--pgo-extended', action='store_true', + help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') + group.add_argument('--tsan-parallel', action='store_true', + help='run a subset of test cases that are appropriate ' + 'for TSAN with `--parallel-threads=N`') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') + group.add_argument('--fail-rerun', action='store_true', + help='if a test failed and then passed when re-run, ' + 'mark the tests as failed') group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', help='writes JUnit-style XML results to the specified ' 'file') - group.add_argument('--tempdir', dest='tempdir', metavar='PATH', + group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') + group.add_argument('--cleanup', action='store_true', + help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') + group.add_argument('--dont-add-python-opts', dest='_add_python_opts', + action='store_false', + help="internal option, don't use it") return parser @@ -307,21 +421,18 @@ def resources_list(string): return u +def priority_list(string): + return string.split(",") + + def _parse_args(args, **kwargs): # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=1, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) + ns = Namespace() for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' 'for this function' % k) setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] parser = _create_parser() # Issue #14191: argparse doesn't support "intermixed" positional and @@ -330,19 +441,81 @@ def _parse_args(args, **kwargs): for arg in ns.args: if arg.startswith('-'): parser.error("unrecognized arguments: %s" % arg) - sys.exit(1) - if ns.findleaks > 1: - # --findleaks implies --fail-env-changed + if ns.timeout is not None: + # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT + # can be used by "make buildbottest" and "make test". + if ns.timeout != "": + try: + ns.timeout = float(ns.timeout) + except ValueError: + parser.error(f"invalid timeout value: {ns.timeout!r}") + else: + ns.timeout = None + + # Continuous Integration (CI): common options for fast/slow CI modes + if ns.slow_ci or ns.fast_ci: + # Similar to options: + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 + if ns.use_mp is None: + ns.use_mp = 0 + ns.randomize = True ns.fail_env_changed = True + if ns.python is None: + ns.rerun = True + ns.print_slow = True + if not ns.verbose: + ns.verbose3 = True + else: + # --verbose has the priority over --verbose3 + pass + else: + ns._add_python_opts = False + + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + + # When both --slow-ci and --fast-ci options are present, + # --slow-ci has the priority + if ns.slow_ci: + # Similar to: -u "all" --timeout=1200 + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all']) + if ns.timeout is None: + ns.timeout = 1200 # 20 minutes + elif ns.fast_ci: + # Similar to: -u "all,-cpu" --timeout=600 + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all', '-cpu']) + if ns.timeout is None: + ns.timeout = 600 # 10 minutes + if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") - if ns.use_mp is not None and ns.trace: - parser.error("-T and -j don't go together!") + if ns.trace: + if ns.use_mp is not None: + if not Py_DEBUG: + parser.error("need --with-pydebug to use -T and -j together") + else: + print( + "Warning: collecting coverage without -j is imprecise. Configure" + " --with-pydebug and run -m test -T -j for best results.", + file=sys.stderr + ) + if ns.python is not None: + if ns.use_mp is None: + parser.error("-p requires -j!") + # The "executable" may be two or more parts, e.g. "node python.js" + ns.python = shlex.split(ns.python) if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") - if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): + if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3): parser.error("--pgo/-v don't go together!") + if ns.pgo_extended: + ns.pgo = True # pgo_extended implies pgo if ns.nowindows: print("Warning: the --nowindows (-n) option is deprecated. " @@ -353,10 +526,6 @@ def _parse_args(args, **kwargs): if ns.timeout is not None: if ns.timeout <= 0: ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) if ns.use: for a in ns.use: for r in a: @@ -377,18 +546,40 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.no_randomize: + ns.randomize = False if ns.verbose: ns.header = True - if ns.huntrleaks and ns.verbose3: + + # When -jN option is used, a worker process does not use --verbose3 + # and so -R 3:3 -jN --verbose3 just works as expected: there is no false + # alarm about memory leak. + if ns.huntrleaks and ns.verbose3 and ns.use_mp is None: + # run_single_test() replaces sys.stdout with io.StringIO if verbose3 + # is true. In this case, huntrleaks sees an write into StringIO as + # a memory leak, whereas it is not (gh-71290). ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " - "--huntrleaks: see http://bugs.python.org/issue27103", + "--huntrleaks without -jN option", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) + + if ns.forever: + # --forever implies --failfast + ns.failfast = True + + if ns.huntrleaks: + warmup, repetitions, _ = ns.huntrleaks + if warmup < 1 or repetitions < 1: + msg = ("Invalid values for the --huntrleaks/-R parameters. The " + "number of warmups and repetitions must be at least 1 " + "each (1:1).") + print(msg, file=sys.stderr, flush=True) + sys.exit(2) + + ns.prioritize = [ + test + for test_list in (ns.prioritize or ()) + for test in test_list + ] return ns diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py new file mode 100644 index 00000000000..41372e427ff --- /dev/null +++ b/Lib/test/libregrtest/filter.py @@ -0,0 +1,77 @@ +import itertools +import operator +import re + + +# By default, don't filter tests +_test_matchers = () +_test_patterns = () + + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result + + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def get_match_tests(): + global _test_patterns + return _test_patterns + + +def set_match_tests(patterns): + global _test_matchers, _test_patterns + + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns + + +def _compile_match_function(patterns): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect_cmd utility only uses such full test identifiers. + return set(patterns).__contains__ + else: + import fnmatch + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id, regex_match=regex_match): + if regex_match(test_id): + # The regex matches the whole identifier, for example + # 'test.test_os.FileTests.test_access'. + return True + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + return match_test_regex diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py new file mode 100644 index 00000000000..f01c1240774 --- /dev/null +++ b/Lib/test/libregrtest/findtests.py @@ -0,0 +1,110 @@ +import os +import sys +import unittest +from collections.abc import Container + +from test import support + +from .filter import match_test, set_match_tests +from .utils import ( + StrPath, TestName, TestTuple, TestList, TestFilter, + abs_module_name, count, printlist) + + +# If these test directories are encountered recurse into them and treat each +# "test_*.py" file or each sub-directory as a separate test module. This can +# increase parallelism. +# +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. +SPLITTESTDIRS: set[TestName] = { + "test_asyncio", + "test_concurrent_futures", + "test_doctests", + "test_future_stmt", + "test_gdb", + "test_inspect", + "test_pydoc", + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", +} + + +def findtestdir(path: StrPath | None = None) -> StrPath: + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir + + +def findtests(*, testdir: StrPath | None = None, exclude: Container[str] = (), + split_test_dirs: set[TestName] = SPLITTESTDIRS, + base_mod: str = "") -> TestList: + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + tests = [] + for name in os.listdir(testdir): + mod, ext = os.path.splitext(name) + if (not mod.startswith("test_")) or (mod in exclude): + continue + if base_mod: + fullname = f"{base_mod}.{mod}" + else: + fullname = mod + if fullname in split_test_dirs: + subdir = os.path.join(testdir, mod) + if not base_mod: + fullname = f"test.{mod}" + tests.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=fullname)) + elif ext in (".py", ""): + tests.append(fullname) + return sorted(tests) + + +def split_test_packages(tests, *, testdir: StrPath | None = None, + exclude: Container[str] = (), + split_test_dirs=SPLITTESTDIRS) -> list[TestName]: + testdir = findtestdir(testdir) + splitted = [] + for name in tests: + if name in split_test_dirs: + subdir = os.path.join(testdir, name) + splitted.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=name)) + else: + splitted.append(name) + return splitted + + +def _list_cases(suite: unittest.TestSuite) -> None: + for test in suite: + if isinstance(test, unittest.loader._FailedTest): # type: ignore[attr-defined] + continue + if isinstance(test, unittest.TestSuite): + _list_cases(test) + elif isinstance(test, unittest.TestCase): + if match_test(test): + print(test.id()) + +def list_cases(tests: TestTuple, *, + match_tests: TestFilter | None = None, + test_dir: StrPath | None = None) -> None: + support.verbose = False + set_match_tests(match_tests) + + skipped = [] + for test_name in tests: + module_name = abs_module_name(test_name, test_dir) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(module_name) + _list_cases(suite) + except unittest.SkipTest: + skipped.append(test_name) + + if skipped: + sys.stdout.flush() + stderr = sys.stderr + print(file=stderr) + print(count(len(skipped), "test"), "skipped:", file=stderr) + printlist(skipped, file=stderr) diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py new file mode 100644 index 00000000000..fa1d4d575c8 --- /dev/null +++ b/Lib/test/libregrtest/logger.py @@ -0,0 +1,89 @@ +import os +import time + +from test.support import MS_WINDOWS +from .results import TestResults +from .runtests import RunTests +from .utils import print_warning + +if MS_WINDOWS: + from .win_utils import WindowsLoadTracker + + +class Logger: + def __init__(self, results: TestResults, quiet: bool, pgo: bool): + self.start_time = time.perf_counter() + self.test_count_text = '' + self.test_count_width = 3 + self.win_load_tracker: WindowsLoadTracker | None = None + self._results: TestResults = results + self._quiet: bool = quiet + self._pgo: bool = pgo + + def log(self, line: str = '') -> None: + empty = not line + + # add the system load prefix: "load avg: 1.80 " + load_avg = self.get_load_avg() + if load_avg is not None: + line = f"load avg: {load_avg:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " + log_time = time.perf_counter() - self.start_time + + mins, secs = divmod(int(log_time), 60) + hours, mins = divmod(mins, 60) + formatted_log_time = "%d:%02d:%02d" % (hours, mins, secs) + + line = f"{formatted_log_time} {line}" + if empty: + line = line[:-1] + + print(line, flush=True) + + def get_load_avg(self) -> float | None: + if hasattr(os, 'getloadavg'): + try: + return os.getloadavg()[0] + except OSError: + pass + if self.win_load_tracker is not None: + return self.win_load_tracker.getloadavg() + return None + + def display_progress(self, test_index: int, text: str) -> None: + if self._quiet: + return + results = self._results + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count_text}" + fails = len(results.bad) + len(results.env_changed) + if fails and not self._pgo: + line = f"{line}/{fails}" + self.log(f"[{line}] {text}") + + def set_tests(self, runtests: RunTests) -> None: + if runtests.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(runtests.tests)) + self.test_count_width = len(self.test_count_text) - 1 + + def start_load_tracker(self) -> None: + if not MS_WINDOWS: + return + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print_warning(f'Failed to create WindowsLoadTracker: {error}') + + def stop_load_tracker(self) -> None: + if self.win_load_tracker is None: + return + self.win_load_tracker.close() + self.win_load_tracker = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1d19e1e4ac..0fc2548789e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,42 +1,33 @@ -import datetime -import faulthandler -import json -import locale import os -import platform import random import re +import shlex import sys import sysconfig -import tempfile import time -import unittest -from test.libregrtest.cmdline import _parse_args -from test.libregrtest.runtest import ( - findtests, runtest, get_abs_module, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, - PROGRESS_MIN_TIME, format_test_result) -from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import removepy, count, format_duration, printlist -from test import support -from test.support import os_helper, import_helper - - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = sysconfig.get_config_var('abs_builddir') - if TEMPDIR is None: - # bpo-30284: On Windows, only srcdir is available. Using abs_builddir - # mostly matters on UNIX when building Python out of the source tree, - # especially when the source tree is read only. - TEMPDIR = sysconfig.get_config_var('srcdir') - TEMPDIR = os.path.join(TEMPDIR, 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) +import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import NoReturn + +from test.support import os_helper, MS_WINDOWS, flush_std_streams + +from .cmdline import _parse_args, Namespace +from .findtests import findtests, split_test_packages, list_cases +from .logger import Logger +from .pgo import setup_pgo_tests +from .result import TestResult +from .results import TestResults, EXITCODE_INTERRUPTED +from .runtests import RunTests, HuntRefleak +from .setup import setup_process, setup_test_dir +from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests, setup_tsan_parallel_tests +from .utils import ( + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, + strip_py_suffix, count, format_duration, + printlist, get_temp_dir, get_work_dir, exit_timeout, + display_header, cleanup_temp_dir, print_warning, + is_cross_compiled, get_host_runner, + EXIT_TIMEOUT) class Regrtest: @@ -57,357 +48,379 @@ class Regrtest: files beginning with test_ will be used. The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, + single, randomize, use_resources, trace, coverdir, print_slow, and random_seed) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ - def __init__(self): - # Namespace of command line options - self.ns = None - - # tests - self.tests = [] - self.selected = [] - - # test results - self.good = [] - self.bad = [] - self.skipped = [] - self.resource_denieds = [] - self.environment_changed = [] - self.run_no_tests = [] - self.rerun = [] - self.first_result = None - self.interrupted = False - - # used by --slow - self.test_times = [] - - # used by --coverage, trace.Trace instance - self.tracer = None - - # used to display the progress bar "[ 3/100]" - self.start_time = time.monotonic() - self.test_count = '' - self.test_count_width = 1 - - # used by --single - self.next_single_test = None - self.next_single_filename = None - - # used by --junit-xml - self.testsuite_xml = None - - self.win_load_tracker = None - - def get_executed(self): - return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denieds) | set(self.environment_changed) - | set(self.run_no_tests)) - - def accumulate_result(self, result, rerun=False): - test_name = result.test_name - ok = result.result - - if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun: - self.test_times.append((result.test_time, test_name)) - - if ok == PASSED: - self.good.append(test_name) - elif ok in (FAILED, CHILD_ERROR): - if not rerun: - self.bad.append(test_name) - elif ok == ENV_CHANGED: - self.environment_changed.append(test_name) - elif ok == SKIPPED: - self.skipped.append(test_name) - elif ok == RESOURCE_DENIED: - self.skipped.append(test_name) - self.resource_denieds.append(test_name) - elif ok == TEST_DID_NOT_RUN: - self.run_no_tests.append(test_name) - elif ok == INTERRUPTED: - self.interrupted = True + def __init__(self, ns: Namespace, _add_python_opts: bool = False): + # Log verbosity + self.verbose: int = int(ns.verbose) + self.quiet: bool = ns.quiet + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan + self.tsan_parallel: bool = ns.tsan_parallel + + # Test results + self.results: TestResults = TestResults() + self.first_state: str | None = None + + # Logger + self.logger = Logger(self.results, self.quiet, self.pgo) + + # Actions + self.want_header: bool = ns.header + self.want_list_tests: bool = ns.list_tests + self.want_list_cases: bool = ns.list_cases + self.want_wait: bool = ns.wait + self.want_cleanup: bool = ns.cleanup + self.want_rerun: bool = ns.rerun + self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect + + self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) + self.want_add_python_opts: bool = (_add_python_opts + and ns._add_python_opts) + + # Select tests + self.match_tests: TestFilter = ns.match_tests + self.exclude: bool = ns.exclude + self.fromfile: StrPath | None = ns.fromfile + self.starting_test: TestName | None = ns.start + self.cmdline_args: TestList = ns.args + + # Workers + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process + elif ns.use_mp <= 0: + num_workers = -1 # run in parallel, use the number of CPUs else: - raise ValueError("invalid test result: %r" % ok) - - if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}: - self.bad.remove(test_name) - - xml_data = result.xml_data - if xml_data: - import xml.etree.ElementTree as ET - for e in xml_data: - try: - self.testsuite_xml.append(ET.fromstring(e)) - except ET.ParseError: - print(xml_data, file=sys.__stderr__) - raise - - def display_progress(self, test_index, text): - if self.ns.quiet: - return - - # "[ 51/405/1] test_tcl passed" - line = f"{test_index:{self.test_count_width}}{self.test_count}" - fails = len(self.bad) + len(self.environment_changed) - if fails and not self.ns.pgo: - line = f"{line}/{fails}" - line = f"[{line}] {text}" - - # add the system load prefix: "load avg: 1.80 " - load_avg = self.getloadavg() - if load_avg is not None: - line = f"load avg: {load_avg:.2f} {line}" - - # add the timestamp prefix: "0:01:05 " - test_time = time.monotonic() - self.start_time - test_time = datetime.timedelta(seconds=int(test_time)) - line = f"{test_time} {line}" - print(line, flush=True) - - def parse_args(self, kwargs): - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later", file=sys.stderr) - ns.timeout = None + num_workers = ns.use_mp # run in parallel + self.num_workers: int = num_workers + self.worker_json: StrJSON | None = ns.worker_json + + # Options to run tests + self.fail_fast: bool = ns.failfast + self.fail_env_changed: bool = ns.fail_env_changed + self.fail_rerun: bool = ns.fail_rerun + self.forever: bool = ns.forever + self.output_on_failure: bool = ns.verbose3 + self.timeout: float | None = ns.timeout + if ns.huntrleaks: + warmups, runs, filename = ns.huntrleaks + filename = os.path.abspath(filename) + self.hunt_refleak: HuntRefleak | None = HuntRefleak(warmups, runs, filename) + else: + self.hunt_refleak = None + self.test_dir: StrPath | None = ns.testdir + self.junit_filename: StrPath | None = ns.xmlpath + self.memory_limit: str | None = ns.memlimit + self.gc_threshold: int | None = ns.threshold + self.use_resources: tuple[str, ...] = tuple(ns.use_resources) + if ns.python: + self.python_cmd: tuple[str, ...] | None = tuple(ns.python) + else: + self.python_cmd = None + self.coverage: bool = ns.trace + self.coverage_dir: StrPath | None = ns.coverdir + self._tmp_dir: StrPath | None = ns.tempdir + + # Randomize + self.randomize: bool = ns.randomize + if ('SOURCE_DATE_EPOCH' in os.environ + # don't use the variable if empty + and os.environ['SOURCE_DATE_EPOCH'] + ): + self.randomize = False + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed + self.prioritize_tests: tuple[str, ...] = tuple(ns.prioritize) - if ns.xmlpath: - support.junit_xml_list = self.testsuite_xml = [] + self.parallel_threads = ns.parallel_threads - # Strip .py extensions. - removepy(ns.args) + # tests + self.first_runtests: RunTests | None = None - return ns + # used by --slowest + self.print_slowest: bool = ns.print_slow - def find_tests(self, tests): - self.tests = tests + # used to display the progress bar "[ 3/100]" + self.start_time = time.perf_counter() - if self.ns.single: - self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + # used by --single + self.single_test_run: bool = ns.single + self.next_single_test: TestName | None = None + self.next_single_filename: StrPath | None = None + + def log(self, line: str = '') -> None: + self.logger.log(line) + + def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList | None]: + if tests is None: + tests = [] + if self.single_test_run: + self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: next_test = fp.read().strip() - self.tests = [next_test] + tests = [next_test] except OSError: pass - if self.ns.fromfile: - self.tests = [] + if self.fromfile: + tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') - with open(os.path.join(os_helper.SAVEDCWD, self.ns.fromfile)) as fp: + with open(os.path.join(os_helper.SAVEDCWD, self.fromfile)) as fp: for line in fp: line = line.split('#', 1)[0] line = line.strip() match = regex.search(line) if match is not None: - self.tests.append(match.group()) - - removepy(self.tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if self.ns.exclude: - for arg in self.ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - self.ns.args = [] - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if self.ns.testdir: - alltests = findtests(self.ns.testdir, list(), set()) - else: - alltests = findtests(self.ns.testdir, stdtests, nottests) + tests.append(match.group()) - if not self.ns.fromfile: - self.selected = self.tests or self.ns.args or alltests + strip_py_suffix(tests) + + exclude_tests = set() + if self.exclude: + for arg in self.cmdline_args: + exclude_tests.add(arg) + self.cmdline_args = [] + + if self.pgo: + # add default PGO tests if no tests are specified + setup_pgo_tests(self.cmdline_args, self.pgo_extended) + + if self.tsan: + setup_tsan_tests(self.cmdline_args) + + if self.tsan_parallel: + setup_tsan_parallel_tests(self.cmdline_args) + + alltests = findtests(testdir=self.test_dir, + exclude=exclude_tests) + + if not self.fromfile: + selected = tests or self.cmdline_args + if exclude_tests: + # Support "--pgo/--tsan -x test_xxx" command + selected = [name for name in selected + if name not in exclude_tests] + if selected: + selected = split_test_packages(selected) + else: + selected = alltests else: - self.selected = self.tests - if self.ns.single: - self.selected = self.selected[:1] + selected = tests + + if self.single_test_run: + selected = selected[:1] try: - pos = alltests.index(self.selected[0]) + pos = alltests.index(selected[0]) self.next_single_test = alltests[pos + 1] except IndexError: pass # Remove all the selected tests that precede start if it's set. - if self.ns.start: + if self.starting_test: try: - del self.selected[:self.selected.index(self.ns.start)] + del selected[:selected.index(self.starting_test)] except ValueError: - print("Couldn't find starting test (%s), using all tests" - % self.ns.start, file=sys.stderr) + print(f"Cannot find starting test: {self.starting_test}") + sys.exit(1) - if self.ns.randomize: - if self.ns.random_seed is None: - self.ns.random_seed = random.randrange(10000000) - random.seed(self.ns.random_seed) - random.shuffle(self.selected) + random.seed(self.random_seed) + if self.randomize: + random.shuffle(selected) - def list_tests(self): - for name in self.selected: - print(name) - - def _list_cases(self, suite): - for test in suite: - if isinstance(test, unittest.loader._FailedTest): - continue - if isinstance(test, unittest.TestSuite): - self._list_cases(test) - elif isinstance(test, unittest.TestCase): - if support.match_test(test): - print(test.id()) - - def list_cases(self): - support.verbose = False - support.set_match_tests(self.ns.match_tests) - - for test_name in self.selected: - abstest = get_abs_module(self.ns, test_name) + for priority_test in reversed(self.prioritize_tests): try: - suite = unittest.defaultTestLoader.loadTestsFromName(abstest) - self._list_cases(suite) - except unittest.SkipTest: - self.skipped.append(test_name) + selected.remove(priority_test) + except ValueError: + print(f"warning: --prioritize={priority_test} used" + f" but test not actually selected") + continue + else: + selected.insert(0, priority_test) - if self.skipped: - print(file=sys.stderr) - print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) - printlist(self.skipped, file=sys.stderr) + return (tuple(selected), tests) - def rerun_failed_tests(self): - self.ns.verbose = True - self.ns.failfast = False - self.ns.verbose3 = False + @staticmethod + def list_tests(tests: TestTuple) -> None: + for name in tests: + print(name) - self.first_result = self.get_tests_result() + def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: + # Configure the runner to re-run tests + if self.num_workers == 0 and not self.single_process: + # Always run tests in fresh processes to have more deterministic + # initial state. Don't re-run tests in parallel but limit to a + # single worker process to have side effects (on the system load + # and timings) between tests. + self.num_workers = 1 + + tests, match_tests_dict = self.results.prepare_rerun() + + # Re-run failed tests + runtests = runtests.copy( + tests=tests, + rerun=True, + verbose=True, + forever=False, + fail_fast=False, + match_tests_dict=match_tests_dict, + output_on_failure=False) + self.logger.set_tests(runtests) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) + return runtests + + def rerun_failed_tests(self, runtests: RunTests) -> None: + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + + if self.python_cmd: + # Temp patch for https://github.com/python/cpython/issues/94052 + self.log( + "Re-running failed tests is not supported with --python " + "host runner option." + ) + return + + self.first_state = self.get_state() print() - print("Re-running failed tests in verbose mode") - self.rerun = self.bad[:] - for test_name in self.rerun: - print(f"Re-running {test_name} in verbose mode", flush=True) - self.ns.verbose = True - result = runtest(self.ns, test_name) + rerun_runtests = self._rerun_failed_tests(runtests) - self.accumulate_result(result, rerun=True) + if self.results.bad: + print( + f"{red}{count(len(self.results.bad), 'test')} " + f"failed again:{reset}" + ) + printlist(self.results.bad) - if result.result == INTERRUPTED: - break + self.display_result(rerun_runtests) - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() - self.display_result() + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return - def display_result(self): + def display_result(self, runtests: RunTests) -> None: # If running the test suite for PGO then no one cares about results. - if self.ns.pgo: + if runtests.pgo: return + state = self.get_state() print() - print("== Tests result: %s ==" % self.get_tests_result()) - - if self.interrupted: - print("Test suite interrupted by signal SIGINT.") - - omitted = set(self.selected) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - - if self.good and not self.ns.quiet: - print() - if (not self.bad - and not self.skipped - and not self.interrupted - and len(self.good) > 1): - print("All", end=' ') - print(count(len(self.good), "test"), "OK.") - - if self.ns.print_slow: - self.test_times.sort(reverse=True) - print() - print("10 slowest tests:") - for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) - - if self.bad: - print() - print(count(len(self.bad), "test"), "failed:") - printlist(self.bad) - - if self.environment_changed: - print() - print("{} altered the execution environment:".format( - count(len(self.environment_changed), "test"))) - printlist(self.environment_changed) - - if self.skipped and not self.ns.quiet: - print() - print(count(len(self.skipped), "test"), "skipped:") - printlist(self.skipped) - - if self.rerun: - print() - print("%s:" % count(len(self.rerun), "re-run test")) - printlist(self.rerun) - - if self.run_no_tests: - print() - print(count(len(self.run_no_tests), "test"), "run no tests:") - printlist(self.run_no_tests) - - def run_tests_sequential(self): - if self.ns.trace: - import trace - self.tracer = trace.Trace(trace=False, count=True) + print(f"== Tests result: {state} ==") + + self.results.display_result(runtests.tests, + self.quiet, self.print_slowest) + + def run_test( + self, test_name: TestName, runtests: RunTests, tracer: trace.Trace | None + ) -> TestResult: + if tracer is not None: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + cmd = ('result = run_single_test(test_name, runtests)') + namespace = dict(locals()) + tracer.runctx(cmd, globals=globals(), locals=namespace) + result = namespace['result'] + result.covered_lines = list(tracer.counts) + else: + result = run_single_test(test_name, runtests) + + self.results.accumulate_result(result, runtests) + + return result + + def run_tests_sequentially(self, runtests: RunTests) -> None: + if self.coverage: + tracer = trace.Trace(trace=False, count=True) + else: + tracer = None save_modules = set(sys.modules) - print("Run tests sequentially") - - previous_test = None - for test_index, test_name in enumerate(self.tests, 1): - start_time = time.monotonic() - - text = test_name - if previous_test: - text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) - - if self.tracer: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name); ' - 'self.accumulate_result(result)') - ns = dict(locals()) - self.tracer.runctx(cmd, globals=globals(), locals=ns) - result = ns['result'] - else: - result = runtest(self.ns, test_name) - self.accumulate_result(result) + jobs = runtests.get_jobs() + if jobs is not None: + tests = count(jobs, 'test') + else: + tests = 'tests' + msg = f"Run {tests} sequentially in a single process" + if runtests.timeout: + msg += " (timeout: %s)" % format_duration(runtests.timeout) + self.log(msg) - if result.result == INTERRUPTED: - break + tests_iter = runtests.iter_tests() + for test_index, test_name in enumerate(tests_iter, 1): + start_time = time.perf_counter() - previous_test = format_test_result(result) - test_time = time.monotonic() - start_time - if test_time >= PROGRESS_MIN_TIME: - previous_test = "%s in %s" % (previous_test, format_duration(test_time)) - elif result[0] == PASSED: - # be quiet: say nothing if the test passed shortly - previous_test = None + self.logger.display_progress(test_index, test_name) + + result = self.run_test(test_name, runtests, tracer) # Unload the newly imported test modules (best effort finalization) new_modules = [module for module in sys.modules @@ -422,95 +435,26 @@ def run_tests_sequential(self): except (KeyError, AttributeError): pass - if previous_test: - print(previous_test) - - def _test_forever(self, tests): - while True: - for test_name in tests: - yield test_name - if self.bad: - return - if self.ns.fail_env_changed and self.environment_changed: - return - - def display_header(self): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - try: - print("==", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - except: - print("== RustPython: Need to fix platform.platform") - print("== cwd:", os.getcwd()) - cpu_count = os.cpu_count() - if cpu_count: - print("== CPU count:", cpu_count) - try: - print("== encodings: locale=%s, FS=%s" - % (locale.getpreferredencoding(False), - sys.getfilesystemencoding())) - except: - print("== RustPython: Need to fix encoding stuff") - - def get_tests_result(self): - result = [] - if self.bad: - result.append("FAILURE") - elif self.ns.fail_env_changed and self.environment_changed: - result.append("ENV CHANGED") - elif not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)): - result.append("NO TEST RUN") - - if self.interrupted: - result.append("INTERRUPTED") - - if not result: - result.append("SUCCESS") - - result = ', '.join(result) - if self.first_result: - result = '%s then %s' % (self.first_result, result) - return result + text = str(result) + test_time = time.perf_counter() - start_time + if test_time >= PROGRESS_MIN_TIME: + text = f"{text} in {format_duration(test_time)}" + self.logger.display_progress(test_index, text) - def run_tests(self): - # For a partial run, we do not need to clutter the output. - if (self.ns.header - or not(self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - self.display_header() - - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) - - if self.ns.forever: - self.tests = self._test_forever(list(self.selected)) - self.test_count = '' - self.test_count_width = 3 - else: - self.tests = iter(self.selected) - self.test_count = '/{}'.format(len(self.selected)) - self.test_count_width = len(self.test_count) - 1 + if result.must_stop(self.fail_fast, self.fail_env_changed): + break - if self.ns.use_mp: - from test.libregrtest.runtest_mp import run_tests_multiprocess - run_tests_multiprocess(self) - else: - self.run_tests_sequential() + def get_state(self) -> str: + state = self.results.get_state(self.fail_env_changed) + if self.first_state: + state = f'{self.first_state} then {state}' + return state - def finalize(self): - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None + def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: + from .run_workers import RunWorkers + RunWorkers(num_workers, runtests, self.logger, self.results).run() + def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -518,141 +462,326 @@ def finalize(self): else: os.unlink(self.next_single_filename) - if self.tracer: - r = self.tracer.results() - r.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + if coverage is not None: + # uses a new-in-Python 3.13 keyword argument that mypy doesn't know about yet: + coverage.write_results(show_missing=True, summary=True, # type: ignore[call-arg] + coverdir=self.coverage_dir, + ignore_missing_files=True) + + if self.want_run_leaks: + os.system("leaks %d" % os.getpid()) + + if self.junit_filename: + self.results.write_junit(self.junit_filename) + def display_summary(self) -> None: + if self.first_runtests is None: + raise ValueError( + "Should never call `display_summary()` before calling `_run_test()`" + ) + + duration = time.perf_counter() - self.logger.start_time + filtered = bool(self.match_tests) + + # Total duration print() - duration = time.monotonic() - self.start_time print("Total duration: %s" % format_duration(duration)) - print("Tests result: %s" % self.get_tests_result()) - if self.ns.runleaks: - os.system("leaks %d" % os.getpid()) + self.results.display_summary(self.first_runtests, filtered) + + # Result + state = self.get_state() + print(f"Result: {state}") + + def create_run_tests(self, tests: TestTuple) -> RunTests: + return RunTests( + tests, + fail_fast=self.fail_fast, + fail_env_changed=self.fail_env_changed, + match_tests=self.match_tests, + match_tests_dict=None, + rerun=False, + forever=self.forever, + pgo=self.pgo, + pgo_extended=self.pgo_extended, + output_on_failure=self.output_on_failure, + timeout=self.timeout, + verbose=self.verbose, + quiet=self.quiet, + hunt_refleak=self.hunt_refleak, + test_dir=self.test_dir, + use_junit=(self.junit_filename is not None), + coverage=self.coverage, + memory_limit=self.memory_limit, + gc_threshold=self.gc_threshold, + use_resources=self.use_resources, + python_cmd=self.python_cmd, + randomize=self.randomize, + random_seed=self.random_seed, + parallel_threads=self.parallel_threads, + ) + + def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + if self.num_workers < 0: + # Use all CPUs + 2 extra worker processes for tests + # that like to sleep + # + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet + self.num_workers = (os.process_cpu_count() or 1) + 2 # type: ignore[attr-defined] - def save_xml_result(self): - if not self.ns.xmlpath and not self.testsuite_xml: - return + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.pgo or self.quiet or self.single_test_run + or tests or self.cmdline_args)): + display_header(self.use_resources, self.python_cmd) - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") + print("Using random seed:", self.random_seed) - # Manually count the totals for the overall summary - totals = {'tests': 0, 'errors': 0, 'failures': 0} - for suite in self.testsuite_xml: - root.append(suite) - for k in totals: - try: - totals[k] += int(suite.get(k, 0)) - except ValueError: - pass + runtests = self.create_run_tests(selected) + self.first_runtests = runtests + self.logger.set_tests(runtests) - for k, v in totals.items(): - root.set(k, str(v)) - - xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath) - with open(xmlpath, 'wb') as f: - for s in ET.tostringlist(root): - f.write(s) - - def main(self, tests=None, **kwargs): - global TEMPDIR - self.ns = self.parse_args(kwargs) - - if self.ns.tempdir: - TEMPDIR = self.ns.tempdir - elif self.ns.worker_args: - ns_dict, _ = json.loads(self.ns.worker_args) - TEMPDIR = ns_dict.get("tempdir") or TEMPDIR - - os.makedirs(TEMPDIR, exist_ok=True) - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from os_helper.SAVEDCWD. - with os_helper.temp_cwd(test_cwd, quiet=True): - self._main(tests, kwargs) - - def getloadavg(self): - if self.win_load_tracker is not None: - return self.win_load_tracker.getloadavg() - - if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] - - return None - - def _main(self, tests, kwargs): - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 1 or repetitions < 1: - msg = ("Invalid values for the --huntrleaks/-R parameters. The " - "number of warmups and repetitions must be at least 1 " - "each (1:1).") - print(msg, file=sys.stderr, flush=True) - sys.exit(2) - - if self.ns.worker_args is not None: - from test.libregrtest.runtest_mp import run_tests_worker - run_tests_worker(self.ns.worker_args) - - if self.ns.wait: - input("Press any key to continue...") + if (runtests.hunt_refleak is not None) and (not self.num_workers): + # gh-109739: WindowsLoadTracker thread interferes with refleak check + use_load_tracker = False + else: + # WindowsLoadTracker is only needed on Windows + use_load_tracker = MS_WINDOWS + + if use_load_tracker: + self.logger.start_load_tracker() + try: + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) + + coverage = self.results.get_coverage_results() + self.display_result(runtests) + + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) + finally: + if use_load_tracker: + self.logger.stop_load_tracker() + + self.display_summary() + self.finalize_tests(coverage) + + return self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + + def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: + os.makedirs(self.tmp_dir, exist_ok=True) + work_dir = get_work_dir(self.tmp_dir) + + # Put a timeout on Python exit + with exit_timeout(): + # Run the tests in a context manager that temporarily changes the + # CWD to a temporary and writable directory. If it's not possible + # to create or change the CWD, the original CWD will be used. + # The original CWD is available from os_helper.SAVEDCWD. + with os_helper.temp_cwd(work_dir, quiet=True): + # When using multiprocessing, worker processes will use + # work_dir as their parent temporary directory. So when the + # main process exit, it removes also subdirectories of worker + # processes. + return self._run_tests(selected, tests) + + def _add_cross_compile_opts(self, regrtest_opts): + # WASM/WASI buildbot builders pass multiple PYTHON environment + # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. + keep_environ = bool(self.python_cmd) + environ = None + + # Are we using cross-compilation? + cross_compile = is_cross_compiled() + + # Get HOSTRUNNER + hostrunner = get_host_runner() + + if cross_compile: + # emulate -E, but keep PYTHONPATH + cross compile env vars, + # so test executable can load correct sysconfigdata file. + keep = { + '_PYTHON_PROJECT_BASE', + '_PYTHON_HOST_PLATFORM', + '_PYTHON_SYSCONFIGDATA_NAME', + "_PYTHON_SYSCONFIGDATA_PATH", + 'PYTHONPATH' + } + old_environ = os.environ + new_environ = { + name: value for name, value in os.environ.items() + if not name.startswith(('PYTHON', '_PYTHON')) or name in keep + } + # Only set environ if at least one variable was removed + if new_environ != old_environ: + environ = new_environ + keep_environ = True + + if cross_compile and hostrunner: + if self.num_workers == 0 and not self.single_process: + # For now use only two cores for cross-compiled builds; + # hostrunner can be expensive. + regrtest_opts.extend(['-j', '2']) + + # If HOSTRUNNER is set and -p/--python option is not given, then + # use hostrunner to execute python binary for tests. + if not self.python_cmd: + buildpython = sysconfig.get_config_var("BUILDPYTHON") + python_cmd = f"{hostrunner} {buildpython}" + regrtest_opts.extend(["--python", python_cmd]) + keep_environ = True + + return (environ, keep_environ) + + def _add_ci_python_opts(self, python_opts, keep_environ): + # --fast-ci and --slow-ci add options to Python. + # + # Some platforms run tests in embedded mode and cannot change options + # after startup, so if this function changes, consider also updating: + # * gradle_task in Android/android.py + + # Unbuffered stdout and stderr. This isn't helpful on Android, because + # it would cause lines to be split into multiple log messages. + if not sys.stdout.write_through and sys.platform != "android": + python_opts.append('-u') + + # Add warnings filter 'error', unless the user specified a different + # filter. Ignore BytesWarning since it's controlled by '-b' below. + if not [ + opt for opt in sys.warnoptions + if not opt.endswith("::BytesWarning") + ]: + python_opts.extend(('-W', 'error')) + + # Error on bytes/str comparison + if sys.flags.bytes_warning < 2: + python_opts.append('-bb') + + if not keep_environ: + # Ignore PYTHON* environment variables + if not sys.flags.ignore_environment: + python_opts.append('-E') + + def _execute_python(self, cmd, environ): + # Make sure that messages before execv() are logged + sys.stdout.flush() + sys.stderr.flush() + + cmd_text = shlex.join(cmd) + try: + # Android and iOS run tests in embedded mode. To update their + # Python options, see the comment in _add_ci_python_opts. + if not cmd[0]: + raise ValueError("No Python executable is present") + + print(f"+ {cmd_text}", flush=True) + if hasattr(os, 'execv') and not MS_WINDOWS: + os.execv(cmd[0], cmd) + # On success, execv() do no return. + # On error, it raises an OSError. + else: + import subprocess + with subprocess.Popen(cmd, env=environ) as proc: + try: + proc.wait() + except KeyboardInterrupt: + # There is no need to call proc.terminate(): on CTRL+C, + # SIGTERM is also sent to the child process. + try: + proc.wait(timeout=EXIT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + sys.exit(EXITCODE_INTERRUPTED) + + sys.exit(proc.returncode) + except Exception as exc: + print_warning(f"Failed to change Python options: {exc!r}\n" + f"Command: {cmd_text}") + # continue executing main() + + def _add_python_opts(self) -> None: + python_opts: list[str] = [] + regrtest_opts: list[str] = [] + + environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) + if self.ci_mode: + self._add_ci_python_opts(python_opts, keep_environ) + + if (not python_opts) and (not regrtest_opts) and (environ is None): + # Nothing changed: nothing to do + return - support.PGO = self.ns.pgo + # Create new command line + cmd = list(sys.orig_argv) + if python_opts: + cmd[1:1] = python_opts + if regrtest_opts: + cmd.extend(regrtest_opts) + cmd.append("--dont-add-python-opts") - setup_tests(self.ns) + self._execute_python(cmd, environ) - self.find_tests(tests) + def _init(self): + setup_process() - if self.ns.list_tests: - self.list_tests() - sys.exit(0) + if self.junit_filename and not os.path.isabs(self.junit_filename): + self.junit_filename = os.path.abspath(self.junit_filename) - if self.ns.list_cases: - self.list_cases() - sys.exit(0) + strip_py_suffix(self.cmdline_args) - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - # TODO: RUSTPYTHON - # if sys.platform == 'win32' and (self.ns.worker_args is None): - # from test.libregrtest.win_utils import WindowsLoadTracker + self._tmp_dir = get_temp_dir(self._tmp_dir) - # try: - # self.win_load_tracker = WindowsLoadTracker() - # except FileNotFoundError as error: - # # Windows IoT Core and Windows Nano Server do not provide - # # typeperf.exe for x64, x86 or ARM - # print(f'Failed to create WindowsLoadTracker: {error}') + @property + def tmp_dir(self) -> StrPath: + if self._tmp_dir is None: + raise ValueError( + "Should never use `.tmp_dir` before calling `.main()`" + ) + return self._tmp_dir - self.run_tests() - self.display_result() + def main(self, tests: TestList | None = None) -> NoReturn: + if self.want_add_python_opts: + self._add_python_opts() - if self.ns.verbose2 and self.bad: - self.rerun_failed_tests() + self._init() - self.finalize() + if self.want_cleanup: + cleanup_temp_dir(self.tmp_dir) + sys.exit(0) + + if self.want_wait: + input("Press any key to continue...") - self.save_xml_result() + setup_test_dir(self.test_dir) + selected, tests = self.find_tests(tests) + + exitcode = 0 + if self.want_list_tests: + self.list_tests(selected) + elif self.want_list_cases: + list_cases(selected, + match_tests=self.match_tests, + test_dir=self.test_dir) + else: + exitcode = self.run_tests(selected, tests) - if self.bad: - sys.exit(2) - if self.interrupted: - sys.exit(130) - if self.ns.fail_env_changed and self.environment_changed: - sys.exit(3) - sys.exit(0) + sys.exit(exitcode) -def main(tests=None, **kwargs): +def main(tests=None, _add_python_opts=False, **kwargs) -> NoReturn: """Run the Python suite.""" - Regrtest().main(tests=tests, **kwargs) + ns = _parse_args(sys.argv[1:], **kwargs) + Regrtest(ns, _add_python_opts=_add_python_opts).main(tests=tests) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini new file mode 100644 index 00000000000..3fa9afcb7a4 --- /dev/null +++ b/Lib/test/libregrtest/mypy.ini @@ -0,0 +1,26 @@ +# Config file for running mypy on libregrtest. +# Run mypy by invoking `mypy --config-file Lib/test/libregrtest/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/test/libregrtest +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_incomplete_defs = False +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False +warn_return_any = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/test/libregrtest/parallel_case.py b/Lib/test/libregrtest/parallel_case.py new file mode 100644 index 00000000000..8eb3c314916 --- /dev/null +++ b/Lib/test/libregrtest/parallel_case.py @@ -0,0 +1,78 @@ +"""Run a test case multiple times in parallel threads.""" + +import copy +import threading +import unittest + +from unittest import TestCase + + +class ParallelTestCase(TestCase): + def __init__(self, test_case: TestCase, num_threads: int): + self.test_case = test_case + self.num_threads = num_threads + self._testMethodName = test_case._testMethodName + self._testMethodDoc = test_case._testMethodDoc + + def __str__(self): + return f"{str(self.test_case)} [threads={self.num_threads}]" + + def run_worker(self, test_case: TestCase, result: unittest.TestResult, + barrier: threading.Barrier): + barrier.wait() + test_case.run(result) + + def run(self, result=None): + if result is None: + result = test_case.defaultTestResult() + startTestRun = getattr(result, 'startTestRun', None) + stopTestRun = getattr(result, 'stopTestRun', None) + if startTestRun is not None: + startTestRun() + else: + stopTestRun = None + + # Called at the beginning of each test. See TestCase.run. + result.startTest(self) + + cases = [copy.copy(self.test_case) for _ in range(self.num_threads)] + results = [unittest.TestResult() for _ in range(self.num_threads)] + + barrier = threading.Barrier(self.num_threads) + threads = [] + for i, (case, r) in enumerate(zip(cases, results)): + thread = threading.Thread(target=self.run_worker, + args=(case, r, barrier), + name=f"{str(self.test_case)}-{i}", + daemon=True) + threads.append(thread) + + for thread in threads: + thread.start() + + for threads in threads: + threads.join() + + # Aggregate test results + if all(r.wasSuccessful() for r in results): + result.addSuccess(self) + + # Note: We can't call result.addError, result.addFailure, etc. because + # we no longer have the original exception, just the string format. + for r in results: + if len(r.errors) > 0 or len(r.failures) > 0: + result._mirrorOutput = True + result.errors.extend(r.errors) + result.failures.extend(r.failures) + result.skipped.extend(r.skipped) + result.expectedFailures.extend(r.expectedFailures) + result.unexpectedSuccesses.extend(r.unexpectedSuccesses) + result.collectedDurations.extend(r.collectedDurations) + + if any(r.shouldStop for r in results): + result.stop() + + # Test has finished running + result.stopTest(self) + if stopTestRun is not None: + stopTestRun() diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py new file mode 100644 index 00000000000..04803ddf644 --- /dev/null +++ b/Lib/test/libregrtest/pgo.py @@ -0,0 +1,55 @@ +# Set of tests run by default if --pgo is specified. The tests below were +# chosen based on the following criteria: either they exercise a commonly used +# C extension module or type, or they run some relatively typical Python code. +# Long running tests should be avoided because the PGO instrumented executable +# runs slowly. +PGO_TESTS = [ + 'test_array', + 'test_base64', + 'test_binascii', + 'test_binop', + 'test_bisect', + 'test_bytes', + 'test_bz2', + 'test_cmath', + 'test_codecs', + 'test_collections', + 'test_complex', + 'test_dataclasses', + 'test_datetime', + 'test_decimal', + 'test_difflib', + 'test_float', + 'test_fstring', + 'test_functools', + 'test_generators', + 'test_hashlib', + 'test_heapq', + 'test_int', + 'test_itertools', + 'test_json', + 'test_long', + 'test_lzma', + 'test_math', + 'test_memoryview', + 'test_operator', + 'test_ordered_dict', + 'test_patma', + 'test_pickle', + 'test_pprint', + 'test_re', + 'test_set', + 'test_sqlite3', + 'test_statistics', + 'test_str', + 'test_struct', + 'test_tabnanny', + 'test_time', + 'test_xml_etree', + 'test_xml_etree_c', +] + +def setup_pgo_tests(cmdline_args, pgo_extended: bool) -> None: + if not cmdline_args and not pgo_extended: + # run default set of tests for PGO training + cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 03747f7f757..5c78515506d 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,9 +1,17 @@ import os -import re import sys import warnings from inspect import isabstract +from typing import Any +import linecache + from test import support +from test.support import os_helper +from test.support import refleak_helper + +from .runtests import HuntRefleak +from .utils import clear_caches + try: from _abc import _get_dump except ImportError: @@ -17,7 +25,33 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) -def dash_R(ns, test_name, test_func): +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + +def runtest_refleak(test_name, test_func, + hunt_refleak: HuntRefleak, + quiet: bool): """Run a test multiple times, looking for reference leaks. Returns: @@ -39,12 +73,19 @@ def dash_R(ns, test_name, test_func): fs = warnings.filters[:] ps = copyreg.dispatch_table.copy() pic = sys.path_importer_cache.copy() + zdc: dict[str, Any] | None + # Linecache holds a cache with the source of interactive code snippets + # (e.g. code typed in the REPL). This cache is not cleared by + # linecache.clearcache(). We need to save and restore it to avoid false + # positives. + linecache_data = linecache.cache.copy(), linecache._interactive_cache.copy() # type: ignore[attr-defined] try: import zipimport except ImportError: zdc = None # Run unmodified on platforms without zipimport support else: - zdc = zipimport._zip_directory_cache.copy() + # private attribute that mypy doesn't know about: + zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined] abcs = {} for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): @@ -60,9 +101,10 @@ def dash_R(ns, test_name, test_func): def get_pooled_int(value): return int_pool.setdefault(value, value) - nwarmup, ntracked, fname = ns.huntrleaks - fname = os.path.join(os_helper.SAVEDCWD, fname) - repcount = nwarmup + ntracked + warmups = hunt_refleak.warmups + runs = hunt_refleak.runs + filename = hunt_refleak.filename + repcount = warmups + runs # Pre-allocate to ensure that the loop doesn't allocate anything new rep_range = list(range(repcount)) @@ -71,45 +113,81 @@ def get_pooled_int(value): fd_deltas = [0] * repcount getallocatedblocks = sys.getallocatedblocks gettotalrefcount = sys.gettotalrefcount - fd_count = support.fd_count - + getunicodeinternedsize = sys.getunicodeinternedsize + fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = 0 + rc_before = alloc_before = fd_before = interned_immortal_before = 0 - if not ns.quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) + if not quiet: + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) - dash_R_cleanup(fs, ps, pic, zdc, abcs) + xml_filename = 'refleak-xml.tmp' + result = None + dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data) for i in rep_range: - test_func() - dash_R_cleanup(fs, ps, pic, zdc, abcs) - - # dash_R_cleanup() ends with collecting cyclic trash: - # read memory statistics immediately after. - alloc_after = getallocatedblocks() + support.gc_collect() + current = refleak_helper._hunting_for_refleaks + refleak_helper._hunting_for_refleaks = True + try: + result = test_func() + finally: + refleak_helper._hunting_for_refleaks = current + + save_support_xml(xml_filename) + dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data) + support.gc_collect() + + # Read memory statistics immediately after the garbage collection. + # Also, readjust the reference counts and alloc blocks by ignoring + # any strings that might have been interned during test_func. These + # strings will be deallocated at runtime shutdown + interned_immortal_after = getunicodeinternedsize( + # Use an internal-only keyword argument that mypy doesn't know yet + _only_immortal=True) # type: ignore[call-arg] + alloc_after = getallocatedblocks() - interned_immortal_after rc_after = gettotalrefcount() fd_after = fd_count() - if not ns.quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after + interned_immortal_before = interned_immortal_after + + restore_support_xml(xml_filename) - if not ns.quiet: + if not quiet: print(file=sys.stderr) # These checkers return False on success, True on failure def check_rc_deltas(deltas): - # Checker for reference counters and memomry blocks. + # Checker for reference counters and memory blocks. # # bpo-30776: Try to ignore false positives: # @@ -133,19 +211,25 @@ def check_fd_deltas(deltas): (fd_deltas, 'file descriptors', check_fd_deltas) ]: # ignore warmup runs - deltas = deltas[nwarmup:] - if checker(deltas): + deltas = deltas[warmups:] + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) + return (failed, result) + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data): import copyreg import collections.abc @@ -155,6 +239,11 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): copyreg.dispatch_table.update(ps) sys.path_importer_cache.clear() sys.path_importer_cache.update(pic) + lcache, linteractive = linecache_data + linecache._interactive_cache.clear() + linecache._interactive_cache.update(linteractive) + linecache.cache.clear() + linecache.cache.update(lcache) try: import zipimport except ImportError: @@ -163,121 +252,28 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.clear() zipimport._zip_directory_cache.update(zdc) - # clear type cache - sys._clear_type_cache() - # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - for ref in abcs.get(obj, set()): - if ref() is not None: - obj.register(ref()) + refs = abcs.get(obj, None) + if refs is not None: + obj._abc_registry_clear() + for ref in refs: + subclass = ref() + if subclass is not None: + obj.register(subclass) obj._abc_caches_clear() + # Clear caches clear_caches() - -def clear_caches(): - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - # Don't worry about resetting the cache if the module is not loaded - try: - distutils_dir_util = sys.modules['distutils.dir_util'] - except KeyError: - pass - else: - distutils_dir_util._path_created.clear() - re.purge() - - try: - _strptime = sys.modules['_strptime'] - except KeyError: - pass - else: - _strptime._regex_cache.clear() - - try: - urllib_parse = sys.modules['urllib.parse'] - except KeyError: - pass - else: - urllib_parse.clear_cache() - - try: - urllib_request = sys.modules['urllib.request'] - except KeyError: - pass - else: - urllib_request.urlcleanup() - - try: - linecache = sys.modules['linecache'] - except KeyError: - pass - else: - linecache.clearcache() - - try: - mimetypes = sys.modules['mimetypes'] - except KeyError: - pass - else: - mimetypes._default_mime_types() - - try: - filecmp = sys.modules['filecmp'] - except KeyError: - pass - else: - filecmp._cache.clear() - - try: - struct = sys.modules['struct'] - except KeyError: - pass - else: - # TODO: fix - # struct._clearcache() - pass - - try: - doctest = sys.modules['doctest'] - except KeyError: - pass - else: - doctest.master = None - - try: - ctypes = sys.modules['ctypes'] - except KeyError: - pass - else: - ctypes._reset_cache() - - try: - typing = sys.modules['typing'] - except KeyError: - pass - else: - for f in typing._cleanups: - f() - - support.gc_collect() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() -def warm_caches(): +def warm_caches() -> None: # char cache s = bytes(range(256)) for i in range(256): diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py new file mode 100644 index 00000000000..daf7624366e --- /dev/null +++ b/Lib/test/libregrtest/result.py @@ -0,0 +1,243 @@ +import dataclasses +import json +from _colorize import get_colors # type: ignore[import-not-found] +from typing import Any + +from .utils import ( + StrJSON, TestName, FilterTuple, + format_duration, normalize_test_name, print_warning) + + +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed, + results.skipped) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + +# Avoid enum.Enum to reduce the number of imports when tests are run +class State: + PASSED = "PASSED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + UNCAUGHT_EXC = "UNCAUGHT_EXC" + REFLEAK = "REFLEAK" + ENV_CHANGED = "ENV_CHANGED" + RESOURCE_DENIED = "RESOURCE_DENIED" + INTERRUPTED = "INTERRUPTED" + WORKER_FAILED = "WORKER_FAILED" # non-zero worker process exit code + WORKER_BUG = "WORKER_BUG" # exception when running a worker + DID_NOT_RUN = "DID_NOT_RUN" + TIMEOUT = "TIMEOUT" + + @staticmethod + def is_failed(state): + return state in { + State.FAILED, + State.UNCAUGHT_EXC, + State.REFLEAK, + State.WORKER_FAILED, + State.WORKER_BUG, + State.TIMEOUT} + + @staticmethod + def has_meaningful_duration(state): + # Consider that the duration is meaningless for these cases. + # For example, if a whole test file is skipped, its duration + # is unlikely to be the duration of executing its tests, + # but just the duration to execute code which skips the test. + return state not in { + State.SKIPPED, + State.RESOURCE_DENIED, + State.INTERRUPTED, + State.WORKER_FAILED, + State.WORKER_BUG, + State.DID_NOT_RUN} + + @staticmethod + def must_stop(state): + return state in { + State.INTERRUPTED, + State.WORKER_BUG, + } + + +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] + + +@dataclasses.dataclass(slots=True) +class TestResult: + test_name: TestName + state: str | None = None + # Test duration in seconds + duration: float | None = None + xml_data: list[str] | None = None + stats: TestStats | None = None + + # errors and failures copied from support.TestFailedWithDetails + errors: list[tuple[str, str]] | None = None + failures: list[tuple[str, str]] | None = None + + # partial coverage in a worker run; not used by sequential in-process runs + covered_lines: list[Location] | None = None + + def is_failed(self, fail_env_changed: bool) -> bool: + if self.state == State.ENV_CHANGED: + return fail_env_changed + return State.is_failed(self.state) + + def _format_failed(self): + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + if self.errors and self.failures: + le = len(self.errors) + lf = len(self.failures) + error_s = "error" + ("s" if le > 1 else "") + failure_s = "failure" + ("s" if lf > 1 else "") + return ( + f"{red}{self.test_name} failed " + f"({le} {error_s}, {lf} {failure_s}){reset}" + ) + + if self.errors: + le = len(self.errors) + error_s = "error" + ("s" if le > 1 else "") + return f"{red}{self.test_name} failed ({le} {error_s}){reset}" + + if self.failures: + lf = len(self.failures) + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{red}{self.test_name} failed ({lf} {failure_s}){reset}" + + return f"{red}{self.test_name} failed{reset}" + + def __str__(self) -> str: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + + match self.state: + case State.PASSED: + return f"{green}{self.test_name} passed{reset}" + case State.FAILED: + return f"{red}{self._format_failed()}{reset}" + case State.SKIPPED: + return f"{yellow}{self.test_name} skipped{reset}" + case State.UNCAUGHT_EXC: + return ( + f"{red}{self.test_name} failed (uncaught exception){reset}" + ) + case State.REFLEAK: + return f"{red}{self.test_name} failed (reference leak){reset}" + case State.ENV_CHANGED: + return f"{red}{self.test_name} failed (env changed){reset}" + case State.RESOURCE_DENIED: + return f"{yellow}{self.test_name} skipped (resource denied){reset}" + case State.INTERRUPTED: + return f"{yellow}{self.test_name} interrupted{reset}" + case State.WORKER_FAILED: + return ( + f"{red}{self.test_name} worker non-zero exit code{reset}" + ) + case State.WORKER_BUG: + return f"{red}{self.test_name} worker bug{reset}" + case State.DID_NOT_RUN: + return f"{yellow}{self.test_name} ran no tests{reset}" + case State.TIMEOUT: + assert self.duration is not None, "self.duration is None" + return f"{self.test_name} timed out ({format_duration(self.duration)})" + case _: + raise ValueError( + f"{red}unknown result state: {{state!r}}{reset}" + ) + + def has_meaningful_duration(self): + return State.has_meaningful_duration(self.state) + + def set_env_changed(self): + if self.state is None or self.state == State.PASSED: + self.state = State.ENV_CHANGED + + def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: + if State.must_stop(self.state): + return True + if fail_fast and self.is_failed(fail_env_changed): + return True + return False + + def get_rerun_match_tests(self) -> FilterTuple | None: + match_tests = [] + + errors = self.errors or [] + failures = self.failures or [] + for error_list, is_error in ( + (errors, True), + (failures, False), + ): + for full_name, *_ in error_list: + match_name = normalize_test_name(full_name, is_error=is_error) + if match_name is None: + # 'setUpModule (test.test_sys)': don't filter tests + return None + if not match_name: + error_type = "ERROR" if is_error else "FAIL" + print_warning(f"rerun failed to parse {error_type} test name: " + f"{full_name!r}: don't filter tests") + return None + match_tests.append(match_name) + + if not match_tests: + return None + return tuple(match_tests) + + def write_json_into(self, file) -> None: + json.dump(self, file, cls=_EncodeTestResult) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'TestResult': + return json.loads(worker_json, object_hook=_decode_test_result) + + +class _EncodeTestResult(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, TestResult): + result = dataclasses.asdict(o) + result["__test_result__"] = o.__class__.__name__ + return result + else: + return super().default(o) + + +def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: + if "__test_result__" in data: + data.pop('__test_result__') + if data['stats'] is not None: + data['stats'] = TestStats(**data['stats']) + if data['covered_lines'] is not None: + data['covered_lines'] = [ + tuple(loc) for loc in data['covered_lines'] + ] + return TestResult(**data) + else: + return data diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py new file mode 100644 index 00000000000..a35934fc2c9 --- /dev/null +++ b/Lib/test/libregrtest/results.py @@ -0,0 +1,309 @@ +import sys +import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import TYPE_CHECKING + +from .runtests import RunTests +from .result import State, TestResult, TestStats, Location +from .utils import ( + StrPath, TestName, TestTuple, TestList, FilterDict, + printlist, count, format_duration) + +if TYPE_CHECKING: + from xml.etree.ElementTree import Element + + +# Python uses exit code 1 when an exception is not caught +# argparse.ArgumentParser.error() uses exit code 2 +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 + + +class TestResults: + def __init__(self) -> None: + self.bad: TestList = [] + self.good: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.env_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] + self.rerun_results: list[TestResult] = [] + + self.interrupted: bool = False + self.worker_bug: bool = False + self.test_times: list[tuple[float, TestName]] = [] + self.stats = TestStats() + # used by --junit-xml + self.testsuite_xml: list['Element'] = [] + # used by -T with -j + self.covered_lines: set[Location] = set() + + def is_all_good(self) -> bool: + return (not self.bad + and not self.skipped + and not self.interrupted + and not self.worker_bug) + + def get_executed(self) -> set[TestName]: + return (set(self.good) | set(self.bad) | set(self.skipped) + | set(self.resource_denied) | set(self.env_changed) + | set(self.run_no_tests)) + + def no_tests_run(self) -> bool: + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.env_changed)) + + def get_state(self, fail_env_changed: bool) -> str: + state = [] + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + if self.bad: + state.append(f"{red}FAILURE{reset}") + elif fail_env_changed and self.env_changed: + state.append(f"{yellow}ENV CHANGED{reset}") + elif self.no_tests_run(): + state.append(f"{yellow}NO TESTS RAN{reset}") + + if self.interrupted: + state.append(f"{yellow}INTERRUPTED{reset}") + if self.worker_bug: + state.append(f"{red}WORKER BUG{reset}") + if not state: + state.append(f"{green}SUCCESS{reset}") + + return ', '.join(state) + + def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: + exitcode = 0 + if self.bad: + exitcode = EXITCODE_BAD_TEST + elif self.interrupted: + exitcode = EXITCODE_INTERRUPTED + elif fail_env_changed and self.env_changed: + exitcode = EXITCODE_ENV_CHANGED + elif self.no_tests_run(): + exitcode = EXITCODE_NO_TESTS_RAN + elif fail_rerun and self.rerun: + exitcode = EXITCODE_RERUN_FAIL + elif self.worker_bug: + exitcode = EXITCODE_BAD_TEST + return exitcode + + def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: + test_name = result.test_name + rerun = runtests.rerun + fail_env_changed = runtests.fail_env_changed + + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.env_changed.append(test_name) + self.rerun_results.append(result) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(fail_env_changed): + self.bad.append(test_name) + self.rerun_results.append(result) + else: + raise ValueError(f"invalid test state: {result.state!r}") + + if result.state == State.WORKER_BUG: + self.worker_bug = True + + if result.has_meaningful_duration() and not rerun: + if result.duration is None: + raise ValueError("result.duration is None") + self.test_times.append((result.duration, test_name)) + if result.stats is not None: + self.stats.accumulate(result.stats) + if rerun: + self.rerun.append(test_name) + if result.covered_lines: + # we don't care about trace counts so we don't have to sum them up + self.covered_lines.update(result.covered_lines) + xml_data = result.xml_data + if xml_data: + self.add_junit(xml_data) + + def get_coverage_results(self) -> trace.CoverageResults: + counts = {loc: 1 for loc in self.covered_lines} + return trace.CoverageResults(counts=counts) + + def need_rerun(self) -> bool: + return bool(self.rerun_results) + + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: + tests: TestList = [] + match_tests_dict = {} + for result in self.rerun_results: + tests.append(result.test_name) + + match_tests = result.get_rerun_match_tests() + # ignore empty match list + if match_tests: + match_tests_dict[result.test_name] = match_tests + + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() + + return (tuple(tests), match_tests_dict) + + def add_junit(self, xml_data: list[str]) -> None: + import xml.etree.ElementTree as ET + for e in xml_data: + try: + self.testsuite_xml.append(ET.fromstring(e)) + except ET.ParseError: + print(xml_data, file=sys.__stderr__) + raise + + def write_junit(self, filename: StrPath) -> None: + if not self.testsuite_xml: + # Don't create empty XML file + return + + import xml.etree.ElementTree as ET + root = ET.Element("testsuites") + + # Manually count the totals for the overall summary + totals = {'tests': 0, 'errors': 0, 'failures': 0} + for suite in self.testsuite_xml: + root.append(suite) + for k in totals: + try: + totals[k] += int(suite.get(k, 0)) + except ValueError: + pass + + for k, v in totals.items(): + root.set(k, str(v)) + + with open(filename, 'wb') as f: + for s in ET.tostringlist(root): + f.write(s) + + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + + if print_slowest: + self.test_times.sort(reverse=True) + print() + print(f"{yellow}10 slowest tests:{reset}") + for test_time, test in self.test_times[:10]: + print(f"- {test}: {format_duration(test_time)}") + + all_tests = [] + omitted = set(tests) - self.get_executed() + + # less important + all_tests.append( + (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") + ) + if not quiet: + all_tests.append( + (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + ) + all_tests.append( + ( + self.resource_denied, + "test", + f"{yellow}{{}} skipped (resource denied):{reset}", + ) + ) + all_tests.append( + (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") + ) + + # more important + all_tests.append( + ( + self.env_changed, + "test", + f"{yellow}{{}} altered the execution environment (env changed):{reset}", + ) + ) + all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) + all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) + + for tests_list, count_text, title_format in all_tests: + if tests_list: + print() + count_text = count(len(tests_list), count_text) + print(title_format.format(count_text)) + printlist(tests_list) + + if self.good and not quiet: + print() + text = count(len(self.good), "test") + text = f"{green}{text} OK.{reset}" + if self.is_all_good() and len(self.good) > 1: + text = f"All {text}" + print(text) + + if self.interrupted: + print() + print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") + + def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: + # Total tests + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW + + stats = self.stats + text = f'run={stats.tests_run:,}' + if filtered: + text = f"{text} (filtered)" + report = [text] + if stats.failures: + report.append(f'{red}failures={stats.failures:,}{reset}') + if stats.skipped: + report.append(f'{yellow}skipped={stats.skipped:,}{reset}') + print(f"Total tests: {' '.join(report)}") + + # Total test files + all_tests = [self.good, self.bad, self.rerun, + self.skipped, + self.env_changed, self.run_no_tests] + run = sum(map(len, all_tests)) + text = f'run={run}' + if not first_runtests.forever: + ntest = len(first_runtests.tests) + text = f"{text}/{ntest}" + if filtered: + text = f"{text} (filtered)" + report = [text] + for name, tests, color in ( + ('failed', self.bad, red), + ('env_changed', self.env_changed, yellow), + ('skipped', self.skipped, yellow), + ('resource_denied', self.resource_denied, yellow), + ('rerun', self.rerun, yellow), + ('run_no_tests', self.run_no_tests, yellow), + ): + if tests: + report.append(f'{color}{name}={len(tests)}{reset}') + print(f"Total test files: {' '.join(report)}") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py new file mode 100644 index 00000000000..424085a0050 --- /dev/null +++ b/Lib/test/libregrtest/run_workers.py @@ -0,0 +1,627 @@ +import contextlib +import dataclasses +import faulthandler +import os.path +import queue +import signal +import subprocess +import sys +import tempfile +import threading +import time +import traceback +from typing import Any, Literal, TextIO + +from test import support +from test.support import os_helper, MS_WINDOWS + +from .logger import Logger +from .result import TestResult, State +from .results import TestResults +from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType +from .single import PROGRESS_MIN_TIME +from .utils import ( + StrPath, TestName, + format_duration, print_warning, count, plural) +from .worker import create_worker_process, USE_PROCESS_GROUP + +if MS_WINDOWS: + import locale + import msvcrt + + + +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 30.0 # seconds +assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME + +# Kill the main process after 5 minutes. It is supposed to write an update +# every PROGRESS_UPDATE seconds. Tolerate 5 minutes for Python slowest +# buildbot workers. +MAIN_PROCESS_TIMEOUT = 5 * 60.0 +assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE + +# Time to wait until a worker completes: should be immediate +WAIT_COMPLETED_TIMEOUT = 30.0 # seconds + +# Time to wait a killed process (in seconds) +WAIT_KILLED_TIMEOUT = 60.0 + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests_iter): + self.lock = threading.Lock() + self.tests_iter = tests_iter + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.tests_iter is None: + raise StopIteration + return next(self.tests_iter) + + def stop(self): + with self.lock: + self.tests_iter = None + + +@dataclasses.dataclass(slots=True, frozen=True) +class MultiprocessResult: + result: TestResult + # bpo-45410: stderr is written into stdout to keep messages order + worker_stdout: str | None = None + err_msg: str | None = None + + +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + +ExcStr = str +QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited + + +class ExitThread(Exception): + pass + + +class WorkerError(Exception): + def __init__(self, + test_name: TestName, + err_msg: str | None, + stdout: str | None, + state: str): + result = TestResult(test_name, state=state) + self.mp_result = MultiprocessResult(result, stdout, err_msg) + super().__init__() + + +_NOT_RUNNING = "" + + +class WorkerThread(threading.Thread): + def __init__(self, worker_id: int, runner: "RunWorkers") -> None: + super().__init__() + self.worker_id = worker_id + self.runtests = runner.runtests + self.pending = runner.pending + self.output = runner.output + self.timeout = runner.worker_timeout + self.log = runner.log + self.test_name = _NOT_RUNNING + self.start_time = time.monotonic() + self._popen: subprocess.Popen[str] | None = None + self._killed = False + self._stopped = False + + def __repr__(self) -> str: + info = [f'WorkerThread #{self.worker_id}'] + if self.is_alive(): + info.append("running") + else: + info.append('stopped') + test = self.test_name + if test: + info.append(f'test={test}') + popen = self._popen + if popen is not None: + dt = time.monotonic() - self.start_time + info.extend((f'pid={popen.pid}', + f'time={format_duration(dt)}')) + return '<%s>' % ' '.join(info) + + def _kill(self) -> None: + popen = self._popen + if popen is None: + return + + if self._killed: + return + self._killed = True + + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: + what = f"{self} process group" + else: + what = f"{self} process" + + print(f"Kill {what}", file=sys.stderr, flush=True) + try: + if use_killpg: + os.killpg(popen.pid, signal.SIGKILL) + else: + popen.kill() + except ProcessLookupError: + # popen.kill(): the process completed, the WorkerThread thread + # read its exit status, but Popen.send_signal() read the returncode + # just before Popen.wait() set returncode. + pass + except OSError as exc: + print_warning(f"Failed to kill {what}: {exc!r}") + + def stop(self) -> None: + # Method called from a different thread to stop this thread + self._stopped = True + self._kill() + + def _run_process(self, runtests: WorkerRunTests, output_fd: int, + tmp_dir: StrPath | None = None) -> int | None: + popen = create_worker_process(runtests, output_fd, tmp_dir) + self._popen = popen + self._killed = False + + try: + if self._stopped: + # If kill() has been called before self._popen is set, + # self._popen is still running. Call again kill() + # to ensure that the process is killed. + self._kill() + raise ExitThread + + try: + # gh-94026: stdout+stderr are written to tempfile + retcode = popen.wait(timeout=self.timeout) + assert retcode is not None + return retcode + except subprocess.TimeoutExpired: + if self._stopped: + # kill() has been called: communicate() fails on reading + # closed stdout + raise ExitThread + + # On timeout, kill the process + self._kill() + + # None means TIMEOUT for the caller + retcode = None + # bpo-38207: Don't attempt to call communicate() again: on it + # can hang until all child processes using stdout + # pipes completes. + except OSError: + if self._stopped: + # kill() has been called: communicate() fails + # on reading closed stdout + raise ExitThread + raise + return None + except: + self._kill() + raise + finally: + self._wait_completed() + self._popen = None + + def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: + """Create stdout temporary file (file descriptor).""" + + if MS_WINDOWS: + # gh-95027: When stdout is not a TTY, Python uses the ANSI code + # page for the sys.stdout encoding. If the main process runs in a + # terminal, sys.stdout uses WindowsConsoleIO with UTF-8 encoding. + encoding = locale.getencoding() + else: + encoding = sys.stdout.encoding + + # gh-94026: Write stdout+stderr to a tempfile as workaround for + # non-blocking pipes on Emscripten with NodeJS. + # gh-109425: Use "backslashreplace" error handler: log corrupted + # stdout+stderr, instead of failing with a UnicodeDecodeError and not + # logging stdout+stderr at all. + stdout_file = tempfile.TemporaryFile('w+', + encoding=encoding, + errors='backslashreplace') + stack.enter_context(stdout_file) + return stdout_file + + def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextIO | None]: + """Create JSON file.""" + + json_file_use_stdout = self.runtests.json_file_use_stdout() + if json_file_use_stdout: + json_file = JsonFile(None, JsonFileType.STDOUT) + json_tmpfile = None + else: + json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') + stack.enter_context(json_tmpfile) + + json_fd = json_tmpfile.fileno() + if MS_WINDOWS: + # The msvcrt module is only available on Windows; + # we run mypy with `--platform=linux` in CI + json_handle: int = msvcrt.get_osfhandle(json_fd) # type: ignore[attr-defined] + json_file = JsonFile(json_handle, + JsonFileType.WINDOWS_HANDLE) + else: + json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) + return (json_file, json_tmpfile) + + def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: + tests = (test_name,) + if self.runtests.rerun: + match_tests = self.runtests.get_match_tests(test_name) + else: + match_tests = None + + kwargs: dict[str, Any] = {} + if match_tests: + kwargs['match_tests'] = [(test, True) for test in match_tests] + if self.runtests.output_on_failure: + kwargs['verbose'] = True + kwargs['output_on_failure'] = False + return self.runtests.create_worker_runtests( + tests=tests, + json_file=json_file, + **kwargs) + + def run_tmp_files(self, worker_runtests: WorkerRunTests, + stdout_fd: int) -> tuple[int | None, list[StrPath]]: + # gh-93353: Check for leaked temporary files in the parent process, + # since the deletion of temporary files can happen late during + # Python finalization: too late for libregrtest. + if not support.is_wasi: + # Don't check for leaked temporary files and directories if Python is + # run on WASI. WASI doesn't pass environment variables like TMPDIR to + # worker processes. + tmp_dir = tempfile.mkdtemp(prefix="test_python_") + tmp_dir = os.path.abspath(tmp_dir) + try: + retcode = self._run_process(worker_runtests, + stdout_fd, tmp_dir) + finally: + tmp_files = os.listdir(tmp_dir) + os_helper.rmtree(tmp_dir) + else: + retcode = self._run_process(worker_runtests, stdout_fd) + tmp_files = [] + + return (retcode, tmp_files) + + def read_stdout(self, stdout_file: TextIO) -> str: + stdout_file.seek(0) + try: + return stdout_file.read().strip() + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + raise WorkerError(self.test_name, + f"Cannot read process stdout: {exc}", + stdout=None, + state=State.WORKER_BUG) + + def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, + stdout: str) -> tuple[TestResult, str]: + try: + if json_tmpfile is not None: + json_tmpfile.seek(0) + worker_json = json_tmpfile.read() + elif json_file.file_type == JsonFileType.STDOUT: + stdout, _, worker_json = stdout.rpartition("\n") + stdout = stdout.rstrip() + else: + with json_file.open(encoding='utf8') as json_fp: + worker_json = json_fp.read() + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Failed to read worker process JSON: {exc}" + raise WorkerError(self.test_name, err_msg, stdout, + state=State.WORKER_BUG) + + if not worker_json: + raise WorkerError(self.test_name, "empty JSON", stdout, + state=State.WORKER_BUG) + + try: + result = TestResult.from_json(worker_json) + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Failed to parse worker process JSON: {exc}" + raise WorkerError(self.test_name, err_msg, stdout, + state=State.WORKER_BUG) + + return (result, stdout) + + def _runtest(self, test_name: TestName) -> MultiprocessResult: + with contextlib.ExitStack() as stack: + stdout_file = self.create_stdout(stack) + json_file, json_tmpfile = self.create_json_file(stack) + worker_runtests = self.create_worker_runtests(test_name, json_file) + + retcode: str | int | None + retcode, tmp_files = self.run_tmp_files(worker_runtests, + stdout_file.fileno()) + + stdout = self.read_stdout(stdout_file) + + if retcode is None: + raise WorkerError(self.test_name, stdout=stdout, + err_msg=None, + state=State.TIMEOUT) + if retcode != 0: + name = support.get_signal_name(retcode) + if name: + retcode = f"{retcode} ({name})" + raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, + state=State.WORKER_FAILED) + + result, stdout = self.read_json(json_file, json_tmpfile, stdout) + + if tmp_files: + msg = (f'\n\n' + f'Warning -- {test_name} leaked temporary files ' + f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}') + stdout += msg + result.set_env_changed() + + return MultiprocessResult(result, stdout) + + def run(self) -> None: + fail_fast = self.runtests.fail_fast + fail_env_changed = self.runtests.fail_env_changed + try: + while not self._stopped: + try: + test_name = next(self.pending) + except StopIteration: + break + + self.start_time = time.monotonic() + self.test_name = test_name + try: + mp_result = self._runtest(test_name) + except WorkerError as exc: + mp_result = exc.mp_result + finally: + self.test_name = _NOT_RUNNING + mp_result.result.duration = time.monotonic() - self.start_time + self.output.put((False, mp_result)) + + if mp_result.result.must_stop(fail_fast, fail_env_changed): + break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) + + def _wait_completed(self) -> None: + popen = self._popen + # only needed for mypy: + if popen is None: + raise ValueError("Should never access `._popen` before calling `.run()`") + + try: + popen.wait(WAIT_COMPLETED_TIMEOUT) + except (subprocess.TimeoutExpired, OSError) as exc: + print_warning(f"Failed to wait for {self} completion " + f"(timeout={format_duration(WAIT_COMPLETED_TIMEOUT)}): " + f"{exc!r}") + + def wait_stopped(self, start_time: float) -> None: + # bpo-38207: RunWorkers.stop_workers() called self.stop() + # which killed the process. Sometimes, killing the process from the + # main thread does not interrupt popen.communicate() in + # WorkerThread thread. This loop with a timeout is a workaround + # for that. + # + # Moreover, if this method fails to join the thread, it is likely + # that Python will hang at exit while calling threading._shutdown() + # which tries again to join the blocked thread. Regrtest.main() + # uses EXIT_TIMEOUT to workaround this second bug. + while True: + # Write a message every second + self.join(1.0) + if not self.is_alive(): + break + dt = time.monotonic() - start_time + self.log(f"Waiting for {self} thread for {format_duration(dt)}") + if dt > WAIT_KILLED_TIMEOUT: + print_warning(f"Failed to join {self} in {format_duration(dt)}") + break + + +def get_running(workers: list[WorkerThread]) -> str | None: + running: list[str] = [] + for worker in workers: + test_name = worker.test_name + if test_name == _NOT_RUNNING: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + text = f'{test_name} ({format_duration(dt)})' + running.append(text) + if not running: + return None + return f"running ({len(running)}): {', '.join(running)}" + + +class RunWorkers: + def __init__(self, num_workers: int, runtests: RunTests, + logger: Logger, results: TestResults) -> None: + self.num_workers = num_workers + self.runtests = runtests + self.log = logger.log + self.display_progress = logger.display_progress + self.results: TestResults = results + self.live_worker_count = 0 + + self.output: queue.Queue[QueueContent] = queue.Queue() + tests_iter = runtests.iter_tests() + self.pending = MultiprocessIterator(tests_iter) + self.timeout = runtests.timeout + if self.timeout is not None: + # Rely on faulthandler to kill a worker process. This timouet is + # when faulthandler fails to kill a worker process. Give a maximum + # of 5 minutes to faulthandler to kill the worker. + self.worker_timeout: float | None = min(self.timeout * 1.5, self.timeout + 5 * 60) + else: + self.worker_timeout = None + self.workers: list[WorkerThread] = [] + + jobs = self.runtests.get_jobs() + if jobs is not None: + # Don't spawn more threads than the number of jobs: + # these worker threads would never get anything to do. + self.num_workers = min(self.num_workers, jobs) + + def start_workers(self) -> None: + self.workers = [WorkerThread(index, self) + for index in range(1, self.num_workers + 1)] + jobs = self.runtests.get_jobs() + if jobs is not None: + tests = count(jobs, 'test') + else: + tests = 'tests' + nworkers = len(self.workers) + processes = plural(nworkers, "process", "processes") + msg = (f"Run {tests} in parallel using " + f"{nworkers} worker {processes}") + if self.timeout and self.worker_timeout is not None: + msg += (" (timeout: %s, worker timeout: %s)" + % (format_duration(self.timeout), + format_duration(self.worker_timeout))) + self.log(msg) + for worker in self.workers: + worker.start() + self.live_worker_count += 1 + + def stop_workers(self) -> None: + start_time = time.monotonic() + for worker in self.workers: + worker.stop() + for worker in self.workers: + worker.wait_stopped(start_time) + + def _get_result(self) -> QueueOutput | None: + pgo = self.runtests.pgo + use_faulthandler = (self.timeout is not None) + + # bpo-46205: check the status of workers every iteration to avoid + # waiting forever on an empty queue. + while self.live_worker_count > 0: + if use_faulthandler: + faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, + exit=True) + + # wait for a thread + try: + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result + except queue.Empty: + pass + + if not pgo: + # display progress + running = get_running(self.workers) + if running: + self.log(running) + return None + + def display_result(self, mp_result: MultiprocessResult) -> None: + result = mp_result.result + pgo = self.runtests.pgo + + text = str(result) + if mp_result.err_msg: + # WORKER_BUG + text += ' (%s)' % mp_result.err_msg + elif (result.duration and result.duration >= PROGRESS_MIN_TIME and not pgo): + text += ' (%s)' % format_duration(result.duration) + if not pgo: + running = get_running(self.workers) + if running: + text += f' -- {running}' + self.display_progress(self.test_index, text) + + def _process_result(self, item: QueueOutput) -> TestResult: + """Returns True if test runner must stop.""" + if item[0]: + # Thread got an exception + format_exc = item[1] + print_warning(f"regrtest worker thread failed: {format_exc}") + result = TestResult("", state=State.WORKER_BUG) + self.results.accumulate_result(result, self.runtests) + return result + + self.test_index += 1 + mp_result = item[1] + result = mp_result.result + self.results.accumulate_result(result, self.runtests) + self.display_result(mp_result) + + # Display worker stdout + if not self.runtests.output_on_failure: + show_stdout = True + else: + # --verbose3 ignores stdout on success + show_stdout = (result.state != State.PASSED) + if show_stdout: + stdout = mp_result.worker_stdout + if stdout: + print(stdout, flush=True) + + return result + + def run(self) -> None: + fail_fast = self.runtests.fail_fast + fail_env_changed = self.runtests.fail_env_changed + + self.start_workers() + + self.test_index = 0 + try: + while True: + item = self._get_result() + if item is None: + break + + result = self._process_result(item) + if result.must_stop(fail_fast, fail_env_changed): + break + except KeyboardInterrupt: + print() + self.results.interrupted = True + finally: + if self.timeout is not None: + faulthandler.cancel_dump_traceback_later() + + # Always ensure that all worker processes are no longer + # worker when we exit this function + self.pending.stop() + self.stop_workers() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py deleted file mode 100644 index e2af18f3499..00000000000 --- a/Lib/test/libregrtest/runtest.py +++ /dev/null @@ -1,328 +0,0 @@ -import collections -import faulthandler -import functools -# import gc -import importlib -import io -import os -import sys -import time -import traceback -import unittest - -from test import support -from test.support import os_helper, import_helper -from test.libregrtest.refleak import dash_R, clear_caches -from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.utils import print_warning - - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process -TEST_DID_NOT_RUN = -6 # error in a child process - -_FORMAT_TEST_RESULT = { - PASSED: '%s passed', - FAILED: '%s failed', - ENV_CHANGED: '%s failed (env changed)', - SKIPPED: '%s skipped', - RESOURCE_DENIED: '%s skipped (resource denied)', - INTERRUPTED: '%s interrupted', - CHILD_ERROR: '%s crashed', - TEST_DID_NOT_RUN: '%s run no tests', -} - -# Minimum duration of a test to display its duration or to mention that -# the test is running in background -PROGRESS_MIN_TIME = 30.0 # seconds - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - # 'test_grammar', - # 'test_opcodes', - # 'test_dict', - # 'test_builtin', - # 'test_exceptions', - # 'test_types', - # 'test_unittest', - # 'test_doctest', - # 'test_doctest2', - # 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - - -# used by --findleaks, store for gc.garbage -FOUND_GARBAGE = [] - - -def format_test_result(result): - fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") - return fmt % result.test_name - - -def findtestdir(path=None): - return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir - - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) - - -def get_abs_module(ns, test_name): - if test_name.startswith('test.') or ns.testdir: - return test_name - else: - # Import it from the test package - return 'test.' + test_name - - -TestResult = collections.namedtuple('TestResult', - 'test_name result test_time xml_data') - -def _runtest(ns, test_name): - # Handle faulthandler timeout, capture stdout+stderr, XML serialization - # and measure time. - - output_on_failure = ns.verbose3 - - use_timeout = (ns.timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(ns.timeout, exit=True) - - start_time = time.perf_counter() - try: - support.set_match_tests(ns.match_tests) - support.junit_xml_list = xml_list = [] if ns.xmlpath else None - if ns.failfast: - support.failfast = True - - if output_on_failure: - support.verbose = True - - stream = io.StringIO() - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = _runtest_inner(ns, test_name, - display_failure=False) - if result != PASSED: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - # Tell tests to be moderately quiet - support.verbose = ns.verbose - - result = _runtest_inner(ns, test_name, - display_failure=not ns.verbose) - - if xml_list: - import xml.etree.ElementTree as ET - xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list] - else: - xml_data = None - - test_time = time.perf_counter() - start_time - - return TestResult(test_name, result, test_time, xml_data) - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - support.junit_xml_list = None - - -def runtest(ns, test_name): - """Run a single test. - - ns -- regrtest namespace of options - test_name -- the name of the test - - Returns the tuple (result, test_time, xml_data), where result is one - of the constants: - - INTERRUPTED KeyboardInterrupt - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - EMPTY_TEST_SUITE test ran no subtests. - - If ns.xmlpath is not None, xml_data is a list containing each - generated testsuite element. - """ - try: - return _runtest(ns, test_name) - except: - if not ns.pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - return TestResult(test_name, FAILED, 0.0, None) - - -def _test_module(the_module): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - - -def _runtest_inner2(ns, test_name): - # Load the test function, run the test function, handle huntrleaks - # and findleaks to detect leaks - - abstest = get_abs_module(ns, test_name) - - # remove the module from sys.module to reload it if it was already imported - import_helper.unload(abstest) - - the_module = importlib.import_module(abstest) - - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - test_runner = functools.partial(_test_module, the_module) - - try: - if ns.huntrleaks: - # Return True if the test leaked references - refleak = dash_R(ns, test_name, test_runner) - else: - test_runner() - refleak = False - finally: - cleanup_test_droppings(test_name, ns.verbose) - - support.gc_collect() - - # if gc.garbage: - # support.environment_altered = True - # print_warning(f"{test_name} created {len(gc.garbage)} " - # f"uncollectable object(s).") - - # # move the uncollectable objects somewhere, - # # so we don't see them again - # FOUND_GARBAGE.extend(gc.garbage) - # gc.garbage.clear() - - support.reap_children() - - return refleak - - -def _runtest_inner(ns, test_name, display_failure=True): - # Detect environment changes, handle exceptions. - - # Reset the environment_altered flag to detect if a test altered - # the environment - support.environment_altered = False - - if ns.pgo: - display_failure = False - - try: - clear_caches() - - # with saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: - refleak = _runtest_inner2(ns, test_name) - except support.ResourceDenied as msg: - if not ns.quiet and not ns.pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - return RESOURCE_DENIED - except unittest.SkipTest as msg: - if not ns.quiet and not ns.pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - return SKIPPED - except support.TestFailed as exc: - msg = f"test {test_name} failed" - if display_failure: - msg = f"{msg} -- {exc}" - print(msg, file=sys.stderr, flush=True) - return FAILED - except support.TestDidNotRun: - return TEST_DID_NOT_RUN - except KeyboardInterrupt: - print() - return INTERRUPTED - except: - if not ns.pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - return FAILED - - if refleak: - return FAILED - # if environment.changed: - # return ENV_CHANGED - return PASSED - - -def cleanup_test_droppings(test_name, verbose): - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - support.gc_collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (os_helper.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - import shutil - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") - - if verbose: - print_warning("%r left behind %s %r" % (test_name, kind, name)) - support.environment_altered = True - - try: - import stat - # fix possible permissions problems that might prevent cleanup - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as exc: - print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py deleted file mode 100644 index c2177d99955..00000000000 --- a/Lib/test/libregrtest/runtest_mp.py +++ /dev/null @@ -1,288 +0,0 @@ -import collections -import faulthandler -import json -import os -import queue -import subprocess -import sys -import threading -import time -import traceback -import types -from test import support - -from test.libregrtest.runtest import ( - runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, - format_test_result, TestResult) -from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import format_duration -from test.support import os_helper - - -# Display the running tests if nothing happened last N seconds -PROGRESS_UPDATE = 30.0 # seconds - - -def must_stop(result): - return result.result in (INTERRUPTED, CHILD_ERROR) - - -def run_test_in_subprocess(testname, ns): - ns_dict = vars(ns) - worker_args = (ns_dict, testname) - worker_args = json.dumps(worker_args) - - cmd = [sys.executable, *support.args_from_interpreter_flags(), - '-u', # Unbuffered stdout and stderr - '-m', 'test.regrtest', - '--worker-args', worker_args] - if ns.pgo: - cmd += ['--pgo'] - - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - return subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=os_helper.SAVEDCWD) - - -def run_tests_worker(worker_args): - ns_dict, testname = json.loads(worker_args) - ns = types.SimpleNamespace(**ns_dict) - - setup_tests(ns) - - result = runtest(ns, testname) - print() # Force a newline (just in case) - print(json.dumps(result), flush=True) - sys.exit(0) - - -# We do not use a generator so multiple threads can call next(). -class MultiprocessIterator: - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - return next(self.tests) - - -MultiprocessResult = collections.namedtuple('MultiprocessResult', - 'result stdout stderr error_msg') - -class MultiprocessThread(threading.Thread): - def __init__(self, pending, output, ns): - super().__init__() - self.pending = pending - self.output = output - self.ns = ns - self.current_test_name = None - self.start_time = None - self._popen = None - - def kill(self): - if not self.is_alive(): - return - if self._popen is not None: - self._popen.kill() - - def _runtest(self, test_name): - try: - self.start_time = time.monotonic() - self.current_test_name = test_name - - popen = run_test_in_subprocess(test_name, self.ns) - self._popen = popen - with popen: - try: - stdout, stderr = popen.communicate() - except: - popen.kill() - popen.wait() - raise - - retcode = popen.wait() - finally: - self.current_test_name = None - self._popen = None - - stdout = stdout.strip() - stderr = stderr.rstrip() - - err_msg = None - if retcode != 0: - err_msg = "Exit code %s" % retcode - else: - stdout, _, result = stdout.rpartition("\n") - stdout = stdout.rstrip() - if not result: - err_msg = "Failed to parse worker stdout" - else: - try: - # deserialize run_tests_worker() output - result = json.loads(result) - result = TestResult(*result) - except Exception as exc: - err_msg = "Failed to parse worker JSON: %s" % exc - - if err_msg is not None: - test_time = time.monotonic() - self.start_time - result = TestResult(test_name, CHILD_ERROR, test_time, None) - - return MultiprocessResult(result, stdout, stderr, err_msg) - - def run(self): - while True: - try: - try: - test_name = next(self.pending) - except StopIteration: - break - - mp_result = self._runtest(test_name) - self.output.put((False, mp_result)) - - if must_stop(mp_result.result): - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break - - -def get_running(workers): - running = [] - for worker in workers: - current_test_name = worker.current_test_name - if not current_test_name: - continue - dt = time.monotonic() - worker.start_time - if dt >= PROGRESS_MIN_TIME: - text = '%s (%s)' % (current_test_name, format_duration(dt)) - running.append(text) - return running - - -class MultiprocessRunner: - def __init__(self, regrtest): - self.regrtest = regrtest - self.ns = regrtest.ns - self.output = queue.Queue() - self.pending = MultiprocessIterator(self.regrtest.tests) - if self.ns.timeout is not None: - self.test_timeout = self.ns.timeout * 1.5 - else: - self.test_timeout = None - self.workers = None - - def start_workers(self): - self.workers = [MultiprocessThread(self.pending, self.output, self.ns) - for _ in range(self.ns.use_mp)] - print("Run tests in parallel using %s child processes" - % len(self.workers)) - for worker in self.workers: - worker.start() - - def wait_workers(self): - for worker in self.workers: - worker.kill() - for worker in self.workers: - worker.join() - - def _get_result(self): - if not any(worker.is_alive() for worker in self.workers): - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None - - while True: - if self.test_timeout is not None: - faulthandler.dump_traceback_later(self.test_timeout, exit=True) - - # wait for a thread - timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) - try: - return self.output.get(timeout=timeout) - except queue.Empty: - pass - - # display progress - running = get_running(self.workers) - if running and not self.ns.pgo: - print('running: %s' % ', '.join(running), flush=True) - - def display_result(self, mp_result): - result = mp_result.result - - text = format_test_result(result) - if mp_result.error_msg is not None: - # CHILD_ERROR - text += ' (%s)' % mp_result.error_msg - elif (result.test_time >= PROGRESS_MIN_TIME and not self.ns.pgo): - text += ' (%s)' % format_duration(result.test_time) - running = get_running(self.workers) - if running and not self.ns.pgo: - text += ' -- running: %s' % ', '.join(running) - self.regrtest.display_progress(self.test_index, text) - - def _process_result(self, item): - if item[0]: - # Thread got an exception - format_exc = item[1] - print(f"regrtest worker thread failed: {format_exc}", - file=sys.stderr, flush=True) - return True - - self.test_index += 1 - mp_result = item[1] - self.regrtest.accumulate_result(mp_result.result) - self.display_result(mp_result) - - if mp_result.stdout: - print(mp_result.stdout, flush=True) - if mp_result.stderr and not self.ns.pgo: - print(mp_result.stderr, file=sys.stderr, flush=True) - - if must_stop(mp_result.result): - return True - - return False - - def run_tests(self): - self.start_workers() - - self.test_index = 0 - try: - while True: - item = self._get_result() - if item is None: - break - - stop = self._process_result(item) - if stop: - break - except KeyboardInterrupt: - print() - self.regrtest.interrupted = True - finally: - if self.test_timeout is not None: - faulthandler.cancel_dump_traceback_later() - - self.wait_workers() - - -def run_tests_multiprocess(regrtest): - MultiprocessRunner(regrtest).run_tests() diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py new file mode 100644 index 00000000000..759f24fc25e --- /dev/null +++ b/Lib/test/libregrtest/runtests.py @@ -0,0 +1,225 @@ +import contextlib +import dataclasses +import json +import os +import shlex +import subprocess +import sys +from typing import Any, Iterator + +from test import support + +from .utils import ( + StrPath, StrJSON, TestTuple, TestName, TestFilter, FilterTuple, FilterDict) + + +class JsonFileType: + UNIX_FD = "UNIX_FD" + WINDOWS_HANDLE = "WINDOWS_HANDLE" + STDOUT = "STDOUT" + + +@dataclasses.dataclass(slots=True, frozen=True) +class JsonFile: + # file type depends on file_type: + # - UNIX_FD: file descriptor (int) + # - WINDOWS_HANDLE: handle (int) + # - STDOUT: use process stdout (None) + file: int | None + file_type: str + + def configure_subprocess(self, popen_kwargs: dict[str, Any]) -> None: + match self.file_type: + case JsonFileType.UNIX_FD: + # Unix file descriptor + popen_kwargs['pass_fds'] = [self.file] + case JsonFileType.WINDOWS_HANDLE: + # Windows handle + # We run mypy with `--platform=linux` so it complains about this: + startupinfo = subprocess.STARTUPINFO() # type: ignore[attr-defined] + startupinfo.lpAttributeList = {"handle_list": [self.file]} + popen_kwargs['startupinfo'] = startupinfo + + @contextlib.contextmanager + def inherit_subprocess(self) -> Iterator[None]: + if sys.platform == 'win32' and self.file_type == JsonFileType.WINDOWS_HANDLE: + os.set_handle_inheritable(self.file, True) + try: + yield + finally: + os.set_handle_inheritable(self.file, False) + else: + yield + + def open(self, mode='r', *, encoding): + if self.file_type == JsonFileType.STDOUT: + raise ValueError("for STDOUT file type, just use sys.stdout") + + file = self.file + if self.file_type == JsonFileType.WINDOWS_HANDLE: + import msvcrt + # Create a file descriptor from the handle + file = msvcrt.open_osfhandle(file, os.O_WRONLY) + return open(file, mode, encoding=encoding) + + +@dataclasses.dataclass(slots=True, frozen=True) +class HuntRefleak: + warmups: int + runs: int + filename: StrPath + + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + + +@dataclasses.dataclass(slots=True, frozen=True) +class RunTests: + tests: TestTuple + fail_fast: bool + fail_env_changed: bool + match_tests: TestFilter + match_tests_dict: FilterDict | None + rerun: bool + forever: bool + pgo: bool + pgo_extended: bool + output_on_failure: bool + timeout: float | None + verbose: int + quiet: bool + hunt_refleak: HuntRefleak | None + test_dir: StrPath | None + use_junit: bool + coverage: bool + memory_limit: str | None + gc_threshold: int | None + use_resources: tuple[str, ...] + python_cmd: tuple[str, ...] | None + randomize: bool + random_seed: int | str + parallel_threads: int | None + + def copy(self, **override) -> 'RunTests': + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + + def create_worker_runtests(self, **override) -> WorkerRunTests: + state = dataclasses.asdict(self) + state.update(override) + return WorkerRunTests(**state) + + def get_match_tests(self, test_name: TestName) -> FilterTuple | None: + if self.match_tests_dict is not None: + return self.match_tests_dict.get(test_name, None) + else: + return None + + def get_jobs(self) -> int | None: + # Number of run_single_test() calls needed to run all tests. + # None means that there is not bound limit (--forever option). + if self.forever: + return None + return len(self.tests) + + def iter_tests(self) -> Iterator[TestName]: + if self.forever: + while True: + yield from self.tests + else: + yield from self.tests + + def json_file_use_stdout(self) -> bool: + # Use STDOUT in two cases: + # + # - If --python command line option is used; + # - On Emscripten and WASI. + # + # On other platforms, UNIX_FD or WINDOWS_HANDLE can be used. + return ( + bool(self.python_cmd) + or support.is_emscripten + or support.is_wasi + ) + + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + if self.parallel_threads: + args.append(f"--parallel-threads={self.parallel_threads}") + args.append(f"--randseed={self.random_seed}") + return args + + +@dataclasses.dataclass(slots=True, frozen=True) +class WorkerRunTests(RunTests): + json_file: JsonFile + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'WorkerRunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + + +class _EncodeRunTests(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, WorkerRunTests): + result = dataclasses.asdict(o) + result["__runtests__"] = True + return result + else: + return super().default(o) + + +def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: + if "__runtests__" in data: + data.pop('__runtests__') + if data['hunt_refleak']: + data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) + if data['json_file']: + data['json_file'] = JsonFile(**data['json_file']) + return WorkerRunTests(**data) + else: + return data diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index b9a1c0b3926..eeb1f17d702 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -1,20 +1,24 @@ -import asyncio import builtins import locale -import logging import os -import shutil import sys -import sysconfig import threading -import warnings + from test import support from test.support import os_helper -from test.libregrtest.utils import print_warning + +from .utils import print_warning + +# Import termios to save and restore terminal echo. This is only available on +# Unix, and it's fine if the module can't be found. try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None + import termios # noqa: F401 +except ModuleNotFoundError: + pass + + +class SkipTestEnvironment(Exception): + pass # Unit tests are supposed to leave the execution environment unchanged @@ -28,21 +32,19 @@ class saved_test_environment: """Save bits of the test environment and restore them at block exit. - with saved_test_environment(testname, verbose, quiet): + with saved_test_environment(test_name, verbose, quiet): #stuff Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. + the saved items was changed by the test. The support.environment_altered + attribute is set to True if a change is detected. If verbose is more than 1, the before and after state of changed items is also printed. """ - changed = False - - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname + def __init__(self, test_name, verbose, quiet, *, pgo): + self.test_name = test_name self.verbose = verbose self.quiet = quiet self.pgo = pgo @@ -69,12 +71,41 @@ def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): 'files', 'locale', 'warnings.showwarning', 'shutil_archive_formats', 'shutil_unpack_formats', 'asyncio.events._event_loop_policy', + 'urllib.requests._url_tempfiles', 'urllib.requests._opener', + 'stty_echo', ) + def get_module(self, name): + # function for restore() methods + return sys.modules[name] + + def try_get_module(self, name): + # function for get() methods + try: + return self.get_module(name) + except KeyError: + raise SkipTestEnvironment + + def get_urllib_requests__url_tempfiles(self): + urllib_request = self.try_get_module('urllib.request') + return list(urllib_request._url_tempfiles) + def restore_urllib_requests__url_tempfiles(self, tempfiles): + for filename in tempfiles: + os_helper.unlink(filename) + + def get_urllib_requests__opener(self): + urllib_request = self.try_get_module('urllib.request') + return urllib_request._opener + def restore_urllib_requests__opener(self, opener): + urllib_request = self.get_module('urllib.request') + urllib_request._opener = opener + def get_asyncio_events__event_loop_policy(self): + self.try_get_module('asyncio') return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): - asyncio.set_event_loop_policy(policy) + asyncio = self.get_module('asyncio') + asyncio.events._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -132,39 +163,46 @@ def restore___import__(self, import_): builtins.__import__ = import_ def get_warnings_filters(self): + warnings = self.try_get_module('warnings') return id(warnings.filters), warnings.filters, warnings.filters[:] def restore_warnings_filters(self, saved_filters): + warnings = self.get_module('warnings') warnings.filters = saved_filters[1] warnings.filters[:] = saved_filters[2] def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') + asyncore = sys.modules.get('test.support.asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') + asyncore = sys.modules.get('test.support.asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) asyncore.socket_map.update(saved_map) def get_shutil_archive_formats(self): + shutil = self.try_get_module('shutil') # we could call get_archives_formats() but that only returns the # registry keys; we want to check the values too (the functions that # are registered) return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() def restore_shutil_archive_formats(self, saved): + shutil = self.get_module('shutil') shutil._ARCHIVE_FORMATS = saved[0] shutil._ARCHIVE_FORMATS.clear() shutil._ARCHIVE_FORMATS.update(saved[1]) def get_shutil_unpack_formats(self): + shutil = self.try_get_module('shutil') return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() def restore_shutil_unpack_formats(self, saved): + shutil = self.get_module('shutil') shutil._UNPACK_FORMATS = saved[0] shutil._UNPACK_FORMATS.clear() shutil._UNPACK_FORMATS.update(saved[1]) def get_logging__handlers(self): + logging = self.try_get_module('logging') # _handlers is a WeakValueDictionary return id(logging._handlers), logging._handlers, logging._handlers.copy() def restore_logging__handlers(self, saved_handlers): @@ -172,6 +210,7 @@ def restore_logging__handlers(self, saved_handlers): pass def get_logging__handlerList(self): + logging = self.try_get_module('logging') # _handlerList is a list of weakrefs to handlers return id(logging._handlerList), logging._handlerList, logging._handlerList[:] def restore_logging__handlerList(self, saved_handlerList): @@ -188,46 +227,54 @@ def restore_sys_warnoptions(self, saved_options): # to track reference leaks. def get_threading__dangling(self): # This copies the weakrefs without making any strong reference - return threading._dangling.copy() + # XXX: RUSTPYTHON - filter out dead threads since gc doesn't clean WeakSet. Revert this line when we have a GC + # return threading._dangling.copy() + return {t for t in threading._dangling if t.is_alive()} def restore_threading__dangling(self, saved): threading._dangling.clear() threading._dangling.update(saved) # Same for Process objects def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None + multiprocessing_process = self.try_get_module('multiprocessing.process') # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() + multiprocessing_process._cleanup() # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() + # TODO: RUSTPYTHON - filter out dead processes since gc doesn't clean WeakSet. Revert this line when we have a GC + # return multiprocessing_process._dangling.copy() + return {p for p in multiprocessing_process._dangling if p.is_alive()} def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) + multiprocessing_process = self.get_module('multiprocessing.process') + multiprocessing_process._dangling.clear() + multiprocessing_process._dangling.update(saved) def get_sysconfig__CONFIG_VARS(self): # make sure the dict is initialized + sysconfig = self.try_get_module('sysconfig') sysconfig.get_config_var('prefix') return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, dict(sysconfig._CONFIG_VARS)) def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig = self.get_module('sysconfig') sysconfig._CONFIG_VARS = saved[1] sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(saved[2]) def get_sysconfig__INSTALL_SCHEMES(self): + sysconfig = self.try_get_module('sysconfig') return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, sysconfig._INSTALL_SCHEMES.copy()) def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig = self.get_module('sysconfig') sysconfig._INSTALL_SCHEMES = saved[1] sysconfig._INSTALL_SCHEMES.clear() sysconfig._INSTALL_SCHEMES.update(saved[2]) def get_files(self): + # XXX: Maybe add an allow-list here? return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) + for fn in os.listdir() + if not fn.startswith(".hypothesis")) def restore_files(self, saved_value): fn = os_helper.TESTFN if fn not in saved_value and (fn + '/') not in saved_value: @@ -251,10 +298,30 @@ def restore_locale(self, saved): locale.setlocale(lc, setting) def get_warnings_showwarning(self): + warnings = self.try_get_module('warnings') return warnings.showwarning def restore_warnings_showwarning(self, fxn): + warnings = self.get_module('warnings') warnings.showwarning = fxn + def get_stty_echo(self): + termios = self.try_get_module('termios') + if not os.isatty(fd := sys.__stdin__.fileno()): + return None + attrs = termios.tcgetattr(fd) + lflags = attrs[3] + return bool(lflags & termios.ECHO) + def restore_stty_echo(self, echo): + termios = self.get_module('termios') + attrs = termios.tcgetattr(fd := sys.__stdin__.fileno()) + if echo: + # Turn echo on. + attrs[3] |= termios.ECHO + else: + # Turn echo off. + attrs[3] &= ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, attrs) + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') @@ -263,29 +330,32 @@ def resource_info(self): yield name, getattr(self, get_name), getattr(self, restore_name) def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) + self.saved_values = [] + for name, get, restore in self.resource_info(): + try: + original = get() + except SkipTestEnvironment: + continue + + self.saved_values.append((name, get, restore, original)) return self def __exit__(self, exc_type, exc_val, exc_tb): saved_values = self.saved_values - del self.saved_values + self.saved_values = None # Some resources use weak references support.gc_collect() - # Read support.environment_altered, set by support helper functions - self.changed |= support.environment_altered - - for name, get, restore in self.resource_info(): + for name, get, restore, original in saved_values: current = get() - original = saved_values.pop(name) # Check for changes to the resource's value if current != original: - self.changed = True + support.environment_altered = True restore(original) if not self.quiet and not self.pgo: - print_warning(f"{name} was modified by {self.testname}") - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print_warning( + f"{name} was modified by {self.test_name}\n" + f" Before: {original}\n" + f" After: {current} ") return False diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b1a5ded5254..b9b76a44e3b 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,17 +1,34 @@ -import atexit import faulthandler +import gc +import io import os +import random import signal import sys import unittest from test import support -try: - import gc -except ImportError: - gc = None +from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from _colorize import can_colorize # type: ignore[import-not-found] +from .filter import set_match_tests +from .runtests import RunTests +from .utils import ( + setup_unraisable_hook, setup_threading_excepthook, + adjust_rlimit_nofile) -def setup_tests(ns): + +UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" + + +def setup_test_dir(testdir: str | None) -> None: + if testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(testdir)) + + +def setup_process() -> None: + assert sys.__stderr__ is not None, "sys.__stderr__ is None" try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -19,13 +36,13 @@ def setup_tests(ns): # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. - stderr_fd = None + pass else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] + signals: list[signal.Signals] = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): @@ -33,13 +50,17 @@ def setup_tests(ns): for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) - # replace_stdout() - # support.record_original_stdout(sys.stdout) + adjust_rlimit_nofile() - if ns.testdir: - # Prepend test directory to sys.path, so runtest() will be able - # to locate tests - sys.path.insert(0, os.path.abspath(ns.testdir)) + support.record_original_stdout(sys.stdout) + + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + # + # Use an assertion to fix mypy error. + assert isinstance(sys.stdout, io.TextIOWrapper) + sys.stdout.reconfigure(errors="backslashreplace") # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -56,79 +77,73 @@ def setup_tests(ns): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: + module.__file__ = os.path.abspath(module.__file__) # type: ignore[type-var] + + if hasattr(sys, 'addaudithook'): + # Add an auditing hook for all tests to ensure PySys_Audit is tested + def _test_audit_hook(name, args): pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + sys.addaudithook(_test_audit_hook) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_unraisable_hook() + setup_threading_excepthook() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) + # Ensure there's a non-ASCII character in env vars at all times to force + # tests consider this case. See BPO-44647 for details. + if TESTFN_UNDECODABLE and os.supports_bytes_environ: + os.environb.setdefault(UNICODE_GUARD_ENV.encode(), TESTFN_UNDECODABLE) + elif FS_NONASCII: + os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) - if ns.threshold is not None: - gc.set_threshold(ns.threshold) - try: - import msvcrt - except ImportError: - pass +def setup_tests(runtests: RunTests) -> None: + support.verbose = runtests.verbose + support.failfast = runtests.fail_fast + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + + set_match_tests(runtests.match_tests) + + if runtests.use_junit: + support.junit_xml_list = [] + from .testresult import RegressionTestResult + RegressionTestResult.USE_XML = True else: - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if ns.verbose and ns.verbose >= 2: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) + support.junit_xml_list = None - support.use_resources = ns.use_resources + if runtests.memory_limit is not None: + support.set_memlimit(runtests.memory_limit) + support.suppress_msvcrt_asserts(runtests.verbose >= 2) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - stdout = sys.stdout - try: - fd = stdout.fileno() - except ValueError: - # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper - # object. Leaving sys.stdout unchanged. - # - # Catch ValueError to catch io.UnsupportedOperation on TextIOBase - # and ValueError on a closed stream. - return - - sys.stdout = open(fd, 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) + support.use_resources = runtests.use_resources + + timeout = runtests.timeout + if timeout is not None: + # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT + support.LOOPBACK_TIMEOUT = max(support.LOOPBACK_TIMEOUT, timeout / 120) + # don't increase INTERNET_TIMEOUT + support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) + support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) + + # If --timeout is short: reduce timeouts + support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, timeout) + support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, timeout) + support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) + support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) + + if runtests.hunt_refleak: + # private attribute that mypy doesn't know about: + unittest.BaseTestSuite._cleanup = False # type: ignore[attr-defined] + + if runtests.gc_threshold is not None: + gc.set_threshold(runtests.gc_threshold) + + random.seed(runtests.random_seed) + + # sys.stdout is redirected to a StringIO in single process mode on which + # color auto-detect fails as StringIO is not a TTY. If the original + # sys.stdout supports color pass that through with FORCE_COLOR so that when + # results are printed, such as with -W, they get color. + if can_colorize(file=sys.stdout): + os.environ['FORCE_COLOR'] = "1" diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py new file mode 100644 index 00000000000..3dfb0b01dc1 --- /dev/null +++ b/Lib/test/libregrtest/single.py @@ -0,0 +1,361 @@ +import faulthandler +import gc +import importlib +import io +import sys +import time +import traceback +import unittest + +from _colorize import get_colors # type: ignore[import-not-found] +from test import support +from test.support import threading_helper + +from .filter import match_test +from .result import State, TestResult, TestStats +from .runtests import RunTests +from .save_env import saved_test_environment +from .setup import setup_tests +from .testresult import get_test_runner +from .parallel_case import ParallelTestCase +from .utils import ( + TestName, + clear_caches, remove_testfn, abs_module_name, print_warning) + + +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_unittest(test_mod, runtests: RunTests): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(test_mod) + + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + _filter_suite(tests, match_test) + if runtests.parallel_threads: + _parallelize_tests(tests, runtests.parallel_threads) + return _run_suite(tests) + +def _filter_suite(suite, pred): + """Recursively filter test cases in a suite based on a predicate.""" + newtests = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _filter_suite(test, pred) + newtests.append(test) + else: + if pred(test): + newtests.append(test) + suite._tests = newtests + +def _parallelize_tests(suite, parallel_threads: int): + def is_thread_unsafe(test): + test_method = getattr(test, test._testMethodName) + instance = test_method.__self__ + return (getattr(test_method, "__unittest_thread_unsafe__", False) or + getattr(instance, "__unittest_thread_unsafe__", False)) + + newtests: list[object] = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _parallelize_tests(test, parallel_threads) + newtests.append(test) + continue + + if is_thread_unsafe(test): + # Don't parallelize thread-unsafe tests + newtests.append(test) + continue + + newtests.append(ParallelTestCase(test, parallel_threads)) + suite._tests = newtests + +def _run_suite(suite): + """Run tests from a unittest.TestSuite-derived class.""" + runner = get_test_runner(sys.stdout, + verbosity=support.verbose, + capture_output=(support.junit_xml_list is not None)) + + result = runner.run(suite) + + if support.junit_xml_list is not None: + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) + + if not result.testsRun and not result.skipped and not result.errors: + raise support.TestDidNotRun + if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "multiple errors occurred" + if not support.verbose: err += "; run in verbose mode for details" + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise support.TestFailedWithDetails(err, errors, failures, stats=stats) + return result + + +def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: + # Run test_func(), collect statistics, and detect reference and memory + # leaks. + if runtests.hunt_refleak: + from .refleak import runtest_refleak + refleak, test_result = runtest_refleak(result.test_name, test_func, + runtests.hunt_refleak, + runtests.quiet) + else: + test_result = test_func() + refleak = False + + if refleak: + result.state = State.REFLEAK + + stats: TestStats | None + + match test_result: + case TestStats(): + stats = test_result + case unittest.TestResult(): + stats = TestStats.from_unittest(test_result) + case None: + print_warning(f"{result.test_name} test runner returned None: {test_func}") + stats = None + case _: + # Don't import doctest at top level since only few tests return + # a doctest.TestResult instance. + import doctest + if isinstance(test_result, doctest.TestResults): + stats = TestStats.from_doctest(test_result) + else: + print_warning(f"Unknown test result type: {type(test_result)}") + stats = None + + result.stats = stats + + +# Storage of uncollectable GC objects (gc.garbage) +GC_GARBAGE = [] + + +def _load_run_test(result: TestResult, runtests: RunTests) -> None: + # Load the test module and run the tests. + test_name = result.test_name + module_name = abs_module_name(test_name, runtests.test_dir) + test_mod = importlib.import_module(module_name) + + if hasattr(test_mod, "test_main"): + # https://github.com/python/cpython/issues/89392 + raise Exception(f"Module {test_name} defines test_main() which " + f"is no longer supported by regrtest") + def test_func(): + return run_unittest(test_mod, runtests) + + try: + regrtest_runner(result, test_func, runtests) + finally: + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce + # failures. + support.gc_collect() + + remove_testfn(test_name, runtests.verbose) + + # XXX: RUSTPYTHON, build a functional garbage collector into the interpreter + # if gc.garbage: + # support.environment_altered = True + # print_warning(f"{test_name} created {len(gc.garbage)} " + # f"uncollectable object(s)") + + # # move the uncollectable objects somewhere, + # # so we don't see them again + # GC_GARBAGE.extend(gc.garbage) + # gc.garbage.clear() + + support.reap_children() + + +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + display_failure: bool = True) -> None: + # Handle exceptions, detect environment changes. + stdout = get_colors(file=sys.stdout) + stderr = get_colors(file=sys.stderr) + + # Reset the environment_altered flag to detect if a test altered + # the environment + support.environment_altered = False + + pgo = runtests.pgo + if pgo: + display_failure = False + quiet = runtests.quiet + + test_name = result.test_name + try: + clear_caches() + support.gc_collect() + + with saved_test_environment(test_name, + runtests.verbose, quiet, pgo=pgo): + _load_run_test(result, runtests) + except support.ResourceDenied as exc: + if not quiet and not pgo: + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) + result.state = State.RESOURCE_DENIED + return + except unittest.SkipTest as exc: + if not quiet and not pgo: + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) + result.state = State.SKIPPED + return + except support.TestFailedWithDetails as exc: + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" + if display_failure: + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.errors = exc.errors + result.failures = exc.failures + result.stats = exc.stats + return + except support.TestFailed as exc: + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" + if display_failure: + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.stats = exc.stats + return + except support.TestDidNotRun: + result.state = State.DID_NOT_RUN + return + except KeyboardInterrupt: + print() + result.state = State.INTERRUPTED + return + except: + if not pgo: + msg = traceback.format_exc() + print( + f"{stderr.RED}test {test_name} crashed -- {msg}{stderr.RESET}", + file=sys.stderr, + flush=True, + ) + result.state = State.UNCAUGHT_EXC + return + + if support.environment_altered: + result.set_env_changed() + # Don't override the state if it was already set (REFLEAK or ENV_CHANGED) + if result.state is None: + result.state = State.PASSED + + +def _runtest(result: TestResult, runtests: RunTests) -> None: + # Capture stdout and stderr, set faulthandler timeout, + # and create JUnit XML report. + verbose = runtests.verbose + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout + + if timeout is not None and threading_helper.can_start_thread: + use_timeout = True + faulthandler.dump_traceback_later(timeout, exit=True) + else: + use_timeout = False + + try: + setup_tests(runtests) + + if output_on_failure or runtests.pgo: + support.verbose = True + + stream = io.StringIO() + orig_stdout = sys.stdout + orig_stderr = sys.stderr + print_warning = support.print_warning + orig_print_warnings_stderr = print_warning.orig_stderr + + output = None + try: + sys.stdout = stream + sys.stderr = stream + # print_warning() writes into the temporary stream to preserve + # messages order. If support.environment_altered becomes true, + # warnings will be written to sys.stderr below. + print_warning.orig_stderr = stream + + _runtest_env_changed_exc(result, runtests, display_failure=False) + # Ignore output if the test passed successfully + if result.state != State.PASSED: + output = stream.getvalue() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + print_warning.orig_stderr = orig_print_warnings_stderr + + if output is not None: + sys.stderr.write(output) + sys.stderr.flush() + else: + # Tell tests to be moderately quiet + support.verbose = verbose + _runtest_env_changed_exc(result, runtests, + display_failure=not verbose) + + xml_list = support.junit_xml_list + if xml_list: + result.xml_data = xml_list + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + support.junit_xml_list = None + + +def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: + """Run a single test. + + test_name -- the name of the test + + Returns a TestResult. + + If runtests.use_junit, xml_data is a list containing each generated + testsuite element. + """ + ansi = get_colors(file=sys.stderr) + red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW + + start_time = time.perf_counter() + result = TestResult(test_name) + pgo = runtests.pgo + try: + _runtest(result, runtests) + except: + if not pgo: + msg = traceback.format_exc() + print(f"{red}test {test_name} crashed -- {msg}{reset}", + file=sys.stderr, flush=True) + result.state = State.UNCAUGHT_EXC + + sys.stdout.flush() + sys.stderr.flush() + + result.duration = time.perf_counter() - start_time + return result diff --git a/Lib/test/support/testresult.py b/Lib/test/libregrtest/testresult.py similarity index 94% rename from Lib/test/support/testresult.py rename to Lib/test/libregrtest/testresult.py index de23fdd59de..1820f354572 100644 --- a/Lib/test/support/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,6 +9,7 @@ import traceback import unittest from test import support +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args): if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-out').text = stdout + ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-err').text = stderr + ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue + e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: - e2.set(k2, str(v2)) + e2.set(k2, sanitize_xml(str(v2))) else: - e2.text = str(v2) + e2.text = sanitize_xml(str(v2)) else: - e2.text = str(v) + e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000..d984a735bdf --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,51 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + 'test_asyncio', + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', + 'test_code', + 'test_ctypes', + # 'test_concurrent_futures', # gh-130605: too many data races + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_opcache', + 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_thread_local_bytecode', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_weakref', + 'test_free_threading', +] + +# Tests that should be run with `--parallel-threads=N` under TSAN. These tests +# typically do not use threads, but are run multiple times in parallel by +# the regression test runner with the `--parallel-threads` option enabled. +TSAN_PARALLEL_TESTS = [ + 'test_abc', + 'test_hashlib', +] + + +def setup_tsan_tests(cmdline_args) -> None: + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] + +def setup_tsan_parallel_tests(cmdline_args) -> None: + if not cmdline_args: + cmdline_args[:] = TSAN_PARALLEL_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fb9971a64f6..d94fb84a743 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,10 +1,63 @@ +import contextlib +import faulthandler +import locale import math import os.path +import platform +import random +import re +import shlex +import subprocess import sys +import sysconfig +import tempfile import textwrap +from collections.abc import Callable, Iterable +from test import support +from test.support import os_helper +from test.support import threading_helper -def format_duration(seconds): + +# All temporary files and temporary directories created by libregrtest should +# use TMP_PREFIX so cleanup_temp_dir() can remove them all. +TMP_PREFIX = 'test_python_' +WORK_DIR_PREFIX = TMP_PREFIX +WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_' + +# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). +# Used to protect against threading._shutdown() hang. +# Must be smaller than buildbot "1200 seconds without output" limit. +EXIT_TIMEOUT = 120.0 + + +ALL_RESOURCES = ('audio', 'console', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') + +# Other resources excluded from --use=all: +# +# - extralagefile (ex: test_zipfile64): really too slow to be enabled +# "by default" +# - tzdata: while needed to validate fully test_datetime, it makes +# test_datetime too slow (15-20 min on some buildbots) and so is disabled by +# default (see bpo-30822). +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') + + +# Types for types hints +StrPath = str +TestName = str +StrJSON = str +TestTuple = tuple[TestName, ...] +TestList = list[TestName] +# --match and --ignore options: list of patterns +# ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] +FilterTuple = tuple[TestName, ...] +FilterDict = dict[TestName, FilterTuple] + + +def format_duration(seconds: float) -> str: ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) minutes, seconds = divmod(seconds, 60) @@ -16,17 +69,20 @@ def format_duration(seconds): if minutes: parts.append('%s min' % minutes) if seconds: - parts.append('%s sec' % seconds) - if ms: - parts.append('%s ms' % ms) + if parts: + # 2 min 1 sec + parts.append('%s sec' % seconds) + else: + # 1.0 sec + parts.append('%.1f sec' % (seconds + ms / 1000)) if not parts: - return '0 ms' + return '%s ms' % ms parts = parts[:2] return ' '.join(parts) -def removepy(names): +def strip_py_suffix(names: list[str] | None) -> None: if not names: return for idx, name in enumerate(names): @@ -35,11 +91,20 @@ def removepy(names): names[idx] = basename -def count(n, word): +def plural(n: int, singular: str, plural: str | None = None) -> str: if n == 1: - return "%d %s" % (n, word) + return singular + elif plural is not None: + return plural else: - return "%d %ss" % (n, word) + return singular + 's' + + +def count(n: int, word: str) -> str: + if n == 1: + return f"{n} {word}" + else: + return f"{n} {word}s" def printlist(x, width=70, indent=4, file=None): @@ -57,5 +122,605 @@ def printlist(x, width=70, indent=4, file=None): file=file) -def print_warning(msg): - print(f"Warning -- {msg}", file=sys.stderr, flush=True) +def print_warning(msg: str) -> None: + support.print_warning(msg) + + +orig_unraisablehook: Callable[..., None] | None = None + + +def regrtest_unraisable_hook(unraisable) -> None: + global orig_unraisablehook + support.environment_altered = True + support.print_warning("Unraisable exception") + old_stderr = sys.stderr + try: + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr + assert orig_unraisablehook is not None, "orig_unraisablehook not set" + orig_unraisablehook(unraisable) + sys.stderr.flush() + finally: + sys.stderr = old_stderr + + +def setup_unraisable_hook() -> None: + global orig_unraisablehook + orig_unraisablehook = sys.unraisablehook + sys.unraisablehook = regrtest_unraisable_hook + + +orig_threading_excepthook: Callable[..., None] | None = None + + +def regrtest_threading_excepthook(args) -> None: + global orig_threading_excepthook + support.environment_altered = True + support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") + old_stderr = sys.stderr + try: + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr + assert orig_threading_excepthook is not None, "orig_threading_excepthook not set" + orig_threading_excepthook(args) + sys.stderr.flush() + finally: + sys.stderr = old_stderr + + +def setup_threading_excepthook() -> None: + global orig_threading_excepthook + import threading + orig_threading_excepthook = threading.excepthook + threading.excepthook = regrtest_threading_excepthook + + +def clear_caches(): + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + try: + re = sys.modules['re'] + except KeyError: + pass + else: + re.purge() + + try: + _strptime = sys.modules['_strptime'] + except KeyError: + pass + else: + _strptime._regex_cache.clear() + + try: + urllib_parse = sys.modules['urllib.parse'] + except KeyError: + pass + else: + urllib_parse.clear_cache() + + try: + urllib_request = sys.modules['urllib.request'] + except KeyError: + pass + else: + urllib_request.urlcleanup() + + try: + linecache = sys.modules['linecache'] + except KeyError: + pass + else: + linecache.clearcache() + + try: + mimetypes = sys.modules['mimetypes'] + except KeyError: + pass + else: + mimetypes._default_mime_types() + + try: + filecmp = sys.modules['filecmp'] + except KeyError: + pass + else: + filecmp._cache.clear() + + try: + struct = sys.modules['struct'] + except KeyError: + pass + else: + struct._clearcache() + + try: + doctest = sys.modules['doctest'] + except KeyError: + pass + else: + doctest.master = None + + try: + ctypes = sys.modules['ctypes'] + except KeyError: + pass + else: + ctypes._reset_cache() + + try: + typing = sys.modules['typing'] + except KeyError: + pass + else: + for f in typing._cleanups: + f() + + import inspect + abs_classes = filter(inspect.isabstract, typing.__dict__.values()) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + obj._abc_caches_clear() + + try: + fractions = sys.modules['fractions'] + except KeyError: + pass + else: + fractions._hash_algorithm.cache_clear() + + try: + inspect = sys.modules['inspect'] + except KeyError: + pass + else: + inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() + + +def get_build_info(): + # Get most important configure and build options as a list of strings. + # Example: ['debug', 'ASAN+MSAN'] or ['release', 'LTO+PGO']. + + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('PY_CFLAGS') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') + ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' + + build = [] + + # --disable-gil + if sysconfig.get_config_var('Py_GIL_DISABLED'): + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) + + if hasattr(sys, 'gettotalrefcount'): + # --with-pydebug + build.append('debug') + + if '-DNDEBUG' in cflags: + build.append('without_assert') + else: + build.append('release') + + if '--with-assertions' in config_args: + build.append('with_assert') + elif '-DNDEBUG' not in cflags: + build.append('with_assert') + + # --enable-experimental-jit + if sys._jit.is_available(): + if sys._jit.is_enabled(): + build.append("JIT") + else: + build.append("JIT (disabled)") + + # --enable-framework=name + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + if framework: + build.append(f'framework={framework}') + + # --enable-shared + shared = int(sysconfig.get_config_var('PY_ENABLE_SHARED') or '0') + if shared: + build.append('shared') + + # --with-lto + optimizations = [] + if '-flto=thin' in ldflags_nodist: + optimizations.append('ThinLTO') + elif '-flto' in ldflags_nodist: + optimizations.append('LTO') + + if support.check_cflags_pgo(): + # PGO (--enable-optimizations) + optimizations.append('PGO') + + if support.check_bolt_optimized(): + # BOLT (--enable-bolt) + optimizations.append('BOLT') + + if optimizations: + build.append('+'.join(optimizations)) + + # --with-address-sanitizer + sanitizers = [] + if support.check_sanitizer(address=True): + sanitizers.append("ASAN") + # --with-memory-sanitizer + if support.check_sanitizer(memory=True): + sanitizers.append("MSAN") + # --with-undefined-behavior-sanitizer + if support.check_sanitizer(ub=True): + sanitizers.append("UBSAN") + # --with-thread-sanitizer + if support.check_sanitizer(thread=True): + sanitizers.append("TSAN") + if sanitizers: + build.append('+'.join(sanitizers)) + + # --with-trace-refs + if hasattr(sys, 'getobjects'): + build.append("TraceRefs") + # --enable-pystats + if hasattr(sys, '_stats_on'): + build.append("pystats") + # --with-valgrind + if sysconfig.get_config_var('WITH_VALGRIND'): + build.append("valgrind") + # --with-dtrace + if sysconfig.get_config_var('WITH_DTRACE'): + build.append("dtrace") + + return build + + +def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: + if tmp_dir: + tmp_dir = os.path.expanduser(tmp_dir) + else: + # When tests are run from the Python build directory, it is best practice + # to keep the test files in a subfolder. This eases the cleanup of leftover + # files using the "make distclean" command. + if sysconfig.is_python_build(): + if not support.is_wasi: + tmp_dir = sysconfig.get_config_var('abs_builddir') + if tmp_dir is None: + tmp_dir = sysconfig.get_config_var('abs_srcdir') + if not tmp_dir: + # gh-74470: On Windows, only srcdir is available. Using + # abs_builddir mostly matters on UNIX when building + # Python out of the source tree, especially when the + # source tree is read only. + tmp_dir = sysconfig.get_config_var('srcdir') + if not tmp_dir: + raise RuntimeError( + "Could not determine the correct value for tmp_dir" + ) + tmp_dir = os.path.join(tmp_dir, 'build') + else: + # WASI platform + tmp_dir = sysconfig.get_config_var('projectbase') + if not tmp_dir: + raise RuntimeError( + "sysconfig.get_config_var('projectbase') " + f"unexpectedly returned {tmp_dir!r} on WASI" + ) + tmp_dir = os.path.join(tmp_dir, 'build') + + # When get_temp_dir() is called in a worker process, + # get_temp_dir() path is different than in the parent process + # which is not a WASI process. So the parent does not create + # the same "tmp_dir" than the test worker process. + os.makedirs(tmp_dir, exist_ok=True) + else: + tmp_dir = tempfile.gettempdir() + + return os.path.abspath(tmp_dir) + + +def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + # Emscripten and WASI have stubbed getpid(), Emscripten has only + # millisecond clock resolution. Use randint() instead. + if support.is_emscripten or support.is_wasi: + nounce = random.randint(0, 1_000_000) + else: + nounce = os.getpid() + + if worker: + work_dir = WORK_DIR_PREFIX + str(nounce) + else: + work_dir = WORKER_WORK_DIR_PREFIX + str(nounce) + work_dir += os_helper.FS_NONASCII + work_dir = os.path.join(parent_dir, work_dir) + return work_dir + + +@contextlib.contextmanager +def exit_timeout(): + try: + yield + except SystemExit as exc: + # bpo-38203: Python can hang at exit in Py_Finalize(), especially + # on threading._shutdown() call: put a timeout + if threading_helper.can_start_thread: + faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) + sys.exit(exc.code) + + +def remove_testfn(test_name: TestName, verbose: int) -> None: + # Try to clean up os_helper.TESTFN if left behind. + # + # While tests shouldn't leave any files or directories behind, when a test + # fails that can be tedious for it to arrange. The consequences can be + # especially nasty on Windows, since if a test leaves a file open, it + # cannot be deleted by name (while there's nothing we can do about that + # here either, we can display the name of the offending test, which is a + # real help). + name = os_helper.TESTFN + if not os.path.exists(name): + return + + nuker: Callable[[str], None] + if os.path.isdir(name): + import shutil + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise RuntimeError(f"os.path says {name!r} exists but is neither " + f"directory nor file") + + if verbose: + print_warning(f"{test_name} left behind {kind} {name!r}") + support.environment_altered = True + + try: + import stat + # fix possible permissions problems that might prevent cleanup + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as exc: + print_warning(f"{test_name} left behind {kind} {name!r} " + f"and it couldn't be removed: {exc}") + + +def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: + if test_name.startswith('test.') or test_dir: + return test_name + else: + # Import it from the test package + return 'test.' + test_name + + +# gh-90681: When rerunning tests, we might need to rerun the whole +# class or module suite if some its life-cycle hooks fail. +# Test level hooks are not affected. +_TEST_LIFECYCLE_HOOKS = frozenset(( + 'setUpClass', 'tearDownClass', + 'setUpModule', 'tearDownModule', +)) + +def normalize_test_name(test_full_name: str, *, + is_error: bool = False) -> str | None: + short_name = test_full_name.split(" ")[0] + if is_error and short_name in _TEST_LIFECYCLE_HOOKS: + if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): + # if setUpModule() or tearDownModule() failed, don't filter + # tests with the test file name, don't use filters. + return None + + # This means that we have a failure in a life-cycle hook, + # we need to rerun the whole module or class suite. + # Basically the error looks like this: + # ERROR: setUpClass (test.test_reg_ex.RegTest) + # or + # ERROR: setUpModule (test.test_reg_ex) + # So, we need to parse the class / module name. + lpar = test_full_name.index('(') + rpar = test_full_name.index(')') + return test_full_name[lpar + 1: rpar].split('.')[-1] + return short_name + + +def adjust_rlimit_nofile() -> None: + """ + On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) + for our test suite to succeed. Raise it to something more reasonable. 1024 + is a common Linux default. + """ + try: + import resource + except ImportError: + return + + fd_limit, max_fds = resource.getrlimit(resource.RLIMIT_NOFILE) + + desired_fds = 1024 + + if fd_limit < desired_fds and fd_limit < max_fds: + new_fd_limit = min(desired_fds, max_fds) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (new_fd_limit, max_fds)) + print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") + except (ValueError, OSError) as err: + print_warning(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " + f"{new_fd_limit}: {err}.") + + +def get_host_runner() -> str: + if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: + hostrunner = sysconfig.get_config_var("HOSTRUNNER") + return hostrunner + + +def is_cross_compiled() -> bool: + return ('_PYTHON_HOST_PLATFORM' in os.environ) + + +def format_resources(use_resources: Iterable[str]) -> str: + use_resources = set(use_resources) + all_resources = set(ALL_RESOURCES) + + # Express resources relative to "all" + relative_all = ['all'] + for name in sorted(all_resources - use_resources): + relative_all.append(f'-{name}') + for name in sorted(use_resources - all_resources): + relative_all.append(f'{name}') + all_text = ','.join(relative_all) + all_text = f"resources: {all_text}" + + # List of enabled resources + text = ','.join(sorted(use_resources)) + text = f"resources ({len(use_resources)}): {text}" + + # Pick the shortest string (prefer relative to all if lengths are equal) + if len(all_text) <= len(text): + return all_text + else: + return text + + +def display_header(use_resources: tuple[str, ...], + python_cmd: tuple[str, ...] | None) -> None: + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) + print("== cwd:", os.getcwd()) + + cpu_count: object = os.cpu_count() + if cpu_count: + # The function is new in Python 3.13; mypy doesn't know about it yet: + process_cpu_count = os.process_cpu_count() # type: ignore[attr-defined] + if process_cpu_count and process_cpu_count != cpu_count: + cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" + print("== CPU count:", cpu_count) + print("== encodings: locale=%s FS=%s" + % (locale.getencoding(), sys.getfilesystemencoding())) + + if use_resources: + text = format_resources(use_resources) + print(f"== {text}") + else: + print("== resources: all test resources are disabled, " + "use -u option to unskip tests") + + cross_compile = is_cross_compiled() + if cross_compile: + print("== cross compiled: Yes") + if python_cmd: + cmd = shlex.join(python_cmd) + print(f"== host python: {cmd}") + + get_cmd = [*python_cmd, '-m', 'platform'] + proc = subprocess.run( + get_cmd, + stdout=subprocess.PIPE, + text=True, + cwd=os_helper.SAVEDCWD) + stdout = proc.stdout.replace('\n', ' ').strip() + if stdout: + print(f"== host platform: {stdout}") + elif proc.returncode: + print(f"== host platform: ") + else: + hostrunner = get_host_runner() + if hostrunner: + print(f"== host runner: {hostrunner}") + + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. + asan = support.check_sanitizer(address=True) + msan = support.check_sanitizer(memory=True) + ubsan = support.check_sanitizer(ub=True) + tsan = support.check_sanitizer(thread=True) + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if tsan: + sanitizers.append("thread") + if sanitizers: + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + (tsan, "TSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") + + print(flush=True) + + +def cleanup_temp_dir(tmp_dir: StrPath) -> None: + import glob + + path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') + print("Cleanup %s directory" % tmp_dir) + for name in glob.glob(path): + if os.path.isdir(name): + print("Remove directory: %s" % name) + os_helper.rmtree(name) + else: + print("Remove file: %s" % name) + os_helper.unlink(name) + + +ILLEGAL_XML_CHARS_RE = re.compile( + '[' + # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal + '\x00-\x08\x0B\x0C\x0E-\x1F' + # Surrogate characters + '\uD800-\uDFFF' + # Special Unicode characters + '\uFFFE' + '\uFFFF' + # Match multiple sequential invalid characters for better efficiency + ']+') + +def _sanitize_xml_replace(regs): + text = regs[0] + return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] + for ch in text) + +def sanitize_xml(text: str) -> str: + return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 95db3def36f..b51fde0af57 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -1,105 +1,128 @@ +import _overlapped +import _thread import _winapi -import msvcrt -import os -import subprocess -import uuid -from test import support +import math +import struct +import winreg -# Max size of asynchronous reads -BUFSIZE = 8192 -# Exponential damping factor (see below) -LOAD_FACTOR_1 = 0.9200444146293232478931553241 # Seconds per measurement -SAMPLING_INTERVAL = 5 -COUNTER_NAME = r'\System\Processor Queue Length' +SAMPLING_INTERVAL = 1 +# Exponential damping factor to compute exponentially weighted moving average +# on 1 minute (60 seconds) +LOAD_FACTOR_1 = 1 / math.exp(SAMPLING_INTERVAL / 60) +# Initialize the load using the arithmetic mean of the first NVALUE values +# of the Processor Queue Length +NVALUE = 5 class WindowsLoadTracker(): """ - This class asynchronously interacts with the `typeperf` command to read - the system load on Windows. Mulitprocessing and threads can't be used - here because they interfere with the test suite's cases for those - modules. + This class asynchronously reads the performance counters to calculate + the system load on Windows. A "raw" thread is used here to prevent + interference with the test suite's cases for the threading module. """ def __init__(self): - self.load = 0.0 - self.start() - - def start(self): - # Create a named pipe which allows for asynchronous IO in Windows - pipe_name = r'\\.\pipe\typeperf_output_' + str(uuid.uuid4()) - - open_mode = _winapi.PIPE_ACCESS_INBOUND - open_mode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE - open_mode |= _winapi.FILE_FLAG_OVERLAPPED - - # This is the read end of the pipe, where we will be grabbing output - self.pipe = _winapi.CreateNamedPipe( - pipe_name, open_mode, _winapi.PIPE_WAIT, - 1, BUFSIZE, BUFSIZE, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL - ) - # The write end of the pipe which is passed to the created process - pipe_write_end = _winapi.CreateFile( - pipe_name, _winapi.GENERIC_WRITE, 0, _winapi.NULL, - _winapi.OPEN_EXISTING, 0, _winapi.NULL - ) - # Open up the handle as a python file object so we can pass it to - # subprocess - command_stdout = msvcrt.open_osfhandle(pipe_write_end, 0) - - # Connect to the read end of the pipe in overlap/async mode - overlap = _winapi.ConnectNamedPipe(self.pipe, overlapped=True) - overlap.GetOverlappedResult(True) - - # Spawn off the load monitor - command = ['typeperf', COUNTER_NAME, '-si', str(SAMPLING_INTERVAL)] - self.p = subprocess.Popen(command, stdout=command_stdout, cwd=os_helper.SAVEDCWD) - - # Close our copy of the write end of the pipe - os.close(command_stdout) - - def close(self): - if self.p is None: + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + + # Pre-flight test for access to the performance data; + # `PermissionError` will be raised if not allowed + winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) + + self._values = [] + self._load = None + self._running = _overlapped.CreateEvent(None, True, False, None) + self._stopped = _overlapped.CreateEvent(None, True, False, None) + + _thread.start_new_thread(self._update_load, (), {}) + + def _update_load(self, + # localize module access to prevent shutdown errors + _wait=_winapi.WaitForSingleObject, + _signal=_overlapped.SetEvent): + # run until signaled to stop + while _wait(self._running, 1000): + self._calculate_load() + # notify stopped + _signal(self._stopped) + + def _calculate_load(self, + # localize module access to prevent shutdown errors + _query=winreg.QueryValueEx, + _hkey=winreg.HKEY_PERFORMANCE_DATA, + _unpack=struct.unpack_from): + # get the 'System' object + data, _ = _query(_hkey, '2') + # PERF_DATA_BLOCK { + # WCHAR Signature[4] 8 + + # DWOWD LittleEndian 4 + + # DWORD Version 4 + + # DWORD Revision 4 + + # DWORD TotalByteLength 4 + + # DWORD HeaderLength = 24 byte offset + # ... + # } + obj_start, = _unpack('L', data, 24) + # PERF_OBJECT_TYPE { + # DWORD TotalByteLength + # DWORD DefinitionLength + # DWORD HeaderLength + # ... + # } + data_start, defn_start = _unpack('4xLL', data, obj_start) + data_base = obj_start + data_start + defn_base = obj_start + defn_start + # find the 'Processor Queue Length' counter (index=44) + while defn_base < data_base: + # PERF_COUNTER_DEFINITION { + # DWORD ByteLength + # DWORD CounterNameTitleIndex + # ... [7 DWORDs/28 bytes] + # DWORD CounterOffset + # } + size, idx, offset = _unpack('LL28xL', data, defn_base) + defn_base += size + if idx == 44: + counter_offset = data_base + offset + # the counter is known to be PERF_COUNTER_RAWCOUNT (DWORD) + processor_queue_length, = _unpack('L', data, counter_offset) + break + else: return - self.p.kill() - self.p.wait() - self.p = None - def __del__(self): - self.close() - - def read_output(self): - import _winapi - - overlapped, _ = _winapi.ReadFile(self.pipe, BUFSIZE, True) - bytes_read, res = overlapped.GetOverlappedResult(False) - if res != 0: - return - - return overlapped.getbuffer().decode() + # We use an exponentially weighted moving average, imitating the + # load calculation on Unix systems. + # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation + # https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + if self._load is not None: + self._load = (self._load * LOAD_FACTOR_1 + + processor_queue_length * (1.0 - LOAD_FACTOR_1)) + elif len(self._values) < NVALUE: + self._values.append(processor_queue_length) + else: + self._load = sum(self._values) / len(self._values) + + def close(self, kill=True): + self.__del__() + return + + def __del__(self, + # localize module access to prevent shutdown errors + _wait=_winapi.WaitForSingleObject, + _close=_winapi.CloseHandle, + _signal=_overlapped.SetEvent): + if self._running is not None: + # tell the update thread to quit + _signal(self._running) + # wait for the update thread to signal done + _wait(self._stopped, -1) + # cleanup events + _close(self._running) + _close(self._stopped) + self._running = self._stopped = None def getloadavg(self): - typeperf_output = self.read_output() - # Nothing to update, just return the current load - if not typeperf_output: - return self.load - - # Process the backlog of load values - for line in typeperf_output.splitlines(): - # typeperf outputs in a CSV format like this: - # "07/19/2018 01:32:26.605","3.000000" - toks = line.split(',') - # Ignore blank lines and the initial header - if line.strip() == '' or (COUNTER_NAME in line) or len(toks) != 2: - continue - - load = float(toks[1].replace('"', '')) - # We use an exponentially weighted moving average, imitating the - # load calculation on Unix systems. - # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation - new_load = self.load * LOAD_FACTOR_1 + load * (1.0 - LOAD_FACTOR_1) - self.load = new_load - - return self.load + return self._load diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py new file mode 100644 index 00000000000..1ad67e1cebf --- /dev/null +++ b/Lib/test/libregrtest/worker.py @@ -0,0 +1,138 @@ +import subprocess +import sys +import os +from _colorize import can_colorize # type: ignore[import-not-found] +from typing import Any, NoReturn + +from test.support import os_helper, Py_DEBUG + +from .setup import setup_process, setup_test_dir +from .runtests import WorkerRunTests, JsonFile, JsonFileType +from .single import run_single_test +from .utils import ( + StrPath, StrJSON, TestFilter, + get_temp_dir, get_work_dir, exit_timeout) + + +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = { + 'test_ioctl', +} + + +def create_worker_process(runtests: WorkerRunTests, output_fd: int, + tmp_dir: StrPath | None = None) -> subprocess.Popen[str]: + worker_json = runtests.as_json() + + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) + + env = dict(os.environ) + if tmp_dir is not None: + env['TMPDIR'] = tmp_dir + env['TEMP'] = tmp_dir + env['TMP'] = tmp_dir + + # The subcommand is run with a temporary output which means it is not a TTY + # and won't auto-color. The test results are printed to stdout so if we can + # color that have the subprocess use color. + if can_colorize(file=sys.stdout): + env['FORCE_COLOR'] = '1' + + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + # + # Emscripten and WASI Python must start in the Python source code directory + # to get 'python.js' or 'python.wasm' file. Then worker_process() changes + # to a temporary directory created to run tests. + work_dir = os_helper.SAVEDCWD + + kwargs: dict[str, Any] = dict( + env=env, + stdout=output_fd, + # bpo-45410: Write stderr into stdout to keep messages order + stderr=output_fd, + text=True, + close_fds=True, + cwd=work_dir, + ) + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: + kwargs['start_new_session'] = True + + # Include the test name in the TSAN log file name + if 'TSAN_OPTIONS' in env: + parts = env['TSAN_OPTIONS'].split(' ') + for i, part in enumerate(parts): + if part.startswith('log_path='): + parts[i] = f'{part}.{test_name}' + break + env['TSAN_OPTIONS'] = ' '.join(parts) + + # Pass json_file to the worker process + json_file = runtests.json_file + json_file.configure_subprocess(kwargs) + + with json_file.inherit_subprocess(): + return subprocess.Popen(cmd, **kwargs) + + +def worker_process(worker_json: StrJSON) -> NoReturn: + runtests = WorkerRunTests.from_json(worker_json) + test_name = runtests.tests[0] + match_tests: TestFilter = runtests.match_tests + json_file: JsonFile = runtests.json_file + + setup_test_dir(runtests.test_dir) + setup_process() + + if runtests.rerun: + if match_tests: + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) + print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) + else: + print(f"Re-running {test_name} in verbose mode", flush=True) + + result = run_single_test(test_name, runtests) + if runtests.coverage: + if "test.cov" in sys.modules: # imported by -Xpresite= + result.covered_lines = list(sys.modules["test.cov"].coverage) + elif not Py_DEBUG: + print( + "Gathering coverage in worker processes requires --with-pydebug", + flush=True, + ) + else: + raise LookupError( + "`test.cov` not found in sys.modules but coverage wanted" + ) + + if json_file.file_type == JsonFileType.STDOUT: + print() + result.write_json_into(sys.stdout) + else: + with json_file.open('w', encoding='utf-8') as json_fp: + result.write_json_into(json_fp) + + sys.exit(0) + + +def main() -> NoReturn: + if len(sys.argv) != 2: + print("usage: python -m test.libregrtest.worker JSON") + sys.exit(1) + worker_json = sys.argv[1] + + tmp_dir = get_temp_dir() + work_dir = get_work_dir(tmp_dir, worker=True) + + with exit_timeout(): + with os_helper.temp_cwd(work_dir, quiet=True): + worker_process(worker_json) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 65dfa41b26e..e76f79c274e 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,8 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit +from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -59,9 +60,11 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) - for i in range(get_c_recursion_limit() + 1): + for i in range(200_000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 09b91147801..8c7a4f76563 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -2,6 +2,7 @@ Various tests for synchronization primitives. """ +import gc import sys import time from _thread import start_new_thread, TIMEOUT_MAX @@ -13,54 +14,79 @@ from test.support import threading_helper -def _wait(): - # A crude wait/yield function not relying on synchronization primitives. - time.sleep(0.01) +requires_fork = unittest.skipUnless(support.has_fork_support, + "platform doesn't support fork " + "(no _at_fork_reinit method)") + + +def wait_threads_blocked(nthread): + # Arbitrary sleep to wait until N threads are blocked, + # like waiting for a lock. + time.sleep(0.010 * nthread) + class Bunch(object): """ A bunch of threads. """ - def __init__(self, f, n, wait_before_exit=False): + def __init__(self, func, nthread, wait_before_exit=False): """ - Construct a bunch of `n` threads running the same function `f`. + Construct a bunch of `nthread` threads running the same function `func`. If `wait_before_exit` is True, the threads won't terminate until do_finish() is called. """ - self.f = f - self.n = n + self.func = func + self.nthread = nthread self.started = [] self.finished = [] + self.exceptions = [] self._can_exit = not wait_before_exit - self.wait_thread = threading_helper.wait_threads_exit() - self.wait_thread.__enter__() + self._wait_thread = None - def task(): - tid = threading.get_ident() - self.started.append(tid) - try: - f() - finally: - self.finished.append(tid) - while not self._can_exit: - _wait() + def task(self): + tid = threading.get_ident() + self.started.append(tid) + try: + self.func() + except BaseException as exc: + self.exceptions.append(exc) + finally: + self.finished.append(tid) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self._can_exit: + break + + def __enter__(self): + self._wait_thread = threading_helper.wait_threads_exit(support.SHORT_TIMEOUT) + self._wait_thread.__enter__() try: - for i in range(n): - start_new_thread(task, ()) + for _ in range(self.nthread): + start_new_thread(self.task, ()) except: self._can_exit = True raise - def wait_for_started(self): - while len(self.started) < self.n: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.started) >= self.nthread: + break + + return self - def wait_for_finished(self): - while len(self.finished) < self.n: - _wait() - # Wait for threads exit - self.wait_thread.__exit__(None, None, None) + def __exit__(self, exc_type, exc_value, traceback): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.finished) >= self.nthread: + break + + # Wait until threads completely exit according to _thread._count() + self._wait_thread.__exit__(None, None, None) + + # Break reference cycle + exceptions = self.exceptions + self.exceptions = None + if exceptions: + raise ExceptionGroup(f"{self.func} threads raised exceptions", + exceptions) def do_finish(self): self._can_exit = True @@ -88,6 +114,12 @@ class BaseLockTests(BaseTestCase): Tests for both recursive and non-recursive locks. """ + def wait_phase(self, phase, expected): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(phase) >= expected: + break + self.assertEqual(len(phase), expected) + def test_constructor(self): lock = self.locktype() del lock @@ -125,44 +157,60 @@ def test_try_acquire_contended(self): result = [] def f(): result.append(lock.acquire(False)) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(result[0]) lock.release() - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_acquire_contended(self): lock = self.locktype() lock.acquire() - N = 5 def f(): lock.acquire() lock.release() - b = Bunch(f, N) - b.wait_for_started() - _wait() - self.assertEqual(len(b.finished), 0) - lock.release() - b.wait_for_finished() - self.assertEqual(len(b.finished), N) + N = 5 + with Bunch(f, N) as bunch: + # Threads block on lock.acquire() + wait_threads_blocked(N) + self.assertEqual(len(bunch.finished), 0) + + # Threads unblocked + lock.release() + + self.assertEqual(len(bunch.finished), N) def test_with(self): lock = self.locktype() def f(): lock.acquire() lock.release() - def _with(err=None): + + def with_lock(err=None): with lock: if err is not None: raise err - _with() - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() - self.assertRaises(TypeError, _with, TypeError) - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + # Acquire the lock, do nothing, with releases the lock + with lock: + pass + + # Check that the lock is unacquired + with Bunch(f, 1): + pass + + # Acquire the lock, raise an exception, with releases the lock + with self.assertRaises(TypeError): + with lock: + raise TypeError + + # Check that the lock is unacquired even if after an exception + # was raised in the previous "with lock:" block + with Bunch(f, 1): + pass + + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign # (non-threading) thread. @@ -170,22 +218,16 @@ def test_thread_leak(self): def f(): lock.acquire() lock.release() - n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't # be recycled. - Bunch(f, 15).wait_for_finished() - if len(threading.enumerate()) != n: - # There is a small window during which a Thread instance's - # target function has finished running, but the Thread is still - # alive and registered. Avoid spurious failures by waiting a - # bit more (seen on a buildbot). - time.sleep(0.4) - self.assertEqual(n, len(threading.enumerate())) + with Bunch(f, 15): + pass def test_timeout(self): lock = self.locktype() # Can't set timeout if not blocking - self.assertRaises(ValueError, lock.acquire, 0, 1) + self.assertRaises(ValueError, lock.acquire, False, 1) # Invalid timeout values self.assertRaises(ValueError, lock.acquire, timeout=-100) self.assertRaises(OverflowError, lock.acquire, timeout=1e100) @@ -204,7 +246,8 @@ def f(): results.append(lock.acquire(timeout=0.5)) t2 = time.monotonic() results.append(t2 - t1) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) @@ -217,6 +260,7 @@ def test_weakref_deleted(self): lock = self.locktype() ref = weakref.ref(lock) del lock + gc.collect() # For PyPy or other GCs. self.assertIsNone(ref()) @@ -237,15 +281,13 @@ def f(): phase.append(None) with threading_helper.wait_threads_exit(): + # Thread blocked on lock.acquire() start_new_thread(f, ()) - while len(phase) == 0: - _wait() - _wait() - self.assertEqual(len(phase), 1) + self.wait_phase(phase, 1) + + # Thread unblocked lock.release() - while len(phase) == 1: - _wait() - self.assertEqual(len(phase), 2) + self.wait_phase(phase, 2) def test_different_thread(self): # Lock can be released from a different thread. @@ -253,8 +295,8 @@ def test_different_thread(self): lock.acquire() def f(): lock.release() - b = Bunch(f, 1) - b.wait_for_finished() + with Bunch(f, 1): + pass lock.acquire() lock.release() @@ -268,11 +310,50 @@ def test_state_after_timeout(self): self.assertFalse(lock.locked()) self.assertTrue(lock.acquire(blocking=False)) + @requires_fork + def test_at_fork_reinit(self): + def use_lock(lock): + # make sure that the lock still works normally + # after _at_fork_reinit() + lock.acquire() + lock.release() + + # unlocked + lock = self.locktype() + lock._at_fork_reinit() + use_lock(lock) + + # locked: _at_fork_reinit() resets the lock to the unlocked state + lock2 = self.locktype() + lock2.acquire() + lock2._at_fork_reinit() + use_lock(lock2) + class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("") + evt.set() + self.assertRegex(repr(evt), r"<\w+\.Event at .*: set>") + class ConditionTests(BaseTestCase): """ @@ -466,15 +631,14 @@ def _check_notify(self, cond): # Note that this test is sensitive to timing. If the worker threads # don't execute in a timely fashion, the main thread may think they # are further along then they are. The main thread therefore issues - # _wait() statements to try to make sure that it doesn't race ahead - # of the workers. + # wait_threads_blocked() statements to try to make sure that it doesn't + # race ahead of the workers. # Secondly, this test assumes that condition variables are not subject # to spurious wakeups. The absence of spurious wakeups is an implementation # detail of Condition Variables in current CPython, but in general, not # a guaranteed property of condition variables as a programming # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. - N = 5 ready = [] results1 = [] results2 = [] @@ -483,58 +647,83 @@ def f(): cond.acquire() ready.append(phase_num) result = cond.wait() + cond.release() results1.append((result, phase_num)) + cond.acquire() ready.append(phase_num) + result = cond.wait() cond.release() results2.append((result, phase_num)) - b = Bunch(f, N) - b.wait_for_started() - # first wait, to ensure all workers settle into cond.wait() before - # we continue. See issues #8799 and #30727. - while len(ready) < 5: - _wait() - ready.clear() - self.assertEqual(results1, []) - # Notify 3 threads at first - cond.acquire() - cond.notify(3) - _wait() - phase_num = 1 - cond.release() - while len(results1) < 3: - _wait() - self.assertEqual(results1, [(True, 1)] * 3) - self.assertEqual(results2, []) - # make sure all awaken workers settle into cond.wait() - while len(ready) < 3: - _wait() - # Notify 5 threads: they might be in their first or second wait - cond.acquire() - cond.notify(5) - _wait() - phase_num = 2 - cond.release() - while len(results1) + len(results2) < 8: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3) - # make sure all workers settle into cond.wait() - while len(ready) < 5: - _wait() - # Notify all threads: they are all in their second wait - cond.acquire() - cond.notify_all() - _wait() - phase_num = 3 - cond.release() - while len(results2) < 5: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True,2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3 + [(True, 3)] * 2) - b.wait_for_finished() + + N = 5 + with Bunch(f, N): + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issues #8799 and #30727. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + + ready.clear() + self.assertEqual(results1, []) + + # Notify 3 threads at first + count1 = 3 + cond.acquire() + cond.notify(count1) + wait_threads_blocked(count1) + + # Phase 1 + phase_num = 1 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) >= count1: + break + + self.assertEqual(results1, [(True, 1)] * count1) + self.assertEqual(results2, []) + + # Wait until awaken workers are blocked on cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= count1 : + break + + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + wait_threads_blocked(N) + + # Phase 2 + phase_num = 2 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= (N + count1): + break + + count2 = N - count1 + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1) + + # Make sure all workers settle into cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + wait_threads_blocked(N) + + # Phase 3 + phase_num = 3 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results2) >= N: + break + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1 + [(True, 3)] * count2) def test_notify(self): cond = self.condtype() @@ -544,19 +733,23 @@ def test_notify(self): def test_timeout(self): cond = self.condtype() + timeout = 0.5 results = [] - N = 5 def f(): cond.acquire() t1 = time.monotonic() - result = cond.wait(0.5) + result = cond.wait(timeout) t2 = time.monotonic() cond.release() results.append((t2 - t1, result)) - Bunch(f, N).wait_for_finished() + + N = 5 + with Bunch(f, N): + pass self.assertEqual(len(results), N) + for dt, result in results: - self.assertTimeout(dt, 0.5) + self.assertTimeout(dt, timeout) # Note that conceptually (that"s the condition variable protocol) # a wait() may succeed even if no one notifies us and before any # timeout occurs. Spurious wakeups can occur. @@ -569,17 +762,16 @@ def test_waitfor(self): state = 0 def f(): with cond: - result = cond.wait_for(lambda : state==4) + result = cond.wait_for(lambda: state == 4) self.assertTrue(result) self.assertEqual(state, 4) - b = Bunch(f, 1) - b.wait_for_started() - for i in range(4): - time.sleep(0.01) - with cond: - state += 1 - cond.notify() - b.wait_for_finished() + + with Bunch(f, 1): + for i in range(4): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() def test_waitfor_timeout(self): cond = self.condtype() @@ -593,15 +785,15 @@ def f(): self.assertFalse(result) self.assertTimeout(dt, 0.1) success.append(None) - b = Bunch(f, 1) - b.wait_for_started() - # Only increment 3 times, so state == 4 is never reached. - for i in range(3): - time.sleep(0.01) - with cond: - state += 1 - cond.notify() - b.wait_for_finished() + + with Bunch(f, 1): + # Only increment 3 times, so state == 4 is never reached. + for i in range(3): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() + self.assertEqual(len(success), 1) @@ -630,41 +822,107 @@ def test_acquire_destroy(self): del sem def test_acquire_contended(self): - sem = self.semtype(7) + sem_value = 7 + sem = self.semtype(sem_value) sem.acquire() - N = 10 + sem_results = [] results1 = [] results2 = [] phase_num = 0 - def f(): + + def func(): sem_results.append(sem.acquire()) results1.append(phase_num) + sem_results.append(sem.acquire()) results2.append(phase_num) - b = Bunch(f, 10) - b.wait_for_started() - while len(results1) + len(results2) < 6: - _wait() - self.assertEqual(results1 + results2, [0] * 6) - phase_num = 1 - for i in range(7): + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + N = 10 + with Bunch(func, N): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + for i in range(sem_value): + sem.release() + count2 = sem_value + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = (sem_value - 1) + for i in range(count3): + sem.release() + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish + count4 = 1 sem.release() - while len(results1) + len(results2) < 13: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) - phase_num = 2 - for i in range(6): + + self.assertEqual(sem_results, + [True] * (count1 + count2 + count3 + count4)) + + def test_multirelease(self): + sem_value = 7 + sem = self.semtype(sem_value) + sem.acquire() + + results1 = [] + results2 = [] + phase_num = 0 + def func(): + sem.acquire() + results1.append(phase_num) + + sem.acquire() + results2.append(phase_num) + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + with Bunch(func, 10): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + count2 = sem_value + sem.release(count2) + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = sem_value - 1 + sem.release(count3) + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish sem.release() - while len(results1) + len(results2) < 19: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) - # The semaphore is still locked - self.assertFalse(sem.acquire(False)) - # Final release, to let the last thread finish - sem.release() - b.wait_for_finished() - self.assertEqual(sem_results, [True] * (6 + 7 + 6 + 1)) def test_try_acquire(self): sem = self.semtype(2) @@ -681,7 +939,8 @@ def test_try_acquire_contended(self): def f(): results.append(sem.acquire(False)) results.append(sem.acquire(False)) - Bunch(f, 5).wait_for_finished() + with Bunch(f, 5): + pass # There can be a thread switch between acquiring the semaphore and # appending the result, therefore results will not necessarily be # ordered. @@ -707,12 +966,14 @@ def test_default_value(self): def f(): sem.acquire() sem.release() - b = Bunch(f, 1) - b.wait_for_started() - _wait() - self.assertFalse(b.finished) - sem.release() - b.wait_for_finished() + + with Bunch(f, 1) as bunch: + # Thread blocked on sem.acquire() + wait_threads_blocked(1) + self.assertFalse(bunch.finished) + + # Thread unblocked + sem.release() def test_with(self): sem = self.semtype(2) @@ -744,6 +1005,15 @@ def test_release_unacquired(self): sem.acquire() sem.release() + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=2>") + sem.release() + sem.release() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=4>") + class BoundedSemaphoreTests(BaseSemaphoreTests): """ @@ -758,6 +1028,12 @@ def test_release_unacquired(self): sem.release() self.assertRaises(ValueError, sem.release) + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=3/3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=2/3>") + class BarrierTests(BaseTestCase): """ @@ -768,13 +1044,13 @@ class BarrierTests(BaseTestCase): def setUp(self): self.barrier = self.barriertype(self.N, timeout=self.defaultTimeout) + def tearDown(self): self.barrier.abort() def run_threads(self, f): - b = Bunch(f, self.N-1) - f() - b.wait_for_finished() + with Bunch(f, self.N): + pass def multipass(self, results, n): m = self.barrier.parties @@ -789,6 +1065,10 @@ def multipass(self, results, n): self.assertEqual(self.barrier.n_waiting, 0) self.assertFalse(self.barrier.broken) + def test_constructor(self): + self.assertRaises(ValueError, self.barriertype, parties=0) + self.assertRaises(ValueError, self.barriertype, parties=-1) + def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep @@ -865,8 +1145,9 @@ def f(): i = self.barrier.wait() if i == self.N//2: # Wait until the other threads are all in the barrier. - while self.barrier.n_waiting < self.N-1: - time.sleep(0.001) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self.barrier.n_waiting >= (self.N - 1): + break self.barrier.reset() else: try: @@ -926,27 +1207,56 @@ def f(): i = self.barrier.wait() if i == self.N // 2: # One thread is late! - time.sleep(1.0) + time.sleep(self.defaultTimeout / 2) # Default timeout is 2.0, so this is shorter. self.assertRaises(threading.BrokenBarrierError, - self.barrier.wait, 0.5) + self.barrier.wait, self.defaultTimeout / 4) self.run_threads(f) def test_default_timeout(self): """ Test the barrier's default timeout """ - # create a barrier with a low default timeout - barrier = self.barriertype(self.N, timeout=0.3) + timeout = 0.100 + barrier = self.barriertype(2, timeout=timeout) def f(): - i = barrier.wait() - if i == self.N // 2: - # One thread is later than the default timeout of 0.3s. - time.sleep(1.0) - self.assertRaises(threading.BrokenBarrierError, barrier.wait) - self.run_threads(f) + self.assertRaises(threading.BrokenBarrierError, + barrier.wait) + + start_time = time.monotonic() + with Bunch(f, 1): + pass + dt = time.monotonic() - start_time + self.assertGreaterEqual(dt, timeout) def test_single_thread(self): b = self.barriertype(1) b.wait() b.wait() + + def test_repr(self): + barrier = self.barriertype(3) + timeout = support.LONG_TIMEOUT + self.assertRegex(repr(barrier), r"<\w+\.Barrier at .*: waiters=0/3>") + def f(): + barrier.wait(timeout) + + N = 2 + with Bunch(f, N): + # Threads blocked on barrier.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if barrier.n_waiting >= N: + break + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=2/3>") + + # Threads unblocked + barrier.wait(timeout) + + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=0/3>") + + # Abort the barrier + barrier.abort() + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: broken>") diff --git a/Lib/test/mathdata/cmath_testcases.txt b/Lib/test/mathdata/cmath_testcases.txt index 0165e17634f..7b98b5a2998 100644 --- a/Lib/test/mathdata/cmath_testcases.txt +++ b/Lib/test/mathdata/cmath_testcases.txt @@ -371,9 +371,9 @@ acosh1002 acosh 0.0 inf -> inf 1.5707963267948966 acosh1003 acosh 2.3 inf -> inf 1.5707963267948966 acosh1004 acosh -0.0 inf -> inf 1.5707963267948966 acosh1005 acosh -2.3 inf -> inf 1.5707963267948966 -acosh1006 acosh 0.0 nan -> nan nan +acosh1006 acosh 0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1007 acosh 2.3 nan -> nan nan -acosh1008 acosh -0.0 nan -> nan nan +acosh1008 acosh -0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1009 acosh -2.3 nan -> nan nan acosh1010 acosh -inf 0.0 -> inf 3.1415926535897931 acosh1011 acosh -inf 2.3 -> inf 3.1415926535897931 @@ -1992,9 +1992,9 @@ tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0 --special values tanh1000 tanh 0.0 0.0 -> 0.0 0.0 -tanh1001 tanh 0.0 inf -> nan nan invalid +tanh1001 tanh 0.0 inf -> 0.0 nan invalid tanh1002 tanh 2.3 inf -> nan nan invalid -tanh1003 tanh 0.0 nan -> nan nan +tanh1003 tanh 0.0 nan -> 0.0 nan tanh1004 tanh 2.3 nan -> nan nan tanh1005 tanh inf 0.0 -> 1.0 0.0 tanh1006 tanh inf 0.7 -> 1.0 0.0 @@ -2009,7 +2009,7 @@ tanh1014 tanh nan 2.3 -> nan nan tanh1015 tanh nan inf -> nan nan tanh1016 tanh nan nan -> nan nan tanh1017 tanh 0.0 -0.0 -> 0.0 -0.0 -tanh1018 tanh 0.0 -inf -> nan nan invalid +tanh1018 tanh 0.0 -inf -> 0.0 nan invalid tanh1019 tanh 2.3 -inf -> nan nan invalid tanh1020 tanh inf -0.0 -> 1.0 -0.0 tanh1021 tanh inf -0.7 -> 1.0 -0.0 @@ -2022,9 +2022,9 @@ tanh1027 tanh nan -0.0 -> nan -0.0 tanh1028 tanh nan -2.3 -> nan nan tanh1029 tanh nan -inf -> nan nan tanh1030 tanh -0.0 -0.0 -> -0.0 -0.0 -tanh1031 tanh -0.0 -inf -> nan nan invalid +tanh1031 tanh -0.0 -inf -> -0.0 nan invalid tanh1032 tanh -2.3 -inf -> nan nan invalid -tanh1033 tanh -0.0 nan -> nan nan +tanh1033 tanh -0.0 nan -> -0.0 nan tanh1034 tanh -2.3 nan -> nan nan tanh1035 tanh -inf -0.0 -> -1.0 -0.0 tanh1036 tanh -inf -0.7 -> -1.0 -0.0 @@ -2035,7 +2035,7 @@ tanh1040 tanh -inf -3.5 -> -1.0 -0.0 tanh1041 tanh -inf -inf -> -1.0 0.0 ignore-imag-sign tanh1042 tanh -inf nan -> -1.0 0.0 ignore-imag-sign tanh1043 tanh -0.0 0.0 -> -0.0 0.0 -tanh1044 tanh -0.0 inf -> nan nan invalid +tanh1044 tanh -0.0 inf -> -0.0 nan invalid tanh1045 tanh -2.3 inf -> nan nan invalid tanh1046 tanh -inf 0.0 -> -1.0 0.0 tanh1047 tanh -inf 0.7 -> -1.0 0.0 @@ -2307,9 +2307,9 @@ tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0 -- special values tan1000 tan -0.0 0.0 -> -0.0 0.0 -tan1001 tan -inf 0.0 -> nan nan invalid +tan1001 tan -inf 0.0 -> nan 0.0 invalid tan1002 tan -inf 2.2999999999999998 -> nan nan invalid -tan1003 tan nan 0.0 -> nan nan +tan1003 tan nan 0.0 -> nan 0.0 tan1004 tan nan 2.2999999999999998 -> nan nan tan1005 tan -0.0 inf -> -0.0 1.0 tan1006 tan -0.69999999999999996 inf -> -0.0 1.0 @@ -2324,7 +2324,7 @@ tan1014 tan -2.2999999999999998 nan -> nan nan tan1015 tan -inf nan -> nan nan tan1016 tan nan nan -> nan nan tan1017 tan 0.0 0.0 -> 0.0 0.0 -tan1018 tan inf 0.0 -> nan nan invalid +tan1018 tan inf 0.0 -> nan 0.0 invalid tan1019 tan inf 2.2999999999999998 -> nan nan invalid tan1020 tan 0.0 inf -> 0.0 1.0 tan1021 tan 0.69999999999999996 inf -> 0.0 1.0 @@ -2337,9 +2337,9 @@ tan1027 tan 0.0 nan -> 0.0 nan tan1028 tan 2.2999999999999998 nan -> nan nan tan1029 tan inf nan -> nan nan tan1030 tan 0.0 -0.0 -> 0.0 -0.0 -tan1031 tan inf -0.0 -> nan nan invalid +tan1031 tan inf -0.0 -> nan -0.0 invalid tan1032 tan inf -2.2999999999999998 -> nan nan invalid -tan1033 tan nan -0.0 -> nan nan +tan1033 tan nan -0.0 -> nan -0.0 tan1034 tan nan -2.2999999999999998 -> nan nan tan1035 tan 0.0 -inf -> 0.0 -1.0 tan1036 tan 0.69999999999999996 -inf -> 0.0 -1.0 @@ -2350,7 +2350,7 @@ tan1040 tan 3.5 -inf -> 0.0 -1.0 tan1041 tan inf -inf -> -0.0 -1.0 ignore-real-sign tan1042 tan nan -inf -> -0.0 -1.0 ignore-real-sign tan1043 tan -0.0 -0.0 -> -0.0 -0.0 -tan1044 tan -inf -0.0 -> nan nan invalid +tan1044 tan -inf -0.0 -> nan -0.0 invalid tan1045 tan -inf -2.2999999999999998 -> nan nan invalid tan1046 tan -0.0 -inf -> -0.0 -1.0 tan1047 tan -0.69999999999999996 -inf -> -0.0 -1.0 diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt index 3e986cdb102..9be667826a6 100644 --- a/Lib/test/mathdata/ieee754.txt +++ b/Lib/test/mathdata/ieee754.txt @@ -116,7 +116,7 @@ inf >>> 0 ** -1 Traceback (most recent call last): ... -ZeroDivisionError: 0.0 cannot be raised to a negative power +ZeroDivisionError: zero to a negative power >>> pow(0, NAN) nan @@ -127,31 +127,31 @@ Trigonometric Functions >>> sin(INF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got inf >>> sin(NINF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got -inf >>> sin(NAN) nan >>> cos(INF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got inf >>> cos(NINF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got -inf >>> cos(NAN) nan >>> tan(INF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got inf >>> tan(NINF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a finite input, got -inf >>> tan(NAN) nan @@ -169,11 +169,11 @@ True >>> asin(INF), asin(NINF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a number in range from -1 up to 1, got inf >>> acos(INF), acos(NINF) Traceback (most recent call last): ... -ValueError: math domain error +ValueError: expected a number in range from -1 up to 1, got inf >>> equal(atan(INF), PI/2), equal(atan(NINF), -PI/2) (True, True) diff --git a/Lib/test/mp_fork_bomb.py b/Lib/test/mp_fork_bomb.py new file mode 100644 index 00000000000..017e010ba0e --- /dev/null +++ b/Lib/test/mp_fork_bomb.py @@ -0,0 +1,18 @@ +import multiprocessing, sys + +def foo(): + print("123") + +# Because "if __name__ == '__main__'" is missing this will not work +# correctly on Windows. However, we should get a RuntimeError rather +# than the Windows equivalent of a fork bomb. + +if len(sys.argv) > 1: + multiprocessing.set_start_method(sys.argv[1]) +else: + multiprocessing.set_start_method('spawn') + +p = multiprocessing.Process(target=foo) +p.start() +p.join() +sys.exit(p.exitcode) diff --git a/Lib/test/mp_preload.py b/Lib/test/mp_preload.py new file mode 100644 index 00000000000..5314e8f0b21 --- /dev/null +++ b/Lib/test/mp_preload.py @@ -0,0 +1,18 @@ +import multiprocessing + +multiprocessing.Lock() + + +def f(): + print("ok") + + +if __name__ == "__main__": + ctx = multiprocessing.get_context("forkserver") + modname = "test.mp_preload" + # Make sure it's importable + __import__(modname) + ctx.set_forkserver_preload([modname]) + proc = ctx.Process(target=f) + proc.start() + proc.join() diff --git a/Lib/test/mp_preload_flush.py b/Lib/test/mp_preload_flush.py new file mode 100644 index 00000000000..c195a9ef6b2 --- /dev/null +++ b/Lib/test/mp_preload_flush.py @@ -0,0 +1,11 @@ +import multiprocessing +import sys + +print(__name__, end='', file=sys.stderr) +print(__name__, end='', file=sys.stdout) +if __name__ == '__main__': + multiprocessing.set_start_method('forkserver') + for _ in range(2): + p = multiprocessing.Process() + p.start() + p.join() diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py new file mode 100644 index 00000000000..6b4c57d0b4b --- /dev/null +++ b/Lib/test/multibytecodec_support.py @@ -0,0 +1,400 @@ +# +# multibytecodec_support.py +# Common Unittest Routines for CJK codecs +# + +import codecs +import os +import re +import sys +import unittest +from http.client import HTTPException +from test import support +from io import BytesIO + +class TestBase: + encoding = '' # codec name + codec = None # codec tuple (with 4 elements) + tstring = None # must set. 2 strings to test StreamReader + + codectests = None # must set. codec test tuple + roundtriptest = 1 # set if roundtrip is possible with unicode + has_iso10646 = 0 # set if this encoding contains whole iso10646 map + xmlcharnametest = None # string to test xmlcharrefreplace + unmappedunicode = '\udeee' # a unicode code point that is not mapped. + + def setUp(self): + if self.codec is None: + self.codec = codecs.lookup(self.encoding) + self.encode = self.codec.encode + self.decode = self.codec.decode + self.reader = self.codec.streamreader + self.writer = self.codec.streamwriter + self.incrementalencoder = self.codec.incrementalencoder + self.incrementaldecoder = self.codec.incrementaldecoder + + def test_chunkcoding(self): + tstring_lines = [] + for b in self.tstring: + lines = b.split(b"\n") + last = lines.pop() + assert last == b"" + lines = [line + b"\n" for line in lines] + tstring_lines.append(lines) + for native, utf8 in zip(*tstring_lines): + u = self.decode(native)[0] + self.assertEqual(u, utf8.decode('utf-8')) + if self.roundtriptest: + self.assertEqual(native, self.encode(u)[0]) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = self.decode + else: + func = self.encode + if expected: + result = func(source, scheme)[0] + if func is self.decode: + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, source, scheme) + + def test_xmlcharrefreplace(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + s = "\u0b13\u0b23\u0b60 nd eggs" + self.assertEqual( + self.encode(s, "xmlcharrefreplace")[0], + b"ଓଣୠ nd eggs" + ) + + def test_customreplace_encode(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + from html.entities import codepoint2name + + def xmlcharnamereplace(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [] + for c in exc.object[exc.start:exc.end]: + if ord(c) in codepoint2name: + l.append("&%s;" % codepoint2name[ord(c)]) + else: + l.append("&#%d;" % ord(c)) + return ("".join(l), exc.end) + + codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace) + + if self.xmlcharnametest: + sin, sout = self.xmlcharnametest + else: + sin = "\xab\u211c\xbb = \u2329\u1234\u232a" + sout = b"«ℜ» = ⟨ሴ⟩" + self.assertEqual(self.encode(sin, + "test.xmlcharnamereplace")[0], sout) + + def test_callback_returns_bytes(self): + def myreplace(exc): + return (b"1234", exc.end) + codecs.register_error("test.cjktest", myreplace) + enc = self.encode("abc" + self.unmappedunicode + "def", "test.cjktest")[0] + self.assertEqual(enc, b"abc1234def") + + def test_callback_wrong_objects(self): + def myreplace(exc): + return (ret, exc.end) + codecs.register_error("test.cjktest", myreplace) + + for ret in ([1, 2, 3], [], None, object()): + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_long_index(self): + def myreplace(exc): + return ('x', int(exc.end)) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdxefgh', 9)) + + def myreplace(exc): + return ('x', sys.maxsize + 1) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_None_index(self): + def myreplace(exc): + return ('x', None) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_backward_index(self): + def myreplace(exc): + if myreplace.limit > 0: + myreplace.limit -= 1 + return ('REPLACED', 0) + else: + return ('TERMINAL', exc.end) + myreplace.limit = 3 + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), + (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9)) + + def test_callback_forward_index(self): + def myreplace(exc): + return ('REPLACED', exc.end + 2) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdREPLACEDgh', 9)) + + def test_callback_index_outofbound(self): + def myreplace(exc): + return ('TERM', 100) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_incrementalencoder(self): + UTF8Reader = codecs.getreader('utf-8') + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = BytesIO() + encoder = self.incrementalencoder() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + e = encoder.encode(data) + ostream.write(e) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_incrementaldecoder(self): + UTF8Writer = codecs.getwriter('utf-8') + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = BytesIO(self.tstring[0]) + ostream = UTF8Writer(BytesIO()) + decoder = self.incrementaldecoder() + while 1: + data = istream.read(sizehint) + if not data: + break + else: + u = decoder.decode(data) + ostream.write(u) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_incrementalencoder_error_callback(self): + inv = self.unmappedunicode + + e = self.incrementalencoder() + self.assertRaises(UnicodeEncodeError, e.encode, inv, True) + + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + e.reset() + def tempreplace(exc): + return ('called', exc.end) + codecs.register_error('test.incremental_error_callback', tempreplace) + e.errors = 'test.incremental_error_callback' + self.assertEqual(e.encode(inv, True), b'called') + + # again + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + def test_streamreader(self): + UTF8Writer = codecs.getwriter('utf-8') + for name in ["read", "readline", "readlines"]: + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = self.reader(BytesIO(self.tstring[0])) + ostream = UTF8Writer(BytesIO()) + func = getattr(istream, name) + while 1: + data = func(sizehint) + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_streamwriter(self): + readfuncs = ('read', 'readline', 'readlines') + UTF8Reader = codecs.getreader('utf-8') + for name in readfuncs: + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = self.writer(BytesIO()) + func = getattr(istream, name) + while 1: + if sizehint is not None: + data = func(sizehint) + else: + data = func() + + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_streamwriter_reset_no_pending(self): + # Issue #23247: Calling reset() on a fresh StreamWriter instance + # (without pending data) must not crash + stream = BytesIO() + writer = self.writer(stream) + writer.reset() + + def test_incrementalencoder_del_segfault(self): + e = self.incrementalencoder() + with self.assertRaises(AttributeError): + del e.errors + + def test_null_terminator(self): + # see gh-101828 + text = "フルーツ" + try: + text.encode(self.encoding) + except UnicodeEncodeError: + text = "Python is cool" + encode_w_null = (text + "\0").encode(self.encoding) + encode_plus_null = text.encode(self.encoding) + "\0".encode(self.encoding) + self.assertTrue(encode_w_null.endswith(b'\x00')) + self.assertEqual(encode_w_null, encode_plus_null) + + encode_w_null_2 = (text + "\0" + text + "\0").encode(self.encoding) + encode_plus_null_2 = encode_plus_null + encode_plus_null + self.assertEqual(encode_w_null_2.count(b'\x00'), 2) + self.assertEqual(encode_w_null_2, encode_plus_null_2) + + +class TestBase_Mapping(unittest.TestCase): + pass_enctest = [] + pass_dectest = [] + supmaps = [] + codectests = [] + + def setUp(self): + try: + self.open_mapping_file().close() # test it to report the error early + except (OSError, HTTPException): + self.skipTest("Could not retrieve "+self.mapfileurl) + + def open_mapping_file(self): + return support.open_urlresource(self.mapfileurl, encoding="utf-8") + + def test_mapping_file(self): + if self.mapfileurl.endswith('.xml'): + self._test_mapping_file_ucm() + else: + self._test_mapping_file_plain() + + def _test_mapping_file_plain(self): + def unichrs(s): + return ''.join(chr(int(x, 16)) for x in s.split('+')) + + urt_wa = {} + + with self.open_mapping_file() as f: + for line in f: + if not line: + break + data = line.split('#')[0].split() + if len(data) != 2: + continue + + if data[0][:2] != '0x': + self.fail(f"Invalid line: {line!r}") + csetch = bytes.fromhex(data[0][2:]) + if len(csetch) == 1 and 0x80 <= csetch[0]: + continue + + unich = unichrs(data[1]) + if ord(unich) == 0xfffd or unich in urt_wa: + continue + urt_wa[unich] = csetch + + self._testpoint(csetch, unich) + + def _test_mapping_file_ucm(self): + with self.open_mapping_file() as f: + ucmdata = f.read() + uc = re.findall('', ucmdata) + for uni, coded in uc: + unich = chr(int(uni, 16)) + codech = bytes.fromhex(coded) + self._testpoint(codech, unich) + + def test_mapping_supplemental(self): + for mapping in self.supmaps: + self._testpoint(*mapping) + + def _testpoint(self, csetch, unich): + if (csetch, unich) not in self.pass_enctest: + self.assertEqual(unich.encode(self.encoding), csetch) + if (csetch, unich) not in self.pass_dectest: + self.assertEqual(str(csetch, self.encoding), unich) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = source.decode + else: + func = source.encode + if expected: + if isinstance(source, bytes): + result = func(self.encoding, scheme) + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + result = func(self.encoding, scheme) + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, self.encoding, scheme) + +def load_teststring(name): + dir = os.path.join(os.path.dirname(__file__), 'cjkencodings') + with open(os.path.join(dir, name + '.txt'), 'rb') as f: + encoded = f.read() + with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f: + utf8 = f.read() + return encoded, utf8 diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c0d4c8f43b9..9a3a26a8400 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1012,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') @@ -1234,24 +1254,37 @@ def test_find_class(self): self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'"): unpickler.find_class('builtins', 'str.upper') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): unpickler.find_class('math', 'spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): unpickler4.find_class('math', 'spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.spam'"): unpickler.find_class('math', 'log.spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.spam' on module 'math'") as cm: unpickler4.find_class('math', 'log.spam') - with self.assertRaises(AttributeError): + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute 'spam'") + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.\.spam'"): unpickler.find_class('math', 'log..spam') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.\.spam' on module 'math'") as cm: unpickler4.find_class('math', 'log..spam') - with self.assertRaises(AttributeError): + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute ''") + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): unpickler.find_class('math', '') - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): unpickler4.find_class('math', '') self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') self.assertRaises(ValueError, unpickler.find_class, '', 'log') @@ -1637,48 +1670,77 @@ def test_bad_reduce_result(self): obj = REX([print, ()]) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + '__reduce__ must return a string or tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((print,)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((print, (), None, None, None, None, None)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_reconstructor(self): obj = REX((42, ())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'first item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_reconstructor(self): obj = REX((UnpickleableCallable(), ())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) def test_bad_reconstructor_args(self): obj = REX((print, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_reconstructor_args(self): obj = REX((print, (1, 2, UNPICKLEABLE))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_bad_newobj_args(self): obj = REX((copyreg.__newobj__, ())) @@ -1686,74 +1748,154 @@ def test_bad_newobj_args(self): with self.subTest(proto=proto): with self.assertRaises((IndexError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'tuple index out of range', + '__newobj__ expected at least 1 argument, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj__, [REX])) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises((IndexError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_newobj_class(self): obj = REX((copyreg.__newobj__, (NoNew(),))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj__() has no __new__', + f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_wrong_newobj_class(self): obj = REX((copyreg.__newobj__, (str,))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj__() must be {REX!r}, not {str!r}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_class(self): class LocalREX(REX): pass obj = LocalREX((copyreg.__newobj__, (LocalREX,))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((pickle.PicklingError, AttributeError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) def test_unpickleable_newobj_args(self): obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_bad_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, ())) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises((ValueError, pickle.PicklingError)): + with self.assertRaises((ValueError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 3, got 0)', + '__newobj_ex__ expected 3 arguments, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, 42)) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) - is_py = self.pickler is pickle._Pickler - for proto in protocols[2:4] if is_py else protocols[2:]: - with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): - self.dumps(obj, proto) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'Value after * must be an iterable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second argument to __newobj_ex__() must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, (), []))) - for proto in protocols[2:4] if is_py else protocols[2:]: - with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): - self.dumps(obj, proto) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'functools.partial() argument after ** must be a mapping, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'third argument to __newobj_ex__() must be a dict, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_bad_newobj_ex__class(self): obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj_ex__() has no __new__', + f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_wrong_newobj_ex_class(self): if self.pickler is not pickle._Pickler: @@ -1761,37 +1903,99 @@ def test_wrong_newobj_ex_class(self): obj = REX((copyreg.__newobj_ex__, (str, (), {}))) for proto in protocols[2:]: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj_ex__() must be {REX}, not {str}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_ex_class(self): class LocalREX(REX): pass obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((pickle.PicklingError, AttributeError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) def test_unpickleable_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_unpickleable_newobj_ex_kwargs(self): obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) def test_unpickleable_state(self): obj = REX_state(UNPICKLEABLE) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_state state', + 'when serializing test.pickletester.REX_state object']) def test_bad_state_setter(self): if self.pickler is pickle._Pickler: @@ -1799,22 +2003,33 @@ def test_bad_state_setter(self): obj = REX((print, (), 'state', None, None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'sixth item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_state_setter(self): obj = REX((print, (), 'state', None, None, UnpickleableCallable())) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state setter', + 'when serializing test.pickletester.REX object']) def test_unpickleable_state_with_state_setter(self): obj = REX((print, (), UNPICKLEABLE, None, None, print)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state', + 'when serializing test.pickletester.REX object']) def test_bad_object_list_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1822,23 +2037,37 @@ def test_bad_object_list_items(self): obj = REX((list, (), None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. obj = REX((list, (), None, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError): self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_object_list_items(self): obj = REX_six([1, 2, UNPICKLEABLE]) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_six item 2', + 'when serializing test.pickletester.REX_six object']) def test_bad_object_dict_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1846,82 +2075,135 @@ def test_bad_object_dict_items(self): obj = REX((dict, (), None, None, 42)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fifth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) for proto in protocols: obj = REX((dict, (), None, None, iter([('a',)]))) with self.subTest(proto=proto): - with self.assertRaises((ValueError, TypeError)): + with self.assertRaises((ValueError, TypeError)) as cm: self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 2, got 1)', + 'dict items iterator must return 2-tuples'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. obj = REX((dict, (), None, None, [])) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((TypeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError): self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'dict items iterator must return 2-tuples') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) def test_unpickleable_object_dict_items(self): obj = REX_seven({'a': UNPICKLEABLE}) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing test.pickletester.REX_seven item 'a'", + 'when serializing test.pickletester.REX_seven object']) def test_unpickleable_list_items(self): obj = [1, [2, 3, UNPICKLEABLE]] for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 2', + 'when serializing list item 1']) for n in [0, 1, 1000, 1005]: obj = [*range(n), UNPICKLEABLE] for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + f'when serializing list item {n}']) def test_unpickleable_tuple_items(self): obj = (1, (2, 3, UNPICKLEABLE)) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1']) obj = (*range(10), UNPICKLEABLE) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 10']) def test_unpickleable_dict_items(self): obj = {'a': {'b': UNPICKLEABLE}} for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'b'", + "when serializing dict item 'a'"]) for n in [0, 1, 1000, 1005]: obj = dict.fromkeys(range(n)) obj['a'] = UNPICKLEABLE for proto in protocols: with self.subTest(proto=proto, n=n): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'"]) def test_unpickleable_set_items(self): obj = {UNPICKLEABLE} for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing set element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing set reconstructor arguments']) def test_unpickleable_frozenset_items(self): obj = frozenset({frozenset({UNPICKLEABLE})}) for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(CustomError): + with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing frozenset element', + 'when serializing frozenset element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments', + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments']) def test_global_lookup_error(self): # Global name does not exist @@ -1929,26 +2211,42 @@ def test_global_lookup_error(self): obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + self.assertEqual(str(cm.exception.__context__), + f"module '{__name__}' has no attribute 'spam'") obj.__module__ = 'nonexisting' for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: No module named 'nonexisting'") + self.assertEqual(str(cm.exception.__context__), + "No module named 'nonexisting'") obj.__module__ = '' for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((ValueError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: Empty module name") + self.assertEqual(str(cm.exception.__context__), + "Empty module name") obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'spam'") def test_nonencodable_global_name_error(self): for proto in protocols[:4]: @@ -1957,8 +2255,11 @@ def test_nonencodable_global_name_error(self): obj = REX(name) obj.__module__ = __name__ with support.swap_item(globals(), name, obj): - with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle global identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) def test_nonencodable_module_name_error(self): for proto in protocols[:4]: @@ -1968,8 +2269,11 @@ def test_nonencodable_module_name_error(self): obj.__module__ = name mod = types.SimpleNamespace(test=obj) with support.swap_item(sys.modules, name, mod): - with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle module identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) def test_nested_lookup_error(self): # Nested name does not exist @@ -1981,14 +2285,24 @@ class A: obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as {__name__}.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "type object 'A' has no attribute 'B'") obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") def test_wrong_object_lookup_error(self): # Name is bound to different object @@ -1999,14 +2313,23 @@ class TestGlobal: obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not the same object as {__name__}.TestGlobal") + self.assertIsNone(cm.exception.__context__) obj.__module__ = None for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") def test_local_lookup_error(self): # Test that whichmodule() errors out cleanly when looking up @@ -2016,21 +2339,27 @@ def f(): # Since the function is local, lookup will fail for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") # Same without a __module__ attribute (exercises a different path # in _pickle.c). del f.__module__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") # Yet a different path. f.__name__ = f.__qualname__ for proto in protocols: with self.subTest(proto=proto): - with self.assertRaises((AttributeError, pickle.PicklingError)): + with self.assertRaises(pickle.PicklingError) as cm: self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") def test_reduce_ex_None(self): c = REX_None() @@ -2744,7 +3073,7 @@ def test_proto(self): pickled = self.dumps(None, proto) if proto >= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) @@ -4640,8 +4969,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): @@ -4680,7 +5012,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 21b0edfd073..dd61b051354 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,13 +6,10 @@ Run this script with -h or --help for documentation. """ -# We import importlib *ASAP* in order to test #15386 -import importlib - import os import sys -from test.libregrtest import main +from test.libregrtest.main import main # Alias for backward compatibility (just in case) main_in_temp_cwd = main diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py new file mode 100644 index 00000000000..9c3d0c7cf4b --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py new file mode 100644 index 00000000000..3dfba253455 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py @@ -0,0 +1,9 @@ +import sys +import unittest + +class Test(unittest.TestCase): + def test(self): + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIn('test_regrtest_b', sys.modules) + self.assertNotIn('test_regrtest_b.util', sys.modules) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py new file mode 100644 index 00000000000..de80769118d --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 54eb5e65ac2..c7497d09f64 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -426,8 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) - @unittest.expectedFailure # TODO: RUSTPYTHON - @support.suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 444ca2219cf..cfeeac6deb4 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3,9 +3,10 @@ if __name__ != 'test.support': raise ImportError('support must be imported from the test package') +import annotationlib import contextlib -import dataclasses import functools +import inspect import logging import _opcode import os @@ -29,10 +30,11 @@ "record_original_stdout", "get_original_stdout", "captured_stdout", "captured_stdin", "captured_stderr", "captured_output", # unittest - "is_resource_enabled", "requires", "requires_freebsd_version", + "is_resource_enabled", "get_resource_value", "requires", "requires_resource", + "requires_freebsd_version", "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "requires_gzip", "requires_bz2", "requires_lzma", + "requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", "has_fork_support", "requires_fork", @@ -41,10 +43,12 @@ "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", - "requires_limited_api", "requires_specialization", + "requires_limited_api", "requires_specialization", "thread_unsafe", + "skip_if_unlimited_stack_size", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", + "support_remote_exec_only", # os "get_pagesize", # network @@ -57,13 +61,16 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", - "skip_on_s390x", - "without_optimizer", + "Py_DEBUG", "exceeds_recursion_limit", "skip_on_s390x", + "requires_jit_enabled", + "requires_jit_disabled", "force_not_colorized", "force_not_colorized_test_class", "make_clean_env", "BrokenIter", + "in_systemd_nspawn_sync_suppressed", + "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", + "reset_code", "on_github_actions" ] @@ -179,7 +186,7 @@ def get_attribute(obj, name): return attribute verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py +use_resources = None # Flag set to {} by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 @@ -294,6 +301,16 @@ def is_resource_enabled(resource): """ return use_resources is None or resource in use_resources +def get_resource_value(resource): + """Test whether a resource is enabled. + + Known resources are set by regrtest.py. If not running under regrtest.py, + all resources are assumed enabled unless use_resources has been set. + """ + if use_resources is None: + return None + return use_resources.get(resource) + def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" if not is_resource_enabled(resource): @@ -389,6 +406,21 @@ def wrapper(*args, **kw): return decorator +def thread_unsafe(reason): + """Mark a test as not thread safe. When the test runner is run with + --parallel-threads=N, the test will be run in a single thread.""" + def decorator(test_item): + test_item.__unittest_thread_unsafe__ = True + # the reason is not currently used + test_item.__unittest_thread_unsafe__why__ = reason + return test_item + if isinstance(reason, types.FunctionType): + test_item = reason + reason = '' + return decorator(test_item) + return decorator + + def skip_if_buildbot(reason=None): """Decorator raising SkipTest if running on a buildbot.""" import getpass @@ -401,7 +433,8 @@ def skip_if_buildbot(reason=None): isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): +def check_sanitizer(*, address=False, memory=False, ub=False, thread=False, + function=True): """Returns True if Python is compiled with sanitizer support""" if not (address or memory or ub or thread): raise ValueError('At least one of address, memory, ub or thread must be True') @@ -425,11 +458,15 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): '-fsanitize=thread' in cflags or '--with-thread-sanitizer' in config_args ) + function_sanitizer = ( + '-fsanitize=function' in cflags + ) return ( (memory and memory_sanitizer) or (address and address_sanitizer) or (ub and ub_sanitizer) or - (thread and thread_sanitizer) + (thread and thread_sanitizer) or + (function and function_sanitizer) ) @@ -514,13 +551,19 @@ def requires_lzma(reason='requires lzma'): lzma = None # XXX: RUSTPYTHON; xz is not supported yet return unittest.skipUnless(lzma, reason) +def requires_zstd(reason='requires zstd'): + try: + from compression import zstd + except ImportError: + zstd = None + return unittest.skipUnless(zstd, reason) + def has_no_debug_ranges(): try: - import _testinternalcapi + import _testcapi except ImportError: raise unittest.SkipTest("_testinternalcapi required") - config = _testinternalcapi.get_config() - return not bool(config['code_debug_ranges']) + return not _testcapi.config_get('code_debug_ranges') def requires_debug_ranges(reason='requires co_positions / debug_ranges'): try: @@ -531,51 +574,6 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(skip, reason) -def can_use_suppress_immortalization(suppress=True): - """Check if suppress_immortalization(suppress) can be used. - - Use this helper in code where SkipTest must be eagerly handled. - """ - if not suppress: - return True - try: - import _testinternalcapi - except ImportError: - return False - return True - - -@contextlib.contextmanager -def suppress_immortalization(suppress=True): - """Suppress immortalization of deferred objects. - - If _testinternalcapi is not available, the decorated test or class - is skipped. Use can_use_suppress_immortalization() outside test cases - to check if this decorator can be used. - """ - if not suppress: - yield # no-op - return - - from .import_helper import import_module - - _testinternalcapi = import_module("_testinternalcapi") - _testinternalcapi.suppress_immortalization(True) - try: - yield - finally: - _testinternalcapi.suppress_immortalization(False) - - -def skip_if_suppress_immortalization(): - try: - import _testinternalcapi - except ImportError: - return - return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), - "requires immortalization of deferred objects") - - MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. @@ -583,6 +581,11 @@ def skip_if_suppress_immortalization(): is_android = sys.platform == "android" +def skip_android_selinux(name): + return unittest.skipIf( + sys.platform == "android", f"Android blocks {name} with SELinux" + ) + if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: @@ -593,6 +596,15 @@ def skip_if_suppress_immortalization(): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +# Use is_wasm32 as a generic check for WebAssembly platforms. +is_wasm32 = is_emscripten or is_wasi + +def skip_emscripten_stack_overflow(): + return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") + +def skip_wasi_stack_overflow(): + return unittest.skipIf(is_wasi, "Exhausts stack on WASI") + is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" @@ -715,9 +727,11 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str) -> dict[str, object]: +def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} + if extra_names: + ns.update(extra_names) exec(textwrap.dedent(code), ns) return ns @@ -953,8 +967,16 @@ def calcvobjsize(fmt): return struct.calcsize(_vheader + fmt + _align) -_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_STATIC_BUILTIN = 1<<1 +_TPFLAGS_DISALLOW_INSTANTIATION = 1<<7 +_TPFLAGS_IMMUTABLETYPE = 1<<8 _TPFLAGS_HEAPTYPE = 1<<9 +_TPFLAGS_BASETYPE = 1<<10 +_TPFLAGS_READY = 1<<12 +_TPFLAGS_READYING = 1<<13 +_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_BASE_EXC_SUBCLASS = 1<<30 +_TPFLAGS_TYPE_SUBCLASS = 1<<31 def check_sizeof(test, o, size): try: @@ -1318,6 +1340,26 @@ def coverage_wrapper(*args, **kwargs): return coverage_wrapper +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + assert not isinstance(func, type), func + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + + def refcount_test(test): """Decorator for tests which involve reference counting. @@ -1331,8 +1373,8 @@ def refcount_test(test): def requires_limited_api(test): try: - import _testcapi - import _testlimitedcapi + import _testcapi # noqa: F401 + import _testlimitedcapi # noqa: F401 except ImportError: return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) return test @@ -1347,6 +1389,18 @@ def requires_specialization(test): _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) +def requires_specialization_ft(test): + return unittest.skipUnless( + _opcode.ENABLE_SPECIALIZATION_FT, "requires specialization")(test) + + +def reset_code(f: types.FunctionType) -> types.FunctionType: + """Clear all specializations, local instrumentation, and JIT code for the given function.""" + f.__code__ = f.__code__.replace() + return f + +on_github_actions = "GITHUB_ACTIONS" in os.environ + #======================================================================= # Check for the presence of docstrings. @@ -1648,6 +1702,25 @@ def skip_if_pgo_task(test): return test if ok else unittest.skip(msg)(test) +def skip_if_unlimited_stack_size(test): + """Skip decorator for tests not run when an unlimited stack size is configured. + + Tests using support.infinite_recursion([...]) may otherwise run into + an infinite loop, running until the memory on the system is filled and + crashing due to OOM. + + See https://github.com/python/cpython/issues/143460. + """ + if is_emscripten or is_wasi or os.name == "nt": + return test + + import resource + curlim, maxlim = resource.getrlimit(resource.RLIMIT_STACK) + unlimited_stack_size_cond = curlim == maxlim and curlim in (-1, 0xFFFF_FFFF_FFFF_FFFF) + reason = "Not run due to unlimited stack size" + return unittest.skipIf(unlimited_stack_size_cond, reason)(test) + + def detect_api_mismatch(ref_api, other_api, *, ignore=()): """Returns the set of items in ref_api not in other_api, except for a defined list of items to be ignored in this check. @@ -2304,7 +2377,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") @@ -2396,8 +2477,9 @@ def clear_ignored_deprecations(*tokens: object) -> None: raise ValueError("Provide token or tokens returned by ignore_deprecations_from") new_filters = [] + old_filters = warnings._get_filters() endswith = tuple(rf"(?#support{id(token)})" for token in tokens) - for action, message, category, module, lineno in warnings.filters: + for action, message, category, module, lineno in old_filters: if action == "ignore" and category is DeprecationWarning: if isinstance(message, re.Pattern): msg = message.pattern @@ -2406,8 +2488,8 @@ def clear_ignored_deprecations(*tokens: object) -> None: if msg.endswith(endswith): continue new_filters.append((action, message, category, module, lineno)) - if warnings.filters != new_filters: - warnings.filters[:] = new_filters + if old_filters != new_filters: + old_filters[:] = new_filters warnings._filters_mutated() @@ -2415,7 +2497,7 @@ def clear_ignored_deprecations(*tokens: object) -> None: def requires_venv_with_pip(): # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) try: - import zlib + import zlib # noqa: F401 except ImportError: return unittest.skipIf(True, "venv: ensurepip requires zlib") @@ -2610,30 +2692,30 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) -class CPUStopwatch: +class Stopwatch: """Context manager to roughly time a CPU-bound operation. - Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of - other processes). + Disables GC. Uses perf_counter, which is a clock with the highest + available resolution. It is chosen even though it does include + time elapsed during sleep and is system-wide, because the + resolution of process_time is too coarse on Windows and + process_time does not exist everywhere (for example, WASM). - N.B.: - - This *includes* time spent in other threads. + Note: + - This *includes* time spent in other threads/processes. - Some systems only have a coarse resolution; check - stopwatch.clock_info.rseolution if. + stopwatch.clock_info.resolution when using the results. Usage: - with ProcessStopwatch() as stopwatch: + with Stopwatch() as stopwatch: ... elapsed = stopwatch.seconds resolution = stopwatch.clock_info.resolution """ def __enter__(self): - get_time = time.process_time - clock_info = time.get_clock_info('process_time') - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic - clock_info = time.get_clock_info('monotonic') + get_time = time.perf_counter + clock_info = time.get_clock_info('perf_counter') self.context = disable_gc() self.context.__enter__() self.get_time = get_time @@ -2661,17 +2743,9 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) -def get_c_recursion_limit(): - try: - import _testcapi - return _testcapi.Py_C_RECURSION_LIMIT - except ImportError: - raise unittest.SkipTest('requires _testcapi') - - def exceeds_recursion_limit(): """For recursion tests, easily exceeds default recursion limit.""" - return get_c_recursion_limit() * 3 + return 150_000 # Windows doesn't have os.uname() but it doesn't support s390x. @@ -2680,21 +2754,9 @@ def exceeds_recursion_limit(): Py_TRACE_REFS = hasattr(sys, 'getobjects') -# Decorator to disable optimizer while a function run -def without_optimizer(func): - try: - from _testinternalcapi import get_optimizer, set_optimizer - except ImportError: - return func - @functools.wraps(func) - def wrapper(*args, **kwargs): - save_opt = get_optimizer() - try: - set_optimizer(None) - return func(*args, **kwargs) - finally: - set_optimizer(save_opt) - return wrapper +_JIT_ENABLED = sys._jit.is_enabled() +requires_jit_enabled = unittest.skipUnless(_JIT_ENABLED, "requires JIT enabled") +requires_jit_disabled = unittest.skipIf(_JIT_ENABLED, "requires JIT disabled") _BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({ @@ -2724,19 +2786,121 @@ def copy_python_src_ignore(path, names): return ignored -def iter_builtin_types(): - for obj in __builtins__.values(): - if not isinstance(obj, type): +# XXX Move this to the inspect module? +def walk_class_hierarchy(top, *, topdown=True): + # This is based on the logic in os.walk(). + assert isinstance(top, type), repr(top) + stack = [top] + while stack: + top = stack.pop() + if isinstance(top, tuple): + yield top continue - cls = obj - if cls.__module__ != 'builtins': + + subs = type(top).__subclasses__(top) + if topdown: + # Yield before subclass traversal if going top down. + yield top, subs + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + else: + # Yield after subclass traversal if going bottom up. + stack.append((top, subs)) + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + + +def iter_builtin_types(): + # First try the explicit route. + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + yield from _testinternalcapi.get_static_builtin_types() + return + + # Fall back to making a best-effort guess. + if hasattr(object, '__flags__'): + # Look for any type object with the Py_TPFLAGS_STATIC_BUILTIN flag set. + import datetime + seen = set() + for cls, subs in walk_class_hierarchy(object): + if cls in seen: + continue + seen.add(cls) + if not (cls.__flags__ & _TPFLAGS_STATIC_BUILTIN): + # Do not walk its subclasses. + subs[:] = [] + continue + yield cls + else: + # Fall back to a naive approach. + seen = set() + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + # XXX? + if cls.__module__ != 'builtins': + continue + if cls == ExceptionGroup: + # It's a heap type. + continue + if cls in seen: + continue + seen.add(cls) + yield cls + + +# XXX Move this to the inspect module? +def iter_name_in_mro(cls, name): + """Yield matching items found in base.__dict__ across the MRO. + + The descriptor protocol is not invoked. + + list(iter_name_in_mro(cls, name))[0] is roughly equivalent to + find_name_in_mro() in Objects/typeobject.c (AKA PyType_Lookup()). + + inspect.getattr_static() is similar. + """ + # This can fail if "cls" is weird. + for base in inspect._static_getmro(cls): + # This can fail if "base" is weird. + ns = inspect._get_dunder_dict_of_class(base) + try: + obj = ns[name] + except KeyError: continue - yield cls + yield obj, base -def iter_slot_wrappers(cls): - assert cls.__module__ == 'builtins', cls +# XXX Move this to the inspect module? +def find_name_in_mro(cls, name, default=inspect._sentinel): + for res in iter_name_in_mro(cls, name): + # Return the first one. + return res + if default is not inspect._sentinel: + return default, None + raise AttributeError(name) + +# XXX The return value should always be exactly the same... +def identify_type_slot_wrappers(): + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + names = {n: None for n in _testinternalcapi.identify_type_slot_wrappers()} + return list(names) + else: + raise NotImplementedError + + +def iter_slot_wrappers(cls): def is_slot_wrapper(name, value): if not isinstance(value, types.WrapperDescriptorType): assert not repr(value).startswith(' dict[str, str]: return clean_env -def initialized_with_pyrepl(): - """Detect whether PyREPL was used during Python initialization.""" - # If the main module has a __file__ attribute it's a Python module, which means PyREPL. - return hasattr(sys.modules["__main__"], "__file__") +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + import signal + + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + # Shell exit code (ex: WASI build) + if 128 < exitcode < 256: + signum = exitcode - 128 + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + return None class BrokenIter: def __init__(self, init_raises=False, next_raises=False, iter_raises=False): @@ -2849,222 +3074,173 @@ def __iter__(self): return self -def linked_to_musl(): +def in_systemd_nspawn_sync_suppressed() -> bool: """ - Test if the Python executable is linked to the musl C library. + Test whether the test suite is runing in systemd-nspawn + with ``--suppress-sync=true``. + + This can be used to skip tests that rely on ``fsync()`` calls + and similar not being intercepted. """ - if sys.platform != 'linux': + + if not hasattr(os, "O_SYNC"): return False - import subprocess - exe = getattr(sys, '_base_executable', sys.executable) - cmd = ['ldd', exe] try: - stdout = subprocess.check_output(cmd, - text=True, - stderr=subprocess.STDOUT) - except (OSError, subprocess.CalledProcessError): + with open("/run/systemd/container", "rb") as fp: + if fp.read().rstrip() != b"systemd-nspawn": + return False + except FileNotFoundError: return False - return ('musl' in stdout) + # If systemd-nspawn is used, O_SYNC flag will immediately + # trigger EINVAL. Otherwise, ENOENT will be given instead. + import errno + try: + fd = os.open(__file__, os.O_RDONLY | os.O_SYNC) + except OSError as err: + if err.errno == errno.EINVAL: + return True + else: + os.close(fd) -# TODO: RUSTPYTHON -# Every line of code below allowed us to update `Lib/test/support/__init__.py` without -# needing to update `libregtest` and its dependencies. -# Ideally we want to remove all code below and update `libregtest`. -# -# Code below was copied from: https://github.com/RustPython/RustPython/blob/9499d39f55b73535e2405bf208d5380241f79ada/Lib/test/support/__init__.py + return False -from .testresult import get_test_runner +def run_no_yield_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests -# By default, don't filter tests -_match_test_func = None +@types.coroutine +def async_yield(v): + return (yield v) -_accept_test_patterns = None -_ignore_test_patterns = None -def match_test(test): - # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) +def run_yielding_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + while True: + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() -def _is_full_match_test(pattern): - # If a pattern contains at least one dot, it's considered - # as a full test identifier. - # Example: 'test.test_os.FileTests.test_access'. - # - # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, ignore 'test_access*'. - return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) - -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) - - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function - -def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): - # Simple case: all patterns are full test identifier. - # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ - else: - import fnmatch - regex = '|'.join(map(fnmatch.translate, patterns)) - # The search *is* case sensitive on purpose: - # don't use flags=re.IGNORECASE - regex_match = re.compile(regex).match - - def match_test_regex(test_id): - if regex_match(test_id): - # The regex matches the whole identifier, for example - # 'test.test_os.FileTests.test_access'. - return True - else: - # Try to match parts of the test identifier. - # For example, split 'test.test_os.FileTests.test_access' - # into: 'test', 'test_os', 'FileTests' and 'test_access'. - return any(map(regex_match, test_id.split("."))) - - func = match_test_regex - - return patterns, func - -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - loader = unittest.TestLoader() - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(loader.loadTestsFromTestCase(cls)) - _filter_suite(suite, match_test) - return _run_suite(suite) - -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - runner = get_test_runner(sys.stdout, - verbosity=verbose, - capture_output=(junit_xml_list is not None)) - - result = runner.run(suite) - - if junit_xml_list is not None: - junit_xml_list.append(result.get_xml_element()) - - if not result.testsRun and not result.skipped and not result.errors: - raise TestDidNotRun - if not result.wasSuccessful(): - stats = TestStats.from_unittest(result) - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - errors = [(str(tc), exc_str) for tc, exc_str in result.errors] - failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures, stats=stats) - return result -@dataclasses.dataclass(slots=True) -class TestStats: - tests_run: int = 0 - failures: int = 0 - skipped: int = 0 +def is_libssl_fips_mode(): + try: + from _hashlib import get_fips_mode # ask _hashopenssl.c + except ImportError: + return False # more of a maybe, unless we add this to the _ssl module. + return get_fips_mode() != 0 - @staticmethod - def from_unittest(result): - return TestStats(result.testsRun, - len(result.failures), - len(result.skipped)) +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False - @staticmethod - def from_doctest(results): - return TestStats(results.attempted, - results.failed) + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass - def accumulate(self, stats): - self.tests_run += stats.tests_run - self.failures += stats.failures - self.skipped += stats.skipped + return PROCESS_VM_READV_SUPPORTED +def _support_remote_exec_only_impl(): + if not sys.is_remote_debug_enabled(): + return unittest.skip("Remote debugging is not enabled") + if sys.platform not in ("darwin", "linux", "win32"): + return unittest.skip("Test only runs on Linux, Windows and macOS") + if sys.platform == "linux" and not _supports_remote_attaching(): + return unittest.skip("Test only runs on Linux with process_vm_readv support") + return _id -def run_doctest(module, verbosity=None, optionflags=0): - """Run doctest on the given module. Return (#failures, #tests). +def support_remote_exec_only(test): + return _support_remote_exec_only_impl()(test) + +class EqualToForwardRef: + """Helper to ease use of annotationlib.ForwardRef in tests. + + This checks only attributes that can be set using the constructor. - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). """ - import doctest + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_class=False, + ): + self.__forward_arg__ = arg + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - results = doctest.testmod(module, - verbose=verbosity, - optionflags=optionflags) - if results.failed: - stats = TestStats.from_doctest(results) - raise TestFailed(f"{results.failed} of {results.attempted} " - f"doctests failed", - stats=stats) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, results.attempted)) - return results + def __eq__(self, other): + if not isinstance(other, (EqualToForwardRef, annotationlib.ForwardRef)): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + and self.__forward_is_class__ == other.__forward_is_class__ + and self.__owner__ == other.__owner__ + ) + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"EqualToForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_linked_to_musl = None +def linked_to_musl(): + """ + Report if the Python executable is linked to the musl C library. + + Return False if we don't think it is, or a version triple otherwise. + """ + # This is can be a relatively expensive check, so we use a cache. + global _linked_to_musl + if _linked_to_musl is not None: + return _linked_to_musl + + # emscripten (at least as far as we're concerned) and wasi use musl, + # but platform doesn't know how to get the version, so set it to zero. + if is_wasm32: + _linked_to_musl = (0, 0, 0) + return _linked_to_musl + + # On all other non-linux platforms assume no musl. + if sys.platform != 'linux': + _linked_to_musl = False + return _linked_to_musl + + # On linux, we'll depend on the platform module to do the check, so new + # musl platforms should add support in that module if possible. + import platform + lib, version = platform.libc_ver() + if lib != 'musl': + _linked_to_musl = False + return _linked_to_musl + _linked_to_musl = tuple(map(int, version.split('.'))) + return _linked_to_musl + + +def control_characters_c0() -> list[str]: + """Returns a list of C0 control characters as strings. + C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. + """ + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py index 6ba5bb814b9..9a57c309616 100644 --- a/Lib/test/support/_hypothesis_stubs/__init__.py +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -1,6 +1,6 @@ -from enum import Enum import functools import unittest +from enum import Enum __all__ = [ "given", diff --git a/Lib/test/support/ast_helper.py b/Lib/test/support/ast_helper.py index 8a0415b6aae..98eaf0b2721 100644 --- a/Lib/test/support/ast_helper.py +++ b/Lib/test/support/ast_helper.py @@ -1,5 +1,6 @@ import ast + class ASTTestMixin: """Test mixing to have basic assertions for AST nodes.""" @@ -16,6 +17,9 @@ def traverse_compare(a, b, missing=object()): self.fail(f"{type(a)!r} is not {type(b)!r}") if isinstance(a, ast.AST): for field in a._fields: + if isinstance(a, ast.Constant) and field == "kind": + # Skip the 'kind' field for ast.Constant + continue value1 = getattr(a, field, missing) value2 = getattr(b, field, missing) # Singletons are equal by definition, so further diff --git a/Lib/test/support/asynchat.py b/Lib/test/support/asynchat.py index 38c47a1fda6..a8c6b28a9e1 100644 --- a/Lib/test/support/asynchat.py +++ b/Lib/test/support/asynchat.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/asyncore.py b/Lib/test/support/asyncore.py index b397aca5568..658c22fdcee 100644 --- a/Lib/test/support/asyncore.py +++ b/Lib/test/support/asyncore.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 @@ -51,17 +51,27 @@ sophisticated high-performance network servers and clients a snap. """ +import os import select import socket import sys import time import warnings - -import os -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ - errorcode - +from errno import ( + EAGAIN, + EALREADY, + EBADF, + ECONNABORTED, + ECONNRESET, + EINPROGRESS, + EINVAL, + EISCONN, + ENOTCONN, + EPIPE, + ESHUTDOWN, + EWOULDBLOCK, + errorcode, +) _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF}) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 85bcd1f0f1c..4a3c8c2c4f1 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -1,9 +1,10 @@ """bytecode_helper - support tools for testing correct bytecode generation""" -import unittest import dis import io import opcode +import unittest + try: import _testinternalcapi except ImportError: @@ -71,7 +72,7 @@ class Label: def assertInstructionsMatch(self, actual_seq, expected): # get an InstructionSequence and an expected list, where each - # entry is a label or an instruction tuple. Construct an expcted + # entry is a label or an instruction tuple. Construct an expected # instruction sequence and compare with the one given. self.assertIsInstance(expected, list) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/channels.py similarity index 73% rename from Lib/test/support/interpreters/channels.py rename to Lib/test/support/channels.py index d2bd93d77f7..3f7b46030fd 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -1,19 +1,23 @@ """Cross-interpreter Channels High Level Module.""" import time +from concurrent.interpreters import _crossinterp +from concurrent.interpreters._crossinterp import ( + UNBOUND_ERROR, + UNBOUND_REMOVE, +) + import _interpchannels as _channels -from . import _crossinterp # aliases: from _interpchannels import ( - ChannelError, ChannelNotFoundError, ChannelClosedError, - ChannelEmptyError, ChannelNotEmptyError, -) -from ._crossinterp import ( - UNBOUND_ERROR, UNBOUND_REMOVE, + ChannelClosedError, + ChannelEmptyError, + ChannelError, + ChannelNotEmptyError, + ChannelNotFoundError, ) - __all__ = [ 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', @@ -55,15 +59,23 @@ def create(*, unbounditems=UNBOUND): """ unbound = _serialize_unbound(unbounditems) unboundop, = unbound - cid = _channels.create(unboundop) - recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) + cid = _channels.create(unboundop, -1) + recv, send = RecvChannel(cid), SendChannel(cid) + send._set_unbound(unboundop, unbounditems) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) - for cid, unbound in _channels.list_all()] + channels = [] + for cid, unboundop, _ in _channels.list_all(): + chan = _, send = RecvChannel(cid), SendChannel(cid) + if not hasattr(send, '_unboundop'): + send._set_unbound(unboundop) + else: + assert send._unbound[0] == unboundop + channels.append(chan) + return channels class _ChannelEnd: @@ -97,12 +109,8 @@ def __eq__(self, other): return other._id == self._id # for pickling: - def __getnewargs__(self): - return (int(self._id),) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (int(self._id),)) @property def id(self): @@ -175,16 +183,33 @@ class SendChannel(_ChannelEnd): _end = 'send' - def __new__(cls, cid, *, _unbound=None): - if _unbound is None: - try: - op = _channels.get_channel_defaults(cid) - _unbound = (op,) - except ChannelNotFoundError: - _unbound = _serialize_unbound(UNBOUND) - self = super().__new__(cls, cid) - self._unbound = _unbound - return self +# def __new__(cls, cid, *, _unbound=None): +# if _unbound is None: +# try: +# op = _channels.get_channel_defaults(cid) +# _unbound = (op,) +# except ChannelNotFoundError: +# _unbound = _serialize_unbound(UNBOUND) +# self = super().__new__(cls, cid) +# self._unbound = _unbound +# return self + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _channels.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items @property def is_closed(self): @@ -192,61 +217,61 @@ def is_closed(self): return info.closed or info.closing def send(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. return _channels.send(self._id, obj, unboundop, blocking=False) def send_buffer(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send_buffer(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_buffer_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index a4e6c92203a..75dc2ba7506 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -1,51 +1,330 @@ import functools import hashlib +import importlib import unittest +from test.support.import_helper import import_module + try: import _hashlib except ImportError: _hashlib = None +try: + import _hmac +except ImportError: + _hmac = None + + +def requires_hashlib(): + return unittest.skipIf(_hashlib is None, "requires _hashlib") + + +def requires_builtin_hmac(): + return unittest.skipIf(_hmac is None, "requires _hmac") + + +def _missing_hash(digestname, implementation=None, *, exc=None): + parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] + msg = " ".join(filter(None, parts)) + raise unittest.SkipTest(msg) from exc + + +def _openssl_availabillity(digestname, *, usedforsecurity): + try: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + except AttributeError: + assert _hashlib is None + _missing_hash(digestname, "OpenSSL") + except ValueError as exc: + _missing_hash(digestname, "OpenSSL", exc=exc) + + +def _decorate_func_or_class(func_or_class, decorator_func): + if not isinstance(func_or_class, type): + return decorator_func(func_or_class) + + decorated_class = func_or_class + setUpClass = decorated_class.__dict__.get('setUpClass') + if setUpClass is None: + def setUpClass(cls): + super(decorated_class, cls).setUpClass() + setUpClass.__qualname__ = decorated_class.__qualname__ + '.setUpClass' + setUpClass.__module__ = decorated_class.__module__ + else: + setUpClass = setUpClass.__func__ + setUpClass = classmethod(decorator_func(setUpClass)) + decorated_class.setUpClass = setUpClass + return decorated_class + def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): - """Decorator raising SkipTest if a hashing algorithm is not available + """Decorator raising SkipTest if a hashing algorithm is not available. - The hashing algorithm could be missing or blocked by a strict crypto - policy. + The hashing algorithm may be missing, blocked by a strict crypto policy, + or Python may be configured with `--with-builtin-hashlib-hashes=no`. If 'openssl' is True, then the decorator checks that OpenSSL provides - the algorithm. Otherwise the check falls back to built-in - implementations. The usedforsecurity flag is passed to the constructor. + the algorithm. Otherwise the check falls back to (optional) built-in + HACL* implementations. + The usedforsecurity flag is passed to the constructor but has no effect + on HACL* implementations. + + Examples of exceptions being suppressed: ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ + if openssl and _hashlib is not None: + def test_availability(): + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + else: + def test_availability(): + hashlib.new(digestname, usedforsecurity=usedforsecurity) + + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + test_availability() + except ValueError as exc: + _missing_hash(digestname, exc=exc) + return func(*args, **kwargs) + return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): + """Decorator raising SkipTest if an OpenSSL hashing algorithm is missing. + + The hashing algorithm may be missing or blocked by a strict crypto policy. + """ + def decorator_func(func): + @requires_hashlib() # avoid checking at each call + @functools.wraps(func) + def wrapper(*args, **kwargs): + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + return func(*args, **kwargs) + return wrapper + def decorator(func_or_class): - if isinstance(func_or_class, type): - setUpClass = func_or_class.__dict__.get('setUpClass') - if setUpClass is None: - def setUpClass(cls): - super(func_or_class, cls).setUpClass() - setUpClass.__qualname__ = func_or_class.__qualname__ + '.setUpClass' - setUpClass.__module__ = func_or_class.__module__ - else: - setUpClass = setUpClass.__func__ - setUpClass = classmethod(decorator(setUpClass)) - func_or_class.setUpClass = setUpClass - return func_or_class - - @functools.wraps(func_or_class) + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): + """Find the OpenSSL hash function constructor by its name.""" + assert isinstance(digestname, str), digestname + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return getattr(_hashlib, f"openssl_{digestname}") + + +def requires_builtin_hashdigest( + module_name, digestname, *, usedforsecurity=True +): + """Decorator raising SkipTest if a HACL* hashing algorithm is missing. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + def decorator_func(func): + @functools.wraps(func) def wrapper(*args, **kwargs): + module = import_module(module_name) try: - if openssl and _hashlib is not None: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - else: - hashlib.new(digestname, usedforsecurity=usedforsecurity) - except ValueError: - raise unittest.SkipTest( - f"hash digest '{digestname}' is not available." - ) - return func_or_class(*args, **kwargs) + getattr(module, digestname) + except AttributeError: + fullname = f'{module_name}.{digestname}' + _missing_hash(fullname, implementation="HACL") + return func(*args, **kwargs) return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) return decorator + + +def find_builtin_hashdigest_constructor( + module_name, digestname, *, usedforsecurity=True +): + """Find the HACL* hash function constructor. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + module = import_module(module_name) + try: + constructor = getattr(module, digestname) + constructor(b'', usedforsecurity=usedforsecurity) + except (AttributeError, TypeError, ValueError): + _missing_hash(f'{module_name}.{digestname}', implementation="HACL") + return constructor + + +class HashFunctionsTrait: + """Mixin trait class containing hash functions. + + This class is assumed to have all unitest.TestCase methods but should + not directly inherit from it to prevent the test suite being run on it. + + Subclasses should implement the hash functions by returning an object + that can be recognized as a valid digestmod parameter for both hashlib + and HMAC. In particular, it cannot be a lambda function as it will not + be recognized by hashlib (it will still be accepted by the pure Python + implementation of HMAC). + """ + + ALGORITHMS = [ + 'md5', 'sha1', + 'sha224', 'sha256', 'sha384', 'sha512', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + ] + + # Default 'usedforsecurity' to use when looking up a hash function. + usedforsecurity = True + + def _find_constructor(self, name): + # By default, a missing algorithm skips the test that uses it. + self.assertIn(name, self.ALGORITHMS) + self.skipTest(f"missing hash function: {name}") + + @property + def md5(self): + return self._find_constructor("md5") + + @property + def sha1(self): + return self._find_constructor("sha1") + + @property + def sha224(self): + return self._find_constructor("sha224") + + @property + def sha256(self): + return self._find_constructor("sha256") + + @property + def sha384(self): + return self._find_constructor("sha384") + + @property + def sha512(self): + return self._find_constructor("sha512") + + @property + def sha3_224(self): + return self._find_constructor("sha3_224") + + @property + def sha3_256(self): + return self._find_constructor("sha3_256") + + @property + def sha3_384(self): + return self._find_constructor("sha3_384") + + @property + def sha3_512(self): + return self._find_constructor("sha3_512") + + +class NamedHashFunctionsTrait(HashFunctionsTrait): + """Trait containing named hash functions. + + Hash functions are available if and only if they are available in hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return name + + +class OpenSSLHashFunctionsTrait(HashFunctionsTrait): + """Trait containing OpenSSL hash functions. + + Hash functions are available if and only if they are available in _hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return find_openssl_hashdigest_constructor( + name, usedforsecurity=self.usedforsecurity + ) + + +class BuiltinHashFunctionsTrait(HashFunctionsTrait): + """Trait containing HACL* hash functions. + + Hash functions are available if and only if they are available in C. + In particular, HACL* HMAC-MD5 may be available even though HACL* md5 + is not since the former is unconditionally built. + """ + + def _find_constructor_in(self, module, name): + self.assertIn(name, self.ALGORITHMS) + return find_builtin_hashdigest_constructor(module, name) + + @property + def md5(self): + return self._find_constructor_in("_md5", "md5") + + @property + def sha1(self): + return self._find_constructor_in("_sha1", "sha1") + + @property + def sha224(self): + return self._find_constructor_in("_sha2", "sha224") + + @property + def sha256(self): + return self._find_constructor_in("_sha2", "sha256") + + @property + def sha384(self): + return self._find_constructor_in("_sha2", "sha384") + + @property + def sha512(self): + return self._find_constructor_in("_sha2", "sha512") + + @property + def sha3_224(self): + return self._find_constructor_in("_sha3", "sha3_224") + + @property + def sha3_256(self): + return self._find_constructor_in("_sha3","sha3_256") + + @property + def sha3_384(self): + return self._find_constructor_in("_sha3","sha3_384") + + @property + def sha3_512(self): + return self._find_constructor_in("_sha3","sha3_512") + + +def find_gil_minsize(modules_names, default=2048): + """Get the largest GIL_MINSIZE value for the given cryptographic modules. + + The valid module names are the following: + + - _hashlib + - _md5, _sha1, _sha2, _sha3, _blake2 + - _hmac + """ + sizes = [] + for module_name in modules_names: + try: + module = importlib.import_module(module_name) + except ImportError: + continue + sizes.append(getattr(module, '_GIL_MINSIZE', default)) + return max(sizes, default=default) diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index a99a4963ffe..6e9e168f63a 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -7,9 +7,10 @@ else: # Regrtest changes to use a tempdir as the working directory, so we have # to tell Hypothesis to use the original in order to persist the database. + from hypothesis.configuration import set_hypothesis_home_dir + from test.support import has_socket_support from test.support.os_helper import SAVEDCWD - from hypothesis.configuration import set_hypothesis_home_dir set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) diff --git a/Lib/test/support/i18n_helper.py b/Lib/test/support/i18n_helper.py index 2e304f29e8b..af97cdc9cb5 100644 --- a/Lib/test/support/i18n_helper.py +++ b/Lib/test/support/i18n_helper.py @@ -3,10 +3,10 @@ import sys import unittest from pathlib import Path + from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess from test.test_tools import skip_if_missing - pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 67f18e530ed..4c7eac0b7eb 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,14 +1,16 @@ import contextlib import _imp import importlib +import importlib.machinery import importlib.util import os import shutil import sys +import textwrap import unittest import warnings -from .os_helper import unlink +from .os_helper import unlink, temp_dir @contextlib.contextmanager @@ -58,8 +60,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc @@ -114,7 +116,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,9 +270,173 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak # memory. from unittest import mock return mock.patch('os.register_at_fork', create=True)(func) + + +@contextlib.contextmanager +def ready_to_import(name=None, source=""): + from test.support import script_helper + + # 1. Sets up a temporary directory and removes it afterwards + # 2. Creates the module file + # 3. Temporarily clears the module from sys.modules (if any) + # 4. Reverts or removes the module when cleaning up + name = name or "spam" + with temp_dir() as tempdir: + path = script_helper.make_script(tempdir, name, source) + old_module = sys.modules.pop(name, None) + try: + sys.path.insert(0, tempdir) + yield name, path + finally: + sys.path.remove(tempdir) + if old_module is not None: + sys.modules[name] = old_module + else: + sys.modules.pop(name, None) + + +def ensure_lazy_imports(imported_module, modules_to_block): + """Test that when imported_module is imported, none of the modules in + modules_to_block are imported as a side effect.""" + modules_to_block = frozenset(modules_to_block) + script = textwrap.dedent( + f""" + import sys + modules_to_block = {modules_to_block} + if unexpected := modules_to_block & sys.modules.keys(): + startup = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported at startup: {{startup}}') + + import {imported_module} + if unexpected := modules_to_block & sys.modules.keys(): + after = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}') + """ + ) + from .script_helper import assert_python_ok + assert_python_ok("-S", "-c", script) + + +@contextlib.contextmanager +def module_restored(name): + """A context manager that restores a module to the original state.""" + missing = object() + orig = sys.modules.get(name, missing) + if orig is None: + mod = importlib.import_module(name) + else: + mod = type(sys)(name) + mod.__dict__.update(orig.__dict__) + sys.modules[name] = mod + try: + yield mod + finally: + if orig is missing: + sys.modules.pop(name, None) + else: + sys.modules[name] = orig + + +def create_module(name, loader=None, *, ispkg=False): + """Return a new, empty module.""" + spec = importlib.machinery.ModuleSpec( + name, + loader, + origin='', + is_package=ispkg, + ) + return importlib.util.module_from_spec(spec) + + +def _ensure_module(name, ispkg, addparent, clearnone): + try: + mod = orig = sys.modules[name] + except KeyError: + mod = orig = None + missing = True + else: + missing = False + if mod is not None: + # It was already imported. + return mod, orig, missing + # Otherwise, None means it was explicitly disabled. + + assert name != '__main__' + if not missing: + assert orig is None, (name, sys.modules[name]) + if not clearnone: + raise ModuleNotFoundError(name) + del sys.modules[name] + # Try normal import, then fall back to adding the module. + try: + mod = importlib.import_module(name) + except ModuleNotFoundError: + if addparent and not clearnone: + addparent = None + mod = _add_module(name, ispkg, addparent) + return mod, orig, missing + + +def _add_module(spec, ispkg, addparent): + if isinstance(spec, str): + name = spec + mod = create_module(name, ispkg=ispkg) + spec = mod.__spec__ + else: + name = spec.name + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + if addparent is not False and spec.parent: + _ensure_module(spec.parent, True, addparent, bool(addparent)) + return mod + + +def add_module(spec, *, parents=True): + """Return the module after creating it and adding it to sys.modules. + + If parents is True then also create any missing parents. + """ + return _add_module(spec, False, parents) + + +def add_package(spec, *, parents=True): + """Return the module after creating it and adding it to sys.modules. + + If parents is True then also create any missing parents. + """ + return _add_module(spec, True, parents) + + +def ensure_module_imported(name, *, clearnone=True): + """Return the corresponding module. + + If it was already imported then return that. Otherwise, try + importing it (optionally clear it first if None). If that fails + then create a new empty module. + + It can be helpful to combine this with ready_to_import() and/or + isolated_modules(). + """ + if sys.modules.get(name) is not None: + mod = sys.modules[name] + else: + mod, _, _ = _ensure_module(name, False, True, clearnone) + return mod diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py deleted file mode 100644 index 5c484d1170d..00000000000 --- a/Lib/test/support/interpreters.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Subinterpreters High Level Module.""" - -import time -import _xxsubinterpreters as _interpreters -import _xxinterpchannels as _channels - -# aliases: -from _xxsubinterpreters import is_shareable, RunFailedError -from _xxinterpchannels import ( - ChannelError, ChannelNotFoundError, ChannelEmptyError, -) - - -__all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] - - -def create(*, isolated=True): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -class Interpreter: - """A single Python interpreter.""" - - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated - - def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - - @property - def id(self): - return self._id - - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - def run(self, src_str, /, *, channels=None): - """Run the given source code in the interpreter. - - This blocks the current Python thread until done. - """ - _interpreters.run_string(self._id, src_str, channels) - - -def create_channel(): - """Return (recv, send) for a new cross-interpreter channel. - - The channel may be used to pass data safely between interpreters. - """ - cid = _channels.create() - recv, send = RecvChannel(cid), SendChannel(cid) - return recv, send - - -def list_all_channels(): - """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid)) - for cid in _channels.list_all()] - - -class _ChannelEnd: - """The base class for RecvChannel and SendChannel.""" - - def __init__(self, id): - if not isinstance(id, (int, _channels.ChannelID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - - def __repr__(self): - return f'{type(self).__name__}(id={int(self._id)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if isinstance(self, RecvChannel): - if not isinstance(other, RecvChannel): - return NotImplemented - elif not isinstance(other, SendChannel): - return NotImplemented - return other._id == self._id - - @property - def id(self): - return self._id - - -_NOT_SET = object() - - -class RecvChannel(_ChannelEnd): - """The receiving end of a cross-interpreter channel.""" - - def recv(self, *, _sentinel=object(), _delay=10 / 1000): # 10 milliseconds - """Return the next object from the channel. - - This blocks until an object has been sent, if none have been - sent already. - """ - obj = _channels.recv(self._id, _sentinel) - while obj is _sentinel: - time.sleep(_delay) - obj = _channels.recv(self._id, _sentinel) - return obj - - def recv_nowait(self, default=_NOT_SET): - """Return the next object from the channel. - - If none have been sent then return the default if one - is provided or fail with ChannelEmptyError. Otherwise this - is the same as recv(). - """ - if default is _NOT_SET: - return _channels.recv(self._id) - else: - return _channels.recv(self._id, default) - - -class SendChannel(_ChannelEnd): - """The sending end of a cross-interpreter channel.""" - - def send(self, obj): - """Send the object (i.e. its data) to the channel's receiving end. - - This blocks until the object is received. - """ - _channels.send(self._id, obj) - # XXX We are missing a low-level channel_send_wait(). - # See bpo-32604 and gh-19829. - # Until that shows up we fake it: - time.sleep(2) - - def send_nowait(self, obj): - """Send the object to the channel's receiving end. - - If the object is immediately received then return True - (else False). Otherwise this is the same as send(). - """ - # XXX Note that at the moment channel_send() only ever returns - # None. This should be fixed when channel_send_wait() is added. - # See bpo-32604 and gh-19829. - return _channels.send(self._id, obj) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py deleted file mode 100644 index e067f259364..00000000000 --- a/Lib/test/support/interpreters/__init__.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Subinterpreters High Level Module.""" - -import threading -import weakref -import _interpreters - -# aliases: -from _interpreters import ( - InterpreterError, InterpreterNotFoundError, NotShareableError, - is_shareable, -) - - -__all__ = [ - 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', - 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', - 'NotShareableError', - 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', -] - - -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - -_EXEC_FAILURE_STR = """ -{superstr} - -Uncaught in the interpreter: - -{formatted} -""".strip() - -class ExecutionFailed(InterpreterError): - """An unhandled exception happened during execution. - - This is raised from Interpreter.exec() and Interpreter.call(). - """ - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f'{excinfo.type.__name__}: {excinfo.msg}' - else: - msg = excinfo.type.__name__ or excinfo.msg - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self): - try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return _EXEC_FAILURE_STR.format( - superstr=super().__str__(), - formatted=formatted, - ) - - -def create(): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(reqrefs=True) - return Interpreter(id, _ownsref=True) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id, _whence=whence) - for id, whence in _interpreters.list_all(require_ready=True)] - - -def get_current(): - """Return the currently running interpreter.""" - id, whence = _interpreters.get_current() - return Interpreter(id, _whence=whence) - - -def get_main(): - """Return the main interpreter.""" - id, whence = _interpreters.get_main() - assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) - return Interpreter(id, _whence=whence) - - -_known = weakref.WeakValueDictionary() - -class Interpreter: - """A single Python interpreter. - - Attributes: - - "id" - the unique process-global ID number for the interpreter - "whence" - indicates where the interpreter was created - - If the interpreter wasn't created by this module - then any method that modifies the interpreter will fail, - i.e. .close(), .prepare_main(), .exec(), and .call() - """ - - _WHENCE_TO_STR = { - _interpreters.WHENCE_UNKNOWN: 'unknown', - _interpreters.WHENCE_RUNTIME: 'runtime init', - _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', - _interpreters.WHENCE_CAPI: 'C-API', - _interpreters.WHENCE_XI: 'cross-interpreter C-API', - _interpreters.WHENCE_STDLIB: '_interpreters module', - } - - def __new__(cls, id, /, _whence=None, _ownsref=None): - # There is only one instance for any given ID. - if not isinstance(id, int): - raise TypeError(f'id must be an int, got {id!r}') - id = int(id) - if _whence is None: - if _ownsref: - _whence = _interpreters.WHENCE_STDLIB - else: - _whence = _interpreters.whence(id) - assert _whence in cls._WHENCE_TO_STR, repr(_whence) - if _ownsref is None: - _ownsref = (_whence == _interpreters.WHENCE_STDLIB) - try: - self = _known[id] - assert hasattr(self, '_ownsref') - except KeyError: - self = super().__new__(cls) - _known[id] = self - self._id = id - self._whence = _whence - self._ownsref = _ownsref - if _ownsref: - # This may raise InterpreterNotFoundError: - _interpreters.incref(id) - return self - - def __repr__(self): - return f'{type(self).__name__}({self.id})' - - def __hash__(self): - return hash(self._id) - - def __del__(self): - self._decref() - - # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None - - def _decref(self): - if not self._ownsref: - return - self._ownsref = False - try: - _interpreters.decref(self._id) - except InterpreterNotFoundError: - pass - - @property - def id(self): - return self._id - - @property - def whence(self): - return self._WHENCE_TO_STR[self._whence] - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - # Everything past here is available only to interpreters created by - # interpreters.create(). - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in an InterpreterError. - """ - return _interpreters.destroy(self._id, restrict=True) - - def prepare_main(self, ns=None, /, **kwargs): - """Bind the given values into the interpreter's __main__. - - The values must be shareable. - """ - ns = dict(ns, **kwargs) if ns is not None else kwargs - _interpreters.set___main___attrs(self._id, ns, restrict=True) - - def exec(self, code, /): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then an ExecutionFailed - exception is raised, which summarizes the unhandled exception. - The actual exception is discarded because objects cannot be - shared between interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - excinfo = _interpreters.exec(self._id, code, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - - def call(self, callable, /): - """Call the object in the interpreter with given args/kwargs. - - Only functions that take no arguments and have no closure - are supported. - - The return value is discarded. - - If the callable raises an exception then the error display - (including full traceback) is send back between the interpreters - and an ExecutionFailed exception is raised, much like what - happens with Interpreter.exec(). - """ - # XXX Support args and kwargs. - # XXX Support arbitrary callables. - # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - - def call_in_thread(self, callable, /): - """Return a new thread that calls the object in the interpreter. - - The return value and any raised exception are discarded. - """ - def task(): - self.call(callable) - t = threading.Thread(target=task) - t.start() - return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py deleted file mode 100644 index 544e197ba4c..00000000000 --- a/Lib/test/support/interpreters/_crossinterp.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Common code between queues and channels.""" - - -class ItemInterpreterDestroyed(Exception): - """Raised when trying to get an item whose interpreter was destroyed.""" - - -class classonly: - """A non-data descriptor that makes a value only visible on the class. - - This is like the "classmethod" builtin, but does not show up on - instances of the class. It may be used as a decorator. - """ - - def __init__(self, value): - self.value = value - self.getter = classmethod(value).__get__ - self.name = None - - def __set_name__(self, cls, name): - if self.name is not None: - raise TypeError('already used') - self.name = name - - def __get__(self, obj, cls): - if obj is not None: - raise AttributeError(self.name) - # called on the class - return self.getter(None, cls) - - -class UnboundItem: - """Represents a cross-interpreter item no longer bound to an interpreter. - - An item is unbound when the interpreter that added it to the - cross-interpreter container is destroyed. - """ - - __slots__ = () - - @classonly - def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) - subclass = type( - f'Unbound{kind.capitalize()}Item', - (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), - ) - return object.__new__(subclass) - - _MODULE = __name__ - _NAME = 'UNBOUND' - - def __new__(cls): - raise Exception(f'use {cls._MODULE}.{cls._NAME}') - - def __repr__(self): - return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' - - -UNBOUND = object.__new__(UnboundItem) -UNBOUND_ERROR = object() -UNBOUND_REMOVE = object() - -_UNBOUND_CONSTANT_TO_FLAG = { - UNBOUND_REMOVE: 1, - UNBOUND_ERROR: 2, - UNBOUND: 3, -} -_UNBOUND_FLAG_TO_CONSTANT = {v: k - for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} - - -def serialize_unbound(unbound): - op = unbound - try: - flag = _UNBOUND_CONSTANT_TO_FLAG[op] - except KeyError: - raise NotImplementedError(f'unsupported unbound replacement op {op!r}') - return flag, - - -def resolve_unbound(flag, exctype_destroyed): - try: - op = _UNBOUND_FLAG_TO_CONSTANT[flag] - except KeyError: - raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') - if op is UNBOUND_REMOVE: - # "remove" not possible here - raise NotImplementedError - elif op is UNBOUND_ERROR: - raise exctype_destroyed("item's original interpreter destroyed") - elif op is UNBOUND: - return UNBOUND - else: - raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py deleted file mode 100644 index deb8e8613af..00000000000 --- a/Lib/test/support/interpreters/queues.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Cross-interpreter Queues High Level Module.""" - -import pickle -import queue -import time -import weakref -import _interpqueues as _queues -from . import _crossinterp - -# aliases: -from _interpqueues import ( - QueueError, QueueNotFoundError, -) -from ._crossinterp import ( - UNBOUND_ERROR, UNBOUND_REMOVE, -) - -__all__ = [ - 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', - 'create', 'list_all', - 'Queue', - 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', - 'ItemInterpreterDestroyed', -] - - -class QueueEmpty(QueueError, queue.Empty): - """Raised from get_nowait() when the queue is empty. - - It is also raised from get() if it times out. - """ - - -class QueueFull(QueueError, queue.Full): - """Raised from put_nowait() when the queue is full. - - It is also raised from put() if it times out. - """ - - -class ItemInterpreterDestroyed(QueueError, - _crossinterp.ItemInterpreterDestroyed): - """Raised from get() and get_nowait().""" - - -_SHARED_ONLY = 0 -_PICKLED = 1 - - -UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) - - -def _serialize_unbound(unbound): - if unbound is UNBOUND: - unbound = _crossinterp.UNBOUND - return _crossinterp.serialize_unbound(unbound) - - -def _resolve_unbound(flag): - resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) - if resolved is _crossinterp.UNBOUND: - resolved = UNBOUND - return resolved - - -def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): - """Return a new cross-interpreter queue. - - The queue may be used to pass data safely between interpreters. - - "syncobj" sets the default for Queue.put() - and Queue.put_nowait(). - - "unbounditems" likewise sets the default. See Queue.put() for - supported values. The default value is UNBOUND, which replaces - the unbound item. - """ - fmt = _SHARED_ONLY if syncobj else _PICKLED - unbound = _serialize_unbound(unbounditems) - unboundop, = unbound - qid = _queues.create(maxsize, fmt, unboundop) - return Queue(qid, _fmt=fmt, _unbound=unbound) - - -def list_all(): - """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) - for qid, fmt, unboundop in _queues.list_all()] - - -_known_queues = weakref.WeakValueDictionary() - -class Queue: - """A cross-interpreter queue.""" - - def __new__(cls, id, /, *, _fmt=None, _unbound=None): - # There is only one instance for any given ID. - if isinstance(id, int): - id = int(id) - else: - raise TypeError(f'id must be an int, got {id!r}') - if _fmt is None: - if _unbound is None: - _fmt, op = _queues.get_queue_defaults(id) - _unbound = (op,) - else: - _fmt, _ = _queues.get_queue_defaults(id) - elif _unbound is None: - _, op = _queues.get_queue_defaults(id) - _unbound = (op,) - try: - self = _known_queues[id] - except KeyError: - self = super().__new__(cls) - self._id = id - self._fmt = _fmt - self._unbound = _unbound - _known_queues[id] = self - _queues.bind(id) - return self - - def __del__(self): - try: - _queues.release(self._id) - except QueueNotFoundError: - pass - try: - del _known_queues[self._id] - except KeyError: - pass - - def __repr__(self): - return f'{type(self).__name__}({self.id})' - - def __hash__(self): - return hash(self._id) - - # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None - - @property - def id(self): - return self._id - - @property - def maxsize(self): - try: - return self._maxsize - except AttributeError: - self._maxsize = _queues.get_maxsize(self._id) - return self._maxsize - - def empty(self): - return self.qsize() == 0 - - def full(self): - return _queues.is_full(self._id) - - def qsize(self): - return _queues.get_count(self._id) - - def put(self, obj, timeout=None, *, - syncobj=None, - unbound=None, - _delay=10 / 1000, # 10 milliseconds - ): - """Add the object to the queue. - - This blocks while the queue is full. - - If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue(). - - If "syncobj" is false then all objects are supported, - at the expense of worse performance. - - If "syncobj" is true then the object must be "shareable". - Examples of "shareable" objects include the builtin singletons, - str, and memoryview. One benefit is that such objects are - passed through the queue efficiently. - - The key difference, though, is conceptual: the corresponding - object returned from Queue.get() will be strictly equivalent - to the given obj. In other words, the two objects will be - effectively indistinguishable from each other, even if the - object is mutable. The received object may actually be the - same object, or a copy (immutable values only), or a proxy. - Regardless, the received object should be treated as though - the original has been shared directly, whether or not it - actually is. That's a slightly different and stronger promise - than just (initial) equality, which is all "syncobj=False" - can promise. - - "unbound" controls the behavior of Queue.get() for the given - object if the current interpreter (calling put()) is later - destroyed. - - If "unbound" is None (the default) then it uses the - queue's default, set with create_queue(), - which is usually UNBOUND. - - If "unbound" is UNBOUND_ERROR then get() will raise an - ItemInterpreterDestroyed exception if the original interpreter - has been destroyed. This does not otherwise affect the queue; - the next call to put() will work like normal, returning the next - item in the queue. - - If "unbound" is UNBOUND_REMOVE then the item will be removed - from the queue as soon as the original interpreter is destroyed. - Be aware that this will introduce an imbalance between put() - and get() calls. - - If "unbound" is UNBOUND then it is returned by get() in place - of the unbound item. - """ - if syncobj is None: - fmt = self._fmt - else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) - if timeout is not None: - timeout = int(timeout) - if timeout < 0: - raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout - if fmt is _PICKLED: - obj = pickle.dumps(obj) - while True: - try: - _queues.put(self._id, obj, fmt, unboundop) - except QueueFull as exc: - if timeout is not None and time.time() >= end: - raise # re-raise - time.sleep(_delay) - else: - break - - def put_nowait(self, obj, *, syncobj=None, unbound=None): - if syncobj is None: - fmt = self._fmt - else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) - if fmt is _PICKLED: - obj = pickle.dumps(obj) - _queues.put(self._id, obj, fmt, unboundop) - - def get(self, timeout=None, *, - _delay=10 / 1000, # 10 milliseconds - ): - """Return the next object from the queue. - - This blocks while the queue is empty. - - If the next item's original interpreter has been destroyed - then the "next object" is determined by the value of the - "unbound" argument to put(). - """ - if timeout is not None: - timeout = int(timeout) - if timeout < 0: - raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout - while True: - try: - obj, fmt, unboundop = _queues.get(self._id) - except QueueEmpty as exc: - if timeout is not None and time.time() >= end: - raise # re-raise - time.sleep(_delay) - else: - break - if unboundop is not None: - assert obj is None, repr(obj) - return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY - return obj - - def get_nowait(self): - """Return the next object from the channel. - - If the queue is empty then raise QueueEmpty. Otherwise this - is the same as get(). - """ - try: - obj, fmt, unboundop = _queues.get(self._id) - except QueueEmpty as exc: - raise # re-raise - if unboundop is not None: - assert obj is None, repr(obj) - return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY - return obj - - -_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/support/logging_helper.py b/Lib/test/support/logging_helper.py index 12fcca4f0f0..db556c7f5ad 100644 --- a/Lib/test/support/logging_helper.py +++ b/Lib/test/support/logging_helper.py @@ -1,5 +1,6 @@ import logging.handlers + class TestHandler(logging.handlers.BufferingHandler): def __init__(self, matcher): # BufferingHandler takes a "capacity" argument diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 70161e90132..d3d6fa632f9 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -1,6 +1,7 @@ import collections.abc import contextlib import errno +import logging import os import re import stat @@ -10,10 +11,8 @@ import unittest import warnings -# From CPython 3.13.5 from test import support - # Filename used for testing TESTFN_ASCII = '@test' @@ -23,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -49,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) @@ -199,10 +198,8 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) -# From CPython 3.13.5 _can_hardlink = None -# From CPython 3.13.5 def can_hardlink(): global _can_hardlink if _can_hardlink is None: @@ -212,7 +209,6 @@ def can_hardlink(): return _can_hardlink -# From CPython 3.13.5 def skip_unless_hardlink(test): ok = can_hardlink() msg = "requires hardlink support" @@ -268,15 +264,15 @@ def can_chmod(): global _can_chmod if _can_chmod is not None: return _can_chmod - if not hasattr(os, "chown"): + if not hasattr(os, "chmod"): _can_chmod = False return _can_chmod try: with open(TESTFN, "wb") as f: try: - os.chmod(TESTFN, 0o777) + os.chmod(TESTFN, 0o555) mode1 = os.stat(TESTFN).st_mode - os.chmod(TESTFN, 0o666) + os.chmod(TESTFN, 0o777) mode2 = os.stat(TESTFN).st_mode except OSError as e: can = False @@ -298,6 +294,33 @@ def skip_unless_working_chmod(test): return test if ok else unittest.skip(msg)(test) +@contextlib.contextmanager +def save_mode(path, *, quiet=False): + """Context manager that restores the mode (permissions) of *path* on exit. + + Arguments: + + path: Path of the file to restore the mode of. + + quiet: if False (the default), the context manager raises an exception + on error. Otherwise, it issues only a warning and keeps the current + working directory the same. + + """ + saved_mode = os.stat(path) + try: + yield + finally: + try: + os.chmod(path, saved_mode.st_mode) + except OSError as exc: + if not quiet: + raise + warnings.warn(f'tests may fail, unable to restore the mode of ' + f'{path!r} to {saved_mode.st_mode}: {exc}', + RuntimeWarning, stacklevel=3) + + # Check whether the current effective user has the capability to override # DAC (discretionary access control). Typically user root is able to # bypass file read, write, and execute permission checks. The capability @@ -323,6 +346,10 @@ def can_dac_override(): else: _can_dac_override = True finally: + try: + os.chmod(TESTFN, 0o700) + except OSError: + pass unlink(TESTFN) return _can_dac_override @@ -378,8 +405,12 @@ def _waitfor(func, pathname, waitall=False): # Increase the timeout and try again time.sleep(timeout) timeout *= 2 - warnings.warn('tests may fail, delete still pending for ' + pathname, - RuntimeWarning, stacklevel=4) + logging.getLogger(__name__).warning( + 'tests may fail, delete still pending for %s', + pathname, + stack_info=True, + stacklevel=4, + ) def _unlink(filename): _waitfor(os.unlink, filename) @@ -494,9 +525,14 @@ def temp_dir(path=None, quiet=False): except OSError as exc: if not quiet: raise - warnings.warn(f'tests may fail, unable to create ' - f'temporary directory {path!r}: {exc}', - RuntimeWarning, stacklevel=3) + logging.getLogger(__name__).warning( + "tests may fail, unable to create temporary directory %r: %s", + path, + exc, + exc_info=exc, + stack_info=True, + stacklevel=3, + ) if dir_created: pid = os.getpid() try: @@ -527,9 +563,15 @@ def change_cwd(path, quiet=False): except OSError as exc: if not quiet: raise - warnings.warn(f'tests may fail, unable to change the current working ' - f'directory to {path!r}: {exc}', - RuntimeWarning, stacklevel=3) + logging.getLogger(__name__).warning( + 'tests may fail, unable to change the current working directory ' + 'to %r: %s', + path, + exc, + exc_info=exc, + stack_info=True, + stacklevel=3, + ) try: yield os.getcwd() finally: @@ -612,11 +654,18 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): + fd_path = "/proc/self/fd" + elif support.is_apple: + fd_path = "/dev/fd" + else: + fd_path = None + + if fd_path is not None: try: - names = os.listdir("/proc/self/fd") + names = os.listdir(fd_path) # Subtract one because listdir() internally opens a file - # descriptor to list the content of the /proc/self/fd/ directory. + # descriptor to list the content of the directory. return len(names) - 1 except FileNotFoundError: pass @@ -686,9 +735,10 @@ def temp_umask(umask): class EnvironmentVarGuard(collections.abc.MutableMapping): + """Class to help protect the environment variable properly. - """Class to help protect the environment variable properly. Can be used as - a context manager.""" + Can be used as a context manager. + """ def __init__(self): self._environ = os.environ @@ -722,7 +772,6 @@ def __len__(self): def set(self, envvar, value): self[envvar] = value - # From CPython 3.13.5 def unset(self, envvar, /, *envvars): """Unset one or more environment variables.""" for ev in (envvar, *envvars): @@ -746,13 +795,16 @@ def __exit__(self, *ignore_exc): try: - import ctypes - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - - ERROR_FILE_NOT_FOUND = 2 - DDD_REMOVE_DEFINITION = 2 - DDD_EXACT_MATCH_ON_REMOVE = 4 - DDD_NO_BROADCAST_SYSTEM = 8 + if support.MS_WINDOWS: + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + ERROR_FILE_NOT_FOUND = 2 + DDD_REMOVE_DEFINITION = 2 + DDD_EXACT_MATCH_ON_REMOVE = 4 + DDD_NO_BROADCAST_SYSTEM = 8 + else: + raise AttributeError except (ImportError, AttributeError): def subst_drive(path): raise unittest.SkipTest('ctypes or kernel32 is not available') diff --git a/Lib/test/support/pty_helper.py b/Lib/test/support/pty_helper.py index 6587fd40333..7e1ae9e59b8 100644 --- a/Lib/test/support/pty_helper.py +++ b/Lib/test/support/pty_helper.py @@ -10,6 +10,7 @@ from test.support.import_helper import import_module + def run_pty(script, input=b"dummy input\r", env=None): pty = import_module('pty') output = bytearray() diff --git a/Lib/test/support/refleak_helper.py b/Lib/test/support/refleak_helper.py new file mode 100644 index 00000000000..2f86c93a1e2 --- /dev/null +++ b/Lib/test/support/refleak_helper.py @@ -0,0 +1,8 @@ +""" +Utilities for changing test behaviour while hunting +for refleaks +""" + +_hunting_for_refleaks = False +def hunting_for_refleaks(): + return _hunting_for_refleaks diff --git a/Lib/test/support/rustpython.py b/Lib/test/support/rustpython.py new file mode 100644 index 00000000000..8ed7bc24dcf --- /dev/null +++ b/Lib/test/support/rustpython.py @@ -0,0 +1,24 @@ +""" +RustPython specific helpers. +""" + +import doctest + + +# copied from https://github.com/RustPython/RustPython/pull/6919 +EXPECTED_FAILURE = doctest.register_optionflag("EXPECTED_FAILURE") + + +class DocTestChecker(doctest.OutputChecker): + """ + Custom output checker that lets us add: `+EXPECTED_FAILURE` for doctest tests. + + We want to be able to mark failing doctest test the same way we do with normal + unit test, without this class we would have to skip the doctest for the CI to pass. + """ + + def check_output(self, want, got, optionflags): + res = super().check_output(want, got, optionflags) + if optionflags & EXPECTED_FAILURE: + res = not res + return res diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index c2b43f4060e..a338f484449 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -3,18 +3,16 @@ import collections import importlib -import sys import os import os.path -import subprocess import py_compile -import zipfile - +import subprocess +import sys from importlib.util import source_from_cache + from test import support from test.support.import_helper import make_legacy_pyc - # Cached result of the expensive test performed in the function below. __cached_interp_requires_environment = None @@ -64,42 +62,59 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] if len(err) > maxlen: err = b'(... truncated stderr ...)' + err[-maxlen:] - out = out.decode('ascii', 'replace').rstrip() - err = err.decode('ascii', 'replace').rstrip() - raise AssertionError("Process return code is %d\n" - "command line: %r\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s\n" - "---" - % (self.rc, cmd_line, - out, - err)) + out = out.decode('utf8', 'replace').rstrip() + err = err.decode('utf8', 'replace').rstrip() + + exitcode = self.rc + signame = support.get_signal_name(exitcode) + if signame: + exitcode = f"{exitcode} ({signame})" + raise AssertionError(f"Process return code is {exitcode}\n" + f"command line: {cmd_line!r}\n" + f"\n" + f"stdout:\n" + f"---\n" + f"{out}\n" + f"---\n" + f"\n" + f"stderr:\n" + f"---\n" + f"{err}\n" + f"---") # Executing the interpreter in a subprocess @support.requires_subprocess() def run_python_until_end(*args, **env_vars): + """Used to implement assert_python_*. + + *args are the command line flags to pass to the python interpreter. + **env_vars keyword arguments are environment variables to set on the process. + + If __run_using_command= is supplied, it must be a list of + command line arguments to prepend to the command line used. + Useful when you want to run another command that should launch the + python interpreter via its own arguments. ["/bin/echo", "--"] for + example could print the unquoted python command line instead of + run it. + """ env_required = interpreter_requires_environment() + run_using_command = env_vars.pop('__run_using_command', None) cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required cmd_line = [sys.executable, '-X', 'faulthandler'] + if run_using_command: + cmd_line = run_using_command + cmd_line if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path @@ -218,14 +233,19 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): if not omit_suffix: script_filename += os.extsep + 'py' script_name = os.path.join(script_dir, script_filename) - # The script should be encoded to UTF-8, the default string encoding - with open(script_name, 'w', encoding='utf-8') as script_file: - script_file.write(source) + if isinstance(source, str): + # The script should be encoded to UTF-8, the default string encoding + with open(script_name, 'w', encoding='utf-8') as script_file: + script_file.write(source) + else: + with open(script_name, 'wb') as script_file: + script_file.write(source) importlib.invalidate_caches() return script_name def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + import zipfile zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) with zipfile.ZipFile(zip_name, 'w') as zip_file: @@ -252,6 +272,7 @@ def make_pkg(pkg_dir, init_source=''): def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source, depth=1, compiled=False): + import zipfile unlink = [] init_name = make_script(zip_dir, '__init__', '') unlink.append(init_name) diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py old mode 100644 new mode 100755 index 6052232ec2b..cf333aaf6b0 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -70,16 +70,17 @@ # - Handle more ESMTP extensions # - handle error codes from the backend smtpd -import sys -import os +import collections import errno import getopt -import time +import os import socket -import collections -from test.support import asyncore, asynchat -from warnings import warn +import sys +import time from email._header_value_parser import get_addr_spec, get_angle_addr +from warnings import warn + +from test.support import asynchat, asyncore __all__ = [ "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", @@ -633,7 +634,8 @@ def __init__(self, localaddr, remoteaddr, " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: - gai_results = socket.getaddrinfo(*localaddr, + family = 0 if socket.has_ipv6 else socket.AF_INET + gai_results = socket.getaddrinfo(*localaddr, family=family, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible @@ -672,9 +674,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -685,7 +687,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 87941ee1791..655ffbea0db 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -2,8 +2,8 @@ import errno import os.path import socket -import sys import subprocess +import sys import tempfile import unittest @@ -259,6 +259,10 @@ def filter_error(err): # raise OSError('socket error', msg) from msg elif len(a) >= 2 and isinstance(a[1], OSError): err = a[1] + # The error can also be wrapped as __cause__: + # raise URLError(f"ftp error: {exp}") from exp + elif isinstance(err, urllib.error.URLError) and err.__cause__: + err = err.__cause__ else: break filter_error(err) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py new file mode 100644 index 00000000000..abc93dee2ce --- /dev/null +++ b/Lib/test/support/strace_helper.py @@ -0,0 +1,210 @@ +import os +import re +import sys +import textwrap +import unittest +from dataclasses import dataclass +from functools import cache + +from test import support +from test.support.script_helper import run_python_until_end + +_strace_binary = "/usr/bin/strace" +_syscall_regex = re.compile( + r"(?P[^(]*)\((?P[^)]*)\)\s*[=]\s*(?P.+)") +_returncode_regex = re.compile( + br"\+\+\+ exited with (?P\d+) \+\+\+") + + +@dataclass +class StraceEvent: + syscall: str + args: list[str] + returncode: str + + +@dataclass +class StraceResult: + strace_returncode: int + python_returncode: int + + """The event messages generated by strace. This is very similar to the + stderr strace produces with returncode marker section removed.""" + event_bytes: bytes + stdout: bytes + stderr: bytes + + def events(self): + """Parse event_bytes data into system calls for easier processing. + + This assumes the program under inspection doesn't print any non-utf8 + strings which would mix into the strace output.""" + decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape') + matches = [ + _syscall_regex.match(event) + for event in decoded_events.splitlines() + ] + return [ + StraceEvent(match["syscall"], + [arg.strip() for arg in (match["args"].split(","))], + match["returncode"]) for match in matches if match + ] + + def sections(self): + """Find all "MARK " writes and use them to make groups of events. + + This is useful to avoid variable / overhead events, like those at + interpreter startup or when opening a file so a test can verify just + the small case under study.""" + current_section = "__startup" + sections = {current_section: []} + for event in self.events(): + if event.syscall == 'write' and len( + event.args) > 2 and event.args[1].startswith("\"MARK "): + # Found a new section, don't include the write in the section + # but all events until next mark should be in that section + current_section = event.args[1].split( + " ", 1)[1].removesuffix('\\n"') + if current_section not in sections: + sections[current_section] = list() + else: + sections[current_section].append(event) + + return sections + +def _filter_memory_call(call): + # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. + # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. + if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + return True + + if call.syscall in ("munmap", "mprotect"): + return True + + return False + + +def filter_memory(syscalls): + """Filter out memory allocation calls from File I/O calls. + + Some calls (mmap, munmap, etc) can be used on files or to just get a block + of memory. Use this function to filter out the memory related calls from + other calls.""" + + return [call for call in syscalls if not _filter_memory_call(call)] + + +@support.requires_subprocess() +def strace_python(code, strace_flags, check=True): + """Run strace and return the trace. + + Sets strace_returncode and python_returncode to `-1` on error.""" + res = None + + def _make_error(reason, details): + return StraceResult( + strace_returncode=-1, + python_returncode=-1, + event_bytes= f"error({reason},details={details!r}) = -1".encode('utf-8'), + stdout=res.out if res else b"", + stderr=res.err if res else b"") + + # Run strace, and get out the raw text + try: + res, cmd_line = run_python_until_end( + "-c", + textwrap.dedent(code), + __run_using_command=[_strace_binary] + strace_flags, + ) + except OSError as err: + return _make_error("Caught OSError", err) + + if check and res.rc: + res.fail(cmd_line) + + # Get out program returncode + stripped = res.err.strip() + output = stripped.rsplit(b"\n", 1) + if len(output) != 2: + return _make_error("Expected strace events and exit code line", + stripped[-50:]) + + returncode_match = _returncode_regex.match(output[1]) + if not returncode_match: + return _make_error("Expected to find returncode in last line.", + output[1][:50]) + + python_returncode = int(returncode_match["returncode"]) + if check and python_returncode: + res.fail(cmd_line) + + return StraceResult(strace_returncode=res.rc, + python_returncode=python_returncode, + event_bytes=output[0], + stdout=res.out, + stderr=res.err) + + +def get_events(code, strace_flags, prelude, cleanup): + # NOTE: The flush is currently required to prevent the prints from getting + # buffered and done all at once at exit + prelude = textwrap.dedent(prelude) + code = textwrap.dedent(code) + cleanup = textwrap.dedent(cleanup) + to_run = f""" +print("MARK prelude", flush=True) +{prelude} +print("MARK code", flush=True) +{code} +print("MARK cleanup", flush=True) +{cleanup} +print("MARK __shutdown", flush=True) + """ + trace = strace_python(to_run, strace_flags) + all_sections = trace.sections() + return all_sections['code'] + + +def get_syscalls(code, strace_flags, prelude="", cleanup="", + ignore_memory=True): + """Get the syscalls which a given chunk of python code generates""" + events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + + if ignore_memory: + events = filter_memory(events) + + return [ev.syscall for ev in events] + + +# Moderately expensive (spawns a subprocess), so share results when possible. +@cache +def _can_strace(): + res = strace_python("import sys; sys.exit(0)", + # --trace option needs strace 5.5 (gh-133741) + ["--trace=%process"], + check=False) + if res.strace_returncode == 0 and res.python_returncode == 0: + assert res.events(), "Should have parsed multiple calls" + return True + return False + + +def requires_strace(): + if sys.platform != "linux": + return unittest.skip("Linux only, requires strace.") + + if "LD_PRELOAD" in os.environ: + # Distribution packaging (ex. Debian `fakeroot` and Gentoo `sandbox`) + # use LD_PRELOAD to intercept system calls, which changes the overall + # set of system calls which breaks tests expecting a specific set of + # system calls). + return unittest.skip("Not supported when LD_PRELOAD is intercepting system calls.") + + if support.check_sanitizer(address=True, memory=True): + return unittest.skip("LeakSanitizer does not work under ptrace (strace, gdb, etc)") + + return unittest.skipUnless(_can_strace(), "Requires working strace") + + +__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace", + "strace_python", "StraceEvent", "StraceResult"] diff --git a/Lib/test/support/testcase.py b/Lib/test/support/testcase.py index fd32457d146..e617b19b6ac 100644 --- a/Lib/test/support/testcase.py +++ b/Lib/test/support/testcase.py @@ -1,6 +1,7 @@ from math import copysign, isnan +# XXX: RUSTPYTHON: removed in 3.14 class ExtraAssertions: def assertIsSubclass(self, cls, superclass, msg=None): diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 7f16050f32b..9b2b8f2dff0 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -8,7 +8,6 @@ from test import support - #======================================================================= # Threading support to prevent reporting refleaks when running regrtest.py -R @@ -22,34 +21,37 @@ def threading_setup(): - return _thread._count(), threading._dangling.copy() + return _thread._count(), len(threading._dangling) def threading_cleanup(*original_values): - _MAX_COUNT = 100 - - for count in range(_MAX_COUNT): - values = _thread._count(), threading._dangling - if values == original_values: - break - - if not count: - # Display a warning at the first iteration - support.environment_altered = True - dangling_threads = values[1] - support.print_warning(f"threading_cleanup() failed to cleanup " - f"{values[0] - original_values[0]} threads " - f"(count: {values[0]}, " - f"dangling: {len(dangling_threads)})") - for thread in dangling_threads: - support.print_warning(f"Dangling thread: {thread!r}") - - # Don't hold references to threads - dangling_threads = None - values = None - - time.sleep(0.01) - support.gc_collect() + orig_count, orig_ndangling = original_values + + timeout = 1.0 + for _ in support.sleeping_retry(timeout, error=False): + # Copy the thread list to get a consistent output. threading._dangling + # is a WeakSet, its value changes when it's read. + dangling_threads = list(threading._dangling) + count = _thread._count() + + if count <= orig_count: + return + + # Timeout! + support.environment_altered = True + support.print_warning( + f"threading_cleanup() failed to clean up threads " + f"in {timeout:.1f} seconds\n" + f" before: thread count={orig_count}, dangling={orig_ndangling}\n" + f" after: thread count={count}, dangling={len(dangling_threads)}") + for thread in dangling_threads: + support.print_warning(f"Dangling thread: {thread!r}") + + # The warning happens when a test spawns threads and some of these threads + # are still running after the test completes. To fix this warning, join + # threads explicitly to wait until they complete. + # + # To make the warning more likely, reduce the timeout. def reap_threads(func): @@ -245,3 +247,27 @@ def requires_working_threading(*, module=False): raise unittest.SkipTest(msg) else: return unittest.skipUnless(can_start_thread, msg) + + +def run_concurrently(worker_func, nthreads, args=(), kwargs={}): + """ + Run the worker function concurrently in multiple threads. + """ + barrier = threading.Barrier(nthreads) + + def wrapper_func(*args, **kwargs): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + worker_func(*args, **kwargs) + + with catch_threading_exception() as cm: + workers = [ + threading.Thread(target=wrapper_func, args=args, kwargs=kwargs) + for _ in range(nthreads) + ] + with start_threads(workers): + pass + + # If a worker thread raises an exception, re-raise it. + if cm.exc_value is not None: + raise cm.exc_value diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py index 78e6a51ec18..b60f6097e65 100644 --- a/Lib/test/support/venv.py +++ b/Lib/test/support/venv.py @@ -1,8 +1,8 @@ import contextlib import logging import os -import subprocess import shlex +import subprocess import sys import sysconfig import tempfile @@ -68,3 +68,14 @@ def run(self, *args, **subprocess_args): raise else: return result + + +class VirtualEnvironmentMixin: + def venv(self, name=None, **venv_create_args): + venv_name = self.id() + if name: + venv_name += f'-{name}' + return VirtualEnvironment.from_tmpdir( + prefix=f'{venv_name}-venv-', + **venv_create_args, + ) diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index c1bf0562300..5f6f14afd74 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -23,8 +23,7 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(len(warns), 1, warns) warn, = warns - testcase.assertTrue(issubclass(warn.category, SyntaxWarning), - warn.category) + testcase.assertIsSubclass(warn.category, SyntaxWarning) if errtext: testcase.assertRegex(str(warn.message), errtext) testcase.assertEqual(warn.filename, '') @@ -160,11 +159,12 @@ def _filterwarnings(filters, quiet=False): registry = frame.f_globals.get('__warningregistry__') if registry: registry.clear() - with warnings.catch_warnings(record=True) as w: - # Set filter "always" to record all warnings. Because - # test_warnings swap the module, we need to look up in - # the sys.modules dictionary. - sys.modules['warnings'].simplefilter("always") + # Because test_warnings swap the module, we need to look up in the + # sys.modules dictionary. + wmod = sys.modules['warnings'] + with wmod.catch_warnings(record=True) as w: + # Set filter "always" to record all warnings. + wmod.simplefilter("always") yield WarningsRecorder(w) # Filter the recorded warnings reraise = list(w) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 7b5356ea02a..8ded9f99248 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,7 +3,6 @@ from test.support import warnings_helper import os import sys -import types if support.check_sanitizer(address=True, memory=True): @@ -38,6 +37,7 @@ def check_all(self, modname): (".* (module|package)", DeprecationWarning), (".* (module|package)", PendingDeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("import %s" % modname, names) @@ -53,6 +53,7 @@ def check_all(self, modname): with warnings_helper.check_warnings( ("", DeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("from %s import *" % modname, names) @@ -71,6 +72,8 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): @@ -94,8 +97,6 @@ def walk_modules(self, basedir, modpath): continue yield path, modpath + modname - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_all(self): # List of denied modules and packages denylist = set([ @@ -106,7 +107,7 @@ def test_all(self): # In case _socket fails to build, make this test fail more gracefully # than an AttributeError somewhere deep in concurrent.futures, email # or unittest. - import _socket + import _socket # noqa: F401 ignored = [] failed_imports = [] diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index b2f0bb1386f..026277267e0 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import sys import unittest @@ -21,6 +22,41 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestTheme(unittest.TestCase): + + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + class TestColorizeFunction(unittest.TestCase): def test_colorized_detection_checks_for_environment_variables(self): def check(env, fallback, expected): @@ -129,6 +165,17 @@ def test_colorized_detection_checks_for_file(self): file.isatty.return_value = False self.assertEqual(_colorize.can_colorize(file=file), False) + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py new file mode 100644 index 00000000000..11b2c9545a1 --- /dev/null +++ b/Lib/test/test__locale.py @@ -0,0 +1,300 @@ +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) +try: + from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) +except ImportError: + nl_langinfo = None + +import locale +import sys +import unittest +from platform import uname + +from test import support + +if uname().system == "Darwin": + maj, min, mic = [int(part) for part in uname().release.split(".")] + if (maj, min, mic) < (8, 0, 0): + raise unittest.SkipTest("locale support broken for OS X < 10.4") + +candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT', + 'et_EE', 'es_PY', 'no_NO', 'nl_NL', 'lv_LV', 'el_GR', 'be_BY', 'fr_BE', + 'ro_RO', 'ru_UA', 'ru_RU', 'es_VE', 'ca_ES', 'se_NO', 'es_EC', 'id_ID', + 'ka_GE', 'es_CL', 'wa_BE', 'hu_HU', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR', + 'es_ES', 'oc_FR', 'gl_ES', 'bg_BG', 'is_IS', 'mk_MK', 'de_AT', 'pt_BR', + 'da_DK', 'nn_NO', 'cs_CZ', 'de_LU', 'es_BO', 'sq_AL', 'sk_SK', 'fr_CH', + 'de_DE', 'sr_YU', 'br_FR', 'nl_BE', 'sv_FI', 'pl_PL', 'fr_CA', 'fo_FO', + 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', + 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', + 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro', + 'ru_RU.KOI8-R', 'ko_KR.eucKR', + 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8', + 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620', +] + +def setUpModule(): + global candidate_locales + # Issue #13441: Skip some locales (e.g. cs_CZ and hu_HU) on Solaris to + # workaround a mbstowcs() bug. For example, on Solaris, the hu_HU locale uses + # the locale encoding ISO-8859-2, the thousands separator is b'\xA0' and it is + # decoded as U+30000020 (an invalid character) by mbstowcs(). + if sys.platform == 'sunos5': + old_locale = locale.setlocale(locale.LC_ALL) + try: + locales = [] + for loc in candidate_locales: + try: + locale.setlocale(locale.LC_ALL, loc) + except Error: + continue + encoding = locale.getencoding() + try: + localeconv() + except Exception as err: + print("WARNING: Skip locale %s (encoding %s): [%s] %s" + % (loc, encoding, type(err), err)) + else: + locales.append(loc) + candidate_locales = locales + finally: + locale.setlocale(locale.LC_ALL, old_locale) + + # Workaround for MSVC6(debug) crash bug + if "MSC v.1200" in sys.version: + def accept(loc): + a = loc.split(".") + return not(len(a) == 2 and len(a[-1]) >= 9) + candidate_locales = [loc for loc in candidate_locales if accept(loc)] + +# List known locale values to test against when available. +# Dict formatted as `` : (, )``. If a +# value is not known, use '' . +known_numerics = { + 'en_US': ('.', ','), + 'de_DE' : (',', '.'), + # The French thousands separator may be a breaking or non-breaking space + # depending on the platform, so do not test it + 'fr_FR' : (',', ''), + 'ps_AF': ('\u066b', '\u066c'), +} + +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), + 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), + 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), +} + +known_era = { + 'C': (0, ''), + 'en_US': (0, ''), + 'ja_JP': (11, '+:1:2019/05/01:2019/12/31:令和:%EC元年'), + 'zh_TW': (3, '+:1:1912/01/01:1912/12/31:民國:%EC元年'), + 'th_TW': (1, '+:1:-543/01/01:+*:พ.ศ.:%EC %Ey'), +} + +if sys.platform == 'win32': + # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) + del known_numerics['ps_AF'] + +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + +class _LocaleTests(unittest.TestCase): + + def setUp(self): + self.oldlocale = setlocale(LC_ALL) + + def tearDown(self): + setlocale(LC_ALL, self.oldlocale) + + # Want to know what value was calculated, what it was compared against, + # what function was used for the calculation, what type of data was used, + # the locale that was supposedly set, and the actual locale that is set. + lc_numeric_err_msg = "%s != %s (%s for %s; set to %s, using %s)" + + def numeric_tester(self, calc_type, calc_value, data_type, used_locale): + """Compare calculation against known value, if available""" + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + known_value = known_numerics.get(used_locale, + ('', ''))[data_type == 'thousands_sep'] + if known_value and calc_value: + self.assertEqual(calc_value, known_value, + self.lc_numeric_err_msg % ( + calc_value, known_value, + calc_type, data_type, set_locale, + used_locale)) + return True + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_nl_langinfo(self): + # Test nl_langinfo against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_localeconv(self): + # Test localeconv against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + formatting = localeconv() + for lc in ("decimal_point", + "thousands_sep"): + if self.numeric_tester('localeconv', formatting[lc], lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + def test_lc_numeric_basic(self): + # Test nl_langinfo against localeconv + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + nl_radixchar = nl_langinfo(li) + li_radixchar = localeconv()[lc] + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + self.assertEqual(nl_radixchar, li_radixchar, + "%s (nl_langinfo) != %s (localeconv) " + "(set to %s, using %s)" % ( + nl_radixchar, li_radixchar, + loc, set_locale)) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, str) + alt_digits = alt_digits.split(';') if alt_digits else [] + if alt_digits: + self.assertGreaterEqual(len(alt_digits), 10, alt_digits) + loc1 = loc.split('.', 1)[0] + if loc1 in known_alt_digits: + count, samples = known_alt_digits[loc1] + if count and not alt_digits: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform') + self.assertEqual(len(alt_digits), count, alt_digits) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ERA'), "requires locale.ERA") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_era_nl_langinfo(self): + # Test nl_langinfo(ERA) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + era = nl_langinfo(locale.ERA) + self.assertIsInstance(era, str) + if era: + self.assertEqual(era.count(':'), (era.count(';') + 1) * 5, era) + + loc1 = loc.split('.', 1)[0] + if loc1 in known_era: + count, sample = known_era[loc1] + if count: + if not era: + self.skipTest(f'ERA is not set for locale {loc!r} on this platform') + self.assertGreaterEqual(era.count(';') + 1, count) + self.assertIn(sample, era) + else: + self.assertEqual(era, '') + tested = True + if not tested: + self.skipTest('no suitable locales') + + def test_float_parsing(self): + # Bug #1391872: Test whether float parsing is okay on European + # locales. + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + + # Ignore buggy locale databases. (Mac OS 10.4 and some other BSDs) + if loc == 'eu_ES' and localeconv()['decimal_point'] == "' ": + continue + + self.assertEqual(int(eval('3.14') * 100), 314, + "using eval('3.14') failed for %s" % loc) + self.assertEqual(int(float('3.14') * 100), 314, + "using float('3.14') failed for %s" % loc) + if localeconv()['decimal_point'] != '.': + self.assertRaises(ValueError, float, + localeconv()['decimal_point'].join(['1', '23'])) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index dd4f30ab17d..43d475baa5d 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -17,7 +17,7 @@ def check_bool_function_result(self, func, ops, expected): self.assertEqual(func(op), expected) def test_invalid_opcodes(self): - invalid = [-100, -1, 255, 512, 513, 1000] + invalid = [-100, -1, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) self.check_bool_function_result(_opcode.has_arg, invalid, False) self.check_bool_function_result(_opcode.has_const, invalid, False) @@ -38,6 +38,14 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 'BINARY_OP_ADD_INT' + def test_opmaps(self): + def check_roundtrip(name, map): + return self.assertEqual(opcode.opname[map[name]], name) + + check_roundtrip('BINARY_OP', opcode.opmap) + check_roundtrip('BINARY_OP_ADD_INT', opcode._specialized_opmap) + def test_oplists(self): def check_function(self, func, expected): for op in [-10, 520]: @@ -56,11 +64,9 @@ def check_function(self, func, expected): class StackEffectTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) - self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) - self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 1), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 2), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2) self.assertRaises(ValueError, stack_effect, 30000) # All defined opcodes @@ -77,7 +83,6 @@ def test_stack_effect(self): self.assertRaises(ValueError, stack_effect, code) self.assertRaises(ValueError, stack_effect, code, 0) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_stack_effect_jump(self): FOR_ITER = dis.opmap['FOR_ITER'] self.assertEqual(stack_effect(FOR_ITER, 0), 1) @@ -111,6 +116,7 @@ def test_stack_effect_jump(self): class SpecializationStatsTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'load_attr' not found in [] def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ @@ -119,7 +125,7 @@ def test_specialization_stats(self): if opcode._inline_cache_entries.get(op, 0) ] self.assertIn('load_attr', specialized_opcodes) - self.assertIn('binary_subscr', specialized_opcodes) + self.assertIn('binary_op', specialized_opcodes) stats = _opcode.get_specialization_stats() if stats is not None: diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 92bd955855b..e29fc5a2394 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -20,7 +20,7 @@ def test_abstractproperty_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") class C(metaclass=abc_ABCMeta): @abc.abstractproperty @@ -89,7 +89,7 @@ def test_abstractmethod_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") def test_abstractproperty_basics(self): @property @@ -168,8 +168,7 @@ def method_two(self): msg = r"class C without an implementation for abstract methods 'method_one', 'method_two'" self.assertRaisesRegex(TypeError, msg, C) - # TODO: RUSTPYTHON; AssertionError: False is not true - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_abstractmethod_integration(self): for abstractthing in [abc.abstractmethod, abc.abstractproperty, abc.abstractclassmethod, @@ -278,21 +277,21 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) B1 = A.register(B) - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) self.assertIs(B1, B) class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) @@ -303,16 +302,16 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) @A.register class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) self.assertIs(C, A.register(C)) @@ -323,14 +322,14 @@ class A(metaclass=abc_ABCMeta): class B: pass b = B() - self.assertFalse(isinstance(b, A)) - self.assertFalse(isinstance(b, (A,))) + self.assertNotIsInstance(b, A) + self.assertNotIsInstance(b, (A,)) token_old = abc_get_cache_token() A.register(B) token_new = abc_get_cache_token() self.assertGreater(token_new, token_old) - self.assertTrue(isinstance(b, A)) - self.assertTrue(isinstance(b, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) def test_registration_builtins(self): class A(metaclass=abc_ABCMeta): @@ -338,18 +337,18 @@ class A(metaclass=abc_ABCMeta): A.register(int) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) - self.assertTrue(issubclass(int, A)) - self.assertTrue(issubclass(int, (A,))) + self.assertIsSubclass(int, A) + self.assertIsSubclass(int, (A,)) class B(A): pass B.register(str) class C(str): pass self.assertIsInstance("", A) self.assertIsInstance("", (A,)) - self.assertTrue(issubclass(str, A)) - self.assertTrue(issubclass(str, (A,))) - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(str, A) + self.assertIsSubclass(str, (A,)) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) def test_registration_edge_cases(self): class A(metaclass=abc_ABCMeta): @@ -377,39 +376,39 @@ class A(metaclass=abc_ABCMeta): def test_registration_transitiveness(self): class A(metaclass=abc_ABCMeta): pass - self.assertTrue(issubclass(A, A)) - self.assertTrue(issubclass(A, (A,))) + self.assertIsSubclass(A, A) + self.assertIsSubclass(A, (A,)) class B(metaclass=abc_ABCMeta): pass - self.assertFalse(issubclass(A, B)) - self.assertFalse(issubclass(A, (B,))) - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(A, B) + self.assertNotIsSubclass(A, (B,)) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertTrue(issubclass(B1, A)) - self.assertTrue(issubclass(B1, (A,))) + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) class C1(C): pass B1.register(C1) - self.assertFalse(issubclass(C, B)) - self.assertFalse(issubclass(C, (B,))) - self.assertFalse(issubclass(C, B1)) - self.assertFalse(issubclass(C, (B1,))) - self.assertTrue(issubclass(C1, A)) - self.assertTrue(issubclass(C1, (A,))) - self.assertTrue(issubclass(C1, B)) - self.assertTrue(issubclass(C1, (B,))) - self.assertTrue(issubclass(C1, B1)) - self.assertTrue(issubclass(C1, (B1,))) + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) C1.register(int) class MyInt(int): pass - self.assertTrue(issubclass(MyInt, A)) - self.assertTrue(issubclass(MyInt, (A,))) + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) @@ -469,16 +468,16 @@ def __subclasshook__(cls, C): if cls is A: return 'foo' in C.__dict__ return NotImplemented - self.assertFalse(issubclass(A, A)) - self.assertFalse(issubclass(A, (A,))) + self.assertNotIsSubclass(A, A) + self.assertNotIsSubclass(A, (A,)) class B: foo = 42 - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) class C: spam = 42 - self.assertFalse(issubclass(C, A)) - self.assertFalse(issubclass(C, (A,))) + self.assertNotIsSubclass(C, A) + self.assertNotIsSubclass(C, (A,)) def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): @@ -495,7 +494,7 @@ class C(A, B): self.assertEqual(B.counter, 1) def test_ABC_has___slots__(self): - self.assertTrue(hasattr(abc.ABC, '__slots__')) + self.assertHasAttr(abc.ABC, '__slots__') def test_tricky_new_works(self): def with_metaclass(meta, *bases): @@ -517,13 +516,14 @@ def foo(self): del A.foo self.assertEqual(A.__abstractmethods__, {'foo'}) - self.assertFalse(hasattr(A, 'foo')) + self.assertNotHasAttr(A, 'foo') abc.update_abstractmethods(A) self.assertEqual(A.__abstractmethods__, set()) A() + def test_update_new_abstractmethods(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -589,7 +589,7 @@ def updated_foo(self): A.foo = updated_foo abc.update_abstractmethods(A) A() - self.assertFalse(hasattr(A, '__abstractmethods__')) + self.assertNotHasAttr(A, '__abstractmethods__') def test_update_del_implementation(self): class A(metaclass=abc_ABCMeta): @@ -685,10 +685,16 @@ class B(A, metaclass=abc_ABCMeta, name="test"): return TestLegacyAPI, TestABC, TestABCWithInitSubclass -TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc.ABCMeta, - abc.get_cache_token) -TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(_py_abc.ABCMeta, - _py_abc.get_cache_token) +TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(_py_abc.ABCMeta, + _py_abc.get_cache_token) +TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(abc.ABCMeta, + abc.get_cache_token) + +# gh-130095: The _py_abc tests are not thread-safe when run with +# `--parallel-threads` +TestLegacyAPI_Py.__unittest_thread_unsafe__ = True +TestABC_Py.__unittest_thread_unsafe__ = True +TestABCWithInitSubclass_Py.__unittest_thread_unsafe__ = True if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 72232b670cd..cf071d2c933 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -24,11 +24,11 @@ def not_implemented(*args, **kwargs): class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Rational)) - self.assertTrue(issubclass(int, Real)) - self.assertTrue(issubclass(int, Complex)) - self.assertTrue(issubclass(int, Number)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -38,11 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Integral)) - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) - self.assertTrue(issubclass(float, Complex)) - self.assertTrue(issubclass(float, Number)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -50,11 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Integral)) - self.assertFalse(issubclass(complex, Rational)) - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) - self.assertTrue(issubclass(complex, Number)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000..8208d0e9c94 --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,2215 @@ +"""Tests for the annotations module.""" + +import textwrap +import annotationlib +import builtins +import collections +import functools +import itertools +import pickle +from string.templatelib import Template, Interpolation +import typing +import sys +import unittest +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + annotations_to_string, + type_repr, +) +from typing import Unpack, get_type_hints, List, Union + +from test import support +from test.support import import_helper +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class MyClass: + def __repr__(self): + return "my repr" + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) + + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS, 2) + + self.assertEqual(Format.FORWARDREF.value, 3) + self.assertEqual(Format.FORWARDREF, 3) + + self.assertEqual(Format.STRING.value, 4) + self.assertEqual(Format.STRING, 4) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.FORWARDREF) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = get_annotations(inner, format=Format.FORWARDREF) + self.assertEqual(anno["arg"], x) + + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + def test_nonexistent_attribute(self): + def f( + x: some.module, + y: some[module], + z: some(module), + alpha: some | obj, + beta: +some, + gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj, module}, + zeta: some | [obj], + eta: some | (), + ): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno, support.EqualToForwardRef("some.module", owner=f)) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno, support.EqualToForwardRef("some[module]", owner=f)) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno, support.EqualToForwardRef("+some", owner=f)) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj, module}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + + def test_partially_nonexistent_union(self): + # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + pipe = annos["pipe"] + self.assertIsInstance(pipe, ForwardRef) + self.assertEqual( + pipe.evaluate(globals={"undefined": int}), + str | int, + ) + union = annos["union"] + self.assertIsInstance(union, Union) + arg1, arg2 = typing.get_args(union) + self.assertIs(arg1, str) + self.assertEqual( + arg2, support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + +class TestStringFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_closure_undefined(self): + if False: + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + gh138558: t"{ 0}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + "gh138558": "t'{ 0}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + + def test_getitem(self): + def f(x: undef1[str, undef2]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "undef1[str, undef2]"}) + + anno = get_annotations(f, format=Format.FORWARDREF) + fwdref = anno["x"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual( + fwdref.evaluate(globals={"undef1": dict, "undef2": float}), dict[str, float] + ) + + def test_slice(self): + def f(x: a[b:c]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c]"}) + + def f(x: a[b:c, d:e]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c, d:e]"}) + + obj = slice(1, 1, 1) + def f(x: obj): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "obj"}) + + def test_literals(self): + def f( + a: 1, + b: 1.0, + c: "hello", + d: b"hello", + e: True, + f: None, + g: ..., + h: 1j, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "1", + "b": "1.0", + "c": 'hello', + "d": "b'hello'", + "e": "True", + "f": "None", + "g": "...", + "h": "1j", + }, + ) + + def test_displays(self): + # Simple case first + def f(x: a[[int, str], float]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[[int, str], float]"}) + + def g( + w: a[[int, str], float], + x: a[{int}, 3], + y: a[{int: str}, 4], + z: a[(int, str), 5], + ): + pass + anno = get_annotations(g, format=Format.STRING) + self.assertEqual( + anno, + { + "w": "a[[int, str], float]", + "x": "a[{int}, 3]", + "y": "a[{int: str}, 4]", + "z": "a[(int, str), 5]", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def test_shenanigans(self): + # In cases like this we can't reconstruct the source; test that we do something + # halfway reasonable. + def f(x: x | (1).__class__, y: (1).__class__): + pass + + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "x | ", "y": ""}, + ) + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(get_annotations(int), {}) + self.assertEqual(get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(get_annotations(C1), {"a": int}) + self.assertEqual( + get_annotations(C1, format=Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + get_annotations(C1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(NoDict), {"b": str}) + self.assertEqual( + get_annotations(NoDict, format=Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + get_annotations(NoDict, format=Format.STRING), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + get_annotations(f1, format=Format.VALUE), + {"a": int}, + ) + self.assertEqual(get_annotations(f1, format=1), {"a": int}) + + fwd = support.EqualToForwardRef("undefined", owner=f2) + self.assertEqual( + get_annotations(f2, format=Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(get_annotations(f2, format=3), {"a": fwd}) + + self.assertEqual( + get_annotations(f1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(f1, format=4), {"a": "int"}) + + with self.assertRaises(ValueError): + get_annotations(f1, format=42) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=2) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + get_annotations(foo, format=Format.STRING, eval_str=True) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(foo, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual(get_annotations(annotationlib, **kwargs), {}) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + get_annotations(isa, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.UnannotatedClass, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.unannotated_function, format=Format.STRING), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.FORWARDREF), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(get_annotations(isa2), {}) + self.assertEqual(get_annotations(isa2, eval_str=True), {}) + self.assertEqual(get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_with_star_unpack(self): + def f(*args: "*tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), + {"x": int}, + ) + + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(get_annotations(f), {"x": int}) + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(get_annotations(f), {"x": str}) + # ... and also in FORWARDREF, which tries __annotations__ if available + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(ha, format=Format.FORWARDREF), {"x": int}) + + self.assertEqual(get_annotations(ha, format=Format.STRING), {"x": "int"}) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual(get_annotations(hra, format=Format.VALUE), {"x": float}) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual(get_annotations(hb, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.FORWARDREF), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.STRING), {"x": str}) + + def test_only_annotate(self): + def f(x: int): + pass + + class OnlyAnnotate: + @property + def __annotate__(self): + return f.__annotate__ + + oa = OnlyAnnotate() + self.assertEqual(get_annotations(oa, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(oa, format=Format.FORWARDREF), {"x": int}) + self.assertEqual( + get_annotations(oa, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + + def test_no_annotations(self): + class CustomClass: + pass + + class MyCallable: + def __call__(self): + pass + + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + for obj in (None, 1, object(), CustomClass()): + with self.subTest(format=format, obj=obj): + with self.assertRaises(TypeError): + get_annotations(obj, format=format) + + # Callables and types with no annotations return an empty dict + for obj in (int, len, MyCallable()): + with self.subTest(format=format, obj=obj): + self.assertEqual(get_annotations(obj, format=format), {}) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars( + self, + ): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None}, + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars( + self, + ): + self.assertEqual( + get_annotations(inspect_stringized_annotations_pep695.E, eval_str=True), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__), + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection( + results.F.__type_params__ + ), + set(), + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + def test_partial_evaluation(self): + def f( + x: builtins.undef, + y: list[int], + z: 1 + int, + a: builtins.int, + b: [builtins.undef, builtins.int], + ): + pass + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("builtins.undef", owner=f), + "y": list[int], + "z": support.EqualToForwardRef("1 + int", owner=f), + "a": int, + "b": [ + support.EqualToForwardRef("builtins.undef", owner=f), + # We can't resolve this because we have to evaluate the whole annotation + support.EqualToForwardRef("builtins.int", owner=f), + ], + }, + ) + + self.assertEqual( + get_annotations(f, format=Format.STRING), + { + "x": "builtins.undef", + "y": "list[int]", + "z": "1 + int", + "a": "builtins.int", + "b": "[builtins.undef, builtins.int]", + }, + ) + + def test_partial_evaluation_error(self): + def f(x: range[1]): + pass + with self.assertRaisesRegex( + TypeError, "type 'range' is not subscriptable" + ): + f.__annotations__ + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("range[1]", owner=f), + }, + ) + + def test_partial_evaluation_cell(self): + obj = object() + + class RaisesAttributeError: + attriberr: obj.missing + + anno = get_annotations(RaisesAttributeError, format=Format.FORWARDREF) + self.assertEqual( + anno, + { + "attriberr": support.EqualToForwardRef( + "obj.missing", is_class=True, owner=RaisesAttributeError + ) + }, + ) + + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format > 2: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), + support.EqualToForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.STRING), + "undefined", + ) + + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + + +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(Meta.__annotate__(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(X.__annotate__, None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(Y.__annotate__(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): + pass + + class X(metaclass=Meta): + a: str + + class Y(X): + pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(Meta.__annotate__, None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(Y.__annotate__, None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(X.__annotate__(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = getattr(c, "__annotate__", None) + if c.expected_annotations: + self.assertEqual( + annotate_func(Format.VALUE), c.expected_annotations + ) + else: + self.assertIs(annotate_func, None) + + +class TestGetAnnotateFromClassNamespace(unittest.TestCase): + def test_with_metaclass(self): + class Meta(type): + def __new__(mcls, name, bases, ns): + annotate = annotationlib.get_annotate_from_class_namespace(ns) + expected = ns["expected_annotate"] + with self.subTest(name=name): + if expected: + self.assertIsNotNone(annotate) + else: + self.assertIsNone(annotate) + return super().__new__(mcls, name, bases, ns) + + class HasAnnotations(metaclass=Meta): + expected_annotate = True + a: int + + class NoAnnotations(metaclass=Meta): + expected_annotate = False + + class CustomAnnotate(metaclass=Meta): + expected_annotate = True + def __annotate__(format): + return {} + + code = """ + from __future__ import annotations + + class HasFutureAnnotations(metaclass=Meta): + expected_annotate = False + a: int + """ + exec(textwrap.dedent(code), {"Meta": Meta}) + + +class TestTypeRepr(unittest.TestCase): + def test_type_repr(self): + class Nested: + pass + + def nested(): + pass + + self.assertEqual(type_repr(int), "int") + self.assertEqual(type_repr(MyClass), f"{__name__}.MyClass") + self.assertEqual( + type_repr(Nested), f"{__name__}.TestTypeRepr.test_type_repr..Nested" + ) + self.assertEqual( + type_repr(nested), f"{__name__}.TestTypeRepr.test_type_repr..nested" + ) + self.assertEqual(type_repr(len), "len") + self.assertEqual(type_repr(type_repr), "annotationlib.type_repr") + self.assertEqual(type_repr(times_three), f"{__name__}.times_three") + self.assertEqual(type_repr(...), "...") + self.assertEqual(type_repr(None), "None") + self.assertEqual(type_repr(1), "1") + self.assertEqual(type_repr("1"), "'1'") + self.assertEqual(type_repr(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(type_repr(MyClass()), "my repr") + # gh138558 tests + self.assertEqual(type_repr(t'''{ 0 + & 1 + | 2 + }'''), 't"""{ 0\n & 1\n | 2}"""') + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42))), + "Template('hi', Interpolation(42, '', None, ''))", + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, " "))), + "Template('hi', Interpolation(42, ' ', None, ''))", + ) + # gh138558: perhaps in the future, we can improve this behavior: + self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") + + +class TestAnnotationsToString(unittest.TestCase): + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + +class A: + pass + +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + + +class TestForwardRefClass(unittest.TestCase): + def test_forwardref_instance_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + isinstance(42, fr) + + def test_forwardref_subclass_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forwardref_only_str_arg(self): + with self.assertRaises(TypeError): + ForwardRef(1) # only `str` type is allowed + + def test_forward_equality(self): + fr = ForwardRef("int") + self.assertEqual(fr, ForwardRef("int")) + self.assertNotEqual(List["int"], List[int]) + self.assertNotEqual(fr, ForwardRef("int", module=__name__)) + frm = ForwardRef("int", module=__name__) + self.assertEqual(frm, ForwardRef("int", module=__name__)) + self.assertNotEqual(frm, ForwardRef("int", module="__other_name__")) + + def test_forward_equality_get_type_hints(self): + c1 = ForwardRef("C") + c1_gth = ForwardRef("C") + c2 = ForwardRef("C") + c2_gth = ForwardRef("C") + + class C: + pass + + def foo(a: c1_gth, b: c2_gth): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), {"a": C, "b": C}) + self.assertEqual(c1, c2) + self.assertEqual(c1, c1_gth) + self.assertEqual(c1_gth, c2_gth) + self.assertEqual(List[c1], List[c1_gth]) + self.assertNotEqual(List[c1], List[C]) + self.assertNotEqual(List[c1_gth], List[C]) + self.assertEqual(Union[c1, c1_gth], Union[c1]) + self.assertEqual(Union[c1, c1_gth, int], Union[c1, int]) + + def test_forward_equality_hash(self): + c1 = ForwardRef("int") + c1_gth = ForwardRef("int") + c2 = ForwardRef("int") + c2_gth = ForwardRef("int") + + def foo(a: c1_gth, b: c2_gth): + pass + + get_type_hints(foo, globals(), locals()) + + self.assertEqual(hash(c1), hash(c2)) + self.assertEqual(hash(c1_gth), hash(c2_gth)) + self.assertEqual(hash(c1), hash(c1_gth)) + + c3 = ForwardRef("int", module=__name__) + c4 = ForwardRef("int", module="__other_name__") + + self.assertNotEqual(hash(c3), hash(c1)) + self.assertNotEqual(hash(c3), hash(c1_gth)) + self.assertNotEqual(hash(c3), hash(c4)) + self.assertEqual(hash(c3), hash(ForwardRef("int", module=__name__))) + + def test_forward_equality_namespace(self): + def namespace1(): + a = ForwardRef("A") + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + + class A: + pass + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + self.assertEqual(namespace1(), namespace1()) + self.assertEqual(namespace1(), namespace2()) + + def test_forward_repr(self): + self.assertEqual(repr(List["int"]), "typing.List[ForwardRef('int')]") + self.assertEqual( + repr(List[ForwardRef("int", module="mod")]), + "typing.List[ForwardRef('int', module='mod')]", + ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) + + def test_forward_recursion_actually(self): + def namespace1(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertEqual(r1, r2) + + def test_syntax_error(self): + + with self.assertRaises(SyntaxError): + typing.Generic["/T"] + + def test_delayed_syntax_error(self): + + def foo(a: "Node[T"): + pass + + with self.assertRaises(SyntaxError): + get_type_hints(foo) + + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[""] + + def test_or(self): + X = ForwardRef("X") + # __or__/__ror__ itself + self.assertEqual(X | "x", Union[X, "x"]) + self.assertEqual("x" | X, Union["x", X]) + + def test_multiple_ways_to_create(self): + X1 = Union["X"] + self.assertIsInstance(X1, ForwardRef) + X2 = ForwardRef("X") + self.assertIsInstance(X2, ForwardRef) + self.assertEqual(X1, X2) + + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = ForwardRef("set[Any]") + self.assertNotHasAttr(fr, "__name__") + self.assertNotHasAttr(fr, "__qualname__") + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + def test_evaluate_string_format(self): + fr = ForwardRef("set[Any]") + self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + + def test_evaluate_forwardref_format(self): + fr = ForwardRef("undef") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertIs(fr, evaluated) + + fr = ForwardRef("set[undefined]") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertEqual( + evaluated, + set[support.EqualToForwardRef("undefined")], + ) + + fr = ForwardRef("a + b") + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef("a + b"), + ) + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF, locals={"a": 1, "b": 2}), + 3, + ) + + fr = ForwardRef('"a" + 1') + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef('"a" + 1'), + ) + + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + + def test_evaluate_with_type_params(self): + class Gen[T]: + alias = int + + with self.assertRaises(NameError): + ForwardRef("T").evaluate() + with self.assertRaises(NameError): + ForwardRef("T").evaluate(type_params=()) + with self.assertRaises(NameError): + ForwardRef("T").evaluate(owner=int) + + (T,) = Gen.__type_params__ + self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) + self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) + + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(type_params=Gen.__type_params__) + self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) + # If you pass custom locals, we don't look at the owner's locals + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(owner=Gen, locals={}) + # But if the name exists in the locals, it works + self.assertIs( + ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str + ) + + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + + def test_fwdref_with_module(self): + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) + + with self.assertRaises(NameError): + # If globals are passed explicitly, we don't look at the module dict + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) + + def test_fwdref_value_is_not_cached(self): + fr = ForwardRef("hello") + with self.assertRaises(NameError): + fr.evaluate() + self.assertIs(fr.evaluate(globals={"hello": str}), str) + with self.assertRaises(NameError): + fr.evaluate() + + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: + ForwardRef("doesntexist").evaluate() + + self.assertEqual(exc.exception.name, "doesntexist") + + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + + +class TestAnnotationLib(unittest.TestCase): + def test__all__(self): + support.check__all__(self, annotationlib) + + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports( + "annotationlib", + { + "typing", + "warnings", + }, + ) diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000..ab5296afad1 --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "") + self.assertEqual(repr(self.log.buffer), "") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index b7e995334fe..f48fb765bb3 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,11 +1,13 @@ # Author: Steven J. Bethard . +import _colorize import contextlib import functools import inspect import io import operator import os +import py_compile import shutil import stat import sys @@ -16,12 +18,22 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) +from test.support import import_helper from test.support import os_helper +from test.support import script_helper from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -631,9 +643,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -777,48 +789,12 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 + def test_invalid_name(self): parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -1036,6 +1012,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -1101,8 +1078,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1135,8 +1112,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1149,9 +1126,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1811,27 +1788,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1885,7 +1878,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1898,8 +1890,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1919,7 +1911,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1938,8 +1930,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1977,8 +1969,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -2000,8 +1992,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -2015,8 +2007,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -2032,8 +2024,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -2045,7 +2037,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -2060,7 +2052,7 @@ def test_open_args(self): def test_invalid_file_type(self): with self.assertRaises(ValueError): - argparse.FileType('b')('-test') + FileType('b')('-test') class TestFileTypeMissingInitialization(TestCase): @@ -2071,7 +2063,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2256,6 +2248,130 @@ class TestActionExtend(ParserTestCase): ] +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + class TestInvalidAction(TestCase): """Test invalid user defined Action""" @@ -2269,17 +2385,31 @@ def test_invalid_type(self): self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) def test_modified_invalid_action(self): - parser = ErrorRaisingArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) action = parser.add_argument('--foo') # Someone got crazy and did this action.type = 1 - self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) # ================ # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -2287,16 +2417,17 @@ def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None, - aliases=False): + aliases=False, usage=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description', prefix_chars=prefix_chars) + prog='PROG', description='main description', usage=usage, + prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description') + prog='PROG', description='main description', usage=usage) parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( @@ -2310,7 +2441,7 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertRaisesRegex(argparse.ArgumentError, + self.assertRaisesRegex(ValueError, 'cannot have multiple subparser arguments', parser.add_subparsers) @@ -2333,7 +2464,8 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser - parser3_kwargs = dict(description='3 description') + parser3_kwargs = dict(description='3 description', + usage='PROG --foo bar 3 t ...') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) @@ -2355,6 +2487,47 @@ def test_parse_args_failures(self): args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) + def test_parse_args_failures_details(self): + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [-h] [--foo] bar {1,2,3} ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + self.parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + + def test_parse_args_failures_details_custom_usage(self): + parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...') + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + def test_parse_args(self): # check some non-failure cases: self.assertEqual( @@ -2508,18 +2681,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2655,6 +2816,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2796,10 +2980,43 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -2832,8 +3049,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2844,7 +3059,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2923,11 +3138,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2943,7 +3157,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2953,15 +3167,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2973,7 +3186,7 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) @@ -3011,6 +3224,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3049,6 +3263,29 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): self.maxDiff = None parser = ErrorRaisingArgumentParser(prog='PROG') @@ -3070,12 +3307,19 @@ def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): ''' self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) class MEMixin(object): @@ -3108,21 +3352,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -3331,25 +3579,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3360,6 +3612,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3413,23 +3666,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3442,57 +3699,9 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - - def get_parser(self, required): - parser = ErrorRaisingArgumentParser(prog='PROG') - group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') - return parser - - usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] - ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) - ''' - - help = '''\ - - options: - -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F - ''' - - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None - class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): def get_parser(self, required=None): @@ -3843,11 +4052,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -3860,6 +4071,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4601,6 +4813,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -4663,25 +4876,6 @@ def test_all_suppressed_mutex_with_optional_nargs(self): usage = 'usage: PROG [-h]\n' self.assertEqual(parser.format_usage(), usage) - def test_nested_mutex_groups(self): - parser = argparse.ArgumentParser(prog='PROG') - g = parser.add_mutually_exclusive_group() - g.add_argument("--spam") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - gg = g.add_mutually_exclusive_group() - gg.add_argument("--hax") - gg.add_argument("--hox", help=argparse.SUPPRESS) - gg.add_argument("--hex") - g.add_argument("--eggs") - parser.add_argument("--num") - - usage = textwrap.dedent('''\ - usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] - [--num NUM] - ''') - self.assertEqual(parser.format_usage(), usage) - def test_long_mutex_groups_wrap(self): parser = argparse.ArgumentParser(prog='PROG') g = parser.add_mutually_exclusive_group() @@ -4700,6 +4894,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -5115,6 +5328,7 @@ class TestHelpArgumentDefaults(HelpTestCase): argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), + Sig('--required', required=True, help='some help'), Sig('--taz', action=argparse.BooleanOptionalAction, help='Whether to taz it', default=True), Sig('--corge', action=argparse.BooleanOptionalAction, @@ -5128,8 +5342,8 @@ class TestHelpArgumentDefaults(HelpTestCase): [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ - usage: PROG [-h] [--foo FOO] [--bar] [--taz | --no-taz] [--corge | --no-corge] - [--quux QUUX] [--baz BAZ] + usage: PROG [-h] [--foo FOO] [--bar] --required REQUIRED [--taz | --no-taz] + [--corge | --no-corge] [--quux QUUX] [--baz BAZ] spam [badger] ''' help = usage + '''\ @@ -5144,6 +5358,7 @@ class TestHelpArgumentDefaults(HelpTestCase): -h, --help show this help message and exit --foo FOO foo help - oh and by the way, None --bar bar help (default: False) + --required REQUIRED some help --taz, --no-taz Whether to taz it (default: True) --corge, --no-corge Whether to corge it --quux QUUX Set the quux (default: 42) @@ -5301,11 +5516,61 @@ def custom_type(string): version = '' -class TestHelpUsageLongSubparserCommand(TestCase): - """Test that subparser commands are formatted correctly in help""" +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): maxDiff = None - def test_parent_help(self): + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=50) @@ -5371,29 +5636,45 @@ def test_missing_destination(self): self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') def test_invalid_prefix(self): - self.assertValueError('--foo', '+foo') + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') self.assertValueError('--foo', action="store-true", errmsg='unknown action') + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') + def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): @@ -5405,8 +5686,11 @@ def test_no_argument_actions(self): with self.subTest(attrs=attrs): self.assertTypeError('-x', action=action, **attrs) self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") self.assertTypeError('-x', action=action, nargs=0) - self.assertTypeError('x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments @@ -5426,7 +5710,7 @@ def test_more_than_one_argument_actions(self): self.assertValueError('-x', nargs=0, action=action, errmsg=f'nargs for {action_name} actions must be != 0') self.assertValueError('spam', nargs=0, action=action, - errmsg=f'nargs for {action_name} actions must be != 0') + errmsg='nargs for positionals must be != 0') # const is disallowed with non-optional arguments for nargs in [1, '*', '+']: @@ -5529,6 +5813,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -5558,20 +5843,25 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5791,6 +6081,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6565,7 +6856,7 @@ class TestImportStar(TestCase): def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -6589,6 +6880,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6754,6 +7046,99 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + # ================= # Translation tests # ================= @@ -6764,6 +7149,278 @@ def test_translations(self): self.assertMsgidsEqual(argparse) +# =========== +# Color tests +# =========== + + +class TestColorized(TestCase): + maxDiff = None + + def setUp(self): + super().setUp() + # Ensure color even if ran with NO_COLOR=1 + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) + self.theme = _colorize.get_theme(force_color=True).argparse + + def test_argparse_color(self): + # Arrange: create a parser with a bit of everything + parser = argparse.ArgumentParser( + color=True, + description="Colorful help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prefix_chars="-+", + prog="PROG", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-v", "--verbose", action="store_true", help="more spam" + ) + group.add_argument( + "-q", "--quiet", action="store_true", help="less spam" + ) + parser.add_argument("x", type=int, help="the base") + parser.add_argument( + "y", type=int, help="the exponent", deprecated=True + ) + parser.add_argument( + "this_indeed_is_a_very_long_action_name", + type=int, + help="the exponent", + ) + parser.add_argument( + "-o", "--optional1", action="store_true", deprecated=True + ) + parser.add_argument("--optional2", help="pick one") + parser.add_argument("--optional3", choices=("X", "Y", "Z")) + parser.add_argument( + "--optional4", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional5", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional6", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "-p", + "--optional7", + choices=("Aaaaa", "Bbbbb", "Ccccc", "Ddddd"), + help="pick one", + ) + + parser.add_argument("+f") + parser.add_argument("++bar") + parser.add_argument("-+baz") + parser.add_argument("-c", "--count") + + subparsers = parser.add_subparsers( + title="subcommands", + description="valid subcommands", + help="additional help", + ) + subparsers.add_parser("sub1", deprecated=True, help="sub1 help") + sub2 = subparsers.add_parser("sub2", deprecated=True, help="sub2 help") + sub2.add_argument("--baz", choices=("X", "Y", "Z"), help="baz help") + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}-v{reset} | {short}-q{reset}] [{short}-o{reset}] [{long}--optional2 {label}OPTIONAL2{reset}] [{long}--optional3 {label}{{X,Y,Z}}{reset}] + [{long}--optional4 {label}{{X,Y,Z}}{reset}] [{long}--optional5 {label}{{X,Y,Z}}{reset}] [{long}--optional6 {label}{{X,Y,Z}}{reset}] + [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] [{long}-+baz {label}BAZ{reset}] + [{short}-c {label}COUNT{reset}] + {pos}x{reset} {pos}y{reset} {pos}this_indeed_is_a_very_long_action_name{reset} {pos}{{sub1,sub2}} ...{reset} + + Colorful help + + {heading}positional arguments:{reset} + {pos_b}x{reset} the base + {pos_b}y{reset} the exponent + {pos_b}this_indeed_is_a_very_long_action_name{reset} + the exponent + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}-v{reset}, {long_b}--verbose{reset} more spam (default: False) + {short_b}-q{reset}, {long_b}--quiet{reset} less spam (default: False) + {short_b}-o{reset}, {long_b}--optional1{reset} + {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} + pick one (default: None) + {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} + pick one (default: None) + {short_b}+f{reset} {label_b}F{reset} + {long_b}++bar{reset} {label_b}BAR{reset} + {long_b}-+baz{reset} {label_b}BAZ{reset} + {short_b}-c{reset}, {long_b}--count{reset} {label_b}COUNT{reset} + + {heading}subcommands:{reset} + valid subcommands + + {pos_b}{{sub1,sub2}}{reset} additional help + {pos_b}sub1{reset} sub1 help + {pos_b}sub2{reset} sub2 help + """ + ), + ) + + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): + # Arrange + parser = argparse.ArgumentParser( + add_help=False, + color=True, + description="Test prog and usage colors", + prog="PROG", + usage="[prefix] %(prog)s [suffix]", + ) + heading = self.theme.heading + prog = self.theme.prog + reset = self.theme.reset + usage = self.theme.prog_extra + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{usage}[prefix] {prog}PROG{reset}{usage} [suffix]{reset} + + Test prog and usage colors + """ + ), + ) + + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. RFile.seen = {} diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 0c20e27cfda..db09e50e8f4 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -8,16 +8,20 @@ from test.support import import_helper from test.support import os_helper from test.support import _2G +from test.support import subTests import weakref import pickle import operator import struct import sys +import warnings import array from array import _array_reconstructor as array_reconstructor -sizeof_wchar = array.array('u').itemsize +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + sizeof_wchar = array.array('u').itemsize class ArraySubclass(array.array): @@ -27,7 +31,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = 'ubBhHiIlLfdqQ' +typecodes = 'uwbBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -93,8 +97,17 @@ def test_empty(self): UTF32_LE = 20 UTF32_BE = 21 + class ArrayReconstructorTest(unittest.TestCase): + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def test_error(self): self.assertRaises(TypeError, array_reconstructor, "", "b", 0, b"") @@ -176,23 +189,23 @@ def test_numbers(self): self.assertEqual(a, b, msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) - # TODO: RUSTPYTHON - requires UTF-32 encoding support in codecs and proper array reconstructor implementation - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_unicode(self): teststr = "Bonne Journ\xe9e \U0002030a\U00020347" testcases = ( (UTF16_LE, "UTF-16-LE"), (UTF16_BE, "UTF-16-BE"), - (UTF32_LE, "UTF-32-LE"), # TODO: RUSTPYTHON - (UTF32_BE, "UTF-32-BE") # TODO: RUSTPYTHON + (UTF32_LE, "UTF-32-LE"), + (UTF32_BE, "UTF-32-BE") ) for testcase in testcases: mformat_code, encoding = testcase - a = array.array('u', teststr) - b = array_reconstructor( - array.array, 'u', mformat_code, teststr.encode(encoding)) - self.assertEqual(a, b, - msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) + for c in 'uw': + a = array.array(c, teststr) + b = array_reconstructor( + array.array, c, mformat_code, teststr.encode(encoding)) + self.assertEqual(a, b, + msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) class BaseTest: @@ -204,6 +217,14 @@ class BaseTest: # outside: An entry that is not in example # minitemsize: the minimum guaranteed itemsize + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def assertEntryEqual(self, entry1, entry2): self.assertEqual(entry1, entry2) @@ -236,7 +257,7 @@ def test_buffer_info(self): self.assertEqual(bi[1], len(a)) def test_byteswap(self): - if self.typecode == 'u': + if self.typecode in ('u', 'w'): example = '\U00100100' else: example = self.example @@ -357,8 +378,6 @@ def test_reverse_iterator(self): self.assertEqual(list(a), list(self.example)) self.assertEqual(list(reversed(a)), list(iter(a))[::-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reverse_iterator_picking(self): orig = array.array(self.typecode, self.example) data = list(orig) @@ -997,6 +1016,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) @@ -1083,7 +1125,7 @@ def test_buffer(self): self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, a.frombytes, a.tobytes()) self.assertEqual(m.tobytes(), expected) - if self.typecode == 'u': + if self.typecode in ('u', 'w'): self.assertRaises(BufferError, a.fromunicode, a.tounicode()) self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, operator.imul, a, 2) @@ -1138,17 +1180,19 @@ def test_sizeof_without_buffer(self): basesize = support.calcvobjsize('Pn2Pi') support.check_sizeof(self, a, basesize) + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_initialize_with_unicode(self): - if self.typecode != 'u': + if self.typecode not in ('u', 'w'): with self.assertRaises(TypeError) as cm: a = array.array(self.typecode, 'foo') self.assertIn("cannot use a str", str(cm.exception)) with self.assertRaises(TypeError) as cm: - a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) self.assertIn("cannot use a unicode array", str(cm.exception)) else: a = array.array(self.typecode, "foo") a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) @support.cpython_only def test_obsolete_write_lock(self): @@ -1156,8 +1200,7 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1177,40 +1220,255 @@ class UnicodeTest(StringTest, unittest.TestCase): smallerexample = '\x01\u263a\x00\ufefe' biggerexample = '\x01\u263a\x01\ufeff' outside = str('\x33') - minitemsize = 2 + minitemsize = sizeof_wchar def test_unicode(self): self.assertRaises(TypeError, array.array, 'b', 'foo') - a = array.array('u', '\xa0\xc2\u1234') + a = array.array(self.typecode, '\xa0\xc2\u1234') a.fromunicode(' ') a.fromunicode('') a.fromunicode('') a.fromunicode('\x11abc\xff\u1234') s = a.tounicode() self.assertEqual(s, '\xa0\xc2\u1234 \x11abc\xff\u1234') - self.assertEqual(a.itemsize, sizeof_wchar) + self.assertEqual(a.itemsize, self.minitemsize) s = '\x00="\'a\\b\x80\xff\u0000\u0001\u1234' - a = array.array('u', s) + a = array.array(self.typecode, s) self.assertEqual( repr(a), - "array('u', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") + f"array('{self.typecode}', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") self.assertRaises(TypeError, a.fromunicode) def test_issue17223(self): - # this used to crash - if sizeof_wchar == 4: - # U+FFFFFFFF is an invalid code point in Unicode 6.0 - invalid_str = b'\xff\xff\xff\xff' - else: + if self.typecode == 'u' and sizeof_wchar == 2: # PyUnicode_FromUnicode() cannot fail with 16-bit wchar_t self.skipTest("specific to 32-bit wchar_t") - a = array.array('u', invalid_str) + + # this used to crash + # U+FFFFFFFF is an invalid code point in Unicode 6.0 + invalid_str = b'\xff\xff\xff\xff' + + a = array.array(self.typecode, invalid_str) self.assertRaises(ValueError, a.tounicode) self.assertRaises(ValueError, str, a) + def test_typecode_u_deprecation(self): + with self.assertWarns(DeprecationWarning): + array.array("u") + + def test_empty_string_mem_leak_gh140474(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + for _ in range(1000): + a = array.array('u', '') + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, 'u') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_add(self): + return super().test_add() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extend(self): + return super().test_extend() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iadd(self): + return super().test_iadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setiadd(self): + return super().test_setiadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setslice(self): + return super().test_setslice() + + +class UCS4Test(UnicodeTest): + typecode = 'w' + minitemsize = 4 + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer(self): + return super().test_buffer() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer_info(self): + return super().test_buffer_info() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_byteswap(self): + return super().test_byteswap() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_clear(self): + return super().test_clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_cmp(self): + return super().test_cmp() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor_with_iterable_argument(self): + return super().test_constructor_with_iterable_argument() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_copy(self): + return super().test_copy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_count(self): + return super().test_count() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_coveritertraverse(self): + return super().test_coveritertraverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_deepcopy(self): + return super().test_deepcopy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_delitem(self): + return super().test_delitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_iterator(self): + return super().test_exhausted_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_reverse_iterator(self): + return super().test_exhausted_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_getslice(self): + return super().test_extended_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_set_del_slice(self): + return super().test_extended_set_del_slice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_filewrite(self): + return super().test_filewrite() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromarray(self): + return super().test_fromarray() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromfile_ioerror(self): + return super().test_fromfile_ioerror() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getitem(self): + return super().test_getitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getslice(self): + return super().test_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_imul(self): + return super().test_imul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_index(self): + return super().test_index() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_insert(self): + return super().test_insert() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_issue17223(self): + return super().test_issue17223() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iterator_pickle(self): + return super().test_iterator_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_len(self): + return super().test_len() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_mul(self): + return super().test_mul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle(self): + return super().test_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle_for_empty_array(self): + return super().test_pickle_for_empty_array() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pop(self): + return super().test_pop() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reduce_ex(self): + return super().test_reduce_ex() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_remove(self): + return super().test_remove() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_repr(self): + return super().test_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse(self): + return super().test_reverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator(self): + return super().test_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator_picking(self): + return super().test_reverse_iterator_picking() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setitem(self): + return super().test_setitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_str(self): + return super().test_str() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofrombytes(self): + return super().test_tofrombytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromfile(self): + return super().test_tofromfile() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromlist(self): + return super().test_tofromlist() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_unicode(self): + return super().test_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_weakref(self): + return super().test_weakref() + + class NumberTest(BaseTest): def test_extslice(self): @@ -1442,8 +1700,8 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() @@ -1615,5 +1873,55 @@ def test_tolist(self, size): self.assertEqual(ls[:8], list(example[:8])) self.assertEqual(ls[-8:], list(example[-8:])) + def test_gh_128961(self): + a = array.array('i') + it = iter(a) + list(it) + it.__setstate__(0) + self.assertRaises(StopIteration, next, it) + + # Tests for NULL pointer dereference in array.__setitem__ + # when the index conversion mutates the array. + # See: https://github.com/python/cpython/issues/142555. + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["b", "B", "h", "H", "i", "l", "q", "I", "L", "Q"]) + def test_setitem_use_after_clear_with_int_data(self, dtype): + victim = array.array(dtype, list(range(64))) + + class Index: + def __index__(self): + victim.clear() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + self.assertEqual(len(victim), 0) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + def test_setitem_use_after_shrink_with_int_data(self): + victim = array.array('b', [1, 2, 3]) + + class Index: + def __index__(self): + victim.pop() + victim.pop() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["f", "d"]) + def test_setitem_use_after_clear_with_float_data(self, dtype): + victim = array.array(dtype, [1.0, 2.0, 3.0]) + + class Float: + def __float__(self): + victim.clear() + return 0.0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Float()) + self.assertEqual(len(victim), 0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000..1c1985519cd --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,214 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py index 28d32b2941f..b76f98901d2 100644 --- a/Lib/test/test_ast/snippets.py +++ b/Lib/test/test_ast/snippets.py @@ -364,6 +364,12 @@ "f'{a:.2f}'", "f'{a!r}'", "f'foo({a})'", + # TemplateStr and Interpolation + "t'{a}'", + "t'{a:.2f}'", + "t'{a!r}'", + "t'{a!r:.2f}'", + "t'foo({a})'", ] @@ -597,5 +603,10 @@ def main(): ('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), ('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), ('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +('Expression', ('TemplateStr', (1, 0, 1, 6), [('Interpolation', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 10), [('Interpolation', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 8), [('Interpolation', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 12), [('Interpolation', (1, 2, 1, 11), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, ('JoinedStr', (1, 6, 1, 10), [('Constant', (1, 7, 1, 10), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('Interpolation', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), 'a', -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), ] main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 09d9444d5d9..e2e619c5a23 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,34 +1,59 @@ +import _ast_unparse import ast import builtins +import contextlib import copy import dis import enum +import itertools import os import re import sys +import tempfile import textwrap import types import unittest -import warnings import weakref -from functools import partial +from io import StringIO +from pathlib import Path from textwrap import dedent - try: import _testinternalcapi except ImportError: _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper +from test.support import os_helper +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_unlimited_stack_size from test.support.ast_helper import ASTTestMixin +from test.support.import_helper import ensure_lazy_imports from test.test_ast.utils import to_tuple from test.test_ast.snippets import ( eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results ) +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) + + +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("ast", {"contextlib", "enum", "inspect", "re", "collections", "argparse"}) + + class AST_Tests(unittest.TestCase): maxDiff = None @@ -37,7 +62,7 @@ def _is_ast_node(self, name, node): return False if "ast" not in node.__module__: return False - return name != "AST" and name[0].isupper() + return name != 'AST' and name[0].isupper() def _assertTrueorder(self, ast_node, parent_pos): if not isinstance(ast_node, ast.AST) or ast_node._fields is None: @@ -50,7 +75,7 @@ def _assertTrueorder(self, ast_node, parent_pos): value = getattr(ast_node, name) if isinstance(value, list): first_pos = parent_pos - if value and name == "decorator_list": + if value and name == 'decorator_list': first_pos = (value[0].lineno, value[0].col_offset) for child in value: self._assertTrueorder(child, first_pos) @@ -72,7 +97,6 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_AST_fields_NULL_check(self): # See: https://github.com/python/cpython/issues/126105 old_value = ast.AST._fields @@ -90,11 +114,10 @@ def cleanup(): with self.assertRaisesRegex(AttributeError, msg): ast.AST() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .X object at 0x7e85c3a80> is not None def test_AST_garbage_collection(self): class X: pass - a = ast.AST() a.x = X() a.x.a = a @@ -103,13 +126,10 @@ class X: support.gc_collect() self.assertIsNone(ref()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_snippets(self): - for input, output, kind in ( - (exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval"), - ): + for input, output, kind in ((exec_tests, exec_results, "exec"), + (single_tests, single_results, "single"), + (eval_tests, eval_results, "eval")): for i, o in zip(input, output): with self.subTest(action="parsing", input=i): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) @@ -118,18 +138,23 @@ def test_snippets(self): with self.subTest(action="compiling", input=i, kind=kind): compile(ast_tree, "?", kind) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests for snippet in snippets_to_validate: tree = ast.parse(snippet) - compile(tree, "", "exec") + compile(tree, '', 'exec') + + def test_parse_invalid_ast(self): + # see gh-130139 + for optval in (-1, 0, 1, 2): + self.assertRaises(TypeError, ast.parse, ast.Constant(42), + optimize=optval) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_optimization_levels__debug__(self): - cases = [(-1, "__debug__"), (0, "__debug__"), (1, False), (2, False)] - for optval, expected in cases: + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] + for (optval, expected) in cases: with self.subTest(optval=optval, expected=expected): res1 = ast.parse("__debug__", optimize=optval) res2 = ast.parse(ast.parse("__debug__"), optimize=optval) @@ -142,33 +167,10 @@ def test_optimization_levels__debug__(self): self.assertIsInstance(res.body[0].value, ast.Name) self.assertEqual(res.body[0].value.id, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_optimization_levels_const_folding(self): - folded = ("Expr", (1, 0, 1, 5), ("Constant", (1, 0, 1, 5), 3, None)) - not_folded = ( - "Expr", - (1, 0, 1, 5), - ( - "BinOp", - (1, 0, 1, 5), - ("Constant", (1, 0, 1, 1), 1, None), - ("Add",), - ("Constant", (1, 4, 1, 5), 2, None), - ), - ) - - cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] - for optval, expected in cases: - with self.subTest(optval=optval): - tree1 = ast.parse("1 + 2", optimize=optval) - tree2 = ast.parse(ast.parse("1 + 2"), optimize=optval) - for tree in [tree1, tree2]: - res = to_tuple(tree.body[0]) - self.assertEqual(res, expected) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_invalid_position_information(self): - invalid_linenos = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + invalid_linenos = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] for lineno, end_lineno in invalid_linenos: with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): @@ -177,42 +179,29 @@ def test_invalid_position_information(self): tree.body[0].lineno = lineno tree.body[0].end_lineno = end_lineno with self.assertRaises(ValueError): - compile(tree, "", "exec") + compile(tree, '', 'exec') - invalid_col_offsets = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + invalid_col_offsets = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] for col_offset, end_col_offset in invalid_col_offsets: - with self.subTest( - f"Check invalid col_offset {col_offset}:{end_col_offset}" - ): + with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): snippet = "a = 1" tree = ast.parse(snippet) tree.body[0].col_offset = col_offset tree.body[0].end_col_offset = end_col_offset with self.assertRaises(ValueError): - compile(tree, "", "exec") + compile(tree, '', 'exec') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compilation_of_ast_nodes_with_default_end_position_values(self): - tree = ast.Module( - body=[ - ast.Import( - names=[ast.alias(name="builtins", lineno=1, col_offset=0)], - lineno=1, - col_offset=0, - ), - ast.Import( - names=[ast.alias(name="traceback", lineno=0, col_offset=0)], - lineno=0, - col_offset=1, - ), - ], - type_ignores=[], - ) + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. compile(tree, "", "exec") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "end_lineno" missing from alias def test_negative_locations_for_compile(self): # See https://github.com/python/cpython/issues/130775 alias = ast.alias(name='traceback', lineno=0, col_offset=0) @@ -233,6 +222,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) @@ -253,7 +367,7 @@ def test_alias(self): im = ast.parse("from bar import y").body[0] self.assertEqual(len(im.names), 1) alias = im.names[0] - self.assertEqual(alias.name, "y") + self.assertEqual(alias.name, 'y') self.assertIsNone(alias.asname) self.assertEqual(alias.lineno, 1) self.assertEqual(alias.end_lineno, 1) @@ -262,7 +376,7 @@ def test_alias(self): im = ast.parse("from bar import *").body[0] alias = im.names[0] - self.assertEqual(alias.name, "*") + self.assertEqual(alias.name, '*') self.assertIsNone(alias.asname) self.assertEqual(alias.lineno, 1) self.assertEqual(alias.end_lineno, 1) @@ -288,45 +402,17 @@ def test_alias(self): self.assertEqual(alias.end_col_offset, 17) def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) - - def test_import_deprecated(self): - ast = import_fresh_module("ast") - depr_regex = ( - r"ast\.{} is deprecated and will be removed in Python 3.14; " - r"use ast\.Constant instead" - ) - for name in "Num", "Str", "Bytes", "NameConstant", "Ellipsis": - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) - for name in ("Num", "Str", "Bytes", "NameConstant", "Ellipsis"): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - @unittest.expectedFailure # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__' def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {"Num", "Str", "Bytes", "NameConstant", "Ellipsis"}: - continue # constructor has a different signature - if name == "Index": + if name == 'Index': continue if self._is_ast_node(name, item): x = self._construct_ast_class(item) @@ -337,42 +423,28 @@ def _construct_ast_class(self, cls): kwargs = {} for name, typ in cls.__annotations__.items(): if typ is str: - kwargs[name] = "capybara" + kwargs[name] = 'capybara' elif typ is int: kwargs[name] = 42 elif typ is object: - kwargs[name] = b"capybara" + kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): kwargs[name] = self._construct_ast_class(typ) return cls(**kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_arguments(self): x = ast.arguments() - self.assertEqual( - x._fields, - ( - "posonlyargs", - "args", - "vararg", - "kwonlyargs", - "kw_defaults", - "kwarg", - "defaults", - ), - ) - self.assertEqual( - x.__annotations__, - { - "posonlyargs": list[ast.arg], - "args": list[ast.arg], - "vararg": ast.arg | None, - "kwonlyargs": list[ast.arg], - "kw_defaults": list[ast.expr], - "kwarg": ast.arg | None, - "defaults": list[ast.expr], - }, - ) + self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', + 'kw_defaults', 'kwarg', 'defaults')) + self.assertEqual(ast.arguments.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) self.assertEqual(x.args, []) self.assertIsNone(x.vararg) @@ -381,117 +453,16 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ("value", "kind")) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ("value", "kind")) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo="bar").foo, "bar") - - with self.assertRaisesRegex( - TypeError, "Num got multiple values for argument 'n'" - ): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str("42").s, "42") - self.assertEqual(ast.Bytes(b"42").s, b"42") - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - "an error in Python 3.15.", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - "an error in Python 3.15.", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " - "arbitrary keyword arguments is deprecated and will be removed in Python " - "3.15.", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() - self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value @@ -510,7 +481,7 @@ def test_classattrs(self): x = ast.Constant(42, lineno=0) self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) self.assertRaises(TypeError, ast.Constant, 1, None, 2) @@ -518,234 +489,32 @@ def test_classattrs(self): # Arbitrary keyword arguments are supported (but deprecated) with self.assertWarns(DeprecationWarning): - self.assertEqual(ast.Constant(1, foo="bar").foo, "bar") + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - with self.assertRaisesRegex( - TypeError, "Constant got multiple values for argument 'value'" - ): + with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) self.assertEqual(ast.Constant(42).value, 42) self.assertEqual(ast.Constant(4.25).value, 4.25) self.assertEqual(ast.Constant(4.25j).value, 4.25j) - self.assertEqual(ast.Constant("42").value, "42") - self.assertEqual(ast.Constant(b"42").value, b"42") + self.assertEqual(ast.Constant('42').value, '42') + self.assertEqual(ast.Constant(b'42').value, b'42') self.assertIs(ast.Constant(True).value, True) self.assertIs(ast.Constant(False).value, False) self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str("42")), ast.Constant) - self.assertIs(type(ast.Bytes(b"42")), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - "ast.{} is deprecated and will be removed in Python 3.14; " - "use ast.Constant instead" - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant"), - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str("42") - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b"42") - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant("42"), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b"42"), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str("42") - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in "42", True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant("42"), Bytes) - assertNameConstantDeprecated( - self.assertNotIsInstance, Constant(42), NameConstant - ) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) - assertNameConstantDeprecated( - self.assertNotIsInstance, Constant(1), NameConstant - ) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) - - class S(str): - pass - - with assertStrDeprecated(): - self.assertIsInstance(Constant(S("42")), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S("42")), Num) - - @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = "spam" - - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, "spam") - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - ], - ) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.z = "spam" - + self.z = 'spam' class N2(ast.Constant): pass n = N(42) self.assertEqual(n.value, 42) - self.assertEqual(n.z, "spam") + self.assertEqual(n.z, 'spam') self.assertEqual(type(n), N) self.assertTrue(isinstance(n, N)) self.assertTrue(isinstance(n, ast.Constant)) @@ -760,12 +529,11 @@ def test_module(self): x = ast.Module(body, []) self.assertEqual(x.body, body) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_nodeclasses(self): # Zero arguments constructor explicitly allowed (but deprecated) with self.assertWarns(DeprecationWarning): x = ast.BinOp() - self.assertEqual(x._fields, ("left", "op", "right")) + self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too x.foobarbaz = 5 @@ -812,15 +580,14 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("but got ", "eval") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_empty_yield_from(self): # Issue 16546: yield from value is not optional. empty_yield_from = ast.parse("def f():\n yield from g()") @@ -849,15 +618,13 @@ def test_issue31592(self): # There shouldn't be an assertion failure in case of a bad # unicodedata.normalize(). import unicodedata - def bad_normalize(*args): return None - - with support.swap_attr(unicodedata, "normalize", bad_normalize): - self.assertRaises(TypeError, ast.parse, "\u03d5") + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') def test_issue18374_binop_col_offset(self): - tree = ast.parse("4+5+6+7") + tree = ast.parse('4+5+6+7') parent_binop = tree.body[0].value child_binop = parent_binop.left grandchild_binop = child_binop.left @@ -868,7 +635,7 @@ def test_issue18374_binop_col_offset(self): self.assertEqual(grandchild_binop.col_offset, 0) self.assertEqual(grandchild_binop.end_col_offset, 3) - tree = ast.parse("4+5-\\\n 6-7") + tree = ast.parse('4+5-\\\n 6-7') parent_binop = tree.body[0].value child_binop = parent_binop.left grandchild_binop = child_binop.left @@ -888,62 +655,264 @@ def test_issue18374_binop_col_offset(self): self.assertEqual(grandchild_binop.end_lineno, 1) def test_issue39579_dotted_name_end_col_offset(self): - tree = ast.parse("@a.b.c\ndef f(): pass") + tree = ast.parse('@a.b.c\ndef f(): pass') attr_b = tree.body[0].decorator_list[0].value self.assertEqual(attr_b.end_col_offset, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ast_asdl_signature(self): - self.assertEqual( - ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)" - ) + self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") self.assertEqual(ast.GtE.__doc__, "GtE") self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") - self.assertEqual( - ast.cmpop.__doc__, - "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", - ) + self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_compare_basics(self): + self.assertTrue(ast.compare(ast.parse("x = 10"), ast.parse("x = 10"))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse(""))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse("x"))) + self.assertFalse( + ast.compare(ast.parse("x = 10;y = 20"), ast.parse("class C:pass")) + ) + + def test_compare_modified_ast(self): + # The ast API is a bit underspecified. The objects are mutable, + # and even _fields and _attributes are mutable. The compare() does + # some simple things to accommodate mutability. + a = ast.parse("m * x + b", mode="eval") + b = ast.parse("m * x + b", mode="eval") + self.assertTrue(ast.compare(a, b)) + + a._fields = a._fields + ("spam",) + a.spam = "Spam" + self.assertNotEqual(a._fields, b._fields) + self.assertFalse(ast.compare(a, b)) + self.assertFalse(ast.compare(b, a)) + + b._fields = a._fields + b.spam = a.spam + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(b, a)) + + b._attributes = b._attributes + ("eggs",) + b.eggs = "eggs" + self.assertNotEqual(a._attributes, b._attributes) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + self.assertFalse(ast.compare(b, a, compare_attributes=True)) + + a._attributes = b._attributes + a.eggs = b.eggs + self.assertTrue(ast.compare(a, b, compare_attributes=True)) + self.assertTrue(ast.compare(b, a, compare_attributes=True)) + + def test_compare_literals(self): + constants = ( + -20, + 20, + 20.0, + 1, + 1.0, + True, + 0, + False, + frozenset(), + tuple(), + "ABCD", + "abcd", + "中文字", + 1e1000, + -1e1000, + ) + for next_index, constant in enumerate(constants[:-1], 1): + next_constant = constants[next_index] + with self.subTest(literal=constant, next_literal=next_constant): + self.assertTrue( + ast.compare(ast.Constant(constant), ast.Constant(constant)) + ) + self.assertFalse( + ast.compare( + ast.Constant(constant), ast.Constant(next_constant) + ) + ) + + same_looking_literal_cases = [ + {1, 1.0, True, 1 + 0j}, + {0, 0.0, False, 0 + 0j}, + ] + for same_looking_literals in same_looking_literal_cases: + for literal in same_looking_literals: + for same_looking_literal in same_looking_literals - {literal}: + self.assertFalse( + ast.compare( + ast.Constant(literal), + ast.Constant(same_looking_literal), + ) + ) + + def test_compare_fieldless(self): + self.assertTrue(ast.compare(ast.Add(), ast.Add())) + self.assertFalse(ast.compare(ast.Sub(), ast.Add())) + + # test that missing runtime fields is handled in ast.compare() + a1, a2 = ast.Name('a'), ast.Name('a') + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2)) + del a1.id + self.assertFalse(ast.compare(a1, a2)) + del a2.id + self.assertTrue(ast.compare(a1, a2)) + + def test_compare_modes(self): + for mode, sources in ( + ("exec", exec_tests), + ("eval", eval_tests), + ("single", single_tests), + ): + for source in sources: + a = ast.parse(source, mode=mode) + b = ast.parse(source, mode=mode) + self.assertTrue( + ast.compare(a, b), f"{ast.dump(a)} != {ast.dump(b)}" + ) + + def test_compare_attributes_option(self): + def parse(a, b): + return ast.parse(a), ast.parse(b) + + a, b = parse("2 + 2", "2+2") + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(a, b, compare_attributes=False)) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + + def test_compare_attributes_option_missing_attribute(self): + # test that missing runtime attributes is handled in ast.compare() + a1, a2 = ast.Name('a', lineno=1), ast.Name('a', lineno=1) + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + del a1.lineno + self.assertFalse(ast.compare(a1, a2, compare_attributes=True)) + del a2.lineno + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + def test_positional_only_feature_version(self): - ast.parse("def foo(x, /): ...", feature_version=(3, 8)) - ast.parse("def bar(x=1, /): ...", feature_version=(3, 8)) + ast.parse('def foo(x, /): ...', feature_version=(3, 8)) + ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) with self.assertRaises(SyntaxError): - ast.parse("def foo(x, /): ...", feature_version=(3, 7)) + ast.parse('def foo(x, /): ...', feature_version=(3, 7)) with self.assertRaises(SyntaxError): - ast.parse("def bar(x=1, /): ...", feature_version=(3, 7)) + ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) - ast.parse("lambda x, /: ...", feature_version=(3, 8)) - ast.parse("lambda x=1, /: ...", feature_version=(3, 8)) + ast.parse('lambda x, /: ...', feature_version=(3, 8)) + ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) with self.assertRaises(SyntaxError): - ast.parse("lambda x, /: ...", feature_version=(3, 7)) + ast.parse('lambda x, /: ...', feature_version=(3, 7)) with self.assertRaises(SyntaxError): - ast.parse("lambda x=1, /: ...", feature_version=(3, 7)) + ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_assignment_expression_feature_version(self): - ast.parse("(x := 0)", feature_version=(3, 8)) + ast.parse('(x := 0)', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('(x := 0)', feature_version=(3, 7)) + + def test_pep750_tstring(self): + code = 't""' + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) with self.assertRaises(SyntaxError): - ast.parse("(x := 0)", feature_version=(3, 7)) + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_with_single_expr(self): + single_expr = textwrap.dedent(""" + try: + ... + except{0} TypeError: + ... + """) + + single_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} TypeError as exc: + ... + """) + + single_tuple_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError,): + ... + """) + + single_tuple_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError,) as exc: + ... + """) + + single_parens_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError): + ... + """) + + single_parens_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError) as exc: + ... + """) + + for code in [ + single_expr, + single_expr_with_as, + single_tuple_expr, + single_tuple_expr_with_as, + single_parens_expr, + single_parens_expr_with_as, + ]: + for star in [True, False]: + code = code.format('*' if star else '') + with self.subTest(code=code, star=star): + ast.parse(code, feature_version=(3, 14)) + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_star_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except* ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) def test_conditional_context_managers_parse_with_low_feature_version(self): # regression test for gh-115881 - ast.parse("with (x() if y else z()): ...", feature_version=(3, 8)) + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_exception_groups_feature_version(self): - code = dedent(""" + code = dedent(''' try: ... except* Exception: ... - """) + ''') ast.parse(code) with self.assertRaises(SyntaxError): ast.parse(code, feature_version=(3, 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_feature_version(self): samples = [ "type X = int", @@ -956,7 +925,7 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_type_params_default_feature_version(self): samples = [ "type X[*Ts=int] = int", @@ -971,21 +940,18 @@ def test_type_params_default_feature_version(self): def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): - ast.parse("pass", feature_version=(2, 7)) + ast.parse('pass', feature_version=(2, 7)) with self.assertRaises(ValueError): - ast.parse("pass", feature_version=(4, 0)) + ast.parse('pass', feature_version=(4, 0)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_name(self): for constant in "True", "False", "None": expr = ast.Expression(ast.Name(constant, ast.Load())) ast.fix_missing_locations(expr) - with self.assertRaisesRegex( - ValueError, f"identifier field can't represent '{constant}' constant" - ): + with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_unicode_name(self): constants = [ ("True", b"Tru\xe1\xb5\x89"), @@ -997,45 +963,44 @@ def test_constant_as_unicode_name(self): f"identifier field can't represent '{constant[0]}' constant"): ast.parse(constant[1], mode="eval") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" - - NAMED_EXPR = enum.auto() # := - TUPLE = enum.auto() # , - YIELD = enum.auto() # 'yield', 'yield from' - TEST = enum.auto() # 'if'-'else', 'lambda' - OR = enum.auto() # 'or' - AND = enum.auto() # 'and' - NOT = enum.auto() # 'not' - CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' + NAMED_EXPR = enum.auto() # := + TUPLE = enum.auto() # , + YIELD = enum.auto() # 'yield', 'yield from' + TEST = enum.auto() # 'if'-'else', 'lambda' + OR = enum.auto() # 'or' + AND = enum.auto() # 'and' + NOT = enum.auto() # 'not' + CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' EXPR = enum.auto() - BOR = EXPR # '|' - BXOR = enum.auto() # '^' - BAND = enum.auto() # '&' - SHIFT = enum.auto() # '<<', '>>' - ARITH = enum.auto() # '+', '-' - TERM = enum.auto() # '*', '@', '/', '%', '//' - FACTOR = enum.auto() # unary '+', '-', '~' - POWER = enum.auto() # '**' - AWAIT = enum.auto() # 'await' + BOR = EXPR # '|' + BXOR = enum.auto() # '^' + BAND = enum.auto() # '&' + SHIFT = enum.auto() # '<<', '>>' + ARITH = enum.auto() # '+', '-' + TERM = enum.auto() # '*', '@', '/', '%', '//' + FACTOR = enum.auto() # unary '+', '-', '~' + POWER = enum.auto() # '**' + AWAIT = enum.auto() # 'await' ATOM = enum.auto() - def next(self): try: return self.__class__(self + 1) except ValueError: return self - - enum._test_simple_enum(_Precedence, ast._Precedence) + enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only + @skip_if_unlimited_stack_size + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): - fail_depth = support.exceeds_recursion_limit() - crash_depth = 100_000 - success_depth = int(support.get_c_recursion_limit() * 0.8) + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 + success_depth = 200 if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) @@ -1043,12 +1008,13 @@ def test_ast_recursion_limit(self): def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format(prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - with support.infinite_recursion(): - ast.parse(broken) + + broken = prefix + repeated * crash_depth + details = "Compiling ({!r} + {!r} * {})".format( + prefix, repeated, crash_depth) + with self.assertRaises(RecursionError, msg=details): + with support.infinite_recursion(): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") @@ -1056,9 +1022,8 @@ def check_limit(prefix, repeated): check_limit("a", "*a") def test_null_bytes(self): - with self.assertRaises( - SyntaxError, msg="source code string cannot contain null bytes" - ): + with self.assertRaises(SyntaxError, + msg="source code string cannot contain null bytes"): ast.parse("a\0b") def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: @@ -1074,7 +1039,7 @@ def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None with self.assertRaisesRegex(ValueError, f"^{e}$"): compile(tree, "", "exec") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_none_checks(self) -> None: tests = [ (ast.alias, "name", "import spam as SPAM"), @@ -1088,11 +1053,63 @@ def test_none_checks(self) -> None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + + def test_tstring(self): + # Test AST structure for simple t-string + tree = ast.parse('t"Hello"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + + # Test AST for t-string with interpolation + tree = ast.parse('t"Hello {name}"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels_const_folding(self): + return super().test_optimization_levels_const_folding() + + @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 + def test_constant_subclasses_deprecated(self): + return super().test_constant_subclasses_deprecated() + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" - @unittest.expectedFailure # TODO: RUSTPYTHON + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + def test_pickling(self): import pickle @@ -1103,6 +1120,7 @@ def test_pickling(self): ast2 = pickle.loads(pickle.dumps(tree, protocol)) self.assertEqual(to_tuple(ast2), to_tuple(tree)) + @skip_if_unlimited_stack_size def test_copy_with_parents(self): # gh-120108 code = """ @@ -1157,66 +1175,317 @@ def test_copy_with_parents(self): for node in ast.walk(tree2): for child in ast.iter_child_nodes(node): - if hasattr(child, "parent") and not isinstance( - child, - ( - ast.expr_context, - ast.boolop, - ast.unaryop, - ast.cmpop, - ast.operator, - ), - ): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): self.assertEqual(to_tuple(child.parent), to_tuple(node)) + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertHasAttr(klass, '__replace__') + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'x' is not 'x' + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ missing\ 1\ keyword\ argument:\ 'id'\." does not match "replace() does not support Name objects" + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_accept_missing_field_with_default(self): + node = ast.FunctionDef(name="foo", args=ast.arguments()) + self.assertIs(node.returns, None) + self.assertEqual(node.decorator_list, []) + node2 = copy.replace(node, name="bar") + self.assertEqual(node2.name, "bar") + self.assertIs(node2.returns, None) + self.assertEqual(node2.decorator_list, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'extra'\." does not match "replace() does not support Name objects" + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertHasAttr(node, 'extra') + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'unknown'\." does not match "replace() does not support Name objects" + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_replace_non_str_kwarg(self): + node = ast.Name(id="x") + errmsg = "got an unexpected keyword argument ", "exec", ast.PyCF_ONLY_AST) + a = ast.parse('foo(1 + 1)') + b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) self.assertEqual(ast.dump(a), ast.dump(b)) def test_parse_in_error(self): try: - 1 / 0 + 1/0 except Exception: with self.assertRaises(SyntaxError) as e: ast.literal_eval(r"'\U'") self.assertIsNotNone(e.exception.__context__) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual( - ast.dump(node), + self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" ) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')]))])", + "Constant('and cheese')]))])" ) - self.assertEqual( - ast.dump(node, include_attributes=True), + self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])", + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_indent(self): node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual( - ast.dump(node, indent=3), - """\ + self.assertEqual(ast.dump(node, indent=3), """\ Module( body=[ Expr( @@ -1224,11 +1493,8 @@ def test_dump_indent(self): func=Name(id='spam', ctx=Load()), args=[ Name(id='eggs', ctx=Load()), - Constant(value='and cheese')]))])""", - ) - self.assertEqual( - ast.dump(node, annotate_fields=False, indent="\t"), - """\ + Constant(value='and cheese')]))])""") + self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ \t\tExpr( @@ -1236,11 +1502,8 @@ def test_dump_indent(self): \t\t\t\tName('spam', Load()), \t\t\t\t[ \t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')]))])""", - ) - self.assertEqual( - ast.dump(node, include_attributes=True, indent=3), - """\ +\t\t\t\t\tConstant('and cheese')]))])""") + self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( body=[ Expr( @@ -1273,78 +1536,72 @@ def test_dump_indent(self): lineno=1, col_offset=0, end_lineno=1, - end_col_offset=24)])""", - ) + end_col_offset=24)])""") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), "Raise()") - self.assertEqual( - ast.dump(node, include_attributes=True), "Raise(lineno=3, col_offset=4)" + self.assertEqual(ast.dump(node), + "Raise()" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(lineno=3, col_offset=4)" + ) + node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise(exc=Name(id='e', ctx=Load()))" ) - node = ast.Raise(exc=ast.Name(id="e", ctx=ast.Load()), lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), "Raise(exc=Name(id='e', ctx=Load()))") - self.assertEqual( - ast.dump(node, annotate_fields=False), "Raise(Name('e', Load()))" + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(Name('e', Load()))" ) - self.assertEqual( - ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)", + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" ) - self.assertEqual( - ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)", + self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), + "Raise(Name('e', Load()), lineno=3, col_offset=4)" ) - node = ast.Raise(cause=ast.Name(id="e", ctx=ast.Load())) - self.assertEqual(ast.dump(node), "Raise(cause=Name(id='e', ctx=Load()))") - self.assertEqual( - ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" + node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) + self.assertEqual(ast.dump(node), + "Raise(cause=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(cause=Name('e', Load()))" ) # Arguments: node = ast.arguments(args=[ast.arg("x")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([], [arg('x')])", ) node = ast.arguments(posonlyargs=[ast.arg("x")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([arg('x')])", ) - node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([arg('x')], kwonlyargs=[arg('y')])", ) - node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) - self.assertEqual( - ast.dump(node, annotate_fields=False), + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments([], [arg('x')], kwonlyargs=[arg('y')])", ) node = ast.arguments() - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "arguments()", ) # Classes: node = ast.ClassDef( - "T", + 'T', [], - [ast.keyword("a", ast.Constant(None))], + [ast.keyword('a', ast.Constant(None))], [], - [ast.Name("dataclass", ctx=ast.Load())], + [ast.Name('dataclass', ctx=ast.Load())], ) - self.assertEqual( - ast.dump(node), + self.assertEqual(ast.dump(node), "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", ) - self.assertEqual( - ast.dump(node, annotate_fields=False), + self.assertEqual(ast.dump(node, annotate_fields=False), "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_show_empty(self): def check_node(node, empty, full, **kwargs): with self.subTest(show_empty=False): @@ -1369,7 +1626,7 @@ def check_text(code, empty, full, **kwargs): check_node( # Corner case: there are no real `Name` instances with `id=''`: - ast.Name(id="", ctx=ast.Load()), + ast.Name(id='', ctx=ast.Load()), empty="Name(id='', ctx=Load())", full="Name(id='', ctx=Load())", ) @@ -1399,11 +1656,23 @@ def check_text(code, empty, full, **kwargs): ) check_node( - ast.Constant(value=""), + ast.Constant(value=''), empty="Constant(value='')", full="Constant(value='')", ) + check_node( + ast.Interpolation(value=ast.Constant(42), str=None, conversion=-1), + empty="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + full="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=[], conversion=-1), + empty="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + full="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + ) + check_text( "def a(b: int = 0, *, c): ...", empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", @@ -1435,37 +1704,31 @@ def check_text(code, empty, full, **kwargs): full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^ def test_copy_location(self): - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Constant(2), src.body.right) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, " - "end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, " - "lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, " - "col_offset=0, end_lineno=1, end_col_offset=5))", - ) - func = ast.Name("spam", ast.Load()) - src = ast.Call( - col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func - ) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' + 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' + 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' + 'col_offset=0, end_lineno=1, end_col_offset=5))' + ) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) self.assertEqual(new.col_offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fix_missing_locations(self): src = ast.parse('write("spam")') - src.body.append( - ast.Expr(ast.Call(ast.Name("spam", ast.Load()), [ast.Constant("eggs")], [])) - ) + src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), + [ast.Constant('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None - self.assertEqual( - ast.dump(src, include_attributes=True), + self.assertEqual(ast.dump(src, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " @@ -1475,29 +1738,27 @@ def test_fix_missing_locations(self): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])", + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ def test_increment_lineno(self): - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " - "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " - "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " - "col_offset=0, end_lineno=4, end_col_offset=5))", + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) # issue10869: do not increment lineno of root twice - src = ast.parse("1 + 1", mode="eval") + src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) - self.assertEqual( - ast.dump(src, include_attributes=True), - "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " - "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " - "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " - "col_offset=0, end_lineno=4, end_col_offset=5))", + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' ) src = ast.Call( func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 @@ -1505,85 +1766,82 @@ def test_increment_lineno(self): self.assertEqual(ast.increment_lineno(src).lineno, 2) self.assertIsNone(ast.increment_lineno(src).end_lineno) - @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range def test_increment_lineno_on_module(self): - src = ast.parse( - dedent("""\ + src = ast.parse(dedent("""\ a = 1 b = 2 # type: ignore c = 3 d = 4 # type: ignore@tag - """), - type_comments=True, - ) + """), type_comments=True) ast.increment_lineno(src, n=5) self.assertEqual(src.type_ignores[0].lineno, 7) self.assertEqual(src.type_ignores[1].lineno, 9) - self.assertEqual(src.type_ignores[1].tag, "@tag") + self.assertEqual(src.type_ignores[1].tag, '@tag') def test_iter_fields(self): - node = ast.parse("foo()", mode="eval") + node = ast.parse('foo()', mode='eval') d = dict(ast.iter_fields(node.body)) - self.assertEqual(d.pop("func").id, "foo") - self.assertEqual(d, {"keywords": [], "args": []}) + self.assertEqual(d.pop('func').id, 'foo') + self.assertEqual(d, {'keywords': [], 'args': []}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_child_nodes(self): - node = ast.parse("spam(23, 42, eggs='leek')", mode="eval") + node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) iterator = ast.iter_child_nodes(node.body) - self.assertEqual(next(iterator).id, "spam") + self.assertEqual(next(iterator).id, 'spam') self.assertEqual(next(iterator).value, 23) self.assertEqual(next(iterator).value, 42) - self.assertEqual( - ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))", + self.assertEqual(ast.dump(next(iterator)), + "keyword(arg='eggs', value=Constant(value='leek'))" ) def test_get_docstring(self): node = ast.parse('"""line one\n line two"""') - self.assertEqual(ast.get_docstring(node), "line one\nline two") + self.assertEqual(ast.get_docstring(node), + 'line one\nline two') node = ast.parse('class foo:\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') node = ast.parse('def foo():\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0]), "spam\nham") + self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0], clean=False), "spam\n ham") + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') - node = ast.parse("x") + node = ast.parse('x') self.assertRaises(TypeError, ast.get_docstring, node.body[0]) def test_get_docstring_none(self): - self.assertIsNone(ast.get_docstring(ast.parse(""))) + self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') self.assertIsNone(ast.get_docstring(node)) - node = ast.parse("def foo():\n pass") + node = ast.parse('def foo():\n pass') self.assertIsNone(ast.get_docstring(node)) - node = ast.parse("class foo:\n pass") + node = ast.parse('class foo:\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('class foo:\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("class foo:\n def bar(self): pass") + node = ast.parse('class foo:\n def bar(self): pass') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("def foo():\n pass") + node = ast.parse('def foo():\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("async def foo():\n pass") + node = ast.parse('async def foo():\n pass') self.assertIsNone(ast.get_docstring(node.body[0])) node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse("async def foo():\n 42") + node = ast.parse('async def foo():\n 42') self.assertIsNone(ast.get_docstring(node.body[0])) def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): @@ -1606,83 +1864,79 @@ def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): self.assertEqual(node.body[2].lineno, 13) def test_elif_stmt_start_position(self): - node = ast.parse("if a:\n pass\nelif b:\n pass\n") + node = ast.parse('if a:\n pass\nelif b:\n pass\n') elif_stmt = node.body[0].orelse[0] self.assertEqual(elif_stmt.lineno, 3) self.assertEqual(elif_stmt.col_offset, 0) def test_elif_stmt_start_position_with_else(self): - node = ast.parse("if a:\n pass\nelif b:\n pass\nelse:\n pass\n") + node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') elif_stmt = node.body[0].orelse[0] self.assertEqual(elif_stmt.lineno, 3) self.assertEqual(elif_stmt.col_offset, 0) def test_starred_expr_end_position_within_call(self): - node = ast.parse("f(*[0, 1])") + node = ast.parse('f(*[0, 1])') starred_expr = node.body[0].value.args[0] self.assertEqual(starred_expr.end_lineno, 1) self.assertEqual(starred_expr.end_col_offset, 9) def test_literal_eval(self): - self.assertEqual(ast.literal_eval("[1, 2, 3]"), [1, 2, 3]) + self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) - self.assertEqual(ast.literal_eval("(True, False, None)"), (True, False, None)) - self.assertEqual(ast.literal_eval("{1, 2, 3}"), {1, 2, 3}) + self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) + self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) self.assertEqual(ast.literal_eval('b"hi"'), b"hi") - self.assertEqual(ast.literal_eval("set()"), set()) - self.assertRaises(ValueError, ast.literal_eval, "foo()") - self.assertEqual(ast.literal_eval("6"), 6) - self.assertEqual(ast.literal_eval("+6"), 6) - self.assertEqual(ast.literal_eval("-6"), -6) - self.assertEqual(ast.literal_eval("3.25"), 3.25) - self.assertEqual(ast.literal_eval("+3.25"), 3.25) - self.assertEqual(ast.literal_eval("-3.25"), -3.25) - self.assertEqual(repr(ast.literal_eval("-0.0")), "-0.0") - self.assertRaises(ValueError, ast.literal_eval, "++6") - self.assertRaises(ValueError, ast.literal_eval, "+True") - self.assertRaises(ValueError, ast.literal_eval, "2+3") - - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + self.assertEqual(ast.literal_eval('set()'), set()) + self.assertRaises(ValueError, ast.literal_eval, 'foo()') + self.assertEqual(ast.literal_eval('6'), 6) + self.assertEqual(ast.literal_eval('+6'), 6) + self.assertEqual(ast.literal_eval('-6'), -6) + self.assertEqual(ast.literal_eval('3.25'), 3.25) + self.assertEqual(ast.literal_eval('+3.25'), 3.25) + self.assertEqual(ast.literal_eval('-3.25'), -3.25) + self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') + self.assertRaises(ValueError, ast.literal_eval, '++6') + self.assertRaises(ValueError, ast.literal_eval, '+True') + self.assertRaises(ValueError, ast.literal_eval, '2+3') + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_literal_eval_str_int_limit(self): with support.adjust_int_max_str_digits(4000): - ast.literal_eval("3" * 4000) # no error + ast.literal_eval('3'*4000) # no error with self.assertRaises(SyntaxError) as err_ctx: - ast.literal_eval("3" * 4001) - self.assertIn("Exceeds the limit ", str(err_ctx.exception)) - self.assertIn(" Consider hexadecimal ", str(err_ctx.exception)) + ast.literal_eval('3'*4001) + self.assertIn('Exceeds the limit ', str(err_ctx.exception)) + self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) def test_literal_eval_complex(self): # Issue #4907 - self.assertEqual(ast.literal_eval("6j"), 6j) - self.assertEqual(ast.literal_eval("-6j"), -6j) - self.assertEqual(ast.literal_eval("6.75j"), 6.75j) - self.assertEqual(ast.literal_eval("-6.75j"), -6.75j) - self.assertEqual(ast.literal_eval("3+6j"), 3 + 6j) - self.assertEqual(ast.literal_eval("-3+6j"), -3 + 6j) - self.assertEqual(ast.literal_eval("3-6j"), 3 - 6j) - self.assertEqual(ast.literal_eval("-3-6j"), -3 - 6j) - self.assertEqual(ast.literal_eval("3.25+6.75j"), 3.25 + 6.75j) - self.assertEqual(ast.literal_eval("-3.25+6.75j"), -3.25 + 6.75j) - self.assertEqual(ast.literal_eval("3.25-6.75j"), 3.25 - 6.75j) - self.assertEqual(ast.literal_eval("-3.25-6.75j"), -3.25 - 6.75j) - self.assertEqual(ast.literal_eval("(3+6j)"), 3 + 6j) - self.assertRaises(ValueError, ast.literal_eval, "-6j+3") - self.assertRaises(ValueError, ast.literal_eval, "-6j+3j") - self.assertRaises(ValueError, ast.literal_eval, "3+-6j") - self.assertRaises(ValueError, ast.literal_eval, "3+(0+6j)") - self.assertRaises(ValueError, ast.literal_eval, "-(3+6j)") + self.assertEqual(ast.literal_eval('6j'), 6j) + self.assertEqual(ast.literal_eval('-6j'), -6j) + self.assertEqual(ast.literal_eval('6.75j'), 6.75j) + self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) + self.assertEqual(ast.literal_eval('3+6j'), 3+6j) + self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) + self.assertEqual(ast.literal_eval('3-6j'), 3-6j) + self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) + self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) + self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) + self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) + self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) + self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) + self.assertRaises(ValueError, ast.literal_eval, '-6j+3') + self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') + self.assertRaises(ValueError, ast.literal_eval, '3+-6j') + self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') + self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') def test_literal_eval_malformed_dict_nodes(self): - malformed = ast.Dict( - keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)] - ) + malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - malformed = ast.Dict( - keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)] - ) + malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: expected an expression def test_literal_eval_trailing_ws(self): self.assertEqual(ast.literal_eval(" -1"), -1) self.assertEqual(ast.literal_eval("\t\t-1"), -1) @@ -1690,59 +1944,52 @@ def test_literal_eval_trailing_ws(self): self.assertRaises(IndentationError, ast.literal_eval, "\n -1") def test_literal_eval_malformed_lineno(self): - msg = r"malformed node or string on line 3:" + msg = r'malformed node or string on line 3:' with self.assertRaisesRegex(ValueError, msg): ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") - node = ast.UnaryOp(ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) - self.assertIsNone(getattr(node, "lineno", None)) - msg = r"malformed node or string:" + node = ast.UnaryOp( + ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) + self.assertIsNone(getattr(node, 'lineno', None)) + msg = r'malformed node or string:' with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "unexpected indent" does not match "expected an expression (, line 2)" def test_literal_eval_syntax_errors(self): with self.assertRaisesRegex(SyntaxError, "unexpected indent"): - ast.literal_eval(r""" + ast.literal_eval(r''' \ (\ - \ """) + \ ''') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "lineno" missing from alias def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values - body = [ - ast.ImportFrom( - module="time", - names=[ast.alias(name="sleep")], - level=None, - lineno=None, - col_offset=None, - ) - ] + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=None, col_offset=None)] mod = ast.Module(body, []) with self.assertRaises(ValueError) as cm: - compile(mod, "test", "exec") + compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_level_as_none(self): - body = [ - ast.ImportFrom( - module="time", - names=[ast.alias(name="sleep", lineno=0, col_offset=0)], - level=None, - lineno=0, - col_offset=0, - ) - ] + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep', + lineno=0, col_offset=0)], + level=None, + lineno=0, col_offset=0)] mod = ast.Module(body, []) - code = compile(mod, "test", "exec") + code = compile(mod, 'test', 'exec') ns = {} exec(code, ns) - self.assertIn("sleep", ns) + self.assertIn('sleep', ns) - @unittest.skip('TODO: RUSTPYTHON; crash') + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1750,7 +1997,9 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") - @unittest.skip('TODO: RUSTPYTHON; crash') + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -1762,6 +2011,7 @@ def test_recursion_indirect(self): class ASTValidatorTests(unittest.TestCase): + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): mod.lineno = mod.col_offset = 0 ast.fix_missing_locations(mod) @@ -1780,7 +2030,6 @@ def stmt(self, stmt, msg=None): mod = ast.Module([stmt], []) self.mod(mod, msg) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_module(self): m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) self.mod(m, "must have Load context", "single") @@ -1788,15 +2037,9 @@ def test_module(self): self.mod(m, "must have Load context", "eval") def _check_arguments(self, fac, check): - def arguments( - args=None, - posonlyargs=None, - vararg=None, - kwonlyargs=None, - kwarg=None, - defaults=None, - kw_defaults=None, - ): + def arguments(args=None, posonlyargs=None, vararg=None, + kwonlyargs=None, kwarg=None, + defaults=None, kw_defaults=None): if args is None: args = [] if posonlyargs is None: @@ -1807,69 +2050,50 @@ def arguments( defaults = [] if kw_defaults is None: kw_defaults = [] - args = ast.arguments( - args, posonlyargs, vararg, kwonlyargs, kw_defaults, kwarg, defaults - ) + args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, + kw_defaults, kwarg, defaults) return fac(args) - args = [ast.arg("x", ast.Name("x", ast.Store()))] check(arguments(args=args), "must have Load context") check(arguments(posonlyargs=args), "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check( - arguments(defaults=[ast.Constant(3)]), "more positional defaults than args" - ) - check( - arguments(kw_defaults=[ast.Constant(4)]), - "length of kwonlyargs is not the same as kw_defaults", - ) + check(arguments(defaults=[ast.Constant(3)]), + "more positional defaults than args") + check(arguments(kw_defaults=[ast.Constant(4)]), + "length of kwonlyargs is not the same as kw_defaults") args = [ast.arg("x", ast.Name("x", ast.Load()))] - check( - arguments(args=args, defaults=[ast.Name("x", ast.Store())]), - "must have Load context", - ) - args = [ - ast.arg("a", ast.Name("x", ast.Load())), - ast.arg("b", ast.Name("y", ast.Load())), - ] - check( - arguments(kwonlyargs=args, kw_defaults=[None, ast.Name("x", ast.Store())]), - "must have Load context", - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context") + args = [ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load()))] + check(arguments(kwonlyargs=args, + kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_funcdef(self): a = ast.arguments([], [], None, [], [], None, []) f = ast.FunctionDef("x", a, [], [], None, None, []) self.stmt(f, "empty body on FunctionDef") - f = ast.FunctionDef( - "x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, [] - ) + f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, []) self.stmt(f, "must have Load context") - f = ast.FunctionDef( - "x", a, [ast.Pass()], [], ast.Name("x", ast.Store()), None, [] - ) + f = ast.FunctionDef("x", a, [ast.Pass()], [], + ast.Name("x", ast.Store()), None, []) self.stmt(f, "must have Load context") f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) self.stmt(f) - def fac(args): return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) - self._check_arguments(fac, self.stmt) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: class pattern defines no positional sub-patterns (__match_args__ missing) def test_funcdef_pattern_matching(self): # gh-104799: New fields on FunctionDef should be added at the end def matcher(node): match node: - case ast.FunctionDef( - "foo", - ast.arguments(args=[ast.arg("bar")]), - [ast.Pass()], - [ast.Name("capybara", ast.Load())], - ast.Name("pacarana", ast.Load()), - ): + case ast.FunctionDef("foo", ast.arguments(args=[ast.arg("bar")]), + [ast.Pass()], + [ast.Name("capybara", ast.Load())], + ast.Name("pacarana", ast.Load())): return True case _: return False @@ -1884,11 +2108,9 @@ def foo(bar) -> pacarana: self.assertIsInstance(funcdef, ast.FunctionDef) self.assertTrue(matcher(funcdef)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_classdef(self): - def cls( - bases=None, keywords=None, body=None, decorator_list=None, type_params=None - ): + def cls(bases=None, keywords=None, body=None, decorator_list=None, type_params=None): if bases is None: bases = [] if keywords is None: @@ -1899,94 +2121,73 @@ def cls( decorator_list = [] if type_params is None: type_params = [] - return ast.ClassDef( - "myclass", bases, keywords, body, decorator_list, type_params - ) - - self.stmt(cls(bases=[ast.Name("x", ast.Store())]), "must have Load context") - self.stmt( - cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), - "must have Load context", - ) + return ast.ClassDef("myclass", bases, keywords, + body, decorator_list, type_params) + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), + "must have Load context") + self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context") self.stmt(cls(body=[]), "empty body on ClassDef") self.stmt(cls(body=[None]), "None disallowed") - self.stmt( - cls(decorator_list=[ast.Name("x", ast.Store())]), "must have Load context" - ) + self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_delete(self): self.stmt(ast.Delete([]), "empty targets on Delete") self.stmt(ast.Delete([None]), "None disallowed") - self.stmt(ast.Delete([ast.Name("x", ast.Load())]), "must have Del context") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), + "must have Del context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assign(self): self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") - self.stmt( - ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), - "must have Store context", - ) - self.stmt( - ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), - "must have Load context", - ) + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), + "must have Store context") + self.stmt(ast.Assign([ast.Name("x", ast.Store())], + ast.Name("y", ast.Store())), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_augassign(self): - aug = ast.AugAssign( - ast.Name("x", ast.Load()), ast.Add(), ast.Name("y", ast.Load()) - ) + aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), + ast.Name("y", ast.Load())) self.stmt(aug, "must have Store context") - aug = ast.AugAssign( - ast.Name("x", ast.Store()), ast.Add(), ast.Name("y", ast.Store()) - ) + aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), + ast.Name("y", ast.Store())) self.stmt(aug, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_for(self): x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) p = ast.Pass() self.stmt(ast.For(x, y, [], []), "empty body on For") - self.stmt( - ast.For(ast.Name("x", ast.Load()), y, [p], []), "must have Store context" - ) - self.stmt( - ast.For(x, ast.Name("y", ast.Store()), [p], []), "must have Load context" - ) + self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), + "must have Store context") + self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), + "must have Load context") e = ast.Expr(ast.Name("x", ast.Store())) self.stmt(ast.For(x, y, [e], []), "must have Load context") self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_while(self): self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") - self.stmt( - ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), - "must have Load context", - ) - self.stmt( - ast.While( - ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] - ), - "must have Load context", - ) + self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context") + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_if(self): self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) self.stmt(i, "must have Load context") i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) self.stmt(i, "must have Load context") - i = ast.If( - ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] - ) + i = ast.If(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") @@ -1997,7 +2198,6 @@ def test_with(self): i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_raise(self): r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") @@ -2006,7 +2206,6 @@ def test_raise(self): r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_try(self): p = ast.Pass() t = ast.Try([], [], [], [p]) @@ -2027,7 +2226,6 @@ def test_try(self): t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_try_star(self): p = ast.Pass() t = ast.TryStar([], [], [], [p]) @@ -2048,38 +2246,32 @@ def test_try_star(self): t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_assert(self): - self.stmt( - ast.Assert(ast.Name("x", ast.Store()), None), "must have Load context" - ) - assrt = ast.Assert(ast.Name("x", ast.Load()), ast.Name("y", ast.Store())) + self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), + "must have Load context") + assrt = ast.Assert(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store())) self.stmt(assrt, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_import(self): self.stmt(ast.Import([]), "empty names on Import") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_importfrom(self): imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) self.stmt(imp, "Negative ImportFrom level") self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_global(self): self.stmt(ast.Global([]), "empty names on Global") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_nonlocal(self): self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_expr(self): e = ast.Expr(ast.Name("x", ast.Store())) self.stmt(e, "must have Load context") - @unittest.skip('TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value') + @unittest.skip("TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value") def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") @@ -2090,36 +2282,33 @@ def test_boolop(self): b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_unaryop(self): u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) self.expr(u, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_lambda(self): a = ast.arguments([], [], None, [], [], None, []) - self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context") - + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), + "must have Load context") def fac(args): return ast.Lambda(args, ast.Name("x", ast.Load())) - self._check_arguments(fac, self.expr) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_ifexp(self): l = ast.Name("x", ast.Load()) s = ast.Name("y", ast.Store()) for args in (s, l, l), (l, s, l), (l, l, s): self.expr(ast.IfExp(*args), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dict(self): d = ast.Dict([], [ast.Name("x", ast.Load())]) self.expr(d, "same number of keys as values") d = ast.Dict([ast.Name("x", ast.Load())], [None]) self.expr(d, "None disallowed") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_set(self): self.expr(ast.Set([None]), "None disallowed") s = ast.Set([ast.Name("x", ast.Store())]) @@ -2127,13 +2316,11 @@ def test_set(self): def _check_comprehension(self, fac): self.expr(fac([]), "comprehension with no generators") - g = ast.comprehension( - ast.Name("x", ast.Load()), ast.Name("x", ast.Load()), [], 0 - ) + g = ast.comprehension(ast.Name("x", ast.Load()), + ast.Name("x", ast.Load()), [], 0) self.expr(fac([g]), "must have Store context") - g = ast.comprehension( - ast.Name("x", ast.Store()), ast.Name("x", ast.Store()), [], 0 - ) + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Store()), [], 0) self.expr(fac([g]), "must have Load context") x = ast.Name("x", ast.Store()) y = ast.Name("y", ast.Load()) @@ -2143,46 +2330,42 @@ def _check_comprehension(self, fac): self.expr(fac([g]), "must have Load context") def _simple_comp(self, fac): - g = ast.comprehension( - ast.Name("x", ast.Store()), ast.Name("x", ast.Load()), [], 0 - ) - self.expr(fac(ast.Name("x", ast.Store()), [g]), "must have Load context") - + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Load()), [], 0) + self.expr(fac(ast.Name("x", ast.Store()), [g]), + "must have Load context") def wrap(gens): return fac(ast.Name("x", ast.Store()), gens) - self._check_comprehension(wrap) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_listcomp(self): self._simple_comp(ast.ListComp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_setcomp(self): self._simple_comp(ast.SetComp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_generatorexp(self): self._simple_comp(ast.GeneratorExp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_dictcomp(self): - g = ast.comprehension( - ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [], 0 - ) - c = ast.DictComp(ast.Name("x", ast.Store()), ast.Name("y", ast.Load()), [g]) + g = ast.comprehension(ast.Name("y", ast.Store()), + ast.Name("p", ast.Load()), [], 0) + c = ast.DictComp(ast.Name("x", ast.Store()), + ast.Name("y", ast.Load()), [g]) self.expr(c, "must have Load context") - c = ast.DictComp(ast.Name("x", ast.Load()), ast.Name("y", ast.Store()), [g]) + c = ast.DictComp(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store()), [g]) self.expr(c, "must have Load context") - def factory(comps): k = ast.Name("x", ast.Load()) v = ast.Name("y", ast.Load()) return ast.DictComp(k, v, comps) - self._check_comprehension(factory) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_yield(self): self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") @@ -2199,7 +2382,7 @@ def test_compare(self): comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_call(self): func = ast.Name("x", ast.Load()) args = [ast.Name("y", ast.Load())] @@ -2212,202 +2395,204 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - - class subint(int): - pass - - class subfloat(float): - pass - - class subcomplex(complex): - pass - - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), + ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Name("y", ast.Store()), ast.Load()) + sub = ast.Subscript(x, ast.Name("y", ast.Store()), + ast.Load()) self.expr(sub, "must have Load context") s = ast.Name("x", ast.Store()) for args in (s, None, None), (None, s, None), (None, None, s): sl = ast.Slice(*args) - self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + self.expr(ast.Subscript(x, sl, ast.Load()), + "must have Load context") sl = ast.Tuple([], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load())) sl = ast.Tuple([s], ast.Load()) self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_starred(self): - left = ast.List( - [ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store() - ) + left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], + ast.Store()) assign = ast.Assign([left], ast.Constant(4)) self.stmt(assign, "must have Store context") def _sequence(self, fac): self.expr(fac([None], ast.Load()), "None disallowed") - self.expr( - fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context" - ) + self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), + "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_list(self): self._sequence(ast.List) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("ignore", "", DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual( - [str(w.message) for w in wlog], - [ - "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", - ], - ) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @support.requires_resource("cpu") + @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) - constant_x = ast.Constant("x") + constant_x = ast.Constant('x') pattern_x = ast.MatchValue(constant_x) constant_true = ast.Constant(True) pattern_true = ast.MatchSingleton(True) - name_carter = ast.Name("carter", ast.Load()) + name_carter = ast.Name('carter', ast.Load()) _MATCH_PATTERNS = [ ast.MatchValue( ast.Attribute( - ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()), - "z", - ast.Load(), + ast.Attribute( + ast.Name('x', ast.Store()), + 'y', ast.Load() + ), + 'z', ast.Load() ) ), ast.MatchValue( ast.Attribute( - ast.Attribute(ast.Name("x", ast.Load()), "y", ast.Store()), - "z", - ast.Load(), + ast.Attribute( + ast.Name('x', ast.Load()), + 'y', ast.Store() + ), + 'z', ast.Load() ) ), - ast.MatchValue(ast.Constant(...)), - ast.MatchValue(ast.Constant(True)), - ast.MatchValue(ast.Constant((1, 2, 3))), - ast.MatchSingleton("string"), - ast.MatchSequence([ast.MatchSingleton("string")]), - ast.MatchSequence([ast.MatchSequence([ast.MatchSingleton("string")])]), - ast.MatchMapping([constant_1, constant_true], [pattern_x]), + ast.MatchValue( + ast.Constant(...) + ), + ast.MatchValue( + ast.Constant(True) + ), + ast.MatchValue( + ast.Constant((1,2,3)) + ), + ast.MatchSingleton('string'), + ast.MatchSequence([ + ast.MatchSingleton('string') + ]), + ast.MatchSequence( + [ + ast.MatchSequence( + [ + ast.MatchSingleton('string') + ] + ) + ] + ), ast.MatchMapping( - [constant_true, constant_1], [pattern_x, pattern_1], rest="True" + [constant_1, constant_true], + [pattern_x] + ), + ast.MatchMapping( + [constant_true, constant_1], + [pattern_x, pattern_1], + rest='True' ), ast.MatchMapping( - [constant_true, ast.Starred(ast.Name("lol", ast.Load()), ast.Load())], + [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], [pattern_x, pattern_1], - rest="legit", + rest='legit' ), ast.MatchClass( - ast.Attribute(ast.Attribute(constant_x, "y", ast.Load()), "z", ast.Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[], + ast.Attribute( + ast.Attribute( + constant_x, + 'y', ast.Load()), + 'z', ast.Load()), + patterns=[], kwd_attrs=[], kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=["True"], kwd_patterns=[pattern_1] + name_carter, + patterns=[], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[pattern_1] + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[pattern_1] ), ast.MatchClass( name_carter, - patterns=[ast.MatchSingleton("string")], + patterns=[ast.MatchSingleton('string')], kwd_attrs=[], - kwd_patterns=[], + kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[ast.MatchStar()], kwd_attrs=[], kwd_patterns=[] + name_carter, + patterns=[ast.MatchStar()], + kwd_attrs=[], + kwd_patterns=[] ), ast.MatchClass( - name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[ast.MatchStar()] + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[ast.MatchStar()] ), ast.MatchClass( constant_true, # invalid name patterns=[], - kwd_attrs=["True"], - kwd_patterns=[pattern_1], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] + ), + ast.MatchSequence( + [ + ast.MatchStar("True") + ] + ), + ast.MatchAs( + name='False' + ), + ast.MatchOr( + [] + ), + ast.MatchOr( + [pattern_1] + ), + ast.MatchOr( + [pattern_1, pattern_x, ast.MatchSingleton('xxx')] ), - ast.MatchSequence([ast.MatchStar("True")]), - ast.MatchAs(name="False"), - ast.MatchOr([]), - ast.MatchOr([pattern_1]), - ast.MatchOr([pattern_1, pattern_x, ast.MatchSingleton("xxx")]), ast.MatchAs(name="_"), ast.MatchStar(name="x"), ast.MatchSequence([ast.MatchStar("_")]), ast.MatchMapping([], [], rest="_"), ] - @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") def test_match_validation_pattern(self): - name_x = ast.Name("x", ast.Load()) + name_x = ast.Name('x', ast.Load()) for pattern in self._MATCH_PATTERNS: with self.subTest(ast.dump(pattern, indent=4)): node = ast.Match( subject=name_x, - cases=[ast.match_case(pattern=pattern, body=[ast.Pass()])], + cases = [ + ast.match_case( + pattern=pattern, + body = [ast.Pass()] + ) + ] ) node = ast.fix_missing_locations(node) module = ast.Module([node], []) @@ -2430,44 +2615,35 @@ def compile_constant(self, value): ns = {} exec(code, ns) - return ns["x"] + return ns['x'] def test_validation(self): with self.assertRaises(TypeError) as cm: self.compile_constant([1, 2, 3]) - self.assertEqual(str(cm.exception), "got an invalid type in Constant: list") + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") - @unittest.expectedFailure # TODO: RUSTPYTHON; b'' is not b'' def test_singletons(self): - for const in (None, False, True, Ellipsis, b"", frozenset()): + for const in (None, False, True, Ellipsis, b''): with self.subTest(const=const): value = self.compile_constant(const) self.assertIs(value, const) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_values(self): nested_tuple = (1,) nested_frozenset = frozenset({1}) for level in range(3): nested_tuple = (nested_tuple, 2) nested_frozenset = frozenset({nested_frozenset, 2}) - values = ( - 123, - 123.0, - 123j, - "unicode", - b"bytes", - tuple("tuple"), - frozenset("frozenset"), - nested_tuple, - nested_frozenset, - ) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) for value in values: with self.subTest(value=value): result = self.compile_constant(value) self.assertEqual(result, value) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal def test_assign_to_constant(self): tree = ast.parse("x = 1") @@ -2478,35 +2654,42 @@ def test_assign_to_constant(self): with self.assertRaises(ValueError) as cm: compile(tree, "string", "exec") - self.assertEqual( - str(cm.exception), - "expression which can't be assigned " "to in Store context", - ) + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") def test_get_docstring(self): tree = ast.parse("'docstring'\nx = 1") - self.assertEqual(ast.get_docstring(tree), "docstring") + self.assertEqual(ast.get_docstring(tree), 'docstring') def get_load_const(self, tree): # Compile to bytecode, disassemble and get parameter of LOAD_CONST # instructions - co = compile(tree, "", "exec") + co = compile(tree, '', 'exec') consts = [] for instr in dis.get_instructions(co): - if instr.opname == "LOAD_CONST" or instr.opname == "RETURN_CONST": + if instr.opcode in dis.hasconst: consts.append(instr.argval) return consts @support.cpython_only def test_load_const(self): - consts = [None, True, False, 124, 2.0, 3j, "unicode", b"bytes", (1, 2, 3)] - - code = "\n".join(["x={!r}".format(const) for const in consts]) - code += "\nx = ..." + consts = [None, + True, False, + 1000, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' consts.extend((Ellipsis, None)) tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), consts) + self.assertEqual(self.get_load_const(tree), + consts) # Replace expression nodes with constants for assign, const in zip(tree.body, consts): @@ -2515,7 +2698,8 @@ def test_load_const(self): ast.copy_location(new_node, assign.value) assign.value = new_node - self.assertEqual(self.get_load_const(tree), consts) + self.assertEqual(self.get_load_const(tree), + consts) def test_literal_eval(self): tree = ast.parse("1 + 2") @@ -2529,22 +2713,22 @@ def test_literal_eval(self): ast.copy_location(new_right, binop.right) binop.right = new_right - self.assertEqual(ast.literal_eval(binop), 10 + 20j) + self.assertEqual(ast.literal_eval(binop), 10+20j) def test_string_kind(self): - c = ast.parse('"x"', mode="eval").body + c = ast.parse('"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, None) - c = ast.parse('u"x"', mode="eval").body + c = ast.parse('u"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, "u") - c = ast.parse('r"x"', mode="eval").body + c = ast.parse('r"x"', mode='eval').body self.assertEqual(c.value, "x") self.assertEqual(c.kind, None) - c = ast.parse('b"x"', mode="eval").body + c = ast.parse('b"x"', mode='eval').body self.assertEqual(c.value, b"x") self.assertEqual(c.kind, None) @@ -2555,7 +2739,6 @@ class EndPositionTests(unittest.TestCase): Testing end positions of nodes requires a bit of extra care because of how LL parsers work. """ - def _check_end_pos(self, ast_node, end_lineno, end_col_offset): self.assertEqual(ast_node.end_lineno, end_lineno) self.assertEqual(ast_node.end_col_offset, end_col_offset) @@ -2569,55 +2752,55 @@ def _parse_value(self, s): return ast.parse(s).body[0].value def test_lambda(self): - s = "lambda x, *y: None" + s = 'lambda x, *y: None' lam = self._parse_value(s) - self._check_content(s, lam.body, "None") - self._check_content(s, lam.args.args[0], "x") - self._check_content(s, lam.args.vararg, "y") + self._check_content(s, lam.body, 'None') + self._check_content(s, lam.args.args[0], 'x') + self._check_content(s, lam.args.vararg, 'y') def test_func_def(self): - s = dedent(""" + s = dedent(''' def func(x: int, *args: str, z: float = 0, **kwargs: Any) -> bool: return True - """).strip() + ''').strip() fdef = ast.parse(s).body[0] self._check_end_pos(fdef, 5, 15) - self._check_content(s, fdef.body[0], "return True") - self._check_content(s, fdef.args.args[0], "x: int") - self._check_content(s, fdef.args.args[0].annotation, "int") - self._check_content(s, fdef.args.kwarg, "kwargs: Any") - self._check_content(s, fdef.args.kwarg.annotation, "Any") + self._check_content(s, fdef.body[0], 'return True') + self._check_content(s, fdef.args.args[0], 'x: int') + self._check_content(s, fdef.args.args[0].annotation, 'int') + self._check_content(s, fdef.args.kwarg, 'kwargs: Any') + self._check_content(s, fdef.args.kwarg.annotation, 'Any') def test_call(self): - s = "func(x, y=2, **kw)" + s = 'func(x, y=2, **kw)' call = self._parse_value(s) - self._check_content(s, call.func, "func") - self._check_content(s, call.keywords[0].value, "2") - self._check_content(s, call.keywords[1].value, "kw") + self._check_content(s, call.func, 'func') + self._check_content(s, call.keywords[0].value, '2') + self._check_content(s, call.keywords[1].value, 'kw') def test_call_noargs(self): - s = "x[0]()" + s = 'x[0]()' call = self._parse_value(s) - self._check_content(s, call.func, "x[0]") + self._check_content(s, call.func, 'x[0]') self._check_end_pos(call, 1, 6) def test_class_def(self): - s = dedent(""" + s = dedent(''' class C(A, B): x: int = 0 - """).strip() + ''').strip() cdef = ast.parse(s).body[0] self._check_end_pos(cdef, 2, 14) - self._check_content(s, cdef.bases[1], "B") - self._check_content(s, cdef.body[0], "x: int = 0") + self._check_content(s, cdef.bases[1], 'B') + self._check_content(s, cdef.body[0], 'x: int = 0') def test_class_kw(self): - s = "class S(metaclass=abc.ABCMeta): pass" + s = 'class S(metaclass=abc.ABCMeta): pass' cdef = ast.parse(s).body[0] - self._check_content(s, cdef.keywords[0].value, "abc.ABCMeta") + self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') def test_multi_line_str(self): s = dedent(''' @@ -2630,10 +2813,10 @@ def test_multi_line_str(self): self._check_end_pos(assign.value, 3, 40) def test_continued_str(self): - s = dedent(""" + s = dedent(''' x = "first part" \\ "second part" - """).strip() + ''').strip() assign = ast.parse(s).body[0] self._check_end_pos(assign, 2, 13) self._check_end_pos(assign.value, 2, 13) @@ -2641,7 +2824,7 @@ def test_continued_str(self): def test_suites(self): # We intentionally put these into the same string to check # that empty lines are not part of the suite. - s = dedent(""" + s = dedent(''' while True: pass @@ -2661,7 +2844,7 @@ def test_suites(self): pass pass - """).strip() + ''').strip() mod = ast.parse(s) while_loop = mod.body[0] if_stmt = mod.body[1] @@ -2675,18 +2858,18 @@ def test_suites(self): self._check_end_pos(try_stmt, 17, 8) self._check_end_pos(pass_stmt, 19, 4) - self._check_content(s, while_loop.test, "True") - self._check_content(s, if_stmt.body[0], "x = None") - self._check_content(s, if_stmt.orelse[0].test, "other()") - self._check_content(s, for_loop.target, "x, y") - self._check_content(s, try_stmt.body[0], "raise RuntimeError") - self._check_content(s, try_stmt.handlers[0].type, "TypeError") + self._check_content(s, while_loop.test, 'True') + self._check_content(s, if_stmt.body[0], 'x = None') + self._check_content(s, if_stmt.orelse[0].test, 'other()') + self._check_content(s, for_loop.target, 'x, y') + self._check_content(s, try_stmt.body[0], 'raise RuntimeError') + self._check_content(s, try_stmt.handlers[0].type, 'TypeError') def test_fstring(self): s = 'x = f"abc {x + y} abc"' fstr = self._parse_value(s) binop = fstr.values[1].value - self._check_content(s, binop, "x + y") + self._check_content(s, binop, 'x + y') def test_fstring_multi_line(self): s = dedent(''' @@ -2701,198 +2884,200 @@ def test_fstring_multi_line(self): fstr = self._parse_value(s) binop = fstr.values[1].value self._check_end_pos(binop, 5, 7) - self._check_content(s, binop.left, "arg_one") - self._check_content(s, binop.right, "arg_two") + self._check_content(s, binop.left, 'arg_one') + self._check_content(s, binop.right, 'arg_two') def test_import_from_multi_line(self): - s = dedent(""" + s = dedent(''' from x.y.z import ( a, b, c as c ) - """).strip() + ''').strip() imp = ast.parse(s).body[0] self._check_end_pos(imp, 3, 1) self._check_end_pos(imp.names[2], 2, 16) def test_slices(self): - s1 = "f()[1, 2] [0]" - s2 = "x[ a.b: c.d]" - sm = dedent(""" + s1 = 'f()[1, 2] [0]' + s2 = 'x[ a.b: c.d]' + sm = dedent(''' x[ a.b: f () , g () : c.d ] - """).strip() + ''').strip() i1, i2, im = map(self._parse_value, (s1, s2, sm)) - self._check_content(s1, i1.value, "f()[1, 2]") - self._check_content(s1, i1.value.slice, "1, 2") - self._check_content(s2, i2.slice.lower, "a.b") - self._check_content(s2, i2.slice.upper, "c.d") - self._check_content(sm, im.slice.elts[0].upper, "f ()") - self._check_content(sm, im.slice.elts[1].lower, "g ()") + self._check_content(s1, i1.value, 'f()[1, 2]') + self._check_content(s1, i1.value.slice, '1, 2') + self._check_content(s2, i2.slice.lower, 'a.b') + self._check_content(s2, i2.slice.upper, 'c.d') + self._check_content(sm, im.slice.elts[0].upper, 'f ()') + self._check_content(sm, im.slice.elts[1].lower, 'g ()') self._check_end_pos(im, 3, 3) def test_binop(self): - s = dedent(""" + s = dedent(''' (1 * 2 + (3 ) + 4 ) - """).strip() + ''').strip() binop = self._parse_value(s) self._check_end_pos(binop, 2, 6) - self._check_content(s, binop.right, "4") - self._check_content(s, binop.left, "1 * 2 + (3 )") - self._check_content(s, binop.left.right, "3") + self._check_content(s, binop.right, '4') + self._check_content(s, binop.left, '1 * 2 + (3 )') + self._check_content(s, binop.left.right, '3') def test_boolop(self): - s = dedent(""" + s = dedent(''' if (one_condition and (other_condition or yet_another_one)): pass - """).strip() + ''').strip() bop = ast.parse(s).body[0].test self._check_end_pos(bop, 2, 44) - self._check_content(s, bop.values[1], "other_condition or yet_another_one") + self._check_content(s, bop.values[1], + 'other_condition or yet_another_one') def test_tuples(self): - s1 = "x = () ;" - s2 = "x = 1 , ;" - s3 = "x = (1 , 2 ) ;" - sm = dedent(""" + s1 = 'x = () ;' + s2 = 'x = 1 , ;' + s3 = 'x = (1 , 2 ) ;' + sm = dedent(''' x = ( a, b, ) - """).strip() + ''').strip() t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) - self._check_content(s1, t1, "()") - self._check_content(s2, t2, "1 ,") - self._check_content(s3, t3, "(1 , 2 )") + self._check_content(s1, t1, '()') + self._check_content(s2, t2, '1 ,') + self._check_content(s3, t3, '(1 , 2 )') self._check_end_pos(tm, 3, 1) def test_attribute_spaces(self): - s = "func(x. y .z)" + s = 'func(x. y .z)' call = self._parse_value(s) self._check_content(s, call, s) - self._check_content(s, call.args[0], "x. y .z") + self._check_content(s, call.args[0], 'x. y .z') def test_redundant_parenthesis(self): - s = "( ( ( a + b ) ) )" + s = '( ( ( a + b ) ) )' v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, "BinOp") - self._check_content(s, v, "a + b") - s2 = "await " + s + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s, v, 'a + b') + s2 = 'await ' + s v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, "BinOp") - self._check_content(s2, v, "a + b") + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s2, v, 'a + b') def test_trailers_with_redundant_parenthesis(self): tests = ( - ("( ( ( a ) ) ) ( )", "Call"), - ("( ( ( a ) ) ) ( b )", "Call"), - ("( ( ( a ) ) ) [ b ]", "Subscript"), - ("( ( ( a ) ) ) . b", "Attribute"), + ('( ( ( a ) ) ) ( )', 'Call'), + ('( ( ( a ) ) ) ( b )', 'Call'), + ('( ( ( a ) ) ) [ b ]', 'Subscript'), + ('( ( ( a ) ) ) . b', 'Attribute'), ) for s, t in tests: with self.subTest(s): v = ast.parse(s).body[0].value self.assertEqual(type(v).__name__, t) self._check_content(s, v, s) - s2 = "await " + s + s2 = 'await ' + s v = ast.parse(s2).body[0].value.value self.assertEqual(type(v).__name__, t) self._check_content(s2, v, s) def test_displays(self): - s1 = "[{}, {1, }, {1, 2,} ]" - s2 = "{a: b, f (): g () ,}" + s1 = '[{}, {1, }, {1, 2,} ]' + s2 = '{a: b, f (): g () ,}' c1 = self._parse_value(s1) c2 = self._parse_value(s2) - self._check_content(s1, c1.elts[0], "{}") - self._check_content(s1, c1.elts[1], "{1, }") - self._check_content(s1, c1.elts[2], "{1, 2,}") - self._check_content(s2, c2.keys[1], "f ()") - self._check_content(s2, c2.values[1], "g ()") + self._check_content(s1, c1.elts[0], '{}') + self._check_content(s1, c1.elts[1], '{1, }') + self._check_content(s1, c1.elts[2], '{1, 2,}') + self._check_content(s2, c2.keys[1], 'f ()') + self._check_content(s2, c2.values[1], 'g ()') def test_comprehensions(self): - s = dedent(""" + s = dedent(''' x = [{x for x, y in stuff if cond.x} for stuff in things] - """).strip() + ''').strip() cmp = self._parse_value(s) self._check_end_pos(cmp, 2, 37) - self._check_content(s, cmp.generators[0].iter, "things") - self._check_content(s, cmp.elt.generators[0].iter, "stuff") - self._check_content(s, cmp.elt.generators[0].ifs[0], "cond.x") - self._check_content(s, cmp.elt.generators[0].target, "x, y") + self._check_content(s, cmp.generators[0].iter, 'things') + self._check_content(s, cmp.elt.generators[0].iter, 'stuff') + self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') + self._check_content(s, cmp.elt.generators[0].target, 'x, y') def test_yield_await(self): - s = dedent(""" + s = dedent(''' async def f(): yield x await y - """).strip() + ''').strip() fdef = ast.parse(s).body[0] - self._check_content(s, fdef.body[0].value, "yield x") - self._check_content(s, fdef.body[1].value, "await y") + self._check_content(s, fdef.body[0].value, 'yield x') + self._check_content(s, fdef.body[1].value, 'await y') def test_source_segment_multi(self): - s_orig = dedent(""" + s_orig = dedent(''' x = ( a, b, ) + () - """).strip() - s_tuple = dedent(""" + ''').strip() + s_tuple = dedent(''' ( a, b, ) - """).strip() + ''').strip() binop = self._parse_value(s_orig) self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) def test_source_segment_padded(self): - s_orig = dedent(""" + s_orig = dedent(''' class C: def fun(self) -> None: "ЖЖЖЖЖ" - """).strip() - s_method = " def fun(self) -> None:\n" ' "ЖЖЖЖЖ"' + ''').strip() + s_method = ' def fun(self) -> None:\n' \ + ' "ЖЖЖЖЖ"' cdef = ast.parse(s_orig).body[0] - self.assertEqual( - ast.get_source_segment(s_orig, cdef.body[0], padded=True), s_method - ) + self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), + s_method) def test_source_segment_endings(self): - s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n" + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' v, w, x, y, z = ast.parse(s).body - self._check_content(s, v, "v = 1") - self._check_content(s, w, "w = 1") - self._check_content(s, x, "x = 1") - self._check_content(s, y, "y = 1") - self._check_content(s, z, "z = 1") + self._check_content(s, v, 'v = 1') + self._check_content(s, w, 'w = 1') + self._check_content(s, x, 'x = 1') + self._check_content(s, y, 'y = 1') + self._check_content(s, z, 'z = 1') def test_source_segment_tabs(self): - s = dedent(""" + s = dedent(''' class C: \t\f def fun(self) -> None: \t\f pass - """).strip() - s_method = " \t\f def fun(self) -> None:\n" " \t\f pass" + ''').strip() + s_method = ' \t\f def fun(self) -> None:\n' \ + ' \t\f pass' cdef = ast.parse(s).body[0] self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) def test_source_segment_newlines(self): - s = "def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n" + s = 'def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n' f, g, h = ast.parse(s).body - self._check_content(s, f, "def f():\n pass") - self._check_content(s, g, "def g():\r pass") - self._check_content(s, h, "def h():\r\n pass") + self._check_content(s, f, 'def f():\n pass') + self._check_content(s, g, 'def g():\r pass') + self._check_content(s, h, 'def h():\r\n pass') - s = "def f():\n a = 1\r b = 2\r\n c = 3\n" + s = 'def f():\n a = 1\r b = 2\r\n c = 3\n' f = ast.parse(s).body[0] self._check_content(s, f, s.rstrip()) def test_source_segment_missing_info(self): - s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n" + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' v, w, x, y = ast.parse(s).body del v.lineno del w.end_lineno @@ -2904,102 +3089,27 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, "Num", node.n)) - - def visit_Str(self, node): - log.append((node.lineno, "Str", node.s)) - - def visit_Bytes(self, node): - log.append((node.lineno, "Bytes", node.s)) - - def visit_NameConstant(self, node): - log.append((node.lineno, "NameConstant", node.value)) - - def visit_Ellipsis(self, node): - log.append((node.lineno, "Ellipsis", ...)) - - mod = ast.parse( - dedent("""\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - """) - ) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings("always", "", DeprecationWarning) - visitor.visit(mod) - self.assertEqual( - log, - [ - (1, "Num", 42), - (2, "Num", 4.25), - (3, "Num", 4.25j), - (4, "Str", "string"), - (5, "Bytes", b"bytes"), - (6, "NameConstant", True), - (7, "NameConstant", None), - (8, "Ellipsis", ...), - ], - ) - self.assertEqual( - [str(w.message) for w in wlog], - [ - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Num is deprecated; add visit_Constant", - "Attribute n is deprecated and will be removed in Python 3.14; use value instead", - "visit_Str is deprecated; add visit_Constant", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "visit_Bytes is deprecated; add visit_Constant", - "Attribute s is deprecated and will be removed in Python 3.14; use value instead", - "visit_NameConstant is deprecated; add visit_Constant", - "visit_NameConstant is deprecated; add visit_Constant", - "visit_Ellipsis is deprecated; add visit_Constant", - ], - ) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, initial_code, expected_code): +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): + def assertASTTransformation(self, transformer_class, + initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) expected_ast = ast.parse(dedent(expected_code)) - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) + transformer = transformer_class() + result_ast = ast.fix_missing_locations(transformer.visit(initial_ast)) self.assertASTEqual(result_ast, expected_ast) - @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_remove_single(self): - code = "def func(arg) -> SomeType: ..." - expected = "def func(arg): ..." + code = 'def func(arg) -> SomeType: ...' + expected = 'def func(arg): ...' # Since `FunctionDef.returns` is defined as a single value, we test # the `if isinstance(old_value, AST):` branch here. class SomeTypeRemover(ast.NodeTransformer): def visit_Name(self, node: ast.Name): self.generic_visit(node) - if node.id == "SomeType": + if node.id == 'SomeType': return None return node @@ -3027,7 +3137,6 @@ def visit_Expr(self, node: ast.Expr): self.assertASTTransformation(YieldRemover, code, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_return_list(self): code = """ class DSL(Base, kw1=True): ... @@ -3039,11 +3148,11 @@ class DSL(Base, kw1=True, kw2=True, kw3=False): ... class ExtendKeywords(ast.NodeTransformer): def visit_keyword(self, node: ast.keyword): self.generic_visit(node) - if node.arg == "kw1": + if node.arg == 'kw1': return [ node, - ast.keyword("kw2", ast.Constant(True)), - ast.keyword("kw3", ast.Constant(False)), + ast.keyword('kw2', ast.Constant(True)), + ast.keyword('kw3', ast.Constant(False)), ] return node @@ -3062,13 +3171,12 @@ def func(arg): class PrintToLog(ast.NodeTransformer): def visit_Call(self, node: ast.Call): self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == "print": - node.func.id = "log" + if isinstance(node.func, ast.Name) and node.func.id == 'print': + node.func.id = 'log' return node self.assertASTTransformation(PrintToLog, code, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; is not def test_node_replace(self): code = """ def func(arg): @@ -3082,15 +3190,15 @@ def func(arg): class PrintToLog(ast.NodeTransformer): def visit_Call(self, node: ast.Call): self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == "print": + if isinstance(node.func, ast.Name) and node.func.id == 'print': return ast.Call( func=ast.Attribute( - ast.Name("logger", ctx=ast.Load()), - attr="log", + ast.Name('logger', ctx=ast.Load()), + attr='log', ctx=ast.Load(), ), args=node.args, - keywords=[ast.keyword("debug", ast.Constant(True))], + keywords=[ast.keyword('debug', ast.Constant(True))], ) return node @@ -3100,23 +3208,19 @@ def visit_Call(self, node: ast.Call): class ASTConstructorTests(unittest.TestCase): """Test the autogenerated constructors for AST nodes.""" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_FunctionDef(self): args = ast.arguments() self.assertEqual(args.args, []) self.assertEqual(args.posonlyargs, []) - with self.assertWarnsRegex( - DeprecationWarning, - r"FunctionDef\.__init__ missing 1 required positional argument: 'name'", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): node = ast.FunctionDef(args=args) - self.assertFalse(hasattr(node, "name")) + self.assertNotHasAttr(node, "name") self.assertEqual(node.decorator_list, []) - node = ast.FunctionDef(name="foo", args=args) - self.assertEqual(node.name, "foo") + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_expr_context(self): name = ast.Name("x") self.assertEqual(name.id, "x") @@ -3130,10 +3234,8 @@ def test_expr_context(self): self.assertEqual(name3.id, "x") self.assertIsInstance(name3.ctx, ast.Del) - with self.assertWarnsRegex( - DeprecationWarning, - r"Name\.__init__ missing 1 required positional argument: 'id'", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): name3 = ast.Name() def test_custom_subclass_with_no_fields(self): @@ -3146,7 +3248,7 @@ class NoInit(ast.AST): def test_fields_but_no_field_types(self): class Fields(ast.AST): - _fields = ("a",) + _fields = ('a',) obj = Fields() with self.assertRaises(AttributeError): @@ -3156,8 +3258,8 @@ class Fields(ast.AST): def test_fields_and_types(self): class FieldsAndTypes(ast.AST): - _fields = ("a",) - _field_types = {"a": int | None} + _fields = ('a',) + _field_types = {'a': int | None} a: int | None = None obj = FieldsAndTypes() @@ -3165,7 +3267,6 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_custom_attributes(self): class MyAttrs(ast.AST): _attributes = ("a", "b") @@ -3174,39 +3275,35 @@ class MyAttrs(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - with self.assertWarnsRegex( - DeprecationWarning, - r"MyAttrs.__init__ got an unexpected keyword argument 'c'.", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): obj = MyAttrs(c=3) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): - _fields = ("a",) - _field_types = {"a": int} + _fields = ('a',) + _field_types = {'a': int} - with self.assertWarnsRegex( - DeprecationWarning, - r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\.", - ): + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): obj = FieldsAndTypesNoDefault() with self.assertRaises(AttributeError): obj.a obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_incomplete_field_types(self): class MoreFieldsThanTypes(ast.AST): - _fields = ("a", "b") - _field_types = {"a": int | None} + _fields = ('a', 'b') + _field_types = {'a': int | None} a: int | None = None b: int | None = None with self.assertWarnsRegex( DeprecationWarning, - r"Field 'b' is missing from MoreFieldsThanTypes\._field_types", + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" ): obj = MoreFieldsThanTypes() self.assertIs(obj.a, None) @@ -3216,11 +3313,20 @@ class MoreFieldsThanTypes(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. + def test_malformed_fields_with_bytes(self): + class BadFields(ast.AST): + _fields = (b'\xff'*64,) + _field_types = {'a': int} + + # This should not crash + with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): + obj = BadFields() + def test_complete_field_types(self): class _AllFieldTypes(ast.AST): - _fields = ("a", "b") - _field_types = {"a": int | None, "b": list[str]} + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} # This must be set explicitly a: int | None = None # This will add an implicit empty list default @@ -3230,6 +3336,28 @@ class _AllFieldTypes(ast.AST): self.assertIs(obj.a, None) self.assertEqual(obj.b, []) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_str_kwarg(self): + warn_msg = "got an unexpected keyword argument int: + x -= 1 + return x + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(flags=args): + self.invoke_ast(*args) + + @support.force_not_colorized + def test_help_message(self): + for flag in ('-h', '--help', '--unknown'): + with self.subTest(flag=flag): + output = StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(output): + ast.main(args=flag) + self.assertStartsWith(output.getvalue(), 'usage: ') + + def test_exec_mode_flag(self): + # test 'python -m ast -m/--mode exec' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)], + type_ignores=[ + TypeIgnore(lineno=1, tag='[assignment]')]) + ''' + for flag in ('-m=exec', '--mode=exec'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_single_mode_flag(self): + # test 'python -m ast -m/--mode single' + source = 'pass' + expect = ''' + Interactive( + body=[ + Pass()]) + ''' + for flag in ('-m=single', '--mode=single'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_eval_mode_flag(self): + # test 'python -m ast -m/--mode eval' + source = 'print(1, 2, 3)' + expect = ''' + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + ''' + for flag in ('-m=eval', '--mode=eval'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_func_type_mode_flag(self): + # test 'python -m ast -m/--mode func_type' + source = '(int, str) -> list[int]' + expect = ''' + FunctionType( + argtypes=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + returns=Subscript( + value=Name(id='list', ctx=Load()), + slice=Name(id='int', ctx=Load()), + ctx=Load())) + ''' + for flag in ('-m=func_type', '--mode=func_type'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_no_type_comments_flag(self): + # test 'python -m ast --no-type-comments' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)]) + ''' + self.check_output(source, expect, '--no-type-comments') + + def test_include_attributes_flag(self): + # test 'python -m ast -a/--include-attributes' + source = 'pass' + expect = ''' + Module( + body=[ + Pass( + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=4)]) + ''' + for flag in ('-a', '--include-attributes'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_indent_flag(self): + # test 'python -m ast -i/--indent 0' + source = 'pass' + expect = ''' + Module( + body=[ + Pass()]) + ''' + for flag in ('-i=0', '--indent=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_feature_version_flag(self): + # test 'python -m ast --feature-version 3.9/3.10' + source = ''' + match x: + case 1: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=1)), + body=[ + Pass()])])]) + ''' + self.check_output(source, expect, '--feature-version=3.10') + with self.assertRaises(SyntaxError): + self.invoke_ast('--feature-version=3.9') + + def test_no_optimize_flag(self): + # test 'python -m ast -O/--optimize -1/0' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=BinOp( + left=Constant(value=1), + op=Add(), + right=Constant(value=2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=-1', '--optimize=-1', '-O=0', '--optimize=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_optimize_flag(self): + # test 'python -m ast -O/--optimize 1/2' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=(1+2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=1', '--optimize=1', '-O=2', '--optimize=2'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_show_empty_flag(self): + # test 'python -m ast --show-empty' + source = 'print(1, 2, 3)' + expect = ''' + Module( + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + keywords=[]))], + type_ignores=[]) + ''' + self.check_output(source, expect, '--show-empty') -def compare(left, right): - return ast.dump(left) == ast.dump(right) class ASTOptimizationTests(unittest.TestCase): - binop = { - "+": ast.Add(), - "-": ast.Sub(), - "*": ast.Mult(), - "/": ast.Div(), - "%": ast.Mod(), - "<<": ast.LShift(), - ">>": ast.RShift(), - "|": ast.BitOr(), - "^": ast.BitXor(), - "&": ast.BitAnd(), - "//": ast.FloorDiv(), - "**": ast.Pow(), - } - - unaryop = { - "~": ast.Invert(), - "+": ast.UAdd(), - "-": ast.USub(), - } - def wrap_expr(self, expr): return ast.Module(body=[ast.Expr(value=expr)]) @@ -3364,112 +3709,31 @@ def wrap_statement(self, statement): return ast.Module(body=[statement]) def assert_ast(self, code, non_optimized_target, optimized_target): - non_optimized_tree = ast.parse(code, optimize=-1) optimized_tree = ast.parse(code, optimize=1) # Is a non-optimized tree equal to a non-optimized target? self.assertTrue( - compare(non_optimized_tree, non_optimized_target), + ast.compare(non_optimized_tree, non_optimized_target), f"{ast.dump(non_optimized_target)} must equal " f"{ast.dump(non_optimized_tree)}", ) # Is a optimized tree equal to a non-optimized target? self.assertFalse( - compare(optimized_tree, non_optimized_target), + ast.compare(optimized_tree, non_optimized_target), f"{ast.dump(non_optimized_target)} must not equal " f"{ast.dump(non_optimized_tree)}" ) # Is a optimized tree is equal to an optimized target? self.assertTrue( - compare(optimized_tree, optimized_target), + ast.compare(optimized_tree, optimized_target), f"{ast.dump(optimized_target)} must equal " f"{ast.dump(optimized_tree)}", ) - def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): - return ast.BinOp(left=left, op=self.binop[operand], right=right) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_binop(self): - code = "1 %s 1" - operators = self.binop.keys() - - for op in operators: - result_code = code % op - non_optimized_target = self.wrap_expr(self.create_binop(op)) - optimized_target = self.wrap_expr(ast.Constant(value=eval(result_code))) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - # Multiplication of constant tuples must be folded - code = "(1,) * 3" - non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) - optimized_target = self.wrap_expr(ast.Constant(eval(code))) - - self.assert_ast(code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_unaryop(self): - code = "%s1" - operators = self.unaryop.keys() - - def create_unaryop(operand): - return ast.UnaryOp(op=self.unaryop[operand], operand=ast.Constant(1)) - - for op in operators: - result_code = code % op - non_optimized_target = self.wrap_expr(create_unaryop(op)) - optimized_target = self.wrap_expr(ast.Constant(eval(result_code))) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_not(self): - code = "not (1 %s (1,))" - operators = { - "in": ast.In(), - "is": ast.Is(), - } - opt_operators = { - "is": ast.IsNot(), - "in": ast.NotIn(), - } - - def create_notop(operand): - return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( - left=ast.Constant(value=1), - ops=[operators[operand]], - comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] - )) - - for op in operators.keys(): - result_code = code % op - non_optimized_target = self.wrap_expr(create_notop(op)) - optimized_target = self.wrap_expr( - ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) - ) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_format(self): code = "'%s' % (a,)" @@ -3489,158 +3753,122 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) + def test_folding_match_case_allowed_expressions(self): + def get_match_case_values(node): + result = [] + if isinstance(node, ast.Constant): + result.append(node.value) + elif isinstance(node, ast.MatchValue): + result.extend(get_match_case_values(node.value)) + elif isinstance(node, ast.MatchMapping): + for key in node.keys: + result.extend(get_match_case_values(key)) + elif isinstance(node, ast.MatchSequence): + for pat in node.patterns: + result.extend(get_match_case_values(pat)) + else: + self.fail(f"Unexpected node {node}") + return result - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_tuple(self): - code = "(1,)" + tests = [ + ("-0", [0]), + ("-0.1", [-0.1]), + ("-0j", [complex(0, 0)]), + ("-0.1j", [complex(0, -0.1)]), + ("1 + 2j", [complex(1, 2)]), + ("1 - 2j", [complex(1, -2)]), + ("1.1 + 2.1j", [complex(1.1, 2.1)]), + ("1.1 - 2.1j", [complex(1.1, -2.1)]), + ("-0 + 1j", [complex(0, 1)]), + ("-0 - 1j", [complex(0, -1)]), + ("-0.1 + 1.1j", [complex(-0.1, 1.1)]), + ("-0.1 - 1.1j", [complex(-0.1, -1.1)]), + ("{-0: 0}", [0]), + ("{-0.1: 0}", [-0.1]), + ("{-0j: 0}", [complex(0, 0)]), + ("{-0.1j: 0}", [complex(0, -0.1)]), + ("{1 + 2j: 0}", [complex(1, 2)]), + ("{1 - 2j: 0}", [complex(1, -2)]), + ("{1.1 + 2.1j: 0}", [complex(1.1, 2.1)]), + ("{1.1 - 2.1j: 0}", [complex(1.1, -2.1)]), + ("{-0 + 1j: 0}", [complex(0, 1)]), + ("{-0 - 1j: 0}", [complex(0, -1)]), + ("{-0.1 + 1.1j: 0}", [complex(-0.1, 1.1)]), + ("{-0.1 - 1.1j: 0}", [complex(-0.1, -1.1)]), + ("{-0: 0, 0 + 1j: 0, 0.1 + 1j: 0}", [0, complex(0, 1), complex(0.1, 1)]), + ("[-0, -0.1, -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[[[-0, -0.1, -0j, -0.1j]]]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], [-0j, -0.1j]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("(-0, -0.1, -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((((-0, -0.1, -0j, -0.1j))))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), (-0j, -0.1j))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ] + for match_expr, constants in tests: + with self.subTest(match_expr): + src = f"match 0:\n\t case {match_expr}: pass" + tree = ast.parse(src, optimize=1) + match_stmt = tree.body[0] + case = match_stmt.cases[0] + values = get_match_case_values(case.pattern) + self.assertListEqual(constants, values) + + def test_match_case_not_folded_in_unoptimized_ast(self): + src = textwrap.dedent(""" + match a: + case 1+2j: + pass + """) - non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) - optimized_target = self.wrap_expr(ast.Constant(value=(1,))) + unfolded = "MatchValue(value=BinOp(left=Constant(value=1), op=Add(), right=Constant(value=2j))" + folded = "MatchValue(value=Constant(value=(1+2j)))" + for optval in (0, 1, 2): + self.assertIn(folded if optval else unfolded, ast.dump(ast.parse(src, optimize=optval))) - self.assert_ast(code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_binop(self): + return super().test_folding_binop() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_comparator(self): - code = "1 %s %s1%s" - operators = [("in", ast.In()), ("not in", ast.NotIn())] - braces = [ - ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), - ] - for left, right, non_optimized_comparator, optimized_comparator in braces: - for op, node in operators: - non_optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] - )) - optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[ast.Constant(value=optimized_comparator)] - )) - self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_iter(self): - code = "for _ in %s1%s: pass" - braces = [ - ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), - ] - - for left, right, ast_cls, optimized_iter in braces: - non_optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast_cls(elts=[ast.Constant(1)]), - body=[ast.Pass()] - )) - optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast.Constant(value=optimized_iter), - body=[ast.Pass()] - )) - - self.assert_ast(code % (left, right), non_optimized_target, optimized_target) - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_subscript(self): - code = "(1,)[0]" - - non_optimized_target = self.wrap_expr( - ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) - ) - optimized_target = self.wrap_expr(ast.Constant(value=1)) + return super().test_folding_comparator() - self.assert_ast(code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_iter(self): + return super().test_folding_iter() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_type_param_in_function_def(self): - code = "def foo[%s = 1 + 1](): pass" + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_not(self): + return super().test_folding_not() - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_subscript(self): + return super().test_folding_subscript() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant(2))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_binop)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_tuple(self): + return super().test_folding_tuple() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_class_def(self): - code = "class foo[%s = 1 + 1]: pass" - - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + return super().test_folding_type_param_in_class_def() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant(2))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_binop)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_function_def(self): + return super().test_folding_type_param_in_function_def() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_folding_type_param_in_type_alias(self): - code = "type foo[%s = 1 + 1] = 1" - - unoptimized_binop = self.create_binop("+") - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] + return super().test_folding_type_param_in_type_alias() - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=ast.Constant(2))], - value=ast.Constant(value=1), - ) - ) - non_optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=unoptimized_binop)], - value=ast.Constant(value=1), - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_unaryop(self): + return super().test_folding_unaryop() -if __name__ == "__main__": +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py index 145e89ee94e..e7054f3f710 100644 --- a/Lib/test/test_ast/utils.py +++ b/Lib/test/test_ast/utils.py @@ -1,5 +1,5 @@ def to_tuple(t): - if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + if t is None or isinstance(t, (str, int, complex, float, bytes, tuple)) or t is Ellipsis: return t elif isinstance(t, list): return [to_tuple(e) for e in t] diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 7039bd7054c..181476e0989 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -4,10 +4,12 @@ import contextlib from test.support.import_helper import import_module -from test.support import gc_collect +from test.support import gc_collect, requires_working_socket asyncio = import_module("asyncio") +requires_working_socket(module=True) + _no_default = object() @@ -375,6 +377,178 @@ async def async_gen_wrapper(): self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + def test_async_gen_exception_12(self): + async def gen(): + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'asend' of '{gen.__qualname__}' " + f"was never awaited"): + await anext(me) + yield 123 + + me = gen() + ai = me.__aiter__() + an = ai.__anext__() + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + an.__next__() + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_3_arg_deprecation_warning(self): + async def gen(): + yield 123 + + with self.assertWarns(DeprecationWarning): + x = gen().athrow(GeneratorExit, GeneratorExit(), None) + with self.assertRaises(GeneratorExit): + x.send(None) + del x + gc_collect() + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -393,8 +567,57 @@ async def gen(): self.assertIsInstance(g.ag_frame, types.FrameType) self.assertFalse(g.ag_running) self.assertIsInstance(g.ag_code, types.CodeType) + aclose = g.aclose() + self.assertTrue(inspect.isawaitable(aclose)) + aclose.close() + + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) - self.assertTrue(inspect.isawaitable(g.aclose())) + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() class AsyncGenAsyncioTest(unittest.TestCase): @@ -406,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): @@ -648,7 +871,7 @@ def test1(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) try: g.send(None) except StopIteration as e: @@ -661,9 +884,9 @@ def test2(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test3(anext): agen = agenfn() @@ -690,9 +913,9 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) - self.assertEqual(g.throw(MyError, MyError(), None), 20) + self.assertEqual(g.throw(MyError()), 20) with self.assertRaisesRegex(MyError, 'val'): - g.throw(MyError, MyError('val'), None) + g.throw(MyError('val')) def test5(anext): @types.coroutine @@ -711,7 +934,7 @@ async def agenfn(): with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) with self.assertRaisesRegex(StopIteration, 'default'): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test6(anext): @types.coroutine @@ -726,7 +949,7 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def run_test(test): with self.subTest('pure-Python anext()'): @@ -929,6 +1152,43 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_tuple_no_exceptions(self): + # StopAsyncIteration exceptions should be cleared. + # See: https://github.com/python/cpython/issues/128078. + + async def foo(): + if False: + yield (1, 2) + + async def run(): + it = foo().__aiter__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + res = await anext(it, ('a', 'b')) + self.assertTupleEqual(res, ('a', 'b')) + + self.loop.run_until_complete(run()) + + def test_sync_anext_raises_exception(self): + # See: https://github.com/python/cpython/issues/131670 + msg = 'custom' + for exc_type in [ + StopAsyncIteration, + StopIteration, + ValueError, + Exception, + ]: + exc = exc_type(msg) + with self.subTest(exc=exc): + class A: + def __anext__(self): + raise exc + + with self.assertRaisesRegex(exc_type, msg): + anext(A()) + with self.assertRaisesRegex(exc_type, msg): + anext(A(), 1) + def test_async_gen_asyncio_anext_stopiteration(self): async def foo(): try: @@ -1026,8 +1286,6 @@ async def run(): fut.cancel() self.loop.run_until_complete(asyncio.sleep(0.01)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_async_gen_asyncio_gc_aclose_09(self): DONE = 0 @@ -1037,8 +1295,7 @@ async def gen(): while True: yield 1 finally: - await asyncio.sleep(0.01) - await asyncio.sleep(0.01) + await asyncio.sleep(0) DONE = 1 async def run(): @@ -1048,7 +1305,10 @@ async def run(): del g gc_collect() # For PyPy or other GCs. - await asyncio.sleep(0.1) + # Starts running the aclose task + await asyncio.sleep(0) + # For asyncio.sleep(0) in finally block + await asyncio.sleep(0) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -1514,8 +1774,6 @@ async def main(): self.assertIn('an error occurred during closing of asynchronous generator', message['message']) - # TODO: RUSTPYTHON, ValueError: not enough values to unpack (expected 1, got 0) - @unittest.expectedFailure def test_async_gen_asyncio_shutdown_exception_02(self): messages = [] @@ -1543,6 +1801,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1560,21 +1820,35 @@ async def run(): res = self.loop.run_until_complete(run()) self.assertEqual(res, [i * 2 for i in range(10)]) - # TODO: RUSTPYTHON: async for gen expression compilation - # def test_async_gen_expression_02(self): - # async def wrap(n): - # await asyncio.sleep(0.01) - # return n + def test_async_gen_expression_02(self): + async def wrap(n): + await asyncio.sleep(0.01) + return n - # def make_arange(n): - # # This syntax is legal starting with Python 3.7 - # return (i * 2 for i in range(n) if await wrap(i)) + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 for i in range(n) if await wrap(i)) - # async def run(): - # return [i async for i in make_arange(10)] + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(1, 10)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ + def test_async_gen_expression_incorrect(self): + async def ag(): + yield 42 - # res = self.loop.run_until_complete(run()) - # self.assertEqual(res, [i * 2 for i in range(1, 10)]) + async def run(arg): + (x async for x in arg) + + err_msg_async = "'async for' requires an object with " \ + "__aiter__ method, got .*" + + self.loop.run_until_complete(run(ag())) + with self.assertRaisesRegex(TypeError, err_msg_async): + self.loop.run_until_complete(run(None)) def test_asyncgen_nonstarted_hooks_are_cancellable(self): # See https://bugs.python.org/issue38013 @@ -1597,6 +1871,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1634,6 +1909,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1676,5 +2007,109 @@ async def run(): self.loop.run_until_complete(run()) +class TestUnawaitedWarnings(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_asend(self): + async def gen(): + yield 1 + + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.asend(None) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_athrow(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'athrow' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.athrow(RuntimeError) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_aclose(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'aclose' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.aclose() + gc_collect() + + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/__init__.py b/Lib/test/test_asyncio/__init__.py new file mode 100644 index 00000000000..ab0b5aa9489 --- /dev/null +++ b/Lib/test/test_asyncio/__init__.py @@ -0,0 +1,12 @@ +import os +from test import support +from test.support import load_package_tests +from test.support import import_helper + +support.requires_working_socket(module=True) + +# Skip tests if we don't have concurrent.futures. +import_helper.import_module('concurrent.futures') + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_asyncio/__main__.py b/Lib/test/test_asyncio/__main__.py new file mode 100644 index 00000000000..40a23a297ec --- /dev/null +++ b/Lib/test/test_asyncio/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_asyncio/echo.py b/Lib/test/test_asyncio/echo.py new file mode 100644 index 00000000000..006364bb007 --- /dev/null +++ b/Lib/test/test_asyncio/echo.py @@ -0,0 +1,8 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + os.write(1, buf) diff --git a/Lib/test/test_asyncio/echo2.py b/Lib/test/test_asyncio/echo2.py new file mode 100644 index 00000000000..e83ca09fb7a --- /dev/null +++ b/Lib/test/test_asyncio/echo2.py @@ -0,0 +1,6 @@ +import os + +if __name__ == '__main__': + buf = os.read(0, 1024) + os.write(1, b'OUT:'+buf) + os.write(2, b'ERR:'+buf) diff --git a/Lib/test/test_asyncio/echo3.py b/Lib/test/test_asyncio/echo3.py new file mode 100644 index 00000000000..064496736bf --- /dev/null +++ b/Lib/test/test_asyncio/echo3.py @@ -0,0 +1,11 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + try: + os.write(1, b'OUT:'+buf) + except OSError as ex: + os.write(2, b'ERR:' + ex.__class__.__name__.encode('ascii')) diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py new file mode 100644 index 00000000000..96dc9ab4401 --- /dev/null +++ b/Lib/test/test_asyncio/functional.py @@ -0,0 +1,268 @@ +import asyncio +import asyncio.events +import contextlib +import os +import pprint +import select +import socket +import tempfile +import threading +from test import support + + +class FunctionalTestCaseMixin: + + def new_loop(self): + return asyncio.new_event_loop() + + def run_loop_briefly(self, *, delay=0.01): + self.loop.run_until_complete(asyncio.sleep(delay)) + + def loop_exception_handler(self, loop, context): + self.__unhandled_exceptions.append(context) + self.loop.default_exception_handler(context) + + def setUp(self): + self.loop = self.new_loop() + asyncio.set_event_loop(None) + + self.loop.set_exception_handler(self.loop_exception_handler) + self.__unhandled_exceptions = [] + + def tearDown(self): + try: + self.loop.close() + + if self.__unhandled_exceptions: + print('Unexpected calls to loop.call_exception_handler():') + pprint.pprint(self.__unhandled_exceptions) + self.fail('unexpected calls to loop.call_exception_handler()') + + finally: + asyncio.set_event_loop(None) + self.loop = None + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=support.LOOPBACK_TIMEOUT, + backlog=1, + max_clients=10): + + if addr is None: + if hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.create_server(addr, family=family, backlog=backlog) + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=support.LOOPBACK_TIMEOUT): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + @contextlib.contextmanager + def unix_sock_name(self): + with tempfile.TemporaryDirectory() as td: + fn = os.path.join(td, 'sock') + try: + yield fn + finally: + try: + os.unlink(fn) + except OSError: + pass + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + +############################################################################## +# Socket Testing Utilities +############################################################################## + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def start_tls(self, ssl_context, *, + server_side=False, + server_hostname=None): + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=False) + + try: + ssl_sock.do_handshake() + except: + ssl_sock.close() + raise + finally: + self.__sock.close() + + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except Exception as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + self._sock.close() + self._s1.close() + self._s2.close() + + + def run(self): + self._sock.setblocking(False) + self._run() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except TimeoutError: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except Exception as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py new file mode 100644 index 00000000000..92895bbb420 --- /dev/null +++ b/Lib/test/test_asyncio/test_base_events.py @@ -0,0 +1,2297 @@ +"""Tests for base_events.py""" + +import concurrent.futures +import errno +import math +import platform +import socket +import sys +import threading +import time +import unittest +from unittest import mock + +import asyncio +from asyncio import base_events +from asyncio import constants +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok +from test.support import os_helper +from test.support import socket_helper +import warnings + +MOCK_ANY = mock.ANY + + +class CustomError(Exception): + pass + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def mock_socket_module(): + m_socket = mock.MagicMock(spec=socket) + for name in ( + 'AF_INET', 'AF_INET6', 'AF_UNSPEC', 'IPPROTO_TCP', 'IPPROTO_UDP', + 'SOCK_STREAM', 'SOCK_DGRAM', 'SOL_SOCKET', 'SO_REUSEADDR', 'inet_pton' + ): + if hasattr(socket, name): + setattr(m_socket, name, getattr(socket, name)) + else: + delattr(m_socket, name) + + m_socket.socket = mock.MagicMock() + m_socket.socket.return_value = test_utils.mock_nonblocking_socket() + + return m_socket + + +def patch_socket(f): + return mock.patch('asyncio.base_events.socket', + new_callable=mock_socket_module)(f) + + +class BaseEventTests(test_utils.TestCase): + + def test_ipaddr_info(self): + UNSPEC = socket.AF_UNSPEC + INET = socket.AF_INET + INET6 = socket.AF_INET6 + STREAM = socket.SOCK_STREAM + DGRAM = socket.SOCK_DGRAM + TCP = socket.IPPROTO_TCP + UDP = socket.IPPROTO_UDP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info(b'1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, TCP)) + + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, UDP)) + + # Socket type STREAM implies TCP protocol. + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, 0)) + + # Socket type DGRAM implies UDP protocol. + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, 0)) + + # No socket type. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0)) + + if socket_helper.IPV6_ENABLED: + # IPv4 address with family IPv6. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), + base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), + base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP)) + + # IPv6 address with family IPv4. + self.assertIsNone( + base_events._ipaddr_info('::3', 1, INET, STREAM, TCP)) + + # IPv6 address with zone index. + self.assertIsNone( + base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP)) + + def test_port_parameter_types(self): + # Test obscure kinds of arguments for "port". + INET = socket.AF_INET + STREAM = socket.SOCK_STREAM + TCP = socket.IPPROTO_TCP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', None, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', b'', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', '', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', '1', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', b'1', INET, STREAM, TCP)) + + @patch_socket + def test_ipaddr_info_no_inet_pton(self, m_socket): + del m_socket.inet_pton + self.assertIsNone(base_events._ipaddr_info('1.2.3.4', 1, + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP)) + + +class BaseEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = base_events.BaseEventLoop() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + def test_not_implemented(self): + m = mock.Mock() + self.assertRaises( + NotImplementedError, + self.loop._make_socket_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_ssl_transport, m, m, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_datagram_transport, m, m) + self.assertRaises( + NotImplementedError, self.loop._process_events, []) + self.assertRaises( + NotImplementedError, self.loop._write_to_self) + self.assertRaises( + NotImplementedError, + self.loop._make_read_pipe_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_write_pipe_transport, m, m) + gen = self.loop._make_subprocess_transport(m, m, m, m, m, m, m) + with self.assertRaises(NotImplementedError): + gen.send(None) + + def test_close(self): + self.assertFalse(self.loop.is_closed()) + self.loop.close() + self.assertTrue(self.loop.is_closed()) + + # it should be possible to call close() more than once + self.loop.close() + self.loop.close() + + # operation blocked when the loop is closed + f = self.loop.create_future() + self.assertRaises(RuntimeError, self.loop.run_forever) + self.assertRaises(RuntimeError, self.loop.run_until_complete, f) + + def test__add_callback_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop, None) + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertIn(h, self.loop._ready) + + def test__add_callback_cancelled_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop, None) + h.cancel() + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertFalse(self.loop._ready) + + def test_set_default_executor(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def submit(self, fn, *args, **kwargs): + raise NotImplementedError( + 'cannot submit into a dummy executor') + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + executor = DummyExecutor() + self.loop.set_default_executor(executor) + self.assertIs(executor, self.loop._default_executor) + + def test_set_default_executor_error(self): + executor = mock.Mock() + + msg = 'executor must be ThreadPoolExecutor instance' + with self.assertRaisesRegex(TypeError, msg): + self.loop.set_default_executor(executor) + + self.assertIsNone(self.loop._default_executor) + + def test_shutdown_default_executor_timeout(self): + event = threading.Event() + + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + event.wait() + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + try: + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + finally: + event.set() + + def test_call_soon(self): + def cb(): + pass + + h = self.loop.call_soon(cb) + self.assertEqual(h._callback, cb) + self.assertIsInstance(h, asyncio.Handle) + self.assertIn(h, self.loop._ready) + + def test_call_soon_non_callable(self): + self.loop.set_debug(True) + with self.assertRaisesRegex(TypeError, 'a callable object'): + self.loop.call_soon(1) + + def test_call_later(self): + def cb(): + pass + + h = self.loop.call_later(10.0, cb) + self.assertIsInstance(h, asyncio.TimerHandle) + self.assertIn(h, self.loop._scheduled) + self.assertNotIn(h, self.loop._ready) + with self.assertRaises(TypeError, msg="delay must not be None"): + self.loop.call_later(None, cb) + + def test_call_later_negative_delays(self): + calls = [] + + def cb(arg): + calls.append(arg) + + self.loop._process_events = mock.Mock() + self.loop.call_later(-1, cb, 'a') + self.loop.call_later(-2, cb, 'b') + test_utils.run_briefly(self.loop) + self.assertEqual(calls, ['b', 'a']) + + def test_time_and_call_at(self): + def cb(): + self.loop.stop() + + self.loop._process_events = mock.Mock() + delay = 0.100 + + when = self.loop.time() + delay + self.loop.call_at(when, cb) + t0 = self.loop.time() + self.loop.run_forever() + dt = self.loop.time() - t0 + + # 50 ms: maximum granularity of the event loop + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) + with self.assertRaises(TypeError, msg="when cannot be None"): + self.loop.call_at(None, cb) + + def check_thread(self, loop, debug): + def cb(): + pass + + loop.set_debug(debug) + if debug: + msg = ("Non-thread-safe operation invoked on an event loop other " + "than the current one") + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_soon(cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_later(60, cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_at(loop.time() + 60, cb) + else: + loop.call_soon(cb) + loop.call_later(60, cb) + loop.call_at(loop.time() + 60, cb) + + def test_check_thread(self): + def check_in_thread(loop, event, debug, create_loop, fut): + # wait until the event loop is running + event.wait() + + try: + if create_loop: + loop2 = base_events.BaseEventLoop() + try: + asyncio.set_event_loop(loop2) + self.check_thread(loop, debug) + finally: + asyncio.set_event_loop(None) + loop2.close() + else: + self.check_thread(loop, debug) + except Exception as exc: + loop.call_soon_threadsafe(fut.set_exception, exc) + else: + loop.call_soon_threadsafe(fut.set_result, None) + + def test_thread(loop, debug, create_loop=False): + event = threading.Event() + fut = loop.create_future() + loop.call_soon(event.set) + args = (loop, event, debug, create_loop, fut) + thread = threading.Thread(target=check_in_thread, args=args) + thread.start() + loop.run_until_complete(fut) + thread.join() + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + # raise RuntimeError if the thread has no event loop + test_thread(self.loop, True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False) + + # raise RuntimeError if the event loop of the thread is not the called + # event loop + test_thread(self.loop, True, create_loop=True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False, create_loop=True) + + def test__run_once(self): + h1 = asyncio.TimerHandle(time.monotonic() + 5.0, lambda: True, (), + self.loop, None) + h2 = asyncio.TimerHandle(time.monotonic() + 10.0, lambda: True, (), + self.loop, None) + + h1.cancel() + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h1) + self.loop._scheduled.append(h2) + self.loop._run_once() + + t = self.loop._selector.select.call_args[0][0] + self.assertTrue(9.5 < t < 10.5, t) + self.assertEqual([h2], self.loop._scheduled) + self.assertTrue(self.loop._process_events.called) + + def test_set_debug(self): + self.loop.set_debug(True) + self.assertTrue(self.loop.get_debug()) + self.loop.set_debug(False) + self.assertFalse(self.loop.get_debug()) + + def test__run_once_schedule_handle(self): + handle = None + processed = False + + def cb(loop): + nonlocal processed, handle + processed = True + handle = loop.call_soon(lambda: True) + + h = asyncio.TimerHandle(time.monotonic() - 1, cb, (self.loop,), + self.loop, None) + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h) + self.loop._run_once() + + self.assertTrue(processed) + self.assertEqual([handle], list(self.loop._ready)) + + def test__run_once_cancelled_event_cleanup(self): + self.loop._process_events = mock.Mock() + + self.assertTrue( + 0 < base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION < 1.0) + + def cb(): + pass + + # Set up one "blocking" event that will not be cancelled to + # ensure later cancelled events do not make it to the head + # of the queue and get cleaned. + not_cancelled_count = 1 + self.loop.call_later(3000, cb) + + # Add less than threshold (base_events._MIN_SCHEDULED_TIMER_HANDLES) + # cancelled handles, ensure they aren't removed + + cancelled_count = 2 + for x in range(2): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Add some cancelled events that will be at head and removed + cancelled_count += 2 + for x in range(2): + h = self.loop.call_later(100, cb) + h.cancel() + + # This test is invalid if _MIN_SCHEDULED_TIMER_HANDLES is too low + self.assertLessEqual(cancelled_count + not_cancelled_count, + base_events._MIN_SCHEDULED_TIMER_HANDLES) + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.loop._run_once() + + cancelled_count -= 2 + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + # Need enough events to pass _MIN_CANCELLED_TIMER_HANDLES_FRACTION + # so that deletion of cancelled events will occur on next _run_once + add_cancel_count = int(math.ceil( + base_events._MIN_SCHEDULED_TIMER_HANDLES * + base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION)) + 1 + + add_not_cancel_count = max(base_events._MIN_SCHEDULED_TIMER_HANDLES - + add_cancel_count, 0) + + # Add some events that will not be cancelled + not_cancelled_count += add_not_cancel_count + for x in range(add_not_cancel_count): + self.loop.call_later(3600, cb) + + # Add enough cancelled events + cancelled_count += add_cancel_count + for x in range(add_cancel_count): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Ensure all handles are still scheduled + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + self.loop._run_once() + + # Ensure cancelled events were removed + self.assertEqual(len(self.loop._scheduled), not_cancelled_count) + + # Ensure only uncancelled events remain scheduled + self.assertTrue(all([not x._cancelled for x in self.loop._scheduled])) + + def test_run_until_complete_type_error(self): + self.assertRaises(TypeError, + self.loop.run_until_complete, 'blah') + + def test_run_until_complete_loop(self): + task = self.loop.create_future() + other_loop = self.new_test_loop() + self.addCleanup(other_loop.close) + self.assertRaises(ValueError, + other_loop.run_until_complete, task) + + def test_run_until_complete_loop_orphan_future_close_loop(self): + class ShowStopper(SystemExit): + pass + + async def foo(delay): + await asyncio.sleep(delay) + + def throw(): + raise ShowStopper + + self.loop._process_events = mock.Mock() + self.loop.call_soon(throw) + with self.assertRaises(ShowStopper): + self.loop.run_until_complete(foo(0.1)) + + # This call fails if run_until_complete does not clean up + # done-callback for the previous future. + self.loop.run_until_complete(foo(0.2)) + + def test_subprocess_exec_invalid_args(self): + args = [sys.executable, '-c', 'pass'] + + # missing program parameter (empty args) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol) + + # expected multiple arguments, not a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, args) + + # program arguments must be strings, not int + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, sys.executable, 123) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, bufsize=4096) + + def test_subprocess_shell_invalid_args(self): + # expected a string, not an int or a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 123) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, [sys.executable, '-c', 'pass']) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', bufsize=4096) + + def test_default_exc_handler_callback(self): + self.loop._process_events = mock.Mock() + + def zero_error(fut): + fut.set_result(True) + 1/0 + + # Test call_soon (events.Handle) + with mock.patch('asyncio.base_events.logger') as log: + fut = self.loop.create_future() + self.loop.call_soon(zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + # Test call_later (events.TimerHandle) + with mock.patch('asyncio.base_events.logger') as log: + fut = self.loop.create_future() + self.loop.call_later(0.01, zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_coro(self): + self.loop._process_events = mock.Mock() + + async def zero_error_coro(): + await asyncio.sleep(0.01) + 1/0 + + # Test Future.__del__ + with mock.patch('asyncio.base_events.logger') as log: + fut = asyncio.ensure_future(zero_error_coro(), loop=self.loop) + fut.add_done_callback(lambda *args: self.loop.stop()) + self.loop.run_forever() + fut = None # Trigger Future.__del__ or futures._TracebackLogger + support.gc_collect() + # Future.__del__ in logs error with an actual exception context + log.error.assert_called_with( + test_utils.MockPattern('.*exception was never retrieved'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + def test_set_exc_handler_invalid(self): + with self.assertRaisesRegex(TypeError, 'A callable object or None'): + self.loop.set_exception_handler('spam') + + def test_set_exc_handler_custom(self): + def zero_error(): + 1/0 + + def run_loop(): + handle = self.loop.call_soon(zero_error) + self.loop._run_once() + return handle + + self.loop.set_debug(True) + self.loop._process_events = mock.Mock() + + self.assertIsNone(self.loop.get_exception_handler()) + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + self.assertIs(self.loop.get_exception_handler(), mock_handler) + handle = run_loop() + mock_handler.assert_called_with(self.loop, { + 'exception': MOCK_ANY, + 'message': test_utils.MockPattern( + 'Exception in callback.*zero_error'), + 'handle': handle, + 'source_traceback': handle._source_traceback, + }) + mock_handler.reset_mock() + + self.loop.set_exception_handler(None) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + self.assertFalse(mock_handler.called) + + def test_set_exc_handler_broken(self): + def run_loop(): + def zero_error(): + 1/0 + self.loop.call_soon(zero_error) + self.loop._run_once() + + def handler(loop, context): + raise AttributeError('spam') + + self.loop._process_events = mock.Mock() + + self.loop.set_exception_handler(handler) + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Unhandled error in exception handler'), + exc_info=(AttributeError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_broken(self): + _context = None + + class Loop(base_events.BaseEventLoop): + + _selector = mock.Mock() + _process_events = mock.Mock() + + def default_exception_handler(self, context): + nonlocal _context + _context = context + # Simulates custom buggy "default_exception_handler" + raise ValueError('spam') + + loop = Loop() + self.addCleanup(loop.close) + asyncio.set_event_loop(loop) + + def run_loop(): + def zero_error(): + 1/0 + loop.call_soon(zero_error) + loop._run_once() + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + 'Exception in default exception handler', + exc_info=True) + + def custom_handler(loop, context): + raise ValueError('ham') + + _context = None + loop.set_exception_handler(custom_handler) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern('Exception in default exception.*' + 'while handling.*in custom'), + exc_info=True) + + # Check that original context was passed to default + # exception handler. + self.assertIn('context', _context) + self.assertIs(type(_context['context']['exception']), + ZeroDivisionError) + + def test_set_task_factory_invalid(self): + with self.assertRaisesRegex( + TypeError, 'task factory must be a callable or None'): + + self.loop.set_task_factory(1) + + self.assertIsNone(self.loop.get_task_factory()) + + def test_set_task_factory(self): + self.loop._process_events = mock.Mock() + + class MyTask(asyncio.Task): + pass + + async def coro(): + pass + + factory = lambda loop, coro: MyTask(coro, loop=loop) + + self.assertIsNone(self.loop.get_task_factory()) + self.loop.set_task_factory(factory) + self.assertIs(self.loop.get_task_factory(), factory) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + self.loop.set_task_factory(None) + self.assertIsNone(self.loop.get_task_factory()) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, asyncio.Task)) + self.assertFalse(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + def test_env_var_debug(self): + code = '\n'.join(( + 'import asyncio', + 'loop = asyncio.new_event_loop()', + 'print(loop.get_debug())')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1') + self.assertEqual(stdout.rstrip(), b'False') + + # -X dev + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + + def test_create_task(self): + class MyTask(asyncio.Task): + pass + + async def test(): + pass + + class EventLoop(base_events.BaseEventLoop): + def create_task(self, coro): + return MyTask(coro, loop=loop) + + loop = EventLoop() + self.set_event_loop(loop) + + coro = test() + task = asyncio.ensure_future(coro, loop=loop) + self.assertIsInstance(task, MyTask) + + # make warnings quiet + task._log_destroy_pending = False + coro.close() + + def test_create_task_error_closes_coro(self): + async def test(): + pass + loop = asyncio.new_event_loop() + loop.close() + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(RuntimeError): + asyncio.ensure_future(test(), loop=loop) + self.assertEqual(len(w), 0) + + + def test_create_named_task_with_default_factory(self): + async def test(): + pass + + loop = asyncio.new_event_loop() + task = loop.create_task(test(), name='test_task') + try: + self.assertEqual(task.get_name(), 'test_task') + finally: + loop.run_until_complete(task) + loop.close() + + def test_create_named_task_with_custom_factory(self): + def task_factory(loop, coro, **kwargs): + return asyncio.Task(coro, loop=loop, **kwargs) + + async def test(): + pass + + loop = asyncio.new_event_loop() + loop.set_task_factory(task_factory) + task = loop.create_task(test(), name='test_task') + try: + self.assertEqual(task.get_name(), 'test_task') + finally: + loop.run_until_complete(task) + loop.close() + + def test_run_forever_keyboard_interrupt(self): + # Python issue #22601: ensure that the temporary task created by + # run_forever() consumes the KeyboardInterrupt and so don't log + # a warning + async def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + try: + self.loop.run_until_complete(raise_keyboard_interrupt()) + except KeyboardInterrupt: + pass + self.loop.close() + support.gc_collect() + + self.assertFalse(self.loop.call_exception_handler.called) + + def test_run_until_complete_baseexception(self): + # Python issue #22429: run_until_complete() must not schedule a pending + # call to stop() if the future raised a BaseException + async def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + + with self.assertRaises(KeyboardInterrupt): + self.loop.run_until_complete(raise_keyboard_interrupt()) + + def func(): + self.loop.stop() + func.called = True + func.called = False + self.loop.call_soon(self.loop.call_soon, func) + self.loop.run_forever() + self.assertTrue(func.called) + + def test_single_selecter_event_callback_after_stopping(self): + # Python issue #25593: A stopped event loop may cause event callbacks + # to run more than once. + event_sentinel = object() + callcount = 0 + doer = None + + def proc_events(event_list): + nonlocal doer + if event_sentinel in event_list: + doer = self.loop.call_soon(do_event) + + def do_event(): + nonlocal callcount + callcount += 1 + self.loop.call_soon(clear_selector) + + def clear_selector(): + doer.cancel() + self.loop._selector.select.return_value = () + + self.loop._process_events = proc_events + self.loop._selector.select.return_value = (event_sentinel,) + + for i in range(1, 3): + with self.subTest('Loop %d/2' % i): + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(callcount, 1) + + def test_run_once(self): + # Simple test for test_utils.run_once(). It may seem strange + # to have a test for this (the function isn't even used!) but + # it's a de-factor standard API for library tests. This tests + # the idiom: loop.call_soon(loop.stop); loop.run_forever(). + count = 0 + + def callback(): + nonlocal count + count += 1 + + self.loop._process_events = mock.Mock() + self.loop.call_soon(callback) + test_utils.run_once(self.loop) + self.assertEqual(count, 1) + + def test_run_forever_pre_stopped(self): + # Test that the old idiom for pre-stopping the loop works. + self.loop._process_events = mock.Mock() + self.loop.stop() + self.loop.run_forever() + self.loop._selector.select.assert_called_once_with(0) + + def test_custom_run_forever_integration(self): + # Test that the run_forever_setup() and run_forever_cleanup() primitives + # can be used to implement a custom run_forever loop. + self.loop._process_events = mock.Mock() + + count = 0 + + def callback(): + nonlocal count + count += 1 + + self.loop.call_soon(callback) + + # Set up the custom event loop + self.loop._run_forever_setup() + + # Confirm the loop has been started + self.assertEqual(asyncio.get_running_loop(), self.loop) + self.assertTrue(self.loop.is_running()) + + # Our custom "event loop" just iterates 10 times before exiting. + for i in range(10): + self.loop._run_once() + + # Clean up the event loop + self.loop._run_forever_cleanup() + + # Confirm the loop has been cleaned up + with self.assertRaises(RuntimeError): + asyncio.get_running_loop() + self.assertFalse(self.loop.is_running()) + + # Confirm the loop actually did run, processing events 10 times, + # and invoking the callback once. + self.assertEqual(self.loop._process_events.call_count, 10) + self.assertEqual(count, 1) + + async def leave_unfinalized_asyncgen(self): + # Create an async generator, iterate it partially, and leave it + # to be garbage collected. + # Used in async generator finalization tests. + # Depends on implementation details of garbage collector. Changes + # in gc may break this function. + status = {'started': False, + 'stopped': False, + 'finalized': False} + + async def agen(): + status['started'] = True + try: + for item in ['ZERO', 'ONE', 'TWO', 'THREE', 'FOUR']: + yield item + finally: + status['finalized'] = True + + ag = agen() + ai = ag.__aiter__() + + async def iter_one(): + try: + item = await ai.__anext__() + except StopAsyncIteration: + return + if item == 'THREE': + status['stopped'] = True + return + asyncio.create_task(iter_one()) + + asyncio.create_task(iter_one()) + return status + + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators + def test_asyncgen_finalization_by_gc(self): + # Async generators should be finalized when garbage collected. + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + with support.disable_gc(): + status = self.loop.run_until_complete(self.leave_unfinalized_asyncgen()) + while not status['stopped']: + test_utils.run_briefly(self.loop) + self.assertTrue(status['started']) + self.assertTrue(status['stopped']) + self.assertFalse(status['finalized']) + support.gc_collect() + test_utils.run_briefly(self.loop) + self.assertTrue(status['finalized']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators + def test_asyncgen_finalization_by_gc_in_other_thread(self): + # Python issue 34769: If garbage collector runs in another + # thread, async generators will not finalize in debug + # mode. + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + self.loop.set_debug(True) + with support.disable_gc(): + status = self.loop.run_until_complete(self.leave_unfinalized_asyncgen()) + while not status['stopped']: + test_utils.run_briefly(self.loop) + self.assertTrue(status['started']) + self.assertTrue(status['stopped']) + self.assertFalse(status['finalized']) + self.loop.run_until_complete( + self.loop.run_in_executor(None, support.gc_collect)) + test_utils.run_briefly(self.loop) + self.assertTrue(status['finalized']) + + +class MyProto(asyncio.Protocol): + done = None + + def __init__(self, create_future=False): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = asyncio.get_running_loop().create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, create_future=False, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + self._assert_state('INITIALIZED') + self.nbytes += len(data) + + def error_received(self, exc): + self._assert_state('INITIALIZED') + + def connection_lost(self, exc): + self._assert_state('INITIALIZED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class BaseEventLoopWithSelectorTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + @mock.patch('socket.getnameinfo') + def test_getnameinfo(self, m_gai): + m_gai.side_effect = lambda *args: 42 + r = self.loop.run_until_complete(self.loop.getnameinfo(('abc', 123))) + self.assertEqual(r, 42) + + @patch_socket + def test_create_connection_multiple_errors(self, m_socket): + + class MyProto(asyncio.Protocol): + pass + + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + idx = -1 + errors = ['err1', 'err2'] + + def _socket(*args, **kw): + nonlocal idx, errors + idx += 1 + raise OSError(errors[idx]) + + m_socket.socket = _socket + + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertEqual(str(cm.exception), 'Multiple exceptions: err1, err2') + + idx = -1 + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + @patch_socket + def test_create_connection_timeout(self, m_socket): + # Ensure that the socket is closed on timeout + sock = mock.Mock() + m_socket.socket.return_value = sock + + def getaddrinfo(*args, **kw): + fut = self.loop.create_future() + addr = (socket.AF_INET, socket.SOCK_STREAM, 0, '', + ('127.0.0.1', 80)) + fut.set_result([addr]) + return fut + self.loop.getaddrinfo = getaddrinfo + + with mock.patch.object(self.loop, 'sock_connect', + side_effect=asyncio.TimeoutError): + coro = self.loop.create_connection(MyProto, '127.0.0.1', 80) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + @patch_socket + def test_create_connection_happy_eyeballs_empty_exceptions(self, m_socket): + # See gh-135836: Fix IndexError when Happy Eyeballs algorithm + # results in empty exceptions list + + async def getaddrinfo(*args, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, 0, '', ('127.0.0.1', 80)), + (socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + + # Mock staggered_race to return empty exceptions list + # This simulates the scenario where Happy Eyeballs algorithm + # cancels all attempts but doesn't properly collect exceptions + with mock.patch('asyncio.staggered.staggered_race') as mock_staggered: + # Return (None, []) - no winner, empty exceptions list + async def mock_race(coro_fns, delay, loop): + return None, [] + mock_staggered.side_effect = mock_race + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, happy_eyeballs_delay=0.1) + + # Should raise TimeoutError instead of IndexError + with self.assertRaisesRegex(TimeoutError, "create_connection failed"): + self.loop.run_until_complete(coro) + + def test_create_connection_host_port_sock(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_wrong_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_connection(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A Stream Socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_server_wrong_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_server(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A Stream Socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_server_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_server( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'no socket.SOCK_NONBLOCK (linux only)') + def test_create_server_stream_bittype(self): + sock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) + with sock: + coro = self.loop.create_server(lambda: None, sock=sock) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'no IPv6 support') + def test_create_server_ipv6(self): + async def main(): + srv = await asyncio.start_server(lambda: None, '::1', 0) + try: + self.assertGreater(len(srv.sockets), 0) + finally: + srv.close() + await srv.wait_closed() + + try: + self.loop.run_until_complete(main()) + except OSError as ex: + if (hasattr(errno, 'EADDRNOTAVAIL') and + ex.errno == errno.EADDRNOTAVAIL): + self.skipTest('failed to bind to ::1') + else: + raise + + def test_create_datagram_endpoint_wrong_sock(self): + sock = socket.socket(socket.AF_INET) + with sock: + coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A datagram socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_connection_no_host_port_sock(self): + coro = self.loop.create_connection(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_no_getaddrinfo(self): + async def getaddrinfo(*args, **kw): + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_connection_connect_err(self): + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + self.assertEqual(len(cm.exception.exceptions), 1) + self.assertIsInstance(cm.exception.exceptions[0], OSError) + + @patch_socket + def test_create_connection_connect_non_os_err_close_err(self, m_socket): + # Test the case when sock_connect() raises non-OSError exception + # and sock.close() raises OSError. + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = CustomError + sock = mock.Mock() + m_socket.socket.return_value = sock + sock.close.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + + def test_create_connection_multiple(self): + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET) + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + @patch_socket + def test_create_connection_multiple_errors_local_addr(self, m_socket): + + def bind(addr): + if addr[0] == '0.0.0.1': + err = OSError('Err') + err.strerror = 'Err' + raise err + + m_socket.socket.return_value.bind = bind + + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError('Err2') + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertStartsWith(str(cm.exception), 'Multiple exceptions: ') + self.assertTrue(m_socket.socket.return_value.close.called) + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080), all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + def _test_create_connection_ip_addr(self, m_socket, allow_inet_pton): + # Test the fallback code, even if this system has inet_pton. + if not allow_inet_pton: + del m_socket.inet_pton + + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + coro = self.loop.create_connection(asyncio.Protocol, '1.2.3.4', 80) + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('1.2.3.4', 80)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + if socket_helper.IPV6_ENABLED: + sock.family = socket.AF_INET6 + coro = self.loop.create_connection(asyncio.Protocol, '::1', 80) + t, p = self.loop.run_until_complete(coro) + try: + # Without inet_pton we use getaddrinfo, which transforms + # ('::1', 80) to ('::1', 80, 0, 0). The last 0s are flow info, + # scope id. + [address] = sock.connect.call_args[0] + host, port = address[:2] + self.assertRegex(host, r'::(0\.)*1') + self.assertEqual(port, 80) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET6) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'no IPv6 support') + @unittest.skipIf(sys.platform.startswith('aix'), + "bpo-25545: IPv6 scope id and getaddrinfo() behave differently on AIX") + @patch_socket + def test_create_connection_ipv6_scope(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + sock.family = socket.AF_INET6 + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + coro = self.loop.create_connection(asyncio.Protocol, 'fe80::1%1', 80) + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('fe80::1', 80, 0, 1)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET6) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + @patch_socket + def test_create_connection_ip_addr(self, m_socket): + self._test_create_connection_ip_addr(m_socket, True) + + @patch_socket + def test_create_connection_no_inet_pton(self, m_socket): + self._test_create_connection_ip_addr(m_socket, False) + + @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) + def test_create_connection_service_name(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + for service, port in ('http', 80), (b'http', 80): + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('127.0.0.1', port)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + for service in 'nonsense', b'nonsense': + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + def test_create_connection_no_local_addr(self): + async def getaddrinfo(host, *args, **kw): + if host == 'example.com': + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + else: + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_connection_bluetooth(self, m_socket): + # See http://bugs.python.org/issue27136, fallback to getaddrinfo when + # we can't recognize an address is resolved, e.g. a Bluetooth address. + addr = ('00:01:02:03:04:05', 1) + + def getaddrinfo(host, port, *args, **kw): + self.assertEqual((host, port), addr) + return [(999, 1, 999, '', (addr, 1))] + + m_socket.getaddrinfo = getaddrinfo + sock = m_socket.socket() + coro = self.loop.sock_connect(sock, addr) + self.loop.run_until_complete(coro) + + def test_create_connection_ssl_server_hostname_default(self): + self.loop.getaddrinfo = mock.Mock() + + def mock_getaddrinfo(*args, **kwds): + f = self.loop.create_future() + f.set_result([(socket.AF_INET, socket.SOCK_STREAM, + socket.SOL_TCP, '', ('1.2.3.4', 80))]) + return f + + self.loop.getaddrinfo.side_effect = mock_getaddrinfo + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.return_value = self.loop.create_future() + self.loop.sock_connect.return_value.set_result(None) + self.loop._make_ssl_transport = mock.Mock() + + class _SelectorTransportMock: + _sock = None + + def get_extra_info(self, key): + return mock.Mock() + + def close(self): + self._sock.close() + + def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, + **kwds): + waiter.set_result(None) + transport = _SelectorTransportMock() + transport._sock = sock + return transport + + self.loop._make_ssl_transport.side_effect = mock_make_ssl_transport + ANY = mock.ANY + handshake_timeout = object() + shutdown_timeout = object() + # First try the default server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='python.org', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + # Next try an explicit server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + # Finally try an explicit empty server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + + def test_create_connection_no_ssl_server_hostname_errors(self): + # When not using ssl, server_hostname must be None. + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='python.org') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_ssl_server_hostname_errors(self): + # When using ssl, server_hostname may be None if host is non-empty. + coro = self.loop.create_connection(MyProto, '', 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, None, 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + sock = socket.socket() + coro = self.loop.create_connection(MyProto, None, None, + ssl=True, sock=sock) + self.addCleanup(sock.close) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + def test_create_server_empty_host(self): + # if host is empty string use None instead + host = object() + + async def getaddrinfo(*args, **kw): + nonlocal host + host = args[0] + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + fut = self.loop.create_server(MyProto, '', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertIsNone(host) + + def test_create_server_host_port_sock(self): + fut = self.loop.create_server( + MyProto, '0.0.0.0', 0, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_host_port_sock(self): + fut = self.loop.create_server(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_getaddrinfo(self): + getaddrinfo = self.loop.getaddrinfo = mock.Mock() + getaddrinfo.return_value = self.loop.create_future() + getaddrinfo.return_value.set_result(None) + + f = self.loop.create_server(MyProto, 'python.org', 0) + self.assertRaises(OSError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_nosoreuseport(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_soreuseport_only_defined(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.return_value = mock.Mock() + m_socket.SO_REUSEPORT = -1 + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_cant_bind(self, m_socket): + + class Err(OSError): + strerror = 'error' + + m_socket.getaddrinfo.return_value = [ + (2, 1, 6, '', ('127.0.0.1', 10100))] + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_server(MyProto, '0.0.0.0', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + @patch_socket + def test_create_datagram_endpoint_no_addrinfo(self, m_socket): + m_socket.getaddrinfo.return_value = [] + + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_addr_error(self): + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr='localhost') + self.assertRaises( + TypeError, self.loop.run_until_complete, coro) + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 1, 2, 3)) + self.assertRaises( + TypeError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_connect_err(self): + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, remote_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_allow_broadcast(self): + protocol = MyDatagramProto(create_future=True, loop=self.loop) + self.loop.sock_connect = sock_connect = mock.Mock() + sock_connect.return_value = [] + + coro = self.loop.create_datagram_endpoint( + lambda: protocol, + remote_addr=('127.0.0.1', 0), + allow_broadcast=True) + + transport, _ = self.loop.run_until_complete(coro) + self.assertFalse(sock_connect.called) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @patch_socket + def test_create_datagram_endpoint_socket_err(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, local_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_no_matching_family(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + remote_addr=('127.0.0.1', 0), local_addr=('::1', 0)) + self.assertRaises( + ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_setblk_err(self, m_socket): + m_socket.socket.return_value.setblocking.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + self.assertTrue( + m_socket.socket.return_value.close.called) + + def test_create_datagram_endpoint_noaddr_nofamily(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_cant_bind(self, m_socket): + class Err(OSError): + pass + + m_socket.getaddrinfo = socket.getaddrinfo + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, + local_addr=('127.0.0.1', 0), family=socket.AF_INET) + self.assertRaises(Err, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + def test_create_datagram_endpoint_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(('127.0.0.1', 0)) + fut = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + sock=sock) + transport, protocol = self.loop.run_until_complete(fut) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_datagram_endpoint_sock_unix(self): + fut = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + family=socket.AF_UNIX) + transport, protocol = self.loop.run_until_complete(fut) + self.assertEqual(transport._sock.family, socket.AF_UNIX) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_datagram_endpoint_existing_sock_unix(self): + with test_utils.unix_socket_path() as path: + sock = socket.socket(socket.AF_UNIX, type=socket.SOCK_DGRAM) + sock.bind(path) + sock.close() + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + path, family=socket.AF_UNIX) + transport, protocol = self.loop.run_until_complete(coro) + transport.close() + self.loop.run_until_complete(protocol.done) + + def test_create_datagram_endpoint_sock_sockopts(self): + class FakeSock: + type = socket.SOCK_DGRAM + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('127.0.0.1', 0), sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, family=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, proto=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, flags=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, reuse_port=True, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, allow_broadcast=True, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + @unittest.skipIf(sys.platform == 'vxworks', + "SO_BROADCAST is enabled by default on VxWorks") + def test_create_datagram_endpoint_sockopts(self): + # Socket options should not be applied unless asked for. + # SO_REUSEPORT is not available on all platforms. + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0)) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + + if reuseport_supported: + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_port=reuseport_supported, + allow_broadcast=True) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR)) + if reuseport_supported: + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @patch_socket + def test_create_datagram_endpoint_nosoreuseport(self, m_socket): + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_ip_addr(self, m_socket): + def getaddrinfo(*args, **kw): + self.fail('should not have called getaddrinfo') + + m_socket.getaddrinfo = getaddrinfo + m_socket.socket.return_value.bind = bind = mock.Mock() + self.loop._add_reader = mock.Mock() + + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('1.2.3.4', 0), + reuse_port=reuseport_supported) + + t, p = self.loop.run_until_complete(coro) + try: + bind.assert_called_with(('1.2.3.4', 0)) + m_socket.socket.assert_called_with(family=m_socket.AF_INET, + proto=m_socket.IPPROTO_UDP, + type=m_socket.SOCK_DGRAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + def test_accept_connection_retry(self): + sock = mock.Mock() + sock.accept.side_effect = BlockingIOError() + + self.loop._accept_connection(MyProto, sock) + self.assertFalse(sock.close.called) + + @mock.patch('asyncio.base_events.logger') + def test_accept_connection_exception(self, m_log): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.accept.side_effect = OSError(errno.EMFILE, 'Too many open files') + self.loop._remove_reader = mock.Mock() + self.loop.call_later = mock.Mock() + + self.loop._accept_connection(MyProto, sock) + self.assertTrue(m_log.error.called) + self.assertFalse(sock.close.called) + self.loop._remove_reader.assert_called_with(10) + self.loop.call_later.assert_called_with( + constants.ACCEPT_RETRY_DELAY, + # self.loop._start_serving + mock.ANY, + MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY) + + def test_call_coroutine(self): + async def simple_coroutine(): + pass + + self.loop.set_debug(True) + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + with self.assertRaises(TypeError): + self.loop.call_soon(func) + with self.assertRaises(TypeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(TypeError): + self.loop.call_later(60, func) + with self.assertRaises(TypeError): + self.loop.call_at(self.loop.time() + 60, func) + with self.assertRaises(TypeError): + self.loop.run_until_complete( + self.loop.run_in_executor(None, func)) + + @mock.patch('asyncio.base_events.logger') + def test_log_slow_callbacks(self, m_logger): + def stop_loop_cb(loop): + loop.stop() + + async def stop_loop_coro(loop): + loop.stop() + + asyncio.set_event_loop(self.loop) + self.loop.set_debug(True) + self.loop.slow_callback_duration = 0.0 + + # slow callback + self.loop.call_soon(stop_loop_cb, self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + # slow task + asyncio.ensure_future(stop_loop_coro(self.loop), loop=self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + +class RunningLoopTests(unittest.TestCase): + + def test_running_loop_within_a_loop(self): + async def runner(loop): + loop.run_forever() + + loop = asyncio.new_event_loop() + outer_loop = asyncio.new_event_loop() + try: + with self.assertRaisesRegex(RuntimeError, + 'while another loop is running'): + outer_loop.run_until_complete(runner(loop)) + finally: + loop.close() + outer_loop.close() + + +class BaseLoopSockSendfileTests(test_utils.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + self.transport = None + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + from asyncio.selector_events import BaseSelectorEventLoop + # BaseSelectorEventLoop() has no native implementation + self.loop = BaseSelectorEventLoop() + self.set_event_loop(self.loop) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + server = self.run_loop(self.loop.create_server( + lambda: proto, socket_helper.HOST, 0, family=socket.AF_INET)) + addr = server.sockets[0].getsockname() + + for _ in range(10): + try: + self.run_loop(self.loop.sock_connect(sock, addr)) + except OSError: + self.run_loop(asyncio.sleep(0.5)) + continue + else: + break + else: + # One last try, so we get the exception + self.run_loop(self.loop.sock_connect(sock, addr)) + + def cleanup(): + server.close() + sock.close() + if proto.transport is not None: + proto.transport.close() + self.run_loop(proto.wait_closed()) + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test__sock_sendfile_native_failure(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + + self.assertEqual(proto.data, b'') + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_no_fallback(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, + fallback=False)) + + self.assertEqual(self.file.tell(), 0) + self.assertEqual(proto.data, b'') + + def test_sock_sendfile_fallback(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(self.file.tell(), len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + + def test_sock_sendfile_fallback_offset_and_count(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 2000) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(proto.data, self.DATA[1000:3000]) + + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_nonbinary_file(self): + sock = self.make_socket() + with open(os_helper.TESTFN, encoding="utf-8") as f: + with self.assertRaisesRegex(ValueError, "binary mode"): + self.run_loop(self.loop.sock_sendfile(sock, f)) + + def test_nonstream_socket(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + self.addCleanup(sock.close) + with self.assertRaisesRegex(ValueError, "only SOCK_STREAM type"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_notint_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, 'count')) + + def test_negative_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, -1)) + + def test_notint_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 'offset')) + + def test_negative_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, -1)) + + +class TestSelectorUtils(test_utils.TestCase): + def check_set_nodelay(self, sock): + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertFalse(opt) + + base_events._set_nodelay(sock) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertTrue(opt) + + @unittest.skipUnless(hasattr(socket, 'TCP_NODELAY'), + 'need socket.TCP_NODELAY') + def test_set_nodelay(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_nodelay(sock) + + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + sock.setblocking(False) + self.check_set_nodelay(sock) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py new file mode 100644 index 00000000000..6d3edcc36f5 --- /dev/null +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -0,0 +1,89 @@ +import asyncio +import unittest + +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class ReceiveStuffProto(asyncio.BufferedProtocol): + def __init__(self, cb, con_lost_fut): + self.cb = cb + self.con_lost_fut = con_lost_fut + + def get_buffer(self, sizehint): + self.buffer = bytearray(100) + return self.buffer + + def buffer_updated(self, nbytes): + self.cb(self.buffer[:nbytes]) + + def connection_lost(self, exc): + if exc is None: + self.con_lost_fut.set_result(None) + else: + self.con_lost_fut.set_exception(exc) + + +class BaseTestBufferedProtocol(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_buffered_proto_create_connection(self): + + NOISE = b'12345678+' * 1024 + + async def client(addr): + data = b'' + + def on_buf(buf): + nonlocal data + data += buf + if data == NOISE: + tr.write(b'1') + + conn_lost_fut = self.loop.create_future() + + tr, pr = await self.loop.create_connection( + lambda: ReceiveStuffProto(on_buf, conn_lost_fut), *addr) + + await conn_lost_fut + + async def on_server_client(reader, writer): + writer.write(NOISE) + await reader.readexactly(1) + writer.close() + await writer.wait_closed() + + srv = self.loop.run_until_complete( + asyncio.start_server( + on_server_client, '127.0.0.1', 0)) + + addr = srv.sockets[0].getsockname() + self.loop.run_until_complete( + asyncio.wait_for(client(addr), 5)) + + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + +class BufferedProtocolSelectorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class BufferedProtocolProactorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py new file mode 100644 index 00000000000..f85f39839cb --- /dev/null +++ b/Lib/test/test_asyncio/test_context.py @@ -0,0 +1,38 @@ +import asyncio +import decimal +import unittest + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +@unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") +class DecimalContextTest(unittest.TestCase): + + def test_asyncio_task_decimal_context(self): + async def fractions(t, precision, x, y): + with decimal.localcontext() as ctx: + ctx.prec = precision + a = decimal.Decimal(x) / decimal.Decimal(y) + await asyncio.sleep(t) + b = decimal.Decimal(x) / decimal.Decimal(y ** 2) + return a, b + + async def main(): + r1, r2 = await asyncio.gather( + fractions(0.1, 3, 1, 3), fractions(0.2, 6, 1, 3)) + + return r1, r2 + + r1, r2 = asyncio.run(main()) + + self.assertEqual(str(r1[0]), '0.333') + self.assertEqual(str(r1[1]), '0.111') + + self.assertEqual(str(r2[0]), '0.333333') + self.assertEqual(str(r2[1]), '0.111111') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py new file mode 100644 index 00000000000..0561b54a3f1 --- /dev/null +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -0,0 +1,545 @@ +"""Tests for base_events.py""" + +import asyncio +import contextvars +import unittest + +from unittest import mock +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test.support.script_helper import assert_python_ok + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class EagerTaskFactoryLoopTests: + + Task = None + + def run_coro(self, coro): + """ + Helper method to run the `coro` coroutine in the test event loop. + It helps with making sure the event loop is running before starting + to execute `coro`. This is important for testing the eager step + functionality, since an eager step is taken only if the event loop + is already running. + """ + + async def coro_runner(): + self.assertTrue(asyncio.get_event_loop().is_running()) + return await coro + + return self.loop.run_until_complete(coro) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.eager_task_factory = asyncio.create_eager_task_factory(self.Task) + self.loop.set_task_factory(self.eager_task_factory) + self.set_event_loop(self.loop) + + def test_eager_task_factory_set(self): + self.assertIsNotNone(self.eager_task_factory) + self.assertIs(self.loop.get_task_factory(), self.eager_task_factory) + + async def noop(): pass + + async def run(): + t = self.loop.create_task(noop()) + self.assertIsInstance(t, self.Task) + await t + + self.run_coro(run()) + + def test_await_future_during_eager_step(self): + + async def set_result(fut, val): + fut.set_result(val) + + async def run(): + fut = self.loop.create_future() + t = self.loop.create_task(set_result(fut, 'my message')) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await fut + + self.assertEqual(self.run_coro(run()), 'my message') + + def test_eager_completion(self): + + async def coro(): + return 'hello' + + async def run(): + t = self.loop.create_task(coro()) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await t + + self.assertEqual(self.run_coro(run()), 'hello') + + def test_block_after_eager_step(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + self.assertFalse(t.done()) + result = await t + self.assertTrue(t.done()) + return result + + self.assertEqual(self.run_coro(run()), 'finished after blocking') + + def test_cancellation_after_eager_completion(self): + + async def coro(): + return 'finished without blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel() + result = await t + # finished task can't be cancelled + self.assertFalse(t.cancelled()) + return result + + self.assertEqual(self.run_coro(run()), 'finished without blocking') + + def test_cancellation_after_eager_step_blocks(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel('cancellation message') + self.assertGreater(t.cancelling(), 0) + result = await t + + with self.assertRaises(asyncio.CancelledError) as cm: + self.run_coro(run()) + + self.assertEqual('cancellation message', cm.exception.args[0]) + + def test_current_task(self): + captured_current_task = None + + async def coro(): + nonlocal captured_current_task + captured_current_task = asyncio.current_task() + # verify the task before and after blocking is identical + await asyncio.sleep(0.1) + self.assertIs(asyncio.current_task(), captured_current_task) + + async def run(): + t = self.loop.create_task(coro()) + self.assertIs(captured_current_task, t) + await t + + self.run_coro(run()) + captured_current_task = None + + def test_all_tasks_with_eager_completion(self): + captured_all_tasks = None + + async def coro(): + nonlocal captured_all_tasks + captured_all_tasks = asyncio.all_tasks() + + async def run(): + t = self.loop.create_task(coro()) + self.assertIn(t, captured_all_tasks) + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_all_tasks_with_blocking(self): + captured_eager_all_tasks = None + + async def coro(fut1, fut2): + nonlocal captured_eager_all_tasks + captured_eager_all_tasks = asyncio.all_tasks() + await fut1 + fut2.set_result(None) + + async def run(): + fut1 = self.loop.create_future() + fut2 = self.loop.create_future() + t = self.loop.create_task(coro(fut1, fut2)) + self.assertIn(t, captured_eager_all_tasks) + self.assertIn(t, asyncio.all_tasks()) + fut1.set_result(None) + await fut2 + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_context_vars(self): + cv = contextvars.ContextVar('cv', default=0) + + coro_first_step_ran = False + coro_second_step_ran = False + + async def coro(): + nonlocal coro_first_step_ran + nonlocal coro_second_step_ran + self.assertEqual(cv.get(), 1) + cv.set(2) + self.assertEqual(cv.get(), 2) + coro_first_step_ran = True + await asyncio.sleep(0.1) + self.assertEqual(cv.get(), 2) + cv.set(3) + self.assertEqual(cv.get(), 3) + coro_second_step_ran = True + + async def run(): + cv.set(1) + t = self.loop.create_task(coro()) + self.assertTrue(coro_first_step_ran) + self.assertFalse(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + await t + self.assertTrue(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks(self): + # See https://github.com/python/cpython/issues/124309 + + async def fail(): + await asyncio.sleep(0) + raise ValueError("no good") + + async def blocked(): + fut = asyncio.Future() + await fut + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: blocked(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: fail() + ], + delay=0.25 + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIsInstance(excs[2], ValueError) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks_no_delay(self): + # See https://github.com/python/cpython/issues/124309 + async def fail(): + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: fail(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: asyncio.sleep(0, result="sleep0"), + ], + delay=None + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], ValueError) + self.assertEqual(len(excs), 2) + + self.run_coro(run()) + + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=False, name="example" + ) + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + self.run_coro(main()) + + +class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._all_tasks = asyncio.all_tasks + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + self._all_tasks = asyncio.all_tasks + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + + def test_issue105987(self): + code = """if 1: + from _asyncio import _swap_current_task, _set_running_loop + + class DummyTask: + pass + + class DummyLoop: + pass + + l = DummyLoop() + _set_running_loop(l) + _swap_current_task(l, DummyTask()) + t = _swap_current_task(l, None) + """ + + _, out, err = assert_python_ok("-c", code) + self.assertFalse(err) + + def test_issue122332(self): + async def coro(): + pass + + async def run(): + task = self.loop.create_task(coro()) + await task + self.assertIsNone(task.get_coro()) + + self.run_coro(run()) + + def test_name(self): + name = None + async def coro(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + task = self.loop.create_task(coro(), name="test name") + self.assertEqual(name, "test name") + await task + + self.run_coro(coro()) + +class AsyncTaskCounter: + def __init__(self, loop, *, task_class, eager): + self.suspense_count = 0 + self.task_count = 0 + + def CountingTask(*args, eager_start=False, **kwargs): + if not eager_start: + self.task_count += 1 + kwargs["eager_start"] = eager_start + return task_class(*args, **kwargs) + + if eager: + factory = asyncio.create_eager_task_factory(CountingTask) + else: + def factory(loop, coro, **kwargs): + return CountingTask(coro, loop=loop, **kwargs) + loop.set_task_factory(factory) + + def get(self): + return self.task_count + + +async def awaitable_chain(depth): + if depth == 0: + return 0 + return 1 + await awaitable_chain(depth - 1) + + +async def recursive_taskgroups(width, depth): + if depth == 0: + return + + async with asyncio.TaskGroup() as tg: + futures = [ + tg.create_task(recursive_taskgroups(width, depth - 1)) + for _ in range(width) + ] + + +async def recursive_gather(width, depth): + if depth == 0: + return + + await asyncio.gather( + *[recursive_gather(width, depth - 1) for _ in range(width)] + ) + + +class BaseTaskCountingTests: + + Task = None + eager = None + expected_task_count = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.counter = AsyncTaskCounter(self.loop, task_class=self.Task, eager=self.eager) + self.set_event_loop(self.loop) + + def test_awaitables_chain(self): + observed_depth = self.loop.run_until_complete(awaitable_chain(100)) + self.assertEqual(observed_depth, 100) + self.assertEqual(self.counter.get(), 0 if self.eager else 1) + + def test_recursive_taskgroups(self): + num_tasks = self.loop.run_until_complete(recursive_taskgroups(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + def test_recursive_gather(self): + self.loop.run_until_complete(recursive_gather(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + +class BaseNonEagerTaskFactoryTests(BaseTaskCountingTests): + eager = False + expected_task_count = 781 # 1 + 5 + 5^2 + 5^3 + 5^4 + + +class BaseEagerTaskFactoryTests(BaseTaskCountingTests): + eager = True + expected_task_count = 0 + + +class NonEagerTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.tasks._CTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + +class EagerTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.tasks._CTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class NonEagerPyTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class EagerPyTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class NonEagerCTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class EagerCTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class DefaultTaskFactoryEagerStart(test_utils.TestCase): + def test_eager_start_true_with_default_factory(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=True, name="example" + ) + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py new file mode 100644 index 00000000000..1a06b426f71 --- /dev/null +++ b/Lib/test/test_asyncio/test_events.py @@ -0,0 +1,3180 @@ +"""Tests for events.py.""" + +import concurrent.futures +import contextlib +import functools +import io +import multiprocessing +import os +import platform +import re +import signal +import socket +try: + import ssl +except ImportError: + ssl = None +import subprocess +import sys +import threading +import time +import types +import errno +import unittest +from unittest import mock +import weakref +if sys.platform not in ('win32', 'vxworks'): + import tty + +import asyncio +from asyncio import coroutines +from asyncio import events +from asyncio import selector_events +from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests +from test.test_asyncio import utils as test_utils +from test import support +from test.support import socket_helper +from test.support import threading_helper +from test.support import ALWAYS_EQ, LARGEST, SMALLEST + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def broken_unix_getsockname(): + """Return True if the platform is Mac OS 10.4 or older.""" + if sys.platform.startswith("aix"): + return True + elif sys.platform != 'darwin': + return False + version = platform.mac_ver()[0] + version = tuple(map(int, version.split('.'))) + return version < (10, 5) + + +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + with contextlib.closing(asyncio.new_event_loop()) as loop: + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + +class CoroLike: + def send(self, v): + pass + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + pass + + +class MyBaseProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyProto(MyBaseProto): + def connection_made(self, transport): + super().connection_made(transport) + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + self._assert_state('INITIALIZED') + self.nbytes += len(data) + + def error_received(self, exc): + self._assert_state('INITIALIZED') + + def connection_lost(self, exc): + self._assert_state('INITIALIZED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyReadPipeProto(asyncio.Protocol): + done = None + + def __init__(self, loop=None): + self.state = ['INITIAL'] + self.nbytes = 0 + self.transport = None + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state(['INITIAL']) + self.state.append('CONNECTED') + + def data_received(self, data): + self._assert_state(['INITIAL', 'CONNECTED']) + self.nbytes += len(data) + + def eof_received(self): + self._assert_state(['INITIAL', 'CONNECTED']) + self.state.append('EOF') + + def connection_lost(self, exc): + if 'EOF' not in self.state: + self.state.append('EOF') # It is okay if EOF is missed. + self._assert_state(['INITIAL', 'CONNECTED', 'EOF']) + self.state.append('CLOSED') + if self.done: + self.done.set_result(None) + + +class MyWritePipeProto(asyncio.BaseProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.transport = None + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + + def connection_lost(self, exc): + self._assert_state('CONNECTED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MySubprocessProtocol(asyncio.SubprocessProtocol): + + def __init__(self, loop): + self.state = 'INITIAL' + self.transport = None + self.connected = loop.create_future() + self.completed = loop.create_future() + self.disconnects = {fd: loop.create_future() for fd in range(3)} + self.data = {1: b'', 2: b''} + self.returncode = None + self.got_data = {1: asyncio.Event(), + 2: asyncio.Event()} + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + self.connected.set_result(None) + + def connection_lost(self, exc): + self._assert_state('CONNECTED') + self.state = 'CLOSED' + self.completed.set_result(None) + + def pipe_data_received(self, fd, data): + self._assert_state('CONNECTED') + self.data[fd] += data + self.got_data[fd].set() + + def pipe_connection_lost(self, fd, exc): + self._assert_state('CONNECTED') + if exc: + self.disconnects[fd].set_exception(exc) + else: + self.disconnects[fd].set_result(exc) + + def process_exited(self): + self._assert_state('CONNECTED') + self.returncode = self.transport.get_returncode() + + +class EventLoopTestsMixin: + + def setUp(self): + super().setUp() + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + @unittest.expectedFailure # TODO: RUSTPYTHON; - RuntimeWarning for unawaited coroutine not triggered + def test_run_until_complete_nesting(self): + async def coro1(): + await asyncio.sleep(0) + + async def coro2(): + self.assertTrue(self.loop.is_running()) + self.loop.run_until_complete(coro1()) + + with self.assertWarnsRegex( + RuntimeWarning, + r"coroutine \S+ was never awaited" + ): + self.assertRaises( + RuntimeError, self.loop.run_until_complete, coro2()) + + # Note: because of the default Windows timing granularity of + # 15.6 msec, we use fairly long sleep times here (~100 msec). + + def test_run_until_complete(self): + delay = 0.100 + t0 = self.loop.time() + self.loop.run_until_complete(asyncio.sleep(delay)) + dt = self.loop.time() - t0 + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) + + def test_run_until_complete_stopped(self): + + async def cb(): + self.loop.stop() + await asyncio.sleep(0.1) + task = cb() + self.assertRaises(RuntimeError, + self.loop.run_until_complete, task) + + def test_call_later(self): + results = [] + + def callback(arg): + results.append(arg) + self.loop.stop() + + self.loop.call_later(0.1, callback, 'hello world') + self.loop.run_forever() + self.assertEqual(results, ['hello world']) + + def test_call_soon(self): + results = [] + + def callback(arg1, arg2): + results.append((arg1, arg2)) + self.loop.stop() + + self.loop.call_soon(callback, 'hello', 'world') + self.loop.run_forever() + self.assertEqual(results, [('hello', 'world')]) + + def test_call_soon_threadsafe(self): + results = [] + lock = threading.Lock() + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + def run_in_thread(): + self.loop.call_soon_threadsafe(callback, 'hello') + lock.release() + + lock.acquire() + t = threading.Thread(target=run_in_thread) + t.start() + + with lock: + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello', 'world']) + + def test_call_soon_threadsafe_handle_block_check_cancelled(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it should block checking for cancellation + # until it finishes + self.assertFalse(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_block_cancellation(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it cannot be cancelled from other thread until + # it finishes + handle.cancel() + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_same_thread(self): + results = [] + callback_started = threading.Event() + callback_finished = threading.Event() + + fut = concurrent.futures.Future() + def callback(arg): + callback_started.set() + handle = fut.result() + handle.cancel() + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + fut.set_result(handle) + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback cancels itself from same thread so it has no effect + # it runs to completion + self.assertTrue(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_other_thread(self): + results = [] + ev = threading.Event() + + callback_finished = threading.Event() + def callback(arg): + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + # handle can be cancelled from other thread if not started yet + self.assertIsInstance(handle, events._ThreadSafeHandle) + handle.cancel() + self.assertTrue(handle.cancelled()) + self.assertFalse(callback_finished.is_set()) + ev.set() + self.loop.call_soon_threadsafe(self.loop.stop) + + # block the main loop until the callback is added and cancelled in the + # other thread + self.loop.call_soon(ev.wait) + t = threading.Thread(target=run_in_thread) + t.start() + self.loop.run_forever() + t.join() + self.assertEqual(results, []) + self.assertFalse(callback_finished.is_set()) + + def test_call_soon_threadsafe_same_thread(self): + results = [] + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + self.loop.call_soon_threadsafe(callback, 'hello') + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + self.assertEqual(results, ['hello', 'world']) + + def test_run_in_executor(self): + def run(arg): + return (arg, threading.get_ident()) + f2 = self.loop.run_in_executor(None, run, 'yo') + res, thread_id = self.loop.run_until_complete(f2) + self.assertEqual(res, 'yo') + self.assertNotEqual(thread_id, threading.get_ident()) + + def test_run_in_executor_cancel(self): + called = False + + def patched_call_soon(*args): + nonlocal called + called = True + + def run(): + time.sleep(0.05) + + f2 = self.loop.run_in_executor(None, run) + f2.cancel() + self.loop.run_until_complete( + self.loop.shutdown_default_executor()) + self.loop.close() + self.loop.call_soon = patched_call_soon + self.loop.call_soon_threadsafe = patched_call_soon + time.sleep(0.4) + self.assertFalse(called) + + def test_reader_callback(self): + r, w = socket.socketpair() + r.setblocking(False) + bytes_read = bytearray() + + def reader(): + try: + data = r.recv(1024) + except BlockingIOError: + # Spurious readiness notifications are possible + # at least on Linux -- see man select. + return + if data: + bytes_read.extend(data) + else: + self.assertTrue(self.loop.remove_reader(r.fileno())) + r.close() + + self.loop.add_reader(r.fileno(), reader) + self.loop.call_soon(w.send, b'abc') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 3) + self.loop.call_soon(w.send, b'def') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 6) + self.loop.call_soon(w.close) + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(bytes_read, b'abcdef') + + def test_writer_callback(self): + r, w = socket.socketpair() + w.setblocking(False) + + def writer(data): + w.send(data) + self.loop.stop() + + data = b'x' * 1024 + self.loop.add_writer(w.fileno(), writer, data) + self.loop.run_forever() + + self.assertTrue(self.loop.remove_writer(w.fileno())) + self.assertFalse(self.loop.remove_writer(w.fileno())) + + w.close() + read = r.recv(len(data) * 2) + r.close() + self.assertEqual(read, data) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - signal handler implementation differs + @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'No SIGKILL') + def test_add_signal_handler(self): + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + + # Check error behavior first. + self.assertRaises( + TypeError, self.loop.add_signal_handler, 'boom', my_handler) + self.assertRaises( + TypeError, self.loop.remove_signal_handler, 'boom') + self.assertRaises( + ValueError, self.loop.add_signal_handler, signal.NSIG+1, + my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, signal.NSIG+1) + self.assertRaises( + ValueError, self.loop.add_signal_handler, 0, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, 0) + self.assertRaises( + ValueError, self.loop.add_signal_handler, -1, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, -1) + self.assertRaises( + RuntimeError, self.loop.add_signal_handler, signal.SIGKILL, + my_handler) + # Removing SIGKILL doesn't raise, since we don't call signal(). + self.assertFalse(self.loop.remove_signal_handler(signal.SIGKILL)) + # Now set a handler and handle it. + self.loop.add_signal_handler(signal.SIGINT, my_handler) + + os.kill(os.getpid(), signal.SIGINT) + test_utils.run_until(self.loop, lambda: caught) + + # Removing it should restore the default handler. + self.assertTrue(self.loop.remove_signal_handler(signal.SIGINT)) + self.assertEqual(signal.getsignal(signal.SIGINT), + signal.default_int_handler) + # Removing again returns False. + self.assertFalse(self.loop.remove_signal_handler(signal.SIGINT)) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + @unittest.skipUnless(hasattr(signal, 'setitimer'), + 'need signal.setitimer()') + def test_signal_handling_while_selecting(self): + # Test with a signal actually arriving during a select() call. + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + self.loop.stop() + + self.loop.add_signal_handler(signal.SIGALRM, my_handler) + + signal.setitimer(signal.ITIMER_REAL, 0.01, 0) # Send SIGALRM once. + self.loop.call_later(60, self.loop.stop) + self.loop.run_forever() + self.assertEqual(caught, 1) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + @unittest.skipUnless(hasattr(signal, 'setitimer'), + 'need signal.setitimer()') + def test_signal_handling_args(self): + some_args = (42,) + caught = 0 + + def my_handler(*args): + nonlocal caught + caught += 1 + self.assertEqual(args, some_args) + self.loop.stop() + + self.loop.add_signal_handler(signal.SIGALRM, my_handler, *some_args) + + signal.setitimer(signal.ITIMER_REAL, 0.1, 0) # Send SIGALRM once. + self.loop.call_later(60, self.loop.stop) + self.loop.run_forever() + self.assertEqual(caught, 1) + + def _basetest_create_connection(self, connection_fut, check_sockname=True): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertIs(pr.transport, tr) + if check_sockname: + self.assertIsNotNone(tr.get_extra_info('sockname')) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def test_create_connection(self): + with test_utils.run_test_server() as httpd: + conn_fut = self.loop.create_connection( + lambda: MyProto(loop=self.loop), *httpd.address) + self._basetest_create_connection(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not broken_unix_getsockname() + + with test_utils.run_test_unix_server() as httpd: + conn_fut = self.loop.create_unix_connection( + lambda: MyProto(loop=self.loop), httpd.address) + self._basetest_create_connection(conn_fut, check_sockname) + + def check_ssl_extra_info(self, client, check_sockname=True, + peername=None, peercert={}): + if check_sockname: + self.assertIsNotNone(client.get_extra_info('sockname')) + if peername: + self.assertEqual(peername, + client.get_extra_info('peername')) + else: + self.assertIsNotNone(client.get_extra_info('peername')) + self.assertEqual(peercert, + client.get_extra_info('peercert')) + + # test SSL cipher + cipher = client.get_extra_info('cipher') + self.assertIsInstance(cipher, tuple) + self.assertEqual(len(cipher), 3, cipher) + self.assertIsInstance(cipher[0], str) + self.assertIsInstance(cipher[1], str) + self.assertIsInstance(cipher[2], int) + + # test SSL object + sslobj = client.get_extra_info('ssl_object') + self.assertIsNotNone(sslobj) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + self.assertEqual(sslobj.cipher(), + client.get_extra_info('cipher')) + self.assertEqual(sslobj.getpeercert(), + client.get_extra_info('peercert')) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + + def _basetest_create_ssl_connection(self, connection_fut, + check_sockname=True, + peername=None): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertTrue('ssl' in tr.__class__.__name__.lower()) + self.check_ssl_extra_info(tr, check_sockname, peername) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def _test_create_ssl_connection(self, httpd, create_connection, + check_sockname=True, peername=None): + conn_fut = create_connection(ssl=test_utils.dummy_ssl_context()) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + # ssl.Purpose was introduced in Python 3.4 + if hasattr(ssl, 'Purpose'): + def _dummy_ssl_create_context(purpose=ssl.Purpose.SERVER_AUTH, *, + cafile=None, capath=None, + cadata=None): + """ + A ssl.create_default_context() replacement that doesn't enable + cert validation. + """ + self.assertEqual(purpose, ssl.Purpose.SERVER_AUTH) + return test_utils.dummy_ssl_context() + + # With ssl=True, ssl.create_default_context() should be called + with mock.patch('ssl.create_default_context', + side_effect=_dummy_ssl_create_context) as m: + conn_fut = create_connection(ssl=True) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + self.assertEqual(m.call_count, 1) + + # With the real ssl.create_default_context(), certificate + # validation will fail + with self.assertRaises(ssl.SSLError) as cm: + conn_fut = create_connection(ssl=True) + # Ignore the "SSL handshake failed" log in debug mode + with test_utils.disable_logger(): + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + self.assertEqual(cm.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_ssl_connection(self): + with test_utils.run_test_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_connection, + lambda: MyProto(loop=self.loop), + *httpd.address) + self._test_create_ssl_connection(httpd, create_connection, + peername=httpd.address) + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_ssl_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not broken_unix_getsockname() + + with test_utils.run_test_unix_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_unix_connection, + lambda: MyProto(loop=self.loop), httpd.address, + server_hostname='127.0.0.1') + + self._test_create_ssl_connection(httpd, create_connection, + check_sockname, + peername=httpd.address) + + def test_create_connection_local_addr(self): + with test_utils.run_test_server() as httpd: + port = socket_helper.find_unused_port() + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=(httpd.address[0], port)) + tr, pr = self.loop.run_until_complete(f) + expected = pr.transport.get_extra_info('sockname')[1] + self.assertEqual(port, expected) + tr.close() + + @socket_helper.skip_if_tcp_blackhole + def test_create_connection_local_addr_skip_different_family(self): + # See https://github.com/python/cpython/issues/86508 + port1 = socket_helper.find_unused_port() + port2 = socket_helper.find_unused_port() + getaddrinfo_orig = self.loop.getaddrinfo + + async def getaddrinfo(host, port, *args, **kwargs): + if port == port2: + return [(socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 0, 0, 0)), + (socket.AF_INET, socket.SOCK_STREAM, 0, '', ('127.0.0.1', 0))] + return await getaddrinfo_orig(host, port, *args, **kwargs) + + self.loop.getaddrinfo = getaddrinfo + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + 'localhost', port1, local_addr=('localhost', port2)) + + with self.assertRaises(OSError): + self.loop.run_until_complete(f) + + @socket_helper.skip_if_tcp_blackhole + def test_create_connection_local_addr_nomatch_family(self): + # See https://github.com/python/cpython/issues/86508 + port1 = socket_helper.find_unused_port() + port2 = socket_helper.find_unused_port() + getaddrinfo_orig = self.loop.getaddrinfo + + async def getaddrinfo(host, port, *args, **kwargs): + if port == port2: + return [(socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 0, 0, 0))] + return await getaddrinfo_orig(host, port, *args, **kwargs) + + self.loop.getaddrinfo = getaddrinfo + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + 'localhost', port1, local_addr=('localhost', port2)) + + with self.assertRaises(OSError): + self.loop.run_until_complete(f) + + def test_create_connection_local_addr_in_use(self): + with test_utils.run_test_server() as httpd: + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=httpd.address) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + self.assertIn(str(httpd.address), cm.exception.strerror) + + def test_connect_accepted_socket(self, server_ssl=None, client_ssl=None): + loop = self.loop + + class MyProto(MyBaseProto): + + def connection_lost(self, exc): + super().connection_lost(exc) + loop.call_soon(loop.stop) + + def data_received(self, data): + super().data_received(data) + self.transport.write(expected_response) + + lsock = socket.create_server(('127.0.0.1', 0), backlog=1) + addr = lsock.getsockname() + + message = b'test data' + response = None + expected_response = b'roger' + + def client(): + nonlocal response + try: + csock = socket.socket() + if client_ssl is not None: + csock = client_ssl.wrap_socket(csock) + csock.connect(addr) + csock.sendall(message) + response = csock.recv(99) + csock.close() + except Exception as exc: + print( + "Failure in client thread in test_connect_accepted_socket", + exc) + + thread = threading.Thread(target=client, daemon=True) + thread.start() + + conn, _ = lsock.accept() + proto = MyProto(loop=loop) + proto.loop = loop + loop.run_until_complete( + loop.connect_accepted_socket( + (lambda: proto), conn, ssl=server_ssl)) + loop.run_forever() + proto.transport.close() + lsock.close() + + threading_helper.join_thread(thread) + self.assertFalse(thread.is_alive()) + self.assertEqual(proto.state, 'CLOSED') + self.assertEqual(proto.nbytes, len(message)) + self.assertEqual(response, expected_response) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_ssl_connect_accepted_socket(self): + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + self.test_connect_accepted_socket(server_context, client_context) + + def test_connect_accepted_socket_ssl_timeout_for_plain_socket(self): + sock = socket.socket() + self.addCleanup(sock.close) + coro = self.loop.connect_accepted_socket( + MyProto, sock, ssl_handshake_timeout=support.LOOPBACK_TIMEOUT) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + @mock.patch('asyncio.base_events.socket') + def create_server_multiple_hosts(self, family, hosts, mock_sock): + async def getaddrinfo(host, port, *args, **kw): + if family == socket.AF_INET: + return [(family, socket.SOCK_STREAM, 6, '', (host, port))] + else: + return [(family, socket.SOCK_STREAM, 6, '', (host, port, 0, 0))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + unique_hosts = set(hosts) + + if family == socket.AF_INET: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80) for host in unique_hosts] + else: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80, 0, 0) for host in unique_hosts] + self.loop.getaddrinfo = getaddrinfo_task + self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() + f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) + server = self.loop.run_until_complete(f) + self.addCleanup(server.close) + server_hosts = {sock.getsockbyname()[0] for sock in server.sockets} + self.assertEqual(server_hosts, unique_hosts) + + def test_create_server_multiple_hosts_ipv4(self): + self.create_server_multiple_hosts(socket.AF_INET, + ['1.2.3.4', '5.6.7.8', '1.2.3.4']) + + def test_create_server_multiple_hosts_ipv6(self): + self.create_server_multiple_hosts(socket.AF_INET6, + ['::1', '::2', '::1']) + + def test_create_server(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('sockname')) + self.assertEqual('127.0.0.1', + proto.transport.get_extra_info('peername')[0]) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + def test_create_server_trsock(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertIsInstance(sock, asyncio.trsock.TransportSocket) + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + dup = sock.dup() + self.addCleanup(dup.close) + self.assertIsInstance(dup, socket.socket) + self.assertFalse(sock.get_inheritable()) + with self.assertRaises(ValueError): + sock.settimeout(1) + sock.settimeout(0) + self.assertEqual(sock.gettimeout(), 0) + with self.assertRaises(ValueError): + sock.setblocking(True) + sock.setblocking(False) + server.close() + + + @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT') + def test_create_server_reuse_port(self): + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + test_utils.run_briefly(self.loop) + + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0, reuse_port=True) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + def _make_unix_server(self, factory, **kwargs): + path = test_utils.gen_unix_socket_path() + self.addCleanup(lambda: os.path.exists(path) and os.unlink(path)) + + f = self.loop.create_unix_server(factory, path, **kwargs) + server = self.loop.run_until_complete(f) + + return server, path + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server(self): + proto = MyProto(loop=self.loop) + server, path = self._make_unix_server(lambda: proto) + self.assertEqual(len(server.sockets), 1) + + client = socket.socket(socket.AF_UNIX) + client.connect(path) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_path_socket_error(self): + proto = MyProto(loop=self.loop) + sock = socket.socket() + with sock: + f = self.loop.create_unix_server(lambda: proto, '/test', sock=sock) + with self.assertRaisesRegex(ValueError, + 'path and sock can not be specified ' + 'at the same time'): + self.loop.run_until_complete(f) + + def _create_ssl_context(self, certfile, keyfile=None): + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.load_cert_chain(certfile, keyfile) + return sslcontext + + def _make_ssl_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + + f = self.loop.create_server(factory, '127.0.0.1', 0, ssl=sslcontext) + server = self.loop.run_until_complete(f) + + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '127.0.0.1') + return server, host, port + + def _make_ssl_unix_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + return self._make_unix_server(factory, ssl=sslcontext) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) + + f_c = self.loop.create_connection(MyBaseProto, host, port, + ssl=test_utils.dummy_ssl_context()) + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.check_ssl_extra_info(client, peername=(host, port)) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) + + f_c = self.loop.create_unix_connection( + MyBaseProto, path, ssl=test_utils.dummy_ssl_context(), + server_hostname='') + + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + + # no CA loaded + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # no CA loaded + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='invalid') + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_match_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations( + cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection + # transport is None because TLS ALERT aborted the handshake + self.assertIsNone(proto.transport) + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + self.loop.run_until_complete(proto.connected) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - SSL peer certificate format differs + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + self.loop.run_until_complete(proto.connected) + + # extra info is available + self.check_ssl_extra_info(client, peername=(host, port), + peercert=test_utils.PEERCERT) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + def test_create_server_sock(self): + proto = self.loop.create_future() + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + proto.set_result(self) + + sock_ob = socket.create_server(('0.0.0.0', 0)) + + f = self.loop.create_server(TestMyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + self.assertEqual(sock.fileno(), sock_ob.fileno()) + + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + server.close() + + def test_create_server_addr_in_use(self): + sock_ob = socket.create_server(('0.0.0.0', 0)) + + f = self.loop.create_server(MyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + f = self.loop.create_server(MyProto, host=host, port=port) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + + server.close() + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_server_dual_stack(self): + f_proto = self.loop.create_future() + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + f_proto.set_result(self) + + try_count = 0 + while True: + try: + port = socket_helper.find_unused_port() + f = self.loop.create_server(TestMyProto, host=None, port=port) + server = self.loop.run_until_complete(f) + except OSError as ex: + if ex.errno == errno.EADDRINUSE: + try_count += 1 + self.assertGreaterEqual(5, try_count) + continue + else: + raise + else: + break + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + f_proto = self.loop.create_future() + client = socket.socket(socket.AF_INET6) + client.connect(('::1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + server.close() + + @socket_helper.skip_if_tcp_blackhole + def test_server_close(self): + f = self.loop.create_server(MyProto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + + server.close() + + client = socket.socket() + self.assertRaises( + ConnectionRefusedError, client.connect, ('127.0.0.1', port)) + client.close() + + def _test_create_datagram_endpoint(self, local_addr, family): + class TestMyDatagramProto(MyDatagramProto): + def __init__(inner_self): + super().__init__(loop=self.loop) + + def datagram_received(self, data, addr): + super().datagram_received(data, addr) + self.transport.sendto(b'resp:'+data, addr) + + coro = self.loop.create_datagram_endpoint( + TestMyDatagramProto, local_addr=local_addr, family=family) + s_transport, server = self.loop.run_until_complete(coro) + sockname = s_transport.get_extra_info('sockname') + host, port = socket.getnameinfo( + sockname, socket.NI_NUMERICHOST|socket.NI_NUMERICSERV) + + self.assertIsInstance(s_transport, asyncio.Transport) + self.assertIsInstance(server, TestMyDatagramProto) + self.assertEqual('INITIALIZED', server.state) + self.assertIs(server.transport, s_transport) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + remote_addr=(host, port)) + transport, client = self.loop.run_until_complete(coro) + + self.assertIsInstance(transport, asyncio.Transport) + self.assertIsInstance(client, MyDatagramProto) + self.assertEqual('INITIALIZED', client.state) + self.assertIs(client.transport, transport) + + transport.sendto(b'xxx') + test_utils.run_until(self.loop, lambda: server.nbytes) + self.assertEqual(3, server.nbytes) + test_utils.run_until(self.loop, lambda: client.nbytes) + + # received + self.assertEqual(8, client.nbytes) + + # extra info is available + self.assertIsNotNone(transport.get_extra_info('sockname')) + + # close connection + transport.close() + self.loop.run_until_complete(client.done) + self.assertEqual('CLOSED', client.state) + server.transport.close() + + def test_create_datagram_endpoint(self): + self._test_create_datagram_endpoint(('127.0.0.1', 0), socket.AF_INET) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_ipv6(self): + self._test_create_datagram_endpoint(('::1', 0), socket.AF_INET6) + + def test_create_datagram_endpoint_sock(self): + sock = None + local_address = ('127.0.0.1', 0) + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *local_address, type=socket.SOCK_DGRAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + sock.bind(address) + except: + pass + else: + break + else: + self.fail('Can not create socket.') + + f = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, MyDatagramProto) + tr.close() + self.loop.run_until_complete(pr.done) + + def test_datagram_send_to_non_listening_address(self): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages after + # sending a message to an address that wasn't listening. + loop = self.loop + + class Protocol(asyncio.DatagramProtocol): + + _received_datagram = None + + def datagram_received(self, data, addr): + self._received_datagram.set_result(data) + + async def wait_for_datagram_received(self): + self._received_datagram = loop.create_future() + result = await asyncio.wait_for(self._received_datagram, 10) + self._received_datagram = None + return result + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + transport_1, protocol_1 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_1) + ) + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + transport_2, protocol_2 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_2) + ) + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address that + # is not listening + socket_3 = create_socket() + transport_3, protocol_3 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_3) + ) + addr_3 = socket_3.getsockname() + transport_3.abort() + + transport_1.sendto(b'a', addr=addr_2) + self.assertEqual(loop.run_until_complete( + protocol_2.wait_for_datagram_received() + ), b'a') + + transport_2.sendto(b'b', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'b') + + # this should send to an address that isn't listening + transport_1.sendto(b'c', addr=addr_3) + loop.run_until_complete(asyncio.sleep(0)) + + # transport 1 should still be able to receive messages after sending to + # an address that wasn't listening + transport_2.sendto(b'd', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'd') + + transport_1.close() + transport_2.close() + + def test_internal_fds(self): + loop = self.create_event_loop() + if not isinstance(loop, selector_events.BaseSelectorEventLoop): + loop.close() + self.skipTest('loop is not a BaseSelectorEventLoop') + + self.assertEqual(1, loop._internal_fds) + loop.close() + self.assertEqual(0, loop._internal_fds) + self.assertIsNone(loop._csock) + self.assertIsNone(loop._ssock) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_read_pipe(self): + proto = MyReadPipeProto(loop=self.loop) + + rpipe, wpipe = os.pipe() + pipeobj = io.open(rpipe, 'rb', 1024) + + async def connect(): + t, p = await self.loop.connect_read_pipe( + lambda: proto, pipeobj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(wpipe, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 1) + self.assertEqual(1, proto.nbytes) + + os.write(wpipe, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(wpipe) + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_unclosed_pipe_transport(self): + # This test reproduces the issue #314 on GitHub + loop = self.create_event_loop() + read_proto = MyReadPipeProto(loop=loop) + write_proto = MyWritePipeProto(loop=loop) + + rpipe, wpipe = os.pipe() + rpipeobj = io.open(rpipe, 'rb', 1024) + wpipeobj = io.open(wpipe, 'w', 1024, encoding="utf-8") + + async def connect(): + read_transport, _ = await loop.connect_read_pipe( + lambda: read_proto, rpipeobj) + write_transport, _ = await loop.connect_write_pipe( + lambda: write_proto, wpipeobj) + return read_transport, write_transport + + # Run and close the loop without closing the transports + read_transport, write_transport = loop.run_until_complete(connect()) + loop.close() + + # These 'repr' calls used to raise an AttributeError + # See Issue #314 on GitHub + self.assertIn('open', repr(read_transport)) + self.assertIn('open', repr(write_transport)) + + # Clean up (avoid ResourceWarning) + rpipeobj.close() + wpipeobj.close() + read_transport._pipe = None + write_transport._pipe = None + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + def test_read_pty_output(self): + proto = MyReadPipeProto(loop=self.loop) + + master, slave = os.openpty() + master_read_obj = io.open(master, 'rb', 0) + + async def connect(): + t, p = await self.loop.connect_read_pipe(lambda: proto, + master_read_obj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(slave, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes) + self.assertEqual(1, proto.nbytes) + + os.write(slave, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(slave) + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe(self): + rpipe, wpipe = os.pipe() + pipeobj = io.open(wpipe, 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(rpipe, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(rpipe) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe_disconnect_on_close(self): + rsock, wsock = socket.socketpair() + rsock.setblocking(False) + pipeobj = io.open(wsock.detach(), 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + data = self.loop.run_until_complete(self.loop.sock_recv(rsock, 1024)) + self.assertEqual(b'1', data) + + rsock.close() + + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_write_pty(self): + master, slave = os.openpty() + slave_write_obj = io.open(slave, 'wb', 0) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, slave_write_obj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(master) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_bidirectional_pty(self): + master, read_slave = os.openpty() + write_slave = os.dup(read_slave) + tty.setraw(read_slave) + + slave_read_obj = io.open(read_slave, 'rb', 0) + read_proto = MyReadPipeProto(loop=self.loop) + read_connect = self.loop.connect_read_pipe(lambda: read_proto, + slave_read_obj) + read_transport, p = self.loop.run_until_complete(read_connect) + self.assertIs(p, read_proto) + self.assertIs(read_transport, read_proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(0, read_proto.nbytes) + + + slave_write_obj = io.open(write_slave, 'wb', 0) + write_proto = MyWritePipeProto(loop=self.loop) + write_connect = self.loop.connect_write_pipe(lambda: write_proto, + slave_write_obj) + write_transport, p = self.loop.run_until_complete(write_connect) + self.assertIs(p, write_proto) + self.assertIs(write_transport, write_proto.transport) + self.assertEqual('CONNECTED', write_proto.state) + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + write_transport.write(b'1') + test_utils.run_until(self.loop, lambda: reader(data) >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'1', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'a') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(1, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + write_transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'12345', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'bcde') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(5, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + os.close(master) + + read_transport.close() + self.loop.run_until_complete(read_proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], read_proto.state) + + write_transport.close() + self.loop.run_until_complete(write_proto.done) + self.assertEqual('CLOSED', write_proto.state) + + def test_prompt_cancellation(self): + r, w = socket.socketpair() + r.setblocking(False) + f = self.loop.create_task(self.loop.sock_recv(r, 1)) + ov = getattr(f, 'ov', None) + if ov is not None: + self.assertTrue(ov.pending) + + async def main(): + try: + self.loop.call_soon(f.cancel) + await f + except asyncio.CancelledError: + res = 'cancelled' + else: + res = None + finally: + self.loop.stop() + return res + + t = self.loop.create_task(main()) + self.loop.run_forever() + + self.assertEqual(t.result(), 'cancelled') + self.assertRaises(asyncio.CancelledError, f.result) + if ov is not None: + self.assertFalse(ov.pending) + self.loop._stop_serving(r) + + r.close() + w.close() + + def test_timeout_rounding(self): + def _run_once(): + self.loop._run_once_counter += 1 + orig_run_once() + + orig_run_once = self.loop._run_once + self.loop._run_once_counter = 0 + self.loop._run_once = _run_once + + async def wait(): + await asyncio.sleep(1e-2) + await asyncio.sleep(1e-4) + await asyncio.sleep(1e-6) + await asyncio.sleep(1e-8) + await asyncio.sleep(1e-10) + + self.loop.run_until_complete(wait()) + # The ideal number of call is 12, but on some platforms, the selector + # may sleep at little bit less than timeout depending on the resolution + # of the clock used by the kernel. Tolerate a few useless calls on + # these platforms. + self.assertLessEqual(self.loop._run_once_counter, 20, + {'clock_resolution': self.loop._clock_resolution, + 'selector': self.loop._selector.__class__.__name__}) + + def test_remove_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = socket.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.add_reader(r, callback) + loop.add_writer(w, callback) + loop.close() + self.assertFalse(loop.remove_reader(r)) + self.assertFalse(loop.remove_writer(w)) + + def test_add_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = socket.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.close() + with self.assertRaises(RuntimeError): + loop.add_reader(r, callback) + with self.assertRaises(RuntimeError): + loop.add_writer(w, callback) + + def test_close_running_event_loop(self): + async def close_loop(loop): + self.loop.close() + + coro = close_loop(self.loop) + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(coro) + + def test_close(self): + self.loop.close() + + async def test(): + pass + + func = lambda: False + coro = test() + self.addCleanup(coro.close) + + # operation blocked when the loop is closed + with self.assertRaises(RuntimeError): + self.loop.run_forever() + with self.assertRaises(RuntimeError): + fut = self.loop.create_future() + self.loop.run_until_complete(fut) + with self.assertRaises(RuntimeError): + self.loop.call_soon(func) + with self.assertRaises(RuntimeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(RuntimeError): + self.loop.call_later(1.0, func) + with self.assertRaises(RuntimeError): + self.loop.call_at(self.loop.time() + .0, func) + with self.assertRaises(RuntimeError): + self.loop.create_task(coro) + with self.assertRaises(RuntimeError): + self.loop.add_signal_handler(signal.SIGTERM, func) + + # run_in_executor test is tricky: the method is a coroutine, + # but run_until_complete cannot be called on closed loop. + # Thus iterate once explicitly. + with self.assertRaises(RuntimeError): + it = self.loop.run_in_executor(None, func).__await__() + next(it) + + +class SubprocessTestsMixin: + + def check_terminated(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + def check_killed(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + @support.requires_subprocess() + def test_subprocess_exec(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + self.assertEqual(b'Python The Winner', proto.data[1]) + + @support.requires_subprocess() + def test_subprocess_interactive(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python ') + self.loop.run_until_complete(proto.got_data[1].wait()) + proto.got_data[1].clear() + self.assertEqual(b'Python ', proto.data[1]) + + stdin.write(b'The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'Python The Winner', proto.data[1]) + + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + @support.requires_subprocess() + def test_subprocess_shell(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'echo Python') + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.get_pipe_transport(0).close() + self.loop.run_until_complete(proto.completed) + self.assertEqual(0, proto.returncode) + self.assertTrue(all(f.done() for f in proto.disconnects.values())) + self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python') + self.assertEqual(proto.data[2], b'') + transp.close() + + @support.requires_subprocess() + def test_subprocess_exitcode(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_close_after_finish(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.assertIsNone(transp.get_pipe_transport(0)) + self.assertIsNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + self.assertIsNone(transp.close()) + + @support.requires_subprocess() + def test_subprocess_kill(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.kill() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_terminate(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.terminate() + self.loop.run_until_complete(proto.completed) + self.check_terminated(proto.returncode) + transp.close() + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() + def test_subprocess_send_signal(self): + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.send_signal(signal.SIGHUP) + self.loop.run_until_complete(proto.completed) + self.assertEqual(-signal.SIGHUP, proto.returncode) + transp.close() + finally: + signal.signal(signal.SIGHUP, old_handler) + + @support.requires_subprocess() + def test_subprocess_stderr(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'test') + + self.loop.run_until_complete(proto.completed) + + transp.close() + self.assertEqual(b'OUT:test', proto.data[1]) + self.assertStartsWith(proto.data[2], b'ERR:test') + self.assertEqual(0, proto.returncode) + + @support.requires_subprocess() + def test_subprocess_stderr_redirect_to_stdout(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog, stderr=subprocess.STDOUT) + + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + self.assertIsNotNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + + stdin.write(b'test') + self.loop.run_until_complete(proto.completed) + self.assertStartsWith(proto.data[1], b'OUT:testERR:test') + self.assertEqual(b'', proto.data[2]) + + transp.close() + self.assertEqual(0, proto.returncode) + + @support.requires_subprocess() + def test_subprocess_close_client_stream(self): + prog = os.path.join(os.path.dirname(__file__), 'echo3.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdout = transp.get_pipe_transport(1) + stdin.write(b'test') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'OUT:test', proto.data[1]) + + stdout.close() + self.loop.run_until_complete(proto.disconnects[1]) + stdin.write(b'xxx') + self.loop.run_until_complete(proto.got_data[2].wait()) + if sys.platform != 'win32': + self.assertEqual(b'ERR:BrokenPipeError', proto.data[2]) + else: + # After closing the read-end of a pipe, writing to the + # write-end using os.write() fails with errno==EINVAL and + # GetLastError()==ERROR_INVALID_NAME on Windows!?! (Using + # WriteFile() we get ERROR_BROKEN_PIPE as expected.) + self.assertEqual(b'ERR:OSError', proto.data[2]) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + @support.requires_subprocess() + def test_subprocess_wait_no_same_group(self): + # start the new process in a new session + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None, + start_new_session=True) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_exec_invalid_args(self): + async def connect(**kwds): + await self.loop.subprocess_exec( + asyncio.SubprocessProtocol, + 'pwd', **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=True)) + + @support.requires_subprocess() + def test_subprocess_shell_invalid_args(self): + + async def connect(cmd=None, **kwds): + if not cmd: + cmd = 'pwd' + await self.loop.subprocess_shell( + asyncio.SubprocessProtocol, + cmd, **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(['ls', '-l'])) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=False)) + + +if sys.platform == 'win32': + + class SelectEventLoopTests(EventLoopTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + class ProactorEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + + def test_reader_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_reader_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_writer_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_writer_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_remove_fds_after_closing(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + # Issue #20667: KqueueEventLoopTests.test_read_pty_output() + # hangs on OpenBSD 5.5 + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'test hangs on OpenBSD') + def test_read_pty_output(self): + super().test_read_pty_output() + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + def test_write_pty(self): + super().test_write_pty() + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +def noop(*args, **kwargs): + pass + + +class HandleTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + self.loop.get_debug.return_value = True + + def test_handle(self): + def callback(*args): + return args + + args = () + h = asyncio.Handle(callback, args, self.loop) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h.cancelled()) + + h.cancel() + self.assertTrue(h.cancelled()) + + def test_callback_with_exception(self): + def callback(): + raise ValueError() + + self.loop = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + h = asyncio.Handle(callback, (), self.loop) + h._run() + + self.loop.call_exception_handler.assert_called_with({ + 'message': test_utils.MockPattern('Exception in callback.*'), + 'exception': mock.ANY, + 'handle': h, + 'source_traceback': h._source_traceback, + }) + + def test_handle_weakref(self): + wd = weakref.WeakValueDictionary() + h = asyncio.Handle(lambda: None, (), self.loop) + wd['h'] = h # Would fail without __weakref__ slot. + + def test_handle_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + # decorated function + cb = types.coroutine(noop) + h = asyncio.Handle(cb, (), self.loop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # partial function + cb = functools.partial(noop, 1, 2) + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial method + method = HandleTests.test_handle_repr + cb = functools.partialmethod(method) + filename, lineno = test_utils.get_function_source(method) + h = asyncio.Handle(cb, (), self.loop) + + cb_regex = r'' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' + regex = fr'^$' + self.assertRegex(repr(h), regex) + + def test_handle_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # double cancellation won't overwrite _repr + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + def test_handle_source_traceback(self): + loop = asyncio.new_event_loop() + loop.set_debug(True) + self.set_event_loop(loop) + + def check_source_traceback(h): + lineno = sys._getframe(1).f_lineno - 1 + self.assertIsInstance(h._source_traceback, list) + self.assertEqual(h._source_traceback[-1][:3], + (__file__, + lineno, + 'test_handle_source_traceback')) + + # call_soon + h = loop.call_soon(noop) + check_source_traceback(h) + + # call_soon_threadsafe + h = loop.call_soon_threadsafe(noop) + check_source_traceback(h) + + # call_later + h = loop.call_later(0, noop) + check_source_traceback(h) + + # call_at + h = loop.call_later(0, noop) + check_source_traceback(h) + + def test_coroutine_like_object_debug_formatting(self): + # Test that asyncio can format coroutines that are instances of + # collections.abc.Coroutine, but lack cr_core or gi_code attributes + # (such as ones compiled with Cython). + + coro = CoroLike() + coro.__name__ = 'AAA' + self.assertTrue(asyncio.iscoroutine(coro)) + self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') + + coro.__qualname__ = 'BBB' + self.assertEqual(coroutines._format_coroutine(coro), 'BBB()') + + coro.cr_running = True + self.assertEqual(coroutines._format_coroutine(coro), 'BBB() running') + + coro.__name__ = coro.__qualname__ = None + self.assertEqual(coroutines._format_coroutine(coro), + '() running') + + coro = CoroLike() + coro.__qualname__ = 'CoroLike' + # Some coroutines might not have '__name__', such as + # built-in async_gen.asend(). + self.assertEqual(coroutines._format_coroutine(coro), 'CoroLike()') + + coro = CoroLike() + coro.__qualname__ = 'AAA' + coro.cr_code = None + self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') + + +class TimerTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + + def test_hash(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(hash(h), hash(when)) + + def test_when(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(when, h.when()) + + def test_timer(self): + def callback(*args): + return args + + args = (1, 2, 3) + when = time.monotonic() + h = asyncio.TimerHandle(when, callback, args, mock.Mock()) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h.cancelled()) + + # cancel + h.cancel() + self.assertTrue(h.cancelled()) + self.assertIsNone(h._callback) + self.assertIsNone(h._args) + + + def test_timer_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.TimerHandle(123, noop, (), self.loop) + src = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' % src) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + def test_timer_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.TimerHandle(123, noop, (), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + + def test_timer_comparison(self): + def callback(*args): + return args + + when = time.monotonic() + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when, callback, (), self.loop) + with self.assertRaises(AssertionError): + self.assertLess(h1, h2) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreater(h2, h1) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, h2) + + self.assertLessEqual(h1, h2) + self.assertLessEqual(h2, h1) + self.assertGreaterEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertEqual(h1, h2) + + h2.cancel() + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + self.assertNotEqual(h1, h2) + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when + 10.0, callback, (), self.loop) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertLessEqual(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, h2) + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + + self.assertLess(h1, h2) + self.assertGreater(h2, h1) + self.assertLessEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertNotEqual(h1, h2) + + h3 = asyncio.Handle(callback, (), self.loop) + self.assertIs(NotImplemented, h1.__eq__(h3)) + self.assertIs(NotImplemented, h1.__ne__(h3)) + + with self.assertRaises(TypeError): + h1 < () + with self.assertRaises(TypeError): + h1 > () + with self.assertRaises(TypeError): + h1 <= () + with self.assertRaises(TypeError): + h1 >= () + with self.assertRaises(AssertionError): + self.assertEqual(h1, ()) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, ALWAYS_EQ) + with self.assertRaises(AssertionError): + self.assertGreater(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertLess(h1, SMALLEST) + with self.assertRaises(AssertionError): + self.assertLessEqual(h1, SMALLEST) + + self.assertNotEqual(h1, ()) + self.assertEqual(h1, ALWAYS_EQ) + self.assertLess(h1, LARGEST) + self.assertLessEqual(h1, LARGEST) + self.assertGreaterEqual(h1, SMALLEST) + self.assertGreater(h1, SMALLEST) + + +class AbstractEventLoopTests(unittest.TestCase): + + def test_not_implemented(self): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + self.assertRaises( + NotImplementedError, loop.run_forever) + self.assertRaises( + NotImplementedError, loop.run_until_complete, None) + self.assertRaises( + NotImplementedError, loop.stop) + self.assertRaises( + NotImplementedError, loop.is_running) + self.assertRaises( + NotImplementedError, loop.is_closed) + self.assertRaises( + NotImplementedError, loop.close) + self.assertRaises( + NotImplementedError, loop.create_task, None) + self.assertRaises( + NotImplementedError, loop.call_later, None, None) + self.assertRaises( + NotImplementedError, loop.call_at, f, f) + self.assertRaises( + NotImplementedError, loop.call_soon, None) + self.assertRaises( + NotImplementedError, loop.time) + self.assertRaises( + NotImplementedError, loop.call_soon_threadsafe, None) + self.assertRaises( + NotImplementedError, loop.set_default_executor, f) + self.assertRaises( + NotImplementedError, loop.add_reader, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_reader, 1) + self.assertRaises( + NotImplementedError, loop.add_writer, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_writer, 1) + self.assertRaises( + NotImplementedError, loop.add_signal_handler, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.set_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.default_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.call_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.get_debug) + self.assertRaises( + NotImplementedError, loop.set_debug, f) + + def test_not_implemented_async(self): + + async def inner(): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + + with self.assertRaises(NotImplementedError): + await loop.run_in_executor(f, f) + with self.assertRaises(NotImplementedError): + await loop.getaddrinfo('localhost', 8080) + with self.assertRaises(NotImplementedError): + await loop.getnameinfo(('localhost', 8080)) + with self.assertRaises(NotImplementedError): + await loop.create_connection(f) + with self.assertRaises(NotImplementedError): + await loop.create_server(f) + with self.assertRaises(NotImplementedError): + await loop.create_datagram_endpoint(f) + with self.assertRaises(NotImplementedError): + await loop.sock_recv(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_recv_into(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_sendall(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_connect(f, f) + with self.assertRaises(NotImplementedError): + await loop.sock_accept(f) + with self.assertRaises(NotImplementedError): + await loop.sock_sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.connect_read_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.connect_write_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.subprocess_shell(f, mock.sentinel) + with self.assertRaises(NotImplementedError): + await loop.subprocess_exec(f) + + loop = asyncio.new_event_loop() + loop.run_until_complete(inner()) + loop.close() + + +class PolicyTests(unittest.TestCase): + + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + + def test_event_loop_policy(self): + policy = asyncio.events._AbstractEventLoopPolicy() + self.assertRaises(NotImplementedError, policy.get_event_loop) + self.assertRaises(NotImplementedError, policy.set_event_loop, object()) + self.assertRaises(NotImplementedError, policy.new_event_loop) + + def test_get_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + self.assertIsNone(policy._local._loop) + + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + self.assertIsNone(policy._local._loop) + + def test_get_event_loop_does_not_call_set_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + + with mock.patch.object( + policy, "set_event_loop", + wraps=policy.set_event_loop) as m_set_event_loop: + + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + + m_set_event_loop.assert_not_called() + + def test_get_event_loop_after_set_none(self): + policy = test_utils.DefaultEventLoopPolicy() + policy.set_event_loop(None) + self.assertRaises(RuntimeError, policy.get_event_loop) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - mock.patch doesn't work correctly with threading.current_thread + @mock.patch('asyncio.events.threading.current_thread') + def test_get_event_loop_thread(self, m_current_thread): + + def f(): + policy = test_utils.DefaultEventLoopPolicy() + self.assertRaises(RuntimeError, policy.get_event_loop) + + th = threading.Thread(target=f) + th.start() + th.join() + + def test_new_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + + loop = policy.new_event_loop() + self.assertIsInstance(loop, asyncio.AbstractEventLoop) + loop.close() + + def test_set_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) + + self.assertRaises(TypeError, policy.set_event_loop, object()) + + loop = policy.new_event_loop() + policy.set_event_loop(loop) + self.assertIs(loop, policy.get_event_loop()) + self.assertIsNot(old_loop, policy.get_event_loop()) + loop.close() + old_loop.close() + + def test_get_event_loop_policy(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio.events._AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) + + def test_set_event_loop_policy(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + self.assertRaises( + TypeError, asyncio.set_event_loop_policy, object()) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + old_policy = asyncio.get_event_loop_policy() + + policy = test_utils.DefaultEventLoopPolicy() + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + asyncio.set_event_loop_policy(policy) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) + + +class GetEventLoopTestsMixin: + + _get_running_loop_impl = None + _set_running_loop_impl = None + get_running_loop_impl = None + get_event_loop_impl = None + + Task = None + Future = None + + def setUp(self): + self._get_running_loop_saved = events._get_running_loop + self._set_running_loop_saved = events._set_running_loop + self.get_running_loop_saved = events.get_running_loop + self.get_event_loop_saved = events.get_event_loop + self._Task_saved = asyncio.Task + self._Future_saved = asyncio.Future + + events._get_running_loop = type(self)._get_running_loop_impl + events._set_running_loop = type(self)._set_running_loop_impl + events.get_running_loop = type(self).get_running_loop_impl + events.get_event_loop = type(self).get_event_loop_impl + + asyncio._get_running_loop = type(self)._get_running_loop_impl + asyncio._set_running_loop = type(self)._set_running_loop_impl + asyncio.get_running_loop = type(self).get_running_loop_impl + asyncio.get_event_loop = type(self).get_event_loop_impl + + asyncio.Task = asyncio.tasks.Task = type(self).Task + asyncio.Future = asyncio.futures.Future = type(self).Future + super().setUp() + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + try: + super().tearDown() + finally: + self.loop.close() + asyncio.set_event_loop(None) + + events._get_running_loop = self._get_running_loop_saved + events._set_running_loop = self._set_running_loop_saved + events.get_running_loop = self.get_running_loop_saved + events.get_event_loop = self.get_event_loop_saved + + asyncio._get_running_loop = self._get_running_loop_saved + asyncio._set_running_loop = self._set_running_loop_saved + asyncio.get_running_loop = self.get_running_loop_saved + asyncio.get_event_loop = self.get_event_loop_saved + + asyncio.Task = asyncio.tasks.Task = self._Task_saved + asyncio.Future = asyncio.futures.Future = self._Future_saved + + if sys.platform != 'win32': + def test_get_event_loop_new_process(self): + # bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.skip_if_broken_multiprocessing_synchronize() + + self.addCleanup(multiprocessing_cleanup_tests) + + async def main(): + if multiprocessing.get_start_method() == 'fork': + # Avoid 'fork' DeprecationWarning. + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + pool = concurrent.futures.ProcessPoolExecutor( + mp_context=mp_context) + result = await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + + def test_get_running_loop_already_running(self): + async def main(): + running_loop = asyncio.get_running_loop() + with contextlib.closing(asyncio.new_event_loop()) as loop: + try: + loop.run_forever() + except RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + + self.assertIs(asyncio.get_running_loop(), running_loop) + + self.loop.run_until_complete(main()) + + + def test_get_event_loop_returns_running_loop(self): + class TestError(Exception): + pass + + class Policy(test_utils.DefaultEventLoopPolicy): + def get_event_loop(self): + raise TestError + + old_policy = asyncio.events._get_event_loop_policy() + try: + asyncio.events._set_event_loop_policy(Policy()) + loop = asyncio.new_event_loop() + + with self.assertRaises(TestError): + asyncio.get_event_loop() + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + self.assertIs(asyncio._get_running_loop(), None) + + async def func(): + self.assertIs(asyncio.get_event_loop(), loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.assertIs(asyncio._get_running_loop(), loop) + + loop.run_until_complete(func()) + + asyncio.set_event_loop(loop) + with self.assertRaises(TestError): + asyncio.get_event_loop() + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + finally: + asyncio.events._set_event_loop_policy(old_policy) + if loop is not None: + loop.close() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + + self.assertIs(asyncio._get_running_loop(), None) + + def test_get_event_loop_returns_running_loop2(self): + old_policy = asyncio.events._get_event_loop_policy() + try: + asyncio.events._set_event_loop_policy(test_utils.DefaultEventLoopPolicy()) + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + async def func(): + self.assertIs(asyncio.get_event_loop(), loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.assertIs(asyncio._get_running_loop(), loop) + + loop.run_until_complete(func()) + + asyncio.set_event_loop(loop) + self.assertIs(asyncio.get_event_loop(), loop) + + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + finally: + asyncio.events._set_event_loop_policy(old_policy) + if loop is not None: + loop.close() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + + self.assertIs(asyncio._get_running_loop(), None) + + +class TestPyGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._py__get_running_loop + _set_running_loop_impl = events._py__set_running_loop + get_running_loop_impl = events._py_get_running_loop + get_event_loop_impl = events._py_get_event_loop + + Task = asyncio.tasks._PyTask + Future = asyncio.futures._PyFuture + +try: + import _asyncio # NoQA +except ImportError: + pass +else: + + class TestCGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._c__get_running_loop + _set_running_loop_impl = events._c__set_running_loop + get_running_loop_impl = events._c_get_running_loop + get_event_loop_impl = events._c_get_event_loop + + Task = asyncio.tasks._CTask + Future = asyncio.futures._CFuture + +class TestServer(unittest.TestCase): + + def test_get_loop(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + proto = MyProto(loop) + server = loop.run_until_complete(loop.create_server(lambda: proto, '0.0.0.0', 0)) + self.assertEqual(server.get_loop(), loop) + server.close() + loop.run_until_complete(server.wait_closed()) + + +class TestAbstractServer(unittest.TestCase): + + def test_close(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().close() + + def test_wait_closed(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + with self.assertRaises(NotImplementedError): + loop.run_until_complete(events.AbstractServer().wait_closed()) + + def test_get_loop(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().get_loop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000..c8de0d24499 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,235 @@ +import asyncio +import threading +import unittest +from threading import Thread +from unittest import TestCase +import weakref +from test import support +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class MyException(Exception): + pass + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class TestFreeThreading: + def test_all_tasks_race(self) -> None: + async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def coro(): + await future + + tasks = set() + + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tasks.add(tg.create_task(coro())) + + all_tasks = asyncio.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in all_tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = asyncio.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) + + threads = [] + + for _ in range(10): + thread = Thread(target=runner) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + def test_all_tasks_different_thread(self) -> None: + loop = None + started = threading.Event() + done = threading.Event() # used for main task not finishing early + async def coro(): + await asyncio.Future() + + lock = threading.Lock() + tasks = set() + + async def main(): + nonlocal tasks, loop + loop = asyncio.get_running_loop() + started.set() + for i in range(1000): + with lock: + asyncio.create_task(coro()) + tasks = asyncio.all_tasks(loop) + done.wait() + + runner = threading.Thread(target=lambda: asyncio.run(main())) + + def check(): + started.wait() + with lock: + self.assertSetEqual(tasks & asyncio.all_tasks(loop), tasks) + + threads = [threading.Thread(target=check) for _ in range(10)] + runner.start() + + with threading_helper.start_threads(threads): + pass + + done.set() + runner.join() + + def test_task_different_thread_finalized(self) -> None: + task = None + async def func(): + nonlocal task + task = asyncio.current_task() + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(func()) + thread = Thread(target=runner) + thread.start() + thread.join() + wr = weakref.ref(task) + del thread + del task + # task finalization in different thread shouldn't crash + support.gc_collect() + self.assertIsNone(wr()) + + def test_run_coroutine_threadsafe(self) -> None: + results = [] + + def in_thread(loop: asyncio.AbstractEventLoop): + coro = asyncio.sleep(0.1, result=42) + fut = asyncio.run_coroutine_threadsafe(coro, loop) + result = fut.result() + self.assertEqual(result, 42) + results.append(result) + + async def main(): + loop = asyncio.get_running_loop() + async with asyncio.TaskGroup() as tg: + for _ in range(10): + tg.create_task(asyncio.to_thread(in_thread, loop)) + self.assertEqual(results, [42] * 10) + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + def test_run_coroutine_threadsafe_exception(self) -> None: + async def coro(): + await asyncio.sleep(0) + raise MyException("test") + + def in_thread(loop: asyncio.AbstractEventLoop): + fut = asyncio.run_coroutine_threadsafe(coro(), loop) + return fut.result() + + async def main(): + loop = asyncio.get_running_loop() + tasks = [] + for _ in range(10): + task = loop.create_task(asyncio.to_thread(in_thread, loop)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, MyException) + self.assertEqual(str(result), "test") + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + +class TestPyFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._PyTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._PyFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.tasks.Future = self._old_Future + return super().tearDown() + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._CTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._CFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.futures.Future = self._old_Future + return super().tearDown() + + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs, eager_start=eager_start) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs, eager_start=eager_start) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py new file mode 100644 index 00000000000..54bf824fef7 --- /dev/null +++ b/Lib/test/test_asyncio/test_futures.py @@ -0,0 +1,1144 @@ +"""Tests for futures.py.""" + +import concurrent.futures +import gc +import re +import sys +import threading +import traceback +import unittest +from unittest import mock +from types import GenericAlias +import asyncio +from asyncio import futures +import warnings +from test.test_asyncio import utils as test_utils +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def _fakefunc(f): + return f + + +def first_cb(): + pass + + +def last_cb(): + pass + + +class ReachableCode(Exception): + """Exception to raise to indicate that some code was reached. + + Use this exception if using mocks is not a good alternative. + """ + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): + """Base class for UAF and other evil stuff requiring an evil event loop.""" + + def get_debug(self): # to suppress tracebacks + return False + + def __del__(self): + # Automatically close the evil event loop to avoid warnings. + if not self.is_closed() and not self.is_running(): + self.close() + + +class DuckFuture: + # Class that does not inherit from Future but aims to be duck-type + # compatible with it. + + _asyncio_future_blocking = False + __cancelled = False + __result = None + __exception = None + + def cancel(self): + if self.done(): + return False + self.__cancelled = True + return True + + def cancelled(self): + return self.__cancelled + + def done(self): + return (self.__cancelled + or self.__result is not None + or self.__exception is not None) + + def result(self): + self.assertFalse(self.cancelled()) + if self.__exception is not None: + raise self.__exception + return self.__result + + def exception(self): + self.assertFalse(self.cancelled()) + return self.__exception + + def set_result(self, result): + self.assertFalse(self.done()) + self.assertIsNotNone(result) + self.__result = result + + def set_exception(self, exception): + self.assertFalse(self.done()) + self.assertIsNotNone(exception) + self.__exception = exception + + def __iter__(self): + if not self.done(): + self._asyncio_future_blocking = True + yield self + self.assertTrue(self.done()) + return self.result() + + +class DuckTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_wrap_future(self): + f = DuckFuture() + g = asyncio.wrap_future(f) + self.assertIs(g, f) + + def test_ensure_future(self): + f = DuckFuture() + g = asyncio.ensure_future(f) + self.assertIs(g, f) + + +class BaseFutureTests: + + def _new_future(self, *args, **kwargs): + return self.cls(*args, **kwargs) + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_generic_alias(self): + future = self.cls[str] + self.assertEqual(future.__args__, (str,)) + self.assertIsInstance(future, GenericAlias) + + def test_isfuture(self): + class MyFuture: + _asyncio_future_blocking = None + + def __init__(self): + self._asyncio_future_blocking = False + + self.assertFalse(asyncio.isfuture(MyFuture)) + self.assertTrue(asyncio.isfuture(MyFuture())) + self.assertFalse(asyncio.isfuture(1)) + + # As `isinstance(Mock(), Future)` returns `False` + self.assertFalse(asyncio.isfuture(mock.Mock())) + + f = self._new_future(loop=self.loop) + self.assertTrue(asyncio.isfuture(f)) + self.assertFalse(asyncio.isfuture(type(f))) + + # As `isinstance(Mock(Future), Future)` returns `True` + self.assertTrue(asyncio.isfuture(mock.Mock(type(f)))) + + f.cancel() + + def test_initial_state(self): + f = self._new_future(loop=self.loop) + self.assertFalse(f.cancelled()) + self.assertFalse(f.done()) + f.cancel() + self.assertTrue(f.cancelled()) + + def test_constructor_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() + + def test_constructor_use_running_loop(self): + async def test(): + return self._new_future() + f = self.loop.run_until_complete(test()) + self.assertIs(f._loop, self.loop) + self.assertIs(f.get_loop(), self.loop) + + def test_constructor_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + f = self._new_future() + self.assertIs(f._loop, self.loop) + self.assertIs(f.get_loop(), self.loop) + + def test_constructor_positional(self): + # Make sure Future doesn't accept a positional argument + self.assertRaises(TypeError, self._new_future, 42) + + def test_uninitialized(self): + # Test that C Future doesn't crash when Future.__init__() + # call was skipped. + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.result) + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.exception) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_result(None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_exception(Exception) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.cancel() + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.add_done_callback(lambda f: None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.remove_done_callback(lambda f: None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + repr(fut) + except (RuntimeError, AttributeError): + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + fut.__await__() + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + iter(fut) + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertFalse(fut.cancelled()) + self.assertFalse(fut.done()) + + def test_future_cancel_message_getter(self): + f = self._new_future(loop=self.loop) + self.assertHasAttr(f, '_cancel_message') + self.assertEqual(f._cancel_message, None) + + f.cancel('my message') + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + self.assertEqual(f._cancel_message, 'my message') + + def test_future_cancel_message_setter(self): + f = self._new_future(loop=self.loop) + f.cancel('my message') + f._cancel_message = 'my new message' + self.assertEqual(f._cancel_message, 'my new message') + + # Also check that the value is used for cancel(). + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + self.assertEqual(f._cancel_message, 'my new message') + + def test_cancel(self): + f = self._new_future(loop=self.loop) + self.assertTrue(f.cancel()) + self.assertTrue(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(asyncio.CancelledError, f.result) + self.assertRaises(asyncio.CancelledError, f.exception) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_result(self): + f = self._new_future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.result) + + f.set_result(42) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 42) + self.assertEqual(f.exception(), None) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_exception(self): + exc = RuntimeError() + f = self._new_future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.exception) + + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + self.assertEqual(f.exception(), exc) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_stop_iteration_exception(self, stop_iteration_class=StopIteration): + exc = stop_iteration_class() + f = self._new_future(loop=self.loop) + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + exc = f.exception() + cause = exc.__cause__ + self.assertIsInstance(exc, RuntimeError) + self.assertRegex(str(exc), 'StopIteration .* cannot be raised') + self.assertIsInstance(cause, stop_iteration_class) + + def test_stop_iteration_subclass_exception(self): + class MyStopIteration(StopIteration): + pass + + self.test_stop_iteration_exception(MyStopIteration) + + def test_exception_class(self): + f = self._new_future(loop=self.loop) + f.set_exception(RuntimeError) + self.assertIsInstance(f.exception(), RuntimeError) + + def test_yield_from_twice(self): + f = self._new_future(loop=self.loop) + + def fixture(): + yield 'A' + x = yield from f + yield 'B', x + y = yield from f + yield 'C', y + + g = fixture() + self.assertEqual(next(g), 'A') # yield 'A'. + self.assertEqual(next(g), f) # First yield from f. + f.set_result(42) + self.assertEqual(next(g), ('B', 42)) # yield 'B', x. + # The second "yield from f" does not yield f. + self.assertEqual(next(g), ('C', 42)) # yield 'C', y. + + def test_future_repr(self): + self.loop.set_debug(True) + f_pending_debug = self._new_future(loop=self.loop) + frame = f_pending_debug._source_traceback[-1] + self.assertEqual( + repr(f_pending_debug), + f'<{self.cls.__name__} pending created at {frame[0]}:{frame[1]}>') + f_pending_debug.cancel() + + self.loop.set_debug(False) + f_pending = self._new_future(loop=self.loop) + self.assertEqual(repr(f_pending), f'<{self.cls.__name__} pending>') + f_pending.cancel() + + f_cancelled = self._new_future(loop=self.loop) + f_cancelled.cancel() + self.assertEqual(repr(f_cancelled), f'<{self.cls.__name__} cancelled>') + + f_result = self._new_future(loop=self.loop) + f_result.set_result(4) + self.assertEqual( + repr(f_result), f'<{self.cls.__name__} finished result=4>') + self.assertEqual(f_result.result(), 4) + + exc = RuntimeError() + f_exception = self._new_future(loop=self.loop) + f_exception.set_exception(exc) + self.assertEqual( + repr(f_exception), + f'<{self.cls.__name__} finished exception=RuntimeError()>') + self.assertIs(f_exception.exception(), exc) + + def func_repr(func): + filename, lineno = test_utils.get_function_source(func) + text = '%s() at %s:%s' % (func.__qualname__, filename, lineno) + return re.escape(text) + + f_one_callbacks = self._new_future(loop=self.loop) + f_one_callbacks.add_done_callback(_fakefunc) + fake_repr = func_repr(_fakefunc) + self.assertRegex( + repr(f_one_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % fake_repr) + f_one_callbacks.cancel() + self.assertEqual(repr(f_one_callbacks), + f'<{self.cls.__name__} cancelled>') + + f_two_callbacks = self._new_future(loop=self.loop) + f_two_callbacks.add_done_callback(first_cb) + f_two_callbacks.add_done_callback(last_cb) + first_repr = func_repr(first_cb) + last_repr = func_repr(last_cb) + self.assertRegex(repr(f_two_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s, %s\]>' + % (first_repr, last_repr)) + + f_many_callbacks = self._new_future(loop=self.loop) + f_many_callbacks.add_done_callback(first_cb) + for i in range(8): + f_many_callbacks.add_done_callback(_fakefunc) + f_many_callbacks.add_done_callback(last_cb) + cb_regex = r'%s, <8 more>, %s' % (first_repr, last_repr) + self.assertRegex( + repr(f_many_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % cb_regex) + f_many_callbacks.cancel() + self.assertEqual(repr(f_many_callbacks), + f'<{self.cls.__name__} cancelled>') + + def test_copy_state(self): + from asyncio.futures import _copy_future_state + + f = self._new_future(loop=self.loop) + f.set_result(10) + + newf = self._new_future(loop=self.loop) + _copy_future_state(f, newf) + self.assertTrue(newf.done()) + self.assertEqual(newf.result(), 10) + + f_exception = self._new_future(loop=self.loop) + f_exception.set_exception(RuntimeError()) + + newf_exception = self._new_future(loop=self.loop) + _copy_future_state(f_exception, newf_exception) + self.assertTrue(newf_exception.done()) + self.assertRaises(RuntimeError, newf_exception.result) + + f_cancelled = self._new_future(loop=self.loop) + f_cancelled.cancel() + + newf_cancelled = self._new_future(loop=self.loop) + _copy_future_state(f_cancelled, newf_cancelled) + self.assertTrue(newf_cancelled.cancelled()) + + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + + def test_iter(self): + fut = self._new_future(loop=self.loop) + + def coro(): + yield from fut + + def test(): + arg1, arg2 = coro() + + with self.assertRaisesRegex(RuntimeError, "await wasn't used"): + test() + fut.cancel() + + def test_log_traceback(self): + fut = self._new_future(loop=self.loop) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + fut._log_traceback = True + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_abandoned(self, m_log): + fut = self._new_future(loop=self.loop) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(Exception()) + fut.cancel() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_unretrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_result(42) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_result(42) + fut.result() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_unretrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + del fut + test_utils.run_briefly(self.loop) + support.gc_collect() + self.assertTrue(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + fut.exception() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_result_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + self.assertRaises(RuntimeError, fut.result) + del fut + self.assertFalse(m_log.error.called) + + def test_wrap_future(self): + + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1, loop=self.loop) + res, ident = self.loop.run_until_complete(f2) + self.assertTrue(asyncio.isfuture(f2)) + self.assertEqual(res, 'oi') + self.assertNotEqual(ident, threading.get_ident()) + ex.shutdown(wait=True) + + def test_wrap_future_future(self): + f1 = self._new_future(loop=self.loop) + f2 = asyncio.wrap_future(f1) + self.assertIs(f1, f2) + + def test_wrap_future_without_loop(self): + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) + ex.shutdown(wait=True) + + def test_wrap_future_use_running_loop(self): + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + async def test(): + return asyncio.wrap_future(f1) + f2 = self.loop.run_until_complete(test()) + self.assertIs(self.loop, f2._loop) + ex.shutdown(wait=True) + + def test_wrap_future_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1) + self.assertIs(self.loop, f2._loop) + ex.shutdown(wait=True) + + def test_wrap_future_cancel(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(f1.cancelled()) + self.assertTrue(f2.cancelled()) + + def test_wrap_future_cancel2(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f1.set_result(42) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertFalse(f1.cancelled()) + self.assertEqual(f1.result(), 42) + self.assertTrue(f2.cancelled()) + + def test_future_source_traceback(self): + self.loop.set_debug(True) + + future = self._new_future(loop=self.loop) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(future._source_traceback, list) + self.assertEqual(future._source_traceback[-2][:3], + (__file__, + lineno, + 'test_future_source_traceback')) + + @mock.patch('asyncio.base_events.logger') + def check_future_exception_never_retrieved(self, debug, m_log): + self.loop.set_debug(debug) + + def memory_error(): + try: + raise MemoryError() + except BaseException as exc: + return exc + exc = memory_error() + + future = self._new_future(loop=self.loop) + future.set_exception(exc) + future = None + test_utils.run_briefly(self.loop) + support.gc_collect() + + regex = f'^{self.cls.__name__} exception was never retrieved\n' + exc_info = (type(exc), exc, exc.__traceback__) + m_log.error.assert_called_once_with(mock.ANY, exc_info=exc_info) + + message = m_log.error.call_args[0][0] + self.assertRegex(message, re.compile(regex, re.DOTALL)) + + def test_future_exception_never_retrieved(self): + self.check_future_exception_never_retrieved(False) + + def test_future_exception_never_retrieved_debug(self): + self.check_future_exception_never_retrieved(True) + + def test_set_result_unless_cancelled(self): + fut = self._new_future(loop=self.loop) + fut.cancel() + futures._set_result_unless_cancelled(fut, 2) + self.assertTrue(fut.cancelled()) + + def test_future_stop_iteration_args(self): + fut = self._new_future(loop=self.loop) + fut.set_result((1, 2)) + fi = fut.__iter__() + result = None + try: + fi.send(None) + except StopIteration as ex: + result = ex.args[0] + else: + self.fail('StopIteration was expected') + self.assertEqual(result, (1, 2)) + + def test_future_iter_throw(self): + fut = self._new_future(loop=self.loop) + fi = iter(fut) + with self.assertWarns(DeprecationWarning): + self.assertRaises(Exception, fi.throw, Exception, Exception("zebra"), None) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + self.assertRaises(TypeError, fi.throw, + Exception, Exception("elephant"), 32) + self.assertRaises(TypeError, fi.throw, + Exception("elephant"), Exception("elephant")) + # https://github.com/python/cpython/issues/101326 + self.assertRaises(ValueError, fi.throw, ValueError, None, None) + self.assertRaises(TypeError, fi.throw, list) + + def test_future_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + for i in range(100): + fut = self._new_future(loop=self.loop) + fut.set_result(Evil()) + + def test_future_cancelled_result_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.result() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + def test_future_cancelled_exception_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.exception() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureTests(BaseFutureTests, test_utils.TestCase): + try: + cls = futures._CFuture + except AttributeError: + cls = None + + def test_future_del_segfault(self): + fut = self._new_future(loop=self.loop) + with self.assertRaises(AttributeError): + del fut._asyncio_future_blocking + with self.assertRaises(AttributeError): + del fut._log_traceback + + def test_callbacks_copy(self): + # See https://github.com/python/cpython/issues/125789 + # In C implementation, the `_callbacks` attribute + # always returns a new list to avoid mutations of internal state + + fut = self._new_future(loop=self.loop) + f1 = lambda _: 1 + f2 = lambda _: 2 + fut.add_done_callback(f1) + fut.add_done_callback(f2) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f1) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f2) + self.assertIsNone(fut._callbacks) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.get_referents not implemented + def test_future_iter_get_referents_segfault(self): + return super().test_future_iter_get_referents_segfault() + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureTests(BaseFutureTests, test_utils.TestCase): + try: + class CSubFuture(futures._CFuture): + pass + + cls = CSubFuture + except AttributeError: + cls = None + + +class PyFutureTests(BaseFutureTests, test_utils.TestCase): + cls = futures._PyFuture + + +class BaseFutureDoneCallbackTests(): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def run_briefly(self): + test_utils.run_briefly(self.loop) + + def _make_callback(self, bag, thing): + # Create a callback function that appends thing to bag. + def bag_appender(future): + bag.append(thing) + return bag_appender + + def _new_future(self): + raise NotImplementedError + + def test_callbacks_remove_first_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [17, 100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_first_and_second_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb2) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_third_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb3) + f.remove_done_callback(cb3) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_invoked_on_set_result(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 42)) + f.add_done_callback(self._make_callback(bag, 17)) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_invoked_on_set_exception(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 100)) + + self.assertEqual(bag, []) + exc = RuntimeError() + f.set_exception(exc) + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.exception(), exc) + + def test_remove_done_callback(self): + bag = [] + f = self._new_future() + cb1 = self._make_callback(bag, 1) + cb2 = self._make_callback(bag, 2) + cb3 = self._make_callback(bag, 3) + + # Add one cb1 and one cb2. + f.add_done_callback(cb1) + f.add_done_callback(cb2) + + # One instance of cb2 removed. Now there's only one cb1. + self.assertEqual(f.remove_done_callback(cb2), 1) + + # Never had any cb3 in there. + self.assertEqual(f.remove_done_callback(cb3), 0) + + # After this there will be 6 instances of cb1 and one of cb2. + f.add_done_callback(cb2) + for i in range(5): + f.add_done_callback(cb1) + + # Remove all instances of cb1. One cb2 remains. + self.assertEqual(f.remove_done_callback(cb1), 6) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [2]) + self.assertEqual(f.result(), 'foo') + + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_remove_done_callbacks_list_clear(self): + # see https://github.com/python/cpython/issues/97592 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(other) + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation_1(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + + def test_schedule_callbacks_list_mutation_2(self): + # see http://bugs.python.org/issue30828 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + max_extra_cbs = 100 + extra_cbs = 0 + + class evil: + def __eq__(self, other): + nonlocal extra_cbs + extra_cbs += 1 + if extra_cbs < max_extra_cbs: + fut.add_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_evil_call_soon_list_mutation(self): + # see: https://github.com/python/cpython/issues/125969 + called_on_fut_callback0 = False + + pad = lambda: ... + + def evil_call_soon(*args, **kwargs): + nonlocal called_on_fut_callback0 + if called_on_fut_callback0: + # Called when handling fut->fut_callbacks[0] + # and mutates the length fut->fut_callbacks. + fut.remove_done_callback(int) + fut.remove_done_callback(pad) + else: + called_on_fut_callback0 = True + + fake_event_loop = SimpleEvilEventLoop() + fake_event_loop.call_soon = evil_call_soon + + with mock.patch.object(self, 'loop', fake_event_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), fake_event_loop) + + fut.add_done_callback(str) # sets fut->fut_callback0 + fut.add_done_callback(int) # sets fut->fut_callbacks[0] + fut.add_done_callback(pad) # sets fut->fut_callbacks[1] + fut.add_done_callback(pad) # sets fut->fut_callbacks[2] + fut.set_result("boom") + + # When there are no more callbacks, the Python implementation + # returns an empty list but the C implementation returns None. + self.assertIn(fut._callbacks, (None, [])) + + def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): + # Special thanks to Nico-Posada for the original PoC. + # See https://github.com/python/cpython/issues/125966. + + fut = self._new_future() + + class cb_pad: + def __eq__(self, other): + return True + + class evil(cb_pad): + def __eq__(self, other): + fut.remove_done_callback(None) + return NotImplemented + + fut.add_done_callback(cb_pad()) + fut.remove_done_callback(evil()) + + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + nonlocal fut_callback_0 + if name == 'call_soon': + fut.remove_done_callback(fut_callback_0) + del fut_callback_0 + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = lambda: ... + fut.add_done_callback(fut_callback_0) + self.assertRaises(ReachableCode, fut.set_result, "boom") + + def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + if name == 'call_soon': + # resets the future's event loop + fut.__init__(loop=SimpleEvilEventLoop()) + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = mock.Mock() + fut_context_0 = mock.Mock() + fut.add_done_callback(fut_callback_0, context=fut_context_0) + del fut_context_0 + del fut_callback_0 + self.assertRaises(ReachableCode, fut.set_result, "boom") + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._CFuture(loop=self.loop) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + class CSubFuture(futures._CFuture): + pass + return CSubFuture(loop=self.loop) + + +class PyFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._PyFuture(loop=self.loop) + + +class BaseFutureInheritanceTests: + + def _get_future_cls(self): + raise NotImplementedError + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_inherit_without_calling_super_init(self): + # See https://bugs.python.org/issue38785 for the context + cls = self._get_future_cls() + + class MyFut(cls): + def __init__(self, *args, **kwargs): + # don't call super().__init__() + pass + + fut = MyFut(loop=self.loop) + with self.assertRaisesRegex( + RuntimeError, + "Future object is not initialized." + ): + fut.get_loop() + + +class PyFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._PyFuture + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._CFuture + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py new file mode 100644 index 00000000000..c7c0ebdac1b --- /dev/null +++ b/Lib/test/test_asyncio/test_futures2.py @@ -0,0 +1,95 @@ +# IsolatedAsyncioTestCase based tests +import asyncio +import contextvars +import traceback +import unittest +from asyncio import tasks + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class FutureTests: + + async def test_future_traceback(self): + + async def raise_exc(): + raise TypeError(42) + + future = self.cls(raise_exc()) + + for _ in range(5): + try: + await future + except TypeError as e: + tb = ''.join(traceback.format_tb(e.__traceback__)) + self.assertEqual(tb.count("await future"), 1) + else: + self.fail('TypeError was not raised') + + async def test_task_exc_handler_correct_context(self): + # see https://github.com/python/cpython/issues/96704 + name = contextvars.ContextVar('name', default='foo') + exc_handler_called = False + + def exc_handler(*args): + self.assertEqual(name.get(), 'bar') + nonlocal exc_handler_called + exc_handler_called = True + + async def task(): + name.set('bar') + 1/0 + + loop = asyncio.get_running_loop() + loop.set_exception_handler(exc_handler) + self.cls(task()) + await asyncio.sleep(0) + self.assertTrue(exc_handler_called) + + async def test_handle_exc_handler_correct_context(self): + # see https://github.com/python/cpython/issues/96704 + name = contextvars.ContextVar('name', default='foo') + exc_handler_called = False + + def exc_handler(*args): + self.assertEqual(name.get(), 'bar') + nonlocal exc_handler_called + exc_handler_called = True + + def callback(): + name.set('bar') + 1/0 + + loop = asyncio.get_running_loop() + loop.set_exception_handler(exc_handler) + loop.call_soon(callback) + await asyncio.sleep(0) + self.assertTrue(exc_handler_called) + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._CTask + +class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._PyTask + +class FutureReprTests(unittest.IsolatedAsyncioTestCase): + + async def test_recursive_repr_for_pending_tasks(self): + # The call crashes if the guard for recursive call + # in base_futures:_future_repr_info is absent + # See Also: https://bugs.python.org/issue42183 + + async def func(): + return asyncio.all_tasks() + + # The repr() call should not raise RecursionError at first. + waiter = await asyncio.wait_for(asyncio.Task(func()),timeout=10) + self.assertIn('...', repr(waiter)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py new file mode 100644 index 00000000000..2f22fbccba4 --- /dev/null +++ b/Lib/test/test_asyncio/test_graph.py @@ -0,0 +1,445 @@ +import asyncio +import io +import unittest + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None, depth=1): + + def walk(s): + ret = [ + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if entry.frame.f_generator is None else + ( + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + buf = io.StringIO() + asyncio.print_call_graph(fut, file=buf, depth=depth+1) + + stack = asyncio.capture_call_graph(fut, depth=depth) + return walk(stack), buf.getvalue() + + +class CallStackTestBase: + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack(depth=2) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5[0], [ + # task name + 'T', + # call stack + ['s c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + self.assertIn( + ' async CallStackTestBase.test_stack_tgroup()', + stack_for_c5[1]) + + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call[0], [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + self.assertIn( + 'async generator CallStackTestBase.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut[0], + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._c_current_task + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._py_current_task + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py new file mode 100644 index 00000000000..e025d2990a3 --- /dev/null +++ b/Lib/test/test_asyncio/test_locks.py @@ -0,0 +1,1825 @@ +"""Tests for locks.py""" + +import unittest +from unittest import mock +import re + +import asyncio +import collections + +STR_RGX_REPR = ( + r'^<(?P.*?) object at (?P
.*?)' + r'\[(?P' + r'(set|unset|locked|unlocked|filling|draining|resetting|broken)' + r'(, value:\d)?' + r'(, waiters:\d+)?' + r'(, waiters:\d+\/\d+)?' # barrier + r')\]>\z' +) +RGX_REPR = re.compile(STR_RGX_REPR) + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class LockTests(unittest.IsolatedAsyncioTestCase): + + async def test_repr(self): + lock = asyncio.Lock() + self.assertEndsWith(repr(lock), '[unlocked]>') + self.assertTrue(RGX_REPR.match(repr(lock))) + + await lock.acquire() + self.assertEndsWith(repr(lock), '[locked]>') + self.assertTrue(RGX_REPR.match(repr(lock))) + + async def test_lock(self): + lock = asyncio.Lock() + + with self.assertRaisesRegex( + TypeError, + "'Lock' object can't be awaited" + ): + await lock + + self.assertFalse(lock.locked()) + + async def test_lock_doesnt_accept_loop_parameter(self): + primitives_cls = [ + asyncio.Lock, + asyncio.Condition, + asyncio.Event, + asyncio.Semaphore, + asyncio.BoundedSemaphore, + ] + + loop = asyncio.get_running_loop() + + for cls in primitives_cls: + with self.assertRaisesRegex( + TypeError, + rf"{cls.__name__}\.__init__\(\) got an unexpected " + rf"keyword argument 'loop'" + ): + cls(loop=loop) + + async def test_lock_by_with_statement(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + for lock in primitives: + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + with self.assertRaisesRegex( + TypeError, + r"'\w+' object can't be awaited" + ): + with await lock: + pass + self.assertFalse(lock.locked()) + + async def test_acquire(self): + lock = asyncio.Lock() + result = [] + + self.assertTrue(await lock.acquire()) + + async def c1(result): + if await lock.acquire(): + result.append(1) + return True + + async def c2(result): + if await lock.acquire(): + result.append(2) + return True + + async def c3(result): + if await lock.acquire(): + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + await asyncio.sleep(0) + self.assertEqual([1], result) + + t3 = asyncio.create_task(c3(result)) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_acquire_cancel(self): + lock = asyncio.Lock() + self.assertTrue(await lock.acquire()) + + task = asyncio.create_task(lock.acquire()) + asyncio.get_running_loop().call_soon(task.cancel) + with self.assertRaises(asyncio.CancelledError): + await task + self.assertFalse(lock._waiters) + + async def test_cancel_race(self): + # Several tasks: + # - A acquires the lock + # - B is blocked in acquire() + # - C is blocked in acquire() + # + # Now, concurrently: + # - B is cancelled + # - A releases the lock + # + # If B's waiter is marked cancelled but not yet removed from + # _waiters, A's release() call will crash when trying to set + # B's waiter; instead, it should move on to C's waiter. + + # Setup: A has the lock, b and c are waiting. + lock = asyncio.Lock() + + async def lockit(name, blocker): + await lock.acquire() + try: + if blocker is not None: + await blocker + finally: + lock.release() + + fa = asyncio.get_running_loop().create_future() + ta = asyncio.create_task(lockit('A', fa)) + await asyncio.sleep(0) + self.assertTrue(lock.locked()) + tb = asyncio.create_task(lockit('B', None)) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 1) + tc = asyncio.create_task(lockit('C', None)) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 2) + + # Create the race and check. + # Without the fix this failed at the last assert. + fa.set_result(None) + tb.cancel() + self.assertTrue(lock._waiters[0].cancelled()) + await asyncio.sleep(0) + self.assertFalse(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + await tc + + async def test_cancel_release_race(self): + # Issue 32734 + # Acquire 4 locks, cancel second, release first + # and 2 locks are taken at once. + loop = asyncio.get_running_loop() + lock = asyncio.Lock() + lock_count = 0 + call_count = 0 + + async def lockit(): + nonlocal lock_count + nonlocal call_count + call_count += 1 + await lock.acquire() + lock_count += 1 + + def trigger(): + t1.cancel() + lock.release() + + await lock.acquire() + + t1 = asyncio.create_task(lockit()) + t2 = asyncio.create_task(lockit()) + t3 = asyncio.create_task(lockit()) + + # Start scheduled tasks + await asyncio.sleep(0) + + loop.call_soon(trigger) + with self.assertRaises(asyncio.CancelledError): + # Wait for cancellation + await t1 + + # Make sure only one lock was taken + self.assertEqual(lock_count, 1) + # While 3 calls were made to lockit() + self.assertEqual(call_count, 3) + self.assertTrue(t1.cancelled() and t2.done()) + + # Cleanup the task that is stuck on acquire. + t3.cancel() + await asyncio.sleep(0) + self.assertTrue(t3.cancelled()) + + async def test_finished_waiter_cancelled(self): + lock = asyncio.Lock() + + await lock.acquire() + self.assertTrue(lock.locked()) + + tb = asyncio.create_task(lock.acquire()) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 1) + + # Create a second waiter, wake up the first, and cancel it. + # Without the fix, the second was not woken up. + tc = asyncio.create_task(lock.acquire()) + tb.cancel() + lock.release() + await asyncio.sleep(0) + + self.assertTrue(lock.locked()) + self.assertTrue(tb.cancelled()) + + # Cleanup + await tc + + async def test_release_not_acquired(self): + lock = asyncio.Lock() + + self.assertRaises(RuntimeError, lock.release) + + async def test_release_no_waiters(self): + lock = asyncio.Lock() + await lock.acquire() + self.assertTrue(lock.locked()) + + lock.release() + self.assertFalse(lock.locked()) + + async def test_context_manager(self): + lock = asyncio.Lock() + self.assertFalse(lock.locked()) + + async with lock: + self.assertTrue(lock.locked()) + + self.assertFalse(lock.locked()) + + +class EventTests(unittest.IsolatedAsyncioTestCase): + + def test_repr(self): + ev = asyncio.Event() + self.assertEndsWith(repr(ev), '[unset]>') + match = RGX_REPR.match(repr(ev)) + self.assertEqual(match.group('extras'), 'unset') + + ev.set() + self.assertEndsWith(repr(ev), '[set]>') + self.assertTrue(RGX_REPR.match(repr(ev))) + + ev._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(ev)) + self.assertTrue(RGX_REPR.match(repr(ev))) + + async def test_wait(self): + ev = asyncio.Event() + self.assertFalse(ev.is_set()) + + result = [] + + async def c1(result): + if await ev.wait(): + result.append(1) + + async def c2(result): + if await ev.wait(): + result.append(2) + + async def c3(result): + if await ev.wait(): + result.append(3) + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + t3 = asyncio.create_task(c3(result)) + + ev.set() + await asyncio.sleep(0) + self.assertEqual([3, 1, 2], result) + + self.assertTrue(t1.done()) + self.assertIsNone(t1.result()) + self.assertTrue(t2.done()) + self.assertIsNone(t2.result()) + self.assertTrue(t3.done()) + self.assertIsNone(t3.result()) + + async def test_wait_on_set(self): + ev = asyncio.Event() + ev.set() + + res = await ev.wait() + self.assertTrue(res) + + async def test_wait_cancel(self): + ev = asyncio.Event() + + wait = asyncio.create_task(ev.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with self.assertRaises(asyncio.CancelledError): + await wait + self.assertFalse(ev._waiters) + + async def test_clear(self): + ev = asyncio.Event() + self.assertFalse(ev.is_set()) + + ev.set() + self.assertTrue(ev.is_set()) + + ev.clear() + self.assertFalse(ev.is_set()) + + async def test_clear_with_waiters(self): + ev = asyncio.Event() + result = [] + + async def c1(result): + if await ev.wait(): + result.append(1) + return True + + t = asyncio.create_task(c1(result)) + await asyncio.sleep(0) + self.assertEqual([], result) + + ev.set() + ev.clear() + self.assertFalse(ev.is_set()) + + ev.set() + ev.set() + self.assertEqual(1, len(ev._waiters)) + + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertEqual(0, len(ev._waiters)) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + +class ConditionTests(unittest.IsolatedAsyncioTestCase): + + async def test_wait(self): + cond = asyncio.Condition() + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + return True + + async def c3(result): + await cond.acquire() + if await cond.wait(): + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + self.assertFalse(cond.locked()) + + self.assertTrue(await cond.acquire()) + cond.notify() + await asyncio.sleep(0) + self.assertEqual([], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.notify(2) + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + self.assertTrue(cond.locked()) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_wait_cancel(self): + cond = asyncio.Condition() + await cond.acquire() + + wait = asyncio.create_task(cond.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with self.assertRaises(asyncio.CancelledError): + await wait + self.assertFalse(cond._waiters) + self.assertTrue(cond.locked()) + + async def test_wait_cancel_contested(self): + cond = asyncio.Condition() + + await cond.acquire() + self.assertTrue(cond.locked()) + + wait_task = asyncio.create_task(cond.wait()) + await asyncio.sleep(0) + self.assertFalse(cond.locked()) + + # Notify, but contest the lock before cancelling + await cond.acquire() + self.assertTrue(cond.locked()) + cond.notify() + asyncio.get_running_loop().call_soon(wait_task.cancel) + asyncio.get_running_loop().call_soon(cond.release) + + try: + await wait_task + except asyncio.CancelledError: + # Should not happen, since no cancellation points + pass + + self.assertTrue(cond.locked()) + + async def test_wait_cancel_after_notify(self): + # See bpo-32841 + waited = False + + cond = asyncio.Condition() + + async def wait_on_cond(): + nonlocal waited + async with cond: + waited = True # Make sure this area was reached + await cond.wait() + + waiter = asyncio.create_task(wait_on_cond()) + await asyncio.sleep(0) # Start waiting + + await cond.acquire() + cond.notify() + await asyncio.sleep(0) # Get to acquire() + waiter.cancel() + await asyncio.sleep(0) # Activate cancellation + cond.release() + await asyncio.sleep(0) # Cancellation should occur + + self.assertTrue(waiter.cancelled()) + self.assertTrue(waited) + + async def test_wait_unacquired(self): + cond = asyncio.Condition() + with self.assertRaises(RuntimeError): + await cond.wait() + + async def test_wait_for(self): + cond = asyncio.Condition() + presult = False + + def predicate(): + return presult + + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait_for(predicate): + result.append(1) + cond.release() + return True + + t = asyncio.create_task(c1(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify() + cond.release() + await asyncio.sleep(0) + self.assertEqual([], result) + + presult = True + await cond.acquire() + cond.notify() + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + async def test_wait_for_unacquired(self): + cond = asyncio.Condition() + + # predicate can return true immediately + res = await cond.wait_for(lambda: [1, 2, 3]) + self.assertEqual([1, 2, 3], res) + + with self.assertRaises(RuntimeError): + await cond.wait_for(lambda: False) + + async def test_notify(self): + cond = asyncio.Condition() + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + cond.release() + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + cond.release() + return True + + async def c3(result): + await cond.acquire() + if await cond.wait(): + result.append(3) + cond.release() + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify(1) + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + await cond.acquire() + cond.notify(1) + cond.notify(2048) + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_notify_all(self): + cond = asyncio.Condition() + + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + cond.release() + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + cond.release() + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify_all() + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + + def test_notify_unacquired(self): + cond = asyncio.Condition() + self.assertRaises(RuntimeError, cond.notify) + + def test_notify_all_unacquired(self): + cond = asyncio.Condition() + self.assertRaises(RuntimeError, cond.notify_all) + + async def test_repr(self): + cond = asyncio.Condition() + self.assertTrue('unlocked' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + await cond.acquire() + self.assertTrue('locked' in repr(cond)) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + async def test_context_manager(self): + cond = asyncio.Condition() + self.assertFalse(cond.locked()) + async with cond: + self.assertTrue(cond.locked()) + self.assertFalse(cond.locked()) + + async def test_explicit_lock(self): + async def f(lock=None, cond=None): + if lock is None: + lock = asyncio.Lock() + if cond is None: + cond = asyncio.Condition(lock) + self.assertIs(cond._lock, lock) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with cond: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with lock: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + + # All should work in the same way. + await f() + await f(asyncio.Lock()) + lock = asyncio.Lock() + await f(lock, asyncio.Condition(lock)) + + async def test_ambiguous_loops(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + async def wrong_loop_in_lock(): + with self.assertRaises(TypeError): + asyncio.Lock(loop=loop) # actively disallowed since 3.10 + lock = asyncio.Lock() + lock._loop = loop # use private API for testing + async with lock: + # acquired immediately via the fast-path + # without interaction with any event loop. + cond = asyncio.Condition(lock) + # cond.acquire() will trigger waiting on the lock + # and it will discover the event loop mismatch. + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.acquire() + + async def wrong_loop_in_cond(): + # Same analogy here with the condition's loop. + lock = asyncio.Lock() + async with lock: + with self.assertRaises(TypeError): + asyncio.Condition(lock, loop=loop) + cond = asyncio.Condition(lock) + cond._loop = loop + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.wait() + + await wrong_loop_in_lock() + await wrong_loop_in_cond() + + async def test_timeout_in_block(self): + condition = asyncio.Condition() + async with condition: + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(condition.wait(), timeout=0.5) + + async def test_cancelled_error_wakeup(self): + # Test that a cancelled error, received when awaiting wakeup, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition, cancel it there. + task.cancel(msg="foo") + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_error_re_aquire(self): + # Test that a cancelled error, received when re-aquiring lock, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition + await cond.acquire() + wake = True + cond.notify() + await asyncio.sleep(0) + # Task is now trying to re-acquire the lock, cancel it there. + task.cancel(msg="foo") + cond.release() + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + +class SemaphoreTests(unittest.IsolatedAsyncioTestCase): + + def test_initial_value_zero(self): + sem = asyncio.Semaphore(0) + self.assertTrue(sem.locked()) + + async def test_repr(self): + sem = asyncio.Semaphore() + self.assertEndsWith(repr(sem), '[unlocked, value:1]>') + self.assertTrue(RGX_REPR.match(repr(sem))) + + await sem.acquire() + self.assertEndsWith(repr(sem), '[locked]>') + self.assertTrue('waiters' not in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + if sem._waiters is None: + sem._waiters = collections.deque() + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + async def test_semaphore(self): + sem = asyncio.Semaphore() + self.assertEqual(1, sem._value) + + with self.assertRaisesRegex( + TypeError, + "'Semaphore' object can't be awaited", + ): + await sem + + self.assertFalse(sem.locked()) + self.assertEqual(1, sem._value) + + def test_semaphore_value(self): + self.assertRaises(ValueError, asyncio.Semaphore, -1) + + async def test_acquire(self): + sem = asyncio.Semaphore(3) + result = [] + + self.assertTrue(await sem.acquire()) + self.assertTrue(await sem.acquire()) + self.assertFalse(sem.locked()) + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + async def c4(result): + await sem.acquire() + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(sem.locked()) + self.assertEqual(2, len(sem._waiters)) + self.assertEqual(0, sem._value) + + t4 = asyncio.create_task(c4(result)) + + sem.release() + sem.release() + self.assertEqual(0, sem._value) + + await asyncio.sleep(0) + self.assertEqual(0, sem._value) + self.assertEqual(3, len(result)) + self.assertTrue(sem.locked()) + self.assertEqual(1, len(sem._waiters)) + self.assertEqual(0, sem._value) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + race_tasks = [t2, t3, t4] + done_tasks = [t for t in race_tasks if t.done() and t.result()] + self.assertEqual(2, len(done_tasks)) + + # cleanup locked semaphore + sem.release() + await asyncio.gather(*race_tasks) + + async def test_acquire_cancel(self): + sem = asyncio.Semaphore() + await sem.acquire() + + acquire = asyncio.create_task(sem.acquire()) + asyncio.get_running_loop().call_soon(acquire.cancel) + with self.assertRaises(asyncio.CancelledError): + await acquire + self.assertTrue((not sem._waiters) or + all(waiter.done() for waiter in sem._waiters)) + + async def test_acquire_cancel_before_awoken(self): + sem = asyncio.Semaphore(value=0) + + t1 = asyncio.create_task(sem.acquire()) + t2 = asyncio.create_task(sem.acquire()) + t3 = asyncio.create_task(sem.acquire()) + t4 = asyncio.create_task(sem.acquire()) + + await asyncio.sleep(0) + + t1.cancel() + t2.cancel() + sem.release() + + await asyncio.sleep(0) + await asyncio.sleep(0) + num_done = sum(t.done() for t in [t3, t4]) + self.assertEqual(num_done, 1) + self.assertTrue(t3.done()) + self.assertFalse(t4.done()) + + t3.cancel() + t4.cancel() + await asyncio.sleep(0) + + async def test_acquire_hang(self): + sem = asyncio.Semaphore(value=0) + + t1 = asyncio.create_task(sem.acquire()) + t2 = asyncio.create_task(sem.acquire()) + await asyncio.sleep(0) + + t1.cancel() + sem.release() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(sem.locked()) + self.assertTrue(t2.done()) + + async def test_acquire_no_hang(self): + + sem = asyncio.Semaphore(1) + + async def c1(): + async with sem: + await asyncio.sleep(0) + t2.cancel() + + async def c2(): + async with sem: + self.assertFalse(True) + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + + r1, r2 = await asyncio.gather(t1, t2, return_exceptions=True) + self.assertTrue(r1 is None) + self.assertTrue(isinstance(r2, asyncio.CancelledError)) + + await asyncio.wait_for(sem.acquire(), timeout=1.0) + + def test_release_not_acquired(self): + sem = asyncio.BoundedSemaphore() + + self.assertRaises(ValueError, sem.release) + + async def test_release_no_waiters(self): + sem = asyncio.Semaphore() + await sem.acquire() + self.assertTrue(sem.locked()) + + sem.release() + self.assertFalse(sem.locked()) + + async def test_acquire_fifo_order(self): + sem = asyncio.Semaphore(1) + result = [] + + async def coro(tag): + await sem.acquire() + result.append(f'{tag}_1') + await asyncio.sleep(0.01) + sem.release() + + await sem.acquire() + result.append(f'{tag}_2') + await asyncio.sleep(0.01) + sem.release() + + async with asyncio.TaskGroup() as tg: + tg.create_task(coro('c1')) + tg.create_task(coro('c2')) + tg.create_task(coro('c3')) + + self.assertEqual( + ['c1_1', 'c2_1', 'c3_1', 'c1_2', 'c2_2', 'c3_2'], + result + ) + + async def test_acquire_fifo_order_2(self): + sem = asyncio.Semaphore(1) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + sem.release() + await sem.acquire() + result.append(4) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) + + async def test_acquire_fifo_order_3(self): + sem = asyncio.Semaphore(0) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + t1.cancel() + + await asyncio.sleep(0) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks, return_exceptions=True) + self.assertEqual([2, 3], result) + + async def test_acquire_fifo_order_4(self): + # Test that a successful `acquire()` will wake up multiple Tasks + # that were waiting in the Semaphore queue due to FIFO rules. + sem = asyncio.Semaphore(0) + result = [] + count = 0 + + async def c1(result): + # First task immediately waits for semaphore. It will be awoken by c2. + self.assertEqual(sem._value, 0) + await sem.acquire() + # We should have woken up all waiting tasks now. + self.assertEqual(sem._value, 0) + # Create a fourth task. It should run after c3, not c2. + nonlocal t4 + t4 = asyncio.create_task(c4(result)) + result.append(1) + return True + + async def c2(result): + # The second task begins by releasing semaphore three times, + # for c1, c2, and c3. + sem.release() + sem.release() + sem.release() + self.assertEqual(sem._value, 2) + # It is locked, because c1 hasn't woken up yet. + self.assertTrue(sem.locked()) + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + self.assertTrue(sem.locked()) + result.append(3) + return True + + async def c4(result): + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + t4 = None + + await asyncio.sleep(0) + # Three tasks are in the queue, the first hasn't woken up yet. + self.assertEqual(sem._value, 2) + self.assertEqual(len(sem._waiters), 3) + await asyncio.sleep(0) + + tasks = [t1, t2, t3, t4] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) + +class BarrierTests(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self): + await super().asyncSetUp() + self.N = 5 + + def make_tasks(self, n, coro): + tasks = [asyncio.create_task(coro()) for _ in range(n)] + return tasks + + async def gather_tasks(self, n, coro): + tasks = self.make_tasks(n, coro) + res = await asyncio.gather(*tasks) + return res, tasks + + async def test_barrier(self): + barrier = asyncio.Barrier(self.N) + self.assertIn("filling", repr(barrier)) + with self.assertRaisesRegex( + TypeError, + "'Barrier' object can't be awaited", + ): + await barrier + + self.assertIn("filling", repr(barrier)) + + async def test_repr(self): + barrier = asyncio.Barrier(self.N) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("filling", repr(barrier)) + + waiters = [] + async def wait(barrier): + await barrier.wait() + + incr = 2 + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"waiters:{incr}/{self.N}" in repr(barrier)) + self.assertIn("filling", repr(barrier)) + + # create missing waiters + for i in range(barrier.parties - barrier.n_waiting): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("draining", repr(barrier)) + + # add a part of waiters + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and reset + await barrier.reset() + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("resetting", repr(barrier)) + + # add a part of waiters again + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and abort + await barrier.abort() + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("broken", repr(barrier)) + self.assertTrue(barrier.broken) + + # suppress unhandled exceptions + await asyncio.gather(*waiters, return_exceptions=True) + + async def test_barrier_parties(self): + self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) + self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + + self.assertIsInstance(asyncio.Barrier(self.N), asyncio.Barrier) + + async def test_context_manager(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier as i: + results.append(i) + + await self.gather_tasks(self.N, coro) + + self.assertListEqual(sorted(results), list(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_one_task(self): + barrier = asyncio.Barrier(1) + + async def f(): + async with barrier as i: + return True + + ret = await f() + + self.assertTrue(ret) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_one_task_twice(self): + barrier = asyncio.Barrier(1) + + t1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 0) + + t2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + + self.assertEqual(t1.result(), t2.result()) + self.assertEqual(t1.done(), t2.done()) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_task_by_task(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + + t1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + self.assertIn("filling", repr(barrier)) + + t2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + self.assertIn("filling", repr(barrier)) + + t3 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + + await asyncio.wait([t1, t2, t3]) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_tasks_wait_twice(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier: + results.append(True) + + async with barrier: + results.append(False) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_tasks_check_return_value(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro(): + async with barrier: + results1.append(True) + + async with barrier as i: + results2.append(True) + return i + + res, _ = await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertListEqual(sorted(res), list(range(self.N))) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_draining_state(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier: + # barrier state change to filling for the last task release + results.append("draining" in repr(barrier)) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results), self.N) + self.assertEqual(results[-1], False) + self.assertTrue(all(results[:self.N-1])) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_blocking_tasks_while_draining(self): + rewait = 2 + barrier = asyncio.Barrier(self.N) + barrier_nowaiting = asyncio.Barrier(self.N - rewait) + results = [] + rewait_n = rewait + counter = 0 + + async def coro(): + nonlocal rewait_n + + # first time waiting + await barrier.wait() + + # after waiting once for all tasks + if rewait_n > 0: + rewait_n -= 1 + # wait again only for rewait tasks + await barrier.wait() + else: + # wait for end of draining state + await barrier_nowaiting.wait() + # wait for other waiting tasks + await barrier.wait() + + # a success means that barrier_nowaiting + # was waited for exactly N-rewait=3 times + await self.gather_tasks(self.N, coro) + + async def test_filling_tasks_cancel_one(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + await barrier.wait() + results.append(True) + + t1 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + + t2 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + + t1.cancel() + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + with self.assertRaises(asyncio.CancelledError): + await t1 + self.assertTrue(t1.cancelled()) + + t3 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + + t4 = asyncio.create_task(coro()) + await asyncio.gather(t2, t3, t4) + + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_reset_barrier(self): + barrier = asyncio.Barrier(1) + + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_while_tasks_waiting(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results.append(True) + + async def coro_reset(): + await barrier.reset() + + # N-1 tasks waiting on barrier with N parties + tasks = self.make_tasks(self.N-1, coro) + await asyncio.sleep(0) + + # reset the barrier + asyncio.create_task(coro_reset()) + await asyncio.gather(*tasks) + + self.assertEqual(len(results), self.N-1) + self.assertTrue(all(results)) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_when_tasks_half_draining(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + rest_of_tasks = self.N//2 + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # catch here waiting tasks + results1.append(True) + else: + # here drained task outside the barrier + if rest_of_tasks == barrier._count: + # tasks outside the barrier + await barrier.reset() + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*rest_of_tasks) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + count = 0 + + async def coro(): + nonlocal count + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch still waiting tasks + results1.append(True) + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + count += 1 + if count > blocking_tasks: + # reset now: raise asyncio.BrokenBarrierError for waiting tasks + await barrier.reset() + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here no catch - blocked tasks go to wait + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, []) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro1(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + await barrier.wait() + results2.append(True) + + async def coro2(): + async with barrier: + results2.append(True) + + tasks = self.make_tasks(self.N-1, coro1) + + # reset barrier, N-1 waiting tasks raise an BrokenBarrierError + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) + + # complete waiting tasks in the `finally` + asyncio.create_task(coro2()) + + await asyncio.gather(*tasks) + + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + + self.assertEqual(barrier.n_waiting, 0) + + + async def test_reset_barrier_while_tasks_draining(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + results3 = [] + count = 0 + + async def coro(): + nonlocal count + + i = await barrier.wait() + count += 1 + if count == self.N: + # last task exited from barrier + await barrier.reset() + + # wait here to reach the `parties` + await barrier.wait() + else: + try: + # second waiting + await barrier.wait() + + # N-1 tasks here + results1.append(True) + except Exception as e: + # never goes here + results2.append(True) + + # Now, pass the barrier again + # last wait, must be completed + k = await barrier.wait() + results3.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertFalse(barrier.broken) + self.assertTrue(all(results1)) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), 0) + self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) + + self.assertEqual(barrier.n_waiting, 0) + + async def test_abort_barrier(self): + barrier = asyncio.Barrier(1) + + asyncio.create_task(barrier.abort()) + await asyncio.sleep(0) + + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) + + async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + count = 0 + + async def coro(): + nonlocal count + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch tasks waiting to drain + results1.append(True) + else: + count += 1 + if count > blocking_tasks: + # abort now: raise asyncio.BrokenBarrierError for all tasks + await barrier.abort() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch blocked tasks (already drained) + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertTrue(barrier.broken) + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + + async def test_abort_barrier_when_exception(self): + # test from threading.Barrier: see `lock_tests.test_reset` + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro(): + try: + async with barrier as i : + if i == self.N//2: + raise RuntimeError + async with barrier: + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier.abort() + + await self.gather_tasks(self.N, coro) + + self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(barrier.n_waiting, 0) + + async def test_abort_barrier_when_exception_then_resetting(self): + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` + barrier1 = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + results3 = [] + + async def coro(): + try: + i = await barrier1.wait() + if i == self.N//2: + raise RuntimeError + await barrier1.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier1.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + i = await barrier2.wait() + if i == self.N//2: + await barrier1.reset() + await barrier2.wait() + await barrier1.wait() + results3.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertFalse(barrier1.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) + + self.assertEqual(barrier1.n_waiting, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py new file mode 100644 index 00000000000..a0c8434c945 --- /dev/null +++ b/Lib/test/test_asyncio/test_pep492.py @@ -0,0 +1,212 @@ +"""Tests support for new syntax introduced by PEP 492.""" + +import sys +import types +import unittest + +from unittest import mock + +import asyncio +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +# Test that asyncio.iscoroutine() uses collections.abc.Coroutine +class FakeCoro: + def send(self, value): + pass + + def throw(self, typ, val=None, tb=None): + pass + + def close(self): + pass + + def __await__(self): + yield + + +class BaseTest(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.BaseEventLoop() + self.loop._process_events = mock.Mock() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + +class LockTests(BaseTest): + + def test_context_manager_async_with(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + async def test(lock): + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + async with lock as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + await asyncio.sleep(0.01) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + def test_context_manager_with_await(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + async def test(lock): + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + with self.assertRaisesRegex( + TypeError, + "can't be awaited" + ): + with await lock: + pass + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + +class StreamReaderTests(BaseTest): + + def test_readline(self): + DATA = b'line1\nline2\nline3' + + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(DATA) + stream.feed_eof() + + async def reader(): + data = [] + async for line in stream: + data.append(line) + return data + + data = self.loop.run_until_complete(reader()) + self.assertEqual(data, [b'line1\n', b'line2\n', b'line3']) + + +class CoroutineTests(BaseTest): + + def test_iscoroutine(self): + async def foo(): pass + + f = foo() + try: + self.assertTrue(asyncio.iscoroutine(f)) + finally: + f.close() # silence warning + + self.assertTrue(asyncio.iscoroutine(FakeCoro())) + + def test_iscoroutine_generator(self): + def foo(): yield + + self.assertFalse(asyncio.iscoroutine(foo())) + + def test_iscoroutinefunction(self): + async def foo(): pass + with self.assertWarns(DeprecationWarning): + self.assertTrue(asyncio.iscoroutinefunction(foo)) + + def test_async_def_coroutines(self): + async def bar(): + return 'spam' + async def foo(): + return await bar() + + # production mode + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + # debug mode + self.loop.set_debug(True) + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + def test_debug_mode_manages_coroutine_origin_tracking(self): + async def start(): + self.assertTrue(sys.get_coroutine_origin_tracking_depth() > 0) + + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) + self.loop.set_debug(True) + self.loop.run_until_complete(start()) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) + + def test_types_coroutine(self): + def gen(): + yield from () + return 'spam' + + @types.coroutine + def func(): + return gen() + + async def coro(): + wrapper = func() + self.assertIsInstance(wrapper, types._GeneratorWrapper) + return await wrapper + + data = self.loop.run_until_complete(coro()) + self.assertEqual(data, 'spam') + + def test_task_print_stack(self): + T = None + + async def foo(): + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + async def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + await T + + self.loop.run_until_complete(runner()) + + def test_double_await(self): + async def afunc(): + await asyncio.sleep(0.1) + + async def runner(): + coro = afunc() + t = self.loop.create_task(coro) + try: + await asyncio.sleep(0) + await coro + finally: + t.cancel() + + self.loop.set_debug(True) + with self.assertRaises( + RuntimeError, + msg='coroutine is being awaited already'): + + self.loop.run_until_complete(runner()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py new file mode 100644 index 00000000000..edfad5e11db --- /dev/null +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -0,0 +1,1094 @@ +"""Tests for proactor_events.py""" + +import io +import socket +import unittest +import sys +from unittest import mock + +import asyncio +from asyncio.proactor_events import BaseProactorEventLoop +from asyncio.proactor_events import _ProactorSocketTransport +from asyncio.proactor_events import _ProactorWritePipeTransport +from asyncio.proactor_events import _ProactorDuplexPipeTransport +from asyncio.proactor_events import _ProactorDatagramTransport +from test.support import os_helper +from test.support import socket_helper +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def close_transport(transport): + # Don't call transport.close() because the event loop and the IOCP proactor + # are mocked + if transport._sock is None: + return + transport._sock.close() + transport._sock = None + + +class ProactorSocketTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.buffer_size = 65536 + + def socket_transport(self, waiter=None): + transport = _ProactorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + fut = self.loop.create_future() + tr = self.socket_transport(waiter=fut) + test_utils.run_briefly(self.loop) + self.assertIsNone(fut.result()) + self.protocol.connection_made(tr) + self.proactor.recv_into.assert_called_with(self.sock, bytearray(self.buffer_size)) + + def test_loop_reading(self): + tr = self.socket_transport() + tr._loop_reading() + self.loop._proactor.recv_into.assert_called_with(self.sock, bytearray(self.buffer_size)) + self.assertFalse(self.protocol.data_received.called) + self.assertFalse(self.protocol.eof_received.called) + + def test_loop_reading_data(self): + buf = b'data' + res = self.loop.create_future() + res.set_result(len(buf)) + + tr = self.socket_transport() + tr._read_fut = res + tr._data[:len(buf)] = buf + tr._loop_reading(res) + called_buf = bytearray(self.buffer_size) + called_buf[:len(buf)] = buf + self.loop._proactor.recv_into.assert_called_with(self.sock, called_buf) + self.protocol.data_received.assert_called_with(buf) + # assert_called_with maps bytearray and bytes to the same thing so check manually + # regression test for https://github.com/python/cpython/issues/99941 + self.assertIsInstance(self.protocol.data_received.call_args.args[0], bytes) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_loop_reading_no_data(self): + res = self.loop.create_future() + res.set_result(0) + + tr = self.socket_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertFalse(self.loop._proactor.recv_into.called) + self.assertTrue(self.protocol.eof_received.called) + self.assertTrue(tr.close.called) + + def test_loop_reading_aborted(self): + err = self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_loop_reading_aborted_closing(self): + self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._closing = True + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + + def test_loop_reading_aborted_is_fatal(self): + self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertTrue(tr._fatal_error.called) + + def test_loop_reading_conn_reset_lost(self): + err = self.loop._proactor.recv_into.side_effect = ConnectionResetError() + + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._force_close = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + tr._force_close.assert_called_with(err) + + def test_loop_reading_exception(self): + err = self.loop._proactor.recv_into.side_effect = (OSError()) + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_write(self): + tr = self.socket_transport() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, None) + tr._loop_writing.assert_called_with(data=b'data') + + def test_write_no_data(self): + tr = self.socket_transport() + tr.write(b'') + self.assertFalse(tr._buffer) + + def test_write_more(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, b'data') + self.assertFalse(tr._loop_writing.called) + + def test_loop_writing(self): + tr = self.socket_transport() + tr._buffer = bytearray(b'data') + tr._loop_writing() + self.loop._proactor.send.assert_called_with(self.sock, b'data') + self.loop._proactor.send.return_value.add_done_callback.\ + assert_called_with(tr._loop_writing) + + @mock.patch('asyncio.proactor_events.logger') + def test_loop_writing_err(self, m_log): + err = self.loop._proactor.send.side_effect = OSError() + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._buffer = [b'da', b'ta'] + tr._loop_writing() + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + tr._conn_lost = 1 + + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + self.assertEqual(tr._buffer, None) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_loop_writing_stop(self): + fut = self.loop.create_future() + fut.set_result(b'data') + + tr = self.socket_transport() + tr._write_fut = fut + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + + def test_loop_writing_closing(self): + fut = self.loop.create_future() + fut.set_result(1) + + tr = self.socket_transport() + tr._write_fut = fut + tr.close() + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_abort(self): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr.abort() + tr._force_close.assert_called_with(None) + + def test_close(self): + tr = self.socket_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertTrue(tr.is_closing()) + self.assertEqual(tr._conn_lost, 1) + + self.protocol.connection_lost.reset_mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_write_fut(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_buffer(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_invalid_sockobj(self): + tr = self.socket_transport() + self.sock.fileno.return_value = -1 + tr.close() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertFalse(self.sock.shutdown.called) + + @mock.patch('asyncio.base_events.logger') + def test_fatal_error(self, m_logging): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr._fatal_error(None) + self.assertTrue(tr._force_close.called) + self.assertTrue(m_logging.error.called) + + def test_force_close(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + read_fut = tr._read_fut = mock.Mock() + write_fut = tr._write_fut = mock.Mock() + tr._force_close(None) + + read_fut.cancel.assert_called_with() + write_fut.cancel.assert_called_with() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + self.assertEqual(tr._conn_lost, 1) + + def test_loop_writing_force_close(self): + exc_handler = mock.Mock() + self.loop.set_exception_handler(exc_handler) + fut = self.loop.create_future() + fut.set_result(1) + self.proactor.send.return_value = fut + + tr = self.socket_transport() + tr.write(b'data') + tr._force_close(None) + test_utils.run_briefly(self.loop) + exc_handler.assert_not_called() + + def test_force_close_idempotent(self): + tr = self.socket_transport() + tr._closing = True + tr._force_close(None) + test_utils.run_briefly(self.loop) + # See https://github.com/python/cpython/issues/89237 + # `protocol.connection_lost` should be called even if + # the transport was closed forcefully otherwise + # the resources held by protocol will never be freed + # and waiters will never be notified leading to hang. + self.assertTrue(self.protocol.connection_lost.called) + + def test_force_close_protocol_connection_lost_once(self): + tr = self.socket_transport() + self.assertFalse(self.protocol.connection_lost.called) + tr._closing = True + # Calling _force_close twice should not call + # protocol.connection_lost twice + tr._force_close(None) + tr._force_close(None) + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) + + def test_close_protocol_connection_lost_once(self): + tr = self.socket_transport() + self.assertFalse(self.protocol.connection_lost.called) + # Calling close twice should not call + # protocol.connection_lost twice + tr.close() + tr.close() + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) + + def test_fatal_error_2(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr._force_close(None) + + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + + def test_call_connection_lost(self): + tr = self.socket_transport() + tr._call_connection_lost(None) + self.assertTrue(self.protocol.connection_lost.called) + self.assertTrue(self.sock.close.called) + + def test_write_eof(self): + tr = self.socket_transport() + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.write_eof() + self.assertEqual(self.sock.shutdown.call_count, 1) + tr.close() + + def test_write_eof_buffer(self): + tr = self.socket_transport() + f = self.loop.create_future() + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr._eof_written) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.close() + + def test_write_eof_write_pipe(self): + tr = _ProactorWritePipeTransport( + self.loop, self.sock, self.protocol) + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_buffer_write_pipe(self): + tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol) + f = self.loop.create_future() + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_duplex_pipe(self): + tr = _ProactorDuplexPipeTransport( + self.loop, self.sock, self.protocol) + self.assertFalse(tr.can_write_eof()) + with self.assertRaises(NotImplementedError): + tr.write_eof() + close_transport(tr) + + def test_pause_resume_reading(self): + tr = self.socket_transport() + index = 0 + msgs = [b'data1', b'data2', b'data3', b'data4', b'data5', b''] + reversed_msgs = list(reversed(msgs)) + + def recv_into(sock, data): + f = self.loop.create_future() + msg = reversed_msgs.pop() + + result = f.result + def monkey(): + data[:len(msg)] = msg + return result() + f.result = monkey + + f.set_result(len(msg)) + return f + + self.loop._proactor.recv_into.side_effect = recv_into + self.loop._run_once() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + + for msg in msgs[:2]: + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msg)) + + tr.pause_reading() + tr.pause_reading() + self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) + for i in range(10): + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msgs[1])) + + tr.resume_reading() + tr.resume_reading() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + + for msg in msgs[2:4]: + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msg)) + + tr.pause_reading() + tr.resume_reading() + self.loop.call_exception_handler = mock.Mock() + self.loop._run_once() + self.loop.call_exception_handler.assert_not_called() + self.protocol.data_received.assert_called_with(bytearray(msgs[4])) + tr.close() + + self.assertFalse(tr.is_reading()) + + def test_pause_reading_connection_made(self): + tr = self.socket_transport() + self.protocol.connection_made.side_effect = lambda _: tr.pause_reading() + test_utils.run_briefly(self.loop) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + self.assertTrue(tr.is_reading()) + + tr.close() + self.assertFalse(tr.is_reading()) + + + def pause_writing_transport(self, high): + tr = self.socket_transport() + tr.set_write_buffer_limits(high=high) + + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + self.assertFalse(self.protocol.resume_writing.called) + return tr + + def test_pause_resume_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk, must pause writing + fut = self.loop.create_future() + self.loop._proactor.send.return_value = fut + tr.write(b'large data') + self.loop._run_once() + self.assertTrue(self.protocol.pause_writing.called) + + # flush the buffer + fut.set_result(None) + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertTrue(self.protocol.resume_writing.called) + + def test_pause_writing_2write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (3 <= 4) + fut1 = self.loop.create_future() + self.loop._proactor.send.return_value = fut1 + tr.write(b'123') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_pause_writing_3write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (1 <= 4) + fut = self.loop.create_future() + self.loop._proactor.send.return_value = fut + tr.write(b'1') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 1) + self.assertFalse(self.protocol.pause_writing.called) + + # second short write, the buffer is not full (3 <= 4) + tr.write(b'23') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_dont_pause_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk which completes immediately, + # it should not pause writing + fut = self.loop.create_future() + fut.set_result(None) + self.loop._proactor.send.return_value = fut + tr.write(b'very large data') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + + +class ProactorDatagramTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + self.sock = mock.Mock(spec_set=socket.socket) + self.sock.fileno.return_value = 7 + + def datagram_transport(self, address=None): + self.sock.getpeername.side_effect = None if address else OSError + transport = _ProactorDatagramTransport(self.loop, self.sock, + self.protocol, + address=address) + self.addCleanup(close_transport, transport) + return transport + + def test_sendto(self): + data = b'data' + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, data, addr=('0.0.0.0', 1234)) + self.assertFalse(transport._buffer) + self.assertEqual(0, transport._buffer_size) + + def test_sendto_bytearray(self): + data = bytearray(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'data', addr=('0.0.0.0', 1234)) + + def test_sendto_memoryview(self): + data = memoryview(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'data', addr=('0.0.0.0', 1234)) + + def test_sendto_no_data(self): + transport = self.datagram_transport() + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) + + def test_sendto_buffer(self): + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(b'data2', ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + + def test_sendto_buffer_bytearray(self): + data2 = bytearray(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_memoryview(self): + data2 = memoryview(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + @mock.patch('asyncio.proactor_events.logger') + def test_sendto_exception(self, m_log): + data = b'data' + err = self.proactor.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertTrue(transport._fatal_error.called) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + transport._conn_lost = 1 + + transport._address = ('123',) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + m_log.warning.assert_called_with('socket.sendto() raised exception.') + + def test_sendto_error_received(self): + data = b'data' + + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertEqual(transport._conn_lost, 0) + self.assertFalse(transport._fatal_error.called) + + def test_sendto_error_received_connected(self): + data = b'data' + + self.proactor.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport.sendto(data) + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + def test_sendto_str(self): + transport = self.datagram_transport() + self.assertRaises(TypeError, transport.sendto, 'str', ()) + + def test_sendto_connected_addr(self): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + self.assertRaises( + ValueError, transport.sendto, b'str', ('0.0.0.0', 2)) + + def test_sendto_closing(self): + transport = self.datagram_transport(address=(1,)) + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.sendto(b'data', (1,)) + self.assertEqual(transport._conn_lost, 2) + + def test__loop_writing_closing(self): + transport = self.datagram_transport() + transport._closing = True + transport._loop_writing() + self.assertIsNone(transport._write_fut) + test_utils.run_briefly(self.loop) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + def test__loop_writing_exception(self): + err = self.proactor.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + + def test__loop_writing_error_received(self): + self.proactor.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + self.assertFalse(transport._fatal_error.called) + + def test__loop_writing_error_received_connection(self): + self.proactor.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected(self, m_exc): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = ConnectionRefusedError() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_not_called() + + +class BaseProactorEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + + self.sock = test_utils.mock_nonblocking_socket() + self.proactor = mock.Mock() + + self.ssock, self.csock = mock.Mock(), mock.Mock() + + with mock.patch('asyncio.proactor_events.socket.socketpair', + return_value=(self.ssock, self.csock)): + with mock.patch('signal.set_wakeup_fd'): + self.loop = BaseProactorEventLoop(self.proactor) + self.set_event_loop(self.loop) + + @mock.patch('asyncio.proactor_events.socket.socketpair') + def test_ctor(self, socketpair): + ssock, csock = socketpair.return_value = ( + mock.Mock(), mock.Mock()) + with mock.patch('signal.set_wakeup_fd'): + loop = BaseProactorEventLoop(self.proactor) + self.assertIs(loop._ssock, ssock) + self.assertIs(loop._csock, csock) + self.assertEqual(loop._internal_fds, 1) + loop.close() + + def test_close_self_pipe(self): + self.loop._close_self_pipe() + self.assertEqual(self.loop._internal_fds, 0) + self.assertTrue(self.ssock.close.called) + self.assertTrue(self.csock.close.called) + self.assertIsNone(self.loop._ssock) + self.assertIsNone(self.loop._csock) + + # Don't call close(): _close_self_pipe() cannot be called twice + self.loop._closed = True + + def test_close(self): + self.loop._close_self_pipe = mock.Mock() + self.loop.close() + self.assertTrue(self.loop._close_self_pipe.called) + self.assertTrue(self.proactor.close.called) + self.assertIsNone(self.loop._proactor) + + self.loop._close_self_pipe.reset_mock() + self.loop.close() + self.assertFalse(self.loop._close_self_pipe.called) + + def test_make_socket_transport(self): + tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol()) + self.assertIsInstance(tr, _ProactorSocketTransport) + close_transport(tr) + + def test_loop_self_reading(self): + self.loop._loop_self_reading() + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_fut(self): + fut = mock.Mock() + self.loop._self_reading_future = fut + self.loop._loop_self_reading(fut) + self.assertTrue(fut.result.called) + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_exception(self): + self.loop.call_exception_handler = mock.Mock() + self.proactor.recv.side_effect = OSError() + self.loop._loop_self_reading() + self.assertTrue(self.loop.call_exception_handler.called) + + def test_write_to_self(self): + self.loop._write_to_self() + self.csock.send.assert_called_with(b'\0') + + def test_process_events(self): + self.loop._process_events([]) + + @mock.patch('asyncio.base_events.logger') + def test_create_server(self, m_log): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + self.assertTrue(call_soon.called) + + # callback + loop = call_soon.call_args[0][0] + loop() + self.proactor.accept.assert_called_with(self.sock) + + # conn + fut = mock.Mock() + fut.result.return_value = (mock.Mock(), mock.Mock()) + + make_tr = self.loop._make_socket_transport = mock.Mock() + loop(fut) + self.assertTrue(fut.result.called) + self.assertTrue(make_tr.called) + + # exception + fut.result.side_effect = OSError() + loop(fut) + self.assertTrue(self.sock.close.called) + self.assertTrue(m_log.error.called) + + def test_create_server_cancel(self): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + loop = call_soon.call_args[0][0] + + # cancelled + fut = self.loop.create_future() + fut.cancel() + loop(fut) + self.assertTrue(self.sock.close.called) + + def test_stop_serving(self): + sock1 = mock.Mock() + future1 = mock.Mock() + sock2 = mock.Mock() + future2 = mock.Mock() + self.loop._accept_futures = { + sock1.fileno(): future1, + sock2.fileno(): future2 + } + + self.loop._stop_serving(sock1) + self.assertTrue(sock1.close.called) + self.assertTrue(future1.cancel.called) + self.proactor._stop_serving.assert_called_with(sock1) + self.assertFalse(sock2.close.called) + self.assertFalse(future2.cancel.called) + + def datagram_transport(self): + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + return self.loop._make_datagram_transport(self.sock, self.protocol) + + def test_make_datagram_transport(self): + tr = self.datagram_transport() + self.assertIsInstance(tr, _ProactorDatagramTransport) + self.assertIsInstance(tr, asyncio.DatagramTransport) + close_transport(tr) + + def test_datagram_loop_writing(self): + tr = self.datagram_transport() + tr._buffer.appendleft((b'data', ('127.0.0.1', 12068))) + tr._loop_writing() + self.loop._proactor.sendto.assert_called_with(self.sock, b'data', addr=('127.0.0.1', 12068)) + self.loop._proactor.sendto.return_value.add_done_callback.\ + assert_called_with(tr._loop_writing) + + close_transport(tr) + + def test_datagram_loop_reading(self): + tr = self.datagram_transport() + tr._loop_reading() + self.loop._proactor.recvfrom.assert_called_with(self.sock, 256 * 1024) + self.assertFalse(self.protocol.datagram_received.called) + self.assertFalse(self.protocol.error_received.called) + close_transport(tr) + + def test_datagram_loop_reading_data(self): + res = self.loop.create_future() + res.set_result((b'data', ('127.0.0.1', 12068))) + + tr = self.datagram_transport() + tr._read_fut = res + tr._loop_reading(res) + self.loop._proactor.recvfrom.assert_called_with(self.sock, 256 * 1024) + self.protocol.datagram_received.assert_called_with(b'data', ('127.0.0.1', 12068)) + close_transport(tr) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_datagram_loop_reading_no_data(self): + res = self.loop.create_future() + res.set_result((b'', ('127.0.0.1', 12068))) + + tr = self.datagram_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertTrue(self.loop._proactor.recvfrom.called) + self.assertFalse(self.protocol.error_received.called) + self.assertFalse(tr.close.called) + close_transport(tr) + + def test_datagram_loop_reading_aborted(self): + err = self.loop._proactor.recvfrom.side_effect = ConnectionAbortedError() + + tr = self.datagram_transport() + tr._fatal_error = mock.Mock() + tr._protocol.error_received = mock.Mock() + tr._loop_reading() + tr._protocol.error_received.assert_called_with(err) + close_transport(tr) + + def test_datagram_loop_writing_aborted(self): + err = self.loop._proactor.sendto.side_effect = ConnectionAbortedError() + + tr = self.datagram_transport() + tr._fatal_error = mock.Mock() + tr._protocol.error_received = mock.Mock() + tr._buffer.appendleft((b'Hello', ('127.0.0.1', 12068))) + tr._loop_writing() + tr._protocol.error_received.assert_called_with(err) + close_transport(tr) + + +@unittest.skipIf(sys.platform != 'win32', + 'Proactor is supported on Windows only') +class ProactorEventLoopUnixSockSendfileTests(test_utils.TestCase): + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + self.addCleanup(self.loop.close) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, cleanup=True, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) + if cleanup: + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind(('127.0.0.1', port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.run_loop(self.loop.sock_connect(sock, srv_sock.getsockname())) + + def cleanup(): + if proto.transport is not None: + # can be None if the task was cancelled before + # connection_made callback + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_not_a_file(self): + sock, proto = self.prepare() + f = object() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_iobuffer(self): + sock, proto = self.prepare() + f = io.BytesIO() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_regular_file(self): + sock, proto = self.prepare() + f = mock.Mock() + f.fileno.return_value = -1 + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py new file mode 100644 index 00000000000..29d3bd22705 --- /dev/null +++ b/Lib/test/test_asyncio/test_protocols.py @@ -0,0 +1,67 @@ +import unittest +from unittest import mock + +import asyncio + + +def tearDownModule(): + # not needed for the test file but added for uniformness with all other + # asyncio test files for the sake of unified cleanup + asyncio.events._set_event_loop_policy(None) + + +class ProtocolsAbsTests(unittest.TestCase): + + def test_base_protocol(self): + f = mock.Mock() + p = asyncio.BaseProtocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_protocol(self): + f = mock.Mock() + p = asyncio.Protocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.data_received(f)) + self.assertIsNone(p.eof_received()) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_buffered_protocol(self): + f = mock.Mock() + p = asyncio.BufferedProtocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.get_buffer(100)) + self.assertIsNone(p.buffer_updated(150)) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_datagram_protocol(self): + f = mock.Mock() + dp = asyncio.DatagramProtocol() + self.assertIsNone(dp.connection_made(f)) + self.assertIsNone(dp.connection_lost(f)) + self.assertIsNone(dp.error_received(f)) + self.assertIsNone(dp.datagram_received(f, f)) + self.assertNotHasAttr(dp, '__dict__') + + def test_subprocess_protocol(self): + f = mock.Mock() + sp = asyncio.SubprocessProtocol() + self.assertIsNone(sp.connection_made(f)) + self.assertIsNone(sp.connection_lost(f)) + self.assertIsNone(sp.pipe_data_received(1, f)) + self.assertIsNone(sp.pipe_connection_lost(1, f)) + self.assertIsNone(sp.process_exited()) + self.assertNotHasAttr(sp, '__dict__') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py new file mode 100644 index 00000000000..54bbe79f81f --- /dev/null +++ b/Lib/test/test_asyncio/test_queues.py @@ -0,0 +1,725 @@ +"""Tests for queues.py""" + +import asyncio +import unittest +from types import GenericAlias + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class QueueBasicTests(unittest.IsolatedAsyncioTestCase): + + async def _test_repr_or_str(self, fn, expect_id): + """Test Queue's repr or str. + + fn is repr or str. expect_id is True if we expect the Queue's id to + appear in fn(Queue()). + """ + q = asyncio.Queue() + self.assertStartsWith(fn(q), ' 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py new file mode 100644 index 00000000000..de489c2dc43 --- /dev/null +++ b/Lib/test/test_asyncio/test_runners.py @@ -0,0 +1,528 @@ +import _thread +import asyncio +import contextvars +import re +import signal +import sys +import threading +import unittest +from test.test_asyncio import utils as test_utils +from unittest import mock +from unittest.mock import patch + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def interrupt_self(): + _thread.interrupt_main() + + +class TestPolicy(asyncio.events._AbstractEventLoopPolicy): + + def __init__(self, loop_factory): + self.loop_factory = loop_factory + self.loop = None + + def get_event_loop(self): + # shouldn't ever be called by asyncio.run() + raise RuntimeError + + def new_event_loop(self): + return self.loop_factory() + + def set_event_loop(self, loop): + if loop is not None: + # we want to check if the loop is closed + # in BaseTest.tearDown + self.loop = loop + + +class BaseTest(unittest.TestCase): + + def new_loop(self): + loop = asyncio.BaseEventLoop() + loop._process_events = mock.Mock() + # Mock waking event loop from select + loop._write_to_self = mock.Mock() + loop._write_to_self.return_value = None + loop._selector = mock.Mock() + loop._selector.select.return_value = () + loop.shutdown_ag_run = False + + async def shutdown_asyncgens(): + loop.shutdown_ag_run = True + loop.shutdown_asyncgens = shutdown_asyncgens + + return loop + + def setUp(self): + super().setUp() + + policy = TestPolicy(self.new_loop) + asyncio.events._set_event_loop_policy(policy) + + def tearDown(self): + policy = asyncio.events._get_event_loop_policy() + if policy.loop is not None: + self.assertTrue(policy.loop.is_closed()) + self.assertTrue(policy.loop.shutdown_ag_run) + + asyncio.events._set_event_loop_policy(None) + super().tearDown() + + +class RunTests(BaseTest): + + def test_asyncio_run_return(self): + async def main(): + await asyncio.sleep(0) + return 42 + + self.assertEqual(asyncio.run(main()), 42) + + def test_asyncio_run_raises(self): + async def main(): + await asyncio.sleep(0) + raise ValueError('spam') + + with self.assertRaisesRegex(ValueError, 'spam'): + asyncio.run(main()) + + def test_asyncio_run_only_coro(self): + for o in {1, lambda: None}: + with self.subTest(obj=o), \ + self.assertRaisesRegex(TypeError, + 'an awaitable is required'): + asyncio.run(o) + + def test_asyncio_run_debug(self): + async def main(expected): + loop = asyncio.get_event_loop() + self.assertIs(loop.get_debug(), expected) + + asyncio.run(main(False), debug=False) + asyncio.run(main(True), debug=True) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): + asyncio.run(main(True)) + asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) + + def test_asyncio_run_from_running_loop(self): + async def main(): + coro = main() + try: + asyncio.run(coro) + finally: + coro.close() # Suppress ResourceWarning + + with self.assertRaisesRegex(RuntimeError, + 'cannot be called from a running'): + asyncio.run(main()) + + def test_asyncio_run_cancels_hanging_tasks(self): + lo_task = None + + async def leftover(): + await asyncio.sleep(0.1) + + async def main(): + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + def test_asyncio_run_reports_hanging_tasks_errors(self): + lo_task = None + call_exc_handler_mock = mock.Mock() + + async def leftover(): + try: + await asyncio.sleep(0.1) + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = call_exc_handler_mock + + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + call_exc_handler_mock.assert_called_with({ + 'message': test_utils.MockPattern(r'asyncio.run.*shutdown'), + 'task': lo_task, + 'exception': test_utils.MockInstanceOf(ZeroDivisionError) + }) + + def test_asyncio_run_closes_gens_after_hanging_tasks_errors(self): + spinner = None + lazyboy = None + + class FancyExit(Exception): + pass + + async def fidget(): + while True: + yield 1 + await asyncio.sleep(1) + + async def spin(): + nonlocal spinner + spinner = fidget() + try: + async for the_meaning_of_life in spinner: # NoQA + pass + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = mock.Mock() + + nonlocal lazyboy + lazyboy = asyncio.create_task(spin()) + raise FancyExit + + with self.assertRaises(FancyExit): + asyncio.run(main()) + + self.assertTrue(lazyboy.done()) + + self.assertIsNone(spinner.ag_frame) + self.assertFalse(spinner.ag_running) + + def test_asyncio_run_set_event_loop(self): + #See https://github.com/python/cpython/issues/93896 + + async def main(): + await asyncio.sleep(0) + return 42 + + policy = asyncio.events._get_event_loop_policy() + policy.set_event_loop = mock.Mock() + asyncio.run(main()) + self.assertTrue(policy.set_event_loop.called) + + def test_asyncio_run_without_uncancel(self): + # See https://github.com/python/cpython/issues/95097 + class Task: + def __init__(self, loop, coro, **kwargs): + self._task = asyncio.Task(coro, loop=loop, **kwargs) + + def cancel(self, *args, **kwargs): + return self._task.cancel(*args, **kwargs) + + def add_done_callback(self, *args, **kwargs): + return self._task.add_done_callback(*args, **kwargs) + + def remove_done_callback(self, *args, **kwargs): + return self._task.remove_done_callback(*args, **kwargs) + + @property + def _asyncio_future_blocking(self): + return self._task._asyncio_future_blocking + + def result(self, *args, **kwargs): + return self._task.result(*args, **kwargs) + + def done(self, *args, **kwargs): + return self._task.done(*args, **kwargs) + + def cancelled(self, *args, **kwargs): + return self._task.cancelled(*args, **kwargs) + + def exception(self, *args, **kwargs): + return self._task.exception(*args, **kwargs) + + def get_loop(self, *args, **kwargs): + return self._task.get_loop(*args, **kwargs) + + def set_name(self, *args, **kwargs): + return self._task.set_name(*args, **kwargs) + + async def main(): + interrupt_self() + await asyncio.Event().wait() + + def new_event_loop(): + loop = self.new_loop() + loop.set_task_factory(Task) + return loop + + asyncio.events._set_event_loop_policy(TestPolicy(new_event_loop)) + with self.assertRaises(asyncio.CancelledError): + asyncio.run(main()) + + def test_asyncio_run_loop_factory(self): + factory = mock.Mock() + loop = factory.return_value = self.new_loop() + + async def main(): + self.assertEqual(asyncio.get_running_loop(), loop) + + asyncio.run(main(), loop_factory=factory) + factory.assert_called_once_with() + + def test_loop_factory_default_event_loop(self): + async def main(): + if sys.platform == "win32": + self.assertIsInstance(asyncio.get_running_loop(), asyncio.ProactorEventLoop) + else: + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) + + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + + +class RunnerTests(BaseTest): + + def test_non_debug(self): + with asyncio.Runner(debug=False) as runner: + self.assertFalse(runner.get_loop().get_debug()) + + def test_debug(self): + with asyncio.Runner(debug=True) as runner: + self.assertTrue(runner.get_loop().get_debug()) + + def test_custom_factory(self): + loop = mock.Mock() + with asyncio.Runner(loop_factory=lambda: loop) as runner: + self.assertIs(runner.get_loop(), loop) + + def test_run(self): + async def f(): + await asyncio.sleep(0) + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(f())) + loop = runner.get_loop() + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + self.assertTrue(loop.is_closed()) + + def test_run_non_coro(self): + with asyncio.Runner() as runner: + with self.assertRaisesRegex( + TypeError, + "an awaitable is required" + ): + runner.run(123) + + def test_run_future(self): + with asyncio.Runner() as runner: + fut = runner.get_loop().create_future() + fut.set_result('done') + self.assertEqual('done', runner.run(fut)) + + def test_run_awaitable(self): + class MyAwaitable: + def __await__(self): + return self.run().__await__() + + @staticmethod + async def run(): + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(MyAwaitable())) + + def test_explicit_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + runner.close() + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + self.assertTrue(loop.is_closed()) + + def test_double_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + + runner.close() + self.assertTrue(loop.is_closed()) + + # the second call is no-op + runner.close() + self.assertTrue(loop.is_closed()) + + def test_second_with_block_raises(self): + ret = [] + + async def f(arg): + ret.append(arg) + + runner = asyncio.Runner() + with runner: + runner.run(f(1)) + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + with runner: + runner.run(f(2)) + + self.assertEqual([1], ret) + + def test_run_keeps_context(self): + cvar = contextvars.ContextVar("cvar", default=-1) + + async def f(val): + old = cvar.get() + await asyncio.sleep(0) + cvar.set(val) + return old + + async def get_context(): + return contextvars.copy_context() + + with asyncio.Runner() as runner: + self.assertEqual(-1, runner.run(f(1))) + self.assertEqual(1, runner.run(f(2))) + + self.assertEqual(2, runner.run(get_context()).get(cvar)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - RuntimeWarning for unawaited coroutine not triggered + def test_recursive_run(self): + async def g(): + pass + + async def f(): + runner.run(g()) + + with asyncio.Runner() as runner: + with self.assertWarnsRegex( + RuntimeWarning, + "coroutine .+ was never awaited", + ): + with self.assertRaisesRegex( + RuntimeError, + re.escape( + "Runner.run() cannot be called from a running event loop" + ), + ): + runner.run(f()) + + def test_interrupt_call_soon(self): + # The only case when task is not suspended by waiting a future + # or another task + assert threading.current_thread() is threading.main_thread() + + async def coro(): + with self.assertRaises(asyncio.CancelledError): + while True: + await asyncio.sleep(0) + raise asyncio.CancelledError() + + with asyncio.Runner() as runner: + runner.get_loop().call_later(0.1, interrupt_self) + with self.assertRaises(KeyboardInterrupt): + runner.run(coro()) + + def test_interrupt_wait(self): + # interrupting when waiting a future cancels both future and main task + assert threading.current_thread() is threading.main_thread() + + async def coro(fut): + with self.assertRaises(asyncio.CancelledError): + await fut + raise asyncio.CancelledError() + + with asyncio.Runner() as runner: + fut = runner.get_loop().create_future() + runner.get_loop().call_later(0.1, interrupt_self) + + with self.assertRaises(KeyboardInterrupt): + runner.run(coro(fut)) + + self.assertTrue(fut.cancelled()) + + def test_interrupt_cancelled_task(self): + # interrupting cancelled main task doesn't raise KeyboardInterrupt + assert threading.current_thread() is threading.main_thread() + + async def subtask(task): + await asyncio.sleep(0) + task.cancel() + interrupt_self() + + async def coro(): + asyncio.create_task(subtask(asyncio.current_task())) + await asyncio.sleep(10) + + with asyncio.Runner() as runner: + with self.assertRaises(asyncio.CancelledError): + runner.run(coro()) + + def test_signal_install_not_supported_ok(self): + # signal.signal() can throw if the "main thread" doesn't have signals enabled + assert threading.current_thread() is threading.main_thread() + + async def coro(): + pass + + with asyncio.Runner() as runner: + with patch.object( + signal, + "signal", + side_effect=ValueError( + "signal only works in main thread of the main interpreter" + ) + ): + runner.run(coro()) + + def test_set_event_loop_called_once(self): + # See https://github.com/python/cpython/issues/95736 + async def coro(): + pass + + policy = asyncio.events._get_event_loop_policy() + policy.set_event_loop = mock.Mock() + runner = asyncio.Runner() + runner.run(coro()) + runner.run(coro()) + + self.assertEqual(1, policy.set_event_loop.call_count) + runner.close() + + def test_no_repr_is_call_on_the_task_result(self): + # See https://github.com/python/cpython/issues/112559. + class MyResult: + def __init__(self): + self.repr_count = 0 + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + async def coro(): + return MyResult() + + + with asyncio.Runner() as runner: + result = runner.run(coro()) + + self.assertEqual(0, result.repr_count) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py new file mode 100644 index 00000000000..4bb5d4fb816 --- /dev/null +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -0,0 +1,1660 @@ +"""Tests for selector_events.py""" + +import collections +import selectors +import socket +import sys +import unittest +from asyncio import selector_events +from unittest import mock + +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from asyncio.selector_events import (BaseSelectorEventLoop, + _SelectorDatagramTransport, + _SelectorSocketTransport, + _SelectorTransport) +from test.test_asyncio import utils as test_utils + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class TestBaseSelectorEventLoop(BaseSelectorEventLoop): + + def _make_self_pipe(self): + self._ssock = mock.Mock() + self._csock = mock.Mock() + self._internal_fds += 1 + + def _close_self_pipe(self): + pass + + +def list_to_buffer(l=()): + buffer = collections.deque() + buffer.extend((memoryview(i) for i in l)) + return buffer + + + +def close_transport(transport): + # Don't call transport.close() because the event loop and the selector + # are mocked + if transport._sock is None: + return + transport._sock.close() + transport._sock = None + + +class BaseSelectorEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.selector = mock.Mock() + self.selector.select.return_value = [] + self.loop = TestBaseSelectorEventLoop(self.selector) + self.set_event_loop(self.loop) + + def test_make_socket_transport(self): + m = mock.Mock() + self.loop.add_reader = mock.Mock() + self.loop._ensure_fd_no_transport = mock.Mock() + transport = self.loop._make_socket_transport(m, asyncio.Protocol()) + self.assertIsInstance(transport, _SelectorSocketTransport) + self.assertEqual(self.loop._ensure_fd_no_transport.call_count, 1) + + # Calling repr() must not fail when the event loop is closed + self.loop.close() + repr(transport) + + close_transport(transport) + + @mock.patch('asyncio.selector_events.ssl', None) + @mock.patch('asyncio.sslproto.ssl', None) + def test_make_ssl_transport_without_ssl_error(self): + m = mock.Mock() + self.loop.add_reader = mock.Mock() + self.loop.add_writer = mock.Mock() + self.loop.remove_reader = mock.Mock() + self.loop.remove_writer = mock.Mock() + self.loop._ensure_fd_no_transport = mock.Mock() + with self.assertRaises(RuntimeError): + self.loop._make_ssl_transport(m, m, m, m) + self.assertEqual(self.loop._ensure_fd_no_transport.call_count, 1) + + def test_close(self): + class EventLoop(BaseSelectorEventLoop): + def _make_self_pipe(self): + self._ssock = mock.Mock() + self._csock = mock.Mock() + self._internal_fds += 1 + + self.loop = EventLoop(self.selector) + self.set_event_loop(self.loop) + + ssock = self.loop._ssock + ssock.fileno.return_value = 7 + csock = self.loop._csock + csock.fileno.return_value = 1 + remove_reader = self.loop._remove_reader = mock.Mock() + + self.loop._selector.close() + self.loop._selector = selector = mock.Mock() + self.assertFalse(self.loop.is_closed()) + + self.loop.close() + self.assertTrue(self.loop.is_closed()) + self.assertIsNone(self.loop._selector) + self.assertIsNone(self.loop._csock) + self.assertIsNone(self.loop._ssock) + selector.close.assert_called_with() + ssock.close.assert_called_with() + csock.close.assert_called_with() + remove_reader.assert_called_with(7) + + # it should be possible to call close() more than once + self.loop.close() + self.loop.close() + + # operation blocked when the loop is closed + f = self.loop.create_future() + self.assertRaises(RuntimeError, self.loop.run_forever) + self.assertRaises(RuntimeError, self.loop.run_until_complete, f) + fd = 0 + def callback(): + pass + self.assertRaises(RuntimeError, self.loop.add_reader, fd, callback) + self.assertRaises(RuntimeError, self.loop.add_writer, fd, callback) + + def test_close_no_selector(self): + self.loop.remove_reader = mock.Mock() + self.loop._selector.close() + self.loop._selector = None + self.loop.close() + self.assertIsNone(self.loop._selector) + + def test_read_from_self_tryagain(self): + self.loop._ssock.recv.side_effect = BlockingIOError + self.assertIsNone(self.loop._read_from_self()) + + def test_read_from_self_exception(self): + self.loop._ssock.recv.side_effect = OSError + self.assertRaises(OSError, self.loop._read_from_self) + + def test_write_to_self_tryagain(self): + self.loop._csock.send.side_effect = BlockingIOError + with test_utils.disable_logger(): + self.assertIsNone(self.loop._write_to_self()) + + def test_write_to_self_exception(self): + # _write_to_self() swallows OSError + self.loop._csock.send.side_effect = RuntimeError() + self.assertRaises(RuntimeError, self.loop._write_to_self) + + @mock.patch('socket.getaddrinfo') + def test_sock_connect_resolve_using_socket_params(self, m_gai): + addr = ('need-resolution.com', 8080) + for sock_type in [socket.SOCK_STREAM, socket.SOCK_DGRAM]: + with self.subTest(sock_type): + sock = test_utils.mock_nonblocking_socket(type=sock_type) + + m_gai.side_effect = \ + lambda *args: [(None, None, None, None, ('127.0.0.1', 0))] + + con = self.loop.create_task(self.loop.sock_connect(sock, addr)) + self.loop.run_until_complete(con) + m_gai.assert_called_with( + addr[0], addr[1], sock.family, sock.type, sock.proto, 0) + + self.loop.run_until_complete(con) + sock.connect.assert_called_with(('127.0.0.1', 0)) + + def test_add_reader(self): + self.loop._selector.get_map.return_value = {} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertTrue(self.loop._selector.register.called) + fd, mask, (r, w) = self.loop._selector.register.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertIsNone(w) + + def test_add_reader_existing(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (reader, writer))} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertTrue(reader.cancel.called) + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertEqual(writer, w) + + def test_add_reader_existing_writer(self): + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, writer))} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertEqual(writer, w) + + def test_remove_reader(self): + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (None, None))} + self.assertFalse(self.loop.remove_reader(1)) + + self.assertTrue(self.loop._selector.unregister.called) + + def test_remove_reader_read_write(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, + (reader, writer))} + self.assertTrue( + self.loop.remove_reader(1)) + + self.assertFalse(self.loop._selector.unregister.called) + self.assertEqual( + (1, selectors.EVENT_WRITE, (None, writer)), + self.loop._selector.modify.call_args[0]) + + def test_remove_reader_unknown(self): + self.loop._selector.get_map.return_value = {} + self.assertFalse( + self.loop.remove_reader(1)) + + def test_add_writer(self): + self.loop._selector.get_map.return_value = {} + cb = lambda: True + self.loop.add_writer(1, cb) + + self.assertTrue(self.loop._selector.register.called) + fd, mask, (r, w) = self.loop._selector.register.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE, mask) + self.assertIsNone(r) + self.assertEqual(cb, w._callback) + + def test_add_writer_existing(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, writer))} + cb = lambda: True + self.loop.add_writer(1, cb) + + self.assertTrue(writer.cancel.called) + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(reader, r) + self.assertEqual(cb, w._callback) + + def test_remove_writer(self): + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, None))} + self.assertFalse(self.loop.remove_writer(1)) + + self.assertTrue(self.loop._selector.unregister.called) + + def test_remove_writer_read_write(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, + (reader, writer))} + self.assertTrue( + self.loop.remove_writer(1)) + + self.assertFalse(self.loop._selector.unregister.called) + self.assertEqual( + (1, selectors.EVENT_READ, (reader, None)), + self.loop._selector.modify.call_args[0]) + + def test_remove_writer_unknown(self): + self.loop._selector.get_map.return_value = {} + self.assertFalse( + self.loop.remove_writer(1)) + + def test_process_events_read(self): + reader = mock.Mock() + reader._cancelled = False + + self.loop._add_callback = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, None)), + selectors.EVENT_READ)]) + self.assertTrue(self.loop._add_callback.called) + self.loop._add_callback.assert_called_with(reader) + + def test_process_events_read_cancelled(self): + reader = mock.Mock() + reader.cancelled = True + + self.loop._remove_reader = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, None)), + selectors.EVENT_READ)]) + self.loop._remove_reader.assert_called_with(1) + + def test_process_events_write(self): + writer = mock.Mock() + writer._cancelled = False + + self.loop._add_callback = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE, + (None, writer)), + selectors.EVENT_WRITE)]) + self.loop._add_callback.assert_called_with(writer) + + def test_process_events_write_cancelled(self): + writer = mock.Mock() + writer.cancelled = True + self.loop._remove_writer = mock.Mock() + + self.loop._process_events( + [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE, + (None, writer)), + selectors.EVENT_WRITE)]) + self.loop._remove_writer.assert_called_with(1) + + def test_accept_connection_zero_one(self): + for backlog in [0, 1]: + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + with self.subTest(backlog): + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + + def test_accept_connection_multiple(self): + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + backlog = 100 + # Mock the coroutine generation for a connection to prevent + # warnings related to un-awaited coroutines. _accept_connection2 + # is an async function that is patched with AsyncMock. create_task + # creates a task out of coroutine returned by AsyncMock, so use + # asyncio.sleep(0) to ensure created tasks are complete to avoid + # task pending warnings. + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + + def test_accept_connection_skip_connectionabortederror(self): + sock = mock.Mock() + + def mock_sock_accept(): + # mock accept(2) returning -ECONNABORTED every-other + # time that it's called. This applies most to OpenBSD + # whose sockets generate this errno more reproducibly than + # Linux and other OS. + if sock.accept.call_count % 2 == 0: + raise ConnectionAbortedError + return (mock.Mock(), mock.Mock()) + + sock.accept.side_effect = mock_sock_accept + backlog = 100 + # test that _accept_connection's loop calls sock.accept + # all 100 times, continuing past ConnectionAbortedError + # instead of unnecessarily returning early + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + # as in test_accept_connection_multiple avoid task pending + # warnings by using asyncio.sleep(0) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + +class SelectorTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.sock.fileno.return_value = 7 + + def create_transport(self): + transport = _SelectorTransport(self.loop, self.sock, self.protocol, + None) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + tr = self.create_transport() + self.assertIs(tr._loop, self.loop) + self.assertIs(tr._sock, self.sock) + self.assertIs(tr._sock_fd, 7) + + def test_abort(self): + tr = self.create_transport() + tr._force_close = mock.Mock() + + tr.abort() + tr._force_close.assert_called_with(None) + + def test_close(self): + tr = self.create_transport() + tr.close() + + self.assertTrue(tr.is_closing()) + self.assertEqual(1, self.loop.remove_reader_count[7]) + self.protocol.connection_lost(None) + self.assertEqual(tr._conn_lost, 1) + + tr.close() + self.assertEqual(tr._conn_lost, 1) + self.assertEqual(1, self.loop.remove_reader_count[7]) + + def test_close_write_buffer(self): + tr = self.create_transport() + tr._buffer.extend(b'data') + tr.close() + + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_force_close(self): + tr = self.create_transport() + tr._buffer.extend(b'1') + self.loop._add_reader(7, mock.sentinel) + self.loop._add_writer(7, mock.sentinel) + tr._force_close(None) + + self.assertTrue(tr.is_closing()) + self.assertEqual(tr._buffer, list_to_buffer()) + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + + # second close should not remove reader + tr._force_close(None) + self.assertFalse(self.loop.readers) + self.assertEqual(1, self.loop.remove_reader_count[7]) + + @mock.patch('asyncio.log.logger.error') + def test_fatal_error(self, m_exc): + exc = OSError() + tr = self.create_transport() + tr._force_close = mock.Mock() + tr._fatal_error(exc) + + m_exc.assert_not_called() + + tr._force_close.assert_called_with(exc) + + @mock.patch('asyncio.log.logger.error') + def test_fatal_error_custom_exception(self, m_exc): + class MyError(Exception): + pass + exc = MyError() + tr = self.create_transport() + tr._force_close = mock.Mock() + tr._fatal_error(exc) + + m_exc.assert_called_with( + test_utils.MockPattern( + 'Fatal error on transport\nprotocol:.*\ntransport:.*'), + exc_info=(MyError, MOCK_ANY, MOCK_ANY)) + + tr._force_close.assert_called_with(exc) + + def test_connection_lost(self): + exc = OSError() + tr = self.create_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + tr._call_connection_lost(exc) + + self.protocol.connection_lost.assert_called_with(exc) + self.sock.close.assert_called_with() + self.assertIsNone(tr._sock) + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__add_reader(self): + tr = self.create_transport() + tr._buffer.extend(b'1') + tr._add_reader(7, mock.sentinel) + self.assertTrue(self.loop.readers) + + tr._force_close(None) + + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + + # can not add readers after closing + tr._add_reader(7, mock.sentinel) + self.assertFalse(self.loop.readers) + + +class SelectorSocketTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.sock_fd = self.sock.fileno.return_value = 7 + + def socket_transport(self, waiter=None, sendmsg=False): + transport = _SelectorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + if sendmsg: + transport._write_ready = transport._write_sendmsg + else: + transport._write_ready = transport._write_send + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.loop.assert_reader(7, tr._read_ready) + test_utils.run_briefly(self.loop) + self.protocol.connection_made.assert_called_with(tr) + + def test_ctor_with_waiter(self): + waiter = self.loop.create_future() + self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.assertIsNone(waiter.result()) + + def test_pause_resume_reading(self): + tr = self.socket_transport() + test_utils.run_briefly(self.loop) + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.pause_reading() + tr.pause_reading() + self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + tr.resume_reading() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.close() + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + def test_pause_reading_connection_made(self): + tr = self.socket_transport() + self.protocol.connection_made.side_effect = lambda _: tr.pause_reading() + test_utils.run_briefly(self.loop) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.close() + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv.return_value = b'' + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_data_received_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.data_received.side_effect = LookupError() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.data_received.called) + + def test_read_ready(self): + transport = self.socket_transport() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.protocol.data_received.assert_called_with(b'data') + + def test_read_ready_eof(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv.return_value = b'' + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + transport.close.assert_called_with() + + def test_read_ready_eof_keep_open(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv.return_value = b'' + self.protocol.eof_received.return_value = True + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertFalse(transport.close.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain(self, m_exc): + self.sock.recv.side_effect = BlockingIOError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain_interrupted(self, m_exc): + self.sock.recv.side_effect = InterruptedError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_conn_reset(self, m_exc): + err = self.sock.recv.side_effect = ConnectionResetError() + + transport = self.socket_transport() + transport._force_close = mock.Mock() + with test_utils.disable_logger(): + transport._read_ready() + transport._force_close.assert_called_with(err) + + @mock.patch('logging.exception') + def test_read_ready_err(self, m_exc): + err = self.sock.recv.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on socket transport') + + def test_write(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + + def test_write_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + + def test_write_no_data(self): + transport = self.socket_transport() + transport._buffer.append(memoryview(b'data')) + transport.write(b'') + self.assertFalse(self.sock.send.called) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_buffer(self): + transport = self.socket_transport() + transport._buffer.append(b'data1') + transport.write(b'data2') + self.assertFalse(self.sock.send.called) + self.assertEqual(list_to_buffer([b'data1', b'data2']), + transport._buffer) + + def test_write_partial(self): + data = b'data' + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_partial_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_partial_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_partial_none(self): + data = b'data' + self.sock.send.return_value = 0 + self.sock.fileno.return_value = 7 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_tryagain(self): + self.sock.send.side_effect = BlockingIOError + + data = b'data' + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_sendmsg_no_data(self): + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = 0 + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(memoryview(b'data')) + transport.write(b'') + self.assertFalse(self.sock.sendmsg.called) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_writelines_sendmsg_full(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = len(data) + + transport = self.socket_transport(sendmsg=True) + transport.writelines([data]) + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_writelines_sendmsg_partial(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport.writelines([data]) + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + + def test_writelines_send_full(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport.writelines([data]) + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + + def test_writelines_send_partial(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport.writelines([data]) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + + def test_writelines_pauses_protocol(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport._high_water = 1 + transport.writelines([data]) + self.assertTrue(self.protocol.pause_writing.called) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + + def test_writelines_after_connection_lost(self): + # GH-136234 + transport = self.socket_transport() + self.sock.send = mock.Mock() + self.sock.send.side_effect = ConnectionResetError + transport.write(b'data1') # Will fail immediately, causing connection lost + + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + + test_utils.run_briefly(self.loop) # Allow _call_connection_lost to run + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_full(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = len(data) + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_partial(self): + + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + # Sent partial data + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_half_buffer(self): + data = [memoryview(b'data1'), memoryview(b'data2')] + self.sock.sendmsg = mock.Mock() + # Sent partial data + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.extend(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + self.assertEqual(list_to_buffer([b'ta1', b'data2']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_OSError(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + err = self.sock.sendmsg.side_effect = OSError() + + transport = self.socket_transport(sendmsg=True) + transport._fatal_error = mock.Mock() + transport._buffer.extend(data) + # Calls _fatal_error and clears the buffer + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + self.assertEqual(list_to_buffer([]), transport._buffer) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + + @mock.patch('asyncio.selector_events.logger') + def test_write_exception(self, m_log): + err = self.sock.send.side_effect = OSError() + + data = b'data' + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport.write(data) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + transport._conn_lost = 1 + + self.sock.reset_mock() + transport.write(data) + self.assertFalse(self.sock.send.called) + self.assertEqual(transport._conn_lost, 2) + transport.write(data) + transport.write(data) + transport.write(data) + transport.write(data) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_write_str(self): + transport = self.socket_transport() + self.assertRaises(TypeError, transport.write, 'str') + + def test_write_closing(self): + transport = self.socket_transport() + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.write(b'data') + self.assertEqual(transport._conn_lost, 2) + + def test_write_ready(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + + def test_write_ready_closing(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport._closing = True + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_write_ready_no_data(self): + transport = self.socket_transport() + # This is an internal error. + self.assertRaises(AssertionError, transport._write_ready) + + def test_write_ready_partial(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_ready_partial_none(self): + data = b'data' + self.sock.send.return_value = 0 + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_ready_tryagain(self): + self.sock.send.side_effect = BlockingIOError + + transport = self.socket_transport() + buffer = list_to_buffer([b'data1', b'data2']) + transport._buffer = buffer + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(buffer, transport._buffer) + + def test_write_ready_exception(self): + err = self.sock.send.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._buffer.extend(b'data') + transport._write_ready() + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + + def test_write_eof(self): + tr = self.socket_transport() + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.write_eof() + self.assertEqual(self.sock.shutdown.call_count, 1) + tr.close() + + def test_write_eof_buffer(self): + tr = self.socket_transport() + self.sock.send.side_effect = BlockingIOError + tr.write(b'data') + tr.write_eof() + self.assertEqual(tr._buffer, list_to_buffer([b'data'])) + self.assertTrue(tr._eof) + self.assertFalse(self.sock.shutdown.called) + self.sock.send.side_effect = lambda _: 4 + tr._write_ready() + self.assertTrue(self.sock.send.called) + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.close() + + def test_write_eof_after_close(self): + tr = self.socket_transport() + tr.close() + self.loop.run_until_complete(asyncio.sleep(0)) + tr.write_eof() + + @mock.patch('asyncio.base_events.logger') + def test_transport_close_remove_writer(self, m_log): + remove_writer = self.loop._remove_writer = mock.Mock() + + transport = self.socket_transport() + transport.close() + remove_writer.assert_called_with(self.sock_fd) + + def test_write_buffer_after_close(self): + # gh-115514: If the transport is closed while: + # * Transport write buffer is not empty + # * Transport is paused + # * Protocol has data in its buffer, like SSLProtocol in self._outgoing + # The data is still written out. + + # Also tested with real SSL transport in + # test.test_asyncio.test_ssl.TestSSL.test_remote_shutdown_receives_trailing_data + + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + def _resume_writing(): + transport.write(b"data") + self.protocol.resume_writing.side_effect = None + + self.protocol.resume_writing.side_effect = _resume_writing + + transport = self.socket_transport() + transport._high_water = 1 + + transport.write(data) + + self.assertTrue(transport._protocol_paused) + self.assertTrue(self.sock.send.called) + self.loop.assert_writer(7, transport._write_ready) + + transport.close() + + # not called, we still have data in write buffer + self.assertFalse(self.protocol.connection_lost.called) + + self.loop.writers[7]._run() + # during this ^ run, the _resume_writing mock above was called and added more data + + self.assertEqual(transport.get_write_buffer_size(), 2) + self.loop.writers[7]._run() + + self.assertEqual(transport.get_write_buffer_size(), 0) + self.assertTrue(self.protocol.connection_lost.called) + +class SelectorSocketTransportBufferedProtocolTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol) + self.buf = bytearray(1) + self.protocol.get_buffer.side_effect = lambda hint: self.buf + + self.sock = mock.Mock(socket.socket) + self.sock_fd = self.sock.fileno.return_value = 7 + + def socket_transport(self, waiter=None): + transport = _SelectorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.loop.assert_reader(7, tr._read_ready) + test_utils.run_briefly(self.loop) + self.protocol.connection_made.assert_called_with(tr) + + def test_get_buffer_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = LookupError() + + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_get_buffer_zerosized(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = lambda hint: bytearray(0) + + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_proto_type_switch(self): + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + transport = self.socket_transport() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.protocol.data_received.assert_called_with(b'data') + + # switch protocol to a BufferedProtocol + + buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol) + buf = bytearray(4) + buf_proto.get_buffer.side_effect = lambda hint: buf + + transport.set_protocol(buf_proto) + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + buf_proto.get_buffer.assert_called_with(-1) + buf_proto.buffer_updated.assert_called_with(10) + + def test_buffer_updated_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.buffer_updated.side_effect = LookupError() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertTrue(self.protocol.buffer_updated.called) + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_read_ready(self): + transport = self.socket_transport() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.protocol.get_buffer.assert_called_with(-1) + self.protocol.buffer_updated.assert_called_with(10) + + def test_read_ready_eof(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + transport.close.assert_called_with() + + def test_read_ready_eof_keep_open(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + self.protocol.eof_received.return_value = True + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertFalse(transport.close.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain(self, m_exc): + self.sock.recv_into.side_effect = BlockingIOError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain_interrupted(self, m_exc): + self.sock.recv_into.side_effect = InterruptedError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_conn_reset(self, m_exc): + err = self.sock.recv_into.side_effect = ConnectionResetError() + + transport = self.socket_transport() + transport._force_close = mock.Mock() + with test_utils.disable_logger(): + transport._read_ready() + transport._force_close.assert_called_with(err) + + @mock.patch('logging.exception') + def test_read_ready_err(self, m_exc): + err = self.sock.recv_into.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on socket transport') + + +class SelectorDatagramTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + self.sock = mock.Mock(spec_set=socket.socket) + self.sock.fileno.return_value = 7 + + def datagram_transport(self, address=None): + self.sock.getpeername.side_effect = None if address else OSError + transport = _SelectorDatagramTransport(self.loop, self.sock, + self.protocol, + address=address) + self.addCleanup(close_transport, transport) + return transport + + def test_read_ready(self): + transport = self.datagram_transport() + + self.sock.recvfrom.return_value = (b'data', ('0.0.0.0', 1234)) + transport._read_ready() + + self.protocol.datagram_received.assert_called_with( + b'data', ('0.0.0.0', 1234)) + + def test_transport_inheritance(self): + transport = self.datagram_transport() + self.assertIsInstance(transport, asyncio.DatagramTransport) + + def test_read_ready_tryagain(self): + transport = self.datagram_transport() + + self.sock.recvfrom.side_effect = BlockingIOError + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + def test_read_ready_err(self): + transport = self.datagram_transport() + + err = self.sock.recvfrom.side_effect = RuntimeError() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on datagram transport') + + def test_read_ready_oserr(self): + transport = self.datagram_transport() + + err = self.sock.recvfrom.side_effect = OSError() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + self.protocol.error_received.assert_called_with(err) + + def test_sendto(self): + data = b'data' + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_bytearray(self): + data = bytearray(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_memoryview(self): + data = memoryview(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_no_data(self): + transport = self.datagram_transport() + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) + + def test_sendto_buffer(self): + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(b'data2', ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + + def test_sendto_buffer_bytearray(self): + data2 = bytearray(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_memoryview(self): + data2 = memoryview(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_tryagain(self): + data = b'data' + + self.sock.sendto.side_effect = BlockingIOError + + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual( + [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + + @mock.patch('asyncio.selector_events.logger') + def test_sendto_exception(self, m_log): + data = b'data' + err = self.sock.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertTrue(transport._fatal_error.called) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + transport._conn_lost = 1 + + transport._address = ('123',) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_sendto_error_received(self): + data = b'data' + + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertEqual(transport._conn_lost, 0) + self.assertFalse(transport._fatal_error.called) + + def test_sendto_error_received_connected(self): + data = b'data' + + self.sock.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport.sendto(data) + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + def test_sendto_str(self): + transport = self.datagram_transport() + self.assertRaises(TypeError, transport.sendto, 'str', ()) + + def test_sendto_connected_addr(self): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + self.assertRaises( + ValueError, transport.sendto, b'str', ('0.0.0.0', 2)) + + def test_sendto_closing(self): + transport = self.datagram_transport(address=(1,)) + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.sendto(b'data', (1,)) + self.assertEqual(transport._conn_lost, 2) + + def test_sendto_sendto_ready(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now let the socket send the buffer + self.sock.sendto.side_effect = None + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + self.assertFalse(transport._buffer) + self.assertEqual(transport._buffer_size, 0) + + def test_sendto_sendto_ready_blocked(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now try to send the buffer, it will be added to buffer again if it fails + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertTrue(self.loop.writers) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + def test_sendto_ready(self): + data = b'data' + self.sock.sendto.return_value = len(data) + + transport = self.datagram_transport() + transport._buffer.append((data, ('0.0.0.0', 12345))) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + + def test_sendto_ready_closing(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.datagram_transport() + transport._closing = True + transport._buffer.append((data, ())) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.sock.sendto.assert_called_with(data, ()) + self.assertFalse(self.loop.writers) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + def test_sendto_ready_no_data(self): + transport = self.datagram_transport() + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.assertFalse(self.sock.sendto.called) + self.assertFalse(self.loop.writers) + + def test_sendto_ready_tryagain(self): + self.sock.sendto.side_effect = BlockingIOError + + transport = self.datagram_transport() + transport._buffer.extend([(b'data1', ()), (b'data2', ())]) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual( + [(b'data1', ()), (b'data2', ())], + list(transport._buffer)) + + def test_sendto_ready_exception(self): + err = self.sock.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + + def test_sendto_ready_error_received(self): + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + self.assertFalse(transport._fatal_error.called) + + def test_sendto_ready_error_received_connection(self): + self.sock.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected(self, m_exc): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = ConnectionRefusedError() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_not_called() + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected_custom_error(self, m_exc): + class MyException(Exception): + pass + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = MyException() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_called_with( + test_utils.MockPattern( + 'Fatal error on transport\nprotocol:.*\ntransport:.*'), + exc_info=(MyException, MOCK_ANY, MOCK_ANY)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py new file mode 100644 index 00000000000..dcd963b3355 --- /dev/null +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -0,0 +1,585 @@ +"""Tests for sendfile functionality.""" + +import asyncio +import errno +import os +import socket +import sys +import tempfile +import unittest +from asyncio import base_events +from asyncio import constants +from unittest import mock +from test import support +from test.support import os_helper +from test.support import socket_helper +from test.test_asyncio import utils as test_utils + +try: + import ssl +except ImportError: + ssl = None + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MySendfileProto(asyncio.Protocol): + + def __init__(self, loop=None, close_after=0): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + self.data = bytearray() + self.close_after = close_after + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + self.data.extend(data) + super().data_received(data) + if self.close_after and self.nbytes >= self.close_after: + self.transport.close() + + +class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + +class SendfileBase: + + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) + # Reduce socket buffer size to test on relative small data sets. + BUF_SIZE = 4 * 1024 # 4 KiB + + def create_event_loop(self): + raise NotImplementedError + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + super().setUp() + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + +class SockSendfileMixin(SendfileBase): + + @classmethod + def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 + super().setUpClass() + + @classmethod + def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize + super().tearDownClass() + + def make_socket(self, cleanup=True): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + if cleanup: + self.addCleanup(sock.close) + return sock + + def reduce_receive_buffer_size(self, sock): + # Reduce receive socket buffer size to test on relative + # small data sets. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.BUF_SIZE) + + def reduce_send_buffer_size(self, sock, transport=None): + # Reduce send socket buffer size to test on relative small data sets. + + # On macOS, SO_SNDBUF is reset by connect(). So this method + # should be called after the socket is connected. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.BUF_SIZE) + + if transport is not None: + transport.set_write_buffer_limits(high=self.BUF_SIZE) + + def prepare_socksendfile(self): + proto = MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.reduce_receive_buffer_size(srv_sock) + + sock = self.make_socket() + self.run_loop(self.loop.sock_connect(sock, ('127.0.0.1', port))) + self.reduce_send_buffer_size(sock) + + def cleanup(): + if proto.transport is not None: + # can be None if the task was cancelled before + # connection_made callback + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_success(self): + sock, proto = self.prepare_socksendfile() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sock_sendfile_with_offset_and_count(self): + sock, proto = self.prepare_socksendfile() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(proto.data, self.DATA[1000:3000]) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(ret, 2000) + + def test_sock_sendfile_zero_size(self): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, + 0, None)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_mix_with_regular_send(self): + buf = b"mix_regular_send" * (4 * 1024) # 64 KiB + sock, proto = self.prepare_socksendfile() + self.run_loop(self.loop.sock_sendall(sock, buf)) + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + self.run_loop(self.loop.sock_sendall(sock, buf)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + expected = buf + self.DATA + buf + self.assertEqual(proto.data, expected) + self.assertEqual(self.file.tell(), len(self.DATA)) + + +class SendfileMixin(SendfileBase): + + # Note: sendfile via SSL transport is equal to sendfile fallback + + def prepare_sendfile(self, *, is_ssl=False, close_after=0): + port = socket_helper.find_unused_port() + srv_proto = MySendfileProto(loop=self.loop, + close_after=close_after) + if is_ssl: + if not ssl: + self.skipTest("No ssl module") + srv_ctx = test_utils.simple_server_sslcontext() + cli_ctx = test_utils.simple_client_sslcontext() + else: + srv_ctx = None + cli_ctx = None + srv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: srv_proto, sock=srv_sock, ssl=srv_ctx)) + self.reduce_receive_buffer_size(srv_sock) + + if is_ssl: + server_hostname = socket_helper.HOST + else: + server_hostname = None + cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cli_sock.connect((socket_helper.HOST, port)) + + cli_proto = MySendfileProto(loop=self.loop) + tr, pr = self.run_loop(self.loop.create_connection( + lambda: cli_proto, sock=cli_sock, + ssl=cli_ctx, server_hostname=server_hostname)) + self.reduce_send_buffer_size(cli_sock, transport=tr) + + def cleanup(): + srv_proto.transport.close() + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.run_loop(cli_proto.done) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + return srv_proto, cli_proto + + @unittest.skipIf(sys.platform == 'win32', "UDP sockets are not supported") + def test_sendfile_not_supported(self): + tr, pr = self.run_loop( + self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + family=socket.AF_INET)) + try: + with self.assertRaisesRegex(RuntimeError, "not supported"): + self.run_loop( + self.loop.sendfile(tr, self.file)) + self.assertEqual(0, self.file.tell()) + finally: + # don't use self.addCleanup because it produces resource warning + tr.close() + + def test_sendfile(self): + srv_proto, cli_proto = self.prepare_sendfile() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_fallback(self): + srv_proto, cli_proto = self.prepare_sendfile() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_unsupported_native(self): + if sys.platform == 'win32': + if isinstance(self.loop, asyncio.ProactorEventLoop): + self.skipTest("Fails on proactor event loop") + srv_proto, cli_proto = self.prepare_sendfile() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not supported"): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, + fallback=False)) + + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_ssl(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_for_closing_transp(self): + srv_proto, cli_proto = self.prepare_sendfile() + cli_proto.transport.close() + with self.assertRaisesRegex(RuntimeError, "is closing"): + self.run_loop(self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare_sendfile() + PREFIX = b'PREFIX__' * 1024 # 8 KiB + SUFFIX = b'--SUFFIX' * 1024 # 8 KiB + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + PREFIX = b'zxcvbnm' * 1024 + SUFFIX = b'0987654321' * 1024 + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_partial(self): + srv_proto, cli_proto = self.prepare_sendfile() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_ssl_partial(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile( + close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile( + is_ssl=True, close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + # On Solaris, lowering SO_RCVBUF on a TCP connection after it has been + # established has no effect. Due to its age, this bug affects both Oracle + # Solaris as well as all other OpenSolaris forks (unless they fixed it + # themselves). + @unittest.skipIf(sys.platform.startswith('sunos'), + "Doesn't work on Solaris") + def test_sendfile_close_peer_in_the_middle_of_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) + with self.assertRaises(ConnectionError): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + if not (sys.platform == 'win32' + and isinstance(self.loop, asyncio.ProactorEventLoop)): + # On Windows, Proactor uses transmitFile, which does not update tell() + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + self.assertTrue(cli_proto.transport.is_closing()) + + def test_sendfile_fallback_close_peer_in_the_middle_of_receiving(self): + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) + with self.assertRaises(ConnectionError): + try: + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + except OSError as e: + # macOS may raise OSError of EPROTOTYPE when writing to a + # socket that is in the process of closing down. + if e.errno == errno.EPROTOTYPE and sys.platform == "darwin": + raise ConnectionError + else: + raise + + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + + @unittest.skipIf(not hasattr(os, 'sendfile'), + "Don't have native sendfile support") + def test_sendfile_prevents_bare_write(self): + srv_proto, cli_proto = self.prepare_sendfile() + fut = self.loop.create_future() + + async def coro(): + fut.set_result(None) + return await self.loop.sendfile(cli_proto.transport, self.file) + + t = self.loop.create_task(coro()) + self.run_loop(fut) + with self.assertRaisesRegex(RuntimeError, + "sendfile is in progress"): + cli_proto.transport.write(b'data') + ret = self.run_loop(t) + self.assertEqual(ret, len(self.DATA)) + + def test_sendfile_no_fallback_for_fallback_transport(self): + transport = mock.Mock() + transport.is_closing.side_effect = lambda: False + transport._sendfile_compatible = constants._SendfileMode.FALLBACK + with self.assertRaisesRegex(RuntimeError, 'fallback is disabled'): + self.loop.run_until_complete( + self.loop.sendfile(transport, None, fallback=False)) + + +class SendfileTestsBase(SendfileMixin, SockSendfileMixin): + pass + + +if sys.platform == 'win32': + + class SelectEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + class ProactorEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py new file mode 100644 index 00000000000..5bd0f7e2af4 --- /dev/null +++ b/Lib/test/test_asyncio/test_server.py @@ -0,0 +1,352 @@ +import asyncio +import os +import socket +import time +import threading +import unittest + +from test.support import socket_helper +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class BaseStartServer(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_start_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + + def client(sock, addr): + for i in range(10): + time.sleep(0.2) + if srv.is_serving(): + break + else: + raise RuntimeError + + sock.settimeout(2) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + await srv.serve_forever() + + srv = self.loop.run_until_complete(asyncio.start_server( + serve, socket_helper.HOSTv4, 0, start_serving=False)) + + self.assertFalse(srv.is_serving()) + + main_task = self.loop.create_task(main(srv)) + + addr = srv.sockets[0].getsockname() + with self.assertRaises(asyncio.CancelledError): + with self.tcp_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, ()) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +class SelectorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + @socket_helper.skip_unless_bind_unix_socket + def test_start_unix_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + started = threading.Event() + + def client(sock, addr): + sock.settimeout(2) + started.wait(5) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + self.assertFalse(srv.is_serving()) + await srv.start_serving() + self.assertTrue(srv.is_serving()) + started.set() + await srv.serve_forever() + + with test_utils.unix_socket_path() as addr: + srv = self.loop.run_until_complete(asyncio.start_unix_server( + serve, addr, start_serving=False)) + + main_task = self.loop.create_task(main(srv)) + + with self.assertRaises(asyncio.CancelledError): + with self.unix_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, ()) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +class TestServer2(unittest.IsolatedAsyncioTestCase): + + async def test_wait_closed_basic(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + # active count = 0, not closed: should block + task1 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + + # active count != 0, not closed: should block + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + task2 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + self.assertFalse(task2.done()) + + srv.close() + await asyncio.sleep(0) + # active count != 0, closed: should block + task3 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + self.assertFalse(task2.done()) + self.assertFalse(task3.done()) + + wr.close() + await wr.wait_closed() + # active count == 0, closed: should unblock + await task1 + await task2 + await task3 + await srv.wait_closed() # Return immediately + + async def test_wait_closed_race(self): + # Test a regression in 3.12.0, should be fixed in 3.12.1 + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + loop = asyncio.get_running_loop() + loop.call_soon(srv.close) + loop.call_soon(wr.close) + await srv.wait_closed() + + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can more reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # kernel stops accepting more data in the send buffer. + # gh-122136: getsockopt() does not reliably report the buffer size + # available for message content. + # We loop until we start filling up the asyncio buffer. + # To avoid an infinite loop we cap at 10 times the expected value + c_bufsize = c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + s_bufsize = s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + for i in range(10): + s_wr.write(b'a' * c_bufsize) + s_wr.write(b'a' * s_bufsize) + if s_wr.transport.get_write_buffer_size() > 0: + break + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + +# Test the various corner cases of Unix server socket removal +class UnixServerCleanupTests(unittest.IsolatedAsyncioTestCase): + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_addr_cleanup(self): + # Default scenario + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_sock_cleanup(self): + # Using already bound socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_gone(self): + # Someone else has already cleaned up the socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + os.unlink(addr) + + srv.close() + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_replaced(self): + # Someone else has replaced the socket with their own + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + os.unlink(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv.close() + self.assertTrue(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_prevented(self): + # Automatic cleanup explicitly disabled + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr, cleanup_socket=False) + + srv.close() + self.assertTrue(os.path.exists(addr)) + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py new file mode 100644 index 00000000000..df4ec794897 --- /dev/null +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -0,0 +1,679 @@ +import socket +import asyncio +import sys +import unittest + +from asyncio import proactor_events +from itertools import cycle, islice +from unittest.mock import Mock +from test.test_asyncio import utils as test_utils +from test import support +from test.support import socket_helper + +if socket_helper.tcp_blackhole(): + raise unittest.SkipTest('Not relevant to ProactorEventLoop') + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class BaseSockTestsMixin: + + def create_event_loop(self): + raise NotImplementedError + + def setUp(self): + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + super().setUp() + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def _basetest_sock_client_ops(self, httpd, sock): + if not isinstance(self.loop, proactor_events.BaseProactorEventLoop): + # in debug mode, socket operations must fail + # if the socket is not in blocking mode + self.loop.set_debug(True) + sock.setblocking(True) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_recv_into(sock, bytearray())) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_accept(sock)) + + # test in non-blocking mode + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + # consume data + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + sock.close() + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + def _basetest_sock_recv_into(self, httpd, sock): + # same as _basetest_sock_client_ops, but using sock_recv_into + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = bytearray(1024) + with memoryview(data) as buf: + nbytes = self.loop.run_until_complete( + self.loop.sock_recv_into(sock, buf[:1024])) + # consume data + self.loop.run_until_complete( + self.loop.sock_recv_into(sock, buf[nbytes:])) + sock.close() + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + def test_sock_client_ops(self): + with test_utils.run_test_server() as httpd: + sock = socket.socket() + self._basetest_sock_client_ops(httpd, sock) + sock = socket.socket() + self._basetest_sock_recv_into(httpd, sock) + + async def _basetest_sock_recv_racing(self, httpd, sock): + sock.setblocking(False) + await self.loop.sock_connect(sock, httpd.address) + + task = asyncio.create_task(self.loop.sock_recv(sock, 1024)) + await asyncio.sleep(0) + task.cancel() + + asyncio.create_task( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = await self.loop.sock_recv(sock, 1024) + # consume data + await self.loop.sock_recv(sock, 1024) + + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + async def _basetest_sock_recv_into_racing(self, httpd, sock): + sock.setblocking(False) + await self.loop.sock_connect(sock, httpd.address) + + data = bytearray(1024) + with memoryview(data) as buf: + task = asyncio.create_task( + self.loop.sock_recv_into(sock, buf[:1024])) + await asyncio.sleep(0) + task.cancel() + + task = asyncio.create_task( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + nbytes = await self.loop.sock_recv_into(sock, buf[:1024]) + # consume data + await self.loop.sock_recv_into(sock, buf[nbytes:]) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + await task + + async def _basetest_sock_send_racing(self, listener, sock): + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + + # make connection + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setblocking(False) + task = asyncio.create_task( + self.loop.sock_connect(sock, listener.getsockname())) + await asyncio.sleep(0) + server = listener.accept()[0] + server.setblocking(False) + + with server: + await task + + # fill the buffer until sending 5 chars would block + size = 8192 + while size >= 4: + with self.assertRaises(BlockingIOError): + while True: + sock.send(b' ' * size) + size = int(size / 2) + + # cancel a blocked sock_sendall + task = asyncio.create_task( + self.loop.sock_sendall(sock, b'hello')) + await asyncio.sleep(0) + task.cancel() + + # receive everything that is not a space + async def recv_all(): + rv = b'' + while True: + buf = await self.loop.sock_recv(server, 8192) + if not buf: + return rv + rv += buf.strip() + task = asyncio.create_task(recv_all()) + + # immediately make another sock_sendall call + await self.loop.sock_sendall(sock, b'world') + sock.shutdown(socket.SHUT_WR) + data = await task + # ProactorEventLoop could deliver hello, so endswith is necessary + self.assertEndsWith(data, b'world') + + # After the first connect attempt before the listener is ready, + # the socket needs time to "recover" to make the next connect call. + # On Linux, a second retry will do. On Windows, the waiting time is + # unpredictable; and on FreeBSD the socket may never come back + # because it's a loopback address. Here we'll just retry for a few + # times, and have to skip the test if it's not working. See also: + # https://stackoverflow.com/a/54437602/3316267 + # https://lists.freebsd.org/pipermail/freebsd-current/2005-May/049876.html + async def _basetest_sock_connect_racing(self, listener, sock): + listener.bind(('127.0.0.1', 0)) + addr = listener.getsockname() + sock.setblocking(False) + + task = asyncio.create_task(self.loop.sock_connect(sock, addr)) + await asyncio.sleep(0) + task.cancel() + + listener.listen(1) + + skip_reason = "Max retries reached" + for i in range(128): + try: + await self.loop.sock_connect(sock, addr) + except ConnectionRefusedError as e: + skip_reason = e + except OSError as e: + skip_reason = e + + # Retry only for this error: + # [WinError 10022] An invalid argument was supplied + if getattr(e, 'winerror', 0) != 10022: + break + else: + # success + return + + self.skipTest(skip_reason) + + def test_sock_client_racing(self): + with test_utils.run_test_server() as httpd: + sock = socket.socket() + with sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_recv_racing(httpd, sock), 10)) + sock = socket.socket() + with sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_recv_into_racing(httpd, sock), 10)) + listener = socket.socket() + sock = socket.socket() + with listener, sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_send_racing(listener, sock), 10)) + + def test_sock_client_connect_racing(self): + listener = socket.socket() + sock = socket.socket() + with listener, sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_connect_racing(listener, sock), 10)) + + async def _basetest_huge_content(self, address): + sock = socket.socket() + sock.setblocking(False) + DATA_SIZE = 10_000_00 + + chunk = b'0123456789' * (DATA_SIZE // 10) + + await self.loop.sock_connect(sock, address) + await self.loop.sock_sendall(sock, + (b'POST /loop HTTP/1.0\r\n' + + b'Content-Length: %d\r\n' % DATA_SIZE + + b'\r\n')) + + task = asyncio.create_task(self.loop.sock_sendall(sock, chunk)) + + data = await self.loop.sock_recv(sock, DATA_SIZE) + # HTTP headers size is less than MTU, + # they are sent by the first packet always + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + while data.find(b'\r\n\r\n') == -1: + data += await self.loop.sock_recv(sock, DATA_SIZE) + # Strip headers + headers = data[:data.index(b'\r\n\r\n') + 4] + data = data[len(headers):] + + size = DATA_SIZE + checker = cycle(b'0123456789') + + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + + while True: + data = await self.loop.sock_recv(sock, DATA_SIZE) + if not data: + break + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + self.assertEqual(size, 0) + + await task + sock.close() + + def test_huge_content(self): + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete( + self._basetest_huge_content(httpd.address)) + + async def _basetest_huge_content_recvinto(self, address): + sock = socket.socket() + sock.setblocking(False) + DATA_SIZE = 10_000_00 + + chunk = b'0123456789' * (DATA_SIZE // 10) + + await self.loop.sock_connect(sock, address) + await self.loop.sock_sendall(sock, + (b'POST /loop HTTP/1.0\r\n' + + b'Content-Length: %d\r\n' % DATA_SIZE + + b'\r\n')) + + task = asyncio.create_task(self.loop.sock_sendall(sock, chunk)) + + array = bytearray(DATA_SIZE) + buf = memoryview(array) + + nbytes = await self.loop.sock_recv_into(sock, buf) + data = bytes(buf[:nbytes]) + # HTTP headers size is less than MTU, + # they are sent by the first packet always + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + while data.find(b'\r\n\r\n') == -1: + nbytes = await self.loop.sock_recv_into(sock, buf) + data = bytes(buf[:nbytes]) + # Strip headers + headers = data[:data.index(b'\r\n\r\n') + 4] + data = data[len(headers):] + + size = DATA_SIZE + checker = cycle(b'0123456789') + + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + + while True: + nbytes = await self.loop.sock_recv_into(sock, buf) + data = buf[:nbytes] + if not data: + break + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + self.assertEqual(size, 0) + + await task + sock.close() + + def test_huge_content_recvinto(self): + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete( + self._basetest_huge_content_recvinto(httpd.address)) + + async def _basetest_datagram_recvfrom(self, server_address): + # Happy path, sock.sendto() returns immediately + data = b'\x01' * 4096 + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + await self.loop.sock_sendto(sock, data, server_address) + received_data, from_addr = await self.loop.sock_recvfrom( + sock, 4096) + self.assertEqual(received_data, data) + self.assertEqual(from_addr, server_address) + + def test_recvfrom(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom(server_address)) + + async def _basetest_datagram_recvfrom_into(self, server_address): + # Happy path, sock.sendto() returns immediately + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + + buf = bytearray(4096) + data = b'\x01' * 4096 + await self.loop.sock_sendto(sock, data, server_address) + num_bytes, from_addr = await self.loop.sock_recvfrom_into( + sock, buf) + self.assertEqual(num_bytes, 4096) + self.assertEqual(buf, data) + self.assertEqual(from_addr, server_address) + + buf = bytearray(8192) + await self.loop.sock_sendto(sock, data, server_address) + num_bytes, from_addr = await self.loop.sock_recvfrom_into( + sock, buf, 4096) + self.assertEqual(num_bytes, 4096) + self.assertEqual(buf[:4096], data[:4096]) + self.assertEqual(from_addr, server_address) + + def test_recvfrom_into(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom_into(server_address)) + + async def _basetest_datagram_sendto_blocking(self, server_address): + # Sad path, sock.sendto() raises BlockingIOError + # This involves patching sock.sendto() to raise BlockingIOError but + # sendto() is not used by the proactor event loop + data = b'\x01' * 4096 + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + mock_sock = Mock(sock) + mock_sock.gettimeout = sock.gettimeout + mock_sock.sendto.configure_mock(side_effect=BlockingIOError) + mock_sock.fileno = sock.fileno + self.loop.call_soon( + lambda: setattr(mock_sock, 'sendto', sock.sendto) + ) + await self.loop.sock_sendto(mock_sock, data, server_address) + + received_data, from_addr = await self.loop.sock_recvfrom( + sock, 4096) + self.assertEqual(received_data, data) + self.assertEqual(from_addr, server_address) + + def test_sendto_blocking(self): + if sys.platform == 'win32': + if isinstance(self.loop, asyncio.ProactorEventLoop): + raise unittest.SkipTest('Not relevant to ProactorEventLoop') + + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_sendto_blocking(server_address)) + + @socket_helper.skip_unless_bind_unix_socket + def test_unix_sock_client_ops(self): + with test_utils.run_test_unix_server() as httpd: + sock = socket.socket(socket.AF_UNIX) + self._basetest_sock_client_ops(httpd, sock) + sock = socket.socket(socket.AF_UNIX) + self._basetest_sock_recv_into(httpd, sock) + + def test_sock_client_fail(self): + # Make sure that we will get an unused port + address = None + try: + s = socket.socket() + s.bind(('127.0.0.1', 0)) + address = s.getsockname() + finally: + s.close() + + sock = socket.socket() + sock.setblocking(False) + with self.assertRaises(ConnectionRefusedError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + sock.close() + + def test_sock_accept(self): + listener = socket.socket() + listener.setblocking(False) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + client = socket.socket() + client.connect(listener.getsockname()) + + f = self.loop.sock_accept(listener) + conn, addr = self.loop.run_until_complete(f) + self.assertEqual(conn.gettimeout(), 0) + self.assertEqual(addr, client.getsockname()) + self.assertEqual(client.getpeername(), listener.getsockname()) + client.close() + conn.close() + listener.close() + + def test_cancel_sock_accept(self): + listener = socket.socket() + listener.setblocking(False) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + sockaddr = listener.getsockname() + f = asyncio.wait_for(self.loop.sock_accept(listener), 0.1) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(f) + + listener.close() + client = socket.socket() + client.setblocking(False) + f = self.loop.sock_connect(client, sockaddr) + with self.assertRaises(ConnectionRefusedError): + self.loop.run_until_complete(f) + + client.close() + + def test_create_connection_sock(self): + with test_utils.run_test_server() as httpd: + sock = None + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *httpd.address, type=socket.SOCK_STREAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + except BaseException: + pass + else: + break + else: + self.fail('Can not create socket.') + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + +if sys.platform == 'win32': + + class SelectEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + + class ProactorEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + + + async def _basetest_datagram_send_to_non_listening_address(self, + recvfrom): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages + # after sending a message to an address that wasn't listening. + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address + # that is not listening + socket_3 = create_socket() + addr_3 = socket_3.getsockname() + socket_3.shutdown(socket.SHUT_RDWR) + socket_3.close() + + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + socket_2_recv_task = self.loop.create_task(recvfrom(socket_2)) + await asyncio.sleep(0) + + await self.loop.sock_sendto(socket_1, b'a', addr_2) + self.assertEqual(await socket_2_recv_task, b'a') + + await self.loop.sock_sendto(socket_2, b'b', addr_1) + self.assertEqual(await socket_1_recv_task, b'b') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # this should send to an address that isn't listening + await self.loop.sock_sendto(socket_1, b'c', addr_3) + self.assertEqual(await socket_1_recv_task, b'') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # socket 1 should still be able to receive messages after sending + # to an address that wasn't listening + socket_2.sendto(b'd', addr_1) + self.assertEqual(await socket_1_recv_task, b'd') + + socket_1.shutdown(socket.SHUT_RDWR) + socket_1.close() + socket_2.shutdown(socket.SHUT_RDWR) + socket_2.close() + + + def test_datagram_send_to_non_listening_address_recvfrom(self): + async def recvfrom(socket): + data, _ = await self.loop.sock_recvfrom(socket, 4096) + return data + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom)) + + + def test_datagram_send_to_non_listening_address_recvfrom_into(self): + async def recvfrom_into(socket): + buf = bytearray(4096) + length, _ = await self.loop.sock_recvfrom_into(socket, buf, + 4096) + return buf[:length] + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom_into)) + +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py new file mode 100644 index 00000000000..e5d3b63b94f --- /dev/null +++ b/Lib/test/test_asyncio/test_ssl.py @@ -0,0 +1,1906 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + +import asyncio +import contextlib +import gc +import logging +import select +import socket +import sys +import tempfile +import threading +import time +import unittest.mock +import weakref +import unittest + +try: + import ssl +except ImportError: + ssl = None + +from test import support +from test.test_asyncio import utils as test_utils + + +MACOS = (sys.platform == 'darwin') +BUF_MULTIPLIER = 1024 if not MACOS else 64 + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyBaseProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = asyncio.Future(loop=loop) + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def data_received(self, data): + assert self.state == 'CONNECTED', self.state + self.nbytes += len(data) + + def eof_received(self): + assert self.state == 'CONNECTED', self.state + self.state = 'EOF' + + def connection_lost(self, exc): + assert self.state in ('CONNECTED', 'EOF'), self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MessageOutFilter(logging.Filter): + def __init__(self, msg): + self.msg = msg + + def filter(self, record): + if self.msg in record.msg: + return False + return True + + +@unittest.skipIf(ssl is None, 'No ssl module') +class TestSSL(test_utils.TestCase): + + PAYLOAD_SIZE = 1024 * 100 + TIMEOUT = support.LONG_TIMEOUT + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + self.addCleanup(self.loop.close) + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=support.SHORT_TIMEOUT, + backlog=1, + max_clients=10): + + if addr is None: + if family == getattr(socket, "AF_UNIX", None): + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + try: + sock.bind(addr) + sock.listen(backlog) + except OSError as ex: + sock.close() + raise ex + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=support.SHORT_TIMEOUT): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + def _create_server_ssl_context(self, certfile, keyfile=None): + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.load_cert_chain(certfile, keyfile) + return sslcontext + + def _create_client_ssl_context(self, *, disable_verify=True): + sslcontext = ssl.create_default_context() + sslcontext.check_hostname = False + if disable_verify: + sslcontext.verify_mode = ssl.CERT_NONE + return sslcontext + + @contextlib.contextmanager + def _silence_eof_received_warning(self): + # TODO This warning has to be fixed in asyncio. + logger = logging.getLogger('asyncio') + filter = MessageOutFilter('has no effect when using ssl') + logger.addFilter(filter) + try: + yield + finally: + logger.removeFilter(filter) + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + def new_loop(self): + return asyncio.new_event_loop() + + def new_policy(self): + return asyncio.DefaultEventLoopPolicy() + + async def wait_closed(self, obj): + if not isinstance(obj, asyncio.StreamWriter): + return + try: + await obj.wait_closed() + except (BrokenPipeError, ConnectionError): + pass + + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_1(self, size): + CNT = 0 # number of clients that were successful + TOTAL_CNT = size # total number of clients that test will create + TIMEOUT = support.LONG_TIMEOUT # timeout for this test + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + + clients = [] + + async def handle_client(reader, writer): + nonlocal CNT + + data = await reader.readexactly(len(A_DATA)) + self.assertEqual(data, A_DATA) + writer.write(b'OK') + + data = await reader.readexactly(len(B_DATA)) + self.assertEqual(data, B_DATA) + writer.writelines([b'SP', bytearray(b'A'), memoryview(b'M')]) + + await writer.drain() + writer.close() + + CNT += 1 + + async def test_client(addr): + fut = asyncio.Future() + + def prog(sock): + try: + sock.starttls(client_sslctx) + sock.connect(addr) + sock.send(A_DATA) + + data = sock.recv_all(2) + self.assertEqual(data, b'OK') + + sock.send(B_DATA) + data = sock.recv_all(4) + self.assertEqual(data, b'SPAM') + + sock.close() + + except Exception as ex: + self.loop.call_soon_threadsafe(fut.set_exception, ex) + else: + self.loop.call_soon_threadsafe(fut.set_result, None) + + client = self.tcp_client(prog) + client.start() + clients.append(client) + + await fut + + async def start_server(): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + srv = await asyncio.start_server( + handle_client, + '127.0.0.1', 0, + family=socket.AF_INET, + ssl=sslctx, + **extras) + + try: + srv_socks = srv.sockets + self.assertTrue(srv_socks) + + addr = srv_socks[0].getsockname() + + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(test_client(addr)) + + await asyncio.wait_for(asyncio.gather(*tasks), TIMEOUT) + + finally: + self.loop.call_soon(srv.close) + await srv.wait_closed() + + with self._silence_eof_received_warning(): + self.loop.run_until_complete(start_server()) + + self.assertEqual(CNT, TOTAL_CNT) + + for client in clients: + client.stop() + + def test_create_connection_ssl_1(self): + self.loop.set_exception_handler(None) + + CNT = 0 + TOTAL_CNT = 25 + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + + def server(sock): + sock.starttls( + sslctx, + server_side=True) + + data = sock.recv_all(len(A_DATA)) + self.assertEqual(data, A_DATA) + sock.send(b'OK') + + data = sock.recv_all(len(B_DATA)) + self.assertEqual(data, B_DATA) + sock.send(b'SPAM') + + sock.close() + + async def client(addr): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + **extras) + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + writer.write(B_DATA) + self.assertEqual(await reader.readexactly(4), b'SPAM') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + + async def client_sock(addr): + sock = socket.socket() + sock.connect(addr) + reader, writer = await asyncio.open_connection( + sock=sock, + ssl=client_sslctx, + server_hostname='') + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + writer.write(B_DATA) + self.assertEqual(await reader.readexactly(4), b'SPAM') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + sock.close() + + def run(coro): + nonlocal CNT + CNT = 0 + + async def _gather(*tasks): + # trampoline + return await asyncio.gather(*tasks) + + with self.tcp_server(server, + max_clients=TOTAL_CNT, + backlog=TOTAL_CNT) as srv: + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(coro(srv.addr)) + + self.loop.run_until_complete(_gather(*tasks)) + + self.assertEqual(CNT, TOTAL_CNT) + + with self._silence_eof_received_warning(): + run(client) + + with self._silence_eof_received_warning(): + run(client_sock) + + def test_create_connection_ssl_slow_handshake(self): + client_sslctx = self._create_client_ssl_context() + + # silence error logger + self.loop.set_exception_handler(lambda *args: None) + + def server(sock): + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=1.0) + writer.close() + await self.wait_closed(writer) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaisesRegex( + ConnectionAbortedError, + r'SSL handshake.*is taking longer'): + + self.loop.run_until_complete(client(srv.addr)) + + def test_create_connection_ssl_failed_certificate(self): + # silence error logger + self.loop.set_exception_handler(lambda *args: None) + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context(disable_verify=False) + + def server(sock): + try: + sock.starttls( + sslctx, + server_side=True) + sock.connect() + except (ssl.SSLError, OSError): + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.SHORT_TIMEOUT) + writer.close() + await self.wait_closed(writer) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ssl.SSLCertVerificationError): + self.loop.run_until_complete(client(srv.addr)) + + def test_ssl_handshake_timeout(self): + # bpo-29970: Check that a connection is aborted if handshake is not + # completed in timeout period, instead of remaining open indefinitely + client_sslctx = test_utils.simple_client_sslcontext() + + # silence error logger + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server_side_aborted = False + + def server(sock): + nonlocal server_side_aborted + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + server_side_aborted = True + finally: + sock.close() + + async def client(addr): + await asyncio.wait_for( + self.loop.create_connection( + asyncio.Protocol, + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=10.0), + 0.5) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(client(srv.addr)) + + self.assertTrue(server_side_aborted) + + # Python issue #23197: cancelling a handshake must not raise an + # exception or log an error, even if the handshake failed + self.assertEqual(messages, []) + + def test_ssl_handshake_connection_lost(self): + # #246: make sure that no connection_lost() is called before + # connection_made() is called first + + client_sslctx = test_utils.simple_client_sslcontext() + + # silence error logger + self.loop.set_exception_handler(lambda loop, ctx: None) + + connection_made_called = False + connection_lost_called = False + + def server(sock): + sock.recv(1024) + # break the connection during handshake + sock.close() + + class ClientProto(asyncio.Protocol): + def connection_made(self, transport): + nonlocal connection_made_called + connection_made_called = True + + def connection_lost(self, exc): + nonlocal connection_lost_called + connection_lost_called = True + + async def client(addr): + await self.loop.create_connection( + ClientProto, + *addr, + ssl=client_sslctx, + server_hostname=''), + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ConnectionResetError): + self.loop.run_until_complete(client(srv.addr)) + + if connection_lost_called: + if connection_made_called: + self.fail("unexpected call to connection_lost()") + else: + self.fail("unexpected call to connection_lost() without" + "calling connection_made()") + elif connection_made_called: + self.fail("unexpected call to connection_made()") + + def test_ssl_connect_accepted_socket(self): + proto = ssl.PROTOCOL_TLS_SERVER + server_context = ssl.SSLContext(proto) + server_context.load_cert_chain(test_utils.ONLYCERT, test_utils.ONLYKEY) + if hasattr(server_context, 'check_hostname'): + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + + client_context = ssl.SSLContext(proto) + if hasattr(server_context, 'check_hostname'): + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + + def test_connect_accepted_socket(self, server_ssl=None, client_ssl=None): + loop = self.loop + + class MyProto(MyBaseProto): + + def connection_lost(self, exc): + super().connection_lost(exc) + loop.call_soon(loop.stop) + + def data_received(self, data): + super().data_received(data) + self.transport.write(expected_response) + + lsock = socket.socket(socket.AF_INET) + lsock.bind(('127.0.0.1', 0)) + lsock.listen(1) + addr = lsock.getsockname() + + message = b'test data' + response = None + expected_response = b'roger' + + def client(): + nonlocal response + try: + csock = socket.socket(socket.AF_INET) + if client_ssl is not None: + csock = client_ssl.wrap_socket(csock) + csock.connect(addr) + csock.sendall(message) + response = csock.recv(99) + csock.close() + except Exception as exc: + print( + "Failure in client thread in test_connect_accepted_socket", + exc) + + thread = threading.Thread(target=client, daemon=True) + thread.start() + + conn, _ = lsock.accept() + proto = MyProto(loop=loop) + proto.loop = loop + + extras = {} + if server_ssl: + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + f = loop.create_task( + loop.connect_accepted_socket( + (lambda: proto), conn, ssl=server_ssl, + **extras)) + loop.run_forever() + conn.close() + lsock.close() + + thread.join(1) + self.assertFalse(thread.is_alive()) + self.assertEqual(proto.state, 'CLOSED') + self.assertEqual(proto.nbytes, len(message)) + self.assertEqual(response, expected_response) + tr, _ = f.result() + + if server_ssl: + self.assertIn('SSL', tr.__class__.__name__) + + tr.close() + # let it close + self.loop.run_until_complete(asyncio.sleep(0.1)) + + def test_start_tls_client_corrupted_ssl(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext() + + def server(sock): + orig_sock = sock.dup() + try: + sock.starttls( + sslctx, + server_side=True) + sock.sendall(b'A\n') + sock.recv_all(1) + orig_sock.send(b'please corrupt the SSL connection') + except ssl.SSLError: + pass + finally: + sock.close() + orig_sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + + self.assertEqual(await reader.readline(), b'A\n') + writer.write(b'B') + with self.assertRaises(ssl.SSLError): + await reader.readline() + writer.close() + try: + await self.wait_closed(writer) + except ssl.SSLError: + pass + return 'OK' + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + res = self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(res, 'OK') + + def test_start_tls_client_reg_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_create_connection_memory_leak(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_context = self._create_client_ssl_context() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + # XXX: We assume user stores the transport in protocol + proto.tr = tr + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr, + ssl=client_context) + + self.assertEqual(await on_data, b'O') + tr.write(HELLO_MSG) + await on_eof + + tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left for SSL client from loop.create_connection, even + # if user stores the SSLTransport in corresponding protocol instance + client_context = weakref.ref(client_context) + self.assertIsNone(client_context()) + + def test_start_tls_client_buf_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + client_con_made_calls = 0 + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.sendall(b'2') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProtoFirst(asyncio.BufferedProtocol): + def __init__(self, on_data): + self.on_data = on_data + self.buf = bytearray(1) + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def get_buffer(self, sizehint): + return self.buf + + def buffer_updated(self, nsize): + assert nsize == 1 + self.on_data.set_result(bytes(self.buf[:nsize])) + + def eof_received(self): + pass + + class ClientProtoSecond(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data1 = self.loop.create_future() + on_data2 = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProtoFirst(on_data1), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data1, b'O') + new_tr.write(HELLO_MSG) + + new_tr.set_protocol(ClientProtoSecond(on_data2, on_eof)) + self.assertEqual(await on_data2, b'2') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + # connection_made() should be called only once -- when + # we establish connection for the first time. Start TLS + # doesn't call connection_made() on application protocols. + self.assertEqual(client_con_made_calls, 1) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=self.TIMEOUT)) + + def test_start_tls_slow_client_cancel(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + client_context = test_utils.simple_client_sslcontext() + server_waits_on_handshake = self.loop.create_future() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + try: + self.loop.call_soon_threadsafe( + server_waits_on_handshake.set_result, None) + data = sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + + await server_waits_on_handshake + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for( + self.loop.start_tls(tr, proto, client_context), + 0.5) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def client(sock, addr): + sock.settimeout(self.TIMEOUT) + + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(client_context) + sock.sendall(HELLO_MSG) + + sock.unwrap() + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_eof, on_con_lost): + self.on_con = on_con + self.on_eof = on_eof + self.on_con_lost = on_con_lost + self.data = b'' + + def connection_made(self, tr): + self.on_con.set_result(tr) + + def data_received(self, data): + self.data += data + + def eof_received(self): + self.on_eof.set_result(1) + + def connection_lost(self, exc): + if exc is None: + self.on_con_lost.set_result(None) + else: + self.on_con_lost.set_exception(exc) + + async def main(proto, on_con, on_eof, on_con_lost): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True, + ssl_handshake_timeout=self.TIMEOUT) + + await on_eof + await on_con_lost + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + async def run_main(): + on_con = self.loop.create_future() + on_eof = self.loop.create_future() + on_con_lost = self.loop.create_future() + proto = ServerProto(on_con, on_eof, on_con_lost) + + server = await self.loop.create_server( + lambda: proto, '127.0.0.1', 0) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr), + timeout=self.TIMEOUT): + await asyncio.wait_for( + main(proto, on_con, on_eof, on_con_lost), + timeout=self.TIMEOUT) + + server.close() + await server.wait_closed() + + self.loop.run_until_complete(run_main()) + + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_over_ssl(self, size): + CNT = 0 # number of clients that were successful + TOTAL_CNT = size # total number of clients that test will create + TIMEOUT = support.LONG_TIMEOUT # timeout for this test + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx_1 = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx_1 = self._create_client_ssl_context() + sslctx_2 = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx_2 = self._create_client_ssl_context() + + clients = [] + + async def handle_client(reader, writer): + nonlocal CNT + + data = await reader.readexactly(len(A_DATA)) + self.assertEqual(data, A_DATA) + writer.write(b'OK') + + data = await reader.readexactly(len(B_DATA)) + self.assertEqual(data, B_DATA) + writer.writelines([b'SP', bytearray(b'A'), memoryview(b'M')]) + + await writer.drain() + writer.close() + + CNT += 1 + + class ServerProtocol(asyncio.StreamReaderProtocol): + def connection_made(self, transport): + super_ = super() + transport.pause_reading() + fut = self._loop.create_task(self._loop.start_tls( + transport, self, sslctx_2, server_side=True)) + + def cb(_): + try: + tr = fut.result() + except Exception as ex: + super_.connection_lost(ex) + else: + super_.connection_made(tr) + fut.add_done_callback(cb) + + def server_protocol_factory(): + reader = asyncio.StreamReader() + protocol = ServerProtocol(reader, handle_client) + return protocol + + async def test_client(addr): + fut = asyncio.Future() + + def prog(sock): + try: + sock.connect(addr) + sock.starttls(client_sslctx_1) + + # because wrap_socket() doesn't work correctly on + # SSLSocket, we have to do the 2nd level SSL manually + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = client_sslctx_2.wrap_bio(incoming, outgoing) + + def do(func, *args): + while True: + try: + rv = func(*args) + break + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(65536)) + if outgoing.pending: + sock.send(outgoing.read()) + return rv + + do(sslobj.do_handshake) + + do(sslobj.write, A_DATA) + data = do(sslobj.read, 2) + self.assertEqual(data, b'OK') + + do(sslobj.write, B_DATA) + data = b'' + while True: + chunk = do(sslobj.read, 4) + if not chunk: + break + data += chunk + self.assertEqual(data, b'SPAM') + + do(sslobj.unwrap) + sock.close() + + except Exception as ex: + self.loop.call_soon_threadsafe(fut.set_exception, ex) + sock.close() + else: + self.loop.call_soon_threadsafe(fut.set_result, None) + + client = self.tcp_client(prog) + client.start() + clients.append(client) + + await fut + + async def start_server(): + extras = {} + + srv = await self.loop.create_server( + server_protocol_factory, + '127.0.0.1', 0, + family=socket.AF_INET, + ssl=sslctx_1, + **extras) + + try: + srv_socks = srv.sockets + self.assertTrue(srv_socks) + + addr = srv_socks[0].getsockname() + + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(test_client(addr)) + + await asyncio.wait_for(asyncio.gather(*tasks), TIMEOUT) + + finally: + self.loop.call_soon(srv.close) + await srv.wait_closed() + + with self._silence_eof_received_warning(): + self.loop.run_until_complete(start_server()) + + self.assertEqual(CNT, TOTAL_CNT) + + for client in clients: + client.stop() + + def test_shutdown_cleanly(self): + CNT = 0 + TOTAL_CNT = 25 + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx = self._create_client_ssl_context() + + def server(sock): + sock.starttls( + sslctx, + server_side=True) + + data = sock.recv_all(len(A_DATA)) + self.assertEqual(data, A_DATA) + sock.send(b'OK') + + sock.unwrap() + + sock.close() + + async def client(addr): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + **extras) + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + self.assertEqual(await reader.read(), b'') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + + def run(coro): + nonlocal CNT + CNT = 0 + + async def _gather(*tasks): + return await asyncio.gather(*tasks) + + with self.tcp_server(server, + max_clients=TOTAL_CNT, + backlog=TOTAL_CNT) as srv: + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(coro(srv.addr)) + + self.loop.run_until_complete( + _gather(*tasks)) + + self.assertEqual(CNT, TOTAL_CNT) + + with self._silence_eof_received_warning(): + run(client) + + def test_flush_before_shutdown(self): + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx = self._create_client_ssl_context() + + future = None + + def server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + time.sleep(0.5) # hopefully stuck the TCP buffer + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + sock.close() + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + async def client(addr): + nonlocal future + future = self.loop.create_future() + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + sslprotocol = writer.transport._ssl_protocol + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + sslprotocol.pause_writing() + for _ in range(SIZE): + writer.write(b'x' * CHUNK) + + writer.close() + sslprotocol.resume_writing() + + await self.wait_closed(writer) + try: + data = await reader.read() + self.assertEqual(data, b'') + except ConnectionResetError: + pass + await future + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_remote_shutdown_receives_trailing_data(self): + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + future = None + + def server(sock): + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = sslctx.wrap_bio(incoming, outgoing, server_side=True) + + while True: + try: + sslobj.do_handshake() + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(16384)) + else: + if outgoing.pending: + sock.send(outgoing.read()) + break + + while True: + try: + data = sslobj.read(4) + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + else: + break + + self.assertEqual(data, b'ping') + sslobj.write(b'pong') + sock.send(outgoing.read()) + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send close_notify but don't wait for response + with self.assertRaises(ssl.SSLWantReadError): + sslobj.unwrap() + sock.send(outgoing.read()) + + # should receive all data + data_len = 0 + while True: + try: + chunk = len(sslobj.read(16384)) + data_len += chunk + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + except ssl.SSLZeroReturnError: + break + + self.assertEqual(data_len, CHUNK * SIZE) + + # verify that close_notify is received + sslobj.unwrap() + + sock.close() + + def eof_server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send EOF + sock.shutdown(socket.SHUT_WR) + + # should receive all data + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + + sock.close() + + async def client(addr): + nonlocal future + future = self.loop.create_future() + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + # fill write backlog in a hacky way - renegotiation won't help + for _ in range(SIZE): + writer.transport._test__append_write_backlog(b'x' * CHUNK) + + try: + data = await reader.read() + self.assertEqual(data, b'') + except (BrokenPipeError, ConnectionResetError): + pass + + await future + + writer.close() + await self.wait_closed(writer) + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + with self.tcp_server(run(eof_server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_remote_shutdown_receives_trailing_data_on_slow_socket(self): + # This test is the same as test_remote_shutdown_receives_trailing_data, + # except it simulates a socket that is not able to write data in time, + # thus triggering different code path in _SelectorSocketTransport. + # This triggers bug gh-115514, also tested using mocks in + # test.test_asyncio.test_selector_events.SelectorSocketTransportTests.test_write_buffer_after_close + # The slow path is triggered here by setting SO_SNDBUF, see code and comment below. + + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + future = None + + def server(sock): + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = sslctx.wrap_bio(incoming, outgoing, server_side=True) + + while True: + try: + sslobj.do_handshake() + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(16384)) + else: + if outgoing.pending: + sock.send(outgoing.read()) + break + + while True: + try: + data = sslobj.read(4) + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + else: + break + + self.assertEqual(data, b'ping') + sslobj.write(b'pong') + sock.send(outgoing.read()) + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send close_notify but don't wait for response + with self.assertRaises(ssl.SSLWantReadError): + sslobj.unwrap() + sock.send(outgoing.read()) + + # should receive all data + data_len = 0 + while True: + try: + chunk = len(sslobj.read(16384)) + data_len += chunk + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + except ssl.SSLZeroReturnError: + break + + self.assertEqual(data_len, CHUNK * SIZE*2) + + # verify that close_notify is received + sslobj.unwrap() + + sock.close() + + def eof_server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send EOF + sock.shutdown(socket.SHUT_WR) + + # should receive all data + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + + sock.close() + + async def client(addr): + nonlocal future + future = self.loop.create_future() + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + # fill write backlog in a hacky way - renegotiation won't help + for _ in range(SIZE*2): + writer.transport._test__append_write_backlog(b'x' * CHUNK) + + try: + data = await reader.read() + self.assertEqual(data, b'') + except (BrokenPipeError, ConnectionResetError): + pass + + # Make sure _SelectorSocketTransport enters the delayed write + # path in its `write` method by wrapping socket in a fake class + # that acts as if there is not enough space in socket buffer. + # This triggers bug gh-115514, also tested using mocks in + # test.test_asyncio.test_selector_events.SelectorSocketTransportTests.test_write_buffer_after_close + socket_transport = writer.transport._ssl_protocol._transport + + class SocketWrapper: + def __init__(self, sock) -> None: + self.sock = sock + + def __getattr__(self, name): + return getattr(self.sock, name) + + def send(self, data): + # Fake that our write buffer is full, send only half + to_send = len(data)//2 + return self.sock.send(data[:to_send]) + + def _fake_full_write_buffer(data): + if socket_transport._read_ready_cb is None and not isinstance(socket_transport._sock, SocketWrapper): + socket_transport._sock = SocketWrapper(socket_transport._sock) + return unittest.mock.DEFAULT + + with unittest.mock.patch.object( + socket_transport, "write", + wraps=socket_transport.write, + side_effect=_fake_full_write_buffer + ): + await future + + writer.close() + await self.wait_closed(writer) + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + with self.tcp_server(run(eof_server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_connect_timeout_warning(self): + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + addr = s.getsockname() + + async def test(): + try: + await asyncio.wait_for( + self.loop.create_connection(asyncio.Protocol, + *addr, ssl=True), + 0.1) + except (ConnectionRefusedError, asyncio.TimeoutError): + pass + else: + self.fail('TimeoutError is not raised') + + with s: + try: + with self.assertWarns(ResourceWarning) as cm: + self.loop.run_until_complete(test()) + gc.collect() + gc.collect() + gc.collect() + except AssertionError as e: + self.assertEqual(str(e), 'ResourceWarning not triggered') + else: + self.fail('Unexpected ResourceWarning: {}'.format(cm.warning)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_handshake_timeout_handler_leak(self): + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen(1) + addr = s.getsockname() + + async def test(ctx): + try: + await asyncio.wait_for( + self.loop.create_connection(asyncio.Protocol, *addr, + ssl=ctx), + 0.1) + except (ConnectionRefusedError, asyncio.TimeoutError): + pass + else: + self.fail('TimeoutError is not raised') + + with s: + ctx = ssl.create_default_context() + self.loop.run_until_complete(test(ctx)) + ctx = weakref.ref(ctx) + + # SSLProtocol should be DECREF to 0 + self.assertIsNone(ctx()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_shutdown_timeout_handler_leak(self): + loop = self.loop + + def server(sock): + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + sock = sslctx.wrap_socket(sock, server_side=True) + sock.recv(32) + sock.close() + + class Protocol(asyncio.Protocol): + def __init__(self): + self.fut = asyncio.Future(loop=loop) + + def connection_lost(self, exc): + self.fut.set_result(None) + + async def client(addr, ctx): + tr, pr = await loop.create_connection(Protocol, *addr, ssl=ctx) + tr.close() + await pr.fut + + with self.tcp_server(server) as srv: + ctx = self._create_client_ssl_context() + loop.run_until_complete(client(srv.addr, ctx)) + ctx = weakref.ref(ctx) + + # asyncio has no shutdown timeout, but it ends up with a circular + # reference loop - not ideal (introduces gc glitches), but at least + # not leaking + gc.collect() + gc.collect() + gc.collect() + + # SSLProtocol should be DECREF to 0 + self.assertIsNone(ctx()) + + def test_shutdown_timeout_handler_not_set(self): + loop = self.loop + eof = asyncio.Event() + extra = None + + def server(sock): + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + sock = sslctx.wrap_socket(sock, server_side=True) + sock.send(b'hello') + assert sock.recv(1024) == b'world' + sock.send(b'extra bytes') + # sending EOF here + sock.shutdown(socket.SHUT_WR) + loop.call_soon_threadsafe(eof.set) + # make sure we have enough time to reproduce the issue + assert sock.recv(1024) == b'' + sock.close() + + class Protocol(asyncio.Protocol): + def __init__(self): + self.fut = asyncio.Future(loop=loop) + self.transport = None + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + if data == b'hello': + self.transport.write(b'world') + # pause reading would make incoming data stay in the sslobj + self.transport.pause_reading() + else: + nonlocal extra + extra = data + + def connection_lost(self, exc): + if exc is None: + self.fut.set_result(None) + else: + self.fut.set_exception(exc) + + async def client(addr): + ctx = self._create_client_ssl_context() + tr, pr = await loop.create_connection(Protocol, *addr, ssl=ctx) + await eof.wait() + tr.resume_reading() + await pr.fut + tr.close() + assert extra == b'extra bytes' + + with self.tcp_server(server) as srv: + loop.run_until_complete(client(srv.addr)) + + +############################################################################### +# Socket Testing Utilities +############################################################################### + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def starttls(self, ssl_context, *, + server_side=False, + server_hostname=None, + do_handshake_on_connect=True): + + assert isinstance(ssl_context, ssl.SSLContext) + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=do_handshake_on_connect) + + if server_side: + ssl_sock.do_handshake() + + self.__sock.close() + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + self._sock.close() + self._s1.close() + self._s2.close() + + def run(self): + self._sock.setblocking(False) + self._run() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except socket.timeout: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py new file mode 100644 index 00000000000..7ab6e1511d7 --- /dev/null +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -0,0 +1,846 @@ +"""Tests for asyncio/sslproto.py.""" + +import logging +import socket +import unittest +import weakref +from test import support +from test.support import socket_helper +from unittest import mock +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from asyncio import log +from asyncio import protocols +from asyncio import sslproto +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +@unittest.skipIf(ssl is None, 'No ssl module') +class SslProtoHandshakeTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def ssl_protocol(self, *, waiter=None, proto=None): + sslcontext = test_utils.dummy_ssl_context() + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): + transport = mock.Mock() + sslobj = mock.Mock() + # emulate reading decompressed data + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + if do_handshake is not None: + sslobj.do_handshake = do_handshake + ssl_proto._sslobj = sslobj + ssl_proto.connection_made(transport) + return transport + + def test_handshake_timeout_zero(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=0) + + def test_handshake_timeout_negative(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=-10) + + def test_eof_received_waiter(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + ssl_proto.eof_received() + test_utils.run_briefly(self.loop) + self.assertIsInstance(waiter.exception(), ConnectionResetError) + + def test_fatal_error_no_name_error(self): + # From issue #363. + # _fatal_error() generates a NameError if sslproto.py + # does not import base_events. + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + # Temporarily turn off error logging so as not to spoil test output. + log_level = log.logger.getEffectiveLevel() + log.logger.setLevel(logging.FATAL) + try: + ssl_proto._fatal_error(None) + finally: + # Restore error logging. + log.logger.setLevel(log_level) + + def test_connection_lost(self): + # From issue #472. + # yield from waiter hang if lost_connection was called. + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + ssl_proto.connection_lost(ConnectionAbortedError) + test_utils.run_briefly(self.loop) + self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + + def test_connection_lost_when_busy(self): + # gh-118950: SSLProtocol.connection_lost not being called when OSError + # is thrown on asyncio.write. + sock = mock.Mock() + sock.fileno = mock.Mock(return_value=12345) + sock.send = mock.Mock(side_effect=BrokenPipeError) + + # construct StreamWriter chain that contains loop dependant logic this emulates + # what _make_ssl_transport() does in BaseSelectorEventLoop + reader = asyncio.StreamReader(limit=2 ** 16, loop=self.loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) + ssl_proto = self.ssl_protocol(proto=protocol) + + # emulate reading decompressed data + sslobj = mock.Mock() + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + ssl_proto._sslobj = sslobj + + # emulate outgoing data + data = b'An interesting message' + + outgoing = mock.Mock() + outgoing.read = mock.Mock(return_value=data) + outgoing.pending = len(data) + ssl_proto._outgoing = outgoing + + # use correct socket transport to initialize the SSLProtocol + self.loop._make_socket_transport(sock, ssl_proto) + + transport = ssl_proto._app_transport + writer = asyncio.StreamWriter(transport, protocol, reader, self.loop) + + async def main(): + # writes data to transport + async def write(): + writer.write(data) + await writer.drain() + + # try to write for the first time + await write() + # try to write for the second time, this raises as the connection_lost + # callback should be done with error + with self.assertRaises(ConnectionResetError): + await write() + + self.loop.run_until_complete(main()) + + def test_close_during_handshake(self): + # bpo-29743 Closing transport during handshake process leaks socket + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + + transport = self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + test_utils.run_briefly(self.loop) + + ssl_proto._app_transport.close() + self.assertTrue(transport._force_close.called) + + def test_close_during_ssl_over_ssl(self): + # gh-113214: passing exceptions from the inner wrapped SSL protocol to the + # shim transport provided by the outer SSL protocol should not raise + # attribute errors + outer = self.ssl_protocol(proto=self.ssl_protocol()) + self.connection_made(outer) + # Closing the outer app transport should not raise an exception + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + outer._app_transport.close() + self.assertEqual(messages, []) + + def test_get_extra_info_on_closed_connection(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + def test_set_new_app_protocol(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + new_app_proto = asyncio.Protocol() + ssl_proto._app_transport.set_protocol(new_app_proto) + self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) + self.assertIs(ssl_proto._app_protocol, new_app_proto) + + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.buffer_updated(5)) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + + +############################################################################## +# Start TLS Tests +############################################################################## + + +class BaseStartTLS(func_tests.FunctionalTestCaseMixin): + + PAYLOAD_SIZE = 1024 * 100 + TIMEOUT = support.LONG_TIMEOUT + + def new_loop(self): + raise NotImplementedError + + def test_buf_feed_data(self): + + class Proto(asyncio.BufferedProtocol): + + def __init__(self, bufsize, usemv): + self.buf = bytearray(bufsize) + self.mv = memoryview(self.buf) + self.data = b'' + self.usemv = usemv + + def get_buffer(self, sizehint): + if self.usemv: + return self.mv + else: + return self.buf + + def buffer_updated(self, nsize): + if self.usemv: + self.data += self.mv[:nsize] + else: + self.data += self.buf[:nsize] + + for usemv in [False, True]: + proto = Proto(1, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(2, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(2, usemv) + protocols._feed_data_to_buffered_proto(proto, b'1234') + self.assertEqual(proto.data, b'1234') + + proto = Proto(4, usemv) + protocols._feed_data_to_buffered_proto(proto, b'1234') + self.assertEqual(proto.data, b'1234') + + proto = Proto(100, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(0, usemv) + with self.assertRaisesRegex(RuntimeError, 'empty buffer'): + protocols._feed_data_to_buffered_proto(proto, b'12345') + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_start_tls_client_reg_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left if SSL is closed uncleanly + client_context = weakref.ref(client_context) + support.gc_collect() + self.assertIsNone(client_context()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_create_connection_memory_leak(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + # XXX: We assume user stores the transport in protocol + proto.tr = tr + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr, + ssl=client_context) + + self.assertEqual(await on_data, b'O') + tr.write(HELLO_MSG) + await on_eof + + tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left for SSL client from loop.create_connection, even + # if user stores the SSLTransport in corresponding protocol instance + client_context = weakref.ref(client_context) + support.gc_collect() + self.assertIsNone(client_context()) + + @socket_helper.skip_if_tcp_blackhole + def test_start_tls_client_buf_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + client_con_made_calls = 0 + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.sendall(b'2') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProtoFirst(asyncio.BufferedProtocol): + def __init__(self, on_data): + self.on_data = on_data + self.buf = bytearray(1) + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def get_buffer(self, sizehint): + return self.buf + + def buffer_updated(slf, nsize): + self.assertEqual(nsize, 1) + slf.on_data.set_result(bytes(slf.buf[:nsize])) + + class ClientProtoSecond(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data1 = self.loop.create_future() + on_data2 = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProtoFirst(on_data1), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data1, b'O') + new_tr.write(HELLO_MSG) + + new_tr.set_protocol(ClientProtoSecond(on_data2, on_eof)) + self.assertEqual(await on_data2, b'2') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + # connection_made() should be called only once -- when + # we establish connection for the first time. Start TLS + # doesn't call connection_made() on application protocols. + self.assertEqual(client_con_made_calls, 1) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=self.TIMEOUT)) + + def test_start_tls_slow_client_cancel(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + client_context = test_utils.simple_client_sslcontext() + server_waits_on_handshake = self.loop.create_future() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + try: + self.loop.call_soon_threadsafe( + server_waits_on_handshake.set_result, None) + data = sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + + await server_waits_on_handshake + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for( + self.loop.start_tls(tr, proto, client_context), + 0.5) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + @socket_helper.skip_if_tcp_blackhole + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + ANSWER = b'answer' + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + answer = None + + def client(sock, addr): + nonlocal answer + sock.settimeout(self.TIMEOUT) + + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(client_context) + sock.sendall(HELLO_MSG) + answer = sock.recv_all(len(ANSWER)) + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_con_lost, on_got_hello): + self.on_con = on_con + self.on_con_lost = on_con_lost + self.on_got_hello = on_got_hello + self.data = b'' + self.transport = None + + def connection_made(self, tr): + self.transport = tr + self.on_con.set_result(tr) + + def replace_transport(self, tr): + self.transport = tr + + def data_received(self, data): + self.data += data + if len(self.data) >= len(HELLO_MSG): + self.on_got_hello.set_result(None) + + def connection_lost(self, exc): + self.transport = None + if exc is None: + self.on_con_lost.set_result(None) + else: + self.on_con_lost.set_exception(exc) + + async def main(proto, on_con, on_con_lost, on_got_hello): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True, + ssl_handshake_timeout=self.TIMEOUT) + proto.replace_transport(new_tr) + + await on_got_hello + new_tr.write(ANSWER) + + await on_con_lost + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + async def run_main(): + on_con = self.loop.create_future() + on_con_lost = self.loop.create_future() + on_got_hello = self.loop.create_future() + proto = ServerProto(on_con, on_con_lost, on_got_hello) + + server = await self.loop.create_server( + lambda: proto, '127.0.0.1', 0) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr), + timeout=self.TIMEOUT): + await asyncio.wait_for( + main(proto, on_con, on_con_lost, on_got_hello), + timeout=self.TIMEOUT) + + server.close() + await server.wait_closed() + self.assertEqual(answer, ANSWER) + + self.loop.run_until_complete(run_main()) + + def test_start_tls_wrong_args(self): + async def main(): + with self.assertRaisesRegex(TypeError, 'SSLContext, got'): + await self.loop.start_tls(None, None, None) + + sslctx = test_utils.simple_server_sslcontext() + with self.assertRaisesRegex(TypeError, 'is not supported'): + await self.loop.start_tls(None, None, sslctx) + + self.loop.run_until_complete(main()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_handshake_timeout(self): + # bpo-29970: Check that a connection is aborted if handshake is not + # completed in timeout period, instead of remaining open indefinitely + client_sslctx = test_utils.simple_client_sslcontext() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server_side_aborted = False + + def server(sock): + nonlocal server_side_aborted + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + server_side_aborted = True + finally: + sock.close() + + async def client(addr): + await asyncio.wait_for( + self.loop.create_connection( + asyncio.Protocol, + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.SHORT_TIMEOUT), + 0.5) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(client(srv.addr)) + + self.assertTrue(server_side_aborted) + + # Python issue #23197: cancelling a handshake must not raise an + # exception or log an error, even if the handshake failed + self.assertEqual(messages, []) + + # The 10s handshake timeout should be cancelled to free related + # objects without really waiting for 10s + client_sslctx = weakref.ref(client_sslctx) + support.gc_collect() + self.assertIsNone(client_sslctx()) + + def test_create_connection_ssl_slow_handshake(self): + client_sslctx = test_utils.simple_client_sslcontext() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + def server(sock): + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=1.0) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaisesRegex( + ConnectionAbortedError, + r'SSL handshake.*is taking longer'): + + self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(messages, []) + + def test_create_connection_ssl_failed_certificate(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext( + disable_verify=False) + + def server(sock): + try: + sock.start_tls( + sslctx, + server_side=True) + except ssl.SSLError: + pass + except OSError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.LOOPBACK_TIMEOUT) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ssl.SSLCertVerificationError): + self.loop.run_until_complete(client(srv.addr)) + + def test_start_tls_client_corrupted_ssl(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext() + + def server(sock): + orig_sock = sock.dup() + try: + sock.start_tls( + sslctx, + server_side=True) + sock.sendall(b'A\n') + sock.recv_all(1) + orig_sock.send(b'please corrupt the SSL connection') + except ssl.SSLError: + pass + finally: + orig_sock.close() + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + + self.assertEqual(await reader.readline(), b'A\n') + writer.write(b'B') + with self.assertRaises(ssl.SSLError): + await reader.readline() + + writer.close() + return 'OK' + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + res = self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(res, 'OK') + + +@unittest.skipIf(ssl is None, 'No ssl module') +class SelectorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipIf(ssl is None, 'No ssl module') +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000..32e4817b70d --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,151 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) + + + async def test_multiple_winners(self): + event = asyncio.Event() + + async def coro(index): + await event.wait() + return index + + async def do_set(): + event.set() + await asyncio.Event().wait() + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + do_set, + ], + delay=0.1, + ) + self.assertIs(winner, 0) + self.assertIs(index, 0) + self.assertEqual(len(excs), 3) + self.assertIsNone(excs[0], None) + self.assertIsInstance(excs[1], asyncio.CancelledError) + self.assertIsInstance(excs[2], asyncio.CancelledError) + + + async def test_cancelled(self): + log = [] + with self.assertRaises(TimeoutError): + async with asyncio.timeout(None) as cs_outer, asyncio.timeout(None) as cs_inner: + async def coro_fn(): + cs_inner.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 1") + + cs_outer.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 2") + try: + await staggered_race([coro_fn], delay=None) + except asyncio.CancelledError: + log.append("cancelled 3") + raise + + self.assertListEqual(log, ["cancelled 1", "cancelled 2", "cancelled 3"]) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py new file mode 100644 index 00000000000..5f0fc6a7a9d --- /dev/null +++ b/Lib/test/test_asyncio/test_streams.py @@ -0,0 +1,1221 @@ +"""Tests for streams.py.""" + +import gc +import queue +import pickle +import socket +import threading +import unittest +from unittest import mock +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from test.test_asyncio import utils as test_utils +from test.support import socket_helper + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class StreamTests(test_utils.TestCase): + + DATA = b'line1\nline2\nline3\n' + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + test_utils.run_briefly(self.loop) + + # set_event_loop() takes care of closing self.loop in a safe way + super().tearDown() + + def _basetest_open_connection(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + reader, writer = self.loop.run_until_complete(open_connection_fut) + writer.write(b'GET / HTTP/1.0\r\n\r\n') + f = reader.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + f = reader.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + writer.close() + self.assertEqual(messages, []) + + def test_open_connection(self): + with test_utils.run_test_server() as httpd: + conn_fut = asyncio.open_connection(*httpd.address) + self._basetest_open_connection(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_open_unix_connection(self): + with test_utils.run_test_unix_server() as httpd: + conn_fut = asyncio.open_unix_connection(httpd.address) + self._basetest_open_connection(conn_fut) + + def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + try: + reader, writer = self.loop.run_until_complete(open_connection_fut) + finally: + asyncio.set_event_loop(None) + writer.write(b'GET / HTTP/1.0\r\n\r\n') + f = reader.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + + writer.close() + self.assertEqual(messages, []) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_open_connection_no_loop_ssl(self): + with test_utils.run_test_server(use_ssl=True) as httpd: + conn_fut = asyncio.open_connection( + *httpd.address, + ssl=test_utils.dummy_ssl_context()) + + self._basetest_open_connection_no_loop_ssl(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_open_unix_connection_no_loop_ssl(self): + with test_utils.run_test_unix_server(use_ssl=True) as httpd: + conn_fut = asyncio.open_unix_connection( + httpd.address, + ssl=test_utils.dummy_ssl_context(), + server_hostname='', + ) + + self._basetest_open_connection_no_loop_ssl(conn_fut) + + def _basetest_open_connection_error(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + reader, writer = self.loop.run_until_complete(open_connection_fut) + writer._protocol.connection_lost(ZeroDivisionError()) + f = reader.read() + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(f) + writer.close() + test_utils.run_briefly(self.loop) + self.assertEqual(messages, []) + + def test_open_connection_error(self): + with test_utils.run_test_server() as httpd: + conn_fut = asyncio.open_connection(*httpd.address) + self._basetest_open_connection_error(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_open_unix_connection_error(self): + with test_utils.run_test_unix_server() as httpd: + conn_fut = asyncio.open_unix_connection(httpd.address) + self._basetest_open_connection_error(conn_fut) + + def test_feed_empty_data(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'') + self.assertEqual(b'', stream._buffer) + + def test_feed_nonempty_data(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(self.DATA) + self.assertEqual(self.DATA, stream._buffer) + + def test_read_zero(self): + # Read zero bytes. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + data = self.loop.run_until_complete(stream.read(0)) + self.assertEqual(b'', data) + self.assertEqual(self.DATA, stream._buffer) + + def test_read(self): + # Read bytes. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(30)) + + def cb(): + stream.feed_data(self.DATA) + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(self.DATA, data) + self.assertEqual(b'', stream._buffer) + + def test_read_line_breaks(self): + # Read bytes without line breaks. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line1') + stream.feed_data(b'line2') + + data = self.loop.run_until_complete(stream.read(5)) + + self.assertEqual(b'line1', data) + self.assertEqual(b'line2', stream._buffer) + + def test_read_eof(self): + # Read bytes, stop at eof. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(1024)) + + def cb(): + stream.feed_eof() + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(b'', data) + self.assertEqual(b'', stream._buffer) + + def test_read_until_eof(self): + # Read all bytes until eof. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(-1)) + + def cb(): + stream.feed_data(b'chunk1\n') + stream.feed_data(b'chunk2') + stream.feed_eof() + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + + self.assertEqual(b'chunk1\nchunk2', data) + self.assertEqual(b'', stream._buffer) + + def test_read_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.read(2)) + self.assertEqual(b'li', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.read(2)) + + def test_invalid_limit(self): + with self.assertRaisesRegex(ValueError, 'imit'): + asyncio.StreamReader(limit=0, loop=self.loop) + + with self.assertRaisesRegex(ValueError, 'imit'): + asyncio.StreamReader(limit=-1, loop=self.loop) + + def test_read_limit(self): + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'chunk') + data = self.loop.run_until_complete(stream.read(5)) + self.assertEqual(b'chunk', data) + self.assertEqual(b'', stream._buffer) + + def test_readline(self): + # Read one line. 'readline' will need to wait for the data + # to come from 'cb' + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'chunk1 ') + read_task = self.loop.create_task(stream.readline()) + + def cb(): + stream.feed_data(b'chunk2 ') + stream.feed_data(b'chunk3 ') + stream.feed_data(b'\n chunk4') + self.loop.call_soon(cb) + + line = self.loop.run_until_complete(read_task) + self.assertEqual(b'chunk1 chunk2 chunk3 \n', line) + self.assertEqual(b' chunk4', stream._buffer) + + def test_readline_limit_with_existing_data(self): + # Read one line. The data is in StreamReader's buffer + # before the event loop is run. + + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'li') + stream.feed_data(b'ne1\nline2\n') + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # The buffer should contain the remaining data after exception + self.assertEqual(b'line2\n', stream._buffer) + + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'li') + stream.feed_data(b'ne1') + stream.feed_data(b'li') + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # No b'\n' at the end. The 'limit' is set to 3. So before + # waiting for the new data in buffer, 'readline' will consume + # the entire buffer, and since the length of the consumed data + # is more than 3, it will raise a ValueError. The buffer is + # expected to be empty now. + self.assertEqual(b'', stream._buffer) + + def test_at_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertFalse(stream.at_eof()) + + stream.feed_data(b'some data\n') + self.assertFalse(stream.at_eof()) + + self.loop.run_until_complete(stream.readline()) + self.assertFalse(stream.at_eof()) + + stream.feed_data(b'some data\n') + stream.feed_eof() + self.loop.run_until_complete(stream.readline()) + self.assertTrue(stream.at_eof()) + + def test_readline_limit(self): + # Read one line. StreamReaders are fed with data after + # their 'readline' methods are called. + + stream = asyncio.StreamReader(limit=7, loop=self.loop) + def cb(): + stream.feed_data(b'chunk1') + stream.feed_data(b'chunk2') + stream.feed_data(b'chunk3\n') + stream.feed_eof() + self.loop.call_soon(cb) + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # The buffer had just one line of data, and after raising + # a ValueError it should be empty. + self.assertEqual(b'', stream._buffer) + + stream = asyncio.StreamReader(limit=7, loop=self.loop) + def cb(): + stream.feed_data(b'chunk1') + stream.feed_data(b'chunk2\n') + stream.feed_data(b'chunk3\n') + stream.feed_eof() + self.loop.call_soon(cb) + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + self.assertEqual(b'chunk3\n', stream._buffer) + + # check strictness of the limit + stream = asyncio.StreamReader(limit=7, loop=self.loop) + stream.feed_data(b'1234567\n') + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'1234567\n', line) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'12345678\n') + with self.assertRaises(ValueError) as cm: + self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'12345678') + with self.assertRaises(ValueError) as cm: + self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', stream._buffer) + + def test_readline_nolimit_nowait(self): + # All needed data for the first 'readline' call will be + # in the buffer. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA[:6]) + stream.feed_data(self.DATA[6:]) + + line = self.loop.run_until_complete(stream.readline()) + + self.assertEqual(b'line1\n', line) + self.assertEqual(b'line2\nline3\n', stream._buffer) + + def test_readline_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data') + stream.feed_eof() + + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'some data', line) + + def test_readline_empty_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_eof() + + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', line) + + def test_readline_read_byte_count(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + self.loop.run_until_complete(stream.readline()) + + data = self.loop.run_until_complete(stream.read(7)) + + self.assertEqual(b'line2\nl', data) + self.assertEqual(b'ine3\n', stream._buffer) + + def test_readline_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'line\n', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=(b'',))) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=())) + + def test_readuntil_multi_chunks(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'lineAAA') + data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'lineAAA') + data = self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'lineAAAxxx') + data = self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'xxx', stream._buffer) + + def test_readuntil_multi_chunks_1(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'QWEaa') + stream.feed_data(b'XYaa') + stream.feed_data(b'a') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'QWEaaXYaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'QWEaa') + stream.feed_data(b'XYa') + stream.feed_data(b'aa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'QWEaaXYaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'aaa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'aaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'Xaaa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'Xaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'XXX') + stream.feed_data(b'a') + stream.feed_data(b'a') + stream.feed_data(b'a') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'XXXaaa', data) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + data = b'some dataAA' + stream.feed_data(data) + stream.feed_eof() + + with self.assertRaisesRegex(asyncio.IncompleteReadError, + 'undefined expected bytes') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(cm.exception.partial, data) + self.assertIsNone(cm.exception.expected) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_limit_found_sep(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataAA') + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'not found') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + + self.assertEqual(b'some dataAA', stream._buffer) + + stream.feed_data(b'A') + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + + self.assertEqual(b'some dataAAA', stream._buffer) + + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil((b'DEF', b'BCDE'))) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil((b'A', b'ome dataA'))) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil((b'A', b'long sep'))) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + + def test_readuntil_bytearray(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data\r\n') + data = self.loop.run_until_complete(stream.readuntil(bytearray(b'\r\n'))) + self.assertEqual(b'some data\r\n', data) + self.assertEqual(b'', stream._buffer) + + def test_readexactly_zero_or_less(self): + # Read exact number of bytes (zero or less). + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + data = self.loop.run_until_complete(stream.readexactly(0)) + self.assertEqual(b'', data) + self.assertEqual(self.DATA, stream._buffer) + + with self.assertRaisesRegex(ValueError, 'less than zero'): + self.loop.run_until_complete(stream.readexactly(-1)) + self.assertEqual(self.DATA, stream._buffer) + + def test_readexactly(self): + # Read exact number of bytes. + stream = asyncio.StreamReader(loop=self.loop) + + n = 2 * len(self.DATA) + read_task = self.loop.create_task(stream.readexactly(n)) + + def cb(): + stream.feed_data(self.DATA) + stream.feed_data(self.DATA) + stream.feed_data(self.DATA) + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(self.DATA + self.DATA, data) + self.assertEqual(self.DATA, stream._buffer) + + def test_readexactly_limit(self): + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'chunk') + data = self.loop.run_until_complete(stream.readexactly(5)) + self.assertEqual(b'chunk', data) + self.assertEqual(b'', stream._buffer) + + def test_readexactly_eof(self): + # Read exact number of bytes (eof). + stream = asyncio.StreamReader(loop=self.loop) + n = 2 * len(self.DATA) + read_task = self.loop.create_task(stream.readexactly(n)) + + def cb(): + stream.feed_data(self.DATA) + stream.feed_eof() + self.loop.call_soon(cb) + + with self.assertRaises(asyncio.IncompleteReadError) as cm: + self.loop.run_until_complete(read_task) + self.assertEqual(cm.exception.partial, self.DATA) + self.assertEqual(cm.exception.expected, n) + self.assertEqual(str(cm.exception), + '18 bytes read on a total of 36 expected bytes') + self.assertEqual(b'', stream._buffer) + + def test_readexactly_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.readexactly(2)) + self.assertEqual(b'li', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readexactly(2)) + + def test_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertIsNone(stream.exception()) + + exc = ValueError() + stream.set_exception(exc) + self.assertIs(stream.exception(), exc) + + def test_exception_waiter(self): + stream = asyncio.StreamReader(loop=self.loop) + + async def set_err(): + stream.set_exception(ValueError()) + + t1 = self.loop.create_task(stream.readline()) + t2 = self.loop.create_task(set_err()) + + self.loop.run_until_complete(asyncio.wait([t1, t2])) + + self.assertRaises(ValueError, t1.result) + + def test_exception_cancel(self): + stream = asyncio.StreamReader(loop=self.loop) + + t = self.loop.create_task(stream.readline()) + test_utils.run_briefly(self.loop) + t.cancel() + test_utils.run_briefly(self.loop) + # The following line fails if set_exception() isn't careful. + stream.set_exception(RuntimeError('message')) + test_utils.run_briefly(self.loop) + self.assertIs(stream._waiter, None) + + def test_start_server(self): + + class MyServer: + + def __init__(self, loop): + self.server = None + self.loop = loop + + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + sock = socket.create_server(('127.0.0.1', 0)) + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, + sock=sock)) + return sock.getsockname() + + def handle_client_callback(self, client_reader, client_writer): + self.loop.create_task(self.handle_client(client_reader, + client_writer)) + + def start_callback(self): + sock = socket.create_server(('127.0.0.1', 0)) + addr = sock.getsockname() + sock.close() + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client_callback, + host=addr[0], port=addr[1])) + return addr + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + # send a line + writer.write(b"hello world!\n") + # read it back + msgback = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + # test the server variant with a coroutine as client handler + server = MyServer(self.loop) + addr = server.start() + msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + # test the server variant with a callback as client handler + server = MyServer(self.loop) + addr = server.start_callback() + msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + self.assertEqual(messages, []) + + @socket_helper.skip_unless_bind_unix_socket + def test_start_unix_server(self): + + class MyServer: + + def __init__(self, loop, path): + self.server = None + self.loop = loop + self.path = path + + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + self.server = self.loop.run_until_complete( + asyncio.start_unix_server(self.handle_client, + path=self.path)) + + def handle_client_callback(self, client_reader, client_writer): + self.loop.create_task(self.handle_client(client_reader, + client_writer)) + + def start_callback(self): + start = asyncio.start_unix_server(self.handle_client_callback, + path=self.path) + self.server = self.loop.run_until_complete(start) + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(path): + reader, writer = await asyncio.open_unix_connection(path) + # send a line + writer.write(b"hello world!\n") + # read it back + msgback = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + # test the server variant with a coroutine as client handler + with test_utils.unix_socket_path() as path: + server = MyServer(self.loop, path) + server.start() + msg = self.loop.run_until_complete( + self.loop.create_task(client(path))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + # test the server variant with a callback as client handler + with test_utils.unix_socket_path() as path: + server = MyServer(self.loop, path) + server.start_callback() + msg = self.loop.run_until_complete( + self.loop.create_task(client(path))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + self.assertEqual(messages, []) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_start_tls(self): + + class MyServer: + + def __init__(self, loop): + self.server = None + self.loop = loop + + async def handle_client(self, client_reader, client_writer): + data1 = await client_reader.readline() + client_writer.write(data1) + await client_writer.drain() + assert client_writer.get_extra_info('sslcontext') is None + await client_writer.start_tls( + test_utils.simple_server_sslcontext()) + assert client_writer.get_extra_info('sslcontext') is not None + data2 = await client_reader.readline() + client_writer.write(data2) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + sock = socket.create_server(('127.0.0.1', 0)) + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, + sock=sock)) + return sock.getsockname() + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + writer.write(b"hello world 1!\n") + await writer.drain() + msgback1 = await reader.readline() + assert writer.get_extra_info('sslcontext') is None + await writer.start_tls(test_utils.simple_client_sslcontext()) + assert writer.get_extra_info('sslcontext') is not None + writer.write(b"hello world 2!\n") + await writer.drain() + msgback2 = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback1, msgback2 + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server = MyServer(self.loop) + addr = server.start() + msg1, msg2 = self.loop.run_until_complete(client(addr)) + server.stop() + + self.assertEqual(messages, []) + self.assertEqual(msg1, b"hello world 1!\n") + self.assertEqual(msg2, b"hello world 2!\n") + + def test_streamreader_constructor_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() + + def test_streamreader_constructor_use_running_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + async def test(): + return asyncio.StreamReader() + + reader = self.loop.run_until_complete(test()) + self.assertIs(reader._loop, self.loop) + + def test_streamreader_constructor_use_global_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + # Deprecated in 3.10, undeprecated in 3.12 + self.addCleanup(asyncio.set_event_loop, None) + asyncio.set_event_loop(self.loop) + reader = asyncio.StreamReader() + self.assertIs(reader._loop, self.loop) + + + def test_streamreaderprotocol_constructor_without_loop(self): + reader = mock.Mock() + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) + + def test_streamreaderprotocol_constructor_use_running_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + reader = mock.Mock() + async def test(): + return asyncio.StreamReaderProtocol(reader) + protocol = self.loop.run_until_complete(test()) + self.assertIs(protocol._loop, self.loop) + + def test_streamreaderprotocol_constructor_use_global_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + # Deprecated in 3.10, undeprecated in 3.12 + self.addCleanup(asyncio.set_event_loop, None) + asyncio.set_event_loop(self.loop) + reader = mock.Mock() + protocol = asyncio.StreamReaderProtocol(reader) + self.assertIs(protocol._loop, self.loop) + + def test_multiple_drain(self): + # See https://github.com/python/cpython/issues/74116 + drained = 0 + + async def drainer(stream): + nonlocal drained + await stream._drain_helper() + drained += 1 + + async def main(): + loop = asyncio.get_running_loop() + stream = asyncio.streams.FlowControlMixin(loop) + stream.pause_writing() + loop.call_later(0.1, stream.resume_writing) + await asyncio.gather(*[drainer(stream) for _ in range(10)]) + self.assertEqual(drained, 10) + + self.loop.run_until_complete(main()) + + def test_drain_raises(self): + # See http://bugs.python.org/issue25441 + + # This test should not use asyncio for the mock server; the + # whole point of the test is to test for a bug in drain() + # where it never gives up the event loop but the socket is + # closed on the server side. + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + q = queue.Queue() + + def server(): + # Runs in a separate thread. + with socket.create_server(('localhost', 0)) as sock: + addr = sock.getsockname() + q.put(addr) + clt, _ = sock.accept() + clt.close() + + async def client(host, port): + reader, writer = await asyncio.open_connection(host, port) + + while True: + writer.write(b"foo\n") + await writer.drain() + + # Start the server thread and wait for it to be listening. + thread = threading.Thread(target=server) + thread.daemon = True + thread.start() + addr = q.get() + + # Should not be stuck in an infinite loop. + with self.assertRaises((ConnectionResetError, ConnectionAbortedError, + BrokenPipeError)): + self.loop.run_until_complete(client(*addr)) + + # Clean up the thread. (Only on success; on failure, it may + # be stuck in accept().) + thread.join() + self.assertEqual([], messages) + + def test___repr__(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertEqual("", repr(stream)) + + def test___repr__nondefault_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=123) + self.assertEqual("", repr(stream)) + + def test___repr__eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_eof() + self.assertEqual("", repr(stream)) + + def test___repr__data(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + self.assertEqual("", repr(stream)) + + def test___repr__exception(self): + stream = asyncio.StreamReader(loop=self.loop) + exc = RuntimeError() + stream.set_exception(exc) + self.assertEqual("", + repr(stream)) + + def test___repr__waiter(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._waiter = asyncio.Future(loop=self.loop) + self.assertRegex( + repr(stream), + r">") + stream._waiter.set_result(None) + self.loop.run_until_complete(stream._waiter) + stream._waiter = None + self.assertEqual("", repr(stream)) + + def test___repr__transport(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._transport = mock.Mock() + stream._transport.__repr__ = mock.Mock() + stream._transport.__repr__.return_value = "" + self.assertEqual(">", repr(stream)) + + def test_IncompleteReadError_pickleable(self): + e = asyncio.IncompleteReadError(b'abc', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.partial, e2.partial) + self.assertEqual(e.expected, e2.expected) + + def test_LimitOverrunError_pickleable(self): + e = asyncio.LimitOverrunError('message', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.consumed, e2.consumed) + + def test_wait_closed_on_close(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + f = rd.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + self.assertFalse(wr.is_closing()) + wr.close() + self.assertTrue(wr.is_closing()) + self.loop.run_until_complete(wr.wait_closed()) + + def test_wait_closed_on_close_with_unread_data(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + wr.close() + self.loop.run_until_complete(wr.wait_closed()) + + def test_async_writer_api(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + wr.close() + await wr.wait_closed() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + def test_async_writer_api_exception_after_close(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + wr.close() + with self.assertRaises(ConnectionResetError): + wr.write(b'data') + await wr.drain() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + def test_eof_feed_when_closing_writer(self): + # See http://bugs.python.org/issue35065 + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.close() + f = wr.wait_closed() + self.loop.run_until_complete(f) + self.assertTrue(rd.at_eof()) + f = rd.read() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'') + + self.assertEqual(messages, []) + + def test_unclosed_resource_warnings(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertStartsWith(str(cm.warnings[0].message), "unclosed " + ) + transport._returncode = None + self.assertEqual( + repr(transport), + "" + ) + transport._pid = None + transport._returncode = None + self.assertEqual( + repr(transport), + "" + ) + transport.close() + + +class SubprocessMixin: + + def test_stdin_stdout(self): + args = PROGRAM_CAT + + async def run(data): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + # feed data + proc.stdin.write(data) + await proc.stdin.drain() + proc.stdin.close() + + # get output and exitcode + data = await proc.stdout.read() + exitcode = await proc.wait() + return (exitcode, data) + + task = run(b'some data') + task = asyncio.wait_for(task, 60.0) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_communicate(self): + args = PROGRAM_CAT + + async def run(data): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate(data) + return proc.returncode, stdout + + task = run(b'some data') + task = asyncio.wait_for(task, support.LONG_TIMEOUT) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_communicate_none_input(self): + args = PROGRAM_CAT + + async def run(): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + return proc.returncode, stdout + + task = run() + task = asyncio.wait_for(task, support.LONG_TIMEOUT) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'') + + def test_shell(self): + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell('exit 7') + ) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 7) + + def test_start_new_session(self): + # start the new process in a new session + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + 'exit 8', + start_new_session=True, + ) + ) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 8) + + def test_kill(self): + args = PROGRAM_BLOCKED + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec(*args) + ) + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_kill_issue43884(self): + if sys.platform == 'win32': + blocking_shell_command = f'"{sys.executable}" -c "import time; time.sleep(2)"' + else: + blocking_shell_command = 'sleep 1; sleep 1' + creationflags = 0 + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + # On windows create a new process group so that killing process + # kills the process and all its children. + creationflags = CREATE_NEW_PROCESS_GROUP + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE, + creationflags=creationflags) + ) + self.loop.run_until_complete(asyncio.sleep(1)) + if sys.platform == 'win32': + proc.send_signal(signal.CTRL_BREAK_EVENT) + # On windows it is an alias of terminate which sets the return code + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_terminate(self): + args = PROGRAM_BLOCKED + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec(*args) + ) + proc.terminate() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + def test_send_signal(self): + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + code = 'import time; print("sleeping", flush=True); time.sleep(3600)' + args = [sys.executable, '-c', code] + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + *args, + stdout=subprocess.PIPE, + ) + ) + + async def send_signal(proc): + # basic synchronization to wait until the program is sleeping + line = await proc.stdout.readline() + self.assertEqual(line, b'sleeping\n') + + proc.send_signal(signal.SIGHUP) + returncode = await proc.wait() + return returncode + + returncode = self.loop.run_until_complete(send_signal(proc)) + self.assertEqual(-signal.SIGHUP, returncode) + finally: + signal.signal(signal.SIGHUP, old_handler) + + def test_stdin_broken_pipe(self): + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + if support.MS_WINDOWS: + handle = msvcrt.get_osfhandle(rfd) + os.set_handle_inheritable(handle, True) + code = textwrap.dedent(f''' + import os, msvcrt + handle = {handle} + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + os.read(fd, 1) + ''') + from subprocess import STARTUPINFO + startupinfo = STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": [handle]} + kwargs = dict(startupinfo=startupinfo) + else: + code = f'import os; fd = {rfd}; os.read(fd, 1)' + kwargs = dict(pass_fds=(rfd,)) + + # the program ends before the stdin can be fed + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=subprocess.PIPE, + **kwargs + ) + ) + + async def write_stdin(proc, data): + proc.stdin.write(data) + # Only exit the child process once the write buffer is filled + os.write(wfd, b'go') + await proc.stdin.drain() + + coro = write_stdin(proc, large_data) + # drain() must raise BrokenPipeError or ConnectionResetError + with test_utils.disable_logger(): + self.assertRaises((BrokenPipeError, ConnectionResetError), + self.loop.run_until_complete, coro) + self.loop.run_until_complete(proc.wait()) + + def test_communicate_ignore_broken_pipe(self): + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + # the program ends before the stdin can be fed + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + sys.executable, '-c', 'pass', + stdin=subprocess.PIPE, + ) + ) + + # communicate() must ignore BrokenPipeError when feeding stdin + self.loop.set_exception_handler(lambda loop, msg: None) + self.loop.run_until_complete(proc.communicate(large_data)) + self.loop.run_until_complete(proc.wait()) + + def test_pause_reading(self): + limit = 10 + size = (limit * 2 + 1) + + async def test_pause_reading(): + code = '\n'.join(( + 'import sys', + 'sys.stdout.write("x" * %s)' % size, + 'sys.stdout.flush()', + )) + + connect_read_pipe = self.loop.connect_read_pipe + + async def connect_read_pipe_mock(*args, **kw): + transport, protocol = await connect_read_pipe(*args, **kw) + transport.pause_reading = mock.Mock() + transport.resume_reading = mock.Mock() + return (transport, protocol) + + self.loop.connect_read_pipe = connect_read_pipe_mock + + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + limit=limit, + ) + stdout_transport = proc._transport.get_pipe_transport(1) + + stdout, stderr = await proc.communicate() + + # The child process produced more than limit bytes of output, + # the stream reader transport should pause the protocol to not + # allocate too much memory. + return (stdout, stdout_transport) + + # Issue #22685: Ensure that the stream reader pauses the protocol + # when the child process produces too much data + stdout, transport = self.loop.run_until_complete(test_pause_reading()) + + self.assertEqual(stdout, b'x' * size) + self.assertTrue(transport.pause_reading.called) + self.assertTrue(transport.resume_reading.called) + + def test_stdin_not_inheritable(self): + # asyncio issue #209: stdin must not be inheritable, otherwise + # the Process.communicate() hangs + async def len_message(message): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(len_message(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + + def test_empty_input(self): + + async def empty_input(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b'') + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_input()) + self.assertEqual(output.rstrip(), b'0') + self.assertEqual(exitcode, 0) + + def test_devnull_input(self): + + async def empty_input(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate() + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_input()) + self.assertEqual(output.rstrip(), b'0') + self.assertEqual(exitcode, 0) + + def test_devnull_output(self): + + async def empty_output(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b"abc") + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_output()) + self.assertEqual(output, None) + self.assertEqual(exitcode, 0) + + def test_devnull_error(self): + + async def empty_error(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b"abc") + exitcode = await proc.wait() + return (stderr, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_error()) + self.assertEqual(output, None) + self.assertEqual(exitcode, 0) + + @unittest.skipIf(sys.platform not in ('linux', 'android'), + "Don't have /dev/stdin") + def test_devstdin_input(self): + + async def devstdin_input(message): + code = 'file = open("/dev/stdin"); data = file.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(devstdin_input(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + + def test_cancel_process_wait(self): + # Issue #23140: cancel Process.wait() + + async def cancel_wait(): + proc = await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED) + + # Create an internal future waiting on the process exit + task = self.loop.create_task(proc.wait()) + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # Cancel the future + task.cancel() + + # Kill the process and wait until it is done + proc.kill() + await proc.wait() + + self.loop.run_until_complete(cancel_wait()) + + def test_cancel_make_subprocess_transport_exec(self): + + async def cancel_make_transport(): + coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + + def test_cancel_post_init(self): + + async def cancel_make_transport(): + coro = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + test_utils.run_briefly(self.loop) + + def test_close_kill_running(self): + + async def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = await create + + kill_called = False + def kill(): + nonlocal kill_called + kill_called = True + orig_kill() + + proc = transport.get_extra_info('subprocess') + orig_kill = proc.kill + proc.kill = kill + returncode = transport.get_returncode() + transport.close() + await asyncio.wait_for(transport._wait(), 5) + return (returncode, kill_called) + + # Ignore "Close running child process: kill ..." log + with test_utils.disable_logger(): + try: + returncode, killed = self.loop.run_until_complete( + kill_running() + ) + except asyncio.TimeoutError: + self.skipTest( + "Timeout failure on waiting for subprocess stopping" + ) + self.assertIsNone(returncode) + + # transport.close() must kill the process if it is still running + self.assertTrue(killed) + test_utils.run_briefly(self.loop) + + def test_close_dont_kill_finished(self): + + async def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = await create + proc = transport.get_extra_info('subprocess') + + # kill the process (but asyncio is not notified immediately) + proc.kill() + proc.wait() + + proc.kill = mock.Mock() + proc_returncode = proc.poll() + transport_returncode = transport.get_returncode() + transport.close() + return (proc_returncode, transport_returncode, proc.kill.called) + + # Ignore "Unknown child process pid ..." log of SafeChildWatcher, + # emitted because the test already consumes the exit status: + # proc.wait() + with test_utils.disable_logger(): + result = self.loop.run_until_complete(kill_running()) + test_utils.run_briefly(self.loop) + + proc_returncode, transport_return_code, killed = result + + self.assertIsNotNone(proc_returncode) + self.assertIsNone(transport_return_code) + + # transport.close() must not kill the process if it finished, even if + # the transport was not notified yet + self.assertFalse(killed) + + async def _test_popen_error(self, stdin): + if sys.platform == 'win32': + target = 'asyncio.windows_utils.Popen' + else: + target = 'subprocess.Popen' + with mock.patch(target) as popen: + exc = ZeroDivisionError + popen.side_effect = exc + + with warnings.catch_warnings(record=True) as warns: + with self.assertRaises(exc): + await asyncio.create_subprocess_exec( + sys.executable, + '-c', + 'pass', + stdin=stdin + ) + self.assertEqual(warns, []) + + def test_popen_error(self): + # Issue #24763: check that the subprocess transport is closed + # when BaseSubprocessTransport fails + self.loop.run_until_complete(self._test_popen_error(stdin=None)) + + def test_popen_error_with_stdin_pipe(self): + # Issue #35721: check that newly created socket pair is closed when + # Popen fails + self.loop.run_until_complete( + self._test_popen_error(stdin=subprocess.PIPE)) + + def test_read_stdout_after_process_exit(self): + + async def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + process = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + ) + + while True: + data = await process.stdout.read(65536) + if data: + await asyncio.sleep(0.3) + else: + break + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_exec_text_mode_fails(self): + async def execute(): + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + text=True) + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + encoding="utf-8") + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + errors="strict") + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_shell_text_mode_fails(self): + + async def execute(): + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + text=True) + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + encoding="utf-8") + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + errors="strict") + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_exec_with_path(self): + async def execute(): + p = await subprocess.create_subprocess_exec( + os_helper.FakePath(sys.executable), '-c', 'pass') + await p.wait() + p = await subprocess.create_subprocess_exec( + sys.executable, '-c', 'pass', os_helper.FakePath('.')) + await p.wait() + + self.assertIsNone(self.loop.run_until_complete(execute())) + + async def check_stdout_output(self, coro, output): + proc = await coro + stdout, _ = await proc.communicate() + self.assertEqual(stdout, output) + self.assertEqual(proc.returncode, 0) + task = asyncio.create_task(proc.wait()) + await asyncio.sleep(0) + self.assertEqual(task.result(), proc.returncode) + + def test_create_subprocess_env_shell(self) -> None: + async def main() -> None: + executable = sys.executable + if sys.platform == "win32": + executable = f'"{executable}"' + cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + env = os.environ.copy() + env["FOO"] = "bar" + proc = await asyncio.create_subprocess_shell( + cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'bar')) + + def test_create_subprocess_env_exec(self) -> None: + async def main() -> None: + cmd = [sys.executable, "-c", + "import os, sys; sys.stdout.write(os.getenv('FOO'))"] + env = os.environ.copy() + env["FOO"] = "baz" + proc = await asyncio.create_subprocess_exec( + *cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'baz')) + + + def test_subprocess_concurrent_wait(self) -> None: + async def main() -> None: + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, _ = await proc.communicate(b'some data') + self.assertEqual(stdout, b"some data") + self.assertEqual(proc.returncode, 0) + self.assertEqual(await asyncio.gather(*[proc.wait() for _ in range(10)]), + [proc.returncode] * 10) + + self.loop.run_until_complete(main()) + + def test_subprocess_protocol_events(self): + # gh-108973: Test that all subprocess protocol methods are called. + # The protocol methods are not called in a deterministic order. + # The order depends on the event loop and the operating system. + events = [] + fds = [1, 2] + expected = [ + ('pipe_data_received', 1, b'stdout'), + ('pipe_data_received', 2, b'stderr'), + ('pipe_connection_lost', 1), + ('pipe_connection_lost', 2), + 'process_exited', + ] + per_fd_expected = [ + 'pipe_data_received', + 'pipe_connection_lost', + ] + + class MyProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future: asyncio.Future) -> None: + self.exit_future = exit_future + + def pipe_data_received(self, fd, data) -> None: + events.append(('pipe_data_received', fd, data)) + self.exit_maybe() + + def pipe_connection_lost(self, fd, exc) -> None: + events.append(('pipe_connection_lost', fd)) + self.exit_maybe() + + def process_exited(self) -> None: + events.append('process_exited') + self.exit_maybe() + + def exit_maybe(self): + # Only exit when we got all expected events + if len(events) >= len(expected): + self.exit_future.set_result(True) + + async def main() -> None: + loop = asyncio.get_running_loop() + exit_future = asyncio.Future() + code = 'import sys; sys.stdout.write("stdout"); sys.stderr.write("stderr")' + transport, _ = await loop.subprocess_exec(lambda: MyProtocol(exit_future), + sys.executable, '-c', code, stdin=None) + await exit_future + transport.close() + + return events + + events = self.loop.run_until_complete(main()) + + # First, make sure that we received all events + self.assertSetEqual(set(events), set(expected)) + + # Second, check order of pipe events per file descriptor + per_fd_events = {fd: [] for fd in fds} + for event in events: + if event == 'process_exited': + continue + name, fd = event[:2] + per_fd_events[fd].append(name) + + for fd in fds: + self.assertEqual(per_fd_events[fd], per_fd_expected, (fd, events)) + + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + + @unittest.skipIf(sys.platform != 'linux', "Linux only") + def test_subprocess_send_signal_race(self): + # See https://github.com/python/cpython/issues/87744 + async def main(): + for _ in range(10): + proc = await asyncio.create_subprocess_exec('sleep', '0.1') + await asyncio.sleep(0.1) + try: + proc.send_signal(signal.SIGUSR1) + except ProcessLookupError: + pass + self.assertNotEqual(await proc.wait(), 255) + + self.loop.run_until_complete(main()) + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() + +if sys.platform != 'win32': + # Unix + class SubprocessWatcherMixin(SubprocessMixin): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) + + + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + def setUp(self): + self._original_can_use_pidfd = unix_events.can_use_pidfd + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() + + def tearDown(self): + unix_events.can_use_pidfd = self._original_can_use_pidfd + return super().tearDown() + + @unittest.skipUnless( + unix_events.can_use_pidfd(), + "operating system does not support pidfds", + ) + class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + pass + +else: + # Windows + class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py new file mode 100644 index 00000000000..d4b2554dda9 --- /dev/null +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -0,0 +1,1166 @@ +# Adapted with permission from the EdgeDB project; +# license: PSFL. + +import weakref +import sys +import gc +import asyncio +import contextvars +import contextlib +from asyncio import taskgroups +import unittest +import warnings + +from test.test_asyncio.utils import await_without_task + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyExc(Exception): + pass + + +class MyBaseExc(BaseException): + pass + + +def get_error_types(eg): + return {type(exc) for exc in eg.exceptions} + + +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +@contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: + + async def test_taskgroup_01(self): + + async def foo1(): + await asyncio.sleep(0.1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + + self.assertEqual(t1.result(), 42) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_02(self): + + async def foo1(): + await asyncio.sleep(0.1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + await asyncio.sleep(0.15) + t2 = g.create_task(foo2()) + + self.assertEqual(t1.result(), 42) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_03(self): + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + await asyncio.sleep(0.15) + # cancel t1 explicitly, i.e. everything should continue + # working as expected. + t1.cancel() + + t2 = g.create_task(foo2()) + + self.assertTrue(t1.cancelled()) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_04(self): + + NUM = 0 + t2_cancel = False + t2 = None + + async def foo1(): + await asyncio.sleep(0.1) + 1 / 0 + + async def foo2(): + nonlocal NUM, t2_cancel + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + t2_cancel = True + raise + NUM += 1 + + async def runner(): + nonlocal NUM, t2 + + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + t2 = g.create_task(foo2()) + + NUM += 10 + + with self.assertRaises(ExceptionGroup) as cm: + await asyncio.create_task(runner()) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + self.assertEqual(NUM, 0) + self.assertTrue(t2_cancel) + self.assertTrue(t2.cancelled()) + + async def test_cancel_children_on_child_error(self): + # When a child task raises an error, the rest of the children + # are cancelled and the errors are gathered into an EG. + + NUM = 0 + t2_cancel = False + runner_cancel = False + + async def foo1(): + await asyncio.sleep(0.1) + 1 / 0 + + async def foo2(): + nonlocal NUM, t2_cancel + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + t2_cancel = True + raise + NUM += 1 + + async def runner(): + nonlocal NUM, runner_cancel + + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + g.create_task(foo1()) + g.create_task(foo1()) + g.create_task(foo2()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + runner_cancel = True + raise + + NUM += 10 + + # The 3 foo1 sub tasks can be racy when the host is busy - if the + # cancellation happens in the middle, we'll see partial sub errors here + with self.assertRaises(ExceptionGroup) as cm: + await asyncio.create_task(runner()) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + self.assertEqual(NUM, 0) + self.assertTrue(t2_cancel) + self.assertTrue(runner_cancel) + + async def test_cancellation(self): + + NUM = 0 + + async def foo(): + nonlocal NUM + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + NUM += 1 + raise + + async def runner(): + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError) as cm: + await r + + self.assertEqual(NUM, 5) + + async def test_taskgroup_07(self): + + NUM = 0 + + async def foo(): + nonlocal NUM + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + NUM += 1 + raise + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError): + await r + + self.assertEqual(NUM, 15) + + async def test_taskgroup_08(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_09(self): + + t1 = t2 = None + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + nonlocal t1, t2 + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + await asyncio.sleep(0.1) + 1 / 0 + + try: + await runner() + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {ZeroDivisionError}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertTrue(t1.cancelled()) + self.assertTrue(t2.cancelled()) + + async def test_taskgroup_10(self): + + t1 = t2 = None + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + nonlocal t1, t2 + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + 1 / 0 + + try: + await runner() + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {ZeroDivisionError}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertTrue(t1.cancelled()) + self.assertTrue(t2.cancelled()) + + async def test_taskgroup_11(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup(): + async with taskgroups.TaskGroup() as g2: + for _ in range(5): + g2.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ZeroDivisionError}) + + async def test_taskgroup_12(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(asyncio.sleep(10)) + + async with taskgroups.TaskGroup() as g2: + for _ in range(5): + g2.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ZeroDivisionError}) + + async def test_taskgroup_13(self): + + async def crash_after(t): + await asyncio.sleep(t) + raise ValueError(t) + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_after(0.1)) + + async with taskgroups.TaskGroup() as g2: + g2.create_task(crash_after(10)) + + r = asyncio.create_task(runner()) + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ValueError}) + + async def test_taskgroup_14(self): + + async def crash_after(t): + await asyncio.sleep(t) + raise ValueError(t) + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_after(10)) + + async with taskgroups.TaskGroup() as g2: + g2.create_task(crash_after(0.1)) + + r = asyncio.create_task(runner()) + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ValueError}) + + async def test_taskgroup_15(self): + + async def crash_soon(): + await asyncio.sleep(0.3) + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_soon()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(0.5) + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_16(self): + + async def crash_soon(): + await asyncio.sleep(0.3) + 1 / 0 + + async def nested_runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_soon()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(0.5) + raise + + async def runner(): + t = asyncio.create_task(nested_runner()) + await t + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_17(self): + NUM = 0 + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError): + await r + + self.assertEqual(NUM, 10) + + async def test_taskgroup_18(self): + NUM = 0 + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + # This isn't a good idea, but we have to support + # this weird case. + raise MyExc + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + + try: + await r + except ExceptionGroup as t: + self.assertEqual(get_error_types(t),{MyExc}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertEqual(NUM, 10) + + async def test_taskgroup_19(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise MyExc + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + r = asyncio.create_task(runner()) + try: + await r + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {MyExc, ZeroDivisionError}) + else: + self.fail('TasgGroupError was not raised') + + async def test_taskgroup_20(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise KeyboardInterrupt + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(KeyboardInterrupt): + await runner() + + async def test_taskgroup_20a(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise MyBaseExc + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(BaseExceptionGroup) as cm: + await runner() + + self.assertEqual( + get_error_types(cm.exception), {MyBaseExc, ZeroDivisionError} + ) + + async def _test_taskgroup_21(self): + # This test doesn't work as asyncio, currently, doesn't + # correctly propagate KeyboardInterrupt (or SystemExit) -- + # those cause the event loop itself to crash. + # (Compare to the previous (passing) test -- that one raises + # a plain exception but raises KeyboardInterrupt in nested(); + # this test does it the other way around.) + + async def crash_soon(): + await asyncio.sleep(0.1) + raise KeyboardInterrupt + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise TypeError + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(KeyboardInterrupt): + await runner() + + async def test_taskgroup_21a(self): + + async def crash_soon(): + await asyncio.sleep(0.1) + raise MyBaseExc + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise TypeError + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(BaseExceptionGroup) as cm: + await runner() + + self.assertEqual(get_error_types(cm.exception), {MyBaseExc, TypeError}) + + async def test_taskgroup_22(self): + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + g.create_task(foo2()) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.05) + r.cancel() + + with self.assertRaises(asyncio.CancelledError): + await r + + async def test_taskgroup_23(self): + + async def do_job(delay): + await asyncio.sleep(delay) + + async with taskgroups.TaskGroup() as g: + for count in range(10): + await asyncio.sleep(0.1) + g.create_task(do_job(0.3)) + if count == 5: + self.assertLess(len(g._tasks), 5) + await asyncio.sleep(1.35) + self.assertEqual(len(g._tasks), 0) + + async def test_taskgroup_24(self): + + async def root(g): + await asyncio.sleep(0.1) + g.create_task(coro1(0.1)) + g.create_task(coro1(0.2)) + + async def coro1(delay): + await asyncio.sleep(delay) + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(root(g)) + + await runner() + + async def test_taskgroup_25(self): + nhydras = 0 + + async def hydra(g): + nonlocal nhydras + nhydras += 1 + await asyncio.sleep(0.01) + g.create_task(hydra(g)) + g.create_task(hydra(g)) + + async def hercules(): + while nhydras < 10: + await asyncio.sleep(0.015) + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(hydra(g)) + g.create_task(hercules()) + + with self.assertRaises(ExceptionGroup) as cm: + await runner() + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + self.assertGreaterEqual(nhydras, 10) + + async def test_taskgroup_task_name(self): + async def coro(): + await asyncio.sleep(0) + async with taskgroups.TaskGroup() as g: + t = g.create_task(coro(), name="yolo") + self.assertEqual(t.get_name(), "yolo") + + async def test_taskgroup_task_context(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async with taskgroups.TaskGroup() as g: + ctx = contextvars.copy_context() + self.assertIsNone(ctx.get(cvar)) + t1 = g.create_task(coro(1), context=ctx) + await t1 + self.assertEqual(1, ctx.get(cvar)) + t2 = g.create_task(coro(2), context=ctx) + await t2 + self.assertEqual(2, ctx.get(cvar)) + + async def test_taskgroup_no_create_task_after_failure(self): + async def coro1(): + await asyncio.sleep(0.001) + 1 / 0 + async def coro2(g): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + with self.assertRaises(RuntimeError): + g.create_task(coro1()) + + with self.assertRaises(ExceptionGroup) as cm: + async with taskgroups.TaskGroup() as g: + g.create_task(coro1()) + g.create_task(coro2(g)) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_context_manager_exit_raises(self): + # See https://github.com/python/cpython/issues/95289 + class CustomException(Exception): + pass + + async def raise_exc(): + raise CustomException + + @contextlib.asynccontextmanager + async def database(): + try: + yield + finally: + raise CustomException + + async def main(): + task = asyncio.current_task() + try: + async with taskgroups.TaskGroup() as tg: + async with database(): + tg.create_task(raise_exc()) + await asyncio.sleep(1) + except* CustomException as err: + self.assertEqual(task.cancelling(), 0) + self.assertEqual(len(err.exceptions), 2) + + else: + self.fail('CustomException not raised') + + await asyncio.create_task(main()) + + async def test_taskgroup_already_entered(self): + tg = taskgroups.TaskGroup() + async with tg: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_double_enter(self): + tg = taskgroups.TaskGroup() + async with tg: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_finished(self): + async def create_task_after_tg_finish(): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + # Make sure the coroutine was closed when submitted to the inactive tg + # (if not closed, a RuntimeWarning should have been raised) + with warnings.catch_warnings(record=True) as w: + await create_task_after_tg_finish() + self.assertEqual(len(w), 0) + + async def test_taskgroup_not_entered(self): + tg = taskgroups.TaskGroup() + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + async def test_taskgroup_without_parent_task(self): + tg = taskgroups.TaskGroup() + with self.assertRaisesRegex(RuntimeError, "parent task"): + await await_without_task(tg.__aenter__()) + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + async def test_coro_closed_when_tg_closed(self): + async def run_coro_after_tg_closes(): + async with taskgroups.TaskGroup() as tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + await run_coro_after_tg_closes() + + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + + async def test_exception_refcycles_direct(self): + """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except ExceptionGroup as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_errors(self): + """Test that TaskGroup deletes self._errors, and __aexit__ args""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except* _Done as excs: + exc = excs.exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task(self): + """Test that TaskGroup deletes self._parent_task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + try: + async with asyncio.TaskGroup() as tg2: + tg2.create_task(coro_fn()) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_propagate_cancellation_error(self): + """Test that TaskGroup deletes propagate_cancellation_error""" + tg = asyncio.TaskGroup() + exc = None + + try: + async with asyncio.timeout(-1): + async with tg: + await asyncio.sleep(0) + except TimeoutError as e: + exc = e.__cause__ + + self.assertIsInstance(exc, asyncio.CancelledError) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_base_error(self): + """Test that TaskGroup deletes self._base_error""" + class MyKeyboardInterrupt(KeyboardInterrupt): + pass + + tg = asyncio.TaskGroup() + exc = None + + try: + async with tg: + raise MyKeyboardInterrupt + except MyKeyboardInterrupt as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_name(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async with asyncio.TaskGroup() as tg: + tg.create_task(asyncfn(), name="example name") + + self.assertEqual(name, "example name") + + + async def test_cancels_task_if_created_during_creation(self): + # regression test for gh-128550 + ran = False + class MyError(Exception): + pass + + exc = None + try: + async with asyncio.TaskGroup() as tg: + async def third_task(): + raise MyError("third task failed") + + async def second_task(): + nonlocal ran + tg.create_task(third_task()) + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(0) # eager tasks cancel here + await asyncio.sleep(0) # lazy tasks cancel here + ran = True + + tg.create_task(second_task()) + except* MyError as excs: + exc = excs.exceptions[0] + + self.assertTrue(ran) + self.assertIsInstance(exc, MyError) + + + async def test_cancellation_does_not_leak_out_of_tg(self): + class MyError(Exception): + pass + + async def throw_error(): + raise MyError + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(throw_error()) + except* MyError: + pass + else: + self.fail("should have raised one MyError in group") + + # if this test fails this current task will be cancelled + # outside the task group and inside unittest internals + # we yield to the event loop with sleep(0) so that + # cancellation happens here and error is more understandable + await asyncio.sleep(0) + + +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py new file mode 100644 index 00000000000..8a291f1cb7e --- /dev/null +++ b/Lib/test/test_asyncio/test_tasks.py @@ -0,0 +1,3763 @@ +"""Tests for tasks.py.""" + +import collections +import contextlib +import contextvars +import gc +import io +import random +import re +import sys +import traceback +import types +import unittest +from unittest import mock +from types import GenericAlias + +import asyncio +from asyncio import futures +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok +from test.support.warnings_helper import ignore_warnings + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +async def coroutine_function(): + pass + + +def format_coroutine(qualname, state, src, source_traceback, generator=False): + if generator: + state = '%s' % state + else: + state = '%s, defined' % state + if source_traceback is not None: + frame = source_traceback[-1] + return ('coro=<%s() %s at %s> created at %s:%s' + % (qualname, state, src, frame[0], frame[1])) + else: + return 'coro=<%s() %s at %s>' % (qualname, state, src) + + +def get_innermost_context(exc): + """ + Return information about the innermost exception context in the chain. + """ + depth = 0 + while True: + context = exc.__context__ + if context is None: + break + + exc = context + depth += 1 + + return (type(exc), exc.args, depth) + + +class Dummy: + + def __repr__(self): + return '' + + def __call__(self, *args): + pass + + +class CoroLikeObject: + def send(self, v): + raise StopIteration(42) + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + return self + + +class BaseTaskTests: + + Task = None + Future = None + all_tasks = None + + def new_task(self, loop, coro, name='TestTask', context=None, eager_start=None): + return self.__class__.Task(coro, loop=loop, name=name, context=context, eager_start=eager_start) + + def new_future(self, loop): + return self.__class__.Future(loop=loop) + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.loop.set_task_factory(self.new_task) + self.loop.create_future = lambda: self.new_future(self.loop) + + def test_generic_alias(self): + task = self.__class__.Task[str] + self.assertEqual(task.__args__, (str,)) + self.assertIsInstance(task, GenericAlias) + + def test_task_cancel_message_getter(self): + async def coro(): + pass + t = self.new_task(self.loop, coro()) + self.assertHasAttr(t, '_cancel_message') + self.assertEqual(t._cancel_message, None) + + t.cancel('my message') + self.assertEqual(t._cancel_message, 'my message') + + with self.assertRaises(asyncio.CancelledError) as cm: + self.loop.run_until_complete(t) + + self.assertEqual('my message', cm.exception.args[0]) + + def test_task_cancel_message_setter(self): + async def coro(): + pass + t = self.new_task(self.loop, coro()) + t.cancel('my message') + t._cancel_message = 'my new message' + self.assertEqual(t._cancel_message, 'my new message') + + with self.assertRaises(asyncio.CancelledError) as cm: + self.loop.run_until_complete(t) + + self.assertEqual('my new message', cm.exception.args[0]) + + def test_task_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + async def run(): + return Evil() + + self.loop.run_until_complete( + asyncio.gather(*[ + self.new_task(self.loop, run()) for _ in range(100) + ])) + + def test_other_loop_future(self): + other_loop = asyncio.new_event_loop() + fut = self.new_future(other_loop) + + async def run(fut): + await fut + + try: + with self.assertRaisesRegex(RuntimeError, + r'Task .* got Future .* attached'): + self.loop.run_until_complete(run(fut)) + finally: + other_loop.close() + + def test_task_awaits_on_itself(self): + + async def test(): + await task + + task = asyncio.ensure_future(test(), loop=self.loop) + + with self.assertRaisesRegex(RuntimeError, + 'Task cannot await on itself'): + self.loop.run_until_complete(task) + + def test_task_class(self): + async def notmuch(): + return 'ok' + t = self.new_task(self.loop, notmuch()) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t._loop, self.loop) + self.assertIs(t.get_loop(), self.loop) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + t = self.new_task(loop, notmuch()) + self.assertIs(t._loop, loop) + loop.run_until_complete(t) + loop.close() + + def test_ensure_future_coroutine(self): + async def notmuch(): + return 'ok' + t = asyncio.ensure_future(notmuch(), loop=self.loop) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + a = notmuch() + self.addCleanup(a.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) + + async def test(): + return asyncio.ensure_future(notmuch()) + t = self.loop.run_until_complete(test()) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + t = asyncio.ensure_future(notmuch()) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + def test_ensure_future_future(self): + f_orig = self.new_future(self.loop) + f_orig.set_result('ko') + + f = asyncio.ensure_future(f_orig) + self.loop.run_until_complete(f) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 'ko') + self.assertIs(f, f_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + f = asyncio.ensure_future(f_orig, loop=loop) + + loop.close() + + f = asyncio.ensure_future(f_orig, loop=self.loop) + self.assertIs(f, f_orig) + + def test_ensure_future_task(self): + async def notmuch(): + return 'ok' + t_orig = self.new_task(self.loop, notmuch()) + t = asyncio.ensure_future(t_orig) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t, t_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + t = asyncio.ensure_future(t_orig, loop=loop) + + loop.close() + + t = asyncio.ensure_future(t_orig, loop=self.loop) + self.assertIs(t, t_orig) + + def test_ensure_future_awaitable(self): + class Aw: + def __init__(self, coro): + self.coro = coro + def __await__(self): + return self.coro.__await__() + + async def coro(): + return 'ok' + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + fut = asyncio.ensure_future(Aw(coro()), loop=loop) + loop.run_until_complete(fut) + self.assertEqual(fut.result(), 'ok') + + def test_ensure_future_task_awaitable(self): + class Aw: + def __await__(self): + return asyncio.sleep(0, result='ok').__await__() + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + task = asyncio.ensure_future(Aw(), loop=loop) + loop.run_until_complete(task) + self.assertTrue(task.done()) + self.assertEqual(task.result(), 'ok') + self.assertIsInstance(task.get_coro(), types.CoroutineType) + loop.close() + + def test_ensure_future_neither(self): + with self.assertRaises(TypeError): + asyncio.ensure_future('ok') + + def test_ensure_future_error_msg(self): + loop = asyncio.new_event_loop() + f = self.new_future(self.loop) + with self.assertRaisesRegex(ValueError, 'The future belongs to a ' + 'different loop than the one specified as ' + 'the loop argument'): + asyncio.ensure_future(f, loop=loop) + loop.close() + + def test_get_stack(self): + T = None + + async def foo(): + await bar() + + async def bar(): + # test get_stack() + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + # test print_stack() + file = io.StringIO() + T.print_stack(limit=1, file=file) + file.seek(0) + tb = file.read() + self.assertRegex(tb, r'foo\(\) running') + + async def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + await T + + self.loop.run_until_complete(runner()) + + def test_task_repr(self): + self.loop.set_debug(False) + + async def notmuch(): + return 'abc' + + # test coroutine function + self.assertEqual(notmuch.__name__, 'notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr..notmuch') + self.assertEqual(notmuch.__module__, __name__) + + filename, lineno = test_utils.get_function_source(notmuch) + src = "%s:%s" % (filename, lineno) + + # test coroutine object + gen = notmuch() + coro_qualname = 'BaseTaskTests.test_task_repr..notmuch' + self.assertEqual(gen.__name__, 'notmuch') + self.assertEqual(gen.__qualname__, coro_qualname) + + # test pending Task + t = self.new_task(self.loop, gen) + t.add_done_callback(Dummy()) + + coro = format_coroutine(coro_qualname, 'running', src, + t._source_traceback, generator=True) + self.assertEqual(repr(t), + "()]>" % coro) + + # test cancelling Task + t.cancel() # Does not take immediate effect! + self.assertEqual(repr(t), + "()]>" % coro) + + # test cancelled Task + self.assertRaises(asyncio.CancelledError, + self.loop.run_until_complete, t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + "" % coro) + + # test finished Task + t = self.new_task(self.loop, notmuch()) + self.loop.run_until_complete(t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + "" % coro) + + def test_task_repr_autogenerated(self): + async def notmuch(): + return 123 + + t1 = self.new_task(self.loop, notmuch(), None) + t2 = self.new_task(self.loop, notmuch(), None) + self.assertNotEqual(repr(t1), repr(t2)) + + match1 = re.match(r"^' % re.escape(repr(fut))) + + fut.set_result(None) + self.loop.run_until_complete(task) + + def test_task_basics(self): + + async def outer(): + a = await inner1() + b = await inner2() + return a+b + + async def inner1(): + return 42 + + async def inner2(): + return 1000 + + t = outer() + self.assertEqual(self.loop.run_until_complete(t), 1042) + + def test_exception_chaining_after_await(self): + # Test that when awaiting on a task when an exception is already + # active, if the task raises an exception it will be chained + # with the original. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def raise_error(): + raise ValueError + + async def run(): + try: + raise KeyError(3) + except Exception as exc: + task = self.new_task(loop, raise_error()) + try: + await task + except Exception as exc: + self.assertEqual(type(exc), ValueError) + chained = exc.__context__ + self.assertEqual((type(chained), chained.args), + (KeyError, (3,))) + + try: + task = self.new_task(loop, run()) + loop.run_until_complete(task) + finally: + loop.close() + + def test_exception_chaining_after_await_with_context_cycle(self): + # Check trying to create an exception context cycle: + # https://bugs.python.org/issue40696 + has_cycle = None + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def process_exc(exc): + raise exc + + async def run(): + nonlocal has_cycle + try: + raise KeyError('a') + except Exception as exc: + task = self.new_task(loop, process_exc(exc)) + try: + await task + except BaseException as exc: + has_cycle = (exc is exc.__context__) + # Prevent a hang if has_cycle is True. + exc.__context__ = None + + try: + task = self.new_task(loop, run()) + loop.run_until_complete(task) + finally: + loop.close() + # This also distinguishes from the initial has_cycle=None. + self.assertEqual(has_cycle, False) + + + def test_cancelling(self): + loop = asyncio.new_event_loop() + + async def task(): + await asyncio.sleep(10) + + try: + t = self.new_task(loop, task()) + self.assertFalse(t.cancelling()) + self.assertNotIn(" cancelling ", repr(t)) + self.assertTrue(t.cancel()) + self.assertTrue(t.cancelling()) + self.assertIn(" cancelling ", repr(t)) + + # Since we commented out two lines from Task.cancel(), + # this t.cancel() call now returns True. + # self.assertFalse(t.cancel()) + self.assertTrue(t.cancel()) + + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + finally: + loop.close() + + def test_uncancel_basic(self): + loop = asyncio.new_event_loop() + + async def task(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + self.current_task().uncancel() + await asyncio.sleep(10) + + try: + t = self.new_task(loop, task()) + loop.run_until_complete(asyncio.sleep(0.01)) + + # Cancel first sleep + self.assertTrue(t.cancel()) + self.assertIn(" cancelling ", repr(t)) + self.assertEqual(t.cancelling(), 1) + self.assertFalse(t.cancelled()) # Task is still not complete + loop.run_until_complete(asyncio.sleep(0.01)) + + # after .uncancel() + self.assertNotIn(" cancelling ", repr(t)) + self.assertEqual(t.cancelling(), 0) + self.assertFalse(t.cancelled()) # Task is still not complete + + # Cancel second sleep + self.assertTrue(t.cancel()) + self.assertEqual(t.cancelling(), 1) + self.assertFalse(t.cancelled()) # Task is still not complete + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + self.assertTrue(t.cancelled()) # Finally, task complete + self.assertTrue(t.done()) + + # uncancel is no longer effective after the task is complete + t.uncancel() + self.assertTrue(t.cancelled()) + self.assertTrue(t.done()) + finally: + loop.close() + + def test_uncancel_structured_blocks(self): + # This test recreates the following high-level structure using uncancel():: + # + # async def make_request_with_timeout(): + # try: + # async with asyncio.timeout(1): + # # Structured block affected by the timeout: + # await make_request() + # await make_another_request() + # except TimeoutError: + # pass # There was a timeout + # # Outer code not affected by the timeout: + # await unrelated_code() + + loop = asyncio.new_event_loop() + + async def make_request_with_timeout(*, sleep: float, timeout: float): + task = self.current_task() + loop = task.get_loop() + + timed_out = False + structured_block_finished = False + outer_code_reached = False + + def on_timeout(): + nonlocal timed_out + timed_out = True + task.cancel() + + timeout_handle = loop.call_later(timeout, on_timeout) + try: + try: + # Structured block affected by the timeout + await asyncio.sleep(sleep) + structured_block_finished = True + finally: + timeout_handle.cancel() + if ( + timed_out + and task.uncancel() == 0 + and type(sys.exception()) is asyncio.CancelledError + ): + # Note the five rules that are needed here to satisfy proper + # uncancellation: + # + # 1. handle uncancellation in a `finally:` block to allow for + # plain returns; + # 2. our `timed_out` flag is set, meaning that it was our event + # that triggered the need to uncancel the task, regardless of + # what exception is raised; + # 3. we can call `uncancel()` because *we* called `cancel()` + # before; + # 4. we call `uncancel()` but we only continue converting the + # CancelledError to TimeoutError if `uncancel()` caused the + # cancellation request count go down to 0. We need to look + # at the counter vs having a simple boolean flag because our + # code might have been nested (think multiple timeouts). See + # commit 7fce1063b6e5a366f8504e039a8ccdd6944625cd for + # details. + # 5. we only convert CancelledError to TimeoutError; for other + # exceptions raised due to the cancellation (like + # a ConnectionLostError from a database client), simply + # propagate them. + # + # Those checks need to take place in this exact order to make + # sure the `cancelling()` counter always stays in sync. + # + # Additionally, the original stimulus to `cancel()` the task + # needs to be unscheduled to avoid re-cancelling the task later. + # Here we do it by cancelling `timeout_handle` in the `finally:` + # block. + raise TimeoutError + except TimeoutError: + self.assertTrue(timed_out) + + # Outer code not affected by the timeout: + outer_code_reached = True + await asyncio.sleep(0) + return timed_out, structured_block_finished, outer_code_reached + + try: + # Test which timed out. + t1 = self.new_task(loop, make_request_with_timeout(sleep=10.0, timeout=0.1)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t1) + ) + self.assertTrue(timed_out) + self.assertFalse(structured_block_finished) # it was cancelled + self.assertTrue(outer_code_reached) # task got uncancelled after leaving + # the structured block and continued until + # completion + self.assertEqual(t1.cancelling(), 0) # no pending cancellation of the outer task + + # Test which did not time out. + t2 = self.new_task(loop, make_request_with_timeout(sleep=0, timeout=10.0)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t2) + ) + self.assertFalse(timed_out) + self.assertTrue(structured_block_finished) + self.assertTrue(outer_code_reached) + self.assertEqual(t2.cancelling(), 0) + finally: + loop.close() + + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + + def test_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + async def task(): + await asyncio.sleep(10.0) + return 12 + + t = self.new_task(loop, task()) + loop.call_soon(t.cancel) + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_with_message_then_future_result(self): + # Test Future.result() after calling cancel() with a message. + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + await asyncio.sleep(0) + task.cancel(*cancel_args) + done, pending = await asyncio.wait([task]) + task.result() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, expected_args) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, expected_args, 0)) + + def test_cancel_with_message_then_future_exception(self): + # Test Future.exception() after calling cancel() with a message. + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + await asyncio.sleep(0) + task.cancel(*cancel_args) + done, pending = await asyncio.wait([task]) + task.exception() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, expected_args) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, expected_args, 0)) + + def test_cancellation_exception_context(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + fut = loop.create_future() + + async def sleep(): + fut.set_result(None) + await asyncio.sleep(10) + + async def coro(): + inner_task = self.new_task(loop, sleep()) + await fut + loop.call_soon(inner_task.cancel, 'msg') + try: + await inner_task + except asyncio.CancelledError as ex: + raise ValueError("cancelled") from ex + + task = self.new_task(loop, coro()) + with self.assertRaises(ValueError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, ('cancelled',)) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, ('msg',), 1)) + + def test_cancel_with_message_before_starting_task(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + # We deliberately leave out the sleep here. + task.cancel('my message') + done, pending = await asyncio.wait([task]) + task.exception() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, ('my message',)) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, ('my message',), 0)) + + def test_cancel_yield(self): + async def task(): + await asyncio.sleep(0) + await asyncio.sleep(0) + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) # start coro + t.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_inner_future(self): + f = self.new_future(self.loop) + + async def task(): + await f + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) # start task + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_both_task_and_inner_future(self): + f = self.new_future(self.loop) + + async def task(): + await f + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + + f.cancel() + t.cancel() + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + + self.assertTrue(t.done()) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_task_catching(self): + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + + async def task(): + await fut1 + try: + await fut2 + except asyncio.CancelledError: + return 42 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(t.cancelled()) + + def test_cancel_task_ignoring(self): + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + fut3 = self.new_future(self.loop) + + async def task(): + await fut1 + try: + await fut2 + except asyncio.CancelledError: + pass + res = await fut3 + return res + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut3) # White-box test. + fut3.set_result(42) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(fut3.cancelled()) + self.assertFalse(t.cancelled()) + + def test_cancel_current_task(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + # The sleep should be cancelled immediately. + await asyncio.sleep(100) + return 12 + + t = self.new_task(loop, task()) + self.assertFalse(t.cancelled()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertFalse(t.cancelled()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + + def test_cancel_awaited_task(self): + # This tests for a relatively rare condition when + # a task cancellation is requested for a task which is not + # currently blocked, such as a task cancelling itself. + # In this situation we must ensure that whatever next future + # or task the cancelled task blocks on is cancelled correctly + # as well. See also bpo-34872. + loop = asyncio.new_event_loop() + self.addCleanup(lambda: loop.close()) + + task = nested_task = None + fut = self.new_future(loop) + + async def nested(): + await fut + + async def coro(): + nonlocal nested_task + # Create a sub-task and wait for it to run. + nested_task = self.new_task(loop, nested()) + await asyncio.sleep(0) + + # Request the current task to be cancelled. + task.cancel() + # Block on the nested task, which should be immediately + # cancelled. + await nested_task + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(task) + + self.assertTrue(task.cancelled()) + self.assertTrue(nested_task.cancelled()) + self.assertTrue(fut.cancelled()) + + def assert_text_contains(self, text, substr): + if substr not in text: + raise RuntimeError(f'text {substr!r} not found in:\n>>>{text}<<<') + + def test_cancel_traceback_for_future_result(self): + # When calling Future.result() on a cancelled task, check that the + # line of code that was interrupted is included in the traceback. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def nested(): + # This will get cancelled immediately. + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, nested()) + await asyncio.sleep(0) + task.cancel() + await task # search target + + task = self.new_task(loop, coro()) + try: + loop.run_until_complete(task) + except asyncio.CancelledError: + tb = traceback.format_exc() + self.assert_text_contains(tb, "await asyncio.sleep(10)") + # The intermediate await should also be included. + self.assert_text_contains(tb, "await task # search target") + else: + self.fail('CancelledError did not occur') + + def test_cancel_traceback_for_future_exception(self): + # When calling Future.exception() on a cancelled task, check that the + # line of code that was interrupted is included in the traceback. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def nested(): + # This will get cancelled immediately. + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, nested()) + await asyncio.sleep(0) + task.cancel() + done, pending = await asyncio.wait([task]) + task.exception() # search target + + task = self.new_task(loop, coro()) + try: + loop.run_until_complete(task) + except asyncio.CancelledError: + tb = traceback.format_exc() + self.assert_text_contains(tb, "await asyncio.sleep(10)") + # The intermediate await should also be included. + self.assert_text_contains(tb, + "task.exception() # search target") + else: + self.fail('CancelledError did not occur') + + def test_stop_while_run_in_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0.1 + self.assertAlmostEqual(0.2, when) + when = yield 0.1 + self.assertAlmostEqual(0.3, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + x = 0 + + async def task(): + nonlocal x + while x < 10: + await asyncio.sleep(0.1) + x += 1 + if x == 2: + loop.stop() + + t = self.new_task(loop, task()) + with self.assertRaises(RuntimeError) as cm: + loop.run_until_complete(t) + self.assertEqual(str(cm.exception), + 'Event loop stopped before Future completed.') + self.assertFalse(t.done()) + self.assertEqual(x, 2) + self.assertAlmostEqual(0.3, loop.time()) + + t.cancel() + self.assertRaises(asyncio.CancelledError, loop.run_until_complete, t) + + def test_log_traceback(self): + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + task._log_traceback = True + self.loop.run_until_complete(task) + + def test_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait([b, a]) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(res, 42) + self.assertAlmostEqual(0.15, loop.time()) + + # Doing it again should take no time and exercise a different path. + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + self.assertEqual(res, 42) + + def test_wait_duplicate_coroutines(self): + + async def coro(s): + return s + c = self.loop.create_task(coro('test')) + task = self.new_task( + self.loop, + asyncio.wait([c, c, self.loop.create_task(coro('spam'))])) + + done, pending = self.loop.run_until_complete(task) + + self.assertFalse(pending) + self.assertEqual(set(f.result() for f in done), {'test', 'spam'}) + + def test_wait_errors(self): + self.assertRaises( + ValueError, self.loop.run_until_complete, + asyncio.wait(set())) + + # -1 is an invalid return_when value + sleep_coro = asyncio.sleep(10.0) + wait_coro = asyncio.wait([sleep_coro], return_when=-1) + self.assertRaises(ValueError, + self.loop.run_until_complete, wait_coro) + + sleep_coro.close() + + def test_wait_first_completed(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(10.0)) + b = self.new_task(loop, asyncio.sleep(0.1)) + task = self.new_task( + loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED)) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertFalse(a.done()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_really_done(self): + # there is possibility that some tasks in the pending list + # became done but their callbacks haven't all been called yet + + async def coro1(): + await asyncio.sleep(0) + + async def coro2(): + await asyncio.sleep(0) + await asyncio.sleep(0) + + a = self.new_task(self.loop, coro1()) + b = self.new_task(self.loop, coro2()) + task = self.new_task( + self.loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED)) + + done, pending = self.loop.run_until_complete(task) + self.assertEqual({a, b}, done) + self.assertTrue(a.done()) + self.assertIsNone(a.result()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + + def test_wait_first_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + # first_exception, task already has exception + a = self.new_task(loop, asyncio.sleep(10.0)) + + async def exc(): + raise ZeroDivisionError('err') + + b = self.new_task(loop, exc()) + task = self.new_task( + loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION)) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_first_exception_in_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.01, when) + yield 0.01 + + loop = self.new_test_loop(gen) + + # first_exception, exception during waiting + a = self.new_task(loop, asyncio.sleep(10.0)) + + async def exc(): + await asyncio.sleep(0.01) + raise ZeroDivisionError('err') + + b = self.new_task(loop, exc()) + task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0.01, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_with_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + + async def sleeper(): + await asyncio.sleep(0.15) + raise ZeroDivisionError('really') + + b = self.new_task(loop, sleeper()) + + async def foo(): + done, pending = await asyncio.wait([b, a]) + self.assertEqual(len(done), 2) + self.assertEqual(pending, set()) + errors = set(f for f in done if f.exception() is not None) + self.assertEqual(len(errors), 1) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + + def test_wait_with_timeout(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.11, when) + yield 0.11 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait([b, a], timeout=0.11) + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.11, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_concurrent_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + done, pending = loop.run_until_complete( + asyncio.wait([b, a], timeout=0.1)) + + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_with_iterator_of_tasks(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait(iter([b, a])) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(res, 42) + self.assertAlmostEqual(0.15, loop.time()) + + + def test_wait_generator(self): + async def func(a): + return a + + loop = self.new_test_loop() + + async def main(): + tasks = (self.new_task(loop, func(i)) for i in range(10)) + done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + self.assertEqual(len(done), 10) + self.assertEqual(len(pending), 0) + + loop.run_until_complete(main()) + + + def test_as_completed(self): + + def gen(): + yield 0 + yield 0 + yield 0.01 + yield 0 + + async def sleeper(dt, x): + nonlocal time_shifted + await asyncio.sleep(dt) + completed.add(x) + if not time_shifted and 'a' in completed and 'b' in completed: + time_shifted = True + loop.advance_time(0.14) + return x + + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values + + async def try_async_iterator(awaitables): + values = [] + async for f in asyncio.as_completed(awaitables): + values.append(await f) + return values + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') + + def test_as_completed_with_timeout(self): + + def gen(): + yield + yield 0 + yield 0 + yield 0.1 + + async def try_iterator(): + values = [] + for f in asyncio.as_completed([a, b], timeout=0.12): + if values: + loop.advance_time(0.02) + try: + v = await f + values.append((1, v)) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values + + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_as_completed_with_unused_timeout(self): + + def gen(): + yield + yield 0 + yield 0.01 + + async def try_iterator(): + for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') + + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. + def gen(): + yield 0 + yield 0.05 + yield 0 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.10, 'b') + fs = {a, b} + + async def test(): + futs = list(asyncio.as_completed(fs)) + self.assertEqual(len(futs), 2) + + x = await futs[1] + self.assertEqual(x, 'a') + self.assertAlmostEqual(0.05, loop.time()) + loop.advance_time(0.05) + y = await futs[0] + self.assertEqual(y, 'b') + self.assertAlmostEqual(0.10, loop.time()) + + loop.run_until_complete(test()) + + def test_as_completed_concurrent(self): + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0 + self.assertAlmostEqual(0.05, when) + yield 0.05 + + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) + + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) + + def test_as_completed_duplicate_coroutines(self): + + async def coro(s): + return s + + async def try_iterator(): + result = [] + c = coro('ham') + for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) + + def test_as_completed_coroutine_without_loop(self): + async def coro(): + return 42 + + a = coro() + self.addCleanup(a.close) + + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) + list(futs) + + def test_as_completed_coroutine_use_running_loop(self): + loop = self.new_test_loop() + + async def coro(): + return 42 + + async def test(): + futs = list(asyncio.as_completed([coro()])) + self.assertEqual(len(futs), 1) + self.assertEqual(await futs[0], 42) + + loop.run_until_complete(test()) + + def test_sleep(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0.05 + self.assertAlmostEqual(0.1, when) + yield 0.05 + + loop = self.new_test_loop(gen) + + async def sleeper(dt, arg): + await asyncio.sleep(dt/2) + res = await asyncio.sleep(dt/2, arg) + return res + + t = self.new_task(loop, sleeper(0.1, 'yeah')) + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'yeah') + self.assertAlmostEqual(0.1, loop.time()) + + def test_sleep_when_delay_is_nan(self): + + def gen(): + yield + + loop = self.new_test_loop(gen) + + async def sleeper(): + await asyncio.sleep(float("nan")) + + t = self.new_task(loop, sleeper()) + + with self.assertRaises(ValueError): + loop.run_until_complete(t) + + def test_sleep_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + t = self.new_task(loop, asyncio.sleep(10.0, 'yeah')) + + handle = None + orig_call_later = loop.call_later + + def call_later(delay, callback, *args): + nonlocal handle + handle = orig_call_later(delay, callback, *args) + return handle + + loop.call_later = call_later + test_utils.run_briefly(loop) + + self.assertFalse(handle._cancelled) + + t.cancel() + test_utils.run_briefly(loop) + self.assertTrue(handle._cancelled) + + def test_task_cancel_sleeping_task(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(5000, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + async def sleep(dt): + await asyncio.sleep(dt) + + async def doit(): + sleeper = self.new_task(loop, sleep(5000)) + loop.call_later(0.1, sleeper.cancel) + try: + await sleeper + except asyncio.CancelledError: + return 'cancelled' + else: + return 'slept in' + + doer = doit() + self.assertEqual(loop.run_until_complete(doer), 'cancelled') + self.assertAlmostEqual(0.1, loop.time()) + + def test_task_cancel_waiter_future(self): + fut = self.new_future(self.loop) + + async def coro(): + await fut + + task = self.new_task(self.loop, coro()) + test_utils.run_briefly(self.loop) + self.assertIs(task._fut_waiter, fut) + + task.cancel() + test_utils.run_briefly(self.loop) + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, task) + self.assertIsNone(task._fut_waiter) + self.assertTrue(fut.cancelled()) + + def test_task_set_methods(self): + async def notmuch(): + return 'ko' + + gen = notmuch() + task = self.new_task(self.loop, gen) + + with self.assertRaisesRegex(RuntimeError, 'not support set_result'): + task.set_result('ok') + + with self.assertRaisesRegex(RuntimeError, 'not support set_exception'): + task.set_exception(ValueError()) + + self.assertEqual( + self.loop.run_until_complete(task), + 'ko') + + def test_step_result_future(self): + # If coroutine returns future, task waits on this future. + + class Fut(asyncio.Future): + def __init__(self, *args, **kwds): + self.cb_added = False + super().__init__(*args, **kwds) + + def add_done_callback(self, *args, **kwargs): + self.cb_added = True + super().add_done_callback(*args, **kwargs) + + fut = Fut(loop=self.loop) + result = None + + async def wait_for_future(): + nonlocal result + result = await fut + + t = self.new_task(self.loop, wait_for_future()) + test_utils.run_briefly(self.loop) + self.assertTrue(fut.cb_added) + + res = object() + fut.set_result(res) + test_utils.run_briefly(self.loop) + self.assertIs(res, result) + self.assertTrue(t.done()) + self.assertIsNone(t.result()) + + def test_baseexception_during_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + async def sleeper(): + await asyncio.sleep(10) + + base_exc = SystemExit() + + async def notmutch(): + try: + await sleeper() + except asyncio.CancelledError: + raise base_exc + + task = self.new_task(loop, notmutch()) + test_utils.run_briefly(loop) + + task.cancel() + self.assertFalse(task.done()) + + self.assertRaises(SystemExit, test_utils.run_briefly, loop) + + self.assertTrue(task.done()) + self.assertFalse(task.cancelled()) + self.assertIs(task.exception(), base_exc) + + @ignore_warnings(category=DeprecationWarning) + def test_iscoroutinefunction(self): + def fn(): + pass + + self.assertFalse(asyncio.iscoroutinefunction(fn)) + + def fn1(): + yield + self.assertFalse(asyncio.iscoroutinefunction(fn1)) + + async def fn2(): + pass + self.assertTrue(asyncio.iscoroutinefunction(fn2)) + + self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) + self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) + + @ignore_warnings(category=DeprecationWarning) + def test_coroutine_non_gen_function(self): + async def func(): + return 'test' + + self.assertTrue(asyncio.iscoroutinefunction(func)) + + coro = func() + self.assertTrue(asyncio.iscoroutine(coro)) + + res = self.loop.run_until_complete(coro) + self.assertEqual(res, 'test') + + def test_coroutine_non_gen_function_return_future(self): + fut = self.new_future(self.loop) + + async def func(): + return fut + + async def coro(): + fut.set_result('test') + + t1 = self.new_task(self.loop, func()) + t2 = self.new_task(self.loop, coro()) + res = self.loop.run_until_complete(t1) + self.assertEqual(res, fut) + self.assertIsNone(t2.result()) + + def test_current_task(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + async def coro(loop): + self.assertIs(self.current_task(), task) + + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) + + task = self.new_task(self.loop, coro(self.loop)) + self.loop.run_until_complete(task) + self.assertIsNone(self.current_task(loop=self.loop)) + + def test_current_task_with_interleaving_tasks(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + + async def coro1(loop): + self.assertTrue(self.current_task() is task1) + await fut1 + self.assertTrue(self.current_task() is task1) + fut2.set_result(True) + + async def coro2(loop): + self.assertTrue(self.current_task() is task2) + fut1.set_result(True) + await fut2 + self.assertTrue(self.current_task() is task2) + + task1 = self.new_task(self.loop, coro1(self.loop)) + task2 = self.new_task(self.loop, coro2(self.loop)) + + self.loop.run_until_complete(asyncio.wait((task1, task2))) + self.assertIsNone(self.current_task(loop=self.loop)) + + # Some thorough tests for cancellation propagation through + # coroutines, tasks and wait(). + + def test_yield_future_passes_cancel(self): + # Cancelling outer() cancels inner() cancels waiter. + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + try: + await waiter + except asyncio.CancelledError: + proof += 1 + raise + else: + self.fail('got past sleep() in inner()') + + async def outer(): + nonlocal proof + try: + await inner() + except asyncio.CancelledError: + proof += 100 # Expect this path. + else: + proof += 10 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.loop.run_until_complete(f) + self.assertEqual(proof, 101) + self.assertTrue(waiter.cancelled()) + + def test_yield_wait_does_not_shield_cancel(self): + # Cancelling outer() makes wait() return early, leaves inner() + # running. + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + async def outer(): + nonlocal proof + with self.assertWarns(DeprecationWarning): + d, p = await asyncio.wait([asyncio.create_task(inner())]) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_result(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + inner.set_result(42) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_shield_exception(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + exc = RuntimeError('expected') + inner.set_exception(exc) + test_utils.run_briefly(self.loop) + self.assertIs(outer.exception(), exc) + + def test_shield_cancel_inner(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + inner.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + + def test_shield_cancel_outer(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_shortcut(self): + fut = self.new_future(self.loop) + fut.set_result(42) + res = self.loop.run_until_complete(asyncio.shield(fut)) + self.assertEqual(res, 42) + + def test_shield_effect(self): + # Cancelling outer() does not affect inner(). + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + async def outer(): + nonlocal proof + await asyncio.shield(inner()) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_gather(self): + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) + parent = asyncio.gather(child1, child2) + outer = asyncio.shield(parent) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + self.assertEqual(parent.result(), [1, 2]) + + def test_gather_shield(self): + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) + inner1 = asyncio.shield(child1) + inner2 = asyncio.shield(child2) + parent = asyncio.gather(inner1, inner2) + test_utils.run_briefly(self.loop) + parent.cancel() + # This should cancel inner1 and inner2 but bot child1 and child2. + test_utils.run_briefly(self.loop) + self.assertIsInstance(parent.exception(), asyncio.CancelledError) + self.assertTrue(inner1.cancelled()) + self.assertTrue(inner2.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + + def test_shield_coroutine_without_loop(self): + async def coro(): + return 42 + + inner = coro() + self.addCleanup(inner.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) + + def test_shield_coroutine_use_running_loop(self): + async def coro(): + return 42 + + async def test(): + return asyncio.shield(coro()) + outer = self.loop.run_until_complete(test()) + self.assertEqual(outer._loop, self.loop) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_shield_coroutine_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + async def coro(): + return 42 + + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + outer = asyncio.shield(coro()) + self.assertEqual(outer._loop, self.loop) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() + + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) + coro = coroutine_function() + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) + coro.close() + + def test_wait_invalid_args(self): + fut = self.new_future(self.loop) + + # wait() expects a list of futures, not a future instance + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(fut)) + coro = coroutine_function() + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(coro)) + coro.close() + + # wait() expects at least a future + self.assertRaises(ValueError, self.loop.run_until_complete, + asyncio.wait([])) + + def test_log_destroyed_pending_task(self): + + async def kill_me(loop): + future = self.new_future(loop) + await future + # at this point, the only reference to kill_me() task is + # the Task._wakeup() method in future._callbacks + raise Exception("code never reached") + + mock_handler = mock.Mock() + self.loop.set_debug(True) + self.loop.set_exception_handler(mock_handler) + + # schedule the task + coro = kill_me(self.loop) + task = self.new_task(self.loop, coro) + + self.assertEqual(self.all_tasks(loop=self.loop), {task}) + + # execute the task so it waits for future + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(len(self.loop._ready), 0) + + coro = None + source_traceback = task._source_traceback + task = None + + # no more reference to kill_me() task: the task is destroyed by the GC + support.gc_collect() + + mock_handler.assert_called_with(self.loop, { + 'message': 'Task was destroyed but it is pending!', + 'task': mock.ANY, + 'source_traceback': source_traceback, + }) + mock_handler.reset_mock() + # task got resurrected by the exception handler + support.gc_collect() + + self.assertEqual(self.all_tasks(loop=self.loop), set()) + + def test_task_not_crash_without_finalization(self): + Task = self.__class__.Task + + class Subclass(Task): + def __del__(self): + pass + + async def corofn(): + await asyncio.sleep(0.01) + + coro = corofn() + task = Subclass(coro, loop = self.loop) + task._log_destroy_pending = False + + del task + + support.gc_collect() + + coro.close() + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def coro(): + raise TypeError + + async def runner(): + task = self.new_task(loop, coro()) + await asyncio.sleep(0.05) + task.cancel() + task = None + + loop.run_until_complete(runner()) + self.assertFalse(m_log.error.called) + + def test_task_source_traceback(self): + self.loop.set_debug(True) + + task = self.new_task(self.loop, coroutine_function()) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(task._source_traceback, list) + self.assertEqual(task._source_traceback[-2][:3], + (__file__, + lineno, + 'test_task_source_traceback')) + self.loop.run_until_complete(task) + + def test_cancel_gather_1(self): + """Ensure that a gathering future refuses to be cancelled once all + children are done""" + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + fut = self.new_future(loop) + async def create(): + # The indirection fut->child_coro is needed since otherwise the + # gathering task is done at the same time as the child future + async def child_coro(): + return await fut + gather_future = asyncio.gather(child_coro()) + return asyncio.ensure_future(gather_future) + gather_task = loop.run_until_complete(create()) + + cancel_result = None + def cancelling_callback(_): + nonlocal cancel_result + cancel_result = gather_task.cancel() + fut.add_done_callback(cancelling_callback) + + fut.set_result(42) # calls the cancelling_callback after fut is done() + + # At this point the task should complete. + loop.run_until_complete(gather_task) + + # Python issue #26923: asyncio.gather drops cancellation + self.assertEqual(cancel_result, False) + self.assertFalse(gather_task.cancelled()) + self.assertEqual(gather_task.result(), [42]) + + def test_cancel_gather_2(self): + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + async def test(): + time = 0 + while True: + time += 0.05 + await asyncio.gather(asyncio.sleep(0.05), + return_exceptions=True) + if time > 1: + return + + async def main(): + qwe = self.new_task(loop, test()) + await asyncio.sleep(0.2) + qwe.cancel(*cancel_args) + await qwe + + try: + loop.run_until_complete(main()) + except asyncio.CancelledError as exc: + self.assertEqual(exc.args, expected_args) + actual = get_innermost_context(exc) + self.assertEqual( + actual, + (asyncio.CancelledError, expected_args, 0), + ) + else: + self.fail( + 'gather() does not propagate CancelledError ' + 'raised by inner task to the gather() caller.' + ) + + def test_exception_traceback(self): + # See http://bugs.python.org/issue28843 + + async def foo(): + 1 / 0 + + async def main(): + task = self.new_task(self.loop, foo()) + await asyncio.sleep(0) # skip one loop iteration + self.assertIsNotNone(task.exception().__traceback__) + + self.loop.run_until_complete(main()) + + @mock.patch('asyncio.base_events.logger') + def test_error_in_call_soon(self, m_log): + def call_soon(callback, *args, **kwargs): + raise ValueError + self.loop.call_soon = call_soon + + async def coro(): + pass + + self.assertFalse(m_log.error.called) + + with self.assertRaises(ValueError): + gen = coro() + try: + self.new_task(self.loop, gen) + finally: + gen.close() + gc.collect() # For PyPy or other GCs. + + self.assertTrue(m_log.error.called) + message = m_log.error.call_args[0][0] + self.assertIn('Task was destroyed but it is pending', message) + + self.assertEqual(self.all_tasks(self.loop), set()) + + def test_create_task_with_noncoroutine(self): + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + def test_create_task_with_async_function(self): + + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_asynclike_function(self): + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + def test_bare_create_task(self): + + async def inner(): + return 1 + + async def coro(): + task = asyncio.create_task(inner()) + self.assertIsInstance(task, self.Task) + ret = await task + self.assertEqual(1, ret) + + self.loop.run_until_complete(coro()) + + def test_bare_create_named_task(self): + + async def coro_noop(): + pass + + async def coro(): + task = asyncio.create_task(coro_noop(), name='No-op') + self.assertEqual(task.get_name(), 'No-op') + await task + + self.loop.run_until_complete(coro()) + + def test_context_1(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def sub(): + await asyncio.sleep(0.01) + self.assertEqual(cvar.get(), 'nope') + cvar.set('something else') + + async def main(): + self.assertEqual(cvar.get(), 'nope') + subtask = self.new_task(loop, sub()) + cvar.set('yes') + self.assertEqual(cvar.get(), 'yes') + await subtask + self.assertEqual(cvar.get(), 'yes') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + def test_context_2(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def main(): + def fut_on_done(fut): + # This change must not pollute the context + # of the "main()" task. + cvar.set('something else') + + self.assertEqual(cvar.get(), 'nope') + + for j in range(2): + fut = self.new_future(loop) + fut.add_done_callback(fut_on_done) + cvar.set(f'yes{j}') + loop.call_soon(fut.set_result, None) + await fut + self.assertEqual(cvar.get(), f'yes{j}') + + for i in range(3): + # Test that task passed its context to add_done_callback: + cvar.set(f'yes{i}-{j}') + await asyncio.sleep(0.001) + self.assertEqual(cvar.get(), f'yes{i}-{j}') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual(cvar.get(), 'nope') + + def test_context_3(self): + # Run 100 Tasks in parallel, each modifying cvar. + + cvar = contextvars.ContextVar('cvar', default=-1) + + async def sub(num): + for i in range(10): + cvar.set(num + i) + await asyncio.sleep(random.uniform(0.001, 0.05)) + self.assertEqual(cvar.get(), num + i) + + async def main(): + tasks = [] + for i in range(100): + task = loop.create_task(sub(random.randint(0, 10))) + tasks.append(task) + + await asyncio.gather(*tasks) + + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + self.assertEqual(cvar.get(), -1) + + def test_context_4(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = self.new_task(loop, coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = self.new_task(loop, coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_5(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = asyncio.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = asyncio.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_6(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = loop.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = loop.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = loop.create_task(main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_eager_start_true(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=True, name="example") + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=False, name="example") + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + + def test_get_coro(self): + loop = asyncio.new_event_loop() + coro = coroutine_function() + try: + task = self.new_task(loop, coro) + loop.run_until_complete(task) + self.assertIs(task.get_coro(), coro) + finally: + loop.close() + + def test_get_context(self): + loop = asyncio.new_event_loop() + coro = coroutine_function() + context = contextvars.copy_context() + try: + task = self.new_task(loop, coro, context=context) + loop.run_until_complete(task) + self.assertIs(task.get_context(), context) + finally: + loop.close() + + def test_proper_refcounts(self): + # see: https://github.com/python/cpython/issues/126083 + class Break: + def __str__(self): + raise RuntimeError("break") + + obj = object() + initial_refcount = sys.getrefcount(obj) + + coro = coroutine_function() + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) + + coro.close() + task._log_destroy_pending = False + del task + + self.assertEqual(sys.getrefcount(obj), initial_refcount) + + +def add_subclass_tests(cls): + BaseTask = cls.Task + BaseFuture = cls.Future + + if BaseTask is None or BaseFuture is None: + return cls + + class CommonFuture: + def __init__(self, *args, **kwargs): + self.calls = collections.defaultdict(lambda: 0) + super().__init__(*args, **kwargs) + + def add_done_callback(self, *args, **kwargs): + self.calls['add_done_callback'] += 1 + return super().add_done_callback(*args, **kwargs) + + class Task(CommonFuture, BaseTask): + pass + + class Future(CommonFuture, BaseFuture): + pass + + def test_subclasses_ctask_cfuture(self): + fut = self.Future(loop=self.loop) + + async def func(): + self.loop.call_soon(lambda: fut.set_result('spam')) + return await fut + + task = self.Task(func(), loop=self.loop) + + result = self.loop.run_until_complete(task) + + self.assertEqual(result, 'spam') + + self.assertEqual( + dict(task.calls), + {'add_done_callback': 1}) + + self.assertEqual( + dict(fut.calls), + {'add_done_callback': 1}) + + # Add patched Task & Future back to the test case + cls.Task = Task + cls.Future = Future + + # Add an extra unit-test + cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture + + # Disable the "test_task_source_traceback" test + # (the test is hardcoded for a particular call stack, which + # is slightly different for Task subclasses) + cls.test_task_source_traceback = None + + return cls + + +class SetMethodsTest: + + def test_set_result_causes_invalid_state(self): + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_result(task, 'spam') + + self.assertEqual( + self.loop.run_until_complete(task), + 'spam') + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + def test_set_exception_causes_invalid_state(self): + class MyExc(Exception): + pass + + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_exception(task, MyExc()) + + with self.assertRaises(MyExc): + self.loop.run_until_complete(task) + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @support.refcount_test + def test_refleaks_in_task___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + async def coro(): + pass + task = self.new_task(self.loop, coro()) + self.loop.run_until_complete(task) + refs_before = gettotalrefcount() + for i in range(100): + task.__init__(coro(), loop=self.loop) + self.loop.run_until_complete(task) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_del__log_destroy_pending_segfault(self): + async def coro(): + pass + task = self.new_task(self.loop, coro()) + self.loop.run_until_complete(task) + with self.assertRaises(AttributeError): + del task._log_destroy_pending + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + + +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +@add_subclass_tests +class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): + + Future = getattr(futures, '_CFuture', None) + Task = tasks._PyTask + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = tasks._PyTask + Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + + Task = tasks._PyTask + Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@add_subclass_tests +class PyTask_PyFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask + Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_Future_Tests(test_utils.TestCase): + + def test_foobar(self): + class Fut(asyncio.Future): + @property + def get_loop(self): + raise AttributeError + + async def coro(): + await fut + return 'spam' + + self.loop = asyncio.new_event_loop() + try: + fut = Fut(loop=self.loop) + self.loop.call_later(0.1, fut.set_result, 1) + task = self.loop.create_task(coro()) + res = self.loop.run_until_complete(task) + finally: + self.loop.close() + + self.assertEqual(res, 'spam') + + +class BaseTaskIntrospectionTests: + _register_task = None + _unregister_task = None + _enter_task = None + _leave_task = None + all_tasks = None + + def test__register_task_1(self): + class TaskLike: + @property + def _loop(self): + return loop + + def done(self): + return False + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__register_task_2(self): + class TaskLike: + def get_loop(self): + return loop + + def done(self): + return False + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__register_task_3(self): + class TaskLike: + def get_loop(self): + return loop + + def done(self): + return True + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), set()) + self._unregister_task(task) + + def test__enter_task(self): + task = mock.Mock() + loop = mock.Mock() + # _enter_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) + self.assertIsNone(self.current_task(loop)) + self._enter_task(loop, task) + self.assertIs(self.current_task(loop), task) + self._leave_task(loop, task) + asyncio._set_running_loop(None) + + def test__enter_task_failure(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._enter_task(loop, task2) + self.assertIs(self.current_task(loop), task1) + self._leave_task(loop, task1) + asyncio._set_running_loop(None) + + def test__leave_task(self): + task = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + self._enter_task(loop, task) + self._leave_task(loop, task) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) + + def test__leave_task_failure1(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + # _leave_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._leave_task(loop, task2) + self.assertIs(self.current_task(loop), task1) + self._leave_task(loop, task1) + asyncio._set_running_loop(None) + + def test__leave_task_failure2(self): + task = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + with self.assertRaises(RuntimeError): + self._leave_task(loop, task) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) + + def test__unregister_task(self): + task = mock.Mock() + loop = mock.Mock() + task.get_loop = lambda: loop + self._register_task(task) + self._unregister_task(task) + self.assertEqual(self.all_tasks(loop), set()) + + def test__unregister_task_not_registered(self): + task = mock.Mock() + loop = mock.Mock() + self._unregister_task(task) + self.assertEqual(self.all_tasks(loop), set()) + + +class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): + _register_task = staticmethod(tasks._py_register_task) + _unregister_task = staticmethod(tasks._py_unregister_task) + _enter_task = staticmethod(tasks._py_enter_task) + _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + +@unittest.skipUnless(hasattr(tasks, '_c_register_task'), + 'requires the C _asyncio module') +class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): + if hasattr(tasks, '_c_register_task'): + _register_task = staticmethod(tasks._c_register_task) + _unregister_task = staticmethod(tasks._c_unregister_task) + _enter_task = staticmethod(tasks._c_enter_task) + _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) + current_task = staticmethod(tasks._c_current_task) + else: + _register_task = _unregister_task = _enter_task = _leave_task = None + + +class BaseCurrentLoopTests: + current_task = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def new_task(self, coro): + raise NotImplementedError + + def test_current_task_no_running_loop(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + def test_current_task_no_running_loop_implicit(self): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): + self.current_task() + + def test_current_task_with_implicit_loop(self): + async def coro(): + self.assertIs(self.current_task(loop=self.loop), task) + + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) + + task = self.new_task(coro()) + self.loop.run_until_complete(task) + self.assertIsNone(self.current_task(loop=self.loop)) + + +class PyCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + current_task = staticmethod(tasks._py_current_task) + + def new_task(self, coro): + return tasks._PyTask(coro, loop=self.loop) + + +@unittest.skipUnless(hasattr(tasks, '_CTask') and + hasattr(tasks, '_c_current_task'), + 'requires the C _asyncio module') +class CCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + if hasattr(tasks, '_c_current_task'): + current_task = staticmethod(tasks._c_current_task) + else: + current_task = None + + def new_task(self, coro): + return getattr(tasks, '_CTask')(coro, loop=self.loop) + + +class GenericTaskTests(test_utils.TestCase): + + def test_future_subclass(self): + self.assertIsSubclass(asyncio.Task, asyncio.Future) + + @support.cpython_only + def test_asyncio_module_compiled(self): + # Because of circular imports it's easy to make _asyncio + # module non-importable. This is a simple test that will + # fail on systems where C modules were successfully compiled + # (hence the test for _functools etc), but _asyncio somehow didn't. + try: + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 + except ImportError: + self.skipTest('C modules are not available') + else: + try: + import _asyncio # noqa: F401 + except ImportError: + self.fail('_asyncio module is missing') + + +class GatherTestsBase: + + def setUp(self): + super().setUp() + self.one_loop = self.new_test_loop() + self.other_loop = self.new_test_loop() + self.set_event_loop(self.one_loop, cleanup=False) + + def _run_loop(self, loop): + while loop._ready: + test_utils.run_briefly(loop) + + def _check_success(self, **kwargs): + a, b, c = [self.one_loop.create_future() for i in range(3)] + fut = self._gather(*self.wrap_futures(a, b, c), **kwargs) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + b.set_result(1) + a.set_result(2) + self._run_loop(self.one_loop) + self.assertEqual(cb.called, False) + self.assertFalse(fut.done()) + c.set_result(3) + self._run_loop(self.one_loop) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [2, 1, 3]) + + def test_success(self): + self._check_success() + self._check_success(return_exceptions=False) + + def test_result_exception_success(self): + self._check_success(return_exceptions=True) + + def test_one_exception(self): + a, b, c, d, e = [self.one_loop.create_future() for i in range(5)] + fut = self._gather(*self.wrap_futures(a, b, c, d, e)) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + a.set_result(1) + b.set_exception(exc) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertIs(fut.exception(), exc) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_return_exceptions(self): + a, b, c, d = [self.one_loop.create_future() for i in range(4)] + fut = self._gather(*self.wrap_futures(a, b, c, d), + return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + exc2 = RuntimeError() + b.set_result(1) + c.set_exception(exc) + a.set_result(3) + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_exception(exc2) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [3, 1, exc, exc2]) + + def test_env_var_debug(self): + code = '\n'.join(( + 'import asyncio.coroutines', + 'print(asyncio.coroutines._is_debug_mode())')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + # -X dev + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + + +class FutureGatherTests(GatherTestsBase, test_utils.TestCase): + + def wrap_futures(self, *futures): + return futures + + def _gather(self, *args, **kwargs): + return asyncio.gather(*args, **kwargs) + + def test_constructor_empty_sequence_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() + + def test_constructor_empty_sequence_use_running_loop(self): + async def gather(): + return asyncio.gather() + fut = self.one_loop.run_until_complete(gather()) + self.assertIsInstance(fut, asyncio.Future) + self.assertIs(fut._loop, self.one_loop) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + self.assertEqual(fut.result(), []) + + def test_constructor_empty_sequence_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.one_loop) + self.addCleanup(asyncio.set_event_loop, None) + fut = asyncio.gather() + self.assertIsInstance(fut, asyncio.Future) + self.assertIs(fut._loop, self.one_loop) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + self.assertEqual(fut.result(), []) + + def test_constructor_heterogenous_futures(self): + fut1 = self.one_loop.create_future() + fut2 = self.other_loop.create_future() + with self.assertRaises(ValueError): + asyncio.gather(fut1, fut2) + + def test_constructor_homogenous_futures(self): + children = [self.other_loop.create_future() for i in range(3)] + fut = asyncio.gather(*children) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + fut = asyncio.gather(*children) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + + def test_one_cancellation(self): + a, b, c, d, e = [self.one_loop.create_future() for i in range(5)] + fut = asyncio.gather(a, b, c, d, e) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + b.cancel() + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertFalse(fut.cancelled()) + self.assertIsInstance(fut.exception(), asyncio.CancelledError) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_result_exception_one_cancellation(self): + a, b, c, d, e, f = [self.one_loop.create_future() + for i in range(6)] + fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + zde = ZeroDivisionError() + b.set_exception(zde) + c.cancel() + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_result(3) + e.cancel() + rte = RuntimeError() + f.set_exception(rte) + res = self.one_loop.run_until_complete(fut) + self.assertIsInstance(res[2], asyncio.CancelledError) + self.assertIsInstance(res[4], asyncio.CancelledError) + res[2] = res[4] = None + self.assertEqual(res, [1, zde, None, 3, None, rte]) + cb.assert_called_once_with(fut) + + +class CoroutineGatherTests(GatherTestsBase, test_utils.TestCase): + + def wrap_futures(self, *futures): + coros = [] + for fut in futures: + async def coro(fut=fut): + return await fut + coros.append(coro()) + return coros + + def _gather(self, *args, **kwargs): + async def coro(): + return asyncio.gather(*args, **kwargs) + return self.one_loop.run_until_complete(coro()) + + def test_constructor_without_loop(self): + async def coro(): + return 'abc' + gen1 = coro() + self.addCleanup(gen1.close) + gen2 = coro() + self.addCleanup(gen2.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) + + def test_constructor_use_running_loop(self): + async def coro(): + return 'abc' + gen1 = coro() + gen2 = coro() + async def gather(): + return asyncio.gather(gen1, gen2) + fut = self.one_loop.run_until_complete(gather()) + self.assertIs(fut._loop, self.one_loop) + self.one_loop.run_until_complete(fut) + + def test_constructor_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + async def coro(): + return 'abc' + asyncio.set_event_loop(self.other_loop) + self.addCleanup(asyncio.set_event_loop, None) + gen1 = coro() + gen2 = coro() + fut = asyncio.gather(gen1, gen2) + self.assertIs(fut._loop, self.other_loop) + self.other_loop.run_until_complete(fut) + + def test_duplicate_coroutines(self): + async def coro(s): + return s + c = coro('abc') + fut = self._gather(c, c, coro('def'), c) + self._run_loop(self.one_loop) + self.assertEqual(fut.result(), ['abc', 'abc', 'def', 'abc']) + + def test_cancellation_broadcast(self): + # Cancelling outer() cancels all children. + proof = 0 + waiter = self.one_loop.create_future() + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + child1 = asyncio.ensure_future(inner(), loop=self.one_loop) + child2 = asyncio.ensure_future(inner(), loop=self.one_loop) + gatherer = None + + async def outer(): + nonlocal proof, gatherer + gatherer = asyncio.gather(child1, child2) + await gatherer + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + self.assertTrue(f.cancel()) + with self.assertRaises(asyncio.CancelledError): + self.one_loop.run_until_complete(f) + self.assertFalse(gatherer.cancel()) + self.assertTrue(waiter.cancelled()) + self.assertTrue(child1.cancelled()) + self.assertTrue(child2.cancelled()) + test_utils.run_briefly(self.one_loop) + self.assertEqual(proof, 0) + + def test_exception_marking(self): + # Test for the first line marked "Mark exception retrieved." + + async def inner(f): + await f + raise RuntimeError('should not be ignored') + + a = self.one_loop.create_future() + b = self.one_loop.create_future() + + async def outer(): + await asyncio.gather(inner(a), inner(b)) + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + a.set_result(None) + test_utils.run_briefly(self.one_loop) + b.set_result(None) + test_utils.run_briefly(self.one_loop) + self.assertIsInstance(f.exception(), RuntimeError) + + def test_issue46672(self): + with mock.patch( + 'asyncio.base_events.BaseEventLoop.call_exception_handler', + ): + async def coro(s): + return s + c = coro('abc') + + with self.assertRaises(TypeError): + self._gather(c, {}) + self._run_loop(self.one_loop) + # NameError should not happen: + self.one_loop.call_exception_handler.assert_not_called() + + +class RunCoroutineThreadsafeTests(test_utils.TestCase): + """Test case for asyncio.run_coroutine_threadsafe.""" + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) # Will cleanup properly + + async def add(self, a, b, fail=False, cancel=False): + """Wait 0.05 second and return a + b.""" + await asyncio.sleep(0.05) + if fail: + raise RuntimeError("Fail!") + if cancel: + asyncio.current_task(self.loop).cancel() + await asyncio.sleep(0) + return a + b + + def target(self, fail=False, cancel=False, timeout=None, + advance_coro=False): + """Run add coroutine in the event loop.""" + coro = self.add(1, 2, fail=fail, cancel=cancel) + future = asyncio.run_coroutine_threadsafe(coro, self.loop) + if advance_coro: + # this is for test_run_coroutine_threadsafe_task_factory_exception; + # otherwise it spills errors and breaks **other** unittests, since + # 'target' is interacting with threads. + + # With this call, `coro` will be advanced. + self.loop.call_soon_threadsafe(coro.send, None) + try: + return future.result(timeout) + finally: + future.done() or future.cancel() + + def test_run_coroutine_threadsafe(self): + """Test coroutine submission from a thread to an event loop.""" + future = self.loop.run_in_executor(None, self.target) + result = self.loop.run_until_complete(future) + self.assertEqual(result, 3) + + def test_run_coroutine_threadsafe_with_exception(self): + """Test coroutine submission from a thread to an event loop + when an exception is raised.""" + future = self.loop.run_in_executor(None, self.target, True) + with self.assertRaises(RuntimeError) as exc_context: + self.loop.run_until_complete(future) + self.assertIn("Fail!", exc_context.exception.args) + + def test_run_coroutine_threadsafe_with_timeout(self): + """Test coroutine submission from a thread to an event loop + when a timeout is raised.""" + callback = lambda: self.target(timeout=0) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(future) + test_utils.run_briefly(self.loop) + # Check that there's no pending task (add has been cancelled) + for task in asyncio.all_tasks(self.loop): + self.assertTrue(task.done()) + + def test_run_coroutine_threadsafe_task_cancelled(self): + """Test coroutine submission from a thread to an event loop + when the task is cancelled.""" + callback = lambda: self.target(cancel=True) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(future) + + def test_run_coroutine_threadsafe_task_factory_exception(self): + """Test coroutine submission from a thread to an event loop + when the task factory raise an exception.""" + + def task_factory(loop, coro): + raise NameError + + run = self.loop.run_in_executor( + None, lambda: self.target(advance_coro=True)) + + # Set exception handler + callback = test_utils.MockCallback() + self.loop.set_exception_handler(callback) + + # Set corrupted task factory + self.addCleanup(self.loop.set_task_factory, + self.loop.get_task_factory()) + self.loop.set_task_factory(task_factory) + + # Run event loop + with self.assertRaises(NameError) as exc_context: + self.loop.run_until_complete(run) + + # Check exceptions + self.assertEqual(len(callback.call_args_list), 1) + (loop, context), kwargs = callback.call_args + self.assertEqual(context['exception'], exc_context.exception) + + +class SleepTests(test_utils.TestCase): + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + def test_sleep_zero(self): + result = 0 + + def inc_result(num): + nonlocal result + result += num + + async def coro(): + self.loop.call_soon(inc_result, 1) + self.assertEqual(result, 0) + num = await asyncio.sleep(0, result=10) + self.assertEqual(result, 1) # inc'ed by call_soon + inc_result(num) # num should be 11 + + self.loop.run_until_complete(coro()) + self.assertEqual(result, 11) + + +class CompatibilityTests(test_utils.TestCase): + # Tests for checking a bridge between old-styled coroutines + # and async/await syntax + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py new file mode 100644 index 00000000000..8ad5f9b2c9e --- /dev/null +++ b/Lib/test/test_asyncio/test_threads.py @@ -0,0 +1,66 @@ +"""Tests for asyncio/threads.py""" + +import asyncio +import unittest + +from contextvars import ContextVar +from unittest import mock + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class ToThreadTests(unittest.IsolatedAsyncioTestCase): + async def test_to_thread(self): + result = await asyncio.to_thread(sum, [40, 2]) + self.assertEqual(result, 42) + + async def test_to_thread_exception(self): + def raise_runtime(): + raise RuntimeError("test") + + with self.assertRaisesRegex(RuntimeError, "test"): + await asyncio.to_thread(raise_runtime) + + async def test_to_thread_once(self): + func = mock.Mock() + + await asyncio.to_thread(func) + func.assert_called_once() + + async def test_to_thread_concurrent(self): + calls = [] + def func(): + calls.append(1) + + futs = [] + for _ in range(10): + fut = asyncio.to_thread(func) + futs.append(fut) + await asyncio.gather(*futs) + + self.assertEqual(sum(calls), 10) + + async def test_to_thread_args_kwargs(self): + # Unlike run_in_executor(), to_thread() should directly accept kwargs. + func = mock.Mock() + + await asyncio.to_thread(func, 'test', something=True) + + func.assert_called_once_with('test', something=True) + + async def test_to_thread_contextvars(self): + test_ctx = ContextVar('test_ctx') + + def get_ctx(): + return test_ctx.get() + + test_ctx.set('parrot') + result = await asyncio.to_thread(get_ctx) + + self.assertEqual(result, 'parrot') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py new file mode 100644 index 00000000000..f60722c48b7 --- /dev/null +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -0,0 +1,411 @@ +"""Tests for asyncio/timeouts.py""" + +import unittest +import time + +import asyncio + +from test.test_asyncio.utils import await_without_task + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + +class TimeoutTests(unittest.IsolatedAsyncioTestCase): + + async def test_timeout_basic(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(10) + self.assertTrue(cm.expired()) + + async def test_timeout_at_basic(self): + loop = asyncio.get_running_loop() + + with self.assertRaises(TimeoutError): + deadline = loop.time() + 0.01 + async with asyncio.timeout_at(deadline) as cm: + await asyncio.sleep(10) + self.assertTrue(cm.expired()) + self.assertEqual(deadline, cm.when()) + + async def test_nested_timeouts(self): + loop = asyncio.get_running_loop() + cancelled = False + with self.assertRaises(TimeoutError): + deadline = loop.time() + 0.01 + async with asyncio.timeout_at(deadline) as cm1: + # Only the topmost context manager should raise TimeoutError + try: + async with asyncio.timeout_at(deadline) as cm2: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled = True + raise + self.assertTrue(cancelled) + self.assertTrue(cm1.expired()) + self.assertTrue(cm2.expired()) + + async def test_waiter_cancelled(self): + cancelled = False + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled = True + raise + self.assertTrue(cancelled) + + async def test_timeout_not_called(self): + loop = asyncio.get_running_loop() + async with asyncio.timeout(10) as cm: + await asyncio.sleep(0.01) + t1 = loop.time() + + self.assertFalse(cm.expired()) + self.assertGreater(cm.when(), t1) + + async def test_timeout_disabled(self): + async with asyncio.timeout(None) as cm: + await asyncio.sleep(0.01) + + self.assertFalse(cm.expired()) + self.assertIsNone(cm.when()) + + async def test_timeout_at_disabled(self): + async with asyncio.timeout_at(None) as cm: + await asyncio.sleep(0.01) + + self.assertFalse(cm.expired()) + self.assertIsNone(cm.when()) + + async def test_timeout_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0) as cm: + await asyncio.sleep(10) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 <= cm.when() <= t1) + + async def test_timeout_zero_sleep_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0) as cm: + await asyncio.sleep(0) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 <= cm.when() <= t1) + + async def test_timeout_in_the_past_sleep_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(-11) as cm: + await asyncio.sleep(0) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 >= cm.when() <= t1) + + async def test_foreign_exception_passed(self): + with self.assertRaises(KeyError): + async with asyncio.timeout(0.01) as cm: + raise KeyError + self.assertFalse(cm.expired()) + + async def test_timeout_exception_context(self): + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.01): + try: + 1/0 + finally: + await asyncio.sleep(1) + e = cm.exception + # Expect TimeoutError caused by CancelledError raised during handling + # of ZeroDivisionError. + e2 = e.__cause__ + self.assertIsInstance(e2, asyncio.CancelledError) + self.assertIs(e.__context__, e2) + self.assertIsNone(e2.__cause__) + self.assertIsInstance(e2.__context__, ZeroDivisionError) + + async def test_foreign_exception_on_timeout(self): + async def crash(): + try: + await asyncio.sleep(1) + finally: + 1/0 + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + await crash() + e = cm.exception + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIs(e2.__context__, e3) + + async def test_foreign_exception_on_timeout_2(self): + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + try: + try: + raise ValueError + finally: + await asyncio.sleep(1) + finally: + try: + raise KeyError + finally: + 1/0 + e = cm.exception + # Expect ZeroDivisionError raised during handling of KeyError + # raised during handling of TimeoutError caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, KeyError) + self.assertIsNone(e2.__cause__) + e3 = e2.__context__ + self.assertIsInstance(e3, TimeoutError) + e4 = e3.__cause__ + self.assertIsInstance(e4, asyncio.CancelledError) + self.assertIsNone(e4.__cause__) + self.assertIsInstance(e4.__context__, ValueError) + self.assertIs(e3.__context__, e4) + + async def test_foreign_cancel_doesnt_timeout_if_not_expired(self): + with self.assertRaises(asyncio.CancelledError): + async with asyncio.timeout(10) as cm: + asyncio.current_task().cancel() + await asyncio.sleep(10) + self.assertFalse(cm.expired()) + + async def test_outer_task_is_not_cancelled(self): + async def outer() -> None: + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.001): + await asyncio.sleep(10) + + task = asyncio.create_task(outer()) + await task + self.assertFalse(task.cancelled()) + self.assertTrue(task.done()) + + async def test_nested_timeouts_concurrent(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.002): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.1): + # Pretend we crunch some numbers. + time.sleep(0.01) + await asyncio.sleep(1) + + async def test_nested_timeouts_loop_busy(self): + # After the inner timeout is an expensive operation which should + # be stopped by the outer timeout. + loop = asyncio.get_running_loop() + # Disable a message about long running task + loop.slow_callback_duration = 10 + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.1): # (1) + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): # (2) + # Pretend the loop is busy for a while. + time.sleep(0.1) + await asyncio.sleep(1) + # TimeoutError was caught by (2) + await asyncio.sleep(10) # This sleep should be interrupted by (1) + t1 = loop.time() + self.assertTrue(t0 <= t1 <= t0 + 1) + + async def test_reschedule(self): + loop = asyncio.get_running_loop() + fut = loop.create_future() + deadline1 = loop.time() + 10 + deadline2 = deadline1 + 20 + + async def f(): + async with asyncio.timeout_at(deadline1) as cm: + fut.set_result(cm) + await asyncio.sleep(50) + + task = asyncio.create_task(f()) + cm = await fut + + self.assertEqual(cm.when(), deadline1) + cm.reschedule(deadline2) + self.assertEqual(cm.when(), deadline2) + cm.reschedule(None) + self.assertIsNone(cm.when()) + + task.cancel() + + with self.assertRaises(asyncio.CancelledError): + await task + self.assertFalse(cm.expired()) + + async def test_repr_active(self): + async with asyncio.timeout(10) as cm: + self.assertRegex(repr(cm), r"") + + async def test_repr_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(10) + self.assertEqual(repr(cm), "") + + async def test_repr_finished(self): + async with asyncio.timeout(10) as cm: + await asyncio.sleep(0) + + self.assertEqual(repr(cm), "") + + async def test_repr_disabled(self): + async with asyncio.timeout(None) as cm: + self.assertEqual(repr(cm), r"") + + async def test_nested_timeout_in_finally(self): + with self.assertRaises(TimeoutError) as cm1: + async with asyncio.timeout(0.01): + try: + await asyncio.sleep(1) + finally: + with self.assertRaises(TimeoutError) as cm2: + async with asyncio.timeout(0.01): + await asyncio.sleep(10) + e1 = cm1.exception + # Expect TimeoutError caused by CancelledError. + e12 = e1.__cause__ + self.assertIsInstance(e12, asyncio.CancelledError) + self.assertIsNone(e12.__cause__) + self.assertIsNone(e12.__context__) + self.assertIs(e1.__context__, e12) + e2 = cm2.exception + # Expect TimeoutError caused by CancelledError raised during + # handling of other CancelledError (which is the same as in + # the above chain). + e22 = e2.__cause__ + self.assertIsInstance(e22, asyncio.CancelledError) + self.assertIsNone(e22.__cause__) + self.assertIs(e22.__context__, e12) + self.assertIs(e2.__context__, e22) + + async def test_timeout_after_cancellation(self): + try: + asyncio.current_task().cancel() + await asyncio.sleep(1) # work which will be cancelled + except asyncio.CancelledError: + pass + finally: + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.0): + await asyncio.sleep(1) # some cleanup + + async def test_cancel_in_timeout_after_cancellation(self): + try: + asyncio.current_task().cancel() + await asyncio.sleep(1) # work which will be cancelled + except asyncio.CancelledError: + pass + finally: + with self.assertRaises(asyncio.CancelledError): + async with asyncio.timeout(1.0): + asyncio.current_task().cancel() + await asyncio.sleep(2) # some cleanup + + async def test_timeout_already_entered(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_double_enter(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_finished(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "finished"): + cm.reschedule(0.02) + + async def test_timeout_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expired"): + cm.reschedule(0.02) + + async def test_timeout_expiring(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expiring"): + cm.reschedule(0.02) + + async def test_timeout_not_entered(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_without_task(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "task"): + await await_without_task(cm.__aenter__()) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_taskgroup(self): + async def task(): + try: + await asyncio.sleep(2) # Will be interrupted after 0.01 second + finally: + 1/0 # Crash in cleanup + + with self.assertRaises(ExceptionGroup) as cm: + async with asyncio.timeout(0.01): + async with asyncio.TaskGroup() as tg: + tg.create_task(task()) + try: + raise ValueError + finally: + await asyncio.sleep(1) + eg = cm.exception + # Expect ExceptionGroup raised during handling of TimeoutError caused + # by CancelledError raised during handling of ValueError. + self.assertIsNone(eg.__cause__) + e_1 = eg.__context__ + self.assertIsInstance(e_1, TimeoutError) + e_2 = e_1.__cause__ + self.assertIsInstance(e_2, asyncio.CancelledError) + self.assertIsNone(e_2.__cause__) + self.assertIsInstance(e_2.__context__, ValueError) + self.assertIs(e_1.__context__, e_2) + + self.assertEqual(len(eg.exceptions), 1, eg) + e1 = eg.exceptions[0] + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError (it is a different CancelledError). + self.assertIsInstance(e1, ZeroDivisionError) + self.assertIsNone(e1.__cause__) + e2 = e1.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIsNone(e3.__context__) + self.assertIsNone(e3.__cause__) + self.assertIs(e2.__context__, e3) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py new file mode 100644 index 00000000000..34e94830204 --- /dev/null +++ b/Lib/test/test_asyncio/test_tools.py @@ -0,0 +1,1706 @@ +import unittest + +from asyncio import tools + +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + + +# mock output of get_all_awaited_by function. +TEST_INPUTS_TREE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── __aexit__", + " └── _aexit", + " ├── (T) root1", + " │ └── bloch", + " │ └── blocho_caller", + " │ └── __aexit__", + " │ └── _aexit", + " │ ├── (T) child1_1", + " │ │ └── awaiter /path/to/app.py:110", + " │ │ └── awaiter2 /path/to/app.py:120", + " │ │ └── awaiter3 /path/to/app.py:130", + " │ │ └── (T) timer", + " │ └── (T) child2_1", + " │ └── awaiterB /path/to/app.py:170", + " │ └── awaiterB2 /path/to/app.py:180", + " │ └── awaiterB3 /path/to/app.py:190", + " │ └── (T) timer", + " └── (T) root2", + " └── bloch", + " └── blocho_caller", + " └── __aexit__", + " └── _aexit", + " ├── (T) child1_2", + " │ └── awaiter /path/to/app.py:110", + " │ └── awaiter2 /path/to/app.py:120", + " │ └── awaiter3 /path/to/app.py:130", + " │ └── (T) timer", + " └── (T) child2_2", + " └── awaiterB /path/to/app.py:170", + " └── awaiterB2 /path/to/app.py:180", + " └── awaiterB3 /path/to/app.py:190", + " └── (T) timer", + ] + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", + ], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + ["└── (T) Task-5"], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], +] + +TEST_INPUTS_CYCLES_TREE = [ + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4]]), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ], +] + +TEST_INPUTS_TABLE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_1", + "0x4", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_2", + "0x5", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_1", + "0x6", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_2", + "0x7", + ], + [ + 1, + "0x8", + "root1", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x9", + "root2", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x4", + "child1_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x6", + "child2_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x7", + "child1_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + [ + 1, + "0x5", + "child2_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], + ] + ), + ], + # CASES WITH CYCLES + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], + ] + ), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "A", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_c", + "C", + "0x5", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_a", + "A", + "0x3", + ], + [ + 1, + "0x5", + "C", + "", + "nested -> nested", + "Task-2", + "0x6", + ], + [ + 1, + "0x6", + "Task-2", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + ] + ), + ], +] + + +class TestAsyncioToolsTree(unittest.TestCase): + def test_asyncio_utils(self): + for input_, tree in TEST_INPUTS_TREE: + with self.subTest(input_): + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) + + def test_asyncio_utils_cycles(self): + for input_, cycles in TEST_INPUTS_CYCLES_TREE: + with self.subTest(input_): + try: + tools.build_async_tree(input_) + except tools.CycleFoundException as e: + self.assertEqual(e.cycles, cycles) + + +class TestAsyncioToolsTable(unittest.TestCase): + def test_asyncio_utils(self): + for input_, table in TEST_INPUTS_TABLE: + with self.subTest(input_): + result = tools.build_task_table(input_) + self.assertEqual(result, table) + + +class TestAsyncioToolsBasic(unittest.TestCase): + def test_empty_input_tree(self): + """Test build_async_tree with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_empty_input_table(self): + """Test build_task_table with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_only_independent_tasks_tree(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [["└── (T) taskA"], ["└── (T) taskB"]] + result = tools.build_async_tree(input_) + self.assertEqual(sorted(result), sorted(expected)) + + def test_only_independent_tasks_table(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + self.assertEqual( + tools.build_task_table(input_), + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] + ) + + def test_single_task_tree(self): + """Test build_async_tree with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_single_task_table(self): + """Test build_task_table with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_cycle_detection(self): + """Test build_async_tree raises CycleFoundException for cyclic input.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as context: + tools.build_async_tree(result) + self.assertEqual(context.exception.cycles, [[3, 2, 3]]) + + def test_complex_tree(self): + """Test build_async_tree with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) Task-2", + " └── main", + " └── (T) Task-3", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_complex_table(self): + """Test build_task_table with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] + ] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_deep_coroutine_chain(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [ + [ + "└── (T) root", + " └── c5", + " └── c4", + " └── c3", + " └── c2", + " └── c1", + " └── (T) leaf", + ] + ] + result = tools.build_async_tree(input_) + self.assertEqual(result, expected) + + def test_multiple_cycles_same_node(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + cycles = ctx.exception.cycles + self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) + + def test_table_output_format(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + table = tools.build_task_table(input_) + for row in table: + self.assertEqual(len(row), 7) + self.assertIsInstance(row[0], int) # thread ID + self.assertTrue( + isinstance(row[1], str) and row[1].startswith("0x") + ) # hex task ID + self.assertIsInstance(row[2], str) # task name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name + self.assertTrue( + isinstance(row[6], str) and row[6].startswith("0x") + ) # hex awaiter ID + + +class TestAsyncioToolsEdgeCases(unittest.TestCase): + + def test_task_awaits_self(self): + """A task directly awaits itself - should raise a cycle.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + self.assertIn([1, 1], ctx.exception.cycles) + + def test_task_with_missing_awaiter_id(self): + """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] + table = tools.build_task_table(input_) + self.assertEqual(len(table), 1) + self.assertEqual(table[0][5], "Unknown") + + def test_duplicate_coroutine_frames(self): + """Same coroutine frame repeated under a parent - should deduplicate.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_) + # Both children should be under the same coroutine node + flat = "\n".join(tree[0]) + self.assertIn("frameA", flat) + self.assertIn("Task-2", flat) + self.assertIn("Task-1", flat) + + flat = "\n".join(tree[1]) + self.assertIn("frameA", flat) + self.assertIn("Task-3", flat) + self.assertIn("Task-1", flat) + + def test_task_with_no_name(self): + """Task with no name in id2name - should still render with fallback.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + # If name is None, fallback to string should not crash + tree = tools.build_async_tree(input_) + self.assertIn("(T) None", "\n".join(tree[0])) + + def test_tree_rendering_with_custom_emojis(self): + """Pass custom emojis to the tree renderer.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + flat = "\n".join(tree[0]) + self.assertIn("🧵 MainTask", flat) + self.assertIn("🔁 f1", flat) + self.assertIn("🔁 f2", flat) + self.assertIn("🧵 SubTask", flat) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py new file mode 100644 index 00000000000..dbb572e2e15 --- /dev/null +++ b/Lib/test/test_asyncio/test_transports.py @@ -0,0 +1,103 @@ +"""Tests for transports.py.""" + +import unittest +from unittest import mock + +import asyncio +from asyncio import transports + + +def tearDownModule(): + # not needed for the test file but added for uniformness with all other + # asyncio test files for the sake of unified cleanup + asyncio.events._set_event_loop_policy(None) + + +class TransportTests(unittest.TestCase): + + def test_ctor_extra_is_none(self): + transport = asyncio.Transport() + self.assertEqual(transport._extra, {}) + + def test_get_extra_info(self): + transport = asyncio.Transport({'extra': 'info'}) + self.assertEqual('info', transport.get_extra_info('extra')) + self.assertIsNone(transport.get_extra_info('unknown')) + + default = object() + self.assertIs(default, transport.get_extra_info('unknown', default)) + + def test_writelines(self): + writer = mock.Mock() + + class MyTransport(asyncio.Transport): + def write(self, data): + writer(data) + + transport = MyTransport() + + transport.writelines([b'line1', + bytearray(b'line2'), + memoryview(b'line3')]) + self.assertEqual(1, writer.call_count) + writer.assert_called_with(b'line1line2line3') + + def test_not_implemented(self): + transport = asyncio.Transport() + + self.assertRaises(NotImplementedError, + transport.set_write_buffer_limits) + self.assertRaises(NotImplementedError, transport.get_write_buffer_size) + self.assertRaises(NotImplementedError, transport.write, 'data') + self.assertRaises(NotImplementedError, transport.write_eof) + self.assertRaises(NotImplementedError, transport.can_write_eof) + self.assertRaises(NotImplementedError, transport.pause_reading) + self.assertRaises(NotImplementedError, transport.resume_reading) + self.assertRaises(NotImplementedError, transport.is_reading) + self.assertRaises(NotImplementedError, transport.close) + self.assertRaises(NotImplementedError, transport.abort) + + def test_dgram_not_implemented(self): + transport = asyncio.DatagramTransport() + + self.assertRaises(NotImplementedError, transport.sendto, 'data') + self.assertRaises(NotImplementedError, transport.abort) + + def test_subprocess_transport_not_implemented(self): + transport = asyncio.SubprocessTransport() + + self.assertRaises(NotImplementedError, transport.get_pid) + self.assertRaises(NotImplementedError, transport.get_returncode) + self.assertRaises(NotImplementedError, transport.get_pipe_transport, 1) + self.assertRaises(NotImplementedError, transport.send_signal, 1) + self.assertRaises(NotImplementedError, transport.terminate) + self.assertRaises(NotImplementedError, transport.kill) + + def test_flowcontrol_mixin_set_write_limits(self): + + class MyTransport(transports._FlowControlMixin, + transports.Transport): + + def get_write_buffer_size(self): + return 512 + + loop = mock.Mock() + transport = MyTransport(loop=loop) + transport._protocol = mock.Mock() + + self.assertFalse(transport._protocol_paused) + + with self.assertRaisesRegex(ValueError, 'high.*must be >= low'): + transport.set_write_buffer_limits(high=0, low=1) + + transport.set_write_buffer_limits(high=1024, low=128) + self.assertFalse(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 1024)) + + transport.set_write_buffer_limits(high=256, low=128) + self.assertTrue(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 256)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py new file mode 100644 index 00000000000..0faf32f79ea --- /dev/null +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -0,0 +1,1334 @@ +"""Tests for unix_events.py.""" + +import contextlib +import errno +import io +import multiprocessing +from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests +import os +import signal +import socket +import stat +import sys +import time +import unittest +from unittest import mock + +from test import support +from test.support import os_helper +from test.support import socket_helper +from test.support import wait_process +from test.support import hashlib_helper + +if sys.platform == 'win32': + raise unittest.SkipTest('UNIX only') + + +import asyncio +from asyncio import unix_events +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +MOCK_ANY = mock.ANY + + +def EXITCODE(exitcode): + return 32768 + exitcode + + +def SIGNAL(signum): + if not 1 <= signum <= 68: + raise AssertionError(f'invalid signum {signum}') + return 32768 - signum + + +def close_pipe_transport(transport): + # Don't call transport.close() because the event loop and the selector + # are mocked + if transport._pipe is None: + return + transport._pipe.close() + transport._pipe = None + + +@unittest.skipUnless(signal, 'Signals are not supported') +class SelectorEventLoopSignalTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + def test_check_signal(self): + self.assertRaises( + TypeError, self.loop._check_signal, '1') + self.assertRaises( + ValueError, self.loop._check_signal, signal.NSIG + 1) + + def test_handle_signal_no_handler(self): + self.loop._handle_signal(signal.NSIG + 1) + + def test_handle_signal_cancelled_handler(self): + h = asyncio.Handle(mock.Mock(), (), + loop=mock.Mock()) + h.cancel() + self.loop._signal_handlers[signal.NSIG + 1] = h + self.loop.remove_signal_handler = mock.Mock() + self.loop._handle_signal(signal.NSIG + 1) + self.loop.remove_signal_handler.assert_called_with(signal.NSIG + 1) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_setup_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + m_signal.set_wakeup_fd.side_effect = ValueError + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_coroutine_error(self, m_signal): + m_signal.NSIG = signal.NSIG + + async def simple_coroutine(): + pass + + # callback must not be a coroutine function + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + self.assertRaisesRegex( + TypeError, 'coroutines cannot be used with add_signal_handler', + self.loop.add_signal_handler, + signal.SIGINT, func) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + cb = lambda: True + self.loop.add_signal_handler(signal.SIGHUP, cb) + h = self.loop._signal_handlers.get(signal.SIGHUP) + self.assertIsInstance(h, asyncio.Handle) + self.assertEqual(h._callback, cb) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_install_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + def set_wakeup_fd(fd): + if fd == -1: + raise ValueError() + m_signal.set_wakeup_fd = set_wakeup_fd + + class Err(OSError): + errno = errno.EFAULT + m_signal.signal.side_effect = Err + + self.assertRaises( + Err, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error2(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.loop._signal_handlers[signal.SIGHUP] = lambda: True + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(1, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error3(self, m_logging, m_signal): + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(2, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGHUP)) + self.assertTrue(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGHUP, m_signal.SIG_DFL), m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.SIGINT = signal.SIGINT + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGINT, lambda: True) + self.loop._signal_handlers[signal.SIGHUP] = object() + m_signal.set_wakeup_fd.reset_mock() + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGINT)) + self.assertFalse(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGINT, m_signal.default_int_handler), + m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_remove_signal_handler_cleanup_error(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.set_wakeup_fd.side_effect = ValueError + + self.loop.remove_signal_handler(signal.SIGHUP) + self.assertTrue(m_logging.info) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.signal.side_effect = OSError + + self.assertRaises( + OSError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.assertRaises( + RuntimeError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_close(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + self.loop.add_signal_handler(signal.SIGCHLD, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 2) + + m_signal.set_wakeup_fd.reset_mock() + + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + m_signal.set_wakeup_fd.assert_called_once_with(-1) + + @mock.patch('asyncio.unix_events.sys') + @mock.patch('asyncio.unix_events.signal') + def test_close_on_finalizing(self, m_signal, m_sys): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 1) + m_sys.is_finalizing.return_value = True + m_signal.signal.reset_mock() + + with self.assertWarnsRegex(ResourceWarning, + "skipping signal handlers removal"): + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + self.assertFalse(m_signal.signal.called) + + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), + 'UNIX Sockets are not supported') +class SelectorEventLoopUnixSocketTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_existing_path_sock(self): + with test_utils.unix_socket_path() as path: + sock = socket.socket(socket.AF_UNIX) + sock.bind(path) + sock.listen(1) + sock.close() + + coro = self.loop.create_unix_server(lambda: None, path) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_pathlike(self): + with test_utils.unix_socket_path() as path: + path = os_helper.FakePath(path) + srv_coro = self.loop.create_unix_server(lambda: None, path) + srv = self.loop.run_until_complete(srv_coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + def test_create_unix_connection_pathlike(self): + with test_utils.unix_socket_path() as path: + path = os_helper.FakePath(path) + coro = self.loop.create_unix_connection(lambda: None, path) + with self.assertRaises(FileNotFoundError): + # If path-like object weren't supported, the exception would be + # different. + self.loop.run_until_complete(coro) + + def test_create_unix_server_existing_path_nonsock(self): + path = test_utils.gen_unix_socket_path() + self.addCleanup(os_helper.unlink, path) + # create the file + open(path, "wb").close() + + coro = self.loop.create_unix_server(lambda: None, path) + with self.assertRaisesRegex(OSError, + 'Address.*is already in use'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_ssl_bool(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl=True) + with self.assertRaisesRegex(TypeError, + 'ssl argument must be an SSLContext'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_nopath_nosock(self): + coro = self.loop.create_unix_server(lambda: None, path=None) + with self.assertRaisesRegex(ValueError, + 'path was not specified, and no sock'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_path_dgram(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'no socket.SOCK_NONBLOCK (linux only)') + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_path_stream_bittype(self): + fn = test_utils.gen_unix_socket_path() + self.addCleanup(os_helper.unlink, fn) + + sock = socket.socket(socket.AF_UNIX, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) + with sock: + sock.bind(fn) + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + def test_create_unix_server_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_connection(lambda: None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + @mock.patch('asyncio.unix_events.socket') + def test_create_unix_server_bind_error(self, m_socket): + # Ensure that the socket is closed on any bind error + sock = mock.Mock() + m_socket.socket.return_value = sock + + sock.bind.side_effect = OSError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + sock.bind.side_effect = MemoryError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(MemoryError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + def test_create_unix_connection_path_sock(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, sock=object()) + with self.assertRaisesRegex(ValueError, 'path and sock can not be'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nopath_nosock(self): + coro = self.loop.create_unix_connection( + lambda: None, None) + with self.assertRaisesRegex(ValueError, + 'no path and sock were specified'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nossl_serverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, server_hostname='spam') + with self.assertRaisesRegex(ValueError, + 'server_hostname is only meaningful'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_ssl_noserverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, ssl=True) + + with self.assertRaisesRegex( + ValueError, 'you have to pass server_hostname when using ssl'): + + self.loop.run_until_complete(coro) + + def test_create_unix_connection_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_connection(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + +@unittest.skipUnless(hasattr(os, 'sendfile'), + 'sendfile is not supported') +class SelectorEventLoopUnixSockSendfileTests(test_utils.TestCase): + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + self._ready = loop.create_future() + + def connection_made(self, transport): + self.started = True + self.transport = transport + self._ready.set_result(None) + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, cleanup=True): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) + if cleanup: + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.run_loop(self.loop.sock_connect(sock, (socket_helper.HOST, port))) + self.run_loop(proto._ready) + + def cleanup(): + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_not_available(self): + sock, proto = self.prepare() + with mock.patch('asyncio.unix_events.os', spec=[]): + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "os[.]sendfile[(][)] is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_a_file(self): + sock, proto = self.prepare() + f = object() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_iobuffer(self): + sock, proto = self.prepare() + f = io.BytesIO() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_regular_file(self): + sock, proto = self.prepare() + f = mock.Mock() + f.fileno.return_value = -1 + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_cancel1(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + with contextlib.suppress(asyncio.CancelledError): + self.run_loop(fut) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_cancel2(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_blocking_error(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = mock.Mock() + fut.cancelled.return_value = False + with mock.patch('os.sendfile', side_effect=BlockingIOError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + key = self.loop._selector.get_key(sock) + self.assertIsNotNone(key) + fut.add_done_callback.assert_called_once_with(mock.ANY) + + def test_sock_sendfile_os_error_first_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + with mock.patch('os.sendfile', side_effect=OSError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIsInstance(exc, asyncio.SendfileNotAvailableError) + self.assertEqual(0, self.file.tell()) + + def test_sock_sendfile_os_error_next_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = OSError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + + def test_sock_sendfile_exception(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = asyncio.SendfileNotAvailableError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + + +class UnixReadPipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('os.set_blocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFIFO + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def read_pipe_transport(self, waiter=None): + transport = unix_events._UnixReadPipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.read_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertIsNone(waiter.result()) + + @mock.patch('os.read') + def test__read_ready(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'data' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.protocol.data_received.assert_called_with(b'data') + + @mock.patch('os.read') + def test__read_ready_eof(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.eof_received.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.read') + def test__read_ready_blocked(self, m_read): + tr = self.read_pipe_transport() + m_read.side_effect = BlockingIOError + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.data_received.called) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.read') + def test__read_ready_error(self, m_read, m_logexc): + tr = self.read_pipe_transport() + err = OSError() + m_read.side_effect = err + tr._close = mock.Mock() + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + tr._close.assert_called_with(err) + m_logexc.assert_called_with( + test_utils.MockPattern( + 'Fatal read error on pipe transport' + '\nprotocol:.*\ntransport:.*'), + exc_info=(OSError, MOCK_ANY, MOCK_ANY)) + + @mock.patch('os.read') + def test_pause_reading(self, m_read): + tr = self.read_pipe_transport() + m = mock.Mock() + self.loop.add_reader(5, m) + tr.pause_reading() + self.assertFalse(self.loop.readers) + + @mock.patch('os.read') + def test_resume_reading(self, m_read): + tr = self.read_pipe_transport() + tr.pause_reading() + tr.resume_reading() + self.loop.assert_reader(5, tr._read_ready) + + @mock.patch('os.read') + def test_close(self, m_read): + tr = self.read_pipe_transport() + tr._close = mock.Mock() + tr.close() + tr._close.assert_called_with(None) + + @mock.patch('os.read') + def test_close_already_closing(self, m_read): + tr = self.read_pipe_transport() + tr._closing = True + tr._close = mock.Mock() + tr.close() + self.assertFalse(tr._close.called) + + @mock.patch('os.read') + def test__close(self, m_read): + tr = self.read_pipe_transport() + err = object() + tr._close(err) + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + def test__call_connection_lost(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test_pause_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.pause_reading() + + def test_pause_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + tr.pause_reading() + # the second call should do nothing + tr.pause_reading() + + def test_resume_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.resume_reading() + + def test_resume_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + # the pipe is not paused + # resuming should do nothing + tr.resume_reading() + + +class UnixWritePipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.BaseProtocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('os.set_blocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFSOCK + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def write_pipe_transport(self, waiter=None): + transport = unix_events._UnixWritePipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.write_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertEqual(None, waiter.result()) + + def test_can_write_eof(self): + tr = self.write_pipe_transport() + self.assertTrue(tr.can_write_eof()) + + @mock.patch('os.write') + def test_write(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 4 + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test_write_no_data(self, m_write): + tr = self.write_pipe_transport() + tr.write(b'') + self.assertFalse(m_write.called) + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(b''), tr._buffer) + + @mock.patch('os.write') + def test_write_partial(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 2 + tr.write(b'data') + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'ta'), tr._buffer) + + @mock.patch('os.write') + def test_write_buffer(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'previous') + tr.write(b'data') + self.assertFalse(m_write.called) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'previousdata'), tr._buffer) + + @mock.patch('os.write') + def test_write_again(self, m_write): + tr = self.write_pipe_transport() + m_write.side_effect = BlockingIOError() + tr.write(b'data') + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.unix_events.logger') + @mock.patch('os.write') + def test_write_err(self, m_write, m_log): + tr = self.write_pipe_transport() + err = OSError() + m_write.side_effect = err + tr._fatal_error = mock.Mock() + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + self.assertEqual(1, tr._conn_lost) + + tr.write(b'data') + self.assertEqual(2, tr._conn_lost) + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + # This is a bit overspecified. :-( + m_log.warning.assert_called_with( + 'pipe closed by peer or os.write(pipe, data) raised exception.') + tr.close() + + @mock.patch('os.write') + def test_write_close(self, m_write): + tr = self.write_pipe_transport() + tr._read_ready() # pipe was closed by peer + + tr.write(b'data') + self.assertEqual(tr._conn_lost, 1) + tr.write(b'data') + self.assertEqual(tr._conn_lost, 2) + + def test__read_ready(self): + tr = self.write_pipe_transport() + tr._read_ready() + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.write') + def test__write_ready(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_partial(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 3 + tr._write_ready() + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'a'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_again(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = BlockingIOError() + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_empty(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 0 + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.write') + def test__write_ready_err(self, m_write, m_logexc): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = err = OSError() + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.assertTrue(tr.is_closing()) + m_logexc.assert_not_called() + self.assertEqual(1, tr._conn_lost) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + @mock.patch('os.write') + def test__write_ready_closing(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._closing = True + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.protocol.connection_lost.assert_called_with(None) + self.pipe.close.assert_called_with() + + @mock.patch('os.write') + def test_abort(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + self.loop.add_reader(5, tr._read_ready) + tr._buffer = [b'da', b'ta'] + tr.abort() + self.assertFalse(m_write.called) + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertEqual([], tr._buffer) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test__call_connection_lost(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test_close(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr.close() + tr.write_eof.assert_called_with() + + # closing the transport twice must not fail + tr.close() + + def test_close_closing(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr._closing = True + tr.close() + self.assertFalse(tr.write_eof.called) + + def test_write_eof(self): + tr = self.write_pipe_transport() + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_write_eof_pending(self): + tr = self.write_pipe_transport() + tr._buffer = [b'data'] + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.protocol.connection_lost.called) + + +class TestFunctional(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + asyncio.set_event_loop(None) + + def test_add_reader_invalid_argument(self): + def assert_raises(): + return self.assertRaisesRegex(ValueError, r'Invalid file object') + + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(object(), cb) + with assert_raises(): + self.loop.add_writer(object(), cb) + + with assert_raises(): + self.loop.remove_reader(object()) + with assert_raises(): + self.loop.remove_writer(object()) + + def test_add_reader_or_writer_transport_fd(self): + def assert_raises(): + return self.assertRaisesRegex( + RuntimeError, + r'File descriptor .* is used by transport') + + async def runner(): + tr, pr = await self.loop.create_connection( + lambda: asyncio.Protocol(), sock=rsock) + + try: + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(rsock, cb) + with assert_raises(): + self.loop.add_reader(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_reader(rsock) + with assert_raises(): + self.loop.remove_reader(rsock.fileno()) + + with assert_raises(): + self.loop.add_writer(rsock, cb) + with assert_raises(): + self.loop.add_writer(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_writer(rsock) + with assert_raises(): + self.loop.remove_writer(rsock.fileno()) + + finally: + tr.close() + + rsock, wsock = socket.socketpair() + try: + self.loop.run_until_complete(runner()) + finally: + rsock.close() + wsock.close() + + +# TODO: RUSTPYTHON, fork() segfaults due to stale parking_lot global state +@unittest.skip("TODO: RUSTPYTHON") +@support.requires_fork() +class TestFork(unittest.TestCase): + + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) + + def test_fork_not_share_event_loop(self): + # The forked process should not share the event loop with the parent + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + loop = asyncio.get_event_loop() + os.write(w, b'LOOP:' + str(id(loop)).encode()) + except RuntimeError: + os.write(w, b'NO LOOP') + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO LOOP') + wait_process(pid, exitcode=0) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_signal_handling(self): + self.addCleanup(multiprocessing_cleanup_tests) + + # Sending signal to the forked process should not affect the parent + # process + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + child_started = manager.Event() + child_handled = manager.Event() + parent_handled = manager.Event() + + def child_main(): + def on_sigterm(*args): + child_handled.set() + sys.exit() + + signal.signal(signal.SIGTERM, on_sigterm) + child_started.set() + while True: + time.sleep(1) + + async def main(): + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, lambda *args: parent_handled.set()) + + process = ctx.Process(target=child_main) + process.start() + child_started.wait() + os.kill(process.pid, signal.SIGTERM) + process.join(timeout=support.SHORT_TIMEOUT) + + async def func(): + await asyncio.sleep(0.1) + return 42 + + # Test parent's loop is still functional + self.assertEqual(await asyncio.create_task(func()), 42) + + asyncio.run(main()) + + child_handled.wait(timeout=support.SHORT_TIMEOUT) + self.assertFalse(parent_handled.is_set()) + self.assertTrue(child_handled.is_set()) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_asyncio_run(self): + self.addCleanup(multiprocessing_cleanup_tests) + + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 0) + + async def child_main(): + await asyncio.sleep(0.1) + result.value = 42 + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 42) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_asyncio_subprocess(self): + self.addCleanup(multiprocessing_cleanup_tests) + + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 1) + + async def child_main(): + proc = await asyncio.create_subprocess_exec(sys.executable, '-c', 'pass') + result.value = await proc.wait() + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 0) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py new file mode 100644 index 00000000000..dedc6bf69d7 --- /dev/null +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -0,0 +1,353 @@ +import asyncio +import unittest +import time +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +# The following value can be used as a very small timeout: +# it passes check "timeout > 0", but has almost +# no effect on the test performance +_EPSILON = 0.0001 + + +class SlowTask: + """ Task will run for this defined time, ignoring cancel requests """ + TASK_TIMEOUT = 0.2 + + def __init__(self): + self.exited = False + + async def run(self): + exitat = time.monotonic() + self.TASK_TIMEOUT + + while True: + tosleep = exitat - time.monotonic() + if tosleep <= 0: + break + + try: + await asyncio.sleep(tosleep) + except asyncio.CancelledError: + pass + + self.exited = True + + +class AsyncioWaitForTest(unittest.IsolatedAsyncioTestCase): + + async def test_asyncio_wait_for_cancelled(self): + t = SlowTask() + + waitfortask = asyncio.create_task( + asyncio.wait_for(t.run(), t.TASK_TIMEOUT * 2)) + await asyncio.sleep(0) + waitfortask.cancel() + await asyncio.wait({waitfortask}) + + self.assertTrue(t.exited) + + async def test_asyncio_wait_for_timeout(self): + t = SlowTask() + + try: + await asyncio.wait_for(t.run(), t.TASK_TIMEOUT / 2) + except asyncio.TimeoutError: + pass + + self.assertTrue(t.exited) + + async def test_wait_for_timeout_less_then_0_or_0_future_done(self): + loop = asyncio.get_running_loop() + + fut = loop.create_future() + fut.set_result('done') + + ret = await asyncio.wait_for(fut, 0) + + self.assertEqual(ret, 'done') + self.assertTrue(fut.done()) + + async def test_wait_for_timeout_less_then_0_or_0_coroutine_do_not_started(self): + foo_started = False + + async def foo(): + nonlocal foo_started + foo_started = True + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(foo(), 0) + + self.assertEqual(foo_started, False) + + async def test_wait_for_timeout_less_then_0_or_0(self): + loop = asyncio.get_running_loop() + + for timeout in [0, -1]: + with self.subTest(timeout=timeout): + foo_running = None + started = loop.create_future() + + async def foo(): + nonlocal foo_running + foo_running = True + started.set_result(None) + try: + await asyncio.sleep(10) + finally: + foo_running = False + return 'done' + + fut = asyncio.create_task(foo()) + await started + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(fut, timeout) + + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertEqual(foo_running, False) + + async def test_wait_for(self): + foo_running = None + + async def foo(): + nonlocal foo_running + foo_running = True + try: + await asyncio.sleep(support.LONG_TIMEOUT) + finally: + foo_running = False + return 'done' + + fut = asyncio.create_task(foo()) + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(fut, 0.1) + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertEqual(foo_running, False) + + async def test_wait_for_blocking(self): + async def coro(): + return 'done' + + res = await asyncio.wait_for(coro(), timeout=None) + self.assertEqual(res, 'done') + + async def test_wait_for_race_condition(self): + loop = asyncio.get_running_loop() + + fut = loop.create_future() + task = asyncio.wait_for(fut, timeout=0.2) + loop.call_soon(fut.set_result, "ok") + res = await task + self.assertEqual(res, "ok") + + async def test_wait_for_cancellation_race_condition(self): + async def inner(): + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(1) + return 1 + + result = await asyncio.wait_for(inner(), timeout=.01) + self.assertEqual(result, 1) + + async def test_wait_for_waits_for_task_cancellation(self): + task_done = False + + async def inner(): + nonlocal task_done + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(_EPSILON) + raise + finally: + task_done = True + + inner_task = asyncio.create_task(inner()) + + with self.assertRaises(asyncio.TimeoutError) as cm: + await asyncio.wait_for(inner_task, timeout=_EPSILON) + + self.assertTrue(task_done) + chained = cm.exception.__context__ + self.assertEqual(type(chained), asyncio.CancelledError) + + async def test_wait_for_waits_for_task_cancellation_w_timeout_0(self): + task_done = False + + async def foo(): + async def inner(): + nonlocal task_done + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(_EPSILON) + raise + finally: + task_done = True + + inner_task = asyncio.create_task(inner()) + await asyncio.sleep(_EPSILON) + await asyncio.wait_for(inner_task, timeout=0) + + with self.assertRaises(asyncio.TimeoutError) as cm: + await foo() + + self.assertTrue(task_done) + chained = cm.exception.__context__ + self.assertEqual(type(chained), asyncio.CancelledError) + + async def test_wait_for_reraises_exception_during_cancellation(self): + class FooException(Exception): + pass + + async def foo(): + async def inner(): + try: + await asyncio.sleep(0.2) + finally: + raise FooException + + inner_task = asyncio.create_task(inner()) + + await asyncio.wait_for(inner_task, timeout=_EPSILON) + + with self.assertRaises(FooException): + await foo() + + async def _test_cancel_wait_for(self, timeout): + loop = asyncio.get_running_loop() + + async def blocking_coroutine(): + fut = loop.create_future() + # Block: fut result is never set + await fut + + task = asyncio.create_task(blocking_coroutine()) + + wait = asyncio.create_task(asyncio.wait_for(task, timeout)) + loop.call_soon(wait.cancel) + + with self.assertRaises(asyncio.CancelledError): + await wait + + # Python issue #23219: cancelling the wait must also cancel the task + self.assertTrue(task.cancelled()) + + async def test_cancel_blocking_wait_for(self): + await self._test_cancel_wait_for(None) + + async def test_cancel_wait_for(self): + await self._test_cancel_wait_for(60.0) + + async def test_wait_for_cancel_suppressed(self): + # GH-86296: Suppressing CancelledError is discouraged + # but if a task suppresses CancelledError and returns a value, + # `wait_for` should return the value instead of raising CancelledError. + # This is the same behavior as `asyncio.timeout`. + + async def return_42(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + return 42 + + res = await asyncio.wait_for(return_42(), timeout=0.1) + self.assertEqual(res, 42) + + + async def test_wait_for_issue86296(self): + # GH-86296: The task should get cancelled and not run to completion. + # inner completes in one cycle of the event loop so it + # completes before the task is cancelled. + + async def inner(): + return 'done' + + inner_task = asyncio.create_task(inner()) + reached_end = False + + async def wait_for_coro(): + await asyncio.wait_for(inner_task, timeout=100) + await asyncio.sleep(1) + nonlocal reached_end + reached_end = True + + task = asyncio.create_task(wait_for_coro()) + self.assertFalse(task.done()) + # Run the task + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertTrue(inner_task.done()) + self.assertEqual(await inner_task, 'done') + self.assertFalse(reached_end) + + +class WaitForShieldTests(unittest.IsolatedAsyncioTestCase): + + async def test_zero_timeout(self): + # `asyncio.shield` creates a new task which wraps the passed in + # awaitable and shields it from cancellation so with timeout=0 + # the task returned by `asyncio.shield` aka shielded_task gets + # cancelled immediately and the task wrapped by it is scheduled + # to run. + + async def coro(): + await asyncio.sleep(0.01) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + shielded_task = asyncio.shield(task) + await asyncio.wait_for(shielded_task, timeout=0) + + # Task is running in background + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertTrue(shielded_task.cancelled()) + + # Wait for the task to complete + await asyncio.sleep(0.1) + self.assertTrue(task.done()) + + + async def test_none_timeout(self): + # With timeout=None the timeout is disabled so it + # runs till completion. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + await asyncio.wait_for(asyncio.shield(task), timeout=None) + + self.assertTrue(task.done()) + self.assertEqual(await task, "done") + + async def test_shielded_timeout(self): + # shield prevents the task from being cancelled. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(task), timeout=0.01) + + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertEqual(await task, "done") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py new file mode 100644 index 00000000000..0af3368627a --- /dev/null +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -0,0 +1,363 @@ +import os +import signal +import socket +import sys +import time +import threading +import unittest +from unittest import mock + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _overlapped +import _winapi + +import asyncio +from asyncio import windows_events +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class UpperProto(asyncio.Protocol): + def __init__(self): + self.buf = [] + + def connection_made(self, trans): + self.trans = trans + + def data_received(self, data): + self.buf.append(data) + if b'\n' in data: + self.trans.write(b''.join(self.buf).upper()) + self.trans.close() + + +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): + + def test_ctrl_c(self): + + def SIGINT_after_delay(): + time.sleep(0.1) + signal.raise_signal(signal.SIGINT) + + thread = threading.Thread(target=SIGINT_after_delay) + loop = asyncio.new_event_loop() + try: + # only start the loop once the event loop is running + loop.call_soon(thread.start) + loop.run_forever() + self.fail("should not fall through 'run_forever'") + except KeyboardInterrupt: + pass + finally: + self.close_loop(loop) + thread.join() + + +class ProactorMultithreading(WindowsEventsTestCase): + def test_run_from_nonmain_thread(self): + finished = False + + async def coro(): + await asyncio.sleep(0) + + def func(): + nonlocal finished + loop = asyncio.new_event_loop() + loop.run_until_complete(coro()) + # close() must not call signal.set_wakeup_fd() + loop.close() + finished = True + + thread = threading.Thread(target=func) + thread.start() + thread.join() + self.assertTrue(finished) + + +class ProactorTests(WindowsEventsTestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + def test_close(self): + a, b = socket.socketpair() + trans = self.loop._make_socket_transport(a, asyncio.Protocol()) + f = asyncio.ensure_future(self.loop.sock_recv(b, 100), loop=self.loop) + trans.close() + self.loop.run_until_complete(f) + self.assertEqual(f.result(), b'') + b.close() + + def test_double_bind(self): + ADDRESS = r'\\.\pipe\test_double_bind-%s' % os.getpid() + server1 = windows_events.PipeServer(ADDRESS) + with self.assertRaises(PermissionError): + windows_events.PipeServer(ADDRESS) + server1.close() + + def test_pipe(self): + res = self.loop.run_until_complete(self._test_pipe()) + self.assertEqual(res, 'done') + + async def _test_pipe(self): + ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid() + + with self.assertRaises(FileNotFoundError): + await self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + [server] = await self.loop.start_serving_pipe( + UpperProto, ADDRESS) + self.assertIsInstance(server, windows_events.PipeServer) + + clients = [] + for i in range(5): + stream_reader = asyncio.StreamReader(loop=self.loop) + protocol = asyncio.StreamReaderProtocol(stream_reader, + loop=self.loop) + trans, proto = await self.loop.create_pipe_connection( + lambda: protocol, ADDRESS) + self.assertIsInstance(trans, asyncio.Transport) + self.assertEqual(protocol, proto) + clients.append((stream_reader, trans)) + + for i, (r, w) in enumerate(clients): + w.write('lower-{}\n'.format(i).encode()) + + for i, (r, w) in enumerate(clients): + response = await r.readline() + self.assertEqual(response, 'LOWER-{}\n'.format(i).encode()) + w.close() + + server.close() + + with self.assertRaises(FileNotFoundError): + await self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + return 'done' + + def test_connect_pipe_cancel(self): + exc = OSError() + exc.winerror = _overlapped.ERROR_PIPE_BUSY + with mock.patch.object(_overlapped, 'ConnectPipe', + side_effect=exc) as connect: + coro = self.loop._proactor.connect_pipe('pipe_address') + task = self.loop.create_task(coro) + + # check that it's possible to cancel connect_pipe() + task.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(task) + + def test_wait_for_handle(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with 0.5s timeout; + # result should be False at timeout + timeout = 0.5 + fut = self.loop._proactor.wait_for_handle(event, timeout) + start = self.loop.time() + done = self.loop.run_until_complete(fut) + elapsed = self.loop.time() - start + + self.assertEqual(done, False) + self.assertFalse(fut.result()) + self.assertGreaterEqual(elapsed, timeout - test_utils.CLOCK_RES) + + _overlapped.SetEvent(event) + + # Wait for set event; + # result should be True immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + done = self.loop.run_until_complete(fut) + + self.assertEqual(done, True) + self.assertTrue(fut.result()) + + # asyncio issue #195: cancelling a done _WaitHandleFuture + # must not crash + fut.cancel() + + def test_wait_for_handle_cancel(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with a cancelled future; + # CancelledError should be raised immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + fut.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(fut) + + # asyncio issue #195: cancelling a _WaitHandleFuture twice + # must not crash + fut = self.loop._proactor.wait_for_handle(event) + fut.cancel() + fut.cancel() + + def test_read_self_pipe_restart(self): + # Regression test for https://bugs.python.org/issue39010 + # Previously, restarting a proactor event loop in certain states + # would lead to spurious ConnectionResetErrors being logged. + self.loop.call_exception_handler = mock.Mock() + # Start an operation in another thread so that the self-pipe is used. + # This is theoretically timing-dependent (the task in the executor + # must complete before our start/stop cycles), but in practice it + # seems to work every time. + f = self.loop.run_in_executor(None, lambda: None) + self.loop.stop() + self.loop.run_forever() + self.loop.stop() + self.loop.run_forever() + + # Shut everything down cleanly. This is an important part of the + # test - in issue 39010, the error occurred during loop.close(), + # so we want to close the loop during the test instead of leaving + # it for tearDown. + # + # First wait for f to complete to avoid a "future's result was never + # retrieved" error. + self.loop.run_until_complete(f) + # Now shut down the loop itself (self.close_loop also shuts down the + # loop's default executor). + self.close_loop(self.loop) + self.assertFalse(self.loop.call_exception_handler.called) + + def test_address_argument_type_error(self): + # Regression test for https://github.com/python/cpython/issues/98793 + proactor = self.loop._proactor + sock = socket.socket(type=socket.SOCK_DGRAM) + bad_address = None + with self.assertRaises(TypeError): + proactor.connect(sock, bad_address) + with self.assertRaises(TypeError): + proactor.sendto(sock, b'abc', addr=bad_address) + sock.close() + + def test_client_pipe_stat(self): + res = self.loop.run_until_complete(self._test_client_pipe_stat()) + self.assertEqual(res, 'done') + + async def _test_client_pipe_stat(self): + # Regression test for https://github.com/python/cpython/issues/100573 + ADDRESS = r'\\.\pipe\test_client_pipe_stat-%s' % os.getpid() + + async def probe(): + # See https://github.com/python/cpython/pull/100959#discussion_r1068533658 + h = _overlapped.ConnectPipe(ADDRESS) + try: + _winapi.CloseHandle(_overlapped.ConnectPipe(ADDRESS)) + except OSError as e: + if e.winerror != _overlapped.ERROR_PIPE_BUSY: + raise + finally: + _winapi.CloseHandle(h) + + with self.assertRaises(FileNotFoundError): + await probe() + + [server] = await self.loop.start_serving_pipe(asyncio.Protocol, ADDRESS) + self.assertIsInstance(server, windows_events.PipeServer) + + errors = [] + self.loop.set_exception_handler(lambda _, data: errors.append(data)) + + for i in range(5): + await self.loop.create_task(probe()) + + self.assertEqual(len(errors), 0, errors) + + server.close() + + with self.assertRaises(FileNotFoundError): + await probe() + + return "done" + + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): + + def test_selector_win_policy(self): + async def main(): + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) + + old_policy = asyncio.events._get_event_loop_policy() + try: + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.run(main()) + finally: + asyncio.events._set_event_loop_policy(old_policy) + + def test_proactor_win_policy(self): + async def main(): + self.assertIsInstance( + asyncio.get_running_loop(), + asyncio.ProactorEventLoop) + + old_policy = asyncio.events._get_event_loop_policy() + try: + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + asyncio.run(main()) + finally: + asyncio.events._set_event_loop_policy(old_policy) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py new file mode 100644 index 00000000000..97f078ff911 --- /dev/null +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -0,0 +1,133 @@ +"""Tests for window_utils""" + +import sys +import unittest +import warnings + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _overlapped +import _winapi + +import asyncio +from asyncio import windows_utils +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class PipeTests(unittest.TestCase): + + def test_pipe_overlapped(self): + h1, h2 = windows_utils.pipe(overlapped=(True, True)) + try: + ov1 = _overlapped.Overlapped() + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, 0) + + ov1.ReadFile(h1, 100) + self.assertTrue(ov1.pending) + self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING) + ERROR_IO_INCOMPLETE = 996 + try: + ov1.getresult() + except OSError as e: + self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE) + else: + raise RuntimeError('expected ERROR_IO_INCOMPLETE') + + ov2 = _overlapped.Overlapped() + self.assertFalse(ov2.pending) + self.assertEqual(ov2.error, 0) + + ov2.WriteFile(h2, b"hello") + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + + res = _winapi.WaitForMultipleObjects([ov2.event], False, 100) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE) + self.assertFalse(ov2.pending) + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + self.assertEqual(ov1.getresult(), b"hello") + finally: + _winapi.CloseHandle(h1) + _winapi.CloseHandle(h2) + + def test_pipe_handle(self): + h, _ = windows_utils.pipe(overlapped=(True, True)) + _winapi.CloseHandle(_) + p = windows_utils.PipeHandle(h) + self.assertEqual(p.fileno(), h) + self.assertEqual(p.handle, h) + + # check garbage collection of p closes handle + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", ResourceWarning) + del p + support.gc_collect() + try: + _winapi.CloseHandle(h) + except OSError as e: + self.assertEqual(e.winerror, 6) # ERROR_INVALID_HANDLE + else: + raise RuntimeError('expected ERROR_INVALID_HANDLE') + + +class PopenTests(unittest.TestCase): + + def test_popen(self): + command = r"""if 1: + import sys + s = sys.stdin.readline() + sys.stdout.write(s.upper()) + sys.stderr.write('stderr') + """ + msg = b"blah\n" + + p = windows_utils.Popen([sys.executable, '-c', command], + stdin=windows_utils.PIPE, + stdout=windows_utils.PIPE, + stderr=windows_utils.PIPE) + + for f in [p.stdin, p.stdout, p.stderr]: + self.assertIsInstance(f, windows_utils.PipeHandle) + + ovin = _overlapped.Overlapped() + ovout = _overlapped.Overlapped() + overr = _overlapped.Overlapped() + + ovin.WriteFile(p.stdin.handle, msg) + ovout.ReadFile(p.stdout.handle, 100) + overr.ReadFile(p.stderr.handle, 100) + + events = [ovin.event, ovout.event, overr.event] + # Super-long timeout for slow buildbots. + res = _winapi.WaitForMultipleObjects(events, True, + int(support.SHORT_TIMEOUT * 1000)) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + self.assertFalse(ovout.pending) + self.assertFalse(overr.pending) + self.assertFalse(ovin.pending) + + self.assertEqual(ovin.getresult(), len(msg)) + out = ovout.getresult().rstrip() + err = overr.getresult().rstrip() + + self.assertGreater(len(out), 0) + self.assertGreater(len(err), 0) + # allow for partial reads... + self.assertStartsWith(msg.upper().rstrip(), out) + self.assertStartsWith(b"stderr", err) + + # The context manager calls wait() and closes resources + with p: + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py new file mode 100644 index 00000000000..a480e16e81b --- /dev/null +++ b/Lib/test/test_asyncio/utils.py @@ -0,0 +1,609 @@ +"""Utilities shared by tests.""" + +import asyncio +import collections +import contextlib +import io +import logging +import os +import re +import selectors +import socket +import socketserver +import sys +import threading +import unittest +import weakref +from ast import literal_eval +from unittest import mock + +from http.server import HTTPServer +from wsgiref.simple_server import WSGIRequestHandler, WSGIServer + +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from asyncio import base_events +from asyncio import events +from asyncio import format_helpers +from asyncio import tasks +from asyncio.log import logger +from test import support +from test.support import socket_helper +from test.support import threading_helper + + +# Use the maximum known clock resolution (gh-75191, gh-110088): Windows +# GetTickCount64() has a resolution of 15.6 ms. Use 50 ms to tolerate rounding +# issues. +CLOCK_RES = 0.050 + + +def data_file(*filename): + fullname = os.path.join(support.TEST_HOME_DIR, *filename) + if os.path.isfile(fullname): + return fullname + fullname = os.path.join(os.path.dirname(__file__), '..', *filename) + if os.path.isfile(fullname): + return fullname + raise FileNotFoundError(os.path.join(filename)) + + +ONLYCERT = data_file('certdata', 'ssl_cert.pem') +ONLYKEY = data_file('certdata', 'ssl_key.pem') +SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') +SIGNING_CA = data_file('certdata', 'pycacert.pem') +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) + +def simple_server_sslcontext(): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(ONLYCERT, ONLYKEY) + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + return server_context + + +def simple_client_sslcontext(*, disable_verify=True): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + if disable_verify: + client_context.verify_mode = ssl.CERT_NONE + return client_context + + +def dummy_ssl_context(): + if ssl is None: + return None + else: + return simple_client_sslcontext(disable_verify=True) + + +def run_briefly(loop): + async def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() + + +def run_until(loop, pred, timeout=support.SHORT_TIMEOUT): + delay = 0.001 + for _ in support.busy_retry(timeout, error=False): + if pred(): + break + loop.run_until_complete(tasks.sleep(delay)) + delay = max(delay * 2, 1.0) + else: + raise TimeoutError() + + +def run_once(loop): + """Legacy API to run once through the event loop. + + This is the recommended pattern for test code. It will poll the + selector once and run all callbacks scheduled in response to I/O + events. + """ + loop.call_soon(loop.stop) + loop.run_forever() + + +class SilentWSGIRequestHandler(WSGIRequestHandler): + + def get_stderr(self): + return io.StringIO() + + def log_message(self, format, *args): + pass + + +class SilentWSGIServer(WSGIServer): + + request_timeout = support.LOOPBACK_TIMEOUT + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + return request, client_addr + + def handle_error(self, request, client_address): + pass + + +class SSLWSGIServerMixin: + + def finish_request(self, request, client_address): + # The relative location of our test directory (which + # contains the ssl key and certificate files) differs + # between the stdlib and stand-alone asyncio. + # Prefer our own if we can find it. + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(ONLYCERT, ONLYKEY) + + ssock = context.wrap_socket(request, server_side=True) + try: + self.RequestHandlerClass(ssock, client_address, self) + ssock.close() + except OSError: + # maybe socket has been closed by peer + pass + + +class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer): + pass + + +def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls): + + def loop(environ): + size = int(environ['CONTENT_LENGTH']) + while size: + data = environ['wsgi.input'].read(min(size, 0x10000)) + yield data + size -= len(data) + + def app(environ, start_response): + status = '200 OK' + headers = [('Content-type', 'text/plain')] + start_response(status, headers) + if environ['PATH_INFO'] == '/loop': + return loop(environ) + else: + return [b'Test message'] + + # Run the test WSGI server in a separate thread in order not to + # interfere with event handling in the main thread + server_class = server_ssl_cls if use_ssl else server_cls + httpd = server_class(address, SilentWSGIRequestHandler) + httpd.set_app(app) + httpd.address = httpd.server_address + server_thread = threading.Thread( + target=lambda: httpd.serve_forever(poll_interval=0.05)) + server_thread.start() + try: + yield httpd + finally: + httpd.shutdown() + httpd.server_close() + server_thread.join() + + +if hasattr(socket, 'AF_UNIX'): + + class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer): + + def server_bind(self): + socketserver.UnixStreamServer.server_bind(self) + self.server_name = '127.0.0.1' + self.server_port = 80 + + + class UnixWSGIServer(UnixHTTPServer, WSGIServer): + + request_timeout = support.LOOPBACK_TIMEOUT + + def server_bind(self): + UnixHTTPServer.server_bind(self) + self.setup_environ() + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + # Code in the stdlib expects that get_request + # will return a socket and a tuple (host, port). + # However, this isn't true for UNIX sockets, + # as the second return value will be a path; + # hence we return some fake data sufficient + # to get the tests going + return request, ('127.0.0.1', '') + + + class SilentUnixWSGIServer(UnixWSGIServer): + + def handle_error(self, request, client_address): + pass + + + class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): + pass + + + def gen_unix_socket_path(): + return socket_helper.create_unix_domain_name() + + + @contextlib.contextmanager + def unix_socket_path(): + path = gen_unix_socket_path() + try: + yield path + finally: + try: + os.unlink(path) + except OSError: + pass + + + @contextlib.contextmanager + def run_test_unix_server(*, use_ssl=False): + with unix_socket_path() as path: + yield from _run_test_server(address=path, use_ssl=use_ssl, + server_cls=SilentUnixWSGIServer, + server_ssl_cls=UnixSSLWSGIServer) + + +@contextlib.contextmanager +def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False): + yield from _run_test_server(address=(host, port), use_ssl=use_ssl, + server_cls=SilentWSGIServer, + server_ssl_cls=SSLWSGIServer) + + +def echo_datagrams(sock): + while True: + data, addr = sock.recvfrom(4096) + if data == b'STOP': + sock.close() + break + else: + sock.sendto(data, addr) + + +@contextlib.contextmanager +def run_udp_echo_server(*, host='127.0.0.1', port=0): + addr_info = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM) + family, type, proto, _, sockaddr = addr_info[0] + sock = socket.socket(family, type, proto) + sock.bind((host, port)) + sockname = sock.getsockname() + thread = threading.Thread(target=lambda: echo_datagrams(sock)) + thread.start() + try: + yield sockname + finally: + # gh-122187: use a separate socket to send the stop message to avoid + # TSan reported race on the same socket. + sock2 = socket.socket(family, type, proto) + sock2.sendto(b'STOP', sockname) + sock2.close() + thread.join() + + +def make_test_protocol(base): + dct = {} + for name in dir(base): + if name.startswith('__') and name.endswith('__'): + # skip magic names + continue + dct[name] = MockCallback(return_value=None) + return type('TestProtocol', (base,) + base.__bases__, dct)() + + +class TestSelector(selectors.BaseSelector): + + def __init__(self): + self.keys = {} + + def register(self, fileobj, events, data=None): + key = selectors.SelectorKey(fileobj, 0, events, data) + self.keys[fileobj] = key + return key + + def unregister(self, fileobj): + return self.keys.pop(fileobj) + + def select(self, timeout): + return [] + + def get_map(self): + return self.keys + + +class TestLoop(base_events.BaseEventLoop): + """Loop for unittests. + + It manages self time directly. + If something scheduled to be executed later then + on next loop iteration after all ready handlers done + generator passed to __init__ is calling. + + Generator should be like this: + + def gen(): + ... + when = yield ... + ... = yield time_advance + + Value returned by yield is absolute time of next scheduled handler. + Value passed to yield is time advance to move loop's time forward. + """ + + def __init__(self, gen=None): + super().__init__() + + if gen is None: + def gen(): + yield + self._check_on_close = False + else: + self._check_on_close = True + + self._gen = gen() + next(self._gen) + self._time = 0 + self._clock_resolution = 1e-9 + self._timers = [] + self._selector = TestSelector() + + self.readers = {} + self.writers = {} + self.reset_counters() + + self._transports = weakref.WeakValueDictionary() + + def time(self): + return self._time + + def advance_time(self, advance): + """Move test time forward.""" + if advance: + self._time += advance + + def close(self): + super().close() + if self._check_on_close: + try: + self._gen.send(0) + except StopIteration: + pass + else: # pragma: no cover + raise AssertionError("Time generator is not finished") + + def _add_reader(self, fd, callback, *args): + self.readers[fd] = events.Handle(callback, args, self, None) + + def _remove_reader(self, fd): + self.remove_reader_count[fd] += 1 + if fd in self.readers: + del self.readers[fd] + return True + else: + return False + + def assert_reader(self, fd, callback, *args): + if fd not in self.readers: + raise AssertionError(f'fd {fd} is not registered') + handle = self.readers[fd] + if handle._callback != callback: + raise AssertionError( + f'unexpected callback: {handle._callback} != {callback}') + if handle._args != args: + raise AssertionError( + f'unexpected callback args: {handle._args} != {args}') + + def assert_no_reader(self, fd): + if fd in self.readers: + raise AssertionError(f'fd {fd} is registered') + + def _add_writer(self, fd, callback, *args): + self.writers[fd] = events.Handle(callback, args, self, None) + + def _remove_writer(self, fd): + self.remove_writer_count[fd] += 1 + if fd in self.writers: + del self.writers[fd] + return True + else: + return False + + def assert_writer(self, fd, callback, *args): + if fd not in self.writers: + raise AssertionError(f'fd {fd} is not registered') + handle = self.writers[fd] + if handle._callback != callback: + raise AssertionError(f'{handle._callback!r} != {callback!r}') + if handle._args != args: + raise AssertionError(f'{handle._args!r} != {args!r}') + + def _ensure_fd_no_transport(self, fd): + if not isinstance(fd, int): + try: + fd = int(fd.fileno()) + except (AttributeError, TypeError, ValueError): + # This code matches selectors._fileobj_to_fd function. + raise ValueError("Invalid file object: " + "{!r}".format(fd)) from None + try: + transport = self._transports[fd] + except KeyError: + pass + else: + raise RuntimeError( + 'File descriptor {!r} is used by transport {!r}'.format( + fd, transport)) + + def add_reader(self, fd, callback, *args): + """Add a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._add_reader(fd, callback, *args) + + def remove_reader(self, fd): + """Remove a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_reader(fd) + + def add_writer(self, fd, callback, *args): + """Add a writer callback..""" + self._ensure_fd_no_transport(fd) + return self._add_writer(fd, callback, *args) + + def remove_writer(self, fd): + """Remove a writer callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_writer(fd) + + def reset_counters(self): + self.remove_reader_count = collections.defaultdict(int) + self.remove_writer_count = collections.defaultdict(int) + + def _run_once(self): + super()._run_once() + for when in self._timers: + advance = self._gen.send(when) + self.advance_time(advance) + self._timers = [] + + def call_at(self, when, callback, *args, context=None): + self._timers.append(when) + return super().call_at(when, callback, *args, context=context) + + def _process_events(self, event_list): + return + + def _write_to_self(self): + pass + + +def MockCallback(**kwargs): + return mock.Mock(spec=['__call__'], **kwargs) + + +class MockPattern(str): + """A regex based str with a fuzzy __eq__. + + Use this helper with 'mock.assert_called_with', or anywhere + where a regex comparison between strings is needed. + + For instance: + mock_call.assert_called_with(MockPattern('spam.*ham')) + """ + def __eq__(self, other): + return bool(re.search(str(self), other, re.S)) + + +class MockInstanceOf: + def __init__(self, type): + self._type = type + + def __eq__(self, other): + return isinstance(other, self._type) + + +def get_function_source(func): + source = format_helpers._get_function_source(func) + if source is None: + raise ValueError("unable to get the source of %r" % (func,)) + return source + + +class TestCase(unittest.TestCase): + @staticmethod + def close_loop(loop): + if loop._default_executor is not None: + if not loop.is_closed(): + loop.run_until_complete(loop.shutdown_default_executor()) + else: + loop._default_executor.shutdown(wait=True) + loop.close() + + def set_event_loop(self, loop, *, cleanup=True): + if loop is None: + raise AssertionError('loop is None') + # ensure that the event loop is passed explicitly in asyncio + events.set_event_loop(None) + if cleanup: + self.addCleanup(self.close_loop, loop) + + def new_test_loop(self, gen=None): + loop = TestLoop(gen) + self.set_event_loop(loop) + return loop + + def setUp(self): + self._thread_cleanup = threading_helper.threading_setup() + + def tearDown(self): + events.set_event_loop(None) + + # Detect CPython bug #23353: ensure that yield/yield-from is not used + # in an except block of a generator + self.assertIsNone(sys.exception()) + + self.doCleanups() + threading_helper.threading_cleanup(*self._thread_cleanup) + support.reap_children() + + +@contextlib.contextmanager +def disable_logger(): + """Context manager to disable asyncio logger. + + For example, it can be used to ignore warnings in debug mode. + """ + old_level = logger.level + try: + logger.setLevel(logging.CRITICAL+1) + yield + finally: + logger.setLevel(old_level) + + +def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, + family=socket.AF_INET): + """Create a mock of a non-blocking socket.""" + sock = mock.MagicMock(socket.socket) + sock.proto = proto + sock.type = type + sock.family = family + sock.gettimeout.return_value = 0.0 + return sock + + +async def await_without_task(coro): + exc = None + def func(): + try: + for _ in coro.__await__(): + pass + except BaseException as err: + nonlocal exc + exc = err + asyncio.get_running_loop().call_soon(func) + await asyncio.sleep(0) + if exc is not None: + raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 409c8c109e8..a6739124571 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -1,10 +1,18 @@ -import unittest import base64 import binascii import os +import unittest from array import array +from test.support import cpython_only from test.support import os_helper from test.support import script_helper +from test.support.import_helper import ensure_lazy_imports + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("base64", {"re", "getopt"}) class LegacyBase64TestCase(unittest.TestCase): @@ -200,18 +208,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -232,6 +228,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -264,9 +274,12 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) def test_b32encode(self): eq = self.assertEqual @@ -321,23 +334,33 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') + def test_b32decode_map01(self): # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] @@ -804,7 +827,7 @@ def test_decode_nonascii_str(self): self.assertRaises(ValueError, f, 'with non-ascii \xcb') def test_ErrorHeritage(self): - self.assertTrue(issubclass(binascii.Error, ValueError)) + self.assertIsSubclass(binascii.Error, ValueError) def test_RFC4648_test_cases(self): # test cases from RFC 4648 section 10 diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 63bf538aa53..5870dc7f9da 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,13 +10,11 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation @@ -65,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index a3abbbb8db2..c41fb763a16 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -586,7 +590,7 @@ def fail(self, msg=None): class StateTestCase(BaseTestCase): """Test the step, next, return, until and quit 'set_' methods.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -597,7 +601,7 @@ def test_step(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step_next_on_last_statement(self): for set_type in ('step', 'next'): with self.subTest(set_type=set_type): @@ -612,7 +616,17 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',), ('quit',)] + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -624,7 +638,7 @@ def test_next(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_over_import(self): code = """ def main(): @@ -639,7 +653,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_on_plain_statement(self): # Check that set_next() is equivalent to set_step() on a plain # statement. @@ -652,7 +666,7 @@ def test_next_on_plain_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_in_caller_frame(self): # Check that set_next() in the caller frame causes the tracer # to stop next in the caller frame. @@ -666,7 +680,7 @@ def test_next_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -679,7 +693,7 @@ def test_return(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -691,7 +705,7 @@ def test_return_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -703,7 +717,7 @@ def test_until(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until_with_too_large_count(self): self.expect_set = [ ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), @@ -714,7 +728,7 @@ def test_until_with_too_large_count(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_until_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -726,7 +740,8 @@ def test_until_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs + @patch_list(sys.meta_path) def test_skip(self): # Check that tracing is skipped over the import statement in # 'tfunc_import()'. @@ -759,7 +774,7 @@ def test_skip_with_no_name_module(self): bdb = Bdb(skip=['anything*']) self.assertIs(bdb.is_skipped_module(None), False) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_down(self): # Check that set_down() raises BdbError at the newest frame. self.expect_set = [ @@ -768,7 +783,7 @@ def test_down(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_up(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -782,7 +797,7 @@ def test_up(self): class BreakpointTestCase(BaseTestCase): """Test the breakpoint set method.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_on_non_existent_module(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) @@ -790,7 +805,7 @@ def test_bp_on_non_existent_module(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_after_last_statement(self): code = """ def main(): @@ -804,7 +819,7 @@ def main(): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_temporary_bp(self): code = """ def func(): @@ -828,7 +843,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_disabled_temporary_bp(self): code = """ def func(): @@ -857,7 +872,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_condition(self): code = """ def func(a): @@ -878,7 +893,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_exception_on_condition_evaluation(self): code = """ def func(a): @@ -898,7 +913,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_bp_ignore_count(self): code = """ def func(): @@ -920,7 +935,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_ignore_count_on_disabled_bp(self): code = """ def func(): @@ -948,7 +963,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_clear_two_bp_on_same_line(self): code = """ def func(): @@ -974,7 +989,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_clear_at_no_bp(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) @@ -1028,7 +1043,7 @@ def test_load_bps_from_previous_Bdb_instance(self): class RunTestCase(BaseTestCase): """Test run, runeval and set_trace.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_run_step(self): # Check that the bdb 'run' method stops at the first line event. code = """ @@ -1041,7 +1056,7 @@ def test_run_step(self): with TracerRun(self) as tracer: tracer.run(compile(textwrap.dedent(code), '', 'exec')) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_runeval_step(self): # Test bdb 'runeval'. code = """ @@ -1064,7 +1079,7 @@ def main(): class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_step_at_return_with_no_trace_in_caller(self): # Issue #13183. # Check that the tracer does step into the caller frame when the @@ -1095,7 +1110,7 @@ def func(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_until_return_in_generator(self): # Issue #16596. # Check that set_next(), set_until() and set_return() do not treat the @@ -1137,7 +1152,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_command_in_generator_for_loop(self): # Issue #16596. code = """ @@ -1169,7 +1184,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_next_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1201,7 +1216,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.expectedFailure # TODO: RUSTPYTHON; Error in atexit._run_exitfuncs def test_return_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1233,6 +1248,20 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',)] + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index aaa9972bc45..8f528812e35 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -638,8 +638,6 @@ def test_encode_utf7(self, size): except MemoryError: pass # acceptable on 32-bit - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_4G // 4 + 5, memuse=ascii_char_size + ucs4_char_size + 4) def test_encode_utf32(self, size): try: diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index cf11ffce7f1..fa027710489 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -38,13 +38,13 @@ def assertConversion(self, original, converted, restored, **kwargs): def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) def test_returned_value(self): @@ -117,7 +117,7 @@ def addnoise(line): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_base64_strict_mode(self): # Test base64 with strict mode on def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c498..b224c3d4e60 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 34ecb45f161..dcdf7bdce03 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -383,6 +383,10 @@ def __len__(self): __bool__ = None self.assertRaises(TypeError, bool, B()) + class C: + __len__ = None + self.assertRaises(TypeError, bool, C()) + def test_real_and_imag(self): self.assertEqual(True.real, 1) self.assertEqual(True.imag, 0) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 468c6ea9def..bc09329e6de 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,12 +17,14 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings import sys, array, io, os from decimal import Decimal from fractions import Fraction +from test.support import warnings_helper try: from _testbuffer import * @@ -64,7 +66,7 @@ '?':0, 'c':0, 'b':0, 'B':0, 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, - 'f':0, 'd':0, 'P':0 + 'e':0, 'f':0, 'd':0, 'P':0 } # NumPy does not have 'n' or 'N': @@ -89,7 +91,8 @@ 'i':(-(1<<31), 1<<31), 'I':(0, 1<<32), 'l':(-(1<<31), 1<<31), 'L':(0, 1<<32), 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), - 'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023) + 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), + 'd':(-(1<<1023), 1<<1023) } def native_type_range(fmt): @@ -98,6 +101,8 @@ def native_type_range(fmt): lh = (0, 256) elif fmt == '?': lh = (0, 2) + elif fmt == 'e': + lh = (-65519, 65520) elif fmt == 'f': lh = (-(1<<63), 1<<63) elif fmt == 'd': @@ -125,7 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() + +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -160,11 +168,11 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) - if char == 'f' or char == 'd': + if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] return x @@ -959,8 +967,10 @@ def check_memoryview(m, expected_readonly=readonly): self.assertEqual(m.strides, tuple(strides)) self.assertEqual(m.suboffsets, tuple(suboffsets)) - n = 1 if ndim == 0 else len(lst) - self.assertEqual(len(m), n) + if ndim == 0: + self.assertRaises(TypeError, len, m) + else: + self.assertEqual(len(m), len(lst)) rep = result.tolist() if fmt else result.tobytes() self.assertEqual(rep, lst) @@ -1019,6 +1029,7 @@ def match(req, flag): ndim=ndim, shape=shape, strides=strides, lst=lst, sliced=sliced) + @support.requires_resource('cpu') def test_ndarray_getbuf(self): requests = ( # distinct flags @@ -1907,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1939,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1985,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) @@ -2246,7 +2257,7 @@ def test_py_buffer_to_contiguous(self): ### ### Fortran output: ### --------------- - ### >>> fortran_buf = nd.tostring(order='F') + ### >>> fortran_buf = nd.tobytes(order='F') ### >>> fortran_buf ### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b' ### @@ -2289,7 +2300,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='C')) + self.assertEqual(b, na.tobytes(order='C')) # 'F' request if f == 0: # 'C' to 'F' @@ -2312,7 +2323,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='F')) + self.assertEqual(b, na.tobytes(order='F')) # 'A' request if f == ND_FORTRAN: @@ -2336,7 +2347,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='A')) + self.assertEqual(b, na.tobytes(order='A')) # multi-dimensional, non-contiguous input nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL) @@ -2750,6 +2761,7 @@ def iter_roundtrip(ex, m, items, fmt): m = memoryview(ex) iter_roundtrip(ex, m, items, fmt) + @support.requires_resource('cpu') def test_memoryview_cast_1D_ND(self): # Cast between C-contiguous buffers. At least one buffer must # be 1D, at least one format must be 'c', 'b' or 'B'. @@ -2867,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith("l:x:>l:y:}" @@ -3227,6 +3233,15 @@ class BEPoint(ctypes.BigEndianStructure): self.assertNotEqual(point, a) self.assertRaises(NotImplementedError, a.tolist) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') + def test_memoryview_compare_special_cases_deprecated_u_type_code(self): + + # Depends on issue #15625: the struct module does not understand 'u'. + a = array.array('u', 'xyz') + v = memoryview(a) + self.assertNotEqual(a, v) + self.assertNotEqual(v, a) + def test_memoryview_compare_ndim_zero(self): nd1 = ndarray(1729, shape=[], format='@L') @@ -3895,6 +3910,8 @@ def test_memoryview_check_released(self): self.assertRaises(ValueError, memoryview, m) # memoryview.cast() self.assertRaises(ValueError, m.cast, 'c') + # memoryview.__iter__() + self.assertRaises(ValueError, m.__iter__) # getbuffer() self.assertRaises(ValueError, ndarray, m) # memoryview.tolist() @@ -4422,6 +4439,14 @@ def test_issue_7385(self): x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL) self.assertRaises(BufferError, memoryview, x) + def test_bytearray_release_buffer_read_flag(self): + # See https://github.com/python/cpython/issues/126980 + obj = bytearray(b'abc') + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.READ) + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.WRITE) + @support.cpython_only def test_pybuffer_size_from_format(self): # basic tests @@ -4429,6 +4454,383 @@ def test_pybuffer_size_from_format(self): self.assertEqual(_testcapi.PyBuffer_SizeFromFormat(format), struct.calcsize(format)) + @support.cpython_only + def test_flags_overflow(self): + # gh-126594: Check for integer overlow on large flags + try: + from _testcapi import INT_MIN, INT_MAX + except ImportError: + INT_MIN = -(2 ** 31) + INT_MAX = 2 ** 31 - 1 + + obj = b'abc' + for flags in (INT_MIN - 1, INT_MAX + 1): + with self.subTest(flags=flags): + with self.assertRaises(OverflowError): + obj.__buffer__(flags) + + +class TestPythonBufferProtocol(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + + def __release_buffer__(self, buffer): + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last_raising(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer_with_exception_set(self): + class A: + def __buffer__(self, flags): + return memoryview(bytes(8)) + def __release_buffer__(self, view): + pass + + b = bytearray(8) + with memoryview(b): + # now b.extend will raise an exception due to exports + with self.assertRaises(BufferError): + b.extend(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py index 989d8cd349b..cb9cb4d0bc7 100644 --- a/Lib/test/test_bufio.py +++ b/Lib/test/test_bufio.py @@ -28,7 +28,7 @@ def try_one(self, s): f.write(b"\n") f.write(s) f.close() - f = open(os_helper.TESTFN, "rb") + f = self.open(os_helper.TESTFN, "rb") line = f.readline() self.assertEqual(line, s + b"\n") line = f.readline() @@ -65,9 +65,6 @@ def test_nullpat(self): class CBufferSizeTest(BufferSizeTest, unittest.TestCase): open = io.open -# TODO: RUSTPYTHON -import sys -@unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, can't cleanup temporary file on Windows") class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 183caa898ef..a2d2e3bb395 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,14 +1,15 @@ # Python test set -- built-in functions import ast -import asyncio import builtins import collections +import contextlib import decimal import fractions import gc import io import locale +import math import os import pickle import platform @@ -17,6 +18,7 @@ import sys import traceback import types +import typing import unittest import warnings from contextlib import ExitStack @@ -27,10 +29,14 @@ from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (swap_attr, maybe_get_event_loop_policy) +from test.support import cpython_only, swap_attr +from test.support import async_yield, run_yielding_async_fn +from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings +from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch try: import pty, signal @@ -38,6 +44,14 @@ pty = signal = None +# Detect evidence of double-rounding: sum() does not always +# get improved accuracy on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + +# used as proof of globals being used +A_GLOBAL_VALUE = 123 + class Squares: def __init__(self, max): @@ -134,7 +148,10 @@ def filter_char(arg): def map_char(arg): return chr(ord(arg)+1) -class BuiltinTest(unittest.TestCase): +def pack(*args): + return args + +class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): itorg = it @@ -153,8 +170,6 @@ def check_iter_pickle(self, it, seq, proto): it = pickle.loads(d) self.assertEqual(list(it), seq[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import(self): __import__('sys') __import__('time') @@ -210,6 +225,8 @@ def test_all(self): self.assertEqual(all(x > 42 for x in S), True) S = [50, 40, 60] self.assertEqual(all(x > 42 for x in S), False) + S = [50, 40, 60, TestFailingBool()] + self.assertEqual(all(x > 42 for x in S), False) def test_any(self): self.assertEqual(any([None, None, None]), False) @@ -223,9 +240,59 @@ def test_any(self): self.assertEqual(any([1, TestFailingBool()]), True) # Short-circuit S = [40, 60, 30] self.assertEqual(any(x > 42 for x in S), True) + S = [40, 60, 30, TestFailingBool()] + self.assertEqual(any(x > 42 for x in S), True) S = [10, 20, 30] self.assertEqual(any(x > 42 for x in S), False) + def test_all_any_tuple_optimization(self): + def f_all(): + return all(x-2 for x in [1,2,3]) + + def f_any(): + return any(x-1 for x in [1,2,3]) + + def f_tuple(): + return tuple(2*x for x in [1,2,3]) + + funcs = [f_all, f_any, f_tuple] + + for f in funcs: + # check that generator code object is not duplicated + code_objs = [c for c in f.__code__.co_consts if isinstance(c, type(f.__code__))] + self.assertEqual(len(code_objs), 1) + + + # check the overriding the builtins works + + global all, any, tuple + saved = all, any, tuple + try: + all = lambda x : "all" + any = lambda x : "any" + tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + # Now repeat, overriding the builtins module as well + saved = all, any, tuple + try: + builtins.all = all = lambda x : "all" + builtins.any = any = lambda x : "any" + builtins.tuple = tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + builtins.all, builtins.any, builtins.tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') @@ -300,15 +367,15 @@ class C3(C2): pass c3 = C3() self.assertTrue(callable(c3)) + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust isize def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -320,10 +387,14 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): - self.assertTrue(not hasattr(builtins, "cmp")) + self.assertNotHasAttr(builtins, "cmp") def test_compile(self): compile('print(1)\n', '', 'exec') @@ -362,19 +433,18 @@ def f(): """doc""" (1, False, 'doc', False, False), (2, False, None, False, False)] for optval, *expected in values: + with self.subTest(optval=optval): # test both direct compilation and compilation via AST - codeobjs = [] - codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) - codeobjs.append(compile(tree, "", "exec", optimize=optval)) - for code in codeobjs: - ns = {} - exec(code, ns) - rv = ns['f']() - self.assertEqual(rv, tuple(expected)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr, optimize=optval) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, tuple(expected)) + def test_compile_top_level_await_no_coro(self): """Make sure top level non-await codes get the correct coroutine flags""" modes = ('single', 'exec') @@ -396,14 +466,9 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -416,12 +481,25 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + + async def sleep(delay, result=None): + assert delay == 0 + await async_yield(None) + return result + modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ - '''a = await asyncio.sleep(0, result=1)''', + '''a = await sleep(0, result=1)''', '''async for i in arange(1): a = 1''', - '''async with asyncio.Lock() as l: + '''async with Lock() as l: a = 1''', '''a = [x async for x in arange(2)][1]''', '''a = 1 in {x async for x in arange(2)}''', @@ -429,45 +507,63 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): source = dedent(code_sample) with self.assertRaises( SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) + compile(source, '?', mode, optimize=optimize) co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + msg=f"source={source} mode={mode}") # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(FunctionType(co, globals_)) self.assertEqual(globals_['a'], 1) # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(lambda: eval(co, globals_)) self.assertEqual(globals_['a'], 1) - finally: - asyncio.set_event_loop_policy(policy) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + modes = ('single', 'exec') code_samples = [ '''def f(): await arange(10)\n''', @@ -478,30 +574,23 @@ async def arange(n): a = 1 ''', '''def f(): - async with asyncio.Lock() as l: + async with Lock() as l: a = 1 ''' ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - finally: - asyncio.set_event_loop_policy(policy) + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_async_generator(self): """ With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to @@ -511,13 +600,35 @@ def test_compile_async_generator(self): code = dedent("""async def ticker(): for i in range(10): yield i - await asyncio.sleep(0)""") + await sleep(0)""") co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) glob = {} exec(co, glob) self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <_ast.Name object at 0xb40000731e3d1360> is not an instance of + def test_compile_ast(self): + args = ("a*__debug__", "f.py", "exec") + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] + opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] + + for tree in (raw, opt1, opt2): + self.assertIsInstance(tree.value, ast.BinOp) + self.assertIsInstance(tree.value.op, ast.Mult) + self.assertIsInstance(tree.value.left, ast.Name) + self.assertEqual(tree.value.left.id, 'a') + + raw_right = raw.value.right + self.assertIsInstance(raw_right, ast.Name) + self.assertEqual(raw_right.id, "__debug__") + + for opt in [opt1, opt2]: + opt_right = opt.value.right + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, __debug__) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') @@ -526,8 +637,7 @@ def test_delattr(self): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, delattr, sys, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__repr__' unexpectedly found in ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__firstlineno__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', 'bar'] def test_dir(self): # dir(wrong number of arguments) self.assertRaises(TypeError, dir, 42, 42) @@ -589,6 +699,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): @@ -610,6 +728,7 @@ def test___ne__(self): self.assertIs(None.__ne__(0), NotImplemented) self.assertIs(None.__ne__("abc"), NotImplemented) + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) self.assertEqual(divmod(-12, 7), (-2, 2)) @@ -627,6 +746,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -651,6 +780,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -744,8 +878,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -755,8 +900,6 @@ def test_exec_globals(self): self.assertRaises(TypeError, exec, code, {'__builtins__': 123}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_frozen(self): class frozendict_error(Exception): pass @@ -789,8 +932,6 @@ def __setitem__(self, key, value): self.assertRaises(frozendict_error, exec, code, namespace) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_error_on_get(self): # custom `globals` or `builtins` can raise errors on item access class setonlyerror(Exception): @@ -810,8 +951,6 @@ def __getitem__(self, key): self.assertRaises(setonlyerror, exec, code, {'__builtins__': setonlydict({'superglobal': 1})}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_dict_subclass(self): class customdict(dict): # this one should not do anything fancy pass @@ -823,6 +962,34 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised by eval + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -834,8 +1001,7 @@ def test_exec_redirected(self): finally: sys.stdout = savestdout - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument closure def test_exec_closure(self): def function_without_closures(): return 3 * 5 @@ -903,8 +1069,24 @@ def four_freevars(): three_freevars.__code__, three_freevars.__globals__, closure=my_closure) + my_closure = tuple(my_closure) + + # should fail: anything passed to closure= isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=int) + + # should fail: correct closure= argument isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=my_closure) # should fail: closure tuple with one non-cell-var + my_closure = list(my_closure) my_closure[0] = int my_closure = tuple(my_closure) self.assertRaises(TypeError, @@ -945,6 +1127,20 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + @unittest.skip("TODO: RUSTPYTHON; Segfault") + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') + def test_filter_dealloc(self): + # Tests recursive deallocation of nested filter objects using the + # thrashcan mechanism. See gh-102356 for more details. + max_iters = 1000000 + i = filter(bool, range(max_iters)) + for _ in range(max_iters): + i = filter(bool, i) + del i + gc.collect() + def test_getattr(self): self.assertTrue(getattr(sys, 'stdout') is sys.stdout) self.assertRaises(TypeError, getattr) @@ -996,6 +1192,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') @@ -1157,6 +1363,130 @@ def test_map_pickle(self): m2 = map(map_char, "Is this the real life?") self.check_iter_pickle(m1, list(m2), proto) + # strict map tests based on strict zip tests + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_pickle_strict(self): + a = (1, 2, 3) + b = (4, 5, 6) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + self.check_iter_pickle(m1, t, proto) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_pickle_strict_fail(self): + a = (1, 2, 3) + b = (4, 5, 6, 7) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + m2 = pickle.loads(pickle.dumps(m1, proto)) + self.assertEqual(self.iter_error(m1, ValueError), t) + self.assertEqual(self.iter_error(m2, ValueError), t) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_strict(self): + self.assertEqual(tuple(map(pack, (1, 2, 3), 'abc', strict=True)), + ((1, 'a'), (2, 'b'), (3, 'c'))) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2, 3, 4), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), (1, 2), 'abc', strict=True)) + + # gh-140517: Testing refleaks with mortal objects. + t1 = (None, object()) + t2 = (object(), object()) + t3 = (object(),) + + self.assertRaises(ValueError, tuple, + map(pack, t1, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, t3, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t1, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t2, t3, strict=True)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_strict_iterators(self): + x = iter(range(5)) + y = [0] + z = iter(range(5)) + self.assertRaises(ValueError, list, + (map(pack, x, y, z, strict=True))) + self.assertEqual(next(x), 2) + self.assertEqual(next(z), 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_strict_error_handling(self): + + class Error(Exception): + pass + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise Error + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), Error) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), Error) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), Error) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), Error) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument strict + def test_map_strict_error_handling_stopiteration(self): + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise StopIteration + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), ValueError) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), ValueError) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + def test_max(self): self.assertEqual(max('123123'), '3') self.assertEqual(max(1, 2, 3), 3) @@ -1174,7 +1504,11 @@ def test_max(self): max() self.assertRaises(TypeError, max, 42) - self.assertRaises(ValueError, max, ()) + with self.assertRaisesRegex( + ValueError, + r'max\(\) iterable argument is empty' + ): + max(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1233,7 +1567,11 @@ def test_min(self): min() self.assertRaises(TypeError, min, 42) - self.assertRaises(ValueError, min, ()) + with self.assertRaisesRegex( + ValueError, + r'min\(\) iterable argument is empty' + ): + min(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1334,18 +1672,13 @@ def test_open(self): self.assertRaises(ValueError, open, 'a\x00b') self.assertRaises(ValueError, open, b'a\x00b') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): - old_environ = dict(os.environ) - try: + with EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that open() uses the current locale # encoding and not the user preferred encoding - for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') self.write_testfile() current_locale_encoding = locale.getencoding() @@ -1354,9 +1687,6 @@ def test_open_default_encoding(self): fp = open(TESTFN, 'w') with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) @support.requires_subprocess() def test_open_non_inheritable(self): @@ -1488,6 +1818,29 @@ def test_input(self): sys.stdout = savestdout fp.close() + def test_input_gh130163(self): + class X(io.StringIO): + def __getattribute__(self, name): + nonlocal patch + if patch: + patch = False + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + support.gc_collect() + return io.StringIO.__getattribute__(self, name) + + with (support.swap_attr(sys, 'stdout', None), + support.swap_attr(sys, 'stderr', None), + support.swap_attr(sys, 'stdin', None)): + patch = False + # the only references: + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + patch = True + input() # should not crash + # test_int(): see test_int.py for tests of built-in function int(). def test_repr(self): @@ -1503,6 +1856,11 @@ def test_repr(self): a[0] = a self.assertEqual(repr(a), '{0: {...}}') + def test_repr_blocked(self): + class C: + __repr__ = None + self.assertRaises(TypeError, repr, C()) + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) @@ -1609,15 +1967,19 @@ def test_bug_27936(self): def test_setattr(self): setattr(sys, 'spam', 1) - self.assertEqual(sys.spam, 1) + try: + self.assertEqual(sys.spam, 1) + finally: + del sys.spam self.assertRaises(TypeError, setattr) self.assertRaises(TypeError, setattr, sys) self.assertRaises(TypeError, setattr, sys, 'spam') msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, setattr, sys, 1, 'spam') - # test_str(): see test_unicode.py and test_bytes.py for str() tests. + # test_str(): see test_str.py and test_bytes.py for str() tests. + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: floats 0.0 and -0.0 are not identical: zeros have different signs def test_sum(self): self.assertEqual(sum([]), 0) self.assertEqual(sum(list(range(2,8))), 27) @@ -1646,6 +2008,8 @@ def test_sum(self): self.assertEqual(repr(sum([-0.0])), '0.0') self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0') self.assertEqual(repr(sum([], -0.0)), '-0.0') + self.assertTrue(math.isinf(sum([float("inf"), float("inf")]))) + self.assertTrue(math.isinf(sum([1e308, 1e308]))) self.assertRaises(TypeError, sum) self.assertRaises(TypeError, sum, 42) @@ -1660,6 +2024,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1670,6 +2036,37 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + + # test that sum() of complex and real numbers doesn't + # smash sign of imaginary 0 + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1, complex(1, -0.0)]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1.0]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1.0, complex(1, -0.0)]), + complex(2, -0.0)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sum accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sum_accuracy(self): + self.assertEqual(sum([0.1] * 10), 1.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) + def test_type(self): self.assertEqual(type(''), type('123')) self.assertNotEqual(type(''), type(())) @@ -1949,7 +2346,7 @@ def __format__(self, format_spec): # tests for object.__format__ really belong elsewhere, but # there's no good place to put them x = object().__format__('') - self.assertTrue(x.startswith(' eval() roundtrip - if stdio_encoding: - expected = terminal_input.decode(stdio_encoding, 'surrogateescape') - else: - expected = terminal_input.decode(sys.stdin.encoding) # what else? + if expected is None: + if stdio_encoding: + expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + else: + expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) - def test_input_tty(self): - # Test input() functionality when wired to a tty (the code path - # is different and invokes GNU readline if available). - self.check_input_tty("prompt", b"quux") - - def skip_if_readline(self): + @contextlib.contextmanager + def detach_readline(self): # bpo-13886: When the readline module is loaded, PyOS_Readline() uses # the readline implementation. In some cases, the Python readline # callback rlhandler() is called by readline with a string without - # non-ASCII characters. Skip tests on non-ASCII characters if the - # readline module is loaded, since test_builtin is not intented to test + # non-ASCII characters. + # Unlink readline temporarily from PyOS_Readline() for those tests, + # since test_builtin is not intended to test # the readline module, but the builtins module. - if 'readline' in sys.modules: - self.skipTest("the readline module is loaded") + if "readline" in sys.modules: + c = import_module("ctypes") + fp_api = "PyOS_ReadlineFunctionPointer" + prev_value = c.c_void_p.in_dll(c.pythonapi, fp_api).value + c.c_void_p.in_dll(c.pythonapi, fp_api).value = None + try: + yield + finally: + c.c_void_p.in_dll(c.pythonapi, fp_api).value = prev_value + else: + yield + + def test_input_tty(self): + # Test input() functionality when wired to a tty + self.check_input_tty("prompt", b"quux") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii(self): - self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii_unicode_errors(self): - self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") + def test_input_tty_null_in_prompt(self): + self.check_input_tty("prompt\0", b"", + expected='ValueError: input: prompt string cannot contain ' + 'null characters') + + def test_input_tty_nonencodable_prompt(self): + self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', + expected="UnicodeEncodeError: 'ascii' codec can't encode " + "character '\\xe9' in position 6: ordinal not in " + "range(128)") + + def test_input_tty_nondecodable_input(self): + self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', + expected="UnicodeDecodeError: 'ascii' codec can't decode " + "byte 0xe9 in position 4: ordinal not in " + "range(128)") + + @unittest.skip("TODO: RUSTPYTHON; FAILURE, WORKER BUG") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor @@ -2330,8 +2799,6 @@ def test_baddecorator(self): class ShutdownTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cleanup(self): # Issue #19255: builtins are still available at shutdown code = """if 1: @@ -2364,6 +2831,35 @@ def __del__(self): self.assertEqual(["before", "after"], out.decode().splitlines()) +@cpython_only +class ImmortalTests(unittest.TestCase): + + if sys.maxsize < (1 << 32): + IMMORTAL_REFCOUNT_MINIMUM = 1 << 30 + else: + IMMORTAL_REFCOUNT_MINIMUM = 1 << 31 + + IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) + + def assert_immortal(self, immortal): + with self.subTest(immortal): + self.assertGreater(sys.getrefcount(immortal), self.IMMORTAL_REFCOUNT_MINIMUM) + + def test_immortals(self): + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_list_repeat_respect_immortality(self): + refs = list(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_tuple_repeat_respect_immortality(self): + refs = tuple(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + class TestType(unittest.TestCase): def test_new_type(self): A = type('A', (), {}) @@ -2372,6 +2868,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2401,8 +2898,6 @@ def test_type_nokwargs(self): with self.assertRaises(TypeError): type('a', (), dict={}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_type_name(self): for name in 'A', '\xc4', '\U0001f40d', 'B.A', '42', '': with self.subTest(name=name): @@ -2452,8 +2947,30 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__firstlineno__' unexpectedly found in mappingproxy({'__firstlineno__': 42, '__module__': 'testmodule', '__dict__': , '__doc__': None}) + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'tuple' but 'str' found. + def test_type_typeparams(self): + class A[T]: + pass + T, = A.__type_params__ + self.assertIsInstance(T, typing.TypeVar) + A.__type_params__ = "whatever" + self.assertEqual(A.__type_params__, "whatever") + with self.assertRaises(TypeError): + del A.__type_params__ + self.assertEqual(A.__type_params__, "whatever") + def test_type_doc(self): for doc in 'x', '\xc4', '\U0001f40d', 'x\x00y', b'x', 42, None: A = type('A', (), {'__doc__': doc}) @@ -2487,8 +3004,6 @@ def test_bad_args(self): with self.assertRaises(TypeError): type('A', (int, str), {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_slots(self): with self.assertRaises(TypeError): type('A', (), {'__slots__': b'x'}) @@ -2527,7 +3042,8 @@ def test_namespace_order(self): def load_tests(loader, tests, pattern): from doctest import DocTestSuite - tests.addTest(DocTestSuite(builtins)) + if sys.float_repr_style == 'short': + tests.addTest(DocTestSuite(builtins)) return tests if __name__ == "__main__": diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 8f01f890309..fcef9c0c972 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1288,11 +1288,20 @@ class SubBytes(bytes): self.assertNotEqual(id(s), id(1 * s)) self.assertNotEqual(id(s), id(s * 2)) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fromhex(self): + return super().test_fromhex() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mod(self): + return super().test_mod() + class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray - _testlimitedcapi = import_helper.import_module('_testlimitedcapi') + # XXX: RUSTPYTHON; import_helper.import_module here cause the entire test stopping + _testlimitedcapi = None # import_helper.import_module('_testlimitedcapi') def test_getitem_error(self): b = bytearray(b'python') @@ -1385,6 +1394,7 @@ def by(s): b = by("Hello, world") self.assertEqual(re.findall(br"\w+", b), [by("Hello"), by("world")]) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_setitem(self): def setitem_as_mapping(b, i, val): b[i] = val @@ -1432,6 +1442,7 @@ def do_tests(setitem): with self.subTest("tp_as_sequence"): do_tests(setitem_as_sequence) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_delitem(self): def del_as_mapping(b, i): del b[i] @@ -1618,6 +1629,7 @@ def g(): alloc = b.__alloc__() self.assertGreaterEqual(alloc, len(b)) # NOTE: RUSTPYTHON patched + @unittest.expectedFailure # TODO: RUSTPYTHON def test_extend(self): orig = b'hello' a = bytearray(orig) @@ -1840,6 +1852,7 @@ def test_repeat_after_setslice(self): self.assertEqual(b1, b) self.assertEqual(b3, b'xcxcxc') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mutating_index(self): # bytearray slice assignment can call into python code # that reallocates the internal buffer @@ -1860,6 +1873,7 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mutating_index_inbounds(self): # gh-91153 continued # Ensure buffer is not broken even if length is correct @@ -1893,6 +1907,14 @@ def __index__(self): self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fromhex(self): + return super().test_fromhex() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mod(self): + return super().test_mod() + class AssortedBytesTest(unittest.TestCase): # @@ -1912,6 +1934,7 @@ def test_bytes_repr(self, f=repr): self.assertEqual(f(b"'\"'"), r"""b'\'"\''""") # '\'"\'' self.assertEqual(f(BytesSubclass(b"abc")), "b'abc'") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bytearray_repr(self, f=repr): self.assertEqual(f(bytearray()), "bytearray(b'')") self.assertEqual(f(bytearray(b'abc')), "bytearray(b'abc')") @@ -1933,6 +1956,7 @@ def test_bytearray_repr(self, f=repr): def test_bytes_str(self): self.test_bytes_repr(str) + @unittest.expectedFailure # TODO: RUSTPYTHON @check_bytes_warnings def test_bytearray_str(self): self.test_bytearray_repr(str) @@ -2138,7 +2162,6 @@ def test_join(self): s3 = s1.join([b"abcd"]) self.assertIs(type(s3), self.basetype) - @unittest.skip('TODO: RUSTPYTHON; Fails on ByteArraySubclassWithSlotsTest') def test_pickle(self): a = self.type2test(b"abcd") a.x = 10 @@ -2153,7 +2176,6 @@ def test_pickle(self): self.assertEqual(type(a.z), type(b.z)) self.assertFalse(hasattr(b, 'y')) - @unittest.skip('TODO: RUSTPYTHON; Fails on ByteArraySubclassWithSlotsTest') def test_copy(self): a = self.type2test(b"abcd") a.x = 10 @@ -2235,6 +2257,14 @@ class ByteArraySubclassWithSlotsTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclassWithSlots + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_copy(self): + return super().test_copy() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickle(self): + return super().test_pickle() + class BytesSubclassTest(SubclassTest, unittest.TestCase): basetype = bytes type2test = BytesSubclass diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 3617eba8e8d..148d8f98c79 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -16,7 +16,7 @@ from test.support import import_helper from test.support import threading_helper from test.support.os_helper import unlink, FakePath -import _compression +from compression._common import _streams import sys @@ -126,15 +126,15 @@ def testReadMultiStream(self): def testReadMonkeyMultiStream(self): # Test BZ2File.read() on a multi-stream archive where a stream # boundary coincides with the end of the raw read buffer. - buffer_size = _compression.BUFFER_SIZE - _compression.BUFFER_SIZE = len(self.DATA) + buffer_size = _streams.BUFFER_SIZE + _streams.BUFFER_SIZE = len(self.DATA) try: self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT * 5) finally: - _compression.BUFFER_SIZE = buffer_size + _streams.BUFFER_SIZE = buffer_size def testReadTrailingJunk(self): self.createTempFile(suffix=self.BAD_DATA) @@ -184,7 +184,7 @@ def testPeek(self): with BZ2File(self.filename) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testReadInto(self): @@ -730,8 +730,7 @@ def testOpenBytesFilename(self): self.assertEqual(f.read(), self.DATA) self.assertEqual(f.name, str_filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != 'Z:\\TEMP\\tmphoipjcen' def testOpenPathLikeFilename(self): filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: @@ -744,7 +743,7 @@ def testOpenPathLikeFilename(self): def testDecompressLimited(self): """Decompressed data buffering should be limited""" bomb = bz2.compress(b'\0' * int(2e6), compresslevel=9) - self.assertLess(len(bomb), _compression.BUFFER_SIZE) + self.assertLess(len(bomb), _streams.BUFFER_SIZE) decomp = BZ2File(BytesIO(bomb)) self.assertEqual(decomp.read(1), b'\0') @@ -770,7 +769,7 @@ def testPeekBytesIO(self): with BZ2File(bio) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testWriteBytesIO(self): @@ -1190,8 +1189,6 @@ def test_encoding_error_handler(self): as f: self.assertEqual(f.read(), "foobar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = self.TEXT.decode("ascii") diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index df102fe1986..7ade4271b7a 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -987,10 +987,11 @@ def assertFailure(self, *args): self.assertCLIFails(*args) self.assertCmdFails(*args) + @support.force_not_colorized def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() @@ -1097,7 +1098,7 @@ def test_option_type(self): output = run('--type', 'text', '2004') self.assertEqual(output, conv(result_2004_text)) output = run('--type', 'html', '2004') - self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 29215f06002..7420f289b16 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +from test import support from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -134,6 +135,7 @@ def __%s__(self, *args): AllTests = type("AllTests", (object,), d) del d, statictests, method, method_template +@support.thread_unsafe("callLst is shared between threads") class ClassTests(unittest.TestCase): def setUp(self): callLst[:] = [] @@ -554,7 +556,9 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) - @unittest.skip("TODO: RUSTPYTHON, segmentation fault") + @unittest.skip("TODO: RUSTPYTHON; segmentation fault") + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 @@ -570,8 +574,7 @@ class A: else: self.fail("Failed to raise RecursionError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testForExceptionsRaisedInInstanceGetattr2(self): # Tests for exceptions raised in instance_getattr2(). @@ -611,7 +614,6 @@ def assertNotOrderable(self, a, b): with self.assertRaises(TypeError): a >= b - @unittest.skip("TODO: RUSTPYTHON; unstable result") def testHashComparisonOfMethods(self): # Test comparison and hash of methods class A: @@ -689,8 +691,7 @@ class A: with self.assertRaisesRegex(AttributeError, error_msg): del A.x - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testObjectAttributeAccessErrorMessages(self): class A: pass @@ -740,8 +741,6 @@ def __setattr__(self, name, value) -> None: with self.assertRaisesRegex(AttributeError, error_msg): del B().z - # TODO: RUSTPYTHON - @unittest.expectedFailure def testConstructorErrorMessages(self): # bpo-31506: Improves the error message logic for object_new & object_init @@ -846,12 +845,28 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) -try: - from _testinternalcapi import has_inline_values -except ImportError: - has_inline_values = None + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + + +# from _testinternalcapi import has_inline_values # XXX: RUSTPYTHON + +Py_TPFLAGS_INLINE_VALUES = (1 << 2) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) + +class NoManagedDict: + __slots__ = ('a',) -Py_TPFLAGS_MANAGED_DICT = (1 << 2) class Plain: pass @@ -866,38 +881,55 @@ def __init__(self): self.d = 4 +class VarSizedSubclass(tuple): + pass + + class TestInlineValues(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flags(self): - self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) - self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name 'has_inline_values' is not defined. + def test_no_flags_for_slots_class(self): + flags = NoManagedDict.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, 0) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(NoManagedDict())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_both_flags_for_regular_class(self): + for cls in (Plain, WithAttrs): + with self.subTest(cls=cls.__name__): + flags = cls.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_INLINE_VALUES) + self.assertTrue(has_inline_values(cls())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_managed_dict_only_for_varsized_subclass(self): + flags = VarSizedSubclass.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(VarSizedSubclass())) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_has_inline_values(self): c = Plain() self.assertTrue(has_inline_values(c)) del c.__dict__ self.assertFalse(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instances(self): self.assertTrue(has_inline_values(Plain())) self.assertTrue(has_inline_values(WithAttrs())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_inspect_dict(self): for cls in (Plain, WithAttrs): c = cls() c.__dict__ self.assertTrue(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_update_dict(self): d = { "e": 5, "f": 6 } for cls in (Plain, WithAttrs): @@ -914,8 +946,7 @@ def check_100(self, obj): for i in range(100): self.assertEqual(getattr(obj, f"a{i}"), i) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_many_attributes(self): class C: pass c = C() @@ -926,8 +957,7 @@ class C: pass c = C() self.assertTrue(has_inline_values(c)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_many_attributes_with_dict(self): class C: pass c = C() @@ -948,8 +978,7 @@ def __init__(self): obj.foo = None # Aborted here self.assertEqual(obj.__dict__, {"foo":None}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_store_attr_deleted_dict(self): class Foo: pass @@ -959,8 +988,7 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_rematerialize_object_dict(self): # gh-121860: rematerializing an object's managed dictionary after it # had been deleted caused a crash. @@ -979,7 +1007,7 @@ class Bar: pass self.assertIsInstance(f, Bar) self.assertEqual(f.__dict__, {}) - @unittest.skip("TODO: RUSTPYTHON, unexpectedly long runtime") + @unittest.skip("TODO: RUSTPYTHON; unexpectedly long runtime") def test_store_attr_type_cache(self): """Verifies that the type cache doesn't provide a value which is inconsistent from the dict.""" @@ -1030,5 +1058,6 @@ def __init__(self): self.assertFalse(out, msg=out.decode('utf-8')) self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 44f1b2da638..a96a5780b31 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -276,7 +276,6 @@ def test_cmath_matches_math(self): self.rAssertAlmostEqual(math.log(v, base), z.real) self.assertEqual(0., z.imag) - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def test_specific_values(self): # Some tests need to be skipped on ancient OS X versions. @@ -530,13 +529,11 @@ def testTanhSign(self): # log1p function; If that system function doesn't respect the sign # of zero, then atan and atanh will also have difficulties with # the sign of complex zeros. - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def testAtanSign(self): for z in complex_zeros: self.assertComplexesAreIdentical(cmath.atan(z), z) - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def testAtanhSign(self): for z in complex_zeros: @@ -583,7 +580,6 @@ def test_complex_near_zero(self): self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03) self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_complex_special(self): self.assertIsNotClose(INF, INF*1j) self.assertIsNotClose(INF*1j, INF) diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py index 46ec82b7049..dbfec42fc21 100644 --- a/Lib/test/test_cmd.py +++ b/Lib/test/test_cmd.py @@ -11,9 +11,15 @@ import io import textwrap from test import support -from test.support.import_helper import import_module +from test.support.import_helper import ensure_lazy_imports, import_module from test.support.pty_helper import run_pty +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("cmd", {"inspect", "string"}) + + class samplecmdclass(cmd.Cmd): """ Instance the sampleclass: @@ -289,6 +295,30 @@ def do_tab_completion_test(self, args): self.assertIn(b'ab_completion_test', output) self.assertIn(b'tab completion success', output) + def test_bang_completion_without_do_shell(self): + script = textwrap.dedent(""" + import cmd + class simplecmd(cmd.Cmd): + def completedefault(self, text, line, begidx, endidx): + return ["hello"] + + def default(self, line): + if line.replace(" ", "") == "!hello": + print('tab completion success') + else: + print('tab completion failure') + return True + + simplecmd().cmdloop() + """) + + # '! h' or '!h' and complete 'ello' to 'hello' + for input in [b"! h\t\n", b"!h\t\n"]: + with self.subTest(input=input): + output = run_pty(script, input) + self.assertIn(b'hello', output) + self.assertIn(b'tab completion success', output) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 833dc6b15d8..f977b97bcd3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -225,8 +225,6 @@ def test_repl_stderr_flush(self): def test_repl_stderr_flush_separate_stderr(self): self.check_repl_stderr_flush(True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_script(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -249,8 +247,6 @@ def test_script_abspath(self): script_dir, None, importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_script_compiled(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -413,8 +409,6 @@ def test_issue8202(self): script_name, script_name, script_dir, 'test_pkg', importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8202_dash_c_file_ignored(self): # Make sure a "-c" file in the current directory # does not alter the value of sys.path[0] @@ -554,8 +548,6 @@ def test_dash_m_main_traceback(self): self.assertIn(b'Exception in __main__ module', err) self.assertIn(b'Traceback', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep_409_verbiage(self): # Make sure PEP 409 syntax properly suppresses # the context of an exception @@ -617,8 +609,6 @@ def test_issue20500_exit_with_exception_value(self): text = stderr.decode('ascii') self.assertEqual(text.rstrip(), "some text") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntaxerror_unindented_caret_position(self): script = "1 + 1 = 2\n" with os_helper.temp_dir() as script_dir: @@ -628,8 +618,6 @@ def test_syntaxerror_unindented_caret_position(self): # Confirm that the caret is located under the '=' sign self.assertIn("\n ^^^^^\n", text) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntaxerror_indented_caret_position(self): script = textwrap.dedent("""\ if True: @@ -715,8 +703,6 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 804cce1dba4..f2ef233a59a 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -425,8 +425,6 @@ def test_co_positions_artificial_instructions(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges(self): # Make sure that if `-X no_debug_ranges` is used, there is # minimal debug info @@ -442,8 +440,6 @@ def f(): """) assert_python_ok('-X', 'no_debug_ranges', '-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges_env(self): # Same as above but using the environment variable opt out. code = textwrap.dedent(""" diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 5ac17ef16ea..39d85d46274 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,20 +1,17 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack from unittest import mock +from test.support import force_not_colorized_test_class from test.support import import_helper - code = import_helper.import_module('code') -class TestInteractiveConsole(unittest.TestCase): - - def setUp(self): - self.console = code.InteractiveConsole() - self.mock_sys() +class MockSys: def mock_sys(self): "Mock system environment for InteractiveConsole" @@ -32,21 +29,58 @@ def mock_sys(self): del self.sysmod.ps1 del self.sysmod.ps2 + +@force_not_colorized_test_class +class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None + + def setUp(self): + self.console = code.InteractiveConsole() + self.mock_sys() + def test_ps1(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps1, '>>> ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('>>> ', output) + self.assertNotHasAttr(self.sysmod, 'ps1') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.sysmod.ps1 = 'custom1> ' self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom1> ', output) self.assertEqual(self.sysmod.ps1, 'custom1> ') def test_ps2(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps2, '... ') - self.sysmod.ps1 = 'custom2> ' + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('... ', output) + self.assertNotHasAttr(self.sysmod, 'ps2') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] + self.sysmod.ps2 = 'custom2> ' self.console.interact() - self.assertEqual(self.sysmod.ps1, 'custom2> ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom2> ', output) + self.assertEqual(self.sysmod.ps2, 'custom2> ') def test_console_stderr(self): self.infunc.side_effect = ["'antioch'", "", EOFError('Finished')] @@ -58,21 +92,154 @@ def test_console_stderr(self): raise AssertionError("no console stdout") def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - 'IndentationError: unexpected indentation'] + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: ' + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertStartsWith(output, 'UnicodeEncodeError: ') + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + 'IndentationError: unexpected indent\n'] + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) + + def test_sysexcepthook_crashing_doesnt_close_repl(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + self.sysmod.excepthook = 1 + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_BaseException(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + s = "not so fast" + def raise_base(*args, **kwargs): + raise BaseException(s) + self.sysmod.excepthook = raise_base + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("not so fast"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_SystemExit_gets_through(self): + self.infunc.side_effect = ["1/0"] + def raise_base(*args, **kwargs): + raise SystemExit + self.sysmod.excepthook = raise_base + with self.assertRaises(SystemExit): + self.console.interact() def test_banner(self): # with banner @@ -115,8 +282,7 @@ def test_exit_msg(self): expected = message + '\n' self.assertEqual(err_msg, ['write', (expected,), {}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_cause_tb(self): self.infunc.side_effect = ["raise ValueError('') from AttributeError", EOFError('Finished')] @@ -132,9 +298,12 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", EOFError('Finished')] @@ -152,6 +321,28 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + +class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole(local_exit=True) + self.mock_sys() + + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") + def test_exit(self): + # default exit message + self.infunc.side_effect = ["exit()"] + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) if __name__ == "__main__": diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 9ca02cea351..763146c94fc 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -281,8 +281,6 @@ def handler2(exc): b"g[<252><223>]" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_longstrings(self): # test long strings to check for memory overflow problems errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", @@ -684,8 +682,6 @@ def test_badandgoodsurrogateescapeexceptions(self): ("\udc80", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodsurrogatepassexceptions(self): surrogatepass_errors = codecs.lookup_error('surrogatepass') # "surrogatepass" complains about a non-exception passed in diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py new file mode 100644 index 00000000000..af32f624d81 --- /dev/null +++ b/Lib/test/test_codecencodings_cn.py @@ -0,0 +1,100 @@ +# +# test_codecencodings_cn.py +# Codec encoding tests for PRC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb2312") +class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb2312' + tstring = multibytecodec_support.load_teststring('gb2312') + codectests = ( + # invalid bytes + (b"abc\x81\x81\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x81\x81\xc1\xc4", "ignore", "abc\u804a"), + (b"\xc1\x64", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gbk") +class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gbk' + tstring = multibytecodec_support.load_teststring('gbk') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"\x83\x34\x83\x31", "strict", None), + ("\u30fb", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb18030") +class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb18030' + tstring = multibytecodec_support.load_teststring('gb18030') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"), + ("\u30fb", "strict", b"\x819\xa79"), + (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), + (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), + (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), + ) + has_iso10646 = True + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: hz") +class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'hz' + tstring = multibytecodec_support.load_teststring('hz') + codectests = ( + # test '~\n' (3 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~{<:Ky2;S{#,~}~\n' + b'~{NpJ)l6HK!#~}Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # test '~\n' (4 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~\n' + b'~{<:Ky2;S{#,NpJ)l6HK!#~}~\n' + b'Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # invalid bytes + (b'ab~cd', 'replace', 'ab\uFFFDcd'), + (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), + (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), + (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py new file mode 100644 index 00000000000..b64d19bca91 --- /dev/null +++ b/Lib/test/test_codecencodings_hk.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_hk.py +# Codec encoding tests for HongKong encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5hkscs") +class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5hkscs' + tstring = multibytecodec_support.load_teststring('big5hkscs') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py new file mode 100644 index 00000000000..fe97aa6977e --- /dev/null +++ b/Lib/test/test_codecencodings_iso2022.py @@ -0,0 +1,92 @@ +# Codec encoding tests for ISO 2022 encodings. + +from test import multibytecodec_support +import unittest + +COMMON_CODEC_TESTS = ( + # invalid bytes + (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), + (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), + (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp") +class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2") +class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'abdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_3") +class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_3' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(O\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(O\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'), + ('ab\u4FF1def', 'replace', b'ab?def'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2004") +class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2004' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(Q\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(Q\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'), + ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_kr") +class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_kr' + tstring = multibytecodec_support.load_teststring('iso2022_kr') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + + # iso2022_kr.txt cannot be used to test "chunk coding": the escape + # sequence is only written on the first line + @unittest.skip('iso2022_kr.txt cannot be used to test "chunk coding"') + def test_chunkcoding(self): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py new file mode 100644 index 00000000000..f78ae229f84 --- /dev/null +++ b/Lib/test/test_codecencodings_jp.py @@ -0,0 +1,133 @@ +# +# test_codecencodings_jp.py +# Codec encoding tests for Japanese encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp932") +class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp932' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = ( + # invalid bytes + (b"abc\x81\x00\x81\x00\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"), + (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"), + (b"abc\x81\x00\x82\x84", "ignore", "abc\x00\uff44"), + (b"ab\xEBxy", "replace", "ab\uFFFDxy"), + (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"), + (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'), + # sjis vs cp932 + (b"\\\x7e", "replace", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"), + ) + +euc_commontests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), + (b"abc\xc8", "strict", None), + (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"), + (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"), + (b"\xc1\x64", "strict", None), + (b"\xa1\xc0", "strict", "\uff3c"), + (b"\xa1\xc0\\", "strict", "\uff3c\\"), + (b"\x8eXY", "replace", "\ufffdXY"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jis_2004") +class Test_EUC_JIS_2004(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jis_2004' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jisx0213") +class Test_EUC_JISX0213(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jisx0213' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jp") +class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jp' + tstring = multibytecodec_support.load_teststring('euc_jp') + codectests = euc_commontests + ( + ("\xa5", "strict", b"\x5c"), + ("\u203e", "strict", b"\x7e"), + ) + +shiftjis_commonenctests = ( + (b"abc\x80\x80\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis") +class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + (b"\\\x7e", "strict", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"), + (b"abc\x81\x39", "replace", "abc\ufffd9"), + (b"abc\xEA\xFC", "replace", "abc\ufffd\ufffd"), + (b"abc\xFF\x58", "replace", "abc\ufffdX"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis_2004") +class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis_2004' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"\\\x7e", "strict", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"), + (b"abc\xEA\xFC", "strict", "abc\u64bf"), + (b"\x81\x39xy", "replace", "\ufffd9xy"), + (b"\xFF\x58xy", "replace", "\ufffdXxy"), + (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"), + (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"), + (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jisx0213") +class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jisx0213' + tstring = multibytecodec_support.load_teststring('shift_jisx0213') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + # sjis vs cp932 + (b"\\\x7e", "replace", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py new file mode 100644 index 00000000000..aee124007ae --- /dev/null +++ b/Lib/test/test_codecencodings_kr.py @@ -0,0 +1,72 @@ +# +# test_codecencodings_kr.py +# Codec encoding tests for ROK encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp949") +class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp949' + tstring = multibytecodec_support.load_teststring('cp949') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_kr") +class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'euc_kr' + tstring = multibytecodec_support.load_teststring('euc_kr') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + + # composed make-up sequence errors + (b"\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"), + (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'), + (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None), + (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'), + (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'), + (b"\xc1\xc4", "strict", "\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: johab") +class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'johab' + tstring = multibytecodec_support.load_teststring('johab') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\ucd27"), + (b"\xD8abc", "replace", "\uFFFDabc"), + (b"\xD8\xFFabc", "replace", "\uFFFD\uFFFDabc"), + (b"\x84bxy", "replace", "\uFFFDbxy"), + (b"\x8CBxy", "replace", "\uFFFDBxy"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py new file mode 100644 index 00000000000..ca56b23234e --- /dev/null +++ b/Lib/test/test_codecencodings_tw.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_tw.py +# Codec encoding tests for ROC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5") +class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5' + tstring = multibytecodec_support.load_teststring('big5') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py new file mode 100644 index 00000000000..de254c6f767 --- /dev/null +++ b/Lib/test/test_codecmaps_cn.py @@ -0,0 +1,38 @@ +# +# test_codecmaps_cn.py +# Codec mapping tests for PRC encodings +# + +from test import multibytecodec_support +import unittest + +class TestGB2312Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb2312' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-CN.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb2312 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGBKMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gbk' + mapfileurl = 'http://www.pythontest.net/unicode/CP936.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gbk + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGB18030Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb18030' + mapfileurl = 'http://www.pythontest.net/unicode/gb-18030-2000.xml' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb18030 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py new file mode 100644 index 00000000000..f02cf486f76 --- /dev/null +++ b/Lib/test/test_codecmaps_hk.py @@ -0,0 +1,19 @@ +# +# test_codecmaps_hk.py +# Codec mapping tests for HongKong encodings +# + +from test import multibytecodec_support +import unittest + +class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5hkscs' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5HKSCS-2004.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5hkscs + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py new file mode 100644 index 00000000000..f2d52f99526 --- /dev/null +++ b/Lib/test/test_codecmaps_jp.py @@ -0,0 +1,84 @@ +# +# test_codecmaps_jp.py +# Codec mapping tests for Japanese encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP932Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp932' + mapfileurl = 'http://www.pythontest.net/unicode/CP932.TXT' + supmaps = [ + (b'\x80', '\u0080'), + (b'\xa0', '\uf8f0'), + (b'\xfd', '\uf8f1'), + (b'\xfe', '\uf8f2'), + (b'\xff', '\uf8f3'), + ] + for i in range(0xa1, 0xe0): + supmaps.append((bytes([i]), chr(i+0xfec0))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_file(self): + return super().test_mapping_file() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_supplemental(self): + return super().test_mapping_supplemental() + + +class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jp' + mapfilename = 'EUC-JP.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JP.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jis' + mapfilename = 'SHIFTJIS.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFTJIS.TXT' + pass_enctest = [ + (b'\x81_', '\\'), + ] + pass_dectest = [ + (b'\\', '\xa5'), + (b'~', '\u203e'), + (b'\x81_', '\\'), + ] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis + def test_mapping_file(self): + return super().test_mapping_file() + +class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jisx0213' + mapfilename = 'EUC-JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jisx0213' + mapfilename = 'SHIFT_JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFT_JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py new file mode 100644 index 00000000000..b8376d36615 --- /dev/null +++ b/Lib/test/test_codecmaps_kr.py @@ -0,0 +1,49 @@ +# +# test_codecmaps_kr.py +# Codec mapping tests for ROK encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP949Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp949' + mapfileurl = 'http://www.pythontest.net/unicode/CP949.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp949 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestEUCKRMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_kr' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-KR.TXT' + + # A4D4 HANGUL FILLER indicates the begin of 8-bytes make-up sequence. + pass_enctest = [(b'\xa4\xd4', '\u3164')] + pass_dectest = [(b'\xa4\xd4', '\u3164')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_kr + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestJOHABMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'johab' + mapfileurl = 'http://www.pythontest.net/unicode/JOHAB.TXT' + # KS X 1001 standard assigned 0x5c as WON SIGN. + # But the early 90s is the only era that used johab widely, + # most software implements it as REVERSE SOLIDUS. + # So, we ignore the standard here. + pass_enctest = [(b'\\', '\u20a9')] + pass_dectest = [(b'\\', '\u20a9')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: johab + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py new file mode 100644 index 00000000000..4a1359ce7be --- /dev/null +++ b/Lib/test/test_codecmaps_tw.py @@ -0,0 +1,39 @@ +# +# test_codecmaps_tw.py +# Codec mapping tests for ROC encodings +# + +from test import multibytecodec_support +import unittest + +class TestBIG5Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestCP950Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp950' + mapfileurl = 'http://www.pythontest.net/unicode/CP950.TXT' + pass_enctest = [ + (b'\xa2\xcc', '\u5341'), + (b'\xa2\xce', '\u5345'), + ] + codectests = ( + (b"\xFFxy", "replace", "\ufffdxy"), + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_errorhandle(self): + return super().test_errorhandle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 0d3d8c9e2d6..fefd062eacb 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,12 +1,15 @@ import codecs import contextlib import copy +import importlib import io import pickle +import os import sys import unittest import encodings from unittest import mock +import warnings from test import support from test.support import os_helper @@ -20,13 +23,12 @@ except ImportError: _testinternalcapi = None -try: - import ctypes -except ImportError: - ctypes = None - SIZEOF_WCHAR_T = -1 -else: - SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) + +def codecs_open_no_warn(*args, **kwargs): + """Call codecs.open(*args, **kwargs) ignoring DeprecationWarning.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return codecs.open(*args, **kwargs) def coding_checker(self, coder): def check(input, expect): @@ -35,13 +37,13 @@ def check(input, expect): # On small versions of Windows like Windows IoT or Windows Nano Server not all codepages are present def is_code_page_present(cp): - from ctypes import POINTER, WINFUNCTYPE, WinDLL + from ctypes import POINTER, WINFUNCTYPE, WinDLL, Structure from ctypes.wintypes import BOOL, BYTE, WCHAR, UINT, DWORD MAX_LEADBYTES = 12 # 5 ranges, 2 bytes ea., 0 term. MAX_DEFAULTCHAR = 2 # single or double byte MAX_PATH = 260 - class CPINFOEXW(ctypes.Structure): + class CPINFOEXW(Structure): _fields_ = [("MaxCharSize", UINT), ("DefaultChar", BYTE*MAX_DEFAULTCHAR), ("LeadByte", BYTE*MAX_LEADBYTES), @@ -388,7 +390,6 @@ def test_bug1098990_b(self): ill_formed_sequence_replace = "\ufffd" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), @@ -464,7 +465,6 @@ class UTF32Test(ReadTest, unittest.TestCase): b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m' b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_only_one_bom(self): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream @@ -480,7 +480,6 @@ def test_only_one_bom(self): f = reader(s) self.assertEqual(f.read(), "spamspam") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) @@ -490,7 +489,6 @@ def test_badbom(self): f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeDecodeError, f.read) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -522,26 +520,22 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_32_decode(b'\x01', 'replace', True)) self.assertEqual(('', 1), codecs.utf_32_decode(b'\x01', 'ignore', True)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -552,40 +546,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -613,16 +578,13 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -630,40 +592,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -691,16 +624,13 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode, b"\xff", "strict", True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -708,34 +638,6 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_be_decode(encoded)[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_a(self): - return super().test_bug1098990_a() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1098990_b(self): - return super().test_bug1098990_b() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_mixed_readline_and_read(self): - return super().test_mixed_readline_and_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readlinequeue(self): - return super().test_readlinequeue() - class UTF16Test(ReadTest, unittest.TestCase): encoding = "utf-16" @@ -771,7 +673,6 @@ def test_badbom(self): f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeDecodeError, f.read) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -793,7 +694,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_16_decode(b'\x01', 'replace', True)) @@ -821,32 +721,27 @@ def test_bug691291(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, 'wb') as fp: fp.write(s) - with codecs.open(os_helper.TESTFN, 'r', + with codecs_open_no_warn(os_helper.TESTFN, 'r', encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) def test_invalid_modes(self): for mode in ('U', 'rU', 'r+U'): with self.assertRaises(ValueError) as cm: - codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) self.assertIn('invalid mode', str(cm.exception)) for mode in ('rt', 'wt', 'at', 'r+t'): with self.assertRaises(ValueError) as cm: - codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) self.assertIn("can't have text and binary mode at once", str(cm.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -866,7 +761,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -888,15 +782,10 @@ def test_nonbmp(self): self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding), "\U00010203") - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -916,7 +805,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -938,10 +826,6 @@ def test_nonbmp(self): self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding), "\U00010203") - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" @@ -987,7 +871,6 @@ def test_decode_error(self): self.assertEqual(data.decode(self.encoding, error_handler), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for @@ -1040,7 +923,6 @@ def test_incremental_errors(self): class UTF7Test(ReadTest, unittest.TestCase): encoding = "utf-7" - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): # Set D (directly encoded characters) set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1067,7 +949,6 @@ def test_ascii(self): b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU' b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( 'a+-b\x00c\x80d\u0100e\U00010000f', @@ -1107,7 +988,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xffb', '\ufffdb'), @@ -1138,7 +1018,6 @@ def test_errors(self): raw, 'strict', True) self.assertEqual(raw.decode('utf-7', 'replace'), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonbmp(self): self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') @@ -1154,7 +1033,6 @@ def test_nonbmp(self): self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding), '\u20ac\u20ac\U000104A0') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): tests = [ (b'a+2AE-b', 'a\ud801b'), @@ -1175,18 +1053,6 @@ def test_lone_surrogates(self): with self.subTest(raw=raw): self.assertEqual(raw.decode('utf-7', 'replace'), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_bug1175396(self): - return super().test_bug1175396() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class UTF16ExTest(unittest.TestCase): @@ -1310,7 +1176,6 @@ def test_raw(self): if b != b'\\': self.assertEqual(decode(b + b'0'), (b + b'0', 2)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1334,7 +1199,7 @@ def test_escape(self): check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_warnings(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1342,32 +1207,31 @@ def test_warnings(self): b = bytes([i]) if b not in b'abfnrtvx': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % i): + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, b"\\" + b) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % (i-32)): + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\8'"): + r'"\\8" is an invalid escape sequence'): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\\xfa'") as cm: + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\%o'" % i): + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, bytes([i & 0o377])) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\z'"): + r'"\\z" is an invalid escape sequence'): self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\501'"): + r'"\\501" is an invalid octal escape sequence'): self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: not raised by escape_decode def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -1999,9 +1863,9 @@ def test_all(self): def test_open(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for mode in ('w', 'r', 'r+', 'w+', 'a', 'a+'): - with self.subTest(mode), \ - codecs.open(os_helper.TESTFN, mode, 'ascii') as file: - self.assertIsInstance(file, codecs.StreamReaderWriter) + with self.subTest(mode), self.assertWarns(DeprecationWarning): + with codecs.open(os_helper.TESTFN, mode, 'ascii') as file: + self.assertIsInstance(file, codecs.StreamReaderWriter) def test_undefined(self): self.assertRaises(UnicodeError, codecs.encode, 'abc', 'undefined') @@ -2018,7 +1882,7 @@ def test_file_closes_if_lookup_error_raised(self): mock_open = mock.mock_open() with mock.patch('builtins.open', mock_open) as file: with self.assertRaises(LookupError): - codecs.open(os_helper.TESTFN, 'wt', 'invalid-encoding') + codecs_open_no_warn(os_helper.TESTFN, 'wt', 'invalid-encoding') file().close.assert_called() @@ -2291,7 +2155,7 @@ def test_basic(self): class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_basics(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: @@ -2411,7 +2275,7 @@ def test_basics_capi(self): self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2427,7 +2291,7 @@ def test_seek(self): data = reader.read() self.assertEqual(s, data) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_decode_args(self): for encoding in all_unicode_encodings: decoder = codecs.getdecoder(encoding) @@ -2435,7 +2299,7 @@ def test_bad_decode_args(self): if encoding not in ("idna", "punycode"): self.assertRaises(TypeError, decoder, 42) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) @@ -2447,7 +2311,7 @@ def test_encoding_map_type_initialized(self): table_type = type(cp1140.encoding_table) self.assertEqual(table_type, table_type) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" @@ -2458,7 +2322,6 @@ def test_decoder_state(self): class CharmapTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_string_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"), @@ -2514,7 +2377,6 @@ def test_decode_with_string_map(self): ("", len(allbytes)) ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2str_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", @@ -2631,7 +2493,6 @@ def test_decode_with_int2str_map(self): b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999} ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2int_map(self): a = ord('a') b = ord('b') @@ -2724,7 +2585,7 @@ def test_streamreaderwriter(self): class TypesTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'? def test_decode_unicode(self): # Most decoders don't accept unicode input decoders = [ @@ -2826,7 +2687,7 @@ def test_escape_decode(self): check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") - @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered def test_decode_warnings(self): decode = codecs.unicode_escape_decode check = coding_checker(self, decode) @@ -2834,30 +2695,30 @@ def test_decode_warnings(self): b = bytes([i]) if b not in b'abfnrtuvx': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % i): + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\%c'" % (i-32)): + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\8'"): + r'"\\8" is an invalid escape sequence'): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\\xfa'") as cm: + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\%o'" % i): + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, chr(i)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid escape sequence '\\z'"): + r'"\\z" is an invalid escape sequence'): self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) with self.assertWarnsRegex(DeprecationWarning, - r"invalid octal escape sequence '\\501'"): + r'"\\501" is an invalid octal escape sequence'): self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) def test_decode_errors(self): @@ -2876,7 +2737,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2916,14 +2776,10 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range def test_incremental_surrogatepass(self): return super().test_incremental_surrogatepass() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): encoding = "raw-unicode-escape" @@ -2977,7 +2833,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -3007,14 +2862,6 @@ def test_partial(self): ] ) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_incremental_surrogatepass(self): - return super().test_incremental_surrogatepass() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_readline(self): - return super().test_readline() - class EscapeEncodeTest(unittest.TestCase): @@ -3057,7 +2904,6 @@ def test_ascii(self): self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"), b"foo\x80bar") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_charmap(self): # bad byte: \xa5 is unmapped in iso-8859-3 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"), @@ -3072,7 +2918,6 @@ def test_latin1(self): class BomTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek0(self): data = "1234567890" tests = ("utf-16", @@ -3084,7 +2929,7 @@ def test_seek0(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for encoding in tests: # Check if the BOM is written only once - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.write(data) f.seek(0) @@ -3093,7 +2938,7 @@ def test_seek0(self): self.assertEqual(f.read(), data * 2) # Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data[0]) self.assertNotEqual(f.tell(), 0) f.seek(0) @@ -3102,7 +2947,7 @@ def test_seek0(self): self.assertEqual(f.read(), data) # (StreamWriter) Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data[0]) self.assertNotEqual(f.writer.tell(), 0) f.writer.seek(0) @@ -3112,7 +2957,7 @@ def test_seek0(self): # Check that the BOM is not written after a seek() at a position # different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.seek(f.tell()) f.write(data) @@ -3121,7 +2966,7 @@ def test_seek0(self): # (StreamWriter) Check that the BOM is not written after a seek() # at a position different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data) f.writer.seek(f.writer.tell()) f.writer.write(data) @@ -3152,7 +2997,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: @@ -3189,7 +3034,6 @@ def test_readline(self): sout = reader.readline() self.assertEqual(sout, b"\x80") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffer_api_usage(self): # We check all the transform codecs accept memoryview input # for encoding and decoding @@ -3252,7 +3096,6 @@ def test_binary_to_text_denylists_text_transforms(self): bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(zlib, "Requires zlib support") def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input @@ -3261,7 +3104,6 @@ def test_custom_zlib_error_is_noted(self): codecs.decode(b"hello", "zlib_codec") self.assertEqual(msg, failure.exception.__notes__[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: 'Error' object has no attribute '__notes__' def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input import binascii @@ -3279,6 +3121,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_mod = f"encodings.{value}" + self.assertIsNotNone(importlib.util.find_spec(codec_mod), + f"Codec module not found: {codec_mod}") + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") @@ -3342,7 +3191,6 @@ def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec raise self.obj_to_raise - @unittest.expectedFailure # TODO: RUSTPYTHON def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): self.obj_to_raise = obj_to_raise self.set_codec(self.raise_obj, self.raise_obj) @@ -3355,55 +3203,46 @@ def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): with self.assertNoted("decoding", exc_type, msg): codecs.decode(b"bytes input", self.codec_name) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_type(self): self.check_note(RuntimeError, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_value(self): msg = "This should be noted" self.check_note(RuntimeError(msg), msg) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_grandchild_subclass_exact_size(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): __slots__ = () self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_subclass_with_weakref_support(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): pass self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_init_override(self): class CustomInit(RuntimeError): def __init__(self): pass self.check_note(CustomInit, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_override(self): class CustomNew(RuntimeError): def __new__(cls): return super().__new__(cls) self.check_note(CustomNew, "") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance_attribute(self): msg = "This should be noted" exc = RuntimeError(msg) exc.attr = 1 self.check_note(exc, "^{}$".format(msg)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_str_arg(self): self.check_note(RuntimeError(1), "1") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_args(self): msg_re = r"^\('a', 'b', 'c'\)$" self.check_note(RuntimeError('a', 'b', 'c'), msg_re) @@ -3420,7 +3259,6 @@ def test_codec_lookup_failure(self): with self.assertRaisesRegex(LookupError, msg): codecs.decode(b"bytes input", self.codec_name) - @unittest.expectedFailure # TODO: RUSTPYTHON; def test_unflagged_non_text_codec_handling(self): # The stdlib non-text codecs are now marked so they're # pre-emptively skipped by the text model related methods @@ -3456,14 +3294,12 @@ def decode_to_bytes(*args, **kwds): class CodePageTest(unittest.TestCase): CP_UTF8 = 65001 - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', codecs.code_page_encode, 932, '\xff') @@ -3473,7 +3309,11 @@ def test_code_page_name(self): codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): - for raw, errors, expected in tests: + for raw, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: decoded = codecs.code_page_decode(cp, raw, errors, True) @@ -3490,8 +3330,21 @@ def check_decode(self, cp, tests): self.assertRaises(UnicodeDecodeError, codecs.code_page_decode, cp, raw, errors, True) + if altexpected is not None: + decoded = raw.decode(f'cp{cp}', errors) + self.assertEqual(decoded, altexpected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded, altexpected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, f'cp{cp}', errors) + def check_encode(self, cp, tests): - for text, errors, expected in tests: + for text, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: encoded = codecs.code_page_encode(cp, text, errors) @@ -3502,19 +3355,27 @@ def check_encode(self, cp, tests): '%a.encode("cp%s", %r)=%a != %a' % (text, cp, errors, encoded[0], expected)) self.assertEqual(encoded[1], len(text)) + + encoded = text.encode(f'cp{cp}', errors) + self.assertEqual(encoded, altexpected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded, altexpected)) else: self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) + self.assertRaises(UnicodeEncodeError, + text.encode, f'cp{cp}', errors) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + ('\uf8f3', 'strict', b'\xff'), # test error handlers ('\xff', 'strict', None), ('[\xff]', 'ignore', b'[]'), - ('[\xff]', 'replace', b'[y]'), + ('[\xff]', 'replace', b'[y]', b'[?]'), ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'namereplace', @@ -3528,19 +3389,18 @@ def test_cp932(self): (b'abc', 'strict', 'abc'), (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), # invalid bytes - (b'[\xff]', 'strict', None), - (b'[\xff]', 'ignore', '[]'), - (b'[\xff]', 'replace', '[\ufffd]'), - (b'[\xff]', 'backslashreplace', '[\\xff]'), - (b'[\xff]', 'surrogateescape', '[\udcff]'), - (b'[\xff]', 'surrogatepass', None), + (b'[\xff]', 'strict', None, '[\uf8f3]'), + (b'[\xff]', 'ignore', '[]', '[\uf8f3]'), + (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'), + (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'), + (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'), + (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp1252(self): self.check_encode(1252, ( ('abc', 'strict', b'abc'), @@ -3549,7 +3409,7 @@ def test_cp1252(self): # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), - ('\u0141', 'replace', b'L'), + ('\u0141', 'replace', b'L', b'?'), ('\udc98', 'surrogateescape', b'\x98'), ('\udc98', 'surrogatepass', None), )) @@ -3559,7 +3419,60 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cp708(self): + self.check_encode(708, ( + ('abc2%', 'strict', b'abc2%'), + ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'), + ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'), + ('\x9a\x9f', 'strict', b'\x9a\x9f'), + ('\u256b', 'strict', b'\xc0'), + # test error handlers + ('[\u0662]', 'strict', None), + ('[\u0662]', 'ignore', b'[]'), + ('[\u0662]', 'replace', b'[?]'), + ('\udca0', 'surrogateescape', b'\xa0'), + ('\udca0', 'surrogatepass', None), + )) + self.check_decode(708, ( + (b'abc2%', 'strict', 'abc2%'), + (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'), + (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'), + (b'\x9a\x9f', 'strict', '\x9a\x9f'), + (b'\xc0', 'strict', '\u256b'), + # test error handlers + (b'\xa0', 'strict', None), + (b'[\xa0]', 'ignore', '[]'), + (b'[\xa0]', 'replace', '[\ufffd]'), + (b'[\xa0]', 'backslashreplace', '[\\xa0]'), + (b'[\xa0]', 'surrogateescape', '[\udca0]'), + (b'[\xa0]', 'surrogatepass', None), + )) + + def test_cp20106(self): + self.check_encode(20106, ( + ('abc', 'strict', b'abc'), + ('\xa7\xc4\xdf', 'strict', b'@[~'), + # test error handlers + ('@', 'strict', None), + ('@', 'ignore', b''), + ('@', 'replace', b'?'), + ('\udcbf', 'surrogateescape', b'\xbf'), + ('\udcbf', 'surrogatepass', None), + )) + self.check_decode(20106, ( + (b'abc', 'strict', 'abc'), + (b'@[~', 'strict', '\xa7\xc4\xdf'), + (b'\xe1\xfe', 'strict', 'a\xdf'), + # test error handlers + (b'(\xbf)', 'strict', None), + (b'(\xbf)', 'ignore', '()'), + (b'(\xbf)', 'replace', '(\ufffd)'), + (b'(\xbf)', 'backslashreplace', '(\\xbf)'), + (b'(\xbf)', 'surrogateescape', '(\udcbf)'), + (b'(\xbf)', 'surrogatepass', None), + )) + + @unittest.expectedFailure # TODO: RUSTPYTHON; # TODO: RUSTPYTHON def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3580,7 +3493,6 @@ def test_cp_utf7(self): (b'[\xff]', 'strict', '[\xff]'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_encoding(self): self.check_decode(932, ( (b'\x84\xe9\x80', 'ignore', '\u9a3e'), @@ -3595,7 +3507,6 @@ def test_multibyte_encoding(self): ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), )) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_decode_flags(self): # Issue #36312: For some code pages (e.g. UTF-7) flags for # MultiByteToWideChar() must be set to 0. @@ -3615,7 +3526,6 @@ def test_code_page_decode_flags(self): self.assertEqual(codecs.code_page_decode(42, b'abc'), ('\uf061\uf062\uf063', 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) self.assertEqual(decoded, ('', 0)) @@ -3635,17 +3545,15 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - def test_mbcs_alias(self): - # Check that looking up our 'default' codepage will return - # mbcs when we don't have a more specific one available - code_page = 99_999 - name = f'cp{code_page}' - with mock.patch('_winapi.GetACP', return_value=code_page): - try: - codec = codecs.lookup(name) - self.assertEqual(codec.name, 'mbcs') - finally: - codecs.unregister(name) + def test_mbcs_code_page(self): + # Check that codec for the current Windows (ANSII) code page is + # always available. + try: + from _winapi import GetACP + except ImportError: + self.skipTest('requires _winapi.GetACP') + cp = GetACP() + codecs.lookup(f'cp{cp}') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) def test_large_input(self, size): @@ -3909,7 +3817,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) @@ -3978,7 +3886,6 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) def test_codecs_lookup(self): FOUND = (1, 2, 3, 4) NOT_FOUND = (None, None, None, None) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index c62e3748e6a..bbc46021406 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -30,8 +30,7 @@ def assertInvalid(self, str, symbol='single', is_syntax=1): except OverflowError: self.assertTrue(not is_syntax) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != at 0xc99532f80 file "", line 1> def test_valid(self): av = self.assertValid @@ -94,8 +93,7 @@ def test_valid(self): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != None def test_incomplete(self): ai = self.assertIncomplete @@ -282,13 +280,12 @@ def test_filename(self): self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename, compile("a = 1\n", "def", 'single').co_filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_warning(self): # Test that the warning is only returned once. with warnings_helper.check_warnings( ('"is" with \'str\' literal', SyntaxWarning), - ("invalid escape sequence", SyntaxWarning), + ('"\\\\e" is an invalid escape sequence', SyntaxWarning), ) as w: compile_command(r"'\e' is 0") self.assertEqual(len(w.warnings), 2) @@ -309,8 +306,7 @@ def test_incomplete_warning(self): self.assertIncomplete("'\\e' + (") self.assertEqual(w, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1 def test_invalid_warning(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') @@ -325,8 +321,6 @@ def assertSyntaxErrorMatches(self, code, message): with self.assertRaisesRegex(SyntaxError, message): compile_command(code, symbol='exec') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntax_errors(self): self.assertSyntaxErrorMatches( dedent("""\ diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 7b11601fc46..0ed51c9f1d8 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -262,7 +262,7 @@ def __contains__(self, key): d = c.new_child(b=20, c=30) self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_union_operators(self): cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4)) cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4)) @@ -469,7 +469,6 @@ def test_module_parameter(self): NT = namedtuple('NT', ['x', 'y'], module=collections) self.assertEqual(NT.__module__, collections) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance(self): Point = namedtuple('Point', 'x y') p = Point(11, 22) @@ -746,11 +745,11 @@ def validate_isinstance(self, abc, name): C = type('C', (object,), {'__hash__': None}) setattr(C, name, stub) self.assertIsInstance(C(), abc) - self.assertTrue(issubclass(C, abc)) + self.assertIsSubclass(C, abc) C = type('C', (object,), {'__hash__': None}) self.assertNotIsInstance(C(), abc) - self.assertFalse(issubclass(C, abc)) + self.assertNotIsSubclass(C, abc) def validate_comparison(self, instance): ops = ['lt', 'gt', 'le', 'ge', 'ne', 'or', 'and', 'xor', 'sub'] @@ -788,7 +787,6 @@ def _test_gen(): class TestOneTrickPonyABCs(ABCTestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_Awaitable(self): def gen(): yield @@ -817,12 +815,12 @@ def __await__(self): non_samples = [None, int(), gen(), object()] for x in non_samples: self.assertNotIsInstance(x, Awaitable) - self.assertFalse(issubclass(type(x), Awaitable), repr(type(x))) + self.assertNotIsSubclass(type(x), Awaitable) samples = [Bar(), MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -836,12 +834,11 @@ def __await__(self): class CoroLike: pass Coroutine.register(CoroLike) - self.assertTrue(isinstance(CoroLike(), Awaitable)) - self.assertTrue(issubclass(CoroLike, Awaitable)) + self.assertIsInstance(CoroLike(), Awaitable) + self.assertIsSubclass(CoroLike, Awaitable) CoroLike = None support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache - @unittest.expectedFailure # TODO: RUSTPYTHON def test_Coroutine(self): def gen(): yield @@ -870,12 +867,12 @@ def __await__(self): non_samples = [None, int(), gen(), object(), Bar()] for x in non_samples: self.assertNotIsInstance(x, Coroutine) - self.assertFalse(issubclass(type(x), Coroutine), repr(type(x))) + self.assertNotIsSubclass(type(x), Coroutine) samples = [MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -896,8 +893,8 @@ def close(self): pass def __await__(self): pass - self.assertTrue(isinstance(CoroLike(), Coroutine)) - self.assertTrue(issubclass(CoroLike, Coroutine)) + self.assertIsInstance(CoroLike(), Coroutine) + self.assertIsSubclass(CoroLike, Coroutine) class CoroLike: def send(self, value): @@ -906,15 +903,15 @@ def close(self): pass def __await__(self): pass - self.assertFalse(isinstance(CoroLike(), Coroutine)) - self.assertFalse(issubclass(CoroLike, Coroutine)) + self.assertNotIsInstance(CoroLike(), Coroutine) + self.assertNotIsSubclass(CoroLike, Coroutine) def test_Hashable(self): # Check some non-hashables non_samples = [bytearray(), list(), set(), dict()] for x in non_samples: self.assertNotIsInstance(x, Hashable) - self.assertFalse(issubclass(type(x), Hashable), repr(type(x))) + self.assertNotIsSubclass(type(x), Hashable) # Check some hashables samples = [None, int(), float(), complex(), @@ -924,14 +921,14 @@ def test_Hashable(self): ] for x in samples: self.assertIsInstance(x, Hashable) - self.assertTrue(issubclass(type(x), Hashable), repr(type(x))) + self.assertIsSubclass(type(x), Hashable) self.assertRaises(TypeError, Hashable) # Check direct subclassing class H(Hashable): def __hash__(self): return super().__hash__() self.assertEqual(hash(H()), 0) - self.assertFalse(issubclass(int, H)) + self.assertNotIsSubclass(int, H) self.validate_abstract_methods(Hashable, '__hash__') self.validate_isinstance(Hashable, '__hash__') @@ -939,13 +936,13 @@ def test_AsyncIterable(self): class AI: def __aiter__(self): return self - self.assertTrue(isinstance(AI(), AsyncIterable)) - self.assertTrue(issubclass(AI, AsyncIterable)) + self.assertIsInstance(AI(), AsyncIterable) + self.assertIsSubclass(AI, AsyncIterable) # Check some non-iterables non_samples = [None, object, []] for x in non_samples: self.assertNotIsInstance(x, AsyncIterable) - self.assertFalse(issubclass(type(x), AsyncIterable), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterable) self.validate_abstract_methods(AsyncIterable, '__aiter__') self.validate_isinstance(AsyncIterable, '__aiter__') @@ -955,13 +952,13 @@ def __aiter__(self): return self async def __anext__(self): raise StopAsyncIteration - self.assertTrue(isinstance(AI(), AsyncIterator)) - self.assertTrue(issubclass(AI, AsyncIterator)) + self.assertIsInstance(AI(), AsyncIterator) + self.assertIsSubclass(AI, AsyncIterator) non_samples = [None, object, []] # Check some non-iterables for x in non_samples: self.assertNotIsInstance(x, AsyncIterator) - self.assertFalse(issubclass(type(x), AsyncIterator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterator) # Similarly to regular iterators (see issue 10565) class AnextOnly: async def __anext__(self): @@ -974,7 +971,7 @@ def test_Iterable(self): non_samples = [None, 42, 3.14, 1j] for x in non_samples: self.assertNotIsInstance(x, Iterable) - self.assertFalse(issubclass(type(x), Iterable), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterable) # Check some iterables samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), @@ -984,13 +981,13 @@ def test_Iterable(self): ] for x in samples: self.assertIsInstance(x, Iterable) - self.assertTrue(issubclass(type(x), Iterable), repr(type(x))) + self.assertIsSubclass(type(x), Iterable) # Check direct subclassing class I(Iterable): def __iter__(self): return super().__iter__() self.assertEqual(list(I()), []) - self.assertFalse(issubclass(str, I)) + self.assertNotIsSubclass(str, I) self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') # Check None blocking @@ -998,22 +995,22 @@ class It: def __iter__(self): return iter([]) class ItBlocked(It): __iter__ = None - self.assertTrue(issubclass(It, Iterable)) - self.assertTrue(isinstance(It(), Iterable)) - self.assertFalse(issubclass(ItBlocked, Iterable)) - self.assertFalse(isinstance(ItBlocked(), Iterable)) + self.assertIsSubclass(It, Iterable) + self.assertIsInstance(It(), Iterable) + self.assertNotIsSubclass(ItBlocked, Iterable) + self.assertNotIsInstance(ItBlocked(), Iterable) def test_Reversible(self): # Check some non-reversibles non_samples = [None, 42, 3.14, 1j, set(), frozenset()] for x in non_samples: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some non-reversible iterables non_reversibles = [_test_gen(), (x for x in []), iter([]), reversed([])] for x in non_reversibles: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some reversible iterables samples = [bytes(), str(), tuple(), list(), OrderedDict(), OrderedDict().keys(), OrderedDict().items(), @@ -1022,11 +1019,11 @@ def test_Reversible(self): dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Reversible) - self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + self.assertIsSubclass(type(x), Reversible) # Check also Mapping, MutableMapping, and Sequence - self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) - self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) - self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + self.assertIsSubclass(Sequence, Reversible) + self.assertNotIsSubclass(Mapping, Reversible) + self.assertNotIsSubclass(MutableMapping, Reversible) # Check direct subclassing class R(Reversible): def __iter__(self): @@ -1034,17 +1031,17 @@ def __iter__(self): def __reversed__(self): return iter(list()) self.assertEqual(list(reversed(R())), []) - self.assertFalse(issubclass(float, R)) + self.assertNotIsSubclass(float, R) self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') # Check reversible non-iterable (which is not Reversible) class RevNoIter: def __reversed__(self): return reversed([]) class RevPlusIter(RevNoIter): def __iter__(self): return iter([]) - self.assertFalse(issubclass(RevNoIter, Reversible)) - self.assertFalse(isinstance(RevNoIter(), Reversible)) - self.assertTrue(issubclass(RevPlusIter, Reversible)) - self.assertTrue(isinstance(RevPlusIter(), Reversible)) + self.assertNotIsSubclass(RevNoIter, Reversible) + self.assertNotIsInstance(RevNoIter(), Reversible) + self.assertIsSubclass(RevPlusIter, Reversible) + self.assertIsInstance(RevPlusIter(), Reversible) # Check None blocking class Rev: def __iter__(self): return iter([]) @@ -1053,39 +1050,38 @@ class RevItBlocked(Rev): __iter__ = None class RevRevBlocked(Rev): __reversed__ = None - self.assertTrue(issubclass(Rev, Reversible)) - self.assertTrue(isinstance(Rev(), Reversible)) - self.assertFalse(issubclass(RevItBlocked, Reversible)) - self.assertFalse(isinstance(RevItBlocked(), Reversible)) - self.assertFalse(issubclass(RevRevBlocked, Reversible)) - self.assertFalse(isinstance(RevRevBlocked(), Reversible)) + self.assertIsSubclass(Rev, Reversible) + self.assertIsInstance(Rev(), Reversible) + self.assertNotIsSubclass(RevItBlocked, Reversible) + self.assertNotIsInstance(RevItBlocked(), Reversible) + self.assertNotIsSubclass(RevRevBlocked, Reversible) + self.assertNotIsInstance(RevRevBlocked(), Reversible) def test_Collection(self): # Check some non-collections non_collections = [None, 42, 3.14, 1j, lambda x: 2*x] for x in non_collections: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some non-collection iterables non_col_iterables = [_test_gen(), iter(b''), iter(bytearray()), (x for x in [])] for x in non_col_iterables: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some collections samples = [set(), frozenset(), dict(), bytes(), str(), tuple(), list(), dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Collection) - self.assertTrue(issubclass(type(x), Collection), repr(type(x))) + self.assertIsSubclass(type(x), Collection) # Check also Mapping, MutableMapping, etc. - self.assertTrue(issubclass(Sequence, Collection), repr(Sequence)) - self.assertTrue(issubclass(Mapping, Collection), repr(Mapping)) - self.assertTrue(issubclass(MutableMapping, Collection), - repr(MutableMapping)) - self.assertTrue(issubclass(Set, Collection), repr(Set)) - self.assertTrue(issubclass(MutableSet, Collection), repr(MutableSet)) - self.assertTrue(issubclass(Sequence, Collection), repr(MutableSet)) + self.assertIsSubclass(Sequence, Collection) + self.assertIsSubclass(Mapping, Collection) + self.assertIsSubclass(MutableMapping, Collection) + self.assertIsSubclass(Set, Collection) + self.assertIsSubclass(MutableSet, Collection) + self.assertIsSubclass(Sequence, Collection) # Check direct subclassing class Col(Collection): def __iter__(self): @@ -1096,13 +1092,13 @@ def __contains__(self, item): return False class DerCol(Col): pass self.assertEqual(list(iter(Col())), []) - self.assertFalse(issubclass(list, Col)) - self.assertFalse(issubclass(set, Col)) - self.assertFalse(issubclass(float, Col)) + self.assertNotIsSubclass(list, Col) + self.assertNotIsSubclass(set, Col) + self.assertNotIsSubclass(float, Col) self.assertEqual(list(iter(DerCol())), []) - self.assertFalse(issubclass(list, DerCol)) - self.assertFalse(issubclass(set, DerCol)) - self.assertFalse(issubclass(float, DerCol)) + self.assertNotIsSubclass(list, DerCol) + self.assertNotIsSubclass(set, DerCol) + self.assertNotIsSubclass(float, DerCol) self.validate_abstract_methods(Collection, '__len__', '__iter__', '__contains__') # Check sized container non-iterable (which is not Collection) etc. @@ -1115,12 +1111,12 @@ def __contains__(self, item): return False class ColNoCont: def __iter__(self): return iter([]) def __len__(self): return 0 - self.assertFalse(issubclass(ColNoIter, Collection)) - self.assertFalse(isinstance(ColNoIter(), Collection)) - self.assertFalse(issubclass(ColNoSize, Collection)) - self.assertFalse(isinstance(ColNoSize(), Collection)) - self.assertFalse(issubclass(ColNoCont, Collection)) - self.assertFalse(isinstance(ColNoCont(), Collection)) + self.assertNotIsSubclass(ColNoIter, Collection) + self.assertNotIsInstance(ColNoIter(), Collection) + self.assertNotIsSubclass(ColNoSize, Collection) + self.assertNotIsInstance(ColNoSize(), Collection) + self.assertNotIsSubclass(ColNoCont, Collection) + self.assertNotIsInstance(ColNoCont(), Collection) # Check None blocking class SizeBlock: def __iter__(self): return iter([]) @@ -1130,10 +1126,10 @@ class IterBlock: def __len__(self): return 0 def __contains__(self): return True __iter__ = None - self.assertFalse(issubclass(SizeBlock, Collection)) - self.assertFalse(isinstance(SizeBlock(), Collection)) - self.assertFalse(issubclass(IterBlock, Collection)) - self.assertFalse(isinstance(IterBlock(), Collection)) + self.assertNotIsSubclass(SizeBlock, Collection) + self.assertNotIsInstance(SizeBlock(), Collection) + self.assertNotIsSubclass(IterBlock, Collection) + self.assertNotIsInstance(IterBlock(), Collection) # Check None blocking in subclass class ColImpl: def __iter__(self): @@ -1144,15 +1140,15 @@ def __contains__(self, item): return False class NonCol(ColImpl): __contains__ = None - self.assertFalse(issubclass(NonCol, Collection)) - self.assertFalse(isinstance(NonCol(), Collection)) + self.assertNotIsSubclass(NonCol, Collection) + self.assertNotIsInstance(NonCol(), Collection) def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: self.assertNotIsInstance(x, Iterator) - self.assertFalse(issubclass(type(x), Iterator), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterator) samples = [iter(bytes()), iter(str()), iter(tuple()), iter(list()), iter(dict()), iter(set()), iter(frozenset()), @@ -1163,7 +1159,7 @@ def test_Iterator(self): ] for x in samples: self.assertIsInstance(x, Iterator) - self.assertTrue(issubclass(type(x), Iterator), repr(type(x))) + self.assertIsSubclass(type(x), Iterator) self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 @@ -1196,7 +1192,7 @@ def throw(self, typ, val=None, tb=None): pass iter(()), iter([]), NonGen1(), NonGen2(), NonGen3()] for x in non_samples: self.assertNotIsInstance(x, Generator) - self.assertFalse(issubclass(type(x), Generator), repr(type(x))) + self.assertNotIsSubclass(type(x), Generator) class Gen: def __iter__(self): return self @@ -1218,7 +1214,7 @@ def gen(): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsInstance(x, Generator) - self.assertTrue(issubclass(type(x), Generator), repr(type(x))) + self.assertIsSubclass(type(x), Generator) self.validate_abstract_methods(Generator, 'send', 'throw') # mixin tests @@ -1267,7 +1263,7 @@ def athrow(self, typ, val=None, tb=None): pass iter(()), iter([]), NonAGen1(), NonAGen2(), NonAGen3()] for x in non_samples: self.assertNotIsInstance(x, AsyncGenerator) - self.assertFalse(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncGenerator) class Gen: def __aiter__(self): return self @@ -1289,7 +1285,7 @@ async def gen(): for x in samples: self.assertIsInstance(x, AsyncIterator) self.assertIsInstance(x, AsyncGenerator) - self.assertTrue(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertIsSubclass(type(x), AsyncGenerator) self.validate_abstract_methods(AsyncGenerator, 'asend', 'athrow') def run_async(coro): @@ -1332,14 +1328,14 @@ def test_Sized(self): ] for x in non_samples: self.assertNotIsInstance(x, Sized) - self.assertFalse(issubclass(type(x), Sized), repr(type(x))) + self.assertNotIsSubclass(type(x), Sized) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), dict().values(), ] for x in samples: self.assertIsInstance(x, Sized) - self.assertTrue(issubclass(type(x), Sized), repr(type(x))) + self.assertIsSubclass(type(x), Sized) self.validate_abstract_methods(Sized, '__len__') self.validate_isinstance(Sized, '__len__') @@ -1350,14 +1346,14 @@ def test_Container(self): ] for x in non_samples: self.assertNotIsInstance(x, Container) - self.assertFalse(issubclass(type(x), Container), repr(type(x))) + self.assertNotIsSubclass(type(x), Container) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), ] for x in samples: self.assertIsInstance(x, Container) - self.assertTrue(issubclass(type(x), Container), repr(type(x))) + self.assertIsSubclass(type(x), Container) self.validate_abstract_methods(Container, '__contains__') self.validate_isinstance(Container, '__contains__') @@ -1369,7 +1365,7 @@ def test_Callable(self): ] for x in non_samples: self.assertNotIsInstance(x, Callable) - self.assertFalse(issubclass(type(x), Callable), repr(type(x))) + self.assertNotIsSubclass(type(x), Callable) samples = [lambda: None, type, int, object, len, @@ -1377,7 +1373,7 @@ def test_Callable(self): ] for x in samples: self.assertIsInstance(x, Callable) - self.assertTrue(issubclass(type(x), Callable), repr(type(x))) + self.assertIsSubclass(type(x), Callable) self.validate_abstract_methods(Callable, '__call__') self.validate_isinstance(Callable, '__call__') @@ -1385,16 +1381,16 @@ def test_direct_subclassing(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass - self.assertTrue(issubclass(C, B)) - self.assertFalse(issubclass(int, C)) + self.assertIsSubclass(C, B) + self.assertNotIsSubclass(int, C) def test_registration(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default - self.assertFalse(issubclass(C, B), B.__name__) + self.assertNotIsSubclass(C, B) B.register(C) - self.assertTrue(issubclass(C, B)) + self.assertIsSubclass(C, B) class WithSet(MutableSet): @@ -1425,7 +1421,7 @@ class TestCollectionABCs(ABCTestCase): def test_Set(self): for sample in [set, frozenset]: self.assertIsInstance(sample(), Set) - self.assertTrue(issubclass(sample, Set)) + self.assertIsSubclass(sample, Set) self.validate_abstract_methods(Set, '__contains__', '__iter__', '__len__') class MySet(Set): def __contains__(self, x): @@ -1506,9 +1502,9 @@ def __len__(self): def test_MutableSet(self): self.assertIsInstance(set(), MutableSet) - self.assertTrue(issubclass(set, MutableSet)) + self.assertIsSubclass(set, MutableSet) self.assertNotIsInstance(frozenset(), MutableSet) - self.assertFalse(issubclass(frozenset, MutableSet)) + self.assertNotIsSubclass(frozenset, MutableSet) self.validate_abstract_methods(MutableSet, '__contains__', '__iter__', '__len__', 'add', 'discard') @@ -1847,7 +1843,7 @@ def test_Set_hash_matches_frozenset(self): def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) - self.assertTrue(issubclass(sample, Mapping)) + self.assertIsSubclass(sample, Mapping) self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): @@ -1862,7 +1858,7 @@ def __iter__(self): def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) - self.assertTrue(issubclass(sample, MutableMapping)) + self.assertIsSubclass(sample, MutableMapping) self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') @@ -1896,12 +1892,12 @@ def test_MutableMapping_subclass(self): def test_Sequence(self): for sample in [tuple, list, bytes, str]: self.assertIsInstance(sample(), Sequence) - self.assertTrue(issubclass(sample, Sequence)) + self.assertIsSubclass(sample, Sequence) self.assertIsInstance(range(10), Sequence) - self.assertTrue(issubclass(range, Sequence)) + self.assertIsSubclass(range, Sequence) self.assertIsInstance(memoryview(b""), Sequence) - self.assertTrue(issubclass(memoryview, Sequence)) - self.assertTrue(issubclass(str, Sequence)) + self.assertIsSubclass(memoryview, Sequence) + self.assertIsSubclass(str, Sequence) self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): @@ -1961,25 +1957,25 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) + @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) - self.assertTrue(issubclass(sample, Buffer)) + self.assertIsSubclass(sample, Buffer) for sample in [str, list, tuple]: self.assertNotIsInstance(sample(), Buffer) - self.assertFalse(issubclass(sample, Buffer)) + self.assertNotIsSubclass(sample, Buffer) self.validate_abstract_methods(Buffer, '__buffer__') def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) - self.assertFalse(issubclass(sample, MutableSequence)) + self.assertNotIsSubclass(sample, MutableSequence) for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) - self.assertTrue(issubclass(sample, MutableSequence)) - self.assertTrue(issubclass(array.array, MutableSequence)) - self.assertFalse(issubclass(str, MutableSequence)) + self.assertIsSubclass(sample, MutableSequence) + self.assertIsSubclass(array.array, MutableSequence) + self.assertNotIsSubclass(str, MutableSequence) self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') @@ -2033,7 +2029,7 @@ def insert(self, index, value): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): @@ -2071,8 +2067,8 @@ def test_basics(self): self.assertEqual(c, Counter(a=3, b=2, c=1)) self.assertIsInstance(c, dict) self.assertIsInstance(c, Mapping) - self.assertTrue(issubclass(Counter, dict)) - self.assertTrue(issubclass(Counter, Mapping)) + self.assertIsSubclass(Counter, dict) + self.assertIsSubclass(Counter, Mapping) self.assertEqual(len(c), 3) self.assertEqual(sum(c.values()), 6) self.assertEqual(list(c.values()), [3, 2, 1]) @@ -2125,6 +2121,19 @@ def test_basics(self): self.assertEqual(c.setdefault('e', 5), 5) self.assertEqual(c['e'], 5) + def test_update_reentrant_add_clears_counter(self): + c = Counter() + key = object() + + class Evil(int): + def __add__(self, other): + c.clear() + return NotImplemented + + c[key] = Evil() + c.update([key]) + self.assertEqual(c[key], 1) + def test_init(self): self.assertEqual(list(Counter(self=42).items()), [('self', 42)]) self.assertEqual(list(Counter(iterable=42).items()), [('iterable', 42)]) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index e4d335a193d..337366b4014 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -108,7 +108,6 @@ def __getitem__(self, key): exec('z = a', g, d) self.assertEqual(d['z'], 12) - @unittest.skip("TODO: RUSTPYTHON; segmentation fault") def test_extended_arg(self): # default: 1000 * 2.5 = 2500 repetitions repeat = int(sys.getrecursionlimit() * 2.5) @@ -153,8 +152,6 @@ def test_indentation(self): pass""" compile(s, "", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure # This test is probably specific to CPython and may not generalize # to other implementations. We are trying to ensure that when # the first line of code starts after 256, correct line numbers @@ -929,8 +926,6 @@ def save_caller_frame(): func(save_caller_frame) self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lineno_after_no_code(self): def no_code1(): "doc string" @@ -1039,8 +1034,6 @@ def test_big_dict_literal(self): the_dict = "{" + ",".join(f"{x}:{x}" for x in range(dict_size)) + "}" self.assertEqual(len(eval(the_dict)), dict_size) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_redundant_jump_in_if_else_break(self): # Check if bytecode containing jumps that simply point to the next line # is generated around if-else-break style structures. See bpo-42615. @@ -1425,8 +1418,6 @@ def test_dict(self): def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_func_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") @@ -1436,8 +1427,6 @@ def test_func_kwargs(self): def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_meth_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("o.m(" + ", ".join(kwargs) + ")") @@ -1536,8 +1525,6 @@ def test_try_except_as(self): """ self.check_stack_size(snippet) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_try_except_star_qualified(self): snippet = """ try: @@ -1549,8 +1536,6 @@ def test_try_except_star_qualified(self): """ self.check_stack_size(snippet) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_try_except_star_as(self): snippet = """ try: @@ -1562,8 +1547,6 @@ def test_try_except_star_as(self): """ self.check_stack_size(snippet) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_try_except_star_finally(self): snippet = """ try: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index f84e67e9cf6..748a2ef7c7f 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -325,8 +325,6 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertEqual(mod_code_obj.co_filename, expected_in) self.assertIn(f'"{expected_in}"', os.fsdecode(err)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_only_one_worker(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -336,8 +334,6 @@ def test_ddir_multiple_workers(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_empty_only_one_worker(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -347,8 +343,6 @@ def test_ddir_empty_multiple_workers(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_only(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -408,8 +402,6 @@ def test_prepend_only(self): str(err, encoding=sys.getdefaultencoding()) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -730,7 +722,6 @@ def test_recursion_limit(self): self.assertCompiled(spamfn) self.assertCompiled(eggfn) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON hangs') @os_helper.skip_unless_symlink def test_symlink_loop(self): # Currently, compileall ignores symlinks to directories. @@ -887,8 +878,6 @@ def test_workers_available_cores(self, compile_dir): self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py new file mode 100644 index 00000000000..99a11e99d56 --- /dev/null +++ b/Lib/test/test_compiler_assemble.py @@ -0,0 +1,149 @@ +import dis +import io +import textwrap +import types + +from test.support.bytecode_helper import AssemblerTestCase + + +# Tests for the code-object creation stage of the compiler. + +class IsolatedAssembleTests(AssemblerTestCase): + + def complete_metadata(self, metadata, filename="myfile.py"): + if metadata is None: + metadata = {} + for key in ['name', 'qualname']: + metadata.setdefault(key, key) + for key in ['consts']: + metadata.setdefault(key, []) + for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + metadata.setdefault(key, {}) + for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: + metadata.setdefault(key, 0) + metadata.setdefault('firstlineno', 1) + metadata.setdefault('filename', filename) + return metadata + + def insts_to_code_object(self, insts, metadata): + metadata = self.complete_metadata(metadata) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) + + def assemble_test(self, insts, metadata, expected): + co = self.insts_to_code_object(insts, metadata) + self.assertIsInstance(co, types.CodeType) + + expected_metadata = {} + for key, value in metadata.items(): + if key == "fasthidden": + # not exposed on code object + continue + if isinstance(value, list): + expected_metadata[key] = tuple(value) + elif isinstance(value, dict): + expected_metadata[key] = tuple(value.keys()) + else: + expected_metadata[key] = value + + for key, value in expected_metadata.items(): + self.assertEqual(getattr(co, "co_" + key), value) + + f = types.FunctionType(co, {}) + for args, res in expected.items(): + self.assertEqual(f(*args), res) + + def test_simple_expr(self): + metadata = { + 'filename' : 'avg.py', + 'name' : 'avg', + 'qualname' : 'stats.avg', + 'consts' : {2 : 0}, + 'argcount' : 2, + 'varnames' : {'x' : 0, 'y' : 1}, + } + + # code for "return (x+y)/2" + insts = [ + ('RESUME', 0), + ('LOAD_FAST', 0, 1), # 'x' + ('LOAD_FAST', 1, 1), # 'y' + ('BINARY_OP', 0, 1), # '+' + ('LOAD_CONST', 0, 1), # 2 + ('BINARY_OP', 11, 1), # '/' + ('RETURN_VALUE', None, 1), + ] + expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} + self.assemble_test(insts, metadata, expected) + + + def test_expression_with_pseudo_instruction_load_closure(self): + + def mod_two(x): + def inner(): + return x + return inner() % 2 + + inner_code = mod_two.__code__.co_consts[0] + assert isinstance(inner_code, types.CodeType) + + metadata = { + 'filename' : 'mod_two.py', + 'name' : 'mod_two', + 'qualname' : 'nested.mod_two', + 'cellvars' : {'x' : 0}, + 'consts': {None: 0, inner_code: 1, 2: 2}, + 'argcount' : 1, + 'varnames' : {'x' : 0}, + } + + instructions = [ + ('RESUME', 0,), + ('LOAD_CLOSURE', 0, 1), + ('BUILD_TUPLE', 1, 1), + ('LOAD_CONST', 1, 1), + ('MAKE_FUNCTION', None, 2), + ('SET_FUNCTION_ATTRIBUTE', 8, 2), + ('PUSH_NULL', None, 1), + ('CALL', 0, 2), # (lambda: x)() + ('LOAD_CONST', 2, 2), # 2 + ('BINARY_OP', 6, 2), # % + ('RETURN_VALUE', None, 2) + ] + + expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} + self.assemble_test(instructions, metadata, expected) + + + def test_exception_table(self): + metadata = { + 'filename' : 'exc.py', + 'name' : 'exc', + 'consts' : {2 : 0}, + } + + # code for "try: pass\n except: pass" + insts = [ + ('RESUME', 0), + ('SETUP_FINALLY', 4), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ('SETUP_CLEANUP', 10), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ('COPY', 3), + ('POP_EXCEPT', None), + ('RERAISE', 1), + ] + co = self.insts_to_code_object(insts, metadata) + output = io.StringIO() + dis.dis(co, file=output) + exc_table = textwrap.dedent(""" + ExceptionTable: + L1 to L2 -> L2 [0] + L2 to L3 -> L3 [1] lasti + """) + self.assertEndsWith(output.getvalue(), exc_table) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py new file mode 100644 index 00000000000..d02937c84d9 --- /dev/null +++ b/Lib/test/test_compiler_codegen.py @@ -0,0 +1,163 @@ + +import textwrap +from test.support.bytecode_helper import CodegenTestCase + +# Tests for the code-generation stage of the compiler. +# Examine the un-optimized code generated from the AST. + +class IsolatedCodeGenTests(CodegenTestCase): + + def assertInstructionsMatch_recursive(self, insts, expected_insts): + expected_nested = [i for i in expected_insts if isinstance(i, list)] + expected_insts = [i for i in expected_insts if not isinstance(i, list)] + self.assertInstructionsMatch(insts, expected_insts) + self.assertEqual(len(insts.get_nested()), len(expected_nested)) + for n_insts, n_expected in zip(insts.get_nested(), expected_nested): + self.assertInstructionsMatch_recursive(n_insts, n_expected) + + def codegen_test(self, snippet, expected_insts): + import ast + a = ast.parse(snippet, "my_file.py", "exec") + insts = self.generate_code(a) + self.assertInstructionsMatch_recursive(insts, expected_insts) + + def test_if_expression(self): + snippet = "42 if True else 24" + false_lbl = self.Label() + expected = [ + ('RESUME', 0, 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_CONST', 0, 1), + ('TO_BOOL', 0, 1), + ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), + ('LOAD_CONST', 1, 1), # 42 + ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), + false_lbl, + ('LOAD_CONST', 2, 1), # 24 + exit_lbl, + ('POP_TOP', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ] + self.codegen_test(snippet, expected) + + def test_for_loop(self): + snippet = "for x in l:\n\tprint(x)" + false_lbl = self.Label() + expected = [ + ('RESUME', 0, 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_NAME', 0, 1), + ('GET_ITER', None, 1), + loop_lbl := self.Label(), + ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOP', None, 1, 1), + ('STORE_NAME', 1, 1), + ('LOAD_NAME', 2, 2), + ('PUSH_NULL', None, 2), + ('LOAD_NAME', 1, 2), + ('CALL', 1, 2), + ('POP_TOP', None), + ('JUMP', loop_lbl), + exit_lbl, + ('END_FOR', None), + ('POP_ITER', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + self.codegen_test(snippet, expected) + + def test_function(self): + snippet = textwrap.dedent(""" + def f(x): + return x + 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_FAST', 0), + ('LOAD_CONST', 42), + ('BINARY_OP', 0), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + ] + self.codegen_test(snippet, expected) + + def test_nested_functions(self): + snippet = textwrap.dedent(""" + def f(): + def h(): + return 12 + def g(): + x = 1 + y = 2 + z = 3 + u = 4 + return 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_CONST', 1), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 1), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + [ + ('RESUME', 0), + ('NOP', None), + ('LOAD_CONST', 12), + ('RETURN_VALUE', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ], + [ + ('RESUME', 0), + ('LOAD_CONST', 1), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('STORE_FAST', 1), + ('LOAD_CONST', 3), + ('STORE_FAST', 2), + ('LOAD_CONST', 4), + ('STORE_FAST', 3), + ('NOP', None), + ('LOAD_CONST', 42), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ], + ], + ] + self.codegen_test(snippet, expected) + + def test_syntax_error__return_not_in_function(self): + snippet = "return 42" + with self.assertRaisesRegex(SyntaxError, "'return' outside function") as cm: + self.codegen_test(snippet, None) + self.assertIsNone(cm.exception.text) + self.assertEqual(cm.exception.offset, 1) + self.assertEqual(cm.exception.end_offset, 10) diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py new file mode 100644 index 00000000000..b38bd38d338 --- /dev/null +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -0,0 +1,18 @@ +import os.path +import unittest +from test import support +from test.support import threading_helper + + +# Adjust if we ever have a platform with processes but not threads. +threading_helper.requires_working_threading(module=True) + + +if support.check_sanitizer(address=True, memory=True): + # gh-90791: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py new file mode 100644 index 00000000000..7442d3bee52 --- /dev/null +++ b/Lib/test/test_concurrent_futures/executor.py @@ -0,0 +1,162 @@ +import threading +import time +import unittest +import weakref +from concurrent import futures +from test import support +from test.support import Py_GIL_DISABLED + + +def mul(x, y): + return x * y + +def capture(*args, **kwargs): + return args, kwargs + + +class MyObject(object): + def my_method(self): + pass + + +def make_dummy_object(_): + return MyObject() + + +# Used in test_swallows_falsey_exceptions +def raiser(exception, msg='std'): + raise exception(msg) + + +class FalseyBoolException(Exception): + def __bool__(self): + return False + + +class FalseyLenException(Exception): + def __len__(self): + return 0 + + +class ExecutorTest: + # Executor.shutdown() and context manager usage is tested by + # ExecutorShutdownTest. + def test_submit(self): + future = self.executor.submit(pow, 2, 8) + self.assertEqual(256, future.result()) + + def test_submit_keyword(self): + future = self.executor.submit(mul, 2, y=8) + self.assertEqual(16, future.result()) + future = self.executor.submit(capture, 1, self=2, fn=3) + self.assertEqual(future.result(), ((1,), {'self': 2, 'fn': 3})) + with self.assertRaises(TypeError): + self.executor.submit(fn=capture, arg=1) + with self.assertRaises(TypeError): + self.executor.submit(arg=1) + + def test_map(self): + self.assertEqual( + list(self.executor.map(pow, range(10), range(10))), + list(map(pow, range(10), range(10)))) + + self.assertEqual( + list(self.executor.map(pow, range(10), range(10), chunksize=3)), + list(map(pow, range(10), range(10)))) + + def test_map_exception(self): + i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) + self.assertEqual(i.__next__(), (0, 1)) + self.assertEqual(i.__next__(), (0, 1)) + self.assertRaises(ZeroDivisionError, i.__next__) + + @support.requires_resource('walltime') + def test_map_timeout(self): + results = [] + try: + for i in self.executor.map(time.sleep, + [0, 0, 6], + timeout=5): + results.append(i) + except futures.TimeoutError: + pass + else: + self.fail('expected TimeoutError') + + self.assertEqual([None, None], results) + + def test_shutdown_race_issue12456(self): + # Issue #12456: race condition at shutdown where trying to post a + # sentinel in the call queue blocks (the queue is full while processes + # have exited). + self.executor.map(str, [2] * (self.worker_count + 1)) + self.executor.shutdown() + + @support.cpython_only + def test_no_stale_references(self): + # Issue #16284: check that the executors don't unnecessarily hang onto + # references. + my_object = MyObject() + my_object_collected = threading.Event() + def set_event(): + if Py_GIL_DISABLED: + # gh-117688 Avoid deadlock by setting the event in a + # background thread. The current thread may be in the middle + # of the my_object_collected.wait() call, which holds locks + # needed by my_object_collected.set(). + threading.Thread(target=my_object_collected.set).start() + else: + my_object_collected.set() + my_object_callback = weakref.ref(my_object, lambda obj: set_event()) + # Deliberately discarding the future. + self.executor.submit(my_object.my_method) + del my_object + + if Py_GIL_DISABLED: + # Due to biased reference counting, my_object might only be + # deallocated while the thread that created it runs -- if the + # thread is paused waiting on an event, it may not merge the + # refcount of the queued object. For that reason, we alternate + # between running the GC and waiting for the event. + wait_time = 0 + collected = False + while not collected and wait_time <= support.SHORT_TIMEOUT: + support.gc_collect() + collected = my_object_collected.wait(timeout=1.0) + wait_time += 1.0 + else: + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) + self.assertTrue(collected, + "Stale reference not collected within timeout.") + + def test_max_workers_negative(self): + for number in (0, -1): + with self.assertRaisesRegex(ValueError, + "max_workers must be greater " + "than 0"): + self.executor_type(max_workers=number) + + def test_free_reference(self): + # Issue #14406: Result iterator should not keep an internal + # reference to result objects. + for obj in self.executor.map(make_dummy_object, range(10)): + wr = weakref.ref(obj) + del obj + support.gc_collect() # For PyPy or other GCs. + + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if wr() is None: + break + + def test_swallows_falsey_exceptions(self): + # see gh-132063: Prevent exceptions that evaluate as falsey + # from being ignored. + # Recall: `x` is falsey if `len(x)` returns 0 or `bool(x)` returns False. + + msg = 'boolbool' + with self.assertRaisesRegex(FalseyBoolException, msg): + self.executor.submit(raiser, FalseyBoolException, msg).result() + + msg = 'lenlen' + with self.assertRaisesRegex(FalseyLenException, msg): + self.executor.submit(raiser, FalseyLenException, msg).result() diff --git a/Lib/test/test_concurrent_futures/test_as_completed.py b/Lib/test/test_concurrent_futures/test_as_completed.py new file mode 100644 index 00000000000..c90b0021d85 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_as_completed.py @@ -0,0 +1,118 @@ +import itertools +import time +import unittest +import weakref +from concurrent import futures +from concurrent.futures._base import ( + CANCELLED_AND_NOTIFIED, FINISHED, Future) + +from test import support + +from .util import ( + PENDING_FUTURE, RUNNING_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, + create_future, create_executor_tests, setup_module) + + +def mul(x, y): + return x * y + + +class AsCompletedTests: + def test_no_timeout(self): + future1 = self.executor.submit(mul, 2, 21) + future2 = self.executor.submit(mul, 7, 6) + + completed = set(futures.as_completed( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1, future2])) + self.assertEqual(set( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future1, future2]), + completed) + + def test_future_times_out(self): + """Test ``futures.as_completed`` timing out before + completing it's final future.""" + already_completed = {CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE} + + # Windows clock resolution is around 15.6 ms + short_timeout = 0.100 + for timeout in (0, short_timeout): + with self.subTest(timeout): + + completed_futures = set() + future = self.executor.submit(time.sleep, short_timeout * 10) + + try: + for f in futures.as_completed( + already_completed | {future}, + timeout + ): + completed_futures.add(f) + except futures.TimeoutError: + pass + + # Check that ``future`` wasn't completed. + self.assertEqual(completed_futures, already_completed) + + def test_duplicate_futures(self): + # Issue 20367. Duplicate futures should not raise exceptions or give + # duplicate responses. + # Issue #31641: accept arbitrary iterables. + future1 = self.executor.submit(time.sleep, 2) + completed = [ + f for f in futures.as_completed(itertools.repeat(future1, 3)) + ] + self.assertEqual(len(completed), 1) + + def test_free_reference_yielded_future(self): + # Issue #14406: Generator should not keep references + # to finished futures. + futures_list = [Future() for _ in range(8)] + futures_list.append(create_future(state=CANCELLED_AND_NOTIFIED)) + futures_list.append(create_future(state=FINISHED, result=42)) + + with self.assertRaises(futures.TimeoutError): + for future in futures.as_completed(futures_list, timeout=0): + futures_list.remove(future) + wr = weakref.ref(future) + del future + support.gc_collect() # For PyPy or other GCs. + self.assertIsNone(wr()) + + futures_list[0].set_result("test") + for future in futures.as_completed(futures_list): + futures_list.remove(future) + wr = weakref.ref(future) + del future + support.gc_collect() # For PyPy or other GCs. + self.assertIsNone(wr()) + if futures_list: + futures_list[0].set_result("test") + + def test_correct_timeout_exception_msg(self): + futures_list = [CANCELLED_AND_NOTIFIED_FUTURE, PENDING_FUTURE, + RUNNING_FUTURE, SUCCESSFUL_FUTURE] + + with self.assertRaises(futures.TimeoutError) as cm: + list(futures.as_completed(futures_list, timeout=0)) + + self.assertEqual(str(cm.exception), '2 (of 4) futures unfinished') + + +create_executor_tests(globals(), AsCompletedTests) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py new file mode 100644 index 00000000000..dcc1d68563a --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -0,0 +1,332 @@ +import contextlib +import queue +import signal +import sys +import time +import unittest +import unittest.mock +from pickle import PicklingError +from concurrent import futures +from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup + +from test import support + +from .util import ( + create_executor_tests, setup_module, + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin) + + +def _crash(delay=None): + """Induces a segfault.""" + if delay: + time.sleep(delay) + import faulthandler + faulthandler.disable() + faulthandler._sigsegv() + + +def _crash_with_data(data): + """Induces a segfault with dummy data in input.""" + _crash() + + +def _exit(): + """Induces a sys exit with exitcode 1.""" + sys.exit(1) + + +def _raise_error(Err): + """Function that raises an Exception in process.""" + raise Err() + + +def _raise_error_ignore_stderr(Err): + """Function that raises an Exception in process and ignores stderr.""" + import io + sys.stderr = io.StringIO() + raise Err() + + +def _return_instance(cls): + """Function that returns a instance of cls.""" + return cls() + + +class CrashAtPickle(object): + """Bad object that triggers a segfault at pickling time.""" + def __reduce__(self): + _crash() + + +class CrashAtUnpickle(object): + """Bad object that triggers a segfault at unpickling time.""" + def __reduce__(self): + return _crash, () + + +class ExitAtPickle(object): + """Bad object that triggers a process exit at pickling time.""" + def __reduce__(self): + _exit() + + +class ExitAtUnpickle(object): + """Bad object that triggers a process exit at unpickling time.""" + def __reduce__(self): + return _exit, () + + +class ErrorAtPickle(object): + """Bad object that triggers an error at pickling time.""" + def __reduce__(self): + from pickle import PicklingError + raise PicklingError("Error in pickle") + + +class ErrorAtUnpickle(object): + """Bad object that triggers an error at unpickling time.""" + def __reduce__(self): + from pickle import UnpicklingError + return _raise_error_ignore_stderr, (UnpicklingError, ) + + +class ExecutorDeadlockTest: + TIMEOUT = support.LONG_TIMEOUT + + def _fail_on_deadlock(self, executor): + # If we did not recover before TIMEOUT seconds, consider that the + # executor is in a deadlock state and forcefully clean all its + # composants. + import faulthandler + from tempfile import TemporaryFile + with TemporaryFile(mode="w+") as f: + faulthandler.dump_traceback(file=f) + f.seek(0) + tb = f.read() + for p in executor._processes.values(): + p.terminate() + # This should be safe to call executor.shutdown here as all possible + # deadlocks should have been broken. + executor.shutdown(wait=True) + print(f"\nTraceback:\n {tb}", file=sys.__stderr__) + self.fail(f"Executor deadlock:\n\n{tb}") + + + def _check_crash(self, error, func, *args, ignore_stderr=False): + # test for deadlock caused by crashes in a pool + self.executor.shutdown(wait=True) + + executor = self.executor_type( + max_workers=2, mp_context=self.get_context()) + res = executor.submit(func, *args) + + if ignore_stderr: + cm = support.captured_stderr() + else: + cm = contextlib.nullcontext() + + try: + with self.assertRaises(error): + with cm: + res.result(timeout=self.TIMEOUT) + except futures.TimeoutError: + # If we did not recover before TIMEOUT seconds, + # consider that the executor is in a deadlock state + self._fail_on_deadlock(executor) + executor.shutdown(wait=True) + + def test_error_at_task_pickle(self): + # Check problem occurring while pickling a task in + # the task_handler thread + self._check_crash(PicklingError, id, ErrorAtPickle()) + + def test_exit_at_task_unpickle(self): + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) + + def test_error_at_task_unpickle(self): + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, ErrorAtUnpickle()) + + def test_crash_at_task_unpickle(self): + # Check problem occurring while unpickling a task on workers + self._check_crash(BrokenProcessPool, id, CrashAtUnpickle()) + + def test_crash_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(BrokenProcessPool, _crash) + + def test_exit_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(SystemExit, _exit) + + def test_error_during_func_exec_on_worker(self): + # Check problem occurring during func execution on workers + self._check_crash(RuntimeError, _raise_error, RuntimeError) + + def test_crash_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(BrokenProcessPool, _return_instance, CrashAtPickle) + + def test_exit_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(SystemExit, _return_instance, ExitAtPickle) + + def test_error_during_result_pickle_on_worker(self): + # Check problem occurring while pickling a task result + # on workers + self._check_crash(PicklingError, _return_instance, ErrorAtPickle) + + def test_error_during_result_unpickle_in_result_handler(self): + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + + # Check problem occurring while unpickling a task in + # the result_handler thread + self._check_crash(BrokenProcessPool, + _return_instance, ErrorAtUnpickle, + ignore_stderr=True) + + def test_exit_during_result_unpickle_in_result_handler(self): + # Check problem occurring while unpickling a task in + # the result_handler thread + self._check_crash(BrokenProcessPool, _return_instance, ExitAtUnpickle) + + def test_shutdown_deadlock(self): + # Test that the pool calling shutdown do not cause deadlock + # if a worker fails after the shutdown call. + self.executor.shutdown(wait=True) + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + f = executor.submit(_crash, delay=.1) + executor.shutdown(wait=True) + with self.assertRaises(BrokenProcessPool): + f.result() + + def test_shutdown_deadlock_pickle(self): + # Test that the pool calling shutdown with wait=False does not cause + # a deadlock if a task fails at pickle after the shutdown call. + # Reported in bpo-39104. + self.executor.shutdown(wait=True) + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + + # Start the executor and get the executor_manager_thread to collect + # the threads and avoid dangling thread that should be cleaned up + # asynchronously. + executor.submit(id, 42).result() + executor_manager = executor._executor_manager_thread + + # Submit a task that fails at pickle and shutdown the executor + # without waiting + f = executor.submit(id, ErrorAtPickle()) + executor.shutdown(wait=False) + with self.assertRaises(PicklingError): + f.result() + + # Make sure the executor is eventually shutdown and do not leave + # dangling threads + executor_manager.join() + + def test_crash_big_data(self): + # Test that there is a clean exception instad of a deadlock when a + # child process crashes while some data is being written into the + # queue. + # https://github.com/python/cpython/issues/94777 + self.executor.shutdown(wait=True) + data = "a" * support.PIPE_MAX_SIZE + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + with self.assertRaises(BrokenProcessPool): + list(executor.map(_crash_with_data, [data] * 10)) + + executor.shutdown(wait=True) + + def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self): + # Issue #105829: The _ExecutorManagerThread wakeup pipe could + # fill up and block. See: https://github.com/python/cpython/issues/105829 + + # Lots of cargo culting while writing this test, apologies if + # something is really stupid... + + self.executor.shutdown(wait=True) + + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + import faulthandler + faulthandler.dump_traceback() + + raise RuntimeError("timed out while submitting jobs?") + + thread_run = futures.process._ExecutorManagerThread.run + def mock_run(self): + # Delay thread startup so the wakeup pipe can fill up and block + time.sleep(3) + thread_run(self) + + class MockWakeup(_ThreadWakeup): + """Mock wakeup object to force the wakeup to block""" + def __init__(self): + super().__init__() + self._dummy_queue = queue.Queue(maxsize=1) + + def wakeup(self): + self._dummy_queue.put(None, block=True) + super().wakeup() + + def clear(self): + super().clear() + try: + while True: + self._dummy_queue.get_nowait() + except queue.Empty: + pass + + with (unittest.mock.patch.object(futures.process._ExecutorManagerThread, + 'run', mock_run), + unittest.mock.patch('concurrent.futures.process._ThreadWakeup', + MockWakeup)): + with self.executor_type(max_workers=2, + mp_context=self.get_context()) as executor: + self.executor = executor # Allow clean up in fail_on_deadlock + + job_num = 100 + job_data = range(job_num) + + # Need to use sigalarm for timeout detection because + # Executor.submit is not guarded by any timeout (both + # self._work_ids.put(self._queue_count) and + # self._executor_manager_thread_wakeup.wakeup() might + # timeout, maybe more?). In this specific case it was + # the wakeup call that deadlocked on a blocking pipe. + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(int(self.TIMEOUT)) + self.assertEqual(job_num, len(list(executor.map(int, job_data)))) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + +create_executor_tests(globals(), ExecutorDeadlockTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_future.py b/Lib/test/test_concurrent_futures/test_future.py new file mode 100644 index 00000000000..4066ea1ee4b --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_future.py @@ -0,0 +1,291 @@ +import threading +import time +import unittest +from concurrent import futures +from concurrent.futures._base import ( + PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future) + +from test import support + +from .util import ( + PENDING_FUTURE, RUNNING_FUTURE, CANCELLED_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, + BaseTestCase, create_future, setup_module) + + +class FutureTests(BaseTestCase): + def test_done_callback_with_result(self): + callback_result = None + def fn(callback_future): + nonlocal callback_result + callback_result = callback_future.result() + + f = Future() + f.add_done_callback(fn) + f.set_result(5) + self.assertEqual(5, callback_result) + + def test_done_callback_with_exception(self): + callback_exception = None + def fn(callback_future): + nonlocal callback_exception + callback_exception = callback_future.exception() + + f = Future() + f.add_done_callback(fn) + f.set_exception(Exception('test')) + self.assertEqual(('test',), callback_exception.args) + + def test_done_callback_with_cancel(self): + was_cancelled = None + def fn(callback_future): + nonlocal was_cancelled + was_cancelled = callback_future.cancelled() + + f = Future() + f.add_done_callback(fn) + self.assertTrue(f.cancel()) + self.assertTrue(was_cancelled) + + def test_done_callback_raises(self): + with support.captured_stderr() as stderr: + raising_was_called = False + fn_was_called = False + + def raising_fn(callback_future): + nonlocal raising_was_called + raising_was_called = True + raise Exception('doh!') + + def fn(callback_future): + nonlocal fn_was_called + fn_was_called = True + + f = Future() + f.add_done_callback(raising_fn) + f.add_done_callback(fn) + f.set_result(5) + self.assertTrue(raising_was_called) + self.assertTrue(fn_was_called) + self.assertIn('Exception: doh!', stderr.getvalue()) + + def test_done_callback_already_successful(self): + callback_result = None + def fn(callback_future): + nonlocal callback_result + callback_result = callback_future.result() + + f = Future() + f.set_result(5) + f.add_done_callback(fn) + self.assertEqual(5, callback_result) + + def test_done_callback_already_failed(self): + callback_exception = None + def fn(callback_future): + nonlocal callback_exception + callback_exception = callback_future.exception() + + f = Future() + f.set_exception(Exception('test')) + f.add_done_callback(fn) + self.assertEqual(('test',), callback_exception.args) + + def test_done_callback_already_cancelled(self): + was_cancelled = None + def fn(callback_future): + nonlocal was_cancelled + was_cancelled = callback_future.cancelled() + + f = Future() + self.assertTrue(f.cancel()) + f.add_done_callback(fn) + self.assertTrue(was_cancelled) + + def test_done_callback_raises_already_succeeded(self): + with support.captured_stderr() as stderr: + def raising_fn(callback_future): + raise Exception('doh!') + + f = Future() + + # Set the result first to simulate a future that runs instantly, + # effectively allowing the callback to be run immediately. + f.set_result(5) + f.add_done_callback(raising_fn) + + self.assertIn('exception calling callback for', stderr.getvalue()) + self.assertIn('doh!', stderr.getvalue()) + + + def test_repr(self): + self.assertRegex(repr(PENDING_FUTURE), + '') + self.assertRegex(repr(RUNNING_FUTURE), + '') + self.assertRegex(repr(CANCELLED_FUTURE), + '') + self.assertRegex(repr(CANCELLED_AND_NOTIFIED_FUTURE), + '') + self.assertRegex( + repr(EXCEPTION_FUTURE), + '') + self.assertRegex( + repr(SUCCESSFUL_FUTURE), + '') + + def test_cancel(self): + f1 = create_future(state=PENDING) + f2 = create_future(state=RUNNING) + f3 = create_future(state=CANCELLED) + f4 = create_future(state=CANCELLED_AND_NOTIFIED) + f5 = create_future(state=FINISHED, exception=OSError()) + f6 = create_future(state=FINISHED, result=5) + + self.assertTrue(f1.cancel()) + self.assertEqual(f1._state, CANCELLED) + + self.assertFalse(f2.cancel()) + self.assertEqual(f2._state, RUNNING) + + self.assertTrue(f3.cancel()) + self.assertEqual(f3._state, CANCELLED) + + self.assertTrue(f4.cancel()) + self.assertEqual(f4._state, CANCELLED_AND_NOTIFIED) + + self.assertFalse(f5.cancel()) + self.assertEqual(f5._state, FINISHED) + + self.assertFalse(f6.cancel()) + self.assertEqual(f6._state, FINISHED) + + def test_cancelled(self): + self.assertFalse(PENDING_FUTURE.cancelled()) + self.assertFalse(RUNNING_FUTURE.cancelled()) + self.assertTrue(CANCELLED_FUTURE.cancelled()) + self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.cancelled()) + self.assertFalse(EXCEPTION_FUTURE.cancelled()) + self.assertFalse(SUCCESSFUL_FUTURE.cancelled()) + + def test_done(self): + self.assertFalse(PENDING_FUTURE.done()) + self.assertFalse(RUNNING_FUTURE.done()) + self.assertTrue(CANCELLED_FUTURE.done()) + self.assertTrue(CANCELLED_AND_NOTIFIED_FUTURE.done()) + self.assertTrue(EXCEPTION_FUTURE.done()) + self.assertTrue(SUCCESSFUL_FUTURE.done()) + + def test_running(self): + self.assertFalse(PENDING_FUTURE.running()) + self.assertTrue(RUNNING_FUTURE.running()) + self.assertFalse(CANCELLED_FUTURE.running()) + self.assertFalse(CANCELLED_AND_NOTIFIED_FUTURE.running()) + self.assertFalse(EXCEPTION_FUTURE.running()) + self.assertFalse(SUCCESSFUL_FUTURE.running()) + + def test_result_with_timeout(self): + self.assertRaises(futures.TimeoutError, + PENDING_FUTURE.result, timeout=0) + self.assertRaises(futures.TimeoutError, + RUNNING_FUTURE.result, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_FUTURE.result, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_AND_NOTIFIED_FUTURE.result, timeout=0) + self.assertRaises(OSError, EXCEPTION_FUTURE.result, timeout=0) + self.assertEqual(SUCCESSFUL_FUTURE.result(timeout=0), 42) + + def test_result_with_success(self): + # TODO(brian@sweetapp.com): This test is timing dependent. + def notification(): + # Wait until the main thread is waiting for the result. + time.sleep(1) + f1.set_result(42) + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertEqual(f1.result(timeout=5), 42) + t.join() + + def test_result_with_cancel(self): + # TODO(brian@sweetapp.com): This test is timing dependent. + def notification(): + # Wait until the main thread is waiting for the result. + time.sleep(1) + f1.cancel() + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertRaises(futures.CancelledError, + f1.result, timeout=support.SHORT_TIMEOUT) + t.join() + + def test_exception_with_timeout(self): + self.assertRaises(futures.TimeoutError, + PENDING_FUTURE.exception, timeout=0) + self.assertRaises(futures.TimeoutError, + RUNNING_FUTURE.exception, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_FUTURE.exception, timeout=0) + self.assertRaises(futures.CancelledError, + CANCELLED_AND_NOTIFIED_FUTURE.exception, timeout=0) + self.assertTrue(isinstance(EXCEPTION_FUTURE.exception(timeout=0), + OSError)) + self.assertEqual(SUCCESSFUL_FUTURE.exception(timeout=0), None) + + def test_exception_with_success(self): + def notification(): + # Wait until the main thread is waiting for the exception. + time.sleep(1) + with f1._condition: + f1._state = FINISHED + f1._exception = OSError() + f1._condition.notify_all() + + f1 = create_future(state=PENDING) + t = threading.Thread(target=notification) + t.start() + + self.assertTrue(isinstance(f1.exception(timeout=support.SHORT_TIMEOUT), OSError)) + t.join() + + def test_multiple_set_result(self): + f = create_future(state=PENDING) + f.set_result(1) + + with self.assertRaisesRegex( + futures.InvalidStateError, + 'FINISHED: ' + ): + f.set_result(2) + + self.assertTrue(f.done()) + self.assertEqual(f.result(), 1) + + def test_multiple_set_exception(self): + f = create_future(state=PENDING) + e = ValueError() + f.set_exception(e) + + with self.assertRaisesRegex( + futures.InvalidStateError, + 'FINISHED: ' + ): + f.set_exception(Exception()) + + self.assertEqual(f.exception(), e) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py new file mode 100644 index 00000000000..df640929309 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -0,0 +1,152 @@ +import contextlib +import logging +import queue +import time +import unittest +import sys +import io +from concurrent.futures._base import BrokenExecutor +from concurrent.futures.process import _check_system_limits + +from logging.handlers import QueueHandler + +from test import support + +from .util import ExecutorMixin, create_executor_tests, setup_module + + +INITIALIZER_STATUS = 'uninitialized' + +def init(x): + global INITIALIZER_STATUS + INITIALIZER_STATUS = x + +def get_init_status(): + return INITIALIZER_STATUS + +def init_fail(log_queue=None): + if log_queue is not None: + logger = logging.getLogger('concurrent.futures') + logger.addHandler(QueueHandler(log_queue)) + logger.setLevel('CRITICAL') + logger.propagate = False + time.sleep(0.1) # let some futures be scheduled + raise ValueError('error in initializer') + + +class InitializerMixin(ExecutorMixin): + worker_count = 2 + + def setUp(self): + global INITIALIZER_STATUS + INITIALIZER_STATUS = 'uninitialized' + self.executor_kwargs = dict(initializer=init, + initargs=('initialized',)) + super().setUp() + + def test_initializer(self): + futures = [self.executor.submit(get_init_status) + for _ in range(self.worker_count)] + + for f in futures: + self.assertEqual(f.result(), 'initialized') + + +class FailingInitializerMixin(ExecutorMixin): + worker_count = 2 + + def setUp(self): + if hasattr(self, "ctx"): + # Pass a queue to redirect the child's logging output + self.mp_context = self.get_context() + self.log_queue = self.mp_context.Queue() + self.executor_kwargs = dict(initializer=init_fail, + initargs=(self.log_queue,)) + else: + # In a thread pool, the child shares our logging setup + # (see _assert_logged()) + self.mp_context = None + self.log_queue = None + self.executor_kwargs = dict(initializer=init_fail) + super().setUp() + + def test_initializer(self): + with self._assert_logged('ValueError: error in initializer'): + try: + future = self.executor.submit(get_init_status) + except BrokenExecutor: + # Perhaps the executor is already broken + pass + else: + with self.assertRaises(BrokenExecutor): + future.result() + + # At some point, the executor should break + for _ in support.sleeping_retry(support.SHORT_TIMEOUT, + "executor not broken"): + if self.executor._broken: + break + + # ... and from this point submit() is guaranteed to fail + with self.assertRaises(BrokenExecutor): + self.executor.submit(get_init_status) + + @contextlib.contextmanager + def _assert_logged(self, msg): + if self.log_queue is not None: + yield + output = [] + try: + while True: + output.append(self.log_queue.get_nowait().getMessage()) + except queue.Empty: + pass + else: + with self.assertLogs('concurrent.futures', 'CRITICAL') as cm: + yield + output = cm.output + self.assertTrue(any(msg in line for line in output), + output) + + +create_executor_tests(globals(), InitializerMixin) +create_executor_tests(globals(), FailingInitializerMixin) + + +@unittest.skipIf(sys.platform == "win32", "Resource Tracker doesn't run on Windows") +class FailingInitializerResourcesTest(unittest.TestCase): + """ + Source: https://github.com/python/cpython/issues/104090 + """ + + def _test(self, test_class): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + + runner = unittest.TextTestRunner(stream=io.StringIO()) + runner.run(test_class('test_initializer')) + + # GH-104090: + # Stop resource tracker manually now, so we can verify there are not leaked resources by checking + # the process exit code + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker._stop() + + self.assertEqual(_resource_tracker._exitcode, 0) + + def test_spawn(self): + self._test(ProcessPoolSpawnFailingInitializerTest) + + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_forkserver(self): + self._test(ProcessPoolForkserverFailingInitializerTest) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py new file mode 100644 index 00000000000..ef318dfc7e1 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -0,0 +1,233 @@ +import os +import sys +import threading +import time +import unittest +from concurrent import futures +from concurrent.futures.process import BrokenProcessPool + +from test import support +from test.support import hashlib_helper + +from .executor import ExecutorTest, mul +from .util import ( + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin, + create_executor_tests, setup_module) + + +class EventfulGCObj(): + def __init__(self, mgr): + self.event = mgr.Event() + + def __del__(self): + self.event.set() + + +class ProcessPoolExecutorTest(ExecutorTest): + + @unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') + def test_max_workers_too_large(self): + with self.assertRaisesRegex(ValueError, + "max_workers must be <= 61"): + futures.ProcessPoolExecutor(max_workers=62) + + def test_killed_child(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken". + futures = [self.executor.submit(time.sleep, 3)] + # Get one of the processes, and terminate (kill) it + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + self.assertRaises(BrokenProcessPool, fut.result) + # Submitting other jobs fails as well. + self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + + def test_map_chunksize(self): + def bad_map(): + list(self.executor.map(pow, range(40), range(40), chunksize=-1)) + + ref = list(map(pow, range(40), range(40))) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=6)), + ref) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=50)), + ref) + self.assertEqual( + list(self.executor.map(pow, range(40), range(40), chunksize=40)), + ref) + self.assertRaises(ValueError, bad_map) + + @classmethod + def _test_traceback(cls): + raise RuntimeError(123) # some comment + + def test_traceback(self): + # We want ensure that the traceback from the child process is + # contained in the traceback raised in the main process. + future = self.executor.submit(self._test_traceback) + with self.assertRaises(Exception) as cm: + future.result() + + exc = cm.exception + self.assertIs(type(exc), RuntimeError) + self.assertEqual(exc.args, (123,)) + cause = exc.__cause__ + self.assertIs(type(cause), futures.process._RemoteTraceback) + self.assertIn('raise RuntimeError(123) # some comment', cause.tb) + + with support.captured_stderr() as f1: + try: + raise exc + except RuntimeError: + sys.excepthook(*sys.exc_info()) + self.assertIn('raise RuntimeError(123) # some comment', + f1.getvalue()) + + @unittest.skip('TODO: RUSTPYTHON flaky EOFError') + @hashlib_helper.requires_hashdigest('md5') + def test_ressources_gced_in_workers(self): + # Ensure that argument for a job are correctly gc-ed after the job + # is finished + mgr = self.get_context().Manager() + obj = EventfulGCObj(mgr) + future = self.executor.submit(id, obj) + future.result() + + self.assertTrue(obj.event.wait(timeout=1)) + + # explicitly destroy the object to ensure that EventfulGCObj.__del__() + # is called while manager is still running. + support.gc_collect() + obj = None + support.gc_collect() + + mgr.shutdown() + mgr.join() + + def test_saturation(self): + executor = self.executor + mp_context = self.get_context() + sem = mp_context.Semaphore(0) + job_count = 15 * executor._max_workers + for _ in range(job_count): + executor.submit(sem.acquire) + self.assertEqual(len(executor._processes), executor._max_workers) + for _ in range(job_count): + sem.release() + + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") + def test_idle_process_reuse_one(self): + executor = self.executor + assert executor._max_workers >= 4 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._processes), 1) + + def test_idle_process_reuse_multiple(self): + executor = self.executor + assert executor._max_workers <= 5 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + executor.submit(mul, 12, 7).result() + executor.submit(mul, 33, 25) + executor.submit(mul, 25, 26).result() + executor.submit(mul, 18, 29) + executor.submit(mul, 1, 2).result() + executor.submit(mul, 0, 9) + self.assertLessEqual(len(executor._processes), 3) + executor.shutdown() + + def test_max_tasks_per_child(self): + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + with self.assertRaises(ValueError): + self.executor_type(1, mp_context=context, max_tasks_per_child=3) + return + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=3) + f1 = executor.submit(os.getpid) + original_pid = f1.result() + # The worker pid remains the same as the worker could be reused + f2 = executor.submit(os.getpid) + self.assertEqual(f2.result(), original_pid) + self.assertEqual(len(executor._processes), 1) + f3 = executor.submit(os.getpid) + self.assertEqual(f3.result(), original_pid) + + # A new worker is spawned, with a statistically different pid, + # while the previous was reaped. + f4 = executor.submit(os.getpid) + new_pid = f4.result() + self.assertNotEqual(original_pid, new_pid) + self.assertEqual(len(executor._processes), 1) + + executor.shutdown() + + def test_max_tasks_per_child_defaults_to_spawn_context(self): + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type(1, max_tasks_per_child=3) + self.assertEqual(executor._mp_context.get_start_method(), "spawn") + + def test_max_tasks_early_shutdown(self): + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + # not using self.executor as we need to control construction. + # arguably this could go in another class w/o that mixin. + executor = self.executor_type( + 3, mp_context=context, max_tasks_per_child=1) + futures = [] + for i in range(6): + futures.append(executor.submit(mul, i, i)) + executor.shutdown() + for i, future in enumerate(futures): + self.assertEqual(future.result(), mul(i, i)) + + def test_python_finalization_error(self): + # gh-109047: Catch RuntimeError on thread creation + # during Python finalization. + + context = self.get_context() + + # gh-109047: Mock the threading.start_joinable_thread() function to inject + # RuntimeError: simulate the error raised during Python finalization. + # Block the second creation: create _ExecutorManagerThread, but block + # QueueFeederThread. + orig_start_new_thread = threading._start_joinable_thread + nthread = 0 + def mock_start_new_thread(func, *args, **kwargs): + nonlocal nthread + if nthread >= 1: + raise RuntimeError("can't create new thread at " + "interpreter shutdown") + nthread += 1 + return orig_start_new_thread(func, *args, **kwargs) + + with support.swap_attr(threading, '_start_joinable_thread', + mock_start_new_thread): + executor = self.executor_type(max_workers=2, mp_context=context) + with executor: + with self.assertRaises(BrokenProcessPool): + list(executor.map(mul, [(2, 3)] * 10)) + executor.shutdown() + + +create_executor_tests(globals(), ProcessPoolExecutorTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py new file mode 100644 index 00000000000..820ea6cf253 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -0,0 +1,410 @@ +import signal +import sys +import threading +import time +import unittest +from concurrent import futures + +from test import support +from test.support.script_helper import assert_python_ok + +from .util import ( + BaseTestCase, ThreadPoolMixin, ProcessPoolForkMixin, + ProcessPoolForkserverMixin, ProcessPoolSpawnMixin, + create_executor_tests, setup_module) + + +def sleep_and_print(t, msg): + time.sleep(t) + print(msg) + sys.stdout.flush() + + +class ExecutorShutdownTest: + def test_run_after_shutdown(self): + self.executor.shutdown() + self.assertRaises(RuntimeError, + self.executor.submit, + pow, 2, 5) + + @unittest.skip('TODO: RUSTPYTHON; hangs') + def test_interpreter_shutdown(self): + # Test the atexit hook for shutdown of worker threads and processes + rc, out, err = assert_python_ok('-c', """if 1: + from concurrent.futures import {executor_type} + from time import sleep + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + context = '{context}' + if context == "": + t = {executor_type}(5) + else: + from multiprocessing import get_context + context = get_context(context) + t = {executor_type}(5, mp_context=context) + t.submit(sleep_and_print, 1.0, "apple") + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, "ctx", ""))) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertFalse(err) + self.assertEqual(out.strip(), b"apple") + + @unittest.skip('TODO: RUSTPYTHON; Hangs') + def test_submit_after_interpreter_shutdown(self): + # Test the atexit hook for shutdown of worker threads and processes + rc, out, err = assert_python_ok('-c', """if 1: + import atexit + @atexit.register + def run_last(): + try: + t.submit(id, None) + except RuntimeError: + print("runtime-error") + raise + from concurrent.futures import {executor_type} + if __name__ == "__main__": + context = '{context}' + if not context: + t = {executor_type}(5) + else: + from multiprocessing import get_context + context = get_context(context) + t = {executor_type}(5, mp_context=context) + t.submit(id, 42).result() + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, "ctx", ""))) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertIn("RuntimeError: cannot schedule new futures", err.decode()) + self.assertEqual(out.strip(), b"runtime-error") + + def test_hang_issue12364(self): + fs = [self.executor.submit(time.sleep, 0.1) for _ in range(50)] + self.executor.shutdown() + for f in fs: + f.result() + + def test_cancel_futures(self): + assert self.worker_count <= 5, "test needs few workers" + fs = [self.executor.submit(time.sleep, .1) for _ in range(50)] + self.executor.shutdown(cancel_futures=True) + # We can't guarantee the exact number of cancellations, but we can + # guarantee that *some* were cancelled. With few workers, many of + # the submitted futures should have been cancelled. + cancelled = [fut for fut in fs if fut.cancelled()] + self.assertGreater(len(cancelled), 20) + + # Ensure the other futures were able to finish. + # Use "not fut.cancelled()" instead of "fut.done()" to include futures + # that may have been left in a pending state. + others = [fut for fut in fs if not fut.cancelled()] + for fut in others: + self.assertTrue(fut.done(), msg=f"{fut._state=}") + self.assertIsNone(fut.exception()) + + # Similar to the number of cancelled futures, we can't guarantee the + # exact number that completed. But, we can guarantee that at least + # one finished. + self.assertGreater(len(others), 0) + + def test_hang_gh83386(self): + """shutdown(wait=False) doesn't hang at exit with running futures. + + See https://github.com/python/cpython/issues/83386. + """ + if self.executor_type == futures.ProcessPoolExecutor: + raise unittest.SkipTest( + "Hangs, see https://github.com/python/cpython/issues/83386") + + rc, out, err = assert_python_ok('-c', """if True: + from concurrent.futures import {executor_type} + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + if {context!r}: multiprocessing.set_start_method({context!r}) + t = {executor_type}(max_workers=3) + t.submit(sleep_and_print, 1.0, "apple") + t.shutdown(wait=False) + """.format(executor_type=self.executor_type.__name__, + context=getattr(self, 'ctx', None))) + self.assertFalse(err) + self.assertEqual(out.strip(), b"apple") + + def test_hang_gh94440(self): + """shutdown(wait=True) doesn't hang when a future was submitted and + quickly canceled right before shutdown. + + See https://github.com/python/cpython/issues/94440. + """ + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + raise RuntimeError("timed out waiting for shutdown") + + kwargs = {} + if getattr(self, 'ctx', None): + kwargs['mp_context'] = self.get_context() + executor = self.executor_type(max_workers=1, **kwargs) + executor.submit(int).result() + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(5) + executor.submit(int).cancel() + executor.shutdown(wait=True) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + +class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): + def test_threads_terminate(self): + def acquire_lock(lock): + lock.acquire() + + sem = threading.Semaphore(0) + for i in range(3): + self.executor.submit(acquire_lock, sem) + self.assertEqual(len(self.executor._threads), 3) + for i in range(3): + sem.release() + self.executor.shutdown() + for t in self.executor._threads: + t.join() + + def test_context_manager_shutdown(self): + with futures.ThreadPoolExecutor(max_workers=5) as e: + executor = e + self.assertEqual(list(e.map(abs, range(-5, 5))), + [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) + + for t in executor._threads: + t.join() + + def test_del_shutdown(self): + executor = futures.ThreadPoolExecutor(max_workers=5) + res = executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + + for t in threads: + t.join() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + def test_shutdown_no_wait(self): + # Ensure that the executor cleans up the threads when calling + # shutdown with wait=False + executor = futures.ThreadPoolExecutor(max_workers=5) + res = executor.map(abs, range(-5, 5)) + threads = executor._threads + executor.shutdown(wait=False) + for t in threads: + t.join() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + + def test_thread_names_assigned(self): + executor = futures.ThreadPoolExecutor( + max_workers=5, thread_name_prefix='SpecialPool') + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + support.gc_collect() # For PyPy or other GCs. + + for t in threads: + self.assertRegex(t.name, r'^SpecialPool_[0-4]$') + t.join() + + def test_thread_names_default(self): + executor = futures.ThreadPoolExecutor(max_workers=5) + executor.map(abs, range(-5, 5)) + threads = executor._threads + del executor + support.gc_collect() # For PyPy or other GCs. + + for t in threads: + # Ensure that our default name is reasonably sane and unique when + # no thread_name_prefix was supplied. + self.assertRegex(t.name, r'ThreadPoolExecutor-\d+_[0-4]$') + t.join() + + def test_cancel_futures_wait_false(self): + # Can only be reliably tested for TPE, since PPE often hangs with + # `wait=False` (even without *cancel_futures*). + rc, out, err = assert_python_ok('-c', """if True: + from concurrent.futures import ThreadPoolExecutor + from test.test_concurrent_futures.test_shutdown import sleep_and_print + if __name__ == "__main__": + t = ThreadPoolExecutor() + t.submit(sleep_and_print, .1, "apple") + t.shutdown(wait=False, cancel_futures=True) + """) + # Errors in atexit hooks don't change the process exit code, check + # stderr manually. + self.assertFalse(err) + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) + + +class ProcessPoolShutdownTest(ExecutorShutdownTest): + # TODO: RUSTPYTHON - flaky, dict changed size during iteration race condition + @unittest.skip("TODO: RUSTPYTHON - flaky race condition on macOS") + def test_cancel_futures(self): + return super().test_cancel_futures() + + def test_processes_terminate(self): + def acquire_lock(lock): + lock.acquire() + + mp_context = self.get_context() + if mp_context.get_start_method(allow_none=False) == "fork": + # fork pre-spawns, not on demand. + expected_num_processes = self.worker_count + else: + expected_num_processes = 3 + + sem = mp_context.Semaphore(0) + for _ in range(3): + self.executor.submit(acquire_lock, sem) + self.assertEqual(len(self.executor._processes), expected_num_processes) + for _ in range(3): + sem.release() + processes = self.executor._processes + self.executor.shutdown() + + for p in processes.values(): + p.join() + + def test_context_manager_shutdown(self): + with futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) as e: + processes = e._processes + self.assertEqual(list(e.map(abs, range(-5, 5))), + [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) + + for p in processes.values(): + p.join() + + def test_del_shutdown(self): + executor = futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) + res = executor.map(abs, range(-5, 5)) + executor_manager_thread = executor._executor_manager_thread + processes = executor._processes + call_queue = executor._call_queue + executor_manager_thread = executor._executor_manager_thread + del executor + support.gc_collect() # For PyPy or other GCs. + + # Make sure that all the executor resources were properly cleaned by + # the shutdown process + executor_manager_thread.join() + for p in processes.values(): + p.join() + call_queue.join_thread() + + # Make sure the results were all computed before the + # executor got shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + def test_shutdown_no_wait(self): + # Ensure that the executor cleans up the processes when calling + # shutdown with wait=False + executor = futures.ProcessPoolExecutor( + max_workers=5, mp_context=self.get_context()) + res = executor.map(abs, range(-5, 5)) + processes = executor._processes + call_queue = executor._call_queue + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + + # Make sure that all the executor resources were properly cleaned by + # the shutdown process + executor_manager_thread.join() + for p in processes.values(): + p.join() + call_queue.join_thread() + + # Make sure the results were all computed before the executor got + # shutdown. + assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + + @classmethod + def _failing_task_gh_132969(cls, n): + raise ValueError("failing task") + + @classmethod + def _good_task_gh_132969(cls, n): + time.sleep(0.1 * n) + return n + + def _run_test_issue_gh_132969(self, max_workers): + # max_workers=2 will repro exception + # max_workers=4 will repro exception and then hang + + # Repro conditions + # max_tasks_per_child=1 + # a task ends abnormally + # shutdown(wait=False) is called + start_method = self.get_context().get_start_method() + if (start_method == "fork" or + (start_method == "forkserver" and sys.platform.startswith("win"))): + self.skipTest(f"Skipping test for {start_method = }") + executor = futures.ProcessPoolExecutor( + max_workers=max_workers, + max_tasks_per_child=1, + mp_context=self.get_context()) + f1 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 1) + f2 = executor.submit(ProcessPoolShutdownTest._failing_task_gh_132969, 2) + f3 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 3) + result = 0 + try: + result += f1.result() + result += f2.result() + result += f3.result() + except ValueError: + # stop processing results upon first exception + pass + + # Ensure that the executor cleans up after called + # shutdown with wait=False + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + time.sleep(0.2) + executor_manager_thread.join() + return result + + def test_shutdown_gh_132969_case_1(self): + # gh-132969: test that exception "object of type 'NoneType' has no len()" + # is not raised when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(2) + self.assertEqual(result, 1) + + def test_shutdown_gh_132969_case_2(self): + # gh-132969: test that process does not hang and + # exception "object of type 'NoneType' has no len()" is not raised + # when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(4) + self.assertEqual(result, 1) + + +create_executor_tests(globals(), ProcessPoolShutdownTest, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py new file mode 100644 index 00000000000..4324241b374 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -0,0 +1,121 @@ +import contextlib +import multiprocessing as mp +import multiprocessing.process +import multiprocessing.util +import os +import threading +import unittest +from concurrent import futures +from test import support + +from .executor import ExecutorTest, mul +from .util import BaseTestCase, ThreadPoolMixin, setup_module + + +class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase): + def test_map_submits_without_iteration(self): + """Tests verifying issue 11777.""" + finished = [] + def record_finished(n): + finished.append(n) + + self.executor.map(record_finished, range(10)) + self.executor.shutdown(wait=True) + self.assertCountEqual(finished, range(10)) + + def test_default_workers(self): + executor = self.executor_type() + expected = min(32, (os.process_cpu_count() or 1) + 4) + self.assertEqual(executor._max_workers, expected) + + def test_saturation(self): + executor = self.executor_type(4) + def acquire_lock(lock): + lock.acquire() + + sem = threading.Semaphore(0) + for i in range(15 * executor._max_workers): + executor.submit(acquire_lock, sem) + self.assertEqual(len(executor._threads), executor._max_workers) + for i in range(15 * executor._max_workers): + sem.release() + executor.shutdown(wait=True) + + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") + def test_idle_thread_reuse(self): + executor = self.executor_type() + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._threads), 1) + executor.shutdown(wait=True) + + @support.requires_fork() + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + @support.requires_resource('cpu') + def test_hang_global_shutdown_lock(self): + # bpo-45021: _global_shutdown_lock should be reinitialized in the child + # process, otherwise it will never exit + def submit(pool): + pool.submit(submit, pool) + + with futures.ThreadPoolExecutor(1) as pool: + pool.submit(submit, pool) + + for _ in range(50): + with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: + workers.submit(tuple) + + @support.requires_fork() + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + def test_process_fork_from_a_threadpool(self): + # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, + # otherwise child process will try to join parent thread + def fork_process_and_return_exitcode(): + # Ignore the warning about fork with threads. + with self.assertWarnsRegex(DeprecationWarning, + r"use of fork\(\) may lead to deadlocks in the child"): + p = mp.get_context('fork').Process(target=lambda: 1) + p.start() + p.join() + return p.exitcode + + with futures.ThreadPoolExecutor(1) as pool: + process_exitcode = pool.submit(fork_process_and_return_exitcode).result() + + self.assertEqual(process_exitcode, 0) + + def test_executor_map_current_future_cancel(self): + stop_event = threading.Event() + log = [] + + def log_n_wait(ident): + log.append(f"{ident=} started") + try: + stop_event.wait() + finally: + log.append(f"{ident=} stopped") + + with self.executor_type(max_workers=1) as pool: + # submit work to saturate the pool + fut = pool.submit(log_n_wait, ident="first") + try: + with contextlib.closing( + pool.map(log_n_wait, ["second", "third"], timeout=0) + ) as gen: + with self.assertRaises(TimeoutError): + next(gen) + finally: + stop_event.set() + fut.result() + # ident='second' is cancelled as a result of raising a TimeoutError + # ident='third' is cancelled because it remained in the collection of futures + self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py new file mode 100644 index 00000000000..818e0d51a2c --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -0,0 +1,219 @@ +import sys +import threading +import unittest +from concurrent import futures +from test import support +from test.support import threading_helper + +from .util import ( + CANCELLED_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + create_executor_tests, setup_module, + BaseTestCase, ThreadPoolMixin, + ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin) + + +def mul(x, y): + return x * y + +def wait_and_raise(e): + e.wait() + raise Exception('this is an exception') + + +class WaitTests: + def test_20369(self): + # See https://bugs.python.org/issue20369 + future = self.executor.submit(mul, 1, 2) + done, not_done = futures.wait([future, future], + return_when=futures.ALL_COMPLETED) + self.assertEqual({future}, done) + self.assertEqual(set(), not_done) + + + def test_first_completed(self): + event = self.create_event() + future1 = self.executor.submit(mul, 21, 2) + future2 = self.executor.submit(event.wait) + + try: + done, not_done = futures.wait( + [CANCELLED_FUTURE, future1, future2], + return_when=futures.FIRST_COMPLETED) + + self.assertEqual(set([future1]), done) + self.assertEqual(set([CANCELLED_FUTURE, future2]), not_done) + finally: + event.set() + future2.result() # wait for job to finish + + def test_first_completed_some_already_completed(self): + event = self.create_event() + future1 = self.executor.submit(event.wait) + + try: + finished, pending = futures.wait( + [CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE, future1], + return_when=futures.FIRST_COMPLETED) + + self.assertEqual( + set([CANCELLED_AND_NOTIFIED_FUTURE, SUCCESSFUL_FUTURE]), + finished) + self.assertEqual(set([future1]), pending) + finally: + event.set() + future1.result() # wait for job to finish + + def test_first_exception(self): + event1 = self.create_event() + event2 = self.create_event() + try: + future1 = self.executor.submit(mul, 2, 21) + future2 = self.executor.submit(wait_and_raise, event1) + future3 = self.executor.submit(event2.wait) + + # Ensure that future1 is completed before future2 finishes + def wait_for_future1(): + future1.result() + event1.set() + + t = threading.Thread(target=wait_for_future1) + t.start() + + finished, pending = futures.wait( + [future1, future2, future3], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([future1, future2]), finished) + self.assertEqual(set([future3]), pending) + + threading_helper.join_thread(t) + finally: + event1.set() + event2.set() + future3.result() # wait for job to finish + + def test_first_exception_some_already_complete(self): + event = self.create_event() + future1 = self.executor.submit(divmod, 21, 0) + future2 = self.executor.submit(event.wait) + + try: + finished, pending = futures.wait( + [SUCCESSFUL_FUTURE, + CANCELLED_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + future1, future2], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + future1]), finished) + self.assertEqual(set([CANCELLED_FUTURE, future2]), pending) + finally: + event.set() + future2.result() # wait for job to finish + + def test_first_exception_one_already_failed(self): + event = self.create_event() + future1 = self.executor.submit(event.wait) + + try: + finished, pending = futures.wait( + [EXCEPTION_FUTURE, future1], + return_when=futures.FIRST_EXCEPTION) + + self.assertEqual(set([EXCEPTION_FUTURE]), finished) + self.assertEqual(set([future1]), pending) + finally: + event.set() + future1.result() # wait for job to finish + + def test_all_completed(self): + future1 = self.executor.submit(divmod, 2, 0) + future2 = self.executor.submit(mul, 2, 21) + + finished, pending = futures.wait( + [SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + future1, + future2], + return_when=futures.ALL_COMPLETED) + + self.assertEqual(set([SUCCESSFUL_FUTURE, + CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + future1, + future2]), finished) + self.assertEqual(set(), pending) + + def test_timeout(self): + short_timeout = 0.050 + + event = self.create_event() + future = self.executor.submit(event.wait) + + try: + finished, pending = futures.wait( + [CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE, + future], + timeout=short_timeout, + return_when=futures.ALL_COMPLETED) + + self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, + EXCEPTION_FUTURE, + SUCCESSFUL_FUTURE]), + finished) + self.assertEqual(set([future]), pending) + finally: + event.set() + future.result() # wait for job to finish + + +class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): + + def test_pending_calls_race(self): + # Issue #14406: multi-threaded race condition when waiting on all + # futures. + event = threading.Event() + def future_func(): + event.wait() + oldswitchinterval = sys.getswitchinterval() + support.setswitchinterval(1e-6) + try: + fs = {self.executor.submit(future_func) for i in range(100)} + event.set() + futures.wait(fs, return_when=futures.ALL_COMPLETED) + finally: + sys.setswitchinterval(oldswitchinterval) + + +create_executor_tests(globals(), WaitTests, + executor_mixins=(ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)) + + +def setUpModule(): + setup_module() + +class ProcessPoolForkWaitTest(ProcessPoolForkWaitTest): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") + def test_first_completed(self): super().test_first_completed() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON Fatal Python error: Segmentation fault") + def test_first_completed_some_already_completed(self): super().test_first_completed_some_already_completed() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON flaky") + def test_first_exception(self): super().test_first_exception() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") + def test_first_exception_one_already_failed(self): super().test_first_exception_one_already_failed() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON flaky") + def test_first_exception_some_already_complete(self): super().test_first_exception_some_already_complete() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON Fatal Python error: Segmentation fault") + def test_timeout(self): super().test_timeout() # TODO: RUSTPYTHON + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py new file mode 100644 index 00000000000..e85ef3b1c91 --- /dev/null +++ b/Lib/test/test_concurrent_futures/util.py @@ -0,0 +1,169 @@ +import multiprocessing +import sys +import threading +import time +import unittest +from concurrent import futures +from concurrent.futures._base import ( + PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future, + ) +from concurrent.futures.process import _check_system_limits + +from test import support +from test.support import threading_helper + + +def create_future(state=PENDING, exception=None, result=None): + f = Future() + f._state = state + f._exception = exception + f._result = result + return f + + +PENDING_FUTURE = create_future(state=PENDING) +RUNNING_FUTURE = create_future(state=RUNNING) +CANCELLED_FUTURE = create_future(state=CANCELLED) +CANCELLED_AND_NOTIFIED_FUTURE = create_future(state=CANCELLED_AND_NOTIFIED) +EXCEPTION_FUTURE = create_future(state=FINISHED, exception=OSError()) +SUCCESSFUL_FUTURE = create_future(state=FINISHED, result=42) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._thread_key = threading_helper.threading_setup() + + def tearDown(self): + support.reap_children() + threading_helper.threading_cleanup(*self._thread_key) + + +class ExecutorMixin: + worker_count = 5 + executor_kwargs = {} + + def setUp(self): + super().setUp() + + self.t1 = time.monotonic() + if hasattr(self, "ctx"): + self.executor = self.executor_type( + max_workers=self.worker_count, + mp_context=self.get_context(), + **self.executor_kwargs) + self.manager = self.get_context().Manager() + else: + self.executor = self.executor_type( + max_workers=self.worker_count, + **self.executor_kwargs) + self.manager = None + + def tearDown(self): + self.executor.shutdown(wait=True) + self.executor = None + if self.manager is not None: + self.manager.shutdown() + self.manager = None + + dt = time.monotonic() - self.t1 + if support.verbose: + print("%.2fs" % dt, end=' ') + self.assertLess(dt, 300, "synchronization issue: test lasted too long") + + super().tearDown() + + def get_context(self): + return multiprocessing.get_context(self.ctx) + + +class ThreadPoolMixin(ExecutorMixin): + executor_type = futures.ThreadPoolExecutor + + def create_event(self): + return threading.Event() + + +class ProcessPoolForkMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "fork" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + if sys.platform == "win32": + self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") + return super().get_context() + + def create_event(self): + return self.manager.Event() + + +class ProcessPoolSpawnMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "spawn" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + return super().get_context() + + def create_event(self): + return self.manager.Event() + + +class ProcessPoolForkserverMixin(ExecutorMixin): + executor_type = futures.ProcessPoolExecutor + ctx = "forkserver" + + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + if sys.platform == "win32": + self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") + return super().get_context() + + def create_event(self): + return self.manager.Event() + + +def create_executor_tests(remote_globals, mixin, bases=(BaseTestCase,), + executor_mixins=(ThreadPoolMixin, + ProcessPoolForkMixin, + ProcessPoolForkserverMixin, + ProcessPoolSpawnMixin)): + def strip_mixin(name): + if name.endswith(('Mixin', 'Tests')): + return name[:-5] + elif name.endswith('Test'): + return name[:-4] + else: + return name + + module = remote_globals['__name__'] + for exe in executor_mixins: + name = ("%s%sTest" + % (strip_mixin(exe.__name__), strip_mixin(mixin.__name__))) + cls = type(name, (mixin,) + (exe,) + bases, {'__module__': module}) + remote_globals[name] = cls + + +def setup_module(): + try: + _check_system_limits() + except NotImplementedError: + pass + else: + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index d793cc58907..1bfb53ccbb1 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -526,7 +526,6 @@ def test_default_case_sensitivity(self): cf.get(self.default_section, "Foo"), "Bar", "could not locate option, expecting case-insensitive defaults") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, universal newlines") def test_parse_errors(self): cf = self.newconfig() self.parse_error(cf, configparser.ParsingError, @@ -762,7 +761,6 @@ def test_read_returns_file_list(self): parsed_files = cf.read([], encoding="utf-8") self.assertEqual(parsed_files, []) - @unittest.skip("TODO: RUSTPYTHON, suspected to make CI hang") def test_read_returns_file_list_with_bytestring_path(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') @@ -988,12 +986,12 @@ def test_add_section_default(self): def test_defaults_keyword(self): """bpo-23835 fix for ConfigParser""" - cf = self.newconfig(defaults={1: 2.4}) - self.assertEqual(cf[self.default_section]['1'], '2.4') - self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.4) - cf = self.newconfig(defaults={"A": 5.2}) - self.assertEqual(cf[self.default_section]['a'], '5.2') - self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.2) + cf = self.newconfig(defaults={1: 2.5}) + self.assertEqual(cf[self.default_section]['1'], '2.5') + self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.5) + cf = self.newconfig(defaults={"A": 5.25}) + self.assertEqual(cf[self.default_section]['a'], '5.25') + self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.25) class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase): @@ -2204,6 +2202,40 @@ def test_no_section(self): self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + def test_empty_unnamed_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.add_section('section') + output = io.StringIO() + cfg.write(output) + self.assertEqual(output.getvalue(), '[section]\n\n') + + def test_add_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.set(configparser.UNNAMED_SECTION, 'a', '1') + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + output = io.StringIO() + cfg.write(output) + self.assertEqual(output.getvalue(), 'a = 1\n\n') + + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg[configparser.UNNAMED_SECTION] = {'a': '1'} + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + output = io.StringIO() + cfg.write(output) + self.assertEqual(output.getvalue(), 'a = 1\n\n') + + def test_disabled_error(self): + with self.assertRaises(configparser.MissingSectionHeaderError): + configparser.ConfigParser().read_string("a = 1") + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser()[configparser.UNNAMED_SECTION] = {'a': '1'} + def test_multiple_configs(self): cfg = configparser.ConfigParser(allow_unnamed_section=True) cfg.read_string('a = 1') @@ -2214,6 +2246,30 @@ def test_multiple_configs(self): self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b']) +class InvalidInputTestCase(unittest.TestCase): + """Tests for issue #65697, where configparser will write configs + it parses back differently. Ex: keys containing delimiters or + matching the section pattern""" + + def test_delimiter_in_key(self): + cfg = configparser.ConfigParser(delimiters=('=')) + cfg.add_section('section1') + cfg.set('section1', 'a=b', 'c') + output = io.StringIO() + with self.assertRaises(configparser.InvalidWriteError): + cfg.write(output) + output.close() + + def test_section_bracket_in_key(self): + cfg = configparser.ConfigParser() + cfg.add_section('section1') + cfg.set('section1', '[this parses back as a section]', 'foo') + output = io.StringIO() + with self.assertRaises(configparser.InvalidWriteError): + cfg.write(output) + output.close() + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 471d04a76ca..7339f16ace2 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -16,6 +16,7 @@ def __getitem__(self, n): return [self.el][n] class TestContains(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_common_tests(self): a = base_set(1) b = myset(1) @@ -24,8 +25,11 @@ def test_common_tests(self): self.assertNotIn(0, b) self.assertIn(1, c) self.assertNotIn(0, c) - self.assertRaises(TypeError, lambda: 1 in a) - self.assertRaises(TypeError, lambda: 1 not in a) + msg = "argument of type 'base_set' is not a container or iterable" + with self.assertRaisesRegex(TypeError, msg): + 1 in a + with self.assertRaisesRegex(TypeError, msg): + 1 not in a # test char in string self.assertIn('c', 'abc') diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 06270e161da..59d2320de85 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -3,6 +3,7 @@ import functools import gc import random +import sys import time import unittest import weakref @@ -216,8 +217,6 @@ def fun(): ctx.run(fun) - # TODO: RUSTPYTHON - @unittest.expectedFailure @isolated_context def test_context_getset_1(self): c = contextvars.ContextVar('c') @@ -316,8 +315,6 @@ def test_context_getset_4(self): with self.assertRaisesRegex(ValueError, 'different Context'): c.reset(tok) - # TODO: RUSTPYTHON - @unittest.expectedFailure @isolated_context def test_context_getset_5(self): c = contextvars.ContextVar('c', default=42) @@ -331,8 +328,6 @@ def fun(): contextvars.copy_context().run(fun) self.assertEqual(c.get(), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_copy_1(self): ctx1 = contextvars.Context() c = contextvars.ContextVar('c', default=42) @@ -358,9 +353,9 @@ def ctx2_fun(): ctx1.run(ctx1_fun) - @unittest.skip("TODO: RUSTPYTHON; threading is not safe") @isolated_context @threading_helper.requires_working_threading() + @unittest.skipIf(sys.platform == 'darwin', 'TODO: RUSTPYTHON; Flaky on Mac, self.assertEqual(cvar.get(), num + i) AssertionError: 8 != 12') def test_context_threads_1(self): cvar = contextvars.ContextVar('cvar') diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 9bb3fd0179b..fabe0c971c1 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -844,8 +844,6 @@ def test_exit_suppress(self): stack.push(lambda *exc: True) 1/0 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exit_exception_traceback(self): # This test captures the current behavior of ExitStack so that we know # if we ever unintendedly change it. It is not a statement of what the @@ -926,8 +924,6 @@ def __exit__(self, *exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exit_exception_chaining(self): # Ensure exception chaining matches the reference behaviour def raise_exc(exc): @@ -959,8 +955,6 @@ def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exit_exception_explicit_none_context(self): # Ensure ExitStack chaining matches actual nested `with` statements # regarding explicit __context__ = None. @@ -1055,8 +1049,6 @@ def gets_the_context_right(exc): self.assertIsNone( exc.__context__.__context__.__context__.__context__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exit_exception_with_existing_context(self): # Addresses a lack of test coverage discovered after checking in a # fix for issue 20317 that still contained debugging code. diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index d7331c4d433..c3c303ad9a7 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -120,8 +120,7 @@ async def woohoo(): raise ZeroDivisionError() self.assertEqual(state, [1, 42, 999]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON async def test_contextmanager_traceback(self): @asynccontextmanager async def f(): @@ -253,8 +252,7 @@ async def woohoo(): raise ZeroDivisionError(999) self.assertEqual(state, [1, 42, 999]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON async def test_contextmanager_except_stopiter(self): @asynccontextmanager async def woohoo(): @@ -652,8 +650,6 @@ async def __aenter__(self): await stack.enter_async_context(LacksExit()) self.assertFalse(stack._exit_callbacks) - # TODO: RUSTPYTHON - @unittest.expectedFailure async def test_async_exit_exception_chaining(self): # Ensure exception chaining matches the reference behaviour async def raise_exc(exc): @@ -685,8 +681,6 @@ async def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) - # TODO: RUSTPYTHON - @unittest.expectedFailure async def test_async_exit_exception_explicit_none_context(self): # Ensure AsyncExitStack chaining matches actual nested `with` statements # regarding explicit __context__ = None. @@ -733,6 +727,7 @@ class Example(object): pass self.assertIs(stack._exit_callbacks[-1][1], cm) + class TestAsyncNullcontext(unittest.IsolatedAsyncioTestCase): async def test_async_nullcontext(self): class C: diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 456767bbe0c..cfef24727e8 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -19,7 +19,7 @@ class TestCopy(unittest.TestCase): def test_exceptions(self): self.assertIs(copy.Error, copy.error) - self.assertTrue(issubclass(copy.Error, Exception)) + self.assertIsSubclass(copy.Error, Exception) # The copy() method @@ -207,8 +207,6 @@ def __eq__(self, other): self.assertIsNot(y, x) self.assertEqual(y.foo, x.foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_inst_getnewargs_ex(self): class C(int): def __new__(cls, *, foo): @@ -373,6 +371,8 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -400,6 +400,8 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -417,6 +419,8 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x @@ -507,8 +511,6 @@ def __eq__(self, other): self.assertEqual(y.foo, x.foo) self.assertIsNot(y.foo, x.foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deepcopy_inst_getnewargs_ex(self): class C(int): def __new__(cls, *, foo): @@ -670,7 +672,7 @@ def __eq__(self, other): def test_reduce_5tuple(self): class C(dict): def __reduce__(self): - return (C, (), self.__dict__, None, self.items()) + return (C, (), self.__dict__, None, iter(self.items())) def __eq__(self, other): return (dict(self) == dict(other) and self.__dict__ == other.__dict__) @@ -938,8 +940,6 @@ def __replace__(self, **changes): self.assertEqual(attrs(copy.replace(a, y=2)), (11, 2, 13)) self.assertEqual(attrs(copy.replace(a, x=1, y=2)), (1, 2, 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_namedtuple(self): from collections import namedtuple from typing import NamedTuple diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py new file mode 100644 index 00000000000..604153354c1 --- /dev/null +++ b/Lib/test/test_coroutines.py @@ -0,0 +1,2547 @@ +import contextlib +import copy +import inspect +import pickle +import sys +import types +import traceback +import unittest +import warnings +from test import support +from test.support import import_helper +from test.support import warnings_helper +from test.support.script_helper import assert_python_ok +try: + import _testcapi +except ImportError: + _testcapi = None + + +class AsyncYieldFrom: + def __init__(self, obj): + self.obj = obj + + def __await__(self): + yield from self.obj + + +class AsyncYield: + def __init__(self, value): + self.value = value + + def __await__(self): + yield self.value + + +async def asynciter(iterable): + """Convert an iterable to an asynchronous iterator.""" + for x in iterable: + yield x + + +def run_async(coro): + assert coro.__class__ in {types.GeneratorType, types.CoroutineType} + + buffer = [] + result = None + while True: + try: + buffer.append(coro.send(None)) + except StopIteration as ex: + result = ex.args[0] if ex.args else None + break + return buffer, result + + +def run_async__await__(coro): + assert coro.__class__ is types.CoroutineType + aw = coro.__await__() + buffer = [] + result = None + i = 0 + while True: + try: + if i % 2: + buffer.append(next(aw)) + else: + buffer.append(aw.send(None)) + i += 1 + except StopIteration as ex: + result = ex.args[0] if ex.args else None + break + return buffer, result + + +@contextlib.contextmanager +def silence_coro_gc(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + yield + support.gc_collect() + + +class AsyncBadSyntaxTest(unittest.TestCase): + + def test_badsyntax_1(self): + samples = [ + """def foo(): + await something() + """, + + """await something()""", + + """async def foo(): + yield from [] + """, + + """async def foo(): + await await fut + """, + + """async def foo(a=await something()): + pass + """, + + """async def foo(a:await something()): + pass + """, + + """async def foo(): + def bar(): + [i async for i in els] + """, + + """async def foo(): + def bar(): + [await i for i in els] + """, + + """async def foo(): + def bar(): + [i for i in els + async for b in els] + """, + + """async def foo(): + def bar(): + [i for i in els + for c in b + async for b in els] + """, + + """async def foo(): + def bar(): + [i for i in els + async for b in els + for c in b] + """, + + """async def foo(): + def bar(): + [[async for i in b] for b in els] + """, + + """async def foo(): + def bar(): + [i for i in els + for b in await els] + """, + + """async def foo(): + def bar(): + [i for i in els + for b in els + if await b] + """, + + """async def foo(): + def bar(): + [i for i in await els] + """, + + """async def foo(): + def bar(): + [i for i in els if await i] + """, + + """def bar(): + [i async for i in els] + """, + + """def bar(): + {i: i async for i in els} + """, + + """def bar(): + {i async for i in els} + """, + + """def bar(): + [await i for i in els] + """, + + """def bar(): + [i for i in els + async for b in els] + """, + + """def bar(): + [i for i in els + for c in b + async for b in els] + """, + + """def bar(): + [i for i in els + async for b in els + for c in b] + """, + + """def bar(): + [i for i in els + for b in await els] + """, + + """def bar(): + [i for i in els + for b in els + if await b] + """, + + """def bar(): + [i for i in await els] + """, + + """def bar(): + [i for i in els if await i] + """, + + """def bar(): + [[i async for i in a] for a in elts] + """, + + """[[i async for i in a] for a in elts] + """, + + """async def foo(): + await + """, + + """async def foo(): + def bar(): pass + await = 1 + """, + + """async def foo(): + + def bar(): pass + await = 1 + """, + + """async def foo(): + def bar(): pass + if 1: + await = 1 + """, + + """def foo(): + async def bar(): pass + if 1: + await a + """, + + """def foo(): + async def bar(): pass + await a + """, + + """def foo(): + def baz(): pass + async def bar(): pass + await a + """, + + """def foo(): + def baz(): pass + # 456 + async def bar(): pass + # 123 + await a + """, + + """async def foo(): + def baz(): pass + # 456 + async def bar(): pass + # 123 + await = 2 + """, + + """def foo(): + + def baz(): pass + + async def bar(): pass + + await a + """, + + """async def foo(): + + def baz(): pass + + async def bar(): pass + + await = 2 + """, + + """async def foo(): + def async(): pass + """, + + """async def foo(): + def await(): pass + """, + + """async def foo(): + def bar(): + await + """, + + """async def foo(): + return lambda async: await + """, + + """async def foo(): + return lambda a: await + """, + + """await a()""", + + """async def foo(a=await b): + pass + """, + + """async def foo(a:await b): + pass + """, + + """def baz(): + async def foo(a=await b): + pass + """, + + """async def foo(async): + pass + """, + + """async def foo(): + def bar(): + def baz(): + async = 1 + """, + + """async def foo(): + def bar(): + def baz(): + pass + async = 1 + """, + + """def foo(): + async def bar(): + + async def baz(): + pass + + def baz(): + 42 + + async = 1 + """, + + """async def foo(): + def bar(): + def baz(): + pass\nawait foo() + """, + + """def foo(): + def bar(): + async def baz(): + pass\nawait foo() + """, + + """async def foo(await): + pass + """, + + """def foo(): + + async def bar(): pass + + await a + """, + + """def foo(): + async def bar(): + pass\nawait a + """, + """def foo(): + async for i in arange(2): + pass + """, + """def foo(): + async with resource: + pass + """, + """async with resource: + pass + """, + """async for i in arange(2): + pass + """, + ] + + for code in samples: + with self.subTest(code=code), self.assertRaises(SyntaxError): + compile(code, "", "exec") + + def test_badsyntax_2(self): + samples = [ + """def foo(): + await = 1 + """, + + """class Bar: + def async(): pass + """, + + """class Bar: + async = 1 + """, + + """class async: + pass + """, + + """class await: + pass + """, + + """import math as await""", + + """def async(): + pass""", + + """def foo(*, await=1): + pass""" + + """async = 1""", + + """print(await=1)""" + ] + + for code in samples: + with self.subTest(code=code), self.assertRaises(SyntaxError): + compile(code, "", "exec") + + def test_badsyntax_3(self): + with self.assertRaises(SyntaxError): + compile("async = 1", "", "exec") + + def test_badsyntax_4(self): + samples = [ + '''def foo(await): + async def foo(): pass + async def foo(): + pass + return await + 1 + ''', + + '''def foo(await): + async def foo(): pass + async def foo(): pass + return await + 1 + ''', + + '''def foo(await): + + async def foo(): pass + + async def foo(): pass + + return await + 1 + ''', + + '''def foo(await): + """spam""" + async def foo(): \ + pass + # 123 + async def foo(): pass + # 456 + return await + 1 + ''', + + '''def foo(await): + def foo(): pass + def foo(): pass + async def bar(): return await_ + await_ = await + try: + bar().send(None) + except StopIteration as ex: + return ex.args[0] + 1 + ''' + ] + + for code in samples: + with self.subTest(code=code), self.assertRaises(SyntaxError): + compile(code, "", "exec") + + +class TokenizerRegrTest(unittest.TestCase): + + def test_oneline_defs(self): + buf = [] + for i in range(500): + buf.append('def i{i}(): return {i}'.format(i=i)) + buf = '\n'.join(buf) + + # Test that 500 consequent, one-line defs is OK + ns = {} + exec(buf, ns, ns) + self.assertEqual(ns['i499'](), 499) + + # Test that 500 consequent, one-line defs *and* + # one 'async def' following them is OK + buf += '\nasync def foo():\n return' + ns = {} + exec(buf, ns, ns) + self.assertEqual(ns['i499'](), 499) + self.assertTrue(inspect.iscoroutinefunction(ns['foo'])) + + +class CoroutineTest(unittest.TestCase): + + def test_gen_1(self): + def gen(): yield + self.assertNotHasAttr(gen, '__await__') + + def test_func_1(self): + async def foo(): + return 10 + + f = foo() + self.assertIsInstance(f, types.CoroutineType) + self.assertTrue(bool(foo.__code__.co_flags & inspect.CO_COROUTINE)) + self.assertFalse(bool(foo.__code__.co_flags & inspect.CO_GENERATOR)) + self.assertTrue(bool(f.cr_code.co_flags & inspect.CO_COROUTINE)) + self.assertFalse(bool(f.cr_code.co_flags & inspect.CO_GENERATOR)) + self.assertEqual(run_async(f), ([], 10)) + + self.assertEqual(run_async__await__(foo()), ([], 10)) + + def bar(): pass + self.assertFalse(bool(bar.__code__.co_flags & inspect.CO_COROUTINE)) + + def test_func_2(self): + async def foo(): + raise StopIteration + + with self.assertRaisesRegex( + RuntimeError, "coroutine raised StopIteration"): + + run_async(foo()) + + def test_func_3(self): + async def foo(): + raise StopIteration + + coro = foo() + self.assertRegex(repr(coro), '^$') + coro.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON; RuntimeError: coroutine raised StopIteration + def test_func_4(self): + async def foo(): + raise StopIteration + coro = foo() + + check = lambda: self.assertRaisesRegex( + TypeError, "'coroutine' object is not iterable") + + with check(): + list(coro) + + with check(): + tuple(coro) + + with check(): + sum(coro) + + with check(): + iter(coro) + + with check(): + for i in coro: + pass + + with check(): + [i for i in coro] + + coro.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised + def test_func_5(self): + @types.coroutine + def bar(): + yield 1 + + async def foo(): + await bar() + + check = lambda: self.assertRaisesRegex( + TypeError, "'coroutine' object is not iterable") + + coro = foo() + with check(): + for el in coro: + pass + coro.close() + + # the following should pass without an error + for el in bar(): + self.assertEqual(el, 1) + self.assertEqual([el for el in bar()], [1]) + self.assertEqual(tuple(bar()), (1,)) + self.assertEqual(next(iter(bar())), 1) + + def test_func_6(self): + @types.coroutine + def bar(): + yield 1 + yield 2 + + async def foo(): + await bar() + + f = foo() + self.assertEqual(f.send(None), 1) + self.assertEqual(f.send(None), 2) + with self.assertRaises(StopIteration): + f.send(None) + + def test_func_7(self): + async def bar(): + return 10 + coro = bar() + + def foo(): + yield from coro + + with self.assertRaisesRegex( + TypeError, + "cannot 'yield from' a coroutine object in " + "a non-coroutine generator"): + list(foo()) + + coro.close() + + def test_func_8(self): + @types.coroutine + def bar(): + return (yield from coro) + + async def foo(): + return 'spam' + + coro = foo() + self.assertEqual(run_async(bar()), ([], 'spam')) + coro.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_func_9(self): + async def foo(): + pass + + with self.assertWarnsRegex( + RuntimeWarning, + r"coroutine '.*test_func_9.*foo' was never awaited"): + + foo() + support.gc_collect() + + with self.assertWarnsRegex( + RuntimeWarning, + r"coroutine '.*test_func_9.*foo' was never awaited"): + + with self.assertRaises(TypeError): + # See bpo-32703. + for _ in foo(): + pass + + support.gc_collect() + + def test_func_10(self): + N = 0 + + @types.coroutine + def gen(): + nonlocal N + try: + a = yield + yield (a ** 2) + except ZeroDivisionError: + N += 100 + raise + finally: + N += 1 + + async def foo(): + await gen() + + coro = foo() + aw = coro.__await__() + self.assertIs(aw, iter(aw)) + next(aw) + self.assertEqual(aw.send(10), 100) + + self.assertEqual(N, 0) + aw.close() + self.assertEqual(N, 1) + + coro = foo() + aw = coro.__await__() + next(aw) + with self.assertRaises(ZeroDivisionError): + aw.throw(ZeroDivisionError()) + self.assertEqual(N, 102) + + coro = foo() + aw = coro.__await__() + next(aw) + with self.assertRaises(ZeroDivisionError): + with self.assertWarns(DeprecationWarning): + aw.throw(ZeroDivisionError, ZeroDivisionError(), None) + + def test_func_11(self): + async def func(): pass + coro = func() + # Test that PyCoro_Type and _PyCoroWrapper_Type types were properly + # initialized + self.assertIn('__await__', dir(coro)) + self.assertIn('__iter__', dir(coro.__await__())) + self.assertIn('coroutine_wrapper', repr(coro.__await__())) + coro.close() # avoid RuntimeWarning + + def test_func_12(self): + async def g(): + me.send(None) + await foo + me = g() + with self.assertRaisesRegex(ValueError, + "coroutine already executing"): + me.send(None) + + def test_func_13(self): + async def g(): + pass + + coro = g() + with self.assertRaisesRegex( + TypeError, + "can't send non-None value to a just-started coroutine"): + coro.send('spam') + + coro.close() + + def test_func_14(self): + @types.coroutine + def gen(): + yield + async def coro(): + try: + await gen() + except GeneratorExit: + await gen() + c = coro() + c.send(None) + with self.assertRaisesRegex(RuntimeError, + "coroutine ignored GeneratorExit"): + c.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON; StopIteration + def test_func_15(self): + # See http://bugs.python.org/issue25887 for details + + async def spammer(): + return 'spam' + async def reader(coro): + return await coro + + spammer_coro = spammer() + + with self.assertRaisesRegex(StopIteration, 'spam'): + reader(spammer_coro).send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + reader(spammer_coro).send(None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; StopIteration + def test_func_16(self): + # See http://bugs.python.org/issue25887 for details + + @types.coroutine + def nop(): + yield + async def send(): + await nop() + return 'spam' + async def read(coro): + await nop() + return await coro + + spammer = send() + + reader = read(spammer) + reader.send(None) + reader.send(None) + with self.assertRaisesRegex(Exception, 'ham'): + reader.throw(Exception('ham')) + + reader = read(spammer) + reader.send(None) + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + reader.send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + reader.throw(Exception('wat')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; StopIteration + def test_func_17(self): + # See http://bugs.python.org/issue25887 for details + + async def coroutine(): + return 'spam' + + coro = coroutine() + with self.assertRaisesRegex(StopIteration, 'spam'): + coro.send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + coro.send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + coro.throw(Exception('wat')) + + # Closing a coroutine shouldn't raise any exception even if it's + # already closed/exhausted (similar to generators) + coro.close() + coro.close() + + def test_func_18(self): + # See http://bugs.python.org/issue25887 for details + + async def coroutine(): + return 'spam' + + coro = coroutine() + await_iter = coro.__await__() + it = iter(await_iter) + + with self.assertRaisesRegex(StopIteration, 'spam'): + it.send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + it.send(None) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + # Although the iterator protocol requires iterators to + # raise another StopIteration here, we don't want to do + # that. In this particular case, the iterator will raise + # a RuntimeError, so that 'yield from' and 'await' + # expressions will trigger the error, instead of silently + # ignoring the call. + next(it) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + it.throw(Exception('wat')) + + with self.assertRaisesRegex(RuntimeError, + 'cannot reuse already awaited coroutine'): + it.throw(Exception('wat')) + + # Closing a coroutine shouldn't raise any exception even if it's + # already closed/exhausted (similar to generators) + it.close() + it.close() + + def test_func_19(self): + CHK = 0 + + @types.coroutine + def foo(): + nonlocal CHK + yield + try: + yield + except GeneratorExit: + CHK += 1 + + async def coroutine(): + await foo() + + coro = coroutine() + + coro.send(None) + coro.send(None) + + self.assertEqual(CHK, 0) + coro.close() + self.assertEqual(CHK, 1) + + for _ in range(3): + # Closing a coroutine shouldn't raise any exception even if it's + # already closed/exhausted (similar to generators) + coro.close() + self.assertEqual(CHK, 1) + + def test_coro_wrapper_send_tuple(self): + async def foo(): + return (10,) + + result = run_async__await__(foo()) + self.assertEqual(result, ([], (10,))) + + def test_coro_wrapper_send_stop_iterator(self): + async def foo(): + return StopIteration(10) + + result = run_async__await__(foo()) + self.assertIsInstance(result[1], StopIteration) + self.assertEqual(result[1].value, 10) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_cr_await(self): + @types.coroutine + def a(): + self.assertEqual(inspect.getcoroutinestate(coro_b), inspect.CORO_RUNNING) + self.assertIsNone(coro_b.cr_await) + yield + self.assertEqual(inspect.getcoroutinestate(coro_b), inspect.CORO_RUNNING) + self.assertIsNone(coro_b.cr_await) + + async def c(): + await a() + + async def b(): + self.assertIsNone(coro_b.cr_await) + await c() + self.assertIsNone(coro_b.cr_await) + + coro_b = b() + self.assertEqual(inspect.getcoroutinestate(coro_b), inspect.CORO_CREATED) + self.assertIsNone(coro_b.cr_await) + + coro_b.send(None) + self.assertEqual(inspect.getcoroutinestate(coro_b), inspect.CORO_SUSPENDED) + self.assertEqual(coro_b.cr_await.cr_await.gi_code.co_name, 'a') + + with self.assertRaises(StopIteration): + coro_b.send(None) # complete coroutine + self.assertEqual(inspect.getcoroutinestate(coro_b), inspect.CORO_CLOSED) + self.assertIsNone(coro_b.cr_await) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not iterable + def test_corotype_1(self): + ct = types.CoroutineType + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('into coroutine', ct.send.__doc__) + self.assertIn('inside coroutine', ct.close.__doc__) + self.assertIn('in coroutine', ct.throw.__doc__) + self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) + self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) + self.assertEqual(ct.__name__, 'coroutine') + + async def f(): pass + c = f() + self.assertIn('coroutine object', repr(c)) + c.close() + + def test_await_1(self): + + async def foo(): + await 1 + with self.assertRaisesRegex(TypeError, "'int' object can.t be awaited"): + run_async(foo()) + + def test_await_2(self): + async def foo(): + await [] + with self.assertRaisesRegex(TypeError, "'list' object can.t be awaited"): + run_async(foo()) + + def test_await_3(self): + async def foo(): + await AsyncYieldFrom([1, 2, 3]) + + self.assertEqual(run_async(foo()), ([1, 2, 3], None)) + self.assertEqual(run_async__await__(foo()), ([1, 2, 3], None)) + + def test_await_4(self): + async def bar(): + return 42 + + async def foo(): + return await bar() + + self.assertEqual(run_async(foo()), ([], 42)) + + def test_await_5(self): + class Awaitable: + def __await__(self): + return + + async def foo(): + return (await Awaitable()) + + with self.assertRaisesRegex( + TypeError, "__await__.*returned non-iterator of type"): + + run_async(foo()) + + def test_await_6(self): + class Awaitable: + def __await__(self): + return iter([52]) + + async def foo(): + return (await Awaitable()) + + self.assertEqual(run_async(foo()), ([52], None)) + + def test_await_7(self): + class Awaitable: + def __await__(self): + yield 42 + return 100 + + async def foo(): + return (await Awaitable()) + + self.assertEqual(run_async(foo()), ([42], 100)) + + def test_await_8(self): + class Awaitable: + pass + + async def foo(): return await Awaitable() + + with self.assertRaisesRegex( + TypeError, "'Awaitable' object can't be awaited"): + + run_async(foo()) + + def test_await_9(self): + def wrap(): + return bar + + async def bar(): + return 42 + + async def foo(): + db = {'b': lambda: wrap} + + class DB: + b = wrap + + return (await bar() + await wrap()() + await db['b']()()() + + await bar() * 1000 + await DB.b()()) + + async def foo2(): + return -await bar() + + self.assertEqual(run_async(foo()), ([], 42168)) + self.assertEqual(run_async(foo2()), ([], -42)) + + def test_await_10(self): + async def baz(): + return 42 + + async def bar(): + return baz() + + async def foo(): + return await (await bar()) + + self.assertEqual(run_async(foo()), ([], 42)) + + def test_await_11(self): + def ident(val): + return val + + async def bar(): + return 'spam' + + async def foo(): + return ident(val=await bar()) + + async def foo2(): + return await bar(), 'ham' + + self.assertEqual(run_async(foo2()), ([], ('spam', 'ham'))) + + def test_await_12(self): + async def coro(): + return 'spam' + c = coro() + + class Awaitable: + def __await__(self): + return c + + async def foo(): + return await Awaitable() + + with self.assertRaisesRegex( + TypeError, r"__await__\(\) returned a coroutine"): + run_async(foo()) + + c.close() + + def test_await_13(self): + class Awaitable: + def __await__(self): + return self + + async def foo(): + return await Awaitable() + + with self.assertRaisesRegex( + TypeError, "__await__.*returned non-iterator of type"): + + run_async(foo()) + + def test_await_14(self): + class Wrapper: + # Forces the interpreter to use CoroutineType.__await__ + def __init__(self, coro): + assert coro.__class__ is types.CoroutineType + self.coro = coro + def __await__(self): + return self.coro.__await__() + + class FutureLike: + def __await__(self): + return (yield) + + class Marker(Exception): + pass + + async def coro1(): + try: + return await FutureLike() + except ZeroDivisionError: + raise Marker + async def coro2(): + return await Wrapper(coro1()) + + c = coro2() + c.send(None) + with self.assertRaisesRegex(StopIteration, 'spam'): + c.send('spam') + + c = coro2() + c.send(None) + with self.assertRaises(Marker): + c.throw(ZeroDivisionError) + + def test_await_15(self): + @types.coroutine + def nop(): + yield + + async def coroutine(): + await nop() + + async def waiter(coro): + await coro + + coro = coroutine() + coro.send(None) + + with self.assertRaisesRegex(RuntimeError, + "coroutine is being awaited already"): + waiter(coro).send(None) + + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except KeyError: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + + def test_await_17(self): + # See https://github.com/python/cpython/issues/131666 for details. + class A: + async def __anext__(self): + raise StopAsyncIteration + def __aiter__(self): + return self + + with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable: + self.assertRaises(TypeError, anext_awaitable.close, 1) + + def test_with_1(self): + class Manager: + def __init__(self, name): + self.name = name + + async def __aenter__(self): + await AsyncYieldFrom(['enter-1-' + self.name, + 'enter-2-' + self.name]) + return self + + async def __aexit__(self, *args): + await AsyncYieldFrom(['exit-1-' + self.name, + 'exit-2-' + self.name]) + + if self.name == 'B': + return True + + + async def foo(): + async with Manager("A") as a, Manager("B") as b: + await AsyncYieldFrom([('managers', a.name, b.name)]) + 1/0 + + f = foo() + result, _ = run_async(f) + + self.assertEqual( + result, ['enter-1-A', 'enter-2-A', 'enter-1-B', 'enter-2-B', + ('managers', 'A', 'B'), + 'exit-1-B', 'exit-2-B', 'exit-1-A', 'exit-2-A'] + ) + + async def foo(): + async with Manager("A") as a, Manager("C") as c: + await AsyncYieldFrom([('managers', a.name, c.name)]) + 1/0 + + with self.assertRaises(ZeroDivisionError): + run_async(foo()) + + def test_with_2(self): + class CM: + def __aenter__(self): + pass + + body_executed = None + async def foo(): + nonlocal body_executed + body_executed = False + async with CM(): + body_executed = True + + with self.assertRaisesRegex(TypeError, 'asynchronous context manager.*__aexit__'): + run_async(foo()) + self.assertIs(body_executed, False) + + def test_with_3(self): + class CM: + def __aexit__(self): + pass + + body_executed = None + async def foo(): + nonlocal body_executed + body_executed = False + async with CM(): + body_executed = True + + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + run_async(foo()) + self.assertIs(body_executed, False) + + def test_with_4(self): + class CM: + pass + + body_executed = None + async def foo(): + nonlocal body_executed + body_executed = False + async with CM(): + body_executed = True + + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + run_async(foo()) + self.assertIs(body_executed, False) + + def test_with_5(self): + # While this test doesn't make a lot of sense, + # it's a regression test for an early bug with opcodes + # generation + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + async def func(): + async with CM(): + self.assertEqual((1, ), 1) + + with self.assertRaises(AssertionError): + run_async(func()) + + def test_with_6(self): + class CM: + def __aenter__(self): + return 123 + + def __aexit__(self, *e): + return 456 + + async def foo(): + async with CM(): + pass + + with self.assertRaisesRegex( + TypeError, + "'async with' received an object from __aenter__ " + "that does not implement __await__: int"): + # it's important that __aexit__ wasn't called + run_async(foo()) + + def test_with_7(self): + class CM: + async def __aenter__(self): + return self + + def __aexit__(self, *e): + return 444 + + # Exit with exception + async def foo(): + async with CM(): + 1/0 + + try: + run_async(foo()) + except TypeError as exc: + self.assertRegex( + exc.args[0], + "'async with' received an object from __aexit__ " + "that does not implement __await__: int") + self.assertTrue(exc.__context__ is not None) + self.assertTrue(isinstance(exc.__context__, ZeroDivisionError)) + else: + self.fail('invalid asynchronous context manager did not fail') + + + def test_with_8(self): + CNT = 0 + + class CM: + async def __aenter__(self): + return self + + def __aexit__(self, *e): + return 456 + + # Normal exit + async def foo(): + nonlocal CNT + async with CM(): + CNT += 1 + with self.assertRaisesRegex( + TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: int"): + run_async(foo()) + self.assertEqual(CNT, 1) + + # Exit with 'break' + async def foo(): + nonlocal CNT + for i in range(2): + async with CM(): + CNT += 1 + break + with self.assertRaisesRegex( + TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: int"): + run_async(foo()) + self.assertEqual(CNT, 2) + + # Exit with 'continue' + async def foo(): + nonlocal CNT + for i in range(2): + async with CM(): + CNT += 1 + continue + with self.assertRaisesRegex( + TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: int"): + run_async(foo()) + self.assertEqual(CNT, 3) + + # Exit with 'return' + async def foo(): + nonlocal CNT + async with CM(): + CNT += 1 + return + with self.assertRaisesRegex( + TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: int"): + run_async(foo()) + self.assertEqual(CNT, 4) + + + def test_with_9(self): + CNT = 0 + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *e): + 1/0 + + async def foo(): + nonlocal CNT + async with CM(): + CNT += 1 + + with self.assertRaises(ZeroDivisionError): + run_async(foo()) + + self.assertEqual(CNT, 1) + + def test_with_10(self): + CNT = 0 + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *e): + 1/0 + + async def foo(): + nonlocal CNT + async with CM(): + async with CM(): + raise RuntimeError + + try: + run_async(foo()) + except ZeroDivisionError as exc: + self.assertTrue(exc.__context__ is not None) + self.assertTrue(isinstance(exc.__context__, ZeroDivisionError)) + self.assertTrue(isinstance(exc.__context__.__context__, + RuntimeError)) + else: + self.fail('exception from __aexit__ did not propagate') + + def test_with_11(self): + CNT = 0 + + class CM: + async def __aenter__(self): + raise NotImplementedError + + async def __aexit__(self, *e): + 1/0 + + async def foo(): + nonlocal CNT + async with CM(): + raise RuntimeError + + try: + run_async(foo()) + except NotImplementedError as exc: + self.assertTrue(exc.__context__ is None) + else: + self.fail('exception from __aenter__ did not propagate') + + def test_with_12(self): + CNT = 0 + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *e): + return True + + async def foo(): + nonlocal CNT + async with CM() as cm: + self.assertIs(cm.__class__, CM) + raise RuntimeError + + run_async(foo()) + + def test_with_13(self): + CNT = 0 + + class CM: + async def __aenter__(self): + 1/0 + + async def __aexit__(self, *e): + return True + + async def foo(): + nonlocal CNT + CNT += 1 + async with CM(): + CNT += 1000 + CNT += 10000 + + with self.assertRaises(ZeroDivisionError): + run_async(foo()) + self.assertEqual(CNT, 1) + + def test_for_1(self): + aiter_calls = 0 + + class AsyncIter: + def __init__(self): + self.i = 0 + + def __aiter__(self): + nonlocal aiter_calls + aiter_calls += 1 + return self + + async def __anext__(self): + self.i += 1 + + if not (self.i % 10): + await AsyncYield(self.i * 10) + + if self.i > 100: + raise StopAsyncIteration + + return self.i, self.i + + + buffer = [] + async def test1(): + async for i1, i2 in AsyncIter(): + buffer.append(i1 + i2) + + yielded, _ = run_async(test1()) + # Make sure that __aiter__ was called only once + self.assertEqual(aiter_calls, 1) + self.assertEqual(yielded, [i * 100 for i in range(1, 11)]) + self.assertEqual(buffer, [i*2 for i in range(1, 101)]) + + + buffer = [] + async def test2(): + nonlocal buffer + async for i in AsyncIter(): + buffer.append(i[0]) + if i[0] == 20: + break + else: + buffer.append('what?') + buffer.append('end') + + yielded, _ = run_async(test2()) + # Make sure that __aiter__ was called only once + self.assertEqual(aiter_calls, 2) + self.assertEqual(yielded, [100, 200]) + self.assertEqual(buffer, [i for i in range(1, 21)] + ['end']) + + + buffer = [] + async def test3(): + nonlocal buffer + async for i in AsyncIter(): + if i[0] > 20: + continue + buffer.append(i[0]) + else: + buffer.append('what?') + buffer.append('end') + + yielded, _ = run_async(test3()) + # Make sure that __aiter__ was called only once + self.assertEqual(aiter_calls, 3) + self.assertEqual(yielded, [i * 100 for i in range(1, 11)]) + self.assertEqual(buffer, [i for i in range(1, 21)] + + ['what?', 'end']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ + def test_for_2(self): + tup = (1, 2, 3) + refs_before = sys.getrefcount(tup) + + async def foo(): + async for i in tup: + print('never going to happen') + + with self.assertRaisesRegex( + TypeError, "async for' requires an object.*__aiter__.*tuple"): + + run_async(foo()) + + self.assertEqual(sys.getrefcount(tup), refs_before) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "that does not implement __anext__" does not match "'async for' requires an iterator with __anext__ method, got I" + def test_for_3(self): + class I: + def __aiter__(self): + return self + + aiter = I() + refs_before = sys.getrefcount(aiter) + + async def foo(): + async for i in aiter: + print('never going to happen') + + with self.assertRaisesRegex( + TypeError, + r"that does not implement __anext__"): + + run_async(foo()) + + self.assertEqual(sys.getrefcount(aiter), refs_before) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "async for' received an invalid object.*__anext__.*tuple" does not match "'tuple' object is not an iterator" + def test_for_4(self): + class I: + def __aiter__(self): + return self + + def __anext__(self): + return () + + aiter = I() + refs_before = sys.getrefcount(aiter) + + async def foo(): + async for i in aiter: + print('never going to happen') + + with self.assertRaisesRegex( + TypeError, + "async for' received an invalid object.*__anext__.*tuple"): + + run_async(foo()) + + self.assertEqual(sys.getrefcount(aiter), refs_before) + + def test_for_6(self): + I = 0 + + class Manager: + async def __aenter__(self): + nonlocal I + I += 10000 + + async def __aexit__(self, *args): + nonlocal I + I += 100000 + + class Iterable: + def __init__(self): + self.i = 0 + + def __aiter__(self): + return self + + async def __anext__(self): + if self.i > 10: + raise StopAsyncIteration + self.i += 1 + return self.i + + ############## + + manager = Manager() + iterable = Iterable() + mrefs_before = sys.getrefcount(manager) + irefs_before = sys.getrefcount(iterable) + + async def main(): + nonlocal I + + async with manager: + async for i in iterable: + I += 1 + I += 1000 + + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that __aiter__ that returns an asynchronous iterator + # directly does not throw any warnings. + run_async(main()) + self.assertEqual(I, 111011) + + self.assertEqual(sys.getrefcount(manager), mrefs_before) + self.assertEqual(sys.getrefcount(iterable), irefs_before) + + ############## + + async def main(): + nonlocal I + + async with Manager(): + async for i in Iterable(): + I += 1 + I += 1000 + + async with Manager(): + async for i in Iterable(): + I += 1 + I += 1000 + + run_async(main()) + self.assertEqual(I, 333033) + + ############## + + async def main(): + nonlocal I + + async with Manager(): + I += 100 + async for i in Iterable(): + I += 1 + else: + I += 10000000 + I += 1000 + + async with Manager(): + I += 100 + async for i in Iterable(): + I += 1 + else: + I += 10000000 + I += 1000 + + run_async(main()) + self.assertEqual(I, 20555255) + + def test_for_7(self): + CNT = 0 + class AI: + def __aiter__(self): + 1/0 + async def foo(): + nonlocal CNT + async for i in AI(): + CNT += 1 + CNT += 10 + with self.assertRaises(ZeroDivisionError): + run_async(foo()) + self.assertEqual(CNT, 0) + + def test_for_8(self): + CNT = 0 + class AI: + def __aiter__(self): + 1/0 + async def foo(): + nonlocal CNT + async for i in AI(): + CNT += 1 + CNT += 10 + with self.assertRaises(ZeroDivisionError): + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that if __aiter__ raises an exception it propagates + # without any kind of warning. + run_async(foo()) + self.assertEqual(CNT, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "an invalid object from __anext__" does not match "'F' object is not an iterator" + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_tuple(self): + class Done(Exception): pass + + class AIter(tuple): + i = 0 + def __aiter__(self): + return self + async def __anext__(self): + if self.i >= len(self): + raise StopAsyncIteration + self.i += 1 + return self[self.i - 1] + + result = [] + async def foo(): + async for i in AIter([42]): + result.append(i) + raise Done + + with self.assertRaises(Done): + foo().send(None) + self.assertEqual(result, [42]) + + def test_for_stop_iteration(self): + class Done(Exception): pass + + class AIter(StopIteration): + i = 0 + def __aiter__(self): + return self + async def __anext__(self): + if self.i: + raise StopAsyncIteration + self.i += 1 + return self.value + + result = [] + async def foo(): + async for i in AIter(42): + result.append(i) + raise Done + + with self.assertRaises(Done): + foo().send(None) + self.assertEqual(result, [42]) + + def test_comp_1(self): + async def f(i): + return i + + async def run_list(): + return [await c for c in [f(1), f(41)]] + + async def run_set(): + return {await c for c in [f(1), f(41)]} + + async def run_dict1(): + return {await c: 'a' for c in [f(1), f(41)]} + + async def run_dict2(): + return {i: await c for i, c in enumerate([f(1), f(41)])} + + self.assertEqual(run_async(run_list()), ([], [1, 41])) + self.assertEqual(run_async(run_set()), ([], {1, 41})) + self.assertEqual(run_async(run_dict1()), ([], {1: 'a', 41: 'a'})) + self.assertEqual(run_async(run_dict2()), ([], {0: 1, 1: 41})) + + def test_comp_2(self): + async def f(i): + return i + + async def run_list(): + return [s for c in [f(''), f('abc'), f(''), f(['de', 'fg'])] + for s in await c] + + self.assertEqual( + run_async(run_list()), + ([], ['a', 'b', 'c', 'de', 'fg'])) + + async def run_set(): + return {d + for c in [f([f([10, 30]), + f([20])])] + for s in await c + for d in await s} + + self.assertEqual( + run_async(run_set()), + ([], {10, 20, 30})) + + async def run_set2(): + return {await s + for c in [f([f(10), f(20)])] + for s in await c} + + self.assertEqual( + run_async(run_set2()), + ([], {10, 20})) + + def test_comp_3(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for i in f([10, 20])] + self.assertEqual( + run_async(run_list()), + ([], [11, 21])) + + async def run_set(): + return {i + 1 async for i in f([10, 20])} + self.assertEqual( + run_async(run_set()), + ([], {11, 21})) + + async def run_dict(): + return {i + 1: i + 2 async for i in f([10, 20])} + self.assertEqual( + run_async(run_dict()), + ([], {11: 12, 21: 22})) + + async def run_gen(): + gen = (i + 1 async for i in f([10, 20])) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 121])) + + def test_comp_4(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for i in f([10, 20]) if i > 10] + self.assertEqual( + run_async(run_list()), + ([], [21])) + + async def run_set(): + return {i + 1 async for i in f([10, 20]) if i > 10} + self.assertEqual( + run_async(run_set()), + ([], {21})) + + async def run_dict(): + return {i + 1: i + 2 async for i in f([10, 20]) if i > 10} + self.assertEqual( + run_async(run_dict()), + ([], {21: 22})) + + async def run_gen(): + gen = (i + 1 async for i in f([10, 20]) if i > 10) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [121])) + + def test_comp_4_2(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 10 async for i in f(range(5)) if 0 < i < 4] + self.assertEqual( + run_async(run_list()), + ([], [11, 12, 13])) + + async def run_set(): + return {i + 10 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_set()), + ([], {11, 12, 13})) + + async def run_dict(): + return {i + 10: i + 100 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_dict()), + ([], {11: 101, 12: 102, 13: 103})) + + async def run_gen(): + gen = (i + 10 async for i in f(range(5)) if 0 < i < 4) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 112, 113])) + + def test_comp_5(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 for pair in ([10, 20], [30, 40]) if pair[0] > 10 + async for i in f(pair) if i > 30] + self.assertEqual( + run_async(run_list()), + ([], [41])) + + def test_comp_6(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 1 async for seq in f([(10, 20), (30,)]) + for i in seq] + + self.assertEqual( + run_async(run_list()), + ([], [11, 21, 31])) + + def test_comp_7(self): + async def f(): + yield 1 + yield 2 + raise Exception('aaa') + + async def run_list(): + return [i async for i in f()] + + with self.assertRaisesRegex(Exception, 'aaa'): + run_async(run_list()) + + def test_comp_8(self): + async def f(): + return [i for i in [1, 2, 3]] + + self.assertEqual( + run_async(f()), + ([], [1, 2, 3])) + + def test_comp_9(self): + async def gen(): + yield 1 + yield 2 + async def f(): + l = [i async for i in gen()] + return [i for i in l] + + self.assertEqual( + run_async(f()), + ([], [1, 2])) + + def test_comp_10(self): + async def f(): + xx = {i for i in [1, 2, 3]} + return {x: x for x in xx} + + self.assertEqual( + run_async(f()), + ([], {1: 1, 2: 2, 3: 3})) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ + def test_nested_comp(self): + async def run_list_inside_list(): + return [[i + j async for i in asynciter([1, 2])] for j in [10, 20]] + self.assertEqual( + run_async(run_list_inside_list()), + ([], [[11, 12], [21, 22]])) + + async def run_set_inside_list(): + return [{i + j async for i in asynciter([1, 2])} for j in [10, 20]] + self.assertEqual( + run_async(run_set_inside_list()), + ([], [{11, 12}, {21, 22}])) + + async def run_list_inside_set(): + return {sum([i async for i in asynciter(range(j))]) for j in [3, 5]} + self.assertEqual( + run_async(run_list_inside_set()), + ([], {3, 10})) + + async def run_dict_inside_dict(): + return {j: {i: i + j async for i in asynciter([1, 2])} for j in [10, 20]} + self.assertEqual( + run_async(run_dict_inside_dict()), + ([], {10: {1: 11, 2: 12}, 20: {1: 21, 2: 22}})) + + async def run_list_inside_gen(): + gen = ([i + j async for i in asynciter([1, 2])] for j in [10, 20]) + return [x async for x in gen] + self.assertEqual( + run_async(run_list_inside_gen()), + ([], [[11, 12], [21, 22]])) + + async def run_gen_inside_list(): + gens = [(i async for i in asynciter(range(j))) for j in [3, 5]] + return [x for g in gens async for x in g] + self.assertEqual( + run_async(run_gen_inside_list()), + ([], [0, 1, 2, 0, 1, 2, 3, 4])) + + async def run_gen_inside_gen(): + gens = ((i async for i in asynciter(range(j))) for j in [3, 5]) + return [x for g in gens async for x in g] + self.assertEqual( + run_async(run_gen_inside_gen()), + ([], [0, 1, 2, 0, 1, 2, 3, 4])) + + async def run_list_inside_list_inside_list(): + return [[[i + j + k async for i in asynciter([1, 2])] + for j in [10, 20]] + for k in [100, 200]] + self.assertEqual( + run_async(run_list_inside_list_inside_list()), + ([], [[[111, 112], [121, 122]], [[211, 212], [221, 222]]])) + + def test_copy(self): + async def func(): pass + coro = func() + with self.assertRaises(TypeError): + copy.copy(coro) + + aw = coro.__await__() + try: + with self.assertRaises(TypeError): + copy.copy(aw) + finally: + aw.close() + + def test_pickle(self): + async def func(): pass + coro = func() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises((TypeError, pickle.PicklingError)): + pickle.dumps(coro, proto) + + aw = coro.__await__() + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises((TypeError, pickle.PicklingError)): + pickle.dumps(aw, proto) + finally: + aw.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'err_msg' + def test_fatal_coro_warning(self): + # Issue 27811 + async def func(): pass + with warnings.catch_warnings(), \ + support.catch_unraisable_exception() as cm: + warnings.filterwarnings("error") + coro = func() + # only store repr() to avoid keeping the coroutine alive + coro_repr = repr(coro) + coro = None + support.gc_collect() + + self.assertEqual(cm.unraisable.err_msg, + f"Exception ignored while finalizing " + f"coroutine {coro_repr}") + self.assertIn("was never awaited", str(cm.unraisable.exc_value)) + + def test_for_assign_raising_stop_async_iteration(self): + class BadTarget: + def __setitem__(self, key, value): + raise StopAsyncIteration(42) + tgt = BadTarget() + async def source(): + yield 10 + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for tgt[0] in source(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for tgt[0] in source()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for tgt[0] in source()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + + def test_for_assign_raising_stop_async_iteration_2(self): + class BadIterable: + def __iter__(self): + raise StopAsyncIteration(42) + async def badpairs(): + yield BadIterable() + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for i, j in badpairs(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for i, j in badpairs()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for i, j in badpairs()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; This would crash the interpreter in 3.11a2 + def test_bpo_45813_1(self): + 'This would crash the interpreter in 3.11a2' + async def f(): + pass + with self.assertWarns(RuntimeWarning): + frame = f().cr_frame + frame.clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON; This would crash the interpreter in 3.11a2 + def test_bpo_45813_2(self): + 'This would crash the interpreter in 3.11a2' + async def f(): + pass + gen = f() + with self.assertWarns(RuntimeWarning): + gen.cr_frame.clear() + gen.close() + + def test_cr_frame_after_close(self): + async def f(): + pass + gen = f() + self.assertIsNotNone(gen.cr_frame) + gen.close() + self.assertIsNone(gen.cr_frame) + + def test_stack_in_coroutine_throw(self): + # Regression test for https://github.com/python/cpython/issues/93592 + async def a(): + return await b() + + async def b(): + return await c() + + @types.coroutine + def c(): + try: + # traceback.print_stack() + yield len(traceback.extract_stack()) + except ZeroDivisionError: + # traceback.print_stack() + yield len(traceback.extract_stack()) + + coro = a() + len_send = coro.send(None) + len_throw = coro.throw(ZeroDivisionError) + # before fixing, visible stack from throw would be shorter than from send. + self.assertEqual(len_send, len_throw) + + def test_call_aiter_once_in_comprehension(self): + + class AsyncIterator: + + def __init__(self): + self.val = 0 + + async def __anext__(self): + if self.val == 2: + raise StopAsyncIteration + self.val += 1 + return self.val + + # No __aiter__ method + + class C: + + def __aiter__(self): + return AsyncIterator() + + async def run_listcomp(): + return [i async for i in C()] + + async def run_asyncgen(): + ag = (i async for i in C()) + return [i async for i in ag] + + self.assertEqual(run_async(run_listcomp()), ([], [1, 2])) + self.assertEqual(run_async(run_asyncgen()), ([], [1, 2])) + + +@unittest.skipIf( + support.is_emscripten or support.is_wasi, + "asyncio does not work under Emscripten/WASI yet." +) +class CoroAsyncIOCompatTest(unittest.TestCase): + + def test_asyncio_1(self): + # asyncio cannot be imported when Python is compiled without thread + # support + asyncio = import_helper.import_module('asyncio') + + class MyException(Exception): + pass + + buffer = [] + + class CM: + async def __aenter__(self): + buffer.append(1) + await asyncio.sleep(0.01) + buffer.append(2) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await asyncio.sleep(0.01) + buffer.append(exc_type.__name__) + + async def f(): + async with CM(): + await asyncio.sleep(0.01) + raise MyException + buffer.append('unreachable') + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(f()) + except MyException: + pass + finally: + loop.close() + asyncio.events._set_event_loop_policy(None) + + self.assertEqual(buffer, [1, 2, 'MyException']) + + +class OriginTrackingTest(unittest.TestCase): + def here(self): + info = inspect.getframeinfo(inspect.currentframe().f_back) + return (info.filename, info.lineno) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != (('/home/fanninpm/Documents/GitHub/RustPy[74 chars]g'),) + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + try: + async def corofn(): + pass + + sys.set_coroutine_origin_tracking_depth(0) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) + + with contextlib.closing(corofn()) as coro: + self.assertIsNone(coro.cr_origin) + + sys.set_coroutine_origin_tracking_depth(1) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 1) + + fname, lineno = self.here() + with contextlib.closing(corofn()) as coro: + self.assertEqual(coro.cr_origin, + ((fname, lineno + 1, "test_origin_tracking"),)) + + sys.set_coroutine_origin_tracking_depth(2) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 2) + + def nested(): + return (self.here(), corofn()) + fname, lineno = self.here() + ((nested_fname, nested_lineno), coro) = nested() + with contextlib.closing(coro): + self.assertEqual(coro.cr_origin, + ((nested_fname, nested_lineno, "nested"), + (fname, lineno + 1, "test_origin_tracking"))) + + # Check we handle running out of frames correctly + sys.set_coroutine_origin_tracking_depth(1000) + with contextlib.closing(corofn()) as coro: + self.assertTrue(2 < len(coro.cr_origin) < 1000) + + # We can't set depth negative + with self.assertRaises(ValueError): + sys.set_coroutine_origin_tracking_depth(-1) + # And trying leaves it unchanged + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 1000) + + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_origin_tracking_warning(self): + async def corofn(): + pass + + a1_filename, a1_lineno = self.here() + def a1(): + return corofn() # comment in a1 + a1_lineno += 2 + + a2_filename, a2_lineno = self.here() + def a2(): + return a1() # comment in a2 + a2_lineno += 2 + + def check(depth, msg): + sys.set_coroutine_origin_tracking_depth(depth) + with self.assertWarns(RuntimeWarning) as cm: + a2() + support.gc_collect() + self.assertEqual(msg, str(cm.warning)) + + orig_depth = sys.get_coroutine_origin_tracking_depth() + try: + check(0, f"coroutine '{corofn.__qualname__}' was never awaited") + check(1, "".join([ + f"coroutine '{corofn.__qualname__}' was never awaited\n", + "Coroutine created at (most recent call last)\n", + f' File "{a1_filename}", line {a1_lineno}, in a1\n', + " return corofn() # comment in a1", + ])) + check(2, "".join([ + f"coroutine '{corofn.__qualname__}' was never awaited\n", + "Coroutine created at (most recent call last)\n", + f' File "{a2_filename}", line {a2_lineno}, in a2\n', + " return a1() # comment in a2\n", + f' File "{a1_filename}", line {a1_lineno}, in a1\n', + " return corofn() # comment in a1", + ])) + + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'err_msg' + def test_unawaited_warning_when_module_broken(self): + # Make sure we don't blow up too bad if + # warnings._warn_unawaited_coroutine is broken somehow (e.g. because + # of shutdown problems) + async def corofn(): + pass + + orig_wuc = warnings._warn_unawaited_coroutine + try: + warnings._warn_unawaited_coroutine = lambda coro: 1/0 + with support.catch_unraisable_exception() as cm, \ + warnings_helper.check_warnings( + (r'coroutine .* was never awaited', + RuntimeWarning)): + # only store repr() to avoid keeping the coroutine alive + coro = corofn() + coro_repr = repr(coro) + + # clear reference to the coroutine without awaiting for it + del coro + support.gc_collect() + + self.assertEqual(cm.unraisable.err_msg, + f"Exception ignored while finalizing " + f"coroutine {coro_repr}") + self.assertEqual(cm.unraisable.exc_type, ZeroDivisionError) + + del warnings._warn_unawaited_coroutine + with warnings_helper.check_warnings( + (r'coroutine .* was never awaited', RuntimeWarning)): + corofn() + support.gc_collect() + + finally: + warnings._warn_unawaited_coroutine = orig_wuc + + +class UnawaitedWarningDuringShutdownTest(unittest.TestCase): + # https://bugs.python.org/issue32591#msg310726 + def test_unawaited_warning_during_shutdown(self): + code = ("import asyncio\n" + "async def f(): pass\n" + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") + assert_python_ok("-c", code) + + code = ("import sys\n" + "async def f(): pass\n" + "sys.coro = f()\n") + assert_python_ok("-c", code) + + code = ("import sys\n" + "async def f(): pass\n" + "sys.corocycle = [f()]\n" + "sys.corocycle.append(sys.corocycle)\n") + assert_python_ok("-c", code) + + +@support.cpython_only +@unittest.skipIf(_testcapi is None, "requires _testcapi") +class CAPITest(unittest.TestCase): + + def test_tp_await_1(self): + from _testcapi import awaitType as at + + async def foo(): + future = at(iter([1])) + return (await future) + + self.assertEqual(foo().send(None), 1) + + def test_tp_await_2(self): + # Test tp_await to __await__ mapping + from _testcapi import awaitType as at + future = at(iter([1])) + self.assertEqual(next(future.__await__()), 1) + + def test_tp_await_3(self): + from _testcapi import awaitType as at + + async def foo(): + future = at(1) + return (await future) + + with self.assertRaisesRegex( + TypeError, "__await__.*returned non-iterator of type 'int'"): + self.assertEqual(foo().send(None), 1) + + +if __name__=="__main__": + unittest.main() diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index b7f93d1bac9..8af2f0b337c 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # csv package unit tests import copy @@ -10,7 +10,8 @@ import gc import pickle from test import support -from test.support import import_helper, check_disallow_instantiation +from test.support import cpython_only, import_helper, check_disallow_instantiation +from test.support.import_helper import ensure_lazy_imports from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -86,12 +87,12 @@ def _test_arg_valid(self, ctor, arg): self.assertRaises(ValueError, ctor, arg, quotechar='\x85', lineterminator='\x85') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) self.assertRaises(OSError, csv.reader, BadIterable()) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_writer_arg_valid(self): self._test_arg_valid(csv.writer, StringIO()) class BadWriter: @@ -212,7 +213,7 @@ def test_write_bigfield(self): self._write_test([bigstring,bigstring], '%s,%s' % \ (bigstring, bigstring)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_quoting(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"') self._write_error_test(csv.Error, ['a',1,'p,q'], @@ -230,7 +231,7 @@ def test_write_quoting(self): self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', escapechar='\\') @@ -262,7 +263,7 @@ def test_write_escape(self): self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_lineterminator(self): for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': with self.subTest(lineterminator=lineterminator): @@ -276,7 +277,7 @@ def test_write_lineterminator(self): f'1,2{lineterminator}' f'"\r","\n"{lineterminator}') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') self._write_test(iter(['a', 1, None]), 'a,1,') @@ -319,7 +320,7 @@ def test_writerows_with_none(self): self.assertEqual(fileobj.read(), 'a\r\n""\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_empty_fields(self): self._write_test((), '') self._write_test([''], '""') @@ -333,7 +334,7 @@ def test_write_empty_fields(self): self._write_test(['', ''], ',') self._write_test([None, None], ',') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_empty_fields_space_delimiter(self): self._write_test([''], '""', delimiter=' ', skipinitialspace=False) self._write_test([''], '""', delimiter=' ', skipinitialspace=True) @@ -374,7 +375,7 @@ def _read_test(self, input, expect, **kwargs): result = list(reader) self.assertEqual(result, expect) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_oddinputs(self): self._read_test([], []) self._read_test([''], [[]]) @@ -385,7 +386,7 @@ def test_read_oddinputs(self): self.assertRaises(csv.Error, self._read_test, [b'abc'], None) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eol(self): self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']]) self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']]) @@ -400,7 +401,7 @@ def test_read_eol(self): with self.assertRaisesRegex(csv.Error, errmsg): next(csv.reader(['a,b\r\nc,d'])) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eof(self): self._read_test(['a,"'], [['a', '']]) self._read_test(['"a'], [['a']]) @@ -410,7 +411,7 @@ def test_read_eof(self): self.assertRaises(csv.Error, self._read_test, ['^'], [], escapechar='^', strict=True) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_nul(self): self._read_test(['\0'], [['\0']]) self._read_test(['a,\0b,c'], [['a', '\0b', 'c']]) @@ -423,7 +424,7 @@ def test_read_delimiter(self): self._read_test(['a;b;c'], [['a', 'b', 'c']], delimiter=';') self._read_test(['a\0b\0c'], [['a', 'b', 'c']], delimiter='\0') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', 'b', 'c']], escapechar='\\') self._read_test(['a,b\\,c'], [['a', 'b,c']], escapechar='\\') @@ -436,7 +437,7 @@ def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', '\\b', 'c']], escapechar=None) self._read_test(['a,\\b,c'], [['a', '\\b', 'c']]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_quoting(self): self._read_test(['1,",3,",5'], [['1', ',3,', '5']]) self._read_test(['1,",3,",5'], [['1', '"', '3', '"', '5']], @@ -473,7 +474,7 @@ def test_read_quoting(self): self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], quoting=csv.QUOTE_STRINGS, escapechar='\\') - @unittest.skip('TODO: RUSTPYTHON; slice index starts at 1 but ends at 0') + @unittest.skip("TODO: RUSTPYTHON; slice index starts at 1 but ends at 0") def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], @@ -488,7 +489,7 @@ def test_read_skipinitialspace(self): [[None, None, None]], skipinitialspace=True, quoting=csv.QUOTE_STRINGS) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_space_delimiter(self): self._read_test(['a b', ' a ', ' ', ''], [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], @@ -528,7 +529,7 @@ def test_read_linenum(self): self.assertRaises(StopIteration, next, r) self.assertEqual(r.line_num, 3) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_quoteed_newlines(self): rows = [ ['\na', 'b\nc', 'd\n'], @@ -547,7 +548,7 @@ def test_roundtrip_quoteed_newlines(self): for i, row in enumerate(csv.reader(fileobj)): self.assertEqual(row, rows[i]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_escaped_unquoted_newlines(self): rows = [ ['\na', 'b\nc', 'd\n'], @@ -662,7 +663,7 @@ def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dialect_apply(self): class testA(csv.excel): delimiter = "\t" @@ -698,7 +699,6 @@ def test_copy(self): dialect = csv.get_dialect(name) self.assertRaises(TypeError, copy.copy, dialect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) @@ -785,7 +785,7 @@ def test_quoted_quote(self): '"I see," said the blind man', 'as he picked up his hammer and saw']]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_quoted_nl(self): input = '''\ 1,2,3,"""I see,"" @@ -826,18 +826,18 @@ class EscapedExcel(csv.excel): class TestEscapedExcel(TestCsvBase): dialect = EscapedExcel() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('abc\\,def\r\n', [['abc,def']]) class TestDialectUnix(TestCsvBase): dialect = 'unix' - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_writer(self): self.writerAssertEqual([[1, 'abc def', 'abc']], '"1","abc def","abc"\n') @@ -854,7 +854,7 @@ class TestQuotedEscapedExcel(TestCsvBase): def test_write_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], '"abc,def"\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']]) @@ -942,6 +942,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -957,6 +965,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1051,7 +1060,7 @@ def test_read_multi(self): "s1": 'abc', "s2": 'def'}) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_with_blanks(self): reader = csv.DictReader(["1,2,abc,4,5,6\r\n","\r\n", "1,2,abc,4,5,6\r\n"], @@ -1103,7 +1112,7 @@ def test_float_write(self): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_char_write(self): import array, string a = array.array('w', string.ascii_letters) @@ -1148,19 +1157,22 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be string or None, not int') + '"quotechar" must be a unicode character or None, ' + 'not int') def test_delimiter(self): class mydialect(csv.Dialect): @@ -1177,31 +1189,32 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, ' + 'not a string of length 3') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, not a string of length 0') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not bytes') + '"delimiter" must be a unicode character, not bytes') mydialect.delimiter = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not int') + '"delimiter" must be a unicode character, not int') mydialect.delimiter = None with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not NoneType') + '"delimiter" must be a unicode character, not NoneType') def test_escapechar(self): class mydialect(csv.Dialect): @@ -1215,20 +1228,32 @@ class mydialect(csv.Dialect): self.assertEqual(d.escapechar, "\\") mydialect.escapechar = "" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.escapechar = "**" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.escapechar = b"*" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not bytes'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not bytes') mydialect.escapechar = 4 - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not int'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not int') def test_lineterminator(self): class mydialect(csv.Dialect): @@ -1249,9 +1274,15 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"lineterminator" must be a string') + '"lineterminator" must be a string, not int') + + mydialect.lineterminator = None + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"lineterminator" must be a string, not NoneType') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_chars(self): def create_invalid(field_name, value, **kwargs): class mydialect(csv.Dialect): @@ -1358,6 +1389,19 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1379,6 +1423,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1428,6 +1477,10 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) def test_doublequote(self): sniffer = csv.Sniffer() @@ -1593,6 +1646,10 @@ class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, csv, ('csv', '_csv')) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("csv", {"re"}) + def test_subclassable(self): # issue 44089 class Foo(csv.Error): ... diff --git a/Lib/test/test_ctypes.py b/Lib/test/test_ctypes.py deleted file mode 100644 index b0a12c97347..00000000000 --- a/Lib/test/test_ctypes.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest -from test.support.import_helper import import_module - - -ctypes_test = import_module('ctypes.test') - -load_tests = ctypes_test.load_tests - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_ctypes/__init__.py b/Lib/test/test_ctypes/__init__.py new file mode 100644 index 00000000000..eb9126cbe18 --- /dev/null +++ b/Lib/test/test_ctypes/__init__.py @@ -0,0 +1,10 @@ +import os +from test import support +from test.support import import_helper + + +# skip tests if the _ctypes extension was not built +import_helper.import_module('ctypes') + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ctypes/__main__.py b/Lib/test/test_ctypes/__main__.py new file mode 100644 index 00000000000..3003d4db890 --- /dev/null +++ b/Lib/test/test_ctypes/__main__.py @@ -0,0 +1,4 @@ +from test.test_ctypes import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py new file mode 100644 index 00000000000..946d654a19a --- /dev/null +++ b/Lib/test/test_ctypes/_support.py @@ -0,0 +1,151 @@ +# Some classes and types are not export to _ctypes module directly. + +import ctypes +from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr +import sys +from test import support + + +_CData = Structure.__base__ +assert _CData.__name__ == "_CData" + +# metaclasses +PyCStructType = type(Structure) +UnionType = type(Union) +PyCPointerType = type(_Pointer) +PyCArrayType = type(Array) +PyCSimpleType = type(_SimpleCData) +PyCFuncPtrType = type(CFuncPtr) + +# type flags +Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 +Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 + + +def is_underaligned(ctype): + """Return true when type's alignment is less than its size. + + A famous example is 64-bit int on 32-bit x86. + """ + return ctypes.alignment(ctype) < ctypes.sizeof(ctype) + + +class StructCheckMixin: + def check_struct(self, structure): + """Assert that a structure is well-formed""" + self._check_struct_or_union(structure, is_struct=True) + + def check_union(self, union): + """Assert that a union is well-formed""" + self._check_struct_or_union(union, is_struct=False) + + def check_struct_or_union(self, cls): + if issubclass(cls, Structure): + self._check_struct_or_union(cls, is_struct=True) + elif issubclass(cls, Union): + self._check_struct_or_union(cls, is_struct=False) + else: + raise TypeError(cls) + + def _check_struct_or_union(self, cls, is_struct): + + # Check that fields are not overlapping (for structs), + # and that their metadata is consistent. + + used_bits = 0 + + is_little_endian = ( + hasattr(cls, '_swappedbytes_') ^ (sys.byteorder == 'little')) + + anon_names = getattr(cls, '_anonymous_', ()) + cls_size = ctypes.sizeof(cls) + for name, requested_type, *rest_of_tuple in cls._fields_: + field = getattr(cls, name) + with self.subTest(name=name, field=field): + is_bitfield = len(rest_of_tuple) > 0 + + # name + self.assertEqual(field.name, name) + + # type + self.assertEqual(field.type, requested_type) + + # offset === byte_offset + self.assertEqual(field.byte_offset, field.offset) + if not is_struct: + self.assertEqual(field.byte_offset, 0) + + # byte_size + self.assertEqual(field.byte_size, ctypes.sizeof(field.type)) + self.assertGreaterEqual(field.byte_size, 0) + + # Check that the field is inside the struct. + # See gh-130410 for why this is skipped for bitfields of + # underaligned types. Later in this function (see `bit_end`) + # we assert that the value *bits* are inside the struct. + if not (field.is_bitfield and is_underaligned(field.type)): + self.assertLessEqual(field.byte_offset + field.byte_size, + cls_size) + + # size + self.assertGreaterEqual(field.size, 0) + if is_bitfield: + # size has backwards-compatible bit-packed info + expected_size = (field.bit_size << 16) + field.bit_offset + self.assertEqual(field.size, expected_size) + else: + # size == byte_size + self.assertEqual(field.size, field.byte_size) + + # is_bitfield (bool) + self.assertIs(field.is_bitfield, is_bitfield) + + # bit_offset + if is_bitfield: + self.assertGreaterEqual(field.bit_offset, 0) + self.assertLessEqual(field.bit_offset + field.bit_size, + field.byte_size * 8) + else: + self.assertEqual(field.bit_offset, 0) + if not is_struct: + if is_little_endian: + self.assertEqual(field.bit_offset, 0) + else: + self.assertEqual(field.bit_offset, + field.byte_size * 8 - field.bit_size) + + # bit_size + if is_bitfield: + self.assertGreaterEqual(field.bit_size, 0) + self.assertLessEqual(field.bit_size, field.byte_size * 8) + [requested_bit_size] = rest_of_tuple + self.assertEqual(field.bit_size, requested_bit_size) + else: + self.assertEqual(field.bit_size, field.byte_size * 8) + + # is_anonymous (bool) + self.assertIs(field.is_anonymous, name in anon_names) + + # In a struct, field should not overlap. + # (Test skipped if the structs is enormous.) + if is_struct and cls_size < 10_000: + # Get a mask indicating where the field is within the struct + if is_little_endian: + tp_shift = field.byte_offset * 8 + else: + tp_shift = (cls_size + - field.byte_offset + - field.byte_size) * 8 + mask = (1 << field.bit_size) - 1 + mask <<= (tp_shift + field.bit_offset) + assert mask.bit_count() == field.bit_size + # Check that these bits aren't shared with previous fields + self.assertEqual(used_bits & mask, 0) + # Mark the bits for future checks + used_bits |= mask + + # field is inside cls + bit_end = (field.byte_offset * 8 + + field.bit_offset + + field.bit_size) + self.assertLessEqual(bit_end, cls_size * 8) diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py new file mode 100644 index 00000000000..50b4d729b9d --- /dev/null +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -0,0 +1,343 @@ +from ctypes import ( + c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, + BigEndianStructure, LittleEndianStructure, + BigEndianUnion, LittleEndianUnion, Structure, +) +import struct +import unittest +from ._support import StructCheckMixin + +class TestAlignedStructures(unittest.TestCase, StructCheckMixin): + def test_aligned_string(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] + self.check_struct(Aligned) + + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(main.first, 7) + self.assertEqual(main.string.value, b'hello world!') + self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0') + self.assertEqual(Main.string.offset, 16) + self.assertEqual(Main.string.size, 16) + self.assertEqual(alignment(main.string), 16) + self.assertEqual(alignment(main), 16) + + def test_aligned_structures(self): + for base, data in ( + (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ): + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] + self.check_struct(SomeBools) + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(alignment(SomeBools), 4) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.y), 4) + self.assertEqual(Main.x.size, 1) + self.assertEqual(Main.y.offset, 4) + self.assertEqual(Main.y.size, 4) + self.assertEqual(main.y.bool1, True) + self.assertEqual(main.y.bool2, False) + self.assertEqual(Main.z.offset, 8) + self.assertEqual(main.z, 7) + + def test_negative_align(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with ( + self.subTest(base=base), + self.assertRaisesRegex( + ValueError, + '_align_ must be a non-negative integer', + ) + ): + class MyStructure(base): + _align_ = -1 + _fields_ = [] + + def test_zero_align_no_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + + def test_zero_align_with_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [ + ("x", c_ubyte), + ] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + + def test_oversized_structure(self): + data = bytearray(b"\0" * 8) + for base in (LittleEndianStructure, BigEndianStructure): + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] + self.check_struct(SomeBoolsTooBig) + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] + self.check_struct(Main) + with self.assertRaises(ValueError) as ctx: + Main.from_buffer(data) + self.assertEqual( + ctx.exception.args[0], + 'Buffer size too small (4 instead of at least 8 bytes)' + ) + + def test_aligned_subclasses(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] + self.check_struct(UnalignedSub) + + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] + self.check_struct(AlignedStruct) + + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(alignment(main.b), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(sizeof(main), 16) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 3) + self.assertEqual(main.b.y, 4) + self.assertEqual(Main.b.offset, 8) + self.assertEqual(Main.b.size, 8) + + def test_aligned_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class AlignedUnion(ubase): + _align_ = 8 + _fields_ = [ + ("a", c_uint32), + ("b", c_ubyte * 7), + ] + self.check_union(AlignedUnion) + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(bytes(main.union.b), data[8:-1]) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + + def test_aligned_struct_in_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] + self.check_struct(Sub) + + class MainUnion(ubase): + _fields_ = [ + ("a", c_uint32), + ("b", Sub), + ] + self.check_union(MainUnion) + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(Main.first.size, 4) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(main.union.b.x, 3) + self.assertEqual(main.union.b.y, 4) + + def test_smaller_aligned_subclassed_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + class SubUnion(ubase): + _align_ = 2 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + self.check_union(SubUnion) + + class MainUnion(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + self.check_union(SubUnion) + + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(main.union.num, 0xD60102D7) + self.assertEqual(main.union.unsigned, data[4]) + self.assertEqual(main.union.signed, data[4] - 256) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.union), 4) + self.assertEqual(Main.union.offset, 4) + self.assertEqual(Main.union.size, 4) + self.assertEqual(Main.first.size, 2) + + def test_larger_aligned_subclassed_union(self): + for ubase, e in ( + (LittleEndianUnion, "<"), + (BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + class SubUnion(ubase): + _align_ = 8 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + self.check_union(SubUnion) + + class Main(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main), 8) + self.assertEqual(main.num, 0xD60102D6) + self.assertEqual(main.unsigned, 0xD6) + self.assertEqual(main.signed, -42) + + def test_aligned_packed_structures(self): + for sbase, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) + + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] + self.check_struct(Inner) + + class Main(sbase): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] + self.check_struct(Main) + + main = Main.from_buffer(data) + self.assertEqual(sizeof(main), 10) + self.assertEqual(Main.b.offset, 1) + # Alignment == 8 because _pack_ wins out. + self.assertEqual(alignment(main.b), 8) + # Size is still 8 though since inside this Structure, it will have + # effect. + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(Main.c.offset, 9) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 2) + self.assertEqual(main.b.y, 3) + self.assertEqual(main.c, 4) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/ctypes/test/test_anon.py b/Lib/test/test_ctypes/test_anon.py similarity index 89% rename from Lib/ctypes/test/test_anon.py rename to Lib/test/test_ctypes/test_anon.py index d378392ebe2..2e16e708635 100644 --- a/Lib/ctypes/test/test_anon.py +++ b/Lib/test/test_ctypes/test_anon.py @@ -1,19 +1,23 @@ import unittest import test.support -from ctypes import * +from ctypes import c_int, Union, Structure, sizeof +from ._support import StructCheckMixin -class AnonTest(unittest.TestCase): + +class AnonTest(unittest.TestCase, StructCheckMixin): def test_anon(self): class ANON(Union): _fields_ = [("a", c_int), ("b", c_int)] + self.check_union(ANON) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.a.offset, sizeof(c_int)) self.assertEqual(Y.b.offset, sizeof(c_int)) @@ -51,17 +55,20 @@ class Name(Structure): def test_nested(self): class ANON_S(Structure): _fields_ = [("a", c_int)] + self.check_struct(ANON_S) class ANON_U(Union): _fields_ = [("_", ANON_S), ("b", c_int)] _anonymous_ = ["_"] + self.check_union(ANON_U) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON_U), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.x.offset, 0) self.assertEqual(Y.a.offset, sizeof(c_int)) @@ -69,5 +76,6 @@ class Y(Structure): self.assertEqual(Y._.offset, sizeof(c_int)) self.assertEqual(Y.y.offset, sizeof(c_int) * 2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_array_in_pointer.py b/Lib/test/test_ctypes/test_array_in_pointer.py similarity index 93% rename from Lib/ctypes/test/test_array_in_pointer.py rename to Lib/test/test_ctypes/test_array_in_pointer.py index ca1edcf6210..b7c96b2fa49 100644 --- a/Lib/ctypes/test/test_array_in_pointer.py +++ b/Lib/test/test_ctypes/test_array_in_pointer.py @@ -1,21 +1,24 @@ -import unittest -from ctypes import * -from binascii import hexlify +import binascii import re +import unittest +from ctypes import c_byte, Structure, POINTER, cast + def dump(obj): # helper function to dump memory contents in hex, with a hyphen # between the bytes. - h = hexlify(memoryview(obj)).decode() + h = binascii.hexlify(memoryview(obj)).decode() return re.sub(r"(..)", r"\1-", h)[:-1] class Value(Structure): _fields_ = [("val", c_byte)] + class Container(Structure): _fields_ = [("pvalues", POINTER(Value))] + class Test(unittest.TestCase): def test(self): # create an array of 4 values @@ -60,5 +63,6 @@ def test_2(self): ([1, 2, 3, 4], "01-02-03-04") ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py similarity index 71% rename from Lib/ctypes/test/test_arrays.py rename to Lib/test/test_ctypes/test_arrays.py index 14603b7049c..7f1f6cf5840 100644 --- a/Lib/ctypes/test/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -1,16 +1,50 @@ -import unittest -from test.support import bigmemtest, _2G +import ctypes import sys -from ctypes import * +import unittest +from ctypes import (Structure, Array, ARRAY, sizeof, addressof, + create_string_buffer, create_unicode_buffer, + c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, + c_long, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED +from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) -from ctypes.test import need_symbol formats = "bBhHiIlLqQfd" formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ c_long, c_ulonglong, c_float, c_double, c_longdouble + class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Array.mro(), [Array, _CData, object]) + + self.assertEqual(PyCArrayType.__name__, "PyCArrayType") + self.assertEqual(type(PyCArrayType), type) + + def test_type_flags(self): + for cls in Array, PyCArrayType: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -34,9 +68,9 @@ def test_simple(self): with self.assertRaises(IndexError): ia[-alen-1] # change the items - from operator import setitem new_values = list(range(42, 42+alen)) - [setitem(ia, n, new_values[n]) for n in range(alen)] + for n in range(alen): + ia[n] = new_values[n] values = [ia[i] for i in range(alen)] self.assertEqual(values, new_values) @@ -66,8 +100,8 @@ def test_simple(self): self.assertEqual(len(ca), 3) # cannot delete items - from operator import delitem - self.assertRaises(TypeError, delitem, ca, 0) + with self.assertRaises(TypeError): + del ca[0] def test_step_overflow(self): a = (c_int * 5)() @@ -117,7 +151,6 @@ def test_from_address(self): self.assertEqual(sz[1:4:2], b"o") self.assertEqual(sz.value, b"foo") - @need_symbol('create_unicode_buffer') def test_from_addressW(self): p = create_unicode_buffer("foo") sz = (c_wchar * 3).from_address(addressof(p)) @@ -178,10 +211,10 @@ def test_bad_subclass(self): class T(Array): pass with self.assertRaises(AttributeError): - class T(Array): + class T2(Array): _type_ = c_int with self.assertRaises(AttributeError): - class T(Array): + class T3(Array): _length_ = 13 def test_bad_length(self): @@ -190,15 +223,15 @@ class T(Array): _type_ = c_int _length_ = - sys.maxsize * 2 with self.assertRaises(ValueError): - class T(Array): + class T2(Array): _type_ = c_int _length_ = -1 with self.assertRaises(TypeError): - class T(Array): + class T3(Array): _type_ = c_int _length_ = 1.87 with self.assertRaises(OverflowError): - class T(Array): + class T4(Array): _type_ = c_int _length_ = sys.maxsize * 2 @@ -212,7 +245,7 @@ def test_empty_element_struct(self): class EmptyStruct(Structure): _fields_ = [] - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + obj = (EmptyStruct * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_empty_element_array(self): @@ -220,7 +253,7 @@ class EmptyArray(Array): _type_ = c_int _length_ = 0 - obj = (EmptyArray * 2)() # bpo37188: Floating point exception + obj = (EmptyArray * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_bpo36504_signed_int_overflow(self): @@ -234,5 +267,26 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size + @threading_helper.requires_working_threading() + @unittest.skipUnless(Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_thread_safety(self): + from threading import Thread + + buffer = (ctypes.c_char_p * 10)() + + def run(): + for i in range(100): + buffer.value = b"hello" + buffer[0] = b"j" + + with threading_helper.catch_threading_exception() as cm: + threads = (Thread(target=run) for _ in range(25)) + with threading_helper.start_threads(threads): + pass + + if cm.exc_value: + raise cm.exc_value + + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py similarity index 83% rename from Lib/ctypes/test/test_as_parameter.py rename to Lib/test/test_ctypes/test_as_parameter.py index 9c39179d2a4..2da1acfcf29 100644 --- a/Lib/ctypes/test/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,24 +1,31 @@ +import ctypes import unittest -from ctypes import * -from ctypes.test import need_symbol -import _ctypes_test +from ctypes import (Structure, CDLL, CFUNCTYPE, + POINTER, pointer, byref, + c_short, c_int, c_long, c_longlong, + c_byte, c_wchar, c_float, c_double, + ArgumentError) +from test.support import import_helper, skip_if_sanitizer +_ctypes_test = import_helper.import_module("_ctypes_test") + dll = CDLL(_ctypes_test.__file__) try: - CALLBACK_FUNCTYPE = WINFUNCTYPE -except NameError: + CALLBACK_FUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux CALLBACK_FUNCTYPE = CFUNCTYPE + class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + class BasicWrapTestCase(unittest.TestCase): def wrap(self, param): return param - @need_symbol('c_wchar') def test_wchar_parm(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] @@ -67,8 +74,6 @@ def callback(v): f(self.wrap(2**18), self.wrap(cb)) self.assertEqual(args, expected) - ################################################################ - def test_callbacks(self): f = dll._testfunc_callback_i_if f.restype = c_int @@ -77,7 +82,6 @@ def test_callbacks(self): MyCallback = CFUNCTYPE(c_int, c_int) def callback(value): - #print "called back with", value return value cb = MyCallback(callback) @@ -114,7 +118,6 @@ def test_callbacks_2(self): f.argtypes = [c_int, MyCallback] def callback(value): - #print "called back with", value self.assertEqual(type(value), int) return value @@ -122,9 +125,7 @@ def callback(value): result = f(self.wrap(-10), self.wrap(cb)) self.assertEqual(result, -18) - @need_symbol('c_longlong') def test_longlong_callbacks(self): - f = dll._testfunc_callback_q_qf f.restype = c_longlong @@ -191,30 +192,34 @@ class S8I(Structure): self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + @skip_if_sanitizer('requires deep stack', thread=True) def test_recursive_as_param(self): - from ctypes import c_int - - class A(object): + class A: pass a = A() a._as_parameter_ = a - with self.assertRaises(RecursionError): - c_int.from_param(a) - - -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -class AsParamWrapper(object): + for c_type in ( + ctypes.c_wchar_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_int, # PyCSimpleType + POINT, # CDataType + ): + with self.subTest(c_type=c_type): + with self.assertRaises(RecursionError): + c_type.from_param(a) + + +class AsParamWrapper: def __init__(self, param): self._as_parameter_ = param class AsParamWrapperTestCase(BasicWrapTestCase): wrap = AsParamWrapper -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -class AsParamPropertyWrapper(object): +class AsParamPropertyWrapper: def __init__(self, param): self._param = param @@ -225,7 +230,17 @@ def getParameter(self): class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py new file mode 100644 index 00000000000..518f838219e --- /dev/null +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -0,0 +1,587 @@ +import os +import sys +import unittest +from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, + LittleEndianStructure, BigEndianStructure, + c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, + c_uint8, c_uint16, c_uint32, c_uint64, + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) +from test import support +from test.support import import_helper +from ._support import StructCheckMixin +_ctypes_test = import_helper.import_module("_ctypes_test") + + +TEST_FIELDS = ( + ("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7), +) + + +class BITS(Structure): + _fields_ = TEST_FIELDS + +func = CDLL(_ctypes_test.__file__).unpack_bitfields +func.argtypes = POINTER(BITS), c_char + + +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = TEST_FIELDS + + +class BITS_gcc(Structure): + _layout_ = "gcc-sysv" + _fields_ = TEST_FIELDS + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + +class C_Test(unittest.TestCase): + + def test_ints(self): + for i in range(512): + for name in "ABCDEFGHI": + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + def test_shorts(self): + b = BITS() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) + + +signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) +unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) +int_types = unsigned_int_types + signed_int_types + +class BitFieldTest(unittest.TestCase, StructCheckMixin): + + def test_generic_checks(self): + self.check_struct(BITS) + self.check_struct(BITS_msvc) + self.check_struct(BITS_gcc) + + def test_longlong(self): + class X(Structure): + _fields_ = [("a", c_longlong, 1), + ("b", c_longlong, 62), + ("c", c_longlong, 1)] + self.check_struct(X) + + self.assertEqual(sizeof(X), sizeof(c_longlong)) + x = X() + x.a, x.b, x.c = -1, 7, -1 + self.assertEqual((x.a, x.b, x.c), (-1, 7, -1)) + + def test_ulonglong(self): + class X(Structure): + _fields_ = [("a", c_ulonglong, 1), + ("b", c_ulonglong, 62), + ("c", c_ulonglong, 1)] + self.check_struct(X) + + self.assertEqual(sizeof(X), sizeof(c_longlong)) + x = X() + self.assertEqual((x.a, x.b, x.c), (0, 0, 0)) + x.a, x.b, x.c = 7, 7, 7 + self.assertEqual((x.a, x.b, x.c), (1, 7, 1)) + + def test_signed(self): + for c_typ in signed_int_types: + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + + + def test_unsigned(self): + for c_typ in unsigned_int_types: + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + + def fail_fields(self, *fields): + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) + + def test_nonint_types(self): + # bit fields are not allowed on non-integer types. + result = self.fail_fields(("a", c_char_p, 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p')) + + result = self.fail_fields(("a", c_void_p, 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p')) + + if c_int != c_long: + result = self.fail_fields(("a", POINTER(c_int), 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int')) + + result = self.fail_fields(("a", c_char, 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) + + class Empty(Structure): + _fields_ = [] + self.check_struct(Empty) + + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + self.check_struct(Dummy) + + result = self.fail_fields(("a", Dummy, 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) + + def test_c_wchar(self): + result = self.fail_fields(("a", c_wchar, 1)) + self.assertEqual(result, + (TypeError, 'bit fields not allowed for type c_wchar')) + + def test_single_bitfield_size(self): + for c_typ in int_types: + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) + + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) + + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) + + def test_multi_bitfields_size(self): + class X(Structure): + _fields_ = [("a", c_short, 1), + ("b", c_short, 14), + ("c", c_short, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_short)) + + class X(Structure): + _fields_ = [("a", c_short, 1), + ("a1", c_short), + ("b", c_short, 14), + ("c", c_short, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_short)*3) + self.assertEqual(X.a.offset, 0) + self.assertEqual(X.a1.offset, sizeof(c_short)) + self.assertEqual(X.b.offset, sizeof(c_short)*2) + self.assertEqual(X.c.offset, sizeof(c_short)*2) + + class X(Structure): + _fields_ = [("a", c_short, 3), + ("b", c_short, 14), + ("c", c_short, 14)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_short)*3) + self.assertEqual(X.a.offset, sizeof(c_short)*0) + self.assertEqual(X.b.offset, sizeof(c_short)*1) + self.assertEqual(X.c.offset, sizeof(c_short)*2) + + def get_except(self, func, *args, **kw): + try: + func(*args, **kw) + except Exception as detail: + return detail.__class__, str(detail) + + def test_mixed_1(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + self.check_struct(X) + if os.name == "nt": + self.assertEqual(sizeof(X), sizeof(c_int)*2) + else: + self.assertEqual(sizeof(X), sizeof(c_int)) + + def test_mixed_2(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 32)] + self.check_struct(X) + self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) + + def test_mixed_3(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_byte)) + + def test_mixed_4(self): + class X(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + self.check_struct(X) + # MSVC does NOT combine c_short and c_int into one field, GCC + # does (unless GCC is run with '-mms-bitfields' which + # produces code compatible with MSVC). + if os.name == "nt": + self.assertEqual(sizeof(X), sizeof(c_int) * 4) + else: + self.assertEqual(sizeof(X), sizeof(c_int) * 2) + + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + self.check_struct(X) + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + self.check_struct(X) + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.check_struct(X) + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + self.check_struct(Foo) + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.check_struct(Bar) + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + self.check_struct(TestStruct) + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + class GoodA(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + self.check_struct(Bad) + self.check_struct(GoodA) + self.check_struct(Good) + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.check_struct(MyStructure) + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + def test_anon_bitfields(self): + # anonymous bit-fields gave a strange error message + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + class Y(Structure): + _anonymous_ = ["_"] + _fields_ = [("_", X)] + + self.check_struct(X) + self.check_struct(Y) + + def test_uint32(self): + class X(Structure): + _fields_ = [("a", c_uint32, 32)] + self.check_struct(X) + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFDCBA987 + self.assertEqual(x.a, 0xFDCBA987) + + def test_uint64(self): + class X(Structure): + _fields_ = [("a", c_uint64, 64)] + self.check_struct(X) + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFEDCBA9876543211 + self.assertEqual(x.a, 0xFEDCBA9876543211) + + def test_uint32_swap_little_endian(self): + # Issue #23319 + class Little(LittleEndianStructure): + _fields_ = [("a", c_uint32, 24), + ("b", c_uint32, 4), + ("c", c_uint32, 4)] + self.check_struct(Little) + b = bytearray(4) + x = Little.from_buffer(b) + x.a = 0xabcdef + x.b = 1 + x.c = 2 + self.assertEqual(b, b'\xef\xcd\xab\x21') + + def test_uint32_swap_big_endian(self): + # Issue #23319 + class Big(BigEndianStructure): + _fields_ = [("a", c_uint32, 24), + ("b", c_uint32, 4), + ("c", c_uint32, 4)] + self.check_struct(Big) + b = bytearray(4) + x = Big.from_buffer(b) + x.a = 0xabcdef + x.b = 1 + x.c = 2 + self.assertEqual(b, b'\xab\xcd\xef\x12') + + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.check_union(BitfieldUnion) + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/ctypes/test/test_buffers.py b/Lib/test/test_ctypes/test_buffers.py similarity index 94% rename from Lib/ctypes/test/test_buffers.py rename to Lib/test/test_ctypes/test_buffers.py index 15782be757c..468f41eb7cf 100644 --- a/Lib/ctypes/test/test_buffers.py +++ b/Lib/test/test_ctypes/test_buffers.py @@ -1,9 +1,9 @@ -from ctypes import * -from ctypes.test import need_symbol import unittest +from ctypes import (create_string_buffer, create_unicode_buffer, sizeof, + c_char, c_wchar) -class StringBufferTestCase(unittest.TestCase): +class StringBufferTestCase(unittest.TestCase): def test_buffer(self): b = create_string_buffer(32) self.assertEqual(len(b), 32) @@ -27,7 +27,6 @@ def test_buffer_interface(self): self.assertEqual(len(bytearray(create_string_buffer(0))), 0) self.assertEqual(len(bytearray(create_string_buffer(1))), 1) - @need_symbol('c_wchar') def test_unicode_buffer(self): b = create_unicode_buffer(32) self.assertEqual(len(b), 32) @@ -47,7 +46,6 @@ def test_unicode_buffer(self): self.assertRaises(TypeError, create_unicode_buffer, b"abc") - @need_symbol('c_wchar') def test_unicode_conversion(self): b = create_unicode_buffer("abc") self.assertEqual(len(b), 4) # trailing nul char @@ -60,7 +58,6 @@ def test_unicode_conversion(self): self.assertEqual(b[::2], "ac") self.assertEqual(b[::5], "a") - @need_symbol('c_wchar') def test_create_unicode_buffer_non_bmp(self): expected = 5 if sizeof(c_wchar) == 2 else 3 for s in '\U00010000\U00100000', '\U00010000\U0010ffff': diff --git a/Lib/ctypes/test/test_bytes.py b/Lib/test/test_ctypes/test_bytes.py similarity index 88% rename from Lib/ctypes/test/test_bytes.py rename to Lib/test/test_ctypes/test_bytes.py index 092ec5af052..0e7f81b9482 100644 --- a/Lib/ctypes/test/test_bytes.py +++ b/Lib/test/test_ctypes/test_bytes.py @@ -1,9 +1,12 @@ """Test where byte objects are accepted""" -import unittest import sys -from ctypes import * +import unittest +from _ctypes import _SimpleCData +from ctypes import Structure, c_char, c_char_p, c_wchar, c_wchar_p +from ._support import StructCheckMixin + -class BytesTest(unittest.TestCase): +class BytesTest(unittest.TestCase, StructCheckMixin): def test_c_char(self): x = c_char(b"x") self.assertRaises(TypeError, c_char, "x") @@ -38,6 +41,7 @@ def test_c_wchar_p(self): def test_struct(self): class X(Structure): _fields_ = [("a", c_char * 3)] + self.check_struct(X) x = X(b"abc") self.assertRaises(TypeError, X, "abc") @@ -47,6 +51,7 @@ class X(Structure): def test_struct_W(self): class X(Structure): _fields_ = [("a", c_wchar * 3)] + self.check_struct(X) x = X("abc") self.assertRaises(TypeError, X, b"abc") @@ -55,7 +60,6 @@ class X(Structure): @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') def test_BSTR(self): - from _ctypes import _SimpleCData class BSTR(_SimpleCData): _type_ = "X" diff --git a/Lib/ctypes/test/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py similarity index 87% rename from Lib/ctypes/test/test_byteswap.py rename to Lib/test/test_ctypes/test_byteswap.py index 7e98559dfbc..ea5951603f9 100644 --- a/Lib/ctypes/test/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,10 +1,22 @@ -import sys, unittest, struct, math, ctypes -from binascii import hexlify +import binascii +import ctypes +import math +import struct +import sys +import unittest +from ctypes import (Structure, Union, LittleEndianUnion, BigEndianUnion, + BigEndianStructure, LittleEndianStructure, + POINTER, sizeof, cast, + c_byte, c_ubyte, c_char, c_wchar, c_void_p, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_uint32, c_float, c_double) +from ._support import StructCheckMixin -from ctypes import * def bin(s): - return hexlify(memoryview(s)).decode().upper() + return binascii.hexlify(memoryview(s)).decode().upper() + # Each *simple* type that supports different byte orders has an # __ctype_be__ attribute that specifies the same type in BIG ENDIAN @@ -13,23 +25,17 @@ def bin(s): # # For Structures and Unions, these types are created on demand. -class Test(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): - print(sys.byteorder, file=sys.stderr) - for i in range(32): - bits = BITS() - setattr(bits, "i%s" % i, 1) - dump(bits) - +class Test(unittest.TestCase, StructCheckMixin): def test_slots(self): class BigPoint(BigEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(BigPoint) class LowPoint(LittleEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(LowPoint) big = BigPoint() little = LowPoint() @@ -197,6 +203,7 @@ def test_struct_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure): _fields_ = fields + [("x", typ)] + self.check_struct(T) def test_struct_struct(self): @@ -216,14 +223,15 @@ def test_struct_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestStructure(parent): _fields_ = [("point", NestedStructure)] + self.check_struct(TestStructure) self.assertEqual(len(data), sizeof(TestStructure)) ptr = POINTER(TestStructure) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestStructure] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) @@ -245,6 +253,7 @@ class S(base): ("h", c_short), ("i", c_int), ("d", c_double)] + self.check_struct(S) s1 = S(0x12, 0x1234, 0x12345678, 3.14) s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) @@ -260,6 +269,7 @@ def test_unaligned_nonnative_struct_fields(self): class S(base): _pack_ = 1 + _layout_ = "ms" _fields_ = [("b", c_byte), ("h", c_short), @@ -268,6 +278,7 @@ class S(base): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -286,6 +297,7 @@ def test_unaligned_native_struct_fields(self): class S(Structure): _pack_ = 1 + _layout_ = "ms" _fields_ = [("b", c_byte), ("h", c_short), @@ -295,6 +307,7 @@ class S(Structure): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -331,6 +344,7 @@ def test_union_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion): _fields_ = fields + [("x", typ)] + self.check_union(T) def test_union_struct(self): # nested structures in unions with different byteorders @@ -349,16 +363,39 @@ def test_union_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestUnion(parent): _fields_ = [("point", NestedStructure)] + self.check_union(TestUnion) self.assertEqual(len(data), sizeof(TestUnion)) ptr = POINTER(TestUnion) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestUnion] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) + def test_build_struct_union_opposite_system_byteorder(self): + # gh-105102 + if sys.byteorder == "little": + _Structure = BigEndianStructure + _Union = BigEndianUnion + else: + _Structure = LittleEndianStructure + _Union = LittleEndianUnion + + class S1(_Structure): + _fields_ = [("a", c_byte), ("b", c_byte)] + self.check_struct(S1) + + class U1(_Union): + _fields_ = [("s1", S1), ("ab", c_short)] + self.check_union(U1) + + class S2(_Structure): + _fields_ = [("u1", U1), ("c", c_byte)] + self.check_struct(S2) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py new file mode 100644 index 00000000000..fd261acf497 --- /dev/null +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -0,0 +1,384 @@ +import unittest +from test.support import MS_WINDOWS +import ctypes +from ctypes import POINTER, Structure, c_void_p + +from ._support import PyCSimpleType, PyCPointerType, PyCStructType + + +def set_non_ctypes_pointer_type(cls, pointer_type): + cls.__pointer_type__ = pointer_type + +class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): + def test_creating_pointer_in_dunder_new_1(self): + # Test metaclass whose instances are C types; when the type is + # created it automatically creates a pointer type for itself. + # The pointer type is also an instance of the metaclass. + # Such an implementation is used in `IUnknown` of the `comtypes` + # project. See gh-124520. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + + # Avoid recursion: don't set up a pointer to + # a pointer (to a pointer...) + if bases == (c_void_p,): + # When creating PtrBase itself, the name + # is not yet available + return self + if issubclass(self, PtrBase): + return self + + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + set_non_ctypes_pointer_type(self, p) + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + ptr_base_pointer = POINTER(PtrBase) + + class CtBase(object, metaclass=ct_meta): + pass + + ct_base_pointer = POINTER(CtBase) + + class Sub(CtBase): + pass + + sub_pointer = POINTER(Sub) + + class Sub2(Sub): + pass + + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + self.assertIs(POINTER(Sub2), sub2_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + + def test_creating_pointer_in_dunder_new_2(self): + # A simpler variant of the above, used in `CoClass` of the `comtypes` + # project. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + if isinstance(self, p_meta): + return self + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + set_non_ctypes_pointer_type(self, p) + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + with self.assertRaisesRegex(TypeError, "must have storage info"): + POINTER(Core) + + class CtBase(Core, metaclass=ct_meta): + pass + + ct_base_pointer = POINTER(CtBase) + + class Sub(CtBase): + pass + + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertIsSubclass(POINTER(Sub), Sub) + + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + set_non_ctypes_pointer_type(self, p) + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + ptr_base_pointer = POINTER(PtrBase) + + class CtBase(object, metaclass=ct_meta): + pass + + ct_base_pointer = POINTER(CtBase) + + class Sub(CtBase): + pass + + sub_pointer = POINTER(Sub) + + class Sub2(Sub): + pass + + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + self.assertIs(POINTER(PtrBase), ptr_base_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(Sub2), sub2_pointer) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + set_non_ctypes_pointer_type(self, p) + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + ct_base_pointer = POINTER(CtBase) + + class Sub(CtBase): + pass + + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertIsSubclass(POINTER(Sub), Sub) + + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + + def test_bad_type_message(self): + """Verify the error message that lists all available type codes""" + # (The string is generated at runtime, so this checks the underlying + # set of types as well as correct construction of the string.) + with self.assertRaises(AttributeError) as cm: + class F(metaclass=PyCSimpleType): + _type_ = "\0" + message = str(cm.exception) + expected_type_chars = list('cbBhHiIlLdDFGfuzZqQPXOv?g') + if not hasattr(ctypes, 'c_float_complex'): + expected_type_chars.remove('F') + expected_type_chars.remove('D') + expected_type_chars.remove('G') + if not MS_WINDOWS: + expected_type_chars.remove('X') + self.assertIn("'" + ''.join(expected_type_chars) + "'", message) + + def test_creating_pointer_in_dunder_init_3(self): + """Check if interfcase subclasses properly creates according internal + pointer types. But not the same as external pointer types. + """ + + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + assert len(bases) == 1, bases + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) + assert self.__pointer_type__ is internal_pointer_type + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + if target is None: + + # Create corresponding interface type and then set it as target + target = StructureMeta( + f"_{name}_", + (bases[0]._type_,), + {}, + create_pointer_type=False + ) + dct['_type_'] = target + + pointer_type = super().__new__(cls, name, bases, dct) + assert not hasattr(target, '__pointer_type__') + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + assert not hasattr(target, '__pointer_type__') + super().__init__(name, bases, dct) + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + pass + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIsNot(POINTER(IUnknown), pIUnknown) + + def test_creating_pointer_in_dunder_init_4(self): + """Check if interfcase subclasses properly creates according internal + pointer types, the same as external pointer types. + """ + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + assert len(bases) == 1, bases + + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) + assert self.__pointer_type__ is internal_pointer_type + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + assert target is not None + pointer_type = getattr(target, '__pointer_type__', None) + + if pointer_type is None: + pointer_type = super().__new__(cls, name, bases, dct) + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + if not hasattr(target, '__pointer_type__'): + # target.__pointer_type__ was created by super().__new__ + super().__init__(name, bases, dct) + + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + _type_ = IUnknown + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIs(POINTER(IUnknown), pIUnknown) + + def test_custom_pointer_cache_for_ctypes_type1(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __new__(). + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, namespace): + namespace["_type_"] = C + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace): + assert not hasattr(C, '__pointer_type__') + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) + + def test_custom_pointer_cache_for_ctypes_type2(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __init__(). + class PointerMeta(PyCPointerType): + def __init__(self, name, bases, namespace): + self._type_ = namespace["_type_"] = C + assert not hasattr(C, '__pointer_type__') + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py similarity index 82% rename from Lib/ctypes/test/test_callbacks.py rename to Lib/test/test_ctypes/test_callbacks.py index 8f95a244439..6c7c2e52707 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,19 +1,25 @@ +import ctypes import functools +import gc +import math +import sys import unittest +from _ctypes import CTYPES_MAX_ARGCOUNT +from ctypes import (CDLL, cdll, Structure, CFUNCTYPE, + ArgumentError, POINTER, sizeof, + c_byte, c_ubyte, c_char, + c_short, c_ushort, c_int, c_uint, + c_long, c_longlong, c_ulonglong, c_ulong, + c_float, c_double, c_longdouble, py_object) +from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -from ctypes import * -from ctypes.test import need_symbol -from _ctypes import CTYPES_MAX_ARGCOUNT -import _ctypes_test class Callbacks(unittest.TestCase): functype = CFUNCTYPE -## def tearDown(self): -## import gc -## gc.collect() - def callback(self, *args): self.got_args = args return args[-1] @@ -35,8 +41,6 @@ def check_type(self, typ, arg): self.assertEqual(self.got_args, (-3, arg)) self.assertEqual(result, arg) - ################ - def test_byte(self): self.check_type(c_byte, 42) self.check_type(c_byte, -42) @@ -65,18 +69,15 @@ def test_long(self): def test_ulong(self): self.check_type(c_ulong, 42) - @need_symbol('c_longlong') def test_longlong(self): self.check_type(c_longlong, 42) self.check_type(c_longlong, -42) - @need_symbol('c_ulonglong') def test_ulonglong(self): self.check_type(c_ulonglong, 42) def test_float(self): # only almost equal: double -> float -> double - import math self.check_type(c_float, math.e) self.check_type(c_float, -math.e) @@ -84,7 +85,6 @@ def test_double(self): self.check_type(c_double, 3.14) self.check_type(c_double, -3.14) - @need_symbol('c_longdouble') def test_longdouble(self): self.check_type(c_longdouble, 3.14) self.check_type(c_longdouble, -3.14) @@ -93,30 +93,21 @@ def test_char(self): self.check_type(c_char, b"x") self.check_type(c_char, b"a") - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. - @unittest.skip('test disabled') - def test_char_p(self): - self.check_type(c_char_p, "abc") - self.check_type(c_char_p, "def") - def test_pyobject(self): o = () - from sys import getrefcount as grc for o in (), [], object(): - initial = grc(o) + initial = sys.getrefcount(o) # This call leaks a reference to 'o'... self.check_type(py_object, o) - before = grc(o) + before = sys.getrefcount(o) # ...but this call doesn't leak any more. Where is the refcount? self.check_type(py_object, o) - after = grc(o) + after = sys.getrefcount(o) self.assertEqual((after, o), (before, o)) def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) @@ -130,12 +121,11 @@ def test_unsupported_restype_2(self): def test_issue_7959(self): proto = self.functype.__func__(None) - class X(object): + class X: def func(self): pass def __init__(self): self.v = proto(self.func) - import gc for i in range(32): X() gc.collect() @@ -144,21 +134,30 @@ def __init__(self): self.assertEqual(len(live), 0) def test_issue12483(self): - import gc class Nasty: def __del__(self): gc.collect() CFUNCTYPE(None)(lambda x=Nasty(): None) + @unittest.skipUnless(hasattr(ctypes, 'WINFUNCTYPE'), + 'ctypes.WINFUNCTYPE is required') + def test_i38748_stackCorruption(self): + callback_funcType = ctypes.WINFUNCTYPE(c_long, c_long, c_longlong) + @callback_funcType + def callback(a, b): + c = a + b + print(f"a={a}, b={b}, c={c}") + return c + dll = cdll[_ctypes_test.__file__] + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") + +if hasattr(ctypes, 'WINFUNCTYPE'): + class StdcallCallbacks(Callbacks): + functype = ctypes.WINFUNCTYPE -@need_symbol('WINFUNCTYPE') -class StdcallCallbacks(Callbacks): - try: - functype = WINFUNCTYPE - except NameError: - pass - -################################################################ class SampleCallbacksTestCase(unittest.TestCase): @@ -183,7 +182,6 @@ def func(x): self.assertLess(diff, 0.01, "%s not less than 0.01" % diff) def test_issue_8959_a(self): - from ctypes.util import find_library libc_path = find_library("c") if not libc_path: self.skipTest('could not find libc') @@ -198,19 +196,21 @@ def cmp_func(a, b): libc.qsort(array, len(array), sizeof(c_int), cmp_func) self.assertEqual(array[:], [1, 5, 7, 33, 99]) - @need_symbol('WINFUNCTYPE') + @unittest.skipUnless(hasattr(ctypes, 'WINFUNCTYPE'), + 'ctypes.WINFUNCTYPE is required') def test_issue_8959_b(self): from ctypes.wintypes import BOOL, HWND, LPARAM global windowCount windowCount = 0 - @WINFUNCTYPE(BOOL, HWND, LPARAM) + @ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) def EnumWindowsCallbackFunc(hwnd, lParam): global windowCount windowCount += 1 return True #Allow windows to keep enumerating - windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + user32 = ctypes.windll.user32 + user32.EnumWindows(EnumWindowsCallbackFunc, 0) def test_callback_register_int(self): # Issue #8275: buggy handling of callback args under Win64 @@ -324,9 +324,9 @@ def func(): self.assertIsInstance(cm.unraisable.exc_value, TypeError) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on converting result " - "of ctypes callback function") - self.assertIs(cm.unraisable.object, func) + f"Exception ignored while converting result " + f"of ctypes callback function {func!r}") + self.assertIsNone(cm.unraisable.object) if __name__ == '__main__': diff --git a/Lib/ctypes/test/test_cast.py b/Lib/test/test_ctypes/test_cast.py similarity index 93% rename from Lib/ctypes/test/test_cast.py rename to Lib/test/test_ctypes/test_cast.py index 6878f973282..604f44f03d6 100644 --- a/Lib/ctypes/test/test_cast.py +++ b/Lib/test/test_ctypes/test_cast.py @@ -1,10 +1,11 @@ -from ctypes import * -from ctypes.test import need_symbol -import unittest import sys +import unittest +from ctypes import (Structure, Union, POINTER, cast, sizeof, addressof, + c_void_p, c_char_p, c_wchar_p, + c_byte, c_short, c_int) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_array2pointer(self): array = (c_int * 3)(42, 17, 2) @@ -12,7 +13,7 @@ def test_array2pointer(self): ptr = cast(array, POINTER(c_int)) self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2]) - if 2*sizeof(c_short) == sizeof(c_int): + if 2 * sizeof(c_short) == sizeof(c_int): ptr = cast(array, POINTER(c_short)) if sys.byteorder == "little": self.assertEqual([ptr[i] for i in range(6)], @@ -76,11 +77,10 @@ def test_char_p(self): self.assertEqual(cast(cast(s, c_void_p), c_char_p).value, b"hiho") - @need_symbol('c_wchar_p') def test_wchar_p(self): s = c_wchar_p("hiho") self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value, - "hiho") + "hiho") def test_bad_type_arg(self): # The type argument must be a ctypes pointer type. @@ -95,5 +95,6 @@ class MyUnion(Union): _fields_ = [("a", c_int)] self.assertRaises(TypeError, cast, array, MyUnion) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py similarity index 84% rename from Lib/ctypes/test/test_cfuncs.py rename to Lib/test/test_ctypes/test_cfuncs.py index 09b06840bf5..937be8eaa95 100644 --- a/Lib/ctypes/test/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,19 +1,23 @@ -# A lot of failures in these tests on Mac OS X. -# Byte order related? - +import ctypes import unittest -from ctypes import * -from ctypes.test import need_symbol +from ctypes import (CDLL, + c_byte, c_ubyte, c_char, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_float, c_double, c_longdouble) +from test import support +from test.support import import_helper, threading_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -import _ctypes_test class CFunctions(unittest.TestCase): _dll = CDLL(_ctypes_test.__file__) def S(self): - return c_longlong.in_dll(self._dll, "last_tf_arg_s").value + return _ctypes_test.get_last_tf_arg_s() + def U(self): - return c_ulonglong.in_dll(self._dll, "last_tf_arg_u").value + return _ctypes_test.get_last_tf_arg_u() def test_byte(self): self._dll.tf_b.restype = c_byte @@ -111,28 +115,24 @@ def test_ulong_plus(self): self.assertEqual(self._dll.tf_bL(b' ', 4294967295), 1431655765) self.assertEqual(self.U(), 4294967295) - @need_symbol('c_longlong') def test_longlong(self): self._dll.tf_q.restype = c_longlong self._dll.tf_q.argtypes = (c_longlong, ) self.assertEqual(self._dll.tf_q(-9223372036854775806), -3074457345618258602) self.assertEqual(self.S(), -9223372036854775806) - @need_symbol('c_longlong') def test_longlong_plus(self): self._dll.tf_bq.restype = c_longlong self._dll.tf_bq.argtypes = (c_byte, c_longlong) self.assertEqual(self._dll.tf_bq(0, -9223372036854775806), -3074457345618258602) self.assertEqual(self.S(), -9223372036854775806) - @need_symbol('c_ulonglong') def test_ulonglong(self): self._dll.tf_Q.restype = c_ulonglong self._dll.tf_Q.argtypes = (c_ulonglong, ) self.assertEqual(self._dll.tf_Q(18446744073709551615), 6148914691236517205) self.assertEqual(self.U(), 18446744073709551615) - @need_symbol('c_ulonglong') def test_ulonglong_plus(self): self._dll.tf_bQ.restype = c_ulonglong self._dll.tf_bQ.argtypes = (c_byte, c_ulonglong) @@ -163,14 +163,12 @@ def test_double_plus(self): self.assertEqual(self._dll.tf_bd(0, 42.), 14.) self.assertEqual(self.S(), 42) - @need_symbol('c_longdouble') def test_longdouble(self): self._dll.tf_D.restype = c_longdouble self._dll.tf_D.argtypes = (c_longdouble,) self.assertEqual(self._dll.tf_D(42.), 14.) self.assertEqual(self.S(), 42) - @need_symbol('c_longdouble') def test_longdouble_plus(self): self._dll.tf_bD.restype = c_longdouble self._dll.tf_bD.argtypes = (c_byte, c_longdouble) @@ -195,14 +193,28 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful on free-threading") + def test_thread_safety(self): + from threading import Thread + + def concurrent(): + for _ in range(100): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + + with threading_helper.catch_threading_exception() as exc: + with threading_helper.start_threads((Thread(target=concurrent) for _ in range(10))): + pass + + self.assertIsNone(exc.exc_value) + + # The following repeats the above tests with stdcall functions (where # they are available) -try: - WinDLL -except NameError: - def stdcall_dll(*_): pass -else: - class stdcall_dll(WinDLL): +if hasattr(ctypes, 'WinDLL'): + class stdcall_dll(ctypes.WinDLL): def __getattr__(self, name): if name[:2] == '__' and name[-2:] == '__': raise AttributeError(name) @@ -210,9 +222,9 @@ def __getattr__(self, name): setattr(self, name, func) return func -@need_symbol('WinDLL') -class stdcallCFunctions(CFunctions): - _dll = stdcall_dll(_ctypes_test.__file__) + class stdcallCFunctions(CFunctions): + _dll = stdcall_dll(_ctypes_test.__file__) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py similarity index 66% rename from Lib/ctypes/test/test_checkretval.py rename to Lib/test/test_ctypes/test_checkretval.py index e9567dc3912..9d6bfdb845e 100644 --- a/Lib/ctypes/test/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,9 @@ +import ctypes import unittest +from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -from ctypes import * -from ctypes.test import need_symbol class CHECKED(c_int): def _check_retval_(value): @@ -9,11 +11,9 @@ def _check_retval_(value): return str(value.value) _check_retval_ = staticmethod(_check_retval_) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_checkretval(self): - - import _ctypes_test dll = CDLL(_ctypes_test.__file__) self.assertEqual(42, dll._testfunc_p_p(42)) @@ -26,11 +26,12 @@ def test_checkretval(self): del dll._testfunc_p_p.restype self.assertEqual(42, dll._testfunc_p_p(42)) - @need_symbol('oledll') + @unittest.skipUnless(hasattr(ctypes, 'oledll'), + 'ctypes.oledll is required') def test_oledll(self): - self.assertRaises(OSError, - oledll.oleaut32.CreateTypeLib2, - 0, None, None) + oleaut32 = ctypes.oledll.oleaut32 + self.assertRaises(OSError, oleaut32.CreateTypeLib2, 0, None, None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_delattr.py b/Lib/test/test_ctypes/test_delattr.py new file mode 100644 index 00000000000..e80b5fa6efb --- /dev/null +++ b/Lib/test/test_ctypes/test_delattr.py @@ -0,0 +1,26 @@ +import unittest +from ctypes import Structure, c_char, c_int + + +class X(Structure): + _fields_ = [("foo", c_int)] + + +class TestCase(unittest.TestCase): + def test_simple(self): + with self.assertRaises(TypeError): + del c_int(42).value + + def test_chararray(self): + chararray = (c_char * 5)() + with self.assertRaises(TypeError): + del chararray.value + + def test_struct(self): + struct = X() + with self.assertRaises(TypeError): + del struct.foo + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py new file mode 100644 index 00000000000..ea2d97d9000 --- /dev/null +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -0,0 +1,182 @@ +import _ctypes +import os +import platform +import sys +import test.support +import unittest +from ctypes import CDLL, c_int +from ctypes.util import find_library + + +FOO_C = r""" +#include + +/* This is a 'GNU indirect function' (IFUNC) that will be called by + dlsym() to resolve the symbol "foo" to an address. Typically, such + a function would return the address of an actual function, but it + can also just return NULL. For some background on IFUNCs, see + https://willnewton.name/uncategorized/using-gnu-indirect-functions. + + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. +*/ + +asm (".type foo STT_GNU_IFUNC"); + +void *foo(void) +{ + write($DESCRIPTOR, "OK", 2); + return NULL; +} +""" + + +@unittest.skipUnless(sys.platform.startswith('linux'), + 'test requires GNU IFUNC support') +@unittest.skipIf(test.support.linked_to_musl(), "Requires glibc") +class TestNullDlsym(unittest.TestCase): + """GH-126554: Ensure that we catch NULL dlsym return values + + In rare cases, such as when using GNU IFUNCs, dlsym(), + the C function that ctypes' CDLL uses to get the address + of symbols, can return NULL. + + The objective way of telling if an error during symbol + lookup happened is to call glibc's dlerror() and check + for a non-NULL return value. + + However, there can be cases where dlsym() returns NULL + and dlerror() is also NULL, meaning that glibc did not + encounter any error. + + In the case of ctypes, we subjectively treat that as + an error, and throw a relevant exception. + + This test case ensures that we correctly enforce + this 'dlsym returned NULL -> throw Error' rule. + """ + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_null_dlsym(self): + import subprocess + import tempfile + + try: + retcode = subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except OSError: + self.skipTest("gcc is missing") + if retcode != 0: + self.skipTest("gcc --version failed") + + pipe_r, pipe_w = os.pipe() + self.addCleanup(os.close, pipe_r) + self.addCleanup(os.close, pipe_w) + + with tempfile.TemporaryDirectory() as d: + # Create a C file with a GNU Indirect Function (FOO_C) + # and compile it into a shared library. + srcname = os.path.join(d, 'foo.c') + dstname = os.path.join(d, 'libfoo.so') + with open(srcname, 'w') as f: + f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + + if p.returncode != 0: + # IFUNC is not supported on all architectures. + if platform.machine() == 'x86_64': + # It should be supported here. Something else went wrong. + p.check_returncode() + else: + # IFUNC might not be supported on this machine. + self.skipTest(f"could not compile indirect function: {p}") + + # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c + L = CDLL(dstname) + with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): + # Try accessing the 'foo' symbol. + # It should resolve via dlsym() to NULL, + # and since we subjectively treat NULL + # addresses as errors, we should get + # an error. + L.foo + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #2: Test 'CDataType_in_dll_impl' from Modules/_ctypes/_ctypes.c + with self.assertRaisesRegex(ValueError, "symbol 'foo' not found"): + c_int.in_dll(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + dlopen = test.support.get_attribute(_ctypes, 'dlopen') + dlsym = test.support.get_attribute(_ctypes, 'dlsym') + L = dlopen(dstname) + with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): + dlsym(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + +@test.support.thread_unsafe('setlocale is not thread-safe') +@unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') +class TestLocalization(unittest.TestCase): + + @staticmethod + def configure_locales(func): + return test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + )(func) + + @classmethod + def setUpClass(cls): + cls.libc_filename = find_library("c") + if cls.libc_filename is None: + raise unittest.SkipTest('cannot find libc') + + @configure_locales + def test_localized_error_from_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(AttributeError): + dll.this_name_does_not_exist + + @configure_locales + def test_localized_error_in_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(ValueError): + c_int.in_dll(dll, 'this_name_does_not_exist') + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @configure_locales + def test_localized_error_dlopen(self): + missing_filename = b'missing\xff.so' + # Depending whether the locale, we may encode '\xff' differently + # but we are only interested in avoiding a UnicodeDecodeError + # when reporting the dlerror() error message which contains + # the localized filename. + filename_pattern = r'missing.*?\.so' + with self.assertRaisesRegex(OSError, filename_pattern): + _ctypes.dlopen(missing_filename, 2) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') + @configure_locales + def test_localized_error_dlsym(self): + dll = _ctypes.dlopen(self.libc_filename) + with self.assertRaises(OSError): + _ctypes.dlsym(dll, 'this_name_does_not_exist') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_dllist.py b/Lib/test/test_ctypes/test_dllist.py new file mode 100644 index 00000000000..0e7c65127f6 --- /dev/null +++ b/Lib/test/test_ctypes/test_dllist.py @@ -0,0 +1,63 @@ +import os +import sys +import unittest +from ctypes import CDLL +import ctypes.util +from test.support import import_helper + + +WINDOWS = os.name == "nt" +APPLE = sys.platform in {"darwin", "ios", "tvos", "watchos"} + +if WINDOWS: + KNOWN_LIBRARIES = ["KERNEL32.DLL"] +elif APPLE: + KNOWN_LIBRARIES = ["libSystem.B.dylib"] +else: + # trickier than it seems, because libc may not be present + # on musl systems, and sometimes goes by different names. + # However, ctypes itself loads libffi + KNOWN_LIBRARIES = ["libc.so", "libffi.so"] + + +@unittest.skipUnless( + hasattr(ctypes.util, "dllist"), + "ctypes.util.dllist is not available on this platform", +) +class ListSharedLibraries(unittest.TestCase): + + # TODO: RUSTPYTHON + @unittest.skipIf(not APPLE, "TODO: RUSTPYTHON") + def test_lists_system(self): + dlls = ctypes.util.dllist() + + self.assertGreater(len(dlls), 0, f"loaded={dlls}") + self.assertTrue( + any(lib in dll for dll in dlls for lib in KNOWN_LIBRARIES), f"loaded={dlls}" + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_lists_updates(self): + dlls = ctypes.util.dllist() + + # this test relies on being able to import a library which is + # not already loaded. + # If it is (e.g. by a previous test in the same process), we skip + if any("_ctypes_test" in dll for dll in dlls): + self.skipTest("Test library is already loaded") + + _ctypes_test = import_helper.import_module("_ctypes_test") + test_module = CDLL(_ctypes_test.__file__) + dlls2 = ctypes.util.dllist() + self.assertIsNotNone(dlls2) + + dlls1 = set(dlls) + dlls2 = set(dlls2) + + self.assertGreater(dlls2, dlls1, f"newly loaded libraries: {dlls2 - dlls1}") + self.assertTrue(any("_ctypes_test" in dll for dll in dlls2), f"loaded={dlls2}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/ctypes/test/test_errno.py b/Lib/test/test_ctypes/test_errno.py similarity index 71% rename from Lib/ctypes/test/test_errno.py rename to Lib/test/test_ctypes/test_errno.py index 3685164dde6..65d99c1e492 100644 --- a/Lib/ctypes/test/test_errno.py +++ b/Lib/test/test_ctypes/test_errno.py @@ -1,14 +1,18 @@ -import unittest, os, errno +import ctypes +import errno +import os import threading - -from ctypes import * +import unittest +from ctypes import CDLL, c_int, c_char_p, c_wchar_p, get_errno, set_errno from ctypes.util import find_library + class Test(unittest.TestCase): def test_open(self): libc_name = find_library("c") if libc_name is None: - raise unittest.SkipTest("Unable to find C library") + self.skipTest("Unable to find C library") + libc = CDLL(libc_name, use_errno=True) if os.name == "nt": libc_open = libc._open @@ -44,33 +48,34 @@ def _worker(): @unittest.skipUnless(os.name == "nt", 'Test specific to Windows') def test_GetLastError(self): - dll = WinDLL("kernel32", use_last_error=True) + dll = ctypes.WinDLL("kernel32", use_last_error=True) GetModuleHandle = dll.GetModuleHandleA GetModuleHandle.argtypes = [c_wchar_p] self.assertEqual(0, GetModuleHandle("foo")) - self.assertEqual(get_last_error(), 126) + self.assertEqual(ctypes.get_last_error(), 126) - self.assertEqual(set_last_error(32), 126) - self.assertEqual(get_last_error(), 32) + self.assertEqual(ctypes.set_last_error(32), 126) + self.assertEqual(ctypes.get_last_error(), 32) def _worker(): - set_last_error(0) + ctypes.set_last_error(0) - dll = WinDLL("kernel32", use_last_error=False) + dll = ctypes.WinDLL("kernel32", use_last_error=False) GetModuleHandle = dll.GetModuleHandleW GetModuleHandle.argtypes = [c_wchar_p] GetModuleHandle("bar") - self.assertEqual(get_last_error(), 0) + self.assertEqual(ctypes.get_last_error(), 0) t = threading.Thread(target=_worker) t.start() t.join() - self.assertEqual(get_last_error(), 32) + self.assertEqual(ctypes.get_last_error(), 32) + + ctypes.set_last_error(0) - set_last_error(0) if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_find.py b/Lib/test/test_ctypes/test_find.py similarity index 53% rename from Lib/ctypes/test/test_find.py rename to Lib/test/test_ctypes/test_find.py index 1ff9d019b13..8bc84c3d2ef 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -1,11 +1,12 @@ -import unittest -import unittest.mock import os.path import sys import test.support -from test.support import os_helper -from ctypes import * +import unittest +import unittest.mock +from ctypes import CDLL, RTLD_GLOBAL from ctypes.util import find_library +from test.support import os_helper, thread_unsafe + # On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. class Test_OpenGL_libs(unittest.TestCase): @@ -22,7 +23,7 @@ def setUpClass(cls): lib_glu = find_library("GLU") lib_gle = find_library("gle") - ## print, for debugging + # print, for debugging if test.support.verbose: print("OpenGL libraries:") for item in (("GL", lib_gl), @@ -36,11 +37,13 @@ def setUpClass(cls): cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) except OSError: pass + if lib_glu: try: cls.glu = CDLL(lib_glu, RTLD_GLOBAL) except OSError: pass + if lib_gle: try: cls.gle = CDLL(lib_gle) @@ -75,6 +78,7 @@ def test_shell_injection(self): @unittest.skipUnless(sys.platform.startswith('linux'), 'Test only valid for Linux') class FindLibraryLinux(unittest.TestCase): + @thread_unsafe('uses setenv') def test_find_on_libpath(self): import subprocess import tempfile @@ -122,6 +126,100 @@ def test_find_library_with_ld(self): unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): self.assertNotEqual(find_library('c'), None) + def test_gh114257(self): + self.assertIsNone(find_library("libc")) + + +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + + +@unittest.skipUnless(test.support.is_emscripten, + 'Test only valid for Emscripten') +class FindLibraryEmscripten(unittest.TestCase): + @classmethod + def setUpClass(cls): + import tempfile + + # A very simple wasm module + # In WAT format: (module) + cls.wasm_module = b'\x00asm\x01\x00\x00\x00\x00\x08\x04name\x02\x01\x00' + + cls.non_wasm_content = b'This is not a WASM file' + + cls.temp_dir = tempfile.mkdtemp() + cls.libdummy_so_path = os.path.join(cls.temp_dir, 'libdummy.so') + with open(cls.libdummy_so_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libother_wasm_path = os.path.join(cls.temp_dir, 'libother.wasm') + with open(cls.libother_wasm_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libnowasm_so_path = os.path.join(cls.temp_dir, 'libnowasm.so') + with open(cls.libnowasm_so_path, 'wb') as f: + f.write(cls.non_wasm_content) + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls.temp_dir) + + def test_find_wasm_file_with_so_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('dummy') + self.assertEqual(result, self.libdummy_so_path) + def test_find_wasm_file_with_wasm_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('other') + self.assertEqual(result, self.libother_wasm_path) + + def test_ignore_non_wasm_file(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('nowasm') + self.assertIsNone(result) + + def test_find_nothing_without_ld_library_path(self): + with os_helper.EnvironmentVarGuard() as env: + if 'LD_LIBRARY_PATH' in env: + del env['LD_LIBRARY_PATH'] + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + def test_find_nothing_with_wrong_ld_library_path(self): + import tempfile + with tempfile.TemporaryDirectory() as empty_dir: + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', empty_dir) + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_frombuffer.py b/Lib/test/test_ctypes/test_frombuffer.py similarity index 97% rename from Lib/ctypes/test/test_frombuffer.py rename to Lib/test/test_ctypes/test_frombuffer.py index 55c244356b3..d4e161f864d 100644 --- a/Lib/ctypes/test/test_frombuffer.py +++ b/Lib/test/test_ctypes/test_frombuffer.py @@ -1,7 +1,10 @@ -from ctypes import * import array import gc import unittest +from ctypes import (Structure, Union, Array, sizeof, + _Pointer, _SimpleCData, _CFuncPtr, + c_char, c_int) + class X(Structure): _fields_ = [("c_int", c_int)] @@ -9,6 +12,7 @@ class X(Structure): def __init__(self): self._init_called = True + class Test(unittest.TestCase): def test_from_buffer(self): a = array.array("i", range(16)) @@ -121,8 +125,6 @@ def test_from_buffer_copy_with_offset(self): (c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int)) def test_abstract(self): - from ctypes import _Pointer, _SimpleCData, _CFuncPtr - self.assertRaises(TypeError, Array.from_buffer, bytearray(10)) self.assertRaises(TypeError, Structure.from_buffer, bytearray(10)) self.assertRaises(TypeError, Union.from_buffer, bytearray(10)) @@ -137,5 +139,6 @@ def test_abstract(self): self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123") self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py similarity index 66% rename from Lib/ctypes/test/test_funcptr.py rename to Lib/test/test_ctypes/test_funcptr.py index e0b9b54e97f..be641da30ea 100644 --- a/Lib/ctypes/test/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,16 +1,41 @@ +import ctypes import unittest -from ctypes import * +from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, + c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE, StructCheckMixin) + try: - WINFUNCTYPE -except NameError: + WINFUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux WINFUNCTYPE = CFUNCTYPE -import _ctypes_test lib = CDLL(_ctypes_test.__file__) -class CFuncPtrTestCase(unittest.TestCase): + +class CFuncPtrTestCase(unittest.TestCase, StructCheckMixin): + def test_inheritance_hierarchy(self): + self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) + + self.assertEqual(PyCFuncPtrType.__name__, "PyCFuncPtrType") + self.assertEqual(type(PyCFuncPtrType), type) + + def test_type_flags(self): + for cls in _CFuncPtr, PyCFuncPtrType: + with self.subTest(cls=cls): + self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) @@ -20,8 +45,8 @@ def func(*args): x = X(func) self.assertEqual(x.restype, c_int) self.assertEqual(x.argtypes, (c_int, c_int)) - self.assertEqual(sizeof(x), sizeof(c_voidp)) - self.assertEqual(sizeof(X), sizeof(c_voidp)) + self.assertEqual(sizeof(x), sizeof(c_void_p)) + self.assertEqual(sizeof(X), sizeof(c_void_p)) def test_first(self): StdCallback = WINFUNCTYPE(c_int, c_int, c_int) @@ -39,7 +64,7 @@ def func(a, b): # possible, as in C, to call cdecl functions with more parameters. #self.assertRaises(TypeError, c, 1, 2, 3) self.assertEqual(c(1, 2, 3, 4, 5, 6), 3) - if not WINFUNCTYPE is CFUNCTYPE: + if WINFUNCTYPE is not CFUNCTYPE: self.assertRaises(TypeError, s, 1, 2, 3) def test_structures(self): @@ -63,23 +88,16 @@ class WNDCLASS(Structure): ("hCursor", HCURSOR), ("lpszMenuName", LPCTSTR), ("lpszClassName", LPCTSTR)] + self.check_struct(WNDCLASS) wndclass = WNDCLASS() wndclass.lpfnWndProc = WNDPROC(wndproc) WNDPROC_2 = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) - # This is no longer true, now that WINFUNCTYPE caches created types internally. - ## # CFuncPtr subclasses are compared by identity, so this raises a TypeError: - ## self.assertRaises(TypeError, setattr, wndclass, - ## "lpfnWndProc", WNDPROC_2(wndproc)) - # instead: - self.assertIs(WNDPROC, WNDPROC_2) - # 'wndclass.lpfnWndProc' leaks 94 references. Why? self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10) - f = wndclass.lpfnWndProc del wndclass @@ -88,24 +106,14 @@ class WNDCLASS(Structure): self.assertEqual(f(10, 11, 12, 13), 46) def test_dllfunctions(self): - - def NoNullHandle(value): - if not value: - raise WinError() - return value - strchr = lib.my_strchr strchr.restype = c_char_p strchr.argtypes = (c_char_p, c_char) self.assertEqual(strchr(b"abcdefghi", b"b"), b"bcdefghi") self.assertEqual(strchr(b"abcdefghi", b"x"), None) - strtok = lib.my_strtok strtok.restype = c_char_p - # Neither of this does work: strtok changes the buffer it is passed -## strtok.argtypes = (c_char_p, c_char_p) -## strtok.argtypes = (c_string, c_char_p) def c_string(init): size = len(init) + 1 @@ -114,19 +122,14 @@ def c_string(init): s = b"a\nb\nc" b = c_string(s) -## b = (c_char * (len(s)+1))() -## b.value = s - -## b = c_string(s) self.assertEqual(strtok(b, b"\n"), b"a") self.assertEqual(strtok(None, b"\n"), b"b") self.assertEqual(strtok(None, b"\n"), b"c") self.assertEqual(strtok(None, b"\n"), None) def test_abstract(self): - from ctypes import _CFuncPtr - self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_functions.py b/Lib/test/test_ctypes/test_functions.py similarity index 73% rename from Lib/ctypes/test/test_functions.py rename to Lib/test/test_ctypes/test_functions.py index fc571700ce3..3454b83d43e 100644 --- a/Lib/ctypes/test/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,30 +1,36 @@ -""" -Here is probably the place to write the docs, since the test-cases -show how the type behave. +import ctypes +import sys +import unittest +from ctypes import (CDLL, Structure, Array, CFUNCTYPE, + byref, POINTER, pointer, ArgumentError, sizeof, + c_char, c_wchar, c_byte, c_char_p, c_wchar_p, + c_short, c_int, c_long, c_longlong, c_void_p, + c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from _ctypes import _Pointer, _SimpleCData -Later... -""" - -from ctypes import * -from ctypes.test import need_symbol -import sys, unittest try: - WINFUNCTYPE -except NameError: + WINFUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux WINFUNCTYPE = CFUNCTYPE -import _ctypes_test dll = CDLL(_ctypes_test.__file__) if sys.platform == "win32": - windll = WinDLL(_ctypes_test.__file__) + windll = ctypes.WinDLL(_ctypes_test.__file__) + class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + + class RECT(Structure): _fields_ = [("left", c_int), ("top", c_int), ("right", c_int), ("bottom", c_int)] + + class FunctionTestCase(unittest.TestCase): def test_mro(self): @@ -40,21 +46,35 @@ class X(object, Array): _length_ = 5 _type_ = "i" - from _ctypes import _Pointer with self.assertRaises(TypeError): - class X(object, _Pointer): + class X2(object, _Pointer): pass - from _ctypes import _SimpleCData with self.assertRaises(TypeError): - class X(object, _SimpleCData): + class X3(object, _SimpleCData): _type_ = "i" with self.assertRaises(TypeError): - class X(object, Structure): + class X4(object, Structure): _fields_ = [] - @need_symbol('c_wchar') + def test_c_char_parm(self): + proto = CFUNCTYPE(c_int, c_char) + def callback(*args): + return 0 + + callback = proto(callback) + + self.assertEqual(callback(b"a"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(b"abc") + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: one character bytes, " + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") + def test_wchar_parm(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] @@ -62,7 +82,79 @@ def test_wchar_parm(self): self.assertEqual(result, 139) self.assertEqual(type(result), int) - @need_symbol('c_wchar') + with self.assertRaises(ArgumentError) as cm: + f(1, 2, 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not instance of int") + + with self.assertRaises(ArgumentError) as cm: + f(1, "abc", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") + + def test_c_char_p_parm(self): + """Test the error message when converting an incompatible type to c_char_p.""" + proto = CFUNCTYPE(c_int, c_char_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(b"abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_char_p") + + def test_c_wchar_p_parm(self): + """Test the error message when converting an incompatible type to c_wchar_p.""" + proto = CFUNCTYPE(c_int, c_wchar_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback("abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_wchar_p") + + def test_c_void_p_parm(self): + """Test the error message when converting an incompatible type to c_void_p.""" + proto = CFUNCTYPE(c_int, c_void_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(5), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(2.5) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'float' object cannot be " + "interpreted as ctypes.c_void_p") + def test_wchar_result(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] @@ -128,7 +220,6 @@ def test_doubleresult(self): self.assertEqual(result, -21) self.assertEqual(type(result), float) - @need_symbol('c_longdouble') def test_longdoubleresult(self): f = dll._testfunc_D_bhilfD f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble] @@ -141,7 +232,6 @@ def test_longdoubleresult(self): self.assertEqual(result, -21) self.assertEqual(type(result), float) - @need_symbol('c_longlong') def test_longlongresult(self): f = dll._testfunc_q_bhilfd f.restype = c_longlong @@ -200,7 +290,6 @@ def test_pointers(self): result = f(byref(c_int(99))) self.assertNotEqual(result.contents, 99) - ################################################################ def test_shorts(self): f = dll._testfunc_callback_i_if @@ -218,9 +307,6 @@ def callback(v): f(2**18, cb) self.assertEqual(args, expected) - ################################################################ - - def test_callbacks(self): f = dll._testfunc_callback_i_if f.restype = c_int @@ -229,7 +315,6 @@ def test_callbacks(self): MyCallback = CFUNCTYPE(c_int, c_int) def callback(value): - #print "called back with", value return value cb = MyCallback(callback) @@ -262,7 +347,6 @@ def test_callbacks_2(self): f.argtypes = [c_int, MyCallback] def callback(value): - #print "called back with", value self.assertEqual(type(value), int) return value @@ -270,7 +354,6 @@ def callback(value): result = f(-10, cb) self.assertEqual(result, -18) - @need_symbol('c_longlong') def test_longlong_callbacks(self): f = dll._testfunc_callback_q_qf @@ -371,7 +454,7 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_sf1651235(self): - # see https://www.python.org/sf/1651235 + # see https://bugs.python.org/issue1651235 proto = CFUNCTYPE(c_int, RECT, POINT) def callback(*args): @@ -380,5 +463,6 @@ def callback(*args): callback = proto(callback) self.assertRaises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT())) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000..aa448fad5bb --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,761 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper, verbose +import re +from dataclasses import dataclass +from functools import cached_property +import sys + +import ctypes +from ctypes import Structure, Union +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + +from test.test_ctypes._support import StructCheckMixin + +# A 64-bit number where each nibble (hex digit) is different and +# has 2-3 bits set. +TEST_PATTERN = 0xae7596db + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + _layout_ = 'ms' + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + _layout_ = 'ms' + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + _layout_ = 'ms' + + +@register() +class Packed4(Structure): + def _maybe_skip(): + # `_pack_` enables MSVC-style packing, but keeps platform-specific + # alignments. + # The C code we generate for GCC/clang currently uses + # `__attribute__((ms_struct))`, which activates MSVC layout *and* + # alignments, that is, sizeof(basic type) == alignment(basic type). + # On a Pentium, int64 is 32-bit aligned, so the two won't match. + # The expected behavior is instead tested in + # StructureTestCase.test_packed, over in test_structures.py. + if sizeof(c_int64) != alignment(c_int64): + raise unittest.SkipTest('cannot test on this platform') + + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + _layout_ = 'ms' + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase, StructCheckMixin): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits unchanged. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + self.check_struct_or_union(cls) + if _maybe_skip := getattr(cls, '_maybe_skip', None): + _maybe_skip() + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, TEST_PATTERN, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + descriptor = field.descriptor + field_mem = py_mem[ + field.byte_offset + : field.byte_offset + descriptor.byte_size] + field_int = int.from_bytes(field_mem, sys.byteorder) + mask = (1 << descriptor.bit_size) - 1 + self.assertEqual( + (field_int >> descriptor.bit_offset) & mask, + value & mask) + + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms': + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + descriptor: object + byte_offset: int + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + descriptor = getattr(tp, f_name) + byte_offset = descriptor.byte_offset + if parent: + byte_offset += parent.byte_offset + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent, descriptor, byte_offset) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output("/* Generated by Lib/test/test_ctypes/test_generated_structs.py */") + output(f"#define TEST_PATTERN {TEST_PATTERN}") + output(""" + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to test values; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, (TYPE)TEST_PATTERN) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py new file mode 100644 index 00000000000..fefdfe9102e --- /dev/null +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -0,0 +1,59 @@ +import ctypes +import unittest +import warnings +from ctypes import Structure, POINTER, pointer, c_char_p + +# String-based "incomplete pointers" were implemented in ctypes 0.6.3 (2003, when +# ctypes was an external project). They made obsolete by the current +# incomplete *types* (setting `_fields_` late) in 0.9.5 (2005). +# ctypes was added to Python 2.5 (2006), without any mention in docs. + +# This tests incomplete pointer example from the old tutorial +# (https://svn.python.org/projects/ctypes/tags/release_0_6_3/ctypes/docs/tutorial.stx) +class TestSetPointerType(unittest.TestCase): + def tearDown(self): + ctypes._pointer_type_cache_fallback.clear() + + def test_incomplete_example(self): + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") + class cell(Structure): + _fields_ = [("name", c_char_p), + ("next", lpcell)] + + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + ctypes.SetPointerType(lpcell, cell) + + self.assertIs(POINTER(cell), lpcell) + + c1 = cell() + c1.name = b"foo" + c2 = cell() + c2.name = b"bar" + + c1.next = pointer(c2) + c2.next = pointer(c1) + + p = c1 + + result = [] + for i in range(8): + result.append(p.name) + p = p.next[0] + self.assertEqual(result, [b"foo", b"bar"] * 4) + + def test_deprecation(self): + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") + class cell(Structure): + _fields_ = [("name", c_char_p), + ("next", lpcell)] + + with self.assertWarns(DeprecationWarning): + ctypes.SetPointerType(lpcell, cell) + + self.assertIs(POINTER(cell), lpcell) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/ctypes/test/test_init.py b/Lib/test/test_ctypes/test_init.py similarity index 96% rename from Lib/ctypes/test/test_init.py rename to Lib/test/test_ctypes/test_init.py index 75fad112a01..113425e5823 100644 --- a/Lib/ctypes/test/test_init.py +++ b/Lib/test/test_ctypes/test_init.py @@ -1,5 +1,6 @@ -from ctypes import * import unittest +from ctypes import Structure, c_int + class X(Structure): _fields_ = [("a", c_int), @@ -15,6 +16,7 @@ def __init__(self): self.a = 9 self.b = 12 + class Y(Structure): _fields_ = [("x", X)] @@ -36,5 +38,6 @@ def test_get(self): self.assertEqual((y.x.a, y.x.b), (9, 12)) self.assertEqual(y.x.new_was_called, False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_internals.py b/Lib/test/test_ctypes/test_internals.py similarity index 85% rename from Lib/ctypes/test/test_internals.py rename to Lib/test/test_ctypes/test_internals.py index 271e3f57f81..778da6573da 100644 --- a/Lib/ctypes/test/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -1,7 +1,4 @@ # This tests the internal _objects attribute -import unittest -from ctypes import * -from sys import getrefcount as grc # XXX This test must be reviewed for correctness!!! @@ -14,22 +11,27 @@ # # What about pointers? +import sys +import unittest +from ctypes import Structure, POINTER, c_char_p, c_int + + class ObjectsTestCase(unittest.TestCase): def assertSame(self, a, b): self.assertEqual(id(a), id(b)) def test_ints(self): i = 42000123 - refcnt = grc(i) + refcnt = sys.getrefcount(i) ci = c_int(i) - self.assertEqual(refcnt, grc(i)) + self.assertEqual(refcnt, sys.getrefcount(i)) self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" - refcnt = grc(s) + s = "Hello, World".encode("ascii") + refcnt = sys.getrefcount(s) cs = c_char_p(s) - self.assertEqual(refcnt + 1, grc(s)) + self.assertEqual(refcnt + 1, sys.getrefcount(s)) self.assertSame(cs._objects, s) def test_simple_struct(self): @@ -78,9 +80,6 @@ class Y(Structure): y = Y() y.x = x self.assertEqual(y._objects, {"0": {"0": s1, "1": s2}}) -## x = y.x -## del y -## print x._b_base_._objects def test_ptr_struct(self): class X(Structure): @@ -92,9 +91,7 @@ class X(Structure): x = X() x.data = a -##XXX print x._objects -##XXX print x.data[0] -##XXX print x.data._objects + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py similarity index 72% rename from Lib/ctypes/test/test_keeprefs.py rename to Lib/test/test_ctypes/test_keeprefs.py index 94c02573fa1..5602460d5ff 100644 --- a/Lib/ctypes/test/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -1,5 +1,6 @@ -from ctypes import * import unittest +from ctypes import (Structure, POINTER, pointer, c_char_p, c_int) + class SimpleTestCase(unittest.TestCase): def test_cint(self): @@ -18,6 +19,7 @@ def test_ccharp(self): x = c_char_p(b"spam") self.assertEqual(x._objects, b"spam") + class StructureTestCase(unittest.TestCase): def test_cint_struct(self): class X(Structure): @@ -64,6 +66,7 @@ class RECT(Structure): r.lr = POINT() self.assertEqual(r._objects, {'0': {}, '1': {}}) + class ArrayTestCase(unittest.TestCase): def test_cint_array(self): INTARR = c_int * 3 @@ -87,43 +90,13 @@ class X(Structure): x.a = ia self.assertEqual(x._objects, {'1': {}}) + class PointerTestCase(unittest.TestCase): def test_p_cint(self): i = c_int(42) x = pointer(i) self.assertEqual(x._objects, {'1': i}) -class DeletePointerTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): - class X(Structure): - _fields_ = [("p", POINTER(c_char_p))] - x = X() - i = c_char_p("abc def") - from sys import getrefcount as grc - print("2?", grc(i)) - x.p = pointer(i) - print("3?", grc(i)) - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) -## del x -## print "2?", grc(i) -## del i - import gc - gc.collect() - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) - print(x.p.contents) -## print x._objects - - x.p[0] = "spam spam" -## print x.p[0] - print("+" * 42) - print(x._objects) class PointerToStructure(unittest.TestCase): def test(self): @@ -137,17 +110,10 @@ class RECT(Structure): r.a = pointer(p1) r.b = pointer(p1) -## from pprint import pprint as pp -## pp(p1._objects) -## pp(r._objects) r.a[0].x = 42 r.a[0].y = 99 - # to avoid leaking when tests are run several times - # clean up the types left in the cache. - from ctypes import _pointer_type_cache - del _pointer_type_cache[POINT] if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py new file mode 100644 index 00000000000..df7dbc0ae26 --- /dev/null +++ b/Lib/test/test_ctypes/test_libc.py @@ -0,0 +1,64 @@ +import ctypes +import math +import unittest +from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, + c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + + +lib = CDLL(_ctypes_test.__file__) + + +def three_way_cmp(x, y): + """Return -1 if x < y, 0 if x == y and 1 if x > y""" + return (x > y) - (x < y) + + +class LibTest(unittest.TestCase): + def test_sqrt(self): + lib.my_sqrt.argtypes = c_double, + lib.my_sqrt.restype = c_double + self.assertEqual(lib.my_sqrt(4.0), 2.0) + self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type and libffi >= 3.3.0") + def test_csqrt(self): + lib.my_csqrt.argtypes = ctypes.c_double_complex, + lib.my_csqrt.restype = ctypes.c_double_complex + self.assertEqual(lib.my_csqrt(4), 2+0j) + self.assertAlmostEqual(lib.my_csqrt(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrt(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtf.argtypes = ctypes.c_float_complex, + lib.my_csqrtf.restype = ctypes.c_float_complex + self.assertAlmostEqual(lib.my_csqrtf(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtf(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtl.argtypes = ctypes.c_longdouble_complex, + lib.my_csqrtl.restype = ctypes.c_longdouble_complex + self.assertAlmostEqual(lib.my_csqrtl(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtl(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + def test_qsort(self): + comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) + lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc + lib.my_qsort.restype = None + + def sort(a, b): + return three_way_cmp(a[0], b[0]) + + chars = create_string_buffer(b"spam, spam, and spam") + lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) + self.assertEqual(chars.raw, b" ,,aaaadmmmnpppsss\x00") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/ctypes/test/test_loading.py b/Lib/test/test_ctypes/test_loading.py similarity index 73% rename from Lib/ctypes/test/test_loading.py rename to Lib/test/test_ctypes/test_loading.py index ea892277c4e..3b8332fbb30 100644 --- a/Lib/ctypes/test/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,16 +1,20 @@ -from ctypes import * +import _ctypes +import ctypes import os import shutil import subprocess import sys -import unittest import test.support -from test.support import import_helper -from test.support import os_helper +import unittest +from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library +from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + libc_name = None + def setUpModule(): global libc_name if os.name == "nt": @@ -23,15 +27,22 @@ def setUpModule(): if test.support.verbose: print("libc_name is", libc_name) + class LoaderTest(unittest.TestCase): unknowndll = "xxrandomnamexx" def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) + if libc_name is not None: + test_lib = libc_name + else: + if os.name == "nt": + test_lib = _ctypes_test.__file__ + else: + self.skipTest('could not find library to load') + CDLL(test_lib) + CDLL(os.path.basename(test_lib)) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): @@ -45,11 +56,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') @@ -62,18 +77,17 @@ def test_load_library(self): print(find_library("user32")) if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW + ctypes.windll.kernel32.GetModuleHandleW + ctypes.windll["kernel32"].GetModuleHandleW + ctypes.windll.LoadLibrary("kernel32").GetModuleHandleW + ctypes.WinDLL("kernel32").GetModuleHandleW # embedded null character - self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") + self.assertRaises(ValueError, ctypes.windll.LoadLibrary, "kernel32\0") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) + dll = ctypes.WinDLL(_ctypes_test.__file__) # We load the same function both via ordinal and name func_ord = dll[2] func_name = dll.GetString @@ -86,16 +100,21 @@ def test_load_ordinal_functions(self): self.assertRaises(AttributeError, dll.__getitem__, 1234) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_load_without_name_and_with_handle(self): + handle = ctypes.windll.kernel32._handle + lib = ctypes.WinDLL(name=None, handle=handle) + self.assertIs(handle, lib._handle) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary # On winXP 64-bit, advapi32 loads at an address that does # NOT fit into a 32-bit integer. FreeLibrary must be able # to accept this address. - # These are tests for https://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) + # These are tests for https://bugs.python.org/issue1703286 + handle = _ctypes.LoadLibrary("advapi32") + _ctypes.FreeLibrary(handle) @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_B(self): @@ -103,25 +122,33 @@ def test_1703286_B(self): # above, the (arbitrarily selected) CloseEventLog function # also has a high address. 'call_function' should accept # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 + + advapi32 = ctypes.windll.advapi32 # Calling CloseEventLog with a NULL argument should fail, # but the call should not segfault or so. self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") + + kernel32 = ctypes.windll.kernel32 + kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + kernel32.GetProcAddress.restype = c_void_p + proc = kernel32.GetProcAddress(advapi32._handle, b"CloseEventLog") self.assertTrue(proc) + # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) + self.assertEqual(0, _ctypes.call_function(proc, (None,))) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertNotHasAttr(ctypes.windll, 'test') @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_dll_with_flags(self): _sqlite3 = import_helper.import_module("_sqlite3") src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): + if os.path.basename(src).partition(".")[0].lower().endswith("_d"): ext = "_d.dll" else: ext = ".dll" @@ -177,6 +204,5 @@ def should_fail(command): "WinDLL('_sqlite3.dll'); p.close()") - if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/test/test_ctypes/test_macholib.py similarity index 99% rename from Lib/ctypes/test/test_macholib.py rename to Lib/test/test_ctypes/test_macholib.py index bc75f1a05a8..9d906179956 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/test/test_ctypes/test_macholib.py @@ -1,7 +1,3 @@ -import os -import sys -import unittest - # Bob Ippolito: # # Ok.. the code to find the filename for __getattr__ should look @@ -31,10 +27,15 @@ # # -bob +import os +import sys +import unittest + from ctypes.macholib.dyld import dyld_find from ctypes.macholib.dylib import dylib_info from ctypes.macholib.framework import framework_info + def find_lib(name): possible = ['lib'+name+'.dylib', name+'.dylib', name+'.framework/'+name] for dylib in possible: @@ -106,5 +107,6 @@ def test_framework_info(self): self.assertEqual(framework_info('P/F.framework/Versions/A/F_debug'), d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug')) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py similarity index 52% rename from Lib/ctypes/test/test_memfunctions.py rename to Lib/test/test_ctypes/test_memfunctions.py index e784b9a7068..e3cb5db775e 100644 --- a/Lib/ctypes/test/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -1,20 +1,25 @@ import sys -from test import support import unittest -from ctypes import * -from ctypes.test import need_symbol +from test import support +from ctypes import (POINTER, sizeof, cast, + create_string_buffer, string_at, + create_unicode_buffer, wstring_at, + memmove, memset, + memoryview_at, c_void_p, + c_char_p, c_byte, c_ubyte, c_wchar, + addressof, byref) + class MemFunctionsTest(unittest.TestCase): - @unittest.skip('test disabled') def test_overflow(self): # string_at and wstring_at must use the Python calling # convention (which acquires the GIL and checks the Python # error flag). Provoke an error and catch it; see also issue - # #3554: + # gh-47804. self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: wstring_at(u"foo", sys.maxint - 1)) + lambda: wstring_at(u"foo", sys.maxsize - 1)) self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: string_at("foo", sys.maxint - 1)) + lambda: string_at("foo", sys.maxsize - 1)) def test_memmove(self): # large buffers apparently increase the chance that the memory @@ -55,15 +60,11 @@ def test_cast(self): @support.refcount_test def test_string_at(self): s = string_at(b"foo bar") - # XXX The following may be wrong, depending on how Python - # manages string instances - self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at(b"foo bar", 7), b"foo bar") self.assertEqual(string_at(b"foo bar", 3), b"foo") - @need_symbol('create_unicode_buffer') def test_wstring_at(self): p = create_unicode_buffer("Hello, World") a = create_unicode_buffer(1000000) @@ -75,5 +76,62 @@ def test_wstring_at(self): self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") self.assertEqual(wstring_at(a, 0), "") + def test_memoryview_at(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to memoryview get reflected in source buffer + v[:] = b"9876543210" + self.assertEqual(bytes(b), b"9876543210") + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, -1) + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, sys.maxsize + 1) + + v0 = memoryview_at(foreign_ptr, 0) + self.assertEqual(bytes(v0), b'') + + def test_memoryview_at_readonly(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size, readonly=True) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to the memoryview are blocked + with self.assertRaises(TypeError): + v[:] = b"9876543210" + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py similarity index 71% rename from Lib/ctypes/test/test_numbers.py rename to Lib/test/test_ctypes/test_numbers.py index a5c661b0e97..c57c58eb002 100644 --- a/Lib/ctypes/test/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,6 +1,16 @@ -from ctypes import * -import unittest +import array +import ctypes import struct +import sys +import unittest +from itertools import combinations +from operator import truth +from ctypes import (byref, sizeof, alignment, + c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_float, c_double, c_longdouble, c_bool) +from test.support.testcase import ComplexesAreIdenticalMixin + def valid_ranges(*types): # given a sequence of numeric types, collect their _type_ @@ -19,38 +29,40 @@ def valid_ranges(*types): result.append((min(a, b, c, d), max(a, b, c, d))) return result + ArgType = type(byref(c_int(0))) -unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong] +unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong] signed_types = [c_byte, c_short, c_int, c_long, c_longlong] - -bool_types = [] - +bool_types = [c_bool] float_types = [c_double, c_float] -try: - c_ulonglong - c_longlong -except NameError: - pass -else: - unsigned_types.append(c_ulonglong) - signed_types.append(c_longlong) - -try: - c_bool -except NameError: - pass -else: - bool_types.append(c_bool) - unsigned_ranges = valid_ranges(*unsigned_types) signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] -################################################################ +class IntLike: + def __int__(self): + return 2 + +class IndexLike: + def __index__(self): + return 2 -class NumberTestCase(unittest.TestCase): +class FloatLike: + def __float__(self): + return 2.0 + +class ComplexLike: + def __complex__(self): + return 1+1j + + +INF = float("inf") +NAN = float("nan") + + +class NumberTestCase(unittest.TestCase, ComplexesAreIdenticalMixin): def test_default_init(self): # default values are set to zero @@ -71,7 +83,6 @@ def test_signed_values(self): self.assertEqual(t(h).value, h) def test_bool_values(self): - from operator import truth for t, v in zip(bool_types, bool_values): self.assertEqual(t(v).value, truth(v)) @@ -98,9 +109,6 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() for t in float_types: self.assertEqual(t(2.0).value, 2.0) @@ -108,18 +116,34 @@ def __float__(self): self.assertEqual(t(2).value, 2.0) self.assertEqual(t(f).value, 2.0) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex(self): + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + self.assertEqual(t(1).value, 1+0j) + self.assertEqual(t(1.0).value, 1+0j) + self.assertEqual(t(1+0.125j).value, 1+0.125j) + self.assertEqual(t(IndexLike()).value, 2+0j) + self.assertEqual(t(FloatLike()).value, 2+0j) + self.assertEqual(t(ComplexLike()).value, 1+1j) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex_round_trip(self): + # Ensure complexes transformed exactly. The CMPLX macro should + # preserve special components (like inf/nan or signed zero). + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + with self.subTest(z=z, type=t): + self.assertComplexesAreIdentical(z, t(z).value) + def test_integers(self): - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() - class IntLike: - def __int__(self): - return 2 d = IntLike() - class IndexLike: - def __index__(self): - return 2 i = IndexLike() # integers cannot be constructed from floats, # but from integer-like objects @@ -147,21 +171,20 @@ def test_alignments(self): # alignment of the type... self.assertEqual((code, alignment(t)), - (code, align)) + (code, align)) # and alignment of an instance self.assertEqual((code, alignment(t())), - (code, align)) + (code, align)) def test_int_from_address(self): - from array import array for t in signed_types + unsigned_types: # the array module doesn't support all format codes # (no 'q' or 'Q') try: - array(t._type_) + array.array(t._type_) except ValueError: continue - a = array(t._type_, [100]) + a = array.array(t._type_, [100]) # v now is an integer at an 'external' memory location v = t.from_address(a.buffer_info()[0]) @@ -174,9 +197,8 @@ def test_int_from_address(self): def test_float_from_address(self): - from array import array for t in float_types: - a = array(t._type_, [3.14]) + a = array.array(t._type_, [3.14]) v = t.from_address(a.buffer_info()[0]) self.assertEqual(v.value, a[0]) self.assertIs(type(v), t) @@ -185,10 +207,7 @@ def test_float_from_address(self): self.assertIs(type(v), t) def test_char_from_address(self): - from ctypes import c_char - from array import array - - a = array('b', [0]) + a = array.array('b', [0]) a[0] = ord('x') v = c_char.from_address(a.buffer_info()[0]) self.assertEqual(v.value, b'x') @@ -204,7 +223,6 @@ def test_init(self): self.assertRaises(TypeError, c_int, c_long(42)) def test_float_overflow(self): - import sys big_int = int(sys.float_info.max) * 2 for t in float_types + [c_longdouble]: self.assertRaises(OverflowError, t, big_int) diff --git a/Lib/ctypes/test/test_objects.py b/Lib/test/test_ctypes/test_objects.py similarity index 78% rename from Lib/ctypes/test/test_objects.py rename to Lib/test/test_ctypes/test_objects.py index 19e3dc1f2d7..fb01421b955 100644 --- a/Lib/ctypes/test/test_objects.py +++ b/Lib/test/test_ctypes/test_objects.py @@ -11,7 +11,7 @@ Here is an array of string pointers: ->>> from ctypes import * +>>> from ctypes import Structure, c_int, c_char_p >>> array = (c_char_p * 5)() >>> print(array._objects) None @@ -42,7 +42,7 @@ of 'x' ('_b_base_' is either None, or the root object owning the memory block): >>> print(x.array._b_base_) # doctest: +ELLIPSIS - + >>> >>> x.array[0] = b'spam spam spam' @@ -51,17 +51,16 @@ >>> x.array._b_base_._objects {'0:2': b'spam spam spam'} >>> - ''' -import unittest, doctest +import doctest +import unittest + -import ctypes.test.test_objects +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests -class TestCase(unittest.TestCase): - def test(self): - failures, tests = doctest.testmod(ctypes.test.test_objects) - self.assertFalse(failures, 'doctests failed, see output above') if __name__ == '__main__': - doctest.testmod(ctypes.test.test_objects) + unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py similarity index 69% rename from Lib/ctypes/test/test_parameters.py rename to Lib/test/test_ctypes/test_parameters.py index 38af7ac13d7..46f8ff93efa 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,11 +1,25 @@ +import sys import unittest -from ctypes.test import need_symbol import test.support +from ctypes import (CDLL, PyDLL, ArgumentError, + Structure, Array, Union, + _Pointer, _SimpleCData, _CFuncPtr, + POINTER, pointer, byref, sizeof, + c_void_p, c_char_p, c_wchar_p, py_object, + c_bool, + c_char, c_wchar, + c_byte, c_ubyte, + c_short, c_ushort, + c_int, c_uint, + c_long, c_ulong, + c_longlong, c_ulonglong, + c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -class SimpleTypesTestCase(unittest.TestCase): +class SimpleTypesTestCase(unittest.TestCase): def setUp(self): - import ctypes try: from _ctypes import set_conversion_mode except ImportError: @@ -22,7 +36,6 @@ def tearDown(self): set_conversion_mode(*self.prev_conv_mode) def test_subclasses(self): - from ctypes import c_void_p, c_char_p # ctypes 0.9.5 and before did overwrite from_param in SimpleType_new class CVOIDP(c_void_p): def from_param(cls, value): @@ -37,10 +50,7 @@ def from_param(cls, value): self.assertEqual(CVOIDP.from_param("abc"), "abcabc") self.assertEqual(CCHARP.from_param("abc"), "abcabcabcabc") - @need_symbol('c_wchar_p') def test_subclasses_c_wchar_p(self): - from ctypes import c_wchar_p - class CWCHARP(c_wchar_p): def from_param(cls, value): return value * 3 @@ -50,8 +60,6 @@ def from_param(cls, value): # XXX Replace by c_char_p tests def test_cstrings(self): - from ctypes import c_char_p - # c_char_p.from_param on a Python String packs the string # into a cparam object s = b"123" @@ -67,10 +75,7 @@ def test_cstrings(self): a = c_char_p(b"123") self.assertIs(c_char_p.from_param(a), a) - @need_symbol('c_wchar_p') def test_cw_strings(self): - from ctypes import c_wchar_p - c_wchar_p.from_param("123") self.assertRaises(TypeError, c_wchar_p.from_param, 42) @@ -79,12 +84,41 @@ def test_cw_strings(self): pa = c_wchar_p.from_param(c_wchar_p("123")) self.assertEqual(type(pa), c_wchar_p) + def test_c_char(self): + with self.assertRaises(TypeError) as cm: + c_char.from_param(b"abc") + self.assertEqual(str(cm.exception), + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") + + def test_c_wchar(self): + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("abc") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 3") + + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") + + with self.assertRaises(TypeError) as cm: + c_wchar.from_param(123) + self.assertEqual(str(cm.exception), + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + + def test_int_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer LPINT = POINTER(c_int) -## p = pointer(c_int(42)) -## x = LPINT.from_param(p) x = LPINT.from_param(pointer(c_int(42))) self.assertEqual(x.contents.value, 42) self.assertEqual(LPINT(c_int(42)).contents.value, 42) @@ -99,7 +133,6 @@ def test_int_pointers(self): def test_byref_pointer(self): # The from_param class method of POINTER(typ) classes accepts what is # returned by byref(obj), it type(obj) == typ - from ctypes import c_short, c_uint, c_int, c_long, POINTER, byref LPINT = POINTER(c_int) LPINT.from_param(byref(c_int(42))) @@ -111,7 +144,6 @@ def test_byref_pointer(self): def test_byref_pointerpointer(self): # See above - from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref LPLPINT = POINTER(POINTER(c_int)) LPLPINT.from_param(byref(pointer(c_int(42)))) @@ -122,7 +154,6 @@ def test_byref_pointerpointer(self): self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_uint(22)))) def test_array_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER INTARRAY = c_int * 3 ia = INTARRAY() self.assertEqual(len(ia), 3) @@ -137,15 +168,12 @@ def test_array_pointers(self): self.assertRaises(TypeError, LPINT.from_param, c_uint*3) def test_noctypes_argtype(self): - import _ctypes_test - from ctypes import CDLL, c_void_p, ArgumentError - func = CDLL(_ctypes_test.__file__)._testfunc_p_p func.restype = c_void_p # TypeError: has no from_param method self.assertRaises(TypeError, setattr, func, "argtypes", (object,)) - class Adapter(object): + class Adapter: def from_param(cls, obj): return None @@ -153,7 +181,7 @@ def from_param(cls, obj): self.assertEqual(func(None), None) self.assertEqual(func(object()), None) - class Adapter(object): + class Adapter: def from_param(cls, obj): return obj @@ -162,7 +190,7 @@ def from_param(cls, obj): self.assertRaises(ArgumentError, func, object()) self.assertEqual(func(c_void_p(42)), 42) - class Adapter(object): + class Adapter: def from_param(cls, obj): raise ValueError(obj) @@ -171,9 +199,6 @@ def from_param(cls, obj): self.assertRaises(ArgumentError, func, 99) def test_abstract(self): - from ctypes import (Array, Structure, Union, _Pointer, - _SimpleCData, _CFuncPtr) - self.assertRaises(TypeError, Array.from_param, 42) self.assertRaises(TypeError, Structure.from_param, 42) self.assertRaises(TypeError, Union.from_param, 42) @@ -185,7 +210,6 @@ def test_abstract(self): def test_issue31311(self): # __setstate__ should neither raise a SystemError nor crash in case # of a bad __dict__. - from ctypes import Structure class BadStruct(Structure): @property @@ -202,27 +226,6 @@ def __dict__(self): WorseStruct().__setstate__({}, b'foo') def test_parameter_repr(self): - from ctypes import ( - c_bool, - c_char, - c_wchar, - c_byte, - c_ubyte, - c_short, - c_ushort, - c_int, - c_uint, - c_long, - c_ulong, - c_longlong, - c_ulonglong, - c_float, - c_double, - c_longdouble, - c_char_p, - c_wchar_p, - c_void_p, - ) self.assertRegex(repr(c_bool.from_param(True)), r"^$") self.assertEqual(repr(c_char.from_param(97)), "") self.assertRegex(repr(c_wchar.from_param('a')), r"^$") @@ -238,13 +241,62 @@ def test_parameter_repr(self): self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") self.assertEqual(repr(c_float.from_param(1.5)), "") self.assertEqual(repr(c_double.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1e300)), "") + if sys.float_repr_style == 'short': + self.assertEqual(repr(c_double.from_param(1e300)), "") self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") -################################################################ + @test.support.cpython_only + def test_from_param_result_refcount(self): + # Issue #99952 + class X(Structure): + """This struct size is <= sizeof(void*).""" + _fields_ = [("a", c_void_p)] + + def __del__(self): + trace.append(4) + + @classmethod + def from_param(cls, value): + trace.append(2) + return cls() + + PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append + PyList_Append.restype = c_int + PyList_Append.argtypes = [py_object, py_object, X] + + trace = [] + trace.append(1) + PyList_Append(trace, 3, "dummy") + trace.append(5) + + self.assertEqual(trace, [1, 2, 3, 4, 5]) + + class Y(Structure): + """This struct size is > sizeof(void*).""" + _fields_ = [("a", c_void_p), ("b", c_void_p)] + + def __del__(self): + trace.append(4) + + @classmethod + def from_param(cls, value): + trace.append(2) + return cls() + + PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append + PyList_Append.restype = c_int + PyList_Append.argtypes = [py_object, py_object, Y] + + trace = [] + trace.append(1) + PyList_Append(trace, 3, "dummy") + trace.append(5) + + self.assertEqual(trace, [1, 2, 3, 4, 5]) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py similarity index 69% rename from Lib/ctypes/test/test_pep3118.py rename to Lib/test/test_ctypes/test_pep3118.py index efffc80a66f..11a0744f5a8 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -1,6 +1,13 @@ +import re +import sys import unittest -from ctypes import * -import re, sys +from ctypes import (CFUNCTYPE, POINTER, sizeof, Union, + Structure, LittleEndianStructure, BigEndianStructure, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_uint64, + c_bool, c_float, c_double, c_longdouble, py_object) + if sys.byteorder == "little": THIS_ENDIAN = "<" @@ -9,6 +16,7 @@ THIS_ENDIAN = ">" OTHER_ENDIAN = "<" + def normalize(format): # Remove current endian specifier and white space from a format # string @@ -17,65 +25,54 @@ def normalize(format): format = format.replace(OTHER_ENDIAN, THIS_ENDIAN) return re.sub(r"\s", "", format) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_native_types(self): for tp, fmt, shape, itemtp in native_types: ob = tp() v = memoryview(ob) - try: - self.assertEqual(normalize(v.format), normalize(fmt)) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides - # if requested. memoryview currently reconstructs missing - # stride information, so this assert will fail. - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n * v.itemsize, len(v.tobytes())) - except: - # so that we can see the failing type - print(tp) - raise + self.assertEqual(normalize(v.format), normalize(fmt)) + if shape: + self.assertEqual(len(v), shape[0]) + else: + self.assertRaises(TypeError, len, v) + self.assertEqual(v.itemsize, sizeof(itemtp)) + self.assertEqual(v.shape, shape) + # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides + # if requested. memoryview currently reconstructs missing + # stride information, so this assert will fail. + # self.assertEqual(v.strides, ()) + + # they are always read/write + self.assertFalse(v.readonly) + + n = 1 + for dim in v.shape: + n = n * dim + self.assertEqual(n * v.itemsize, len(v.tobytes())) def test_endian_types(self): for tp, fmt, shape, itemtp in endian_types: ob = tp() v = memoryview(ob) - try: - self.assertEqual(v.format, fmt) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851 - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n, len(v)) - except: - # so that we can see the failing type - print(tp) - raise + self.assertEqual(v.format, fmt) + if shape: + self.assertEqual(len(v), shape[0]) + else: + self.assertRaises(TypeError, len, v) + self.assertEqual(v.itemsize, sizeof(itemtp)) + self.assertEqual(v.shape, shape) + # XXX Issue #12851 + # self.assertEqual(v.strides, ()) + + # they are always read/write + self.assertFalse(v.readonly) + + n = 1 + for dim in v.shape: + n = n * dim + self.assertEqual(n * v.itemsize, len(v.tobytes())) + # define some structure classes @@ -84,8 +81,25 @@ class Point(Structure): class PackedPoint(Structure): _pack_ = 2 + _layout_ = 'ms' _fields_ = [("x", c_long), ("y", c_long)] +class PointMidPad(Structure): + _fields_ = [("x", c_byte), ("y", c_uint)] + +class PackedPointMidPad(Structure): + _pack_ = 2 + _layout_ = 'ms' + _fields_ = [("x", c_byte), ("y", c_uint64)] + +class PointEndPad(Structure): + _fields_ = [("x", c_uint), ("y", c_byte)] + +class PackedPointEndPad(Structure): + _pack_ = 2 + _layout_ = 'ms' + _fields_ = [("x", c_uint64), ("y", c_byte)] + class Point2(Structure): pass Point2._fields_ = [("x", c_long), ("y", c_long)] @@ -107,6 +121,7 @@ class Complete(Structure): PComplete = POINTER(Complete) Complete._fields_ = [("a", c_long)] + ################################################################ # # This table contains format strings as they look on little endian @@ -185,11 +200,14 @@ class Complete(Structure): ## structures and unions - (Point, "T{l:x:>l:y:}".replace('l', s_long), (), BEPoint), - (LEPoint, "T{l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)), (POINTER(LEPoint), "&T{ has no attribute '__pointer_type__'"): + Cls.__pointer_type__ + + p = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, p) + + def test_arbitrary_pointer_type_attribute(self): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + garbage = 'garbage' + + P = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, P) + Cls.__pointer_type__ = garbage + self.assertIs(Cls.__pointer_type__, garbage) + self.assertIs(POINTER(Cls), garbage) + self.assertIs(P._type_, Cls) + + instance = Cls(1, 2.0) + pointer = P(instance) + self.assertEqual(pointer[0].a, 1) + self.assertEqual(pointer[0].b, 2) + + del Cls.__pointer_type__ + + NewP = POINTER(Cls) + self.assertIsNot(NewP, P) + self.assertIs(Cls.__pointer_type__, NewP) + self.assertIs(P._type_, Cls) + + def test_pointer_types_factory(self): + """Shouldn't leak""" + def factory(): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + return Cls + + ws_typ = WeakSet() + ws_ptr = WeakSet() + for _ in range(10): + typ = factory() + ptr = POINTER(typ) + + ws_typ.add(typ) + ws_ptr.add(ptr) + + typ = None + ptr = None + + gc.collect() + + self.assertEqual(len(ws_typ), 0, ws_typ) + self.assertEqual(len(ws_ptr), 0, ws_ptr) + + def test_pointer_proto_missing_argtypes_error(self): + class BadType(ctypes._Pointer): + # _type_ is intentionally missing + pass + + func = ctypes.pythonapi.Py_GetVersion + func.argtypes = (BadType,) + + with self.assertRaises(ctypes.ArgumentError): + func(object()) + +class PointerTypeCacheTestCase(unittest.TestCase): + # dummy tests to check warnings and base behavior + def tearDown(self): + _pointer_type_cache_fallback.clear() + + def test_deprecated_cache_with_not_ctypes_type(self): + class C: + pass + + with self.assertWarns(DeprecationWarning): + P = POINTER("C") + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache["C"], P) + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P + self.assertIs(C.__pointer_type__, P) + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P) + + def test_deprecated_cache_with_ints(self): + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[123] = 456 + + with self.assertWarns(DeprecationWarning): + self.assertEqual(_pointer_type_cache[123], 456) + + def test_deprecated_cache_with_ctypes_type(self): + class C(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int)] + + P1 = POINTER(C) + with self.assertWarns(DeprecationWarning): + P2 = POINTER("C") + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P2 + + self.assertIs(C.__pointer_type__, P2) + self.assertIsNot(C.__pointer_type__, P1) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P2) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache.get(C), P2) + + def test_get_not_registered(self): + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str)) + + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str, None)) + + def test_repeated_set_type(self): + # Regression test for gh-133290 + class C(Structure): + _fields_ = [('a', c_int)] + ptr = POINTER(C) + # Read _type_ several times to warm up cache + for i in range(5): + self.assertIs(ptr._type_, C) + ptr.set_type(c_int) + self.assertIs(ptr._type_, c_int) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/ctypes/test/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py similarity index 78% rename from Lib/ctypes/test/test_prototypes.py rename to Lib/test/test_ctypes/test_prototypes.py index cd0c649de3e..d976e8da0e2 100644 --- a/Lib/ctypes/test/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -1,7 +1,3 @@ -from ctypes import * -from ctypes.test import need_symbol -import unittest - # IMPORTANT INFO: # # Consider this call: @@ -22,9 +18,18 @@ # # In this case, there would have to be an additional reference to the argument... -import _ctypes_test +import unittest +from ctypes import (CDLL, CFUNCTYPE, POINTER, ArgumentError, + pointer, byref, sizeof, addressof, create_string_buffer, + c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, + c_short, c_int, c_long, c_longlong, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + + testdll = CDLL(_ctypes_test.__file__) + # Return machine address `a` as a (possibly long) non-negative integer. # Starting with Python 2.5, id(anything) is always non-negative, and # the ctypes addressof() inherits that via PyLong_FromVoidPtr(). @@ -38,12 +43,13 @@ def positive_address(a): assert a >= 0 return a + def c_wbuffer(init): n = len(init) + 1 return (c_wchar * n)(*init) -class CharPointersTestCase(unittest.TestCase): +class CharPointersTestCase(unittest.TestCase): def setUp(self): func = testdll._testfunc_p_p func.restype = c_long @@ -66,6 +72,32 @@ def test_paramflags(self): self.assertEqual(func(None), None) self.assertEqual(func(input=None), None) + def test_invalid_paramflags(self): + proto = CFUNCTYPE(c_int, c_char_p) + with self.assertRaises(ValueError): + func = proto(("myprintf", testdll), ((1, "fmt"), (1, "arg1"))) + + def test_invalid_setattr_argtypes(self): + proto = CFUNCTYPE(c_int, c_char_p) + func = proto(("myprintf", testdll), ((1, "fmt"),)) + + with self.assertRaisesRegex(TypeError, "_argtypes_ must be a sequence of types"): + func.argtypes = 123 + self.assertEqual(func.argtypes, (c_char_p,)) + + with self.assertRaisesRegex(ValueError, "paramflags must have the same length as argtypes"): + func.argtypes = (c_char_p, c_int) + self.assertEqual(func.argtypes, (c_char_p,)) + + def test_paramflags_outarg(self): + proto = CFUNCTYPE(c_int, c_char_p, c_int) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + + proto = CFUNCTYPE(c_int, c_char_p, c_void_p) + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func.argtypes = (c_char_p, c_int) def test_int_pointer_arg(self): func = testdll._testfunc_p_p @@ -100,7 +132,7 @@ def test_POINTER_c_char_arg(self): self.assertEqual(None, func(c_char_p(None))) self.assertEqual(b"123", func(c_char_p(b"123"))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -115,7 +147,7 @@ def test_c_char_p_arg(self): self.assertEqual(None, func(c_char_p(None))) self.assertEqual(b"123", func(c_char_p(b"123"))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -130,7 +162,7 @@ def test_c_void_p_arg(self): self.assertEqual(b"123", func(c_char_p(b"123"))) self.assertEqual(None, func(c_char_p(None))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -139,7 +171,6 @@ def test_c_void_p_arg(self): func(pointer(c_int())) func((c_int * 3)()) - @need_symbol('c_wchar_p') def test_c_void_p_arg_with_c_wchar_p(self): func = testdll._testfunc_p_p func.restype = c_wchar_p @@ -161,9 +192,8 @@ class X: func.argtypes = None self.assertEqual(None, func(X())) -@need_symbol('c_wchar') -class WCharPointersTestCase(unittest.TestCase): +class WCharPointersTestCase(unittest.TestCase): def setUp(self): func = testdll._testfunc_p_p func.restype = c_int @@ -203,6 +233,7 @@ def test_c_wchar_p_arg(self): self.assertEqual("a", func(pointer(ca))[0]) self.assertEqual("a", func(byref(ca))[0]) + class ArrayTest(unittest.TestCase): def test(self): func = testdll._testfunc_ai8 @@ -216,7 +247,6 @@ def test(self): def func(): pass CFUNCTYPE(None, c_int * 3)(func) -################################################################ if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py similarity index 67% rename from Lib/ctypes/test/test_python_api.py rename to Lib/test/test_ctypes/test_python_api.py index 49571f97bbe..a1ee8a0de1e 100644 --- a/Lib/ctypes/test/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -1,18 +1,14 @@ -from ctypes import * +import _ctypes +import sys import unittest from test import support +from ctypes import (pythonapi, POINTER, create_string_buffer, sizeof, + py_object, c_char_p, c_char, c_long, c_size_t) -################################################################ -# This section should be moved into ctypes\__init__.py, when it's ready. - -from _ctypes import PyObj_FromPtr - -################################################################ - -from sys import getrefcount as grc class PythonAPITestCase(unittest.TestCase): - + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_PyBytes_FromStringAndSize(self): PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize @@ -27,46 +23,49 @@ def test_PyString_FromString(self): pythonapi.PyBytes_FromString.argtypes = (c_char_p,) s = b"abc" - refcnt = grc(s) + refcnt = sys.getrefcount(s) pyob = pythonapi.PyBytes_FromString(s) - self.assertEqual(grc(s), refcnt) + self.assertEqual(sys.getrefcount(s), refcnt) self.assertEqual(s, pyob) del pyob - self.assertEqual(grc(s), refcnt) + self.assertEqual(sys.getrefcount(s), refcnt) @support.refcount_test def test_PyLong_Long(self): - ref42 = grc(42) + ref42 = sys.getrefcount(42) pythonapi.PyLong_FromLong.restype = py_object self.assertEqual(pythonapi.PyLong_FromLong(42), 42) - self.assertEqual(grc(42), ref42) + self.assertEqual(sys.getrefcount(42), ref42) pythonapi.PyLong_AsLong.argtypes = (py_object,) pythonapi.PyLong_AsLong.restype = c_long res = pythonapi.PyLong_AsLong(42) - self.assertEqual(grc(res), ref42 + 1) + # Small int refcnts don't change + self.assertEqual(sys.getrefcount(res), ref42) del res - self.assertEqual(grc(42), ref42) + self.assertEqual(sys.getrefcount(42), ref42) @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" - ref = grc(s) + s = object() + ref = sys.getrefcount(s) # id(python-object) is the address - pyobj = PyObj_FromPtr(id(s)) + pyobj = _ctypes.PyObj_FromPtr(id(s)) self.assertIs(s, pyobj) - self.assertEqual(grc(s), ref + 1) + self.assertEqual(sys.getrefcount(s), ref + 1) del pyobj - self.assertEqual(grc(s), ref) + self.assertEqual(sys.getrefcount(s), ref) + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_PyOS_snprintf(self): PyOS_snprintf = pythonapi.PyOS_snprintf PyOS_snprintf.argtypes = POINTER(c_char), c_size_t, c_char_p - buf = c_buffer(256) + buf = create_string_buffer(256) PyOS_snprintf(buf, sizeof(buf), b"Hello from %s", b"ctypes") self.assertEqual(buf.value, b"Hello from ctypes") @@ -81,5 +80,6 @@ def test_pyobject_repr(self): self.assertEqual(repr(py_object(42)), "py_object(42)") self.assertEqual(repr(py_object(object)), "py_object(%r)" % object) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_random_things.py b/Lib/test/test_ctypes/test_random_things.py similarity index 74% rename from Lib/ctypes/test/test_random_things.py rename to Lib/test/test_ctypes/test_random_things.py index 2988e275cf4..73ff57d925e 100644 --- a/Lib/ctypes/test/test_random_things.py +++ b/Lib/test/test_ctypes/test_random_things.py @@ -1,30 +1,34 @@ -from ctypes import * +import _ctypes import contextlib -from test import support -import unittest +import ctypes import sys +import unittest +from test import support +from ctypes import CFUNCTYPE, c_void_p, c_char_p, c_int, c_double def callback_func(arg): 42 / arg raise ValueError(arg) + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class call_function_TestCase(unittest.TestCase): # _ctypes.call_function is deprecated and private, but used by # Gary Bishp's readline module. If we have it, we must test it as well. def test(self): - from _ctypes import call_function - windll.kernel32.LoadLibraryA.restype = c_void_p - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p + kernel32 = ctypes.windll.kernel32 + kernel32.LoadLibraryA.restype = c_void_p + kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + kernel32.GetProcAddress.restype = c_void_p + + hdll = kernel32.LoadLibraryA(b"kernel32") + funcaddr = kernel32.GetProcAddress(hdll, b"GetModuleHandleA") - hdll = windll.kernel32.LoadLibraryA(b"kernel32") - funcaddr = windll.kernel32.GetProcAddress(hdll, b"GetModuleHandleA") + self.assertEqual(_ctypes.call_function(funcaddr, (None,)), + kernel32.GetModuleHandleA(None)) - self.assertEqual(call_function(funcaddr, (None,)), - windll.kernel32.GetModuleHandleA(None)) class CallbackTracbackTestCase(unittest.TestCase): # When an exception is raised in a ctypes callback function, the C @@ -47,9 +51,9 @@ def expect_unraisable(self, exc_type, exc_msg=None): if exc_msg is not None: self.assertEqual(str(cm.unraisable.exc_value), exc_msg) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on calling ctypes " - "callback function") - self.assertIs(cm.unraisable.object, callback_func) + f"Exception ignored while calling ctypes " + f"callback function {callback_func!r}") + self.assertIsNone(cm.unraisable.object) def test_ValueError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py similarity index 54% rename from Lib/ctypes/test/test_refcounts.py rename to Lib/test/test_ctypes/test_refcounts.py index 48958cd2a60..1815649ceb5 100644 --- a/Lib/ctypes/test/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,60 +1,58 @@ -import unittest -from test import support import ctypes import gc +import sys +import unittest +from test import support +from test.support import import_helper, thread_unsafe +from test.support import script_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) OtherCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong) -import _ctypes_test dll = ctypes.CDLL(_ctypes_test.__file__) +@thread_unsafe('not thread safe') class RefcountTestCase(unittest.TestCase): - @support.refcount_test def test_1(self): - from sys import getrefcount as grc - f = dll._testfunc_callback_i_if f.restype = ctypes.c_int f.argtypes = [ctypes.c_int, MyCallback] def callback(value): - #print "called back with", value return value - self.assertEqual(grc(callback), 2) + orig_refcount = sys.getrefcount(callback) cb = MyCallback(callback) - self.assertGreater(grc(callback), 2) + self.assertGreater(sys.getrefcount(callback), orig_refcount) result = f(-10, cb) self.assertEqual(result, -18) cb = None gc.collect() - self.assertEqual(grc(callback), 2) - + self.assertEqual(sys.getrefcount(callback), orig_refcount) @support.refcount_test def test_refcount(self): - from sys import getrefcount as grc def func(*args): pass - # this is the standard refcount for func - self.assertEqual(grc(func), 2) + orig_refcount = sys.getrefcount(func) # the CFuncPtr instance holds at least one refcount on func: f = OtherCallback(func) - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # and may release it again del f - self.assertGreaterEqual(grc(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), orig_refcount) # but now it must be gone gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), orig_refcount) class X(ctypes.Structure): _fields_ = [("a", OtherCallback)] @@ -62,32 +60,31 @@ class X(ctypes.Structure): x.a = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # and may release it again del x - self.assertGreaterEqual(grc(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), orig_refcount) # and now it must be gone again gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), orig_refcount) f = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # create a cycle f.cycle = f del f gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), orig_refcount) +@thread_unsafe('not thread safe') class AnotherLeak(unittest.TestCase): def test_callback(self): - import sys - proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): return a * b * 2 @@ -112,5 +109,34 @@ def func(): for _ in range(10000): func() + +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + +class PyObjectRestypeTest(unittest.TestCase): + def test_restype_py_object_with_null_return(self): + # Test that a function which returns a NULL PyObject * + # without setting an exception does not crash. + PyErr_Occurred = ctypes.pythonapi.PyErr_Occurred + PyErr_Occurred.argtypes = [] + PyErr_Occurred.restype = ctypes.py_object + + # At this point, there's no exception set, so PyErr_Occurred + # returns NULL. Given the restype is py_object, the + # ctypes machinery will raise a custom error. + with self.assertRaisesRegex(ValueError, "PyObject is NULL"): + PyErr_Occurred() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_repr.py b/Lib/test/test_ctypes/test_repr.py similarity index 61% rename from Lib/ctypes/test/test_repr.py rename to Lib/test/test_ctypes/test_repr.py index 60a2c803453..8c85e6cbe70 100644 --- a/Lib/ctypes/test/test_repr.py +++ b/Lib/test/test_ctypes/test_repr.py @@ -1,5 +1,8 @@ -from ctypes import * import unittest +from ctypes import (c_byte, c_short, c_int, c_long, c_longlong, + c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong, + c_float, c_double, c_longdouble, c_bool, c_char) + subclasses = [] for base in [c_byte, c_short, c_int, c_long, c_longlong, @@ -9,21 +12,23 @@ class X(base): pass subclasses.append(X) + class X(c_char): pass -# This test checks if the __repr__ is correct for subclasses of simple types +# This test checks if the __repr__ is correct for subclasses of simple types class ReprTest(unittest.TestCase): def test_numbers(self): for typ in subclasses: base = typ.__bases__[0] - self.assertTrue(repr(base(42)).startswith(base.__name__)) - self.assertEqual(" 8\)", + ): + CField( + name="x", + type=c_byte, + byte_size=1, + byte_offset=0, + index=0, + _internal_use=True, + bit_size=7, + bit_offset=2, + ) + + # __set__ and __get__ should raise a TypeError in case their self + # argument is not a ctype instance. + def test___set__(self): + class MyCStruct(self.cls): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCStruct.field.__set__, 'wrong type self', 42) + + def test___get__(self): + class MyCStruct(self.cls): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCStruct.field.__get__, 'wrong type self', 42) + +class StructFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Structure + + def test_cfield_type_flags(self): + self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + + def test_cfield_inheritance_hierarchy(self): + self.assertEqual(CField.mro(), [CField, object]) + +class UnionFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Union + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_structunion.py b/Lib/test/test_ctypes/test_structunion.py new file mode 100644 index 00000000000..5b21d48d99c --- /dev/null +++ b/Lib/test/test_ctypes/test_structunion.py @@ -0,0 +1,477 @@ +"""Common tests for ctypes.Structure and ctypes.Union""" + +import unittest +import sys +from ctypes import (Structure, Union, POINTER, sizeof, alignment, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, + c_int8, c_int16, c_int32) +from ._support import (_CData, PyCStructType, UnionType, + Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) +from struct import calcsize +import contextlib +from test.support import MS_WINDOWS + + +class StructUnionTestBase: + formats = {"c": c_char, + "b": c_byte, + "B": c_ubyte, + "h": c_short, + "H": c_ushort, + "i": c_int, + "I": c_uint, + "l": c_long, + "L": c_ulong, + "q": c_longlong, + "Q": c_ulonglong, + "f": c_float, + "d": c_double, + } + + def test_subclass(self): + class X(self.cls): + _fields_ = [("a", c_int)] + + class Y(X): + _fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_subclass_delayed(self): + class X(self.cls): + pass + self.assertEqual(sizeof(X), 0) + X._fields_ = [("a", c_int)] + + class Y(X): + pass + self.assertEqual(sizeof(Y), sizeof(X)) + Y._fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_inheritance_hierarchy(self): + self.assertEqual(self.cls.mro(), [self.cls, _CData, object]) + self.assertEqual(type(self.metacls), type) + + def test_type_flags(self): + for cls in self.cls, self.metacls: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewClass = self.metacls.__new__(self.metacls, 'NewClass', + (self.cls,), {}) + for cls in self.cls, NewClass: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(self.cls): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + self.metacls.__init__(T, 'ptr', (), {}) + + def test_alignment(self): + class X(self.cls): + _fields_ = [("x", c_char * 3)] + self.assertEqual(alignment(X), calcsize("s")) + self.assertEqual(sizeof(X), calcsize("3s")) + + class Y(self.cls): + _fields_ = [("x", c_char * 3), + ("y", c_int)] + self.assertEqual(alignment(Y), alignment(c_int)) + self.check_sizeof(Y, + struct_size=calcsize("3s i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class SI(self.cls): + _fields_ = [("a", X), + ("b", Y)] + self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) + self.check_sizeof(SI, + struct_size=calcsize("3s0i 3si 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class IS(self.cls): + _fields_ = [("b", Y), + ("a", X)] + + self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) + self.check_sizeof(IS, + struct_size=calcsize("3si 3s 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + self.assertEqual(alignment(XX), alignment(X)) + self.check_sizeof(XX, + struct_size=calcsize("3s 3s 0s"), + union_size=calcsize("3s")) + + def test_empty(self): + # I had problems with these + # + # Although these are pathological cases: Empty Structures! + class X(self.cls): + _fields_ = [] + + # Is this really the correct alignment, or should it be 0? + self.assertTrue(alignment(X) == 1) + self.assertTrue(sizeof(X) == 0) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + + self.assertEqual(alignment(XX), 1) + self.assertEqual(sizeof(XX), 0) + + def test_fields(self): + # test the offset and size attributes of Structure/Union fields. + class X(self.cls): + _fields_ = [("x", c_int), + ("y", c_char)] + + self.assertEqual(X.x.offset, 0) + self.assertEqual(X.x.size, sizeof(c_int)) + + if self.cls == Structure: + self.assertEqual(X.y.offset, sizeof(c_int)) + else: + self.assertEqual(X.y.offset, 0) + self.assertEqual(X.y.size, sizeof(c_char)) + + # readonly + self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) + self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) + + # XXX Should we check nested data types also? + # offset is always relative to the class... + + def test_field_descriptor_attributes(self): + """Test information provided by the descriptors""" + class Inner(Structure): + _fields_ = [ + ("a", c_int16), + ("b", c_int8, 1), + ("c", c_int8, 2), + ] + class X(self.cls): + _fields_ = [ + ("x", c_int32), + ("y", c_int16, 1), + ("_", Inner), + ] + _anonymous_ = ["_"] + + field_names = "xy_abc" + + # name + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).name, name) + + # type + + expected_types = dict( + x=c_int32, + y=c_int16, + _=Inner, + a=c_int16, + b=c_int8, + c=c_int8, + ) + assert set(expected_types) == set(field_names) + for name, tp in expected_types.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).type, tp) + self.assertEqual(getattr(X, name).byte_size, sizeof(tp)) + + # offset, byte_offset + + expected_offsets = dict( + x=(0, 0), + y=(0, 4), + _=(0, 6), + a=(0, 6), + b=(2, 8), + c=(2, 8), + ) + assert set(expected_offsets) == set(field_names) + for name, (union_offset, struct_offset) in expected_offsets.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).offset, + getattr(X, name).byte_offset) + if self.cls == Structure: + self.assertEqual(getattr(X, name).offset, struct_offset) + else: + self.assertEqual(getattr(X, name).offset, union_offset) + + # is_bitfield, bit_size, bit_offset + # size + + little_endian = (sys.byteorder == 'little') + expected_bitfield_info = dict( + # (bit_size, bit_offset) + b=(1, 0 if little_endian else 7), + c=(2, 1 if little_endian else 5), + y=(1, 0 if little_endian else 15), + ) + for name in field_names: + with self.subTest(name=name): + if info := expected_bitfield_info.get(name): + self.assertEqual(getattr(X, name).is_bitfield, True) + expected_bit_size, expected_bit_offset = info + self.assertEqual(getattr(X, name).bit_size, + expected_bit_size) + self.assertEqual(getattr(X, name).bit_offset, + expected_bit_offset) + self.assertEqual(getattr(X, name).size, + (expected_bit_size << 16) + | expected_bit_offset) + else: + self.assertEqual(getattr(X, name).is_bitfield, False) + type_size = sizeof(expected_types[name]) + self.assertEqual(getattr(X, name).bit_size, type_size * 8) + self.assertEqual(getattr(X, name).bit_offset, 0) + self.assertEqual(getattr(X, name).size, type_size) + + # is_anonymous + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).is_anonymous, (name == '_')) + + + def test_invalid_field_types(self): + class POINT(self.cls): + pass + self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) + + def test_invalid_name(self): + # field name must be string + for name in b"x", 3, None: + with self.subTest(name=name): + with self.assertRaises(TypeError): + class S(self.cls): + _fields_ = [(name, c_int)] + + def test_str_name(self): + class WeirdString(str): + def __str__(self): + return "unwanted value" + class S(self.cls): + _fields_ = [(WeirdString("f"), c_int)] + self.assertEqual(S.f.name, "f") + + def test_intarray_fields(self): + class SomeInts(self.cls): + _fields_ = [("a", c_int * 4)] + + # can use tuple to initialize array (but not list!) + self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) + self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) + self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) + self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) + # too long + # XXX Should raise ValueError?, not RuntimeError + self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) + + def test_huge_field_name(self): + # issue12881: segfault with large structure field names + def create_class(length): + class S(self.cls): + _fields_ = [('x' * length, c_int)] + + for length in [10 ** i for i in range(0, 8)]: + try: + create_class(length) + except MemoryError: + # MemoryErrors are OK, we just don't want to segfault + pass + + def test_abstract_class(self): + class X(self.cls): + _abstract_ = "something" + with self.assertRaisesRegex(TypeError, r"^abstract class$"): + X() + + def test_methods(self): + self.assertIn("in_dll", dir(type(self.cls))) + self.assertIn("from_address", dir(type(self.cls))) + self.assertIn("in_dll", dir(type(self.cls))) + + def test_pack_layout_switch(self): + # Setting _pack_ implicitly sets default layout to MSVC; + # this is deprecated on non-Windows platforms. + if MS_WINDOWS: + warn_context = contextlib.nullcontext() + else: + warn_context = self.assertWarns(DeprecationWarning) + with warn_context: + class X(self.cls): + _pack_ = 1 + # _layout_ missing + _fields_ = [('a', c_int8, 1), ('b', c_int16, 2)] + + # Check MSVC layout (bitfields of different types aren't combined) + self.check_sizeof(X, struct_size=3, union_size=2) + + +class StructureTestCase(unittest.TestCase, StructUnionTestBase): + cls = Structure + metacls = PyCStructType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "PyCStructType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), struct_size) + + def test_simple_structs(self): + for code, tp in self.formats.items(): + class X(Structure): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("c%c0%c" % (code, code)), code)) + + +class UnionTestCase(unittest.TestCase, StructUnionTestBase): + cls = Union + metacls = UnionType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "UnionType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), union_size) + + def test_simple_unions(self): + for code, tp in self.formats.items(): + class X(Union): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("%c" % (code)), code)) + + +class PointerMemberTestBase: + def test(self): + # a Structure/Union with a POINTER field + class S(self.cls): + _fields_ = [("array", POINTER(c_int))] + + s = S() + # We can assign arrays of the correct type + s.array = (c_int * 3)(1, 2, 3) + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + + s.array[0] = 42 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [42, 2, 3]) + + s.array[0] = 1 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + +class PointerMemberTestCase_Struct(unittest.TestCase, PointerMemberTestBase): + cls = Structure + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertEqual(s.x, 12345678) + +class PointerMemberTestCase_Union(unittest.TestCase, PointerMemberTestBase): + cls = Union + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertFalse(s.p) # NULL pointers are falsy + + +class TestRecursiveBase: + def test_contains_itself(self): + class Recursive(self.cls): + pass + + try: + Recursive._fields_ = [("next", Recursive)] + except AttributeError as details: + self.assertIn("Structure or union cannot contain itself", + str(details)) + else: + self.fail("Structure or union cannot contain itself") + + + def test_vice_versa(self): + class First(self.cls): + pass + class Second(self.cls): + pass + + First._fields_ = [("second", Second)] + + try: + Second._fields_ = [("first", First)] + except AttributeError as details: + self.assertIn("_fields_ is final", str(details)) + else: + self.fail("AttributeError not raised") + +class TestRecursiveStructure(unittest.TestCase, TestRecursiveBase): + cls = Structure + +class TestRecursiveUnion(unittest.TestCase, TestRecursiveBase): + cls = Union diff --git a/Lib/ctypes/test/test_structures.py b/Lib/test/test_ctypes/test_structures.py similarity index 58% rename from Lib/ctypes/test/test_structures.py rename to Lib/test/test_ctypes/test_structures.py index f95d5a99a3a..92d4851d739 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,178 +1,32 @@ -import platform +"""Tests for ctypes.Structure + +Features common with Union should go in test_structunion.py instead. +""" + +from platform import architecture as _architecture +import struct import sys import unittest -from ctypes import * -from ctypes.test import need_symbol -from struct import calcsize -import _ctypes_test +from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, + c_void_p, c_char, c_wchar, c_byte, c_ubyte, + c_uint8, c_uint16, c_uint32, c_int, c_uint, + c_long, c_ulong, c_longlong, c_float, c_double) +from ctypes.util import find_library +from collections import namedtuple from test import support +from test.support import import_helper +from ._support import StructCheckMixin +_ctypes_test = import_helper.import_module("_ctypes_test") -# The following definition is meant to be used from time to time to assist -# temporarily disabling tests on specific architectures while investigations -# are in progress, to keep buildbots happy. -MACHINE = platform.machine() - -class SubclassesTest(unittest.TestCase): - def test_subclass(self): - class X(Structure): - _fields_ = [("a", c_int)] - - class Y(X): - _fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - - def test_subclass_delayed(self): - class X(Structure): - pass - self.assertEqual(sizeof(X), 0) - X._fields_ = [("a", c_int)] - - class Y(X): - pass - self.assertEqual(sizeof(Y), sizeof(X)) - Y._fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - -class StructureTestCase(unittest.TestCase): - formats = {"c": c_char, - "b": c_byte, - "B": c_ubyte, - "h": c_short, - "H": c_ushort, - "i": c_int, - "I": c_uint, - "l": c_long, - "L": c_ulong, - "q": c_longlong, - "Q": c_ulonglong, - "f": c_float, - "d": c_double, - } - - def test_simple_structs(self): - for code, tp in self.formats.items(): - class X(Structure): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("c%c0%c" % (code, code)), code)) - - def test_unions(self): - for code, tp in self.formats.items(): - class X(Union): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("%c" % (code)), code)) - - def test_struct_alignment(self): - class X(Structure): - _fields_ = [("x", c_char * 3)] - self.assertEqual(alignment(X), calcsize("s")) - self.assertEqual(sizeof(X), calcsize("3s")) - - class Y(Structure): - _fields_ = [("x", c_char * 3), - ("y", c_int)] - self.assertEqual(alignment(Y), alignment(c_int)) - self.assertEqual(sizeof(Y), calcsize("3si")) - - class SI(Structure): - _fields_ = [("a", X), - ("b", Y)] - self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) - self.assertEqual(sizeof(SI), calcsize("3s0i 3si 0i")) - - class IS(Structure): - _fields_ = [("b", Y), - ("a", X)] - - self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) - self.assertEqual(sizeof(IS), calcsize("3si 3s 0i")) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - self.assertEqual(alignment(XX), alignment(X)) - self.assertEqual(sizeof(XX), calcsize("3s 3s 0s")) - - def test_empty(self): - # I had problems with these - # - # Although these are pathological cases: Empty Structures! - class X(Structure): - _fields_ = [] - - class Y(Union): - _fields_ = [] - - # Is this really the correct alignment, or should it be 0? - self.assertTrue(alignment(X) == alignment(Y) == 1) - self.assertTrue(sizeof(X) == sizeof(Y) == 0) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - - self.assertEqual(alignment(XX), 1) - self.assertEqual(sizeof(XX), 0) - - def test_fields(self): - # test the offset and size attributes of Structure/Union fields. - class X(Structure): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, sizeof(c_int)) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - class X(Union): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, 0) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - # XXX Should we check nested data types also? - # offset is always relative to the class... +class StructureTestCase(unittest.TestCase, StructCheckMixin): def test_packed(self): class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 1 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), 9) self.assertEqual(X.b.offset, 1) @@ -181,10 +35,11 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 2 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), 10) self.assertEqual(X.b.offset, 2) - import struct longlong_size = struct.calcsize("q") longlong_align = struct.calcsize("bq") - longlong_size @@ -192,6 +47,8 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 4 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), min(4, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(4, longlong_align)) @@ -199,26 +56,33 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 8 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), min(8, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(8, longlong_align)) - - d = {"_fields_": [("a", "b"), - ("b", "q")], - "_pack_": -1} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", "b"), ("b", "q")] + _pack_ = -1 + _layout_ = "ms" @support.cpython_only def test_packed_c_limits(self): # Issue 15989 import _testcapi - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.INT_MAX + 1} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.UINT_MAX + 2} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", c_byte)] + _pack_ = _testcapi.INT_MAX + 1 + _layout_ = "ms" + + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", c_byte)] + _pack_ = _testcapi.UINT_MAX + 2 + _layout_ = "ms" def test_initializers(self): class Person(Structure): @@ -239,6 +103,7 @@ class Person(Structure): def test_conflicting_initializers(self): class POINT(Structure): _fields_ = [("phi", c_float), ("rho", c_float)] + self.check_struct(POINT) # conflicting positional and keyword args self.assertRaisesRegex(TypeError, "phi", POINT, 2, 3, phi=4) self.assertRaisesRegex(TypeError, "rho", POINT, 2, 3, rho=4) @@ -249,52 +114,25 @@ class POINT(Structure): def test_keyword_initializers(self): class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(POINT) pt = POINT(1, 2) self.assertEqual((pt.x, pt.y), (1, 2)) pt = POINT(y=2, x=1) self.assertEqual((pt.x, pt.y), (1, 2)) - def test_invalid_field_types(self): - class POINT(Structure): - pass - self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) - - def test_invalid_name(self): - # field name must be string - def declare_with_name(name): - class S(Structure): - _fields_ = [(name, c_int)] - - self.assertRaises(TypeError, declare_with_name, b"x") - - def test_intarray_fields(self): - class SomeInts(Structure): - _fields_ = [("a", c_int * 4)] - - # can use tuple to initialize array (but not list!) - self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) - self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) - self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) - self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) - # too long - # XXX Should raise ValueError?, not RuntimeError - self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) - def test_nested_initializers(self): # test initializing nested structures class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) p = Person(b"Someone", (b"1234", b"5678"), 5) @@ -303,11 +141,11 @@ class Person(Structure): self.assertEqual(p.phone.number, b"5678") self.assertEqual(p.age, 5) - @need_symbol('c_wchar') def test_structures_with_wchar(self): class PersonW(Structure): _fields_ = [("name", c_wchar * 12), ("age", c_int)] + self.check_struct(PersonW) p = PersonW("Someone \xe9") self.assertEqual(p.name, "Someone \xe9") @@ -323,11 +161,13 @@ def test_init_errors(self): class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) cls, msg = self.get_except(Person, b"Someone", (1, 2)) self.assertEqual(cls, RuntimeError) @@ -340,59 +180,29 @@ class Person(Structure): self.assertEqual(msg, "(Phone) TypeError: too many initializers") - def test_huge_field_name(self): - # issue12881: segfault with large structure field names - def create_class(length): - class S(Structure): - _fields_ = [('x' * length, c_int)] - - for length in [10 ** i for i in range(0, 8)]: - try: - create_class(length) - except MemoryError: - # MemoryErrors are OK, we just don't want to segfault - pass - def get_except(self, func, *args): try: func(*args) except Exception as detail: return detail.__class__, str(detail) - @unittest.skip('test disabled') - def test_subclass_creation(self): - meta = type(Structure) - # same as 'class X(Structure): pass' - # fails, since we need either a _fields_ or a _abstract_ attribute - cls, msg = self.get_except(meta, "X", (Structure,), {}) - self.assertEqual((cls, msg), - (AttributeError, "class must define a '_fields_' attribute")) - - def test_abstract_class(self): - class X(Structure): - _abstract_ = "something" - # try 'X()' - cls, msg = self.get_except(eval, "X()", locals()) - self.assertEqual((cls, msg), (TypeError, "abstract class")) - - def test_methods(self): -## class X(Structure): -## _fields_ = [] - - self.assertIn("in_dll", dir(type(Structure))) - self.assertIn("from_address", dir(type(Structure))) - self.assertIn("in_dll", dir(type(Structure))) - def test_positional_args(self): # see also http://bugs.python.org/issue5042 class W(Structure): _fields_ = [("a", c_int), ("b", c_int)] + self.check_struct(W) + class X(W): _fields_ = [("c", c_int)] + self.check_struct(X) + class Y(X): pass + self.check_struct(Y) + class Z(Y): _fields_ = [("d", c_int), ("e", c_int), ("f", c_int)] + self.check_struct(Z) z = Z(1, 2, 3, 4, 5, 6) self.assertEqual((z.a, z.b, z.c, z.d, z.e, z.f), @@ -411,6 +221,7 @@ class Test(Structure): ('second', c_ulong), ('third', c_ulong), ] + self.check_struct(Test) s = Test() s.first = 0xdeadbeef @@ -440,6 +251,7 @@ class Test(Structure): ] def __del__(self): finalizer_calls.append("called") + self.check_struct(Test) s = Test(1, 2, 3) # Test the StructUnionType_paramfunc() code path which copies the @@ -469,6 +281,7 @@ class X(Structure): ('first', c_uint), ('second', c_uint) ] + self.check_struct(X) s = X() s.first = 0xdeadbeef @@ -480,41 +293,131 @@ class X(Structure): func(s) self.assertEqual(s.first, 0xdeadbeef) self.assertEqual(s.second, 0xcafebabe) - got = X.in_dll(dll, "last_tfrsuv_arg") + dll.get_last_tfrsuv_arg.argtypes = () + dll.get_last_tfrsuv_arg.restype = X + got = dll.get_last_tfrsuv_arg() self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + def _test_issue18060(self, Vector): + # Regression tests for gh-62260 + + # The call to atan2() should succeed if the + # class fields were correctly cloned in the + # subclasses. Otherwise, it will segfault. + if sys.platform == 'win32': + libm = CDLL(find_library('msvcrt.dll')) + else: + libm = CDLL(find_library('m')) + + libm.atan2.argtypes = [Vector] + libm.atan2.restype = c_double + + arg = Vector(y=0.0, x=-1.0) + self.assertAlmostEqual(libm.atan2(arg), 3.141592653589793) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_a(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment, and PyCStgInfo_clone() + # for the Mid and Vector class definitions. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + pass + Mid._fields_ = [] + class Vector(Mid): pass + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_b(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [] + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_c(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [('x', c_double)] + self._test_issue18060(Vector) + def test_array_in_struct(self): # See bpo-22273 + # Load the shared library + dll = CDLL(_ctypes_test.__file__) + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] + self.check_struct(Test2) + + class Test3AParent(Structure): + _fields_ = [ + ('data', c_float * 2), + ] + self.check_struct(Test3AParent) + + class Test3A(Test3AParent): + _fields_ = [ + ('more_data', c_float * 2), + ] + self.check_struct(Test3A) - class Test3(Structure): + class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] + self.check_struct(Test3B) - class Test3A(Structure): + class Test3C(Structure): _fields_ = [ - ('data', c_float * 2), + ("data", c_double * 4) ] + self.check_struct(Test3C) - class Test3B(Test3A): + class Test3D(Structure): _fields_ = [ - ('more_data', c_float * 2), + ("data", c_double * 8) + ] + self.check_struct(Test3D) + + class Test3E(Structure): + _fields_ = [ + ("data", c_double * 9) ] + self.check_struct(Test3E) + + # Tests for struct Test2 s = Test2() expected = 0 for i in range(16): s.data[i] = i expected += i - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_array_in_struct1 + func = dll._testfunc_array_in_struct2 func.restype = c_int func.argtypes = (Test2,) result = func(s) @@ -523,29 +426,16 @@ class Test3B(Test3A): for i in range(16): self.assertEqual(s.data[i], i) - s = Test3() - s.data[0] = 3.14159 - s.data[1] = 2.71828 - expected = 3.14159 + 2.71828 - func = dll._testfunc_array_in_struct2 - func.restype = c_double - func.argtypes = (Test3,) - result = func(s) - self.assertEqual(result, expected) - # check the passed-in struct hasn't changed - self.assertEqual(s.data[0], 3.14159) - self.assertEqual(s.data[1], 2.71828) - - s = Test3B() + # Tests for struct Test3A + s = Test3A() s.data[0] = 3.14159 s.data[1] = 2.71828 s.more_data[0] = -3.0 s.more_data[1] = -2.0 - - expected = 3.14159 + 2.71828 - 5.0 - func = dll._testfunc_array_in_struct2a + expected = 3.14159 + 2.71828 - 3.0 - 2.0 + func = dll._testfunc_array_in_struct3A func.restype = c_double - func.argtypes = (Test3B,) + func.argtypes = (Test3A,) result = func(s) self.assertAlmostEqual(result, expected, places=6) # check the passed-in struct hasn't changed @@ -554,13 +444,71 @@ class Test3B(Test3A): self.assertAlmostEqual(s.more_data[0], -3.0, places=6) self.assertAlmostEqual(s.more_data[1], -2.0, places=6) + # Test3B, Test3C, Test3D, Test3E have the same logic with different + # sizes hence putting them in a loop. + StructCtype = namedtuple( + "StructCtype", + ["cls", "cfunc1", "cfunc2", "items"] + ) + structs_to_test = [ + StructCtype( + Test3B, + dll._testfunc_array_in_struct3B, + dll._testfunc_array_in_struct3B_set_defaults, + 2), + StructCtype( + Test3C, + dll._testfunc_array_in_struct3C, + dll._testfunc_array_in_struct3C_set_defaults, + 4), + StructCtype( + Test3D, + dll._testfunc_array_in_struct3D, + dll._testfunc_array_in_struct3D_set_defaults, + 8), + StructCtype( + Test3E, + dll._testfunc_array_in_struct3E, + dll._testfunc_array_in_struct3E_set_defaults, + 9), + ] + + for sut in structs_to_test: + s = sut.cls() + + # Test for cfunc1 + expected = 0 + for i in range(sut.items): + float_i = float(i) + s.data[i] = float_i + expected += float_i + func = sut.cfunc1 + func.restype = c_double + func.argtypes = (sut.cls,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + for i in range(sut.items): + self.assertEqual(s.data[i], float(i)) + + # Test for cfunc2 + func = sut.cfunc2 + func.restype = sut.cls + result = func() + # check if the default values have been set correctly + for i in range(sut.items): + self.assertEqual(result.data[i], float(i+1)) + def test_38368(self): + # Regression test for gh-82549 class U(Union): _fields_ = [ ('f1', c_uint8 * 16), ('f2', c_uint16 * 8), ('f3', c_uint32 * 4), ] + self.check_union(U) + u = U() u.f3[0] = 0x01234567 u.f3[1] = 0x89ABCDEF @@ -576,9 +524,9 @@ class U(Union): self.assertEqual(f2, [0x4567, 0x0123, 0xcdef, 0x89ab, 0x3210, 0x7654, 0xba98, 0xfedc]) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_union_by_value(self): - # See bpo-16575 + # See gh-60779 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -587,18 +535,21 @@ class Nested1(Structure): ('an_int', c_int), ('another_int', c_int), ] + self.check_struct(Nested1) class Test4(Union): _fields_ = [ ('a_long', c_long), ('a_struct', Nested1), ] + self.check_struct(Test4) class Nested2(Structure): _fields_ = [ ('an_int', c_int), ('a_union', Test4), ] + self.check_struct(Nested2) class Test5(Structure): _fields_ = [ @@ -606,6 +557,7 @@ class Test5(Structure): ('nested', Nested2), ('another_int', c_int), ] + self.check_struct(Test5) test4 = Test4() dll = CDLL(_ctypes_test.__file__) @@ -657,9 +609,9 @@ class Test5(Structure): self.assertEqual(test5.nested.an_int, 0) self.assertEqual(test5.another_int, 0) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_bitfield_by_value(self): - # See bpo-16576 + # See gh-60780 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -670,6 +622,7 @@ class Test6(Structure): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_struct(Test6) test6 = Test6() # As these are signed int fields, all are logically -1 due to sign @@ -705,6 +658,8 @@ class Test7(Structure): ('C', c_uint, 3), ('D', c_uint, 2), ] + self.check_struct(Test7) + test7 = Test7() test7.A = 1 test7.B = 3 @@ -728,6 +683,7 @@ class Test8(Union): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_union(Test8) test8 = Test8() with self.assertRaises(TypeError) as ctx: @@ -738,75 +694,30 @@ class Test8(Union): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') -class PointerMemberTestCase(unittest.TestCase): - - def test(self): - # a Structure with a POINTER field - class S(Structure): - _fields_ = [("array", POINTER(c_int))] - - s = S() - # We can assign arrays of the correct type - s.array = (c_int * 3)(1, 2, 3) - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - # The following are bugs, but are included here because the unittests - # also describe the current behaviour. - # - # This fails with SystemError: bad arg to internal function - # or with IndexError (with a patch I have) - - s.array[0] = 42 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [42, 2, 3]) - - s.array[0] = 1 - -## s.array[1] = 42 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - def test_none_to_pointer_fields(self): - class S(Structure): - _fields_ = [("x", c_int), - ("p", POINTER(c_int))] - - s = S() - s.x = 12345678 - s.p = None - self.assertEqual(s.x, 12345678) - -class TestRecursiveStructure(unittest.TestCase): - def test_contains_itself(self): - class Recursive(Structure): + def test_do_not_share_pointer_type_cache_via_stginfo_clone(self): + # This test case calls PyCStgInfo_clone() + # for the Mid and Vector class definitions + # and checks that pointer_type cache not shared + # between subclasses. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + base_ptr = POINTER(Base) + + class Mid(Base): pass + Mid._fields_ = [] + mid_ptr = POINTER(Mid) - try: - Recursive._fields_ = [("next", Recursive)] - except AttributeError as details: - self.assertIn("Structure or union cannot contain itself", - str(details)) - else: - self.fail("Structure or union cannot contain itself") - - - def test_vice_versa(self): - class First(Structure): - pass - class Second(Structure): + class Vector(Mid): pass - First._fields_ = [("second", Second)] + vector_ptr = POINTER(Vector) + + self.assertIsNot(base_ptr, mid_ptr) + self.assertIsNot(base_ptr, vector_ptr) + self.assertIsNot(mid_ptr, vector_ptr) - try: - Second._fields_ = [("first", First)] - except AttributeError as details: - self.assertIn("_fields_ is final", str(details)) - else: - self.fail("AttributeError not raised") if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_unaligned_structures.py b/Lib/test/test_ctypes/test_unaligned_structures.py similarity index 79% rename from Lib/ctypes/test/test_unaligned_structures.py rename to Lib/test/test_ctypes/test_unaligned_structures.py index ee7fb45809b..b5fb4c0df77 100644 --- a/Lib/ctypes/test/test_unaligned_structures.py +++ b/Lib/test/test_ctypes/test_unaligned_structures.py @@ -1,5 +1,9 @@ import sys, unittest -from ctypes import * +from ctypes import (Structure, BigEndianStructure, LittleEndianStructure, + c_byte, c_short, c_int, c_long, c_longlong, + c_float, c_double, + c_ushort, c_uint, c_ulong, c_ulonglong) + structures = [] byteswapped_structures = [] @@ -15,15 +19,18 @@ c_ushort, c_uint, c_ulong, c_ulonglong]: class X(Structure): _pack_ = 1 + _layout_ = 'ms' _fields_ = [("pad", c_byte), ("value", typ)] class Y(SwappedStructure): _pack_ = 1 + _layout_ = 'ms' _fields_ = [("pad", c_byte), ("value", typ)] structures.append(X) byteswapped_structures.append(Y) + class TestStructures(unittest.TestCase): def test_native(self): for typ in structures: @@ -39,5 +46,6 @@ def test_swapped(self): o.value = 4 self.assertEqual(o.value, 4) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_unicode.py b/Lib/test/test_ctypes/test_unicode.py similarity index 95% rename from Lib/ctypes/test/test_unicode.py rename to Lib/test/test_ctypes/test_unicode.py index 60c75424b76..d9e17371d13 100644 --- a/Lib/ctypes/test/test_unicode.py +++ b/Lib/test/test_ctypes/test_unicode.py @@ -1,10 +1,9 @@ -import unittest import ctypes -from ctypes.test import need_symbol +import unittest +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -import _ctypes_test -@need_symbol('c_wchar') class UnicodeTestCase(unittest.TestCase): def test_wcslen(self): dll = ctypes.CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py new file mode 100644 index 00000000000..e2dff0f22a9 --- /dev/null +++ b/Lib/test/test_ctypes/test_unions.py @@ -0,0 +1,35 @@ +import unittest +from ctypes import Union, c_char +from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) + + +class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Union.mro(), [Union, _CData, object]) + + self.assertEqual(UnionType.__name__, "UnionType") + self.assertEqual(type(UnionType), type) + + def test_type_flags(self): + for cls in Union, UnionType: + with self.subTest(cls=Union): + self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewUnion = UnionType.__new__(UnionType, 'NewUnion', + (Union,), {}) + for cls in Union, NewUnion: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Union): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + UnionType.__init__(T, 'ptr', (), {}) diff --git a/Lib/ctypes/test/test_values.py b/Lib/test/test_ctypes/test_values.py similarity index 84% rename from Lib/ctypes/test/test_values.py rename to Lib/test/test_ctypes/test_values.py index 435fdd22ea2..18554e193be 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -4,19 +4,24 @@ import _imp import importlib.util -import unittest import sys -from ctypes import * -from test.support import import_helper +import unittest +from ctypes import (Structure, CDLL, POINTER, pythonapi, + c_ubyte, c_char_p, c_int) +from test.support import import_helper, thread_unsafe -import _ctypes_test class ValuesTestCase(unittest.TestCase): + def setUp(self): + _ctypes_test = import_helper.import_module("_ctypes_test") + self.ctdll = CDLL(_ctypes_test.__file__) + + @thread_unsafe("static global variables aren't thread-safe") def test_an_integer(self): # This test checks and changes an integer stored inside the # _ctypes_test dll/shared lib. - ctdll = CDLL(_ctypes_test.__file__) + ctdll = self.ctdll an_integer = c_int.in_dll(ctdll, "an_integer") x = an_integer.value self.assertEqual(x, ctdll.get_an_integer()) @@ -28,12 +33,14 @@ def test_an_integer(self): self.assertEqual(x, ctdll.get_an_integer()) def test_undefined(self): - ctdll = CDLL(_ctypes_test.__file__) - self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + self.assertRaises(ValueError, c_int.in_dll, self.ctdll, "Undefined_Symbol") + class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_optimizeflag(self): # This test accesses the Py_OptimizeFlag integer, which is # exported by the Python dll and should match the sys.flags value @@ -41,6 +48,9 @@ def test_optimizeflag(self): opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value self.assertEqual(opt, sys.flags.optimize) + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure + @thread_unsafe('overrides frozen modules') def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a # pointer to an array of struct _frozen entries. The end of the @@ -55,7 +65,6 @@ class struct_frozen(Structure): ("code", POINTER(c_ubyte)), ("size", c_int), ("is_package", c_int), - ("get_code", POINTER(c_ubyte)), # Function ptr ] FrozenTable = POINTER(struct_frozen) @@ -92,12 +101,10 @@ class struct_frozen(Structure): "_PyImport_FrozenBootstrap example " "in Doc/library/ctypes.rst may be out of date") - from ctypes import _pointer_type_cache - del _pointer_type_cache[struct_frozen] - def test_undefined(self): self.assertRaises(ValueError, c_int.in_dll, pythonapi, "Undefined_Symbol") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_varsize_struct.py b/Lib/test/test_ctypes/test_varsize_struct.py similarity index 97% rename from Lib/ctypes/test/test_varsize_struct.py rename to Lib/test/test_ctypes/test_varsize_struct.py index f409500f013..3e6ba6fed07 100644 --- a/Lib/ctypes/test/test_varsize_struct.py +++ b/Lib/test/test_ctypes/test_varsize_struct.py @@ -1,5 +1,6 @@ -from ctypes import * import unittest +from ctypes import Structure, sizeof, resize, c_int + class VarSizeTest(unittest.TestCase): def test_resize(self): @@ -46,5 +47,6 @@ def test_zerosized_array(self): self.assertRaises(IndexError, array.__setitem__, -1, None) self.assertRaises(IndexError, array.__getitem__, -1) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_win32.py b/Lib/test/test_ctypes/test_win32.py similarity index 72% rename from Lib/ctypes/test/test_win32.py rename to Lib/test/test_ctypes/test_win32.py index e51bdc8ad6b..bb2fc0ca222 100644 --- a/Lib/ctypes/test/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -1,33 +1,43 @@ # Windows specific tests -from ctypes import * -import unittest, sys +import ctypes +import errno +import sys +import unittest +from ctypes import (CDLL, Structure, POINTER, pointer, sizeof, byref, + c_void_p, c_char, c_int, c_long) from test import support +from test.support import import_helper +from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE -import _ctypes_test @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): + # TODO: RUSTPYTHON: SEH not implemented, crashes with STATUS_ACCESS_VIOLATION + @unittest.skip("TODO: RUSTPYTHON") @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") def test_SEH(self): # Disable faulthandler to prevent logging the warning: # "Windows fatal exception: access violation" + kernel32 = ctypes.windll.kernel32 with support.disable_faulthandler(): # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. - self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) + self.assertRaises(OSError, kernel32.GetModuleHandleA, 32) def test_noargs(self): # This is a special case on win32 x64 - windll.user32.GetDesktopWindow() + user32 = ctypes.windll.user32 + user32.GetDesktopWindow() @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class ReturnStructSizesTestCase(unittest.TestCase): def test_sizes(self): + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) for i in range(1, 11): fields = [ (f"f{f}", c_char) for f in range(1, i + 1)] @@ -42,7 +52,6 @@ class S(Structure): self.assertEqual(value, expected) - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWintypes(unittest.TestCase): def test_HWND(self): @@ -57,39 +66,47 @@ def test_PARAM(self): sizeof(c_void_p)) def test_COMError(self): - from _ctypes import COMError + from ctypes import COMError if support.HAVE_DOCSTRINGS: self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") - ex = COMError(-1, "text", ("details",)) + ex = COMError(-1, "text", ("descr", "source", "helpfile", 0, "progid")) self.assertEqual(ex.hresult, -1) self.assertEqual(ex.text, "text") - self.assertEqual(ex.details, ("details",)) + self.assertEqual(ex.details, + ("descr", "source", "helpfile", 0, "progid")) + + self.assertEqual(COMError.mro(), + [COMError, Exception, BaseException, object]) + self.assertFalse(COMError.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + self.assertTrue(COMError.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWinError(unittest.TestCase): def test_winerror(self): # see Issue 16169 - import errno ERROR_INVALID_PARAMETER = 87 - msg = FormatError(ERROR_INVALID_PARAMETER).strip() + msg = ctypes.FormatError(ERROR_INVALID_PARAMETER).strip() args = (errno.EINVAL, msg, None, ERROR_INVALID_PARAMETER) - e = WinError(ERROR_INVALID_PARAMETER) + e = ctypes.WinError(ERROR_INVALID_PARAMETER) self.assertEqual(e.args, args) self.assertEqual(e.errno, errno.EINVAL) self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER) - windll.kernel32.SetLastError(ERROR_INVALID_PARAMETER) + kernel32 = ctypes.windll.kernel32 + kernel32.SetLastError(ERROR_INVALID_PARAMETER) try: - raise WinError() + raise ctypes.WinError() except OSError as exc: e = exc self.assertEqual(e.args, args) self.assertEqual(e.errno, errno.EINVAL) self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER) + class Structures(unittest.TestCase): def test_struct_by_value(self): class POINT(Structure): @@ -102,6 +119,7 @@ class RECT(Structure): ("right", c_long), ("bottom", c_long)] + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) pt = POINT(15, 25) @@ -128,9 +146,9 @@ class RECT(Structure): self.assertEqual(ret.top, top.value) self.assertEqual(ret.bottom, bottom.value) - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[RECT] + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[2]) + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[5]) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py new file mode 100644 index 00000000000..01db602149b --- /dev/null +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -0,0 +1,286 @@ +import ctypes +import gc +import sys +import unittest +from ctypes import POINTER, byref, c_void_p +from ctypes.wintypes import BYTE, DWORD, WORD + +if sys.platform != "win32": + raise unittest.SkipTest("Windows-specific test") + + +from ctypes import COMError, CopyComPointer, HRESULT + + +COINIT_APARTMENTTHREADED = 0x2 +CLSCTX_SERVER = 5 +S_OK = 0 +OUT = 2 +TRUE = 1 +E_NOINTERFACE = -2147467262 + + +class GUID(ctypes.Structure): + # https://learn.microsoft.com/en-us/windows/win32/api/guiddef/ns-guiddef-guid + _fields_ = [ + ("Data1", DWORD), + ("Data2", WORD), + ("Data3", WORD), + ("Data4", BYTE * 8), + ] + + +def create_proto_com_method(name, index, restype, *argtypes): + proto = ctypes.WINFUNCTYPE(restype, *argtypes) + + def make_method(*args): + foreign_func = proto(index, name, *args) + + def call(self, *args, **kwargs): + return foreign_func(self, *args, **kwargs) + + return call + + return make_method + + +def create_guid(name): + guid = GUID() + # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-clsidfromstring + ole32.CLSIDFromString(name, byref(guid)) + return guid + + +def is_equal_guid(guid1, guid2): + # https://learn.microsoft.com/en-us/windows/win32/api/objbase/nf-objbase-isequalguid + return ole32.IsEqualGUID(byref(guid1), byref(guid2)) + + +ole32 = ctypes.oledll.ole32 + +IID_IUnknown = create_guid("{00000000-0000-0000-C000-000000000046}") +IID_IStream = create_guid("{0000000C-0000-0000-C000-000000000046}") +IID_IPersist = create_guid("{0000010C-0000-0000-C000-000000000046}") +CLSID_ShellLink = create_guid("{00021401-0000-0000-C000-000000000046}") + +# https://learn.microsoft.com/en-us/windows/win32/api/unknwn/nf-unknwn-iunknown-queryinterface(refiid_void) +proto_query_interface = create_proto_com_method( + "QueryInterface", 0, HRESULT, POINTER(GUID), POINTER(c_void_p) +) +# https://learn.microsoft.com/en-us/windows/win32/api/unknwn/nf-unknwn-iunknown-addref +proto_add_ref = create_proto_com_method("AddRef", 1, ctypes.c_long) +# https://learn.microsoft.com/en-us/windows/win32/api/unknwn/nf-unknwn-iunknown-release +proto_release = create_proto_com_method("Release", 2, ctypes.c_long) +# https://learn.microsoft.com/en-us/windows/win32/api/objidl/nf-objidl-ipersist-getclassid +proto_get_class_id = create_proto_com_method( + "GetClassID", 3, HRESULT, POINTER(GUID) +) + + +def create_shelllink_persist(typ): + ppst = typ() + # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-cocreateinstance + ole32.CoCreateInstance( + byref(CLSID_ShellLink), + None, + CLSCTX_SERVER, + byref(IID_IPersist), + byref(ppst), + ) + return ppst + + +class ForeignFunctionsThatWillCallComMethodsTests(unittest.TestCase): + def setUp(self): + # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-coinitializeex + ole32.CoInitializeEx(None, COINIT_APARTMENTTHREADED) + + def tearDown(self): + # https://learn.microsoft.com/en-us/windows/win32/api/combaseapi/nf-combaseapi-couninitialize + ole32.CoUninitialize() + gc.collect() + + def test_without_paramflags_and_iid(self): + class IUnknown(c_void_p): + QueryInterface = proto_query_interface() + AddRef = proto_add_ref() + Release = proto_release() + + class IPersist(IUnknown): + GetClassID = proto_get_class_id() + + ppst = create_shelllink_persist(IPersist) + + clsid = GUID() + hr_getclsid = ppst.GetClassID(byref(clsid)) + self.assertEqual(S_OK, hr_getclsid) + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + self.assertEqual(2, ppst.AddRef()) + self.assertEqual(3, ppst.AddRef()) + + punk = IUnknown() + hr_qi = ppst.QueryInterface(IID_IUnknown, punk) + self.assertEqual(S_OK, hr_qi) + self.assertEqual(3, punk.Release()) + + with self.assertRaises(OSError) as e: + punk.QueryInterface(IID_IStream, IUnknown()) + self.assertEqual(E_NOINTERFACE, e.exception.winerror) + + self.assertEqual(2, ppst.Release()) + self.assertEqual(1, ppst.Release()) + self.assertEqual(0, ppst.Release()) + + def test_with_paramflags_and_without_iid(self): + class IUnknown(c_void_p): + QueryInterface = proto_query_interface(None) + AddRef = proto_add_ref() + Release = proto_release() + + class IPersist(IUnknown): + GetClassID = proto_get_class_id(((OUT, "pClassID"),)) + + ppst = create_shelllink_persist(IPersist) + + clsid = ppst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + punk = IUnknown() + hr_qi = ppst.QueryInterface(IID_IUnknown, punk) + self.assertEqual(S_OK, hr_qi) + self.assertEqual(1, punk.Release()) + + with self.assertRaises(OSError) as e: + ppst.QueryInterface(IID_IStream, IUnknown()) + self.assertEqual(E_NOINTERFACE, e.exception.winerror) + + self.assertEqual(0, ppst.Release()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - COM iid parameter handling not implemented + def test_with_paramflags_and_iid(self): + class IUnknown(c_void_p): + QueryInterface = proto_query_interface(None, IID_IUnknown) + AddRef = proto_add_ref() + Release = proto_release() + + class IPersist(IUnknown): + GetClassID = proto_get_class_id(((OUT, "pClassID"),), IID_IPersist) + + ppst = create_shelllink_persist(IPersist) + + clsid = ppst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + punk = IUnknown() + hr_qi = ppst.QueryInterface(IID_IUnknown, punk) + self.assertEqual(S_OK, hr_qi) + self.assertEqual(1, punk.Release()) + + with self.assertRaises(COMError) as e: + ppst.QueryInterface(IID_IStream, IUnknown()) + self.assertEqual(E_NOINTERFACE, e.exception.hresult) + + self.assertEqual(0, ppst.Release()) + + +class CopyComPointerTests(unittest.TestCase): + def setUp(self): + ole32.CoInitializeEx(None, COINIT_APARTMENTTHREADED) + + class IUnknown(c_void_p): + QueryInterface = proto_query_interface(None, IID_IUnknown) + AddRef = proto_add_ref() + Release = proto_release() + + class IPersist(IUnknown): + GetClassID = proto_get_class_id(((OUT, "pClassID"),), IID_IPersist) + + self.IUnknown = IUnknown + self.IPersist = IPersist + + def tearDown(self): + ole32.CoUninitialize() + gc.collect() + + def test_both_are_null(self): + src = self.IPersist() + dst = self.IPersist() + + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + + self.assertIsNone(src.value) + self.assertIsNone(dst.value) + + def test_src_is_nonnull_and_dest_is_null(self): + # The reference count of the COM pointer created by `CoCreateInstance` + # is initially 1. + src = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + + # `CopyComPointer` calls `AddRef` explicitly in the C implementation. + # The refcount of `src` is incremented from 1 to 2 here. + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertEqual(src.value, dst.value) + + # This indicates that the refcount was 2 before the `Release` call. + self.assertEqual(1, src.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + self.assertEqual(0, dst.Release()) + + def test_src_is_null_and_dest_is_nonnull(self): + src = self.IPersist() + dst_orig = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + CopyComPointer(dst_orig, byref(dst)) + self.assertEqual(1, dst_orig.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + # This does NOT affects the refcount of `dst_orig`. + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertIsNone(dst.value) + + with self.assertRaises(ValueError): + dst.GetClassID() # NULL COM pointer access + + # This indicates that the refcount was 1 before the `Release` call. + self.assertEqual(0, dst_orig.Release()) + + def test_both_are_nonnull(self): + src = create_shelllink_persist(self.IPersist) + dst_orig = create_shelllink_persist(self.IPersist) + dst = self.IPersist() + CopyComPointer(dst_orig, byref(dst)) + self.assertEqual(1, dst_orig.Release()) + + self.assertEqual(dst.value, dst_orig.value) + self.assertNotEqual(src.value, dst.value) + + hr = CopyComPointer(src, byref(dst)) + + self.assertEqual(S_OK, hr) + self.assertEqual(src.value, dst.value) + self.assertNotEqual(dst.value, dst_orig.value) + + self.assertEqual(1, src.Release()) + + clsid = dst.GetClassID() + self.assertEqual(TRUE, is_equal_guid(CLSID_ShellLink, clsid)) + + self.assertEqual(0, dst.Release()) + self.assertEqual(0, dst_orig.Release()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/ctypes/test/test_wintypes.py b/Lib/test/test_ctypes/test_wintypes.py similarity index 64% rename from Lib/ctypes/test/test_wintypes.py rename to Lib/test/test_ctypes/test_wintypes.py index 243d5962ffa..a04d725a473 100644 --- a/Lib/ctypes/test/test_wintypes.py +++ b/Lib/test/test_ctypes/test_wintypes.py @@ -1,8 +1,10 @@ -import unittest - -# also work on POSIX +# See +# for reference. +# +# Tests also work on POSIX -from ctypes import * +import unittest +from ctypes import POINTER, cast, c_int16 from ctypes import wintypes @@ -38,6 +40,22 @@ def test_variant_bool(self): vb.value = [] self.assertIs(vb.value, False) + def assertIsSigned(self, ctype): + self.assertLess(ctype(-1).value, 0) + + def assertIsUnsigned(self, ctype): + self.assertGreater(ctype(-1).value, 0) + + def test_signedness(self): + for ctype in (wintypes.BYTE, wintypes.WORD, wintypes.DWORD, + wintypes.BOOLEAN, wintypes.UINT, wintypes.ULONG): + with self.subTest(ctype=ctype): + self.assertIsUnsigned(ctype) + + for ctype in (wintypes.BOOL, wintypes.INT, wintypes.LONG): + with self.subTest(ctype=ctype): + self.assertIsSigned(ctype) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 1b16da42648..12db84a1209 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -5,6 +5,7 @@ from dataclasses import * import abc +import annotationlib import io import pickle import inspect @@ -12,18 +13,21 @@ import types import weakref import traceback +import sys +import textwrap import unittest from unittest.mock import Mock from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict from copy import deepcopy -from functools import total_ordering +from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. from test import support +from test.support import import_helper # Just any custom exception we can catch. class CustomError(Exception): pass @@ -61,7 +65,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -69,6 +73,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -115,7 +120,7 @@ class Some: pass for param in inspect.signature(dataclass).parameters: if param == 'cls': continue - self.assertTrue(hasattr(Some.__dataclass_params__, param), msg=param) + self.assertHasAttr(Some.__dataclass_params__, param) def test_named_init_params(self): @dataclass @@ -666,7 +671,7 @@ class C: self.assertEqual(the_fields[0].name, 'x') self.assertEqual(the_fields[0].type, int) - self.assertFalse(hasattr(C, 'x')) + self.assertNotHasAttr(C, 'x') self.assertTrue (the_fields[0].init) self.assertTrue (the_fields[0].repr) self.assertEqual(the_fields[1].name, 'y') @@ -676,7 +681,7 @@ class C: self.assertTrue (the_fields[1].repr) self.assertEqual(the_fields[2].name, 'z') self.assertEqual(the_fields[2].type, str) - self.assertFalse(hasattr(C, 'z')) + self.assertNotHasAttr(C, 'z') self.assertTrue (the_fields[2].init) self.assertFalse(the_fields[2].repr) @@ -727,8 +732,8 @@ class C: z: object = default t: int = field(default=100) - self.assertFalse(hasattr(C, 'x')) - self.assertFalse(hasattr(C, 'y')) + self.assertNotHasAttr(C, 'x') + self.assertNotHasAttr(C, 'y') self.assertIs (C.z, default) self.assertEqual(C.t, 100) @@ -922,6 +927,20 @@ class C: validate_class(C) + def test_incomplete_annotations(self): + # gh-142214 + @dataclass + class C: + "doc" # needed because otherwise we fetch the annotations at the wrong time + x: int + + C.__annotate__ = lambda _: {} + + self.assertEqual( + annotationlib.get_annotations(C.__init__), + {"return": None} + ) + def test_missing_default(self): # Test that MISSING works the same as a default not being # specified. @@ -1776,8 +1795,7 @@ class C: self.assertIsNot(d['f'], t) self.assertEqual(d['f'].my_a(), 6) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_helper_asdict_defaultdict(self): # Ensure asdict() does not throw exceptions when a # defaultdict is a member of a dataclass @@ -1920,8 +1938,7 @@ class C: t = astuple(c, tuple_factory=list) self.assertEqual(t, ['outer', T(1, ['inner', T(11, 12, 13)], 2)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_helper_astuple_defaultdict(self): # Ensure astuple() does not throw exceptions when a # defaultdict is a member of a dataclass @@ -2316,7 +2333,7 @@ def test_docstring_one_field_with_default_none(self): class C: x: Union[int, type(None)] = None - self.assertDocStrEqual(C.__doc__, "C(x:Optional[int]=None)") + self.assertDocStrEqual(C.__doc__, "C(x:int|None=None)") def test_docstring_list_field(self): @dataclass @@ -2346,6 +2363,31 @@ class C: self.assertDocStrEqual(C.__doc__, "C(x:collections.deque=)") + def test_docstring_undefined_name(self): + @dataclass + class C: + x: undef + + self.assertDocStrEqual(C.__doc__, "C(x:undef)") + + def test_docstring_with_unsolvable_forward_ref_in_init(self): + # See: https://github.com/python/cpython/issues/128184 + ns = {} + exec( + textwrap.dedent( + """ + from dataclasses import dataclass + + @dataclass + class C: + def __init__(self, x: X, num: int) -> None: ... + """, + ), + ns, + ) + + self.assertDocStrEqual(ns['C'].__doc__, "C(x:X,num:int)") + def test_docstring_with_no_signature(self): # See https://github.com/python/cpython/issues/103449 class Meta(type): @@ -2445,6 +2487,149 @@ def __init__(self, a): self.assertEqual(D(5).a, 10) +class TestInitAnnotate(unittest.TestCase): + # Tests for the generated __annotate__ function for __init__ + # See: https://github.com/python/cpython/issues/137530 + + def test_annotate_function(self): + # No forward references + @dataclass + class A: + a: int + + value_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'a': int, 'return': None}) + self.assertEqual(forwardref_annos, {'a': int, 'return': None}) + self.assertEqual(string_annos, {'a': 'int', 'return': 'None'}) + + self.assertTrue(getattr(A.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_annotate_function_forwardref(self): + # With forward references + @dataclass + class B: + b: undefined + + # VALUE annotations should raise while unresolvable + with self.assertRaises(NameError): + _ = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(forwardref_annos, {'b': support.EqualToForwardRef('undefined', owner=B, is_class=True), 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + # Now VALUE and FORWARDREF should resolve, STRING should be unchanged + undefined = int + + value_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'b': int, 'return': None}) + self.assertEqual(forwardref_annos, {'b': int, 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + def test_annotate_function_init_false(self): + # Check `init=False` attributes don't get into the annotations of the __init__ function + @dataclass + class C: + c: str = field(init=False) + + self.assertEqual(annotationlib.get_annotations(C.__init__), {'return': None}) + + def test_annotate_function_contains_forwardref(self): + # Check string annotations on objects containing a ForwardRef + @dataclass + class D: + d: list[undefined] + + with self.assertRaises(NameError): + annotationlib.get_annotations(D.__init__) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[support.EqualToForwardRef("undefined", is_class=True, owner=D)], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + # Now test when it is defined + undefined = str + + # VALUE should now resolve + self.assertEqual( + annotationlib.get_annotations(D.__init__), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + def test_annotate_function_not_replaced(self): + # Check that __annotate__ is not replaced on non-generated __init__ functions + @dataclass(slots=True) + class E: + x: str + def __init__(self, x: int) -> None: + self.x = x + + self.assertEqual( + annotationlib.get_annotations(E.__init__), {"x": int, "return": None} + ) + + self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_slots_true_init_false(self): + # Test that slots=True and init=False work together and + # that __annotate__ is not added to __init__. + + @dataclass(slots=True, init=False) + class F: + x: int + + f = F() + f.x = 10 + self.assertEqual(f.x, 10) + + self.assertFalse(hasattr(F.__init__, "__annotate__")) + + def test_init_false_forwardref(self): + # Test forward references in fields not required for __init__ annotations. + + # At the moment this raises a NameError for VALUE annotations even though the + # undefined annotation is not required for the __init__ annotations. + # Ideally this will be fixed but currently there is no good way to resolve this + + @dataclass + class F: + not_in_init: list[undefined] = field(init=False, default=None) + in_init: int + + annos = annotationlib.get_annotations(F.__init__, format=annotationlib.Format.FORWARDREF) + self.assertEqual( + annos, + {"in_init": int, "return": None}, + ) + + with self.assertRaises(NameError): + annos = annotationlib.get_annotations(F.__init__) # NameError on not_in_init + + class TestRepr(unittest.TestCase): def test_repr(self): @dataclass @@ -2886,10 +3071,10 @@ class C: pass c = C() - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): c.i = 5 - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): del c.i @@ -3118,7 +3303,7 @@ class S(D): del s.y self.assertEqual(s.y, 10) del s.cached - self.assertFalse(hasattr(s, 'cached')) + self.assertNotHasAttr(s, 'cached') with self.assertRaises(AttributeError) as cm: del s.cached self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3132,12 +3317,12 @@ class S(D): pass s = S() - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') s.x = 5 self.assertEqual(s.x, 5) del s.x - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') with self.assertRaises(AttributeError) as cm: del s.x self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3308,7 +3493,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3318,7 +3503,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3326,6 +3511,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: @@ -3349,8 +3552,8 @@ class A: B = dataclass(A, slots=True) self.assertIsNot(A, B) - self.assertFalse(hasattr(A, "__slots__")) - self.assertTrue(hasattr(B, "__slots__")) + self.assertNotHasAttr(A, "__slots__") + self.assertHasAttr(B, "__slots__") # Can't be local to test_frozen_pickle. @dataclass(frozen=True, slots=True) @@ -3469,8 +3672,7 @@ class A: self.assertEqual(obj.a, 'a') self.assertEqual(obj.b, 'b') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots_no_weakref(self): @dataclass(slots=True) class A: @@ -3485,8 +3687,7 @@ class A: with self.assertRaises(AttributeError): a.__weakref__ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots_weakref(self): @dataclass(slots=True, weakref_slot=True) class A: @@ -3547,8 +3748,7 @@ def test_weakref_slot_make_dataclass(self): "weakref_slot is True but slots is False"): B = make_dataclass('B', [('a', int),], weakref_slot=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weakref_slot_subclass_weakref_slot(self): @dataclass(slots=True, weakref_slot=True) class Base: @@ -3567,8 +3767,7 @@ class A(Base): a_ref = weakref.ref(a) self.assertIs(a.__weakref__, a_ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weakref_slot_subclass_no_weakref_slot(self): @dataclass(slots=True, weakref_slot=True) class Base: @@ -3586,8 +3785,7 @@ class A(Base): a_ref = weakref.ref(a) self.assertIs(a.__weakref__, a_ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weakref_slot_normal_base_weakref_slot(self): class Base: __slots__ = ('__weakref__',) @@ -3632,8 +3830,7 @@ class B[T2]: self.assertTrue(B.__weakref__) B() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dataclass_derived_generic_from_base(self): T = typing.TypeVar('T') @@ -3725,7 +3922,6 @@ class A(WithDictSlot): ... @support.cpython_only def test_dataclass_slot_dict_ctype(self): # https://github.com/python/cpython/issues/123935 - from test.support import import_helper # Skips test if `_testcapi` is not present: _testcapi = import_helper.import_module('_testcapi') @@ -3773,6 +3969,50 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + @unittest.skip("TODO: RUSTPYTHON; Crash - static type name must be already interned but async_generator_wrapped_value is not") + def test_original_class_is_gced(self): + # gh-135228: Make sure when we replace the class with slots=True, the original class + # gets garbage collected. + def make_simple(): + @dataclass(slots=True) + class SlotsTest: + pass + + return SlotsTest + + def make_with_annotations(): + @dataclass(slots=True) + class SlotsTest: + x: int + + return SlotsTest + + def make_with_annotations_and_method(): + @dataclass(slots=True) + class SlotsTest: + x: int + + def method(self) -> int: + return self.x + + return SlotsTest + + def make_with_forwardref(): + @dataclass(slots=True) + class SlotsTest: + x: undefined + y: list[undefined] + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method, make_with_forwardref): + with self.subTest(make=make): + C = make() + support.gc_collect() + candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' + and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] + self.assertEqual(candidates, [C]) + class TestDescriptors(unittest.TestCase): def test_set_name(self): @@ -4217,16 +4457,56 @@ def test_no_types(self): C = make_dataclass('Point', ['x', 'y', 'z']) c = C(1, 2, 3) self.assertEqual(vars(c), {'x': 1, 'y': 2, 'z': 3}) - self.assertEqual(C.__annotations__, {'x': 'typing.Any', - 'y': 'typing.Any', - 'z': 'typing.Any'}) + self.assertEqual(C.__annotations__, {'x': typing.Any, + 'y': typing.Any, + 'z': typing.Any}) C = make_dataclass('Point', ['x', ('y', int), 'z']) c = C(1, 2, 3) self.assertEqual(vars(c), {'x': 1, 'y': 2, 'z': 3}) - self.assertEqual(C.__annotations__, {'x': 'typing.Any', + self.assertEqual(C.__annotations__, {'x': typing.Any, 'y': int, - 'z': 'typing.Any'}) + 'z': typing.Any}) + + def test_no_types_get_annotations(self): + C = make_dataclass('C', ['x', ('y', int), 'z']) + + self.assertEqual( + annotationlib.get_annotations(C, format=annotationlib.Format.VALUE), + {'x': typing.Any, 'y': int, 'z': typing.Any}, + ) + self.assertEqual( + annotationlib.get_annotations( + C, format=annotationlib.Format.FORWARDREF), + {'x': typing.Any, 'y': int, 'z': typing.Any}, + ) + self.assertEqual( + annotationlib.get_annotations( + C, format=annotationlib.Format.STRING), + {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, + ) + + def test_no_types_no_typing_import(self): + with import_helper.CleanImport('typing'): + self.assertNotIn('typing', sys.modules) + C = make_dataclass('C', ['x', ('y', int)]) + + self.assertNotIn('typing', sys.modules) + self.assertEqual( + C.__annotate__(annotationlib.Format.FORWARDREF), + { + 'x': annotationlib.ForwardRef('Any', module='typing'), + 'y': int, + }, + ) + self.assertNotIn('typing', sys.modules) + + for field in fields(C): + if field.name == "x": + self.assertEqual(field.type, annotationlib.ForwardRef('Any', module='typing')) + else: + self.assertEqual(field.name, "y") + self.assertIs(field.type, int) def test_module_attr(self): self.assertEqual(ByMakeDataClass.__module__, __name__) @@ -4313,6 +4593,23 @@ def test_funny_class_names_names(self): C = make_dataclass(classname, ['a', 'b']) self.assertEqual(C.__name__, classname) + def test_dataclass_decorator_default(self): + C = make_dataclass('C', [('x', int)], decorator=dataclass) + c = C(10) + self.assertEqual(c.x, 10) + + def test_dataclass_custom_decorator(self): + def custom_dataclass(cls, *args, **kwargs): + dc = dataclass(cls, *args, **kwargs) + dc.__custom__ = True + return dc + + C = make_dataclass('C', [('x', int)], decorator=custom_dataclass) + c = C(10) + self.assertEqual(c.x, 10) + self.assertEqual(c.__custom__, True) + + class TestReplace(unittest.TestCase): def test(self): @dataclass(frozen=True) @@ -4561,8 +4858,7 @@ class Date(Ordered): self.assertFalse(inspect.isabstract(Date)) self.assertGreater(Date(2020,12,25), Date(2020,8,31)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_maintain_abc(self): class A(abc.ABC): @abc.abstractmethod @@ -4728,8 +5024,6 @@ class C: b: int = field(kw_only=True) self.assertEqual(C(42, b=10).__match_args__, ('a',)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_KW_ONLY(self): @dataclass class A: @@ -4920,6 +5214,140 @@ def test_make_dataclass(self): self.assertTrue(fields(B)[0].kw_only) self.assertFalse(fields(B)[1].kw_only) + def test_deferred_annotations(self): + @dataclass + class A: + x: undefined + y: ClassVar[undefined] + + fs = fields(A) + self.assertEqual(len(fs), 1) + self.assertEqual(fs[0].name, 'x') + + +class TestZeroArgumentSuperWithSlots(unittest.TestCase): + def test_zero_argument_super(self): + @dataclass(slots=True) + class A: + def foo(self): + super() + + A().foo() + + def test_dunder_class_with_old_property(self): + @dataclass(slots=True) + class A: + def _get_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + return __class__ + + def _set_foo(slf, value): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + def _del_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + foo = property(_get_foo, _set_foo, _del_foo) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + def test_dunder_class_with_new_property(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return slf.__class__ + + @foo.setter + def foo(slf, value): + self.assertIs(__class__, type(slf)) + + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + # Test the parts of a property individually. + def test_slots_dunder_class_property_getter(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return __class__ + + a = A() + self.assertIs(a.foo, A) + + def test_slots_dunder_class_property_setter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.setter + def foo(slf, val): + self.assertIs(__class__, type(slf)) + + a = A() + a.foo = 4 + + def test_slots_dunder_class_property_deleter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + del a.foo + + def test_wrapped(self): + def mydecorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + @dataclass(slots=True) + class A: + @mydecorator + def foo(self): + super() + + A().foo() + + def test_remembered_class(self): + # Apply the dataclass decorator manually (not when the class + # is created), so that we can keep a reference to the + # undecorated class. + class A: + def cls(self): + return __class__ + + self.assertIs(A().cls(), A) + + B = dataclass(slots=True)(A) + self.assertIs(B().cls(), B) + + # This is undesirable behavior, but is a function of how + # modifying __class__ in the closure works. I'm not sure this + # should be tested or not: I don't really want to guarantee + # this behavior, but I don't want to lose the point that this + # is how it works. + + # The underlying class is "broken" by changing its __class__ + # in A.foo() to B. This normally isn't a problem, because no + # one will be keeping a reference to the underlying class A. + self.assertIs(A().cls(), B) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e615d284cd3..e68eca29564 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -1,23 +1,29 @@ """Test script for the dbm.open function based on testdumbdbm.py""" +import sys import unittest -import glob -import test.support -from test.support import os_helper, import_helper +import dbm +import os +from test.support import import_helper +from test.support import os_helper + + +try: + from dbm import sqlite3 as dbm_sqlite3 +except ImportError: + dbm_sqlite3 = None -# Skip tests if dbm module doesn't exist. -dbm = import_helper.import_module('dbm') try: from dbm import ndbm except ImportError: ndbm = None -_fname = os_helper.TESTFN +dirname = os_helper.TESTFN +_fname = os.path.join(dirname, os_helper.TESTFN) # -# Iterates over every database module supported by dbm currently available, -# setting dbm to use each in turn, and yielding that module +# Iterates over every database module supported by dbm currently available. # def dbm_iterator(): for name in dbm._names: @@ -31,11 +37,12 @@ def dbm_iterator(): # # Clean up all scratch databases we might have created during testing # -def delete_files(): - # we don't know the precise name the underlying database uses - # so we use glob to locate all names - for f in glob.glob(glob.escape(_fname) + "*"): - os_helper.unlink(f) +def cleaunup_test_dir(): + os_helper.rmtree(dirname) + +def setup_test_dir(): + cleaunup_test_dir() + os.mkdir(dirname) class AnyDBMTestCase: @@ -129,85 +136,130 @@ def test_anydbm_access(self): assert(f[key] == b"Python:") f.close() + def test_open_with_bytes(self): + dbm.open(os.fsencode(_fname), "c").close() + + def test_open_with_pathlib_path(self): + dbm.open(os_helper.FakePath(_fname), "c").close() + + def test_open_with_pathlib_path_bytes(self): + dbm.open(os_helper.FakePath(os.fsencode(_fname)), "c").close() + def read_helper(self, f): keys = self.keys_helper(f) for key in self._dict: self.assertEqual(self._dict[key], f[key.encode("ascii")]) - def tearDown(self): - delete_files() + def test_keys(self): + with dbm.open(_fname, 'c') as d: + self.assertEqual(d.keys(), []) + a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] + for k, v in a: + d[k] = v + self.assertEqual(sorted(d.keys()), sorted(k for (k, v) in a)) + for k, v in a: + self.assertIn(k, d) + self.assertEqual(d[k], v) + self.assertNotIn(b'xxx', d) + self.assertRaises(KeyError, lambda: d[b'xxx']) + + def test_clear(self): + with dbm.open(_fname, 'c') as d: + self.assertEqual(d.keys(), []) + a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] + for k, v in a: + d[k] = v + for k, _ in a: + self.assertIn(k, d) + self.assertEqual(len(d), len(a)) + + d.clear() + self.assertEqual(len(d), 0) + for k, _ in a: + self.assertNotIn(k, d) def setUp(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) dbm._defaultmod = self.module - delete_files() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() class WhichDBTestCase(unittest.TestCase): def test_whichdb(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) + _bytes_fname = os.fsencode(_fname) + fnames = [_fname, os_helper.FakePath(_fname), + _bytes_fname, os_helper.FakePath(_bytes_fname)] for module in dbm_iterator(): # Check whether whichdb correctly guesses module name # for databases opened with "module" module. - # Try with empty files first name = module.__name__ - if name == 'dbm.dumb': - continue # whichdb can't support dbm.dumb - delete_files() - f = module.open(_fname, 'c') - f.close() - self.assertEqual(name, self.dbm.whichdb(_fname)) + setup_test_dir() + dbm._defaultmod = module + # Try with empty files first + with module.open(_fname, 'c'): pass + for path in fnames: + self.assertEqual(name, self.dbm.whichdb(path)) # Now add a key - f = module.open(_fname, 'w') - f[b"1"] = b"1" - # and test that we can find it - self.assertIn(b"1", f) - # and read it - self.assertEqual(f[b"1"], b"1") - f.close() - self.assertEqual(name, self.dbm.whichdb(_fname)) + with module.open(_fname, 'w') as f: + f[b"1"] = b"1" + # and test that we can find it + self.assertIn(b"1", f) + # and read it + self.assertEqual(f[b"1"], b"1") + for path in fnames: + self.assertEqual(name, self.dbm.whichdb(path)) @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - db_file = '{}_ndbm.db'.format(_fname) - with open(db_file, 'w'): - self.addCleanup(os_helper.unlink, db_file) - self.assertIsNone(self.dbm.whichdb(db_file[:-3])) + with open(_fname + '.db', 'wb') as f: + f.write(b'spam') + _bytes_fname = os.fsencode(_fname) + fnames = [_fname, os_helper.FakePath(_fname), + _bytes_fname, os_helper.FakePath(_bytes_fname)] + for path in fnames: + self.assertIsNone(self.dbm.whichdb(path)) + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3(self): + # Databases created by dbm.sqlite3 are detected correctly. + with dbm_sqlite3.open(_fname, "c") as db: + db["key"] = "value" + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3_existing_db(self): + # Existing sqlite3 databases are detected correctly. + sqlite3 = import_helper.import_module("sqlite3") + try: + # Create an empty database. + with sqlite3.connect(_fname) as cx: + cx.execute("CREATE TABLE dummy(database)") + cx.commit() + finally: + cx.close() + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") - def tearDown(self): - delete_files() def setUp(self): - delete_files() - self.filename = os_helper.TESTFN - self.d = dbm.open(self.filename, 'c') - self.d.close() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() self.dbm = import_helper.import_fresh_module('dbm') - def test_keys(self): - self.d = dbm.open(self.filename, 'c') - self.assertEqual(self.d.keys(), []) - a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] - for k, v in a: - self.d[k] = v - self.assertEqual(sorted(self.d.keys()), sorted(k for (k, v) in a)) - for k, v in a: - self.assertIn(k, self.d) - self.assertEqual(self.d[k], v) - self.assertNotIn(b'xxx', self.d) - self.assertRaises(KeyError, lambda: self.d[b'xxx']) - self.d.close() - - -def load_tests(loader, tests, pattern): - classes = [] - for mod in dbm_iterator(): - classes.append(type("TestCase-" + mod.__name__, - (AnyDBMTestCase, unittest.TestCase), - {'module': mod})) - suites = [unittest.makeSuite(c) for c in classes] - - tests.addTests(suites) - return tests + +for mod in dbm_iterator(): + assert mod.__name__.startswith('dbm.') + suffix = mod.__name__[4:] + testname = f'TestCase_{suffix}' + cls = type(testname, + (AnyDBMTestCase, unittest.TestCase), + {'module': mod}) + # TODO: RUSTPYTHON; sqlite3 file locking prevents cleanup on Windows + if suffix == 'sqlite3' and sys.platform == 'win32': + cls = unittest.skip("TODO: RUSTPYTHON; sqlite3 file locking on Windows")(cls) + globals()[testname] = cls if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index 0dc489362b2..672f9092207 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -15,6 +15,7 @@ _fname = os_helper.TESTFN + def _delete_files(): for ext in [".dir", ".dat", ".bak"]: try: @@ -41,6 +42,7 @@ def test_dumbdbm_creation(self): self.read_helper(f) @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + @os_helper.skip_unless_working_chmod def test_dumbdbm_creation_mode(self): try: old_umask = os.umask(0o002) @@ -231,7 +233,7 @@ def test_create_new(self): self.assertEqual(f.keys(), []) def test_eval(self): - with open(_fname + '.dir', 'w') as stream: + with open(_fname + '.dir', 'w', encoding="utf-8") as stream: stream.write("str(print('Hacked!')), 0\n") with support.captured_stdout() as stdout: with self.assertRaises(ValueError): @@ -244,9 +246,27 @@ def test_missing_data(self): _delete_files() with self.assertRaises(FileNotFoundError): dumbdbm.open(_fname, value) + self.assertFalse(os.path.exists(_fname + '.dat')) self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + def test_missing_index(self): with dumbdbm.open(_fname, 'n') as f: pass @@ -257,6 +277,60 @@ def test_missing_index(self): self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + + def test_sync_empty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + def test_sync_nonempty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + def test_invalid_flag(self): for flag in ('x', 'rf', None): with self.assertRaisesRegex(ValueError, @@ -264,6 +338,7 @@ def test_invalid_flag(self): "'r', 'w', 'c', or 'n'"): dumbdbm.open(_fname, flag) + @os_helper.skip_unless_working_chmod def test_readonly_files(self): with os_helper.temp_dir() as dir: fname = os.path.join(dir, 'db') @@ -293,6 +368,15 @@ def test_nonascii_filename(self): self.assertTrue(b'key' in db) self.assertEqual(db[b'key'], b'value') + def test_open_with_pathlib_path(self): + dumbdbm.open(os_helper.FakePath(_fname), "c").close() + + def test_open_with_bytes_path(self): + dumbdbm.open(os.fsencode(_fname), "c").close() + + def test_open_with_pathlib_bytes_path(self): + dumbdbm.open(os_helper.FakePath(os.fsencode(_fname)), "c").close() + def tearDown(self): _delete_files() diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py new file mode 100644 index 00000000000..f367a98865d --- /dev/null +++ b/Lib/test/test_dbm_sqlite3.py @@ -0,0 +1,361 @@ +import os +import stat +import sys +import unittest +from contextlib import closing +from functools import partial +from pathlib import Path +from test.support import import_helper, os_helper + +dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") +# N.B. The test will fail on some platforms without sqlite3 +# if the sqlite3 import is above the import of dbm.sqlite3. +# This is deliberate: if the import helper managed to import dbm.sqlite3, +# we must inevitably be able to import sqlite3. Else, we have a problem. +import sqlite3 +from dbm.sqlite3 import _normalize_uri + + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + + +class _SQLiteDbmTests(unittest.TestCase): + + def setUp(self): + self.filename = os_helper.TESTFN + db = dbm_sqlite3.open(self.filename, "c") + db.close() + + def tearDown(self): + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) + + +class URI(unittest.TestCase): + + def test_uri_substitutions(self): + dataset = ( + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3F%25%25%23"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + self.assertEndsWith(_normalize_uri(path), normalized) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_uri_windows(self): + dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), + # Absolute with drive letter. + (r"C:\Projects\apilibrary\apilibrary.sln", + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") + self.assertEndsWith(_normalize_uri(path), normalized) + + +class ReadOnly(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + with dbm_sqlite3.open(self.filename, "w") as db: + db[b"key1"] = "value1" + db[b"key2"] = "value2" + self.db = dbm_sqlite3.open(self.filename, "r") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_readonly_read(self): + self.assertEqual(self.db[b"key1"], b"value1") + self.assertEqual(self.db[b"key2"], b"value2") + + def test_readonly_write(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"new"] = "value" + + def test_readonly_delete(self): + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"key1"] + + def test_readonly_keys(self): + self.assertEqual(self.db.keys(), [b"key1", b"key2"]) + + def test_readonly_iter(self): + self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + + +@unittest.skipIf(root_in_posix, "test is meanless with root privilege") +class ReadOnlyFilesystem(unittest.TestCase): + + def setUp(self): + self.test_dir = os_helper.TESTFN + self.addCleanup(os_helper.rmtree, self.test_dir) + os.mkdir(self.test_dir) + self.db_path = os.path.join(self.test_dir, "test.db") + + db = dbm_sqlite3.open(self.db_path, "c") + db[b"key"] = b"value" + db.close() + + def test_readonly_file_read(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_file_write(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "w") as db: + with self.assertRaises(dbm_sqlite3.error): + db[b"newkey"] = b"newvalue" + + def test_readonly_dir_read(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_dir_write(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "w") as db: + try: + db[b"newkey"] = b"newvalue" + modified = True # on Windows and macOS + except dbm_sqlite3.error: + modified = False + with dbm_sqlite3.open(self.db_path, "r") as db: + if modified: + self.assertEqual(db[b"newkey"], b"newvalue") + else: + self.assertNotIn(b"newkey", db) + + +class ReadWrite(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def db_content(self): + with closing(sqlite3.connect(self.filename)) as cx: + keys = [r[0] for r in cx.execute("SELECT key FROM Dict")] + vals = [r[0] for r in cx.execute("SELECT value FROM Dict")] + return keys, vals + + def test_readwrite_unique_key(self): + self.db["key"] = "value" + self.db["key"] = "other" + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"other"]) + + def test_readwrite_delete(self): + self.db["key"] = "value" + self.db["new"] = "other" + + del self.db[b"new"] + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"value"]) + + del self.db[b"key"] + keys, vals = self.db_content() + self.assertEqual(keys, []) + self.assertEqual(vals, []) + + def test_readwrite_null_key(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[None] = "value" + + def test_readwrite_null_value(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"key"] = None + + +class Misuse(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_misuse_double_create(self): + self.db["key"] = "value" + with dbm_sqlite3.open(self.filename, "c") as db: + self.assertEqual(db[b"key"], b"value") + + def test_misuse_double_close(self): + self.db.close() + + def test_misuse_invalid_flag(self): + regex = "must be.*'r'.*'w'.*'c'.*'n', not 'invalid'" + with self.assertRaisesRegex(ValueError, regex): + dbm_sqlite3.open(self.filename, flag="invalid") + + def test_misuse_double_delete(self): + self.db["key"] = "value" + del self.db[b"key"] + with self.assertRaises(KeyError): + del self.db[b"key"] + + def test_misuse_invalid_key(self): + with self.assertRaises(KeyError): + self.db[b"key"] + + def test_misuse_iter_close1(self): + self.db["1"] = 1 + it = iter(self.db) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_iter_close2(self): + self.db["1"] = 1 + self.db["2"] = 2 + it = iter(self.db) + next(it) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_use_after_close(self): + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + self.db[b"read"] + with self.assertRaises(dbm_sqlite3.error): + self.db[b"write"] = "value" + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"del"] + with self.assertRaises(dbm_sqlite3.error): + len(self.db) + with self.assertRaises(dbm_sqlite3.error): + self.db.keys() + + def test_misuse_reinit(self): + with self.assertRaises(dbm_sqlite3.error): + self.db.__init__("new.db", flag="n", mode=0o666) + + def test_misuse_empty_filename(self): + for flag in "r", "w", "c", "n": + with self.assertRaises(dbm_sqlite3.error): + db = dbm_sqlite3.open("", flag="c") + + +class DataTypes(_SQLiteDbmTests): + + dataset = ( + # (raw, coerced) + (42, b"42"), + (3.14, b"3.14"), + ("string", b"string"), + (b"bytes", b"bytes"), + ) + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_datatypes_values(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db["key"] = raw + self.assertEqual(self.db[b"key"], coerced) + + def test_datatypes_keys(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db[raw] = "value" + self.assertEqual(self.db[coerced], b"value") + # Raw keys are silently coerced to bytes. + self.assertEqual(self.db[raw], b"value") + del self.db[raw] + + def test_datatypes_replace_coerced(self): + self.db["10"] = "value" + self.db[b"10"] = "value" + self.db[10] = "value" + self.assertEqual(self.db.keys(), [b"10"]) + + +class CorruptDatabase(_SQLiteDbmTests): + """Verify that database exceptions are raised as dbm.sqlite3.error.""" + + def setUp(self): + super().setUp() + with closing(sqlite3.connect(self.filename)) as cx: + with cx: + cx.execute("DROP TABLE IF EXISTS Dict") + cx.execute("CREATE TABLE Dict (invalid_schema)") + + def check(self, flag, fn, should_succeed=False): + with closing(dbm_sqlite3.open(self.filename, flag)) as db: + with self.assertRaises(dbm_sqlite3.error): + fn(db) + + @staticmethod + def read(db): + return db["key"] + + @staticmethod + def write(db): + db["key"] = "value" + + @staticmethod + def iter(db): + next(iter(db)) + + @staticmethod + def keys(db): + db.keys() + + @staticmethod + def del_(db): + del db["key"] + + @staticmethod + def len_(db): + len(db) + + def test_corrupt_readwrite(self): + for flag in "r", "w", "c": + with self.subTest(flag=flag): + check = partial(self.check, flag=flag) + check(fn=self.read) + check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + check(fn=self.len_) + + def test_corrupt_force_new(self): + with closing(dbm_sqlite3.open(self.filename, "n")) as db: + db["foo"] = "write" + _ = db[b"foo"] + next(iter(db)) + del db[b"foo"] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 01b0c06196c..3017c3337c7 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -45,6 +45,7 @@ import random import inspect import threading +import contextvars if sys.platform == 'darwin': @@ -753,7 +754,7 @@ def test_explicit_context_create_decimal(self): for v in [-2**63-1, -2**63, -2**31-1, -2**31, 0, 2**31-1, 2**31, 2**63-1, 2**63]: d = nc.create_decimal(v) - self.assertTrue(isinstance(d, Decimal)) + self.assertIsInstance(d, Decimal) self.assertEqual(int(d), v) nc.prec = 3 @@ -813,6 +814,29 @@ def test_explicit_context_create_from_float(self): x = random.expovariate(0.01) * (random.random() * 2.0 - 1.0) self.assertEqual(x, float(nc.create_decimal(x))) # roundtrip + def test_from_number(self, cls=None): + Decimal = self.decimal.Decimal + if cls is None: + cls = Decimal + + def check(arg, expected): + d = cls.from_number(arg) + self.assertIs(type(d), cls) + self.assertEqual(d, expected) + + check(314, Decimal(314)) + check(3.14, Decimal.from_float(3.14)) + check(Decimal('3.14'), Decimal('3.14')) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '314') + self.assertRaises(TypeError, cls.from_number, (0, (3, 1, 4), 0)) + self.assertRaises(TypeError, cls.from_number, object()) + + def test_from_number_subclass(self, cls=None): + class DecimalSubclass(self.decimal.Decimal): + pass + self.test_from_number(DecimalSubclass) + def test_unicode_digits(self): Decimal = self.decimal.Decimal @@ -830,9 +854,8 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_unicode_digits(self): # TODO(RUSTPYTHON): Remove this test when it pass + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unicode_digits(self): return super().test_unicode_digits() class ImplicitConstructionTest: @@ -963,6 +986,7 @@ def test_formatting(self): ('.0f', '0e-2', '0'), ('.0f', '3.14159265', '3'), ('.1f', '3.14159265', '3.1'), + ('.01f', '3.14159265', '3.1'), # leading zero in precision ('.4f', '3.14159265', '3.1416'), ('.6f', '3.14159265', '3.141593'), ('.7f', '3.14159265', '3.1415926'), # round-half-even! @@ -1048,6 +1072,7 @@ def test_formatting(self): ('8,', '123456', ' 123,456'), ('08,', '123456', '0,123,456'), # special case: extra 0 needed ('+08,', '123456', '+123,456'), # but not if there's a sign + ('008,', '123456', '0,123,456'), # leading zero in width (' 08,', '123456', ' 123,456'), ('08,', '-123456', '-123,456'), ('+09,', '123456', '+0,123,456'), @@ -1064,6 +1089,20 @@ def test_formatting(self): (',%', '123.456789', '12,345.6789%'), (',e', '123456', '1.23456e+5'), (',E', '123456', '1.23456E+5'), + # ... with '_' instead + ('_', '1234567', '1_234_567'), + ('07_', '1234.56', '1_234.56'), + ('_', '1.23456789', '1.23456789'), + ('_%', '123.456789', '12_345.6789%'), + # and now for something completely different... + ('.,', '1.23456789', '1.234,567,89'), + ('._', '1.23456789', '1.234_567_89'), + ('.6_f', '12345.23456789', '12345.234_568'), + (',._%', '123.456789', '12,345.678_9%'), + (',._e', '123456', '1.234_56e+5'), + (',.4_e', '123456', '1.234_6e+5'), + (',.3_e', '123456', '1.235e+5'), + (',._E', '123456', '1.234_56E+5'), # negative zero: default behavior ('.1f', '-0', '-0.0'), @@ -1137,6 +1176,10 @@ def test_formatting(self): # bytes format argument self.assertRaises(TypeError, Decimal(1).__format__, b'-020') + # precision or fractional part separator should follow after dot + self.assertRaises(ValueError, format, Decimal(1), '.f') + self.assertRaises(ValueError, format, Decimal(1), '._6f') + def test_negative_zero_format_directed_rounding(self): with self.decimal.localcontext() as ctx: ctx.rounding = ROUND_CEILING @@ -1708,8 +1751,13 @@ def test_threading(self): self.finish1 = threading.Event() self.finish2 = threading.Event() - th1 = threading.Thread(target=thfunc1, args=(self,)) - th2 = threading.Thread(target=thfunc2, args=(self,)) + # This test wants to start threads with an empty context, no matter + # the setting of sys.flags.thread_inherit_context. We pass the + # 'context' argument explicitly with an empty context instance. + th1 = threading.Thread(target=thfunc1, args=(self,), + context=contextvars.Context()) + th2 = threading.Thread(target=thfunc2, args=(self,), + context=contextvars.Context()) th1.start() th2.start() @@ -2573,8 +2621,8 @@ class PythonAPItests: def test_abc(self): Decimal = self.decimal.Decimal - self.assertTrue(issubclass(Decimal, numbers.Number)) - self.assertFalse(issubclass(Decimal, numbers.Real)) + self.assertIsSubclass(Decimal, numbers.Number) + self.assertNotIsSubclass(Decimal, numbers.Real) self.assertIsInstance(Decimal(0), numbers.Number) self.assertNotIsInstance(Decimal(0), numbers.Real) @@ -2673,7 +2721,7 @@ class MyDecimal(Decimal): def __init__(self, _): self.x = 'y' - self.assertTrue(issubclass(MyDecimal, Decimal)) + self.assertIsSubclass(MyDecimal, Decimal) r = MyDecimal.from_float(0.1) self.assertEqual(type(r), MyDecimal) @@ -2891,31 +2939,31 @@ def test_exception_hierarchy(self): Rounded = decimal.Rounded Clamped = decimal.Clamped - self.assertTrue(issubclass(DecimalException, ArithmeticError)) - - self.assertTrue(issubclass(InvalidOperation, DecimalException)) - self.assertTrue(issubclass(FloatOperation, DecimalException)) - self.assertTrue(issubclass(FloatOperation, TypeError)) - self.assertTrue(issubclass(DivisionByZero, DecimalException)) - self.assertTrue(issubclass(DivisionByZero, ZeroDivisionError)) - self.assertTrue(issubclass(Overflow, Rounded)) - self.assertTrue(issubclass(Overflow, Inexact)) - self.assertTrue(issubclass(Overflow, DecimalException)) - self.assertTrue(issubclass(Underflow, Inexact)) - self.assertTrue(issubclass(Underflow, Rounded)) - self.assertTrue(issubclass(Underflow, Subnormal)) - self.assertTrue(issubclass(Underflow, DecimalException)) - - self.assertTrue(issubclass(Subnormal, DecimalException)) - self.assertTrue(issubclass(Inexact, DecimalException)) - self.assertTrue(issubclass(Rounded, DecimalException)) - self.assertTrue(issubclass(Clamped, DecimalException)) - - self.assertTrue(issubclass(decimal.ConversionSyntax, InvalidOperation)) - self.assertTrue(issubclass(decimal.DivisionImpossible, InvalidOperation)) - self.assertTrue(issubclass(decimal.DivisionUndefined, InvalidOperation)) - self.assertTrue(issubclass(decimal.DivisionUndefined, ZeroDivisionError)) - self.assertTrue(issubclass(decimal.InvalidContext, InvalidOperation)) + self.assertIsSubclass(DecimalException, ArithmeticError) + + self.assertIsSubclass(InvalidOperation, DecimalException) + self.assertIsSubclass(FloatOperation, DecimalException) + self.assertIsSubclass(FloatOperation, TypeError) + self.assertIsSubclass(DivisionByZero, DecimalException) + self.assertIsSubclass(DivisionByZero, ZeroDivisionError) + self.assertIsSubclass(Overflow, Rounded) + self.assertIsSubclass(Overflow, Inexact) + self.assertIsSubclass(Overflow, DecimalException) + self.assertIsSubclass(Underflow, Inexact) + self.assertIsSubclass(Underflow, Rounded) + self.assertIsSubclass(Underflow, Subnormal) + self.assertIsSubclass(Underflow, DecimalException) + + self.assertIsSubclass(Subnormal, DecimalException) + self.assertIsSubclass(Inexact, DecimalException) + self.assertIsSubclass(Rounded, DecimalException) + self.assertIsSubclass(Clamped, DecimalException) + + self.assertIsSubclass(decimal.ConversionSyntax, InvalidOperation) + self.assertIsSubclass(decimal.DivisionImpossible, InvalidOperation) + self.assertIsSubclass(decimal.DivisionUndefined, InvalidOperation) + self.assertIsSubclass(decimal.DivisionUndefined, ZeroDivisionError) + self.assertIsSubclass(decimal.InvalidContext, InvalidOperation) @requires_cdecimal class CPythonAPItests(PythonAPItests, unittest.TestCase): @@ -2923,7 +2971,6 @@ class CPythonAPItests(PythonAPItests, unittest.TestCase): class PyPythonAPItests(PythonAPItests, unittest.TestCase): decimal = P - class ContextAPItests: def test_none_args(self): @@ -4371,6 +4418,51 @@ class CContextSubclassing(ContextSubclassing, unittest.TestCase): class PyContextSubclassing(ContextSubclassing, unittest.TestCase): decimal = P +class IEEEContexts: + + def test_ieee_context(self): + # issue 8786: Add support for IEEE 754 contexts to decimal module. + IEEEContext = self.decimal.IEEEContext + + def assert_rest(self, context): + self.assertEqual(context.clamp, 1) + assert_signals(self, context, 'traps', []) + assert_signals(self, context, 'flags', []) + + c = IEEEContext(32) + self.assertEqual(c.prec, 7) + self.assertEqual(c.Emax, 96) + self.assertEqual(c.Emin, -95) + assert_rest(self, c) + + c = IEEEContext(64) + self.assertEqual(c.prec, 16) + self.assertEqual(c.Emax, 384) + self.assertEqual(c.Emin, -383) + assert_rest(self, c) + + c = IEEEContext(128) + self.assertEqual(c.prec, 34) + self.assertEqual(c.Emax, 6144) + self.assertEqual(c.Emin, -6143) + assert_rest(self, c) + + # Invalid values + self.assertRaises(ValueError, IEEEContext, -1) + self.assertRaises(ValueError, IEEEContext, 123) + self.assertRaises(ValueError, IEEEContext, 1024) + + def test_constants(self): + # IEEEContext + IEEE_CONTEXT_MAX_BITS = self.decimal.IEEE_CONTEXT_MAX_BITS + self.assertIn(IEEE_CONTEXT_MAX_BITS, {256, 512}) + +@requires_cdecimal +class CIEEEContexts(IEEEContexts, unittest.TestCase): + decimal = C +class PyIEEEContexts(IEEEContexts, unittest.TestCase): + decimal = P + @skip_if_extra_functionality @requires_cdecimal class CheckAttributes(unittest.TestCase): @@ -4382,6 +4474,7 @@ def test_module_attributes(self): self.assertEqual(C.MAX_EMAX, P.MAX_EMAX) self.assertEqual(C.MIN_EMIN, P.MIN_EMIN) self.assertEqual(C.MIN_ETINY, P.MIN_ETINY) + self.assertEqual(C.IEEE_CONTEXT_MAX_BITS, P.IEEE_CONTEXT_MAX_BITS) self.assertTrue(C.HAVE_THREADS is True or C.HAVE_THREADS is False) self.assertTrue(P.HAVE_THREADS is True or P.HAVE_THREADS is False) @@ -4465,12 +4558,10 @@ def test_implicit_context(self): self.assertIs(Decimal("NaN").fma(7, 1).is_nan(), True) # three arg power self.assertEqual(pow(Decimal(10), 2, 7), 2) + self.assertEqual(pow(10, Decimal(2), 7), 2) if self.decimal == C: - self.assertEqual(pow(10, Decimal(2), 7), 2) self.assertEqual(pow(10, 2, Decimal(7)), 2) else: - # XXX: Three-arg power doesn't use __rpow__. - self.assertRaises(TypeError, pow, 10, Decimal(2), 7) # XXX: There is no special method to dispatch on the # third arg of three-arg power. self.assertRaises(TypeError, pow, 10, 2, Decimal(7)) @@ -4684,10 +4775,6 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_int_limit) super().tearDown() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_implicit_context(self): # TODO(RUSTPYTHON): Remove this test when it pass - return super().test_implicit_context() class PyFunctionality(unittest.TestCase): """Extra functionality in decimal.py""" @@ -4872,42 +4959,6 @@ def test_py__round(self): class CFunctionality(unittest.TestCase): """Extra functionality in _decimal""" - @requires_extra_functionality - def test_c_ieee_context(self): - # issue 8786: Add support for IEEE 754 contexts to decimal module. - IEEEContext = C.IEEEContext - DECIMAL32 = C.DECIMAL32 - DECIMAL64 = C.DECIMAL64 - DECIMAL128 = C.DECIMAL128 - - def assert_rest(self, context): - self.assertEqual(context.clamp, 1) - assert_signals(self, context, 'traps', []) - assert_signals(self, context, 'flags', []) - - c = IEEEContext(DECIMAL32) - self.assertEqual(c.prec, 7) - self.assertEqual(c.Emax, 96) - self.assertEqual(c.Emin, -95) - assert_rest(self, c) - - c = IEEEContext(DECIMAL64) - self.assertEqual(c.prec, 16) - self.assertEqual(c.Emax, 384) - self.assertEqual(c.Emin, -383) - assert_rest(self, c) - - c = IEEEContext(DECIMAL128) - self.assertEqual(c.prec, 34) - self.assertEqual(c.Emax, 6144) - self.assertEqual(c.Emin, -6143) - assert_rest(self, c) - - # Invalid values - self.assertRaises(OverflowError, IEEEContext, 2**63) - self.assertRaises(ValueError, IEEEContext, -1) - self.assertRaises(ValueError, IEEEContext, 1024) - @requires_extra_functionality def test_c_context(self): Context = C.Context @@ -4928,12 +4979,6 @@ def test_constants(self): C.DecSubnormal, C.DecUnderflow ) - # IEEEContext - self.assertEqual(C.DECIMAL32, 32) - self.assertEqual(C.DECIMAL64, 64) - self.assertEqual(C.DECIMAL128, 128) - self.assertEqual(C.IEEE_CONTEXT_MAX_BITS, 512) - # Conditions for i, v in enumerate(cond): self.assertEqual(v, 1< int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + with self.assertRaises(AttributeError): + del unannotated.__annotations__ + + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + + def test_staticmethod_annotations_without_dict_access(self): + # gh-125017: this used to crash + class Spam: + def __new__(cls, x, y): + pass + + self.assertEqual(Spam.__new__.__annotations__, {}) + obj = Spam.__dict__['__new__'] + self.assertIsInstance(obj, staticmethod) + self.assertEqual(obj.__annotations__, {}) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') @@ -1734,7 +1785,7 @@ class D(C): class E: # *not* subclassing from C foo = C.foo self.assertEqual(E().foo.__func__, C.foo) # i.e., unbound - self.assertTrue(repr(C.foo.__get__(C())).startswith("__... @@ -4384,7 +4456,7 @@ class D(C): self.assertIsInstance(a, C) # Baseline self.assertIsInstance(pa, C) # Test - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_super(self): # Testing super() for a proxy object... class Proxy(object): @@ -4408,7 +4480,7 @@ def f(self): p = Proxy(obj) self.assertEqual(C.__dict__["f"](p), "B.f->C.f") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_carloverre(self): # Testing prohibition of Carlo Verre's hack... try: @@ -4441,7 +4513,7 @@ class C(B, A): except TypeError: self.fail("setattr through direct base types should be legal") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_carloverre_multi_inherit_invalid(self): class A(type): def __setattr__(cls, key, value): @@ -4480,7 +4552,9 @@ class Oops(object): o.whatever = Provoker(o) del o - @unittest.skip('TODO: RUSTPYTHON; rustpython segmentation fault') + @unittest.skip("TODO: RUSTPYTHON; rustpython segmentation fault") + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_wrapper_segfault(self): # SF 927248: deeply nested wrappers could cause stack overflow @@ -4555,7 +4629,7 @@ def assertNotOrderable(self, a, b): with self.assertRaises(TypeError): a >= b - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_method_wrapper(self): # Testing method-wrapper objects... # did not support any reflection before 2.5 @@ -4768,7 +4842,7 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ - @unittest.skip('TODO: RUSTPYTHON; crash. "dict has non-string keys: [PyObject PyInt { value: 1 }]"') + @unittest.skip("TODO: RUSTPYTHON; crash. \"dict has non-string keys: [PyObject PyInt { value: 1 }]\"") def test_gh55664(self): # gh-55664: issue a warning when the # __dict__ of a class contains non-string keys @@ -4826,6 +4900,8 @@ class Thing: # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. @@ -4857,7 +4933,7 @@ class A(int): with self.assertRaises(TypeError): a + a - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slot_shadows_class_variable(self): with self.assertRaises(ValueError) as cm: class X: @@ -4866,7 +4942,7 @@ class X: m = str(cm.exception) self.assertEqual("'foo' in __slots__ conflicts with class variable", m) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set_doc(self): class X: "elephant" @@ -4882,7 +4958,7 @@ class X: self.assertIn("cannot delete '__doc__' attribute of immutable type 'X'", str(cm.exception)) self.assertEqual(X.__doc__, "banana") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_qualname(self): descriptors = [str.lower, complex.real, float.real, int.__add__] types = ['method', 'member', 'getset', 'wrapper'] @@ -4915,7 +4991,7 @@ class Inside: self.assertEqual(Y.__qualname__, 'Y') self.assertEqual(Y.Inside.__qualname__, 'Y.Inside') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_qualname_dict(self): ns = {'__qualname__': 'some.name'} tp = type('Foo', (), ns) @@ -4926,7 +5002,7 @@ def test_qualname_dict(self): ns = {'__qualname__': 1} self.assertRaises(TypeError, type, 'Foo', (), ns) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle_through_dict(self): # See bug #1469629 class X(dict): @@ -4942,7 +5018,6 @@ def __init__(self): for o in gc.get_objects(): self.assertIsNot(type(o), X) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_object_new_and_init_with_parameters(self): # See issue #1683368 class OverrideNeither: @@ -4963,7 +5038,6 @@ class OverrideBoth(OverrideNew, OverrideInit): self.assertRaises(TypeError, case, 1, 2, 3) self.assertRaises(TypeError, case, 1, 2, foo=3) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclassing_does_not_duplicate_dict_descriptors(self): class Base: pass @@ -5043,7 +5117,7 @@ def __new__(cls): cls.lst = [2**i for i in range(10000)] X.descr - @support.suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_remove_subclass(self): # bpo-46417: when the last subclass of a type is deleted, # remove_subclass() clears the internal dictionary of subclasses: @@ -5061,7 +5135,7 @@ class Child(Parent): gc.collect() self.assertEqual(Parent.__subclasses__(), []) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance_method_get_behavior(self): # test case for gh-113157 @@ -5111,7 +5185,7 @@ def meth(self): pass self.C = C - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_keys(self): @@ -5125,7 +5199,7 @@ def test_iter_keys(self): '__static_attributes__', '__weakref__', 'meth']) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 5 != 7 + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 5 != 7 @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_values(self): @@ -5135,7 +5209,7 @@ def test_iter_values(self): values = list(it) self.assertEqual(len(values), 7) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_items(self): @@ -5165,8 +5239,8 @@ def test_repr(self): # We can't blindly compare with the repr of another dict as ordering # of keys and values is arbitrary and may differ. r = repr(self.C.__dict__) - self.assertTrue(r.startswith('mappingproxy('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, 'mappingproxy(') + self.assertEndsWith(r, ')') for k, v in self.C.__dict__.items(): self.assertIn('{!r}: {!r}'.format(k, v), r) @@ -5202,6 +5276,7 @@ def test_type_lookup_mro_reference(self): # Issue #14199: _PyType_Lookup() has to keep a strong reference to # the type MRO because it may be modified during the lookup, if # __bases__ is set during the lookup for example. + code = textwrap.dedent(""" class MyKey(object): def __hash__(self): return hash('mykey') @@ -5217,17 +5292,29 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - with self.assertWarnsRegex(RuntimeWarning, 'X'): - X = type('X', (Base,), {MyKey(): 5}) + X = type('X', (Base,), {MyKey(): 5}) + + bases_before = ",".join([c.__name__ for c in X.__bases__]) + print(f"before={bases_before}") + + # mykey is initially read from Base, however, the lookup will be perfomed + # again if specialization fails. The second lookup will use the new + # mro set by __eq__. + print(X.mykey) - # Note that the access below uses getattr() rather than normally - # accessing the attribute. That is done to avoid the bytecode - # specializer activating on repeated runs of the test. + bases_after = ",".join([c.__name__ for c in X.__bases__]) + print(f"after={bases_after}") - # mykey is read from Base - self.assertEqual(getattr(X, 'mykey'), 'from Base') - # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ - self.assertEqual(getattr(X, 'mykey2'), 'from Base2') + # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_ + print(f"mykey2={X.mykey2}") + """) + _, out, err = assert_python_ok("-c", code) + err = err.decode() + self.assertRegex(err, "RuntimeWarning: .*X") + out = out.decode() + self.assertRegex(out, "before=Base") + self.assertRegex(out, "after=Base2") + self.assertRegex(out, "mykey2=from Base2") class PicklingTests(unittest.TestCase): @@ -5262,7 +5349,6 @@ def _check_reduce(self, proto, obj, args=(), kwargs={}, state=None, self.assertEqual(obj.__reduce_ex__(proto), reduce_value) self.assertEqual(obj.__reduce__(), reduce_value) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_reduce(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) args = (-101, "spam") @@ -5386,7 +5472,6 @@ class C16(list): for proto in protocols: self._check_reduce(proto, obj, listitems=list(obj)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_method_lookup(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) class Picky: @@ -5452,6 +5537,7 @@ def __repr__(self): {pickle.dumps, pickle._dumps}, {pickle.loads, pickle._loads})) + @support.thread_unsafe def test_pickle_slots(self): # Tests pickling of classes with __slots__. @@ -5519,7 +5605,7 @@ class E(C): y = pickle_copier.copy(x) self._assert_is_copy(x, y) - @unittest.expectedFailure # TODO: RUSTPYTHON + @support.thread_unsafe def test_reduce_copying(self): # Tests pickling and copying new-style classes and objects. global C1 @@ -5644,7 +5730,7 @@ def __repr__(self): objcopy2 = deepcopy(objcopy) self._assert_is_copy(obj, objcopy2) - @unittest.skip('TODO: RUSTPYTHON') + @unittest.skip("TODO: RUSTPYTHON") def test_issue24097(self): # Slot name is freed inside __getattr__ and is later used. class S(str): # Not interned @@ -5785,7 +5871,7 @@ class B(A): class C(B): pass - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reent_set_bases_tp_base_cycle(self): """ type_set_bases must check for an inheritance cycle not only through @@ -5822,7 +5908,7 @@ class B2(A): with self.assertRaises(TypeError): B1.__bases__ += () - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tp_subclasses_cycle_in_update_slots(self): """ type_set_bases must check for reentrancy upon finishing its job @@ -5859,7 +5945,7 @@ class C(A): self.assertEqual(B1.__bases__, (C,)) self.assertEqual(C.__subclasses__(), [B1]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tp_subclasses_cycle_error_return_path(self): """ The same as test_tp_subclasses_cycle_in_update_slots, but tests @@ -5928,7 +6014,7 @@ def mro(cls): class A(metaclass=M): pass - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_disappearing_custom_mro(self): """ gh-92112: A custom mro() returning a result conflicting with diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 4c128f770e2..15c3a6ebeed 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -8,7 +8,7 @@ # of much interest anymore), and a few were fiddled to make the output # deterministic. -from test.support import sortdict +from test.support import sortdict # noqa: F401 import doctest import unittest @@ -34,22 +34,21 @@ def merge(self, other): if key not in self: self[key] = other[key] - test_1 = """ Here's the new type at work: >>> print(defaultdict) # show our type - + >>> print(type(defaultdict)) # its metatype >>> a = defaultdict(default=0.0) # create an instance >>> print(a) # show the instance {} >>> print(type(a)) # show its type - + >>> print(a.__class__) # show its class - + >>> print(type(a) is a.__class__) # its type is its class True >>> a[1] = 3.25 # modify the instance @@ -100,7 +99,7 @@ def merge(self, other): >>> print(sortdict(a.__dict__)) {'default': -1000, 'x1': 100, 'x2': 200} >>> -""" +""" % {'modname': __name__} class defaultdict2(dict): __slots__ = ['default'] @@ -137,10 +136,10 @@ def merge(self, other): >>> a.default = -1 >>> a[1] -1 - >>> a.x1 = 1 + >>> a.x1 = 1 # TODO: RUSTPYTHON; # doctest: +EXPECTED_FAILURE Traceback (most recent call last): File "", line 1, in ? - AttributeError: 'defaultdict2' object has no attribute 'x1' + AttributeError: 'defaultdict2' object has no attribute 'x1' and no __dict__ for setting new attributes >>> """ @@ -265,19 +264,19 @@ def merge(self, other): ... print("classmethod", cls, y) >>> C.foo(1) - classmethod 1 + classmethod 1 >>> c = C() >>> c.foo(1) - classmethod 1 + classmethod 1 >>> class D(C): ... pass >>> D.foo(1) - classmethod 1 + classmethod 1 >>> d = D() >>> d.foo(1) - classmethod 1 + classmethod 1 This prints "classmethod __main__.D 1" both times; in other words, the class passed as the first argument of foo() is the class involved in the @@ -293,18 +292,18 @@ class passed as the first argument of foo() is the class involved in the >>> E.foo(1) E.foo() called - classmethod 1 + classmethod 1 >>> e = E() >>> e.foo(1) E.foo() called - classmethod 1 + classmethod 1 In this example, the call to C.foo() from E.foo() will see class C as its first argument, not class E. This is to be expected, since the call specifies the class C. But it stresses the difference between these class methods and methods defined in metaclasses (where an upcall to a metamethod would pass the target class as an explicit first argument). -""" +""" % {'modname': __name__} test_5 = """ @@ -465,18 +464,18 @@ def m(self): called A.foo() """ -# TODO: RUSTPYTHON -__test__ = {# "tut1": test_1, - # "tut2": test_2, - # "tut3": test_3, - # "tut4": test_4, +__test__ = {"tut1": test_1, + "tut2": test_2, + "tut3": test_3, + "tut4": test_4, "tut5": test_5, "tut6": test_6, "tut7": test_7, "tut8": test_8} def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # TODO: RUSTPYTHON return tests diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 9598a7ab962..85d15830dcd 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -3,12 +3,22 @@ import gc import pickle import random +import re import string import sys import unittest import weakref from test import support -from test.support import import_helper, get_c_recursion_limit +from test.support import import_helper + + +class CustomHash: + def __init__(self, hash): + self.hash = hash + def __hash__(self): + return self.hash + def __repr__(self): + return f'' class DictTest(unittest.TestCase): @@ -265,6 +275,64 @@ def __next__(self): self.assertRaises(ValueError, {}.update, [(1, 2, 3)]) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_update_type_error(self): + with self.assertRaises(TypeError) as cm: + {}.update([object() for _ in range(3)]) + + self.assertEqual(str(cm.exception), "object is not iterable") + self.assertEqual( + cm.exception.__notes__, + ['Cannot convert dictionary update sequence element #0 to a sequence'], + ) + + def badgen(): + yield "key" + raise TypeError("oops") + yield "value" + + with self.assertRaises(TypeError) as cm: + dict([badgen() for _ in range(3)]) + + self.assertEqual(str(cm.exception), "oops") + self.assertEqual( + cm.exception.__notes__, + ['Cannot convert dictionary update sequence element #0 to a sequence'], + ) + + def test_update_shared_keys(self): + class MyClass: pass + + # Subclass str to enable us to create an object during the + # dict.update() call. + class MyStr(str): + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + # Create an object that shares the same PyDictKeysObject as + # obj.__dict__. + obj2 = MyClass() + obj2.a = "a" + obj2.b = "b" + obj2.c = "c" + return super().__eq__(other) + + obj = MyClass() + obj.a = "a" + obj.b = "b" + + x = {} + x[MyStr("a")] = MyStr("a") + + # gh-132617: this previously raised "dict mutated during update" error + x.update(obj.__dict__) + + self.assertEqual(x, { + MyStr("a"): "a", + "b": "b", + }) + def test_fromkeys(self): self.assertEqual(dict.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) d = {} @@ -369,8 +437,6 @@ def test_copy_fuzz(self): self.assertNotEqual(d, d2) self.assertEqual(len(d2), len(d) + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_maintains_tracking(self): class A: pass @@ -613,9 +679,12 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @unittest.skip("TODO: RUSTPYTHON; segfault") + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() def test_repr_deep(self): d = {} - for i in range(get_c_recursion_limit() + 1): + for i in range(support.exceeds_recursion_limit()): d = {1: d} self.assertRaises(RecursionError, repr, d) @@ -761,8 +830,8 @@ def test_dictview_mixed_set_operations(self): def test_missing(self): # Make sure dict doesn't have a __missing__ method - self.assertFalse(hasattr(dict, "__missing__")) - self.assertFalse(hasattr({}, "__missing__")) + self.assertNotHasAttr(dict, "__missing__") + self.assertNotHasAttr({}, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -883,8 +952,7 @@ def test_empty_presized_dict_in_freelist(self): 'f': None, 'g': None, 'h': None} d = {} - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_container_iterator(self): # Bug #3680: tp_traverse was not implemented for dictiter and # dictview objects. @@ -901,127 +969,6 @@ class C(object): gc.collect() self.assertIs(ref(), None, "Cycle was not collected") - def _not_tracked(self, t): - # Nested containers can take several collections to untrack - gc.collect() - gc.collect() - self.assertFalse(gc.is_tracked(t), t) - - def _tracked(self, t): - self.assertTrue(gc.is_tracked(t), t) - gc.collect() - gc.collect() - self.assertTrue(gc.is_tracked(t), t) - - def test_string_keys_can_track_values(self): - # Test that this doesn't leak. - for i in range(10): - d = {} - for j in range(10): - d[str(j)] = j - d["foo"] = d - - @support.cpython_only - def test_track_literals(self): - # Test GC-optimization of dict literals - x, y, z, w = 1.5, "a", (1, None), [] - - self._not_tracked({}) - self._not_tracked({x:(), y:x, z:1}) - self._not_tracked({1: "a", "b": 2}) - self._not_tracked({1: 2, (None, True, False, ()): int}) - self._not_tracked({1: object()}) - - # Dicts with mutable elements are always tracked, even if those - # elements are not tracked right now. - self._tracked({1: []}) - self._tracked({1: ([],)}) - self._tracked({1: {}}) - self._tracked({1: set()}) - - @support.cpython_only - def test_track_dynamic(self): - # Test GC-optimization of dynamically-created dicts - class MyObject(object): - pass - x, y, z, w, o = 1.5, "a", (1, object()), [], MyObject() - - d = dict() - self._not_tracked(d) - d[1] = "a" - self._not_tracked(d) - d[y] = 2 - self._not_tracked(d) - d[z] = 3 - self._not_tracked(d) - self._not_tracked(d.copy()) - d[4] = w - self._tracked(d) - self._tracked(d.copy()) - d[4] = None - self._not_tracked(d) - self._not_tracked(d.copy()) - - # dd isn't tracked right now, but it may mutate and therefore d - # which contains it must be tracked. - d = dict() - dd = dict() - d[1] = dd - self._not_tracked(dd) - self._tracked(d) - dd[1] = d - self._tracked(dd) - - d = dict.fromkeys([x, y, z]) - self._not_tracked(d) - dd = dict() - dd.update(d) - self._not_tracked(dd) - d = dict.fromkeys([x, y, z, o]) - self._tracked(d) - dd = dict() - dd.update(d) - self._tracked(dd) - - d = dict(x=x, y=y, z=z) - self._not_tracked(d) - d = dict(x=x, y=y, z=z, w=w) - self._tracked(d) - d = dict() - d.update(x=x, y=y, z=z) - self._not_tracked(d) - d.update(w=w) - self._tracked(d) - - d = dict([(x, y), (z, 1)]) - self._not_tracked(d) - d = dict([(x, y), (z, w)]) - self._tracked(d) - d = dict() - d.update([(x, y), (z, 1)]) - self._not_tracked(d) - d.update([(x, y), (z, w)]) - self._tracked(d) - - @support.cpython_only - def test_track_subtypes(self): - # Dict subtypes are always tracked - class MyDict(dict): - pass - self._tracked(MyDict()) - - @support.cpython_only - def test_track_lazy_instance_dicts(self): - class C: - pass - o = C() - d = o.__dict__ - self._not_tracked(d) - o.untracked = 42 - self._not_tracked(d) - o.tracked = [] - self._tracked(d) - def make_shared_key_dict(self, n): class C: pass @@ -1312,16 +1259,14 @@ def __eq__(self, o): d = {X(): 0, 1: 1} self.assertRaises(RuntimeError, d.update, other) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, dict) support.check_free_after_iterating(self, lambda d: iter(d.keys()), dict) support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_equal_operator_modifying_operand(self): # test fix for seg fault reported in bpo-27945 part 3. class X(): @@ -1629,6 +1574,118 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unhashable_key(self): + d = {'a': 1} + key = [1, 2, 3] + + def check_unhashable_key(): + msg = "cannot use 'list' as a dict key (unhashable type: 'list')" + return self.assertRaisesRegex(TypeError, re.escape(msg)) + + with check_unhashable_key(): + key in d + with check_unhashable_key(): + d[key] + with check_unhashable_key(): + d[key] = 2 + with check_unhashable_key(): + d.setdefault(key, 2) + with check_unhashable_key(): + d.pop(key) + with check_unhashable_key(): + d.get(key) + + # Only TypeError exception is overriden, + # other exceptions are left unchanged. + class HashError: + def __hash__(self): + raise KeyError('error') + + key2 = HashError() + with self.assertRaises(KeyError): + key2 in d + with self.assertRaises(KeyError): + d[key2] + with self.assertRaises(KeyError): + d[key2] = 2 + with self.assertRaises(KeyError): + d.setdefault(key2, 2) + with self.assertRaises(KeyError): + d.pop(key2) + with self.assertRaises(KeyError): + d.get(key2) + + def test_clear_at_lookup(self): + # gh-140551 dict crash if clear is called at lookup stage + class X: + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal d + d.clear() + + d = {} + for _ in range(10): + d[X()] = None + + self.assertEqual(len(d), 1) + + d = {} + for _ in range(10): + d.setdefault(X(), None) + + self.assertEqual(len(d), 1) + + def test_split_table_update_with_str_subclass(self): + # gh-142218: inserting into a split table dictionary with a non str + # key that matches an existing key. + class MyStr(str): pass + class MyClass: pass + obj = MyClass() + obj.attr = 1 + obj.__dict__[MyStr('attr')] = 2 + self.assertEqual(obj.attr, 2) + + def test_split_table_insert_with_str_subclass(self): + # gh-143189: inserting into split table dictionary with a non str + # key that matches an existing key in the shared table but not in + # the dict yet. + + class MyStr(str): pass + class MyClass: pass + + obj = MyClass() + obj.attr1 = 1 + + obj2 = MyClass() + d = obj2.__dict__ + d[MyStr("attr1")] = 2 + self.assertIsInstance(list(d)[0], MyStr) + + def test_hash_collision_remove_add(self): + self.maxDiff = None + # There should be enough space, so all elements with unique hash + # will be placed in corresponding cells without collision. + n = 64 + items = [(CustomHash(h), h) for h in range(n)] + # Keys with hash collision. + a = CustomHash(n) + b = CustomHash(n) + items += [(a, 'a'), (b, 'b')] + d = dict(items) + self.assertEqual(len(d), len(items), d) + del d[a] + # "a" has been replaced with a dummy. + del items[n] + self.assertEqual(len(d), len(items), d) + self.assertEqual(d, dict(items)) + d[b] = 'c' + # "b" should not replace the dummy. + items[n] = (b, 'c') + self.assertEqual(len(d), len(items), d) + self.assertEqual(d, dict(items)) + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 233faa90c72..fc7ebb0f5a6 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -1,5 +1,8 @@ +import traceback import unittest +from test.support import BrokenIter + # For scope testing. g = "Global variable" @@ -72,8 +75,7 @@ def test_local_visibility(self): self.assertEqual(actual, expected) self.assertEqual(v, "Local variable") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_assignment(self): with self.assertRaisesRegex(SyntaxError, "cannot assign"): compile("{x: y for y, x in ((1, 2), (3, 4))} = 5", "", @@ -129,6 +131,41 @@ def test_star_expression(self): self.assertEqual({i: i*i for i in [*range(4)]}, expected) self.assertEqual({i: i*i for i in (*range(4),)}, expected) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + def init_raises(): + try: + {x:x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x:x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x:x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 667cccd6cd7..2809417ffa1 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import get_c_recursion_limit +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit class DictSetTest(unittest.TestCase): @@ -277,11 +277,12 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; segfault") + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): d = {} - for i in range(get_c_recursion_limit()//2 + 100): + for i in range(exceeds_recursion_limit()): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) @@ -291,8 +292,7 @@ def test_copy(self): self.assertRaises(TypeError, copy.copy, d.values()) self.assertRaises(TypeError, copy.copy, d.items()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compare_error(self): class Exc(Exception): pass diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 943d7a659b1..36be5c99c3f 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -282,6 +282,26 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) +class TestDiffer(unittest.TestCase): + def test_close_matches_aligned(self): + # Of the 4 closely matching pairs, we want 1 to match with 3, + # and 2 with 4, to align with a "top to bottom" mental model. + a = ["cat\n", "dog\n", "close match 1\n", "close match 2\n"] + b = ["close match 3\n", "close match 4\n", "kitten\n", "puppy\n"] + m = difflib.Differ().compare(a, b) + self.assertEqual(list(m), + ['- cat\n', + '- dog\n', + '- close match 1\n', + '? ^\n', + '+ close match 3\n', + '? ^\n', + '- close match 2\n', + '? ^\n', + '+ close match 4\n', + '? ^\n', + '+ kitten\n', + '+ puppy\n']) def test_one_insert(self): m = difflib.Differ().compare('b' * 2, 'a' + 'b' * 2) @@ -294,7 +314,7 @@ def test_one_delete(self): class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -306,7 +326,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -446,6 +466,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -494,10 +536,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Lib/test/test_difflib_expect.html b/Lib/test/test_difflib_expect.html index 12091206a28..9f33a9e9c9c 100644 --- a/Lib/test/test_difflib_expect.html +++ b/Lib/test/test_difflib_expect.html @@ -9,13 +9,22 @@ content="text/html; charset=utf-8" /> diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8bbba86a464..ec851f939c9 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,59 +1,2649 @@ -import subprocess +# Minimal tests for dis module + +import ast +import contextlib +import dis +import functools +import io +import itertools +import opcode +import re import sys +import tempfile +import textwrap +import types import unittest +from test.support import (captured_stdout, requires_debug_ranges, + requires_specialization, cpython_only, + os_helper, import_helper, reset_code) +from test.support.bytecode_helper import BytecodeTestCase + + +CACHE = dis.opmap["CACHE"] -# This only tests that it prints something in order -# to avoid changing this test if the bytecode changes +def get_tb(): + def _error(): + try: + 1 / 0 + except Exception as e: + tb = e.__traceback__ + return tb -# These tests start a new process instead of redirecting stdout because -# stdout is being written to by rust code, which currently can't be -# redirected by reassigning sys.stdout + tb = _error() + while tb.tb_next: + tb = tb.tb_next + return tb +TRACEBACK_CODE = get_tb().tb_frame.f_code + +class _C: + def __init__(self, x): + self.x = x == 1 + + @staticmethod + def sm(x): + x = x == 1 -class TestDis(unittest.TestCase): @classmethod - def setUpClass(cls): - cls.setup = """ -import dis -def tested_func(): pass + def cm(cls, x): + cls.x = x == 1 + +dis_c_instance_method = """\ +%3d RESUME 0 + +%3d LOAD_FAST_BORROW 1 (x) + LOAD_SMALL_INT 1 + COMPARE_OP 72 (==) + LOAD_FAST_BORROW 0 (self) + STORE_ATTR 0 (x) + LOAD_CONST 1 (None) + RETURN_VALUE +""" % (_C.__init__.__code__.co_firstlineno, _C.__init__.__code__.co_firstlineno + 1,) + +dis_c_instance_method_bytes = """\ + RESUME 0 + LOAD_FAST_BORROW 1 + LOAD_SMALL_INT 1 + COMPARE_OP 72 (==) + LOAD_FAST_BORROW 0 + STORE_ATTR 0 + LOAD_CONST 1 + RETURN_VALUE +""" + +dis_c_class_method = """\ +%3d RESUME 0 + +%3d LOAD_FAST_BORROW 1 (x) + LOAD_SMALL_INT 1 + COMPARE_OP 72 (==) + LOAD_FAST_BORROW 0 (cls) + STORE_ATTR 0 (x) + LOAD_CONST 1 (None) + RETURN_VALUE +""" % (_C.cm.__code__.co_firstlineno, _C.cm.__code__.co_firstlineno + 2,) + +dis_c_static_method = """\ +%3d RESUME 0 + +%3d LOAD_FAST_BORROW 0 (x) + LOAD_SMALL_INT 1 + COMPARE_OP 72 (==) + STORE_FAST 0 (x) + LOAD_CONST 1 (None) + RETURN_VALUE +""" % (_C.sm.__code__.co_firstlineno, _C.sm.__code__.co_firstlineno + 2,) + +# Class disassembling info has an extra newline at end. +dis_c = """\ +Disassembly of %s: +%s +Disassembly of %s: +%s +Disassembly of %s: +%s +""" % (_C.__init__.__name__, dis_c_instance_method, + _C.cm.__name__, dis_c_class_method, + _C.sm.__name__, dis_c_static_method) + +def _f(a): + print(a) + return 1 + +dis_f = """\ +%3d RESUME 0 + +%3d LOAD_GLOBAL 1 (print + NULL) + LOAD_FAST_BORROW 0 (a) + CALL 1 + POP_TOP + +%3d LOAD_SMALL_INT 1 + RETURN_VALUE +""" % (_f.__code__.co_firstlineno, + _f.__code__.co_firstlineno + 1, + _f.__code__.co_firstlineno + 2) + +dis_f_with_offsets = """\ +%3d 0 RESUME 0 + +%3d 2 LOAD_GLOBAL 1 (print + NULL) + 12 LOAD_FAST_BORROW 0 (a) + 14 CALL 1 + 22 POP_TOP + +%3d 24 LOAD_SMALL_INT 1 + 26 RETURN_VALUE +""" % (_f.__code__.co_firstlineno, + _f.__code__.co_firstlineno + 1, + _f.__code__.co_firstlineno + 2) + +dis_f_with_positions_format = f"""\ +%-14s RESUME 0 + +%-14s LOAD_GLOBAL 1 (print + NULL) +%-14s LOAD_FAST_BORROW 0 (a) +%-14s CALL 1 +%-14s POP_TOP + +%-14s LOAD_SMALL_INT 1 +%-14s RETURN_VALUE +""" + +dis_f_co_code = """\ + RESUME 0 + LOAD_GLOBAL 1 + LOAD_FAST_BORROW 0 + CALL 1 + POP_TOP + LOAD_SMALL_INT 1 + RETURN_VALUE +""" + +def bug708901(): + for res in range(1, + 10): + pass + +dis_bug708901 = """\ +%3d RESUME 0 + +%3d LOAD_GLOBAL 1 (range + NULL) + LOAD_SMALL_INT 1 + +%3d LOAD_SMALL_INT 10 + +%3d CALL 2 + GET_ITER + L1: FOR_ITER 3 (to L2) + STORE_FAST 0 (res) + +%3d JUMP_BACKWARD 5 (to L1) + +%3d L2: END_FOR + POP_ITER + LOAD_CONST 1 (None) + RETURN_VALUE +""" % (bug708901.__code__.co_firstlineno, + bug708901.__code__.co_firstlineno + 1, + bug708901.__code__.co_firstlineno + 2, + bug708901.__code__.co_firstlineno + 1, + bug708901.__code__.co_firstlineno + 3, + bug708901.__code__.co_firstlineno + 1) + + +def bug1333982(x=[]): + assert 0, ((s for s in x) + + 1) + pass + +dis_bug1333982 = """\ +%3d RESUME 0 + +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + MAKE_FUNCTION + LOAD_FAST_BORROW 0 (x) + GET_ITER + CALL 0 + +%3d LOAD_SMALL_INT 1 + +%3d BINARY_OP 0 (+) + CALL 0 + RAISE_VARARGS 1 +""" % (bug1333982.__code__.co_firstlineno, + bug1333982.__code__.co_firstlineno + 1, + __file__, + bug1333982.__code__.co_firstlineno + 1, + bug1333982.__code__.co_firstlineno + 2, + bug1333982.__code__.co_firstlineno + 1) + + +def bug42562(): + pass + + +# Set line number for 'pass' to None +bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\xf8') + + +dis_bug42562 = """\ + RESUME 0 + LOAD_CONST 0 (None) + RETURN_VALUE +""" + +# Extended arg followed by NOP +code_bug_45757 = bytes([ + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['NOP'], 0xFF, + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['LOAD_CONST'], 0x29, + opcode.opmap['RETURN_VALUE'], 0x00, + ]) + +dis_bug_45757 = """\ + EXTENDED_ARG 1 + NOP + EXTENDED_ARG 1 + LOAD_CONST 297 + RETURN_VALUE +""" + +# [255, 255, 255, 252] is -4 in a 4 byte signed integer +bug46724 = bytes([ + opcode.EXTENDED_ARG, 255, + opcode.EXTENDED_ARG, 255, + opcode.EXTENDED_ARG, 255, + opcode.opmap['JUMP_FORWARD'], 252, +]) + + +dis_bug46724 = """\ + L1: EXTENDED_ARG 255 + EXTENDED_ARG 65535 + EXTENDED_ARG 16777215 + JUMP_FORWARD -4 (to L1) +""" + +def func_w_kwargs(a, b, **c): + pass + +def wrap_func_w_kwargs(): + func_w_kwargs(1, 2, c=5) + +dis_kw_names = """\ +%3d RESUME 0 + +%3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) + LOAD_SMALL_INT 1 + LOAD_SMALL_INT 2 + LOAD_SMALL_INT 5 + LOAD_CONST 1 (('c',)) + CALL_KW 3 + POP_TOP + LOAD_CONST 2 (None) + RETURN_VALUE +""" % (wrap_func_w_kwargs.__code__.co_firstlineno, + wrap_func_w_kwargs.__code__.co_firstlineno + 1) + +dis_intrinsic_1_2 = """\ + 0 RESUME 0 + + 1 LOAD_SMALL_INT 0 + LOAD_CONST 1 (('*',)) + IMPORT_NAME 0 (math) + CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) + POP_TOP + LOAD_CONST 2 (None) + RETURN_VALUE +""" + +dis_intrinsic_1_5 = """\ + 0 RESUME 0 + + 1 LOAD_NAME 0 (a) + CALL_INTRINSIC_1 5 (INTRINSIC_UNARY_POSITIVE) + RETURN_VALUE +""" + +dis_intrinsic_1_6 = """\ + 0 RESUME 0 + + 1 BUILD_LIST 0 + LOAD_NAME 0 (a) + LIST_EXTEND 1 + CALL_INTRINSIC_1 6 (INTRINSIC_LIST_TO_TUPLE) + RETURN_VALUE +""" + +_BIG_LINENO_FORMAT = """\ + 1 RESUME 0 + +%3d LOAD_GLOBAL 0 (spam) + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE +""" + +_BIG_LINENO_FORMAT2 = """\ + 1 RESUME 0 + +%4d LOAD_GLOBAL 0 (spam) + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE +""" + +dis_module_expected_results = """\ +Disassembly of f: + 4 RESUME 0 + LOAD_CONST 0 (None) + RETURN_VALUE + +Disassembly of g: + 5 RESUME 0 + LOAD_CONST 0 (None) + RETURN_VALUE + +""" + +expr_str = "x + 1" + +dis_expr_str = """\ + 0 RESUME 0 + + 1 LOAD_NAME 0 (x) + LOAD_SMALL_INT 1 + BINARY_OP 0 (+) + RETURN_VALUE +""" + +simple_stmt_str = "x = x + 1" + +dis_simple_stmt_str = """\ + 0 RESUME 0 + + 1 LOAD_NAME 0 (x) + LOAD_SMALL_INT 1 + BINARY_OP 0 (+) + STORE_NAME 0 (x) + LOAD_CONST 1 (None) + RETURN_VALUE +""" + +annot_stmt_str = """\ + +x: int = 1 +y: fun(1) +lst[fun(0)]: int = 1 +""" +# leading newline is for a reason (tests lineno) + +dis_annot_stmt_str = """\ + -- MAKE_CELL 0 (__conditional_annotations__) + + 0 RESUME 0 + + 2 LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + STORE_NAME 4 (__annotate__) + BUILD_SET 0 + STORE_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 1 + STORE_NAME 1 (x) + LOAD_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 0 + SET_ADD 1 + POP_TOP + + 3 LOAD_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 1 + SET_ADD 1 + POP_TOP + + 4 LOAD_SMALL_INT 1 + LOAD_NAME 2 (lst) + LOAD_NAME 3 (fun) + PUSH_NULL + LOAD_SMALL_INT 0 + CALL 1 + STORE_SUBSCR + LOAD_CONST 2 (None) + RETURN_VALUE +""" + +fn_with_annotate_str = """ +def foo(a: int, b: str) -> str: + return a * b +""" + +dis_fn_with_annotate_str = """\ + 0 RESUME 0 + + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 16 (annotate) + STORE_NAME 0 (foo) + LOAD_CONST 2 (None) + RETURN_VALUE +""" + +compound_stmt_str = """\ +x = 0 +while 1: + x += 1""" +# Trailing newline has been deliberately omitted + +dis_compound_stmt_str = """\ + 0 RESUME 0 + + 1 LOAD_SMALL_INT 0 + STORE_NAME 0 (x) + + 2 L1: NOP + + 3 LOAD_NAME 0 (x) + LOAD_SMALL_INT 1 + BINARY_OP 13 (+=) + STORE_NAME 0 (x) + JUMP_BACKWARD 12 (to L1) """ - cls.command = (sys.executable, "-c") +dis_traceback = """\ +%4d RESUME 0 + +%4d NOP + +%4d L1: LOAD_SMALL_INT 1 + LOAD_SMALL_INT 0 + --> BINARY_OP 11 (/) + POP_TOP + +%4d L2: LOAD_FAST_CHECK 1 (tb) + RETURN_VALUE + + -- L3: PUSH_EXC_INFO + +%4d LOAD_GLOBAL 0 (Exception) + CHECK_EXC_MATCH + POP_JUMP_IF_FALSE 24 (to L7) + NOT_TAKEN + STORE_FAST 0 (e) + +%4d L4: LOAD_FAST 0 (e) + LOAD_ATTR 2 (__traceback__) + STORE_FAST 1 (tb) + L5: POP_EXCEPT + LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + +%4d LOAD_FAST 1 (tb) + RETURN_VALUE + + -- L6: LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + RERAISE 1 + +%4d L7: RERAISE 0 + + -- L8: COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: + L1 to L2 -> L3 [0] + L3 to L4 -> L8 [1] lasti + L4 to L5 -> L6 [1] lasti + L6 to L8 -> L8 [1] lasti +""" % (TRACEBACK_CODE.co_firstlineno, + TRACEBACK_CODE.co_firstlineno + 1, + TRACEBACK_CODE.co_firstlineno + 2, + TRACEBACK_CODE.co_firstlineno + 5, + TRACEBACK_CODE.co_firstlineno + 3, + TRACEBACK_CODE.co_firstlineno + 4, + TRACEBACK_CODE.co_firstlineno + 5, + TRACEBACK_CODE.co_firstlineno + 3) + +def _fstring(a, b, c, d): + return f'{a} {b:4} {c!r} {d!r:4}' + +dis_fstring = """\ +%3d RESUME 0 + +%3d LOAD_FAST_BORROW 0 (a) + FORMAT_SIMPLE + LOAD_CONST 0 (' ') + LOAD_FAST_BORROW 1 (b) + LOAD_CONST 1 ('4') + FORMAT_WITH_SPEC + LOAD_CONST 0 (' ') + LOAD_FAST_BORROW 2 (c) + CONVERT_VALUE 2 (repr) + FORMAT_SIMPLE + LOAD_CONST 0 (' ') + LOAD_FAST_BORROW 3 (d) + CONVERT_VALUE 2 (repr) + LOAD_CONST 1 ('4') + FORMAT_WITH_SPEC + BUILD_STRING 7 + RETURN_VALUE +""" % (_fstring.__code__.co_firstlineno, _fstring.__code__.co_firstlineno + 1) + +def _with(c): + with c: + x = 1 + y = 2 + +dis_with = """\ +%4d RESUME 0 + +%4d LOAD_FAST_BORROW 0 (c) + COPY 1 + LOAD_SPECIAL 1 (__exit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 0 (__enter__) + CALL 0 + L1: POP_TOP + +%4d LOAD_SMALL_INT 1 + STORE_FAST 1 (x) + +%4d L2: LOAD_CONST 1 (None) + LOAD_CONST 1 (None) + LOAD_CONST 1 (None) + CALL 3 + POP_TOP + +%4d LOAD_SMALL_INT 2 + STORE_FAST 2 (y) + LOAD_CONST 1 (None) + RETURN_VALUE + +%4d L3: PUSH_EXC_INFO + WITH_EXCEPT_START + TO_BOOL + POP_JUMP_IF_TRUE 2 (to L4) + NOT_TAKEN + RERAISE 2 + L4: POP_TOP + L5: POP_EXCEPT + POP_TOP + POP_TOP + POP_TOP + +%4d LOAD_SMALL_INT 2 + STORE_FAST 2 (y) + LOAD_CONST 1 (None) + RETURN_VALUE + + -- L6: COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: + L1 to L2 -> L3 [2] lasti + L3 to L5 -> L6 [4] lasti +""" % (_with.__code__.co_firstlineno, + _with.__code__.co_firstlineno + 1, + _with.__code__.co_firstlineno + 2, + _with.__code__.co_firstlineno + 1, + _with.__code__.co_firstlineno + 3, + _with.__code__.co_firstlineno + 1, + _with.__code__.co_firstlineno + 3, + ) + +async def _asyncwith(c): + async with c: + x = 1 + y = 2 + +dis_asyncwith = """\ +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + +%4d LOAD_FAST 0 (c) + COPY 1 + LOAD_SPECIAL 3 (__aexit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 2 (__aenter__) + CALL 0 + GET_AWAITABLE 1 + LOAD_CONST 0 (None) + L2: SEND 3 (to L5) + L3: YIELD_VALUE 1 + L4: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L2) + L5: END_SEND + L6: POP_TOP + +%4d LOAD_SMALL_INT 1 + STORE_FAST 1 (x) + +%4d L7: LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 3 + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + L8: SEND 3 (to L11) + L9: YIELD_VALUE 1 + L10: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L8) + L11: END_SEND + POP_TOP + +%4d LOAD_SMALL_INT 2 + STORE_FAST 2 (y) + LOAD_CONST 0 (None) + RETURN_VALUE + +%4d L12: CLEANUP_THROW + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) + L14: CLEANUP_THROW + L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) + L16: PUSH_EXC_INFO + WITH_EXCEPT_START + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + L17: SEND 4 (to L21) + L18: YIELD_VALUE 1 + L19: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L17) + L20: CLEANUP_THROW + L21: END_SEND + TO_BOOL + POP_JUMP_IF_TRUE 2 (to L24) + L22: NOT_TAKEN + L23: RERAISE 2 + L24: POP_TOP + L25: POP_EXCEPT + POP_TOP + POP_TOP + POP_TOP + +%4d LOAD_SMALL_INT 2 + STORE_FAST 2 (y) + LOAD_CONST 0 (None) + RETURN_VALUE + + -- L26: COPY 3 + POP_EXCEPT + RERAISE 1 + L27: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 +ExceptionTable: + L1 to L3 -> L27 [0] lasti + L3 to L4 -> L12 [4] + L4 to L6 -> L27 [0] lasti + L6 to L7 -> L16 [2] lasti + L7 to L9 -> L27 [0] lasti + L9 to L10 -> L14 [2] + L10 to L13 -> L27 [0] lasti + L14 to L15 -> L27 [0] lasti + L16 to L18 -> L26 [4] lasti + L18 to L19 -> L20 [7] + L19 to L22 -> L26 [4] lasti + L23 to L25 -> L26 [4] lasti + L25 to L27 -> L27 [0] lasti +""" % (_asyncwith.__code__.co_firstlineno, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 2, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 3, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 3, + ) + + +def _tryfinally(a, b): + try: + return a + finally: + b() + +def _tryfinallyconst(b): + try: + return 1 + finally: + b() + +dis_tryfinally = """\ +%4d RESUME 0 + +%4d NOP + +%4d L1: LOAD_FAST_BORROW 0 (a) + +%4d L2: LOAD_FAST_BORROW 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RETURN_VALUE + + -- L3: PUSH_EXC_INFO + +%4d LOAD_FAST 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 + + -- L4: COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: + L1 to L2 -> L3 [0] + L3 to L4 -> L4 [1] lasti +""" % (_tryfinally.__code__.co_firstlineno, + _tryfinally.__code__.co_firstlineno + 1, + _tryfinally.__code__.co_firstlineno + 2, + _tryfinally.__code__.co_firstlineno + 4, + _tryfinally.__code__.co_firstlineno + 4, + ) + +dis_tryfinallyconst = """\ +%4d RESUME 0 + +%4d NOP + +%4d NOP + +%4d LOAD_FAST_BORROW 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + LOAD_SMALL_INT 1 + RETURN_VALUE + + -- L1: PUSH_EXC_INFO + +%4d LOAD_FAST 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 + + -- L2: COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: + L1 to L2 -> L2 [1] lasti +""" % (_tryfinallyconst.__code__.co_firstlineno, + _tryfinallyconst.__code__.co_firstlineno + 1, + _tryfinallyconst.__code__.co_firstlineno + 2, + _tryfinallyconst.__code__.co_firstlineno + 4, + _tryfinallyconst.__code__.co_firstlineno + 4, + ) + +def _g(x): + yield x + +async def _ag(x): + yield x + +async def _co(x): + async for item in _ag(x): + pass + +def _h(y): + def foo(x): + '''funcdoc''' + return list(x + z for z in y) + return foo + +dis_nested_0 = """\ + -- MAKE_CELL 0 (y) + +%4d RESUME 0 + +%4d LOAD_FAST_BORROW 0 (y) + BUILD_TUPLE 1 + LOAD_CONST 0 () + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + STORE_FAST 1 (foo) + +%4d LOAD_FAST_BORROW 1 (foo) + RETURN_VALUE +""" % (_h.__code__.co_firstlineno, + _h.__code__.co_firstlineno + 1, + __file__, + _h.__code__.co_firstlineno + 1, + _h.__code__.co_firstlineno + 4, +) + +dis_nested_1 = """%s +Disassembly of : + -- COPY_FREE_VARS 1 + MAKE_CELL 0 (x) + +%4d RESUME 0 + +%4d LOAD_GLOBAL 1 (list + NULL) + LOAD_FAST_BORROW 0 (x) + BUILD_TUPLE 1 + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + LOAD_DEREF 1 (y) + GET_ITER + CALL 0 + CALL 1 + RETURN_VALUE +""" % (dis_nested_0, + __file__, + _h.__code__.co_firstlineno + 1, + _h.__code__.co_firstlineno + 1, + _h.__code__.co_firstlineno + 3, + __file__, + _h.__code__.co_firstlineno + 3, +) + +dis_nested_2 = """%s +Disassembly of at 0x..., file "%s", line %d>: + -- COPY_FREE_VARS 1 + +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + LOAD_FAST 0 (.0) + L2: FOR_ITER 14 (to L3) + STORE_FAST 1 (z) + LOAD_DEREF 2 (x) + LOAD_FAST_BORROW 1 (z) + BINARY_OP 0 (+) + YIELD_VALUE 0 + RESUME 5 + POP_TOP + JUMP_BACKWARD 16 (to L2) + L3: END_FOR + POP_ITER + LOAD_CONST 0 (None) + RETURN_VALUE + + -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 +ExceptionTable: + L1 to L4 -> L4 [0] lasti +""" % (dis_nested_1, + __file__, + _h.__code__.co_firstlineno + 3, + _h.__code__.co_firstlineno + 3, +) + +def load_test(x, y=0): + a, b = x, y + return a, b + +dis_load_test_quickened_code = """\ +%3d RESUME_CHECK 0 + +%3d LOAD_FAST_LOAD_FAST 1 (x, y) + STORE_FAST_STORE_FAST 50 (b, a) + +%3d LOAD_FAST_BORROW_LOAD_FAST_BORROW 35 (a, b) + BUILD_TUPLE 2 + RETURN_VALUE +""" % (load_test.__code__.co_firstlineno, + load_test.__code__.co_firstlineno + 1, + load_test.__code__.co_firstlineno + 2) + +def loop_test(): + for i in [1, 2, 3] * 3: + load_test(i) + +dis_loop_test_quickened_code = """\ +%3d RESUME_CHECK 0 + +%3d BUILD_LIST 0 + LOAD_CONST_MORTAL 2 ((1, 2, 3)) + LIST_EXTEND 1 + LOAD_SMALL_INT 3 + BINARY_OP 5 (*) + GET_ITER + L1: FOR_ITER_LIST 14 (to L2) + STORE_FAST 0 (i) + +%3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) + LOAD_FAST_BORROW 0 (i) + CALL_PY_GENERAL 1 + POP_TOP + JUMP_BACKWARD_{: <6} 16 (to L1) + +%3d L2: END_FOR + POP_ITER + LOAD_CONST_IMMORTAL 1 (None) + RETURN_VALUE +""" % (loop_test.__code__.co_firstlineno, + loop_test.__code__.co_firstlineno + 1, + loop_test.__code__.co_firstlineno + 2, + loop_test.__code__.co_firstlineno + 1,) + +def extended_arg_quick(): + *_, _ = ... + +dis_extended_arg_quick_code = """\ +%3d RESUME 0 + +%3d LOAD_CONST 0 (Ellipsis) + EXTENDED_ARG 1 + UNPACK_EX 256 + POP_TOP + STORE_FAST 0 (_) + LOAD_CONST 1 (None) + RETURN_VALUE +"""% (extended_arg_quick.__code__.co_firstlineno, + extended_arg_quick.__code__.co_firstlineno + 1,) + +class DisTestBase(unittest.TestCase): + "Common utilities for DisTests and TestDisTraceback" + + def strip_addresses(self, text): + return re.sub(r'\b0x[0-9A-Fa-f]+\b', '0x...', text) + + def assert_exception_table_increasing(self, lines): + prev_start, prev_end = -1, -1 + count = 0 + for line in lines: + m = re.match(r' L(\d+) to L(\d+) -> L\d+ \[\d+\]', line) + start, end = [int(g) for g in m.groups()] + self.assertGreaterEqual(end, start) + self.assertGreaterEqual(start, prev_end) + prev_start, prev_end = start, end + count += 1 + return count + + def do_disassembly_compare(self, got, expected): + if got != expected: + got = self.strip_addresses(got) + self.assertEqual(got, expected) + + +class DisTests(DisTestBase): + + maxDiff = None + + def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): + # We want to test the default printing behaviour, not the file arg + output = io.StringIO() + with contextlib.redirect_stdout(output): + if wrapper: + dis.dis(func, **kwargs) + else: + dis.disassemble(func, lasti, **kwargs) + return output.getvalue() + + def get_disassemble_as_string(self, func, lasti=-1): + return self.get_disassembly(func, lasti, False) + + def do_disassembly_test(self, func, expected, **kwargs): + self.maxDiff = None + got = self.get_disassembly(func, depth=0, **kwargs) + self.do_disassembly_compare(got, expected) + # Add checks for dis.disco + if hasattr(func, '__code__'): + got_disco = io.StringIO() + with contextlib.redirect_stdout(got_disco): + dis.disco(func.__code__, **kwargs) + self.do_disassembly_compare(got_disco.getvalue(), expected) + + def test_opmap(self): + self.assertEqual(dis.opmap["CACHE"], 0) + self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst) + self.assertIn(dis.opmap["STORE_NAME"], dis.hasname) + + def test_opname(self): + self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST") + + def test_boundaries(self): + self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 29 not less than or equal to 20 + def test_widths(self): + long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', + 'INSTRUMENTED_CALL_FUNCTION_EX', + 'ANNOTATIONS_PLACEHOLDER']) + for op, opname in enumerate(dis.opname): + if opname in long_opcodes or opname.startswith("INSTRUMENTED"): + continue + if opname in opcode._specialized_opmap: + continue + with self.subTest(opname=opname): + width = dis._OPNAME_WIDTH + if op in dis.hasarg: + width += 1 + dis._OPARG_WIDTH + self.assertLessEqual(len(opname), width) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dis(self): - test_code = f""" -{self.setup} -dis.dis(tested_func) -dis.dis("x = 2; print(x)") -""" - - result = subprocess.run( - self.command + (test_code,), capture_output=True - ) - self.assertNotEqual("", result.stdout.decode()) - self.assertEqual("", result.stderr.decode()) - - def test_disassemble(self): - test_code = f""" -{self.setup} -dis.disassemble(tested_func) -""" - result = subprocess.run( - self.command + (test_code,), capture_output=True - ) - # In CPython this would raise an AttributeError, not a - # TypeError because dis is implemented in python in CPython and - # as such the type mismatch wouldn't be caught immeadiately - self.assertIn("TypeError", result.stderr.decode()) - - test_code = f""" -{self.setup} -dis.disassemble(tested_func.__code__) -""" - result = subprocess.run( - self.command + (test_code,), capture_output=True - ) - self.assertNotEqual("", result.stdout.decode()) - self.assertEqual("", result.stderr.decode()) + self.do_disassembly_test(_f, dis_f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dis_with_offsets(self): + self.do_disassembly_test(_f, dis_f_with_offsets, show_offsets=True) + + @requires_debug_ranges() + def test_dis_with_all_positions(self): + def format_instr_positions(instr): + values = tuple('?' if p is None else p for p in instr.positions) + return '%s:%s-%s:%s' % (values[0], values[2], values[1], values[3]) + + instrs = list(dis.get_instructions(_f)) + for instr in instrs: + with self.subTest(instr=instr): + self.assertTrue(all(p is not None for p in instr.positions)) + positions = tuple(map(format_instr_positions, instrs)) + expected = dis_f_with_positions_format % positions + self.do_disassembly_test(_f, expected, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_some_positions(self): + code = ("def f():\n" + " try: pass\n" + " finally:pass") + f = compile(ast.parse(code), "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:3-3:15 NOP', + '', + '3:11-3:15 LOAD_CONST 0 (None)', + '3:11-3:15 RETURN_VALUE', + '', + ' -- L1: PUSH_EXC_INFO', + '', + '3:11-3:15 RERAISE 0', + '', + ' -- L2: COPY 3', + ' -- POP_EXCEPT', + ' -- RERAISE 1', + 'ExceptionTable:', + ' L1 to L2 -> L2 [1] lasti', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_linenos_but_no_columns(self): + code = "def f():\n\tx = 1" + tree = ast.parse(code) + func = tree.body[0] + ass_x = func.body[0].targets[0] + # remove columns information but keep line information + ass_x.col_offset = ass_x.end_col_offset = -1 + f = compile(tree, "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:5-2:6 LOAD_SMALL_INT 1', + '2:?-2:? STORE_FAST 0 (x)', + '2:?-2:? LOAD_CONST 1 (None)', + '2:?-2:? RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + def test_dis_with_no_positions(self): + def f(): + pass + + f.__code__ = f.__code__.replace(co_linetable=b'') + expect = '\n'.join([ + ' RESUME 0', + ' LOAD_CONST 0 (None)', + ' RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bug_708901(self): + self.do_disassembly_test(bug708901, dis_bug708901) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bug_1333982(self): + # This one is checking bytecodes generated for an `assert` statement, + # so fails if the tests are run with -O. Skip this test then. + if not __debug__: + self.skipTest('need asserts, run without -O') + + self.do_disassembly_test(bug1333982, dis_bug1333982) + + def test_bug_42562(self): + self.do_disassembly_test(bug42562, dis_bug42562) + + def test_bug_45757(self): + # Extended arg followed by NOP + self.do_disassembly_test(code_bug_45757, dis_bug_45757) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bug_46724(self): + # Test that negative operargs are handled properly + self.do_disassembly_test(bug46724, dis_bug46724) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_kw_names(self): + # Test that value is displayed for keyword argument names: + self.do_disassembly_test(wrap_func_w_kwargs, dis_kw_names) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_intrinsic_1(self): + # Test that argrepr is displayed for CALL_INTRINSIC_1 + self.do_disassembly_test("from math import *", dis_intrinsic_1_2) + self.do_disassembly_test("+a", dis_intrinsic_1_5) + self.do_disassembly_test("(*a,)", dis_intrinsic_1_6) + + def test_intrinsic_2(self): + self.assertIn("CALL_INTRINSIC_2 1 (INTRINSIC_PREP_RERAISE_STAR)", + self.get_disassembly("try: pass\nexcept* Exception: x")) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_big_linenos(self): + def func(count): + namespace = {} + func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"]) + exec(func, namespace) + return namespace['foo'] + + # Test all small ranges + for i in range(1, 300): + expected = _BIG_LINENO_FORMAT % (i + 2) + self.do_disassembly_test(func(i), expected) + + # Test some larger ranges too + for i in range(300, 1000, 10): + expected = _BIG_LINENO_FORMAT % (i + 2) + self.do_disassembly_test(func(i), expected) + + for i in range(1000, 5000, 10): + expected = _BIG_LINENO_FORMAT2 % (i + 2) + self.do_disassembly_test(func(i), expected) + + from test import dis_module + self.do_disassembly_test(dis_module, dis_module_expected_results) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_str(self): + self.do_disassembly_test(expr_str, dis_expr_str) + self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) + self.do_disassembly_test(annot_stmt_str, dis_annot_stmt_str) + self.do_disassembly_test(fn_with_annotate_str, dis_fn_with_annotate_str) + self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_bytes(self): + self.do_disassembly_test(_f.__code__.co_code, dis_f_co_code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_class(self): + self.do_disassembly_test(_C, dis_c) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_instance_method(self): + self.do_disassembly_test(_C(1).__init__, dis_c_instance_method) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_instance_method_bytes(self): + method_bytecode = _C(1).__init__.__code__.co_code + self.do_disassembly_test(method_bytecode, dis_c_instance_method_bytes) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_static_method(self): + self.do_disassembly_test(_C.sm, dis_c_static_method) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_class_method(self): + self.do_disassembly_test(_C.cm, dis_c_class_method) + + def test_disassemble_generator(self): + gen_func_disas = self.get_disassembly(_g) # Generator function + gen_disas = self.get_disassembly(_g(1)) # Generator iterator + self.assertEqual(gen_disas, gen_func_disas) + + def test_disassemble_async_generator(self): + agen_func_disas = self.get_disassembly(_ag) # Async generator function + agen_disas = self.get_disassembly(_ag(1)) # Async generator iterator + self.assertEqual(agen_disas, agen_func_disas) + + def test_disassemble_coroutine(self): + coro_func_disas = self.get_disassembly(_co) # Coroutine function + coro = _co(1) # Coroutine object + coro.close() # Avoid a RuntimeWarning (never awaited) + coro_disas = self.get_disassembly(coro) + self.assertEqual(coro_disas, coro_func_disas) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_fstring(self): + self.do_disassembly_test(_fstring, dis_fstring) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_with(self): + self.do_disassembly_test(_with, dis_with) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_asyncwith(self): + self.do_disassembly_test(_asyncwith, dis_asyncwith) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_try_finally(self): + self.do_disassembly_test(_tryfinally, dis_tryfinally) + self.do_disassembly_test(_tryfinallyconst, dis_tryfinallyconst) + + def test_dis_none(self): + try: + del sys.last_exc + except AttributeError: + pass + try: + del sys.last_traceback + except AttributeError: + pass + self.assertRaises(RuntimeError, dis.dis, None) + + def test_dis_traceback(self): + self.maxDiff = None + try: + del sys.last_traceback + except AttributeError: + pass + + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + sys.last_exc = e + + tb_dis = self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) + self.do_disassembly_test(None, tb_dis) + + def test_dis_object(self): + self.assertRaises(TypeError, dis.dis, object()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassemble_recursive(self): + def check(expected, **kwargs): + dis = self.get_disassembly(_h, **kwargs) + dis = self.strip_addresses(dis) + self.assertEqual(dis, expected) + + check(dis_nested_0, depth=0) + check(dis_nested_1, depth=1) + check(dis_nested_2, depth=2) + check(dis_nested_2, depth=3) + check(dis_nested_2, depth=None) + check(dis_nested_2) + + def test__try_compile_no_context_exc_on_error(self): + # see gh-102114 + try: + dis._try_compile(")", "") + except Exception as e: + self.assertIsNone(e.__context__) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: No END_ASYNC_FOR in disassembly of async for + def test_async_for_presentation(self): + + async def afunc(): + async for letter in async_iter1: + l2 + l3 + + disassembly = self.get_disassembly(afunc) + for line in disassembly.split("\n"): + if "END_ASYNC_FOR" in line: + break + else: + self.fail("No END_ASYNC_FOR in disassembly of async for") + self.assertNotIn("to", line) + self.assertIn("from", line) + + + @staticmethod + def code_quicken(f): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + + @cpython_only + @requires_specialization + def test_super_instructions(self): + self.code_quicken(lambda: load_test(0, 0)) + got = self.get_disassembly(load_test, adaptive=True) + self.do_disassembly_compare(got, dis_load_test_quickened_code) + + @cpython_only + @requires_specialization + def test_load_attr_specialize(self): + load_attr_quicken = """\ + 0 RESUME_CHECK 0 + + 1 LOAD_CONST_IMMORTAL 0 ('a') + LOAD_ATTR_SLOT 0 (__class__) + RETURN_VALUE +""" + co = compile("'a'.__class__", "", "eval") + self.code_quicken(lambda: exec(co, {}, {})) + got = self.get_disassembly(co, adaptive=True) + self.do_disassembly_compare(got, load_attr_quicken) + + @cpython_only + @requires_specialization + def test_call_specialize(self): + call_quicken = """\ + 0 RESUME_CHECK 0 + + 1 LOAD_NAME 0 (str) + PUSH_NULL + LOAD_SMALL_INT 1 + CALL_STR_1 1 + RETURN_VALUE +""" + co = compile("str(1)", "", "eval") + self.code_quicken(lambda: exec(co, {}, {})) + got = self.get_disassembly(co, adaptive=True) + self.do_disassembly_compare(got, call_quicken) + + @cpython_only + @requires_specialization + def test_loop_quicken(self): + # Loop can trigger a quicken where the loop is located + self.code_quicken(loop_test) + got = self.get_disassembly(loop_test, adaptive=True) + jit = sys._jit.is_enabled() + expected = dis_loop_test_quickened_code.format("JIT" if jit else "NO_JIT") + self.do_disassembly_compare(got, expected) + + @cpython_only + @requires_specialization + def test_loop_with_conditional_at_end_is_quickened(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + def for_loop_true(x): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if x: + pass + + for_loop_true(True) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_true, adaptive=True)) + + def for_loop_false(x): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if x: + pass + + for_loop_false(False) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_false, adaptive=True)) + + def while_loop(): + i = 0 + while i < _testinternalcapi.SPECIALIZATION_THRESHOLD: + i += 1 + + while_loop() + self.assertIn('COMPARE_OP_INT', + self.get_disassembly(while_loop, adaptive=True)) + + @cpython_only + def test_extended_arg_quick(self): + got = self.get_disassembly(extended_arg_quick) + self.do_disassembly_compare(got, dis_extended_arg_quick_code) + + def get_cached_values(self, quickened, adaptive): + def f(): + l = [] + for i in range(42): + l.append(i) + if quickened: + self.code_quicken(f) + else: + # "copy" the code to un-quicken it: + reset_code(f) + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( + f, show_caches=True, adaptive=adaptive + ), show_caches=True): + if instruction.opname == "CACHE": + yield instruction.argrepr + + @cpython_only + def test_show_caches(self): + for quickened in (False, True): + for adaptive in (False, True): + with self.subTest(f"{quickened=}, {adaptive=}"): + if adaptive: + pattern = r"^(\w+: \d+)?$" + else: + pattern = r"^(\w+: 0)?$" + caches = list(self.get_cached_values(quickened, adaptive)) + for cache in caches: + self.assertRegex(cache, pattern) + total_caches = 21 + empty_caches = 7 + self.assertEqual(caches.count(""), empty_caches) + self.assertEqual(len(caches), total_caches) + + @cpython_only + def test_show_currinstr_with_cache(self): + """ + Make sure that with lasti pointing to CACHE, it still shows the current + line correctly + """ + def f(): + print(a) + # The code above should generate a LOAD_GLOBAL which has CACHE instr after + # However, this might change in the future. So we explicitly try to find + # a CACHE entry in the instructions. If we can't do that, fail the test + + for inst in _unroll_caches_as_Instructions( + dis.get_instructions(f, show_caches=True), show_caches=True): + if inst.opname == "CACHE": + op_offset = inst.offset - 2 + cache_offset = inst.offset + break + else: + opname = inst.opname + else: + self.fail("Can't find a CACHE entry in the function provided to do the test") + + assem_op = self.get_disassembly(f.__code__, lasti=op_offset, wrapper=False) + assem_cache = self.get_disassembly(f.__code__, lasti=cache_offset, wrapper=False) + + # Make sure --> exists and points to the correct op + self.assertRegex(assem_op, fr"--> {opname}") + # Make sure when lasti points to cache, it shows the same disassembly + self.assertEqual(assem_op, assem_cache) + + +class DisWithFileTests(DisTests): + + # Run the tests again, using the file arg instead of print + def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): + output = io.StringIO() + if wrapper: + dis.dis(func, file=output, **kwargs) + else: + dis.disassemble(func, lasti, file=output, **kwargs) + return output.getvalue() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: No END_ASYNC_FOR in disassembly of async for + def test_async_for_presentation(self): + return super().test_async_for_presentation() + + +if dis.code_info.__doc__ is None: + code_info_consts = "0: None" +else: + code_info_consts = "0: 'Formatted details of methods, functions, or code.'" + +code_info_code_info = f"""\ +Name: code_info +Filename: (.*) +Argument count: 1 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 1 +Stack size: \\d+ +Flags: OPTIMIZED, NEWLOCALS, HAS_DOCSTRING +Constants: + {code_info_consts} +Names: + 0: _format_code_info + 1: _get_code_object +Variable names: + 0: x""" + + +@staticmethod +def tricky(a, b, /, x, y, z=True, *args, c, d, e=[], **kwds): + def f(c=c): + print(a, b, x, y, z, c, d, e, f) + yield a, b, x, y, z, c, d, e, f + +code_info_tricky = """\ +Name: tricky +Filename: (.*) +Argument count: 5 +Positional-only arguments: 2 +Kw-only arguments: 3 +Number of locals: 10 +Stack size: \\d+ +Flags: OPTIMIZED, NEWLOCALS, VARARGS, VARKEYWORDS, GENERATOR +Constants: + 0: + 1: None +Variable names: + 0: a + 1: b + 2: x + 3: y + 4: z + 5: c + 6: d + 7: e + 8: args + 9: kwds +Cell variables: + 0: [abedfxyz] + 1: [abedfxyz] + 2: [abedfxyz] + 3: [abedfxyz] + 4: [abedfxyz] + 5: [abedfxyz] + 6: [abedfxyz] + 7: [abedfxyz]""" +# NOTE: the order of the cell variables above depends on dictionary order! + +co_tricky_nested_f = tricky.__func__.__code__.co_consts[0] + +code_info_tricky_nested_f = """\ +Filename: (.*) +Argument count: 1 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 1 +Stack size: \\d+ +Flags: OPTIMIZED, NEWLOCALS, NESTED +Constants: + 0: None +Names: + 0: print +Variable names: + 0: c +Free variables: + 0: [abedfxyz] + 1: [abedfxyz] + 2: [abedfxyz] + 3: [abedfxyz] + 4: [abedfxyz] + 5: [abedfxyz]""" + +code_info_expr_str = """\ +Name: +Filename: +Argument count: 0 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 0 +Stack size: \\d+ +Flags: 0x0 +Constants: + 0: 1 +Names: + 0: x""" + +code_info_simple_stmt_str = """\ +Name: +Filename: +Argument count: 0 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 0 +Stack size: \\d+ +Flags: 0x0 +Constants: + 0: 1 + 1: None +Names: + 0: x""" + +code_info_compound_stmt_str = """\ +Name: +Filename: +Argument count: 0 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 0 +Stack size: \\d+ +Flags: 0x0 +Constants: + 0: 0 +Names: + 0: x""" + + +async def async_def(): + await 1 + async for a in b: pass + async with c as d: pass + +code_info_async_def = """\ +Name: async_def +Filename: (.*) +Argument count: 0 +Positional-only arguments: 0 +Kw-only arguments: 0 +Number of locals: 2 +Stack size: \\d+ +Flags: OPTIMIZED, NEWLOCALS, COROUTINE +Constants: + 0: 1 + 1: None +Names: + 0: b + 1: c +Variable names: + 0: a + 1: d""" + +class CodeInfoTests(unittest.TestCase): + test_pairs = [ + (dis.code_info, code_info_code_info), + (tricky, code_info_tricky), + (co_tricky_nested_f, code_info_tricky_nested_f), + (expr_str, code_info_expr_str), + (simple_stmt_str, code_info_simple_stmt_str), + (compound_stmt_str, code_info_compound_stmt_str), + (async_def, code_info_async_def) + ] + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_code_info(self): + self.maxDiff = 1000 + for x, expected in self.test_pairs: + self.assertRegex(dis.code_info(x), expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_show_code(self): + self.maxDiff = 1000 + for x, expected in self.test_pairs: + with captured_stdout() as output: + dis.show_code(x) + self.assertRegex(output.getvalue(), expected+"\n") + output = io.StringIO() + dis.show_code(x, file=output) + self.assertRegex(output.getvalue(), expected) + + def test_code_info_object(self): + self.assertRaises(TypeError, dis.code_info, object()) + + def test_pretty_flags_no_flags(self): + self.assertEqual(dis.pretty_flags(0), '0x0') + + +# Fodder for instruction introspection tests +# Editing any of these may require recalculating the expected output +def outer(a=1, b=2): + def f(c=3, d=4): + def inner(e=5, f=6): + print(a, b, c, d, e, f) + print(a, b, c, d) + return inner + print(a, b, '', 1, [], {}, "Hello world!") + return f + +def jumpy(): + # This won't actually run (but that's OK, we only disassemble it) + for i in range(10): + print(i) + if i < 4: + continue + if i > 6: + break + else: + print("I can haz else clause?") + while i: + print(i) + i -= 1 + if i > 6: + continue + if i < 4: + break + else: + print("Who let lolcatz into this test suite?") + try: + 1 / 0 + except ZeroDivisionError: + print("Here we go, here we go, here we go...") + else: + with i as dodgy: + print("Never reach this") + finally: + print("OK, now we're done") + +# End fodder for opinfo generation tests +expected_outer_line = 1 +_line_offset = outer.__code__.co_firstlineno - 1 +code_object_f = outer.__code__.co_consts[1] +expected_f_line = code_object_f.co_firstlineno - _line_offset +code_object_inner = code_object_f.co_consts[1] +expected_inner_line = code_object_inner.co_firstlineno - _line_offset +expected_jumpy_line = 1 + +# The following lines are useful to regenerate the expected results after +# either the fodder is modified or the bytecode generation changes +# After regeneration, update the references to code_object_f and +# code_object_inner before rerunning the tests + +def _stringify_instruction(instr): + # Since postions offsets change a lot for these test cases, ignore them. + base = ( + f" make_inst(opname={instr.opname!r}, arg={instr.arg!r}, argval={instr.argval!r}, " + + f"argrepr={instr.argrepr!r}, offset={instr.offset}, start_offset={instr.start_offset}, " + + f"starts_line={instr.starts_line!r}, line_number={instr.line_number}" + ) + if instr.label is not None: + base += f", label={instr.label!r}" + if instr.cache_info: + base += f", cache_info={instr.cache_info!r}" + return base + ")," + +def _prepare_test_cases(): + ignore = io.StringIO() + with contextlib.redirect_stdout(ignore): + f = outer() + inner = f() + _instructions_outer = dis.get_instructions(outer, first_line=expected_outer_line) + _instructions_f = dis.get_instructions(f, first_line=expected_f_line) + _instructions_inner = dis.get_instructions(inner, first_line=expected_inner_line) + _instructions_jumpy = dis.get_instructions(jumpy, first_line=expected_jumpy_line) + result = "\n".join( + [ + "expected_opinfo_outer = [", + *map(_stringify_instruction, _instructions_outer), + "]", + "", + "expected_opinfo_f = [", + *map(_stringify_instruction, _instructions_f), + "]", + "", + "expected_opinfo_inner = [", + *map(_stringify_instruction, _instructions_inner), + "]", + "", + "expected_opinfo_jumpy = [", + *map(_stringify_instruction, _instructions_jumpy), + "]", + ] + ) + result = result.replace(repr(repr(code_object_f)), "repr(code_object_f)") + result = result.replace(repr(code_object_f), "code_object_f") + result = result.replace(repr(repr(code_object_inner)), "repr(code_object_inner)") + result = result.replace(repr(code_object_inner), "code_object_inner") + print(result) + +# from test.test_dis import _prepare_test_cases; _prepare_test_cases() + +make_inst = dis.Instruction.make + +expected_opinfo_outer = [ + make_inst(opname='MAKE_CELL', arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None), + make_inst(opname='MAKE_CELL', arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None), + make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1), + make_inst(opname='LOAD_CONST', arg=4, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2), + make_inst(opname='LOAD_FAST_BORROW', arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2), + make_inst(opname='BUILD_TUPLE', arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2), + make_inst(opname='LOAD_CONST', arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2), + make_inst(opname='MAKE_FUNCTION', arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2), + make_inst(opname='SET_FUNCTION_ATTRIBUTE', arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2), + make_inst(opname='SET_FUNCTION_ATTRIBUTE', arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2), + make_inst(opname='STORE_FAST', arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2), + make_inst(opname='LOAD_GLOBAL', arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_DEREF', arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7), + make_inst(opname='LOAD_DEREF', arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7), + make_inst(opname='LOAD_CONST', arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7), + make_inst(opname='BUILD_LIST', arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7), + make_inst(opname='BUILD_MAP', arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7), + make_inst(opname='LOAD_CONST', arg=3, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7), + make_inst(opname='CALL', arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7), + make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8), +] + +expected_opinfo_f = [ + make_inst(opname='COPY_FREE_VARS', arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None), + make_inst(opname='MAKE_CELL', arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None), + make_inst(opname='MAKE_CELL', arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None), + make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2), + make_inst(opname='LOAD_CONST', arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3), + make_inst(opname='LOAD_FAST_BORROW', arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3), + make_inst(opname='LOAD_FAST_BORROW', arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3), + make_inst(opname='LOAD_FAST_BORROW', arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3), + make_inst(opname='BUILD_TUPLE', arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3), + make_inst(opname='LOAD_CONST', arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3), + make_inst(opname='MAKE_FUNCTION', arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3), + make_inst(opname='SET_FUNCTION_ATTRIBUTE', arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3), + make_inst(opname='SET_FUNCTION_ATTRIBUTE', arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3), + make_inst(opname='STORE_FAST', arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_DEREF', arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5), + make_inst(opname='LOAD_DEREF', arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5), + make_inst(opname='LOAD_DEREF', arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5), + make_inst(opname='LOAD_DEREF', arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5), + make_inst(opname='CALL', arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5), + make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6), +] + +expected_opinfo_inner = [ + make_inst(opname='COPY_FREE_VARS', arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None), + make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_DEREF', arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4), + make_inst(opname='LOAD_DEREF', arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4), + make_inst(opname='LOAD_DEREF', arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4), + make_inst(opname='LOAD_DEREF', arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4), + make_inst(opname='LOAD_FAST_BORROW_LOAD_FAST_BORROW', arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4), + make_inst(opname='CALL', arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4), + make_inst(opname='LOAD_CONST', arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4), +] + +expected_opinfo_jumpy = [ + make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1), + make_inst(opname='LOAD_GLOBAL', arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='GET_ITER', arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3), + make_inst(opname='FOR_ITER', arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5), + make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5), + make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=70, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=64, start_offset=64, starts_line=False, line_number=5), + make_inst(opname='JUMP_BACKWARD', arg=23, argval=24, argrepr='to L1', offset=66, start_offset=66, starts_line=True, line_number=6, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=70, start_offset=70, starts_line=True, line_number=7, label=2), + make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=72, start_offset=72, starts_line=False, line_number=7), + make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=74, start_offset=74, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=88, argrepr='to L3', offset=78, start_offset=78, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7), + make_inst(opname='JUMP_BACKWARD', arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3), + make_inst(opname='JUMP_FORWARD', arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8), + make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4), + make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10), + make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13), + make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14), + make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14), + make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14), + make_inst(opname='JUMP_BACKWARD', arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6), + make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16), + make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16), + make_inst(opname='JUMP_BACKWARD', arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='JUMP_FORWARD', arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19), + make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21), + make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21), + make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25), + make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25), + make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25), + make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22), + make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), +] + +# One last piece of inspect fodder to check the default line number handling +def simple(): pass +expected_opinfo_simple = [ + make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno), + make_inst(opname='LOAD_CONST', arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno), +] + + +class InstructionTestCase(BytecodeTestCase): + + def assertInstructionsEqual(self, instrs_1, instrs_2, /): + instrs_1 = [instr_1._replace(positions=None, cache_info=None) for instr_1 in instrs_1] + instrs_2 = [instr_2._replace(positions=None, cache_info=None) for instr_2 in instrs_2] + self.assertEqual(instrs_1, instrs_2) + +class InstructionTests(InstructionTestCase): + + def __init__(self, *args): + super().__init__(*args) + self.maxDiff = None + + def test_instruction_str(self): + # smoke test for __str__ + instrs = dis.get_instructions(simple) + for instr in instrs: + str(instr) + + def test_default_first_line(self): + actual = dis.get_instructions(simple) + self.assertInstructionsEqual(list(actual), expected_opinfo_simple) + + def test_first_line_set_to_None(self): + actual = dis.get_instructions(simple, first_line=None) + self.assertInstructionsEqual(list(actual), expected_opinfo_simple) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_outer(self): + actual = dis.get_instructions(outer, first_line=expected_outer_line) + self.assertInstructionsEqual(list(actual), expected_opinfo_outer) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_nested(self): + with captured_stdout(): + f = outer() + actual = dis.get_instructions(f, first_line=expected_f_line) + self.assertInstructionsEqual(list(actual), expected_opinfo_f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_doubly_nested(self): + with captured_stdout(): + inner = outer()() + actual = dis.get_instructions(inner, first_line=expected_inner_line) + self.assertInstructionsEqual(list(actual), expected_opinfo_inner) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_jumpy(self): + actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line) + self.assertInstructionsEqual(list(actual), expected_opinfo_jumpy) + + @requires_debug_ranges() + def test_co_positions(self): + code = compile('f(\n x, y, z\n)', '', 'exec') + positions = [ + instr.positions + for instr in dis.get_instructions(code) + ] + expected = [ + (0, 1, 0, 0), + (1, 1, 0, 1), + (1, 1, 0, 1), + (2, 2, 2, 3), + (2, 2, 5, 6), + (2, 2, 8, 9), + (1, 3, 0, 1), + (1, 3, 0, 1), + (1, 3, 0, 1), + (1, 3, 0, 1) + ] + self.assertEqual(positions, expected) + + named_positions = [ + (pos.lineno, pos.end_lineno, pos.col_offset, pos.end_col_offset) + for pos in positions + ] + self.assertEqual(named_positions, expected) + + @requires_debug_ranges() + def test_co_positions_missing_info(self): + code = compile('x, y, z', '', 'exec') + code_without_location_table = code.replace(co_linetable=b'') + actual = dis.get_instructions(code_without_location_table) + for instruction in actual: + with self.subTest(instruction=instruction): + positions = instruction.positions + self.assertEqual(len(positions), 4) + if instruction.opname == "RESUME": + continue + self.assertIsNone(positions.lineno) + self.assertIsNone(positions.end_lineno) + self.assertIsNone(positions.col_offset) + self.assertIsNone(positions.end_col_offset) + + @requires_debug_ranges() + def test_co_positions_with_lots_of_caches(self): + def roots(a, b, c): + d = b**2 - 4 * a * c + yield (-b - cmath.sqrt(d)) / (2 * a) + if d: + yield (-b + cmath.sqrt(d)) / (2 * a) + code = roots.__code__ + ops = code.co_code[::2] + cache_opcode = opcode.opmap["CACHE"] + caches = sum(op == cache_opcode for op in ops) + non_caches = len(ops) - caches + # Make sure we have "lots of caches". If not, roots should be changed: + assert 1 / 3 <= caches / non_caches, "this test needs more caches!" + for show_caches in (False, True): + for adaptive in (False, True): + with self.subTest(f"{adaptive=}, {show_caches=}"): + co_positions = [ + positions + for op, positions in zip(ops, code.co_positions(), strict=True) + if show_caches or op != cache_opcode + ] + dis_positions = [ + None if instruction.positions is None else ( + instruction.positions.lineno, + instruction.positions.end_lineno, + instruction.positions.col_offset, + instruction.positions.end_col_offset, + ) + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( + code, adaptive=adaptive, show_caches=show_caches + ), show_caches=show_caches) + ] + self.assertEqual(co_positions, dis_positions) + + def test_oparg_alias(self): + instruction = make_inst(opname="NOP", arg=None, argval=None, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, + positions=None) + self.assertEqual(instruction.arg, instruction.oparg) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_show_caches_with_label(self): + def f(x, y, z): + if x: + res = y + else: + res = z + return res + + output = io.StringIO() + dis.dis(f.__code__, file=output, show_caches=True) + self.assertIn("L1:", output.getvalue()) + + def test_is_op_format(self): + output = io.StringIO() + dis.dis("a is b", file=output, show_caches=True) + self.assertIn("IS_OP 0 (is)", output.getvalue()) + + output = io.StringIO() + dis.dis("a is not b", file=output, show_caches=True) + self.assertIn("IS_OP 1 (is not)", output.getvalue()) + + def test_contains_op_format(self): + output = io.StringIO() + dis.dis("a in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 0 (in)", output.getvalue()) + + output = io.StringIO() + dis.dis("a not in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 1 (not in)", output.getvalue()) + + def test_baseopname_and_baseopcode(self): + # Standard instructions + for name in dis.opmap: + instruction = make_inst(opname=name, arg=None, argval=None, argrepr='', offset=0, + start_offset=0, starts_line=True, line_number=1, label=None, positions=None) + baseopname = instruction.baseopname + baseopcode = instruction.baseopcode + self.assertIsNotNone(baseopname) + self.assertIsNotNone(baseopcode) + self.assertEqual(name, baseopname) + self.assertEqual(instruction.opcode, baseopcode) + + # Specialized instructions + for name in opcode._specialized_opmap: + instruction = make_inst(opname=name, arg=None, argval=None, argrepr='', + offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None) + baseopname = instruction.baseopname + baseopcode = instruction.baseopcode + self.assertIn(name, opcode._specializations[baseopname]) + self.assertEqual(opcode.opmap[baseopname], baseopcode) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - JUMP_BACKWARD/JUMP_FORWARD are placeholders + def test_jump_target(self): + # Non-jump instructions should return None + instruction = make_inst(opname="NOP", arg=None, argval=None, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, + positions=None) + self.assertIsNone(instruction.jump_target) + + delta = 100 + instruction = make_inst(opname="JUMP_FORWARD", arg=delta, argval=delta, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, + positions=None) + self.assertEqual(10 + 2 + 100*2, instruction.jump_target) + + # Test negative deltas + instruction = make_inst(opname="JUMP_BACKWARD", arg=delta, argval=delta, + argrepr='', offset=200, start_offset=200, starts_line=True, line_number=1, label=None, + positions=None) + self.assertEqual(200 + 2 - 100*2 + 2*1, instruction.jump_target) + + # Make sure cache entries are handled + instruction = make_inst(opname="SEND", arg=delta, argval=delta, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, + positions=None) + self.assertEqual(10 + 2 + 1*2 + 100*2, instruction.jump_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - JUMP_BACKWARD is a placeholder + def test_argval_argrepr(self): + def f(opcode, oparg, offset, *init_args): + arg_resolver = dis.ArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + + offset = 42 + co_consts = (0, 1, 2, 3) + names = {1: 'a', 2: 'b'} + varname_from_oparg = lambda i : names[i] + labels_map = {24: 1} + args = (offset, co_consts, names, varname_from_oparg, labels_map) + self.assertEqual(f(opcode.opmap["POP_TOP"], None, *args), (None, '')) + self.assertEqual(f(opcode.opmap["LOAD_CONST"], 1, *args), (1, '1')) + self.assertEqual(f(opcode.opmap["LOAD_GLOBAL"], 2, *args), ('a', 'a')) + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 11, *args), (24, 'to L1')) + self.assertEqual(f(opcode.opmap["COMPARE_OP"], 3, *args), ('<', '<')) + self.assertEqual(f(opcode.opmap["SET_FUNCTION_ATTRIBUTE"], 2, *args), (2, 'kwdefaults')) + self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) + self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - JUMP_BACKWARD is a placeholder + def test_custom_arg_resolver(self): + class MyArgResolver(dis.ArgResolver): + def offset_from_jump_arg(self, op, arg, offset): + return arg + 1 + + def get_label_for_offset(self, offset): + return 2 * offset + + def f(opcode, oparg, offset, *init_args): + arg_resolver = MyArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + offset = 42 + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to L4')) + self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to L6')) + + + def get_instructions(self, code): + return dis._get_instructions_bytes(code) + + def test_start_offset(self): + # When no extended args are present, + # start_offset should be equal to offset + + instructions = list(dis.Bytecode(_f)) + for instruction in instructions: + self.assertEqual(instruction.offset, instruction.start_offset) + + def last_item(iterable): + return functools.reduce(lambda a, b : b, iterable) + + code = bytes([ + opcode.opmap["LOAD_FAST"], 0x00, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, + ]) + labels_map = dis._make_labels_map(code) + jump = last_item(self.get_instructions(code)) + self.assertEqual(4, jump.offset) + self.assertEqual(2, jump.start_offset) + + code = bytes([ + opcode.opmap["LOAD_FAST"], 0x00, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, + opcode.opmap["CACHE"], 0x00, + ]) + jump = last_item(self.get_instructions(code)) + self.assertEqual(8, jump.offset) + self.assertEqual(2, jump.start_offset) + + code = bytes([ + opcode.opmap["LOAD_FAST"], 0x00, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["EXTENDED_ARG"], 0x01, + opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, + opcode.opmap["CACHE"], 0x00, + ]) + instructions = list(self.get_instructions(code)) + # 1st jump + self.assertEqual(4, instructions[2].offset) + self.assertEqual(2, instructions[2].start_offset) + # 2nd jump + self.assertEqual(14, instructions[6].offset) + self.assertEqual(8, instructions[6].start_offset) + + def test_cache_offset_and_end_offset(self): + code = bytes([ + opcode.opmap["LOAD_GLOBAL"], 0x01, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["LOAD_FAST"], 0x00, + opcode.opmap["CALL"], 0x01, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["CACHE"], 0x00, + opcode.opmap["CACHE"], 0x00 + ]) + instructions = list(self.get_instructions(code)) + self.assertEqual(2, instructions[0].cache_offset) + self.assertEqual(10, instructions[0].end_offset) + self.assertEqual(12, instructions[1].cache_offset) + self.assertEqual(12, instructions[1].end_offset) + self.assertEqual(14, instructions[2].cache_offset) + self.assertEqual(20, instructions[2].end_offset) + + # end_offset of the previous instruction should be equal to the + # start_offset of the following instruction + instructions = list(dis.Bytecode(self.test_cache_offset_and_end_offset)) + for prev, curr in zip(instructions, instructions[1:]): + self.assertEqual(prev.end_offset, curr.start_offset) + + +# get_instructions has its own tests above, so can rely on it to validate +# the object oriented API +class BytecodeTests(InstructionTestCase, DisTestBase): + + def test_instantiation(self): + # Test with function, method, code string and code object + for obj in [_f, _C(1).__init__, "a=1", _f.__code__]: + with self.subTest(obj=obj): + b = dis.Bytecode(obj) + self.assertIsInstance(b.codeobj, types.CodeType) + + self.assertRaises(TypeError, dis.Bytecode, object()) + + def test_iteration(self): + for obj in [_f, _C(1).__init__, "a=1", _f.__code__]: + with self.subTest(obj=obj): + via_object = list(dis.Bytecode(obj)) + via_generator = list(dis.get_instructions(obj)) + self.assertInstructionsEqual(via_object, via_generator) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_explicit_first_line(self): + actual = dis.Bytecode(outer, first_line=expected_outer_line) + self.assertInstructionsEqual(list(actual), expected_opinfo_outer) + + def test_source_line_in_disassembly(self): + # Use the line in the source code + actual = dis.Bytecode(simple).dis() + actual = actual.strip().partition(" ")[0] # extract the line no + expected = str(simple.__code__.co_firstlineno) + self.assertEqual(actual, expected) + # Use an explicit first line number + actual = dis.Bytecode(simple, first_line=350).dis() + actual = actual.strip().partition(" ")[0] # extract the line no + self.assertEqual(actual, "350") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_info(self): + self.maxDiff = 1000 + for x, expected in CodeInfoTests.test_pairs: + b = dis.Bytecode(x) + self.assertRegex(b.info(), expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_disassembled(self): + actual = dis.Bytecode(_f).dis() + self.do_disassembly_compare(actual, dis_f) + + def test_from_traceback(self): + tb = get_tb() + b = dis.Bytecode.from_traceback(tb) + while tb.tb_next: tb = tb.tb_next + + self.assertEqual(b.current_offset, tb.tb_lasti) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_from_traceback_dis(self): + self.maxDiff = None + tb = get_tb() + b = dis.Bytecode.from_traceback(tb) + self.assertEqual(b.dis(), dis_traceback) + + @requires_debug_ranges() + def test_bytecode_co_positions(self): + bytecode = dis.Bytecode("a=1") + for instr, positions in zip(bytecode, bytecode.codeobj.co_positions()): + assert instr.positions == positions + +class TestBytecodeTestCase(BytecodeTestCase): + def test_assert_not_in_with_op_not_in_bytecode(self): + code = compile("a = 1", "", "exec") + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) + self.assertNotInBytecode(code, "LOAD_NAME") + self.assertNotInBytecode(code, "LOAD_NAME", "a") + + def test_assert_not_in_with_arg_not_in_bytecode(self): + code = compile("a = 1", "", "exec") + self.assertInBytecode(code, "LOAD_SMALL_INT") + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) + self.assertNotInBytecode(code, "LOAD_CONST", 2) + + def test_assert_not_in_with_arg_in_bytecode(self): + code = compile("a = 1", "", "exec") + with self.assertRaises(AssertionError): + self.assertNotInBytecode(code, "LOAD_SMALL_INT", 1) + +class TestFinderMethods(unittest.TestCase): + def test__find_imports(self): + cases = [ + ("import a.b.c", ('a.b.c', 0, None)), + ("from a.b import c", ('a.b', 0, ('c',))), + ("from a.b import c as d", ('a.b', 0, ('c',))), + ("from a.b import *", ('a.b', 0, ('*',))), + ("from ...a.b import c as d", ('a.b', 3, ('c',))), + ("from ..a.b import c as d, e as f", ('a.b', 2, ('c', 'e'))), + ("from ..a.b import *", ('a.b', 2, ('*',))), + ] + for src, expected in cases: + with self.subTest(src=src): + code = compile(src, "", "exec") + res = tuple(dis._find_imports(code)) + self.assertEqual(len(res), 1) + self.assertEqual(res[0], expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test__find_store_names(self): + cases = [ + ("x+y", ()), + ("x=y=1", ('x', 'y')), + ("x+=y", ('x',)), + ("global x\nx=y=1", ('x', 'y')), + ("global x\nz=x", ('z',)), + ] + for src, expected in cases: + with self.subTest(src=src): + code = compile(src, "", "exec") + res = tuple(dis._find_store_names(code)) + self.assertEqual(res, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_findlabels(self): + labels = dis.findlabels(jumpy.__code__.co_code) + jumps = [ + instr.offset + for instr in expected_opinfo_jumpy + if instr.is_jump_target + ] + + self.assertEqual(sorted(labels), sorted(jumps)) + + def test_findlinestarts(self): + def func(): + pass + + code = func.__code__ + offsets = [linestart[0] for linestart in dis.findlinestarts(code)] + self.assertEqual(offsets, [0, 2]) + + +class TestDisTraceback(DisTestBase): + def setUp(self) -> None: + try: # We need to clean up existing tracebacks + del sys.last_exc + except AttributeError: + pass + try: # We need to clean up existing tracebacks + del sys.last_traceback + except AttributeError: + pass + return super().setUp() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def get_disassembly(self, tb): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.distb(tb) + return output.getvalue() + + def test_distb_empty(self): + with self.assertRaises(RuntimeError): + dis.distb() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_distb_last_traceback(self): + self.maxDiff = None + # We need to have an existing last traceback in `sys`: + tb = get_tb() + sys.last_traceback = tb + + self.do_disassembly_compare(self.get_disassembly(None), dis_traceback) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_distb_explicit_arg(self): + self.maxDiff = None + tb = get_tb() + + self.do_disassembly_compare(self.get_disassembly(tb), dis_traceback) + + +class TestDisTracebackWithFile(TestDisTraceback): + # Run the `distb` tests again, using the file arg instead of print + def get_disassembly(self, tb): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.distb(tb, file=output) + return output.getvalue() + +def _unroll_caches_as_Instructions(instrs, show_caches=False): + # Cache entries are no longer reported by dis as fake instructions, + # but some tests assume that do. We should rewrite the tests to assume + # the new API, but it will be clearer to keep the tests working as + # before and do that in a separate PR. + + for instr in instrs: + yield instr + if not show_caches: + continue + + offset = instr.offset + for name, size, data in (instr.cache_info or ()): + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + + yield make_inst("CACHE", 0, None, argrepr, offset, offset, + False, None, None, instr.positions) + + +class TestDisCLI(unittest.TestCase): + + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return textwrap.dedent(string).strip() + + def set_source(self, content): + with open(self.filename, 'w') as fp: + fp.write(self.text_normalize(content)) + + def invoke_dis(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def check_output(self, source, expect, *flags): + with self.subTest(source=source, flags=flags): + self.set_source(source) + res = self.invoke_dis(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + base_flags = [ + ('-C', '--show-caches'), + ('-O', '--show-offsets'), + ('-P', '--show-positions'), + ('-S', '--specialized'), + ] + + self.set_source(''' + def f(): + print(x) + return None + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(args=args[1:]): + _ = self.invoke_dis(*args) + + with self.assertRaises(SystemExit): + # suppress argparse error message + with contextlib.redirect_stderr(io.StringIO()): + _ = self.invoke_dis('--unknown') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_show_cache(self): + # test 'python -m dis -C/--show-caches' + source = 'print()' + expect = ''' + 0 RESUME 0 + + 1 LOAD_NAME 0 (print) + PUSH_NULL + CALL 0 + CACHE 0 (counter: 0) + CACHE 0 (func_version: 0) + CACHE 0 + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-C', '--show-caches']: + self.check_output(source, expect, flag) + + def test_show_offsets(self): + # test 'python -m dis -O/--show-offsets' + source = 'pass' + expect = ''' + 0 0 RESUME 0 + + 1 2 LOAD_CONST 0 (None) + 4 RETURN_VALUE + ''' + for flag in ['-O', '--show-offsets']: + self.check_output(source, expect, flag) + + def test_show_positions(self): + # test 'python -m dis -P/--show-positions' + source = 'pass' + expect = ''' + 0:0-1:0 RESUME 0 + + 1:0-1:4 LOAD_CONST 0 (None) + 1:0-1:4 RETURN_VALUE + ''' + for flag in ['-P', '--show-positions']: + self.check_output(source, expect, flag) + + def test_specialized_code(self): + # test 'python -m dis -S/--specialized' + source = 'pass' + expect = ''' + 0 RESUME 0 + + 1 LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-S', '--specialized']: + self.check_output(source, expect, flag) if __name__ == "__main__": diff --git a/Lib/test/test_doctest/__init__.py b/Lib/test/test_doctest/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_doctest/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_doctest/decorator_mod.py b/Lib/test/test_doctest/decorator_mod.py new file mode 100644 index 00000000000..9f106888411 --- /dev/null +++ b/Lib/test/test_doctest/decorator_mod.py @@ -0,0 +1,10 @@ +# This module is used in `doctest_lineno.py`. +import functools + + +def decorator(f): + @functools.wraps(f) + def inner(): + return f() + + return inner diff --git a/Lib/test/test_doctest/doctest_aliases.py b/Lib/test/test_doctest/doctest_aliases.py new file mode 100644 index 00000000000..30cefafa83e --- /dev/null +++ b/Lib/test/test_doctest/doctest_aliases.py @@ -0,0 +1,13 @@ +# Used by test_doctest.py. + +class TwoNames: + '''f() and g() are two names for the same method''' + + def f(self): + ''' + >>> print(TwoNames().f()) + f + ''' + return 'f' + + g = f # define an alias for f diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py new file mode 100644 index 00000000000..0bd402e9828 --- /dev/null +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -0,0 +1,107 @@ +# This module is used in `test_doctest`. +# It must not have a docstring. + +def func_with_docstring(): + """Some unrelated info.""" + + +def func_without_docstring(): + pass + + +def func_with_doctest(): + """ + This function really contains a test case. + + >>> func_with_doctest.__name__ + 'func_with_doctest' + """ + return 3 + + +class ClassWithDocstring: + """Some unrelated class information.""" + + +class ClassWithoutDocstring: + pass + + +class ClassWithDoctest: + """This class really has a test case in it. + + >>> ClassWithDoctest.__name__ + 'ClassWithDoctest' + """ + + +class MethodWrapper: + def method_with_docstring(self): + """Method with a docstring.""" + + def method_without_docstring(self): + pass + + def method_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.method_with_doctest.__name__ + 'method_with_doctest' + """ + + @classmethod + def classmethod_with_doctest(cls): + """ + This has a doctest! + >>> MethodWrapper.classmethod_with_doctest.__name__ + 'classmethod_with_doctest' + """ + + @property + def property_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.property_with_doctest.__name__ + 'property_with_doctest' + """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ + + +# https://github.com/python/cpython/issues/115392 +from test.test_doctest.decorator_mod import decorator + +@decorator +@decorator +def func_with_docstring_wrapped(): + """Some unrelated info.""" + + +# https://github.com/python/cpython/issues/136914 +import functools + + +@functools.cache +def cached_func_with_doctest(value): + """ + >>> cached_func_with_doctest(1) + -1 + """ + return -value + + +@functools.cache +def cached_func_without_docstring(value): + return value + 1 + + +class ClassWithACachedProperty: + + @functools.cached_property + def cached(self): + """ + >>> X().cached + -1 + """ + return 0 diff --git a/Lib/test/test_doctest/sample_doctest.py b/Lib/test/test_doctest/sample_doctest.py new file mode 100644 index 00000000000..049f737a0a4 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest.py @@ -0,0 +1,76 @@ +"""This is a sample module that doesn't really test anything all that + interesting. + +It simply has a few tests, some of which succeed and some of which fail. + +It's important that the numbers remain constant as another test is +testing the running of these tests. + + +>>> 2+2 +4 +""" + + +def foo(): + """ + + >>> 2+2 + 5 + + >>> 2+2 + 4 + """ + +def bar(): + """ + + >>> 2+2 + 4 + """ + +def test_silly_setup(): + """ + + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup + True + """ + +def w_blank(): + """ + >>> if 1: + ... print('a') + ... print() + ... print('b') + a + + b + """ + +x = 1 +def x_is_one(): + """ + >>> x + 1 + """ + +def y_is_one(): + """ + >>> y + 1 + """ + +__test__ = {'good': """ + >>> 42 + 42 + """, + 'bad': """ + >>> 42 + 666 + """, + } + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/sample_doctest_errors.py b/Lib/test/test_doctest/sample_doctest_errors.py new file mode 100644 index 00000000000..4a6f07af2d4 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_errors.py @@ -0,0 +1,46 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving errors. + +>>> 2 + 2 +5 +>>> 1/0 +1 +""" + +def g(): + [][0] # line 12 + +def errors(): + """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> g() + 1 + """ + +def syntax_error(): + """ + >>> 2+*3 + 5 + """ + +__test__ = { + 'bad': """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + """, +} + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/sample_doctest_no_docstrings.py b/Lib/test/test_doctest/sample_doctest_no_docstrings.py new file mode 100644 index 00000000000..e4201edbce9 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_no_docstrings.py @@ -0,0 +1,12 @@ +# This is a sample module used for testing doctest. +# +# This module is for testing how doctest handles a module with no +# docstrings. + + +class Foo(object): + + # A class with no docstring. + + def __init__(self): + pass diff --git a/Lib/test/test_doctest/sample_doctest_no_doctests.py b/Lib/test/test_doctest/sample_doctest_no_doctests.py new file mode 100644 index 00000000000..7daa57231c8 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_no_doctests.py @@ -0,0 +1,15 @@ +"""This is a sample module used for testing doctest. + +This module is for testing how doctest handles a module with docstrings +but no doctest examples. + +""" + + +class Foo(object): + """A docstring with no doctest examples. + + """ + + def __init__(self): + pass diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000..1b83dec1f8c --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py new file mode 100644 index 00000000000..ec008ed38b6 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest.py @@ -0,0 +1,3864 @@ +""" +Test script for doctest. +""" + +from test import support +from test.support import import_helper +import doctest +import functools +import os +import sys +import importlib +import importlib.abc +import importlib.util +import unittest +import tempfile +import types +import contextlib + + +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator + + +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. +# There are also related tests in `test_doctest2` module. + +###################################################################### +## Sample Objects (used by test cases) +###################################################################### + +def sample_func(v): + """ + Blah blah + + >>> print(sample_func(22)) + 44 + + Yee ha! + """ + return v+v + +class SampleClass: + """ + >>> print(1) + 1 + + >>> # comments get ignored. so are empty PS1 and PS2 prompts: + >>> + ... + + Multiline example: + >>> sc = SampleClass(3) + >>> for i in range(10): + ... sc = sc.double() + ... print(' ', sc.get(), sep='', end='') + 6 12 24 48 96 192 384 768 1536 3072 + """ + def __init__(self, val): + """ + >>> print(SampleClass(12).get()) + 12 + """ + self.val = val + + def double(self): + """ + >>> print(SampleClass(12).double().get()) + 24 + """ + return SampleClass(self.val + self.val) + + def get(self): + """ + >>> print(SampleClass(-5).get()) + -5 + """ + return self.val + + def setter(self, val): + """ + >>> s = SampleClass(-5) + >>> s.setter(1) + >>> print(s.val) + 1 + """ + self.val = val + + def a_staticmethod(v): + """ + >>> print(SampleClass.a_staticmethod(10)) + 11 + """ + return v+1 + a_staticmethod = staticmethod(a_staticmethod) + + def a_classmethod(cls, v): + """ + >>> print(SampleClass.a_classmethod(10)) + 12 + >>> print(SampleClass(0).a_classmethod(10)) + 12 + """ + return v+2 + a_classmethod = classmethod(a_classmethod) + + a_property = property(get, setter, doc=""" + >>> print(SampleClass(22).a_property) + 22 + """) + + a_class_attribute = 42 + + @functools.cached_property + def a_cached_property(self): + """ + >>> print(SampleClass(29).get()) + 29 + """ + return "hello" + + class NestedClass: + """ + >>> x = SampleClass.NestedClass(5) + >>> y = x.square() + >>> print(y.get()) + 25 + """ + def __init__(self, val=0): + """ + >>> print(SampleClass.NestedClass().get()) + 0 + """ + self.val = val + def square(self): + return SampleClass.NestedClass(self.val*self.val) + def get(self): + return self.val + +class SampleNewStyleClass(object): + r""" + >>> print('1\n2\n3') + 1 + 2 + 3 + """ + def __init__(self, val): + """ + >>> print(SampleNewStyleClass(12).get()) + 12 + """ + self.val = val + + def double(self): + """ + >>> print(SampleNewStyleClass(12).double().get()) + 24 + """ + return SampleNewStyleClass(self.val + self.val) + + def get(self): + """ + >>> print(SampleNewStyleClass(-5).get()) + -5 + """ + return self.val + +###################################################################### +## Test Cases +###################################################################### + +def test_Example(): r""" +Unit tests for the `Example` class. + +Example is a simple container class that holds: + - `source`: A source string. + - `want`: An expected output string. + - `exc_msg`: An expected exception message string (or None if no + exception is expected). + - `lineno`: A line number (within the docstring). + - `indent`: The example's indentation in the input string. + - `options`: An option dictionary, mapping option flags to True or + False. + +These attributes are set by the constructor. `source` and `want` are +required; the other attributes all have default values: + + >>> example = doctest.Example('print(1)', '1\n') + >>> (example.source, example.want, example.exc_msg, + ... example.lineno, example.indent, example.options) + ('print(1)\n', '1\n', None, 0, 0, {}) + +The first three attributes (`source`, `want`, and `exc_msg`) may be +specified positionally; the remaining arguments should be specified as +keyword arguments: + + >>> exc_msg = 'IndexError: pop from an empty list' + >>> example = doctest.Example('[].pop()', '', exc_msg, + ... lineno=5, indent=4, + ... options={doctest.ELLIPSIS: True}) + >>> (example.source, example.want, example.exc_msg, + ... example.lineno, example.indent, example.options) + ('[].pop()\n', '', 'IndexError: pop from an empty list\n', 5, 4, {8: True}) + +The constructor normalizes the `source` string to end in a newline: + + Source spans a single line: no terminating newline. + >>> e = doctest.Example('print(1)', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print(1)\n', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + Source spans multiple lines: require terminating newline. + >>> e = doctest.Example('print(1);\nprint(2)\n', '1\n2\n') + >>> e.source, e.want + ('print(1);\nprint(2)\n', '1\n2\n') + + >>> e = doctest.Example('print(1);\nprint(2)', '1\n2\n') + >>> e.source, e.want + ('print(1);\nprint(2)\n', '1\n2\n') + + Empty source string (which should never appear in real examples) + >>> e = doctest.Example('', '') + >>> e.source, e.want + ('\n', '') + +The constructor normalizes the `want` string to end in a newline, +unless it's the empty string: + + >>> e = doctest.Example('print(1)', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print(1)', '1') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print', '') + >>> e.source, e.want + ('print\n', '') + +The constructor normalizes the `exc_msg` string to end in a newline, +unless it's `None`: + + Message spans one line + >>> exc_msg = 'IndexError: pop from an empty list' + >>> e = doctest.Example('[].pop()', '', exc_msg) + >>> e.exc_msg + 'IndexError: pop from an empty list\n' + + >>> exc_msg = 'IndexError: pop from an empty list\n' + >>> e = doctest.Example('[].pop()', '', exc_msg) + >>> e.exc_msg + 'IndexError: pop from an empty list\n' + + Message spans multiple lines + >>> exc_msg = 'ValueError: 1\n 2' + >>> e = doctest.Example('raise ValueError("1\n 2")', '', exc_msg) + >>> e.exc_msg + 'ValueError: 1\n 2\n' + + >>> exc_msg = 'ValueError: 1\n 2\n' + >>> e = doctest.Example('raise ValueError("1\n 2")', '', exc_msg) + >>> e.exc_msg + 'ValueError: 1\n 2\n' + + Empty (but non-None) exception message (which should never appear + in real examples) + >>> exc_msg = '' + >>> e = doctest.Example('raise X()', '', exc_msg) + >>> e.exc_msg + '\n' + +Compare `Example`: + >>> example = doctest.Example('print 1', '1\n') + >>> same_example = doctest.Example('print 1', '1\n') + >>> other_example = doctest.Example('print 42', '42\n') + >>> example == same_example + True + >>> example != same_example + False + >>> hash(example) == hash(same_example) + True + >>> example == other_example + False + >>> example != other_example + True +""" + +def test_DocTest(): r""" +Unit tests for the `DocTest` class. + +DocTest is a collection of examples, extracted from a docstring, along +with information about where the docstring comes from (a name, +filename, and line number). The docstring is parsed by the `DocTest` +constructor: + + >>> docstring = ''' + ... >>> print(12) + ... 12 + ... + ... Non-example text. + ... + ... >>> print('another\\example') + ... another + ... example + ... ''' + >>> globs = {} # globals to run the test in. + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_file', 20) + >>> print(test) + + >>> len(test.examples) + 2 + >>> e1, e2 = test.examples + >>> (e1.source, e1.want, e1.lineno) + ('print(12)\n', '12\n', 1) + >>> (e2.source, e2.want, e2.lineno) + ("print('another\\example')\n", 'another\nexample\n', 6) + +Source information (name, filename, and line number) is available as +attributes on the doctest object: + + >>> (test.name, test.filename, test.lineno) + ('some_test', 'some_file', 20) + +The line number of an example within its containing file is found by +adding the line number of the example and the line number of its +containing test: + + >>> test.lineno + e1.lineno + 21 + >>> test.lineno + e2.lineno + 26 + +If the docstring contains inconsistent leading whitespace in the +expected output of an example, then `DocTest` will raise a ValueError: + + >>> docstring = r''' + ... >>> print('bad\nindentation') + ... bad + ... indentation + ... ''' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 4 of the docstring for some_test has inconsistent leading whitespace: 'indentation' + +If the docstring contains inconsistent leading whitespace on +continuation lines, then `DocTest` will raise a ValueError: + + >>> docstring = r''' + ... >>> print(('bad indentation', + ... ... 2)) + ... ('bad', 'indentation') + ... ''' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 2 of the docstring for some_test has inconsistent leading whitespace: '... 2))' + +If there's no blank space after a PS1 prompt ('>>>'), then `DocTest` +will raise a ValueError: + + >>> docstring = '>>>print(1)\n1' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 1 of the docstring for some_test lacks blank after >>>: '>>>print(1)' + +If there's no blank space after a PS2 prompt ('...'), then `DocTest` +will raise a ValueError: + + >>> docstring = '>>> if 1:\n...print(1)\n1' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 2 of the docstring for some_test lacks blank after ...: '...print(1)' + +Compare `DocTest`: + + >>> docstring = ''' + ... >>> print 12 + ... 12 + ... ''' + >>> test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', 20) + >>> same_test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', 20) + >>> test == same_test + True + >>> test != same_test + False + >>> hash(test) == hash(same_test) + True + >>> docstring = ''' + ... >>> print 42 + ... 42 + ... ''' + >>> other_test = parser.get_doctest(docstring, globs, 'other_test', + ... 'other_file', 10) + >>> test == other_test + False + >>> test != other_test + True + >>> test < other_test + False + >>> other_test < test + True + +Test comparison with lineno None on one side + + >>> no_lineno = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', None) + >>> test.lineno is None + False + >>> no_lineno.lineno is None + True + >>> test < no_lineno + False + >>> no_lineno < test + True + +Compare `DocTestCase`: + + >>> DocTestCase = doctest.DocTestCase + >>> test_case = DocTestCase(test) + >>> same_test_case = DocTestCase(same_test) + >>> other_test_case = DocTestCase(other_test) + >>> test_case == same_test_case + True + >>> test_case != same_test_case + False + >>> hash(test_case) == hash(same_test_case) + True + >>> test == other_test_case + False + >>> test != other_test_case + True + +""" + +class test_DocTestFinder: + def basics(): r""" +Unit tests for the `DocTestFinder` class. + +DocTestFinder is used to extract DocTests from an object's docstring +and the docstrings of its contained objects. It can be used with +modules, functions, classes, methods, staticmethods, classmethods, and +properties. + +Finding Tests in Functions +~~~~~~~~~~~~~~~~~~~~~~~~~~ +For a function whose docstring contains examples, DocTestFinder.find() +will return a single test (for that function's docstring): + + >>> finder = doctest.DocTestFinder() + +We'll simulate a __file__ attr that ends in pyc: + + >>> from test.test_doctest import test_doctest + >>> old = test_doctest.__file__ + >>> test_doctest.__file__ = 'test_doctest.pyc' + + >>> tests = finder.find(sample_func) + + >>> print(tests) # doctest: +ELLIPSIS + [] + +The exact name depends on how test_doctest was invoked, so allow for +leading path components. + + >>> tests[0].filename # doctest: +ELLIPSIS + '...test_doctest.py' + + >>> test_doctest.__file__ = old + + + >>> e = tests[0].examples[0] + >>> (e.source, e.want, e.lineno) + ('print(sample_func(22))\n', '44\n', 3) + +By default, tests are created for objects with no docstring: + + >>> def no_docstring(v): + ... pass + >>> finder.find(no_docstring) + [] + +However, the optional argument `exclude_empty` to the DocTestFinder +constructor can be used to exclude tests for objects with empty +docstrings: + + >>> def no_docstring(v): + ... pass + >>> excl_empty_finder = doctest.DocTestFinder(exclude_empty=True) + >>> excl_empty_finder.find(no_docstring) + [] + +If the function has a docstring with no examples, then a test with no +examples is returned. (This lets `DocTestRunner` collect statistics +about which functions have no tests -- but is that useful? And should +an empty test also be created when there's no docstring?) + + >>> def no_examples(v): + ... ''' no doctest examples ''' + >>> finder.find(no_examples) # doctest: +ELLIPSIS + [] + +Finding Tests in Classes +~~~~~~~~~~~~~~~~~~~~~~~~ +For a class, DocTestFinder will create a test for the class's +docstring, and will recursively explore its contents, including +methods, classmethods, staticmethods, properties, and nested classes. + + >>> finder = doctest.DocTestFinder() + >>> tests = finder.find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +New-style classes are also supported: + + >>> tests = finder.find(SampleNewStyleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 1 SampleNewStyleClass + 1 SampleNewStyleClass.__init__ + 1 SampleNewStyleClass.double + 1 SampleNewStyleClass.get + +Finding Tests in Modules +~~~~~~~~~~~~~~~~~~~~~~~~ +For a module, DocTestFinder will create a test for the class's +docstring, and will recursively explore its contents, including +functions, classes, and the `__test__` dictionary, if it exists: + + >>> # A module + >>> import types + >>> m = types.ModuleType('some_module') + >>> def triple(val): + ... ''' + ... >>> print(triple(11)) + ... 33 + ... ''' + ... return val*3 + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... '__doc__': ''' + ... Module docstring. + ... >>> print('module') + ... module + ... ''', + ... '__test__': { + ... 'd': '>>> print(6)\n6\n>>> print(7)\n7\n', + ... 'c': triple}}) + + >>> finder = doctest.DocTestFinder() + >>> # Use module=test_doctest, to prevent doctest from + >>> # ignoring the objects since they weren't defined in m. + >>> from test.test_doctest import test_doctest + >>> tests = finder.find(m, module=test_doctest) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 1 some_module + 3 some_module.SampleClass + 3 some_module.SampleClass.NestedClass + 1 some_module.SampleClass.NestedClass.__init__ + 1 some_module.SampleClass.__init__ + 1 some_module.SampleClass.a_cached_property + 2 some_module.SampleClass.a_classmethod + 1 some_module.SampleClass.a_property + 1 some_module.SampleClass.a_staticmethod + 1 some_module.SampleClass.double + 1 some_module.SampleClass.get + 3 some_module.SampleClass.setter + 1 some_module.__test__.c + 2 some_module.__test__.d + 1 some_module.sample_func + +However, doctest will ignore imported objects from other modules +(without proper `module=`): + + >>> import types + >>> m = types.ModuleType('poluted_namespace') + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... }) + + >>> finder = doctest.DocTestFinder() + >>> finder.find(m) + [] + +Duplicate Removal +~~~~~~~~~~~~~~~~~ +If a single object is listed twice (under different names), then tests +will only be generated for it once: + + >>> from test.test_doctest import doctest_aliases + >>> assert doctest_aliases.TwoNames.f + >>> assert doctest_aliases.TwoNames.g + >>> tests = excl_empty_finder.find(doctest_aliases) + >>> print(len(tests)) + 2 + >>> print(tests[0].name) + test.test_doctest.doctest_aliases.TwoNames + + TwoNames.f and TwoNames.g are bound to the same object. + We can't guess which will be found in doctest's traversal of + TwoNames.__dict__ first, so we have to allow for either. + + >>> tests[1].name.split('.')[-1] in ['f', 'g'] + True + +Empty Tests +~~~~~~~~~~~ +By default, an object with no doctests doesn't create any tests: + + >>> tests = doctest.DocTestFinder().find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +By default, that excluded objects with no doctests. exclude_empty=False +tells it to include (empty) tests for objects with no doctests. This feature +is really to support backward compatibility in what doctest.master.summarize() +displays. + + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 0 SampleClass.NestedClass.get + 0 SampleClass.NestedClass.square + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +When used with `exclude_empty=False` we are also interested in line numbers +of doctests that are empty. +It used to be broken for quite some time until `bpo-28249`. + + >>> from test.test_doctest import doctest_lineno + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(doctest_lineno) + >>> for t in tests: + ... print('%5s %s' % (t.lineno, t.name)) + None test.test_doctest.doctest_lineno + None test.test_doctest.doctest_lineno.ClassWithACachedProperty + 102 test.test_doctest.doctest_lineno.ClassWithACachedProperty.cached + 22 test.test_doctest.doctest_lineno.ClassWithDocstring + 30 test.test_doctest.doctest_lineno.ClassWithDoctest + None test.test_doctest.doctest_lineno.ClassWithoutDocstring + None test.test_doctest.doctest_lineno.MethodWrapper + 53 test.test_doctest.doctest_lineno.MethodWrapper.classmethod_with_doctest + 39 test.test_doctest.doctest_lineno.MethodWrapper.method_with_docstring + 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest + None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 86 test.test_doctest.doctest_lineno.cached_func_with_doctest + None test.test_doctest.doctest_lineno.cached_func_without_docstring + 4 test.test_doctest.doctest_lineno.func_with_docstring + 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped + 12 test.test_doctest.doctest_lineno.func_with_doctest + None test.test_doctest.doctest_lineno.func_without_docstring + +Turning off Recursion +~~~~~~~~~~~~~~~~~~~~~ +DocTestFinder can be told not to look for tests in contained objects +using the `recurse` flag: + + >>> tests = doctest.DocTestFinder(recurse=False).find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + +Line numbers +~~~~~~~~~~~~ +DocTestFinder finds the line number of each example: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... + ... some text + ... + ... >>> # examples are not created for comments & bare prompts. + ... >>> + ... ... + ... + ... >>> for x in range(10): + ... ... print(x, end=' ') + ... 0 1 2 3 4 5 6 7 8 9 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> [e.lineno for e in test.examples] + [1, 9, 12] +""" + + if int.__doc__: # simple check for --without-doc-strings, skip if lacking + def non_Python_modules(): r""" + +Finding Doctests in Modules Not Written in Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +DocTestFinder can also find doctests in most modules not written in Python. +We'll use builtins as an example, since it almost certainly isn't written in +plain ol' Python and is guaranteed to be available. + + >>> import builtins + >>> tests = doctest.DocTestFinder().find(builtins) + >>> 750 < len(tests) < 800 # approximate number of objects with docstrings # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + True + >>> real_tests = [t for t in tests if len(t.examples) > 0] + >>> len(real_tests) # objects that actually have doctests # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + 14 + >>> for t in real_tests: # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + ... print('{} {}'.format(len(t.examples), t.name)) + ... + 1 builtins.bin + 5 builtins.bytearray.hex + 5 builtins.bytes.hex + 3 builtins.float.as_integer_ratio + 2 builtins.float.fromhex + 2 builtins.float.hex + 1 builtins.hex + 1 builtins.int + 3 builtins.int.as_integer_ratio + 2 builtins.int.bit_count + 2 builtins.int.bit_length + 5 builtins.memoryview.hex + 1 builtins.oct + 1 builtins.zip + +Note here that 'bin', 'oct', and 'hex' are functions; 'float.as_integer_ratio', +'float.hex', and 'int.bit_length' are methods; 'float.fromhex' is a classmethod, +and 'int' is a type. +""" + + +class TestDocTest(unittest.TestCase): + + def test_run(self): + test = ''' + >>> 1 + 1 + 11 + >>> 2 + 3 # doctest: +SKIP + "23" + >>> 5 + 7 + 57 + ''' + + def myfunc(): + pass + myfunc.__doc__ = test + + # test DocTestFinder.run() + test = doctest.DocTestFinder().find(myfunc)[0] + with support.captured_stdout(): + with support.captured_stderr(): + results = doctest.DocTestRunner(verbose=False).run(test) + + # test TestResults + self.assertIsInstance(results, doctest.TestResults) + self.assertEqual(results.failed, 2) + self.assertEqual(results.attempted, 3) + self.assertEqual(results.skipped, 1) + self.assertEqual(tuple(results), (2, 3)) + x, y = results + self.assertEqual((x, y), (2, 3)) + + +class TestDocTestFinder(unittest.TestCase): + + def test_issue35753(self): + # This import of `call` should trigger issue35753 when + # DocTestFinder.find() is called due to inspect.unwrap() failing, + # however with a patched doctest this should succeed. + from unittest.mock import call + dummy_module = types.ModuleType("dummy") + dummy_module.__dict__['inject_call'] = call + finder = doctest.DocTestFinder() + self.assertEqual(finder.find(dummy_module), []) + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + with tempfile.TemporaryDirectory() as parent_dir: + pkg_dir = os.path.join(parent_dir, pkg_name) + os.mkdir(pkg_dir) + sys.path.append(parent_dir) + try: + mod = importlib.import_module(pkg_name) + finally: + import_helper.forget(pkg_name) + sys.path.pop() + + include_empty_finder = doctest.DocTestFinder(exclude_empty=False) + exclude_empty_finder = doctest.DocTestFinder(exclude_empty=True) + + self.assertEqual(len(include_empty_finder.find(mod)), 1) + self.assertEqual(len(exclude_empty_finder.find(mod)), 0) + +def test_DocTestParser(): r""" +Unit tests for the `DocTestParser` class. + +DocTestParser is used to parse docstrings containing doctest examples. + +The `parse` method divides a docstring into examples and intervening +text: + + >>> s = ''' + ... >>> x, y = 2, 3 # no output expected + ... >>> if 1: + ... ... print(x) + ... ... print(y) + ... 2 + ... 3 + ... + ... Some text. + ... >>> x+y + ... 5 + ... ''' + >>> parser = doctest.DocTestParser() + >>> for piece in parser.parse(s): + ... if isinstance(piece, doctest.Example): + ... print('Example:', (piece.source, piece.want, piece.lineno)) + ... else: + ... print(' Text:', repr(piece)) + Text: '\n' + Example: ('x, y = 2, 3 # no output expected\n', '', 1) + Text: '' + Example: ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + Text: '\nSome text.\n' + Example: ('x+y\n', '5\n', 9) + Text: '' + +The `get_examples` method returns just the examples: + + >>> for piece in parser.get_examples(s): + ... print((piece.source, piece.want, piece.lineno)) + ('x, y = 2, 3 # no output expected\n', '', 1) + ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + ('x+y\n', '5\n', 9) + +The `get_doctest` method creates a Test from the examples, along with the +given arguments: + + >>> test = parser.get_doctest(s, {}, 'name', 'filename', lineno=5) + >>> (test.name, test.filename, test.lineno) + ('name', 'filename', 5) + >>> for piece in test.examples: + ... print((piece.source, piece.want, piece.lineno)) + ('x, y = 2, 3 # no output expected\n', '', 1) + ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + ('x+y\n', '5\n', 9) +""" + +class test_DocTestRunner: + def basics(): r""" +Unit tests for the `DocTestRunner` class. + +DocTestRunner is used to run DocTest test cases, and to accumulate +statistics. Here's a simple DocTest case we can use: + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 12 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + +The main DocTestRunner interface is the `run` method, which runs a +given DocTest case in a given namespace (globs). It returns a tuple +`(f,t)`, where `f` is the number of failed tests and `t` is the number +of tried tests. + + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=3) + +If any example produces incorrect output, then the test runner reports +the failure and proceeds to the next example: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 14 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=True).run(test) + ... # doctest: +ELLIPSIS + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 14 + ********************************************************************** + File ..., line 4, in f + Failed example: + print(x) + Expected: + 14 + Got: + 12 + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize +""" + def verbose_flag(): r""" +The `verbose` flag makes the test runner generate more detailed +output: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 12 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + + >>> doctest.DocTestRunner(verbose=True).run(test) + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 12 + ok + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=0, attempted=3) + +If the `verbose` flag is unspecified, then the output will be verbose +iff `-v` appears in sys.argv: + + >>> # Save the real sys.argv list. + >>> old_argv = sys.argv + + >>> # If -v does not appear in sys.argv, then output isn't verbose. + >>> sys.argv = ['test'] + >>> doctest.DocTestRunner().run(test) + TestResults(failed=0, attempted=3) + + >>> # If -v does appear in sys.argv, then output is verbose. + >>> sys.argv = ['test', '-v'] + >>> doctest.DocTestRunner().run(test) + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 12 + ok + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=0, attempted=3) + + >>> # Restore sys.argv + >>> sys.argv = old_argv + +In the remaining examples, the test runner's verbosity will be +explicitly set, to ensure that the test behavior is consistent. + """ + def exceptions(): r""" +Tests of `DocTestRunner`'s exception handling. + +An expected exception is specified with a traceback message. The +lines between the first line and the type/value may be omitted or +replaced with any other string: + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x//0) + ... Traceback (most recent call last): + ... ZeroDivisionError: division by zero + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +An example may not generate output before it raises an exception; if +it does, then the traceback message will not be recognized as +signaling an expected exception, so the example will be reported as an +unexpected exception: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print('pre-exception output', x//0) + ... pre-exception output + ... Traceback (most recent call last): + ... ZeroDivisionError: division by zero + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 4, in f + Failed example: + print('pre-exception output', x//0) + Exception raised: + ... + ZeroDivisionError: division by zero + TestResults(failed=1, attempted=2) + +Exception messages may contain newlines: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('multi\nline\nmessage') + ... Traceback (most recent call last): + ... ValueError: multi + ... line + ... message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +If an exception is expected, but an exception with the wrong type or +message is raised, then it is reported as a failure: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') + ... Traceback (most recent call last): + ... ValueError: wrong message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + raise ValueError('message') + Expected: + Traceback (most recent call last): + ValueError: wrong message + Got: + Traceback (most recent call last): + ... + ValueError: message + TestResults(failed=1, attempted=1) + +However, IGNORE_EXCEPTION_DETAIL can be used to allow a mismatch in the +detail: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... ValueError: wrong message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +IGNORE_EXCEPTION_DETAIL also ignores difference in exception formatting +between Python versions. For example, in Python 2.x, the module path of +the exception is not in the output, but this will fail under Python 3: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') + ... Traceback (most recent call last): + ... HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 4, in f + Failed example: + raise HTTPException('message') + Expected: + Traceback (most recent call last): + HTTPException: message + Got: + Traceback (most recent call last): + ... + http.client.HTTPException: message + TestResults(failed=1, attempted=2) + +But in Python 3 the module path is included, and therefore a test must look +like the following test to succeed in Python 3. But that test will fail under +Python 2. + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') + ... Traceback (most recent call last): + ... http.client.HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +However, with IGNORE_EXCEPTION_DETAIL, the module name of the exception +(or its unexpected absence) will be ignored: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +The module path will be completely ignored, so two different module paths will +still pass if IGNORE_EXCEPTION_DETAIL is given. This is intentional, so it can +be used when exceptions have changed module. + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +But IGNORE_EXCEPTION_DETAIL does not allow a mismatch in the exception type: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... TypeError: wrong type + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + Expected: + Traceback (most recent call last): + TypeError: wrong type + Got: + Traceback (most recent call last): + ... + ValueError: message + TestResults(failed=1, attempted=1) + +If the exception does not have a message, you can still use +IGNORE_EXCEPTION_DETAIL to normalize the modules between Python 2 and 3: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException() #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +Note that a trailing colon doesn't matter either: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException() #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException: + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +If an exception is raised but not expected, then it is reported as an +unexpected exception: + + >>> def f(x): + ... r''' + ... >>> 1//0 + ... 0 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + 1//0 + Exception raised: + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize +""" + def displayhook(): r""" +Test that changing sys.displayhook doesn't matter for doctest. + + >>> import sys + >>> orig_displayhook = sys.displayhook + >>> def my_displayhook(x): + ... print('hi!') + >>> sys.displayhook = my_displayhook + >>> def f(): + ... ''' + ... >>> 3 + ... 3 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> r = doctest.DocTestRunner(verbose=False).run(test) + >>> post_displayhook = sys.displayhook + + We need to restore sys.displayhook now, so that we'll be able to test + results. + + >>> sys.displayhook = orig_displayhook + + Ok, now we can check that everything is ok. + + >>> r + TestResults(failed=0, attempted=1) + >>> post_displayhook is my_displayhook + True +""" + def optionflags(): r""" +Tests of `DocTestRunner`'s option flag handling. + +Several option flags can be used to customize the behavior of the test +runner. These are defined as module constants in doctest, and passed +to the DocTestRunner constructor (multiple constants should be ORed +together). + +The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False +and 1/0: + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... '>>> True\n1\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.DONT_ACCEPT_TRUE_FOR_1 + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + True + Expected: + 1 + Got: + True + TestResults(failed=1, attempted=1) + +The DONT_ACCEPT_BLANKLINE flag disables the match between blank lines +and the '' marker: + + >>> def f(x): + ... '>>> print("a\\n\\nb")\na\n\nb\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.DONT_ACCEPT_BLANKLINE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print("a\n\nb") + Expected: + a + + b + Got: + a + + b + TestResults(failed=1, attempted=1) + +The NORMALIZE_WHITESPACE flag causes all sequences of whitespace to be +treated as equal: + + >>> def f(x): + ... '\n>>> print(1, 2, 3)\n 1 2\n 3' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print(1, 2, 3) + Expected: + 1 2 + 3 + Got: + 1 2 3 + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.NORMALIZE_WHITESPACE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + TestResults(failed=0, attempted=1) + + An example from the docs: + >>> print(list(range(20))) #doctest: +NORMALIZE_WHITESPACE + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] + +The ELLIPSIS flag causes ellipsis marker ("...") in the expected +output to match any substring in the actual output: + + >>> def f(x): + ... '>>> print(list(range(15)))\n[0, 1, 2, ..., 14]\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(15))) + Expected: + [0, 1, 2, ..., 14] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.ELLIPSIS + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + TestResults(failed=0, attempted=1) + + ... also matches nothing: + + >>> if 1: + ... for i in range(100): + ... print(i**2, end=' ') #doctest: +ELLIPSIS + ... print('!') + 0 1...4...9 16 ... 36 49 64 ... 9801 ! + + ... can be surprising; e.g., this test passes: + + >>> if 1: #doctest: +ELLIPSIS + ... for i in range(20): + ... print(i, end=' ') + ... print(20) + 0 1 2 ...1...2...0 + + Examples from the docs: + + >>> print(list(range(20))) # doctest:+ELLIPSIS + [0, 1, ..., 18, 19] + + >>> print(list(range(20))) # doctest: +ELLIPSIS + ... # doctest: +NORMALIZE_WHITESPACE + [0, 1, ..., 18, 19] + +The SKIP flag causes an example to be skipped entirely. I.e., the +example is not run. It can be useful in contexts where doctest +examples serve as both documentation and test cases, and an example +should be included for documentation purposes, but should not be +checked (e.g., because its output is random, or depends on resources +which would be unavailable.) The SKIP flag can also be used for +'commenting out' broken examples. + + >>> import unavailable_resource # doctest: +SKIP + >>> unavailable_resource.do_something() # doctest: +SKIP + >>> unavailable_resource.blow_up() # doctest: +SKIP + Traceback (most recent call last): + ... + UncheckedBlowUpError: Nobody checks me. + + >>> import random + >>> print(random.random()) # doctest: +SKIP + 0.721216923889 + +The REPORT_UDIFF flag causes failures that involve multi-line expected +and actual outputs to be displayed using a unified diff: + + >>> def f(x): + ... r''' + ... >>> print('\n'.join('abcdefg')) + ... a + ... B + ... c + ... d + ... f + ... g + ... h + ... ''' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Expected: + a + B + c + d + f + g + h + Got: + a + b + c + d + e + f + g + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_UDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Differences (unified diff with -expected +actual): + @@ -1,7 +1,7 @@ + a + -B + +b + c + d + +e + f + g + -h + TestResults(failed=1, attempted=1) + +The REPORT_CDIFF flag causes failures that involve multi-line expected +and actual outputs to be displayed using a context diff: + + >>> # Reuse f() from the REPORT_UDIFF example, above. + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_CDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Differences (context diff with expected followed by actual): + *************** + *** 1,7 **** + a + ! B + c + d + f + g + - h + --- 1,7 ---- + a + ! b + c + d + + e + f + g + TestResults(failed=1, attempted=1) + + +The REPORT_NDIFF flag causes failures to use the difflib.Differ algorithm +used by the popular ndiff.py utility. This does intraline difference +marking, as well as interline differences. + + >>> def f(x): + ... r''' + ... >>> print("a b c d e f g h i j k l m") + ... a b c d e f g h i j k 1 m + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_NDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print("a b c d e f g h i j k l m") + Differences (ndiff with -expected +actual): + - a b c d e f g h i j k 1 m + ? ^ + + a b c d e f g h i j k l m + ? + ++ ^ + TestResults(failed=1, attempted=1) + +The REPORT_ONLY_FIRST_FAILURE suppresses result output after the first +failing example: + + >>> def f(x): + ... r''' + ... >>> print(1) # first success + ... 1 + ... >>> print(2) # first failure + ... 200 + ... >>> print(3) # second failure + ... 300 + ... >>> print(4) # second success + ... 4 + ... >>> print(5) # third failure + ... 500 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=3, attempted=5) + +However, output from `report_start` is not suppressed: + + >>> doctest.DocTestRunner(verbose=True, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + Trying: + print(1) # first success + Expecting: + 1 + ok + Trying: + print(2) # first failure + Expecting: + 200 + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=3, attempted=5) + +The FAIL_FAST flag causes the runner to exit after the first failing example, +so subsequent examples are not even attempted: + + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +Specifying both FAIL_FAST and REPORT_ONLY_FIRST_FAILURE is equivalent to +FAIL_FAST only: + + >>> flags = doctest.FAIL_FAST | doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +For the purposes of both REPORT_ONLY_FIRST_FAILURE and FAIL_FAST, unexpected +exceptions count as failures: + + >>> def f(x): + ... r''' + ... >>> print(1) # first success + ... 1 + ... >>> raise ValueError(2) # first failure + ... 200 + ... >>> print(3) # second failure + ... 300 + ... >>> print(4) # second success + ... 4 + ... >>> print(5) # third failure + ... 500 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + raise ValueError(2) # first failure + Exception raised: + ... + ValueError: 2 + TestResults(failed=3, attempted=5) + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + raise ValueError(2) # first failure + Exception raised: + ... + ValueError: 2 + TestResults(failed=1, attempted=2) + +New option flags can also be registered, via register_optionflag(). Here +we reach into doctest's internals a bit. + + >>> unlikely = "UNLIKELY_OPTION_NAME" + >>> unlikely in doctest.OPTIONFLAGS_BY_NAME + False + >>> new_flag_value = doctest.register_optionflag(unlikely) + >>> unlikely in doctest.OPTIONFLAGS_BY_NAME + True + +Before 2.4.4/2.5, registering a name more than once erroneously created +more than one flag value. Here we verify that's fixed: + + >>> redundant_flag_value = doctest.register_optionflag(unlikely) + >>> redundant_flag_value == new_flag_value + True + +Clean up. + >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] + >>> _colorize.COLORIZE = save_colorize + + """ + + def option_directives(): r""" +Tests of `DocTestRunner`'s option directive mechanism. + +Option directives can be used to turn option flags on or off for a +single example. To turn an option on for an example, follow that +example with a comment of the form ``# doctest: +OPTION``: + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): r''' + ... >>> print(list(range(10))) # should fail: no ellipsis + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +To turn an option off for an example, follow that example with a +comment of the form ``# doctest: -OPTION``: + + >>> def f(x): r''' + ... >>> print(list(range(10))) + ... [0, 1, ..., 9] + ... + ... >>> # should fail: no ellipsis + ... >>> print(list(range(10))) # doctest: -ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False, + ... optionflags=doctest.ELLIPSIS).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 6, in f + Failed example: + print(list(range(10))) # doctest: -ELLIPSIS + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +Option directives affect only the example that they appear with; they +do not change the options for surrounding examples: + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail: no ellipsis + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # Should fail: no ellipsis + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ********************************************************************** + File ..., line 8, in f + Failed example: + print(list(range(10))) # Should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=2, attempted=3) + +Multiple options may be modified by a single option directive. They +may be separated by whitespace, commas, or both: + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +The option directive may be put on the line following the source, as +long as a continuation prompt is used: + + >>> def f(x): r''' + ... >>> print(list(range(10))) + ... ... # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +For examples with multi-line source, the option directive may appear +at the end of any line: + + >>> def f(x): r''' + ... >>> for x in range(10): # doctest: +ELLIPSIS + ... ... print(' ', x, end='', sep='') + ... 0 1 2 ... 9 + ... + ... >>> for x in range(10): + ... ... print(' ', x, end='', sep='') # doctest: +ELLIPSIS + ... 0 1 2 ... 9 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +If more than one line of an example with multi-line source has an +option directive, then they are combined: + + >>> def f(x): r''' + ... Should fail (option directive not on the last line): + ... >>> for x in range(10): # doctest: +ELLIPSIS + ... ... print(x, end=' ') # doctest: +NORMALIZE_WHITESPACE + ... 0 1 2...9 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +It is an error to have a comment of the form ``# doctest:`` that is +*not* followed by words of the form ``+OPTION`` or ``-OPTION``, where +``OPTION`` is an option that has been registered with +`register_option`: + + >>> # Error: Option not registered + >>> s = '>>> print(12) #doctest: +BADOPTION' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 1 of the doctest for s has an invalid option: '+BADOPTION' + + >>> # Error: No + or - prefix + >>> s = '>>> print(12) #doctest: ELLIPSIS' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 1 of the doctest for s has an invalid option: 'ELLIPSIS' + +It is an error to use an option directive on a line that contains no +source: + + >>> s = '>>> # doctest: +ELLIPSIS' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' + + >>> _colorize.COLORIZE = save_colorize +""" + +def test_testsource(): r""" +Unit tests for `testsource()`. + +The testsource() function takes a module and a name, finds the (first) +test with that name in that module, and converts it to a script. The +example code is converted to regular Python code. The surrounding +words and expected output are converted to comments: + + >>> from test.test_doctest import test_doctest + >>> name = 'test.test_doctest.test_doctest.sample_func' + >>> print(doctest.testsource(test_doctest, name)) + # Blah blah + # + print(sample_func(22)) + # Expected: + ## 44 + # + # Yee ha! + + + >>> name = 'test.test_doctest.test_doctest.SampleNewStyleClass' + >>> print(doctest.testsource(test_doctest, name)) + print('1\n2\n3') + # Expected: + ## 1 + ## 2 + ## 3 + + + >>> name = 'test.test_doctest.test_doctest.SampleClass.a_classmethod' + >>> print(doctest.testsource(test_doctest, name)) + print(SampleClass.a_classmethod(10)) + # Expected: + ## 12 + print(SampleClass(0).a_classmethod(10)) + # Expected: + ## 12 + +""" + +def test_debug(): r""" + +Create a docstring that we want to debug: + + >>> s = ''' + ... >>> x = 12 + ... >>> print(x) + ... 12 + ... ''' + +Create some fake stdin input, to feed to the debugger: + + >>> from test.support.pty_helper import FakeInput + >>> real_stdin = sys.stdin + >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) + +Run the debugger on the docstring, and then restore sys.stdin. + + >>> try: doctest.debug_src(s) + ... finally: sys.stdin = real_stdin + > (1)() + (Pdb) next + 12 + --Return-- + > (1)()->None + (Pdb) print(x) + 12 + (Pdb) continue + +""" + +if not hasattr(sys, 'gettrace') or not sys.gettrace(): + def test_pdb_set_trace(): + """Using pdb.set_trace from a doctest. + + You can use pdb.set_trace from a doctest. To do so, you must + retrieve the set_trace function from the pdb module at the time + you use it. The doctest module changes sys.stdout so that it can + capture program output. It also temporarily replaces pdb.set_trace + with a version that restores stdout. This is necessary for you to + see debugger output. + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> doc = ''' + ... >>> x = 42 + ... >>> raise Exception('clé') + ... Traceback (most recent call last): + ... Exception: clé + ... >>> import pdb; pdb.set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) + >>> runner = doctest.DocTestRunner(verbose=False) + + To demonstrate this, we'll create a fake standard input that + captures our debugger input: + + >>> from test.support.pty_helper import FakeInput + >>> real_stdin = sys.stdin + >>> sys.stdin = FakeInput([ + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: runner.run(test) # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + ... finally: sys.stdin = real_stdin + > (1)() + -> import pdb; pdb.set_trace() + (Pdb) print(x) + 42 + (Pdb) continue + TestResults(failed=0, attempted=3) + + You can also put pdb.set_trace in a function called from a test: + + >>> def calls_set_trace(): + ... y=2 + ... import pdb; pdb.set_trace() + + >>> doc = ''' + ... >>> x=1 + ... >>> calls_set_trace() + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = FakeInput([ + ... 'print(y)', # print data defined in the function + ... 'up', # out of function + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + > (3)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) print(y) + 2 + (Pdb) up + > (1)() + -> calls_set_trace() + (Pdb) print(x) + 1 + (Pdb) continue + TestResults(failed=0, attempted=2) + + During interactive debugging, source code is shown, even for + doctest examples: + + >>> doc = ''' + ... >>> def f(x): + ... ... g(x*2) + ... >>> def g(x): + ... ... print(x+3) + ... ... import pdb; pdb.set_trace() + ... >>> f(3) + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = FakeInput([ + ... 'step', # return event of g + ... 'list', # list source from example 2 + ... 'next', # return from g() + ... 'list', # list source from example 1 + ... 'next', # return from f() + ... 'list', # list source from example 3 + ... 'continue', # stop debugging + ... '']) + >>> try: runner.run(test) # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + ... finally: sys.stdin = real_stdin + ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step + --Return-- + > (3)g()->None + -> import pdb; pdb.set_trace() + (Pdb) list + 1 def g(x): + 2 print(x+3) + 3 -> import pdb; pdb.set_trace() + [EOF] + (Pdb) next + --Return-- + > (2)f()->None + -> g(x*2) + (Pdb) list + 1 def f(x): + 2 -> g(x*2) + [EOF] + (Pdb) next + --Return-- + > (1)()->None + -> f(3) + (Pdb) list + 1 -> f(3) + [EOF] + (Pdb) continue + ********************************************************************** + File "foo-bar@baz.py", line 7, in foo-bar@baz + Failed example: + f(3) + Expected nothing + Got: + 9 + TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize + """ + + def test_pdb_set_trace_nested(): + """This illustrates more-demanding use of set_trace with nested functions. + + >>> class C(object): + ... def calls_set_trace(self): + ... y = 1 + ... import pdb; pdb.set_trace() + ... self.f1() + ... y = 2 + ... def f1(self): + ... x = 1 + ... self.f2() + ... x = 2 + ... def f2(self): + ... z = 1 + ... z = 2 + + >>> calls_set_trace = C().calls_set_trace + + >>> doc = ''' + ... >>> a = 1 + ... >>> calls_set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> runner = doctest.DocTestRunner(verbose=False) + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> from test.support.pty_helper import FakeInput + >>> real_stdin = sys.stdin + >>> sys.stdin = FakeInput([ + ... 'step', + ... 'print(y)', # print data defined in the function + ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', + ... 'up', 'print(x)', + ... 'up', 'print(y)', + ... 'up', 'print(foo)', + ... 'continue', # stop debugging + ... '']) + + >>> try: # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + ... # doctest: +REPORT_NDIFF + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step + > (5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) step + --Call-- + > (7)f1() + -> def f1(self): + (Pdb) step + > (8)f1() + -> x = 1 + (Pdb) step + > (9)f1() + -> self.f2() + (Pdb) step + --Call-- + > (11)f2() + -> def f2(self): + (Pdb) step + > (12)f2() + -> z = 1 + (Pdb) step + > (13)f2() + -> z = 2 + (Pdb) print(z) + 1 + (Pdb) up + > (9)f1() + -> self.f2() + (Pdb) print(x) + 1 + (Pdb) up + > (5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) up + > (1)() + -> calls_set_trace() + (Pdb) print(foo) + *** NameError: name 'foo' is not defined + (Pdb) continue + TestResults(failed=0, attempted=2) + """ + +def test_DocTestSuite(): + """DocTestSuite creates a unittest test suite from a doctest. + + We create a Suite by providing a module. A module can be provided + by passing a module object: + + >>> import unittest + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> for tst, _ in result.failures: + ... print(tst) + bad (test.test_doctest.sample_doctest.__test__) + foo (test.test_doctest.sample_doctest) + test_silly_setup (test.test_doctest.sample_doctest) + y_is_one (test.test_doctest.sample_doctest) + + We can also supply the module by name: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') + >>> result = suite.run(unittest.TestResult()) + >>> result + + + The module need not contain any doctest examples: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_doctests') + >>> suite.run(unittest.TestResult()) + + + The module need not contain any docstrings either: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings') + >>> suite.run(unittest.TestResult()) + + + If all examples in a docstring are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 2 + >>> for tst, _ in result.skipped: + ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) + single_skip (test.test_doctest.sample_doctest_skip) + >>> for tst, _ in result.failures: + ... print(tst) + no_skip_fail (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) + + We can use the current module: + + >>> suite = test.test_doctest.sample_doctest.test_suite() + >>> suite.run(unittest.TestResult()) + + + We can also provide a DocTestFinder: + + >>> finder = doctest.DocTestFinder() + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', + ... test_finder=finder) + >>> suite.run(unittest.TestResult()) + + + The DocTestFinder need not return any tests: + + >>> finder = doctest.DocTestFinder() + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings', + ... test_finder=finder) + >>> suite.run(unittest.TestResult()) + + + We can supply global variables. If we pass globs, they will be + used instead of the module globals. Here we'll pass an empty + globals, triggering an extra error: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) + >>> suite.run(unittest.TestResult()) + + + Alternatively, we can provide extra globals. Here we'll make an + error go away by providing an extra global variable: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', + ... extraglobs={'y': 1}) + >>> suite.run(unittest.TestResult()) + + + You can pass option flags. Here we'll cause an extra error + by disabling the blank-line feature: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) + >>> suite.run(unittest.TestResult()) + + + You can supply setUp and tearDown functions: + + >>> def setUp(t): + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True + + >>> def tearDown(t): + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup + + Here, we installed a silly variable that the test expects: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', + ... setUp=setUp, tearDown=tearDown) + >>> suite.run(unittest.TestResult()) + + + But the tearDown restores sanity: + + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup + Traceback (most recent call last): + ... + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' + + The setUp and tearDown functions are passed test objects. Here + we'll use the setUp function to supply the missing variable y: + + >>> def setUp(test): + ... test.globs['y'] = 1 + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) + >>> suite.run(unittest.TestResult()) + + + Here, we didn't need to use a tearDown function because we + modified the test globals, which are a copy of the + sample_doctest module dictionary. The test globals are + automatically cleared for us after a test. + """ + +def test_DocTestSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> import test.test_doctest.sample_doctest_errors as mod + >>> suite = doctest.DocTestSuite(mod) + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors + File "...sample_doctest_errors.py", line 0, in sample_doctest_errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.failures[1][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line unknown line number, in bad + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.failures[2][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors + File "...sample_doctest_errors.py", line 14, in errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + + >>> print(result.failures[3][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error + File "...sample_doctest_errors.py", line 29, in syntax_error + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + """ + +def test_DocFileSuite(): + """We can test tests found in text files using a DocFileSuite. + + We create a suite by providing the names of one or more text + files that include examples: + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt') + >>> suite.run(unittest.TestResult()) + + + The test files are looked for in the directory containing the + calling module. A package keyword argument can be provided to + specify a different relative location. + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt', + ... package='test.test_doctest') + >>> suite.run(unittest.TestResult()) + + + '/' should be used as a path separator. It will be converted + to a native separator at run time: + + >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') + >>> suite.run(unittest.TestResult()) + + + If DocFileSuite is used from an interactive session, then files + are resolved relative to the directory of sys.argv[0]: + + >>> import types, os.path + >>> from test.test_doctest import test_doctest + >>> save_argv = sys.argv + >>> sys.argv = [test_doctest.__file__] + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... package=types.ModuleType('__main__')) + >>> sys.argv = save_argv + + By setting `module_relative=False`, os-specific paths may be + used (including absolute paths and paths relative to the + working directory): + + >>> # Get the absolute path of the test package. + >>> test_doctest_path = os.path.abspath(test_doctest.__file__) + >>> test_pkg_path = os.path.split(test_doctest_path)[0] + + >>> # Use it to find the absolute path of test_doctest.txt. + >>> test_file = os.path.join(test_pkg_path, 'test_doctest.txt') + + >>> suite = doctest.DocFileSuite(test_file, module_relative=False) + >>> suite.run(unittest.TestResult()) + + + It is an error to specify `package` when `module_relative=False`: + + >>> suite = doctest.DocFileSuite(test_file, module_relative=False, + ... package='test') + Traceback (most recent call last): + ValueError: Package may only be specified for module-relative paths. + + If all examples in a file are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest4.txt', + ... 'test_doctest_skip.txt', + ... 'test_doctest_skip2.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 1 + >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS + ... print('=', tst) + = ...test_doctest_skip.txt + + You can specify initial global variables: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt', + ... globs={'favorite_color': 'blue'}) + >>> suite.run(unittest.TestResult()) + + + In this case, we supplied a missing favorite color. You can + provide doctest options: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE, + ... globs={'favorite_color': 'blue'}) + >>> suite.run(unittest.TestResult()) + + + And, you can provide setUp and tearDown functions: + + >>> def setUp(t): + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True + + >>> def tearDown(t): + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup + + Here, we installed a silly variable that the test expects: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt', + ... setUp=setUp, tearDown=tearDown) + >>> suite.run(unittest.TestResult()) + + + But the tearDown restores sanity: + + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup + Traceback (most recent call last): + ... + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' + + The setUp and tearDown functions are passed test objects. + Here, we'll use a setUp function to set the favorite color in + test_doctest.txt: + + >>> def setUp(test): + ... test.globs['favorite_color'] = 'blue' + + >>> suite = doctest.DocFileSuite('test_doctest.txt', setUp=setUp) + >>> suite.run(unittest.TestResult()) + + + Here, we didn't need to use a tearDown function because we + modified the test globals. The test globals are + automatically cleared for us after a test. + + Tests in a file run using `DocFileSuite` can also access the + `__file__` global, which is set to the name of the file + containing the tests: + + >>> suite = doctest.DocFileSuite('test_doctest3.txt') + >>> suite.run(unittest.TestResult()) + + + If the tests contain non-ASCII characters, we have to specify which + encoding the file is encoded with. We do so by using the `encoding` + parameter: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest2.txt', + ... 'test_doctest4.txt', + ... encoding='utf-8') + >>> suite.run(unittest.TestResult()) + + """ + +def test_DocFileSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test_doctest_errors.txt + File "...test_doctest_errors.txt", line 0 + + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + """ + +def test_trailing_space_in_test(): + """ + Trailing spaces in expected output are significant: + + >>> x, y = 'foo', '' + >>> print(x, y) + foo \n + """ + +class Wrapper: + def __init__(self, func): + self.func = func + functools.update_wrapper(self, func) + + def __call__(self, *args, **kwargs): + self.func(*args, **kwargs) + +@Wrapper +def wrapped(): + """ + Docstrings in wrapped functions must be detected as well. + + >>> 'one other test' + 'one other test' + """ + +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + +def test_unittest_reportflags(): + """Default unittest reporting flags can be set to control reporting + + Here, we'll set the REPORT_ONLY_FIRST_FAILURE option so we see + only the first failure of each test. First, we'll look at the + output without the flag. The file test_doctest.txt file has two + tests. They both fail if blank lines are disabled: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) + >>> import unittest + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test_doctest.txt + ... + Failed example: + favorite_color + ... + Failed example: + if 1: + ... + + Note that we see both failures displayed. + + >>> old = doctest.set_unittest_reportflags( + ... doctest.REPORT_ONLY_FIRST_FAILURE) + + Now, when we run the test: + + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test_doctest.txt + ... + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + + + We get only the first failure. + + If we give any reporting options when we set up the tests, + however: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE | doctest.REPORT_NDIFF) + + Then the default eporting options are ignored: + + >>> result = suite.run(unittest.TestResult()) + >>> result + + + *NOTE*: These doctest are intentionally not placed in raw string to depict + the trailing whitespace using `\x20` in the diff below. + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test_doctest.txt + ... + Failed example: + favorite_color + ... + Failed example: + if 1: + print('a') + print() + print('b') + Differences (ndiff with -expected +actual): + a + - + +\x20 + b + + + + Test runners can restore the formatting flags after they run: + + >>> ignored = doctest.set_unittest_reportflags(old) + + """ + +def test_testfile(): r""" +Tests for the `testfile()` function. This function runs all the +doctest examples in a given file. In its simple invocation, it is +called with the name of a file, which is taken to be relative to the +calling module. The return value is (#failures, #tests). + +We don't want color or `-v` in sys.argv for these tests. + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> save_argv = sys.argv + >>> if '-v' in sys.argv: + ... sys.argv = [arg for arg in save_argv if arg != '-v'] + + + >>> doctest.testfile('test_doctest.txt') # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + ********************************************************************** + 1 item had failures: + 1 of 2 in test_doctest.txt + ***Test Failed*** 1 failure. + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +(Note: we'll be clearing doctest.master after each call to +`doctest.testfile`, to suppress warnings about multiple tests with the +same name.) + +Globals may be specified with the `globs` and `extraglobs` parameters: + + >>> globs = {'favorite_color': 'blue'} + >>> doctest.testfile('test_doctest.txt', globs=globs) + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + + >>> extraglobs = {'favorite_color': 'red'} + >>> doctest.testfile('test_doctest.txt', globs=globs, + ... extraglobs=extraglobs) # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Expected: + 'blue' + Got: + 'red' + ********************************************************************** + 1 item had failures: + 1 of 2 in test_doctest.txt + ***Test Failed*** 1 failure. + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The file may be made relative to a given module or package, using the +optional `module_relative` parameter: + + >>> doctest.testfile('test_doctest.txt', globs=globs, + ... module_relative='test') + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +Verbosity can be increased with the optional `verbose` parameter: + + >>> doctest.testfile('test_doctest.txt', globs=globs, verbose=True) + Trying: + favorite_color + Expecting: + 'blue' + ok + Trying: + if 1: + print('a') + print() + print('b') + Expecting: + a + + b + ok + 1 item passed all tests: + 2 tests in test_doctest.txt + 2 tests in 1 item. + 2 passed. + Test passed. + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +The name of the test may be specified with the optional `name` +parameter: + + >>> doctest.testfile('test_doctest.txt', name='newname') + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in newname + ... + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The summary report may be suppressed with the optional `report` +parameter: + + >>> doctest.testfile('test_doctest.txt', report=False) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The optional keyword argument `raise_on_error` can be used to raise an +exception on the first error (which may be useful for postmortem +debugging): + + >>> doctest.testfile('test_doctest.txt', raise_on_error=True) + ... # doctest: +ELLIPSIS + Traceback (most recent call last): + doctest.UnexpectedException: ... + >>> doctest.master = None # Reset master. + +If the tests contain non-ASCII characters, the tests might fail, since +it's unknown which encoding is used. The encoding can be specified +using the optional keyword argument `encoding`: + + >>> doctest.testfile('test_doctest4.txt', encoding='latin-1') # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 7, in test_doctest4.txt + Failed example: + '...' + Expected: + 'f\xf6\xf6' + Got: + 'f\xc3\xb6\xc3\xb6' + ********************************************************************** + ... + ********************************************************************** + 1 item had failures: + 2 of 2 in test_doctest4.txt + ***Test Failed*** 2 failures. + TestResults(failed=2, attempted=2) + >>> doctest.master = None # Reset master. + + >>> doctest.testfile('test_doctest4.txt', encoding='utf-8') + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +Test the verbose output: + + >>> doctest.testfile('test_doctest4.txt', encoding='utf-8', verbose=True) + Trying: + 'föö' + Expecting: + 'f\xf6\xf6' + ok + Trying: + 'bąr' + Expecting: + 'b\u0105r' + ok + 1 item passed all tests: + 2 tests in test_doctest4.txt + 2 tests in 1 item. + 2 passed. + Test passed. + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + >>> sys.argv = save_argv + >>> _colorize.COLORIZE = save_colorize +""" + +def test_testfile_errors(): r""" +Tests for error reporting in the testfile() function. + + >>> doctest.testfile('test_doctest_errors.txt', verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 1 item had failures: + 4 of 5 in test_doctest_errors.txt + ***Test Failed*** 4 failures. + TestResults(failed=4, attempted=5) +""" + +class TestImporter(importlib.abc.MetaPathFinder): + + def find_spec(self, fullname, path, target=None): + return importlib.util.spec_from_file_location(fullname, path, loader=self) + + def get_data(self, path): + with open(path, mode='rb') as f: + return f.read() + + def exec_module(self, module): + raise ImportError + + def create_module(self, spec): + return None + +class TestHook: + + def __init__(self, pathdir): + self.sys_path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path.append(pathdir) + sys.path_importer_cache.clear() + self.modules_before = sys.modules.copy() + self.importer = TestImporter() + sys.meta_path.append(self.importer) + + def remove(self): + sys.path[:] = self.sys_path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + sys.modules.clear() + sys.modules.update(self.modules_before) + + +@contextlib.contextmanager +def test_hook(pathdir): + hook = TestHook(pathdir) + try: + yield hook + finally: + hook.remove() + + +def test_lineendings(): r""" +*nix systems use \n line endings, while Windows systems use \r\n, and +old Mac systems used \r, which Python still recognizes as a line ending. Python +handles this using universal newline mode for reading files. Let's make +sure doctest does so (issue 8473) by creating temporary test files using each +of the three line disciplines. At least one will not match either the universal +newline \n or os.linesep for the platform the test is run on. + +Windows line endings first: + + >>> import tempfile, os + >>> fn = tempfile.mktemp() + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\r\n\r\n >>> x = 1 + 1\r\n\r\nDone.\r\n') + 35 + >>> doctest.testfile(fn, module_relative=False, verbose=False) + TestResults(failed=0, attempted=1) + >>> os.remove(fn) + +And now *nix line endings: + + >>> fn = tempfile.mktemp() + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\n\n >>> x = 1 + 1\n\nDone.\n') + 30 + >>> doctest.testfile(fn, module_relative=False, verbose=False) + TestResults(failed=0, attempted=1) + >>> os.remove(fn) + +And finally old Mac line endings: + + >>> fn = tempfile.mktemp() + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\r\r >>> x = 1 + 1\r\rDone.\r') + 30 + >>> doctest.testfile(fn, module_relative=False, verbose=False) + TestResults(failed=0, attempted=1) + >>> os.remove(fn) + +Now we test with a package loader that has a get_data method, since that +bypasses the standard universal newline handling so doctest has to do the +newline conversion itself; let's make sure it does so correctly (issue 1812). +We'll write a file inside the package that has all three kinds of line endings +in it, and use a package hook to install a custom loader; on any platform, +at least one of the line endings will raise a ValueError for inconsistent +whitespace if doctest does not correctly do the newline conversion. + + >>> from test.support import os_helper + >>> import shutil + >>> dn = tempfile.mkdtemp() + >>> pkg = os.path.join(dn, "doctest_testpkg") + >>> os.mkdir(pkg) + >>> os_helper.create_empty_file(os.path.join(pkg, "__init__.py")) + >>> fn = os.path.join(pkg, "doctest_testfile.txt") + >>> with open(fn, 'wb') as f: + ... f.write( + ... b'Test:\r\n\r\n' + ... b' >>> x = 1 + 1\r\n\r\n' + ... b'Done.\r\n' + ... b'Test:\n\n' + ... b' >>> x = 1 + 1\n\n' + ... b'Done.\n' + ... b'Test:\r\r' + ... b' >>> x = 1 + 1\r\r' + ... b'Done.\r' + ... ) + 95 + >>> with test_hook(dn): + ... doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) + TestResults(failed=0, attempted=3) + >>> shutil.rmtree(dn) + +""" + +def test_testmod(): r""" +Tests for the testmod function. More might be useful, but for now we're just +testing the case raised by Issue 6195, where trying to doctest a C module would +fail with a UnicodeDecodeError because doctest tried to read the "source" lines +out of the binary module. + + >>> import unicodedata + >>> doctest.testmod(unicodedata, verbose=False) + TestResults(failed=0, attempted=0) +""" + +def test_testmod_errors(): r""" +Tests for error reporting in the testmod() function. + + >>> import test.test_doctest.sample_doctest_errors as mod + >>> doctest.testmod(mod, verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + ********************************************************************** + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 4 items had failures: + 2 of 2 in test.test_doctest.sample_doctest_errors + 2 of 2 in test.test_doctest.sample_doctest_errors.__test__.bad + 4 of 5 in test.test_doctest.sample_doctest_errors.errors + 1 of 1 in test.test_doctest.sample_doctest_errors.syntax_error + ***Test Failed*** 9 failures. + TestResults(failed=9, attempted=10) +""" + +try: + os.fsencode("foo-bär@baz.py") + supports_unicode = True +except UnicodeEncodeError: + # Skip the test: the filesystem encoding is unable to encode the filename + supports_unicode = False + +if supports_unicode: + def test_unicode(): """ +Check doctest with a non-ascii filename: + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> doc = ''' + ... >>> raise Exception('clé') + ... ''' + ... + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(doc, {}, "foo-bär@baz", "foo-bär@baz.py", 0) + >>> test + + >>> runner = doctest.DocTestRunner(verbose=False) + >>> runner.run(test) # doctest: +ELLIPSIS + ********************************************************************** + File "foo-bär@baz.py", line 2, in foo-bär@baz + Failed example: + raise Exception('clé') + Exception raised: + Traceback (most recent call last): + File "", line 1, in + raise Exception('clé') + Exception: clé + TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize + """ + + +@doctest_skip_if(not support.has_subprocess_support) +def test_CLI(): r""" +The doctest module can be used to run doctests against an arbitrary file. +These tests test this CLI functionality. + +We'll use the support module's script_helpers for this, and write a test files +to a temp dir to run the command against. Due to a current limitation in +script_helpers, though, we need a little utility function to turn the returned +output into something we can doctest against: + + >>> def normalize(s): + ... return '\n'.join(s.decode().splitlines()) + +With those preliminaries out of the way, we'll start with a file with two +simple tests and no errors. We'll run both the unadorned doctest command, and +the verbose version, and then check the output: + + >>> from test.support import script_helper + >>> from test.support.os_helper import temp_dir + >>> with temp_dir() as tmpdir: + ... fn = os.path.join(tmpdir, 'myfile.doc') + ... with open(fn, 'w', encoding='utf-8') as f: + ... _ = f.write('This is a very simple test file.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "a"\n') + ... _ = f.write(" 'a'\n") + ... _ = f.write('\n') + ... _ = f.write('And that is it.\n') + ... rc1, out1, err1 = script_helper.assert_python_ok( + ... '-m', 'doctest', fn) + ... rc2, out2, err2 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-v', fn) + +With no arguments and passing tests, we should get no output: + + >>> rc1, out1, err1 + (0, b'', b'') + +With the verbose flag, we should see the test output, but no error output: + + >>> rc2, err2 + (0, b'') + >>> print(normalize(out2)) + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "a" + Expecting: + 'a' + ok + 1 item passed all tests: + 2 tests in myfile.doc + 2 tests in 1 item. + 2 passed. + Test passed. + +Now we'll write a couple files, one with three tests, the other a python module +with two tests, both of the files having "errors" in the tests that can be made +non-errors by applying the appropriate doctest options to the run (ELLIPSIS in +the first file, NORMALIZE_WHITESPACE in the second). This combination will +allow thoroughly testing the -f and -o flags, as well as the doctest command's +ability to process more than one file on the command line and, since the second +file ends in '.py', its handling of python module files (as opposed to straight +text files). + + >>> from test.support import script_helper + >>> from test.support.os_helper import temp_dir + >>> with temp_dir() as tmpdir: + ... fn = os.path.join(tmpdir, 'myfile.doc') + ... with open(fn, 'w', encoding="utf-8") as f: + ... _ = f.write('This is another simple test file.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "abcdef"\n') + ... _ = f.write(" 'a...f'\n") + ... _ = f.write(' >>> "ajkml"\n') + ... _ = f.write(" 'a...l'\n") + ... _ = f.write('\n') + ... _ = f.write('And that is it.\n') + ... fn2 = os.path.join(tmpdir, 'myfile2.py') + ... with open(fn2, 'w', encoding='utf-8') as f: + ... _ = f.write('def test_func():\n') + ... _ = f.write(' \"\"\"\n') + ... _ = f.write(' This is simple python test function.\n') + ... _ = f.write(' >>> 1 + 1\n') + ... _ = f.write(' 2\n') + ... _ = f.write(' >>> "abc def"\n') + ... _ = f.write(" 'abc def'\n") + ... _ = f.write("\n") + ... _ = f.write(' \"\"\"\n') + ... rc1, out1, err1 = script_helper.assert_python_failure( + ... '-m', 'doctest', fn, fn2) + ... rc2, out2, err2 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-o', 'ELLIPSIS', fn) + ... rc3, out3, err3 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-o', 'ELLIPSIS', + ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2) + ... rc4, out4, err4 = script_helper.assert_python_failure( + ... '-m', 'doctest', '-f', fn, fn2) + ... rc5, out5, err5 = script_helper.assert_python_ok( + ... '-m', 'doctest', '-v', '-o', 'ELLIPSIS', + ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2) + +Our first test run will show the errors from the first file (doctest stops if a +file has errors). Note that doctest test-run error output appears on stdout, +not stderr: + + >>> rc1, err1 + (1, b'') + >>> print(normalize(out1)) # doctest: +ELLIPSIS + ********************************************************************** + File "...myfile.doc", line 4, in myfile.doc + Failed example: + "abcdef" + Expected: + 'a...f' + Got: + 'abcdef' + ********************************************************************** + File "...myfile.doc", line 6, in myfile.doc + Failed example: + "ajkml" + Expected: + 'a...l' + Got: + 'ajkml' + ********************************************************************** + 1 item had failures: + 2 of 3 in myfile.doc + ***Test Failed*** 2 failures. + +With -o ELLIPSIS specified, the second run, against just the first file, should +produce no errors, and with -o NORMALIZE_WHITESPACE also specified, neither +should the third, which ran against both files: + + >>> rc2, out2, err2 + (0, b'', b'') + >>> rc3, out3, err3 + (0, b'', b'') + +The fourth run uses FAIL_FAST, so we should see only one error: + + >>> rc4, err4 + (1, b'') + >>> print(normalize(out4)) # doctest: +ELLIPSIS + ********************************************************************** + File "...myfile.doc", line 4, in myfile.doc + Failed example: + "abcdef" + Expected: + 'a...f' + Got: + 'abcdef' + ********************************************************************** + 1 item had failures: + 1 of 2 in myfile.doc + ***Test Failed*** 1 failure. + +The fifth test uses verbose with the two options, so we should get verbose +success output for the tests in both files: + + >>> rc5, err5 + (0, b'') + >>> print(normalize(out5)) + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "abcdef" + Expecting: + 'a...f' + ok + Trying: + "ajkml" + Expecting: + 'a...l' + ok + 1 item passed all tests: + 3 tests in myfile.doc + 3 tests in 1 item. + 3 passed. + Test passed. + Trying: + 1 + 1 + Expecting: + 2 + ok + Trying: + "abc def" + Expecting: + 'abc def' + ok + 1 item had no tests: + myfile2 + 1 item passed all tests: + 2 tests in myfile2.test_func + 2 tests in 2 items. + 2 passed. + Test passed. + +We should also check some typical error cases. + +Invalid file name: + + >>> rc, out, err = script_helper.assert_python_failure( + ... '-m', 'doctest', 'nosuchfile') + >>> rc, out + (1, b'') + >>> # The exact error message changes depending on the platform. + >>> print(normalize(err)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + FileNotFoundError: [Errno ...] ...nosuchfile... + +Invalid doctest option: + + >>> rc, out, err = script_helper.assert_python_failure( + ... '-m', 'doctest', '-o', 'nosuchoption') + >>> rc, out + (2, b'') + >>> print(normalize(err)) # doctest: +ELLIPSIS + usage...invalid...nosuchoption... + +""" + +def test_no_trailing_whitespace_stripping(): + r""" + The fancy reports had a bug for a long time where any trailing whitespace on + the reported diff lines was stripped, making it impossible to see the + differences in line reported as different that differed only in the amount of + trailing whitespace. The whitespace still isn't particularly visible unless + you use NDIFF, but at least it is now there to be found. + + *NOTE*: This snippet was intentionally put inside a raw string to get rid of + leading whitespace error in executing the example below + + >>> def f(x): + ... r''' + ... >>> print('\n'.join(['a ', 'b'])) + ... a + ... b + ... ''' + """ + """ + *NOTE*: These doctest are not placed in raw string to depict the trailing whitespace + using `\x20` + + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_NDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join(['a ', 'b'])) + Differences (ndiff with -expected +actual): + - a + + a + b + TestResults(failed=1, attempted=1) + + *NOTE*: `\x20` is for checking the trailing whitespace on the +a line above. + We cannot use actual spaces there, as a commit hook prevents from committing + patches that contain trailing whitespace. More info on Issue 24746. + """ + + +def test_run_doctestsuite_multiple_times(): + """ + It was not possible to run the same DocTestSuite multiple times + http://bugs.python.org/issue2604 + http://bugs.python.org/issue9736 + + >>> import unittest + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) + >>> suite.run(unittest.TestResult()) + + >>> suite.run(unittest.TestResult()) + + """ + + +def test_exception_with_note(note): + """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> test_exception_with_note('Note') + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('Note') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('''Note + ... multiline + ... example''') + Traceback (most recent call last): + ValueError: Text + Note + multiline + example + + Different note will fail the test: + + >>> def f(x): + ... r''' + ... >>> exc = ValueError('message') + ... >>> exc.add_note('note') + ... >>> raise exc + ... Traceback (most recent call last): + ... ValueError: message + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 5, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + ValueError: message + wrong note + Got: + Traceback (most recent call last): + ... + ValueError: message + note + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + exc = ValueError('Text') + exc.add_note(note) + raise exc + + +def test_exception_with_multiple_notes(): + """ + >>> test_exception_with_multiple_notes() + Traceback (most recent call last): + ... + ValueError: Text + One + Two + """ + exc = ValueError('Text') + exc.add_note('One') + exc.add_note('Two') + raise exc + + +def test_syntax_error_with_note(cls, multiline=False): + """ + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + File "x.py", line 23 + bad syntax + SyntaxError: error + Note + + >>> test_syntax_error_with_note(IndentationError) + Traceback (most recent call last): + ... + IndentationError: error + Note + + >>> test_syntax_error_with_note(TabError, multiline=True) + Traceback (most recent call last): + ... + TabError: error + Note + Line + """ + exc = cls("error", ("x.py", 23, None, "bad syntax")) + exc.add_note('Note\nLine' if multiline else 'Note') + raise exc + + +def test_syntax_error_subclass_from_stdlib(): + """ + `ParseError` is a subclass of `SyntaxError`, but it is not a builtin: + + >>> test_syntax_error_subclass_from_stdlib() + Traceback (most recent call last): + ... + xml.etree.ElementTree.ParseError: error + error + Note + Line + """ + from xml.etree.ElementTree import ParseError + exc = ParseError("error\nerror") + exc.add_note('Note\nLine') + raise exc + + +def test_syntax_error_with_incorrect_expected_note(): + """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... r''' + ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + ... >>> exc.add_note('note1') + ... >>> exc.add_note('note2') + ... >>> raise exc + ... Traceback (most recent call last): + ... SyntaxError: error + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + SyntaxError: error + wrong note + Got: + Traceback (most recent call last): + ... + SyntaxError: error + note1 + note2 + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(doctest)) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # TODO: RUSTPYTHON + return tests + + +if __name__ == '__main__': + unittest.main(module='test.test_doctest.test_doctest') diff --git a/Lib/test/test_doctest/test_doctest.txt b/Lib/test/test_doctest/test_doctest.txt new file mode 100644 index 00000000000..23446d1d224 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest.txt @@ -0,0 +1,17 @@ +This is a sample doctest in a text file. + +In this example, we'll rely on a global variable being set for us +already: + + >>> favorite_color + 'blue' + +We can make this fail by disabling the blank-line feature. + + >>> if 1: + ... print('a') + ... print() + ... print('b') + a + + b diff --git a/Lib/test/test_doctest/test_doctest2.py b/Lib/test/test_doctest/test_doctest2.py new file mode 100644 index 00000000000..ab8a0696736 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest2.py @@ -0,0 +1,126 @@ +"""A module to test whether doctest recognizes some 2.2 features, +like static and class methods. + +>>> print('yup') # 1 +yup + +We include some (random) encoded (utf-8) text in the text surrounding +the example. It should be ignored: + +ЉЊЈЁЂ + +""" + +import sys +import unittest +if sys.flags.optimize >= 2: + raise unittest.SkipTest("Cannot test docstrings with -O2") + +class C(object): + """Class C. + + >>> print(C()) # 2 + 42 + + + We include some (random) encoded (utf-8) text in the text surrounding + the example. It should be ignored: + + ЉЊЈЁЂ + + """ + + def __init__(self): + """C.__init__. + + >>> print(C()) # 3 + 42 + """ + + def __str__(self): + """ + >>> print(C()) # 4 + 42 + """ + return "42" + + class D(object): + """A nested D class. + + >>> print("In D!") # 5 + In D! + """ + + def nested(self): + """ + >>> print(3) # 6 + 3 + """ + + def getx(self): + """ + >>> c = C() # 7 + >>> c.x = 12 # 8 + >>> print(c.x) # 9 + -12 + """ + return -self._x + + def setx(self, value): + """ + >>> c = C() # 10 + >>> c.x = 12 # 11 + >>> print(c.x) # 12 + -12 + """ + self._x = value + + x = property(getx, setx, doc="""\ + >>> c = C() # 13 + >>> c.x = 12 # 14 + >>> print(c.x) # 15 + -12 + """) + + @staticmethod + def statm(): + """ + A static method. + + >>> print(C.statm()) # 16 + 666 + >>> print(C().statm()) # 17 + 666 + """ + return 666 + + @classmethod + def clsm(cls, val): + """ + A class method. + + >>> print(C.clsm(22)) # 18 + 22 + >>> print(C().clsm(23)) # 19 + 23 + """ + return val + + +class Test(unittest.TestCase): + def test_testmod(self): + import doctest, sys + EXPECTED = 19 + f, t = doctest.testmod(sys.modules[__name__]) + if f: + self.fail("%d of %d doctests failed" % (f, t)) + if t != EXPECTED: + self.fail("expected %d tests to run, not %d" % (EXPECTED, t)) + + +# Pollute the namespace with a bunch of imported functions and classes, +# to make sure they don't get tested. +from doctest import * + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_doctest/test_doctest2.txt b/Lib/test/test_doctest/test_doctest2.txt new file mode 100644 index 00000000000..76dab94a9c0 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest2.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file. + +In this example, we'll rely on some silly setup: + + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup + True + +This test also has some (random) encoded (utf-8) unicode text: + + ЉЊЈЁЂ + +This doesn't cause a problem in the tect surrounding the examples, but +we include it here (in this test text file) to make sure. :) diff --git a/Lib/test/test_doctest/test_doctest3.txt b/Lib/test/test_doctest/test_doctest3.txt new file mode 100644 index 00000000000..dd8557e57a5 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest3.txt @@ -0,0 +1,5 @@ + +Here we check that `__file__` is provided: + + >>> type(__file__) + diff --git a/Lib/test/test_doctest/test_doctest4.txt b/Lib/test/test_doctest/test_doctest4.txt new file mode 100644 index 00000000000..0428e6f9632 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest4.txt @@ -0,0 +1,11 @@ +This is a sample doctest in a text file that contains non-ASCII characters. +This file is encoded using UTF-8. + +In order to get this test to pass, we have to manually specify the +encoding. + + >>> 'föö' + 'f\xf6\xf6' + + >>> 'bąr' + 'b\u0105r' diff --git a/Lib/test/test_doctest/test_doctest_errors.txt b/Lib/test/test_doctest/test_doctest_errors.txt new file mode 100644 index 00000000000..93c3c106e60 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_errors.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file, in which all examples fail +or raise an exception. + + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> 2+*3 + 5 diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000..06c23d06e60 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 # doctest: +SKIP + 4 diff --git a/Lib/test/test_doctest/test_doctest_skip2.txt b/Lib/test/test_doctest/test_doctest_skip2.txt new file mode 100644 index 00000000000..85e4938c346 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip2.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which some examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 + 4 diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 99469a58496..2ad422079b7 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -88,8 +88,6 @@ def tearDown(self): self.thread.join() self.serv.server_close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid_get_response(self): self.client.request("GET", "/") response = self.client.getresponse() @@ -100,8 +98,6 @@ def test_valid_get_response(self): # Server raises an exception if we don't start to read the data response.read() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_css(self): self.client.request("GET", "/pydoc.css") response = self.client.getresponse() @@ -135,8 +131,6 @@ def test_lambda(self): b'<lambda>(x, y)'), response.read()) - # TODO: RUSTPYTHON - @unittest.expectedFailure @make_request_and_skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_autolinking(self): diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py new file mode 100644 index 00000000000..455dc48facf --- /dev/null +++ b/Lib/test/test_email/__init__.py @@ -0,0 +1,167 @@ +import os +import unittest +import collections +import email +from email.message import Message +from email._policybase import compat32 +from test.support import load_package_tests +from test.support.testcase import ExtraAssertions +from test.test_email import __file__ as landmark + +# Load all tests in package +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) + + +# helper code used by a number of test modules. + +def openfile(filename, *args, **kws): + path = os.path.join(os.path.dirname(landmark), 'data', filename) + return open(path, *args, **kws) + + +# Base test class +class TestEmailBase(unittest.TestCase, ExtraAssertions): + + maxDiff = None + # Currently the default policy is compat32. By setting that as the default + # here we make minimal changes in the test_email tests compared to their + # pre-3.3 state. + policy = compat32 + # Likewise, the default message object is Message. + message = Message + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.addTypeEqualityFunc(bytes, self.assertBytesEqual) + + # Backward compatibility to minimize test_email test changes. + ndiffAssertEqual = unittest.TestCase.assertEqual + + def _msgobj(self, filename): + with openfile(filename, encoding="utf-8") as fp: + return email.message_from_file(fp, policy=self.policy) + + def _str_msg(self, string, message=None, policy=None): + if policy is None: + policy = self.policy + if message is None: + message = self.message + return email.message_from_string(string, message, policy=policy) + + def _bytes_msg(self, bytestring, message=None, policy=None): + if policy is None: + policy = self.policy + if message is None: + message = self.message + return email.message_from_bytes(bytestring, message, policy=policy) + + def _make_message(self): + return self.message(policy=self.policy) + + def _bytes_repr(self, b): + return [repr(x) for x in b.splitlines(keepends=True)] + + def assertBytesEqual(self, first, second, msg): + """Our byte strings are really encoded strings; improve diff output""" + self.assertEqual(self._bytes_repr(first), self._bytes_repr(second)) + + def assertDefectsEqual(self, actual, expected): + self.assertEqual(len(actual), len(expected), actual) + for i in range(len(actual)): + self.assertIsInstance(actual[i], expected[i], + 'item {}'.format(i)) + + +def parameterize(cls): + """A test method parameterization class decorator. + + Parameters are specified as the value of a class attribute that ends with + the string '_params'. Call the portion before '_params' the prefix. Then + a method to be parameterized must have the same prefix, the string + '_as_', and an arbitrary suffix. + + The value of the _params attribute may be either a dictionary or a list. + The values in the dictionary and the elements of the list may either be + single values, or a list. If single values, they are turned into single + element tuples. However derived, the resulting sequence is passed via + *args to the parameterized test function. + + In a _params dictionary, the keys become part of the name of the generated + tests. In a _params list, the values in the list are converted into a + string by joining the string values of the elements of the tuple by '_' and + converting any blanks into '_'s, and this become part of the name. + The full name of a generated test is a 'test_' prefix, the portion of the + test function name after the '_as_' separator, plus an '_', plus the name + derived as explained above. + + For example, if we have: + + count_params = range(2) + + def count_as_foo_arg(self, foo): + self.assertEqual(foo+1, myfunc(foo)) + + we will get parameterized test methods named: + test_foo_arg_0 + test_foo_arg_1 + test_foo_arg_2 + + Or we could have: + + example_params = {'foo': ('bar', 1), 'bing': ('bang', 2)} + + def example_as_myfunc_input(self, name, count): + self.assertEqual(name+str(count), myfunc(name, count)) + + and get: + test_myfunc_input_foo + test_myfunc_input_bing + + Note: if and only if the generated test name is a valid identifier can it + be used to select the test individually from the unittest command line. + + The values in the params dict can be a single value, a tuple, or a + dict. If a single value of a tuple, it is passed to the test function + as positional arguments. If a dict, it is a passed via **kw. + + """ + paramdicts = {} + testers = collections.defaultdict(list) + for name, attr in cls.__dict__.items(): + if name.endswith('_params'): + if not hasattr(attr, 'keys'): + d = {} + for x in attr: + if not hasattr(x, '__iter__'): + x = (x,) + n = '_'.join(str(v) for v in x).replace(' ', '_') + d[n] = x + attr = d + paramdicts[name[:-7] + '_as_'] = attr + if '_as_' in name: + testers[name.split('_as_')[0] + '_as_'].append(name) + testfuncs = {} + for name in paramdicts: + if name not in testers: + raise ValueError("No tester found for {}".format(name)) + for name in testers: + if name not in paramdicts: + raise ValueError("No params found for {}".format(name)) + for name, attr in cls.__dict__.items(): + for paramsname, paramsdict in paramdicts.items(): + if name.startswith(paramsname): + testnameroot = 'test_' + name[len(paramsname):] + for paramname, params in paramsdict.items(): + if hasattr(params, 'keys'): + test = (lambda self, name=name, params=params: + getattr(self, name)(**params)) + else: + test = (lambda self, name=name, params=params: + getattr(self, name)(*params)) + testname = testnameroot + '_' + paramname + test.__name__ = testname + testfuncs[testname] = test + for key, value in testfuncs.items(): + setattr(cls, key, value) + return cls diff --git a/Lib/test/test_email/__main__.py b/Lib/test/test_email/__main__.py new file mode 100644 index 00000000000..4b14f773db4 --- /dev/null +++ b/Lib/test/test_email/__main__.py @@ -0,0 +1,4 @@ +from test.test_email import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_email/data/msg_01.txt b/Lib/test/test_email/data/msg_01.txt new file mode 100644 index 00000000000..7e33bcf96af --- /dev/null +++ b/Lib/test/test_email/data/msg_01.txt @@ -0,0 +1,19 @@ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_02.txt b/Lib/test/test_email/data/msg_02.txt new file mode 100644 index 00000000000..5d0a7e16c82 --- /dev/null +++ b/Lib/test/test_email/data/msg_02.txt @@ -0,0 +1,136 @@ +MIME-version: 1.0 +From: ppp-request@zzz.org +Sender: ppp-admin@zzz.org +To: ppp@zzz.org +Subject: Ppp digest, Vol 1 #2 - 5 msgs +Date: Fri, 20 Apr 2001 20:18:00 -0400 (EDT) +X-Mailer: Mailman v2.0.4 +X-Mailman-Version: 2.0.4 +Content-Type: multipart/mixed; boundary="192.168.1.2.889.32614.987812255.500.21814" + +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Masthead (Ppp digest, Vol 1 #2) + +Send Ppp mailing list submissions to + ppp@zzz.org + +To subscribe or unsubscribe via the World Wide Web, visit + http://www.zzz.org/mailman/listinfo/ppp +or, via email, send a message with subject or body 'help' to + ppp-request@zzz.org + +You can reach the person managing the list at + ppp-admin@zzz.org + +When replying, please edit your Subject line so it is more specific +than "Re: Contents of Ppp digest..." + + +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Today's Topics (5 msgs) + +Today's Topics: + + 1. testing #1 (Barry A. Warsaw) + 2. testing #2 (Barry A. Warsaw) + 3. testing #3 (Barry A. Warsaw) + 4. testing #4 (Barry A. Warsaw) + 5. testing #5 (Barry A. Warsaw) + +--192.168.1.2.889.32614.987812255.500.21814 +Content-Type: multipart/digest; boundary="__--__--" + +--__--__-- + +Message: 1 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Date: Fri, 20 Apr 2001 20:16:13 -0400 +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #1 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 2 +Date: Fri, 20 Apr 2001 20:16:21 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Precedence: bulk + + +hello + + +--__--__-- + +Message: 3 +Date: Fri, 20 Apr 2001 20:16:25 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #3 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 4 +Date: Fri, 20 Apr 2001 20:16:28 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #4 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 5 +Date: Fri, 20 Apr 2001 20:16:32 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #5 +Precedence: bulk + + +hello + + + + +--__--__---- + +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Digest Footer + +_______________________________________________ +Ppp mailing list +Ppp@zzz.org +http://www.zzz.org/mailman/listinfo/ppp + + +--192.168.1.2.889.32614.987812255.500.21814-- + +End of Ppp Digest + diff --git a/Lib/test/test_email/data/msg_03.txt b/Lib/test/test_email/data/msg_03.txt new file mode 100644 index 00000000000..c748ebf1176 --- /dev/null +++ b/Lib/test/test_email/data/msg_03.txt @@ -0,0 +1,16 @@ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_04.txt b/Lib/test/test_email/data/msg_04.txt new file mode 100644 index 00000000000..1f633c4496f --- /dev/null +++ b/Lib/test/test_email/data/msg_04.txt @@ -0,0 +1,37 @@ +Return-Path: +Delivered-To: barry@python.org +Received: by mail.python.org (Postfix, from userid 889) + id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="h90VIIIKmx" +Content-Transfer-Encoding: 7bit +Message-ID: <15261.36209.358846.118674@anthem.python.org> +From: barry@python.org (Barry A. Warsaw) +To: barry@python.org +Subject: a simple multipart +Date: Tue, 11 Sep 2001 00:05:05 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Make a door into a window + + +--h90VIIIKmx +Content-Type: text/plain +Content-Disposition: inline; + filename="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx +Content-Type: text/plain +Content-Disposition: inline; + filename="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx-- + diff --git a/Lib/test/test_email/data/msg_05.txt b/Lib/test/test_email/data/msg_05.txt new file mode 100644 index 00000000000..87d5e9cbf8b --- /dev/null +++ b/Lib/test/test_email/data/msg_05.txt @@ -0,0 +1,28 @@ +From: foo +Subject: bar +To: baz +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="D1690A7AC1.996856090/mail.example.com" +Message-Id: <20010803162810.0CA8AA7ACC@mail.example.com> + +This is a MIME-encapsulated message. + +--D1690A7AC1.996856090/mail.example.com +Content-Type: text/plain + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com +Content-Type: message/rfc822 + +From: nobody@python.org + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com-- + diff --git a/Lib/test/test_email/data/msg_06.txt b/Lib/test/test_email/data/msg_06.txt new file mode 100644 index 00000000000..f51ac96114b --- /dev/null +++ b/Lib/test/test_email/data/msg_06.txt @@ -0,0 +1,33 @@ +Return-Path: +Delivered-To: barry@python.org +MIME-Version: 1.0 +Content-Type: message/rfc822 +Content-Description: forwarded message +Content-Transfer-Encoding: 7bit +Message-ID: <15265.9482.641338.555352@python.org> +From: barry@python.org (Barry A. Warsaw) +Sender: barry@python.org +To: barry@python.org +Subject: forwarded message from Barry A. Warsaw +Date: Thu, 13 Sep 2001 17:28:42 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Be dirty +X-Url: http://barry.wooz.org + +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Return-Path: +Delivered-To: barry@python.org +Message-ID: <15265.9468.713530.98441@python.org> +From: barry@python.org (Barry A. Warsaw) +Sender: barry@python.org +To: barry@python.org +Subject: testing +Date: Thu, 13 Sep 2001 17:28:28 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Spectrum analysis +X-Url: http://barry.wooz.org + + diff --git a/Lib/test/test_email/data/msg_07.txt b/Lib/test/test_email/data/msg_07.txt new file mode 100644 index 00000000000..721f3a0d316 --- /dev/null +++ b/Lib/test/test_email/data/msg_07.txt @@ -0,0 +1,83 @@ +MIME-Version: 1.0 +From: Barry +To: Dingus Lovers +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + +Hi there, + +This is the dingus fish. + +--BOUNDARY +Content-Type: image/gif; name="dingusfish.gif" +Content-Transfer-Encoding: base64 +content-disposition: attachment; filename="dingusfish.gif" + +R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 +7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f +qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI +oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR +RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA +/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ +YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb +6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC +M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA +B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D +z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX +JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 +ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR +n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX +eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ +qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe +wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j +Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 +doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x +lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A +E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq +Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic +isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD +ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv +Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn +I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff +BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG +pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n +B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ +1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 +YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW +FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 +ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu +zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s +ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D +oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 +cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ +7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz +GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ +FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt +pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb +6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS +kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe +UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 +9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y +xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI +fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV +SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH +VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR +fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE +J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN +l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq +I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi +yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w +qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi +MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky +uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR +WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM +YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I +MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 +cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa +BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_08.txt b/Lib/test/test_email/data/msg_08.txt new file mode 100644 index 00000000000..132ce7ada9b --- /dev/null +++ b/Lib/test/test_email/data/msg_08.txt @@ -0,0 +1,24 @@ +MIME-Version: 1.0 +From: Barry Warsaw +To: Dingus Lovers +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-2" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_09.txt b/Lib/test/test_email/data/msg_09.txt new file mode 100644 index 00000000000..0cfa6bab2ba --- /dev/null +++ b/Lib/test/test_email/data/msg_09.txt @@ -0,0 +1,24 @@ +MIME-Version: 1.0 +From: Barry Warsaw +To: Dingus Lovers +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: text/plain + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_10.txt b/Lib/test/test_email/data/msg_10.txt new file mode 100644 index 00000000000..d49e477a818 --- /dev/null +++ b/Lib/test/test_email/data/msg_10.txt @@ -0,0 +1,39 @@ +MIME-Version: 1.0 +From: Barry Warsaw +To: Dingus Lovers +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +This is a 7bit encoded message. + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" +Content-Transfer-Encoding: Quoted-Printable + +=A1This is a Quoted Printable encoded message! + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: Base64 + +VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2Uu + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: Base64 + +VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2UuCg== + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" + +This has no Content-Transfer-Encoding: header. + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_11.txt b/Lib/test/test_email/data/msg_11.txt new file mode 100644 index 00000000000..8f7f1991cbd --- /dev/null +++ b/Lib/test/test_email/data/msg_11.txt @@ -0,0 +1,7 @@ +Content-Type: message/rfc822 +MIME-Version: 1.0 +Subject: The enclosing message + +Subject: An enclosed message + +Here is the body of the message. diff --git a/Lib/test/test_email/data/msg_12.txt b/Lib/test/test_email/data/msg_12.txt new file mode 100644 index 00000000000..b109b985c8b --- /dev/null +++ b/Lib/test/test_email/data/msg_12.txt @@ -0,0 +1,36 @@ +MIME-Version: 1.0 +From: Barry Warsaw +To: Dingus Lovers +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: multipart/mixed; boundary="ANOTHER" + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-2" + + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-3" + +--ANOTHER-- + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_12a.txt b/Lib/test/test_email/data/msg_12a.txt new file mode 100644 index 00000000000..2092aa0c351 --- /dev/null +++ b/Lib/test/test_email/data/msg_12a.txt @@ -0,0 +1,38 @@ +MIME-Version: 1.0 +From: Barry Warsaw +To: Dingus Lovers +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: multipart/mixed; boundary="ANOTHER" + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-2" + + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-3" + + +--ANOTHER-- + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_13.txt b/Lib/test/test_email/data/msg_13.txt new file mode 100644 index 00000000000..8e6d52d5bef --- /dev/null +++ b/Lib/test/test_email/data/msg_13.txt @@ -0,0 +1,94 @@ +MIME-Version: 1.0 +From: Barry +To: Dingus Lovers +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="OUTER" + +--OUTER +Content-Type: text/plain; charset="us-ascii" + +A text/plain part + +--OUTER +Content-Type: multipart/mixed; boundary=BOUNDARY + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + +Hi there, + +This is the dingus fish. + +--BOUNDARY +Content-Type: image/gif; name="dingusfish.gif" +Content-Transfer-Encoding: base64 +content-disposition: attachment; filename="dingusfish.gif" + +R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 +7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f +qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI +oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR +RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA +/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ +YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb +6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC +M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA +B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D +z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX +JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 +ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR +n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX +eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ +qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe +wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j +Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 +doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x +lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A +E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq +Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic +isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD +ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv +Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn +I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff +BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG +pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n +B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ +1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 +YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW +FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 +ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu +zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s +ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D +oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 +cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ +7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz +GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ +FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt +pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb +6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS +kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe +UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 +9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y +xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI +fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV +SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH +VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR +fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE +J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN +l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq +I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi +yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w +qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi +MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky +uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR +WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM +YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I +MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 +cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa +BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= + +--BOUNDARY-- + +--OUTER-- diff --git a/Lib/test/test_email/data/msg_14.txt b/Lib/test/test_email/data/msg_14.txt new file mode 100644 index 00000000000..5d98d2fd145 --- /dev/null +++ b/Lib/test/test_email/data/msg_14.txt @@ -0,0 +1,23 @@ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +I'm sorry but I'm using a drainbread ISP, which although big and +wealthy can't seem to generate standard compliant email. :( + +This message has a Content-Type: header with no subtype. I hope you +can still read it. + +-Me diff --git a/Lib/test/test_email/data/msg_15.txt b/Lib/test/test_email/data/msg_15.txt new file mode 100644 index 00000000000..0025624e750 --- /dev/null +++ b/Lib/test/test_email/data/msg_15.txt @@ -0,0 +1,52 @@ +Return-Path: +Received: from fepD.post.tele.dk (195.41.46.149) by mail.groupcare.dk (LSMTP for Windows NT v1.1b) with SMTP id <0.0014F8A2@mail.groupcare.dk>; Mon, 30 Apr 2001 12:17:50 +0200 +User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.02.2106 +Subject: XX +From: xx@xx.dk +To: XX +Message-ID: +Mime-version: 1.0 +Content-type: multipart/mixed; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + +> Denne meddelelse er i MIME-format. Da dit postl + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: multipart/alternative; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: text/plain; charset="ISO-8859-1" +Content-transfer-encoding: quoted-printable + +Some removed test. + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: text/html; charset="ISO-8859-1" +Content-transfer-encoding: quoted-printable + + + +Some removed HTML + + +Some removed text. + + + + +--MS_Mac_OE_3071477847_720252_MIME_Part-- + + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: image/gif; name="xx.gif"; + x-mac-creator="6F676C65"; + x-mac-type="47494666" +Content-disposition: attachment +Content-transfer-encoding: base64 + +Some removed base64 encoded chars. + +--MS_Mac_OE_3071477847_720252_MIME_Part-- + diff --git a/Lib/test/test_email/data/msg_16.txt b/Lib/test/test_email/data/msg_16.txt new file mode 100644 index 00000000000..56167e9f5b4 --- /dev/null +++ b/Lib/test/test_email/data/msg_16.txt @@ -0,0 +1,123 @@ +Return-Path: <> +Delivered-To: scr-admin@socal-raves.org +Received: from cougar.noc.ucla.edu (cougar.noc.ucla.edu [169.232.10.18]) + by babylon.socal-raves.org (Postfix) with ESMTP id CCC2C51B84 + for ; Sun, 23 Sep 2001 20:13:54 -0700 (PDT) +Received: from sims-ms-daemon by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8Y@cougar.noc.ucla.edu> for scr-admin@socal-raves.org; Sun, + 23 Sep 2001 20:14:35 -0700 (PDT) +Received: from cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8X@cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +Date: Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +From: Internet Mail Delivery +Subject: Delivery Notification: Delivery has failed +To: scr-admin@socal-raves.org +Message-id: <0GK500B04D0B8X@cougar.noc.ucla.edu> +MIME-version: 1.0 +Sender: scr-owner@socal-raves.org +Errors-To: scr-owner@socal-raves.org +X-BeenThere: scr@socal-raves.org +X-Mailman-Version: 2.1a3 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: SoCal-Raves +List-Unsubscribe: , + +List-Archive: +Content-Type: multipart/report; boundary="Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)" + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: text/plain; charset=ISO-8859-1 + +This report relates to a message you sent with the following header fields: + + Message-id: <002001c144a6$8752e060$56104586@oxy.edu> + Date: Sun, 23 Sep 2001 20:10:55 -0700 + From: "Ian T. Henry" + To: SoCal Raves + Subject: [scr] yeah for Ians!! + +Your message cannot be delivered to the following recipients: + + Recipient address: jangel1@cougar.noc.ucla.edu + Reason: recipient reached disk quota + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: message/DELIVERY-STATUS + +Original-envelope-id: 0GK500B4HD0888@cougar.noc.ucla.edu +Reporting-MTA: dns; cougar.noc.ucla.edu + +Action: failed +Status: 5.0.0 (recipient reached disk quota) +Original-recipient: rfc822;jangel1@cougar.noc.ucla.edu +Final-recipient: rfc822;jangel1@cougar.noc.ucla.edu + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: MESSAGE/RFC822 + +Return-path: scr-admin@socal-raves.org +Received: from sims-ms-daemon by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8X@cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +Received: from panther.noc.ucla.edu by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + with ESMTP id <0GK500B4GD0888@cougar.noc.ucla.edu> for jangel1@sims-ms-daemon; + Sun, 23 Sep 2001 20:14:33 -0700 (PDT) +Received: from babylon.socal-raves.org + (ip-209-85-222-117.dreamhost.com [209.85.222.117]) + by panther.noc.ucla.edu (8.9.1a/8.9.1) with ESMTP id UAA09793 for + ; Sun, 23 Sep 2001 20:14:32 -0700 (PDT) +Received: from babylon (localhost [127.0.0.1]) by babylon.socal-raves.org + (Postfix) with ESMTP id D3B2951B70; Sun, 23 Sep 2001 20:13:47 -0700 (PDT) +Received: by babylon.socal-raves.org (Postfix, from userid 60001) + id A611F51B82; Sun, 23 Sep 2001 20:13:46 -0700 (PDT) +Received: from tiger.cc.oxy.edu (tiger.cc.oxy.edu [134.69.3.112]) + by babylon.socal-raves.org (Postfix) with ESMTP id ADA7351B70 for + ; Sun, 23 Sep 2001 20:13:44 -0700 (PDT) +Received: from ent (n16h86.dhcp.oxy.edu [134.69.16.86]) + by tiger.cc.oxy.edu (8.8.8/8.8.8) with SMTP id UAA08100 for + ; Sun, 23 Sep 2001 20:14:24 -0700 (PDT) +Date: Sun, 23 Sep 2001 20:10:55 -0700 +From: "Ian T. Henry" +Subject: [scr] yeah for Ians!! +Sender: scr-admin@socal-raves.org +To: SoCal Raves +Errors-to: scr-admin@socal-raves.org +Message-id: <002001c144a6$8752e060$56104586@oxy.edu> +MIME-version: 1.0 +X-Mailer: Microsoft Outlook Express 5.50.4522.1200 +Content-type: text/plain; charset=us-ascii +Precedence: bulk +Delivered-to: scr-post@babylon.socal-raves.org +Delivered-to: scr@socal-raves.org +X-Converted-To-Plain-Text: from multipart/alternative by demime 0.98e +X-Converted-To-Plain-Text: Alternative section used was text/plain +X-BeenThere: scr@socal-raves.org +X-Mailman-Version: 2.1a3 +List-Help: +List-Post: +List-Subscribe: , + +List-Id: SoCal-Raves +List-Unsubscribe: , + +List-Archive: + +I always love to find more Ian's that are over 3 years old!! + +Ian +_______________________________________________ +For event info, list questions, or to unsubscribe, see http://www.socal-raves.org/ + + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)-- + diff --git a/Lib/test/test_email/data/msg_17.txt b/Lib/test/test_email/data/msg_17.txt new file mode 100644 index 00000000000..8d86e4180dd --- /dev/null +++ b/Lib/test/test_email/data/msg_17.txt @@ -0,0 +1,12 @@ +MIME-Version: 1.0 +From: Barry +To: Dingus Lovers +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +Hi there, + +This is the dingus fish. + +[Non-text (image/gif) part of message omitted, filename dingusfish.gif] diff --git a/Lib/test/test_email/data/msg_18.txt b/Lib/test/test_email/data/msg_18.txt new file mode 100644 index 00000000000..f9f4904d366 --- /dev/null +++ b/Lib/test/test_email/data/msg_18.txt @@ -0,0 +1,6 @@ +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" + diff --git a/Lib/test/test_email/data/msg_19.txt b/Lib/test/test_email/data/msg_19.txt new file mode 100644 index 00000000000..49bf7fccdd9 --- /dev/null +++ b/Lib/test/test_email/data/msg_19.txt @@ -0,0 +1,43 @@ +Send Ppp mailing list submissions to + ppp@zzz.org + +To subscribe or unsubscribe via the World Wide Web, visit + http://www.zzz.org/mailman/listinfo/ppp +or, via email, send a message with subject or body 'help' to + ppp-request@zzz.org + +You can reach the person managing the list at + ppp-admin@zzz.org + +When replying, please edit your Subject line so it is more specific +than "Re: Contents of Ppp digest..." + +Today's Topics: + + 1. testing #1 (Barry A. Warsaw) + 2. testing #2 (Barry A. Warsaw) + 3. testing #3 (Barry A. Warsaw) + 4. testing #4 (Barry A. Warsaw) + 5. testing #5 (Barry A. Warsaw) + +hello + + +hello + + +hello + + +hello + + +hello + + + +_______________________________________________ +Ppp mailing list +Ppp@zzz.org +http://www.zzz.org/mailman/listinfo/ppp + diff --git a/Lib/test/test_email/data/msg_20.txt b/Lib/test/test_email/data/msg_20.txt new file mode 100644 index 00000000000..1a6a88783ee --- /dev/null +++ b/Lib/test/test_email/data/msg_20.txt @@ -0,0 +1,22 @@ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Cc: ccc@zzz.org +CC: ddd@zzz.org +cc: eee@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_21.txt b/Lib/test/test_email/data/msg_21.txt new file mode 100644 index 00000000000..23590b255dd --- /dev/null +++ b/Lib/test/test_email/data/msg_21.txt @@ -0,0 +1,20 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: Test +Content-Type: multipart/mixed; boundary="BOUNDARY" + +MIME message +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +One +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Two +--BOUNDARY-- +End of MIME message diff --git a/Lib/test/test_email/data/msg_22.txt b/Lib/test/test_email/data/msg_22.txt new file mode 100644 index 00000000000..af9de5fa27b --- /dev/null +++ b/Lib/test/test_email/data/msg_22.txt @@ -0,0 +1,46 @@ +Mime-Version: 1.0 +Message-Id: +Date: Tue, 16 Oct 2001 13:59:25 +0300 +To: a@example.com +From: b@example.com +Content-Type: multipart/mixed; boundary="============_-1208892523==_============" + +--============_-1208892523==_============ +Content-Type: text/plain; charset="us-ascii" ; format="flowed" + +Text text text. +--============_-1208892523==_============ +Content-Id: +Content-Type: image/jpeg; name="wibble.JPG" + ; x-mac-type="4A504547" + ; x-mac-creator="474B4F4E" +Content-Disposition: attachment; filename="wibble.JPG" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABIEBAREA +g6bCjjw/pIZSjO6FWFpldjySOmCNrO7DBZibUXhTwtCixw+GtAijVdqxxaPp0aKvmGXa +qrbBQvms0mAMeYS/3iTV1dG0hHaRNK01XblnWxtVdjkHLMIgTyqnk9VB7CrP2KzIINpa +4O7I+zxYO9WV8jZg71Zlb+8rMDkEirAVQFAUAKAFAAAUAYAUDgADgY6DjpRtXj5RxjHA +4wQRj0wQCMdCAewpaKKK/9k= +--============_-1208892523==_============ +Content-Id: +Content-Type: image/jpeg; name="wibble2.JPG" + ; x-mac-type="4A504547" + ; x-mac-creator="474B4F4E" +Content-Disposition: attachment; filename="wibble2.JPG" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABJ0BAREA +/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQA +W6NFJJBEkU10kKGTcWMDwxuU+0JHvk8qAtOpNwqSR0n8c3BlDyXHlqsUltHEiTvdXLxR +7vMiGDNJAJWkAMk8ZkCFp5G2oo5W++INrbQtNfTQxJAuXlupz9oS4d5Y1W+E2XlWZJJE +Y7LWYQxTLE1zuMbfBPxw8X2fibVdIbSbI6nLZxX635t9TjtYreWR7WGKJTLJFFKSlozO +0ShxIXM43uC3/9k= +--============_-1208892523==_============ +Content-Type: text/plain; charset="us-ascii" ; format="flowed" + +Text text text. +--============_-1208892523==_============-- + diff --git a/Lib/test/test_email/data/msg_23.txt b/Lib/test/test_email/data/msg_23.txt new file mode 100644 index 00000000000..bb2e8ec36bb --- /dev/null +++ b/Lib/test/test_email/data/msg_23.txt @@ -0,0 +1,8 @@ +From: aperson@dom.ain +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain + +A message part +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_24.txt b/Lib/test/test_email/data/msg_24.txt new file mode 100644 index 00000000000..4e52339e86d --- /dev/null +++ b/Lib/test/test_email/data/msg_24.txt @@ -0,0 +1,10 @@ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_25.txt b/Lib/test/test_email/data/msg_25.txt new file mode 100644 index 00000000000..9e35275fe0d --- /dev/null +++ b/Lib/test/test_email/data/msg_25.txt @@ -0,0 +1,117 @@ +From MAILER-DAEMON Fri Apr 06 16:46:09 2001 +Received: from [204.245.199.98] (helo=zinfandel.lacita.com) + by www.linux.org.uk with esmtp (Exim 3.13 #1) + id 14lYR6-0008Iv-00 + for linuxuser-admin@www.linux.org.uk; Fri, 06 Apr 2001 16:46:09 +0100 +Received: from localhost (localhost) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with internal id JAB03225; Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) +Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) +From: Mail Delivery Subsystem +Subject: Returned mail: Too many hops 19 (17 max): from via [199.164.235.226], to +Message-Id: <200104061723.JAB03225@zinfandel.lacita.com> +To: +To: postmaster@zinfandel.lacita.com +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + bo +Auto-Submitted: auto-generated (failure) + +This is a MIME-encapsulated message + +--JAB03225.986577786/zinfandel.lacita.com + +The original message was received at Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) +from [199.164.235.226] + + ----- The following addresses have delivery notifications ----- + (unrecoverable error) + + ----- Transcript of session follows ----- +554 Too many hops 19 (17 max): from via [199.164.235.226], to + +--JAB03225.986577786/zinfandel.lacita.com +Content-Type: message/delivery-status + +Reporting-MTA: dns; zinfandel.lacita.com +Received-From-MTA: dns; [199.164.235.226] +Arrival-Date: Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) + +Final-Recipient: rfc822; scoffman@wellpartner.com +Action: failed +Status: 5.4.6 +Last-Attempt-Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) + +--JAB03225.986577786/zinfandel.lacita.com +Content-Type: text/rfc822-headers + +Return-Path: linuxuser-admin@www.linux.org.uk +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03225 for ; Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03221 for ; Fri, 6 Apr 2001 09:22:18 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03217 for ; Fri, 6 Apr 2001 09:21:37 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03213 for ; Fri, 6 Apr 2001 09:20:56 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03209 for ; Fri, 6 Apr 2001 09:20:15 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03205 for ; Fri, 6 Apr 2001 09:19:33 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03201 for ; Fri, 6 Apr 2001 09:18:52 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03197 for ; Fri, 6 Apr 2001 09:17:54 -0800 (GMT-0800) +Received: from www.linux.org.uk (parcelfarce.linux.theplanet.co.uk [195.92.249.252]) + by + fo +Received: from localhost.localdomain + ([ + by + id +Received: from [212.1.130.11] (helo=s1.uklinux.net ident=root) + by + id + fo +Received: from server (ppp-2-22.cvx4.telinco.net [212.1.149.22]) + by + fo +From: Daniel James +Organization: LinuxUser +To: linuxuser@www.linux.org.uk +X-Mailer: KMail [version 1.1.99] +Content-Type: text/plain; + c +MIME-Version: 1.0 +Message-Id: <01040616033903.00962@server> +Content-Transfer-Encoding: 8bit +Subject: [LinuxUser] bulletin no. 45 +Sender: linuxuser-admin@www.linux.org.uk +Errors-To: linuxuser-admin@www.linux.org.uk +X-BeenThere: linuxuser@www.linux.org.uk +X-Mailman-Version: 2.0.3 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Unsubscribe: , + +Date: Fri, 6 Apr 2001 16:03:39 +0100 + +--JAB03225.986577786/zinfandel.lacita.com-- + + diff --git a/Lib/test/test_email/data/msg_26.txt b/Lib/test/test_email/data/msg_26.txt new file mode 100644 index 00000000000..58efaa9c9a8 --- /dev/null +++ b/Lib/test/test_email/data/msg_26.txt @@ -0,0 +1,46 @@ +Received: from xcar [192.168.0.2] by jeeves.wooster.local + (SMTPD32-7.07 EVAL) id AFF92F0214; Sun, 12 May 2002 08:55:37 +0100 +Date: Sun, 12 May 2002 08:56:15 +0100 +From: Father Time +To: timbo@jeeves.wooster.local +Subject: IMAP file test +Message-ID: <6df65d354b.father.time@rpc.wooster.local> +X-Organization: Home +User-Agent: Messenger-Pro/2.50a (MsgServe/1.50) (RISC-OS/4.02) POPstar/2.03 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="1618492860--2051301190--113853680" +Status: R +X-UIDL: 319998302 + +This message is in MIME format which your mailer apparently does not support. +You either require a newer version of your software which supports MIME, or +a separate MIME decoding utility. Alternatively, ask the sender of this +message to resend it in a different format. + +--1618492860--2051301190--113853680 +Content-Type: text/plain; charset=us-ascii + +Simple email with attachment. + + +--1618492860--2051301190--113853680 +Content-Type: application/riscos; name="clock.bmp,69c"; type=BMP; + load=&fff69c4b; exec=&355dd4d1; access=&03 +Content-Disposition: attachment; filename="clock.bmp" +Content-Transfer-Encoding: base64 + +Qk12AgAAAAAAAHYAAAAoAAAAIAAAACAAAAABAAQAAAAAAAAAAADXDQAA1w0AAAAAAAAA +AAAAAAAAAAAAiAAAiAAAAIiIAIgAAACIAIgAiIgAALu7uwCIiIgAERHdACLuIgAz//8A +zAAAAN0R3QDu7iIA////AAAAAAAAAAAAAAAAAAAAAAAAAAi3AAAAAAAAADeAAAAAAAAA +C3ADMzMzMANwAAAAAAAAAAAHMAAAAANwAAAAAAAAAACAMAd3zPfwAwgAAAAAAAAIAwd/ +f8x/f3AwgAAAAAAAgDB0x/f3//zPAwgAAAAAAAcHfM9////8z/AwAAAAAAiwd/f3//// +////A4AAAAAAcEx/f///////zAMAAAAAiwfM9////3///8zwOAAAAAcHf3////B///// +8DAAAAALB/f3///wd3d3//AwAAAABwTPf//wCQAAD/zAMAAAAAsEx/f///B////8wDAA +AAAHB39////wf/////AwAAAACwf39///8H/////wMAAAAIcHfM9///B////M8DgAAAAA +sHTH///wf///xAMAAAAACHB3f3//8H////cDgAAAAAALB3zH//D//M9wMAAAAAAAgLB0 +z39///xHAwgAAAAAAAgLB3d3RHd3cDCAAAAAAAAAgLAHd0R3cAMIAAAAAAAAgAgLcAAA +AAMwgAgAAAAACDAAAAu7t7cwAAgDgAAAAABzcIAAAAAAAAgDMwAAAAAAN7uwgAAAAAgH +MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA +AAAAAAAIAAAAAAAAAIAAAAAA + +--1618492860--2051301190--113853680-- diff --git a/Lib/test/test_email/data/msg_27.txt b/Lib/test/test_email/data/msg_27.txt new file mode 100644 index 00000000000..d0191769d73 --- /dev/null +++ b/Lib/test/test_email/data/msg_27.txt @@ -0,0 +1,15 @@ +Return-Path: +Received: by mail.dom.ain (Postfix, from userid 889) + id B9D0AD35DB; Tue, 4 Jun 2002 21:46:59 -0400 (EDT) +Message-ID: <15613.28051.707126.569693@dom.ain> +Date: Tue, 4 Jun 2002 21:46:59 -0400 +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Subject: bug demonstration + 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 + more text +From: aperson@dom.ain (Anne P. Erson) +To: bperson@dom.ain (Barney P. Erson) + +test diff --git a/Lib/test/test_email/data/msg_28.txt b/Lib/test/test_email/data/msg_28.txt new file mode 100644 index 00000000000..1e4824cabb0 --- /dev/null +++ b/Lib/test/test_email/data/msg_28.txt @@ -0,0 +1,25 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/digest; boundary=BOUNDARY + +--BOUNDARY +Content-Type: message/rfc822 + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 1 + +--BOUNDARY +Content-Type: message/rfc822 + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_29.txt b/Lib/test/test_email/data/msg_29.txt new file mode 100644 index 00000000000..1fab5616173 --- /dev/null +++ b/Lib/test/test_email/data/msg_29.txt @@ -0,0 +1,22 @@ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii; + title*0*="us-ascii'en'This%20is%20even%20more%20"; + title*1*="%2A%2A%2Afun%2A%2A%2A%20"; + title*2="isn't it!" +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_30.txt b/Lib/test/test_email/data/msg_30.txt new file mode 100644 index 00000000000..4334bb6e608 --- /dev/null +++ b/Lib/test/test_email/data/msg_30.txt @@ -0,0 +1,23 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/digest; boundary=BOUNDARY + +--BOUNDARY + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 1 + +--BOUNDARY + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_31.txt b/Lib/test/test_email/data/msg_31.txt new file mode 100644 index 00000000000..1e58e56cf52 --- /dev/null +++ b/Lib/test/test_email/data/msg_31.txt @@ -0,0 +1,15 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=BOUNDARY_ + +--BOUNDARY +Content-Type: text/plain + +message 1 + +--BOUNDARY +Content-Type: text/plain + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_32.txt b/Lib/test/test_email/data/msg_32.txt new file mode 100644 index 00000000000..07ec5af9a3d --- /dev/null +++ b/Lib/test/test_email/data/msg_32.txt @@ -0,0 +1,14 @@ +Delivered-To: freebsd-isp@freebsd.org +Date: Tue, 26 Sep 2000 12:23:03 -0500 +From: Anne Person +To: Barney Dude +Subject: Re: Limiting Perl CPU Utilization... +Mime-Version: 1.0 +Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii +Content-Disposition: inline +User-Agent: Mutt/1.3.8i +Sender: owner-freebsd-isp@FreeBSD.ORG +Precedence: bulk +X-Loop: FreeBSD.org + +Some message. diff --git a/Lib/test/test_email/data/msg_33.txt b/Lib/test/test_email/data/msg_33.txt new file mode 100644 index 00000000000..042787a4fd8 --- /dev/null +++ b/Lib/test/test_email/data/msg_33.txt @@ -0,0 +1,29 @@ +Delivered-To: freebsd-isp@freebsd.org +Date: Wed, 27 Sep 2000 11:11:09 -0500 +From: Anne Person +To: Barney Dude +Subject: Re: Limiting Perl CPU Utilization... +Mime-Version: 1.0 +Content-Type: multipart/signed; micalg*=ansi-x3.4-1968''pgp-md5; + protocol*=ansi-x3.4-1968''application%2Fpgp-signature; + boundary*="ansi-x3.4-1968''EeQfGwPcQSOJBaQU" +Content-Disposition: inline +Sender: owner-freebsd-isp@FreeBSD.ORG +Precedence: bulk +X-Loop: FreeBSD.org + + +--EeQfGwPcQSOJBaQU +Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +part 1 + +--EeQfGwPcQSOJBaQU +Content-Type: text/plain +Content-Disposition: inline + +part 2 + +--EeQfGwPcQSOJBaQU-- diff --git a/Lib/test/test_email/data/msg_34.txt b/Lib/test/test_email/data/msg_34.txt new file mode 100644 index 00000000000..055dfea5310 --- /dev/null +++ b/Lib/test/test_email/data/msg_34.txt @@ -0,0 +1,19 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Content-Type: multipart/digest; boundary=XYZ + +--XYZ +Content-Type: text/plain + + +This is a text plain part that is counter to recommended practice in +RFC 2046, $5.1.5, but is not illegal + +--XYZ + +From: cperson@dom.ain +To: dperson@dom.ain + +A submessage + +--XYZ-- diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt new file mode 100644 index 00000000000..0e2bbcaf718 --- /dev/null +++ b/Lib/test/test_email/data/msg_35.txt @@ -0,0 +1,4 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: here's something interesting +counter to RFC 5322, there's no separating newline here diff --git a/Lib/test/test_email/data/msg_36.txt b/Lib/test/test_email/data/msg_36.txt new file mode 100644 index 00000000000..5632c3062c9 --- /dev/null +++ b/Lib/test/test_email/data/msg_36.txt @@ -0,0 +1,40 @@ +Mime-Version: 1.0 +Content-Type: Multipart/Mixed; Boundary="NextPart" +To: IETF-Announce:; +From: Internet-Drafts@ietf.org +Subject: I-D ACTION:draft-ietf-mboned-mix-00.txt +Date: Tue, 22 Dec 1998 16:55:06 -0500 + +--NextPart + +Blah blah blah + +--NextPart +Content-Type: Multipart/Alternative; Boundary="OtherAccess" + +--OtherAccess +Content-Type: Message/External-body; + access-type="mail-server"; + server="mailserv@ietf.org" + +Content-Type: text/plain +Content-ID: <19981222151406.I-D@ietf.org> + +ENCODING mime +FILE /internet-drafts/draft-ietf-mboned-mix-00.txt + +--OtherAccess +Content-Type: Message/External-body; + name="draft-ietf-mboned-mix-00.txt"; + site="ftp.ietf.org"; + access-type="anon-ftp"; + directory="internet-drafts" + +Content-Type: text/plain +Content-ID: <19981222151406.I-D@ietf.org> + + +--OtherAccess-- + +--NextPart-- + diff --git a/Lib/test/test_email/data/msg_37.txt b/Lib/test/test_email/data/msg_37.txt new file mode 100644 index 00000000000..038d34a1a42 --- /dev/null +++ b/Lib/test/test_email/data/msg_37.txt @@ -0,0 +1,22 @@ +Content-Type: multipart/mixed; boundary=ABCDE + +--ABCDE +Content-Type: text/x-one + +Blah + +--ABCDE +--ABCDE +Content-Type: text/x-two + +Blah + +--ABCDE +--ABCDE +--ABCDE +--ABCDE +Content-Type: text/x-two + +Blah + +--ABCDE-- diff --git a/Lib/test/test_email/data/msg_38.txt b/Lib/test/test_email/data/msg_38.txt new file mode 100644 index 00000000000..006df81cb59 --- /dev/null +++ b/Lib/test/test_email/data/msg_38.txt @@ -0,0 +1,101 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" + +------- =_aaaaaaaaaa0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.1@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa2" +Content-ID: <20592.1022586929.2@example.com> + +------- =_aaaaaaaaaa2 +Content-Type: text/plain +Content-ID: <20592.1022586929.3@example.com> +Content-Description: very tricky +Content-Transfer-Encoding: 7bit + + +Unlike the test test_nested-multiples-with-internal-boundary, this +piece of text not only contains the outer boundary tags +------- =_aaaaaaaaaa1 +and +------- =_aaaaaaaaaa0 +but puts them at the start of a line! And, to be even nastier, it +even includes a couple of end tags, such as this one: + +------- =_aaaaaaaaaa1-- + +and this one, which is from a multipart we haven't even seen yet! + +------- =_aaaaaaaaaa4-- + +This will, I'm sure, cause much breakage of MIME parsers. But, as +far as I can tell, it's perfectly legal. I have not yet ever seen +a case of this in the wild, but I've seen *similar* things. + + +------- =_aaaaaaaaaa2 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.4@example.com> +Content-Description: patch2 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa2-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa3" +Content-ID: <20592.1022586929.6@example.com> + +------- =_aaaaaaaaaa3 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.7@example.com> +Content-Description: patch3 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa3 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.8@example.com> +Content-Description: patch4 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa3-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa4" +Content-ID: <20592.1022586929.10@example.com> + +------- =_aaaaaaaaaa4 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.11@example.com> +Content-Description: patch5 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa4 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.12@example.com> +Content-Description: patch6 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa4-- + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa0 +Content-Type: text/plain; charset="us-ascii" +Content-ID: <20592.1022586929.15@example.com> + +-- +It's never too late to have a happy childhood. + +------- =_aaaaaaaaaa0-- diff --git a/Lib/test/test_email/data/msg_39.txt b/Lib/test/test_email/data/msg_39.txt new file mode 100644 index 00000000000..124b2691927 --- /dev/null +++ b/Lib/test/test_email/data/msg_39.txt @@ -0,0 +1,83 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" + +------- =_aaaaaaaaaa0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.1@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.2@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.3@example.com> +Content-Description: patch1 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.4@example.com> +Content-Description: patch2 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.6@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.7@example.com> +Content-Description: patch3 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.8@example.com> +Content-Description: patch4 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.10@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.11@example.com> +Content-Description: patch5 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.12@example.com> +Content-Description: patch6 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa0 +Content-Type: text/plain; charset="us-ascii" +Content-ID: <20592.1022586929.15@example.com> + +-- +It's never too late to have a happy childhood. + +------- =_aaaaaaaaaa0-- diff --git a/Lib/test/test_email/data/msg_40.txt b/Lib/test/test_email/data/msg_40.txt new file mode 100644 index 00000000000..1435fa1e1a0 --- /dev/null +++ b/Lib/test/test_email/data/msg_40.txt @@ -0,0 +1,10 @@ +MIME-Version: 1.0 +Content-Type: text/html; boundary="--961284236552522269" + +----961284236552522269 +Content-Type: text/html; +Content-Transfer-Encoding: 7Bit + + + +----961284236552522269-- diff --git a/Lib/test/test_email/data/msg_41.txt b/Lib/test/test_email/data/msg_41.txt new file mode 100644 index 00000000000..76cdd1cb7f2 --- /dev/null +++ b/Lib/test/test_email/data/msg_41.txt @@ -0,0 +1,8 @@ +From: "Allison Dunlap" +To: yyy@example.com +Subject: 64423 +Date: Sun, 11 Jul 2004 16:09:27 -0300 +MIME-Version: 1.0 +Content-Type: multipart/alternative; + +Blah blah blah diff --git a/Lib/test/test_email/data/msg_42.txt b/Lib/test/test_email/data/msg_42.txt new file mode 100644 index 00000000000..a75f8f4a020 --- /dev/null +++ b/Lib/test/test_email/data/msg_42.txt @@ -0,0 +1,20 @@ +Content-Type: multipart/mixed; boundary="AAA" +From: Mail Delivery Subsystem +To: yyy@example.com + +This is a MIME-encapsulated message + +--AAA + +Stuff + +--AAA +Content-Type: message/rfc822 + +From: webmaster@python.org +To: zzz@example.com +Content-Type: multipart/mixed; boundary="BBB" + +--BBB-- + +--AAA-- diff --git a/Lib/test/test_email/data/msg_43.txt b/Lib/test/test_email/data/msg_43.txt new file mode 100644 index 00000000000..797d12c5688 --- /dev/null +++ b/Lib/test/test_email/data/msg_43.txt @@ -0,0 +1,217 @@ +From SRS0=aO/p=ON=bag.python.org=None@bounce2.pobox.com Fri Nov 26 21:40:36 2004 +X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] + [nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil] + nil) +MIME-Version: 1.0 +Message-Id: +Content-Type: multipart/report; report-type=delivery-status; + charset=utf-8; + boundary="----------=_1101526904-1956-5" +X-Virus-Scanned: by XS4ALL Virus Scanner +X-UIDL: 4\G!!! +To: +Subject: Banned file: auto__mail.python.bat in mail from you +Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +This is a multi-part message in MIME format... + +------------=_1101526904-1956-5 +Content-Type: text/plain; charset="utf-8" +Content-Disposition: inline +Content-Transfer-Encoding: 7bit + +BANNED FILENAME ALERT + +Your message to: xxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxxxx@dot.ca.gov +was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED: + +Subject: Delivery_failure_notice + +An attachment in that mail was of a file type that the Spam Firewall is set to block. + + + +------------=_1101526904-1956-5 +Content-Type: message/delivery-status +Content-Disposition: inline +Content-Transfer-Encoding: 7bit +Content-Description: Delivery error report + +Reporting-MTA: dns; sacspam01.dot.ca.gov +Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1]) +Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +------------=_1101526904-1956-5 +Content-Type: text/rfc822-headers +Content-Disposition: inline +Content-Transfer-Encoding: 7bit +Content-Description: Undelivered-message headers + +Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63]) + by sacspam01.dot.ca.gov (Spam Firewall) with SMTP + id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST) +From: webmaster@python.org +To: xxxxx@dot.ca.gov +Date: Sat, 27 Nov 2004 03:35:30 UTC +Subject: Delivery_failure_notice +Importance: Normal +X-Priority: 3 (Normal) +X-MSMail-Priority: Normal +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7" +Content-Transfer-Encoding: 7bit + +------------=_1101526904-1956-5-- + diff --git a/Lib/test/test_email/data/msg_44.txt b/Lib/test/test_email/data/msg_44.txt new file mode 100644 index 00000000000..15a225287bd --- /dev/null +++ b/Lib/test/test_email/data/msg_44.txt @@ -0,0 +1,33 @@ +Return-Path: +Delivered-To: barry@python.org +Received: by mail.python.org (Postfix, from userid 889) + id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="h90VIIIKmx" +Content-Transfer-Encoding: 7bit +Message-ID: <15261.36209.358846.118674@anthem.python.org> +From: barry@python.org (Barry A. Warsaw) +To: barry@python.org +Subject: a simple multipart +Date: Tue, 11 Sep 2001 00:05:05 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Make a door into a window + + +--h90VIIIKmx +Content-Type: text/plain; name="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx +Content-Type: text/plain; name="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx-- + diff --git a/Lib/test/test_email/data/msg_45.txt b/Lib/test/test_email/data/msg_45.txt new file mode 100644 index 00000000000..58fde956e71 --- /dev/null +++ b/Lib/test/test_email/data/msg_45.txt @@ -0,0 +1,33 @@ +From: +To: +Subject: test +X-Long-Line: Some really long line contains a lot of text and thus has to be rewrapped because it is some + really long + line +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="borderline"; + protocol="application/pgp-signature"; micalg=pgp-sha1 + +This is an OpenPGP/MIME signed message (RFC 2440 and 3156) +--borderline +Content-Type: text/plain +X-Long-Line: Another really long line contains a lot of text and thus has to be rewrapped because it is another + really long + line + +This is the signed contents. + +--borderline +Content-Type: application/pgp-signature; name="signature.asc" +Content-Description: OpenPGP digital signature +Content-Disposition: attachment; filename="signature.asc" + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.0.6 (GNU/Linux) + +iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj +FGwfnRHFBFO/S4/DKysm0lI= +=t7+s +-----END PGP SIGNATURE----- + +--borderline-- diff --git a/Lib/test/test_email/data/msg_46.txt b/Lib/test/test_email/data/msg_46.txt new file mode 100644 index 00000000000..1e22c4f600a --- /dev/null +++ b/Lib/test/test_email/data/msg_46.txt @@ -0,0 +1,23 @@ +Return-Path: +Delivery-Date: Mon, 08 Feb 2010 14:05:16 +0100 +Received: from example.org (example.org [64.5.53.58]) + by example.net (node=mxbap2) with ESMTP (Nemesis) + id UNIQUE for someone@example.com; Mon, 08 Feb 2010 14:05:16 +0100 +Date: Mon, 01 Feb 2010 12:21:16 +0100 +From: "Sender" +To: +Subject: GroupwiseForwardingTest +Mime-Version: 1.0 +Content-Type: message/rfc822 + +Return-path: +Message-ID: <4B66B890.4070408@teconcept.de> +Date: Mon, 01 Feb 2010 12:18:40 +0100 +From: "Dr. Sender" +MIME-Version: 1.0 +To: "Recipient" +Subject: GroupwiseForwardingTest +Content-Type: text/plain; charset=ISO-8859-15 +Content-Transfer-Encoding: 7bit + +Testing email forwarding with Groupwise 1.2.2010 diff --git a/Lib/test/test_email/data/msg_47.txt b/Lib/test/test_email/data/msg_47.txt new file mode 100644 index 00000000000..bb48b47d96b --- /dev/null +++ b/Lib/test/test_email/data/msg_47.txt @@ -0,0 +1,14 @@ +Date: 01 Jan 2001 00:01+0000 +From: arthur@example.example +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=foo + +--foo +Content-Type: text/plain +bar + +--foo +Content-Type: text/html +

baz

+ +--foo-- \ No newline at end of file diff --git a/Lib/test/test_email/data/python.bmp b/Lib/test/test_email/data/python.bmp new file mode 100644 index 00000000000..675f95191a4 Binary files /dev/null and b/Lib/test/test_email/data/python.bmp differ diff --git a/Lib/test/test_email/data/python.exr b/Lib/test/test_email/data/python.exr new file mode 100644 index 00000000000..773c81ee1fb Binary files /dev/null and b/Lib/test/test_email/data/python.exr differ diff --git a/Lib/test/test_email/data/python.gif b/Lib/test/test_email/data/python.gif new file mode 100644 index 00000000000..efa0be3861d Binary files /dev/null and b/Lib/test/test_email/data/python.gif differ diff --git a/Lib/test/test_email/data/python.jpg b/Lib/test/test_email/data/python.jpg new file mode 100644 index 00000000000..21222c09f5a Binary files /dev/null and b/Lib/test/test_email/data/python.jpg differ diff --git a/Lib/test/test_email/data/python.pbm b/Lib/test/test_email/data/python.pbm new file mode 100644 index 00000000000..1848ba7ff06 --- /dev/null +++ b/Lib/test/test_email/data/python.pbm @@ -0,0 +1,3 @@ +P4 +16 16 +[a_X? \ No newline at end of file diff --git a/Lib/test/test_email/data/python.pgm b/Lib/test/test_email/data/python.pgm new file mode 100644 index 00000000000..8349f2a53a9 Binary files /dev/null and b/Lib/test/test_email/data/python.pgm differ diff --git a/Lib/test/test_email/data/python.png b/Lib/test/test_email/data/python.png new file mode 100644 index 00000000000..1a987f79fcd Binary files /dev/null and b/Lib/test/test_email/data/python.png differ diff --git a/Lib/test/test_email/data/python.ppm b/Lib/test/test_email/data/python.ppm new file mode 100644 index 00000000000..7d9cdb32158 Binary files /dev/null and b/Lib/test/test_email/data/python.ppm differ diff --git a/Lib/test/test_email/data/python.ras b/Lib/test/test_email/data/python.ras new file mode 100644 index 00000000000..130e96f817e Binary files /dev/null and b/Lib/test/test_email/data/python.ras differ diff --git a/Lib/test/test_email/data/python.sgi b/Lib/test/test_email/data/python.sgi new file mode 100644 index 00000000000..ffe9081c7a5 Binary files /dev/null and b/Lib/test/test_email/data/python.sgi differ diff --git a/Lib/test/test_email/data/python.tiff b/Lib/test/test_email/data/python.tiff new file mode 100644 index 00000000000..39d0bfcec02 Binary files /dev/null and b/Lib/test/test_email/data/python.tiff differ diff --git a/Lib/test/test_email/data/python.webp b/Lib/test/test_email/data/python.webp new file mode 100644 index 00000000000..e824ec7fb1c Binary files /dev/null and b/Lib/test/test_email/data/python.webp differ diff --git a/Lib/test/test_email/data/python.xbm b/Lib/test/test_email/data/python.xbm new file mode 100644 index 00000000000..cfbee2e9806 --- /dev/null +++ b/Lib/test/test_email/data/python.xbm @@ -0,0 +1,6 @@ +#define python_width 16 +#define python_height 16 +static char python_bits[] = { + 0xDF, 0xFE, 0x8F, 0xFD, 0x5F, 0xFB, 0xAB, 0xFE, 0xB5, 0x8D, 0xDA, 0x8F, + 0xA5, 0x86, 0xFA, 0x83, 0x1A, 0x80, 0x0D, 0x80, 0x0D, 0x80, 0x0F, 0xE0, + 0x0F, 0xF8, 0x0F, 0xF8, 0x0F, 0xFC, 0xFF, 0xFF, }; diff --git a/Lib/test/test_email/data/sndhdr.aifc b/Lib/test/test_email/data/sndhdr.aifc new file mode 100644 index 00000000000..8aae4e730bd Binary files /dev/null and b/Lib/test/test_email/data/sndhdr.aifc differ diff --git a/Lib/test/test_email/data/sndhdr.aiff b/Lib/test/test_email/data/sndhdr.aiff new file mode 100644 index 00000000000..8c279a762f1 Binary files /dev/null and b/Lib/test/test_email/data/sndhdr.aiff differ diff --git a/Lib/test/test_email/data/sndhdr.au b/Lib/test/test_email/data/sndhdr.au new file mode 100644 index 00000000000..f76b0501b8c Binary files /dev/null and b/Lib/test/test_email/data/sndhdr.au differ diff --git a/Lib/test/test_email/data/sndhdr.wav b/Lib/test/test_email/data/sndhdr.wav new file mode 100644 index 00000000000..0dca36739cd Binary files /dev/null and b/Lib/test/test_email/data/sndhdr.wav differ diff --git a/Lib/test/test_email/test__encoded_words.py b/Lib/test/test_email/test__encoded_words.py new file mode 100644 index 00000000000..dcac7d34e95 --- /dev/null +++ b/Lib/test/test_email/test__encoded_words.py @@ -0,0 +1,208 @@ +import unittest +from email import _encoded_words as _ew +from email import errors +from test.test_email import TestEmailBase + + +class TestDecodeQ(TestEmailBase): + + def _test(self, source, ex_result, ex_defects=[]): + result, defects = _ew.decode_q(source) + self.assertEqual(result, ex_result) + self.assertDefectsEqual(defects, ex_defects) + + def test_no_encoded(self): + self._test(b'foobar', b'foobar') + + def test_spaces(self): + self._test(b'foo=20bar=20', b'foo bar ') + self._test(b'foo_bar_', b'foo bar ') + + def test_run_of_encoded(self): + self._test(b'foo=20=20=21=2Cbar', b'foo !,bar') + + +class TestDecodeB(TestEmailBase): + + def _test(self, source, ex_result, ex_defects=[]): + result, defects = _ew.decode_b(source) + self.assertEqual(result, ex_result) + self.assertDefectsEqual(defects, ex_defects) + + def test_simple(self): + self._test(b'Zm9v', b'foo') + + def test_missing_padding(self): + # 1 missing padding character + self._test(b'dmk', b'vi', [errors.InvalidBase64PaddingDefect]) + # 2 missing padding characters + self._test(b'dg', b'v', [errors.InvalidBase64PaddingDefect]) + + def test_invalid_character(self): + self._test(b'dm\x01k===', b'vi', [errors.InvalidBase64CharactersDefect]) + + def test_invalid_character_and_bad_padding(self): + self._test(b'dm\x01k', b'vi', [errors.InvalidBase64CharactersDefect, + errors.InvalidBase64PaddingDefect]) + + def test_invalid_length(self): + self._test(b'abcde', b'abcde', [errors.InvalidBase64LengthDefect]) + + +class TestDecode(TestEmailBase): + + def test_wrong_format_input_raises(self): + with self.assertRaises(ValueError): + _ew.decode('=?badone?=') + with self.assertRaises(ValueError): + _ew.decode('=?') + with self.assertRaises(ValueError): + _ew.decode('') + with self.assertRaises(KeyError): + _ew.decode('=?utf-8?X?somevalue?=') + + def _test(self, source, result, charset='us-ascii', lang='', defects=[]): + res, char, l, d = _ew.decode(source) + self.assertEqual(res, result) + self.assertEqual(char, charset) + self.assertEqual(l, lang) + self.assertDefectsEqual(d, defects) + + def test_simple_q(self): + self._test('=?us-ascii?q?foo?=', 'foo') + + def test_simple_b(self): + self._test('=?us-ascii?b?dmk=?=', 'vi') + + def test_q_case_ignored(self): + self._test('=?us-ascii?Q?foo?=', 'foo') + + def test_b_case_ignored(self): + self._test('=?us-ascii?B?dmk=?=', 'vi') + + def test_non_trivial_q(self): + self._test('=?latin-1?q?=20F=fcr=20Elise=20?=', ' Für Elise ', 'latin-1') + + def test_q_escaped_bytes_preserved(self): + self._test(b'=?us-ascii?q?=20\xACfoo?='.decode('us-ascii', + 'surrogateescape'), + ' \uDCACfoo', + defects = [errors.UndecodableBytesDefect]) + + def test_b_undecodable_bytes_ignored_with_defect(self): + self._test(b'=?us-ascii?b?dm\xACk?='.decode('us-ascii', + 'surrogateescape'), + 'vi', + defects = [ + errors.InvalidBase64CharactersDefect, + errors.InvalidBase64PaddingDefect]) + + def test_b_invalid_bytes_ignored_with_defect(self): + self._test('=?us-ascii?b?dm\x01k===?=', + 'vi', + defects = [errors.InvalidBase64CharactersDefect]) + + def test_b_invalid_bytes_incorrect_padding(self): + self._test('=?us-ascii?b?dm\x01k?=', + 'vi', + defects = [ + errors.InvalidBase64CharactersDefect, + errors.InvalidBase64PaddingDefect]) + + def test_b_padding_defect(self): + self._test('=?us-ascii?b?dmk?=', + 'vi', + defects = [errors.InvalidBase64PaddingDefect]) + + def test_nonnull_lang(self): + self._test('=?us-ascii*jive?q?test?=', 'test', lang='jive') + + def test_unknown_8bit_charset(self): + self._test('=?unknown-8bit?q?foo=ACbar?=', + b'foo\xacbar'.decode('ascii', 'surrogateescape'), + charset = 'unknown-8bit', + defects = []) + + def test_unknown_charset(self): + self._test('=?foobar?q?foo=ACbar?=', + b'foo\xacbar'.decode('ascii', 'surrogateescape'), + charset = 'foobar', + # XXX Should this be a new Defect instead? + defects = [errors.CharsetError]) + + @unittest.skip("TODO: RUSTPYTHON; str has surrogates") + def test_invalid_character_in_charset(self): + self._test('=?utf-8\udce2\udc80\udc9d?q?foo=ACbar?=', + b'foo\xacbar'.decode('ascii', 'surrogateescape'), + charset = 'utf-8\udce2\udc80\udc9d', + # XXX Should this be a new Defect instead? + defects = [errors.CharsetError]) + + def test_q_nonascii(self): + self._test('=?utf-8?q?=C3=89ric?=', + 'Éric', + charset='utf-8') + + +class TestEncodeQ(TestEmailBase): + + def _test(self, src, expected): + self.assertEqual(_ew.encode_q(src), expected) + + def test_all_safe(self): + self._test(b'foobar', 'foobar') + + def test_spaces(self): + self._test(b'foo bar ', 'foo_bar_') + + def test_run_of_encodables(self): + self._test(b'foo ,,bar', 'foo__=2C=2Cbar') + + +class TestEncodeB(TestEmailBase): + + def test_simple(self): + self.assertEqual(_ew.encode_b(b'foo'), 'Zm9v') + + def test_padding(self): + self.assertEqual(_ew.encode_b(b'vi'), 'dmk=') + + +class TestEncode(TestEmailBase): + + def test_q(self): + self.assertEqual(_ew.encode('foo', 'utf-8', 'q'), '=?utf-8?q?foo?=') + + def test_b(self): + self.assertEqual(_ew.encode('foo', 'utf-8', 'b'), '=?utf-8?b?Zm9v?=') + + def test_auto_q(self): + self.assertEqual(_ew.encode('foo', 'utf-8'), '=?utf-8?q?foo?=') + + def test_auto_q_if_short_mostly_safe(self): + self.assertEqual(_ew.encode('vi.', 'utf-8'), '=?utf-8?q?vi=2E?=') + + def test_auto_b_if_enough_unsafe(self): + self.assertEqual(_ew.encode('.....', 'utf-8'), '=?utf-8?b?Li4uLi4=?=') + + def test_auto_b_if_long_unsafe(self): + self.assertEqual(_ew.encode('vi.vi.vi.vi.vi.', 'utf-8'), + '=?utf-8?b?dmkudmkudmkudmkudmku?=') + + def test_auto_q_if_long_mostly_safe(self): + self.assertEqual(_ew.encode('vi vi vi.vi ', 'utf-8'), + '=?utf-8?q?vi_vi_vi=2Evi_?=') + + def test_utf8_default(self): + self.assertEqual(_ew.encode('foo'), '=?utf-8?q?foo?=') + + def test_lang(self): + self.assertEqual(_ew.encode('foo', lang='jive'), '=?utf-8*jive?q?foo?=') + + def test_unknown_8bit(self): + self.assertEqual(_ew.encode('foo\uDCACbar', charset='unknown-8bit'), + '=?unknown-8bit?q?foo=ACbar?=') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py new file mode 100644 index 00000000000..64bc3677e87 --- /dev/null +++ b/Lib/test/test_email/test__header_value_parser.py @@ -0,0 +1,3259 @@ +import string +import unittest +from email import _header_value_parser as parser +from email import errors +from email import policy +from test.test_email import TestEmailBase, parameterize + +class TestTokens(TestEmailBase): + + # EWWhiteSpaceTerminal + + def test_EWWhiteSpaceTerminal(self): + x = parser.EWWhiteSpaceTerminal(' \t', 'fws') + self.assertEqual(x, ' \t') + self.assertEqual(str(x), '') + self.assertEqual(x.value, '') + self.assertEqual(x.token_type, 'fws') + + +class TestParserMixin: + + def _assert_results(self, tl, rest, string, value, defects, remainder, + comments=None): + self.assertEqual(str(tl), string) + self.assertEqual(tl.value, value) + self.assertDefectsEqual(tl.all_defects, defects) + self.assertEqual(rest, remainder) + if comments is not None: + self.assertEqual(tl.comments, comments) + + def _test_get_x(self, method, source, string, value, defects, + remainder, comments=None): + tl, rest = method(source) + self._assert_results(tl, rest, string, value, defects, remainder, + comments=None) + return tl + + def _test_parse_x(self, method, input, string, value, defects, + comments=None): + tl = method(input) + self._assert_results(tl, '', string, value, defects, '', comments) + return tl + + +class TestParser(TestParserMixin, TestEmailBase): + + # _wsp_splitter + + rfc_printable_ascii = bytes(range(33, 127)).decode('ascii') + rfc_atext_chars = (string.ascii_letters + string.digits + + "!#$%&\'*+-/=?^_`{}|~") + rfc_dtext_chars = rfc_printable_ascii.translate(str.maketrans('','',r'\[]')) + + def test__wsp_splitter_one_word(self): + self.assertEqual(parser._wsp_splitter('foo', 1), ['foo']) + + def test__wsp_splitter_two_words(self): + self.assertEqual(parser._wsp_splitter('foo def', 1), + ['foo', ' ', 'def']) + + def test__wsp_splitter_ws_runs(self): + self.assertEqual(parser._wsp_splitter('foo \t def jik', 1), + ['foo', ' \t ', 'def jik']) + + + # get_fws + + def test_get_fws_only(self): + fws = self._test_get_x(parser.get_fws, ' \t ', ' \t ', ' ', [], '') + self.assertEqual(fws.token_type, 'fws') + + def test_get_fws_space(self): + self._test_get_x(parser.get_fws, ' foo', ' ', ' ', [], 'foo') + + def test_get_fws_ws_run(self): + self._test_get_x(parser.get_fws, ' \t foo ', ' \t ', ' ', [], 'foo ') + + # get_encoded_word + + def test_get_encoded_word_missing_start_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_encoded_word('abc') + + def test_get_encoded_word_missing_end_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_encoded_word('=?abc') + + def test_get_encoded_word_missing_middle_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_encoded_word('=?abc?=') + + def test_get_encoded_word_invalid_cte(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_encoded_word('=?utf-8?X?somevalue?=') + + def test_get_encoded_word_valid_ew(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?this_is_a_test?= bird', + 'this is a test', + 'this is a test', + [], + ' bird') + + def test_get_encoded_word_internal_spaces(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?this is a test?= bird', + 'this is a test', + 'this is a test', + [errors.InvalidHeaderDefect], + ' bird') + + def test_get_encoded_word_gets_first(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?first?= =?utf-8?q?second?=', + 'first', + 'first', + [], + ' =?utf-8?q?second?=') + + def test_get_encoded_word_gets_first_even_if_no_space(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?first?==?utf-8?q?second?=', + 'first', + 'first', + [errors.InvalidHeaderDefect], + '=?utf-8?q?second?=') + + def test_get_encoded_word_sets_extra_attributes(self): + ew = self._test_get_x(parser.get_encoded_word, + '=?us-ascii*jive?q?first_second?=', + 'first second', + 'first second', + [], + '') + self.assertEqual(ew.charset, 'us-ascii') + self.assertEqual(ew.lang, 'jive') + + def test_get_encoded_word_lang_default_is_blank(self): + ew = self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?first_second?=', + 'first second', + 'first second', + [], + '') + self.assertEqual(ew.charset, 'us-ascii') + self.assertEqual(ew.lang, '') + + def test_get_encoded_word_non_printable_defect(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?first\x02second?=', + 'first\x02second', + 'first\x02second', + [errors.NonPrintableDefect], + '') + + def test_get_encoded_word_leading_internal_space(self): + self._test_get_x(parser.get_encoded_word, + '=?us-ascii?q?=20foo?=', + ' foo', + ' foo', + [], + '') + + def test_get_encoded_word_quopri_utf_escape_follows_cte(self): + # Issue 18044 + self._test_get_x(parser.get_encoded_word, + '=?utf-8?q?=C3=89ric?=', + 'Éric', + 'Éric', + [], + '') + + # get_unstructured + + def _get_unst(self, value): + token = parser.get_unstructured(value) + return token, '' + + def test_get_unstructured_null(self): + self._test_get_x(self._get_unst, '', '', '', [], '') + + def test_get_unstructured_one_word(self): + self._test_get_x(self._get_unst, 'foo', 'foo', 'foo', [], '') + + def test_get_unstructured_normal_phrase(self): + self._test_get_x(self._get_unst, 'foo bar bird', + 'foo bar bird', + 'foo bar bird', + [], + '') + + def test_get_unstructured_normal_phrase_with_whitespace(self): + self._test_get_x(self._get_unst, 'foo \t bar bird', + 'foo \t bar bird', + 'foo bar bird', + [], + '') + + def test_get_unstructured_leading_whitespace(self): + self._test_get_x(self._get_unst, ' foo bar', + ' foo bar', + ' foo bar', + [], + '') + + def test_get_unstructured_trailing_whitespace(self): + self._test_get_x(self._get_unst, 'foo bar ', + 'foo bar ', + 'foo bar ', + [], + '') + + def test_get_unstructured_leading_and_trailing_whitespace(self): + self._test_get_x(self._get_unst, ' foo bar ', + ' foo bar ', + ' foo bar ', + [], + '') + + def test_get_unstructured_one_valid_ew_no_ws(self): + self._test_get_x(self._get_unst, '=?us-ascii?q?bar?=', + 'bar', + 'bar', + [], + '') + + def test_get_unstructured_one_ew_trailing_ws(self): + self._test_get_x(self._get_unst, '=?us-ascii?q?bar?= ', + 'bar ', + 'bar ', + [], + '') + + def test_get_unstructured_one_valid_ew_trailing_text(self): + self._test_get_x(self._get_unst, '=?us-ascii?q?bar?= bird', + 'bar bird', + 'bar bird', + [], + '') + + def test_get_unstructured_phrase_with_ew_in_middle_of_text(self): + self._test_get_x(self._get_unst, 'foo =?us-ascii?q?bar?= bird', + 'foo bar bird', + 'foo bar bird', + [], + '') + + def test_get_unstructured_phrase_with_two_ew(self): + self._test_get_x(self._get_unst, + 'foo =?us-ascii?q?bar?= =?us-ascii?q?bird?=', + 'foo barbird', + 'foo barbird', + [], + '') + + def test_get_unstructured_phrase_with_two_ew_trailing_ws(self): + self._test_get_x(self._get_unst, + 'foo =?us-ascii?q?bar?= =?us-ascii?q?bird?= ', + 'foo barbird ', + 'foo barbird ', + [], + '') + + def test_get_unstructured_phrase_with_ew_with_leading_ws(self): + self._test_get_x(self._get_unst, + ' =?us-ascii?q?bar?=', + ' bar', + ' bar', + [], + '') + + def test_get_unstructured_phrase_with_two_ew_extra_ws(self): + self._test_get_x(self._get_unst, + 'foo =?us-ascii?q?bar?= \t =?us-ascii?q?bird?=', + 'foo barbird', + 'foo barbird', + [], + '') + + def test_get_unstructured_two_ew_extra_ws_trailing_text(self): + self._test_get_x(self._get_unst, + '=?us-ascii?q?test?= =?us-ascii?q?foo?= val', + 'testfoo val', + 'testfoo val', + [], + '') + + def test_get_unstructured_ew_with_internal_ws(self): + self._test_get_x(self._get_unst, + '=?iso-8859-1?q?hello=20world?=', + 'hello world', + 'hello world', + [], + '') + + def test_get_unstructured_ew_with_internal_leading_ws(self): + self._test_get_x(self._get_unst, + ' =?us-ascii?q?=20test?= =?us-ascii?q?=20foo?= val', + ' test foo val', + ' test foo val', + [], + '') + + def test_get_unstructured_invalid_ew(self): + self._test_get_x(self._get_unst, + '=?test val', + '=?test val', + '=?test val', + [], + '') + + def test_get_unstructured_undecodable_bytes(self): + self._test_get_x(self._get_unst, + b'test \xACfoo val'.decode('ascii', 'surrogateescape'), + 'test \uDCACfoo val', + 'test \uDCACfoo val', + [errors.UndecodableBytesDefect], + '') + + def test_get_unstructured_undecodable_bytes_in_EW(self): + self._test_get_x(self._get_unst, + (b'=?us-ascii?q?=20test?= =?us-ascii?q?=20\xACfoo?=' + b' val').decode('ascii', 'surrogateescape'), + ' test \uDCACfoo val', + ' test \uDCACfoo val', + [errors.UndecodableBytesDefect]*2, + '') + + def test_get_unstructured_missing_base64_padding(self): + self._test_get_x(self._get_unst, + '=?utf-8?b?dmk?=', + 'vi', + 'vi', + [errors.InvalidBase64PaddingDefect], + '') + + def test_get_unstructured_invalid_base64_character(self): + self._test_get_x(self._get_unst, + '=?utf-8?b?dm\x01k===?=', + 'vi', + 'vi', + [errors.InvalidBase64CharactersDefect], + '') + + def test_get_unstructured_invalid_base64_character_and_bad_padding(self): + self._test_get_x(self._get_unst, + '=?utf-8?b?dm\x01k?=', + 'vi', + 'vi', + [errors.InvalidBase64CharactersDefect, + errors.InvalidBase64PaddingDefect], + '') + + def test_get_unstructured_invalid_base64_length(self): + # bpo-27397: Return the encoded string since there's no way to decode. + self._test_get_x(self._get_unst, + '=?utf-8?b?abcde?=', + 'abcde', + 'abcde', + [errors.InvalidBase64LengthDefect], + '') + + def test_get_unstructured_no_whitespace_between_ews(self): + self._test_get_x(self._get_unst, + '=?utf-8?q?foo?==?utf-8?q?bar?=', + 'foobar', + 'foobar', + [errors.InvalidHeaderDefect, + errors.InvalidHeaderDefect], + '') + + def test_get_unstructured_ew_without_leading_whitespace(self): + self._test_get_x( + self._get_unst, + 'nowhitespace=?utf-8?q?somevalue?=', + 'nowhitespacesomevalue', + 'nowhitespacesomevalue', + [errors.InvalidHeaderDefect], + '') + + def test_get_unstructured_ew_without_trailing_whitespace(self): + self._test_get_x( + self._get_unst, + '=?utf-8?q?somevalue?=nowhitespace', + 'somevaluenowhitespace', + 'somevaluenowhitespace', + [errors.InvalidHeaderDefect], + '') + + def test_get_unstructured_without_trailing_whitespace_hang_case(self): + self._test_get_x(self._get_unst, + '=?utf-8?q?somevalue?=aa', + 'somevalueaa', + 'somevalueaa', + [errors.InvalidHeaderDefect], + '') + + def test_get_unstructured_invalid_ew2(self): + self._test_get_x(self._get_unst, + '=?utf-8?q?=somevalue?=', + '=?utf-8?q?=somevalue?=', + '=?utf-8?q?=somevalue?=', + [], + '') + + def test_get_unstructured_invalid_ew_cte(self): + self._test_get_x(self._get_unst, + '=?utf-8?X?=somevalue?=', + '=?utf-8?X?=somevalue?=', + '=?utf-8?X?=somevalue?=', + [], + '') + + # get_qp_ctext + + def test_get_qp_ctext_only(self): + ptext = self._test_get_x(parser.get_qp_ctext, + 'foobar', 'foobar', ' ', [], '') + self.assertEqual(ptext.token_type, 'ptext') + + def test_get_qp_ctext_all_printables(self): + with_qp = self.rfc_printable_ascii.replace('\\', '\\\\') + with_qp = with_qp. replace('(', r'\(') + with_qp = with_qp.replace(')', r'\)') + ptext = self._test_get_x(parser.get_qp_ctext, + with_qp, self.rfc_printable_ascii, ' ', [], '') + + def test_get_qp_ctext_two_words_gets_first(self): + self._test_get_x(parser.get_qp_ctext, + 'foo de', 'foo', ' ', [], ' de') + + def test_get_qp_ctext_following_wsp_preserved(self): + self._test_get_x(parser.get_qp_ctext, + 'foo \t\tde', 'foo', ' ', [], ' \t\tde') + + def test_get_qp_ctext_up_to_close_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + 'foo)', 'foo', ' ', [], ')') + + def test_get_qp_ctext_wsp_before_close_paren_preserved(self): + self._test_get_x(parser.get_qp_ctext, + 'foo )', 'foo', ' ', [], ' )') + + def test_get_qp_ctext_close_paren_mid_word(self): + self._test_get_x(parser.get_qp_ctext, + 'foo)bar', 'foo', ' ', [], ')bar') + + def test_get_qp_ctext_up_to_open_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + 'foo(', 'foo', ' ', [], '(') + + def test_get_qp_ctext_wsp_before_open_paren_preserved(self): + self._test_get_x(parser.get_qp_ctext, + 'foo (', 'foo', ' ', [], ' (') + + def test_get_qp_ctext_open_paren_mid_word(self): + self._test_get_x(parser.get_qp_ctext, + 'foo(bar', 'foo', ' ', [], '(bar') + + def test_get_qp_ctext_non_printables(self): + ptext = self._test_get_x(parser.get_qp_ctext, + 'foo\x00bar)', 'foo\x00bar', ' ', + [errors.NonPrintableDefect], ')') + self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + + def test_get_qp_ctext_close_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + ')', '', ' ', [], ')') + + def test_get_qp_ctext_open_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + '(', '', ' ', [], '(') + + def test_get_qp_ctext_no_end_char(self): + self._test_get_x(parser.get_qp_ctext, + '', '', ' ', [], '') + + + # get_qcontent + + def test_get_qcontent_only(self): + ptext = self._test_get_x(parser.get_qcontent, + 'foobar', 'foobar', 'foobar', [], '') + self.assertEqual(ptext.token_type, 'ptext') + + def test_get_qcontent_all_printables(self): + with_qp = self.rfc_printable_ascii.replace('\\', '\\\\') + with_qp = with_qp. replace('"', r'\"') + ptext = self._test_get_x(parser.get_qcontent, with_qp, + self.rfc_printable_ascii, + self.rfc_printable_ascii, [], '') + + def test_get_qcontent_two_words_gets_first(self): + self._test_get_x(parser.get_qcontent, + 'foo de', 'foo', 'foo', [], ' de') + + def test_get_qcontent_following_wsp_preserved(self): + self._test_get_x(parser.get_qcontent, + 'foo \t\tde', 'foo', 'foo', [], ' \t\tde') + + def test_get_qcontent_up_to_dquote_only(self): + self._test_get_x(parser.get_qcontent, + 'foo"', 'foo', 'foo', [], '"') + + def test_get_qcontent_wsp_before_close_paren_preserved(self): + self._test_get_x(parser.get_qcontent, + 'foo "', 'foo', 'foo', [], ' "') + + def test_get_qcontent_close_paren_mid_word(self): + self._test_get_x(parser.get_qcontent, + 'foo"bar', 'foo', 'foo', [], '"bar') + + def test_get_qcontent_non_printables(self): + ptext = self._test_get_x(parser.get_qcontent, + 'foo\x00fg"', 'foo\x00fg', 'foo\x00fg', + [errors.NonPrintableDefect], '"') + self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + + def test_get_qcontent_empty(self): + self._test_get_x(parser.get_qcontent, + '"', '', '', [], '"') + + def test_get_qcontent_no_end_char(self): + self._test_get_x(parser.get_qcontent, + '', '', '', [], '') + + # get_atext + + def test_get_atext_only(self): + atext = self._test_get_x(parser.get_atext, + 'foobar', 'foobar', 'foobar', [], '') + self.assertEqual(atext.token_type, 'atext') + + def test_get_atext_all_atext(self): + atext = self._test_get_x(parser.get_atext, self.rfc_atext_chars, + self.rfc_atext_chars, + self.rfc_atext_chars, [], '') + + def test_get_atext_two_words_gets_first(self): + self._test_get_x(parser.get_atext, + 'foo bar', 'foo', 'foo', [], ' bar') + + def test_get_atext_following_wsp_preserved(self): + self._test_get_x(parser.get_atext, + 'foo \t\tbar', 'foo', 'foo', [], ' \t\tbar') + + def test_get_atext_up_to_special(self): + self._test_get_x(parser.get_atext, + 'foo@bar', 'foo', 'foo', [], '@bar') + + def test_get_atext_non_printables(self): + atext = self._test_get_x(parser.get_atext, + 'foo\x00bar(', 'foo\x00bar', 'foo\x00bar', + [errors.NonPrintableDefect], '(') + self.assertEqual(atext.defects[0].non_printables[0], '\x00') + + # get_bare_quoted_string + + def test_get_bare_quoted_string_only(self): + bqs = self._test_get_x(parser.get_bare_quoted_string, + '"foo"', '"foo"', 'foo', [], '') + self.assertEqual(bqs.token_type, 'bare-quoted-string') + + def test_get_bare_quoted_string_must_start_with_dquote(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_bare_quoted_string('foo"') + with self.assertRaises(errors.HeaderParseError): + parser.get_bare_quoted_string(' "foo"') + + def test_get_bare_quoted_string_only_quotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '""', '""', '', [], '') + + def test_get_bare_quoted_string_missing_endquotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '"', '""', '', [errors.InvalidHeaderDefect], '') + + def test_get_bare_quoted_string_following_wsp_preserved(self): + self._test_get_x(parser.get_bare_quoted_string, + '"foo"\t bar', '"foo"', 'foo', [], '\t bar') + + def test_get_bare_quoted_string_multiple_words(self): + self._test_get_x(parser.get_bare_quoted_string, + '"foo bar moo"', '"foo bar moo"', 'foo bar moo', [], '') + + def test_get_bare_quoted_string_multiple_words_wsp_preserved(self): + self._test_get_x(parser.get_bare_quoted_string, + '" foo moo\t"', '" foo moo\t"', ' foo moo\t', [], '') + + def test_get_bare_quoted_string_end_dquote_mid_word(self): + self._test_get_x(parser.get_bare_quoted_string, + '"foo"bar', '"foo"', 'foo', [], 'bar') + + def test_get_bare_quoted_string_quoted_dquote(self): + self._test_get_x(parser.get_bare_quoted_string, + r'"foo\"in"a', r'"foo\"in"', 'foo"in', [], 'a') + + def test_get_bare_quoted_string_non_printables(self): + self._test_get_x(parser.get_bare_quoted_string, + '"a\x01a"', '"a\x01a"', 'a\x01a', + [errors.NonPrintableDefect], '') + + def test_get_bare_quoted_string_no_end_dquote(self): + self._test_get_x(parser.get_bare_quoted_string, + '"foo', '"foo"', 'foo', + [errors.InvalidHeaderDefect], '') + self._test_get_x(parser.get_bare_quoted_string, + '"foo ', '"foo "', 'foo ', + [errors.InvalidHeaderDefect], '') + + def test_get_bare_quoted_string_empty_quotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '""', '""', '', [], '') + + # Issue 16983: apply postel's law to some bad encoding. + def test_encoded_word_inside_quotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '"=?utf-8?Q?not_really_valid?="', + '"not really valid"', + 'not really valid', + [errors.InvalidHeaderDefect, + errors.InvalidHeaderDefect], + '') + + # get_comment + + def test_get_comment_only(self): + comment = self._test_get_x(parser.get_comment, + '(comment)', '(comment)', ' ', [], '', ['comment']) + self.assertEqual(comment.token_type, 'comment') + + def test_get_comment_must_start_with_paren(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_comment('foo"') + with self.assertRaises(errors.HeaderParseError): + parser.get_comment(' (foo"') + + def test_get_comment_following_wsp_preserved(self): + self._test_get_x(parser.get_comment, + '(comment) \t', '(comment)', ' ', [], ' \t', ['comment']) + + def test_get_comment_multiple_words(self): + self._test_get_x(parser.get_comment, + '(foo bar) \t', '(foo bar)', ' ', [], ' \t', ['foo bar']) + + def test_get_comment_multiple_words_wsp_preserved(self): + self._test_get_x(parser.get_comment, + '( foo bar\t ) \t', '( foo bar\t )', ' ', [], ' \t', + [' foo bar\t ']) + + def test_get_comment_end_paren_mid_word(self): + self._test_get_x(parser.get_comment, + '(foo)bar', '(foo)', ' ', [], 'bar', ['foo']) + + def test_get_comment_quoted_parens(self): + self._test_get_x(parser.get_comment, + r'(foo\) \(\)bar)', r'(foo\) \(\)bar)', ' ', [], '', ['foo) ()bar']) + + def test_get_comment_non_printable(self): + self._test_get_x(parser.get_comment, + '(foo\x7Fbar)', '(foo\x7Fbar)', ' ', + [errors.NonPrintableDefect], '', ['foo\x7Fbar']) + + def test_get_comment_no_end_paren(self): + self._test_get_x(parser.get_comment, + '(foo bar', '(foo bar)', ' ', + [errors.InvalidHeaderDefect], '', ['foo bar']) + self._test_get_x(parser.get_comment, + '(foo bar ', '(foo bar )', ' ', + [errors.InvalidHeaderDefect], '', ['foo bar ']) + + def test_get_comment_nested_comment(self): + comment = self._test_get_x(parser.get_comment, + '(foo(bar))', '(foo(bar))', ' ', [], '', ['foo(bar)']) + self.assertEqual(comment[1].content, 'bar') + + def test_get_comment_nested_comment_wsp(self): + comment = self._test_get_x(parser.get_comment, + '(foo ( bar ) )', '(foo ( bar ) )', ' ', [], '', ['foo ( bar ) ']) + self.assertEqual(comment[2].content, ' bar ') + + def test_get_comment_empty_comment(self): + self._test_get_x(parser.get_comment, + '()', '()', ' ', [], '', ['']) + + def test_get_comment_multiple_nesting(self): + comment = self._test_get_x(parser.get_comment, + '(((((foo)))))', '(((((foo)))))', ' ', [], '', ['((((foo))))']) + for i in range(4, 0, -1): + self.assertEqual(comment[0].content, '('*(i-1)+'foo'+')'*(i-1)) + comment = comment[0] + self.assertEqual(comment.content, 'foo') + + def test_get_comment_missing_end_of_nesting(self): + self._test_get_x(parser.get_comment, + '(((((foo)))', '(((((foo)))))', ' ', + [errors.InvalidHeaderDefect]*2, '', ['((((foo))))']) + + def test_get_comment_qs_in_nested_comment(self): + comment = self._test_get_x(parser.get_comment, + r'(foo (b\)))', r'(foo (b\)))', ' ', [], '', [r'foo (b\))']) + self.assertEqual(comment[2].content, 'b)') + + # get_cfws + + def test_get_cfws_only_ws(self): + cfws = self._test_get_x(parser.get_cfws, + ' \t \t', ' \t \t', ' ', [], '', []) + self.assertEqual(cfws.token_type, 'cfws') + + def test_get_cfws_only_comment(self): + cfws = self._test_get_x(parser.get_cfws, + '(foo)', '(foo)', ' ', [], '', ['foo']) + self.assertEqual(cfws[0].content, 'foo') + + def test_get_cfws_only_mixed(self): + cfws = self._test_get_x(parser.get_cfws, + ' (foo ) ( bar) ', ' (foo ) ( bar) ', ' ', [], '', + ['foo ', ' bar']) + self.assertEqual(cfws[1].content, 'foo ') + self.assertEqual(cfws[3].content, ' bar') + + def test_get_cfws_ends_at_non_leader(self): + cfws = self._test_get_x(parser.get_cfws, + '(foo) bar', '(foo) ', ' ', [], 'bar', ['foo']) + self.assertEqual(cfws[0].content, 'foo') + + def test_get_cfws_ends_at_non_printable(self): + cfws = self._test_get_x(parser.get_cfws, + '(foo) \x07', '(foo) ', ' ', [], '\x07', ['foo']) + self.assertEqual(cfws[0].content, 'foo') + + def test_get_cfws_non_printable_in_comment(self): + cfws = self._test_get_x(parser.get_cfws, + '(foo \x07) "test"', '(foo \x07) ', ' ', + [errors.NonPrintableDefect], '"test"', ['foo \x07']) + self.assertEqual(cfws[0].content, 'foo \x07') + + def test_get_cfws_header_ends_in_comment(self): + cfws = self._test_get_x(parser.get_cfws, + ' (foo ', ' (foo )', ' ', + [errors.InvalidHeaderDefect], '', ['foo ']) + self.assertEqual(cfws[1].content, 'foo ') + + def test_get_cfws_multiple_nested_comments(self): + cfws = self._test_get_x(parser.get_cfws, + '(foo (bar)) ((a)(a))', '(foo (bar)) ((a)(a))', ' ', [], + '', ['foo (bar)', '(a)(a)']) + self.assertEqual(cfws[0].comments, ['foo (bar)']) + self.assertEqual(cfws[2].comments, ['(a)(a)']) + + # get_quoted_string + + def test_get_quoted_string_only(self): + qs = self._test_get_x(parser.get_quoted_string, + '"bob"', '"bob"', 'bob', [], '') + self.assertEqual(qs.token_type, 'quoted-string') + self.assertEqual(qs.quoted_value, '"bob"') + self.assertEqual(qs.content, 'bob') + + def test_get_quoted_string_with_wsp(self): + qs = self._test_get_x(parser.get_quoted_string, + '\t "bob" ', '\t "bob" ', ' bob ', [], '') + self.assertEqual(qs.quoted_value, ' "bob" ') + self.assertEqual(qs.content, 'bob') + + def test_get_quoted_string_with_comments_and_wsp(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (foo) "bob"(bar)', ' (foo) "bob"(bar)', ' bob ', [], '') + self.assertEqual(qs[0][1].content, 'foo') + self.assertEqual(qs[2][0].content, 'bar') + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob" ') + + def test_get_quoted_string_with_multiple_comments(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (foo) (bar) "bob"(bird)', ' (foo) (bar) "bob"(bird)', ' bob ', + [], '') + self.assertEqual(qs[0].comments, ['foo', 'bar']) + self.assertEqual(qs[2].comments, ['bird']) + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob" ') + + def test_get_quoted_string_non_printable_in_comment(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (\x0A) "bob"', ' (\x0A) "bob"', ' bob', + [errors.NonPrintableDefect], '') + self.assertEqual(qs[0].comments, ['\x0A']) + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob"') + + def test_get_quoted_string_non_printable_in_qcontent(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (a) "a\x0B"', ' (a) "a\x0B"', ' a\x0B', + [errors.NonPrintableDefect], '') + self.assertEqual(qs[0].comments, ['a']) + self.assertEqual(qs.content, 'a\x0B') + self.assertEqual(qs.quoted_value, ' "a\x0B"') + + def test_get_quoted_string_internal_ws(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (a) "foo bar "', ' (a) "foo bar "', ' foo bar ', + [], '') + self.assertEqual(qs[0].comments, ['a']) + self.assertEqual(qs.content, 'foo bar ') + self.assertEqual(qs.quoted_value, ' "foo bar "') + + def test_get_quoted_string_header_ends_in_comment(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (a) "bob" (a', ' (a) "bob" (a)', ' bob ', + [errors.InvalidHeaderDefect], '') + self.assertEqual(qs[0].comments, ['a']) + self.assertEqual(qs[2].comments, ['a']) + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob" ') + + def test_get_quoted_string_header_ends_in_qcontent(self): + qs = self._test_get_x(parser.get_quoted_string, + ' (a) "bob', ' (a) "bob"', ' bob', + [errors.InvalidHeaderDefect], '') + self.assertEqual(qs[0].comments, ['a']) + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob"') + + def test_get_quoted_string_cfws_only_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_quoted_string(' (foo) ') + + def test_get_quoted_string_no_quoted_string(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_quoted_string(' (ab) xyz') + + def test_get_quoted_string_qs_ends_at_noncfws(self): + qs = self._test_get_x(parser.get_quoted_string, + '\t "bob" fee', '\t "bob" ', ' bob ', [], 'fee') + self.assertEqual(qs.content, 'bob') + self.assertEqual(qs.quoted_value, ' "bob" ') + + # get_atom + + def test_get_atom_only(self): + atom = self._test_get_x(parser.get_atom, + 'bob', 'bob', 'bob', [], '') + self.assertEqual(atom.token_type, 'atom') + + def test_get_atom_with_wsp(self): + self._test_get_x(parser.get_atom, + '\t bob ', '\t bob ', ' bob ', [], '') + + def test_get_atom_with_comments_and_wsp(self): + atom = self._test_get_x(parser.get_atom, + ' (foo) bob(bar)', ' (foo) bob(bar)', ' bob ', [], '') + self.assertEqual(atom[0][1].content, 'foo') + self.assertEqual(atom[2][0].content, 'bar') + + def test_get_atom_with_multiple_comments(self): + atom = self._test_get_x(parser.get_atom, + ' (foo) (bar) bob(bird)', ' (foo) (bar) bob(bird)', ' bob ', + [], '') + self.assertEqual(atom[0].comments, ['foo', 'bar']) + self.assertEqual(atom[2].comments, ['bird']) + + def test_get_atom_non_printable_in_comment(self): + atom = self._test_get_x(parser.get_atom, + ' (\x0A) bob', ' (\x0A) bob', ' bob', + [errors.NonPrintableDefect], '') + self.assertEqual(atom[0].comments, ['\x0A']) + + def test_get_atom_non_printable_in_atext(self): + atom = self._test_get_x(parser.get_atom, + ' (a) a\x0B', ' (a) a\x0B', ' a\x0B', + [errors.NonPrintableDefect], '') + self.assertEqual(atom[0].comments, ['a']) + + def test_get_atom_header_ends_in_comment(self): + atom = self._test_get_x(parser.get_atom, + ' (a) bob (a', ' (a) bob (a)', ' bob ', + [errors.InvalidHeaderDefect], '') + self.assertEqual(atom[0].comments, ['a']) + self.assertEqual(atom[2].comments, ['a']) + + def test_get_atom_no_atom(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_atom(' (ab) ') + + def test_get_atom_no_atom_before_special(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_atom(' (ab) @') + + def test_get_atom_atom_ends_at_special(self): + atom = self._test_get_x(parser.get_atom, + ' (foo) bob(bar) @bang', ' (foo) bob(bar) ', ' bob ', [], '@bang') + self.assertEqual(atom[0].comments, ['foo']) + self.assertEqual(atom[2].comments, ['bar']) + + def test_get_atom_atom_ends_at_noncfws(self): + self._test_get_x(parser.get_atom, + 'bob fred', 'bob ', 'bob ', [], 'fred') + + def test_get_atom_rfc2047_atom(self): + self._test_get_x(parser.get_atom, + '=?utf-8?q?=20bob?=', ' bob', ' bob', [], '') + + # get_dot_atom_text + + def test_get_dot_atom_text(self): + dot_atom_text = self._test_get_x(parser.get_dot_atom_text, + 'foo.bar.bang', 'foo.bar.bang', 'foo.bar.bang', [], '') + self.assertEqual(dot_atom_text.token_type, 'dot-atom-text') + self.assertEqual(len(dot_atom_text), 5) + + def test_get_dot_atom_text_lone_atom_is_valid(self): + dot_atom_text = self._test_get_x(parser.get_dot_atom_text, + 'foo', 'foo', 'foo', [], '') + + def test_get_dot_atom_text_raises_on_leading_dot(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom_text('.foo.bar') + + def test_get_dot_atom_text_raises_on_trailing_dot(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom_text('foo.bar.') + + def test_get_dot_atom_text_raises_on_leading_non_atext(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom_text(' foo.bar') + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom_text('@foo.bar') + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom_text('"foo.bar"') + + def test_get_dot_atom_text_trailing_text_preserved(self): + dot_atom_text = self._test_get_x(parser.get_dot_atom_text, + 'foo@bar', 'foo', 'foo', [], '@bar') + + def test_get_dot_atom_text_trailing_ws_preserved(self): + dot_atom_text = self._test_get_x(parser.get_dot_atom_text, + 'foo .bar', 'foo', 'foo', [], ' .bar') + + # get_dot_atom + + def test_get_dot_atom_only(self): + dot_atom = self._test_get_x(parser.get_dot_atom, + 'foo.bar.bing', 'foo.bar.bing', 'foo.bar.bing', [], '') + self.assertEqual(dot_atom.token_type, 'dot-atom') + self.assertEqual(len(dot_atom), 1) + + def test_get_dot_atom_with_wsp(self): + self._test_get_x(parser.get_dot_atom, + '\t foo.bar.bing ', '\t foo.bar.bing ', ' foo.bar.bing ', [], '') + + def test_get_dot_atom_with_comments_and_wsp(self): + self._test_get_x(parser.get_dot_atom, + ' (sing) foo.bar.bing (here) ', ' (sing) foo.bar.bing (here) ', + ' foo.bar.bing ', [], '') + + def test_get_dot_atom_space_ends_dot_atom(self): + self._test_get_x(parser.get_dot_atom, + ' (sing) foo.bar .bing (here) ', ' (sing) foo.bar ', + ' foo.bar ', [], '.bing (here) ') + + def test_get_dot_atom_no_atom_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom(' (foo) ') + + def test_get_dot_atom_leading_dot_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom(' (foo) .bar') + + def test_get_dot_atom_two_dots_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom('bar..bang') + + def test_get_dot_atom_trailing_dot_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_dot_atom(' (foo) bar.bang. foo') + + def test_get_dot_atom_rfc2047_atom(self): + self._test_get_x(parser.get_dot_atom, + '=?utf-8?q?=20bob?=', ' bob', ' bob', [], '') + + # get_word (if this were black box we'd repeat all the qs/atom tests) + + def test_get_word_atom_yields_atom(self): + word = self._test_get_x(parser.get_word, + ' (foo) bar (bang) :ah', ' (foo) bar (bang) ', ' bar ', [], ':ah') + self.assertEqual(word.token_type, 'atom') + self.assertEqual(word[0].token_type, 'cfws') + + def test_get_word_all_CFWS(self): + # bpo-29412: Test that we don't raise IndexError when parsing CFWS only + # token. + with self.assertRaises(errors.HeaderParseError): + parser.get_word('(Recipients list suppressed') + + def test_get_word_qs_yields_qs(self): + word = self._test_get_x(parser.get_word, + '"bar " (bang) ah', '"bar " (bang) ', 'bar ', [], 'ah') + self.assertEqual(word.token_type, 'quoted-string') + self.assertEqual(word[0].token_type, 'bare-quoted-string') + self.assertEqual(word[0].value, 'bar ') + self.assertEqual(word.content, 'bar ') + + def test_get_word_ends_at_dot(self): + self._test_get_x(parser.get_word, + 'foo.', 'foo', 'foo', [], '.') + + # get_phrase + + def test_get_phrase_simple(self): + phrase = self._test_get_x(parser.get_phrase, + '"Fred A. Johnson" is his name, oh.', + '"Fred A. Johnson" is his name', + 'Fred A. Johnson is his name', + [], + ', oh.') + self.assertEqual(phrase.token_type, 'phrase') + + def test_get_phrase_complex(self): + phrase = self._test_get_x(parser.get_phrase, + ' (A) bird (in (my|your)) "hand " is messy\t<>\t', + ' (A) bird (in (my|your)) "hand " is messy\t', + ' bird hand is messy ', + [], + '<>\t') + self.assertEqual(phrase[0][0].comments, ['A']) + self.assertEqual(phrase[0][2].comments, ['in (my|your)']) + + def test_get_phrase_obsolete(self): + phrase = self._test_get_x(parser.get_phrase, + 'Fred A.(weird).O Johnson', + 'Fred A.(weird).O Johnson', + 'Fred A. .O Johnson', + [errors.ObsoleteHeaderDefect]*3, + '') + self.assertEqual(len(phrase), 7) + self.assertEqual(phrase[3].comments, ['weird']) + + def test_get_phrase_pharse_must_start_with_word(self): + phrase = self._test_get_x(parser.get_phrase, + '(even weirder).name', + '(even weirder).name', + ' .name', + [errors.InvalidHeaderDefect] + [errors.ObsoleteHeaderDefect]*2, + '') + self.assertEqual(len(phrase), 3) + self.assertEqual(phrase[0].comments, ['even weirder']) + + def test_get_phrase_ending_with_obsolete(self): + phrase = self._test_get_x(parser.get_phrase, + 'simple phrase.(with trailing comment):boo', + 'simple phrase.(with trailing comment)', + 'simple phrase. ', + [errors.ObsoleteHeaderDefect]*2, + ':boo') + self.assertEqual(len(phrase), 4) + self.assertEqual(phrase[3].comments, ['with trailing comment']) + + def get_phrase_cfws_only_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_phrase(' (foo) ') + + # get_local_part + + def test_get_local_part_simple(self): + local_part = self._test_get_x(parser.get_local_part, + 'dinsdale@python.org', 'dinsdale', 'dinsdale', [], '@python.org') + self.assertEqual(local_part.token_type, 'local-part') + self.assertEqual(local_part.local_part, 'dinsdale') + + def test_get_local_part_with_dot(self): + local_part = self._test_get_x(parser.get_local_part, + 'Fred.A.Johnson@python.org', + 'Fred.A.Johnson', + 'Fred.A.Johnson', + [], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson') + + def test_get_local_part_with_whitespace(self): + local_part = self._test_get_x(parser.get_local_part, + ' Fred.A.Johnson @python.org', + ' Fred.A.Johnson ', + ' Fred.A.Johnson ', + [], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson') + + def test_get_local_part_with_cfws(self): + local_part = self._test_get_x(parser.get_local_part, + ' (foo) Fred.A.Johnson (bar (bird)) @python.org', + ' (foo) Fred.A.Johnson (bar (bird)) ', + ' Fred.A.Johnson ', + [], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson') + self.assertEqual(local_part[0][0].comments, ['foo']) + self.assertEqual(local_part[0][2].comments, ['bar (bird)']) + + def test_get_local_part_simple_quoted(self): + local_part = self._test_get_x(parser.get_local_part, + '"dinsdale"@python.org', '"dinsdale"', '"dinsdale"', [], '@python.org') + self.assertEqual(local_part.token_type, 'local-part') + self.assertEqual(local_part.local_part, 'dinsdale') + + def test_get_local_part_with_quoted_dot(self): + local_part = self._test_get_x(parser.get_local_part, + '"Fred.A.Johnson"@python.org', + '"Fred.A.Johnson"', + '"Fred.A.Johnson"', + [], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson') + + def test_get_local_part_quoted_with_whitespace(self): + local_part = self._test_get_x(parser.get_local_part, + ' "Fred A. Johnson" @python.org', + ' "Fred A. Johnson" ', + ' "Fred A. Johnson" ', + [], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred A. Johnson') + + def test_get_local_part_quoted_with_cfws(self): + local_part = self._test_get_x(parser.get_local_part, + ' (foo) " Fred A. Johnson " (bar (bird)) @python.org', + ' (foo) " Fred A. Johnson " (bar (bird)) ', + ' " Fred A. Johnson " ', + [], + '@python.org') + self.assertEqual(local_part.local_part, ' Fred A. Johnson ') + self.assertEqual(local_part[0][0].comments, ['foo']) + self.assertEqual(local_part[0][2].comments, ['bar (bird)']) + + + def test_get_local_part_simple_obsolete(self): + local_part = self._test_get_x(parser.get_local_part, + 'Fred. A.Johnson@python.org', + 'Fred. A.Johnson', + 'Fred. A.Johnson', + [errors.ObsoleteHeaderDefect], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson') + + def test_get_local_part_complex_obsolete_1(self): + local_part = self._test_get_x(parser.get_local_part, + ' (foo )Fred (bar).(bird) A.(sheep)Johnson."and dogs "@python.org', + ' (foo )Fred (bar).(bird) A.(sheep)Johnson."and dogs "', + ' Fred . A. Johnson.and dogs ', + [errors.ObsoleteHeaderDefect], + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson.and dogs ') + + def test_get_local_part_complex_obsolete_invalid(self): + local_part = self._test_get_x(parser.get_local_part, + ' (foo )Fred (bar).(bird) A.(sheep)Johnson "and dogs"@python.org', + ' (foo )Fred (bar).(bird) A.(sheep)Johnson "and dogs"', + ' Fred . A. Johnson and dogs', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, 'Fred.A.Johnson and dogs') + + def test_get_local_part_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part('') + + def test_get_local_part_no_part_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part(' (foo) ') + + def test_get_local_part_special_instead_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part(' (foo) @python.org') + + def test_get_local_part_trailing_dot(self): + local_part = self._test_get_x(parser.get_local_part, + ' borris.@python.org', + ' borris.', + ' borris.', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, 'borris.') + + def test_get_local_part_trailing_dot_with_ws(self): + local_part = self._test_get_x(parser.get_local_part, + ' borris. @python.org', + ' borris. ', + ' borris. ', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, 'borris.') + + def test_get_local_part_leading_dot(self): + local_part = self._test_get_x(parser.get_local_part, + '.borris@python.org', + '.borris', + '.borris', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, '.borris') + + def test_get_local_part_leading_dot_after_ws(self): + local_part = self._test_get_x(parser.get_local_part, + ' .borris@python.org', + ' .borris', + ' .borris', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, '.borris') + + def test_get_local_part_double_dot_raises(self): + local_part = self._test_get_x(parser.get_local_part, + ' borris.(foo).natasha@python.org', + ' borris.(foo).natasha', + ' borris. .natasha', + [errors.InvalidHeaderDefect]*2, + '@python.org') + self.assertEqual(local_part.local_part, 'borris..natasha') + + def test_get_local_part_quoted_strings_in_atom_list(self): + local_part = self._test_get_x(parser.get_local_part, + '""example" example"@example.com', + '""example" example"', + 'example example', + [errors.InvalidHeaderDefect]*3, + '@example.com') + self.assertEqual(local_part.local_part, 'example example') + + def test_get_local_part_valid_and_invalid_qp_in_atom_list(self): + local_part = self._test_get_x(parser.get_local_part, + r'"\\"example\\" example"@example.com', + r'"\\"example\\" example"', + r'\example\\ example', + [errors.InvalidHeaderDefect]*5, + '@example.com') + self.assertEqual(local_part.local_part, r'\example\\ example') + + def test_get_local_part_unicode_defect(self): + # Currently this only happens when parsing unicode, not when parsing + # stuff that was originally binary. + local_part = self._test_get_x(parser.get_local_part, + 'exámple@example.com', + 'exámple', + 'exámple', + [errors.NonASCIILocalPartDefect], + '@example.com') + self.assertEqual(local_part.local_part, 'exámple') + + # get_dtext + + def test_get_dtext_only(self): + dtext = self._test_get_x(parser.get_dtext, + 'foobar', 'foobar', 'foobar', [], '') + self.assertEqual(dtext.token_type, 'ptext') + + def test_get_dtext_all_dtext(self): + dtext = self._test_get_x(parser.get_dtext, self.rfc_dtext_chars, + self.rfc_dtext_chars, + self.rfc_dtext_chars, [], '') + + def test_get_dtext_two_words_gets_first(self): + self._test_get_x(parser.get_dtext, + 'foo bar', 'foo', 'foo', [], ' bar') + + def test_get_dtext_following_wsp_preserved(self): + self._test_get_x(parser.get_dtext, + 'foo \t\tbar', 'foo', 'foo', [], ' \t\tbar') + + def test_get_dtext_non_printables(self): + dtext = self._test_get_x(parser.get_dtext, + 'foo\x00bar]', 'foo\x00bar', 'foo\x00bar', + [errors.NonPrintableDefect], ']') + self.assertEqual(dtext.defects[0].non_printables[0], '\x00') + + def test_get_dtext_with_qp(self): + ptext = self._test_get_x(parser.get_dtext, + r'foo\]\[\\bar\b\e\l\l', + r'foo][\barbell', + r'foo][\barbell', + [errors.ObsoleteHeaderDefect], + '') + + def test_get_dtext_up_to_close_bracket_only(self): + self._test_get_x(parser.get_dtext, + 'foo]', 'foo', 'foo', [], ']') + + def test_get_dtext_wsp_before_close_bracket_preserved(self): + self._test_get_x(parser.get_dtext, + 'foo ]', 'foo', 'foo', [], ' ]') + + def test_get_dtext_close_bracket_mid_word(self): + self._test_get_x(parser.get_dtext, + 'foo]bar', 'foo', 'foo', [], ']bar') + + def test_get_dtext_up_to_open_bracket_only(self): + self._test_get_x(parser.get_dtext, + 'foo[', 'foo', 'foo', [], '[') + + def test_get_dtext_wsp_before_open_bracket_preserved(self): + self._test_get_x(parser.get_dtext, + 'foo [', 'foo', 'foo', [], ' [') + + def test_get_dtext_open_bracket_mid_word(self): + self._test_get_x(parser.get_dtext, + 'foo[bar', 'foo', 'foo', [], '[bar') + + def test_get_dtext_open_bracket_only(self): + self._test_get_x(parser.get_dtext, + '[', '', '', [], '[') + + def test_get_dtext_close_bracket_only(self): + self._test_get_x(parser.get_dtext, + ']', '', '', [], ']') + + def test_get_dtext_empty(self): + self._test_get_x(parser.get_dtext, + '', '', '', [], '') + + # get_domain_literal + + def test_get_domain_literal_only(self): + domain_literal = domain_literal = self._test_get_x(parser.get_domain_literal, + '[127.0.0.1]', + '[127.0.0.1]', + '[127.0.0.1]', + [], + '') + self.assertEqual(domain_literal.token_type, 'domain-literal') + self.assertEqual(domain_literal.domain, '[127.0.0.1]') + self.assertEqual(domain_literal.ip, '127.0.0.1') + + def test_get_domain_literal_with_internal_ws(self): + domain_literal = self._test_get_x(parser.get_domain_literal, + '[ 127.0.0.1\t ]', + '[ 127.0.0.1\t ]', + '[ 127.0.0.1 ]', + [], + '') + self.assertEqual(domain_literal.domain, '[127.0.0.1]') + self.assertEqual(domain_literal.ip, '127.0.0.1') + + def test_get_domain_literal_with_surrounding_cfws(self): + domain_literal = self._test_get_x(parser.get_domain_literal, + '(foo)[ 127.0.0.1] (bar)', + '(foo)[ 127.0.0.1] (bar)', + ' [ 127.0.0.1] ', + [], + '') + self.assertEqual(domain_literal.domain, '[127.0.0.1]') + self.assertEqual(domain_literal.ip, '127.0.0.1') + + def test_get_domain_literal_no_start_char_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain_literal('(foo) ') + + def test_get_domain_literal_no_start_char_before_special_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain_literal('(foo) @') + + def test_get_domain_literal_bad_dtext_char_before_special_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain_literal('(foo) [abc[@') + + # get_domain + + def test_get_domain_regular_domain_only(self): + domain = self._test_get_x(parser.get_domain, + 'example.com', + 'example.com', + 'example.com', + [], + '') + self.assertEqual(domain.token_type, 'domain') + self.assertEqual(domain.domain, 'example.com') + + def test_get_domain_domain_literal_only(self): + domain = self._test_get_x(parser.get_domain, + '[127.0.0.1]', + '[127.0.0.1]', + '[127.0.0.1]', + [], + '') + self.assertEqual(domain.token_type, 'domain') + self.assertEqual(domain.domain, '[127.0.0.1]') + + def test_get_domain_with_cfws(self): + domain = self._test_get_x(parser.get_domain, + '(foo) example.com(bar)\t', + '(foo) example.com(bar)\t', + ' example.com ', + [], + '') + self.assertEqual(domain.domain, 'example.com') + + def test_get_domain_domain_literal_with_cfws(self): + domain = self._test_get_x(parser.get_domain, + '(foo)[127.0.0.1]\t(bar)', + '(foo)[127.0.0.1]\t(bar)', + ' [127.0.0.1] ', + [], + '') + self.assertEqual(domain.domain, '[127.0.0.1]') + + def test_get_domain_domain_with_cfws_ends_at_special(self): + domain = self._test_get_x(parser.get_domain, + '(foo)example.com\t(bar), next', + '(foo)example.com\t(bar)', + ' example.com ', + [], + ', next') + self.assertEqual(domain.domain, 'example.com') + + def test_get_domain_domain_literal_with_cfws_ends_at_special(self): + domain = self._test_get_x(parser.get_domain, + '(foo)[127.0.0.1]\t(bar), next', + '(foo)[127.0.0.1]\t(bar)', + ' [127.0.0.1] ', + [], + ', next') + self.assertEqual(domain.domain, '[127.0.0.1]') + + def test_get_domain_obsolete(self): + domain = self._test_get_x(parser.get_domain, + '(foo) example . (bird)com(bar)\t', + '(foo) example . (bird)com(bar)\t', + ' example . com ', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(domain.domain, 'example.com') + + def test_get_domain_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain("") + + def test_get_domain_no_non_cfws_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain(" (foo)\t") + + def test_get_domain_no_atom_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain(" (foo)\t, broken") + + + # get_addr_spec + + def test_get_addr_spec_normal(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + 'dinsdale@example.com', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + '') + self.assertEqual(addr_spec.token_type, 'addr-spec') + self.assertEqual(addr_spec.local_part, 'dinsdale') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, 'dinsdale@example.com') + + def test_get_addr_spec_with_doamin_literal(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + 'dinsdale@[127.0.0.1]', + 'dinsdale@[127.0.0.1]', + 'dinsdale@[127.0.0.1]', + [], + '') + self.assertEqual(addr_spec.local_part, 'dinsdale') + self.assertEqual(addr_spec.domain, '[127.0.0.1]') + self.assertEqual(addr_spec.addr_spec, 'dinsdale@[127.0.0.1]') + + def test_get_addr_spec_with_cfws(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + '(foo) dinsdale(bar)@ (bird) example.com (bog)', + '(foo) dinsdale(bar)@ (bird) example.com (bog)', + ' dinsdale@example.com ', + [], + '') + self.assertEqual(addr_spec.local_part, 'dinsdale') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, 'dinsdale@example.com') + + def test_get_addr_spec_with_qouoted_string_and_cfws(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + '(foo) "roy a bug"(bar)@ (bird) example.com (bog)', + '(foo) "roy a bug"(bar)@ (bird) example.com (bog)', + ' "roy a bug"@example.com ', + [], + '') + self.assertEqual(addr_spec.local_part, 'roy a bug') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, '"roy a bug"@example.com') + + def test_get_addr_spec_ends_at_special(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + '(foo) "roy a bug"(bar)@ (bird) example.com (bog) , next', + '(foo) "roy a bug"(bar)@ (bird) example.com (bog) ', + ' "roy a bug"@example.com ', + [], + ', next') + self.assertEqual(addr_spec.local_part, 'roy a bug') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, '"roy a bug"@example.com') + + def test_get_addr_spec_quoted_strings_in_atom_list(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + '""example" example"@example.com', + '""example" example"@example.com', + 'example example@example.com', + [errors.InvalidHeaderDefect]*3, + '') + self.assertEqual(addr_spec.local_part, 'example example') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, '"example example"@example.com') + + def test_get_addr_spec_dot_atom(self): + addr_spec = self._test_get_x(parser.get_addr_spec, + 'star.a.star@example.com', + 'star.a.star@example.com', + 'star.a.star@example.com', + [], + '') + self.assertEqual(addr_spec.local_part, 'star.a.star') + self.assertEqual(addr_spec.domain, 'example.com') + self.assertEqual(addr_spec.addr_spec, 'star.a.star@example.com') + + def test_get_addr_spec_multiple_domains(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@a.star@example.com') + + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@a@example.com') + + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@172.17.0.1@example.com') + + # get_obs_route + + def test_get_obs_route_simple(self): + obs_route = self._test_get_x(parser.get_obs_route, + '@example.com, @two.example.com:', + '@example.com, @two.example.com:', + '@example.com, @two.example.com:', + [], + '') + self.assertEqual(obs_route.token_type, 'obs-route') + self.assertEqual(obs_route.domains, ['example.com', 'two.example.com']) + + def test_get_obs_route_complex(self): + obs_route = self._test_get_x(parser.get_obs_route, + '(foo),, (blue)@example.com (bar),@two.(foo) example.com (bird):', + '(foo),, (blue)@example.com (bar),@two.(foo) example.com (bird):', + ' ,, @example.com ,@two. example.com :', + [errors.ObsoleteHeaderDefect], # This is the obs-domain + '') + self.assertEqual(obs_route.token_type, 'obs-route') + self.assertEqual(obs_route.domains, ['example.com', 'two.example.com']) + + def test_get_obs_route_no_route_before_end_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com,') + + def test_get_obs_route_no_route_before_end_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com, (foo) ') + + def test_get_obs_route_no_route_before_special_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) [abc],') + + def test_get_obs_route_no_route_before_special_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com [abc],') + + def test_get_obs_route_no_domain_after_at_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@') + + def test_get_obs_route_no_domain_after_at_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@example.com, @') + + # get_angle_addr + + def test_get_angle_addr_simple(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '', + '', + '', + [], + '') + self.assertEqual(angle_addr.token_type, 'angle-addr') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_empty(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '<>', + '<>', + '<>', + [errors.InvalidHeaderDefect], + '') + self.assertEqual(angle_addr.token_type, 'angle-addr') + self.assertIsNone(angle_addr.local_part) + self.assertIsNone(angle_addr.domain) + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, '<>') + + def test_get_angle_addr_qs_only_quotes(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '<""@example.com>', + '<""@example.com>', + '<""@example.com>', + [], + '') + self.assertEqual(angle_addr.token_type, 'angle-addr') + self.assertEqual(angle_addr.local_part, '') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, '""@example.com') + + def test_get_angle_addr_with_cfws(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + ' (foo) (bar)', + ' (foo) (bar)', + ' ', + [], + '') + self.assertEqual(angle_addr.token_type, 'angle-addr') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_qs_and_domain_literal(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '<"Fred Perfect"@[127.0.0.1]>', + '<"Fred Perfect"@[127.0.0.1]>', + '<"Fred Perfect"@[127.0.0.1]>', + [], + '') + self.assertEqual(angle_addr.local_part, 'Fred Perfect') + self.assertEqual(angle_addr.domain, '[127.0.0.1]') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, '"Fred Perfect"@[127.0.0.1]') + + def test_get_angle_addr_internal_cfws(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '<(foo) dinsdale@example.com(bar)>', + '<(foo) dinsdale@example.com(bar)>', + '< dinsdale@example.com >', + [], + '') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_obs_route(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '(foo)<@example.com, (bird) @two.example.com: dinsdale@example.com> (bar) ', + '(foo)<@example.com, (bird) @two.example.com: dinsdale@example.com> (bar) ', + ' <@example.com, @two.example.com: dinsdale@example.com> ', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertEqual(angle_addr.route, ['example.com', 'two.example.com']) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_missing_closing_angle(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '', + '', + [errors.InvalidHeaderDefect], + '') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_missing_closing_angle_with_cfws(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '', + '', + [errors.InvalidHeaderDefect], + '') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_ends_at_special(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + ' (foo), next', + ' (foo)', + ' ', + [], + ', next') + self.assertEqual(angle_addr.local_part, 'dinsdale') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + + def test_get_angle_addr_empty_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('') + + def test_get_angle_addr_left_angle_only_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('<') + + def test_get_angle_addr_no_angle_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('(foo) ') + + def test_get_angle_addr_no_angle_before_special_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('(foo) , next') + + def test_get_angle_addr_no_angle_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('bar') + + def test_get_angle_addr_special_after_angle_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('(foo) <, bar') + + # get_display_name This is phrase but with a different value. + + def test_get_display_name_simple(self): + display_name = self._test_get_x(parser.get_display_name, + 'Fred A Johnson', + 'Fred A Johnson', + 'Fred A Johnson', + [], + '') + self.assertEqual(display_name.token_type, 'display-name') + self.assertEqual(display_name.display_name, 'Fred A Johnson') + + def test_get_display_name_complex1(self): + display_name = self._test_get_x(parser.get_display_name, + '"Fred A. Johnson" is his name, oh.', + '"Fred A. Johnson" is his name', + '"Fred A. Johnson is his name"', + [], + ', oh.') + self.assertEqual(display_name.token_type, 'display-name') + self.assertEqual(display_name.display_name, 'Fred A. Johnson is his name') + + def test_get_display_name_complex2(self): + display_name = self._test_get_x(parser.get_display_name, + ' (A) bird (in (my|your)) "hand " is messy\t<>\t', + ' (A) bird (in (my|your)) "hand " is messy\t', + ' "bird hand is messy" ', + [], + '<>\t') + self.assertEqual(display_name[0][0].comments, ['A']) + self.assertEqual(display_name[0][2].comments, ['in (my|your)']) + self.assertEqual(display_name.display_name, 'bird hand is messy') + + def test_get_display_name_obsolete(self): + display_name = self._test_get_x(parser.get_display_name, + 'Fred A.(weird).O Johnson', + 'Fred A.(weird).O Johnson', + '"Fred A. .O Johnson"', + [errors.ObsoleteHeaderDefect]*3, + '') + self.assertEqual(len(display_name), 7) + self.assertEqual(display_name[3].comments, ['weird']) + self.assertEqual(display_name.display_name, 'Fred A. .O Johnson') + + def test_get_display_name_pharse_must_start_with_word(self): + display_name = self._test_get_x(parser.get_display_name, + '(even weirder).name', + '(even weirder).name', + ' ".name"', + [errors.InvalidHeaderDefect] + [errors.ObsoleteHeaderDefect]*2, + '') + self.assertEqual(len(display_name), 3) + self.assertEqual(display_name[0].comments, ['even weirder']) + self.assertEqual(display_name.display_name, '.name') + + def test_get_display_name_ending_with_obsolete(self): + display_name = self._test_get_x(parser.get_display_name, + 'simple phrase.(with trailing comment):boo', + 'simple phrase.(with trailing comment)', + '"simple phrase." ', + [errors.ObsoleteHeaderDefect]*2, + ':boo') + self.assertEqual(len(display_name), 4) + self.assertEqual(display_name[3].comments, ['with trailing comment']) + self.assertEqual(display_name.display_name, 'simple phrase.') + + def test_get_display_name_for_invalid_address_field(self): + # bpo-32178: Test that address fields starting with `:` don't cause + # IndexError when parsing the display name. + display_name = self._test_get_x( + parser.get_display_name, + ':Foo ', '', '', [errors.InvalidHeaderDefect], ':Foo ') + self.assertEqual(display_name.value, '') + + # get_name_addr + + def test_get_name_addr_angle_addr_only(self): + name_addr = self._test_get_x(parser.get_name_addr, + '', + '', + '', + [], + '') + self.assertEqual(name_addr.token_type, 'name-addr') + self.assertIsNone(name_addr.display_name) + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_atom_name(self): + name_addr = self._test_get_x(parser.get_name_addr, + 'Dinsdale ', + 'Dinsdale ', + 'Dinsdale ', + [], + '') + self.assertEqual(name_addr.token_type, 'name-addr') + self.assertEqual(name_addr.display_name, 'Dinsdale') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_atom_name_with_cfws(self): + name_addr = self._test_get_x(parser.get_name_addr, + '(foo) Dinsdale (bar) (bird)', + '(foo) Dinsdale (bar) (bird)', + ' Dinsdale ', + [], + '') + self.assertEqual(name_addr.display_name, 'Dinsdale') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_name_with_cfws_and_dots(self): + name_addr = self._test_get_x(parser.get_name_addr, + '(foo) Roy.A.Bear (bar) (bird)', + '(foo) Roy.A.Bear (bar) (bird)', + ' "Roy.A.Bear" ', + [errors.ObsoleteHeaderDefect]*2, + '') + self.assertEqual(name_addr.display_name, 'Roy.A.Bear') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_qs_name(self): + name_addr = self._test_get_x(parser.get_name_addr, + '"Roy.A.Bear" ', + '"Roy.A.Bear" ', + '"Roy.A.Bear" ', + [], + '') + self.assertEqual(name_addr.display_name, 'Roy.A.Bear') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_ending_with_dot_without_space(self): + name_addr = self._test_get_x(parser.get_name_addr, + 'John X.', + 'John X.', + '"John X."', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, 'John X.') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + + def test_get_name_addr_starting_with_dot(self): + name_addr = self._test_get_x(parser.get_name_addr, + '. Doe ', + '. Doe ', + '". Doe" ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, '. Doe') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + + def test_get_name_addr_with_route(self): + name_addr = self._test_get_x(parser.get_name_addr, + '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', + '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', + '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, 'Roy.A.Bear') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertEqual(name_addr.route, ['two.example.com']) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_ends_at_special(self): + name_addr = self._test_get_x(parser.get_name_addr, + '"Roy.A.Bear" , next', + '"Roy.A.Bear" ', + '"Roy.A.Bear" ', + [], + ', next') + self.assertEqual(name_addr.display_name, 'Roy.A.Bear') + self.assertEqual(name_addr.local_part, 'dinsdale') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + + def test_get_name_addr_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr('') + + def test_get_name_addr_no_content_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr(' (foo) ') + + def test_get_name_addr_no_content_before_special_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr(' (foo) ,') + + def test_get_name_addr_no_angle_after_display_name_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr('foo bar') + + # get_mailbox + + def test_get_mailbox_addr_spec_only(self): + mailbox = self._test_get_x(parser.get_mailbox, + 'dinsdale@example.com', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + '') + self.assertEqual(mailbox.token_type, 'mailbox') + self.assertIsNone(mailbox.display_name) + self.assertEqual(mailbox.local_part, 'dinsdale') + self.assertEqual(mailbox.domain, 'example.com') + self.assertIsNone(mailbox.route) + self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com') + + def test_get_mailbox_angle_addr_only(self): + mailbox = self._test_get_x(parser.get_mailbox, + '', + '', + '', + [], + '') + self.assertEqual(mailbox.token_type, 'mailbox') + self.assertIsNone(mailbox.display_name) + self.assertEqual(mailbox.local_part, 'dinsdale') + self.assertEqual(mailbox.domain, 'example.com') + self.assertIsNone(mailbox.route) + self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com') + + def test_get_mailbox_name_addr(self): + mailbox = self._test_get_x(parser.get_mailbox, + '"Roy A. Bear" ', + '"Roy A. Bear" ', + '"Roy A. Bear" ', + [], + '') + self.assertEqual(mailbox.token_type, 'mailbox') + self.assertEqual(mailbox.display_name, 'Roy A. Bear') + self.assertEqual(mailbox.local_part, 'dinsdale') + self.assertEqual(mailbox.domain, 'example.com') + self.assertIsNone(mailbox.route) + self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com') + + def test_get_mailbox_ends_at_special(self): + mailbox = self._test_get_x(parser.get_mailbox, + '"Roy A. Bear" , rest', + '"Roy A. Bear" ', + '"Roy A. Bear" ', + [], + ', rest') + self.assertEqual(mailbox.token_type, 'mailbox') + self.assertEqual(mailbox.display_name, 'Roy A. Bear') + self.assertEqual(mailbox.local_part, 'dinsdale') + self.assertEqual(mailbox.domain, 'example.com') + self.assertIsNone(mailbox.route) + self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com') + + def test_get_mailbox_quoted_strings_in_atom_list(self): + mailbox = self._test_get_x(parser.get_mailbox, + '""example" example"@example.com', + '""example" example"@example.com', + 'example example@example.com', + [errors.InvalidHeaderDefect]*3, + '') + self.assertEqual(mailbox.local_part, 'example example') + self.assertEqual(mailbox.domain, 'example.com') + self.assertEqual(mailbox.addr_spec, '"example example"@example.com') + + # get_mailbox_list + + def test_get_mailbox_list_single_addr(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + 'dinsdale@example.com', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + '') + self.assertEqual(mailbox_list.token_type, 'mailbox-list') + self.assertEqual(len(mailbox_list.mailboxes), 1) + mailbox = mailbox_list.mailboxes[0] + self.assertIsNone(mailbox.display_name) + self.assertEqual(mailbox.local_part, 'dinsdale') + self.assertEqual(mailbox.domain, 'example.com') + self.assertIsNone(mailbox.route) + self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com') + self.assertEqual(mailbox_list.mailboxes, + mailbox_list.all_mailboxes) + + def test_get_mailbox_list_two_simple_addr(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + 'dinsdale@example.com, dinsdale@test.example.com', + 'dinsdale@example.com, dinsdale@test.example.com', + 'dinsdale@example.com, dinsdale@test.example.com', + [], + '') + self.assertEqual(mailbox_list.token_type, 'mailbox-list') + self.assertEqual(len(mailbox_list.mailboxes), 2) + self.assertEqual(mailbox_list.mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(mailbox_list.mailboxes[1].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes, + mailbox_list.all_mailboxes) + + def test_get_mailbox_list_two_name_addr(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + ('"Roy A. Bear" ,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" ,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" ,' + ' "Fred Flintstone" '), + [], + '') + self.assertEqual(len(mailbox_list.mailboxes), 2) + self.assertEqual(mailbox_list.mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(mailbox_list.mailboxes[0].display_name, + 'Roy A. Bear') + self.assertEqual(mailbox_list.mailboxes[1].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes[1].display_name, + 'Fred Flintstone') + self.assertEqual(mailbox_list.mailboxes, + mailbox_list.all_mailboxes) + + def test_get_mailbox_list_two_complex(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + ('(foo) "Roy A. Bear" (bar),' + ' "Fred Flintstone" '), + ('(foo) "Roy A. Bear" (bar),' + ' "Fred Flintstone" '), + (' "Roy A. Bear" ,' + ' "Fred Flintstone" '), + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(len(mailbox_list.mailboxes), 2) + self.assertEqual(mailbox_list.mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(mailbox_list.mailboxes[0].display_name, + 'Roy A. Bear') + self.assertEqual(mailbox_list.mailboxes[1].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes[1].display_name, + 'Fred Flintstone') + self.assertEqual(mailbox_list.mailboxes, + mailbox_list.all_mailboxes) + + def test_get_mailbox_list_unparseable_mailbox_null(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + ('"Roy A. Bear"[] dinsdale@example.com,' + ' "Fred Flintstone" '), + ('"Roy A. Bear"[] dinsdale@example.com,' + ' "Fred Flintstone" '), + ('"Roy A. Bear"[] dinsdale@example.com,' + ' "Fred Flintstone" '), + [errors.InvalidHeaderDefect, # the 'extra' text after the local part + errors.InvalidHeaderDefect, # the local part with no angle-addr + errors.ObsoleteHeaderDefect, # period in extra text (example.com) + errors.ObsoleteHeaderDefect], # (bird) in valid address. + '') + self.assertEqual(len(mailbox_list.mailboxes), 1) + self.assertEqual(len(mailbox_list.all_mailboxes), 2) + self.assertEqual(mailbox_list.all_mailboxes[0].token_type, + 'invalid-mailbox') + self.assertIsNone(mailbox_list.all_mailboxes[0].display_name) + self.assertEqual(mailbox_list.all_mailboxes[0].local_part, + 'Roy A. Bear') + self.assertIsNone(mailbox_list.all_mailboxes[0].domain) + self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec, + '"Roy A. Bear"') + self.assertIs(mailbox_list.all_mailboxes[1], + mailbox_list.mailboxes[0]) + self.assertEqual(mailbox_list.mailboxes[0].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes[0].display_name, + 'Fred Flintstone') + + def test_get_mailbox_list_junk_after_valid_address(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + ('"Roy A. Bear" @@,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" @@,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" @@,' + ' "Fred Flintstone" '), + [errors.InvalidHeaderDefect], + '') + self.assertEqual(len(mailbox_list.mailboxes), 1) + self.assertEqual(len(mailbox_list.all_mailboxes), 2) + self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(mailbox_list.all_mailboxes[0].display_name, + 'Roy A. Bear') + self.assertEqual(mailbox_list.all_mailboxes[0].token_type, + 'invalid-mailbox') + self.assertIs(mailbox_list.all_mailboxes[1], + mailbox_list.mailboxes[0]) + self.assertEqual(mailbox_list.mailboxes[0].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes[0].display_name, + 'Fred Flintstone') + + def test_get_mailbox_list_empty_list_element(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + ('"Roy A. Bear" , (bird),,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" , (bird),,' + ' "Fred Flintstone" '), + ('"Roy A. Bear" , ,,' + ' "Fred Flintstone" '), + [errors.ObsoleteHeaderDefect]*2, + '') + self.assertEqual(len(mailbox_list.mailboxes), 2) + self.assertEqual(mailbox_list.all_mailboxes, + mailbox_list.mailboxes) + self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(mailbox_list.all_mailboxes[0].display_name, + 'Roy A. Bear') + self.assertEqual(mailbox_list.mailboxes[1].addr_spec, + 'dinsdale@test.example.com') + self.assertEqual(mailbox_list.mailboxes[1].display_name, + 'Fred Flintstone') + + def test_get_mailbox_list_only_empty_elements(self): + mailbox_list = self._test_get_x(parser.get_mailbox_list, + '(foo),, (bar)', + '(foo),, (bar)', + ' ,, ', + [errors.ObsoleteHeaderDefect]*3, + '') + self.assertEqual(len(mailbox_list.mailboxes), 0) + self.assertEqual(mailbox_list.all_mailboxes, + mailbox_list.mailboxes) + + # get_group_list + + def test_get_group_list_cfws_only(self): + group_list = self._test_get_x(parser.get_group_list, + '(hidden);', + '(hidden)', + ' ', + [], + ';') + self.assertEqual(group_list.token_type, 'group-list') + self.assertEqual(len(group_list.mailboxes), 0) + self.assertEqual(group_list.mailboxes, + group_list.all_mailboxes) + + def test_get_group_list_mailbox_list(self): + group_list = self._test_get_x(parser.get_group_list, + 'dinsdale@example.org, "Fred A. Bear" ', + 'dinsdale@example.org, "Fred A. Bear" ', + 'dinsdale@example.org, "Fred A. Bear" ', + [], + '') + self.assertEqual(group_list.token_type, 'group-list') + self.assertEqual(len(group_list.mailboxes), 2) + self.assertEqual(group_list.mailboxes, + group_list.all_mailboxes) + self.assertEqual(group_list.mailboxes[1].display_name, + 'Fred A. Bear') + + def test_get_group_list_obs_group_list(self): + group_list = self._test_get_x(parser.get_group_list, + ', (foo),,(bar)', + ', (foo),,(bar)', + ', ,, ', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(group_list.token_type, 'group-list') + self.assertEqual(len(group_list.mailboxes), 0) + self.assertEqual(group_list.mailboxes, + group_list.all_mailboxes) + + def test_get_group_list_comment_only_invalid(self): + group_list = self._test_get_x(parser.get_group_list, + '(bar)', + '(bar)', + ' ', + [errors.InvalidHeaderDefect], + '') + self.assertEqual(group_list.token_type, 'group-list') + self.assertEqual(len(group_list.mailboxes), 0) + self.assertEqual(group_list.mailboxes, + group_list.all_mailboxes) + + # get_group + + def test_get_group_empty(self): + group = self._test_get_x(parser.get_group, + 'Monty Python:;', + 'Monty Python:;', + 'Monty Python:;', + [], + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 0) + self.assertEqual(group.mailboxes, + group.all_mailboxes) + + def test_get_group_null_addr_spec(self): + group = self._test_get_x(parser.get_group, + 'foo: <>;', + 'foo: <>;', + 'foo: <>;', + [errors.InvalidHeaderDefect], + '') + self.assertEqual(group.display_name, 'foo') + self.assertEqual(len(group.mailboxes), 0) + self.assertEqual(len(group.all_mailboxes), 1) + self.assertEqual(group.all_mailboxes[0].value, '<>') + + def test_get_group_cfws_only(self): + group = self._test_get_x(parser.get_group, + 'Monty Python: (hidden);', + 'Monty Python: (hidden);', + 'Monty Python: ;', + [], + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 0) + self.assertEqual(group.mailboxes, + group.all_mailboxes) + + def test_get_group_single_mailbox(self): + group = self._test_get_x(parser.get_group, + 'Monty Python: "Fred A. Bear" ;', + 'Monty Python: "Fred A. Bear" ;', + 'Monty Python: "Fred A. Bear" ;', + [], + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 1) + self.assertEqual(group.mailboxes, + group.all_mailboxes) + self.assertEqual(group.mailboxes[0].addr_spec, + 'dinsdale@example.com') + + def test_get_group_mixed_list(self): + group = self._test_get_x(parser.get_group, + ('Monty Python: "Fred A. Bear" ,' + '(foo) Roger , x@test.example.com;'), + ('Monty Python: "Fred A. Bear" ,' + '(foo) Roger , x@test.example.com;'), + ('Monty Python: "Fred A. Bear" ,' + ' Roger , x@test.example.com;'), + [], + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 3) + self.assertEqual(group.mailboxes, + group.all_mailboxes) + self.assertEqual(group.mailboxes[0].display_name, + 'Fred A. Bear') + self.assertEqual(group.mailboxes[1].display_name, + 'Roger') + self.assertEqual(group.mailboxes[2].local_part, 'x') + + def test_get_group_one_invalid(self): + group = self._test_get_x(parser.get_group, + ('Monty Python: "Fred A. Bear" ,' + '(foo) Roger ping@exampele.com, x@test.example.com;'), + ('Monty Python: "Fred A. Bear" ,' + '(foo) Roger ping@exampele.com, x@test.example.com;'), + ('Monty Python: "Fred A. Bear" ,' + ' Roger ping@exampele.com, x@test.example.com;'), + [errors.InvalidHeaderDefect, # non-angle addr makes local part invalid + errors.InvalidHeaderDefect], # and its not obs-local either: no dots. + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 2) + self.assertEqual(len(group.all_mailboxes), 3) + self.assertEqual(group.mailboxes[0].display_name, + 'Fred A. Bear') + self.assertEqual(group.mailboxes[1].local_part, 'x') + self.assertIsNone(group.all_mailboxes[1].display_name) + + def test_get_group_missing_final_semicol(self): + group = self._test_get_x(parser.get_group, + ('Monty Python:"Fred A. Bear" ,' + 'eric@where.test,John '), + ('Monty Python:"Fred A. Bear" ,' + 'eric@where.test,John ;'), + ('Monty Python:"Fred A. Bear" ,' + 'eric@where.test,John ;'), + [errors.InvalidHeaderDefect], + '') + self.assertEqual(group.token_type, 'group') + self.assertEqual(group.display_name, 'Monty Python') + self.assertEqual(len(group.mailboxes), 3) + self.assertEqual(group.mailboxes, + group.all_mailboxes) + self.assertEqual(group.mailboxes[0].addr_spec, + 'dinsdale@example.com') + self.assertEqual(group.mailboxes[0].display_name, + 'Fred A. Bear') + self.assertEqual(group.mailboxes[1].addr_spec, + 'eric@where.test') + self.assertEqual(group.mailboxes[2].display_name, + 'John') + self.assertEqual(group.mailboxes[2].addr_spec, + 'jdoe@test') + # get_address + + def test_get_address_simple(self): + address = self._test_get_x(parser.get_address, + 'dinsdale@example.com', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].domain, + 'example.com') + self.assertEqual(address[0].token_type, + 'mailbox') + + def test_get_address_complex(self): + address = self._test_get_x(parser.get_address, + '(foo) "Fred A. Bear" <(bird)dinsdale@example.com>', + '(foo) "Fred A. Bear" <(bird)dinsdale@example.com>', + ' "Fred A. Bear" < dinsdale@example.com>', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Fred A. Bear') + self.assertEqual(address[0].token_type, + 'mailbox') + + def test_get_address_rfc2047_display_name(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?=C3=89ric?= ', + 'Éric ', + 'Éric ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Éric') + self.assertEqual(address[0].token_type, + 'mailbox') + + def test_get_address_empty_group(self): + address = self._test_get_x(parser.get_address, + 'Monty Python:;', + 'Monty Python:;', + 'Monty Python:;', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address[0].token_type, + 'group') + self.assertEqual(address[0].display_name, + 'Monty Python') + + def test_get_address_group(self): + address = self._test_get_x(parser.get_address, + 'Monty Python: x@example.com, y@example.com;', + 'Monty Python: x@example.com, y@example.com;', + 'Monty Python: x@example.com, y@example.com;', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 2) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address[0].token_type, + 'group') + self.assertEqual(address[0].display_name, + 'Monty Python') + self.assertEqual(address.mailboxes[0].local_part, 'x') + + def test_get_address_quoted_local_part(self): + address = self._test_get_x(parser.get_address, + '"foo bar"@example.com', + '"foo bar"@example.com', + '"foo bar"@example.com', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].domain, + 'example.com') + self.assertEqual(address.mailboxes[0].local_part, + 'foo bar') + self.assertEqual(address[0].token_type, 'mailbox') + + def test_get_address_ends_at_special(self): + address = self._test_get_x(parser.get_address, + 'dinsdale@example.com, next', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + ', next') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].domain, + 'example.com') + self.assertEqual(address[0].token_type, 'mailbox') + + def test_get_address_invalid_mailbox_invalid(self): + address = self._test_get_x(parser.get_address, + 'ping example.com, next', + 'ping example.com', + 'ping example.com', + [errors.InvalidHeaderDefect, # addr-spec with no domain + errors.InvalidHeaderDefect, # invalid local-part + errors.InvalidHeaderDefect, # missing .s in local-part + ], + ', next') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertIsNone(address.all_mailboxes[0].domain) + self.assertEqual(address.all_mailboxes[0].local_part, 'ping example.com') + self.assertEqual(address[0].token_type, 'invalid-mailbox') + + def test_get_address_quoted_strings_in_atom_list(self): + address = self._test_get_x(parser.get_address, + '""example" example"@example.com', + '""example" example"@example.com', + 'example example@example.com', + [errors.InvalidHeaderDefect]*3, + '') + self.assertEqual(address.all_mailboxes[0].local_part, 'example example') + self.assertEqual(address.all_mailboxes[0].domain, 'example.com') + self.assertEqual(address.all_mailboxes[0].addr_spec, '"example example"@example.com') + + def test_get_address_with_invalid_domain(self): + address = self._test_get_x(parser.get_address, + '', + '', + [errors.InvalidHeaderDefect, # missing trailing '>' on angle-addr + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[]') + self.assertEqual(address.all_mailboxes[0].local_part, 'T') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'invalid-mailbox') + + address = self._test_get_x(parser.get_address, + '!an??:=m==fr2@[C', + '!an??:=m==fr2@[C];', + '!an??:=m==fr2@[C];', + [errors.InvalidHeaderDefect, # end of header in group + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[C]') + self.assertEqual(address.all_mailboxes[0].local_part, '=m==fr2') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'group') + + # get_address_list + + def test_get_address_list_CFWS(self): + address_list = self._test_get_x(parser.get_address_list, + '(Recipient list suppressed)', + '(Recipient list suppressed)', + ' ', + [errors.ObsoleteHeaderDefect], # no content in address list + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 0) + self.assertEqual(address_list.mailboxes, address_list.all_mailboxes) + + def test_get_address_list_mailboxes_simple(self): + address_list = self._test_get_x(parser.get_address_list, + 'dinsdale@example.com', + 'dinsdale@example.com', + 'dinsdale@example.com', + [], + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 1) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual([str(x) for x in address_list.mailboxes], + [str(x) for x in address_list.addresses]) + self.assertEqual(address_list.mailboxes[0].domain, 'example.com') + self.assertEqual(address_list[0].token_type, 'address') + self.assertIsNone(address_list[0].display_name) + + def test_get_address_list_mailboxes_two_simple(self): + address_list = self._test_get_x(parser.get_address_list, + 'foo@example.com, "Fred A. Bar" ', + 'foo@example.com, "Fred A. Bar" ', + 'foo@example.com, "Fred A. Bar" ', + [], + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 2) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual([str(x) for x in address_list.mailboxes], + [str(x) for x in address_list.addresses]) + self.assertEqual(address_list.mailboxes[0].local_part, 'foo') + self.assertEqual(address_list.mailboxes[1].display_name, "Fred A. Bar") + + def test_get_address_list_mailboxes_complex(self): + address_list = self._test_get_x(parser.get_address_list, + ('"Roy A. Bear" , ' + '(ping) Foo ,' + 'Nobody Is. Special '), + ('"Roy A. Bear" , ' + '(ping) Foo ,' + 'Nobody Is. Special '), + ('"Roy A. Bear" , ' + 'Foo ,' + '"Nobody Is. Special" '), + [errors.ObsoleteHeaderDefect, # period in Is. + errors.ObsoleteHeaderDefect], # cfws in domain + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 3) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual([str(x) for x in address_list.mailboxes], + [str(x) for x in address_list.addresses]) + self.assertEqual(address_list.mailboxes[0].domain, 'example.com') + self.assertEqual(address_list.mailboxes[0].token_type, 'mailbox') + self.assertEqual(address_list.addresses[0].token_type, 'address') + self.assertEqual(address_list.mailboxes[1].local_part, 'x') + self.assertEqual(address_list.mailboxes[2].display_name, + 'Nobody Is. Special') + + def test_get_address_list_mailboxes_invalid_addresses(self): + address_list = self._test_get_x(parser.get_address_list, + ('"Roy A. Bear" , ' + '(ping) Foo x@example.com[],' + 'Nobody Is. Special <(bird)example.(bad)com>'), + ('"Roy A. Bear" , ' + '(ping) Foo x@example.com[],' + 'Nobody Is. Special <(bird)example.(bad)com>'), + ('"Roy A. Bear" , ' + 'Foo x@example.com[],' + '"Nobody Is. Special" < example. com>'), + [errors.InvalidHeaderDefect, # invalid address in list + errors.InvalidHeaderDefect, # 'Foo x' local part invalid. + errors.InvalidHeaderDefect, # Missing . in 'Foo x' local part + errors.ObsoleteHeaderDefect, # period in 'Is.' disp-name phrase + errors.InvalidHeaderDefect, # no domain part in addr-spec + errors.ObsoleteHeaderDefect], # addr-spec has comment in it + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 1) + self.assertEqual(len(address_list.all_mailboxes), 3) + self.assertEqual([str(x) for x in address_list.all_mailboxes], + [str(x) for x in address_list.addresses]) + self.assertEqual(address_list.mailboxes[0].domain, 'example.com') + self.assertEqual(address_list.mailboxes[0].token_type, 'mailbox') + self.assertEqual(address_list.addresses[0].token_type, 'address') + self.assertEqual(address_list.addresses[1].token_type, 'address') + self.assertEqual(len(address_list.addresses[0].mailboxes), 1) + self.assertEqual(len(address_list.addresses[1].mailboxes), 0) + self.assertEqual(len(address_list.addresses[1].mailboxes), 0) + self.assertEqual( + address_list.addresses[1].all_mailboxes[0].local_part, 'Foo x') + self.assertEqual( + address_list.addresses[2].all_mailboxes[0].display_name, + "Nobody Is. Special") + + def test_get_address_list_group_empty(self): + address_list = self._test_get_x(parser.get_address_list, + 'Monty Python: ;', + 'Monty Python: ;', + 'Monty Python: ;', + [], + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 0) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual(len(address_list.addresses), 1) + self.assertEqual(address_list.addresses[0].token_type, 'address') + self.assertEqual(address_list.addresses[0].display_name, 'Monty Python') + self.assertEqual(len(address_list.addresses[0].mailboxes), 0) + + def test_get_address_list_group_simple(self): + address_list = self._test_get_x(parser.get_address_list, + 'Monty Python: dinsdale@example.com;', + 'Monty Python: dinsdale@example.com;', + 'Monty Python: dinsdale@example.com;', + [], + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 1) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual(address_list.mailboxes[0].domain, 'example.com') + self.assertEqual(address_list.addresses[0].display_name, + 'Monty Python') + self.assertEqual(address_list.addresses[0].mailboxes[0].domain, + 'example.com') + + def test_get_address_list_group_and_mailboxes(self): + address_list = self._test_get_x(parser.get_address_list, + ('Monty Python: dinsdale@example.com, "Fred" ;, ' + 'Abe , Bee '), + ('Monty Python: dinsdale@example.com, "Fred" ;, ' + 'Abe , Bee '), + ('Monty Python: dinsdale@example.com, "Fred" ;, ' + 'Abe , Bee '), + [], + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 4) + self.assertEqual(address_list.mailboxes, + address_list.all_mailboxes) + self.assertEqual(len(address_list.addresses), 3) + self.assertEqual(address_list.mailboxes[0].local_part, 'dinsdale') + self.assertEqual(address_list.addresses[0].display_name, + 'Monty Python') + self.assertEqual(address_list.addresses[0].mailboxes[0].domain, + 'example.com') + self.assertEqual(address_list.addresses[0].mailboxes[1].local_part, + 'flint') + self.assertEqual(address_list.addresses[1].mailboxes[0].local_part, + 'x') + self.assertEqual(address_list.addresses[2].mailboxes[0].local_part, + 'y') + self.assertEqual(str(address_list.addresses[1]), + str(address_list.mailboxes[2])) + + def test_invalid_content_disposition(self): + content_disp = self._test_parse_x( + parser.parse_content_disposition_header, + ";attachment", "; attachment", ";attachment", + [errors.InvalidHeaderDefect]*2 + ) + + def test_invalid_content_transfer_encoding(self): + cte = self._test_parse_x( + parser.parse_content_transfer_encoding_header, + ";foo", ";foo", ";foo", [errors.InvalidHeaderDefect]*3 + ) + + # get_msg_id + + def test_get_msg_id_empty(self): + # bpo-38708: Test that HeaderParseError is raised and not IndexError. + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id('') + + def test_get_msg_id_valid(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "", + "", + "", + [], + '', + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + def test_get_msg_id_obsolete_local(self): + msg_id = self._test_get_x( + parser.get_msg_id, + '<"simeple.local"@example.com>', + '<"simeple.local"@example.com>', + '', + [errors.ObsoleteHeaderDefect], + '', + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + def test_get_msg_id_non_folding_literal_domain(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "", + "", + "", + [], + "", + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + + def test_get_msg_id_obsolete_domain_part(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "", + "", + "", + [errors.ObsoleteHeaderDefect], + "" + ) + + def test_get_msg_id_no_id_right_part(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "", + "", + "", + [errors.InvalidHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + def test_get_msg_id_invalid_expected_msg_id_not_found(self): + text = "935-XPB-567:0:45327:9:90305:17843586-40@example.com" + msg_id = parser.parse_message_id(text) + self.assertDefectsEqual( + msg_id.all_defects, + [errors.InvalidHeaderDefect]) + + def test_parse_invalid_message_id(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "935-XPB-567:0:45327:9:90305:17843586-40@example.com", + "935-XPB-567:0:45327:9:90305:17843586-40@example.com", + "935-XPB-567:0:45327:9:90305:17843586-40@example.com", + [errors.InvalidHeaderDefect], + ) + self.assertEqual(message_id.token_type, 'invalid-message-id') + + def test_parse_valid_message_id(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "", + "", + "", + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + + def test_parse_message_id_with_invalid_domain(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "", + "", + [errors.ObsoleteHeaderDefect] + [errors.InvalidHeaderDefect] * 2, + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + self.assertEqual(str(message_id.all_defects[-1]), + "end of input inside domain-literal") + + def test_parse_message_id_with_remaining(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "thensomething", + "", + "", + [errors.InvalidHeaderDefect], + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + self.assertEqual(str(message_id.all_defects[0]), + "Unexpected 'thensomething'") + + def test_get_msg_id_no_angle_start(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("msgwithnoankle") + + def test_get_msg_id_no_angle_end(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "", + "", + [errors.InvalidHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + def test_get_msg_id_empty_id_left(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("<@domain>") + + def test_get_msg_id_empty_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_no_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_ws_only_local(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "< @domain>", + "< @domain>", + "< @domain>", + [errors.ObsoleteHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + + + +@parameterize +class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): + + def mime_parameters_as_value(self, + value, + tl_str, + tl_value, + params, + defects): + mime_parameters = self._test_parse_x(parser.parse_mime_parameters, + value, tl_str, tl_value, defects) + self.assertEqual(mime_parameters.token_type, 'mime-parameters') + self.assertEqual(list(mime_parameters.params), params) + + + mime_parameters_params = { + + 'simple': ( + 'filename="abc.py"', + ' filename="abc.py"', + 'filename=abc.py', + [('filename', 'abc.py')], + []), + + 'multiple_keys': ( + 'filename="abc.py"; xyz=abc', + ' filename="abc.py"; xyz="abc"', + 'filename=abc.py; xyz=abc', + [('filename', 'abc.py'), ('xyz', 'abc')], + []), + + 'split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', '201.tif')], + []), + + # Note that it is undefined what we should do for error recovery when + # there are duplicate parameter names or duplicate parts in a split + # part. We choose to ignore all duplicate parameters after the first + # and to take duplicate or missing rfc 2231 parts in appearance order. + # This is backward compatible with get_param's behavior, but the + # decisions are arbitrary. + + 'duplicate_key': ( + 'filename=abc.gif; filename=def.tiff', + ' filename="abc.gif"', + "filename=abc.gif; filename=def.tiff", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value_other_order': ( + "filename=abc.gif; " + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="abc.gif"', + "filename=abc.gif;" + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_in_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]), + + 'missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_and_missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]*2), + + # Here we depart from get_param and assume the *0* was missing. + 'duplicate_with_broken_split_value': ( + "filename=abc.gif; " + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + ' filename="abc.gif201.tif"', + "filename=abc.gif;" + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + [('filename', 'abc.gif201.tif')], + # Defects are apparent missing *0*, and two 'out of sequence'. + [errors.InvalidHeaderDefect]*3), + + # bpo-37461: Check that we don't go into an infinite loop. + 'extra_dquote': ( + 'r*="\'a\'\\"', + ' r="\\""', + 'r*=\'a\'"', + [('r', '"')], + [errors.InvalidHeaderDefect]*2), + } + +@parameterize +class Test_parse_mime_version(TestParserMixin, TestEmailBase): + + def mime_version_as_value(self, + value, + tl_str, + tl_value, + major, + minor, + defects): + mime_version = self._test_parse_x(parser.parse_mime_version, + value, tl_str, tl_value, defects) + self.assertEqual(mime_version.major, major) + self.assertEqual(mime_version.minor, minor) + + mime_version_params = { + + 'rfc_2045_1': ( + '1.0', + '1.0', + '1.0', + 1, + 0, + []), + + 'RFC_2045_2': ( + '1.0 (produced by MetaSend Vx.x)', + '1.0 (produced by MetaSend Vx.x)', + '1.0 ', + 1, + 0, + []), + + 'RFC_2045_3': ( + '(produced by MetaSend Vx.x) 1.0', + '(produced by MetaSend Vx.x) 1.0', + ' 1.0', + 1, + 0, + []), + + 'RFC_2045_4': ( + '1.(produced by MetaSend Vx.x)0', + '1.(produced by MetaSend Vx.x)0', + '1. 0', + 1, + 0, + []), + + 'empty': ( + '', + '', + '', + None, + None, + [errors.HeaderMissingRequiredValue]), + + } + + + +class TestFolding(TestEmailBase): + + policy = policy.default + + def _test(self, tl, folded, policy=policy): + self.assertEqual(tl.fold(policy=policy), folded, tl.ppstr()) + + def test_simple_unstructured_no_folds(self): + self._test(parser.get_unstructured("This is a test"), + "This is a test\n") + + def test_simple_unstructured_folded(self): + self._test(parser.get_unstructured("This is also a test, but this " + "time there are enough words (and even some " + "symbols) to make it wrap; at least in theory."), + "This is also a test, but this time there are enough " + "words (and even some\n" + " symbols) to make it wrap; at least in theory.\n") + + def test_unstructured_with_unicode_no_folds(self): + self._test(parser.get_unstructured("hübsch kleiner beißt"), + "=?utf-8?q?h=C3=BCbsch_kleiner_bei=C3=9Ft?=\n") + + def test_one_ew_on_each_of_two_wrapped_lines(self): + self._test(parser.get_unstructured("Mein kleiner Kaktus ist sehr " + "hübsch. Es hat viele Stacheln " + "und oft beißt mich."), + "Mein kleiner Kaktus ist sehr =?utf-8?q?h=C3=BCbsch=2E?= " + "Es hat viele Stacheln\n" + " und oft =?utf-8?q?bei=C3=9Ft?= mich.\n") + + def test_ews_combined_before_wrap(self): + self._test(parser.get_unstructured("Mein Kaktus ist hübsch. " + "Es beißt mich. " + "And that's all I'm sayin."), + "Mein Kaktus ist =?utf-8?q?h=C3=BCbsch=2E__Es_bei=C3=9Ft?= " + "mich. And that's\n" + " all I'm sayin.\n") + + def test_unicode_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=\xa4"), + "=?unknown-8bit?q?=A4?==?utf-8?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=\xa4"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?=C2=A4?=\n") + + def test_ascii_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=abc"), + "=?unknown-8bit?q?=A4?=abc\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=abc"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?abc?=\n") + + def test_unknown_after_unicode_not_combined(self): + self._test(parser.get_unstructured("\xa4" + "=?unknown-8bit?q?=A4?="), + "=?utf-8?q?=C2=A4?==?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "\xa4=?unknown-8bit?q?=A4?="), + prefix + "=?utf-8?q?=C2=A4?=\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_ascii_not_combined(self): + self._test(parser.get_unstructured("abc" + "=?unknown-8bit?q?=A4?="), + "abc=?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "abcd=?unknown-8bit?q?=A4?="), + prefix + "abcd\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_unknown(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + "=?unknown-8bit?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + prefix + "=?unknown-8bit?q?=C2?=\n =?unknown-8bit?q?=A4?=\n") + + # XXX Need test of an encoded word so long that it needs to be wrapped + + def test_simple_address(self): + self._test(parser.get_address_list("abc ")[0], + "abc \n") + + def test_address_list_folding_at_commas(self): + self._test(parser.get_address_list('abc , ' + '"Fred Blunt" , ' + '"J.P.Cool" , ' + '"K<>y" , ' + 'Firesale , ' + '')[0], + 'abc , "Fred Blunt" ,\n' + ' "J.P.Cool" , "K<>y" ,\n' + ' Firesale , \n') + + def test_address_list_with_unicode_names(self): + self._test(parser.get_address_list( + 'Hübsch Kaktus , ' + 'beißt beißt ')[0], + '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' + ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + + def test_address_list_with_unicode_names_in_quotes(self): + self._test(parser.get_address_list( + '"Hübsch Kaktus" , ' + '"beißt" beißt ')[0], + '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' + ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + + def test_address_list_with_specials_in_encoded_word(self): + # An encoded-word parsed from a structured header must remain + # encoded when it contains specials. Regression for gh-121284. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('=?utf-8?q?A_v=C3=A9ry_long_name_with=2C_comma?= ', + 'A =?utf-8?q?v=C3=A9ry_long_name_with?=\n' + ' =?utf-8?q?=2C?= comma \n'), + ('=?utf-8?q?This_long_name_does_not_need_encoded=2Dword?= ', + 'This long name does not need\n' + ' encoded-word \n'), + ('"A véry long name with, comma" ', + # (This isn't the best fold point, but it's not invalid.) + 'A =?utf-8?q?v=C3=A9ry_long_name_with?=\n' + ' =?utf-8?q?=2C?= comma \n'), + ('"A véry long name containing a, comma" ', + 'A =?utf-8?q?v=C3=A9ry?= long name\n' + ' containing =?utf-8?q?a=2C?= comma\n' + ' \n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + + def test_address_list_with_list_separator_after_fold(self): + a = 'x' * 66 + '@example.com' + to = f'{a}, "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus \n') + + a = '.' * 79 # ('.' is a special, so must be in quoted-string.) + to = f'"{a}" , "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'"{a}"\n' + ' , =?utf-8?q?H=C3=BCbsch?= Kaktus ' + '\n') + + def test_address_list_with_specials_in_long_quoted_string(self): + # Regression for gh-80222. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('"Exfiltrator (unclosed comment?" ', + '"Exfiltrator (unclosed\n' + ' comment?" \n'), + ('"Escaped \\" chars \\\\ in quoted-string stay escaped" ', + '"Escaped \\" chars \\\\ in quoted-string\n' + ' stay escaped" \n'), + ('This long display name does not need quotes ', + 'This long display name does not need\n' + ' quotes \n'), + ('"Quotes are not required but are retained here" ', + '"Quotes are not required but are\n' + ' retained here" \n'), + ('"A quoted-string, it can be a valid local-part"@example.com', + '"A quoted-string, it can be a valid\n' + ' local-part"@example.com\n'), + ('"local-part-with-specials@but-no-fws.cannot-fold"@example.com', + '"local-part-with-specials@but-no-fws.cannot-fold"@example.com\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + + # XXX Need tests with comments on various sides of a unicode token, + # and with unicode tokens in the comments. Spaces inside the quotes + # currently don't do the right thing. + + def test_split_at_whitespace_after_header_before_long_token(self): + body = parser.get_unstructured(' ' + 'x'*77) + header = parser.Header([ + parser.HeaderLabel([parser.ValueTerminal('test:', 'atext')]), + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), body]) + self._test(header, 'test: \n ' + 'x'*77 + '\n') + + def test_split_at_whitespace_before_long_token(self): + self._test(parser.get_unstructured('xxx ' + 'y'*77), + 'xxx \n ' + 'y'*77 + '\n') + + def test_overlong_encodeable_is_wrapped(self): + first_token_with_whitespace = 'xxx ' + chrome_leader = '=?utf-8?q?' + len_chrome = len(chrome_leader) + 2 + len_non_y = len_chrome + len(first_token_with_whitespace) + self._test(parser.get_unstructured(first_token_with_whitespace + + 'y'*80), + first_token_with_whitespace + chrome_leader + + 'y'*(78-len_non_y) + '?=\n' + + ' ' + chrome_leader + 'y'*(80-(78-len_non_y)) + '?=\n') + + def test_long_filename_attachment(self): + self._test(parser.parse_content_disposition_header( + 'attachment; filename="TEST_TEST_TEST_TEST' + '_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"'), + "attachment;\n" + " filename*0*=us-ascii''TEST_TEST_TEST_TEST_TEST_TEST" + "_TEST_TEST_TEST_TEST_TEST;\n" + " filename*1*=_TEST_TES.txt\n", + ) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py new file mode 100644 index 00000000000..1e0caeeaed0 --- /dev/null +++ b/Lib/test/test_email/test_asian_codecs.py @@ -0,0 +1,81 @@ +# Copyright (C) 2002-2006 Python Software Foundation +# Contact: email-sig@python.org +# email package unit tests for (optional) Asian codecs + +import unittest + +from test.test_email import TestEmailBase +from email.charset import Charset +from email.header import Header, decode_header +from email.message import Message + +# We're compatible with Python 2.3, but it doesn't have the built-in Asian +# codecs, so we have to skip all these tests. +try: + str(b'foo', 'euc-jp') +except LookupError: + raise unittest.SkipTest + + + +class TestEmailAsianCodecs(TestEmailBase): + def test_japanese_codecs(self): + eq = self.ndiffAssertEqual + jcode = "euc-jp" + gcode = "iso-8859-1" + j = Charset(jcode) + g = Charset(gcode) + h = Header("Hello World!") + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) + ghello = str(b'Gr\xfc\xdf Gott!', gcode) + h.append(jhello, j) + h.append(ghello, g) + # BAW: This used to -- and maybe should -- fold the two iso-8859-1 + # chunks into a single encoded word. However it doesn't violate the + # standard to have them as two encoded chunks and maybe it's + # reasonable for each .append() call to result in a separate + # encoded word. + eq(h.encode(), """\ +Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?= + =?iso-8859-1?q?Gr=FC=DF_Gott!?=""") + eq(decode_header(h.encode()), + [(b'Hello World! ', None), + (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'), + (b'Gr\xfc\xdf Gott!', gcode)]) + subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5' + b'\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2' + b'\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3' + b'\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9') + subject = str(subject_bytes, jcode) + h = Header(subject, j, header_name="Subject") + # test a very long header + enc = h.encode() + # TK: splitting point may differ by codec design and/or Header encoding + eq(enc , """\ +=?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKGyhC?= + =?iso-2022-jp?b?GyRCMnE8VCROPjVHJyRyQlQkQyRGJCQkXiQ5GyhC?=""") + # TK: full decode comparison + eq(str(h).encode(jcode), subject_bytes) + + def test_payload_encoding_utf8(self): + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') + msg = Message() + msg.set_payload(jhello, 'utf-8') + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) + + def test_payload_encoding(self): + jcode = 'euc-jp' + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) + msg = Message() + msg.set_payload(jhello, jcode) + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py new file mode 100644 index 00000000000..dceb54f15e4 --- /dev/null +++ b/Lib/test/test_email/test_contentmanager.py @@ -0,0 +1,836 @@ +import unittest +from test.test_email import TestEmailBase, parameterize +import textwrap +from email import policy +from email.message import EmailMessage +from email.contentmanager import ContentManager, raw_data_manager + + +@parameterize +class TestContentManager(TestEmailBase): + + policy = policy.default + message = EmailMessage + + get_key_params = { + 'full_type': (1, 'text/plain',), + 'maintype_only': (2, 'text',), + 'null_key': (3, '',), + } + + def get_key_as_get_content_key(self, order, key): + def foo_getter(msg, foo=None): + bar = msg['X-Bar-Header'] + return foo, bar + cm = ContentManager() + cm.add_get_handler(key, foo_getter) + m = self._make_message() + m['Content-Type'] = 'text/plain' + m['X-Bar-Header'] = 'foo' + self.assertEqual(cm.get_content(m, foo='bar'), ('bar', 'foo')) + + def get_key_as_get_content_key_order(self, order, key): + def bar_getter(msg): + return msg['X-Bar-Header'] + def foo_getter(msg): + return msg['X-Foo-Header'] + cm = ContentManager() + cm.add_get_handler(key, foo_getter) + for precedence, key in self.get_key_params.values(): + if precedence > order: + cm.add_get_handler(key, bar_getter) + m = self._make_message() + m['Content-Type'] = 'text/plain' + m['X-Bar-Header'] = 'bar' + m['X-Foo-Header'] = 'foo' + self.assertEqual(cm.get_content(m), ('foo')) + + def test_get_content_raises_if_unknown_mimetype_and_no_default(self): + cm = ContentManager() + m = self._make_message() + m['Content-Type'] = 'text/plain' + with self.assertRaisesRegex(KeyError, 'text/plain'): + cm.get_content(m) + + class BaseThing(str): + pass + baseobject_full_path = __name__ + '.' + 'TestContentManager.BaseThing' + class Thing(BaseThing): + pass + testobject_full_path = __name__ + '.' + 'TestContentManager.Thing' + + set_key_params = { + 'type': (0, Thing,), + 'full_path': (1, testobject_full_path,), + 'qualname': (2, 'TestContentManager.Thing',), + 'name': (3, 'Thing',), + 'base_type': (4, BaseThing,), + 'base_full_path': (5, baseobject_full_path,), + 'base_qualname': (6, 'TestContentManager.BaseThing',), + 'base_name': (7, 'BaseThing',), + 'str_type': (8, str,), + 'str_full_path': (9, 'builtins.str',), + 'str_name': (10, 'str',), # str name and qualname are the same + 'null_key': (11, None,), + } + + def set_key_as_set_content_key(self, order, key): + def foo_setter(msg, obj, foo=None): + msg['X-Foo-Header'] = foo + msg.set_payload(obj) + cm = ContentManager() + cm.add_set_handler(key, foo_setter) + m = self._make_message() + msg_obj = self.Thing() + cm.set_content(m, msg_obj, foo='bar') + self.assertEqual(m['X-Foo-Header'], 'bar') + self.assertEqual(m.get_payload(), msg_obj) + + def set_key_as_set_content_key_order(self, order, key): + def foo_setter(msg, obj): + msg['X-FooBar-Header'] = 'foo' + msg.set_payload(obj) + def bar_setter(msg, obj): + msg['X-FooBar-Header'] = 'bar' + cm = ContentManager() + cm.add_set_handler(key, foo_setter) + for precedence, key in self.get_key_params.values(): + if precedence > order: + cm.add_set_handler(key, bar_setter) + m = self._make_message() + msg_obj = self.Thing() + cm.set_content(m, msg_obj) + self.assertEqual(m['X-FooBar-Header'], 'foo') + self.assertEqual(m.get_payload(), msg_obj) + + def test_set_content_raises_if_unknown_type_and_no_default(self): + cm = ContentManager() + m = self._make_message() + msg_obj = self.Thing() + with self.assertRaisesRegex(KeyError, self.testobject_full_path): + cm.set_content(m, msg_obj) + + def test_set_content_raises_if_called_on_multipart(self): + cm = ContentManager() + m = self._make_message() + m['Content-Type'] = 'multipart/foo' + with self.assertRaises(TypeError): + cm.set_content(m, 'test') + + def test_set_content_calls_clear_content(self): + m = self._make_message() + m['Content-Foo'] = 'bar' + m['Content-Type'] = 'text/html' + m['To'] = 'test' + m.set_payload('abc') + cm = ContentManager() + cm.add_set_handler(str, lambda *args, **kw: None) + m.set_content('xyz', content_manager=cm) + self.assertIsNone(m['Content-Foo']) + self.assertIsNone(m['Content-Type']) + self.assertEqual(m['To'], 'test') + self.assertIsNone(m.get_payload()) + + +@parameterize +class TestRawDataManager(TestEmailBase): + # Note: these tests are dependent on the order in which headers are added + # to the message objects by the code. There's no defined ordering in + # RFC5322/MIME, so this makes the tests more fragile than the standards + # require. However, if the header order changes it is best to understand + # *why*, and make sure it isn't a subtle bug in whatever change was + # applied. + + policy = policy.default.clone(max_line_length=60, + content_manager=raw_data_manager) + message = EmailMessage + + def test_get_text_plain(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain + + Basic text. + """)) + self.assertEqual(raw_data_manager.get_content(m), "Basic text.\n") + + def test_get_text_html(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/html + +

Basic text.

+ """)) + self.assertEqual(raw_data_manager.get_content(m), + "

Basic text.

\n") + + def test_get_text_plain_latin1(self): + m = self._bytes_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset=latin1 + + Basìc tëxt. + """).encode('latin1')) + self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n") + + def test_get_text_plain_latin1_quoted_printable(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset="latin-1" + Content-Transfer-Encoding: quoted-printable + + Bas=ECc t=EBxt. + """)) + self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n") + + def test_get_text_plain_utf8_base64(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset="utf8" + Content-Transfer-Encoding: base64 + + QmFzw6xjIHTDq3h0Lgo= + """)) + self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n") + + def test_get_text_plain_bad_utf8_quoted_printable(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset="utf8" + Content-Transfer-Encoding: quoted-printable + + Bas=c3=acc t=c3=abxt=fd. + """)) + self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt�.\n") + + def test_get_text_plain_bad_utf8_quoted_printable_ignore_errors(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset="utf8" + Content-Transfer-Encoding: quoted-printable + + Bas=c3=acc t=c3=abxt=fd. + """)) + self.assertEqual(raw_data_manager.get_content(m, errors='ignore'), + "Basìc tëxt.\n") + + def test_get_text_plain_utf8_base64_recoverable_bad_CTE_data(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain; charset="utf8" + Content-Transfer-Encoding: base64 + + QmFzw6xjIHTDq3h0Lgo\xFF= + """)) + self.assertEqual(raw_data_manager.get_content(m, errors='ignore'), + "Basìc tëxt.\n") + + def test_get_text_invalid_keyword(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: text/plain + + Basic text. + """)) + with self.assertRaises(TypeError): + raw_data_manager.get_content(m, foo='ignore') + + def test_get_non_text(self): + template = textwrap.dedent("""\ + Content-Type: {} + Content-Transfer-Encoding: base64 + + Ym9ndXMgZGF0YQ== + """) + for maintype in 'audio image video application'.split(): + with self.subTest(maintype=maintype): + m = self._str_msg(template.format(maintype+'/foo')) + self.assertEqual(raw_data_manager.get_content(m), b"bogus data") + + def test_get_non_text_invalid_keyword(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: image/jpg + Content-Transfer-Encoding: base64 + + Ym9ndXMgZGF0YQ== + """)) + with self.assertRaises(TypeError): + raw_data_manager.get_content(m, errors='ignore') + + def test_get_raises_on_multipart(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: multipart/mixed; boundary="===" + + --=== + --===-- + """)) + with self.assertRaises(KeyError): + raw_data_manager.get_content(m) + + def test_get_message_rfc822_and_external_body(self): + template = textwrap.dedent("""\ + Content-Type: message/{} + + To: foo@example.com + From: bar@example.com + Subject: example + + an example message + """) + for subtype in 'rfc822 external-body'.split(): + with self.subTest(subtype=subtype): + m = self._str_msg(template.format(subtype)) + sub_msg = raw_data_manager.get_content(m) + self.assertIsInstance(sub_msg, self.message) + self.assertEqual(raw_data_manager.get_content(sub_msg), + "an example message\n") + self.assertEqual(sub_msg['to'], 'foo@example.com') + self.assertEqual(sub_msg['from'].addresses[0].username, 'bar') + + def test_get_message_non_rfc822_or_external_body_yields_bytes(self): + m = self._str_msg(textwrap.dedent("""\ + Content-Type: message/partial + + To: foo@example.com + From: bar@example.com + Subject: example + + The real body is in another message. + """)) + self.assertStartsWith(raw_data_manager.get_content(m), b'To: foo@ex') + + def test_set_text_plain(self): + m = self._make_message() + content = "Simple message.\n" + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + + Simple message. + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_plain_null(self): + m = self._make_message() + content = '' + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + + + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), '\n') + self.assertEqual(m.get_content(), '\n') + + def test_set_text_html(self): + m = self._make_message() + content = "

Simple message.

\n" + raw_data_manager.set_content(m, content, subtype='html') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/html; charset="utf-8" + Content-Transfer-Encoding: 7bit + +

Simple message.

+ """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_charset_latin_1(self): + m = self._make_message() + content = "Simple message.\n" + raw_data_manager.set_content(m, content, charset='latin-1') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: 7bit + + Simple message. + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_plain_long_line_heuristics(self): + m = self._make_message() + content = ("Simple but long message that is over 78 characters" + " long to force transfer encoding.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + + Simple but long message that is over 78 characters long to = + force transfer encoding. + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_short_line_minimal_non_ascii_heuristics(self): + m = self._make_message() + content = "et là il est monté sur moi et il commence à m'éto.\n" + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + et là il est monté sur moi et il commence à m'éto. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_long_line_minimal_non_ascii_heuristics(self): + m = self._make_message() + content = ("j'ai un problème de python. il est sorti de son" + " vivarium. et là il est monté sur moi et il commence" + " à m'éto.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + + j'ai un probl=C3=A8me de python. il est sorti de son vivari= + um. et l=C3=A0 il est mont=C3=A9 sur moi et il commence = + =C3=A0 m'=C3=A9to. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_11_lines_long_line_minimal_non_ascii_heuristics(self): + m = self._make_message() + content = '\n'*10 + ( + "j'ai un problème de python. il est sorti de son" + " vivarium. et là il est monté sur moi et il commence" + " à m'éto.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + """ + '\n'*10 + """ + j'ai un probl=C3=A8me de python. il est sorti de son vivari= + um. et l=C3=A0 il est mont=C3=A9 sur moi et il commence = + =C3=A0 m'=C3=A9to. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_maximal_non_ascii_heuristics(self): + m = self._make_message() + content = "áàäéèęöő.\n" + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + áàäéèęöő. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_11_lines_maximal_non_ascii_heuristics(self): + m = self._make_message() + content = '\n'*10 + "áàäéèęöő.\n" + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + """ + '\n'*10 + """ + áàäéèęöő. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_long_line_maximal_non_ascii_heuristics(self): + m = self._make_message() + content = ("áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + w6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TDqcOoxJnD + tsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TDqcOo + xJnDtsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TD + qcOoxJnDtsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOg + w6TDqcOoxJnDtsWRLgo= + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_11_lines_long_line_maximal_non_ascii_heuristics(self): + # Yes, it chooses "wrong" here. It's a heuristic. So this result + # could change if we come up with a better heuristic. + m = self._make_message() + content = ('\n'*10 + + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n") + raw_data_manager.set_content(m, "\n"*10 + + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő" + "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n") + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + """ + '\n'*10 + """ + =C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3= + =A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4= + =C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3= + =A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99= + =C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5= + =91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1= + =C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3= + =A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9= + =C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4= + =99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6= + =C5=91. + """).encode('utf-8')) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + + def test_set_text_non_ascii_with_cte_7bit_raises(self): + m = self._make_message() + with self.assertRaises(UnicodeError): + raw_data_manager.set_content(m,"áàäéèęöő.\n", cte='7bit') + + def test_set_text_non_ascii_with_charset_ascii_raises(self): + m = self._make_message() + with self.assertRaises(UnicodeError): + raw_data_manager.set_content(m,"áàäéèęöő.\n", charset='ascii') + + def test_set_text_non_ascii_with_cte_7bit_and_charset_ascii_raises(self): + m = self._make_message() + with self.assertRaises(UnicodeError): + raw_data_manager.set_content(m,"áàäéèęöő.\n", cte='7bit', charset='ascii') + + def test_set_message(self): + m = self._make_message() + m['Subject'] = "Forwarded message" + content = self._make_message() + content['To'] = 'python@vivarium.org' + content['From'] = 'police@monty.org' + content['Subject'] = "get back in your box" + content.set_content("Or face the comfy chair.") + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Subject: Forwarded message + Content-Type: message/rfc822 + Content-Transfer-Encoding: 8bit + + To: python@vivarium.org + From: police@monty.org + Subject: get back in your box + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + MIME-Version: 1.0 + + Or face the comfy chair. + """)) + payload = m.get_payload(0) + self.assertIsInstance(payload, self.message) + self.assertEqual(str(payload), str(content)) + self.assertIsInstance(m.get_content(), self.message) + self.assertEqual(str(m.get_content()), str(content)) + + def test_set_message_with_non_ascii_and_coercion_to_7bit(self): + m = self._make_message() + m['Subject'] = "Escape report" + content = self._make_message() + content['To'] = 'police@monty.org' + content['From'] = 'victim@monty.org' + content['Subject'] = "Help" + content.set_content("j'ai un problème de python. il est sorti de son" + " vivarium.") + raw_data_manager.set_content(m, content) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Subject: Escape report + Content-Type: message/rfc822 + Content-Transfer-Encoding: 8bit + + To: police@monty.org + From: victim@monty.org + Subject: Help + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + j'ai un problème de python. il est sorti de son vivarium. + """).encode('utf-8')) + # The choice of base64 for the body encoding is because generator + # doesn't bother with heuristics and uses it unconditionally for utf-8 + # text. + # XXX: the first cte should be 7bit, too...that's a generator bug. + # XXX: the line length in the body also looks like a generator bug. + self.assertEqual(m.as_string(maxheaderlen=self.policy.max_line_length), + textwrap.dedent("""\ + Subject: Escape report + Content-Type: message/rfc822 + Content-Transfer-Encoding: 8bit + + To: police@monty.org + From: victim@monty.org + Subject: Help + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + MIME-Version: 1.0 + + aidhaSB1biBwcm9ibMOobWUgZGUgcHl0aG9uLiBpbCBlc3Qgc29ydGkgZGUgc29uIHZpdmFyaXVt + Lgo= + """)) + self.assertIsInstance(m.get_content(), self.message) + self.assertEqual(str(m.get_content()), str(content)) + + def test_set_message_invalid_cte_raises(self): + m = self._make_message() + content = self._make_message() + for cte in 'quoted-printable base64'.split(): + for subtype in 'rfc822 external-body'.split(): + with self.subTest(cte=cte, subtype=subtype): + with self.assertRaises(ValueError) as ar: + m.set_content(content, subtype, cte=cte) + exc = str(ar.exception) + self.assertIn(cte, exc) + self.assertIn(subtype, exc) + subtype = 'external-body' + for cte in '8bit binary'.split(): + with self.subTest(cte=cte, subtype=subtype): + with self.assertRaises(ValueError) as ar: + m.set_content(content, subtype, cte=cte) + exc = str(ar.exception) + self.assertIn(cte, exc) + self.assertIn(subtype, exc) + + def test_set_image_jpg(self): + for content in (b"bogus content", + bytearray(b"bogus content"), + memoryview(b"bogus content")): + with self.subTest(content=content): + m = self._make_message() + raw_data_manager.set_content(m, content, 'image', 'jpeg') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: image/jpeg + Content-Transfer-Encoding: base64 + + Ym9ndXMgY29udGVudA== + """)) + self.assertEqual(m.get_payload(decode=True), content) + self.assertEqual(m.get_content(), content) + + def test_set_audio_aif_with_quoted_printable_cte(self): + # Why you would use qp, I don't know, but it is technically supported. + # XXX: the incorrect line length is because binascii.b2a_qp doesn't + # support a line length parameter, but we must use it to get newline + # encoding. + # XXX: what about that lack of tailing newline? Do we actually handle + # that correctly in all cases? That is, if the *source* has an + # unencoded newline, do we add an extra newline to the returned payload + # or not? And can that actually be disambiguated based on the RFC? + m = self._make_message() + content = b'b\xFFgus\tcon\nt\rent ' + b'z'*100 + m.set_content(content, 'audio', 'aif', cte='quoted-printable') + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: audio/aif + Content-Transfer-Encoding: quoted-printable + MIME-Version: 1.0 + + b=FFgus=09con=0At=0Dent=20zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz= + zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz""").encode('latin-1')) + self.assertEqual(m.get_payload(decode=True), content) + self.assertEqual(m.get_content(), content) + + def test_set_video_mpeg_with_binary_cte(self): + m = self._make_message() + content = b'b\xFFgus\tcon\nt\rent ' + b'z'*100 + m.set_content(content, 'video', 'mpeg', cte='binary') + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: video/mpeg + Content-Transfer-Encoding: binary + MIME-Version: 1.0 + + """).encode('ascii') + + # XXX: the second \n ought to be a \r, but generator gets it wrong. + # THIS MEANS WE DON'T ACTUALLY SUPPORT THE 'binary' CTE. + b'b\xFFgus\tcon\nt\nent zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz' + + b'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz') + self.assertEqual(m.get_payload(decode=True), content) + self.assertEqual(m.get_content(), content) + + def test_set_application_octet_stream_with_8bit_cte(self): + # In 8bit mode, universal line end logic applies. It is up to the + # application to make sure the lines are short enough; we don't check. + m = self._make_message() + content = b'b\xFFgus\tcon\nt\rent\n' + b'z'*60 + b'\n' + m.set_content(content, 'application', 'octet-stream', cte='8bit') + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: application/octet-stream + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + """).encode('ascii') + + b'b\xFFgus\tcon\nt\nent\n' + + b'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n') + self.assertEqual(m.get_payload(decode=True), content) + self.assertEqual(m.get_content(), content) + + def test_set_headers_from_header_objects(self): + m = self._make_message() + content = "Simple message.\n" + header_factory = self.policy.header_factory + raw_data_manager.set_content(m, content, headers=( + header_factory("To", "foo@example.com"), + header_factory("From", "foo@example.com"), + header_factory("Subject", "I'm talking to myself."))) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + To: foo@example.com + From: foo@example.com + Subject: I'm talking to myself. + Content-Transfer-Encoding: 7bit + + Simple message. + """)) + + def test_set_headers_from_strings(self): + m = self._make_message() + content = "Simple message.\n" + raw_data_manager.set_content(m, content, headers=( + "X-Foo-Header: foo", + "X-Bar-Header: bar",)) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + X-Foo-Header: foo + X-Bar-Header: bar + Content-Transfer-Encoding: 7bit + + Simple message. + """)) + + def test_set_headers_with_invalid_duplicate_string_header_raises(self): + m = self._make_message() + content = "Simple message.\n" + with self.assertRaisesRegex(ValueError, 'Content-Type'): + raw_data_manager.set_content(m, content, headers=( + "Content-Type: foo/bar",) + ) + + def test_set_headers_with_invalid_duplicate_header_header_raises(self): + m = self._make_message() + content = "Simple message.\n" + header_factory = self.policy.header_factory + with self.assertRaisesRegex(ValueError, 'Content-Type'): + raw_data_manager.set_content(m, content, headers=( + header_factory("Content-Type", " foo/bar"),) + ) + + def test_set_headers_with_defective_string_header_raises(self): + m = self._make_message() + content = "Simple message.\n" + with self.assertRaisesRegex(ValueError, 'a@fairly@@invalid@address'): + raw_data_manager.set_content(m, content, headers=( + 'To: a@fairly@@invalid@address',) + ) + print(m['To'].defects) + + def test_set_headers_with_defective_header_header_raises(self): + m = self._make_message() + content = "Simple message.\n" + header_factory = self.policy.header_factory + with self.assertRaisesRegex(ValueError, 'a@fairly@@invalid@address'): + raw_data_manager.set_content(m, content, headers=( + header_factory('To', 'a@fairly@@invalid@address'),) + ) + print(m['To'].defects) + + def test_set_disposition_inline(self): + m = self._make_message() + m.set_content('foo', disposition='inline') + self.assertEqual(m['Content-Disposition'], 'inline') + + def test_set_disposition_attachment(self): + m = self._make_message() + m.set_content('foo', disposition='attachment') + self.assertEqual(m['Content-Disposition'], 'attachment') + + def test_set_disposition_foo(self): + m = self._make_message() + m.set_content('foo', disposition='foo') + self.assertEqual(m['Content-Disposition'], 'foo') + + # XXX: we should have a 'strict' policy mode (beyond raise_on_defect) that + # would cause 'foo' above to raise. + + def test_set_filename(self): + m = self._make_message() + m.set_content('foo', filename='bar.txt') + self.assertEqual(m['Content-Disposition'], + 'attachment; filename="bar.txt"') + + def test_set_filename_and_disposition_inline(self): + m = self._make_message() + m.set_content('foo', disposition='inline', filename='bar.txt') + self.assertEqual(m['Content-Disposition'], 'inline; filename="bar.txt"') + + def test_set_non_ascii_filename(self): + m = self._make_message() + m.set_content('foo', filename='ábárî.txt') + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; + filename*=utf-8''%C3%A1b%C3%A1r%C3%AE.txt + MIME-Version: 1.0 + + foo + """).encode('ascii')) + + def test_set_content_bytes_cte_7bit(self): + m = self._make_message() + m.set_content(b'ASCII-only message.\n', + maintype='application', subtype='octet-stream', cte='7bit') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: application/octet-stream + Content-Transfer-Encoding: 7bit + MIME-Version: 1.0 + + ASCII-only message. + """)) + + content_object_params = { + 'text_plain': ('content', ()), + 'text_html': ('content', ('html',)), + 'application_octet_stream': (b'content', + ('application', 'octet_stream')), + 'image_jpeg': (b'content', ('image', 'jpeg')), + 'message_rfc822': (message(), ()), + 'message_external_body': (message(), ('external-body',)), + } + + def content_object_as_header_receiver(self, obj, mimetype): + m = self._make_message() + m.set_content(obj, *mimetype, headers=( + 'To: foo@example.com', + 'From: bar@simple.net')) + self.assertEqual(m['to'], 'foo@example.com') + self.assertEqual(m['from'], 'bar@simple.net') + + def content_object_as_disposition_inline_receiver(self, obj, mimetype): + m = self._make_message() + m.set_content(obj, *mimetype, disposition='inline') + self.assertEqual(m['Content-Disposition'], 'inline') + + def content_object_as_non_ascii_filename_receiver(self, obj, mimetype): + m = self._make_message() + m.set_content(obj, *mimetype, disposition='inline', filename='bár.txt') + self.assertEqual(m['Content-Disposition'], 'inline; filename="bár.txt"') + self.assertEqual(m.get_filename(), "bár.txt") + self.assertEqual(m['Content-Disposition'].params['filename'], "bár.txt") + + def content_object_as_cid_receiver(self, obj, mimetype): + m = self._make_message() + m.set_content(obj, *mimetype, cid='some_random_stuff') + self.assertEqual(m['Content-ID'], 'some_random_stuff') + + def content_object_as_params_receiver(self, obj, mimetype): + m = self._make_message() + params = {'foo': 'bár', 'abc': 'xyz'} + m.set_content(obj, *mimetype, params=params) + if isinstance(obj, str): + params['charset'] = 'utf-8' + self.assertEqual(m['Content-Type'].params, params) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py new file mode 100644 index 00000000000..44e76c8ce5e --- /dev/null +++ b/Lib/test/test_email/test_defect_handling.py @@ -0,0 +1,337 @@ +import textwrap +import unittest +import contextlib +from email import policy +from email import errors +from test.test_email import TestEmailBase + + +class TestDefectsBase: + + policy = policy.default + raise_expected = False + + @contextlib.contextmanager + def _raise_point(self, defect): + yield + + def test_same_boundary_inner_outer(self): + source = textwrap.dedent("""\ + Subject: XX + From: xx@xx.dk + To: XX + Mime-version: 1.0 + Content-type: multipart/mixed; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: multipart/alternative; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: text/plain; charset="ISO-8859-1" + Content-transfer-encoding: quoted-printable + + text + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: text/html; charset="ISO-8859-1" + Content-transfer-encoding: quoted-printable + + + + --MS_Mac_OE_3071477847_720252_MIME_Part-- + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: image/gif; name="xx.gif"; + Content-disposition: attachment + Content-transfer-encoding: base64 + + Some removed base64 encoded chars. + + --MS_Mac_OE_3071477847_720252_MIME_Part-- + + """) + # XXX better would be to actually detect the duplicate. + with self._raise_point(errors.StartBoundaryNotFoundDefect): + msg = self._str_msg(source) + if self.raise_expected: return + inner = msg.get_payload(0) + self.assertHasAttr(inner, 'defects') + self.assertEqual(len(self.get_defects(inner)), 1) + self.assertIsInstance(self.get_defects(inner)[0], + errors.StartBoundaryNotFoundDefect) + + def test_multipart_no_boundary(self): + source = textwrap.dedent("""\ + Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) + From: foobar + Subject: broken mail + MIME-Version: 1.0 + Content-Type: multipart/report; report-type=delivery-status; + + --JAB03225.986577786/zinfandel.lacita.com + + One part + + --JAB03225.986577786/zinfandel.lacita.com + Content-Type: message/delivery-status + + Header: Another part + + --JAB03225.986577786/zinfandel.lacita.com-- + """) + with self._raise_point(errors.NoBoundaryInMultipartDefect): + msg = self._str_msg(source) + if self.raise_expected: return + self.assertIsInstance(msg.get_payload(), str) + self.assertEqual(len(self.get_defects(msg)), 2) + self.assertIsInstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect) + self.assertIsInstance(self.get_defects(msg)[1], + errors.MultipartInvariantViolationDefect) + + multipart_msg = textwrap.dedent("""\ + Date: Wed, 14 Nov 2007 12:56:23 GMT + From: foo@bar.invalid + To: foo@bar.invalid + Subject: Content-Transfer-Encoding: base64 and multipart + MIME-Version: 1.0 + Content-Type: multipart/mixed; + boundary="===============3344438784458119861=="{} + + --===============3344438784458119861== + Content-Type: text/plain + + Test message + + --===============3344438784458119861== + Content-Type: application/octet-stream + Content-Transfer-Encoding: base64 + + YWJj + + --===============3344438784458119861==-- + """) + + def test_multipart_invalid_cte(self): + with self._raise_point( + errors.InvalidMultipartContentTransferEncodingDefect): + msg = self._str_msg( + self.multipart_msg.format( + "\nContent-Transfer-Encoding: base64")) + if self.raise_expected: return + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertIsInstance(self.get_defects(msg)[0], + errors.InvalidMultipartContentTransferEncodingDefect) + + def test_multipart_no_cte_no_defect(self): + if self.raise_expected: return + msg = self._str_msg(self.multipart_msg.format('')) + self.assertEqual(len(self.get_defects(msg)), 0) + + def test_multipart_valid_cte_no_defect(self): + if self.raise_expected: return + for cte in ('7bit', '8bit', 'BINary'): + msg = self._str_msg( + self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte)) + self.assertEqual(len(self.get_defects(msg)), 0, "cte="+cte) + + def test_lying_multipart(self): + source = textwrap.dedent("""\ + From: "Allison Dunlap" + To: yyy@example.com + Subject: 64423 + Date: Sun, 11 Jul 2004 16:09:27 -0300 + MIME-Version: 1.0 + Content-Type: multipart/alternative; + + Blah blah blah + """) + with self._raise_point(errors.NoBoundaryInMultipartDefect): + msg = self._str_msg(source) + if self.raise_expected: return + self.assertHasAttr(msg, 'defects') + self.assertEqual(len(self.get_defects(msg)), 2) + self.assertIsInstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect) + self.assertIsInstance(self.get_defects(msg)[1], + errors.MultipartInvariantViolationDefect) + + def test_missing_start_boundary(self): + source = textwrap.dedent("""\ + Content-Type: multipart/mixed; boundary="AAA" + From: Mail Delivery Subsystem + To: yyy@example.com + + --AAA + + Stuff + + --AAA + Content-Type: message/rfc822 + + From: webmaster@python.org + To: zzz@example.com + Content-Type: multipart/mixed; boundary="BBB" + + --BBB-- + + --AAA-- + + """) + # The message structure is: + # + # multipart/mixed + # text/plain + # message/rfc822 + # multipart/mixed [*] + # + # [*] This message is missing its start boundary + with self._raise_point(errors.StartBoundaryNotFoundDefect): + outer = self._str_msg(source) + if self.raise_expected: return + bad = outer.get_payload(1).get_payload(0) + self.assertEqual(len(self.get_defects(bad)), 1) + self.assertIsInstance(self.get_defects(bad)[0], + errors.StartBoundaryNotFoundDefect) + + def test_first_line_is_continuation_header(self): + with self._raise_point(errors.FirstHeaderLineIsContinuationDefect): + msg = self._str_msg(' Line 1\nSubject: test\n\nbody') + if self.raise_expected: return + self.assertEqual(msg.keys(), ['Subject']) + self.assertEqual(msg.get_payload(), 'body') + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual(self.get_defects(msg), + [errors.FirstHeaderLineIsContinuationDefect]) + self.assertEqual(self.get_defects(msg)[0].line, ' Line 1\n') + + def test_missing_header_body_separator(self): + # Our heuristic if we see a line that doesn't look like a header (no + # leading whitespace but no ':') is to assume that the blank line that + # separates the header from the body is missing, and to stop parsing + # headers and start parsing the body. + with self._raise_point(errors.MissingHeaderBodySeparatorDefect): + msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') + if self.raise_expected: return + self.assertEqual(msg.keys(), ['Subject']) + self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') + self.assertDefectsEqual(self.get_defects(msg), + [errors.MissingHeaderBodySeparatorDefect]) + + def test_bad_padding_in_base64_payload(self): + source = textwrap.dedent("""\ + Subject: test + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + dmk + """) + msg = self._str_msg(source) + with self._raise_point(errors.InvalidBase64PaddingDefect): + payload = msg.get_payload(decode=True) + if self.raise_expected: return + self.assertEqual(payload, b'vi') + self.assertDefectsEqual(self.get_defects(msg), + [errors.InvalidBase64PaddingDefect]) + + def test_invalid_chars_in_base64_payload(self): + source = textwrap.dedent("""\ + Subject: test + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + dm\x01k=== + """) + msg = self._str_msg(source) + with self._raise_point(errors.InvalidBase64CharactersDefect): + payload = msg.get_payload(decode=True) + if self.raise_expected: return + self.assertEqual(payload, b'vi') + self.assertDefectsEqual(self.get_defects(msg), + [errors.InvalidBase64CharactersDefect]) + + def test_invalid_length_of_base64_payload(self): + source = textwrap.dedent("""\ + Subject: test + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + abcde + """) + msg = self._str_msg(source) + with self._raise_point(errors.InvalidBase64LengthDefect): + payload = msg.get_payload(decode=True) + if self.raise_expected: return + self.assertEqual(payload, b'abcde') + self.assertDefectsEqual(self.get_defects(msg), + [errors.InvalidBase64LengthDefect]) + + def test_missing_ending_boundary(self): + source = textwrap.dedent("""\ + To: 1@harrydomain4.com + Subject: Fwd: 1 + MIME-Version: 1.0 + Content-Type: multipart/alternative; + boundary="------------000101020201080900040301" + + --------------000101020201080900040301 + Content-Type: text/plain; charset=ISO-8859-1 + Content-Transfer-Encoding: 7bit + + Alternative 1 + + --------------000101020201080900040301 + Content-Type: text/html; charset=ISO-8859-1 + Content-Transfer-Encoding: 7bit + + Alternative 2 + + """) + with self._raise_point(errors.CloseBoundaryNotFoundDefect): + msg = self._str_msg(source) + if self.raise_expected: return + self.assertEqual(len(msg.get_payload()), 2) + self.assertEqual(msg.get_payload(1).get_payload(), 'Alternative 2\n') + self.assertDefectsEqual(self.get_defects(msg), + [errors.CloseBoundaryNotFoundDefect]) + + +class TestDefectDetection(TestDefectsBase, TestEmailBase): + + def get_defects(self, obj): + return obj.defects + + +class TestDefectCapture(TestDefectsBase, TestEmailBase): + + class CapturePolicy(policy.EmailPolicy): + captured = None + def register_defect(self, obj, defect): + self.captured.append(defect) + + def setUp(self): + self.policy = self.CapturePolicy(captured=list()) + + def get_defects(self, obj): + return self.policy.captured + + +class TestDefectRaising(TestDefectsBase, TestEmailBase): + + policy = TestDefectsBase.policy + policy = policy.clone(raise_on_defect=True) + raise_expected = True + + @contextlib.contextmanager + def _raise_point(self, defect): + with self.assertRaises(defect): + yield + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py new file mode 100644 index 00000000000..2d843c7d723 --- /dev/null +++ b/Lib/test/test_email/test_email.py @@ -0,0 +1,5907 @@ +# Copyright (C) 2001-2010 Python Software Foundation +# Contact: email-sig@python.org +# email package unit tests + +import re +import time +import base64 +import unittest +import textwrap + +from io import StringIO, BytesIO +from itertools import chain +from random import choice +from threading import Thread +from unittest.mock import patch + +import email +import email.policy +import email.utils + +from email.charset import Charset +from email.generator import Generator, DecodedGenerator, BytesGenerator +from email.header import Header, decode_header, make_header +from email.headerregistry import HeaderRegistry +from email.message import Message +from email.mime.application import MIMEApplication +from email.mime.audio import MIMEAudio +from email.mime.base import MIMEBase +from email.mime.image import MIMEImage +from email.mime.message import MIMEMessage +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.parser import Parser, HeaderParser +from email import base64mime +from email import encoders +from email import errors +from email import iterators +from email import quoprimime +from email import utils + +from test import support +from test.support import threading_helper +from test.support.os_helper import unlink +from test.test_email import openfile, TestEmailBase + +# These imports are documented to work, but we are testing them using a +# different path, so we import them here just to make sure they are importable. +from email.parser import FeedParser + +NL = '\n' +EMPTYSTRING = '' +SPACE = ' ' + + +# Test various aspects of the Message class's API +class TestMessageAPI(TestEmailBase): + def test_get_all(self): + eq = self.assertEqual + msg = self._msgobj('msg_20.txt') + eq(msg.get_all('cc'), ['ccc@zzz.org', 'ddd@zzz.org', 'eee@zzz.org']) + eq(msg.get_all('xx', 'n/a'), 'n/a') + + def test_getset_charset(self): + eq = self.assertEqual + msg = Message() + eq(msg.get_charset(), None) + charset = Charset('iso-8859-1') + msg.set_charset(charset) + eq(msg['mime-version'], '1.0') + eq(msg.get_content_type(), 'text/plain') + eq(msg['content-type'], 'text/plain; charset="iso-8859-1"') + eq(msg.get_param('charset'), 'iso-8859-1') + eq(msg['content-transfer-encoding'], 'quoted-printable') + eq(msg.get_charset().input_charset, 'iso-8859-1') + # Remove the charset + msg.set_charset(None) + eq(msg.get_charset(), None) + eq(msg['content-type'], 'text/plain') + # Try adding a charset when there's already MIME headers present + msg = Message() + msg['MIME-Version'] = '2.0' + msg['Content-Type'] = 'text/x-weird' + msg['Content-Transfer-Encoding'] = 'quinted-puntable' + msg.set_charset(charset) + eq(msg['mime-version'], '2.0') + eq(msg['content-type'], 'text/x-weird; charset="iso-8859-1"') + eq(msg['content-transfer-encoding'], 'quinted-puntable') + + def test_set_charset_from_string(self): + eq = self.assertEqual + msg = Message() + msg.set_charset('us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + + def test_set_payload_with_charset(self): + msg = Message() + charset = Charset('iso-8859-1') + msg.set_payload('This is a string payload', charset) + self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') + + def test_set_payload_with_8bit_data_and_charset(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + msg = Message() + msg.set_payload(data, charset) + self.assertEqual(msg['content-transfer-encoding'], 'base64') + self.assertEqual(msg.get_payload(decode=True), data) + self.assertEqual(msg.get_payload(), '0JDQkdCS\n') + + def test_set_payload_with_non_ascii_and_charset_body_encoding_none(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + charset.body_encoding = None # Disable base64 encoding + msg = Message() + msg.set_payload(data.decode('utf-8'), charset) + self.assertEqual(msg['content-transfer-encoding'], '8bit') + self.assertEqual(msg.get_payload(decode=True), data) + + def test_set_payload_with_8bit_data_and_charset_body_encoding_none(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + charset.body_encoding = None # Disable base64 encoding + msg = Message() + msg.set_payload(data, charset) + self.assertEqual(msg['content-transfer-encoding'], '8bit') + self.assertEqual(msg.get_payload(decode=True), data) + + def test_set_payload_to_list(self): + msg = Message() + msg.set_payload([]) + self.assertEqual(msg.get_payload(), []) + + def test_attach_when_payload_is_string(self): + msg = Message() + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload('string payload') + sub_msg = MIMEMessage(Message()) + self.assertRaisesRegex(TypeError, "[Aa]ttach.*non-multipart", + msg.attach, sub_msg) + + def test_get_charsets(self): + eq = self.assertEqual + + msg = self._msgobj('msg_08.txt') + charsets = msg.get_charsets() + eq(charsets, [None, 'us-ascii', 'iso-8859-1', 'iso-8859-2', 'koi8-r']) + + msg = self._msgobj('msg_09.txt') + charsets = msg.get_charsets('dingbat') + eq(charsets, ['dingbat', 'us-ascii', 'iso-8859-1', 'dingbat', + 'koi8-r']) + + msg = self._msgobj('msg_12.txt') + charsets = msg.get_charsets() + eq(charsets, [None, 'us-ascii', 'iso-8859-1', None, 'iso-8859-2', + 'iso-8859-3', 'us-ascii', 'koi8-r']) + + def test_get_filename(self): + eq = self.assertEqual + + msg = self._msgobj('msg_04.txt') + filenames = [p.get_filename() for p in msg.get_payload()] + eq(filenames, ['msg.txt', 'msg.txt']) + + msg = self._msgobj('msg_07.txt') + subpart = msg.get_payload(1) + eq(subpart.get_filename(), 'dingusfish.gif') + + def test_get_filename_with_name_parameter(self): + eq = self.assertEqual + + msg = self._msgobj('msg_44.txt') + filenames = [p.get_filename() for p in msg.get_payload()] + eq(filenames, ['msg.txt', 'msg.txt']) + + def test_get_boundary(self): + eq = self.assertEqual + msg = self._msgobj('msg_07.txt') + # No quotes! + eq(msg.get_boundary(), 'BOUNDARY') + + def test_set_boundary(self): + eq = self.assertEqual + # This one has no existing boundary parameter, but the Content-Type: + # header appears fifth. + msg = self._msgobj('msg_01.txt') + msg.set_boundary('BOUNDARY') + header, value = msg.items()[4] + eq(header.lower(), 'content-type') + eq(value, 'text/plain; charset="us-ascii"; boundary="BOUNDARY"') + # This one has a Content-Type: header, with a boundary, stuck in the + # middle of its headers. Make sure the order is preserved; it should + # be fifth. + msg = self._msgobj('msg_04.txt') + msg.set_boundary('BOUNDARY') + header, value = msg.items()[4] + eq(header.lower(), 'content-type') + eq(value, 'multipart/mixed; boundary="BOUNDARY"') + # And this one has no Content-Type: header at all. + msg = self._msgobj('msg_03.txt') + self.assertRaises(errors.HeaderParseError, + msg.set_boundary, 'BOUNDARY') + + def test_make_boundary(self): + msg = MIMEMultipart('form-data') + # Note that when the boundary gets created is an implementation + # detail and might change. + self.assertEqual(msg.items()[0][1], 'multipart/form-data') + # Trigger creation of boundary + msg.as_string() + self.assertStartsWith(msg.items()[0][1], + 'multipart/form-data; boundary="==') + # XXX: there ought to be tests of the uniqueness of the boundary, too. + + def test_message_rfc822_only(self): + # Issue 7970: message/rfc822 not in multipart parsed by + # HeaderParser caused an exception when flattened. + with openfile('msg_46.txt', encoding="utf-8") as fp: + msgdata = fp.read() + parser = HeaderParser() + msg = parser.parsestr(msgdata) + out = StringIO() + gen = Generator(out, True, 0) + gen.flatten(msg, False) + self.assertEqual(out.getvalue(), msgdata) + + def test_byte_message_rfc822_only(self): + # Make sure new bytes header parser also passes this. + with openfile('msg_46.txt', encoding="utf-8") as fp: + msgdata = fp.read().encode('ascii') + parser = email.parser.BytesHeaderParser() + msg = parser.parsebytes(msgdata) + out = BytesIO() + gen = email.generator.BytesGenerator(out) + gen.flatten(msg) + self.assertEqual(out.getvalue(), msgdata) + + def test_get_decoded_payload(self): + eq = self.assertEqual + msg = self._msgobj('msg_10.txt') + # The outer message is a multipart + eq(msg.get_payload(decode=True), None) + # Subpart 1 is 7bit encoded + eq(msg.get_payload(0).get_payload(decode=True), + b'This is a 7bit encoded message.\n') + # Subpart 2 is quopri + eq(msg.get_payload(1).get_payload(decode=True), + b'\xa1This is a Quoted Printable encoded message!\n') + # Subpart 3 is base64 + eq(msg.get_payload(2).get_payload(decode=True), + b'This is a Base64 encoded message.') + # Subpart 4 is base64 with a trailing newline, which + # used to be stripped (issue 7143). + eq(msg.get_payload(3).get_payload(decode=True), + b'This is a Base64 encoded message.\n') + # Subpart 5 has no Content-Transfer-Encoding: header. + eq(msg.get_payload(4).get_payload(decode=True), + b'This has no Content-Transfer-Encoding: header.\n') + + def test_get_decoded_uu_payload(self): + eq = self.assertEqual + msg = Message() + msg.set_payload('begin 666 -\n+:&5L;&\\@=V]R;&0 \n \nend\n') + for cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): + msg['content-transfer-encoding'] = cte + eq(msg.get_payload(decode=True), b'hello world') + # Now try some bogus data + msg.set_payload('foo') + eq(msg.get_payload(decode=True), b'foo') + + def test_get_payload_n_raises_on_non_multipart(self): + msg = Message() + self.assertRaises(TypeError, msg.get_payload, 1) + + def test_decoded_generator(self): + eq = self.assertEqual + msg = self._msgobj('msg_07.txt') + with openfile('msg_17.txt', encoding="utf-8") as fp: + text = fp.read() + s = StringIO() + g = DecodedGenerator(s) + g.flatten(msg) + eq(s.getvalue(), text) + + def test__contains__(self): + msg = Message() + msg['From'] = 'Me' + msg['to'] = 'You' + # Check for case insensitivity + self.assertIn('from', msg) + self.assertIn('From', msg) + self.assertIn('FROM', msg) + self.assertIn('to', msg) + self.assertIn('To', msg) + self.assertIn('TO', msg) + + def test_as_string(self): + msg = self._msgobj('msg_01.txt') + with openfile('msg_01.txt', encoding="utf-8") as fp: + text = fp.read() + self.assertEqual(text, str(msg)) + fullrepr = msg.as_string(unixfrom=True) + lines = fullrepr.split('\n') + self.assertStartsWith(lines[0], 'From ') + self.assertEqual(text, NL.join(lines[1:])) + + def test_as_string_policy(self): + msg = self._msgobj('msg_01.txt') + newpolicy = msg.policy.clone(linesep='\r\n') + fullrepr = msg.as_string(policy=newpolicy) + s = StringIO() + g = Generator(s, policy=newpolicy) + g.flatten(msg) + self.assertEqual(fullrepr, s.getvalue()) + + def test_nonascii_as_string_without_cte(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="iso-8859-1" + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + F=F6=F6 b=E4r + """) + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + + def test_nonascii_as_string_with_ascii_charset(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 8bit + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = source.decode('ascii', 'replace') + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + + def test_nonascii_as_string_without_content_type_and_cte(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + + Test if non-ascii messages with no Content-Type nor + Content-Transfer-Encoding set can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = source.decode('ascii', 'replace') + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + + def test_as_bytes(self): + msg = self._msgobj('msg_01.txt') + with openfile('msg_01.txt', encoding="utf-8") as fp: + data = fp.read().encode('ascii') + self.assertEqual(data, bytes(msg)) + fullrepr = msg.as_bytes(unixfrom=True) + lines = fullrepr.split(b'\n') + self.assertStartsWith(lines[0], b'From ') + self.assertEqual(data, b'\n'.join(lines[1:])) + + def test_as_bytes_policy(self): + msg = self._msgobj('msg_01.txt') + newpolicy = msg.policy.clone(linesep='\r\n') + fullrepr = msg.as_bytes(policy=newpolicy) + s = BytesIO() + g = BytesGenerator(s,policy=newpolicy) + g.flatten(msg) + self.assertEqual(fullrepr, s.getvalue()) + + # test_headerregistry.TestContentTypeHeader.bad_params + def test_bad_param(self): + msg = email.message_from_string("Content-Type: blarg; baz; boo\n") + self.assertEqual(msg.get_param('baz'), '') + + def test_continuation_sorting_part_order(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"ignored\"; " + "filename*0*=\"utf-8''foo%20\"; " + "filename*1*=\"bar.txt\"\n" + ) + filename = msg.get_filename() + self.assertEqual(filename, 'foo bar.txt') + + def test_sorting_no_continuations(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"bar.txt\"; " + ) + filename = msg.get_filename() + self.assertEqual(filename, 'bar.txt') + + def test_missing_filename(self): + msg = email.message_from_string("From: foo\n") + self.assertEqual(msg.get_filename(), None) + + def test_bogus_filename(self): + msg = email.message_from_string( + "Content-Disposition: blarg; filename\n") + self.assertEqual(msg.get_filename(), '') + + def test_missing_boundary(self): + msg = email.message_from_string("From: foo\n") + self.assertEqual(msg.get_boundary(), None) + + def test_get_params(self): + eq = self.assertEqual + msg = email.message_from_string( + 'X-Header: foo=one; bar=two; baz=three\n') + eq(msg.get_params(header='x-header'), + [('foo', 'one'), ('bar', 'two'), ('baz', 'three')]) + msg = email.message_from_string( + 'X-Header: foo; bar=one; baz=two\n') + eq(msg.get_params(header='x-header'), + [('foo', ''), ('bar', 'one'), ('baz', 'two')]) + eq(msg.get_params(), None) + msg = email.message_from_string( + 'X-Header: foo; bar="one"; baz=two\n') + eq(msg.get_params(header='x-header'), + [('foo', ''), ('bar', 'one'), ('baz', 'two')]) + + # test_headerregistry.TestContentTypeHeader.spaces_around_param_equals + def test_get_param_liberal(self): + msg = Message() + msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"' + self.assertEqual(msg.get_param('boundary'), 'CPIMSSMTPC06p5f3tG') + + def test_get_param(self): + eq = self.assertEqual + msg = email.message_from_string( + "X-Header: foo=one; bar=two; baz=three\n") + eq(msg.get_param('bar', header='x-header'), 'two') + eq(msg.get_param('quuz', header='x-header'), None) + eq(msg.get_param('quuz'), None) + msg = email.message_from_string( + 'X-Header: foo; bar="one"; baz=two\n') + eq(msg.get_param('foo', header='x-header'), '') + eq(msg.get_param('bar', header='x-header'), 'one') + eq(msg.get_param('baz', header='x-header'), 'two') + # XXX: We are not RFC-2045 compliant! We cannot parse: + # msg["Content-Type"] = 'text/plain; weird="hey; dolly? [you] @ <\\"home\\">?"' + # msg.get_param("weird") + # yet. + + # test_headerregistry.TestContentTypeHeader.spaces_around_semis + def test_get_param_funky_continuation_lines(self): + msg = self._msgobj('msg_22.txt') + self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG') + + # test_headerregistry.TestContentTypeHeader.semis_inside_quotes + def test_get_param_with_semis_in_quotes(self): + msg = email.message_from_string( + 'Content-Type: image/pjpeg; name="Jim&&Jill"\n') + self.assertEqual(msg.get_param('name'), 'Jim&&Jill') + self.assertEqual(msg.get_param('name', unquote=False), + '"Jim&&Jill"') + + # test_headerregistry.TestContentTypeHeader.quotes_inside_rfc2231_value + def test_get_param_with_quotes(self): + msg = email.message_from_string( + 'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"') + self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + msg = email.message_from_string( + "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") + self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + + @unittest.skip('TODO: RUSTPYTHON; Takes a long time to the point of timeouting') + def test_get_param_linear_complexity(self): + # Ensure that email.message._parseparam() is fast. + # See https://github.com/python/cpython/issues/136063. + N = 100_000 + for s, r in [ + ("", ""), + ("foo=bar", "foo=bar"), + (" FOO = bar ", "foo=bar"), + ]: + with self.subTest(s=s, r=r, N=N): + src = f'{s};' * (N - 1) + s + res = email.message._parseparam(src) + self.assertEqual(len(res), N) + self.assertEqual(len(set(res)), 1) + self.assertEqual(res[0], r) + + # This will be considered as a single parameter. + malformed = 's="' + ';' * (N - 1) + res = email.message._parseparam(malformed) + self.assertEqual(res, [malformed]) + + def test_field_containment(self): + msg = email.message_from_string('Header: exists') + self.assertIn('header', msg) + self.assertIn('Header', msg) + self.assertIn('HEADER', msg) + self.assertNotIn('headerx', msg) + + def test_set_param(self): + eq = self.assertEqual + msg = Message() + msg.set_param('charset', 'iso-2022-jp') + eq(msg.get_param('charset'), 'iso-2022-jp') + msg.set_param('importance', 'high value') + eq(msg.get_param('importance'), 'high value') + eq(msg.get_param('importance', unquote=False), '"high value"') + eq(msg.get_params(), [('text/plain', ''), + ('charset', 'iso-2022-jp'), + ('importance', 'high value')]) + eq(msg.get_params(unquote=False), [('text/plain', ''), + ('charset', '"iso-2022-jp"'), + ('importance', '"high value"')]) + msg.set_param('charset', 'iso-9999-xx', header='X-Jimmy') + eq(msg.get_param('charset', header='X-Jimmy'), 'iso-9999-xx') + + def test_del_param(self): + eq = self.assertEqual + msg = self._msgobj('msg_05.txt') + eq(msg.get_params(), + [('multipart/report', ''), ('report-type', 'delivery-status'), + ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) + old_val = msg.get_param("report-type") + msg.del_param("report-type") + eq(msg.get_params(), + [('multipart/report', ''), + ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) + msg.set_param("report-type", old_val) + eq(msg.get_params(), + [('multipart/report', ''), + ('boundary', 'D1690A7AC1.996856090/mail.example.com'), + ('report-type', old_val)]) + + def test_del_param_on_other_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') + msg.del_param('filename', 'content-disposition') + self.assertEqual(msg['content-disposition'], 'attachment') + + def test_del_param_on_nonexistent_header(self): + msg = Message() + # Deleting param on empty msg should not raise exception. + msg.del_param('filename', 'content-disposition') + + def test_del_nonexistent_param(self): + msg = Message() + msg.add_header('Content-Type', 'text/plain', charset='utf-8') + existing_header = msg['Content-Type'] + msg.del_param('foobar', header='Content-Type') + self.assertEqual(msg['Content-Type'], existing_header) + + def test_set_type(self): + eq = self.assertEqual + msg = Message() + self.assertRaises(ValueError, msg.set_type, 'text') + msg.set_type('text/plain') + eq(msg['content-type'], 'text/plain') + msg.set_param('charset', 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + msg.set_type('text/html') + eq(msg['content-type'], 'text/html; charset="us-ascii"') + + def test_set_type_on_other_header(self): + msg = Message() + msg['X-Content-Type'] = 'text/plain' + msg.set_type('application/octet-stream', 'X-Content-Type') + self.assertEqual(msg['x-content-type'], 'application/octet-stream') + + def test_get_content_type_missing(self): + msg = Message() + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_type_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_type(), 'message/rfc822') + + def test_get_content_type_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_type(), + 'message/rfc822') + + def test_get_content_type_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_type(), + 'message/rfc822') + + def test_get_content_type_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_type_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_maintype_missing(self): + msg = Message() + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_maintype_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_maintype_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_subtype_missing(self): + msg = Message() + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_subtype_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_subtype_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_maintype_error(self): + msg = Message() + msg['Content-Type'] = 'no-slash-in-this-string' + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_subtype_error(self): + msg = Message() + msg['Content-Type'] = 'no-slash-in-this-string' + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_replace_header(self): + eq = self.assertEqual + msg = Message() + msg.add_header('First', 'One') + msg.add_header('Second', 'Two') + msg.add_header('Third', 'Three') + eq(msg.keys(), ['First', 'Second', 'Third']) + eq(msg.values(), ['One', 'Two', 'Three']) + msg.replace_header('Second', 'Twenty') + eq(msg.keys(), ['First', 'Second', 'Third']) + eq(msg.values(), ['One', 'Twenty', 'Three']) + msg.add_header('First', 'Eleven') + msg.replace_header('First', 'One Hundred') + eq(msg.keys(), ['First', 'Second', 'Third', 'First']) + eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven']) + self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing') + + def test_get_content_disposition(self): + msg = Message() + self.assertIsNone(msg.get_content_disposition()) + msg.add_header('Content-Disposition', 'attachment', + filename='random.avi') + self.assertEqual(msg.get_content_disposition(), 'attachment') + msg.replace_header('Content-Disposition', 'inline') + self.assertEqual(msg.get_content_disposition(), 'inline') + msg.replace_header('Content-Disposition', 'InlinE') + self.assertEqual(msg.get_content_disposition(), 'inline') + + # test_defect_handling:test_invalid_chars_in_base64_payload + def test_broken_base64_payload(self): + x = 'AwDp0P7//y6LwKEAcPa/6Q=9' + msg = Message() + msg['content-type'] = 'audio/x-midi' + msg['content-transfer-encoding'] = 'base64' + msg.set_payload(x) + self.assertEqual(msg.get_payload(decode=True), + (b'\x03\x00\xe9\xd0\xfe\xff\xff.\x8b\xc0' + b'\xa1\x00p\xf6\xbf\xe9\x0f')) + self.assertIsInstance(msg.defects[0], + errors.InvalidBase64CharactersDefect) + + def test_broken_unicode_payload(self): + # This test improves coverage but is not a compliance test. + # The behavior in this situation is currently undefined by the API. + x = 'this is a br\xf6ken thing to do' + msg = Message() + msg['content-type'] = 'text/plain' + msg['content-transfer-encoding'] = '8bit' + msg.set_payload(x) + self.assertEqual(msg.get_payload(decode=True), + bytes(x, 'raw-unicode-escape')) + + def test_questionable_bytes_payload(self): + # This test improves coverage but is not a compliance test, + # since it involves poking inside the black box. + x = 'this is a quéstionable thing to do'.encode('utf-8') + msg = Message() + msg['content-type'] = 'text/plain; charset="utf-8"' + msg['content-transfer-encoding'] = '8bit' + msg._payload = x + self.assertEqual(msg.get_payload(decode=True), x) + + # Issue 1078919 + def test_ascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename='bud.gif') + self.assertEqual('attachment; filename="bud.gif"', + msg['Content-Disposition']) + + def test_noascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="Fußballer.ppt") + self.assertEqual( + 'attachment; filename*=utf-8\'\'Fu%C3%9Fballer.ppt', + msg['Content-Disposition']) + + def test_nonascii_add_header_via_triple(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename=('iso-8859-1', '', 'Fußballer.ppt')) + self.assertEqual( + 'attachment; filename*=iso-8859-1\'\'Fu%DFballer.ppt', + msg['Content-Disposition']) + + def test_ascii_add_header_with_tspecial(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="windows [filename].ppt") + self.assertEqual( + 'attachment; filename="windows [filename].ppt"', + msg['Content-Disposition']) + + def test_nonascii_add_header_with_tspecial(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="Fußballer [filename].ppt") + self.assertEqual( + "attachment; filename*=utf-8''Fu%C3%9Fballer%20%5Bfilename%5D.ppt", + msg['Content-Disposition']) + + def test_binary_quopri_payload(self): + for charset in ('latin-1', 'ascii'): + msg = Message() + msg['content-type'] = 'text/plain; charset=%s' % charset + msg['content-transfer-encoding'] = 'quoted-printable' + msg.set_payload(b'foo=e6=96=87bar') + self.assertEqual( + msg.get_payload(decode=True), + b'foo\xe6\x96\x87bar', + 'get_payload returns wrong result with charset %s.' % charset) + + def test_binary_base64_payload(self): + for charset in ('latin-1', 'ascii'): + msg = Message() + msg['content-type'] = 'text/plain; charset=%s' % charset + msg['content-transfer-encoding'] = 'base64' + msg.set_payload(b'Zm9v5paHYmFy') + self.assertEqual( + msg.get_payload(decode=True), + b'foo\xe6\x96\x87bar', + 'get_payload returns wrong result with charset %s.' % charset) + + def test_binary_uuencode_payload(self): + for charset in ('latin-1', 'ascii'): + for encoding in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): + msg = Message() + msg['content-type'] = 'text/plain; charset=%s' % charset + msg['content-transfer-encoding'] = encoding + msg.set_payload(b"begin 666 -\n)9F]OYI:'8F%R\n \nend\n") + self.assertEqual( + msg.get_payload(decode=True), + b'foo\xe6\x96\x87bar', + str(('get_payload returns wrong result ', + 'with charset {0} and encoding {1}.')).\ + format(charset, encoding)) + + def test_add_header_with_name_only_param(self): + msg = Message() + msg.add_header('Content-Disposition', 'inline', foo_bar=None) + self.assertEqual("inline; foo-bar", msg['Content-Disposition']) + + def test_add_header_with_no_value(self): + msg = Message() + msg.add_header('X-Status', None) + self.assertEqual('', msg['X-Status']) + + # Issue 5871: reject an attempt to embed a header inside a header value + # (header injection attack). + def test_embedded_header_via_Header_rejected(self): + msg = Message() + msg['Dummy'] = Header('dummy\nX-Injected-Header: test') + self.assertRaises(errors.HeaderParseError, msg.as_string) + + def test_embedded_header_via_string_rejected(self): + msg = Message() + msg['Dummy'] = 'dummy\nX-Injected-Header: test' + self.assertRaises(errors.HeaderParseError, msg.as_string) + + def test_unicode_header_defaults_to_utf8_encoding(self): + # Issue 14291 + m = MIMEText('abc\n') + m['Subject'] = 'É test' + self.assertEqual(str(m),textwrap.dedent("""\ + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Subject: =?utf-8?q?=C3=89_test?= + + abc + """)) + + def test_unicode_body_defaults_to_utf8_encoding(self): + # Issue 14291 + m = MIMEText('É testabc\n') + self.assertEqual(str(m),textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + MIME-Version: 1.0 + Content-Transfer-Encoding: base64 + + w4kgdGVzdGFiYwo= + """)) + + def test_string_payload_with_base64_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + + +# Test the email.encoders module +class TestEncoders(unittest.TestCase): + + def test_EncodersEncode_base64(self): + with openfile('python.gif', 'rb') as fp: + bindata = fp.read() + mimed = email.mime.image.MIMEImage(bindata) + base64ed = mimed.get_payload() + # the transfer-encoded body lines should all be <=76 characters + lines = base64ed.split('\n') + self.assertLessEqual(max([ len(x) for x in lines ]), 76) + + def test_encode_empty_payload(self): + eq = self.assertEqual + msg = Message() + msg.set_charset('us-ascii') + eq(msg['content-transfer-encoding'], '7bit') + + def test_default_cte(self): + eq = self.assertEqual + # 7bit data and the default us-ascii _charset + msg = MIMEText('hello world') + eq(msg['content-transfer-encoding'], '7bit') + # Similar, but with 8bit data + msg = MIMEText('hello \xf8 world') + eq(msg['content-transfer-encoding'], 'base64') + # And now with a different charset + msg = MIMEText('hello \xf8 world', _charset='iso-8859-1') + eq(msg['content-transfer-encoding'], 'quoted-printable') + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp + def test_encode7or8bit(self): + # Make sure a charset whose input character set is 8bit but + # whose output character set is 7bit gets a transfer-encoding + # of 7bit. + eq = self.assertEqual + msg = MIMEText('文\n', _charset='euc-jp') + eq(msg['content-transfer-encoding'], '7bit') + eq(msg.as_string(), textwrap.dedent("""\ + MIME-Version: 1.0 + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \x1b$BJ8\x1b(B + """)) + + def test_qp_encode_latin1(self): + msg = MIMEText('\xe1\xf6\n', 'text', 'ISO-8859-1') + self.assertEqual(str(msg), textwrap.dedent("""\ + MIME-Version: 1.0 + Content-Type: text/text; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + =E1=F6 + """)) + + def test_qp_encode_non_latin1(self): + # Issue 16948 + msg = MIMEText('\u017c\n', 'text', 'ISO-8859-2') + self.assertEqual(str(msg), textwrap.dedent("""\ + MIME-Version: 1.0 + Content-Type: text/text; charset="iso-8859-2" + Content-Transfer-Encoding: quoted-printable + + =BF + """)) + + +# Test long header wrapping +class TestLongHeaders(TestEmailBase): + + maxDiff = None + + def test_split_long_continuation(self): + eq = self.ndiffAssertEqual + msg = email.message_from_string("""\ +Subject: bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text + +test +""") + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +Subject: bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text + +test +""") + + def test_another_long_almost_unsplittable_header(self): + eq = self.ndiffAssertEqual + hstr = """\ +bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text""" + h = Header(hstr, continuation_ws='\t') + eq(h.encode(), """\ +bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text""") + h = Header(hstr.replace('\t', ' ')) + eq(h.encode(), """\ +bug demonstration + 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 + more text""") + + def test_long_nonstring(self): + eq = self.ndiffAssertEqual + g = Charset("iso-8859-1") + cz = Charset("iso-8859-2") + utf8 = Charset("utf-8") + g_head = (b'Die Mieter treten hier ein werden mit einem Foerderband ' + b'komfortabel den Korridor entlang, an s\xfcdl\xfcndischen ' + b'Wandgem\xe4lden vorbei, gegen die rotierenden Klingen ' + b'bef\xf6rdert. ') + cz_head = (b'Finan\xe8ni metropole se hroutily pod tlakem jejich ' + b'd\xf9vtipu.. ') + utf8_head = ('\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das ' + 'Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder ' + 'die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066' + '\u3044\u307e\u3059\u3002') + h = Header(g_head, g, header_name='Subject') + h.append(cz_head, cz) + h.append(utf8_head, utf8) + msg = Message() + msg['Subject'] = h + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +Subject: =?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderb?= + =?iso-8859-1?q?and_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen?= + =?iso-8859-1?q?_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef?= + =?iso-8859-1?q?=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hrouti?= + =?iso-8859-2?q?ly_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= + =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC5LiA?= + =?utf-8?b?6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn44Gf44KJ?= + =?utf-8?b?44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFzIE51bnN0dWNr?= + =?utf-8?b?IGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWloZXJodW5kIGRhcyBPZGVyIGRp?= + =?utf-8?b?ZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI3jgajoqIDjgaPjgabjgYTjgb7jgZk=?= + =?utf-8?b?44CC?= + +""") + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerde?= + =?iso-8859-1?q?rband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndis?= + =?iso-8859-1?q?chen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klinge?= + =?iso-8859-1?q?n_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se?= + =?iso-8859-2?q?_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= + =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb?= + =?utf-8?b?44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go?= + =?utf-8?b?44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBp?= + =?utf-8?b?c3QgZGFzIE51bnN0dWNrIGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWlo?= + =?utf-8?b?ZXJodW5kIGRhcyBPZGVyIGRpZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI0=?= + =?utf-8?b?44Go6KiA44Gj44Gm44GE44G+44GZ44CC?=""") + + def test_long_header_encode(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + def test_long_header_encode_with_tab_continuation_is_just_a_hint(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit', + continuation_ws='\t') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + def test_long_header_encode_with_tab_continuation(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals";\t' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit', + continuation_ws='\t') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; +\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp + def test_header_encode_with_different_output_charset(self): + h = Header('文', 'euc-jp') + self.assertEqual(h.encode(), "=?iso-2022-jp?b?GyRCSjgbKEI=?=") + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc-jp + def test_long_header_encode_with_different_output_charset(self): + h = Header(b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4' + b'\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4' + b'\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3\xa4\xc6\xa4' + b'\xa4\xa4\xde\xa4\xb9'.decode('euc-jp'), 'euc-jp') + res = """\ +=?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKMnE8VCROPjUbKEI=?= + =?iso-2022-jp?b?GyRCRyckckJUJEMkRiQkJF4kORsoQg==?=""" + self.assertEqual(h.encode(), res) + + def test_header_splitter(self): + eq = self.ndiffAssertEqual + msg = MIMEText('') + # It'd be great if we could use add_header() here, but that doesn't + # guarantee an order of the parameters. + msg['X-Foobar-Spoink-Defrobnit'] = ( + 'wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), '''\ +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" + +''') + + def test_no_semis_header_splitter(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join('<%d@dom.ain>' % i for i in range(10)) + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: <0@dom.ain> <1@dom.ain> <2@dom.ain> <3@dom.ain> <4@dom.ain> + <5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain> + +Test""") + + def test_last_split_chunk_does_not_fit(self): + eq = self.ndiffAssertEqual + h = Header('Subject: the first part of this is short, but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +Subject: the first part of this is short, + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_splittable_leading_char_followed_by_overlong_unsplittable(self): + eq = self.ndiffAssertEqual + h = Header(', but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +, + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_multiple_splittable_leading_char_followed_by_overlong_unsplittable(self): + eq = self.ndiffAssertEqual + h = Header(', , but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +, , + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_trailing_splittable_on_overlong_unsplittable(self): + eq = self.ndiffAssertEqual + h = Header('this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself;') + eq(h.encode(), "this_part_does_not_fit_within_maxlinelen_and_thus_should_" + "be_on_a_line_all_by_itself;") + + def test_trailing_splittable_on_overlong_unsplittable_with_leading_splittable(self): + eq = self.ndiffAssertEqual + h = Header('; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_long_header_with_multiple_sequential_split_chars(self): + eq = self.ndiffAssertEqual + h = Header('This is a long line that has two whitespaces in a row. ' + 'This used to cause truncation of the header when folded') + eq(h.encode(), """\ +This is a long line that has two whitespaces in a row. This used to cause + truncation of the header when folded""") + + def test_splitter_split_on_punctuation_only_if_fws_with_header(self): + eq = self.ndiffAssertEqual + h = Header('thisverylongheaderhas;semicolons;and,commas,but' + 'they;arenotlegal;fold,points') + eq(h.encode(), "thisverylongheaderhas;semicolons;and,commas,butthey;" + "arenotlegal;fold,points") + + def test_leading_splittable_in_the_middle_just_before_overlong_last_part(self): + eq = self.ndiffAssertEqual + h = Header('this is a test where we need to have more than one line ' + 'before; our final line that is just too big to fit;; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself;') + eq(h.encode(), """\ +this is a test where we need to have more than one line before; + our final line that is just too big to fit;; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself;""") + + def test_overlong_last_part_followed_by_split_point(self): + eq = self.ndiffAssertEqual + h = Header('this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself ') + eq(h.encode(), "this_part_does_not_fit_within_maxlinelen_and_thus_" + "should_be_on_a_line_all_by_itself ") + + def test_multiline_with_overlong_parts_separated_by_two_split_points(self): + eq = self.ndiffAssertEqual + h = Header('this_is_a__test_where_we_need_to_have_more_than_one_line_' + 'before_our_final_line_; ; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +this_is_a__test_where_we_need_to_have_more_than_one_line_before_our_final_line_; + ; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_multiline_with_overlong_last_part_followed_by_split_point(self): + eq = self.ndiffAssertEqual + h = Header('this is a test where we need to have more than one line ' + 'before our final line; ; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +this is a test where we need to have more than one line before our final line; + ; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_long_header_with_whitespace_runs(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join([' '] * 10) + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: + + \x20\x20 + +Test""") + + def test_long_run_with_semi_header_splitter(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join([''] * 10) + '; abc' + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: + + ; abc + +Test""") + + def test_splitter_split_on_punctuation_only_if_fws(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = ('thisverylongheaderhas;semicolons;and,commas,but' + 'they;arenotlegal;fold,points') + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + # XXX the space after the header should not be there. + eq(sfp.getvalue(), """\ +From: test@dom.ain +References:\x20 + thisverylongheaderhas;semicolons;and,commas,butthey;arenotlegal;fold,points + +Test""") + + def test_no_split_long_header(self): + eq = self.ndiffAssertEqual + hstr = 'References: ' + 'x' * 80 + h = Header(hstr) + # These come on two lines because Headers are really field value + # classes and don't really know about their field names. + eq(h.encode(), """\ +References: + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""") + h = Header('x' * 80) + eq(h.encode(), 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') + + def test_splitting_multiple_long_lines(self): + eq = self.ndiffAssertEqual + hstr = """\ +from babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for ; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for ; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for ; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +""" + h = Header(hstr, continuation_ws='\t') + eq(h.encode(), """\ +from babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for ; + Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for ; + Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for ; + Sat, 2 Feb 2002 17:00:06 -0800 (PST)""") + + def test_splitting_first_line_only_is_long(self): + eq = self.ndiffAssertEqual + hstr = """\ +from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] helo=cthulhu.gerg.ca) +\tby kronos.mems-exchange.org with esmtp (Exim 4.05) +\tid 17k4h5-00034i-00 +\tfor test@mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""" + h = Header(hstr, maxlinelen=78, header_name='Received', + continuation_ws='\t') + eq(h.encode(), """\ +from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] + helo=cthulhu.gerg.ca) +\tby kronos.mems-exchange.org with esmtp (Exim 4.05) +\tid 17k4h5-00034i-00 +\tfor test@mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""") + + def test_long_8bit_header(self): + eq = self.ndiffAssertEqual + msg = Message() + h = Header('Britische Regierung gibt', 'iso-8859-1', + header_name='Subject') + h.append('gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte') + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offs?= + =?iso-8859-1?q?hore-Windkraftprojekte?=""") + msg['Subject'] = h + eq(msg.as_string(maxheaderlen=76), """\ +Subject: =?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offs?= + =?iso-8859-1?q?hore-Windkraftprojekte?= + +""") + eq(msg.as_string(maxheaderlen=0), """\ +Subject: =?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offshore-Windkraftprojekte?= + +""") + + def test_long_8bit_header_no_charset(self): + eq = self.ndiffAssertEqual + msg = Message() + header_string = ('Britische Regierung gibt gr\xfcnes Licht ' + 'f\xfcr Offshore-Windkraftprojekte ' + '') + msg['Reply-To'] = header_string + eq(msg.as_string(maxheaderlen=78), """\ +Reply-To: =?utf-8?q?Britische_Regierung_gibt_gr=C3=BCnes_Licht_f=C3=BCr_Offs?= + =?utf-8?q?hore-Windkraftprojekte_=3Ca-very-long-address=40example=2Ecom=3E?= + +""") + msg = Message() + msg['Reply-To'] = Header(header_string, + header_name='Reply-To') + eq(msg.as_string(maxheaderlen=78), """\ +Reply-To: =?utf-8?q?Britische_Regierung_gibt_gr=C3=BCnes_Licht_f=C3=BCr_Offs?= + =?utf-8?q?hore-Windkraftprojekte_=3Ca-very-long-address=40example=2Ecom=3E?= + +""") + + def test_long_to_header(self): + eq = self.ndiffAssertEqual + to = ('"Someone Test #A" ,' + ', ' + '"Someone Test #B" , ' + '"Someone Test #C" , ' + '"Someone Test #D" ') + msg = Message() + msg['To'] = to + eq(msg.as_string(maxheaderlen=78), '''\ +To: "Someone Test #A" ,, + "Someone Test #B" , + "Someone Test #C" , + "Someone Test #D" + +''') + + def test_long_line_after_append(self): + eq = self.ndiffAssertEqual + s = 'This is an example of string which has almost the limit of header length.' + h = Header(s) + h.append('Add another line.') + eq(h.encode(maxlinelen=76), """\ +This is an example of string which has almost the limit of header length. + Add another line.""") + + def test_shorter_line_with_append(self): + eq = self.ndiffAssertEqual + s = 'This is a shorter line.' + h = Header(s) + h.append('Add another sentence. (Surprise?)') + eq(h.encode(), + 'This is a shorter line. Add another sentence. (Surprise?)') + + def test_long_field_name(self): + eq = self.ndiffAssertEqual + fn = 'X-Very-Very-Very-Long-Header-Name' + gs = ('Die Mieter treten hier ein werden mit einem Foerderband ' + 'komfortabel den Korridor entlang, an s\xfcdl\xfcndischen ' + 'Wandgem\xe4lden vorbei, gegen die rotierenden Klingen ' + 'bef\xf6rdert. ') + h = Header(gs, 'iso-8859-1', header_name=fn) + # BAW: this seems broken because the first line is too long + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_e?= + =?iso-8859-1?q?in_werden_mit_einem_Foerderband_komfortabel_den_Korridor_e?= + =?iso-8859-1?q?ntlang=2C_an_s=FCdl=FCndischen_Wandgem=E4lden_vorbei=2C_ge?= + =?iso-8859-1?q?gen_die_rotierenden_Klingen_bef=F6rdert=2E_?=""") + + def test_long_received_header(self): + h = ('from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) ' + 'by hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; ' + 'Wed, 05 Mar 2003 18:10:18 -0700') + msg = Message() + msg['Received-1'] = Header(h, continuation_ws='\t') + msg['Received-2'] = h + # This should be splitting on spaces not semicolons. + self.ndiffAssertEqual(msg.as_string(maxheaderlen=78), """\ +Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by + hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; + Wed, 05 Mar 2003 18:10:18 -0700 +Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by + hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; + Wed, 05 Mar 2003 18:10:18 -0700 + +""") + + def test_string_headerinst_eq(self): + h = ('<15975.17901.207240.414604@sgigritzmann1.mathematik.' + 'tu-muenchen.de> (David Bremner\'s message of ' + '"Thu, 6 Mar 2003 13:58:21 +0100")') + msg = Message() + msg['Received-1'] = Header(h, header_name='Received-1', + continuation_ws='\t') + msg['Received-2'] = h + # XXX The space after the ':' should not be there. + self.ndiffAssertEqual(msg.as_string(maxheaderlen=78), """\ +Received-1:\x20 + <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David + Bremner's message of \"Thu, 6 Mar 2003 13:58:21 +0100\") +Received-2:\x20 + <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David + Bremner's message of \"Thu, 6 Mar 2003 13:58:21 +0100\") + +""") + + def test_long_unbreakable_lines_with_continuation(self): + eq = self.ndiffAssertEqual + msg = Message() + t = """\ +iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp""" + msg['Face-1'] = t + msg['Face-2'] = Header(t, header_name='Face-2') + msg['Face-3'] = ' ' + t + # XXX This splitting is all wrong. It the first value line should be + # snug against the field name or the space after the header not there. + eq(msg.as_string(maxheaderlen=78), """\ +Face-1:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp +Face-2:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp +Face-3:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp + +""") + + def test_another_long_multiline_header(self): + eq = self.ndiffAssertEqual + m = ('Received: from siimage.com ' + '([172.25.1.3]) by zima.siliconimage.com with ' + 'Microsoft SMTPSVC(5.0.2195.4905); ' + 'Wed, 16 Oct 2002 07:41:11 -0700') + msg = email.message_from_string(m) + eq(msg.as_string(maxheaderlen=78), '''\ +Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with + Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700 + +''') + + def test_long_lines_with_different_header(self): + eq = self.ndiffAssertEqual + h = ('List-Unsubscribe: ' + ',' + ' ') + msg = Message() + msg['List'] = h + msg['List'] = Header(h, header_name='List') + eq(msg.as_string(maxheaderlen=78), """\ +List: List-Unsubscribe: + , + +List: List-Unsubscribe: + , + + +""") + + def test_long_rfc2047_header_with_embedded_fws(self): + h = Header(textwrap.dedent("""\ + We're going to pretend this header is in a non-ascii character set + \tto see if line wrapping with encoded words and embedded + folding white space works"""), + charset='utf-8', + header_name='Test') + self.assertEqual(h.encode()+'\n', textwrap.dedent("""\ + =?utf-8?q?We=27re_going_to_pretend_this_header_is_in_a_non-ascii_chara?= + =?utf-8?q?cter_set?= + =?utf-8?q?_to_see_if_line_wrapping_with_encoded_words_and_embedded?= + =?utf-8?q?_folding_white_space_works?=""")+'\n') + + + +# Test mangling of "From " lines in the body of a message +class TestFromMangling(unittest.TestCase): + def setUp(self): + self.msg = Message() + self.msg['From'] = 'aaa@bbb.org' + self.msg.set_payload("""\ +From the desk of A.A.A.: +Blah blah blah +""") + + def test_mangled_from(self): + s = StringIO() + g = Generator(s, mangle_from_=True) + g.flatten(self.msg) + self.assertEqual(s.getvalue(), """\ +From: aaa@bbb.org + +>From the desk of A.A.A.: +Blah blah blah +""") + + def test_dont_mangle_from(self): + s = StringIO() + g = Generator(s, mangle_from_=False) + g.flatten(self.msg) + self.assertEqual(s.getvalue(), """\ +From: aaa@bbb.org + +From the desk of A.A.A.: +Blah blah blah +""") + + def test_mangle_from_in_preamble_and_epilog(self): + s = StringIO() + g = Generator(s, mangle_from_=True) + msg = email.message_from_string(textwrap.dedent("""\ + From: foo@bar.com + Mime-Version: 1.0 + Content-Type: multipart/mixed; boundary=XXX + + From somewhere unknown + + --XXX + Content-Type: text/plain + + foo + + --XXX-- + + From somewhere unknowable + """)) + g.flatten(msg) + self.assertEqual(len([1 for x in s.getvalue().split('\n') + if x.startswith('>From ')]), 2) + + def test_mangled_from_with_bad_bytes(self): + source = textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + MIME-Version: 1.0 + Content-Transfer-Encoding: 8bit + From: aaa@bbb.org + + """).encode('utf-8') + msg = email.message_from_bytes(source + b'From R\xc3\xb6lli\n') + b = BytesIO() + g = BytesGenerator(b, mangle_from_=True) + g.flatten(msg) + self.assertEqual(b.getvalue(), source + b'>From R\xc3\xb6lli\n') + + def test_multipart_with_bad_bytes_in_cte(self): + # bpo30835 + source = textwrap.dedent("""\ + From: aperson@example.com + Content-Type: multipart/mixed; boundary="1" + Content-Transfer-Encoding: \xc8 + """).encode('utf-8') + msg = email.message_from_bytes(source) + + +# Test the basic MIMEAudio class +class TestMIMEAudio(unittest.TestCase): + def _make_audio(self, ext): + with openfile(f'sndhdr.{ext}', 'rb') as fp: + self._audiodata = fp.read() + self._au = MIMEAudio(self._audiodata) + + def test_guess_minor_type(self): + for ext, subtype in { + 'aifc': 'x-aiff', + 'aiff': 'x-aiff', + 'wav': 'x-wav', + 'au': 'basic', + }.items(): + self._make_audio(ext) + subtype = ext if subtype is None else subtype + self.assertEqual(self._au.get_content_type(), f'audio/{subtype}') + + def test_encoding(self): + self._make_audio('au') + payload = self._au.get_payload() + self.assertEqual(base64.decodebytes(bytes(payload, 'ascii')), + self._audiodata) + + def test_checkSetMinor(self): + self._make_audio('au') + au = MIMEAudio(self._audiodata, 'fish') + self.assertEqual(au.get_content_type(), 'audio/fish') + + def test_add_header(self): + self._make_audio('au') + eq = self.assertEqual + self._au.add_header('Content-Disposition', 'attachment', + filename='sndhdr.au') + eq(self._au['content-disposition'], + 'attachment; filename="sndhdr.au"') + eq(self._au.get_params(header='content-disposition'), + [('attachment', ''), ('filename', 'sndhdr.au')]) + eq(self._au.get_param('filename', header='content-disposition'), + 'sndhdr.au') + missing = [] + eq(self._au.get_param('attachment', header='content-disposition'), '') + self.assertIs(self._au.get_param( + 'foo', failobj=missing, + header='content-disposition'), missing) + # Try some missing stuff + self.assertIs(self._au.get_param('foobar', missing), missing) + self.assertIs(self._au.get_param('attachment', missing, + header='foobar'), missing) + + + +# Test the basic MIMEImage class +class TestMIMEImage(unittest.TestCase): + def _make_image(self, ext): + with openfile(f'python.{ext}', 'rb') as fp: + self._imgdata = fp.read() + self._im = MIMEImage(self._imgdata) + + def test_guess_minor_type(self): + for ext, subtype in { + 'bmp': None, + 'exr': None, + 'gif': None, + 'jpg': 'jpeg', + 'pbm': None, + 'pgm': None, + 'png': None, + 'ppm': None, + 'ras': 'rast', + 'sgi': 'rgb', + 'tiff': None, + 'webp': None, + 'xbm': None, + }.items(): + self._make_image(ext) + subtype = ext if subtype is None else subtype + self.assertEqual(self._im.get_content_type(), f'image/{subtype}') + + def test_encoding(self): + self._make_image('gif') + payload = self._im.get_payload() + self.assertEqual(base64.decodebytes(bytes(payload, 'ascii')), + self._imgdata) + + def test_checkSetMinor(self): + self._make_image('gif') + im = MIMEImage(self._imgdata, 'fish') + self.assertEqual(im.get_content_type(), 'image/fish') + + def test_add_header(self): + self._make_image('gif') + eq = self.assertEqual + self._im.add_header('Content-Disposition', 'attachment', + filename='dingusfish.gif') + eq(self._im['content-disposition'], + 'attachment; filename="dingusfish.gif"') + eq(self._im.get_params(header='content-disposition'), + [('attachment', ''), ('filename', 'dingusfish.gif')]) + eq(self._im.get_param('filename', header='content-disposition'), + 'dingusfish.gif') + missing = [] + eq(self._im.get_param('attachment', header='content-disposition'), '') + self.assertIs(self._im.get_param('foo', failobj=missing, + header='content-disposition'), missing) + # Try some missing stuff + self.assertIs(self._im.get_param('foobar', missing), missing) + self.assertIs(self._im.get_param('attachment', missing, + header='foobar'), missing) + + +# Test the basic MIMEApplication class +class TestMIMEApplication(unittest.TestCase): + def test_headers(self): + eq = self.assertEqual + msg = MIMEApplication(b'\xfa\xfb\xfc\xfd\xfe\xff') + eq(msg.get_content_type(), 'application/octet-stream') + eq(msg['content-transfer-encoding'], 'base64') + + def test_body(self): + eq = self.assertEqual + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata) + # whitespace in the cte encoded block is RFC-irrelevant. + eq(msg.get_payload().strip(), '+vv8/f7/') + eq(msg.get_payload(decode=True), bytesdata) + + def test_binary_body_with_encode_7or8bit(self): + # Issue 17171. + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata, _encoder=encoders.encode_7or8bit) + # Treated as a string, this will be invalid code points. + self.assertEqual(msg.get_payload(), '\uFFFD' * len(bytesdata)) + self.assertEqual(msg.get_payload(decode=True), bytesdata) + self.assertEqual(msg['Content-Transfer-Encoding'], '8bit') + s = BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + wireform = s.getvalue() + msg2 = email.message_from_bytes(wireform) + self.assertEqual(msg.get_payload(), '\uFFFD' * len(bytesdata)) + self.assertEqual(msg2.get_payload(decode=True), bytesdata) + self.assertEqual(msg2['Content-Transfer-Encoding'], '8bit') + + def test_binary_body_with_encode_noop(self): + # Issue 16564: This does not produce an RFC valid message, since to be + # valid it should have a CTE of binary. But the below works in + # Python2, and is documented as working this way. + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata, _encoder=encoders.encode_noop) + # Treated as a string, this will be invalid code points. + self.assertEqual(msg.get_payload(), '\uFFFD' * len(bytesdata)) + self.assertEqual(msg.get_payload(decode=True), bytesdata) + s = BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + wireform = s.getvalue() + msg2 = email.message_from_bytes(wireform) + self.assertEqual(msg.get_payload(), '\uFFFD' * len(bytesdata)) + self.assertEqual(msg2.get_payload(decode=True), bytesdata) + + def test_binary_body_with_unicode_linend_encode_noop(self): + # Issue 19003: This is a variation on #16564. + bytesdata = b'\x0b\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata, _encoder=encoders.encode_noop) + self.assertEqual(msg.get_payload(decode=True), bytesdata) + s = BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + wireform = s.getvalue() + msg2 = email.message_from_bytes(wireform) + self.assertEqual(msg2.get_payload(decode=True), bytesdata) + + def test_binary_body_with_encode_quopri(self): + # Issue 14360. + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff ' + msg = MIMEApplication(bytesdata, _encoder=encoders.encode_quopri) + self.assertEqual(msg.get_payload(), '=FA=FB=FC=FD=FE=FF=20') + self.assertEqual(msg.get_payload(decode=True), bytesdata) + self.assertEqual(msg['Content-Transfer-Encoding'], 'quoted-printable') + s = BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + wireform = s.getvalue() + msg2 = email.message_from_bytes(wireform) + self.assertEqual(msg.get_payload(), '=FA=FB=FC=FD=FE=FF=20') + self.assertEqual(msg2.get_payload(decode=True), bytesdata) + self.assertEqual(msg2['Content-Transfer-Encoding'], 'quoted-printable') + + def test_binary_body_with_encode_base64(self): + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata, _encoder=encoders.encode_base64) + self.assertEqual(msg.get_payload(), '+vv8/f7/\n') + self.assertEqual(msg.get_payload(decode=True), bytesdata) + s = BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + wireform = s.getvalue() + msg2 = email.message_from_bytes(wireform) + self.assertEqual(msg.get_payload(), '+vv8/f7/\n') + self.assertEqual(msg2.get_payload(decode=True), bytesdata) + + +# Test the basic MIMEText class +class TestMIMEText(unittest.TestCase): + def setUp(self): + self._msg = MIMEText('hello there') + + def test_types(self): + eq = self.assertEqual + eq(self._msg.get_content_type(), 'text/plain') + eq(self._msg.get_param('charset'), 'us-ascii') + missing = [] + self.assertIs(self._msg.get_param('foobar', missing), missing) + self.assertIs(self._msg.get_param('charset', missing, header='foobar'), + missing) + + def test_payload(self): + self.assertEqual(self._msg.get_payload(), 'hello there') + self.assertFalse(self._msg.is_multipart()) + + def test_charset(self): + eq = self.assertEqual + msg = MIMEText('hello there', _charset='us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + # Also accept a Charset instance + charset = Charset('utf-8') + charset.body_encoding = None + msg = MIMEText('hello there', _charset=charset) + eq(msg.get_charset().input_charset, 'utf-8') + eq(msg['content-type'], 'text/plain; charset="utf-8"') + eq(msg.get_payload(), 'hello there') + + def test_7bit_input(self): + eq = self.assertEqual + msg = MIMEText('hello there', _charset='us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + + def test_7bit_input_no_charset(self): + eq = self.assertEqual + msg = MIMEText('hello there') + eq(msg.get_charset(), 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + self.assertIn('hello there', msg.as_string()) + + def test_utf8_input(self): + teststr = '\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' + eq = self.assertEqual + msg = MIMEText(teststr, _charset='utf-8') + eq(msg.get_charset().output_charset, 'utf-8') + eq(msg['content-type'], 'text/plain; charset="utf-8"') + eq(msg.get_payload(decode=True), teststr.encode('utf-8')) + + @unittest.skip("can't fix because of backward compat in email5, " + "will fix in email6") + def test_utf8_input_no_charset(self): + teststr = '\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' + self.assertRaises(UnicodeEncodeError, MIMEText, teststr) + + + +# Test complicated multipart/* messages +class TestMultipart(TestEmailBase): + def setUp(self): + with openfile('python.gif', 'rb') as fp: + data = fp.read() + container = MIMEBase('multipart', 'mixed', boundary='BOUNDARY') + image = MIMEImage(data, name='dingusfish.gif') + image.add_header('content-disposition', 'attachment', + filename='dingusfish.gif') + intro = MIMEText('''\ +Hi there, + +This is the dingus fish. +''') + container.attach(intro) + container.attach(image) + container['From'] = 'Barry ' + container['To'] = 'Dingus Lovers ' + container['Subject'] = 'Here is your dingus fish' + + now = 987809702.54848599 + timetuple = time.localtime(now) + if timetuple[-1] == 0: + tzsecs = time.timezone + else: + tzsecs = time.altzone + if tzsecs > 0: + sign = '-' + else: + sign = '+' + tzoffset = ' %s%04d' % (sign, tzsecs / 36) + container['Date'] = time.strftime( + '%a, %d %b %Y %H:%M:%S', + time.localtime(now)) + tzoffset + self._msg = container + self._im = image + self._txt = intro + + def test_hierarchy(self): + # convenience + eq = self.assertEqual + raises = self.assertRaises + # tests + m = self._msg + self.assertTrue(m.is_multipart()) + eq(m.get_content_type(), 'multipart/mixed') + eq(len(m.get_payload()), 2) + raises(IndexError, m.get_payload, 2) + m0 = m.get_payload(0) + m1 = m.get_payload(1) + self.assertIs(m0, self._txt) + self.assertIs(m1, self._im) + eq(m.get_payload(), [m0, m1]) + self.assertFalse(m0.is_multipart()) + self.assertFalse(m1.is_multipart()) + + def test_empty_multipart_idempotent(self): + text = """\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY + + +--BOUNDARY-- +""" + msg = Parser().parsestr(text) + self.ndiffAssertEqual(text, msg.as_string()) + + def test_no_parts_in_a_multipart_with_none_epilogue(self): + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.set_boundary('BOUNDARY') + self.ndiffAssertEqual(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY + +--BOUNDARY-- +''') + + def test_no_parts_in_a_multipart_with_empty_epilogue(self): + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = '' + outer.epilogue = '' + outer.set_boundary('BOUNDARY') + self.ndiffAssertEqual(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY + +--BOUNDARY-- +''') + + def test_one_part_in_a_multipart(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.set_boundary('BOUNDARY') + msg = MIMEText('hello world') + outer.attach(msg) + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + def test_seq_parts_in_a_multipart_with_empty_preamble(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = '' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + + def test_seq_parts_in_a_multipart_with_none_preamble(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = None + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + + def test_seq_parts_in_a_multipart_with_none_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = None + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + + def test_seq_parts_in_a_multipart_with_empty_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = '' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + + def test_seq_parts_in_a_multipart_with_nl_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = '\n' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- + +''') + + def test_message_external_body(self): + eq = self.assertEqual + msg = self._msgobj('msg_36.txt') + eq(len(msg.get_payload()), 2) + msg1 = msg.get_payload(1) + eq(msg1.get_content_type(), 'multipart/alternative') + eq(len(msg1.get_payload()), 2) + for subpart in msg1.get_payload(): + eq(subpart.get_content_type(), 'message/external-body') + eq(len(subpart.get_payload()), 1) + subsubpart = subpart.get_payload(0) + eq(subsubpart.get_content_type(), 'text/plain') + + def test_double_boundary(self): + # msg_37.txt is a multipart that contains two dash-boundary's in a + # row. Our interpretation of RFC 2046 calls for ignoring the second + # and subsequent boundaries. + msg = self._msgobj('msg_37.txt') + self.assertEqual(len(msg.get_payload()), 3) + + def test_nested_inner_contains_outer_boundary(self): + eq = self.ndiffAssertEqual + # msg_38.txt has an inner part that contains outer boundaries. My + # interpretation of RFC 2046 (based on sections 5.1 and 5.1.2) say + # these are illegal and should be interpreted as unterminated inner + # parts. + msg = self._msgobj('msg_38.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/mixed + multipart/mixed + multipart/alternative + text/plain + text/plain + text/plain + text/plain +""") + + def test_nested_with_same_boundary(self): + eq = self.ndiffAssertEqual + # msg 39.txt is similarly evil in that it's got inner parts that use + # the same boundary as outer parts. Again, I believe the way this is + # parsed is closest to the spirit of RFC 2046 + msg = self._msgobj('msg_39.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/mixed + multipart/mixed + multipart/alternative + application/octet-stream + application/octet-stream + text/plain +""") + + def test_boundary_in_non_multipart(self): + msg = self._msgobj('msg_40.txt') + self.assertEqual(msg.as_string(), '''\ +MIME-Version: 1.0 +Content-Type: text/html; boundary="--961284236552522269" + +----961284236552522269 +Content-Type: text/html; +Content-Transfer-Encoding: 7Bit + + + +----961284236552522269-- +''') + + def test_boundary_with_leading_space(self): + eq = self.assertEqual + msg = email.message_from_string('''\ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=" XXXX" + +-- XXXX +Content-Type: text/plain + + +-- XXXX +Content-Type: text/plain + +-- XXXX-- +''') + self.assertTrue(msg.is_multipart()) + eq(msg.get_boundary(), ' XXXX') + eq(len(msg.get_payload()), 2) + + def test_boundary_without_trailing_newline(self): + m = Parser().parsestr("""\ +Content-Type: multipart/mixed; boundary="===============0012394164==" +MIME-Version: 1.0 + +--===============0012394164== +Content-Type: image/file1.jpg +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 + +YXNkZg== +--===============0012394164==--""") + self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') + + def test_mimebase_default_policy(self): + m = MIMEBase('multipart', 'mixed') + self.assertIs(m.policy, email.policy.compat32) + + def test_mimebase_custom_policy(self): + m = MIMEBase('multipart', 'mixed', policy=email.policy.default) + self.assertIs(m.policy, email.policy.default) + +# Test some badly formatted messages +class TestNonConformant(TestEmailBase): + + def test_parse_missing_minor_type(self): + eq = self.assertEqual + msg = self._msgobj('msg_14.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + + # test_defect_handling + def test_same_boundary_inner_outer(self): + msg = self._msgobj('msg_15.txt') + # XXX We can probably eventually do better + inner = msg.get_payload(0) + self.assertHasAttr(inner, 'defects') + self.assertEqual(len(inner.defects), 1) + self.assertIsInstance(inner.defects[0], + errors.StartBoundaryNotFoundDefect) + + # test_defect_handling + def test_multipart_no_boundary(self): + msg = self._msgobj('msg_25.txt') + self.assertIsInstance(msg.get_payload(), str) + self.assertEqual(len(msg.defects), 2) + self.assertIsInstance(msg.defects[0], + errors.NoBoundaryInMultipartDefect) + self.assertIsInstance(msg.defects[1], + errors.MultipartInvariantViolationDefect) + + multipart_msg = textwrap.dedent("""\ + Date: Wed, 14 Nov 2007 12:56:23 GMT + From: foo@bar.invalid + To: foo@bar.invalid + Subject: Content-Transfer-Encoding: base64 and multipart + MIME-Version: 1.0 + Content-Type: multipart/mixed; + boundary="===============3344438784458119861=="{} + + --===============3344438784458119861== + Content-Type: text/plain + + Test message + + --===============3344438784458119861== + Content-Type: application/octet-stream + Content-Transfer-Encoding: base64 + + YWJj + + --===============3344438784458119861==-- + """) + + # test_defect_handling + def test_multipart_invalid_cte(self): + msg = self._str_msg( + self.multipart_msg.format("\nContent-Transfer-Encoding: base64")) + self.assertEqual(len(msg.defects), 1) + self.assertIsInstance(msg.defects[0], + errors.InvalidMultipartContentTransferEncodingDefect) + + # test_defect_handling + def test_multipart_no_cte_no_defect(self): + msg = self._str_msg(self.multipart_msg.format('')) + self.assertEqual(len(msg.defects), 0) + + # test_defect_handling + def test_multipart_valid_cte_no_defect(self): + for cte in ('7bit', '8bit', 'BINary'): + msg = self._str_msg( + self.multipart_msg.format( + "\nContent-Transfer-Encoding: {}".format(cte))) + self.assertEqual(len(msg.defects), 0) + + # test_headerregistry.TestContentTypeHeader invalid_1 and invalid_2. + def test_invalid_content_type(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + msg = Message() + # RFC 2045, $5.2 says invalid yields text/plain + msg['Content-Type'] = 'text' + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_content_type(), 'text/plain') + # Clear the old value and try something /really/ invalid + del msg['content-type'] + msg['Content-Type'] = 'foo' + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_content_type(), 'text/plain') + # Still, make sure that the message is idempotently generated + s = StringIO() + g = Generator(s) + g.flatten(msg) + neq(s.getvalue(), 'Content-Type: foo\n\n') + + def test_no_start_boundary(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_31.txt') + eq(msg.get_payload(), """\ +--BOUNDARY +Content-Type: text/plain + +message 1 + +--BOUNDARY +Content-Type: text/plain + +message 2 + +--BOUNDARY-- +""") + + def test_no_separating_blank_line(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_35.txt') + eq(msg.as_string(), """\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: here's something interesting + +counter to RFC 5322, there's no separating newline here +""") + + # test_defect_handling + def test_lying_multipart(self): + msg = self._msgobj('msg_41.txt') + self.assertHasAttr(msg, 'defects') + self.assertEqual(len(msg.defects), 2) + self.assertIsInstance(msg.defects[0], + errors.NoBoundaryInMultipartDefect) + self.assertIsInstance(msg.defects[1], + errors.MultipartInvariantViolationDefect) + + # test_defect_handling + def test_missing_start_boundary(self): + outer = self._msgobj('msg_42.txt') + # The message structure is: + # + # multipart/mixed + # text/plain + # message/rfc822 + # multipart/mixed [*] + # + # [*] This message is missing its start boundary + bad = outer.get_payload(1).get_payload(0) + self.assertEqual(len(bad.defects), 1) + self.assertIsInstance(bad.defects[0], + errors.StartBoundaryNotFoundDefect) + + # test_defect_handling + def test_first_line_is_continuation_header(self): + eq = self.assertEqual + m = ' Line 1\nSubject: test\n\nbody' + msg = email.message_from_string(m) + eq(msg.keys(), ['Subject']) + eq(msg.get_payload(), 'body') + eq(len(msg.defects), 1) + self.assertDefectsEqual(msg.defects, + [errors.FirstHeaderLineIsContinuationDefect]) + eq(msg.defects[0].line, ' Line 1\n') + + # test_defect_handling + def test_missing_header_body_separator(self): + # Our heuristic if we see a line that doesn't look like a header (no + # leading whitespace but no ':') is to assume that the blank line that + # separates the header from the body is missing, and to stop parsing + # headers and start parsing the body. + msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') + self.assertEqual(msg.keys(), ['Subject']) + self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') + self.assertDefectsEqual(msg.defects, + [errors.MissingHeaderBodySeparatorDefect]) + + def test_string_payload_with_extra_space_after_cte(self): + # https://github.com/python/cpython/issues/98188 + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + def test_string_payload_with_extra_text_after_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 some text + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + cte = msg['content-transfer-encoding'] + self.assertDefectsEqual(cte.defects, [email.errors.InvalidHeaderDefect]) + + def test_string_payload_with_extra_space_after_cte_compat32(self): + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.compat32) + pasted_cte = msg['content-transfer-encoding'] + self.assertEqual(pasted_cte, cte) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg.defects, []) + + + +# Test RFC 2047 header encoding and decoding +class TestRFC2047(TestEmailBase): + def test_rfc2047_multiline(self): + eq = self.assertEqual + s = """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz + foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""" + dh = decode_header(s) + eq(dh, [ + (b'Re: ', None), + (b'r\x8aksm\x9arg\x8cs', 'mac-iceland'), + (b' baz foo bar ', None), + (b'r\x8aksm\x9arg\x8cs', 'mac-iceland')]) + header = make_header(dh) + eq(str(header), + 'Re: r\xe4ksm\xf6rg\xe5s baz foo bar r\xe4ksm\xf6rg\xe5s') + self.ndiffAssertEqual(header.encode(maxlinelen=76), """\ +Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?= + =?mac-iceland?q?=9Arg=8Cs?=""") + + def test_whitespace_keeper_unicode(self): + eq = self.assertEqual + s = '=?ISO-8859-1?Q?Andr=E9?= Pirard ' + dh = decode_header(s) + eq(dh, [(b'Andr\xe9', 'iso-8859-1'), + (b' Pirard ', None)]) + header = str(make_header(dh)) + eq(header, 'Andr\xe9 Pirard ') + + def test_whitespace_keeper_unicode_2(self): + eq = self.assertEqual + s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?=' + dh = decode_header(s) + eq(dh, [(b'The ', None), (b'quick brown fox', 'iso-8859-1'), + (b' jumped over the ', None), (b'lazy dog', 'iso-8859-1')]) + hu = str(make_header(dh)) + eq(hu, 'The quick brown fox jumped over the lazy dog') + + def test_rfc2047_missing_whitespace(self): + s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord' + dh = decode_header(s) + self.assertEqual(dh, [(b'Sm', None), (b'\xf6', 'iso-8859-1'), + (b'rg', None), (b'\xe5', 'iso-8859-1'), + (b'sbord', None)]) + + def test_rfc2047_with_whitespace(self): + s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord' + dh = decode_header(s) + self.assertEqual(dh, [(b'Sm ', None), (b'\xf6', 'iso-8859-1'), + (b' rg ', None), (b'\xe5', 'iso-8859-1'), + (b' sbord', None)]) + + def test_rfc2047_B_bad_padding(self): + s = '=?iso-8859-1?B?%s?=' + data = [ # only test complete bytes + ('dm==', b'v'), ('dm=', b'v'), ('dm', b'v'), + ('dmk=', b'vi'), ('dmk', b'vi') + ] + for q, a in data: + dh = decode_header(s % q) + self.assertEqual(dh, [(a, 'iso-8859-1')]) + + def test_rfc2047_Q_invalid_digits(self): + # issue 10004. + s = '=?iso-8859-1?Q?andr=e9=zz?=' + self.assertEqual(decode_header(s), + [(b'andr\xe9=zz', 'iso-8859-1')]) + + def test_rfc2047_rfc2047_1(self): + # 1st testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_2(self): + # 2nd testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a?= b)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)]) + + def test_rfc2047_rfc2047_3(self): + # 3rd testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_4(self): + # 4th testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_5a(self): + # 5th testcase at end of RFC 2047 newline is \r\n + s = '(=?ISO-8859-1?Q?a?=\r\n =?ISO-8859-1?Q?b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_5b(self): + # 5th testcase at end of RFC 2047 newline is \n + s = '(=?ISO-8859-1?Q?a?=\n =?ISO-8859-1?Q?b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_6(self): + # 6th testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a_b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)]) + + def test_rfc2047_rfc2047_7(self): + # 7th testcase at end of RFC 2047 + s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)' + self.assertEqual(decode_header(s), + [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'), + (b')', None)]) + self.assertEqual(make_header(decode_header(s)).encode(), s.lower()) + self.assertEqual(str(make_header(decode_header(s))), '(a b)') + + def test_multiline_header(self): + s = '=?windows-1252?q?=22M=FCller_T=22?=\r\n ' + self.assertEqual(decode_header(s), + [(b'"M\xfcller T"', 'windows-1252'), + (b'', None)]) + self.assertEqual(make_header(decode_header(s)).encode(), + ''.join(s.splitlines())) + self.assertEqual(str(make_header(decode_header(s))), + '"Müller T" ') + + def test_unencoded_ascii(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header without encoded words' + self.assertEqual(decode_header(s), + [('header without encoded words', None)]) + + def test_unencoded_utf8(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header with unexpected non ASCII caract\xe8res' + self.assertEqual(decode_header(s), + [('header with unexpected non ASCII caract\xe8res', None)]) + + +# Test the MIMEMessage class +class TestMIMEMessage(TestEmailBase): + def setUp(self): + with openfile('msg_11.txt', encoding="utf-8") as fp: + self._text = fp.read() + + def test_type_error(self): + self.assertRaises(TypeError, MIMEMessage, 'a plain string') + + def test_valid_argument(self): + eq = self.assertEqual + subject = 'A sub-message' + m = Message() + m['Subject'] = subject + r = MIMEMessage(m) + eq(r.get_content_type(), 'message/rfc822') + payload = r.get_payload() + self.assertIsInstance(payload, list) + eq(len(payload), 1) + subpart = payload[0] + self.assertIs(subpart, m) + eq(subpart['subject'], subject) + + def test_bad_multipart(self): + msg1 = Message() + msg1['Subject'] = 'subpart 1' + msg2 = Message() + msg2['Subject'] = 'subpart 2' + r = MIMEMessage(msg1) + self.assertRaises(errors.MultipartConversionError, r.attach, msg2) + + def test_generate(self): + # First craft the message to be encapsulated + m = Message() + m['Subject'] = 'An enclosed message' + m.set_payload('Here is the body of the message.\n') + r = MIMEMessage(m) + r['Subject'] = 'The enclosing message' + s = StringIO() + g = Generator(s) + g.flatten(r) + self.assertEqual(s.getvalue(), """\ +Content-Type: message/rfc822 +MIME-Version: 1.0 +Subject: The enclosing message + +Subject: An enclosed message + +Here is the body of the message. +""") + + def test_parse_message_rfc822(self): + eq = self.assertEqual + msg = self._msgobj('msg_11.txt') + eq(msg.get_content_type(), 'message/rfc822') + payload = msg.get_payload() + self.assertIsInstance(payload, list) + eq(len(payload), 1) + submsg = payload[0] + self.assertIsInstance(submsg, Message) + eq(submsg['subject'], 'An enclosed message') + eq(submsg.get_payload(), 'Here is the body of the message.\n') + + def test_dsn(self): + eq = self.assertEqual + # msg 16 is a Delivery Status Notification, see RFC 1894 + msg = self._msgobj('msg_16.txt') + eq(msg.get_content_type(), 'multipart/report') + self.assertTrue(msg.is_multipart()) + eq(len(msg.get_payload()), 3) + # Subpart 1 is a text/plain, human readable section + subpart = msg.get_payload(0) + eq(subpart.get_content_type(), 'text/plain') + eq(subpart.get_payload(), """\ +This report relates to a message you sent with the following header fields: + + Message-id: <002001c144a6$8752e060$56104586@oxy.edu> + Date: Sun, 23 Sep 2001 20:10:55 -0700 + From: "Ian T. Henry" + To: SoCal Raves + Subject: [scr] yeah for Ians!! + +Your message cannot be delivered to the following recipients: + + Recipient address: jangel1@cougar.noc.ucla.edu + Reason: recipient reached disk quota + +""") + # Subpart 2 contains the machine parsable DSN information. It + # consists of two blocks of headers, represented by two nested Message + # objects. + subpart = msg.get_payload(1) + eq(subpart.get_content_type(), 'message/delivery-status') + eq(len(subpart.get_payload()), 2) + # message/delivery-status should treat each block as a bunch of + # headers, i.e. a bunch of Message objects. + dsn1 = subpart.get_payload(0) + self.assertIsInstance(dsn1, Message) + eq(dsn1['original-envelope-id'], '0GK500B4HD0888@cougar.noc.ucla.edu') + eq(dsn1.get_param('dns', header='reporting-mta'), '') + # Try a missing one + eq(dsn1.get_param('nsd', header='reporting-mta'), None) + dsn2 = subpart.get_payload(1) + self.assertIsInstance(dsn2, Message) + eq(dsn2['action'], 'failed') + eq(dsn2.get_params(header='original-recipient'), + [('rfc822', ''), ('jangel1@cougar.noc.ucla.edu', '')]) + eq(dsn2.get_param('rfc822', header='final-recipient'), '') + # Subpart 3 is the original message + subpart = msg.get_payload(2) + eq(subpart.get_content_type(), 'message/rfc822') + payload = subpart.get_payload() + self.assertIsInstance(payload, list) + eq(len(payload), 1) + subsubpart = payload[0] + self.assertIsInstance(subsubpart, Message) + eq(subsubpart.get_content_type(), 'text/plain') + eq(subsubpart['message-id'], + '<002001c144a6$8752e060$56104586@oxy.edu>') + + def test_epilogue(self): + eq = self.ndiffAssertEqual + with openfile('msg_21.txt', encoding="utf-8") as fp: + text = fp.read() + msg = Message() + msg['From'] = 'aperson@dom.ain' + msg['To'] = 'bperson@dom.ain' + msg['Subject'] = 'Test' + msg.preamble = 'MIME message' + msg.epilogue = 'End of MIME message\n' + msg1 = MIMEText('One') + msg2 = MIMEText('Two') + msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') + msg.attach(msg1) + msg.attach(msg2) + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), text) + + def test_no_nl_preamble(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'aperson@dom.ain' + msg['To'] = 'bperson@dom.ain' + msg['Subject'] = 'Test' + msg.preamble = 'MIME message' + msg.epilogue = '' + msg1 = MIMEText('One') + msg2 = MIMEText('Two') + msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') + msg.attach(msg1) + msg.attach(msg2) + eq(msg.as_string(), """\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: Test +Content-Type: multipart/mixed; boundary="BOUNDARY" + +MIME message +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +One +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Two +--BOUNDARY-- +""") + + def test_default_type(self): + eq = self.assertEqual + with openfile('msg_30.txt', encoding="utf-8") as fp: + msg = email.message_from_file(fp) + container1 = msg.get_payload(0) + eq(container1.get_default_type(), 'message/rfc822') + eq(container1.get_content_type(), 'message/rfc822') + container2 = msg.get_payload(1) + eq(container2.get_default_type(), 'message/rfc822') + eq(container2.get_content_type(), 'message/rfc822') + container1a = container1.get_payload(0) + eq(container1a.get_default_type(), 'text/plain') + eq(container1a.get_content_type(), 'text/plain') + container2a = container2.get_payload(0) + eq(container2a.get_default_type(), 'text/plain') + eq(container2a.get_content_type(), 'text/plain') + + def test_default_type_with_explicit_container_type(self): + eq = self.assertEqual + with openfile('msg_28.txt', encoding="utf-8") as fp: + msg = email.message_from_file(fp) + container1 = msg.get_payload(0) + eq(container1.get_default_type(), 'message/rfc822') + eq(container1.get_content_type(), 'message/rfc822') + container2 = msg.get_payload(1) + eq(container2.get_default_type(), 'message/rfc822') + eq(container2.get_content_type(), 'message/rfc822') + container1a = container1.get_payload(0) + eq(container1a.get_default_type(), 'text/plain') + eq(container1a.get_content_type(), 'text/plain') + container2a = container2.get_payload(0) + eq(container2a.get_default_type(), 'text/plain') + eq(container2a.get_content_type(), 'text/plain') + + def test_default_type_non_parsed(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + # Set up container + container = MIMEMultipart('digest', 'BOUNDARY') + container.epilogue = '' + # Set up subparts + subpart1a = MIMEText('message 1\n') + subpart2a = MIMEText('message 2\n') + subpart1 = MIMEMessage(subpart1a) + subpart2 = MIMEMessage(subpart2a) + container.attach(subpart1) + container.attach(subpart2) + eq(subpart1.get_content_type(), 'message/rfc822') + eq(subpart1.get_default_type(), 'message/rfc822') + eq(subpart2.get_content_type(), 'message/rfc822') + eq(subpart2.get_default_type(), 'message/rfc822') + neq(container.as_string(0), '''\ +Content-Type: multipart/digest; boundary="BOUNDARY" +MIME-Version: 1.0 + +--BOUNDARY +Content-Type: message/rfc822 +MIME-Version: 1.0 + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 1 + +--BOUNDARY +Content-Type: message/rfc822 +MIME-Version: 1.0 + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 2 + +--BOUNDARY-- +''') + del subpart1['content-type'] + del subpart1['mime-version'] + del subpart2['content-type'] + del subpart2['mime-version'] + eq(subpart1.get_content_type(), 'message/rfc822') + eq(subpart1.get_default_type(), 'message/rfc822') + eq(subpart2.get_content_type(), 'message/rfc822') + eq(subpart2.get_default_type(), 'message/rfc822') + neq(container.as_string(0), '''\ +Content-Type: multipart/digest; boundary="BOUNDARY" +MIME-Version: 1.0 + +--BOUNDARY + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 1 + +--BOUNDARY + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 2 + +--BOUNDARY-- +''') + + def test_mime_attachments_in_constructor(self): + eq = self.assertEqual + text1 = MIMEText('') + text2 = MIMEText('') + msg = MIMEMultipart(_subparts=(text1, text2)) + eq(len(msg.get_payload()), 2) + eq(msg.get_payload(0), text1) + eq(msg.get_payload(1), text2) + + def test_default_multipart_constructor(self): + msg = MIMEMultipart() + self.assertTrue(msg.is_multipart()) + + def test_multipart_default_policy(self): + msg = MIMEMultipart() + msg['To'] = 'a@b.com' + msg['To'] = 'c@d.com' + self.assertEqual(msg.get_all('to'), ['a@b.com', 'c@d.com']) + + def test_multipart_custom_policy(self): + msg = MIMEMultipart(policy=email.policy.default) + msg['To'] = 'a@b.com' + with self.assertRaises(ValueError) as cm: + msg['To'] = 'c@d.com' + self.assertEqual(str(cm.exception), + 'There may be at most 1 To headers in a message') + + +# Test the NonMultipart class +class TestNonMultipart(TestEmailBase): + def test_nonmultipart_is_not_multipart(self): + msg = MIMENonMultipart('text', 'plain') + self.assertFalse(msg.is_multipart()) + + def test_attach_raises_exception(self): + msg = Message() + msg['Subject'] = 'subpart 1' + r = MIMENonMultipart('text', 'plain') + self.assertRaises(errors.MultipartConversionError, r.attach, msg) + + +# A general test of parser->model->generator idempotency. IOW, read a message +# in, parse it into a message object tree, then without touching the tree, +# regenerate the plain text. The original text and the transformed text +# should be identical. Note: that we ignore the Unix-From since that may +# contain a changed date. +class TestIdempotent(TestEmailBase): + + linesep = '\n' + + def _msgobj(self, filename): + with openfile(filename, encoding="utf-8") as fp: + data = fp.read() + msg = email.message_from_string(data) + return msg, data + + def _idempotent(self, msg, text, unixfrom=False): + eq = self.ndiffAssertEqual + s = StringIO() + g = Generator(s, maxheaderlen=0) + g.flatten(msg, unixfrom=unixfrom) + eq(text, s.getvalue()) + + def test_parse_text_message(self): + eq = self.assertEqual + msg, text = self._msgobj('msg_01.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_params()[1], ('charset', 'us-ascii')) + eq(msg.get_param('charset'), 'us-ascii') + eq(msg.preamble, None) + eq(msg.epilogue, None) + self._idempotent(msg, text) + + def test_parse_untyped_message(self): + eq = self.assertEqual + msg, text = self._msgobj('msg_03.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_params(), None) + eq(msg.get_param('charset'), None) + self._idempotent(msg, text) + + def test_simple_multipart(self): + msg, text = self._msgobj('msg_04.txt') + self._idempotent(msg, text) + + def test_MIME_digest(self): + msg, text = self._msgobj('msg_02.txt') + self._idempotent(msg, text) + + def test_long_header(self): + msg, text = self._msgobj('msg_27.txt') + self._idempotent(msg, text) + + def test_MIME_digest_with_part_headers(self): + msg, text = self._msgobj('msg_28.txt') + self._idempotent(msg, text) + + def test_mixed_with_image(self): + msg, text = self._msgobj('msg_06.txt') + self._idempotent(msg, text) + + def test_multipart_report(self): + msg, text = self._msgobj('msg_05.txt') + self._idempotent(msg, text) + + def test_dsn(self): + msg, text = self._msgobj('msg_16.txt') + self._idempotent(msg, text) + + def test_preamble_epilogue(self): + msg, text = self._msgobj('msg_21.txt') + self._idempotent(msg, text) + + def test_multipart_one_part(self): + msg, text = self._msgobj('msg_23.txt') + self._idempotent(msg, text) + + def test_multipart_no_parts(self): + msg, text = self._msgobj('msg_24.txt') + self._idempotent(msg, text) + + def test_no_start_boundary(self): + msg, text = self._msgobj('msg_31.txt') + self._idempotent(msg, text) + + def test_rfc2231_charset(self): + msg, text = self._msgobj('msg_32.txt') + self._idempotent(msg, text) + + def test_more_rfc2231_parameters(self): + msg, text = self._msgobj('msg_33.txt') + self._idempotent(msg, text) + + def test_text_plain_in_a_multipart_digest(self): + msg, text = self._msgobj('msg_34.txt') + self._idempotent(msg, text) + + def test_nested_multipart_mixeds(self): + msg, text = self._msgobj('msg_12a.txt') + self._idempotent(msg, text) + + def test_message_external_body_idempotent(self): + msg, text = self._msgobj('msg_36.txt') + self._idempotent(msg, text) + + def test_message_delivery_status(self): + msg, text = self._msgobj('msg_43.txt') + self._idempotent(msg, text, unixfrom=True) + + def test_message_signed_idempotent(self): + msg, text = self._msgobj('msg_45.txt') + self._idempotent(msg, text) + + def test_content_type(self): + eq = self.assertEqual + # Get a message object and reset the seek pointer for other tests + msg, text = self._msgobj('msg_05.txt') + eq(msg.get_content_type(), 'multipart/report') + # Test the Content-Type: parameters + params = {} + for pk, pv in msg.get_params(): + params[pk] = pv + eq(params['report-type'], 'delivery-status') + eq(params['boundary'], 'D1690A7AC1.996856090/mail.example.com') + eq(msg.preamble, 'This is a MIME-encapsulated message.' + self.linesep) + eq(msg.epilogue, self.linesep) + eq(len(msg.get_payload()), 3) + # Make sure the subparts are what we expect + msg1 = msg.get_payload(0) + eq(msg1.get_content_type(), 'text/plain') + eq(msg1.get_payload(), 'Yadda yadda yadda' + self.linesep) + msg2 = msg.get_payload(1) + eq(msg2.get_content_type(), 'text/plain') + eq(msg2.get_payload(), 'Yadda yadda yadda' + self.linesep) + msg3 = msg.get_payload(2) + eq(msg3.get_content_type(), 'message/rfc822') + self.assertIsInstance(msg3, Message) + payload = msg3.get_payload() + self.assertIsInstance(payload, list) + eq(len(payload), 1) + msg4 = payload[0] + self.assertIsInstance(msg4, Message) + eq(msg4.get_payload(), 'Yadda yadda yadda' + self.linesep) + + def test_parser(self): + eq = self.assertEqual + msg, text = self._msgobj('msg_06.txt') + # Check some of the outer headers + eq(msg.get_content_type(), 'message/rfc822') + # Make sure the payload is a list of exactly one sub-Message, and that + # that submessage has a type of text/plain + payload = msg.get_payload() + self.assertIsInstance(payload, list) + eq(len(payload), 1) + msg1 = payload[0] + self.assertIsInstance(msg1, Message) + eq(msg1.get_content_type(), 'text/plain') + self.assertIsInstance(msg1.get_payload(), str) + eq(msg1.get_payload(), self.linesep) + + + +# Test various other bits of the package's functionality +class TestMiscellaneous(TestEmailBase): + def test_message_from_string(self): + with openfile('msg_01.txt', encoding="utf-8") as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + # Don't wrap/continue long headers since we're trying to test + # idempotency. + g = Generator(s, maxheaderlen=0) + g.flatten(msg) + self.assertEqual(text, s.getvalue()) + + def test_message_from_file(self): + with openfile('msg_01.txt', encoding="utf-8") as fp: + text = fp.read() + fp.seek(0) + msg = email.message_from_file(fp) + s = StringIO() + # Don't wrap/continue long headers since we're trying to test + # idempotency. + g = Generator(s, maxheaderlen=0) + g.flatten(msg) + self.assertEqual(text, s.getvalue()) + + def test_message_from_string_with_class(self): + with openfile('msg_01.txt', encoding="utf-8") as fp: + text = fp.read() + + # Create a subclass + class MyMessage(Message): + pass + + msg = email.message_from_string(text, MyMessage) + self.assertIsInstance(msg, MyMessage) + # Try something more complicated + with openfile('msg_02.txt', encoding="utf-8") as fp: + text = fp.read() + msg = email.message_from_string(text, MyMessage) + for subpart in msg.walk(): + self.assertIsInstance(subpart, MyMessage) + + def test_message_from_file_with_class(self): + # Create a subclass + class MyMessage(Message): + pass + + with openfile('msg_01.txt', encoding="utf-8") as fp: + msg = email.message_from_file(fp, MyMessage) + self.assertIsInstance(msg, MyMessage) + # Try something more complicated + with openfile('msg_02.txt', encoding="utf-8") as fp: + msg = email.message_from_file(fp, MyMessage) + for subpart in msg.walk(): + self.assertIsInstance(subpart, MyMessage) + + def test_custom_message_does_not_require_arguments(self): + class MyMessage(Message): + def __init__(self): + super().__init__() + msg = self._str_msg("Subject: test\n\ntest", MyMessage) + self.assertIsInstance(msg, MyMessage) + + def test__all__(self): + module = __import__('email') + self.assertEqual(sorted(module.__all__), [ + 'base64mime', 'charset', 'encoders', 'errors', 'feedparser', + 'generator', 'header', 'iterators', 'message', + 'message_from_binary_file', 'message_from_bytes', + 'message_from_file', 'message_from_string', 'mime', 'parser', + 'quoprimime', 'utils', + ]) + + def test_formatdate(self): + now = time.time() + self.assertEqual(utils.parsedate(utils.formatdate(now))[:6], + time.gmtime(now)[:6]) + + def test_formatdate_localtime(self): + now = time.time() + self.assertEqual( + utils.parsedate(utils.formatdate(now, localtime=True))[:6], + time.localtime(now)[:6]) + + def test_formatdate_usegmt(self): + now = time.time() + self.assertEqual( + utils.formatdate(now, localtime=False), + time.strftime('%a, %d %b %Y %H:%M:%S -0000', time.gmtime(now))) + self.assertEqual( + utils.formatdate(now, localtime=False, usegmt=True), + time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now))) + + # parsedate and parsedate_tz will become deprecated interfaces someday + def test_parsedate_returns_None_for_invalid_strings(self): + # See also test_parsedate_to_datetime_with_invalid_raises_valueerror + # in test_utils. + invalid_dates = [ + '', + ' ', + '0', + 'A Complete Waste of Time', + 'Wed, 3 Apr 2002 12.34.56.78+0800', + '17 June , 2022', + 'Friday, -Nov-82 16:14:55 EST', + 'Friday, Nov--82 16:14:55 EST', + 'Friday, 19-Nov- 16:14:55 EST', + ] + for dtstr in invalid_dates: + with self.subTest(dtstr=dtstr): + self.assertIsNone(utils.parsedate(dtstr)) + self.assertIsNone(utils.parsedate_tz(dtstr)) + # Not a part of the spec but, but this has historically worked: + self.assertIsNone(utils.parsedate(None)) + self.assertIsNone(utils.parsedate_tz(None)) + + def test_parsedate_compact(self): + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58:26 +0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + # The FWS after the comma is optional + self.assertEqual(utils.parsedate_tz('Wed,3 Apr 2002 14:58:26 +0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + # The comma is optional + self.assertEqual(utils.parsedate_tz('Wed 3 Apr 2002 14:58:26 +0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + + def test_parsedate_no_dayofweek(self): + eq = self.assertEqual + eq(utils.parsedate_tz('5 Feb 2003 13:47:26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + eq(utils.parsedate_tz('February 5, 2003 13:47:26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + + def test_parsedate_no_space_before_positive_offset(self): + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58:26+0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + + def test_parsedate_no_space_before_negative_offset(self): + # Issue 1155362: we already handled '+' for this case. + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58:26-0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, -28800)) + + def test_parsedate_accepts_time_with_dots(self): + eq = self.assertEqual + eq(utils.parsedate_tz('5 Feb 2003 13.47.26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + eq(utils.parsedate_tz('5 Feb 2003 13.47 -0800'), + (2003, 2, 5, 13, 47, 0, 0, 1, -1, -28800)) + + def test_parsedate_rfc_850(self): + self.assertEqual(utils.parsedate_tz('Friday, 19-Nov-82 16:14:55 EST'), + (1982, 11, 19, 16, 14, 55, 0, 1, -1, -18000)) + + def test_parsedate_no_seconds(self): + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58 +0800'), + (2002, 4, 3, 14, 58, 0, 0, 1, -1, 28800)) + + def test_parsedate_dot_time_delimiter(self): + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14.58.26 +0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14.58 +0800'), + (2002, 4, 3, 14, 58, 0, 0, 1, -1, 28800)) + + def test_parsedate_acceptable_to_time_functions(self): + eq = self.assertEqual + timetup = utils.parsedate('5 Feb 2003 13:47:26 -0800') + t = int(time.mktime(timetup)) + eq(time.localtime(t)[:6], timetup[:6]) + eq(int(time.strftime('%Y', timetup)), 2003) + timetup = utils.parsedate_tz('5 Feb 2003 13:47:26 -0800') + t = int(time.mktime(timetup[:9])) + eq(time.localtime(t)[:6], timetup[:6]) + eq(int(time.strftime('%Y', timetup[:9])), 2003) + + def test_mktime_tz(self): + self.assertEqual(utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 0)), 0) + self.assertEqual(utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 1234)), -1234) + + def test_parsedate_y2k(self): + """Test for parsing a date with a two-digit year. + + Parsing a date with a two-digit year should return the correct + four-digit year. RFC 822 allows two-digit years, but RFC 5322 (which + obsoletes RFC 2822, which obsoletes RFC 822) requires four-digit years. + + """ + self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), + utils.parsedate_tz('25 Feb 2003 13:47:26 -0800')) + self.assertEqual(utils.parsedate_tz('25 Feb 71 13:47:26 -0800'), + utils.parsedate_tz('25 Feb 1971 13:47:26 -0800')) + + def test_parseaddr_empty(self): + self.assertEqual(utils.parseaddr('<>'), ('', '')) + self.assertEqual(utils.formataddr(utils.parseaddr('<>')), '') + + def test_parseaddr_multiple_domains(self): + self.assertEqual( + utils.parseaddr('a@b@c'), + ('', '') + ) + self.assertEqual( + utils.parseaddr('a@b.c@c'), + ('', '') + ) + self.assertEqual( + utils.parseaddr('a@172.17.0.1@c'), + ('', '') + ) + + def test_noquote_dump(self): + self.assertEqual( + utils.formataddr(('A Silly Person', 'person@dom.ain')), + 'A Silly Person ') + + def test_escape_dump(self): + self.assertEqual( + utils.formataddr(('A (Very) Silly Person', 'person@dom.ain')), + r'"A (Very) Silly Person" ') + self.assertEqual( + utils.parseaddr(r'"A \(Very\) Silly Person" '), + ('A (Very) Silly Person', 'person@dom.ain')) + a = r'A \(Special\) Person' + b = 'person@dom.ain' + self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) + + def test_escape_backslashes(self): + self.assertEqual( + utils.formataddr((r'Arthur \Backslash\ Foobar', 'person@dom.ain')), + r'"Arthur \\Backslash\\ Foobar" ') + a = r'Arthur \Backslash\ Foobar' + b = 'person@dom.ain' + self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) + + def test_quotes_unicode_names(self): + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " + latin1_quopri = "=?iso-8859-1?q?H=E4ns_W=FCrst?= " + self.assertEqual(utils.formataddr((name, addr)), utf8_base64) + self.assertEqual(utils.formataddr((name, addr), 'iso-8859-1'), + latin1_quopri) + + def test_accepts_any_charset_like_object(self): + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " + foobar = "FOOBAR" + class CharsetMock: + def header_encode(self, string): + return foobar + mock = CharsetMock() + mock_expected = "%s <%s>" % (foobar, addr) + self.assertEqual(utils.formataddr((name, addr), mock), mock_expected) + self.assertEqual(utils.formataddr((name, addr), Charset('utf-8')), + utf8_base64) + + def test_invalid_charset_like_object_raises_error(self): + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + # An object without a header_encode method: + bad_charset = object() + self.assertRaises(AttributeError, utils.formataddr, (name, addr), + bad_charset) + + def test_unicode_address_raises_error(self): + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. + addr = 'pers\u00f6n@dom.in' + self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) + self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) + + def test_name_with_dot(self): + x = 'John X. Doe ' + y = '"John X. Doe" ' + a, b = ('John X. Doe', 'jxd@example.com') + self.assertEqual(utils.parseaddr(x), (a, b)) + self.assertEqual(utils.parseaddr(y), (a, b)) + # formataddr() quotes the name if there's a dot in it + self.assertEqual(utils.formataddr((a, b)), y) + + def test_parseaddr_preserves_quoted_pairs_in_addresses(self): + # issue 10005. Note that in the third test the second pair of + # backslashes is not actually a quoted pair because it is not inside a + # comment or quoted string: the address being parsed has a quoted + # string containing a quoted backslash, followed by 'example' and two + # backslashes, followed by another quoted string containing a space and + # the word 'example'. parseaddr copies those two backslashes + # literally. Per RFC 5322 this is not technically correct since a \ may + # not appear in an address outside of a quoted string. It is probably + # a sensible Postel interpretation, though. + eq = self.assertEqual + eq(utils.parseaddr('""example" example"@example.com'), + ('', '""example" example"@example.com')) + eq(utils.parseaddr('"\\"example\\" example"@example.com'), + ('', '"\\"example\\" example"@example.com')) + eq(utils.parseaddr('"\\\\"example\\\\" example"@example.com'), + ('', '"\\\\"example\\\\" example"@example.com')) + + def test_parseaddr_preserves_spaces_in_local_part(self): + # issue 9286. A normal RFC 5322 local part should not contain any + # folding white space, but legacy local parts can (they are a sequence + # of atoms, not dotatoms). On the other hand we strip whitespace from + # before the @ and around dots, on the assumption that the whitespace + # around the punctuation is a mistake in what would otherwise be + # an RFC 5322 local part. Leading whitespace is, usual, stripped as well. + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr("merwok wok@xample.com")) + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr("merwok wok@xample.com")) + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr(" merwok wok @xample.com")) + self.assertEqual(('', 'merwok"wok" wok@xample.com'), + utils.parseaddr('merwok"wok" wok@xample.com')) + self.assertEqual(('', 'merwok.wok.wok@xample.com'), + utils.parseaddr('merwok. wok . wok@xample.com')) + + def test_formataddr_does_not_quote_parens_in_quoted_string(self): + addr = ("'foo@example.com' (foo@example.com)", + 'foo@example.com') + addrstr = ('"\'foo@example.com\' ' + '(foo@example.com)" ') + self.assertEqual(utils.parseaddr(addrstr), addr) + self.assertEqual(utils.formataddr(addr), addrstr) + + + def test_multiline_from_comment(self): + x = """\ +Foo +\tBar """ + self.assertEqual(utils.parseaddr(x), ('Foo Bar', 'foo@example.com')) + + def test_quote_dump(self): + self.assertEqual( + utils.formataddr(('A Silly; Person', 'person@dom.ain')), + r'"A Silly; Person" ') + + def test_charset_richcomparisons(self): + eq = self.assertEqual + ne = self.assertNotEqual + cset1 = Charset() + cset2 = Charset() + eq(cset1, 'us-ascii') + eq(cset1, 'US-ASCII') + eq(cset1, 'Us-AsCiI') + eq('us-ascii', cset1) + eq('US-ASCII', cset1) + eq('Us-AsCiI', cset1) + ne(cset1, 'usascii') + ne(cset1, 'USASCII') + ne(cset1, 'UsAsCiI') + ne('usascii', cset1) + ne('USASCII', cset1) + ne('UsAsCiI', cset1) + eq(cset1, cset2) + eq(cset2, cset1) + + def test_getaddresses(self): + eq = self.assertEqual + eq(utils.getaddresses(['aperson@dom.ain (Al Person)', + 'Bud Person ']), + [('Al Person', 'aperson@dom.ain'), + ('Bud Person', 'bperson@dom.ain')]) + + def test_getaddresses_comma_in_name(self): + """GH-106669 regression test.""" + self.assertEqual( + utils.getaddresses( + [ + '"Bud, Person" ', + 'aperson@dom.ain (Al Person)', + '"Mariusz Felisiak" ', + ] + ), + [ + ('Bud, Person', 'bperson@dom.ain'), + ('Al Person', 'aperson@dom.ain'), + ('Mariusz Felisiak', 'to@example.com'), + ], + ) + + def test_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056""" + alice = 'alice@example.org' + bob = 'bob@example.com' + empty = ('', '') + + # Test utils.getaddresses() and utils.parseaddr() on malformed email + # addresses: default behavior (strict=True) rejects malformed address, + # and strict=False which tolerates malformed address. + for invalid_separator, expected_non_strict in ( + ('(', [(f'<{bob}>', alice)]), + (')', [('', alice), empty, ('', bob)]), + ('<', [('', alice), empty, ('', bob), empty]), + ('>', [('', alice), empty, ('', bob)]), + ('[', [('', f'{alice}[<{bob}>]')]), + (']', [('', alice), empty, ('', bob)]), + ('@', [empty, empty, ('', bob)]), + (';', [('', alice), empty, ('', bob)]), + (':', [('', alice), ('', bob)]), + ('.', [('', alice + '.'), ('', bob)]), + ('"', [('', alice), ('', f'<{bob}>')]), + ): + address = f'{alice}{invalid_separator}<{bob}>' + with self.subTest(address=address): + self.assertEqual(utils.getaddresses([address]), + [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + expected_non_strict) + + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Comma (',') is treated differently depending on strict parameter. + # Comma without quotes. + address = f'{alice},<{bob}>' + self.assertEqual(utils.getaddresses([address]), + [('', alice), ('', bob)]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', alice), ('', bob)]) + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Real name between quotes containing comma. + address = '"Alice, alice@example.org" ' + expected_strict = ('Alice, alice@example.org', 'bob@example.com') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Valid parenthesis in comments. + address = 'alice@example.org (Alice)' + expected_strict = ('Alice', 'alice@example.org') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Invalid parenthesis in comments. + address = 'alice@example.org )Alice(' + self.assertEqual(utils.getaddresses([address]), [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', 'alice@example.org'), ('', ''), ('', 'Alice')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Two addresses with quotes separated by comma. + address = '"Jane Doe" , "John Doe" ' + self.assertEqual(utils.getaddresses([address]), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Test email.utils.supports_strict_parsing attribute + self.assertEqual(email.utils.supports_strict_parsing, True) + + def test_getaddresses_nasty(self): + for addresses, expected in ( + (['"Sürname, Firstname" '], + [('Sürname, Firstname', 'to@example.com')]), + + (['foo: ;'], + [('', '')]), + + (['foo: ;', '"Jason R. Mastaler" '], + [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]), + + ([r'Pete(A nice \) chap) '], + [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]), + + (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'], + [('', '')]), + + (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'], + [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]), + + (['John Doe '], + [('John Doe (comment)', 'jdoe@machine.example')]), + + (['"Mary Smith: Personal Account" '], + [('Mary Smith: Personal Account', 'smith@home.example')]), + + (['Undisclosed recipients:;'], + [('', '')]), + + ([r', "Giant; \"Big\" Box" '], + [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]), + ): + with self.subTest(addresses=addresses): + self.assertEqual(utils.getaddresses(addresses), + expected) + self.assertEqual(utils.getaddresses(addresses, strict=False), + expected) + + addresses = ['[]*-- =~$'] + self.assertEqual(utils.getaddresses(addresses), + [('', '')]) + self.assertEqual(utils.getaddresses(addresses, strict=False), + [('', ''), ('', ''), ('', '*--')]) + + def test_getaddresses_embedded_comment(self): + """Test proper handling of a nested comment""" + eq = self.assertEqual + addrs = utils.getaddresses(['User ((nested comment)) ']) + eq(addrs[0][1], 'foo@bar.com') + + def test_getaddresses_header_obj(self): + """Test the handling of a Header object.""" + addrs = utils.getaddresses([Header('Al Person ')]) + self.assertEqual(addrs[0][1], 'aperson@dom.ain') + + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') + def test_make_msgid_collisions(self): + # Test make_msgid uniqueness, even with multiple threads + class MsgidsThread(Thread): + def run(self): + # generate msgids for 3 seconds + self.msgids = [] + append = self.msgids.append + make_msgid = utils.make_msgid + clock = time.monotonic + tfin = clock() + 3.0 + while clock() < tfin: + append(make_msgid(domain='testdomain-string')) + + threads = [MsgidsThread() for i in range(5)] + with threading_helper.start_threads(threads): + pass + all_ids = sum([t.msgids for t in threads], []) + self.assertEqual(len(set(all_ids)), len(all_ids)) + + def test_utils_quote_unquote(self): + eq = self.assertEqual + msg = Message() + msg.add_header('content-disposition', 'attachment', + filename='foo\\wacky"name') + eq(msg.get_filename(), 'foo\\wacky"name') + + def test_get_body_encoding_with_bogus_charset(self): + charset = Charset('not a charset') + self.assertEqual(charset.get_body_encoding(), 'base64') + + def test_get_body_encoding_with_uppercase_charset(self): + eq = self.assertEqual + msg = Message() + msg['Content-Type'] = 'text/plain; charset=UTF-8' + eq(msg['content-type'], 'text/plain; charset=UTF-8') + charsets = msg.get_charsets() + eq(len(charsets), 1) + eq(charsets[0], 'utf-8') + charset = Charset(charsets[0]) + eq(charset.get_body_encoding(), 'base64') + msg.set_payload(b'hello world', charset=charset) + eq(msg.get_payload(), 'aGVsbG8gd29ybGQ=\n') + eq(msg.get_payload(decode=True), b'hello world') + eq(msg['content-transfer-encoding'], 'base64') + # Try another one + msg = Message() + msg['Content-Type'] = 'text/plain; charset="US-ASCII"' + charsets = msg.get_charsets() + eq(len(charsets), 1) + eq(charsets[0], 'us-ascii') + charset = Charset(charsets[0]) + eq(charset.get_body_encoding(), encoders.encode_7or8bit) + msg.set_payload('hello world', charset=charset) + eq(msg.get_payload(), 'hello world') + eq(msg['content-transfer-encoding'], '7bit') + + def test_charsets_case_insensitive(self): + lc = Charset('us-ascii') + uc = Charset('US-ASCII') + self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding()) + + def test_partial_falls_inside_message_delivery_status(self): + eq = self.ndiffAssertEqual + # The Parser interface provides chunks of data to FeedParser in 8192 + # byte gulps. SF bug #1076485 found one of those chunks inside + # message/delivery-status header block, which triggered an + # unreadline() of NeedMoreData. + msg = self._msgobj('msg_43.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/report + text/plain + message/delivery-status + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/rfc822-headers +""") + + def test_make_msgid_domain(self): + self.assertEqual( + email.utils.make_msgid(domain='testdomain-string')[-19:], + '@testdomain-string>') + + def test_make_msgid_idstring(self): + self.assertEqual( + email.utils.make_msgid(idstring='test-idstring', + domain='testdomain-string')[-33:], + '.test-idstring@testdomain-string>') + + def test_make_msgid_default_domain(self): + with patch('socket.getfqdn') as mock_getfqdn: + mock_getfqdn.return_value = domain = 'pythontest.example.com' + self.assertEndsWith(email.utils.make_msgid(), '@' + domain + '>') + + def test_Generator_linend(self): + # Issue 14645. + with openfile('msg_26.txt', encoding="utf-8", newline='\n') as f: + msgtxt = f.read() + msgtxt_nl = msgtxt.replace('\r\n', '\n') + msg = email.message_from_string(msgtxt) + s = StringIO() + g = email.generator.Generator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), msgtxt_nl) + + def test_BytesGenerator_linend(self): + # Issue 14645. + with openfile('msg_26.txt', encoding="utf-8", newline='\n') as f: + msgtxt = f.read() + msgtxt_nl = msgtxt.replace('\r\n', '\n') + msg = email.message_from_string(msgtxt_nl) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue().decode('ascii'), msgtxt) + + def test_BytesGenerator_linend_with_non_ascii(self): + # Issue 14645. + with openfile('msg_26.txt', 'rb') as f: + msgtxt = f.read() + msgtxt = msgtxt.replace(b'with attachment', b'fo\xf6') + msgtxt_nl = msgtxt.replace(b'\r\n', b'\n') + msg = email.message_from_bytes(msgtxt_nl) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue(), msgtxt) + + def test_mime_classes_policy_argument(self): + with openfile('sndhdr.au', 'rb') as fp: + audiodata = fp.read() + with openfile('python.gif', 'rb') as fp: + bindata = fp.read() + classes = [ + (MIMEApplication, ('',)), + (MIMEAudio, (audiodata,)), + (MIMEImage, (bindata,)), + (MIMEMessage, (Message(),)), + (MIMENonMultipart, ('multipart', 'mixed')), + (MIMEText, ('',)), + ] + for cls, constructor in classes: + with self.subTest(cls=cls.__name__, policy='compat32'): + m = cls(*constructor) + self.assertIs(m.policy, email.policy.compat32) + with self.subTest(cls=cls.__name__, policy='default'): + m = cls(*constructor, policy=email.policy.default) + self.assertIs(m.policy, email.policy.default) + + def test_iter_escaped_chars(self): + self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')), + [(0, 'a'), + (2, '\\\\'), + (3, 'b'), + (5, '\\"'), + (6, 'c'), + (8, '\\\\'), + (9, '"'), + (10, 'd')]) + self.assertEqual(list(utils._iter_escaped_chars('a\\')), + [(0, 'a'), (1, '\\')]) + + def test_strip_quoted_realnames(self): + def check(addr, expected): + self.assertEqual(utils._strip_quoted_realnames(addr), expected) + + check('"Jane Doe" , "John Doe" ', + ' , ') + check(r'"Jane \"Doe\"." ', + ' ') + + # special cases + check(r'before"name"after', 'beforeafter') + check(r'before"name"', 'before') + check(r'b"name"', 'b') # single char + check(r'"name"after', 'after') + check(r'"name"a', 'a') # single char + check(r'"name"', '') + + # no change + for addr in ( + 'Jane Doe , John Doe ', + 'lone " quote', + ): + self.assertEqual(utils._strip_quoted_realnames(addr), addr) + + + def test_check_parenthesis(self): + addr = 'alice@example.net' + self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)')) + self.assertFalse(utils._check_parenthesis(f'{addr} )Alice(')) + self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))')) + self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)')) + + # Ignore real name between quotes + self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}')) + + +# Test the iterator/generators +class TestIterators(TestEmailBase): + def test_body_line_iterator(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + # First a simple non-multipart message + msg = self._msgobj('msg_01.txt') + it = iterators.body_line_iterator(msg) + lines = list(it) + eq(len(lines), 6) + neq(EMPTYSTRING.join(lines), msg.get_payload()) + # Now a more complicated multipart + msg = self._msgobj('msg_02.txt') + it = iterators.body_line_iterator(msg) + lines = list(it) + eq(len(lines), 43) + with openfile('msg_19.txt', encoding="utf-8") as fp: + neq(EMPTYSTRING.join(lines), fp.read()) + + def test_typed_subpart_iterator(self): + eq = self.assertEqual + msg = self._msgobj('msg_04.txt') + it = iterators.typed_subpart_iterator(msg, 'text') + lines = [] + subparts = 0 + for subpart in it: + subparts += 1 + lines.append(subpart.get_payload()) + eq(subparts, 2) + eq(EMPTYSTRING.join(lines), """\ +a simple kind of mirror +to reflect upon our own +a simple kind of mirror +to reflect upon our own +""") + + def test_typed_subpart_iterator_default_type(self): + eq = self.assertEqual + msg = self._msgobj('msg_03.txt') + it = iterators.typed_subpart_iterator(msg, 'text', 'plain') + lines = [] + subparts = 0 + for subpart in it: + subparts += 1 + lines.append(subpart.get_payload()) + eq(subparts, 1) + eq(EMPTYSTRING.join(lines), """\ + +Hi, + +Do you like this message? + +-Me +""") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pushCR_LF(self): + '''FeedParser BufferedSubFile.push() assumed it received complete + line endings. A CR ending one push() followed by a LF starting + the next push() added an empty line. + ''' + imt = [ + ("a\r \n", 2), + ("b", 0), + ("c\n", 1), + ("", 0), + ("d\r\n", 1), + ("e\r", 0), + ("\nf", 1), + ("\r\n", 1), + ] + from email.feedparser import BufferedSubFile, NeedMoreData + bsf = BufferedSubFile() + om = [] + nt = 0 + for il, n in imt: + bsf.push(il) + nt += n + n1 = 0 + for ol in iter(bsf.readline, NeedMoreData): + om.append(ol) + n1 += 1 + self.assertEqual(n, n1) + self.assertEqual(len(om), nt) + self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_empty_header_name_handled(self): + # Issue 19996 + msg = self.parse("First: val\n: bad\nSecond: val") + self.assertEqual(msg['First'], 'val') + self.assertEqual(msg['Second'], 'val') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Feedparser.feed -> Feedparser._input.push, Feedparser._call_parse -> Feedparser._parse does not keep _input state between calls + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + # Only CR and LF should break header fields + m = self.parse(['a:\x85b:\u2028c:\n']) + self.assertEqual(m.items(), [('a', '\x85b:\u2028c:')]) + m = self.parse(['a:\r', 'b:\x85', 'c:\n']) + self.assertEqual(m.items(), [('a', ''), ('b', '\x85c:')]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 6*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M+'\x85'] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), ('x'*M+'\x85')*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) + + +class TestParsers(TestEmailBase): + + def test_header_parser(self): + eq = self.assertEqual + # Parse only the headers of a complex multipart MIME document + with openfile('msg_02.txt', encoding="utf-8") as fp: + msg = HeaderParser().parse(fp) + eq(msg['from'], 'ppp-request@zzz.org') + eq(msg['to'], 'ppp@zzz.org') + eq(msg.get_content_type(), 'multipart/mixed') + self.assertFalse(msg.is_multipart()) + self.assertIsInstance(msg.get_payload(), str) + + def test_bytes_header_parser(self): + eq = self.assertEqual + # Parse only the headers of a complex multipart MIME document + with openfile('msg_02.txt', 'rb') as fp: + msg = email.parser.BytesHeaderParser().parse(fp) + eq(msg['from'], 'ppp-request@zzz.org') + eq(msg['to'], 'ppp@zzz.org') + eq(msg.get_content_type(), 'multipart/mixed') + self.assertFalse(msg.is_multipart()) + self.assertIsInstance(msg.get_payload(), str) + self.assertIsInstance(msg.get_payload(decode=True), bytes) + + def test_header_parser_multipart_is_valid(self): + # Don't flag valid multipart emails as having defects + with openfile('msg_47.txt', encoding="utf-8") as fp: + msgdata = fp.read() + + parser = email.parser.Parser(policy=email.policy.default) + parsed_msg = parser.parsestr(msgdata, headersonly=True) + + self.assertEqual(parsed_msg.defects, []) + + def test_bytes_parser_does_not_close_file(self): + with openfile('msg_02.txt', 'rb') as fp: + email.parser.BytesParser().parse(fp) + self.assertFalse(fp.closed) + + def test_bytes_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', 'rb') as fp: + bytesParser = email.parser.BytesParser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + bytesParser(policy=email.policy.strict).parse, + fp) + self.assertFalse(fp.closed) + + def test_parser_does_not_close_file(self): + with openfile('msg_02.txt', encoding="utf-8") as fp: + email.parser.Parser().parse(fp) + self.assertFalse(fp.closed) + + def test_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', encoding="utf-8") as fp: + parser = email.parser.Parser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + parser(policy=email.policy.strict).parse, fp) + self.assertFalse(fp.closed) + + def test_whitespace_continuation(self): + eq = self.assertEqual + # This message contains a line after the Subject: header that has only + # whitespace, but it is not empty! + msg = email.message_from_string("""\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: the next line has a space on it +\x20 +Date: Mon, 8 Apr 2002 15:09:19 -0400 +Message-ID: spam + +Here's the message body +""") + eq(msg['subject'], 'the next line has a space on it\n ') + eq(msg['message-id'], 'spam') + eq(msg.get_payload(), "Here's the message body\n") + + def test_whitespace_continuation_last_header(self): + eq = self.assertEqual + # Like the previous test, but the subject line is the last + # header. + msg = email.message_from_string("""\ +From: aperson@dom.ain +To: bperson@dom.ain +Date: Mon, 8 Apr 2002 15:09:19 -0400 +Message-ID: spam +Subject: the next line has a space on it +\x20 + +Here's the message body +""") + eq(msg['subject'], 'the next line has a space on it\n ') + eq(msg['message-id'], 'spam') + eq(msg.get_payload(), "Here's the message body\n") + + def test_crlf_separation(self): + eq = self.assertEqual + with openfile('msg_26.txt', encoding="utf-8", newline='\n') as fp: + msg = Parser().parse(fp) + eq(len(msg.get_payload()), 2) + part1 = msg.get_payload(0) + eq(part1.get_content_type(), 'text/plain') + eq(part1.get_payload(), 'Simple email with attachment.\r\n\r\n') + part2 = msg.get_payload(1) + eq(part2.get_content_type(), 'application/riscos') + + def test_crlf_flatten(self): + # Using newline='\n' preserves the crlfs in this input file. + with openfile('msg_26.txt', encoding="utf-8", newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = Generator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue(), text) + + maxDiff = None + + def test_multipart_digest_with_extra_mime_headers(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + with openfile('msg_28.txt', encoding="utf-8") as fp: + msg = email.message_from_file(fp) + # Structure is: + # multipart/digest + # message/rfc822 + # text/plain + # message/rfc822 + # text/plain + eq(msg.is_multipart(), 1) + eq(len(msg.get_payload()), 2) + part1 = msg.get_payload(0) + eq(part1.get_content_type(), 'message/rfc822') + eq(part1.is_multipart(), 1) + eq(len(part1.get_payload()), 1) + part1a = part1.get_payload(0) + eq(part1a.is_multipart(), 0) + eq(part1a.get_content_type(), 'text/plain') + neq(part1a.get_payload(), 'message 1\n') + # next message/rfc822 + part2 = msg.get_payload(1) + eq(part2.get_content_type(), 'message/rfc822') + eq(part2.is_multipart(), 1) + eq(len(part2.get_payload()), 1) + part2a = part2.get_payload(0) + eq(part2a.is_multipart(), 0) + eq(part2a.get_content_type(), 'text/plain') + neq(part2a.get_payload(), 'message 2\n') + + def test_three_lines(self): + # A bug report by Andrew McNamara + lines = ['From: Andrew Person From', 'From']) + eq(msg.get_payload(), 'body') + + def test_rfc2822_space_not_allowed_in_header(self): + eq = self.assertEqual + m = '>From foo@example.com 11:25:53\nFrom: bar\n!"#QUX;~: zoo\n\nbody' + msg = email.message_from_string(m) + eq(len(msg.keys()), 0) + + def test_rfc2822_one_character_header(self): + eq = self.assertEqual + m = 'A: first header\nB: second header\nCC: third header\n\nbody' + msg = email.message_from_string(m) + headers = msg.keys() + headers.sort() + eq(headers, ['A', 'B', 'CC']) + eq(msg.get_payload(), 'body') + + def test_CRLFLF_at_end_of_part(self): + # issue 5610: feedparser should not eat two chars from body part ending + # with "\r\n\n". + m = ( + "From: foo@bar.com\n" + "To: baz\n" + "Mime-Version: 1.0\n" + "Content-Type: multipart/mixed; boundary=BOUNDARY\n" + "\n" + "--BOUNDARY\n" + "Content-Type: text/plain\n" + "\n" + "body ending with CRLF newline\r\n" + "\n" + "--BOUNDARY--\n" + ) + msg = email.message_from_string(m) + self.assertEndsWith(msg.get_payload(0).get_payload(), '\r\n') + + +class Test8BitBytesHandling(TestEmailBase): + # In Python3 all input is string, but that doesn't work if the actual input + # uses an 8bit transfer encoding. To hack around that, in email 5.1 we + # decode byte streams using the surrogateescape error handler, and + # reconvert to binary at appropriate places if we detect surrogates. This + # doesn't allow us to transform headers with 8bit bytes (they get munged), + # but it does allow us to parse and preserve them, and to decode body + # parts that use an 8bit CTE. + + bodytest_msg = textwrap.dedent("""\ + From: foo@bar.com + To: baz + Mime-Version: 1.0 + Content-Type: text/plain; charset={charset} + Content-Transfer-Encoding: {cte} + + {bodyline} + """) + + def test_known_8bit_CTE(self): + m = self.bodytest_msg.format(charset='utf-8', + cte='8bit', + bodyline='pöstal').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), "pöstal\n") + self.assertEqual(msg.get_payload(decode=True), + "pöstal\n".encode('utf-8')) + + def test_unknown_8bit_CTE(self): + m = self.bodytest_msg.format(charset='notavalidcharset', + cte='8bit', + bodyline='pöstal').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), "p\uFFFD\uFFFDstal\n") + self.assertEqual(msg.get_payload(decode=True), + "pöstal\n".encode('utf-8')) + + def test_8bit_in_quopri_body(self): + # This is non-RFC compliant data...without 'decode' the library code + # decodes the body using the charset from the headers, and because the + # source byte really is utf-8 this works. This is likely to fail + # against real dirty data (ie: produce mojibake), but the data is + # invalid anyway so it is as good a guess as any. But this means that + # this test just confirms the current behavior; that behavior is not + # necessarily the best possible behavior. With 'decode' it is + # returning the raw bytes, so that test should be of correct behavior, + # or at least produce the same result that email4 did. + m = self.bodytest_msg.format(charset='utf-8', + cte='quoted-printable', + bodyline='p=C3=B6stál').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), 'p=C3=B6stál\n') + self.assertEqual(msg.get_payload(decode=True), + 'pöstál\n'.encode('utf-8')) + + def test_invalid_8bit_in_non_8bit_cte_uses_replace(self): + # This is similar to the previous test, but proves that if the 8bit + # byte is undecodeable in the specified charset, it gets replaced + # by the unicode 'unknown' character. Again, this may or may not + # be the ideal behavior. Note that if decode=False none of the + # decoders will get involved, so this is the only test we need + # for this behavior. + m = self.bodytest_msg.format(charset='ascii', + cte='quoted-printable', + bodyline='p=C3=B6stál').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), 'p=C3=B6st\uFFFD\uFFFDl\n') + self.assertEqual(msg.get_payload(decode=True), + 'pöstál\n'.encode('utf-8')) + + # test_defect_handling:test_invalid_chars_in_base64_payload + def test_8bit_in_base64_body(self): + # If we get 8bit bytes in a base64 body, we can just ignore them + # as being outside the base64 alphabet and decode anyway. But + # we register a defect. + m = self.bodytest_msg.format(charset='utf-8', + cte='base64', + bodyline='cMO2c3RhbAá=').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(decode=True), + 'pöstal'.encode('utf-8')) + self.assertIsInstance(msg.defects[0], + errors.InvalidBase64CharactersDefect) + + def test_8bit_in_uuencode_body(self): + # Sticking an 8bit byte in a uuencode block makes it undecodable by + # normal means, so the block is returned undecoded, but as bytes. + m = self.bodytest_msg.format(charset='utf-8', + cte='uuencode', + bodyline='<,.V7bit conversion. + self.assertEqual(out.getvalue(), + self.latin_bin_msg.decode('latin-1')+'\n') + + def test_bytes_feedparser(self): + bfp = email.feedparser.BytesFeedParser() + for i in range(0, len(self.latin_bin_msg), 10): + bfp.feed(self.latin_bin_msg[i:i+10]) + m = bfp.close() + self.assertEqual(str(m), self.latin_bin_msg_as7bit) + + def test_crlf_flatten(self): + with openfile('msg_26.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue(), text) + + def test_8bit_multipart(self): + # Issue 11605 + source = textwrap.dedent("""\ + Date: Fri, 18 Mar 2011 17:15:43 +0100 + To: foo@example.com + From: foodwatch-Newsletter + Subject: Aktuelles zu Japan, Klonfleisch und Smiley-System + Message-ID: <76a486bee62b0d200f33dc2ca08220ad@localhost.localdomain> + MIME-Version: 1.0 + Content-Type: multipart/alternative; + boundary="b1_76a486bee62b0d200f33dc2ca08220ad" + + --b1_76a486bee62b0d200f33dc2ca08220ad + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Guten Tag, , + + mit großer Betroffenheit verfolgen auch wir im foodwatch-Team die + Nachrichten aus Japan. + + + --b1_76a486bee62b0d200f33dc2ca08220ad + Content-Type: text/html; charset="utf-8" + Content-Transfer-Encoding: 8bit + + + + + foodwatch - Newsletter + + +

mit großer Betroffenheit verfolgen auch wir im foodwatch-Team + die Nachrichten aus Japan.

+ + + --b1_76a486bee62b0d200f33dc2ca08220ad-- + + """).encode('utf-8') + msg = email.message_from_bytes(source) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), source) + + def test_bytes_generator_b_encoding_linesep(self): + # Issue 14062: b encoding was tacking on an extra \n. + m = Message() + # This has enough non-ascii that it should always end up b encoded. + m['Subject'] = Header('žluťoučký kůň') + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(m, linesep='\r\n') + self.assertEqual( + s.getvalue(), + b'Subject: =?utf-8?b?xb5sdcWlb3XEjWvDvSBrxa/FiA==?=\r\n\r\n') + + def test_generator_b_encoding_linesep(self): + # Since this broke in ByteGenerator, test Generator for completeness. + m = Message() + # This has enough non-ascii that it should always end up b encoded. + m['Subject'] = Header('žluťoučký kůň') + s = StringIO() + g = email.generator.Generator(s) + g.flatten(m, linesep='\r\n') + self.assertEqual( + s.getvalue(), + 'Subject: =?utf-8?b?xb5sdcWlb3XEjWvDvSBrxa/FiA==?=\r\n\r\n') + + maxDiff = None + + +class BaseTestBytesGeneratorIdempotent: + + maxDiff = None + + def _msgobj(self, filename): + with openfile(filename, 'rb') as fp: + data = fp.read() + data = self.normalize_linesep_regex.sub(self.blinesep, data) + msg = email.message_from_bytes(data) + return msg, data + + def _idempotent(self, msg, data, unixfrom=False): + b = BytesIO() + g = email.generator.BytesGenerator(b, maxheaderlen=0) + g.flatten(msg, unixfrom=unixfrom, linesep=self.linesep) + self.assertEqual(data, b.getvalue()) + + +class TestBytesGeneratorIdempotentNL(BaseTestBytesGeneratorIdempotent, + TestIdempotent): + linesep = '\n' + blinesep = b'\n' + normalize_linesep_regex = re.compile(br'\r\n') + + +class TestBytesGeneratorIdempotentCRLF(BaseTestBytesGeneratorIdempotent, + TestIdempotent): + linesep = '\r\n' + blinesep = b'\r\n' + normalize_linesep_regex = re.compile(br'(? A+B==2', 'A=1,B=A ==> A+B==2') + + def _test_encode(self, body, expected_encoded_body, maxlinelen=None, eol=None): + kwargs = {} + if maxlinelen is None: + # Use body_encode's default. + maxlinelen = 76 + else: + kwargs['maxlinelen'] = maxlinelen + if eol is None: + # Use body_encode's default. + eol = '\n' + else: + kwargs['eol'] = eol + encoded_body = quoprimime.body_encode(body, **kwargs) + self.assertEqual(encoded_body, expected_encoded_body) + if eol == '\n' or eol == '\r\n': + # We know how to split the result back into lines, so maxlinelen + # can be checked. + for line in encoded_body.splitlines(): + self.assertLessEqual(len(line), maxlinelen) + + def test_encode_null(self): + self._test_encode('', '') + + def test_encode_null_lines(self): + self._test_encode('\n\n', '\n\n') + + def test_encode_one_line(self): + self._test_encode('hello\n', 'hello\n') + + def test_encode_one_line_crlf(self): + self._test_encode('hello\r\n', 'hello\n') + + def test_encode_one_line_eol(self): + self._test_encode('hello\n', 'hello\r\n', eol='\r\n') + + def test_encode_one_line_eol_after_non_ascii(self): + # issue 20206; see changeset 0cf700464177 for why the encode/decode. + self._test_encode('hello\u03c5\n'.encode('utf-8').decode('latin1'), + 'hello=CF=85\r\n', eol='\r\n') + + def test_encode_one_space(self): + self._test_encode(' ', '=20') + + def test_encode_one_line_one_space(self): + self._test_encode(' \n', '=20\n') + +# XXX: body_encode() expect strings, but uses ord(char) from these strings +# to index into a 256-entry list. For code points above 255, this will fail. +# Should there be a check for 8-bit only ord() values in body, or at least +# a comment about the expected input? + + def test_encode_two_lines_one_space(self): + self._test_encode(' \n \n', '=20\n=20\n') + + def test_encode_one_word_trailing_spaces(self): + self._test_encode('hello ', 'hello =20') + + def test_encode_one_line_trailing_spaces(self): + self._test_encode('hello \n', 'hello =20\n') + + def test_encode_one_word_trailing_tab(self): + self._test_encode('hello \t', 'hello =09') + + def test_encode_one_line_trailing_tab(self): + self._test_encode('hello \t\n', 'hello =09\n') + + def test_encode_trailing_space_before_maxlinelen(self): + self._test_encode('abcd \n1234', 'abcd =\n\n1234', maxlinelen=6) + + def test_encode_trailing_space_at_maxlinelen(self): + self._test_encode('abcd \n1234', 'abcd=\n=20\n1234', maxlinelen=5) + + def test_encode_trailing_space_beyond_maxlinelen(self): + self._test_encode('abcd \n1234', 'abc=\nd=20\n1234', maxlinelen=4) + + def test_encode_whitespace_lines(self): + self._test_encode(' \n' * 5, '=20\n' * 5) + + def test_encode_quoted_equals(self): + self._test_encode('a = b', 'a =3D b') + + def test_encode_one_long_string(self): + self._test_encode('x' * 100, 'x' * 75 + '=\n' + 'x' * 25) + + def test_encode_one_long_line(self): + self._test_encode('x' * 100 + '\n', 'x' * 75 + '=\n' + 'x' * 25 + '\n') + + def test_encode_one_very_long_line(self): + self._test_encode('x' * 200 + '\n', + 2 * ('x' * 75 + '=\n') + 'x' * 50 + '\n') + + def test_encode_shortest_maxlinelen(self): + self._test_encode('=' * 5, '=3D=\n' * 4 + '=3D', maxlinelen=4) + + def test_encode_maxlinelen_too_small(self): + self.assertRaises(ValueError, self._test_encode, '', '', maxlinelen=3) + + def test_encode(self): + eq = self.assertEqual + eq(quoprimime.body_encode(''), '') + eq(quoprimime.body_encode('hello'), 'hello') + # Test the binary flag + eq(quoprimime.body_encode('hello\r\nworld'), 'hello\nworld') + # Test the maxlinelen arg + eq(quoprimime.body_encode('xxxx ' * 20, maxlinelen=40), """\ +xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx= + xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx= +x xxxx xxxx xxxx xxxx=20""") + # Test the eol argument + eq(quoprimime.body_encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), + """\ +xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx=\r + xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx=\r +x xxxx xxxx xxxx xxxx=20""") + eq(quoprimime.body_encode("""\ +one line + +two line"""), """\ +one line + +two line""") + + + +# Test the Charset class +class TestCharset(unittest.TestCase): + def tearDown(self): + from email import charset as CharsetModule + try: + del CharsetModule.CHARSETS['fake'] + except KeyError: + pass + + def test_codec_encodeable(self): + eq = self.assertEqual + # Make sure us-ascii = no Unicode conversion + c = Charset('us-ascii') + eq(c.header_encode('Hello World!'), 'Hello World!') + # Test 8-bit idempotency with us-ascii + s = '\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa' + self.assertRaises(UnicodeError, c.header_encode, s) + c = Charset('utf-8') + eq(c.header_encode(s), '=?utf-8?b?wqTCosKkwqTCpMKmwqTCqMKkwqo=?=') + + def test_body_encode(self): + eq = self.assertEqual + # Try a charset with QP body encoding + c = Charset('iso-8859-1') + eq('hello w=F6rld', c.body_encode('hello w\xf6rld')) + # Try a charset with Base64 body encoding + c = Charset('utf-8') + eq('aGVsbG8gd29ybGQ=\n', c.body_encode(b'hello world')) + # Try a charset with None body encoding + c = Charset('us-ascii') + eq('hello world', c.body_encode('hello world')) + # Try the convert argument, where input codec != output codec + c = Charset('euc-jp') + # With apologies to Tokio Kikuchi ;) + # XXX FIXME +## try: +## eq('\x1b$B5FCO;~IW\x1b(B', +## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) +## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', +## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) +## except LookupError: +## # We probably don't have the Japanese codecs installed +## pass + # Testing SF bug #625509, which we have to fake, since there are no + # built-in encodings where the header encoding is QP but the body + # encoding is not. + from email import charset as CharsetModule + CharsetModule.add_charset('fake', CharsetModule.QP, None, 'utf-8') + c = Charset('fake') + eq('hello world', c.body_encode('hello world')) + + def test_unicode_charset_name(self): + charset = Charset('us-ascii') + self.assertEqual(str(charset), 'us-ascii') + self.assertRaises(errors.CharsetError, Charset, 'asc\xffii') + + + +# Test multilingual MIME headers. +class TestHeader(TestEmailBase): + def test_simple(self): + eq = self.ndiffAssertEqual + h = Header('Hello World!') + eq(h.encode(), 'Hello World!') + h.append(' Goodbye World!') + eq(h.encode(), 'Hello World! Goodbye World!') + + def test_simple_surprise(self): + eq = self.ndiffAssertEqual + h = Header('Hello World!') + eq(h.encode(), 'Hello World!') + h.append('Goodbye World!') + eq(h.encode(), 'Hello World! Goodbye World!') + + def test_header_needs_no_decoding(self): + h = 'no decoding needed' + self.assertEqual(decode_header(h), [(h, None)]) + + def test_long(self): + h = Header("I am the very model of a modern Major-General; I've information vegetable, animal, and mineral; I know the kings of England, and I quote the fights historical from Marathon to Waterloo, in order categorical; I'm very well acquainted, too, with matters mathematical; I understand equations, both the simple and quadratical; about binomial theorem I'm teeming with a lot o' news, with many cheerful facts about the square of the hypotenuse.", + maxlinelen=76) + for l in h.encode(splitchars=' ').split('\n '): + self.assertLessEqual(len(l), 76) + + def test_multilingual(self): + eq = self.ndiffAssertEqual + g = Charset("iso-8859-1") + cz = Charset("iso-8859-2") + utf8 = Charset("utf-8") + g_head = (b'Die Mieter treten hier ein werden mit einem ' + b'Foerderband komfortabel den Korridor entlang, ' + b'an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, ' + b'gegen die rotierenden Klingen bef\xf6rdert. ') + cz_head = (b'Finan\xe8ni metropole se hroutily pod tlakem jejich ' + b'd\xf9vtipu.. ') + utf8_head = ('\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das ' + 'Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder ' + 'die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066' + '\u3044\u307e\u3059\u3002') + h = Header(g_head, g) + h.append(cz_head, cz) + h.append(utf8_head, utf8) + enc = h.encode(maxlinelen=76) + eq(enc, """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderband_kom?= + =?iso-8859-1?q?fortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen_Wand?= + =?iso-8859-1?q?gem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef=F6r?= + =?iso-8859-1?q?dert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hroutily?= + =?iso-8859-2?q?_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= + =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC?= + =?utf-8?b?5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn?= + =?utf-8?b?44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFz?= + =?utf-8?b?IE51bnN0dWNrIGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWloZXJodW5k?= + =?utf-8?b?IGRhcyBPZGVyIGRpZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI3jgajoqIA=?= + =?utf-8?b?44Gj44Gm44GE44G+44GZ44CC?=""") + decoded = decode_header(enc) + eq(len(decoded), 3) + eq(decoded[0], (g_head, 'iso-8859-1')) + eq(decoded[1], (cz_head, 'iso-8859-2')) + eq(decoded[2], (utf8_head.encode('utf-8'), 'utf-8')) + ustr = str(h) + eq(ustr, + (b'Die Mieter treten hier ein werden mit einem Foerderband ' + b'komfortabel den Korridor entlang, an s\xc3\xbcdl\xc3\xbcndischen ' + b'Wandgem\xc3\xa4lden vorbei, gegen die rotierenden Klingen ' + b'bef\xc3\xb6rdert. Finan\xc4\x8dni metropole se hroutily pod ' + b'tlakem jejich d\xc5\xafvtipu.. \xe6\xad\xa3\xe7\xa2\xba\xe3\x81' + b'\xab\xe8\xa8\x80\xe3\x81\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3' + b'\xe3\x81\xaf\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3' + b'\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' + b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8\xaa\x9e' + b'\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\xe3\x81\x82\xe3' + b'\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81\x9f\xe3\x82\x89\xe3\x82' + b'\x81\xe3\x81\xa7\xe3\x81\x99\xe3\x80\x82\xe5\xae\x9f\xe9\x9a\x9b' + b'\xe3\x81\xab\xe3\x81\xaf\xe3\x80\x8cWenn ist das Nunstuck git ' + b'und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt ' + b'gersput.\xe3\x80\x8d\xe3\x81\xa8\xe8\xa8\x80\xe3\x81\xa3\xe3\x81' + b'\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80\x82' + ).decode('utf-8')) + # Test make_header() + newh = make_header(decode_header(enc)) + eq(newh, h) + + def test_empty_header_encode(self): + h = Header() + self.assertEqual(h.encode(), '') + + def test_header_ctor_default_args(self): + eq = self.ndiffAssertEqual + h = Header() + eq(h, '') + h.append('foo', Charset('iso-8859-1')) + eq(h, 'foo') + + def test_explicit_maxlinelen(self): + eq = self.ndiffAssertEqual + hstr = ('A very long line that must get split to something other ' + 'than at the 76th character boundary to test the non-default ' + 'behavior') + h = Header(hstr) + eq(h.encode(), '''\ +A very long line that must get split to something other than at the 76th + character boundary to test the non-default behavior''') + eq(str(h), hstr) + h = Header(hstr, header_name='Subject') + eq(h.encode(), '''\ +A very long line that must get split to something other than at the + 76th character boundary to test the non-default behavior''') + eq(str(h), hstr) + h = Header(hstr, maxlinelen=1024, header_name='Subject') + eq(h.encode(), hstr) + eq(str(h), hstr) + + def test_quopri_splittable(self): + eq = self.ndiffAssertEqual + h = Header(charset='iso-8859-1', maxlinelen=20) + x = 'xxxx ' * 20 + h.append(x) + s = h.encode() + eq(s, """\ +=?iso-8859-1?q?xxx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_?=""") + eq(x, str(make_header(decode_header(s)))) + h = Header(charset='iso-8859-1', maxlinelen=40) + h.append('xxxx ' * 20) + s = h.encode() + eq(s, """\ +=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xxx?= + =?iso-8859-1?q?x_xxxx_xxxx_xxxx_xxxx_?= + =?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?= + =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?= + =?iso-8859-1?q?_xxxx_xxxx_?=""") + eq(x, str(make_header(decode_header(s)))) + + def test_base64_splittable(self): + eq = self.ndiffAssertEqual + h = Header(charset='koi8-r', maxlinelen=20) + x = 'xxxx ' * 20 + h.append(x) + s = h.encode() + eq(s, """\ +=?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IA==?=""") + eq(x, str(make_header(decode_header(s)))) + h = Header(charset='koi8-r', maxlinelen=40) + h.append(x) + s = h.encode() + eq(s, """\ +=?koi8-r?b?eHh4eCB4eHh4IHh4eHggeHh4?= + =?koi8-r?b?eCB4eHh4IHh4eHggeHh4eCB4?= + =?koi8-r?b?eHh4IHh4eHggeHh4eCB4eHh4?= + =?koi8-r?b?IHh4eHggeHh4eCB4eHh4IHh4?= + =?koi8-r?b?eHggeHh4eCB4eHh4IHh4eHgg?= + =?koi8-r?b?eHh4eCB4eHh4IA==?=""") + eq(x, str(make_header(decode_header(s)))) + + def test_us_ascii_header(self): + eq = self.assertEqual + s = 'hello' + x = decode_header(s) + eq(x, [('hello', None)]) + h = make_header(x) + eq(s, h.encode()) + + def test_string_charset(self): + eq = self.assertEqual + h = Header() + h.append('hello', 'iso-8859-1') + eq(h, 'hello') + +## def test_unicode_error(self): +## raises = self.assertRaises +## raises(UnicodeError, Header, u'[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, Header, '[P\xf6stal]', 'us-ascii') +## h = Header() +## raises(UnicodeError, h.append, u'[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, h.append, '[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, Header, u'\u83ca\u5730\u6642\u592b', 'iso-8859-1') + + def test_utf8_shortest(self): + eq = self.assertEqual + h = Header('p\xf6stal', 'utf-8') + eq(h.encode(), '=?utf-8?q?p=C3=B6stal?=') + h = Header('\u83ca\u5730\u6642\u592b', 'utf-8') + eq(h.encode(), '=?utf-8?b?6I+K5Zyw5pmC5aSr?=') + + def test_bad_8bit_header(self): + raises = self.assertRaises + eq = self.assertEqual + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + raises(UnicodeError, Header, x) + h = Header() + raises(UnicodeError, h.append, x) + e = x.decode('utf-8', 'replace') + eq(str(Header(x, errors='replace')), e) + h.append(x, errors='replace') + eq(str(h), e) + + def test_escaped_8bit_header(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + e = x.decode('ascii', 'surrogateescape') + h = Header(e, charset=email.charset.UNKNOWN8BIT) + self.assertEqual(str(h), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h), [(x, 'unknown-8bit')]) + + def test_header_handles_binary_unknown8bit(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + h = Header(x, charset=email.charset.UNKNOWN8BIT) + self.assertEqual(str(h), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h), [(x, 'unknown-8bit')]) + + def test_make_header_handles_binary_unknown8bit(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + h = Header(x, charset=email.charset.UNKNOWN8BIT) + h2 = email.header.make_header(email.header.decode_header(h)) + self.assertEqual(str(h2), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h2), [(x, 'unknown-8bit')]) + + def test_modify_returned_list_does_not_change_header(self): + h = Header('test') + chunks = email.header.decode_header(h) + chunks.append(('ascii', 'test2')) + self.assertEqual(str(h), 'test') + + def test_encoded_adjacent_nonencoded(self): + eq = self.assertEqual + h = Header() + h.append('hello', 'iso-8859-1') + h.append('world') + s = h.encode() + eq(s, '=?iso-8859-1?q?hello?= world') + h = make_header(decode_header(s)) + eq(h.encode(), s) + + def test_whitespace_keeper(self): + eq = self.assertEqual + s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.' + parts = decode_header(s) + eq(parts, [(b'Subject: ', None), (b'\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), (b' zz.', None)]) + hdr = make_header(parts) + eq(hdr.encode(), + 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.') + + def test_broken_base64_header(self): + raises = self.assertRaises + s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' + raises(errors.HeaderParseError, decode_header, s) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp + def test_shift_jis_charset(self): + h = Header('文', charset='shift_jis') + self.assertEqual(h.encode(), '=?iso-2022-jp?b?GyRCSjgbKEI=?=') + + def test_flatten_header_with_no_value(self): + # Issue 11401 (regression from email 4.x) Note that the space after + # the header doesn't reflect the input, but this is also the way + # email 4.x behaved. At some point it would be nice to fix that. + msg = email.message_from_string("EmptyHeader:") + self.assertEqual(str(msg), "EmptyHeader: \n\n") + + def test_encode_preserves_leading_ws_on_value(self): + msg = Message() + msg['SomeHeader'] = ' value with leading ws' + self.assertEqual(str(msg), "SomeHeader: value with leading ws\n\n") + + def test_whitespace_header(self): + self.assertEqual(Header(' ').encode(), ' ') + + + +# Test RFC 2231 header parameters (en/de)coding +class TestRFC2231(TestEmailBase): + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_with_double_quotes + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_inside_double_quotes + def test_get_param(self): + eq = self.assertEqual + msg = self._msgobj('msg_29.txt') + eq(msg.get_param('title'), + ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) + eq(msg.get_param('title', unquote=False), + ('us-ascii', 'en', '"This is even more ***fun*** isn\'t it!"')) + + def test_set_param(self): + eq = self.ndiffAssertEqual + msg = Message() + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii') + eq(msg.get_param('title'), + ('us-ascii', '', 'This is even more ***fun*** isn\'t it!')) + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + eq(msg.get_param('title'), + ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) + msg = self._msgobj('msg_01.txt') + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + eq(msg.as_string(maxheaderlen=78), """\ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) +\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 +Content-Type: text/plain; charset=us-ascii; + title*=us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21 + + +Hi, + +Do you like this message? + +-Me +""") + + def test_set_param_requote(self): + msg = Message() + msg.set_param('title', 'foo') + self.assertEqual(msg['content-type'], 'text/plain; title="foo"') + msg.set_param('title', 'bar', requote=False) + self.assertEqual(msg['content-type'], 'text/plain; title=bar') + # tspecial is still quoted. + msg.set_param('title', "(bar)bell", requote=False) + self.assertEqual(msg['content-type'], 'text/plain; title="(bar)bell"') + + def test_del_param(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_01.txt') + msg.set_param('foo', 'bar', charset='us-ascii', language='en') + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + msg.del_param('foo', header='Content-Type') + eq(msg.as_string(maxheaderlen=78), """\ +Return-Path: +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) +\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 +Content-Type: text/plain; charset="us-ascii"; + title*=us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21 + + +Hi, + +Do you like this message? + +-Me +""") + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_charset + # I changed the charset name, though, because the one in the file isn't + # a legal charset name. Should add a test for an illegal charset. + def test_rfc2231_get_content_charset(self): + eq = self.assertEqual + msg = self._msgobj('msg_32.txt') + eq(msg.get_content_charset(), 'us-ascii') + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_no_double_quotes + def test_rfc2231_parse_rfc_quoting(self): + m = textwrap.dedent('''\ + Content-Disposition: inline; + \tfilename*0*=''This%20is%20even%20more%20; + \tfilename*1*=%2A%2A%2Afun%2A%2A%2A%20; + \tfilename*2="is it not.pdf" + + ''') + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + self.assertEqual(m, msg.as_string()) + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_with_double_quotes + def test_rfc2231_parse_extra_quoting(self): + m = textwrap.dedent('''\ + Content-Disposition: inline; + \tfilename*0*="''This%20is%20even%20more%20"; + \tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; + \tfilename*2="is it not.pdf" + + ''') + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + self.assertEqual(m, msg.as_string()) + + # test_headerregistry.TestContentTypeHeader.rfc2231_no_language_or_charset + # but new test uses *0* because otherwise lang/charset is not valid. + # test_headerregistry.TestContentTypeHeader.rfc2231_segmented_normal_values + def test_rfc2231_no_language_or_charset(self): + m = '''\ +Content-Transfer-Encoding: 8bit +Content-Disposition: inline; filename="file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm" +Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEM; NAME*1=P_nsmail.htm + +''' + msg = email.message_from_string(m) + param = msg.get_param('NAME') + self.assertNotIsInstance(param, tuple) + self.assertEqual( + param, + 'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm') + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_no_charset + def test_rfc2231_no_language_or_charset_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + # Duplicate of previous test? + def test_rfc2231_no_language_or_charset_in_filename_encoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + # test_headerregistry.TestContentTypeHeader.rfc2231_partly_encoded, + # but the test below is wrong (the first part should be decoded). + def test_rfc2231_partly_encoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + 'This%20is%20even%20more%20***fun*** is it not.pdf') + + def test_rfc2231_partly_nonencoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0="This%20is%20even%20more%20"; +\tfilename*1="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + 'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20is it not.pdf') + + def test_rfc2231_no_language_or_charset_in_boundary(self): + m = '''\ +Content-Type: multipart/alternative; +\tboundary*0*="''This%20is%20even%20more%20"; +\tboundary*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tboundary*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_boundary(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_no_language_or_charset_in_charset(self): + # This is a nonsensical charset value, but tests the code anyway + m = '''\ +Content-Type: text/plain; +\tcharset*0*="This%20is%20even%20more%20"; +\tcharset*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tcharset*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_content_charset(), + 'this is even more ***fun*** is it not.pdf') + + # test_headerregistry.TestContentTypeHeader.rfc2231_unknown_charset_treated_as_ascii + def test_rfc2231_bad_encoding_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="bogus'xx'This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_bad_encoding_in_charset(self): + m = """\ +Content-Type: text/plain; charset*=bogus''utf-8%E2%80%9D + +""" + msg = email.message_from_string(m) + # This should return None because non-ascii characters in the charset + # are not allowed. + self.assertEqual(msg.get_content_charset(), None) + + def test_rfc2231_bad_character_in_charset(self): + m = """\ +Content-Type: text/plain; charset*=ascii''utf-8%E2%80%9D + +""" + msg = email.message_from_string(m) + # This should return None because non-ascii characters in the charset + # are not allowed. + self.assertEqual(msg.get_content_charset(), None) + + def test_rfc2231_bad_character_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="ascii'xx'This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2*="is it not.pdf%E2" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf\ufffd') + + def test_rfc2231_unknown_encoding(self): + m = """\ +Content-Transfer-Encoding: 8bit +Content-Disposition: inline; filename*=X-UNKNOWN''myfile.txt + +""" + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), 'myfile.txt') + + def test_rfc2231_bad_character_in_encoding(self): + m = """\ +Content-Transfer-Encoding: 8bit +Content-Disposition: inline; filename*=utf-8\udce2\udc80\udc9d''myfile.txt + +""" + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), 'myfile.txt') + + def test_rfc2231_single_tick_in_filename_extended(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"Frank's\"; name*1*=\" Document\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, None) + eq(language, None) + eq(s, "Frank's Document") + + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_inside_double_quotes + def test_rfc2231_single_tick_in_filename(self): + m = """\ +Content-Type: application/x-foo; name*0=\"Frank's\"; name*1=\" Document\" + +""" + msg = email.message_from_string(m) + param = msg.get_param('name') + self.assertNotIsInstance(param, tuple) + self.assertEqual(param, "Frank's Document") + + def test_rfc2231_missing_tick(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="'This%20is%20broken"; +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + "'This is broken") + + def test_rfc2231_missing_tick_with_encoded_non_ascii(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="'This%20is%E2broken"; +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + "'This is\ufffdbroken") + + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_in_value_with_charset_and_lang + def test_rfc2231_tick_attack_extended(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"us-ascii'en-us'Frank's\"; name*1*=\" Document\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, "Frank's Document") + + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_in_non_encoded_value + def test_rfc2231_tick_attack(self): + m = """\ +Content-Type: application/x-foo; +\tname*0=\"us-ascii'en-us'Frank's\"; name*1=\" Document\" + +""" + msg = email.message_from_string(m) + param = msg.get_param('name') + self.assertNotIsInstance(param, tuple) + self.assertEqual(param, "us-ascii'en-us'Frank's Document") + + # test_headerregistry.TestContentTypeHeader.rfc2231_single_quotes_inside_quotes + def test_rfc2231_no_extended_values(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; name=\"Frank's Document\" + +""" + msg = email.message_from_string(m) + eq(msg.get_param('name'), "Frank's Document") + + # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_then_unencoded_segments + def test_rfc2231_encoded_then_unencoded_segments(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"us-ascii'en-us'My\"; +\tname*1=\" Document\"; +\tname*2*=\" For You\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, 'My Document For You') + + # test_headerregistry.TestContentTypeHeader.rfc2231_unencoded_then_encoded_segments + # test_headerregistry.TestContentTypeHeader.rfc2231_quoted_unencoded_then_encoded_segments + def test_rfc2231_unencoded_then_encoded_segments(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0=\"us-ascii'en-us'My\"; +\tname*1*=\" Document\"; +\tname*2*=\" For You\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, 'My Document For You') + + def test_should_not_hang_on_invalid_ew_messages(self): + messages = ["""From: user@host.com +To: user@host.com +Bad-Header: + =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?= + =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?= + =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?= + +Hello! +""", """From: ����� �������� +To: "xxx" +Subject: ��� ���������� ����� ����� � ��������� �� ���� +MIME-Version: 1.0 +Content-Type: text/plain; charset="windows-1251"; +Content-Transfer-Encoding: 8bit + +�� ����� � ���� ������ ��� �������� +"""] + for m in messages: + with self.subTest(m=m): + msg = email.message_from_string(m) + + +# Tests to ensure that signed parts of an email are completely preserved, as +# required by RFC1847 section 2.1. Note that these are incomplete, because the +# email package does not currently always preserve the body. See issue 1670765. +class TestSigned(TestEmailBase): + + def _msg_and_obj(self, filename): + with openfile(filename, encoding="utf-8") as fp: + original = fp.read() + msg = email.message_from_string(original) + return original, msg + + def _signed_parts_eq(self, original, result): + # Extract the first mime part of each message + import re + repart = re.compile(r'^--([^\n]+)\n(.*?)\n--\1$', re.S | re.M) + inpart = repart.search(original).group(2) + outpart = repart.search(result).group(2) + self.assertEqual(outpart, inpart) + + def test_long_headers_as_string(self): + original, msg = self._msg_and_obj('msg_45.txt') + result = msg.as_string() + self._signed_parts_eq(original, result) + + def test_long_headers_as_string_maxheaderlen(self): + original, msg = self._msg_and_obj('msg_45.txt') + result = msg.as_string(maxheaderlen=60) + self._signed_parts_eq(original, result) + + def test_long_headers_flatten(self): + original, msg = self._msg_and_obj('msg_45.txt') + fp = StringIO() + Generator(fp).flatten(msg) + result = fp.getvalue() + self._signed_parts_eq(original, result) + +class TestHeaderRegistry(TestEmailBase): + # See issue gh-93010. + def test_HeaderRegistry(self): + reg = HeaderRegistry() + a = reg('Content-Disposition', 'attachment; 0*00="foo"') + self.assertIsInstance(a.defects[0], errors.InvalidHeaderDefect) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py new file mode 100644 index 00000000000..c75a842c335 --- /dev/null +++ b/Lib/test/test_email/test_generator.py @@ -0,0 +1,477 @@ +import io +import textwrap +import unittest +from email import message_from_string, message_from_bytes +from email.message import EmailMessage +from email.generator import Generator, BytesGenerator +from email.headerregistry import Address +from email import policy +import email.errors +from test.test_email import TestEmailBase, parameterize + + +@parameterize +class TestGeneratorBase: + + policy = policy.default + + def msgmaker(self, msg, policy=None): + policy = self.policy if policy is None else policy + return self.msgfunc(msg, policy=policy) + + refold_long_expected = { + 0: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From: nobody_you_want_to_know@example.com + Subject: We the willing led by the unknowing are doing the + impossible for the ungrateful. We have done so much for so long with so little + we are now qualified to do anything with nothing. + + None + """), + 40: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From: + nobody_you_want_to_know@example.com + Subject: We the willing led by the + unknowing are doing the impossible for + the ungrateful. We have done so much + for so long with so little we are now + qualified to do anything with nothing. + + None + """), + 20: textwrap.dedent("""\ + To: + whom_it_may_concern@example.com + From: + nobody_you_want_to_know@example.com + Subject: We the + willing led by the + unknowing are doing + the impossible for + the ungrateful. We + have done so much + for so long with so + little we are now + qualified to do + anything with + nothing. + + None + """), + } + refold_long_expected[100] = refold_long_expected[0] + + refold_all_expected = refold_long_expected.copy() + refold_all_expected[0] = ( + "To: whom_it_may_concern@example.com\n" + "From: nobody_you_want_to_know@example.com\n" + "Subject: We the willing led by the unknowing are doing the " + "impossible for the ungrateful. We have done so much for " + "so long with so little we are now qualified to do anything " + "with nothing.\n" + "\n" + "None\n") + refold_all_expected[100] = ( + "To: whom_it_may_concern@example.com\n" + "From: nobody_you_want_to_know@example.com\n" + "Subject: We the willing led by the unknowing are doing the " + "impossible for the ungrateful. We have\n" + " done so much for so long with so little we are now qualified " + "to do anything with nothing.\n" + "\n" + "None\n") + + length_params = [n for n in refold_long_expected] + + def length_as_maxheaderlen_parameter(self, n): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n, policy=self.policy) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n])) + + def length_as_max_line_length_policy(self, n): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=n)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n])) + + def length_as_maxheaderlen_parm_overrides_policy(self, n): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n, + policy=self.policy.clone(max_line_length=10)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n])) + + def length_as_max_line_length_with_refold_none_does_not_fold(self, n): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(refold_source='none', + max_line_length=n)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + + def length_as_max_line_length_with_refold_all_folds(self, n): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(refold_source='all', + max_line_length=n)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_all_expected[n])) + + def test_crlf_control_via_policy(self): + source = "Subject: test\r\n\r\ntest body\r\n" + expected = source + msg = self.msgmaker(self.typ(source)) + s = self.ioclass() + g = self.genclass(s, policy=policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_flatten_linesep_overrides_policy(self): + source = "Subject: test\n\ntest body\n" + expected = source + msg = self.msgmaker(self.typ(source)) + s = self.ioclass() + g = self.genclass(s, policy=policy.SMTP) + g.flatten(msg, linesep='\n') + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_flatten_linesep(self): + source = 'Subject: one\n two\r three\r\n four\r\n\r\ntest body\r\n' + msg = self.msgmaker(self.typ(source)) + self.assertEqual(msg['Subject'], 'one two three four') + + expected = 'Subject: one\n two\n three\n four\n\ntest body\n' + s = self.ioclass() + g = self.genclass(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + expected = 'Subject: one two three four\n\ntest body\n' + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(refold_source='all')) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_flatten_control_linesep(self): + source = 'Subject: one\v two\f three\x1c four\x1d five\x1e six\r\n\r\ntest body\r\n' + msg = self.msgmaker(self.typ(source)) + self.assertEqual(msg['Subject'], 'one\v two\f three\x1c four\x1d five\x1e six') + + expected = 'Subject: one\v two\f three\x1c four\x1d five\x1e six\n\ntest body\n' + s = self.ioclass() + g = self.genclass(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(refold_source='all')) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_set_mangle_from_via_policy(self): + source = textwrap.dedent("""\ + Subject: test that + from is mangled in the body! + + From time to time I write a rhyme. + """) + variants = ( + (None, True), + (policy.compat32, True), + (policy.default, False), + (policy.default.clone(mangle_from_=True), True), + ) + for p, mangle in variants: + expected = source.replace('From ', '>From ') if mangle else source + with self.subTest(policy=p, mangle_from_=mangle): + msg = self.msgmaker(self.typ(source)) + s = self.ioclass() + g = self.genclass(s, policy=p) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_compat32_max_line_length_does_not_fold_when_none(self): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=policy.compat32.clone(max_line_length=None)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + + def test_rfc2231_wrapping(self): + # This is pretty much just to make sure we don't have an infinite + # loop; I don't expect anyone to hit this in the field. + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename*0*=us-ascii''afilename; + filename*1*=longenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=33)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_rfc2231_wrapping_switches_to_default_len_if_too_narrow(self): + # This is just to make sure we don't have an infinite loop; I don't + # expect anyone to hit this in the field, so I'm not bothering to make + # the result optimal (the encoding isn't needed). + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: + attachment; + filename*0*=us-ascii''afilenamelongenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=20)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_keep_encoded_newlines(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=80)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_keep_long_encoded_newlines(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Subject: Bad subject + =?utf-8?q?=0A?=Bcc: + injection@example.com + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + +class TestGenerator(TestGeneratorBase, TestEmailBase): + + msgfunc = staticmethod(message_from_string) + genclass = Generator + ioclass = io.StringIO + typ = str + + def test_flatten_unicode_linesep(self): + source = 'Subject: one\x85 two\u2028 three\u2029 four\r\n\r\ntest body\r\n' + msg = self.msgmaker(self.typ(source)) + self.assertEqual(msg['Subject'], 'one\x85 two\u2028 three\u2029 four') + + expected = 'Subject: =?utf-8?b?b25lwoUgdHdv4oCoIHRocmVl4oCp?= four\n\ntest body\n' + s = self.ioclass() + g = self.genclass(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(refold_source='all')) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_verify_generated_headers(self): + """gh-121650: by default the generator prevents header injection""" + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + for text in ( + 'Value\r\nBad Injection\r\n', + 'NoNewLine' + ): + with self.subTest(text=text): + message = message_from_string( + "Header: Value\r\n\r\nBody", + policy=self.policy, + ) + + del message['Header'] + message['Header'] = LiteralHeader(text) + + with self.assertRaises(email.errors.HeaderWriteError): + message.as_string() + + +class TestBytesGenerator(TestGeneratorBase, TestEmailBase): + + msgfunc = staticmethod(message_from_bytes) + genclass = BytesGenerator + ioclass = io.BytesIO + typ = lambda self, x: x.encode('ascii') + + def test_defaults_handle_spaces_between_encoded_words_when_folded(self): + source = ("Уведомление о принятии в работу обращения для" + " подключения услуги") + expected = ('Subject: =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtSDQviDQv9GA0LjQvdGP0YLQuNC4?=\n' + ' =?utf-8?b?INCyINGA0LDQsdC+0YLRgyDQvtCx0YDQsNGJ0LXQvdC40Y8g0LTQu9GPINC/0L4=?=\n' + ' =?utf-8?b?0LTQutC70Y7Rh9C10L3QuNGPINGD0YHQu9GD0LPQuA==?=\n\n').encode('ascii') + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_defaults_handle_spaces_when_encoded_words_is_folded_in_middle(self): + source = ('A very long long long long long long long long long long long long ' + 'long long long long long long long long long long long súmmäry') + expected = ('Subject: A very long long long long long long long long long long long long\n' + ' long long long long long long long long long long long =?utf-8?q?s=C3=BAmm?=\n' + ' =?utf-8?q?=C3=A4ry?=\n\n').encode('ascii') + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_defaults_handle_spaces_at_start_of_subject(self): + source = " Уведомление" + expected = b"Subject: =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtQ==?=\n\n" + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_defaults_handle_spaces_at_start_of_continuation_line(self): + source = " ф ффффффффффффффффффф ф ф" + expected = (b"Subject: " + b"=?utf-8?b?0YQg0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YQ=?=\n" + b" =?utf-8?b?INGEINGE?=\n\n") + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_cte_type_7bit_handles_unknown_8bit(self): + source = ("Subject: Maintenant je vous présente mon " + "collègue\n\n").encode('utf-8') + expected = ('Subject: Maintenant je vous =?unknown-8bit?q?' + 'pr=C3=A9sente_mon_coll=C3=A8gue?=\n\n').encode('ascii') + msg = message_from_bytes(source) + s = io.BytesIO() + g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit')) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_cte_type_7bit_transforms_8bit_cte(self): + source = textwrap.dedent("""\ + From: foo@bar.com + To: Dinsdale + Subject: Nudge nudge, wink, wink + Mime-Version: 1.0 + Content-Type: text/plain; charset="latin-1" + Content-Transfer-Encoding: 8bit + + oh là là, know what I mean, know what I mean? + """).encode('latin1') + msg = message_from_bytes(source) + expected = textwrap.dedent("""\ + From: foo@bar.com + To: Dinsdale + Subject: Nudge nudge, wink, wink + Mime-Version: 1.0 + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + oh l=E0 l=E0, know what I mean, know what I mean? + """).encode('ascii') + s = io.BytesIO() + g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit', + linesep='\n')) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_smtputf8_policy(self): + msg = EmailMessage() + msg['From'] = "Páolo " + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + msg.set_content("oh là là, know what I mean, know what I mean?") + expected = textwrap.dedent("""\ + From: Páolo + To: Dinsdale + Subject: Nudge nudge, wink, wink \u1F609 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + oh là là, know what I mean, know what I mean? + """).encode('utf-8').replace(b'\n', b'\r\n') + s = io.BytesIO() + g = BytesGenerator(s, policy=policy.SMTPUTF8) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_smtp_policy(self): + msg = EmailMessage() + msg["From"] = Address(addr_spec="foo@bar.com", display_name="Páolo") + msg["To"] = Address(addr_spec="bar@foo.com", display_name="Dinsdale") + msg["Subject"] = "Nudge nudge, wink, wink" + msg.set_content("oh boy, know what I mean, know what I mean?") + expected = textwrap.dedent("""\ + From: =?utf-8?q?P=C3=A1olo?= + To: Dinsdale + Subject: Nudge nudge, wink, wink + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + MIME-Version: 1.0 + + oh boy, know what I mean, know what I mean? + """).encode().replace(b"\n", b"\r\n") + s = io.BytesIO() + g = BytesGenerator(s, policy=policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py new file mode 100644 index 00000000000..d2c571299bc --- /dev/null +++ b/Lib/test/test_email/test_headerregistry.py @@ -0,0 +1,1822 @@ +import datetime +import textwrap +import unittest +from email import errors +from email import policy +from email.message import Message +from test.test_email import TestEmailBase, parameterize +from email import headerregistry +from email.headerregistry import Address, Group +from email.header import decode_header +from test.support import ALWAYS_EQ + + +DITTO = object() + + +class TestHeaderRegistry(TestEmailBase): + + def test_arbitrary_name_unstructured(self): + factory = headerregistry.HeaderRegistry() + h = factory('foobar', 'test') + self.assertIsInstance(h, headerregistry.BaseHeader) + self.assertIsInstance(h, headerregistry.UnstructuredHeader) + + def test_name_case_ignored(self): + factory = headerregistry.HeaderRegistry() + # Whitebox check that test is valid + self.assertNotIn('Subject', factory.registry) + h = factory('Subject', 'test') + self.assertIsInstance(h, headerregistry.BaseHeader) + self.assertIsInstance(h, headerregistry.UniqueUnstructuredHeader) + + class FooBase: + def __init__(self, *args, **kw): + pass + + def test_override_default_base_class(self): + factory = headerregistry.HeaderRegistry(base_class=self.FooBase) + h = factory('foobar', 'test') + self.assertIsInstance(h, self.FooBase) + self.assertIsInstance(h, headerregistry.UnstructuredHeader) + + class FooDefault: + parse = headerregistry.UnstructuredHeader.parse + + def test_override_default_class(self): + factory = headerregistry.HeaderRegistry(default_class=self.FooDefault) + h = factory('foobar', 'test') + self.assertIsInstance(h, headerregistry.BaseHeader) + self.assertIsInstance(h, self.FooDefault) + + def test_override_default_class_only_overrides_default(self): + factory = headerregistry.HeaderRegistry(default_class=self.FooDefault) + h = factory('subject', 'test') + self.assertIsInstance(h, headerregistry.BaseHeader) + self.assertIsInstance(h, headerregistry.UniqueUnstructuredHeader) + + def test_dont_use_default_map(self): + factory = headerregistry.HeaderRegistry(use_default_map=False) + h = factory('subject', 'test') + self.assertIsInstance(h, headerregistry.BaseHeader) + self.assertIsInstance(h, headerregistry.UnstructuredHeader) + + def test_map_to_type(self): + factory = headerregistry.HeaderRegistry() + h1 = factory('foobar', 'test') + factory.map_to_type('foobar', headerregistry.UniqueUnstructuredHeader) + h2 = factory('foobar', 'test') + self.assertIsInstance(h1, headerregistry.BaseHeader) + self.assertIsInstance(h1, headerregistry.UnstructuredHeader) + self.assertIsInstance(h2, headerregistry.BaseHeader) + self.assertIsInstance(h2, headerregistry.UniqueUnstructuredHeader) + + +class TestHeaderBase(TestEmailBase): + + factory = headerregistry.HeaderRegistry() + + def make_header(self, name, value): + return self.factory(name, value) + + +class TestBaseHeaderFeatures(TestHeaderBase): + + def test_str(self): + h = self.make_header('subject', 'this is a test') + self.assertIsInstance(h, str) + self.assertEqual(h, 'this is a test') + self.assertEqual(str(h), 'this is a test') + + def test_substr(self): + h = self.make_header('subject', 'this is a test') + self.assertEqual(h[5:7], 'is') + + def test_has_name(self): + h = self.make_header('subject', 'this is a test') + self.assertEqual(h.name, 'subject') + + def _test_attr_ro(self, attr): + h = self.make_header('subject', 'this is a test') + with self.assertRaises(AttributeError): + setattr(h, attr, 'foo') + + def test_name_read_only(self): + self._test_attr_ro('name') + + def test_defects_read_only(self): + self._test_attr_ro('defects') + + def test_defects_is_tuple(self): + h = self.make_header('subject', 'this is a test') + self.assertEqual(len(h.defects), 0) + self.assertIsInstance(h.defects, tuple) + # Make sure it is still true when there are defects. + h = self.make_header('date', '') + self.assertEqual(len(h.defects), 1) + self.assertIsInstance(h.defects, tuple) + + # XXX: FIXME + #def test_CR_in_value(self): + # # XXX: this also re-raises the issue of embedded headers, + # # need test and solution for that. + # value = '\r'.join(['this is', ' a test']) + # h = self.make_header('subject', value) + # self.assertEqual(h, value) + # self.assertDefectsEqual(h.defects, [errors.ObsoleteHeaderDefect]) + + +@parameterize +class TestUnstructuredHeader(TestHeaderBase): + + def string_as_value(self, + source, + decoded, + *args): + # TODO: RUSTPYTHON; RustPython currently does not support non-utf8 encoding + if source == '=?gb2312?b?1eLKx9bQzsSy4srUo6E=?=': + raise unittest.SkipTest("TODO: RUSTPYTHON; RustPython currently does not support non-utf8 encoding") + # RUSTPYTHON: End + # ------------------------------------------------------------------ + l = len(args) + defects = args[0] if l>0 else [] + header = 'Subject:' + (' ' if source else '') + folded = header + (args[1] if l>1 else source) + '\n' + h = self.make_header('Subject', source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + self.assertEqual(h.fold(policy=policy.default), folded) + + string_params = { + + 'rfc2047_simple_quopri': ( + '=?utf-8?q?this_is_a_test?=', + 'this is a test', + [], + 'this is a test'), + + 'rfc2047_gb2312_base64': ( + '=?gb2312?b?1eLKx9bQzsSy4srUo6E=?=', + '\u8fd9\u662f\u4e2d\u6587\u6d4b\u8bd5\uff01', + [], + '=?utf-8?b?6L+Z5piv5Lit5paH5rWL6K+V77yB?='), + + 'rfc2047_simple_nonascii_quopri': ( + '=?utf-8?q?=C3=89ric?=', + 'Éric'), + + 'rfc2047_quopri_with_regular_text': ( + 'The =?utf-8?q?=C3=89ric=2C?= Himself', + 'The Éric, Himself'), + + } + + +@parameterize +class TestDateHeader(TestHeaderBase): + + datestring = 'Sun, 23 Sep 2001 20:10:55 -0700' + utcoffset = datetime.timedelta(hours=-7) + tz = datetime.timezone(utcoffset) + dt = datetime.datetime(2001, 9, 23, 20, 10, 55, tzinfo=tz) + + def test_parse_date(self): + h = self.make_header('date', self.datestring) + self.assertEqual(h, self.datestring) + self.assertEqual(h.datetime, self.dt) + self.assertEqual(h.datetime.utcoffset(), self.utcoffset) + self.assertEqual(h.defects, ()) + + def test_set_from_datetime(self): + h = self.make_header('date', self.dt) + self.assertEqual(h, self.datestring) + self.assertEqual(h.datetime, self.dt) + self.assertEqual(h.defects, ()) + + def test_date_header_properties(self): + h = self.make_header('date', self.datestring) + self.assertIsInstance(h, headerregistry.UniqueDateHeader) + self.assertEqual(h.max_count, 1) + self.assertEqual(h.defects, ()) + + def test_resent_date_header_properties(self): + h = self.make_header('resent-date', self.datestring) + self.assertIsInstance(h, headerregistry.DateHeader) + self.assertEqual(h.max_count, None) + self.assertEqual(h.defects, ()) + + def test_no_value_is_defect(self): + h = self.make_header('date', '') + self.assertEqual(len(h.defects), 1) + self.assertIsInstance(h.defects[0], errors.HeaderMissingRequiredValue) + + def test_invalid_date_format(self): + s = 'Not a date header' + h = self.make_header('date', s) + self.assertEqual(h, s) + self.assertIsNone(h.datetime) + self.assertEqual(len(h.defects), 1) + self.assertIsInstance(h.defects[0], errors.InvalidDateDefect) + + def test_invalid_date_value(self): + s = 'Tue, 06 Jun 2017 27:39:33 +0600' + h = self.make_header('date', s) + self.assertEqual(h, s) + self.assertIsNone(h.datetime) + self.assertEqual(len(h.defects), 1) + self.assertIsInstance(h.defects[0], errors.InvalidDateDefect) + + def test_datetime_read_only(self): + h = self.make_header('date', self.datestring) + with self.assertRaises(AttributeError): + h.datetime = 'foo' + + def test_set_date_header_from_datetime(self): + m = Message(policy=policy.default) + m['Date'] = self.dt + self.assertEqual(m['Date'], self.datestring) + self.assertEqual(m['Date'].datetime, self.dt) + + +@parameterize +class TestContentTypeHeader(TestHeaderBase): + + def content_type_as_value(self, + source, + content_type, + maintype, + subtype, + *args): + l = len(args) + parmdict = args[0] if l>0 else {} + defects = args[1] if l>1 else [] + decoded = args[2] if l>2 and args[2] is not DITTO else source + header = 'Content-Type:' + ' ' if source else '' + folded = args[3] if l>3 else header + decoded + '\n' + h = self.make_header('Content-Type', source) + self.assertEqual(h.content_type, content_type) + self.assertEqual(h.maintype, maintype) + self.assertEqual(h.subtype, subtype) + self.assertEqual(h.params, parmdict) + with self.assertRaises(TypeError): + h.params['abc'] = 'xyz' # make sure params is read-only. + self.assertDefectsEqual(h.defects, defects) + self.assertEqual(h, decoded) + self.assertEqual(h.fold(policy=policy.default), folded) + + content_type_params = { + + # Examples from RFC 2045. + + 'RFC_2045_1': ( + 'text/plain; charset=us-ascii (Plain text)', + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii'}, + [], + 'text/plain; charset="us-ascii"'), + + 'RFC_2045_2': ( + 'text/plain; charset=us-ascii', + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii'}, + [], + 'text/plain; charset="us-ascii"'), + + 'RFC_2045_3': ( + 'text/plain; charset="us-ascii"', + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii'}), + + # RFC 2045 5.2 says syntactically invalid values are to be treated as + # text/plain. + + 'no_subtype_in_content_type': ( + 'text/', + 'text/plain', + 'text', + 'plain', + {}, + [errors.InvalidHeaderDefect]), + + 'no_slash_in_content_type': ( + 'foo', + 'text/plain', + 'text', + 'plain', + {}, + [errors.InvalidHeaderDefect]), + + 'junk_text_in_content_type': ( + '', + 'text/plain', + 'text', + 'plain', + {}, + [errors.InvalidHeaderDefect]), + + 'too_many_slashes_in_content_type': ( + 'image/jpeg/foo', + 'text/plain', + 'text', + 'plain', + {}, + [errors.InvalidHeaderDefect]), + + # But unknown names are OK. We could make non-IANA names a defect, but + # by not doing so we make ourselves future proof. The fact that they + # are unknown will be detectable by the fact that they don't appear in + # the mime_registry...and the application is free to extend that list + # to handle them even if the core library doesn't. + + 'unknown_content_type': ( + 'bad/names', + 'bad/names', + 'bad', + 'names'), + + # The content type is case insensitive, and CFWS is ignored. + + 'mixed_case_content_type': ( + 'ImAge/JPeg', + 'image/jpeg', + 'image', + 'jpeg'), + + 'spaces_in_content_type': ( + ' text / plain ', + 'text/plain', + 'text', + 'plain'), + + 'cfws_in_content_type': ( + '(foo) text (bar)/(baz)plain(stuff)', + 'text/plain', + 'text', + 'plain'), + + # test some parameters (more tests could be added for parameters + # associated with other content types, but since parameter parsing is + # generic they would be redundant for the current implementation). + + 'charset_param': ( + 'text/plain; charset="utf-8"', + 'text/plain', + 'text', + 'plain', + {'charset': 'utf-8'}), + + 'capitalized_charset': ( + 'text/plain; charset="US-ASCII"', + 'text/plain', + 'text', + 'plain', + {'charset': 'US-ASCII'}), + + 'unknown_charset': ( + 'text/plain; charset="fOo"', + 'text/plain', + 'text', + 'plain', + {'charset': 'fOo'}), + + 'capitalized_charset_param_name_and_comment': ( + 'text/plain; (interjection) Charset="utf-8"', + 'text/plain', + 'text', + 'plain', + {'charset': 'utf-8'}, + [], + # Should the parameter name be lowercased here? + 'text/plain; Charset="utf-8"'), + + # Since this is pretty much the ur-mimeheader, we'll put all the tests + # that exercise the parameter parsing and formatting here. Note that + # when we refold we may canonicalize, so things like whitespace, + # quoting, and rfc2231 encoding may change from what was in the input + # header. + + 'unquoted_param_value': ( + 'text/plain; title=foo', + 'text/plain', + 'text', + 'plain', + {'title': 'foo'}, + [], + 'text/plain; title="foo"', + ), + + 'param_value_with_tspecials': ( + 'text/plain; title="(bar)foo blue"', + 'text/plain', + 'text', + 'plain', + {'title': '(bar)foo blue'}), + + 'param_with_extra_quoted_whitespace': ( + 'text/plain; title=" a loong way \t home "', + 'text/plain', + 'text', + 'plain', + {'title': ' a loong way \t home '}), + + 'bad_params': ( + 'blarg; baz; boo', + 'text/plain', + 'text', + 'plain', + {'baz': '', 'boo': ''}, + [errors.InvalidHeaderDefect]*3), + + 'spaces_around_param_equals': ( + 'Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"', + 'multipart/mixed', + 'multipart', + 'mixed', + {'boundary': 'CPIMSSMTPC06p5f3tG'}, + [], + 'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"', + ), + + 'spaces_around_semis': ( + ('image/jpeg; name="wibble.JPG" ; x-mac-type="4A504547" ; ' + 'x-mac-creator="474B4F4E"'), + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'wibble.JPG', + 'x-mac-type': '4A504547', + 'x-mac-creator': '474B4F4E'}, + [], + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"'), + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' + ' x-mac-creator="474B4F4E"\n'), + ), + + 'lots_of_mime_params': ( + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'wibble.JPG', + 'x-mac-type': '4A504547', + 'x-mac-creator': '474B4F4E', + 'x-extrastuff': 'make it longer'}, + [], + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + # In this case the whole of the MimeParameters does *not* fit + # one one line, so we break at a lower syntactic level. + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' + ' x-mac-creator="474B4F4E"; x-extrastuff="make it longer"\n'), + ), + + 'semis_inside_quotes': ( + 'image/jpeg; name="Jim&&Jill"', + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'Jim&&Jill'}), + + 'single_quotes_inside_quotes': ( + 'image/jpeg; name="Jim \'Bob\' Jill"', + 'image/jpeg', + 'image', + 'jpeg', + {'name': "Jim 'Bob' Jill"}), + + 'double_quotes_inside_quotes': ( + r'image/jpeg; name="Jim \"Bob\" Jill"', + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'Jim "Bob" Jill'}, + [], + r'image/jpeg; name="Jim \"Bob\" Jill"'), + + 'non_ascii_in_params': ( + ('foo\xa7/bar; b\xa7r=two; ' + 'baz=thr\xa7e'.encode('latin-1').decode('us-ascii', + 'surrogateescape')), + 'foo\uFFFD/bar', + 'foo\uFFFD', + 'bar', + {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'}, + [errors.UndecodableBytesDefect]*3, + 'foo�/bar; b�r="two"; baz="thr�e"', + # XXX Two bugs here: the mime type is not allowed to be an encoded + # word, and we shouldn't be emitting surrogates in the parameter + # names. But I don't know what the behavior should be here, so I'm + # punting for now. In practice this is unlikely to be encountered + # since headers with binary in them only come from a binary source + # and are almost certain to be re-emitted without refolding. + 'Content-Type: =?unknown-8bit?q?foo=A7?=/bar; b\udca7r="two";\n' + " baz*=unknown-8bit''thr%A7e\n", + ), + + # RFC 2231 parameter tests. + + 'rfc2231_segmented_normal_values': ( + 'image/jpeg; name*0="abc"; name*1=".html"', + 'image/jpeg', + 'image', + 'jpeg', + {'name': "abc.html"}, + [], + 'image/jpeg; name="abc.html"'), + + 'quotes_inside_rfc2231_value': ( + r'image/jpeg; bar*0="baz\"foobar"; bar*1="\"baz"', + 'image/jpeg', + 'image', + 'jpeg', + {'bar': 'baz"foobar"baz'}, + [], + r'image/jpeg; bar="baz\"foobar\"baz"'), + + 'non_ascii_rfc2231_value': ( + ('text/plain; charset=us-ascii; ' + "title*=us-ascii'en'This%20is%20" + 'not%20f\xa7n').encode('latin-1').decode('us-ascii', + 'surrogateescape'), + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'}, + [errors.UndecodableBytesDefect], + 'text/plain; charset="us-ascii"; title="This is not f�n"', + 'Content-Type: text/plain; charset="us-ascii";\n' + " title*=unknown-8bit''This%20is%20not%20f%A7n\n", + ), + + 'rfc2231_encoded_charset': ( + 'text/plain; charset*=ansi-x3.4-1968\'\'us-ascii', + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii'}, + [], + 'text/plain; charset="us-ascii"'), + + # This follows the RFC: no double quotes around encoded values. + 'rfc2231_encoded_no_double_quotes': ( + ("text/plain;" + "\tname*0*=''This%20is%20;" + "\tname*1*=%2A%2A%2Afun%2A%2A%2A%20;" + '\tname*2="is it not.pdf"'), + 'text/plain', + 'text', + 'plain', + {'name': 'This is ***fun*** is it not.pdf'}, + [], + 'text/plain; name="This is ***fun*** is it not.pdf"', + ), + + # Make sure we also handle it if there are spurious double quotes. + 'rfc2231_encoded_with_double_quotes': ( + ("text/plain;" + '\tname*0*="us-ascii\'\'This%20is%20even%20more%20";' + '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";' + '\tname*2="is it not.pdf"'), + 'text/plain', + 'text', + 'plain', + {'name': 'This is even more ***fun*** is it not.pdf'}, + [errors.InvalidHeaderDefect]*2, + 'text/plain; name="This is even more ***fun*** is it not.pdf"', + ), + + 'rfc2231_single_quote_inside_double_quotes': ( + ('text/plain; charset=us-ascii;' + '\ttitle*0*="us-ascii\'en\'This%20is%20really%20";' + '\ttitle*1*="%2A%2A%2Afun%2A%2A%2A%20";' + '\ttitle*2="isn\'t it!"'), + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii', 'title': "This is really ***fun*** isn't it!"}, + [errors.InvalidHeaderDefect]*2, + ('text/plain; charset="us-ascii"; ' + 'title="This is really ***fun*** isn\'t it!"'), + ('Content-Type: text/plain; charset="us-ascii";\n' + ' title="This is really ***fun*** isn\'t it!"\n'), + ), + + 'rfc2231_single_quote_in_value_with_charset_and_lang': ( + ('application/x-foo;' + "\tname*0*=\"us-ascii'en-us'Frank's\"; name*1*=\" Document\""), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "Frank's Document"}, + [errors.InvalidHeaderDefect]*2, + 'application/x-foo; name="Frank\'s Document"', + ), + + 'rfc2231_single_quote_in_non_encoded_value': ( + ('application/x-foo;' + "\tname*0=\"us-ascii'en-us'Frank's\"; name*1=\" Document\""), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "us-ascii'en-us'Frank's Document"}, + [], + 'application/x-foo; name="us-ascii\'en-us\'Frank\'s Document"', + ), + + 'rfc2231_no_language_or_charset': ( + 'text/plain; NAME*0*=english_is_the_default.html', + 'text/plain', + 'text', + 'plain', + {'name': 'english_is_the_default.html'}, + [errors.InvalidHeaderDefect], + 'text/plain; NAME="english_is_the_default.html"'), + + 'rfc2231_encoded_no_charset': ( + ("text/plain;" + '\tname*0*="\'\'This%20is%20even%20more%20";' + '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";' + '\tname*2="is it.pdf"'), + 'text/plain', + 'text', + 'plain', + {'name': 'This is even more ***fun*** is it.pdf'}, + [errors.InvalidHeaderDefect]*2, + 'text/plain; name="This is even more ***fun*** is it.pdf"', + ), + + 'rfc2231_partly_encoded': ( + ("text/plain;" + '\tname*0*="\'\'This%20is%20even%20more%20";' + '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";' + '\tname*2="is it.pdf"'), + 'text/plain', + 'text', + 'plain', + {'name': 'This is even more ***fun*** is it.pdf'}, + [errors.InvalidHeaderDefect]*2, + 'text/plain; name="This is even more ***fun*** is it.pdf"', + ), + + 'rfc2231_partly_encoded_2': ( + ("text/plain;" + '\tname*0*="\'\'This%20is%20even%20more%20";' + '\tname*1="%2A%2A%2Afun%2A%2A%2A%20";' + '\tname*2="is it.pdf"'), + 'text/plain', + 'text', + 'plain', + {'name': 'This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf'}, + [errors.InvalidHeaderDefect], + ('text/plain;' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"'), + ('Content-Type: text/plain;\n' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is' + ' it.pdf"\n'), + ), + + 'rfc2231_unknown_charset_treated_as_ascii': ( + "text/plain; name*0*=bogus'xx'ascii_is_the_default", + 'text/plain', + 'text', + 'plain', + {'name': 'ascii_is_the_default'}, + [], + 'text/plain; name="ascii_is_the_default"'), + + 'rfc2231_bad_character_in_charset_parameter_value': ( + "text/plain; charset*=ascii''utf-8%F1%F2%F3", + 'text/plain', + 'text', + 'plain', + {'charset': 'utf-8\uFFFD\uFFFD\uFFFD'}, + [errors.UndecodableBytesDefect], + 'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"', + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%F1%F2%F3\n", + ), + + 'rfc2231_utf8_in_supposedly_ascii_charset_parameter_value': ( + "text/plain; charset*=ascii''utf-8%E2%80%9D", + 'text/plain', + 'text', + 'plain', + {'charset': 'utf-8”'}, + [errors.UndecodableBytesDefect], + 'text/plain; charset="utf-8”"', + # XXX Should folding change the charset to utf8? Currently it just + # reproduces the original, which is arguably fine. + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%E2%80%9D\n", + ), + + 'rfc2231_nonascii_in_charset_of_charset_parameter_value': ( + "text/plain; charset*=utf-8”''utf-8%E2%80%9D", + 'text/plain', + 'text', + 'plain', + {'charset': 'utf-8”'}, + [], + 'text/plain; charset="utf-8”"', + "Content-Type: text/plain;" + " charset*=utf-8''utf-8%E2%80%9D\n", + ), + + 'rfc2231_encoded_then_unencoded_segments': ( + ('application/x-foo;' + '\tname*0*="us-ascii\'en-us\'My";' + '\tname*1=" Document";' + '\tname*2=" For You"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': 'My Document For You'}, + [errors.InvalidHeaderDefect], + 'application/x-foo; name="My Document For You"', + ), + + # My reading of the RFC is that this is an invalid header. The RFC + # says that if charset and language information is given, the first + # segment *must* be encoded. + 'rfc2231_unencoded_then_encoded_segments': ( + ('application/x-foo;' + '\tname*0=us-ascii\'en-us\'My;' + '\tname*1*=" Document";' + '\tname*2*=" For You"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': 'My Document For You'}, + [errors.InvalidHeaderDefect]*3, + 'application/x-foo; name="My Document For You"', + ), + + # XXX: I would say this one should default to ascii/en for the + # "encoded" segment, since the first segment is not encoded and is + # in double quotes, making the value a valid non-encoded string. The + # old parser decodes this just like the previous case, which may be the + # better Postel rule, but could equally result in borking headers that + # intentionally have quoted quotes in them. We could get this 98% + # right if we treat it as a quoted string *unless* it matches the + # charset'lang'value pattern exactly *and* there is at least one + # encoded segment. Implementing that algorithm will require some + # refactoring, so I haven't done it (yet). + 'rfc2231_quoted_unencoded_then_encoded_segments': ( + ('application/x-foo;' + '\tname*0="us-ascii\'en-us\'My";' + '\tname*1*=" Document";' + '\tname*2*=" For You"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "us-ascii'en-us'My Document For You"}, + [errors.InvalidHeaderDefect]*2, + 'application/x-foo; name="us-ascii\'en-us\'My Document For You"', + ), + + # Make sure our folding algorithm produces multiple sections correctly. + # We could mix encoded and non-encoded segments, but we don't, we just + # make them all encoded. It might be worth fixing that, since the + # sections can get used for wrapping ascii text. + 'rfc2231_folded_segments_correctly_formatted': ( + ('application/x-foo;' + '\tname="' + "with spaces"*8 + '"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "with spaces"*8}, + [], + 'application/x-foo; name="' + "with spaces"*8 + '"', + "Content-Type: application/x-foo;\n" + " name*0*=us-ascii''with%20spaceswith%20spaceswith%20spaceswith" + "%20spaceswith;\n" + " name*1*=%20spaceswith%20spaceswith%20spaceswith%20spaces\n" + ), + + } + + +@parameterize +class TestContentTransferEncoding(TestHeaderBase): + + def cte_as_value(self, + source, + cte, + *args): + l = len(args) + defects = args[0] if l>0 else [] + decoded = args[1] if l>1 and args[1] is not DITTO else source + header = 'Content-Transfer-Encoding:' + ' ' if source else '' + folded = args[2] if l>2 else header + source + '\n' + h = self.make_header('Content-Transfer-Encoding', source) + self.assertEqual(h.cte, cte) + self.assertDefectsEqual(h.defects, defects) + self.assertEqual(h, decoded) + self.assertEqual(h.fold(policy=policy.default), folded) + + cte_params = { + + 'RFC_2183_1': ( + 'base64', + 'base64',), + + 'no_value': ( + '', + '7bit', + [errors.HeaderMissingRequiredValue], + '', + 'Content-Transfer-Encoding:\n', + ), + + 'junk_after_cte': ( + '7bit and a bunch more', + '7bit', + [errors.InvalidHeaderDefect]), + + 'extra_space_after_cte': ( + 'base64 ', + 'base64', + []), + + } + + +@parameterize +class TestContentDisposition(TestHeaderBase): + + def content_disp_as_value(self, + source, + content_disposition, + *args): + l = len(args) + parmdict = args[0] if l>0 else {} + defects = args[1] if l>1 else [] + decoded = args[2] if l>2 and args[2] is not DITTO else source + header = 'Content-Disposition:' + ' ' if source else '' + folded = args[3] if l>3 else header + source + '\n' + h = self.make_header('Content-Disposition', source) + self.assertEqual(h.content_disposition, content_disposition) + self.assertEqual(h.params, parmdict) + self.assertDefectsEqual(h.defects, defects) + self.assertEqual(h, decoded) + self.assertEqual(h.fold(policy=policy.default), folded) + + content_disp_params = { + + # Examples from RFC 2183. + + 'RFC_2183_1': ( + 'inline', + 'inline',), + + 'RFC_2183_2': ( + ('attachment; filename=genome.jpeg;' + ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500";'), + 'attachment', + {'filename': 'genome.jpeg', + 'modification-date': 'Wed, 12 Feb 1997 16:29:51 -0500'}, + [], + ('attachment; filename="genome.jpeg"; ' + 'modification-date="Wed, 12 Feb 1997 16:29:51 -0500"'), + ('Content-Disposition: attachment; filename="genome.jpeg";\n' + ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500"\n'), + ), + + 'no_value': ( + '', + None, + {}, + [errors.HeaderMissingRequiredValue], + '', + 'Content-Disposition:\n'), + + 'invalid_value': ( + 'ab./k', + 'ab.', + {}, + [errors.InvalidHeaderDefect]), + + 'invalid_value_with_params': ( + 'ab./k; filename="foo"', + 'ab.', + {'filename': 'foo'}, + [errors.InvalidHeaderDefect]), + + 'invalid_parameter_value_with_fws_between_ew': ( + 'attachment; filename="=?UTF-8?Q?Schulbesuchsbest=C3=A4ttigung=2E?=' + ' =?UTF-8?Q?pdf?="', + 'attachment', + {'filename': 'Schulbesuchsbestättigung.pdf'}, + [errors.InvalidHeaderDefect]*3, + ('attachment; filename="Schulbesuchsbestättigung.pdf"'), + ('Content-Disposition: attachment;\n' + ' filename*=utf-8\'\'Schulbesuchsbest%C3%A4ttigung.pdf\n'), + ), + + 'parameter_value_with_fws_between_tokens': ( + 'attachment; filename="File =?utf-8?q?Name?= With Spaces.pdf"', + 'attachment', + {'filename': 'File Name With Spaces.pdf'}, + [errors.InvalidHeaderDefect], + 'attachment; filename="File Name With Spaces.pdf"', + ('Content-Disposition: attachment; filename="File Name With Spaces.pdf"\n'), + ) + } + + +@parameterize +class TestMIMEVersionHeader(TestHeaderBase): + + def version_string_as_MIME_Version(self, + source, + decoded, + version, + major, + minor, + defects): + h = self.make_header('MIME-Version', source) + self.assertEqual(h, decoded) + self.assertEqual(h.version, version) + self.assertEqual(h.major, major) + self.assertEqual(h.minor, minor) + self.assertDefectsEqual(h.defects, defects) + if source: + source = ' ' + source + self.assertEqual(h.fold(policy=policy.default), + 'MIME-Version:' + source + '\n') + + version_string_params = { + + # Examples from the RFC. + + 'RFC_2045_1': ( + '1.0', + '1.0', + '1.0', + 1, + 0, + []), + + 'RFC_2045_2': ( + '1.0 (produced by MetaSend Vx.x)', + '1.0 (produced by MetaSend Vx.x)', + '1.0', + 1, + 0, + []), + + 'RFC_2045_3': ( + '(produced by MetaSend Vx.x) 1.0', + '(produced by MetaSend Vx.x) 1.0', + '1.0', + 1, + 0, + []), + + 'RFC_2045_4': ( + '1.(produced by MetaSend Vx.x)0', + '1.(produced by MetaSend Vx.x)0', + '1.0', + 1, + 0, + []), + + # Other valid values. + + '1_1': ( + '1.1', + '1.1', + '1.1', + 1, + 1, + []), + + '2_1': ( + '2.1', + '2.1', + '2.1', + 2, + 1, + []), + + 'whitespace': ( + '1 .0', + '1 .0', + '1.0', + 1, + 0, + []), + + 'leading_trailing_whitespace_ignored': ( + ' 1.0 ', + ' 1.0 ', + '1.0', + 1, + 0, + []), + + # Recoverable invalid values. We can recover here only because we + # already have a valid value by the time we encounter the garbage. + # Anywhere else, and we don't know where the garbage ends. + + 'non_comment_garbage_after': ( + '1.0 ', + '1.0 ', + '1.0', + 1, + 0, + [errors.InvalidHeaderDefect]), + + # Unrecoverable invalid values. We *could* apply more heuristics to + # get something out of the first two, but doing so is not worth the + # effort. + + 'non_comment_garbage_before': ( + ' 1.0', + ' 1.0', + None, + None, + None, + [errors.InvalidHeaderDefect]), + + 'non_comment_garbage_inside': ( + '1.0', + '1.0', + None, + None, + None, + [errors.InvalidHeaderDefect]), + + 'two_periods': ( + '1..0', + '1..0', + None, + None, + None, + [errors.InvalidHeaderDefect]), + + '2_x': ( + '2.x', + '2.x', + None, # This could be 2, but it seems safer to make it None. + None, + None, + [errors.InvalidHeaderDefect]), + + 'foo': ( + 'foo', + 'foo', + None, + None, + None, + [errors.InvalidHeaderDefect]), + + 'missing': ( + '', + '', + None, + None, + None, + [errors.HeaderMissingRequiredValue]), + + } + + +@parameterize +class TestAddressHeader(TestHeaderBase): + + example_params = { + + 'empty': + ('<>', + [errors.InvalidHeaderDefect], + '<>', + '', + '<>', + '', + '', + None), + + 'address_only': + ('zippy@pinhead.com', + [], + 'zippy@pinhead.com', + '', + 'zippy@pinhead.com', + 'zippy', + 'pinhead.com', + None), + + 'name_and_address': + ('Zaphrod Beblebrux ', + [], + 'Zaphrod Beblebrux ', + 'Zaphrod Beblebrux', + 'zippy@pinhead.com', + 'zippy', + 'pinhead.com', + None), + + 'quoted_local_part': + ('Zaphrod Beblebrux <"foo bar"@pinhead.com>', + [], + 'Zaphrod Beblebrux <"foo bar"@pinhead.com>', + 'Zaphrod Beblebrux', + '"foo bar"@pinhead.com', + 'foo bar', + 'pinhead.com', + None), + + 'quoted_parens_in_name': + (r'"A \(Special\) Person" ', + [], + '"A (Special) Person" ', + 'A (Special) Person', + 'person@dom.ain', + 'person', + 'dom.ain', + None), + + 'quoted_backslashes_in_name': + (r'"Arthur \\Backslash\\ Foobar" ', + [], + r'"Arthur \\Backslash\\ Foobar" ', + r'Arthur \Backslash\ Foobar', + 'person@dom.ain', + 'person', + 'dom.ain', + None), + + 'name_with_dot': + ('John X. Doe ', + [errors.ObsoleteHeaderDefect], + '"John X. Doe" ', + 'John X. Doe', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + 'quoted_strings_in_local_part': + ('""example" example"@example.com', + [errors.InvalidHeaderDefect]*3, + '"example example"@example.com', + '', + '"example example"@example.com', + 'example example', + 'example.com', + None), + + 'escaped_quoted_strings_in_local_part': + (r'"\"example\" example"@example.com', + [], + r'"\"example\" example"@example.com', + '', + r'"\"example\" example"@example.com', + r'"example" example', + 'example.com', + None), + + 'escaped_escapes_in_local_part': + (r'"\\"example\\" example"@example.com', + [errors.InvalidHeaderDefect]*5, + r'"\\example\\\\ example"@example.com', + '', + r'"\\example\\\\ example"@example.com', + r'\example\\ example', + 'example.com', + None), + + 'spaces_in_unquoted_local_part_collapsed': + ('merwok wok @example.com', + [errors.InvalidHeaderDefect]*2, + '"merwok wok"@example.com', + '', + '"merwok wok"@example.com', + 'merwok wok', + 'example.com', + None), + + 'spaces_around_dots_in_local_part_removed': + ('merwok. wok . wok@example.com', + [errors.ObsoleteHeaderDefect], + 'merwok.wok.wok@example.com', + '', + 'merwok.wok.wok@example.com', + 'merwok.wok.wok', + 'example.com', + None), + + 'rfc2047_atom_is_decoded': + ('=?utf-8?q?=C3=89ric?= ', + [], + 'Éric ', + 'Éric', + 'foo@example.com', + 'foo', + 'example.com', + None), + + 'rfc2047_atom_in_phrase_is_decoded': + ('The =?utf-8?q?=C3=89ric=2C?= Himself ', + [], + '"The Éric, Himself" ', + 'The Éric, Himself', + 'foo@example.com', + 'foo', + 'example.com', + None), + + 'rfc2047_atom_in_quoted_string_is_decoded': + ('"=?utf-8?q?=C3=89ric?=" ', + [errors.InvalidHeaderDefect, + errors.InvalidHeaderDefect], + 'Éric ', + 'Éric', + 'foo@example.com', + 'foo', + 'example.com', + None), + + 'name_ending_with_dot_without_space': + ('John X.', + [errors.ObsoleteHeaderDefect], + '"John X." ', + 'John X.', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + 'name_starting_with_dot': + ('. Doe ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '". Doe" ', + '. Doe', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + } + + # XXX: Need many more examples, and in particular some with names in + # trailing comments, which aren't currently handled. comments in + # general are not handled yet. + + def example_as_address(self, source, defects, decoded, display_name, + addr_spec, username, domain, comment): + h = self.make_header('sender', source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + a = h.address + self.assertEqual(str(a), decoded) + self.assertEqual(len(h.groups), 1) + self.assertEqual([a], list(h.groups[0].addresses)) + self.assertEqual([a], list(h.addresses)) + self.assertEqual(a.display_name, display_name) + self.assertEqual(a.addr_spec, addr_spec) + self.assertEqual(a.username, username) + self.assertEqual(a.domain, domain) + # XXX: we have no comment support yet. + #self.assertEqual(a.comment, comment) + + def example_as_group(self, source, defects, decoded, display_name, + addr_spec, username, domain, comment): + source = 'foo: {};'.format(source) + gdecoded = 'foo: {};'.format(decoded) if decoded else 'foo:;' + h = self.make_header('to', source) + self.assertEqual(h, gdecoded) + self.assertDefectsEqual(h.defects, defects) + self.assertEqual(h.groups[0].addresses, h.addresses) + self.assertEqual(len(h.groups), 1) + self.assertEqual(len(h.addresses), 1) + a = h.addresses[0] + self.assertEqual(str(a), decoded) + self.assertEqual(a.display_name, display_name) + self.assertEqual(a.addr_spec, addr_spec) + self.assertEqual(a.username, username) + self.assertEqual(a.domain, domain) + + def test_simple_address_list(self): + value = ('Fred , foo@example.com, ' + '"Harry W. Hastings" ') + h = self.make_header('to', value) + self.assertEqual(h, value) + self.assertEqual(len(h.groups), 3) + self.assertEqual(len(h.addresses), 3) + for i in range(3): + self.assertEqual(h.groups[i].addresses[0], h.addresses[i]) + self.assertEqual(str(h.addresses[0]), 'Fred ') + self.assertEqual(str(h.addresses[1]), 'foo@example.com') + self.assertEqual(str(h.addresses[2]), + '"Harry W. Hastings" ') + self.assertEqual(h.addresses[2].display_name, + 'Harry W. Hastings') + + def test_complex_address_list(self): + examples = list(self.example_params.values()) + source = ('dummy list:;, another: (empty);,' + + ', '.join([x[0] for x in examples[:4]]) + ', ' + + r'"A \"list\"": ' + + ', '.join([x[0] for x in examples[4:6]]) + ';,' + + ', '.join([x[0] for x in examples[6:]]) + ) + # XXX: the fact that (empty) disappears here is a potential API design + # bug. We don't currently have a way to preserve comments. + expected = ('dummy list:;, another:;, ' + + ', '.join([x[2] for x in examples[:4]]) + ', ' + + r'"A \"list\"": ' + + ', '.join([x[2] for x in examples[4:6]]) + ';, ' + + ', '.join([x[2] for x in examples[6:]]) + ) + + h = self.make_header('to', source) + self.assertEqual(h.split(','), expected.split(',')) + self.assertEqual(h, expected) + self.assertEqual(len(h.groups), 7 + len(examples) - 6) + self.assertEqual(h.groups[0].display_name, 'dummy list') + self.assertEqual(h.groups[1].display_name, 'another') + self.assertEqual(h.groups[6].display_name, 'A "list"') + self.assertEqual(len(h.addresses), len(examples)) + for i in range(4): + self.assertIsNone(h.groups[i+2].display_name) + self.assertEqual(str(h.groups[i+2].addresses[0]), examples[i][2]) + for i in range(7, 7 + len(examples) - 6): + self.assertIsNone(h.groups[i].display_name) + self.assertEqual(str(h.groups[i].addresses[0]), examples[i-1][2]) + for i in range(len(examples)): + self.assertEqual(str(h.addresses[i]), examples[i][2]) + self.assertEqual(h.addresses[i].addr_spec, examples[i][4]) + + def test_address_read_only(self): + h = self.make_header('sender', 'abc@xyz.com') + with self.assertRaises(AttributeError): + h.address = 'foo' + + def test_addresses_read_only(self): + h = self.make_header('sender', 'abc@xyz.com') + with self.assertRaises(AttributeError): + h.addresses = 'foo' + + def test_groups_read_only(self): + h = self.make_header('sender', 'abc@xyz.com') + with self.assertRaises(AttributeError): + h.groups = 'foo' + + def test_addresses_types(self): + source = 'me ' + h = self.make_header('to', source) + self.assertIsInstance(h.addresses, tuple) + self.assertIsInstance(h.addresses[0], Address) + + def test_groups_types(self): + source = 'me ' + h = self.make_header('to', source) + self.assertIsInstance(h.groups, tuple) + self.assertIsInstance(h.groups[0], Group) + + def test_set_from_Address(self): + h = self.make_header('to', Address('me', 'foo', 'example.com')) + self.assertEqual(h, 'me ') + + def test_set_from_Address_list(self): + h = self.make_header('to', [Address('me', 'foo', 'example.com'), + Address('you', 'bar', 'example.com')]) + self.assertEqual(h, 'me , you ') + + def test_set_from_Address_and_Group_list(self): + h = self.make_header('to', [Address('me', 'foo', 'example.com'), + Group('bing', [Address('fiz', 'z', 'b.com'), + Address('zif', 'f', 'c.com')]), + Address('you', 'bar', 'example.com')]) + self.assertEqual(h, 'me , bing: fiz , ' + 'zif ;, you ') + self.assertEqual(h.fold(policy=policy.default.clone(max_line_length=40)), + 'to: me ,\n' + ' bing: fiz , zif ;,\n' + ' you \n') + + def test_set_from_Group_list(self): + h = self.make_header('to', [Group('bing', [Address('fiz', 'z', 'b.com'), + Address('zif', 'f', 'c.com')])]) + self.assertEqual(h, 'bing: fiz , zif ;') + + +class TestAddressAndGroup(TestEmailBase): + + def _test_attr_ro(self, obj, attr): + with self.assertRaises(AttributeError): + setattr(obj, attr, 'foo') + + def test_address_display_name_ro(self): + self._test_attr_ro(Address('foo', 'bar', 'baz'), 'display_name') + + def test_address_username_ro(self): + self._test_attr_ro(Address('foo', 'bar', 'baz'), 'username') + + def test_address_domain_ro(self): + self._test_attr_ro(Address('foo', 'bar', 'baz'), 'domain') + + def test_group_display_name_ro(self): + self._test_attr_ro(Group('foo'), 'display_name') + + def test_group_addresses_ro(self): + self._test_attr_ro(Group('foo'), 'addresses') + + def test_address_from_username_domain(self): + a = Address('foo', 'bar', 'baz') + self.assertEqual(a.display_name, 'foo') + self.assertEqual(a.username, 'bar') + self.assertEqual(a.domain, 'baz') + self.assertEqual(a.addr_spec, 'bar@baz') + self.assertEqual(str(a), 'foo ') + + def test_address_from_addr_spec(self): + a = Address('foo', addr_spec='bar@baz') + self.assertEqual(a.display_name, 'foo') + self.assertEqual(a.username, 'bar') + self.assertEqual(a.domain, 'baz') + self.assertEqual(a.addr_spec, 'bar@baz') + self.assertEqual(str(a), 'foo ') + + def test_address_with_no_display_name(self): + a = Address(addr_spec='bar@baz') + self.assertEqual(a.display_name, '') + self.assertEqual(a.username, 'bar') + self.assertEqual(a.domain, 'baz') + self.assertEqual(a.addr_spec, 'bar@baz') + self.assertEqual(str(a), 'bar@baz') + + def test_null_address(self): + a = Address() + self.assertEqual(a.display_name, '') + self.assertEqual(a.username, '') + self.assertEqual(a.domain, '') + self.assertEqual(a.addr_spec, '<>') + self.assertEqual(str(a), '<>') + + def test_domain_only(self): + # This isn't really a valid address. + a = Address(domain='buzz') + self.assertEqual(a.display_name, '') + self.assertEqual(a.username, '') + self.assertEqual(a.domain, 'buzz') + self.assertEqual(a.addr_spec, '@buzz') + self.assertEqual(str(a), '@buzz') + + def test_username_only(self): + # This isn't really a valid address. + a = Address(username='buzz') + self.assertEqual(a.display_name, '') + self.assertEqual(a.username, 'buzz') + self.assertEqual(a.domain, '') + self.assertEqual(a.addr_spec, 'buzz') + self.assertEqual(str(a), 'buzz') + + def test_display_name_only(self): + a = Address('buzz') + self.assertEqual(a.display_name, 'buzz') + self.assertEqual(a.username, '') + self.assertEqual(a.domain, '') + self.assertEqual(a.addr_spec, '<>') + self.assertEqual(str(a), 'buzz <>') + + def test_quoting(self): + # Ideally we'd check every special individually, but I'm not up for + # writing that many tests. + a = Address('Sara J.', 'bad name', 'example.com') + self.assertEqual(a.display_name, 'Sara J.') + self.assertEqual(a.username, 'bad name') + self.assertEqual(a.domain, 'example.com') + self.assertEqual(a.addr_spec, '"bad name"@example.com') + self.assertEqual(str(a), '"Sara J." <"bad name"@example.com>') + + def test_il8n(self): + a = Address('Éric', 'wok', 'exàmple.com') + self.assertEqual(a.display_name, 'Éric') + self.assertEqual(a.username, 'wok') + self.assertEqual(a.domain, 'exàmple.com') + self.assertEqual(a.addr_spec, 'wok@exàmple.com') + self.assertEqual(str(a), 'Éric ') + + # XXX: there is an API design issue that needs to be solved here. + #def test_non_ascii_username_raises(self): + # with self.assertRaises(ValueError): + # Address('foo', 'wők', 'example.com') + + def test_crlf_in_constructor_args_raises(self): + cases = ( + dict(display_name='foo\r'), + dict(display_name='foo\n'), + dict(display_name='foo\r\n'), + dict(domain='example.com\r'), + dict(domain='example.com\n'), + dict(domain='example.com\r\n'), + dict(username='wok\r'), + dict(username='wok\n'), + dict(username='wok\r\n'), + dict(addr_spec='wok@example.com\r'), + dict(addr_spec='wok@example.com\n'), + dict(addr_spec='wok@example.com\r\n') + ) + for kwargs in cases: + with self.subTest(kwargs=kwargs), self.assertRaisesRegex(ValueError, "invalid arguments"): + Address(**kwargs) + + def test_non_ascii_username_in_addr_spec_raises(self): + with self.assertRaises(ValueError): + Address('foo', addr_spec='wők@example.com') + + def test_address_addr_spec_and_username_raises(self): + with self.assertRaises(TypeError): + Address('foo', username='bing', addr_spec='bar@baz') + + def test_address_addr_spec_and_domain_raises(self): + with self.assertRaises(TypeError): + Address('foo', domain='bing', addr_spec='bar@baz') + + def test_address_addr_spec_and_username_and_domain_raises(self): + with self.assertRaises(TypeError): + Address('foo', username='bong', domain='bing', addr_spec='bar@baz') + + def test_space_in_addr_spec_username_raises(self): + with self.assertRaises(ValueError): + Address('foo', addr_spec="bad name@example.com") + + def test_bad_addr_sepc_raises(self): + with self.assertRaises(ValueError): + Address('foo', addr_spec="name@ex[]ample.com") + + def test_empty_group(self): + g = Group('foo') + self.assertEqual(g.display_name, 'foo') + self.assertEqual(g.addresses, tuple()) + self.assertEqual(str(g), 'foo:;') + + def test_empty_group_list(self): + g = Group('foo', addresses=[]) + self.assertEqual(g.display_name, 'foo') + self.assertEqual(g.addresses, tuple()) + self.assertEqual(str(g), 'foo:;') + + def test_null_group(self): + g = Group() + self.assertIsNone(g.display_name) + self.assertEqual(g.addresses, tuple()) + self.assertEqual(str(g), 'None:;') + + def test_group_with_addresses(self): + addrs = [Address('b', 'b', 'c'), Address('a', 'b','c')] + g = Group('foo', addrs) + self.assertEqual(g.display_name, 'foo') + self.assertEqual(g.addresses, tuple(addrs)) + self.assertEqual(str(g), 'foo: b , a ;') + + def test_group_with_addresses_no_display_name(self): + addrs = [Address('b', 'b', 'c'), Address('a', 'b','c')] + g = Group(addresses=addrs) + self.assertIsNone(g.display_name) + self.assertEqual(g.addresses, tuple(addrs)) + self.assertEqual(str(g), 'None: b , a ;') + + def test_group_with_one_address_no_display_name(self): + addrs = [Address('b', 'b', 'c')] + g = Group(addresses=addrs) + self.assertIsNone(g.display_name) + self.assertEqual(g.addresses, tuple(addrs)) + self.assertEqual(str(g), 'b ') + + def test_display_name_quoting(self): + g = Group('foo.bar') + self.assertEqual(g.display_name, 'foo.bar') + self.assertEqual(g.addresses, tuple()) + self.assertEqual(str(g), '"foo.bar":;') + + def test_display_name_blanks_not_quoted(self): + g = Group('foo bar') + self.assertEqual(g.display_name, 'foo bar') + self.assertEqual(g.addresses, tuple()) + self.assertEqual(str(g), 'foo bar:;') + + def test_set_message_header_from_address(self): + a = Address('foo', 'bar', 'example.com') + m = Message(policy=policy.default) + m['To'] = a + self.assertEqual(m['to'], 'foo ') + self.assertEqual(m['to'].addresses, (a,)) + + def test_set_message_header_from_group(self): + g = Group('foo bar') + m = Message(policy=policy.default) + m['To'] = g + self.assertEqual(m['to'], 'foo bar:;') + self.assertEqual(m['to'].addresses, g.addresses) + + def test_address_comparison(self): + a = Address('foo', 'bar', 'example.com') + self.assertEqual(Address('foo', 'bar', 'example.com'), a) + self.assertNotEqual(Address('baz', 'bar', 'example.com'), a) + self.assertNotEqual(Address('foo', 'baz', 'example.com'), a) + self.assertNotEqual(Address('foo', 'bar', 'baz'), a) + self.assertFalse(a == object()) + self.assertTrue(a == ALWAYS_EQ) + + def test_group_comparison(self): + a = Address('foo', 'bar', 'example.com') + g = Group('foo bar', [a]) + self.assertEqual(Group('foo bar', (a,)), g) + self.assertNotEqual(Group('baz', [a]), g) + self.assertNotEqual(Group('foo bar', []), g) + self.assertFalse(g == object()) + self.assertTrue(g == ALWAYS_EQ) + + +class TestFolding(TestHeaderBase): + + def test_address_display_names(self): + """Test the folding and encoding of address headers.""" + for name, result in ( + ('Foo Bar, France', '"Foo Bar, France"'), + ('Foo Bar (France)', '"Foo Bar (France)"'), + ('Foo Bar, España', 'Foo =?utf-8?q?Bar=2C_Espa=C3=B1a?='), + ('Foo Bar (España)', 'Foo Bar =?utf-8?b?KEVzcGHDsWEp?='), + ('Foo, Bar España', '=?utf-8?q?Foo=2C_Bar_Espa=C3=B1a?='), + ('Foo, Bar [España]', '=?utf-8?q?Foo=2C_Bar_=5BEspa=C3=B1a=5D?='), + ('Foo Bär, France', 'Foo =?utf-8?q?B=C3=A4r=2C?= France'), + ('Foo Bär ', 'Foo =?utf-8?q?B=C3=A4r_=3CFrance=3E?='), + ( + 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' + 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', + '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' + '=C3=B4nsectetuer?=\n =?utf-8?q?_adipiscing=2E_Suspendisse' + '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' + '?q?p=C3=B4tenti=2E?=', + ), + ): + h = self.make_header('To', Address(name, addr_spec='a@b.com')) + self.assertEqual(h.fold(policy=policy.default), + 'To: %s \n' % result) + + def test_short_unstructured(self): + h = self.make_header('subject', 'this is a test') + self.assertEqual(h.fold(policy=policy.default), + 'subject: this is a test\n') + + def test_long_unstructured(self): + h = self.make_header('Subject', 'This is a long header ' + 'line that will need to be folded into two lines ' + 'and will demonstrate basic folding') + self.assertEqual(h.fold(policy=policy.default), + 'Subject: This is a long header line that will ' + 'need to be folded into two lines\n' + ' and will demonstrate basic folding\n') + + def test_unstructured_short_max_line_length(self): + h = self.make_header('Subject', 'this is a short header ' + 'that will be folded anyway') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + textwrap.dedent("""\ + Subject: this is a + short header that + will be folded + anyway + """)) + + def test_fold_unstructured_single_word(self): + h = self.make_header('Subject', 'test') + self.assertEqual(h.fold(policy=policy.default), 'Subject: test\n') + + def test_fold_unstructured_short(self): + h = self.make_header('Subject', 'test test test') + self.assertEqual(h.fold(policy=policy.default), + 'Subject: test test test\n') + + def test_fold_unstructured_with_overlong_word(self): + h = self.make_header('Subject', 'thisisaverylonglineconsistingofa' + 'singlewordthatwontfit') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) + + def test_fold_unstructured_with_two_overlong_words(self): + h = self.make_header('Subject', 'thisisaverylonglineconsistingofa' + 'singlewordthatwontfit plusanotherverylongwordthatwontfit') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit_pl?=\n' + ' =?utf-8?q?usanoth?=\n' + ' =?utf-8?q?erveryl?=\n' + ' =?utf-8?q?ongword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) + + # XXX Need test for when max_line_length is less than the chrome size. + + def test_fold_unstructured_with_slightly_long_word(self): + h = self.make_header('Subject', 'thislongwordislessthanmaxlinelen') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=35)), + 'Subject:\n thislongwordislessthanmaxlinelen\n') + + def test_fold_unstructured_with_commas(self): + # The old wrapper would fold this at the commas. + h = self.make_header('Subject', "This header is intended to " + "demonstrate, in a fairly succinct way, that we now do " + "not give a , special treatment in unstructured headers.") + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=60)), + textwrap.dedent("""\ + Subject: This header is intended to demonstrate, in a fairly + succinct way, that we now do not give a , special treatment + in unstructured headers. + """)) + + def test_fold_address_list(self): + h = self.make_header('To', '"Theodore H. Perfect" , ' + '"My address is very long because my name is long" , ' + '"Only A. Friend" ') + self.assertEqual(h.fold(policy=policy.default), textwrap.dedent("""\ + To: "Theodore H. Perfect" , + "My address is very long because my name is long" , + "Only A. Friend" + """)) + + def test_fold_date_header(self): + h = self.make_header('Date', 'Sat, 2 Feb 2002 17:00:06 -0800') + self.assertEqual(h.fold(policy=policy.default), + 'Date: Sat, 02 Feb 2002 17:00:06 -0800\n') + + def test_fold_overlong_words_using_RFC2047(self): + h = self.make_header( + 'X-Report-Abuse', + '') + self.assertEqual( + h.fold(policy=policy.default), + 'X-Report-Abuse: =?utf-8?q?=3Chttps=3A//www=2Emailitapp=2E' + 'com/report=5Fabuse?=\n' + ' =?utf-8?q?=2Ephp=3Fmid=3Dxxx-xxx-xxxx' + 'xxxxxxxxxxxxxxxxxxxx=3D=3D-xxx-xx-xx?=\n' + ' =?utf-8?q?=3E?=\n') + + def test_message_id_header_is_not_folded(self): + h = self.make_header( + 'Message-ID', + '') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'Message-ID: \n') + + # Test message-id isn't folded when id-right is no-fold-literal. + h = self.make_header( + 'Message-ID', + '') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'Message-ID: \n') + + # Test message-id isn't folded when id-right is non-ascii characters. + h = self.make_header('Message-ID', '<ईमेल@wők.com>') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=30)), + 'Message-ID: <ईमेल@wők.com>\n') + + # Test message-id is folded without breaking the msg-id token into + # encoded words, *even* if they don't fit into max_line_length. + h = self.make_header('Message-ID', '<ईमेलfromMessage@wők.com>') + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'Message-ID:\n <ईमेलfromMessage@wők.com>\n') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_inversion.py b/Lib/test/test_email/test_inversion.py new file mode 100644 index 00000000000..7bd7f2a7206 --- /dev/null +++ b/Lib/test/test_email/test_inversion.py @@ -0,0 +1,78 @@ +"""Test the parser and generator are inverses. + +Note that this is only strictly true if we are parsing RFC valid messages and +producing RFC valid messages. +""" + +import io +import unittest +from email import policy, message_from_bytes +from email.message import EmailMessage +from email.generator import BytesGenerator +from test.test_email import TestEmailBase, parameterize + +# This is like textwrap.dedent for bytes, except that it uses \r\n for the line +# separators on the rebuilt string. +def dedent(bstr): + lines = bstr.splitlines() + if not lines[0].strip(): + raise ValueError("First line must contain text") + stripamt = len(lines[0]) - len(lines[0].lstrip()) + return b'\r\n'.join( + [x[stripamt:] if len(x)>=stripamt else b'' + for x in lines]) + + +@parameterize +class TestInversion(TestEmailBase): + + policy = policy.default + message = EmailMessage + + def msg_as_input(self, msg): + m = message_from_bytes(msg, policy=policy.SMTP) + b = io.BytesIO() + g = BytesGenerator(b) + g.flatten(m) + self.assertEqual(b.getvalue(), msg) + + # XXX: spaces are not preserved correctly here yet in the general case. + msg_params = { + 'header_with_one_space_body': (dedent(b"""\ + From: abc@xyz.com + X-Status:\x20 + Subject: test + + foo + """),), + + 'header_with_invalid_date': (dedent(b"""\ + Date: Tue, 06 Jun 2017 27:39:33 +0600 + From: abc@xyz.com + Subject: timezones + + How do they work even? + """),), + + } + + payload_params = { + 'plain_text': dict(payload='This is a test\n'*20), + 'base64_text': dict(payload=(('xy a'*40+'\n')*5), cte='base64'), + 'qp_text': dict(payload=(('xy a'*40+'\n')*5), cte='quoted-printable'), + } + + def payload_as_body(self, payload, **kw): + msg = self._make_message() + msg['From'] = 'foo' + msg['To'] = 'bar' + msg['Subject'] = 'payload round trip test' + msg.set_content(payload, **kw) + b = bytes(msg) + msg2 = message_from_bytes(b, policy=self.policy) + self.assertEqual(bytes(msg2), b) + self.assertEqual(msg2.get_content(), payload) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py new file mode 100644 index 00000000000..966615dcc1d --- /dev/null +++ b/Lib/test/test_email/test_message.py @@ -0,0 +1,1094 @@ +import textwrap +import unittest +from email import message_from_bytes, message_from_string, policy +from email.message import EmailMessage, MIMEPart +from test.test_email import TestEmailBase, parameterize + + +# Helper. +def first(iterable): + return next(filter(lambda x: x is not None, iterable), None) + + +class Test(TestEmailBase): + + policy = policy.default + + def test_error_on_setitem_if_max_count_exceeded(self): + m = self._str_msg("") + m['To'] = 'abc@xyz' + with self.assertRaises(ValueError): + m['To'] = 'xyz@abc' + + def test_rfc2043_auto_decoded_and_emailmessage_used(self): + m = message_from_string(textwrap.dedent("""\ + Subject: Ayons asperges pour le =?utf-8?q?d=C3=A9jeuner?= + From: =?utf-8?q?Pep=C3=A9?= Le Pew + To: "Penelope Pussycat" <"penelope@example.com"> + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + + sample text + """), policy=policy.default) + self.assertEqual(m['subject'], "Ayons asperges pour le déjeuner") + self.assertEqual(m['from'], "Pepé Le Pew ") + self.assertIsInstance(m, EmailMessage) + + +@parameterize +class TestEmailMessageBase: + + policy = policy.default + + # The first argument is a triple (related, html, plain) of indices into the + # list returned by 'walk' called on a Message constructed from the third. + # The indices indicate which part should match the corresponding part-type + # when passed to get_body (ie: the "first" part of that type in the + # message). The second argument is a list of indices into the 'walk' list + # of the attachments that should be returned by a call to + # 'iter_attachments'. The third argument is a list of indices into 'walk' + # that should be returned by a call to 'iter_parts'. Note that the first + # item returned by 'walk' is the Message itself. + + message_params = { + + 'empty_message': ( + (None, None, 0), + (), + (), + ""), + + 'non_mime_plain': ( + (None, None, 0), + (), + (), + textwrap.dedent("""\ + To: foo@example.com + + simple text body + """)), + + 'mime_non_text': ( + (None, None, None), + (), + (), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: image/jpg + + bogus body. + """)), + + 'plain_html_alternative': ( + (None, 2, 1), + (), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/alternative; boundary="===" + + preamble + + --=== + Content-Type: text/plain + + simple body + + --=== + Content-Type: text/html + +

simple body

+ --===-- + """)), + + 'plain_html_mixed': ( + (None, 2, 1), + (), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + preamble + + --=== + Content-Type: text/plain + + simple body + + --=== + Content-Type: text/html + +

simple body

+ + --===-- + """)), + + 'plain_html_attachment_mixed': ( + (None, None, 1), + (2,), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: text/plain + + simple body + + --=== + Content-Type: text/html + Content-Disposition: attachment + +

simple body

+ + --===-- + """)), + + 'html_text_attachment_mixed': ( + (None, 2, None), + (1,), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: text/plain + Content-Disposition: AtTaChment + + simple body + + --=== + Content-Type: text/html + +

simple body

+ + --===-- + """)), + + 'html_text_attachment_inline_mixed': ( + (None, 2, 1), + (), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: text/plain + Content-Disposition: InLine + + simple body + + --=== + Content-Type: text/html + Content-Disposition: inline + +

simple body

+ + --===-- + """)), + + # RFC 2387 + 'related': ( + (0, 1, None), + (2,), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/related; boundary="==="; type=text/html + + --=== + Content-Type: text/html + +

simple body

+ + --=== + Content-Type: image/jpg + Content-ID: + + bogus data + + --===-- + """)), + + # This message structure will probably never be seen in the wild, but + # it proves we distinguish between text parts based on 'start'. The + # content would not, of course, actually work :) + 'related_with_start': ( + (0, 2, None), + (1,), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/related; boundary="==="; type=text/html; + start="" + + --=== + Content-Type: text/html + Content-ID: + + useless text + + --=== + Content-Type: text/html + Content-ID: + +

simple body

+ + + --===-- + """)), + + + 'mixed_alternative_plain_related': ( + (3, 4, 2), + (6, 7), + (1, 6, 7), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: multipart/alternative; boundary="+++" + + --+++ + Content-Type: text/plain + + simple body + + --+++ + Content-Type: multipart/related; boundary="___" + + --___ + Content-Type: text/html + +

simple body

+ + --___ + Content-Type: image/jpg + Content-ID: + + bogus jpg body + + --___-- + + --+++-- + + --=== + Content-Type: image/jpg + Content-Disposition: attachment + + bogus jpg body + + --=== + Content-Type: image/jpg + Content-Disposition: AttacHmenT + + another bogus jpg body + + --===-- + """)), + + # This structure suggested by Stephen J. Turnbull...may not exist/be + # supported in the wild, but we want to support it. + 'mixed_related_alternative_plain_html': ( + (1, 4, 3), + (6, 7), + (1, 6, 7), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: multipart/related; boundary="+++" + + --+++ + Content-Type: multipart/alternative; boundary="___" + + --___ + Content-Type: text/plain + + simple body + + --___ + Content-Type: text/html + +

simple body

+ + --___-- + + --+++ + Content-Type: image/jpg + Content-ID: + + bogus jpg body + + --+++-- + + --=== + Content-Type: image/jpg + Content-Disposition: attachment + + bogus jpg body + + --=== + Content-Type: image/jpg + Content-Disposition: attachment + + another bogus jpg body + + --===-- + """)), + + # Same thing, but proving we only look at the root part, which is the + # first one if there isn't any start parameter. That is, this is a + # broken related. + 'mixed_related_alternative_plain_html_wrong_order': ( + (1, None, None), + (6, 7), + (1, 6, 7), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: multipart/related; boundary="+++" + + --+++ + Content-Type: image/jpg + Content-ID: + + bogus jpg body + + --+++ + Content-Type: multipart/alternative; boundary="___" + + --___ + Content-Type: text/plain + + simple body + + --___ + Content-Type: text/html + +

simple body

+ + --___-- + + --+++-- + + --=== + Content-Type: image/jpg + Content-Disposition: attachment + + bogus jpg body + + --=== + Content-Type: image/jpg + Content-Disposition: attachment + + another bogus jpg body + + --===-- + """)), + + 'message_rfc822': ( + (None, None, None), + (), + (), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: message/rfc822 + + To: bar@example.com + From: robot@examp.com + + this is a message body. + """)), + + 'mixed_text_message_rfc822': ( + (None, None, 1), + (2,), + (1, 2), + textwrap.dedent("""\ + To: foo@example.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="===" + + --=== + Content-Type: text/plain + + Your message has bounced, sir. + + --=== + Content-Type: message/rfc822 + + To: bar@example.com + From: robot@examp.com + + this is a message body. + + --===-- + """)), + + } + + def message_as_get_body(self, body_parts, attachments, parts, msg): + m = self._str_msg(msg) + allparts = list(m.walk()) + expected = [None if n is None else allparts[n] for n in body_parts] + related = 0; html = 1; plain = 2 + self.assertEqual(m.get_body(), first(expected)) + self.assertEqual(m.get_body(preferencelist=( + 'related', 'html', 'plain')), + first(expected)) + self.assertEqual(m.get_body(preferencelist=('related', 'html')), + first(expected[related:html+1])) + self.assertEqual(m.get_body(preferencelist=('related', 'plain')), + first([expected[related], expected[plain]])) + self.assertEqual(m.get_body(preferencelist=('html', 'plain')), + first(expected[html:plain+1])) + self.assertEqual(m.get_body(preferencelist=['related']), + expected[related]) + self.assertEqual(m.get_body(preferencelist=['html']), expected[html]) + self.assertEqual(m.get_body(preferencelist=['plain']), expected[plain]) + self.assertEqual(m.get_body(preferencelist=('plain', 'html')), + first(expected[plain:html-1:-1])) + self.assertEqual(m.get_body(preferencelist=('plain', 'related')), + first([expected[plain], expected[related]])) + self.assertEqual(m.get_body(preferencelist=('html', 'related')), + first(expected[html::-1])) + self.assertEqual(m.get_body(preferencelist=('plain', 'html', 'related')), + first(expected[::-1])) + self.assertEqual(m.get_body(preferencelist=('html', 'plain', 'related')), + first([expected[html], + expected[plain], + expected[related]])) + + def message_as_iter_attachment(self, body_parts, attachments, parts, msg): + m = self._str_msg(msg) + allparts = list(m.walk()) + attachments = [allparts[n] for n in attachments] + self.assertEqual(list(m.iter_attachments()), attachments) + + def message_as_iter_parts(self, body_parts, attachments, parts, msg): + def _is_multipart_msg(msg): + return 'Content-Type: multipart' in msg + + m = self._str_msg(msg) + allparts = list(m.walk()) + parts = [allparts[n] for n in parts] + iter_parts = list(m.iter_parts()) if _is_multipart_msg(msg) else [] + self.assertEqual(iter_parts, parts) + + class _TestContentManager: + def get_content(self, msg, *args, **kw): + return msg, args, kw + def set_content(self, msg, *args, **kw): + self.msg = msg + self.args = args + self.kw = kw + + def test_get_content_with_cm(self): + m = self._str_msg('') + cm = self._TestContentManager() + self.assertEqual(m.get_content(content_manager=cm), (m, (), {})) + msg, args, kw = m.get_content('foo', content_manager=cm, bar=1, k=2) + self.assertEqual(msg, m) + self.assertEqual(args, ('foo',)) + self.assertEqual(kw, dict(bar=1, k=2)) + + def test_get_content_default_cm_comes_from_policy(self): + p = policy.default.clone(content_manager=self._TestContentManager()) + m = self._str_msg('', policy=p) + self.assertEqual(m.get_content(), (m, (), {})) + msg, args, kw = m.get_content('foo', bar=1, k=2) + self.assertEqual(msg, m) + self.assertEqual(args, ('foo',)) + self.assertEqual(kw, dict(bar=1, k=2)) + + def test_set_content_with_cm(self): + m = self._str_msg('') + cm = self._TestContentManager() + m.set_content(content_manager=cm) + self.assertEqual(cm.msg, m) + self.assertEqual(cm.args, ()) + self.assertEqual(cm.kw, {}) + m.set_content('foo', content_manager=cm, bar=1, k=2) + self.assertEqual(cm.msg, m) + self.assertEqual(cm.args, ('foo',)) + self.assertEqual(cm.kw, dict(bar=1, k=2)) + + def test_set_content_default_cm_comes_from_policy(self): + cm = self._TestContentManager() + p = policy.default.clone(content_manager=cm) + m = self._str_msg('', policy=p) + m.set_content() + self.assertEqual(cm.msg, m) + self.assertEqual(cm.args, ()) + self.assertEqual(cm.kw, {}) + m.set_content('foo', bar=1, k=2) + self.assertEqual(cm.msg, m) + self.assertEqual(cm.args, ('foo',)) + self.assertEqual(cm.kw, dict(bar=1, k=2)) + + # outcome is whether xxx_method should raise ValueError error when called + # on multipart/subtype. Blank outcome means it depends on xxx (add + # succeeds, make raises). Note: 'none' means there are content-type + # headers but payload is None...this happening in practice would be very + # unusual, so treating it as if there were content seems reasonable. + # method subtype outcome + subtype_params = ( + ('related', 'no_content', 'succeeds'), + ('related', 'none', 'succeeds'), + ('related', 'plain', 'succeeds'), + ('related', 'related', ''), + ('related', 'alternative', 'raises'), + ('related', 'mixed', 'raises'), + ('alternative', 'no_content', 'succeeds'), + ('alternative', 'none', 'succeeds'), + ('alternative', 'plain', 'succeeds'), + ('alternative', 'related', 'succeeds'), + ('alternative', 'alternative', ''), + ('alternative', 'mixed', 'raises'), + ('mixed', 'no_content', 'succeeds'), + ('mixed', 'none', 'succeeds'), + ('mixed', 'plain', 'succeeds'), + ('mixed', 'related', 'succeeds'), + ('mixed', 'alternative', 'succeeds'), + ('mixed', 'mixed', ''), + ) + + def _make_subtype_test_message(self, subtype): + m = self.message() + payload = None + msg_headers = [ + ('To', 'foo@bar.com'), + ('From', 'bar@foo.com'), + ] + if subtype != 'no_content': + ('content-shadow', 'Logrus'), + msg_headers.append(('X-Random-Header', 'Corwin')) + if subtype == 'text': + payload = '' + msg_headers.append(('Content-Type', 'text/plain')) + m.set_payload('') + elif subtype != 'no_content': + payload = [] + msg_headers.append(('Content-Type', 'multipart/' + subtype)) + msg_headers.append(('X-Trump', 'Random')) + m.set_payload(payload) + for name, value in msg_headers: + m[name] = value + return m, msg_headers, payload + + def _check_disallowed_subtype_raises(self, m, method_name, subtype, method): + with self.assertRaises(ValueError) as ar: + getattr(m, method)() + exc_text = str(ar.exception) + self.assertIn(subtype, exc_text) + self.assertIn(method_name, exc_text) + + def _check_make_multipart(self, m, msg_headers, payload): + count = 0 + for name, value in msg_headers: + if not name.lower().startswith('content-'): + self.assertEqual(m[name], value) + count += 1 + self.assertEqual(len(m), count+1) # +1 for new Content-Type + part = next(m.iter_parts()) + count = 0 + for name, value in msg_headers: + if name.lower().startswith('content-'): + self.assertEqual(part[name], value) + count += 1 + self.assertEqual(len(part), count) + self.assertEqual(part.get_payload(), payload) + + def subtype_as_make(self, method, subtype, outcome): + m, msg_headers, payload = self._make_subtype_test_message(subtype) + make_method = 'make_' + method + if outcome in ('', 'raises'): + self._check_disallowed_subtype_raises(m, method, subtype, make_method) + return + getattr(m, make_method)() + self.assertEqual(m.get_content_maintype(), 'multipart') + self.assertEqual(m.get_content_subtype(), method) + if subtype == 'no_content': + self.assertEqual(len(m.get_payload()), 0) + self.assertEqual(m.items(), + msg_headers + [('Content-Type', + 'multipart/'+method)]) + else: + self.assertEqual(len(m.get_payload()), 1) + self._check_make_multipart(m, msg_headers, payload) + + def subtype_as_make_with_boundary(self, method, subtype, outcome): + # Doing all variation is a bit of overkill... + m = self.message() + if outcome in ('', 'raises'): + m['Content-Type'] = 'multipart/' + subtype + with self.assertRaises(ValueError) as cm: + getattr(m, 'make_' + method)() + return + if subtype == 'plain': + m['Content-Type'] = 'text/plain' + elif subtype != 'no_content': + m['Content-Type'] = 'multipart/' + subtype + getattr(m, 'make_' + method)(boundary="abc") + self.assertTrue(m.is_multipart()) + self.assertEqual(m.get_boundary(), 'abc') + + def test_policy_on_part_made_by_make_comes_from_message(self): + for method in ('make_related', 'make_alternative', 'make_mixed'): + m = self.message(policy=self.policy.clone(content_manager='foo')) + m['Content-Type'] = 'text/plain' + getattr(m, method)() + self.assertEqual(m.get_payload(0).policy.content_manager, 'foo') + + class _TestSetContentManager: + def set_content(self, msg, content, *args, **kw): + msg['Content-Type'] = 'text/plain' + msg.set_payload(content) + + def subtype_as_add(self, method, subtype, outcome): + m, msg_headers, payload = self._make_subtype_test_message(subtype) + cm = self._TestSetContentManager() + add_method = 'add_attachment' if method=='mixed' else 'add_' + method + if outcome == 'raises': + self._check_disallowed_subtype_raises(m, method, subtype, add_method) + return + getattr(m, add_method)('test', content_manager=cm) + self.assertEqual(m.get_content_maintype(), 'multipart') + self.assertEqual(m.get_content_subtype(), method) + if method == subtype or subtype == 'no_content': + self.assertEqual(len(m.get_payload()), 1) + for name, value in msg_headers: + self.assertEqual(m[name], value) + part = m.get_payload()[0] + else: + self.assertEqual(len(m.get_payload()), 2) + self._check_make_multipart(m, msg_headers, payload) + part = m.get_payload()[1] + self.assertEqual(part.get_content_type(), 'text/plain') + self.assertEqual(part.get_payload(), 'test') + if method=='mixed': + self.assertEqual(part['Content-Disposition'], 'attachment') + elif method=='related': + self.assertEqual(part['Content-Disposition'], 'inline') + else: + # Otherwise we don't guess. + self.assertIsNone(part['Content-Disposition']) + + class _TestSetRaisingContentManager: + class CustomError(Exception): + pass + def set_content(self, msg, content, *args, **kw): + raise self.CustomError('test') + + def test_default_content_manager_for_add_comes_from_policy(self): + cm = self._TestSetRaisingContentManager() + m = self.message(policy=self.policy.clone(content_manager=cm)) + for method in ('add_related', 'add_alternative', 'add_attachment'): + with self.assertRaises(self._TestSetRaisingContentManager.CustomError) as ar: + getattr(m, method)('') + self.assertEqual(str(ar.exception), 'test') + + def message_as_clear(self, body_parts, attachments, parts, msg): + m = self._str_msg(msg) + m.clear() + self.assertEqual(len(m), 0) + self.assertEqual(list(m.items()), []) + self.assertIsNone(m.get_payload()) + self.assertEqual(list(m.iter_parts()), []) + + def message_as_clear_content(self, body_parts, attachments, parts, msg): + m = self._str_msg(msg) + expected_headers = [h for h in m.keys() + if not h.lower().startswith('content-')] + m.clear_content() + self.assertEqual(list(m.keys()), expected_headers) + self.assertIsNone(m.get_payload()) + self.assertEqual(list(m.iter_parts()), []) + + def test_is_attachment(self): + m = self._make_message() + self.assertFalse(m.is_attachment()) + m['Content-Disposition'] = 'inline' + self.assertFalse(m.is_attachment()) + m.replace_header('Content-Disposition', 'attachment') + self.assertTrue(m.is_attachment()) + m.replace_header('Content-Disposition', 'AtTachMent') + self.assertTrue(m.is_attachment()) + m.set_param('filename', 'abc.png', 'Content-Disposition') + self.assertTrue(m.is_attachment()) + + def test_iter_attachments_mutation(self): + # We had a bug where iter_attachments was mutating the list. + m = self._make_message() + m.set_content('arbitrary text as main part') + m.add_related('more text as a related part') + m.add_related('yet more text as a second "attachment"') + orig = m.get_payload().copy() + self.assertEqual(len(list(m.iter_attachments())), 2) + self.assertEqual(m.get_payload(), orig) + + get_payload_surrogate_params = { + + 'good_surrogateescape': ( + "String that can be encod\udcc3\udcabd with surrogateescape", + b'String that can be encod\xc3\xabd with surrogateescape' + ), + + 'string_with_utf8': ( + "String with utf-8 charactër", + b'String with utf-8 charact\xebr' + ), + + 'surrogate_and_utf8': ( + "String that cannot be ëncod\udcc3\udcabd with surrogateescape", + b'String that cannot be \xebncod\\udcc3\\udcabd with surrogateescape' + ), + + 'out_of_range_surrogate': ( + "String with \udfff cannot be encoded with surrogateescape", + b'String with \\udfff cannot be encoded with surrogateescape' + ), + } + + def get_payload_surrogate_as_gh_94606(self, msg, expected): + """test for GH issue 94606""" + m = self._str_msg(msg) + payload = m.get_payload(decode=True) + self.assertEqual(expected, payload) + + +class TestEmailMessage(TestEmailMessageBase, TestEmailBase): + message = EmailMessage + + def test_set_content_adds_MIME_Version(self): + m = self._str_msg('') + cm = self._TestContentManager() + self.assertNotIn('MIME-Version', m) + m.set_content(content_manager=cm) + self.assertEqual(m['MIME-Version'], '1.0') + + class _MIME_Version_adding_CM: + def set_content(self, msg, *args, **kw): + msg['MIME-Version'] = '1.0' + + def test_set_content_does_not_duplicate_MIME_Version(self): + m = self._str_msg('') + cm = self._MIME_Version_adding_CM() + self.assertNotIn('MIME-Version', m) + m.set_content(content_manager=cm) + self.assertEqual(m['MIME-Version'], '1.0') + + def test_as_string_uses_max_header_length_by_default(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(m.as_string().strip().splitlines()), 3) + + def test_as_string_allows_maxheaderlen(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(m.as_string(maxheaderlen=0).strip().splitlines()), + 1) + self.assertEqual(len(m.as_string(maxheaderlen=34).strip().splitlines()), + 6) + + def test_as_string_unixform(self): + m = self._str_msg('test') + m.set_unixfrom('From foo@bar Thu Jan 1 00:00:00 1970') + self.assertEqual(m.as_string(unixfrom=True), + 'From foo@bar Thu Jan 1 00:00:00 1970\n\ntest') + self.assertEqual(m.as_string(unixfrom=False), '\ntest') + + def test_str_defaults_to_policy_max_line_length(self): + m = self._str_msg('Subject: long line' + ' ab'*50 + '\n\n') + self.assertEqual(len(str(m).strip().splitlines()), 3) + + def test_str_defaults_to_utf8(self): + m = EmailMessage() + m['Subject'] = 'unicöde' + self.assertEqual(str(m), 'Subject: unicöde\n\n') + + def test_folding_with_utf8_encoding_1(self): + # bpo-36520 + # + # Fold a line that contains UTF-8 words before + # and after the whitespace fold point, where the + # line length limit is reached within an ASCII + # word. + + m = EmailMessage() + m['Subject'] = 'Hello Wörld! Hello Wörld! ' \ + 'Hello Wörld! Hello Wörld!Hello Wörld!' + self.assertEqual(bytes(m), + b'Subject: Hello =?utf-8?q?W=C3=B6rld!_Hello_W' + b'=C3=B6rld!_Hello_W=C3=B6rld!?=\n' + b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n') + + + def test_folding_with_utf8_encoding_2(self): + # bpo-36520 + # + # Fold a line that contains UTF-8 words before + # and after the whitespace fold point, where the + # line length limit is reached at the end of an + # encoded word. + + m = EmailMessage() + m['Subject'] = 'Hello Wörld! Hello Wörld! ' \ + 'Hello Wörlds123! Hello Wörld!Hello Wörld!' + self.assertEqual(bytes(m), + b'Subject: Hello =?utf-8?q?W=C3=B6rld!_Hello_W' + b'=C3=B6rld!_Hello_W=C3=B6rlds123!?=\n' + b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n') + + def test_folding_with_utf8_encoding_3(self): + # bpo-36520 + # + # Fold a line that contains UTF-8 words before + # and after the whitespace fold point, where the + # line length limit is reached at the end of the + # first word. + + m = EmailMessage() + m['Subject'] = 'Hello-Wörld!-Hello-Wörld!-Hello-Wörlds123! ' \ + 'Hello Wörld!Hello Wörld!' + self.assertEqual(bytes(m), \ + b'Subject: =?utf-8?q?Hello-W=C3=B6rld!-Hello-W' + b'=C3=B6rld!-Hello-W=C3=B6rlds123!?=\n' + b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n') + + def test_folding_with_utf8_encoding_4(self): + # bpo-36520 + # + # Fold a line that contains UTF-8 words before + # and after the fold point, where the first + # word is UTF-8 and the fold point is within + # the word. + + m = EmailMessage() + m['Subject'] = 'Hello-Wörld!-Hello-Wörld!-Hello-Wörlds123!-Hello' \ + ' Wörld!Hello Wörld!' + self.assertEqual(bytes(m), + b'Subject: =?utf-8?q?Hello-W=C3=B6rld!-Hello-W' + b'=C3=B6rld!-Hello-W=C3=B6rlds123!?=\n' + b' =?utf-8?q?-Hello_W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n') + + def test_folding_with_utf8_encoding_5(self): + # bpo-36520 + # + # Fold a line that contains a UTF-8 word after + # the fold point. + + m = EmailMessage() + m['Subject'] = '123456789 123456789 123456789 123456789 123456789' \ + ' 123456789 123456789 Hello Wörld!' + self.assertEqual(bytes(m), + b'Subject: 123456789 123456789 123456789 123456789' + b' 123456789 123456789 123456789\n' + b' Hello =?utf-8?q?W=C3=B6rld!?=\n\n') + + def test_folding_with_utf8_encoding_6(self): + # bpo-36520 + # + # Fold a line that contains a UTF-8 word before + # the fold point and ASCII words after + + m = EmailMessage() + m['Subject'] = '123456789 123456789 123456789 123456789 Hello Wörld!' \ + ' 123456789 123456789 123456789 123456789 123456789' \ + ' 123456789' + self.assertEqual(bytes(m), + b'Subject: 123456789 123456789 123456789 123456789' + b' Hello =?utf-8?q?W=C3=B6rld!?=\n 123456789 ' + b'123456789 123456789 123456789 123456789 ' + b'123456789\n\n') + + def test_folding_with_utf8_encoding_7(self): + # bpo-36520 + # + # Fold a line twice that contains UTF-8 words before + # and after the first fold point, and ASCII words + # after the second fold point. + + m = EmailMessage() + m['Subject'] = '123456789 123456789 Hello Wörld! Hello Wörld! ' \ + '123456789-123456789 123456789 Hello Wörld! 123456789' \ + ' 123456789' + self.assertEqual(bytes(m), + b'Subject: 123456789 123456789 Hello =?utf-8?q?' + b'W=C3=B6rld!_Hello_W=C3=B6rld!?=\n' + b' 123456789-123456789 123456789 Hello ' + b'=?utf-8?q?W=C3=B6rld!?= 123456789\n 123456789\n\n') + + def test_folding_with_utf8_encoding_8(self): + # bpo-36520 + # + # Fold a line twice that contains UTF-8 words before + # the first fold point, and ASCII words after the + # first fold point, and UTF-8 words after the second + # fold point. + + m = EmailMessage() + m['Subject'] = '123456789 123456789 Hello Wörld! Hello Wörld! ' \ + '123456789 123456789 123456789 123456789 123456789 ' \ + '123456789-123456789 123456789 Hello Wörld! 123456789' \ + ' 123456789' + self.assertEqual(bytes(m), + b'Subject: 123456789 123456789 Hello ' + b'=?utf-8?q?W=C3=B6rld!_Hello_W=C3=B6rld!?=\n 123456789 ' + b'123456789 123456789 123456789 123456789 ' + b'123456789-123456789\n 123456789 Hello ' + b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n') + + def test_folding_with_short_nospace_1(self): + # bpo-36520 + # + # Fold a line that contains a long whitespace after + # the fold point. + + m = EmailMessage(policy.default) + m['Message-ID'] = '123456789' * 3 + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_default_policy_1(self): + # Fixed: https://github.com/python/cpython/issues/124452 + # + # When the value is too long, it should be converted back + # to its original form without any modifications. + + m = EmailMessage(policy.default) + message = '123456789' * 10 + m['Message-ID'] = message + self.assertEqual(m.as_bytes(), + f'Message-ID:\n {message}\n\n'.encode()) + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_compat32_policy_1(self): + m = EmailMessage(policy.compat32) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_smtp_policy_1(self): + m = EmailMessage(policy.SMTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_http_policy_1(self): + m = EmailMessage(policy.HTTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_no_wrapping_max_line_length(self): + # Test that falsey 'max_line_length' are converted to sys.maxsize. + for n in [0, None]: + with self.subTest(max_line_length=n): + self.do_test_no_wrapping_max_line_length(n) + + def do_test_no_wrapping_max_line_length(self, falsey): + self.assertFalse(falsey) + pol = policy.default.clone(max_line_length=falsey) + subj = "S" * 100 + body = "B" * 100 + msg = EmailMessage(policy=pol) + msg["From"] = "a@ex.com" + msg["To"] = "b@ex.com" + msg["Subject"] = subj + msg.set_content(body) + + raw = msg.as_bytes() + self.assertNotIn(b"=\n", raw, + "Found fold indicator; wrapping not disabled") + + parsed = message_from_bytes(raw, policy=policy.default) + self.assertEqual(parsed["Subject"], subj) + parsed_body = parsed.get_body().get_content().rstrip('\n') + self.assertEqual(parsed_body, body) + + def test_get_body_malformed(self): + """test for bpo-42892""" + msg = textwrap.dedent("""\ + Message-ID: <674392CA.4347091@email.au> + Date: Wed, 08 Nov 2017 08:50:22 +0700 + From: Foo Bar + MIME-Version: 1.0 + To: email@email.com + Subject: Python Email + Content-Type: multipart/mixed; + boundary="------------879045806563892972123996" + X-Global-filter:Messagescannedforspamandviruses:passedalltests + + This is a multi-part message in MIME format. + --------------879045806563892972123996 + Content-Type: text/plain; charset=ISO-8859-1; format=flowed + Content-Transfer-Encoding: 7bit + + Your message is ready to be sent with the following file or link + attachments: + XU89 - 08.11.2017 + """) + m = self._str_msg(msg) + # In bpo-42892, this would raise + # AttributeError: 'str' object has no attribute 'is_attachment' + m.get_body() + + def test_get_bytes_payload_with_quoted_printable_encoding(self): + # We use a memoryview to avoid directly changing the private payload + # and to prevent using the dedicated paths for string or bytes objects. + payload = memoryview(b'Some payload') + m = self._make_message() + m.add_header('Content-Transfer-Encoding', 'quoted-printable') + m.set_payload(payload) + self.assertEqual(m.get_payload(decode=True), payload) + + +class TestMIMEPart(TestEmailMessageBase, TestEmailBase): + # Doing the full test run here may seem a bit redundant, since the two + # classes are almost identical. But what if they drift apart? So we do + # the full tests so that any future drift doesn't introduce bugs. + message = MIMEPart + + def test_set_content_does_not_add_MIME_Version(self): + m = self._str_msg('') + cm = self._TestContentManager() + self.assertNotIn('MIME-Version', m) + m.set_content(content_manager=cm) + self.assertNotIn('MIME-Version', m) + + def test_string_payload_with_multipart_content_type(self): + msg = message_from_string(textwrap.dedent("""\ + Content-Type: multipart/mixed; charset="utf-8" + + sample text + """), policy=policy.default) + attachments = msg.iter_attachments() + self.assertEqual(list(attachments), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_parser.py b/Lib/test/test_email/test_parser.py new file mode 100644 index 00000000000..06c86408ab5 --- /dev/null +++ b/Lib/test/test_email/test_parser.py @@ -0,0 +1,110 @@ +import io +import email +import unittest +from email.message import Message, EmailMessage +from email.policy import default +from test.test_email import TestEmailBase + + +class TestCustomMessage(TestEmailBase): + + class MyMessage(Message): + def __init__(self, policy): + self.check_policy = policy + super().__init__() + + MyPolicy = TestEmailBase.policy.clone(linesep='boo') + + def test_custom_message_gets_policy_if_possible_from_string(self): + msg = email.message_from_string("Subject: bogus\n\nmsg\n", + self.MyMessage, + policy=self.MyPolicy) + self.assertIsInstance(msg, self.MyMessage) + self.assertIs(msg.check_policy, self.MyPolicy) + + def test_custom_message_gets_policy_if_possible_from_file(self): + source_file = io.StringIO("Subject: bogus\n\nmsg\n") + msg = email.message_from_file(source_file, + self.MyMessage, + policy=self.MyPolicy) + self.assertIsInstance(msg, self.MyMessage) + self.assertIs(msg.check_policy, self.MyPolicy) + + # XXX add tests for other functions that take Message arg. + + +class TestParserBase: + + def test_only_split_on_cr_lf(self): + # The unicode line splitter splits on unicode linebreaks, which are + # more numerous than allowed by the email RFCs; make sure we are only + # splitting on those two. + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + msg = parser( + "Next-Line: not\x85broken\r\n" + "Null: not\x00broken\r\n" + "Vertical-Tab: not\vbroken\r\n" + "Form-Feed: not\fbroken\r\n" + "File-Separator: not\x1Cbroken\r\n" + "Group-Separator: not\x1Dbroken\r\n" + "Record-Separator: not\x1Ebroken\r\n" + "Line-Separator: not\u2028broken\r\n" + "Paragraph-Separator: not\u2029broken\r\n" + "\r\n", + policy=default, + ) + self.assertEqual(msg.items(), [ + ("Next-Line", "not\x85broken"), + ("Null", "not\x00broken"), + ("Vertical-Tab", "not\vbroken"), + ("Form-Feed", "not\fbroken"), + ("File-Separator", "not\x1Cbroken"), + ("Group-Separator", "not\x1Dbroken"), + ("Record-Separator", "not\x1Ebroken"), + ("Line-Separator", "not\u2028broken"), + ("Paragraph-Separator", "not\u2029broken"), + ]) + self.assertEqual(msg.get_payload(), "") + + class MyMessage(EmailMessage): + pass + + def test_custom_message_factory_on_policy(self): + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + MyPolicy = default.clone(message_factory=self.MyMessage) + msg = parser("To: foo\n\ntest", policy=MyPolicy) + self.assertIsInstance(msg, self.MyMessage) + + def test_factory_arg_overrides_policy(self): + for parser in self.parsers: + with self.subTest(parser=parser.__name__): + MyPolicy = default.clone(message_factory=self.MyMessage) + msg = parser("To: foo\n\ntest", Message, policy=MyPolicy) + self.assertNotIsInstance(msg, self.MyMessage) + self.assertIsInstance(msg, Message) + +# Play some games to get nice output in subTest. This code could be clearer +# if staticmethod supported __name__. + +def message_from_file(s, *args, **kw): + f = io.StringIO(s) + return email.message_from_file(f, *args, **kw) + +class TestParser(TestParserBase, TestEmailBase): + parsers = (email.message_from_string, message_from_file) + +def message_from_bytes(s, *args, **kw): + return email.message_from_bytes(s.encode(), *args, **kw) + +def message_from_binary_file(s, *args, **kw): + f = io.BytesIO(s.encode()) + return email.message_from_binary_file(f, *args, **kw) + +class TestBytesParser(TestParserBase, TestEmailBase): + parsers = (message_from_bytes, message_from_binary_file) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_pickleable.py b/Lib/test/test_email/test_pickleable.py new file mode 100644 index 00000000000..16b44671146 --- /dev/null +++ b/Lib/test/test_email/test_pickleable.py @@ -0,0 +1,76 @@ +import unittest +import textwrap +import copy +import pickle +import email +import email.message +from email import policy +from email.headerregistry import HeaderRegistry +from test.test_email import TestEmailBase, parameterize + + +@parameterize +class TestPickleCopyHeader(TestEmailBase): + + header_factory = HeaderRegistry() + + unstructured = header_factory('subject', 'this is a test') + + header_params = { + 'subject': ('subject', 'this is a test'), + 'from': ('from', 'frodo@mordor.net'), + 'to': ('to', 'a: k@b.com, y@z.com;, j@f.com'), + 'date': ('date', 'Tue, 29 May 2012 09:24:26 +1000'), + } + + def header_as_deepcopy(self, name, value): + header = self.header_factory(name, value) + h = copy.deepcopy(header) + self.assertEqual(str(h), str(header)) + + def header_as_pickle(self, name, value): + header = self.header_factory(name, value) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(header, proto) + h = pickle.loads(p) + self.assertEqual(str(h), str(header)) + + +@parameterize +class TestPickleCopyMessage(TestEmailBase): + + # Message objects are a sequence, so we have to make them a one-tuple in + # msg_params so they get passed to the parameterized test method as a + # single argument instead of as a list of headers. + msg_params = {} + + # Note: there will be no custom header objects in the parsed message. + msg_params['parsed'] = (email.message_from_string(textwrap.dedent("""\ + Date: Tue, 29 May 2012 09:24:26 +1000 + From: frodo@mordor.net + To: bilbo@underhill.org + Subject: help + + I think I forgot the ring. + """), policy=policy.default),) + + msg_params['created'] = (email.message.Message(policy=policy.default),) + msg_params['created'][0]['Date'] = 'Tue, 29 May 2012 09:24:26 +1000' + msg_params['created'][0]['From'] = 'frodo@mordor.net' + msg_params['created'][0]['To'] = 'bilbo@underhill.org' + msg_params['created'][0]['Subject'] = 'help' + msg_params['created'][0].set_payload('I think I forgot the ring.') + + def msg_as_deepcopy(self, msg): + msg2 = copy.deepcopy(msg) + self.assertEqual(msg2.as_string(), msg.as_string()) + + def msg_as_pickle(self, msg): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(msg, proto) + msg2 = pickle.loads(p) + self.assertEqual(msg2.as_string(), msg.as_string()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py new file mode 100644 index 00000000000..baa35fd68e4 --- /dev/null +++ b/Lib/test/test_email/test_policy.py @@ -0,0 +1,429 @@ +import io +import types +import textwrap +import unittest +import email.errors +import email.policy +import email.parser +import email.generator +import email.message +from email import headerregistry + +def make_defaults(base_defaults, differences): + defaults = base_defaults.copy() + defaults.update(differences) + return defaults + +class PolicyAPITests(unittest.TestCase): + + longMessage = True + + # Base default values. + compat32_defaults = { + 'max_line_length': 78, + 'linesep': '\n', + 'cte_type': '8bit', + 'raise_on_defect': False, + 'mangle_from_': True, + 'message_factory': None, + 'verify_generated_headers': True, + } + # These default values are the ones set on email.policy.default. + # If any of these defaults change, the docs must be updated. + policy_defaults = compat32_defaults.copy() + policy_defaults.update({ + 'utf8': False, + 'raise_on_defect': False, + 'header_factory': email.policy.EmailPolicy.header_factory, + 'refold_source': 'long', + 'content_manager': email.policy.EmailPolicy.content_manager, + 'mangle_from_': False, + 'message_factory': email.message.EmailMessage, + }) + + # For each policy under test, we give here what we expect the defaults to + # be for that policy. The second argument to make defaults is the + # difference between the base defaults and that for the particular policy. + new_policy = email.policy.EmailPolicy() + policies = { + email.policy.compat32: make_defaults(compat32_defaults, {}), + email.policy.default: make_defaults(policy_defaults, {}), + email.policy.SMTP: make_defaults(policy_defaults, + {'linesep': '\r\n'}), + email.policy.SMTPUTF8: make_defaults(policy_defaults, + {'linesep': '\r\n', + 'utf8': True}), + email.policy.HTTP: make_defaults(policy_defaults, + {'linesep': '\r\n', + 'max_line_length': None}), + email.policy.strict: make_defaults(policy_defaults, + {'raise_on_defect': True}), + new_policy: make_defaults(policy_defaults, {}), + } + # Creating a new policy creates a new header factory. There is a test + # later that proves this. + policies[new_policy]['header_factory'] = new_policy.header_factory + + def test_defaults(self): + for policy, expected in self.policies.items(): + for attr, value in expected.items(): + with self.subTest(policy=policy, attr=attr): + self.assertEqual(getattr(policy, attr), value, + ("change {} docs/docstrings if defaults have " + "changed").format(policy)) + + def test_all_attributes_covered(self): + for policy, expected in self.policies.items(): + for attr in dir(policy): + with self.subTest(policy=policy, attr=attr): + if (attr.startswith('_') or + isinstance(getattr(email.policy.EmailPolicy, attr), + types.FunctionType)): + continue + else: + self.assertIn(attr, expected, + "{} is not fully tested".format(attr)) + + def test_abc(self): + with self.assertRaises(TypeError) as cm: + email.policy.Policy() + msg = str(cm.exception) + abstract_methods = ('fold', + 'fold_binary', + 'header_fetch_parse', + 'header_source_parse', + 'header_store_parse') + for method in abstract_methods: + self.assertIn(method, msg) + + def test_policy_is_immutable(self): + for policy, defaults in self.policies.items(): + for attr in defaults: + with self.assertRaisesRegex(AttributeError, attr+".*read-only"): + setattr(policy, attr, None) + with self.assertRaisesRegex(AttributeError, 'no attribute.*foo'): + policy.foo = None + + def test_set_policy_attrs_when_cloned(self): + # None of the attributes has a default value of None, so we set them + # all to None in the clone call and check that it worked. + for policyclass, defaults in self.policies.items(): + testattrdict = {attr: None for attr in defaults} + policy = policyclass.clone(**testattrdict) + for attr in defaults: + self.assertIsNone(getattr(policy, attr)) + + def test_reject_non_policy_keyword_when_called(self): + for policyclass in self.policies: + with self.assertRaises(TypeError): + policyclass(this_keyword_should_not_be_valid=None) + with self.assertRaises(TypeError): + policyclass(newtline=None) + + def test_policy_addition(self): + expected = self.policy_defaults.copy() + p1 = email.policy.default.clone(max_line_length=100) + p2 = email.policy.default.clone(max_line_length=50) + added = p1 + p2 + expected.update(max_line_length=50) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = p2 + p1 + expected.update(max_line_length=100) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = added + email.policy.default + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + + def test_fold_utf8(self): + expected_ascii = 'Subject: =?utf-8?q?=C3=A1?=\n' + expected_utf8 = 'Subject: á\n' + + msg = email.message.EmailMessage() + s = 'á' + msg['Subject'] = s + + p_ascii = email.policy.default.clone() + p_utf8 = email.policy.default.clone(utf8=True) + + self.assertEqual(p_ascii.fold('Subject', msg['Subject']), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', msg['Subject']), expected_utf8) + + self.assertEqual(p_ascii.fold('Subject', s), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', s), expected_utf8) + + def test_fold_zero_max_line_length(self): + expected = 'Subject: =?utf-8?q?=C3=A1?=\n' + + msg = email.message.EmailMessage() + msg['Subject'] = 'á' + + p1 = email.policy.default.clone(max_line_length=0) + p2 = email.policy.default.clone(max_line_length=None) + + self.assertEqual(p1.fold('Subject', msg['Subject']), expected) + self.assertEqual(p2.fold('Subject', msg['Subject']), expected) + + def test_register_defect(self): + class Dummy: + def __init__(self): + self.defects = [] + obj = Dummy() + defect = object() + policy = email.policy.EmailPolicy() + policy.register_defect(obj, defect) + self.assertEqual(obj.defects, [defect]) + defect2 = object() + policy.register_defect(obj, defect2) + self.assertEqual(obj.defects, [defect, defect2]) + + class MyObj: + def __init__(self): + self.defects = [] + + class MyDefect(Exception): + pass + + def test_handle_defect_raises_on_strict(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + email.policy.strict.handle_defect(foo, defect) + + def test_handle_defect_registers_defect(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + email.policy.default.handle_defect(foo, defect1) + self.assertEqual(foo.defects, [defect1]) + defect2 = self.MyDefect("two") + email.policy.default.handle_defect(foo, defect2) + self.assertEqual(foo.defects, [defect1, defect2]) + + class MyPolicy(email.policy.EmailPolicy): + defects = None + def __init__(self, *args, **kw): + super().__init__(*args, defects=[], **kw) + def register_defect(self, obj, defect): + self.defects.append(defect) + + def test_overridden_register_defect_still_raises(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + self.MyPolicy(raise_on_defect=True).handle_defect(foo, defect) + + def test_overridden_register_defect_works(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + my_policy = self.MyPolicy() + my_policy.handle_defect(foo, defect1) + self.assertEqual(my_policy.defects, [defect1]) + self.assertEqual(foo.defects, []) + defect2 = self.MyDefect("two") + my_policy.handle_defect(foo, defect2) + self.assertEqual(my_policy.defects, [defect1, defect2]) + self.assertEqual(foo.defects, []) + + def test_default_header_factory(self): + h = email.policy.default.header_factory('Test', 'test') + self.assertEqual(h.name, 'Test') + self.assertIsInstance(h, headerregistry.UnstructuredHeader) + self.assertIsInstance(h, headerregistry.BaseHeader) + + class Foo: + parse = headerregistry.UnstructuredHeader.parse + + def test_each_Policy_gets_unique_factory(self): + policy1 = email.policy.EmailPolicy() + policy2 = email.policy.EmailPolicy() + policy1.header_factory.map_to_type('foo', self.Foo) + h = policy1.header_factory('foo', 'test') + self.assertIsInstance(h, self.Foo) + self.assertNotIsInstance(h, headerregistry.UnstructuredHeader) + h = policy2.header_factory('foo', 'test') + self.assertNotIsInstance(h, self.Foo) + self.assertIsInstance(h, headerregistry.UnstructuredHeader) + + def test_clone_copies_factory(self): + policy1 = email.policy.EmailPolicy() + policy2 = policy1.clone() + policy1.header_factory.map_to_type('foo', self.Foo) + h = policy1.header_factory('foo', 'test') + self.assertIsInstance(h, self.Foo) + h = policy2.header_factory('foo', 'test') + self.assertIsInstance(h, self.Foo) + + def test_new_factory_overrides_default(self): + mypolicy = email.policy.EmailPolicy() + myfactory = mypolicy.header_factory + newpolicy = mypolicy + email.policy.strict + self.assertEqual(newpolicy.header_factory, myfactory) + newpolicy = email.policy.strict + mypolicy + self.assertEqual(newpolicy.header_factory, myfactory) + + def test_adding_default_policies_preserves_default_factory(self): + newpolicy = email.policy.default + email.policy.strict + self.assertEqual(newpolicy.header_factory, + email.policy.EmailPolicy.header_factory) + self.assertEqual(newpolicy.__dict__, {'raise_on_defect': True}) + + def test_non_ascii_chars_do_not_cause_inf_loop(self): + policy = email.policy.default.clone(max_line_length=20) + actual = policy.fold('Subject', 'ą' * 12) + self.assertEqual( + actual, + 'Subject: \n' + + 12 * ' =?utf-8?q?=C4=85?=\n') + + def test_short_maxlen_error(self): + # RFC 2047 chrome takes up 7 characters, plus the length of the charset + # name, so folding should fail if maxlen is lower than the minimum + # required length for a line. + + # Note: This is only triggered when there is a single word longer than + # max_line_length, hence the 1234567890 at the end of this whimsical + # subject. This is because when we encounter a word longer than + # max_line_length, it is broken down into encoded words to fit + # max_line_length. If the max_line_length isn't large enough to even + # contain the RFC 2047 chrome (`?=?q??=`), we fail. + subject = "Melt away the pounds with this one simple trick! 1234567890" + + for maxlen in [3, 7, 9]: + with self.subTest(maxlen=maxlen): + policy = email.policy.default.clone(max_line_length=maxlen) + with self.assertRaises(email.errors.HeaderParseError): + policy.fold("Subject", subject) + + def test_verify_generated_headers(self): + """Turning protection off allows header injection""" + policy = email.policy.default.clone(verify_generated_headers=False) + for text in ( + 'Header: Value\r\nBad: Injection\r\n', + 'Header: NoNewLine' + ): + with self.subTest(text=text): + message = email.message_from_string( + "Header: Value\r\n\r\nBody", + policy=policy, + ) + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + del message['Header'] + message['Header'] = LiteralHeader(text) + + self.assertEqual( + message.as_string(), + f"{text}\nBody", + ) + + # XXX: Need subclassing tests. + # For adding subclassed objects, make sure the usual rules apply (subclass + # wins), but that the order still works (right overrides left). + + +class TestException(Exception): + pass + +class TestPolicyPropagation(unittest.TestCase): + + # The abstract methods are used by the parser but not by the wrapper + # functions that call it, so if the exception gets raised we know that the + # policy was actually propagated all the way to feedparser. + class MyPolicy(email.policy.Policy): + def badmethod(self, *args, **kw): + raise TestException("test") + fold = fold_binary = header_fetch_parser = badmethod + header_source_parse = header_store_parse = badmethod + + def test_message_from_string(self): + with self.assertRaisesRegex(TestException, "^test$"): + email.message_from_string("Subject: test\n\n", + policy=self.MyPolicy) + + def test_message_from_bytes(self): + with self.assertRaisesRegex(TestException, "^test$"): + email.message_from_bytes(b"Subject: test\n\n", + policy=self.MyPolicy) + + def test_message_from_file(self): + f = io.StringIO('Subject: test\n\n') + with self.assertRaisesRegex(TestException, "^test$"): + email.message_from_file(f, policy=self.MyPolicy) + + def test_message_from_binary_file(self): + f = io.BytesIO(b'Subject: test\n\n') + with self.assertRaisesRegex(TestException, "^test$"): + email.message_from_binary_file(f, policy=self.MyPolicy) + + # These are redundant, but we need them for black-box completeness. + + def test_parser(self): + p = email.parser.Parser(policy=self.MyPolicy) + with self.assertRaisesRegex(TestException, "^test$"): + p.parsestr('Subject: test\n\n') + + def test_bytes_parser(self): + p = email.parser.BytesParser(policy=self.MyPolicy) + with self.assertRaisesRegex(TestException, "^test$"): + p.parsebytes(b'Subject: test\n\n') + + # Now that we've established that all the parse methods get the + # policy in to feedparser, we can use message_from_string for + # the rest of the propagation tests. + + def _make_msg(self, source='Subject: test\n\n', policy=None): + self.policy = email.policy.default.clone() if policy is None else policy + return email.message_from_string(source, policy=self.policy) + + def test_parser_propagates_policy_to_message(self): + msg = self._make_msg() + self.assertIs(msg.policy, self.policy) + + def test_parser_propagates_policy_to_sub_messages(self): + msg = self._make_msg(textwrap.dedent("""\ + Subject: mime test + MIME-Version: 1.0 + Content-Type: multipart/mixed, boundary="XXX" + + --XXX + Content-Type: text/plain + + test + --XXX + Content-Type: text/plain + + test2 + --XXX-- + """)) + for part in msg.walk(): + self.assertIs(part.policy, self.policy) + + def test_message_policy_propagates_to_generator(self): + msg = self._make_msg("Subject: test\nTo: foo\n\n", + policy=email.policy.default.clone(linesep='X')) + s = io.StringIO() + g = email.generator.Generator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), "Subject: testXTo: fooXX") + + def test_message_policy_used_by_as_string(self): + msg = self._make_msg("Subject: test\nTo: foo\n\n", + policy=email.policy.default.clone(linesep='X')) + self.assertEqual(msg.as_string(), "Subject: testXTo: fooXX") + + +class TestConcretePolicies(unittest.TestCase): + + def test_header_store_parse_rejects_newlines(self): + instance = email.policy.EmailPolicy() + self.assertRaises(ValueError, + instance.header_store_parse, + 'From', 'spam\negg@foo.py') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py new file mode 100644 index 00000000000..d04b3909efa --- /dev/null +++ b/Lib/test/test_email/test_utils.py @@ -0,0 +1,186 @@ +import datetime +from email import utils +import test.support +import time +import unittest +import sys +import os.path +import zoneinfo + +class DateTimeTests(unittest.TestCase): + + datestring = 'Sun, 23 Sep 2001 20:10:55' + dateargs = (2001, 9, 23, 20, 10, 55) + offsetstring = ' -0700' + utcoffset = datetime.timedelta(hours=-7) + tz = datetime.timezone(utcoffset) + naive_dt = datetime.datetime(*dateargs) + aware_dt = datetime.datetime(*dateargs, tzinfo=tz) + + def test_naive_datetime(self): + self.assertEqual(utils.format_datetime(self.naive_dt), + self.datestring + ' -0000') + + def test_aware_datetime(self): + self.assertEqual(utils.format_datetime(self.aware_dt), + self.datestring + self.offsetstring) + + def test_usegmt(self): + utc_dt = datetime.datetime(*self.dateargs, + tzinfo=datetime.timezone.utc) + self.assertEqual(utils.format_datetime(utc_dt, usegmt=True), + self.datestring + ' GMT') + + def test_usegmt_with_naive_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.naive_dt, usegmt=True) + + def test_usegmt_with_non_utc_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.aware_dt, usegmt=True) + + def test_parsedate_to_datetime(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + self.offsetstring), + self.aware_dt) + + def test_parsedate_to_datetime_naive(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + ' -0000'), + self.naive_dt) + + def test_parsedate_to_datetime_with_invalid_raises_valueerror(self): + # See also test_parsedate_returns_None_for_invalid_strings in test_email. + invalid_dates = [ + '', + ' ', + '0', + 'A Complete Waste of Time', + 'Wed, 3 Apr 2002 12.34.56.78+0800' + 'Tue, 06 Jun 2017 27:39:33 +0600', + 'Tue, 06 Jun 2017 07:39:33 +2600', + 'Tue, 06 Jun 2017 27:39:33', + '17 June , 2022', + 'Friday, -Nov-82 16:14:55 EST', + 'Friday, Nov--82 16:14:55 EST', + 'Friday, 19-Nov- 16:14:55 EST', + ] + for dtstr in invalid_dates: + with self.subTest(dtstr=dtstr): + self.assertRaises(ValueError, utils.parsedate_to_datetime, dtstr) + +class LocaltimeTests(unittest.TestCase): + + def test_localtime_is_tz_aware_daylight_true(self): + test.support.patch(self, time, 'daylight', True) + t = utils.localtime() + self.assertIsNotNone(t.tzinfo) + + def test_localtime_is_tz_aware_daylight_false(self): + test.support.patch(self, time, 'daylight', False) + t = utils.localtime() + self.assertIsNotNone(t.tzinfo) + + def test_localtime_daylight_true_dst_false(self): + test.support.patch(self, time, 'daylight', True) + t0 = datetime.datetime(2012, 3, 12, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t1) + self.assertEqual(t1, t2) + + def test_localtime_daylight_false_dst_false(self): + test.support.patch(self, time, 'daylight', False) + t0 = datetime.datetime(2012, 3, 12, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t1) + self.assertEqual(t1, t2) + + @test.support.run_with_tz('Europe/Minsk') + def test_localtime_daylight_true_dst_true(self): + test.support.patch(self, time, 'daylight', True) + t0 = datetime.datetime(2012, 3, 12, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t1) + self.assertEqual(t1, t2) + + @test.support.run_with_tz('Europe/Minsk') + def test_localtime_daylight_false_dst_true(self): + test.support.patch(self, time, 'daylight', False) + t0 = datetime.datetime(2012, 3, 12, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t1) + self.assertEqual(t1, t2) + + @test.support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_localtime_epoch_utc_daylight_true(self): + test.support.patch(self, time, 'daylight', True) + t0 = datetime.datetime(1990, 1, 1, tzinfo = datetime.timezone.utc) + t1 = utils.localtime(t0) + t2 = t0 - datetime.timedelta(hours=5) + t2 = t2.replace(tzinfo = datetime.timezone(datetime.timedelta(hours=-5))) + self.assertEqual(t1, t2) + + @test.support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_localtime_epoch_utc_daylight_false(self): + test.support.patch(self, time, 'daylight', False) + t0 = datetime.datetime(1990, 1, 1, tzinfo = datetime.timezone.utc) + t1 = utils.localtime(t0) + t2 = t0 - datetime.timedelta(hours=5) + t2 = t2.replace(tzinfo = datetime.timezone(datetime.timedelta(hours=-5))) + self.assertEqual(t1, t2) + + def test_localtime_epoch_notz_daylight_true(self): + test.support.patch(self, time, 'daylight', True) + t0 = datetime.datetime(1990, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t0.replace(tzinfo=None)) + self.assertEqual(t1, t2) + + def test_localtime_epoch_notz_daylight_false(self): + test.support.patch(self, time, 'daylight', False) + t0 = datetime.datetime(1990, 1, 1) + t1 = utils.localtime(t0) + t2 = utils.localtime(t0.replace(tzinfo=None)) + self.assertEqual(t1, t2) + + @test.support.run_with_tz('Europe/Kyiv') + def test_variable_tzname(self): + t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) + t1 = utils.localtime(t0) + if t1.tzname() in ('Europe', 'UTC'): + self.skipTest("Can't find a Kyiv timezone database") + self.assertEqual(t1.tzname(), 'MSK') + t0 = datetime.datetime(1994, 1, 1, tzinfo=datetime.timezone.utc) + t1 = utils.localtime(t0) + self.assertEqual(t1.tzname(), 'EET') + + def test_isdst_deprecation(self): + with self.assertWarns(DeprecationWarning): + t0 = datetime.datetime(1990, 1, 1) + t1 = utils.localtime(t0, isdst=True) + +# Issue #24836: The timezone files are out of date (pre 2011k) +# on Mac OS X Snow Leopard. +@test.support.requires_mac_ver(10, 7) +class FormatDateTests(unittest.TestCase): + + @test.support.run_with_tz('Europe/Minsk') + def test_formatdate(self): + timeval = time.mktime((2011, 12, 1, 18, 0, 0, 4, 335, 0)) + string = utils.formatdate(timeval, localtime=False, usegmt=False) + self.assertEqual(string, 'Thu, 01 Dec 2011 15:00:00 -0000') + string = utils.formatdate(timeval, localtime=False, usegmt=True) + self.assertEqual(string, 'Thu, 01 Dec 2011 15:00:00 GMT') + + @test.support.run_with_tz('Europe/Minsk') + def test_formatdate_with_localtime(self): + timeval = time.mktime((2011, 1, 1, 18, 0, 0, 6, 1, 0)) + string = utils.formatdate(timeval, localtime=True) + self.assertEqual(string, 'Sat, 01 Jan 2011 18:00:00 +0200') + # Minsk moved from +0200 (with DST) to +0300 (without DST) in 2011 + timeval = time.mktime((2011, 12, 1, 18, 0, 0, 4, 335, 0)) + string = utils.formatdate(timeval, localtime=True) + self.assertEqual(string, 'Thu, 01 Dec 2011 18:00:00 +0300') + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py new file mode 100644 index 00000000000..9cf9362c9b7 --- /dev/null +++ b/Lib/test/test_email/torture_test.py @@ -0,0 +1,127 @@ +# Copyright (C) 2002-2004 Python Software Foundation +# +# A torture test of the email package. This should not be run as part of the +# standard Python test suite since it requires several meg of email messages +# collected in the wild. These source messages are not checked into the +# Python distro, but are available as part of the standalone email package at +# http://sf.net/projects/mimelib + +import sys +import os +import unittest +from io import StringIO + +from test.test_email import TestEmailBase + +import email +from email import __file__ as testfile +from email.iterators import _structure + +def openfile(filename): + from os.path import join, dirname, abspath + path = abspath(join(dirname(testfile), os.pardir, 'moredata', filename)) + return open(path, 'r') + +# Prevent this test from running in the Python distro +def setUpModule(): + try: + openfile('crispin-torture.txt') + except OSError: + raise unittest.SkipTest + + + +class TortureBase(TestEmailBase): + def _msgobj(self, filename): + fp = openfile(filename) + try: + msg = email.message_from_file(fp) + finally: + fp.close() + return msg + + + +class TestCrispinTorture(TortureBase): + # Mark Crispin's torture test from the SquirrelMail project + def test_mondo_message(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + msg = self._msgobj('crispin-torture.txt') + payload = msg.get_payload() + eq(type(payload), list) + eq(len(payload), 12) + eq(msg.preamble, None) + eq(msg.epilogue, '\n') + # Probably the best way to verify the message is parsed correctly is to + # dump its structure and compare it against the known structure. + fp = StringIO() + _structure(msg, fp=fp) + neq(fp.getvalue(), """\ +multipart/mixed + text/plain + message/rfc822 + multipart/alternative + text/plain + multipart/mixed + text/richtext + application/andrew-inset + message/rfc822 + audio/basic + audio/basic + image/pbm + message/rfc822 + multipart/mixed + multipart/mixed + text/plain + audio/x-sun + multipart/mixed + image/gif + image/gif + application/x-be2 + application/atomicmail + audio/x-sun + message/rfc822 + multipart/mixed + text/plain + image/pgm + text/plain + message/rfc822 + multipart/mixed + text/plain + image/pbm + message/rfc822 + application/postscript + image/gif + message/rfc822 + multipart/mixed + audio/basic + audio/basic + message/rfc822 + multipart/mixed + application/postscript + text/plain + message/rfc822 + multipart/mixed + text/plain + multipart/parallel + image/gif + audio/basic + application/atomicmail + message/rfc822 + audio/x-sun +""") + +def _testclasses(): + mod = sys.modules[__name__] + return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')] + + +def load_tests(loader, tests, pattern): + suite = loader.suiteClass() + for testclass in _testclasses(): + suite.addTest(loader.loadTestsFromTestCase(testclass)) + return suite + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a4b36a90d88..6d3c91b0b6d 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,7 +6,6 @@ import test.support import unittest import unittest.mock -from importlib.resources.abc import Traversable from pathlib import Path import ensurepip diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 32a3c1dee07..75e2353ea16 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -14,30 +14,55 @@ from enum import Enum, EnumMeta, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember, _iter_bits_lsb +from enum import member, nonmember, _iter_bits_lsb, EnumDict from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support -from test.support import ALWAYS_EQ -from test.support import threading_helper +from test.support import ALWAYS_EQ, REPO_ROOT +from test.support import threading_helper, cpython_only +from test.support.import_helper import ensure_lazy_imports from datetime import timedelta python_version = sys.version_info[:2] def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(enum)) - if os.path.exists('Doc/library/enum.rst'): + + lib_tests = os.path.join(REPO_ROOT, 'Doc/library/enum.rst') + if os.path.exists(lib_tests): tests.addTests(doctest.DocFileSuite( - '../../Doc/library/enum.rst', + lib_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) - if os.path.exists('Doc/howto/enum.rst'): + howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') + if os.path.exists(howto_tests) and sys.float_repr_style == 'short': tests.addTests(doctest.DocFileSuite( - '../../Doc/howto/enum.rst', + howto_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) return tests +def reraise_if_not_enum(*enum_types_or_exceptions): + from functools import wraps + + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + excs = [ + e + for e in enum_types_or_exceptions + if isinstance(e, Exception) + ] + if len(excs) == 1: + raise excs[0] + elif excs: + raise ExceptionGroup('Enum Exceptions', excs) + return func(*args, **kwargs) + return inner + return decorator + MODULE = __name__ SHORT_MODULE = MODULE.split('.')[-1] @@ -75,30 +100,42 @@ class FlagStooges(Flag): except Exception as exc: FlagStooges = exc -class FlagStoogesWithZero(Flag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStooges(IntFlag): - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStoogesWithZero(IntFlag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 +try: + class FlagStoogesWithZero(Flag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + FlagStoogesWithZero = exc + +try: + class IntFlagStooges(IntFlag): + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStooges = exc + +try: + class IntFlagStoogesWithZero(IntFlag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStoogesWithZero = exc # for pickle test and subclass tests -class Name(StrEnum): - BDFL = 'Guido van Rossum' - FLUFL = 'Barry Warsaw' +try: + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception as exc: + Name = exc try: Question = Enum('Question', 'who what when where why', module=__name__) @@ -140,7 +177,7 @@ class TestHelpers(unittest.TestCase): sunder_names = '_bad_', '_good_', '_what_ho_' dunder_names = '__mal__', '__bien__', '__que_que__' - private_names = '_MyEnum__private', '_MyEnum__still_private' + private_names = '_MyEnum__private', '_MyEnum__still_private', '_MyEnum___triple_private' private_and_sunder_names = '_MyEnum__private_', '_MyEnum__also_private_' random_names = 'okay', '_semi_private', '_weird__', '_MyEnum__' @@ -204,26 +241,35 @@ def __get__(self, instance, ownerclass): # for global repr tests -@enum.global_enum -class HeadlightsK(IntFlag, boundary=enum.KEEP): - OFF_K = 0 - LOW_BEAM_K = auto() - HIGH_BEAM_K = auto() - FOG_K = auto() +try: + @enum.global_enum + class HeadlightsK(IntFlag, boundary=enum.KEEP): + OFF_K = 0 + LOW_BEAM_K = auto() + HIGH_BEAM_K = auto() + FOG_K = auto() +except Exception as exc: + HeadlightsK = exc -@enum.global_enum -class HeadlightsC(IntFlag, boundary=enum.CONFORM): - OFF_C = 0 - LOW_BEAM_C = auto() - HIGH_BEAM_C = auto() - FOG_C = auto() +try: + @enum.global_enum + class HeadlightsC(IntFlag, boundary=enum.CONFORM): + OFF_C = 0 + LOW_BEAM_C = auto() + HIGH_BEAM_C = auto() + FOG_C = auto() +except Exception as exc: + HeadlightsC = exc -@enum.global_enum -class NoName(Flag): - ONE = 1 - TWO = 2 +try: + @enum.global_enum + class NoName(Flag): + ONE = 1 + TWO = 2 +except Exception as exc: + NoName = exc # tests @@ -388,9 +434,9 @@ class Season(self.enum_type): def spam(cls): pass # - self.assertTrue(hasattr(Season, 'spam')) + self.assertHasAttr(Season, 'spam') del Season.spam - self.assertFalse(hasattr(Season, 'spam')) + self.assertNotHasAttr(Season, 'spam') # with self.assertRaises(AttributeError): del Season.SPRING @@ -402,7 +448,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -417,6 +463,7 @@ def test_basics(self): self.assertEqual(str(TE), "") self.assertEqual(format(TE), "") self.assertTrue(TE(5) is self.dupe2) + self.assertTrue(7 in TE) else: self.assertEqual(repr(TE), "") self.assertEqual(str(TE), "") @@ -469,6 +516,7 @@ def test_contains_tf(self): self.assertFalse('first' in MainEnum) val = MainEnum.dupe self.assertIn(val, MainEnum) + self.assertNotIn(float('nan'), MainEnum) # class OtherEnum(Enum): one = auto() @@ -609,9 +657,6 @@ def __repr__(self): self.assertEqual(str(Generic.item), 'item.test') def test_overridden_str(self): - # TODO: RUSTPYTHON, format(NS.first) does not use __str__ - if self.__class__ in (TestIntFlagFunction, TestIntFlagClass, TestIntEnumFunction, TestIntEnumClass, TestMinimalFloatFunction, TestMinimalFloatClass): - self.skipTest("format(NS.first) does not use __str__") NS = self.NewStrEnum self.assertEqual(str(NS.first), NS.first.name.upper()) self.assertEqual(format(NS.first), NS.first.name.upper()) @@ -1005,6 +1050,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -1292,7 +1353,7 @@ class Color(Enum): red = 1 green = 2 blue = 3 - def red(self): + def red(self): # noqa: F811 return 'red' # with self.assertRaises(TypeError): @@ -1300,13 +1361,12 @@ class Color(Enum): @enum.property def red(self): return 'redder' - red = 1 + red = 1 # noqa: F811 green = 2 blue = 3 + @reraise_if_not_enum(Theory) def test_enum_function_with_qualname(self): - if isinstance(Theory, Exception): - raise Theory self.assertEqual(Theory.__qualname__, 'spanish_inquisition') def test_enum_of_types(self): @@ -1369,12 +1429,10 @@ class Inner(Enum): [Outer.a, Outer.b, Outer.Inner], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_nested_classes_in_enum_are_not_members(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1439,6 +1497,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1491,6 +1570,17 @@ class IntFlag1(IntFlag): self.assertIn(IntEnum1.X, IntFlag1) self.assertIn(IntFlag1.X, IntEnum1) + def test_contains_does_not_call_missing(self): + class AnEnum(Enum): + UNKNOWN = None + LUCKY = 3 + @classmethod + def _missing_(cls, *values): + return cls.UNKNOWN + self.assertTrue(None in AnEnum) + self.assertTrue(3 in AnEnum) + self.assertFalse(7 in AnEnum) + def test_inherited_data_type(self): class HexInt(int): __qualname__ = 'HexInt' @@ -1537,6 +1627,7 @@ class MyUnBrokenEnum(UnBrokenInt, Enum): test_pickle_dump_load(self.assertIs, MyUnBrokenEnum.I) test_pickle_dump_load(self.assertIs, MyUnBrokenEnum) + @reraise_if_not_enum(FloatStooges) def test_floatenum_fromhex(self): h = float.hex(FloatStooges.MOE.value) self.assertIs(FloatStooges.fromhex(h), FloatStooges.MOE) @@ -1657,8 +1748,8 @@ class ThreePart(Enum): self.assertIs(ThreePart((3, 3.0, 'three')), ThreePart.THREE) self.assertIs(ThreePart(3, 3.0, 'three'), ThreePart.THREE) - # TODO: RUSTPYTHON, AssertionError: is not - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: is not + @reraise_if_not_enum(IntStooges) def test_intenum_from_bytes(self): self.assertIs(IntStooges.from_bytes(b'\x00\x03', 'big'), IntStooges.MOE) with self.assertRaises(ValueError): @@ -1687,33 +1778,28 @@ def repr(self): class Huh(MyStr, MyInt, Enum): One = 1 + @reraise_if_not_enum(Stooges) def test_pickle_enum(self): - if isinstance(Stooges, Exception): - raise Stooges test_pickle_dump_load(self.assertIs, Stooges.CURLY) test_pickle_dump_load(self.assertIs, Stooges) + @reraise_if_not_enum(IntStooges) def test_pickle_int(self): - if isinstance(IntStooges, Exception): - raise IntStooges test_pickle_dump_load(self.assertIs, IntStooges.CURLY) test_pickle_dump_load(self.assertIs, IntStooges) + @reraise_if_not_enum(FloatStooges) def test_pickle_float(self): - if isinstance(FloatStooges, Exception): - raise FloatStooges test_pickle_dump_load(self.assertIs, FloatStooges.CURLY) test_pickle_dump_load(self.assertIs, FloatStooges) + @reraise_if_not_enum(Answer) def test_pickle_enum_function(self): - if isinstance(Answer, Exception): - raise Answer test_pickle_dump_load(self.assertIs, Answer.him) test_pickle_dump_load(self.assertIs, Answer) + @reraise_if_not_enum(Question) def test_pickle_enum_function_with_module(self): - if isinstance(Question, Exception): - raise Question test_pickle_dump_load(self.assertIs, Question.who) test_pickle_dump_load(self.assertIs, Question) @@ -1776,9 +1862,8 @@ class Season(Enum): [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], ) + @reraise_if_not_enum(Name) def test_subclassing(self): - if isinstance(Name, Exception): - raise Name self.assertEqual(Name.BDFL, 'Guido van Rossum') self.assertTrue(Name.BDFL, Name('Guido van Rossum')) self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) @@ -1817,6 +1902,25 @@ def test_wrong_inheritance_order(self): class Wrong(Enum, str): NotHere = 'error before this point' + def test_raise_custom_error_on_creation(self): + class InvalidRgbColorError(ValueError): + def __init__(self, r, g, b): + self.r = r + self.g = g + self.b = b + super().__init__(f'({r}, {g}, {b}) is not a valid RGB color') + + with self.assertRaises(InvalidRgbColorError): + class RgbColor(Enum): + RED = (255, 0, 0) + GREEN = (0, 255, 0) + BLUE = (0, 0, 255) + INVALID = (256, 0, 0) + + def __init__(self, r, g, b): + if not all(0 <= val <= 255 for val in (r, g, b)): + raise InvalidRgbColorError(r, g, b) + def test_intenum_transitivity(self): class number(IntEnum): one = 1 @@ -2003,8 +2107,6 @@ class NEI(NamedInt, Enum): test_pickle_dump_load(self.assertIs, NEI.y) test_pickle_dump_load(self.assertIs, NEI) - # TODO: RUSTPYTHON, fails on pickle - @unittest.expectedFailure def test_subclasses_with_getnewargs_ex(self): class NamedInt(int): __qualname__ = 'NamedInt' # needed for pickle protocol 4 @@ -2312,6 +2414,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () @@ -2517,12 +2653,12 @@ def __new__(cls, value, period): OneDay = day_1 OneWeek = week_1 OneMonth = month_1 - self.assertFalse(hasattr(Period, '_ignore_')) - self.assertFalse(hasattr(Period, 'Period')) - self.assertFalse(hasattr(Period, 'i')) - self.assertTrue(isinstance(Period.day_1, timedelta)) - self.assertTrue(Period.month_1 is Period.day_30) - self.assertTrue(Period.week_4 is Period.day_28) + self.assertNotHasAttr(Period, '_ignore_') + self.assertNotHasAttr(Period, 'Period') + self.assertNotHasAttr(Period, 'i') + self.assertIsInstance(Period.day_1, timedelta) + self.assertIs(Period.month_1, Period.day_30) + self.assertIs(Period.week_4, Period.day_28) def test_nonhash_value(self): class AutoNumberInAList(Enum): @@ -2742,7 +2878,7 @@ class ReformedColor(StrMixin, IntEnum, SomeEnum, AnotherEnum): self.assertEqual(str(ReformedColor.BLUE), 'blue') self.assertEqual(ReformedColor.RED.behavior(), 'booyah') self.assertEqual(ConfusedColor.RED.social(), "what's up?") - self.assertTrue(issubclass(ReformedColor, int)) + self.assertIsSubclass(ReformedColor, int) def test_multiple_inherited_mixin(self): @unique @@ -2900,8 +3036,7 @@ class ThirdFailedStrEnum(StrEnum): one = '1' two = b'2', 'ascii', 9 - # TODO: RUSTPYTHON, fails on encoding testing : TypeError: Expected type 'str' but 'builtin_function_or_method' found - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fails on encoding testing : TypeError: Expected type 'str' but 'builtin_function_or_method' found def test_custom_strenum(self): class CustomStrEnum(str, Enum): pass @@ -2952,11 +3087,13 @@ class SecondFailedStrEnum(CustomStrEnum): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = 2 # this will become '2' - with self.assertRaisesRegex(TypeError, '.encoding. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (2|'encoding') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', sys.getdefaultencoding - with self.assertRaisesRegex(TypeError, '.errors. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (3|'errors') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', 'ascii', 9 @@ -3170,6 +3307,37 @@ class NTEnum(Enum): [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], ) + self.assertRaises(AttributeError, getattr, NTEnum.NONE, 'id') + # + class NTCEnum(TTuple, Enum): + NONE = 0, 0, [] + A = 1, 2, [4] + B = 2, 4, [0, 1, 2] + self.assertEqual(repr(NTCEnum.NONE), "") + self.assertEqual(NTCEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual(NTCEnum.NONE.id, 0) + self.assertEqual(NTCEnum.A.a, 2) + self.assertEqual(NTCEnum.B.blist, [0, 1 ,2]) + self.assertEqual( + [x.value for x in NTCEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) + # + class NTDEnum(Enum): + def __new__(cls, id, a, blist): + member = object.__new__(cls) + member.id = id + member.a = a + member.blist = blist + return member + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTDEnum.NONE), "") + self.assertEqual(NTDEnum.NONE.id, 0) + self.assertEqual(NTDEnum.A.a, 2) + self.assertEqual(NTDEnum.B.blist, [0, 1 ,2]) + def test_flag_with_custom_new(self): class FlagFromChar(IntFlag): def __new__(cls, c): @@ -3237,6 +3405,102 @@ def __new__(cls, value): member._value_ = Base(value) return member + def test_extra_member_creation(self): + class IDEnumMeta(EnumMeta): + def __new__(metacls, cls, bases, classdict, **kwds): + # add new entries to classdict + for name in classdict.member_names: + classdict[f'{name}_DESC'] = f'-{classdict[name]}' + return super().__new__(metacls, cls, bases, classdict, **kwds) + class IDEnum(StrEnum, metaclass=IDEnumMeta): + pass + class MyEnum(IDEnum): + ID = 'id' + NAME = 'name' + self.assertEqual(list(MyEnum), [MyEnum.ID, MyEnum.NAME, MyEnum.ID_DESC, MyEnum.NAME_DESC]) + + def test_add_alias(self): + class mixin: + @property + def ORG(self): + return 'huh' + class Color(mixin, Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_alias_('ROJO') + self.assertIs(Color.RED, Color['ROJO']) + self.assertIs(Color.RED, Color.ROJO) + Color.BLUE._add_alias_('ORG') + self.assertIs(Color.BLUE, Color['ORG']) + self.assertIs(Color.BLUE, Color.ORG) + self.assertEqual(Color.RED.ORG, 'huh') + self.assertEqual(Color.GREEN.ORG, 'huh') + self.assertEqual(Color.BLUE.ORG, 'huh') + self.assertEqual(Color.ORG.ORG, 'huh') + + def test_add_value_alias_after_creation(self): + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_value_alias_(5) + self.assertIs(Color.RED, Color(5)) + + def test_add_value_alias_during_creation(self): + class Types(Enum): + Unknown = 0, + Source = 1, 'src' + NetList = 2, 'nl' + def __new__(cls, int_value, *value_aliases): + member = object.__new__(cls) + member._value_ = int_value + for alias in value_aliases: + member._add_value_alias_(alias) + return member + self.assertIs(Types(0), Types.Unknown) + self.assertIs(Types(1), Types.Source) + self.assertIs(Types('src'), Types.Source) + self.assertIs(Types(2), Types.NetList) + self.assertIs(Types('nl'), Types.NetList) + + def test_second_tuple_item_is_falsey(self): + class Cardinal(Enum): + RIGHT = (1, 0) + UP = (0, 1) + LEFT = (-1, 0) + DOWN = (0, -1) + self.assertIs(Cardinal(1, 0), Cardinal.RIGHT) + self.assertIs(Cardinal(-1, 0), Cardinal.LEFT) + + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) + + def test_empty_names(self): + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) + + def test_nonhashable_matches_hashable(self): # issue 125710 + class Directions(Enum): + DOWN_ONLY = frozenset({"sc"}) + UP_ONLY = frozenset({"cs"}) + UNRESTRICTED = frozenset({"sc", "cs"}) + self.assertIs(Directions({"sc"}), Directions.DOWN_ONLY) + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -3518,9 +3782,13 @@ def test_programatic_function_from_dict(self): self.assertIn(e, Perm) self.assertIs(type(e), Perm) + @reraise_if_not_enum( + FlagStooges, + FlagStoogesWithZero, + IntFlagStooges, + IntFlagStoogesWithZero, + ) def test_pickle(self): - if isinstance(FlagStooges, Exception): - raise FlagStooges test_pickle_dump_load(self.assertIs, FlagStooges.CURLY) test_pickle_dump_load(self.assertEqual, FlagStooges.CURLY|FlagStooges.MOE) @@ -3826,6 +4094,7 @@ def test_type(self): self.assertTrue(isinstance(Open.WO | Open.RW, Open)) self.assertEqual(Open.WO | Open.RW, 3) + @reraise_if_not_enum(HeadlightsK) def test_global_repr_keep(self): self.assertEqual( repr(HeadlightsK(0)), @@ -3840,6 +4109,7 @@ def test_global_repr_keep(self): '%(m)s.HeadlightsK(8)' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(HeadlightsC) def test_global_repr_conform1(self): self.assertEqual( repr(HeadlightsC(0)), @@ -3854,13 +4124,13 @@ def test_global_repr_conform1(self): '%(m)s.OFF_C' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') - - # TODO: RUSTPYTHON, format(NewPerm.R) does not use __str__ - @unittest.expectedFailure def test_format(self): Perm = self.Perm self.assertEqual(format(Perm.R, ''), '4') @@ -4548,35 +4818,28 @@ class Color(Enum): red = 'red' blue = 2 green = auto() - yellow = auto() - self.assertEqual(list(Color), - [Color.red, Color.blue, Color.green, Color.yellow]) + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) self.assertEqual(Color.red.value, 'red') self.assertEqual(Color.blue.value, 2) self.assertEqual(Color.green.value, 3) - self.assertEqual(Color.yellow.value, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', + ) def test_auto_garbage_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, "unable to increment 'red'"): class Color(Enum): red = 'red' blue = auto() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', + ) def test_auto_garbage_corrected_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, 'unable to sort non-numeric values'): class Color(Enum): red = 'red' blue = 2 @@ -4604,9 +4867,9 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Color.blue.value, 'blue') @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'auto() will return highest value + 1 in 3.13', + ) def test_auto_with_aliases(self): class Color(Enum): red = auto() @@ -4670,8 +4933,6 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Huh.TWO.value, (2, 2)) self.assertEqual(Huh.THREE.value, (3, 3, 3)) -class TestEnumTypeSubclassing(unittest.TestCase): - pass expected_help_output_with_docs = """\ Help on class Color in module %s: @@ -4702,22 +4963,23 @@ class Color(enum.Enum) | The value of the Enum member. | | ---------------------------------------------------------------------- - | Methods inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: | - | __contains__(value) from enum.EnumType + | __contains__(value) | Return True if `value` is in `cls`. | | `value` is in `cls` if: | 1) `value` is a member of `cls`, or | 2) `value` is the value of one of the `cls`'s members. + | 3) `value` is a pseudo-member (flags) | - | __getitem__(name) from enum.EnumType + | __getitem__(name) | Return the member matching `name`. | - | __iter__() from enum.EnumType + | __iter__() | Return members in definition order. | - | __len__() from enum.EnumType + | __len__() | Return the number of members (no aliases) | | ---------------------------------------------------------------------- @@ -4742,11 +5004,11 @@ class Color(enum.Enum) | | Data and other attributes defined here: | - | YELLOW = + | CYAN = | | MAGENTA = | - | CYAN = + | YELLOW = | | ---------------------------------------------------------------------- | Data descriptors inherited from enum.Enum: @@ -4756,7 +5018,18 @@ class Color(enum.Enum) | value | | ---------------------------------------------------------------------- - | Data descriptors inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: + | + | __contains__(value) + | + | __getitem__(name) + | + | __iter__() + | + | __len__() + | + | ---------------------------------------------------------------------- + | Readonly properties inherited from enum.EnumType: | | __members__""" @@ -4769,8 +5042,6 @@ class Color(Enum): MAGENTA = 2 YELLOW = 3 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pydoc(self): # indirectly test __objclass__ if StrEnum.__doc__ is None: @@ -4903,8 +5174,6 @@ def test_inspect_signatures(self): ]), ) - # TODO: RUSTPYTHON, len is often/always > 256 - @unittest.expectedFailure def test_test_simple_enum(self): @_simple_enum(Enum) class SimpleColor: @@ -4921,12 +5190,14 @@ class CheckedColor(Enum): @bltns.property def zeroth(self): return 'zeroed %s' % self.name - self.assertTrue(_test_simple_enum(CheckedColor, SimpleColor) is None) + _test_simple_enum(CheckedColor, SimpleColor) SimpleColor.MAGENTA._value_ = 9 self.assertRaisesRegex( TypeError, "enum mismatch", _test_simple_enum, CheckedColor, SimpleColor, ) + # + # class CheckedMissing(IntFlag, boundary=KEEP): SIXTY_FOUR = 64 ONE_TWENTY_EIGHT = 128 @@ -4943,8 +5214,78 @@ class Missing: ALL = 2048 + 128 + 64 + 12 M = Missing self.assertEqual(list(CheckedMissing), [M.SIXTY_FOUR, M.ONE_TWENTY_EIGHT, M.TWENTY_FORTY_EIGHT]) - # _test_simple_enum(CheckedMissing, Missing) + # + # + class CheckedUnhashable(Enum): + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), CheckedUnhashable) + self.assertIn('python', CheckedUnhashable) + self.assertEqual(CheckedUnhashable.name.value, 'python') + self.assertEqual(CheckedUnhashable.name.name, 'name') + # + @_simple_enum() + class Unhashable: + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), Unhashable) + self.assertIn('python', Unhashable) + self.assertEqual(Unhashable.name.value, 'python') + self.assertEqual(Unhashable.name.name, 'name') + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): @@ -4952,6 +5293,10 @@ class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, enum, not_exported={'bin', 'show_flag_values'}) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("enum", {"functools", "warnings", "inspect", "re"}) + def test_doc_1(self): class Single(Enum): ONE = 1 @@ -5031,7 +5376,7 @@ def test_convert_value_lookup_priority(self): filter=lambda x: x.startswith('CONVERT_TEST_')) # We don't want the reverse lookup value to vary when there are # multiple possible names for a given value. It should always - # report the first lexigraphical name in that case. + # report the first lexicographical name in that case. self.assertEqual(test_type(5).name, 'CONVERT_TEST_NAME_A') def test_convert_int(self): @@ -5117,6 +5462,37 @@ def test_convert_repr_and_str(self): self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5') +class TestEnumDict(unittest.TestCase): + def test_enum_dict_in_metaclass(self): + """Test that EnumDict is usable as a class namespace""" + class Meta(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return EnumDict(cls) + + class MyClass(metaclass=Meta): + a = 1 + + with self.assertRaises(TypeError): + a = 2 # duplicate + + with self.assertRaises(ValueError): + _a_sunder_ = 3 + + def test_enum_dict_standalone(self): + """Test that EnumDict is usable on its own""" + enumdict = EnumDict() + enumdict['a'] = 1 + + with self.assertRaises(TypeError): + enumdict['a'] = 'other value' + + # Only MutableMapping interface is overridden for now. + # If this stops passing, update the documentation. + enumdict |= {'a': 'other value'} + self.assertEqual(enumdict['a'], 'other value') + + # helpers def enum_dir(cls): @@ -5151,7 +5527,7 @@ def member_dir(member): allowed.add(name) else: allowed.discard(name) - else: + elif name not in member._member_map_: allowed.add(name) return sorted(allowed) diff --git a/Lib/test/test_errno.py b/Lib/test/test_errno.py index 5c437e9ccea..e7f185c6b1a 100644 --- a/Lib/test/test_errno.py +++ b/Lib/test/test_errno.py @@ -12,14 +12,12 @@ class ErrnoAttributeTests(unittest.TestCase): def test_for_improper_attributes(self): # No unexpected attributes should be on the module. for error_code in std_c_errors: - self.assertTrue(hasattr(errno, error_code), - "errno is missing %s" % error_code) + self.assertHasAttr(errno, error_code) def test_using_errorcode(self): # Every key value in errno.errorcode should be on the module. for value in errno.errorcode.values(): - self.assertTrue(hasattr(errno, value), - 'no %s attr in errno' % value) + self.assertHasAttr(errno, value) class ErrorcodeTests(unittest.TestCase): diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py new file mode 100644 index 00000000000..807e7c5a5d6 --- /dev/null +++ b/Lib/test/test_except_star.py @@ -0,0 +1,1221 @@ +import sys +import unittest +import textwrap +from test.support.testcase import ExceptionIsLikeMixin + +class TestInvalidExceptStar(unittest.TestCase): + def test_mixed_except_and_except_star_is_syntax_error(self): + errors = [ + "try: pass\nexcept ValueError: pass\nexcept* TypeError: pass\n", + "try: pass\nexcept* ValueError: pass\nexcept TypeError: pass\n", + "try: pass\nexcept ValueError as e: pass\nexcept* TypeError: pass\n", + "try: pass\nexcept* ValueError as e: pass\nexcept TypeError: pass\n", + "try: pass\nexcept ValueError: pass\nexcept* TypeError as e: pass\n", + "try: pass\nexcept* ValueError: pass\nexcept TypeError as e: pass\n", + "try: pass\nexcept ValueError: pass\nexcept*: pass\n", + "try: pass\nexcept* ValueError: pass\nexcept: pass\n", + ] + + for err in errors: + with self.assertRaises(SyntaxError): + compile(err, "", "exec") + + def test_except_star_ExceptionGroup_is_runtime_error_single(self): + with self.assertRaises(TypeError): + try: + raise OSError("blah") + except* ExceptionGroup as e: + pass + + def test_except_star_ExceptionGroup_is_runtime_error_tuple(self): + with self.assertRaises(TypeError): + try: + raise ExceptionGroup("eg", [ValueError(42)]) + except* (TypeError, ExceptionGroup): + pass + + def test_except_star_invalid_exception_type(self): + with self.assertRaises(TypeError): + try: + raise ValueError + except* 42: + pass + + with self.assertRaises(TypeError): + try: + raise ValueError + except* (ValueError, 42): + pass + + +class TestBreakContinueReturnInExceptStarBlock(unittest.TestCase): + MSG = (r"'break', 'continue' and 'return'" + r" cannot appear in an except\* block") + + def check_invalid(self, src): + with self.assertRaisesRegex(SyntaxError, self.MSG): + compile(textwrap.dedent(src), "", "exec") + + def test_break_in_except_star(self): + self.check_invalid( + """ + try: + raise ValueError + except* Exception as e: + break + """) + + self.check_invalid( + """ + for i in range(5): + try: + pass + except* Exception as e: + if i == 2: + break + """) + + self.check_invalid( + """ + for i in range(5): + try: + pass + except* Exception as e: + if i == 2: + break + finally: + pass + return 0 + """) + + + def test_continue_in_except_star_block_invalid(self): + self.check_invalid( + """ + for i in range(5): + try: + raise ValueError + except* Exception as e: + continue + """) + + self.check_invalid( + """ + for i in range(5): + try: + pass + except* Exception as e: + if i == 2: + continue + """) + + self.check_invalid( + """ + for i in range(5): + try: + pass + except* Exception as e: + if i == 2: + continue + finally: + pass + return 0 + """) + + def test_return_in_except_star_block_invalid(self): + self.check_invalid( + """ + def f(): + try: + raise ValueError + except* Exception as e: + return 42 + """) + + self.check_invalid( + """ + def f(): + try: + pass + except* Exception as e: + return 42 + finally: + finished = True + """) + + def test_break_continue_in_except_star_block_valid(self): + try: + raise ValueError(42) + except* Exception as e: + count = 0 + for i in range(5): + if i == 0: + continue + if i == 4: + break + count += 1 + + self.assertEqual(count, 3) + self.assertEqual(i, 4) + exc = e + self.assertIsInstance(exc, ExceptionGroup) + + def test_return_in_except_star_block_valid(self): + try: + raise ValueError(42) + except* Exception as e: + def f(x): + return 2*x + r = f(3) + exc = e + self.assertEqual(r, 6) + self.assertIsInstance(exc, ExceptionGroup) + + +class ExceptStarTest(ExceptionIsLikeMixin, unittest.TestCase): + def assertMetadataEqual(self, e1, e2): + if e1 is None or e2 is None: + self.assertTrue(e1 is None and e2 is None) + else: + self.assertEqual(e1.__context__, e2.__context__) + self.assertEqual(e1.__cause__, e2.__cause__) + self.assertEqual(e1.__traceback__, e2.__traceback__) + + def assertMetadataNotEqual(self, e1, e2): + if e1 is None or e2 is None: + self.assertNotEqual(e1, e2) + else: + return not (e1.__context__ == e2.__context__ + and e1.__cause__ == e2.__cause__ + and e1.__traceback__ == e2.__traceback__) + + +class TestExceptStarSplitSemantics(ExceptStarTest): + def doSplitTestNamed(self, exc, T, match_template, rest_template): + initial_sys_exception = sys.exception() + sys_exception = match = rest = None + try: + try: + raise exc + except* T as e: + sys_exception = sys.exception() + match = e + except BaseException as e: + rest = e + + self.assertEqual(sys_exception, match) + self.assertExceptionIsLike(match, match_template) + self.assertExceptionIsLike(rest, rest_template) + self.assertEqual(sys.exception(), initial_sys_exception) + + def doSplitTestUnnamed(self, exc, T, match_template, rest_template): + initial_sys_exception = sys.exception() + sys_exception = match = rest = None + try: + try: + raise exc + except* T: + sys_exception = match = sys.exception() + else: + if rest_template: + self.fail("Exception not raised") + except BaseException as e: + rest = e + self.assertExceptionIsLike(match, match_template) + self.assertExceptionIsLike(rest, rest_template) + self.assertEqual(sys.exception(), initial_sys_exception) + + def doSplitTestInExceptHandler(self, exc, T, match_template, rest_template): + try: + raise ExceptionGroup('eg', [TypeError(1), ValueError(2)]) + except Exception: + self.doSplitTestNamed(exc, T, match_template, rest_template) + self.doSplitTestUnnamed(exc, T, match_template, rest_template) + + def doSplitTestInExceptStarHandler(self, exc, T, match_template, rest_template): + try: + raise ExceptionGroup('eg', [TypeError(1), ValueError(2)]) + except* Exception: + self.doSplitTestNamed(exc, T, match_template, rest_template) + self.doSplitTestUnnamed(exc, T, match_template, rest_template) + + def doSplitTest(self, exc, T, match_template, rest_template): + self.doSplitTestNamed(exc, T, match_template, rest_template) + self.doSplitTestUnnamed(exc, T, match_template, rest_template) + self.doSplitTestInExceptHandler(exc, T, match_template, rest_template) + self.doSplitTestInExceptStarHandler(exc, T, match_template, rest_template) + + def test_no_match_single_type(self): + self.doSplitTest( + ExceptionGroup("test1", [ValueError("V"), TypeError("T")]), + SyntaxError, + None, + ExceptionGroup("test1", [ValueError("V"), TypeError("T")])) + + def test_match_single_type(self): + self.doSplitTest( + ExceptionGroup("test2", [ValueError("V1"), ValueError("V2")]), + ValueError, + ExceptionGroup("test2", [ValueError("V1"), ValueError("V2")]), + None) + + def test_match_single_type_partial_match(self): + self.doSplitTest( + ExceptionGroup( + "test3", + [ValueError("V1"), OSError("OS"), ValueError("V2")]), + ValueError, + ExceptionGroup("test3", [ValueError("V1"), ValueError("V2")]), + ExceptionGroup("test3", [OSError("OS")])) + + def test_match_single_type_nested(self): + self.doSplitTest( + ExceptionGroup( + "g1", [ + ValueError("V1"), + OSError("OS1"), + ExceptionGroup( + "g2", [ + OSError("OS2"), + ValueError("V2"), + TypeError("T")])]), + ValueError, + ExceptionGroup( + "g1", [ + ValueError("V1"), + ExceptionGroup("g2", [ValueError("V2")])]), + ExceptionGroup("g1", [ + OSError("OS1"), + ExceptionGroup("g2", [ + OSError("OS2"), TypeError("T")])])) + + def test_match_type_tuple_nested(self): + self.doSplitTest( + ExceptionGroup( + "h1", [ + ValueError("V1"), + OSError("OS1"), + ExceptionGroup( + "h2", [OSError("OS2"), ValueError("V2"), TypeError("T")])]), + (ValueError, TypeError), + ExceptionGroup( + "h1", [ + ValueError("V1"), + ExceptionGroup("h2", [ValueError("V2"), TypeError("T")])]), + ExceptionGroup( + "h1", [ + OSError("OS1"), + ExceptionGroup("h2", [OSError("OS2")])])) + + def test_empty_groups_removed(self): + self.doSplitTest( + ExceptionGroup( + "eg", [ + ExceptionGroup("i1", [ValueError("V1")]), + ExceptionGroup("i2", [ValueError("V2"), TypeError("T1")]), + ExceptionGroup("i3", [TypeError("T2")])]), + TypeError, + ExceptionGroup("eg", [ + ExceptionGroup("i2", [TypeError("T1")]), + ExceptionGroup("i3", [TypeError("T2")])]), + ExceptionGroup("eg", [ + ExceptionGroup("i1", [ValueError("V1")]), + ExceptionGroup("i2", [ValueError("V2")])])) + + def test_singleton_groups_are_kept(self): + self.doSplitTest( + ExceptionGroup("j1", [ + ExceptionGroup("j2", [ + ExceptionGroup("j3", [ValueError("V1")]), + ExceptionGroup("j4", [TypeError("T")])])]), + TypeError, + ExceptionGroup( + "j1", + [ExceptionGroup("j2", [ExceptionGroup("j4", [TypeError("T")])])]), + ExceptionGroup( + "j1", + [ExceptionGroup("j2", [ExceptionGroup("j3", [ValueError("V1")])])])) + + def test_naked_exception_matched_wrapped1(self): + self.doSplitTest( + ValueError("V"), + ValueError, + ExceptionGroup("", [ValueError("V")]), + None) + + def test_naked_exception_matched_wrapped2(self): + self.doSplitTest( + ValueError("V"), + Exception, + ExceptionGroup("", [ValueError("V")]), + None) + + def test_exception_group_except_star_Exception_not_wrapped(self): + self.doSplitTest( + ExceptionGroup("eg", [ValueError("V")]), + Exception, + ExceptionGroup("eg", [ValueError("V")]), + None) + + def test_plain_exception_not_matched(self): + self.doSplitTest( + ValueError("V"), + TypeError, + None, + ValueError("V")) + + def test_match__supertype(self): + self.doSplitTest( + ExceptionGroup("st", [BlockingIOError("io"), TypeError("T")]), + OSError, + ExceptionGroup("st", [BlockingIOError("io")]), + ExceptionGroup("st", [TypeError("T")])) + + def test_multiple_matches_named(self): + try: + raise ExceptionGroup("mmn", [OSError("os"), BlockingIOError("io")]) + except* BlockingIOError as e: + self.assertExceptionIsLike(e, + ExceptionGroup("mmn", [BlockingIOError("io")])) + except* OSError as e: + self.assertExceptionIsLike(e, + ExceptionGroup("mmn", [OSError("os")])) + else: + self.fail("Exception not raised") + + def test_multiple_matches_unnamed(self): + try: + raise ExceptionGroup("mmu", [OSError("os"), BlockingIOError("io")]) + except* BlockingIOError: + e = sys.exception() + self.assertExceptionIsLike(e, + ExceptionGroup("mmu", [BlockingIOError("io")])) + except* OSError: + e = sys.exception() + self.assertExceptionIsLike(e, + ExceptionGroup("mmu", [OSError("os")])) + else: + self.fail("Exception not raised") + + def test_first_match_wins_named(self): + try: + raise ExceptionGroup("fst", [BlockingIOError("io")]) + except* OSError as e: + self.assertExceptionIsLike(e, + ExceptionGroup("fst", [BlockingIOError("io")])) + except* BlockingIOError: + self.fail("Should have been matched as OSError") + else: + self.fail("Exception not raised") + + def test_first_match_wins_unnamed(self): + try: + raise ExceptionGroup("fstu", [BlockingIOError("io")]) + except* OSError: + e = sys.exception() + self.assertExceptionIsLike(e, + ExceptionGroup("fstu", [BlockingIOError("io")])) + except* BlockingIOError: + pass + else: + self.fail("Exception not raised") + + def test_nested_except_stars(self): + try: + raise ExceptionGroup("n", [BlockingIOError("io")]) + except* BlockingIOError: + try: + raise ExceptionGroup("n", [ValueError("io")]) + except* ValueError: + pass + else: + self.fail("Exception not raised") + e = sys.exception() + self.assertExceptionIsLike(e, + ExceptionGroup("n", [BlockingIOError("io")])) + else: + self.fail("Exception not raised") + + def test_nested_in_loop(self): + for _ in range(2): + try: + raise ExceptionGroup("nl", [BlockingIOError("io")]) + except* BlockingIOError: + pass + else: + self.fail("Exception not raised") + + +class TestExceptStarReraise(ExceptStarTest): + def test_reraise_all_named(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2), OSError(3)]) + except* TypeError as e: + raise + except* ValueError as e: + raise + # OSError not handled + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup("eg", [TypeError(1), ValueError(2), OSError(3)])) + + def test_reraise_all_unnamed(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2), OSError(3)]) + except* TypeError: + raise + except* ValueError: + raise + # OSError not handled + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup("eg", [TypeError(1), ValueError(2), OSError(3)])) + + def test_reraise_some_handle_all_named(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2), OSError(3)]) + except* TypeError as e: + raise + except* ValueError as e: + pass + # OSError not handled + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [TypeError(1), OSError(3)])) + + def test_reraise_partial_handle_all_unnamed(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2)]) + except* TypeError: + raise + except* ValueError: + pass + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [TypeError(1)])) + + def test_reraise_partial_handle_some_named(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2), OSError(3)]) + except* TypeError as e: + raise + except* ValueError as e: + pass + # OSError not handled + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [TypeError(1), OSError(3)])) + + def test_reraise_partial_handle_some_unnamed(self): + try: + try: + raise ExceptionGroup( + "eg", [TypeError(1), ValueError(2), OSError(3)]) + except* TypeError: + raise + except* ValueError: + pass + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [TypeError(1), OSError(3)])) + + def test_reraise_plain_exception_named(self): + try: + try: + raise ValueError(42) + except* ValueError as e: + raise + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [ValueError(42)])) + + def test_reraise_plain_exception_unnamed(self): + try: + try: + raise ValueError(42) + except* ValueError: + raise + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [ValueError(42)])) + + +class TestExceptStarRaise(ExceptStarTest): + def test_raise_named(self): + orig = ExceptionGroup("eg", [ValueError(1), OSError(2)]) + try: + try: + raise orig + except* OSError as e: + raise TypeError(3) + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup( + "", [TypeError(3), ExceptionGroup("eg", [ValueError(1)])])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + + def test_raise_unnamed(self): + orig = ExceptionGroup("eg", [ValueError(1), OSError(2)]) + try: + try: + raise orig + except* OSError: + raise TypeError(3) + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup( + "", [TypeError(3), ExceptionGroup("eg", [ValueError(1)])])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + + def test_raise_handle_all_raise_one_named(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* (TypeError, ValueError) as e: + raise SyntaxError(3) + except SyntaxError as e: + exc = e + + self.assertExceptionIsLike(exc, SyntaxError(3)) + + self.assertExceptionIsLike( + exc.__context__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.__context__) + + def test_raise_handle_all_raise_one_unnamed(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* (TypeError, ValueError) as e: + raise SyntaxError(3) + except SyntaxError as e: + exc = e + + self.assertExceptionIsLike(exc, SyntaxError(3)) + + self.assertExceptionIsLike( + exc.__context__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.__context__) + + def test_raise_handle_all_raise_two_named(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* TypeError as e: + raise SyntaxError(3) + except* ValueError as e: + raise SyntaxError(4) + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [SyntaxError(3), SyntaxError(4)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__context__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[1].__context__) + + def test_raise_handle_all_raise_two_unnamed(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* TypeError: + raise SyntaxError(3) + except* ValueError: + raise SyntaxError(4) + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [SyntaxError(3), SyntaxError(4)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__context__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[1].__context__) + + +class TestExceptStarRaiseFrom(ExceptStarTest): + def test_raise_named(self): + orig = ExceptionGroup("eg", [ValueError(1), OSError(2)]) + try: + try: + raise orig + except* OSError as e: + raise TypeError(3) from e + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup( + "", [TypeError(3), ExceptionGroup("eg", [ValueError(1)])])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__cause__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[0].__cause__) + self.assertMetadataNotEqual(orig, exc.exceptions[1].__context__) + self.assertMetadataNotEqual(orig, exc.exceptions[1].__cause__) + + def test_raise_unnamed(self): + orig = ExceptionGroup("eg", [ValueError(1), OSError(2)]) + try: + try: + raise orig + except* OSError: + e = sys.exception() + raise TypeError(3) from e + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, + ExceptionGroup( + "", [TypeError(3), ExceptionGroup("eg", [ValueError(1)])])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__cause__, + ExceptionGroup("eg", [OSError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[0].__cause__) + self.assertMetadataNotEqual(orig, exc.exceptions[1].__context__) + self.assertMetadataNotEqual(orig, exc.exceptions[1].__cause__) + + def test_raise_handle_all_raise_one_named(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* (TypeError, ValueError) as e: + raise SyntaxError(3) from e + except SyntaxError as e: + exc = e + + self.assertExceptionIsLike(exc, SyntaxError(3)) + + self.assertExceptionIsLike( + exc.__context__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertExceptionIsLike( + exc.__cause__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.__context__) + self.assertMetadataEqual(orig, exc.__cause__) + + def test_raise_handle_all_raise_one_unnamed(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* (TypeError, ValueError) as e: + e = sys.exception() + raise SyntaxError(3) from e + except SyntaxError as e: + exc = e + + self.assertExceptionIsLike(exc, SyntaxError(3)) + + self.assertExceptionIsLike( + exc.__context__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertExceptionIsLike( + exc.__cause__, + ExceptionGroup("eg", [TypeError(1), ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.__context__) + self.assertMetadataEqual(orig, exc.__cause__) + + def test_raise_handle_all_raise_two_named(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* TypeError as e: + raise SyntaxError(3) from e + except* ValueError as e: + raise SyntaxError(4) from e + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [SyntaxError(3), SyntaxError(4)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__cause__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__context__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__cause__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[0].__cause__) + + def test_raise_handle_all_raise_two_unnamed(self): + orig = ExceptionGroup("eg", [TypeError(1), ValueError(2)]) + try: + try: + raise orig + except* TypeError: + e = sys.exception() + raise SyntaxError(3) from e + except* ValueError: + e = sys.exception() + raise SyntaxError(4) from e + except ExceptionGroup as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("", [SyntaxError(3), SyntaxError(4)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__context__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[0].__cause__, + ExceptionGroup("eg", [TypeError(1)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__context__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertExceptionIsLike( + exc.exceptions[1].__cause__, + ExceptionGroup("eg", [ValueError(2)])) + + self.assertMetadataNotEqual(orig, exc) + self.assertMetadataEqual(orig, exc.exceptions[0].__context__) + self.assertMetadataEqual(orig, exc.exceptions[0].__cause__) + self.assertMetadataEqual(orig, exc.exceptions[1].__context__) + self.assertMetadataEqual(orig, exc.exceptions[1].__cause__) + + +class TestExceptStarExceptionGroupSubclass(ExceptStarTest): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_star_EG_subclass(self): + class EG(ExceptionGroup): + def __new__(cls, message, excs, code): + obj = super().__new__(cls, message, excs) + obj.code = code + return obj + + def derive(self, excs): + return EG(self.message, excs, self.code) + + try: + try: + try: + try: + raise TypeError(2) + except TypeError as te: + raise EG("nested", [te], 101) from None + except EG as nested: + try: + raise ValueError(1) + except ValueError as ve: + raise EG("eg", [ve, nested], 42) + except* ValueError as eg: + veg = eg + except EG as eg: + teg = eg + + self.assertIsInstance(veg, EG) + self.assertIsInstance(teg, EG) + self.assertIsInstance(teg.exceptions[0], EG) + self.assertMetadataEqual(veg, teg) + self.assertEqual(veg.code, 42) + self.assertEqual(teg.code, 42) + self.assertEqual(teg.exceptions[0].code, 101) + + def test_falsy_exception_group_subclass(self): + class FalsyEG(ExceptionGroup): + def __bool__(self): + return False + + def derive(self, excs): + return FalsyEG(self.message, excs) + + try: + try: + raise FalsyEG("eg", [TypeError(1), ValueError(2)]) + except *TypeError as e: + tes = e + raise + except *ValueError as e: + ves = e + pass + except Exception as e: + exc = e + + for e in [tes, ves, exc]: + self.assertFalse(e) + self.assertIsInstance(e, FalsyEG) + + self.assertExceptionIsLike(exc, FalsyEG("eg", [TypeError(1)])) + self.assertExceptionIsLike(tes, FalsyEG("eg", [TypeError(1)])) + self.assertExceptionIsLike(ves, FalsyEG("eg", [ValueError(2)])) + + def test_exception_group_subclass_with_bad_split_func(self): + # see gh-128049. + class BadEG1(ExceptionGroup): + def split(self, *args): + return "NOT A 2-TUPLE!" + + class BadEG2(ExceptionGroup): + def split(self, *args): + return ("NOT A 2-TUPLE!",) + + eg_list = [ + (BadEG1("eg", [OSError(123), ValueError(456)]), + r"split must return a tuple, not str"), + (BadEG2("eg", [OSError(123), ValueError(456)]), + r"split must return a 2-tuple, got tuple of size 1") + ] + + for eg_class, msg in eg_list: + with self.assertRaisesRegex(TypeError, msg) as m: + try: + raise eg_class + except* ValueError: + pass + except* OSError: + pass + + self.assertExceptionIsLike(m.exception.__context__, eg_class) + + # we allow tuples of length > 2 for backwards compatibility + class WeirdEG(ExceptionGroup): + def split(self, *args): + return super().split(*args) + ("anything", 123456, None) + + try: + raise WeirdEG("eg", [OSError(123), ValueError(456)]) + except* OSError as e: + oeg = e + except* ValueError as e: + veg = e + + self.assertExceptionIsLike(oeg, WeirdEG("eg", [OSError(123)])) + self.assertExceptionIsLike(veg, WeirdEG("eg", [ValueError(456)])) + + +class TestExceptStarCleanup(ExceptStarTest): + def test_sys_exception_restored(self): + try: + try: + raise ValueError(42) + except: + try: + raise TypeError(int) + except* Exception: + pass + 1/0 + except Exception as e: + exc = e + + self.assertExceptionIsLike(exc, ZeroDivisionError('division by zero')) + self.assertExceptionIsLike(exc.__context__, ValueError(42)) + self.assertEqual(sys.exception(), None) + + +class TestExceptStar_WeirdLeafExceptions(ExceptStarTest): + # Test that except* works when leaf exceptions are + # unhashable or have a bad custom __eq__ + + class UnhashableExc(ValueError): + __hash__ = None + + class AlwaysEqualExc(ValueError): + def __eq__(self, other): + return True + + class NeverEqualExc(ValueError): + def __eq__(self, other): + return False + + class BrokenEqualExc(ValueError): + def __eq__(self, other): + raise RuntimeError() + + def setUp(self): + self.bad_types = [self.UnhashableExc, + self.AlwaysEqualExc, + self.NeverEqualExc, + self.BrokenEqualExc] + + def except_type(self, eg, type): + match, rest = None, None + try: + try: + raise eg + except* type as e: + match = e + except Exception as e: + rest = e + return match, rest + + def test_catch_unhashable_leaf_exception(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, Bad) + self.assertExceptionIsLike( + match, ExceptionGroup("eg", [Bad(2)])) + self.assertExceptionIsLike( + rest, ExceptionGroup("eg", [TypeError(1)])) + + def test_propagate_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, TypeError) + self.assertExceptionIsLike( + match, ExceptionGroup("eg", [TypeError(1)])) + self.assertExceptionIsLike( + rest, ExceptionGroup("eg", [Bad(2)])) + + def test_catch_nothing_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, OSError) + self.assertIsNone(match) + self.assertExceptionIsLike(rest, eg) + + def test_catch_everything_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, Exception) + self.assertExceptionIsLike(match, eg) + self.assertIsNone(rest) + + def test_reraise_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup( + "eg", [TypeError(1), Bad(2), ValueError(3)]) + + try: + try: + raise eg + except* TypeError: + pass + except* Bad: + raise + except Exception as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [Bad(2), ValueError(3)])) + + +class TestExceptStar_WeirdExceptionGroupSubclass(ExceptStarTest): + # Test that except* works with exception groups that are + # unhashable or have a bad custom __eq__ + + class UnhashableEG(ExceptionGroup): + __hash__ = None + + def derive(self, excs): + return type(self)(self.message, excs) + + class AlwaysEqualEG(ExceptionGroup): + def __eq__(self, other): + return True + + def derive(self, excs): + return type(self)(self.message, excs) + + class NeverEqualEG(ExceptionGroup): + def __eq__(self, other): + return False + + def derive(self, excs): + return type(self)(self.message, excs) + + class BrokenEqualEG(ExceptionGroup): + def __eq__(self, other): + raise RuntimeError() + + def derive(self, excs): + return type(self)(self.message, excs) + + def setUp(self): + self.bad_types = [self.UnhashableEG, + self.AlwaysEqualEG, + self.NeverEqualEG, + self.BrokenEqualEG] + + def except_type(self, eg, type): + match, rest = None, None + try: + try: + raise eg + except* type as e: + match = e + except Exception as e: + rest = e + return match, rest + + def test_catch_some_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, TypeError) + self.assertExceptionIsLike(match, BadEG("eg", [TypeError(1)])) + self.assertExceptionIsLike(rest, + BadEG("eg", [BadEG("nested", [ValueError(2)])])) + + def test_catch_none_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, OSError) + self.assertIsNone(match) + self.assertExceptionIsLike(rest, eg) + + def test_catch_all_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, Exception) + self.assertExceptionIsLike(match, eg) + self.assertIsNone(rest) + + def test_reraise_unhashable_eg(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), ValueError(2), + BadEG("nested", [ValueError(3), OSError(4)])]) + + try: + try: + raise eg + except* ValueError: + pass + except* OSError: + raise + except Exception as e: + exc = e + + self.assertExceptionIsLike( + exc, BadEG("eg", [TypeError(1), + BadEG("nested", [OSError(4)])])) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2b48530a309..507bbc2ecbc 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,16 +1,14 @@ import collections.abc import types import unittest -from test.support import get_c_recursion_limit +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): - self.assertTrue(issubclass(ExceptionGroup, Exception)) - self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) - self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + self.assertIsSubclass(ExceptionGroup, Exception) + self.assertIsSubclass(ExceptionGroup, BaseExceptionGroup) + self.assertIsSubclass(BaseExceptionGroup, BaseException) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_is_not_generic_type(self): with self.assertRaisesRegex(TypeError, 'Exception'): Exception[OSError] @@ -22,8 +20,7 @@ def test_exception_group_is_generic_type(self): class BadConstructorArgs(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_EG_construction__too_many_args(self): MSG = r'BaseExceptionGroup.__new__\(\) takes exactly 2 arguments' with self.assertRaisesRegex(TypeError, MSG): @@ -464,15 +461,21 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(get_c_recursion_limit() + 1): + for i in range(exceeds_recursion_limit()): e = ExceptionGroup('eg', [e]) return e + @unittest.skip("TODO: RUSTPYTHON; Segfault") + @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) + @unittest.skip("TODO: RUSTPYTHON; Segfault") + @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): @@ -814,8 +817,8 @@ def test_split_does_not_copy_non_sequence_notes(self): eg = ExceptionGroup("eg", [ValueError(1), TypeError(2)]) eg.__notes__ = 123 match, rest = eg.split(TypeError) - self.assertFalse(hasattr(match, '__notes__')) - self.assertFalse(hasattr(rest, '__notes__')) + self.assertNotHasAttr(match, '__notes__') + self.assertNotHasAttr(rest, '__notes__') def test_drive_invalid_return_value(self): class MyEg(ExceptionGroup): diff --git a/Lib/test/test_exception_hierarchy.py b/Lib/test/test_exception_hierarchy.py index e8c1c7fd1e7..3472019ea13 100644 --- a/Lib/test/test_exception_hierarchy.py +++ b/Lib/test/test_exception_hierarchy.py @@ -146,8 +146,7 @@ def test_errno_translation(self): self.assertEqual(e.strerror, "File already exists") self.assertEqual(e.filename, "foo.txt") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_blockingioerror(self): args = ("a", "b", "c", "d", "e") for n in range(6): @@ -182,8 +181,7 @@ def test_init_kwdargs(self): self.assertEqual(e.bar, "baz") self.assertEqual(e.args, ("some message",)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_overridden(self): e = SubOSErrorWithNew("some message", "baz") self.assertEqual(e.baz, "baz") diff --git a/Lib/test/test_exception_variations.py b/Lib/test/test_exception_variations.py index e103eaf8466..a83a41d2975 100644 --- a/Lib/test/test_exception_variations.py +++ b/Lib/test/test_exception_variations.py @@ -294,8 +294,6 @@ def test_nested_exception_in_finally_with_exception(self): self.assertTrue(hit_except) -# TODO: RUSTPYTHON -''' class ExceptStarTestCases(unittest.TestCase): def test_try_except_else_finally(self): hit_except = False @@ -571,7 +569,7 @@ def test_nested_else_mixed2(self): self.assertFalse(hit_else) self.assertTrue(hit_finally) self.assertTrue(hit_except) -''' + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 61f4156dc6d..e7b0e8850a1 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -60,7 +60,7 @@ def raise_catch(self, exc, excname): self.assertEqual(buf1, buf2) self.assertEqual(exc.__name__, excname) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def testRaising(self): self.raise_catch(AttributeError, "AttributeError") self.assertRaises(AttributeError, getattr, sys, "undefined_attribute") @@ -145,7 +145,7 @@ def testRaising(self): self.raise_catch(StopAsyncIteration, "StopAsyncIteration") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorMessage(self): # make sure the right exception message is raised for each of # these code fragments @@ -170,7 +170,7 @@ def ckmsg(src, msg): ckmsg("continue\n", "'continue' not properly in loop") ckmsg("f'{6 0}'", "invalid syntax. Perhaps you forgot a comma?") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorMissingParens(self): def ckmsg(src, msg, exception=SyntaxError): try: @@ -227,14 +227,16 @@ def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding= if not isinstance(src, str): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] + if lineno == 1: + line = line.removeprefix('\ufeff') self.assertIn(line, cm.exception.text) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_offset_continuation_characters(self): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) @@ -244,7 +246,9 @@ def testSyntaxErrorOffset(self): check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', 2, 19, encoding='cp1251') - check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 10) + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 12) + check(b'\n\n\nPython = "\xcf\xb3\xf2\xee\xed" +', 4, 12) + check(b'\xef\xbb\xbfPython = "\xcf\xb3\xf2\xee\xed" +', 1, 12) check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) @@ -292,7 +296,7 @@ def baz(): check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) check("[interesting\nfoo()\n", 1, 1) - check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 1, 0) check("""f''' { (123_a) @@ -362,7 +366,7 @@ def test_capi1(): except TypeError as err: co = err.__traceback__.tb_frame.f_code self.assertEqual(co.co_name, "test_capi1") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') else: self.fail("Expected exception") @@ -374,7 +378,7 @@ def test_capi2(): tb = err.__traceback__.tb_next co = tb.tb_frame.f_code self.assertEqual(co.co_name, "__init__") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') co2 = tb.tb_frame.f_back.f_code self.assertEqual(co2.co_name, "test_capi2") else: @@ -428,6 +432,7 @@ def test_WindowsError(self): self.assertEqual(w.filename, None) self.assertEqual(w.filename2, None) + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.platform == 'win32', 'test specific to Windows') def test_windows_message(self): @@ -438,7 +443,7 @@ def test_windows_message(self): with self.assertRaisesRegex(OSError, 'Windows Error 0x%x' % code): ctypes.pythonapi.PyErr_SetFromWindowsErr(code) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def testAttributes(self): # test that exception attributes are happy @@ -604,7 +609,7 @@ def test_invalid_setstate(self): def test_notes(self): for e in [BaseException(1), Exception(2), ValueError(3)]: with self.subTest(e=e): - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("My Note") self.assertEqual(e.__notes__, ["My Note"]) @@ -616,7 +621,7 @@ def test_notes(self): self.assertEqual(e.__notes__, ["My Note", "Your Note"]) del e.__notes__ - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("Our Note") self.assertEqual(e.__notes__, ["Our Note"]) @@ -657,7 +662,7 @@ def testInvalidTraceback(self): else: self.fail("No exception raised") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_setattr(self): TE = TypeError exc = Exception() @@ -670,7 +675,7 @@ def test_invalid_setattr(self): msg = "exception context must be None or derive from BaseException" self.assertRaisesRegex(TE, msg, setattr, exc, '__context__', 1) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_delattr(self): TE = TypeError try: @@ -742,7 +747,6 @@ def __init__(self, fancy_arg): x = DerivedException(fancy_arg=42) self.assertEqual(x.fancy_arg, 42) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Windows') @no_tracing def testInfiniteRecursion(self): def f(): @@ -1151,7 +1155,6 @@ class C(Exception): self.assertIs(c.__context__, b) self.assertIsNone(b.__context__) - @unittest.skip("TODO: RUSTPYTHON; Infinite loop") def test_no_hang_on_context_chain_cycle1(self): # See issue 25782. Cycle in context chain. @@ -1207,7 +1210,6 @@ class C(Exception): self.assertIs(b.__context__, a) self.assertIs(a.__context__, c) - @unittest.skip("TODO: RUSTPYTHON; Infinite loop") def test_no_hang_on_context_chain_cycle3(self): # See issue 25782. Longer context chain with cycle. @@ -1305,7 +1307,7 @@ def test_context_of_exception_in_else_and_finally(self): self.assertIs(exc, oe) self.assertIs(exc.__context__, ve) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_change_attributes(self): # See issue 7309. This was a crasher. @@ -1349,7 +1351,7 @@ def test_unicode_errors_no_object(self): for klass in klasses: self.assertEqual(str(klass.__new__(klass)), "") - @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust usize + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust usize def test_unicode_error_str_does_not_crash(self): # Test that str(UnicodeError(...)) does not crash. # See https://github.com/python/cpython/issues/123378. @@ -1373,7 +1375,45 @@ def test_unicode_error_str_does_not_crash(self): exc = UnicodeDecodeError('utf-8', encoded, start, end, '') self.assertIsInstance(str(exc), str) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Windows') + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unicode_error_evil_str_set_none_object(self): + def side_effect(exc): + exc.object = None + self.do_test_unicode_error_mutate(side_effect) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unicode_error_evil_str_del_self_object(self): + def side_effect(exc): + del exc.object + self.do_test_unicode_error_mutate(side_effect) + + def do_test_unicode_error_mutate(self, side_effect): + # Test that str(UnicodeError(...)) does not crash when + # side-effects mutate the underlying 'object' attribute. + # See https://github.com/python/cpython/issues/128974. + + class Evil(str): + def __str__(self): + side_effect(exc) + return self + + for reason, encoding in [ + ("reason", Evil("utf-8")), + (Evil("reason"), "utf-8"), + (Evil("reason"), Evil("utf-8")), + ]: + with self.subTest(encoding=encoding, reason=reason): + with self.subTest(UnicodeEncodeError): + exc = UnicodeEncodeError(encoding, "x", 0, 1, reason) + self.assertRaises(TypeError, str, exc) + with self.subTest(UnicodeDecodeError): + exc = UnicodeDecodeError(encoding, b"x", 0, 1, reason) + self.assertRaises(TypeError, str, exc) + + with self.subTest(UnicodeTranslateError): + exc = UnicodeTranslateError("x", 0, 1, Evil("reason")) + self.assertRaises(TypeError, str, exc) + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, @@ -1405,7 +1445,8 @@ def g(): self.assertIsInstance(exc, RecursionError, type(exc)) self.assertIn("maximum recursion depth exceeded", str(exc)) - + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @cpython_only @support.requires_resource('cpu') def test_trashcan_recursion(self): @@ -1421,6 +1462,7 @@ def foo(): foo() support.gc_collect() + @support.skip_emscripten_stack_overflow() @cpython_only def test_recursion_normalizing_exception(self): import_module("_testinternalcapi") @@ -1493,11 +1535,12 @@ def test_recursion_normalizing_infinite_exception(self): """ rc, out, err = script_helper.assert_python_failure("-c", code) self.assertEqual(rc, 1) - expected = b'RecursionError: maximum recursion depth exceeded' + expected = b'RecursionError' self.assertTrue(expected in err, msg=f"{expected!r} not found in {err[:3_000]!r}... (truncated)") self.assertIn(b'Done.', out) + @support.skip_emscripten_stack_overflow() def test_recursion_in_except_handler(self): def set_relative_recursion_limit(n): @@ -1603,7 +1646,7 @@ def test_exception_with_doc(self): # test basic usage of PyErr_NewException error1 = _testcapi.make_exception_with_doc("_testcapi.error1") self.assertIs(type(error1), type) - self.assertTrue(issubclass(error1, Exception)) + self.assertIsSubclass(error1, Exception) self.assertIsNone(error1.__doc__) # test with given docstring @@ -1613,21 +1656,21 @@ def test_exception_with_doc(self): # test with explicit base (without docstring) error3 = _testcapi.make_exception_with_doc("_testcapi.error3", base=error2) - self.assertTrue(issubclass(error3, error2)) + self.assertIsSubclass(error3, error2) # test with explicit base tuple class C(object): pass error4 = _testcapi.make_exception_with_doc("_testcapi.error4", doc4, (error3, C)) - self.assertTrue(issubclass(error4, error3)) - self.assertTrue(issubclass(error4, C)) + self.assertIsSubclass(error4, error3) + self.assertIsSubclass(error4, C) self.assertEqual(error4.__doc__, doc4) # test with explicit dictionary error5 = _testcapi.make_exception_with_doc("_testcapi.error5", "", error4, {'a': 1}) - self.assertTrue(issubclass(error5, error4)) + self.assertIsSubclass(error5, error4) self.assertEqual(error5.a, 1) self.assertEqual(error5.__doc__, "") @@ -1655,7 +1698,6 @@ def inner(): gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Windows') @no_tracing def test_recursion_error_cleanup(self): # Same test as above, but with "recursion exceeded" errors @@ -1677,13 +1719,13 @@ def inner(): gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; error specific to cpython') def test_errno_ENOTDIR(self): # Issue #12802: "not a directory" errors are ENOTDIR even on Windows with self.assertRaises(OSError) as cm: os.listdir(__file__) self.assertEqual(cm.exception.errno, errno.ENOTDIR, cm.exception) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'Exception ignored while calling dealloca[83 chars]200>' def test_unraisable(self): # Issue #22836: PyErr_WriteUnraisable() should give sensible reports class BrokenDel: @@ -1694,13 +1736,15 @@ def __del__(self): obj = BrokenDel() with support.catch_unraisable_exception() as cm: + obj_repr = repr(type(obj).__del__) del obj gc_collect() # For PyPy or other GCs. - self.assertEqual(cm.unraisable.object, BrokenDel.__del__) + self.assertEqual(cm.unraisable.err_msg, + f"Exception ignored while calling " + f"deallocator {obj_repr}") self.assertIsNotNone(cm.unraisable.exc_traceback) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unhandled(self): # Check for sensible reporting of unhandled exceptions for exc_type in (ValueError, BrokenStrException): @@ -1720,7 +1764,7 @@ def test_unhandled(self): self.assertIn("", report) else: self.assertIn("test message", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") @cpython_only # Python built with Py_TRACE_REFS fail with a fatal error in @@ -1799,7 +1843,6 @@ def g(): next(i) next(i) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(__debug__, "Won't work if __debug__ is False") def test_assert_shadowing(self): # Shadowing AssertionError would cause the assert statement to @@ -1861,6 +1904,30 @@ def test_memory_error_in_subinterp(self): rc, _, err = script_helper.assert_python_ok("-c", code) self.assertIn(b'MemoryError', err) + def test_keyerror_context(self): + # Make sure that _PyErr_SetKeyError() chains exceptions + try: + err1 = None + err2 = None + try: + d = {} + try: + raise ValueError("bug") + except Exception as exc: + err1 = exc + d[1] + except Exception as exc: + err2 = exc + + self.assertIsInstance(err1, ValueError) + self.assertIsInstance(err2, KeyError) + self.assertEqual(err2.__context__, err1) + finally: + # Break any potential reference cycle + exc1 = None + exc2 = None + + @cpython_only # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @@ -1874,7 +1941,7 @@ def test_exec_set_nomemory_hang(self): # PyLong_FromLong() from returning cached integers, which # don't require a memory allocation. Prepend some dummy code # to artificially increase the instruction index. - warmup_code = "a = list(range(0, 1))\n" * 20 + warmup_code = "a = list(range(0, 1))\n" * 60 user_input = warmup_code + dedent(""" try: import _testcapi @@ -1989,7 +2056,7 @@ def blech(self): class ImportErrorTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attributes(self): # Setting 'name' and 'path' should not be a problem. exc = ImportError('test') @@ -2079,7 +2146,7 @@ class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_assertion_error_location(self): cases = [ @@ -2178,7 +2245,7 @@ def test_assertion_error_location(self): result = run_script(source) self.assertEqual(result[-3:], expected) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_multiline_not_highlighted(self): cases = [ @@ -2215,7 +2282,6 @@ def test_multiline_not_highlighted(self): class SyntaxErrorTests(unittest.TestCase): maxDiff = None - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_range_of_offsets(self): cases = [ @@ -2307,6 +2373,7 @@ def test_range_of_offsets(self): self.assertIn(expected, err.getvalue()) the_exception = exc + @force_not_colorized def test_subclass(self): class MySyntaxError(SyntaxError): pass @@ -2322,7 +2389,7 @@ class MySyntaxError(SyntaxError): ^^^^^ """, err.getvalue()) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encodings(self): self.addCleanup(unlink, TESTFN) source = ( @@ -2331,7 +2398,7 @@ def test_encodings(self): ) err = run_script(source.encode('cp437')) self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') - self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') + self.assertEqual(err[-2], ' ^^^') # Check backwards tokenizer errors source = '# -*- coding: ascii -*-\n\n(\n' @@ -2339,14 +2406,14 @@ def test_encodings(self): self.assertEqual(err[-3], ' (') self.assertEqual(err[-2], ' ^') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_utf8(self): # Check non utf-8 characters self.addCleanup(unlink, TESTFN) err = run_script(b"\x89") self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_source(self): def try_compile(source): with self.assertRaises(SyntaxError) as cm: @@ -2389,7 +2456,7 @@ def try_compile(source): self.assertEqual(exc.offset, 1) self.assertEqual(exc.end_offset, 12) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_file_source(self): self.addCleanup(unlink, TESTFN) err = run_script('return "ä"') @@ -2452,12 +2519,12 @@ def test_attributes_old_constructor(self): self.assertEqual(error, the_exception.text) self.assertEqual("bad bad", the_exception.msg) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incorrect_constructor(self): args = ("bad.py", 1, 2) self.assertRaises(TypeError, SyntaxError, "bad bad", args) - args = ("bad.py", 1, 2, 4, 5, 6, 7) + args = ("bad.py", 1, 2, 4, 5, 6, 7, 8) self.assertRaises(TypeError, SyntaxError, "bad bad", args) args = ("bad.py", 1, 2, "abcdefg", 1) @@ -2514,7 +2581,7 @@ def in_except(): pass self.lineno_after_raise(in_except, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_other_except(self): def other_except(): try: @@ -2532,7 +2599,7 @@ def in_named_except(): pass self.lineno_after_raise(in_named_except, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_in_try(self): def in_try(): try: @@ -2559,7 +2626,6 @@ def in_finally_except(): pass self.lineno_after_raise(in_finally_except, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_with(self): class Noop: def __enter__(self): @@ -2572,7 +2638,7 @@ def after_with(): pass self.lineno_after_raise(after_with, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_lineno_shows_as_none(self): def f(): 1/0 diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py new file mode 100644 index 00000000000..483d5ad5f2b --- /dev/null +++ b/Lib/test/test_extcall.py @@ -0,0 +1,561 @@ + +"""Doctest for method/function calls. + +We're going the use these types for extra testing + + >>> from collections import UserList + >>> from collections import UserDict + +We're defining four helper functions + + >>> from test import support + >>> def e(a,b): + ... print(a, b) + + >>> def f(*a, **k): + ... print(a, support.sortdict(k)) + + >>> def g(x, *y, **z): + ... print(x, y, support.sortdict(z)) + + >>> def h(j=1, a=2, h=3): + ... print(j, a, h) + +Argument list examples + + >>> f() + () {} + >>> f(1) + (1,) {} + >>> f(1, 2) + (1, 2) {} + >>> f(1, 2, 3) + (1, 2, 3) {} + >>> f(1, 2, 3, *(4, 5)) + (1, 2, 3, 4, 5) {} + >>> f(1, 2, 3, *[4, 5]) + (1, 2, 3, 4, 5) {} + >>> f(*[1, 2, 3], 4, 5) + (1, 2, 3, 4, 5) {} + >>> f(1, 2, 3, *UserList([4, 5])) + (1, 2, 3, 4, 5) {} + >>> f(1, 2, 3, *[4, 5], *[6, 7]) + (1, 2, 3, 4, 5, 6, 7) {} + >>> f(1, *[2, 3], 4, *[5, 6], 7) + (1, 2, 3, 4, 5, 6, 7) {} + >>> f(*UserList([1, 2]), *UserList([3, 4]), 5, *UserList([6, 7])) + (1, 2, 3, 4, 5, 6, 7) {} + +Here we add keyword arguments + + >>> f(1, 2, 3, **{'a':4, 'b':5}) + (1, 2, 3) {'a': 4, 'b': 5} + >>> f(1, 2, **{'a': -1, 'b': 5}, **{'a': 4, 'c': 6}) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' + >>> f(1, 2, **{'a': -1, 'b': 5}, a=4, c=6) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' + >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5}) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' + >>> f(1, 2, 3, *[4, 5], **{'a':6, 'b':7}) + (1, 2, 3, 4, 5) {'a': 6, 'b': 7} + >>> f(1, 2, 3, x=4, y=5, *(6, 7), **{'a':8, 'b': 9}) + (1, 2, 3, 6, 7) {'a': 8, 'b': 9, 'x': 4, 'y': 5} + >>> f(1, 2, 3, *[4, 5], **{'c': 8}, **{'a':6, 'b':7}) + (1, 2, 3, 4, 5) {'a': 6, 'b': 7, 'c': 8} + >>> f(1, 2, 3, *(4, 5), x=6, y=7, **{'a':8, 'b': 9}) + (1, 2, 3, 4, 5) {'a': 8, 'b': 9, 'x': 6, 'y': 7} + + >>> f(1, 2, 3, **UserDict(a=4, b=5)) + (1, 2, 3) {'a': 4, 'b': 5} + >>> f(1, 2, 3, *(4, 5), **UserDict(a=6, b=7)) + (1, 2, 3, 4, 5) {'a': 6, 'b': 7} + >>> f(1, 2, 3, x=4, y=5, *(6, 7), **UserDict(a=8, b=9)) + (1, 2, 3, 6, 7) {'a': 8, 'b': 9, 'x': 4, 'y': 5} + >>> f(1, 2, 3, *(4, 5), x=6, y=7, **UserDict(a=8, b=9)) + (1, 2, 3, 4, 5) {'a': 8, 'b': 9, 'x': 6, 'y': 7} + +Mix keyword arguments and dict unpacking + + >>> d1 = {'a':1} + + >>> d2 = {'c':3} + + >>> f(b=2, **d1, **d2) + () {'a': 1, 'b': 2, 'c': 3} + + >>> f(**d1, b=2, **d2) + () {'a': 1, 'b': 2, 'c': 3} + + >>> f(**d1, **d2, b=2) + () {'a': 1, 'b': 2, 'c': 3} + + >>> f(**d1, b=2, **d2, d=4) + () {'a': 1, 'b': 2, 'c': 3, 'd': 4} + +Examples with invalid arguments (TypeErrors). We're also testing the function +names in the exception messages. + +Verify clearing of SF bug #733667 + + >>> e(c=4) + Traceback (most recent call last): + ... + TypeError: e() got an unexpected keyword argument 'c' + + >>> g() + Traceback (most recent call last): + ... + TypeError: g() missing 1 required positional argument: 'x' + + >>> g(*()) + Traceback (most recent call last): + ... + TypeError: g() missing 1 required positional argument: 'x' + + >>> g(*(), **{}) + Traceback (most recent call last): + ... + TypeError: g() missing 1 required positional argument: 'x' + + >>> g(1) + 1 () {} + >>> g(1, 2) + 1 (2,) {} + >>> g(1, 2, 3) + 1 (2, 3) {} + >>> g(1, 2, 3, *(4, 5)) + 1 (2, 3, 4, 5) {} + + >>> class Nothing: pass + ... + >>> g(*Nothing()) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing + + >>> class Nothing: + ... def __len__(self): return 5 + ... + + >>> g(*Nothing()) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing + + >>> class Nothing(): + ... def __len__(self): return 5 + ... def __getitem__(self, i): + ... if i<3: return i + ... else: raise IndexError(i) + ... + + >>> g(*Nothing()) + 0 (1, 2) {} + + >>> class Nothing: + ... def __init__(self): self.c = 0 + ... def __iter__(self): return self + ... def __next__(self): + ... if self.c == 4: + ... raise StopIteration + ... c = self.c + ... self.c += 1 + ... return c + ... + + >>> g(*Nothing()) + 0 (1, 2, 3) {} + +Check for issue #4806: Does a TypeError in a generator get propagated with the +right error message? (Also check with other iterables.) + + >>> def broken(): raise TypeError("myerror") + ... + + >>> g(*(broken() for i in range(1))) + Traceback (most recent call last): + ... + TypeError: myerror + >>> g(*range(1), *(broken() for i in range(1))) + Traceback (most recent call last): + ... + TypeError: myerror + + >>> class BrokenIterable1: + ... def __iter__(self): + ... raise TypeError('myerror') + ... + >>> g(*BrokenIterable1()) + Traceback (most recent call last): + ... + TypeError: myerror + >>> g(*range(1), *BrokenIterable1()) + Traceback (most recent call last): + ... + TypeError: myerror + + >>> class BrokenIterable2: + ... def __iter__(self): + ... yield 0 + ... raise TypeError('myerror') + ... + >>> g(*BrokenIterable2()) + Traceback (most recent call last): + ... + TypeError: myerror + >>> g(*range(1), *BrokenIterable2()) + Traceback (most recent call last): + ... + TypeError: myerror + + >>> class BrokenSequence: + ... def __getitem__(self, idx): + ... raise TypeError('myerror') + ... + >>> g(*BrokenSequence()) + Traceback (most recent call last): + ... + TypeError: myerror + >>> g(*range(1), *BrokenSequence()) + Traceback (most recent call last): + ... + TypeError: myerror + +Make sure that the function doesn't stomp the dictionary + + >>> d = {'a': 1, 'b': 2, 'c': 3} + >>> d2 = d.copy() + >>> g(1, d=4, **d) + 1 () {'a': 1, 'b': 2, 'c': 3, 'd': 4} + >>> d == d2 + True + +What about willful misconduct? + + >>> def saboteur(**kw): + ... kw['x'] = 'm' + ... return kw + + >>> d = {} + >>> kw = saboteur(a=1, **d) + >>> d + {} + + + >>> g(1, 2, 3, **{'x': 4, 'y': 5}) + Traceback (most recent call last): + ... + TypeError: g() got multiple values for argument 'x' + + >>> f(**{1:2}) + Traceback (most recent call last): + ... + TypeError: keywords must be strings + + >>> h(**{'e': 2}) + Traceback (most recent call last): + ... + TypeError: h() got an unexpected keyword argument 'e' + + >>> h(*h) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after * must be an iterable, not function + + >>> h(1, *h) + Traceback (most recent call last): + ... + TypeError: Value after * must be an iterable, not function + + >>> h(*[1], *h) + Traceback (most recent call last): + ... + TypeError: Value after * must be an iterable, not function + + >>> dir(*h) + Traceback (most recent call last): + ... + TypeError: dir() argument after * must be an iterable, not function + + >>> nothing = None + >>> nothing(*h) + Traceback (most recent call last): + ... + TypeError: None argument after * must be an iterable, \ +not function + + >>> h(**h) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + + >>> h(**[]) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + + >>> h(a=1, **h) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + + >>> h(a=1, **[]) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + + >>> h(**{'a': 1}, **h) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + + >>> h(**{'a': 1}, **[]) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + + >>> dir(**h) + Traceback (most recent call last): + ... + TypeError: dir() argument after ** must be a mapping, not function + + >>> nothing(**h) + Traceback (most recent call last): + ... + TypeError: None argument after ** must be a mapping, \ +not function + + >>> dir(b=1, **{'b': 1}) + Traceback (most recent call last): + ... + TypeError: dir() got multiple values for keyword argument 'b' + +Test a kwargs mapping with duplicated keys. + + >>> from collections.abc import Mapping + >>> class MultiDict(Mapping): + ... def __init__(self, items): + ... self._items = items + ... + ... def __iter__(self): + ... return (k for k, v in self._items) + ... + ... def __getitem__(self, key): + ... for k, v in self._items: + ... if k == key: + ... return v + ... raise KeyError(key) + ... + ... def __len__(self): + ... return len(self._items) + ... + ... def keys(self): + ... return [k for k, v in self._items] + ... + ... def values(self): + ... return [v for k, v in self._items] + ... + ... def items(self): + ... return [(k, v) for k, v in self._items] + ... + >>> g(**MultiDict([('x', 1), ('y', 2)])) + 1 () {'y': 2} + + >>> g(**MultiDict([('x', 1), ('x', 2)])) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' + + >>> g(a=3, **MultiDict([('x', 1), ('x', 2)])) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' + + >>> g(**MultiDict([('a', 3)]), **MultiDict([('x', 1), ('x', 2)])) + Traceback (most recent call last): + ... + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' + +Call with dict subtype: + + >>> class MyDict(dict): + ... pass + + >>> def s1(**kwargs): + ... return kwargs + >>> def s2(*args, **kwargs): + ... return (args, kwargs) + >>> def s3(*, n, **kwargs): + ... return (n, kwargs) + + >>> md = MyDict({'a': 1, 'b': 2}) + >>> assert s1(**md) == {'a': 1, 'b': 2} + >>> assert s2(*(1, 2), **md) == ((1, 2), {'a': 1, 'b': 2}) + >>> assert s3(**MyDict({'n': 1, 'b': 2})) == (1, {'b': 2}) + >>> s3(**md) + Traceback (most recent call last): + ... + TypeError: s3() missing 1 required keyword-only argument: 'n' + +Another helper function + + >>> def f2(*a, **b): + ... return a, b + + + >>> d = {} + >>> for i in range(512): + ... key = 'k%d' % i + ... d[key] = i + >>> a, b = f2(1, *(2,3), **d) + >>> len(a), len(b), b == d + (3, 512, True) + + >>> class Foo: + ... def method(self, arg1, arg2): + ... return arg1+arg2 + + >>> x = Foo() + >>> Foo.method(*(x, 1, 2)) + 3 + >>> Foo.method(x, *(1, 2)) + 3 + >>> Foo.method(*(1, 2, 3)) + 5 + >>> Foo.method(1, *[2, 3]) + 5 + +A PyCFunction that takes only positional parameters should allow an +empty keyword dictionary to pass without a complaint, but raise a +TypeError if te dictionary is not empty + + >>> try: + ... silence = id(1, *{}) + ... True + ... except: + ... False + True + + >>> id(1, **{'foo': 1}) # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + TypeError: id() takes no keyword arguments + +A corner case of keyword dictionary items being deleted during +the function call setup. See . + + >>> class Name(str): + ... def __eq__(self, other): + ... try: + ... del x[self] + ... except KeyError: + ... pass + ... return str.__eq__(self, other) + ... def __hash__(self): + ... return str.__hash__(self) + + >>> x = {Name("a"):1, Name("b"):2} + >>> def f(a, b): + ... print(a,b) + >>> f(**x) + 1 2 + +Too many arguments: + + >>> def f(): pass + >>> f(1) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 was given + >>> def f(a): pass + >>> f(1, 2) + Traceback (most recent call last): + ... + TypeError: f() takes 1 positional argument but 2 were given + >>> def f(a, b=1): pass + >>> f(1, 2, 3) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 were given + >>> def f(*, kw): pass + >>> f(1, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given + >>> def f(*, kw, b): pass + >>> f(1, 2, 3, b=3, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 3 positional arguments (and 2 keyword-only arguments) were given + >>> def f(a, b=2, *, kw): pass + >>> f(2, 3, 4, kw=4) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 positional arguments (and 1 keyword-only argument) were given + +Too few and missing arguments: + + >>> def f(a): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' + >>> def f(a, b): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 2 required positional arguments: 'a' and 'b' + >>> def f(a, b, c): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 3 required positional arguments: 'a', 'b', and 'c' + >>> def f(a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required positional arguments: 'a', 'b', 'c', 'd', and 'e' + >>> def f(a, b=4, c=5, d=5): pass + >>> f(c=12, b=9) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' + +Same with keyword only args: + + >>> def f(*, w): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 1 required keyword-only argument: 'w' + >>> def f(*, a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required keyword-only arguments: 'a', 'b', 'c', 'd', and 'e' + +""" + +import doctest +import unittest + +EXPECTED_FAILURE = doctest.register_optionflag('EXPECTED_FAILURE') # TODO: RUSTPYTHON +class CustomOutputChecker(doctest.OutputChecker): # TODO: RUSTPYTHON + def check_output(self, want, got, optionflags): # TODO: RUSTPYTHON + if optionflags & EXPECTED_FAILURE: # TODO: RUSTPYTHON + return not super().check_output(want, got, optionflags) # TODO: RUSTPYTHON + return super().check_output(want, got, optionflags) # TODO: RUSTPYTHON + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(checker=CustomOutputChecker())) # TODO: RUSTPYTHON + return tests + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 5596e5b1669..090fb3a1484 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -22,6 +22,16 @@ TIMEOUT = 0.5 +STACK_HEADER_STR = r'Stack (most recent call first):' + +# Regular expressions +STACK_HEADER = re.escape(STACK_HEADER_STR) +THREAD_NAME = r'( \[.*\])?' +THREAD_ID = fr'Thread 0x[0-9a-f]+{THREAD_NAME}' +THREAD_HEADER = fr'{THREAD_ID} \(most recent call first\):' +CURRENT_THREAD_ID = fr'Current thread 0x[0-9a-f]+{THREAD_NAME}' +CURRENT_THREAD_HEADER = fr'{CURRENT_THREAD_ID} \(most recent call first\):' + def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header @@ -45,6 +55,13 @@ def temporary_filename(): finally: os_helper.unlink(filename) + +ADDRESS_EXPR = "0x[0-9a-f]+" +C_STACK_REGEX = [ + r"Current thread's C stack trace \(most recent call first\):", + fr'( Binary file ".+"(, at .*(\+|-){ADDRESS_EXPR})? \[{ADDRESS_EXPR}\])|(<.+>)' +] + class FaultHandlerTests(unittest.TestCase): def get_output(self, code, filename=None, fd=None): @@ -93,6 +110,7 @@ def check_error(self, code, lineno, fatal_error, *, fd=None, know_current_thread=True, py_fatal_error=False, garbage_collecting=False, + c_stack=True, function=''): """ Check that the fault handler for fatal errors is enabled and check the @@ -100,21 +118,32 @@ def check_error(self, code, lineno, fatal_error, *, Raise an error if the output doesn't match the expected format. """ - if all_threads: + all_threads_disabled = ( + all_threads + and (not sys._is_gil_enabled()) + ) + if all_threads and not all_threads_disabled: if know_current_thread: - header = 'Current thread 0x[0-9a-f]+' + header = CURRENT_THREAD_HEADER else: - header = 'Thread 0x[0-9a-f]+' + header = THREAD_HEADER else: - header = 'Stack' + header = STACK_HEADER regex = [f'^{fatal_error}'] if py_fatal_error: regex.append("Python runtime state: initialized") regex.append('') - regex.append(fr'{header} \(most recent call first\):') - if garbage_collecting: - regex.append(' Garbage-collecting') - regex.append(fr' File "", line {lineno} in {function}') + if all_threads_disabled and not py_fatal_error: + regex.append("") + regex.append(fr'{header}') + if support.Py_GIL_DISABLED and py_fatal_error and not know_current_thread: + regex.append(" ") + else: + if garbage_collecting and not all_threads_disabled: + regex.append(' Garbage-collecting') + regex.append(fr' File "", line {lineno} in {function}') + if c_stack: + regex.extend(C_STACK_REGEX) regex = '\n'.join(regex) if other_regex: @@ -137,8 +166,6 @@ def check_windows_exception(self, code, line_number, name_regex, **kw): fatal_error = 'Windows fatal exception: %s' % name_regex self.check_error(code, line_number, fatal_error, **kw) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform.startswith('aix'), "the first page of memory is a mapped read-only on AIX") def test_read_null(self): @@ -162,8 +189,6 @@ def test_read_null(self): 3, 'access violation') - # TODO: RUSTPYTHON, AssertionError: Regex didn't match - @unittest.expectedFailure @skip_segfault_on_android def test_sigsegv(self): self.check_fatal_error(""" @@ -174,8 +199,7 @@ def test_sigsegv(self): 3, 'Segmentation fault') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Regex didn't match: '(?m)^Fatal Python error: Segmentation fault\n\n\nStack\\ \\(most\\ recent\\ call\\ first\\):\n File "", line 9 in __del__\nCurrent thread\'s C stack trace \\(most recent call first\\):\n( Binary file ".+"(, at .*(\\+|-)0x[0-9a-f]+)? \\[0x[0-9a-f]+\\])|(<.+>)' not found in 'exit' @skip_segfault_on_android def test_gc(self): # bpo-44466: Detect if the GC is running @@ -212,8 +236,7 @@ def __del__(self): function='__del__', garbage_collecting=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 == 0 def test_fatal_error_c_thread(self): self.check_fatal_error(""" import faulthandler @@ -226,8 +249,7 @@ def test_fatal_error_c_thread(self): func='faulthandler_fatal_error_thread', py_fatal_error=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.skip_if_sanitizer("TSAN itercepts SIGABRT", thread=True) def test_sigabrt(self): self.check_fatal_error(""" import faulthandler @@ -237,10 +259,9 @@ def test_sigabrt(self): 3, 'Aborted') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == 'win32', "SIGFPE cannot be caught on Windows") + @support.skip_if_sanitizer("TSAN itercepts SIGFPE", thread=True) def test_sigfpe(self): self.check_fatal_error(""" import faulthandler @@ -252,6 +273,7 @@ def test_sigfpe(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') + @support.skip_if_sanitizer("TSAN itercepts SIGBUS", thread=True) @skip_segfault_on_android def test_sigbus(self): self.check_fatal_error(""" @@ -266,6 +288,7 @@ def test_sigbus(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGILL'), 'need signal.SIGILL') + @support.skip_if_sanitizer("TSAN itercepts SIGILL", thread=True) @skip_segfault_on_android def test_sigill(self): self.check_fatal_error(""" @@ -291,13 +314,9 @@ def check_fatal_error_func(self, release_gil): func='_testcapi_fatal_error_impl', py_fatal_error=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fatal_error(self): self.check_fatal_error_func(False) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fatal_error_without_gil(self): self.check_fatal_error_func(True) @@ -316,8 +335,6 @@ def test_stack_overflow(self): '(?:Segmentation fault|Bus error)', other_regex='unable to raise a stack overflow') - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_segfault_on_android def test_gil_released(self): self.check_fatal_error(""" @@ -328,8 +345,6 @@ def test_gil_released(self): 3, 'Segmentation fault') - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: @@ -343,8 +358,6 @@ def test_enable_file(self): 'Segmentation fault', filename=filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") @skip_segfault_on_android @@ -361,8 +374,6 @@ def test_enable_fd(self): 'Segmentation fault', fd=fd) - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_segfault_on_android def test_enable_single_thread(self): self.check_fatal_error(""" @@ -389,8 +400,7 @@ def test_disable(self): "%r is present in %r" % (not_expected, stderr)) self.assertNotEqual(exitcode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Cannot find 'Extension modules:' in 'Fatal Python error: Segmentation fault\n\nCurrent thread 0x0000000000004284 (most recent call first):\n File "", line 6 in ' @skip_segfault_on_android def test_dump_ext_modules(self): code = """ @@ -511,7 +521,7 @@ def funcA(): else: lineno = 14 expected = [ - 'Stack (most recent call first):', + f'{STACK_HEADER_STR}', ' File "", line %s in funcB' % lineno, ' File "", line 17 in funcA', ' File "", line 19 in ' @@ -523,14 +533,10 @@ def funcA(): def test_dump_traceback(self): self.check_dump_traceback() - # TODO: RUSTPYTHON - binary file write needs different handling - @unittest.expectedFailure def test_dump_traceback_file(self): with temporary_filename() as filename: self.check_dump_traceback(filename=filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") def test_dump_traceback_fd(self): @@ -553,7 +559,7 @@ def {func_name}(): func_name=func_name, ) expected = [ - 'Stack (most recent call first):', + f'{STACK_HEADER_STR}', ' File "", line 4 in %s' % truncated, ' File "", line 6 in ' ] @@ -607,28 +613,24 @@ def run(self): lineno = 10 # When the traceback is dumped, the waiter thread may be in the # `self.running.set()` call or in `self.stop.wait()`. - regex = r""" - ^Thread 0x[0-9a-f]+ \(most recent call first\): + regex = fr""" + ^{THREAD_HEADER} (?: File ".*threading.py", line [0-9]+ in [_a-z]+ ){{1,3}} File "", line (?:22|23) in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap - Current thread 0x[0-9a-f]+ \(most recent call first\): + {CURRENT_THREAD_HEADER} File "", line {lineno} in dump File "", line 28 in $ """ - regex = dedent(regex.format(lineno=lineno)).strip() + regex = dedent(regex).strip() self.assertRegex(output, regex) self.assertEqual(exitcode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_threads(self): self.check_dump_traceback_threads(None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_threads_file(self): with temporary_filename() as filename: self.check_dump_traceback_threads(filename) @@ -688,44 +690,33 @@ def func(timeout, repeat, cancel, file, loops): count = loops if repeat: count *= 2 - header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+ \(most recent call first\):\n' % timeout_str + header = (fr'Timeout \({timeout_str}\)!\n' + fr'{THREAD_HEADER}\n') regex = expected_traceback(17, 26, header, min_count=count) self.assertRegex(trace, regex) else: self.assertEqual(trace, '') self.assertEqual(exitcode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_later(self): self.check_dump_traceback_later() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_later_repeat(self): self.check_dump_traceback_later(repeat=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_later_cancel(self): self.check_dump_traceback_later(cancel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_traceback_later_file(self): with temporary_filename() as filename: self.check_dump_traceback_later(filename=filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") def test_dump_traceback_later_fd(self): with tempfile.TemporaryFile('wb+') as fp: self.check_dump_traceback_later(fd=fp.fileno()) - # TODO: RUSTPYTHON - @unittest.expectedFailure @support.requires_resource('walltime') def test_dump_traceback_later_twice(self): self.check_dump_traceback_later(loops=2) @@ -801,9 +792,9 @@ def handler(signum, frame): trace = '\n'.join(trace) if not unregister: if all_threads: - regex = r'Current thread 0x[0-9a-f]+ \(most recent call first\):\n' + regex = fr'{CURRENT_THREAD_HEADER}\n' else: - regex = r'Stack \(most recent call first\):\n' + regex = fr'{STACK_HEADER}\n' regex = expected_traceback(14, 32, regex) self.assertRegex(trace, regex) else: @@ -813,37 +804,26 @@ def handler(signum, frame): else: self.assertEqual(exitcode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_register(self): self.check_register() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unregister(self): self.check_register(unregister=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_register_file(self): with temporary_filename() as filename: self.check_register(filename=filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") def test_register_fd(self): with tempfile.TemporaryFile('wb+') as fp: self.check_register(fd=fp.fileno()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_register_threads(self): self.check_register(all_threads=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.skip_if_sanitizer("gh-129825: hangs under TSAN", thread=True) def test_register_chain(self): self.check_register(chain=True) @@ -871,8 +851,6 @@ def test_stderr_None(self): with self.check_stderr_none(): faulthandler.register(signal.SIGUSR1) - # TODO: RUSTPYTHON, AttributeError: module 'msvcrt' has no attribute 'GetErrorMode' - @unittest.expectedFailure @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') def test_raise_exception(self): for exc, name in ( @@ -985,5 +963,37 @@ def run(self): _, exitcode = self.get_output(code) self.assertEqual(exitcode, 0) + def check_c_stack(self, output): + starting_line = output.pop(0) + self.assertRegex(starting_line, C_STACK_REGEX[0]) + self.assertGreater(len(output), 0) + + for line in output: + with self.subTest(line=line): + if line != '': # Ignore trailing or leading newlines + self.assertRegex(line, C_STACK_REGEX[1]) + + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 0 + def test_dump_c_stack(self): + code = dedent(""" + import faulthandler + faulthandler.dump_c_stack() + """) + output, exitcode = self.get_output(code) + self.assertEqual(exitcode, 0) + self.check_c_stack(output) + + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'faulthandler' has no attribute 'dump_c_stack' + def test_dump_c_stack_file(self): + import tempfile + + with tempfile.TemporaryFile("w+") as tmp: + faulthandler.dump_c_stack(file=tmp) + tmp.flush() # Just in case + tmp.seek(0) + self.check_c_stack(tmp.read().split("\n")) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index d64f3f797d8..36aea52a5a9 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -344,9 +344,6 @@ def testIteration(self): class COtherFileTests(OtherFileTests, unittest.TestCase): open = io.open - @unittest.expectedFailure # TODO: RUSTPYTHON - def testSetBufferSize(self): - return super().testSetBufferSize() class PyOtherFileTests(OtherFileTests, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_file_eintr.py index 55cc31dc59f..fa9c6637fd4 100644 --- a/Lib/test/test_file_eintr.py +++ b/Lib/test/test_file_eintr.py @@ -152,7 +152,6 @@ def _test_reading(self, data_to_write, read_and_verify_code): '"got data %r\\nexpected %r" % (got, expected))' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readline(self): """readline() must handle signals and not lose data.""" self._test_reading( @@ -161,7 +160,6 @@ def test_readline(self): read_method_name='readline', expected=b'hello, world!\n')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlines(self): """readlines() must handle signals and not lose data.""" self._test_reading( @@ -170,7 +168,6 @@ def test_readlines(self): read_method_name='readlines', expected=[b'hello\n', b'world!\n'])) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readall(self): """readall() must handle signals and not lose data.""" self._test_reading( @@ -189,6 +186,11 @@ def test_readall(self): class CTestFileIOSignalInterrupt(TestFileIOSignalInterrupt, unittest.TestCase): modname = '_io' + # TODO: RUSTPYTHON - _io.FileIO.readall uses read_to_end which differs from _pyio.FileIO.readall + @unittest.expectedFailure + def test_readall(self): + super().test_readall() + class PyTestFileIOSignalInterrupt(TestFileIOSignalInterrupt, unittest.TestCase): modname = '_pyio' @@ -200,7 +202,6 @@ def _generate_infile_setup_code(self): 'assert isinstance(infile, io.BufferedReader)' % self.modname) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readall(self): """BufferedReader.read() must handle signals and not lose data.""" self._test_reading( @@ -224,7 +225,6 @@ def _generate_infile_setup_code(self): 'assert isinstance(infile, io.TextIOWrapper)' % self.modname) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readline(self): """readline() must handle signals and not lose data.""" self._test_reading( @@ -233,7 +233,6 @@ def test_readline(self): read_method_name='readline', expected='hello, world!\n')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlines(self): """readlines() must handle signals and not lose data.""" self._test_reading( @@ -242,7 +241,6 @@ def test_readlines(self): read_method_name='readlines', expected=['hello\n', 'world!\n'])) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_readall(self): """read() must handle signals and not lose data.""" self._test_reading( diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 9b5ac12bccc..2c83667b22f 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -8,11 +9,24 @@ from test.support import os_helper +def _create_file_shallow_equal(template_path, new_path): + """create a file with the same size and mtime but different content.""" + shutil.copy2(template_path, new_path) + with open(new_path, 'r+b') as f: + next_char = bytearray(f.read(1)) + next_char[0] = (next_char[0] + 1) % 256 + f.seek(0) + f.write(next_char) + shutil.copystat(template_path, new_path) + assert os.stat(new_path).st_size == os.stat(template_path).st_size + assert os.stat(new_path).st_mtime == os.stat(template_path).st_mtime + class FileCompareTestCase(unittest.TestCase): def setUp(self): self.name = os_helper.TESTFN self.name_same = os_helper.TESTFN + '-same' self.name_diff = os_helper.TESTFN + '-diff' + self.name_same_shallow = os_helper.TESTFN + '-same-shallow' data = 'Contents of file go here.\n' for name in [self.name, self.name_same, self.name_diff]: with open(name, 'w', encoding="utf-8") as output: @@ -20,12 +34,19 @@ def setUp(self): with open(self.name_diff, 'a+', encoding="utf-8") as output: output.write('An extra line.\n') + + for name in [self.name_same, self.name_diff]: + shutil.copystat(self.name, name) + + _create_file_shallow_equal(self.name, self.name_same_shallow) + self.dir = tempfile.gettempdir() def tearDown(self): os.unlink(self.name) os.unlink(self.name_same) os.unlink(self.name_diff) + os.unlink(self.name_same_shallow) def test_matching(self): self.assertTrue(filecmp.cmp(self.name, self.name), @@ -36,12 +57,17 @@ def test_matching(self): "Comparing file to identical file fails") self.assertTrue(filecmp.cmp(self.name, self.name_same, shallow=False), "Comparing file to identical file fails") + self.assertTrue(filecmp.cmp(self.name, self.name_same_shallow), + "Shallow identical files should be considered equal") def test_different(self): self.assertFalse(filecmp.cmp(self.name, self.name_diff), "Mismatched files compare as equal") self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + self.assertFalse(filecmp.cmp(self.name, self.name_same_shallow, + shallow=False), + "Mismatched file to shallow identical file compares as equal") def test_cache_clear(self): first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) @@ -56,6 +82,8 @@ def setUp(self): self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + self.dir_diff_file = os.path.join(tmpdir, 'dir-diff-file') + self.dir_same_shallow = os.path.join(tmpdir, 'dir-same-shallow') # Another dir is created under dir_same, but it has a name from the # ignored list so it should not affect testing results. @@ -63,7 +91,17 @@ def setUp(self): self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): + + shutil.rmtree(self.dir, True) + os.mkdir(self.dir) + subdir_path = os.path.join(self.dir, 'subdir') + os.mkdir(subdir_path) + dir_file_path = os.path.join(self.dir, "file") + with open(dir_file_path, 'w', encoding="utf-8") as output: + output.write(data) + + for dir in (self.dir_same, self.dir_same_shallow, + self.dir_diff, self.dir_diff_file): shutil.rmtree(dir, True) os.mkdir(dir) subdir_path = os.path.join(dir, 'subdir') @@ -72,14 +110,25 @@ def setUp(self): fn = 'FiLe' # Verify case-insensitive comparison else: fn = 'file' - with open(os.path.join(dir, fn), 'w', encoding="utf-8") as output: - output.write(data) + + file_path = os.path.join(dir, fn) + + if dir is self.dir_same_shallow: + _create_file_shallow_equal(dir_file_path, file_path) + else: + shutil.copy2(dir_file_path, file_path) with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: output.write('An extra file.\n') + # Add different file2 with respect to dir_diff + with open(os.path.join(self.dir_diff_file, 'file2'), 'w', encoding="utf-8") as output: + output.write('Different contents.\n') + + def tearDown(self): - for dir in (self.dir, self.dir_same, self.dir_diff): + for dir in (self.dir, self.dir_same, self.dir_diff, + self.dir_same_shallow, self.dir_diff_file): shutil.rmtree(dir) def test_default_ignores(self): @@ -102,25 +151,65 @@ def test_cmpfiles(self): shallow=False), "Comparing directory to same fails") - # Add different file2 - with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - - self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_same, + self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_diff_file, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails") + def test_cmpfiles_invalid_names(self): + # See https://github.com/python/cpython/issues/122400. + for file, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long filename'), + ]: + for other_dir in [self.dir, self.dir_same, self.dir_diff]: + with self.subTest(f'cmpfiles: {desc}', other_dir=other_dir): + res = filecmp.cmpfiles(self.dir, other_dir, [file]) + self.assertTupleEqual(res, ([], [], [file])) + + def test_dircmp_invalid_names(self): + for bad_dir, desc in [ + ('\x00', 'NUL bytes dirname'), + (f'Top{os.sep}Mid\x00', 'dirname with embedded NUL bytes'), + ("\uD834\uDD1E", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long dirname'), + ]: + d1 = filecmp.dircmp(self.dir, bad_dir) + d2 = filecmp.dircmp(bad_dir, self.dir) + for target in [ + # attributes where os.listdir() raises OSError or ValueError + 'left_list', 'right_list', + 'left_only', 'right_only', 'common', + ]: + with self.subTest(f'dircmp(ok, bad): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d1, target) + with self.subTest(f'dircmp(bad, ok): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d2, target) def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" self.assertEqual(sorted(actual), sorted(expected)) + def test_dircmp_identical_directories(self): + self._assert_dircmp_identical_directories() + self._assert_dircmp_identical_directories(shallow=False) + + def test_dircmp_different_file(self): + self._assert_dircmp_different_file() + self._assert_dircmp_different_file(shallow=False) - def test_dircmp(self): + def test_dircmp_different_directories(self): + self._assert_dircmp_different_directories() + self._assert_dircmp_different_directories(shallow=False) + + def _assert_dircmp_identical_directories(self, **options): # Check attributes for comparison of two identical directories left_dir, right_dir = self.dir, self.dir_same - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) if self.caseinsensitive: @@ -142,9 +231,10 @@ def test_dircmp(self): ] self._assert_report(d.report, expected_report) + def _assert_dircmp_different_directories(self, **options): # Check attributes for comparison of two different directories (right) left_dir, right_dir = self.dir, self.dir_diff - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'subdir']) @@ -164,12 +254,8 @@ def test_dircmp(self): self._assert_report(d.report, expected_report) # Check attributes for comparison of two different directories (left) - left_dir, right_dir = self.dir, self.dir_diff - shutil.move( - os.path.join(self.dir_diff, 'file2'), - os.path.join(self.dir, 'file2') - ) - d = filecmp.dircmp(left_dir, right_dir) + left_dir, right_dir = self.dir_diff, self.dir + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'file2', 'subdir']) @@ -180,27 +266,62 @@ def test_dircmp(self): self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, []) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), - "Only in {} : ['file2']".format(self.dir), + "diff {} {}".format(self.dir_diff, self.dir), + "Only in {} : ['file2']".format(self.dir_diff), "Identical files : ['file']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) - # Add different file2 - with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - d = filecmp.dircmp(self.dir, self.dir_diff) + + def _assert_dircmp_different_file(self, **options): + # A different file2 + d = filecmp.dircmp(self.dir_diff, self.dir_diff_file, **options) self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, ['file2']) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), + "diff {} {}".format(self.dir_diff, self.dir_diff_file), "Identical files : ['file']", "Differing files : ['file2']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) + def test_dircmp_no_shallow_different_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow, shallow=False) + self.assertEqual(d.same_files, []) + self.assertEqual(d.diff_files, ['file']) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Differing files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_same_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow) + self.assertEqual(d.same_files, ['file']) + self.assertEqual(d.diff_files, []) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Identical files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 1a6ef3cd275..b340ef7ed16 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -980,8 +980,6 @@ def check(errors, expected_lines): check('replace', ['\ufffdabc']) check('backslashreplace', ['\\x80abc']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_modes(self): with open(TESTFN, 'wb') as f: # UTF-7 is a convenient, seldom used encoding diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index edc29b34d54..4f195cf045c 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,8 +10,8 @@ from functools import wraps from test.support import ( - cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, - infinite_recursion, + cpython_only, swap_attr, gc_collect, is_wasi, + infinite_recursion, strace_helper ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, @@ -24,6 +24,9 @@ import _pyio # Python implementation of io +_strace_flags=["--trace=%file,%desc"] + + class AutoFileTests: # file tests for which a test file is automatically set up @@ -359,31 +362,148 @@ def testErrnoOnClosedReadinto(self, f): a = array('b', b'x'*10) f.readinto(a) -class CAutoFileTests(AutoFileTests, unittest.TestCase): - FileIO = _io.FileIO - modulename = '_io' + @unittest.skip("TODO: RUSTPYTHON; extra ioctl(TCGETS) syscall for isatty check") + @strace_helper.requires_strace() + def test_syscalls_read(self): + """Check set of system calls during common I/O patterns + + It's expected as bits of the I/O implementation change, this will need + to change. The goal is to catch changes that unintentionally add + additional systemcalls (ex. additional calls have been looked at in + bpo-21679 and gh-120754). + """ + self.f.write(b"Hello, World!") + self.f.close() - @unittest.expectedFailure # TODO: RUSTPYTHON - def testBlksize(self): - return super().testBlksize() - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def testErrnoOnClosedTruncate(self): - return super().testErrnoOnClosedTruncate() + def check_readall(name, code, prelude="", cleanup="", + extra_checks=None): + with self.subTest(name=name): + syscalls = strace_helper.get_events(code, _strace_flags, + prelude=prelude, + cleanup=cleanup) + + # Some system calls (ex. mmap) can be used for both File I/O and + # memory allocation. Filter out the ones used for memory + # allocation. + syscalls = strace_helper.filter_memory(syscalls) + + # The first call should be an open that returns a + # file descriptor (fd). Afer that calls may vary. Once the file + # is opened, check calls refer to it by fd as the filename + # could be removed from the filesystem, renamed, etc. See: + # Time-of-check time-of-use (TOCTOU) software bug class. + # + # There are a number of related but distinct open system calls + # so not checking precise name here. + self.assertGreater( + len(syscalls), + 1, + f"Should have had at least an open call|calls={syscalls}") + fd_str = syscalls[0].returncode + + # All other calls should contain the fd in their argument set. + for ev in syscalls[1:]: + self.assertIn( + fd_str, + ev.args, + f"Looking for file descriptor in arguments|ev={ev}" + ) + + # There are a number of related syscalls used to implement + # behaviors in a libc (ex. fstat, newfstatat, statx, open, openat). + # Allow any that use the same substring. + def count_similarname(name): + return len([ev for ev in syscalls if name in ev.syscall]) + + checks = [ + # Should open and close the file exactly once + ("open", 1), + ("close", 1), + # There should no longer be an isatty call (All files being + # tested are block devices / not character devices). + ('ioctl', 0), + # Should only have one fstat (bpo-21679, gh-120754) + # note: It's important this uses a fd rather than filename, + # That is validated by the `fd` check above. + # note: fstat, newfstatat, and statx have all been observed + # here in the underlying C library implementations. + ("stat", 1) + ] + + if extra_checks: + checks += extra_checks + + for call, count in checks: + self.assertEqual( + count_similarname(call), + count, + msg=f"call={call}|count={count}|syscalls={syscalls}" + ) + + # "open, read, close" file using different common patterns. + check_readall( + "open builtin with default options", + f""" + f = open('{TESTFN}') + f.read() + f.close() + """ + ) + + check_readall( + "open in binary mode", + f""" + f = open('{TESTFN}', 'rb') + f.read() + f.close() + """ + ) + + check_readall( + "open in text mode", + f""" + f = open('{TESTFN}', 'rt') + f.read() + f.close() + """, + # GH-122111: read_text uses BufferedIO which requires looking up + # position in file. `read_bytes` disables that buffering and avoids + # these calls which is tested the `pathlib read_bytes` case. + extra_checks=[("seek", 1)] + ) + + check_readall( + "pathlib read_bytes", + "p.read_bytes()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""", + # GH-122111: Buffering is disabled so these calls are avoided. + extra_checks=[("seek", 0)] + ) + + check_readall( + "pathlib read_text", + "p.read_text()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""" + ) + + # Focus on just `read()`. + calls = strace_helper.get_syscalls( + prelude=f"f = open('{TESTFN}')", + code="f.read()", + cleanup="f.close()", + strace_flags=_strace_flags + ) + # One to read all the bytes + # One to read the EOF and get a size 0 return. + self.assertEqual(calls.count("read"), 2) - @unittest.expectedFailure # TODO: RUSTPYTHON - def testMethods(self): - return super().testMethods() - @unittest.expectedFailure # TODO: RUSTPYTHON - def testOpenDirFD(self): - return super().testOpenDirFD() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_subclass_repr(self): - return super().test_subclass_repr() +class CAutoFileTests(AutoFileTests, unittest.TestCase): + FileIO = _io.FileIO + modulename = '_io' -@unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, test setUp errors on Windows") class PyAutoFileTests(AutoFileTests, unittest.TestCase): FileIO = _pyio.FileIO modulename = '_pyio' @@ -412,7 +532,7 @@ def testAbles(self): self.assertEqual(f.isatty(), False) f.close() - if sys.platform != "win32" and not is_emscripten: + if sys.platform != "win32": try: f = self.FileIO("/dev/tty", "a") except OSError: @@ -506,7 +626,6 @@ def testInvalidFd(self): import msvcrt self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) - @unittest.expectedFailure # TODO: RUSTPYTHON def testBooleanFd(self): for fd in False, True: with self.assertWarnsRegex(RuntimeWarning, @@ -634,10 +753,6 @@ def test_open_code(self): actual = f.read() self.assertEqual(expected, actual) - @unittest.expectedFailure # TODO: RUSTPYTHON - def testUnclosedFDOnException(self): - return super().testUnclosedFDOnException() - class PyOtherFileTests(OtherFileTests, unittest.TestCase): FileIO = _pyio.FileIO diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 4ca4103f420..53f454d1891 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -35,6 +35,28 @@ class FloatSubclass(float): class OtherFloatSubclass(float): pass +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class FloatLike: + def __init__(self, value): + self.value = value + + def __float__(self): + return self.value + + class GeneralFloatCases(unittest.TestCase): def test_float(self): @@ -184,10 +206,6 @@ def test_float_with_comma(self): def test_floatconversion(self): # Make sure that calls to __float__() work properly - class Foo1(object): - def __float__(self): - return 42. - class Foo2(float): def __float__(self): return 42. @@ -209,52 +227,35 @@ class FooStr(str): def __float__(self): return float(str(self)) + 1 - self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(FloatLike(42.)), 42.) self.assertEqual(float(Foo2()), 42.) with self.assertWarns(DeprecationWarning): self.assertEqual(float(Foo3(21)), 42.) self.assertRaises(TypeError, float, Foo4(42)) self.assertEqual(float(FooStr('8')), 9.) - class Foo5: - def __float__(self): - return "" - self.assertRaises(TypeError, time.sleep, Foo5()) + self.assertRaises(TypeError, time.sleep, FloatLike("")) # Issue #24731 - class F: - def __float__(self): - return OtherFloatSubclass(42.) + f = FloatLike(OtherFloatSubclass(42.)) with self.assertWarns(DeprecationWarning): - self.assertEqual(float(F()), 42.) + self.assertEqual(float(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(float(F())), float) + self.assertIs(type(float(f)), float) with self.assertWarns(DeprecationWarning): - self.assertEqual(FloatSubclass(F()), 42.) + self.assertEqual(FloatSubclass(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(FloatSubclass(F())), FloatSubclass) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value + self.assertIs(type(FloatSubclass(f)), FloatSubclass) self.assertEqual(float(MyIndex(42)), 42.0) self.assertRaises(OverflowError, float, MyIndex(2**2000)) - - class MyInt: - def __int__(self): - return 42 - - self.assertRaises(TypeError, float, MyInt()) + self.assertRaises(TypeError, float, MyInt(42)) def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): float(x='3.14') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument newarg def test_keywords_in_subclass(self): class subclass(float): pass @@ -282,6 +283,37 @@ def __new__(cls, arg, newarg=None): self.assertEqual(float(u), 2.5) self.assertEqual(u.newarg, 3) + def assertEqualAndType(self, actual, expected_value, expected_type): + self.assertEqual(actual, expected_value) + self.assertIs(type(actual), expected_type) + + def test_from_number(self, cls=float): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14) + eq(cls.from_number(314), 314.0) + eq(cls.from_number(OtherFloatSubclass(3.14)), 3.14) + eq(cls.from_number(FloatLike(3.14)), 3.14) + eq(cls.from_number(MyIndex(314)), 314.0) + + x = cls.from_number(NAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is float: + self.assertIs(cls.from_number(NAN), NAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, 3.14j) + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(FloatSubclass) + def test_is_integer(self): self.assertFalse((1.1).is_integer()) self.assertTrue((1.).is_integer()) @@ -400,9 +432,8 @@ def test_float_mod(self): self.assertEqualAndEqualSign(mod(1e-100, -1.0), -1.0) self.assertEqualAndEqualSign(mod(1.0, -1.0), -0.0) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: must be real number, not complex @support.requires_IEEE_754 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_float_pow(self): # test builtin pow and ** operator for IEEE 754 special cases. # Special cases taken from section F.9.4.4 of the C99 specification @@ -622,6 +653,24 @@ class F(float, H): value = F('nan') self.assertEqual(hash(value), object.__hash__(value)) + def test_issue_gh143006(self): + # When comparing negative non-integer float and int with the + # same number of bits in the integer part, __neg__() in the + # int subclass returning not an int caused an assertion error. + class EvilInt(int): + def __neg__(self): + return "" + + i = -1 << 50 + f = float(i) - 0.5 + i = EvilInt(i) + self.assertFalse(f == i) + self.assertTrue(f != i) + self.assertTrue(f < i) + self.assertTrue(f <= i) + self.assertFalse(f > i) + self.assertFalse(f >= i) + @unittest.skipUnless(hasattr(float, "__getformat__"), "requires __getformat__") class FormatFunctionsTestCase(unittest.TestCase): @@ -676,6 +725,7 @@ def test_serialized_float_rounding(self): class FormatTestCase(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Invalid format specifier def test_format(self): # these should be rewritten to use both format(x, spec) and # x.__format__(spec) @@ -727,6 +777,44 @@ def test_format(self): self.assertEqual(format(INF, 'f'), 'inf') self.assertEqual(format(INF, 'F'), 'INF') + # thousands separators + x = 123_456.123_456 + self.assertEqual(format(x, '_f'), '123_456.123456') + self.assertEqual(format(x, ',f'), '123,456.123456') + self.assertEqual(format(x, '._f'), '123456.123_456') + self.assertEqual(format(x, '.,f'), '123456.123,456') + self.assertEqual(format(x, '_._f'), '123_456.123_456') + self.assertEqual(format(x, ',.,f'), '123,456.123,456') + self.assertEqual(format(x, '.10_f'), '123456.123_456_000_0') + self.assertEqual(format(x, '.10,f'), '123456.123,456,000,0') + self.assertEqual(format(x, '>21._f'), ' 123456.123_456') + self.assertEqual(format(x, '<21._f'), '123456.123_456 ') + self.assertEqual(format(x, '+.11_e'), '+1.234_561_234_56e+05') + self.assertEqual(format(x, '+.11,e'), '+1.234,561,234,56e+05') + self.assertEqual(format(x, '021_._f'), '0_000_123_456.123_456') + self.assertEqual(format(x, '020_._f'), '0_000_123_456.123_456') + self.assertEqual(format(x, '+021_._f'), '+0_000_123_456.123_456') + self.assertEqual(format(x, '21_._f'), ' 123_456.123_456') + self.assertEqual(format(x, '>021_._f'), '000000123_456.123_456') + self.assertEqual(format(x, '<021_._f'), '123_456.123_456000000') + self.assertEqual(format(x, '023_.10_f'), '0_123_456.123_456_000_0') + self.assertEqual(format(x, '022_.10_f'), '0_123_456.123_456_000_0') + self.assertEqual(format(x, '+023_.10_f'), '+0_123_456.123_456_000_0') + self.assertEqual(format(x, '023_.9_f'), '000_123_456.123_456_000') + self.assertEqual(format(x, '021_._e'), '0_000_001.234_561e+05') + self.assertEqual(format(x, '020_._e'), '0_000_001.234_561e+05') + self.assertEqual(format(x, '+021_._e'), '+0_000_001.234_561e+05') + self.assertEqual(format(x, '023_.10_e'), '0_001.234_561_234_6e+05') + self.assertEqual(format(x, '022_.10_e'), '0_001.234_561_234_6e+05') + self.assertEqual(format(x, '023_.9_e'), '000_001.234_561_235e+05') + + self.assertRaises(ValueError, format, x, '._6f') + self.assertRaises(ValueError, format, x, '.,_f') + self.assertRaises(ValueError, format, x, '.6,_f') + self.assertRaises(ValueError, format, x, '.6_,f') + self.assertRaises(ValueError, format, x, '.6_n') + self.assertRaises(ValueError, format, x, '.6,n') + @support.requires_IEEE_754 @unittest.skipUnless(sys.float_repr_style == 'short', "applies only when using short float repr style") @@ -877,10 +965,9 @@ def test_overflow(self): self.assertRaises(OverflowError, round, 1.6e308, -308) self.assertRaises(OverflowError, round, -1.7e308, -308) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 56294995342131.51 != 56294995342131.5 @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short', "applies only when using short float repr style") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_previous_round_bugs(self): # particular cases that have occurred in bug reports self.assertEqual(round(562949953421312.5, 1), @@ -897,10 +984,9 @@ def test_previous_round_bugs(self): self.assertEqual(round(85.0, -1), 80.0) self.assertEqual(round(95.0, -1), 100.0) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0.01 != 0.0 @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short', "applies only when using short float repr style") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_matches_float_format(self): # round should give the same results as float formatting for i in range(500): @@ -962,6 +1048,13 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1.0).__round__(None), + round(1.0), + round(1.0, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) # Beginning with Python 2.6 float has cross platform compatible # ways to create and represent inf and nan @@ -1171,8 +1264,7 @@ def test_whitespace(self): self.identical(got, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: invalid hexadecimal floating-point string def test_from_hex(self): MIN = self.MIN MAX = self.MAX diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py new file mode 100644 index 00000000000..bd6267d45ae --- /dev/null +++ b/Lib/test/test_flufl.py @@ -0,0 +1,69 @@ +import __future__ +import unittest + + +class FLUFLTests(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_barry_as_bdfl(self): + code = "from __future__ import barry_as_FLUFL\n2 {0} 3" + compile(code.format('<>'), '', 'exec', + __future__.CO_FUTURE_BARRY_AS_BDFL) + with self.assertRaises(SyntaxError) as cm: + compile(code.format('!='), '', 'exec', + __future__.CO_FUTURE_BARRY_AS_BDFL) + self.assertRegex(str(cm.exception), + "with Barry as BDFL, use '<>' instead of '!='") + self.assertIn('2 != 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + + self.assertEqual(cm.exception.lineno, 2) + # The old parser reports the end of the token and the new + # parser reports the start of the token + self.assertEqual(cm.exception.offset, 3) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_guido_as_bdfl(self): + code = '2 {0} 3' + compile(code.format('!='), '', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile(code.format('<>'), '', 'exec') + self.assertRegex(str(cm.exception), "invalid syntax") + self.assertIn('2 <> 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + self.assertEqual(cm.exception.lineno, 1) + # The old parser reports the end of the token and the new + # parser reports the start of the token + self.assertEqual(cm.exception.offset, 3) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): + # Check that the future import is handled by the parser + # even if the compiler flags are not passed. + code = "from __future__ import barry_as_FLUFL;2 {0} 3" + compile(code.format('<>'), '', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile(code.format('!='), '', 'exec') + self.assertRegex(str(cm.exception), "with Barry as BDFL, use '<>' instead of '!='") + self.assertIn('2 != 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + self.assertEqual(cm.exception.lineno, 1) + self.assertEqual(cm.exception.offset, len(code) - 4) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_barry_as_bdfl_relative_import(self): + code = "from .__future__ import barry_as_FLUFL;2 {0} 3" + compile(code.format('!='), '', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile(code.format('<>'), '', 'exec') + self.assertRegex(str(cm.exception), "") + self.assertIn('2 <> 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + self.assertEqual(cm.exception.lineno, 1) + self.assertEqual(cm.exception.offset, len(code) - 4) + + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index 10ed496d4e2..5daaf3b3fdd 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -1,11 +1,15 @@ """Test cases for the fnmatch module.""" -import unittest import os import string +import unittest import warnings +from fnmatch import fnmatch, fnmatchcase, translate, filter, filterfalse + + +IGNORECASE = os.path.normcase('P') == os.path.normcase('p') +NORMSEP = os.path.normcase('\\') == os.path.normcase('/') -from fnmatch import fnmatch, fnmatchcase, translate, filter class FnmatchTestCase(unittest.TestCase): @@ -77,23 +81,20 @@ def test_bytes(self): self.check_match(b'foo\nbar', b'foo*') def test_case(self): - ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') check = self.check_match check('abc', 'abc') - check('AbC', 'abc', ignorecase) - check('abc', 'AbC', ignorecase) + check('AbC', 'abc', IGNORECASE) + check('abc', 'AbC', IGNORECASE) check('AbC', 'AbC') def test_sep(self): - normsep = os.path.normcase('\\') == os.path.normcase('/') check = self.check_match check('usr/bin', 'usr/bin') - check('usr\\bin', 'usr/bin', normsep) - check('usr/bin', 'usr\\bin', normsep) + check('usr\\bin', 'usr/bin', NORMSEP) + check('usr/bin', 'usr\\bin', NORMSEP) check('usr\\bin', 'usr\\bin') def test_char_set(self): - ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') check = self.check_match tescases = string.ascii_lowercase + string.digits + string.punctuation for c in tescases: @@ -101,11 +102,11 @@ def test_char_set(self): check(c, '[!az]', c not in 'az') # Case insensitive. for c in tescases: - check(c, '[AZ]', (c in 'az') and ignorecase) - check(c, '[!AZ]', (c not in 'az') or not ignorecase) + check(c, '[AZ]', (c in 'az') and IGNORECASE) + check(c, '[!AZ]', (c not in 'az') or not IGNORECASE) for c in string.ascii_uppercase: - check(c, '[az]', (c in 'AZ') and ignorecase) - check(c, '[!az]', (c not in 'AZ') or not ignorecase) + check(c, '[az]', (c in 'AZ') and IGNORECASE) + check(c, '[!az]', (c not in 'AZ') or not IGNORECASE) # Repeated same character. for c in tescases: check(c, '[aa]', c == 'a') @@ -120,8 +121,6 @@ def test_char_set(self): check('[!]', '[!]') def test_range(self): - ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') - normsep = os.path.normcase('\\') == os.path.normcase('/') check = self.check_match tescases = string.ascii_lowercase + string.digits + string.punctuation for c in tescases: @@ -131,11 +130,11 @@ def test_range(self): check(c, '[!b-dx-z]', c not in 'bcdxyz') # Case insensitive. for c in tescases: - check(c, '[B-D]', (c in 'bcd') and ignorecase) - check(c, '[!B-D]', (c not in 'bcd') or not ignorecase) + check(c, '[B-D]', (c in 'bcd') and IGNORECASE) + check(c, '[!B-D]', (c not in 'bcd') or not IGNORECASE) for c in string.ascii_uppercase: - check(c, '[b-d]', (c in 'BCD') and ignorecase) - check(c, '[!b-d]', (c not in 'BCD') or not ignorecase) + check(c, '[b-d]', (c in 'BCD') and IGNORECASE) + check(c, '[!b-d]', (c not in 'BCD') or not IGNORECASE) # Upper bound == lower bound. for c in tescases: check(c, '[b-b]', c == 'b') @@ -144,7 +143,7 @@ def test_range(self): check(c, '[!-#]', c not in '-#') check(c, '[!--.]', c not in '-.') check(c, '[^-`]', c in '^_`') - if not (normsep and c == '/'): + if not (NORMSEP and c == '/'): check(c, '[[-^]', c in r'[\]^') check(c, r'[\-^]', c in r'\]^') check(c, '[b-]', c in '-b') @@ -160,47 +159,45 @@ def test_range(self): check(c, '[d-bx-z]', c in 'xyz') check(c, '[!d-bx-z]', c not in 'xyz') check(c, '[d-b^-`]', c in '^_`') - if not (normsep and c == '/'): + if not (NORMSEP and c == '/'): check(c, '[d-b[-^]', c in r'[\]^') def test_sep_in_char_set(self): - normsep = os.path.normcase('\\') == os.path.normcase('/') check = self.check_match check('/', r'[/]') check('\\', r'[\]') - check('/', r'[\]', normsep) - check('\\', r'[/]', normsep) + check('/', r'[\]', NORMSEP) + check('\\', r'[/]', NORMSEP) check('[/]', r'[/]', False) check(r'[\\]', r'[/]', False) check('\\', r'[\t]') - check('/', r'[\t]', normsep) + check('/', r'[\t]', NORMSEP) check('t', r'[\t]') check('\t', r'[\t]', False) def test_sep_in_range(self): - normsep = os.path.normcase('\\') == os.path.normcase('/') check = self.check_match - check('a/b', 'a[.-0]b', not normsep) + check('a/b', 'a[.-0]b', not NORMSEP) check('a\\b', 'a[.-0]b', False) - check('a\\b', 'a[Z-^]b', not normsep) + check('a\\b', 'a[Z-^]b', not NORMSEP) check('a/b', 'a[Z-^]b', False) - check('a/b', 'a[/-0]b', not normsep) + check('a/b', 'a[/-0]b', not NORMSEP) check(r'a\b', 'a[/-0]b', False) check('a[/-0]b', 'a[/-0]b', False) check(r'a[\-0]b', 'a[/-0]b', False) check('a/b', 'a[.-/]b') - check(r'a\b', 'a[.-/]b', normsep) + check(r'a\b', 'a[.-/]b', NORMSEP) check('a[.-/]b', 'a[.-/]b', False) check(r'a[.-\]b', 'a[.-/]b', False) check(r'a\b', r'a[\-^]b') - check('a/b', r'a[\-^]b', normsep) + check('a/b', r'a[\-^]b', NORMSEP) check(r'a[\-^]b', r'a[\-^]b', False) check('a[/-^]b', r'a[\-^]b', False) - check(r'a\b', r'a[Z-\]b', not normsep) + check(r'a\b', r'a[Z-\]b', not NORMSEP) check('a/b', r'a[Z-\]b', False) check(r'a[Z-\]b', r'a[Z-\]b', False) check('a[Z-/]b', r'a[Z-\]b', False) @@ -221,24 +218,24 @@ class TranslateTestCase(unittest.TestCase): def test_translate(self): import re - self.assertEqual(translate('*'), r'(?s:.*)\Z') - self.assertEqual(translate('?'), r'(?s:.)\Z') - self.assertEqual(translate('a?b*'), r'(?s:a.b.*)\Z') - self.assertEqual(translate('[abc]'), r'(?s:[abc])\Z') - self.assertEqual(translate('[]]'), r'(?s:[]])\Z') - self.assertEqual(translate('[!x]'), r'(?s:[^x])\Z') - self.assertEqual(translate('[^x]'), r'(?s:[\^x])\Z') - self.assertEqual(translate('[x'), r'(?s:\[x)\Z') + self.assertEqual(translate('*'), r'(?s:.*)\z') + self.assertEqual(translate('?'), r'(?s:.)\z') + self.assertEqual(translate('a?b*'), r'(?s:a.b.*)\z') + self.assertEqual(translate('[abc]'), r'(?s:[abc])\z') + self.assertEqual(translate('[]]'), r'(?s:[]])\z') + self.assertEqual(translate('[!x]'), r'(?s:[^x])\z') + self.assertEqual(translate('[^x]'), r'(?s:[\^x])\z') + self.assertEqual(translate('[x'), r'(?s:\[x)\z') # from the docs - self.assertEqual(translate('*.txt'), r'(?s:.*\.txt)\Z') + self.assertEqual(translate('*.txt'), r'(?s:.*\.txt)\z') # squash consecutive stars - self.assertEqual(translate('*********'), r'(?s:.*)\Z') - self.assertEqual(translate('A*********'), r'(?s:A.*)\Z') - self.assertEqual(translate('*********A'), r'(?s:.*A)\Z') - self.assertEqual(translate('A*********?[?]?'), r'(?s:A.*.[?].)\Z') + self.assertEqual(translate('*********'), r'(?s:.*)\z') + self.assertEqual(translate('A*********'), r'(?s:A.*)\z') + self.assertEqual(translate('*********A'), r'(?s:.*A)\z') + self.assertEqual(translate('A*********?[?]?'), r'(?s:A.*.[?].)\z') # fancy translation to prevent exponential-time match failure t = translate('**a*a****a') - self.assertEqual(t, r'(?s:(?>.*?a)(?>.*?a).*a)\Z') + self.assertEqual(t, r'(?s:(?>.*?a)(?>.*?a).*a)\z') # and try pasting multiple translate results - it's an undocumented # feature that this works r1 = translate('**a**a**a*') @@ -250,6 +247,75 @@ def test_translate(self): self.assertTrue(re.match(fatre, 'cbabcaxc')) self.assertFalse(re.match(fatre, 'dabccbad')) + def test_translate_wildcards(self): + for pattern, expect in [ + ('ab*', r'(?s:ab.*)\z'), + ('ab*cd', r'(?s:ab.*cd)\z'), + ('ab*cd*', r'(?s:ab(?>.*?cd).*)\z'), + ('ab*cd*12', r'(?s:ab(?>.*?cd).*12)\z'), + ('ab*cd*12*', r'(?s:ab(?>.*?cd)(?>.*?12).*)\z'), + ('ab*cd*12*34', r'(?s:ab(?>.*?cd)(?>.*?12).*34)\z'), + ('ab*cd*12*34*', r'(?s:ab(?>.*?cd)(?>.*?12)(?>.*?34).*)\z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + for pattern, expect in [ + ('*ab', r'(?s:.*ab)\z'), + ('*ab*', r'(?s:(?>.*?ab).*)\z'), + ('*ab*cd', r'(?s:(?>.*?ab).*cd)\z'), + ('*ab*cd*', r'(?s:(?>.*?ab)(?>.*?cd).*)\z'), + ('*ab*cd*12', r'(?s:(?>.*?ab)(?>.*?cd).*12)\z'), + ('*ab*cd*12*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*)\z'), + ('*ab*cd*12*34', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*34)\z'), + ('*ab*cd*12*34*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12)(?>.*?34).*)\z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_translate_expressions(self): + for pattern, expect in [ + ('[', r'(?s:\[)\z'), + ('[!', r'(?s:\[!)\z'), + ('[]', r'(?s:\[\])\z'), + ('[abc', r'(?s:\[abc)\z'), + ('[!abc', r'(?s:\[!abc)\z'), + ('[abc]', r'(?s:[abc])\z'), + ('[!abc]', r'(?s:[^abc])\z'), + ('[!abc][!def]', r'(?s:[^abc][^def])\z'), + # with [[ + ('[[', r'(?s:\[\[)\z'), + ('[[a', r'(?s:\[\[a)\z'), + ('[[]', r'(?s:[\[])\z'), + ('[[]a', r'(?s:[\[]a)\z'), + ('[[]]', r'(?s:[\[]\])\z'), + ('[[]a]', r'(?s:[\[]a\])\z'), + ('[[a]', r'(?s:[\[a])\z'), + ('[[a]]', r'(?s:[\[a]\])\z'), + ('[[a]b', r'(?s:[\[a]b)\z'), + # backslashes + ('[\\', r'(?s:\[\\)\z'), + (r'[\]', r'(?s:[\\])\z'), + (r'[\\]', r'(?s:[\\\\])\z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_star_indices_locations(self): + from fnmatch import _translate + + blocks = ['a^b', '***', '?', '?', '[a-z]', '[1-9]', '*', '++', '[[a'] + parts, star_indices = _translate(''.join(blocks), '*', '.') + expect_parts = ['a', r'\^', 'b', '*', + '.', '.', '[a-z]', '[1-9]', '*', + r'\+', r'\+', r'\[', r'\[', 'a'] + self.assertListEqual(parts, expect_parts) + self.assertListEqual(star_indices, [3, 8]) + + class FilterTestCase(unittest.TestCase): def test_filter(self): @@ -263,18 +329,41 @@ def test_mix_bytes_str(self): self.assertRaises(TypeError, filter, [b'test'], '*') def test_case(self): - ignorecase = os.path.normcase('P') == os.path.normcase('p') self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.p*'), - ['Test.py', 'Test.PL'] if ignorecase else ['Test.py']) + ['Test.py', 'Test.PL'] if IGNORECASE else ['Test.py']) self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.P*'), - ['Test.py', 'Test.PL'] if ignorecase else ['Test.PL']) + ['Test.py', 'Test.PL'] if IGNORECASE else ['Test.PL']) def test_sep(self): - normsep = os.path.normcase('\\') == os.path.normcase('/') self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr/*'), - ['usr/bin', 'usr\\lib'] if normsep else ['usr/bin']) + ['usr/bin', 'usr\\lib'] if NORMSEP else ['usr/bin']) self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr\\*'), - ['usr/bin', 'usr\\lib'] if normsep else ['usr\\lib']) + ['usr/bin', 'usr\\lib'] if NORMSEP else ['usr\\lib']) + + +class FilterFalseTestCase(unittest.TestCase): + + def test_filterfalse(self): + actual = filterfalse(['Python', 'Ruby', 'Perl', 'Tcl'], 'P*') + self.assertListEqual(actual, ['Ruby', 'Tcl']) + actual = filterfalse([b'Python', b'Ruby', b'Perl', b'Tcl'], b'P*') + self.assertListEqual(actual, [b'Ruby', b'Tcl']) + + def test_mix_bytes_str(self): + self.assertRaises(TypeError, filterfalse, ['test'], b'*') + self.assertRaises(TypeError, filterfalse, [b'test'], '*') + + def test_case(self): + self.assertEqual(filterfalse(['Test.py', 'Test.rb', 'Test.PL'], '*.p*'), + ['Test.rb'] if IGNORECASE else ['Test.rb', 'Test.PL']) + self.assertEqual(filterfalse(['Test.py', 'Test.rb', 'Test.PL'], '*.P*'), + ['Test.rb'] if IGNORECASE else ['Test.py', 'Test.rb',]) + + def test_sep(self): + self.assertEqual(filterfalse(['usr/bin', 'usr', 'usr\\lib'], 'usr/*'), + ['usr'] if NORMSEP else ['usr', 'usr\\lib']) + self.assertEqual(filterfalse(['usr/bin', 'usr', 'usr\\lib'], 'usr\\*'), + ['usr'] if NORMSEP else ['usr/bin', 'usr']) if __name__ == "__main__": diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py new file mode 100644 index 00000000000..4f4a5ee0507 --- /dev/null +++ b/Lib/test/test_fork1.py @@ -0,0 +1,103 @@ +"""This test checks for correct fork() behavior. +""" + +import _imp as imp +import os +import signal +import sys +import threading +import time +import unittest + +from test.fork_wait import ForkWait +from test import support + + +# Skip test if fork does not exist. +if not support.has_fork_support: + raise unittest.SkipTest("test module requires working os.fork") + + +class ForkTest(ForkWait): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: process 44587 exited with code 1, but exit code 42 is expected + def test_threaded_import_lock_fork(self): + """Check fork() in main thread works while a subthread is doing an import""" + import_started = threading.Event() + fake_module_name = "fake test module" + partial_module = "partial" + complete_module = "complete" + def importer(): + imp.acquire_lock() + sys.modules[fake_module_name] = partial_module + import_started.set() + time.sleep(0.01) # Give the other thread time to try and acquire. + sys.modules[fake_module_name] = complete_module + imp.release_lock() + t = threading.Thread(target=importer) + t.start() + import_started.wait() + exitcode = 42 + pid = os.fork() + try: + # PyOS_BeforeFork should have waited for the import to complete + # before forking, so the child can recreate the import lock + # correctly, but also won't see a partially initialised module + if not pid: + m = __import__(fake_module_name) + if m == complete_module: + os._exit(exitcode) + else: + if support.verbose > 1: + print("Child encountered partial module") + os._exit(1) + else: + t.join() + # Exitcode 1 means the child got a partial module (bad.) No + # exitcode (but a hang, which manifests as 'got pid 0') + # means the child deadlocked (also bad.) + self.wait_impl(pid, exitcode=exitcode) + finally: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + + + def test_nested_import_lock_fork(self): + """Check fork() in main thread works while the main thread is doing an import""" + exitcode = 42 + # Issue 9573: this used to trigger RuntimeError in the child process + def fork_with_import_lock(level): + release = 0 + in_child = False + try: + try: + for i in range(level): + imp.acquire_lock() + release += 1 + pid = os.fork() + in_child = not pid + finally: + for i in range(release): + imp.release_lock() + except RuntimeError: + if in_child: + if support.verbose > 1: + print("RuntimeError in child") + os._exit(1) + raise + if in_child: + os._exit(exitcode) + self.wait_impl(pid, exitcode=exitcode) + + # Check this works with various levels of nested + # import in the main thread + for level in range(5): + fork_with_import_lock(level) + + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 5c74e36a182..49fc9c2ba23 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1,8 +1,7 @@ """Tests for Lib/fractions.py.""" -import cmath from decimal import Decimal -# from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, adjust_int_max_str_digits import math import numbers import operator @@ -19,7 +18,7 @@ #locate file with float format test values test_dir = os.path.dirname(__file__) or os.curdir -format_testfile = os.path.join(test_dir, 'formatfloat_testcases.txt') +format_testfile = os.path.join(test_dir, 'mathdata', 'formatfloat_testcases.txt') class DummyFloat(object): """Dummy float class for testing comparisons with Fractions""" @@ -97,7 +96,7 @@ def typed_approx_eq(a, b): class Symbolic: """Simple non-numeric class for testing mixed arithmetic. - It is not Integral, Rational, Real or Complex, and cannot be conveted + It is not Integral, Rational, Real or Complex, and cannot be converted to int, float or complex. but it supports some arithmetic operations. """ def __init__(self, value): @@ -284,6 +283,13 @@ def __repr__(self): class RectComplex(Rect, complex): pass +class Ratio: + def __init__(self, ratio): + self._ratio = ratio + def as_integer_ratio(self): + return self._ratio + + class FractionTest(unittest.TestCase): def assertTypedEquals(self, expected, actual): @@ -331,7 +337,7 @@ def testInit(self): self.assertRaises(TypeError, F, 3, 1j) self.assertRaises(TypeError, F, 1, 2, 3) - # @requires_IEEE_754 + @requires_IEEE_754 def testInitFromFloat(self): self.assertEqual((5, 2), _components(F(2.5))) self.assertEqual((0, 1), _components(F(-0.0))) @@ -355,14 +361,48 @@ def testInitFromDecimal(self): self.assertRaises(OverflowError, F, Decimal('inf')) self.assertRaises(OverflowError, F, Decimal('-inf')) + def testInitFromIntegerRatio(self): + self.assertEqual((7, 3), _components(F(Ratio((7, 3))))) + errmsg = (r"argument should be a string or a Rational instance or " + r"have the as_integer_ratio\(\) method") + # the type also has an "as_integer_ratio" attribute. + self.assertRaisesRegex(TypeError, errmsg, F, Ratio) + # bad ratio + self.assertRaises(TypeError, F, Ratio(7)) + self.assertRaises(ValueError, F, Ratio((7,))) + self.assertRaises(ValueError, F, Ratio((7, 3, 1))) + # only single-argument form + self.assertRaises(TypeError, F, Ratio((3, 7)), 11) + self.assertRaises(TypeError, F, 2, Ratio((-10, 9))) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + self.assertEqual((9, 5), _components(F(a))) + + # as_integer_ratio defined in a metaclass + class M(type): + def as_integer_ratio(self): + return (11, 9) + class B(metaclass=M): + pass + self.assertRaisesRegex(TypeError, errmsg, F, B) + self.assertRaisesRegex(TypeError, errmsg, F, B()) + self.assertRaises(TypeError, F.from_number, B) + self.assertRaises(TypeError, F.from_number, B()) + def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((5, 1), _components(F("005"))) self.assertEqual((3, 2), _components(F("3/2"))) self.assertEqual((3, 2), _components(F("3 / 2"))) self.assertEqual((3, 2), _components(F(" \n +3/2"))) self.assertEqual((-3, 2), _components(F("-3/2 "))) - self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((13, 2), _components(F(" 0013/002 \n "))) self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((16, 5), _components(F("003.2"))) self.assertEqual((-16, 5), _components(F(" -3.2 "))) self.assertEqual((-3, 1), _components(F(" -3. "))) self.assertEqual((3, 5), _components(F(" .6 "))) @@ -381,116 +421,102 @@ def testFromString(self): self.assertRaisesMessage( ZeroDivisionError, "Fraction(3, 0)", F, "3/0") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '3/'", - F, "3/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/2'", - F, "/2") - self.assertRaisesMessage( - # Denominators don't need a sign. - ValueError, "Invalid literal for Fraction: '3/+2'", - F, "3/+2") - self.assertRaisesMessage( - # Imitate float's parsing. - ValueError, "Invalid literal for Fraction: '+ 3/2'", - F, "+ 3/2") - self.assertRaisesMessage( - # Avoid treating '.' as a regex special character. - ValueError, "Invalid literal for Fraction: '3a2'", - F, "3a2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3/7.2'", - F, "3/7.2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3.2/7'", - F, "3.2/7") - self.assertRaisesMessage( - # Allow 3. and .3, but not . - ValueError, "Invalid literal for Fraction: '.'", - F, ".") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_'", - F, "_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1'", - F, "_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2'", - F, "1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/_'", - F, "/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1_/'", - F, "1_/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1/'", - F, "_1/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2/'", - F, "1__2/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_'", - F, "1/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_1'", - F, "1/_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/1__2'", - F, "1/1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1._111'", - F, "1._111") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1__1'", - F, "1.1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+_1'", - F, "1.1e+_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+1__1'", - F, "1.1e+1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.dd'", - F, "123.dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.5_dd'", - F, "123.5_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: 'dd.5'", - F, "dd.5") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '7_dd'", - F, "7_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/dd'", - F, "1/dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/123_dd'", - F, "1/123_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789edd'", - F, "789edd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789e2_dd'", - F, "789e2_dd") + + def check_invalid(s): + msg = "Invalid literal for Fraction: " + repr(s) + self.assertRaisesMessage(ValueError, msg, F, s) + + check_invalid("3/") + check_invalid("/2") + # Denominators don't need a sign. + check_invalid("3/+2") + check_invalid("3/-2") + # Imitate float's parsing. + check_invalid("+ 3/2") + check_invalid("- 3/2") + # Avoid treating '.' as a regex special character. + check_invalid("3a2") + # Don't accept combinations of decimals and rationals. + check_invalid("3/7.2") + check_invalid("3.2/7") + # No space around dot. + check_invalid("3 .2") + check_invalid("3. 2") + # No space around e. + check_invalid("3.2 e1") + check_invalid("3.2e 1") + # Fractional part don't need a sign. + check_invalid("3.+2") + check_invalid("3.-2") + # Only accept base 10. + check_invalid("0x10") + check_invalid("0x10/1") + check_invalid("1/0x10") + check_invalid("0x10.") + check_invalid("0x10.1") + check_invalid("1.0x10") + check_invalid("1.0e0x10") + # Only accept decimal digits. + check_invalid("³") + check_invalid("³/2") + check_invalid("3/²") + check_invalid("³.2") + check_invalid("3.²") + check_invalid("3.2e²") + check_invalid("¼") + # Allow 3. and .3, but not . + check_invalid(".") + check_invalid("_") + check_invalid("_1") + check_invalid("1__2") + check_invalid("/_") + check_invalid("1_/") + check_invalid("_1/") + check_invalid("1__2/") + check_invalid("1/_") + check_invalid("1/_1") + check_invalid("1/1__2") + check_invalid("1._111") + check_invalid("1.1__1") + check_invalid("1.1e+_1") + check_invalid("1.1e+1__1") + check_invalid("123.dd") + check_invalid("123.5_dd") + check_invalid("dd.5") + check_invalid("7_dd") + check_invalid("1/dd") + check_invalid("1/123_dd") + check_invalid("789edd") + check_invalid("789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '" + val + "'", - F, val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/" + val + "'", - F, "1/" + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1." + val + "'", - F, "1." + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1+e" + val + "'", - F, "1.1+e" + val) + check_invalid(val) + check_invalid("1/" + val) + check_invalid("1." + val) + check_invalid("." + val) + check_invalid("1.1+e" + val) + check_invalid("1.1e" + val) + + def test_limit_int(self): + maxdigits = 5000 + with adjust_int_max_str_digits(maxdigits): + msg = 'Exceeds the limit' + val = '1' * maxdigits + num = (10**maxdigits - 1)//9 + self.assertEqual((num, 1), _components(F(val))) + self.assertRaisesRegex(ValueError, msg, F, val + '1') + self.assertEqual((num, 2), _components(F(val + '/2'))) + self.assertRaisesRegex(ValueError, msg, F, val + '1/2') + self.assertEqual((1, num), _components(F('1/' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1/1' + val) + self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), + _components(F('1.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1.1' + val) + self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1e1' + val) + self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) + self.assertRaisesRegex(ValueError, msg, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) @@ -560,6 +586,37 @@ def testFromDecimal(self): ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) + def testFromNumber(self, cls=F): + def check(arg, numerator, denominator): + f = cls.from_number(arg) + self.assertIs(type(f), cls) + self.assertEqual(f.numerator, numerator) + self.assertEqual(f.denominator, denominator) + + check(10, 10, 1) + check(2.5, 5, 2) + check(Decimal('2.5'), 5, 2) + check(F(22, 7), 22, 7) + check(DummyFraction(22, 7), 22, 7) + check(Rat(22, 7), 22, 7) + check(Ratio((22, 7)), 22, 7) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '5/2') + self.assertRaises(TypeError, cls.from_number, []) + self.assertRaises(OverflowError, cls.from_number, float('inf')) + self.assertRaises(OverflowError, cls.from_number, Decimal('inf')) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + check(a, 9, 5) + + def testFromNumber_subclass(self): + self.testFromNumber(DummyFraction) + + def test_is_integer(self): self.assertTrue(F(1, 1).is_integer()) self.assertTrue(F(-1, 1).is_integer()) @@ -806,7 +863,7 @@ def testMixedMultiplication(self): self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) - self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0+0j, 4.5+0j)) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0, 4.5)) self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) @@ -922,21 +979,21 @@ def testMixedPower(self): self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) - self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(2.25, 0.0)) - self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) - self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) def testMixingWithDecimal(self): @@ -1165,12 +1222,50 @@ def denominator(self): self.assertEqual(type(f.denominator), myint) def test_format_no_presentation_type(self): - # Triples (fraction, specification, expected_result) + # Triples (fraction, specification, expected_result). testcases = [ - (F(1, 3), '', '1/3'), - (F(-1, 3), '', '-1/3'), - (F(3), '', '3'), - (F(-3), '', '-3'), + # Explicit sign handling + (F(2, 3), '+', '+2/3'), + (F(-2, 3), '+', '-2/3'), + (F(3), '+', '+3'), + (F(-3), '+', '-3'), + (F(2, 3), ' ', ' 2/3'), + (F(-2, 3), ' ', '-2/3'), + (F(3), ' ', ' 3'), + (F(-3), ' ', '-3'), + (F(2, 3), '-', '2/3'), + (F(-2, 3), '-', '-2/3'), + (F(3), '-', '3'), + (F(-3), '-', '-3'), + # Padding + (F(0), '5', ' 0'), + (F(2, 3), '5', ' 2/3'), + (F(-2, 3), '5', ' -2/3'), + (F(2, 3), '0', '2/3'), + (F(2, 3), '1', '2/3'), + (F(2, 3), '2', '2/3'), + # Alignment + (F(2, 3), '<5', '2/3 '), + (F(2, 3), '>5', ' 2/3'), + (F(2, 3), '^5', ' 2/3 '), + (F(2, 3), '=5', ' 2/3'), + (F(-2, 3), '<5', '-2/3 '), + (F(-2, 3), '>5', ' -2/3'), + (F(-2, 3), '^5', '-2/3 '), + (F(-2, 3), '=5', '- 2/3'), + # Fill + (F(2, 3), 'X>5', 'XX2/3'), + (F(-2, 3), '.<5', '-2/3.'), + (F(-2, 3), '\n^6', '\n-2/3\n'), + # Thousands separators + (F(1234, 5679), ',', '1,234/5,679'), + (F(-1234, 5679), '_', '-1_234/5_679'), + (F(1234567), '_', '1_234_567'), + (F(-1234567), ',', '-1,234,567'), + # Alternate form forces a slash in the output + (F(123), '#', '123/1'), + (F(-123), '#', '-123/1'), + (F(0), '#', '0/1'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1227,6 +1322,8 @@ def test_format_e_presentation_type(self): # Thousands separators (F('1234567.123456'), ',.5e', '1.23457e+06'), (F('123.123456'), '012_.2e', '0_001.23e+02'), + # Thousands separators for fractional part (or for integral too) + (F('1234567.123456'), '.5_e', '1.234_57e+06'), # z flag is legal, but never makes a difference to the output (F(-1, 7**100), 'z.6e', '-3.091690e-85'), ] @@ -1352,6 +1449,12 @@ def test_format_f_presentation_type(self): (F('1234567'), ',.2f', '1,234,567.00'), (F('12345678'), ',.2f', '12,345,678.00'), (F('12345678'), ',f', '12,345,678.000000'), + # Thousands separators for fractional part (or for integral too) + (F('123456.789123123'), '._f', '123456.789_123'), + (F('123456.789123123'), '.7_f', '123456.789_123_1'), + (F('123456.789123123'), '.9_f', '123456.789_123_123'), + (F('123456.789123123'), '.,f', '123456.789,123'), + (F('123456.789123123'), '_.,f', '123_456.789,123'), # Underscore as thousands separator (F(2, 3), '_.2f', '0.67'), (F(2, 3), '_.7f', '0.6666667'), @@ -1385,11 +1488,8 @@ def test_format_f_presentation_type(self): (F('-1234.5678'), '08,.0f', '-001,235'), (F('-1234.5678'), '09,.0f', '-0,001,235'), # Corner-case - zero-padding specified through fill and align - # instead of the zero-pad character - in this case, treat '0' as a - # regular fill character and don't attempt to insert commas into - # the filled portion. This differs from the int and float - # behaviour. - (F('1234.5678'), '0=12,.2f', '00001,234.57'), + # instead of the zero-pad character. + (F('1234.5678'), '0=12,.2f', '0,001,234.57'), # Corner case where it's not clear whether the '0' indicates zero # padding or gives the minimum width, but there's still an obvious # answer to give. We want this to work in case the minimum width @@ -1423,6 +1523,8 @@ def test_format_f_presentation_type(self): (F(51, 1000), '.1f', '0.1'), (F(149, 1000), '.1f', '0.1'), (F(151, 1000), '.1f', '0.2'), + (F(22, 7), '.02f', '3.14'), # issue gh-130662 + (F(22, 7), '005.02f', '03.14'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1521,26 +1623,29 @@ def test_invalid_formats(self): '=010%', '>00.2f', '>00f', - # Too many zeros - minimum width should not have leading zeros - '006f', - # Leading zeros in precision - '.010f', - '.02f', - '.000f', # Missing precision '.e', '.f', '.g', '.%', + # Thousands separators before precision + '._6e', + '._6f', + '._6g', + '._6%', # Z instead of z for negative zero suppression 'Z.2f' + # z flag not supported for general formatting + 'z', + # zero padding not supported for general formatting + '05', ] for spec in invalid_specs: with self.subTest(spec=spec): with self.assertRaises(ValueError): format(fraction, spec) - # @requires_IEEE_754 + @requires_IEEE_754 def test_float_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: @@ -1564,6 +1669,47 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: '%' not supported between instances of 'Fraction' and 'complex' + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_three_argument_pow(self): + message = "unsupported operand type(s) for ** or pow(): '%s', '%s', '%s'" + self.assertRaisesMessage(TypeError, + message % ("Fraction", "int", "int"), + pow, F(3), 4, 5) + self.assertRaisesMessage(TypeError, + message % ("int", "Fraction", "int"), + pow, 3, F(4), 5) + self.assertRaisesMessage(TypeError, + message % ("int", "int", "Fraction"), + pow, 3, 4, F(5)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py new file mode 100644 index 00000000000..0b4a12bcf40 --- /dev/null +++ b/Lib/test/test_frozen.py @@ -0,0 +1,56 @@ +"""Basic test of the frozen module (source is in Python/frozen.c).""" + +# The Python/frozen.c source code contains a marshalled Python module +# and therefore depends on the marshal format as well as the bytecode +# format. If those formats have been changed then frozen.c needs to be +# updated. +# +# The test_importlib also tests this module but because those tests +# are much more complicated, it might be unclear why they are failing. +# Invalid marshalled data in frozen.c could case the interpreter to +# crash when __hello__ is imported. + +import importlib.machinery +import sys +import unittest +from test.support import captured_stdout, import_helper + + +class TestFrozen(unittest.TestCase): + def test_frozen(self): + name = '__hello__' + if name in sys.modules: + del sys.modules[name] + with import_helper.frozen_modules(): + import __hello__ + with captured_stdout() as out: + __hello__.main() + self.assertEqual(out.getvalue(), 'Hello world!\n') + + def test_frozen_submodule_in_unfrozen_package(self): + with import_helper.CleanImport('__phello__', '__phello__.spam'): + with import_helper.frozen_modules(enabled=False): + import __phello__ + with import_helper.frozen_modules(enabled=True): + import __phello__.spam as spam + self.assertIs(spam, __phello__.spam) + self.assertIsNot(__phello__.__spec__.loader, + importlib.machinery.FrozenImporter) + self.assertIs(spam.__spec__.loader, + importlib.machinery.FrozenImporter) + + def test_unfrozen_submodule_in_frozen_package(self): + with import_helper.CleanImport('__phello__', '__phello__.spam'): + with import_helper.frozen_modules(enabled=True): + import __phello__ + with import_helper.frozen_modules(enabled=False): + import __phello__.spam as spam + self.assertIs(spam, __phello__.spam) + self.assertIs(__phello__.__spec__.loader, + importlib.machinery.FrozenImporter) + self.assertIsNot(spam.__spec__.loader, + importlib.machinery.FrozenImporter) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 930e409fb2e..f4fca1caec7 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -383,7 +383,7 @@ def test_ast_line_numbers_multiline_fstring(self): self.assertEqual(t.body[0].value.values[1].value.col_offset, 11) self.assertEqual(t.body[0].value.values[1].value.end_col_offset, 16) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 4 != 5 def test_ast_line_numbers_with_parentheses(self): expr = """ x = ( @@ -587,7 +587,6 @@ def test_ast_compile_time_concat(self): exec(c) self.assertEqual(x[0], 'foo3') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_time_concat_errors(self): self.assertAllRaise(SyntaxError, 'cannot mix bytes and nonbytes literals', @@ -600,7 +599,6 @@ def test_literal(self): self.assertEqual(f'a', 'a') self.assertEqual(f' ', ' ') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unterminated_string(self): self.assertAllRaise(SyntaxError, 'unterminated string', [r"""f'{"x'""", @@ -609,7 +607,7 @@ def test_unterminated_string(self): r"""f'{("x}'""", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " @@ -632,14 +630,24 @@ def test_mismatched_parens(self): r"does not match opening parenthesis '\('", ["f'{a(4}'", ]) - self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + self.assertRaises(SyntaxError, eval, "f'{" + "("*20 + "}'") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: No exception raised @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_fstring_nested_too_deeply(self): - self.assertAllRaise(SyntaxError, - "f-string: expressions nested too deeply", - ['f"{1+2:{1+2:{1+1:{1}}}}"']) + def raises_syntax_or_memory_error(txt): + try: + eval(txt) + except SyntaxError: + pass + except MemoryError: + pass + except Exception as ex: + self.fail(f"Should raise SyntaxError or MemoryError, not {type(ex)}") + else: + self.fail("No exception raised") + + raises_syntax_or_memory_error('f"{1+2:{1+2:{1+1:{1}}}}"') def create_nested_fstring(n): if n == 0: @@ -647,9 +655,10 @@ def create_nested_fstring(n): prev = create_nested_fstring(n-1) return f'f"{{{prev}}}"' - self.assertAllRaise(SyntaxError, - "too many nested f-strings", - [create_nested_fstring(160)]) + raises_syntax_or_memory_error(create_nested_fstring(160)) + raises_syntax_or_memory_error("f'{" + "("*100 + "}'") + raises_syntax_or_memory_error("f'{" + "("*1000 + "}'") + raises_syntax_or_memory_error("f'{" + "("*10_000 + "}'") def test_syntax_error_in_nested_fstring(self): # See gh-104016 for more information on this crash @@ -692,7 +701,7 @@ def test_double_braces(self): ["f'{ {{}} }'", # dict in a set ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_time_concat(self): x = 'def' self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') @@ -730,7 +739,7 @@ def test_compile_time_concat(self): ['''f'{3' f"}"''', # can't concat to get a valid f-string ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_comments(self): # These aren't comments, since they're in strings. d = {'#': 'hash'} @@ -807,7 +816,7 @@ def build_fstr(n, extra=''): s = "f'{1}' 'x' 'y'" * 1024 self.assertEqual(eval(s), '1xy' * 1024) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_specifier_expressions(self): width = 10 precision = 4 @@ -841,7 +850,6 @@ def test_format_specifier_expressions(self): """f'{"s"!{"r"}}'""", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_format_specifier(self): class CustomFormat: def __format__(self, format_spec): @@ -863,7 +871,7 @@ def __format__(self, spec): x = X() self.assertEqual(f'{x} {x}', '1 2') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_expression(self): self.assertAllRaise(SyntaxError, "f-string: valid expression required before '}'", @@ -926,7 +934,7 @@ def test_missing_expression(self): "\xa0", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parens_in_expressions(self): self.assertEqual(f'{3,}', '(3,)') @@ -939,13 +947,12 @@ def test_parens_in_expressions(self): ["f'{3)+(4}'", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newlines_before_syntax_error(self): self.assertAllRaise(SyntaxError, "f-string: expecting a valid expression after '{'", ["f'{.}'", "\nf'{.}'", "\n\nf'{.}'"]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_backslashes_in_string_part(self): self.assertEqual(f'\t', '\t') self.assertEqual(r'\t', '\\t') @@ -1004,7 +1011,7 @@ def test_backslashes_in_string_part(self): self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') self.assertEqual(f'\\\N{AMPERSAND}', '\\&') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. @@ -1024,7 +1031,7 @@ def test_misformed_unicode_character_name(self): r"'\N{GREEK CAPITAL LETTER DELTA'", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_backslashes_in_expression_part(self): self.assertEqual(f"{( 1 + @@ -1040,7 +1047,6 @@ def test_backslashes_in_expression_part(self): ["f'{\n}'", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_backslashes_inside_fstring_context(self): # All of these variations are invalid python syntax, # so they are also invalid in f-strings as well. @@ -1075,7 +1081,7 @@ def test_newlines_in_expressions(self): self.assertEqual(rf'''{3+ 4}''', '7') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "f-string: expecting a valid expression after '{'" does not match "invalid syntax (, line 1)" def test_lambda(self): x = 5 self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") @@ -1118,7 +1124,6 @@ def test_roundtrip_raw_quotes(self): self.assertEqual(fr'\'\"\'', '\\\'\\"\\\'') self.assertEqual(fr'\"\'\"\'', '\\"\\\'\\"\\\'') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fstring_backslash_before_double_bracket(self): deprecated_cases = [ (r"f'\{{\}}'", '\\{\\}'), @@ -1138,7 +1143,6 @@ def test_fstring_backslash_before_double_bracket(self): self.assertEqual(fr'\}}{1+1}', '\\}2') self.assertEqual(fr'{1+1}\}}', '2\\}') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fstring_backslash_before_double_bracket_warns_once(self): with self.assertWarns(SyntaxWarning) as w: eval(r"f'\{{'") @@ -1288,6 +1292,7 @@ def test_nested_fstrings(self): self.assertEqual(f'{f"{0}"*3}', '000') self.assertEqual(f'{f"{y}"*3}', '555') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_string_prefixes(self): single_quote_cases = ["fu''", "uf''", @@ -1312,7 +1317,7 @@ def test_invalid_string_prefixes(self): "Bf''", "BF''",] double_quote_cases = [case.replace("'", '"') for case in single_quote_cases] - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, 'prefixes are incompatible', single_quote_cases + double_quote_cases) def test_leading_trailing_spaces(self): @@ -1342,7 +1347,7 @@ def test_equal_equal(self): self.assertEqual(f'{0==1}', 'False') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') self.assertEqual(f'{1.25!s:10.10}', '1.25 ') @@ -1367,7 +1372,6 @@ def test_conversions(self): self.assertAllRaise(SyntaxError, "f-string: expecting '}'", ["f'{3!'", "f'{3!s'", - "f'{3!g'", ]) self.assertAllRaise(SyntaxError, 'f-string: missing conversion character', @@ -1408,14 +1412,13 @@ def test_assignment(self): "f'{x}' = x", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_del(self): self.assertAllRaise(SyntaxError, 'invalid syntax', ["del f''", "del '' f''", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mismatched_braces(self): self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", ["f'{{}'", @@ -1514,7 +1517,6 @@ def test_str_format_differences(self): self.assertEqual('{d[a]}'.format(d=d), 'string') self.assertEqual('{d[0]}'.format(d=d), 'integer') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): # see issue 26287 self.assertAllRaise(TypeError, 'unsupported', @@ -1557,7 +1559,6 @@ def test_backslash_char(self): self.assertEqual(eval('f"\\\n"'), '') self.assertEqual(eval('f"\\\r"'), '') - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '1+2 = # my comment\n 3' != '1+2 = \n 3' def test_debug_conversion(self): x = 'A string' self.assertEqual(f'{x=}', 'x=' + repr(x)) @@ -1705,7 +1706,7 @@ def test_walrus(self): self.assertEqual(f'{(x:=10)}', '10') self.assertEqual(x, 10) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "f-string: expecting '=', or '!', or ':', or '}'" does not match "invalid syntax (?, line 1)" def test_invalid_syntax_error_message(self): with self.assertRaisesRegex(SyntaxError, "f-string: expecting '=', or '!', or ':', or '}'"): @@ -1731,7 +1732,7 @@ def test_with_an_underscore_and_a_comma_in_format_specifier(self): with self.assertRaisesRegex(ValueError, error_msg): f'{1:_,}' - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "f-string: expecting a valid expression after '{'" does not match "invalid syntax (?, line 1)" def test_syntax_error_for_starred_expressions(self): with self.assertRaisesRegex(SyntaxError, "can't use starred expression here"): compile("f'{*a}'", "?", "exec") @@ -1740,7 +1741,7 @@ def test_syntax_error_for_starred_expressions(self): "f-string: expecting a valid expression after '{'"): compile("f'{**a}'", "?", "exec") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; - def test_not_closing_quotes(self): self.assertAllRaise(SyntaxError, "unterminated f-string literal", ['f"', "f'"]) self.assertAllRaise(SyntaxError, "unterminated triple-quoted f-string literal", @@ -1760,7 +1761,7 @@ def test_not_closing_quotes(self): except SyntaxError as e: self.assertEqual(e.text, 'z = f"""') self.assertEqual(e.lineno, 3) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntax_error_after_debug(self): self.assertAllRaise(SyntaxError, "f-string: expecting a valid expression after '{'", [ @@ -1788,7 +1789,6 @@ def test_debug_in_file(self): self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), "3\n=3") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_syntax_warning_infinite_recursion_in_file(self): with temp_cwd(): script = 'script.py' @@ -1878,6 +1878,13 @@ def __format__(self, format): # Test multiple format specs in same raw f-string self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + def test_gh139516(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'wb') as f: + f.write('''def f(a): pass\nf"{f(a=lambda: 'à'\n)}"'''.encode()) + assert_python_ok(script) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index b3e4e776a47..684f5d438b3 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -32,7 +33,7 @@ DEFAULT_ENCODING = 'utf-8' # the dummy data returned by server over the data channel when # RETR, LIST, NLST, MLSD commands are issued -RETR_DATA = 'abcde12345\r\n' * 1000 + 'non-ascii char \xAE\r\n' +RETR_DATA = 'abcde\xB9\xB2\xB3\xA4\xA6\r\n' * 1000 LIST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' NLST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" @@ -67,11 +68,11 @@ class DummyDTPHandler(asynchat.async_chat): def __init__(self, conn, baseclass): asynchat.async_chat.__init__(self, conn) self.baseclass = baseclass - self.baseclass.last_received_data = '' + self.baseclass.last_received_data = bytearray() self.encoding = baseclass.encoding def handle_read(self): - new_data = self.recv(1024).decode(self.encoding, 'replace') + new_data = self.recv(1024) self.baseclass.last_received_data += new_data def handle_close(self): @@ -80,7 +81,7 @@ def handle_close(self): # (behaviour witnessed with test_data_connection) if not self.dtp_conn_closed: self.baseclass.push('226 transfer complete') - self.close() + self.shutdown() self.dtp_conn_closed = True def push(self, what): @@ -94,6 +95,9 @@ def push(self, what): def handle_error(self): default_error_handler() + def shutdown(self): + self.close() + class DummyFTPHandler(asynchat.async_chat): @@ -107,7 +111,7 @@ def __init__(self, conn, encoding=DEFAULT_ENCODING): self.in_buffer = [] self.dtp = None self.last_received_cmd = None - self.last_received_data = '' + self.last_received_data = bytearray() self.next_response = '' self.next_data = None self.rest = None @@ -226,7 +230,7 @@ def cmd_type(self, arg): def cmd_quit(self, arg): self.push('221 quit ok') - self.close() + self.shutdown() def cmd_abor(self, arg): self.push('226 abor ok') @@ -313,7 +317,7 @@ def handle_accepted(self, conn, addr): self.handler_instance = self.handler(conn, encoding=self.encoding) def handle_connect(self): - self.close() + self.shutdown() handle_read = handle_connect def writable(self): @@ -325,8 +329,8 @@ def handle_error(self): if ssl is not None: - CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") - CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + CERTFILE = os.path.join(os.path.dirname(__file__), "certdata", "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "certdata", "pycacert.pem") class SSLConnection(asyncore.dispatcher): """An asyncore.dispatcher subclass supporting TLS/SSL.""" @@ -425,12 +429,12 @@ def recv(self, buffer_size): def handle_error(self): default_error_handler() - def close(self): + def shutdown(self): if (isinstance(self.socket, ssl.SSLSocket) and self.socket._sslobj is not None): self._do_ssl_shutdown() else: - super(SSLConnection, self).close() + self.close() class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): @@ -542,8 +546,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') @@ -590,30 +594,28 @@ def test_abort(self): self.client.abort() def test_retrbinary(self): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.check_data(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.check_data(b''.join(received), + RETR_DATA.encode(self.client.encoding)) def test_retrbinary_rest(self): - def callback(data): - received.append(data.decode(self.client.encoding)) for rest in (0, 10, 20): received = [] - self.client.retrbinary('retr', callback, rest=rest) - self.check_data(''.join(received), RETR_DATA[rest:]) + self.client.retrbinary('retr', received.append, rest=rest) + self.check_data(b''.join(received), + RETR_DATA[rest:].encode(self.client.encoding)) def test_retrlines(self): received = [] self.client.retrlines('retr', received.append) self.check_data(''.join(received), RETR_DATA.replace('\r\n', '')) - @unittest.skip("TODO: RUSTPYTHON; weird limiting to 8192, something w/ buffering?") def test_storbinary(self): f = io.BytesIO(RETR_DATA.encode(self.client.encoding)) self.client.storbinary('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) @@ -632,7 +634,8 @@ def test_storlines(self): data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding) f = io.BytesIO(data) self.client.storlines('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) @@ -650,7 +653,7 @@ def test_nlst(self): def test_dir(self): l = [] - self.client.dir(lambda x: l.append(x)) + self.client.dir(l.append) self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) def test_mlsd(self): @@ -890,20 +893,19 @@ def test_makepasv(self): def test_transfer(self): def retr(): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.assertEqual(len(''.join(received)), len(RETR_DATA)) - self.assertEqual(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.assertEqual(b''.join(received), + RETR_DATA.encode(self.client.encoding)) self.client.set_pasv(True) retr() self.client.set_pasv(False) retr() +@unittest.skip("TODO: RUSTPYTHON; SSL + asyncore has problem") @skipUnless(ssl, "SSL not available") -@unittest.skip("TODO: RUSTPYTHON; figure out why do_handshake() is throwing 'ssl session has been shut down'. SslSession object?") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -920,7 +922,8 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") -@unittest.skip("TODO: RUSTPYTHON; fix ssl") +@unittest.skip("TODO: RUSTPYTHON; SSL + asyncore has problem") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" @@ -968,6 +971,7 @@ def test_data_connection(self): LIST_DATA.encode(self.client.encoding)) self.assertEqual(self.client.voidresp(), "226 transfer complete") + @unittest.skip('TODO: RUSTPYTHON flaky TimeoutError') def test_login(self): # login() is supposed to implicitly secure the control connection self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) @@ -980,6 +984,7 @@ def test_auth_issued_twice(self): self.client.auth() self.assertRaises(ValueError, self.client.auth) + @unittest.skip('TODO: RUSTPYTHON flaky TimeoutError') def test_context(self): self.client.quit() ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index e06e9f7f4a9..f245213418e 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -1,6 +1,9 @@ import textwrap import types +import typing import unittest +import warnings +from test import support def global_function(): @@ -47,8 +50,7 @@ class FunctionPropertiesTest(FuncAttrsTest): def test_module(self): self.assertEqual(self.b.__module__, __name__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dir_includes_correct_attrs(self): self.b.known_attr = 7 self.assertIn('known_attr', dir(self.b), @@ -71,15 +73,40 @@ def test(): pass test.__code__ = self.b.__code__ self.assertEqual(test(), 3) # self.b always returns 3, arbitrarily + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: DeprecationWarning not triggered + def test_invalid___code___assignment(self): + def A(): pass + def B(): yield + async def C(): yield + async def D(x): await x + + for src in [A, B, C, D]: + for dst in [A, B, C, D]: + if src == dst: + continue + + assert src.__code__.co_flags != dst.__code__.co_flags + prev = dst.__code__ + try: + with self.assertWarnsRegex(DeprecationWarning, 'code object of non-matching type'): + dst.__code__ = src.__code__ + finally: + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + dst.__code__ = prev + def test___globals__(self): self.assertIs(self.b.__globals__, globals()) self.cannot_set_attr(self.b, '__globals__', 2, (AttributeError, TypeError)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___builtins__(self): - self.assertIs(self.b.__builtins__, __builtins__) + if __name__ == "__main__": + builtins_dict = __builtins__.__dict__ + else: + builtins_dict = __builtins__ + + self.assertIs(self.b.__builtins__, builtins_dict) self.cannot_set_attr(self.b, '__builtins__', 2, (AttributeError, TypeError)) @@ -89,7 +116,7 @@ def func(s): return len(s) ns = {} func2 = type(func)(func.__code__, ns) self.assertIs(func2.__globals__, ns) - self.assertIs(func2.__builtins__, __builtins__) + self.assertIs(func2.__builtins__, builtins_dict) # Make sure that the function actually works. self.assertEqual(func2("abc"), 3) @@ -194,8 +221,24 @@ def test___qualname__(self): # __qualname__ must be a string self.cannot_set_attr(self.b, '__qualname__', 7, TypeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test___type_params__(self): + def generic[T](): pass + def not_generic(): pass + lambda_ = lambda: ... + T, = generic.__type_params__ + self.assertIsInstance(T, typing.TypeVar) + self.assertEqual(generic.__type_params__, (T,)) + for func in (not_generic, lambda_): + with self.subTest(func=func): + self.assertEqual(func.__type_params__, ()) + with self.assertRaises(TypeError): + del func.__type_params__ + with self.assertRaises(TypeError): + func.__type_params__ = 42 + func.__type_params__ = (T,) + self.assertEqual(func.__type_params__, (T,)) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test___code__(self): num_one, num_two = 7, 8 def a(): pass @@ -226,15 +269,13 @@ def e(): return num_one, num_two self.fail("__code__ with different numbers of free vars should " "not be possible") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_blank_func_defaults(self): self.assertEqual(self.b.__defaults__, None) del self.b.__defaults__ self.assertEqual(self.b.__defaults__, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_func_default_args(self): def first_func(a, b): return a+b @@ -395,8 +436,7 @@ def f(): class CellTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_comparison(self): # These tests are here simply to exercise the comparison code; # their presence should not be interpreted as providing any @@ -445,6 +485,33 @@ def test_builtin__qualname__(self): self.assertEqual([1, 2, 3].append.__qualname__, 'list.append') self.assertEqual({'foo': 'bar'}.pop.__qualname__, 'dict.pop') + @support.cpython_only + def test_builtin__self__(self): + # See https://github.com/python/cpython/issues/58211. + import builtins + import time + + # builtin function: + self.assertIs(len.__self__, builtins) + self.assertIs(time.sleep.__self__, time) + + # builtin classmethod: + self.assertIs(dict.fromkeys.__self__, dict) + self.assertIs(float.__getformat__.__self__, float) + + # builtin staticmethod: + self.assertIsNone(str.maketrans.__self__) + self.assertIsNone(bytes.maketrans.__self__) + + # builtin bound instance method: + l = [1, 2, 3] + self.assertIs(l.append.__self__, l) + + d = {'foo': 'bar'} + self.assertEqual(d.pop.__self__, d) + + self.assertIsNone(None.__repr__.__self__) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6de5d14bf73..21b94cfd2bc 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1,4 +1,5 @@ import abc +from annotationlib import Format, get_annotations import builtins import collections import collections.abc @@ -6,6 +7,7 @@ from itertools import permutations import pickle from random import choice +import re import sys from test import support import threading @@ -19,8 +21,11 @@ import contextlib from inspect import Signature +from test.support import ALWAYS_EQ from test.support import import_helper from test.support import threading_helper +from test.support import cpython_only +from test.support import EqualToForwardRef import functools @@ -60,6 +65,14 @@ def __add__(self, other): class MyDict(dict): pass +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "functools", {"os", "weakref", "typing", "annotationlib", "warnings"} + ) + class TestPartial: @@ -210,6 +223,69 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') + def test_placeholders_trailing_raise(self): + PH = self.module.Placeholder + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) + + def test_placeholders(self): + PH = self.module.Placeholder + # 1 Placeholder + args = (PH, 0) + p = self.partial(capture, *args) + actual_args, actual_kwds = p('x') + self.assertEqual(actual_args, ('x', 0)) + self.assertEqual(actual_kwds, {}) + # 2 Placeholders + args = (PH, 0, PH, 1) + p = self.partial(capture, *args) + with self.assertRaises(TypeError): + p('x') + actual_args, actual_kwds = p('x', 'y') + self.assertEqual(actual_args, ('x', 0, 'y', 1)) + self.assertEqual(actual_kwds, {}) + # Checks via `is` and not `eq` + # thus ALWAYS_EQ isn't treated as Placeholder + p = self.partial(capture, ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(len(actual_args), 1) + self.assertIs(actual_args[0], ALWAYS_EQ) + self.assertEqual(actual_kwds, {}) + + def test_placeholders_optimization(self): + PH = self.module.Placeholder + p = self.partial(capture, PH, 0) + p2 = self.partial(p, PH, 1, 2, 3) + self.assertEqual(p2.args, (PH, 0, 1, 2, 3)) + p3 = self.partial(p2, -1, 4) + actual_args, actual_kwds = p3(5) + self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5)) + self.assertEqual(actual_kwds, {}) + # inner partial has placeholders and outer partial has no args case + p = self.partial(capture, PH, 0) + p2 = self.partial(p) + self.assertEqual(p2.args, (PH, 0)) + self.assertEqual(p2(1), ((1, 0), {})) + + def test_placeholders_kw_restriction(self): + PH = self.module.Placeholder + with self.assertRaisesRegex(TypeError, "Placeholder"): + self.partial(capture, a=PH) + # Passes, as checks via `is` and not `eq` + p = self.partial(capture, a=ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(actual_args, ()) + self.assertEqual(len(actual_kwds), 1) + self.assertIs(actual_kwds['a'], ALWAYS_EQ) + + def test_construct_placeholder_singleton(self): + PH = self.module.Placeholder + tp = type(PH) + self.assertIs(tp(), PH) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) @@ -311,8 +387,26 @@ def test_setstate(self): self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) + # Set State with placeholders + PH = self.module.Placeholder + f = self.partial(signature) + f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) + self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) + msg_regex = re.escape("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f() + self.assertEqual(f(2), ((2, 1), dict(a=10))) + + # Trailing Placeholder error + f = self.partial(signature) + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, {}, None)) self.assertRaises(TypeError, f.__setstate__, [capture, (), {}, None]) @@ -320,6 +414,8 @@ def test_setstate_errors(self): self.assertRaises(TypeError, f.__setstate__, (capture, None, {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, [], {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, (), [], None)) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, ())) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, 'test')) def test_setstate_subclasses(self): f = self.partial(signature) @@ -341,6 +437,9 @@ def test_setstate_subclasses(self): self.assertEqual(r, ((1, 2), {})) self.assertIs(type(r[0]), tuple) + @support.skip_if_sanitizer("thread sanitizer crashes in __tsan::FuncEntry", thread=True) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) @@ -395,7 +494,6 @@ def __getitem__(self, key): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial_as_method(self): class A: meth = self.partial(capture, 1, a=2) @@ -406,9 +504,7 @@ class A: self.assertEqual(A.meth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) self.assertEqual(A.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) self.assertEqual(A.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) - with self.assertWarns(FutureWarning) as w: - self.assertEqual(a.meth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) - self.assertEqual(w.filename, __file__) + self.assertEqual(a.meth(3, b=4), ((1, a, 3), {'a': 2, 'b': 4})) self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) @@ -465,13 +561,18 @@ def __str__(self): self.assertIn('astr', r) self.assertIn("['sth']", r) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_repr(self): - return super().test_repr() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_recursive_repr(self): - return super().test_recursive_repr() + def test_placeholders_refcount_smoke(self): + PH = self.module.Placeholder + # sum supports vector call + lst1, start = [], [] + sum_lists = self.partial(sum, PH, start) + for i in range(10): + sum_lists([lst1, lst1]) + # collections.ChainMap initializer does not support vectorcall + map1, map2 = {}, {} + partial_cm = self.partial(collections.ChainMap, PH, map1) + for i in range(10): + partial_cm(map2, map2) class TestPartialPy(TestPartial, unittest.TestCase): @@ -497,6 +598,19 @@ class TestPartialCSubclass(TestPartialC): class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass + def test_subclass_optimization(self): + # `partial` input to `partial` subclass + p = py_functools.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + # `partial` subclass input to `partial` subclass + p = self.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + + class TestPartialMethod(unittest.TestCase): class A(object): @@ -566,11 +680,11 @@ def test_bound_method_introspection(self): def test_unbound_method_retrieval(self): obj = self.A - self.assertFalse(hasattr(obj.both, "__self__")) - self.assertFalse(hasattr(obj.nested, "__self__")) - self.assertFalse(hasattr(obj.over_partial, "__self__")) - self.assertFalse(hasattr(obj.static, "__self__")) - self.assertFalse(hasattr(self.a.static, "__self__")) + self.assertNotHasAttr(obj.both, "__self__") + self.assertNotHasAttr(obj.nested, "__self__") + self.assertNotHasAttr(obj.over_partial, "__self__") + self.assertNotHasAttr(obj.static, "__self__") + self.assertNotHasAttr(self.a.static, "__self__") def test_descriptors(self): for obj in [self.A, self.a]: @@ -634,6 +748,20 @@ def f(a, b, /): p = functools.partial(f, 1) self.assertEqual(p(2), f(1, 2)) + def test_subclass_optimization(self): + class PartialMethodSubclass(functools.partialmethod): + pass + # `partialmethod` input to `partialmethod` subclass + p = functools.partialmethod(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + # `partialmethod` subclass input to `partialmethod` subclass + p = PartialMethodSubclass(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + class TestUpdateWrapper(unittest.TestCase): @@ -698,7 +826,7 @@ def wrapper(): self.assertNotEqual(wrapper.__qualname__, f.__qualname__) self.assertEqual(wrapper.__doc__, None) self.assertEqual(wrapper.__annotations__, {}) - self.assertFalse(hasattr(wrapper, 'attr')) + self.assertNotHasAttr(wrapper, 'attr') def test_selective_update(self): def f(): @@ -747,7 +875,7 @@ def wrapper(): pass functools.update_wrapper(wrapper, max) self.assertEqual(wrapper.__name__, 'max') - self.assertTrue(wrapper.__doc__.startswith('max(')) + self.assertStartsWith(wrapper.__doc__, 'max(') self.assertEqual(wrapper.__annotations__, {}) def test_update_type_wrapper(self): @@ -758,6 +886,26 @@ def wrapper(*args): pass self.assertEqual(wrapper.__annotations__, {}) self.assertEqual(wrapper.__type_params__, ()) + def test_update_wrapper_annotations(self): + def inner(x: int): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, inner) + self.assertEqual(wrapper.__annotations__, {'x': int}) + self.assertIs(wrapper.__annotate__, inner.__annotate__) + + def with_forward_ref(x: undefined): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, with_forward_ref) + + self.assertIs(wrapper.__annotate__, with_forward_ref.__annotate__) + with self.assertRaises(NameError): + wrapper.__annotations__ + + undefined = str + self.assertEqual(wrapper.__annotations__, {'x': undefined}) + class TestWraps(TestUpdateWrapper): @@ -797,7 +945,7 @@ def wrapper(): self.assertEqual(wrapper.__name__, 'wrapper') self.assertNotEqual(wrapper.__qualname__, f.__qualname__) self.assertEqual(wrapper.__doc__, None) - self.assertFalse(hasattr(wrapper, 'attr')) + self.assertNotHasAttr(wrapper, 'attr') def test_selective_update(self): def f(): @@ -899,6 +1047,29 @@ def __getitem__(self, i): d = {"one": 1, "two": 2, "three": 3} self.assertEqual(self.reduce(add, d), "".join(d.keys())) + # test correctness of keyword usage of `initial` in `reduce` + def test_initial_keyword(self): + def add(x, y): + return x + y + self.assertEqual( + self.reduce(add, ['a', 'b', 'c'], ''), + self.reduce(add, ['a', 'b', 'c'], initial=''), + ) + self.assertEqual( + self.reduce(add, [['a', 'c'], [], ['d', 'w']], []), + self.reduce(add, [['a', 'c'], [], ['d', 'w']], initial=[]), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,8), 1), + self.reduce(lambda x, y: x*y, range(2,8), initial=1), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,21), 1), + self.reduce(lambda x, y: x*y, range(2,21), initial=1), + ) + self.assertRaises(TypeError, self.reduce, add, [0, 1], initial="") + self.assertEqual(self.reduce(42, "", initial="1"), "1") # func is never called with one item + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduceC(TestReduce, unittest.TestCase): @@ -909,6 +1080,12 @@ class TestReduceC(TestReduce, unittest.TestCase): class TestReducePy(TestReduce, unittest.TestCase): reduce = staticmethod(py_functools.reduce) + def test_reduce_with_kwargs(self): + with self.assertWarns(DeprecationWarning): + self.reduce(function=lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + with self.assertWarns(DeprecationWarning): + self.reduce(lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + class TestCmpToKey: @@ -1016,35 +1193,35 @@ def test_disallow_instantiation(self): self, type(c_functools.cmp_to_key(None)) ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + (mycmp) + def test_cmp_to_signature(self): + return super().test_cmp_to_signature() + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() got multiple values for argument 'mycmp' + def test_cmp_to_key_arguments(self): + return super().test_cmp_to_key_arguments() + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() got multiple values for argument 'mycmp' + def test_obj_field(self): + return super().test_obj_field() + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() takes 1 positional argument but 2 were given def test_bad_cmp(self): return super().test_bad_cmp() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() takes 1 positional argument but 2 were given def test_cmp_to_key(self): return super().test_cmp_to_key() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_cmp_to_key_arguments(self): - return super().test_cmp_to_key_arguments() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_cmp_to_signature(self): - return super().test_cmp_to_signature() - - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() takes 1 positional argument but 2 were given def test_hash(self): return super().test_hash() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_obj_field(self): - return super().test_obj_field() - - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() takes 1 positional argument but 2 were given def test_sort_int(self): return super().test_sort_int() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: cmp_to_key() takes 1 positional argument but 2 were given def test_sort_int_str(self): return super().test_sort_int_str() @@ -1534,6 +1711,7 @@ def f(x): f(0, **{}) self.assertEqual(f.cache_info().hits, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON; Python lru_cache impl doesn't cache hash like C impl def test_lru_hash_only_once(self): # To protect against weird reentrancy bugs and to improve # efficiency when faced with slow __hash__ methods, the @@ -1927,7 +2105,7 @@ def f(): return 1 self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True}) - @support.suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON; GC behavior differs from CPython's refcounting def test_lru_cache_weakrefable(self): @self.module.lru_cache def test_function(x): @@ -1965,8 +2143,39 @@ def orig(a, /, b, c=True): ... self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + def test_get_annotations(self): + def orig(a: int) -> str: ... + lru = self.module.lru_cache(1)(orig) + + self.assertEqual( + get_annotations(orig), {"a": int, "return": str}, + ) + self.assertEqual( + get_annotations(lru), {"a": int, "return": str}, + ) + + def test_get_annotations_with_forwardref(self): + def orig(a: int) -> nonexistent: ... + lru = self.module.lru_cache(1)(orig) + + self.assertEqual( + get_annotations(orig, format=Format.FORWARDREF), + {"a": int, "return": EqualToForwardRef('nonexistent', owner=orig)}, + ) + self.assertEqual( + get_annotations(lru, format=Format.FORWARDREF), + {"a": int, "return": EqualToForwardRef('nonexistent', owner=lru)}, + ) + with self.assertRaises(NameError): + get_annotations(orig, format=Format.VALUE) + with self.assertRaises(NameError): + get_annotations(lru, format=Format.VALUE) + @support.skip_on_s390x @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + @support.skip_if_sanitizer("requires deep stack", ub=True, thread=True) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() def test_lru_recursion(self): @self.module.lru_cache @@ -1975,15 +2184,12 @@ def fib(n): return n return fib(n-1) + fib(n-2) - if not support.Py_DEBUG: - depth = support.get_c_recursion_limit()*2//7 - with support.infinite_recursion(): - fib(depth) + fib(100) if self.module == c_functools: fib.cache_clear() with support.infinite_recursion(): with self.assertRaises(RecursionError): - fib(10000) + fib(support.exceeds_recursion_limit()) @py_functools.lru_cache() @@ -2565,15 +2771,15 @@ def _(self, arg): a.t(0) self.assertEqual(a.arg, "int") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') a.t('') self.assertEqual(a.arg, "str") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') a.t(0.0) self.assertEqual(a.arg, "base") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') def test_staticmethod_register(self): class A: @@ -2808,6 +3014,8 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): + self.assertEqual(meth.__module__, __name__) + self.assertEqual(type(meth).__module__, 'functools') self.assertEqual(meth.__qualname__, prefix + meth.__name__) self.assertEqual(meth.__doc__, ('My function docstring' @@ -2822,6 +3030,67 @@ def static_func(arg: int) -> str: self.assertEqual(A.static_func.__name__, 'static_func') self.assertEqual(A().static_func.__name__, 'static_func') + def test_method_repr(self): + class Callable: + def __call__(self, *args): + pass + + class CallableWithName: + __name__ = 'NOQUALNAME' + def __call__(self, *args): + pass + + class A: + @functools.singledispatchmethod + def func(self, arg): + pass + @functools.singledispatchmethod + @classmethod + def cls_func(cls, arg): + pass + @functools.singledispatchmethod + @staticmethod + def static_func(arg): + pass + # No __qualname__, only __name__ + no_qualname = functools.singledispatchmethod(CallableWithName()) + # No __qualname__, no __name__ + no_name = functools.singledispatchmethod(Callable()) + + self.assertEqual(repr(A.__dict__['func']), + f'') + self.assertEqual(repr(A.__dict__['cls_func']), + f'') + self.assertEqual(repr(A.__dict__['static_func']), + f'') + self.assertEqual(repr(A.__dict__['no_qualname']), + f'') + self.assertEqual(repr(A.__dict__['no_name']), + f'') + + self.assertEqual(repr(A.func), + f'') + self.assertEqual(repr(A.cls_func), + f'') + self.assertEqual(repr(A.static_func), + f'') + self.assertEqual(repr(A.no_qualname), + f'') + self.assertEqual(repr(A.no_name), + f'') + + a = A() + self.assertEqual(repr(a.func), + f'') + self.assertEqual(repr(a.cls_func), + f'') + self.assertEqual(repr(a.static_func), + f'') + self.assertEqual(repr(a.no_qualname), + f'') + self.assertEqual(repr(a.no_name), + f'') + def test_double_wrapped_methods(self): def classmethod_friendly_decorator(func): wrapped = func.__func__ @@ -2838,7 +3107,8 @@ def cls_context_manager(cls, arg: int) -> str: try: yield str(arg) finally: - return 'Done' + pass + return 'Done' @classmethod_friendly_decorator @classmethod @@ -2854,7 +3124,8 @@ def cls_context_manager(cls, arg: int) -> str: try: yield str(arg) finally: - return 'Done' + pass + return 'Done' @functools.singledispatchmethod @classmethod_friendly_decorator @@ -2937,16 +3208,16 @@ def i(arg): @i.register(42) def _(arg): return "I annotated with a non-type" - self.assertTrue(str(exc.exception).startswith(msg_prefix + "42")) - self.assertTrue(str(exc.exception).endswith(msg_suffix)) + self.assertStartsWith(str(exc.exception), msg_prefix + "42") + self.assertEndsWith(str(exc.exception), msg_suffix) with self.assertRaises(TypeError) as exc: @i.register def _(arg): return "I forgot to annotate" - self.assertTrue(str(exc.exception).startswith(msg_prefix + + self.assertStartsWith(str(exc.exception), msg_prefix + "._" - )) - self.assertTrue(str(exc.exception).endswith(msg_suffix)) + ) + self.assertEndsWith(str(exc.exception), msg_suffix) with self.assertRaises(TypeError) as exc: @i.register @@ -2956,23 +3227,23 @@ def _(arg: typing.Iterable[str]): # types from `typing`. Instead, annotate with regular types # or ABCs. return "I annotated with a generic collection" - self.assertTrue(str(exc.exception).startswith( + self.assertStartsWith(str(exc.exception), "Invalid annotation for 'arg'." - )) - self.assertTrue(str(exc.exception).endswith( + ) + self.assertEndsWith(str(exc.exception), 'typing.Iterable[str] is not a class.' - )) + ) with self.assertRaises(TypeError) as exc: @i.register def _(arg: typing.Union[int, typing.Iterable[str]]): return "Invalid Union" - self.assertTrue(str(exc.exception).startswith( + self.assertStartsWith(str(exc.exception), "Invalid annotation for 'arg'." - )) - self.assertTrue(str(exc.exception).endswith( - 'typing.Union[int, typing.Iterable[str]] not all arguments are classes.' - )) + ) + self.assertEndsWith(str(exc.exception), + 'int | typing.Iterable[str] not all arguments are classes.' + ) def test_invalid_positional_argument(self): @functools.singledispatch @@ -3119,6 +3390,28 @@ def _(arg: typing.List[float] | bytes): self.assertEqual(f(""), "default") self.assertEqual(f(b""), "default") + def test_forward_reference(self): + @functools.singledispatch + def f(arg, arg2=None): + return "default" + + @f.register + def _(arg: str, arg2: undefined = None): + return "forward reference" + + self.assertEqual(f(1), "default") + self.assertEqual(f(""), "forward reference") + + def test_unresolved_forward_reference(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "is an unresolved forward reference"): + @f.register + def _(arg: undefined): + return "forward reference" + def test_method_equal_instances(self): # gh-127750: Reference to self was cached class A: @@ -3189,16 +3482,11 @@ def _(item: int, arg: bytes) -> str: def test_method_signatures(self): class A: - def m(self, item, arg: int) -> str: - return str(item) - @classmethod - def cm(cls, item, arg: int) -> str: - return str(item) @functools.singledispatchmethod def func(self, item, arg: int) -> str: return str(item) @func.register - def _(self, item, arg: bytes) -> str: + def _(self, item: int, arg: bytes) -> str: return str(item) @functools.singledispatchmethod @@ -3207,7 +3495,7 @@ def cls_func(cls, item, arg: int) -> str: return str(arg) @func.register @classmethod - def _(cls, item, arg: bytes) -> str: + def _(cls, item: int, arg: bytes) -> str: return str(item) @functools.singledispatchmethod @@ -3216,7 +3504,7 @@ def static_func(item, arg: int) -> str: return str(arg) @func.register @staticmethod - def _(item, arg: bytes) -> str: + def _(item: int, arg: bytes) -> str: return str(item) self.assertEqual(str(Signature.from_callable(A.func)), @@ -3301,7 +3589,6 @@ class MyClass(metaclass=MyMeta): ): MyClass.prop - @unittest.expectedFailure # TODO: RUSTPYTHON def test_reuse_different_names(self): """Disallow this case because decorated function a would not be cached.""" with self.assertRaises(TypeError) as ctx: diff --git a/Lib/test/test_future_stmt/badsyntax_future10.py b/Lib/test/test_future_stmt/badsyntax_future.py similarity index 100% rename from Lib/test/test_future_stmt/badsyntax_future10.py rename to Lib/test/test_future_stmt/badsyntax_future.py diff --git a/Lib/test/test_future_stmt/badsyntax_future3.py b/Lib/test/test_future_stmt/badsyntax_future3.py deleted file mode 100644 index f1c8417edaa..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future3.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -from __future__ import rested_snopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future4.py b/Lib/test/test_future_stmt/badsyntax_future4.py deleted file mode 100644 index b5f4c98e922..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future4.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -import __future__ -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future5.py b/Lib/test/test_future_stmt/badsyntax_future5.py deleted file mode 100644 index 8a7e5fcb70f..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future5.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -import foo -from __future__ import nested_scopes - - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future6.py b/Lib/test/test_future_stmt/badsyntax_future6.py deleted file mode 100644 index 5a8b55a02c4..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future6.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -"this isn't a doc string" -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future7.py b/Lib/test/test_future_stmt/badsyntax_future7.py deleted file mode 100644 index 131db2c2164..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future7.py +++ /dev/null @@ -1,11 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes; import string; from __future__ import \ - nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future8.py b/Lib/test/test_future_stmt/badsyntax_future8.py deleted file mode 100644 index ca45289e2e5..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future8.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import * - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/badsyntax_future9.py b/Lib/test/test_future_stmt/badsyntax_future9.py deleted file mode 100644 index 916de06ab71..00000000000 --- a/Lib/test/test_future_stmt/badsyntax_future9.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes, braces - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/future_test1.py b/Lib/test/test_future_stmt/import_nested_scope_twice.py similarity index 100% rename from Lib/test/test_future_stmt/future_test1.py rename to Lib/test/test_future_stmt/import_nested_scope_twice.py diff --git a/Lib/test/test_future_stmt/future_test2.py b/Lib/test/test_future_stmt/nested_scope.py similarity index 60% rename from Lib/test/test_future_stmt/future_test2.py rename to Lib/test/test_future_stmt/nested_scope.py index 3d7fc860a37..a8433a42cbb 100644 --- a/Lib/test/test_future_stmt/future_test2.py +++ b/Lib/test/test_future_stmt/nested_scope.py @@ -1,6 +1,6 @@ """This is a test""" -from __future__ import nested_scopes; import site +from __future__ import nested_scopes; import site # noqa: F401 def f(x): def g(y): diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 9c30054963b..faa5f4cc683 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -10,6 +10,8 @@ import re import sys +TOP_LEVEL_MSG = 'from __future__ imports must occur at the beginning of the file' + rx = re.compile(r'\((\S+).py, line (\d+)') def get_error_location(msg): @@ -18,81 +20,145 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): - def check_syntax_error(self, err, basename, lineno, offset=1): - self.assertIn('%s.py, line %d' % (basename, lineno), str(err)) - self.assertEqual(os.path.basename(err.filename), basename + '.py') + def check_syntax_error(self, err, basename, + *, + lineno, + message=TOP_LEVEL_MSG, offset=1): + if basename != '': + basename += '.py' + + self.assertEqual(f'{message} ({basename}, line {lineno})', str(err)) + self.assertEqual(os.path.basename(err.filename), basename) self.assertEqual(err.lineno, lineno) self.assertEqual(err.offset, offset) - def test_future1(self): - with import_helper.CleanImport('test.test_future_stmt.future_test1'): - from test.test_future_stmt import future_test1 - self.assertEqual(future_test1.result, 6) + def assertSyntaxError(self, code, + *, + lineno=1, + message=TOP_LEVEL_MSG, offset=1, + parametrize_docstring=True): + code = dedent(code.lstrip('\n')) + for add_docstring in ([False, True] if parametrize_docstring else [False]): + with self.subTest(code=code, add_docstring=add_docstring): + if add_docstring: + code = '"""Docstring"""\n' + code + lineno += 1 + with self.assertRaises(SyntaxError) as cm: + exec(code) + self.check_syntax_error(cm.exception, "", + lineno=lineno, + message=message, + offset=offset) + + def test_import_nested_scope_twice(self): + # Import the name nested_scopes twice to trigger SF bug #407394 + with import_helper.CleanImport( + 'test.test_future_stmt.import_nested_scope_twice', + ): + from test.test_future_stmt import import_nested_scope_twice + self.assertEqual(import_nested_scope_twice.result, 6) - def test_future2(self): - with import_helper.CleanImport('test.test_future_stmt.future_test2'): - from test.test_future_stmt import future_test2 - self.assertEqual(future_test2.result, 6) + def test_nested_scope(self): + with import_helper.CleanImport('test.test_future_stmt.nested_scope'): + from test.test_future_stmt import nested_scope + self.assertEqual(nested_scope.result, 6) def test_future_single_import(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_single_import', ): - from test.test_future_stmt import test_future_single_import + from test.test_future_stmt import test_future_single_import # noqa: F401 def test_future_multiple_imports(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_multiple_imports', ): - from test.test_future_stmt import test_future_multiple_imports + from test.test_future_stmt import test_future_multiple_imports # noqa: F401 def test_future_multiple_features(self): with import_helper.CleanImport( "test.test_future_stmt.test_future_multiple_features", ): - from test.test_future_stmt import test_future_multiple_features + from test.test_future_stmt import test_future_multiple_features # noqa: F401 - def test_badfuture3(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future3 - self.check_syntax_error(cm.exception, "badsyntax_future3", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 24 + def test_unknown_future_flag(self): + code = """ + from __future__ import nested_scopes + from __future__ import rested_snopes # typo error here: nested => rested + """ + self.assertSyntaxError( + code, lineno=2, + message='future feature rested_snopes is not defined', offset=24, + ) - def test_badfuture4(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future4 - self.check_syntax_error(cm.exception, "badsyntax_future4", 3) + def test_future_import_not_on_top(self): + code = """ + import some_module + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - def test_badfuture5(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future5 - self.check_syntax_error(cm.exception, "badsyntax_future5", 4) + code = """ + import __future__ + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_badfuture6(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future6 - self.check_syntax_error(cm.exception, "badsyntax_future6", 3) + code = """ + from __future__ import absolute_import + "spam, bar, blah" + from __future__ import print_function + """ + self.assertSyntaxError(code, lineno=3) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised + def test_future_import_with_extra_string(self): + code = """ + '''Docstring''' + "this isn't a doc string" + from __future__ import nested_scopes + """ + self.assertSyntaxError(code, lineno=3, parametrize_docstring=False) - def test_badfuture7(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future7 - self.check_syntax_error(cm.exception, "badsyntax_future7", 3, 54) + def test_multiple_import_statements_on_same_line(self): + # With `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import \ + nested_scopes + """ + self.assertSyntaxError(code, offset=54) - def test_badfuture8(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future8 - self.check_syntax_error(cm.exception, "badsyntax_future8", 3) + # Without `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import nested_scopes + """ + self.assertSyntaxError(code, offset=54) - def test_badfuture9(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future9 - self.check_syntax_error(cm.exception, "badsyntax_future9", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 24 + def test_future_import_star(self): + code = """ + from __future__ import * + """ + self.assertSyntaxError(code, message='future feature * is not defined', offset=24) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_future_import_braces(self): + code = """ + from __future__ import braces + """ + # Congrats, you found an easter egg! + self.assertSyntaxError(code, message='not a chance', offset=24) - def test_badfuture10(self): + code = """ + from __future__ import nested_scopes, braces + """ + self.assertSyntaxError(code, message='not a chance', offset=39) + + def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future10 - self.check_syntax_error(cm.exception, "badsyntax_future10", 3) + from test.test_future_stmt import badsyntax_future # noqa: F401 + self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): # bpo-39562: test that future flags and compiler flags doesn't clash @@ -109,31 +175,12 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) self.assertIsInstance(scope["x"], str) + @unittest.expectedFailure # TODO: RUSTPYTHON; barry_as_FLUFL (<> operator) not supported def test_syntactical_future_repl(self): p = spawn_python('-i') p.stdin.write(b"from __future__ import barry_as_FLUFL\n") @@ -141,6 +188,26 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ @@ -198,6 +265,7 @@ def _exec_future(self, code): ) return scope + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "t'{a + b}'" != "t'{a + b}'" def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -361,6 +429,11 @@ def test_annotations(self): eq('(((a)))', 'a') eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + eq("t''") + eq("t'{a + b}'") + eq("t'{a!s}'") + eq("t'{a:b}'") + eq("t'{a:b=}'") def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected @@ -384,8 +457,6 @@ def test_infinity_numbers(self): self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})") self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_annotation_with_complex_target(self): with self.assertRaises(SyntaxError): exec( @@ -409,8 +480,7 @@ def bar(): self.assertEqual(foo.__code__.co_cellvars, ()) self.assertEqual(foo().__code__.co_freevars, ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised def test_annotations_forbidden(self): with self.assertRaises(SyntaxError): self._exec_future("test: (yield)") diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index c6f66085c74..c6069f6fb2e 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -6,6 +6,7 @@ import unittest import weakref import inspect +import textwrap import types from test import support @@ -48,7 +49,6 @@ def test_raise_and_yield_from(self): class FinalizationTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_frame_resurrect(self): # A generator frame can be resurrected by a generator's finalization. def gen(): @@ -68,7 +68,7 @@ def gen(): del frame support.gc_collect() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_refcycle(self): # A generator caught in a refcycle gets finalized anyway. old_garbage = gc.garbage[:] @@ -84,7 +84,7 @@ def gen(): g = gen() next(g) g.send(g) - self.assertGreater(sys.getrefcount(g), 2) + self.assertGreaterEqual(sys.getrefcount(g), 2) self.assertFalse(finalized) del g support.gc_collect() @@ -114,6 +114,28 @@ def g3(): return (yield from f()) gen.send(2) self.assertEqual(cm.exception.value, 2) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1 + def test_generator_resurrect(self): + # Test that a resurrected generator still has a valid gi_code + resurrected = [] + + # Resurrect a generator in a finalizer + exec(textwrap.dedent(""" + def gen(): + try: + yield + except: + resurrected.append(g) + + g = gen() + next(g) + """), {"resurrected": resurrected}) + + support.gc_collect() + + self.assertEqual(len(resurrected), 1) + self.assertIsInstance(resurrected[0].gi_code, types.CodeType) + class GeneratorTest(unittest.TestCase): @@ -176,7 +198,6 @@ def f(): g.send(0) self.assertEqual(next(g), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; NotImplementedError def test_handle_frame_object_in_creation(self): #Attempt to expose partially constructed frames @@ -249,6 +270,28 @@ def loop(): #This should not raise loop() + def test_genexpr_only_calls_dunder_iter_once(self): + + class Iterator: + + def __init__(self): + self.val = 0 + + def __next__(self): + if self.val == 2: + raise StopIteration + self.val += 1 + return self.val + + # No __iter__ method + + class C: + + def __iter__(self): + return Iterator() + + self.assertEqual([1, 2], list(i for i in C())) + class ModifyUnderlyingIterableTest(unittest.TestCase): iterables = [ @@ -277,23 +320,27 @@ def gen(it): yield x return gen(range(10)) - def process_tests(self, get_generator): + def process_tests(self, get_generator, is_expr): + err_iterator = "'.*' object is not an iterator" + err_iterable = "'.*' object is not iterable" for obj in self.iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj, obj=obj): - self.assertListEqual(list(g_obj), list(obj)) + if is_expr: + self.assertRaisesRegex(TypeError, err_iterator, list, g_obj) + else: + self.assertListEqual(list(g_obj), list(obj)) g_iter = get_generator(iter(obj)) with self.subTest(g_iter=g_iter, obj=obj): self.assertListEqual(list(g_iter), list(obj)) - err_regex = "'.*' object is not iterable" for obj in self.non_iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj): - self.assertRaisesRegex(TypeError, err_regex, list, g_obj) + err = err_iterator if is_expr else err_iterable + self.assertRaisesRegex(TypeError, err, list, g_obj) - @unittest.expectedFailure # AssertionError: TypeError not raised by list def test_modify_f_locals(self): def modify_f_locals(g, local, obj): g.gi_frame.f_locals[local] = obj @@ -305,10 +352,9 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return modify_f_locals(self.genfunc(), 'it', obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) - @unittest.expectedFailure # AssertionError: "'.*' object is not iterable" does not match "'complex' object is not an iterator" def test_new_gen_from_gi_code(self): def new_gen_from_gi_code(g, obj): generator_func = types.FunctionType(g.gi_code, {}) @@ -320,8 +366,8 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return new_gen_from_gi_code(self.genfunc(), obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) class ExceptionTest(unittest.TestCase): @@ -452,7 +498,6 @@ def gen(): self.assertEqual(next(g), "done") self.assertIsNone(sys.exception()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_except_throw_bad_exception(self): class E(Exception): def __new__(cls, *args, **kwargs): @@ -479,7 +524,6 @@ def generator(): with self.assertRaises(StopIteration): gen.throw(E) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: DeprecationWarning not triggered def test_gen_3_arg_deprecation_warning(self): def g(): yield 42 @@ -545,7 +589,6 @@ def f(): gen.send(None) self.assertIsNone(gen.close()) - @unittest.expectedFailure # AssertionError: None != 0 def test_close_return_value(self): def f(): try: @@ -592,7 +635,6 @@ def f(): next(gen) self.assertIsNone(gen.close()) - @unittest.expectedFailure # AssertionError: None != 0 def test_close_closed(self): def f(): try: @@ -618,7 +660,7 @@ def f(): with self.assertRaises(RuntimeError): gen.close() - @unittest.expectedFailure # AssertionError: .Foo object at 0xb400007e3c212160> is not None + @unittest.expectedFailure # TODO: RUSTPYTHON; no deterministic GC finalization def test_close_releases_frame_locals(self): # See gh-118272 @@ -681,7 +723,7 @@ def get_frame(index): self.assertIn('a', frame_locals) self.assertEqual(frame_locals['a'], 42) - @unittest.expectedFailure # AssertionError: 'a' not found in {'frame_locals1': None} + @unittest.expectedFailure # TODO: RUSTPYTHON; frame locals don't survive generator deallocation def test_frame_locals_outlive_generator(self): frame_locals1 = None @@ -726,7 +768,6 @@ def g(): class GeneratorThrowTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_exception_context_with_yield(self): def f(): try: @@ -763,7 +804,6 @@ def f(): # This ensures that the assertions inside were executed. self.assertEqual(actual, 'b') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_exception_context_with_yield_from(self): def f(): yield @@ -832,7 +872,8 @@ def check_stack_names(self, frame, expected): while frame: name = frame.f_code.co_name # Stop checking frames when we get to our test helper. - if name.startswith('check_') or name.startswith('call_'): + if (name.startswith('check_') or name.startswith('call_') + or name.startswith('test')): break names.append(name) @@ -873,9 +914,27 @@ def call_throw(gen): self.check_yield_from_example(call_throw) + def test_throw_with_yield_from_custom_generator(self): + + class CustomGen: + def __init__(self, test): + self.test = test + def throw(self, *args): + self.test.check_stack_names(sys._getframe(), ['throw', 'g']) + def __iter__(self): + return self + def __next__(self): + return 42 + + def g(target): + yield from target + + gen = g(CustomGen(self)) + gen.send(None) + gen.throw(RuntimeError) + class YieldFromTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_generator_gi_yieldfrom(self): def a(): self.assertEqual(inspect.getgeneratorstate(gen_b), inspect.GEN_RUNNING) @@ -1081,7 +1140,7 @@ def b(): File "", line 1, in ? File "", line 2, in g File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(k) # and the generator cannot be resumed Traceback (most recent call last): File "", line 1, in ? @@ -1275,7 +1334,7 @@ def b(): >>> [s for s in dir(i) if not s.startswith('_')] ['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_suspended', 'gi_yieldfrom', 'send', 'throw'] >>> from test.support import HAVE_DOCSTRINGS ->>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).') +>>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).') # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE Implement next(self). >>> iter(i) is i True @@ -2427,17 +2486,17 @@ def printsolution(self, x): ... SyntaxError: 'yield from' outside function ->>> def f(): x = yield = y +>>> def f(): x = yield = y # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE Traceback (most recent call last): ... SyntaxError: assignment to yield expression not possible ->>> def f(): (yield bar) = y +>>> def f(): (yield bar) = y # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE Traceback (most recent call last): ... SyntaxError: cannot assign to yield expression here. Maybe you meant '==' instead of '='? ->>> def f(): (yield bar) += y +>>> def f(): (yield bar) += y # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE Traceback (most recent call last): ... SyntaxError: 'yield expression' is an illegal expression for augmented assignment @@ -2633,17 +2692,21 @@ def printsolution(self, x): Our ill-behaved code should be invoked during GC: ->>> with support.catch_unraisable_exception() as cm: +>>> with support.catch_unraisable_exception() as cm: # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE ... g = f() ... next(g) +... gen_repr = repr(g) ... del g ... +... cm.unraisable.err_msg == (f'Exception ignored while closing ' +... f'generator {gen_repr}') ... cm.unraisable.exc_type == RuntimeError ... "generator ignored GeneratorExit" in str(cm.unraisable.exc_value) ... cm.unraisable.exc_traceback is not None True True True +True And errors thrown during closing should propagate: @@ -2747,11 +2810,13 @@ def printsolution(self, x): ... raise RuntimeError(message) ... invoke("del failed") ... ->>> with support.catch_unraisable_exception() as cm: -... l = Leaker() -... del l +>>> with support.catch_unraisable_exception() as cm: # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE +... leaker = Leaker() +... del_repr = repr(type(leaker).__del__) +... del leaker ... -... cm.unraisable.object == Leaker.__del__ +... cm.unraisable.err_msg == (f'Exception ignored while ' +... f'calling deallocator {del_repr}') ... cm.unraisable.exc_type == RuntimeError ... str(cm.unraisable.exc_value) == "del failed" ... cm.unraisable.exc_traceback is not None @@ -2778,7 +2843,8 @@ def printsolution(self, x): } def load_tests(loader, tests, pattern): - # tests.addTest(doctest.DocTestSuite()) # TODO: RUSTPYTHON + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # TODO: RUSTPYTHON return tests diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index c1c49dc29d8..4f5b10650ac 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -57,6 +57,11 @@ from weakref import WeakSet, ReferenceType, ref import typing from typing import Unpack +try: + from tkinter import Event +except ImportError: + Event = None +from string.templatelib import Template, Interpolation from typing import TypeVar T = TypeVar('T') @@ -96,7 +101,7 @@ class BaseTest(unittest.TestCase): """Test basics.""" - generic_types = [type, tuple, list, dict, set, frozenset, enumerate, + generic_types = [type, tuple, list, dict, set, frozenset, enumerate, memoryview, defaultdict, deque, SequenceMatcher, dircmp, @@ -133,12 +138,19 @@ class BaseTest(unittest.TestCase): Future, _WorkItem, Morsel, DictReader, DictWriter, - array] + array, + staticmethod, + classmethod, + Template, + Interpolation, + ] if ctypes is not None: - generic_types.extend((ctypes.Array, ctypes.LibraryLoader)) + generic_types.extend((ctypes.Array, ctypes.LibraryLoader, ctypes.py_object)) if ValueProxy is not None: generic_types.extend((ValueProxy, DictProxy, ListProxy, ApplyResult, MPSimpleQueue, MPQueue, MPJoinableQueue)) + if Event is not None: + generic_types.append(Event) def test_subscriptable(self): for t in self.generic_types: @@ -151,7 +163,6 @@ def test_subscriptable(self): self.assertEqual(alias.__args__, (int,)) self.assertEqual(alias.__parameters__, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_unsubscriptable(self): for t in int, str, float, Sized, Hashable: tname = t.__name__ @@ -210,7 +221,6 @@ class MyList(list): self.assertEqual(t.__args__, (int,)) self.assertEqual(t.__parameters__, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_repr(self): class MyList(list): pass @@ -226,13 +236,63 @@ class MyGeneric: self.assertEqual(repr(x2), 'tuple[*tuple[int, str]]') x3 = tuple[*tuple[int, ...]] self.assertEqual(repr(x3), 'tuple[*tuple[int, ...]]') - self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) + self.assertEndsWith(repr(MyList[int]), '.BaseTest.test_repr..MyList[int]') self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr # gh-105488 - self.assertTrue(repr(MyGeneric[int]).endswith('MyGeneric[int]')) - self.assertTrue(repr(MyGeneric[[]]).endswith('MyGeneric[[]]')) - self.assertTrue(repr(MyGeneric[[int, str]]).endswith('MyGeneric[[int, str]]')) + self.assertEndsWith(repr(MyGeneric[int]), 'MyGeneric[int]') + self.assertEndsWith(repr(MyGeneric[[]]), 'MyGeneric[[]]') + self.assertEndsWith(repr(MyGeneric[[int, str]]), 'MyGeneric[[int, str]]') + + def test_evil_repr1(self): + # gh-143635 + class Zap: + def __init__(self, container): + self.container = container + def __getattr__(self, name): + if name == "__origin__": + self.container.clear() + return None + if name == "__args__": + return () + raise AttributeError + + params = [] + params.append(Zap(params)) + alias = GenericAlias(list, (params,)) + repr_str = repr(alias) + self.assertTrue(repr_str.startswith("list[["), repr_str) + + def test_evil_repr2(self): + class Zap: + def __init__(self, container): + self.container = container + def __getattr__(self, name): + if name == "__qualname__": + self.container.clear() + return "abcd" + if name == "__module__": + return None + raise AttributeError + + params = [] + params.append(Zap(params)) + alias = GenericAlias(list, (params,)) + repr_str = repr(alias) + self.assertTrue(repr_str.startswith("list[["), repr_str) + + def test_evil_repr3(self): + # gh-143823 + lst = [] + class X: + def __repr__(self): + lst.clear() + return "x" + + lst += [X(), 1] + ga = GenericAlias(int, lst) + with self.assertRaises(IndexError): + repr(ga) def test_exposed_type(self): import types @@ -334,7 +394,6 @@ def test_parameter_chaining(self): with self.assertRaises(TypeError): dict[T, T][str, int] - @unittest.expectedFailure # TODO: RUSTPYTHON def test_equality(self): self.assertEqual(list[int], list[int]) self.assertEqual(dict[str, int], dict[str, int]) @@ -353,7 +412,7 @@ def test_isinstance(self): def test_issubclass(self): class L(list): ... - self.assertTrue(issubclass(L, list)) + self.assertIsSubclass(L, list) with self.assertRaises(TypeError): issubclass(L, list[str]) @@ -365,7 +424,6 @@ def test_type_generic(self): self.assertEqual(t(test), Test) self.assertEqual(t(0), int) - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_type_subclass_generic(self): class MyType(type): pass @@ -426,7 +484,6 @@ def test_union_generic(self): self.assertEqual(a.__args__, (list[T], tuple[T, ...])) self.assertEqual(a.__parameters__, (T,)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dir(self): ga = list[int] dir_of_gen_alias = set(dir(ga)) @@ -492,6 +549,76 @@ def test_del_iter(self): iter_x = iter(t) del iter_x + def test_paramspec_specialization(self): + # gh-124445 + T = TypeVar("T") + U = TypeVar("U") + type X[**P] = Callable[P, int] + + generic = X[[T]] + self.assertEqual(generic.__args__, ([T],)) + self.assertEqual(generic.__parameters__, (T,)) + specialized = generic[str] + self.assertEqual(specialized.__args__, ([str],)) + self.assertEqual(specialized.__parameters__, ()) + + generic = X[(T,)] + self.assertEqual(generic.__args__, (T,)) + self.assertEqual(generic.__parameters__, (T,)) + specialized = generic[str] + self.assertEqual(specialized.__args__, (str,)) + self.assertEqual(specialized.__parameters__, ()) + + generic = X[[T, U]] + self.assertEqual(generic.__args__, ([T, U],)) + self.assertEqual(generic.__parameters__, (T, U)) + specialized = generic[str, int] + self.assertEqual(specialized.__args__, ([str, int],)) + self.assertEqual(specialized.__parameters__, ()) + + generic = X[(T, U)] + self.assertEqual(generic.__args__, (T, U)) + self.assertEqual(generic.__parameters__, (T, U)) + specialized = generic[str, int] + self.assertEqual(specialized.__args__, (str, int)) + self.assertEqual(specialized.__parameters__, ()) + + def test_nested_paramspec_specialization(self): + # gh-124445 + type X[**P, T] = Callable[P, T] + + x_list = X[[int, str], float] + self.assertEqual(x_list.__args__, ([int, str], float)) + self.assertEqual(x_list.__parameters__, ()) + + x_tuple = X[(int, str), float] + self.assertEqual(x_tuple.__args__, ((int, str), float)) + self.assertEqual(x_tuple.__parameters__, ()) + + U = TypeVar("U") + V = TypeVar("V") + + multiple_params_list = X[[int, U], V] + self.assertEqual(multiple_params_list.__args__, ([int, U], V)) + self.assertEqual(multiple_params_list.__parameters__, (U, V)) + multiple_params_list_specialized = multiple_params_list[str, float] + self.assertEqual(multiple_params_list_specialized.__args__, ([int, str], float)) + self.assertEqual(multiple_params_list_specialized.__parameters__, ()) + + multiple_params_tuple = X[(int, U), V] + self.assertEqual(multiple_params_tuple.__args__, ((int, U), V)) + self.assertEqual(multiple_params_tuple.__parameters__, (U, V)) + multiple_params_tuple_specialized = multiple_params_tuple[str, float] + self.assertEqual(multiple_params_tuple_specialized.__args__, ((int, str), float)) + self.assertEqual(multiple_params_tuple_specialized.__parameters__, ()) + + deeply_nested = X[[U, [V], int], V] + self.assertEqual(deeply_nested.__args__, ([U, [V], int], V)) + self.assertEqual(deeply_nested.__parameters__, (U, V)) + deeply_nested_specialized = deeply_nested[str, float] + self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) + self.assertEqual(deeply_nested_specialized.__parameters__, ()) + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index aa843f6d9f0..e530b463966 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,5 +1,6 @@ import unittest from test import support +from test.support.import_helper import import_module class TestMROEntry(unittest.TestCase): @@ -98,7 +99,7 @@ def __mro_entries__(self): return () d = C_too_few() with self.assertRaises(TypeError): - class D(d): ... + class E(d): ... def test_mro_entry_errors_2(self): class C_not_callable: @@ -111,7 +112,7 @@ def __mro_entries__(self): return object c = C_not_tuple() with self.assertRaises(TypeError): - class D(c): ... + class E(c): ... def test_mro_entry_metaclass(self): meta_args = [] @@ -227,8 +228,6 @@ def __class_getitem__(cls, one, two): with self.assertRaises(TypeError): C_too_many[int] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_class_getitem_errors_2(self): class C: def __class_getitem__(cls, item): @@ -279,7 +278,9 @@ def __class_getitem__(cls, item): class CAPITest(unittest.TestCase): def test_c_class(self): - from _testcapi import Generic, GenericAlias + _testcapi = import_module("_testcapi") + Generic = _testcapi.Generic + GenericAlias = _testcapi.GenericAlias self.assertIsInstance(Generic.__class_getitem__(int), GenericAlias) IntGeneric = Generic[int] diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 89e4fe1882e..1a44cedcd36 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import ( - is_apple, is_emscripten, os_helper, warnings_helper -) +from test import support +from test.support import os_helper +from test.support import warnings_helper from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -92,8 +92,8 @@ def test_commonprefix(self): for s1 in testlist: for s2 in testlist: p = commonprefix([s1, s2]) - self.assertTrue(s1.startswith(p)) - self.assertTrue(s2.startswith(p)) + self.assertStartsWith(s1, p) + self.assertStartsWith(s2, p) if s1 != s2: n = len(p) self.assertNotEqual(s1[n:n+1], s2[n:n+1]) @@ -161,7 +161,6 @@ def test_exists(self): self.assertIs(self.pathmodule.lexists(path=filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") def test_exists_fd(self): r, w = os.pipe() try: @@ -171,8 +170,6 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exists_bool(self): for fd in False, True: with self.assertWarnsRegex(RuntimeWarning, @@ -352,7 +349,6 @@ def test_invalid_paths(self): with self.assertRaisesRegex(ValueError, 'embedded null'): func(b'/tmp\x00abcds') - # Following TestCase is not supposed to be run from test_genericpath. # It is inherited by other test modules (ntpath, posixpath). @@ -449,6 +445,19 @@ def check(value, expected): os.fsencode('$bar%s bar' % nonascii)) check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = self.pathmodule.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('$A'*n), 'B'*n) + self.assertEqual(expandvars('${A}'*n), 'B'*n) + self.assertEqual(expandvars('$A!'*n), 'B!'*n) + self.assertEqual(expandvars('${A}A'*n), 'BA'*n) + self.assertEqual(expandvars('${'*10*n), '${'*10*n) + def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) with warnings.catch_warnings(): @@ -506,7 +515,7 @@ def test_nonascii_abspath(self): # directory (when the bytes name is used). and sys.platform not in { "win32", "emscripten", "wasi" - } and not is_apple + } and not support.is_apple ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index 295a2c81363..8d0d5084abb 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -19,21 +19,34 @@ def assertError(self, *args, **kwargs): self.assertRaises(getopt.GetoptError, *args, **kwargs) def test_short_has_arg(self): - self.assertTrue(getopt.short_has_arg('a', 'a:')) - self.assertFalse(getopt.short_has_arg('a', 'a')) + self.assertIs(getopt.short_has_arg('a', 'a:'), True) + self.assertIs(getopt.short_has_arg('a', 'a'), False) + self.assertEqual(getopt.short_has_arg('a', 'a::'), '?') self.assertError(getopt.short_has_arg, 'a', 'b') def test_long_has_args(self): has_arg, option = getopt.long_has_args('abc', ['abc=']) - self.assertTrue(has_arg) + self.assertIs(has_arg, True) self.assertEqual(option, 'abc') has_arg, option = getopt.long_has_args('abc', ['abc']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) self.assertEqual(option, 'abc') + has_arg, option = getopt.long_has_args('abc', ['abc=?']) + self.assertEqual(has_arg, '?') + self.assertEqual(option, 'abc') + + has_arg, option = getopt.long_has_args('abc', ['abcd=']) + self.assertIs(has_arg, True) + self.assertEqual(option, 'abcd') + has_arg, option = getopt.long_has_args('abc', ['abcd']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) + self.assertEqual(option, 'abcd') + + has_arg, option = getopt.long_has_args('abc', ['abcd=?']) + self.assertEqual(has_arg, '?') self.assertEqual(option, 'abcd') self.assertError(getopt.long_has_args, 'abc', ['def']) @@ -49,9 +62,9 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, []) - #opts, args = getopt.do_shorts([], 'a=1', 'a:', []) - #self.assertEqual(opts, [('-a', '1')]) - #self.assertEqual(args, []) + opts, args = getopt.do_shorts([], 'a=1', 'a:', []) + self.assertEqual(opts, [('-a', '=1')]) + self.assertEqual(args, []) opts, args = getopt.do_shorts([], 'a', 'a:', ['1']) self.assertEqual(opts, [('-a', '1')]) @@ -61,6 +74,14 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, ['2']) + opts, args = getopt.do_shorts([], 'a', 'a::', ['1']) + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_shorts([], 'a1', 'a::', []) + self.assertEqual(opts, [('-a', '1')]) + self.assertEqual(args, []) + self.assertError(getopt.do_shorts, [], 'a1', 'a', []) self.assertError(getopt.do_shorts, [], 'a', 'a:', []) @@ -77,6 +98,22 @@ def test_do_longs(self): self.assertEqual(opts, [('--abcd', '1')]) self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['abc=?'], ['1']) + self.assertEqual(opts, [('--abc', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc', ['abcd=?'], ['1']) + self.assertEqual(opts, [('--abcd', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc=1', ['abc=?'], []) + self.assertEqual(opts, [('--abc', '1')]) + self.assertEqual(args, []) + + opts, args = getopt.do_longs([], 'abc=1', ['abcd=?'], []) + self.assertEqual(opts, [('--abcd', '1')]) + self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['ab', 'abc', 'abcd'], []) self.assertEqual(opts, [('--abc', '')]) self.assertEqual(args, []) @@ -95,7 +132,7 @@ def test_getopt(self): # note: the empty string between '-a' and '--beta' is significant: # it simulates an empty string option argument ('-a ""') on the # command line. - cmdline = ['-a', '1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', + cmdline = ['-a1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', '', '--beta', 'arg1', 'arg2'] opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) @@ -106,33 +143,53 @@ def test_getopt(self): # accounted for in the code that calls getopt(). self.assertEqual(args, ['arg1', 'arg2']) + cmdline = ['-a1', '--alpha=2', '--alpha=', '-a', '--alpha', 'arg1', 'arg2'] + opts, args = getopt.getopt(cmdline, 'a::', ['alpha=?']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): # Test handling of GNU style scanning mode. - cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2'] + cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2', '--beta', + '3', 'arg2'] # GNU style opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) - self.assertEqual(args, ['arg1']) - self.assertEqual(opts, [('-a', ''), ('-b', '1'), - ('--alpha', ''), ('--beta', '2')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + + opts, args = getopt.gnu_getopt(cmdline, 'ab::', ['alpha', 'beta=?']) + self.assertEqual(args, ['arg1', '1', '3', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', ''), ('--alpha', ''), + ('--beta', '2'), ('--beta', '')]) # recognize "-" as an argument opts, args = getopt.gnu_getopt(['-a', '-', '-b', '-'], 'ab:', []) self.assertEqual(args, ['-']) self.assertEqual(opts, [('-a', ''), ('-b', '-')]) + # Return positional arguments intermixed with options. + opts, args = getopt.gnu_getopt(cmdline, '-ab:', ['alpha', 'beta=']) + self.assertEqual(args, ['arg2']) + self.assertEqual(opts, [('-a', ''), (None, ['arg1']), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + # Posix style via + opts, args = getopt.gnu_getopt(cmdline, '+ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) # Posix style via POSIXLY_CORRECT self.env["POSIXLY_CORRECT"] = "1" opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 80dda2caaa3..9c3def2c3be 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -161,6 +161,81 @@ def test_falls_back_to_stdin(self): self.assertIn('Warning', stderr.getvalue()) self.assertIn('Password:', stderr.getvalue()) + def test_echo_char_replaces_input_with_asterisks(self): + mock_result = '*************' + with mock.patch('os.open') as os_open, \ + mock.patch('io.FileIO'), \ + mock.patch('io.TextIOWrapper') as textio, \ + mock.patch('termios.tcgetattr'), \ + mock.patch('termios.tcsetattr'), \ + mock.patch('getpass._raw_input') as mock_input: + os_open.return_value = 3 + mock_input.return_value = mock_result + + result = getpass.unix_getpass(echo_char='*') + mock_input.assert_called_once_with('Password: ', textio(), + input=textio(), echo_char='*') + self.assertEqual(result, mock_result) + + def test_raw_input_with_echo_char(self): + passwd = 'my1pa$$word!' + mock_input = StringIO(f'{passwd}\n') + mock_output = StringIO() + with mock.patch('sys.stdin', mock_input), \ + mock.patch('sys.stdout', mock_output): + result = getpass._raw_input('Password: ', mock_output, mock_input, + '*') + self.assertEqual(result, passwd) + self.assertEqual('Password: ************', mock_output.getvalue()) + + def test_control_chars_with_echo_char(self): + passwd = 'pass\twd\b' + expect_result = 'pass\tw' + mock_input = StringIO(f'{passwd}\n') + mock_output = StringIO() + with mock.patch('sys.stdin', mock_input), \ + mock.patch('sys.stdout', mock_output): + result = getpass._raw_input('Password: ', mock_output, mock_input, + '*') + self.assertEqual(result, expect_result) + self.assertEqual('Password: *******\x08 \x08', mock_output.getvalue()) + + +class GetpassEchoCharTest(unittest.TestCase): + + def test_accept_none(self): + getpass._check_echo_char(None) + + @support.subTests('echo_char', ["*", "A", " "]) + def test_accept_single_printable_ascii(self, echo_char): + getpass._check_echo_char(echo_char) + + def test_reject_empty_string(self): + self.assertRaises(ValueError, getpass.getpass, echo_char="") + + @support.subTests('echo_char', ["***", "AA", "aA*!"]) + def test_reject_multi_character_strings(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + '\N{LATIN CAPITAL LETTER AE}', # non-ASCII single character + '\N{HEAVY BLACK HEART}', # non-ASCII multibyte character + ]) + def test_reject_non_ascii(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + ch for ch in map(chr, range(0, 128)) + if not ch.isprintable() + ]) + def test_reject_non_printable_characters(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + # TypeError Rejection + @support.subTests('echo_char', [b"*", 0, 0.0, [], {}]) + def test_reject_non_string(self, echo_char): + self.assertRaises(TypeError, getpass.getpass, echo_char=echo_char) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 0653bb762a9..f4069082969 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -6,13 +6,12 @@ from functools import partial from test import support -from test.support import os_helper +from test.support import cpython_only, os_helper +from test.support.import_helper import ensure_lazy_imports # TODO: # - Add new tests, for example for "dgettext" -# - Remove dummy tests, for example testing for single and double quotes -# has no sense, it would have if we were testing a parser (i.e. pygettext) # - Tests should have only one assert. GNU_MO_DATA = b'''\ @@ -231,30 +230,6 @@ def test_some_translations_with_context(self): eq(pgettext('my other context', 'nudge nudge'), 'wink wink (in "my other context")') - def test_double_quotes(self): - eq = self.assertEqual - # double quotes - eq(_("albatross"), 'albatross') - eq(_("mullusk"), 'bacon') - eq(_(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove') - eq(_(r"nudge nudge"), 'wink wink') - - def test_triple_single_quotes(self): - eq = self.assertEqual - # triple single quotes - eq(_('''albatross'''), 'albatross') - eq(_('''mullusk'''), 'bacon') - eq(_(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove') - eq(_(r'''nudge nudge'''), 'wink wink') - - def test_triple_double_quotes(self): - eq = self.assertEqual - # triple double quotes - eq(_("""albatross"""), 'albatross') - eq(_("""mullusk"""), 'bacon') - eq(_(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove') - eq(_(r"""nudge nudge"""), 'wink wink') - def test_multiline_strings(self): eq = self.assertEqual # multiline strings @@ -367,30 +342,6 @@ def test_some_translations_with_context_and_domain(self): eq(gettext.dpgettext('gettext', 'my other context', 'nudge nudge'), 'wink wink (in "my other context")') - def test_double_quotes(self): - eq = self.assertEqual - # double quotes - eq(self._("albatross"), 'albatross') - eq(self._("mullusk"), 'bacon') - eq(self._(r"Raymond Luxury Yach-t"), 'Throatwobbler Mangrove') - eq(self._(r"nudge nudge"), 'wink wink') - - def test_triple_single_quotes(self): - eq = self.assertEqual - # triple single quotes - eq(self._('''albatross'''), 'albatross') - eq(self._('''mullusk'''), 'bacon') - eq(self._(r'''Raymond Luxury Yach-t'''), 'Throatwobbler Mangrove') - eq(self._(r'''nudge nudge'''), 'wink wink') - - def test_triple_double_quotes(self): - eq = self.assertEqual - # triple double quotes - eq(self._("""albatross"""), 'albatross') - eq(self._("""mullusk"""), 'bacon') - eq(self._(r"""Raymond Luxury Yach-t"""), 'Throatwobbler Mangrove') - eq(self._(r"""nudge nudge"""), 'wink wink') - def test_multiline_strings(self): eq = self.assertEqual # multiline strings @@ -434,8 +385,7 @@ def _test_plural_forms(self, ngettext, gettext, x = ngettext(singular, plural, None) self.assertEqual(x, tplural) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms(self): self._test_plural_forms( self.ngettext, self.gettext, @@ -446,8 +396,7 @@ def test_plural_forms(self): '%d file deleted', '%d files deleted', '%d file deleted', '%d files deleted') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms(self): ngettext = partial(self.npgettext, 'With context') gettext = partial(self.pgettext, 'With context') @@ -460,8 +409,7 @@ def test_plural_context_forms(self): '%d file deleted', '%d files deleted', '%d file deleted', '%d files deleted') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_wrong_context_forms(self): self._test_plural_forms( partial(self.npgettext, 'Unknown context'), @@ -494,8 +442,7 @@ def setUp(self): self.pgettext = partial(gettext.dpgettext, 'gettext') self.npgettext = partial(gettext.dnpgettext, 'gettext') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms_wrong_domain(self): self._test_plural_forms( partial(gettext.dngettext, 'unknown'), @@ -504,8 +451,7 @@ def test_plural_forms_wrong_domain(self): 'There is %s file', 'There are %s files', numbers_only=False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms_wrong_domain(self): self._test_plural_forms( partial(gettext.dnpgettext, 'unknown', 'With context'), @@ -526,8 +472,7 @@ def setUp(self): self.pgettext = t.pgettext self.npgettext = t.npgettext - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_forms_null_translations(self): t = gettext.NullTranslations() self._test_plural_forms( @@ -536,8 +481,7 @@ def test_plural_forms_null_translations(self): 'There is %s file', 'There are %s files', numbers_only=False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_plural_context_forms_null_translations(self): t = gettext.NullTranslations() self._test_plural_forms( @@ -630,6 +574,7 @@ def test_ar(self): s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "01233333333444444444444444444444444444444444444444444444444444444444444444444444444444444444444444445553333333344444444444444444444444444444444444444444444444444444444444444444444444444444444444444444") + @support.skip_wasi_stack_overflow() def test_security(self): raises = self.assertRaises # Test for a dangerous expression @@ -994,6 +939,17 @@ def test__all__(self): support.check__all__(self, gettext, not_exported={'c2py', 'ENOENT'}) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) + + +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index c3fb8939a69..d0ed5129253 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -459,111 +459,59 @@ def test_translate_matching(self): def test_translate(self): def fn(pat): return glob.translate(pat, seps='/') - self.assertEqual(fn('foo'), r'(?s:foo)\Z') - self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') - self.assertEqual(fn('*'), r'(?s:[^/.][^/]*)\Z') - self.assertEqual(fn('?'), r'(?s:(?!\.)[^/])\Z') - self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') - self.assertEqual(fn('*a'), r'(?s:(?!\.)[^/]*a)\Z') - self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') - self.assertEqual(fn('?aa'), r'(?s:(?!\.)[^/]aa)\Z') - self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') - self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') - self.assertEqual(fn('**'), r'(?s:(?!\.)[^/]*)\Z') - self.assertEqual(fn('***'), r'(?s:(?!\.)[^/]*)\Z') - self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') - self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\Z') + self.assertEqual(fn('foo'), r'(?s:foo)\z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\z') + self.assertEqual(fn('*'), r'(?s:[^/.][^/]*)\z') + self.assertEqual(fn('?'), r'(?s:(?!\.)[^/])\z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\z') + self.assertEqual(fn('*a'), r'(?s:(?!\.)[^/]*a)\z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\z') + self.assertEqual(fn('?aa'), r'(?s:(?!\.)[^/]aa)\z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\z') + self.assertEqual(fn('**'), r'(?s:(?!\.)[^/]*)\z') + self.assertEqual(fn('***'), r'(?s:(?!\.)[^/]*)\z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z') + self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\z') self.assertEqual(fn('/**/*/*.*/**'), - r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\Z') + r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\z') def test_translate_include_hidden(self): def fn(pat): return glob.translate(pat, include_hidden=True, seps='/') - self.assertEqual(fn('foo'), r'(?s:foo)\Z') - self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') - self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') - self.assertEqual(fn('?'), r'(?s:[^/])\Z') - self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') - self.assertEqual(fn('*a'), r'(?s:[^/]*a)\Z') - self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') - self.assertEqual(fn('?aa'), r'(?s:[^/]aa)\Z') - self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') - self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') - self.assertEqual(fn('**'), r'(?s:[^/]*)\Z') - self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') - self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') - self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') - self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\Z') + self.assertEqual(fn('foo'), r'(?s:foo)\z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\z') + self.assertEqual(fn('*'), r'(?s:[^/]+)\z') + self.assertEqual(fn('?'), r'(?s:[^/])\z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\z') + self.assertEqual(fn('*a'), r'(?s:[^/]*a)\z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\z') + self.assertEqual(fn('?aa'), r'(?s:[^/]aa)\z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\z') + self.assertEqual(fn('**'), r'(?s:[^/]*)\z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\z') def test_translate_recursive(self): def fn(pat): return glob.translate(pat, recursive=True, include_hidden=True, seps='/') - self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') - self.assertEqual(fn('?'), r'(?s:[^/])\Z') - self.assertEqual(fn('**'), r'(?s:.*)\Z') - self.assertEqual(fn('**/**'), r'(?s:.*)\Z') - self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') - self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') - self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') - self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') + self.assertEqual(fn('*'), r'(?s:[^/]+)\z') + self.assertEqual(fn('?'), r'(?s:[^/])\z') + self.assertEqual(fn('**'), r'(?s:.*)\z') + self.assertEqual(fn('**/**'), r'(?s:.*)\z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\z') def test_translate_seps(self): def fn(pat): return glob.translate(pat, recursive=True, include_hidden=True, seps=['/', '\\']) - self.assertEqual(fn('foo/bar\\baz'), r'(?s:foo[/\\]bar[/\\]baz)\Z') - self.assertEqual(fn('**/*'), r'(?s:(?:.+[/\\])?[^/\\]+)\Z') - - -@skip_unless_symlink -class SymlinkLoopGlobTests(unittest.TestCase): - - # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False - # randomly when checking the "link/" symbolic link. - # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 - @unittest.skip("flaky test") - def test_selflink(self): - tempdir = TESTFN + "_dir" - os.makedirs(tempdir) - self.addCleanup(shutil.rmtree, tempdir) - with change_cwd(tempdir): - os.makedirs('dir') - create_empty_file(os.path.join('dir', 'file')) - os.symlink(os.curdir, os.path.join('dir', 'link')) - - results = glob.glob('**', recursive=True) - self.assertEqual(len(results), len(set(results))) - results = set(results) - depth = 0 - while results: - path = os.path.join(*(['dir'] + ['link'] * depth)) - self.assertIn(path, results) - results.remove(path) - if not results: - break - path = os.path.join(path, 'file') - self.assertIn(path, results) - results.remove(path) - depth += 1 - - results = glob.glob(os.path.join('**', 'file'), recursive=True) - self.assertEqual(len(results), len(set(results))) - results = set(results) - depth = 0 - while results: - path = os.path.join(*(['dir'] + ['link'] * depth + ['file'])) - self.assertIn(path, results) - results.remove(path) - depth += 1 - - results = glob.glob(os.path.join('**', ''), recursive=True) - self.assertEqual(len(results), len(set(results))) - results = set(results) - depth = 0 - while results: - path = os.path.join(*(['dir'] + ['link'] * depth + [''])) - self.assertIn(path, results) - results.remove(path) - depth += 1 + self.assertEqual(fn('foo/bar\\baz'), r'(?s:foo[/\\]bar[/\\]baz)\z') + self.assertEqual(fn('**/*'), r'(?s:(?:.+[/\\])?[^/\\]+)\z') if __name__ == "__main__": diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 323f4ee4c61..77bd5a163ce 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -221,7 +221,6 @@ def test_ellipsis(self): self.assertTrue(x is Ellipsis) self.assertRaises(SyntaxError, eval, ".. .") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_eof_error(self): samples = ("def foo(", "\ndef foo(", "def foo(\n") for s in samples: @@ -374,6 +373,7 @@ class F(C, A): self.assertEqual(F.__annotations__, {}) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod @@ -403,6 +403,7 @@ def test_var_annot_in_module(self): with self.assertRaises(NameError): ann_module3.D_bad_ann(5) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_simple_exec(self): gns = {}; lns= {} exec("'docstring'\n" diff --git a/Lib/test/test_graphlib.py b/Lib/test/test_graphlib.py index 5f38af4024c..66722e0b049 100644 --- a/Lib/test/test_graphlib.py +++ b/Lib/test/test_graphlib.py @@ -140,9 +140,21 @@ def test_calls_before_prepare(self): def test_prepare_multiple_times(self): ts = graphlib.TopologicalSorter() ts.prepare() - with self.assertRaisesRegex(ValueError, r"cannot prepare\(\) more than once"): + ts.prepare() + + def test_prepare_after_pass_out(self): + ts = graphlib.TopologicalSorter({'a': 'bc'}) + ts.prepare() + self.assertEqual(set(ts.get_ready()), {'b', 'c'}) + with self.assertRaisesRegex(ValueError, r"cannot prepare\(\) after starting sort"): ts.prepare() + def test_prepare_cycleerror_each_time(self): + ts = graphlib.TopologicalSorter({'a': 'b', 'b': 'a'}) + for attempt in range(1, 4): + with self.assertRaises(graphlib.CycleError, msg=f"{attempt=}"): + ts.prepare() + def test_invalid_nodes_in_done(self): ts = graphlib.TopologicalSorter() ts.add(1, 2, 3, 4) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b0d9613cdbd..ccbacc7c19b 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -9,6 +9,7 @@ import struct import sys import unittest +import warnings from subprocess import PIPE, Popen from test.support import catch_unraisable_exception from test.support import import_helper @@ -143,6 +144,38 @@ def test_read1(self): self.assertEqual(f.tell(), nread) self.assertEqual(b''.join(blocks), data1 * 50) + def test_readinto(self): + # 10MB of uncompressible data to ensure multiple reads + large_data = os.urandom(10 * 2**20) + with gzip.GzipFile(self.filename, 'wb') as f: + f.write(large_data) + + buf = bytearray(len(large_data)) + with gzip.GzipFile(self.filename, 'r') as f: + nbytes = f.readinto(buf) + self.assertEqual(nbytes, len(large_data)) + self.assertEqual(buf, large_data) + + def test_readinto1(self): + # 10MB of uncompressible data to ensure multiple reads + large_data = os.urandom(10 * 2**20) + with gzip.GzipFile(self.filename, 'wb') as f: + f.write(large_data) + + nread = 0 + buf = bytearray(len(large_data)) + memview = memoryview(buf) # Simplifies slicing + with gzip.GzipFile(self.filename, 'r') as f: + for count in range(200): + nbytes = f.readinto1(memview[nread:]) + if not nbytes: + break + nread += nbytes + self.assertEqual(f.tell(), nread) + self.assertEqual(buf, large_data) + # readinto1() should require multiple loops + self.assertGreater(count, 1) + @bigmemtest(size=_4G, memuse=1) def test_read_large(self, size): # Read chunk size over UINT_MAX should be supported, despite zlib's @@ -298,13 +331,13 @@ def test_mode(self): def test_1647484(self): for mode in ('wb', 'rb'): with gzip.GzipFile(self.filename, mode) as f: - self.assertTrue(hasattr(f, "name")) + self.assertHasAttr(f, "name") self.assertEqual(f.name, self.filename) def test_paddedfile_getattr(self): self.test_write() with gzip.GzipFile(self.filename, 'rb') as f: - self.assertTrue(hasattr(f.fileobj, "name")) + self.assertHasAttr(f.fileobj, "name") self.assertEqual(f.fileobj.name, self.filename) def test_mtime(self): @@ -312,7 +345,7 @@ def test_mtime(self): with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite: fWrite.write(data1) with gzip.GzipFile(self.filename) as fRead: - self.assertTrue(hasattr(fRead, 'mtime')) + self.assertHasAttr(fRead, 'mtime') self.assertIsNone(fRead.mtime) dataRead = fRead.read() self.assertEqual(dataRead, data1) @@ -427,7 +460,7 @@ def test_zero_padded_file(self): self.assertEqual(d, data1 * 50, "Incorrect data in file") def test_gzip_BadGzipFile_exception(self): - self.assertTrue(issubclass(gzip.BadGzipFile, OSError)) + self.assertIsSubclass(gzip.BadGzipFile, OSError) def test_bad_gzip_file(self): with open(self.filename, 'wb') as file: @@ -715,6 +748,17 @@ def test_compress_mtime(self): f.read(1) # to set mtime attribute self.assertEqual(f.mtime, mtime) + def test_compress_mtime_default(self): + # test for gh-125260 + datac = gzip.compress(data1, mtime=0) + datac2 = gzip.compress(data1) + self.assertEqual(datac, datac2) + datac3 = gzip.compress(data1, mtime=None) + self.assertNotEqual(datac, datac3) + with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f: + f.read(1) # to set mtime attribute + self.assertGreater(f.mtime, 1) + def test_compress_correct_level(self): for mtime in (0, 42): with self.subTest(mtime=mtime): @@ -856,9 +900,10 @@ def test_refloop_unraisable(self): # fileobj would be closed before the GzipFile as the result of a # reference loop. See issue gh-129726 with catch_unraisable_exception() as cm: - gzip.GzipFile(fileobj=io.BytesIO(), mode="w") - gc.collect() - self.assertIsNone(cm.unraisable) + with self.assertWarns(ResourceWarning): + gzip.GzipFile(fileobj=io.BytesIO(), mode="w") + gc.collect() + self.assertIsNone(cm.unraisable) class TestOpen(BaseTest): @@ -991,8 +1036,6 @@ def test_encoding_error_handler(self): as f: self.assertEqual(f.read(), "foobar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newline(self): # Test with explicit newline (universal newline mode disabled). uncompressed = data1.decode("ascii") * 50 diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index cd9cbe9bff7..5c38813f3f5 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -1,6 +1,4 @@ -# Test hashlib module -# -# $Id$ +# Test the hashlib module. # # Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. @@ -12,50 +10,46 @@ import importlib import io import itertools +import logging import os +import re import sys import sysconfig +import tempfile import threading import unittest import warnings from test import support from test.support import _4G, bigmemtest +from test.support import hashlib_helper from test.support.import_helper import import_fresh_module -from test.support import os_helper +from test.support import requires_resource from test.support import threading_helper -from test.support import warnings_helper from http.client import HTTPException -# Were we compiled --with-pydebug or with #define Py_DEBUG? -COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') -# default builtin hash module -default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} +default_builtin_hashes = {'md5', 'sha1', 'sha2', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") if builtin_hashes is None: builtin_hashes = default_builtin_hashes else: - builtin_hashes = { - m.strip() for m in builtin_hashes.strip('"').lower().split(",") - } + builtin_hash_names = builtin_hashes.strip('"').lower().split(",") + builtin_hashes = set(map(str.strip, builtin_hash_names)) -# hashlib with and without OpenSSL backend for PBKDF2 -# only import builtin_hashlib when all builtin hashes are available. -# Otherwise import prints noise on stderr +# Public 'hashlib' module with OpenSSL backend for PBKDF2. openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) -if builtin_hashes == default_builtin_hashes: - builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -else: - builtin_hashlib = None try: - from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode + import _hashlib except ImportError: - HASH = None - HASHXOF = None - openssl_md_meth_names = frozenset() - + _hashlib = None +# The extension module may exist but only define some of these. gh-141907 +HASH = getattr(_hashlib, 'HASH', None) +HASHXOF = getattr(_hashlib, 'HASHXOF', None) +openssl_md_meth_names = getattr(_hashlib, 'openssl_md_meth_names', frozenset()) +get_fips_mode = getattr(_hashlib, 'get_fips_mode', None) +if not get_fips_mode: def get_fips_mode(): return 0 @@ -66,9 +60,12 @@ def get_fips_mode(): requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2') -# bpo-46913: Don't test the _sha3 extension on a Python UBSAN build -SKIP_SHA3 = support.check_sanitizer(ub=True) -requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3') +try: + import _sha3 +except ImportError: + _sha3 = None + +requires_sha3 = unittest.skipUnless(_sha3, 'requires _sha3') def hexstr(s): @@ -108,8 +105,8 @@ class HashLibTestCase(unittest.TestCase): shakes = {'shake_128', 'shake_256'} - # Issue #14693: fallback modules are always compiled under POSIX - _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG + # gh-58898: Fallback modules are always compiled under POSIX. + _warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG) def _conditional_import_module(self, module_name): """Import a module and return a reference to it or None on failure.""" @@ -117,7 +114,11 @@ def _conditional_import_module(self, module_name): return importlib.import_module(module_name) except ModuleNotFoundError as error: if self._warn_on_extension_import and module_name in builtin_hashes: - warnings.warn('Did a C extension fail to compile? %s' % error) + logging.getLogger(__name__).warning( + 'Did a C extension fail to compile? %s', + error, + exc_info=error, + ) return None def __init__(self, *args, **kwargs): @@ -131,27 +132,24 @@ def __init__(self, *args, **kwargs): self.constructors_to_test = {} for algorithm in algorithms: - if SKIP_SHA3 and algorithm.startswith('sha3_'): - continue self.constructors_to_test[algorithm] = set() # For each algorithm, test the direct constructor and the use # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): constructors.add(getattr(hashlib, algorithm)) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): - if data is None: - return hashlib.new(_alg, **kwargs) - return hashlib.new(_alg, data, **kwargs) - constructors.add(_test_algorithm_via_hashlib_new) + def c(*args, __algorithm_name=algorithm, **kwargs): + return hashlib.new(__algorithm_name, *args, **kwargs) + c.__name__ = f'do_test_algorithm_via_hashlib_new_{algorithm}' + constructors.add(c) _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib if _hashlib: - # These two algorithms should always be present when this module + # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. - self.assertTrue(hasattr(_hashlib, 'openssl_md5')) - self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) + self.assertHasAttr(_hashlib, 'openssl_md5') + self.assertHasAttr(_hashlib, 'openssl_sha1') for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: @@ -173,28 +171,24 @@ def add_builtin_constructor(name): _sha1 = self._conditional_import_module('_sha1') if _sha1: add_builtin_constructor('sha1') - _sha256 = self._conditional_import_module('_sha256') - if _sha256: + _sha2 = self._conditional_import_module('_sha2') + if _sha2: add_builtin_constructor('sha224') add_builtin_constructor('sha256') - _sha512 = self._conditional_import_module('_sha512') - if _sha512: add_builtin_constructor('sha384') add_builtin_constructor('sha512') + _sha3 = self._conditional_import_module('_sha3') + if _sha3: + add_builtin_constructor('sha3_224') + add_builtin_constructor('sha3_256') + add_builtin_constructor('sha3_384') + add_builtin_constructor('sha3_512') + add_builtin_constructor('shake_128') + add_builtin_constructor('shake_256') if _blake2: add_builtin_constructor('blake2s') add_builtin_constructor('blake2b') - if not SKIP_SHA3: - _sha3 = self._conditional_import_module('_sha3') - if _sha3: - add_builtin_constructor('sha3_224') - add_builtin_constructor('sha3_256') - add_builtin_constructor('sha3_384') - add_builtin_constructor('sha3_512') - add_builtin_constructor('shake_128') - add_builtin_constructor('shake_256') - super(HashLibTestCase, self).__init__(*args, **kwargs) @property @@ -252,6 +246,80 @@ def test_usedforsecurity_false(self): self._hashlib.new("md5", usedforsecurity=False) self._hashlib.openssl_md5(usedforsecurity=False) + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature(self): + for constructor in self.hash_constructors: + with self.subTest(constructor.__name__): + constructor(b'') + constructor(data=b'') + constructor(string=b'') # should be deprecated in the future + + digest_name = constructor(b'').name + with self.subTest(digest_name): + hashlib.new(digest_name, b'') + hashlib.new(digest_name, data=b'') + hashlib.new(digest_name, string=b'') + # Make sure that _hashlib contains the constructor + # to test when using a combination of libcrypto and + # interned hash implementations. + if self._hashlib and digest_name in self._hashlib._constructors: + self._hashlib.new(digest_name, b'') + self._hashlib.new(digest_name, data=b'') + self._hashlib.new(digest_name, string=b'') + + @unittest.expectedFailure # TODO: RUSTPYTHON; duplicate positional/keyword arg error message differs + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature_errors(self): + nomsg = b'' + mymsg = b'msg' + conflicting_call = re.escape( + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version." + ) + duplicated_param = re.escape("given by name ('data') and position") + unexpected_param = re.escape("got an unexpected keyword argument '_'") + for args, kwds, errmsg in [ + # Reject duplicated arguments before unknown keyword arguments. + ((nomsg,), dict(data=nomsg, _=nomsg), duplicated_param), + ((mymsg,), dict(data=nomsg, _=nomsg), duplicated_param), + # Reject duplicated arguments before conflicting ones. + *itertools.product( + [[nomsg], [mymsg]], + [dict(data=nomsg), dict(data=nomsg, string=nomsg)], + [duplicated_param] + ), + # Reject unknown keyword arguments before conflicting ones. + *itertools.product( + [()], + [ + dict(_=None), + dict(data=nomsg, _=None), + dict(string=nomsg, _=None), + dict(string=nomsg, data=nomsg, _=None), + ], + [unexpected_param] + ), + ((nomsg,), dict(_=None), unexpected_param), + ((mymsg,), dict(_=None), unexpected_param), + # Reject conflicting arguments. + [(nomsg,), dict(string=nomsg), conflicting_call], + [(mymsg,), dict(string=nomsg), conflicting_call], + [(), dict(data=nomsg, string=nomsg), conflicting_call], + ]: + for constructor in self.hash_constructors: + digest_name = constructor(b'').name + with self.subTest(constructor.__name__, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + constructor(*args, **kwds) + with self.subTest(digest_name, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + hashlib.new(digest_name, *args, **kwds) + if (self._hashlib and + digest_name in self._hashlib._constructors): + with self.assertRaisesRegex(TypeError, errmsg): + self._hashlib.new(digest_name, *args, **kwds) + def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') self.assertRaises(TypeError, hashlib.new, 1) @@ -259,6 +327,7 @@ def test_unknown_hash(self): def test_new_upper_to_lower(self): self.assertEqual(hashlib.new("SHA256").name, "sha256") + @support.thread_unsafe("modifies sys.modules") def test_get_builtin_constructor(self): get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') @@ -295,8 +364,6 @@ def test_hexdigest(self): self.assertIsInstance(h.digest(), bytes) self.assertEqual(hexstr(h.digest()), h.hexdigest()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_digest_length_overflow(self): # See issue #34922 large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) @@ -358,6 +425,35 @@ def test_large_update(self): self.assertEqual(m1.digest(*args), m4_copy.digest(*args)) self.assertEqual(m4.digest(*args), m4_digest) + @requires_resource('cpu') + def test_sha256_update_over_4gb(self): + zero_1mb = b"\0" * 1024 * 1024 + h = hashlib.sha256() + for i in range(0, 4096): + h.update(zero_1mb) + h.update(b"hello world") + self.assertEqual(h.hexdigest(), "a5364f7a52ebe2e25f1838a4ca715a893b6fd7a23f2a0d9e9762120da8b1bf53") + + @requires_resource('cpu') + def test_sha3_256_update_over_4gb(self): + zero_1mb = b"\0" * 1024 * 1024 + h = hashlib.sha3_256() + for i in range(0, 4096): + h.update(zero_1mb) + h.update(b"hello world") + self.assertEqual(h.hexdigest(), "e2d4535e3b613135c14f2fe4e026d7ad8d569db44901740beffa30d430acb038") + + @requires_resource('cpu') + def test_blake2_update_over_4gb(self): + # blake2s or blake2b doesn't matter based on how our C code is structured, this tests the + # common loop macro logic. + zero_1mb = b"\0" * 1024 * 1024 + h = hashlib.blake2s() + for i in range(0, 4096): + h.update(zero_1mb) + h.update(b"hello world") + self.assertEqual(h.hexdigest(), "8a268e83dd30528bc0907fa2008c91de8f090a0b6e0e60a5ff0d999d8485526f") + def check(self, name, data, hexdigest, shake=False, **kwargs): length = len(hexdigest)//2 hexdigest = hexdigest.lower() @@ -393,21 +489,18 @@ def check_file_digest(self, name, data, hexdigest): digests = [name] digests.extend(self.constructors_to_test[name]) - with open(os_helper.TESTFN, "wb") as f: + with tempfile.TemporaryFile() as f: f.write(data) - try: for digest in digests: buf = io.BytesIO(data) buf.seek(0) self.assertEqual( hashlib.file_digest(buf, digest).hexdigest(), hexdigest ) - with open(os_helper.TESTFN, "rb") as f: - digestobj = hashlib.file_digest(f, digest) + f.seek(0) + digestobj = hashlib.file_digest(f, digest) self.assertEqual(digestobj.hexdigest(), hexdigest) - finally: - os.unlink(os_helper.TESTFN) def check_no_unicode(self, algorithm_name): # Unicode objects are not allowed as input. @@ -454,9 +547,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, self.assertEqual(len(m.hexdigest()), 2*digest_size) self.assertEqual(m.name, name) # split for sha3_512 / _sha3.sha3 object - self.assertIn(name.split("_")[0], repr(m)) + self.assertIn(name.split("_")[0], repr(m).lower()) - def test_blocksize_name(self): + def test_blocksize_and_name(self): self.check_blocksize_name('md5', 64, 16) self.check_blocksize_name('sha1', 64, 20) self.check_blocksize_name('sha224', 64, 28) @@ -464,8 +557,6 @@ def test_blocksize_name(self): self.check_blocksize_name('sha384', 128, 48) self.check_blocksize_name('sha512', 128, 64) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_sha3 def test_blocksize_name_sha3(self): self.check_blocksize_name('sha3_224', 144, 28) @@ -479,16 +570,19 @@ def check_sha3(self, name, capacity, rate, suffix): constructors = self.constructors_to_test[name] for hash_object_constructor in constructors: m = hash_object_constructor() - if HASH is not None and isinstance(m, HASH): - # _hashopenssl's variant does not have extra SHA3 attributes - continue + if name.startswith('shake_'): + if HASHXOF is not None and isinstance(m, HASHXOF): + # _hashopenssl's variant does not have extra SHA3 attributes + continue + else: + if HASH is not None and isinstance(m, HASH): + # _hashopenssl's variant does not have extra SHA3 attributes + continue self.assertEqual(capacity + rate, 1600) self.assertEqual(m._capacity_bits, capacity) self.assertEqual(m._rate_bits, rate) self.assertEqual(m._suffix, suffix) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_sha3 def test_extra_sha3(self): self.check_sha3('sha3_224', 448, 1152, b'\x06') @@ -700,8 +794,6 @@ def check_blake2(self, constructor, salt_size, person_size, key_size, self.assertRaises(ValueError, constructor, node_offset=-1) self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) - self.assertRaises(TypeError, constructor, data=b'') - self.assertRaises(TypeError, constructor, string=b'') self.assertRaises(TypeError, constructor, '') constructor( @@ -741,8 +833,7 @@ def selftest_seq(length, seed): outer.update(keyed.digest()) return outer.hexdigest() - # TODO: RUSTPYTHON add to constructor const value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; add to constructor const value @requires_blake2 def test_blake2b(self): self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1) @@ -764,8 +855,7 @@ def test_case_blake2b_1(self): "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+ "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923") - # TODO: RUSTPYTHON implement all blake2 fields - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; implement all blake2 fields @requires_blake2 def test_case_blake2b_all_parameters(self): # This checks that all the parameters work in general, and also that @@ -784,15 +874,14 @@ def test_case_blake2b_all_parameters(self): inner_size=7, last_node=True) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; blake2 key parameter not supported @requires_blake2 def test_blake2b_vectors(self): for msg, key, md in read_vectors('blake2b'): key = bytes.fromhex(key) self.check('blake2b', msg, md, key=key) - # TODO: RUSTPYTHON add to constructor const value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; add to constructor const value @requires_blake2 def test_blake2s(self): self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1) @@ -812,8 +901,7 @@ def test_case_blake2s_1(self): self.check('blake2s', b"abc", "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982") - # TODO: RUSTPYTHON implement all blake2 fields - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; implement all blake2 fields @requires_blake2 def test_case_blake2s_all_parameters(self): # This checks that all the parameters work in general, and also that @@ -832,7 +920,7 @@ def test_case_blake2s_all_parameters(self): inner_size=7, last_node=True) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; blake2 key parameter not supported @requires_blake2 def test_blake2s_vectors(self): for msg, key, md in read_vectors('blake2s'): @@ -903,10 +991,13 @@ def test_case_shake256_vector(self): def test_gil(self): # Check things work fine with an input larger than the size required - # for multithreaded operation (which is hardwired to 2048). - gil_minsize = 2048 - + # for multithreaded operation. Currently, all cryptographic modules + # have the same constant value (2048) but in the future it might not + # be the case. + mods = ['_md5', '_sha1', '_sha2', '_sha3', '_blake2', '_hashlib'] + gil_minsize = hashlib_helper.find_gil_minsize(mods) for cons in self.hash_constructors: + # constructors belong to one of the above modules m = cons(usedforsecurity=False) m.update(b'1') m.update(b'#' * gil_minsize) @@ -915,6 +1006,8 @@ def test_gil(self): m = cons(b'x' * gil_minsize, usedforsecurity=False) m.update(b'1') + def test_sha256_gil(self): + gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib']) m = hashlib.sha256() m.update(b'1') m.update(b'#' * gil_minsize) @@ -992,10 +1085,9 @@ def test_disallow_instantiation(self): def test_hash_disallow_instantiation(self): # internal types like _hashlib.HASH are not constructable support.check_disallow_instantiation(self, HASH) - support.check_disallow_instantiation(self, HASHXOF) + if HASHXOF is not None: + support.check_disallow_instantiation(self, HASHXOF) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_readonly_types(self): for algorithm, constructors in self.constructors_to_test.items(): # all other types have DISALLOW_INSTANTIATION @@ -1116,15 +1208,7 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): iterations=1, dklen=None) self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) - @unittest.skipIf(builtin_hashlib is None, "test requires builtin_hashlib") - def test_pbkdf2_hmac_py(self): - with warnings_helper.check_warnings(): - self._test_pbkdf2_hmac( - builtin_hashlib.pbkdf2_hmac, builtin_hashes - ) - - @unittest.skipUnless(hasattr(openssl_hashlib, 'pbkdf2_hmac'), - ' test requires OpenSSL > 1.0') + @unittest.skipIf(openssl_hashlib is None, "requires OpenSSL bindings") def test_pbkdf2_hmac_c(self): self._test_pbkdf2_hmac(openssl_hashlib.pbkdf2_hmac, openssl_md_meth_names) @@ -1175,29 +1259,38 @@ def test_normalized_name(self): def test_file_digest(self): data = b'a' * 65536 d1 = hashlib.sha256() - self.addCleanup(os.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "wb") as f: + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: for _ in range(10): d1.update(data) - f.write(data) + fp.write(data) + fp.close() + + with open(fp.name, "rb") as f: + d2 = hashlib.file_digest(f, hashlib.sha256) - with open(os_helper.TESTFN, "rb") as f: - d2 = hashlib.file_digest(f, hashlib.sha256) + self.assertEqual(d1.hexdigest(), d2.hexdigest()) + self.assertEqual(d1.name, d2.name) + self.assertIs(type(d1), type(d2)) - self.assertEqual(d1.hexdigest(), d2.hexdigest()) - self.assertEqual(d1.name, d2.name) - self.assertIs(type(d1), type(d2)) + with self.assertRaises(ValueError): + with open(fp.name, "r") as f: + hashlib.file_digest(f, "sha256") + + with self.assertRaises(ValueError): + with open(fp.name, "wb") as f: + hashlib.file_digest(f, "sha256") with self.assertRaises(ValueError): hashlib.file_digest(None, "sha256") - with self.assertRaises(ValueError): - with open(os_helper.TESTFN, "r") as f: - hashlib.file_digest(f, "sha256") + class NonBlocking: + def readinto(self, buf): + return None + def readable(self): + return True - with self.assertRaises(ValueError): - with open(os_helper.TESTFN, "wb") as f: - hashlib.file_digest(f, "sha256") + with self.assertRaises(BlockingIOError): + hashlib.file_digest(NonBlocking(), hashlib.sha256) if __name__ == "__main__": diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index 1aa8e4e2897..d6623fee9bb 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -13,8 +13,9 @@ # _heapq.nlargest/nsmallest are saved in heapq._nlargest/_smallest when # _heapq is imported, so check them there -func_names = ['heapify', 'heappop', 'heappush', 'heappushpop', 'heapreplace', - '_heappop_max', '_heapreplace_max', '_heapify_max'] +func_names = ['heapify', 'heappop', 'heappush', 'heappushpop', 'heapreplace'] +# Add max-heap variants +func_names += [func + '_max' for func in func_names] class TestModules(TestCase): def test_py_functions(self): @@ -24,7 +25,7 @@ def test_py_functions(self): @skipUnless(c_heapq, 'requires _heapq') def test_c_functions(self): for fname in func_names: - self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq') + self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq', fname) def load_tests(loader, tests, ignore): @@ -74,6 +75,34 @@ def test_push_pop(self): except AttributeError: pass + def test_max_push_pop(self): + # 1) Push 256 random numbers and pop them off, verifying all's OK. + heap = [] + data = [] + self.check_max_invariant(heap) + for i in range(256): + item = random.random() + data.append(item) + self.module.heappush_max(heap, item) + self.check_max_invariant(heap) + results = [] + while heap: + item = self.module.heappop_max(heap) + self.check_max_invariant(heap) + results.append(item) + data_sorted = data[:] + data_sorted.sort(reverse=True) + + self.assertEqual(data_sorted, results) + # 2) Check that the invariant holds for a sorted array + self.check_max_invariant(results) + + self.assertRaises(TypeError, self.module.heappush_max, []) + + exc_types = (AttributeError, TypeError) + self.assertRaises(exc_types, self.module.heappush_max, None, None) + self.assertRaises(exc_types, self.module.heappop_max, None) + def check_invariant(self, heap): # Check the heap invariant. for pos, item in enumerate(heap): @@ -81,6 +110,11 @@ def check_invariant(self, heap): parentpos = (pos-1) >> 1 self.assertTrue(heap[parentpos] <= item) + def check_max_invariant(self, heap): + for pos, item in enumerate(heap[1:], start=1): + parentpos = (pos - 1) >> 1 + self.assertGreaterEqual(heap[parentpos], item) + def test_heapify(self): for size in list(range(30)) + [20000]: heap = [random.random() for dummy in range(size)] @@ -89,6 +123,14 @@ def test_heapify(self): self.assertRaises(TypeError, self.module.heapify, None) + def test_heapify_max(self): + for size in list(range(30)) + [20000]: + heap = [random.random() for dummy in range(size)] + self.module.heapify_max(heap) + self.check_max_invariant(heap) + + self.assertRaises(TypeError, self.module.heapify_max, None) + def test_naive_nbest(self): data = [random.randrange(2000) for i in range(1000)] heap = [] @@ -109,10 +151,7 @@ def heapiter(self, heap): def test_nbest(self): # Less-naive "N-best" algorithm, much faster (if len(data) is big - # enough ) than sorting all of data. However, if we had a max - # heap instead of a min heap, it could go faster still via - # heapify'ing all of data (linear time), then doing 10 heappops - # (10 log-time steps). + # enough ) than sorting all of data. data = [random.randrange(2000) for i in range(1000)] heap = data[:10] self.module.heapify(heap) @@ -125,6 +164,17 @@ def test_nbest(self): self.assertRaises(TypeError, self.module.heapreplace, None, None) self.assertRaises(IndexError, self.module.heapreplace, [], None) + def test_nbest_maxheap(self): + # With a max heap instead of a min heap, the "N-best" algorithm can + # go even faster still via heapify'ing all of data (linear time), then + # doing 10 heappops (10 log-time steps). + data = [random.randrange(2000) for i in range(1000)] + heap = data[:] + self.module.heapify_max(heap) + result = [self.module.heappop_max(heap) for _ in range(10)] + result.reverse() + self.assertEqual(result, sorted(data)[-10:]) + def test_nbest_with_pushpop(self): data = [random.randrange(2000) for i in range(1000)] heap = data[:10] @@ -134,6 +184,62 @@ def test_nbest_with_pushpop(self): self.assertEqual(list(self.heapiter(heap)), sorted(data)[-10:]) self.assertEqual(self.module.heappushpop([], 'x'), 'x') + def test_naive_nworst(self): + # Max-heap variant of "test_naive_nbest" + data = [random.randrange(2000) for i in range(1000)] + heap = [] + for item in data: + self.module.heappush_max(heap, item) + if len(heap) > 10: + self.module.heappop_max(heap) + heap.sort() + expected = sorted(data)[:10] + self.assertEqual(heap, expected) + + def heapiter_max(self, heap): + # An iterator returning a max-heap's elements, largest-first. + try: + while 1: + yield self.module.heappop_max(heap) + except IndexError: + pass + + def test_nworst(self): + # Max-heap variant of "test_nbest" + data = [random.randrange(2000) for i in range(1000)] + heap = data[:10] + self.module.heapify_max(heap) + for item in data[10:]: + if item < heap[0]: # this gets rarer the longer we run + self.module.heapreplace_max(heap, item) + expected = sorted(data, reverse=True)[-10:] + self.assertEqual(list(self.heapiter_max(heap)), expected) + + self.assertRaises(TypeError, self.module.heapreplace_max, None) + self.assertRaises(TypeError, self.module.heapreplace_max, None, None) + self.assertRaises(IndexError, self.module.heapreplace_max, [], None) + + def test_nworst_minheap(self): + # Min-heap variant of "test_nbest_maxheap" + data = [random.randrange(2000) for i in range(1000)] + heap = data[:] + self.module.heapify(heap) + result = [self.module.heappop(heap) for _ in range(10)] + result.reverse() + expected = sorted(data, reverse=True)[-10:] + self.assertEqual(result, expected) + + def test_nworst_with_pushpop(self): + # Max-heap variant of "test_nbest_with_pushpop" + data = [random.randrange(2000) for i in range(1000)] + heap = data[:10] + self.module.heapify_max(heap) + for item in data[10:]: + self.module.heappushpop_max(heap, item) + expected = sorted(data, reverse=True)[-10:] + self.assertEqual(list(self.heapiter_max(heap)), expected) + self.assertEqual(self.module.heappushpop_max([], 'x'), 'x') + def test_heappushpop(self): h = [] x = self.module.heappushpop(h, 10) @@ -153,12 +259,31 @@ def test_heappushpop(self): x = self.module.heappushpop(h, 11) self.assertEqual((h, x), ([11], 10)) + def test_heappushpop_max(self): + h = [] + x = self.module.heappushpop_max(h, 10) + self.assertTupleEqual((h, x), ([], 10)) + + h = [10] + x = self.module.heappushpop_max(h, 10.0) + self.assertTupleEqual((h, x), ([10], 10.0)) + self.assertIsInstance(h[0], int) + self.assertIsInstance(x, float) + + h = [10] + x = self.module.heappushpop_max(h, 11) + self.assertTupleEqual((h, x), ([10], 11)) + + h = [10] + x = self.module.heappushpop_max(h, 9) + self.assertTupleEqual((h, x), ([9], 10)) + def test_heappop_max(self): - # _heapop_max has an optimization for one-item lists which isn't + # heapop_max has an optimization for one-item lists which isn't # covered in other tests, so test that case explicitly here h = [3, 2] - self.assertEqual(self.module._heappop_max(h), 3) - self.assertEqual(self.module._heappop_max(h), 2) + self.assertEqual(self.module.heappop_max(h), 3) + self.assertEqual(self.module.heappop_max(h), 2) def test_heapsort(self): # Exercise everything with repeated heapsort checks @@ -175,6 +300,20 @@ def test_heapsort(self): heap_sorted = [self.module.heappop(heap) for i in range(size)] self.assertEqual(heap_sorted, sorted(data)) + def test_heapsort_max(self): + for trial in range(100): + size = random.randrange(50) + data = [random.randrange(25) for i in range(size)] + if trial & 1: # Half of the time, use heapify_max + heap = data[:] + self.module.heapify_max(heap) + else: # The rest of the time, use heappush_max + heap = [] + for item in data: + self.module.heappush_max(heap, item) + heap_sorted = [self.module.heappop_max(heap) for i in range(size)] + self.assertEqual(heap_sorted, sorted(data, reverse=True)) + def test_merge(self): inputs = [] for i in range(random.randrange(25)): @@ -377,16 +516,20 @@ def __lt__(self, other): class TestErrorHandling: def test_non_sequence(self): - for f in (self.module.heapify, self.module.heappop): + for f in (self.module.heapify, self.module.heappop, + self.module.heapify_max, self.module.heappop_max): self.assertRaises((TypeError, AttributeError), f, 10) for f in (self.module.heappush, self.module.heapreplace, + self.module.heappush_max, self.module.heapreplace_max, self.module.nlargest, self.module.nsmallest): self.assertRaises((TypeError, AttributeError), f, 10, 10) def test_len_only(self): - for f in (self.module.heapify, self.module.heappop): + for f in (self.module.heapify, self.module.heappop, + self.module.heapify_max, self.module.heappop_max): self.assertRaises((TypeError, AttributeError), f, LenOnly()) - for f in (self.module.heappush, self.module.heapreplace): + for f in (self.module.heappush, self.module.heapreplace, + self.module.heappush_max, self.module.heapreplace_max): self.assertRaises((TypeError, AttributeError), f, LenOnly(), 10) for f in (self.module.nlargest, self.module.nsmallest): self.assertRaises(TypeError, f, 2, LenOnly()) @@ -395,7 +538,8 @@ def test_cmp_err(self): seq = [CmpErr(), CmpErr(), CmpErr()] for f in (self.module.heapify, self.module.heappop): self.assertRaises(ZeroDivisionError, f, seq) - for f in (self.module.heappush, self.module.heapreplace): + for f in (self.module.heappush, self.module.heapreplace, + self.module.heappush_max, self.module.heapreplace_max): self.assertRaises(ZeroDivisionError, f, seq, 10) for f in (self.module.nlargest, self.module.nsmallest): self.assertRaises(ZeroDivisionError, f, 2, seq) @@ -403,6 +547,8 @@ def test_cmp_err(self): def test_arg_parsing(self): for f in (self.module.heapify, self.module.heappop, self.module.heappush, self.module.heapreplace, + self.module.heapify_max, self.module.heappop_max, + self.module.heappush_max, self.module.heapreplace_max, self.module.nlargest, self.module.nsmallest): self.assertRaises((TypeError, AttributeError), f, 10) @@ -424,6 +570,10 @@ def test_heappush_mutating_heap(self): # Python version raises IndexError, C version RuntimeError with self.assertRaises((IndexError, RuntimeError)): self.module.heappush(heap, SideEffectLT(5, heap)) + heap = [] + heap.extend(SideEffectLT(i, heap) for i in range(200)) + with self.assertRaises((IndexError, RuntimeError)): + self.module.heappush_max(heap, SideEffectLT(5, heap)) def test_heappop_mutating_heap(self): heap = [] @@ -431,8 +581,12 @@ def test_heappop_mutating_heap(self): # Python version raises IndexError, C version RuntimeError with self.assertRaises((IndexError, RuntimeError)): self.module.heappop(heap) + heap = [] + heap.extend(SideEffectLT(i, heap) for i in range(200)) + with self.assertRaises((IndexError, RuntimeError)): + self.module.heappop_max(heap) - def test_comparison_operator_modifiying_heap(self): + def test_comparison_operator_modifying_heap(self): # See bpo-39421: Strong references need to be taken # when comparing objects as they can alter the heap class EvilClass(int): @@ -444,7 +598,7 @@ def __lt__(self, o): self.module.heappush(heap, EvilClass(0)) self.assertRaises(IndexError, self.module.heappushpop, heap, 1) - def test_comparison_operator_modifiying_heap_two_heaps(self): + def test_comparison_operator_modifying_heap_two_heaps(self): class h(int): def __lt__(self, o): @@ -464,6 +618,17 @@ def __lt__(self, o): self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1)) self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1)) + list1, list2 = [], [] + + self.module.heappush_max(list1, h(0)) + self.module.heappush_max(list2, g(0)) + self.module.heappush_max(list1, g(1)) + self.module.heappush_max(list2, h(1)) + + self.assertRaises((IndexError, RuntimeError), self.module.heappush_max, list1, g(1)) + self.assertRaises((IndexError, RuntimeError), self.module.heappush_max, list2, h(1)) + + class TestErrorHandlingPython(TestErrorHandling, TestCase): module = py_heapq diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 8e1a4a204c5..1506bb7982a 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -1,149 +1,429 @@ +"""Test suite for HMAC. + +Python provides three different implementations of HMAC: + +- OpenSSL HMAC using OpenSSL hash functions. +- HACL* HMAC using HACL* hash functions. +- Generic Python HMAC using user-defined hash functions. + +The generic Python HMAC implementation is able to use OpenSSL +callables or names, HACL* named hash functions or arbitrary +objects implementing PEP 247 interface. + +In the two first cases, Python HMAC wraps a C HMAC object (either OpenSSL +or HACL*-based). As a last resort, HMAC is re-implemented in pure Python. +It is however interesting to test the pure Python implementation against +the OpenSSL and HACL* hash functions. +""" + import binascii import functools import hmac import hashlib +import random +import types import unittest -import unittest.mock import warnings - -from test.support import hashlib_helper, check_disallow_instantiation - from _operator import _compare_digest as operator_compare_digest +from test.support import _4G, bigmemtest +from test.support import check_disallow_instantiation +from test.support import hashlib_helper, import_helper +from test.support.hashlib_helper import ( + BuiltinHashFunctionsTrait, + HashFunctionsTrait, + NamedHashFunctionsTrait, + OpenSSLHashFunctionsTrait, +) +from test.support.import_helper import import_fresh_module +from unittest.mock import patch try: - import _hashlib as _hashopenssl - from _hashlib import HMAC as C_HMAC - from _hashlib import hmac_new as c_hmac_new + import _hashlib from _hashlib import compare_digest as openssl_compare_digest except ImportError: - _hashopenssl = None - C_HMAC = None - c_hmac_new = None + _hashlib = None openssl_compare_digest = None try: - import _sha256 as sha256_module + import _sha2 as sha2 except ImportError: - sha256_module = None + sha2 = None -def ignore_warning(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", - category=DeprecationWarning) - return func(*args, **kwargs) - return wrapper +def requires_builtin_sha2(): + return unittest.skipIf(sha2 is None, "requires _sha2") -class TestVectorsTestCase(unittest.TestCase): +class ModuleMixin: + """Mixin with a HMAC module implementation.""" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def assert_hmac_internals( - self, h, digest, hashname, digest_size, block_size - ): - self.assertEqual(h.hexdigest().upper(), digest.upper()) - self.assertEqual(h.digest(), binascii.unhexlify(digest)) + hmac = None + + +class PyModuleMixin(ModuleMixin): + """Pure Python implementation of HMAC. + + The underlying hash functions may be OpenSSL-based or HACL* based, + depending on whether OpenSSL is present or not. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hmac = import_fresh_module('hmac', blocked=['_hashlib', '_hmac']) + + +@hashlib_helper.requires_builtin_hmac() +class BuiltinModuleMixin(ModuleMixin): + """Built-in HACL* implementation of HMAC.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hmac = import_fresh_module('_hmac') + + +# Sentinel object used to detect whether a digestmod is given or not. +DIGESTMOD_SENTINEL = object() + + +class CreatorMixin: + """Mixin exposing a method creating a HMAC object.""" + + def hmac_new(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """Create a new HMAC object. + + Implementations should accept arbitrary 'digestmod' as this + method can be used to test which exceptions are being raised. + """ + raise NotImplementedError + + def bind_hmac_new(self, digestmod): + """Return a specialization of hmac_new() with a bound digestmod.""" + return functools.partial(self.hmac_new, digestmod=digestmod) + + +class DigestMixin: + """Mixin exposing a method computing a HMAC digest.""" + + def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """Compute a HMAC digest. + + Implementations should accept arbitrary 'digestmod' as this + method can be used to test which exceptions are being raised. + """ + raise NotImplementedError + + def bind_hmac_digest(self, digestmod): + """Return a specialization of hmac_digest() with a bound digestmod.""" + return functools.partial(self.hmac_digest, digestmod=digestmod) + + +def _call_newobj_func(new_func, key, msg, digestmod): + if digestmod is DIGESTMOD_SENTINEL: # to test when digestmod is missing + return new_func(key, msg) # expected to raise + # functions creating HMAC objects take a 'digestmod' keyword argument + return new_func(key, msg, digestmod=digestmod) + + +def _call_digest_func(digest_func, key, msg, digestmod): + if digestmod is DIGESTMOD_SENTINEL: # to test when digestmod is missing + return digest_func(key, msg) # expected to raise + # functions directly computing digests take a 'digest' keyword argument + return digest_func(key, msg, digest=digestmod) + + +class ThroughObjectMixin(ModuleMixin, CreatorMixin, DigestMixin): + """Mixin delegating to .HMAC() and .HMAC(...).digest(). + + Both the C implementation and the Python implementation of HMAC should + expose a HMAC class with the same functionalities. + """ + + def hmac_new(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """Create a HMAC object via a module-level class constructor.""" + return _call_newobj_func(self.hmac.HMAC, key, msg, digestmod) + + def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """Call the digest() method on a HMAC object obtained by hmac_new().""" + return _call_newobj_func(self.hmac_new, key, msg, digestmod).digest() + + +class ThroughModuleAPIMixin(ModuleMixin, CreatorMixin, DigestMixin): + """Mixin delegating to .new() and .digest().""" + + def hmac_new(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """Create a HMAC object via a module-level function.""" + return _call_newobj_func(self.hmac.new, key, msg, digestmod) + + def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + """One-shot HMAC digest computation.""" + return _call_digest_func(self.hmac.digest, key, msg, digestmod) + + +@hashlib_helper.requires_hashlib() +class ThroughOpenSSLAPIMixin(CreatorMixin, DigestMixin): + """Mixin delegating to _hashlib.hmac_new() and _hashlib.hmac_digest().""" + + def hmac_new(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + return _call_newobj_func(_hashlib.hmac_new, key, msg, digestmod) + + def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + return _call_digest_func(_hashlib.hmac_digest, key, msg, digestmod) + + +class ThroughBuiltinAPIMixin(BuiltinModuleMixin, CreatorMixin, DigestMixin): + """Mixin delegating to _hmac.new() and _hmac.compute_digest().""" + + def hmac_new(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + return _call_newobj_func(self.hmac.new, key, msg, digestmod) + + def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): + return _call_digest_func(self.hmac.compute_digest, key, msg, digestmod) + + +class ObjectCheckerMixin: + """Mixin for checking HMAC objects (pure Python, OpenSSL or built-in).""" + + def check_object(self, h, hexdigest, hashname, digest_size, block_size): + """Check a HMAC object 'h' against the given values.""" + self.check_internals(h, hashname, digest_size, block_size) + self.check_hexdigest(h, hexdigest, digest_size) + + def check_internals(self, h, hashname, digest_size, block_size): + """Check the constant attributes of a HMAC object.""" self.assertEqual(h.name, f"hmac-{hashname}") self.assertEqual(h.digest_size, digest_size) self.assertEqual(h.block_size, block_size) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def check_hexdigest(self, h, hexdigest, digest_size): + """Check the HMAC digest of 'h' and its size.""" + self.assertEqual(len(h.digest()), digest_size) + self.assertEqual(h.digest(), binascii.unhexlify(hexdigest)) + self.assertEqual(h.hexdigest().upper(), hexdigest.upper()) + + +class AssertersMixin(CreatorMixin, DigestMixin, ObjectCheckerMixin): + """Mixin class for common tests.""" + + def hmac_new_by_name(self, key, msg=None, *, hashname): + """Alternative implementation of hmac_new(). + + This is typically useful when one needs to test against an HMAC + implementation which only recognizes underlying hash functions + by their name (all HMAC implementations must at least recognize + hash functions by their names but some may use aliases such as + `hashlib.sha1` instead of "sha1"). + + Unlike hmac_new(), this method may assert the type of 'hashname' + as it should only be used in tests that are expected to create + a HMAC object. + """ + self.assertIsInstance(hashname, str) + return self.hmac_new(key, msg, digestmod=hashname) + + def hmac_digest_by_name(self, key, msg=None, *, hashname): + """Alternative implementation of hmac_digest(). + + Unlike hmac_digest(), this method may assert the type of 'hashname' + as it should only be used in tests that are expected to compute a + HMAC digest. + """ + self.assertIsInstance(hashname, str) + return self.hmac_digest(key, msg, digestmod=hashname) + def assert_hmac( - self, key, data, digest, hashfunc, hashname, digest_size, block_size + self, key, msg, hexdigest, hashfunc, hashname, digest_size, block_size ): - h = hmac.HMAC(key, data, digestmod=hashfunc) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size + """Check that HMAC(key, msg) == digest. + + The 'hashfunc' and 'hashname' are used as 'digestmod' values, + thereby allowing to test the underlying dispatching mechanism. + + Note that 'hashfunc' may be a string, a callable, or a PEP-257 + module. Note that not all HMAC implementations may recognize the + same set of types for 'hashfunc', but they should always accept + a hash function by its name. + """ + if hashfunc == hashname: + choices = [hashname] + else: + choices = [hashfunc, hashname] + + for digestmod in choices: + with self.subTest(digestmod=digestmod): + self.assert_hmac_new( + key, msg, hexdigest, digestmod, + hashname, digest_size, block_size + ) + self.assert_hmac_hexdigest( + key, msg, hexdigest, digestmod, digest_size + ) + self.assert_hmac_common_cases( + key, msg, hexdigest, digestmod, + hashname, digest_size, block_size + ) + self.assert_hmac_extra_cases( + key, msg, hexdigest, digestmod, + hashname, digest_size, block_size + ) + + self.assert_hmac_new_by_name( + key, msg, hexdigest, hashname, digest_size, block_size + ) + self.assert_hmac_hexdigest_by_name( + key, msg, hexdigest, hashname, digest_size ) - h = hmac.HMAC(key, data, digestmod=hashname) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size + def assert_hmac_new( + self, key, msg, hexdigest, digestmod, hashname, digest_size, block_size + ): + """Check that HMAC(key, msg) == digest. + + This test uses the `hmac_new()` method to create HMAC objects. + """ + self.check_hmac_new( + key, msg, hexdigest, hashname, digest_size, block_size, + hmac_new_func=self.hmac_new, + hmac_new_kwds={'digestmod': digestmod}, ) - h = hmac.HMAC(key, digestmod=hashname) - h2 = h.copy() - h2.update(b"test update") - h.update(data) - self.assertEqual(h.hexdigest().upper(), digest.upper()) + def assert_hmac_new_by_name( + self, key, msg, hexdigest, hashname, digest_size, block_size + ): + """Check that HMAC(key, msg) == digest. + + This test uses the `hmac_new_by_name()` method to create HMAC objects. + """ + self.check_hmac_new( + key, msg, hexdigest, hashname, digest_size, block_size, + hmac_new_func=self.hmac_new_by_name, + hmac_new_kwds={'hashname': hashname}, + ) - h = hmac.new(key, data, digestmod=hashname) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size + def check_hmac_new( + self, key, msg, hexdigest, hashname, digest_size, block_size, + hmac_new_func, hmac_new_kwds=types.MappingProxyType({}), + ): + """Check that HMAC(key, msg) == digest. + + This also tests that using an empty/None initial message and + then calling `h.update(msg)` produces the same result, namely + that HMAC(key, msg) is equivalent to HMAC(key).update(msg). + """ + h = hmac_new_func(key, msg, **hmac_new_kwds) + self.check_object(h, hexdigest, hashname, digest_size, block_size) + + def hmac_new_feed(*args): + h = hmac_new_func(key, *args, **hmac_new_kwds) + h.update(msg) + self.check_hexdigest(h, hexdigest, digest_size) + + with self.subTest('no initial message'): + hmac_new_feed() + with self.subTest('initial message is empty'): + hmac_new_feed(b'') + with self.subTest('initial message is None'): + hmac_new_feed(None) + + def assert_hmac_hexdigest( + self, key, msg, hexdigest, digestmod, digest_size, + ): + """Check a HMAC digest computed by hmac_digest().""" + self.check_hmac_hexdigest( + key, msg, hexdigest, digest_size, + hmac_digest_func=self.hmac_digest, + hmac_digest_kwds={'digestmod': digestmod}, ) - h = hmac.new(key, None, digestmod=hashname) - h.update(data) - self.assertEqual(h.hexdigest().upper(), digest.upper()) + def assert_hmac_hexdigest_by_name( + self, key, msg, hexdigest, hashname, digest_size + ): + """Check a HMAC digest computed by hmac_digest_by_name().""" + self.assertIsInstance(hashname, str) + self.check_hmac_hexdigest( + key, msg, hexdigest, digest_size, + hmac_digest_func=self.hmac_digest_by_name, + hmac_digest_kwds={'hashname': hashname}, + ) - h = hmac.new(key, digestmod=hashname) - h.update(data) - self.assertEqual(h.hexdigest().upper(), digest.upper()) + def check_hmac_hexdigest( + self, key, msg, hexdigest, digest_size, + hmac_digest_func, hmac_digest_kwds=types.MappingProxyType({}), + ): + """Check and return a HMAC digest computed by hmac_digest_func(). - h = hmac.new(key, data, digestmod=hashfunc) - self.assertEqual(h.hexdigest().upper(), digest.upper()) + This HMAC digest is computed by: - self.assertEqual( - hmac.digest(key, data, digest=hashname), - binascii.unhexlify(digest) - ) - self.assertEqual( - hmac.digest(key, data, digest=hashfunc), - binascii.unhexlify(digest) - ) + hmac_digest_func(key, msg, **hmac_digest_kwds) - h = hmac.HMAC.__new__(hmac.HMAC) - h._init_old(key, data, digestmod=hashname) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size - ) + This is typically useful for checking one-shot HMAC functions. + """ + d = hmac_digest_func(key, msg, **hmac_digest_kwds) + self.assertEqual(len(d), digest_size) + self.assertEqual(d, binascii.unhexlify(hexdigest)) + return d - if c_hmac_new is not None: - h = c_hmac_new(key, data, digestmod=hashname) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size - ) + def assert_hmac_common_cases( + self, key, msg, hexdigest, digestmod, hashname, digest_size, block_size + ): + """Common tests executed by all subclasses.""" + h1 = self.hmac_new_by_name(key, hashname=hashname) + h2 = h1.copy() + h2.update(b"test update should not affect original") + h1.update(msg) + self.check_object(h1, hexdigest, hashname, digest_size, block_size) - h = c_hmac_new(key, digestmod=hashname) - h2 = h.copy() - h2.update(b"test update") - h.update(data) - self.assertEqual(h.hexdigest().upper(), digest.upper()) + def assert_hmac_extra_cases( + self, key, msg, hexdigest, digestmod, hashname, digest_size, block_size + ): + """Extra tests that can be added in subclasses.""" - func = getattr(_hashopenssl, f"openssl_{hashname}") - h = c_hmac_new(key, data, digestmod=func) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size - ) - h = hmac.HMAC.__new__(hmac.HMAC) - h._init_hmac(key, data, digestmod=hashname) - self.assert_hmac_internals( - h, digest, hashname, digest_size, block_size - ) +class PyAssertersMixin(PyModuleMixin, AssertersMixin): - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('md5', openssl=True) - def test_md5_vectors(self): - # Test the HMAC module against test vectors from the RFC. + def assert_hmac_extra_cases( + self, key, msg, hexdigest, digestmod, hashname, digest_size, block_size + ): + h = self.hmac.HMAC.__new__(self.hmac.HMAC) + h._init_old(key, msg, digestmod=digestmod) + self.check_object(h, hexdigest, hashname, digest_size, block_size) + + +class OpenSSLAssertersMixin(ThroughOpenSSLAPIMixin, AssertersMixin): + + def hmac_new_by_name(self, key, msg=None, *, hashname): + self.assertIsInstance(hashname, str) + openssl_func = getattr(_hashlib, f"openssl_{hashname}") + return self.hmac_new(key, msg, digestmod=openssl_func) + + def hmac_digest_by_name(self, key, msg=None, *, hashname): + self.assertIsInstance(hashname, str) + openssl_func = getattr(_hashlib, f"openssl_{hashname}") + return self.hmac_digest(key, msg, digestmod=openssl_func) + + +class BuiltinAssertersMixin(ThroughBuiltinAPIMixin, AssertersMixin): + pass - def md5test(key, data, digest): - self.assert_hmac( - key, data, digest, - hashfunc=hashlib.md5, - hashname="md5", - digest_size=16, - block_size=64 - ) + +class RFCTestCaseMixin(HashFunctionsTrait, AssertersMixin): + """Test HMAC implementations against RFC 2202/4231 and NIST test vectors. + + - Test vectors for MD5 and SHA-1 are taken from RFC 2202. + - Test vectors for SHA-2 are taken from RFC 4231. + - Test vectors for SHA-3 are NIST's test vectors [1]. + + [1] https://csrc.nist.gov/projects/message-authentication-codes + """ + + def test_md5_rfc2202(self): + def md5test(key, msg, hexdigest): + self.assert_hmac(key, msg, hexdigest, self.md5, "md5", 16, 64) md5test(b"\x0b" * 16, b"Hi There", - "9294727A3638BB1C13F48EF8158BFC9D") + "9294727a3638bb1c13f48ef8158bfc9d") md5test(b"Jefe", b"what do ya want for nothing?", @@ -170,18 +450,9 @@ def md5test(key, data, digest): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha1', openssl=True) - def test_sha_vectors(self): - def shatest(key, data, digest): - self.assert_hmac( - key, data, digest, - hashfunc=hashlib.sha1, - hashname="sha1", - digest_size=20, - block_size=64 - ) + def test_sha1_rfc2202(self): + def shatest(key, msg, hexdigest): + self.assert_hmac(key, msg, hexdigest, self.sha1, "sha1", 20, 64) shatest(b"\x0b" * 20, b"Hi There", @@ -212,498 +483,1160 @@ def shatest(key, data, digest): b"and Larger Than One Block-Size Data"), "e8e99d0f45237d786d6bbaa7965c7808bbff1a91") - def _rfc4231_test_cases(self, hashfunc, hash_name, digest_size, block_size): - def hmactest(key, data, hexdigests): - digest = hexdigests[hashfunc] + def test_sha2_224_rfc4231(self): + self._test_sha2_rfc4231(self.sha224, 'sha224', 28, 64) + + def test_sha2_256_rfc4231(self): + self._test_sha2_rfc4231(self.sha256, 'sha256', 32, 64) + + def test_sha2_384_rfc4231(self): + self._test_sha2_rfc4231(self.sha384, 'sha384', 48, 128) + + def test_sha2_512_rfc4231(self): + self._test_sha2_rfc4231(self.sha512, 'sha512', 64, 128) + + def _test_sha2_rfc4231(self, hashfunc, hashname, digest_size, block_size): + def hmactest(key, msg, hexdigests): + hexdigest = hexdigests[hashname] self.assert_hmac( - key, data, digest, + key, msg, hexdigest, hashfunc=hashfunc, - hashname=hash_name, + hashname=hashname, digest_size=digest_size, block_size=block_size ) # 4.2. Test Case 1 - hmactest(key = b'\x0b'*20, - data = b'Hi There', - hexdigests = { - hashlib.sha224: '896fb1128abbdf196832107cd49df33f' - '47b4b1169912ba4f53684b22', - hashlib.sha256: 'b0344c61d8db38535ca8afceaf0bf12b' - '881dc200c9833da726e9376c2e32cff7', - hashlib.sha384: 'afd03944d84895626b0825f4ab46907f' - '15f9dadbe4101ec682aa034c7cebc59c' - 'faea9ea9076ede7f4af152e8b2fa9cb6', - hashlib.sha512: '87aa7cdea5ef619d4ff0b4241a1d6cb0' - '2379f4e2ce4ec2787ad0b30545e17cde' - 'daa833b7d6b8a702038b274eaea3f4e4' - 'be9d914eeb61f1702e696c203a126854', + hmactest(key=b'\x0b' * 20, + msg=b'Hi There', + hexdigests={ + 'sha224': '896fb1128abbdf196832107cd49df33f' + '47b4b1169912ba4f53684b22', + 'sha256': 'b0344c61d8db38535ca8afceaf0bf12b' + '881dc200c9833da726e9376c2e32cff7', + 'sha384': 'afd03944d84895626b0825f4ab46907f' + '15f9dadbe4101ec682aa034c7cebc59c' + 'faea9ea9076ede7f4af152e8b2fa9cb6', + 'sha512': '87aa7cdea5ef619d4ff0b4241a1d6cb0' + '2379f4e2ce4ec2787ad0b30545e17cde' + 'daa833b7d6b8a702038b274eaea3f4e4' + 'be9d914eeb61f1702e696c203a126854', }) # 4.3. Test Case 2 - hmactest(key = b'Jefe', - data = b'what do ya want for nothing?', - hexdigests = { - hashlib.sha224: 'a30e01098bc6dbbf45690f3a7e9e6d0f' - '8bbea2a39e6148008fd05e44', - hashlib.sha256: '5bdcc146bf60754e6a042426089575c7' - '5a003f089d2739839dec58b964ec3843', - hashlib.sha384: 'af45d2e376484031617f78d2b58a6b1b' - '9c7ef464f5a01b47e42ec3736322445e' - '8e2240ca5e69e2c78b3239ecfab21649', - hashlib.sha512: '164b7a7bfcf819e2e395fbe73b56e0a3' - '87bd64222e831fd610270cd7ea250554' - '9758bf75c05a994a6d034f65f8f0e6fd' - 'caeab1a34d4a6b4b636e070a38bce737', + hmactest(key=b'Jefe', + msg=b'what do ya want for nothing?', + hexdigests={ + 'sha224': 'a30e01098bc6dbbf45690f3a7e9e6d0f' + '8bbea2a39e6148008fd05e44', + 'sha256': '5bdcc146bf60754e6a042426089575c7' + '5a003f089d2739839dec58b964ec3843', + 'sha384': 'af45d2e376484031617f78d2b58a6b1b' + '9c7ef464f5a01b47e42ec3736322445e' + '8e2240ca5e69e2c78b3239ecfab21649', + 'sha512': '164b7a7bfcf819e2e395fbe73b56e0a3' + '87bd64222e831fd610270cd7ea250554' + '9758bf75c05a994a6d034f65f8f0e6fd' + 'caeab1a34d4a6b4b636e070a38bce737', }) # 4.4. Test Case 3 - hmactest(key = b'\xaa'*20, - data = b'\xdd'*50, - hexdigests = { - hashlib.sha224: '7fb3cb3588c6c1f6ffa9694d7d6ad264' - '9365b0c1f65d69d1ec8333ea', - hashlib.sha256: '773ea91e36800e46854db8ebd09181a7' - '2959098b3ef8c122d9635514ced565fe', - hashlib.sha384: '88062608d3e6ad8a0aa2ace014c8a86f' - '0aa635d947ac9febe83ef4e55966144b' - '2a5ab39dc13814b94e3ab6e101a34f27', - hashlib.sha512: 'fa73b0089d56a284efb0f0756c890be9' - 'b1b5dbdd8ee81a3655f83e33b2279d39' - 'bf3e848279a722c806b485a47e67c807' - 'b946a337bee8942674278859e13292fb', + hmactest(key=b'\xaa' * 20, + msg=b'\xdd' * 50, + hexdigests={ + 'sha224': '7fb3cb3588c6c1f6ffa9694d7d6ad264' + '9365b0c1f65d69d1ec8333ea', + 'sha256': '773ea91e36800e46854db8ebd09181a7' + '2959098b3ef8c122d9635514ced565fe', + 'sha384': '88062608d3e6ad8a0aa2ace014c8a86f' + '0aa635d947ac9febe83ef4e55966144b' + '2a5ab39dc13814b94e3ab6e101a34f27', + 'sha512': 'fa73b0089d56a284efb0f0756c890be9' + 'b1b5dbdd8ee81a3655f83e33b2279d39' + 'bf3e848279a722c806b485a47e67c807' + 'b946a337bee8942674278859e13292fb', }) # 4.5. Test Case 4 - hmactest(key = bytes(x for x in range(0x01, 0x19+1)), - data = b'\xcd'*50, - hexdigests = { - hashlib.sha224: '6c11506874013cac6a2abc1bb382627c' - 'ec6a90d86efc012de7afec5a', - hashlib.sha256: '82558a389a443c0ea4cc819899f2083a' - '85f0faa3e578f8077a2e3ff46729665b', - hashlib.sha384: '3e8a69b7783c25851933ab6290af6ca7' - '7a9981480850009cc5577c6e1f573b4e' - '6801dd23c4a7d679ccf8a386c674cffb', - hashlib.sha512: 'b0ba465637458c6990e5a8c5f61d4af7' - 'e576d97ff94b872de76f8050361ee3db' - 'a91ca5c11aa25eb4d679275cc5788063' - 'a5f19741120c4f2de2adebeb10a298dd', + hmactest(key=bytes(x for x in range(0x01, 0x19 + 1)), + msg=b'\xcd' * 50, + hexdigests={ + 'sha224': '6c11506874013cac6a2abc1bb382627c' + 'ec6a90d86efc012de7afec5a', + 'sha256': '82558a389a443c0ea4cc819899f2083a' + '85f0faa3e578f8077a2e3ff46729665b', + 'sha384': '3e8a69b7783c25851933ab6290af6ca7' + '7a9981480850009cc5577c6e1f573b4e' + '6801dd23c4a7d679ccf8a386c674cffb', + 'sha512': 'b0ba465637458c6990e5a8c5f61d4af7' + 'e576d97ff94b872de76f8050361ee3db' + 'a91ca5c11aa25eb4d679275cc5788063' + 'a5f19741120c4f2de2adebeb10a298dd', }) # 4.7. Test Case 6 - hmactest(key = b'\xaa'*131, - data = b'Test Using Larger Than Block-Siz' - b'e Key - Hash Key First', - hexdigests = { - hashlib.sha224: '95e9a0db962095adaebe9b2d6f0dbce2' - 'd499f112f2d2b7273fa6870e', - hashlib.sha256: '60e431591ee0b67f0d8a26aacbf5b77f' - '8e0bc6213728c5140546040f0ee37f54', - hashlib.sha384: '4ece084485813e9088d2c63a041bc5b4' - '4f9ef1012a2b588f3cd11f05033ac4c6' - '0c2ef6ab4030fe8296248df163f44952', - hashlib.sha512: '80b24263c7c1a3ebb71493c1dd7be8b4' - '9b46d1f41b4aeec1121b013783f8f352' - '6b56d037e05f2598bd0fd2215d6a1e52' - '95e64f73f63f0aec8b915a985d786598', + hmactest(key=b'\xaa' * 131, + msg=b'Test Using Larger Than Block-Siz' + b'e Key - Hash Key First', + hexdigests={ + 'sha224': '95e9a0db962095adaebe9b2d6f0dbce2' + 'd499f112f2d2b7273fa6870e', + 'sha256': '60e431591ee0b67f0d8a26aacbf5b77f' + '8e0bc6213728c5140546040f0ee37f54', + 'sha384': '4ece084485813e9088d2c63a041bc5b4' + '4f9ef1012a2b588f3cd11f05033ac4c6' + '0c2ef6ab4030fe8296248df163f44952', + 'sha512': '80b24263c7c1a3ebb71493c1dd7be8b4' + '9b46d1f41b4aeec1121b013783f8f352' + '6b56d037e05f2598bd0fd2215d6a1e52' + '95e64f73f63f0aec8b915a985d786598', }) # 4.8. Test Case 7 - hmactest(key = b'\xaa'*131, - data = b'This is a test using a larger th' - b'an block-size key and a larger t' - b'han block-size data. The key nee' - b'ds to be hashed before being use' - b'd by the HMAC algorithm.', - hexdigests = { - hashlib.sha224: '3a854166ac5d9f023f54d517d0b39dbd' - '946770db9c2b95c9f6f565d1', - hashlib.sha256: '9b09ffa71b942fcb27635fbcd5b0e944' - 'bfdc63644f0713938a7f51535c3a35e2', - hashlib.sha384: '6617178e941f020d351e2f254e8fd32c' - '602420feb0b8fb9adccebb82461e99c5' - 'a678cc31e799176d3860e6110c46523e', - hashlib.sha512: 'e37b6a775dc87dbaa4dfa9f96e5e3ffd' - 'debd71f8867289865df5a32d20cdc944' - 'b6022cac3c4982b10d5eeb55c3e4de15' - '134676fb6de0446065c97440fa8c6a58', + hmactest(key=b'\xaa' * 131, + msg=b'This is a test using a larger th' + b'an block-size key and a larger t' + b'han block-size data. The key nee' + b'ds to be hashed before being use' + b'd by the HMAC algorithm.', + hexdigests={ + 'sha224': '3a854166ac5d9f023f54d517d0b39dbd' + '946770db9c2b95c9f6f565d1', + 'sha256': '9b09ffa71b942fcb27635fbcd5b0e944' + 'bfdc63644f0713938a7f51535c3a35e2', + 'sha384': '6617178e941f020d351e2f254e8fd32c' + '602420feb0b8fb9adccebb82461e99c5' + 'a678cc31e799176d3860e6110c46523e', + 'sha512': 'e37b6a775dc87dbaa4dfa9f96e5e3ffd' + 'debd71f8867289865df5a32d20cdc944' + 'b6022cac3c4982b10d5eeb55c3e4de15' + '134676fb6de0446065c97440fa8c6a58', }) - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha224', openssl=True) - def test_sha224_rfc4231(self): - self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256', openssl=True) - def test_sha256_rfc4231(self): - self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha384', openssl=True) - def test_sha384_rfc4231(self): - self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha512', openssl=True) - def test_sha512_rfc4231(self): - self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) + def test_sha3_224_nist(self): + for key, msg, hexdigest in [ + ( + bytes(range(28)), + b'Sample message for keylenblocklen', + '078695eecc227c636ad31d063a15dd05a7e819a66ec6d8de1e193e59' + ) + ]: + self.assert_hmac( + key, msg, hexdigest, + hashfunc=self.sha3_224, hashname='sha3_224', + digest_size=28, block_size=144 + ) - @hashlib_helper.requires_hashdigest('sha256') - def test_legacy_block_size_warnings(self): - class MockCrazyHash(object): - """Ain't no block_size attribute here.""" - def __init__(self, *args): - self._x = hashlib.sha256(*args) - self.digest_size = self._x.digest_size - def update(self, v): - self._x.update(v) - def digest(self): - return self._x.digest() + def test_sha3_256_nist(self): + for key, msg, hexdigest in [ + ( + bytes(range(32)), + b'Sample message for keylenblocklen', + '9bcf2c238e235c3ce88404e813bd2f3a' + '97185ac6f238c63d6229a00b07974258' + ) + ]: + self.assert_hmac( + key, msg, hexdigest, + hashfunc=self.sha3_256, hashname='sha3_256', + digest_size=32, block_size=136 + ) - with warnings.catch_warnings(): - warnings.simplefilter('error', RuntimeWarning) - with self.assertRaises(RuntimeWarning): - hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash) - self.fail('Expected warning about missing block_size') + def test_sha3_384_nist(self): + for key, msg, hexdigest in [ + ( + bytes(range(48)), + b'Sample message for keylenblocklen', + 'e5ae4c739f455279368ebf36d4f5354c' + '95aa184c899d3870e460ebc288ef1f94' + '70053f73f7c6da2a71bcaec38ce7d6ac' + ) + ]: + self.assert_hmac( + key, msg, hexdigest, + hashfunc=self.sha3_384, hashname='sha3_384', + digest_size=48, block_size=104 + ) - MockCrazyHash.block_size = 1 - with self.assertRaises(RuntimeWarning): - hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash) - self.fail('Expected warning about small block_size') + def test_sha3_512_nist(self): + for key, msg, hexdigest in [ + ( + bytes(range(64)), + b'Sample message for keylenblocklen', + '5f464f5e5b7848e3885e49b2c385f069' + '4985d0e38966242dc4a5fe3fea4b37d4' + '6b65ceced5dcf59438dd840bab22269f' + '0ba7febdb9fcf74602a35666b2a32915' + ) + ]: + self.assert_hmac( + key, msg, hexdigest, + hashfunc=self.sha3_512, hashname='sha3_512', + digest_size=64, block_size=72 + ) - def test_with_digestmod_no_default(self): - """The digestmod parameter is required as of Python 3.8.""" - with self.assertRaisesRegex(TypeError, r'required.*digestmod'): - key = b"\x0b" * 16 - data = b"Hi There" - hmac.HMAC(key, data, digestmod=None) - with self.assertRaisesRegex(TypeError, r'required.*digestmod'): - hmac.new(key, data) - with self.assertRaisesRegex(TypeError, r'required.*digestmod'): - hmac.HMAC(key, msg=data, digestmod='') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_with_fallback(self): - cache = getattr(hashlib, '__builtin_constructor_cache') - try: - cache['foo'] = hashlib.sha256 - hexdigest = hmac.digest(b'key', b'message', 'foo').hex() - expected = '6e9ef29b75fffc5b7abae527d58fdadb2fe42e7219011976917343065f58ed4a' - self.assertEqual(hexdigest, expected) - finally: - cache.pop('foo') +class PurePythonInitHMAC(PyModuleMixin, HashFunctionsTrait): + + @classmethod + def setUpClass(cls): + super().setUpClass() + for meth in ['_init_openssl_hmac', '_init_builtin_hmac']: + fn = getattr(cls.hmac.HMAC, meth) + cm = patch.object(cls.hmac.HMAC, meth, autospec=True, wraps=fn) + cls.enterClassContext(cm) + + @classmethod + def tearDownClass(cls): + cls.hmac.HMAC._init_openssl_hmac.assert_not_called() + cls.hmac.HMAC._init_builtin_hmac.assert_not_called() + # Do not assert that HMAC._init_old() has been called as it's tricky + # to determine whether a test for a specific hash function has been + # executed or not. On regular builds, it will be called but if a + # hash function is not available, it's hard to detect for which + # test we should checj HMAC._init_old() or not. + super().tearDownClass() + + +class PyRFCOpenSSLTestCase(ThroughObjectMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.HMAC(). + + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class PyRFCBuiltinTestCase(ThroughObjectMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.HMAC(). + + The underlying hash functions are HACL*-based but + _init_old() is used instead of _init_builtin_hmac(). + """ + + +class PyDotNewOpenSSLRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.new(). + + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class PyDotNewBuiltinRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.new(). + + The underlying hash functions are HACL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class OpenSSLRFCTestCase(OpenSSLAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + unittest.TestCase): + """OpenSSL implementation of HMAC. + + The underlying hash functions are also OpenSSL-based. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_md5_rfc2202(self): + return super().test_md5_rfc2202() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha1_rfc2202(self): + return super().test_sha1_rfc2202() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha2_224_rfc4231(self): + return super().test_sha2_224_rfc4231() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha2_256_rfc4231(self): + return super().test_sha2_256_rfc4231() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha2_384_rfc4231(self): + return super().test_sha2_384_rfc4231() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha2_512_rfc4231(self): + return super().test_sha2_512_rfc4231() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha3_224_nist(self): + return super().test_sha3_224_nist() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha3_256_nist(self): + return super().test_sha3_256_nist() + + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha3_384_nist(self): + return super().test_sha3_384_nist() -class ConstructorTestCase(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; _hashlib.UnsupportedDigestmodError: unsupported hash type + def test_sha3_512_nist(self): + return super().test_sha3_512_nist() + + +class BuiltinRFCTestCase(BuiltinAssertersMixin, + NamedHashFunctionsTrait, + RFCTestCaseMixin, + unittest.TestCase): + """Built-in HACL* implementation of HMAC. - expected = ( - "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb" - ) + The underlying hash functions are also HACL*-based. + """ - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_normal(self): - # Standard constructor call. - try: - hmac.HMAC(b"key", digestmod='sha256') - except Exception: - self.fail("Standard constructor call raised exception.") + def assert_hmac_extra_cases( + self, key, msg, hexdigest, digestmod, hashname, digest_size, block_size + ): + # assert one-shot HMAC at the same time + with self.subTest(key=key, msg=msg, hashname=hashname): + func = getattr(self.hmac, f'compute_{hashname}') + self.assertTrue(callable(func)) + self.check_hmac_hexdigest(key, msg, hexdigest, digest_size, func) + + +class DigestModTestCaseMixin(CreatorMixin, DigestMixin): + """Tests for the 'digestmod' parameter for hmac_new() and hmac_digest().""" + + def assert_raises_missing_digestmod(self): + """A context manager catching errors when a digestmod is missing.""" + return self.assertRaisesRegex(TypeError, + "[M|m]issing.*required.*digestmod") + + def assert_raises_unknown_digestmod(self): + """A context manager catching errors when a digestmod is unknown.""" + return self.assertRaisesRegex(ValueError, "[Uu]nsupported.*") + + def test_constructor_missing_digestmod(self): + catcher = self.assert_raises_missing_digestmod + self.do_test_constructor_missing_digestmod(catcher) + + def test_constructor_unknown_digestmod(self): + catcher = self.assert_raises_unknown_digestmod + self.do_test_constructor_unknown_digestmod(catcher) - @hashlib_helper.requires_hashdigest('sha256') - def test_with_str_key(self): - # Pass a key of type str, which is an error, because it expects a key - # of type bytes - with self.assertRaises(TypeError): - h = hmac.HMAC("key", digestmod='sha256') + def do_test_constructor_missing_digestmod(self, catcher): + for func, args, kwds in self.cases_missing_digestmod_in_constructor(): + with self.subTest(args=args, kwds=kwds), catcher(): + func(*args, **kwds) - @hashlib_helper.requires_hashdigest('sha256') - def test_dot_new_with_str_key(self): - # Pass a key of type str, which is an error, because it expects a key - # of type bytes - with self.assertRaises(TypeError): - h = hmac.new("key", digestmod='sha256') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_withtext(self): - # Constructor call with text. - try: - h = hmac.HMAC(b"key", b"hash this!", digestmod='sha256') - except Exception: - self.fail("Constructor call with text argument raised exception.") - self.assertEqual(h.hexdigest(), self.expected) + def do_test_constructor_unknown_digestmod(self, catcher): + for func, args, kwds in self.cases_unknown_digestmod_in_constructor(): + with self.subTest(args=args, kwds=kwds), catcher(): + func(*args, **kwds) - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_with_bytearray(self): - try: - h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"), - digestmod="sha256") - except Exception: - self.fail("Constructor call with bytearray arguments raised exception.") - self.assertEqual(h.hexdigest(), self.expected) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_with_memoryview_msg(self): - try: - h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256") - except Exception: - self.fail("Constructor call with memoryview msg raised exception.") - self.assertEqual(h.hexdigest(), self.expected) + def cases_missing_digestmod_in_constructor(self): + raise NotImplementedError + + def make_missing_digestmod_cases(self, func, missing_like=()): + """Generate cases for missing digestmod tests. + + Only the Python implementation should consider "falsey" 'digestmod' + values as being equivalent to a missing one. + """ + key, msg = b'unused key', b'unused msg' + choices = [DIGESTMOD_SENTINEL, *missing_like] + return self._invalid_digestmod_cases(func, key, msg, choices) + + def cases_unknown_digestmod_in_constructor(self): + raise NotImplementedError + + def make_unknown_digestmod_cases(self, func, bad_digestmods): + """Generate cases for unknown digestmod tests.""" + key, msg = b'unused key', b'unused msg' + return self._invalid_digestmod_cases(func, key, msg, bad_digestmods) + + def _invalid_digestmod_cases(self, func, key, msg, choices): + cases = [] + for digestmod in choices: + kwargs = {'digestmod': digestmod} + cases.append((func, (key,), kwargs)) + cases.append((func, (key, msg), kwargs)) + cases.append((func, (key,), kwargs | {'msg': msg})) + return cases - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_withmodule(self): - # Constructor call with text and digest module. - try: - h = hmac.HMAC(b"key", b"", hashlib.sha256) - except Exception: - self.fail("Constructor call with hashlib.sha256 raised exception.") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipUnless(C_HMAC is not None, 'need _hashlib') +class ConstructorTestCaseMixin(CreatorMixin, DigestMixin, ObjectCheckerMixin): + """HMAC constructor tests based on HMAC-SHA-2/256.""" + + key = b"key" + msg = b"hash this!" + res = "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb" + + def do_test_constructor(self, hmac_on_key_and_msg): + self.do_test_constructor_invalid_types(hmac_on_key_and_msg) + self.do_test_constructor_supported_types(hmac_on_key_and_msg) + + def do_test_constructor_invalid_types(self, hmac_on_key_and_msg): + self.assertRaises(TypeError, hmac_on_key_and_msg, 1) + self.assertRaises(TypeError, hmac_on_key_and_msg, "key") + + self.assertRaises(TypeError, hmac_on_key_and_msg, b"key", 1) + self.assertRaises(TypeError, hmac_on_key_and_msg, b"key", "msg") + + def do_test_constructor_supported_types(self, hmac_on_key_and_msg): + for tp_key in [bytes, bytearray]: + for tp_msg in [bytes, bytearray, memoryview]: + with self.subTest(tp_key=tp_key, tp_msg=tp_msg): + h = hmac_on_key_and_msg(tp_key(self.key), tp_msg(self.msg)) + self.assertEqual(h.name, "hmac-sha256") + self.assertEqual(h.hexdigest(), self.res) + + @hashlib_helper.requires_hashdigest("sha256") + def test_constructor(self): + self.do_test_constructor(self.bind_hmac_new("sha256")) + + @hashlib_helper.requires_hashdigest("sha256") + def test_digest(self): + digest = self.hmac_digest(self.key, self.msg, "sha256") + self.assertEqual(digest, binascii.unhexlify(self.res)) + + +class PyConstructorBaseMixin(PyModuleMixin, + DigestModTestCaseMixin, + ConstructorTestCaseMixin): + + def cases_missing_digestmod_in_constructor(self): + func, choices = self.hmac_new, ['', None, False] + return self.make_missing_digestmod_cases(func, choices) + + def cases_unknown_digestmod_in_constructor(self): + func, choices = self.hmac_new, ['unknown'] + return self.make_unknown_digestmod_cases(func, choices) + + @requires_builtin_sha2() + def test_constructor_with_module(self): + self.do_test_constructor(self.bind_hmac_new(sha2.sha256)) + + @requires_builtin_sha2() + def test_digest_with_module(self): + digest = self.hmac_digest(self.key, self.msg, sha2.sha256) + self.assertEqual(digest, binascii.unhexlify(self.res)) + + +class PyConstructorTestCase(ThroughObjectMixin, PyConstructorBaseMixin, + unittest.TestCase): + """Test the hmac.HMAC() pure Python constructor.""" + + +class PyModuleConstructorTestCase(ThroughModuleAPIMixin, PyConstructorBaseMixin, + unittest.TestCase): + """Test the hmac.new() and hmac.digest() functions. + + Note that "self.hmac" is imported by blocking "_hashlib" and "_hmac". + For testing functions in "hmac", extend PyMiscellaneousTests instead. + """ + + def test_hmac_digest_digestmod_parameter(self): + func = self.hmac_digest + + def raiser(): + raise RuntimeError("custom exception") + + with self.assertRaisesRegex(RuntimeError, "custom exception"): + func(b'key', b'msg', raiser) + + with self.assertRaisesRegex(ValueError, 'hash type'): + func(b'key', b'msg', 'unknown') + + with self.assertRaisesRegex(AttributeError, 'new'): + func(b'key', b'msg', 1234) + with self.assertRaisesRegex(AttributeError, 'new'): + func(b'key', b'msg', None) + + +class ExtensionConstructorTestCaseMixin(DigestModTestCaseMixin, + ConstructorTestCaseMixin): + + @property + def obj_type(self): + """The underlying (non-instantiable) C class.""" + raise NotImplementedError + + @property + def exc_type(self): + """The exact exception class raised upon invalid 'digestmod' values.""" + raise NotImplementedError + def test_internal_types(self): - # internal types like _hashlib.C_HMAC are not constructable - check_disallow_instantiation(self, C_HMAC) + # internal C types are immutable and cannot be instantiated + check_disallow_instantiation(self, self.obj_type) with self.assertRaisesRegex(TypeError, "immutable type"): - C_HMAC.value = None - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipUnless(sha256_module is not None, 'need _sha256') - def test_with_sha256_module(self): - h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256) - self.assertEqual(h.hexdigest(), self.expected) - self.assertEqual(h.name, "hmac-sha256") + self.obj_type.value = None + + def assert_raises_unknown_digestmod(self): + self.assertIsSubclass(self.exc_type, ValueError) + return self.assertRaises(self.exc_type) + + def cases_missing_digestmod_in_constructor(self): + return self.make_missing_digestmod_cases(self.hmac_new) + + def cases_unknown_digestmod_in_constructor(self): + func, choices = self.hmac_new, ['unknown', 1234] + return self.make_unknown_digestmod_cases(func, choices) + + +class OpenSSLConstructorTestCase(ThroughOpenSSLAPIMixin, + ExtensionConstructorTestCaseMixin, + unittest.TestCase): + + @property + def obj_type(self): + return _hashlib.HMAC + + @property + def exc_type(self): + return _hashlib.UnsupportedDigestmodError + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_hmac_digest_digestmod_parameter(self): + for value in [object, 'unknown', 1234, None]: + with ( + self.subTest(value=value), + self.assert_raises_unknown_digestmod(), + ): + self.hmac_digest(b'key', b'msg', value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module '_hashlib' has no attribute 'hmac_digest' + def test_digest(self): + return super().test_digest() + + +class BuiltinConstructorTestCase(ThroughBuiltinAPIMixin, + ExtensionConstructorTestCaseMixin, + unittest.TestCase): + + @property + def obj_type(self): + return self.hmac.HMAC + + @property + def exc_type(self): + return self.hmac.UnknownHashError + + def test_hmac_digest_digestmod_parameter(self): + for value in [object, 'unknown', 1234, None]: + with ( + self.subTest(value=value), + self.assert_raises_unknown_digestmod(), + ): + self.hmac_digest(b'key', b'msg', value) + + +class SanityTestCaseMixin(CreatorMixin): + """Sanity checks for HMAC objects and their object interface. + + The tests here use a common digestname and do not check all supported + hash functions. + """ + + # The underlying HMAC class to test. May be in C or in Python. + hmac_class: type + # The underlying hash function name (should be accepted by the HMAC class). + digestname: str + # The expected digest and block sizes (must be hardcoded). + digest_size: int + block_size: int + + def test_methods(self): + h = self.hmac_new(b"my secret key", digestmod=self.digestname) + self.assertIsInstance(h, self.hmac_class) + self.assertIsNone(h.update(b"compute the hash of this text!")) + self.assertIsInstance(h.digest(), bytes) + self.assertIsInstance(h.hexdigest(), str) + self.assertIsInstance(h.copy(), self.hmac_class) + + def test_properties(self): + h = self.hmac_new(b"my secret key", digestmod=self.digestname) + self.assertEqual(h.name, f"hmac-{self.digestname}") + self.assertEqual(h.digest_size, self.digest_size) + self.assertEqual(h.block_size, self.block_size) + + def test_copy(self): + # Test a generic copy() and the attributes it exposes. + # See https://github.com/python/cpython/issues/142451. + h1 = self.hmac_new(b"my secret key", digestmod=self.digestname) + h2 = h1.copy() + self.assertEqual(h1.name, h2.name) + self.assertEqual(h1.digest_size, h2.digest_size) + self.assertEqual(h1.block_size, h2.block_size) + + def test_repr(self): + # HMAC object representation may differ across implementations + raise NotImplementedError - digest = hmac.digest(b"key", b"hash this!", sha256_module.sha256) - self.assertEqual(digest, binascii.unhexlify(self.expected)) +@hashlib_helper.requires_hashdigest('sha256') +class PySanityTestCase(ThroughObjectMixin, PyModuleMixin, SanityTestCaseMixin, + unittest.TestCase): -class SanityTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hmac_class = cls.hmac.HMAC + cls.digestname = 'sha256' + cls.digest_size = 32 + cls.block_size = 64 - # TODO: RUSTPYTHON - @unittest.expectedFailure - @hashlib_helper.requires_hashdigest('sha256') - def test_exercise_all_methods(self): - # Exercising all methods once. - # This must not raise any exceptions - try: - h = hmac.HMAC(b"my secret key", digestmod="sha256") - h.update(b"compute the hash of this text!") - h.digest() - h.hexdigest() - h.copy() - except Exception: - self.fail("Exception raised during normal usage of HMAC class.") + def test_repr(self): + h = self.hmac_new(b"my secret key", digestmod=self.digestname) + self.assertStartsWith(repr(h), "' + "" + '' + '' + '' + '\u2603' +) + +SAMPLE_RAWTEXT = SAMPLE_RCDATA + '&☺' + + class EventCollector(html.parser.HTMLParser): def __init__(self, *args, autocdata=False, **kw): @@ -97,12 +109,13 @@ def get_events(self): class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -116,7 +129,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -175,10 +188,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -293,30 +383,20 @@ def test_get_starttag_text(self): 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', '', - 'foo = ""', - 'foo = ""', - 'foo = ""', - 'foo = ""', - 'foo = ""', - 'foo = ""', ]) def test_script_content(self, content): s = f'' - self._run_check(s, [("starttag", "script", []), - ("data", content), - ("endtag", "script")]) + self._run_check(s, [ + ("starttag", "script", []), + ("data", content), + ("endtag", "script"), + ]) @support.subTests('content', [ 'a::before { content: ""; }', 'a::before { content: "¬-an-entity-ref;"; }', 'a::before { content: ""; }', 'a::before { content: "\u2603"; }', - 'a::before { content: "< /style>"; }', - 'a::before { content: ""; }', - 'a::before { content: ""; }', - 'a::before { content: ""; }', - 'a::before { content: ""; }', - 'a::before { content: ""; }', ]) def test_style_content(self, content): s = f'' @@ -324,47 +404,59 @@ def test_style_content(self, content): ("data", content), ("endtag", "style")]) - @support.subTests('content', [ - '', - "", - '', - '', - '', - '\u2603', - '< /title>', - '', - '', - '', - '', - '', + @support.subTests('tag', ['title', 'textarea']) + def test_rcdata_content(self, tag): + source = f"<{tag}>{SAMPLE_RCDATA}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RCDATA), + ("endtag", tag), ]) - def test_title_content(self, content): - source = f"{content}" + source = f"<{tag}>&" self._run_check(source, [ - ("starttag", "title", []), - ("data", content), - ("endtag", "title"), + ("starttag", tag, []), + ('entityref', 'amp'), + ("endtag", tag), ]) - @support.subTests('content', [ - '', - "", - '', - '', - '', - '\u2603', - '< /textarea>', - '', - '', - '', - '', + @support.subTests('tag', + ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script']) + def test_rawtext_content(self, tag): + source = f"<{tag}>{SAMPLE_RAWTEXT}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RAWTEXT), + ("endtag", tag), + ]) + + def test_noscript_content(self): + source = f"" + # scripting=False -- normal mode + self._run_check(source, [ + ('starttag', 'noscript', []), + ('comment', ' not a comment '), + ('starttag', 'not', [('a', 'start tag')]), + ('unknown decl', 'CDATA[not a cdata'), + ('comment', 'not a bogus comment'), + ('endtag', 'not'), + ('data', '☃'), + ('entityref', 'amp'), + ('charref', '9786'), + ('endtag', 'noscript'), ]) - def test_textarea_content(self, content): - source = f"" + # scripting=True -- RAWTEXT mode self._run_check(source, [ - ("starttag", "textarea", []), + ("starttag", "noscript", []), + ("data", SAMPLE_RAWTEXT), + ("endtag", "noscript"), + ], collector=EventCollector(scripting=True)) + + def test_plaintext_content(self): + content = SAMPLE_RAWTEXT + '' # not closing + source = f"{content}" + self._run_check(source, [ + ("starttag", "plaintext", []), ("data", content), - ("endtag", "textarea"), ]) @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', @@ -381,52 +473,65 @@ def test_script_closing_tag(self, endtag): ("endtag", "script")], collector=EventCollectorNoNormalize(convert_charrefs=False)) - @support.subTests('endtag', ['style', 'STYLE', 'style ', 'style\n', - 'style/', 'style foo=bar', 'style foo=">"']) - def test_style_closing_tag(self, endtag): - content = """ - b::before { content: "<!-- not a comment -->"; } - p::before { content: "&not-an-entity-ref;"; } - a::before { content: "<i>"; } - a::after { content: "</i>"; } - """ - s = f'<StyLE>{content}</{endtag}>' - self._run_check(s, [("starttag", "style", []), - ("data", content), - ("endtag", "style")], - collector=EventCollectorNoNormalize(convert_charrefs=False)) - - @support.subTests('endtag', ['title', 'TITLE', 'title ', 'title\n', - 'title/', 'title foo=bar', 'title foo=">"']) - def test_title_closing_tag(self, endtag): - content = "<!-- not a comment --><i>Egg &amp; Spam</i>" - s = f'<TitLe>{content}</{endtag}>' - self._run_check(s, [("starttag", "title", []), - ('data', '<!-- not a comment --><i>Egg & Spam</i>'), - ("endtag", "title")], - collector=EventCollectorNoNormalize(convert_charrefs=True)) - self._run_check(s, [("starttag", "title", []), - ('data', '<!-- not a comment --><i>Egg '), - ('entityref', 'amp'), - ('data', ' Spam</i>'), - ("endtag", "title")], - collector=EventCollectorNoNormalize(convert_charrefs=False)) - - @support.subTests('endtag', ['textarea', 'TEXTAREA', 'textarea ', 'textarea\n', - 'textarea/', 'textarea foo=bar', 'textarea foo=">"']) - def test_textarea_closing_tag(self, endtag): - content = "<!-- not a comment --><i>Egg &amp; Spam</i>" - s = f'<TexTarEa>{content}</{endtag}>' - self._run_check(s, [("starttag", "textarea", []), - ('data', '<!-- not a comment --><i>Egg & Spam</i>'), - ("endtag", "textarea")], - collector=EventCollectorNoNormalize(convert_charrefs=True)) - self._run_check(s, [("starttag", "textarea", []), - ('data', '<!-- not a comment --><i>Egg '), - ('entityref', 'amp'), - ('data', ' Spam</i>'), - ("endtag", "textarea")], - collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_closing_tag(self, tag): + for endtag in [tag, tag.upper(), f'{tag} ', f'{tag}\n', + f'{tag}/', f'{tag} foo=bar', f'{tag} foo=">"']: + content = "<!-- not a comment --><i>Spam</i>" + s = f'<{tag.upper()}>{content}</{endtag}>' + self._run_check(s, [ + ("starttag", tag, []), + ('data', content), + ("endtag", tag), + ], collector=EventCollectorNoNormalize(convert_charrefs=False, scripting=True)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_invalid_closing_tag(self, tag): + content = ( + f'< /{tag}>' + f'</ {tag}>' + f'</{tag}x>' + f'</{tag}\v>' + f'</{tag}\xa0>' + ) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tag,endtag', [ + ('title', 'tıtle'), + ('style', 'ſtyle'), + ('style', 'ſtyle'), + ('style', 'style'), + ('iframe', 'ıframe'), + ('noframes', 'noframeſ'), + ('noscript', 'noſcript'), + ('noscript', 'noscrıpt'), + ('script', 'ſcript'), + ('script', 'scrıpt'), + ]) + def test_invalid_nonascii_closing_tag(self, tag, endtag): + content = f"<br></{endtag}>" + source = f"<{tag}>{content}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) @support.subTests('tail,end', [ ('', False), @@ -735,20 +840,6 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these - data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), - ] - for html, expected in data: - self._run_check(html, expected) - def test_eof_in_comments(self): data = [ ('<!--', [('comment', '')]), diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index f4b9dc6a282..51fa4a3d413 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1,14 +1,16 @@ """Tests for http/cookiejar.py.""" import os +import stat +import sys import re -import test.support +from test import support from test.support import os_helper from test.support import warnings_helper +from test.support.testcase import ExtraAssertions import time import unittest import urllib.request -import pathlib from http.cookiejar import (time2isoz, http2time, iso2time, time2netscape, parse_ns_headers, join_header_words, split_header_words, Cookie, @@ -17,6 +19,7 @@ reach, is_HDN, domain_match, user_domain_match, request_path, request_port, request_host) +mswindows = (sys.platform == "win32") class DateTimeTests(unittest.TestCase): @@ -104,8 +107,7 @@ def test_http2time_formats(self): self.assertEqual(http2time(s.lower()), test_t, s.lower()) self.assertEqual(http2time(s.upper()), test_t, s.upper()) - def test_http2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Mandag 16. September 1996', @@ -120,12 +122,10 @@ def test_http2time_garbage(self): '08-01-3697739', '09 Feb 19942632 22:23:32 GMT', 'Wed, 09 Feb 1994834 22:23:32 GMT', - ]: - self.assertIsNone(http2time(test), - "http2time(%s) is not None\n" - "http2time(test) %s" % (test, http2time(test))) + ]) + def test_http2time_garbage(self, test): + self.assertIsNone(http2time(test)) - @unittest.skip("TODO: RUSTPYTHON, regressed to cubic complexity") def test_http2time_redos_regression_actually_completes(self): # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). # If we regress to cubic complexity, this test will take a very long time to succeed. @@ -149,9 +149,7 @@ def parse_date(text): self.assertEqual(parse_date("1994-02-03 19:45:29 +0530"), (1994, 2, 3, 14, 15, 29)) - def test_iso2time_formats(self): - # test iso2time for supported dates. - tests = [ + @support.subTests('s', [ '1994-02-03 00:00:00 -0000', # ISO 8601 format '1994-02-03 00:00:00 +0000', # ISO 8601 format '1994-02-03 00:00:00', # zone is optional @@ -164,16 +162,15 @@ def test_iso2time_formats(self): # A few tests with extra space at various places ' 1994-02-03 ', ' 1994-02-03T00:00:00 ', - ] - + ]) + def test_iso2time_formats(self, s): + # test iso2time for supported dates. test_t = 760233600 # assume broken POSIX counting of seconds - for s in tests: - self.assertEqual(iso2time(s), test_t, s) - self.assertEqual(iso2time(s.lower()), test_t, s.lower()) - self.assertEqual(iso2time(s.upper()), test_t, s.upper()) + self.assertEqual(iso2time(s), test_t, s) + self.assertEqual(iso2time(s.lower()), test_t, s.lower()) + self.assertEqual(iso2time(s.upper()), test_t, s.upper()) - def test_iso2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Thursday, 03-Feb-94 00:00:00 GMT', @@ -186,11 +183,10 @@ def test_iso2time_garbage(self): '01-01-1980 00:00:62', '01-01-1980T00:00:62', '19800101T250000Z', - ]: - self.assertIsNone(iso2time(test), - "iso2time(%r)" % test) + ]) + def test_iso2time_garbage(self, test): + self.assertIsNone(iso2time(test)) - @unittest.skip("TODO, RUSTPYTHON, regressed to quadratic complexity") def test_iso2time_performance_regression(self): # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. # If fixed, it should complete within a fraction of a second. @@ -200,24 +196,23 @@ def test_iso2time_performance_regression(self): class HeaderTests(unittest.TestCase): - def test_parse_ns_headers(self): - # quotes should be stripped - expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] - for hdr in [ + @support.subTests('hdr', [ 'foo=bar; expires=01 Jan 2040 22:23:32 GMT', 'foo=bar; expires="01 Jan 2040 22:23:32 GMT"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) - - def test_parse_ns_headers_version(self): - + ]) + def test_parse_ns_headers(self, hdr): # quotes should be stripped - expected = [[('foo', 'bar'), ('version', '1')]] - for hdr in [ + expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] + self.assertEqual(parse_ns_headers([hdr]), expected) + + @support.subTests('hdr', [ 'foo=bar; version="1"', 'foo=bar; Version="1"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) + ]) + def test_parse_ns_headers_version(self, hdr): + # quotes should be stripped + expected = [[('foo', 'bar'), ('version', '1')]] + self.assertEqual(parse_ns_headers([hdr]), expected) def test_parse_ns_headers_special_names(self): # names such as 'expires' are not special in first name=value pair @@ -233,8 +228,7 @@ def test_join_header_words(self): self.assertEqual(join_header_words([[]]), "") - def test_split_header_words(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", [[("foo", None)]]), ("foo=bar", [[("foo", "bar")]]), (" foo ", [[("foo", None)]]), @@ -251,24 +245,22 @@ def test_split_header_words(self): (r'foo; bar=baz, spam=, foo="\,\;\"", bar= ', [[("foo", None), ("bar", "baz")], [("spam", "")], [("foo", ',;"')], [("bar", "")]]), - ] - - for arg, expect in tests: - try: - result = split_header_words([arg]) - except: - import traceback, io - f = io.StringIO() - traceback.print_exc(None, f) - result = "(error -- traceback follows)\n\n%s" % f.getvalue() - self.assertEqual(result, expect, """ + ]) + def test_split_header_words(self, arg, expect): + try: + result = split_header_words([arg]) + except: + import traceback, io + f = io.StringIO() + traceback.print_exc(None, f) + result = "(error -- traceback follows)\n\n%s" % f.getvalue() + self.assertEqual(result, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' """ % (arg, expect, result)) - def test_roundtrip(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", "foo"), ("foo=bar", "foo=bar"), (" foo ", "foo"), @@ -277,23 +269,35 @@ def test_roundtrip(self): ("foo=bar;bar=baz", "foo=bar; bar=baz"), ('foo bar baz', "foo; bar; baz"), (r'foo="\"" bar="\\"', r'foo="\""; bar="\\"'), + ("föo=bär", 'föo="bär"'), ('foo,,,bar', 'foo, bar'), ('foo=bar,bar=baz', 'foo=bar, bar=baz'), + ("foo=\n", 'foo=""'), + ('foo="\n"', 'foo="\n"'), + ('foo=bar\n', 'foo=bar'), + ('foo="bar\n"', 'foo="bar\n"'), + ('foo=bar\nbaz', 'foo=bar; baz'), + ('foo="bar\nbaz"', 'foo="bar\nbaz"'), ('text/html; charset=iso-8859-1', - 'text/html; charset="iso-8859-1"'), + 'text/html; charset=iso-8859-1'), + + ('text/html; charset="iso-8859/1"', + 'text/html; charset="iso-8859/1"'), ('foo="bar"; port="80,81"; discard, bar=baz', 'foo=bar; port="80,81"; discard, bar=baz'), (r'Basic realm="\"foo\\\\bar\""', - r'Basic; realm="\"foo\\\\bar\""') - ] - - for arg, expect in tests: - input = split_header_words([arg]) - res = join_header_words(input) - self.assertEqual(res, expect, """ + r'Basic; realm="\"foo\\\\bar\""'), + + ('n; foo="foo;_", bar="foo,_"', + 'n; foo="foo;_", bar="foo,_"'), + ]) + def test_roundtrip(self, arg, expect): + input = split_header_words([arg]) + res = join_header_words(input) + self.assertEqual(res, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' @@ -337,9 +341,9 @@ def test_constructor_with_str(self): self.assertEqual(c.filename, filename) def test_constructor_with_path_like(self): - filename = pathlib.Path(os_helper.TESTFN) - c = LWPCookieJar(filename) - self.assertEqual(c.filename, os.fspath(filename)) + filename = os_helper.TESTFN + c = LWPCookieJar(os_helper.FakePath(filename)) + self.assertEqual(c.filename, filename) def test_constructor_with_none(self): c = LWPCookieJar(None) @@ -366,10 +370,63 @@ def test_lwp_valueless_cookie(self): c = LWPCookieJar() c.load(filename, ignore_discard=True) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) self.assertEqual(c._cookies["www.acme.com"]["/"]["boo"].value, None) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_lwp_filepermissions(self): + # Cookie file should only be readable by the creator + filename = os_helper.TESTFN + c = LWPCookieJar() + interact_netscape(c, "http://www.acme.com/", 'boo') + try: + c.save(filename, ignore_discard=True) + st = os.stat(filename) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) + finally: + os_helper.unlink(filename) + + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_mozilla_filepermissions(self): + # Cookie file should only be readable by the creator + filename = os_helper.TESTFN + c = MozillaCookieJar() + interact_netscape(c, "http://www.acme.com/", 'boo') + try: + c.save(filename, ignore_discard=True) + st = os.stat(filename) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) + finally: + os_helper.unlink(filename) + + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN @@ -393,8 +450,7 @@ def test_bad_magic(self): c = cookiejar_class() self.assertRaises(LoadError, c.load, filename) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) class CookieTests(unittest.TestCase): # XXX @@ -443,14 +499,7 @@ class CookieTests(unittest.TestCase): ## just the 7 special TLD's listed in their spec. And folks rely on ## that... - def test_domain_return_ok(self): - # test optimization: .domain_return_ok() should filter out most - # domains in the CookieJar before we try to access them (because that - # may require disk access -- in particular, with MSIECookieJar) - # This is only a rough check for performance reasons, so it's not too - # critical as long as it's sufficiently liberal. - pol = DefaultCookiePolicy() - for url, domain, ok in [ + @support.subTests('url,domain,ok', [ ("http://foo.bar.com/", "blah.com", False), ("http://foo.bar.com/", "rhubarb.blah.com", False), ("http://foo.bar.com/", "rhubarb.foo.bar.com", False), @@ -470,11 +519,18 @@ def test_domain_return_ok(self): ("http://foo/", ".local", True), ("http://barfoo.com", ".foo.com", False), ("http://barfoo.com", "foo.com", False), - ]: - request = urllib.request.Request(url) - r = pol.domain_return_ok(domain, request) - if ok: self.assertTrue(r) - else: self.assertFalse(r) + ]) + def test_domain_return_ok(self, url, domain, ok): + # test optimization: .domain_return_ok() should filter out most + # domains in the CookieJar before we try to access them (because that + # may require disk access -- in particular, with MSIECookieJar) + # This is only a rough check for performance reasons, so it's not too + # critical as long as it's sufficiently liberal. + pol = DefaultCookiePolicy() + request = urllib.request.Request(url) + r = pol.domain_return_ok(domain, request) + if ok: self.assertTrue(r) + else: self.assertFalse(r) def test_missing_value(self): # missing = sign in Cookie: header is regarded by Mozilla as a missing @@ -490,7 +546,7 @@ def test_missing_value(self): self.assertIsNone(cookie.value) self.assertEqual(cookie.name, '"spam"') self.assertEqual(lwp_cookie_str(cookie), ( - r'"spam"; path="/foo/"; domain="www.acme.com"; ' + r'"spam"; path="/foo/"; domain=www.acme.com; ' 'path_spec; discard; version=0')) old_str = repr(c) c.save(ignore_expires=True, ignore_discard=True) @@ -498,7 +554,7 @@ def test_missing_value(self): c = MozillaCookieJar(filename) c.revert(ignore_expires=True, ignore_discard=True) finally: - os.unlink(c.filename) + os_helper.unlink(c.filename) # cookies unchanged apart from lost info re. whether path was specified self.assertEqual( repr(c), @@ -508,10 +564,7 @@ def test_missing_value(self): self.assertEqual(interact_netscape(c, "http://www.acme.com/foo/"), '"spam"; eggs') - def test_rfc2109_handling(self): - # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, - # dependent on policy settings - for rfc2109_as_netscape, rfc2965, version in [ + @support.subTests('rfc2109_as_netscape,rfc2965,version', [ # default according to rfc2965 if not explicitly specified (None, False, 0), (None, True, 1), @@ -520,24 +573,27 @@ def test_rfc2109_handling(self): (False, True, 1), (True, False, 0), (True, True, 0), - ]: - policy = DefaultCookiePolicy( - rfc2109_as_netscape=rfc2109_as_netscape, - rfc2965=rfc2965) - c = CookieJar(policy) - interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") - try: - cookie = c._cookies["www.example.com"]["/"]["ni"] - except KeyError: - self.assertIsNone(version) # didn't expect a stored cookie - else: - self.assertEqual(cookie.version, version) - # 2965 cookies are unaffected - interact_2965(c, "http://www.example.com/", - "foo=bar; Version=1") - if rfc2965: - cookie2965 = c._cookies["www.example.com"]["/"]["foo"] - self.assertEqual(cookie2965.version, 1) + ]) + def test_rfc2109_handling(self, rfc2109_as_netscape, rfc2965, version): + # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, + # dependent on policy settings + policy = DefaultCookiePolicy( + rfc2109_as_netscape=rfc2109_as_netscape, + rfc2965=rfc2965) + c = CookieJar(policy) + interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") + try: + cookie = c._cookies["www.example.com"]["/"]["ni"] + except KeyError: + self.assertIsNone(version) # didn't expect a stored cookie + else: + self.assertEqual(cookie.version, version) + # 2965 cookies are unaffected + interact_2965(c, "http://www.example.com/", + "foo=bar; Version=1") + if rfc2965: + cookie2965 = c._cookies["www.example.com"]["/"]["foo"] + self.assertEqual(cookie2965.version, 1) def test_ns_parser(self): c = CookieJar() @@ -598,8 +654,6 @@ def test_ns_parser_special_names(self): self.assertIn('expires', cookies) self.assertIn('version', cookies) - # TODO: RUSTPYTHON; need to update http library to remove warnings - @unittest.expectedFailure def test_expires(self): # if expires is in future, keep cookie... c = CookieJar() @@ -707,8 +761,7 @@ def test_default_path_with_query(self): # Cookie is sent back to the same URI. self.assertEqual(interact_netscape(cj, uri), value) - def test_escape_path(self): - cases = [ + @support.subTests('arg,result', [ # quoted safe ("/foo%2f/bar", "/foo%2F/bar"), ("/foo%2F/bar", "/foo%2F/bar"), @@ -728,9 +781,9 @@ def test_escape_path(self): ("/foo/bar\u00fc", "/foo/bar%C3%BC"), # UTF-8 encoded # unicode ("/foo/bar\uabcd", "/foo/bar%EA%AF%8D"), # UTF-8 encoded - ] - for arg, result in cases: - self.assertEqual(escape_path(arg), result) + ]) + def test_escape_path(self, arg, result): + self.assertEqual(escape_path(arg), result) def test_request_path(self): # with parameters @@ -924,6 +977,48 @@ def test_two_component_domain_ns(self): ## self.assertEqual(len(c), 2) self.assertEqual(len(c), 4) + def test_localhost_domain(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=localhost;") + + self.assertEqual(len(c), 1) + + def test_localhost_domain_contents(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=localhost;") + + self.assertEqual(c._cookies[".localhost"]["/"]["foo"].value, "bar") + + def test_localhost_domain_contents_2(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar;") + + self.assertEqual(c._cookies["localhost.local"]["/"]["foo"].value, "bar") + + def test_evil_nonlocal_domain(self): + c = CookieJar() + + interact_netscape(c, "http://evil.com", "foo=bar; domain=.localhost") + + self.assertEqual(len(c), 0) + + def test_evil_local_domain(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=.evil.com") + + self.assertEqual(len(c), 0) + + def test_evil_local_domain_2(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=.someother.local") + + self.assertEqual(len(c), 0) + def test_two_component_domain_rfc2965(self): pol = DefaultCookiePolicy(rfc2965=True) c = CookieJar(pol) @@ -1255,11 +1350,11 @@ def test_Cookie_iterator(self): r'port="90,100, 80,8080"; ' r'max-age=100; Comment = "Just kidding! (\"|\\\\) "') - versions = [1, 1, 1, 0, 1] - names = ["bang", "foo", "foo", "spam", "foo"] - domains = [".sol.no", "blah.spam.org", "www.acme.com", - "www.acme.com", "www.acme.com"] - paths = ["/", "/", "/", "/blah", "/blah/"] + versions = [1, 0, 1, 1, 1] + names = ["foo", "spam", "foo", "foo", "bang"] + domains = ["blah.spam.org", "www.acme.com", "www.acme.com", + "www.acme.com", ".sol.no"] + paths = ["/", "/blah", "/blah/", "/", "/"] for i in range(4): i = 0 @@ -1332,7 +1427,7 @@ def cookiejar_from_cookie_headers(headers): self.assertIsNone(cookie.expires) -class LWPCookieTests(unittest.TestCase): +class LWPCookieTests(unittest.TestCase, ExtraAssertions): # Tests taken from libwww-perl, with a few modifications and additions. def test_netscape_example_1(self): @@ -1424,7 +1519,7 @@ def test_netscape_example_1(self): h = req.get_header("Cookie") self.assertIn("PART_NUMBER=ROCKET_LAUNCHER_0001", h) self.assertIn("CUSTOMER=WILE_E_COYOTE", h) - self.assertTrue(h.startswith("SHIPPING=FEDEX;")) + self.assertStartsWith(h, "SHIPPING=FEDEX;") def test_netscape_example_2(self): # Second Example transaction sequence: @@ -1728,8 +1823,7 @@ def test_rejection(self): c = LWPCookieJar(policy=pol) c.load(filename, ignore_discard=True) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) self.assertEqual(old, repr(c)) @@ -1788,8 +1882,7 @@ def save_and_restore(cj, ignore_discard): DefaultCookiePolicy(rfc2965=True)) new_c.load(ignore_discard=ignore_discard) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) return new_c new_c = save_and_restore(c, True) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 6072c7e15e9..3e0b4d1d5ca 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,13 +1,15 @@ # Simple test suite for http/cookies.py import copy -from test.support import run_unittest, run_doctest import unittest +import doctest from http import cookies import pickle +from test import support +from test.support.testcase import ExtraAssertions -class CookieTests(unittest.TestCase): +class CookieTests(unittest.TestCase, ExtraAssertions): def test_basic(self): cases = [ @@ -58,6 +60,90 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_obsolete_rfc850_date_format(self): + # Test cases with different days and dates in obsolete RFC 850 format + test_cases = [ + # from RFC 850, change EST to GMT + # https://datatracker.ietf.org/doc/html/rfc850#section-2 + { + 'data': 'key=value; expires=Saturday, 01-Jan-83 00:00:00 GMT', + 'output': 'Saturday, 01-Jan-83 00:00:00 GMT' + }, + { + 'data': 'key=value; expires=Friday, 19-Nov-82 16:59:30 GMT', + 'output': 'Friday, 19-Nov-82 16:59:30 GMT' + }, + # from RFC 9110 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.7-6 + { + 'data': 'key=value; expires=Sunday, 06-Nov-94 08:49:37 GMT', + 'output': 'Sunday, 06-Nov-94 08:49:37 GMT' + }, + # other test cases + { + 'data': 'key=value; expires=Wednesday, 09-Nov-94 08:49:37 GMT', + 'output': 'Wednesday, 09-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Friday, 11-Nov-94 08:49:37 GMT', + 'output': 'Friday, 11-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Monday, 14-Nov-94 08:49:37 GMT', + 'output': 'Monday, 14-Nov-94 08:49:37 GMT' + }, + ] + + for case in test_cases: + with self.subTest(data=case['data']): + C = cookies.SimpleCookie() + C.load(case['data']) + + # Extract the cookie name from the data string + cookie_name = case['data'].split('=')[0] + + # Check if the cookie is loaded correctly + self.assertIn(cookie_name, C) + self.assertEqual(C[cookie_name].get('expires'), case['output']) + + def test_unquote(self): + cases = [ + (r'a="b=\""', 'b="'), + (r'a="b=\\"', 'b=\\'), + (r'a="b=\="', 'b=='), + (r'a="b=\n"', 'b=n'), + (r'a="b=\042"', 'b="'), + (r'a="b=\134"', 'b=\\'), + (r'a="b=\377"', 'b=\xff'), + (r'a="b=\400"', 'b=400'), + (r'a="b=\42"', 'b=42'), + (r'a="b=\\042"', 'b=\\042'), + (r'a="b=\\134"', 'b=\\134'), + (r'a="b=\\\""', 'b=\\"'), + (r'a="b=\\\042"', 'b=\\"'), + (r'a="b=\134\""', 'b=\\"'), + (r'a="b=\134\042"', 'b=\\"'), + ] + for encoded, decoded in cases: + with self.subTest(encoded): + C = cookies.SimpleCookie() + C.load(encoded) + self.assertEqual(C['a'].value, decoded) + + @support.requires_resource('cpu') + def test_unquote_large(self): + #n = 10**6 + n = 10**4 # XXX: RUSTPYTHON; This takes more than 10 minutes to run. lower to 4 + for encoded in r'\\', r'\134': + with self.subTest(encoded): + data = 'a="b=' + encoded*n + ';"' + C = cookies.SimpleCookie() + C.load(data) + value = C['a'].value + self.assertEqual(value[:3], 'b=\\') + self.assertEqual(value[-2:], '\\;') + self.assertEqual(len(value), n + 3) + def test_load(self): C = cookies.SimpleCookie() C.load('Customer="WILE_E_COYOTE"; Version=1; Path=/acme') @@ -96,7 +182,7 @@ def test_special_attrs(self): C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"') C['Customer']['expires'] = 0 # can't test exact output, it always depends on current date/time - self.assertTrue(C.output().endswith('GMT')) + self.assertEndsWith(C.output(), 'GMT') # loading 'expires' C = cookies.SimpleCookie() @@ -479,9 +565,11 @@ def test_repr(self): r'Set-Cookie: key=coded_val; ' r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+') -def test_main(): - run_unittest(CookieTests, MorselTests) - run_doctest(cookies) + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(cookies)) + return tests + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index d4a6eefe322..5267d2fe011 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1,4 +1,4 @@ -import sys +import enum import errno from http import client, HTTPStatus import io @@ -8,7 +8,6 @@ import re import socket import threading -import warnings import unittest from unittest import mock @@ -17,16 +16,19 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper +from test.support.testcase import ExtraAssertions +support.requires_working_socket(module=True) here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' -CERT_localhost = os.path.join(here, 'certdata/keycert.pem') +CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem') # Self-signed cert file for 'fakehostname' -CERT_fakehostname = os.path.join(here, 'certdata/keycert2.pem') +CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem') # Self-signed cert file for self-signed.pythontest.net -CERT_selfsigned_pythontestdotnet = os.path.join(here, 'certdata/selfsigned_pythontestdotnet.pem') +CERT_selfsigned_pythontestdotnet = os.path.join( + here, 'certdata', 'selfsigned_pythontestdotnet.pem', +) # constants for testing chunked encoding chunked_start = ( @@ -133,7 +135,7 @@ def connect(self): def create_connection(self, *pos, **kw): return FakeSocket(*self.fake_socket_args) -class HeaderTests(TestCase): +class HeaderTests(TestCase, ExtraAssertions): def test_auto_headers(self): # Some headers are added automatically, but should not be added by # .request() if they are explicitly set. @@ -272,7 +274,7 @@ def test_ipv6host_header(self): sock = FakeSocket('') conn.sock = sock conn.request('GET', '/foo') - self.assertTrue(sock.data.startswith(expected)) + self.assertStartsWith(sock.data, expected) expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ b'Accept-Encoding: identity\r\n\r\n' @@ -280,7 +282,23 @@ def test_ipv6host_header(self): sock = FakeSocket('') conn.sock = sock conn.request('GET', '/foo') - self.assertTrue(sock.data.startswith(expected)) + self.assertStartsWith(sock.data, expected) + + expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[fe80::%2]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertStartsWith(sock.data, expected) + + expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[fe80::%2]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertStartsWith(sock.data, expected) def test_malformed_headers_coped_with(self): # Issue 19996 @@ -318,9 +336,9 @@ def test_parse_all_octets(self): self.assertIsNotNone(resp.getheader('obs-text')) self.assertIn('obs-text', resp.msg) for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): - self.assertTrue(folded.startswith('text')) + self.assertStartsWith(folded, 'text') self.assertIn(' folded with space', folded) - self.assertTrue(folded.endswith('folded with tab')) + self.assertEndsWith(folded, 'folded with tab') def test_invalid_headers(self): conn = client.HTTPConnection('example.com') @@ -520,11 +538,203 @@ def _parse_chunked(self, data): return b''.join(body) -class BasicTest(TestCase): +class BasicTest(TestCase, ExtraAssertions): def test_dir_with_added_behavior_on_status(self): # see issue40084 self.assertTrue({'description', 'name', 'phrase', 'value'} <= set(dir(HTTPStatus(404)))) + def test_simple_httpstatus(self): + class CheckedHTTPStatus(enum.IntEnum): + """HTTP status codes and reason phrases + + Status codes from the following RFCs are all observed: + + * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 6585: Additional HTTP Status Codes + * RFC 3229: Delta encoding in HTTP + * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 + * RFC 5842: Binding Extensions to WebDAV + * RFC 7238: Permanent Redirect + * RFC 2295: Transparent Content Negotiation in HTTP + * RFC 2774: An HTTP Extension Framework + * RFC 7725: An HTTP Status Code to Report Legal Obstacles + * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) + * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) + * RFC 8297: An HTTP Status Code for Indicating Hints + * RFC 8470: Using Early Data in HTTP + """ + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + + obj.phrase = phrase + obj.description = description + return obj + + @property + def is_informational(self): + return 100 <= self <= 199 + + @property + def is_success(self): + return 200 <= self <= 299 + + @property + def is_redirection(self): + return 300 <= self <= 399 + + @property + def is_client_error(self): + return 400 <= self <= 499 + + @property + def is_server_error(self): + return 500 <= self <= 599 + + # informational + CONTINUE = 100, 'Continue', 'Request received, please continue' + SWITCHING_PROTOCOLS = (101, 'Switching Protocols', + 'Switching to new protocol; obey Upgrade header') + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + # success + OK = 200, 'OK', 'Request fulfilled, document follows' + CREATED = 201, 'Created', 'Document created, URL follows' + ACCEPTED = (202, 'Accepted', + 'Request accepted, processing continues off-line') + NON_AUTHORITATIVE_INFORMATION = (203, + 'Non-Authoritative Information', 'Request fulfilled from cache') + NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' + RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' + PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' + MULTI_STATUS = 207, 'Multi-Status' + ALREADY_REPORTED = 208, 'Already Reported' + IM_USED = 226, 'IM Used' + # redirection + MULTIPLE_CHOICES = (300, 'Multiple Choices', + 'Object has several resources -- see URI list') + MOVED_PERMANENTLY = (301, 'Moved Permanently', + 'Object moved permanently -- see URI list') + FOUND = 302, 'Found', 'Object moved temporarily -- see URI list' + SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list' + NOT_MODIFIED = (304, 'Not Modified', + 'Document has not changed since given time') + USE_PROXY = (305, 'Use Proxy', + 'You must use proxy specified in Location to access this resource') + TEMPORARY_REDIRECT = (307, 'Temporary Redirect', + 'Object moved temporarily -- see URI list') + PERMANENT_REDIRECT = (308, 'Permanent Redirect', + 'Object moved permanently -- see URI list') + # client error + BAD_REQUEST = (400, 'Bad Request', + 'Bad request syntax or unsupported method') + UNAUTHORIZED = (401, 'Unauthorized', + 'No permission -- see authorization schemes') + PAYMENT_REQUIRED = (402, 'Payment Required', + 'No payment -- see charging schemes') + FORBIDDEN = (403, 'Forbidden', + 'Request forbidden -- authorization will not help') + NOT_FOUND = (404, 'Not Found', + 'Nothing matches the given URI') + METHOD_NOT_ALLOWED = (405, 'Method Not Allowed', + 'Specified method is invalid for this resource') + NOT_ACCEPTABLE = (406, 'Not Acceptable', + 'URI not available in preferred format') + PROXY_AUTHENTICATION_REQUIRED = (407, + 'Proxy Authentication Required', + 'You must authenticate with this proxy before proceeding') + REQUEST_TIMEOUT = (408, 'Request Timeout', + 'Request timed out; try again later') + CONFLICT = 409, 'Conflict', 'Request conflict' + GONE = (410, 'Gone', + 'URI no longer exists and has been permanently removed') + LENGTH_REQUIRED = (411, 'Length Required', + 'Client must specify Content-Length') + PRECONDITION_FAILED = (412, 'Precondition Failed', + 'Precondition in headers is false') + CONTENT_TOO_LARGE = (413, 'Content Too Large', + 'Content is too large') + REQUEST_ENTITY_TOO_LARGE = CONTENT_TOO_LARGE + URI_TOO_LONG = (414, 'URI Too Long', 'URI is too long') + REQUEST_URI_TOO_LONG = URI_TOO_LONG + UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', + 'Entity body in unsupported format') + RANGE_NOT_SATISFIABLE = (416, + 'Range Not Satisfiable', + 'Cannot satisfy request range') + REQUESTED_RANGE_NOT_SATISFIABLE = RANGE_NOT_SATISFIABLE + EXPECTATION_FAILED = (417, 'Expectation Failed', + 'Expect condition could not be satisfied') + IM_A_TEAPOT = (418, 'I\'m a Teapot', + 'Server refuses to brew coffee because it is a teapot.') + MISDIRECTED_REQUEST = (421, 'Misdirected Request', + 'Server is not able to produce a response') + UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT + LOCKED = 423, 'Locked' + FAILED_DEPENDENCY = 424, 'Failed Dependency' + TOO_EARLY = 425, 'Too Early' + UPGRADE_REQUIRED = 426, 'Upgrade Required' + PRECONDITION_REQUIRED = (428, 'Precondition Required', + 'The origin server requires the request to be conditional') + TOO_MANY_REQUESTS = (429, 'Too Many Requests', + 'The user has sent too many requests in ' + 'a given amount of time ("rate limiting")') + REQUEST_HEADER_FIELDS_TOO_LARGE = (431, + 'Request Header Fields Too Large', + 'The server is unwilling to process the request because its header ' + 'fields are too large') + UNAVAILABLE_FOR_LEGAL_REASONS = (451, + 'Unavailable For Legal Reasons', + 'The server is denying access to the ' + 'resource as a consequence of a legal demand') + # server errors + INTERNAL_SERVER_ERROR = (500, 'Internal Server Error', + 'Server got itself in trouble') + NOT_IMPLEMENTED = (501, 'Not Implemented', + 'Server does not support this operation') + BAD_GATEWAY = (502, 'Bad Gateway', + 'Invalid responses from another server/proxy') + SERVICE_UNAVAILABLE = (503, 'Service Unavailable', + 'The server cannot process the request due to a high load') + GATEWAY_TIMEOUT = (504, 'Gateway Timeout', + 'The gateway server did not receive a timely response') + HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', + 'Cannot fulfill request') + VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' + INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' + LOOP_DETECTED = 508, 'Loop Detected' + NOT_EXTENDED = 510, 'Not Extended' + NETWORK_AUTHENTICATION_REQUIRED = (511, + 'Network Authentication Required', + 'The client needs to authenticate to gain network access') + enum._test_simple_enum(CheckedHTTPStatus, HTTPStatus) + + def test_httpstatus_range(self): + """Checks that the statuses are in the 100-599 range""" + + for member in HTTPStatus.__members__.values(): + self.assertGreaterEqual(member, 100) + self.assertLessEqual(member, 599) + + def test_httpstatus_category(self): + """Checks that the statuses belong to the standard categories""" + + categories = ( + ((100, 199), "is_informational"), + ((200, 299), "is_success"), + ((300, 399), "is_redirection"), + ((400, 499), "is_client_error"), + ((500, 599), "is_server_error"), + ) + for member in HTTPStatus.__members__.values(): + for (lower, upper), category in categories: + category_indicator = getattr(member, category) + if lower <= member <= upper: + self.assertTrue(category_indicator) + else: + self.assertFalse(category_indicator) + def test_status_lines(self): # Test HTTP status lines @@ -780,8 +990,7 @@ def test_send_file(self): sock = FakeSocket(body) conn.sock = sock conn.request('GET', '/foo', body) - self.assertTrue(sock.data.startswith(expected), '%r != %r' % - (sock.data[:len(expected)], expected)) + self.assertStartsWith(sock.data, expected) def test_send(self): expected = b'this is a test this is only a test' @@ -872,6 +1081,25 @@ def test_chunked(self): self.assertEqual(resp.read(), expected) resp.close() + # Explicit full read + for n in (-123, -1, None): + with self.subTest('full read', n=n): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read(n), expected) + resp.close() + + # Read first chunk + with self.subTest('read1(-1)'): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read1(-1), b"hello worl") + resp.close() + # Various read sizes for n in range(1, 12): sock = FakeSocket(chunked_start + last_chunk + chunked_end) @@ -1227,6 +1455,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation @@ -1266,7 +1560,7 @@ def _encode_request(self, str_url): conn.putrequest('GET', '/☃') -class ExtendedReadTest(TestCase): +class ExtendedReadTest(TestCase, ExtraAssertions): """ Test peek(), read1(), readline() """ @@ -1325,7 +1619,7 @@ def mypeek(n=-1): # then unbounded peek p2 = resp.peek() self.assertGreaterEqual(len(p2), len(p)) - self.assertTrue(p2.startswith(p)) + self.assertStartsWith(p2, p) next = resp.read(len(p2)) self.assertEqual(next, p2) else: @@ -1340,18 +1634,22 @@ def test_readline(self): resp = self.resp self._verify_readline(self.resp.readline, self.lines_expected) - def _verify_readline(self, readline, expected): + def test_readline_without_limit(self): + self._verify_readline(self.resp.readline, self.lines_expected, limit=-1) + + def _verify_readline(self, readline, expected, limit=5): all = [] while True: # short readlines - line = readline(5) + line = readline(limit) if line and line != b"foo": if len(line) < 5: - self.assertTrue(line.endswith(b"\n")) + self.assertEndsWith(line, b"\n") all.append(line) if not line: break self.assertEqual(b"".join(all), expected) + self.assertTrue(self.resp.isclosed()) def test_read1(self): resp = self.resp @@ -1371,6 +1669,7 @@ def test_read1_unbounded(self): break all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_bounded(self): resp = self.resp @@ -1382,15 +1681,22 @@ def test_read1_bounded(self): self.assertLessEqual(len(data), 10) all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_0(self): self.assertEqual(self.resp.read1(0), b"") + self.assertFalse(self.resp.isclosed()) def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) +class ExtendedReadTestContentLengthKnown(ExtendedReadTest): + _header, _body = ExtendedReadTest.lines.split('\r\n\r\n', 1) + lines = _header + f'\r\nContent-Length: {len(_body)}\r\n\r\n' + _body + + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode @@ -1447,7 +1753,7 @@ def readline(self, limit): raise -class OfflineTest(TestCase): +class OfflineTest(TestCase, ExtraAssertions): def test_all(self): # Documented objects defined in the module should be in __all__ expected = {"responses"} # Allowlist documented dict() object @@ -1500,13 +1806,17 @@ def test_client_constants(self): 'GONE', 'LENGTH_REQUIRED', 'PRECONDITION_FAILED', + 'CONTENT_TOO_LARGE', 'REQUEST_ENTITY_TOO_LARGE', + 'URI_TOO_LONG', 'REQUEST_URI_TOO_LONG', 'UNSUPPORTED_MEDIA_TYPE', + 'RANGE_NOT_SATISFIABLE', 'REQUESTED_RANGE_NOT_SATISFIABLE', 'EXPECTATION_FAILED', 'IM_A_TEAPOT', 'MISDIRECTED_REQUEST', + 'UNPROCESSABLE_CONTENT', 'UNPROCESSABLE_ENTITY', 'LOCKED', 'FAILED_DEPENDENCY', @@ -1529,7 +1839,7 @@ def test_client_constants(self): ] for const in expected: with self.subTest(constant=const): - self.assertTrue(hasattr(client, const)) + self.assertHasAttr(client, const) class SourceAddressTest(TestCase): @@ -1766,6 +2076,7 @@ def test_networked_good_cert(self): h.close() self.assertIn('nginx', server_string) + @support.requires_resource('walltime') def test_networked_bad_cert(self): # We feed a "CA" cert that is unrelated to the server's cert import ssl @@ -1778,7 +2089,6 @@ def test_networked_bad_cert(self): h.request('GET', '/') self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') - @unittest.skipIf(sys.platform == 'darwin', 'Occasionally success on macOS') def test_local_unknown_cert(self): # The custom cert isn't known to the default trust bundle import ssl @@ -1789,7 +2099,7 @@ def test_local_unknown_cert(self): self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') def test_local_good_hostname(self): - # The (valid) cert validates the HTTP hostname + # The (valid) cert validates the HTTPS hostname import ssl server = self.make_server(CERT_localhost) context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -1802,7 +2112,7 @@ def test_local_good_hostname(self): self.assertEqual(resp.status, 404) def test_local_bad_hostname(self): - # The (valid) cert doesn't validate the HTTP hostname + # The (valid) cert doesn't validate the HTTPS hostname import ssl server = self.make_server(CERT_fakehostname) context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -1810,38 +2120,21 @@ def test_local_bad_hostname(self): h = client.HTTPSConnection('localhost', server.port, context=context) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') - # Same with explicit check_hostname=True - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=True) + + # Same with explicit context.check_hostname=True + context.check_hostname = True + h = client.HTTPSConnection('localhost', server.port, context=context) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') - # With check_hostname=False, the mismatching is ignored - context.check_hostname = False - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=False) - h.request('GET', '/nonexistent') - resp = h.getresponse() - resp.close() - h.close() - self.assertEqual(resp.status, 404) - # The context's check_hostname setting is used if one isn't passed to - # HTTPSConnection. + + # With context.check_hostname=False, the mismatching is ignored context.check_hostname = False h = client.HTTPSConnection('localhost', server.port, context=context) h.request('GET', '/nonexistent') resp = h.getresponse() - self.assertEqual(resp.status, 404) resp.close() h.close() - # Passing check_hostname to HTTPSConnection should override the - # context's setting. - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=True) - with self.assertRaises(ssl.CertificateError): - h.request('GET', '/') + self.assertEqual(resp.status, 404) @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), 'http.client.HTTPSConnection not available') @@ -1877,11 +2170,9 @@ def test_tls13_pha(self): self.assertIs(h._context, context) self.assertFalse(h._context.post_handshake_auth) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'key_file, cert_file and check_hostname are deprecated', - DeprecationWarning) - h = client.HTTPSConnection('localhost', 443, context=context, - cert_file=CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT, cert_file=CERT_localhost) + context.post_handshake_auth = True + h = client.HTTPSConnection('localhost', 443, context=context) self.assertTrue(h._context.post_handshake_auth) @@ -2016,14 +2307,15 @@ def test_getting_header_defaultint(self): header = self.resp.getheader('No-Such-Header',default=42) self.assertEqual(header, 42) -class TunnelTests(TestCase): +class TunnelTests(TestCase, ExtraAssertions): def setUp(self): response_text = ( - 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n\r\n' # Reply to CONNECT 'HTTP/1.1 200 OK\r\n' # Reply to HEAD 'Content-Length: 42\r\n\r\n' ) self.host = 'proxy.com' + self.port = client.HTTP_PORT self.conn = client.HTTPConnection(self.host) self.conn._create_connection = self._create_connection(response_text) @@ -2035,15 +2327,45 @@ def create_connection(address, timeout=None, source_address=None): return FakeSocket(response_text, host=address[0], port=address[1]) return create_connection - def test_set_tunnel_host_port_headers(self): + def test_set_tunnel_host_port_headers_add_host_missing(self): tunnel_host = 'destination.com' tunnel_port = 8888 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + tunnel_headers_after = tunnel_headers.copy() + tunnel_headers_after['Host'] = '%s:%d' % (tunnel_host, tunnel_port) self.conn.set_tunnel(tunnel_host, port=tunnel_port, headers=tunnel_headers) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) - self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers_after) + + def test_set_tunnel_host_port_headers_set_host_identical(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % (tunnel_host, tunnel_port)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_set_tunnel_host_port_headers_set_host_different(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % ('example.com', 4200)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) self.assertEqual(self.conn._tunnel_host, tunnel_host) self.assertEqual(self.conn._tunnel_port, tunnel_port) self.assertEqual(self.conn._tunnel_headers, tunnel_headers) @@ -2055,17 +2377,96 @@ def test_disallow_set_tunnel_after_connect(self): 'destination.com') def test_connect_with_tunnel(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_default_port(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_nonstandard_port(self): + d = { + b'host': b'destination.com', + b'port': 8888, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s:%(port)d\r\n' % d, + self.conn.sock.data) + + # This request is not RFC-valid, but it's been possible with the library + # for years, so don't break it unexpectedly... This also tests + # case-insensitivity when injecting Host: headers if they're missing. + def test_connect_with_tunnel_with_different_host_header(self): + d = { + b'host': b'destination.com', + b'tunnel_host_header': b'example.com:9876', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel( + d[b'host'].decode('ascii'), + headers={'HOST': d[b'tunnel_host_header'].decode('ascii')}) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'HOST: %(tunnel_host_header)s\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_different_host(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_idna(self): + dest = '\u03b4\u03c0\u03b8.gr' + dest_port = b'%s:%d' % (dest.encode('idna'), client.HTTP_PORT) + expected = b'CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n' % ( + dest_port, dest_port) + self.conn.set_tunnel(dest) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - # issue22095 - self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) - - # This test should be removed when CONNECT gets the HTTP/1.1 blessing - self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + self.assertIn(expected, self.conn.sock.data) def test_tunnel_connect_single_send_connection_setup(self): """Regresstion test for https://bugs.python.org/issue43332.""" @@ -2080,17 +2481,39 @@ def test_tunnel_connect_single_send_connection_setup(self): msg=f'unexpected number of send calls: {mock_send.mock_calls}') proxy_setup_data_sent = mock_send.mock_calls[0][1][0] self.assertIn(b'CONNECT destination.com', proxy_setup_data_sent) - self.assertTrue( - proxy_setup_data_sent.endswith(b'\r\n\r\n'), + self.assertEndsWith(proxy_setup_data_sent, b'\r\n\r\n', msg=f'unexpected proxy data sent {proxy_setup_data_sent!r}') def test_connect_put_request(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_put_request_ipv6(self): + self.conn.set_tunnel('[1:2:3::4]', 1234) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + + def test_connect_put_request_ipv6_port(self): + self.conn.set_tunnel('[1:2:3::4]:1234') self.conn.request('PUT', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' @@ -2105,6 +2528,56 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIn(expected_header, headers.items()) + + def test_no_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIsNone(headers) + + def test_tunnel_leak(self): + sock = None + + def _create_connection(address, timeout=None, source_address=None): + nonlocal sock + sock = FakeSocket( + 'HTTP/1.1 404 NOT FOUND\r\n\r\n', + host=address[0], + port=address[1], + ) + return sock + + self.conn._create_connection = _create_connection + self.conn.set_tunnel('destination.com') + exc = None + try: + self.conn.request('HEAD', '/', '') + except OSError as e: + # keeping a reference to exc keeps response alive in the traceback + exc = e + self.assertIsNotNone(exc) + self.assertTrue(sock.file_closed) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index cd689492ca3..0022e0e5d71 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -8,6 +8,7 @@ SimpleHTTPRequestHandler, CGIHTTPRequestHandler from http import server, HTTPStatus +import contextlib import os import socket import sys @@ -26,13 +27,16 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support -from test.support import os_helper -from test.support import threading_helper +from test.support import ( + is_apple, os_helper, requires_subprocess, threading_helper +) +from test.support.testcase import ExtraAssertions +support.requires_working_socket(module=True) class NoLogRequestHandler: def log_message(self, *args): @@ -64,7 +68,7 @@ def stop(self): self.join() -class BaseTestCase(unittest.TestCase): +class BaseTestCase(unittest.TestCase, ExtraAssertions): def setUp(self): self._threads = threading_helper.threading_setup() os.environ = os_helper.EnvironmentVarGuard() @@ -163,6 +167,27 @@ def test_version_digits(self): res = self.con.getresponse() self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_signs_and_underscores(self): + self.con._http_vsn_str = 'HTTP/-9_9_9.+9_9_9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_major_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/909876543210.0' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_minor_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/1.909876543210' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_none_get(self): self.con._http_vsn_str = '' self.con.putrequest('GET', '/') @@ -292,6 +317,44 @@ def test_head_via_send_error(self): self.assertEqual(b'', data) +class HTTP09ServerTestCase(BaseTestCase): + + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + """Request handler for HTTP/0.9 server.""" + + def do_GET(self): + self.wfile.write(f'OK: here is {self.path}\r\n'.encode()) + + def setUp(self): + super().setUp() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = self.enterContext(self.sock) + self.sock.connect((self.HOST, self.PORT)) + + def test_simple_get(self): + self.sock.send(b'GET /index.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /index.html\r\n") + + def test_invalid_request(self): + self.sock.send(b'POST /index.html\r\n') + res = self.sock.recv(1024) + self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) + + def test_single_request(self): + self.sock.send(b'GET /foo.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /foo.html\r\n") + + # Ignore errors if the connection is already closed, + # as this is the expected behavior of HTTP/0.9. + with contextlib.suppress(OSError): + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server should not process our request. + self.assertEqual(res, b'') + + class RequestHandlerLoggingTestCase(BaseTestCase): class request_handler(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' @@ -312,8 +375,7 @@ def test_get(self): self.con.request('GET', '/') self.con.getresponse() - self.assertTrue( - err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n')) + self.assertEndsWith(err.getvalue(), '"GET / HTTP/1.1" 200 -\n') def test_err(self): self.con = http.client.HTTPConnection(self.HOST, self.PORT) @@ -324,8 +386,8 @@ def test_err(self): self.con.getresponse() lines = err.getvalue().split('\n') - self.assertTrue(lines[0].endswith('code 404, message File not found')) - self.assertTrue(lines[1].endswith('"ERROR / HTTP/1.1" 404 -')) + self.assertEndsWith(lines[0], 'code 404, message File not found') + self.assertEndsWith(lines[1], '"ERROR / HTTP/1.1" 404 -') class SimpleHTTPServerTestCase(BaseTestCase): @@ -333,7 +395,7 @@ class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): pass def setUp(self): - BaseTestCase.setUp(self) + super().setUp() self.cwd = os.getcwd() basetempdir = tempfile.gettempdir() os.chdir(basetempdir) @@ -361,7 +423,7 @@ def tearDown(self): except: pass finally: - BaseTestCase.tearDown(self) + super().tearDown() def check_status_and_reason(self, response, status, data=None): def close_conn(): @@ -388,34 +450,169 @@ def close_conn(): reader.close() return body - @unittest.skipIf(sys.platform == 'darwin', - 'undecodable name cannot always be decoded on macOS') - @unittest.skipIf(sys.platform == 'win32', - 'undecodable name cannot be decoded on win32') - @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, - 'need os_helper.TESTFN_UNDECODABLE') - def test_undecodable_filename(self): + def check_list_dir_dirname(self, dirname, quotedname=None): + fullpath = os.path.join(self.tempdir, dirname) + try: + os.mkdir(os.path.join(self.tempdir, dirname)) + except (OSError, UnicodeEncodeError): + self.skipTest(f'Can not create directory {dirname!a} ' + f'on current file system') + + if quotedname is None: + quotedname = urllib.parse.quote(dirname, errors='surrogatepass') + response = self.request(self.base_url + '/' + quotedname + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + displaypath = html.escape(f'{self.base_url}/{dirname}/', quote=False) enc = sys.getfilesystemencoding() - filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' - with open(os.path.join(self.tempdir, filename), 'wb') as f: - f.write(os_helper.TESTFN_UNDECODABLE) + prefix = f'listing for {displaypath}</'.encode(enc, 'surrogateescape') + self.assertIn(prefix + b'title>', body) + self.assertIn(prefix + b'h1>', body) + + def check_list_dir_filename(self, filename): + fullpath = os.path.join(self.tempdir, filename) + content = ascii(fullpath).encode() + (os_helper.TESTFN_UNDECODABLE or b'\xff') + try: + with open(fullpath, 'wb') as f: + f.write(content) + except OSError: + self.skipTest(f'Can not create file {filename!a} ' + f'on current file system') + response = self.request(self.base_url + '/') - if sys.platform == 'darwin': - # On Mac OS the HFS+ filesystem replaces bytes that aren't valid - # UTF-8 into a percent-encoded value. - for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). - filename = name - break body = self.check_status_and_reason(response, HTTPStatus.OK) quotedname = urllib.parse.quote(filename, errors='surrogatepass') - self.assertIn(('href="%s"' % quotedname) - .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename, quote=False)) - .encode(enc, 'surrogateescape'), body) + enc = response.headers.get_content_charset() + self.assertIsNotNone(enc) + self.assertIn((f'href="{quotedname}"').encode('ascii'), body) + displayname = html.escape(filename, quote=False) + self.assertIn(f'>{displayname}<'.encode(enc, 'surrogateescape'), body) + response = self.request(self.base_url + '/' + quotedname) - self.check_status_and_reason(response, HTTPStatus.OK, - data=os_helper.TESTFN_UNDECODABLE) + self.check_status_and_reason(response, HTTPStatus.OK, data=content) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_dirname(self): + dirname = os_helper.TESTFN_NONASCII + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_filename(self): + filename = os_helper.TESTFN_NONASCII + '.txt' + self.check_list_dir_filename(filename) + + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_dirname(self): + dirname = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_filename(self): + filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_undecodable_dirname2(self): + dirname = '\ufffd.dir' + self.check_list_dir_dirname(dirname, quotedname='%ff.dir') + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_dirname(self): + dirname = os_helper.TESTFN_UNENCODABLE + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_filename(self): + filename = os_helper.TESTFN_UNENCODABLE + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_escape_dirname(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + dirname = name + '.dir' + self.check_list_dir_dirname(dirname, + quotedname=urllib.parse.quote(dirname, safe='&<>\'"')) + + def test_list_dir_escape_filename(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + filename = name + '.txt' + self.check_list_dir_filename(filename) + os_helper.unlink(os.path.join(self.tempdir, filename)) + + def test_list_dir_with_query_and_fragment(self): + prefix = f'listing for {self.base_url}/</'.encode('latin1') + response = self.request(self.base_url + '/#123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) + response = self.request(self.base_url + '/?x=123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) + + def test_get_dir_redirect_location_domain_injection_bug(self): + """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. + + //netloc/ in a Location header is a redirect to a new host. + https://github.com/python/cpython/issues/87389 + + This checks that a path resolving to a directory on our server cannot + resolve into a redirect to another server. + """ + os.mkdir(os.path.join(self.tempdir, 'existing_directory')) + url = f'/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../{self.tempdir_name}/existing_directory' + expected_location = f'{url}/' # /python.org.../ single slash single prefix, trailing slash + # Canonicalizes to /tmp/tempdir_name/existing_directory which does + # exist and is a dir, triggering the 301 redirect logic. + response = self.request(url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + self.assertEqual(location, expected_location, msg='non-attack failed!') + + # //python.org... multi-slash prefix, no trailing slash + attack_url = f'/{url}' + response = self.request(attack_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + self.assertNotStartsWith(location, '//') + self.assertEqual(location, expected_location, + msg='Expected Location header to start with a single / and ' + 'end with a / as this is a directory redirect.') + + # ///python.org... triple-slash prefix, no trailing slash + attack3_url = f'//{url}' + response = self.request(attack3_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader('Location'), expected_location) + + # If the second word in the http request (Request-URI for the http + # method) is a full URI, we don't worry about it, as that'll be parsed + # and reassembled as a full URI within BaseHTTPRequestHandler.send_head + # so no errant scheme-less //netloc//evil.co/ domain mixup can happen. + attack_scheme_netloc_2slash_url = f'https://pypi.org/{url}' + expected_scheme_netloc_location = f'{attack_scheme_netloc_2slash_url}/' + response = self.request(attack_scheme_netloc_2slash_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + # We're just ensuring that the scheme and domain make it through, if + # there are or aren't multiple slashes at the start of the path that + # follows that isn't important in this Location: header. + self.assertStartsWith(location, 'https://pypi.org/') def test_get(self): #constructs the path relative to the root directory of the HTTPServer @@ -424,10 +621,19 @@ def test_get(self): # check for trailing "/" which should return 404. See Issue17324 response = self.request(self.base_url + '/test/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2f') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2F') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request(self.base_url + '/') self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2f') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2F') + self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.base_url) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), self.base_url + "/") self.assertEqual(response.getheader("Content-Length"), "0") response = self.request(self.base_url + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) @@ -439,6 +645,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: @@ -530,6 +739,8 @@ def test_path_without_leading_slash(self): self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/") response = self.request(self.tempdir_name + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name + '?hi=1') @@ -537,27 +748,6 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - def test_html_escape_filename(self): - filename = '<test&>.txt' - fullpath = os.path.join(self.tempdir, filename) - - try: - open(fullpath, 'wb').close() - except OSError: - raise unittest.SkipTest('Can not create file %s on current file ' - 'system' % filename) - - try: - response = self.request(self.base_url + '/') - body = self.check_status_and_reason(response, HTTPStatus.OK) - enc = response.headers.get_content_charset() - finally: - os.unlink(fullpath) # avoid affecting test_undecodable_filename - - self.assertIsNotNone(enc) - html_text = '>%s<' % html.escape(filename, quote=False) - self.assertIn(html_text.encode(enc), body) - cgi_file1 = """\ #!%s @@ -569,14 +759,19 @@ def test_html_escape_filename(self): cgi_file2 = """\ #!%s -import cgi +import os +import sys +import urllib.parse print("Content-type: text/html") print() -form = cgi.FieldStorage() -print("%%s, %%s, %%s" %% (form.getfirst("spam"), form.getfirst("eggs"), - form.getfirst("bacon"))) +content_length = int(os.environ["CONTENT_LENGTH"]) +query_string = sys.stdin.buffer.read(content_length) +params = {key.decode("utf-8"): val.decode("utf-8") + for key, val in urllib.parse.parse_qsl(query_string)} + +print("%%s, %%s, %%s" %% (params["spam"], params["eggs"], params["bacon"])) """ cgi_file4 = """\ @@ -607,17 +802,40 @@ def test_html_escape_filename(self): print("</pre>") """ -@unittest.skipIf(not hasattr(os, '_exit'), - "TODO: RUSTPYTHON, run_cgi in http/server.py gets stuck as os._exit(127) doesn't currently kill forked processes") +cgi_file7 = """\ +#!%s +import os +import sys + +print("Content-type: text/plain") +print() + +content_length = int(os.environ["CONTENT_LENGTH"]) +body = sys.stdin.buffer.read(content_length) + +print(f"{content_length} {len(body)}") +""" + + @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") +@requires_subprocess() class CGIHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): - pass + _test_case_self = None # populated by each setUp() method call. + + def __init__(self, *args, **kwargs): + with self._test_case_self.assertWarnsRegex( + DeprecationWarning, + r'http\.server\.CGIHTTPRequestHandler'): + # This context also happens to catch and silence the + # threading DeprecationWarning from os.fork(). + super().__init__(*args, **kwargs) linesep = os.linesep.encode('ascii') def setUp(self): + self.request_handler._test_case_self = self # practical, but yuck. BaseTestCase.setUp(self) self.cwd = os.getcwd() self.parent_dir = tempfile.mkdtemp() @@ -637,12 +855,13 @@ def setUp(self): self.file3_path = None self.file4_path = None self.file5_path = None + self.file6_path = None + self.file7_path = None # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. self._pythonexe_symlink = None - # TODO: RUSTPYTHON; dl_nt not supported yet - if os_helper.can_symlink() and sys.platform != 'win32': + if os_helper.can_symlink(): self.pythonexe = os.path.join(self.parent_dir, 'python') self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__() else: @@ -692,9 +911,15 @@ def setUp(self): file6.write(cgi_file6 % self.pythonexe) os.chmod(self.file6_path, 0o777) + self.file7_path = os.path.join(self.cgi_dir, 'file7.py') + with open(self.file7_path, 'w', encoding='utf-8') as file7: + file7.write(cgi_file7 % self.pythonexe) + os.chmod(self.file7_path, 0o777) + os.chdir(self.parent_dir) def tearDown(self): + self.request_handler._test_case_self = None try: os.chdir(self.cwd) if self._pythonexe_symlink: @@ -713,11 +938,16 @@ def tearDown(self): os.remove(self.file5_path) if self.file6_path: os.remove(self.file6_path) + if self.file7_path: + os.remove(self.file7_path) os.rmdir(self.cgi_child_dir) os.rmdir(self.cgi_dir) os.rmdir(self.cgi_dir_in_sub_dir) os.rmdir(self.sub_dir_2) os.rmdir(self.sub_dir_1) + # The 'gmon.out' file can be written in the current working + # directory if C-level code profiling with gprof is enabled. + os_helper.unlink(os.path.join(self.parent_dir, 'gmon.out')) os.rmdir(self.parent_dir) finally: BaseTestCase.tearDown(self) @@ -764,8 +994,6 @@ def test_url_collapse_path(self): msg='path = %r\nGot: %r\nWanted: %r' % (path, actual, expected)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_headers_and_content(self): res = self.request('/cgi-bin/file1.py') self.assertEqual( @@ -776,9 +1004,6 @@ def test_issue19435(self): res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") - @unittest.expectedFailure def test_post(self): params = urllib.parse.urlencode( {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) @@ -787,13 +1012,27 @@ def test_post(self): self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) + def test_large_content_length(self): + for w in range(15, 25): + size = 1 << w + body = b'X' * size + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file7.py', 'POST', body, headers) + self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep) + + def test_large_content_length_truncated(self): + with support.swap_attr(self.request_handler, 'timeout', 0.001): + for w in range(18, 65): + size = 1 << w + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers) + self.assertEqual(res.read(), b'Hello World' + self.linesep) + def test_invaliduri(self): res = self.request('/cgi-bin/invalid') res.read() self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_authorization(self): headers = {b'Authorization' : b'Basic ' + base64.b64encode(b'username:pass')} @@ -802,8 +1041,6 @@ def test_authorization(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_no_leading_slash(self): # http://bugs.python.org/issue2254 res = self.request('cgi-bin/file1.py') @@ -811,8 +1048,6 @@ def test_no_leading_slash(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_os_environ_is_not_altered(self): signature = "Test CGI Server" os.environ['SERVER_SOFTWARE'] = signature @@ -822,32 +1057,24 @@ def test_os_environ_is_not_altered(self): (res.read(), res.getheader('Content-type'), res.status)) self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_urlquote_decoding_in_cgi_check(self): res = self.request('/cgi-bin%2ffile1.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_nested_cgi_path_issue21323(self): res = self.request('/cgi-bin/child-dir/file3.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_query_with_multiple_question_mark(self): res = self.request('/cgi-bin/file4.py?a=b?c=d') self.assertEqual( (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_query_with_continuous_slashes(self): res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') self.assertEqual( @@ -855,8 +1082,6 @@ def test_query_with_continuous_slashes(self): 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_cgi_path_in_sub_directories(self): try: CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') @@ -867,8 +1092,6 @@ def test_cgi_path_in_sub_directories(self): finally: CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin') - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_accept(self): browser_accept = \ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' @@ -929,7 +1152,7 @@ def numWrites(self): return len(self.datas) -class BaseHTTPRequestHandlerTestCase(unittest.TestCase): +class BaseHTTPRequestHandlerTestCase(unittest.TestCase, ExtraAssertions): """Test the functionality of the BaseHTTPServer. Test the support for the Expect 100-continue header. @@ -960,6 +1183,27 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) @@ -996,7 +1240,7 @@ def test_extra_space(self): b'Host: dummy\r\n' b'\r\n' ) - self.assertTrue(result[0].startswith(b'HTTP/1.1 400 ')) + self.assertStartsWith(result[0], b'HTTP/1.1 400 ') self.verify_expected_headers(result[1:result.index(b'\r\n')]) self.assertFalse(self.handler.get_called) @@ -1110,7 +1354,7 @@ def test_request_length(self): # Issue #10714: huge request lines are discarded, to avoid Denial # of Service attacks. result = self.send_typical_request(b'GET ' + b'x' * 65537) - self.assertEqual(result[0], b'HTTP/1.1 414 Request-URI Too Long\r\n') + self.assertEqual(result[0], b'HTTP/1.1 414 URI Too Long\r\n') self.assertFalse(self.handler.get_called) self.assertIsInstance(self.handler.requestline, str) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py new file mode 100644 index 00000000000..9155a43a06e --- /dev/null +++ b/Lib/test/test_imaplib.py @@ -0,0 +1,1128 @@ +from test import support +from test.support import socket_helper + +from contextlib import contextmanager +import imaplib +import os.path +import socketserver +import time +import calendar +import threading +import re +import socket + +from test.support import verbose, run_with_tz, run_with_locale, cpython_only +from test.support import hashlib_helper +from test.support import threading_helper +import unittest +from unittest import mock +from datetime import datetime, timezone, timedelta +try: + import ssl +except ImportError: + ssl = None + +support.requires_working_socket(module=True) + +CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "keycert3.pem") +CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "pycacert.pem") + + +class TestImaplib(unittest.TestCase): + + def test_Internaldate2tuple(self): + t0 = calendar.timegm((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "01-Jan-2000 00:00:00 +0000")') + self.assertEqual(time.mktime(tt), t0) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "01-Jan-2000 11:30:00 +1130")') + self.assertEqual(time.mktime(tt), t0) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "31-Dec-1999 12:30:00 -1130")') + self.assertEqual(time.mktime(tt), t0) + + @run_with_tz('MST+07MDT,M4.1.0,M10.5.0') + def test_Internaldate2tuple_issue10941(self): + self.assertNotEqual(imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "02-Apr-2000 02:30:00 +0000")'), + imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "02-Apr-2000 03:30:00 +0000")')) + + def timevalues(self): + return [2000000000, 2000000000.0, time.localtime(2000000000), + (2033, 5, 18, 5, 33, 20, -1, -1, -1), + (2033, 5, 18, 5, 33, 20, -1, -1, 1), + datetime.fromtimestamp(2000000000, + timezone(timedelta(0, 2 * 60 * 60))), + '"18-May-2033 05:33:20 +0200"'] + + @run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') + # DST rules included to work around quirk where the Gnu C library may not + # otherwise restore the previous time zone + @run_with_tz('STD-1DST,M3.2.0,M11.1.0') + def test_Time2Internaldate(self): + expected = '"18-May-2033 05:33:20 +0200"' + + for t in self.timevalues(): + internal = imaplib.Time2Internaldate(t) + self.assertEqual(internal, expected) + + def test_that_Time2Internaldate_returns_a_result(self): + # Without tzset, we can check only that it successfully + # produces a result, not the correctness of the result itself, + # since the result depends on the timezone the machine is in. + for t in self.timevalues(): + imaplib.Time2Internaldate(t) + + @socket_helper.skip_if_tcp_blackhole + def test_imap4_host_default_value(self): + # Check whether the IMAP4_PORT is truly unavailable. + with socket.socket() as s: + try: + s.connect(('', imaplib.IMAP4_PORT)) + self.skipTest( + "Cannot run the test with local IMAP server running.") + except socket.error: + pass + + # This is the exception that should be raised. + expected_errnos = socket_helper.get_socket_conn_refused_errs() + with self.assertRaises(OSError) as cm: + imaplib.IMAP4() + self.assertIn(cm.exception.errno, expected_errnos) + + +if ssl: + class SecureTCPServer(socketserver.TCPServer): + + def get_request(self): + newsocket, fromaddr = self.socket.accept() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(CERTFILE) + connstream = context.wrap_socket(newsocket, server_side=True) + return connstream, fromaddr + + IMAP4_SSL = imaplib.IMAP4_SSL + +else: + + class SecureTCPServer: + pass + + IMAP4_SSL = None + + +class SimpleIMAPHandler(socketserver.StreamRequestHandler): + timeout = support.LOOPBACK_TIMEOUT + continuation = None + capabilities = '' + + def setup(self): + super().setup() + self.server.is_selected = False + self.server.logged = None + + def _send(self, message): + if verbose: + print("SENT: %r" % message.strip()) + self.wfile.write(message) + + def _send_line(self, message): + self._send(message + b'\r\n') + + def _send_textline(self, message): + self._send_line(message.encode('ASCII')) + + def _send_tagged(self, tag, code, message): + self._send_textline(' '.join((tag, code, message))) + + def handle(self): + # Send a welcome message. + self._send_textline('* OK IMAP4rev1') + while 1: + # Gather up input until we receive a line terminator or we timeout. + # Accumulate read(1) because it's simpler to handle the differences + # between naked sockets and SSL sockets. + line = b'' + while 1: + try: + part = self.rfile.read(1) + if part == b'': + # Naked sockets return empty strings.. + return + line += part + except OSError: + # ..but SSLSockets raise exceptions. + return + if line.endswith(b'\r\n'): + break + + if verbose: + print('GOT: %r' % line.strip()) + if self.continuation: + try: + self.continuation.send(line) + except StopIteration: + self.continuation = None + continue + splitline = line.decode('ASCII').split() + tag = splitline[0] + cmd = splitline[1] + args = splitline[2:] + + if hasattr(self, 'cmd_' + cmd): + continuation = getattr(self, 'cmd_' + cmd)(tag, args) + if continuation: + self.continuation = continuation + next(continuation) + else: + self._send_tagged(tag, 'BAD', cmd + ' unknown') + + def cmd_CAPABILITY(self, tag, args): + caps = ('IMAP4rev1 ' + self.capabilities + if self.capabilities + else 'IMAP4rev1') + self._send_textline('* CAPABILITY ' + caps) + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + + def cmd_LOGOUT(self, tag, args): + self.server.logged = None + self._send_textline('* BYE IMAP4ref1 Server logging out') + self._send_tagged(tag, 'OK', 'LOGOUT completed') + + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'OK', 'LOGIN completed') + + def cmd_SELECT(self, tag, args): + self.server.is_selected = True + self._send_line(b'* 2 EXISTS') + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_UNSELECT(self, tag, args): + if self.server.is_selected: + self.server.is_selected = False + self._send_tagged(tag, 'OK', 'Returned to authenticated state. (Success)') + else: + self._send_tagged(tag, 'BAD', 'No mailbox selected') + + +class IdleCmdDenyHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_tagged(tag, 'NO', 'IDLE is not allowed at this time') + + +class IdleCmdHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + # pre-idle-continuation response + self._send_line(b'* 0 EXISTS') + self._send_textline('+ idling') + # simple response + self._send_line(b'* 2 EXISTS') + # complex response: fragmented data due to literal string + self._send_line(b'* 1 FETCH (BODY[HEADER.FIELDS (DATE)] {41}') + self._send(b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n') + self._send_line(b')') + # simple response following a fragmented one + self._send_line(b'* 3 EXISTS') + # response arriving later + time.sleep(1) + self._send_line(b'* 1 RECENT') + r = yield + if r == b'DONE\r\n': + self._send_line(b'* 9 RECENT') + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + +class IdleCmdDelayedPacketHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_textline('+ idling') + # response line spanning multiple packets, the last one delayed + self._send(b'* 1 EX') + time.sleep(0.2) + self._send(b'IS') + time.sleep(1) + self._send(b'TS\r\n') + r = yield + if r == b'DONE\r\n': + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + +class AuthHandler_CRAM_MD5(SimpleIMAPHandler): + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + +class NewIMAPTestsMixin: + client = None + + def _setup(self, imap_handler, connect=True): + """ + Sets up imap_handler for tests. imap_handler should inherit from either: + - SimpleIMAPHandler - for testing IMAP commands, + - socketserver.StreamRequestHandler - if raw access to stream is needed. + Returns (client, server). + """ + class TestTCPServer(self.server_class): + def handle_error(self, request, client_address): + """ + End request and raise the error if one occurs. + """ + self.close_request(request) + self.server_close() + raise + + self.addCleanup(self._cleanup) + self.server = self.server_class((socket_helper.HOST, 0), imap_handler) + self.thread = threading.Thread( + name=self._testMethodName+'-server', + target=self.server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval': 0.01}) + self.thread.daemon = True # In case this function raises. + self.thread.start() + + if connect: + self.client = self.imap_class(*self.server.server_address) + + return self.client, self.server + + def _cleanup(self): + """ + Cleans up the test server. This method should not be called manually, + it is added to the cleanup queue in the _setup method already. + """ + # if logout was called already we'd raise an exception trying to + # shutdown the client once again + if self.client is not None and self.client.state != 'LOGOUT': + self.client.shutdown() + # cleanup the server + self.server.shutdown() + self.server.server_close() + threading_helper.join_thread(self.thread) + # Explicitly clear the attribute to prevent dangling thread + self.thread = None + + def test_EOF_without_complete_welcome_message(self): + # http://bugs.python.org/issue5949 + class EOFHandler(socketserver.StreamRequestHandler): + def handle(self): + self.wfile.write(b'* OK') + _, server = self._setup(EOFHandler, connect=False) + self.assertRaises(imaplib.IMAP4.abort, self.imap_class, + *server.server_address) + + def test_line_termination(self): + class BadNewlineHandler(SimpleIMAPHandler): + def cmd_CAPABILITY(self, tag, args): + self._send(b'* CAPABILITY IMAP4rev1 AUTH\n') + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + _, server = self._setup(BadNewlineHandler, connect=False) + self.assertRaises(imaplib.IMAP4.abort, self.imap_class, + *server.server_address) + + def test_enable_raises_error_if_not_AUTH(self): + class EnableHandler(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + client, _ = self._setup(EnableHandler) + self.assertFalse(client.utf8_enabled) + with self.assertRaisesRegex(imaplib.IMAP4.error, 'ENABLE.*NONAUTH'): + client.enable('foo') + self.assertFalse(client.utf8_enabled) + + def test_enable_raises_error_if_no_capability(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support ENABLE'): + client.enable('foo') + + def test_enable_UTF8_raises_error_if_not_supported(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support ENABLE'): + client.enable('UTF8=ACCEPT') + + def test_enable_UTF8_True_append(self): + class UTF8AppendServer(SimpleIMAPHandler): + capabilities = 'ENABLE UTF8=ACCEPT' + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + def cmd_APPEND(self, tag, args): + self._send_textline('+') + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) + self._send_tagged(tag, 'OK', 'okay') + client, server = self._setup(UTF8AppendServer) + self.assertEqual(client._encoding, 'ascii') + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + code, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(code, 'OK') + self.assertEqual(client._encoding, 'utf-8') + msg_string = 'Subject: üñí©öðé' + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) + self.assertEqual(typ, 'OK') + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) + + def test_search_disallows_charset_in_utf8_mode(self): + class UTF8Server(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, _ = self._setup(UTF8Server) + typ, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(typ, 'OK') + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + self.assertTrue(client.utf8_enabled) + with self.assertRaisesRegex(imaplib.IMAP4.error, 'charset.*UTF8'): + client.search('foo', 'bar') + + def test_bad_auth_name(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_tagged(tag, 'NO', + 'unrecognized authentication type {}'.format(args[0])) + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'unrecognized authentication type METHOD'): + client.authenticate('METHOD', lambda: 1) + + def test_invalid_authentication(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] invalid') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[AUTHENTICATIONFAILED\] invalid'): + client.authenticate('MYAUTH', lambda x: b'fake') + + def test_valid_authentication_bytes(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, server = self._setup(MyServer) + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + def test_valid_authentication_plain_text(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, server = self._setup(MyServer) + code, _ = client.authenticate('MYAUTH', lambda x: 'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5_bytes(self): + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5_plain_text(self): + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + ret, _ = client.login_cram_md5("tim", "tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + def test_login_cram_md5_blocked(self): + def side_effect(*a, **kw): + raise ValueError + + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + msg = re.escape("CRAM-MD5 authentication is not supported") + with ( + mock.patch("hmac.HMAC", side_effect=side_effect), + self.assertRaisesRegex(imaplib.IMAP4.error, msg) + ): + client.login_cram_md5("tim", b"tanstaaftanstaaf") + + def test_aborted_authentication(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + if self.response == b'*\r\n': + self._send_tagged( + tag, + 'NO', + '[AUTHENTICATIONFAILED] aborted') + else: + self._send_tagged(tag, 'OK', 'MYAUTH successful') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[AUTHENTICATIONFAILED\] aborted'): + client.authenticate('MYAUTH', lambda x: None) + + @mock.patch('imaplib._MAXLINE', 10) + def test_linetoolong(self): + class TooLongHandler(SimpleIMAPHandler): + def handle(self): + # send response line longer than the limit set in the next line + self.wfile.write(b'* OK ' + 11 * b'x' + b'\r\n') + _, server = self._setup(TooLongHandler, connect=False) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'got more than 10 bytes'): + self.imap_class(*server.server_address) + + def test_simple_with_statement(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address): + pass + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'socket' object has no attribute 'timeout'. Did you mean: 'gettimeout'? + def test_imaplib_timeout_test(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address, timeout=None) as client: + self.assertEqual(client.sock.timeout, None) + with self.imap_class(*server.server_address, timeout=support.LOOPBACK_TIMEOUT) as client: + self.assertEqual(client.sock.timeout, support.LOOPBACK_TIMEOUT) + with self.assertRaises(ValueError): + self.imap_class(*server.server_address, timeout=0) + + def test_imaplib_timeout_functionality_test(self): + class TimeoutHandler(SimpleIMAPHandler): + def handle(self): + time.sleep(1) + SimpleIMAPHandler.handle(self) + + _, server = self._setup(TimeoutHandler) + addr = server.server_address[1] + with self.assertRaises(TimeoutError): + client = self.imap_class("localhost", addr, timeout=0.001) + + def test_with_statement(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + self.assertIsNone(server.logged) + + def test_with_statement_logout(self): + # It is legal to log out explicitly inside the with block + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + imap.logout() + self.assertIsNone(server.logged) + self.assertIsNone(server.logged) + + # command tests + + def test_idle_capability(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support IMAP4 IDLE'): + with client.idle(): + pass + + def test_idle_denied(self): + client, _ = self._setup(IdleCmdDenyHandler) + client.login('user', 'pass') + with self.assertRaises(imaplib.IMAP4.error): + with client.idle() as idler: + pass + + def test_idle_iter(self): + client, _ = self._setup(IdleCmdHandler) + client.login('user', 'pass') + with client.idle() as idler: + # iteration should include response between 'IDLE' & '+ idling' + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'0'])) + # iteration should produce responses + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'2'])) + # fragmented response (with literal string) should arrive whole + expected_fetch_data = [ + (b'1 (BODY[HEADER.FIELDS (DATE)] {41}', + b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n'), + b')'] + typ, data = next(idler) + self.assertEqual(typ, 'FETCH') + self.assertEqual(data, expected_fetch_data) + # response after a fragmented one should arrive separately + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'3'])) + # iteration should have consumed untagged responses + _, data = client.response('EXISTS') + self.assertEqual(data, [None]) + # responses not iterated should be available after idle + _, data = client.response('RECENT') + self.assertEqual(data[0], b'1') + # responses received after 'DONE' should be available after idle + self.assertEqual(data[1], b'9') + + def test_idle_burst(self): + client, _ = self._setup(IdleCmdHandler) + client.login('user', 'pass') + # burst() should yield immediately available responses + with client.idle() as idler: + batch = list(idler.burst()) + self.assertEqual(len(batch), 4) + # burst() should not have consumed later responses + _, data = client.response('RECENT') + self.assertEqual(data, [b'1', b'9']) + + def test_idle_delayed_packet(self): + client, _ = self._setup(IdleCmdDelayedPacketHandler) + client.login('user', 'pass') + # If our readline() implementation fails to preserve line fragments + # when idle timeouts trigger, a response spanning delayed packets + # can be corrupted, leaving the protocol stream in a bad state. + try: + with client.idle(0.5) as idler: + self.assertRaises(StopIteration, next, idler) + except client.abort as err: + self.fail('multi-packet response was corrupted by idle timeout') + + def test_login(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + self.assertEqual(client.state, 'AUTH') + + def test_logout(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + typ, data = client.logout() + self.assertEqual(typ, 'BYE', (typ, data)) + self.assertEqual(data[0], b'IMAP4ref1 Server logging out', (typ, data)) + self.assertEqual(client.state, 'LOGOUT') + + def test_lsub(self): + class LsubCmd(SimpleIMAPHandler): + def cmd_LSUB(self, tag, args): + self._send_textline('* LSUB () "." directoryA') + return self._send_tagged(tag, 'OK', 'LSUB completed') + client, _ = self._setup(LsubCmd) + client.login('user', 'pass') + typ, data = client.lsub() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'() "." directoryA') + + def test_unselect(self): + client, _ = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + typ, data = client.select() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'2') + + typ, data = client.unselect() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'Returned to authenticated state. (Success)') + self.assertEqual(client.state, 'AUTH') + + # property tests + + def test_file_property_should_not_be_accessed(self): + client, _ = self._setup(SimpleIMAPHandler) + # the 'file' property replaced a private attribute that is now unsafe + with self.assertWarns(RuntimeWarning): + client.file + + +class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): + imap_class = imaplib.IMAP4 + server_class = socketserver.TCPServer + + +@unittest.skipUnless(ssl, "SSL not available") +class NewIMAPSSLTests(NewIMAPTestsMixin, unittest.TestCase): + imap_class = IMAP4_SSL + server_class = SecureTCPServer + + def test_ssl_raises(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ssl_context.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler, connect=False) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) + client.shutdown() + + def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + + _, server = self._setup(SimpleIMAPHandler, connect=False) + client = self.imap_class("localhost", server.server_address[1], + ssl_context=ssl_context) + client.shutdown() + +class ThreadedNetworkedTests(unittest.TestCase): + server_class = socketserver.TCPServer + imap_class = imaplib.IMAP4 + + def make_server(self, addr, hdlr): + + class MyServer(self.server_class): + def handle_error(self, request, client_address): + self.close_request(request) + self.server_close() + raise + + if verbose: + print("creating server") + server = MyServer(addr, hdlr) + self.assertEqual(server.server_address, server.socket.getsockname()) + + if verbose: + print("server created") + print("ADDR =", addr) + print("CLASS =", self.server_class) + print("HDLR =", server.RequestHandlerClass) + + t = threading.Thread( + name='%s serving' % self.server_class, + target=server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval': 0.01}) + t.daemon = True # In case this function raises. + t.start() + if verbose: + print("server running") + return server, t + + def reap_server(self, server, thread): + if verbose: + print("waiting for server") + server.shutdown() + server.server_close() + thread.join() + if verbose: + print("done") + + @contextmanager + def reaped_server(self, hdlr): + server, thread = self.make_server((socket_helper.HOST, 0), hdlr) + try: + yield server + finally: + self.reap_server(server, thread) + + @contextmanager + def reaped_pair(self, hdlr): + with self.reaped_server(hdlr) as server: + client = self.imap_class(*server.server_address) + try: + yield server, client + finally: + client.logout() + + @threading_helper.reap_threads + def test_connect(self): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address) + client.shutdown() + + @threading_helper.reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @threading_helper.reap_threads + def test_issue5949(self): + + class EOFHandler(socketserver.StreamRequestHandler): + def handle(self): + # EOF without sending a complete welcome message. + self.wfile.write(b'* OK') + + with self.reaped_server(EOFHandler) as server: + self.assertRaises(imaplib.IMAP4.abort, + self.imap_class, *server.server_address) + + @threading_helper.reap_threads + def test_line_termination(self): + + class BadNewlineHandler(SimpleIMAPHandler): + + def cmd_CAPABILITY(self, tag, args): + self._send(b'* CAPABILITY IMAP4rev1 AUTH\n') + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + + with self.reaped_server(BadNewlineHandler) as server: + self.assertRaises(imaplib.IMAP4.abort, + self.imap_class, *server.server_address) + + class UTF8Server(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + @threading_helper.reap_threads + def test_enable_raises_error_if_not_AUTH(self): + with self.reaped_pair(self.UTF8Server) as (server, client): + self.assertFalse(client.utf8_enabled) + self.assertRaises(imaplib.IMAP4.error, client.enable, 'foo') + self.assertFalse(client.utf8_enabled) + + # XXX Also need a test that enable after SELECT raises an error. + + @threading_helper.reap_threads + def test_enable_raises_error_if_no_capability(self): + class NoEnableServer(self.UTF8Server): + capabilities = 'AUTH' + with self.reaped_pair(NoEnableServer) as (server, client): + self.assertRaises(imaplib.IMAP4.error, client.enable, 'foo') + + @threading_helper.reap_threads + def test_enable_UTF8_raises_error_if_not_supported(self): + class NonUTF8Server(SimpleIMAPHandler): + pass + with self.assertRaises(imaplib.IMAP4.error): + with self.reaped_pair(NonUTF8Server) as (server, client): + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + client.enable('UTF8=ACCEPT') + + @threading_helper.reap_threads + def test_enable_UTF8_True_append(self): + + class UTF8AppendServer(self.UTF8Server): + def cmd_APPEND(self, tag, args): + self._send_textline('+') + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) + self._send_tagged(tag, 'OK', 'okay') + + with self.reaped_pair(UTF8AppendServer) as (server, client): + self.assertEqual(client._encoding, 'ascii') + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + code, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(code, 'OK') + self.assertEqual(client._encoding, 'utf-8') + msg_string = 'Subject: üñí©öðé' + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) + self.assertEqual(typ, 'OK') + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) + + # XXX also need a test that makes sure that the Literal and Untagged_status + # regexes uses unicode in UTF8 mode instead of the default ASCII. + + @threading_helper.reap_threads + def test_search_disallows_charset_in_utf8_mode(self): + with self.reaped_pair(self.UTF8Server) as (server, client): + typ, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(typ, 'OK') + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + self.assertTrue(client.utf8_enabled) + self.assertRaises(imaplib.IMAP4.error, client.search, 'foo', 'bar') + + @threading_helper.reap_threads + def test_bad_auth_name(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_tagged(tag, 'NO', 'unrecognized authentication ' + 'type {}'.format(args[0])) + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + client.authenticate('METHOD', lambda: 1) + + @threading_helper.reap_threads + def test_invalid_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] invalid') + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + + @threading_helper.reap_threads + def test_valid_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + with self.reaped_pair(MyServer) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + with self.reaped_pair(MyServer) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: 'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + @threading_helper.reap_threads + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5(self): + + class AuthHandler(SimpleIMAPHandler): + + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + with self.reaped_pair(AuthHandler) as (server, client): + self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + ret, data = client.login_cram_md5("tim", "tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + with self.reaped_pair(AuthHandler) as (server, client): + self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + ret, data = client.login_cram_md5("tim", b"tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + + @threading_helper.reap_threads + def test_aborted_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + + if self.response == b'*\r\n': + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] aborted') + else: + self._send_tagged(tag, 'OK', 'MYAUTH successful') + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + code, data = client.authenticate('MYAUTH', lambda x: None) + + + def test_linetoolong(self): + class TooLongHandler(SimpleIMAPHandler): + def handle(self): + # Send a very long response line + self.wfile.write(b'* OK ' + imaplib._MAXLINE * b'x' + b'\r\n') + + with self.reaped_server(TooLongHandler) as server: + self.assertRaises(imaplib.IMAP4.error, + self.imap_class, *server.server_address) + + def test_truncated_large_literal(self): + size = 0 + class BadHandler(SimpleIMAPHandler): + def handle(self): + self._send_textline('* OK {%d}' % size) + self._send_textline('IMAP4rev1') + + for exponent in range(15, 64): + size = 1 << exponent + with self.subTest(f"size=2e{size}"): + with self.reaped_server(BadHandler) as server: + with self.assertRaises(imaplib.IMAP4.abort): + self.imap_class(*server.server_address) + + @threading_helper.reap_threads + def test_simple_with_statement(self): + # simplest call + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address): + pass + + @threading_helper.reap_threads + def test_with_statement(self): + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + self.assertIsNone(server.logged) + + @threading_helper.reap_threads + def test_with_statement_logout(self): + # what happens if already logout in the block? + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + imap.logout() + self.assertIsNone(server.logged) + self.assertIsNone(server.logged) + + @threading_helper.reap_threads + @cpython_only + @unittest.skipUnless(__debug__, "Won't work if __debug__ is False") + def test_dump_ur(self): + # See: http://bugs.python.org/issue26543 + untagged_resp_dict = {'READ-WRITE': [b'']} + + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + with mock.patch.object(imap, '_mesg') as mock_mesg: + imap._dump_ur(untagged_resp_dict) + mock_mesg.assert_called_with( + "untagged responses dump:READ-WRITE: [b'']" + ) + + +@unittest.skipUnless(ssl, "SSL not available") +class ThreadedNetworkedTestsSSL(ThreadedNetworkedTests): + server_class = SecureTCPServer + imap_class = IMAP4_SSL + + @threading_helper.reap_threads + def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) + client.shutdown() + + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class("localhost", server.server_address[1], + ssl_context=ssl_context) + client.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 44e7da1033d..6920cf45533 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1,9 +1,16 @@ import builtins -import contextlib import errno import glob +import json import importlib.util from importlib._bootstrap_external import _get_sourcefile +from importlib.machinery import ( + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, +) import marshal import os import py_compile @@ -15,27 +22,125 @@ import textwrap import threading import time +import types import unittest from unittest import mock +import _imp from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi) + STDLIB_DIR, + swap_attr, + swap_item, + cpython_only, + is_apple_mobile, + is_emscripten, + is_wasm32, + run_in_subinterp, + run_in_subinterp_with_config, + Py_TRACE_REFS, + requires_gil_enabled, + Py_GIL_DISABLED, + no_rerun, + force_not_colorized_test_class, +) from test.support.import_helper import ( - forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) + forget, make_legacy_pyc, unlink, unload, ready_to_import, + DirsOnSysPath, CleanImport, import_module) from test.support.os_helper import ( - TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir) + TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper from test.test_importlib.util import uncache from types import ModuleType +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None skip_if_dont_write_bytecode = unittest.skipIf( sys.dont_write_bytecode, "test meaningful only when writing bytecode") + +def _require_loader(module, loader, skip): + if isinstance(module, str): + module = __import__(module) + + MODULE_KINDS = { + BuiltinImporter: 'built-in', + ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', + FrozenImporter: 'frozen', + SourceFileLoader: 'pure Python', + } + + expected = loader + assert isinstance(expected, type), expected + expected = MODULE_KINDS[expected] + + actual = module.__spec__.loader + if not isinstance(actual, type): + actual = type(actual) + actual = MODULE_KINDS[actual] + + if actual != expected: + err = f'expected module to be {expected}, got {module.__spec__}' + if skip: + raise unittest.SkipTest(err) + raise Exception(err) + return module + +def require_builtin(module, *, skip=False): + module = _require_loader(module, BuiltinImporter, skip) + assert module.__spec__.origin == 'built-in', module.__spec__ + +def require_extension(module, *, skip=False): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) + +def require_frozen(module, *, skip=True): + module = _require_loader(module, FrozenImporter, skip) + assert module.__spec__.origin == 'frozen', module.__spec__ + +def require_pure_python(module, *, skip=False): + _require_loader(module, SourceFileLoader, skip) + +def create_extension_loader(modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + +def import_extension_from_file(modname, filename, *, put_in_sys_modules=True): + loader = create_extension_loader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + if put_in_sys_modules: + sys.modules[modname] = module + return module + + def remove_files(name): for f in (name + ".py", name + ".pyc", @@ -45,27 +150,202 @@ def remove_files(name): rmtree('__pycache__') -@contextlib.contextmanager -def _ready_to_import(name=None, source=""): - # sets up a temporary directory and removes it - # creates the module file - # temporarily clears the module from sys.modules (if any) - # reverts or removes the module when cleaning up - name = name or "spam" - with temp_dir() as tempdir: - path = script_helper.make_script(tempdir, name, source) - old_module = sys.modules.pop(name, None) +if _testsinglephase is not None: + def restore__testsinglephase(*, _orig=_testsinglephase): + # We started with the module imported and want to restore + # it to its nominal state. + sys.modules.pop('_testsinglephase', None) + _orig._clear_globals() + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) + import _testsinglephase + + +def requires_singlephase_init(meth): + """Decorator to skip if single-phase init modules are not supported.""" + if not isinstance(meth, type): + def meth(self, _meth=meth): + try: + return _meth(self) + finally: + restore__testsinglephase() + meth = cpython_only(meth) + msg = "gh-117694: free-threaded build does not currently support single-phase init modules in sub-interpreters" + meth = requires_gil_enabled(msg)(meth) + return unittest.skipIf(_testsinglephase is None, + 'test requires _testsinglephase module')(meth) + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + +class ModuleSnapshot(types.SimpleNamespace): + """A representation of a module for testing. + + Fields: + + * id - the module's object ID + * module - the actual module or an adequate substitute + * __file__ + * __spec__ + * name + * origin + * ns - a copy (dict) of the module's __dict__ (or None) + * ns_id - the object ID of the module's __dict__ + * cached - the sys.modules[mod.__spec__.name] entry (or None) + * cached_id - the object ID of the sys.modules entry (or None) + + In cases where the value is not available (e.g. due to serialization), + the value will be None. + """ + _fields = tuple('id module ns ns_id cached cached_id'.split()) + + @classmethod + def from_module(cls, mod): + name = mod.__spec__.name + cached = sys.modules.get(name) + return cls( + id=id(mod), + module=mod, + ns=types.SimpleNamespace(**mod.__dict__), + ns_id=id(mod.__dict__), + cached=cached, + cached_id=id(cached), + ) + + SCRIPT = textwrap.dedent(''' + {imports} + + name = {name!r} + + {prescript} + + mod = {name} + + {body} + + {postscript} + ''') + IMPORTS = textwrap.dedent(''' + import sys + ''').strip() + SCRIPT_BODY = textwrap.dedent(''' + # Capture the snapshot data. + cached = sys.modules.get(name) + snapshot = dict( + id=id(mod), + module=dict( + __file__=mod.__file__, + __spec__=dict( + name=mod.__spec__.name, + origin=mod.__spec__.origin, + ), + ), + ns=None, + ns_id=id(mod.__dict__), + cached=None, + cached_id=id(cached) if cached else None, + ) + ''').strip() + CLEANUP_SCRIPT = textwrap.dedent(''' + # Clean up the module. + sys.modules.pop(name, None) + ''').strip() + + @classmethod + def build_script(cls, name, *, + prescript=None, + import_first=False, + postscript=None, + postcleanup=False, + ): + if postcleanup is True: + postcleanup = cls.CLEANUP_SCRIPT + elif isinstance(postcleanup, str): + postcleanup = textwrap.dedent(postcleanup).strip() + postcleanup = cls.CLEANUP_SCRIPT + os.linesep + postcleanup + else: + postcleanup = '' + prescript = textwrap.dedent(prescript).strip() if prescript else '' + postscript = textwrap.dedent(postscript).strip() if postscript else '' + + if postcleanup: + if postscript: + postscript = postscript + os.linesep * 2 + postcleanup + else: + postscript = postcleanup + + if import_first: + prescript += textwrap.dedent(f''' + + # Now import the module. + assert name not in sys.modules + import {name}''') + + return cls.SCRIPT.format( + imports=cls.IMPORTS.strip(), + name=name, + prescript=prescript.strip(), + body=cls.SCRIPT_BODY.strip(), + postscript=postscript, + ) + + @classmethod + def parse(cls, text): + raw = json.loads(text) + mod = raw['module'] + mod['__spec__'] = types.SimpleNamespace(**mod['__spec__']) + raw['module'] = types.SimpleNamespace(**mod) + return cls(**raw) + + @classmethod + def from_subinterp(cls, name, interpid=None, *, pipe=None, **script_kwds): + if pipe is not None: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + pipe = os.pipe() try: - sys.path.insert(0, tempdir) - yield name, path - sys.path.remove(tempdir) + return cls._from_subinterp(name, interpid, pipe, script_kwds) finally: - if old_module is not None: - sys.modules[name] = old_module - elif name in sys.modules: - del sys.modules[name] + r, w = pipe + os.close(r) + os.close(w) + + @classmethod + def _from_subinterp(cls, name, interpid, pipe, script_kwargs): + r, w = pipe + + # Build the script. + postscript = textwrap.dedent(f''' + # Send the result over the pipe. + import json + import os + os.write({w}, json.dumps(snapshot).encode()) + + ''') + _postscript = script_kwargs.get('postscript') + if _postscript: + _postscript = textwrap.dedent(_postscript).lstrip() + postscript += _postscript + script_kwargs['postscript'] = postscript.strip() + script = cls.build_script(name, **script_kwargs) + + # Run the script. + if interpid is None: + ret = run_in_subinterp(script) + if ret != 0: + raise AssertionError(f'{ret} != 0') + else: + _interpreters.run_string(interpid, script) + # Parse the results. + text = os.read(r, 1000) + return cls.parse(text.decode()) + +@force_not_colorized_test_class class ImportTests(unittest.TestCase): def setUp(self): @@ -87,8 +367,6 @@ def test_from_import_missing_attr_raises_ImportError(self): with self.assertRaises(ImportError): from importlib import something_that_should_not_exist_anywhere - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_has_name_and_path(self): with CleanImport('os'): import os @@ -100,15 +378,19 @@ def test_from_import_missing_attr_has_name_and_path(self): @cpython_only def test_from_import_missing_attr_has_name_and_so_path(self): - import _testcapi + _testcapi = import_module("_testcapi") with self.assertRaises(ImportError) as cm: from _testcapi import i_dont_exist self.assertEqual(cm.exception.name, '_testcapi') if hasattr(_testcapi, "__file__"): - self.assertEqual(cm.exception.path, _testcapi.__file__) + # The path on the exception is strictly the spec origin, not the + # module's __file__. For most cases, these are the same; but on + # iOS, the Framework relocation process results in the exception + # being raised from the spec location. + self.assertEqual(cm.exception.path, _testcapi.__spec__.origin) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*(\.(so|pyd))?\)" ) else: self.assertEqual( @@ -123,19 +405,15 @@ def test_from_import_missing_attr_has_name(self): self.assertEqual(cm.exception.name, '_warning') self.assertIsNone(cm.exception.path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_path_is_canonical(self): with self.assertRaises(ImportError) as cm: from os.path import i_dont_exist self.assertIn(cm.exception.name, {'posixpath', 'ntpath'}) self.assertIsNotNone(cm.exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_star_invalid_type(self): import re - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") globals = {} @@ -144,7 +422,7 @@ def test_from_import_star_invalid_type(self): ): exec(f"from {name} import *", globals) self.assertNotIn(b"invalid_type", globals) - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("globals()[b'invalid_type'] = object()") globals = {} @@ -161,18 +439,18 @@ def test_case_sensitivity(self): import RAnDoM def test_double_const(self): - # Another brief digression to test the accuracy of manifest float - # constants. - from test import double_const # don't blink -- that *was* the test + # Importing double_const checks that float constants + # serialized by marshal as PYC files don't lose precision + # (SF bug 422177). + from test.test_import.data import double_const + unload('test.test_import.data.double_const') + from test.test_import.data import double_const # noqa: F811 def test_import(self): def test_with_extension(ext): # The extension is normally ".py", perhaps ".pyw". source = TESTFN + ext - if is_jython: - pyc = TESTFN + "$py.class" - else: - pyc = TESTFN + ".pyc" + pyc = TESTFN + ".pyc" with open(source, "w", encoding='utf-8') as f: print("# This tests Python's ability to import a", @@ -274,7 +552,7 @@ def test_import_name_binding(self): import test as x import test.support self.assertIs(x, test, x.__name__) - self.assertTrue(hasattr(test.support, "__file__")) + self.assertHasAttr(test.support, "__file__") # import x.y.z as w binds z as w import test.support as y @@ -295,7 +573,7 @@ def test_issue31286(self): # import in a 'for' loop resulted in segmentation fault for i in range(2): - import test.support.script_helper as x + import test.support.script_helper as x # noqa: F811 def test_failing_reload(self): # A failing reload should leave the module object in sys.modules. @@ -345,7 +623,7 @@ def test_file_to_source(self): sys.path.insert(0, os.curdir) try: mod = __import__(TESTFN) - self.assertTrue(mod.__file__.endswith('.py')) + self.assertEndsWith(mod.__file__, '.py') os.remove(source) del sys.modules[TESTFN] make_legacy_pyc(source) @@ -427,8 +705,6 @@ def test_from_import_message_for_existing_module(self): with self.assertRaisesRegex(ImportError, "^cannot import name 'bogus'"): from re import bogus - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_AttributeError(self): # Issue #24492: trying to import an attribute that raises an # AttributeError should lead to an ImportError. @@ -492,7 +768,7 @@ def run(): finally: del sys.path[0] - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; no C extension support @unittest.skipUnless(sys.platform == "win32", "Windows-specific") def test_dll_dependency_import(self): from _winapi import GetModuleFileName @@ -538,6 +814,445 @@ def test_dll_dependency_import(self): env=env, cwd=os.path.dirname(pyexe)) + def test_issue105979(self): + # this used to crash + with self.assertRaises(ImportError) as cm: + _imp.get_frozen_object("x", b"6\'\xd5Cu\x12") + self.assertIn("Frozen object named 'x' is invalid", + str(cm.exception)) + + def test_frozen_module_from_import_error(self): + with self.assertRaises(ImportError) as cm: + from os import this_will_never_exist + self.assertIn( + f"cannot import name 'this_will_never_exist' from 'os' ({os.__file__})", + str(cm.exception), + ) + with self.assertRaises(ImportError) as cm: + from sys import this_will_never_exist + self.assertIn( + "cannot import name 'this_will_never_exist' from 'sys' (unknown location)", + str(cm.exception), + ) + + scripts = [ + """ +import os +os.__spec__.has_location = False +os.__file__ = [] +from os import this_will_never_exist +""", + """ +import os +os.__spec__.has_location = False +del os.__file__ +from os import this_will_never_exist +""", + """ +import os +os.__spec__.origin = [] +os.__file__ = [] +from os import this_will_never_exist +""" + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"cannot import name 'this_will_never_exist' " + b"from 'os' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_non_module_from_import_error(self): + prefix = """ +import sys +class NotAModule: ... +nm = NotAModule() +nm.symbol = 123 +sys.modules["not_a_module"] = nm +from not_a_module import symbol +""" + scripts = [ + prefix + "from not_a_module import missing_symbol", + prefix + "nm.__spec__ = []\nfrom not_a_module import missing_symbol", + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"ImportError: cannot import name 'missing_symbol' from " + b"'<unknown module name>' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_script_shadowing_stdlib(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no error at all when using -P + popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') + + tmp_child = os.path.join(tmp, "child") + os.mkdir(tmp_child) + + # test the logic with different cwd + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + def test_package_shadowing_stdlib_module(self): + script_errors = [ + ( + "fractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + os.mkdir(os.path.join(tmp, "fractions")) + with open( + os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8' + ) as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write("import fractions; fractions.shadowing_module\n") + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*[\\/]fractions[\\/]+__init__.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'main', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no shadowing at all when using -P + popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") + + def test_script_shadowing_third_party(self): + script_errors = [ + ( + "import numpy\nnumpy.array", + rb"AttributeError: module 'numpy' has no attribute 'array'" + ), + ( + "from numpy import array", + rb"ImportError: cannot import name 'array' from 'numpy'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*numpy.py' if it has the " + rb"same name as a library you intended to import\)\s+\z" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_maybe_not_shadowing_third_party(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write("this_script_does_not_attempt_to_import_numpy = True") + + expected_error = ( + rb"AttributeError: module 'numpy' has no attribute 'attr'\s+\z" + ) + popen = script_helper.spawn_python('-c', 'import numpy; numpy.attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + expected_error = ( + rb"ImportError: cannot import name 'attr' from 'numpy' \(.*\)\s+\z" + ) + popen = script_helper.spawn_python('-c', 'from numpy import attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_edge_cases(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + + # Unhashable str subclass + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + fractions.Fraction +except TypeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertIn(b"unhashable type: 'substr'", stdout.rstrip()) + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + from fractions import Fraction +except TypeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertIn(b"unhashable type: 'substr'", stdout.rstrip()) + + # Various issues with sys module + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +sys.path = [0] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +sys.path = [0] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + # Various issues with origin + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') + @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') + def test_script_shadowing_stdlib_cwd_failure(self): + with os_helper.temp_dir() as tmp: + subtmp = os.path.join(tmp, "subtmp") + os.mkdir(subtmp) + with open(os.path.join(subtmp, "main.py"), "w", encoding='utf-8') as f: + f.write(f""" +import sys +assert sys.path[0] == '' + +import os +import shutil +shutil.rmtree(os.getcwd()) + +os.does_not_exist +""") + # Use -c to ensure sys.path[0] is "" + popen = script_helper.spawn_python("-c", "import main", cwd=subtmp) + stdout, stderr = popen.communicate() + expected_error = rb"AttributeError: module 'os' has no attribute 'does_not_exist'" + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_sys_path_modification(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write('import sys; sys.path.insert(0, "this_folder_does_not_exist")\n') + f.write(script) + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # TODO: RUSTPYTHON: _imp.create_dynamic is for C extensions, not applicable + @unittest.skip("TODO: RustPython _imp.create_dynamic not implemented") + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -546,12 +1261,12 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): mask = 0o022 - with temp_umask(mask), _ready_to_import() as (name, path): + with temp_umask(mask), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) module = __import__(name) if not os.path.exists(cached_path): @@ -570,7 +1285,7 @@ def test_creation_mode(self): def test_cached_mode_issue_2051(self): # permissions of .pyc should match those of .py, regardless of mask mode = 0o600 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -586,7 +1301,7 @@ def test_cached_mode_issue_2051(self): @os_helper.skip_unless_working_chmod def test_cached_readonly(self): mode = 0o400 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -601,7 +1316,7 @@ def test_cached_readonly(self): def test_pyc_always_writable(self): # Initially read-only .pyc files on Windows used to cause problems # with later updates, see issue #6074 for details - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): # Write a Python file, make it read-only and import it with open(path, 'w', encoding='utf-8') as f: f.write("x = 'original'\n") @@ -639,7 +1354,7 @@ class PycRewritingTests(unittest.TestCase): import sys code_filename = sys._getframe().f_code.co_filename module_filename = __file__ -constant = 1 +constant = 1000 def func(): pass func_filename = func.__code__.co_filename @@ -683,8 +1398,6 @@ def test_basics(self): self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_incorrect_code_name(self): py_compile.compile(self.file_name, dfile="another_module.py") mod = self.import_module() @@ -710,7 +1423,7 @@ def test_foreign_code(self): code = marshal.load(f) constants = list(code.co_consts) foreign_code = importlib.import_module.__code__ - pos = constants.index(1) + pos = constants.index(1000) constants[pos] = foreign_code code = code.replace(co_consts=tuple(constants)) with open(self.compiled_name, "wb") as f: @@ -770,7 +1483,7 @@ def test_UNC_path(self): self.fail("could not import 'test_unc_path' from %r: %r" % (unc, e)) self.assertEqual(mod.testdata, 'test_unc_path') - self.assertTrue(mod.__file__.startswith(unc), mod.__file__) + self.assertStartsWith(mod.__file__, unc) unload("test_unc_path") @@ -783,7 +1496,7 @@ def tearDown(self): def test_relimport_star(self): # This will import * from .test_import. from .. import relimport - self.assertTrue(hasattr(relimport, "RelativeImportTests")) + self.assertHasAttr(relimport, "RelativeImportTests") def test_issue3221(self): # Note for mergers: the 'absolute' tests from the 2.x branch @@ -842,10 +1555,36 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_override_builtin(self): # Test that overriding builtins.__import__ can bypass sys.modules. import os @@ -1087,7 +1826,7 @@ def test_frozen_importlib_is_bootstrap(self): self.assertIs(mod, _bootstrap) self.assertEqual(mod.__name__, 'importlib._bootstrap') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap.py') def test_frozen_importlib_external_is_bootstrap_external(self): from importlib import _bootstrap_external @@ -1095,7 +1834,7 @@ def test_frozen_importlib_external_is_bootstrap_external(self): self.assertIs(mod, _bootstrap_external) self.assertEqual(mod.__name__, 'importlib._bootstrap_external') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap_external.py') def test_there_can_be_only_one(self): # Issue #15386 revealed a tricky loophole in the bootstrapping @@ -1305,8 +2044,7 @@ def exec_module(*args): else: importlib.SourceLoader.exec_module = old_exec_module - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; subprocess fails on Windows @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE') def test_unencodable_filename(self): # Issue #11619: The Python parser and the import machinery must not @@ -1357,8 +2095,6 @@ def test_rebinding(self): from test.test_import.data.circular_imports.subpkg import util self.assertIs(util.util, rebinding.util) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binding(self): try: import test.test_import.data.circular_imports.binding @@ -1369,8 +2105,6 @@ def test_crossreference1(self): import test.test_import.data.circular_imports.use import test.test_import.data.circular_imports.source - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_crossreference2(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.source @@ -1390,8 +2124,14 @@ def test_circular_from_import(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_circular_import(self): + with self.assertRaisesRegex( + AttributeError, + r"partially initialized module 'test.test_import.data.circular_imports.import_cycle' " + r"from '.*' has no attribute 'some_attribute' \(most likely due to a circular import\)" + ): + import test.test_import.data.circular_imports.import_cycle + def test_absolute_circular_submodule(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.subpkg2.parent @@ -1402,8 +2142,37 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @requires_singlephase_init + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_circular(self): + """Regression test for gh-123950 + + Import a single-phase-init module that imports itself + from the PyInit_* function (before it's added to sys.modules). + Manages its own cache (which is `static`, and so incompatible + with multiple interpreters or interpreter reset). + """ + name = '_testsinglephase_circular' + helper_name = 'test.test_import.data.circular_imports.singlephase' + with uncache(name, helper_name): + filename = _testsinglephase.__file__ + # We don't put the module in sys.modules: that the *inner* + # import should do that. + mod = import_extension_from_file(name, filename, + put_in_sys_modules=False) + + self.assertEqual(mod.helper_mod_name, helper_name) + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + self.assertNotIn(name, sys.modules) + self.assertNotIn(helper_name, sys.modules) + self.assertIs(mod.clear_static_var(), mod) + _testinternalcapi.clear_extension('_testsinglephase_circular', + mod.__spec__.origin) + def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") @@ -1418,6 +2187,1198 @@ def test_unwritable_module(self): unwritable.x = 42 +class SubinterpImportTests(unittest.TestCase): + + RUN_KWARGS = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + # Isolation-related config values aren't included here. + ) + ISOLATED = dict( + use_main_obmalloc=False, + gil=2, + ) + NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} + NOT_ISOLATED['gil'] = 1 + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def pipe(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + if hasattr(os, 'set_blocking'): + os.set_blocking(r, False) + return (r, w) + + def import_script(self, name, fd, filename=None, check_override=None): + override_text = '' + if check_override is not None: + override_text = f''' + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import {loader} + import os, sys + {override_text} + loader = {loader}({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + + def run_here(self, name, filename=None, *, + check_singlephase_setting=False, + check_singlephase_override=None, + isolated=False, + ): + """ + Try importing the named module in a subinterpreter. + + The subinterpreter will be in the current process. + The module will have already been imported in the main interpreter. + Thus, for extension/builtin modules, the module definition will + have been loaded already and cached globally. + + "check_singlephase_setting" determines whether or not + the interpreter will be configured to check for modules + that are not compatible with use in multiple interpreters. + + This should always return "okay" for all modules if the + setting is False (with no override). + """ + __import__(name) + + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=check_singlephase_setting, + ) + + r, w = self.pipe() + script = self.import_script(name, w, filename, + check_singlephase_override) + + ret = run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + return os.read(r, 100) + + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): + # Verify that the named module may be imported in a subinterpreter. + # (See run_here() for more info.) + out = self.run_here(name, filename, + check_singlephase_setting=strict, + isolated=isolated, + ) + self.assertEqual(out, b'okay') + + def check_incompatible_here(self, name, filename=None, *, isolated=False): + # Differences from check_compatible_here(): + # * verify that import fails + # * "strict" is always True + out = self.run_here(name, filename, + check_singlephase_setting=True, + isolated=isolated, + ) + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def check_compatible_fresh(self, name, *, strict=False, isolated=False): + # Differences from check_compatible_here(): + # * subinterpreter in a new process + # * module has never been imported before in that process + # * this tests importing the module for the first time + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=strict, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert ( + {name!r} in sys.builtin_module_names or + {name!r} not in sys.modules + ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual(out, b'okay') + + def check_incompatible_fresh(self, name, *, isolated=False): + # Differences from check_compatible_fresh(): + # * verify that import fails + # * "strict" is always True + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=True, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_builtin_compat(self): + # For now we avoid using sys or builtins + # since they still don't implement multi-phase init. + module = '_imp' + require_builtin(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @cpython_only + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_frozen_compat(self): + module = '_frozen_importlib' + require_frozen(module, skip=True) + if __import__(module).__spec__.origin != 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly not frozen') + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @requires_singlephase_init + def test_single_init_extension_compat(self): + module = '_testsinglephase' + require_extension(module) + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_incompatible_here(module) + with self.subTest(f'{module}: strict, fresh'): + self.check_incompatible_fresh(module) + with self.subTest(f'{module}: isolated, fresh'): + self.check_incompatible_fresh(module, isolated=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_compat(self): + # Module with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + module = '_testmultiphase' + require_extension(module) + + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + # Module with Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + # and Py_MOD_GIL_NOT_USED + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + + # _test_shared_gil_only: + # Explicit Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED (default) + # and Py_MOD_GIL_NOT_USED + # _test_no_multiple_interpreter_slot: + # No Py_mod_multiple_interpreters slot + # and Py_MOD_GIL_NOT_USED + for modname in ('_test_shared_gil_only', + '_test_no_multiple_interpreter_slot'): + with self.subTest(modname=modname): + + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, + isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here( + modname, filename, strict=False, isolated=False) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_python_compat(self): + module = 'threading' + require_pure_python(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @requires_singlephase_init + def test_singlephase_check_with_setting_and_override(self): + module = '_testsinglephase' + require_extension(module) + + def check_compatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertEqual(out, b'okay') + + def check_incompatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertNotEqual(out, b'okay') + + with self.subTest('config: check enabled; override: enabled'): + check_incompatible(True, 1) + with self.subTest('config: check enabled; override: use config'): + check_incompatible(True, 0) + with self.subTest('config: check enabled; override: disabled'): + check_compatible(True, -1) + + with self.subTest('config: check disabled; override: enabled'): + check_incompatible(False, 1) + with self.subTest('config: check disabled; override: use config'): + check_compatible(False, 0) + with self.subTest('config: check disabled; override: disabled'): + check_compatible(False, -1) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_isolated_config(self): + module = 'threading' + require_pure_python(module) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True, isolated=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True, isolated=True) + + @requires_subinterpreters + @requires_singlephase_init + def test_disallowed_reimport(self): + # See https://github.com/python/cpython/issues/104621. + script = textwrap.dedent(''' + import _testsinglephase + print(_testsinglephase) + ''') + interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + +class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ + + @classmethod + def from_module(cls, mod): + self = super().from_module(mod) + self.summed = mod.sum(1, 2) + self.lookedup = mod.look_up_self() + self.lookedup_id = id(self.lookedup) + self.state_initialized = mod.state_initialized() + if hasattr(mod, 'initialized_count'): + self.init_count = mod.initialized_count() + return self + + SCRIPT_BODY = ModuleSnapshot.SCRIPT_BODY + textwrap.dedent(''' + snapshot['module'].update(dict( + int_const=mod.int_const, + str_const=mod.str_const, + _module_initialized=mod._module_initialized, + )) + snapshot.update(dict( + summed=mod.sum(1, 2), + lookedup_id=id(mod.look_up_self()), + state_initialized=mod.state_initialized(), + init_count=mod.initialized_count(), + has_spam=hasattr(mod, 'spam'), + spam=getattr(mod, 'spam', None), + )) + ''').rstrip() + + @classmethod + def parse(cls, text): + self = super().parse(text) + if not self.has_spam: + del self.spam + del self.has_spam + return self + + +@requires_singlephase_init +class SinglephaseInitTests(unittest.TestCase): + + NAME = '_testsinglephase' + + @classmethod + def setUpClass(cls): + spec = importlib.util.find_spec(cls.NAME) + cls.LOADER = type(spec.loader) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin + + # Start fresh. + cls.clean_up() + + def tearDown(self): + # Clean up the module. + self.clean_up() + + @classmethod + def clean_up(cls): + name = cls.NAME + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, cls.ORIGIN) + + ######################### + # helpers + + def add_module_cleanup(self, name): + def clean_up(): + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.ORIGIN) + self.addCleanup(clean_up) + + def _load_dynamic(self, name, path): + """ + Load an extension module. + """ + # This is essentially copied from the old imp module. + from importlib._bootstrap import _load + loader = self.LOADER(name, path) + + # Issue bpo-24748: Skip the sys.modules check in _load_module_shim; + # always load new extension. + spec = importlib.util.spec_from_file_location(name, path, + loader=loader) + return _load(spec) + + def load(self, name): + try: + already_loaded = self.already_loaded + except AttributeError: + already_loaded = self.already_loaded = {} + assert name not in already_loaded + mod = self._load_dynamic(name, self.ORIGIN) + self.assertNotIn(mod, already_loaded.values()) + already_loaded[name] = mod + return types.SimpleNamespace( + name=name, + module=mod, + snapshot=TestSinglePhaseSnapshot.from_module(mod), + ) + + def re_load(self, name, mod): + assert sys.modules[name] is mod + assert mod.__dict__ == mod.__dict__ + reloaded = self._load_dynamic(name, self.ORIGIN) + return types.SimpleNamespace( + name=name, + module=reloaded, + snapshot=TestSinglePhaseSnapshot.from_module(reloaded), + ) + + # subinterpreters + + def add_subinterpreter(self): + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' + import sys + import _testinternalcapi + ''')) + def clean_up(): + _interpreters.exec(interpid, textwrap.dedent(f''' + name = {self.NAME!r} + if name in sys.modules: + sys.modules.pop(name)._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''')) + _interpreters.destroy(interpid) + self.addCleanup(clean_up) + return interpid + + def import_in_subinterp(self, interpid=None, *, + postscript=None, + postcleanup=False, + ): + name = self.NAME + + if postcleanup: + import_ = 'import _testinternalcapi' if interpid is None else '' + postcleanup = f''' + {import_} + mod._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''' + + try: + pipe = self._pipe + except AttributeError: + r, w = pipe = self._pipe = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + snapshot = TestSinglePhaseSnapshot.from_subinterp( + name, + interpid, + pipe=pipe, + import_first=True, + postscript=postscript, + postcleanup=postcleanup, + ) + + return types.SimpleNamespace( + name=name, + module=None, + snapshot=snapshot, + ) + + # checks + + def check_common(self, loaded): + isolated = False + + mod = loaded.module + if not mod: + # It came from a subinterpreter. + isolated = True + mod = loaded.snapshot.module + # mod.__name__ might not match, but the spec will. + self.assertEqual(mod.__spec__.name, loaded.name) + self.assertEqual(mod.__file__, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) + if not isolated: + self.assertIsSubclass(mod.error, Exception) + self.assertEqual(mod.int_const, 1969) + self.assertEqual(mod.str_const, 'something different') + self.assertIsInstance(mod._module_initialized, float) + self.assertGreater(mod._module_initialized, 0) + + snap = loaded.snapshot + self.assertEqual(snap.summed, 3) + if snap.state_initialized is not None: + self.assertIsInstance(snap.state_initialized, float) + self.assertGreater(snap.state_initialized, 0) + if isolated: + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snap.lookedup_id, snap.id) + self.assertEqual(snap.cached_id, snap.id) + with self.assertRaises(AttributeError): + snap.spam + else: + self.assertIs(snap.lookedup, mod) + self.assertIs(snap.cached, mod) + + def check_direct(self, loaded): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_indirect(self, loaded, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not loaded.module + assert orig.__name__ != loaded.name + self.assertNotEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_basic(self, loaded, expected_init_count): + # m_size == -1 + # The module loads fresh the first time and copies m_copy after. + snap = loaded.snapshot + self.assertIsNot(snap.state_initialized, None) + self.assertIsInstance(snap.init_count, int) + self.assertGreater(snap.init_count, 0) + self.assertEqual(snap.init_count, expected_init_count) + + def check_with_reinit(self, loaded): + # m_size >= 0 + # The module loads fresh every time. + pass + + def check_fresh(self, loaded): + """ + The module had not been loaded before (at least since fully reset). + """ + snap = loaded.snapshot + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snap.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + + def check_semi_fresh(self, loaded, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + snap = loaded.snapshot + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertNotEqual(snap.id, prev.snapshot.id) + self.assertEqual(snap.init_count, prev.snapshot.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertNotEqual(snap.state_initialized, + base.snapshot.state_initialized) + self.assertNotEqual(snap.state_initialized, + prev.snapshot.state_initialized) + + def check_copied(self, loaded, base): + """ + The module had been loaded before and never reset. + """ + snap = loaded.snapshot + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertEqual(snap.init_count, base.snapshot.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertEqual(snap.state_initialized, + base.snapshot.state_initialized) + + ######################### + # the tests + + def test_cleared_globals(self): + loaded = self.load(self.NAME) + _testsinglephase = loaded.module + init_before = _testsinglephase.state_initialized() + + _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() + init_count = _testsinglephase.initialized_count() + + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) + self.assertEqual(init_count, -1) + + def test_variants(self): + # Exercise the most meaningful variants described in Python/import.c. + self.maxDiff = None + + # Check the "basic" module. + + name = self.NAME + expected_init_count = 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + basic = loaded.module + + # Check its indirect variants. + + name = f'{self.NAME}_basic_wrapper' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_indirect(loaded, basic) + self.check_basic(loaded, expected_init_count) + + # Currently PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), loaded.module) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't change after this point. + basic_lookedup = loaded.module + + # Check its direct variant. + + name = f'{self.NAME}_basic_copy' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{self.NAME}_with_reinit' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIs(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{self.NAME}_with_state' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + self.addCleanup(loaded.module._clear_module_state) + + self.check_common(loaded) + self.assertIsNot(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + def test_basic_reloaded(self): + # m_copy is copied into the existing module object. + # Global state is not changed. + self.maxDiff = None + + for name in [ + self.NAME, # the "basic" module + f'{self.NAME}_basic_wrapper', # the indirect variant + f'{self.NAME}_basic_copy', # the direct variant + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIs(reloaded.module, loaded.module) + self.assertIs(reloaded.module.__dict__, loaded.module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(reloaded.snapshot.lookedup, + loaded.snapshot.lookedup) + self.assertEqual(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + self.assertEqual(reloaded.snapshot.init_count, + loaded.snapshot.init_count) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + def test_with_reinit_reloaded(self): + # The module's m_init func is run again. + self.maxDiff = None + + # Keep a reference around. + basic = self.load(self.NAME) + + for name, has_state in [ + (f'{self.NAME}_with_reinit', False), # m_size == 0 + (f'{self.NAME}_with_state', True), # m_size > 0 + ]: + self.add_module_cleanup(name) + with self.subTest(name=name, has_state=has_state): + loaded = self.load(name) + if has_state: + self.addCleanup(loaded.module._clear_module_state) + + reloaded = self.re_load(name, loaded.module) + if has_state: + self.addCleanup(reloaded.module._clear_module_state) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIsNot(reloaded.module, loaded.module) + self.assertNotEqual(reloaded.module.__dict__, + loaded.module.__dict__) + self.assertIs(reloaded.snapshot.lookedup, reloaded.module) + if loaded.snapshot.state_initialized is None: + self.assertIs(reloaded.snapshot.state_initialized, None) + else: + self.assertGreater(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) + + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + # Also, we test with a single-phase module that has global state, + # which is shared by all interpreters. + + @no_rerun(reason="module state is not cleared (see gh-140657)") + @requires_subinterpreters + def test_basic_multiple_interpreters_main_no_reset(self): + # without resetting; already loaded in main interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + main_loaded = self.load(self.NAME) + _testsinglephase = main_loaded.module + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + + self.check_common(main_loaded) + self.check_fresh(main_loaded) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # At this point: + # * alive in 1 interpreter (main) + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy was copied from the main interpreter (was NULL) + # * module's global state was initialized + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp() + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 1 interpreter (main) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (cleared when the interpreter was destroyed) + # (was from main interpreter) + # * module's global state was updated, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 2 interpreters (main, interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 3 interpreters (main, interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") + @requires_subinterpreters + def test_basic_multiple_interpreters_deleted_no_reset(self): + # without resetting; already loaded in a deleted interpreter + + if Py_TRACE_REFS: + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # First, load in the main interpreter but then completely clear it. + loaded_main = self.load(self.NAME) + loaded_main.module._clear_globals() + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) + + # At this point: + # * alive in 0 interpreters + # * module def loaded already + # * module def was in _PyRuntime.imports.extensions, but cleared + # * mod init func ran for the first time (since reset, at least) + # * m_copy was set, but cleared (was NULL) + # * module's global state was initialized but cleared + + # Start with an interpreter that gets destroyed right away. + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') + self.check_common(base) + self.check_fresh(base) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset) + # * m_copy is still set (owned by main interpreter) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded_interp1 = self.import_in_subinterp(interpid1) + self.check_common(loaded_interp1) + self.check_copied(loaded_interp1, base) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + # Use a subinterpreter while the previous one is still alive. + loaded_interp2 = self.import_in_subinterp(interpid2) + self.check_common(loaded_interp2) + self.check_copied(loaded_interp2, loaded_interp1) + + # At this point: + # * alive in 2 interpreters (interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + @requires_subinterpreters + def test_basic_multiple_interpreters_reset_each(self): + # resetting between each interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''', + postcleanup=True, + ) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy is NULL (cleared when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was initialized, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 2 interpreters (interp2, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was initialized, not reset + + +@cpython_only +class TestMagicNumber(unittest.TestCase): + def test_magic_number_endianness(self): + magic_number_bytes = _imp.pyc_magic_number_token.to_bytes(4, 'little') + self.assertEqual(magic_number_bytes[2:], b'\r\n') + # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. + magic_number = int.from_bytes(magic_number_bytes[:2], 'little') + start = 2900 + sys.version_info.minor * 50 + self.assertIn(magic_number, range(start, start + 50)) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_import/data/circular_imports/import_cycle.py b/Lib/test/test_import/data/circular_imports/import_cycle.py new file mode 100644 index 00000000000..cd9507b5f69 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/import_cycle.py @@ -0,0 +1,3 @@ +import test.test_import.data.circular_imports.import_cycle as m + +m.some_attribute diff --git a/Lib/test/test_import/data/circular_imports/singlephase.py b/Lib/test/test_import/data/circular_imports/singlephase.py new file mode 100644 index 00000000000..05618bc72f9 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/singlephase.py @@ -0,0 +1,13 @@ +"""Circular import involving a single-phase-init extension. + +This module is imported from the _testsinglephase_circular module from +_testsinglephase, and imports that module again. +""" + +import importlib +import _testsinglephase +from test.test_import import import_extension_from_file + +name = '_testsinglephase_circular' +filename = _testsinglephase.__file__ +mod = import_extension_from_file(name, filename) diff --git a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py +++ b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/double_const.py b/Lib/test/test_import/data/double_const.py new file mode 100644 index 00000000000..67852aaf982 --- /dev/null +++ b/Lib/test/test_import/data/double_const.py @@ -0,0 +1,30 @@ +from test.support import TestFailed + +# A test for SF bug 422177: manifest float constants varied way too much in +# precision depending on whether Python was loading a module for the first +# time, or reloading it from a precompiled .pyc. The "expected" failure +# mode is that when test_import imports this after all .pyc files have been +# erased, it passes, but when test_import imports this from +# double_const.pyc, it fails. This indicates a woeful loss of precision in +# the marshal format for doubles. It's also possible that repr() doesn't +# produce enough digits to get reasonable precision for this box. + +PI = 3.14159265358979324 +TWOPI = 6.28318530717958648 + +PI_str = "3.14159265358979324" +TWOPI_str = "6.28318530717958648" + +# Verify that the double x is within a few bits of eval(x_str). +def check_ok(x, x_str): + assert x > 0.0 + x2 = eval(x_str) + assert x2 > 0.0 + diff = abs(x - x2) + # If diff is no larger than 3 ULP (wrt x2), then diff/8 is no larger + # than 0.375 ULP, so adding diff/8 to x2 should have no effect. + if x2 + (diff / 8.) != x2: + raise TestFailed("Manifest const %s lost too much precision " % x_str) + +check_ok(PI, PI_str) +check_ok(TWOPI, TWOPI_str) diff --git a/Lib/test/test_import/data/package/submodule.py b/Lib/test/test_import/data/package/submodule.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/package/submodule.py +++ b/Lib/test/test_import/data/package/submodule.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/package2/submodule2.py b/Lib/test/test_import/data/package2/submodule2.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/package2/submodule2.py +++ b/Lib/test/test_import/data/package2/submodule2.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 00000000000..7033c22a719 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 00000000000..cd7b30db15e --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 00000000000..d8af60ab38a --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 00000000000..c861417aece --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_import/data/unwritable/x.py b/Lib/test/test_import/data/unwritable/x.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/unwritable/x.py +++ b/Lib/test/test_import/data/unwritable/x.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index 111c4af1ea7..1fb1d2f9efa 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -4,7 +4,6 @@ import sys import unittest -import warnings @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') diff --git a/Lib/test/test_importlib/extension/_test_nonmodule_cases.py b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py new file mode 100644 index 00000000000..8ffd18d221d --- /dev/null +++ b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py @@ -0,0 +1,44 @@ +import types +import unittest +from test.test_importlib import util + +machinery = util.import_importlib('importlib.machinery') + +from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests + + +class NonModuleExtensionTests: + setUp = MultiPhaseExtensionModuleTests.setUp + load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name + + def _test_nonmodule(self): + # Test returning a non-module object from create works. + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + # Test creating a non-module object with methods defined. + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + # Test that NULL slots aren't a problem. + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + +(Frozen_NonModuleExtensionTests, + Source_NonModuleExtensionTests + ) = util.test_both(NonModuleExtensionTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 0bb74fff5fc..5183719162e 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,4 +1,3 @@ -from importlib import _bootstrap_external from test.support import os_helper import unittest import sys @@ -8,7 +7,8 @@ machinery = util.import_importlib('importlib.machinery') -@unittest.skipIf(util.EXTENSIONS.filename is None, f'{util.EXTENSIONS.name} not available') +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 35ff9fbef58..cdc8884d668 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -11,7 +12,7 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -19,14 +20,30 @@ def setUp(self): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] + else: + loaders = [ + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) return importer.find_spec(fullname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): self.assertTrue(self.find_spec(util.EXTENSIONS.name)) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index d06558f2ade..0dd21e079eb 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,4 +1,4 @@ -from warnings import catch_warnings +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -10,7 +10,8 @@ import warnings import importlib.util import importlib -from test.support.script_helper import assert_python_failure +from test import support +from test.support import MISSING_C_DOCSTRINGS, script_helper class LoaderTests: @@ -18,14 +19,21 @@ class LoaderTests: """Test ExtensionFileLoader.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -33,13 +41,11 @@ def load_module(self, fullname): return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -51,8 +57,6 @@ def test_load_module_API(self): with self.assertRaises(ImportError): self.load_module('XXX') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): with util.uncache(util.EXTENSIONS.name): module = self.load_module(util.EXTENSIONS.name) @@ -61,8 +65,7 @@ def test_module(self): ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -85,13 +88,11 @@ def test_module_reuse(self): module2 = self.load_module(util.EXTENSIONS.name) self.assertIs(module1, module2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) @@ -104,8 +105,16 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules without multi-phase initialization. def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -114,8 +123,8 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -125,7 +134,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -142,8 +151,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -181,13 +189,20 @@ def test_unloadable_nonascii(self): ) = util.test_both(SinglePhaseExtensionModuleTests, machinery=machinery) -# @unittest.skip("TODO: RUSTPYTHON, AssertionError") class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -196,8 +211,7 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -208,7 +222,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -234,8 +248,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. @@ -313,29 +326,6 @@ def test_unloadable_nonascii(self): self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) - def test_nonmodule(self): - # Test returning a non-module object from create works. - name = self.name + '_nonmodule' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - - # issue 27782 - def test_nonmodule_with_methods(self): - # Test creating a non-module object with methods defined. - name = self.name + '_nonmodule_with_methods' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - self.assertEqual(mod.bar(10, 1), 9) - - def test_null_slots(self): - # Test that NULL slots aren't a problem. - name = self.name + '_null_slots' - module = self.load_module_by_name(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - def test_bad_modules(self): # Test SystemError is raised for misbehaving extensions. for name_base in [ @@ -380,7 +370,8 @@ def test_nonascii(self): with self.subTest(name): module = self.load_module_by_name(name) self.assertEqual(module.__name__, name) - self.assertEqual(module.__doc__, "Module named in %s" % lang) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(module.__doc__, "Module named in %s" % lang) (Frozen_MultiPhaseExtensionModuleTests, @@ -388,5 +379,14 @@ def test_nonascii(self): ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) +class NonModuleExtensionTests(unittest.TestCase): + def test_nonmodule_cases(self): + # The test cases in this file cause the GIL to be enabled permanently + # in free-threaded builds, so they are run in a subprocess to isolate + # this effect. + script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py") + script_helper.run_test_script(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index ec9644dc520..941dcd5432c 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -5,6 +5,8 @@ import unittest +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') class PathHookTests: """Test the path hook for extension modules.""" @@ -19,7 +21,7 @@ def hook(self, entry): def test_success(self): # Path hook should handle a directory where a known extension module # exists. - self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_spec')) + self.assertHasAttr(self.hook(util.EXTENSIONS.path), 'find_spec') (Frozen_PathHooksTests, diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 5bb075f3770..971cc28b6d3 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -2,11 +2,8 @@ machinery = util.import_importlib('importlib.machinery') -import _imp -import marshal import os.path import unittest -import warnings from test.support import import_helper, REPO_ROOT, STDLIB_DIR @@ -70,8 +67,6 @@ def check_search_locations(self, spec): expected = [os.path.dirname(filename)] self.assertListEqual(spec.submodule_search_locations, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): modules = [ '__hello__', @@ -114,8 +109,6 @@ def test_module(self): self.check_basic(spec, name) self.check_loader_state(spec, origname, filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_package(self): packages = [ '__phello__', @@ -170,8 +163,6 @@ def test_failure(self): spec = self.find('<not real>') self.assertIsNone(spec) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_not_using_frozen(self): finder = self.machinery.FrozenImporter with import_helper.frozen_modules(enabled=False): diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index b1eb399d937..0824af53e05 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,9 +3,7 @@ machinery = util.import_importlib('importlib.machinery') from test.support import captured_stdout, import_helper, STDLIB_DIR -import _imp import contextlib -import marshal import os.path import sys import types @@ -64,7 +62,7 @@ def exec_module(self, name, origname=None): module.main() self.assertTrue(module.initialized) - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.origin, 'frozen') return module, stdout.getvalue() @@ -75,7 +73,7 @@ def test_module(self): for attr, value in check.items(): self.assertEqual(getattr(module, attr), value) self.assertEqual(output, 'Hello world!\n') - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.loader_state.origname, name) @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") @@ -92,7 +90,6 @@ def test_package(self): self.assertEqual(output, 'Hello world!\n') self.assertEqual(module.__spec__.loader_state.origname, name) - @unittest.skipIf(sys.platform == 'win32', "TODO:RUSTPYTHON Flaky on Windows") def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): @@ -141,7 +138,7 @@ def test_get_code(self): exec(code, mod.__dict__) with captured_stdout() as stdout: mod.main() - self.assertTrue(hasattr(mod, 'initialized')) + self.assertHasAttr(mod, 'initialized') self.assertEqual(stdout.getvalue(), 'Hello world!\n') def test_get_source(self): diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index a14163919af..858b37effc6 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -1,8 +1,5 @@ from importlib import machinery -import sys -import types import unittest -import warnings from test.test_importlib import util diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 431faea5b4e..7130c99a6fc 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -56,8 +56,6 @@ def test_using___name__(self): '__path__': []}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_using___name__(self): with self.assertWarns(ImportWarning): self.import_module({'__name__': 'pkg.fake', '__path__': []}) @@ -75,8 +73,6 @@ def test_spec_fallback(self): module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_package_and_spec_disagree(self): # Raise a DeprecationWarning if __package__ != __spec__.parent. with self.assertWarns(DeprecationWarning): diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index aedf0fd4f9d..718e7d041b0 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -78,7 +78,7 @@ def test_using_cache_for_assigning_to_attribute(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.module') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) @@ -88,7 +88,7 @@ def test_using_cache_for_fromlist(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 4b4b9bc3f5e..feccc7be09a 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -63,7 +63,7 @@ def test_nonexistent_object(self): with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['non_existent']) self.assertEqual(module.__name__, 'module') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package(self): # [module] @@ -71,7 +71,7 @@ def test_module_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_nonexistent_from_package(self): @@ -79,7 +79,7 @@ def test_nonexistent_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['non_existent']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package_triggers_ModuleNotFoundError(self): # If a submodule causes an ModuleNotFoundError because it tries @@ -107,7 +107,7 @@ def basic_star_test(self, fromlist=['*']): mock['pkg'].__all__ = ['module'] module = self.__import__('pkg', fromlist=fromlist) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_using_star(self): @@ -125,8 +125,8 @@ def test_star_with_others(self): mock['pkg'].__all__ = ['module1'] module = self.__import__('pkg', fromlist=['module2', '*']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module1')) - self.assertTrue(hasattr(module, 'module2')) + self.assertHasAttr(module, 'module1') + self.assertHasAttr(module, 'module2') self.assertEqual(module.module1.__name__, 'pkg.module1') self.assertEqual(module.module2.__name__, 'pkg.module2') @@ -136,7 +136,7 @@ def test_nonexistent_in_all(self): importer['pkg'].__all__ = ['non_existent'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_star_in_all(self): with util.mock_spec('pkg.__init__') as importer: @@ -144,7 +144,7 @@ def test_star_in_all(self): importer['pkg'].__all__ = ['*'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, '*')) + self.assertNotHasAttr(module, '*') def test_invalid_type(self): with util.mock_spec('pkg.__init__') as importer: diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py index 28cdc0e526e..550f88d1d7a 100644 --- a/Lib/test/test_importlib/import_/test_helpers.py +++ b/Lib/test/test_importlib/import_/test_helpers.py @@ -126,8 +126,6 @@ def test_gh86298_loader_is_none_and_spec_loader_is_none(self): ValueError, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -137,8 +135,6 @@ def test_gh86298_no_spec(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_spec_is_none(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -148,8 +144,6 @@ def test_gh86298_spec_is_none(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec_loader(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -159,8 +153,6 @@ def test_gh86298_no_spec_loader(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_loader_and_spec_loader_disagree(self): bar = ModuleType('bar') bar.__loader__ = object() diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 26e7b070b95..4c00f60681a 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -30,8 +30,6 @@ def test_continuing(self): with util.import_state(meta_path=[first, second]): self.assertIs(self.__import__(mod_name), second.modules[mod_name]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty(self): # Raise an ImportWarning if sys.meta_path is empty. module_name = 'nothing' @@ -45,7 +43,7 @@ def test_empty(self): self.assertIsNone(importlib._bootstrap._find_spec('nothing', None)) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) (Frozen_CallingOrder, diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index eb0831f7d6d..0c29d608326 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,7 +1,6 @@ from test.test_importlib import util import sys import unittest -from test import support from test.support import import_helper diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 9cf3a77cb84..79e0bdca94c 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -1,3 +1,4 @@ +from test.support import os_helper from test.test_importlib import util importlib = util.import_importlib('importlib') @@ -68,8 +69,6 @@ def test_path_hooks(self): self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_path_hooks(self): # Test that if sys.path_hooks is empty a warning is raised, # sys.path_importer_cache gets None set, and PathFinder returns None. @@ -82,7 +81,7 @@ def test_empty_path_hooks(self): self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) def test_path_importer_cache_empty_string(self): # The empty string should create a finder using the cwd. @@ -155,6 +154,32 @@ def test_deleted_cwd(self): # Do not want FileNotFoundError raised. self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + @os_helper.skip_unless_working_chmod + def test_permission_error_cwd(self): + # gh-115911: Test that an unreadable CWD does not break imports, in + # particular during early stages of interpreter startup. + + def noop_hook(*args): + raise ImportError + + with ( + os_helper.temp_dir() as new_dir, + os_helper.save_mode(new_dir), + os_helper.change_cwd(new_dir), + util.import_state(path=[''], path_hooks=[noop_hook]), + ): + # chmod() is done here (inside the 'with' block) because the order + # of teardown operations cannot be the reverse of setup order. See + # https://github.com/python/cpython/pull/116131#discussion_r1739649390 + try: + os.chmod(new_dir, 0o000) + except OSError: + self.skipTest("platform does not allow " + "changing mode of the cwd") + + # Do not want PermissionError raised. + self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + def test_invalidate_caches_finders(self): # Finders with an invalidate_caches() method have it called. class FakeFinder: diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 99c24f1fd94..1549cbe96ce 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -81,7 +81,7 @@ def callback(global_): self.__import__('pkg') # For __import__(). module = self.__import__('', global_, fromlist=['mod2'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'mod2')) + self.assertHasAttr(module, 'mod2') self.assertEqual(module.mod2.attr, 'pkg.mod2') self.relative_import_test(create, globals_, callback) @@ -107,7 +107,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['module'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.attr, 'pkg.module') self.relative_import_test(create, globals_, callback) @@ -131,7 +131,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['subpkg2'], level=2) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'subpkg2')) + self.assertHasAttr(module, 'subpkg2') self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') self.relative_import_test(create, globals_, callback) @@ -223,6 +223,21 @@ def test_relative_import_no_package_exists_absolute(self): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Lib/test/test_importlib/metadata/__init__.py b/Lib/test/test_importlib/metadata/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/Lib/test/test_importlib/metadata/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/_context.py b/Lib/test/test_importlib/metadata/_context.py new file mode 100644 index 00000000000..8a53eb55d15 --- /dev/null +++ b/Lib/test/test_importlib/metadata/_context.py @@ -0,0 +1,13 @@ +import contextlib + + +# from jaraco.context 4.3 +class suppress(contextlib.suppress, contextlib.ContextDecorator): + """ + A version of contextlib.suppress with decorator support. + + >>> @suppress(KeyError) + ... def key_error(): + ... {}[''] + >>> key_error() + """ diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py new file mode 100644 index 00000000000..b3cfb9cd549 --- /dev/null +++ b/Lib/test/test_importlib/metadata/_path.py @@ -0,0 +1,115 @@ +# from jaraco.path 3.7 + +import functools +import pathlib +from typing import Dict, Protocol, Union +from typing import runtime_checkable + + +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover + + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore +): + """ + Build a set of files/directories, as described by the spec. + + Each key represents a pathname, and the value represents + the content. Content may be a nested directory. + + >>> spec = { + ... 'README.txt': "A README file", + ... "foo": { + ... "__init__.py": "", + ... "bar": { + ... "__init__.py": "", + ... }, + ... "baz.py": "# Some code", + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), + ... } + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' + """ + for name, contents in spec.items(): + create(contents, _ensure_tree_maker(prefix) / name) + + +@functools.singledispatch +def create(content: Union[str, bytes, FilesSpec], path): + path.mkdir(exist_ok=True) + build(content, prefix=path) # type: ignore + + +@create.register +def _(content: bytes, path): + path.write_bytes(content) + + +@create.register +def _(content: str, path): + path.write_text(content, encoding='utf-8') + + +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + +class Recording: + """ + A TreeMaker object that records everything that would be written. + + >>> r = Recording() + >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r) + >>> r.record + ['foo/foo1.txt', 'bar.txt'] + """ + + def __init__(self, loc=pathlib.PurePosixPath(), record=None): + self.loc = loc + self.record = record if record is not None else [] + + def __truediv__(self, other): + return Recording(self.loc / other, self.record) + + def write_text(self, content, **kwargs): + self.record.append(str(self.loc)) + + write_bytes = write_text + + def mkdir(self, **kwargs): + return + + def symlink_to(self, target): + pass diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl new file mode 100644 index 00000000000..641ab07f7aa Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl differ diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg new file mode 100644 index 00000000000..cdb298a19b0 Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg differ diff --git a/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl new file mode 100644 index 00000000000..5ca93657f81 Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl differ diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000..ba73b743394 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000..479488a0348 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000..de645c2e8bc --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000..011f4751fb9 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py new file mode 100644 index 00000000000..826b1b3259b --- /dev/null +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -0,0 +1,395 @@ +import sys +import copy +import json +import shutil +import pathlib +import textwrap +import functools +import contextlib + +from test.support import import_helper +from test.support import os_helper +from test.support import requires_zlib + +from . import _path +from ._path import FilesSpec + + +try: + from importlib import resources # type: ignore + + getattr(resources, 'files') + getattr(resources, 'as_file') +except (ImportError, AttributeError): + import importlib_resources as resources # type: ignore + + +@contextlib.contextmanager +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) + + +@contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super().setUp() + self.site_dir = self.fixtures.enter_context(tmp_path()) + + +class OnSysPath(Fixtures): + @staticmethod + @contextlib.contextmanager + def add_sys_path(dir): + sys.path[:0] = [str(dir)] + try: + yield + finally: + sys.path.remove(str(dir)) + + def setUp(self): + super().setUp() + self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) + + +class SiteBuilder(SiteDir): + def setUp(self): + super().setUp() + for cls in self.__class__.mro(): + with contextlib.suppress(AttributeError): + build_files(cls.files, prefix=self.site_dir) + + +class DistInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + ns:sub = mod:main + """, + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + + +class DistInfoPkgEditable(DistInfoPkg): + """ + Package with a PEP 660 direct_url.json. + """ + + some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' + files: FilesSpec = { + 'distinfo_pkg-1.0.0.dist-info': { + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) + }, + } + + +class DistInfoPkgWithDot(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg_dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg.dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + "pkg.lot.egg-info": { + "METADATA": """ + Name: pkg.lot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgOffPath(SiteBuilder): + files = DistInfoPkg.files + + +class EggInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n", + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_no_modules_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_no_modules-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + setup.py + egg_with_no_modules_pkg.egg-info/PKG-INFO + egg_with_no_modules_pkg.egg-info/SOURCES.txt + egg_with_no_modules_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + PKG-INFO + SOURCES.txt + top_level.txt + """, + # top_level.txt correctly reflects that no modules are installed + "top_level.txt": b"\n", + }, + } + + +class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): + files: FilesSpec = { + "sources_fallback_pkg.egg-info": { + "PKG-INFO": "Name: sources_fallback-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + sources_fallback.py + setup.py + sources_fallback_pkg.egg-info/PKG-INFO + sources_fallback_pkg.egg-info/SOURCES.txt + """, + # missing installed-files.txt (i.e. not installed by pip) and + # missing top_level.txt (to trigger fallback to SOURCES.txt) + }, + "sources_fallback.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoFile(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_file.egg-info": """ + Metadata-Version: 1.0 + Name: egginfo_file + Version: 0.1 + Summary: An example package + Home-page: www.example.com + Author: Eric Haffa-Vee + Author-email: eric@example.coms + License: UNKNOWN + Description: UNKNOWN + Platform: UNKNOWN + """, + } + + +# dedent all text strings before writing +orig = _path.create.registry[str] +_path.create.register(str, lambda content, path: orig(DALS(content), path)) + + +build_files = _path.build + + +def build_record(file_defs): + return ''.join(f'{name},,\n' for name in record_names(file_defs)) + + +def record_names(file_defs): + recording = _path.Recording() + _path.build(file_defs, recording) + return recording.record + + +class FileBuilder: + def unicode_filename(self): + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + + +@requires_zlib() +class ZipFixtures: + root = 'test.test_importlib.metadata.data' + + def _fixture_on_path(self, filename): + pkg_file = resources.files(self.root).joinpath(filename) + file = self.resources.enter_context(resources.as_file(pkg_file)) + assert file.name.startswith('example'), file.name + sys.path.insert(0, str(file)) + self.resources.callback(sys.path.pop, 0) + + def setUp(self): + # Add self.zip_name to the front of sys.path. + self.resources = contextlib.ExitStack() + self.addCleanup(self.resources.close) + + +def parameterize(*args_set): + """Run test method with a series of parameters.""" + + def wrapper(func): + @functools.wraps(func) + def _inner(self): + for args in args_set: + with self.subTest(**args): + func(self, **args) + + return _inner + + return wrapper diff --git a/Lib/test/test_importlib/metadata/stubs.py b/Lib/test/test_importlib/metadata/stubs.py new file mode 100644 index 00000000000..e5b011c399f --- /dev/null +++ b/Lib/test/test_importlib/metadata/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py new file mode 100644 index 00000000000..2256e0c502e --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -0,0 +1,323 @@ +import re +import textwrap +import unittest +import warnings +import importlib +import contextlib + +from . import fixtures +from importlib.metadata import ( + Distribution, + PackageNotFoundError, + distribution, + entry_points, + files, + metadata, + requires, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class APITests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + fixtures.DistInfoPkgWithDot, + fixtures.EggInfoFile, + unittest.TestCase, +): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + pkg_version = version('egginfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_retrieves_version_of_distinfo_pkg(self): + pkg_version = version('distinfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + distribution('does-not-exist') + + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_prefix_not_matched(self): + prefixes = 'p', 'pkg', 'pkg.' + for prefix in prefixes: + with self.subTest(prefix): + with self.assertRaises(PackageNotFoundError): + distribution(prefix) + + def test_for_top_level(self): + tests = [ + ('egginfo-pkg', 'mod'), + ('egg_with_no_modules-pkg', ''), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + self.assertEqual( + distribution(pkg_name).read_text('top_level.txt').strip(), + expect_content, + ) + + def test_read_text(self): + tests = [ + ('egginfo-pkg', 'mod\n'), + ('egg_with_no_modules-pkg', '\n'), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + top_level = [ + path for path in files(pkg_name) if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), expect_content) + + def test_entry_points(self): + eps = entry_points() + assert 'entries' in eps.groups + entries = eps.select(group='entries') + assert 'main' in entries.names + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_entry_points_distribution(self): + entries = entry_points(group='entries') + for entry in ("main", "ns:sub"): + ep = entries[entry] + self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) + self.assertEqual(ep.dist.version, "1.0.0") + + def test_entry_points_unique_packages_normalized(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name (even when normalized). + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "DistInfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = entry_points(group='entries') + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries.names + + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points(group='entries')['missing'] + + def test_entry_points_missing_group(self): + assert entry_points(group='missing') == () + + def test_entry_points_allows_no_attributes(self): + ep = entry_points().select(group='entries', name='main') + with self.assertRaises(AttributeError): + ep.foo = 4 + + def test_metadata_for_this_package(self): + md = metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + def test_missing_key_legacy(self): + """ + Requesting a missing key will still return None, but warn. + """ + md = metadata('distinfo-pkg') + with suppress_known_deprecation(): + assert md['does-not-exist'] is None + + def test_get_key(self): + """ + Getting a key gets the key. + """ + md = metadata('egginfo-pkg') + assert md.get('Name') == 'egginfo-pkg' + + def test_get_missing_key(self): + """ + Requesting a missing key will return None. + """ + md = metadata('distinfo-pkg') + assert md.get('does-not-exist') is None + + @staticmethod + def _test_files(files): + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] + self.assertRegex(repr(util.hash), '<FileHash mode: sha256 value: .*>') + + def test_files_dist_info(self): + self._test_files(files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(files('egginfo-pkg')) + self._test_files(files('egg_with_module-pkg')) + self._test_files(files('egg_with_no_modules-pkg')) + self._test_files(files('sources_fallback-pkg')) + + def test_version_egg_info_file(self): + self.assertEqual(version('egginfo-file'), '0.1') + + def test_requires_egg_info_file(self): + requirements = requires('egginfo-file') + self.assertIsNone(requirements) + + def test_requires_egg_info(self): + deps = requires('egginfo-pkg') + assert len(deps) == 2 + assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) + + def test_requires_egg_info_empty(self): + fixtures.build_files( + { + 'requires.txt': '', + }, + self.site_dir.joinpath('egginfo_pkg.egg-info'), + ) + deps = requires('egginfo-pkg') + assert deps == [] + + def test_requires_dist_info(self): + deps = requires('distinfo-pkg') + assert len(deps) == 2 + assert all(deps) + assert 'wheel >= 1.0' in deps + assert "pytest; extra == 'test'" in deps + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent( + """ + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + dep6@ git+https://example.com/python/dep.git@v1.0.0 + + [extra2:python_version < "3"] + dep5 + """ + ) + deps = sorted(Distribution._deps_from_requires_text(requires)) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + def test_as_json(self): + md = metadata('distinfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 + + def test_as_json_egg_info(self): + md = metadata('egginfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 + + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + + +class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_name_normalization_versionless_egg_info(self): + names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.lot' + + +class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): + def test_find_distributions_specified_path(self): + dists = Distribution.discover(path=[str(self.site_dir)]) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_distribution_at_pathlib(self): + """Demonstrate how to load metadata direct from a directory.""" + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(dist_info_path) + assert dist.version == '1.0.0' + + def test_distribution_at_str(self): + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(str(dist_info_path)) + assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py new file mode 100644 index 00000000000..e4218076f8c --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -0,0 +1,468 @@ +import re +import pickle +import unittest +import warnings +import importlib +import importlib.metadata +import contextlib +from test.support import os_helper + +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + +from . import fixtures +from ._context import suppress +from ._path import Symlink +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + _unique, + distributions, + entry_points, + metadata, + packages_distributions, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + Distribution.from_name('does-not-exist') + + def test_package_not_found_mentions_metadata(self): + """ + When a package is not found, that could indicate that the + package is not installed or that it is installed without + metadata. Ensure the exception mentions metadata to help + guide users toward the cause. See #124. + """ + with self.assertRaises(PackageNotFoundError) as ctx: + Distribution.from_name('does-not-exist') + + assert "metadata" in str(ctx.exception) + + # expected to fail until ABC is enforced + @suppress(AssertionError) + @suppress_known_deprecation() + def test_abc_enforced(self): + with self.assertRaises(TypeError): + type('DistributionSubclass', (Distribution,), {})() + + @fixtures.parameterize( + dict(name=None), + dict(name=''), + ) + def test_invalid_inputs_to_from_name(self, name): + with self.assertRaises(Exception): + Distribution.from_name(name) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existent module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + ep = entry_points(group='entries')['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_entrypoint_with_colon_in_name(self): + ep = entry_points(group='entries')['ns:sub'] + self.assertEqual(ep.value, 'mod:main') + + def test_resolve_without_attr(self): + ep = EntryPoint( + name='ep', + value='importlib.metadata', + group='grp', + ) + assert ep.load() is importlib.metadata + + +class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name): + """ + Create minimal metadata for a dist-info package with + the indicated name on the file system. + """ + return { + f'{name}.dist-info': { + 'METADATA': 'VERSION: 1.0\n', + }, + } + + def test_dashes_in_dist_name_found_as_underscores(self): + """ + For a package with a dash in the name, the dist-info metadata + uses underscores in the name. Ensure the metadata loads. + """ + fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) + assert version('my-pkg') == '1.0' + + def test_dist_name_found_as_any_case(self): + """ + Ensure the metadata loads when queried with any case. + """ + pkg_name = 'CherryPy' + fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) + assert version(pkg_name) == '1.0' + assert version(pkg_name.lower()) == '1.0' + assert version(pkg_name.upper()) == '1.0' + + def test_unique_distributions(self): + """ + Two distributions varying only by non-normalized name on + the file system should resolve as the same. + """ + fixtures.build_files(self.make_pkg('abc'), self.site_dir) + before = list(_unique(distributions())) + + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) + after = list(_unique(distributions())) + + assert len(after) == len(before) + + +class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': 'Description: pôrˈtend', + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': """ + Name: portend + + pôrˈtend""", + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + +class DiscoveryTests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + unittest.TestCase, +): + def test_package_discovery(self): + dists = list(distributions()) + assert all(isinstance(dist, Distribution) for dist in dists) + assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_module-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_no_modules-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'sources_fallback-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_invalid_usage(self): + with self.assertRaises(ValueError): + list(distributions(context='something', name='else')) + + def test_interleaved_discovery(self): + """ + Ensure interleaved searches are safe. + + When the search is cached, it is possible for searches to be + interleaved, so make sure those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('egginfo-pkg') + next(dists) + + +class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_egg_info(self): + # make an `EGG-INFO` directory that's unrelated + self.site_dir.joinpath('EGG-INFO').mkdir() + # used to crash with `IsADirectoryError` + with self.assertRaises(PackageNotFoundError): + version('unknown-package') + + def test_egg(self): + egg = self.site_dir.joinpath('foo-3.6.egg') + egg.mkdir() + with self.add_sys_path(egg): + with self.assertRaises(PackageNotFoundError): + version('foo') + + +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super().setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + +class TestEntryPoints(unittest.TestCase): + def __init__(self, *args): + super().__init__(*args) + self.ep = importlib.metadata.EntryPoint( + name='name', value='value', group='group' + ) + + def test_entry_point_pickleable(self): + revived = pickle.loads(pickle.dumps(self.ep)) + assert revived == self.ep + + def test_positional_args(self): + """ + Capture legacy (namedtuple) construction, discouraged. + """ + EntryPoint('name', 'value', 'group') + + def test_immutable(self): + """EntryPoints should be immutable""" + with self.assertRaises(AttributeError): + self.ep.name = 'badactor' + + def test_repr(self): + assert 'EntryPoint' in repr(self.ep) + assert 'name=' in repr(self.ep) + assert "'name'" in repr(self.ep) + + def test_hashable(self): + """EntryPoints should be hashable""" + hash(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None + + def test_sortable(self): + """ + EntryPoint objects are sortable, but result is undefined. + """ + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) + + +class FileSystem( + fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase +): + def test_unicode_dir_on_sys_path(self): + """ + Ensure a Unicode subdirectory of a directory on sys.path + does not crash. + """ + fixtures.build_files( + {self.unicode_filename(): {}}, + prefix=self.site_dir, + ) + list(distributions()) + + +class PackagesDistributionsPrebuiltTest(fixtures.ZipFixtures, unittest.TestCase): + def test_packages_distributions_example(self): + self._fixture_on_path('example-21.12-py3-none-any.whl') + assert packages_distributions()['example'] == ['example'] + + def test_packages_distributions_example2(self): + """ + Test packages_distributions on a wheel built + by trampolim. + """ + self._fixture_on_path('example2-1.0.0-py3-none-any.whl') + assert packages_distributions()['example2'] == ['example2'] + + +class PackagesDistributionsTest( + fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase +): + def test_packages_distributions_neither_toplevel_nor_files(self): + """ + Test a package built without 'top-level.txt' or a file list. + """ + fixtures.build_files( + { + 'trim_example-1.0.0.dist-info': { + 'METADATA': """ + Name: trim_example + Version: 1.0.0 + """, + } + }, + prefix=self.site_dir, + ) + packages_distributions() + + def test_packages_distributions_all_module_types(self): + """ + Test top-level modules detected on a package without 'top-level.txt'. + """ + suffixes = importlib.machinery.all_suffixes() + metadata = dict( + METADATA=""" + Name: all_distributions + Version: 1.0.0 + """, + ) + files = { + 'all_distributions-1.0.0.dist-info': metadata, + } + for i, suffix in enumerate(suffixes): + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) + metadata.update(RECORD=fixtures.build_record(files)) + fixtures.build_files(files, prefix=self.site_dir) + + distributions = packages_distributions() + + for i in range(len(suffixes)): + assert distributions[f'importable-name {i}'] == ['all_distributions'] + assert distributions[f'in_namespace_{i}'] == ['all_distributions'] + assert distributions[f'in_package_{i}'] == ['all_distributions'] + + assert not any(name.endswith('.dist-info') for name in distributions) + + @os_helper.skip_unless_symlink + def test_packages_distributions_symlinked_top_level(self) -> None: + """ + Distribution is resolvable from a simple top-level symlink in RECORD. + See #452. + """ + + files: fixtures.FilesSpec = { + "symlinked_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: symlinked-pkg + Version: 1.0.0 + """, + "RECORD": "symlinked,,\n", + }, + ".symlink.target": {}, + "symlinked": Symlink(".symlink.target"), + } + + fixtures.build_files(files, self.site_dir) + assert packages_distributions()['symlinked'] == ['symlinked-pkg'] + + +class PackagesDistributionsEggTest( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + unittest.TestCase, +): + def test_packages_distributions_on_eggs(self): + """ + Test old-style egg packages with a variation of 'top_level.txt', + 'SOURCES.txt', and 'installed-files.txt', available. + """ + distributions = packages_distributions() + + def import_names_from_package(package_name): + return { + import_name + for import_name, package_names in distributions.items() + if package_name in package_names + } + + # egginfo-pkg declares one import ('mod') via top_level.txt + assert import_names_from_package('egginfo-pkg') == {'mod'} + + # egg_with_module-pkg has one import ('egg_with_module') inferred from + # installed-files.txt (top_level.txt is missing) + assert import_names_from_package('egg_with_module-pkg') == {'egg_with_module'} + + # egg_with_no_modules-pkg should not be associated with any import names + # (top_level.txt is empty, and installed-files.txt has no .py files) + assert import_names_from_package('egg_with_no_modules-pkg') == set() + + # sources_fallback-pkg has one import ('sources_fallback') inferred from + # SOURCES.txt (top_level.txt and installed-files.txt is missing) + assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} + + +class EditableDistributionTest(fixtures.DistInfoPkgEditable, unittest.TestCase): + def test_origin(self): + dist = Distribution.from_name('distinfo-pkg') + assert dist.origin.url.endswith('.whl') + assert dist.origin.archive_info.hashes.sha256 diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py new file mode 100644 index 00000000000..276f6288c91 --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -0,0 +1,62 @@ +import sys +import unittest + +from . import fixtures +from importlib.metadata import ( + PackageNotFoundError, + distribution, + distributions, + entry_points, + files, + version, +) + + +class TestZip(fixtures.ZipFixtures, unittest.TestCase): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3-none-any.whl') + + def test_zip_version(self): + self.assertEqual(version('example'), '21.12') + + def test_zip_version_does_not_match(self): + with self.assertRaises(PackageNotFoundError): + version('definitely-not-installed') + + def test_zip_entry_points(self): + scripts = entry_points(group='console_scripts') + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + entry_point = scripts['Example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + self.assertIsNone(distribution('example').read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(version('Example'), '21.12') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path + + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + + +class TestEgg(TestZip): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3.6.egg') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.egg/' in path, path + + def test_normalized_name(self): + dist = distribution('example') + assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_importlib/resources/__init__.py +++ b/Lib/test/test_importlib/resources/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index 1f97c961469..b144628cb73 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -2,15 +2,44 @@ import functools from typing import Dict, Union +from typing import runtime_checkable +from typing import Protocol #### -# from jaraco.path 3.4.1 +# from jaraco.path 3.7.1 -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover -def build(spec: FilesSpec, prefix=pathlib.Path()): + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): """ Build a set of files/directories, as described by the spec. @@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()): ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): - create(contents, pathlib.Path(prefix) / name) + create(contents, _ensure_tree_maker(prefix) / name) @functools.singledispatch def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + build(content, prefix=path) # type: ignore[arg-type] @create.register @@ -52,5 +85,10 @@ def _(content: str, path): path.write_text(content, encoding='utf-8') +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + # end from jaraco.path #### diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86..4e4e0e9c337 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,25 +18,21 @@ def test_contents(self): assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a27..640f90fc0dd 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1d04cda1a8f..3ce44999f98 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,5 @@ -import typing +import pathlib +import py_compile import textwrap import unittest import warnings @@ -7,11 +8,8 @@ from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -32,13 +30,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -48,73 +47,145 @@ def test_old_parameter(self): resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 - - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_read_bytes(self): - super().test_read_bytes() +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' - self.data = namespacedata01 + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. + + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_read_bytes(self): - super().test_read_bytes() + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + + +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ + + MODULE = 'unused' + + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } -class ModulesFilesTests(SiteDir, unittest.TestCase): def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) - import mod + import mod # type: ignore[import-not-found] actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] + + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass + + +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } + + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000..e8d25fa4d9f --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,249 @@ +import unittest +import os +import importlib + +from test.support import warnings_helper + +from importlib import resources + +from . import util + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaa..8c00378ad3c 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -24,7 +23,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -65,21 +64,21 @@ def test_open_text_FileNotFoundError(self): target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 - +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58..903911f57b3 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ import io +import pathlib import unittest from importlib import resources -from . import data01 from . import util @@ -15,23 +15,16 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) - + self.assertIsInstance(path, pathlib.Path) + self.assertEndsWith(path.name, "utf-8.file") + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8..59c237d9641 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -51,30 +51,42 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334..ed5693ab416 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,17 +65,16 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,10 +86,8 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -105,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -124,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -133,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03..7e5e5903fde 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,8 @@ -import contextlib -import sys +import os import unittest -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -29,9 +22,8 @@ def test_is_dir(self): self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -42,33 +34,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -77,7 +75,7 @@ def test_package_has_no_reader_fallback(self): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -88,43 +86,39 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +126,49 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') + @unittest.skipIf("RUSTPYTHON_SKIP_ENV_POLLUTERS" in os.environ, "TODO: RUSTPYTHON; environment pollution when running rustpython -m test --fail-env-changed due to tmpfile leak") def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +187,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +197,41 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476..e2d995f5963 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib -from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -67,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -83,34 +84,34 @@ def test_package_name(self): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -140,40 +141,66 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass - +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) + + +class ZipSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) + + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100644 index 00000000000..fc453f02060 --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,24 @@ +""" +Generate zip test data files. +""" + +import zipfile + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 6a06313319d..e52829e6280 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -9,7 +9,6 @@ import os from test.support import os_helper import unittest -import warnings @util.case_insensitive_tests diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index d487fc9b82d..f35adec1a8e 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -359,23 +359,6 @@ def test_overridden_unchecked_hash_based_pyc(self): ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SimpleTest(Source_SimpleTest): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_checked_hash_based_pyc(self): - super().test_checked_hash_based_pyc() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_unchecked_hash_based_pyc(self): - super().test_unchecked_hash_based_pyc() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_overridden_unchecked_hash_based_pyc(self): - super().test_overridden_unchecked_hash_based_pyc() - class SourceDateEpochTestMeta(SourceDateEpochTestMeta, type(Source_SimpleTest)): @@ -697,23 +680,6 @@ class SourceLoaderBadBytecodeTestPEP451( machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourceBadBytecodePEP451(Source_SourceBadBytecodePEP451): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_bad_marshal(self): - super().test_bad_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_non_code_marshal(self): - super().test_non_code_marshal() - class SourceLoaderBadBytecodeTestPEP302( SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): @@ -726,23 +692,6 @@ class SourceLoaderBadBytecodeTestPEP302( machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourceBadBytecodePEP302(Source_SourceBadBytecodePEP302): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_bad_marshal(self): - super().test_bad_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_non_code_marshal(self): - super().test_non_code_marshal() - class SourcelessLoaderBadBytecodeTest: @@ -829,38 +778,6 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourcelessBadBytecodePEP451(Source_SourcelessBadBytecodePEP451): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_magic_only(self): - super().test_magic_only() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_flags(self): - super().test_partial_flags() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_hash(self): - super().test_partial_hash() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_size(self): - super().test_partial_size() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_timestamp(self): - super().test_partial_timestamp() - class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, BadBytecodeTestPEP302): @@ -873,38 +790,6 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourcelessBadBytecodePEP302(Source_SourcelessBadBytecodePEP302): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_magic_only(self): - super().test_magic_only() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_flags(self): - super().test_partial_flags() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_hash(self): - super().test_partial_hash() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_size(self): - super().test_partial_size() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_timestamp(self): - super().test_partial_timestamp() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 12db7c7d352..4de736a6bf3 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -10,7 +10,6 @@ import tempfile from test.support.import_helper import make_legacy_pyc import unittest -import warnings class FinderTests(abc.FinderTests): @@ -74,7 +73,7 @@ def run_test(self, test, create=None, *, compile_=None, unlink=None): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') return loader def test_module(self): @@ -101,7 +100,7 @@ def test_module_in_package(self): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [sub package] def test_package_in_package(self): @@ -109,7 +108,7 @@ def test_package_in_package(self): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [package over modules] def test_package_over_module(self): @@ -130,7 +129,7 @@ def test_empty_string_for_dir(self): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') finally: os.unlink('mod.py') diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index f274330e0b3..6e1c23e6a98 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -15,12 +15,12 @@ def path_hook(self): def test_success(self): with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_spec')) + self.assertHasAttr(self.path_hook()(mapping['.root']), + 'find_spec') def test_empty_string(self): # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + self.assertHasAttr(self.path_hook()(''), 'find_spec') (Frozen_PathHookTest, diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 4f206accf97..d65d51d0cca 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -61,17 +61,15 @@ def test_non_obvious_encoding(self): def test_default_encoding(self): self.run_test(self.source_line.encode('utf-8')) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 17 - @unittest.expectedFailure # [encoding first line] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 17 def test_encoding_on_first_line(self): encoding = 'Latin-1' source = self.create_source(encoding) self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 34 - @unittest.expectedFailure # [encoding second line] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 34 def test_encoding_on_second_line(self): source = b"#/usr/bin/python\n" + self.create_source('Latin-1') self.run_test(source) @@ -85,9 +83,8 @@ def test_bom_and_utf_8(self): source = codecs.BOM_UTF8 + self.create_source('utf-8') self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 20 - @unittest.expectedFailure # [BOM conflict] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 20 def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') with self.assertRaises(SyntaxError): diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 603125f6d92..dd943210ffc 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -43,14 +43,12 @@ def setUp(self): def test_subclasses(self): # Test that the expected subclasses inherit. for subclass in self.subclasses: - self.assertTrue(issubclass(subclass, self.__test), - "{0} is not a subclass of {1}".format(subclass, self.__test)) + self.assertIsSubclass(subclass, self.__test) def test_superclasses(self): # Test that the class inherits from the expected superclasses. for superclass in self.superclasses: - self.assertTrue(issubclass(self.__test, superclass), - "{0} is not a superclass of {1}".format(superclass, self.__test)) + self.assertIsSubclass(self.__test, superclass) class MetaPathFinder(InheritanceTests): @@ -416,14 +414,14 @@ def test_source_to_code_source(self): # Since compile() can handle strings, so should source_to_code(). source = 'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_bytes(self): # Since compile() can handle bytes, so should source_to_code(). source = b'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_path(self): @@ -757,7 +755,7 @@ def test_package_settings(self): warnings.simplefilter('ignore', DeprecationWarning) module = self.loader.load_module(self.name) self.verify_module(module) - self.assertFalse(hasattr(module, '__path__')) + self.assertNotHasAttr(module, '__path__') def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise @@ -795,6 +793,9 @@ def verify_code(self, code_object, *, bytecode_written=False): data.extend(self.init._pack_uint32(0)) data.extend(self.init._pack_uint32(self.loader.source_mtime)) data.extend(self.init._pack_uint32(self.loader.source_size)) + # Make sure there's > 1 reference to code_object so that the + # marshaled representation below matches the cached representation + l = [code_object] data.extend(marshal.dumps(code_object)) self.assertEqual(self.loader.written[self.cached], bytes(data)) @@ -913,5 +914,30 @@ def test_universal_newlines(self): SourceOnlyLoaderMock=SPLIT_SOL) +class SourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests SourceLoader deprecation warnings.""" + + def test_deprecated_path_mtime(self): + from importlib.abc import SourceLoader + class DummySourceLoader(SourceLoader): + def get_data(self, path): + return b'' + + def get_filename(self, fullname): + return 'foo.py' + + def path_stats(self, path): + return {'mtime': 1} + + loader = DummySourceLoader() + + with self.assertWarnsRegex( + DeprecationWarning, + r"SourceLoader\.path_mtime is deprecated in favour of " + r"SourceLoader\.path_stats\(\)\." + ): + loader.path_mtime('foo.py') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index ecf2c47c462..1bc531a2fe3 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,11 +6,12 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper +import traceback import types import unittest -import warnings class ImportModuleTests: @@ -354,6 +355,20 @@ def test_module_missing_spec(self): with self.assertRaises(ModuleNotFoundError): self.init.reload(module) + def test_reload_traceback_with_non_str(self): + # gh-125519 + with support.captured_stdout() as stdout: + try: + self.init.reload("typing") + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + else: + self.fail("Expected TypeError to be raised") + printed_traceback = stdout.getvalue() + self.assertIn("TypeError", printed_traceback) + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("module.__spec__.name", printed_traceback) + (Frozen_ReloadTests, Source_ReloadTests @@ -415,8 +430,7 @@ def test_everyone_has___loader__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__loader__'), - '{!r} lacks a __loader__ attribute'.format(name)) + self.assertHasAttr(module, '__loader__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__loader__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -426,7 +440,7 @@ def test_everyone_has___spec__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__spec__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -438,5 +452,57 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + +class TestDeprecations(unittest.TestCase): + def test_machinery_deprecated_attributes(self): + from importlib import machinery + attributes = ( + 'DEBUG_BYTECODE_SUFFIXES', + 'OPTIMIZED_BYTECODE_SUFFIXES', + ) + for attr in attributes: + with self.subTest(attr=attr): + with self.assertWarns(DeprecationWarning): + getattr(machinery, attr) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333e3..e48fad8898f 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -118,12 +125,12 @@ def test_delete_eventual_attr(self): # Deleting an attribute should stay deleted. module = self.new_module() del module.attr - self.assertFalse(hasattr(module, 'attr')) + self.assertNotHasAttr(module, 'attr') def test_delete_preexisting_attr(self): module = self.new_module() del module.__name__ - self.assertFalse(hasattr(module, '__name__')) + self.assertNotHasAttr(module, '__name__') def test_module_substitution_error(self): with test_util.uncache(TestingImporter.module_name): @@ -140,6 +147,83 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 17cce741cce..b1f5f9d6c8b 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -29,9 +29,12 @@ class ModuleLockAsRLockTests: test_timeout = None # _release_save() unsupported test_release_save_unacquired = None + # _recursion_count() unsupported + test_recursion_count = None # lock status in repr unsupported test_repr = None test_locked_repr = None + test_repr_count = None def tearDown(self): for splitinit in init.values(): @@ -47,7 +50,6 @@ def tearDown(self): LockType=LOCK_TYPES) -@unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON") class DeadlockAvoidanceTests: def setUp(self): @@ -92,11 +94,12 @@ def f(): b.release() if ra: a.release() - lock_tests.Bunch(f, NTHREADS).wait_for_finished() + with lock_tests.Bunch(f, NTHREADS): + pass self.assertEqual(len(results), NTHREADS) return results - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_deadlock(self): results = self.run_deadlock_avoidance_test(True) # At least one of the threads detected a potential deadlock on its @@ -106,7 +109,7 @@ def test_deadlock(self): self.assertGreaterEqual(nb_deadlocks, 1) self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks) - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_no_deadlock(self): results = self.run_deadlock_avoidance_test(False) self.assertEqual(results.count((True, False)), 0) @@ -145,10 +148,14 @@ def test_all_locks(self): self.assertEqual(0, len(self.bootstrap._module_locks), self.bootstrap._module_locks) -# TODO: RUSTPYTHON -# (Frozen_LifetimeTests, -# Source_LifetimeTests -# ) = test_util.test_both(LifetimeTests, init=init) + +(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) + +# TODO: RUSTPYTHON; dead weakref module locks not cleaned up in frozen bootstrap +Frozen_LifetimeTests.test_all_locks = unittest.skip("TODO: RUSTPYTHON")( + Frozen_LifetimeTests.test_all_locks) def setUpModule(): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 65428c3d3ea..6ca0978f9bc 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -6,7 +6,6 @@ import sys import tempfile import unittest -import warnings from test.test_importlib import util @@ -81,7 +80,7 @@ def test_cant_import_other(self): def test_simple_repr(self): import foo.one - assert repr(foo).startswith("<module 'foo' (namespace) from [") + self.assertStartsWith(repr(foo), "<module 'foo' (namespace) from [") class DynamicPathNamespacePackage(NamespacePackageTest): @@ -287,25 +286,24 @@ def test_project3_succeeds(self): class ZipWithMissingDirectory(NamespacePackageTest): paths = ['missing_directory.zip'] + # missing_directory.zip contains: + # Length Date Time Name + # --------- ---------- ----- ---- + # 29 2012-05-03 18:13 foo/one.py + # 0 2012-05-03 20:57 bar/ + # 38 2012-05-03 20:57 bar/two.py + # --------- ------- + # 67 3 files - @unittest.expectedFailure def test_missing_directory(self): - # This will fail because missing_directory.zip contains: - # Length Date Time Name - # --------- ---------- ----- ---- - # 29 2012-05-03 18:13 foo/one.py - # 0 2012-05-03 20:57 bar/ - # 38 2012-05-03 20:57 bar/two.py - # --------- ------- - # 67 3 files - - # Because there is no 'foo/', the zipimporter currently doesn't - # know that foo is a namespace package - import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_missing_directory2(self): + import foo + self.assertNotHasAttr(foo, 'one') def test_present_directory(self): - # This succeeds because there is a "bar/" in the zip file import bar.two self.assertEqual(bar.two.attr, 'missing_directory foo two') diff --git a/Lib/test/test_importlib/test_pkg_import.py b/Lib/test/test_importlib/test_pkg_import.py index 66f5f8bc253..5ffae6222ba 100644 --- a/Lib/test/test_importlib/test_pkg_import.py +++ b/Lib/test/test_importlib/test_pkg_import.py @@ -55,7 +55,7 @@ def test_package_import__semantics(self): except SyntaxError: pass else: raise RuntimeError('Failed to induce SyntaxError') # self.fail()? self.assertNotIn(self.module_name, sys.modules) - self.assertFalse(hasattr(sys.modules[self.package_name], 'foo')) + self.assertNotHasAttr(sys.modules[self.package_name], 'foo') # ...make up a variable name that isn't bound in __builtins__ var = 'a' diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index 921b6bbece0..aebeabaf83f 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -237,7 +237,7 @@ def test_exec(self): self.spec.loader = NewLoader() module = self.util.module_from_spec(self.spec) sys.modules[self.name] = module - self.assertFalse(hasattr(module, 'eggs')) + self.assertNotHasAttr(module, 'eggs') self.bootstrap._exec(self.spec, module) self.assertEqual(module.eggs, 1) @@ -348,9 +348,9 @@ def test_reload_init_module_attrs(self): self.assertIs(loaded.__loader__, self.spec.loader) self.assertEqual(loaded.__package__, self.spec.parent) self.assertIs(loaded.__spec__, self.spec) - self.assertFalse(hasattr(loaded, '__path__')) - self.assertFalse(hasattr(loaded, '__file__')) - self.assertFalse(hasattr(loaded, '__cached__')) + self.assertNotHasAttr(loaded, '__path__') + self.assertNotHasAttr(loaded, '__file__') + self.assertNotHasAttr(loaded, '__cached__') (Frozen_ModuleSpecMethodsTests, @@ -502,7 +502,8 @@ def test_spec_from_loader_is_package_true_with_fileloader(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -601,7 +602,8 @@ def test_spec_from_file_location_smsl_empty(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -626,7 +628,8 @@ def test_spec_from_file_location_smsl_default(self): self.assertEqual(spec.loader, self.pkgloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -686,10 +689,9 @@ def test_spec_from_file_location_relative_path(self): self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) -# TODO: RUSTPYTHON -# (Frozen_FactoryTests, -# Source_FactoryTests -# ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) +(Frozen_FactoryTests, + Source_FactoryTests + ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 148b2e4370b..c8d156f7f7c 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -6,7 +6,6 @@ # randrange, and then Python hangs. import _imp as imp -import _multiprocessing # TODO: RUSTPYTHON import os import importlib import sys @@ -14,7 +13,7 @@ import shutil import threading import unittest -from unittest import mock +from test import support from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) @@ -136,14 +135,13 @@ def check_parallel_module_init(self, mock_os): if verbose: print("OK.") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parallel_module_init(self): + @unittest.skip("TODO: RUSTPYTHON; flaky") + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_module_init(self, size): self.check_parallel_module_init() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parallel_meta_path(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_meta_path(self, size): finder = Finder() sys.meta_path.insert(0, finder) try: @@ -153,9 +151,9 @@ def test_parallel_meta_path(self): finally: sys.meta_path.remove(finder) - # TODO: RUSTPYTHON; maybe hang? - @unittest.expectedFailure - def test_parallel_path_hooks(self): + @unittest.expectedFailure # TODO: RUSTPYTHON; maybe hang? + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_path_hooks(self, size): # Here the Finder instance is only used to check concurrent calls # to path_hook(). finder = Finder() @@ -249,16 +247,17 @@ def target(): __import__(TESTFN) del sys.modules[TESTFN] - @unittest.skip("TODO: RUSTPYTHON; hang") - def test_concurrent_futures_circular_import(self): + @unittest.skip("TODO: RUSTPYTHON; hang; Suspected cause of crashes in Windows CI - PermissionError: [WinError 32] Permission denied: \"C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\test_python_0cdrhhs_\\test_python_6340æ\"") + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_concurrent_futures_circular_import(self, size): # Regression test for bpo-43515 fn = os.path.join(os.path.dirname(__file__), 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - @unittest.skipUnless(hasattr(_multiprocessing, "SemLock"), "TODO: RUSTPYTHON, pool_in_threads.py needs _multiprocessing.SemLock") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_multiprocessing_pool_circular_import(self): + @unittest.skip("TODO: RUSTPYTHON; hang") + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_multiprocessing_pool_circular_import(self, size): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), 'partial', 'pool_in_threads.py') @@ -271,7 +270,7 @@ def setUpModule(): try: old_switchinterval = sys.getswitchinterval() unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval) - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) except AttributeError: pass diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 201f5069114..8c14b96271a 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -6,12 +6,13 @@ importlib_util = util.import_importlib('importlib.util') import importlib.util +from importlib import _bootstrap_external import os import pathlib -import re import string import sys from test import support +from test.support import os_helper import textwrap import types import unittest @@ -27,7 +28,7 @@ except ImportError: _testmultiphase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None @@ -319,7 +320,7 @@ def test_length(self): def test_incorporates_rn(self): # The magic number uses \r\n to come out wrong when splitting on lines. - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) + self.assertEndsWith(self.util.MAGIC_NUMBER, b'\r\n') (Frozen_MagicNumberTests, @@ -327,15 +328,6 @@ def test_incorporates_rn(self): ) = util.test_both(MagicNumberTests, util=importlib_util) -# TODO: RUSTPYTHON -@unittest.expectedFailure -def test_incorporates_rn_MONKEYPATCH(self): - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) - -# TODO: RUSTPYTHON -Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH - - class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" @@ -367,8 +359,6 @@ def test_cache_from_source_no_dot(self): self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cache_from_source_debug_override(self): # Given the path to a .py file, return the path to its PEP 3147/PEP 488 # defined .pyc file (i.e. under __pycache__). @@ -588,7 +578,19 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): with util.temporary_pycache_prefix(pycache_prefix): self.assertEqual( self.util.cache_from_source(path, optimization=''), - expect) + os.path.normpath(expect)) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_in_root_with_pycache_prefix(self): + # Regression test for gh-82916 + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = 'qux.py' + expect = os.path.join(os.path.sep, 'tmp', 'bytecode', + f'qux.{self.tag}.pyc') + with util.temporary_pycache_prefix(pycache_prefix): + with os_helper.change_cwd('/'): + self.assertEqual(self.util.cache_from_source(path), expect) @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') @@ -645,7 +647,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3531 + EXPECTED_MAGIC_NUMBER = 3627 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -666,27 +668,36 @@ def test_magic_number(self): @unittest.skipIf(_interpreters is None, 'subinterpreters required') class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): - ERROR = re.compile("^<class 'ImportError'>: module (.*) does not support loading in subinterpreters") - def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('isolated') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + # gh-117649: single-phase init modules are not currently supported in + # subinterpreters in the free-threaded build + @support.expected_failure_if_gil_disabled() def test_single_phase_init_module(self): script = textwrap.dedent(''' from importlib.util import _incompatible_extension_module_restrictions @@ -711,14 +722,22 @@ def test_single_phase_init_module(self): self.run_with_own_gil(script) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + @support.requires_gil_enabled("gh-117649: not supported in free-threaded build") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') @@ -769,5 +788,74 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): + import _pyio + + oldwrite = os.write + + # Emulate an os.write that only writes partial data. + def write(fd, data): + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) + + # Need to patch _io to be _pyio, so that io.FileIO is affected by the + # os.write patch. + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + with self.assertRaises(OSError): + os.stat(support.os_helper.TESTFN) # Check that the file did not get written. + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index f8a9ead9ac8..0ae911bc43d 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -5,7 +5,7 @@ import re import sys import unittest -import warnings +from test import support from test.support import import_helper from contextlib import contextmanager from test.test_importlib.util import temp_module @@ -91,31 +91,61 @@ class WindowsRegistryFinderTests: test_module = "spamham{}".format(os.getpid()) def test_find_spec_missing(self): - spec = self.machinery.WindowsRegistryFinder.find_spec('spam') + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec('spam') self.assertIsNone(spec) def test_module_found(self): with setup_module(self.machinery, self.test_module): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNotNone(spec) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNone(spec) + def test_raises_deprecation_warning(self): + # WindowsRegistryFinder is not meant to be instantiated, so the + # deprecation warning is raised in the 'find_spec' method instead. + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + self.machinery.WindowsRegistryFinder.find_spec('spam') + (Frozen_WindowsRegistryFinderTests, Source_WindowsRegistryFinderTests ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery) @unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') class WindowsExtensionSuffixTests: - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; no C extension (.pyd) support def test_tagged_suffix(self): suffixes = self.machinery.EXTENSION_SUFFIXES - expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info, - re.sub('[^a-zA-Z0-9]', '_', get_platform())) + abi_flags = "t" if support.Py_GIL_DISABLED else "" + ver = sys.version_info + platform = re.sub('[^a-zA-Z0-9]', '_', get_platform()) + expected_tag = f".cp{ver.major}{ver.minor}{abi_flags}-{platform}.pyd" try: untagged_i = suffixes.index(".pyd") except ValueError: diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e52..85e7ffcb608 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -6,13 +6,20 @@ import marshal import os import os.path +from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys import tempfile import types +try: + _testsinglephase = import_helper.import_module("_testsinglephase") +except unittest.SkipTest: + _testsinglephase = None # TODO: RUSTPYTHON + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -22,25 +29,39 @@ if 'importlib' not in sys.builtin_module_names: BUILTINS.bad_name = 'importlib' -EXTENSIONS = types.SimpleNamespace() -EXTENSIONS.path = None -EXTENSIONS.ext = None -EXTENSIONS.filename = None -EXTENSIONS.file_path = None -EXTENSIONS.name = '_testsinglephase' - -def _extension_details(): - global EXTENSIONS - for path in sys.path: - for ext in machinery.EXTENSION_SUFFIXES: - filename = EXTENSIONS.name + ext - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - EXTENSIONS.path = path - EXTENSIONS.ext = ext - EXTENSIONS.filename = filename - EXTENSIONS.file_path = file_path - return +if support.is_wasi: + # dlopen() is a shim for WASI as of WASI SDK which fails by default. + # We don't provide an implementation, so tests will fail. + # But we also don't want to turn off dynamic loading for those that provide + # a working implementation. + def _extension_details(): + global EXTENSIONS + EXTENSIONS = None +else: + EXTENSIONS = types.SimpleNamespace() + EXTENSIONS.path = None + EXTENSIONS.ext = None + EXTENSIONS.filename = None + EXTENSIONS.file_path = None + EXTENSIONS.name = '_testsinglephase' + + def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return _extension_details() diff --git a/Lib/test/test_inspect/inspect_deferred_annotations.py b/Lib/test/test_inspect/inspect_deferred_annotations.py new file mode 100644 index 00000000000..bb59ef1035b --- /dev/null +++ b/Lib/test/test_inspect/inspect_deferred_annotations.py @@ -0,0 +1,2 @@ +def f(x: undefined): + pass diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d0fec18250e..0aaf9176d8d 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,3 +1,4 @@ +from annotationlib import Format, ForwardRef import asyncio import builtins import collections @@ -12,6 +13,7 @@ import os import dis from os.path import normcase +# import _pickle # TODO: RUSTPYTHON import pickle import shutil import stat @@ -21,45 +23,36 @@ import types import tempfile import textwrap -from typing import Unpack import unicodedata import unittest import unittest.mock import warnings import weakref -# XXX: RUSTPYTHON; skip _pickle tests if _pickle is not available -try: - import _pickle -except ImportError: - _pickle = None - try: from concurrent.futures import ThreadPoolExecutor except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only, import_helper, suppress_immortalization +from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ -# XXX: RUSTPYTHON; test.support is not updated yet -from test.support.import_helper import DirsOnSysPath #, ready_to_import +from test.support import run_no_yield_async_fn, EqualToForwardRef +from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python -from test.support import has_subprocess_support, SuppressCrashReport +from test.support import has_subprocess_support from test import support from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 -from test.test_inspect import inspect_stock_annotations from test.test_inspect import inspect_stringized_annotations -from test.test_inspect import inspect_stringized_annotations_2 -from test.test_inspect import inspect_stringized_annotations_pep695 +from test.test_inspect import inspect_deferred_annotations # Functions tested in this suite: # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, -# isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, +# isbuiltin, isroutine, isgenerator, ispackage, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, # getclasstree, getargvalues, formatargvalues, currentframe, # stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper @@ -81,11 +74,6 @@ def revise(filename, *args): git = mod.StupidGit() -def tearDownModule(): - if support.has_socket_support: - asyncio.set_event_loop_policy(None) - - def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -113,7 +101,7 @@ def unsorted_keyword_only_parameters_fn(*, throw, out, the, baby, with_, class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, - inspect.ismodule, inspect.istraceback, + inspect.ismodule, inspect.istraceback, inspect.ispackage, inspect.isgenerator, inspect.isgeneratorfunction, inspect.iscoroutine, inspect.iscoroutinefunction, inspect.isasyncgen, inspect.isasyncgenfunction, @@ -129,12 +117,14 @@ def istest(self, predicate, exp): predicate == inspect.iscoroutinefunction) and \ other == inspect.isfunction: continue - self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) + if predicate == inspect.ispackage and other == inspect.ismodule: + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + else: + self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; First has 0, Second has 1: 'iskeyword' def test__all__(self): - support.check__all__(self, inspect, not_exported=("modulesbyfile",)) + support.check__all__(self, inspect, not_exported=("modulesbyfile",), extra=("get_annotations",)) def generator_function_example(self): for i in range(2): @@ -185,8 +175,7 @@ def __get__(self, instance, owner): class TestPredicates(IsTestBase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'defaultdict' has no attribute 'default_factory' def test_excluding_predicates(self): global tb self.istest(inspect.isbuiltin, 'sys.exit') @@ -236,10 +225,19 @@ def test_excluding_predicates(self): self.assertFalse(inspect.ismethodwrapper(int)) self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) + def test_ispackage(self): + self.istest(inspect.ispackage, 'unittest') + self.istest(inspect.ispackage, 'importlib') + self.assertFalse(inspect.ispackage(inspect)) + self.assertFalse(inspect.ispackage(mod)) + self.assertFalse(inspect.ispackage(':)')) + class FakePackage: + __path__ = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertFalse(inspect.ispackage(FakePackage())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_iscoroutine(self): async_gen_coro = async_generator_function_example(1) gen_coro = gen_coroutine_function_example(1) @@ -384,8 +382,6 @@ def do_something_static(): coro.close(); gen_coro.close(); # silence warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_isawaitable(self): def gen(): yield self.assertFalse(inspect.isawaitable(gen())) @@ -443,7 +439,7 @@ def test_isroutine(self): self.assertFalse(inspect.isroutine(int)) self.assertFalse(inspect.isroutine(type('some_class', (), {}))) # partial - self.assertFalse(inspect.isroutine(functools.partial(mod.spam))) + self.assertTrue(inspect.isroutine(functools.partial(mod.spam))) def test_isroutine_singledispatch(self): self.assertTrue(inspect.isroutine(functools.singledispatch(mod.spam))) @@ -484,8 +480,7 @@ class C(object): self.assertIn('a', members) self.assertNotIn('b', members) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_isabstract(self): from abc import ABCMeta, abstractmethod @@ -508,8 +503,7 @@ def foo(self): self.assertFalse(inspect.isabstract(int)) self.assertFalse(inspect.isabstract(5)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + [True, False] def test_isabstract_during_init_subclass(self): from abc import ABCMeta, abstractmethod isabstract_checks = [] @@ -545,8 +539,6 @@ def test_abuse_done(self): self.istest(inspect.istraceback, 'git.ex.__traceback__') self.istest(inspect.isframe, 'mod.fr') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_stack(self): self.assertTrue(len(mod.st) >= 5) frame1, frame2, frame3, frame4, *_ = mod.st @@ -575,8 +567,6 @@ def test_stack(self): self.assertIn('inspect.stack()', record.code_context[0]) self.assertEqual(record.index, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_trace(self): self.assertEqual(len(git.tr), 3) frame1, frame2, frame3, = git.tr @@ -599,8 +589,7 @@ def test_frame(self): self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), '(x=11, y=14)') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'f_code' def test_previous_frame(self): args, varargs, varkw, locals = inspect.getargvalues(mod.fr.f_back) self.assertEqual(args, ['a', 'b', 'c', 'd', 'e', 'f']) @@ -682,8 +671,6 @@ def test_getfunctions(self): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getdoc(self): self.assertEqual(inspect.getdoc(mod), 'A module docstring.') self.assertEqual(inspect.getdoc(mod.StupidGit), @@ -828,12 +815,12 @@ def test_getfile(self): def test_getfile_builtin_module(self): with self.assertRaises(TypeError) as e: inspect.getfile(sys) - self.assertTrue(str(e.exception).startswith('<module')) + self.assertStartsWith(str(e.exception), '<module') def test_getfile_builtin_class(self): with self.assertRaises(TypeError) as e: inspect.getfile(int) - self.assertTrue(str(e.exception).startswith('<class')) + self.assertStartsWith(str(e.exception), '<class') def test_getfile_builtin_function_or_method(self): with self.assertRaises(TypeError) as e_abs: @@ -843,7 +830,6 @@ def test_getfile_builtin_function_or_method(self): inspect.getfile(list.append) self.assertIn('expected, got', str(e_append.exception)) - @suppress_immortalization() def test_getfile_class_without_module(self): class CM(type): @property @@ -902,8 +888,7 @@ def test_getsource_on_generated_class(self): self.assertRaises(OSError, inspect.getsourcelines, A) self.assertIsNone(inspect.getcomments(A)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by getsource def test_getsource_on_class_without_firstlineno(self): __firstlineno__ = 1 class C: @@ -913,8 +898,6 @@ class C: class TestGetsourceStdlib(unittest.TestCase): # Test Python implementations of the stdlib modules - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getsource_stdlib_collections_abc(self): import collections.abc lines, lineno = inspect.getsourcelines(collections.abc.Sequence) @@ -927,8 +910,7 @@ def test_getsource_stdlib_tomllib(self): self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by getsource def test_getsource_stdlib_abc(self): # Pure Python implementation abc = import_helper.import_fresh_module('abc', blocked=['_abc']) @@ -957,8 +939,6 @@ def test_getsource_stdlib_decimal(self): class TestGetsourceInteractive(unittest.TestCase): @support.force_not_colorized - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; # there is no `__file__` in the __main__ module @@ -982,8 +962,7 @@ def test_range_traceback_toplevel_frame(self): class TestDecorators(GetSourceBase): fodderModule = mod2 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; pass def test_wrapped_decorator(self): self.assertSourceEqual(mod2.wrapped, 14, 17) @@ -1184,8 +1163,7 @@ def test_nested_class_definition(self): self.assertSourceEqual(mod2.cls183, 183, 188) self.assertSourceEqual(mod2.cls183.cls185, 185, 188) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; pass def test_class_decorator(self): self.assertSourceEqual(mod2.cls196, 194, 201) self.assertSourceEqual(mod2.cls196.cls200, 198, 201) @@ -1239,17 +1217,13 @@ def f(self): # This is necessary when the test is run multiple times. sys.modules.pop("inspect_actual") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_nested_class_definition_inside_async_function(self): - import asyncio - self.addCleanup(asyncio.set_event_loop_policy, None) - self.assertSourceEqual(asyncio.run(mod2.func225()), 226, 227) + run = run_no_yield_async_fn + + self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) self.assertSourceEqual(mod2.cls226.func232, 232, 235) - self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) @@ -1286,8 +1260,7 @@ def test_class(self): class TestComplexDecorator(GetSourceBase): fodderModule = mod2 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; return foo + bar() def test_parens_in_decorator(self): self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) @@ -1399,9 +1372,9 @@ def test(): pass spec = inspect.getfullargspec(test) self.assertEqual(test.__annotations__, spec.annotations) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - @unittest.skipIf(_pickle is None, "requires _pickle") def test_getfullargspec_builtin_methods(self): self.assertFullArgSpecEquals(_pickle.Pickler.dump, ['self', 'obj']) @@ -1460,7 +1433,7 @@ def test_getfullargspec_builtin_func_no_signature(self): (dict.__class_getitem__, meth_type_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -1483,8 +1456,6 @@ def test_getfullargspec_definition_order_preserved_on_kwonly(self): l = list(signature.kwonlyargs) self.assertEqual(l, unsorted_keyword_only_parameters) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_classify_newstyle(self): class A(object): @@ -1566,8 +1537,7 @@ def m1(self): pass self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ('to_bytes', 'method', <class 'int'>) not found in [('__abs__', 'method', <class 'bool'>), ('__add__', 'method', <class 'bool'>), ('__and__', 'method', <class 'bool'>), ('__bool__', 'method', <class 'bool'>), ('__ceil__', 'class method', <class 'int'>), ('__class__', 'data', <class 'object'>), ('__delattr__', 'class method', <class 'object'>), ('__dir__', 'class method', <class 'object'>), ('__divmod__', 'method', <class 'bool'>), ('__doc__', 'data', <class 'bool'>), ('__eq__', 'class method', <class 'int'>), ('__float__', 'method', <class 'bool'>), ('__floor__', 'class method', <class 'int'>), ('__floordiv__', 'method', <class 'bool'>), ('__format__', 'class method', <class 'bool'>), ('__ge__', 'class method', <class 'int'>), ('__getattribute__', 'class method', <class 'object'>), ('__getnewargs__', 'class method', <class 'int'>), ('__getstate__', 'class method', <class 'object'>), ('__gt__', 'class method', <class 'int'>), ('__hash__', 'method', <class 'int'>), ('__index__', 'method', <class 'bool'>), ('__init__', 'method', <class 'object'>), ('__init_subclass__', 'class method', <class 'object'>), ('__int__', 'method', <class 'bool'>), ('__invert__', 'method', <class 'bool'>), ('__le__', 'class method', <class 'int'>), ('__lshift__', 'method', <class 'bool'>), ('__lt__', 'class method', <class 'int'>), ('__mod__', 'method', <class 'bool'>), ('__mul__', 'method', <class 'bool'>), ('__ne__', 'class method', <class 'int'>), ('__neg__', 'method', <class 'bool'>), ('__new__', 'static method', <class 'bool'>), ('__or__', 'method', <class 'bool'>), ('__pos__', 'method', <class 'bool'>), ('__pow__', 'method', <class 'bool'>), ('__radd__', 'method', <class 'bool'>), ('__rand__', 'method', <class 'bool'>), ('__rdivmod__', 'method', <class 'bool'>), ('__reduce__', 'class method', <class 'object'>), ('__reduce_ex__', 'class method', <class 'object'>), ('__repr__', 'method', <class 'bool'>), ('__rfloordiv__', 'method', <class 'bool'>), ('__rlshift__', 'method', <class 'bool'>), ('__rmod__', 'method', <class 'bool'>), ('__rmul__', 'method', <class 'bool'>), ('__ror__', 'method', <class 'bool'>), ('__round__', 'class method', <class 'int'>), ('__rpow__', 'method', <class 'bool'>), ('__rrshift__', 'method', <class 'bool'>), ('__rshift__', 'method', <class 'bool'>), ('__rsub__', 'method', <class 'bool'>), ('__rtruediv__', 'method', <class 'bool'>), ('__rxor__', 'method', <class 'bool'>), ('__setattr__', 'class method', <class 'object'>), ('__sizeof__', 'class method', <class 'int'>), ('__str__', 'method', <class 'object'>), ('__sub__', 'method', <class 'bool'>), ('__subclasshook__', 'class method', <class 'object'>), ('__truediv__', 'method', <class 'bool'>), ('__trunc__', 'class method', <class 'int'>), ('__xor__', 'method', <class 'bool'>), ('as_integer_ratio', 'class method', <class 'int'>), ('bit_count', 'class method', <class 'int'>), ('bit_length', 'class method', <class 'int'>), ('conjugate', 'class method', <class 'int'>), ('denominator', 'data', <class 'int'>), ('from_bytes', 'class method', <class 'int'>), ('imag', 'data', <class 'int'>), ('is_integer', 'class method', <class 'int'>), ('numerator', 'data', <class 'int'>), ('real', 'data', <class 'int'>), ('to_bytes', 'class method', <class 'int'>)] : missing plain method def test_classify_builtin_types(self): # Simple sanity check that all built-in types can have their # attributes classified. @@ -1822,230 +1792,24 @@ class C(metaclass=M): attrs = [a[0] for a in inspect.getmembers(C)] self.assertNotIn('missing', attrs) - def test_get_annotations_with_stock_annotations(self): - def foo(a:int, b:str): pass - self.assertEqual(inspect.get_annotations(foo), {'a': int, 'b': str}) - - foo.__annotations__ = {'a': 'foo', 'b':'str'} - self.assertEqual(inspect.get_annotations(foo), {'a': 'foo', 'b': 'str'}) - - self.assertEqual(inspect.get_annotations(foo, eval_str=True, locals=locals()), {'a': foo, 'b': str}) - self.assertEqual(inspect.get_annotations(foo, eval_str=True, globals=locals()), {'a': foo, 'b': str}) - - isa = inspect_stock_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect), {}) # inspect module has no annotations - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - - def test_get_annotations_with_stringized_annotations(self): - isa = inspect_stringized_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - isa2 = inspect_stringized_annotations_2 - self.assertEqual(inspect.get_annotations(isa2), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - - # test that local namespace lookups work - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) - - def test_pep695_generic_class_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) - A_type_params = ann_module695.A.__type_params__ - self.assertIs(A_annotations["x"], A_type_params[0]) - self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) - self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) - - def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): - B_annotations = inspect.get_annotations( - inspect_stringized_annotations_pep695.B, eval_str=True - ) - self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) - - def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): - ann_module695 = inspect_stringized_annotations_pep695 - C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) - self.assertEqual( - set(C_annotations.values()), - set(ann_module695.C.__type_params__) - ) - - def test_pep_695_generic_function_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - generic_func_annotations = inspect.get_annotations( - ann_module695.generic_function, eval_str=True - ) - func_t_params = ann_module695.generic_function.__type_params__ - self.assertEqual( - generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} - ) - self.assertIs(generic_func_annotations["x"], func_t_params[0]) - self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) - self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) - self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) - - def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): - self.assertEqual( - set( - inspect.get_annotations( - inspect_stringized_annotations_pep695.generic_function_2, - eval_str=True - ).values() - ), - set( - inspect_stringized_annotations_pep695.generic_function_2.__type_params__ - ) - ) - - def test_pep_695_generic_method_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - generic_method_annotations = inspect.get_annotations( - ann_module695.D.generic_method, eval_str=True - ) - params = { - param.__name__: param - for param in ann_module695.D.generic_method.__type_params__ - } - self.assertEqual( - generic_method_annotations, - {"x": params["Foo"], "y": params["Bar"], "return": None} - ) - - def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): - self.assertEqual( - set( - inspect.get_annotations( - inspect_stringized_annotations_pep695.D.generic_method_2, - eval_str=True - ).values() - ), - set( - inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ - ) - ) - - def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): - self.assertEqual( - inspect.get_annotations( - inspect_stringized_annotations_pep695.E, eval_str=True - ), - {"x": str}, - ) - - def test_pep_695_generics_with_future_annotations_nested_in_function(self): - results = inspect_stringized_annotations_pep695.nested() - - self.assertEqual( - set(results.F_annotations.values()), - set(results.F.__type_params__) - ) - self.assertEqual( - set(results.F_meth_annotations.values()), - set(results.F.generic_method.__type_params__) - ) - self.assertNotEqual( - set(results.F_meth_annotations.values()), - set(results.F.__type_params__) - ) - self.assertEqual( - set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), - set() - ) - - self.assertEqual(results.G_annotations, {"x": str}) - - self.assertEqual( - set(results.generic_func_annotations.values()), - set(results.generic_func.__type_params__) - ) - class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): from test.typinganndata.ann_module9 import A, ann, ann1 - self.assertEqual(inspect.formatannotation(ann), 'Union[List[str], int]') - self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') + self.assertEqual(inspect.formatannotation(ann), 'List[str] | int') + self.assertEqual(inspect.formatannotation(ann1), 'List[testModule.typing.A] | int') self.assertEqual(inspect.formatannotation(A, 'testModule.typing'), 'A') self.assertEqual(inspect.formatannotation(A, 'other'), 'testModule.typing.A') self.assertEqual( inspect.formatannotation(ann1, 'testModule.typing'), - 'Union[List[testModule.typing.A], int]', + 'List[testModule.typing.A] | int', ) + def test_forwardref(self): + fwdref = ForwardRef('fwdref') + self.assertEqual(inspect.formatannotation(fwdref), 'fwdref') + def test_formatannotationrelativeto(self): from test.typinganndata.ann_module9 import A, ann1 @@ -2077,7 +1841,7 @@ class B: ... # Not an instance of "type": self.assertEqual( inspect.formatannotationrelativeto(A)(ann1), - 'Union[List[testModule.typing.A], int]', + 'List[testModule.typing.A] | int', ) @@ -2128,8 +1892,6 @@ class DataDescriptorSub(DataDescriptorWithNoGet, self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_builtin_descriptors(self): builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. class Owner: @@ -2176,7 +1938,7 @@ def function(): self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) self.assertFalse(inspect.ismethoddescriptor(function)) self.assertFalse(inspect.ismethoddescriptor(a_lambda)) - self.assertFalse(inspect.ismethoddescriptor(functools.partial(function))) + self.assertTrue(inspect.ismethoddescriptor(functools.partial(function))) def test_descriptor_being_a_class(self): class MethodDescriptorMeta(type): @@ -2219,8 +1981,6 @@ class DataDescriptor2: self.assertTrue(inspect.isdatadescriptor(DataDescriptor2()), 'class with __set__ = None is a data descriptor') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slot(self): class Slotted: __slots__ = 'foo', @@ -2260,8 +2020,7 @@ def function(): _global_ref = object() class TestGetClosureVars(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[132 chars]={'print': <built-in function print>}, unbound=set()) != Closu[132 chars]={'print': <built-in function print>}, unbound={'unbound_ref'}) def test_name_resolution(self): # Basic test of the 4 different resolution mechanisms def f(nonlocal_ref): @@ -2277,8 +2036,7 @@ def g(local_ref): builtin_vars, unbound_names) self.assertEqual(inspect.getclosurevars(f(_arg)), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[132 chars]={'print': <built-in function print>}, unbound=set()) != Closu[132 chars]={'print': <built-in function print>}, unbound={'unbound_ref'}) def test_generator_closure(self): def f(nonlocal_ref): def g(local_ref): @@ -2294,8 +2052,7 @@ def g(local_ref): builtin_vars, unbound_names) self.assertEqual(inspect.getclosurevars(f(_arg)), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[132 chars]={'print': <built-in function print>}, unbound=set()) != Closu[132 chars]={'print': <built-in function print>}, unbound={'unbound_ref'}) def test_method_closure(self): class C: def f(self, nonlocal_ref): @@ -2311,8 +2068,7 @@ def g(local_ref): builtin_vars, unbound_names) self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[139 chars]als={}, builtins={'print': <built-in function [18 chars]et()) != Closu[139 chars]als={'_global_ref': <object object at 0xa4ffee[73 chars]ef'}) def test_attribute_same_name_as_global_var(self): class C: _global_ref = object() @@ -2382,24 +2138,21 @@ def _private_globals(self): exec(code, ns) return ns["f"], ns - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[34 chars]uiltins={'print': <built-in function print>}, unbound=set()) != Closu[34 chars]uiltins={'print': <built-in function print>}, unbound={'path'}) def test_builtins_fallback(self): f, ns = self._private_globals() ns.pop("__builtins__", None) expected = inspect.ClosureVars({}, {}, {"print":print}, {"path"}) self.assertEqual(inspect.getclosurevars(f), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ClosureVars(nonlocals={}, globals={}, builtins={}, unbound={'print'}) != ClosureVars(nonlocals={}, globals={}, builtins={'path': 1}, unbound={'print'}) def test_builtins_as_dict(self): f, ns = self._private_globals() ns["__builtins__"] = {"path":1} expected = inspect.ClosureVars({}, {}, {"path":1}, {"print"}) self.assertEqual(inspect.getclosurevars(f), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Closu[38 chars]ins={}, unbound={'print'}) != Closu[38 chars]ins={'path': <module 'posixpath' from '/Users/[79 chars]nt'}) def test_builtins_as_module(self): f, ns = self._private_globals() ns["__builtins__"] = os @@ -2494,8 +2247,6 @@ def test_varkw_only(self): self.assertEqualCallArgs(f, '**collections.UserDict(a=1, b=2)') self.assertEqualCallArgs(f, 'c=3, **collections.UserDict(a=1, b=2)') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_keyword_only(self): f = self.makeCallable('a=3, *, c, d=2') self.assertEqualCallArgs(f, 'c=3') @@ -2536,8 +2287,7 @@ def test_multiple_features(self): '(4,[5,6])]), q=0, **collections.UserDict(' 'y=9, z=10)') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + <lambda>() got an unexpected keyword argument 'x' def test_errors(self): f0 = self.makeCallable('') f1 = self.makeCallable('a, b') @@ -2964,7 +2714,7 @@ def __getattribute__(self, attr): self.assertFalse(test.called) - @suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'test.test_inspect.test_inspect.TestGetattrStatic.test_cache_does_not_cause_classes_to_persist.<locals>.Foo'> is not None def test_cache_does_not_cause_classes_to_persist(self): # regression test for gh-118013: # check that the internal _shadowed_dict cache does not cause @@ -2992,33 +2742,23 @@ def number_generator(): def _generatorstate(self): return inspect.getgeneratorstate(self.generator) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_created(self): self.assertEqual(self._generatorstate(), inspect.GEN_CREATED) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_suspended(self): next(self.generator) self.assertEqual(self._generatorstate(), inspect.GEN_SUSPENDED) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_closed_after_exhaustion(self): for i in self.generator: pass self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_closed_after_immediate_exception(self): with self.assertRaises(RuntimeError): self.generator.throw(RuntimeError) self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_closed_after_close(self): self.generator.close() self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) @@ -3039,6 +2779,31 @@ def running_check_generator(): # Running after the first yield next(self.generator) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: '_GeneratorWrapper' object has no attribute 'gi_suspended' + def test_types_coroutine_wrapper_state(self): + def gen(): + yield 1 + yield 2 + + @types.coroutine + def wrapped_generator_coro(): + # return a generator iterator so types.coroutine + # wraps it into types._GeneratorWrapper. + return gen() + + g = wrapped_generator_coro() + self.addCleanup(g.close) + self.assertIs(type(g), types._GeneratorWrapper) + + # _GeneratorWrapper must provide gi_suspended/cr_suspended + # so inspect.get*state() doesn't raise AttributeError. + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CREATED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_CREATED) + + next(g) + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_SUSPENDED) + def test_easy_debugging(self): # repr() and str() of a generator state should contain the state name names = 'GEN_CREATED GEN_RUNNING GEN_SUSPENDED GEN_CLOSED'.split() @@ -3047,8 +2812,6 @@ def test_easy_debugging(self): self.assertIn(name, repr(state)) self.assertIn(name, str(state)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getgeneratorlocals(self): def each(lst, a=None): b=(1, 2, 3) @@ -3113,19 +2876,16 @@ def tearDown(self): def _coroutinestate(self): return inspect.getcoroutinestate(self.coroutine) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' def test_created(self): self.assertEqual(self._coroutinestate(), inspect.CORO_CREATED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' def test_suspended(self): self.coroutine.send(None) self.assertEqual(self._coroutinestate(), inspect.CORO_SUSPENDED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' def test_closed_after_exhaustion(self): while True: try: @@ -3135,15 +2895,13 @@ def test_closed_after_exhaustion(self): self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' def test_closed_after_immediate_exception(self): with self.assertRaises(RuntimeError): self.coroutine.throw(RuntimeError) self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' def test_closed_after_close(self): self.coroutine.close() self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) @@ -3186,23 +2944,24 @@ async def number_asyncgen(): async def asyncTearDown(self): await self.asyncgen.aclose() + @classmethod + def tearDownClass(cls): + asyncio.events._set_event_loop_policy(None) + def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' def test_created(self): self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' async def test_suspended(self): value = await anext(self.asyncgen) self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) self.assertEqual(value, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' async def test_closed_after_exhaustion(self): countdown = 7 with self.assertRaises(StopAsyncIteration): @@ -3211,15 +2970,13 @@ async def test_closed_after_exhaustion(self): self.assertEqual(countdown, 1) self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' async def test_closed_after_immediate_exception(self): with self.assertRaises(RuntimeError): await self.asyncgen.athrow(RuntimeError) self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' async def test_running(self): async def running_check_asyncgen(): for number in range(5): @@ -3242,8 +2999,6 @@ def test_easy_debugging(self): self.assertIn(name, repr(state)) self.assertIn(name, str(state)) - # TODO: RUSTPYTHON - @unittest.expectedFailure async def test_getasyncgenlocals(self): async def each(lst, a=None): b=(1, 2, 3) @@ -3325,7 +3080,7 @@ def test(po, /, pk, pkd=100, *args, ko, kod=10, **kwargs): pass sig = inspect.signature(test) - self.assertTrue(repr(sig).startswith('<Signature')) + self.assertStartsWith(repr(sig), '<Signature') self.assertTrue('(po, /, pk' in repr(sig)) # We need two functions, because it is impossible to represent @@ -3334,7 +3089,7 @@ def test2(pod=42, /): pass sig2 = inspect.signature(test2) - self.assertTrue(repr(sig2).startswith('<Signature')) + self.assertStartsWith(repr(sig2), '<Signature') self.assertTrue('(pod=42, /)' in repr(sig2)) po = sig.parameters['po'] @@ -3390,6 +3145,17 @@ def test2(pod=42, /): with self.assertRaisesRegex(ValueError, 'follows default argument'): S((pkd, pk)) + second_args = args.replace(name="second_args") + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, second_args)) + + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, ko, second_args)) + + second_kwargs = kwargs.replace(name="second_kwargs") + with self.assertRaisesRegex(ValueError, 'more than one variadic keyword parameter'): + S((kwargs, second_kwargs)) + def test_signature_object_pickle(self): def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass foo_partial = functools.partial(foo, a=1) @@ -3641,7 +3407,7 @@ def test_signature_on_builtins_no_signature(self): (dict.__class_getitem__, meth_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -3776,8 +3542,7 @@ def m1d(*args, **kwargs): ('arg2', 1, ..., "positional_or_keyword")), int)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: no signature found for builtin type <class 'classmethod'> def test_signature_on_classmethod(self): if not support.MISSING_C_DOCSTRINGS: self.assertEqual(self.signature(classmethod), @@ -3801,8 +3566,7 @@ def foo(cls, arg1, *, arg2=1): ('arg2', 1, ..., "keyword_only")), ...)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: no signature found for builtin type <class 'staticmethod'> def test_signature_on_staticmethod(self): if not support.MISSING_C_DOCSTRINGS: self.assertEqual(self.signature(staticmethod), @@ -3827,7 +3591,7 @@ def foo(cls, *, arg): ...)) def test_signature_on_partial(self): - from functools import partial + from functools import partial, Placeholder def test(): pass @@ -3882,6 +3646,25 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass @@ -3929,6 +3712,15 @@ def test(a, *args, b, **kwargs): ('kwargs', ..., ..., "var_keyword")), ...)) + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + def test(a, b, c:int) -> 42: pass @@ -4033,6 +3825,34 @@ def foo(a, b, /, c, d, **kwargs): ('kwargs', ..., ..., 'var_keyword')), ...)) + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), + ...)) + def test_signature_on_partialmethod(self): from functools import partialmethod @@ -4045,18 +3865,32 @@ def test(): inspect.signature(Spam.ham) class Spam: - def test(it, a, *, c) -> 'spam': + def test(it, a, b, *, c) -> 'spam': pass ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((('it', ..., ..., 'positional_or_keyword'), ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().ham, eval_str=False), ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), + ((('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) @@ -4146,8 +3980,7 @@ def wrapped_foo_call(): ('b', ..., ..., "positional_or_keyword")), ...)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_signature_on_class(self): class C: def __init__(self, a): @@ -4213,17 +4046,15 @@ def __init__(self, b): with self.subTest('partial'): class CM(type): - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) class C(metaclass=CM): - def __init__(self, b): + def __init__(self, c): pass - with self.assertWarns(FutureWarning): - self.assertEqual(C(1), (2, 1)) - with self.assertWarns(FutureWarning): - self.assertEqual(self.signature(C), - ((('a', ..., ..., "positional_or_keyword"),), - ...)) + self.assertEqual(C(1), (2, C, 1)) + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) with self.subTest('partialmethod'): class CM(type): @@ -4406,8 +4237,7 @@ def __init__(self, b): self.assertEqual(self.signature(C.__call__, follow_wrapped=False), varargs_signature) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_signature_on_class_with_wrapped_init(self): class C: @identity_wrapper @@ -4459,14 +4289,12 @@ class C: with self.subTest('partial'): class C: - __init__ = functools.partial(identity_wrapper(lambda x, a: None), 2) + __init__ = functools.partial(identity_wrapper(lambda x, a, b: None), 2) - with self.assertWarns(FutureWarning): - C(1) # does not raise - with self.assertWarns(FutureWarning): - self.assertEqual(self.signature(C), - ((('a', ..., ..., "positional_or_keyword"),), - ...)) + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) with self.subTest('partialmethod'): class C: @@ -4507,8 +4335,7 @@ def __init__(self, a): self.assertEqual(self.signature(C.__new__, follow_wrapped=False), varargs_signature) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_signature_on_class_with_wrapped_new(self): with self.subTest('FunctionType'): class C: @@ -4597,8 +4424,6 @@ def __new__(cls, a): self.assertEqual(self.signature(C.__new__, follow_wrapped=False), varargs_signature) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_signature_on_class_with_init(self): class C: def __init__(self, b): @@ -4645,14 +4470,12 @@ class C: with self.subTest('partial'): class C: - __init__ = functools.partial(lambda x, a: None, 2) + __init__ = functools.partial(lambda x, a, b: None, 2) - with self.assertWarns(FutureWarning): - C(1) # does not raise - with self.assertWarns(FutureWarning): - self.assertEqual(self.signature(C), - ((('a', ..., ..., "positional_or_keyword"),), - ...)) + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) with self.subTest('partialmethod'): class C: @@ -4665,8 +4488,7 @@ def _init(self, x, a): ((('a', ..., ..., "positional_or_keyword"),), ...)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_signature_on_class_with_new(self): with self.subTest('FunctionType'): class C: @@ -4827,9 +4649,9 @@ class D(C): pass with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - @unittest.skipIf(_pickle is None, "requires _pickle") def test_signature_on_builtin_class(self): expected = ('(file, protocol=None, fix_imports=True, ' 'buffer_callback=None)') @@ -4853,8 +4675,7 @@ class P4(P2, metaclass=MetaP): pass self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_signature_on_callable_objects(self): class Foo: def __call__(self, a): @@ -4910,15 +4731,13 @@ class C: with self.subTest('partial'): class C: - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) c = C() - with self.assertWarns(FutureWarning): - self.assertEqual(c(1), (2, 1)) - with self.assertWarns(FutureWarning): - self.assertEqual(self.signature(c), - ((('a', ..., ..., "positional_or_keyword"),), - ...)) + self.assertEqual(c(1), (2, c, 1)) + self.assertEqual(self.signature(C()), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) with self.subTest('partialmethod'): class C: @@ -5184,6 +5003,11 @@ def foo(a: list[str]) -> Tuple[str, float]: self.assertEqual(str(inspect.signature(foo)), inspect.signature(foo).format()) + def foo(x: undef): + pass + sig = inspect.signature(foo, annotation_format=Format.FORWARDREF) + self.assertEqual(str(sig), '(x: undef)') + def test_signature_str_positional_only(self): P = inspect.Parameter S = inspect.Signature @@ -5273,6 +5097,18 @@ def func( expected_multiline, ) + def test_signature_format_unquote(self): + def func(x: 'int') -> 'str': ... + + self.assertEqual( + inspect.signature(func).format(), + "(x: 'int') -> 'str'" + ) + self.assertEqual( + inspect.signature(func).format(quote_annotation_strings=False), + "(x: int) -> str" + ) + def test_signature_replace_parameters(self): def test(a, b) -> 42: pass @@ -5325,8 +5161,6 @@ def test(): sig = test.__signature__ = inspect.Signature(parameters=(spam_param,)) self.assertEqual(sig, inspect.signature(test)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_signature_on_mangled_parameters(self): class Spam: def foo(self, __p1:1=2, *, __p2:2=3): @@ -5358,9 +5192,9 @@ class foo: pass foo_sig = MySignature.from_callable(foo) self.assertIsInstance(foo_sig, MySignature) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - @unittest.skipIf(_pickle is None, "requires _pickle") def test_signature_from_callable_builtin_obj(self): class MySignature(inspect.Signature): pass sig = MySignature.from_callable(_pickle.Pickler) @@ -5508,6 +5342,60 @@ def test_signature_eval_str(self): par('b', PORK, annotation=tuple), ))) + def test_signature_annotation_format(self): + ida = inspect_deferred_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func=signature_func): + self.assertEqual( + signature_func(ida.f, annotation_format=Format.STRING), + sig([par("x", PORK, annotation="undefined")]) + ) + s1 = signature_func(ida.f, annotation_format=Format.FORWARDREF) + s2 = sig([par("x", PORK, annotation=EqualToForwardRef("undefined", owner=ida.f))]) + #breakpoint() + self.assertEqual( + signature_func(ida.f, annotation_format=Format.FORWARDREF), + sig([par("x", PORK, annotation=EqualToForwardRef("undefined", owner=ida.f))]) + ) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f, annotation_format=Format.VALUE) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f) + + def test_signature_deferred_annotations(self): + def f(x: undef): + pass + + class C: + x: undef + + def __init__(self, x: undef): + self.x = x + + sig = inspect.signature(f, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + sig = inspect.signature(C, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + + class CallableWrapper: + def __init__(self, func): + self.func = func + self.__annotate__ = func.__annotate__ + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __annotations__(self): + return self.__annotate__(Format.VALUE) + + cw = CallableWrapper(f) + sig = inspect.signature(cw, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['args', 'kwargs']) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -5533,38 +5421,6 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) - def test_signature_as_str(self): - self.maxDiff = None - class S: - __signature__ = '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - def test_signature_as_callable(self): - # __signature__ should be either a staticmethod or a bound classmethod - class S: - @classmethod - def __signature__(cls): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - class S: - @staticmethod - def __signature__(): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - def test_signature_on_derived_classes(self): # gh-105080: Make sure that signatures are consistent on derived classes @@ -5645,7 +5501,7 @@ def test_signature_parameter_object(self): with self.assertRaisesRegex(ValueError, 'cannot have default values'): p.replace(kind=inspect.Parameter.VAR_POSITIONAL) - self.assertTrue(repr(p).startswith('<Parameter')) + self.assertStartsWith(repr(p), '<Parameter') self.assertTrue('"a=42"' in repr(p)) def test_signature_parameter_hashable(self): @@ -6019,7 +5875,7 @@ def test_signature_bind_implicit_arg(self): # Issue #19611: getcallargs should work with comprehensions def make_set(): return set(z * z for z in range(5)) - gencomp_code = make_set.__code__.co_consts[1] + gencomp_code = make_set.__code__.co_consts[0] gencomp_func = types.FunctionType(gencomp_code, {}) iterator = iter(range(5)) @@ -6147,8 +6003,7 @@ def _strip_non_python_syntax(self, input, self.assertEqual(computed_clean_signature, clean_signature) self.assertEqual(computed_self_parameter, self_parameter) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + (module, /, path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True) def test_signature_strip_non_python_syntax(self): self._strip_non_python_syntax( "($module, /, path, mode, *, dir_fd=None, " + @@ -6287,7 +6142,6 @@ def test_builtins_have_signatures(self): 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, 'dict': {'pop'}, - 'int': {'__round__'}, 'memoryview': {'cast', 'hex'}, 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, } @@ -6351,14 +6205,14 @@ def test_errno_module_has_signatures(self): def test_faulthandler_module_has_signatures(self): import faulthandler - unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable'} + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable', 'dump_c_stack'} unsupported_signature |= {name for name in ['register'] if hasattr(faulthandler, name)} self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - no_signature = {'reduce'} - self._test_module_has_signatures(functools, no_signature) + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) def test_gc_module_has_signatures(self): import gc @@ -6392,7 +6246,7 @@ def test_operator_module_has_signatures(self): def test_os_module_has_signatures(self): unsupported_signature = {'chmod', 'utime'} unsupported_signature |= {name for name in - ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + ['get_terminal_size', 'link', 'posix_spawn', 'posix_spawnp', 'register_at_fork', 'startfile'] if hasattr(os, name)} self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) @@ -6453,8 +6307,7 @@ def test_tokenize_module_has_signatures(self): import tokenize self._test_module_has_signatures(tokenize) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'tracemalloc' def test_tracemalloc_module_has_signatures(self): import tracemalloc self._test_module_has_signatures(tracemalloc) @@ -6482,8 +6335,7 @@ def test_weakref_module_has_signatures(self): no_signature = {'ReferenceType', 'ref'} self._test_module_has_signatures(weakref, no_signature) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: <function TestSignatureDefinitions.test_python_function_override_signature.<locals>.func at 0xa4c07a580> builtin has invalid signature def test_python_function_override_signature(self): def func(*args, **kwargs): pass @@ -6514,9 +6366,8 @@ def func(*args, **kwargs): with self.assertRaises(ValueError): inspect.signature(func) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != '(raw, buffer_size=DEFAULT_BUFFER_SIZE)' @support.requires_docstrings - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_base_class_have_text_signature(self): # see issue 43118 from test.typinganndata.ann_module7 import BufferedReader @@ -6680,8 +6531,6 @@ def assertInspectEqual(self, path, source): inspected_src.splitlines(True) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getsource_reload(self): # see issue 1218234 with ready_to_import('reload_bug', self.src_before) as (name, path): @@ -6736,9 +6585,8 @@ def run_on_interactive_mode(self, source): raise ValueError("Process didn't exit properly.") return output + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'The source is: <<<def f():\n print(0)\n return 1 + 2\n>>>' not found in 'Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 1161, in getsource\n lines, lnum = getsourcelines(object)\n ~~~~~~~~~~~~~~^^^^^^^^\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 1143, in getsourcelines\n lines, lnum = findsource(object)\n ~~~~~~~~~~^^^^^^^^\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 978, in findsource\n raise OSError(\'could not get source code\')\nOSError: could not get source code\n' @unittest.skipIf(not has_subprocess_support, "test requires subprocess") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getsource(self): output = self.run_on_interactive_mode(textwrap.dedent("""\ def f(): @@ -6756,4 +6604,4 @@ def f(): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 1ab7a1fb6dd..f7b26e37e3a 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,5 +1,4 @@ import sys -import time import unittest # TODO: RUSTPYTHON @@ -18,6 +17,11 @@ except ImportError: _pylong = None +try: + import _decimal +except ImportError: + _decimal = None + L = [ ('0', 0), ('1', 1), @@ -242,8 +246,7 @@ def test_invalid_signs(self): with self.assertRaises(ValueError): int(' + 1 ') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode(self): self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) @@ -376,6 +379,7 @@ def test_int_memoryview(self): def test_string_float(self): self.assertRaises(ValueError, int, '1.2') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_intconversion(self): # Test __int__() class ClassicMissingMethods: @@ -405,68 +409,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError), \ - self.assertWarns(DeprecationWarning): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - with self.assertWarns(DeprecationWarning): - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError), \ - self.assertWarns(DeprecationWarning): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -517,18 +461,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -552,26 +484,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - with self.assertWarns(DeprecationWarning): - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - with self.assertWarns(DeprecationWarning): - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, @@ -612,6 +524,13 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) class IntStrDigitLimitsTests(unittest.TestCase): @@ -654,8 +573,7 @@ def check(self, i, base=None): else: self.int_class(i, base) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_max_str_digits(self): maxdigits = sys.get_int_max_str_digits() @@ -670,8 +588,7 @@ def test_max_str_digits(self): with self.assertRaises(ValueError): str(i) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() @@ -681,7 +598,7 @@ def test_denial_of_service_prevented_int_to_str(self): digits = 78_268 with ( support.adjust_int_max_str_digits(digits), - support.CPUStopwatch() as sw_convert): + support.Stopwatch() as sw_convert): huge_decimal = str(huge_int) self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. @@ -696,7 +613,7 @@ def test_denial_of_service_prevented_int_to_str(self): with support.adjust_int_max_str_digits(int(.995 * digits)): with ( self.assertRaises(ValueError) as err, - support.CPUStopwatch() as sw_fail_huge): + support.Stopwatch() as sw_fail_huge): str(huge_int) self.assertIn('conversion', str(err.exception)) self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) @@ -706,13 +623,13 @@ def test_denial_of_service_prevented_int_to_str(self): extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. with ( self.assertRaises(ValueError) as err, - support.CPUStopwatch() as sw_fail_extra_huge): + support.Stopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) self.assertIn('conversion', str(err.exception)) self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) - @unittest.skip('TODO: RUSTPYTHON; flaky test') + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() @@ -722,7 +639,7 @@ def test_denial_of_service_prevented_str_to_int(self): huge = '8'*digits with ( support.adjust_int_max_str_digits(digits), - support.CPUStopwatch() as sw_convert): + support.Stopwatch() as sw_convert): int(huge) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. @@ -734,7 +651,7 @@ def test_denial_of_service_prevented_str_to_int(self): with support.adjust_int_max_str_digits(digits - 1): with ( self.assertRaises(ValueError) as err, - support.CPUStopwatch() as sw_fail_huge): + support.Stopwatch() as sw_fail_huge): int(huge) self.assertIn('conversion', str(err.exception)) self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) @@ -744,7 +661,7 @@ def test_denial_of_service_prevented_str_to_int(self): extra_huge = '7'*1_200_000 with ( self.assertRaises(ValueError) as err, - support.CPUStopwatch() as sw_fail_extra_huge): + support.Stopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) self.assertIn('conversion', str(err.exception)) @@ -796,8 +713,7 @@ def _other_base_helper(self, base): with self.assertRaises(ValueError) as err: int_class(f'{s}1', base) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_int_from_other_bases(self): base = 3 with self.subTest(base=base): @@ -872,6 +788,18 @@ def test_pylong_int_divmod(self): a, b = divmod(n*3 + 1, n) assert a == 3 and b == 1 + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_int_divmod_crash(self): + # Regression test for https://github.com/python/cpython/issues/142554. + bad_int_divmod = lambda a, b: (1,) + # 'k' chosen such that divmod(2**(2*k), 2**k) uses _pylong.int_divmod() + k = 10_000 + a, b = (1 << (2 * k)), (1 << k) + with mock.patch.object(_pylong, "int_divmod", wraps=bad_int_divmod): + msg = r"tuple of length 2 is required from int_divmod\(\)" + self.assertRaisesRegex(ValueError, msg, divmod, a, b) + def test_pylong_str_to_int(self): v1 = 1 << 100_000 s = str(v1) @@ -890,7 +818,7 @@ def test_pylong_str_to_int(self): @support.cpython_only # tests implementation details of CPython. @unittest.skipUnless(_pylong, "_pylong module required") - # @mock.patch.object(_pylong, "int_to_decimal_string") # NOTE(RUSTPYTHON): See comment at top of file + #@mock.patch.object(_pylong, "int_to_decimal_string") # NOTE(RUSTPYTHON): See comment at top of file def test_pylong_misbehavior_error_path_to_str( self, mock_int_to_str): with support.adjust_int_max_str_digits(20_000): @@ -906,7 +834,7 @@ def test_pylong_misbehavior_error_path_to_str( @support.cpython_only # tests implementation details of CPython. @unittest.skipUnless(_pylong, "_pylong module required") - # @mock.patch.object(_pylong, "int_from_string") # NOTE(RUSTPYTHON): See comment at top of file + #@mock.patch.object(_pylong, "int_from_string") # NOTE(RUSTPYTHON): See comment at top of file def test_pylong_misbehavior_error_path_from_str( self, mock_int_from_str): big_value = '7'*19_999 @@ -930,9 +858,91 @@ def test_pylong_roundtrip(self): n = hibit | getrandbits(bits - 1) assert n.bit_length() == bits sn = str(n) - self.assertFalse(sn.startswith('0')) + self.assertNotStartsWith(sn, '0') self.assertEqual(n, int(sn)) bits <<= 1 + @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 0142208427b..e747b9dd03f 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -445,9 +445,25 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) self.assertRaises(exc, fp.seek, -1, self.SEEK_END) - @unittest.skipIf( - support.is_emscripten, "fstat() of a pipe fd is not supported" - ) + @support.cpython_only + def test_startup_optimization(self): + # gh-132952: Test that `io` is not imported at startup and that the + # __module__ of UnsupportedOperation is set to "io". + assert_python_ok("-S", "-c", textwrap.dedent( + """ + import sys + assert "io" not in sys.modules + try: + sys.stdin.truncate() + except Exception as e: + typ = type(e) + assert typ.__module__ == "io", (typ, typ.__module__) + assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) + else: + raise AssertionError("Expected UnsupportedOperation") + """ + )) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_optional_abilities(self): # Test for OSError when optional APIs are not supported @@ -501,57 +517,65 @@ class UnseekableWriter(self.MockUnseekableIO): (text_reader, "r"), (text_writer, "w"), (self.BytesIO, "rws"), (self.StringIO, "rws"), ) - for [test, abilities] in tests: - with self.subTest(test), test() as obj: - readable = "r" in abilities - self.assertEqual(obj.readable(), readable) - writable = "w" in abilities - self.assertEqual(obj.writable(), writable) - - if isinstance(obj, self.TextIOBase): - data = "3" - elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): - data = b"3" - else: - self.fail("Unknown base class") - if "f" in abilities: - obj.fileno() - else: - self.assertRaises(OSError, obj.fileno) + def do_test(test, obj, abilities): + readable = "r" in abilities + self.assertEqual(obj.readable(), readable) + writable = "w" in abilities + self.assertEqual(obj.writable(), writable) - if readable: - obj.read(1) - obj.read() - else: - self.assertRaises(OSError, obj.read, 1) - self.assertRaises(OSError, obj.read) + if isinstance(obj, self.TextIOBase): + data = "3" + elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): + data = b"3" + else: + self.fail("Unknown base class") - if writable: - obj.write(data) - else: - self.assertRaises(OSError, obj.write, data) - - if sys.platform.startswith("win") and test in ( - pipe_reader, pipe_writer): - # Pipes seem to appear as seekable on Windows - continue - seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) - - if seekable: - obj.tell() - obj.seek(0) - else: - self.assertRaises(OSError, obj.tell) - self.assertRaises(OSError, obj.seek, 0) + if "f" in abilities: + obj.fileno() + else: + self.assertRaises(OSError, obj.fileno) + + if readable: + obj.read(1) + obj.read() + else: + self.assertRaises(OSError, obj.read, 1) + self.assertRaises(OSError, obj.read) + + if writable: + obj.write(data) + else: + self.assertRaises(OSError, obj.write, data) + + if sys.platform.startswith("win") and test in ( + pipe_reader, pipe_writer): + # Pipes seem to appear as seekable on Windows + return + seekable = "s" in abilities + self.assertEqual(obj.seekable(), seekable) + + if seekable: + obj.tell() + obj.seek(0) + else: + self.assertRaises(OSError, obj.tell) + self.assertRaises(OSError, obj.seek, 0) + + if writable and seekable: + obj.truncate() + obj.truncate(0) + else: + self.assertRaises(OSError, obj.truncate) + self.assertRaises(OSError, obj.truncate, 0) + + for [test, abilities] in tests: + with self.subTest(test): + if test == pipe_writer and not threading_helper.can_start_thread: + self.skipTest("Need threads") + with test() as obj: + do_test(test, obj, abilities) - if writable and seekable: - obj.truncate() - obj.truncate(0) - else: - self.assertRaises(OSError, obj.truncate) - self.assertRaises(OSError, obj.truncate, 0) def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' @@ -780,8 +804,8 @@ def test_closefd_attr(self): file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) self.assertEqual(file.buffer.raw.closefd, False) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. @@ -871,6 +895,22 @@ def test_RawIOBase_read(self): self.assertEqual(rawio.read(2), None) self.assertEqual(rawio.read(2), b"") + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + def test_types_have_dict(self): test = ( self.IOBase(), @@ -880,7 +920,7 @@ def test_types_have_dict(self): self.BytesIO() ) for obj in test: - self.assertTrue(hasattr(obj, "__dict__")) + self.assertHasAttr(obj, "__dict__") def test_opener(self): with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: @@ -1074,7 +1114,7 @@ def reader(file, barrier): class CIOTest(IOTest): - @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_IOBase_finalize(self): # Issue #12149: segmentation fault on _PyIOBase_finalize when both a # class which inherits IOBase and an object of this class are caught @@ -1093,10 +1133,6 @@ def close(self): support.gc_collect() self.assertIsNone(wr(), wr) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning - def test_destructor(self): - return super().test_destructor() - @support.cpython_only class TestIOCTypes(unittest.TestCase): def setUp(self): @@ -1131,7 +1167,7 @@ def test_class_hierarchy(self): def check_subs(types, base): for tp in types: with self.subTest(tp=tp, base=base): - self.assertTrue(issubclass(tp, base)) + self.assertIsSubclass(tp, base) def recursive_check(d): for k, v in d.items(): @@ -1204,14 +1240,6 @@ def test_stringio_setstate(self): class PyIOTest(IOTest): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor - def test_bad_opener_negative_1(): - return super().test_bad_opener_negative_1() - - @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor - def test_bad_opener_other_negative(): - return super().test_bad_opener_other_negative() - @support.cpython_only class APIMismatchTest(unittest.TestCase): @@ -1288,7 +1316,6 @@ def _with(): # a ValueError. self.assertRaises(ValueError, _with) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -1796,8 +1823,8 @@ def test_misbehaved_io_read(self): # checking this is not so easy. self.assertRaises(OSError, bufio.read, 10) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead @@ -1824,7 +1851,6 @@ def test_bad_readinto_value(self): bufio.readline() self.assertIsNone(cm.exception.__cause__) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'bytes' object cannot be interpreted as an integer") def test_bad_readinto_type(self): rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: b'' @@ -1833,26 +1859,6 @@ def test_bad_readinto_type(self): bufio.readline() self.assertIsInstance(cm.exception.__cause__, TypeError) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_seek_character_device_file(self): - return super().test_seek_character_device_file() - - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UnsupportedOperation not raised by truncate - def test_truncate_on_read_only(self): - return super().test_truncate_on_read_only() - - @unittest.skip('TODO: RUSTPYTHON; fallible allocation') - def test_constructor(self): - return super().test_constructor() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_pickling_subclass(self): - return super().test_pickling_subclass() - class PyBufferedReaderTest(BufferedReaderTest): tp = pyio.BufferedReader @@ -1916,7 +1922,7 @@ def test_write_overflow(self): flushed = b"".join(writer._write_stack) # At least (total - 8) bytes were implicitly flushed, perhaps more # depending on the implementation. - self.assertTrue(flushed.startswith(contents[:-8]), flushed) + self.assertStartsWith(flushed, contents[:-8]) def check_writes(self, intermediate_func): # Lots of writes, test the flushed output is as expected. @@ -1961,7 +1967,6 @@ def _seekrel(bufio): def test_writes_and_truncates(self): self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_non_blocking(self): raw = self.MockNonBlockWriterIO() bufio = self.tp(raw, 8) @@ -1987,7 +1992,7 @@ def test_write_non_blocking(self): self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) s = raw.pop_written() # Previously buffered bytes were flushed - self.assertTrue(s.startswith(b"01234567A"), s) + self.assertStartsWith(s, b"01234567A") def test_write_and_rewind(self): raw = self.BytesIO() @@ -2168,8 +2173,8 @@ def test_initialization(self): self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) self.assertRaises(ValueError, bufio.write, b"def") - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C BufferedWriter objects are collected, and collecting them flushes # all data to disk. @@ -2192,17 +2197,6 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedWriter"): self.tp(self.BytesIO(), 1024, 1024, 1024) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() - - @unittest.skip('TODO: RUSTPYTHON; fallible allocation') - def test_constructor(self): - return super().test_constructor() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_pickling_subclass(self): - return super().test_pickling_subclass() class PyBufferedWriterTest(BufferedWriterTest): tp = pyio.BufferedWriter @@ -2296,7 +2290,7 @@ def test_write(self): def test_peek(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - self.assertTrue(pair.peek(3).startswith(b"abc")) + self.assertStartsWith(pair.peek(3), b"abc") self.assertEqual(pair.read(3), b"abc") def test_readable(self): @@ -2680,8 +2674,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2691,26 +2685,6 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedRandom"): self.tp(self.BytesIO(), 1024, 1024, 1024) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_seek_character_device_file(self): - return super().test_seek_character_device_file() - - @unittest.expectedFailure # TODO: RUSTPYTHON; f.read1(1) returns b'a' - def test_read1_after_write(self): - return super().test_read1_after_write() - - @unittest.skip('TODO: RUSTPYTHON; fallible allocation') - def test_constructor(self): - return super().test_constructor() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_pickling_subclass(self): - return super().test_pickling_subclass() - class PyBufferedRandomTest(BufferedRandomTest): tp = pyio.BufferedRandom @@ -3020,14 +2994,11 @@ def test_reconfigure_line_buffering(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_default_encoding(self): - old_environ = dict(os.environ) - try: + with os_helper.EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that TextIOWrapper() uses the current # locale encoding and not the user preferred encoding - for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') current_locale_encoding = locale.getencoding() b = self.BytesIO() @@ -3035,9 +3006,6 @@ def test_default_encoding(self): warnings.simplefilter("ignore", EncodingWarning) t = self.TextIOWrapper(b) self.assertEqual(t.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) def test_encoding(self): # Check the encoding attribute is always set, and valid @@ -3205,7 +3173,6 @@ def flush(self): support.gc_collect() self.assertEqual(record, [1, 2, 3]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -3364,7 +3331,7 @@ def test_seek_and_tell_with_data(data, min_pos=0): finally: StatefulIncrementalDecoder.codecEnabled = 0 - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp def test_multibyte_seek_and_tell(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") f.write("AB\n\u3046\u3048\n") @@ -3380,7 +3347,24 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() - @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -3394,7 +3378,6 @@ def test_seek_with_encoder_state(self): self.assertEqual(f.readline(), "\u00e6\u0300\u0300") f.close() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoded_writes(self): data = "1234567890" tests = ("utf-16", @@ -3533,7 +3516,6 @@ def test_issue2282(self): self.assertEqual(buffer.seekable(), txt.seekable()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_append_bom(self): # The BOM is not written again when appending to a non-empty file filename = os_helper.TESTFN @@ -3549,7 +3531,6 @@ def test_append_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_bom(self): # Same test, but when seeking manually filename = os_helper.TESTFN @@ -3565,7 +3546,6 @@ def test_seek_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_append_bom(self): # Same test, but first seek to the start and then to the end filename = os_helper.TESTFN @@ -3833,7 +3813,7 @@ def __del__(self): """.format(iomod=iomod, kwargs=kwargs) return assert_python_ok("-c", code) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError during module teardown in __del__ def test_create_at_shutdown_without_encoding(self): rc, out, err = self._check_create_at_shutdown() if err: @@ -3843,7 +3823,7 @@ def test_create_at_shutdown_without_encoding(self): else: self.assertEqual("ok", out.decode().strip()) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError during module teardown in __del__ def test_create_at_shutdown_with_encoding(self): rc, out, err = self._check_create_at_shutdown(encoding='utf-8', errors='strict') @@ -4091,6 +4071,22 @@ def __setstate__(slf, state): self.assertEqual(newtxt.tag, 'ham') del MyTextIO + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -4115,7 +4111,11 @@ class CTextIOWrapperTest(TextIOWrapperTest): io = io shutdown_error = "LookupError: unknown encoding: ascii" - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.expectedFailure + def test_read_non_blocking(self): + return super().test_read_non_blocking() + def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) @@ -4126,8 +4126,8 @@ def test_initialization(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) self.assertRaises(Exception, repr, t) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. @@ -4191,7 +4191,6 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue119506(self): chunk_size = 8192 @@ -4214,92 +4213,49 @@ def write(self, data): self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], buf._write_stack) - def test_basic_io(self): - return super().test_basic_io() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_constructor(self): - return super().test_constructor() - - def test_detach(self): - return super().test_detach() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_newlines(self): - return super().test_newlines() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_newlines_input(self): - return super().test_newlines_input() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_non_text_encoding_codecs_are_rejected(self): - return super().test_non_text_encoding_codecs_are_rejected() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_defaults(self): - return super().test_reconfigure_defaults() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_encoding_read(self): - return super().test_reconfigure_encoding_read() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_errors(self): - return super().test_reconfigure_errors() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_line_buffering(self): - return super().test_reconfigure_line_buffering() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_locale(self): - return super().test_reconfigure_locale() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_newline(self): - return super().test_reconfigure_newline() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_write(self): - return super().test_reconfigure_write() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_write_fromascii(self): - return super().test_reconfigure_write_fromascii() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_reconfigure_write_through(self): - return super().test_reconfigure_write_through() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_repr(self): - return super().test_repr() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'closed' + def test_issue142594(self): + wrapper = None + detached = False + class ReentrantRawIO(self.RawIOBase): + @property + def closed(self): + nonlocal detached + if wrapper is not None and not detached: + detached = True + wrapper.detach() + return False - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_uninitialized(self): - return super().test_uninitialized() + raw = ReentrantRawIO() + wrapper = self.TextIOWrapper(raw) + wrapper.close() # should not crash - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_recursive_repr(self): - return super().test_recursive_repr() + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004 + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_pickling_subclass(self): - return super().test_pickling_subclass() + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'NoneType' + def test_read_non_blocking(self): + return super().test_read_non_blocking() class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio shutdown_error = "LookupError: unknown encoding: ascii" - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised - def test_constructor(self): - return super().test_constructor() + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.set_blocking not available on Windows") + def test_read_non_blocking(self): + return super().test_read_non_blocking() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_newlines(self): - return super().test_newlines() + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004 + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() + + if sys.platform == "win32": + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.expectedFailure + def test_read_non_blocking(self): + return super().test_read_non_blocking() class IncrementalNewlineDecoderTest(unittest.TestCase): @@ -4379,7 +4335,6 @@ def _decode_bytewise(s): self.assertEqual(decoder.decode(input), "abc") self.assertEqual(decoder.newlines, None) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_newline_decoder(self): encodings = ( # None meaning the IncrementalNewlineDecoder takes unicode input @@ -4477,9 +4432,6 @@ def test_removed_u_mode(self): self.open(os_helper.TESTFN, mode) self.assertIn('invalid mode', str(cm.exception)) - @unittest.skipIf( - support.is_emscripten, "fstat() of a pipe fd is not supported" - ) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_open_pipe_with_append(self): # bpo-27805: Ignore ESPIPE from lseek() in open(). @@ -4541,7 +4493,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str): @@ -4596,7 +4548,6 @@ def _check_warn_on_dealloc(self, *args, **kwargs): support.gc_collect() self.assertIn(r, str(cm.warning.args[0])) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_warn_on_dealloc(self): self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) self._check_warn_on_dealloc(os_helper.TESTFN, "wb") @@ -4621,7 +4572,6 @@ def cleanup_fds(): with warnings_helper.check_no_resource_warning(self): self.open(r, *args, closefd=False, **kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): self._check_warn_on_dealloc_fd("rb", buffering=0) @@ -4651,17 +4601,11 @@ def test_pickling(self): with self.assertRaisesRegex(TypeError, msg): pickle.dumps(f, protocol) - @unittest.expectedFailure # TODO: RUSTPYTHON - @unittest.skipIf( - support.is_emscripten, "fstat() of a pipe fd is not supported" - ) + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") def test_nonblock_pipe_write_bigbuf(self): self._test_nonblock_pipe_write(16*1024) - @unittest.expectedFailure # TODO: RUSTPYTHON - @unittest.skipIf( - support.is_emscripten, "fstat() of a pipe fd is not supported" - ) + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") def test_nonblock_pipe_write_smallbuf(self): self._test_nonblock_pipe_write(1024) @@ -4780,7 +4724,6 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_check_encoding_warning(self): # PEP 597: Raise warning when encoding is not specified # and sys.flags.warn_default_encoding is set. @@ -4799,12 +4742,9 @@ def test_check_encoding_warning(self): proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() self.assertEqual(len(warnings), 2) - self.assertTrue( - warnings[0].startswith(b"<string>:5: EncodingWarning: ")) - self.assertTrue( - warnings[1].startswith(b"<string>:8: EncodingWarning: ")) + self.assertStartsWith(warnings[0], b"<string>:5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b"<string>:8: EncodingWarning: ") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" # based on sys.flags.utf8_mode @@ -4872,20 +4812,18 @@ def run(): else: self.assertFalse(err.strip('.!')) + @unittest.expectedFailure # TODO: RUSTPYTHON; without GC+GIL, finalize_modules clears __main__ globals while daemon threads are still running @threading_helper.requires_working_threading() @support.requires_resource('walltime') def test_daemon_threads_shutdown_stdout_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stdout') + @unittest.expectedFailure # TODO: RUSTPYTHON; without GC+GIL, finalize_modules clears __main__ globals while daemon threads are still running @threading_helper.requires_working_threading() @support.requires_resource('walltime') def test_daemon_threads_shutdown_stderr_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stderr') - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') - def test_check_encoding_errors(self): - return super().test_check_encoding_errors() - class PyMiscIOTest(MiscIOTest): io = pyio @@ -5020,7 +4958,7 @@ def on_alarm(*args): os.read(r, len(data) * 100) exc = cm.exception if isinstance(exc, RuntimeError): - self.assertTrue(str(exc).startswith("reentrant call"), str(exc)) + self.assertStartsWith(str(exc), "reentrant call") finally: signal.alarm(0) wio.close() @@ -5058,14 +4996,12 @@ def alarm_handler(sig, frame): os.close(w) os.close(r) - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_buffered(self): self.check_interrupted_read_retry(lambda x: x.decode('latin1'), mode="rb") - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_text(self): @@ -5140,13 +5076,13 @@ def alarm2(sig, frame): if e.errno != errno.EBADF: raise - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_buffered(self): self.check_interrupted_write_retry(b"x", mode="wb") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_text(self): @@ -5156,6 +5092,10 @@ def test_interrupted_write_retry_text(self): class CSignalsTest(SignalsTest): io = io + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") + def test_interrupted_read_retry_buffered(self): + return super().test_interrupted_read_retry_buffered() + class PySignalsTest(SignalsTest): io = pyio @@ -5165,6 +5105,24 @@ class PySignalsTest(SignalsTest): test_reentrant_write_text = None +class ProtocolsTest(unittest.TestCase): + class MyReader: + def read(self, sz=-1): + return b"" + + class MyWriter: + def write(self, b: bytes): + pass + + def test_reader_subclass(self): + self.assertIsSubclass(self.MyReader, io.Reader) + self.assertNotIsSubclass(str, io.Reader) + + def test_writer_subclass(self): + self.assertIsSubclass(self.MyWriter, io.Writer) + self.assertNotIsSubclass(str, io.Writer) + + def load_tests(loader, tests, pattern): tests = (CIOTest, PyIOTest, APIMismatchTest, CBufferedReaderTest, PyBufferedReaderTest, @@ -5176,6 +5134,7 @@ def load_tests(loader, tests, pattern): CTextIOWrapperTest, PyTextIOWrapperTest, CMiscIOTest, PyMiscIOTest, CSignalsTest, PySignalsTest, TestIOCTypes, + ProtocolsTest, ) # Put the namespaces of the IO module we are testing and some useful mock diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index e69e12495ad..8af91e857d8 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -1472,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1494,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1503,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1512,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1520,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -2214,12 +2248,18 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('ffff::c0a8:ffff%scope')) def testIPVersion(self): + self.assertEqual(ipaddress.IPv4Address.version, 4) + self.assertEqual(ipaddress.IPv6Address.version, 6) + self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) self.assertEqual(self.ipv6_scoped_address.version, 6) self.assertEqual(self.ipv6_with_ipv4_part.version, 6) def testMaxPrefixLength(self): + self.assertEqual(ipaddress.IPv4Address.max_prefixlen, 32) + self.assertEqual(ipaddress.IPv6Address.max_prefixlen, 128) + self.assertEqual(self.ipv4_interface.max_prefixlen, 32) self.assertEqual(self.ipv6_interface.max_prefixlen, 128) self.assertEqual(self.ipv6_scoped_interface.max_prefixlen, 128) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 42b94a55c1d..9c26eb08583 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -4,11 +4,14 @@ import unittest from test.support import cpython_only from test.support.os_helper import TESTFN, unlink -# XXX: RUSTPYTHON -# from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ +from test.support import BrokenIter import pickle import collections.abc +import functools +import contextlib +import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -83,6 +86,22 @@ class BadIterableClass: def __iter__(self): raise ZeroDivisionError +class CallableIterClass: + def __init__(self): + self.i = 0 + def __call__(self): + i = self.i + self.i = i + 1 + if i > 100: + raise IndexError # Emergency stop + return i + +class EmptyIterClass: + def __len__(self): + return 0 + def __getitem__(self, i): + raise StopIteration + # Main test suite class TestCase(unittest.TestCase): @@ -230,6 +249,78 @@ def test_mutating_seq_class_exhausted_iter(self): self.assertEqual(list(empit), [5, 6]) self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6]) + def test_reduce_mutating_builtins_iter(self): + # This is a reproducer of issue #101765 + # where iter `__reduce__` calls could lead to a segfault or SystemError + # depending on the order of C argument evaluation, which is undefined + + # Backup builtins + builtins_dict = builtins.__dict__ + orig = {"iter": iter, "reversed": reversed} + + def run(builtin_name, item, sentinel=None): + it = iter(item) if sentinel is None else iter(item, sentinel) + + class CustomStr: + def __init__(self, name, iterator): + self.name = name + self.iterator = iterator + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + # Here we exhaust our iterator, possibly changing + # its `it_seq` pointer to NULL + # The `__reduce__` call should correctly get + # the pointers after this call + list(self.iterator) + return other == self.name + + # del is required here + # to not prematurely call __eq__ from + # the hash collision with the old key + del builtins_dict[builtin_name] + builtins_dict[CustomStr(builtin_name, it)] = orig[builtin_name] + + return it.__reduce__() + + types = [ + (EmptyIterClass(),), + (bytes(8),), + (bytearray(8),), + ((1, 2, 3),), + (lambda: 0, 0), + (tuple[int],) # GenericAlias + ] + + try: + run_iter = functools.partial(run, "iter") + # The returned value of `__reduce__` should not only be valid + # but also *empty*, as `it` was exhausted during `__eq__` + # i.e "xyz" returns (iter, ("",)) + self.assertEqual(run_iter("xyz"), (orig["iter"], ("",))) + self.assertEqual(run_iter([1, 2, 3]), (orig["iter"], ([],))) + + # _PyEval_GetBuiltin is also called for `reversed` in a branch of + # listiter_reduce_general + self.assertEqual( + run("reversed", orig["reversed"](list(range(8)))), + (reversed, ([],)) + ) + + for case in types: + self.assertEqual(run_iter(*case), (orig["iter"], ((),))) + finally: + # Restore original builtins + for key, func in orig.items(): + # need to suppress KeyErrors in case + # a failed test deletes the key without setting anything + with contextlib.suppress(KeyError): + # del is required here + # to not invoke our custom __eq__ from + # the hash collision with the old key + del builtins_dict[key] + builtins_dict[key] = func + # Test a new_style class with __iter__ but no next() method def test_new_style_iter_class(self): class IterClass(object): @@ -239,16 +330,7 @@ def __iter__(self): # Test two-argument iter() with callable instance def test_iter_callable(self): - class C: - def __init__(self): - self.i = 0 - def __call__(self): - i = self.i - self.i = i + 1 - if i > 100: - raise IndexError # Emergency stop - return i - self.check_iterator(iter(C(), 10), list(range(10)), pickle=False) + self.check_iterator(iter(CallableIterClass(), 10), list(range(10)), pickle=True) # Test two-argument iter() with function def test_iter_function(self): @@ -268,6 +350,31 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + def test_iter_function_concealing_reentrant_exhaustion(self): + # gh-101892: Test two-argument iter() with a function that + # exhausts its associated iterator but forgets to either return + # a sentinel value or raise StopIteration. + HAS_MORE = 1 + NO_MORE = 2 + + def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + + def spam(): + # Touching the iterator with exhaust() below will call + # spam() once again so protect against recursion. + if spam.is_recursive_call: + return NO_MORE + spam.is_recursive_call = True + exhaust(spam.iterator) + return HAS_MORE + + spam.is_recursive_call = False + spam.iterator = iter(spam, NO_MORE) + with self.assertRaises(StopIteration): + next(spam.iterator) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): @@ -1030,8 +1137,7 @@ def test_iter_neg_setstate(self): self.assertEqual(next(it), 0) self.assertEqual(next(it), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_free_after_iterating(self): check_free_after_iterating(self, iter, SequenceClass, (0,)) @@ -1040,6 +1146,46 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should be the iterator expression + + def init_raises(): + try: + for x in BrokenIter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in BrokenIter(next_raises=True): + pass + except Exception as e: + return e + + def iter_raises(): + try: + for x in BrokenIter(iter_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 03dadb71f78..44addde1948 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -16,27 +16,6 @@ import struct import threading import gc -import warnings - -def pickle_deprecated(testfunc): - """ Run the test three times. - First, verify that a Deprecation Warning is raised. - Second, run normally but with DeprecationWarnings temporarily disabled. - Third, run with warnings promoted to errors. - """ - def inner(self): - with self.assertWarns(DeprecationWarning): - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - testfunc(self) - # XXX: RUSTPYTHON; Patch to make tests pass. It will be removed once 3.14 is released anyway. - # with warnings.catch_warnings(): - # warnings.simplefilter("error", category=DeprecationWarning) - # with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - # testfunc(self) - - return inner maxsize = support.MAX_Py_ssize_t minsize = -maxsize-1 @@ -146,8 +125,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @unittest.expectedFailure # TODO: RUSTPYTHON; [7, 7, 8, 10] != <itertools.accumulate object at 0xb4000073f32b4480> - @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -174,9 +151,6 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) @@ -248,60 +222,12 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @pickle_deprecated - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -355,9 +281,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -369,8 +292,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -378,15 +299,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -444,23 +356,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -521,9 +428,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -567,7 +471,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -584,24 +487,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -650,19 +535,10 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @unittest.expectedFailure # TODO: RUSTPYTHON; 'count(10.5)' != 'count(10.5, 1.0)' - @pickle_deprecated + @unittest.expectedFailure # TODO: RUSTPYTHON; 'count(10.5)' != 'count(10.5, 1.0)' def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), @@ -712,17 +588,6 @@ def test_count_with_step(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) c = count(maxsize -2, 2) self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') @@ -767,117 +632,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -896,15 +650,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -915,18 +660,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -946,13 +679,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -1035,7 +761,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -1046,8 +771,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -1066,33 +789,12 @@ def test_zip(self): lzip('abc', 'def')) @support.impl_detail("tuple reuse is specific to CPython") - @pickle_deprecated def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1142,14 +844,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @pickle_deprecated - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1365,35 +1059,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @pickle_deprecated - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - - @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1412,22 +1077,13 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated + @unittest.expectedFailure # TODO: RUSTPYTHON def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1445,20 +1101,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - - @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1473,21 +1115,7 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated + @unittest.expectedFailure # TODO: RUSTPYTHON def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1544,21 +1172,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1582,7 +1195,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1596,15 +1208,7 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1615,15 +1219,7 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tee(self): n = 200 @@ -1704,7 +1300,7 @@ def test_tee(self): t3 = tnew(t1) self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) - # test that tee objects are weak referencable + # test that tee objects are weak referenceable a, b = tee(range(10)) p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) @@ -1739,41 +1335,6 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) - def test_tee_dealloc_segfault(self): # gh-115874: segfaults when accessing module state in tp_dealloc. script = ( @@ -1807,7 +1368,6 @@ def __next__(self): with self.assertRaisesRegex(RuntimeError, "tee"): next(a) - @unittest.skip("TODO: RUSTPYTHON; , hangs") @threading_helper.requires_working_threading() def test_tee_concurrent(self): start = threading.Event() @@ -1942,34 +1502,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @pickle_deprecated - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @pickle_deprecated - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') @@ -2077,27 +1609,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -2769,7 +2443,7 @@ def __eq__(self, other): class SubclassWithKwargsTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): # count is not subclassable... testcases = [ @@ -2797,10 +2471,10 @@ class subclass(cls): subclass(*args, newarg=3) for cls, args, result in testcases: - # Constructors of repeat, zip, compress accept keyword arguments. + # Constructors of repeat, zip, map, compress accept keyword arguments. # Their subclasses need overriding __new__ to support new # keyword arguments. - if cls in [repeat, zip, compress]: + if cls in [repeat, zip, map, compress]: continue with self.subTest(cls): class subclass_with_init(cls): diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index b919af2328f..41c06beaa38 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -41,8 +41,7 @@ def test_pyjson(self): 'json.encoder') class TestCTest(CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cjson(self): self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') self.assertEqual(self.json.decoder.scanstring.__module__, '_json') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index f07f7d55339..7b3b30ce449 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -4,7 +4,7 @@ from test.test_json import PyTest, CTest from test import support -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class TestDecode: @@ -18,7 +18,6 @@ def test_float(self): self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }") def test_nonascii_digits_rejected(self): # JSON specifies only ascii digits, see gh-125687 @@ -136,14 +135,7 @@ def test_limit_int(self): class TestPyDecode(TestDecode, PyTest): pass - class TestCDecode(TestDecode, CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_keys_reuse(self): - return super().test_keys_reuse() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_limit_int(self): return super().test_limit_int() diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 3ce16684a08..4d569dadfa4 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,6 +1,8 @@ import collections from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestDefault: def test_default(self): @@ -8,6 +10,25 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bad_default(self): + def default(obj): + if obj is NotImplemented: + raise ValueError + if obj is ...: + return NotImplemented + if obj is type: + return collections + return [...] + + with self.assertRaises(ValueError) as cm: + self.dumps(type, default=default) + self.assertEqual(cm.exception.__notes__, + ['when serializing ellipsis object', + 'when serializing list item 0', + 'when serializing module object', + 'when serializing type object']) + def test_ordereddict(self): od = collections.OrderedDict(a=1, b=2, c=3, d=4) od.move_to_end('b') diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 6a39b72a09d..c90d3e968e5 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 7a85665c816..4adfcb17c4d 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,6 +1,6 @@ from test.test_json import PyTest, CTest -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests # 2007-10-05 JSONDOCS = [ @@ -102,8 +102,27 @@ def test_non_string_keys_dict(self): def test_not_serializable(self): import sys with self.assertRaisesRegex(TypeError, - 'Object of type module is not JSON serializable'): + 'Object of type module is not JSON serializable') as cm: self.dumps(sys) + self.assertNotHasAttr(cm.exception, '__notes__') + + with self.assertRaises(TypeError) as cm: + self.dumps([1, [2, 3, sys]]) + self.assertEqual(cm.exception.__notes__, + ['when serializing list item 2', + 'when serializing list item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps((1, (2, 3, sys))) + self.assertEqual(cm.exception.__notes__, + ['when serializing tuple item 2', + 'when serializing tuple item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps({'a': {'b': sys}}) + self.assertEqual(cm.exception.__notes__, + ["when serializing dict item 'b'", + "when serializing dict item 'a'"]) def test_truncated_input(self): test_cases = [ @@ -220,9 +239,7 @@ def test_linecol(self): (line, col, idx)) class TestPyFail(TestFail, PyTest): pass - class TestCFail(TestFail, CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_failures(self): return super().test_failures() diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 59f6f2c4b19..2fa8d3f528c 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,7 +1,7 @@ from test import support from test.test_json import PyTest, CTest -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class JSONTestObject: @@ -14,8 +14,8 @@ def test_listrecursion(self): x.append(x) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]) else: self.fail("didn't raise ValueError on list recursion") x = [] @@ -23,8 +23,8 @@ def test_listrecursion(self): x.append(y) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2) else: self.fail("didn't raise ValueError on alternating list recursion") y = [] @@ -37,8 +37,8 @@ def test_dictrecursion(self): x["test"] = x try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"]) else: self.fail("didn't raise ValueError on dict recursion") x = {} @@ -62,31 +62,40 @@ def default(self, o): enc.recurse = True try: enc.encode(JSONTestObject) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, + ["when serializing list item 0", + "when serializing type object"]) else: self.fail("didn't raise ValueError on default recursion") - # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; crashes") + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): + very_deep = 500_000 # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('{"a":' * 100000 + '1' + '}' * 100000) + self.loads('{"a":' * very_deep + '1' + '}' * very_deep) with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + self.loads('{"a":' * very_deep + '[1]' + '}' * very_deep) with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('[' * 100000 + '1' + ']' * 100000) + self.loads('[' * very_deep + '1' + ']' * very_deep) + @support.skip_if_unlimited_stack_size + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} - for x in range(100000): + for x in range(500_000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): with support.infinite_recursion(5000): @@ -95,6 +104,9 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index a5c46bb64b4..e77ec152280 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -1,7 +1,8 @@ import sys from test.test_json import PyTest, CTest -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestScanstring: def test_scanstring(self): @@ -143,11 +144,10 @@ def test_bad_escapes(self): with self.assertRaises(self.JSONDecodeError, msg=s): scanstring(s, 1, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_overflow(self): with self.assertRaises(OverflowError): - self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + self.json.decoder.scanstring("xxx", sys.maxsize+1) class TestPyScanstring(TestScanstring, PyTest): pass diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index ada96729123..370a2539d10 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,6 +1,6 @@ from test.test_json import CTest -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class BadBool: @@ -40,8 +40,7 @@ def test_make_encoder(self): b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) - # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable def test_bad_str_encoder(self): # Issue #31505: There shouldn't be an assertion failure in case # c_make_encoder() receives a bad encoder() argument. @@ -63,8 +62,7 @@ def bad_encoder2(*args): with self.assertRaises(ZeroDivisionError): enc('spam', 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_markers_argument_to_encoder(self): # https://bugs.python.org/issue45269 with self.assertRaisesRegex( @@ -74,8 +72,7 @@ def test_bad_markers_argument_to_encoder(self): self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ', False, False, False) - # TODO: RUSTPYTHON; ZeroDivisionError not raised by test - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ZeroDivisionError not raised by test def test_bad_bool_args(self): def test(name): self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) @@ -88,3 +85,35 @@ def test(name): def test_unsortable_keys(self): with self.assertRaises(TypeError): self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable + def test_current_indent_level(self): + enc = self.json.encoder.c_make_encoder( + markers=None, + default=str, + encoder=self.json.encoder.c_encode_basestring, + indent='\t', + key_separator=': ', + item_separator=', ', + sort_keys=False, + skipkeys=False, + allow_nan=False) + expected = ( + '[\n' + '\t"spam", \n' + '\t{\n' + '\t\t"ham": "eggs"\n' + '\t}\n' + ']') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 0)[0], expected) + self.assertEqual(enc(['spam', {'ham': 'eggs'}], -3)[0], expected) + expected2 = ( + '[\n' + '\t\t\t\t"spam", \n' + '\t\t\t\t{\n' + '\t\t\t\t\t"ham": "eggs"\n' + '\t\t\t\t}\n' + '\t\t\t]') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 3)[0], expected2) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}], 3.0) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}]) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 2b63810d539..7b5d217a215 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -6,12 +6,15 @@ import subprocess from test import support -from test.support import os_helper +from test.support import force_colorized, force_not_colorized, os_helper from test.support.script_helper import assert_python_ok +from _colorize import get_theme + @support.requires_subprocess() -class TestTool(unittest.TestCase): +@support.skip_if_pgo_task +class TestMain(unittest.TestCase): data = """ [["blorpie"],[ "whoops" ] , [ @@ -19,6 +22,7 @@ class TestTool(unittest.TestCase): "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" :"yes"} ] """ + module = 'json' expect_without_sort_keys = textwrap.dedent("""\ [ @@ -86,8 +90,9 @@ class TestTool(unittest.TestCase): } """) + @force_not_colorized def test_stdin_stdout(self): - args = sys.executable, '-m', 'json.tool' + args = sys.executable, '-m', self.module process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.expect) self.assertEqual(process.stderr, '') @@ -101,7 +106,8 @@ def _create_infile(self, data=None): def test_infile_stdout(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') @@ -115,7 +121,8 @@ def test_non_ascii_infile(self): ''').encode() infile = self._create_infile(data) - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), expect.splitlines()) @@ -124,7 +131,8 @@ def test_non_ascii_infile(self): def test_infile_outfile(self): infile = self._create_infile() outfile = os_helper.TESTFN + '.out' - rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) + rc, out, err = assert_python_ok('-m', self.module, infile, outfile, + PYTHON_COLORS='0') self.addCleanup(os.remove, outfile) with open(outfile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) @@ -134,33 +142,38 @@ def test_infile_outfile(self): def test_writing_in_place(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile) + rc, out, err = assert_python_ok('-m', self.module, infile, infile, + PYTHON_COLORS='0') with open(infile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) self.assertEqual(out, b'') self.assertEqual(err, b'') + @force_not_colorized def test_jsonlines(self): - args = sys.executable, '-m', 'json.tool', '--json-lines' + args = sys.executable, '-m', self.module, '--json-lines' process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') def test_help_flag(self): - rc, out, err = assert_python_ok('-m', 'json.tool', '-h') + rc, out, err = assert_python_ok('-m', self.module, '-h', + PYTHON_COLORS='0') self.assertEqual(rc, 0) - self.assertTrue(out.startswith(b'usage: ')) + self.assertStartsWith(out, b'usage: ') self.assertEqual(err, b'') def test_sort_keys_flag(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', '--sort-keys', infile) + rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) self.assertEqual(err, b'') + @force_not_colorized def test_indent(self): input_ = '[1, 2]' expect = textwrap.dedent('''\ @@ -169,31 +182,34 @@ def test_indent(self): 2 ] ''') - args = sys.executable, '-m', 'json.tool', '--indent', '2' + args = sys.executable, '-m', self.module, '--indent', '2' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_no_indent(self): input_ = '[1,\n2]' expect = '[1, 2]\n' - args = sys.executable, '-m', 'json.tool', '--no-indent' + args = sys.executable, '-m', self.module, '--no-indent' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_tab(self): input_ = '[1, 2]' expect = '[\n\t1,\n\t2\n]\n' - args = sys.executable, '-m', 'json.tool', '--tab' + args = sys.executable, '-m', self.module, '--tab' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_compact(self): input_ = '[ 1 ,\n 2]' expect = '[1,2]\n' - args = sys.executable, '-m', 'json.tool', '--compact' + args = sys.executable, '-m', self.module, '--compact' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -202,7 +218,8 @@ def test_no_ensure_ascii_flag(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile) + assert_python_ok('-m', self.module, '--no-ensure-ascii', infile, + outfile, PYTHON_COLORS='0') with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting utf-8 encoded output file @@ -213,20 +230,100 @@ def test_ensure_ascii_default(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', infile, outfile) + assert_python_ok('-m', self.module, infile, outfile, PYTHON_COLORS='0') with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting an ascii encoded output file expected = [b'{', rb' "key": "\ud83d\udca9"', b"}"] self.assertEqual(lines, expected) + @force_not_colorized @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") def test_broken_pipe_error(self): - cmd = [sys.executable, '-m', 'json.tool'] + cmd = [sys.executable, '-m', self.module] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - # bpo-39828: Closing before json.tool attempts to write into stdout. + # bpo-39828: Closing before json attempts to write into stdout. proc.stdout.close() proc.communicate(b'"{}"') self.assertEqual(proc.returncode, errno.EPIPE) + + @force_colorized + def test_colors(self): + infile = os_helper.TESTFN + self.addCleanup(os.remove, infile) + + t = get_theme().syntax + ob = "{" + cb = "}" + + cases = ( + ('{}', '{}'), + ('[]', '[]'), + ('null', f'{t.keyword}null{t.reset}'), + ('true', f'{t.keyword}true{t.reset}'), + ('false', f'{t.keyword}false{t.reset}'), + ('NaN', f'{t.number}NaN{t.reset}'), + ('Infinity', f'{t.number}Infinity{t.reset}'), + ('-Infinity', f'{t.number}-Infinity{t.reset}'), + ('"foo"', f'{t.string}"foo"{t.reset}'), + (r'" \"foo\" "', f'{t.string}" \\"foo\\" "{t.reset}'), + ('"α"', f'{t.string}"\\u03b1"{t.reset}'), + ('123', f'{t.number}123{t.reset}'), + ('-1.25e+23', f'{t.number}-1.25e+23{t.reset}'), + (r'{"\\": ""}', + f'''\ +{ob} + {t.definition}"\\\\"{t.reset}: {t.string}""{t.reset} +{cb}'''), + (r'{"\\\\": ""}', + f'''\ +{ob} + {t.definition}"\\\\\\\\"{t.reset}: {t.string}""{t.reset} +{cb}'''), + ('''\ +{ + "foo": "bar", + "baz": 1234, + "qux": [true, false, null], + "xyz": [NaN, -Infinity, Infinity] +}''', + f'''\ +{ob} + {t.definition}"foo"{t.reset}: {t.string}"bar"{t.reset}, + {t.definition}"baz"{t.reset}: {t.number}1234{t.reset}, + {t.definition}"qux"{t.reset}: [ + {t.keyword}true{t.reset}, + {t.keyword}false{t.reset}, + {t.keyword}null{t.reset} + ], + {t.definition}"xyz"{t.reset}: [ + {t.number}NaN{t.reset}, + {t.number}-Infinity{t.reset}, + {t.number}Infinity{t.reset} + ] +{cb}'''), + ) + + for input_, expected in cases: + with self.subTest(input=input_): + with open(infile, "w", encoding="utf-8") as fp: + fp.write(input_) + _, stdout_b, _ = assert_python_ok( + '-m', self.module, infile, FORCE_COLOR='1', __isolated='1' + ) + stdout = stdout_b.decode() + stdout = stdout.replace('\r\n', '\n') # normalize line endings + stdout = stdout.strip() + self.assertEqual(stdout, expected) + + +@support.requires_subprocess() +@support.skip_if_pgo_task +class TestTool(TestMain): + module = 'json.tool' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 4bdb607e7da..ab1be6ea6e8 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -2,7 +2,7 @@ from collections import OrderedDict from test.test_json import PyTest, CTest -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class TestUnicode: @@ -34,6 +34,29 @@ def test_encoding7(self): j = self.dumps(u + "\n", ensure_ascii=False) self.assertEqual(j, f'"{u}\\n"') + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) + def test_big_unicode_encode(self): u = '\U0001d120' self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') @@ -50,6 +73,18 @@ def test_unicode_decode(self): s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) @@ -59,8 +94,6 @@ def test_bytes_encode(self): self.assertRaises(TypeError, self.dumps, b"hi") self.assertRaises(TypeError, self.dumps, [b"hi"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bytes_decode(self): for encoding, bom in [ ('utf-8', codecs.BOM_UTF8), @@ -104,4 +137,15 @@ def test_object_pairs_hook_with_unicode(self): class TestPyUnicode(TestUnicode, PyTest): pass -class TestCUnicode(TestUnicode, CTest): pass +class TestCUnicode(TestUnicode, CTest): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ascii_non_printable_encode(self): + return super().test_ascii_non_printable_encode() + + @unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'") + def test_single_surrogate_decode(self): + return super().test_single_surrogate_decode() + + @unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'") + def test_single_surrogate_encode(self): + return super().test_single_surrogate_encode() diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index e41e7c051f6..918f953cae5 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -58,7 +58,6 @@ def testSyntaxForManyArguments(self): fundef = "def f(*, %s):\n pass\n" % ', '.join('i%d' % i for i in range(300)) compile(fundef, "<test>", "single") - @unittest.expectedFailure # TODO: RUSTPYTHON def testTooManyPositionalErrorMessage(self): def f(a, b=None, *, c=None): pass @@ -157,7 +156,6 @@ def test_issue13343(self): # used to fail with a SystemError. lambda *, k1=unittest: None - @unittest.expectedFailure # TODO: RUSTPYTHON def test_mangling(self): class X: def f(self, *, __a=42): diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py new file mode 100644 index 00000000000..caa1603c78e --- /dev/null +++ b/Lib/test/test_launcher.py @@ -0,0 +1,796 @@ +import contextlib +import itertools +import os +import re +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import unittest +from pathlib import Path +from test import support + +if sys.platform != "win32": + raise unittest.SkipTest("test only applies to Windows") + +# Get winreg after the platform check +import winreg + + +PY_EXE = "py.exe" +DEBUG_BUILD = False +if sys.executable.casefold().endswith("_d.exe".casefold()): + PY_EXE = "py_d.exe" + DEBUG_BUILD = True + +# Registry data to create. On removal, everything beneath top-level names will +# be deleted. +TEST_DATA = { + "PythonTestSuite": { + "DisplayName": "Python Test Suite", + "SupportUrl": "https://www.python.org/", + "3.100": { + "DisplayName": "X.Y version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y.exe", + } + }, + "3.100-32": { + "DisplayName": "X.Y-32 version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y-32.exe", + } + }, + "3.100-arm64": { + "DisplayName": "X.Y-arm64 version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y-arm64.exe", + "ExecutableArguments": "-X fake_arg_for_test", + } + }, + "ignored": { + "DisplayName": "Ignored because no ExecutablePath", + "InstallPath": { + None: sys.prefix, + } + }, + }, + "PythonTestSuite1": { + "DisplayName": "Python Test Suite Single", + "3.100": { + "DisplayName": "Single Interpreter", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": sys.executable, + } + } + }, +} + + +TEST_PY_ENV = dict( + PY_PYTHON="PythonTestSuite/3.100", + PY_PYTHON2="PythonTestSuite/3.100-32", + PY_PYTHON3="PythonTestSuite/3.100-arm64", +) + + +TEST_PY_DEFAULTS = "\n".join([ + "[defaults]", + *[f"{k[3:].lower()}={v}" for k, v in TEST_PY_ENV.items()], +]) + + +TEST_PY_COMMANDS = "\n".join([ + "[commands]", + "test-command=TEST_EXE.exe", +]) + + +def quote(s): + s = str(s) + return f'"{s}"' if " " in s else s + + +def create_registry_data(root, data): + def _create_registry_data(root, key, value): + if isinstance(value, dict): + # For a dict, we recursively create keys + with winreg.CreateKeyEx(root, key) as hkey: + for k, v in value.items(): + _create_registry_data(hkey, k, v) + elif isinstance(value, str): + # For strings, we set values. 'key' may be None in this case + winreg.SetValueEx(root, key, None, winreg.REG_SZ, value) + else: + raise TypeError("don't know how to create data for '{}'".format(value)) + + for k, v in data.items(): + _create_registry_data(root, k, v) + + +def enum_keys(root): + for i in itertools.count(): + try: + yield winreg.EnumKey(root, i) + except OSError as ex: + if ex.winerror == 259: + break + raise + + +def delete_registry_data(root, keys): + ACCESS = winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS + for key in list(keys): + with winreg.OpenKey(root, key, access=ACCESS) as hkey: + delete_registry_data(hkey, enum_keys(hkey)) + winreg.DeleteKey(root, key) + + +def is_installed(tag): + key = rf"Software\Python\PythonCore\{tag}\InstallPath" + for root, flag in [ + (winreg.HKEY_CURRENT_USER, 0), + (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_64KEY), + (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_32KEY), + ]: + try: + winreg.CloseKey(winreg.OpenKey(root, key, access=winreg.KEY_READ | flag)) + return True + except OSError: + pass + return False + + +class PreservePyIni: + def __init__(self, path, content): + self.path = Path(path) + self.content = content + self._preserved = None + + def __enter__(self): + try: + self._preserved = self.path.read_bytes() + except FileNotFoundError: + self._preserved = None + self.path.write_text(self.content, encoding="utf-16") + + def __exit__(self, *exc_info): + if self._preserved is None: + self.path.unlink() + else: + self.path.write_bytes(self._preserved) + + +class RunPyMixin: + py_exe = None + + @classmethod + def find_py(cls): + py_exe = None + if sysconfig.is_python_build(): + py_exe = Path(sys.executable).parent / PY_EXE + else: + for p in os.getenv("PATH").split(";"): + if p: + py_exe = Path(p) / PY_EXE + if py_exe.is_file(): + break + else: + py_exe = None + + # Test launch and check version, to exclude installs of older + # releases when running outside of a source tree + if py_exe: + try: + with subprocess.Popen( + [py_exe, "-h"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="ascii", + errors="ignore", + ) as p: + p.stdin.close() + version = next(p.stdout, "\n").splitlines()[0].rpartition(" ")[2] + p.stdout.read() + p.wait(10) + if not sys.version.startswith(version): + py_exe = None + except OSError: + py_exe = None + + if not py_exe: + raise unittest.SkipTest( + "cannot locate '{}' for test".format(PY_EXE) + ) + return py_exe + + def get_py_exe(self): + if not self.py_exe: + self.py_exe = self.find_py() + return self.py_exe + + def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=None): + if not self.py_exe: + self.py_exe = self.find_py() + + ignore = {"VIRTUAL_ENV", "PY_PYTHON", "PY_PYTHON2", "PY_PYTHON3"} + env = { + **{k.upper(): v for k, v in os.environ.items() if k.upper() not in ignore}, + "PYLAUNCHER_DEBUG": "1", + "PYLAUNCHER_DRYRUN": "1", + "PYLAUNCHER_LIMIT_TO_COMPANY": "", + **{k.upper(): v for k, v in (env or {}).items()}, + } + if not argv: + argv = [self.py_exe, *args] + with subprocess.Popen( + argv, + env=env, + executable=self.py_exe, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.stdin.close() + p.wait(10) + out = p.stdout.read().decode("utf-8", "replace") + err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") + if p.returncode != expect_returncode and support.verbose and not allow_fail: + print("++ COMMAND ++") + print([self.py_exe, *args]) + print("++ STDOUT ++") + print(out) + print("++ STDERR ++") + print(err) + if allow_fail and p.returncode != expect_returncode: + raise subprocess.CalledProcessError(p.returncode, [self.py_exe, *args], out, err) + else: + self.assertEqual(expect_returncode, p.returncode) + data = { + s.partition(":")[0]: s.partition(":")[2].lstrip() + for s in err.splitlines() + if not s.startswith("#") and ":" in s + } + data["stdout"] = out + data["stderr"] = err + return data + + def py_ini(self, content): + local_appdata = os.environ.get("LOCALAPPDATA") + if not local_appdata: + raise unittest.SkipTest("LOCALAPPDATA environment variable is " + "missing or empty") + return PreservePyIni(Path(local_appdata) / "py.ini", content) + + @contextlib.contextmanager + def script(self, content, encoding="utf-8"): + file = Path(tempfile.mktemp(dir=os.getcwd()) + ".py") + if isinstance(content, bytes): + file.write_bytes(content) + else: + file.write_text(content, encoding=encoding) + try: + yield file + finally: + file.unlink() + + @contextlib.contextmanager + def fake_venv(self): + venv = Path.cwd() / "Scripts" + venv.mkdir(exist_ok=True, parents=True) + venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) + venv_exe.touch() + try: + yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} + finally: + shutil.rmtree(venv) + + +class TestLauncher(unittest.TestCase, RunPyMixin): + @classmethod + def setUpClass(cls): + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, rf"Software\Python") as key: + create_registry_data(key, TEST_DATA) + + if support.verbose: + p = subprocess.check_output("reg query HKCU\\Software\\Python /s") + #print(p.decode('mbcs')) + + + @classmethod + def tearDownClass(cls): + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, rf"Software\Python", access=winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS) as key: + delete_registry_data(key, TEST_DATA) + + + def test_version(self): + data = self.run_py(["-0"]) + self.assertEqual(self.py_exe, Path(data["argv0"])) + self.assertEqual(sys.version.partition(" ")[0], data["version"]) + + def test_help_option(self): + data = self.run_py(["-h"]) + self.assertEqual("True", data["SearchInfo.help"]) + + def test_list_option(self): + for opt, v1, v2 in [ + ("-0", "True", "False"), + ("-0p", "False", "True"), + ("--list", "True", "False"), + ("--list-paths", "False", "True"), + ]: + with self.subTest(opt): + data = self.run_py([opt]) + self.assertEqual(v1, data["SearchInfo.list"]) + self.assertEqual(v2, data["SearchInfo.listPaths"]) + + def test_list(self): + data = self.run_py(["--list"]) + found = {} + expect = {} + for line in data["stdout"].splitlines(): + m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) + if m: + found[m.group(1)] = m.group(3) + for company in TEST_DATA: + company_data = TEST_DATA[company] + tags = [t for t in company_data if isinstance(company_data[t], dict)] + for tag in tags: + arg = f"-V:{company}/{tag}" + expect[arg] = company_data[tag]["DisplayName"] + expect.pop(f"-V:{company}/ignored", None) + + actual = {k: v for k, v in found.items() if k in expect} + try: + self.assertDictEqual(expect, actual) + except: + if support.verbose: + print("*** STDOUT ***") + print(data["stdout"]) + raise + + def test_list_paths(self): + data = self.run_py(["--list-paths"]) + found = {} + expect = {} + for line in data["stdout"].splitlines(): + m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) + if m: + found[m.group(1)] = m.group(3) + for company in TEST_DATA: + company_data = TEST_DATA[company] + tags = [t for t in company_data if isinstance(company_data[t], dict)] + for tag in tags: + arg = f"-V:{company}/{tag}" + install = company_data[tag]["InstallPath"] + try: + expect[arg] = install["ExecutablePath"] + try: + expect[arg] += " " + install["ExecutableArguments"] + except KeyError: + pass + except KeyError: + expect[arg] = str(Path(install[None]) / Path(sys.executable).name) + + expect.pop(f"-V:{company}/ignored", None) + + actual = {k: v for k, v in found.items() if k in expect} + try: + self.assertDictEqual(expect, actual) + except: + if support.verbose: + print("*** STDOUT ***") + print(data["stdout"]) + raise + + def test_filter_to_company(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_to_company_with_default(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/"], env=dict(PY_PYTHON="3.0")) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_to_tag(self): + company = "PythonTestSuite" + data = self.run_py(["-V:3.100"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + data = self.run_py(["-V:3.100-32"]) + self.assertEqual("X.Y-32.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100-32", data["env.tag"]) + + data = self.run_py(["-V:3.100-arm64"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100-arm64", data["env.tag"]) + + def test_filter_to_company_and_tag(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/3.1"], expect_returncode=103) + + data = self.run_py([f"-V:{company}/3.100"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_with_single_install(self): + company = "PythonTestSuite1" + data = self.run_py( + ["-V:Nonexistent"], + env={"PYLAUNCHER_LIMIT_TO_COMPANY": company}, + expect_returncode=103, + ) + + def test_search_major_3(self): + try: + data = self.run_py(["-3"], allow_fail=True) + except subprocess.CalledProcessError: + raise unittest.SkipTest("requires at least one Python 3.x install") + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "3.") + + def test_search_major_3_32(self): + try: + data = self.run_py(["-3-32"], allow_fail=True) + except subprocess.CalledProcessError: + if not any(is_installed(f"3.{i}-32") for i in range(5, 11)): + raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") + raise + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "3.") + self.assertEndsWith(data["env.tag"], "-32") + + def test_search_major_2(self): + try: + data = self.run_py(["-2"], allow_fail=True) + except subprocess.CalledProcessError: + if not is_installed("2.7"): + raise unittest.SkipTest("requires at least one Python 2.x install") + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "2.") + + def test_py_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) + + def test_py2_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-2", "-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) + + def test_py3_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-3", "-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) + + def test_py_default_env(self): + data = self.run_py(["-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) + + def test_py2_default_env(self): + data = self.run_py(["-2", "-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) + + def test_py3_default_env(self): + data = self.run_py(["-3", "-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) + + def test_py_default_short_argv0(self): + with self.py_ini(TEST_PY_DEFAULTS): + for argv0 in ['"py.exe"', 'py.exe', '"py"', 'py']: + with self.subTest(argv0): + data = self.run_py(["--version"], argv=f'{argv0} --version') + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe --version", data["stdout"].strip()) + + def test_py_default_in_list(self): + data = self.run_py(["-0"], env=TEST_PY_ENV) + default = None + for line in data["stdout"].splitlines(): + m = re.match(r"\s*-V:(.+?)\s+?\*\s+(.+)$", line) + if m: + default = m.group(1) + break + self.assertEqual("PythonTestSuite/3.100", default) + + def test_virtualenv_in_list(self): + with self.fake_venv() as (venv_exe, env): + data = self.run_py(["-0p"], env=env) + for line in data["stdout"].splitlines(): + m = re.match(r"\s*\*\s+(.+)$", line) + if m: + self.assertEqual(str(venv_exe), m.group(1)) + break + else: + if support.verbose: + print(data["stdout"]) + print(data["stderr"]) + self.fail("did not find active venv path") + + data = self.run_py(["-0"], env=env) + for line in data["stdout"].splitlines(): + m = re.match(r"\s*\*\s+(.+)$", line) + if m: + self.assertEqual("Active venv", m.group(1)) + break + else: + self.fail("did not find active venv entry") + + def test_virtualenv_with_env(self): + with self.fake_venv() as (venv_exe, env): + data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + # Compare stdout, because stderr goes via ascii + self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) + self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") + # Ensure passing the argument doesn't trigger the same behaviour + self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) + self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") + + def test_py_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_python_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! python -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_py2_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python2 -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py3_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python3 -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py2_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python2 -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py3_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python3 -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py_shebang_short_argv0(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg") as script: + # Override argv to only pass "py.exe" as the command + data = self.run_py([script, "-postarg"], argv=f'"py.exe" "{script}" -postarg') + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f'X.Y.exe -prearg "{script}" -postarg', data["stdout"].strip()) + + def test_py_shebang_valid_bom(self): + with self.py_ini(TEST_PY_DEFAULTS): + content = "#! /usr/bin/python -prearg".encode("utf-8") + with self.script(b"\xEF\xBB\xBF" + content) as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_py_shebang_invalid_bom(self): + with self.py_ini(TEST_PY_DEFAULTS): + content = "#! /usr/bin/python3 -prearg".encode("utf-8") + with self.script(b"\xEF\xAA\xBF" + content) as script: + data = self.run_py([script, "-postarg"]) + self.assertIn("Invalid BOM", data["stderr"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe {quote(script)} -postarg", data["stdout"].strip()) + + def test_py_handle_64_in_ini(self): + with self.py_ini("\n".join(["[defaults]", "python=3.999-64"])): + # Expect this to fail, but should get oldStyleTag flipped on + data = self.run_py([], allow_fail=True, expect_returncode=103) + self.assertEqual("3.999-64", data["SearchInfo.tag"]) + self.assertEqual("True", data["SearchInfo.oldStyleTag"]) + + def test_search_path(self): + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: + data = self.run_py( + [script, "-postarg"], + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, + ) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_search_path_exe(self): + # Leave the .exe on the name to ensure we don't add it a second time + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: + data = self.run_py( + [script, "-postarg"], + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, + ) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_recursive_search_path(self): + stem = self.get_py_exe().stem + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {stem}") as script: + data = self.run_py( + [script], + env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, + ) + # The recursive search is ignored and we get normal "py" behavior + self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) + + def test_install(self): + data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) + cmd = data["stdout"].strip() + # If winget is runnable, we should find it. Otherwise, we'll be trying + # to open the Store. + try: + subprocess.check_call(["winget.exe", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except FileNotFoundError: + self.assertIn("ms-windows-store://", cmd) + else: + self.assertIn("winget.exe", cmd) + # Both command lines include the store ID + self.assertIn("9PJPW5LDXLZ5", cmd) + + def test_literal_shebang_absolute(self): + with self.script("#! C:/some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"C:\\some_random_app -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_relative(self): + with self.script("#! ..\\some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted(self): + with self.script('#! "some random app" -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + with self.script('#! some" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted_escape(self): + with self.script('#! some\\" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_command(self): + with self.py_ini(TEST_PY_COMMANDS): + with self.script('#! test-command arg1') as script: + data = self.run_py([script]) + self.assertEqual( + f"TEST_EXE.exe arg1 {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_invalid_template(self): + with self.script('#! /usr/bin/not-python arg1') as script: + data = self.run_py([script]) + expect = script.parent / "/usr/bin/not-python" + self.assertEqual( + f"{quote(expect)} arg1 {quote(script)}", + data["stdout"].strip(), + ) + + def test_shebang_command_in_venv(self): + stem = "python-that-is-not-on-path" + + # First ensure that our test name doesn't exist, and the launcher does + # not match any installed env + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], expect_returncode=103) + + with self.fake_venv() as (venv_exe, env): + # Put a "normal" Python on PATH as a distraction. + # The active VIRTUAL_ENV should be preferred when the name isn't an + # exact match. + exe = Path(Path(venv_exe).name).absolute() + exe.touch() + self.addCleanup(exe.unlink) + env["PATH"] = f"{exe.parent};{os.environ['PATH']}" + + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") + + with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e23e1cc9428..02f65338428 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,10 +4,12 @@ import unittest import os.path import tempfile +import threading import tokenize from importlib.machinery import ModuleSpec from test import support from test.support import os_helper +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -281,6 +283,19 @@ def test_loader(self): self.assertEqual(linecache.getlines(filename, module_globals), ['source for x.y.z\n']) + def test_frozen(self): + filename = '<frozen fakemodule>' + module_globals = {'__file__': FILENAME} + empty = linecache.getlines(filename) + self.assertEqual(empty, []) + lines = linecache.getlines(filename, module_globals) + self.assertGreater(len(lines), 0) + lines_cached = linecache.getlines(filename) + self.assertEqual(lines, lines_cached) + linecache.clearcache() + empty = linecache.getlines(filename) + self.assertEqual(empty, []) + def test_invalid_names(self): for name, desc in [ ('\x00', 'NUL bytes filename'), @@ -361,5 +376,40 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class MultiThreadingTest(unittest.TestCase): + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_read_write_safety(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + filenames = [] + for i in range(10): + name = os.path.join(tmpdirname, f"test_{i}.py") + with open(name, "w") as h: + h.write("import time\n") + h.write("import system\n") + filenames.append(name) + + def linecache_get_line(b): + b.wait() + for _ in range(100): + for name in filenames: + linecache.getline(name, 1) + + def check(funcs): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([linecache_get_line] * 20) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index ed061384f17..6dbe2d7a144 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,8 +1,10 @@ +import signal import sys import textwrap -from test import list_tests +from test import list_tests, support from test.support import cpython_only -from test.support.script_helper import assert_python_ok +from test.support.import_helper import import_module +from test.support.script_helper import assert_python_failure, assert_python_ok import pickle import unittest @@ -48,7 +50,7 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): list(sequence=[]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): class subclass(list): pass @@ -329,6 +331,25 @@ def test_tier2_invalidates_iterator(self): a.append(4) self.assertEqual(list(it), []) + @support.cpython_only + def test_no_memory(self): + # gh-118331: Make sure we don't crash if list allocation fails + import_module("_testcapi") + code = textwrap.dedent(""" + import _testcapi, sys + # Prime the freelist + l = [None] + del l + _testcapi.set_nomemory(0) + l = [None] + """) + rc, _, _ = assert_python_failure("-c", code) + if support.MS_WINDOWS: + # STATUS_ACCESS_VIOLATION + self.assertNotEqual(rc, 0xC0000005) + else: + self.assertNotEqual(rc, -int(signal.SIGSEGV)) + def test_deopt_from_append_list(self): # gh-132011: it used to crash, because # of `CALL_LIST_APPEND` specialization failure. diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1380c08d28b..964383966c2 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -609,7 +609,7 @@ def test_comp_in_try_except(self): result = snapshot = None try: result = [{func}(value) for value in value] - except: + except ValueError: snapshot = value raise """ @@ -643,13 +643,12 @@ def test_exception_in_post_comp_call(self): value = [1, None] try: [v for v in value].sort() - except: + except TypeError: pass """ self._check_in_scopes(code, {"value": [1, None]}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_frame_locals(self): code = """ val = "a" in [sys._getframe().f_locals for a in [0]][0] @@ -716,11 +715,9 @@ def test_multiple_comprehension_name_reuse(self): self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: @@ -754,6 +751,28 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_only_calls_dunder_iter_once(self): + + class Iterator: + + def __init__(self): + self.val = 0 + + def __next__(self): + if self.val == 2: + raise StopIteration + self.val += 1 + return self.val + + # No __iter__ method + + class C: + + def __iter__(self): + return Iterator() + + self.assertEqual([1, 2], [i for i in C()]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9004e9ed744..6c0cb49f78b 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -736,6 +736,7 @@ def remove_loop(fname, tries): @threading_helper.requires_working_threading() @skip_if_asan_fork @skip_if_tsan_fork + @unittest.skip("TODO: RUSTPYTHON; Flaky") def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): @@ -1115,7 +1116,6 @@ class SMTPHandlerTest(BaseTest): # bpo-14314, bpo-19665, bpo-34092: don't wait forever TIMEOUT = support.LONG_TIMEOUT - @unittest.skip("TODO: RUSTPYTHON; hangs") def test_basic(self): sockmap = {} server = TestSMTPServer((socket_helper.HOST, 0), self.process_message, 0.001, @@ -2153,7 +2153,6 @@ def handle_request(self, request): request.end_headers() self.handled.set() - @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_output(self): # The log message sent to the HTTPHandler is properly received. logger = logging.getLogger("http") @@ -4058,7 +4057,9 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) - @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' + @unittest.skip('TODO: RUSTPYTHON, flaky EOFError') + # TODO: RUSTPYTHON - SemLock not implemented on Windows + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): @@ -4118,7 +4119,8 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) - @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' + # TODO: RUSTPYTHON - SemLock not implemented on Windows + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_111615(self): # See gh-111615 import_helper.import_module('_multiprocessing') # see gh-113692 @@ -5163,7 +5165,7 @@ def __init__(self, name='MyLogger', level=logging.NOTSET): h.close() logging.setLoggerClass(logging.Logger) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError during module teardown in __del__ def test_logging_at_shutdown(self): # bpo-20037: Doing text I/O late at interpreter shutdown must not crash code = textwrap.dedent(""" @@ -5183,7 +5185,7 @@ def __del__(self): self.assertIn("exception in __del__", err) self.assertIn("ValueError: some error", err) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError during module teardown in __del__ def test_logging_at_shutdown_open(self): # bpo-26789: FileHandler keeps a reference to the builtin open() # function to be able to open or reopen the file during Python diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index dbcc85bd69e..a879c6f5df8 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -386,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. @@ -412,7 +403,7 @@ def check_float_conversion(self, n): "Got {}, expected {}.".format(n, actual, expected)) self.assertEqual(actual, expected, msg) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_float_conversion(self): @@ -483,6 +474,12 @@ def test_float_conversion(self): self.check_float_conversion(value) self.check_float_conversion(-value) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_float_conversion_huge_integer(self, size): + v = 1 << size + self.assertRaises(OverflowError, float, v) + def test_float_overflow(self): for x in -2.0, -1.0, 0.0, 1.0, 2.0: self.assertEqual(float(int(x)), x) @@ -624,6 +621,56 @@ def __lt__(self, other): eq(x > y, Rcmp > 0) eq(x >= y, Rcmp >= 0) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_mixed_compares_huge_integer(self, size): + v = 1 << size + f = sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + + del v + v = (-1) << size + f = -sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('-inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + def test__format__(self): self.assertEqual(format(123456789, 'd'), '123456789') self.assertEqual(format(123456789, 'd'), '123456789') @@ -823,7 +870,7 @@ def check_truediv(self, a, b, skip_small=True): self.assertEqual(expected, got, "Incorrectly rounded division {}/{}: " "expected {}, got {}".format(a, b, expected, got)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_correctly_rounded_true_division(self): # more stringent tests than those above, checking that the @@ -944,9 +991,12 @@ def test_huge_lshift_of_zero(self): self.assertEqual(0 << (sys.maxsize + 1), 0) @support.cpython_only - @support.bigmemtest(sys.maxsize + 1000, memuse=2/15 * 2, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_lshift(self, size): - self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize) + v = 5 << size + self.assertEqual(v.bit_length(), size + 3) + self.assertEqual(v.bit_count(), 2) + self.assertEqual(v >> size, 5) def test_huge_rshift(self): huge_shift = 1 << 1000 @@ -958,11 +1008,13 @@ def test_huge_rshift(self): self.assertEqual(-2**128 >> huge_shift, -1) @support.cpython_only - @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_rshift_of_huge(self, size): - huge = ((1 << 500) + 11) << sys.maxsize - self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5) - self.assertEqual(huge >> (sys.maxsize + 1000), 0) + huge = ((1 << 500) + 11) << size + self.assertEqual(huge.bit_length(), size + 501) + self.assertEqual(huge.bit_count(), 4) + self.assertEqual(huge >> (size + 1), (1 << 499) + 5) + self.assertEqual(huge >> (size + 1000), 0) def test_small_rshift(self): self.assertEqual(42 >> 1, 21) @@ -1347,7 +1399,7 @@ class SubStr(str): self.assertEqual((0).to_bytes(1, SubStr('big')), b'\x00') self.assertEqual((0).to_bytes(0, SubStr('little')), b'') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_from_bytes(self): def check(tests, byteorder, signed=False): def equivalent_python(byte_array, byteorder, signed=False): @@ -1426,7 +1478,6 @@ def equivalent_python(byte_array, byteorder, signed=False): b'\x00': 0, b'\x00\x00': 0, b'\x01': 1, - b'\x00\x01': 256, b'\xff': -1, b'\xff\xff': -1, b'\x81': -127, @@ -1609,7 +1660,7 @@ def test_square(self): self.assertEqual(n**2, (1 << (2 * bitlen)) - (1 << (bitlen + 1)) + 1) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test___sizeof__(self): self.assertEqual(int.__itemsize__, sys.int_info.sizeof_digit) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 1bac61f59e1..75ab8c42e92 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1,4 +1,3 @@ -import _compression import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os @@ -7,6 +6,7 @@ import sys from test import support import unittest +from compression._common import _streams from test.support import _4G, bigmemtest from test.support.import_helper import import_module @@ -22,8 +22,7 @@ class CompressorDecompressorTestCase(unittest.TestCase): # Test error cases. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_simple_bad_args(self): self.assertRaises(TypeError, LZMACompressor, []) self.assertRaises(TypeError, LZMACompressor, format=3.45) @@ -64,8 +63,7 @@ def test_simple_bad_args(self): lzd.decompress(empty) self.assertRaises(EOFError, lzd.decompress, b"quux") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder def test_bad_filter_spec(self): self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"]) self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}]) @@ -82,8 +80,7 @@ def test_decompressor_after_eof(self): lzd.decompress(COMPRESSED_XZ) self.assertRaises(EOFError, lzd.decompress, b"nyan") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument memlimit def test_decompressor_memlimit(self): lzd = LZMADecompressor(memlimit=1024) self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) @@ -104,8 +101,7 @@ def _test_decompressor(self, lzd, data, check, unused_data=b""): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, unused_data) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_auto(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) @@ -113,44 +109,37 @@ def test_decompressor_auto(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_xz(self): lzd = LZMADecompressor(lzma.FORMAT_XZ) self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_alone(self): lzd = LZMADecompressor(lzma.FORMAT_ALONE) self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_1(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_2(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2) self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_3(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3) self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_4(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_chunks(self): lzd = LZMADecompressor() out = [] @@ -163,8 +152,7 @@ def test_decompressor_chunks(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; EOFError: End of stream already reached def test_decompressor_chunks_empty(self): lzd = LZMADecompressor() out = [] @@ -180,8 +168,7 @@ def test_decompressor_chunks_empty(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_chunks_maxsize(self): lzd = LZMADecompressor() max_length = 100 @@ -273,16 +260,14 @@ def test_decompressor_inputbuf_3(self): out.append(lzd.decompress(COMPRESSED_XZ[300:])) self.assertEqual(b''.join(out), INPUT) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_unused_data(self): lzd = LZMADecompressor() extra = b"fooblibar" self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64, unused_data=extra) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompressor_bad_input(self): lzd = LZMADecompressor() self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) @@ -296,8 +281,7 @@ def test_decompressor_bad_input(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompressor_bug_28275(self): # Test coverage for Issue 28275 lzd = LZMADecompressor() @@ -307,32 +291,28 @@ def test_decompressor_bug_28275(self): # Test that LZMACompressor->LZMADecompressor preserves the input data. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_xz(self): lzc = LZMACompressor() cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_alone(self): lzc = LZMACompressor(lzma.FORMAT_ALONE) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip_raw(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip_raw_empty(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) @@ -343,8 +323,7 @@ def test_roundtrip_raw_empty(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_chunks(self): lzc = LZMACompressor() cdata = [] @@ -355,8 +334,7 @@ def test_roundtrip_chunks(self): lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_empty_chunks(self): lzc = LZMACompressor() cdata = [] @@ -372,8 +350,7 @@ def test_roundtrip_empty_chunks(self): # LZMADecompressor intentionally does not handle concatenated streams. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_multistream(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE, @@ -409,8 +386,6 @@ def test_decompressor_bigmem(self, size): # Pickling raises an exception; there's no way to serialize an lzma_stream. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): @@ -436,8 +411,7 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): # Test error cases: - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder def test_bad_args(self): self.assertRaises(TypeError, lzma.compress) self.assertRaises(TypeError, lzma.compress, []) @@ -465,24 +439,22 @@ def test_bad_args(self): lzma.decompress(b"", format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1) with self.assertRaises(ValueError): lzma.decompress( - b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: memory limit reached def test_decompress_memlimit(self): with self.assertRaises(LZMAError): lzma.decompress(COMPRESSED_XZ, memlimit=1024) with self.assertRaises(LZMAError): lzma.decompress( - COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024) + COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024) with self.assertRaises(LZMAError): lzma.decompress( - COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024) + COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024) # Test LZMADecompressor on known-good input data. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompress_good_input(self): ddata = lzma.decompress(COMPRESSED_XZ) self.assertEqual(ddata, INPUT) @@ -497,23 +469,22 @@ def test_decompress_good_input(self): self.assertEqual(ddata, INPUT) ddata = lzma.decompress( - COMPRESSED_RAW_1, lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + COMPRESSED_RAW_1, lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self.assertEqual(ddata, INPUT) ddata = lzma.decompress( - COMPRESSED_RAW_2, lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + COMPRESSED_RAW_2, lzma.FORMAT_RAW, filters=FILTERS_RAW_2) self.assertEqual(ddata, INPUT) ddata = lzma.decompress( - COMPRESSED_RAW_3, lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + COMPRESSED_RAW_3, lzma.FORMAT_RAW, filters=FILTERS_RAW_3) self.assertEqual(ddata, INPUT) ddata = lzma.decompress( - COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self.assertEqual(ddata, INPUT) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompress_incomplete_input(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128]) self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128]) @@ -526,8 +497,7 @@ def test_decompress_incomplete_input(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128], format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompress_bad_input(self): with self.assertRaises(LZMAError): lzma.decompress(COMPRESSED_BOGUS) @@ -543,8 +513,7 @@ def test_decompress_bad_input(self): # Test that compress()->decompress() preserves the input data. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip(self): cdata = lzma.compress(INPUT) ddata = lzma.decompress(cdata) @@ -570,14 +539,12 @@ def test_decompress_multistream(self): # Test robust handling of non-LZMA data following the compressed stream(s). - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompress_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompress_multistream_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT * 3) @@ -614,8 +581,7 @@ def test_init(self): self.assertIsInstance(f, LZMAFile) self.assertEqual(f.mode, "wb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <FakePath '@test_23396_tmp챈'> != '@test_23396_tmp챈' def test_init_with_PathLike_filename(self): filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): @@ -696,8 +662,7 @@ def test_init_bad_mode(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), "rw") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Invalid check value def test_init_bad_check(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", check=b"asd") @@ -718,6 +683,7 @@ def test_init_bad_check(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_UNKNOWN) + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u32 def test_init_bad_preset(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", preset=4.39) @@ -725,18 +691,19 @@ def test_init_bad_preset(self): LZMAFile(BytesIO(), "w", preset=10) with self.assertRaises(LZMAError): LZMAFile(BytesIO(), "w", preset=23) - with self.assertRaises(OverflowError): + with self.assertRaises(ValueError): LZMAFile(BytesIO(), "w", preset=-1) - with self.assertRaises(OverflowError): + with self.assertRaises(ValueError): LZMAFile(BytesIO(), "w", preset=-7) + with self.assertRaises(OverflowError): + LZMAFile(BytesIO(), "w", preset=2**1000) with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", preset="foo") # Cannot specify a preset with mode="r". with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), preset=3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder def test_init_bad_filter_spec(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", filters=[b"wobsite"]) @@ -754,8 +721,7 @@ def test_init_bad_filter_spec(self): LZMAFile(BytesIO(), "w", filters=[{"id": lzma.FILTER_X86, "foo": 0}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_init_with_preset_and_filters(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW, @@ -874,8 +840,7 @@ def test_writable(self): f.close() self.assertRaises(ValueError, f.writable) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_read(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: self.assertEqual(f.read(), INPUT) @@ -923,8 +888,7 @@ def test_read_10(self): chunks.append(result) self.assertEqual(b"".join(chunks), INPUT) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_read_multistream(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: self.assertEqual(f.read(), INPUT * 5) @@ -937,22 +901,20 @@ def test_read_multistream(self): def test_read_multistream_buffer_size_aligned(self): # Test the case where a stream boundary coincides with the end # of the raw read buffer. - saved_buffer_size = _compression.BUFFER_SIZE - _compression.BUFFER_SIZE = len(COMPRESSED_XZ) + saved_buffer_size = _streams.BUFFER_SIZE + _streams.BUFFER_SIZE = len(COMPRESSED_XZ) try: with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: self.assertEqual(f.read(), INPUT * 5) finally: - _compression.BUFFER_SIZE = saved_buffer_size + _streams.BUFFER_SIZE = saved_buffer_size - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_read_trailing_junk(self): with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f: self.assertEqual(f.read(), INPUT) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_read_multistream_trailing_junk(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f: self.assertEqual(f.read(), INPUT * 5) @@ -1058,8 +1020,7 @@ def test_read_bad_args(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: self.assertRaises(TypeError, f.read, float()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_read_bad_data(self): with LZMAFile(BytesIO(COMPRESSED_BOGUS)) as f: self.assertRaises(LZMAError, f.read) @@ -1105,20 +1066,19 @@ def test_peek(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek() self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek(10) self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) def test_peek_bad_args(self): with LZMAFile(BytesIO(), "w") as f: self.assertRaises(ValueError, f.peek) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_iterator(self): with BytesIO(INPUT) as f: lines = f.readlines() @@ -1150,16 +1110,15 @@ def test_readlines(self): def test_decompress_limited(self): """Decompressed data buffering should be limited""" bomb = lzma.compress(b'\0' * int(2e6), preset=6) - self.assertLess(len(bomb), _compression.BUFFER_SIZE) + self.assertLess(len(bomb), _streams.BUFFER_SIZE) decomp = LZMAFile(BytesIO(bomb)) self.assertEqual(decomp.read(1), b'\0') max_decomp = 1 + DEFAULT_BUFFER_SIZE self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, - "Excessive amount of data was decompressed") + "Excessive amount of data was decompressed") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: @@ -1428,8 +1387,7 @@ def test_tell_bad_args(self): f.close() self.assertRaises(ValueError, f.tell) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: True is not false def test_issue21872(self): # sometimes decompress data incompletely @@ -1513,8 +1471,7 @@ def test_filename(self): with lzma.open(TESTFN, "rb") as f: self.assertEqual(f.read(), INPUT * 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <FakePath '@test_23396_tmp챈'> != '@test_23396_tmp챈' def test_with_pathlike_filename(self): filename = FakePath(TESTFN) with TempFile(filename): @@ -1541,8 +1498,7 @@ def test_bad_params(self): with self.assertRaises(ValueError): lzma.open(TESTFN, "rb", newline="\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_format_and_filters(self): # Test non-default format and filter chain. options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1} @@ -1573,8 +1529,6 @@ def test_encoding_error_handler(self): with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f: self.assertEqual(f.read(), "foobar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = INPUT.decode("ascii") @@ -1599,8 +1553,6 @@ def test_x_mode(self): class MiscellaneousTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_check_supported(self): # CHECK_NONE and CHECK_CRC32 should always be supported, # regardless of the options liblzma was compiled with. @@ -1613,8 +1565,7 @@ def test_is_check_supported(self): # This value should not be a valid check ID. self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 0 arguments, got 1 def test__encode_filter_properties(self): with self.assertRaises(TypeError): lzma._encode_filter_properties(b"not a dict") @@ -1624,20 +1575,19 @@ def test__encode_filter_properties(self): lzma._encode_filter_properties({"id": lzma.FILTER_LZMA2, "junk": 12}) with self.assertRaises(lzma.LZMAError): lzma._encode_filter_properties({"id": lzma.FILTER_DELTA, - "dist": 9001}) + "dist": 9001}) # Test with parameters used by zipfile module. props = lzma._encode_filter_properties({ - "id": lzma.FILTER_LZMA1, - "pb": 2, - "lp": 0, - "lc": 3, - "dict_size": 8 << 20, - }) + "id": lzma.FILTER_LZMA1, + "pb": 2, + "lp": 0, + "lc": 3, + "dict_size": 8 << 20, + }) self.assertEqual(props, b"]\x00\x00\x80\x00") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: LZMAError not raised def test__decode_filter_properties(self): with self.assertRaises(TypeError): lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes}) @@ -1646,7 +1596,7 @@ def test__decode_filter_properties(self): # Test with parameters used by zipfile module. filterspec = lzma._decode_filter_properties( - lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") self.assertEqual(filterspec["id"], lzma.FILTER_LZMA1) self.assertEqual(filterspec["pb"], 2) self.assertEqual(filterspec["lp"], 0) @@ -1661,11 +1611,10 @@ def test__decode_filter_properties(self): filterspec = lzma._decode_filter_properties(f, b"") self.assertEqual(filterspec, {"id": f}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 0 arguments, got 1 def test_filter_properties_roundtrip(self): spec1 = lzma._decode_filter_properties( - lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") reencoded = lzma._encode_filter_properties(spec1) spec2 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, reencoded) self.assertEqual(spec1, spec2) @@ -2194,4 +2143,4 @@ def test_filter_properties_roundtrip(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py new file mode 100644 index 00000000000..0169948e453 --- /dev/null +++ b/Lib/test/test_mailbox.py @@ -0,0 +1,2493 @@ +import os +import sys +import time +import socket +import email +import email.message +import re +import io +import tempfile +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import refleak_helper +from test.support import socket_helper +import unittest +import textwrap +import mailbox +import glob + + +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + +class TestBase: + + all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage, + mailbox.mboxMessage, mailbox.MHMessage, + mailbox.BabylMessage, mailbox.MMDFMessage) + + def _check_sample(self, msg): + # Inspect a mailbox.Message representation of the sample message + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + for key, value in _sample_headers: + self.assertIn(value, msg.get_all(key)) + self.assertTrue(msg.is_multipart()) + self.assertEqual(len(msg.get_payload()), len(_sample_payloads)) + for i, payload in enumerate(_sample_payloads): + part = msg.get_payload(i) + self.assertIsInstance(part, email.message.Message) + self.assertNotIsInstance(part, mailbox.Message) + self.assertEqual(part.get_payload(), payload) + + def _delete_recursively(self, target): + # Delete a file or delete a directory recursively + if os.path.isdir(target): + os_helper.rmtree(target) + elif os.path.exists(target): + os_helper.unlink(target) + + +class TestMailbox(TestBase): + + maxDiff = None + + _factory = None # Overridden by subclasses to reuse tests + _template = 'From: foo\n\n%s\n' + + def setUp(self): + self._path = os_helper.TESTFN + self._delete_recursively(self._path) + self._box = self._factory(self._path) + + def tearDown(self): + self._box.close() + self._delete_recursively(self._path) + + def test_add(self): + # Add copies of a sample message + keys = [] + keys.append(self._box.add(self._template % 0)) + self.assertEqual(len(self._box), 1) + keys.append(self._box.add(mailbox.Message(_sample_message))) + self.assertEqual(len(self._box), 2) + keys.append(self._box.add(email.message_from_string(_sample_message))) + self.assertEqual(len(self._box), 3) + keys.append(self._box.add(io.BytesIO(_bytes_sample_message))) + self.assertEqual(len(self._box), 4) + keys.append(self._box.add(_sample_message)) + self.assertEqual(len(self._box), 5) + keys.append(self._box.add(_bytes_sample_message)) + self.assertEqual(len(self._box), 6) + with self.assertWarns(DeprecationWarning): + keys.append(self._box.add( + io.TextIOWrapper(io.BytesIO(_bytes_sample_message), encoding="utf-8"))) + self.assertEqual(len(self._box), 7) + self.assertEqual(self._box.get_string(keys[0]), self._template % 0) + for i in (1, 2, 3, 4, 5, 6): + self._check_sample(self._box[keys[i]]) + + _nonascii_msg = textwrap.dedent("""\ + From: foo + Subject: Falinaptár házhozszállítással. Már rendeltél? + + 0 + """) + + def test_add_invalid_8bit_bytes_header(self): + key = self._box.add(self._nonascii_msg.encode('latin-1')) + self.assertEqual(len(self._box), 1) + self.assertEqual(self._box.get_bytes(key), + self._nonascii_msg.encode('latin-1')) + + def test_invalid_nonascii_header_as_string(self): + subj = self._nonascii_msg.splitlines()[1] + key = self._box.add(subj.encode('latin-1')) + self.assertEqual(self._box.get_string(key), + 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' + 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') + + def test_add_nonascii_string_header_raises(self): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(self._nonascii_msg) + self._box.flush() + self.assertEqual(len(self._box), 0) + self.assertMailboxEmpty() + + def test_add_that_raises_leaves_mailbox_empty(self): + class CustomError(Exception): ... + exc_msg = "a fake error" + + def raiser(*args, **kw): + raise CustomError(exc_msg) + support.patch(self, email.generator.BytesGenerator, 'flatten', raiser) + with self.assertRaisesRegex(CustomError, exc_msg): + self._box.add(email.message_from_string("From: Alphöso")) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + _non_latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: báz + Subject: Maintenant je vous présente mon collègue, le pouf célèbre + \tJean de Baddie + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Да, они летят. + """).encode('utf-8') + + def test_add_8bit_body(self): + key = self._box.add(self._non_latin_bin_msg) + self.assertEqual(self._box.get_bytes(key), + self._non_latin_bin_msg) + with self._box.get_file(key) as f: + self.assertEqual(f.read(), + self._non_latin_bin_msg.replace(b'\n', + os.linesep.encode())) + self.assertEqual(self._box[key].get_payload(), + "Да, они летят.\n") + + def test_add_binary_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_binary_nonascii_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(self._non_latin_bin_msg) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + self._non_latin_bin_msg.split(b'\n')) + + def test_add_text_file_warns(self): + with tempfile.TemporaryFile('w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + with self.assertWarns(DeprecationWarning): + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_StringIO_warns(self): + with self.assertWarns(DeprecationWarning): + key = self._box.add(io.StringIO(self._template % "0")) + self.assertEqual(self._box.get_string(key), self._template % "0") + + def test_add_nonascii_StringIO_raises(self): + with self.assertWarns(DeprecationWarning): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(io.StringIO(self._nonascii_msg)) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + def test_remove(self): + # Remove messages using remove() + self._test_remove_or_delitem(self._box.remove) + + def test_delitem(self): + # Remove messages using __delitem__() + self._test_remove_or_delitem(self._box.__delitem__) + + def _test_remove_or_delitem(self, method): + # (Used by test_remove() and test_delitem().) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + method(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self.assertRaises(KeyError, lambda: method(key0)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + key2 = self._box.add(self._template % 2) + self.assertEqual(len(self._box), 2) + method(key2) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key2]) + self.assertRaises(KeyError, lambda: method(key2)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + method(key1) + self.assertEqual(len(self._box), 0) + self.assertRaises(KeyError, lambda: self._box[key1]) + self.assertRaises(KeyError, lambda: method(key1)) + + def test_discard(self, repetitions=10): + # Discard messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get(self): + # Retrieve messages using get() + key0 = self._box.add(self._template % 0) + msg = self._box.get(key0) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertIsNone(self._box.get('foo')) + self.assertIs(self._box.get('foo', False), False) + self._box.close() + self._box = self._factory(self._path) + key1 = self._box.add(self._template % 1) + msg = self._box.get(key1) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '1\n') + + def test_getitem(self): + # Retrieve message using __getitem__() + key0 = self._box.add(self._template % 0) + msg = self._box[key0] + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertRaises(KeyError, lambda: self._box['foo']) + self._box.discard(key0) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get_message(self): + # Get Message representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + msg0 = self._box.get_message(key0) + self.assertIsInstance(msg0, mailbox.Message) + self.assertEqual(msg0['from'], 'foo') + self.assertEqual(msg0.get_payload(), '0\n') + self._check_sample(self._box.get_message(key1)) + + def test_get_bytes(self): + # Get bytes representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_bytes(key0), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1), _bytes_sample_message) + + def test_get_string(self): + # Get string representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_string(key0), self._template % 0) + self.assertEqual(self._box.get_string(key1).split('\n'), + _sample_message.split('\n')) + + def test_get_file(self): + # Get file representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + with self._box.get_file(key0) as file: + data0 = file.read() + with self._box.get_file(key1) as file: + data1 = file.read() + self.assertEqual(data0.decode('ascii').replace(os.linesep, '\n'), + self._template % 0) + self.assertEqual(data1.decode('ascii').replace(os.linesep, '\n'), + _sample_message) + + def test_get_file_can_be_closed_twice(self): + # Issue 11700 + key = self._box.add(_sample_message) + f = self._box.get_file(key) + f.close() + f.close() + + def test_iterkeys(self): + # Get keys using iterkeys() + self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False) + + def test_keys(self): + # Get keys using keys() + self._check_iteration(self._box.keys, do_keys=True, do_values=False) + + def test_itervalues(self): + # Get values using itervalues() + self._check_iteration(self._box.itervalues, do_keys=False, + do_values=True) + + def test_iter(self): + # Get values using __iter__() + self._check_iteration(self._box.__iter__, do_keys=False, + do_values=True) + + def test_values(self): + # Get values using values() + self._check_iteration(self._box.values, do_keys=False, do_values=True) + + def test_iteritems(self): + # Get keys and values using iteritems() + self._check_iteration(self._box.iteritems, do_keys=True, + do_values=True) + + def test_items(self): + # Get keys and values using items() + self._check_iteration(self._box.items, do_keys=True, do_values=True) + + def _check_iteration(self, method, do_keys, do_values, repetitions=10): + for value in method(): + self.fail("Not empty") + keys, values = [], [] + for i in range(repetitions): + keys.append(self._box.add(self._template % i)) + values.append(self._template % i) + if do_keys and not do_values: + returned_keys = list(method()) + elif do_values and not do_keys: + returned_values = list(method()) + else: + returned_keys, returned_values = [], [] + for key, value in method(): + returned_keys.append(key) + returned_values.append(value) + if do_keys: + self.assertEqual(len(keys), len(returned_keys)) + self.assertEqual(set(keys), set(returned_keys)) + if do_values: + count = 0 + for value in returned_values: + self.assertEqual(value['from'], 'foo') + self.assertLess(int(value.get_payload()), repetitions) + count += 1 + self.assertEqual(len(values), count) + + def test_contains(self): + # Check existence of keys using __contains__() + self.assertNotIn('foo', self._box) + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key0) + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key1) + self.assertNotIn(key1, self._box) + self.assertNotIn(key0, self._box) + self.assertNotIn('foo', self._box) + + def test_len(self, repetitions=10): + # Get message count + keys = [] + for i in range(repetitions): + self.assertEqual(len(self._box), i) + keys.append(self._box.add(self._template % i)) + self.assertEqual(len(self._box), i + 1) + for i in range(repetitions): + self.assertEqual(len(self._box), repetitions - i) + self._box.remove(keys[i]) + self.assertEqual(len(self._box), repetitions - i - 1) + + def test_set_item(self): + # Modify messages using __setitem__() + key0 = self._box.add(self._template % 'original 0') + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._box[key0] = self._template % 'changed 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self._box[key1] = self._template % 'changed 1' + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self._box[key0] = _sample_message + self._check_sample(self._box[key0]) + self._box[key1] = self._box[key0] + self._check_sample(self._box[key1]) + self._box[key0] = self._template % 'original 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self._check_sample(self._box[key1]) + self.assertRaises(KeyError, + lambda: self._box.__setitem__('foo', 'bar')) + self.assertRaises(KeyError, lambda: self._box['foo']) + self.assertEqual(len(self._box), 2) + + def test_clear(self, iterations=10): + # Remove all messages using clear() + keys = [] + for i in range(iterations): + self._box.add(self._template % i) + for i, key in enumerate(keys): + self.assertEqual(self._box.get_string(key), self._template % i) + self._box.clear() + self.assertEqual(len(self._box), 0) + for i, key in enumerate(keys): + self.assertRaises(KeyError, lambda: self._box.get_string(key)) + + def test_pop(self): + # Get and remove a message using pop() + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key0).get_payload(), '0\n') + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + key2 = self._box.add(self._template % 2) + self.assertIn(key2, self._box) + self.assertEqual(self._box.pop(key2).get_payload(), '2\n') + self.assertNotIn(key2, self._box) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key1).get_payload(), '1\n') + self.assertNotIn(key1, self._box) + self.assertEqual(len(self._box), 0) + + def test_popitem(self, iterations=10): + # Get and remove an arbitrary (key, message) using popitem() + keys = [] + for i in range(10): + keys.append(self._box.add(self._template % i)) + seen = [] + for i in range(10): + key, msg = self._box.popitem() + self.assertIn(key, keys) + self.assertNotIn(key, seen) + seen.append(key) + self.assertEqual(int(msg.get_payload()), keys.index(key)) + self.assertEqual(len(self._box), 0) + for key in keys: + self.assertRaises(KeyError, lambda: self._box[key]) + + def test_update(self): + # Modify multiple messages using update() + key0 = self._box.add(self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + key2 = self._box.add(self._template % 'original 2') + self._box.update({key0: self._template % 'changed 0', + key2: _sample_message}) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._check_sample(self._box[key2]) + self._box.update([(key2, self._template % 'changed 2'), + (key1, self._template % 'changed 1'), + (key0, self._template % 'original 0')]) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self.assertEqual(self._box.get_string(key2), + self._template % 'changed 2') + self.assertRaises(KeyError, + lambda: self._box.update({'foo': 'bar', + key0: self._template % "changed 0"})) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % "changed 0") + self.assertEqual(self._box.get_string(key1), + self._template % "changed 1") + self.assertEqual(self._box.get_string(key2), + self._template % "changed 2") + + def test_flush(self): + # Write changes to disk + self._test_flush_or_close(self._box.flush, True) + + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + + def test_lock_unlock(self): + # Lock and unlock the mailbox + self.assertFalse(os.path.exists(self._get_lock_path())) + self._box.lock() + self.assertTrue(os.path.exists(self._get_lock_path())) + self._box.unlock() + self.assertFalse(os.path.exists(self._get_lock_path())) + + def test_close(self): + # Close mailbox and flush changes to disk + self._test_flush_or_close(self._box.close, False) + + def _test_flush_or_close(self, method, should_call_close): + contents = [self._template % i for i in range(3)] + self._box.add(contents[0]) + self._box.add(contents[1]) + self._box.add(contents[2]) + oldbox = self._box + method() + if should_call_close: + self._box.close() + self._box = self._factory(self._path) + keys = self._box.keys() + self.assertEqual(len(keys), 3) + for key in keys: + self.assertIn(self._box.get_string(key), contents) + oldbox.close() + + def test_dump_message(self): + # Write message representations to disk + for input in (email.message_from_string(_sample_message), + _sample_message, io.BytesIO(_bytes_sample_message)): + output = io.BytesIO() + self._box._dump_message(input, output) + self.assertEqual(output.getvalue(), + _bytes_sample_message.replace(b'\n', os.linesep.encode())) + output = io.BytesIO() + self.assertRaises(TypeError, + lambda: self._box._dump_message(None, output)) + + def _get_lock_path(self): + # Return the path of the dot lock file. May be overridden. + return self._path + '.lock' + + +class TestMailboxSuperclass(TestBase, unittest.TestCase): + + def test_notimplemented(self): + # Test that all Mailbox methods raise NotImplementedException. + box = mailbox.Mailbox('path') + self.assertRaises(NotImplementedError, lambda: box.add('')) + self.assertRaises(NotImplementedError, lambda: box.remove('')) + self.assertRaises(NotImplementedError, lambda: box.__delitem__('')) + self.assertRaises(NotImplementedError, lambda: box.discard('')) + self.assertRaises(NotImplementedError, lambda: box.__setitem__('', '')) + self.assertRaises(NotImplementedError, lambda: box.iterkeys()) + self.assertRaises(NotImplementedError, lambda: box.keys()) + self.assertRaises(NotImplementedError, lambda: box.itervalues().__next__()) + self.assertRaises(NotImplementedError, lambda: box.__iter__().__next__()) + self.assertRaises(NotImplementedError, lambda: box.values()) + self.assertRaises(NotImplementedError, lambda: box.iteritems().__next__()) + self.assertRaises(NotImplementedError, lambda: box.items()) + self.assertRaises(NotImplementedError, lambda: box.get('')) + self.assertRaises(NotImplementedError, lambda: box.__getitem__('')) + self.assertRaises(NotImplementedError, lambda: box.get_message('')) + self.assertRaises(NotImplementedError, lambda: box.get_string('')) + self.assertRaises(NotImplementedError, lambda: box.get_bytes('')) + self.assertRaises(NotImplementedError, lambda: box.get_file('')) + self.assertRaises(NotImplementedError, lambda: '' in box) + self.assertRaises(NotImplementedError, lambda: box.__contains__('')) + self.assertRaises(NotImplementedError, lambda: box.__len__()) + self.assertRaises(NotImplementedError, lambda: box.clear()) + self.assertRaises(NotImplementedError, lambda: box.pop('')) + self.assertRaises(NotImplementedError, lambda: box.popitem()) + self.assertRaises(NotImplementedError, lambda: box.update((('', ''),))) + self.assertRaises(NotImplementedError, lambda: box.flush()) + self.assertRaises(NotImplementedError, lambda: box.lock()) + self.assertRaises(NotImplementedError, lambda: box.unlock()) + self.assertRaises(NotImplementedError, lambda: box.close()) + + +class TestMaildir(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory) + + def setUp(self): + TestMailbox.setUp(self) + if (os.name == 'nt') or (sys.platform == 'cygwin'): + self._box.colon = '!' + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(os.path.join(self._path, 'tmp')), []) + + def test_add_MM(self): + # Add a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_info('foo') + key = self._box.add(msg) + self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' % + (key, self._box.colon)))) + + def test_get_MM(self): + # Get a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + msg_returned = self._box.get_message(key) + self.assertIsInstance(msg_returned, mailbox.MaildirMessage) + self.assertEqual(msg_returned.get_subdir(), 'cur') + self.assertEqual(msg_returned.get_flags(), 'FR') + + def test_set_MM(self): + # Set with a MaildirMessage instance + msg0 = mailbox.MaildirMessage(self._template % 0) + msg0.set_flags('TP') + key = self._box.add(msg0) + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'PT') + msg1 = mailbox.MaildirMessage(self._template % 1) + self._box[key] = msg1 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), '') + self.assertEqual(msg_returned.get_payload(), '1\n') + msg2 = mailbox.MaildirMessage(self._template % 2) + msg2.set_info('2,S') + self._box[key] = msg2 + self._box[key] = self._template % 3 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'S') + self.assertEqual(msg_returned.get_payload(), '3\n') + + def test_consistent_factory(self): + # Add a message. + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + + # Create new mailbox with + class FakeMessage(mailbox.MaildirMessage): + pass + box = mailbox.Maildir(self._path, factory=FakeMessage) + box.colon = self._box.colon + msg2 = box.get_message(key) + self.assertIsInstance(msg2, FakeMessage) + + def test_initialize_new(self): + # Initialize a non-existent mailbox + self.tearDown() + self._box = mailbox.Maildir(self._path) + self._check_basics() + self._delete_recursively(self._path) + self._box = self._factory(self._path, factory=None) + self._check_basics() + + def test_initialize_existing(self): + # Initialize an existing mailbox + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + self._box = mailbox.Maildir(self._path) + self._check_basics() + + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + + def _check_basics(self, factory=None): + # (Used by test_open_new() and test_open_existing().) + self.assertEqual(self._box._path, os.path.abspath(self._path)) + self.assertEqual(self._box._factory, factory) + for subdir in '', 'tmp', 'new', 'cur': + path = os.path.join(self._path, subdir) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_clean(self): + # Remove old files from 'tmp' + foo_path = os.path.join(self._path, 'tmp', 'foo') + bar_path = os.path.join(self._path, 'tmp', 'bar') + with open(foo_path, 'w', encoding='utf-8') as f: + f.write("@") + with open(bar_path, 'w', encoding='utf-8') as f: + f.write("@") + self._box.clean() + self.assertTrue(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + foo_stat = os.stat(foo_path) + os.utime(foo_path, (time.time() - 129600 - 2, + foo_stat.st_mtime)) + self._box.clean() + self.assertFalse(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + + def test_create_tmp(self, repetitions=10): + # Create files in tmp directory + hostname = socket.gethostname() + if '/' in hostname: + hostname = hostname.replace('/', r'\057') + if ':' in hostname: + hostname = hostname.replace(':', r'\072') + pid = os.getpid() + pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)" + r"Q(?P<Q>\d+)\.(?P<host>[^:/]*)") + previous_groups = None + for x in range(repetitions): + tmp_file = self._box._create_tmp() + head, tail = os.path.split(tmp_file.name) + self.assertEqual(head, os.path.abspath(os.path.join(self._path, + "tmp")), + "File in wrong location: '%s'" % head) + match = pattern.match(tail) + self.assertIsNotNone(match, "Invalid file name: '%s'" % tail) + groups = match.groups() + if previous_groups is not None: + self.assertGreaterEqual(int(groups[0]), int(previous_groups[0]), + "Non-monotonic seconds: '%s' before '%s'" % + (previous_groups[0], groups[0])) + if int(groups[0]) == int(previous_groups[0]): + self.assertGreaterEqual(int(groups[1]), int(previous_groups[1]), + "Non-monotonic milliseconds: '%s' before '%s'" % + (previous_groups[1], groups[1])) + self.assertEqual(int(groups[2]), pid, + "Process ID mismatch: '%s' should be '%s'" % + (groups[2], pid)) + self.assertEqual(int(groups[3]), int(previous_groups[3]) + 1, + "Non-sequential counter: '%s' before '%s'" % + (previous_groups[3], groups[3])) + self.assertEqual(groups[4], hostname, + "Host name mismatch: '%s' should be '%s'" % + (groups[4], hostname)) + previous_groups = groups + tmp_file.write(_bytes_sample_message) + tmp_file.seek(0) + self.assertEqual(tmp_file.read(), _bytes_sample_message) + tmp_file.close() + file_count = len(os.listdir(os.path.join(self._path, "tmp"))) + self.assertEqual(file_count, repetitions, + "Wrong file count: '%s' should be '%s'" % + (file_count, repetitions)) + + def test_refresh(self): + # Update the table of contents + self.assertEqual(self._box._toc, {}) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(self._box._toc, {}) + self._box._refresh() + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1)}) + key2 = self._box.add(self._template % 2) + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1)}) + self._box._refresh() + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1), + key2: os.path.join('new', key2)}) + + def test_refresh_after_safety_period(self): + # Issue #13254: Call _refresh after the "file system safety + # period" of 2 seconds has passed; _toc should still be + # updated because this is the first call to _refresh. + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + + self._box = self._factory(self._path) + self.assertEqual(self._box._toc, {}) + + # Emulate sleeping. Instead of sleeping for 2 seconds, use the + # skew factor to make _refresh think that the filesystem + # safety period has passed and re-reading the _toc is only + # required if mtimes differ. + self._box._skewfactor = -3 + + self._box._refresh() + self.assertEqual(sorted(self._box._toc.keys()), sorted([key0, key1])) + + def test_lookup(self): + # Look up message subpaths in the TOC + self.assertRaises(KeyError, lambda: self._box._lookup('foo')) + key0 = self._box.add(self._template % 0) + self.assertEqual(self._box._lookup(key0), os.path.join('new', key0)) + os.remove(os.path.join(self._path, 'new', key0)) + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0)}) + # Be sure that the TOC is read back from disk (see issue #6896 + # about bad mtime behaviour on some systems). + self._box.flush() + self.assertRaises(KeyError, lambda: self._box._lookup(key0)) + self.assertEqual(self._box._toc, {}) + + def test_lock_unlock(self): + # Lock and unlock the mailbox. For Maildir, this does nothing. + self._box.lock() + self._box.unlock() + + def test_get_info(self): + # Test getting message info from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_info(key), '') + msg.set_info('OurTestInfo') + self._box[key] = msg + self.assertEqual(self._box.get_info(key), 'OurTestInfo') + + def test_set_info(self): + # Test setting message info from Maildir, not the message. + # This should immediately rename the message file. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + def check_info(oldinfo, newinfo): + oldfilename = os.path.join(self._box._path, self._box._lookup(key)) + newsubpath = self._box._lookup(key).split(self._box.colon)[0] + if newinfo: + newsubpath += self._box.colon + newinfo + newfilename = os.path.join(self._box._path, newsubpath) + # assert initial conditions + self.assertEqual(self._box.get_info(key), oldinfo) + if not oldinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + self.assertTrue(os.path.exists(oldfilename)) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(newfilename)) + # do the rename + self._box.set_info(key, newinfo) + # assert post conditions + if not newinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(oldfilename)) + self.assertTrue(os.path.exists(newfilename)) + self.assertEqual(self._box.get_info(key), newinfo) + # none -> has info + check_info('', 'info1') + # has info -> same info + check_info('info1', 'info1') + # has info -> different info + check_info('info1', 'info2') + # has info -> none + check_info('info2', '') + # none -> none + check_info('', '') + + def test_get_flags(self): + # Test getting message flags from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + msg.set_flags('T') + self._box[key] = msg + self.assertEqual(self._box.get_flags(key), 'T') + + def test_set_flags(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.set_flags(key, 'S') + self.assertEqual(self._box.get_flags(key), 'S') + + def test_add_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'AC') + self.assertEqual(self._box.get_flags(key), 'ABC') + + def test_remove_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self._box.set_flags(key, 'abc') + self.assertEqual(self._box.get_flags(key), 'abc') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'ac') + self.assertEqual(self._box.get_flags(key), '') + + def test_folder (self): + # Test for bug #1569790: verify that folders returned by .get_folder() + # use the same factory function. + def dummy_factory (s): + return None + box = self._factory(self._path, factory=dummy_factory) + folder = box.add_folder('folder1') + self.assertIs(folder._factory, dummy_factory) + + folder1_alias = box.get_folder('folder1') + self.assertIs(folder1_alias._factory, dummy_factory) + + def test_directory_in_folder (self): + # Test that mailboxes still work if there's a stray extra directory + # in a folder. + for i in range(10): + self._box.add(mailbox.Message(_sample_message)) + + # Create a stray directory + os.mkdir(os.path.join(self._path, 'cur', 'stray-dir')) + + # Check that looping still works with the directory present. + for msg in self._box: + pass + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_file_permissions(self): + # Verify that message files are created without execute permissions + msg = mailbox.MaildirMessage(self._template % 0) + orig_umask = os.umask(0) + try: + key = self._box.add(msg) + finally: + os.umask(orig_umask) + path = os.path.join(self._path, self._box._lookup(key)) + mode = os.stat(path).st_mode + self.assertFalse(mode & 0o111) + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_folder_file_perms(self): + # From bug #3228, we want to verify that the file created inside a Maildir + # subfolder isn't marked as executable. + orig_umask = os.umask(0) + try: + subfolder = self._box.add_folder('subfolder') + finally: + os.umask(orig_umask) + + path = os.path.join(subfolder._path, 'maildirfolder') + st = os.stat(path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. + + def test_reread(self): + # Do an initial unconditional refresh + self._box._refresh() + + # Put the last modified times more than two seconds into the past + # (because mtime may have a two second granularity) + for subdir in ('cur', 'new'): + os.utime(os.path.join(self._box._path, subdir), + (time.time()-5,)*2) + + # Because mtime has a two second granularity in worst case (FAT), a + # refresh is done unconditionally if called for within + # two-second-plus-a-bit of the last one, just in case the mbox has + # changed; so now we have to wait for that interval to expire. + # + # Because this is a test, emulate sleeping. Instead of + # sleeping for 2 seconds, use the skew factor to make _refresh + # think that 2 seconds have passed and re-reading the _toc is + # only required if mtimes differ. + self._box._skewfactor = -3 + + # Re-reading causes the ._toc attribute to be assigned a new dictionary + # object, so we'll check that the ._toc attribute isn't a different + # object. + orig_toc = self._box._toc + def refreshed(): + return self._box._toc is not orig_toc + + self._box._refresh() + self.assertFalse(refreshed()) + + # Now, write something into cur and remove it. This changes + # the mtime and should cause a re-read. Note that "sleep + # emulation" is still in effect, as skewfactor is -3. + filename = os.path.join(self._path, 'cur', 'stray-file') + os_helper.create_empty_file(filename) + os.unlink(filename) + self._box._refresh() + self.assertTrue(refreshed()) + + +class _TestSingleFile(TestMailbox): + '''Common tests for single-file mailboxes''' + + def test_add_doesnt_rewrite(self): + # When only adding messages, flush() should not rewrite the + # mailbox file. See issue #9559. + + # Inode number changes if the contents are written to another + # file which is then renamed over the original file. So we + # must check that the inode number doesn't change. + inode_before = os.stat(self._path).st_ino + + self._box.add(self._template % 0) + self._box.flush() + + inode_after = os.stat(self._path).st_ino + self.assertEqual(inode_before, inode_after) + + # Make sure the message was really added + self._box.close() + self._box = self._factory(self._path) + self.assertEqual(len(self._box), 1) + + def test_permissions_after_flush(self): + # See issue #5346 + + # Make the mailbox world writable. It's unlikely that the new + # mailbox file would have these permissions after flush(), + # because umask usually prevents it. + mode = os.stat(self._path).st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + self.assertEqual(os.stat(self._path).st_mode, mode) + + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + + +class _TestMboxMMDF(_TestSingleFile): + + def tearDown(self): + super().tearDown() + self._box.close() + self._delete_recursively(self._path) + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): + os_helper.unlink(lock_remnant) + + def assertMailboxEmpty(self): + with open(self._path, 'rb') as f: + self.assertEqual(f.readlines(), []) + + def test_get_bytes_from(self): + # Get bytes representations of messages with _unixfrom. + unixfrom = 'From foo@bar blah\n' + key0 = self._box.add(unixfrom + self._template % 0) + key1 = self._box.add(unixfrom + _sample_message) + self.assertEqual(self._box.get_bytes(key0, from_=False), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1, from_=False), + _bytes_sample_message) + self.assertEqual(self._box.get_bytes(key0, from_=True), + (unixfrom + self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1, from_=True), + unixfrom.encode('ascii') + _bytes_sample_message) + + def test_get_string_from(self): + # Get string representations of messages with _unixfrom. + unixfrom = 'From foo@bar blah\n' + key0 = self._box.add(unixfrom + self._template % 0) + key1 = self._box.add(unixfrom + _sample_message) + self.assertEqual(self._box.get_string(key0, from_=False), + self._template % 0) + self.assertEqual(self._box.get_string(key1, from_=False).split('\n'), + _sample_message.split('\n')) + self.assertEqual(self._box.get_string(key0, from_=True), + unixfrom + self._template % 0) + self.assertEqual(self._box.get_string(key1, from_=True).split('\n'), + (unixfrom + _sample_message).split('\n')) + + def test_add_from_string(self): + # Add a string starting with 'From ' to the mailbox + key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') + self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') + self.assertEqual(self._box[key].get_payload(), '0\n') + + def test_add_from_bytes(self): + # Add a byte string starting with 'From ' to the mailbox + key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') + self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') + self.assertEqual(self._box[key].get_payload(), '0\n') + + def test_add_mbox_or_mmdf_message(self): + # Add an mboxMessage or MMDFMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = class_('From foo@bar blah\nFrom: foo\n\n0\n') + key = self._box.add(msg) + + def test_open_close_open(self): + # Open and inspect previously-created mailbox + values = [self._template % i for i in range(3)] + for value in values: + self._box.add(value) + self._box.close() + mtime = os.path.getmtime(self._path) + self._box = self._factory(self._path) + self.assertEqual(len(self._box), 3) + for key in self._box.iterkeys(): + self.assertIn(self._box.get_string(key), values) + self._box.close() + self.assertEqual(mtime, os.path.getmtime(self._path)) + + def test_add_and_close(self): + # Verifying that closing a mailbox doesn't change added items + self._box.add(_sample_message) + for i in range(3): + self._box.add(self._template % i) + self._box.add(_sample_message) + self._box._file.flush() + self._box._file.seek(0) + contents = self._box._file.read() + self._box.close() + with open(self._path, 'rb') as f: + self.assertEqual(contents, f.read()) + self._box = self._factory(self._path) + + @support.requires_fork() + @unittest.skipUnless(hasattr(socket, 'socketpair'), "Test needs socketpair().") + def test_lock_conflict(self): + # Fork off a child process that will lock the mailbox temporarily, + # unlock it and exit. + c, p = socket.socketpair() + self.addCleanup(c.close) + self.addCleanup(p.close) + + pid = os.fork() + if pid == 0: + # child + try: + # lock the mailbox, and signal the parent it can proceed + self._box.lock() + c.send(b'c') + + # wait until the parent is done, and unlock the mailbox + c.recv(1) + self._box.unlock() + finally: + os._exit(0) + + # In the parent, wait until the child signals it locked the mailbox. + p.recv(1) + try: + self.assertRaises(mailbox.ExternalClashError, + self._box.lock) + finally: + # Signal the child it can now release the lock and exit. + p.send(b'p') + # Wait for child to exit. Locking should now succeed. + support.wait_process(pid, exitcode=0) + + self._box.lock() + self._box.unlock() + + def test_relock(self): + # Test case for bug #1575506: the mailbox class was locking the + # wrong file object in its flush() method. + msg = "Subject: sub\n\nbody\n" + key1 = self._box.add(msg) + self._box.flush() + self._box.close() + + self._box = self._factory(self._path) + self._box.lock() + key2 = self._box.add(msg) + self._box.flush() + self.assertTrue(self._box._locked) + self._box.close() + + +class TestMbox(_TestMboxMMDF, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.mbox(path, factory) + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_file_perms(self): + # From bug #3228, we want to verify that the mailbox file isn't executable, + # even if the umask is set to something that would leave executable bits set. + # We only run this test on platforms that support umask. + try: + old_umask = os.umask(0o077) + self._box.close() + os.unlink(self._path) + self._box = mailbox.mbox(self._path, create=True) + self._box.add('') + self._box.close() + finally: + os.umask(old_umask) + + st = os.stat(self._path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. + + def test_terminating_newline(self): + message = email.message.Message() + message['From'] = 'john@example.com' + message.set_payload('No newline at the end') + i = self._box.add(message) + + # A newline should have been appended to the payload + message = self._box.get(i) + self.assertEqual(message.get_payload(), 'No newline at the end\n') + + def test_message_separator(self): + # Check there's always a single blank line after each message + self._box.add('From: foo\n\n0') # No newline at the end + with open(self._path, encoding='utf-8') as f: + data = f.read() + self.assertEndsWith(data, '0\n\n') + + self._box.add('From: foo\n\n0\n') # Newline at the end + with open(self._path, encoding='utf-8') as f: + data = f.read() + self.assertEndsWith(data, '0\n\n') + + +class TestMMDF(_TestMboxMMDF, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.MMDF(path, factory) + + +class TestMH(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.MH(path, factory) + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(self._path), ['.mh_sequences']) + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + def dummy_factory (s): + return None + self._box = self._factory(self._path, dummy_factory) + + new_folder = self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, 'foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + # Test for bug #1569790: verify that folders returned by .get_folder() + # use the same factory function. + self.assertIs(new_folder._factory, self._box._factory) + self.assertIs(folder0._factory, self._box._factory) + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_sequences(self): + # Get and set sequences + self.assertEqual(self._box.get_sequences(), {}) + msg0 = mailbox.MHMessage(self._template % 0) + msg0.add_sequence('foo') + key0 = self._box.add(msg0) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + msg1 = mailbox.MHMessage(self._template % 1) + msg1.set_sequences(['bar', 'replied', 'foo']) + key1 = self._box.add(msg1) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0, key1], 'bar':[key1], 'replied':[key1]}) + msg0.set_sequences(['flagged']) + self._box[key0] = msg0 + self.assertEqual(self._box.get_sequences(), + {'foo':[key1], 'bar':[key1], 'replied':[key1], + 'flagged':[key0]}) + self._box.remove(key1) + self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_lock_unlock_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + box.lock() + box.unlock() + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_issue2625(self): + msg0 = mailbox.MHMessage(self._template % 0) + msg0.add_sequence('foo') + key0 = self._box.add(msg0) + refmsg0 = self._box.get_message(key0) + + def test_issue7627(self): + msg0 = mailbox.MHMessage(self._template % 0) + key0 = self._box.add(msg0) + self._box.lock() + self._box.remove(key0) + self._box.unlock() + + def test_pack(self): + # Pack the contents of the mailbox + msg0 = mailbox.MHMessage(self._template % 0) + msg1 = mailbox.MHMessage(self._template % 1) + msg2 = mailbox.MHMessage(self._template % 2) + msg3 = mailbox.MHMessage(self._template % 3) + msg0.set_sequences(['foo', 'unseen']) + msg1.set_sequences(['foo']) + msg2.set_sequences(['foo', 'flagged']) + msg3.set_sequences(['foo', 'bar', 'replied']) + key0 = self._box.add(msg0) + key1 = self._box.add(msg1) + key2 = self._box.add(msg2) + key3 = self._box.add(msg3) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0,key1,key2,key3], 'unseen':[key0], + 'flagged':[key2], 'bar':[key3], 'replied':[key3]}) + self._box.remove(key2) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0,key1,key3], 'unseen':[key0], 'bar':[key3], + 'replied':[key3]}) + self._box.pack() + self.assertEqual(self._box.keys(), [1, 2, 3]) + key0 = key0 + key1 = key0 + 1 + key2 = key1 + 1 + self.assertEqual(self._box.get_sequences(), + {'foo':[1, 2, 3], 'unseen':[1], 'bar':[3], 'replied':[3]}) + + # Test case for packing while holding the mailbox locked. + key0 = self._box.add(msg1) + key1 = self._box.add(msg1) + key2 = self._box.add(msg1) + key3 = self._box.add(msg1) + + self._box.remove(key0) + self._box.remove(key2) + self._box.lock() + self._box.pack() + self._box.unlock() + self.assertEqual(self._box.get_sequences(), + {'foo':[1, 2, 3, 4, 5], + 'unseen':[1], 'bar':[3], 'replied':[3]}) + + def _get_lock_path(self): + return os.path.join(self._path, '.mh_sequences.lock') + + +class TestBabyl(_TestSingleFile, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Babyl(path, factory) + + def assertMailboxEmpty(self): + with open(self._path, 'rb') as f: + self.assertEqual(f.readlines(), []) + + def tearDown(self): + super().tearDown() + self._box.close() + self._delete_recursively(self._path) + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): + os_helper.unlink(lock_remnant) + + def test_labels(self): + # Get labels from the mailbox + self.assertEqual(self._box.get_labels(), []) + msg0 = mailbox.BabylMessage(self._template % 0) + msg0.add_label('foo') + key0 = self._box.add(msg0) + self.assertEqual(self._box.get_labels(), ['foo']) + msg1 = mailbox.BabylMessage(self._template % 1) + msg1.set_labels(['bar', 'answered', 'foo']) + key1 = self._box.add(msg1) + self.assertEqual(set(self._box.get_labels()), set(['foo', 'bar'])) + msg0.set_labels(['blah', 'filed']) + self._box[key0] = msg0 + self.assertEqual(set(self._box.get_labels()), + set(['foo', 'bar', 'blah'])) + self._box.remove(key1) + self.assertEqual(set(self._box.get_labels()), set(['blah'])) + + +class FakeFileLikeObject: + + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +class FakeMailBox(mailbox.Mailbox): + + def __init__(self): + mailbox.Mailbox.__init__(self, '', lambda file: None) + self.files = [FakeFileLikeObject() for i in range(10)] + + def get_file(self, key): + return self.files[key] + + +class TestFakeMailBox(unittest.TestCase): + + def test_closing_fd(self): + box = FakeMailBox() + for i in range(10): + self.assertFalse(box.files[i].closed) + for i in range(10): + box[i] + for i in range(10): + self.assertTrue(box.files[i].closed) + + +class TestMessage(TestBase, unittest.TestCase): + + _factory = mailbox.Message # Overridden by subclasses to reuse tests + + def setUp(self): + self._path = os_helper.TESTFN + + def tearDown(self): + self._delete_recursively(self._path) + + def test_initialize_with_eMM(self): + # Initialize based on email.message.Message instance + eMM = email.message_from_string(_sample_message) + msg = self._factory(eMM) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_string(self): + # Initialize based on string + msg = self._factory(_sample_message) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_file(self): + # Initialize based on contents of file + with open(self._path, 'w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + msg = self._factory(f) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_binary_file(self): + # Initialize based on contents of binary file + with open(self._path, 'wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + msg = self._factory(f) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_nothing(self): + # Initialize without arguments + msg = self._factory() + self._post_initialize_hook(msg) + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + self.assertIsInstance(msg, self._factory) + self.assertEqual(msg.keys(), []) + self.assertFalse(msg.is_multipart()) + self.assertIsNone(msg.get_payload()) + + def test_initialize_incorrectly(self): + # Initialize with invalid argument + self.assertRaises(TypeError, lambda: self._factory(object())) + + def test_all_eMM_attributes_exist(self): + # Issue 12537 + eMM = email.message_from_string(_sample_message) + msg = self._factory(_sample_message) + for attr in eMM.__dict__: + self.assertIn(attr, msg.__dict__, + '{} attribute does not exist'.format(attr)) + + def test_become_message(self): + # Take on the state of another message + eMM = email.message_from_string(_sample_message) + msg = self._factory() + msg._become_message(eMM) + self._check_sample(msg) + + def test_explain_to(self): + # Copy self's format-specific data to other message formats. + # This test is superficial; better ones are in TestMessageConversion. + msg = self._factory() + for class_ in self.all_mailbox_types: + other_msg = class_() + msg._explain_to(other_msg) + other_msg = email.message.Message() + self.assertRaises(TypeError, lambda: msg._explain_to(other_msg)) + + def _post_initialize_hook(self, msg): + # Overridden by subclasses to check extra things after initialization + pass + + +class TestMaildirMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.MaildirMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._subdir, 'new') + self.assertEqual(msg._info, '') + + def test_subdir(self): + # Use get_subdir() and set_subdir() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_subdir(), 'new') + msg.set_subdir('cur') + self.assertEqual(msg.get_subdir(), 'cur') + msg.set_subdir('new') + self.assertEqual(msg.get_subdir(), 'new') + self.assertRaises(ValueError, lambda: msg.set_subdir('tmp')) + self.assertEqual(msg.get_subdir(), 'new') + msg.set_subdir('new') + self.assertEqual(msg.get_subdir(), 'new') + self._check_sample(msg) + + def test_flags(self): + # Use get_flags(), set_flags(), add_flag(), remove_flag() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_subdir(), 'new') + msg.set_flags('F') + self.assertEqual(msg.get_subdir(), 'new') + self.assertEqual(msg.get_flags(), 'F') + msg.set_flags('SDTP') + self.assertEqual(msg.get_flags(), 'DPST') + msg.add_flag('FT') + self.assertEqual(msg.get_flags(), 'DFPST') + msg.remove_flag('TDRP') + self.assertEqual(msg.get_flags(), 'FS') + self.assertEqual(msg.get_subdir(), 'new') + self._check_sample(msg) + + def test_date(self): + # Use get_date() and set_date() + msg = mailbox.MaildirMessage(_sample_message) + self.assertLess(abs(msg.get_date() - time.time()), 60) + msg.set_date(0.0) + self.assertEqual(msg.get_date(), 0.0) + + def test_info(self): + # Use get_info() and set_info() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_info(), '') + msg.set_info('1,foo=bar') + self.assertEqual(msg.get_info(), '1,foo=bar') + self.assertRaises(TypeError, lambda: msg.set_info(None)) + self._check_sample(msg) + + def test_info_and_flags(self): + # Test interaction of info and flag methods + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_info(), '') + msg.set_flags('SF') + self.assertEqual(msg.get_flags(), 'FS') + self.assertEqual(msg.get_info(), '2,FS') + msg.set_info('1,') + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_info(), '1,') + msg.remove_flag('RPT') + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_info(), '1,') + msg.add_flag('D') + self.assertEqual(msg.get_flags(), 'D') + self.assertEqual(msg.get_info(), '2,D') + self._check_sample(msg) + + +class _TestMboxMMDFMessage: + + _factory = mailbox._mboxMMDFMessage + + def _post_initialize_hook(self, msg): + self._check_from(msg) + + def test_initialize_with_unixfrom(self): + # Initialize with a message that already has a _unixfrom attribute + msg = mailbox.Message(_sample_message) + msg.set_unixfrom('From foo@bar blah') + msg = mailbox.mboxMessage(msg) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') + + def test_from(self): + # Get and set "From " line + msg = mailbox.mboxMessage(_sample_message) + self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('foo bar') + self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('foo@bar', True) + self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('blah@temp', time.localtime()) + self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) + + def test_flags(self): + # Use get_flags(), set_flags(), add_flag(), remove_flag() + msg = mailbox.mboxMessage(_sample_message) + self.assertEqual(msg.get_flags(), '') + msg.set_flags('F') + self.assertEqual(msg.get_flags(), 'F') + msg.set_flags('XODR') + self.assertEqual(msg.get_flags(), 'RODX') + msg.add_flag('FA') + self.assertEqual(msg.get_flags(), 'RODFAX') + msg.remove_flag('FDXA') + self.assertEqual(msg.get_flags(), 'RO') + self._check_sample(msg) + + def _check_from(self, msg, sender=None): + # Check contents of "From " line + if sender is None: + sender = "MAILER-DAEMON" + self.assertIsNotNone(re.match( + sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:\d{2} \d{4}", + msg.get_from())) + + +class TestMboxMessage(_TestMboxMMDFMessage, TestMessage): + + _factory = mailbox.mboxMessage + + +class TestMHMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.MHMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._sequences, []) + + def test_sequences(self): + # Get, set, join, and leave sequences + msg = mailbox.MHMessage(_sample_message) + self.assertEqual(msg.get_sequences(), []) + msg.set_sequences(['foobar']) + self.assertEqual(msg.get_sequences(), ['foobar']) + msg.set_sequences([]) + self.assertEqual(msg.get_sequences(), []) + msg.add_sequence('unseen') + self.assertEqual(msg.get_sequences(), ['unseen']) + msg.add_sequence('flagged') + self.assertEqual(msg.get_sequences(), ['unseen', 'flagged']) + msg.add_sequence('flagged') + self.assertEqual(msg.get_sequences(), ['unseen', 'flagged']) + msg.remove_sequence('unseen') + self.assertEqual(msg.get_sequences(), ['flagged']) + msg.add_sequence('foobar') + self.assertEqual(msg.get_sequences(), ['flagged', 'foobar']) + msg.remove_sequence('replied') + self.assertEqual(msg.get_sequences(), ['flagged', 'foobar']) + msg.set_sequences(['foobar', 'replied']) + self.assertEqual(msg.get_sequences(), ['foobar', 'replied']) + + +class TestBabylMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.BabylMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._labels, []) + + def test_labels(self): + # Get, set, join, and leave labels + msg = mailbox.BabylMessage(_sample_message) + self.assertEqual(msg.get_labels(), []) + msg.set_labels(['foobar']) + self.assertEqual(msg.get_labels(), ['foobar']) + msg.set_labels([]) + self.assertEqual(msg.get_labels(), []) + msg.add_label('filed') + self.assertEqual(msg.get_labels(), ['filed']) + msg.add_label('resent') + self.assertEqual(msg.get_labels(), ['filed', 'resent']) + msg.add_label('resent') + self.assertEqual(msg.get_labels(), ['filed', 'resent']) + msg.remove_label('filed') + self.assertEqual(msg.get_labels(), ['resent']) + msg.add_label('foobar') + self.assertEqual(msg.get_labels(), ['resent', 'foobar']) + msg.remove_label('unseen') + self.assertEqual(msg.get_labels(), ['resent', 'foobar']) + msg.set_labels(['foobar', 'answered']) + self.assertEqual(msg.get_labels(), ['foobar', 'answered']) + + def test_visible(self): + # Get, set, and update visible headers + msg = mailbox.BabylMessage(_sample_message) + visible = msg.get_visible() + self.assertEqual(visible.keys(), []) + self.assertIsNone(visible.get_payload()) + visible['User-Agent'] = 'FooBar 1.0' + visible['X-Whatever'] = 'Blah' + self.assertEqual(msg.get_visible().keys(), []) + msg.set_visible(visible) + visible = msg.get_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) + self.assertEqual(visible['User-Agent'], 'FooBar 1.0') + self.assertEqual(visible['X-Whatever'], 'Blah') + self.assertIsNone(visible.get_payload()) + msg.update_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) + self.assertIsNone(visible.get_payload()) + visible = msg.get_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To', + 'Subject']) + for header in ('User-Agent', 'Date', 'From', 'To', 'Subject'): + self.assertEqual(visible[header], msg[header]) + + +class TestMMDFMessage(_TestMboxMMDFMessage, TestMessage): + + _factory = mailbox.MMDFMessage + + +class TestMessageConversion(TestBase, unittest.TestCase): + + def test_plain_to_x(self): + # Convert Message to all formats + for class_ in self.all_mailbox_types: + msg_plain = mailbox.Message(_sample_message) + msg = class_(msg_plain) + self._check_sample(msg) + + def test_x_to_plain(self): + # Convert all formats to Message + for class_ in self.all_mailbox_types: + msg = class_(_sample_message) + msg_plain = mailbox.Message(msg) + self._check_sample(msg_plain) + + def test_x_from_bytes(self): + # Convert all formats to Message + for class_ in self.all_mailbox_types: + msg = class_(_bytes_sample_message) + self._check_sample(msg) + + def test_x_to_invalid(self): + # Convert all formats to an invalid format + for class_ in self.all_mailbox_types: + self.assertRaises(TypeError, lambda: class_(False)) + + def test_type_specific_attributes_removed_on_conversion(self): + reference = {class_: class_(_sample_message).__dict__ + for class_ in self.all_mailbox_types} + for class1 in self.all_mailbox_types: + for class2 in self.all_mailbox_types: + if class1 is class2: + continue + source = class1(_sample_message) + target = class2(source) + type_specific = [a for a in reference[class1] + if a not in reference[class2]] + for attr in type_specific: + self.assertNotIn(attr, target.__dict__, + "while converting {} to {}".format(class1, class2)) + + def test_maildir_to_maildir(self): + # Convert MaildirMessage to MaildirMessage + msg_maildir = mailbox.MaildirMessage(_sample_message) + msg_maildir.set_flags('DFPRST') + msg_maildir.set_subdir('cur') + date = msg_maildir.get_date() + msg = mailbox.MaildirMessage(msg_maildir) + self._check_sample(msg) + self.assertEqual(msg.get_flags(), 'DFPRST') + self.assertEqual(msg.get_subdir(), 'cur') + self.assertEqual(msg.get_date(), date) + + def test_maildir_to_mboxmmdf(self): + # Convert MaildirMessage to mboxmessage and MMDFMessage + pairs = (('D', ''), ('F', 'F'), ('P', ''), ('R', 'A'), ('S', 'R'), + ('T', 'D'), ('DFPRST', 'RDFA')) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_maildir = mailbox.MaildirMessage(_sample_message) + msg_maildir.set_date(0.0) + for setting, result in pairs: + msg_maildir.set_flags(setting) + msg = class_(msg_maildir) + self.assertEqual(msg.get_flags(), result) + self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % + time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) + msg_maildir.set_subdir('cur') + self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') + + def test_maildir_to_mh(self): + # Convert MaildirMessage to MHMessage + msg_maildir = mailbox.MaildirMessage(_sample_message) + pairs = (('D', ['unseen']), ('F', ['unseen', 'flagged']), + ('P', ['unseen']), ('R', ['unseen', 'replied']), ('S', []), + ('T', ['unseen']), ('DFPRST', ['replied', 'flagged'])) + for setting, result in pairs: + msg_maildir.set_flags(setting) + self.assertEqual(mailbox.MHMessage(msg_maildir).get_sequences(), + result) + + def test_maildir_to_babyl(self): + # Convert MaildirMessage to Babyl + msg_maildir = mailbox.MaildirMessage(_sample_message) + pairs = (('D', ['unseen']), ('F', ['unseen']), + ('P', ['unseen', 'forwarded']), ('R', ['unseen', 'answered']), + ('S', []), ('T', ['unseen', 'deleted']), + ('DFPRST', ['deleted', 'answered', 'forwarded'])) + for setting, result in pairs: + msg_maildir.set_flags(setting) + self.assertEqual(mailbox.BabylMessage(msg_maildir).get_labels(), + result) + + def test_mboxmmdf_to_maildir(self): + # Convert mboxMessage and MMDFMessage to MaildirMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + msg_mboxMMDF.set_from('foo@bar', time.gmtime(0.0)) + pairs = (('R', 'S'), ('O', ''), ('D', 'T'), ('F', 'F'), ('A', 'R'), + ('RODFA', 'FRST')) + for setting, result in pairs: + msg_mboxMMDF.set_flags(setting) + msg = mailbox.MaildirMessage(msg_mboxMMDF) + self.assertEqual(msg.get_flags(), result) + self.assertEqual(msg.get_date(), 0.0) + msg_mboxMMDF.set_flags('O') + self.assertEqual(mailbox.MaildirMessage(msg_mboxMMDF).get_subdir(), + 'cur') + + def test_mboxmmdf_to_mboxmmdf(self): + # Convert mboxMessage and MMDFMessage to mboxMessage and MMDFMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + msg_mboxMMDF.set_flags('RODFA') + msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) + for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg2 = class2_(msg_mboxMMDF) + self.assertEqual(msg2.get_flags(), 'RODFA') + self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) + + def test_mboxmmdf_to_mh(self): + # Convert mboxMessage and MMDFMessage to MHMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + pairs = (('R', []), ('O', ['unseen']), ('D', ['unseen']), + ('F', ['unseen', 'flagged']), + ('A', ['unseen', 'replied']), + ('RODFA', ['replied', 'flagged'])) + for setting, result in pairs: + msg_mboxMMDF.set_flags(setting) + self.assertEqual(mailbox.MHMessage(msg_mboxMMDF).get_sequences(), + result) + + def test_mboxmmdf_to_babyl(self): + # Convert mboxMessage and MMDFMessage to BabylMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = class_(_sample_message) + pairs = (('R', []), ('O', ['unseen']), + ('D', ['unseen', 'deleted']), ('F', ['unseen']), + ('A', ['unseen', 'answered']), + ('RODFA', ['deleted', 'answered'])) + for setting, result in pairs: + msg.set_flags(setting) + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result) + + def test_mh_to_maildir(self): + # Convert MHMessage to MaildirMessage + pairs = (('unseen', ''), ('replied', 'RS'), ('flagged', 'FS')) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result) + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'FR') + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + + def test_mh_to_mboxmmdf(self): + # Convert MHMessage to mboxMessage and MMDFMessage + pairs = (('unseen', 'O'), ('replied', 'ROA'), ('flagged', 'ROF')) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), result) + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), 'OFA') + + def test_mh_to_mh(self): + # Convert MHMessage to MHMessage + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), + ['unseen', 'replied', 'flagged']) + + def test_mh_to_babyl(self): + # Convert MHMessage to BabylMessage + pairs = (('unseen', ['unseen']), ('replied', ['answered']), + ('flagged', [])) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result) + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), + ['unseen', 'answered']) + + def test_babyl_to_maildir(self): + # Convert BabylMessage to MaildirMessage + pairs = (('unseen', ''), ('deleted', 'ST'), ('filed', 'S'), + ('answered', 'RS'), ('forwarded', 'PS'), ('edited', 'S'), + ('resent', 'PS')) + for setting, result in pairs: + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result) + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'PRT') + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + + def test_babyl_to_mboxmmdf(self): + # Convert BabylMessage to mboxMessage and MMDFMessage + pairs = (('unseen', 'O'), ('deleted', 'ROD'), ('filed', 'RO'), + ('answered', 'ROA'), ('forwarded', 'RO'), ('edited', 'RO'), + ('resent', 'RO')) + for setting, result in pairs: + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(class_(msg).get_flags(), result) + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), 'ODA') + + def test_babyl_to_mh(self): + # Convert BabylMessage to MHMessage + pairs = (('unseen', ['unseen']), ('deleted', []), ('filed', []), + ('answered', ['replied']), ('forwarded', []), ('edited', []), + ('resent', [])) + for setting, result in pairs: + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), result) + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), + ['unseen', 'replied']) + + def test_babyl_to_babyl(self): + # Convert BabylMessage to BabylMessage + msg = mailbox.BabylMessage(_sample_message) + msg.update_visible() + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + msg2 = mailbox.BabylMessage(msg) + self.assertEqual(msg2.get_labels(), ['unseen', 'deleted', 'filed', + 'answered', 'forwarded', 'edited', + 'resent']) + self.assertEqual(msg.get_visible().keys(), msg2.get_visible().keys()) + for key in msg.get_visible().keys(): + self.assertEqual(msg.get_visible()[key], msg2.get_visible()[key]) + + +class TestProxyFileBase(TestBase): + + def _test_read(self, proxy): + # Read by byte + proxy.seek(0) + self.assertEqual(proxy.read(), b'bar') + proxy.seek(1) + self.assertEqual(proxy.read(), b'ar') + proxy.seek(0) + self.assertEqual(proxy.read(2), b'ba') + proxy.seek(1) + self.assertEqual(proxy.read(-1), b'ar') + proxy.seek(2) + self.assertEqual(proxy.read(1000), b'r') + + def _test_readline(self, proxy): + # Read by line + linesep = os.linesep.encode() + proxy.seek(0) + self.assertEqual(proxy.readline(), b'foo' + linesep) + self.assertEqual(proxy.readline(), b'bar' + linesep) + self.assertEqual(proxy.readline(), b'fred' + linesep) + self.assertEqual(proxy.readline(), b'bob') + proxy.seek(2) + self.assertEqual(proxy.readline(), b'o' + linesep) + proxy.seek(6 + 2 * len(os.linesep)) + self.assertEqual(proxy.readline(), b'fred' + linesep) + proxy.seek(6 + 2 * len(os.linesep)) + self.assertEqual(proxy.readline(2), b'fr') + self.assertEqual(proxy.readline(-10), b'ed' + linesep) + + def _test_readlines(self, proxy): + # Read multiple lines + linesep = os.linesep.encode() + proxy.seek(0) + self.assertEqual(proxy.readlines(), [b'foo' + linesep, + b'bar' + linesep, + b'fred' + linesep, b'bob']) + proxy.seek(0) + self.assertEqual(proxy.readlines(2), [b'foo' + linesep]) + proxy.seek(3 + len(linesep)) + self.assertEqual(proxy.readlines(4 + len(linesep)), + [b'bar' + linesep, b'fred' + linesep]) + proxy.seek(3) + self.assertEqual(proxy.readlines(1000), [linesep, b'bar' + linesep, + b'fred' + linesep, b'bob']) + + def _test_iteration(self, proxy): + # Iterate by line + linesep = os.linesep.encode() + proxy.seek(0) + iterator = iter(proxy) + self.assertEqual(next(iterator), b'foo' + linesep) + self.assertEqual(next(iterator), b'bar' + linesep) + self.assertEqual(next(iterator), b'fred' + linesep) + self.assertEqual(next(iterator), b'bob') + self.assertRaises(StopIteration, next, iterator) + + def _test_seek_and_tell(self, proxy): + # Seek and use tell to check position + linesep = os.linesep.encode() + proxy.seek(3) + self.assertEqual(proxy.tell(), 3) + self.assertEqual(proxy.read(len(linesep)), linesep) + proxy.seek(2, 1) + self.assertEqual(proxy.read(1 + len(linesep)), b'r' + linesep) + proxy.seek(-3 - len(linesep), 2) + self.assertEqual(proxy.read(3), b'bar') + proxy.seek(2, 0) + self.assertEqual(proxy.read(), b'o' + linesep + b'bar' + linesep) + proxy.seek(100) + self.assertFalse(proxy.read()) + + def _test_close(self, proxy): + # Close a file + self.assertFalse(proxy.closed) + proxy.close() + self.assertTrue(proxy.closed) + # Issue 11700 subsequent closes should be a no-op. + proxy.close() + self.assertTrue(proxy.closed) + + +class TestProxyFile(TestProxyFileBase, unittest.TestCase): + + def setUp(self): + self._path = os_helper.TESTFN + self._file = open(self._path, 'wb+') + + def tearDown(self): + self._file.close() + self._delete_recursively(self._path) + + def test_initialize(self): + # Initialize and check position + self._file.write(b'foo') + pos = self._file.tell() + proxy0 = mailbox._ProxyFile(self._file) + self.assertEqual(proxy0.tell(), pos) + self.assertEqual(self._file.tell(), pos) + proxy1 = mailbox._ProxyFile(self._file, 0) + self.assertEqual(proxy1.tell(), 0) + self.assertEqual(self._file.tell(), pos) + + def test_read(self): + self._file.write(b'bar') + self._test_read(mailbox._ProxyFile(self._file)) + + def test_readline(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_readline(mailbox._ProxyFile(self._file)) + + def test_readlines(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_readlines(mailbox._ProxyFile(self._file)) + + def test_iteration(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_iteration(mailbox._ProxyFile(self._file)) + + def test_seek_and_tell(self): + self._file.write(bytes('foo%sbar%s' % (os.linesep, os.linesep), 'ascii')) + self._test_seek_and_tell(mailbox._ProxyFile(self._file)) + + def test_close(self): + self._file.write(bytes('foo%sbar%s' % (os.linesep, os.linesep), 'ascii')) + self._test_close(mailbox._ProxyFile(self._file)) + + +class TestPartialFile(TestProxyFileBase, unittest.TestCase): + + def setUp(self): + self._path = os_helper.TESTFN + self._file = open(self._path, 'wb+') + + def tearDown(self): + self._file.close() + self._delete_recursively(self._path) + + def test_initialize(self): + # Initialize and check position + self._file.write(bytes('foo' + os.linesep + 'bar', 'ascii')) + pos = self._file.tell() + proxy = mailbox._PartialFile(self._file, 2, 5) + self.assertEqual(proxy.tell(), 0) + self.assertEqual(self._file.tell(), pos) + + def test_read(self): + self._file.write(bytes('***bar***', 'ascii')) + self._test_read(mailbox._PartialFile(self._file, 3, 6)) + + def test_readline(self): + self._file.write(bytes('!!!!!foo%sbar%sfred%sbob!!!!!' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_readline(mailbox._PartialFile(self._file, 5, + 18 + 3 * len(os.linesep))) + + def test_readlines(self): + self._file.write(bytes('foo%sbar%sfred%sbob?????' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_readlines(mailbox._PartialFile(self._file, 0, + 13 + 3 * len(os.linesep))) + + def test_iteration(self): + self._file.write(bytes('____foo%sbar%sfred%sbob####' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_iteration(mailbox._PartialFile(self._file, 4, + 17 + 3 * len(os.linesep))) + + def test_seek_and_tell(self): + self._file.write(bytes('(((foo%sbar%s$$$' % (os.linesep, os.linesep), 'ascii')) + self._test_seek_and_tell(mailbox._PartialFile(self._file, 3, + 9 + 2 * len(os.linesep))) + + def test_close(self): + self._file.write(bytes('&foo%sbar%s^' % (os.linesep, os.linesep), 'ascii')) + self._test_close(mailbox._PartialFile(self._file, 1, + 6 + 3 * len(os.linesep))) + + +## Start: tests from the original module (for backward compatibility). + +FROM_ = "From some.body@dummy.domain Sat Jul 24 13:43:35 2004\n" +DUMMY_MESSAGE = """\ +From: some.body@dummy.domain +To: me@my.domain +Subject: Simple Test + +This is a dummy message. +""" + +class MaildirTestCase(unittest.TestCase): + + def setUp(self): + # create a new maildir mailbox to work with: + self._dir = os_helper.TESTFN + if os.path.isdir(self._dir): + os_helper.rmtree(self._dir) + elif os.path.isfile(self._dir): + os_helper.unlink(self._dir) + os.mkdir(self._dir) + os.mkdir(os.path.join(self._dir, "cur")) + os.mkdir(os.path.join(self._dir, "tmp")) + os.mkdir(os.path.join(self._dir, "new")) + self._counter = 1 + self._msgfiles = [] + + def tearDown(self): + list(map(os.unlink, self._msgfiles)) + os_helper.rmdir(os.path.join(self._dir, "cur")) + os_helper.rmdir(os.path.join(self._dir, "tmp")) + os_helper.rmdir(os.path.join(self._dir, "new")) + os_helper.rmdir(self._dir) + + def createMessage(self, dir, mbox=False): + t = int(time.time() % 1000000) + pid = self._counter + self._counter += 1 + filename = ".".join((str(t), str(pid), "myhostname", "mydomain")) + tmpname = os.path.join(self._dir, "tmp", filename) + newname = os.path.join(self._dir, dir, filename) + with open(tmpname, "w", encoding="utf-8") as fp: + self._msgfiles.append(tmpname) + if mbox: + fp.write(FROM_) + fp.write(DUMMY_MESSAGE) + try: + os.link(tmpname, newname) + except (AttributeError, PermissionError): + with open(newname, "w") as fp: + fp.write(DUMMY_MESSAGE) + self._msgfiles.append(newname) + return tmpname + + def test_empty_maildir(self): + """Test an empty maildir mailbox""" + # Test for regression on bug #117490: + # Make sure the boxes attribute actually gets set. + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertHasAttr(self.mbox, "boxes") + #self.assertEqual(len(self.mbox.boxes), 0) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_cur(self): + self.createMessage("cur") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 1) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_new(self): + self.createMessage("new") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 1) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_both(self): + self.createMessage("cur") + self.createMessage("new") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 2) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + +## End: tests from the original module (for backward compatibility). + + +_sample_message = """\ +Return-Path: <gkj@gregorykjohnson.com> +X-Original-To: gkj+person@localhost +Delivered-To: gkj+person@localhost +Received: from localhost (localhost [127.0.0.1]) + by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17 + for <gkj+person@localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT) +Delivered-To: gkj@sundance.gregorykjohnson.com +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.5) + for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT) +Received: from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) + by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746 + for <gkj@gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT) +Received: by andy.gregorykjohnson.com (Postfix, from userid 1000) + id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT) +Date: Wed, 13 Jul 2005 17:23:11 -0400 +From: "Gregory K. Johnson" <gkj@gregorykjohnson.com> +To: gkj@gregorykjohnson.com +Subject: Sample message +Message-ID: <20050713212311.GC4701@andy.gregorykjohnson.com> +Mime-Version: 1.0 +Content-Type: multipart/mixed; boundary="NMuMz9nt05w80d4+" +Content-Disposition: inline +User-Agent: Mutt/1.5.9i + + +--NMuMz9nt05w80d4+ +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline + +This is a sample message. + +-- +Gregory K. Johnson + +--NMuMz9nt05w80d4+ +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="text.gz" +Content-Transfer-Encoding: base64 + +H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs +3FYlAAAA + +--NMuMz9nt05w80d4+-- +""" + +_bytes_sample_message = _sample_message.encode('ascii') + +_sample_headers = [ + ("Return-Path", "<gkj@gregorykjohnson.com>"), + ("X-Original-To", "gkj+person@localhost"), + ("Delivered-To", "gkj+person@localhost"), + ("Received", """from localhost (localhost [127.0.0.1]) + by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17 + for <gkj+person@localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Delivered-To", "gkj@sundance.gregorykjohnson.com"), + ("Received", """from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.5) + for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Received", """from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) + by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746 + for <gkj@gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Received", """by andy.gregorykjohnson.com (Postfix, from userid 1000) + id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Date", "Wed, 13 Jul 2005 17:23:11 -0400"), + ("From", """"Gregory K. Johnson" <gkj@gregorykjohnson.com>"""), + ("To", "gkj@gregorykjohnson.com"), + ("Subject", "Sample message"), + ("Mime-Version", "1.0"), + ("Content-Type", """multipart/mixed; boundary="NMuMz9nt05w80d4+\""""), + ("Content-Disposition", "inline"), + ("User-Agent", "Mutt/1.5.9i"), +] + +_sample_payloads = ("""This is a sample message. + +-- +Gregory K. Johnson +""", +"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs +3FYlAAAA +""") + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, mailbox, + not_exported={"linesep", "fcntl"}) + + +def tearDownModule(): + support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 2161e06b2f2..142b45f6a46 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -35,7 +35,7 @@ def test_ints(self): self.helper(expected) n = n >> 1 - @unittest.skip("TODO: RUSTPYTHON; hang") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_int64(self): # Simulate int marshaling with TYPE_INT64. maxint64 = (1 << 63) - 1 @@ -232,7 +232,7 @@ def check(s): self.assertRaises(ValueError, marshal.loads, s) run_tests(2**20, check) - @unittest.skip("TODO: RUSTPYTHON; segfault") + @unittest.expectedFailure # TODO: RUSTPYTHON; segfault def test_recursion_limit(self): # Create a deeply nested structure. head = last = [] diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 1a4d257586b..d14336f8bac 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -573,6 +573,8 @@ def testFloor(self): #self.assertEqual(math.ceil(NINF), NINF) #self.assertTrue(math.isnan(math.floor(NAN))) + class TestFloorIsNone(float): + __floor__ = None class TestFloor: def __floor__(self): return 42 @@ -588,6 +590,7 @@ class TestBadFloor: self.assertEqual(math.floor(FloatLike(41.9)), 41) self.assertRaises(TypeError, math.floor, TestNoFloor()) self.assertRaises(ValueError, math.floor, TestBadFloor()) + self.assertRaises(TypeError, math.floor, TestFloorIsNone(3.5)) t = TestNoFloor() t.__floor__ = lambda *args: args @@ -1125,6 +1128,15 @@ def __index__(self): with self.assertRaises(TypeError): math.isqrt(value) + @support.bigmemtest(2**32, memuse=0.85) + def test_isqrt_huge(self, size): + if size & 1: + size += 1 + v = 1 << size + w = math.isqrt(v) + self.assertEqual(w.bit_length(), size // 2 + 1) + self.assertEqual(w.bit_count(), 1) + def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) @@ -1272,6 +1284,13 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + @support.bigmemtest(2**32, memuse=0.2) + def test_log_huge_integer(self, size): + v = 1 << size + self.assertAlmostEqual(math.log2(v), size) + self.assertAlmostEqual(math.log(v), size * 0.6931471805599453) + self.assertAlmostEqual(math.log10(v), size * 0.3010299956639812) + def testSumProd(self): sumprod = math.sumprod Decimal = decimal.Decimal @@ -1380,7 +1399,6 @@ def test_sumprod_accuracy(self): self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) - @unittest.skip("TODO: RUSTPYTHON, Taking a few minutes.") @support.requires_resource('cpu') def test_sumprod_stress(self): sumprod = math.sumprod @@ -2020,7 +2038,6 @@ def test_exceptions(self): else: self.fail("sqrt(-1) didn't raise ValueError") - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def test_testfile(self): # Some tests need to be skipped on ancient OS X versions. @@ -2495,6 +2512,46 @@ def test_input_exceptions(self): self.assertRaises(TypeError, math.atan2, 1.0) self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + def test_exception_messages(self): + x = -1.1 + with self.assertRaisesRegex(ValueError, + f"expected a nonnegative input, got {x}"): + math.sqrt(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(123, x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x, 123) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log2(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log10(x) + x = decimal.Decimal('-1.1') + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + x = fractions.Fraction(1, 10**400) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {float(x)}"): + math.log(x) + x = -123 + with self.assertRaisesRegex(ValueError, + "expected a positive input$"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a noninteger or positive integer, got {x}"): + math.gamma(x) + x = 1.0 + with self.assertRaisesRegex(ValueError, + f"expected a number between -1 and 1, got {x}"): + math.atanh(x) + # Custom assertions. def assertIsNaN(self, value): @@ -2724,6 +2781,9 @@ def test_fma_infinities(self): or (sys.platform == "android" and platform.machine() == "x86_64") or support.linked_to_musl(), # gh-131032 f"this platform doesn't implement IEE 754-2008 properly") + # gh-131032: musl is fixed but the fix is not yet released; when the fixed + # version is known change this to: + # or support.linked_to_musl() < (1, <m>, <p>) def test_fma_zero_result(self): nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 61d9b180e26..00f646e5a94 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -6,10 +6,12 @@ import unittest from test import support +import gc import io import _pyio as pyio import pickle import sys +import weakref class IntLike: def __init__(self, num): @@ -52,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -263,8 +271,8 @@ def test_iterator(self): memio = self.ioclass(buf * 10) self.assertEqual(iter(memio), memio) - self.assertTrue(hasattr(memio, '__iter__')) - self.assertTrue(hasattr(memio, '__next__')) + self.assertHasAttr(memio, '__iter__') + self.assertHasAttr(memio, '__next__') i = 0 for line in memio: self.assertEqual(line, buf) @@ -463,6 +471,40 @@ def test_getbuffer(self): memio.close() self.assertRaises(ValueError, memio.getbuffer) + def test_getbuffer_empty(self): + memio = self.ioclass() + buf = memio.getbuffer() + self.assertEqual(bytes(buf), b"") + # Trying to change the size of the BytesIO while a buffer is exported + # raises a BufferError. + self.assertRaises(BufferError, memio.write, b'x') + buf2 = memio.getbuffer() + self.assertRaises(BufferError, memio.write, b'x') + buf.release() + self.assertRaises(BufferError, memio.write, b'x') + buf2.release() + memio.write(b'x') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <memory at 0xbb894d200> is not None + def test_getbuffer_gc_collect(self): + memio = self.ioclass(b"1234567890") + buf = memio.getbuffer() + memiowr = weakref.ref(memio) + bufwr = weakref.ref(buf) + # Create a reference loop. + a = [buf] + a.append(a) + # The Python implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + del memio + del buf + del a + # The C implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + gc.collect() + self.assertIsNone(memiowr()) + self.assertIsNone(bufwr()) + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) @@ -517,6 +559,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() @@ -538,6 +588,75 @@ def test_issue5449(self): self.ioclass(initial_bytes=buf) self.assertRaises(TypeError, self.ioclass, buf, foo=None) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.write, B()) + + # Prevent crashes when memio.write() or memio.writelines() + # concurrently mutates (e.g., closes or exports) 'memio'. + # See: https://github.com/python/cpython/issues/143378. + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_writelines_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.writelines, [B()]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.write, B()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_writelines_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.writelines, [B()]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_mutating_buffer(self): + # Test that buffer is exported only once during write(). + # See: https://github.com/python/cpython/issues/143602. + class B: + count = 0 + def __buffer__(self, flags): + self.count += 1 + if self.count == 1: + return memoryview(b"AAA") + else: + return memoryview(b"BBBBBBBBB") + + memio = self.ioclass(b'0123456789') + memio.seek(2) + b = B() + n = memio.write(b) + + self.assertEqual(b.count, 1) + self.assertEqual(n, 3) + self.assertEqual(memio.getvalue(), b"01AAA56789") + self.assertEqual(memio.tell(), 5) + class TextIOTestMixin: @@ -724,67 +843,6 @@ class CBytesIOTest(PyBytesIOTest): ioclass = io.BytesIO UnsupportedOperation = io.UnsupportedOperation - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_array(self): - super().test_bytes_array() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flags(self): - super().test_flags() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_getbuffer(self): - super().test_getbuffer() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_init(self): - super().test_init() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5449(self): - super().test_issue5449() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - - def test_read(self): - super().test_read() - - def test_readline(self): - super().test_readline() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_seek(self): - super().test_seek() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_subclassing(self): - super().test_subclassing() - - def test_truncate(self): - super().test_truncate() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write(self): - super().test_write() - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -796,8 +854,7 @@ def test_getstate(self): memio.close() self.assertRaises(ValueError, memio.__getstate__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'bytes' but 'bytearray' found. def test_setstate(self): # This checks whether __setstate__ does proper input validation. memio = self.ioclass() @@ -829,7 +886,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) @@ -870,86 +927,25 @@ def test_cow_mutable(self): memio = self.ioclass(ba) self.assertEqual(sys.getrefcount(ba), old_rc) -class CStringIOTest(PyStringIOTest): - ioclass = io.StringIO - UnsupportedOperation = io.UnsupportedOperation - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_detach(self): - super().test_detach() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable def test_flags(self): - super().test_flags() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_init(self): - super().test_init() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5265(self): - super().test_issue5265() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_cr(self): - super().test_newline_cr() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_crlf(self): - super().test_newline_crlf() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_empty(self): - super().test_newline_empty() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_none(self): - super().test_newline_none() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_property(self): - super().test_newlines_property() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - - def test_read(self): - super().test_read() - - def test_readline(self): - super().test_readline() + return super().test_flags() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write + def test_write(self): + return super().test_write() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u64 def test_seek(self): - super().test_seek() - - def test_textio_properties(self): - super().test_textio_properties() + return super().test_seek() - def test_truncate(self): - super().test_truncate() +class CStringIOTest(PyStringIOTest): + ioclass = io.StringIO + UnsupportedOperation = io.UnsupportedOperation # XXX: For the Python version of io.StringIO, this is highly # dependent on the encoding used for the underlying buffer. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 8 != 2 def test_widechar(self): buf = self.buftype("\U0002030a\U00020347") memio = self.ioclass(buf) @@ -962,8 +958,6 @@ def test_widechar(self): self.assertEqual(memio.tell(), len(buf) * 2) self.assertEqual(memio.getvalue(), buf + buf) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -976,8 +970,7 @@ def test_getstate(self): memio.close() self.assertRaises(ValueError, memio.__getstate__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by __setstate__ def test_setstate(self): # This checks whether __setstate__ does proper input validation. memio = self.ioclass() @@ -995,57 +988,49 @@ def test_setstate(self): memio.close() self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) + @unittest.expectedFailure # TODO: RUSTPYTHON; + + def test_issue5265(self): + return super().test_issue5265() -class CStringIOPickleTest(PyStringIOPickleTest): - UnsupportedOperation = io.UnsupportedOperation + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ + def test_newline_empty(self): + return super().test_newline_empty() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5265(self): - super().test_issue5265() + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ + def test_newline_none(self): + return super().test_newline_none() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_cr(self): - super().test_newline_cr() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek + def test_relative_seek(self): + return super().test_relative_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_crlf(self): - super().test_newline_crlf() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable + def test_flags(self): + return super().test_flags() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_default(self): - super().test_newline_default() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'detach' + def test_detach(self): + return super().test_detach() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_empty(self): - super().test_newline_empty() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? + def test_newlines_property(self): + return super().test_newlines_property() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_lf(self): - super().test_newline_lf() + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u64 + def test_seek(self): + return super().test_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_none(self): - super().test_newline_none() + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_cr(self): + return super().test_newline_cr() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_property(self): - super().test_newlines_property() + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_crlf(self): + return super().test_newline_crlf() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() - def test_textio_properties(self): - super().test_textio_properties() +class CStringIOPickleTest(PyStringIOPickleTest): + UnsupportedOperation = io.UnsupportedOperation class ioclass(io.StringIO): def __new__(cls, *args, **kwargs): @@ -1053,6 +1038,34 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass + @unittest.expectedFailure # TODO: RUSTPYTHON; + + def test_issue5265(self): + return super().test_issue5265() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ + def test_newline_empty(self): + return super().test_newline_empty() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ + def test_newline_none(self): + return super().test_newline_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek + def test_relative_seek(self): + return super().test_relative_seek() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? + def test_newlines_property(self): + return super().test_newlines_property() + + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_cr(self): + return super().test_newline_cr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_crlf(self): + return super().test_newline_crlf() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 5ab9441da4f..891c4d76745 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -13,8 +13,15 @@ import io import copy import pickle +import struct -from test.support import import_helper +from itertools import product +from test import support +from test.support import import_helper, threading_helper + + +class MyObject: + pass class AbstractMemoryTests: @@ -53,12 +60,53 @@ def test_getitem(self): for tp in self._types: self.check_getitem_with_type(tp) + def test_index(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) # may be a sub-view + l = m.tolist() + k = 2 * len(self._source) + + for chi in self._source: + if chi in l: + self.assertEqual(m.index(chi), l.index(chi)) + else: + self.assertRaises(ValueError, m.index, chi) + + for start, stop in product(range(-k, k), range(-k, k)): + index = -1 + try: + index = l.index(chi, start, stop) + except ValueError: + pass + + if index == -1: + self.assertRaises(ValueError, m.index, chi, start, stop) + else: + self.assertEqual(m.index(chi, start, stop), index) + def test_iter(self): for tp in self._types: b = tp(self._source) m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_count(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + l = m.tolist() + for ch in list(m): + self.assertEqual(m.count(ch), l.count(ch)) + + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # may be sliced + l = m.tolist() + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), l.count(ord('a'))) + self.assertEqual(m.count(ord('b')), l.count(ord('b'))) + self.assertEqual(m.count(ord('c')), l.count(ord('c'))) + def test_setitem_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") @@ -191,16 +239,12 @@ def check_attributes_with_type(self, tp): self.assertEqual(m.suboffsets, ()) return m - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attributes_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") m = self.check_attributes_with_type(self.ro_type) self.assertEqual(m.readonly, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attributes_writable(self): if not self.rw_type: self.skipTest("no writable type to test") @@ -231,8 +275,6 @@ def __init__(self, base): self.m = memoryview(base) class MySource(tp): pass - class MyObject: - pass # Create a reference cycle through a memoryview object. # This exercises mbuf_clear(). @@ -345,6 +387,21 @@ def test_hash_writable(self): m = self._view(b) self.assertRaises(ValueError, hash, m) + @unittest.expectedFailure # TODO: RUSTPYTHON; re-entrant buffer release not detected + def test_hash_use_after_free(self): + # Prevent crash in memoryview(v).__hash__ with re-entrant v.__hash__. + # Regression test for https://github.com/python/cpython/issues/142664. + class E(array.array): + def __hash__(self): + mv.release() + self.clear() + return 123 + + v = E('B', b'A' * 4096) + mv = memoryview(v).toreadonly() # must be read-only for hash() + self.assertRaises(BufferError, hash, mv) + self.assertRaises(BufferError, mv.__hash__) + def test_weakref(self): # Check memoryviews are weakrefable for tp in self._types: @@ -400,6 +457,21 @@ def test_issue22668(self): self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") + @unittest.expectedFailure # TODO: RUSTPYTHON; re-entrant buffer release not detected + def test_hex_use_after_free(self): + # Prevent UAF in memoryview.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'A' * 1024) + mv = memoryview(ba) + + class S(bytes): + def __len__(self): + mv.release() + ba.clear() + return 1 + + self.assertRaises(BufferError, mv.hex, S(b':')) + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. @@ -439,6 +511,18 @@ def _view(self, obj): def _check_contents(self, tp, obj, contents): self.assertEqual(obj, tp(contents)) + def test_count(self): + super().test_count() + for tp in self._types: + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # should not be sliced + self.assertEqual(len(b), len(m)) + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), 5) + self.assertEqual(m.count(ord('b')), 0) + self.assertEqual(m.count(ord('c')), 3) + + class BaseMemorySliceTests: source_bytes = b"XabcdefY" @@ -472,11 +556,6 @@ def _check_contents(self, tp, obj, contents): class BytesMemoryviewTest(unittest.TestCase, BaseMemoryviewTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() - def test_constructor(self): for tp in self._types: ob = tp(self._source) @@ -487,6 +566,10 @@ def test_constructor(self): self.assertRaises(TypeError, memoryview, argument=ob) self.assertRaises(TypeError, memoryview, ob, argument=True) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ecb1920> + def test_gc(self): + return super().test_gc() + class ArrayMemoryviewTest(unittest.TestCase, BaseMemoryviewTests, BaseArrayMemoryTests): @@ -501,24 +584,24 @@ def test_array_assign(self): class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() pass + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ecb13e0> + def test_gc(self): + return super().test_gc() + class ArrayMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseArrayMemoryTests): pass class BytesMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() pass + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ddca1c0> + def test_gc(self): + return super().test_gc() + class ArrayMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseArrayMemoryTests): pass @@ -544,6 +627,14 @@ def test_ctypes_cast(self): m[2:] = memoryview(p6).cast(format)[2:] self.assertEqual(d.value, 0.6) + def test_half_float(self): + half_data = struct.pack('eee', 0.0, -1.5, 1.5) + float_data = struct.pack('fff', 0.0, -1.5, 1.5) + half_view = memoryview(half_data).cast('e') + float_view = memoryview(float_data).cast('f') + self.assertEqual(half_view.nbytes * 2, float_view.nbytes) + self.assertListEqual(half_view.tolist(), float_view.tolist()) + def test_memoryview_hex(self): # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers. x = b'0' * 200000 @@ -551,6 +642,26 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument sep + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): @@ -562,8 +673,7 @@ def test_pickle(self): with self.assertRaises(TypeError): pickle.dumps(m, proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_use_released_memory(self): # gh-92888: Previously it was possible to use a memoryview even after # backing buffer is freed in certain cases. This tests that those @@ -666,5 +776,56 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__' + def test_buffer_reference_loop(self): + m = memoryview(b'abc').__buffer__(0) + o = MyObject() + o.m = m + o.o = o + wr = weakref.ref(o) + del m, o + gc.collect() + self.assertIsNone(wr()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'pickle' has no attribute 'PickleBuffer' + def test_picklebuffer_reference_loop(self): + pb = pickle.PickleBuffer(memoryview(b'abc')) + o = MyObject() + o.pb = pb + o.o = o + wr = weakref.ref(o) + del pb, o + gc.collect() + self.assertIsNone(wr()) + + +@threading_helper.requires_working_threading() +@support.requires_resource("cpu") +class RacingTest(unittest.TestCase): + def test_racing_getbuf_and_releasebuf(self): + """Repeatly access the memoryview for racing.""" + try: + from multiprocessing.managers import SharedMemoryManager + except ImportError: + self.skipTest("Test requires multiprocessing") + from threading import Thread, Event + + start = Event() + with SharedMemoryManager() as smm: + obj = smm.ShareableList(range(100)) + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] + with threading_helper.start_threads(threads): + start.set() + + del obj + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py new file mode 100644 index 00000000000..1707df9075a --- /dev/null +++ b/Lib/test/test_metaclass.py @@ -0,0 +1,321 @@ +import doctest +import unittest + + +doctests = """ + +Basic class construction. + + >>> class C: + ... def meth(self): print("Hello") + ... + >>> C.__class__ is type + True + >>> a = C() + >>> a.__class__ is C + True + >>> a.meth() + Hello + >>> + +Use *args notation for the bases. + + >>> class A: pass + >>> class B: pass + >>> bases = (A, B) + >>> class C(*bases): pass + >>> C.__bases__ == bases + True + >>> + +Use a trivial metaclass. + + >>> class M(type): + ... pass + ... + >>> class C(metaclass=M): + ... def meth(self): print("Hello") + ... + >>> C.__class__ is M + True + >>> a = C() + >>> a.__class__ is C + True + >>> a.meth() + Hello + >>> + +Use **kwds notation for the metaclass keyword. + + >>> kwds = {'metaclass': M} + >>> class C(**kwds): pass + ... + >>> C.__class__ is M + True + >>> a = C() + >>> a.__class__ is C + True + >>> + +Use a metaclass with a __prepare__ static method. + + >>> class M(type): + ... @staticmethod + ... def __prepare__(*args, **kwds): + ... print("Prepare called:", args, kwds) + ... return dict() + ... def __new__(cls, name, bases, namespace, **kwds): + ... print("New called:", kwds) + ... return type.__new__(cls, name, bases, namespace) + ... def __init__(cls, *args, **kwds): + ... pass + ... + >>> class C(metaclass=M): + ... def meth(self): print("Hello") + ... + Prepare called: ('C', ()) {} + New called: {} + >>> + +Also pass another keyword. + + >>> class C(object, metaclass=M, other="haha"): + ... pass + ... + Prepare called: ('C', (<class 'object'>,)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (object,) + True + >>> a = C() + >>> a.__class__ is C + True + >>> + +Check that build_class doesn't mutate the kwds dict. + + >>> kwds = {'metaclass': type} + >>> class C(**kwds): pass + ... + >>> kwds == {'metaclass': type} + True + >>> + +Use various combinations of explicit keywords and **kwds. + + >>> bases = (object,) + >>> kwds = {'metaclass': M, 'other': 'haha'} + >>> class C(*bases, **kwds): pass + ... + Prepare called: ('C', (<class 'object'>,)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (object,) + True + >>> class B: pass + >>> kwds = {'other': 'haha'} + >>> class C(B, metaclass=M, *bases, **kwds): pass + ... + Prepare called: ('C', (<class 'test.test_metaclass.B'>, <class 'object'>)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (B, object) + True + >>> + +Check for duplicate keywords. + + # TODO: RUSTPYTHON + >>> class C(metaclass=type, metaclass=type): pass # doctest: +SKIP + ... + Traceback (most recent call last): + [...] + SyntaxError: keyword argument repeated: metaclass + >>> + +Another way. + + >>> kwds = {'metaclass': type} + + # TODO: RUSTPYTHON + >>> class C(metaclass=type, **kwds): pass # doctest: +SKIP + ... + Traceback (most recent call last): + [...] + TypeError: __build_class__() got multiple values for keyword argument 'metaclass' + >>> + +Use a __prepare__ method that returns an instrumented dict. + + >>> class LoggingDict(dict): + ... def __setitem__(self, key, value): + ... print("d[%r] = %r" % (key, value)) + ... dict.__setitem__(self, key, value) + ... + >>> class Meta(type): + ... @staticmethod + ... def __prepare__(name, bases): + ... return LoggingDict() + ... + + # TODO: RUSTPYTHON + >>> class C(metaclass=Meta): # doctest: +SKIP + ... foo = 2+2 + ... foo = 42 + ... bar = 123 + ... + d['__module__'] = 'test.test_metaclass' + d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 + d['foo'] = 4 + d['foo'] = 42 + d['bar'] = 123 + d['__static_attributes__'] = () + >>> + +Use a metaclass that doesn't derive from type. + + >>> def meta(name, bases, namespace, **kwds): + ... print("meta:", name, bases) + ... print("ns:", sorted(namespace.items())) + ... print("kw:", sorted(kwds.items())) + ... return namespace + ... + + # TODO: RUSTPYTHON + >>> class C(metaclass=meta): # doctest: +SKIP + ... a = 42 + ... b = 24 + ... + meta: C () + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + kw: [] + + # TODO: RUSTPYTHON + >>> type(C) is dict # doctest: +SKIP + True + + # TODO: RUSTPYTHON + >>> print(sorted(C.items())) # doctest: +SKIP + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + >>> + +And again, with a __prepare__ attribute. + + >>> def prepare(name, bases, **kwds): + ... print("prepare:", name, bases, sorted(kwds.items())) + ... return LoggingDict() + ... + >>> meta.__prepare__ = prepare + + # TODO: RUSTPYTHON + >>> class C(metaclass=meta, other="booh"): # doctest: +SKIP + ... a = 1 + ... a = 2 + ... b = 3 + ... + prepare: C () [('other', 'booh')] + d['__module__'] = 'test.test_metaclass' + d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 + d['a'] = 1 + d['a'] = 2 + d['b'] = 3 + d['__static_attributes__'] = () + meta: C () + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] + kw: [('other', 'booh')] + >>> + +The default metaclass must define a __prepare__() method. + + >>> type.__prepare__() + {} + >>> + +Make sure it works with subclassing. + + >>> class M(type): + ... @classmethod + ... def __prepare__(cls, *args, **kwds): + ... d = super().__prepare__(*args, **kwds) + ... d["hello"] = 42 + ... return d + ... + >>> class C(metaclass=M): + ... print(hello) + ... + 42 + >>> print(C.hello) + 42 + >>> + +Test failures in looking up the __prepare__ method work. + >>> class ObscureException(Exception): + ... pass + >>> class FailDescr: + ... def __get__(self, instance, owner): + ... raise ObscureException + >>> class Meta(type): + ... __prepare__ = FailDescr() + >>> class X(metaclass=Meta): + ... pass + Traceback (most recent call last): + [...] + test.test_metaclass.ObscureException + +Test setting attributes with a non-base type in mro() (gh-127773). + + >>> class Base: + ... value = 1 + ... + >>> class Meta(type): + ... def mro(cls): + ... return (cls, Base, object) + ... + >>> class WeirdClass(metaclass=Meta): + ... pass + ... + >>> Base.value + 1 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 1 + >>> Base.value = 2 + >>> Base.value + 2 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 2 + >>> Base.value = 3 + >>> Base.value + 3 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 3 + +""" + +import sys + +# Trace function introduces __locals__ which causes various tests to fail. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + + +if __name__ == "__main__": + # set __name__ to match doctest expectations + __name__ = "test.test_metaclass" + unittest.main() diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 23092ffd0f3..c1806b1c133 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,12 +1,18 @@ import io -import locale import mimetypes -import pathlib +import os +import shlex import sys -import unittest - -from test import support +import unittest.mock from platform import win32_edition +from test import support +from test.support import cpython_only, force_not_colorized, os_helper +from test.support.import_helper import ensure_lazy_imports + +try: + import _winapi +except ImportError: + _winapi = None def setUpModule(): @@ -28,15 +34,30 @@ class MimeTypesTestCase(unittest.TestCase): def setUp(self): self.db = mimetypes.MimeTypes() + def test_case_sensitivity(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) + def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -50,12 +71,10 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_mime_types(self): eq = self.assertEqual @@ -64,32 +83,40 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data) + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. - with os_helper.temp_dir() as directory: - data = "application/no-mans-land Fran\u00E7ais" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding='utf-8') - import _bootlocale - with support.swap_attr(_bootlocale, 'getpreferredencoding', lambda do_setlocale=True: 'ASCII'): - mime_dict = mimetypes.read_mime_types(file) - eq(mime_dict[".Français"], "application/no-mans-land") + data = "application/no-mans-land Fran\u00E7ais" + filename = "filename" + fp = io.StringIO(data) + with unittest.mock.patch.object(mimetypes, 'open', + return_value=fp) as mock_open: + mime_dict = mimetypes.read_mime_types(filename) + mock_open.assert_called_with(filename, encoding='utf-8') + eq(mime_dict[".Français"], "application/no-mans-land") def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -99,37 +126,77 @@ def test_filename_with_url_delimiters(self): # compared to when interpreted as filename because of the semicolon. eq = self.assertEqual gzip_expected = ('application/x-tar', 'gzip') - eq(self.db.guess_type(";1.tar.gz"), gzip_expected) - eq(self.db.guess_type("?1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) - eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + def test_guess_all_types(self): - eq = self.assertEqual - unless = self.assertTrue # First try strict. Use a set here for testing the results because if # test_urllib2 is run before test_mimetypes, global state is modified # such that the 'all' set will have more items in it. - all = set(self.db.guess_all_extensions('text/plain', strict=True)) - unless(all >= set(['.bat', '.c', '.h', '.ksh', '.pl', '.txt'])) + all = self.db.guess_all_extensions('text/plain', strict=True) + self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) + self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict all = self.db.guess_all_extensions('image/jpg', strict=False) - all.sort() - eq(all, ['.jpg']) + self.assertEqual(all, ['.jpg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) - eq(all, []) + self.assertEqual(all, []) + # And now for type existing in both strict and non-strict mappings. + self.db.add_type('test-type', '.strict-ext') + self.db.add_type('test-type', '.non-strict-ext', strict=False) + all = self.db.guess_all_extensions('test-type', strict=False) + self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) + all = self.db.guess_all_extensions('test-type') + self.assertEqual(all, ['.strict-ext']) + # Test that changing the result list does not affect the global state + all.append('.no-such-ext') + all = self.db.guess_all_extensions('test-type') + self.assertNotIn('.no-such-ext', all) def test_encoding(self): - getpreferredencoding = locale.getpreferredencoding - self.addCleanup(setattr, locale, 'getpreferredencoding', - getpreferredencoding) - locale.getpreferredencoding = lambda: 'ascii' - filename = support.findfile("mime.types") mimes = mimetypes.MimeTypes([filename]) exts = mimes.guess_all_extensions('application/vnd.geocube+xml', @@ -146,29 +213,110 @@ def test_init_reinitializes(self): # Poison should be gone. self.assertEqual(mimetypes.guess_extension('foo/bar'), None) + @unittest.skipIf(sys.platform.startswith("win"), "Non-Windows only") + def test_guess_known_extensions(self): + # Issue 37529 + # The test fails on Windows because Windows adds mime types from the Registry + # and that creates some duplicates. + from mimetypes import types_map + for v in types_map.values(): + self.assertIsNotNone(mimetypes.guess_extension(v)) + def test_preferred_extension(self): def check_extensions(): - self.assertEqual(mimetypes.guess_extension('application/octet-stream'), '.bin') - self.assertEqual(mimetypes.guess_extension('application/postscript'), '.ps') - self.assertEqual(mimetypes.guess_extension('application/vnd.apple.mpegurl'), '.m3u') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-excel'), '.xls') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-powerpoint'), '.ppt') - self.assertEqual(mimetypes.guess_extension('application/x-texinfo'), '.texi') - self.assertEqual(mimetypes.guess_extension('application/x-troff'), '.roff') - self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') - self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') - self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') - self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') - self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') - self.assertEqual(mimetypes.guess_extension('text/html'), '.html') - self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') - self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') - self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') + for mime_type, ext in ( + ("application/epub+zip", ".epub"), + ("application/octet-stream", ".bin"), + ("application/gzip", ".gz"), + ("application/ogg", ".ogx"), + ("application/postscript", ".ps"), + ("application/vnd.apple.mpegurl", ".m3u"), + ("application/vnd.ms-excel", ".xls"), + ("application/vnd.ms-fontobject", ".eot"), + ("application/vnd.ms-powerpoint", ".ppt"), + ("application/vnd.oasis.opendocument.graphics", ".odg"), + ("application/vnd.oasis.opendocument.presentation", ".odp"), + ("application/vnd.oasis.opendocument.spreadsheet", ".ods"), + ("application/vnd.oasis.opendocument.text", ".odt"), + ("application/vnd.openxmlformats-officedocument.presentationml.presentation", ".pptx"), + ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xlsx"), + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"), + ("application/vnd.rar", ".rar"), + ("application/x-7z-compressed", ".7z"), + ("application/x-debian-package", ".deb"), + ("application/x-httpd-php", ".php"), + ("application/x-rpm", ".rpm"), + ("application/x-texinfo", ".texi"), + ("application/x-troff", ".roff"), + ("application/xml", ".xsl"), + ("application/yaml", ".yaml"), + ("audio/flac", ".flac"), + ("audio/matroska", ".mka"), + ("audio/mp4", ".m4a"), + ("audio/mpeg", ".mp3"), + ("audio/ogg", ".ogg"), + ("audio/vnd.wave", ".wav"), + ("audio/webm", ".weba"), + ("font/otf", ".otf"), + ("font/ttf", ".ttf"), + ("font/woff", ".woff"), + ("font/woff2", ".woff2"), + ("image/avif", ".avif"), + ("image/emf", ".emf"), + ("image/fits", ".fits"), + ("image/g3fax", ".g3"), + ("image/jp2", ".jp2"), + ("image/jpeg", ".jpg"), + ("image/jpm", ".jpm"), + ("image/t38", ".t38"), + ("image/tiff", ".tiff"), + ("image/tiff-fx", ".tfx"), + ("image/webp", ".webp"), + ("image/wmf", ".wmf"), + ("message/rfc822", ".eml"), + ("model/gltf+json", ".gltf"), + ("model/gltf-binary", ".glb"), + ("model/stl", ".stl"), + ("text/html", ".html"), + ("text/plain", ".txt"), + ("text/rtf", ".rtf"), + ("text/x-rst", ".rst"), + ("video/matroska", ".mkv"), + ("video/matroska-3d", ".mk3d"), + ("video/mpeg", ".mpeg"), + ("video/ogg", ".ogv"), + ("video/quicktime", ".mov"), + ("video/vnd.avi", ".avi"), + ("video/x-m4v", ".m4v"), + ("video/x-ms-wmv", ".wmv"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + self.assertEqual(mimetypes.guess_extension(mime_type), ext) check_extensions() mimetypes.init() check_extensions() + def test_guess_file_type(self): + def check_file_type(): + for mime_type, ext in ( + ("application/yaml", ".yaml"), + ("application/yaml", ".yml"), + ("audio/mpeg", ".mp2"), + ("audio/mpeg", ".mp3"), + ("video/mpeg", ".m1v"), + ("video/mpeg", ".mpe"), + ("video/mpeg", ".mpeg"), + ("video/mpeg", ".mpg"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + result, _ = mimetypes.guess_file_type(f"filename{ext}") + self.assertEqual(result, mime_type) + + check_file_type() + mimetypes.init() + check_file_type() + def test_init_stability(self): mimetypes.init() @@ -189,27 +337,59 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( type='image/jpg', strict=False), '.jpg') + def test_added_types_are_used(self): + mimetypes.add_type('testing/default-type', '') + mime_type, _ = mimetypes.guess_type('') + self.assertEqual(mime_type, 'testing/default-type') + + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, None) + + mimetypes.add_type('testing/type', '.myext') + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, 'testing/type') + + def test_add_type_with_undotted_extension_deprecated(self): + with self.assertWarns(DeprecationWarning): + mimetypes.add_type("testing/type", "undotted") + @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): @@ -236,58 +416,94 @@ def test_registry_parsing(self): eq(self.db.guess_type("image.jpg"), ("image/jpeg", None)) eq(self.db.guess_type("image.png"), ("image/png", None)) + @unittest.skipIf(not hasattr(_winapi, "_mimetypes_read_windows_registry"), + "read_windows_registry accelerator unavailable") + def test_registry_accelerator(self): + from_accel = {} + from_reg = {} + _winapi._mimetypes_read_windows_registry( + lambda v, k: from_accel.setdefault(k, set()).add(v) + ) + mimetypes.MimeTypes._read_windows_registry( + lambda v, k: from_reg.setdefault(k, set()).add(v) + ) + self.assertEqual(list(from_reg), list(from_accel)) + for k in from_reg: + self.assertEqual(from_reg[k], from_accel[k]) + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, mimetypes) - -class MimetypesCliTestCase(unittest.TestCase): - - def mimetypes_cmd(self, *args, **kwargs): - support.patch(self, sys, "argv", [sys.executable, *args]) - with support.captured_stdout() as output: - mimetypes._main() - return output.getvalue().strip() - - def test_help_option(self): - support.patch(self, sys, "argv", [sys.executable, "-h"]) - with support.captured_stdout() as output: - with self.assertRaises(SystemExit) as cm: - mimetypes._main() - - self.assertIn("Usage: mimetypes.py", output.getvalue()) - self.assertEqual(cm.exception.code, 0) - - def test_invalid_option(self): - support.patch(self, sys, "argv", [sys.executable, "--invalid"]) - with support.captured_stdout() as output: - with self.assertRaises(SystemExit) as cm: - mimetypes._main() - - self.assertIn("Usage: mimetypes.py", output.getvalue()) - self.assertEqual(cm.exception.code, 1) - - def test_guess_extension(self): - eq = self.assertEqual - - extension = self.mimetypes_cmd("-l", "-e", "image/jpg") - eq(extension, ".jpg") - - extension = self.mimetypes_cmd("-e", "image/jpg") - eq(extension, "I don't know anything about type image/jpg") - - extension = self.mimetypes_cmd("-e", "image/jpeg") - eq(extension, ".jpg") - - def test_guess_type(self): - eq = self.assertEqual - - type_info = self.mimetypes_cmd("-l", "foo.pic") - eq(type_info, "type: image/pict encoding: None") - - type_info = self.mimetypes_cmd("foo.pic") - eq(type_info, "I don't know anything about type foo.pic") + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("mimetypes", {"os", "posixpath", "urllib.parse", "argparse"}) + + +class CommandLineTest(unittest.TestCase): + @force_not_colorized + def test_parse_args(self): + args, help_text = mimetypes._parse_args("-h") + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = mimetypes._parse_args("--invalid") + self.assertTrue(help_text.startswith("usage: ")) + + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + self.assertTrue(args.extension) + self.assertTrue(args.lenient) + self.assertEqual(args.type, ["image/jpg"]) + + args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + self.assertTrue(args.extension) + self.assertFalse(args.lenient) + self.assertEqual(args.type, ["image/jpg"]) + + args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) + self.assertFalse(args.extension) + self.assertTrue(args.lenient) + self.assertEqual(args.type, ["foo.webp"]) + + args, _ = mimetypes._parse_args(shlex.split("foo.pic")) + self.assertFalse(args.extension) + self.assertFalse(args.lenient) + self.assertEqual(args.type, ["foo.pic"]) + + def test_multiple_inputs(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.png"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "type: image/png encoding: None" + ) + + def test_multiple_inputs_error(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.bar_ext"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "error: media type unknown for foo.bar_ext" + ) + + + def test_invocation(self): + for command, expected in [ + ("-l -e image/jpg", ".jpg"), + ("-e image/jpeg", ".jpg"), + ("-l foo.webp", "type: image/webp encoding: None"), + ]: + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) + + def test_invocation_error(self): + for command, expected in [ + ("-e image/jpg", "error: unknown type image/jpg"), + ("foo.bar_ext", "error: media type unknown for foo.bar_ext"), + ]: + with self.subTest(command=command): + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) if __name__ == "__main__": diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index b599c6d8c8d..59c74fd0d41 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -293,8 +293,6 @@ class M(ModuleType): melon = Descr() self.assertRaises(RuntimeError, getattr, M("mymod"), "melon") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_create_annotations(self): # module objects lazy create their __annotations__ dict on demand. # the annotations dict is stored in module.__dict__. @@ -334,7 +332,11 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.typinganndata.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py new file mode 100644 index 00000000000..1c6905bd1ee --- /dev/null +++ b/Lib/test/test_msvcrt.py @@ -0,0 +1,120 @@ +import os +import subprocess +import sys +import unittest +from textwrap import dedent + +from test.support import os_helper, requires_resource +from test.support.os_helper import TESTFN, TESTFN_ASCII + +if sys.platform != "win32": + raise unittest.SkipTest("windows related tests") + +import _winapi +import msvcrt + + +class TestFileOperations(unittest.TestCase): + def test_locking(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + self.assertRaises(OSError, msvcrt.locking, f.fileno(), msvcrt.LK_NBLCK, 1) + + def test_unlockfile(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + + def test_setmode(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.setmode(f.fileno(), os.O_BINARY) + msvcrt.setmode(f.fileno(), os.O_TEXT) + + def test_open_osfhandle(self): + h = _winapi.CreateFile(TESTFN_ASCII, _winapi.GENERIC_WRITE, 0, 0, 1, 128, 0) + self.addCleanup(os_helper.unlink, TESTFN_ASCII) + + try: + fd = msvcrt.open_osfhandle(h, os.O_RDONLY) + h = None + os.close(fd) + finally: + if h: + _winapi.CloseHandle(h) + + def test_get_osfhandle(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.get_osfhandle(f.fileno()) + + +c = '\u5b57' # unicode CJK char (meaning 'character') for 'wide-char' tests +c_encoded = b'\x57\x5b' # utf-16-le (which windows internally used) encoded char for this CJK char + + +class TestConsoleIO(unittest.TestCase): + # CREATE_NEW_CONSOLE creates a "popup" window. + @requires_resource('gui') + def run_in_separated_process(self, code): + # Run test in a separated process to avoid stdin conflicts. + # See: gh-110147 + cmd = [sys.executable, '-c', code] + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + + def test_kbhit(self): + code = dedent(''' + import msvcrt + assert msvcrt.kbhit() == 0 + ''') + self.run_in_separated_process(code) + + def test_getch(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getch(), b'c') + + def check_getwch(self, funcname): + code = dedent(f''' + import msvcrt + from _testconsole import write_input + with open("CONIN$", "rb", buffering=0) as stdin: + write_input(stdin, {ascii(c_encoded)}) + assert msvcrt.{funcname}() == "{c}" + ''') + self.run_in_separated_process(code) + + def test_getwch(self): + self.check_getwch('getwch') + + def test_getche(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getche(), b'c') + + def test_getwche(self): + self.check_getwch('getwche') + + def test_putch(self): + msvcrt.putch(b'c') + + def test_putwch(self): + msvcrt.putwch(c) + + +class TestOther(unittest.TestCase): + def test_heap_min(self): + try: + msvcrt.heapmin() + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_manager.py b/Lib/test/test_multiprocessing_fork/test_manager.py index 9efbb83bbb7..f8d7eddd652 100644 --- a/Lib/test/test_multiprocessing_fork/test_manager.py +++ b/Lib/test/test_multiprocessing_fork/test_manager.py @@ -3,5 +3,22 @@ install_tests_in_module_dict(globals(), 'fork', only_type="manager") +import sys # TODO: RUSTPYTHON +class WithManagerTestCondition(WithManagerTestCondition): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON, times out') + def test_notify_all(self): super().test_notify_all() # TODO: RUSTPYTHON + +class WithManagerTestQueue(WithManagerTestQueue): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON, times out') + def test_fork(self): super().test_fork() # TODO: RUSTPYTHON + +local_globs = globals().copy() # TODO: RUSTPYTHON +for name, base in local_globs.items(): # TODO: RUSTPYTHON + if name.startswith('WithManagerTest') and issubclass(base, unittest.TestCase): # TODO: RUSTPYTHON + base = unittest.skipIf( # TODO: RUSTPYTHON + sys.platform == 'linux', # TODO: RUSTPYTHON + 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError' + )(base) # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_misc.py b/Lib/test/test_multiprocessing_fork/test_misc.py index 891a494020c..bcf0858258e 100644 --- a/Lib/test/test_multiprocessing_fork/test_misc.py +++ b/Lib/test/test_multiprocessing_fork/test_misc.py @@ -3,5 +3,24 @@ install_tests_in_module_dict(globals(), 'fork', exclude_types=True) +import sys # TODO: RUSTPYTHON +class TestManagerExceptions(TestManagerExceptions): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") + def test_queue_get(self): super().test_queue_get() # TODO: RUSTPYTHON + +@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") +class TestInitializers(TestInitializers): pass # TODO: RUSTPYTHON + +class TestStartMethod(TestStartMethod): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") + def test_nested_startmethod(self): super().test_nested_startmethod() # TODO: RUSTPYTHON + +@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") +class TestSyncManagerTypes(TestSyncManagerTypes): pass # TODO: RUSTPYTHON + +class MiscTestCase(MiscTestCase): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky") + def test_forked_thread_not_started(self): super().test_forked_thread_not_started() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_processes.py b/Lib/test/test_multiprocessing_fork/test_processes.py index e64e9afc010..02b7256e41b 100644 --- a/Lib/test/test_multiprocessing_fork/test_processes.py +++ b/Lib/test/test_multiprocessing_fork/test_processes.py @@ -3,5 +3,40 @@ install_tests_in_module_dict(globals(), 'fork', only_type="processes") +import os, sys # TODO: RUSTPYTHON +class WithProcessesTestCondition(WithProcessesTestCondition): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_notify_all(self): super().test_notify_all() # TODO: RUSTPYTHON + +class WithProcessesTestLock(WithProcessesTestLock): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError') + def test_repr_lock(self): super().test_repr_lock() # TODO: RUSTPYTHON + +class WithProcessesTestManagerRestart(WithProcessesTestManagerRestart): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError') + def test_rapid_restart(self): super().test_rapid_restart() # TODO: RUSTPYTHON + +class WithProcessesTestProcess(WithProcessesTestProcess): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_args_argument(self): super().test_args_argument() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_process(self): super().test_process() # TODO: RUSTPYTHON + +class WithProcessesTestPoolWorkerLifetime(WithProcessesTestPoolWorkerLifetime): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_pool_worker_lifetime(self): super().test_pool_worker_lifetime() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_pool_worker_lifetime_early_close(self): super().test_pool_worker_lifetime_early_close() # TODO: RUSTPYTHON + +class WithProcessesTestQueue(WithProcessesTestQueue): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_fork(self): super().test_fork() # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky timeout') + def test_get(self): super().test_get() # TODO: RUSTPYTHON + +class WithProcessesTestSharedMemory(WithProcessesTestSharedMemory): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError') + def test_shared_memory_SharedMemoryManager_basics(self): super().test_shared_memory_SharedMemoryManager_basics() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_threads.py b/Lib/test/test_multiprocessing_fork/test_threads.py index 1670e34cb17..1065ebf7fe4 100644 --- a/Lib/test/test_multiprocessing_fork/test_threads.py +++ b/Lib/test/test_multiprocessing_fork/test_threads.py @@ -3,5 +3,14 @@ install_tests_in_module_dict(globals(), 'fork', only_type="threads") +import os, sys # TODO: RUSTPYTHON +class WithThreadsTestPool(WithThreadsTestPool): # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; flaky environment pollution when running rustpython -m test --fail-env-changed due to unknown reason") + def test_terminate(self): super().test_terminate() # TODO: RUSTPYTHON + +class WithThreadsTestManagerRestart(WithThreadsTestManagerRestart): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError') + def test_rapid_restart(self): super().test_rapid_restart() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_processes.py b/Lib/test/test_multiprocessing_forkserver/test_processes.py index 360967cf1ae..6f6b8f56837 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_processes.py +++ b/Lib/test/test_multiprocessing_forkserver/test_processes.py @@ -3,5 +3,19 @@ install_tests_in_module_dict(globals(), 'forkserver', only_type="processes") +import os, sys # TODO: RUSTPYTHON +class WithProcessesTestCondition(WithProcessesTestCondition): # TODO: RUSTPYTHON + @unittest.skip('TODO: RUSTPYTHON flaky timeout') + def test_notify(self): super().test_notify() + @unittest.skip('TODO: RUSTPYTHON flaky timeout') + def test_notify_n(self): super().test_notify_n() + +class WithProcessesTestLock(WithProcessesTestLock): # TODO: RUSTPYTHON + @unittest.skipIf( # TODO: RUSTPYTHON + sys.platform == 'linux', # TODO: RUSTPYTHON + 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError' + ) # TODO: RUSTPYTHON + def test_repr_rlock(self): super().test_repr_rlock() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_processes.py b/Lib/test/test_multiprocessing_spawn/test_processes.py index af764b0d848..21fd6abd655 100644 --- a/Lib/test/test_multiprocessing_spawn/test_processes.py +++ b/Lib/test/test_multiprocessing_spawn/test_processes.py @@ -3,5 +3,17 @@ install_tests_in_module_dict(globals(), 'spawn', only_type="processes") +import os, sys # TODO: RUSTPYTHON +class WithProcessesTestCondition(WithProcessesTestCondition): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'darwin', 'TODO: RUSTPYTHON flaky timeout') + def test_notify(self): super().test_notify() + +class WithProcessesTestLock(WithProcessesTestLock): # TODO: RUSTPYTHON + @unittest.skipIf( # TODO: RUSTPYTHON + sys.platform == 'linux', # TODO: RUSTPYTHON + 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError' + ) # TODO: RUSTPYTHON + def test_repr_rlock(self): super().test_repr_rlock() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_threads.py b/Lib/test/test_multiprocessing_spawn/test_threads.py index c1257749b9c..54c52c4188b 100644 --- a/Lib/test/test_multiprocessing_spawn/test_threads.py +++ b/Lib/test/test_multiprocessing_spawn/test_threads.py @@ -3,5 +3,10 @@ install_tests_in_module_dict(globals(), 'spawn', only_type="threads") +import os, sys # TODO: RUSTPYTHON +class WithThreadsTestPool(WithThreadsTestPool): # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; flaky environment pollution when running rustpython -m test --fail-env-changed due to unknown reason") + def test_terminate(self): super().test_terminate() # TODO: RUSTPYTHON + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 03f5384b632..fea86fe4308 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -44,32 +44,24 @@ def test_named_expression_invalid_06(self): with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_07(self): code = """def spam(a = b := 42): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_08(self): code = """def spam(a: b := 42 = 5): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_09(self): code = """spam(a=b := 'c')""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_10(self): code = """spam(x = y := f(x))""" @@ -103,8 +95,6 @@ def test_named_expression_invalid_13(self): "positional argument follows keyword argument"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_14(self): code = """(x := lambda: y := 1)""" @@ -120,8 +110,6 @@ def test_named_expression_invalid_15(self): "cannot use assignment expressions with lambda"): exec(code, {}, {}) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure def test_named_expression_invalid_16(self): code = "[i + 1 for i in i := [1,2]]" @@ -165,8 +153,6 @@ def test_named_expression_invalid_rebinding_list_comprehension_iteration_variabl with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message def test_named_expression_invalid_rebinding_list_comprehension_inner_loop(self): cases = [ ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"), @@ -223,8 +209,6 @@ def test_named_expression_invalid_rebinding_set_comprehension_iteration_variable with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_expression_invalid_rebinding_set_comprehension_inner_loop(self): cases = [ ("Inner reuse", 'j', "{i for i in range(5) if (j := 0) for j in range(5)}"), diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index ed9d8b5d281..9270f325706 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -6,10 +6,9 @@ import sys import unittest import warnings -from ntpath import ALLOW_MISSING from test import support -from test.support import cpython_only, os_helper -from test.support import TestFailed, is_emscripten +from test.support import os_helper +from ntpath import ALLOW_MISSING from test.support.os_helper import FakePath from test import test_genericpath from tempfile import TemporaryFile @@ -59,7 +58,7 @@ def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), str(gotResult))) # then with bytes @@ -75,7 +74,7 @@ def tester(fn, wantResult): warnings.simplefilter("ignore", DeprecationWarning) gotResult = eval(fn) if _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), repr(gotResult))) @@ -1022,6 +1021,19 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') @@ -1229,7 +1241,6 @@ def check_error(paths, expected): self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) - @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") def test_sameopenfile(self): with TemporaryFile() as tf1, TemporaryFile() as tf2: # Make sure the same file is really the same @@ -1279,8 +1290,6 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; crash") def test_ismount_invalid_paths(self): ismount = ntpath.ismount self.assertFalse(ismount("c:\\\udfff")) @@ -1442,7 +1451,7 @@ def test_con_device(self): self.assertTrue(os.path.exists(r"\\.\CON")) @unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32") - @cpython_only + @support.cpython_only def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in @@ -1464,8 +1473,6 @@ def test_fast_paths_in_use(self): self.assertTrue(os.path.lexists is nt._path_lexists) self.assertFalse(inspect.isfunction(os.path.lexists)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): # Result may be True or False, but shouldn't raise diff --git a/Lib/test/test_nturl2path.py b/Lib/test/test_nturl2path.py new file mode 100644 index 00000000000..a6a3422a0f7 --- /dev/null +++ b/Lib/test/test_nturl2path.py @@ -0,0 +1,107 @@ +import unittest + +from test.support import warnings_helper + + +nturl2path = warnings_helper.import_deprecated("nturl2path") + + +class NTURL2PathTest(unittest.TestCase): + """Test pathname2url() and url2pathname()""" + + def test_basic(self): + # Make sure simple tests pass + expected_path = r"parts\of\a\path" + expected_url = "parts/of/a/path" + result = nturl2path.pathname2url(expected_path) + self.assertEqual(expected_url, result, + "pathname2url() failed; %s != %s" % + (result, expected_url)) + result = nturl2path.url2pathname(expected_url) + self.assertEqual(expected_path, result, + "url2pathame() failed; %s != %s" % + (result, expected_path)) + + def test_pathname2url(self): + # Test special prefixes are correctly handled in pathname2url() + fn = nturl2path.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') + self.assertEqual(fn('c:\\a\\b.c'), '///c:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # NTFS alternate data streams + self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') + self.assertEqual(fn('foo:bar'), 'foo%3Abar') + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(nturl2path.url2pathname(url)), url) + + def test_url2pathname(self): + fn = nturl2path.url2pathname + self.assertEqual(fn('/'), '\\') + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('c:/path/to/file'), 'c:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Colons in URI + self.assertEqual(fn('///\u00e8|/'), '\u00e8:\\') + self.assertEqual(fn('//host/share/spam.txt:eggs'), '\\\\host\\share\\spam.txt:eggs') + self.assertEqual(fn('///c:/spam.txt:eggs'), 'c:\\spam.txt:eggs') + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/path/to/file'), '\\path\\to\\file') + self.assertEqual(fn('//localhost//server/path/to/file'), '\\\\server\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') + # Round-tripping + paths = ['C:', + r'\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(nturl2path.pathname2url(path)), path) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b..f7cc8331b8d 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 05b7a7462db..1f89986c777 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -347,6 +347,26 @@ def test_is_not(self): self.assertFalse(operator.is_not(a, b)) self.assertTrue(operator.is_not(a,c)) + def test_is_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_none) + self.assertFalse(operator.is_none(a)) + self.assertFalse(operator.is_none(b)) + self.assertTrue(operator.is_none(c)) + + def test_is_not_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_not_none) + self.assertTrue(operator.is_not_none(a)) + self.assertTrue(operator.is_not_none(b)) + self.assertFalse(operator.is_not_none(c)) + def test_attrgetter(self): operator = self.module class A: @@ -462,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -478,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): @@ -635,22 +668,8 @@ class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): class COperatorTestCase(OperatorTestCase, unittest.TestCase): module = c_operator - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_attrgetter_signature(self): - super().test_attrgetter_signature() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_itemgetter_signature(self): - super().test_itemgetter_signature() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_methodcaller_signature(self): - super().test_methodcaller_signature() - +@support.thread_unsafe("swaps global operator module") class OperatorPickleTestCase: def copy(self, obj, proto): with support.swap_item(sys.modules, 'operator', self.module): diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index c68214961e3..e476e472780 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -14,8 +14,9 @@ from io import StringIO from test import support -from test.support import os_helper +from test.support import cpython_only, os_helper from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.import_helper import ensure_lazy_imports import optparse from optparse import make_option, Option, \ @@ -1655,6 +1656,10 @@ def test__all__(self): not_exported = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} support.check__all__(self, optparse, not_exported=not_exported) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("optparse", {"textwrap"}) + class TestTranslations(TestTranslationsBase): def test_translations(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 105629bda19..0fd4f66df28 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -104,7 +104,7 @@ def create_file(filename, content=b'content'): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MiscTests(unittest.TestCase): @@ -187,10 +187,6 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") - @unittest.skipIf( - support.is_emscripten, "Test is unstable under Emscripten." - ) @unittest.skipIf( support.is_wasi, "WASI does not support dup." ) @@ -233,6 +229,94 @@ def test_read(self): self.assertEqual(type(s), bytes) self.assertEqual(s, b"spam") + def test_readinto(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + # Oversized so readinto without hitting end. + buffer = bytearray(7) + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 4) + # Should overwrite the first 4 bytes of the buffer. + self.assertEqual(buffer[:4], b"spam") + + # Readinto at EOF should return 0 and not touch buffer. + buffer[:] = b"notspam" + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + s = os.readinto(fd, buffer) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + + # Readinto a 0 length bytearray when at EOF should return 0 + self.assertEqual(os.readinto(fd, bytearray()), 0) + + # Readinto a 0 length bytearray with data available should return 0. + os.lseek(fd, 0, 0) + self.assertEqual(os.readinto(fd, bytearray()), 0) + + @unittest.skipUnless(hasattr(os, 'get_blocking'), + 'needs os.get_blocking() and os.set_blocking()') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.skipIf(support.is_emscripten, "set_blocking does not work correctly") + def test_readinto_non_blocking(self): + # Verify behavior of a readinto which would block on a non-blocking fd. + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # Pass some data through + os.write(w, b"spam") + self.assertEqual(os.readinto(r, bytearray(4)), 4) + + # Still don't block or return 0. + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # At EOF should return size 0 + os.close(w) + w = None + self.assertEqual(os.readinto(r, bytearray(5)), 0) + self.assertEqual(os.readinto(r, bytearray(5)), 0) # Still EOF + + finally: + os.close(r) + if w is not None: + os.close(w) + + def test_readinto_badarg(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + + for bad_arg in ("test", bytes(), 14): + with self.subTest(f"bad buffer {type(bad_arg)}"): + with self.assertRaises(TypeError): + os.readinto(fd, bad_arg) + + with self.subTest("doesn't work on file objects"): + with self.assertRaises(TypeError): + os.readinto(fobj, bytearray(5)) + + # takes two args + with self.assertRaises(TypeError): + os.readinto(fd) + + # No data should have been read with the bad arguments. + buffer = bytearray(4) + s = os.readinto(fd, buffer) + self.assertEqual(s, 4) + self.assertEqual(buffer, b"spam") + @support.cpython_only # Skip the test on 32-bit platforms: the number of bytes must fit in a # Py_ssize_t type @@ -252,6 +336,29 @@ def test_large_read(self, size): # operating system is free to return less bytes than requested. self.assertEqual(data, b'test') + + @support.cpython_only + # Skip the test on 32-bit platforms: the number of bytes must fit in a + # Py_ssize_t type + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, + "needs INT_MAX < PY_SSIZE_T_MAX") + @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) + def test_large_readinto(self, size): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + create_file(os_helper.TESTFN, b'test') + + # Issue #21932: For readinto the buffer contains the length rather than + # a length being passed explicitly to read, should still get capped to a + # valid size / not raise an OverflowError for sizes larger than INT_MAX. + buffer = bytearray(INT_MAX + 10) + with open(os_helper.TESTFN, "rb") as fp: + length = os.readinto(fp.fileno(), buffer) + + # The test does not try to read more than 2 GiB at once because the + # operating system is free to return less bytes than requested. + self.assertEqual(length, 4) + self.assertEqual(buffer[:4], b'test') + def test_write(self): # os.write() accepts bytes- and buffer-like objects but not strings fd = os.open(os_helper.TESTFN, os.O_CREAT | os.O_WRONLY) @@ -710,7 +817,7 @@ def test_15261(self): self.assertEqual(ctx.exception.errno, errno.EBADF) def check_file_attributes(self, result): - self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertHasAttr(result, 'st_file_attributes') self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) @@ -805,14 +912,28 @@ def _test_utime(self, set_time, filename=None): set_time(filename, (atime_ns, mtime_ns)) st = os.stat(filename) - if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) - self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + if support.is_emscripten: + # Emscripten timestamps are roundtripped through a 53 bit integer of + # nanoseconds. If we want to represent ~50 years which is an 11 + # digits number of seconds: + # 2*log10(60) + log10(24) + log10(365) + log10(60) + log10(50) + # is about 11. Because 53 * log10(2) is about 16, we only have 5 + # digits worth of sub-second precision. + # Some day it would be good to fix this upstream. + delta=1e-5 + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) + self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) - self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) - self.assertEqual(st.st_mtime_ns, mtime_ns) + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): def set_time(filename, ns): @@ -825,9 +946,7 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` - # return (ns * 1e-9) + 0.5e-9 - return (ns * 1e-9) + return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter @@ -939,7 +1058,6 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -1301,6 +1419,52 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_reload_environ(self): + # Test os.reload_environ() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.reload_environ() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.reload_environ() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test reload_environ() on os.environb with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.reload_environ() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test reload_environ() on os.environb with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.reload_environ() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" @@ -1371,9 +1535,7 @@ def setUp(self): else: self.sub2_tree = (sub2_path, ["SUB21"], ["tmp3"]) - if not support.is_emscripten: - # Emscripten fails with inaccessible directory - os.chmod(sub21_path, 0) + os.chmod(sub21_path, 0) try: os.listdir(sub21_path) except PermissionError: @@ -1669,9 +1831,6 @@ def test_yields_correct_dir_fd(self): # check that listdir() returns consistent information self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files)) - @unittest.skipIf( - support.is_emscripten, "Cannot dup stdout on Emscripten" - ) @unittest.skipIf( support.is_android, "dup return value is unpredictable on Android" ) @@ -1688,9 +1847,6 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) - @unittest.skipIf( - support.is_emscripten, "Cannot dup stdout on Emscripten" - ) @unittest.skipIf( support.is_android, "dup return value is unpredictable on Android" ) @@ -1726,17 +1882,6 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) - def test_compare_to_walk(self): - return super().test_compare_to_walk() - - @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) - def test_dir_fd(self): - return super().test_dir_fd() - - @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) - def test_yields_correct_dir_fd(self): - return super().test_yields_correct_dir_fd() @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class BytesFwalkTests(FwalkTests): @@ -1771,10 +1916,12 @@ def test_makedir(self): os.makedirs(path) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') @@ -1787,8 +1934,8 @@ def test_mode(self): self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_existing_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -1805,8 +1952,8 @@ def test_exist_ok_existing_directory(self): os.makedirs(os.path.abspath('/'), exist_ok=True) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_s_isgid_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -2036,7 +2183,7 @@ def test_getrandom0(self): self.assertEqual(empty, b'') def test_getrandom_random(self): - self.assertTrue(hasattr(os, 'GRND_RANDOM')) + self.assertHasAttr(os, 'GRND_RANDOM') # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare # resource /dev/random @@ -2320,9 +2467,13 @@ def test_chmod(self): @unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): - singles = ["fchdir", "dup", "fdatasync", "fstat", - "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] - singles_fildes = {"fchdir", "fdatasync", "fsync"} + singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir"} + # systemd-nspawn --suppress-sync=true does not verify fd passed + # fdatasync() and fsync(), and always returns success + if not support.in_systemd_nspawn_sync_suppressed(): + singles += ["fdatasync", "fsync"] + singles_fildes |= {"fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): @@ -2351,7 +2502,6 @@ def check_bool(self, f, *args, **kwargs): with self.assertRaises(RuntimeWarning): f(fd, *args, **kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") self.check_bool(os.fdopen, encoding="utf-8") @@ -2381,10 +2531,6 @@ def test_dup2(self): self.check(os.dup2, 20) @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') - @unittest.skipIf( - support.is_emscripten, - "dup2() with negative fds is broken on Emscripten (see gh-102179)" - ) def test_dup2_negative_fd(self): valid_fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, valid_fd) @@ -2408,20 +2554,21 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) def test_fpathconf(self): self.assertIn("PC_NAME_MAX", os.pathconf_names) - self.check(os.pathconf, "PC_NAME_MAX") - self.check(os.fpathconf, "PC_NAME_MAX") self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') + @unittest.skipIf( + support.linked_to_musl(), + 'musl pathconf ignores the file descriptor and returns a constant', + ) + def test_fpathconf_bad_fd(self): + self.check(os.pathconf, "PC_NAME_MAX") + self.check(os.fpathconf, "PC_NAME_MAX") + @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) @@ -2436,6 +2583,10 @@ def test_lseek(self): def test_read(self): self.check(os.read, 1) + @unittest.skipUnless(hasattr(os, 'readinto'), 'test needs os.readinto()') + def test_readinto(self): + self.check(os.readinto, bytearray(5)) + @unittest.skipUnless(hasattr(os, 'readv'), 'test needs os.readv()') def test_readv(self): buf = bytearray(10) @@ -2464,13 +2615,8 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_fchdir(self): - return super().test_fchdir() - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_fsync(self): - return super().test_fsync() + @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') @@ -2712,12 +2858,10 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2776,7 +2920,6 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3360,9 +3503,6 @@ def test_bad_fd(self): @unittest.skipUnless(os.isatty(0) and not win32_is_iot() and (sys.platform.startswith('win') or (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), 'test requires a tty and either Windows or nl_langinfo(CODESET)') - @unittest.skipIf( - support.is_emscripten, "Cannot get encoding of stdin on Emscripten" - ) def test_device_encoding(self): encoding = os.device_encoding(0) self.assertIsNotNone(encoding) @@ -3542,7 +3682,6 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -4502,7 +4641,6 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>) def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4906,7 +5044,6 @@ def setUp(self): def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - @unittest.expectedFailure # TODO: RUSTPYTHON; (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry) def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -4993,7 +5130,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON; flaky test') + @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON; flaky test") def test_attributes(self): link = os_helper.can_hardlink() symlink = os_helper.can_symlink() @@ -5175,7 +5312,7 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <builtin_function_or_method object at 0xba3106920> not found in {<builtin_function_or_method object at 0xba31078e0>, <builtin_function_or_method object at 0xba31079c0>, <builtin_function_or_method object at 0xba3107b10>, <builtin_function_or_method object at 0xba3159500>, <builtin_function_or_method object at 0xba3159570>, <builtin_function_or_method object at 0xba3107800>, <builtin_function_or_method object at 0xba3106760>, <builtin_function_or_method object at 0xba3106a00>, <builtin_function_or_method object at 0xba3106990>, <builtin_function_or_method object at 0xba3107330>, <builtin_function_or_method object at 0xba31072c0>, <builtin_function_or_method object at 0xba31064c0>} @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -5257,7 +5394,6 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - @unittest.expectedFailure # TODO: RUSTPYTHON def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -5297,8 +5433,8 @@ def test_fsencode_fsdecode(self): def test_pathlike(self): self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) - self.assertTrue(issubclass(FakePath, os.PathLike)) - self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + self.assertIsSubclass(FakePath, os.PathLike) + self.assertIsInstance(FakePath('x'), os.PathLike) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -5324,8 +5460,8 @@ def test_pathlike_subclasshook(self): # true on abstract implementation. class A(os.PathLike): pass - self.assertFalse(issubclass(FakePath, A)) - self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertNotIsSubclass(FakePath, A) + self.assertIsSubclass(FakePath, os.PathLike) def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) @@ -5335,9 +5471,8 @@ class A(os.PathLike): __slots__ = () def __fspath__(self): return '' - self.assertFalse(hasattr(A(), '__dict__')) + self.assertNotHasAttr(A(), '__dict__') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_fspath_set_to_None(self): class Foo: __fspath__ = None @@ -5440,7 +5575,7 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b"can't fork at interpreter shutdown" not found in b"Exception ignored in: <function AtFinalization.__del__ at 0xc508b30c0>\nAttributeError: 'NoneType' object has no attribute 'fork'\n" def test_fork_at_finalization(self): code = """if 1: import atexit diff --git a/Lib/test/test_osx_env.py b/Lib/test/test_osx_env.py new file mode 100644 index 00000000000..80198edcb80 --- /dev/null +++ b/Lib/test/test_osx_env.py @@ -0,0 +1,34 @@ +""" +Test suite for OS X interpreter environment variables. +""" + +from test.support.os_helper import EnvironmentVarGuard +import subprocess +import sys +import sysconfig +import unittest + +@unittest.skipUnless(sys.platform == 'darwin' and + sysconfig.get_config_var('WITH_NEXT_FRAMEWORK'), + 'unnecessary on this platform') +class OSXEnvironmentVariableTestCase(unittest.TestCase): + def _check_sys(self, ev, cond, sv, val = sys.executable + 'dummy'): + with EnvironmentVarGuard() as evg: + subpc = [str(sys.executable), '-c', + 'import sys; sys.exit(2 if "%s" %s %s else 3)' % (val, cond, sv)] + # ensure environment variable does not exist + evg.unset(ev) + # test that test on sys.xxx normally fails + rc = subprocess.call(subpc) + self.assertEqual(rc, 3, "expected %s not %s %s" % (ev, cond, sv)) + # set environ variable + evg.set(ev, val) + # test that sys.xxx has been influenced by the environ value + rc = subprocess.call(subpc) + self.assertEqual(rc, 2, "expected %s %s %s" % (ev, cond, sv)) + + def test_pythonexecutable_sets_sys_executable(self): + self._check_sys('PYTHONEXECUTABLE', '==', 'sys.executable') + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/__init__.py b/Lib/test/test_pathlib/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_pathlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pathlib/support/__init__.py b/Lib/test/test_pathlib/support/__init__.py new file mode 100644 index 00000000000..dcaef654d77 --- /dev/null +++ b/Lib/test/test_pathlib/support/__init__.py @@ -0,0 +1,2 @@ +# Set to 'True' if the tests are run against the pathlib-abc PyPI package. +is_pypi = False diff --git a/Lib/test/test_pathlib/support/lexical_path.py b/Lib/test/test_pathlib/support/lexical_path.py new file mode 100644 index 00000000000..f29a521af9b --- /dev/null +++ b/Lib/test/test_pathlib/support/lexical_path.py @@ -0,0 +1,51 @@ +""" +Simple implementation of JoinablePath, for use in pathlib tests. +""" + +import ntpath +import os.path +import posixpath + +from . import is_pypi + +if is_pypi: + from pathlib_abc import _JoinablePath +else: + from pathlib.types import _JoinablePath + + +class LexicalPath(_JoinablePath): + __slots__ = ('_segments',) + parser = os.path + + def __init__(self, *pathsegments): + self._segments = pathsegments + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + if not isinstance(other, LexicalPath): + return NotImplemented + return str(self) == str(other) + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + + +class LexicalPosixPath(LexicalPath): + __slots__ = () + parser = posixpath + + +class LexicalWindowsPath(LexicalPath): + __slots__ = () + parser = ntpath diff --git a/Lib/test/test_pathlib/support/local_path.py b/Lib/test/test_pathlib/support/local_path.py new file mode 100644 index 00000000000..d481fd45ead --- /dev/null +++ b/Lib/test/test_pathlib/support/local_path.py @@ -0,0 +1,177 @@ +""" +Implementations of ReadablePath and WritablePath for local paths, for use in +pathlib tests. + +LocalPathGround is also defined here. It helps establish the "ground truth" +about local paths in tests. +""" + +import os + +from . import is_pypi +from .lexical_path import LexicalPath + +if is_pypi: + from shutil import rmtree + from pathlib_abc import PathInfo, _ReadablePath, _WritablePath + can_symlink = True + testfn = "TESTFN" +else: + from pathlib.types import PathInfo, _ReadablePath, _WritablePath + from test.support import os_helper + can_symlink = os_helper.can_symlink() + testfn = os_helper.TESTFN + rmtree = os_helper.rmtree + + +class LocalPathGround: + can_symlink = can_symlink + + def __init__(self, path_cls): + self.path_cls = path_cls + + def setup(self, local_suffix=""): + root = self.path_cls(testfn + local_suffix) + os.mkdir(root) + return root + + def teardown(self, root): + rmtree(root) + + def create_file(self, p, data=b''): + with open(p, 'wb') as f: + f.write(data) + + def create_dir(self, p): + os.mkdir(p) + + def create_symlink(self, p, target): + os.symlink(target, p) + + def create_hierarchy(self, p): + os.mkdir(os.path.join(p, 'dirA')) + os.mkdir(os.path.join(p, 'dirB')) + os.mkdir(os.path.join(p, 'dirC')) + os.mkdir(os.path.join(p, 'dirC', 'dirD')) + with open(os.path.join(p, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(p, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(p, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(p, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(p, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(p, 'linkA')) + os.symlink('non-existing', os.path.join(p, 'brokenLink')) + os.symlink('dirB', + os.path.join(p, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(p, 'dirA', 'linkC'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(p, 'brokenLinkLoop')) + + isdir = staticmethod(os.path.isdir) + isfile = staticmethod(os.path.isfile) + islink = staticmethod(os.path.islink) + readlink = staticmethod(os.readlink) + + def readtext(self, p): + with open(p, 'r', encoding='utf-8') as f: + return f.read() + + def readbytes(self, p): + with open(p, 'rb') as f: + return f.read() + + +class LocalPathInfo(PathInfo): + """ + Simple implementation of PathInfo for a local path + """ + __slots__ = ('_path', '_exists', '_is_dir', '_is_file', '_is_symlink') + + def __init__(self, path): + self._path = str(path) + self._exists = None + self._is_dir = None + self._is_file = None + self._is_symlink = None + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks and self.is_symlink(): + return True + if self._exists is None: + self._exists = os.path.exists(self._path) + return self._exists + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + if not follow_symlinks and self.is_symlink(): + return False + if self._is_dir is None: + self._is_dir = os.path.isdir(self._path) + return self._is_dir + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + if not follow_symlinks and self.is_symlink(): + return False + if self._is_file is None: + self._is_file = os.path.isfile(self._path) + return self._is_file + + def is_symlink(self): + """Whether this path is a symbolic link.""" + if self._is_symlink is None: + self._is_symlink = os.path.islink(self._path) + return self._is_symlink + + +class ReadableLocalPath(_ReadablePath, LexicalPath): + """ + Simple implementation of a ReadablePath class for local filesystem paths. + """ + __slots__ = ('info',) + + def __init__(self, *pathsegments): + super().__init__(*pathsegments) + self.info = LocalPathInfo(self) + + def __fspath__(self): + return str(self) + + def __open_rb__(self, buffering=-1): + return open(self, 'rb') + + def iterdir(self): + return (self / name for name in os.listdir(self)) + + def readlink(self): + return self.with_segments(os.readlink(self)) + + +class WritableLocalPath(_WritablePath, LexicalPath): + """ + Simple implementation of a WritablePath class for local filesystem paths. + """ + + __slots__ = () + + def __fspath__(self): + return str(self) + + def __open_wb__(self, buffering=-1): + return open(self, 'wb') + + def mkdir(self, mode=0o777): + os.mkdir(self, mode) + + def symlink_to(self, target, target_is_directory=False): + os.symlink(target, self, target_is_directory) diff --git a/Lib/test/test_pathlib/support/zip_path.py b/Lib/test/test_pathlib/support/zip_path.py new file mode 100644 index 00000000000..2905260c9df --- /dev/null +++ b/Lib/test/test_pathlib/support/zip_path.py @@ -0,0 +1,336 @@ +""" +Implementations of ReadablePath and WritablePath for zip file members, for use +in pathlib tests. + +ZipPathGround is also defined here. It helps establish the "ground truth" +about zip file members in tests. +""" + +import errno +import io +import posixpath +import stat +import zipfile +from stat import S_IFMT, S_ISDIR, S_ISREG, S_ISLNK + +from . import is_pypi + +if is_pypi: + from pathlib_abc import PathInfo, _ReadablePath, _WritablePath +else: + from pathlib.types import PathInfo, _ReadablePath, _WritablePath + + +class ZipPathGround: + can_symlink = True + + def __init__(self, path_cls): + self.path_cls = path_cls + + def setup(self, local_suffix=""): + return self.path_cls(zip_file=zipfile.ZipFile(io.BytesIO(), "w")) + + def teardown(self, root): + root.zip_file.close() + + def create_file(self, path, data=b''): + path.zip_file.writestr(str(path), data) + + def create_dir(self, path): + zip_info = zipfile.ZipInfo(str(path) + '/') + zip_info.external_attr |= stat.S_IFDIR << 16 + zip_info.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY + path.zip_file.writestr(zip_info, '') + + def create_symlink(self, path, target): + zip_info = zipfile.ZipInfo(str(path)) + zip_info.external_attr = stat.S_IFLNK << 16 + path.zip_file.writestr(zip_info, target.encode()) + + def create_hierarchy(self, p): + # Add regular files + self.create_file(p.joinpath('fileA'), b'this is file A\n') + self.create_file(p.joinpath('dirB/fileB'), b'this is file B\n') + self.create_file(p.joinpath('dirC/fileC'), b'this is file C\n') + self.create_file(p.joinpath('dirC/dirD/fileD'), b'this is file D\n') + self.create_file(p.joinpath('dirC/novel.txt'), b'this is a novel\n') + # Add symlinks + self.create_symlink(p.joinpath('linkA'), 'fileA') + self.create_symlink(p.joinpath('linkB'), 'dirB') + self.create_symlink(p.joinpath('dirA/linkC'), '../dirB') + self.create_symlink(p.joinpath('brokenLink'), 'non-existing') + self.create_symlink(p.joinpath('brokenLinkLoop'), 'brokenLinkLoop') + + def readtext(self, p): + with p.zip_file.open(str(p), 'r') as f: + f = io.TextIOWrapper(f, encoding='utf-8') + return f.read() + + def readbytes(self, p): + with p.zip_file.open(str(p), 'r') as f: + return f.read() + + readlink = readtext + + def isdir(self, p): + path_str = str(p) + "/" + return path_str in p.zip_file.NameToInfo + + def isfile(self, p): + info = p.zip_file.NameToInfo.get(str(p)) + if info is None: + return False + return not stat.S_ISLNK(info.external_attr >> 16) + + def islink(self, p): + info = p.zip_file.NameToInfo.get(str(p)) + if info is None: + return False + return stat.S_ISLNK(info.external_attr >> 16) + + +class MissingZipPathInfo(PathInfo): + """ + PathInfo implementation that is used when a zip file member is missing. + """ + __slots__ = () + + def exists(self, follow_symlinks=True): + return False + + def is_dir(self, follow_symlinks=True): + return False + + def is_file(self, follow_symlinks=True): + return False + + def is_symlink(self): + return False + + def resolve(self): + return self + + +missing_zip_path_info = MissingZipPathInfo() + + +class ZipPathInfo(PathInfo): + """ + PathInfo implementation for an existing zip file member. + """ + __slots__ = ('zip_file', 'zip_info', 'parent', 'children') + + def __init__(self, zip_file, parent=None): + self.zip_file = zip_file + self.zip_info = None + self.parent = parent or self + self.children = {} + + def exists(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().exists() + return True + + def is_dir(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().is_dir() + elif self.zip_info is None: + return True + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISDIR(fmt) + else: + return self.zip_info.filename.endswith('/') + + def is_file(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().is_file() + elif self.zip_info is None: + return False + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISREG(fmt) + else: + return not self.zip_info.filename.endswith('/') + + def is_symlink(self): + if self.zip_info is None: + return False + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISLNK(fmt) + else: + return False + + def resolve(self, path=None, create=False, follow_symlinks=True): + """ + Traverse zip hierarchy (parents, children and symlinks) starting + from this PathInfo. This is called from three places: + + - When a zip file member is added to ZipFile.filelist, this method + populates the ZipPathInfo tree (using create=True). + - When ReadableZipPath.info is accessed, this method is finds a + ZipPathInfo entry for the path without resolving any final symlink + (using follow_symlinks=False) + - When ZipPathInfo methods are called with follow_symlinks=True, this + method resolves any symlink in the final path position. + """ + link_count = 0 + stack = path.split('/')[::-1] if path else [] + info = self + while True: + if info.is_symlink() and (follow_symlinks or stack): + link_count += 1 + if link_count >= 40: + return missing_zip_path_info # Symlink loop! + path = info.zip_file.read(info.zip_info).decode() + stack += path.split('/')[::-1] if path else [] + info = info.parent + + if stack: + name = stack.pop() + else: + return info + + if name == '..': + info = info.parent + elif name and name != '.': + if name not in info.children: + if create: + info.children[name] = ZipPathInfo(info.zip_file, info) + else: + return missing_zip_path_info # No such child! + info = info.children[name] + + +class ZipFileList: + """ + `list`-like object that we inject as `ZipFile.filelist`. We maintain a + tree of `ZipPathInfo` objects representing the zip file members. + """ + + __slots__ = ('tree', '_items') + + def __init__(self, zip_file): + self.tree = ZipPathInfo(zip_file) + self._items = [] + for item in zip_file.filelist: + self.append(item) + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items) + + def append(self, item): + self._items.append(item) + self.tree.resolve(item.filename, create=True).zip_info = item + + +class ReadableZipPath(_ReadablePath): + """ + Simple implementation of a ReadablePath class for .zip files. + """ + + __slots__ = ('_segments', 'zip_file') + parser = posixpath + + def __init__(self, *pathsegments, zip_file): + self._segments = pathsegments + self.zip_file = zip_file + if not isinstance(zip_file.filelist, ZipFileList): + zip_file.filelist = ZipFileList(zip_file) + + def __hash__(self): + return hash((str(self), self.zip_file)) + + def __eq__(self, other): + if not isinstance(other, ReadableZipPath): + return NotImplemented + return str(self) == str(other) and self.zip_file is other.zip_file + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, zip_file=self.zip_file) + + @property + def info(self): + tree = self.zip_file.filelist.tree + return tree.resolve(str(self), follow_symlinks=False) + + def __open_rb__(self, buffering=-1): + info = self.info.resolve() + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif info.is_dir(): + raise IsADirectoryError(errno.EISDIR, "Is a directory", self) + return self.zip_file.open(info.zip_info, 'r') + + def iterdir(self): + info = self.info.resolve() + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif not info.is_dir(): + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", self) + return (self / name for name in info.children) + + def readlink(self): + info = self.info + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif not info.is_symlink(): + raise OSError(errno.EINVAL, "Not a symlink", self) + return self.with_segments(self.zip_file.read(info.zip_info).decode()) + + +class WritableZipPath(_WritablePath): + """ + Simple implementation of a WritablePath class for .zip files. + """ + + __slots__ = ('_segments', 'zip_file') + parser = posixpath + + def __init__(self, *pathsegments, zip_file): + self._segments = pathsegments + self.zip_file = zip_file + + def __hash__(self): + return hash((str(self), self.zip_file)) + + def __eq__(self, other): + if not isinstance(other, WritableZipPath): + return NotImplemented + return str(self) == str(other) and self.zip_file is other.zip_file + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, zip_file=self.zip_file) + + def __open_wb__(self, buffering=-1): + return self.zip_file.open(str(self), 'w') + + def mkdir(self, mode=0o777): + zinfo = zipfile.ZipInfo(str(self) + '/') + zinfo.external_attr |= stat.S_IFDIR << 16 + zinfo.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY + self.zip_file.writestr(zinfo, '') + + def symlink_to(self, target, target_is_directory=False): + zinfo = zipfile.ZipInfo(str(self)) + zinfo.external_attr = stat.S_IFLNK << 16 + if target_is_directory: + zinfo.external_attr |= 0x10 + self.zip_file.writestr(zinfo, str(target)) diff --git a/Lib/test/test_pathlib/test_copy.py b/Lib/test/test_pathlib/test_copy.py new file mode 100644 index 00000000000..5f4cf82a031 --- /dev/null +++ b/Lib/test/test_pathlib/test_copy.py @@ -0,0 +1,174 @@ +""" +Tests for copying from pathlib.types._ReadablePath to _WritablePath. +""" + +import contextlib +import unittest + +from .support import is_pypi +from .support.local_path import LocalPathGround +from .support.zip_path import ZipPathGround, ReadableZipPath, WritableZipPath + + +class CopyTestBase: + def setUp(self): + self.source_root = self.source_ground.setup() + self.source_ground.create_hierarchy(self.source_root) + self.target_root = self.target_ground.setup(local_suffix="_target") + + def tearDown(self): + self.source_ground.teardown(self.source_root) + self.target_ground.teardown(self.target_root) + + def test_copy_file(self): + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_file_empty(self): + source = self.source_root / 'empty' + target = self.target_root / 'copyA' + self.source_ground.create_file(source, b'') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.target_ground.readbytes(result), b'') + + def test_copy_file_to_existing_file(self): + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + self.target_ground.create_file(target, b'this is a copy\n') + with contextlib.ExitStack() as stack: + if isinstance(target, WritableZipPath): + stack.enter_context(self.assertWarns(UserWarning)) + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_file_to_directory(self): + if isinstance(self.target_root, WritableZipPath): + self.skipTest('needs local target') + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + self.target_ground.create_dir(target) + self.assertRaises(OSError, source.copy, target) + + def test_copy_file_to_itself(self): + source = self.source_root / 'fileA' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir(self): + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertTrue(self.target_ground.isfile(target / 'fileC')) + self.assertEqual(self.target_ground.readtext(target / 'fileC'), 'this is file C\n') + self.assertTrue(self.target_ground.isdir(target / 'dirD')) + self.assertTrue(self.target_ground.isfile(target / 'dirD' / 'fileD')) + self.assertEqual(self.target_ground.readtext(target / 'dirD' / 'fileD'), 'this is file D\n') + + def test_copy_dir_follow_symlinks_true(self): + if not self.source_ground.can_symlink: + self.skipTest('needs symlink support on source') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.source_ground.create_symlink(source / 'linkC', 'fileC') + self.source_ground.create_symlink(source / 'linkD', 'dirD') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertFalse(self.target_ground.islink(target / 'linkC')) + self.assertTrue(self.target_ground.isfile(target / 'linkC')) + self.assertEqual(self.target_ground.readtext(target / 'linkC'), 'this is file C\n') + self.assertFalse(self.target_ground.islink(target / 'linkD')) + self.assertTrue(self.target_ground.isdir(target / 'linkD')) + self.assertTrue(self.target_ground.isfile(target / 'linkD' / 'fileD')) + self.assertEqual(self.target_ground.readtext(target / 'linkD' / 'fileD'), 'this is file D\n') + + def test_copy_dir_follow_symlinks_false(self): + if not self.source_ground.can_symlink: + self.skipTest('needs symlink support on source') + if not self.target_ground.can_symlink: + self.skipTest('needs symlink support on target') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.source_ground.create_symlink(source / 'linkC', 'fileC') + self.source_ground.create_symlink(source / 'linkD', 'dirD') + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertTrue(self.target_ground.islink(target / 'linkC')) + self.assertEqual(self.target_ground.readlink(target / 'linkC'), 'fileC') + self.assertTrue(self.target_ground.islink(target / 'linkD')) + self.assertEqual(self.target_ground.readlink(target / 'linkD'), 'dirD') + + def test_copy_dir_to_existing_directory(self): + if isinstance(self.target_root, WritableZipPath): + self.skipTest('needs local target') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.target_ground.create_dir(target) + self.assertRaises(FileExistsError, source.copy, target) + + def test_copy_dir_to_itself(self): + source = self.source_root / 'dirC' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_into_itself(self): + source = self.source_root / 'dirC' + target = self.source_root / 'dirC' / 'dirD' / 'copyC' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + def test_copy_into(self): + source = self.source_root / 'fileA' + target_dir = self.target_root / 'dirA' + self.target_ground.create_dir(target_dir) + result = source.copy_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertTrue(self.target_ground.isfile(result)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_into_empty_name(self): + source = self.source_root.with_segments() + target_dir = self.target_root / 'dirA' + self.target_ground.create_dir(target_dir) + self.assertRaises(ValueError, source.copy_into, target_dir) + + +class ZipToZipPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = ZipPathGround(ReadableZipPath) + target_ground = ZipPathGround(WritableZipPath) + + +if not is_pypi: + from pathlib import Path + + class ZipToLocalPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = ZipPathGround(ReadableZipPath) + target_ground = LocalPathGround(Path) + + + class LocalToZipPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = LocalPathGround(Path) + target_ground = ZipPathGround(WritableZipPath) + + + class LocalToLocalPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = LocalPathGround(Path) + target_ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py new file mode 100644 index 00000000000..6b51a09e5ac --- /dev/null +++ b/Lib/test/test_pathlib/test_join.py @@ -0,0 +1,395 @@ +""" +Tests for pathlib.types._JoinablePath +""" + +import unittest +import threading +from test.support import threading_helper + +from .support import is_pypi +from .support.lexical_path import LexicalPath + +if is_pypi: + from pathlib_abc import _PathParser, _JoinablePath +else: + from pathlib.types import _PathParser, _JoinablePath + + +class JoinTestBase: + def test_is_joinable(self): + p = self.cls() + self.assertIsInstance(p, _JoinablePath) + + def test_parser(self): + self.assertIsInstance(self.cls.parser, _PathParser) + + def test_constructor(self): + P = self.cls + p = P('a') + self.assertIsInstance(p, P) + P() + P('a', 'b', 'c') + P('/a', 'b', 'c') + P('a/b/c') + P('/a/b/c') + + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.parent.session_id) + for parent in p.parents: + self.assertEqual(42, parent.session_id) + + def test_join(self): + P = self.cls + sep = self.cls.parser.sep + p = P(f'a{sep}b') + pp = p.joinpath('c') + self.assertEqual(pp, P(f'a{sep}b{sep}c')) + self.assertIs(type(pp), type(p)) + pp = p.joinpath('c', 'd') + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = p.joinpath(f'{sep}c') + self.assertEqual(pp, P(f'{sep}c')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + sep = self.cls.parser.sep + p = P(f'a{sep}b') + pp = p / 'c' + self.assertEqual(pp, P(f'a{sep}b{sep}c')) + self.assertIs(type(pp), type(p)) + pp = p / f'c{sep}d' + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = p / 'c' / 'd' + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = 'c' / p / 'd' + self.assertEqual(pp, P(f'c{sep}a{sep}b{sep}d')) + pp = p/ f'{sep}c' + self.assertEqual(pp, P(f'{sep}c')) + + def test_full_match(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').full_match('b.py')) + self.assertFalse(P('a/b.py').full_match('b.py')) + self.assertFalse(P('/a/b.py').full_match('b.py')) + self.assertFalse(P('a.py').full_match('b.py')) + self.assertFalse(P('b/py').full_match('b.py')) + self.assertFalse(P('/a.py').full_match('b.py')) + self.assertFalse(P('b.py/c').full_match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').full_match('*.py')) + self.assertFalse(P('a/b.py').full_match('*.py')) + self.assertFalse(P('/a/b.py').full_match('*.py')) + self.assertFalse(P('b.pyc').full_match('*.py')) + self.assertFalse(P('b./py').full_match('*.py')) + self.assertFalse(P('b.py/c').full_match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('/d/ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('a.py').full_match('a*/*.py')) + self.assertFalse(P('/dab/c.py').full_match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').full_match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').full_match('/*.py')) + self.assertFalse(P('b.py').full_match('/*.py')) + self.assertFalse(P('a/b.py').full_match('/*.py')) + self.assertFalse(P('/a/b.py').full_match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').full_match('/a/*.py')) + self.assertFalse(P('/ab.py').full_match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').full_match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').full_match('**')) + self.assertTrue(P('c.py').full_match('**')) + self.assertTrue(P('a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('/**')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**')) + self.assertTrue(P('/a/b/c.py').full_match('**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').full_match('**/a.py')) + self.assertFalse(P('c.py').full_match('c/**')) + self.assertFalse(P('a/b/c.py').full_match('**/a')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) + # Matching against empty path + self.assertFalse(P('').full_match('*')) + self.assertTrue(P('').full_match('**')) + self.assertFalse(P('').full_match('**/*')) + # Matching with empty pattern + self.assertTrue(P('').full_match('')) + self.assertTrue(P('.').full_match('.')) + self.assertFalse(P('/').full_match('')) + self.assertFalse(P('/').full_match('.')) + self.assertFalse(P('foo').full_match('')) + self.assertFalse(P('foo').full_match('.')) + + def test_parts(self): + # `parts` returns a tuple. + sep = self.cls.parser.sep + P = self.cls + p = P(f'a{sep}b') + parts = p.parts + self.assertEqual(parts, ('a', 'b')) + # When the path is absolute, the anchor is a separate part. + p = P(f'{sep}a{sep}b') + parts = p.parts + self.assertEqual(parts, (sep, 'a', 'b')) + + @threading_helper.requires_working_threading() + def test_parts_multithreaded(self): + P = self.cls + + NUM_THREADS = 10 + NUM_ITERS = 10 + + for _ in range(NUM_ITERS): + b = threading.Barrier(NUM_THREADS) + path = P('a') / 'b' / 'c' / 'd' / 'e' + expected = ('a', 'b', 'c', 'd', 'e') + + def check_parts(): + b.wait() + self.assertEqual(path.parts, expected) + + threads = [threading.Thread(target=check_parts) for _ in range(NUM_THREADS)] + with threading_helper.start_threads(threads): + pass + + def test_parent(self): + # Relative + P = self.cls + p = P('a/b/c') + self.assertEqual(p.parent, P('a/b')) + self.assertEqual(p.parent.parent, P('a')) + self.assertEqual(p.parent.parent.parent, P('')) + self.assertEqual(p.parent.parent.parent.parent, P('')) + # Anchored + p = P('/a/b/c') + self.assertEqual(p.parent, P('/a/b')) + self.assertEqual(p.parent.parent, P('/a')) + self.assertEqual(p.parent.parent.parent, P('/')) + self.assertEqual(p.parent.parent.parent.parent, P('/')) + + def test_parents(self): + # Relative + P = self.cls + p = P('a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('a/b')) + self.assertEqual(par[1], P('a')) + self.assertEqual(par[2], P('')) + self.assertEqual(par[-1], P('')) + self.assertEqual(par[-2], P('a')) + self.assertEqual(par[-3], P('a/b')) + self.assertEqual(par[0:1], (P('a/b'),)) + self.assertEqual(par[:2], (P('a/b'), P('a'))) + self.assertEqual(par[:-1], (P('a/b'), P('a'))) + self.assertEqual(par[1:], (P('a'), P(''))) + self.assertEqual(par[::2], (P('a/b'), P(''))) + self.assertEqual(par[::-1], (P(''), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + with self.assertRaises(TypeError): + par[0] = p + # Anchored + p = P('/a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('/a/b')) + self.assertEqual(par[1], P('/a')) + self.assertEqual(par[2], P('/')) + self.assertEqual(par[-1], P('/')) + self.assertEqual(par[-2], P('/a')) + self.assertEqual(par[-3], P('/a/b')) + self.assertEqual(par[0:1], (P('/a/b'),)) + self.assertEqual(par[:2], (P('/a/b'), P('/a'))) + self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) + self.assertEqual(par[1:], (P('/a'), P('/'))) + self.assertEqual(par[::2], (P('/a/b'), P('/'))) + self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) + self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + + def test_anchor(self): + P = self.cls + sep = self.cls.parser.sep + self.assertEqual(P('').anchor, '') + self.assertEqual(P(f'a{sep}b').anchor, '') + self.assertEqual(P(sep).anchor, sep) + self.assertEqual(P(f'{sep}a{sep}b').anchor, sep) + + def test_name(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('/').name, '') + self.assertEqual(P('a/b').name, 'b') + self.assertEqual(P('/a/b').name, 'b') + self.assertEqual(P('a/b.py').name, 'b.py') + self.assertEqual(P('/a/b.py').name, 'b.py') + + def test_suffix(self): + P = self.cls + self.assertEqual(P('').suffix, '') + self.assertEqual(P('.').suffix, '') + self.assertEqual(P('..').suffix, '') + self.assertEqual(P('/').suffix, '') + self.assertEqual(P('a/b').suffix, '') + self.assertEqual(P('/a/b').suffix, '') + self.assertEqual(P('/a/b/.').suffix, '') + self.assertEqual(P('a/b.py').suffix, '.py') + self.assertEqual(P('/a/b.py').suffix, '.py') + self.assertEqual(P('a/.hgrc').suffix, '') + self.assertEqual(P('/a/.hgrc').suffix, '') + self.assertEqual(P('a/.hg.rc').suffix, '.rc') + self.assertEqual(P('/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') + + def test_suffixes(self): + P = self.cls + self.assertEqual(P('').suffixes, []) + self.assertEqual(P('.').suffixes, []) + self.assertEqual(P('/').suffixes, []) + self.assertEqual(P('a/b').suffixes, []) + self.assertEqual(P('/a/b').suffixes, []) + self.assertEqual(P('/a/b/.').suffixes, []) + self.assertEqual(P('a/b.py').suffixes, ['.py']) + self.assertEqual(P('/a/b.py').suffixes, ['.py']) + self.assertEqual(P('a/.hgrc').suffixes, []) + self.assertEqual(P('/a/.hgrc').suffixes, []) + self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) + + def test_stem(self): + P = self.cls + self.assertEqual(P('..').stem, '..') + self.assertEqual(P('').stem, '') + self.assertEqual(P('/').stem, '') + self.assertEqual(P('a/b').stem, 'b') + self.assertEqual(P('a/b.py').stem, 'b') + self.assertEqual(P('a/.hgrc').stem, '.hgrc') + self.assertEqual(P('a/.hg.rc').stem, '.hg') + self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') + + def test_with_name(self): + P = self.cls + self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + self.assertRaises(ValueError, P('a/b').with_name, '/c') + self.assertRaises(ValueError, P('a/b').with_name, 'c/') + self.assertRaises(ValueError, P('a/b').with_name, 'c/d') + + def test_with_stem(self): + P = self.cls + self.assertEqual(P('a/b').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) + self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) + self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '/c') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') + + def test_with_suffix(self): + P = self.cls + self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) + self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) + # Stripping suffix. + self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) + self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('a/b').with_suffix, '/') + self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') + self.assertRaises(ValueError, P('a/b').with_suffix, './.d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + self.assertRaises(TypeError, P('a/b').with_suffix, None) + + +class LexicalPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalPath + + +if not is_pypi: + from pathlib import PurePath, Path + + class PurePathJoinTest(JoinTestBase, unittest.TestCase): + cls = PurePath + + class PathJoinTest(JoinTestBase, unittest.TestCase): + cls = Path + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join_posix.py b/Lib/test/test_pathlib/test_join_posix.py new file mode 100644 index 00000000000..d24fb1087c9 --- /dev/null +++ b/Lib/test/test_pathlib/test_join_posix.py @@ -0,0 +1,51 @@ +""" +Tests for Posix-flavoured pathlib.types._JoinablePath +""" + +import os +import unittest + +from .support import is_pypi +from .support.lexical_path import LexicalPosixPath + + +class JoinTestBase: + def test_join(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + +class LexicalPosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalPosixPath + + +if not is_pypi: + from pathlib import PurePosixPath, PosixPath + + class PurePosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PurePosixPath + + if os.name != 'nt': + class PosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PosixPath + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join_windows.py b/Lib/test/test_pathlib/test_join_windows.py new file mode 100644 index 00000000000..2cc634f25ef --- /dev/null +++ b/Lib/test/test_pathlib/test_join_windows.py @@ -0,0 +1,290 @@ +""" +Tests for Windows-flavoured pathlib.types._JoinablePath +""" + +import os +import unittest + +from .support import is_pypi +from .support.lexical_path import LexicalWindowsPath + + +class JoinTestBase: + def test_join(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P(r'C:/a/b\x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P(r'c:/a/b\x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('c:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath('./d:s') + self.assertEqual(pp, P(r'C:/a/b\./d:s')) + pp = p.joinpath('./dd:s') + self.assertEqual(pp, P(r'C:/a/b\./dd:s')) + pp = p.joinpath('E:d:s') + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//server').joinpath('share') + self.assertEqual(pp, P(r'//server\share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P(r'//./BootPartition\Windows')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P(r'C:/a/b\x/y')) + self.assertEqual(p / 'x' / 'y', P(r'C:/a/b\x\y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P(r'C:/x\y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P(r'c:/a/b\x/y')) + self.assertEqual(p / 'c:/x/y', P('c:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / './d:s', P(r'C:/a/b\./d:s')) + self.assertEqual(p / './dd:s', P(r'C:/a/b\./dd:s')) + self.assertEqual(p / 'E:d:s', P('E:d:s')) + + def test_str(self): + p = self.cls(r'a\b\c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls(r'c:\a\b\c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('\\\\a\\b\\') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls(r'\\a\b\c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls(r'\\a\b\c\d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + + def test_parts(self): + P = self.cls + p = P(r'c:a\b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P(r'c:\a\b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P(r'\\a\b\c\d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) + + def test_parent(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b/')) + self.assertEqual(p.parent.parent.parent, P('//a/b/')) + + def test_parents(self): + # Anchored + P = self.cls + p = P('z:a/b') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b/')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b/'))) + self.assertEqual(par[1:], (P('//a/b/'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b/'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b/')]) + with self.assertRaises(IndexError): + par[2] + + def test_anchor(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:\\').anchor, 'c:\\') + self.assertEqual(P('c:\\a\\b\\').anchor, 'c:\\') + self.assertEqual(P('\\\\a\\b\\').anchor, '\\\\a\\b\\') + self.assertEqual(P('\\\\a\\b\\c\\d').anchor, '\\\\a\\b\\') + + def test_name(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + + def test_stem(self): + P = self.cls + self.assertEqual(P('c:').stem, '') + self.assertEqual(P('c:..').stem, '..') + self.assertEqual(P('c:/').stem, '') + self.assertEqual(P('c:a/b').stem, 'b') + self.assertEqual(P('c:a/b.py').stem, 'b') + self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') + self.assertEqual(P('c:a/.hg.rc').stem, '.hg') + self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + + def test_suffix(self): + P = self.cls + self.assertEqual(P('c:').suffix, '') + self.assertEqual(P('c:/').suffix, '') + self.assertEqual(P('c:a/b').suffix, '') + self.assertEqual(P('c:/a/b').suffix, '') + self.assertEqual(P('c:a/b.py').suffix, '.py') + self.assertEqual(P('c:/a/b.py').suffix, '.py') + self.assertEqual(P('c:a/.hgrc').suffix, '') + self.assertEqual(P('c:/a/.hgrc').suffix, '') + self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') + self.assertEqual(P('//My.py/Share.php').suffix, '') + self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') + + def test_suffixes(self): + P = self.cls + self.assertEqual(P('c:').suffixes, []) + self.assertEqual(P('c:/').suffixes, []) + self.assertEqual(P('c:a/b').suffixes, []) + self.assertEqual(P('c:/a/b').suffixes, []) + self.assertEqual(P('c:a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:a/.hgrc').suffixes, []) + self.assertEqual(P('c:/a/.hgrc').suffixes, []) + self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('//My.py/Share.php').suffixes, []) + self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) + + def test_with_name(self): + P = self.cls + self.assertEqual(P(r'c:a\b').with_name('d.xml'), P(r'c:a\d.xml')) + self.assertEqual(P(r'c:\a\b').with_name('d.xml'), P(r'c:\a\d.xml')) + self.assertEqual(P(r'c:a\Dot ending.').with_name('d.xml'), P(r'c:a\d.xml')) + self.assertEqual(P(r'c:\a\Dot ending.').with_name('d.xml'), P(r'c:\a\d.xml')) + self.assertRaises(ValueError, P(r'c:a\b').with_name, r'd:\e') + self.assertRaises(ValueError, P(r'c:a\b').with_name, r'\\My\Share') + + def test_with_stem(self): + P = self.cls + self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) + self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') + + def test_with_suffix(self): + P = self.cls + self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) + self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) + + +class LexicalWindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalWindowsPath + + +if not is_pypi: + from pathlib import PureWindowsPath, WindowsPath + + class PureWindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PureWindowsPath + + if os.name == 'nt': + class WindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = WindowsPath + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py similarity index 52% rename from Lib/test/test_pathlib.py rename to Lib/test/test_pathlib/test_pathlib.py index c1696bb3737..a1ea69a6b90 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,43 +1,101 @@ +import collections import contextlib -import collections.abc import io import os import sys import errno +import ntpath import pathlib import pickle +import posixpath import socket import stat import tempfile import unittest from unittest import mock +from urllib.request import pathname2url from test.support import import_helper -from test.support import set_recursion_limit -from test.support import is_emscripten, is_wasi +from test.support import cpython_only +from test.support import is_emscripten, is_wasi, is_wasm32 +from test.support import infinite_recursion from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath - +from test.support.os_helper import TESTFN, FS_NONASCII, FakePath +try: + import fcntl +except ImportError: + fcntl = None try: import grp, pwd except ImportError: grp = pwd = None +try: + import posix +except ImportError: + posix = None + + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + + +def patch_replace(old_test): + def new_replace(self, target): + raise OSError(errno.EXDEV, "Cross-device link", self, target) + + def new_test(self): + old_replace = self.cls.replace + self.cls.replace = new_replace + try: + old_test(self) + finally: + self.cls.replace = old_replace + return new_test + + +_tests_needing_posix = set() +_tests_needing_windows = set() +_tests_needing_symlinks = set() + +def needs_posix(fn): + """Decorator that marks a test as requiring a POSIX-flavoured path class.""" + _tests_needing_posix.add(fn.__name__) + return fn + +def needs_windows(fn): + """Decorator that marks a test as requiring a Windows-flavoured path class.""" + _tests_needing_windows.add(fn.__name__) + return fn + +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertIsSubclass(pathlib.UnsupportedOperation, NotImplementedError) + self.assertIsInstance(pathlib.UnsupportedOperation(), NotImplementedError) + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports("pathlib", {"shutil"}) # # Tests for the pure classes. # -class _BasePurePathSubclass(object): - def __init__(self, *pathsegments, session_id): - super().__init__(*pathsegments) - self.session_id = session_id - - def with_segments(self, *pathsegments): - return type(self)(*pathsegments, session_id=self.session_id) - +class PurePathTest(unittest.TestCase): + cls = pathlib.PurePath -class _BasePurePathTest(object): + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) # Keys are canonical paths, values are list of tuples of arguments # supposed to produce equal paths. @@ -56,52 +114,15 @@ class _BasePurePathTest(object): } def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_posix and self.cls.parser is not posixpath: + self.skipTest('requires POSIX-flavoured path class') + if name in _tests_needing_windows and self.cls.parser is posixpath: + self.skipTest('requires Windows-flavoured path class') p = self.cls('a') - self.flavour = p._flavour - self.sep = self.flavour.sep - self.altsep = self.flavour.altsep - - def test_constructor_common(self): - P = self.cls - p = P('a') - self.assertIsInstance(p, P) - P('a', 'b', 'c') - P('/a', 'b', 'c') - P('a/b/c') - P('/a/b/c') - P(FakePath("a/b/c")) - self.assertEqual(P(P('a')), P('a')) - self.assertEqual(P(P('a'), 'b'), P('a/b')) - self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) - self.assertEqual(P(P('./a:b')), P('./a:b')) - - def test_bytes(self): - P = self.cls - message = (r"argument should be a str or an os\.PathLike object " - r"where __fspath__ returns a str, not 'bytes'") - with self.assertRaisesRegex(TypeError, message): - P(b'a') - with self.assertRaisesRegex(TypeError, message): - P(b'a', 'b') - with self.assertRaisesRegex(TypeError, message): - P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object @@ -122,93 +143,18 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') - @unittest.skip("TODO: RUSTPYTHON; PyObject::set_slot index out of bounds") - def test_with_segments_common(self): - class P(_BasePurePathSubclass, self.cls): - pass - p = P('foo', 'bar', session_id=42) - self.assertEqual(42, (p / 'foo').session_id) - self.assertEqual(42, ('foo' / p).session_id) - self.assertEqual(42, p.joinpath('foo').session_id) - self.assertEqual(42, p.with_name('foo').session_id) - self.assertEqual(42, p.with_stem('foo').session_id) - self.assertEqual(42, p.with_suffix('.foo').session_id) - self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) - self.assertEqual(42, p.parent.session_id) - for parent in p.parents: - self.assertEqual(42, parent.session_id) - - def _get_drive_root_parts(self, parts): - path = self.cls(*parts) - return path.drive, path.root, path.parts - - def _check_drive_root_parts(self, arg, *expected): - sep = self.flavour.sep - actual = self._get_drive_root_parts([x.replace('/', sep) for x in arg]) - self.assertEqual(actual, expected) - if altsep := self.flavour.altsep: - actual = self._get_drive_root_parts([x.replace('/', altsep) for x in arg]) - self.assertEqual(actual, expected) - - def test_drive_root_parts_common(self): - check = self._check_drive_root_parts - sep = self.flavour.sep - # Unanchored parts. - check((), '', '', ()) - check(('a',), '', '', ('a',)) - check(('a/',), '', '', ('a',)) - check(('a', 'b'), '', '', ('a', 'b')) - # Expansion. - check(('a/b',), '', '', ('a', 'b')) - check(('a/b/',), '', '', ('a', 'b')) - check(('a', 'b/c', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Collapsing and stripping excess slashes. - check(('a', 'b//c', 'd'), '', '', ('a', 'b', 'c', 'd')) - check(('a', 'b/c/', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Eliminating standalone dots. - check(('.',), '', '', ()) - check(('.', '.', 'b'), '', '', ('b',)) - check(('a', '.', 'b'), '', '', ('a', 'b')) - check(('a', '.', '.'), '', '', ('a',)) - # The first part is anchored. - check(('/a/b',), '', sep, (sep, 'a', 'b')) - check(('/a', 'b'), '', sep, (sep, 'a', 'b')) - check(('/a/', 'b'), '', sep, (sep, 'a', 'b')) - # Ignoring parts before an anchored part. - check(('a', '/b', 'c'), '', sep, (sep, 'b', 'c')) - check(('a', '/b', '/c'), '', sep, (sep, 'c')) - - def test_join_common(self): - P = self.cls - p = P('a/b') - pp = p.joinpath('c') - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p.joinpath('c', 'd') - self.assertEqual(pp, P('a/b/c/d')) - pp = p.joinpath(P('c')) - self.assertEqual(pp, P('a/b/c')) - pp = p.joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div_common(self): - # Basically the same as joinpath(). - P = self.cls - p = P('a/b') - pp = p / 'c' - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p / 'c/d' - self.assertEqual(pp, P('a/b/c/d')) - pp = p / 'c' / 'd' - self.assertEqual(pp, P('a/b/c/d')) - pp = 'c' / p / 'd' - self.assertEqual(pp, P('c/a/b/d')) - pp = p / P('c') - self.assertEqual(pp, P('a/b/c')) - pp = p/ '/c' - self.assertEqual(pp, P('/c')) + @needs_windows + def test_str_subclass_windows(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') def _check_str(self, expected, args): p = self.cls(*args) @@ -218,27 +164,140 @@ def test_str_common(self): # Canonicalized paths roundtrip. for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): self._check_str(pathstr, (pathstr,)) + # Other tests for str() are in test_equivalences(). + + @needs_windows + def test_str_windows(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_concrete_parser(self): + if self.cls is pathlib.PurePosixPath: + expected = posixpath + elif self.cls is pathlib.PureWindowsPath: + expected = ntpath + else: + expected = os.path + p = self.cls('a') + self.assertIs(p.parser, expected) + + def test_different_parsers_unequal(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_parsers_unordered(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + + def test_constructor_nested(self): + P = self.cls + P(FakePath("a/b/c")) + self.assertEqual(P(P('a')), P('a')) + self.assertEqual(P(P('a'), 'b'), P('a/b')) + self.assertEqual(P(P('a'), P('b')), P('a/b')) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) + + @needs_windows + def test_constructor_nested_foreign_flavour(self): + # See GH-125069. + p1 = pathlib.PurePosixPath('b/c:\\d') + p2 = pathlib.PurePosixPath('b/', 'c:\\d') + self.assertEqual(p1, p2) + self.assertEqual(self.cls(p1), self.cls('b/c:/d')) + self.assertEqual(self.cls(p2), self.cls('b/c:/d')) + + def _check_parse_path(self, raw_path, *expected): + sep = self.parser.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.parser.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.parser.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') # Special case for the empty path. self._check_str('.', ('',)) - # Other tests for str() are in test_equivalences(). - def test_as_posix_common(self): + def test_join_nested(self): P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). + p = P('a/b').joinpath(P('c')) + self.assertEqual(p, P('a/b/c')) - def test_as_bytes_common(self): - sep = os.fsencode(self.sep) + def test_div_nested(self): P = self.cls - self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + p = P('a/b') / P('c') + self.assertEqual(p, P('a/b/c')) - def test_as_uri_common(self): + def test_pickling_common(self): P = self.cls - with self.assertRaises(ValueError): - P('a').as_uri() - with self.assertRaises(ValueError): - P().as_uri() + for pathstr in ('a', 'a/', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c', 'a/b/c/'): + with self.subTest(pathstr=pathstr): + p = P(pathstr) + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertIs(pp.__class__, p.__class__) + self.assertEqual(pp, p) + self.assertEqual(hash(pp), hash(p)) + self.assertEqual(str(pp), str(p)) + + def test_unpicking_3_13(self): + data = (b"\x80\x04\x95'\x00\x00\x00\x00\x00\x00\x00\x8c\x0e" + b"pathlib._local\x94\x8c\rPurePosixPath\x94\x93\x94)R\x94.") + p = pickle.loads(data) + self.assertIsInstance(p, pathlib.PurePosixPath) def test_repr_common(self): for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): @@ -247,21 +306,63 @@ def test_repr_common(self): clsname = p.__class__.__name__ r = repr(p) # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, clsname + '(') + self.assertEndsWith(r, ')') inner = r[len(clsname) + 1 : -1] self.assertEqual(eval(inner), p.as_posix()) - def test_repr_roundtrips(self): + def test_fspath_common(self): + P = self.cls + p = P('a/b') + self._check_str(p.__fspath__(), ('a/b',)) + self._check_str(os.fspath(p), ('a/b',)) + + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + + def test_bytes_exc_message(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaisesRegex(TypeError, message): + P(b'a', 'b') + with self.assertRaisesRegex(TypeError, message): + P('a', b'b') + + def test_as_bytes_common(self): + sep = os.fsencode(self.sep) + P = self.cls + self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + + def test_as_posix_common(self): + P = self.cls for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - r = repr(p) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). def test_eq_common(self): P = self.cls @@ -276,58 +377,31 @@ def test_eq_common(self): self.assertNotEqual(P(), {}) self.assertNotEqual(P(), int) - def test_match_common(self): - P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') - # Simple relative pattern. - self.assertTrue(P('b.py').match('b.py')) - self.assertTrue(P('a/b.py').match('b.py')) - self.assertTrue(P('/a/b.py').match('b.py')) - self.assertFalse(P('a.py').match('b.py')) - self.assertFalse(P('b/py').match('b.py')) - self.assertFalse(P('/a.py').match('b.py')) - self.assertFalse(P('b.py/c').match('b.py')) - # Wildcard relative pattern. - self.assertTrue(P('b.py').match('*.py')) - self.assertTrue(P('a/b.py').match('*.py')) - self.assertTrue(P('/a/b.py').match('*.py')) - self.assertFalse(P('b.pyc').match('*.py')) - self.assertFalse(P('b./py').match('*.py')) - self.assertFalse(P('b.py/c').match('*.py')) - # Multi-part relative pattern. - self.assertTrue(P('ab/c.py').match('a*/*.py')) - self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) - self.assertFalse(P('a.py').match('a*/*.py')) - self.assertFalse(P('/dab/c.py').match('a*/*.py')) - self.assertFalse(P('ab/c.py/d').match('a*/*.py')) - # Absolute pattern. - self.assertTrue(P('/b.py').match('/*.py')) - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('a/b.py').match('/*.py')) - self.assertFalse(P('/a/b.py').match('/*.py')) - # Multi-part absolute pattern. - self.assertTrue(P('/a/b.py').match('/a/*.py')) - self.assertFalse(P('/ab.py').match('/a/*.py')) - self.assertFalse(P('/a/b/c.py').match('/a/*.py')) - # Multi-part glob-style pattern. - self.assertFalse(P('/a/b/c.py').match('/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - # Case-sensitive flag - self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) - self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) - self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) - self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) - # Matching against empty path - self.assertFalse(P().match('*')) - self.assertTrue(P().match('**')) - self.assertFalse(P().match('**/*')) - - def test_ordering_common(self): - # Ordering is tuple-alike. - def assertLess(a, b): - self.assertLess(a, b) - self.assertGreater(b, a) + def test_equivalences(self, equivalences=None): + if equivalences is None: + equivalences = self.equivalences + for k, tuples in equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + + def test_ordering_common(self): + # Ordering is tuple-alike. + def assertLess(a, b): + self.assertLess(a, b) + self.assertGreater(b, a) P = self.cls a = P('a') b = P('a/b') @@ -351,103 +425,29 @@ def assertLess(a, b): with self.assertRaises(TypeError): P() < {} - def test_parts_common(self): - # `parts` returns a tuple. - sep = self.sep - P = self.cls - p = P('a/b') - parts = p.parts - self.assertEqual(parts, ('a', 'b')) - # When the path is absolute, the anchor is a separate part. - p = P('/a/b') - parts = p.parts - self.assertEqual(parts, (sep, 'a', 'b')) + def make_uri(self, path): + if isinstance(path, pathlib.Path): + return path.as_uri() + with self.assertWarns(DeprecationWarning): + return path.as_uri() - def test_fspath_common(self): + def test_as_uri_common(self): P = self.cls - p = P('a/b') - self._check_str(p.__fspath__(), ('a/b',)) - self._check_str(os.fspath(p), ('a/b',)) - - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) + with self.assertRaises(ValueError): + self.make_uri(P('a')) + with self.assertRaises(ValueError): + self.make_uri(P()) - def test_parent_common(self): - # Relative - P = self.cls - p = P('a/b/c') - self.assertEqual(p.parent, P('a/b')) - self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) - # Anchored - p = P('/a/b/c') - self.assertEqual(p.parent, P('/a/b')) - self.assertEqual(p.parent.parent, P('/a')) - self.assertEqual(p.parent.parent.parent, P('/')) - self.assertEqual(p.parent.parent.parent.parent, P('/')) - - def test_parents_common(self): - # Relative - P = self.cls - p = P('a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('a/b')) - self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) - self.assertEqual(par[-2], P('a')) - self.assertEqual(par[-3], P('a/b')) - self.assertEqual(par[0:1], (P('a/b'),)) - self.assertEqual(par[:2], (P('a/b'), P('a'))) - self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - with self.assertRaises(TypeError): - par[0] = p - # Anchored - p = P('/a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('/a/b')) - self.assertEqual(par[1], P('/a')) - self.assertEqual(par[2], P('/')) - self.assertEqual(par[-1], P('/')) - self.assertEqual(par[-2], P('/a')) - self.assertEqual(par[-3], P('/a/b')) - self.assertEqual(par[0:1], (P('/a/b'),)) - self.assertEqual(par[:2], (P('/a/b'), P('/a'))) - self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) - self.assertEqual(par[1:], (P('/a'), P('/'))) - self.assertEqual(par[::2], (P('/a/b'), P('/'))) - self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) - self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) def test_drive_common(self): P = self.cls @@ -455,6 +455,19 @@ def test_drive_common(self): self.assertEqual(P('/a/b').drive, '') self.assertEqual(P('').drive, '') + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + def test_root_common(self): P = self.cls sep = self.sep @@ -463,298 +476,199 @@ def test_root_common(self): self.assertEqual(P('/').root, sep) self.assertEqual(P('/a/b').root, sep) - def test_anchor_common(self): + @needs_posix + def test_root_posix(self): P = self.cls - sep = self.sep - self.assertEqual(P('').anchor, '') - self.assertEqual(P('a/b').anchor, '') - self.assertEqual(P('/').anchor, sep) - self.assertEqual(P('/a/b').anchor, sep) + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') - def test_name_common(self): + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') self.assertEqual(P('.').name, '') - self.assertEqual(P('/').name, '') - self.assertEqual(P('a/b').name, 'b') - self.assertEqual(P('/a/b').name, 'b') self.assertEqual(P('/a/b/.').name, 'b') - self.assertEqual(P('a/b.py').name, 'b.py') - self.assertEqual(P('/a/b.py').name, 'b.py') - - def test_suffix_common(self): - P = self.cls - self.assertEqual(P('').suffix, '') - self.assertEqual(P('.').suffix, '') - self.assertEqual(P('..').suffix, '') - self.assertEqual(P('/').suffix, '') - self.assertEqual(P('a/b').suffix, '') - self.assertEqual(P('/a/b').suffix, '') - self.assertEqual(P('/a/b/.').suffix, '') - self.assertEqual(P('a/b.py').suffix, '.py') - self.assertEqual(P('/a/b.py').suffix, '.py') - self.assertEqual(P('a/.hgrc').suffix, '') - self.assertEqual(P('/a/.hgrc').suffix, '') - self.assertEqual(P('a/.hg.rc').suffix, '.rc') - self.assertEqual(P('/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') - - def test_suffixes_common(self): - P = self.cls - self.assertEqual(P('').suffixes, []) - self.assertEqual(P('.').suffixes, []) - self.assertEqual(P('/').suffixes, []) - self.assertEqual(P('a/b').suffixes, []) - self.assertEqual(P('/a/b').suffixes, []) - self.assertEqual(P('/a/b/.').suffixes, []) - self.assertEqual(P('a/b.py').suffixes, ['.py']) - self.assertEqual(P('/a/b.py').suffixes, ['.py']) - self.assertEqual(P('a/.hgrc').suffixes, []) - self.assertEqual(P('/a/.hgrc').suffixes, []) - self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem_common(self): + + def test_stem_empty(self): P = self.cls self.assertEqual(P('').stem, '') self.assertEqual(P('.').stem, '') - self.assertEqual(P('..').stem, '..') - self.assertEqual(P('/').stem, '') - self.assertEqual(P('a/b').stem, 'b') - self.assertEqual(P('a/b.py').stem, 'b') - self.assertEqual(P('a/.hgrc').stem, '.hgrc') - self.assertEqual(P('a/.hg.rc').stem, '.hg') - self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name_common(self): - P = self.cls - self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + + @needs_windows + def test_with_name_windows(self): + P = self.cls + self.assertRaises(ValueError, P(r'c:').with_name, 'd.xml') + self.assertRaises(ValueError, P(r'c:\\').with_name, 'd.xml') + self.assertRaises(ValueError, P(r'\\My\Share').with_name, 'd.xml') + # NTFS alternate data streams + self.assertEqual(str(P('a').with_name('d:')), '.\\d:') + self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') + self.assertEqual(P(r'c:a\b').with_name('d:'), P(r'c:a\d:')) + self.assertEqual(P(r'c:a\b').with_name('d:e'), P(r'c:a\d:e')) + + def test_with_name_empty(self): + P = self.cls self.assertRaises(ValueError, P('').with_name, 'd.xml') self.assertRaises(ValueError, P('.').with_name, 'd.xml') self.assertRaises(ValueError, P('/').with_name, 'd.xml') self.assertRaises(ValueError, P('a/b').with_name, '') self.assertRaises(ValueError, P('a/b').with_name, '.') - self.assertRaises(ValueError, P('a/b').with_name, '/c') - self.assertRaises(ValueError, P('a/b').with_name, 'c/') - self.assertRaises(ValueError, P('a/b').with_name, 'c/d') - - def test_with_stem_common(self): - P = self.cls - self.assertEqual(P('a/b').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) - self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) - self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) - self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + + @needs_windows + def test_with_stem_windows(self): + P = self.cls + self.assertRaises(ValueError, P('c:').with_stem, 'd') + self.assertRaises(ValueError, P('c:/').with_stem, 'd') + self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') + # NTFS alternate data streams + self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') + self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) + + def test_with_stem_empty(self): + P = self.cls self.assertRaises(ValueError, P('').with_stem, 'd') self.assertRaises(ValueError, P('.').with_stem, 'd') self.assertRaises(ValueError, P('/').with_stem, 'd') self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - self.assertRaises(ValueError, P('a/b').with_stem, '/c') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') - - def test_with_suffix_common(self): - P = self.cls - self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) - self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) - # Stripping suffix. - self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) - self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') - self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') - self.assertRaises(ValueError, P('a/b').with_suffix, './.d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - self.assertRaises(ValueError, P('a/b').with_suffix, - (self.flavour.sep, 'd')) - def test_relative_to_common(self): + def test_is_reserved_deprecated(self): P = self.cls p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # With several args. with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p.is_reserved() - def test_is_relative_to_common(self): + def test_full_match_case_sensitive(self): P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) + self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False)) - def test_pickling_common(self): + def test_match_empty(self): P = self.cls - p = P('/a/b') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertIs(pp.__class__, p.__class__) - self.assertEqual(pp, p) - self.assertEqual(hash(pp), hash(p)) - self.assertEqual(str(pp), str(p)) + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + def test_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').match('b.py')) + self.assertTrue(P('a/b.py').match('b.py')) + self.assertTrue(P('/a/b.py').match('b.py')) + self.assertFalse(P('a.py').match('b.py')) + self.assertFalse(P('b/py').match('b.py')) + self.assertFalse(P('/a.py').match('b.py')) + self.assertFalse(P('b.py/c').match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').match('*.py')) + self.assertTrue(P('a/b.py').match('*.py')) + self.assertTrue(P('/a/b.py').match('*.py')) + self.assertFalse(P('b.pyc').match('*.py')) + self.assertFalse(P('b./py').match('*.py')) + self.assertFalse(P('b.py/c').match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').match('a*/*.py')) + self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) + self.assertFalse(P('a.py').match('a*/*.py')) + self.assertFalse(P('/dab/c.py').match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').match('/*.py')) + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('a/b.py').match('/*.py')) + self.assertFalse(P('/a/b.py').match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').match('/a/*.py')) + self.assertFalse(P('/ab.py').match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) + # Case-sensitive flag + self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').match('*')) + self.assertFalse(P('').match('**')) + self.assertFalse(P('').match('**/*')) -class PurePosixPathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PurePosixPath + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) - def test_drive_root_parts(self): - check = self._check_drive_root_parts + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + @needs_posix + def test_parse_path_posix(self): + check = self._check_parse_path # Collapsing of excess leading slashes, except for the double-slash # special case. - check(('//a', 'b'), '', '//', ('//', 'a', 'b')) - check(('///a', 'b'), '', '/', ('/', 'a', 'b')) - check(('////a', 'b'), '', '/', ('/', 'a', 'b')) + check('//a/b', '', '//', ['a', 'b']) + check('///a/b', '', '/', ['a', 'b']) + check('////a/b', '', '/', ['a', 'b']) # Paths which look like NT paths aren't treated specially. - check(('c:a',), '', '', ('c:a',)) - check(('c:\\a',), '', '', ('c:\\a',)) - check(('\\a',), '', '', ('\\a',)) + check('c:a', '', '', ['c:a',]) + check('c:\\a', '', '', ['c:\\a',]) + check('\\a', '', '', ['\\a',]) - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): + @needs_posix + def test_eq_posix(self): P = self.cls self.assertNotEqual(P('a/b'), P('A/b')) self.assertEqual(P('/a'), P('///a')) self.assertNotEqual(P('/a'), P('//a')) - def test_as_uri(self): + @needs_posix + def test_as_uri_posix(self): P = self.cls - self.assertEqual(P('/').as_uri(), 'file:///') - self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') - self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') + self.assertEqual(self.make_uri(P('/')), 'file:///') + self.assertEqual(self.make_uri(P('/a/b.c')), 'file:///a/b.c') + self.assertEqual(self.make_uri(P('/a/b%#c')), 'file:///a/b%25%23c') + @needs_posix def test_as_uri_non_ascii(self): from urllib.parse import quote_from_bytes P = self.cls @@ -762,64 +676,17 @@ def test_as_uri_non_ascii(self): os.fsencode('\xe9') except UnicodeEncodeError: self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") - self.assertEqual(P('/a/b\xe9').as_uri(), + self.assertEqual(self.make_uri(P('/a/b\xe9')), 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - + @needs_posix def test_parse_windows_path(self): P = self.cls p = P('c:', 'a', 'b') pp = P(pathlib.PureWindowsPath('c:\\a\\b')) self.assertEqual(p, pp) - -class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PureWindowsPath - - equivalences = _BasePurePathTest.equivalences.copy() - equivalences.update({ + windows_equivalences = { './a:b': [ ('./a:b',) ], 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ @@ -830,95 +697,63 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): '//a/b/c': [ ('//a/b', 'c'), ('//a/b/', 'c'), ], - }) + } + + @needs_windows + def test_equivalences_windows(self): + self.test_equivalences(self.windows_equivalences) - def test_drive_root_parts(self): - check = self._check_drive_root_parts + @needs_windows + def test_parse_path_windows(self): + check = self._check_parse_path # First part is anchored. - check(('c:',), 'c:', '', ('c:',)) - check(('c:/',), 'c:', '\\', ('c:\\',)) - check(('/',), '', '\\', ('\\',)) - check(('c:a',), 'c:', '', ('c:', 'a')) - check(('c:/a',), 'c:', '\\', ('c:\\', 'a')) - check(('/a',), '', '\\', ('\\', 'a')) - # UNC paths. - check(('//',), '\\\\', '', ('\\\\',)) - check(('//a',), '\\\\a', '', ('\\\\a',)) - check(('//a/',), '\\\\a\\', '', ('\\\\a\\',)) - check(('//a/b',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/c',), '\\\\a\\b', '\\', ('\\\\a\\b\\', 'c')) - # Second part is anchored, so that the first part is ignored. - check(('a', 'Z:b', 'c'), 'Z:', '', ('Z:', 'b', 'c')) - check(('a', 'Z:/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) + check('c:', 'c:', '', []) + check('c:/', 'c:', '\\', []) + check('/', '', '\\', []) + check('c:a', 'c:', '', ['a']) + check('c:/a', 'c:', '\\', ['a']) + check('/a', '', '\\', ['a']) # UNC paths. - check(('a', '//b/c', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) + check('//', '\\\\', '', []) + check('//a', '\\\\a', '', []) + check('//a/', '\\\\a\\', '', []) + check('//a/b', '\\\\a\\b', '\\', []) + check('//a/b/', '\\\\a\\b', '\\', []) + check('//a/b/c', '\\\\a\\b', '\\', ['c']) # Collapsing and stripping excess slashes. - check(('a', 'Z://b//c/', 'd/'), 'Z:', '\\', ('Z:\\', 'b', 'c', 'd')) + check('Z://b//c/d/', 'Z:', '\\', ['b', 'c', 'd']) # UNC paths. - check(('a', '//b/c//', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) + check('//b/c//d', '\\\\b\\c', '\\', ['d']) # Extended paths. - check(('//./c:',), '\\\\.\\c:', '', ('\\\\.\\c:',)) - check(('//?/c:/',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\',)) - check(('//?/c:/a',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'a')) - check(('//?/c:/a', '/b'), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'b')) + check('//./c:', '\\\\.\\c:', '', []) + check('//?/c:/', '\\\\?\\c:', '\\', []) + check('//?/c:/a', '\\\\?\\c:', '\\', ['a']) # Extended UNC paths (format is "\\?\UNC\server\share"). - check(('//?',), '\\\\?', '', ('\\\\?',)) - check(('//?/',), '\\\\?\\', '', ('\\\\?\\',)) - check(('//?/UNC',), '\\\\?\\UNC', '', ('\\\\?\\UNC',)) - check(('//?/UNC/',), '\\\\?\\UNC\\', '', ('\\\\?\\UNC\\',)) - check(('//?/UNC/b',), '\\\\?\\UNC\\b', '', ('\\\\?\\UNC\\b',)) - check(('//?/UNC/b/',), '\\\\?\\UNC\\b\\', '', ('\\\\?\\UNC\\b\\',)) - check(('//?/UNC/b/c',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/d',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\', 'd')) + check('//?', '\\\\?', '', []) + check('//?/', '\\\\?\\', '', []) + check('//?/UNC', '\\\\?\\UNC', '', []) + check('//?/UNC/', '\\\\?\\UNC\\', '', []) + check('//?/UNC/b', '\\\\?\\UNC\\b', '', []) + check('//?/UNC/b/', '\\\\?\\UNC\\b\\', '', []) + check('//?/UNC/b/c', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/d', '\\\\?\\UNC\\b\\c', '\\', ['d']) # UNC device paths - check(('//./BootPartition/',), '\\\\.\\BootPartition', '\\', ('\\\\.\\BootPartition\\',)) - check(('//?/BootPartition/',), '\\\\?\\BootPartition', '\\', ('\\\\?\\BootPartition\\',)) - check(('//./PhysicalDrive0',), '\\\\.\\PhysicalDrive0', '', ('\\\\.\\PhysicalDrive0',)) - check(('//?/Volume{}/',), '\\\\?\\Volume{}', '\\', ('\\\\?\\Volume{}\\',)) - check(('//./nul',), '\\\\.\\nul', '', ('\\\\.\\nul',)) - # Second part has a root but not drive. - check(('a', '/b', 'c'), '', '\\', ('\\', 'b', 'c')) - check(('Z:/a', '/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) - check(('//?/Z:/a', '/b', 'c'), '\\\\?\\Z:', '\\', ('\\\\?\\Z:\\', 'b', 'c')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - check(('c:/a/b', 'c:x/y'), 'c:', '\\', ('c:\\', 'a', 'b', 'x', 'y')) - check(('c:/a/b', 'c:/x/y'), 'c:', '\\', ('c:\\', 'x', 'y')) + check('//./BootPartition/', '\\\\.\\BootPartition', '\\', []) + check('//?/BootPartition/', '\\\\?\\BootPartition', '\\', []) + check('//./PhysicalDrive0', '\\\\.\\PhysicalDrive0', '', []) + check('//?/Volume{}/', '\\\\?\\Volume{}', '\\', []) + check('//./nul', '\\\\.\\nul', '', []) # Paths to files with NTFS alternate data streams - check(('./c:s',), '', '', ('c:s',)) - check(('cc:s',), '', '', ('cc:s',)) - check(('C:c:s',), 'C:', '', ('C:', 'c:s')) - check(('C:/c:s',), 'C:', '\\', ('C:\\', 'c:s')) - check(('D:a', './c:b'), 'D:', '', ('D:', 'a', 'c:b')) - check(('D:/a', './c:b'), 'D:', '\\', ('D:\\', 'a', 'c:b')) - - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') + check('./c:s', '', '', ['c:s']) + check('cc:s', '', '', ['cc:s']) + check('C:c:s', 'C:', '', ['c:s']) + check('C:/c:s', 'C:', '\\', ['c:s']) + check('D:a/c:b', 'D:', '', ['a', 'c:b']) + check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) - def test_str_subclass(self): - self._check_str_subclass('.\\a:b') - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): + @needs_windows + def test_eq_windows(self): P = self.cls self.assertEqual(P('c:a/b'), P('c:a/b')) self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) @@ -931,50 +766,29 @@ def test_eq(self): self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) self.assertEqual(P('\u0130'), P('i\u0307')) - def test_as_uri(self): + @needs_windows + def test_as_uri_windows(self): P = self.cls with self.assertRaises(ValueError): - P('/a/b').as_uri() + self.make_uri(P('/a/b')) with self.assertRaises(ValueError): - P('c:a/b').as_uri() - self.assertEqual(P('c:/').as_uri(), 'file:///c:/') - self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') - self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') - self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') - self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') - self.assertEqual(P('//some/share/a/b.c').as_uri(), + self.make_uri(P('c:a/b')) + self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/') + self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c') + self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c') + self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/') + self.assertEqual(self.make_uri(P('//some/share/a/b.c')), 'file://some/share/a/b.c') - self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), - 'file://some/share/a/b%25%23c%C3%A9') - - def test_match(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - def test_ordering_common(self): + from urllib.parse import quote_from_bytes + QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII)) + self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)), + 'file:///c:/a/b' + QUOTED_FS_NONASCII) + self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)), + 'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII) + + @needs_windows + def test_ordering_windows(self): # Case-insensitivity. def assertOrderedEqual(a, b): self.assertLessEqual(a, b) @@ -991,233 +805,121 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): + @needs_posix + def test_is_absolute_posix(self): P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) - def test_root(self): + @needs_windows + def test_is_absolute_windows(self): P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') - - def test_suffix(self): - P = self.cls - self.assertEqual(P('c:').suffix, '') - self.assertEqual(P('c:/').suffix, '') - self.assertEqual(P('c:a/b').suffix, '') - self.assertEqual(P('c:/a/b').suffix, '') - self.assertEqual(P('c:a/b.py').suffix, '.py') - self.assertEqual(P('c:/a/b.py').suffix, '.py') - self.assertEqual(P('c:a/.hgrc').suffix, '') - self.assertEqual(P('c:/a/.hgrc').suffix, '') - self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('//My.py/Share.php').suffix, '') - self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - - def test_suffixes(self): - P = self.cls - self.assertEqual(P('c:').suffixes, []) - self.assertEqual(P('c:/').suffixes, []) - self.assertEqual(P('c:a/b').suffixes, []) - self.assertEqual(P('c:/a/b').suffixes, []) - self.assertEqual(P('c:a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:a/.hgrc').suffixes, []) - self.assertEqual(P('c:/a/.hgrc').suffixes, []) - self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('//My.py/Share.php').suffixes, []) - self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem(self): - P = self.cls - self.assertEqual(P('c:').stem, '') - self.assertEqual(P('c:.').stem, '') - self.assertEqual(P('c:..').stem, '..') - self.assertEqual(P('c:/').stem, '') - self.assertEqual(P('c:a/b').stem, 'b') - self.assertEqual(P('c:a/b.py').stem, 'b') - self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') - self.assertEqual(P('c:a/.hg.rc').stem, '.hg') - self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name(self): - P = self.cls - self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) - self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) - self.assertRaises(ValueError, P('c:').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') - self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') - self.assertEqual(str(P('a').with_name('d:')), '.\\d:') - self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) - def test_with_stem(self): + def test_relative_to_common(self): P = self.cls - self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) - self.assertRaises(ValueError, P('c:').with_stem, 'd') - self.assertRaises(ValueError, P('c:/').with_stem, 'd') - self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') - self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') - self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - - def test_with_suffix(self): - P = self.cls - self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) - self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') - - def test_relative_to(self): + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + @needs_windows + def test_relative_to_windows(self): P = self.cls p = P('C:Foo/Bar') self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) @@ -1322,7 +1024,40 @@ def test_relative_to(self): self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - def test_is_relative_to(self): + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): P = self.cls p = P('C:Foo/Bar') self.assertTrue(p.is_relative_to(P('c:'))) @@ -1375,251 +1110,74 @@ def test_is_relative_to(self): self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - def test_is_absolute(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - pp = p.joinpath(P('./d:s')) - self.assertEqual(pp, P('C:/a/b/d:s')) - pp = p.joinpath(P('./dd:s')) - self.assertEqual(pp, P('C:/a/b/dd:s')) - pp = p.joinpath(P('E:d:s')) - self.assertEqual(pp, P('E:d:s')) - # Joining onto a UNC path with no root - pp = P('//').joinpath('server') - self.assertEqual(pp, P('//server')) - pp = P('//server').joinpath('share') - self.assertEqual(pp, P('//server/share')) - pp = P('//./BootPartition').joinpath('Windows') - self.assertEqual(pp, P('//./BootPartition/Windows')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) - self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) - self.assertEqual(p / P('E:d:s'), P('E:d:s')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) - -class PurePathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PurePath +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath - def test_concrete_class(self): - p = self.cls('a') - self.assertIs(type(p), - pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath) - def test_different_flavours_unequal(self): - p = pathlib.PurePosixPath('a') - q = pathlib.PureWindowsPath('a') - self.assertNotEqual(p, q) +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath - def test_different_flavours_unordered(self): - p = pathlib.PurePosixPath('a') - q = pathlib.PureWindowsPath('a') - with self.assertRaises(TypeError): - p < q - with self.assertRaises(TypeError): - p <= q - with self.assertRaises(TypeError): - p > q - with self.assertRaises(TypeError): - p >= q + +class PurePathSubclassTest(PurePathTest): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None # # Tests for the concrete classes. # -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) -rel_join = lambda *x: os.path.join(TESTFN, *x) - -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - -@only_posix -class PosixPathAsPureTest(PurePosixPathTest): - cls = pathlib.PosixPath - -@only_nt -class WindowsPathAsPureTest(PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_owner(self): - P = self.cls - with self.assertRaises(NotImplementedError): - P('c:/').owner() - - def test_group(self): - P = self.cls - with self.assertRaises(NotImplementedError): - P('c:/').group() - - -class _BasePathTest(object): +class PathTest(PurePathTest): """Tests for the FS-accessing functionalities of the Path classes.""" - - # (BASE) - # | - # |-- brokenLink -> non-existing - # |-- dirA - # | `-- linkC -> ../dirB - # |-- dirB - # | |-- fileB - # | `-- linkD -> ../dirB - # |-- dirC - # | |-- dirD - # | | `-- fileD - # | `-- fileC - # | `-- novel.txt - # |-- dirE # No permissions - # |-- fileA - # |-- linkA -> fileA - # |-- linkB -> dirB - # `-- brokenLinkLoop -> brokenLinkLoop - # + cls = pathlib.Path + can_symlink = os_helper.can_symlink() def setUp(self): - def cleanup(): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - self.addCleanup(cleanup) - os.mkdir(BASE) - os.mkdir(join('dirA')) - os.mkdir(join('dirB')) - os.mkdir(join('dirC')) - os.mkdir(join('dirC', 'dirD')) - os.mkdir(join('dirE')) - with open(join('fileA'), 'wb') as f: + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + super().setUp() + os.mkdir(self.base) + os.mkdir(os.path.join(self.base, 'dirA')) + os.mkdir(os.path.join(self.base, 'dirB')) + os.mkdir(os.path.join(self.base, 'dirC')) + os.mkdir(os.path.join(self.base, 'dirC', 'dirD')) + os.mkdir(os.path.join(self.base, 'dirE')) + with open(os.path.join(self.base, 'fileA'), 'wb') as f: f.write(b"this is file A\n") - with open(join('dirB', 'fileB'), 'wb') as f: + with open(os.path.join(self.base, 'dirB', 'fileB'), 'wb') as f: f.write(b"this is file B\n") - with open(join('dirC', 'fileC'), 'wb') as f: + with open(os.path.join(self.base, 'dirC', 'fileC'), 'wb') as f: f.write(b"this is file C\n") - with open(join('dirC', 'novel.txt'), 'wb') as f: + with open(os.path.join(self.base, 'dirC', 'novel.txt'), 'wb') as f: f.write(b"this is a novel\n") - with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: + with open(os.path.join(self.base, 'dirC', 'dirD', 'fileD'), 'wb') as f: f.write(b"this is file D\n") - os.chmod(join('dirE'), 0) - if os_helper.can_symlink(): + os.chmod(os.path.join(self.base, 'dirE'), 0) + if self.can_symlink: # Relative symlinks. - os.symlink('fileA', join('linkA')) - os.symlink('non-existing', join('brokenLink')) - self.dirlink('dirB', join('linkB')) - self.dirlink(os.path.join('..', 'dirB'), join('dirA', 'linkC')) + os.symlink('fileA', os.path.join(self.base, 'linkA')) + os.symlink('non-existing', os.path.join(self.base, 'brokenLink')) + os.symlink('dirB', + os.path.join(self.base, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirA', 'linkC'), + target_is_directory=True) # This one goes upwards, creating a loop. - self.dirlink(os.path.join('..', 'dirB'), join('dirB', 'linkD')) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirB', 'linkD'), + target_is_directory=True) # Broken symlink (pointing to itself). - os.symlink('brokenLinkLoop', join('brokenLinkLoop')) + os.symlink('brokenLinkLoop', os.path.join(self.base, 'brokenLinkLoop')) - if os.name == 'nt': - # Workaround for http://bugs.python.org/issue13772. - def dirlink(self, src, dest): - os.symlink(src, dest, target_is_directory=True) - else: - def dirlink(self, src, dest): - os.symlink(src, dest) - - def assertSame(self, path_a, path_b): - self.assertTrue(os.path.samefile(str(path_a), str(path_b)), - "%r and %r don't point to the same file" % - (path_a, path_b)) + def tearDown(self): + os.chmod(os.path.join(self.base, 'dirE'), 0o777) + os_helper.rmtree(self.base) def assertFileNotFound(self, func, *args, **kwargs): with self.assertRaises(FileNotFoundError) as cm: @@ -1627,7 +1185,39 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) + normcase = self.parser.normcase + self.assertEqual(normcase(path_a), normcase(path_b)) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + + def test_matches_writablepath_docstrings(self): + path_names = {name for name in dir(pathlib.types._WritablePath) if name[0] != '_'} + for attr_name in path_names: + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but WritablePath.parser is + # posixpath, and so their docstrings differ. + continue + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib.types._WritablePath, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + def test_concrete_class(self): + if self.cls is pathlib.Path: + expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") + else: + self.assertRaises(pathlib.UnsupportedOperation, self.cls) def _test_cwd(self, p): q = self.cls(os.getcwd()) @@ -1644,23 +1234,23 @@ def test_absolute_common(self): P = self.cls with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = BASE + getcwd.return_value = self.base # Simple relative paths. - self.assertEqual(str(P().absolute()), BASE) - self.assertEqual(str(P('.').absolute()), BASE) - self.assertEqual(str(P('a').absolute()), os.path.join(BASE, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(BASE, 'a', 'b', 'c')) + self.assertEqual(str(P().absolute()), self.base) + self.assertEqual(str(P('.').absolute()), self.base) + self.assertEqual(str(P('a').absolute()), os.path.join(self.base, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(self.base, 'a', 'b', 'c')) # Symlinks should not be resolved. - self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(BASE, 'linkB', 'fileB')) - self.assertEqual(str(P('brokenLink').absolute()), os.path.join(BASE, 'brokenLink')) - self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(BASE, 'brokenLinkLoop')) + self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(self.base, 'linkB', 'fileB')) + self.assertEqual(str(P('brokenLink').absolute()), os.path.join(self.base, 'brokenLink')) + self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(self.base, 'brokenLinkLoop')) # '..' entries should be preserved and not normalised. - self.assertEqual(str(P('..').absolute()), os.path.join(BASE, '..')) - self.assertEqual(str(P('a', '..').absolute()), os.path.join(BASE, 'a', '..')) - self.assertEqual(str(P('..', 'b').absolute()), os.path.join(BASE, '..', 'b')) + self.assertEqual(str(P('..').absolute()), os.path.join(self.base, '..')) + self.assertEqual(str(P('a', '..').absolute()), os.path.join(self.base, 'a', '..')) + self.assertEqual(str(P('..', 'b').absolute()), os.path.join(self.base, '..', 'b')) def _test_home(self, p): q = self.cls(os.path.expanduser('~')) @@ -1671,65 +1261,18 @@ def _test_home(self, p): @unittest.skipIf( pwd is None, reason="Test requires pwd module to get homedir." - ) - def test_home(self): - with os_helper.EnvironmentVarGuard() as env: - self._test_home(self.cls.home()) - - env.clear() - env['USERPROFILE'] = os.path.join(BASE, 'userprofile') - self._test_home(self.cls.home()) - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = os.path.join(BASE, 'home') - self._test_home(self.cls.home()) - - @unittest.skip("TODO: RUSTPYTHON; PyObject::set_slot index out of bounds") - def test_with_segments(self): - class P(_BasePurePathSubclass, self.cls): - pass - p = P(BASE, session_id=42) - self.assertEqual(42, p.absolute().session_id) - self.assertEqual(42, p.resolve().session_id) - if not is_wasi: # WASI has no user accounts. - self.assertEqual(42, p.with_segments('~').expanduser().session_id) - self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) - self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) - if os_helper.can_symlink(): - self.assertEqual(42, (p / 'linkA').readlink().session_id) - for path in p.iterdir(): - self.assertEqual(42, path.session_id) - for path in p.glob('*'): - self.assertEqual(42, path.session_id) - for path in p.rglob('*'): - self.assertEqual(42, path.session_id) - for dirpath, dirnames, filenames in p.walk(): - self.assertEqual(42, dirpath.session_id) - - def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) + ) + def test_home(self): + with os_helper.EnvironmentVarGuard() as env: + self._test_home(self.cls.home()) - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(p.stat(), os.stat('.')) + env.clear() + env['USERPROFILE'] = os.path.join(self.base, 'userprofile') + self._test_home(self.cls.home()) + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = os.path.join(self.base, 'home') + self._test_home(self.cls.home()) @unittest.skipIf(is_wasi, "WASI has no user accounts.") def test_expanduser_common(self): @@ -1747,277 +1290,561 @@ def test_expanduser_common(self): p = P('~/a:b') self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) - def test_exists(self): - P = self.cls - p = P(BASE) - self.assertIs(True, p.exists()) - self.assertIs(True, (p / 'dirA').exists()) - self.assertIs(True, (p / 'fileA').exists()) - self.assertIs(False, (p / 'fileA' / 'bah').exists()) - if os_helper.can_symlink(): - self.assertIs(True, (p / 'linkA').exists()) - self.assertIs(True, (p / 'linkB').exists()) - self.assertIs(True, (p / 'linkB' / 'fileB').exists()) - self.assertIs(False, (p / 'linkA' / 'bah').exists()) - self.assertIs(False, (p / 'brokenLink').exists()) - self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) - self.assertIs(False, (p / 'foo').exists()) - self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P(self.base, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if self.can_symlink: + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) def test_open_common(self): - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'fileA').open('r') as f: self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.read(), "this is file A\n") with (p / 'fileA').open('rb') as f: self.assertIsInstance(f, io.BufferedIOBase) self.assertEqual(f.read().strip(), b"this is file A") + + def test_open_unbuffered(self): + p = self.cls(self.base) with (p / 'fileA').open('rb', buffering=0) as f: self.assertIsInstance(f, io.RawIOBase) self.assertEqual(f.read().strip(), b"this is file A") - def test_read_write_bytes(self): - p = self.cls(BASE) - (p / 'fileA').write_bytes(b'abcdefg') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - # Check that trying to write str does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - - def test_read_write_text(self): - p = self.cls(BASE) - (p / 'fileA').write_text('äbcdefg', encoding='latin-1') - self.assertEqual((p / 'fileA').read_text( - encoding='utf-8', errors='ignore'), 'bcdefg') - # Check that trying to write bytes does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') - self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') - - def test_write_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\rfghlk\r\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\r\nfghlk\r\n\rmnopq') - # Check that no argument passed will change `\n` to `os.linesep` - os_linesep_byte = bytes(os.linesep, encoding='ascii') - (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') - - def test_iterdir(self): - P = self.cls - p = P(BASE) - it = p.iterdir() - paths = set(it) - expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] - if os_helper.can_symlink(): - expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) - - @os_helper.skip_unless_symlink - def test_iterdir_symlink(self): - # __iter__ on a symlink to a directory. - P = self.cls - p = P(BASE, 'linkB') - paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - - def test_iterdir_nodir(self): - # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') - with self.assertRaises(OSError) as cm: - next(p.iterdir()) - # ENOENT or EINVAL under Windows, ENOTDIR otherwise - # (see issue #12802). - self.assertIn(cm.exception.errno, (errno.ENOTDIR, - errno.ENOENT, errno.EINVAL)) - - def test_glob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - it = p.glob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.glob("fileB"), []) - _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) - if not os_helper.can_symlink(): - _check(p.glob("*A"), ['dirA', 'fileA']) - else: - _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) - if not os_helper.can_symlink(): - _check(p.glob("*B/*"), ['dirB/fileB']) - else: - _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', - 'linkB/fileB', 'linkB/linkD']) - if not os_helper.can_symlink(): - _check(p.glob("*/fileB"), ['dirB/fileB']) - else: - _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) - if os_helper.can_symlink(): - _check(p.glob("brokenLink"), ['brokenLink']) - - if not os_helper.can_symlink(): - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE"]) - else: - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE", "linkB"]) - - def test_glob_case_sensitive(self): - P = self.cls - def _check(path, pattern, case_sensitive, expected): - actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} - expected = {str(P(BASE, q)) for q in expected} - self.assertEqual(actual, expected) - path = P(BASE) - _check(path, "DIRB/FILE*", True, []) - _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) - _check(path, "dirb/file*", True, []) - _check(path, "dirb/file*", False, ["dirB/fileB"]) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) - P = self.cls - p = P(BASE) - it = p.rglob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not os_helper.can_symlink(): - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not os_helper.can_symlink(): - _check(p.rglob("*/"), [ - "dirA", "dirB", "dirC", "dirC/dirD", "dirE", - ]) + def test_copy_file_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'fileA' + if hasattr(os, 'chmod'): + os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source, stat.UF_NODUMP) + source_st = source.stat() + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + target_st = target.stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + + @needs_symlinks + def test_copy_link_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'linkA' + if hasattr(os, 'lchmod'): + os.lchmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(source, stat.UF_NODUMP) + source_st = source.lstat() + target = base / 'copyA' + source.copy(target, follow_symlinks=False, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + target_st = target.lstat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + def test_copy_error_handling(self): + def make_raiser(err): + def raiser(*args, **kwargs): + raise OSError(err, os.strerror(err)) + return raiser + + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + + # Raise non-fatal OSError from all available fast copy functions. + with contextlib.ExitStack() as ctx: + if fcntl and hasattr(fcntl, 'FICLONE'): + ctx.enter_context(mock.patch('fcntl.ioctl', make_raiser(errno.EXDEV))) + if posix and hasattr(posix, '_fcopyfile'): + ctx.enter_context(mock.patch('posix._fcopyfile', make_raiser(errno.ENOTSUP))) + if hasattr(os, 'copy_file_range'): + ctx.enter_context(mock.patch('os.copy_file_range', make_raiser(errno.EXDEV))) + if hasattr(os, 'sendfile'): + ctx.enter_context(mock.patch('os.sendfile', make_raiser(errno.ENOTSOCK))) + + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + # Raise fatal OSError from first available fast copy function. + if fcntl and hasattr(fcntl, 'FICLONE'): + patchpoint = 'fcntl.ioctl' + elif posix and hasattr(posix, '_fcopyfile'): + patchpoint = 'posix._fcopyfile' + elif hasattr(os, 'copy_file_range'): + patchpoint = 'os.copy_file_range' + elif hasattr(os, 'sendfile'): + patchpoint = 'os.sendfile' else: - _check(p.rglob("*/"), [ - "dirA", "dirA/linkC", "dirB", "dirB/linkD", "dirC", - "dirC/dirD", "dirE", "linkB", - ]) - _check(p.rglob(""), ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) - - p = P(BASE, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD"]) - _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**"), ["dirC", "dirC/dirD"]) - # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) - - @os_helper.skip_unless_symlink - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - P = self.cls - p = P(BASE) - given = set(p.rglob('*')) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - base = P(BASE) / 'deep' - p = P(base, *(['d']*depth)) - p.mkdir(parents=True) - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) + return + with mock.patch(patchpoint, make_raiser(errno.ENOENT)): + self.assertRaises(FileNotFoundError, source.copy, target) + + @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") + @unittest.skipIf(root_in_posix, "test fails with root privilege") + def test_copy_dir_no_read_permission(self): + base = self.cls(self.base) + source = base / 'dirE' + target = base / 'copyE' + self.assertRaises(PermissionError, source.copy, target) + self.assertFalse(target.exists()) + + def test_copy_dir_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'dirC' + if hasattr(os, 'chmod'): + os.chmod(source / 'dirD', stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source / 'fileC', stat.UF_NODUMP) + target = base / 'copyA' + + subpaths = ['.', 'fileC', 'dirD', 'dirD/fileD'] + source_sts = [source.joinpath(subpath).stat() for subpath in subpaths] + source.copy(target, preserve_metadata=True) + target_sts = [target.joinpath(subpath).stat() for subpath in subpaths] + + for source_st, target_st in zip(source_sts, target_sts): + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_dir_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'dirC' + source_file = source.joinpath('dirD', 'fileD') + os.setxattr(source_file, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + target_file = target.joinpath('dirD', 'fileD') + self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + def test_move_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'fileA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_other_fs(self): + self.test_move_file() + + def test_move_file_to_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'dirB' / 'fileB' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_to_file_other_fs(self): + self.test_move_file_to_file() + + def test_move_file_to_dir(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + + @patch_replace + def test_move_file_to_dir_other_fs(self): + self.test_move_file_to_dir() + + def test_move_file_to_itself(self): + base = self.cls(self.base) + source = base / 'fileA' + self.assertRaises(OSError, source.move, source) + + def test_move_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + @patch_replace + def test_move_dir_other_fs(self): + self.test_move_dir() + + def test_move_dir_to_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertTrue(target.exists()) - @os_helper.skip_unless_symlink - def test_glob_permissions(self): - # See bpo-38894 - P = self.cls - base = P(BASE) / 'permissions' - base.mkdir() - self.addCleanup(os_helper.rmtree, base) + @patch_replace + def test_move_dir_to_dir_other_fs(self): + self.test_move_dir_to_dir() + + def test_move_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.move, source) + self.assertTrue(source.exists()) + + def test_move_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'bar' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertFalse(target.exists()) + + @patch_replace + def test_move_dir_into_itself_other_fs(self): + self.test_move_dir_into_itself() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_other_fs(self): + self.test_move_file_symlink() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_to_itself_other_fs(self): + self.test_move_file_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_other_fs(self): + self.test_move_dir_symlink() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_to_itself_other_fs(self): + self.test_move_dir_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dangling_symlink_other_fs(self): + self.test_move_dangling_symlink() + + def test_move_into(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target_dir = base / 'dirA' + result = source.move_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertFalse(source.exists()) + self.assertTrue(result.exists()) + self.assertEqual(source_text, result.read_text()) + + @patch_replace + def test_move_into_other_os(self): + self.test_move_into() + + def test_move_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.move_into, target_dir) + + @patch_replace + def test_move_into_empty_name_other_os(self): + self.test_move_into_empty_name() + + @needs_symlinks + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(BASE, "dirE", "nonexistent")) - else: - link.symlink_to(P(BASE, "dirC")) + @needs_symlinks + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) + @needs_symlinks + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.parser.join('dirA', '..')) - @os_helper.skip_unless_symlink - def test_glob_long_symlink(self): - # See gh-87695 - base = self.cls(BASE) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + def _check_complex_symlinks(self, link0_target): + # Test solving a non-looping chain of symlinks (issue #19887). + parser = self.parser + P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - def test_glob_above_recursion_limit(self): - recursion_limit = 50 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = pathlib.Path(os_helper.TESTFN, 'deep') - path = pathlib.Path(base, *(['d'] * directory_depth)) - path.mkdir(parents=True) + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) - with set_recursion_limit(recursion_limit): - list(base.glob('**')) + # Resolve relative paths. + old_path = os.getcwd() + os.chdir(self.base) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + finally: + os.chdir(old_path) def _check_resolve(self, p, expected, strict=True): q = p.resolve(strict) @@ -2026,74 +1853,74 @@ def _check_resolve(self, p, expected, strict=True): # This can be used to check both relative and absolute resolutions. _check_resolve_relative = _check_resolve_absolute = _check_resolve - @os_helper.skip_unless_symlink + @needs_symlinks def test_resolve_common(self): P = self.cls - p = P(BASE, 'foo') + p = P(self.base, 'foo') with self.assertRaises(OSError) as cm: p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict + parser = self.parser self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') + p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) # Now create absolute symlinks. - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) - os.symlink(os.path.join(d), join('dirA', 'linkX')) - os.symlink(join('dirB'), os.path.join(d, 'linkY')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - @os_helper.skip_unless_symlink + @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - p = self.cls(BASE) - self.dirlink('.', join('0')) - self.dirlink(os.path.join('0', '0'), join('1')) - self.dirlink(os.path.join('1', '1'), join('2')) + parser = self.parser + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) q = p / '2' self.assertEqual(q.resolve(strict=True), p) r = q / '3' / '4' @@ -2101,39 +1928,67 @@ def test_resolve_dot(self): # Non-strict self.assertEqual(r.resolve(strict=False), p / '3' / '4') + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + @needs_posix + @needs_symlinks + def test_resolve_loop(self): + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') old_cwd = os.getcwd() - os.chdir(BASE) + os.chdir(self.base) try: - self.assertEqual(p.resolve(), self.cls(BASE, p)) + self.assertEqual(p.resolve(), self.cls(self.base, p)) finally: os.chdir(old_cwd) - def test_with(self): - p = self.cls(BASE) - it = p.iterdir() - it2 = p.iterdir() - next(it2) - # bpo-46556: path context managers are deprecated in Python 3.11. - with self.assertWarns(DeprecationWarning): - with p: - pass - # Using a path as a context manager is a no-op, thus the following - # operations should still succeed after the context manage exits. - next(it) - next(it2) - p.exists() - p.resolve() - p.absolute() - with self.assertWarns(DeprecationWarning): - with p: - pass + @needs_symlinks + def test_readlink(self): + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) @os_helper.skip_unless_working_chmod def test_chmod(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' mode = p.stat().st_mode # Clear writable bit. new_mode = mode & ~0o222 @@ -2145,10 +2000,10 @@ def test_chmod(self): self.assertEqual(p.stat().st_mode, new_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix + @needs_posix @os_helper.skip_unless_working_chmod def test_chmod_follow_symlinks_true(self): - p = self.cls(BASE) / 'linkA' + p = self.cls(self.base) / 'linkA' q = p.resolve() mode = q.stat().st_mode # Clear writable bit. @@ -2162,82 +2017,266 @@ def test_chmod_follow_symlinks_true(self): # XXX also need a test for lchmod. - @os_helper.skip_unless_working_chmod - def test_stat(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(p.stat(), st) - # Change file mode by flipping write bit. - p.chmod(st.st_mode ^ 0o222) - self.addCleanup(p.chmod, st.st_mode) - self.assertNotEqual(p.stat(), st) + def _get_pw_name_or_skip_test(self, uid): + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + self.skipTest( + "user %d doesn't have an entry in the system database" % uid) - @os_helper.skip_unless_symlink - def test_stat_no_follow_symlinks(self): - p = self.cls(BASE) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + def test_owner(self): + p = self.cls(self.base) / 'fileA' + expected_uid = p.stat().st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) + self.assertEqual(expected_name, p.owner()) - @os_helper.skip_unless_symlink - def test_lstat(self): - p = self.cls(BASE)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_owner_no_follow_symlinks(self): + all_users = [u.pw_uid for u in pwd.getpwall()] + if len(all_users) < 2: + self.skipTest("test needs more than one user") - def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' - @unittest.skipUnless(pwd, "the pwd module is needed for this test") - def test_owner(self): - p = self.cls(BASE) / 'fileA' - uid = p.stat().st_uid + uid_1, uid_2 = all_users[:2] + os.chown(target, uid_1, -1) + os.chown(link, uid_2, -1, follow_symlinks=False) + + expected_uid = link.stat(follow_symlinks=False).st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_uid, uid_2) + self.assertEqual(expected_name, link.owner(follow_symlinks=False)) + + def _get_gr_name_or_skip_test(self, gid): try: - name = pwd.getpwuid(uid).pw_name + return grp.getgrgid(gid).gr_name except KeyError: self.skipTest( - "user %d doesn't have an entry in the system database" % uid) - self.assertEqual(name, p.owner()) + "group %d doesn't have an entry in the system database" % gid) @unittest.skipUnless(grp, "the grp module is needed for this test") def test_group(self): - p = self.cls(BASE) / 'fileA' - gid = p.stat().st_gid - try: - name = grp.getgrgid(gid).gr_name - except KeyError: - self.skipTest( - "group %d doesn't have an entry in the system database" % gid) - self.assertEqual(name, p.group()) + p = self.cls(self.base) / 'fileA' + expected_gid = p.stat().st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_name, p.group()) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_group_no_follow_symlinks(self): + all_groups = [g.gr_gid for g in grp.getgrall()] + if len(all_groups) < 2: + self.skipTest("test needs more than one group") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + gid_1, gid_2 = all_groups[:2] + os.chown(target, -1, gid_1) + os.chown(link, -1, gid_2, follow_symlinks=False) + + expected_gid = link.stat(follow_symlinks=False).st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_gid, gid_2) + self.assertEqual(expected_name, link.group(follow_symlinks=False)) def test_unlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' p.unlink() self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) def test_unlink_missing_ok(self): - p = self.cls(BASE) / 'fileAAA' + p = self.cls(self.base) / 'fileAAA' self.assertFileNotFound(p.unlink) p.unlink(missing_ok=True) - def test_rmdir(self): - p = self.cls(BASE) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p._delete() + self.assertFalse(p.exists()) + self.assertFileNotFound(p._delete) + + def test_delete_dir(self): + base = self.cls(self.base) + base.joinpath('dirA')._delete() + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) + base.joinpath('dirB')._delete() + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) + base.joinpath('dirC')._delete() + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) + + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename._delete) + + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_delete_unwritable(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + child_file_path = tmp / 'a' + child_dir_path = tmp / 'b' + child_file_path.write_text("") + child_dir_path.mkdir() + old_dir_mode = tmp.stat().st_mode + old_child_file_mode = child_file_path.stat().st_mode + old_child_dir_mode = child_dir_path.stat().st_mode + # Make unwritable. + new_mode = stat.S_IREAD | stat.S_IEXEC + try: + child_file_path.chmod(new_mode) + child_dir_path.chmod(new_mode) + tmp.chmod(new_mode) + + self.assertRaises(PermissionError, tmp._delete) + finally: + tmp.chmod(old_dir_mode) + child_file_path.chmod(old_child_file_mode) + child_dir_path.chmod(old_child_dir_mode) + + @needs_windows + def test_delete_inner_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + _winapi.CreateJunction(str(dir2), str(link1)) + link2 = dir1 / 'link2' + _winapi.CreateJunction(str(dir3), str(link2)) + link3 = dir1 / 'link3' + _winapi.CreateJunction(str(file1), str(link3)) + # make sure junctions are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @needs_windows + def test_delete_outer_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + src = tmp / 'cheese' + dst = tmp / 'shop' + src.mkdir() + spam = src / 'spam' + spam.write_text('') + _winapi.CreateJunction(str(src), str(dst)) + dst._delete() + self.assertFalse(dst.exists()) + self.assertTrue(spam.exists()) + self.assertTrue(src.exists()) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_delete_on_named_pipe(self): + p = self.cls(self.base, 'pipe') + os.mkfifo(p) + p._delete() + self.assertFalse(p.exists()) + + p = self.cls(self.base, 'dir') + p.mkdir() + os.mkfifo(p / 'mypipe') + p._delete() + self.assertFalse(p.exists()) + + def test_delete_does_not_choke_on_failing_lstat(self): + try: + orig_lstat = os.lstat + tmp = self.cls(self.base, 'delete') + + def raiser(fn, *args, **kwargs): + if fn != tmp: + raise OSError() + else: + return orig_lstat(fn) + + os.lstat = raiser + + tmp.mkdir() + foo = tmp / 'foo' + foo.write_text('') + tmp._delete() + finally: + os.lstat = orig_lstat - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") + @os_helper.skip_unless_hardlink def test_hardlink_to(self): - P = self.cls(BASE) + P = self.cls(self.base) target = P / 'fileA' size = target.stat().st_size # linking to another path. @@ -2248,22 +2287,22 @@ def test_hardlink_to(self): self.assertTrue(target.exists()) # Linking to a str of a relative path. link2 = P / 'dirA' / 'fileAAA' - target2 = rel_join('fileA') + target2 = self.parser.join(TESTFN, 'fileA') link2.hardlink_to(target2) self.assertEqual(os.stat(target2).st_size, size) self.assertTrue(link2.exists()) @unittest.skipIf(hasattr(os, "link"), "os.link() is present") - def test_link_to_not_implemented(self): - P = self.cls(BASE) + def test_hardlink_to_unsupported(self): + P = self.cls(self.base) p = P / 'fileA' # linking to another path. q = P / 'dirA' / 'fileAA' - with self.assertRaises(NotImplementedError): + with self.assertRaises(pathlib.UnsupportedOperation): q.hardlink_to(p) def test_rename(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' size = p.stat().st_size # Renaming to another path. @@ -2273,14 +2312,14 @@ def test_rename(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Renaming to a str of a relative path. - r = rel_join('fileAAA') + r = self.parser.join(TESTFN, 'fileAAA') renamed_q = q.rename(r) self.assertEqual(renamed_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) self.assertFileNotFound(q.stat) def test_replace(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' size = p.stat().st_size # Replacing a non-existing path. @@ -2290,24 +2329,14 @@ def test_replace(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Replacing another (existing) path. - r = rel_join('dirB', 'fileB') + r = self.parser.join(TESTFN, 'dirB', 'fileB') replaced_q = q.replace(r) self.assertEqual(replaced_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) self.assertFileNotFound(q.stat) - @os_helper.skip_unless_symlink - def test_readlink(self): - P = self.cls(BASE) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - def test_touch_common(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'newfileA' self.assertFalse(p.exists()) p.touch() @@ -2331,14 +2360,14 @@ def test_touch_common(self): self.assertRaises(OSError, p.touch, exist_ok=False) def test_touch_nochange(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' p.touch() with p.open('rb') as f: self.assertEqual(f.read().strip(), b"this is file A") def test_mkdir(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'newdirA' self.assertFalse(p.exists()) p.mkdir() @@ -2350,7 +2379,7 @@ def test_mkdir(self): def test_mkdir_parents(self): # Creating a chain of directories. - p = self.cls(BASE, 'newdirB', 'newdirC') + p = self.cls(self.base, 'newdirB', 'newdirC') self.assertFalse(p.exists()) with self.assertRaises(OSError) as cm: p.mkdir() @@ -2363,7 +2392,7 @@ def test_mkdir_parents(self): self.assertEqual(cm.exception.errno, errno.EEXIST) # Test `mode` arg. mode = stat.S_IMODE(p.stat().st_mode) # Default mode. - p = self.cls(BASE, 'newdirD', 'newdirE') + p = self.cls(self.base, 'newdirD', 'newdirE') p.mkdir(0o555, parents=True) self.assertTrue(p.exists()) self.assertTrue(p.is_dir()) @@ -2374,7 +2403,7 @@ def test_mkdir_parents(self): self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) def test_mkdir_exist_ok(self): - p = self.cls(BASE, 'dirB') + p = self.cls(self.base, 'dirB') st_ctime_first = p.stat().st_ctime self.assertTrue(p.exists()) self.assertTrue(p.is_dir()) @@ -2386,7 +2415,7 @@ def test_mkdir_exist_ok(self): self.assertEqual(p.stat().st_ctime, st_ctime_first) def test_mkdir_exist_ok_with_parent(self): - p = self.cls(BASE, 'dirC') + p = self.cls(self.base, 'dirC') self.assertTrue(p.exists()) with self.assertRaises(FileExistsError) as cm: p.mkdir() @@ -2402,13 +2431,12 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) - @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") def test_mkdir_exist_ok_root(self): # Issue #25803: A drive root could raise PermissionError on Windows. self.cls('/').resolve().mkdir(exist_ok=True) self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - @only_nt # XXX: not sure how to test this on POSIX. + @needs_windows # XXX: not sure how to test this on POSIX. def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': p = self.cls(d + ':\\') @@ -2420,7 +2448,7 @@ def test_mkdir_with_unknown_drive(self): (p / 'child' / 'path').mkdir(parents=True) def test_mkdir_with_child_file(self): - p = self.cls(BASE, 'dirB', 'fileB') + p = self.cls(self.base, 'dirB', 'fileB') self.assertTrue(p.exists()) # An exception is raised when the last path component is an existing # regular file, regardless of whether exist_ok is true or not. @@ -2432,7 +2460,7 @@ def test_mkdir_with_child_file(self): self.assertEqual(cm.exception.errno, errno.EEXIST) def test_mkdir_no_parents_file(self): - p = self.cls(BASE, 'fileA') + p = self.cls(self.base, 'fileA') self.assertTrue(p.exists()) # An exception is raised when the last path component is an existing # regular file, regardless of whether exist_ok is true or not. @@ -2445,7 +2473,7 @@ def test_mkdir_no_parents_file(self): def test_mkdir_concurrent_parent_creation(self): for pattern_num in range(32): - p = self.cls(BASE, 'dirCPC%d' % pattern_num) + p = self.cls(self.base, 'dirCPC%d' % pattern_num) self.assertFalse(p.exists()) real_mkdir = os.mkdir @@ -2473,9 +2501,9 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) - @os_helper.skip_unless_symlink + @needs_symlinks def test_symlink_to(self): - P = self.cls(BASE) + P = self.cls(self.base) target = P / 'fileA' # Symlinking a path target. link = P / 'dirA' / 'linkAA' @@ -2497,72 +2525,202 @@ def test_symlink_to(self): self.assertTrue(link.is_dir()) self.assertTrue(list(link.iterdir())) + @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") + def test_symlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.symlink_to(p) + + def test_info_exists_caching(self): + p = self.cls(self.base) + q = p / 'myfile' + self.assertFalse(q.info.exists()) + self.assertFalse(q.info.exists(follow_symlinks=False)) + q.write_text('hullo') + self.assertFalse(q.info.exists()) + self.assertFalse(q.info.exists(follow_symlinks=False)) + + def test_info_is_dir_caching(self): + p = self.cls(self.base) + q = p / 'mydir' + self.assertFalse(q.info.is_dir()) + self.assertFalse(q.info.is_dir(follow_symlinks=False)) + q.mkdir() + self.assertFalse(q.info.is_dir()) + self.assertFalse(q.info.is_dir(follow_symlinks=False)) + + def test_info_is_file_caching(self): + p = self.cls(self.base) + q = p / 'myfile' + self.assertFalse(q.info.is_file()) + self.assertFalse(q.info.is_file(follow_symlinks=False)) + q.write_text('hullo') + self.assertFalse(q.info.is_file()) + self.assertFalse(q.info.is_file(follow_symlinks=False)) + + @needs_symlinks + def test_info_is_symlink_caching(self): + p = self.cls(self.base) + q = p / 'mylink' + self.assertFalse(q.info.is_symlink()) + q.symlink_to('blah') + self.assertFalse(q.info.is_symlink()) + + q = p / 'mylink' # same path, new instance. + self.assertTrue(q.info.is_symlink()) + q.unlink() + self.assertTrue(q.info.is_symlink()) + + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_stat_no_follow_symlinks(self): + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_lstat(self): + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_exists(self): + P = self.cls + p = P(self.base) + self.assertIs(True, p.exists()) + self.assertIs(True, (p / 'dirA').exists()) + self.assertIs(True, (p / 'fileA').exists()) + self.assertIs(False, (p / 'fileA' / 'bah').exists()) + if self.can_symlink: + self.assertIs(True, (p / 'linkA').exists()) + self.assertIs(True, (p / 'linkB').exists()) + self.assertIs(True, (p / 'linkB' / 'fileB').exists()) + self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) + self.assertIs(False, (p / 'foo').exists()) + self.assertIs(False, P('/xyzzy').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) + def test_is_dir(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) self.assertFalse((P / 'fileA').is_dir()) self.assertFalse((P / 'non-existing').is_dir()) self.assertFalse((P / 'fileA' / 'bah').is_dir()) - if os_helper.can_symlink(): + if self.can_symlink: self.assertFalse((P / 'linkA').is_dir()) self.assertTrue((P / 'linkB').is_dir()) - self.assertFalse((P/ 'brokenLink').is_dir(), False) - self.assertIs((P / 'dirA\udfff').is_dir(), False) - self.assertIs((P / 'dirA\x00').is_dir(), False) + self.assertFalse((P/ 'brokenLink').is_dir()) + self.assertFalse((P / 'dirA\udfff').is_dir()) + self.assertFalse((P / 'dirA\x00').is_dir()) + + def test_is_dir_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) def test_is_file(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'fileA').is_file()) self.assertFalse((P / 'dirA').is_file()) self.assertFalse((P / 'non-existing').is_file()) self.assertFalse((P / 'fileA' / 'bah').is_file()) - if os_helper.can_symlink(): + if self.can_symlink: self.assertTrue((P / 'linkA').is_file()) self.assertFalse((P / 'linkB').is_file()) self.assertFalse((P/ 'brokenLink').is_file()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - - def test_is_mount(self): - P = self.cls(BASE) - if os.name == 'nt': - R = self.cls('c:\\') - else: - R = self.cls('/') - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - self.assertTrue(R.is_mount()) - if os_helper.can_symlink(): - self.assertFalse((P / 'linkA').is_mount()) - self.assertIs((R / '\udfff').is_mount(), False) + self.assertFalse((P / 'fileA\udfff').is_file()) + self.assertFalse((P / 'fileA\x00').is_file()) + + def test_is_file_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) def test_is_symlink(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) self.assertFalse((P / 'dirA').is_symlink()) self.assertFalse((P / 'non-existing').is_symlink()) self.assertFalse((P / 'fileA' / 'bah').is_symlink()) - if os_helper.can_symlink(): + if self.can_symlink: self.assertTrue((P / 'linkA').is_symlink()) self.assertTrue((P / 'linkB').is_symlink()) self.assertTrue((P/ 'brokenLink').is_symlink()) self.assertIs((P / 'fileA\udfff').is_file(), False) self.assertIs((P / 'fileA\x00').is_file(), False) - if os_helper.can_symlink(): + if self.can_symlink: self.assertIs((P / 'linkA\udfff').is_file(), False) self.assertIs((P / 'linkA\x00').is_file(), False) - def test_is_junction(self): - P = self.cls(BASE) + def test_is_junction_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) - with mock.patch.object(P._flavour, 'isjunction'): - self.assertEqual(P.is_junction(), P._flavour.isjunction.return_value) - P._flavour.isjunction.assert_called_once_with(P) + def test_is_junction_true(self): + P = self.cls(self.base) + + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) def test_is_fifo_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_fifo()) self.assertFalse((P / 'dirA').is_fifo()) self.assertFalse((P / 'non-existing').is_fifo()) @@ -2574,7 +2732,7 @@ def test_is_fifo_false(self): @unittest.skipIf(sys.platform == "vxworks", "fifo requires special path on VxWorks") def test_is_fifo_true(self): - P = self.cls(BASE, 'myfifo') + P = self.cls(self.base, 'myfifo') try: os.mkfifo(str(P)) except PermissionError as e: @@ -2582,11 +2740,11 @@ def test_is_fifo_true(self): self.assertTrue(P.is_fifo()) self.assertFalse(P.is_socket()) self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'myfifo\udfff').is_fifo(), False) - self.assertIs(self.cls(BASE, 'myfifo\x00').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) def test_is_socket_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_socket()) self.assertFalse((P / 'dirA').is_socket()) self.assertFalse((P / 'non-existing').is_socket()) @@ -2602,7 +2760,7 @@ def test_is_socket_false(self): is_wasi, "Cannot create socket on WASI." ) def test_is_socket_true(self): - P = self.cls(BASE, 'mysock') + P = self.cls(self.base, 'mysock') sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.addCleanup(sock.close) try: @@ -2614,11 +2772,11 @@ def test_is_socket_true(self): self.assertTrue(P.is_socket()) self.assertFalse(P.is_fifo()) self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'mysock\udfff').is_socket(), False) - self.assertIs(self.cls(BASE, 'mysock\x00').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) def test_is_block_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_block_device()) self.assertFalse((P / 'dirA').is_block_device()) self.assertFalse((P / 'non-existing').is_block_device()) @@ -2627,7 +2785,7 @@ def test_is_block_device_false(self): self.assertIs((P / 'fileA\x00').is_block_device(), False) def test_is_char_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_char_device()) self.assertFalse((P / 'dirA').is_char_device()) self.assertFalse((P / 'non-existing').is_char_device()) @@ -2646,329 +2804,361 @@ def test_is_char_device_true(self): self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) - def test_pickling_common(self): - p = self.cls(BASE, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - - def _check_complex_symlinks(self, link0_target): - # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) - self.dirlink(os.path.join('link0', 'link0'), join('link1')) - self.dirlink(os.path.join('link1', 'link1'), join('link2')) - self.dirlink(os.path.join('link2', 'link2'), join('link3')) - self.dirlink(link0_target, join('link0')) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - - # Resolve relative paths. - old_path = os.getcwd() - os.chdir(BASE) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - finally: - os.chdir(old_path) - - @os_helper.skip_unless_symlink - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) - - @os_helper.skip_unless_symlink - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) + if os.name == 'nt': + R = self.cls('c:\\') + else: + R = self.cls('/') + self.assertTrue(R.is_mount()) + self.assertFalse((R / '\udfff').is_mount()) - @os_helper.skip_unless_symlink - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) + def test_samefile(self): + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = parser.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): self.cls(foo="bar") + @needs_symlinks + def test_iterdir_symlink(self): + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) -class WalkTests(unittest.TestCase): - - def setUp(self): - self.addCleanup(os_helper.rmtree, os_helper.TESTFN) - - # Build: - # TESTFN/ - # TEST1/ a file kid and two directory kids - # tmp1 - # SUB1/ a file kid and a directory kid - # tmp2 - # SUB11/ no kids - # SUB2/ a file kid and a dirsymlink kid - # tmp3 - # SUB21/ not readable - # tmp5 - # link/ a symlink to TEST2 - # broken_link - # broken_link2 - # broken_link3 - # TEST2/ - # tmp4 a lone file - self.walk_path = pathlib.Path(os_helper.TESTFN, "TEST1") - self.sub1_path = self.walk_path / "SUB1" - self.sub11_path = self.sub1_path / "SUB11" - self.sub2_path = self.walk_path / "SUB2" - sub21_path= self.sub2_path / "SUB21" - tmp1_path = self.walk_path / "tmp1" - tmp2_path = self.sub1_path / "tmp2" - tmp3_path = self.sub2_path / "tmp3" - tmp5_path = sub21_path / "tmp3" - self.link_path = self.sub2_path / "link" - t2_path = pathlib.Path(os_helper.TESTFN, "TEST2") - tmp4_path = pathlib.Path(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = self.sub2_path / "broken_link" - broken_link2_path = self.sub2_path / "broken_link2" - broken_link3_path = self.sub2_path / "broken_link3" - - os.makedirs(self.sub11_path) - os.makedirs(self.sub2_path) - os.makedirs(sub21_path) - os.makedirs(t2_path) - - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: - with open(path, "x", encoding='utf-8') as f: - f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") - - if os_helper.can_symlink(): - os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) - os.symlink(pathlib.Path('tmp3', 'broken'), broken_link2_path, True) - os.symlink(pathlib.Path('SUB21', 'tmp5'), broken_link3_path, True) - self.sub2_tree = (self.sub2_path, ["SUB21"], - ["broken_link", "broken_link2", "broken_link3", - "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, ["SUB21"], ["tmp3"]) - - if not is_emscripten: - # Emscripten fails with inaccessible directories. - os.chmod(sub21_path, 0) - try: - os.listdir(sub21_path) - except PermissionError: - self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU) - else: - os.chmod(sub21_path, stat.S_IRWXU) - os.unlink(tmp5_path) - os.rmdir(sub21_path) - del self.sub2_tree[1][:1] - - def test_walk_topdown(self): - walker = self.walk_path.walk() - entry = next(walker) - entry[1].sort() # Ensure we visit SUB1 before SUB2 - self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) - entry = next(walker) - self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"])) - entry = next(walker) - self.assertEqual(entry, (self.sub11_path, [], [])) - entry = next(walker) - entry[1].sort() - entry[2].sort() - self.assertEqual(entry, self.sub2_tree) - with self.assertRaises(StopIteration): - next(walker) - - def test_walk_prune(self, walk_path=None): - if walk_path is None: - walk_path = self.walk_path - # Prune the search. - all = [] - for root, dirs, files in walk_path.walk(): - all.append((root, dirs, files)) - if 'SUB1' in dirs: - # Note that this also mutates the dirs we appended to all! - dirs.remove('SUB1') - - self.assertEqual(len(all), 2) - self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) - - all[1][-1].sort() - all[1][1].sort() - self.assertEqual(all[1], self.sub2_tree) - - def test_file_like_path(self): - self.test_walk_prune(FakePath(self.walk_path).__fspath__()) - - def test_walk_bottom_up(self): - seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False - for path, dirnames, filenames in self.walk_path.walk(top_down=False): - if path == self.walk_path: - self.assertFalse(seen_testfn) - self.assertTrue(seen_sub1) - self.assertTrue(seen_sub2) - self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"]) - self.assertEqual(filenames, ["tmp1"]) - seen_testfn = True - elif path == self.sub1_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub1) - self.assertTrue(seen_sub11) - self.assertEqual(dirnames, ["SUB11"]) - self.assertEqual(filenames, ["tmp2"]) - seen_sub1 = True - elif path == self.sub11_path: - self.assertFalse(seen_sub1) - self.assertFalse(seen_sub11) - self.assertEqual(dirnames, []) - self.assertEqual(filenames, []) - seen_sub11 = True - elif path == self.sub2_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub2) - self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1])) - self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2])) - seen_sub2 = True - else: - raise AssertionError(f"Unexpected path: {path}") - self.assertTrue(seen_testfn) - - @os_helper.skip_unless_symlink - def test_walk_follow_symlinks(self): - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - @os_helper.skip_unless_symlink - def test_walk_symlink_location(self): - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") + @needs_posix + def test_glob_posix(self): + P = self.cls + p = P(self.base) + q = p / "FILEa" + given = set(p.glob("FILEa")) + expect = {q} if q.info.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break + @needs_windows + def test_glob_windows(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) - def test_walk_bad_dir(self): - errors = [] - walk_it = self.walk_path.walk(on_error=errors.append) - root, dirs, files = next(walk_it) - self.assertEqual(errors, []) - dir1 = 'SUB1' - path1 = root / dir1 - path1new = (root / dir1).with_suffix(".new") - path1.rename(path1new) - try: - roots = [r for r, _, _ in walk_it] - self.assertTrue(errors) - self.assertNotIn(path1, roots) - self.assertNotIn(path1new, roots) - for dir2 in dirs: - if dir2 != dir1: - self.assertIn(root / dir2, roots) - finally: - path1new.rename(path1) + def test_glob_empty_pattern(self): + p = self.cls('') + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('.')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('./')) - def test_walk_many_open_files(self): + def test_glob_many_open_files(self): depth = 30 - base = pathlib.Path(os_helper.TESTFN, 'deep') - path = pathlib.Path(base, *(['d']*depth)) - path.mkdir(parents=True) - - iters = [base.walk(top_down=False) for _ in range(100)] - for i in range(depth + 1): - expected = (path, ['d'] if i else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path.parent - - iters = [base.walk(top_down=True) for _ in range(100)] - path = base - for i in range(depth + 1): - expected = (path, ['d'] if i < depth else [], []) + P = self.cls + p = base = P(self.base) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' for it in iters: - self.assertEqual(next(it), expected) - path = path / 'd' + self.assertEqual(next(it), p) - def test_walk_above_recursion_limit(self): - recursion_limit = 40 + def test_glob_above_recursion_limit(self): + recursion_limit = 50 # directory_depth > recursion_limit directory_depth = recursion_limit + 10 - base = pathlib.Path(os_helper.TESTFN, 'deep') - path = pathlib.Path(base, *(['d'] * directory_depth)) + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) + with infinite_recursion(recursion_limit): + list(base.glob('**/')) + def test_glob_pathlike(self): + P = self.cls + p = P(self.base) + pattern = "dir*/file*" + expect = {p / "dirB/fileB", p / "dirC/fileC"} + self.assertEqual(expect, set(p.glob(P(pattern)))) + self.assertEqual(expect, set(p.glob(FakePath(pattern)))) -class PathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.Path + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(self.base, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(self.base) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) - def test_concrete_class(self): - p = self.cls('a') - self.assertIs(type(p), - pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath) + @needs_symlinks + def test_glob_dot(self): + P = self.cls + with os_helper.change_cwd(P(self.base, "dirC")): + self.assertEqual( + set(P('.').glob('*')), {P("fileC"), P("novel.txt"), P("dirD")}) + self.assertEqual( + set(P('.').glob('**')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD"), P(".")}) + self.assertEqual( + set(P('.').glob('**/*')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD")}) + self.assertEqual( + set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + if sys.platform == "emscripten": + # Emscripten will return ELOOP if there are 49 or more ..'s. + # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. + NDOTDOTS = 48 + else: + NDOTDOTS = 50 + self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) - def test_unsupported_flavour(self): - if os.name == 'nt': - self.assertRaises(NotImplementedError, pathlib.PosixPath) + def test_glob_inaccessible(self): + P = self.cls + p = P(self.base, "mydir1", "mydir2") + p.mkdir(parents=True) + p.parent.chmod(0) + self.assertEqual(set(p.glob('*')), set()) + + def test_rglob_pathlike(self): + P = self.cls + p = P(self.base, "dirC") + pattern = "**/file*" + expect = {p / "fileC", p / "dirD/fileD"} + self.assertEqual(expect, set(p.rglob(P(pattern)))) + self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + + @needs_symlinks + def test_glob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + @needs_symlinks + def test_rglob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_recurse_symlinks_false(self): + def _check(path, glob, expected): + actual = set(path.rglob(glob, recurse_symlinks=False)) + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + it = p.rglob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(p, "fileA", ["fileA"]) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "**/fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + + if self.can_symlink: + _check(p, "*/fileB", ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p, "*/", [ + "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", + "dirC/dirD/", "dirE/", "linkB/"]) else: - self.assertRaises(NotImplementedError, pathlib.WindowsPath) + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "**/file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "dir*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/**/", ["dirC/dirD/"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + _check(p, "**", ["dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + _check(p, "**/", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + @needs_posix + def test_rglob_posix(self): + P = self.cls + p = P(self.base, "dirC") + q = p / "dirD" / "FILEd" + given = set(p.rglob("FILEd")) + expect = {q} if q.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + @needs_windows + def test_rglob_windows(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) -@only_posix -class PosixPathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.PosixPath + @needs_symlinks + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + P = self.cls + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + @needs_symlinks + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + @needs_symlinks + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - def test_absolute(self): + @needs_posix + def test_absolute_posix(self): P = self.cls self.assertEqual(str(P('/').absolute()), '/') self.assertEqual(str(P('/a').absolute()), '/a') @@ -2979,29 +3169,28 @@ def test_absolute(self): self.assertEqual(str(P('//a').absolute()), '//a') self.assertEqual(str(P('//a/b').absolute()), '//a/b') - def _check_symlink_loop(self, *args, strict=True): - path = self.cls(*args) - with self.assertRaises(RuntimeError): - print(path.resolve(strict)) - @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_open_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'new_file').open('wb'): pass - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) with (p / 'other_new_file').open('wb'): pass - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) + @needs_posix def test_resolve_root(self): current_directory = os.getcwd() try: @@ -3012,66 +3201,33 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) + @needs_posix def test_touch_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) + p = self.cls(self.base) (p / 'new_file').touch() - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) (p / 'other_new_file').touch() - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(join('masked_new_file')) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - @os_helper.skip_unless_symlink - def test_resolve_loop(self): - # Loops with relative symlinks. - os.symlink('linkX/inside', join('linkX')) - self._check_symlink_loop(BASE, 'linkX') - os.symlink('linkY', join('linkY')) - self._check_symlink_loop(BASE, 'linkY') - os.symlink('linkZ/../linkZ', join('linkZ')) - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - self._check_symlink_loop(BASE, 'linkZ', 'foo', strict=False) - # Loops with absolute symlinks. - os.symlink(join('linkU/inside'), join('linkU')) - self._check_symlink_loop(BASE, 'linkU') - os.symlink(join('linkV'), join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - os.symlink(join('linkW/../linkW'), join('linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - self._check_symlink_loop(BASE, 'linkW', 'foo', strict=False) - - def test_glob(self): - P = self.cls - p = P(BASE) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - @unittest.skipUnless(hasattr(pwd, 'getpwall'), 'pwd module does not expose getpwall()') @unittest.skipIf(sys.platform == "vxworks", "no home directory on VxWorks") - def test_expanduser(self): + @needs_posix + def test_expanduser_posix(self): P = self.cls import_helper.import_module('pwd') import pwd @@ -3105,7 +3261,7 @@ def test_expanduser(self): p7 = P(f'~{fakename}/Documents') with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) + env.unset('HOME') self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') @@ -3126,6 +3282,7 @@ def test_expanduser(self): @unittest.skipIf(sys.platform != "darwin", "Bad file descriptor in /dev/fd affects only macOS") + @needs_posix def test_handling_bad_descriptor(self): try: file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] @@ -3147,12 +3304,31 @@ def test_handling_bad_descriptor(self): self.fail("Bad file descriptor not handled.") raise + @needs_posix + def test_from_uri_posix(self): + P = self.cls + self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'file://foo/bar') + self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) + if not is_wasi: + self.assertEqual(P.from_uri(f'file://{socket.gethostname()}/foo/bar'), + P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, '/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') -@only_nt -class WindowsPathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.WindowsPath + @needs_posix + def test_from_uri_pathname2url_posix(self): + P = self.cls + self.assertEqual(P.from_uri(pathname2url('/foo/bar', add_scheme=True)), P('/foo/bar')) + self.assertEqual(P.from_uri(pathname2url('//foo/bar', add_scheme=True)), P('//foo/bar')) - def test_absolute(self): + @needs_windows + def test_absolute_windows(self): P = self.cls # Simple absolute paths. @@ -3176,17 +3352,17 @@ def test_absolute(self): self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(share, 'a', 'b', 'c')) - drive = os.path.splitdrive(BASE)[0] - with os_helper.change_cwd(BASE): + drive = os.path.splitdrive(self.base)[0] + with os_helper.change_cwd(self.base): # Relative path with root self.assertEqual(str(P('\\').absolute()), drive + '\\') self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') # Relative path on current drive - self.assertEqual(str(P(drive).absolute()), BASE) - self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) + self.assertEqual(str(P(drive).absolute()), self.base) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(self.base, 'foo')) - with os_helper.subst_drive(BASE) as other_drive: + with os_helper.subst_drive(self.base) as other_drive: # Set the working directory on the substitute drive saved_cwd = os.getcwd() other_cwd = f'{other_drive}\\dirA' @@ -3197,29 +3373,11 @@ def test_absolute(self): self.assertEqual(str(P(other_drive).absolute()), other_cwd) self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') - def test_glob(self): - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA") }) - self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - def test_expanduser(self): + @needs_windows + def test_expanduser_windows(self): P = self.cls with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - env.pop('USERPROFILE', None) - env.pop('HOMEPATH', None) - env.pop('HOMEDRIVE', None) + env.unset('HOME', 'USERPROFILE', 'HOMEPATH', 'HOMEDRIVE') env['USERNAME'] = 'alice' # test that the path returns unchanged @@ -3257,8 +3415,7 @@ def check(): env['HOMEPATH'] = 'Users\\alice' check() - env.pop('HOMEDRIVE', None) - env.pop('HOMEPATH', None) + env.unset('HOMEDRIVE', 'HOMEPATH') env['USERPROFILE'] = 'C:\\Users\\alice' check() @@ -3266,16 +3423,218 @@ def check(): env['HOME'] = 'C:\\Users\\eve' check() + @needs_windows + def test_from_uri_windows(self): + P = self.cls + # DOS drive paths + self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) + # UNC paths + self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) + # Localhost paths + self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) + # Invalid paths + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + @needs_windows + def test_from_uri_pathname2url_windows(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) + + @needs_windows + def test_owner_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').owner() -class PurePathSubclassTest(_BasePurePathTest, unittest.TestCase): - class cls(pathlib.PurePath): - pass + @needs_windows + def test_group_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').group() - # repr() roundtripping is not supported in custom subclass. - test_repr_roundtrips = None + +class PathWalkTest(unittest.TestCase): + cls = pathlib.Path + base = PathTest.base + can_symlink = PathTest.can_symlink + + def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + self.walk_path = self.cls(self.base, "TEST1") + self.sub1_path = self.walk_path / "SUB1" + self.sub11_path = self.sub1_path / "SUB11" + self.sub2_path = self.walk_path / "SUB2" + self.link_path = self.sub2_path / "link" + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + t2_path = self.cls(self.base, "TEST2") + os.makedirs(self.sub11_path) + os.makedirs(self.sub2_path) + os.makedirs(t2_path) + + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + tmp4_path = self.cls(self.base, "TEST2", "tmp4") + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with open(path, "w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + os.symlink(t2_path, self.link_path, target_is_directory=True) + os.symlink('broken', broken_link_path) + os.symlink(os.path.join('tmp3', 'broken'), broken_link2_path) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) + sub21_path= self.sub2_path / "SUB21" + tmp5_path = sub21_path / "tmp3" + broken_link3_path = self.sub2_path / "broken_link3" + + os.makedirs(sub21_path) + tmp5_path.write_text("I am tmp5, blame test_pathlib.") + if self.can_symlink: + os.symlink(tmp5_path, broken_link3_path) + self.sub2_tree[2].append('broken_link3') + self.sub2_tree[2].sort() + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.sub2_tree[1].append('SUB21') + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + + def tearDown(self): + if 'SUB21' in self.sub2_tree[1]: + os.chmod(self.sub2_path / "SUB21", stat.S_IRWXU) + os_helper.rmtree(self.base) + + def test_walk_bad_dir(self): + errors = [] + walk_it = self.walk_path.walk(on_error=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = root / dir1 + path1new = (root / dir1).with_suffix(".new") + path1.rename(path1new) + try: + roots = [r for r, _, _ in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(root / dir2, roots) + finally: + path1new.rename(path1) + + def test_walk_many_open_files(self): + depth = 30 + base = self.cls(self.base, 'deep') + path = self.cls(base, *(['d']*depth)) + path.mkdir(parents=True) + + iters = [base.walk(top_down=False) for _ in range(100)] + for i in range(depth + 1): + expected = (path, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path.parent + + iters = [base.walk(top_down=True) for _ in range(100)] + path = base + for i in range(depth + 1): + expected = (path, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path / 'd' + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + @needs_symlinks + def test_walk_follow_symlinks(self): + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + @needs_symlinks + def test_walk_symlink_location(self): + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + + +@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + +@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath -class PathSubclassTest(_BasePathTest, unittest.TestCase): +class PathSubclassTest(PathTest): class cls(pathlib.Path): pass diff --git a/Lib/test/test_pathlib/test_read.py b/Lib/test/test_pathlib/test_read.py new file mode 100644 index 00000000000..482203c290a --- /dev/null +++ b/Lib/test/test_pathlib/test_read.py @@ -0,0 +1,343 @@ +""" +Tests for pathlib.types._ReadablePath +""" + +import collections.abc +import io +import sys +import unittest + +from .support import is_pypi +from .support.local_path import ReadableLocalPath, LocalPathGround +from .support.zip_path import ReadableZipPath, ZipPathGround + +if is_pypi: + from pathlib_abc import PathInfo, _ReadablePath + from pathlib_abc._os import magic_open +else: + from pathlib.types import PathInfo, _ReadablePath + from pathlib._os import magic_open + + +class ReadTestBase: + def setUp(self): + self.root = self.ground.setup() + self.ground.create_hierarchy(self.root) + + def tearDown(self): + self.ground.teardown(self.root) + + def test_is_readable(self): + self.assertIsInstance(self.root, _ReadablePath) + + def test_open_r(self): + p = self.root / 'fileA' + with magic_open(p, 'r', encoding='utf-8') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), 'this is file A\n') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_open_r_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + with magic_open(p, 'r'): + pass + self.assertEqual(wc.filename, __file__) + + def test_open_rb(self): + p = self.root / 'fileA' + with magic_open(p, 'rb') as f: + self.assertEqual(f.read(), b'this is file A\n') + self.assertRaises(ValueError, magic_open, p, 'rb', encoding='utf8') + self.assertRaises(ValueError, magic_open, p, 'rb', errors='strict') + self.assertRaises(ValueError, magic_open, p, 'rb', newline='') + + def test_read_bytes(self): + p = self.root / 'fileA' + self.assertEqual(p.read_bytes(), b'this is file A\n') + + def test_read_text(self): + p = self.root / 'fileA' + self.assertEqual(p.read_text(encoding='utf-8'), 'this is file A\n') + q = self.root / 'abc' + self.ground.create_file(q, b'\xe4bcdefg') + self.assertEqual(q.read_text(encoding='latin-1'), 'äbcdefg') + self.assertEqual(q.read_text(encoding='utf-8', errors='ignore'), 'bcdefg') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_read_text_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + p.read_text() + self.assertEqual(wc.filename, __file__) + + def test_read_text_with_newlines(self): + p = self.root / 'abc' + self.ground.create_file(p, b'abcde\r\nfghlk\n\rmnopq') + # Check that `\n` character change nothing + self.assertEqual(p.read_text(encoding='utf-8', newline='\n'), 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + self.assertEqual(p.read_text(encoding='utf-8', newline='\r'), 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + self.assertEqual(p.read_text(encoding='utf-8', newline='\r\n'), 'abcde\r\nfghlk\n\rmnopq') + + def test_iterdir(self): + expected = ['dirA', 'dirB', 'dirC', 'fileA'] + if self.ground.can_symlink: + expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] + expected = {self.root.joinpath(name) for name in expected} + actual = set(self.root.iterdir()) + self.assertEqual(actual, expected) + + def test_iterdir_nodir(self): + p = self.root / 'fileA' + self.assertRaises(OSError, p.iterdir) + + def test_iterdir_info(self): + for child in self.root.iterdir(): + self.assertIsInstance(child.info, PathInfo) + self.assertTrue(child.info.exists(follow_symlinks=False)) + + def test_glob(self): + if not self.ground.can_symlink: + self.skipTest("requires symlinks") + + p = self.root + sep = self.root.parser.sep + altsep = self.root.parser.altsep + def check(pattern, expected): + if altsep: + expected = {name.replace(altsep, sep) for name in expected} + expected = {p.joinpath(name) for name in expected} + actual = set(p.glob(pattern, recurse_symlinks=True)) + self.assertEqual(actual, expected) + + it = p.glob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + self.assertEqual(list(it), [p.joinpath("fileA")]) + check("*A", ["dirA", "fileA", "linkA"]) + check("*A", ['dirA', 'fileA', 'linkA']) + check("*B/*", ["dirB/fileB", "linkB/fileB"]) + check("*B/*", ['dirB/fileB', 'linkB/fileB']) + check("brokenLink", ['brokenLink']) + check("brokenLinkLoop", ['brokenLinkLoop']) + check("**/", ["", "dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/", "linkB/"]) + check("**/*/", ["dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/", "linkB/"]) + check("*/", ["dirA/", "dirB/", "dirC/", "linkB/"]) + check("*/dirD/**/", ["dirC/dirD/"]) + check("*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + check("dir*/**", ["dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirB/", "dirB/fileB", "dirC/", + "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + check("dir*/**/", ["dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/"]) + check("dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", "dirC/..", "dirC/dirD/.."]) + check("dir*/*/**", ["dirA/linkC/", "dirA/linkC/fileB", "dirC/dirD/", "dirC/dirD/fileD"]) + check("dir*/*/**/", ["dirA/linkC/", "dirC/dirD/"]) + check("dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + check("dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) + check("dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + check("dir*/**/fileC", ["dirC/fileC"]) + check("dir*/file*", ["dirB/fileB", "dirC/fileC"]) + check("**/*/fileA", []) + check("fileB", []) + check("**/*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + check("**/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + check("*/fileB", ["dirB/fileB", "linkB/fileB"]) + check("*/fileB", ['dirB/fileB', 'linkB/fileB']) + check("**/file*", + ["fileA", "dirA/linkC/fileB", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", + "linkB/fileB"]) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + + def test_walk_top_down(self): + it = self.root.walk() + + path, dirnames, filenames = next(it) + dirnames.sort() + filenames.sort() + self.assertEqual(path, self.root) + self.assertEqual(dirnames, ['dirA', 'dirB', 'dirC']) + self.assertEqual(filenames, ['brokenLink', 'brokenLinkLoop', 'fileA', 'linkA', 'linkB'] + if self.ground.can_symlink else ['fileA']) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirA') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['linkC'] if self.ground.can_symlink else []) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirB') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileB']) + + path, dirnames, filenames = next(it) + filenames.sort() + self.assertEqual(path, self.root / 'dirC') + self.assertEqual(dirnames, ['dirD']) + self.assertEqual(filenames, ['fileC', 'novel.txt']) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirC' / 'dirD') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileD']) + + self.assertRaises(StopIteration, next, it) + + def test_walk_prune(self): + expected = {self.root, self.root / 'dirA', self.root / 'dirC', self.root / 'dirC' / 'dirD'} + actual = set() + for path, dirnames, filenames in self.root.walk(): + actual.add(path) + if path == self.root: + dirnames.remove('dirB') + self.assertEqual(actual, expected) + + def test_walk_bottom_up(self): + seen_root = seen_dira = seen_dirb = seen_dirc = seen_dird = False + for path, dirnames, filenames in self.root.walk(top_down=False): + if path == self.root: + self.assertFalse(seen_root) + self.assertTrue(seen_dira) + self.assertTrue(seen_dirb) + self.assertTrue(seen_dirc) + self.assertEqual(sorted(dirnames), ['dirA', 'dirB', 'dirC']) + self.assertEqual(sorted(filenames), + ['brokenLink', 'brokenLinkLoop', 'fileA', 'linkA', 'linkB'] + if self.ground.can_symlink else ['fileA']) + seen_root = True + elif path == self.root / 'dirA': + self.assertFalse(seen_root) + self.assertFalse(seen_dira) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['linkC'] if self.ground.can_symlink else []) + seen_dira = True + elif path == self.root / 'dirB': + self.assertFalse(seen_root) + self.assertFalse(seen_dirb) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileB']) + seen_dirb = True + elif path == self.root / 'dirC': + self.assertFalse(seen_root) + self.assertFalse(seen_dirc) + self.assertTrue(seen_dird) + self.assertEqual(dirnames, ['dirD']) + self.assertEqual(sorted(filenames), ['fileC', 'novel.txt']) + seen_dirc = True + elif path == self.root / 'dirC' / 'dirD': + self.assertFalse(seen_root) + self.assertFalse(seen_dirc) + self.assertFalse(seen_dird) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileD']) + seen_dird = True + else: + raise AssertionError(f"Unexpected path: {path}") + self.assertTrue(seen_root) + + def test_info_exists(self): + p = self.root + self.assertTrue(p.info.exists()) + self.assertTrue((p / 'dirA').info.exists()) + self.assertTrue((p / 'dirA').info.exists(follow_symlinks=False)) + self.assertTrue((p / 'fileA').info.exists()) + self.assertTrue((p / 'fileA').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.exists()) + self.assertFalse((p / 'non-existing').info.exists(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.exists()) + self.assertTrue((p / 'linkA').info.exists(follow_symlinks=False)) + self.assertTrue((p / 'linkB').info.exists()) + self.assertTrue((p / 'linkB').info.exists(follow_symlinks=True)) + self.assertFalse((p / 'brokenLink').info.exists()) + self.assertTrue((p / 'brokenLink').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.exists()) + self.assertTrue((p / 'brokenLinkLoop').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'fileA\udfff').info.exists()) + self.assertFalse((p / 'fileA\udfff').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'fileA\x00').info.exists()) + self.assertFalse((p / 'fileA\x00').info.exists(follow_symlinks=False)) + + def test_info_is_dir(self): + p = self.root + self.assertTrue((p / 'dirA').info.is_dir()) + self.assertTrue((p / 'dirA').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'fileA').info.is_dir()) + self.assertFalse((p / 'fileA').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.is_dir()) + self.assertFalse((p / 'non-existing').info.is_dir(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertFalse((p / 'linkA').info.is_dir()) + self.assertFalse((p / 'linkA').info.is_dir(follow_symlinks=False)) + self.assertTrue((p / 'linkB').info.is_dir()) + self.assertFalse((p / 'linkB').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'brokenLink').info.is_dir()) + self.assertFalse((p / 'brokenLink').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.is_dir()) + self.assertFalse((p / 'brokenLinkLoop').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'dirA\udfff').info.is_dir()) + self.assertFalse((p / 'dirA\udfff').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'dirA\x00').info.is_dir()) + self.assertFalse((p / 'dirA\x00').info.is_dir(follow_symlinks=False)) + + def test_info_is_file(self): + p = self.root + self.assertTrue((p / 'fileA').info.is_file()) + self.assertTrue((p / 'fileA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'dirA').info.is_file()) + self.assertFalse((p / 'dirA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.is_file()) + self.assertFalse((p / 'non-existing').info.is_file(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.is_file()) + self.assertFalse((p / 'linkA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'linkB').info.is_file()) + self.assertFalse((p / 'linkB').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'brokenLink').info.is_file()) + self.assertFalse((p / 'brokenLink').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.is_file()) + self.assertFalse((p / 'brokenLinkLoop').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'fileA\udfff').info.is_file()) + self.assertFalse((p / 'fileA\udfff').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'fileA\x00').info.is_file()) + self.assertFalse((p / 'fileA\x00').info.is_file(follow_symlinks=False)) + + def test_info_is_symlink(self): + p = self.root + self.assertFalse((p / 'fileA').info.is_symlink()) + self.assertFalse((p / 'dirA').info.is_symlink()) + self.assertFalse((p / 'non-existing').info.is_symlink()) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.is_symlink()) + self.assertTrue((p / 'linkB').info.is_symlink()) + self.assertTrue((p / 'brokenLink').info.is_symlink()) + self.assertFalse((p / 'linkA\udfff').info.is_symlink()) + self.assertFalse((p / 'linkA\x00').info.is_symlink()) + self.assertTrue((p / 'brokenLinkLoop').info.is_symlink()) + self.assertFalse((p / 'fileA\udfff').info.is_symlink()) + self.assertFalse((p / 'fileA\x00').info.is_symlink()) + + +class ZipPathReadTest(ReadTestBase, unittest.TestCase): + ground = ZipPathGround(ReadableZipPath) + + +class LocalPathReadTest(ReadTestBase, unittest.TestCase): + ground = LocalPathGround(ReadableLocalPath) + + +if not is_pypi: + from pathlib import Path + + class PathReadTest(ReadTestBase, unittest.TestCase): + ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_write.py b/Lib/test/test_pathlib/test_write.py new file mode 100644 index 00000000000..15054e804ec --- /dev/null +++ b/Lib/test/test_pathlib/test_write.py @@ -0,0 +1,143 @@ +""" +Tests for pathlib.types._WritablePath +""" + +import io +import os +import sys +import unittest + +from .support import is_pypi +from .support.local_path import WritableLocalPath, LocalPathGround +from .support.zip_path import WritableZipPath, ZipPathGround + +if is_pypi: + from pathlib_abc import _WritablePath + from pathlib_abc._os import magic_open +else: + from pathlib.types import _WritablePath + from pathlib._os import magic_open + + +class WriteTestBase: + def setUp(self): + self.root = self.ground.setup() + + def tearDown(self): + self.ground.teardown(self.root) + + def test_is_writable(self): + self.assertIsInstance(self.root, _WritablePath) + + def test_open_w(self): + p = self.root / 'fileA' + with magic_open(p, 'w', encoding='utf-8') as f: + self.assertIsInstance(f, io.TextIOBase) + f.write('this is file A\n') + self.assertEqual(self.ground.readtext(p), 'this is file A\n') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_open_w_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + with magic_open(p, 'w'): + pass + self.assertEqual(wc.filename, __file__) + + def test_open_wb(self): + p = self.root / 'fileA' + with magic_open(p, 'wb') as f: + #self.assertIsInstance(f, io.BufferedWriter) + f.write(b'this is file A\n') + self.assertEqual(self.ground.readbytes(p), b'this is file A\n') + self.assertRaises(ValueError, magic_open, p, 'wb', encoding='utf8') + self.assertRaises(ValueError, magic_open, p, 'wb', errors='strict') + self.assertRaises(ValueError, magic_open, p, 'wb', newline='') + + def test_write_bytes(self): + p = self.root / 'fileA' + data = b'abcdefg' + self.assertEqual(len(data), p.write_bytes(data)) + self.assertEqual(self.ground.readbytes(p), data) + # Check that trying to write str does not truncate the file. + self.assertRaises(TypeError, p.write_bytes, 'somestr') + self.assertEqual(self.ground.readbytes(p), data) + + def test_write_text(self): + p = self.root / 'fileA' + data = 'äbcdefg' + self.assertEqual(len(data), p.write_text(data, encoding='latin-1')) + self.assertEqual(self.ground.readbytes(p), b'\xe4bcdefg') + # Check that trying to write bytes does not truncate the file. + self.assertRaises(TypeError, p.write_text, b'somebytes', encoding='utf-8') + self.assertEqual(self.ground.readbytes(p), b'\xe4bcdefg') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_write_text_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + p.write_text('abcdefg') + self.assertEqual(wc.filename, __file__) + + def test_write_text_with_newlines(self): + # Check that `\n` character change nothing + p = self.root / 'fileA' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\n') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\nfghlk\n\rmnopq') + + # Check that `\r` character replaces `\n` + p = self.root / 'fileB' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\r') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\rfghlk\r\rmnopq') + + # Check that `\r\n` character replaces `\n` + p = self.root / 'fileC' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\r\n') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\r\nfghlk\r\n\rmnopq') + + # Check that no argument passed will change `\n` to `os.linesep` + os_linesep_byte = bytes(os.linesep, encoding='ascii') + p = self.root / 'fileD' + p.write_text('abcde\nfghlk\n\rmnopq', encoding='utf-8') + self.assertEqual(self.ground.readbytes(p), + b'abcde' + os_linesep_byte + + b'fghlk' + os_linesep_byte + b'\rmnopq') + + def test_mkdir(self): + p = self.root / 'newdirA' + self.assertFalse(self.ground.isdir(p)) + p.mkdir() + self.assertTrue(self.ground.isdir(p)) + + def test_symlink_to(self): + if not self.ground.can_symlink: + self.skipTest('needs symlinks') + link = self.root.joinpath('linkA') + link.symlink_to('fileA') + self.assertTrue(self.ground.islink(link)) + self.assertEqual(self.ground.readlink(link), 'fileA') + + +class ZipPathWriteTest(WriteTestBase, unittest.TestCase): + ground = ZipPathGround(WritableZipPath) + + +class LocalPathWriteTest(WriteTestBase, unittest.TestCase): + ground = LocalPathGround(WritableLocalPath) + + +if not is_pypi: + from pathlib import Path + + class PathWriteTest(WriteTestBase, unittest.TestCase): + ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 847ca001e43..6ca1fa0ba40 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -3394,7 +3394,6 @@ class Keys: self.assertIs(z, None) class TestSourceLocations(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_jump_threading(self): # See gh-123048 def f(): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py new file mode 100644 index 00000000000..c5c10cb5318 --- /dev/null +++ b/Lib/test/test_peepholer.py @@ -0,0 +1,2761 @@ +import dis +import gc +from itertools import combinations, product +import opcode +import sys +import textwrap +import unittest +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +from test import support +from test.support.bytecode_helper import ( + BytecodeTestCase, CfgOptimizationTestCase, CompilationStepTestCase) + + +def compile_pattern_with_fast_locals(pattern): + source = textwrap.dedent( + f""" + def f(x): + match x: + case {pattern}: + pass + """ + ) + namespace = {} + exec(source, namespace) + return namespace["f"].__code__ + + +def count_instr_recursively(f, opname): + count = 0 + for instr in dis.get_instructions(f): + if instr.opname == opname: + count += 1 + if hasattr(f, '__code__'): + f = f.__code__ + for c in f.co_consts: + if hasattr(c, 'co_code'): + count += count_instr_recursively(c, opname) + return count + + +def get_binop_argval(arg): + for i, nb_op in enumerate(opcode._nb_ops): + if arg == nb_op[0]: + return i + assert False, f"{arg} is not a valid BINARY_OP argument." + + +class TestTranforms(BytecodeTestCase): + + def check_jump_targets(self, code): + instructions = list(dis.get_instructions(code)) + targets = {instr.offset: instr for instr in instructions} + for instr in instructions: + if 'JUMP_' not in instr.opname: + continue + tgt = targets[instr.argval] + # jump to unconditional jump + if tgt.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD'): + self.fail(f'{instr.opname} at {instr.offset} ' + f'jumps to {tgt.opname} at {tgt.offset}') + # unconditional jump to RETURN_VALUE + if (instr.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD') and + tgt.opname == 'RETURN_VALUE'): + self.fail(f'{instr.opname} at {instr.offset} ' + f'jumps to {tgt.opname} at {tgt.offset}') + + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unot(self): + # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' + def unot(x): + if not x == 2: + del x + self.assertNotInBytecode(unot, 'UNARY_NOT') + self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') + self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_inversion_of_is_or_in(self): + for line, cmp_op, invert in ( + ('not a is b', 'IS_OP', 1,), + ('not a is not b', 'IS_OP', 0,), + ('not a in b', 'CONTAINS_OP', 1,), + ('not a not in b', 'CONTAINS_OP', 0,), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertInBytecode(code, cmp_op, invert) + self.check_lnotab(code) + + def test_global_as_constant(self): + # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False + def f(): + x = None + x = None + return x + def g(): + x = True + return x + def h(): + x = False + return x + + for func, elem in ((f, None), (g, True), (h, False)): + with self.subTest(func=func): + self.assertNotInBytecode(func, 'LOAD_GLOBAL') + self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) + + def f(): + 'Adding a docstring made this test fail in Py2.5.0' + return None + + self.assertNotInBytecode(f, 'LOAD_GLOBAL') + self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_while_one(self): + # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx + def f(): + while 1: + pass + return list + for elem in ('LOAD_CONST', 'POP_JUMP_IF_FALSE'): + self.assertNotInBytecode(f, elem) + for elem in ('JUMP_BACKWARD',): + self.assertInBytecode(f, elem) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pack_unpack(self): + for line, elem in ( + ('a, = a,', 'LOAD_CONST',), + ('a, b = a, b', 'SWAP',), + ('a, b, c = a, b, c', 'SWAP',), + ): + with self.subTest(line=line): + code = compile(line,'','single') + self.assertInBytecode(code, elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.assertNotInBytecode(code, 'UNPACK_SEQUENCE') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_tuples_of_constants(self): + for line, elem in ( + ('a = 1,2,3', (1, 2, 3)), + ('("a","b","c")', ('a', 'b', 'c')), + ('a,b,c,d = 1,2,3,4', (1, 2, 3, 4)), + ('(None, 1, None)', (None, 1, None)), + ('((1, 2), 3, 4)', ((1, 2), 3, 4)), + ): + with self.subTest(line=line): + code = compile(line,'','single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) + + # Long tuples should be folded too. + code = compile(repr(tuple(range(10000))),'','single') + self.assertNotInBytecode(code, 'BUILD_TUPLE') + # One LOAD_CONST for the tuple, one for the None return value + load_consts = [instr for instr in dis.get_instructions(code) + if instr.opname == 'LOAD_CONST'] + self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) + + # Bug 1053819: Tuple of constants misidentified when presented with: + # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . + # The following would segfault upon compilation + def crater(): + (~[ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + ],) + self.check_lnotab(crater) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_lists_of_constants(self): + for line, elem in ( + # in/not in constants with BUILD_LIST should be folded to a tuple: + ('a in [1,2,3]', (1, 2, 3)), + ('a not in ["a","b","c"]', ('a', 'b', 'c')), + ('a in [None, 1, None]', (None, 1, None)), + ('a not in [(1, 2), 3, 4]', ((1, 2), 3, 4)), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_sets_of_constants(self): + for line, elem in ( + # in/not in constants with BUILD_SET should be folded to a frozenset: + ('a in {1,2,3}', frozenset({1, 2, 3})), + ('a not in {"a","b","c"}', frozenset({'a', 'c', 'b'})), + ('a in {None, 1, None}', frozenset({1, None})), + ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), + ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertNotInBytecode(code, 'BUILD_SET') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) + + # Ensure that the resulting code actually works: + def f(a): + return a in {1, 2, 3} + + def g(a): + return a not in {1, 2, 3} + + self.assertTrue(f(3)) + self.assertTrue(not f(4)) + self.check_lnotab(f) + + self.assertTrue(not g(3)) + self.assertTrue(g(4)) + self.check_lnotab(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_small_int(self): + tests = [ + ('(0, )[0]', 0), + ('(1 + 2, )[0]', 3), + ('(2 + 2 * 2, )[0]', 6), + ('(1, (1 + 2 + 3, ))[1][0]', 6), + ('1 + 2', 3), + ('2 + 2 * 2 // 2 - 2', 2), + ('(255, )[0]', 255), + ('(256, )[0]', None), + ('(1000, )[0]', None), + ('(1 - 2, )[0]', None), + ('255 + 0', 255), + ('255 + 1', None), + ('-1', None), + ('--1', 1), + ('--255', 255), + ('--256', None), + ('~1', None), + ('~~1', 1), + ('~~255', 255), + ('~~256', None), + ('++255', 255), + ('++256', None), + ] + for expr, oparg in tests: + with self.subTest(expr=expr, oparg=oparg): + code = compile(expr, '', 'single') + if oparg is not None: + self.assertInBytecode(code, 'LOAD_SMALL_INT', oparg) + else: + self.assertNotInBytecode(code, 'LOAD_SMALL_INT') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_unaryop(self): + intrinsic_positive = 5 + tests = [ + ('-0', 'UNARY_NEGATIVE', None, True, 'LOAD_SMALL_INT', 0), + ('-0.0', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.0), + ('-(1.0-1.0)', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.0), + ('-0.5', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.5), + ('---1', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -1), + ('---""', 'UNARY_NEGATIVE', None, False, None, None), + ('~~~1', 'UNARY_INVERT', None, True, 'LOAD_CONST', -2), + ('~~~""', 'UNARY_INVERT', None, False, None, None), + ('not not True', 'UNARY_NOT', None, True, 'LOAD_CONST', True), + ('not not x', 'UNARY_NOT', None, True, 'LOAD_NAME', 'x'), # this should be optimized regardless of constant or not + ('+++1', 'CALL_INTRINSIC_1', intrinsic_positive, True, 'LOAD_SMALL_INT', 1), + ('---x', 'UNARY_NEGATIVE', None, False, None, None), + ('~~~x', 'UNARY_INVERT', None, False, None, None), + ('+++x', 'CALL_INTRINSIC_1', intrinsic_positive, False, None, None), + ('~True', 'UNARY_INVERT', None, False, None, None), + ] + + for ( + expr, + original_opcode, + original_argval, + is_optimized, + optimized_opcode, + optimized_argval, + ) in tests: + with self.subTest(expr=expr, is_optimized=is_optimized): + code = compile(expr, "", "single") + if is_optimized: + self.assertNotInBytecode(code, original_opcode, argval=original_argval) + self.assertInBytecode(code, optimized_opcode, argval=optimized_argval) + else: + self.assertInBytecode(code, original_opcode, argval=original_argval) + self.check_lnotab(code) + + # Check that -0.0 works after marshaling + def negzero(): + return -(1.0-1.0) + + for instr in dis.get_instructions(negzero): + self.assertNotStartsWith(instr.opname, 'UNARY_') + self.check_lnotab(negzero) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_binop(self): + tests = [ + ('1 + 2', 'NB_ADD', True, 'LOAD_SMALL_INT', 3), + ('1 + 2 + 3', 'NB_ADD', True, 'LOAD_SMALL_INT', 6), + ('1 + ""', 'NB_ADD', False, None, None), + ('1 - 2', 'NB_SUBTRACT', True, 'LOAD_CONST', -1), + ('1 - 2 - 3', 'NB_SUBTRACT', True, 'LOAD_CONST', -4), + ('1 - ""', 'NB_SUBTRACT', False, None, None), + ('2 * 2', 'NB_MULTIPLY', True, 'LOAD_SMALL_INT', 4), + ('2 * 2 * 2', 'NB_MULTIPLY', True, 'LOAD_SMALL_INT', 8), + ('2 / 2', 'NB_TRUE_DIVIDE', True, 'LOAD_CONST', 1.0), + ('2 / 2 / 2', 'NB_TRUE_DIVIDE', True, 'LOAD_CONST', 0.5), + ('2 / ""', 'NB_TRUE_DIVIDE', False, None, None), + ('2 // 2', 'NB_FLOOR_DIVIDE', True, 'LOAD_SMALL_INT', 1), + ('2 // 2 // 2', 'NB_FLOOR_DIVIDE', True, 'LOAD_SMALL_INT', 0), + ('2 // ""', 'NB_FLOOR_DIVIDE', False, None, None), + ('2 % 2', 'NB_REMAINDER', True, 'LOAD_SMALL_INT', 0), + ('2 % 2 % 2', 'NB_REMAINDER', True, 'LOAD_SMALL_INT', 0), + ('2 % ()', 'NB_REMAINDER', False, None, None), + ('2 ** 2', 'NB_POWER', True, 'LOAD_SMALL_INT', 4), + ('2 ** 2 ** 2', 'NB_POWER', True, 'LOAD_SMALL_INT', 16), + ('2 ** ""', 'NB_POWER', False, None, None), + ('2 << 2', 'NB_LSHIFT', True, 'LOAD_SMALL_INT', 8), + ('2 << 2 << 2', 'NB_LSHIFT', True, 'LOAD_SMALL_INT', 32), + ('2 << ""', 'NB_LSHIFT', False, None, None), + ('2 >> 2', 'NB_RSHIFT', True, 'LOAD_SMALL_INT', 0), + ('2 >> 2 >> 2', 'NB_RSHIFT', True, 'LOAD_SMALL_INT', 0), + ('2 >> ""', 'NB_RSHIFT', False, None, None), + ('2 | 2', 'NB_OR', True, 'LOAD_SMALL_INT', 2), + ('2 | 2 | 2', 'NB_OR', True, 'LOAD_SMALL_INT', 2), + ('2 | ""', 'NB_OR', False, None, None), + ('2 & 2', 'NB_AND', True, 'LOAD_SMALL_INT', 2), + ('2 & 2 & 2', 'NB_AND', True, 'LOAD_SMALL_INT', 2), + ('2 & ""', 'NB_AND', False, None, None), + ('2 ^ 2', 'NB_XOR', True, 'LOAD_SMALL_INT', 0), + ('2 ^ 2 ^ 2', 'NB_XOR', True, 'LOAD_SMALL_INT', 2), + ('2 ^ ""', 'NB_XOR', False, None, None), + ('(1, )[0]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 1), + ('(1, )[-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 1), + ('(1 + 2, )[0]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 3), + ('(1, (1, 2))[1][1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, 2)[2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, (1, 2))[1][2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, (1, 2))[1:6][0][2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('"a"[0]', 'NB_SUBSCR', True, 'LOAD_CONST', 'a'), + ('("a" + "b")[1]', 'NB_SUBSCR', True, 'LOAD_CONST', 'b'), + ('("a" + "b", )[0][1]', 'NB_SUBSCR', True, 'LOAD_CONST', 'b'), + ('("a" * 10)[9]', 'NB_SUBSCR', True, 'LOAD_CONST', 'a'), + ('(1, )[1]', 'NB_SUBSCR', False, None, None), + ('(1, )[-2]', 'NB_SUBSCR', False, None, None), + ('"a"[1]', 'NB_SUBSCR', False, None, None), + ('"a"[-2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b")[2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b", )[0][2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b", )[1][0]', 'NB_SUBSCR', False, None, None), + ('("a" * 10)[10]', 'NB_SUBSCR', False, None, None), + ('(1, (1, 2))[2:6][0][2-1]', 'NB_SUBSCR', False, None, None), + ] + + for ( + expr, + nb_op, + is_optimized, + optimized_opcode, + optimized_argval + ) in tests: + with self.subTest(expr=expr, is_optimized=is_optimized): + code = compile(expr, '', 'single') + nb_op_val = get_binop_argval(nb_op) + if is_optimized: + self.assertNotInBytecode(code, 'BINARY_OP', argval=nb_op_val) + self.assertInBytecode(code, optimized_opcode, argval=optimized_argval) + else: + self.assertInBytecode(code, 'BINARY_OP', argval=nb_op_val) + self.check_lnotab(code) + + # Verify that large sequences do not result from folding + code = compile('"x"*10000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 10000) + self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) + code = compile('1<<1000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 1000) + self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) + code = compile('2**1000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 1000) + self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) + + # Test binary subscript on unicode + # valid code get optimized + code = compile('"foo"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 'f') + self.assertNotInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + code = compile('"\u0061\uffff"[1]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\uffff') + self.assertNotInBytecode(code,'BINARY_OP') + self.check_lnotab(code) + + # With PEP 393, non-BMP char get optimized + code = compile('"\U00012345"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') + self.assertNotInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + + # invalid code doesn't get optimized + # out of range + code = compile('"fuu"[10]', '', 'single') + self.assertInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + + + def test_constant_folding_remove_nop_location(self): + sources = [ + """ + (- + - + - + 1) + """, + + """ + (1 + + + 2 + + + 3) + """, + + """ + (1, + 2, + 3)[0] + """, + + """ + [1, + 2, + 3] + """, + + """ + {1, + 2, + 3} + """, + + """ + 1 in [ + 1, + 2, + 3 + ] + """, + + """ + 1 in { + 1, + 2, + 3 + } + """, + + """ + for _ in [1, + 2, + 3]: + pass + """, + + """ + for _ in [1, + 2, + x]: + pass + """, + + """ + for _ in {1, + 2, + 3}: + pass + """ + ] + + for source in sources: + code = compile(textwrap.dedent(source), '', 'single') + self.assertNotInBytecode(code, 'NOP') + + def test_elim_extra_return(self): + # RETURN LOAD_CONST None RETURN --> RETURN + def f(x): + return x + self.assertNotInBytecode(f, 'LOAD_CONST', None) + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 1) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_to_return(self): + # JUMP_FORWARD to RETURN --> RETURN + def f(cond, true_value, false_value): + # Intentionally use two-line expression to test issue37213. + return (true_value if cond + else false_value) + self.check_jump_targets(f) + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_BACKWARD') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 2) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_to_uncond_jump(self): + # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump + def f(): + if a: + # Intentionally use two-line expression to test issue37213. + if (c + or d): + foo() + else: + baz() + self.check_jump_targets(f) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_to_uncond_jump2(self): + # POP_JUMP_IF_FALSE to JUMP_BACKWARD --> POP_JUMP_IF_FALSE to non-jump + def f(): + while a: + # Intentionally use two-line expression to test issue37213. + if (c + or d): + a = foo() + self.check_jump_targets(f) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_to_uncond_jump3(self): + # Intentionally use two-line expressions to test issue37213. + # POP_JUMP_IF_FALSE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_FALSE to non-jump + def f(a, b, c): + return ((a and b) + and c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 2) + # POP_JUMP_IF_TRUE to POP_JUMP_IF_TRUE --> POP_JUMP_IF_TRUE to non-jump + def f(a, b, c): + return ((a or b) + or c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 2) + # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump + def f(a, b, c): + return ((a and b) + or c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) + # POP_JUMP_IF_TRUE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE to non-jump + def f(a, b, c): + return ((a or b) + and c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_to_uncond_jump4(self): + def f(): + for i in range(5): + if i > 3: + print(i) + self.check_jump_targets(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_jump_after_return1(self): + # Eliminate dead code: jumps immediately after returns can't be reached + def f(cond1, cond2): + if cond1: return 1 + if cond2: return 2 + while 1: + return 3 + while 1: + if cond1: return 4 + return 5 + return 6 + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_BACKWARD') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) + + def test_make_function_doesnt_bail(self): + def f(): + def g()->1+1: + pass + return g + self.assertNotInBytecode(f, 'BINARY_OP') + self.check_lnotab(f) + + def test_in_literal_list(self): + def containtest(): + return x in [a, b] + self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) + + def test_iterate_literal_list(self): + def forloop(): + for x in [a, b]: + pass + self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) + + def test_condition_with_binop_with_bools(self): + def f(): + if True or False: + return 1 + return 0 + self.assertEqual(f(), 1) + self.check_lnotab(f) + + def test_if_with_if_expression(self): + # Check bpo-37289 + def f(x): + if (True if x else False): + return True + return False + self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_assignment_idiom_in_comprehensions(self): + def listcomp(): + return [y for x in a for y in [f(x)]] + self.assertEqual(count_instr_recursively(listcomp, 'FOR_ITER'), 1) + def setcomp(): + return {y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(setcomp, 'FOR_ITER'), 1) + def dictcomp(): + return {y: y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(dictcomp, 'FOR_ITER'), 1) + def genexpr(): + return (y for x in a for y in [f(x)]) + self.assertEqual(count_instr_recursively(genexpr, 'FOR_ITER'), 1) + + @support.requires_resource('cpu') + def test_format_combinations(self): + flags = '-+ #0' + testcases = [ + *product(('', '1234', 'абвг'), 'sra'), + *product((1234, -1234), 'duioxX'), + *product((1234.5678901, -1234.5678901), 'duifegFEG'), + *product((float('inf'), -float('inf')), 'fegFEG'), + ] + width_precs = [ + *product(('', '1', '30'), ('', '.', '.0', '.2')), + ('', '.40'), + ('30', '.40'), + ] + for value, suffix in testcases: + for width, prec in width_precs: + for r in range(len(flags) + 1): + for spec in combinations(flags, r): + fmt = '%' + ''.join(spec) + width + prec + suffix + with self.subTest(fmt=fmt, value=value): + s1 = fmt % value + s2 = eval(f'{fmt!r} % (x,)', {'x': value}) + self.assertEqual(s2, s1, f'{fmt = }') + + def test_format_misc(self): + def format(fmt, *values): + vars = [f'x{i+1}' for i in range(len(values))] + if len(vars) == 1: + args = '(' + vars[0] + ',)' + else: + args = '(' + ', '.join(vars) + ')' + return eval(f'{fmt!r} % {args}', dict(zip(vars, values))) + + self.assertEqual(format('string'), 'string') + self.assertEqual(format('x = %s!', 1234), 'x = 1234!') + self.assertEqual(format('x = %d!', 1234), 'x = 1234!') + self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') + self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') + self.assertEqual(format('x = %s!', 1234.0000625), 'x = 1234.0000625!') + self.assertEqual(format('x = %f!', 1234.0000625), 'x = 1234.000063!') + self.assertEqual(format('x = %d!', 1234.0000625), 'x = 1234!') + self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') + self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') + self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_format_errors(self): + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s' % ()") + with self.assertRaisesRegex(TypeError, + 'not all arguments converted during string formatting'): + eval("'%s' % (x, y)", {'x': 1, 'y': 2}) + with self.assertRaisesRegex(ValueError, 'incomplete format'): + eval("'%s%' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(ValueError, 'incomplete format'): + eval("'%s%%%' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s%z' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(ValueError, 'unsupported format character'): + eval("'%s%z' % (x, 5)", {'x': 1234}) + with self.assertRaisesRegex(TypeError, 'a real number is required, not str'): + eval("'%d' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, 'an integer is required, not float'): + eval("'%x' % (x,)", {'x': 1234.56}) + with self.assertRaisesRegex(TypeError, 'an integer is required, not str'): + eval("'%x' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, 'must be real number, not str'): + eval("'%f' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s, %s' % (x, *y)", {'x': 1, 'y': []}) + with self.assertRaisesRegex(TypeError, + 'not all arguments converted during string formatting'): + eval("'%s, %s' % (x, *y)", {'x': 1, 'y': [2, 3]}) + + def test_static_swaps_unpack_two(self): + def f(a, b): + a, b = a, b + b, a = a, b + self.assertNotInBytecode(f, "SWAP") + + def test_static_swaps_unpack_three(self): + def f(a, b, c): + a, b, c = a, b, c + a, c, b = a, b, c + b, a, c = a, b, c + b, c, a = a, b, c + c, a, b = a, b, c + c, b, a = a, b, c + self.assertNotInBytecode(f, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_mapping(self): + for a, b, c in product("_a", "_b", "_c"): + pattern = f"{{'a': {a}, 'b': {b}, 'c': {c}}}" + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + self.assertNotInBytecode(code, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_class(self): + forms = [ + "C({}, {}, {})", + "C({}, {}, c={})", + "C({}, b={}, c={})", + "C(a={}, b={}, c={})" + ] + for a, b, c in product("_a", "_b", "_c"): + for form in forms: + pattern = form.format(a, b, c) + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + self.assertNotInBytecode(code, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_sequence(self): + swaps = {"*_, b, c", "a, *_, c", "a, b, *_"} + forms = ["{}, {}, {}", "{}, {}, *{}", "{}, *{}, {}", "*{}, {}, {}"] + for a, b, c in product("_a", "_b", "_c"): + for form in forms: + pattern = form.format(a, b, c) + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + if pattern in swaps: + # If this fails... great! Remove this pattern from swaps + # to prevent regressing on any improvement: + self.assertInBytecode(code, "SWAP") + else: + self.assertNotInBytecode(code, "SWAP") + + +class TestBuglets(unittest.TestCase): + + def test_bug_11510(self): + # folded constant set optimization was commingled with the tuple + # unpacking optimization which would fail if the set had duplicate + # elements so that the set length was unexpected + def f(): + x, y = {1, 1} + return x, y + with self.assertRaises(ValueError): + f() + + def test_bpo_42057(self): + for i in range(10): + try: + raise Exception + except Exception or Exception: + pass + + def test_bpo_45773_pop_jump_if_true(self): + compile("while True or spam: pass", "<test>", "exec") + + def test_bpo_45773_pop_jump_if_false(self): + compile("while True or not spam: pass", "<test>", "exec") + + +class TestMarkingVariablesAsUnKnown(BytecodeTestCase): + + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_known_simple(self): + def f(): + x = 1 + y = x + x + self.assertInBytecode(f, 'LOAD_FAST_BORROW_LOAD_FAST_BORROW') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_unknown_simple(self): + def f(): + if condition(): + x = 1 + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_unknown_because_del(self): + def f(): + x = 1 + del x + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_known_because_parameter(self): + def f1(x): + print(x) + self.assertInBytecode(f1, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f1, 'LOAD_FAST_CHECK') + + def f2(*, x): + print(x) + self.assertInBytecode(f2, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f2, 'LOAD_FAST_CHECK') + + def f3(*args): + print(args) + self.assertInBytecode(f3, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f3, 'LOAD_FAST_CHECK') + + def f4(**kwargs): + print(kwargs) + self.assertInBytecode(f4, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f4, 'LOAD_FAST_CHECK') + + def f5(x=0): + print(x) + self.assertInBytecode(f5, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f5, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_known_because_already_loaded(self): + def f(): + if condition(): + x = 1 + print(x) + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertInBytecode(f, 'LOAD_FAST_BORROW') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_known_multiple_branches(self): + def f(): + if condition(): + x = 1 + else: + x = 2 + print(x) + self.assertInBytecode(f, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_unknown_after_error(self): + def f(): + try: + res = 1 / 0 + except ZeroDivisionError: + pass + return res + # LOAD_FAST (known) still occurs in the no-exception branch. + # Assert that it doesn't occur in the LOAD_FAST_CHECK branch. + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_unknown_after_error_2(self): + def f(): + try: + 1 / 0 + except ZeroDivisionError: + print(a, b, c, d, e, f, g) + a = b = c = d = e = f = g = 1 + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_load_fast_too_many_locals(self): + # When there get to be too many locals to analyze completely, + # later locals are all converted to LOAD_FAST_CHECK, except + # when a store or prior load occurred in the same basicblock. + def f(): + a00 = a01 = a02 = a03 = a04 = a05 = a06 = a07 = a08 = a09 = 1 + a10 = a11 = a12 = a13 = a14 = a15 = a16 = a17 = a18 = a19 = 1 + a20 = a21 = a22 = a23 = a24 = a25 = a26 = a27 = a28 = a29 = 1 + a30 = a31 = a32 = a33 = a34 = a35 = a36 = a37 = a38 = a39 = 1 + a40 = a41 = a42 = a43 = a44 = a45 = a46 = a47 = a48 = a49 = 1 + a50 = a51 = a52 = a53 = a54 = a55 = a56 = a57 = a58 = a59 = 1 + a60 = a61 = a62 = a63 = a64 = a65 = a66 = a67 = a68 = a69 = 1 + a70 = a71 = a72 = a73 = a74 = a75 = a76 = a77 = a78 = a79 = 1 + del a72, a73 + print(a73) + print(a70, a71, a72, a73) + while True: + print(a00, a01, a62, a63) + print(a64, a65, a78, a79) + + self.assertInBytecode(f, 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', ("a00", "a01")) + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a00") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a01") + for i in 62, 63: + # First 64 locals: analyze completely + self.assertInBytecode(f, 'LOAD_FAST_BORROW', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + for i in 64, 65, 78, 79: + # Locals >=64 not in the same basicblock + self.assertInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST', f"a{i:02}") + for i in 70, 71: + # Locals >=64 in the same basicblock + self.assertInBytecode(f, 'LOAD_FAST_BORROW', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + # del statements should invalidate within basicblocks. + self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a72") + self.assertNotInBytecode(f, 'LOAD_FAST', "a72") + # previous checked loads within a basicblock enable unchecked loads + self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a73") + self.assertInBytecode(f, 'LOAD_FAST_BORROW', "a73") + + def test_setting_lineno_no_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + L = 7 + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + result = f() + self.assertIsNone(result) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_setting_lineno_one_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + del x + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + e = r"assigning None to 1 unbound local" + with self.assertWarnsRegex(RuntimeWarning, e): + sys.settrace(trace) + result = f() + self.assertEqual(result, 4) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_setting_lineno_two_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + del x, y + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + e = r"assigning None to 2 unbound locals" + with self.assertWarnsRegex(RuntimeWarning, e): + sys.settrace(trace) + result = f() + self.assertEqual(result, 4) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + def make_function_with_no_checks(self): + code = textwrap.dedent("""\ + def f(): + x = 2 + L = 3 + L = 4 + L = 5 + if not L: + x + 7 + y = 2 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + return f + + def test_modifying_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["x"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + def test_initializing_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["y"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + +class DirectCfgOptimizerTests(CfgOptimizationTestCase): + + def cfg_optimization_test(self, insts, expected_insts, + consts=None, expected_consts=None, + nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + + if expected_consts is None: + expected_consts = consts + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() + self.assertInstructionsMatch(opt_insts, expected_insts) + self.assertEqual(opt_consts, expected_consts) + + def test_conditional_jump_forward_non_const_condition(self): + insts = [ + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', None, 13), + lbl, + ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', None, 14), + ] + expected_insts = [ + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_SMALL_INT', 2, 13), + ('RETURN_VALUE', None, 13), + lbl, + ('LOAD_SMALL_INT', 3, 14), + ('RETURN_VALUE', None, 14), + ] + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0]) + + def test_list_exceeding_stack_use_guideline(self): + def f(): + return [ + 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39 + ] + self.assertEqual(f(), list(range(40))) + + def test_set_exceeding_stack_use_guideline(self): + def f(): + return { + 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39 + } + self.assertEqual(f(), frozenset(range(40))) + + def test_nested_const_foldings(self): + # (1, (--2 + ++2 * 2 // 2 - 2, )[0], ~~3, not not True) ==> (1, 2, 3, True) + intrinsic_positive = 5 + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_NEGATIVE', None, 0), + ('NOP', None, 0), + ('UNARY_NEGATIVE', None, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('CALL_INTRINSIC_1', intrinsic_positive, 0), + ('NOP', None, 0), + ('CALL_INTRINSIC_1', intrinsic_positive, 0), + ('BINARY_OP', get_binop_argval('NB_MULTIPLY')), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('BINARY_OP', get_binop_argval('NB_FLOOR_DIVIDE')), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', get_binop_argval('NB_ADD')), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('BINARY_OP', get_binop_argval('NB_SUBTRACT')), + ('NOP', None, 0), + ('BUILD_TUPLE', 1, 0), + ('LOAD_SMALL_INT', 0, 0), + ('BINARY_OP', get_binop_argval('NB_SUBSCR'), 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('UNARY_INVERT', None, 0), + ('NOP', None, 0), + ('UNARY_INVERT', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('UNARY_NOT', None, 0), + ('NOP', None, 0), + ('UNARY_NOT', None, 0), + ('NOP', None, 0), + ('BUILD_TUPLE', 4, 0), + ('NOP', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2, (1, 2, 3, True)]) + + def test_build_empty_tuple(self): + before = [ + ('BUILD_TUPLE', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + + def test_fold_tuple_of_constants(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_TUPLE', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_TUPLE', 3, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_fold_constant_intrinsic_list_to_tuple(self): + INTRINSIC_LIST_TO_TUPLE = 6 + + # long tuple + consts = 1000 + before = ( + [('BUILD_LIST', 0, 0)] + + [('LOAD_CONST', 0, 0), ('LIST_APPEND', 1, 0)] * consts + + [('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), ('RETURN_VALUE', None, 0)] + ) + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + result_const = tuple(["test"] * consts) + self.cfg_optimization_test(before, after, consts=["test"], expected_consts=["test", result_const]) + + # empty list + before = [ + ('BUILD_LIST', 0, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + + # multiple BUILD_LIST 0: ([], 1, [], 2) + same = [ + ('BUILD_LIST', 0, 0), + ('BUILD_LIST', 0, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LIST_APPEND', 1, 0), + ('BUILD_LIST', 0, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('LIST_APPEND', 1, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(same, same, consts=[]) + + # nested folding: (1, 1+1, 3) + before = [ + ('BUILD_LIST', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', get_binop_argval('NB_ADD'), 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 3, 0), + ('LIST_APPEND', 1, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # NOP's in between: (1, 2, 3) + before = [ + ('BUILD_LIST', 0, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + def test_optimize_if_const_list(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_LIST', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('BUILD_LIST', 0, 0), + ('LOAD_CONST', 0, 0), + ('LIST_EXTEND', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # need minimum 3 consts to optimize + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 3, 0), + ('BUILD_LIST', 3, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_optimize_if_const_set(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_SET', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('BUILD_SET', 0, 0), + ('LOAD_CONST', 0, 0), + ('SET_UPDATE', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[frozenset({1, 2, 3})]) + + # need minimum 3 consts to optimize + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 3, 0), + ('BUILD_SET', 3, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_optimize_literal_list_for_iter(self): + # for _ in [1, 2]: pass ==> for _ in (1, 2): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, (1, 2)]) + + # for _ in [1, x]: pass ==> for _ in (1, x): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_LIST', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_TUPLE', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None]) + + def test_optimize_literal_set_for_iter(self): + # for _ in {1, 2}: pass ==> for _ in (1, 2): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, frozenset({1, 2})]) + + # non constant literal set is not changed + # for _ in {1, x}: pass ==> for _ in {1, x}: pass + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_SET', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[None], expected_consts=[None]) + + def test_optimize_literal_list_contains(self): + # x in [1, 2] ==> x in (1, 2) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_CONST', 1, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, (1, 2)]) + + # x in [1, y] ==> x in (1, y) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_LIST', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_TUPLE', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None]) + + def test_optimize_literal_set_contains(self): + # x in {1, 2} ==> x in (1, 2) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_CONST', 1, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, frozenset({1, 2})]) + + # non constant literal set is not changed + # x in {1, y} ==> x in {1, y} + same = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_SET', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[None], expected_consts=[None]) + + def test_optimize_unary_not(self): + # test folding + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True, False]) + + # test cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test folding & cancel out + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True]) + + # test folding & eliminate to bool + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True, False]) + + # test cancel out & eliminate to bool (to bool stays as we are not iterating to a fixed point) + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + is_ = in_ = 0 + isnot = notin = 1 + + # test is/isnot + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', isnot, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', isnot, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot cancel out & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', notin, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', notin, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin cancel out & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + def test_optimize_if_const_unaryop(self): + # test unary negative + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_NEGATIVE', None, 0), + ('UNARY_NEGATIVE', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2]) + + # test unary invert + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_INVERT', None, 0), + ('UNARY_INVERT', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-3]) + + # test unary positive + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('CALL_INTRINSIC_1', 5, 0), + ('CALL_INTRINSIC_1', 5, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + def test_optimize_if_const_binop(self): + add = get_binop_argval('NB_ADD') + sub = get_binop_argval('NB_SUBTRACT') + mul = get_binop_argval('NB_MULTIPLY') + div = get_binop_argval('NB_TRUE_DIVIDE') + floor = get_binop_argval('NB_FLOOR_DIVIDE') + rem = get_binop_argval('NB_REMAINDER') + pow = get_binop_argval('NB_POWER') + lshift = get_binop_argval('NB_LSHIFT') + rshift = get_binop_argval('NB_RSHIFT') + or_ = get_binop_argval('NB_OR') + and_ = get_binop_argval('NB_AND') + xor = get_binop_argval('NB_XOR') + subscr = get_binop_argval('NB_SUBSCR') + + # test add + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', add, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', add, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 6, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test sub + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', sub, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', sub, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2]) + + # test mul + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', mul, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', mul, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 8, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test div + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', div, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', div, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[1.0, 0.5]) + + # test floor + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', floor, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', floor, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test rem + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', rem, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', rem, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test pow + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', pow, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', pow, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 16, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test lshift + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', lshift, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', lshift, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 4, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test rshift + before = [ + ('LOAD_SMALL_INT', 4, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', rshift, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', rshift, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test or + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', or_, 0), + ('LOAD_SMALL_INT', 4, 0), + ('BINARY_OP', or_, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 7, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test and + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', and_, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', and_, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test xor + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', xor, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', xor, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test subscr + before = [ + ('LOAD_CONST', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', subscr, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', subscr, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 3, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[(1, (1, 2, 3))], expected_consts=[(1, (1, 2, 3))]) + + + def test_conditional_jump_forward_const_condition(self): + # The unreachable branch of the jump is removed, the jump + # becomes redundant and is replaced by a NOP (for the lineno) + + insts = [ + ('LOAD_CONST', 3, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_CONST', 2, 13), + lbl, + ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', None, 14), + ] + expected_insts = [ + ('NOP', None, 11), + ('NOP', None, 12), + ('LOAD_SMALL_INT', 3, 14), + ('RETURN_VALUE', None, 14), + ] + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0]) + + def test_conditional_jump_backward_non_const_condition(self): + insts = [ + lbl1 := self.Label(), + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl1, 12), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', None, 13), + ] + expected = [ + lbl := self.Label(), + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl, 12), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', None, 13), + ] + self.cfg_optimization_test(insts, expected, consts=list(range(5))) + + def test_conditional_jump_backward_const_condition(self): + # The unreachable branch of the jump is removed + insts = [ + lbl1 := self.Label(), + ('LOAD_CONST', 3, 11), + ('POP_JUMP_IF_TRUE', lbl1, 12), + ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', None, 13), + ] + expected_insts = [ + lbl := self.Label(), + ('NOP', None, 11), + ('JUMP', lbl, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_except_handler_label(self): + insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('POP_BLOCK', None, -1), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), + ] + expected_insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('LOAD_SMALL_INT', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_SMALL_INT', 2, 12), + ('RETURN_VALUE', None, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_no_unsafe_static_swap(self): + # We can't change order of two stores to the same location + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('SWAP', 3, 4), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('POP_TOP', None, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('NOP', None, 3), + ('STORE_FAST', 1, 4), + ('POP_TOP', None, 4), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_dead_store_elimination_in_same_lineno(self): + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('NOP', None, 3), + ('POP_TOP', None, 4), + ('STORE_FAST', 1, 4), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_no_dead_store_elimination_in_different_lineno(self): + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 5), + ('STORE_FAST', 1, 6), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('LOAD_SMALL_INT', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 5), + ('STORE_FAST', 1, 6), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_unconditional_jump_threading(self): + + def get_insts(lno1, lno2, op1, op2): + return [ + lbl2 := self.Label(), + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + (op1, lbl1 := self.Label(), lno1), + ('LOAD_NAME', 1, 20), + lbl1, + (op2, lbl2, lno2), + ] + + for op1 in ('JUMP', 'JUMP_NO_INTERRUPT'): + for op2 in ('JUMP', 'JUMP_NO_INTERRUPT'): + # different lines + lno1, lno2 = (4, 5) + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + ('NOP', None, 4), + (op, 0, 5), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + # Threading + for lno1, lno2 in [(-1, -1), (-1, 5), (6, -1), (7, 7)]: + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + lno = lno1 if lno1 != -1 else lno2 + if lno == -1: + lno = 10 # Propagated from the line before + + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + (op, 0, lno), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_list_to_tuple_get_iter(self): + # for _ in (*foo, *bar) -> for _ in [*foo, *bar] + INTRINSIC_LIST_TO_TUPLE = 6 + insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + expected_insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST_BORROW", 1, 4), + ("LIST_EXTEND", 1, 5), + ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + self.cfg_optimization_test(insts, expected_insts, consts=[None]) + + def test_list_to_tuple_get_iter_is_safe(self): + a, b = [], [] + for item in (*(items := [0, 1, 2, 3]),): + a.append(item) + b.append(items.pop()) + self.assertEqual(a, [0, 1, 2, 3]) + self.assertEqual(b, [3, 2, 1, 0]) + self.assertEqual(items, []) + + +class OptimizeLoadFastTestCase(DirectCfgOptimizerTests): + def make_bb(self, insts): + last_loc = insts[-1][2] + maxconst = 0 + for op, arg, _ in insts: + if op == "LOAD_CONST": + maxconst = max(maxconst, arg) + consts = [None for _ in range(maxconst + 1)] + return insts + [ + ("LOAD_CONST", 0, last_loc + 1), + ("RETURN_VALUE", None, last_loc + 2), + ], consts + + def check(self, insts, expected_insts, consts=None): + insts_bb, insts_consts = self.make_bb(insts) + expected_insts_bb, exp_consts = self.make_bb(expected_insts) + self.cfg_optimization_test(insts_bb, expected_insts_bb, + consts=insts_consts, expected_consts=exp_consts) + + def test_optimized(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("BINARY_OP", 2, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("BINARY_OP", 2, 3), + ] + self.check(insts, expected) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 1, 2), + ("SWAP", 2, 3), + ("POP_TOP", None, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_CONST", 1, 2), + ("SWAP", 2, 3), + ("POP_TOP", None, 4), + ] + self.check(insts, expected) + + def test_unoptimized_if_unconsumed(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + insts = [ + ("LOAD_FAST", 0, 1), + ("COPY", 1, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("NOP", None, 2), + ("NOP", None, 3), + ] + self.check(insts, expected) + + def test_unoptimized_if_support_killed(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 2), + ("STORE_FAST", 0, 3), + ("POP_TOP", None, 4), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 2), + ("LOAD_CONST", 0, 3), + ("STORE_FAST_STORE_FAST", ((0 << 4) | 1), 4), + ("POP_TOP", None, 5), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("DELETE_FAST", 0, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, insts) + + def test_unoptimized_if_aliased(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("STORE_FAST", 1, 2), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 3), + ("STORE_FAST_STORE_FAST", ((0 << 4) | 1), 4), + ] + self.check(insts, insts) + + def test_consume_no_inputs(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_LEN", None, 2), + ("STORE_FAST", 1 , 3), + ("STORE_FAST", 2, 4), + ] + self.check(insts, insts) + + def test_consume_some_inputs_no_outputs(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_LEN", None, 2), + ("LIST_APPEND", 0, 3), + ] + self.check(insts, insts) + + def test_check_exc_match(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("CHECK_EXC_MATCH", None, 3) + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("CHECK_EXC_MATCH", None, 3) + ] + self.check(insts, expected) + + def test_for_iter(self): + insts = [ + ("LOAD_FAST", 0, 1), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 2), + ("STORE_FAST", 2, 3), + ("JUMP", top, 4), + end, + ("END_FOR", None, 5), + ("POP_TOP", None, 6), + ("LOAD_CONST", 0, 7), + ("RETURN_VALUE", None, 8), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_load_attr(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 0, 2), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_ATTR", 0, 2), + ] + self.check(insts, expected) + + # Method call, leaves self on stack unconsumed + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 1, 2), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 1, 2), + ] + self.check(insts, expected) + + def test_super_attr(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 0, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("LOAD_FAST_BORROW", 2, 3), + ("LOAD_SUPER_ATTR", 0, 4), + ] + self.check(insts, expected) + + # Method call, leaves self on stack unconsumed + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 1, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 1, 4), + ] + self.check(insts, expected) + + def test_send(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("SEND", end := self.Label(), 3), + ("LOAD_CONST", 0, 4), + ("RETURN_VALUE", None, 5), + end, + ("LOAD_CONST", 0, 6), + ("RETURN_VALUE", None, 7) + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("SEND", end := self.Label(), 3), + ("LOAD_CONST", 0, 4), + ("RETURN_VALUE", None, 5), + end, + ("LOAD_CONST", 0, 6), + ("RETURN_VALUE", None, 7) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_format_simple(self): + # FORMAT_SIMPLE will leave its operand on the stack if it's a unicode + # object. We treat it conservatively and assume that it always leaves + # its operand on the stack. + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + def test_set_function_attribute(self): + # SET_FUNCTION_ATTRIBUTE leaves the function on the stack + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("STORE_FAST", 1, 4), + ("LOAD_CONST", 0, 5), + ("RETURN_VALUE", None, 6) + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + expected = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_get_yield_from_iter(self): + # GET_YIELD_FROM_ITER may leave its operand on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_YIELD_FROM_ITER", None, 2), + ("LOAD_CONST", 0, 3), + send := self.Label(), + ("SEND", end := self.Label(), 5), + ("YIELD_VALUE", 1, 6), + ("RESUME", 2, 7), + ("JUMP", send, 8), + end, + ("END_SEND", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_push_exc_info(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("PUSH_EXC_INFO", None, 2), + ] + self.check(insts, insts) + + def test_load_special(self): + # LOAD_SPECIAL may leave self on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_SPECIAL", 0, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + + def test_del_in_finally(self): + # This loads `obj` onto the stack, executes `del obj`, then returns the + # `obj` from the stack. See gh-133371 for more details. + def create_obj(): + obj = [42] + try: + return obj + finally: + del obj + + obj = create_obj() + # The crash in the linked issue happens while running GC during + # interpreter finalization, so run it here manually. + gc.collect() + self.assertEqual(obj, [42]) + + def test_format_simple_unicode(self): + # Repro from gh-134889 + def f(): + var = f"{1}" + var = f"{var}" + return var + self.assertEqual(f(), "1") + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pep646_syntax.py b/Lib/test/test_pep646_syntax.py new file mode 100644 index 00000000000..d9a0aa9a90e --- /dev/null +++ b/Lib/test/test_pep646_syntax.py @@ -0,0 +1,338 @@ +import doctest +import unittest + +doctests = """ + +Setup + + >>> class AClass: + ... def __init__(self): + ... self._setitem_name = None + ... self._setitem_val = None + ... self._delitem_name = None + ... def __setitem__(self, name, val): + ... self._delitem_name = None + ... self._setitem_name = name + ... self._setitem_val = val + ... def __repr__(self): + ... if self._setitem_name is not None: + ... return f"A[{self._setitem_name}]={self._setitem_val}" + ... elif self._delitem_name is not None: + ... return f"delA[{self._delitem_name}]" + ... def __getitem__(self, name): + ... return ParameterisedA(name) + ... def __delitem__(self, name): + ... self._setitem_name = None + ... self._delitem_name = name + ... + >>> class ParameterisedA: + ... def __init__(self, name): + ... self._name = name + ... def __repr__(self): + ... return f"A[{self._name}]" + ... def __iter__(self): + ... for p in self._name: + ... yield p + >>> class B: + ... def __iter__(self): + ... yield StarredB() + ... def __repr__(self): + ... return "B" + >>> class StarredB: + ... def __repr__(self): + ... return "StarredB" + >>> A = AClass() + >>> b = B() + +Slices that are supposed to work, starring our custom B class + + >>> A[*b] + A[(StarredB,)] + >>> A[*b] = 1; A + A[(StarredB,)]=1 + >>> del A[*b]; A + delA[(StarredB,)] + + >>> A[*b, *b] + A[(StarredB, StarredB)] + >>> A[*b, *b] = 1; A + A[(StarredB, StarredB)]=1 + >>> del A[*b, *b]; A + delA[(StarredB, StarredB)] + + >>> A[b, *b] + A[(B, StarredB)] + >>> A[b, *b] = 1; A + A[(B, StarredB)]=1 + >>> del A[b, *b]; A + delA[(B, StarredB)] + + >>> A[*b, b] + A[(StarredB, B)] + >>> A[*b, b] = 1; A + A[(StarredB, B)]=1 + >>> del A[*b, b]; A + delA[(StarredB, B)] + + >>> A[b, b, *b] + A[(B, B, StarredB)] + >>> A[b, b, *b] = 1; A + A[(B, B, StarredB)]=1 + >>> del A[b, b, *b]; A + delA[(B, B, StarredB)] + + >>> A[*b, b, b] + A[(StarredB, B, B)] + >>> A[*b, b, b] = 1; A + A[(StarredB, B, B)]=1 + >>> del A[*b, b, b]; A + delA[(StarredB, B, B)] + + >>> A[b, *b, b] + A[(B, StarredB, B)] + >>> A[b, *b, b] = 1; A + A[(B, StarredB, B)]=1 + >>> del A[b, *b, b]; A + delA[(B, StarredB, B)] + + >>> A[b, b, *b, b] + A[(B, B, StarredB, B)] + >>> A[b, b, *b, b] = 1; A + A[(B, B, StarredB, B)]=1 + >>> del A[b, b, *b, b]; A + delA[(B, B, StarredB, B)] + + >>> A[b, *b, b, b] + A[(B, StarredB, B, B)] + >>> A[b, *b, b, b] = 1; A + A[(B, StarredB, B, B)]=1 + >>> del A[b, *b, b, b]; A + delA[(B, StarredB, B, B)] + + >>> A[A[b, *b, b]] + A[A[(B, StarredB, B)]] + >>> A[A[b, *b, b]] = 1; A + A[A[(B, StarredB, B)]]=1 + >>> del A[A[b, *b, b]]; A + delA[A[(B, StarredB, B)]] + + >>> A[*A[b, *b, b]] + A[(B, StarredB, B)] + >>> A[*A[b, *b, b]] = 1; A + A[(B, StarredB, B)]=1 + >>> del A[*A[b, *b, b]]; A + delA[(B, StarredB, B)] + + >>> A[b, ...] + A[(B, Ellipsis)] + >>> A[b, ...] = 1; A + A[(B, Ellipsis)]=1 + >>> del A[b, ...]; A + delA[(B, Ellipsis)] + + >>> A[*A[b, ...]] + A[(B, Ellipsis)] + >>> A[*A[b, ...]] = 1; A + A[(B, Ellipsis)]=1 + >>> del A[*A[b, ...]]; A + delA[(B, Ellipsis)] + +Slices that are supposed to work, starring a list + + >>> l = [1, 2, 3] + + >>> A[*l] + A[(1, 2, 3)] + >>> A[*l] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*l]; A + delA[(1, 2, 3)] + + >>> A[*l, 4] + A[(1, 2, 3, 4)] + >>> A[*l, 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*l, 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *l] + A[(0, 1, 2, 3)] + >>> A[0, *l] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *l]; A + delA[(0, 1, 2, 3)] + + >>> A[1:2, *l] + A[(slice(1, 2, None), 1, 2, 3)] + >>> A[1:2, *l] = 1; A + A[(slice(1, 2, None), 1, 2, 3)]=1 + >>> del A[1:2, *l]; A + delA[(slice(1, 2, None), 1, 2, 3)] + + >>> repr(A[1:2, *l]) == repr(A[1:2, 1, 2, 3]) + True + +Slices that are supposed to work, starring a tuple + + >>> t = (1, 2, 3) + + >>> A[*t] + A[(1, 2, 3)] + >>> A[*t] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*t]; A + delA[(1, 2, 3)] + + >>> A[*t, 4] + A[(1, 2, 3, 4)] + >>> A[*t, 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*t, 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *t] + A[(0, 1, 2, 3)] + >>> A[0, *t] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *t]; A + delA[(0, 1, 2, 3)] + + >>> A[1:2, *t] + A[(slice(1, 2, None), 1, 2, 3)] + >>> A[1:2, *t] = 1; A + A[(slice(1, 2, None), 1, 2, 3)]=1 + >>> del A[1:2, *t]; A + delA[(slice(1, 2, None), 1, 2, 3)] + + >>> repr(A[1:2, *t]) == repr(A[1:2, 1, 2, 3]) + True + +Starring an expression (rather than a name) in a slice + + >>> def returns_list(): + ... return [1, 2, 3] + + >>> A[returns_list()] + A[[1, 2, 3]] + >>> A[returns_list()] = 1; A + A[[1, 2, 3]]=1 + >>> del A[returns_list()]; A + delA[[1, 2, 3]] + + >>> A[returns_list(), 4] + A[([1, 2, 3], 4)] + >>> A[returns_list(), 4] = 1; A + A[([1, 2, 3], 4)]=1 + >>> del A[returns_list(), 4]; A + delA[([1, 2, 3], 4)] + + >>> A[*returns_list()] + A[(1, 2, 3)] + >>> A[*returns_list()] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*returns_list()]; A + delA[(1, 2, 3)] + + >>> A[*returns_list(), 4] + A[(1, 2, 3, 4)] + >>> A[*returns_list(), 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*returns_list(), 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *returns_list()] + A[(0, 1, 2, 3)] + >>> A[0, *returns_list()] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *returns_list()]; A + delA[(0, 1, 2, 3)] + + >>> A[*returns_list(), *returns_list()] + A[(1, 2, 3, 1, 2, 3)] + >>> A[*returns_list(), *returns_list()] = 1; A + A[(1, 2, 3, 1, 2, 3)]=1 + >>> del A[*returns_list(), *returns_list()]; A + delA[(1, 2, 3, 1, 2, 3)] + +Using both a starred object and a start:stop in a slice +(See also tests in test_syntax confirming that starring *inside* a start:stop +is *not* valid syntax.) + + >>> A[1:2, *b] + A[(slice(1, 2, None), StarredB)] + >>> A[*b, 1:2] + A[(StarredB, slice(1, 2, None))] + >>> A[1:2, *b, 1:2] + A[(slice(1, 2, None), StarredB, slice(1, 2, None))] + >>> A[*b, 1:2, *b] + A[(StarredB, slice(1, 2, None), StarredB)] + + >>> A[1:, *b] + A[(slice(1, None, None), StarredB)] + >>> A[*b, 1:] + A[(StarredB, slice(1, None, None))] + >>> A[1:, *b, 1:] + A[(slice(1, None, None), StarredB, slice(1, None, None))] + >>> A[*b, 1:, *b] + A[(StarredB, slice(1, None, None), StarredB)] + + >>> A[:1, *b] + A[(slice(None, 1, None), StarredB)] + >>> A[*b, :1] + A[(StarredB, slice(None, 1, None))] + >>> A[:1, *b, :1] + A[(slice(None, 1, None), StarredB, slice(None, 1, None))] + >>> A[*b, :1, *b] + A[(StarredB, slice(None, 1, None), StarredB)] + + >>> A[:, *b] + A[(slice(None, None, None), StarredB)] + >>> A[*b, :] + A[(StarredB, slice(None, None, None))] + >>> A[:, *b, :] + A[(slice(None, None, None), StarredB, slice(None, None, None))] + >>> A[*b, :, *b] + A[(StarredB, slice(None, None, None), StarredB)] + +*args annotated as starred expression + + >>> def f1(*args: *b): pass + >>> f1.__annotations__ + {'args': StarredB} + + >>> def f2(*args: *b, arg1): pass + >>> f2.__annotations__ + {'args': StarredB} + + >>> def f3(*args: *b, arg1: int): pass + >>> f3.__annotations__ # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + {'args': StarredB, 'arg1': <class 'int'>} + + >>> def f4(*args: *b, arg1: int = 2): pass + >>> f4.__annotations__ # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + {'args': StarredB, 'arg1': <class 'int'>} + + >>> def f5(*args: *b = (1,)): pass # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: invalid syntax +""" + +__test__ = {'doctests' : doctests} + +EXPECTED_FAILURE = doctest.register_optionflag('EXPECTED_FAILURE') # TODO: RUSTPYTHON +class CustomOutputChecker(doctest.OutputChecker): # TODO: RUSTPYTHON + def check_output(self, want, got, optionflags): # TODO: RUSTPYTHON + if optionflags & EXPECTED_FAILURE: # TODO: RUSTPYTHON + if want == got: # TODO: RUSTPYTHON + return False # TODO: RUSTPYTHON + return True # TODO: RUSTPYTHON + return super().check_output(want, got, optionflags) # TODO: RUSTPYTHON + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(checker=CustomOutputChecker())) # TODO: RUSTPYTHON + return tests + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 070e277c2a7..a09037034ad 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -1,18 +1,22 @@ from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING, NAME_MAPPING, REVERSE_NAME_MAPPING) import builtins -import pickle -import io import collections +import contextlib +import io +import pickle import struct import sys +import tempfile import warnings import weakref +from textwrap import dedent import doctest import unittest from test import support -from test.support import import_helper +from test.support import cpython_only, import_helper, os_helper +from test.support.import_helper import ensure_lazy_imports from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests @@ -33,6 +37,12 @@ has_c_implementation = False +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("pickle", {"re"}) + + class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): dump = staticmethod(pickle._dump) dumps = staticmethod(pickle._dumps) @@ -41,14 +51,12 @@ class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): Pickler = pickle._Pickler Unpickler = pickle._Unpickler - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dump_load_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump_load_oob_buffers(self): return super().test_dump_load_oob_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dumps_loads_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dumps_loads_oob_buffers(self): return super().test_dumps_loads_oob_buffers() @@ -65,21 +73,6 @@ def loads(self, buf, **kwds): u = self.unpickler(f, **kwds) return u.load() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_badly_escaped_string() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_correctly_quoted_string() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_load_python2_str_as_bytes() - class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): @@ -92,26 +85,22 @@ def dumps(self, arg, proto=None, **kwargs): f.seek(0) return bytes(f.read()) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_picklebuffer_error() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_getattr(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bad_getattr() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bad_newobj_args(self): + return super().test_bad_newobj_args() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffer_callback_error(self): return super().test_buffer_callback_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_continuous_buffer(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_continuous_buffer(self): return super().test_non_continuous_buffer() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_picklebuffer_error(self): + return super().test_picklebuffer_error() + class PyPicklerTests(AbstractPickleTests, unittest.TestCase): @@ -130,101 +119,34 @@ def loads(self, buf, **kwds): u = self.unpickler(f, **kwds) return u.load() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_c_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): return super().test_bytearray_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): return super().test_bytes_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_in_band_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_optional_frames() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_buffers_error() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytearray_memoization() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytes_memoization() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): return super().test_in_band_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): return super().test_oob_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): return super().test_oob_buffers_writable_to_readonly() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_optional_frames() - class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, BigmemPickleTests, unittest.TestCase): @@ -244,75 +166,34 @@ def loads(self, buf, **kwds): test_find_class = None test_custom_find_class = None - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_c_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_badly_escaped_string() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_correctly_quoted_string() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_load_python2_str_as_bytes() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): return super().test_bytearray_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): return super().test_bytes_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): return super().test_in_band_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): return super().test_oob_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_optional_frames() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): + return super().test_oob_buffers_writable_to_readonly() + class PersistentPicklerUnpicklerMixin(object): @@ -536,6 +417,7 @@ def _persistent_load(subself, pid): del unpickler.persistent_load self.assertEqual(unpickler.persistent_load, old_persistent_load) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): pickler_class = pickle._Pickler @@ -611,6 +493,46 @@ def test_issue18339(self): unpickler.memo = {-1: None} unpickler.memo = {1: None} + def test_concurrent_pickler_dump(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.dump, 42) + return list, () + pickler.dump(X()) # should not crash + self.assertEqual(pickle.loads(f.getvalue()), []) + + def test_concurrent_pickler_dump_and_init(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.__init__, f) + return list, () + pickler.dump([X()]) # should not fail + self.assertEqual(pickle.loads(f.getvalue()), [[]]) + + def test_concurrent_unpickler_load(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.load) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not fail + self.assertEqual(unpickled, [42]) + + def test_concurrent_unpickler_load_and_init(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.__init__, f) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not crash + self.assertEqual(unpickled, [42]) + class CDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): pickler_class = pickle.Pickler def get_dispatch_table(self): @@ -659,7 +581,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('7P2n3i2n3i2P') + basesize = support.calcobjsize('7P2n3i2n4i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -676,7 +598,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n3i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -809,10 +731,10 @@ def test_name_mapping(self): with self.subTest(((module3, name3), (module2, name2))): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, OSError)) + self.assertIsSubclass(attr, OSError) elif (module2, name2) == ('exceptions', 'ImportError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, ImportError)) + self.assertIsSubclass(attr, ImportError) else: module, name = mapping(module2, name2) if module3[:1] != '_': @@ -857,8 +779,7 @@ def test_reverse_name_mapping(self): module, name = mapping(module, name) self.assertEqual((module, name), (module3, name3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exceptions(self): self.assertEqual(mapping('exceptions', 'StandardError'), ('builtins', 'Exception')) @@ -908,6 +829,60 @@ def test_multiprocessing_exceptions(self): self.assertEqual(mapping('multiprocessing', name), ('multiprocessing.context', name)) + +class CommandLineTest(unittest.TestCase): + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return dedent(string).strip() + + def set_pickle_data(self, data): + with open(self.filename, 'wb') as f: + pickle.dump(data, f) + + def invoke_pickle(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + pickle._main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def test_invocation(self): + # test 'python -m pickle pickle_file' + data = { + 'a': [1, 2.0, 3+4j], + 'b': ('character string', b'byte string'), + 'c': 'string' + } + expect = ''' + {'a': [1, 2.0, (3+4j)], + 'b': ('character string', b'byte string'), + 'c': 'string'} + ''' + self.set_pickle_data(data) + + with self.subTest(data=data): + res = self.invoke_pickle() + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + @support.force_not_colorized + def test_unknown_flag(self): + stderr = io.StringIO() + with self.assertRaises(SystemExit): + # check that the parser help is shown + with contextlib.redirect_stderr(stderr): + _ = self.invoke_pickle('--unknown') + self.assertStartsWith(stderr.getvalue(), 'usage: ') + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(pickle)) return tests diff --git a/Lib/test/test_picklebuffer.py b/Lib/test/test_picklebuffer.py index a14f6a86b4f..f63be69cfc8 100644 --- a/Lib/test/test_picklebuffer.py +++ b/Lib/test/test_picklebuffer.py @@ -34,8 +34,7 @@ def check_memoryview(self, pb, equiv): self.assertEqual(m.format, expected.format) self.assertEqual(m.tobytes(), expected.tobytes()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_failure(self): with self.assertRaises(TypeError): PickleBuffer() @@ -47,8 +46,7 @@ def test_constructor_failure(self): with self.assertRaises(ValueError): PickleBuffer(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basics(self): pb = PickleBuffer(b"foo") self.assertEqual(b"foo", bytes(pb)) @@ -62,8 +60,7 @@ def test_basics(self): m[0] = 48 self.assertEqual(b"0oo", bytes(pb)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_release(self): pb = PickleBuffer(b"foo") pb.release() @@ -74,8 +71,7 @@ def test_release(self): # Idempotency pb.release() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle(self): b = B(b"foo") pb = PickleBuffer(b) @@ -85,8 +81,7 @@ def test_cycle(self): gc.collect() self.assertIsNone(wpb()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ndarray_2d(self): # C-contiguous ndarray = import_helper.import_module("_testbuffer").ndarray @@ -110,23 +105,20 @@ def test_ndarray_2d(self): # Tests for PickleBuffer.raw() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def check_raw(self, obj, equiv): pb = PickleBuffer(obj) with pb.raw() as m: self.assertIsInstance(m, memoryview) self.check_memoryview(m, equiv) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raw(self): for obj in (b"foo", bytearray(b"foo")): with self.subTest(obj=obj): self.check_raw(obj, obj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raw_ndarray(self): # 1-D, contiguous ndarray = import_helper.import_module("_testbuffer").ndarray @@ -148,15 +140,13 @@ def test_raw_ndarray(self): equiv = b'\xc8\x01\x00\x00' self.check_raw(arr, equiv) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def check_raw_non_contiguous(self, obj): pb = PickleBuffer(obj) with self.assertRaisesRegex(BufferError, "non-contiguous"): pb.raw() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raw_non_contiguous(self): # 1-D ndarray = import_helper.import_module("_testbuffer").ndarray @@ -166,8 +156,7 @@ def test_raw_non_contiguous(self): arr = ndarray(list(range(12)), shape=(4, 3), format='<i')[::2] self.check_raw_non_contiguous(arr) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raw_released(self): pb = PickleBuffer(b"foo") pb.release() diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 6c38bef3d31..1a6256e097a 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -62,61 +62,34 @@ def test_optimize_binput_and_memoize(self): self.assertIs(unpickled2[1], unpickled2[2]) self.assertNotIn(pickle.BINPUT, pickled2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): return super().test_bytearray_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): return super().test_bytes_memoization() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): return super().test_in_band_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): return super().test_oob_buffers() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): return super().test_oob_buffers_writable_to_readonly() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_optional_frames() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_c_methods() - class SimpleReader: def __init__(self, data): @@ -261,7 +234,7 @@ def test_mark_without_pos(self): def test_no_mark(self): self.check_dis_error(b'Nt.', '''\ 0: N NONE - 1: t TUPLE no MARK exists on stack + 1: t TUPLE ''', 'no MARK exists on stack') def test_put(self): @@ -276,26 +249,16 @@ def test_put(self): ''') def test_put_redefined(self): - self.check_dis_error(b'Np1\np1\n.', '''\ + self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\ 0: N NONE 1: p PUT 1 4: p PUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\nq\x01.', '''\ - 0: N NONE - 1: p PUT 1 - 4: q BINPUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\nr\x01\x00\x00\x00.', '''\ - 0: N NONE - 1: p PUT 1 - 4: r LONG_BINPUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\n\x94.', '''\ - 0: N NONE - 1: p PUT 1 - 4: \\x94 MEMOIZE (as 1) -''', 'memo key None already defined') + 7: q BINPUT 1 + 9: r LONG_BINPUT 1 + 14: \\x94 MEMOIZE (as 1) + 15: . STOP +highest protocol among opcodes = 4 +''') def test_put_empty_stack(self): self.check_dis_error(b'p0\n', '''\ @@ -449,13 +412,13 @@ def test_string_without_quotes(self): self.check_dis_error(b'Sabc"\n.', '', "no string quotes around b'abc\"'") self.check_dis_error(b"S'abc\n.", '', - '''strinq quote b"'" not found at both ends of b"'abc"''') + '''string quote b"'" not found at both ends of b"'abc"''') self.check_dis_error(b'S"abc\n.', '', - r"""strinq quote b'"' not found at both ends of b'"abc'""") + r"""string quote b'"' not found at both ends of b'"abc'""") self.check_dis_error(b"S'abc\"\n.", '', - r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + r"""string quote b"'" not found at both ends of b'\\'abc"'""") self.check_dis_error(b"S\"abc'\n.", '', - r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + r"""string quote b'"' not found at both ends of b'"abc\\''""") def test_binstring(self): self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ @@ -508,6 +471,44 @@ def test_persid(self): highest protocol among opcodes = 0 ''') + def test_constants(self): + self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\ + 0: ( MARK + 1: N NONE + 2: I INT False + 6: I INT True + 10: \\x89 NEWFALSE + 11: \\x88 NEWTRUE + 12: t TUPLE (MARK at 0) + 13: . STOP +highest protocol among opcodes = 2 +''') + + def test_integers(self): + self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\ + 0: ( MARK + 1: I INT 0 + 4: I INT 1 + 7: I INT 10 + 11: I INT 11 + 16: L LONG 12 + 20: L LONG 13 + 25: L LONG 14 + 30: L LONG 15 + 36: t TUPLE (MARK at 0) + 37: . STOP +highest protocol among opcodes = 0 +''') + + def test_nondecimal_integers(self): + self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int') + + class MiscTestCase(unittest.TestCase): def test__all__(self): not_exported = { @@ -542,8 +543,8 @@ def test__all__(self): def load_tests(loader, tests, pattern): - # TODO: RUSTPYTHON - # tests.addTest(doctest.DocTestSuite(pickletools)) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON; Remove this + tests.addTest(doctest.DocTestSuite(pickletools, checker=DocTestChecker())) # XXX: RUSTPYTHON return tests diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index ece4cf2d05f..0d44092dabc 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,5 +1,6 @@ +from pathlib import Path from test.support.import_helper import unload, CleanImport -from test.support.warnings_helper import check_warnings +from test.support.warnings_helper import check_warnings, ignore_warnings import unittest import sys import importlib @@ -11,6 +12,10 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -91,6 +96,45 @@ def test_getdata_zipfile(self): del sys.modules[pkg] + def test_issue44061_iter_modules(self): + #see: issue44061 + zip = 'test_getdata_zipfile.zip' + pkg = 'test_getdata_zipfile' + + # Include a LF and a CRLF, to test that binary data is read back + RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line' + + # Make a package with some resources + zip_file = os.path.join(self.dirname, zip) + z = zipfile.ZipFile(zip_file, 'w') + + # Empty init.py + z.writestr(pkg + '/__init__.py', "") + # Resource files, res.txt + z.writestr(pkg + '/res.txt', RESOURCE_DATA) + z.close() + + # Check we can read the resources + sys.path.insert(0, zip_file) + try: + res = pkgutil.get_data(pkg, 'res.txt') + self.assertEqual(res, RESOURCE_DATA) + + # make sure iter_modules accepts Path objects + names = [] + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): + self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) + names.append(moduleinfo.name) + self.assertEqual(names, [pkg]) + finally: + del sys.path[0] + sys.modules.pop(pkg, None) + + # assert path must be None or list of paths + expected_msg = "path must be None or list of paths to look for modules in" + with self.assertRaisesRegex(ValueError, expected_msg): + list(pkgutil.iter_modules("invalid_path")) + def test_unreadable_dir_on_syspath(self): # issue7367 - walk_packages failed if unreadable dir on sys.path package_name = "unreadable_package" @@ -187,8 +231,7 @@ def test_walk_packages_raises_on_string_or_bytes_input(self): with self.assertRaises((TypeError, ValueError)): list(pkgutil.walk_packages(bytes_input)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_name_resolution(self): import logging import logging.handlers @@ -280,6 +323,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): @@ -391,7 +466,7 @@ def test_iter_importers(self): importers = list(iter_importers(fullname)) expected_importer = get_importer(pathitem) for finder in importers: - spec = pkgutil._get_spec(finder, fullname) + spec = finder.find_spec(fullname) loader = spec.loader try: loader = loader.loader @@ -403,7 +478,7 @@ def test_iter_importers(self): self.assertEqual(finder, expected_importer) self.assertIsInstance(loader, importlib.machinery.SourceFileLoader) - self.assertIsNone(pkgutil._get_spec(finder, pkgname)) + self.assertIsNone(finder.find_spec(pkgname)) with self.assertRaises(ImportError): list(iter_importers('invalid.module')) @@ -448,7 +523,43 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] - # XXX: test .pkg files + + def test_extend_path_argument_types(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + # If the input path is not a list it is returned unchanged + self.assertEqual('notalist', pkgutil.extend_path('notalist', 'foo')) + self.assertEqual(('not', 'a', 'list'), pkgutil.extend_path(('not', 'a', 'list'), 'foo')) + self.assertEqual(123, pkgutil.extend_path(123, 'foo')) + self.assertEqual(None, pkgutil.extend_path(None, 'foo')) + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] + + + def test_extend_path_pkg_files(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + with open(os.path.join(dirname_0, 'bar.pkg'), 'w') as pkg_file: + pkg_file.write('\n'.join([ + 'baz', + '/foo/bar/baz', + '', + '#comment' + ])) + + extended_paths = pkgutil.extend_path(sys.path, 'bar') + + self.assertEqual(extended_paths[:-2], sys.path) + self.assertEqual(extended_paths[-2], 'baz') + self.assertEqual(extended_paths[-1], '/foo/bar/baz') + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] class NestedNamespacePackageTest(unittest.TestCase): @@ -491,36 +602,50 @@ def test_nested(self): self.assertEqual(c, 1) self.assertEqual(d, 2) + +class ImportlibMigrationTests(unittest.TestCase): + # With full PEP 302 support in the standard import machinery, the + # PEP 302 emulation in this module is in the process of being + # deprecated in favour of importlib proper + @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_loader_attribute(self): global __loader__ this_loader = __loader__ del __loader__ try: - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - self.assertEqual(len(w.warnings), 0) + self.assertIsNotNone(pkgutil.get_loader(__name__)) finally: __loader__ = this_loader + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_spec_attribute(self): name = 'spam' mod = type(sys)(name) del mod.__spec__ with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) + try: + sys.modules[name] = mod + loader = pkgutil.get_loader(name) + finally: + sys.modules.pop(name, None) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_spec_attribute_none(self): name = 'spam' mod = type(sys)(name) mod.__spec__ = None with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) + try: + sys.modules[name] = mod + loader = pkgutil.get_loader(name) + finally: + sys.modules.pop(name, None) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_None_in_sys_modules(self): name = 'totally bogus' sys.modules[name] = None @@ -530,24 +655,38 @@ def test_get_loader_None_in_sys_modules(self): del sys.modules[name] self.assertIsNone(loader) + def test_get_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), + ): + res = pkgutil.get_loader("sys") + self.assertIsNotNone(res) + + def test_find_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), + ): + res = pkgutil.find_loader("sys") + self.assertIsNotNone(res) + + @ignore_warnings(category=DeprecationWarning) def test_find_loader_missing_module(self): name = 'totally bogus' loader = pkgutil.find_loader(name) self.assertIsNone(loader) - def test_find_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.find_loader("sys")) - self.assertIsNotNone(pkgutil.find_loader("os")) - self.assertIsNotNone(pkgutil.find_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: self.assertIsNone(pkgutil.get_importer("*??")) self.assertEqual(len(w.warnings), 0) + def test_issue44061(self): + try: + pkgutil.get_importer(Path("/home")) + except AttributeError: + self.fail("Unexpected AttributeError when calling get_importer") + def test_iter_importers_avoids_emulation(self): with check_warnings() as w: for importer in pkgutil.iter_importers(): pass diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index c9f27575b51..ed277276b51 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -123,10 +131,6 @@ def test_sys_version(self): for input, output in ( ('2.4.3 (#1, Jun 21 2006, 13:54:21) \n[GCC 3.3.4 (pre 3.3.5 20040809)]', ('CPython', '2.4.3', '', '', '1', 'Jun 21 2006 13:54:21', 'GCC 3.3.4 (pre 3.3.5 20040809)')), - ('IronPython 1.0.60816 on .NET 2.0.50727.42', - ('IronPython', '1.0.60816', '', '', '', '', '.NET 2.0.50727.42')), - ('IronPython 1.0 (1.0.61005.1977) on .NET 2.0.50727.42', - ('IronPython', '1.0.0', '', '', '', '', '.NET 2.0.50727.42')), ('2.4.3 (truncation, date, t) \n[GCC]', ('CPython', '2.4.3', '', '', 'truncation', 'date t', 'GCC')), ('2.4.3 (truncation, date, ) \n[GCC]', @@ -161,20 +165,11 @@ def test_sys_version(self): ('r261:67515', 'Dec 6 2008 15:26:00'), 'GCC 4.0.1 (Apple Computer, Inc. build 5370)'), - ("IronPython 2.0 (2.0.0.0) on .NET 2.0.50727.3053", None, "cli") - : - ("IronPython", "2.0.0", "", "", ("", ""), - ".NET 2.0.50727.3053"), - - ("2.6.1 (IronPython 2.6.1 (2.6.10920.0) on .NET 2.0.50727.1433)", None, "cli") + ("3.10.8 (tags/v3.10.8:aaaf517424, Feb 14 2023, 16:28:12) [GCC 9.4.0]", + None, "linux") : - ("IronPython", "2.6.1", "", "", ("", ""), - ".NET 2.0.50727.1433"), - - ("2.7.4 (IronPython 2.7.4 (2.7.0.40) on Mono 4.0.30319.1 (32-bit))", None, "cli") - : - ("IronPython", "2.7.4", "", "", ("", ""), - "Mono 4.0.30319.1 (32-bit)"), + ('CPython', '3.10.8', '', '', + ('tags/v3.10.8:aaaf517424', 'Feb 14 2023 16:28:12'), 'GCC 9.4.0'), ("2.5 (trunk:6107, Mar 26 2009, 13:02:18) \n[Java HotSpot(TM) Client VM (\"Apple Computer, Inc.\")]", ('Jython', 'trunk', '6107'), "java1.5.0_16") @@ -205,6 +200,9 @@ def test_sys_version(self): self.assertEqual(platform.python_build(), info[4]) self.assertEqual(platform.python_compiler(), info[5]) + with self.assertRaises(ValueError): + platform._sys_version('2. 4.3 (truncation) \n[GCC]') + def test_system_alias(self): res = platform.system_alias( platform.system(), @@ -229,6 +227,38 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + + + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") + def test_uname_win32_without_wmi(self): + def raises_oserror(*a): + raise OSError() + + with support.swap_attr(platform, '_wmi_query', raises_oserror): + self.test_uname() + def test_uname_cast_to_tuple(self): res = platform.uname() expected = ( @@ -297,28 +327,66 @@ def test_uname_win32_ARCHITEW6432(self): # on 64 bit Windows: if PROCESSOR_ARCHITEW6432 exists we should be # using it, per # http://blogs.msdn.com/david.wang/archive/2006/03/26/HOWTO-Detect-Process-Bitness.aspx - try: + + # We also need to suppress WMI checks, as those are reliable and + # overrule the environment variables + def raises_oserror(*a): + raise OSError() + + with support.swap_attr(platform, '_wmi_query', raises_oserror): with os_helper.EnvironmentVarGuard() as environ: - if 'PROCESSOR_ARCHITEW6432' in environ: + try: del environ['PROCESSOR_ARCHITEW6432'] - environ['PROCESSOR_ARCHITECTURE'] = 'foo' - platform._uname_cache = None - system, node, release, version, machine, processor = platform.uname() - self.assertEqual(machine, 'foo') - environ['PROCESSOR_ARCHITEW6432'] = 'bar' - platform._uname_cache = None - system, node, release, version, machine, processor = platform.uname() - self.assertEqual(machine, 'bar') - finally: - platform._uname_cache = None + environ['PROCESSOR_ARCHITECTURE'] = 'foo' + platform._uname_cache = None + system, node, release, version, machine, processor = platform.uname() + self.assertEqual(machine, 'foo') + environ['PROCESSOR_ARCHITEW6432'] = 'bar' + platform._uname_cache = None + system, node, release, version, machine, processor = platform.uname() + self.assertEqual(machine, 'bar') + finally: + platform._uname_cache = None def test_java_ver(self): - res = platform.java_ver() - if sys.platform == 'java': - self.assertTrue(all(res)) + import re + msg = re.escape( + "'java_ver' is deprecated and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, msg): + res = platform.java_ver() + self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() @@ -372,6 +440,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception @@ -421,6 +539,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V @@ -467,7 +622,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index c6d4cfe5c6b..389da145e6d 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -6,10 +6,15 @@ import unittest import plistlib import os +import sys +import json import datetime import codecs +import subprocess import binascii import collections +import time +import zoneinfo from test import support from test.support import os_helper from io import BytesIO @@ -505,6 +510,19 @@ def test_bytes(self): data2 = plistlib.dumps(pl2) self.assertEqual(data, data2) + def test_loads_str_with_xml_fmt(self): + pl = self._create() + b = plistlib.dumps(pl) + s = b.decode() + self.assertIsInstance(s, str) + pl2 = plistlib.loads(s) + self.assertEqual(pl, pl2) + + def test_loads_str_with_binary_fmt(self): + msg = "value must be bytes-like object when fmt is FMT_BINARY" + with self.assertRaisesRegex(TypeError, msg): + plistlib.loads('test', fmt=plistlib.FMT_BINARY) + def test_indentation_array(self): data = [[[[[[[[{'test': b'aaaaaa'}]]]]]]]] self.assertEqual(plistlib.loads(plistlib.dumps(data)), data) @@ -734,8 +752,6 @@ def test_non_bmp_characters(self): data = plistlib.dumps(pl, fmt=fmt) self.assertEqual(plistlib.loads(data), pl) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lone_surrogates(self): for fmt in ALL_FORMATS: with self.subTest(fmt=fmt): @@ -754,8 +770,7 @@ def test_nondictroot(self): self.assertEqual(test1, result1) self.assertEqual(test2, result2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidarray(self): for i in ["<key>key inside an array</key>", "<key>key inside an array2</key><real>3</real>", @@ -763,8 +778,7 @@ def test_invalidarray(self): self.assertRaises(ValueError, plistlib.loads, ("<plist><array>%s</array></plist>"%i).encode()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invaliddict(self): for i in ["<key><true/>k</key><string>compound key</string>", "<key>single key</key>", @@ -776,14 +790,12 @@ def test_invaliddict(self): self.assertRaises(ValueError, plistlib.loads, ("<plist><array><dict>%s</dict></array></plist>"%i).encode()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidinteger(self): self.assertRaises(ValueError, plistlib.loads, b"<plist><integer>not integer</integer></plist>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidreal(self): self.assertRaises(ValueError, plistlib.loads, b"<plist><integer>not real</integer></plist>") @@ -840,18 +852,64 @@ def test_modified_uid_huge(self): with self.assertRaises(OverflowError): plistlib.dumps(huge_uid, fmt=plistlib.FMT_BINARY) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xml_plist_with_entity_decl(self): with self.assertRaisesRegex(plistlib.InvalidFileException, "XML entity declarations are not supported"): plistlib.loads(XML_PLIST_WITH_ENTITY, fmt=plistlib.FMT_XML) + def test_load_aware_datetime(self): + dt = plistlib.loads(b"<plist><date>2023-12-10T08:03:30Z</date></plist>", + aware_datetime=True) + self.assertEqual(dt.tzinfo, datetime.UTC) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + def test_dump_utc_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, tzinfo=datetime.UTC) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_naive_datetime_with_aware_datetime_option(self): + # Save a naive datetime with aware_datetime set to true. This will lead + # to having different time as compared to the current machine's + # timezone, which is UTC. + dt = datetime.datetime(2003, 6, 7, 8, tzinfo=None) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + parsed = plistlib.loads(s, aware_datetime=False) + expected = dt.astimezone(datetime.UTC).replace(tzinfo=None) + self.assertEqual(parsed, expected) + class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -863,7 +921,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -917,12 +979,12 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - for N in [300, 100000]: + for N in [50, 300, 100_000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) except RecursionError: - pass + self.assertGreater(N, sys.getrecursionlimit()) else: for i in range(N): self.assertIsInstance(result, list) @@ -934,7 +996,7 @@ def test_large_timestamp(self): # Issue #26709: 32-bit timestamp out of range for ts in -2**31-1, 2**31: with self.subTest(ts=ts): - d = (datetime.datetime.utcfromtimestamp(0) + + d = (datetime.datetime(1970, 1, 1, 0, 0) + datetime.timedelta(seconds=ts)) data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY) self.assertEqual(plistlib.loads(data), d) @@ -971,6 +1033,60 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + + def test_load_aware_datetime(self): + data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' + b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11') + self.assertEqual(plistlib.loads(data, aware_datetime=True), + datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC)) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + + # TODO: RUSTPYTHON + # The error message is different + # In CPython, there is a separate .c file for datetime, which raises a different error message + @unittest.expectedFailure + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + class TestKeyedArchive(unittest.TestCase): def test_keyed_archive_data(self): @@ -1009,6 +1125,78 @@ def test__all__(self): not_exported = {"PlistFormat", "PLISTHEADER"} support.check__all__(self, plistlib, not_exported=not_exported) +@unittest.skipUnless(sys.platform == "darwin", "plutil utility is for Mac os") +class TestPlutil(unittest.TestCase): + file_name = "plutil_test.plist" + properties = { + "fname" : "H", + "lname":"A", + "marks" : {"a":100, "b":0x10} + } + exptected_properties = { + "fname" : "H", + "lname": "A", + "marks" : {"a":100, "b":16} + } + pl = { + "HexType" : 0x0100000c, + "IntType" : 0o123 + } + + @classmethod + def setUpClass(cls) -> None: + ## Generate plist file with plistlib and parse with plutil + with open(cls.file_name,'wb') as f: + plistlib.dump(cls.properties, f, fmt=plistlib.FMT_BINARY) + + @classmethod + def tearDownClass(cls) -> None: + os.remove(cls.file_name) + + def get_lint_status(self): + return subprocess.run(['plutil', "-lint", self.file_name], capture_output=True, text=True).stdout + + def convert_to_json(self): + """Convert binary file to json using plutil + """ + subprocess.run(['plutil', "-convert", 'json', self.file_name]) + + def convert_to_bin(self): + """Convert file to binary using plutil + """ + subprocess.run(['plutil', "-convert", 'binary1', self.file_name]) + + def write_pl(self): + """Write Hex properties to file using writePlist + """ + with open(self.file_name, 'wb') as f: + plistlib.dump(self.pl, f, fmt=plistlib.FMT_BINARY) + + def test_lint_status(self): + # check lint status of file using plutil + self.assertEqual(f"{self.file_name}: OK\n", self.get_lint_status()) + + def check_content(self): + # check file content with plutil converting binary to json + self.convert_to_json() + with open(self.file_name) as f: + ff = json.loads(f.read()) + self.assertEqual(ff, self.exptected_properties) + + def check_plistlib_parse(self): + # Generate plist files with plutil and parse with plistlib + self.convert_to_bin() + with open(self.file_name, 'rb') as f: + self.assertEqual(plistlib.load(f), self.exptected_properties) + + def test_octal_and_hex(self): + self.write_pl() + self.convert_to_json() + with open(self.file_name, 'r') as f: + p = json.loads(f.read()) + self.assertEqual(p.get("HexType"), 16777228) + self.assertEqual(p.get("IntType"), 83) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_popen.py b/Lib/test/test_popen.py index e6bfc480cbd..34cda35b17b 100644 --- a/Lib/test/test_popen.py +++ b/Lib/test/test_popen.py @@ -57,14 +57,21 @@ def test_return_code(self): def test_contextmanager(self): with os.popen("echo hello") as f: self.assertEqual(f.read(), "hello\n") + self.assertFalse(f.closed) + self.assertTrue(f.closed) def test_iterating(self): with os.popen("echo hello") as f: self.assertEqual(list(f), ["hello\n"]) + self.assertFalse(f.closed) + self.assertTrue(f.closed) def test_keywords(self): - with os.popen(cmd="exit 0", mode="w", buffering=-1): - pass + with os.popen(cmd="echo hello", mode="r", buffering=-1) as f: + self.assertEqual(f.read(), "hello\n") + self.assertFalse(f.closed) + self.assertTrue(f.closed) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index e0d784325ba..e002babab44 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -22,13 +23,13 @@ def assertRaisesSyntaxError(self, codestr, regex="invalid syntax"): with self.assertRaisesRegex(SyntaxError, regex): compile(codestr + "\n", "<test>", "single") - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_errors(self): - check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a, /, b = 5, c): pass", "parameter without a default follows parameter with a default") check_syntax_error(self, "def f(*args, /): pass") check_syntax_error(self, "def f(*args, a, /): pass") check_syntax_error(self, "def f(**kwargs, /): pass") @@ -45,13 +46,13 @@ def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(a, /, c, /, d, *, e): pass") check_syntax_error(self, "def f(a, *, c, /, d, e): pass") - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_errors_async(self): - check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a, /, b = 5, c): pass", "parameter without a default follows parameter with a default") check_syntax_error(self, "async def f(*args, /): pass") check_syntax_error(self, "async def f(*args, a, /): pass") check_syntax_error(self, "async def f(**kwargs, /): pass") @@ -152,8 +153,6 @@ def f(a, b, /, c): with self.assertRaisesRegex(TypeError, r"f\(\) takes 3 positional arguments but 4 were given"): f(1, 2, 3, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_and_optional_arg_invalid_calls(self): def f(a, b, /, c=3): pass @@ -165,8 +164,6 @@ def f(a, b, /, c=3): with self.assertRaisesRegex(TypeError, r"f\(\) takes from 2 to 3 positional arguments but 4 were given"): f(1, 2, 3, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_and_kwonlyargs_invalid_calls(self): def f(a, b, /, c, *, d, e): pass @@ -198,8 +195,6 @@ def f(a, b, /): with self.assertRaisesRegex(TypeError, r"f\(\) takes 2 positional arguments but 3 were given"): f(1, 2, 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_with_optional_invalid_calls(self): def f(a, b=2, /): pass @@ -240,12 +235,13 @@ def test_lambdas(self): x = lambda a, b, /, : a + b self.assertEqual(x(1, 2), 3) - # TODO: RUSTPYTHON: wrong error message - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_lambda(self): - check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument") + check_syntax_error(self, "lambda a, b = 5, /, c: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b, /, c: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b=1, /, c, *, d=2: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b, /: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a, /, b = 5, c: None", "parameter without a default follows parameter with a default") check_syntax_error(self, "lambda *args, /: None") check_syntax_error(self, "lambda *args, a, /: None") check_syntax_error(self, "lambda **kwargs, /: None") @@ -342,8 +338,6 @@ def f(something,/,**kwargs): self.assertEqual(f(42), (42, {})) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mangling(self): class X: def f(self, __a=42, /): @@ -443,8 +437,7 @@ def method(self, /): self.assertEqual(C().method(), sentinel) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_annotations_constant_fold(self): def g(): def f(x: not (int is int), /): ... @@ -452,7 +445,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a4809a23798..8b3cbc2f093 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -583,7 +583,6 @@ def test_confstr(self): self.assertGreater(len(path), 0) self.assertEqual(posix.confstr(posix.confstr_names["CS_PATH"]), path) - @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), '''TODO: RUSTPYTHON; AssertionError: "configuration names must be strings or integers" does not match "Expected type 'str' but 'float' found."''') @unittest.skipUnless(hasattr(posix, 'sysconf'), 'test needs posix.sysconf()') def test_sysconf(self): @@ -1018,7 +1017,7 @@ def test_chmod_dir(self): target = self.tempdir() self.check_chmod(posix.chmod, target) - @unittest.skipIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; crash') + @unittest.skipIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; crash") @os_helper.skip_unless_working_chmod def test_fchmod_file(self): with open(os_helper.TESTFN, 'wb+') as f: @@ -1075,7 +1074,6 @@ def test_chmod_file_symlink(self): self.check_chmod_link(posix.chmod, target, link) self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; flaky') @os_helper.skip_unless_symlink def test_chmod_dir_symlink(self): target = self.tempdir() @@ -1110,7 +1108,7 @@ def test_lchmod_dir_symlink(self): def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) - self.assertTrue(hasattr(st, 'st_flags')) + self.assertHasAttr(st, 'st_flags') # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = st.st_flags | stat.UF_IMMUTABLE @@ -1146,7 +1144,7 @@ def test_lchflags_regular_file(self): def test_lchflags_symlink(self): testfn_st = os.stat(os_helper.TESTFN) - self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.assertHasAttr(testfn_st, 'st_flags') self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) @@ -1350,7 +1348,6 @@ def test_get_and_set_scheduler_and_param(self): param = posix.sched_param(sched_priority=-large) self.assertRaises(OverflowError, posix.sched_setparam, 0, param) - @unittest.expectedFailureIf(sys.platform == 'linux', "TODO: RUSTPYTHON; TypeError: cannot pickle 'sched_param' object") @requires_sched def test_sched_param(self): param = posix.sched_param(1) @@ -1370,6 +1367,14 @@ def test_sched_param(self): self.assertNotEqual(newparam, param) self.assertEqual(newparam.sched_priority, 0) + @requires_sched + def test_bug_140634(self): + sched_priority = float('inf') # any new reference + param = posix.sched_param(sched_priority) + param.__reduce__() + del sched_priority, param # should not crash + support.gc_collect() # just to be sure + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1525,6 +1530,51 @@ def test_pidfd_open(self): self.assertEqual(cm.exception.errno, errno.EINVAL) os.close(os.pidfd_open(os.getpid(), 0)) + @os_helper.skip_unless_hardlink + @os_helper.skip_unless_symlink + def test_link_follow_symlinks(self): + default_follow = sys.platform.startswith( + ('darwin', 'freebsd', 'netbsd', 'openbsd', 'dragonfly', 'sunos5')) + default_no_follow = sys.platform.startswith(('win32', 'linux')) + orig = os_helper.TESTFN + symlink = orig + 'symlink' + posix.symlink(orig, symlink) + self.addCleanup(os_helper.unlink, symlink) + + with self.subTest('no follow_symlinks'): + # no follow_symlinks -> platform depending + link = orig + 'link' + posix.link(symlink, link) + self.addCleanup(os_helper.unlink, link) + if os.link in os.supports_follow_symlinks or default_follow: + self.assertEqual(posix.lstat(link), posix.lstat(orig)) + elif default_no_follow: + self.assertEqual(posix.lstat(link), posix.lstat(symlink)) + + with self.subTest('follow_symlinks=False'): + # follow_symlinks=False -> duplicate the symlink itself + link = orig + 'link_nofollow' + try: + posix.link(symlink, link, follow_symlinks=False) + except NotImplementedError: + if os.link in os.supports_follow_symlinks or default_no_follow: + raise + else: + self.addCleanup(os_helper.unlink, link) + self.assertEqual(posix.lstat(link), posix.lstat(symlink)) + + with self.subTest('follow_symlinks=True'): + # follow_symlinks=True -> duplicate the target file + link = orig + 'link_following' + try: + posix.link(symlink, link, follow_symlinks=True) + except NotImplementedError: + if os.link in os.supports_follow_symlinks or default_follow: + raise + else: + self.addCleanup(os_helper.unlink, link) + self.assertEqual(posix.lstat(link), posix.lstat(orig)) + # tests for the posix *at functions follow class TestPosixDirFd(unittest.TestCase): @@ -1570,7 +1620,6 @@ def test_chown_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) - @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered') @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") def test_stat_dir_fd(self): with self.prepare() as (dir_fd, name, fullname): @@ -1973,7 +2022,7 @@ def test_setsigdef_wrong_type(self): [sys.executable, "-c", "pass"], os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) - @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented') + @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @@ -1994,14 +2043,15 @@ def test_setscheduler_only_param(self): ) support.wait_process(pid, exitcode=0) - @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented') + @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @unittest.skipIf(platform.libc_ver()[0] == 'glibc' and os.sched_getscheduler(0) in [ os.SCHED_BATCH, - os.SCHED_IDLE], + os.SCHED_IDLE, + os.SCHED_DEADLINE], "Skip test due to glibc posix_spawn policy") def test_setscheduler_with_policy(self): policy = os.sched_getscheduler(0) @@ -2081,7 +2131,7 @@ def test_open_file(self): with open(outfile, encoding="utf-8") as f: self.assertEqual(f.read(), 'hello') - @unittest.expectedFailure # TODO: RUSTPYTHON; the rust runtime reopens closed stdio fds at startup, so this test fails, even though POSIX_SPAWN_CLOSE does actually have an effect + @unittest.expectedFailure # TODO: RUSTPYTHON; the rust runtime reopens closed stdio fds at startup, so this test fails, even though POSIX_SPAWN_CLOSE does actually have an effect def test_close_file(self): closefile = os_helper.TESTFN self.addCleanup(os_helper.unlink, closefile) @@ -2186,12 +2236,12 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertHasAttr(os, "pwritev") + self.assertHasAttr(os, "preadv") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertNotHasAttr(os, "pwritev") + self.assertNotHasAttr(os, "preadv") def test_stat(self): self._verify_available("HAVE_FSTATAT") diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 0dc0211eada..21f06712548 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1084,15 +1084,11 @@ def check_error(exc, paths): ['usr/lib/', b'/usr/lib/python3']) -# TODO: RUSTPYTHON -@unittest.skip("TODO: RUSTPYTHON, flaky tests") class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = posixpath attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat'] -# TODO: RUSTPYTHON -@unittest.skipIf(os.getenv("CI"), "TODO: RUSTPYTHON, FileExistsError: (17, 'File exists (os error 17)')") class PathLikeTests(unittest.TestCase): path = posixpath diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index ace75561f25..403d2e90084 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -8,10 +8,12 @@ import pprint import random import re -import test.support import types import unittest +from test.support import cpython_only +from test.support.import_helper import ensure_lazy_imports + # list, tuple and dict subclasses that do or don't overwrite __repr__ class list2(list): pass @@ -130,6 +132,10 @@ def setUp(self): self.b = list(range(200)) self.a[-12] = self.b + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("pprint", {"dataclasses", "re"}) + def test_init(self): pp = pprint.PrettyPrinter() pp = pprint.PrettyPrinter(indent=4, width=40, depth=5, diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 6107b7032f9..7c71c5eefe2 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -146,7 +146,7 @@ class TestPy2MigrationHint(unittest.TestCase): if print statement is executed as in Python 2. """ - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_normal_string(self): python2_print_str = 'print "Hello World"' with self.assertRaises(SyntaxError) as context: @@ -155,7 +155,7 @@ def test_normal_string(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_soft_space(self): python2_print_str = 'print "Hello World",' with self.assertRaises(SyntaxError) as context: @@ -164,7 +164,7 @@ def test_string_with_soft_space(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_excessive_whitespace(self): python2_print_str = 'print "Hello World", ' with self.assertRaises(SyntaxError) as context: @@ -173,7 +173,7 @@ def test_string_with_excessive_whitespace(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_leading_whitespace(self): python2_print_str = '''if 1: print "Hello World" @@ -187,7 +187,7 @@ def test_string_with_leading_whitespace(self): # bpo-32685: Suggestions for print statement should be proper when # it is in the same line as the header of a compound statement # and/or followed by a semicolon - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_semicolon(self): python2_print_str = 'print p;' with self.assertRaises(SyntaxError) as context: @@ -196,7 +196,7 @@ def test_string_with_semicolon(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_in_loop_on_same_line(self): python2_print_str = 'for i in s: print i' with self.assertRaises(SyntaxError) as context: @@ -205,39 +205,6 @@ def test_string_in_loop_on_same_line(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_stream_redirection_hint_for_py2_migration(self): - # Test correct hint produced for Py2 redirection syntax - with self.assertRaises(TypeError) as context: - print >> sys.stderr, "message" - self.assertIn('Did you mean "print(<message>, ' - 'file=<output_stream>)"?', str(context.exception)) - - # Test correct hint is produced in the case where RHS implements - # __rrshift__ but returns NotImplemented - with self.assertRaises(TypeError) as context: - print >> 42 - self.assertIn('Did you mean "print(<message>, ' - 'file=<output_stream>)"?', str(context.exception)) - - # Test stream redirection hint is specific to print - with self.assertRaises(TypeError) as context: - max >> sys.stderr - self.assertNotIn('Did you mean ', str(context.exception)) - - # Test stream redirection hint is specific to rshift - with self.assertRaises(TypeError) as context: - print << sys.stderr - self.assertNotIn('Did you mean', str(context.exception)) - - # Ensure right operand implementing rrshift still works - class OverrideRRShift: - def __rrshift__(self, lhs): - return 42 # Force result independent of LHS - - self.assertEqual(print >> OverrideRRShift(), 42) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index cea241b0f20..26aefdbf042 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -87,8 +87,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index b1c1f6abffb..6d3c47195c6 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,10 +1,16 @@ -from test import support -from test.support import verbose, reap_children +import unittest +from test.support import ( + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") + import errno import os import pty @@ -14,21 +20,12 @@ import signal import socket import io # readline -import unittest - -import struct -import fcntl import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" TEST_STRING_2 = b"For my pet fish, Eric.\n" -try: - _TIOCGWINSZ = tty.TIOCGWINSZ - _TIOCSWINSZ = tty.TIOCSWINSZ - _HAVE_WINSZ = True -except AttributeError: - _HAVE_WINSZ = False +_HAVE_WINSZ = hasattr(tty, "TIOCGWINSZ") and hasattr(tty, "TIOCSWINSZ") if verbose: def debug(msg): @@ -82,90 +79,78 @@ def expectedFailureIfStdinIsTTY(fun): pass return fun -def _get_term_winsz(fd): - s = struct.pack("HHHH", 0, 0, 0, 0) - return fcntl.ioctl(fd, _TIOCGWINSZ, s) -def _set_term_winsz(fd, winsz): - fcntl.ioctl(fd, _TIOCSWINSZ, winsz) +def write_all(fd, data): + written = os.write(fd, data) + if written != len(data): + # gh-73256, gh-110673: It should never happen, but check just in case + raise Exception(f"short write: os.write({fd}, {len(data)} bytes) " + f"wrote {written} bytes") # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. class PtyTest(unittest.TestCase): def setUp(self): - old_alarm = signal.signal(signal.SIGALRM, self.handle_sig) - self.addCleanup(signal.signal, signal.SIGALRM, old_alarm) - old_sighup = signal.signal(signal.SIGHUP, self.handle_sighup) self.addCleanup(signal.signal, signal.SIGHUP, old_sighup) - # isatty() and close() can hang on some platforms. Set an alarm - # before running the test to make sure we don't hang forever. - self.addCleanup(signal.alarm, 0) - signal.alarm(10) - - # Save original stdin window size - self.stdin_rows = None - self.stdin_cols = None + # Save original stdin window size. + self.stdin_dim = None if _HAVE_WINSZ: try: - stdin_dim = os.get_terminal_size(pty.STDIN_FILENO) - self.stdin_rows = stdin_dim.lines - self.stdin_cols = stdin_dim.columns - old_stdin_winsz = struct.pack("HHHH", self.stdin_rows, - self.stdin_cols, 0, 0) - self.addCleanup(_set_term_winsz, pty.STDIN_FILENO, old_stdin_winsz) - except OSError: + self.stdin_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.addCleanup(tty.tcsetwinsize, pty.STDIN_FILENO, + self.stdin_dim) + except tty.error: pass - def handle_sig(self, sig, frame): - self.fail("isatty hung") - @staticmethod def handle_sighup(signum, frame): pass @expectedFailureIfStdinIsTTY + @unittest.skip('TODO: RUSTPYTHON; "Not runnable. tty.tcgetwinsize" is required to setUp') def test_openpty(self): try: mode = tty.tcgetattr(pty.STDIN_FILENO) except tty.error: - # not a tty or bad/closed fd + # Not a tty or bad/closed fd. debug("tty.tcgetattr(pty.STDIN_FILENO) failed") mode = None - new_stdin_winsz = None - if self.stdin_rows is not None and self.stdin_cols is not None: + new_dim = None + if self.stdin_dim: try: # Modify pty.STDIN_FILENO window size; we need to # check if pty.openpty() is able to set pty slave # window size accordingly. - debug("Setting pty.STDIN_FILENO window size") - debug(f"original size: (rows={self.stdin_rows}, cols={self.stdin_cols})") - target_stdin_rows = self.stdin_rows + 1 - target_stdin_cols = self.stdin_cols + 1 - debug(f"target size: (rows={target_stdin_rows}, cols={target_stdin_cols})") - target_stdin_winsz = struct.pack("HHHH", target_stdin_rows, - target_stdin_cols, 0, 0) - _set_term_winsz(pty.STDIN_FILENO, target_stdin_winsz) + debug("Setting pty.STDIN_FILENO window size.") + debug(f"original size: (row, col) = {self.stdin_dim}") + target_dim = (self.stdin_dim[0] + 1, self.stdin_dim[1] + 1) + debug(f"target size: (row, col) = {target_dim}") + tty.tcsetwinsize(pty.STDIN_FILENO, target_dim) # Were we able to set the window size # of pty.STDIN_FILENO successfully? - new_stdin_winsz = _get_term_winsz(pty.STDIN_FILENO) - self.assertEqual(new_stdin_winsz, target_stdin_winsz, + new_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.assertEqual(new_dim, target_dim, "pty.STDIN_FILENO window size unchanged") - except OSError: - warnings.warn("Failed to set pty.STDIN_FILENO window size") + except OSError as e: + logging.getLogger(__name__).warning( + "Failed to set pty.STDIN_FILENO window size.", exc_info=e, + ) pass try: debug("Calling pty.openpty()") try: - master_fd, slave_fd = pty.openpty(mode, new_stdin_winsz) + master_fd, slave_fd, slave_name = pty.openpty(mode, new_dim, + True) except TypeError: master_fd, slave_fd = pty.openpty() - debug(f"Got master_fd '{master_fd}', slave_fd '{slave_fd}'") + slave_name = None + debug(f"Got {master_fd=}, {slave_fd=}, {slave_name=}") except OSError: # " An optional feature could not be imported " ... ? raise unittest.SkipTest("Pseudo-terminals (seemingly) not functional.") @@ -181,8 +166,8 @@ def test_openpty(self): if mode: self.assertEqual(tty.tcgetattr(slave_fd), mode, "openpty() failed to set slave termios") - if new_stdin_winsz: - self.assertEqual(_get_term_winsz(slave_fd), new_stdin_winsz, + if new_dim: + self.assertEqual(tty.tcgetwinsize(slave_fd), new_dim, "openpty() failed to set slave window size") # Ensure the fd is non-blocking in case there's nothing to read. @@ -200,18 +185,18 @@ def test_openpty(self): os.set_blocking(master_fd, blocking) debug("Writing to slave_fd") - os.write(slave_fd, TEST_STRING_1) + write_all(slave_fd, TEST_STRING_1) s1 = _readline(master_fd) self.assertEqual(b'I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") - os.write(slave_fd, TEST_STRING_2[:5]) - os.write(slave_fd, TEST_STRING_2[5:]) + write_all(slave_fd, TEST_STRING_2[:5]) + write_all(slave_fd, TEST_STRING_2[5:]) s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) - @support.requires_fork() + @unittest.skip('TODO: RUSTPYTHON; "Not runnable. tty.tcgetwinsize" is required to setUp') def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() @@ -294,6 +279,7 @@ def test_fork(self): ##else: ## raise TestFailed("Read from master_fd did not raise exception") + @unittest.skip('TODO: RUSTPYTHON; AttributeError: module "tty" has no attribute "tcgetwinsize"') def test_master_read(self): # XXX(nnorwitz): this test leaks fds when there is an error. debug("Calling pty.openpty()") @@ -313,8 +299,28 @@ def test_master_read(self): self.assertEqual(data, b"") + @unittest.skip('TODO: RUSTPYTHON; AttributeError: module "tty" has no attribute "tcgetwinsize"') def test_spawn_doesnt_hang(self): - pty.spawn([sys.executable, '-c', 'print("hi there")']) + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f: + STDOUT_FILENO = 1 + dup_stdout = os.dup(STDOUT_FILENO) + os.dup2(f.fileno(), STDOUT_FILENO) + buf = b'' + def master_read(fd): + nonlocal buf + data = os.read(fd, 1024) + buf += data + return data + try: + pty.spawn([sys.executable, '-c', 'print("hi there")'], + master_read) + finally: + os.dup2(dup_stdout, STDOUT_FILENO) + os.close(dup_stdout) + self.assertEqual(buf, b'hi there\r\n') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'hi there\r\n') class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" @@ -332,8 +338,8 @@ def setUp(self): self.orig_pty_waitpid = pty.waitpid self.fds = [] # A list of file descriptors to close. self.files = [] - self.select_rfds_lengths = [] - self.select_rfds_results = [] + self.select_input = [] + self.select_output = [] self.tcsetattr_mode_setting = None def tearDown(self): @@ -368,11 +374,10 @@ def _socketpair(self): self.files.extend(socketpair) return socketpair - def _mock_select(self, rfds, wfds, xfds, timeout=0): + def _mock_select(self, rfds, wfds, xfds): # This will raise IndexError when no more expected calls exist. - # This ignores the timeout - self.assertEqual(self.select_rfds_lengths.pop(0), len(rfds)) - return self.select_rfds_results.pop(0), [], [] + self.assertEqual((rfds, wfds, xfds), self.select_input.pop(0)) + return self.select_output.pop(0) def _make_mock_fork(self, pid): def mock_fork(): @@ -392,14 +397,16 @@ def test__copy_to_each(self): masters = [s.fileno() for s in socketpair] # Feed data. Smaller than PIPEBUF. These writes will not block. - os.write(masters[1], b'from master') - os.write(write_to_stdin_fd, b'from stdin') + write_all(masters[1], b'from master') + write_all(write_to_stdin_fd, b'from stdin') - # Expect two select calls, the last one will cause IndexError + # Expect three select calls, the last one will cause IndexError pty.select = self._mock_select - self.select_rfds_lengths.append(2) - self.select_rfds_results.append([mock_stdin_fd, masters[0]]) - self.select_rfds_lengths.append(2) + self.select_input.append(([mock_stdin_fd, masters[0]], [], [])) + self.select_output.append(([mock_stdin_fd, masters[0]], [], [])) + self.select_input.append(([mock_stdin_fd, masters[0]], [mock_stdout_fd, masters[0]], [])) + self.select_output.append(([], [mock_stdout_fd, masters[0]], [])) + self.select_input.append(([mock_stdin_fd, masters[0]], [], [])) with self.assertRaises(IndexError): pty._copy(masters[0]) @@ -410,28 +417,6 @@ def test__copy_to_each(self): self.assertEqual(os.read(read_from_stdout_fd, 20), b'from master') self.assertEqual(os.read(masters[1], 20), b'from stdin') - def test__copy_eof_on_all(self): - """Test the empty read EOF case on both master_fd and stdin.""" - read_from_stdout_fd, mock_stdout_fd = self._pipe() - pty.STDOUT_FILENO = mock_stdout_fd - mock_stdin_fd, write_to_stdin_fd = self._pipe() - pty.STDIN_FILENO = mock_stdin_fd - socketpair = self._socketpair() - masters = [s.fileno() for s in socketpair] - - socketpair[1].close() - os.close(write_to_stdin_fd) - - pty.select = self._mock_select - self.select_rfds_lengths.append(2) - self.select_rfds_results.append([mock_stdin_fd, masters[0]]) - # We expect that both fds were removed from the fds list as they - # both encountered an EOF before the second select call. - self.select_rfds_lengths.append(0) - - # We expect the function to return without error. - self.assertEqual(pty._copy(masters[0]), None) - def test__restore_tty_mode_normal_return(self): """Test that spawn resets the tty mode no when _copy returns normally.""" diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 54302eba4df..54786505d00 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -109,18 +109,17 @@ def test_cwd(self): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - import platform - @unittest.expectedFailureIf(sys.platform == "darwin" and int(platform.release().split(".")[0]) < 20, "TODO: RUSTPYTHON") + @unittest.expectedFailureIf(sys.platform == "darwin" and int(__import__("platform").release().split(".")[0]) < 20, "TODO: RUSTPYTHON") def test_relative_path(self): py_compile.compile(os.path.relpath(self.source_path), os.path.relpath(self.pyc_path)) self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'non-root user required') + @os_helper.skip_if_dac_override @unittest.skipIf(os.name == 'nt', 'cannot control directory permissions on Windows') + @os_helper.skip_unless_working_chmod def test_exceptions_propagate(self): # Make sure that exceptions raised thanks to issues with writing # bytecode. @@ -133,10 +132,11 @@ def test_exceptions_propagate(self): finally: os.chmod(self.directory, mode.st_mode) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_coding(self): - bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') + bad_coding = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'bad_coding2.py') with support.captured_stderr(): self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) self.assertFalse(os.path.exists( @@ -198,6 +198,18 @@ def test_invalidation_mode(self): fp.read(), 'test', {}) self.assertEqual(flags, 0b1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_quiet(self): + bad_coding = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'bad_coding2.py') + with support.captured_stderr() as stderr: + self.assertIsNone(py_compile.compile(bad_coding, doraise=False, quiet=2)) + self.assertIsNone(py_compile.compile(bad_coding, doraise=True, quiet=2)) + self.assertEqual(stderr.getvalue(), '') + with self.assertRaises(py_compile.PyCompileError): + py_compile.compile(bad_coding, doraise=True, quiet=1) + class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, unittest.TestCase, @@ -218,27 +230,31 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') def tearDown(self): os_helper.rmtree(self.directory) + @support.requires_subprocess() def pycompilecmd(self, *args, **kwargs): # assert_python_* helpers don't return proc object. We'll just use # subprocess.run() instead of spawn_python() and its friends to test # stdin support of the CLI. + opts = '-m' if __debug__ else '-Om' if args and args[0] == '-' and 'input' in kwargs: - return subprocess.run([sys.executable, '-m', 'py_compile', '-'], + return subprocess.run([sys.executable, opts, 'py_compile', '-'], input=kwargs['input'].encode(), capture_output=True) - return script_helper.assert_python_ok('-m', 'py_compile', *args, **kwargs) + return script_helper.assert_python_ok(opts, 'py_compile', *args, **kwargs) def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') @@ -253,14 +269,18 @@ def test_with_files(self): self.assertTrue(os.path.exists(self.cache_path)) def test_bad_syntax(self): - bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py') + bad_syntax = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'badsyntax_3131.py') rc, stdout, stderr = self.pycompilecmd_failure(bad_syntax) self.assertEqual(rc, 1) self.assertEqual(stdout, b'') self.assertIn(b'SyntaxError', stderr) def test_bad_syntax_with_quiet(self): - bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py') + bad_syntax = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'badsyntax_3131.py') rc, stdout, stderr = self.pycompilecmd_failure('-q', bad_syntax) self.assertEqual(rc, 1) self.assertEqual(stdout, b'') diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index ad26fe1dba1..9e7a67ebee5 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -3,16 +3,17 @@ Nick Mathewson ''' +import importlib.machinery import sys +from contextlib import contextmanager from textwrap import dedent from types import FunctionType, MethodType, BuiltinFunctionType import pyclbr from unittest import TestCase, main as unittest_main from test.test_importlib import util as test_importlib_util import warnings -from test.support.testcase import ExtraAssertions -import unittest # TODO: RUSTPYTHON +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests StaticMethodType = type(staticmethod(lambda: None)) @@ -25,7 +26,30 @@ # is imperfect (as designed), testModule is called with a set of # members to ignore. -class PyclbrTest(TestCase, ExtraAssertions): + +@contextmanager +def temporary_main_spec(): + """ + A context manager that temporarily sets the `__spec__` attribute + of the `__main__` module if it's missing. + """ + main_mod = sys.modules.get("__main__") + if main_mod is None: + yield # Do nothing if __main__ is not present + return + + original_spec = getattr(main_mod, "__spec__", None) + if original_spec is None: + main_mod.__spec__ = importlib.machinery.ModuleSpec( + name="__main__", loader=None, origin="built-in" + ) + try: + yield + finally: + main_mod.__spec__ = original_spec + + +class PyclbrTest(TestCase): def assertListEq(self, l1, l2, ignore): ''' succeed iff {l1} - {ignore} == {l2} - {ignore} ''' @@ -81,7 +105,7 @@ def ismethod(oclass, obj, name): for name, value in dict.items(): if name in ignore: continue - self.assertHasAttr(module, name, ignore) + self.assertHasAttr(module, name) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) @@ -105,6 +129,8 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) @@ -142,24 +168,20 @@ def defined_in(item, module): if defined_in(item, module): self.assertHaskey(dict, name, ignore) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_easy(self): self.checkModule('pyclbr') # XXX: Metaclasses are not supported # self.checkModule('ast') - self.checkModule('doctest', ignore=("TestResults", "_SpoofOut", - "DocTestCase", '_DocTestSuite')) + with temporary_main_spec(): + self.checkModule('doctest', ignore=("TestResults", "_SpoofOut", + "DocTestCase", '_DocTestSuite')) self.checkModule('difflib', ignore=("Match",)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cases(self): # see test.pyclbr_input for the rationale behind the ignored symbols self.checkModule('test.pyclbr_input', ignore=['om', 'f']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nested(self): mb = pyclbr # Set arguments for descriptor creation and _creat_tree call. @@ -221,8 +243,7 @@ def compare(parent1, children1, parent2, children2): compare(None, actual, None, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_others(self): cm = self.checkModule @@ -232,12 +253,14 @@ def test_others(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property - cm( - 'pdb', - # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), - ) - cm('pydoc', ignore=('input', 'output',)) # properties + with temporary_main_spec(): + cm( + 'pdb', + # pyclbr does not handle elegantly `typing` or properties + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget', 'curframe_locals', + '_InteractState', 'rlcompleter'), + ) + cm('pydoc', ignore=('input', 'output',)) # properties # Tests for modules inside packages cm('email.parser') diff --git a/Lib/test/test_pydoc/__init__.py b/Lib/test/test_pydoc/__init__.py new file mode 100644 index 00000000000..f2a39a3fe29 --- /dev/null +++ b/Lib/test/test_pydoc/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pydoc/module_none.py b/Lib/test/test_pydoc/module_none.py new file mode 100644 index 00000000000..ebb50fc86e2 --- /dev/null +++ b/Lib/test/test_pydoc/module_none.py @@ -0,0 +1,8 @@ +def func(): + pass +func.__module__ = None + +class A: + def method(self): + pass + method.__module__ = None diff --git a/Lib/test/test_pydoc/pydoc_mod.py b/Lib/test/test_pydoc/pydoc_mod.py new file mode 100644 index 00000000000..80c287fb10c --- /dev/null +++ b/Lib/test/test_pydoc/pydoc_mod.py @@ -0,0 +1,51 @@ +"""This is a test module for test_pydoc""" + +from __future__ import print_function + +import types +import typing + +__author__ = "Benjamin Peterson" +__credits__ = "Nobody" +__version__ = "1.2.3.4" +__xyz__ = "X, Y and Z" + +class A: + """Hello and goodbye""" + def __init__(): + """Wow, I have no function!""" + pass + +class B(object): + NO_MEANING: str = "eggs" + pass + +class C(object): + def say_no(self): + return "no" + def get_answer(self): + """ Return say_no() """ + return self.say_no() + def is_it_true(self): + """ Return self.get_answer() """ + return self.get_answer() + def __class_getitem__(self, item): + return types.GenericAlias(self, item) + +def doc_func(): + """ + This function solves all of the world's problems: + hunger + lack of Python + war + """ + +def nodoc_func(): + pass + + +list_alias1 = typing.List[int] +list_alias2 = list[int] +c_alias = C[int] +type_union1 = typing.Union[int, str] +type_union2 = int | str diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py new file mode 100644 index 00000000000..412aa3743e4 --- /dev/null +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -0,0 +1,191 @@ +"""Something just to look at via pydoc.""" + +import types + +def global_func(x, y): + """Module global function""" + +def global_func2(x, y): + """Module global function 2""" + +class A: + "A class." + + def A_method(self): + "Method defined in A." + def AB_method(self): + "Method defined in A and B." + def AC_method(self): + "Method defined in A and C." + def AD_method(self): + "Method defined in A and D." + def ABC_method(self): + "Method defined in A, B and C." + def ABD_method(self): + "Method defined in A, B and D." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + + def A_classmethod(cls, x): + "A class method defined in A." + A_classmethod = classmethod(A_classmethod) + + def A_staticmethod(x, y): + "A static method defined in A." + A_staticmethod = staticmethod(A_staticmethod) + + def _getx(self): + "A property getter function." + def _setx(self, value): + "A property setter function." + def _delx(self): + "A property deleter function." + A_property = property(fdel=_delx, fget=_getx, fset=_setx, + doc="A sample property defined in A.") + + A_int_alias = int + +class B(A): + "A class, derived from A." + + def AB_method(self): + "Method defined in A and B." + def ABC_method(self): + "Method defined in A, B and C." + def ABD_method(self): + "Method defined in A, B and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def B_method(self): + "Method defined in B." + def BC_method(self): + "Method defined in B and C." + def BD_method(self): + "Method defined in B and D." + def BCD_method(self): + "Method defined in B, C and D." + + @classmethod + def B_classmethod(cls, x): + "A class method defined in B." + + global_func = global_func # same name + global_func_alias = global_func + global_func2_alias = global_func2 + B_classmethod_alias = B_classmethod + A_classmethod_ref = A.A_classmethod + A_staticmethod = A.A_staticmethod # same name + A_staticmethod_alias = A.A_staticmethod + A_method_ref = A().A_method + A_method_alias = A.A_method + B_method_alias = B_method + count = list.count # same name + list_count = list.count + __repr__ = object.__repr__ # same name + object_repr = object.__repr__ + get = {}.get # same name + dict_get = {}.get + from math import sin + + +B.B_classmethod_ref = B.B_classmethod + + +class C(A): + "A class, derived from A." + + def AC_method(self): + "Method defined in A and C." + def ABC_method(self): + "Method defined in A, B and C." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def BC_method(self): + "Method defined in B and C." + def BCD_method(self): + "Method defined in B, C and D." + def C_method(self): + "Method defined in C." + def CD_method(self): + "Method defined in C and D." + +class D(B, C): + """A class, derived from B and C. + """ + + def AD_method(self): + "Method defined in A and D." + def ABD_method(self): + "Method defined in A, B and D." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def BD_method(self): + "Method defined in B and D." + def BCD_method(self): + "Method defined in B, C and D." + def CD_method(self): + "Method defined in C and D." + def D_method(self): + "Method defined in D." + +class FunkyProperties(object): + """From SF bug 472347, by Roeland Rengelink. + + Property getters etc may not be vanilla functions or methods, + and this used to make GUI pydoc blow up. + """ + + def __init__(self): + self.desc = {'x':0} + + class get_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst): + print('Get called', self, inst) + return inst.desc[self.attr] + class set_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst, val): + print('Set called', self, inst, val) + inst.desc[self.attr] = val + class del_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst): + print('Del called', self, inst) + del inst.desc[self.attr] + + x = property(get_desc('x'), set_desc('x'), del_desc('x'), 'prop x') + + +submodule = types.ModuleType(__name__ + '.submodule', + """A submodule, which should appear in its parent's summary""") + +global_func_alias = global_func +A_classmethod = A.A_classmethod # same name +A_classmethod2 = A.A_classmethod +A_classmethod3 = B.A_classmethod +A_staticmethod = A.A_staticmethod # same name +A_staticmethod_alias = A.A_staticmethod +A_staticmethod_ref = A().A_staticmethod +A_staticmethod_ref2 = B().A_staticmethod +A_method = A().A_method # same name +A_method2 = A().A_method +A_method3 = B().A_method +B_method = B.B_method # same name +B_method2 = B.B_method +count = list.count # same name +list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ +get = {}.get # same name +dict_get = {}.get +from math import sin # noqa: F401 diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py new file mode 100644 index 00000000000..e91c36aed19 --- /dev/null +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -0,0 +1,2459 @@ +import datetime +import os +import sys +import contextlib +import importlib.util +import inspect +import io +import pydoc +import py_compile +import keyword +try: # TODO: RUSTPYTHON; Implement `_pickle` + import _pickle +except ImportError: + _pickle = None +import pkgutil +import re +import tempfile +import test.support +import time +import types +import typing +import unittest +import unittest.mock +import urllib.parse +import xml.etree +import xml.etree.ElementTree +import textwrap +from io import StringIO +from collections import namedtuple +from urllib.request import urlopen, urlcleanup +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support.script_helper import (assert_python_ok, + assert_python_failure, spawn_python) +from test.support import threading_helper +from test.support import (reap_children, captured_stdout, + captured_stderr, is_wasm32, + requires_docstrings, MISSING_C_DOCSTRINGS) +from test.support.os_helper import (TESTFN, rmtree, unlink) +from test.test_pydoc import pydoc_mod +from test.test_pydoc import pydocfodder + + +class nonascii: + 'Це не латиниця' + pass + +if test.support.HAVE_DOCSTRINGS: + expected_data_docstrings = ( + 'dictionary for instance variables', + 'list of weak references to the object', + ) * 2 +else: + expected_data_docstrings = ('', '', '', '') + +expected_text_pattern = """ +NAME + test.test_pydoc.pydoc_mod - This is a test module for test_pydoc +%s +CLASSES + builtins.object + A + B + C + + class A(builtins.object) + | Hello and goodbye + | + | Methods defined here: + | + | __init__() + | Wow, I have no function! + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + + class B(builtins.object) + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + | + | ---------------------------------------------------------------------- + | Data and other attributes defined here: + | + | NO_MEANING = 'eggs' + + class C(builtins.object) + | Methods defined here: + | + | get_answer(self) + | Return say_no() + | + | is_it_true(self) + | Return self.get_answer() + | + | say_no(self) + | + | ---------------------------------------------------------------------- + | Class methods defined here: + | + | __class_getitem__(item) + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__ + | dictionary for instance variables + | + | __weakref__ + | list of weak references to the object + +FUNCTIONS + doc_func() + This function solves all of the world's problems: + hunger + lack of Python + war + + nodoc_func() + +DATA + __xyz__ = 'X, Y and Z' + c_alias = test.test_pydoc.pydoc_mod.C[int] + list_alias1 = typing.List[int] + list_alias2 = list[int] + type_union1 = int | str + type_union2 = int | str + +VERSION + 1.2.3.4 + +AUTHOR + Benjamin Peterson + +CREDITS + Nobody + +FILE + %s +""".strip() + +expected_text_data_docstrings = tuple('\n | ' + s if s else '' + for s in expected_data_docstrings) + +html2text_of_expected = """ +test.test_pydoc.pydoc_mod (version 1.2.3.4) +This is a test module for test_pydoc + +Modules + types + typing + +Classes + builtins.object + A + B + C + +class A(builtins.object) + Hello and goodbye + + Methods defined here: + __init__() + Wow, I have no function! + ---------------------------------------------------------------------- + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + +class B(builtins.object) + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + ---------------------------------------------------------------------- + Data and other attributes defined here: + NO_MEANING = 'eggs' + + +class C(builtins.object) + Methods defined here: + get_answer(self) + Return say_no() + is_it_true(self) + Return self.get_answer() + say_no(self) + ---------------------------------------------------------------------- + Class methods defined here: + __class_getitem__(item) + ---------------------------------------------------------------------- + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + +Functions + doc_func() + This function solves all of the world's problems: + hunger + lack of Python + war + nodoc_func() + +Data + __xyz__ = 'X, Y and Z' + c_alias = test.test_pydoc.pydoc_mod.C[int] + list_alias1 = typing.List[int] + list_alias2 = list[int] + type_union1 = int | str + type_union2 = int | str + +Author + Benjamin Peterson + +Credits + Nobody +""" + +expected_html_data_docstrings = tuple(s.replace(' ', '&nbsp;') + for s in expected_data_docstrings) + +# output pattern for missing module +missing_pattern = '''\ +No Python documentation found for %r. +Use help() to get the interactive help utility. +Use help(str) for help on the str class.'''.replace('\n', os.linesep) + +# output pattern for module with bad imports +badimport_pattern = "problem in %s - ModuleNotFoundError: No module named %r" + +expected_dynamicattribute_pattern = """ +Help on class DA in module %s: + +class DA(builtins.object) + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + | + | ham + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta: + | + | ham = 'spam' +""".strip() + +expected_virtualattribute_pattern1 = """ +Help on class Class in module %s: + +class Class(builtins.object) + | Data and other attributes inherited from Meta: + | + | LIFE = 42 +""".strip() + +expected_virtualattribute_pattern2 = """ +Help on class Class1 in module %s: + +class Class1(builtins.object) + | Data and other attributes inherited from Meta1: + | + | one = 1 +""".strip() + +expected_virtualattribute_pattern3 = """ +Help on class Class2 in module %s: + +class Class2(Class1) + | Method resolution order: + | Class2 + | Class1 + | builtins.object + | + | Data and other attributes inherited from Meta1: + | + | one = 1 + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta3: + | + | three = 3 + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta2: + | + | two = 2 +""".strip() + +expected_missingattribute_pattern = """ +Help on class C in module %s: + +class C(builtins.object) + | Data and other attributes defined here: + | + | here = 'present!' +""".strip() + +def run_pydoc(module_name, *args, **env): + """ + Runs pydoc on the specified module. Returns the stripped + output of pydoc. + """ + args = args + (module_name,) + # do not write bytecode files to avoid caching errors + rc, out, err = assert_python_ok('-B', pydoc.__file__, *args, **env) + return out.strip() + +def run_pydoc_fail(module_name, *args, **env): + """ + Runs pydoc on the specified module expecting a failure. + """ + args = args + (module_name,) + rc, out, err = assert_python_failure('-B', pydoc.__file__, *args, **env) + return out.strip() + +def get_pydoc_html(module): + "Returns pydoc generated output as html" + doc = pydoc.HTMLDoc() + output = doc.docmodule(module) + loc = doc.getdocloc(pydoc_mod) or "" + if loc: + loc = "<br><a href=\"" + loc + "\">Module Docs</a>" + return output.strip(), loc + +def clean_text(doc): + # clean up the extra text formatting that pydoc performs + return re.sub('\b.', '', doc) + +def get_pydoc_link(module): + "Returns a documentation web link of a module" + abspath = os.path.abspath + dirname = os.path.dirname + basedir = dirname(dirname(dirname(abspath(__file__)))) + doc = pydoc.TextDoc() + loc = doc.getdocloc(module, basedir=basedir) + return loc + +def get_pydoc_text(module): + "Returns pydoc generated output as text" + doc = pydoc.TextDoc() + loc = doc.getdocloc(pydoc_mod) or "" + if loc: + loc = "\nMODULE DOCS\n " + loc + "\n" + + output = doc.docmodule(module) + output = clean_text(output) + return output.strip(), loc + +def get_html_title(text): + # Bit of hack, but good enough for test purposes + header, _, _ = text.partition("</head>") + _, _, title = header.partition("<title>") + title, _, _ = title.partition("</title>") + return title + + +def html2text(html): + """A quick and dirty implementation of html2text. + + Tailored for pydoc tests only. + """ + html = html.replace("<dd>", "\n") + html = html.replace("<hr>", "-"*70) + html = re.sub("<.*?>", "", html) + html = pydoc.replace(html, "&nbsp;", " ", "&gt;", ">", "&lt;", "<") + return html + + +class PydocBaseTest(unittest.TestCase): + def tearDown(self): + # Self-testing. Mocking only works if sys.modules['pydoc'] and pydoc + # are the same. But some pydoc functions reload the module and change + # sys.modules, so check that it was restored. + self.assertIs(sys.modules['pydoc'], pydoc) + + def _restricted_walk_packages(self, walk_packages, path=None): + """ + A version of pkgutil.walk_packages() that will restrict itself to + a given path. + """ + default_path = path or [os.path.dirname(__file__)] + def wrapper(path=None, prefix='', onerror=None): + return walk_packages(path or default_path, prefix, onerror) + return wrapper + + @contextlib.contextmanager + def restrict_walk_packages(self, path=None): + walk_packages = pkgutil.walk_packages + pkgutil.walk_packages = self._restricted_walk_packages(walk_packages, + path) + try: + yield + finally: + pkgutil.walk_packages = walk_packages + + def call_url_handler(self, url, expected_title): + text = pydoc._url_handler(url, "text/html") + result = get_html_title(text) + # Check the title to ensure an unexpected error page was not returned + self.assertEqual(result, expected_title, text) + return text + + +class PydocDocTest(unittest.TestCase): + maxDiff = None + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_html_doc(self): + result, doc_loc = get_pydoc_html(pydoc_mod) + text_result = html2text(result) + text_lines = [line.strip() for line in text_result.splitlines()] + text_lines = [line for line in text_lines if line] + del text_lines[1] + expected_lines = html2text_of_expected.splitlines() + expected_lines = [line.strip() for line in expected_lines if line] + self.assertEqual(text_lines, expected_lines) + mod_file = inspect.getabsfile(pydoc_mod) + mod_url = urllib.parse.quote(mod_file) + self.assertIn(mod_url, result) + self.assertIn(mod_file, result) + self.assertIn(doc_loc, result) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_text_doc(self): + result, doc_loc = get_pydoc_text(pydoc_mod) + expected_text = expected_text_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual(expected_text, result) + + def test_text_enum_member_with_value_zero(self): + # Test issue #20654 to ensure enum member with value 0 can be + # displayed. It used to throw KeyError: 'zero'. + import enum + class BinaryInteger(enum.IntEnum): + zero = 0 + one = 1 + doc = pydoc.render_doc(BinaryInteger) + self.assertIn('BinaryInteger.zero', doc) + + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + + def test_mixed_case_module_names_are_lower_cased(self): + # issue16484 + doc_link = get_pydoc_link(xml.etree.ElementTree) + self.assertIn('xml.etree.elementtree', doc_link) + + def test_issue8225(self): + # Test issue8225 to ensure no doc link appears for xml.etree + result, doc_loc = get_pydoc_text(xml.etree) + self.assertEqual(doc_loc, "", "MODULE DOCS incorrectly includes a link") + + def test_online_docs_link(self): + import encodings.idna + import importlib._bootstrap + + module_docs = { + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + } + + with unittest.mock.patch('pydoc_data.module_docs.module_docs', module_docs): + doc = pydoc.TextDoc() + + basedir = os.path.dirname(encodings.__file__) + doc_link = doc.getdocloc(encodings, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings', doc_link) + self.assertNotIn('encodings.html', doc_link) + + doc_link = doc.getdocloc(encodings.idna, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings.idna', doc_link) + self.assertNotIn('encodings.idna.html', doc_link) + + doc_link = doc.getdocloc(importlib._bootstrap, basedir=basedir) + self.assertIsNone(doc_link) + + def test_getpager_with_stdin_none(self): + previous_stdin = sys.stdin + try: + sys.stdin = None + pydoc.getpager() # Shouldn't fail. + finally: + sys.stdin = previous_stdin + + def test_non_str_name(self): + # issue14638 + # Treat illegal (non-str) name like no name + + class A: + __name__ = 42 + class B: + pass + adoc = pydoc.render_doc(A()) + bdoc = pydoc.render_doc(B()) + self.assertEqual(adoc.replace("A", "B"), bdoc) + + def test_not_here(self): + missing_module = "test.i_am_not_here" + result = str(run_pydoc_fail(missing_module), 'ascii') + expected = missing_pattern % missing_module + self.assertEqual(expected, result, + "documentation for missing module found") + + @requires_docstrings + def test_not_ascii(self): + result = run_pydoc('test.test_pydoc.test_pydoc.nonascii', PYTHONIOENCODING='ascii') + encoded = nonascii.__doc__.encode('ascii', 'backslashreplace') + self.assertIn(encoded, result) + + def test_input_strip(self): + missing_module = " test.i_am_not_here " + result = str(run_pydoc_fail(missing_module), 'ascii') + expected = missing_pattern % missing_module.strip() + self.assertEqual(expected, result) + + def test_stripid(self): + # test with strings, other implementations might have different repr() + stripid = pydoc.stripid + # strip the id + self.assertEqual(stripid('<function stripid at 0x88dcee4>'), + '<function stripid>') + self.assertEqual(stripid('<function stripid at 0x01F65390>'), + '<function stripid>') + # nothing to strip, return the same text + self.assertEqual(stripid('42'), '42') + self.assertEqual(stripid("<type 'exceptions.Exception'>"), + "<type 'exceptions.Exception'>") + + @unittest.skip("TODO: RUSTPYTHON; Panic") + def test_builtin_with_more_than_four_children(self): + """Tests help on builtin object which have more than four child classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section and only 4 classes + should be displayed with a hint on how many more subclasses are present. + For example: + + >>> help(object) + Help on class object in module builtins: + + class object + | The most base type + | + | Built-in subclasses: + | async_generator + | BaseException + | builtin_function_or_method + | bytearray + | ... and 82 other subclasses + """ + doc = pydoc.TextDoc() + try: + # Make sure HeapType, which has no __module__ attribute, is one + # of the known subclasses of object. (doc.docclass() used to + # fail if HeapType was imported before running this test, like + # when running tests sequentially.) + from _testcapi import HeapType + except ImportError: + pass + text = doc.docclass(object) + snip = (" | Built-in subclasses:\n" + " | async_generator\n" + " | BaseException\n" + " | builtin_function_or_method\n" + " | bytearray\n" + " | ... and \\d+ other subclasses") + self.assertRegex(text, snip) + + def test_builtin_with_child(self): + """Tests help on builtin object which have only child classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section. For example: + + >>> help(ArithmeticError) + Help on class ArithmeticError in module builtins: + + class ArithmeticError(Exception) + | Base class for arithmetic errors. + | + ... + | + | Built-in subclasses: + | FloatingPointError + | OverflowError + | ZeroDivisionError + """ + doc = pydoc.TextDoc() + text = doc.docclass(ArithmeticError) + snip = (" | Built-in subclasses:\n" + " | FloatingPointError\n" + " | OverflowError\n" + " | ZeroDivisionError") + self.assertIn(snip, text) + + def test_builtin_with_grandchild(self): + """Tests help on builtin classes which have grandchild classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section. However, if it also has + grandchildren, these should not show up on the subclasses section. + For example: + + >>> help(Exception) + Help on class Exception in module builtins: + + class Exception(BaseException) + | Common base class for all non-exit exceptions. + | + ... + | + | Built-in subclasses: + | ArithmeticError + | AssertionError + | AttributeError + ... + """ + doc = pydoc.TextDoc() + text = doc.docclass(Exception) + snip = (" | Built-in subclasses:\n" + " | ArithmeticError\n" + " | AssertionError\n" + " | AttributeError") + self.assertIn(snip, text) + # Testing that the grandchild ZeroDivisionError does not show up + self.assertNotIn('ZeroDivisionError', text) + + def test_builtin_no_child(self): + """Tests help on builtin object which have no child classes. + + When running help() on a builtin class which has no child classes, it + should not contain any "Built-in subclasses" section. For example: + + >>> help(ZeroDivisionError) + + Help on class ZeroDivisionError in module builtins: + + class ZeroDivisionError(ArithmeticError) + | Second argument to a division or modulo operation was zero. + | + | Method resolution order: + | ZeroDivisionError + | ArithmeticError + | Exception + | BaseException + | object + | + | Methods defined here: + ... + """ + doc = pydoc.TextDoc() + text = doc.docclass(ZeroDivisionError) + # Testing that the subclasses section does not appear + self.assertNotIn('Built-in subclasses', text) + + def test_builtin_on_metaclasses(self): + """Tests help on metaclasses. + + When running help() on a metaclasses such as type, it + should not contain any "Built-in subclasses" section. + """ + doc = pydoc.TextDoc() + text = doc.docclass(type) + # Testing that the subclasses section does not appear + self.assertNotIn('Built-in subclasses', text) + + def test_fail_help_cli(self): + elines = (missing_pattern % 'abd').splitlines() + with spawn_python("-c" "help()") as proc: + out, _ = proc.communicate(b"abd") + olines = out.decode().splitlines()[-9:-6] + olines[0] = olines[0].removeprefix('help> ') + self.assertEqual(elines, olines) + + def test_fail_help_output_redirect(self): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help("abd") + expected = missing_pattern % "abd" + self.assertEqual(expected, buf.getvalue().strip().replace('\n', os.linesep)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') + @requires_docstrings + def test_help_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + self.maxDiff = None + + unused, doc_loc = get_pydoc_text(pydoc_mod) + module = "test.test_pydoc.pydoc_mod" + help_header = """ + Help on module test.test_pydoc.pydoc_mod in test.test_pydoc: + + """.lstrip() + help_header = textwrap.dedent(help_header) + expected_help_pattern = help_header + expected_text_pattern + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(request) + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) + self.assertEqual('', err.getvalue()) + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', + 'Help on method descriptor upper in str:') + run_pydoc_pager(str.upper, 'str.upper', + 'Help on method descriptor upper:') + run_pydoc_pager(''.upper, 'str.upper', + 'Help on built-in function upper:') + run_pydoc_pager(str.__add__, + 'str.__add__', 'Help on method descriptor __add__:') + run_pydoc_pager(''.__add__, + 'str.__add__', 'Help on method wrapper __add__:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() + + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) + + def test_namedtuple_fields(self): + Person = namedtuple('Person', ['nickname', 'firstname']) + with captured_stdout() as help_io: + pydoc.help(Person) + helptext = help_io.getvalue() + self.assertIn("nickname", helptext) + self.assertIn("firstname", helptext) + self.assertIn("Alias for field number 0", helptext) + self.assertIn("Alias for field number 1", helptext) + + def test_namedtuple_public_underscore(self): + NT = namedtuple('NT', ['abc', 'def'], rename=True) + with captured_stdout() as help_io: + pydoc.help(NT) + helptext = help_io.getvalue() + self.assertIn('_1', helptext) + self.assertIn('_replace', helptext) + self.assertIn('_asdict', helptext) + + def test_synopsis(self): + self.addCleanup(unlink, TESTFN) + for encoding in ('ISO-8859-1', 'UTF-8'): + with open(TESTFN, 'w', encoding=encoding) as script: + if encoding != 'UTF-8': + print('#coding: {}'.format(encoding), file=script) + print('"""line 1: h\xe9', file=script) + print('line 2: hi"""', file=script) + synopsis = pydoc.synopsis(TESTFN, {}) + self.assertEqual(synopsis, 'line 1: h\xe9') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + + @requires_docstrings + def test_synopsis_sourceless(self): + os = import_helper.import_fresh_module('os') + expected = os.__doc__.splitlines()[0] + filename = os.__spec__.cached + synopsis = pydoc.synopsis(filename) + + self.assertEqual(synopsis, expected) + + def test_synopsis_sourceless_empty_doc(self): + with os_helper.temp_cwd() as test_dir: + init_path = os.path.join(test_dir, 'foomod42.py') + cached_path = importlib.util.cache_from_source(init_path) + with open(init_path, 'w') as fobj: + fobj.write("foo = 1") + py_compile.compile(init_path) + synopsis = pydoc.synopsis(init_path, {}) + self.assertIsNone(synopsis) + synopsis_cached = pydoc.synopsis(cached_path, {}) + self.assertIsNone(synopsis_cached) + + def test_splitdoc_with_description(self): + example_string = "I Am A Doc\n\n\nHere is my description" + self.assertEqual(pydoc.splitdoc(example_string), + ('I Am A Doc', '\nHere is my description')) + + def test_is_package_when_not_package(self): + with os_helper.temp_cwd() as test_dir: + with self.assertWarns(DeprecationWarning) as cm: + self.assertFalse(pydoc.ispackage(test_dir)) + self.assertEqual(cm.filename, __file__) + + def test_is_package_when_is_package(self): + with os_helper.temp_cwd() as test_dir: + init_path = os.path.join(test_dir, '__init__.py') + open(init_path, 'w').close() + with self.assertWarns(DeprecationWarning) as cm: + self.assertTrue(pydoc.ispackage(test_dir)) + os.remove(init_path) + self.assertEqual(cm.filename, __file__) + + def test_allmethods(self): + # issue 17476: allmethods was no longer returning unbound methods. + # This test is a bit fragile in the face of changes to object and type, + # but I can't think of a better way to do it without duplicating the + # logic of the function under test. + + class TestClass(object): + def method_returning_true(self): + return True + + # What we expect to get back: everything on object... + expected = dict(vars(object)) + # ...plus our unbound method... + expected['method_returning_true'] = TestClass.method_returning_true + # ...but not the non-methods on object. + del expected['__doc__'] + del expected['__class__'] + # inspect resolves descriptors on type into methods, but vars doesn't, + # so we need to update __subclasshook__ and __init_subclass__. + expected['__subclasshook__'] = TestClass.__subclasshook__ + expected['__init_subclass__'] = TestClass.__init_subclass__ + + methods = pydoc.allmethods(TestClass) + self.assertDictEqual(methods, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_method_aliases(self): + class A: + def tkraise(self, aboveThis=None): + """Raise this widget in the stacking order.""" + lift = tkraise + def a_size(self): + """Return size""" + class B(A): + def itemconfigure(self, tagOrId, cnf=None, **kw): + """Configure resources of an item TAGORID.""" + itemconfig = itemconfigure + b_size = A.a_size + + doc = pydoc.render_doc(B) + doc = clean_text(doc) + self.assertEqual(doc, '''\ +Python Library Documentation: class B in module %s + +class B(A) + | Method resolution order: + | B + | A + | builtins.object + | + | Methods defined here: + | + | b_size = a_size(self) + | + | itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw) + | + | itemconfigure(self, tagOrId, cnf=None, **kw) + | Configure resources of an item TAGORID. + | + | ---------------------------------------------------------------------- + | Methods inherited from A: + | + | a_size(self) + | Return size + | + | lift = tkraise(self, aboveThis=None) + | + | tkraise(self, aboveThis=None) + | Raise this widget in the stacking order. + | + | ---------------------------------------------------------------------- + | Data descriptors inherited from A: + | + | __dict__ + | dictionary for instance variables + | + | __weakref__ + | list of weak references to the object +''' % __name__) + + doc = pydoc.render_doc(B, renderer=pydoc.HTMLDoc()) + expected_text = f""" +Python Library Documentation + +class B in module {__name__} +class B(A) + Method resolution order: + B + A + builtins.object + + Methods defined here: + b_size = a_size(self) + itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw) + itemconfigure(self, tagOrId, cnf=None, **kw) + Configure resources of an item TAGORID. + + Methods inherited from A: + a_size(self) + Return size + lift = tkraise(self, aboveThis=None) + tkraise(self, aboveThis=None) + Raise this widget in the stacking order. + + Data descriptors inherited from A: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object +""" + as_text = html2text(doc) + expected_lines = [line.strip() for line in expected_text.split("\n") if line] + for expected_line in expected_lines: + self.assertIn(expected_line, as_text) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_long_signatures(self): + from collections.abc import Callable + from typing import Literal, Annotated + + class A: + def __init__(self, + arg1: Callable[[int, int, int], str], + arg2: Literal['some value', 'other value'], + arg3: Annotated[int, 'some docs about this type'], + ) -> None: + ... + + doc = pydoc.render_doc(A) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: class A in module %s + +class A(builtins.object) + | A( + | arg1: Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | Methods defined here: + | + | __init__( + | self, + | arg1: Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s +''' % (__name__, + '' if MISSING_C_DOCSTRINGS else '\n | dictionary for instance variables', + '' if MISSING_C_DOCSTRINGS else '\n | list of weak references to the object', + )) + + def func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8], + ) -> Annotated[int, 'Some other']: + ... + + doc = pydoc.render_doc(func) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function func in module %s + +func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] +) -> Annotated[int, 'Some other'] +''' % __name__) + + def function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str, + ): + ... + + doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s + +function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str +) +''' % __name__) + + does_not_have_name = lambda \ + very_long_parameter_name_that_should_not_fit_into_a_single_line, \ + second_very_long_parameter_name: ... + + doc = pydoc.render_doc(does_not_have_name) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function <lambda> in module %s + +<lambda> lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name +''' % __name__) + + def test__future__imports(self): + # __future__ features are excluded from module help, + # except when it's the __future__ module itself + import __future__ + future_text, _ = get_pydoc_text(__future__) + future_html, _ = get_pydoc_html(__future__) + pydoc_mod_text, _ = get_pydoc_text(pydoc_mod) + pydoc_mod_html, _ = get_pydoc_html(pydoc_mod) + + for feature in __future__.all_feature_names: + txt = f"{feature} = _Feature" + html = f"<strong>{feature}</strong> = _Feature" + self.assertIn(txt, future_text) + self.assertIn(html, future_html) + self.assertNotIn(txt, pydoc_mod_text) + self.assertNotIn(html, pydoc_mod_html) + + +class PydocImportTest(PydocBaseTest): + + def setUp(self): + self.test_dir = os.mkdir(TESTFN) + self.addCleanup(rmtree, TESTFN) + importlib.invalidate_caches() + + def test_badimport(self): + # This tests the fix for issue 5230, where if pydoc found the module + # but the module had an internal import error pydoc would report no doc + # found. + modname = 'testmod_xyzzy' + testpairs = ( + ('i_am_not_here', 'i_am_not_here'), + ('test.i_am_not_here_either', 'test.i_am_not_here_either'), + ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'), + ('i_am_not_here.{}'.format(modname), 'i_am_not_here'), + ('test.{}'.format(modname), 'test.{}'.format(modname)), + ) + + sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py" + for importstring, expectedinmsg in testpairs: + with open(sourcefn, 'w') as f: + f.write("import {}\n".format(importstring)) + result = run_pydoc_fail(modname, PYTHONPATH=TESTFN).decode("ascii") + expected = badimport_pattern % (modname, expectedinmsg) + self.assertEqual(expected, result) + + def test_apropos_with_bad_package(self): + # Issue 7425 - pydoc -k failed when bad package on path + pkgdir = os.path.join(TESTFN, "syntaxerr") + os.mkdir(pkgdir) + badsyntax = os.path.join(pkgdir, "__init__") + os.extsep + "py" + with open(badsyntax, 'w') as f: + f.write("invalid python syntax = $1\n") + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('xyzzy') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + # The package name is still matched + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('syntaxerr') + self.assertEqual(out.getvalue().strip(), 'syntaxerr') + self.assertEqual(err.getvalue(), '') + + def test_apropos_with_unreadable_dir(self): + # Issue 7367 - pydoc -k failed when unreadable dir on path + self.unreadable_dir = os.path.join(TESTFN, "unreadable") + os.mkdir(self.unreadable_dir, 0) + self.addCleanup(os.rmdir, self.unreadable_dir) + # Note, on Windows the directory appears to be still + # readable so this is not really testing the issue there + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('SOMEKEY') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + + def test_apropos_empty_doc(self): + pkgdir = os.path.join(TESTFN, 'walkpkg') + os.mkdir(pkgdir) + self.addCleanup(rmtree, pkgdir) + init_path = os.path.join(pkgdir, '__init__.py') + with open(init_path, 'w') as fobj: + fobj.write("foo = 1") + with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: + pydoc.apropos('') + self.assertIn('walkpkg', stdout.getvalue()) + + def test_url_search_package_error(self): + # URL handler search should cope with packages that raise exceptions + pkgdir = os.path.join(TESTFN, "test_error_package") + os.mkdir(pkgdir) + init = os.path.join(pkgdir, "__init__.py") + with open(init, "wt", encoding="ascii") as f: + f.write("""raise ValueError("ouch")\n""") + with self.restrict_walk_packages(path=[TESTFN]): + # Package has to be importable for the error to have any effect + saved_paths = tuple(sys.path) + sys.path.insert(0, TESTFN) + try: + with self.assertRaisesRegex(ValueError, "ouch"): + # Sanity check + import test_error_package # noqa: F401 + + text = self.call_url_handler("search?key=test_error_package", + "Pydoc: Search Results") + found = ('<a href="test_error_package.html">' + 'test_error_package</a>') + self.assertIn(found, text) + finally: + sys.path[:] = saved_paths + + @unittest.skip('causes undesirable side-effects (#20128)') + def test_modules(self): + # See Helper.listmodules(). + num_header_lines = 2 + num_module_lines_min = 5 # Playing it safe. + num_footer_lines = 3 + expected = num_header_lines + num_module_lines_min + num_footer_lines + + output = StringIO() + helper = pydoc.Helper(output=output) + helper('modules') + result = output.getvalue().strip() + num_lines = len(result.splitlines()) + + self.assertGreaterEqual(num_lines, expected) + + @unittest.skip('causes undesirable side-effects (#20128)') + def test_modules_search(self): + # See Helper.listmodules(). + expected = 'pydoc - ' + + output = StringIO() + helper = pydoc.Helper(output=output) + with captured_stdout() as help_io: + helper('modules pydoc') + result = help_io.getvalue() + + self.assertIn(expected, result) + + @unittest.skip('some buildbots are not cooperating (#20128)') + def test_modules_search_builtin(self): + expected = 'gc - ' + + output = StringIO() + helper = pydoc.Helper(output=output) + with captured_stdout() as help_io: + helper('modules garbage') + result = help_io.getvalue() + + self.assertStartsWith(result, expected) + + def test_importfile(self): + try: + loaded_pydoc = pydoc.importfile(pydoc.__file__) + + self.assertIsNot(loaded_pydoc, pydoc) + self.assertEqual(loaded_pydoc.__name__, 'pydoc') + self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) + self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + finally: + sys.modules['pydoc'] = pydoc + + +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h + + +class Square(Rect): + area = property(lambda self: self.side**2) + + +class TestDescriptions(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def test_module(self): + # Check that pydocfodder module can be described + doc = pydoc.render_doc(pydocfodder) + self.assertIn("pydocfodder", doc) + + def test_class(self): + class C: "New-style class" + c = C() + + self.assertEqual(pydoc.describe(C), 'class C') + self.assertEqual(pydoc.describe(c), 'C') + expected = 'C in module %s object' % __name__ + self.assertIn(expected, pydoc.render_doc(c)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_generic_alias(self): + self.assertEqual(pydoc.describe(typing.List[int]), '_GenericAlias') + doc = pydoc.render_doc(typing.List[int], renderer=pydoc.plaintext) + self.assertIn('_GenericAlias in module typing', doc) + self.assertIn('List = class list(object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) + + self.assertEqual(pydoc.describe(list[int]), 'GenericAlias') + doc = pydoc.render_doc(list[int], renderer=pydoc.plaintext) + self.assertIn('GenericAlias in module builtins', doc) + self.assertIn('\nclass list(object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_union_type(self): + self.assertEqual(pydoc.describe(typing.Union[int, str]), 'Union') + doc = pydoc.render_doc(typing.Union[int, str], renderer=pydoc.plaintext) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) + if typing.Union.__doc__: + self.assertIn(typing.Union.__doc__.strip().splitlines()[0], doc) + + self.assertEqual(pydoc.describe(int | str), 'Union') + doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) + + def test_special_form(self): + self.assertEqual(pydoc.describe(typing.NoReturn), '_SpecialForm') + doc = pydoc.render_doc(typing.NoReturn, renderer=pydoc.plaintext) + self.assertIn('_SpecialForm in module typing', doc) + if typing.NoReturn.__doc__: + self.assertIn('NoReturn = typing.NoReturn', doc) + self.assertIn(typing.NoReturn.__doc__.strip().splitlines()[0], doc) + else: + self.assertIn('NoReturn = class _SpecialForm(_Final)', doc) + + def test_typing_pydoc(self): + def foo(data: typing.List[typing.Any], + x: int) -> typing.Iterator[typing.Tuple[int, typing.Any]]: + ... + T = typing.TypeVar('T') + class C(typing.Generic[T], typing.Mapping[int, str]): ... + self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], + 'f\x08fo\x08oo\x08o(data: typing.List[typing.Any], x: int)' + ' -> typing.Iterator[typing.Tuple[int, typing.Any]]') + self.assertEqual(pydoc.render_doc(C).splitlines()[2], + 'class C\x08C(collections.abc.Mapping, typing.Generic)') + + def test_builtin(self): + for name in ('str', 'str.translate', 'builtins.str', + 'builtins.str.translate'): + # test low-level function + self.assertIsNotNone(pydoc.locate(name)) + # test high-level function + try: + pydoc.render_doc(name) + except ImportError: + self.fail('finding the doc of {!r} failed'.format(name)) + + for name in ('notbuiltins', 'strrr', 'strr.translate', + 'str.trrrranslate', 'builtins.strrr', + 'builtins.str.trrranslate'): + self.assertIsNone(pydoc.locate(name)) + self.assertRaises(ImportError, pydoc.render_doc, name) + + @staticmethod + def _get_summary_line(o): + text = pydoc.plain(pydoc.render_doc(o)) + lines = text.split('\n') + assert len(lines) >= 2 + return lines[2] + + @staticmethod + def _get_summary_lines(o): + text = pydoc.plain(pydoc.render_doc(o)) + lines = text.split('\n') + return '\n'.join(lines[2:]) + + # these should include "self" + def test_unbound_python_method(self): + self.assertEqual(self._get_summary_line(textwrap.TextWrapper.wrap), + "wrap(self, text)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_unbound_builtin_method(self): + self.assertEqual(self._get_summary_line(_pickle.Pickler.dump), + "dump(self, obj, /) unbound _pickle.Pickler method") + + # these no longer include "self" + def test_bound_python_method(self): + t = textwrap.TextWrapper() + self.assertEqual(self._get_summary_line(t.wrap), + "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_bound_builtin_method(self): + s = StringIO() + p = _pickle.Pickler(s) + self.assertEqual(self._get_summary_line(p.dump), + "dump(obj, /) method of _pickle.Pickler instance") + + # this should *never* include self! + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_module_level_callable(self): + self.assertEqual(self._get_summary_line(os.stat), + "stat(path, *, dir_fd=None, follow_symlinks=True)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_module_level_callable_noargs(self): + self.assertEqual(self._get_summary_line(time.time), + "time()") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_module_level_callable_o(self): + try: + import _stat + except ImportError: + # stat.S_IMODE() and _stat.S_IMODE() have a different signature + self.skipTest('_stat extension is missing') + + self.assertEqual(self._get_summary_line(_stat.S_IMODE), + "S_IMODE(object, /)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(str.lower), + "lower(self, /) unbound builtins.str method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(''.lower), + "lower() method of builtins.str instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set.add), + "add(self, object, /) unbound builtins.set method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set().add), + "add(object, /) method of builtins.set instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set.__contains__), + "__contains__(self, object, /) unbound builtins.set method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set().__contains__), + "__contains__(object, /) method of builtins.set instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), + "utcnow(type, /) unbound datetime.datetime method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), + "utcnow() class method of datetime.datetime") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), + "__class_getitem__(type, object, /) unbound builtins.dict method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__class_getitem__), + "__class_getitem__(object, /) class method of builtins.dict") + + @support.cpython_only + @requires_docstrings + def test_module_level_callable_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.func_with_unrepresentable_signature + self.assertEqual(self._get_summary_line(builtin), + "func_with_unrepresentable_signature(a, b=<x>)") + + @support.cpython_only + @requires_docstrings + def test_builtin_staticmethod_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(str.maketrans), + "maketrans(x, y=<unrepresentable>, z=<unrepresentable>, /)") + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.staticmeth), + "staticmeth(a, b=<x>)") + + @support.cpython_only + @requires_docstrings + def test_unbound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(dict.pop), + "pop(self, key, default=<unrepresentable>, /) " + "unbound builtins.dict method") + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.meth), + "meth(self, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") + + @support.cpython_only + @requires_docstrings + def test_bound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line({}.pop), + "pop(key, default=<unrepresentable>, /) " + "method of builtins.dict instance") + _testcapi = import_helper.import_module("_testcapi") + obj = _testcapi.DocStringUnrepresentableSignatureTest() + self.assertEqual(self._get_summary_line(obj.meth), + "meth(a, b=<x>) " + "method of _testcapi.DocStringUnrepresentableSignatureTest instance") + + @support.cpython_only + @requires_docstrings + def test_unbound_builtin_classmethod_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + descr = cls.__dict__['classmeth'] + self.assertEqual(self._get_summary_line(descr), + "classmeth(type, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") + + @support.cpython_only + @requires_docstrings + def test_bound_builtin_classmethod_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.classmeth), + "classmeth(a, b=<x>) class method of " + "_testcapi.DocStringUnrepresentableSignatureTest") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_overridden_text_signature(self): + class C: + def meth(*args, **kwargs): + pass + @classmethod + def cmeth(*args, **kwargs): + pass + @staticmethod + def smeth(*args, **kwargs): + pass + for text_signature, unbound, bound in [ + ("($slf)", "(slf, /)", "()"), + ("($slf, /)", "(slf, /)", "()"), + ("($slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("($slf, /, arg=<x>)", "(slf, /, arg=<x>)", "(arg=<x>)"), + ("($slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("($slf, arg=<x>, /)", "(slf, arg=<x>, /)", "(arg=<x>, /)"), + ("(/, slf, arg)", "(/, slf, arg)", "(/, slf, arg)"), + ("(/, slf, arg=<x>)", "(/, slf, arg=<x>)", "(/, slf, arg=<x>)"), + ("(slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("(slf, /, arg=<x>)", "(slf, /, arg=<x>)", "(arg=<x>)"), + ("(slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("(slf, arg=<x>, /)", "(slf, arg=<x>, /)", "(arg=<x>, /)"), + ]: + with self.subTest(text_signature): + C.meth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.meth), + "meth" + unbound) + self.assertEqual(self._get_summary_line(C().meth), + "meth" + bound + " method of test.test_pydoc.test_pydoc.C instance") + C.cmeth.__func__.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.cmeth), + "cmeth" + bound + " class method of test.test_pydoc.test_pydoc.C") + C.smeth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.smeth), + "smeth" + unbound) + + @requires_docstrings + def test_staticmethod(self): + class X: + @staticmethod + def sm(x, y): + '''A static method''' + ... + self.assertEqual(self._get_summary_lines(X.__dict__['sm']), + 'sm(x, y)\n' + ' A static method\n') + self.assertEqual(self._get_summary_lines(X.sm), """\ +sm(x, y) + A static method +""") + self.assertIn(""" + | Static methods defined here: + | + | sm(x, y) + | A static method +""", pydoc.plain(pydoc.render_doc(X))) + + @requires_docstrings + def test_classmethod(self): + class X: + @classmethod + def cm(cls, x): + '''A class method''' + ... + self.assertEqual(self._get_summary_lines(X.__dict__['cm']), + 'cm(...)\n' + ' A class method\n') + self.assertEqual(self._get_summary_lines(X.cm), """\ +cm(x) class method of test.test_pydoc.test_pydoc.X + A class method +""") + self.assertIn(""" + | Class methods defined here: + | + | cm(x) + | A class method +""", pydoc.plain(pydoc.render_doc(X))) + + @requires_docstrings + def test_getset_descriptor(self): + # Currently these attributes are implemented as getset descriptors + # in CPython. + self.assertEqual(self._get_summary_line(int.numerator), "numerator") + self.assertEqual(self._get_summary_line(float.real), "real") + self.assertEqual(self._get_summary_line(Exception.args), "args") + self.assertEqual(self._get_summary_line(memoryview.obj), "obj") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_member_descriptor(self): + # Currently these attributes are implemented as member descriptors + # in CPython. + self.assertEqual(self._get_summary_line(complex.real), "real") + self.assertEqual(self._get_summary_line(range.start), "start") + self.assertEqual(self._get_summary_line(slice.start), "start") + self.assertEqual(self._get_summary_line(property.fget), "fget") + self.assertEqual(self._get_summary_line(StopIteration.value), "value") + + @requires_docstrings + def test_slot_descriptor(self): + class Point: + __slots__ = 'x', 'y' + self.assertEqual(self._get_summary_line(Point.x), "x") + + @requires_docstrings + def test_dict_attr_descriptor(self): + class NS: + pass + self.assertEqual(self._get_summary_line(NS.__dict__['__dict__']), + "__dict__") + + @requires_docstrings + def test_structseq_member_descriptor(self): + self.assertEqual(self._get_summary_line(type(sys.hash_info).width), + "width") + self.assertEqual(self._get_summary_line(type(sys.flags).debug), + "debug") + self.assertEqual(self._get_summary_line(type(sys.version_info).major), + "major") + self.assertEqual(self._get_summary_line(type(sys.float_info).max), + "max") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_namedtuple_field_descriptor(self): + Box = namedtuple('Box', ('width', 'height')) + self.assertEqual(self._get_summary_lines(Box.width), """\ + Alias for field number 0 +""") + + @requires_docstrings + def test_property(self): + self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area + Area of the rect +""") + self.assertIn(""" + | area + | Area of the rect +""", pydoc.plain(pydoc.render_doc(Rect))) + + @requires_docstrings + def test_custom_non_data_descriptor(self): + class Descr: + def __get__(self, obj, cls): + if obj is None: + return self + return 42 + class X: + attr = Descr() + + self.assertEqual(self._get_summary_lines(X.attr), f"""\ +<{__name__}.TestDescriptions.test_custom_non_data_descriptor.<locals>.Descr object>""") + + X.attr.__doc__ = 'Custom descriptor' + self.assertEqual(self._get_summary_lines(X.attr), f"""\ +<{__name__}.TestDescriptions.test_custom_non_data_descriptor.<locals>.Descr object> + Custom descriptor +""") + + X.attr.__name__ = 'foo' + self.assertEqual(self._get_summary_lines(X.attr), """\ +foo(...) + Custom descriptor +""") + + @requires_docstrings + def test_custom_data_descriptor(self): + class Descr: + def __get__(self, obj, cls): + if obj is None: + return self + return 42 + def __set__(self, obj, cls): + 1/0 + class X: + attr = Descr() + + self.assertEqual(self._get_summary_lines(X.attr), "") + + X.attr.__doc__ = 'Custom descriptor' + self.assertEqual(self._get_summary_lines(X.attr), """\ + Custom descriptor +""") + + X.attr.__name__ = 'foo' + self.assertEqual(self._get_summary_lines(X.attr), """\ +foo + Custom descriptor +""") + + def test_async_annotation(self): + async def coro_function(ign) -> int: + return 1 + + text = pydoc.plain(pydoc.plaintext.document(coro_function)) + self.assertIn('async coro_function', text) + + html = pydoc.HTMLDoc().document(coro_function) + self.assertIn( + 'async <a name="-coro_function"><strong>coro_function', + html) + + def test_async_generator_annotation(self): + async def an_async_generator(): + yield 1 + + text = pydoc.plain(pydoc.plaintext.document(an_async_generator)) + self.assertIn('async an_async_generator', text) + + html = pydoc.HTMLDoc().document(an_async_generator) + self.assertIn( + 'async <a name="-an_async_generator"><strong>an_async_generator', + html) + + @requires_docstrings + def test_html_for_https_links(self): + def a_fn_with_https_link(): + """a link https://localhost/""" + pass + + html = pydoc.HTMLDoc().document(a_fn_with_https_link) + self.assertIn( + '<a href="https://localhost/">https://localhost/</a>', + html + ) + + def test_module_none(self): + # Issue #128772 + from test.test_pydoc import module_none + pydoc.render_doc(module_none) + + +class PydocFodderTest(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def getsection(self, text, beginline, endline): + lines = text.splitlines() + beginindex, endindex = 0, None + if beginline is not None: + beginindex = lines.index(beginline) + if endline is not None: + endindex = lines.index(endline, beginindex) + return lines[beginindex:endindex] + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.TextDoc() + result = doc.docclass(cls) + result = clean_text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f' | Methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_method_alias = A_method(self)', lines) + self.assertIn(' | B_method_alias = B_method(self)', lines) + self.assertIn(' | A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn(' | A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | global_func_alias = global_func(x, y)', lines) + self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) + self.assertIn(' | __repr__(self, /) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(self, /)', lines) + else: + self.assertIn(' | count(self, object, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, object, /)', lines) + self.assertIn(' | __repr__(...) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(...)', lines) + + lines = self.getsection(result, f' | Static methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn(' | B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn(' | A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | sin(x, /)', lines) + else: + self.assertIn(' | get(...) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(...) method of builtins.dict instance', lines) + self.assertIn(' | sin(object, /)', lines) + + lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) + self.assertIn(' | B_classmethod(x)', lines) + self.assertIn(' | B_classmethod_alias = B_classmethod(x)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.HTMLDoc() + result = doc.docclass(cls) + result = html2text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f'Methods {where}:', '-'*70) + self.assertIn('A_method_alias = A_method(self)', lines) + self.assertIn('B_method_alias = B_method(self)', lines) + self.assertIn('A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn('A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('global_func_alias = global_func(x, y)', lines) + self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) + self.assertIn('__repr__(self, /) from builtins.object', lines) + self.assertIn('object_repr = __repr__(self, /)', lines) + else: + self.assertIn('count(self, object, /) from builtins.list', lines) + self.assertIn('list_count = count(self, object, /)', lines) + self.assertIn('__repr__(...) from builtins.object', lines) + self.assertIn('object_repr = __repr__(...)', lines) + + lines = self.getsection(result, f'Static methods {where}:', '-'*70) + self.assertIn('A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn('B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn('A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + + lines = self.getsection(result, f'Class methods {where}:', '-'*70) + self.assertIn('B_classmethod(x)', lines) + self.assertIn('B_classmethod_alias = B_classmethod(x)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_inherited_routines_in_class(self): + self.test_text_doc_routines_in_class(pydocfodder.D) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_inherited_routines_in_class(self): + self.test_html_doc_routines_in_class(pydocfodder.D) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_routines_in_module(self): + doc = pydoc.TextDoc() + result = doc.docmodule(pydocfodder) + result = clean_text(result) + lines = self.getsection(result, 'FUNCTIONS', 'FILE') + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn(' A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + else: + self.assertIn(' get(...) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(...) method of builtins.dict instance', lines) + + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + else: + self.assertIn(' count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(...) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_routines_in_module(self): + doc = pydoc.HTMLDoc() + result = doc.docmodule(pydocfodder) + result = html2text(result) + lines = self.getsection(result, ' Functions', None) + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn('A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + else: + self.assertIn(' get(...) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(...) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + else: + self.assertIn(' count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(...) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + + +@unittest.skipIf( + is_wasm32, + "Socket server not available on Emscripten/WASI." +) +class PydocServerTest(unittest.TestCase): + """Tests for pydoc._start_server""" + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def test_server(self): + # Minimal test that starts the server, checks that it works, then stops + # it and checks its cleanup. + def my_url_handler(url, content_type): + text = 'the URL sent was: (%s, %s)' % (url, content_type) + return text + + serverthread = pydoc._start_server( + my_url_handler, + hostname='localhost', + port=0, + ) + self.assertEqual(serverthread.error, None) + self.assertTrue(serverthread.serving) + self.addCleanup( + lambda: serverthread.stop() if serverthread.serving else None + ) + self.assertIn('localhost', serverthread.url) + + self.addCleanup(urlcleanup) + self.assertEqual( + b'the URL sent was: (/test, text/html)', + urlopen(urllib.parse.urljoin(serverthread.url, '/test')).read(), + ) + self.assertEqual( + b'the URL sent was: (/test.css, text/css)', + urlopen(urllib.parse.urljoin(serverthread.url, '/test.css')).read(), + ) + + serverthread.stop() + self.assertFalse(serverthread.serving) + self.assertIsNone(serverthread.docserver) + self.assertIsNone(serverthread.url) + + +class PydocUrlHandlerTest(PydocBaseTest): + """Tests for pydoc._url_handler""" + + @unittest.skip("TODO: RUSTPYTHON; Panic") + def test_content_type_err(self): + f = pydoc._url_handler + self.assertRaises(TypeError, f, 'A', '') + self.assertRaises(TypeError, f, 'B', 'foobar') + + def test_url_requests(self): + # Test for the correct title in the html pages returned. + # This tests the different parts of the URL handler without + # getting too picky about the exact html. + requests = [ + ("", "Pydoc: Index of Modules"), + ("get?key=", "Pydoc: Index of Modules"), + ("index", "Pydoc: Index of Modules"), + ("topics", "Pydoc: Topics"), + ("keywords", "Pydoc: Keywords"), + ("pydoc", "Pydoc: module pydoc"), + ("get?key=pydoc", "Pydoc: module pydoc"), + ("search?key=pydoc", "Pydoc: Search Results"), + ("topic?key=def", "Pydoc: KEYWORD def"), + ("topic?key=STRINGS", "Pydoc: TOPIC STRINGS"), + ("foobar", "Pydoc: Error - foobar"), + ] + + self.assertIs(sys.modules['pydoc'], pydoc) + try: + with self.restrict_walk_packages(): + for url, title in requests: + self.call_url_handler(url, title) + finally: + # Some requests reload the module and change sys.modules. + sys.modules['pydoc'] = pydoc + + +class TestHelper(unittest.TestCase): + def mock_interactive_session(self, inputs): + """ + Given a list of inputs, run an interactive help session. Returns a string + of what would be shown on screen. + """ + input_iter = iter(inputs) + + def mock_getline(prompt): + output.write(prompt) + next_input = next(input_iter) + output.write(next_input + os.linesep) + return next_input + + with captured_stdout() as output: + helper = pydoc.Helper(output=output) + with unittest.mock.patch.object(helper, "getline", mock_getline): + helper.interact() + + # handle different line endings across platforms consistently + return output.getvalue().strip().splitlines(keepends=False) + + def test_keywords(self): + self.assertEqual(sorted(pydoc.Helper.keywords), + sorted(keyword.kwlist)) + + def test_interact_empty_line_continues(self): + # gh-138568: test pressing Enter without input should continue in help session + self.assertEqual( + self.mock_interactive_session(["", " ", "quit"]), + ["help> ", "help> ", "help> quit"], + ) + + def test_interact_quit_commands_exit(self): + quit_commands = ["quit", "q", "exit"] + for quit_cmd in quit_commands: + with self.subTest(quit_command=quit_cmd): + self.assertEqual( + self.mock_interactive_session([quit_cmd]), + [f"help> {quit_cmd}"], + ) + + +class PydocWithMetaClasses(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_DynamicClassAttribute(self): + class Meta(type): + def __getattr__(self, name): + if name == 'ham': + return 'spam' + return super().__getattr__(name) + class DA(metaclass=Meta): + @types.DynamicClassAttribute + def ham(self): + return 'eggs' + expected_text_data_docstrings = tuple('\n | ' + s if s else '' + for s in expected_data_docstrings) + output = StringIO() + helper = pydoc.Helper(output=output) + helper(DA) + expected_text = expected_dynamicattribute_pattern % ( + (__name__,) + expected_text_data_docstrings[:2]) + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_virtualClassAttributeWithOneMeta(self): + class Meta(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'LIFE'] + def __getattr__(self, name): + if name =='LIFE': + return 42 + return super().__getattr(name) + class Class(metaclass=Meta): + pass + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class) + expected_text = expected_virtualattribute_pattern1 % __name__ + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_virtualClassAttributeWithTwoMeta(self): + class Meta1(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'one'] + def __getattr__(self, name): + if name =='one': + return 1 + return super().__getattr__(name) + class Meta2(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'two'] + def __getattr__(self, name): + if name =='two': + return 2 + return super().__getattr__(name) + class Meta3(Meta1, Meta2): + def __dir__(cls): + return list(sorted(set( + ['__class__', '__module__', '__name__', 'three'] + + Meta1.__dir__(cls) + Meta2.__dir__(cls)))) + def __getattr__(self, name): + if name =='three': + return 3 + return super().__getattr__(name) + class Class1(metaclass=Meta1): + pass + class Class2(Class1, metaclass=Meta3): + pass + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class1) + expected_text1 = expected_virtualattribute_pattern2 % __name__ + result1 = output.getvalue().strip() + self.assertEqual(expected_text1, result1) + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class2) + expected_text2 = expected_virtualattribute_pattern3 % __name__ + result2 = output.getvalue().strip() + self.assertEqual(expected_text2, result2) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing', 'here'] + class C(metaclass=M): + here = 'present!' + output = StringIO() + helper = pydoc.Helper(output=output) + helper(C) + expected_text = expected_missingattribute_pattern % __name__ + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + def test_resolve_false(self): + # Issue #23008: pydoc enum.{,Int}Enum failed + # because bool(enum.Enum) is False. + with captured_stdout() as help_io: + pydoc.help('enum.Enum') + helptext = help_io.getvalue() + self.assertIn('class Enum', helptext) + + +class TestInternalUtilities(unittest.TestCase): + + def setUp(self): + tmpdir = tempfile.TemporaryDirectory() + self.argv0dir = tmpdir.name + self.argv0 = os.path.join(tmpdir.name, "nonexistent") + self.addCleanup(tmpdir.cleanup) + self.abs_curdir = abs_curdir = os.getcwd() + self.curdir_spellings = ["", os.curdir, abs_curdir] + + def _get_revised_path(self, given_path, argv0=None): + # Checking that pydoc.cli() actually calls pydoc._get_revised_path() + # is handled via code review (at least for now). + if argv0 is None: + argv0 = self.argv0 + return pydoc._get_revised_path(given_path, argv0) + + def _get_starting_path(self): + # Get a copy of sys.path without the current directory. + clean_path = sys.path.copy() + for spelling in self.curdir_spellings: + for __ in range(clean_path.count(spelling)): + clean_path.remove(spelling) + return clean_path + + def test_sys_path_adjustment_adds_missing_curdir(self): + clean_path = self._get_starting_path() + expected_path = [self.abs_curdir] + clean_path + self.assertEqual(self._get_revised_path(clean_path), expected_path) + + def test_sys_path_adjustment_removes_argv0_dir(self): + clean_path = self._get_starting_path() + expected_path = [self.abs_curdir] + clean_path + leading_argv0dir = [self.argv0dir] + clean_path + self.assertEqual(self._get_revised_path(leading_argv0dir), expected_path) + trailing_argv0dir = clean_path + [self.argv0dir] + self.assertEqual(self._get_revised_path(trailing_argv0dir), expected_path) + + def test_sys_path_adjustment_protects_pydoc_dir(self): + def _get_revised_path(given_path): + return self._get_revised_path(given_path, argv0=pydoc.__file__) + clean_path = self._get_starting_path() + leading_argv0dir = [self.argv0dir] + clean_path + expected_path = [self.abs_curdir] + leading_argv0dir + self.assertEqual(_get_revised_path(leading_argv0dir), expected_path) + trailing_argv0dir = clean_path + [self.argv0dir] + expected_path = [self.abs_curdir] + trailing_argv0dir + self.assertEqual(_get_revised_path(trailing_argv0dir), expected_path) + + def test_sys_path_adjustment_when_curdir_already_included(self): + clean_path = self._get_starting_path() + for spelling in self.curdir_spellings: + with self.subTest(curdir_spelling=spelling): + # If curdir is already present, no alterations are made at all + leading_curdir = [spelling] + clean_path + self.assertIsNone(self._get_revised_path(leading_curdir)) + trailing_curdir = clean_path + [spelling] + self.assertIsNone(self._get_revised_path(trailing_curdir)) + leading_argv0dir = [self.argv0dir] + leading_curdir + self.assertIsNone(self._get_revised_path(leading_argv0dir)) + trailing_argv0dir = trailing_curdir + [self.argv0dir] + self.assertIsNone(self._get_revised_path(trailing_argv0dir)) + + +def setUpModule(): + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + unittest.addModuleCleanup(reap_children) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 80485cc74b9..d360e8cd65d 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -20,28 +20,24 @@ class SetAttributeTest(unittest.TestCase): def setUp(self): self.parser = expat.ParserCreate(namespace_separator='!') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffer_text(self): self.assertIs(self.parser.buffer_text, False) for x in 0, 1, 2, 0: self.parser.buffer_text = x self.assertIs(self.parser.buffer_text, bool(x)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespace_prefixes(self): self.assertIs(self.parser.namespace_prefixes, False) for x in 0, 1, 2, 0: self.parser.namespace_prefixes = x self.assertIs(self.parser.namespace_prefixes, bool(x)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_ordered_attributes(self): self.assertIs(self.parser.ordered_attributes, False) for x in 0, 1, 2, 0: self.parser.ordered_attributes = x self.assertIs(self.parser.ordered_attributes, bool(x)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_specified_attributes(self): self.assertIs(self.parser.specified_attributes, False) for x in 0, 1, 2, 0: @@ -231,7 +227,6 @@ def _verify_parse_output(self, operations): for operation, expected_operation in zip(operations, expected_operations): self.assertEqual(operation, expected_operation) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_bytes(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -244,7 +239,6 @@ def test_parse_bytes(self): # Issue #6697. self.assertRaises(AttributeError, getattr, parser, '\uD800') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_str(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -255,7 +249,6 @@ def test_parse_str(self): operations = out.out self._verify_parse_output(operations) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_file(self): # Try parsing a file out = self.Outputter() @@ -282,7 +275,6 @@ def test_parse_again(self): expat.errors.XML_ERROR_FINISHED) class NamespaceSeparatorTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_legal(self): # Tests that make sure we get errors when the namespace_separator value # is illegal, and that we don't for good values: @@ -415,14 +407,12 @@ def test2(self): self.assertEqual(self.stuff, ["1<2> \n 3"], "buffered text not properly collapsed") - @unittest.expectedFailure # TODO: RUSTPYTHON def test3(self): self.setHandlers(["StartElementHandler"]) self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"], "buffered text not properly split") - @unittest.expectedFailure # TODO: RUSTPYTHON def test4(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.CharacterDataHandler = None @@ -430,14 +420,12 @@ def test4(self): self.assertEqual(self.stuff, ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test5(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test6(self): self.setHandlers(["CommentHandler", "EndElementHandler", "StartElementHandler"]) @@ -535,7 +523,6 @@ def check_pos(self, event): 'Expected position %s, got position %s' %(pos, expected)) self.upto += 1 - @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): self.parser = expat.ParserCreate() self.parser.StartElementHandler = self.StartElementHandler diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 93cbe1fe230..c855fb8fe2b 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -2,12 +2,11 @@ # to ensure the Queue locks remain stable. import itertools import random -import sys import threading import time import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper @@ -964,33 +963,33 @@ def test_order(self): # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) - def test_many_threads(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads(self, size): # Test multiple concurrent put() and get() - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, self.feed, self.consume) + results = self.run_threads(size, q, inputs, self.feed, self.consume) # Multiple consumers without synchronization append the # results in random order self.assertEqual(sorted(results), inputs) - def test_many_threads_nonblock(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_nonblock(self, size): # Test multiple concurrent put() and get(block=False) - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_nonblock) self.assertEqual(sorted(results), inputs) - def test_many_threads_timeout(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_timeout(self, size): # Test multiple concurrent put() and get(timeout=...) - N = 50 q = self.q inputs = list(range(1000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 53fce0a501d..ca253585e1a 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,7 +185,6 @@ def test_class_cause(self): else: self.fail("No exception raised") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'classmethod' object is not callable def test_class_cause_nonexception_result(self): # See https://github.com/python/cpython/issues/140530. class ConstructMortal(BaseException): @@ -244,7 +243,6 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError - @unittest.expectedFailure # TODO: RUSTPYTHON def test_attrs(self): try: self.raiser() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 3c0b6af3c6b..b415c5907fa 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,12 +1,11 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, - check_disallow_instantiation, is_emscripten, is_wasi, - SHORT_TIMEOUT, requires_resource) + check_disallow_instantiation, linked_to_musl, + warnings_helper, SHORT_TIMEOUT, Stopwatch, requires_resource) import locale import re import string import sys -import time import unittest import warnings from re import Scanner @@ -14,7 +13,7 @@ # some platforms lack working multiprocessing try: - import _multiprocessing + import _multiprocessing # noqa: F401 except ImportError: multiprocessing = None else: @@ -47,7 +46,7 @@ def recurse(actual, expect): recurse(actual, expect) def checkPatternError(self, pattern, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(pattern) with self.subTest(pattern=pattern): err = cm.exception @@ -56,7 +55,7 @@ def checkPatternError(self, pattern, errmsg, pos=None): self.assertEqual(err.pos, pos) def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.sub(pattern, repl, string) with self.subTest(pattern=pattern, repl=repl): err = cm.exception @@ -64,8 +63,10 @@ def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): if pos is not None: self.assertEqual(err.pos, pos) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_error_is_PatternError_alias(self): + assert re.error is re.PatternError + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keep_buffer(self): # See bug 14212 b = bytearray(b'x') @@ -129,8 +130,10 @@ def test_basic_re_sub(self): self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), '9.3 -3 24x100y') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), - '9.3 -3 23x99y') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + '9.3 -3 23x99y') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', count=3), '9.3 -3 23x99y') @@ -154,7 +157,7 @@ def test_basic_re_sub(self): (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': with self.subTest(c): - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') @@ -237,9 +240,42 @@ def test_sub_template_numeric_escape(self): def test_qualified_re_sub(self): self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') - self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa') + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'count'"): + re.sub('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'flags'"): + re.sub('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"sub\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.sub('a', 'b', 'aaaaa', 1, 0, 0) + + def test_misuse_flags(self): + with self.assertWarns(DeprecationWarning) as w: + result = re.sub('a', 'b', 'aaaaa', re.I) + self.assertEqual(result, re.sub('a', 'b', 'aaaaa', count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.subn("b*", "x", "xyz", re.I) + self.assertEqual(result, re.subn("b*", "x", "xyz", count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.split(":", ":a:b::c", re.I) + self.assertEqual(result, re.split(":", ":a:b::c", maxsplit=int(re.I))) + self.assertEqual(str(w.warning), + "'maxsplit' is passed as positional argument") + self.assertEqual(w.filename, __file__) + def test_bug_114660(self): self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), 'hello there') @@ -346,9 +382,22 @@ def test_re_subn(self): self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) - self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + self.assertEqual(w.filename, __file__) self.assertEqual(re.subn("b*", "x", "xyz", count=2), ('xxxyz', 2)) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'count'"): + re.subn('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'flags'"): + re.subn('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"subn\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.subn('a', 'b', 'aaaaa', 1, 0, 0) + def test_re_split(self): for string in ":a:b::c", S(":a:b::c"): self.assertTypedEqual(re.split(":", string), @@ -403,7 +452,9 @@ def test_re_split(self): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) def test_qualified_re_split(self): - self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + self.assertEqual(w.filename, __file__) self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), @@ -413,6 +464,17 @@ def test_qualified_re_split(self): self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), ['', ':', '', '', 'a:b::c']) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'maxsplit'"): + re.split(":", ":a:b::c", 2, maxsplit=2) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'flags'"): + re.split(":", ":a:b::c", 2, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"split\(\) takes from 2 to 4 positional arguments but 5 " + r"were given"): + re.split(":", ":a:b::c", 2, 0, 0) + def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) for string in "a:b::c:::d", S("a:b::c:::d"): @@ -558,6 +620,7 @@ def test_re_fullmatch(self): self.assertEqual(re.fullmatch(r"a.*?b", "axxb").span(), (0, 4)) self.assertIsNone(re.fullmatch(r"a+", "ab")) self.assertIsNone(re.fullmatch(r"abc$", "abc\n")) + self.assertIsNone(re.fullmatch(r"abc\z", "abc\n")) self.assertIsNone(re.fullmatch(r"abc\Z", "abc\n")) self.assertIsNone(re.fullmatch(r"(?m)abc$", "abc\n")) self.assertEqual(re.fullmatch(r"ab(?=c)cd", "abcd").span(), (0, 4)) @@ -663,8 +726,7 @@ def test_groupdict(self): 'first second').groupdict(), {'first':'first', 'second':'second'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expand(self): self.assertEqual(re.match("(?P<first>first) (?P<second>second)", "first second") @@ -711,8 +773,7 @@ def test_repeat_minmax(self): self.checkPatternError(r'x{2,1}', 'min repeat greater than max repeat', 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_getattr(self): self.assertEqual(re.compile("(?i)(a)(b)").pattern, "(?i)(a)(b)") self.assertEqual(re.compile("(?i)(a)(b)").flags, re.I | re.U) @@ -745,6 +806,8 @@ def test_special_escapes(self): self.assertEqual(re.search(r"\B(b.)\B", "abc bcd bc abxd", re.ASCII).group(1), "bx") self.assertEqual(re.search(r"^abc$", "\nabc\n", re.M).group(0), "abc") + self.assertEqual(re.search(r"^\Aabc\z$", "abc", re.M).group(0), "abc") + self.assertIsNone(re.search(r"^\Aabc\z$", "\nabc\n", re.M)) self.assertEqual(re.search(r"^\Aabc\Z$", "abc", re.M).group(0), "abc") self.assertIsNone(re.search(r"^\Aabc\Z$", "\nabc\n", re.M)) self.assertEqual(re.search(br"\b(b.)\b", @@ -756,6 +819,8 @@ def test_special_escapes(self): self.assertEqual(re.search(br"\B(b.)\B", b"abc bcd bc abxd", re.LOCALE).group(1), b"bx") self.assertEqual(re.search(br"^abc$", b"\nabc\n", re.M).group(0), b"abc") + self.assertEqual(re.search(br"^\Aabc\z$", b"abc", re.M).group(0), b"abc") + self.assertIsNone(re.search(br"^\Aabc\z$", b"\nabc\n", re.M)) self.assertEqual(re.search(br"^\Aabc\Z$", b"abc", re.M).group(0), b"abc") self.assertIsNone(re.search(br"^\Aabc\Z$", b"\nabc\n", re.M)) self.assertEqual(re.search(r"\d\D\w\W\s\S", @@ -779,15 +844,13 @@ def test_other_escapes(self): self.assertEqual(re.match(r"[\^a]+", 'a^').group(), 'a^') self.assertIsNone(re.match(r"[\^a]+", 'b')) re.purge() # for warnings - for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': + for c in 'ceghijklmopqyCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - self.assertRaises(re.error, re.compile, '\\%c' % c) + self.assertRaises(re.PatternError, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - self.assertRaises(re.error, re.compile, '[\\%c]' % c) + self.assertRaises(re.PatternError, re.compile, '[\\%c]' % c) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_unicode_escapes(self): # test individual Unicode named escapes self.assertTrue(re.match(r'\N{LESS-THAN SIGN}', '<')) @@ -828,31 +891,136 @@ def test_named_unicode_escapes(self): self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) - def test_string_boundaries(self): + # TODO: RUSTPYTHON; re.search(r"\B", "") now returns a match in CPython 3.14 + @unittest.expectedFailure + def test_word_boundaries(self): # See http://bugs.python.org/issue10713 - self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), - "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc", re.ASCII).group(1), "abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc").group(1), b"abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc", re.LOCALE).group(1), b"abc") + self.assertEqual(re.search(r"\b(ьюя)\b", "ьюя").group(1), "ьюя") + self.assertIsNone(re.search(r"\b(ьюя)\b", "ьюя", re.ASCII)) + # There's a word boundary between a word and a non-word. + self.assertTrue(re.match(r".\b", "a=")) + self.assertTrue(re.match(r".\b", "a=", re.ASCII)) + self.assertTrue(re.match(br".\b", b"a=")) + self.assertTrue(re.match(br".\b", b"a=", re.LOCALE)) + self.assertTrue(re.match(r".\b", "я=")) + self.assertIsNone(re.match(r".\b", "я=", re.ASCII)) + # There's a word boundary between a non-word and a word. + self.assertTrue(re.match(r".\b", "=a")) + self.assertTrue(re.match(r".\b", "=a", re.ASCII)) + self.assertTrue(re.match(br".\b", b"=a")) + self.assertTrue(re.match(br".\b", b"=a", re.LOCALE)) + self.assertTrue(re.match(r".\b", "=я")) + self.assertIsNone(re.match(r".\b", "=я", re.ASCII)) + # There is no word boundary inside a word. + self.assertIsNone(re.match(r".\b", "ab")) + self.assertIsNone(re.match(r".\b", "ab", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"ab")) + self.assertIsNone(re.match(br".\b", b"ab", re.LOCALE)) + self.assertIsNone(re.match(r".\b", "юя")) + self.assertIsNone(re.match(r".\b", "юя", re.ASCII)) + # There is no word boundary between a non-word characters. + self.assertIsNone(re.match(r".\b", "=-")) + self.assertIsNone(re.match(r".\b", "=-", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"=-")) + self.assertIsNone(re.match(br".\b", b"=-", re.LOCALE)) + # There is no non-boundary match between a word and a non-word. + self.assertIsNone(re.match(r".\B", "a=")) + self.assertIsNone(re.match(r".\B", "a=", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"a=")) + self.assertIsNone(re.match(br".\B", b"a=", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "я=")) + self.assertTrue(re.match(r".\B", "я=", re.ASCII)) + # There is no non-boundary match between a non-word and a word. + self.assertIsNone(re.match(r".\B", "=a")) + self.assertIsNone(re.match(r".\B", "=a", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"=a")) + self.assertIsNone(re.match(br".\B", b"=a", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "=я")) + self.assertTrue(re.match(r".\B", "=я", re.ASCII)) + # There's a non-boundary match inside a word. + self.assertTrue(re.match(r".\B", "ab")) + self.assertTrue(re.match(r".\B", "ab", re.ASCII)) + self.assertTrue(re.match(br".\B", b"ab")) + self.assertTrue(re.match(br".\B", b"ab", re.LOCALE)) + self.assertTrue(re.match(r".\B", "юя")) + self.assertTrue(re.match(r".\B", "юя", re.ASCII)) + # There's a non-boundary match between a non-word characters. + self.assertTrue(re.match(r".\B", "=-")) + self.assertTrue(re.match(r".\B", "=-", re.ASCII)) + self.assertTrue(re.match(br".\B", b"=-")) + self.assertTrue(re.match(br".\B", b"=-", re.LOCALE)) # There's a word boundary at the start of a string. self.assertTrue(re.match(r"\b", "abc")) + self.assertTrue(re.match(r"\b", "abc", re.ASCII)) + self.assertTrue(re.match(br"\b", b"abc")) + self.assertTrue(re.match(br"\b", b"abc", re.LOCALE)) + self.assertTrue(re.match(r"\b", "ьюя")) + self.assertIsNone(re.match(r"\b", "ьюя", re.ASCII)) + # There's a word boundary at the end of a string. + self.assertTrue(re.fullmatch(r".+\b", "abc")) + self.assertTrue(re.fullmatch(r".+\b", "abc", re.ASCII)) + self.assertTrue(re.fullmatch(br".+\b", b"abc")) + self.assertTrue(re.fullmatch(br".+\b", b"abc", re.LOCALE)) + self.assertTrue(re.fullmatch(r".+\b", "ьюя")) + self.assertIsNone(re.search(r"\b", "ьюя", re.ASCII)) # A non-empty string includes a non-boundary zero-length match. - self.assertTrue(re.search(r"\B", "abc")) + self.assertEqual(re.search(r"\B", "abc").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "abc", re.ASCII).span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc").span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc", re.LOCALE).span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя", re.ASCII).span(), (0, 0)) # There is no non-boundary match at the start of a string. - self.assertFalse(re.match(r"\B", "abc")) - # However, an empty string contains no word boundaries, and also no - # non-boundaries. - self.assertIsNone(re.search(r"\B", "")) - # This one is questionable and different from the perlre behaviour, - # but describes current behavior. + self.assertIsNone(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc", re.ASCII)) + self.assertIsNone(re.match(br"\B", b"abc")) + self.assertIsNone(re.match(br"\B", b"abc", re.LOCALE)) + self.assertIsNone(re.match(r"\B", "ьюя")) + self.assertTrue(re.match(r"\B", "ьюя", re.ASCII)) + # There is no non-boundary match at the end of a string. + self.assertIsNone(re.fullmatch(r".+\B", "abc")) + self.assertIsNone(re.fullmatch(r".+\B", "abc", re.ASCII)) + self.assertIsNone(re.fullmatch(br".+\B", b"abc")) + self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) + self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) + self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) + # However, an empty string contains no word boundaries. self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\b", "", re.ASCII)) + self.assertIsNone(re.search(br"\b", b"")) + self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) + self.assertTrue(re.search(r"\B", "")) + self.assertTrue(re.search(r"\B", "", re.ASCII)) + self.assertTrue(re.search(br"\B", b"")) + self.assertTrue(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\b", "a", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\b", b"a")), 2) + self.assertEqual(len(re.findall(br"\b", b"a", re.LOCALE)), 2) self.assertEqual(len(re.findall(r"\B", "a")), 0) + self.assertEqual(len(re.findall(r"\B", "a", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\B", b"a")), 0) + self.assertEqual(len(re.findall(br"\B", b"a", re.LOCALE)), 0) # If there are no words, there are no boundaries self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) + self.assertEqual(len(re.findall(r"\B", " ", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\B", b" ")), 2) + self.assertEqual(len(re.findall(br"\B", b" ", re.LOCALE)), 2) def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", @@ -917,14 +1085,14 @@ def test_lookbehind(self): self.assertIsNone(re.match(r'(?:(a)|(x))b(?<=(?(1)c|x))c', 'abc')) self.assertTrue(re.match(r'(?:(a)|(x))b(?<=(?(1)b|x))c', 'abc')) # Group used before defined. - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?(2)b|x))(c)') self.assertIsNone(re.match(r'(a)b(?<=(?(1)c|x))(c)', 'abc')) self.assertTrue(re.match(r'(a)b(?<=(?(1)b|x))(c)', 'abc')) # Group defined in the same lookbehind pattern - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)\2)(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?P<a>.)(?P=a))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)\2)(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?P<a>.)(?P=a))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") @@ -975,6 +1143,39 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'a', re.I)) self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I)) + + self.assertTrue(re.match(br'[19A]', b'A', re.I)) + self.assertTrue(re.match(br'[19a]', b'a', re.I)) + self.assertTrue(re.match(br'[19a]', b'A', re.I)) + self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19A]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19A]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xc7]', '\xe7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xe7]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0400]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0450]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010400]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010428]', '\U00010400', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I|re.A)) # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' @@ -1011,8 +1212,10 @@ def test_ignore_case_range(self): self.assertTrue(re.match(br'[9-a]', b'_', re.I)) self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xe7', re.I)) self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xc7', re.I)) self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) @@ -1023,6 +1226,26 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0430-\u045f]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0400-\u042f]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I|re.A)) + + self.assertTrue(re.match(r'[N-\x7f]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\x7f]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\uffff]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\uffff]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\U00010000]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\U00010000]', 'Z', re.I|re.A)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) @@ -1060,47 +1283,76 @@ def test_not_literal(self): def test_possible_set_operations(self): s = bytes(range(128)).decode() - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[0-9--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('-./0123456789')) + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: + self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) + self.assertEqual(w.filename, __file__) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[%--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-1")) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: p = re.compile(r'[%--]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-")) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[0-9&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: + self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) + self.assertEqual(w.filename, __file__) + + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[\d&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[0-9||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[\d||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[\d~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[0-9]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789[]')) + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) + self.assertEqual(w.filename, __file__) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[:digit:]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list(':[]dgit')) def test_search_coverage(self): @@ -1173,10 +1425,9 @@ def test_pickling(self): newpat = pickle.loads(pickled) self.assertEqual(newpat, oldpat) # current pickle expects the _compile() reconstructor in re module - from re import _compile + from re import _compile # noqa: F401 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_copying(self): import copy p = re.compile(r'(?P<int>\d+)(?:\.(?P<frac>\d*))?') @@ -1267,8 +1518,8 @@ def test_sre_byte_literals(self): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - self.assertRaises(re.error, re.compile, br"\u1234") - self.assertRaises(re.error, re.compile, br"\U00012345") + self.assertRaises(re.PatternError, re.compile, br"\u1234") + self.assertRaises(re.PatternError, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1290,8 +1541,8 @@ def test_sre_byte_class_literals(self): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - self.assertRaises(re.error, re.compile, br"[\u1234]") - self.assertRaises(re.error, re.compile, br"[\U00012345]") + self.assertRaises(re.PatternError, re.compile, br"[\u1234]") + self.assertRaises(re.PatternError, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1484,8 +1735,7 @@ def test_bug_817234(self): self.assertEqual(next(iter).span(), (4, 4)) self.assertRaises(StopIteration, next, iter) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_6561(self): # '\d' should match characters in Unicode category 'Nd' # (Number, Decimal Digit), but not those in 'Nl' (Number, @@ -1507,10 +1757,12 @@ def test_bug_6561(self): for x in not_decimal_digits: self.assertIsNone(re.match(r'^\d$', x)) + @unittest.expectedFailure # TODO: RUSTPYTHON; a = array.array(typecode)\n ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') def test_empty_array(self): # SF buf 1647541 import array - for typecode in 'bBuhHiIlLfd': + for typecode in 'bBhuwHiIlLfd': a = array.array(typecode) self.assertIsNone(re.compile(b"bla").match(a)) self.assertEqual(re.compile(b"").match(a).groups(), ()) @@ -1625,11 +1877,11 @@ def test_ascii_and_unicode_flag(self): self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) - self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(re.PatternError, re.compile, br'(?u)\w') self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) - self.assertRaises(re.error, re.compile, r'(?au)\w') + self.assertRaises(re.PatternError, re.compile, r'(?au)\w') def test_locale_flag(self): enc = locale.getpreferredencoding() @@ -1670,11 +1922,11 @@ def test_locale_flag(self): self.assertIsNone(pat.match(bletter)) # Incompatibilities self.assertRaises(ValueError, re.compile, '', re.LOCALE) - self.assertRaises(re.error, re.compile, '(?L)') + self.assertRaises(re.PatternError, re.compile, '(?L)') self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) - self.assertRaises(re.error, re.compile, b'(?aL)') + self.assertRaises(re.PatternError, re.compile, b'(?aL)') def test_scoped_flags(self): self.assertTrue(re.match(r'(?i:a)b', 'Ab')) @@ -1854,8 +2106,6 @@ def test_issue17998(self): self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), [b'xyz'], msg=pattern) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_match_repr(self): for string in '[abracadabra]', S('[abracadabra]'): m = re.search(r'(.+)(.*?)\1', string) @@ -1935,10 +2185,10 @@ def test_bug_20998(self): # with ignore case. self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_caching(self): # Issue #22410 oldlocale = locale.setlocale(locale.LC_CTYPE) @@ -1975,10 +2225,9 @@ def check_en_US_utf8(self): self.assertIsNone(re.match(b'(?Li)\xc5', b'\xe5')) self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5')) - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_compiled(self): oldlocale = locale.setlocale(locale.LC_CTYPE) self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) @@ -2012,7 +2261,7 @@ def test_locale_compiled(self): self.assertIsNone(p4.match(b'\xc5\xc5')) def test_error(self): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile('(\u20ac))') err = cm.exception self.assertIsInstance(err.pattern, str) @@ -2024,14 +2273,14 @@ def test_error(self): self.assertIn(' at position 3', str(err)) self.assertNotIn(' at position 3', err.msg) # Bytes pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(b'(\xa4))') err = cm.exception self.assertIsInstance(err.pattern, bytes) self.assertEqual(err.pattern, b'(\xa4))') self.assertEqual(err.pos, 3) # Multiline pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(""" ( abc @@ -2231,24 +2480,24 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) - @unittest.skip("TODO: RUSTPYTHON: flaky, improve perf") + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @unittest.skip('TODO: RUSTPYTHON; flaky, improve perf') @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with Stopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.2) + self.assertLess(stopwatch.seconds, 0.1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_possessive_quantifiers(self): """Test Possessive Quantifiers Test quantifiers of the form @+ for some repetition operator @, @@ -2288,8 +2537,7 @@ def test_possessive_quantifiers(self): self.assertIsNone(re.match("^x{}+$", "xxx")) self.assertTrue(re.match("^x{}+$", "x{}")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fullmatch_possessive_quantifiers(self): self.assertTrue(re.fullmatch(r'a++', 'a')) self.assertTrue(re.fullmatch(r'a*+', 'a')) @@ -2342,8 +2590,7 @@ def test_atomic_grouping(self): self.assertIsNone(re.match(r'(?>x)++x', 'xxx')) self.assertIsNone(re.match(r'(?>x++)x', 'xxx')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fullmatch_atomic_grouping(self): self.assertTrue(re.fullmatch(r'(?>a+)', 'a')) self.assertTrue(re.fullmatch(r'(?>a*)', 'a')) @@ -2382,39 +2629,12 @@ def test_findall_atomic_grouping(self): self.assertEqual(re.findall(r'(?>(?:ab)?)', 'ababc'), ['ab', 'ab', '', '']) self.assertEqual(re.findall(r'(?>(?:ab){1,3})', 'ababc'), ['abab']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_gh91616(self): - self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer - self.assertTrue(re.fullmatch(r'(?s:(?=(?P<g0>.*?\.))(?P=g0).*)\Z', "a.txt")) + self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\z', "a.txt")) # reproducer + self.assertTrue(re.fullmatch(r'(?s:(?=(?P<g0>.*?\.))(?P=g0).*)\z', "a.txt")) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_template_function_and_flag_is_deprecated(self): - with self.assertWarns(DeprecationWarning) as cm: - template_re1 = re.template(r'a') - self.assertIn('re.template()', str(cm.warning)) - self.assertIn('is deprecated', str(cm.warning)) - self.assertIn('function', str(cm.warning)) - self.assertNotIn('flag', str(cm.warning)) - - with self.assertWarns(DeprecationWarning) as cm: - # we deliberately use more flags here to test that that still - # triggers the warning - # if paranoid, we could test multiple different combinations, - # but it's probably not worth it - template_re2 = re.compile(r'a', flags=re.TEMPLATE|re.UNICODE) - self.assertIn('re.TEMPLATE', str(cm.warning)) - self.assertIn('is deprecated', str(cm.warning)) - self.assertIn('flag', str(cm.warning)) - self.assertNotIn('function', str(cm.warning)) - - # while deprecated, is should still function - self.assertEqual(template_re1, template_re2) - self.assertTrue(template_re1.match('ahoy')) - self.assertFalse(template_re1.match('nope')) - - def test_bug_gh106052(self): + def test_bug_gh100061(self): # gh-100061 self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) @@ -2434,8 +2654,13 @@ def test_bug_gh106052(self): self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) - # TODO: RUSTPYTHON - @unittest.skipUnless(sys.platform == 'linux', 'multiprocessing related issue') + @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x'))\n AssertionError: Tuples differ: ('x', 'x') != ('z', 'x') + def test_bug_gh101955(self): + # Possessive quantifier with nested alternative with capture groups + self.assertEqual(re.match('((x)|y|z)*+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3,}+', 'xyz').groups(), ('z', 'x')) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2456,6 +2681,54 @@ def test_regression_gh94675(self): p.terminate() p.join() + def test_fail(self): + self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + + def test_character_set_any(self): + # The union of complementary character sets matches any character + # and is equivalent to "(?s:.)". + s = '1x\n' + for p in r'[\s\S]', r'[\d\D]', r'[\w\W]', r'[\S\s]', r'\s|\S': + with self.subTest(pattern=p): + self.assertEqual(re.findall(p, s), list(s)) + self.assertEqual(re.fullmatch('(?:' + p + ')+', s).group(), s) + + def test_character_set_none(self): + # Negation of the union of complementary character sets does not match + # any character. + s = '1x\n' + for p in r'[^\s\S]', r'[^\d\D]', r'[^\w\W]', r'[^\S\s]': + with self.subTest(pattern=p): + self.assertIsNone(re.search(p, s)) + self.assertIsNone(re.search('(?s:.)' + p, s)) + + def check_interrupt(self, pattern, string, maxcount): + class Interrupt(Exception): + pass + p = re.compile(pattern) + for n in range(maxcount): + try: + p._fail_after(n, Interrupt) + p.match(string) + return n + except Interrupt: + pass + finally: + p._fail_after(-1, None) + + @unittest.skipUnless(hasattr(re.Pattern, '_fail_after'), 'requires debug build') + def test_memory_leaks(self): + self.check_interrupt(r'(.)*:', 'abc:', 100) + self.check_interrupt(r'([^:])*?:', 'abc:', 100) + self.check_interrupt(r'([^:])*+:', 'abc:', 100) + self.check_interrupt(r'(.){2,4}:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}?:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}+:', 'abc:', 100) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_template_function_and_flag_is_deprecated(self): + return super().test_template_function_and_flag_is_deprecated() + def get_debug_out(pat): with captured_stdout() as out: @@ -2590,8 +2863,7 @@ def test_inline_flags(self): self.check('(?i)pattern', "re.compile('(?i)pattern', re.IGNORECASE)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unknown_flags(self): self.check_flags('random pattern', 0x123000, "re.compile('random pattern', 0x123000)") @@ -2620,14 +2892,12 @@ def test_long_pattern(self): pattern = 'Very %spattern' % ('long ' * 1000) r = repr(re.compile(pattern)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertStartsWith(r, "re.compile('Very long long lon") r = repr(re.compile(pattern, re.I)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") - self.assertEqual(r[-16:], ", re.IGNORECASE)") + self.assertStartsWith(r, "re.compile('Very long long lon") + self.assertEndsWith(r, ", re.IGNORECASE)") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_flags_repr(self): self.assertEqual(repr(re.I), "re.IGNORECASE") self.assertEqual(repr(re.I|re.S|re.X), @@ -2636,11 +2906,11 @@ def test_flags_repr(self): "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000") self.assertEqual( repr(~re.I), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.TEMPLATE|re.DEBUG") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.DEBUG|0x1") self.assertEqual(repr(~(re.I|re.S|re.X)), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0x1") self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG|0xffe00") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0xffe01") class ImplementationTest(unittest.TestCase): @@ -2681,8 +2951,7 @@ def test_disallow_instantiation(self): pat = re.compile("") check_disallow_instantiation(self, type(pat.scanner(""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_deprecated_modules(self): deprecated = { 'sre_compile': ['compile', 'error', @@ -2707,7 +2976,7 @@ def test_deprecated_modules(self): self.assertEqual(mod.__name__, name) self.assertEqual(mod.__package__, '') for attr in deprecated[name]: - self.assertTrue(hasattr(mod, attr)) + self.assertHasAttr(mod, attr) del sys.modules[name] @cpython_only @@ -2813,7 +3082,7 @@ def test_re_tests(self): with self.subTest(pattern=pattern, string=s): if outcome == SYNTAX_ERROR: # Expected a syntax error - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): re.compile(pattern) continue diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d9ae2f35487..82939108b12 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,29 +4,61 @@ Note: test_regrtest cannot be run twice in parallel. """ +import _colorize import contextlib -import faulthandler +import dataclasses import glob import io +import locale import os.path import platform +import random import re +import shlex +import signal import subprocess import sys import sysconfig import tempfile import textwrap import unittest -from test import libregrtest +import unittest.mock +from xml.etree import ElementTree + from test import support +from test.support import import_helper from test.support import os_helper +from test.libregrtest import cmdline +from test.libregrtest import main +from test.libregrtest import setup from test.libregrtest import utils +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test +from test.libregrtest.result import TestStats +from test.libregrtest.utils import normalize_test_name +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") -Py_DEBUG = hasattr(sys, 'gettotalrefcount') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +RESULT_REGEX = ( + 'passed', + 'failed', + 'skipped', + 'interrupted', + 'env changed', + 'timed out', + 'ran no tests', + 'worker non-zero exit code', +) +RESULT_REGEX = fr'(?:{"|".join(RESULT_REGEX)})' + +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT, raise_signal @@ -43,9 +75,13 @@ class ParseArgsTestCase(unittest.TestCase): Test regrtest's argument parsing, function _parse_args(). """ + @staticmethod + def parse_args(args): + return cmdline._parse_args(args) + def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - libregrtest._parse_args(args) + self.parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -53,94 +89,130 @@ def test_help(self): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - libregrtest._parse_args([opt]) + self.parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) - @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), - "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = libregrtest._parse_args(['--timeout', '4.2']) + ns = self.parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) + + # negative, zero and empty string are treated as "no timeout" + for value in ('-1', '0', ''): + with self.subTest(value=value): + ns = self.parse_args([f'--timeout={value}']) + self.assertEqual(ns.timeout, None) + self.checkError(['--timeout'], 'expected one argument') - self.checkError(['--timeout', 'foo'], 'invalid float value') + self.checkError(['--timeout', 'foo'], 'invalid timeout value:') def test_wait(self): - ns = libregrtest._parse_args(['--wait']) + ns = self.parse_args(['--wait']) self.assertTrue(ns.wait) - def test_worker_args(self): - ns = libregrtest._parse_args(['--worker-args', '[[], {}]']) - self.assertEqual(ns.worker_args, '[[], {}]') - self.checkError(['--worker-args'], 'expected one argument') - def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = libregrtest._parse_args(['-v']) + ns = self.parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['-vvv']) + ns = self.parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['--verbose'] * 3) + ns = self.parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args([]) + ns = self.parse_args([]) self.assertEqual(ns.verbose, 0) - def test_verbose2(self): - for opt in '-w', '--verbose2': + def test_rerun(self): + for opt in '-w', '--rerun', '--verbose2': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) - self.assertTrue(ns.verbose2) + ns = self.parse_args([opt]) + self.assertTrue(ns.rerun) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slowest(self): for opt in '-o', '--slowest': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = libregrtest._parse_args(['--header']) + ns = self.parse_args(['--header']) self.assertTrue(ns.header) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.randomize) + with os_helper.EnvironmentVarGuard() as env: + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertFalse(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) + + def test_no_randomize(self): + ns = self.parse_args([]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize"]) + self.assertIs(ns.randomize, True) + + ns = self.parse_args(["--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize", "--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--no-randomize", "--randomize"]) + self.assertIs(ns.randomize, False) + def test_randseed(self): - ns = libregrtest._parse_args(['--randseed', '12345']) + ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') self.checkError(['--randseed', 'foo'], 'invalid int value') + ns = self.parse_args(['--randseed', '12345', '--no-randomize']) + self.assertEqual(ns.random_seed, 12345) + self.assertFalse(ns.randomize) + def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -148,46 +220,37 @@ def test_fromfile(self): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + ns = self.parse_args([opt, 'pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + ns = self.parse_args([opt, 'pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = libregrtest._parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -195,73 +258,76 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) + self.assertEqual(ns.match_tests, + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '-v']) + ns = self.parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = libregrtest._parse_args([opt, '-W']) + ns = self.parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'gui,network']) + ns = self.parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = libregrtest._parse_args([opt, 'gui,none,network']) + ns = self.parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.ALL_RESOURCES) + expected = list(cmdline.ALL_RESOURCES) expected.remove('gui') - ns = libregrtest._parse_args([opt, 'all,-gui']) + ns = self.parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') # all + a resource not part of "all" - ns = libregrtest._parse_args([opt, 'all,tzdata']) + ns = self.parse_args([opt, 'all,tzdata']) self.assertEqual(ns.use_resources, - list(libregrtest.ALL_RESOURCES) + ['tzdata']) + list(cmdline.ALL_RESOURCES) + ['tzdata']) # test another resource which is not part of "all" - ns = libregrtest._parse_args([opt, 'extralargefile']) + ns = self.parse_args([opt, 'extralargefile']) self.assertEqual(ns.use_resources, ['extralargefile']) def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '4G']) + ns = self.parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = libregrtest._parse_args(['--testdir', 'foo']) + ns = self.parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, ':']) + ns = self.parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:']) + ns = self.parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, ':3']) + ns = self.parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:3:leaks.log']) + ns = self.parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -272,23 +338,33 @@ def test_huntrleaks(self): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '2']) + ns = self.parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') - self.checkError([opt, '2', '-T'], "don't go together") - self.checkError([opt, '0', '-T'], "don't go together") - def test_coverage(self): + def test_coverage_sequential(self): + for opt in '-T', '--coverage': + with self.subTest(opt=opt): + with support.captured_stderr() as stderr: + ns = self.parse_args([opt]) + self.assertTrue(ns.trace) + self.assertIn( + "collecting coverage without -j is imprecise", + stderr.getvalue(), + ) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_coverage_mp(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt, '-j1']) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -296,13 +372,13 @@ def test_coverdir(self): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '1000']) + ns = self.parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -311,7 +387,7 @@ def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): with contextlib.redirect_stderr(io.StringIO()) as stderr: - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.nowindows) err = stderr.getvalue() self.assertIn('the --nowindows (-n) option is deprecated', err) @@ -319,39 +395,39 @@ def test_nowindows(self): def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.forever) def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = libregrtest._parse_args(['--qui']) + ns = self.parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = libregrtest._parse_args(['--quiet', '--exclude']) + ns = self.parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = libregrtest._parse_args(['--start', '']) + ns = self.parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = libregrtest._parse_args(['foo']) + ns = self.parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = libregrtest._parse_args(['--quiet', 'foo']) + ns = self.parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) def test_arg_option_arg(self): - ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) + ns = self.parse_args(['test_unaryop', '-v', 'test_binop']) self.assertEqual(ns.verbose, 1) self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) @@ -359,6 +435,118 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') + def create_regrtest(self, args): + ns = cmdline._parse_args(args) + + # Check Regrtest attributes which are more reliable than Namespace + # which has an unclear API + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + + return regrtest + + def check_ci_mode(self, args, use_resources, + *, rerun=True, randomize=True, output_on_failure=True): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, -1) + self.assertEqual(regrtest.want_rerun, rerun) + self.assertEqual(regrtest.fail_rerun, False) + self.assertEqual(regrtest.randomize, randomize) + self.assertIsInstance(regrtest.random_seed, int) + self.assertTrue(regrtest.fail_env_changed) + self.assertTrue(regrtest.print_slowest) + self.assertEqual(regrtest.output_on_failure, output_on_failure) + self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + return regrtest + + def test_fast_ci(self): + args = ['--fast-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 10 * 60) + + def test_fast_ci_python_cmd(self): + args = ['--fast-ci', '--python', 'python -X dev'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, rerun=False) + self.assertEqual(regrtest.timeout, 10 * 60) + self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) + + def test_fast_ci_resource(self): + # it should be possible to override resources individually + args = ['--fast-ci', '-u-network'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + use_resources.remove('network') + self.check_ci_mode(args, use_resources) + + def test_fast_ci_verbose(self): + args = ['--fast-ci', '--verbose'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, + output_on_failure=False) + self.assertEqual(regrtest.verbose, True) + + def test_slow_ci(self): + args = ['--slow-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 20 * 60) + + def test_ci_no_randomize(self): + all_resources = set(cmdline.ALL_RESOURCES) + self.check_ci_mode( + ["--slow-ci", "--no-randomize"], all_resources, randomize=False + ) + self.check_ci_mode( + ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False + ) + + def test_dont_add_python_opts(self): + args = ['--dont-add-python-opts'] + ns = cmdline._parse_args(args) + self.assertFalse(ns._add_python_opts) + + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + + def test_verbose3_huntrleaks(self): + args = ['-R', '3:10', '--verbose3'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertIsNotNone(regrtest.hunt_refleak) + self.assertEqual(regrtest.hunt_refleak.warmups, 3) + self.assertEqual(regrtest.hunt_refleak.runs, 10) + self.assertFalse(regrtest.output_on_failure) + + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + +@dataclasses.dataclass(slots=True) +class Rerun: + name: str + match: str | None + success: bool + class BaseTestCase(unittest.TestCase): TEST_UNIQUE_ID = 1 @@ -407,41 +595,61 @@ def regex_search(self, regex, output): self.fail("%r not found in %r" % (regex, output)) return match - def check_line(self, output, regex): - regex = re.compile(r'^' + regex, re.MULTILINE) + def check_line(self, output, pattern, full=False, regex=True): + if not regex: + pattern = re.escape(pattern) + if full: + pattern += '\n' + regex = re.compile(r'^' + pattern, re.MULTILINE) self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^%s\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' - % (LOG_PREFIX, self.TESTNAME_REGEX)) + regex = (fr'^{LOG_PREFIX}\[ *[0-9]+(?:/ *[0-9]+)*\] ' + fr'({self.TESTNAME_REGEX}) {RESULT_REGEX}') parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=(), failed=(), + def check_executed_tests(self, output, tests, *, stats, + skipped=(), failed=(), env_changed=(), omitted=(), - rerun=(), no_test_ran=(), - randomize=False, interrupted=False, - fail_env_changed=False): + rerun=None, run_no_tests=(), + resource_denied=(), + randomize=False, parallel=False, interrupted=False, + fail_env_changed=False, + forever=False, filtered=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] + if isinstance(resource_denied, str): + resource_denied = [resource_denied] if isinstance(failed, str): failed = [failed] if isinstance(env_changed, str): env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - if isinstance(rerun, str): - rerun = [rerun] - if isinstance(no_test_ran, str): - no_test_ran = [no_test_ran] + if isinstance(run_no_tests, str): + run_no_tests = [run_no_tests] + if isinstance(stats, int): + stats = TestStats(stats) + if parallel: + randomize = True + + rerun_failed = [] + if rerun is not None and not env_changed: + failed = [rerun.name] + if not rerun.success: + rerun_failed.append(rerun.name) executed = self.parse_executed_tests(output) + total_tests = list(tests) + if rerun is not None: + total_tests.append(rerun.name) if randomize: - self.assertEqual(set(executed), set(tests), output) + self.assertEqual(set(executed), set(total_tests), output) else: - self.assertEqual(executed, tests, output) + self.assertEqual(executed, total_tests, output) def plural(count): return 's' if count != 1 else '' @@ -457,12 +665,17 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s skipped', skipped) self.check_line(output, regex) + if resource_denied: + regex = list_regex(r'%s test%s skipped \(resource denied\)', resource_denied) + self.check_line(output, regex) + if failed: regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) if env_changed: - regex = list_regex('%s test%s altered the execution environment', + regex = list_regex(r'%s test%s altered the execution environment ' + r'\(env changed\)', env_changed) self.check_line(output, regex) @@ -470,73 +683,120 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s omitted', omitted) self.check_line(output, regex) - if rerun: - regex = list_regex('%s re-run test%s', rerun) + if rerun is not None: + regex = list_regex('%s re-run test%s', [rerun.name]) self.check_line(output, regex) - regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" + regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode" + self.check_line(output, regex) + regex = fr"Re-running {rerun.name} in verbose mode" + if rerun.match: + regex = fr"{regex} \(matching: {rerun.match}\)" self.check_line(output, regex) - for test_name in rerun: - regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode" - self.check_line(output, regex) - if no_test_ran: - regex = list_regex('%s test%s run no tests', no_test_ran) + if run_no_tests: + regex = list_regex('%s test%s run no tests', run_no_tests) self.check_line(output, regex) - good = (len(tests) - len(skipped) - len(failed) - - len(omitted) - len(env_changed) - len(no_test_ran)) + good = (len(tests) - len(skipped) - len(resource_denied) - len(failed) + - len(omitted) - len(env_changed) - len(run_no_tests)) if good: - regex = r'%s test%s OK\.$' % (good, plural(good)) - if not skipped and not failed and good > 1: + regex = r'%s test%s OK\.' % (good, plural(good)) + if not skipped and not failed and (rerun is None or rerun.success) and good > 1: regex = 'All %s' % regex - self.check_line(output, regex) + self.check_line(output, regex, full=True) if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') - result = [] + # Total tests + text = f'run={stats.tests_run:,}' + if filtered: + text = fr'{text} \(filtered\)' + parts = [text] + if stats.failures: + parts.append(f'failures={stats.failures:,}') + if stats.skipped: + parts.append(f'skipped={stats.skipped:,}') + line = fr'Total tests: {" ".join(parts)}' + self.check_line(output, line, full=True) + + # Total test files + run = len(total_tests) - len(resource_denied) + if rerun is not None: + total_failed = len(rerun_failed) + total_rerun = 1 + else: + total_failed = len(failed) + total_rerun = 0 + if interrupted: + run = 0 + text = f'run={run}' + if not forever: + text = f'{text}/{len(tests)}' + if filtered: + text = fr'{text} \(filtered\)' + report = [text] + for name, ntest in ( + ('failed', total_failed), + ('env_changed', len(env_changed)), + ('skipped', len(skipped)), + ('resource_denied', len(resource_denied)), + ('rerun', total_rerun), + ('run_no_tests', len(run_no_tests)), + ): + if ntest: + report.append(f'{name}={ntest}') + line = fr'Total test files: {" ".join(report)}' + self.check_line(output, line, full=True) + + # Result + state = [] if failed: - result.append('FAILURE') + state.append('FAILURE') elif fail_env_changed and env_changed: - result.append('ENV CHANGED') + state.append('ENV CHANGED') if interrupted: - result.append('INTERRUPTED') - if not any((good, result, failed, interrupted, skipped, + state.append('INTERRUPTED') + if not any((good, failed, interrupted, skipped, env_changed, fail_env_changed)): - result.append("NO TEST RUN") - elif not result: - result.append('SUCCESS') - result = ', '.join(result) - if rerun: - self.check_line(output, 'Tests result: FAILURE') - result = 'FAILURE then %s' % result - - self.check_line(output, 'Tests result: %s' % result) - - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) - return randseed + state.append("NO TESTS RAN") + elif not state: + state.append('SUCCESS') + state = ', '.join(state) + if rerun is not None: + new_state = 'SUCCESS' if rerun.success else 'FAILURE' + state = f'{state} then {new_state}' + self.check_line(output, f'Result: {state}', full=True) + + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: - kw['stderr'] = subprocess.PIPE + kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, - universal_newlines=True, + text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: - msg = ("Command %s failed with exit code %s\n" + msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" "\n" "stdout:\n" "---\n" "%s\n" "---\n" - % (str(args), proc.returncode, proc.stdout)) + % (str(args), proc.returncode, exitcode, proc.stdout)) if proc.stderr: msg += ("\n" "stderr:\n" @@ -547,18 +807,24 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): - args = [sys.executable, '-X', 'faulthandler', '-I', *args] - proc = self.run_command(args, **kw) + def run_python(self, args, isolated=True, **kw): + extraargs = [] + if 'uops' in sys._xoptions: + # Pass -X uops along + extraargs.extend(['-X', 'uops']) + cmd = [sys.executable, *extraargs, '-X', 'faulthandler'] + if isolated: + cmd.append('-I') + cmd.extend(args) + proc = self.run_command(cmd, **kw) return proc.stdout class CheckActualTests(BaseTestCase): - """ - Check that regrtest appears to find the expected set of tests. - """ - def test_finds_expected_number_of_tests(self): + """ + Check that regrtest appears to find the expected set of tests. + """ args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] output = self.run_python(args) rough_number_of_tests_found = len(output.splitlines()) @@ -578,6 +844,7 @@ def test_finds_expected_number_of_tests(self): f'{", ".join(output.splitlines())}') +@support.force_not_colorized_test_class class ProgramsTestCase(BaseTestCase): """ Test various ways to run the Python test suite. Use options close @@ -595,17 +862,19 @@ def setUp(self): self.python_args = ['-Wd', '-E', '-bb'] self.regrtest_args = ['-uall', '-rwW', '--testdir=%s' % self.tmptestdir] - if hasattr(faulthandler, 'dump_traceback_later'): - self.regrtest_args.extend(('--timeout', '3600', '-j4')) + self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) - self.check_executed_tests(output, self.tests, randomize=True) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) - def run_tests(self, args): - output = self.run_python(args) + self.check_executed_tests(output, self.tests, + randomize=True, stats=len(self.tests)) + + def run_tests(self, args, env=None, isolated=True): + output = self.run_python(args, env=env, isolated=isolated) self.check_output(output) def test_script_regrtest(self): @@ -615,6 +884,7 @@ def test_script_regrtest(self): args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_test(self): # -m test args = [*self.python_args, '-m', 'test', @@ -627,16 +897,14 @@ def test_module_regrtest(self): *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_autotest(self): # -m test.autotest args = [*self.python_args, '-m', 'test.autotest', *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_from_test_autotest(self): # from test import autotest code = 'from test import autotest' @@ -644,24 +912,18 @@ def test_module_from_test_autotest(self): *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_script_autotest(self): # Lib/test/autotest.py script = os.path.join(self.testdir, 'autotest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) - @unittest.skipUnless(sysconfig.is_python_build(), - 'run_tests.py script is not installed') - def test_tools_script_run_tests(self): - # Tools/scripts/run_tests.py - script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, *self.regrtest_args, *self.tests] - self.run_tests(args) - def run_batch(self, *args): - proc = self.run_command(args) + proc = self.run_command(args, + # gh-133711: cmd.exe uses the OEM code page + # to display the non-ASCII current directory + errors="backslashreplace") self.check_output(proc.stdout) @unittest.skipUnless(sysconfig.is_python_build(), @@ -673,10 +935,14 @@ def test_tools_buildbot_test(self): test_args = ['--testdir=%s' % self.tmptestdir] if platform.machine() == 'ARM64': test_args.append('-arm64') # ARM 64-bit build + elif platform.machine() == 'ARM': + test_args.append('-arm32') # 32-bit ARM build elif platform.architecture()[0] == '64bit': test_args.append('-x64') # 64-bit build - if not Py_DEBUG: + if not support.Py_DEBUG: test_args.append('+d') # Release build, use python.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + test_args.append('--disable-gil') self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -688,13 +954,18 @@ def test_pcbuild_rt(self): rt_args = ["-q"] # Quick, don't run tests twice if platform.machine() == 'ARM64': rt_args.append('-arm64') # ARM 64-bit build + elif platform.machine() == 'ARM': + rt_args.append('-arm32') # 32-bit ARM build elif platform.architecture()[0] == '64bit': rt_args.append('-x64') # 64-bit build - if Py_DEBUG: + if support.Py_DEBUG: rt_args.append('-d') # Debug build, use python_d.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + rt_args.append('--disable-gil') self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) +@support.force_not_colorized_test_class class ArgsTestCase(BaseTestCase): """ Test arguments of the Python test suite. @@ -704,6 +975,40 @@ def run_tests(self, *testargs, **kw): cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] return self.run_python(cmdargs, **kw) + def test_success(self): + code = textwrap.dedent(""" + import unittest + + class PassingTests(unittest.TestCase): + def test_test1(self): + pass + + def test_test2(self): + pass + + def test_test3(self): + pass + """) + tests = [self.create_test(f'ok{i}', code=code) for i in range(1, 6)] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + stats=3 * len(tests)) + + def test_skip(self): + code = textwrap.dedent(""" + import unittest + raise unittest.SkipTest("nope") + """) + test_ok = self.create_test('ok') + test_skip = self.create_test('skip', code=code) + tests = [test_ok, test_skip] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + skipped=[test_skip], + stats=1) + def test_failing_test(self): # test a failing test code = textwrap.dedent(""" @@ -717,8 +1022,9 @@ def test_failing(self): test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] - output = self.run_tests(*tests, exitcode=2) - self.check_executed_tests(output, tests, failed=test_failing) + output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, tests, failed=test_failing, + stats=TestStats(2, 1)) def test_resources(self): # test -u command line option @@ -737,17 +1043,19 @@ def test_pass(self): # -u all: 2 resources enabled output = self.run_tests('-u', 'all', *test_names) - self.check_executed_tests(output, test_names) + self.check_executed_tests(output, test_names, stats=2) # -u audio: 1 resource enabled output = self.run_tests('-uaudio', *test_names) self.check_executed_tests(output, test_names, - skipped=tests['network']) + resource_denied=tests['network'], + stats=1) # no option: 0 resources enabled - output = self.run_tests(*test_names) + output = self.run_tests(*test_names, exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, test_names, - skipped=test_names) + resource_denied=test_names, + stats=0) def test_random(self): # test -r and --randseed command line option @@ -758,13 +1066,14 @@ def test_random(self): test = self.create_test('random', code) # first run to get the output with the random seed - output = self.run_tests('-r', test) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed = self.parse_random_seed(output) match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test) + output = self.run_tests('-r', f'--randseed={randseed}', test, + exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -772,6 +1081,35 @@ def test_random(self): test_random2 = int(match.group(1)) self.assertEqual(test_random2, test_random) + # check that random.seed is used by default + output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string): ignore the env var + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + def test_fromfile(self): # test --fromfile tests = [self.create_test() for index in range(5)] @@ -794,7 +1132,8 @@ def test_fromfile(self): previous = name output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + stats = len(tests) + self.check_executed_tests(output, tests, stats=stats) # test format '[2/7] test_opcodes' with open(filename, "w") as fp: @@ -802,7 +1141,7 @@ def test_fromfile(self): print("[%s/%s] %s" % (index, len(tests), name), file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'test_opcodes' with open(filename, "w") as fp: @@ -810,7 +1149,7 @@ def test_fromfile(self): print(name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'Lib/test/test_opcodes.py' with open(filename, "w") as fp: @@ -818,20 +1157,20 @@ def test_fromfile(self): print('Lib/test/%s.py' % name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) - output = self.run_tests(test, exitcode=130) + output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, omitted=test, - interrupted=True) + interrupted=True, stats=0) def test_slowest(self): # test --slowest tests = [self.create_test() for index in range(3)] output = self.run_tests("--slowest", *tests) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=len(tests)) regex = ('10 slowest tests:\n' '(?:- %s: .*\n){%s}' % (self.TESTNAME_REGEX, len(tests))) @@ -848,22 +1187,22 @@ def test_slowest_interrupted(self): args = ("--slowest", "-j2", test) else: args = ("--slowest", test) - output = self.run_tests(*args, exitcode=130) + output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, - omitted=test, interrupted=True) + omitted=test, interrupted=True, + stats=0) regex = ('10 slowest tests:\n') self.check_line(output, regex) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Regex didn't match: '^lines +cov% +module +\\(path\\)\\n(?: *[0-9]+ *[0-9]{1,2}\\.[0-9]% *[^ ]+ +\\([^)]+\\)+)+' not found in 'Warning: collecting coverage without -j is imprecise. Configure --with-pydebug and run -m test -T -j for best results.\nUsing random seed: 2780369491\n0:00:00 Run 1 test sequentially in a single process\n0:00:00 [1/1] test_regrtest_coverage\n0:00:00 [1/1] test_regrtest_coverage passed\n\n== Tests result: SUCCESS ==\n\n1 test OK.\n\nTotal duration: 102 ms\nTotal tests: run=1\nTotal test files: run=1/1\nResult: SUCCESS\n' def test_coverage(self): # test --coverage test = self.create_test('coverage') output = self.run_tests("--coverage", test) - self.check_executed_tests(output, [test]) + self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' - r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) def test_wait(self): @@ -890,21 +1229,39 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) - output = self.run_tests('--forever', test, exitcode=2) - self.check_executed_tests(output, [test]*3, failed=test) - def check_leak(self, code, what): + # --forever + output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [test]*3, failed=test, + stats=TestStats(3, 1), + forever=True) + + # --forever --rerun + output = self.run_tests('--forever', '--rerun', test, exitcode=0) + self.check_executed_tests(output, [test]*3, + rerun=Rerun(test, + match='test_run', + success=True), + stats=TestStats(4, 1), + forever=True) + + @support.requires_jit_disabled + def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) - output = self.run_tests('--huntrleaks', '3:3:', test, - exitcode=2, + cmd = ['--huntrleaks', '3:3:'] + if run_workers: + cmd.append('-j1') + cmd.append(test) + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test) + self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -913,8 +1270,8 @@ def check_leak(self, code, what): reflog = fp.read() self.assertIn(line2, reflog) - @unittest.skipUnless(Py_DEBUG, 'need a debug build') - def test_huntrleaks(self): + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def check_huntrleaks(self, *, run_workers: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -925,9 +1282,56 @@ class RefLeakTest(unittest.TestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references') + self.check_leak(code, 'references', run_workers=run_workers) - @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks(self): + self.check_huntrleaks(run_workers=False) + + def test_huntrleaks_mp(self): + self.check_huntrleaks(run_workers=True) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" @@ -981,16 +1385,14 @@ def test_crashed(self): crash_test = self.create_test(name="crash", code=code) tests = [crash_test] - output = self.run_tests("-j2", *tests, exitcode=2) + output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, tests, failed=crash_test, - randomize=True) + parallel=True, stats=0) def parse_methods(self, output): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) return [match.group(1) for match in regex.finditer(output)] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignorefile(self): code = textwrap.dedent(""" import unittest @@ -1005,8 +1407,6 @@ def test_method3(self): def test_method4(self): pass """) - all_methods = ['test_method1', 'test_method2', - 'test_method3', 'test_method4'] testname = self.create_test(code=code) # only run a subset @@ -1068,8 +1468,6 @@ def test_method4(self): subset = ['test_method1', 'test_method3'] self.assertEqual(methods, subset) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_env_changed(self): code = textwrap.dedent(""" import unittest @@ -1082,52 +1480,265 @@ def test_env_changed(self): # don't fail by default output = self.run_tests(testname) - self.check_executed_tests(output, [testname], env_changed=testname) + self.check_executed_tests(output, [testname], + env_changed=testname, stats=1) # fail with --fail-env-changed - output = self.run_tests("--fail-env-changed", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=testname, - fail_env_changed=True) + fail_env_changed=True, stats=1) + + # rerun + output = self.run_tests("--rerun", testname) + self.check_executed_tests(output, [testname], + env_changed=testname, + rerun=Rerun(testname, + match=None, + success=True), + stats=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" import unittest class Tests(unittest.TestCase): - def test_bug(self): - # test always fail + def test_succeed(self): + return + + def test_fail_always(self): + # test that always fails self.fail("bug") """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=2) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname, rerun=testname) + rerun=Rerun(testname, + "test_fail_always", + success=False), + stats=TestStats(3, 2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rerun_success(self): # FAILURE then SUCCESS - code = textwrap.dedent(""" - import builtins + marker_filename = os.path.abspath("regrtest_marker_filename") + self.addCleanup(os_helper.unlink, marker_filename) + self.assertFalse(os.path.exists(marker_filename)) + + code = textwrap.dedent(f""" + import os.path import unittest + marker_filename = {marker_filename!r} + class Tests(unittest.TestCase): - failed = False + def test_succeed(self): + return def test_fail_once(self): - if not hasattr(builtins, '_test_failed'): - builtins._test_failed = True + if not os.path.exists(marker_filename): + open(marker_filename, "w").close() self.fail("bug") """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=0) + # FAILURE then SUCCESS => exit code 0 + output = self.run_tests("--rerun", testname, exitcode=0) self.check_executed_tests(output, [testname], - rerun=testname) + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) + + # with --fail-rerun, exit code EXITCODE_RERUN_FAIL + # on "FAILURE then SUCCESS" state. + output = self.run_tests("--rerun", "--fail-rerun", testname, + exitcode=EXITCODE_RERUN_FAIL) + self.check_executed_tests(output, [testname], + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) + + def test_rerun_setup_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def setUpClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="ExampleTests", + success=False), + stats=0) + + def test_rerun_teardown_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def tearDownClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="ExampleTests", + success=False), + stats=2) + + def test_rerun_setup_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def setUpModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match=None, + success=False), + stats=0) + + def test_rerun_teardown_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def tearDownModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + rerun=Rerun(testname, + match=None, + success=False), + stats=2) + + def test_rerun_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def setUp(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def tearDown(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_async_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_async_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncTearDown(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_no_tests_ran(self): code = textwrap.dedent(""" @@ -1139,8 +1750,11 @@ def test_bug(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0) - self.check_executed_tests(output, [testname], no_test_ran=testname) + output = self.run_tests(testname, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) + self.check_executed_tests(output, [testname], + run_no_tests=testname, + stats=0, filtered=True) def test_no_tests_ran_skip(self): code = textwrap.dedent(""" @@ -1152,8 +1766,9 @@ def test_skipped(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, exitcode=0) - self.check_executed_tests(output, [testname]) + output = self.run_tests(testname) + self.check_executed_tests(output, [testname], + stats=TestStats(1, skipped=1)) def test_no_tests_ran_multiple_tests_nonexistent(self): code = textwrap.dedent(""" @@ -1166,9 +1781,11 @@ def test_bug(self): testname = self.create_test(code=code) testname2 = self.create_test(code=code) - output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0) + output = self.run_tests(testname, testname2, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname, testname2]) + run_no_tests=[testname, testname2], + stats=0, filtered=True) def test_no_test_ran_some_test_exist_some_not(self): code = textwrap.dedent(""" @@ -1191,10 +1808,14 @@ def test_other_bug(self): output = self.run_tests(testname, testname2, "-m", "nosuchtest", "-m", "test_other_bug", exitcode=0) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname]) + run_no_tests=[testname], + stats=1, filtered=True) @support.cpython_only - def test_findleaks(self): + def test_uncollectable(self): + # Skip test if _testcapi is missing + import_helper.import_module('_testcapi') + code = textwrap.dedent(r""" import _testcapi import gc @@ -1214,19 +1835,13 @@ def test_garbage(self): """) testname = self.create_test(code=code) - output = self.run_tests("--fail-env-changed", testname, exitcode=3) - self.check_executed_tests(output, [testname], - env_changed=[testname], - fail_env_changed=True) - - # --findleaks is now basically an alias to --fail-env-changed - output = self.run_tests("--findleaks", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiprocessing_timeout(self): code = textwrap.dedent(r""" import time @@ -1248,14 +1863,131 @@ def test_sleep(self): """) testname = self.create_test(code=code) - output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2) + output = self.run_tests("-j2", "--timeout=1.0", testname, + exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname) + failed=testname, stats=0) self.assertRegex(output, re.compile('%s timed out' % testname, re.MULTILINE)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; test_unraisable_exc (test_regrtest_noop39.Tests.test_unraisable_exc) ... ok + def test_unraisable_exc(self): + # --fail-env-changed must catch unraisable exception. + # The exception must be displayed even if sys.stderr is redirected. + code = textwrap.dedent(r""" + import unittest + import weakref + from test.support import captured_stderr + + class MyObject: + pass + + def weakref_callback(obj): + raise Exception("weakref callback bug") + + class Tests(unittest.TestCase): + def test_unraisable_exc(self): + obj = MyObject() + ref = weakref.ref(obj, weakref_callback) + with captured_stderr() as stderr: + # call weakref_callback() which logs + # an unraisable exception + obj = None + self.assertEqual(stderr.getvalue(), '') + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertIn("Warning -- Unraisable exception", output) + self.assertIn("Exception: weakref callback bug", output) + + def test_threading_excepthook(self): + # --fail-env-changed must catch uncaught thread exception. + # The exception must be displayed even if sys.stderr is redirected. + code = textwrap.dedent(r""" + import threading + import unittest + from test.support import captured_stderr + + class MyObject: + pass + + def func_bug(): + raise Exception("bug in thread") + + class Tests(unittest.TestCase): + def test_threading_excepthook(self): + with captured_stderr() as stderr: + thread = threading.Thread(target=func_bug) + thread.start() + thread.join() + self.assertEqual(stderr.getvalue(), '') + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertIn("Warning -- Uncaught thread exception", output) + self.assertIn("Exception: bug in thread", output) + + def test_print_warning(self): + # bpo-45410: The order of messages must be preserved when -W and + # support.print_warning() are used. + code = textwrap.dedent(r""" + import sys + import unittest + from test import support + + class MyObject: + pass + + def func_bug(): + raise Exception("bug in thread") + + class Tests(unittest.TestCase): + def test_print_warning(self): + print("msg1: stdout") + support.print_warning("msg2: print_warning") + # Fail with ENV CHANGED to see print_warning() log + support.environment_altered = True + """) + testname = self.create_test(code=code) + + # Expect an output like: + # + # test_threading_excepthook (test.test_x.Tests) ... msg1: stdout + # Warning -- msg2: print_warning + # ok + regex = (r"test_print_warning.*msg1: stdout\n" + r"Warning -- msg2: print_warning\n" + r"ok\n") + for option in ("-v", "-W"): + with self.subTest(option=option): + cmd = ["--fail-env-changed", option, testname] + output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertRegex(output, regex) + + def test_unicode_guard_env(self): + guard = os.environ.get(setup.UNICODE_GUARD_ENV) + self.assertIsNotNone(guard, f"{setup.UNICODE_GUARD_ENV} not set") + if guard.isascii(): + # Skip to signify that the env var value was changed by the user; + # possibly to something ASCII to work around Unicode issues. + self.skipTest("Modified guard") + def test_cleanup(self): dirname = os.path.join(self.tmptestdir, "test_python_123") os.mkdir(dirname) @@ -1271,10 +2003,411 @@ def test_cleanup(self): for name in names: self.assertFalse(os.path.exists(name), name) + @unittest.skip("TODO: RUSTPYTHON; flaky") + @unittest.skipIf(support.is_wasi, + 'checking temp files is not implemented on WASI') + def test_leak_tmp_file(self): + code = textwrap.dedent(r""" + import os.path + import tempfile + import unittest + + class FileTests(unittest.TestCase): + def test_leak_tmp_file(self): + filename = os.path.join(tempfile.gettempdir(), 'mytmpfile') + with open(filename, "wb") as fp: + fp.write(b'content') + """) + testnames = [self.create_test(code=code) for _ in range(3)] + + output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, testnames, + env_changed=testnames, + fail_env_changed=True, + parallel=True, + stats=len(testnames)) + for testname in testnames: + self.assertIn(f"Warning -- {testname} leaked temporary " + f"files (1): mytmpfile", + output) + + def test_worker_decode_error(self): + # gh-109425: Use "backslashreplace" error handler to decode stdout. + if sys.platform == 'win32': + encoding = locale.getencoding() + else: + encoding = sys.stdout.encoding + if encoding is None: + encoding = sys.__stdout__.encoding + if encoding is None: + self.skipTest("cannot get regrtest worker encoding") + + nonascii = bytes(ch for ch in range(128, 256)) + corrupted_output = b"nonascii:%s\n" % (nonascii,) + # gh-108989: On Windows, assertion errors are written in UTF-16: when + # decoded each letter is follow by a NUL character. + assertion_failed = 'Assertion failed: tstate_is_alive(tstate)\n' + corrupted_output += assertion_failed.encode('utf-16-le') + try: + corrupted_output.decode(encoding) + except UnicodeDecodeError: + pass + else: + self.skipTest(f"{encoding} can decode non-ASCII bytes") + + expected_line = corrupted_output.decode(encoding, 'backslashreplace') + + code = textwrap.dedent(fr""" + import sys + import unittest + + class Tests(unittest.TestCase): + def test_pass(self): + pass + + # bytes which cannot be decoded from UTF-8 + corrupted_output = {corrupted_output!a} + sys.stdout.buffer.write(corrupted_output) + sys.stdout.buffer.flush() + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname) + self.check_executed_tests(output, [testname], + parallel=True, + stats=1) + self.check_line(output, expected_line, regex=False) + + def test_doctest(self): + code = textwrap.dedent(r''' + import doctest + import sys + from test import support + + def my_function(): + """ + Pass: + + >>> 1 + 1 + 2 + + Failure: + + >>> 2 + 3 + 23 + >>> 1 + 1 + 11 + + Skipped test (ignored): + + >>> id(1.0) # doctest: +SKIP + 7948648 + """ + + def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + ''') + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + parallel=True, + stats=TestStats(1, 1, 0)) + + def _check_random_seed(self, run_workers: bool): + # gh-109276: When -r/--randomize is used, random.seed() is called + # with the same random seed before running each test file. + code = textwrap.dedent(r''' + import random + import unittest + + class RandomSeedTest(unittest.TestCase): + def test_randint(self): + numbers = [random.randint(0, 1000) for _ in range(10)] + print(f"Random numbers: {numbers}") + ''') + tests = [self.create_test(name=f'test_random{i}', code=code) + for i in range(1, 3+1)] + + random_seed = 856_656_202 + cmd = ["--randomize", f"--randseed={random_seed}"] + if run_workers: + # run as many worker processes than the number of tests + cmd.append(f'-j{len(tests)}') + cmd.extend(tests) + output = self.run_tests(*cmd) + + random.seed(random_seed) + # Make the assumption that nothing consume entropy between libregrest + # setup_tests() which calls random.seed() and RandomSeedTest calling + # random.randint(). + numbers = [random.randint(0, 1000) for _ in range(10)] + expected = f"Random numbers: {numbers}" + + regex = r'^Random numbers: .*$' + matches = re.findall(regex, output, flags=re.MULTILINE) + self.assertEqual(matches, [expected] * len(tests)) + + def test_random_seed(self): + self._check_random_seed(run_workers=False) + + @unittest.skip("TODO: RUSTPYTHON; flaky") + def test_random_seed_workers(self): + self._check_random_seed(run_workers=True) + + @unittest.skip("TODO: RUSTPYTHON; flaky") + def test_python_command(self): + code = textwrap.dedent(r""" + import sys + import unittest + + class WorkerTests(unittest.TestCase): + def test_dev_mode(self): + self.assertTrue(sys.flags.dev_mode) + """) + tests = [self.create_test(code=code) for _ in range(3)] + + # Custom Python command: "python -X dev" + python_cmd = [sys.executable, '-X', 'dev'] + # test.libregrtest.cmdline uses shlex.split() to parse the Python + # command line string + python_cmd = shlex.join(python_cmd) + + output = self.run_tests("--python", python_cmd, "-j0", *tests) + self.check_executed_tests(output, tests, + stats=len(tests), parallel=True) + + def test_unload_tests(self): + # Test that unloading test modules does not break tests + # that import from other tests. + # The test execution order matters for this test. + # Both test_regrtest_a and test_regrtest_c which are executed before + # and after test_regrtest_b import a submodule from the test_regrtest_b + # package and use it in testing. test_regrtest_b itself does not import + # that submodule. + # Previously test_regrtest_c failed because test_regrtest_b.util in + # sys.modules was left after test_regrtest_a (making the import + # statement no-op), but new test_regrtest_b without the util attribute + # was imported for test_regrtest_b. + testdir = os.path.join(os.path.dirname(__file__), + 'regrtestdata', 'import_from_tests') + tests = [f'test_regrtest_{name}' for name in ('a', 'b', 'c')] + args = ['-Wd', '-E', '-bb', '-m', 'test', '--testdir=%s' % testdir, *tests] + output = self.run_python(args) + self.check_executed_tests(output, tests, stats=3) + + def check_add_python_opts(self, option): + # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + + # Skip test if _testinternalcapi is missing + import_helper.import_module('_testinternalcapi') + + code = textwrap.dedent(r""" + import sys + import unittest + from test import support + try: + from _testcapi import config_get + except ImportError: + config_get = None + + # WASI/WASM buildbots don't use -E option + use_environment = (support.is_emscripten or support.is_wasi) + + class WorkerTests(unittest.TestCase): + @unittest.skipUnless(config_get is None, 'need config_get()') + def test_config(self): + config = config_get() + # -u option + self.assertEqual(config_get('buffered_stdio'), 0) + # -W default option + self.assertTrue(config_get('warnoptions'), ['default']) + # -bb option + self.assertTrue(config_get('bytes_warning'), 2) + # -E option + self.assertTrue(config_get('use_environment'), use_environment) + + def test_python_opts(self): + # -u option + self.assertTrue(sys.__stdout__.write_through) + self.assertTrue(sys.__stderr__.write_through) + + # -W default option + self.assertTrue(sys.warnoptions, ['default']) + + # -bb option + self.assertEqual(sys.flags.bytes_warning, 2) + + # -E option + self.assertEqual(not sys.flags.ignore_environment, + use_environment) + """) + testname = self.create_test(code=code) + + # Use directly subprocess to control the exact command line + cmd = [sys.executable, + "-m", "test", option, + f'--testdir={self.tmptestdir}', + testname] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + self.assertEqual(proc.returncode, 0, proc) + + def test_add_python_opts(self): + for opt in ("--fast-ci", "--slow-ci"): + with self.subTest(opt=opt): + self.check_add_python_opts(opt) + + # gh-76319: Raising SIGSEGV on Android may not cause a crash. + @unittest.skipIf(support.is_android, + 'raising SIGSEGV on Android is unreliable') + def test_worker_output_on_failure(self): + # Skip test if faulthandler is missing + import_helper.import_module('faulthandler') + + code = textwrap.dedent(r""" + import faulthandler + import unittest + from test import support + + class CrashTests(unittest.TestCase): + def test_crash(self): + print("just before crash!", flush=True) + + with support.SuppressCrashReport(): + faulthandler._sigsegv(True) + """) + testname = self.create_test(code=code) + + # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) + env = dict(os.environ) + option = 'handle_segv=0' + support.set_sanitizer_env_var(env, option) + + output = self.run_tests("-j1", testname, + exitcode=EXITCODE_BAD_TEST, + env=env) + self.check_executed_tests(output, testname, + failed=[testname], + stats=0, parallel=True) + if not support.MS_WINDOWS: + exitcode = -int(signal.SIGSEGV) + self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) + self.check_line(output, "just before crash!", full=True, regex=False) + + def test_verbose3(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_pass(self): + print("SPAM SPAM SPAM") + """) + testname = self.create_test(code=code) + + # Run sequentially + output = self.run_tests("--verbose3", testname) + self.check_executed_tests(output, testname, stats=1) + self.assertNotIn('SPAM SPAM SPAM', output) + + # -R option needs a debug build + if support.Py_DEBUG: + # Check for reference leaks, run in parallel + output = self.run_tests("-R", "3:3", "-j1", "--verbose3", testname) + self.check_executed_tests(output, testname, stats=1, parallel=True) + self.assertNotIn('SPAM SPAM SPAM', output) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType' + def test_xml(self): + code = textwrap.dedent(r""" + import unittest + + class VerboseTests(unittest.TestCase): + def test_failed(self): + print("abc \x1b def") + self.fail() + """) + testname = self.create_test(code=code) + + # Run sequentially + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + output = self.run_tests(testname, "--junit-xml", filename, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=testname, + stats=TestStats(1, 1, 0)) + + # Test generated XML + with open(filename, encoding="utf8") as fp: + content = fp.read() + + testsuite = ElementTree.fromstring(content) + self.assertEqual(int(testsuite.get('tests')), 1) + self.assertEqual(int(testsuite.get('errors')), 0) + self.assertEqual(int(testsuite.get('failures')), 1) + + testcase = testsuite[0][0] + self.assertEqual(testcase.get('status'), 'run') + self.assertEqual(testcase.get('result'), 'completed') + self.assertGreater(float(testcase.get('time')), 0) + for out in testcase.iter('system-out'): + self.assertEqual(out.text, r"abc \x1b def") + + def test_nonascii(self): + code = textwrap.dedent(r""" + import unittest + + class NonASCIITests(unittest.TestCase): + def test_docstring(self): + '''docstring:\u20ac''' + + def test_subtest(self): + with self.subTest(param='subtest:\u20ac'): + pass + + def test_skip(self): + self.skipTest('skipped:\u20ac') + """) + testname = self.create_test(code=code) + + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'ascii' + + def check(output): + self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1)) + self.assertIn(r'docstring:\u20ac', output) + self.assertIn(r'skipped:\u20ac', output) + + # Run sequentially + output = self.run_tests('-v', testname, env=env, isolated=False) + check(output) + + # Run in parallel + output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) + check(output) + + def test_pgo_exclude(self): + # Get PGO tests + output = self.run_tests('--pgo', '--list-tests') + pgo_tests = output.strip().split() + + # Exclude test_re + output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re') + tests = output.strip().split() + self.assertNotIn('test_re', tests) + self.assertEqual(len(tests), len(pgo_tests) - 1) + class TestUtils(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') @@ -1297,6 +2430,219 @@ def test_format_duration(self): self.assertEqual(utils.format_duration(3 * 3600 + 1), '3 hour 1 sec') + def test_normalize_test_name(self): + normalize = normalize_test_name + self.assertEqual(normalize('test_access (test.test_os.FileTests.test_access)'), + 'test_access') + self.assertEqual(normalize('setUpClass (test.test_os.ChownFileTests)', is_error=True), + 'ChownFileTests') + self.assertEqual(normalize('test_success (test.test_bug.ExampleTests.test_success)', is_error=True), + 'test_success') + self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) + self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) + + def test_format_resources(self): + format_resources = utils.format_resources + ALL_RESOURCES = utils.ALL_RESOURCES + self.assertEqual( + format_resources(("network",)), + 'resources (1): network') + self.assertEqual( + format_resources(("audio", "decimal", "network")), + 'resources (3): audio,decimal,network') + self.assertEqual( + format_resources(ALL_RESOURCES), + 'resources: all') + self.assertEqual( + format_resources(tuple(name for name in ALL_RESOURCES + if name != "cpu")), + 'resources: all,-cpu') + self.assertEqual( + format_resources((*ALL_RESOURCES, "tzdata")), + 'resources: all,tzdata') + + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') + + # Test acceptance + with support.swap_attr(support, '_test_matchers', ()): + # match all + set_match_tests([]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match all using None + set_match_tests(None) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the full test identifier + set_match_tests([(test_access.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', True)]) + self.assertFalse(match_test(test_access)) + set_match_tests([('FileTests', True)]) + self.assertTrue(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + set_match_tests([('test_access', True), ('DONTMATCH', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test rejection + with support.swap_attr(support, '_test_matchers', ()): + # match the full test identifier + set_match_tests([(test_access.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', False)]) + self.assertTrue(match_test(test_access)) + set_match_tests([('FileTests', False)]) + self.assertFalse(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('test_access', False), ('DONTMATCH', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('*test_os', True), ('test_access', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + def test_sanitize_xml(self): + sanitize_xml = utils.sanitize_xml + + # escape invalid XML characters + self.assertEqual(sanitize_xml('abc \x1b\x1f def'), + r'abc \x1b\x1f def') + self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'), + r'nul:\x00, bell:\x07') + self.assertEqual(sanitize_xml('surrogate:\uDC80'), + r'surrogate:\udc80') + self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'), + r'illegal \ufffe and \uffff') + + # no escape for valid XML characters + self.assertEqual(sanitize_xml('a\n\tb'), + 'a\n\tb') + self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), + 'valid t\xe9xt \u20ac') + + +from test.libregrtest.results import TestResults + + +class TestColorized(unittest.TestCase): + def test_test_result_get_state(self): + # Arrange + green = _colorize.ANSIColors.GREEN + red = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + yellow = _colorize.ANSIColors.YELLOW + + good_results = TestResults() + good_results.good = ["good1", "good2"] + bad_results = TestResults() + bad_results.bad = ["bad1", "bad2"] + no_results = TestResults() + no_results.bad = [] + interrupted_results = TestResults() + interrupted_results.interrupted = True + interrupted_worker_bug = TestResults() + interrupted_worker_bug.interrupted = True + interrupted_worker_bug.worker_bug = True + + for results, expected in ( + (good_results, f"{green}SUCCESS{reset}"), + (bad_results, f"{red}FAILURE{reset}"), + (no_results, f"{yellow}NO TESTS RAN{reset}"), + (interrupted_results, f"{yellow}INTERRUPTED{reset}"), + ( + interrupted_worker_bug, + f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}", + ), + ): + with self.subTest(results=results, expected=expected): + # Act + with unittest.mock.patch( + "_colorize.can_colorize", return_value=True + ): + result = results.get_state(fail_env_changed=False) + + # Assert + self.assertEqual(result, expected) + if __name__ == '__main__': + setup.setup_process() unittest.main() diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 738b48f5623..3396b54cc9f 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -3,6 +3,7 @@ Nick Mathewson """ +import annotationlib import sys import os import shutil @@ -11,7 +12,7 @@ import unittest import textwrap -from test.support import verbose +from test.support import verbose, EqualToForwardRef from test.support.os_helper import create_empty_file from reprlib import repr as r # Don't shadow builtin repr from reprlib import Repr @@ -149,15 +150,40 @@ def test_frozenset(self): eq(r(frozenset({1, 2, 3, 4, 5, 6})), "frozenset({1, 2, 3, 4, 5, 6})") eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_numbers(self): - eq = self.assertEqual - eq(r(123), repr(123)) - eq(r(123), repr(123)) - eq(r(1.0/3), repr(1.0/3)) - - n = 10**100 - expected = repr(n)[:18] + "..." + repr(n)[-19:] - eq(r(n), expected) + for x in [123, 1.0 / 3]: + self.assertEqual(r(x), repr(x)) + + max_digits = sys.get_int_max_str_digits() + for k in [100, max_digits - 1]: + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + expected = repr(n)[:18] + "..." + repr(n)[-19:] + self.assertEqual(r(n), expected) + + def re_msg(n, d): + return (rf'<{n.__class__.__name__} instance with roughly {d} ' + rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>') + + k = max_digits + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + + for k in [max_digits + 1, 2 * max_digits]: + self.assertGreater(k, 100) + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + with self.subTest(f'10 ** {k} - 1', k=k): + n = 10 ** k - 1 + # Here, since math.log10(n) == math.log10(n-1), + # the number of digits of n - 1 is overestimated. + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) def test_instance(self): eq = self.assertEqual @@ -172,24 +198,22 @@ def test_instance(self): eq(r(i3), ("<ClassWithFailingRepr instance at %#x>"%id(i3))) s = r(ClassWithFailingRepr) - self.assertTrue(s.startswith("<class ")) - self.assertTrue(s.endswith(">")) + self.assertStartsWith(s, "<class ") + self.assertEndsWith(s, ">") self.assertIn(s.find("..."), [12, 13]) def test_lambda(self): r = repr(lambda x: x) - self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r) + self.assertStartsWith(r, "<function ReprTests.test_lambda.<locals>.<lambda") # XXX anonymous functions? see func_repr - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_builtin_function(self): eq = self.assertEqual # Functions eq(repr(hash), '<built-in function hash>') # Methods - self.assertTrue(repr(''.split).startswith( - '<built-in method split of str object at 0x')) + self.assertStartsWith(repr(''.split), + '<built-in method split of str object at 0x') def test_range(self): eq = self.assertEqual @@ -214,8 +238,7 @@ def test_nesting(self): eq(r([[[[[[{}]]]]]]), "[[[[[[{}]]]]]]") eq(r([[[[[[[{}]]]]]]]), "[[[[[[[...]]]]]]]") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cell(self): def get_cell(): x = 42 @@ -376,20 +399,20 @@ def test_valid_indent(self): 'object': { 1: 'two', b'three': [ - (4.5, 6.7), + (4.5, 6.25), [set((8, 9)), frozenset((10, 11))], ], }, 'tests': ( (dict(indent=None), '''\ - {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''), + {1: 'two', b'three': [(4.5, 6.25), [{8, 9}, frozenset({10, 11})]]}'''), (dict(indent=False), '''\ { 1: 'two', b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -409,7 +432,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -429,7 +452,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -449,7 +472,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -469,7 +492,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -497,7 +520,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -517,7 +540,7 @@ def test_valid_indent(self): -->b'three': [ -->-->( -->-->-->4.5, - -->-->-->6.7, + -->-->-->6.25, -->-->), -->-->[ -->-->-->{ @@ -537,7 +560,7 @@ def test_valid_indent(self): ....b'three': [ ........( ............4.5, - ............6.7, + ............6.25, ........), ........[ ............{ @@ -733,8 +756,8 @@ class baz: importlib.invalidate_caches() from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import baz ibaz = baz.baz() - self.assertTrue(repr(ibaz).startswith( - "<%s.baz object at 0x" % baz.__name__)) + self.assertStartsWith(repr(ibaz), + "<%s.baz object at 0x" % baz.__name__) def test_method(self): self._check_path_limitations('qux') @@ -747,13 +770,13 @@ def amethod(self): pass from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import qux # Unbound methods first r = repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod) - self.assertTrue(r.startswith('<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod'), r) + self.assertStartsWith(r, '<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod') # Bound method next iqux = qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() r = repr(iqux.amethod) - self.assertTrue(r.startswith( + self.assertStartsWith(r, '<bound method aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod of <%s.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa object at 0x' \ - % (qux.__name__,) ), r) + % (qux.__name__,) ) @unittest.skip('needs a built-in function with a really long name') def test_builtin_function(self): @@ -822,8 +845,6 @@ def __repr__(self): self.assertIs(X.f, X.__repr__.__wrapped__) - # TODO: RUSTPYTHON: AttributeError: 'TypeVar' object has no attribute '__name__' - @unittest.expectedFailure def test__type_params__(self): class My: @recursive_repr() @@ -835,5 +856,19 @@ def __repr__[T: str](self, default: T = '') -> str: self.assertEqual(type_params[0].__name__, 'T') self.assertEqual(type_params[0].__bound__, str) + def test_annotations(self): + class My: + @recursive_repr() + def __repr__(self, default: undefined = ...): + return default + + annotations = annotationlib.get_annotations( + My.__repr__, format=annotationlib.Format.FORWARDREF + ) + self.assertEqual( + annotations, + {'default': EqualToForwardRef("undefined", owner=My.__repr__)} + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 5f449cdc05c..b967c7623c5 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -28,9 +28,6 @@ def __gt__(self, other): def __ge__(self, other): return self.x >= other - def __cmp__(self, other): - raise support.TestFailed("Number.__cmp__() should not be called") - def __repr__(self): return "Number(%r)" % (self.x, ) @@ -53,9 +50,6 @@ def __setitem__(self, i, v): def __bool__(self): raise TypeError("Vectors cannot be used in Boolean contexts") - def __cmp__(self, other): - raise support.TestFailed("Vector.__cmp__() should not be called") - def __repr__(self): return "Vector(%r)" % (self.data, ) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b0bed431d4b..e33723cc70c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -16,6 +16,14 @@ class BaseRobotTest: bad = [] site_maps = None + def __init_subclass__(cls): + super().__init_subclass__() + # Remove tests that do nothing. + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + def setUp(self): lines = io.StringIO(self.robots_txt).readlines() self.parser = urllib.robotparser.RobotFileParser() @@ -231,9 +239,16 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * Disallow: /some/path?name=value +Disallow: /another/path? +Disallow: /yet/one/path?name=value&more """ - good = ['/some/path'] - bad = ['/some/path?name=value'] + good = ['/some/path', '/some/path?', + '/some/path%3Fname=value', '/some/path?name%3Dvalue', + '/another/path', '/another/path%3F', + '/yet/one/path?name=value%26more'] + bad = ['/some/path?name=value' + '/another/path?', '/another/path?name=value', + '/yet/one/path?name=value&more'] class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): @@ -249,15 +264,79 @@ class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): bad = ['/some/path'] -class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): - # normalize the URL first (#17403) +class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * -Allow: /some/path? -Disallow: /another/path? - """ - good = ['/some/path?'] - bad = ['/another/path?'] +Disallow: /a1/Z-._~ # unreserved characters +Disallow: /a2/%5A%2D%2E%5F%7E # percent-encoded unreserved characters +Disallow: /u1/%F0%9F%90%8D # percent-encoded ASCII Unicode character +Disallow: /u2/%f0%9f%90%8d +Disallow: /u3/\U0001f40d # raw non-ASCII Unicode character +Disallow: /v1/%F0 # percent-encoded non-ASCII octet +Disallow: /v2/%f0 +Disallow: /v3/\udcf0 # raw non-ASCII octet +Disallow: /p1%xy # raw percent +Disallow: /p2% +Disallow: /p3%25xy # percent-encoded percent +Disallow: /p4%2525xy # double percent-encoded percent +Disallow: /john%20smith # space +Disallow: /john doe +Disallow: /trailingspace%20 +Disallow: /question%3Fq=v # not query +Disallow: /hash%23f # not fragment +Disallow: /dollar%24 +Disallow: /asterisk%2A +Disallow: /sub/dir +Disallow: /slash%2F +Disallow: /query/question?q=%3F +Disallow: /query/raw/question?q=? +Disallow: /query/eq?q%3Dv +Disallow: /query/amp?q=v%26a +""" + good = [ + '/u1/%F0', '/u1/%f0', + '/u2/%F0', '/u2/%f0', + '/u3/%F0', '/u3/%f0', + '/p1%2525xy', '/p2%f0', '/p3%2525xy', '/p4%xy', '/p4%25xy', + '/question?q=v', + '/dollar', '/asterisk', + '/query/eq?q=v', + '/query/amp?q=v&a', + ] + bad = [ + '/a1/Z-._~', '/a1/%5A%2D%2E%5F%7E', + '/a2/Z-._~', '/a2/%5A%2D%2E%5F%7E', + '/u1/%F0%9F%90%8D', '/u1/%f0%9f%90%8d', '/u1/\U0001f40d', + '/u2/%F0%9F%90%8D', '/u2/%f0%9f%90%8d', '/u2/\U0001f40d', + '/u3/%F0%9F%90%8D', '/u3/%f0%9f%90%8d', '/u3/\U0001f40d', + '/v1/%F0', '/v1/%f0', '/v1/\udcf0', '/v1/\U0001f40d', + '/v2/%F0', '/v2/%f0', '/v2/\udcf0', '/v2/\U0001f40d', + '/v3/%F0', '/v3/%f0', '/v3/\udcf0', '/v3/\U0001f40d', + '/p1%xy', '/p1%25xy', + '/p2%', '/p2%25', '/p2%2525', '/p2%xy', + '/p3%xy', '/p3%25xy', + '/p4%2525xy', + '/john%20smith', '/john smith', + '/john%20doe', '/john doe', + '/trailingspace%20', '/trailingspace ', + '/question%3Fq=v', + '/hash#f', '/hash%23f', + '/dollar$', '/dollar%24', + '/asterisk*', '/asterisk%2A', + '/sub/dir', '/sub%2Fdir', + '/slash%2F', '/slash/', + '/query/question?q=?', '/query/question?q=%3F', + '/query/raw/question?q=?', '/query/raw/question?q=%3F', + '/query/eq?q%3Dv', + '/query/amp?q=v%26a', + ] + # other reserved characters + for c in ":/#[]@!$&'()*+,;=": + robots_txt += f'Disallow: /raw{c}\nDisallow: /pc%{ord(c):02X}\n' + bad.append(f'/raw{c}') + bad.append(f'/raw%{ord(c):02X}') + bad.append(f'/pc{c}') + bad.append(f'/pc%{ord(c):02X}') class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): @@ -299,22 +378,17 @@ def test_string_formatting(self): self.assertEqual(str(self.parser), self.expected_output) -class RobotHandler(BaseHTTPRequestHandler): - - def do_GET(self): - self.send_error(403, "Forbidden access") - - def log_message(self, format, *args): - pass - - -class PasswordProtectedSiteTestCase(unittest.TestCase): +@unittest.skipUnless( + support.has_socket_support, + "Socket server requires working socket." +) +class BaseLocalNetworkTestCase: def setUp(self): # clear _opener global variable self.addCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), RobotHandler) + self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) self.t = threading.Thread( name='HTTPServer serving', @@ -331,6 +405,57 @@ def tearDown(self): self.t.join() self.server.server_close() + +SAMPLE_ROBOTS_TXT = b'''\ +User-agent: test_robotparser +Disallow: /utf8/\xf0\x9f\x90\x8d +Disallow: /non-utf8/\xf0 +Disallow: //[spam]/path +''' + + +class LocalNetworkTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(SAMPLE_ROBOTS_TXT) + + def log_message(self, format, *args): + pass + + @threading_helper.reap_threads + def testRead(self): + # Test that reading a weird robots.txt doesn't fail. + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + '/robots.txt' + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + # And it can even interpret the weird paths in some reasonable way. + agent = 'test_robotparser' + self.assertTrue(parser.can_fetch(agent, robots_url)) + self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) + + +class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(403, "Forbidden access") + + def log_message(self, format, *args): + pass + @threading_helper.reap_threads def testPasswordProtectedSite(self): addr = self.server.server_address @@ -342,6 +467,7 @@ def testPasswordProtectedSite(self): self.assertFalse(parser.can_fetch("*", robots_url)) +@support.requires_working_socket() class NetworkTestCase(unittest.TestCase): base_url = 'http://www.pythontest.net/' diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index c1e255e7af3..1b77b102577 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -680,7 +680,6 @@ def test_basic_script_no_suffix(self): self._check_script(script_name, "<run_path>", script_name, script_name, expect_spec=False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; weird panic in lz4-flex") def test_script_compiled(self): with temp_dir() as script_dir: mod_name = 'script' @@ -798,7 +797,7 @@ def assertSigInt(self, cmd, *args, **kwargs): # Use -E to ignore PYTHONSAFEPATH cmd = [sys.executable, '-E', *cmd] proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n"), proc.stderr) + self.assertEndsWith(proc.stderr, "\nKeyboardInterrupt\n") self.assertEqual(proc.returncode, self.EXPECTED_CODE) def test_pymain_run_file(self): diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 662d8eefbc0..952afb7e0d3 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -692,7 +692,7 @@ def dec(self): self.assertEqual(c.dec(), 1) self.assertEqual(c.dec(), 0) - @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope def testGlobalInParallelNestedFunctions(self): # A symbol table bug leaked the global statement from one # function to other nested functions in the same block. @@ -779,7 +779,7 @@ class X: class X: locals()["x"] = 43 del x - self.assertFalse(hasattr(X, "x")) + self.assertNotHasAttr(X, "x") self.assertEqual(x, 42) @cpython_only diff --git a/Lib/test/test_script_helper.py b/Lib/test/test_script_helper.py index e7b54fd7798..4ade2cbc0d4 100644 --- a/Lib/test/test_script_helper.py +++ b/Lib/test/test_script_helper.py @@ -82,7 +82,6 @@ def tearDown(self): # Reset the private cached state. script_helper.__dict__['__cached_interp_requires_environment'] = None - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_true(self, mock_check_call): with mock.patch.dict(os.environ): @@ -92,7 +91,6 @@ def test_interpreter_requires_environment_true(self, mock_check_call): self.assertTrue(script_helper.interpreter_requires_environment()) self.assertEqual(1, mock_check_call.call_count) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_false(self, mock_check_call): with mock.patch.dict(os.environ): @@ -102,7 +100,6 @@ def test_interpreter_requires_environment_false(self, mock_check_call): self.assertFalse(script_helper.interpreter_requires_environment()) self.assertEqual(1, mock_check_call.call_count) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_details(self, mock_check_call): with mock.patch.dict(os.environ): @@ -115,7 +112,6 @@ def test_interpreter_requires_environment_details(self, mock_check_call): self.assertEqual(sys.executable, check_call_command[0]) self.assertIn('-E', check_call_command) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_with_pythonhome(self, mock_check_call): with mock.patch.dict(os.environ): diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py new file mode 100644 index 00000000000..6ce8cd423f7 --- /dev/null +++ b/Lib/test/test_select.py @@ -0,0 +1,107 @@ +import errno +import select +import subprocess +import sys +import textwrap +import unittest +from test import support + +support.requires_working_socket(module=True) + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: unexpectedly identical: [] + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + @support.requires_fork() + def test_select(self): + code = textwrap.dedent(''' + import time + for i in range(10): + print("testing...", flush=True) + time.sleep(0.050) + ''') + cmd = [sys.executable, '-I', '-c', code] + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + pipe = proc.stdout + for timeout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print(f'timeout = {timeout}') + rfd, wfd, xfd = select.select([pipe], [], [], timeout) + self.assertEqual(wfd, []) + self.assertEqual(xfd, []) + if not rfd: + continue + if rfd == [pipe]: + line = pipe.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', + rfd, wfd, xfd) + + # Issue 16230: Crash on select resized list + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot select a fd multiple times." + ) + @unittest.skip("TODO: RUSTPYTHON timed out") + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by poll + def test_disallow_instantiation(self): + support.check_disallow_instantiation(self, type(select.poll())) + + if hasattr(select, 'devpoll'): + support.check_disallow_instantiation(self, type(select.devpoll())) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index e8c0c33e980..0bb02ef11f6 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -152,7 +152,6 @@ """ class SetComprehensionTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' def test_exception_locations(self): # The location of an exception raised from __init__ or # __next__ should should be the iterator expression diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index ac25eee2e52..08c6562f2a2 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,7 +1,9 @@ import unittest +import dbm import shelve -import glob -from test import support +import pickle +import os + from test.support import os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -41,12 +43,8 @@ def copy(self): class TestCase(unittest.TestCase): - - fn = "shelftemp.db" - - def tearDown(self): - for f in glob.glob(self.fn+"*"): - os_helper.unlink(f) + dirname = os_helper.TESTFN + fn = os.path.join(os_helper.TESTFN, "shelftemp.db") def test_close(self): d1 = {} @@ -63,29 +61,34 @@ def test_close(self): else: self.fail('Closed shelf should not find a key') - def test_ascii_file_shelf(self): - s = shelve.open(self.fn, protocol=0) + def test_open_template(self, filename=None, protocol=None): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + s = shelve.open(filename=filename if filename is not None else self.fn, + protocol=protocol) try: s['key1'] = (1,2,3,4) self.assertEqual(s['key1'], (1,2,3,4)) finally: s.close() + def test_ascii_file_shelf(self): + self.test_open_template(protocol=0) + def test_binary_file_shelf(self): - s = shelve.open(self.fn, protocol=1) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=1) def test_proto2_file_shelf(self): - s = shelve.open(self.fn, protocol=2) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=2) + + def test_pathlib_path_file_shelf(self): + self.test_open_template(filename=os_helper.FakePath(self.fn)) + + def test_bytes_path_file_shelf(self): + self.test_open_template(filename=os.fsencode(self.fn)) + + def test_pathlib_bytes_path_file_shelf(self): + self.test_open_template(filename=os_helper.FakePath(os.fsencode(self.fn))) def test_in_memory_shelf(self): d1 = byteskeydict() @@ -160,65 +163,54 @@ def test_with(self): def test_default_protocol(self): with shelve.Shelf({}) as s: - self.assertEqual(s._protocol, 3) + self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) -from test import mapping_tests -class TestShelveBase(mapping_tests.BasicTestMappingProtocol): - fn = "shelftemp.db" - counter = 0 - def __init__(self, *args, **kw): - self._db = [] - mapping_tests.BasicTestMappingProtocol.__init__(self, *args, **kw) +class TestShelveBase: type2test = shelve.Shelf + def _reference(self): return {"key1":"value1", "key2":2, "key3":(1,2,3)} + + +class TestShelveInMemBase(TestShelveBase): def _empty_mapping(self): - if self._in_mem: - x= shelve.Shelf(byteskeydict(), **self._args) - else: - self.counter+=1 - x= shelve.open(self.fn+str(self.counter), **self._args) - self._db.append(x) + return shelve.Shelf(byteskeydict(), **self._args) + + +class TestShelveFileBase(TestShelveBase): + counter = 0 + + def _empty_mapping(self): + self.counter += 1 + x = shelve.open(self.base_path + str(self.counter), **self._args) + self.addCleanup(x.close) return x - def tearDown(self): - for db in self._db: - db.close() - self._db = [] - if not self._in_mem: - for f in glob.glob(self.fn+"*"): - os_helper.unlink(f) - -class TestAsciiFileShelve(TestShelveBase): - _args={'protocol':0} - _in_mem = False -class TestBinaryFileShelve(TestShelveBase): - _args={'protocol':1} - _in_mem = False -class TestProto2FileShelve(TestShelveBase): - _args={'protocol':2} - _in_mem = False -class TestAsciiMemShelve(TestShelveBase): - _args={'protocol':0} - _in_mem = True -class TestBinaryMemShelve(TestShelveBase): - _args={'protocol':1} - _in_mem = True -class TestProto2MemShelve(TestShelveBase): - _args={'protocol':2} - _in_mem = True - -def test_main(): - for module in dbm_iterator(): - support.run_unittest( - TestAsciiFileShelve, - TestBinaryFileShelve, - TestProto2FileShelve, - TestAsciiMemShelve, - TestBinaryMemShelve, - TestProto2MemShelve, - TestCase - ) + + def setUp(self): + dirname = os_helper.TESTFN + os.mkdir(dirname) + self.addCleanup(os_helper.rmtree, dirname) + self.base_path = os.path.join(dirname, "shelftemp.db") + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) + dbm._defaultmod = self.dbm_mod + + +from test import mapping_tests + +for proto in range(pickle.HIGHEST_PROTOCOL + 1): + bases = (TestShelveInMemBase, mapping_tests.BasicTestMappingProtocol) + name = f'TestProto{proto}MemShelve' + globals()[name] = type(name, bases, + {'_args': {'protocol': proto}}) + bases = (TestShelveFileBase, mapping_tests.BasicTestMappingProtocol) + for dbm_mod in dbm_iterator(): + assert dbm_mod.__name__.startswith('dbm.') + suffix = dbm_mod.__name__[4:] + name = f'TestProto{proto}File_{suffix}Shelve' + globals()[name] = type(name, bases, + {'dbm_mod': dbm_mod, '_args': {'protocol': proto}}) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index baabccf19f4..7c41432b82f 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,6 +3,8 @@ import shlex import string import unittest +from test.support import cpython_only +from test.support import import_helper # The original test data set was from shellwords, by Hartmut Goebel. @@ -165,14 +167,12 @@ def testSplitNone(self): with self.assertRaises(ValueError): shlex.split(None) - # TODO: RUSTPYTHON; ValueError: Error Retrieving Value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testSplitPosix(self): """Test data splitting with posix parser""" self.splitTest(self.posix_data, comments=True) - # TODO: RUSTPYTHON; ValueError: Error Retrieving Value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testCompat(self): """Test compatibility interface""" for i in range(len(self.data)): @@ -313,8 +313,7 @@ def testEmptyStringHandling(self): s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) - # TODO: RUSTPYTHON; ValueError: Error Retrieving Value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testUnicodeHandling(self): """Test punctuation_chars and whitespace_split handle unicode.""" ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24" @@ -334,6 +333,7 @@ def testQuote(self): unsafe = '"`$\\!' + unicode_sample self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(None), "''") self.assertEqual(shlex.quote(safeunquoted), safeunquoted) self.assertEqual(shlex.quote('test file name'), "'test file name'") for u in unsafe: @@ -342,6 +342,8 @@ def testQuote(self): for u in unsafe: self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + self.assertRaises(TypeError, shlex.quote, 42) + self.assertRaises(TypeError, shlex.quote, b"abc") def testJoin(self): for split_command, command in [ @@ -354,8 +356,7 @@ def testJoin(self): joined = shlex.join(split_command) self.assertEqual(joined, command) - # TODO: RUSTPYTHON; ValueError: Error Retrieving Value - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testJoinRoundtrip(self): all_data = self.data + self.posix_data for command, *split_command in all_data: @@ -371,6 +372,10 @@ def testPunctuationCharsReadOnly(self): with self.assertRaises(AttributeError): shlex_instance.punctuation_chars = False + @cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'}) + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index ad67583d438..62c80aab4b3 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,7 +10,6 @@ import os.path import errno import functools -import pathlib import subprocess import random import string @@ -23,7 +22,6 @@ unregister_unpack_format, get_unpack_formats, SameFileError, _GiveupOnFastCopy) import tarfile -import warnings import zipfile try: import posix @@ -33,7 +31,6 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN, FakePath -from test.support import warnings_helper TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" @@ -71,18 +68,17 @@ def wrap(*args, **kwargs): os.rename = builtin_rename return wrap -def write_file(path, content, binary=False): +def create_file(path, content=b''): """Write *content* to a file located at *path*. If *path* is a tuple instead of a string, os.path.join will be used to - make a path. If *binary* is true, the file will be opened in binary - mode. + make a path. """ if isinstance(path, tuple): path = os.path.join(*path) - mode = 'wb' if binary else 'w' - encoding = None if binary else "utf-8" - with open(path, mode, encoding=encoding) as fp: + if isinstance(content, str): + content = content.encode() + with open(path, 'xb') as fp: fp.write(content) def write_test_file(path, size): @@ -191,12 +187,11 @@ def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') os.mkdir(victim) - write_file(os.path.join(victim, 'somefile'), 'foo') + create_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) shutil.rmtree(victim) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; flaky') @os_helper.skip_unless_symlink def test_rmtree_fails_on_symlink_onerror(self): tmp = self.mkdtemp() @@ -244,7 +239,7 @@ def test_rmtree_works_on_symlinks(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') os.symlink(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -306,7 +301,7 @@ def test_rmtree_works_on_junctions(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') _winapi.CreateJunction(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -319,7 +314,7 @@ def test_rmtree_works_on_junctions(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - def test_rmtree_errors_onerror(self): + def test_rmtree_errors(self): # filename is guaranteed not to exist filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) @@ -328,8 +323,8 @@ def test_rmtree_errors_onerror(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -337,6 +332,19 @@ def test_rmtree_errors_onerror(self): # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) self.assertTrue(os.path.exists(filename)) + + self.assertRaises(TypeError, shutil.rmtree, None) + self.assertRaises(TypeError, shutil.rmtree, None, ignore_errors=True) + exc = TypeError if shutil.rmtree.avoids_symlink_attacks else NotImplementedError + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid') + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid', ignore_errors=True) + + def test_rmtree_errors_onerror(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + create_file(filename) errors = [] def onerror(*args): errors.append(args) @@ -352,23 +360,9 @@ def onerror(*args): self.assertEqual(errors[1][2][1].filename, filename) def test_rmtree_errors_onexc(self): - # filename is guaranteed not to exist - filename = tempfile.mktemp(dir=self.mkdtemp()) - self.assertRaises(FileNotFoundError, shutil.rmtree, filename) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - - # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") - with self.assertRaises(NotADirectoryError) as cm: - shutil.rmtree(filename) - self.assertEqual(cm.exception.filename, filename) - self.assertTrue(os.path.exists(filename)) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - self.assertTrue(os.path.exists(filename)) + create_file(filename) errors = [] def onexc(*args): errors.append(args) @@ -433,12 +427,12 @@ def check_args_to_onerror(self, func, arg, exc): else: self.assertIs(func, os.listdir) self.assertIn(arg, [TESTFN, self.child_dir_path]) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState += 1 else: self.assertEqual(func, os.rmdir) self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState = 3 @unittest.skipIf(sys.platform[:6] == 'cygwin', @@ -550,7 +544,7 @@ def raiser(fn, *args, **kwargs): os.lstat = raiser os.mkdir(TESTFN) - write_file((TESTFN, 'foo'), 'foo') + create_file((TESTFN, 'foo'), 'foo') shutil.rmtree(TESTFN) finally: os.lstat = orig_lstat @@ -561,25 +555,23 @@ def test_rmtree_uses_safe_fd_version_if_available(self): os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) self.assertTrue(shutil.rmtree.avoids_symlink_attacks) tmp_dir = self.mkdtemp() d = os.path.join(tmp_dir, 'a') os.mkdir(d) try: - real_rmtree = shutil._rmtree_safe_fd + real_open = os.open class Called(Exception): pass def _raiser(*args, **kwargs): raise Called - shutil._rmtree_safe_fd = _raiser + os.open = _raiser self.assertRaises(Called, shutil.rmtree, d) finally: - shutil._rmtree_safe_fd = real_rmtree + os.open = real_open else: - self.assertFalse(shutil._use_fd_functions) self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - @unittest.skipUnless(shutil._use_fd_functions, "requires safe rmtree") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "requires safe rmtree") def test_rmtree_fails_on_close(self): # Test that the error handler is called for failed os.close() and that # os.close() is only called once for a file descriptor. @@ -614,7 +606,7 @@ def onexc(*args): self.assertEqual(errors[1][1], dir1) self.assertEqual(close_count, 2) - @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "dir_fd is not supported") def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() victim = 'killme' @@ -623,12 +615,12 @@ def test_rmtree_with_dir_fd(self): self.addCleanup(os.close, dir_fd) os.mkdir(fullname) os.mkdir(os.path.join(fullname, 'subdir')) - write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + create_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') self.assertTrue(os.path.exists(fullname)) shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) - @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported") + @unittest.skipIf(shutil.rmtree.avoids_symlink_attacks, "dir_fd is supported") def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() with self.assertRaises(NotImplementedError): @@ -663,7 +655,7 @@ def test_rmtree_on_junction(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() + create_file(os.path.join(src, 'spam')) _winapi.CreateJunction(src, dst) self.assertRaises(OSError, shutil.rmtree, dst) shutil.rmtree(dst, ignore_errors=True) @@ -687,6 +679,73 @@ def test_rmtree_on_named_pipe(self): shutil.rmtree(TESTFN) self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def _onexc(fn, path, exc): + assert fn in (os.rmdir, os.unlink) + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + os.chmod(p, mode) + # Remove other dirs except one. + keep = next(p for p in dirs if p != path) + for p in dirs: + if p != keep: + os.rmdir(p) + # Remove other files except one. + keep = next(p for p in files if p != path) + for p in files: + if p != keep: + os.unlink(p) + + os.mkdir(TESTFN) + paths = [TESTFN] + [os.path.join(TESTFN, f'child{i}') + for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + os.mkdir(path) + for path in files: + create_file(path) + + old_modes = [os.stat(path).st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD|stat.S_IEXEC + for path in reversed(paths): + os.chmod(path, new_mode) + + try: + shutil.rmtree(TESTFN, onexc=_onexc) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + os.chmod(path, mode) + except OSError: + pass + shutil.rmtree(TESTFN) + raise + + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): @@ -695,9 +754,9 @@ def test_copytree_simple(self): dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) @@ -715,11 +774,11 @@ def test_copytree_dirs_exist_ok(self): self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, dst_dir) - write_file((src_dir, 'nonexisting.txt'), '123') + create_file((src_dir, 'nonexisting.txt'), '123') os.mkdir(os.path.join(src_dir, 'existing_dir')) os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + create_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + create_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) @@ -742,7 +801,7 @@ def test_copytree_symlinks(self): sub_dir = os.path.join(src_dir, 'sub') os.mkdir(src_dir) os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') + create_file((src_dir, 'file.txt'), 'foo') src_link = os.path.join(sub_dir, 'link') dst_link = os.path.join(dst_dir, 'sub/link') os.symlink(os.path.join(src_dir, 'file.txt'), @@ -773,16 +832,16 @@ def test_copytree_with_exclude(self): src_dir = self.mkdtemp() try: dst_dir = join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') + create_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + create_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -841,12 +900,12 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(join(src_dir)) os.mkdir(join(src_dir, 'test_dir')) os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - invokations = [] + invocations = [] def _ignore(src, names): - invokations.append(src) + invocations.append(src) self.assertIsInstance(src, str) self.assertIsInstance(names, list) self.assertEqual(len(names), len(set(names))) @@ -860,7 +919,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -871,7 +930,7 @@ def _ignore(src, names): self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) - self.assertEqual(len(invokations), 9) + self.assertEqual(len(invocations), 9) def test_copytree_retains_permissions(self): tmp_dir = self.mkdtemp() @@ -881,9 +940,9 @@ def test_copytree_retains_permissions(self): self.addCleanup(shutil.rmtree, tmp_dir) os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') + create_file((src_dir, 'permissive.txt'), '123') os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') + create_file((src_dir, 'restrictive.txt'), '456') os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) restrictive_subdir = tempfile.mkdtemp(dir=src_dir) self.addCleanup(os_helper.rmtree, restrictive_subdir) @@ -926,8 +985,7 @@ def custom_cpfun(a, b): flag = [] src = self.mkdtemp() dst = tempfile.mktemp(dir=self.mkdtemp()) - with open(os.path.join(src, 'foo'), 'w', encoding='utf-8') as f: - f.close() + create_file(os.path.join(src, 'foo')) shutil.copytree(src, dst, copy_function=custom_cpfun) self.assertEqual(len(flag), 1) @@ -962,9 +1020,9 @@ def test_copytree_named_pipe(self): def test_copytree_special_func(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -977,7 +1035,7 @@ def _copy(src, dst): def test_copytree_dangling_symlinks(self): src_dir = self.mkdtemp() valid_file = os.path.join(src_dir, 'test.txt') - write_file(valid_file, 'abc') + create_file(valid_file, 'abc') dir_a = os.path.join(src_dir, 'dir_a') os.mkdir(dir_a) for d in src_dir, dir_a: @@ -1005,8 +1063,7 @@ def test_copytree_symlink_dir(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'wb'): - pass + create_file(os.path.join(src_dir, 'real_dir', 'test.txt')) os.symlink(os.path.join(src_dir, 'real_dir'), os.path.join(src_dir, 'link_to_dir'), target_is_directory=True) @@ -1026,7 +1083,7 @@ def test_copytree_return_value(self): dst_dir = src_dir + "dest" self.addCleanup(shutil.rmtree, dst_dir, True) src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['foo'], os.listdir(rv)) @@ -1038,7 +1095,7 @@ def test_copytree_subdirectory(self): dst_dir = os.path.join(src_dir, "somevendor", "1.0") os.makedirs(src_dir) src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') + create_file(src, 'pol') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['pol'], os.listdir(rv)) @@ -1053,8 +1110,8 @@ def test_copymode_follow_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1077,35 +1134,34 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src_link, dst_link) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - @unittest.skipUnless(hasattr(os, 'lchmod') or os.name == 'nt', 'requires os.lchmod') + @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') @os_helper.skip_unless_symlink def test_copymode_symlink_to_symlink(self): - _lchmod = os.chmod if os.name == 'nt' else os.lchmod tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) os.chmod(dst, stat.S_IRWXU) - _lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) + os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) # link to link - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) old_mode = os.stat(dst).st_mode shutil.copymode(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.lstat(src_link).st_mode, os.lstat(dst_link).st_mode) self.assertEqual(os.stat(dst).st_mode, old_mode) # src link - use chmod - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src_link, dst, follow_symlinks=False) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) # dst link - use chmod - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src, dst_link, follow_symlinks=False) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) @@ -1117,8 +1173,8 @@ def test_copymode_symlink_to_symlink_wo_lchmod(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail @@ -1132,23 +1188,21 @@ def test_copystat_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') + create_file(src, 'foo') src_stat = os.stat(src) os.utime(src, (src_stat.st_atime, src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') + create_file(dst, 'bar') self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) os.symlink(src, src_link) os.symlink(dst, dst_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXO) - elif os.name == 'nt': - os.chmod(src_link, stat.S_IRWXO) if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): os.lchflags(src_link, stat.UF_NODUMP) src_link_stat = os.lstat(src_link) # follow - if hasattr(os, 'lchmod') or os.name == 'nt': + if hasattr(os, 'lchmod'): shutil.copystat(src_link, dst_link, follow_symlinks=True) self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) # don't follow @@ -1159,7 +1213,7 @@ def test_copystat_symlinks(self): # The modification times may be truncated in the new file. self.assertLessEqual(getattr(src_link_stat, attr), getattr(dst_link_stat, attr) + 1) - if hasattr(os, 'lchmod') or os.name == 'nt': + if hasattr(os, 'lchmod'): self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode) if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags) @@ -1176,8 +1230,8 @@ def test_copystat_handles_harmless_chflags_errors(self): tmpdir = self.mkdtemp() file1 = os.path.join(tmpdir, 'file1') file2 = os.path.join(tmpdir, 'file2') - write_file(file1, 'xxx') - write_file(file2, 'xxx') + create_file(file1, 'xxx') + create_file(file2, 'xxx') def make_chflags_raiser(err): ex = OSError() @@ -1203,9 +1257,9 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True): def test_copyxattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(dst, 'bar') + create_file(dst, 'bar') # no xattr == no problem shutil._copyxattr(src, dst) @@ -1219,7 +1273,7 @@ def test_copyxattr(self): os.getxattr(dst, 'user.foo')) # check errors don't affect other attrs os.remove(dst) - write_file(dst, 'bar') + create_file(dst, 'bar') os_error = OSError(errno.EPERM, 'EPERM') def _raise_on_user_foo(fname, attr, val, **kwargs): @@ -1249,15 +1303,15 @@ def _raise_on_src(fname, *, follow_symlinks=True): # test that shutil.copystat copies xattrs src = os.path.join(tmp_dir, 'the_original') srcro = os.path.join(tmp_dir, 'the_original_ro') - write_file(src, src) - write_file(srcro, srcro) + create_file(src, src) + create_file(srcro, srcro) os.setxattr(src, 'user.the_value', b'fiddly') os.setxattr(srcro, 'user.the_value', b'fiddly') os.chmod(srcro, 0o444) dst = os.path.join(tmp_dir, 'the_copy') dstro = os.path.join(tmp_dir, 'the_copy_ro') - write_file(dst, dst) - write_file(dstro, dstro) + create_file(dst, dst) + create_file(dstro, dstro) shutil.copystat(src, dst) shutil.copystat(srcro, dstro) self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') @@ -1273,13 +1327,13 @@ def test_copyxattr_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) os.setxattr(src, 'trusted.foo', b'42') os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) dst = os.path.join(tmp_dir, 'bar') dst_link = os.path.join(tmp_dir, 'qux') - write_file(dst, 'bar') + create_file(dst, 'bar') os.symlink(dst, dst_link) shutil._copyxattr(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') @@ -1292,7 +1346,7 @@ def test_copyxattr_symlinks(self): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') + create_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -1311,7 +1365,7 @@ def test_copy_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1353,7 +1407,7 @@ def test_copy2_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1387,7 +1441,7 @@ def test_copy2_xattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') + create_file(src, 'foo') os.setxattr(src, 'user.foo', b'42') shutil.copy2(src, dst) self.assertEqual( @@ -1401,7 +1455,7 @@ def test_copy_return_value(self): src_dir = self.mkdtemp() dst_dir = self.mkdtemp() src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = fn(src, dst_dir) self.assertEqual(rv, os.path.join(dst_dir, 'foo')) rv = fn(src, os.path.join(dst_dir, 'bar')) @@ -1418,7 +1472,7 @@ def _test_copy_dir(self, copy_func): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1438,7 +1492,7 @@ def test_copyfile_symlinks(self): dst = os.path.join(tmp_dir, 'dst') dst_link = os.path.join(tmp_dir, 'dst_link') link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, link) # don't follow shutil.copyfile(link, dst_link, follow_symlinks=False) @@ -1455,8 +1509,7 @@ def test_dont_copy_file_onto_link_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') try: os.link(src, dst) except PermissionError as e: @@ -1475,8 +1528,7 @@ def test_dont_copy_file_onto_symlink_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. @@ -1511,7 +1563,7 @@ def test_copyfile_return_value(self): dst_dir = self.mkdtemp() dst_file = os.path.join(dst_dir, 'bar') src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') rv = shutil.copyfile(src_file, dst_file) self.assertTrue(os.path.exists(rv)) self.assertEqual(read_file(src_file), read_file(dst_file)) @@ -1521,7 +1573,7 @@ def test_copyfile_same_file(self): # are the same. src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) # But Error should work too, to stay backward compatible. self.assertRaises(Error, shutil.copyfile, src_file, src_file) @@ -1538,7 +1590,7 @@ def test_copyfile_nonexistent_dir(self): src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) def test_copyfile_copy_dir(self): @@ -1549,7 +1601,7 @@ def test_copyfile_copy_dir(self): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1564,42 +1616,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1611,15 +1627,101 @@ def _create_files(self, base_dir='dist'): root_dir = self.mkdtemp() dist = os.path.join(root_dir, base_dir) os.makedirs(dist, exist_ok=True) - write_file((dist, 'file1'), 'xxx') - write_file((dist, 'file2'), 'xxx') + create_file((dist, 'file1'), 'xxx') + create_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - write_file((dist, 'sub', 'file3'), 'xxx') + create_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) if base_dir: - write_file((root_dir, 'outer'), 'xxx') + create_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1669,40 +1771,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1751,7 +1902,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) @@ -1872,17 +2026,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1903,10 +2059,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1915,6 +2072,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1925,10 +2083,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1937,6 +2096,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1951,7 +2111,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -1981,9 +2141,6 @@ def check_unpack_archive_with_converter(self, format, converter, **kwargs): def check_unpack_tarball(self, format): self.check_unpack_archive(format, filter='fully_trusted') self.check_unpack_archive(format, filter='data') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - self.check_unpack_archive(format) def test_unpack_archive_tar(self): self.check_unpack_tarball('tar') @@ -1996,6 +2153,10 @@ def test_unpack_archive_gztar(self): def test_unpack_archive_bztar(self): self.check_unpack_tarball('bztar') + @support.requires_zstd() + def test_unpack_archive_zstdtar(self): + self.check_unpack_tarball('zstdtar') + @support.requires_lzma() @unittest.skipIf(AIX and not _maxdataOK(), "AIX MAXDATA must be 0x20000000 or larger") def test_unpack_archive_xztar(self): @@ -2056,7 +2217,9 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) - write_file(filename, 'testing chown function') + linkname = os.path.join(dirname, "chown_link") + create_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -2077,7 +2240,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -2113,41 +2276,76 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, user, group) check_chown(dirname, uid, gid) + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) + + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) + + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) + +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = self.mkdtemp(prefix="Tmp") + temp_dir = self.mkdtemp(prefix="Tmp") + base_dir = os.path.join(temp_dir, TESTFN + '-basedir') + os.mkdir(base_dir) + self.dir = os.path.join(base_dir, TESTFN + '-dir') + os.mkdir(self.dir) + self.other_dir = os.path.join(base_dir, TESTFN + '-dir2') + os.mkdir(self.other_dir) # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. - self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp", - suffix=".Exe") - os.chmod(self.temp_file.name, stat.S_IXUSR) - self.addCleanup(self.temp_file.close) - self.dir, self.file = os.path.split(self.temp_file.name) + self.file = TESTFN + '.Exe' + self.filepath = os.path.join(self.dir, self.file) + self.create_file(self.filepath) self.env_path = self.dir self.curdir = os.curdir self.ext = ".EXE" - def to_text_type(self, s): - ''' - In this class we're testing with str, so convert s to a str - ''' - if isinstance(s, bytes): - return s.decode() - return s + to_text_type = staticmethod(os.fsdecode) + + def create_file(self, path): + create_file(path) + os.chmod(path, 0o755) + + def assertNormEqual(self, actual, expected): + self.assertEqual(os.path.normcase(actual), os.path.normcase(expected)) def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. - rv = shutil.which(self.temp_file.name, path=self.temp_dir) - self.assertEqual(rv, self.temp_file.name) + rv = shutil.which(self.filepath, path=self.other_dir) + self.assertEqual(rv, self.filepath) def test_relative_cmd(self): # When given the relative path with a directory part to an executable @@ -2155,7 +2353,7 @@ def test_relative_cmd(self): base_dir, tail_dir = os.path.split(self.dir) relpath = os.path.join(tail_dir, self.file) with os_helper.change_cwd(path=base_dir): - rv = shutil.which(relpath, path=self.temp_dir) + rv = shutil.which(relpath, path=self.other_dir) self.assertEqual(rv, relpath) # But it shouldn't be searched in PATH directories (issue #16957). with os_helper.change_cwd(path=self.dir): @@ -2166,9 +2364,8 @@ def test_relative_cmd(self): "test is for non win32") def test_cwd_non_win32(self): # Issue #16957 - base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # non-win32: shouldn't match in the current directory. self.assertIsNone(rv) @@ -2178,57 +2375,32 @@ def test_cwd_win32(self): base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory not on PATH self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", "test is for win32") def test_cwd_win32_added_before_all_other_path(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - - elsewhere_in_path_dir = base_dir / 'dir1' - elsewhere_in_path_dir.mkdir() - match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' - match_elsewhere_in_path.touch() - - exe_in_cwd = base_dir / 'hello.exe' - exe_in_cwd.touch() - - with os_helper.change_cwd(path=base_dir): - with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) - - self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) - - @unittest.skipUnless(sys.platform == "win32", - "test is for win32") - def test_pathext_match_before_path_full_match(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - dir1 = base_dir / 'dir1' - dir2 = base_dir / 'dir2' - dir1.mkdir() - dir2.mkdir() - - pathext_match = dir1 / 'hello.com.exe' - path_match = dir2 / 'hello.com' - pathext_match.touch() - path_match.touch() - - test_path = os.pathsep.join([str(dir1), str(dir2)]) - assert os.path.basename(shutil.which( - 'hello.com', path=test_path, mode = os.F_OK - )).lower() == 'hello.com.exe' + other_file_path = os.path.join(self.other_dir, self.file) + self.create_file(other_file_path) + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + with os_helper.change_cwd(path=self.other_dir): + rv = shutil.which(self.file, path=self.dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. - os.chmod(self.temp_file.name, stat.S_IREAD) - if os.access(self.temp_file.name, os.W_OK): + os.chmod(self.filepath, stat.S_IREAD) + if os.access(self.filepath, os.W_OK): self.skipTest("can't set the file read-only") rv = shutil.which(self.file, path=self.dir, mode=os.W_OK) self.assertIsNone(rv) @@ -2250,13 +2422,13 @@ def test_pathext_checking(self): # Ask for the file without the ".exe" extension, then ensure that # it gets found properly with the extension. rv = shutil.which(self.file[:-4], path=self.dir) - self.assertEqual(rv, self.temp_file.name[:-4] + self.ext) + self.assertEqual(rv, self.filepath[:-4] + self.ext) def test_environ_path(self): with os_helper.EnvironmentVarGuard() as env: env['PATH'] = self.env_path rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_environ_path_empty(self): # PATH='': no match @@ -2270,12 +2442,9 @@ def test_environ_path_empty(self): self.assertIsNone(rv) def test_environ_path_cwd(self): - expected_cwd = os.path.basename(self.temp_file.name) + expected_cwd = self.file if sys.platform == "win32": - curdir = os.curdir - if isinstance(expected_cwd, bytes): - curdir = os.fsencode(curdir) - expected_cwd = os.path.join(curdir, expected_cwd) + expected_cwd = os.path.join(self.curdir, expected_cwd) # PATH=':': explicitly looks in the current directory with os_helper.EnvironmentVarGuard() as env: @@ -2293,21 +2462,21 @@ def test_environ_path_cwd(self): def test_environ_path_missing(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] # without confstr with unittest.mock.patch('os.confstr', side_effect=ValueError, \ create=True), \ support.swap_attr(os, 'defpath', self.dir): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) # with confstr with unittest.mock.patch('os.confstr', return_value=self.dir, \ create=True), \ support.swap_attr(os, 'defpath', ''): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_empty_path(self): base_dir = os.path.dirname(self.dir) @@ -2319,56 +2488,94 @@ def test_empty_path(self): def test_empty_path_no_PATH(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] rv = shutil.which(self.file) self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - os.chmod(temp_filexyz.name, stat.S_IXUSR) - self.addCleanup(temp_filexyz.close) - - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] - + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) # Issue 40592: See https://bugs.python.org/issue40592 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_with_empty_str(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - self.addCleanup(temp_filexyz.close) + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + ';' # note the ; + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_multidot_extension(self): + ext = '.foo.bar' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_null_extension(self): + cmd = self.to_text_type(TESTFN2) + cmddot = cmd + self.to_text_type('.') + filepath = os.path.join(self.dir, cmd) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = '.xyz' + self.assertIsNone(shutil.which(cmd, path=self.dir)) + self.assertIsNone(shutil.which(cmddot, path=self.dir)) + env['PATHEXT'] = '.xyz;.' # note the . + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + env['PATHEXT'] = '.xyz;..' # multiple dots + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_extension_ends_with_dot(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + dot = self.to_text_type('.') + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + '.' + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) # cmd.exe hangs here + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + self.assertIsNone(shutil.which(cmd + dot, path=self.dir)) + self.assertIsNone(shutil.which(cmdext + dot, path=self.dir)) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_applied_on_files_in_path(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) - open(test_path, 'w').close() - os.chmod(test_path, 0o755) - - self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) + env["PATH"] = os.fsdecode(self.dir) + env["PATHEXT"] = ext + self.assertEqual(shutil.which(cmd), filepath) + self.assertEqual(shutil.which(cmdext), filepath) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') @@ -2384,49 +2591,107 @@ def test_win_path_needs_curdir(self): self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) need_curdir_mock.assert_called_once_with('dontcare') - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_preferred_for_execute(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) - - # default behavior allows a direct match if nothing in PATHEXT matches - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - - dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) - open(dot_test, 'w').close() - os.chmod(dot_test, 0o755) - - # now we have a PATHEXT match, so it take precedence - self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) + def test_same_dir_with_pathext_extension(self): + cmd = self.file # with .exe extension + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.dir, cmd2) + self.create_file(other_file_path) + + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + self.assertNormEqual(shutil.which(cmd2, path=self.dir), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=self.dir, mode=os.F_OK), + other_file_path) - # but if we don't use os.X_OK we don't change the order based off PATHEXT - # and therefore get the direct match. - self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) - - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_given_extension_preferred(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".exe2;.exe" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) + def test_same_dir_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + # pathext match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.dir, cmd) + self.create_file(other_file_path) + + # pathext match if mode contains X_OK + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + other_file_path) + self.assertNormEqual(shutil.which(self.file, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=self.dir, mode=os.F_OK), + self.filepath) - exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) - open(exe2, 'w').close() - os.chmod(exe2, 0o755) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_with_pathext_extension(self): + cmd = self.file # with .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # full match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.other_dir, cmd2) + self.create_file(other_file_path) + + # pathext match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd2, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=search_path, mode=os.F_OK), + other_file_path) + + # full match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) - # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.other_dir, cmd) + self.create_file(other_file_path) + + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the second directory + self.assertNormEqual(shutil.which(self.file, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=search_path, mode=os.F_OK), + self.filepath) + + # pathext match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) class TestWhichBytes(TestWhich): @@ -2434,18 +2699,12 @@ def setUp(self): TestWhich.setUp(self) self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) - self.temp_file.name = os.fsencode(self.temp_file.name) - self.temp_dir = os.fsencode(self.temp_dir) + self.filepath = os.fsencode(self.filepath) + self.other_dir = os.fsencode(self.other_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) - def to_text_type(self, s): - ''' - In this class we're testing with bytes, so convert s to a bytes - ''' - if isinstance(s, str): - return s.encode() - return s + to_text_type = staticmethod(os.fsencode) class TestMove(BaseTest, unittest.TestCase): @@ -2456,8 +2715,7 @@ def setUp(self): self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") + create_file(self.src_file, b"spam") def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: @@ -2483,12 +2741,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename @@ -2535,8 +2793,7 @@ def test_move_dir_altsep_to_dir(self): def test_existing_file_inside_dest_dir(self): # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass + create_file(self.dst_file) self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) def test_dont_move_dir_in_itself(self): @@ -2951,8 +3208,7 @@ def test_empty_file(self): dstname = TESTFN + 'dst' self.addCleanup(lambda: os_helper.unlink(srcname)) self.addCleanup(lambda: os_helper.unlink(dstname)) - with open(srcname, "wb"): - pass + create_file(srcname) with open(srcname, "rb") as src: with open(dstname, "wb") as dst: @@ -2985,12 +3241,8 @@ def test_filesystem_full(self): self.assertRaises(OSError, self.zerocopy_fun, src, dst) -@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported') -class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase): - PATCHPOINT = "os.sendfile" - - def zerocopy_fun(self, fsrc, fdst): - return shutil._fastcopy_sendfile(fsrc, fdst) +class _ZeroCopyFileLinuxTest(_ZeroCopyFileTest): + BLOCKSIZE_INDEX = None def test_non_regular_file_src(self): with io.BytesIO(self.FILEDATA) as src: @@ -3011,77 +3263,87 @@ def test_non_regular_file_dst(self): self.assertEqual(dst.read(), self.FILEDATA) def test_exception_on_second_call(self): - def sendfile(*args, **kwargs): + def syscall(*args, **kwargs): if not flag: flag.append(None) - return orig_sendfile(*args, **kwargs) + return orig_syscall(*args, **kwargs) else: raise OSError(errno.EBADF, "yo") flag = [] - orig_sendfile = os.sendfile - with unittest.mock.patch('os.sendfile', create=True, - side_effect=sendfile): + orig_syscall = eval(self.PATCHPOINT) + with unittest.mock.patch(self.PATCHPOINT, create=True, + side_effect=syscall): with self.get_files() as (src, dst): with self.assertRaises(OSError) as cm: - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert flag self.assertEqual(cm.exception.errno, errno.EBADF) def test_cant_get_size(self): # Emulate a case where src file size cannot be determined. # Internally bufsize will be set to a small value and - # sendfile() will be called repeatedly. + # a system call will be called repeatedly. with unittest.mock.patch('os.fstat', side_effect=OSError) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_small_chunks(self): # Force internal file size detection to be smaller than the - # actual file size. We want to force sendfile() to be called + # actual file size. We want to force a system call to be called # multiple times, also in order to emulate a src fd which gets # bigger while it is being copied. mock = unittest.mock.Mock() mock.st_size = 65536 + 1 with unittest.mock.patch('os.fstat', return_value=mock) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_big_chunk(self): # Force internal file size detection to be +100MB bigger than - # the actual file size. Make sure sendfile() does not rely on + # the actual file size. Make sure a system call does not rely on # file size value except for (maybe) a better throughput / # performance. mock = unittest.mock.Mock() mock.st_size = self.FILESIZE + (100 * 1024 * 1024) with unittest.mock.patch('os.fstat', return_value=mock) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_blocksize_arg(self): - with unittest.mock.patch('os.sendfile', + with unittest.mock.patch(self.PATCHPOINT, side_effect=ZeroDivisionError) as m: self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN, TESTFN2) - blocksize = m.call_args[0][3] + blocksize = m.call_args[0][self.BLOCKSIZE_INDEX] # Make sure file size and the block size arg passed to # sendfile() are the same. self.assertEqual(blocksize, os.path.getsize(TESTFN)) # ...unless we're dealing with a small file. os_helper.unlink(TESTFN2) - write_file(TESTFN2, b"hello", binary=True) + create_file(TESTFN2, b"hello") self.addCleanup(os_helper.unlink, TESTFN2 + '3') self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN2, TESTFN2 + '3') - blocksize = m.call_args[0][3] + blocksize = m.call_args[0][self.BLOCKSIZE_INDEX] self.assertEqual(blocksize, 2 ** 23) + +@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported') +@unittest.mock.patch.object(shutil, "_USE_CP_COPY_FILE_RANGE", False) +class TestZeroCopySendfile(_ZeroCopyFileLinuxTest, unittest.TestCase): + PATCHPOINT = "os.sendfile" + BLOCKSIZE_INDEX = 3 + + def zerocopy_fun(self, fsrc, fdst): + return shutil._fastcopy_sendfile(fsrc, fdst) + def test_file2file_not_supported(self): # Emulate a case where sendfile() only support file->socket # fds. In such a case copyfile() is supposed to skip the @@ -3104,6 +3366,29 @@ def test_file2file_not_supported(self): shutil._USE_CP_SENDFILE = True +@unittest.skipUnless(shutil._USE_CP_COPY_FILE_RANGE, "os.copy_file_range() not supported") +class TestZeroCopyCopyFileRange(_ZeroCopyFileLinuxTest, unittest.TestCase): + PATCHPOINT = "os.copy_file_range" + BLOCKSIZE_INDEX = 2 + + def zerocopy_fun(self, fsrc, fdst): + return shutil._fastcopy_copy_file_range(fsrc, fdst) + + def test_empty_file(self): + srcname = f"{TESTFN}src" + dstname = f"{TESTFN}dst" + self.addCleanup(lambda: os_helper.unlink(srcname)) + self.addCleanup(lambda: os_helper.unlink(dstname)) + with open(srcname, "wb"): + pass + + with open(srcname, "rb") as src, open(dstname, "wb") as dst: + # _fastcopy_copy_file_range gives up copying empty files due + # to a bug in older Linux. + with self.assertRaises(shutil._GiveupOnFastCopy): + self.zerocopy_fun(src, dst) + + @unittest.skipIf(not MACOS, 'macOS only') class TestZeroCopyMACOS(_ZeroCopyFileTest, unittest.TestCase): PATCHPOINT = "posix._fcopyfile" @@ -3147,6 +3432,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): @@ -3164,8 +3450,7 @@ def test_stty_match(self): expected = (int(size[1]), int(size[0])) # reversed order with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') actual = shutil.get_terminal_size() self.assertEqual(expected, actual) @@ -3173,8 +3458,7 @@ def test_stty_match(self): @unittest.skipIf(support.is_wasi, "WASI has no /dev/null") def test_fallback(self): with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') # sys.__stdout__ has no fileno() with support.swap_attr(sys, '__stdout__', None): @@ -3195,10 +3479,10 @@ class PublicAPITests(unittest.TestCase): """Ensures that the correct values are exposed in the public API.""" def test_module_all_attribute(self): - self.assertTrue(hasattr(shutil, '__all__')) + self.assertHasAttr(shutil, '__all__') target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', - 'SpecialFileError', 'ExecError', 'make_archive', + 'SpecialFileError', 'make_archive', 'get_archive_formats', 'register_archive_format', 'unregister_archive_format', 'get_unpack_formats', 'register_unpack_format', 'unregister_unpack_format', @@ -3207,6 +3491,8 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) + with self.assertWarns(DeprecationWarning): + from shutil import ExecError if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 34ef13adf33..07fc97cb6a1 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -13,9 +13,10 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python -from test.support import threading_helper try: import _testcapi except ImportError: @@ -24,8 +25,6 @@ class GenericTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_enums(self): for name in dir(signal): sig = getattr(signal, name) @@ -124,13 +123,13 @@ def __repr__(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) self.assertEqual(0, argument.repr_count) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124083: strsignal is not supported on NetBSD") def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) self.assertIn("Hangup", signal.strsignal(signal.SIGHUP)) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): dirname = os.path.dirname(__file__) @@ -193,8 +192,6 @@ def test_valid_signals(self): self.assertNotIn(signal.NSIG, s) self.assertLess(len(s), signal.NSIG) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue9324(self): # Updated for issue #10003, adding SIGBREAK handler = lambda x, y: None @@ -256,9 +253,6 @@ def test_invalid_socket(self): self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - # Emscripten does not support fstat on pipes yet. - # https://github.com/emscripten-core/emscripten/issues/16414 - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_result(self): r1, w1 = os.pipe() @@ -277,7 +271,6 @@ def test_set_wakeup_fd_result(self): self.assertEqual(signal.set_wakeup_fd(-1), w2) self.assertEqual(signal.set_wakeup_fd(-1), -1) - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_set_wakeup_fd_socket_result(self): sock1 = socket.socket() @@ -298,7 +291,6 @@ def test_set_wakeup_fd_socket_result(self): # On Windows, files are always blocking and Windows does not provide a # function to test if a socket is in non-blocking mode. @unittest.skipIf(sys.platform == "win32", "tests specific to POSIX") - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_blocking(self): rfd, wfd = os.pipe() @@ -388,7 +380,7 @@ def handler(signum, frame): except ZeroDivisionError: # An ignored exception should have been printed out on stderr err = err.getvalue() - if ('Exception ignored when trying to write to the signal wakeup fd' + if ('Exception ignored while trying to write to the signal wakeup fd' not in err): raise AssertionError(err) if ('OSError: [Errno %d]' % errno.EBADF) not in err: @@ -577,7 +569,7 @@ def handler(signum, frame): signal.raise_signal(signum) err = err.getvalue() - if ('Exception ignored when trying to {action} to the signal wakeup fd' + if ('Exception ignored while trying to {action} to the signal wakeup fd' not in err): raise AssertionError(err) """.format(action=action) @@ -647,7 +639,7 @@ def handler(signum, frame): "buffer" % written) # By default, we get a warning when a signal arrives - msg = ('Exception ignored when trying to {action} ' + msg = ('Exception ignored while trying to {action} ' 'to the signal wakeup fd') signal.set_wakeup_fd(write.fileno()) @@ -703,7 +695,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -751,7 +743,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -763,8 +755,6 @@ def handler(signum, frame): % (exitcode, stdout)) return (exitcode == 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_without_siginterrupt(self): # If a signal handler is installed and siginterrupt is not called # at all, when that signal arrives, it interrupts a syscall that's in @@ -772,8 +762,6 @@ def test_without_siginterrupt(self): interrupted = self.readpipe_interrupted(None) self.assertTrue(interrupted) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_siginterrupt_on(self): # If a signal handler is installed and siginterrupt is called with # a true value for the second argument, when that signal arrives, it @@ -786,7 +774,7 @@ def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) @@ -826,8 +814,6 @@ def sig_prof(self, *args): self.hndl_called = True signal.setitimer(signal.ITIMER_PROF, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_exc(self): # XXX I'm assuming -1 is an invalid itimer, but maybe some platform # defines it ? @@ -837,27 +823,23 @@ def test_itimer_exc(self): self.assertRaises(signal.ItimerError, signal.setitimer, signal.ITIMER_REAL, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_real(self): self.itimer = signal.ITIMER_REAL signal.setitimer(self.itimer, 1.0) signal.pause() self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL signal.signal(signal.SIGVTALRM, self.sig_vtalrm) - signal.setitimer(self.itimer, 0.3, 0.2) + signal.setitimer(self.itimer, 0.001, 0.001) for _ in support.busy_retry(support.LONG_TIMEOUT): # use up some virtual time by doing real work - _ = pow(12345, 67890, 10000019) + _ = sum(i * i for i in range(10**5)) if signal.getitimer(self.itimer) == (0.0, 0.0): # sig_vtalrm handler stopped this itimer break @@ -867,8 +849,6 @@ def test_itimer_virtual(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_prof(self): self.itimer = signal.ITIMER_PROF signal.signal(signal.SIGPROF, self.sig_prof) @@ -876,7 +856,7 @@ def test_itimer_prof(self): for _ in support.busy_retry(support.LONG_TIMEOUT): # do some work - _ = pow(12345, 67890, 10000019) + _ = sum(i * i for i in range(10**5)) if signal.getitimer(self.itimer) == (0.0, 0.0): # sig_prof handler stopped this itimer break @@ -886,8 +866,6 @@ def test_itimer_prof(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setitimer_tiny(self): # bpo-30807: C setitimer() takes a microsecond-resolution interval. # Check that float -> timeval conversion doesn't round @@ -1344,15 +1322,18 @@ def test_stress_delivery_simultaneous(self): def handler(signum, frame): sigs.append(signum) - self.setsig(signal.SIGUSR1, handler) + # On Android, SIGUSR1 is unreliable when used in close proximity to + # another signal – see Android/testbed/app/src/main/python/main.py. + # So we use a different signal. + self.setsig(signal.SIGUSR2, handler) self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during - # initial processing of SIGUSR1. + # initial processing of SIGUSR2. signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) - os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR2) expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing @@ -1364,8 +1345,8 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @unittest.skip("TODO: RUSTPYTHON; hang") - @unittest.skipIf(sys.platform == "darwin", "crashes due to system bug (FB13453490)") + @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() @@ -1432,8 +1413,6 @@ def test_sigint(self): with self.assertRaises(KeyboardInterrupt): signal.raise_signal(signal.SIGINT) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform != "win32", "Windows specific test") def test_invalid_argument(self): try: @@ -1457,8 +1436,7 @@ def handler(a, b): signal.raise_signal(signal.SIGINT) self.assertTrue(is_ok) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test__thread_interrupt_main(self): # See https://github.com/python/cpython/issues/102397 code = """if 1: diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index df279bd9652..01951e6247b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -8,6 +8,7 @@ import test.support from test import support from test.support.script_helper import assert_python_ok +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -308,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertTrue(site.getuserbase().startswith('xoxo')) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -319,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertTrue(user_site.startswith(site.USER_BASE)) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -362,11 +362,10 @@ def test_no_home_directory(self): environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertTrue(user_base.startswith('~' + os.sep)) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertTrue(user_site.startswith(user_base)) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -515,7 +514,7 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: @@ -578,10 +577,20 @@ def test_license_exists_at_url(self): code = e.code self.assertEqual(code, 200, msg="Can't find " + url) + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports("site", [ + "io", + "locale", + "traceback", + "atexit", + "warnings", + "textwrap", + ]) + class StartupImportTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) @@ -843,12 +852,15 @@ def get_excepted_output(self, *args): return 10, None def invoke_command_line(self, *args): - args = ["-m", "site", *args] + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) with EnvironmentVarGuard() as env: env["PYTHONUTF8"] = "1" env["PYTHONIOENCODING"] = "utf-8" - proc = spawn_python(*args, text=True, env=env, + proc = spawn_python(*cmd_args, text=True, env=env, encoding='utf-8', errors='replace') output = kill_python(proc) diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 9b787950fc2..fb3ea34d766 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -17,6 +17,7 @@ import threading import unittest +import unittest.mock as mock from test import support, mock_socket from test.support import hashlib_helper from test.support import socket_helper @@ -350,7 +351,7 @@ def testVRFY(self): timeout=support.LOOPBACK_TIMEOUT) self.addCleanup(smtp.close) expected = (252, b'Cannot VRFY user, but will accept message ' + \ - b'and attempt delivery') + b'and attempt delivery') self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) smtp.quit() @@ -371,7 +372,7 @@ def testHELP(self): timeout=support.LOOPBACK_TIMEOUT) self.addCleanup(smtp.close) self.assertEqual(smtp.help(), b'Supported commands: EHLO HELO MAIL ' + \ - b'RCPT DATA RSET NOOP QUIT VRFY') + b'RCPT DATA RSET NOOP QUIT VRFY') smtp.quit() def testSend(self): @@ -527,7 +528,7 @@ def testSendMessageWithAddresses(self): smtp.quit() # make sure the Bcc header is still in the message. self.assertEqual(m['Bcc'], 'John Root <root@localhost>, "Dinsdale" ' - '<warped@silly.walks.com>') + '<warped@silly.walks.com>') self.client_evt.set() self.serv_evt.wait() @@ -766,7 +767,7 @@ def tearDown(self): def testFailingHELO(self): self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, - HOST, self.port, 'localhost', 3) + HOST, self.port, 'localhost', 3) class TooLongLineTests(unittest.TestCase): @@ -804,14 +805,14 @@ def testLineTooLong(self): sim_users = {'Mr.A@somewhere.com':'John A', 'Ms.B@xn--fo-fka.com':'Sally B', 'Mrs.C@somewhereesle.com':'Ruth C', - } + } sim_auth = ('Mr.A@somewhere.com', 'somepassword') sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], 'list-2':['Ms.B@xn--fo-fka.com',], - } + } # Simulated SMTP channel & server class ResponseException(Exception): pass @@ -830,6 +831,7 @@ class SimSMTPChannel(smtpd.SMTPChannel): def __init__(self, extra_features, *args, **kw): self._extrafeatures = ''.join( [ "250-{0}\r\n".format(x) for x in extra_features ]) + self.all_received_lines = [] super(SimSMTPChannel, self).__init__(*args, **kw) # AUTH related stuff. It would be nice if support for this were in smtpd. @@ -844,6 +846,7 @@ def found_terminator(self): self.smtp_state = self.COMMAND self.push('%s %s' % (e.smtp_code, e.smtp_error)) return + self.all_received_lines.append(self.received_lines) super().found_terminator() @@ -924,11 +927,14 @@ def _auth_cram_md5(self, arg=None): except ValueError as e: self.push('535 Splitting response {!r} into user and password ' 'failed: {}'.format(logpass, e)) - return False - valid_hashed_pass = hmac.HMAC( - sim_auth[1].encode('ascii'), - self._decode_base64(sim_cram_md5_challenge).encode('ascii'), - 'md5').hexdigest() + return + pwd = sim_auth[1].encode('ascii') + msg = self._decode_base64(sim_cram_md5_challenge).encode('ascii') + try: + valid_hashed_pass = hmac.HMAC(pwd, msg, 'md5').hexdigest() + except ValueError: + self.push('504 CRAM-MD5 is not supported') + return self._authenticated(user, hashed_pass == valid_hashed_pass) # end AUTH related stuff. @@ -1170,8 +1176,6 @@ def auth_buggy(challenge=None): finally: smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_CRAM_MD5(self): self.serv.add_feature("AUTH CRAM-MD5") @@ -1181,8 +1185,39 @@ def testAUTH_CRAM_MD5(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @mock.patch("hmac.HMAC") + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked(self, hmac_constructor): + # CRAM-MD5 is the only "known" method by the server, + # but it is not supported by the client. In particular, + # no challenge will ever be sent. + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + msg = re.escape("No suitable authentication method found.") + with self.assertRaisesRegex(smtplib.SMTPException, msg): + smtp.login(sim_auth[0], sim_auth[1]) + hmac_constructor.assert_not_called() # call has been bypassed + + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked_and_fallback(self): + # Test that PLAIN is tried after CRAM-MD5 failed + self.serv.add_feature("AUTH CRAM-MD5 PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with ( + mock.patch.object(smtp, "auth_cram_md5") as smtp_auth_cram_md5, + mock.patch.object( + smtp, "auth_plain", wraps=smtp.auth_plain + ) as smtp_auth_plain + ): + resp = smtp.login(sim_auth[0], sim_auth[1]) + smtp_auth_plain.assert_called_once() + smtp_auth_cram_md5.assert_not_called() # no call to HMAC constructor + self.assertEqual(resp, (235, b'Authentication Succeeded')) + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_multiple(self): # Test that multiple authentication methods are tried. @@ -1193,8 +1228,6 @@ def testAUTH_multiple(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_auth_function(self): supported = {'PLAIN', 'LOGIN'} try: @@ -1354,6 +1387,18 @@ def test_name_field_not_included_in_envelop_addresses(self): self.assertEqual(self.serv._addresses['from'], 'michael@example.com') self.assertEqual(self.serv._addresses['tos'], ['rene@example.com']) + def test_lowercase_mail_from_rcpt_to(self): + m = 'A test message' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + + smtp.sendmail('John', 'Sally', m) + + self.assertIn(['mail from:<John> size=14'], self.serv._SMTPchannel.all_received_lines) + self.assertIn(['rcpt to:<Sally>'], self.serv._SMTPchannel.all_received_lines) + class SimSMTPUTF8Server(SimSMTPServer): @@ -1372,7 +1417,7 @@ def handle_accepted(self, conn, addr): ) def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, - rcpt_options=None): + rcpt_options=None): self.last_peer = peer self.last_mailfrom = mailfrom self.last_rcpttos = rcpttos diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index be25e961f74..d765746987b 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -2,6 +2,7 @@ from test import support from test.support import import_helper from test.support import socket_helper +import os import smtplib import socket @@ -9,6 +10,8 @@ support.requires("network") +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + def check_ssl_verifiy(host, port): context = ssl.create_default_context() with socket.create_connection((host, port)) as sock: @@ -22,7 +25,7 @@ def check_ssl_verifiy(host, port): class SmtpTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 587 def test_connect_starttls(self): @@ -44,7 +47,7 @@ def test_connect_starttls(self): class SmtpSSLTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 465 def test_connect(self): @@ -87,4 +90,4 @@ def test_connect_using_sslcontext_verified(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 5ccbfa7ff83..e792d4f30a9 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,9 +1,10 @@ import unittest +from unittest import mock from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper - +from test.support import ( + cpython_only, is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) +from test.support.import_helper import ensure_lazy_imports import _thread as thread import array import contextlib @@ -28,6 +29,7 @@ import threading import time import traceback +import warnings from weakref import proxy try: import multiprocessing @@ -37,6 +39,10 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None support.requires_working_socket(module=True) @@ -46,6 +52,7 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +SOLARIS = sys.platform.startswith("sunos") WSL = "microsoft-standard-WSL" in platform.release() try: @@ -53,6 +60,35 @@ except ImportError: _socket = None +def skipForRefleakHuntinIf(condition, issueref): + if not condition: + def decorator(f): + f.client_skip = lambda f: f + return f + + else: + def decorator(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + raise unittest.SkipTest(f"ignore while hunting for refleaks, see {issueref}") + + return f(*args, **kwds) + + def client_skip(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + return + + return f(*args, **kwds) + + return wrapper + wrapper.client_skip = client_skip + return wrapper + + return decorator + def get_cid(): if fcntl is None: return None @@ -128,8 +164,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -145,6 +181,17 @@ def _have_socket_bluetooth(): return True +def _have_socket_bluetooth_l2cap(): + """Check whether BTPROTO_L2CAP sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) + except (AttributeError, OSError): + return False + else: + s.close() + return True + + def _have_socket_hyperv(): """Check whether AF_HYPERV sockets are supported on this host.""" try: @@ -166,6 +213,24 @@ def socket_setdefaulttimeout(timeout): socket.setdefaulttimeout(old_timeout) +@contextlib.contextmanager +def downgrade_malformed_data_warning(): + # This warning happens on macos and win, but does not always happen on linux. + if sys.platform not in {"win32", "darwin"}: + yield + return + + with warnings.catch_warnings(): + # TODO: gh-110012, we should investigate why this warning is happening + # and fix it properly. + warnings.filterwarnings( + action="always", + message="received malformed or improperly-truncated ancillary data", + category=RuntimeWarning, + ) + yield + + HAVE_SOCKET_CAN = _have_socket_can() HAVE_SOCKET_CAN_ISOTP = _have_socket_can_isotp() @@ -180,15 +245,26 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() +HAVE_SOCKET_BLUETOOTH_L2CAP = _have_socket_bluetooth_l2cap() + HAVE_SOCKET_HYPERV = _have_socket_hyperv() # Size in bytes of the int type SIZEOF_INT = array.array("i").itemsize +class TestLazyImport(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("socket", {"array", "selectors"}) + + class SocketTCPTest(unittest.TestCase): def setUp(self): @@ -485,8 +561,8 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") +@unittest.skipUnless(get_cid() != 2, # VMADDR_CID_HOST + "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -508,10 +584,16 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): - msg = self.conn.recv(1024) + try: + msg = self.conn.recv(1024) + except PermissionError as exc: + self.skipTest(repr(exc)) self.assertEqual(msg, MSG) def _testStream(self): @@ -556,19 +638,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): unittest.TestCase.__init__(self, methodName=methodName) ThreadableTest.__init__(self) + self.cli = None + self.serv = None + + def socketpair(self): + # To be overridden by some child classes. + return socket.socketpair() def setUp(self): - self.serv, self.cli = socket.socketpair() + self.serv, self.cli = self.socketpair() def tearDown(self): - self.serv.close() + if self.serv: + self.serv.close() self.serv = None def clientSetUp(self): pass def clientTearDown(self): - self.cli.close() + if self.cli: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -821,8 +911,7 @@ def requireSocket(*args): class GeneralModuleTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; gc.is_tracked not implemented @unittest.skipUnless(_socket is not None, 'need _socket module') def test_socket_type(self): self.assertTrue(gc.is_tracked(_socket.socket)) @@ -886,8 +975,7 @@ def testSocketError(self): with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): raise socket.gaierror - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; error message format differs def testSendtoErrors(self): # Testing that sendto doesn't mask failures. See #10169. s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -973,8 +1061,6 @@ def test_socket_methods(self): if not hasattr(socket.socket, name): self.fail(f"socket method {name} is missing") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.platform == 'darwin', 'macOS specific test') @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') def test3542SocketOptions(self): @@ -1002,9 +1088,7 @@ def test3542SocketOptions(self): 'IPV6_USE_MIN_MTU', } for opt in opts: - self.assertTrue( - hasattr(socket, opt), f"Missing RFC3542 socket option '{opt}'" - ) + self.assertHasAttr(socket, opt) def testHostnameRes(self): # Testing hostname resolution mechanisms @@ -1073,6 +1157,7 @@ def test_sethostname(self): @unittest.skipUnless(hasattr(socket, 'if_nameindex'), 'socket.if_nameindex() not available.') + @support.skip_android_selinux('if_nameindex') def testInterfaceNameIndex(self): interfaces = socket.if_nameindex() for index, name in interfaces: @@ -1087,11 +1172,16 @@ def testInterfaceNameIndex(self): self.assertIsInstance(_name, str) self.assertEqual(name, _name) + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u32 @unittest.skipUnless(hasattr(socket, 'if_indextoname'), 'socket.if_indextoname() not available.') + @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) - self.assertRaises(OverflowError, socket.if_indextoname, -1) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + + self.assertRaises(ValueError, socket.if_indextoname, -1) self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') if hasattr(socket, 'if_nameindex'): @@ -1108,9 +1198,13 @@ def testInvalidInterfaceIndexToName(self): @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') + @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') @@ -1146,23 +1240,24 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) - @support.cpython_only + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u16 def testNtoHErrors(self): - import _testcapi s_good_values = [0, 1, 2, 0xffff] l_good_values = s_good_values + [0xffffffff] - l_bad_values = [-1, -2, 1<<32, 1<<1000] - s_bad_values = ( - l_bad_values + - [_testcapi.INT_MIN-1, _testcapi.INT_MAX+1] + - [1 << 16, _testcapi.INT_MAX] - ) + neg_values = [-1, -2, -(1<<15)-1, -(1<<31)-1, -(1<<63)-1, -1<<1000] + l_bad_values = [1<<32, 1<<1000] + s_bad_values = l_bad_values + [1 << 16, (1<<31)-1, 1<<31] for k in s_good_values: socket.ntohs(k) socket.htons(k) for k in l_good_values: socket.ntohl(k) socket.htonl(k) + for k in neg_values: + self.assertRaises(ValueError, socket.ntohs, k) + self.assertRaises(ValueError, socket.htons, k) + self.assertRaises(ValueError, socket.ntohl, k) + self.assertRaises(ValueError, socket.htonl, k) for k in s_bad_values: self.assertRaises(OverflowError, socket.ntohs, k) self.assertRaises(OverflowError, socket.htons, k) @@ -1175,8 +1270,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -1191,9 +1289,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1204,8 +1301,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1503,14 +1601,12 @@ def test_getsockaddrarg(self): break @unittest.skipUnless(os.name == "nt", "Windows specific") - # TODO: RUSTPYTHON, windows ioctls - @unittest.expectedFailure def test_sock_ioctl(self): - self.assertTrue(hasattr(socket.socket, 'ioctl')) - self.assertTrue(hasattr(socket, 'SIO_RCVALL')) - self.assertTrue(hasattr(socket, 'RCVALL_ON')) - self.assertTrue(hasattr(socket, 'RCVALL_OFF')) - self.assertTrue(hasattr(socket, 'SIO_KEEPALIVE_VALS')) + self.assertHasAttr(socket.socket, 'ioctl') + self.assertHasAttr(socket, 'SIO_RCVALL') + self.assertHasAttr(socket, 'RCVALL_ON') + self.assertHasAttr(socket, 'RCVALL_OFF') + self.assertHasAttr(socket, 'SIO_KEEPALIVE_VALS') s = socket.socket() self.addCleanup(s.close) self.assertRaises(ValueError, s.ioctl, -1, None) @@ -1519,8 +1615,6 @@ def test_sock_ioctl(self): @unittest.skipUnless(os.name == "nt", "Windows specific") @unittest.skipUnless(hasattr(socket, 'SIO_LOOPBACK_FAST_PATH'), 'Loopback fast path support required for this test') - # TODO: RUSTPYTHON, AttributeError: 'socket' object has no attribute 'ioctl' - @unittest.expectedFailure def test_sio_loopback_fast_path(self): s = socket.socket() self.addCleanup(s.close) @@ -1554,9 +1648,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -1602,11 +1695,13 @@ def testGetaddrinfo(self): flags=socket.AI_PASSIVE) self.assertEqual(a, b) # Issue #6697. - # XXX RUSTPYTHON TODO: surrogates in str - # self.assertRaises(UnicodeEncodeError, socket.getaddrinfo, 'localhost', '\uD800') + self.assertRaises(UnicodeEncodeError, socket.getaddrinfo, 'localhost', '\uD800') - # Issue 17269: test workaround for OS X platform bug segfault if hasattr(socket, 'AI_NUMERICSERV'): + self.assertRaises(socket.gaierror, socket.getaddrinfo, "localhost", "http", + flags=socket.AI_NUMERICSERV) + + # Issue 17269: test workaround for OS X platform bug segfault try: # The arguments here are undefined and the call may succeed # or fail. All we care here is that it doesn't segfault. @@ -1615,8 +1710,7 @@ def testGetaddrinfo(self): except socket.gaierror: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1634,7 +1728,7 @@ def test_getaddrinfo_int_port_overflow(self): try: socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) except OverflowError: - # Platforms differ as to what values consitute a getaddrinfo() error + # Platforms differ as to what values constitute a getaddrinfo() error # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows # silently accepts such huge "port" aka "service" numeric values. self.fail("Either no error or socket.gaierror expected.") @@ -1669,7 +1763,6 @@ def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) - @unittest.skip("TODO: RUSTPYTHON: flaky on CI?") @unittest.skipUnless(support.is_resource_enabled('network'), 'network is not enabled') def test_idna(self): @@ -1724,8 +1817,6 @@ def test_sendall_interrupted(self): def test_sendall_interrupted_with_timeout(self): self.check_sendall_interrupted(True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dealloc_warn(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) r = repr(sock) @@ -1813,6 +1904,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -1845,6 +1937,7 @@ def test_getfqdn_filter_localhost(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getaddrinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface (Linux, Mac OS X) (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1878,6 +1971,7 @@ def test_getaddrinfo_ipv6_scopeid_numeric(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getnameinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface. (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1905,7 +1999,6 @@ def test_str_for_enums(self): self.assertEqual(str(s.family), str(s.family.value)) self.assertEqual(str(s.type), str(s.type.value)) - @unittest.expectedFailureIf(sys.platform.startswith("linux"), "TODO: RUSTPYTHON, AssertionError: 526337 != <SocketKind.SOCK_STREAM: 1>") def test_socket_consistent_sock_type(self): SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) SOCK_CLOEXEC = getattr(socket, 'SOCK_CLOEXEC', 0) @@ -2052,8 +2145,6 @@ def test_socket_fileno_requires_socket_fd(self): fileno=afile.fileno()) self.assertEqual(cm.exception.errno, errno.ENOTSOCK) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_addressfamily_enum(self): import _socket, enum CheckedAddressFamily = enum._old_convert_( @@ -2063,8 +2154,6 @@ def test_addressfamily_enum(self): ) enum._test_simple_enum(CheckedAddressFamily, socket.AddressFamily) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_socketkind_enum(self): import _socket, enum CheckedSocketKind = enum._old_convert_( @@ -2074,8 +2163,6 @@ def test_socketkind_enum(self): ) enum._test_simple_enum(CheckedSocketKind, socket.SocketKind) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_msgflag_enum(self): import _socket, enum CheckedMsgFlag = enum._old_convert_( @@ -2085,8 +2172,6 @@ def test_msgflag_enum(self): ) enum._test_simple_enum(CheckedMsgFlag, socket.MsgFlag) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_addressinfo_enum(self): import _socket, enum CheckedAddressInfo = enum._old_convert_( @@ -2106,8 +2191,6 @@ def testCrucialConstants(self): @unittest.skipUnless(hasattr(socket, "CAN_BCM"), 'socket.CAN_BCM required for this test.') - # TODO: RUSTPYTHON, AttributeError: module 'socket' has no attribute 'CAN_BCM_TX_SETUP' - @unittest.expectedFailure def testBCMConstants(self): socket.CAN_BCM @@ -2148,16 +2231,12 @@ def testCreateBCMSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: pass - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def testBindAny(self): with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: address = ('', ) s.bind(address) self.assertEqual(s.getsockname(), address) - # TODO: RUSTPYTHON, AssertionError: "interface name too long" does not match "bind(): bad family" - @unittest.expectedFailure def testTooLongInterfaceName(self): # most systems limit IFNAMSIZ to 16, take 1024 to be sure with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: @@ -2300,16 +2379,14 @@ def testCreateISOTPSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: pass - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: AF_CAN address must be a tuple (interface,) or (interface, addr) def testTooLongInterfaceName(self): # most systems limit IFNAMSIZ to 16, take 1024 to be sure with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: with self.assertRaisesRegex(OSError, 'interface name too long'): s.bind(('x' * 1024, 1, 2)) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: AF_CAN address must be a tuple (interface,) or (interface, addr) def testBind(self): try: with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: @@ -2331,8 +2408,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.interface = "vcan0" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - J1939 constants not fully implemented @unittest.skipUnless(hasattr(socket, "CAN_J1939"), 'socket.CAN_J1939 required for this test.') def testJ1939Constants(self): @@ -2374,8 +2450,7 @@ def testCreateJ1939Socket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939) as s: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_CAN J1939 address format not fully implemented def testBind(self): try: with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939) as s: @@ -2514,6 +2589,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID @@ -2549,23 +2625,88 @@ def testSocketBufferSize(self): socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE)) -@unittest.skipUnless(HAVE_SOCKET_BLUETOOTH, +@unittest.skipUnless(hasattr(socket, 'AF_BLUETOOTH'), 'Bluetooth sockets required for this test.') class BasicBluetoothTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'socket' has no attribute 'BTPROTO_RFCOMM' def testBluetoothConstants(self): socket.BDADDR_ANY socket.BDADDR_LOCAL socket.AF_BLUETOOTH socket.BTPROTO_RFCOMM + socket.SOL_RFCOMM + + if sys.platform == "win32": + socket.SO_BTH_ENCRYPT + socket.SO_BTH_MTU + socket.SO_BTH_MTU_MAX + socket.SO_BTH_MTU_MIN if sys.platform != "win32": socket.BTPROTO_HCI socket.SOL_HCI socket.BTPROTO_L2CAP + socket.SOL_L2CAP + socket.BTPROTO_SCO + socket.SOL_SCO + socket.HCI_DATA_DIR + + if sys.platform == "linux": + socket.SOL_BLUETOOTH + socket.HCI_DEV_NONE + socket.HCI_CHANNEL_RAW + socket.HCI_CHANNEL_USER + socket.HCI_CHANNEL_MONITOR + socket.HCI_CHANNEL_CONTROL + socket.HCI_CHANNEL_LOGGING + socket.HCI_TIME_STAMP + socket.BT_SECURITY + socket.BT_SECURITY_SDP + socket.BT_FLUSHABLE + socket.BT_POWER + socket.BT_CHANNEL_POLICY + socket.BT_CHANNEL_POLICY_BREDR_ONLY + if hasattr(socket, 'BT_PHY'): + socket.BT_PHY_BR_1M_1SLOT + if hasattr(socket, 'BT_MODE'): + socket.BT_MODE_BASIC + if hasattr(socket, 'BT_VOICE'): + socket.BT_VOICE_TRANSPARENT + socket.BT_VOICE_CVSD_16BIT + socket.L2CAP_LM + socket.L2CAP_LM_MASTER + socket.L2CAP_LM_AUTH + + if sys.platform in ("linux", "freebsd"): + socket.BDADDR_BREDR + socket.BDADDR_LE_PUBLIC + socket.BDADDR_LE_RANDOM + socket.HCI_FILTER + + if sys.platform.startswith(("freebsd", "netbsd", "dragonfly")): + socket.SO_L2CAP_IMTU + socket.SO_L2CAP_FLUSH + socket.SO_RFCOMM_MTU + socket.SO_RFCOMM_FC_INFO + socket.SO_SCO_MTU + + if sys.platform == "freebsd": + socket.SO_SCO_CONNINFO + + if sys.platform.startswith(("netbsd", "dragonfly")): + socket.SO_HCI_EVT_FILTER + socket.SO_HCI_PKT_FILTER + socket.SO_L2CAP_IQOS + socket.SO_L2CAP_LM + socket.L2CAP_LM_AUTH + socket.SO_RFCOMM_LM + socket.RFCOMM_LM_AUTH + socket.SO_SCO_HANDLE - if not sys.platform.startswith("freebsd"): - socket.BTPROTO_SCO +@unittest.skipUnless(HAVE_SOCKET_BLUETOOTH, + 'Bluetooth sockets required for this test.') +class BluetoothTest(unittest.TestCase): def testCreateRfcommSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: @@ -2581,12 +2722,193 @@ def testCreateHciSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: pass - @unittest.skipIf(sys.platform == "win32" or sys.platform.startswith("freebsd"), - "windows and freebsd do not support SCO sockets") + @unittest.skipIf(sys.platform == "win32", "windows does not support SCO sockets") def testCreateScoSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: pass + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindLeAttL2capSocket(self): + BDADDR_LE_PUBLIC = support.get_attribute(socket, 'BDADDR_LE_PUBLIC') + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # ATT is the only CID allowed in userspace by the Linux kernel + CID_ATT = 4 + f.bind((socket.BDADDR_ANY, 0, CID_ATT, BDADDR_LE_PUBLIC)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, 0, CID_ATT, BDADDR_LE_PUBLIC)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindLePsmL2capSocket(self): + BDADDR_LE_RANDOM = support.get_attribute(socket, 'BDADDR_LE_RANDOM') + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # First user PSM in LE L2CAP + psm = 0x80 + f.bind((socket.BDADDR_ANY, psm, 0, BDADDR_LE_RANDOM)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, psm, 0, BDADDR_LE_RANDOM)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindBrEdrL2capSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # First user PSM in BR/EDR L2CAP + psm = 0x1001 + f.bind((socket.BDADDR_ANY, psm)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, psm)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBadL2capAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY, 0, 0, 0, 0)) + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + f.bind(socket.BDADDR_ANY) + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY.encode(), 0x1001)) + with self.assertRaises(OSError): + f.bind(('\ud812', 0x1001)) + + def testBindRfcommSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + channel = 0 + try: + s.bind((socket.BDADDR_ANY, channel)) + except OSError as err: + if sys.platform == 'win32' and err.winerror == 10050: + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, (mock.ANY, channel)) + self.assertRegex(addr[0], r'(?i)[0-9a-f]{2}(?::[0-9a-f]{2}){4}') + if sys.platform != 'win32': + self.assertEqual(addr, (socket.BDADDR_ANY, channel)) + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + s.bind(addr) + addr2 = s.getsockname() + self.assertEqual(addr2, addr) + + def testBadRfcommAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + channel = 0 + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY.encode(), channel)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY, channel, 0)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY + '\0', channel)) + with self.assertRaises(OSError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind(('invalid', channel)) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_HCI'), 'Bluetooth HCI sockets required for this test') + def testBindHciSocket(self): + if sys.platform.startswith(('netbsd', 'dragonfly', 'freebsd')): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + s.bind(socket.BDADDR_ANY) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + else: + dev = 0 + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + try: + s.bind((dev,)) + except OSError as err: + if err.errno in (errno.EINVAL, errno.ENODEV): + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('integer'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + s.bind(dev) + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('channel=HCI_CHANNEL_RAW'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + channel = socket.HCI_CHANNEL_RAW + s.bind((dev, channel)) + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('channel=HCI_CHANNEL_USER'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + channel = socket.HCI_CHANNEL_USER + try: + s.bind((dev, channel)) + except OSError as err: + # Needs special permissions. + if err.errno in (errno.EPERM, errno.EBUSY, errno.ERFKILL): + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, (dev, channel)) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_HCI'), 'Bluetooth HCI sockets required for this test') + def testBadHciAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + if sys.platform.startswith(('netbsd', 'dragonfly', 'freebsd')): + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY.encode()) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY + '\0') + with self.assertRaises((ValueError, OSError)): + s.bind(socket.BDADDR_ANY + ' '*100) + with self.assertRaises(OSError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind('invalid') + with self.assertRaises(OSError): + s.bind(b'invalid') + else: + dev = 0 + with self.assertRaises(OSError): + s.bind(()) + with self.assertRaises(OSError): + s.bind((dev, socket.HCI_CHANNEL_RAW, 0, 0)) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY.encode()) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_SCO'), 'Bluetooth SCO sockets required for this test') + def testBindScoSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + s.bind(socket.BDADDR_ANY) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + s.bind(socket.BDADDR_ANY.encode()) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_SCO'), 'Bluetooth SCO sockets required for this test') + def testBadScoAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY.encode(),)) + with self.assertRaises(ValueError): + s.bind(socket.BDADDR_ANY + '\0') + with self.assertRaises(ValueError): + s.bind(socket.BDADDR_ANY.encode() + b'\0') + with self.assertRaises(UnicodeEncodeError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind('invalid') + with self.assertRaises(OSError): + s.bind(b'invalid') + @unittest.skipUnless(HAVE_SOCKET_HYPERV, 'Hyper-V sockets required for this test.') @@ -2717,22 +3039,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -3203,7 +3532,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -3575,6 +3904,10 @@ def testCMSG_SPACE(self): # Test CMSG_SPACE() with various valid and invalid values, # checking the assumptions used by sendmsg(). toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + if SOLARIS and platform.processor() == "sparc": + # On Solaris SPARC, number of bytes returned by socket.CMSG_SPACE + # increases at different lengths; see gh-91214. + toobig -= 3 values = list(range(257)) + list(range(toobig - 257, toobig)) last = socket.CMSG_SPACE(0) @@ -3720,7 +4053,8 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3731,7 +4065,8 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3744,7 +4079,8 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3758,7 +4094,8 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3782,7 +4119,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. @@ -3856,6 +4193,7 @@ def checkTruncatedHeader(self, result, ignoreflags=0): self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, ignore=ignoreflags) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncNoBufSize(self): # Check that no ancillary data is received when no buffer size # is specified. @@ -3865,26 +4203,32 @@ def testCmsgTruncNoBufSize(self): # received. ignoreflags=socket.MSG_CTRUNC) + @testCmsgTruncNoBufSize.client_skip def _testCmsgTruncNoBufSize(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc0(self): # Check that no ancillary data is received when buffer size is 0. self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), ignoreflags=socket.MSG_CTRUNC) + @testCmsgTrunc0.client_skip def _testCmsgTrunc0(self): self.createAndSendFDs(1) # Check that no ancillary data is returned for various non-zero # (but still too small) buffer sizes. + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + @testCmsgTrunc1.client_skip def _testCmsgTrunc1(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -3892,13 +4236,16 @@ def testCmsgTrunc2Int(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), SIZEOF_INT * 2)) + @testCmsgTrunc2Int.client_skip def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) + @testCmsgTruncLen0Minus1.client_skip def _testCmsgTruncLen0Minus1(self): self.createAndSendFDs(1) @@ -3910,8 +4257,9 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): # mindata and maxdata bytes when received with buffer size # ancbuf, and that any complete file descriptor numbers are # valid. - msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, - len(MSG), ancbuf) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) @@ -3929,29 +4277,38 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) self.checkFDs(fds) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + @testCmsgTruncLen0.client_skip def _testCmsgTruncLen0(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + @testCmsgTruncLen0Plus1.client_skip def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) + @testCmsgTruncLen1.client_skip def _testCmsgTruncLen1(self): self.createAndSendFDs(2) + + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) + @testCmsgTruncLen2Minus1.client_skip def _testCmsgTruncLen2Minus1(self): self.createAndSendFDs(2) @@ -4253,8 +4610,9 @@ def testSingleCmsgTruncInData(self): self.serv_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_RECVHOPLIMIT, 1) self.misc_event.set() - msg, ancdata, flags, addr = self.doRecvmsg( - self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) @@ -4357,9 +4715,10 @@ def testSecondCmsgTruncInData(self): self.serv_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_RECVTCLASS, 1) self.misc_event.set() - msg, ancdata, flags, addr = self.doRecvmsg( - self.serv_sock, len(MSG), - socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) @@ -4601,7 +4960,6 @@ class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, ConnectedStreamTestMixin, UnixStreamBase): pass -@unittest.skipIf(sys.platform == "darwin", "flaky on macOS") @requireAttrs(socket.socket, "sendmsg") @requireAttrs(socket, "AF_UNIX") class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): @@ -4770,15 +5128,13 @@ def testInterruptedSendmsgTimeout(self): class TCPCloserTest(ThreadedTCPSocketTest): - def testClose(self): - conn, addr = self.serv.accept() - conn.close() + conn, _ = self.serv.accept() - sd = self.cli - read, write, err = select.select([sd], [], [], 1.0) - self.assertEqual(read, [sd]) - self.assertEqual(sd.recv(1), b'') + read, _, _ = select.select([conn], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [conn]) + self.assertEqual(conn.recv(1), b'x') + conn.close() # Calling close() many times should be safe. conn.close() @@ -4786,7 +5142,10 @@ def testClose(self): def _testClose(self): self.cli.connect((HOST, self.port)) - time.sleep(1.0) + self.cli.send(b'x') + read, _, _ = select.select([self.cli], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [self.cli]) + self.assertEqual(self.cli.recv(1), b'') class BasicSocketPairTest(SocketPairTest): @@ -4824,6 +5183,112 @@ def _testSend(self): self.assertEqual(msg, MSG) +class PurePythonSocketPairTest(SocketPairTest): + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with + # _socket.socketpair on many platforms). + def socketpair(self): + # called by super().setUp(). + try: + return socket.socketpair(socket.AF_INET6) + except OSError: + return socket.socketpair(socket.AF_INET) + + # Local imports in this class make for easy security fix backporting. + + def setUp(self): + if hasattr(_socket, "socketpair"): + self._orig_sp = socket.socketpair + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. + socket.socketpair = socket._fallback_socketpair + else: + # This platform already uses the non-OS provided version. + self._orig_sp = None + super().setUp() + + def tearDown(self): + super().tearDown() + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. + socket.socketpair = self._orig_sp + + def test_recv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _test_recv(self): + self.cli.send(MSG) + + def test_send(self): + self.serv.send(MSG) + + def _test_send(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + def test_ipv4(self): + cli, srv = socket.socketpair(socket.AF_INET) + cli.close() + srv.close() + + def _test_ipv4(self): + pass + + @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or + not hasattr(_socket, 'IPV6_V6ONLY'), + "IPV6_V6ONLY option not supported") + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') + def test_ipv6(self): + cli, srv = socket.socketpair(socket.AF_INET6) + cli.close() + srv.close() + + def _test_ipv6(self): + pass + + def test_injected_authentication_failure(self): + orig_getsockname = socket.socket.getsockname + inject_sock = None + + def inject_getsocketname(self): + nonlocal inject_sock + sockname = orig_getsockname(self) + # Connect to the listening socket ahead of the + # client socket. + if inject_sock is None: + inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + inject_sock.setblocking(False) + try: + inject_sock.connect(sockname[:2]) + except (BlockingIOError, InterruptedError): + pass + inject_sock.setblocking(True) + return sockname + + sock1 = sock2 = None + try: + socket.socket.getsockname = inject_getsocketname + with self.assertRaises(OSError): + sock1, sock2 = socket.socketpair() + finally: + socket.socket.getsockname = orig_getsockname + if inject_sock: + inject_sock.close() + if sock1: # This cleanup isn't needed on a successful test. + sock1.close() + if sock2: + sock2.close() + + def _test_injected_authentication_failure(self): + # No-op. Exists for base class threading infrastructure to call. + # We could refactor this test into its own lesser class along with the + # setUp and tearDown code to construct an ideal; it is simpler to keep + # it here and live with extra overhead one this _one_ failure test. + pass + + class NonBlockingTCPTests(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): @@ -4871,6 +5336,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi @@ -4888,8 +5354,6 @@ def testSetBlocking_overflow(self): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'test needs socket.SOCK_NONBLOCK') @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: None != 0 - @unittest.expectedFailure def testInitNonBlocking(self): # create a socket with SOCK_NONBLOCK self.serv.close() @@ -4985,6 +5449,39 @@ def _testRecv(self): # send data: recv() will no longer block self.cli.sendall(MSG) + def testLargeTimeout(self): + # gh-126876: Check that a timeout larger than INT_MAX is replaced with + # INT_MAX in the poll() code path. The following assertion must not + # fail: assert(INT_MIN <= ms && ms <= INT_MAX). + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + + # test recv() with large timeout + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + try: + conn.settimeout(large_timeout) + except OverflowError: + # On Windows, settimeout() fails with OverflowError, whereas + # we want to test recv(). Just give up silently. + return + msg = conn.recv(len(MSG)) + + def _testLargeTimeout(self): + # test sendall() with large timeout + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + self.cli.connect((HOST, self.port)) + try: + self.cli.settimeout(large_timeout) + except OverflowError: + return + self.cli.sendall(MSG) + class FileObjectClassTestCase(SocketConnectedTest): """Unit tests for the object returned by socket.makefile() @@ -5187,6 +5684,8 @@ def _testMakefileClose(self): self.write_file.write(self.write_msg) self.write_file.flush() + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') def testMakefileCloseSocketDestroy(self): refcount_before = sys.getrefcount(self.cli_conn) self.read_file.close() @@ -5605,10 +6104,10 @@ def testTimeoutZero(self): class TestExceptions(unittest.TestCase): def testExceptionTree(self): - self.assertTrue(issubclass(OSError, Exception)) - self.assertTrue(issubclass(socket.herror, OSError)) - self.assertTrue(issubclass(socket.gaierror, OSError)) - self.assertTrue(issubclass(socket.timeout, OSError)) + self.assertIsSubclass(OSError, Exception) + self.assertIsSubclass(socket.herror, OSError) + self.assertIsSubclass(socket.gaierror, OSError) + self.assertIsSubclass(socket.timeout, OSError) self.assertIs(socket.error, OSError) self.assertIs(socket.timeout, TimeoutError) @@ -5625,7 +6124,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5750,7 +6249,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") @@ -5976,8 +6476,6 @@ class InheritanceTest(unittest.TestCase): @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), "SOCK_CLOEXEC not defined") @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: 524289 != <SocketKind.SOCK_STREAM: 1> - @unittest.expectedFailure def test_SOCK_CLOEXEC(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: @@ -6070,8 +6568,6 @@ def checkNonblock(self, s, nonblock=True, timeout=0.0): self.assertTrue(s.getblocking()) @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: 2049 != <SocketKind.SOCK_STREAM: 1> - @unittest.expectedFailure def test_SOCK_NONBLOCK(self): # a lot of it seems silly and redundant, but I wanted to test that # changing back and forth worked ok @@ -6107,7 +6603,6 @@ def test_SOCK_NONBLOCK(self): @unittest.skipUnless(os.name == "nt", "Windows specific") @unittest.skipUnless(multiprocessing, "need multiprocessing") -@unittest.skip("TODO: RUSTPYTHON, socket sharing") class TestSocketSharing(SocketTCPTest): # This must be classmethod and not staticmethod or multiprocessing # won't be able to bootstrap it. @@ -6125,6 +6620,7 @@ def remoteProcessServer(cls, q): s2.close() s.close() + @unittest.expectedFailure # TODO: RUSTPYTHON; multiprocessing.SemLock not implemented def testShare(self): # Transfer the listening server socket to another process # and service it from there. @@ -6348,7 +6844,6 @@ def _testCount(self): self.assertEqual(sent, count) self.assertEqual(file.tell(), count) - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON, killed (for OOM?)") def testCount(self): count = 5000007 conn = self.accept_conn() @@ -6424,7 +6919,6 @@ def _testWithTimeout(self): sent = meth(file) self.assertEqual(sent, self.FILESIZE) - @unittest.skip("TODO: RUSTPYTHON") def testWithTimeout(self): conn = self.accept_conn() data = self.recv_data(conn) @@ -6485,6 +6979,14 @@ class SendfileUsingSendfileTest(SendfileUsingSendTest): def meth_from_sock(self, sock): return getattr(sock, "_sendfile_use_sendfile") + @unittest.skip("TODO: RUSTPYTHON; os.sendfile count parameter not handled correctly; flaky") + def testCount(self): + return super().testCount() + + @unittest.skip("TODO: RUSTPYTHON; os.sendfile count parameter not handled correctly; flaky") + def testWithTimeout(self): + return super().testWithTimeout() + @unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') class LinuxKernelCryptoAPI(unittest.TestCase): @@ -6502,9 +7004,8 @@ def create_alg(self, typ, name): # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, # at least on ppc64le architecture + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 5) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_sha256(self): expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" "177a9cb410ff61f20015ad") @@ -6522,8 +7023,7 @@ def test_sha256(self): op.send(b'') self.assertEqual(op.recv(512), expected) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented def test_hmac_sha1(self): # gh-109396: In FIPS mode, Linux 6.5 requires a key # of at least 112 bits. Use a key of 152 bits. @@ -6539,9 +7039,8 @@ def test_hmac_sha1(self): # Although it should work with 3.19 and newer the test blocks on # Ubuntu 15.10 with Kernel 4.2.0-19. + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 3) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_aes_cbc(self): key = bytes.fromhex('06a9214036b8a15b512e03d534120006') iv = bytes.fromhex('3dafba429d9eb430b422da802c9fac41') @@ -6582,10 +7081,15 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 9) # see issue29324 - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented + @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): + kernel_version = support._get_kernel_version("Linux") + if kernel_version is not None: + if kernel_version >= (6, 16) and kernel_version < (6, 18): + # See https://github.com/python/cpython/issues/139310. + self.skipTest("upstream Linux kernel issue") + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') @@ -6647,9 +7151,8 @@ def test_aead_aes_gcm(self): res = op.recv(len(msg) - taglen) self.assertEqual(plain, res[assoclen:]) + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 3) # see test_aes_cbc - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_drbg_pr_sha256(self): # deterministic random bit generator, prediction resistance, sha256 with self.create_alg('rng', 'drbg_pr_sha256') as algo: @@ -6660,8 +7163,6 @@ def test_drbg_pr_sha256(self): rn = op.recv(32) self.assertEqual(len(rn), 32) - # TODO: RUSTPYTHON, AttributeError: 'socket' object has no attribute 'sendmsg_afalg' - @unittest.expectedFailure def test_sendmsg_afalg_args(self): sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) with sock: @@ -6680,8 +7181,6 @@ def test_sendmsg_afalg_args(self): with self.assertRaises(TypeError): sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=-1) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_length_restriction(self): # bpo-35050, off-by-one error in length check sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) @@ -6705,6 +7204,28 @@ class TestMacOSTCPFlags(unittest.TestCase): def test_tcp_keepalive(self): self.assertTrue(socket.TCP_KEEPALIVE) +@unittest.skipUnless(hasattr(socket, 'TCP_QUICKACK'), 'need socket.TCP_QUICKACK') +class TestQuickackFlag(unittest.TestCase): + def check_set_quickack(self, sock): + # quickack already true by default on some OS distributions + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + if opt: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 0) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertFalse(opt) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 1) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertTrue(opt) + + def test_set_quickack(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_quickack(sock) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") class TestMSWindowsTCPFlags(unittest.TestCase): @@ -6718,7 +7239,9 @@ class TestMSWindowsTCPFlags(unittest.TestCase): 'TCP_KEEPCNT', # available starting with Windows 10 1709 'TCP_KEEPIDLE', - 'TCP_KEEPINTVL' + 'TCP_KEEPINTVL', + # available starting with Windows 7 / Server 2008 R2 + 'TCP_QUICKACK', } def test_new_tcp_flags(self): @@ -6886,6 +7409,26 @@ def close_fds(fds): self.assertEqual(data, str(index).encode()) +class FreeThreadingTests(unittest.TestCase): + + def test_close_detach_race(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def close(): + for _ in range(1000): + s.close() + + def detach(): + for _ in range(1000): + s.detach() + + t1 = threading.Thread(target=close) + t2 = threading.Thread(target=detach) + + with threading_helper.start_threads([t1, t2]): + pass + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index cdbf341b9cb..6235c8e74cf 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -8,7 +8,6 @@ import select import signal import socket -import tempfile import threading import unittest import socketserver @@ -21,6 +20,8 @@ test.support.requires("network") +test.support.requires_working_socket(module=True) + TEST_STR = b"hello world\n" HOST = socket_helper.HOST @@ -28,14 +29,9 @@ HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, 'requires Unix sockets') -HAVE_FORKING = hasattr(os, "fork") +HAVE_FORKING = test.support.has_fork_support requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') -def signal_alarm(n): - """Call signal.alarm when it exists (i.e. not on Windows).""" - if hasattr(signal, 'alarm'): - signal.alarm(n) - # Remember real select() to avoid interferences with mocking _real_select = select.select @@ -46,14 +42,6 @@ def receive(sock, n, timeout=test.support.SHORT_TIMEOUT): else: raise RuntimeError("timed out on %r" % (sock,)) -if HAVE_UNIX_SOCKETS and HAVE_FORKING: - class ForkingUnixStreamServer(socketserver.ForkingMixIn, - socketserver.UnixStreamServer): - pass - - class ForkingUnixDatagramServer(socketserver.ForkingMixIn, - socketserver.UnixDatagramServer): - pass @test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems @contextlib.contextmanager @@ -75,12 +63,10 @@ class SocketServerTest(unittest.TestCase): """Test all socket servers.""" def setUp(self): - signal_alarm(60) # Kill deadlocks after 60 seconds. self.port_seed = 0 self.test_files = [] def tearDown(self): - signal_alarm(0) # Didn't deadlock. reap_children() for fn in self.test_files: @@ -96,8 +82,7 @@ def pickaddr(self, proto): else: # XXX: We need a way to tell AF_UNIX to pick its own name # like AF_INET provides port==0. - dir = None - fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + fn = socket_helper.create_unix_domain_name() self.test_files.append(fn) return fn @@ -211,7 +196,7 @@ def test_ThreadingUnixStreamServer(self): @requires_forking def test_ForkingUnixStreamServer(self): with simple_subprocess(self): - self.run_server(ForkingUnixStreamServer, + self.run_server(socketserver.ForkingUnixStreamServer, socketserver.StreamRequestHandler, self.stream_examine) @@ -247,7 +232,7 @@ def test_ThreadingUnixDatagramServer(self): @requires_unix_sockets @requires_forking def test_ForkingUnixDatagramServer(self): - self.run_server(ForkingUnixDatagramServer, + self.run_server(socketserver.ForkingUnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index be3d4a8461f..a01f5fc9cee 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): @@ -149,7 +170,7 @@ def __lt__(self, other): L = [C() for i in range(50)] self.assertRaises(ValueError, L.sort) - @unittest.skip("TODO: RUSTPYTHON; figure out how to detect sort mutation that doesn't change list length") + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to detect sort mutation that doesn't change list length def test_undetected_mutation(self): # Python 2.4a1 did not always detect mutation memorywaster = [] @@ -307,8 +328,7 @@ def test_safe_object_compare(self): for L in float_int_lists: check_against_PyObject_RichCompareBool(self, L) - # XXX RUSTPYTHON: added by us but it seems like an implementation detail - @support.cpython_only + @support.cpython_only # XXX RUSTPYTHON: added by us but it seems like an implementation detail def test_unsafe_object_compare(self): # This test is by ppperry. It ensures that unsafe_object_compare is diff --git a/Lib/test/test_sqlite3/__init__.py b/Lib/test/test_sqlite3/__init__.py index d777fca82da..78a1e2078a5 100644 --- a/Lib/test/test_sqlite3/__init__.py +++ b/Lib/test/test_sqlite3/__init__.py @@ -8,8 +8,7 @@ # Implement the unittest "load tests" protocol. def load_tests(*args): + if verbose: + print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") pkg_dir = os.path.dirname(__file__) return load_package_tests(pkg_dir, *args) - -if verbose: - print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index fb3a83e3b0e..9d31978b1ad 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -1,6 +1,8 @@ import sqlite3 as sqlite import unittest +from .util import memory_database + class BackupTests(unittest.TestCase): def setUp(self): @@ -32,34 +34,32 @@ def test_bad_target_same_connection(self): self.cx.backup(self.cx) def test_bad_target_closed_connection(self): - bck = sqlite.connect(':memory:') - bck.close() - with self.assertRaises(sqlite.ProgrammingError): - self.cx.backup(bck) + with memory_database() as bck: + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) def test_bad_source_closed_connection(self): - bck = sqlite.connect(':memory:') - source = sqlite.connect(":memory:") - source.close() - with self.assertRaises(sqlite.ProgrammingError): - source.backup(bck) + with memory_database() as bck: + source = sqlite.connect(":memory:") + source.close() + with self.assertRaises(sqlite.ProgrammingError): + source.backup(bck) def test_bad_target_in_transaction(self): - bck = sqlite.connect(':memory:') - bck.execute('CREATE TABLE bar (key INTEGER)') - bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) - with self.assertRaises(sqlite.OperationalError) as cm: - self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 8): - self.assertEqual(str(cm.exception), 'target is in transaction') + with memory_database() as bck: + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) def test_keyword_only_args(self): with self.assertRaises(TypeError): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, 1) def test_simple(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck) self.verify_backup(bck) @@ -69,7 +69,7 @@ def test_progress(self): def progress(status, remaining, total): journal.append(status) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -83,7 +83,7 @@ def test_progress_all_pages_at_once_1(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.verify_backup(bck) @@ -96,18 +96,17 @@ def test_progress_all_pages_at_once_2(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=-1, progress=progress) self.verify_backup(bck) self.assertEqual(len(journal), 1) self.assertEqual(journal[0], 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_callable_progress(self): with self.assertRaises(TypeError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress='bar') self.assertEqual(str(cm.exception), 'progress argument must be a callable') @@ -120,7 +119,7 @@ def progress(status, remaining, total): self.cx.commit() journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -139,17 +138,17 @@ def progress(status, remaining, total): raise SystemError('nearly out of space') with self.assertRaises(SystemError) as err: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.assertEqual(str(err.exception), 'nearly out of space') def test_database_source_name(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='main') - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='temp') with self.assertRaises(sqlite.OperationalError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='non-existing') self.assertIn("unknown database", str(cm.exception)) @@ -157,7 +156,7 @@ def test_database_source_name(self): self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) self.cx.commit() - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='attached_db') self.verify_backup(bck) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 560cd9efc5c..a03d7cbe16b 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,52 +1,52 @@ """sqlite3 CLI tests.""" - -import sqlite3 as sqlite -import subprocess -import sys +import sqlite3 import unittest -from test.support import SHORT_TIMEOUT#, requires_subprocess +from sqlite3.__main__ import main as cli from test.support.os_helper import TESTFN, unlink +from test.support import ( + captured_stdout, + captured_stderr, + captured_stdin, + force_not_colorized, +) -# TODO: RUSTPYTHON -#@requires_subprocess() class CommandLineInterface(unittest.TestCase): def _do_test(self, *args, expect_success=True): - with subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - proc.wait() - if expect_success == bool(proc.returncode): - self.fail("".join(proc.stderr)) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - if expect_success: - self.assertEqual(stderr, "") - else: - self.assertEqual(stdout, "") - return stdout, stderr + with ( + captured_stdout() as out, + captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + cli(args) + return out.getvalue(), err.getvalue(), cm.exception.code def expect_success(self, *args): - out, _ = self._do_test(*args) + out, err, code = self._do_test(*args) + self.assertEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") return out def expect_failure(self, *args): - _, err = self._do_test(*args, expect_success=False) + out, err, code = self._do_test(*args, expect_success=False) + self.assertNotEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(out, "") return err + @force_not_colorized def test_cli_help(self): out = self.expect_success("-h") - self.assertIn("usage: python -m sqlite3", out) + self.assertIn("usage: ", out) + self.assertIn(" [-h] [-v] [filename] [sql]", out) + self.assertIn("Python sqlite3 CLI", out) def test_cli_version(self): out = self.expect_success("-v") - self.assertIn(sqlite.sqlite_version, out) + self.assertIn(sqlite3.sqlite_version, out) def test_cli_execute_sql(self): out = self.expect_success(":memory:", "select 1") @@ -69,88 +69,126 @@ def test_cli_on_disk_db(self): self.assertIn("(0,)", out) -# TODO: RUSTPYTHON -#@requires_subprocess() class InteractiveSession(unittest.TestCase): - TIMEOUT = SHORT_TIMEOUT / 10. MEMORY_DB_MSG = "Connected to a transient in-memory database" PS1 = "sqlite> " PS2 = "... " - def start_cli(self, *args): - return subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdin=subprocess.PIPE, - # Note: the banner is printed to stderr, the prompt to stdout. - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - def expect_success(self, proc): - proc.wait() - if proc.returncode: - self.fail("".join(proc.stderr)) + def run_cli(self, *args, commands=()): + with ( + captured_stdin() as stdin, + captured_stdout() as stdout, + captured_stderr() as stderr, + self.assertRaises(SystemExit) as cm + ): + for cmd in commands: + stdin.write(cmd + "\n") + stdin.seek(0) + cli(args) + + out = stdout.getvalue() + err = stderr.getvalue() + self.assertEqual(cm.exception.code, 0, + f"Unexpected failure: {args=}\n{out}\n{err}") + return out, err def test_interact(self): - with self.start_cli() as proc: - out, err = proc.communicate(timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli() + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_quit(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".quit", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".quit",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_version(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".version", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(sqlite.sqlite_version, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".version",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(sqlite3.sqlite_version + "\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn(sqlite3.sqlite_version, out) + + def test_interact_empty_source(self): + out, err = self.run_cli(commands=("", " ")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_unknown(self): + out, err = self.run_cli(commands=(".unknown_command", )) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn("Error", err) + # test "unknown_command" is pointed out in the error message + self.assertIn("unknown_command", err) + + def test_interact_dot_commands_empty(self): + out, err = self.run_cli(commands=(".")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_with_whitespaces(self): + out, err = self.run_cli(commands=(".version ", ". version")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEqual(out.count(sqlite3.sqlite_version + "\n"), 2) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) def test_interact_valid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("(1,)\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_incomplete_multiline_sql(self): + out, err = self.run_cli(commands=("SELECT 1",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS2) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 1) def test_interact_valid_multiline_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1\n;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS2, out) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1\n;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.PS2, out) + self.assertIn("(1,)\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 1) def test_interact_invalid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="sel;", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("OperationalError (SQLITE_ERROR)", err) - self.expect_success(proc) + out, err = self.run_cli(commands=("sel;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("OperationalError (SQLITE_ERROR)", err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) def test_interact_on_disk_file(self): self.addCleanup(unlink, TESTFN) - with self.start_cli(TESTFN) as proc: - out, err = proc.communicate(input="create table t(t);", - timeout=self.TIMEOUT) - self.assertIn(TESTFN, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) - with self.start_cli(TESTFN, "select count(t) from t") as proc: - out = proc.stdout.read() - err = proc.stderr.read() - self.assertIn("(0,)", out) - self.expect_success(proc) + + out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) + self.assertIn(TESTFN, err) + self.assertEndsWith(out, self.PS1) + + out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) + self.assertIn("(0,)\n", out) if __name__ == "__main__": diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9a95c489a31..46f098e4655 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import functools import os import sqlite3 as sqlite import subprocess @@ -28,33 +29,18 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation,# requires_subprocess, - #is_emscripten, is_wasi -# TODO: RUSTPYTHON + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) -from test.support import threading_helper -# TODO: RUSTPYTHON -#from _testcapi import INT_MAX, ULLONG_MAX +from test.support import gc_collect +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath - -# Helper for temporary memory databases -def memory_database(*args, **kwargs): - cx = sqlite.connect(":memory:", *args, **kwargs) - return contextlib.closing(cx) - - -# Temporarily limit a database connection parameter -@contextlib.contextmanager -def cx_limit(cx, category=sqlite.SQLITE_LIMIT_SQL_LENGTH, limit=128): - try: - _prev = cx.setlimit(category, limit) - yield limit - finally: - cx.setlimit(category, _prev) +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin class ModuleTests(unittest.TestCase): @@ -62,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % @@ -84,45 +59,34 @@ def test_param_style(self): sqlite.paramstyle) def test_warning(self): - self.assertTrue(issubclass(sqlite.Warning, Exception), - "Warning is not a subclass of Exception") + self.assertIsSubclass(sqlite.Warning, Exception) def test_error(self): - self.assertTrue(issubclass(sqlite.Error, Exception), - "Error is not a subclass of Exception") + self.assertIsSubclass(sqlite.Error, Exception) def test_interface_error(self): - self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error), - "InterfaceError is not a subclass of Error") + self.assertIsSubclass(sqlite.InterfaceError, sqlite.Error) def test_database_error(self): - self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error), - "DatabaseError is not a subclass of Error") + self.assertIsSubclass(sqlite.DatabaseError, sqlite.Error) def test_data_error(self): - self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError), - "DataError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.DataError, sqlite.DatabaseError) def test_operational_error(self): - self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError), - "OperationalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.OperationalError, sqlite.DatabaseError) def test_integrity_error(self): - self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError), - "IntegrityError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.IntegrityError, sqlite.DatabaseError) def test_internal_error(self): - self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError), - "InternalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.InternalError, sqlite.DatabaseError) def test_programming_error(self): - self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError), - "ProgrammingError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.ProgrammingError, sqlite.DatabaseError) def test_not_supported_error(self): - self.assertTrue(issubclass(sqlite.NotSupportedError, - sqlite.DatabaseError), - "NotSupportedError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.NotSupportedError, sqlite.DatabaseError) def test_module_constants(self): consts = [ @@ -167,6 +131,7 @@ def test_module_constants(self): "SQLITE_INTERNAL", "SQLITE_INTERRUPT", "SQLITE_IOERR", + "SQLITE_LIMIT_WORKER_THREADS", "SQLITE_LOCKED", "SQLITE_MISMATCH", "SQLITE_MISUSE", @@ -174,6 +139,7 @@ def test_module_constants(self): "SQLITE_NOMEM", "SQLITE_NOTADB", "SQLITE_NOTFOUND", + "SQLITE_NOTICE", "SQLITE_OK", "SQLITE_PERM", "SQLITE_PRAGMA", @@ -181,6 +147,7 @@ def test_module_constants(self): "SQLITE_RANGE", "SQLITE_READ", "SQLITE_READONLY", + "SQLITE_RECURSIVE", "SQLITE_REINDEX", "SQLITE_ROW", "SQLITE_SAVEPOINT", @@ -189,6 +156,7 @@ def test_module_constants(self): "SQLITE_TOOBIG", "SQLITE_TRANSACTION", "SQLITE_UPDATE", + "SQLITE_WARNING", # Run-time limit categories "SQLITE_LIMIT_LENGTH", "SQLITE_LIMIT_SQL_LENGTH", @@ -202,32 +170,43 @@ def test_module_constants(self): "SQLITE_LIMIT_VARIABLE_NUMBER", "SQLITE_LIMIT_TRIGGER_DEPTH", ] - if sqlite.sqlite_version_info >= (3, 7, 17): - consts += ["SQLITE_NOTICE", "SQLITE_WARNING"] - if sqlite.sqlite_version_info >= (3, 8, 3): - consts.append("SQLITE_RECURSIVE") - if sqlite.sqlite_version_info >= (3, 8, 7): - consts.append("SQLITE_LIMIT_WORKER_THREADS") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] # Extended result codes consts += [ "SQLITE_ABORT_ROLLBACK", + "SQLITE_AUTH_USER", "SQLITE_BUSY_RECOVERY", + "SQLITE_BUSY_SNAPSHOT", + "SQLITE_CANTOPEN_CONVPATH", "SQLITE_CANTOPEN_FULLPATH", "SQLITE_CANTOPEN_ISDIR", "SQLITE_CANTOPEN_NOTEMPDIR", + "SQLITE_CONSTRAINT_CHECK", + "SQLITE_CONSTRAINT_COMMITHOOK", + "SQLITE_CONSTRAINT_FOREIGNKEY", + "SQLITE_CONSTRAINT_FUNCTION", + "SQLITE_CONSTRAINT_NOTNULL", + "SQLITE_CONSTRAINT_PRIMARYKEY", + "SQLITE_CONSTRAINT_ROWID", + "SQLITE_CONSTRAINT_TRIGGER", + "SQLITE_CONSTRAINT_UNIQUE", + "SQLITE_CONSTRAINT_VTAB", "SQLITE_CORRUPT_VTAB", "SQLITE_IOERR_ACCESS", + "SQLITE_IOERR_AUTH", "SQLITE_IOERR_BLOCKED", "SQLITE_IOERR_CHECKRESERVEDLOCK", "SQLITE_IOERR_CLOSE", + "SQLITE_IOERR_CONVPATH", "SQLITE_IOERR_DELETE", "SQLITE_IOERR_DELETE_NOENT", "SQLITE_IOERR_DIR_CLOSE", "SQLITE_IOERR_DIR_FSYNC", "SQLITE_IOERR_FSTAT", "SQLITE_IOERR_FSYNC", + "SQLITE_IOERR_GETTEMPPATH", "SQLITE_IOERR_LOCK", + "SQLITE_IOERR_MMAP", "SQLITE_IOERR_NOMEM", "SQLITE_IOERR_RDLOCK", "SQLITE_IOERR_READ", @@ -239,50 +218,18 @@ def test_module_constants(self): "SQLITE_IOERR_SHORT_READ", "SQLITE_IOERR_TRUNCATE", "SQLITE_IOERR_UNLOCK", + "SQLITE_IOERR_VNODE", "SQLITE_IOERR_WRITE", "SQLITE_LOCKED_SHAREDCACHE", + "SQLITE_NOTICE_RECOVER_ROLLBACK", + "SQLITE_NOTICE_RECOVER_WAL", + "SQLITE_OK_LOAD_PERMANENTLY", "SQLITE_READONLY_CANTLOCK", + "SQLITE_READONLY_DBMOVED", "SQLITE_READONLY_RECOVERY", + "SQLITE_READONLY_ROLLBACK", + "SQLITE_WARNING_AUTOINDEX", ] - if sqlite.sqlite_version_info >= (3, 7, 16): - consts += [ - "SQLITE_CONSTRAINT_CHECK", - "SQLITE_CONSTRAINT_COMMITHOOK", - "SQLITE_CONSTRAINT_FOREIGNKEY", - "SQLITE_CONSTRAINT_FUNCTION", - "SQLITE_CONSTRAINT_NOTNULL", - "SQLITE_CONSTRAINT_PRIMARYKEY", - "SQLITE_CONSTRAINT_TRIGGER", - "SQLITE_CONSTRAINT_UNIQUE", - "SQLITE_CONSTRAINT_VTAB", - "SQLITE_READONLY_ROLLBACK", - ] - if sqlite.sqlite_version_info >= (3, 7, 17): - consts += [ - "SQLITE_IOERR_MMAP", - "SQLITE_NOTICE_RECOVER_ROLLBACK", - "SQLITE_NOTICE_RECOVER_WAL", - ] - if sqlite.sqlite_version_info >= (3, 8, 0): - consts += [ - "SQLITE_BUSY_SNAPSHOT", - "SQLITE_IOERR_GETTEMPPATH", - "SQLITE_WARNING_AUTOINDEX", - ] - if sqlite.sqlite_version_info >= (3, 8, 1): - consts += ["SQLITE_CANTOPEN_CONVPATH", "SQLITE_IOERR_CONVPATH"] - if sqlite.sqlite_version_info >= (3, 8, 2): - consts.append("SQLITE_CONSTRAINT_ROWID") - if sqlite.sqlite_version_info >= (3, 8, 3): - consts.append("SQLITE_READONLY_DBMOVED") - if sqlite.sqlite_version_info >= (3, 8, 7): - consts.append("SQLITE_AUTH_USER") - if sqlite.sqlite_version_info >= (3, 9, 0): - consts.append("SQLITE_IOERR_VNODE") - if sqlite.sqlite_version_info >= (3, 10, 0): - consts.append("SQLITE_IOERR_AUTH") - if sqlite.sqlite_version_info >= (3, 14, 1): - consts.append("SQLITE_OK_LOAD_PERMANENTLY") if sqlite.sqlite_version_info >= (3, 21, 0): consts += [ "SQLITE_IOERR_BEGIN_ATOMIC", @@ -316,7 +263,7 @@ def test_module_constants(self): consts.append("SQLITE_IOERR_CORRUPTFS") for const in consts: with self.subTest(const=const): - self.assertTrue(hasattr(sqlite, const)) + self.assertHasAttr(sqlite, const) def test_error_code_on_exception(self): err_msg = "unable to open database file" @@ -330,10 +277,8 @@ def test_error_code_on_exception(self): sqlite.connect(db) e = cm.exception self.assertEqual(e.sqlite_errorcode, err_code) - self.assertTrue(e.sqlite_errorname.startswith("SQLITE_CANTOPEN")) + self.assertStartsWith(e.sqlite_errorname, "SQLITE_CANTOPEN") - @unittest.skipIf(sqlite.sqlite_version_info <= (3, 7, 16), - "Requires SQLite 3.7.16 or newer") def test_extended_error_code_on_exception(self): with memory_database() as con: with con: @@ -347,9 +292,9 @@ def test_extended_error_code_on_exception(self): self.assertEqual(exc.sqlite_errorname, "SQLITE_CONSTRAINT_CHECK") def test_disallow_instantiation(self): - cx = sqlite.connect(":memory:") - check_disallow_instantiation(self, type(cx("select 1"))) - check_disallow_instantiation(self, sqlite.Blob) + with memory_database() as cx: + check_disallow_instantiation(self, type(cx("select 1"))) + check_disallow_instantiation(self, sqlite.Blob) def test_complete_statement(self): self.assertFalse(sqlite.complete_statement("select t")) @@ -363,6 +308,7 @@ def setUp(self): cu = self.cx.cursor() cu.execute("create table test(id integer primary key, name text)") cu.execute("insert into test(name) values (?)", ("foo",)) + cu.close() def tearDown(self): self.cx.close() @@ -418,8 +364,7 @@ def test_use_after_close(self): with self.cx: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exceptions(self): # Optional DB-API extension. self.assertEqual(self.cx.Warning, sqlite.Warning) @@ -435,28 +380,28 @@ def test_exceptions(self): def test_in_transaction(self): # Can't use db from setUp because we want to test initial state. - cx = sqlite.connect(":memory:") - cu = cx.cursor() - self.assertEqual(cx.in_transaction, False) - cu.execute("create table transactiontest(id integer primary key, name text)") - self.assertEqual(cx.in_transaction, False) - cu.execute("insert into transactiontest(name) values (?)", ("foo",)) - self.assertEqual(cx.in_transaction, True) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, True) - cx.commit() - self.assertEqual(cx.in_transaction, False) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, False) + with memory_database() as cx: + cu = cx.cursor() + self.assertEqual(cx.in_transaction, False) + cu.execute("create table transactiontest(id integer primary key, name text)") + self.assertEqual(cx.in_transaction, False) + cu.execute("insert into transactiontest(name) values (?)", ("foo",)) + self.assertEqual(cx.in_transaction, True) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, True) + cx.commit() + self.assertEqual(cx.in_transaction, False) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, False) + cu.close() def test_in_transaction_ro(self): with self.assertRaises(AttributeError): self.cx.in_transaction = True - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_exceptions(self): exceptions = [ "DataError", @@ -471,14 +416,13 @@ def test_connection_exceptions(self): ] for exc in exceptions: with self.subTest(exc=exc): - self.assertTrue(hasattr(self.cx, exc)) + self.assertHasAttr(self.cx, exc) self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): - cx = sqlite.connect(":memory:") - cx.close() + self.cx.close() with self.assertRaises(sqlite.ProgrammingError): - cx.interrupt() + self.cx.interrupt() def test_interrupt(self): self.assertIsNone(self.cx.interrupt()) @@ -545,33 +489,30 @@ def test_connection_init_good_isolation_levels(self): cx.isolation_level = level self.assertEqual(cx.isolation_level, level) - # TODO: RUSTPYTHON - # @unittest.expectedFailure - @unittest.skip("TODO: RUSTPYTHON deadlock") def test_connection_reinit(self): - db = ":memory:" - cx = sqlite.connect(db) - cx.text_factory = bytes - cx.row_factory = sqlite.Row - cu = cx.cursor() - cu.execute("create table foo (bar)") - cu.executemany("insert into foo (bar) values (?)", - ((str(v),) for v in range(4))) - cu.execute("select bar from foo") - - rows = [r for r in cu.fetchmany(2)] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], [b"0", b"1"]) - - cx.__init__(db) - cx.execute("create table foo (bar)") - cx.executemany("insert into foo (bar) values (?)", - ((v,) for v in ("a", "b", "c", "d"))) - - # This uses the old database, old row factory, but new text factory - rows = [r for r in cu.fetchall()] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], ["2", "3"]) + with memory_database() as cx: + cx.text_factory = bytes + cx.row_factory = sqlite.Row + cu = cx.cursor() + cu.execute("create table foo (bar)") + cu.executemany("insert into foo (bar) values (?)", + ((str(v),) for v in range(4))) + cu.execute("select bar from foo") + + rows = [r for r in cu.fetchmany(2)] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], [b"0", b"1"]) + + cx.__init__(":memory:") + cx.execute("create table foo (bar)") + cx.executemany("insert into foo (bar) values (?)", + ((v,) for v in ("a", "b", "c", "d"))) + + # This uses the old database, old row factory, but new text factory + rows = [r for r in cu.fetchall()] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], ["2", "3"]) + cu.close() def test_connection_bad_reinit(self): cx = sqlite.connect(":memory:") @@ -586,12 +527,64 @@ def test_connection_bad_reinit(self): cx.executemany, "insert into t values(?)", ((v,) for v in range(3))) + @unittest.expectedFailure # TODO: RUSTPYTHON SQLITE_DBCONFIG constants not implemented + def test_connection_config(self): + op = sqlite.SQLITE_DBCONFIG_ENABLE_FKEY + with memory_database() as cx: + with self.assertRaisesRegex(ValueError, "unknown"): + cx.getconfig(-1) + + # Toggle and verify. + old = cx.getconfig(op) + new = not old + cx.setconfig(op, new) + self.assertEqual(cx.getconfig(op), new) + + cx.setconfig(op) # defaults to True + self.assertTrue(cx.getconfig(op)) + + # Check that foreign key support was actually enabled. + with cx: + cx.executescript(""" + create table t(t integer primary key); + create table u(u, foreign key(u) references t(t)); + """) + with self.assertRaisesRegex(sqlite.IntegrityError, "constraint"): + cx.execute("insert into u values(0)") + + @unittest.expectedFailure # TODO: RUSTPYTHON deprecation warning not emitted for positional args + def test_connect_positional_arguments(self): + regex = ( + r"Passing more than 1 positional argument to sqlite3.connect\(\)" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + cx = sqlite.connect(":memory:", 1.0) + cx.close() + self.assertEqual(cm.filename, __file__) + + @unittest.expectedFailure # TODO: RUSTPYTHON ResourceWarning not emitted + def test_connection_resource_warning(self): + with self.assertWarns(ResourceWarning): + cx = sqlite.connect(":memory:") + del cx + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON Connection signature inspection not working + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + -@unittest.skip("TODO: RUSTPYTHON") class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): self.cx = sqlite.Connection.__new__(sqlite.Connection) + @unittest.skip('TODO: RUSTPYTHON') def test_uninit_operations(self): funcs = ( lambda: self.cx.isolation_level, @@ -616,7 +609,6 @@ def test_serialize_deserialize(self): with cx: cx.execute("create table t(t)") data = cx.serialize() - self.assertEqual(len(data), 8192) # Remove test table, verify that it was removed. with cx: @@ -654,6 +646,14 @@ def test_deserialize_corrupt_database(self): class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" + def test_open_with_bytes_path(self): + path = os.fsencode(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ @@ -664,15 +664,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + def get_undecodable_path(self): + path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") - # TODO: RUSTPYTHON - # @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -712,21 +718,15 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") - # TODO: RUSTPYTHON - # @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_factory_database_arg(self): def factory(database, *args, **kwargs): nonlocal database_arg @@ -875,6 +875,34 @@ def __getitem__(slf, x): with self.assertRaises(ZeroDivisionError): self.cu.execute("select name from test where name=?", L()) + @unittest.expectedFailure # TODO: RUSTPYTHON mixed named and positional parameters not validated + def test_execute_named_param_and_sequence(self): + dataset = ( + ("select :a", (1,)), + ("select :a, ?, ?", (1, 2, 3)), + ("select ?, :b, ?", (1, 2, 3)), + ("select ?, ?, :c", (1, 2, 3)), + ("select :a, :b, ?", (1, 2, 3)), + ) + msg = "Binding.*is a named parameter" + for query, params in dataset: + with self.subTest(query=query, params=params): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: + self.cu.execute(query, params) + + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) + def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" @@ -1050,7 +1078,7 @@ def test_array_size(self): # now set to 2 self.cu.arraysize = 2 - # now make the query return 3 rows + # now make the query return 2 rows from a table of 3 rows self.cu.execute("delete from test") self.cu.execute("insert into test(name) values ('A')") self.cu.execute("insert into test(name) values ('B')") @@ -1060,13 +1088,53 @@ def test_array_size(self): self.assertEqual(len(res), 2) + @unittest.expectedFailure # TODO: RUSTPYTHON arraysize validation not implemented + def test_invalid_array_size(self): + UINT32_MAX = (1 << 32) - 1 + setter = functools.partial(setattr, self.cu, 'arraysize') + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(ValueError, setter, -3) + self.assertRaises(OverflowError, setter, UINT32_MAX + 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON fetchmany behavior with exhausted cursor differs def test_fetchmany(self): + # no active SQL statement + res = self.cu.fetchmany() + self.assertEqual(res, []) + res = self.cu.fetchmany(1000) + self.assertEqual(res, []) + + # test default parameter + self.cu.execute("select name from test") + res = self.cu.fetchmany() + self.assertEqual(len(res), 1) + + # test when the number of requested rows exceeds the actual count self.cu.execute("select name from test") res = self.cu.fetchmany(100) self.assertEqual(len(res), 1) res = self.cu.fetchmany(100) self.assertEqual(res, []) + # test when size = 0 + self.cu.execute("select name from test") + res = self.cu.fetchmany(0) + self.assertEqual(res, []) + res = self.cu.fetchmany(100) + self.assertEqual(len(res), 1) + res = self.cu.fetchmany(100) + self.assertEqual(res, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON fetchmany size validation not implemented + def test_invalid_fetchmany(self): + UINT32_MAX = (1 << 32) - 1 + fetchmany = self.cu.fetchmany + + self.assertRaises(TypeError, fetchmany, 1.0) + self.assertRaises(ValueError, fetchmany, -3) + self.assertRaises(OverflowError, fetchmany, UINT32_MAX + 1) + def test_fetchmany_kw_arg(self): """Checks if fetchmany works with keyword arguments""" self.cu.execute("select name from test") @@ -1186,12 +1254,9 @@ def test_blob_seek_and_tell(self): self.blob.seek(-10, SEEK_END) self.assertEqual(self.blob.tell(), 40) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1203,12 +1268,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1362,24 +1430,24 @@ def test_blob_mapping_invalid_index_type(self): with self.assertRaisesRegex(TypeError, msg): self.blob["a"] = b"b" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_get_item_error(self): dataset = [len(self.blob), 105, -105] for idx in dataset: with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" @@ -1416,7 +1484,7 @@ def test_blob_sequence_not_supported(self): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): @@ -1477,9 +1545,16 @@ def test_blob_closed_db_read(self): "Cannot operate on a closed database", blob.read) + def test_blob_32bit_rowid(self): + # gh-100370: we should not get an OverflowError for 32-bit rowids + with memory_database() as cx: + rowid = 2**32 + cx.execute("create table t(t blob)") + cx.execute("insert into t(rowid, t) values (?, zeroblob(1))", (rowid,)) + cx.blobopen('t', 't', rowid) -# TODO: RUSTPYTHON -# @threading_helper.requires_working_threading() + +@threading_helper.requires_working_threading() class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -1532,8 +1607,7 @@ def test_check_connection_thread(self): with self.subTest(fn=fn): self._run_test(fn) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_cursor_thread(self): fns = [ lambda: self.cur.execute("insert into test(name) values('a')"), @@ -1554,12 +1628,12 @@ def run(con, err): except sqlite.Error: err.append("multi-threading not allowed") - con = sqlite.connect(":memory:", check_same_thread=False) - err = [] - t = threading.Thread(target=run, kwargs={"con": con, "err": err}) - t.start() - t.join() - self.assertEqual(len(err), 0, "\n".join(err)) + with memory_database(check_same_thread=False) as con: + err = [] + t = threading.Thread(target=run, kwargs={"con": con, "err": err}) + t.start() + t.join() + self.assertEqual(len(err), 0, "\n".join(err)) class ConstructorTests(unittest.TestCase): @@ -1585,9 +1659,16 @@ def test_binary(self): b = sqlite.Binary(b"\0'") class ExtensionTests(unittest.TestCase): + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + def test_script_string_sql(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cur cur.executescript(""" -- bla bla /* a stupid comment */ @@ -1599,40 +1680,40 @@ def test_script_string_sql(self): self.assertEqual(res, 5) def test_script_syntax_error(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(x); asdf; create table test2(x)") + self.cur.executescript(""" + CREATE TABLE test(x); + asdf; + CREATE TABLE test2(x) + """) def test_script_error_normal(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(sadfsadfdsa); select foo from hurz;") + self.cur.executescript(""" + CREATE TABLE test(sadfsadfdsa); + SELECT foo FROM hurz; + """) def test_cursor_executescript_as_bytes(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(TypeError): - cur.executescript(b"create table test(foo); insert into test(foo) values (5);") + self.cur.executescript(b""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) def test_cursor_executescript_with_null_characters(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(ValueError): - cur.executescript(""" - create table a(i);\0 - insert into a(i) values (5); - """) + self.cur.executescript(""" + CREATE TABLE a(i);\0 + INSERT INTO a(i) VALUES (5); + """) def test_cursor_executescript_with_surrogates(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(UnicodeEncodeError): - cur.executescript(""" - create table a(s); - insert into a(s) values ('\ud8ff'); - """) + self.cur.executescript(""" + CREATE TABLE a(s); + INSERT INTO a(s) VALUES ('\ud8ff'); + """) def test_cursor_executescript_too_large_script(self): msg = "query string is too large" @@ -1642,19 +1723,18 @@ def test_cursor_executescript_too_large_script(self): cx.executescript("select 'too large'".ljust(lim+1)) def test_cursor_executescript_tx_control(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("begin") self.assertTrue(con.in_transaction) con.executescript("select 1") self.assertFalse(con.in_transaction) def test_connection_execute(self): - con = sqlite.connect(":memory:") - result = con.execute("select 5").fetchone()[0] + result = self.con.execute("select 5").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.execute") def test_connection_executemany(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table test(foo)") con.executemany("insert into test(foo) values (?)", [(3,), (4,)]) result = con.execute("select foo from test order by foo").fetchall() @@ -1662,47 +1742,50 @@ def test_connection_executemany(self): self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany") def test_connection_executescript(self): - con = sqlite.connect(":memory:") - con.executescript("create table test(foo); insert into test(foo) values (5);") + con = self.con + con.executescript(""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) result = con.execute("select foo from test").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.executescript") + class ClosedConTests(unittest.TestCase): + def check(self, fn, *args, **kwds): + regex = "Cannot operate on a closed database." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + fn(*args, **kwds) + + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + self.con.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_cursor(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur = con.cursor() + self.check(self.con.cursor) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_commit(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.commit() + self.check(self.con.commit) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_rollback(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.rollback() + self.check(self.con.rollback) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_cur_execute(self): - con = sqlite.connect(":memory:") - cur = con.cursor() - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur.execute("select 4") + self.check(self.cur.execute, "select 4") + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_create_function(self): - con = sqlite.connect(":memory:") - con.close() - def f(x): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_function("foo", 1, f) + def f(x): + return 17 + self.check(self.con.create_function, "foo", 1, f) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_create_aggregate(self): - con = sqlite.connect(":memory:") - con.close() class Agg: def __init__(self): pass @@ -1710,34 +1793,28 @@ def step(self, x): pass def finalize(self): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_aggregate("foo", 1, Agg) + self.check(self.con.create_aggregate, "foo", 1, Agg) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_set_authorizer(self): - con = sqlite.connect(":memory:") - con.close() def authorizer(*args): return sqlite.DENY - with self.assertRaises(sqlite.ProgrammingError): - con.set_authorizer(authorizer) + self.check(self.con.set_authorizer, authorizer) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_set_progress_callback(self): - con = sqlite.connect(":memory:") - con.close() - def progress(): pass - with self.assertRaises(sqlite.ProgrammingError): - con.set_progress_handler(progress, 100) + def progress(): + pass + self.check(self.con.set_progress_handler, progress, 100) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_call(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con() + self.check(self.con) + -class ClosedCurTests(unittest.TestCase): +class ClosedCurTests(MemoryDatabaseMixin, unittest.TestCase): def test_closed(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cx.cursor() cur.close() for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"): @@ -1846,14 +1923,14 @@ def test_on_conflict_replace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) -# TODO: RUSTPYTHON -# @requires_subprocess() +@requires_subprocess() class MultiprocessTests(unittest.TestCase): - CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms + CONNECTION_TIMEOUT = 0 # Disable the busy timeout. def tearDown(self): unlink(TESTFN) + @unittest.expectedFailure # TODO: RUSTPYTHON multiprocess test fails def test_ctx_mgr_rollback_if_commit_failed(self): # bpo-27334: ctx manager does not rollback if commit fails SCRIPT = f"""if 1: @@ -1919,5 +1996,71 @@ def wait(): self.assertEqual(proc.returncode, 0) +class RowTests(unittest.TestCase): + + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.row_factory = sqlite.Row + + def tearDown(self): + self.cx.close() + + def test_row_keys(self): + cu = self.cx.execute("SELECT 1 as first, 2 as second") + row = cu.fetchone() + self.assertEqual(row.keys(), ["first", "second"]) + + def test_row_length(self): + cu = self.cx.execute("SELECT 1, 2, 3") + row = cu.fetchone() + self.assertEqual(len(row), 3) + + def test_row_getitem(self): + cu = self.cx.execute("SELECT 1 as a, 2 as b") + row = cu.fetchone() + self.assertEqual(row[0], 1) + self.assertEqual(row[1], 2) + self.assertEqual(row["a"], 1) + self.assertEqual(row["b"], 2) + for key in "nokey", 4, 1.2: + with self.subTest(key=key): + with self.assertRaises(IndexError): + row[key] + + def test_row_equality(self): + c1 = self.cx.execute("SELECT 1 as a") + r1 = c1.fetchone() + + c2 = self.cx.execute("SELECT 1 as a") + r2 = c2.fetchone() + + self.assertIsNot(r1, r2) + self.assertEqual(r1, r2) + + c3 = self.cx.execute("SELECT 1 as b") + r3 = c3.fetchone() + + self.assertNotEqual(r1, r3) + + @unittest.expectedFailure # TODO: RUSTPYTHON Row with no description fails + def test_row_no_description(self): + cu = self.cx.cursor() + self.assertIsNone(cu.description) + + row = sqlite.Row(cu, ()) + self.assertEqual(row.keys(), []) + with self.assertRaisesRegex(IndexError, "nokey"): + row["nokey"] + + def test_row_is_a_sequence(self): + from collections.abc import Sequence + + cu = self.cx.execute("SELECT 1") + row = cu.fetchone() + + self.assertIsSubclass(sqlite.Row, Sequence) + self.assertIsInstance(row, Sequence) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index ec4a11da8b0..74aacc05c2b 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -1,22 +1,18 @@ # Author: Paul Kippes <kippesp@gmail.com> import unittest -import sqlite3 as sqlite -from .test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin +from .util import requires_virtual_table -# TODO: RUSTPYTHON -@unittest.expectedFailure -class DumpTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cu = self.cx.cursor() - def tearDown(self): - self.cx.close() +class DumpTests(MemoryDatabaseMixin, unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test_table_dump(self): expected_sqls = [ + "PRAGMA foreign_keys=OFF;", """CREATE TABLE "index"("index" blob);""" , """INSERT INTO "index" VALUES(X'01');""" @@ -27,7 +23,8 @@ def test_table_dump(self): , "CREATE TABLE t1(id integer primary key, s1 text, " \ "t1_i1 integer not null, i2 integer, unique (s1), " \ - "constraint t1_idx1 unique (i2));" + "constraint t1_idx1 unique (i2), " \ + "constraint t1_i1_idx1 unique (t1_i1));" , "INSERT INTO \"t1\" VALUES(1,'foo',10,20);" , @@ -37,6 +34,9 @@ def test_table_dump(self): "t2_i2 integer, primary key (id)," \ "foreign key(t2_i1) references t1(t1_i1));" , + # Foreign key violation. + "INSERT INTO \"t2\" VALUES(1,2,3);" + , "CREATE TRIGGER trigger_1 update of t1_i1 on t1 " \ "begin " \ "update t2 set t2_i1 = new.t1_i1 where t2_i1 = old.t1_i1; " \ @@ -48,11 +48,87 @@ def test_table_dump(self): [self.cu.execute(s) for s in expected_sqls] i = self.cx.iterdump() actual_sqls = [s for s in i] - expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ - ['COMMIT;'] + expected_sqls = [ + "PRAGMA foreign_keys=OFF;", + "BEGIN TRANSACTION;", + *expected_sqls[1:], + "COMMIT;", + ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + @unittest.expectedFailure # TODO: RUSTPYTHON iterdump filter parameter not implemented + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', @@ -73,6 +149,7 @@ def test_dump_autoincrement(self): actual = [stmt for stmt in self.cx.iterdump()] self.assertEqual(expected, actual) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_dump_autoincrement_create_new_db(self): self.cu.execute("BEGIN TRANSACTION") self.cu.execute("CREATE TABLE t1 (id integer primary key autoincrement)") @@ -98,6 +175,7 @@ def test_dump_autoincrement_create_new_db(self): rows = res.fetchall() self.assertEqual(rows[0][0], seq) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_unorderable_row(self): # iterdump() should be able to cope with unorderable row types (issue #15545) class UnorderableRow: @@ -119,6 +197,44 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented + @requires_virtual_table("fts4") + def test_dump_virtual_tables(self): + # gh-64662 + expected = [ + "BEGIN TRANSACTION;", + "PRAGMA writable_schema=ON;", + ("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','test','test',0,'CREATE VIRTUAL TABLE test USING fts4(example)');"), + "CREATE TABLE 'test_content'(docid INTEGER PRIMARY KEY, 'c0example');", + "CREATE TABLE 'test_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + ("CREATE TABLE 'test_segdir'(level INTEGER,idx INTEGER,start_block INTEGER," + "leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));"), + "CREATE TABLE 'test_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + "CREATE TABLE 'test_stat'(id INTEGER PRIMARY KEY, value BLOB);", + "PRAGMA writable_schema=OFF;", + "COMMIT;" + ] + self.cu.execute("CREATE VIRTUAL TABLE test USING fts4(example)") + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index e52c10fe944..d19b98b2056 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -24,6 +24,9 @@ import sqlite3 as sqlite from collections.abc import Sequence +from .util import memory_database +from .util import MemoryDatabaseMixin + def dict_factory(cursor, row): d = {} @@ -37,6 +40,7 @@ def __init__(self, *args, **kwargs): self.row_factory = dict_factory class ConnectionFactoryTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factories(self): class DefectFactory(sqlite.Connection): def __init__(self, *args, **kwargs): @@ -45,13 +49,14 @@ class OkFactory(sqlite.Connection): def __init__(self, *args, **kwargs): sqlite.Connection.__init__(self, *args, **kwargs) - for factory in DefectFactory, OkFactory: - with self.subTest(factory=factory): - con = sqlite.connect(":memory:", factory=factory) - self.assertIsInstance(con, factory) + with memory_database(factory=OkFactory) as con: + self.assertIsInstance(con, OkFactory) + regex = "Base Connection.__init__ not called." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + with memory_database(factory=DefectFactory) as con: + self.assertIsInstance(con, DefectFactory) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factory_relayed_call(self): # gh-95132: keyword args must not be passed as positional args class Factory(sqlite.Connection): @@ -59,29 +64,35 @@ def __init__(self, *args, **kwargs): kwargs["isolation_level"] = None super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", factory=Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + with memory_database(factory=Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factory_as_positional_arg(self): class Factory(sqlite.Connection): def __init__(self, *args, **kwargs): super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + regex = ( + r"Passing more than 1 positional argument to _sqlite3.Connection\(\) " + r"is deprecated. Parameters 'timeout', 'detect_types', " + r"'isolation_level', 'check_same_thread', 'factory', " + r"'cached_statements' and 'uri' will become keyword-only " + r"parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with memory_database(5.0, 0, None, True, Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) + self.assertEqual(cm.filename, __file__) -class CursorFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class CursorFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def tearDown(self): self.con.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_instance(self): cur = self.con.cursor() self.assertIsInstance(cur, sqlite.Cursor) @@ -98,12 +109,10 @@ def test_invalid_factory(self): # invalid callable returning non-cursor self.assertRaises(TypeError, self.con.cursor, lambda con: None) -class RowFactoryTestsBackwardsCompat(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - # TODO: RUSTPYTHON - @unittest.expectedFailure +class RowFactoryTestsBackwardsCompat(MemoryDatabaseMixin, unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_is_produced_by_factory(self): cur = self.con.cursor(factory=MyCursor) cur.execute("select 4+5 as foo") @@ -111,22 +120,20 @@ def test_is_produced_by_factory(self): self.assertIsInstance(row, dict) cur.close() - def tearDown(self): - self.con.close() -class RowFactoryTests(unittest.TestCase): +class RowFactoryTests(MemoryDatabaseMixin, unittest.TestCase): + def setUp(self): - self.con = sqlite.connect(":memory:") + super().setUp() + self.con.row_factory = sqlite.Row def test_custom_factory(self): self.con.row_factory = lambda cur, row: list(row) row = self.con.execute("select 1, 2").fetchone() self.assertIsInstance(row, list) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sqlite_row_index(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a_1, 2 as b").fetchone() self.assertIsInstance(row, sqlite.Row) @@ -156,10 +163,8 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sqlite_row_index_unicode(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) with self.assertRaises(IndexError): @@ -169,7 +174,6 @@ def test_sqlite_row_index_unicode(self): def test_sqlite_row_slice(self): # A sqlite.Row can be sliced like a list. - self.con.row_factory = sqlite.Row row = self.con.execute("select 1, 2, 3, 4").fetchone() self.assertEqual(row[0:0], ()) self.assertEqual(row[0:1], (1,)) @@ -186,30 +190,32 @@ def test_sqlite_row_slice(self): self.assertEqual(row[3:0:-2], (4, 2)) def test_sqlite_row_iter(self): - """Checks if the row object is iterable""" - self.con.row_factory = sqlite.Row + # Checks if the row object is iterable. row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): - """Checks if the row object can be converted to a tuple""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be converted to a tuple. row = self.con.execute("select 1 as a, 2 as b").fetchone() t = tuple(row) self.assertEqual(t, (row['a'], row['b'])) def test_sqlite_row_as_dict(self): - """Checks if the row object can be correctly converted to a dictionary""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be correctly converted to a dictionary. row = self.con.execute("select 1 as a, 2 as b").fetchone() d = dict(row) self.assertEqual(d["a"], row["a"]) self.assertEqual(d["b"], row["b"]) def test_sqlite_row_hash_cmp(self): - """Checks if the row object compares and hashes correctly""" - self.con.row_factory = sqlite.Row + # Checks if the row object compares and hashes correctly. row_1 = self.con.execute("select 1 as a, 2 as b").fetchone() row_2 = self.con.execute("select 1 as a, 2 as b").fetchone() row_3 = self.con.execute("select 1 as a, 3 as b").fetchone() @@ -241,33 +247,30 @@ def test_sqlite_row_hash_cmp(self): self.assertEqual(hash(row_1), hash(row_2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sqlite_row_as_sequence(self): - """ Checks if the row object can act like a sequence """ - self.con.row_factory = sqlite.Row + # Checks if the row object can act like a sequence. row = self.con.execute("select 1 as a, 2 as b").fetchone() as_tuple = tuple(row) self.assertEqual(list(reversed(row)), list(reversed(as_tuple))) self.assertIsInstance(row, Sequence) + def test_sqlite_row_keys(self): + # Checks if the row object can return a list of columns as strings. + row = self.con.execute("select 1 as a, 2 as b").fetchone() + self.assertEqual(row.keys(), ['a', 'b']) + def test_fake_cursor_class(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # segmentation fault. # Issue #27861: Also applies for cursor factory. class FakeCursor(str): __class__ = sqlite.Cursor - self.con.row_factory = sqlite.Row self.assertRaises(TypeError, self.con.cursor, FakeCursor) self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ()) - def tearDown(self): - self.con.close() -class TextFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TextFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_unicode(self): austria = "Österreich" @@ -286,17 +289,19 @@ def test_custom(self): austria = "Österreich" row = self.con.execute("select ?", (austria,)).fetchone() self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") - self.assertTrue(row[0].endswith("reich"), "column must contain original data") + self.assertEndsWith(row[0], "reich", "column must contain original data") - def tearDown(self): - self.con.close() class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): + def setUp(self): self.con = sqlite.connect(":memory:") self.con.execute("create table test (value text)") self.con.execute("insert into test (value) values (?)", ("a\x00b",)) + def tearDown(self): + self.con.close() + def test_string(self): # text_factory defaults to str row = self.con.execute("select value from test").fetchone() @@ -322,9 +327,6 @@ def test_custom(self): self.assertIs(type(row[0]), bytes) self.assertEqual(row[0], b"a\x00b") - def tearDown(self): - self.con.close() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 21042b9bf10..c47cfab180d 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -26,34 +26,31 @@ from test.support.os_helper import TESTFN, unlink -from test.test_sqlite3.test_dbapi import memory_database, cx_limit -from test.test_sqlite3.test_userfunctions import with_tracebacks +from .util import memory_database, cx_limit, with_tracebacks +from .util import MemoryDatabaseMixin -class CollationTests(unittest.TestCase): +class CollationTests(MemoryDatabaseMixin, unittest.TestCase): + def test_create_collation_not_string(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError): - con.create_collation(None, lambda x, y: (x > y) - (x < y)) + self.con.create_collation(None, lambda x, y: (x > y) - (x < y)) def test_create_collation_not_callable(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError) as cm: - con.create_collation("X", 42) + self.con.create_collation("X", 42) self.assertEqual(str(cm.exception), 'parameter must be callable') def test_create_collation_not_ascii(self): - con = sqlite.connect(":memory:") - con.create_collation("collä", lambda x, y: (x > y) - (x < y)) + self.con.create_collation("collä", lambda x, y: (x > y) - (x < y)) def test_create_collation_bad_upper(self): class BadUpperStr(str): def upper(self): return None - con = sqlite.connect(":memory:") mycoll = lambda x, y: -((x > y) - (x < y)) - con.create_collation(BadUpperStr("mycoll"), mycoll) - result = con.execute(""" + self.con.create_collation(BadUpperStr("mycoll"), mycoll) + result = self.con.execute(""" select x from ( select 'a' as x union @@ -68,8 +65,7 @@ def mycoll(x, y): # reverse order return -((x > y) - (x < y)) - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -79,21 +75,20 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg='the expected order was not returned') - con.create_collation("mycoll", None) + self.con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') def test_collation_returns_large_integer(self): def mycoll(x, y): # reverse order return -((x > y) - (x < y)) * 2**32 - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -103,7 +98,7 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg="the expected order was not returned") @@ -112,7 +107,7 @@ def test_collation_register_twice(self): Register two different collation functions under the same name. Verify that the last one is actually used. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", lambda x, y: -((x > y) - (x < y))) result = con.execute(""" @@ -126,25 +121,26 @@ def test_deregister_collation(self): Register a collation, then deregister it. Make sure an error is raised if we try to use it. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: con.execute("select 'a' as x union select 'b' as x order by x collate mycoll") self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') -class ProgressTests(unittest.TestCase): + +class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. """ - con = sqlite.connect(":memory:") progress_calls = [] def progress(): progress_calls.append(None) return 0 - con.set_progress_handler(progress, 1) - con.execute(""" + self.con.set_progress_handler(progress, 1) + self.con.execute(""" create table foo(a, b) """) self.assertTrue(progress_calls) @@ -153,7 +149,7 @@ def test_opcode_count(self): """ Test that the opcode argument is respected. """ - con = sqlite.connect(":memory:") + con = self.con progress_calls = [] def progress(): progress_calls.append(None) @@ -176,11 +172,10 @@ def test_cancel_operation(self): """ Test that returning a non-zero value stops the operation in progress. """ - con = sqlite.connect(":memory:") def progress(): return 1 - con.set_progress_handler(progress, 1) - curs = con.cursor() + self.con.set_progress_handler(progress, 1) + curs = self.con.cursor() self.assertRaises( sqlite.OperationalError, curs.execute, @@ -190,7 +185,7 @@ def test_clear_handler(self): """ Test that setting the progress handler to None clears the previously set handler. """ - con = sqlite.connect(":memory:") + con = self.con action = 0 def progress(): nonlocal action @@ -201,33 +196,47 @@ def progress(): con.execute("select 1 union select 2 union select 3").fetchall() self.assertEqual(action, 0, "progress handler was not cleared") - @with_tracebacks(ZeroDivisionError, name="bad_progress") + @unittest.expectedFailure # TODO: RUSTPYTHON + @with_tracebacks(ZeroDivisionError, msg_regex="bad_progress") def test_error_in_progress_handler(self): - con = sqlite.connect(":memory:") def bad_progress(): 1 / 0 - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) - @with_tracebacks(ZeroDivisionError, name="bad_progress") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="bad_progress") def test_error_in_progress_handler_result(self): - con = sqlite.connect(":memory:") class BadBool: def __bool__(self): 1 / 0 def bad_progress(): return BadBool() - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_progress_handler + def test_progress_handler_keyword_args(self): + regex = ( + r"Passing keyword argument 'progress_handler' to " + r"_sqlite3.Connection.set_progress_handler\(\) is deprecated. " + r"Parameter 'progress_handler' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_progress_handler(progress_handler=lambda: None, n=1) + self.assertEqual(cm.filename, __file__) + + +class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): -class TraceCallbackTests(unittest.TestCase): @contextlib.contextmanager def check_stmt_trace(self, cx, expected): try: @@ -242,12 +251,11 @@ def test_trace_callback_used(self): """ Test that the trace callback is invoked once it is set. """ - con = sqlite.connect(":memory:") traced_statements = [] def trace(statement): traced_statements.append(statement) - con.set_trace_callback(trace) - con.execute("create table foo(a, b)") + self.con.set_trace_callback(trace) + self.con.execute("create table foo(a, b)") self.assertTrue(traced_statements) self.assertTrue(any("create table foo" in stmt for stmt in traced_statements)) @@ -255,7 +263,7 @@ def test_clear_trace_callback(self): """ Test that setting the trace callback to None clears the previously set callback. """ - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -269,7 +277,7 @@ def test_unicode_content(self): Test that the statement can contain unicode literals. """ unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac' - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -317,13 +325,14 @@ def test_trace_expanded_sql(self): cx.execute("create table t(t)") cx.executemany("insert into t values(?)", ((v,) for v in range(3))) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks( sqlite.DataError, regex="Expanded SQL string exceeds the maximum string length" ) def test_trace_too_much_expanded_sql(self): # If the expanded string is too large, we'll fall back to the - # unexpanded SQL statement (for SQLite 3.14.0 and newer). + # unexpanded SQL statement. # The resulting string length is limited by the runtime limit # SQLITE_LIMIT_LENGTH. template = "select 1 as a where a=" @@ -334,8 +343,6 @@ def test_trace_too_much_expanded_sql(self): unexpanded_query = template + "?" expected = [unexpanded_query] - if sqlite.sqlite_version_info < (3, 14, 0): - expected = [] with self.check_stmt_trace(cx, expected): cx.execute(unexpanded_query, (bad_param,)) @@ -343,12 +350,26 @@ def test_trace_too_much_expanded_sql(self): with self.check_stmt_trace(cx, [expanded_query]): cx.execute(unexpanded_query, (ok_param,)) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(ZeroDivisionError, regex="division by zero") def test_trace_bad_handler(self): with memory_database() as cx: cx.set_trace_callback(lambda stmt: 5/0) cx.execute("select 1") + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_trace_callback + def test_trace_keyword_args(self): + regex = ( + r"Passing keyword argument 'trace_callback' to " + r"_sqlite3.Connection.set_trace_callback\(\) is deprecated. " + r"Parameter 'trace_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_trace_callback(trace_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index d746be647c6..0ebd6d5e9da 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,15 +28,12 @@ from test import support from unittest.mock import patch -from test.test_sqlite3.test_dbapi import memory_database, cx_limit +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin -class RegressionTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - def tearDown(self): - self.con.close() +class RegressionTests(MemoryDatabaseMixin, unittest.TestCase): def test_pragma_user_version(self): # This used to crash pysqlite because this pragma command returns NULL for the column name @@ -45,28 +42,24 @@ def test_pragma_user_version(self): def test_pragma_schema_version(self): # This still crashed pysqlite <= 2.2.1 - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES) - try: + with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con: cur = self.con.cursor() cur.execute("pragma schema_version") - finally: - cur.close() - con.close() def test_statement_reset(self): # pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are # reset before a rollback, but only those that are still in the # statement cache. The others are not accessible from the connection object. - con = sqlite.connect(":memory:", cached_statements=5) - cursors = [con.cursor() for x in range(5)] - cursors[0].execute("create table test(x)") - for i in range(10): - cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) + with memory_database(cached_statements=5) as con: + cursors = [con.cursor() for x in range(5)] + cursors[0].execute("create table test(x)") + for i in range(10): + cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) - for i in range(5): - cursors[i].execute(" " * i + "select x from test") + for i in range(5): + cursors[i].execute(" " * i + "select x from test") - con.rollback() + con.rollback() def test_column_name_with_spaces(self): cur = self.con.cursor() @@ -81,17 +74,15 @@ def test_statement_finalization_on_close_db(self): # cache when closing the database. statements that were still # referenced in cursors weren't closed and could provoke " # "OperationalError: Unable to close due to unfinalised statements". - con = sqlite.connect(":memory:") cursors = [] # default statement cache size is 100 for i in range(105): - cur = con.cursor() + cur = self.con.cursor() cursors.append(cur) cur.execute("select 1 x union select " + str(i)) - con.close() def test_on_conflict_rollback(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table foo(x, unique(x) on conflict rollback)") con.execute("insert into foo(x) values (1)") try: @@ -126,16 +117,16 @@ def test_type_map_usage(self): a statement. This test exhibits the problem. """ SELECT = "select * from foo" - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("create table foo(bar timestamp)") - with self.assertWarnsRegex(DeprecationWarning, "adapter"): - cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) - cur.execute(SELECT) - cur.execute("drop table foo") - cur.execute("create table foo(bar integer)") - cur.execute("insert into foo(bar) values (5)") - cur.execute(SELECT) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("create table foo(bar timestamp)") + with self.assertWarnsRegex(DeprecationWarning, "adapter"): + cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) + cur.execute(SELECT) + cur.execute("drop table foo") + cur.execute("create table foo(bar integer)") + cur.execute("insert into foo(bar) values (5)") + cur.execute(SELECT) def test_bind_mutating_list(self): # Issue41662: Crash when mutate a list of parameters during iteration. @@ -144,11 +135,11 @@ def __conform__(self, protocol): parameters.clear() return "..." parameters = [X(), 0] - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - con.execute("create table foo(bar X, baz integer)") - # Should not crash - with self.assertRaises(IndexError): - con.execute("insert into foo(bar, baz) values (?, ?)", parameters) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + con.execute("create table foo(bar X, baz integer)") + # Should not crash + with self.assertRaises(IndexError): + con.execute("insert into foo(bar, baz) values (?, ?)", parameters) def test_error_msg_decode_error(self): # When porting the module to Python 3.0, the error message about @@ -173,7 +164,7 @@ def upper(self): def __del__(self): con.isolation_level = "" - con = sqlite.connect(":memory:") + con = self.con con.isolation_level = None for level in "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE": with self.subTest(level=level): @@ -204,8 +195,7 @@ class Cursor(sqlite.Cursor): def __init__(self, con): pass - con = sqlite.connect(":memory:") - cur = Cursor(con) + cur = Cursor(self.con) with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4+5").fetchall() with self.assertRaisesRegex(sqlite.ProgrammingError, @@ -238,7 +228,9 @@ def test_auto_commit(self): 2.5.3 introduced a regression so that these could no longer be created. """ - con = sqlite.connect(":memory:", isolation_level=None) + with memory_database(isolation_level=None) as con: + self.assertIsNone(con.isolation_level) + self.assertFalse(con.in_transaction) def test_pragma_autocommit(self): """ @@ -266,7 +258,7 @@ def collation_cb(a, b): # Lone surrogate cannot be encoded to the default encoding (utf8) "\uDC80", collation_cb) - @unittest.skip("TODO: RUSTPYTHON deadlock") + @unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') def test_recursive_cursor_use(self): """ http://bugs.python.org/issue10811 @@ -274,9 +266,7 @@ def test_recursive_cursor_use(self): Recursively using a cursor, such as when reusing it from a generator led to segfaults. Now we catch recursive cursor usage and raise a ProgrammingError. """ - con = sqlite.connect(":memory:") - - cur = con.cursor() + cur = self.con.cursor() cur.execute("create table a (bar)") cur.execute("create table b (baz)") @@ -296,29 +286,31 @@ def test_convert_timestamp_microsecond_padding(self): since the microsecond string "456" actually represents "456000". """ - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("CREATE TABLE t (x TIMESTAMP)") + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("CREATE TABLE t (x TIMESTAMP)") - # Microseconds should be 456000 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") + # Microseconds should be 456000 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") - # Microseconds should be truncated to 123456 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") + # Microseconds should be truncated to 123456 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") - cur.execute("SELECT * FROM t") - with self.assertWarnsRegex(DeprecationWarning, "converter"): - values = [x[0] for x in cur.fetchall()] + cur.execute("SELECT * FROM t") + with self.assertWarnsRegex(DeprecationWarning, "converter"): + values = [x[0] for x in cur.fetchall()] - self.assertEqual(values, [ - datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), - datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), - ]) + self.assertEqual(values, [ + datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), + datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), + ]) + @unittest.expectedFailure # TODO: RUSTPYTHON; error message mismatch def test_invalid_isolation_level_type(self): # isolation level is a string, not an integer - self.assertRaises(TypeError, - sqlite.connect, ":memory:", isolation_level=123) + regex = "isolation_level must be str or None" + with self.assertRaisesRegex(TypeError, regex): + memory_database(isolation_level=123).__enter__() def test_null_character(self): @@ -334,7 +326,7 @@ def test_null_character(self): cur.execute, query) def test_surrogates(self): - con = sqlite.connect(":memory:") + con = self.con self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, con, "select '\udcff'") cur = con.cursor() @@ -360,7 +352,7 @@ def test_commit_cursor_reset(self): to return rows multiple times when fetched from cursors after commit. See issues 10513 and 23129 for details. """ - con = sqlite.connect(":memory:") + con = self.con con.executescript(""" create table t(c); create table t2(c); @@ -392,10 +384,9 @@ def test_bpo31770(self): """ def callback(*args): pass - con = sqlite.connect(":memory:") - cur = sqlite.Cursor(con) + cur = sqlite.Cursor(self.con) ref = weakref.ref(cur, callback) - cur.__init__(con) + cur.__init__(self.con) del cur # The interpreter shouldn't crash when ref is collected. del ref @@ -405,8 +396,7 @@ def test_del_isolation_level_segfault(self): with self.assertRaises(AttributeError): del self.con.isolation_level - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bpo37347(self): class Printer: def log(self, *args): @@ -428,6 +418,7 @@ def test_return_empty_bytestring(self): def test_table_lock_cursor_replace_stmt(self): with memory_database() as con: + con = self.con cur = con.cursor() cur.execute("create table t(t)") cur.executemany("insert into t values(?)", @@ -445,10 +436,11 @@ def test_table_lock_cursor_dealloc(self): con.commit() cur = con.execute("select t from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() - @unittest.skip("TODO: RUSTPYTHON deadlock") + @unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') def test_table_lock_cursor_non_readonly_select(self): with memory_database() as con: con.execute("create table t(t)") @@ -461,6 +453,7 @@ def dup(v): con.create_function("dup", 1, dup) cur = con.execute("select dup(t) from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() @@ -476,7 +469,7 @@ def test_executescript_step_through_select(self): self.assertEqual(steps, values) -@unittest.skip("TODO: RUSTPYTHON deadlock") +@unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') class RecursiveUseOfCursors(unittest.TestCase): # GH-80254: sqlite3 should not segfault for recursive use of cursors. msg = "Recursive use of cursors not allowed" @@ -496,21 +489,21 @@ def tearDown(self): def test_recursive_cursor_init(self): conv = lambda x: self.cur.__init__(self.con) with patch.dict(sqlite.converters, {"INIT": conv}): - self.cur.execute(f'select x as "x [INIT]", x from test') + self.cur.execute('select x as "x [INIT]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) def test_recursive_cursor_close(self): conv = lambda x: self.cur.close() with patch.dict(sqlite.converters, {"CLOSE": conv}): - self.cur.execute(f'select x as "x [CLOSE]", x from test') + self.cur.execute('select x as "x [CLOSE]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) def test_recursive_cursor_iter(self): conv = lambda x, l=[]: self.cur.fetchone() if l else l.append(None) with patch.dict(sqlite.converters, {"ITER": conv}): - self.cur.execute(f'select x as "x [ITER]", x from test') + self.cur.execute('select x as "x [ITER]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index 9c3d19e79bd..f38d042e598 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -22,22 +22,24 @@ import unittest import sqlite3 as sqlite +from contextlib import contextmanager -from test.support import LOOPBACK_TIMEOUT from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok -from test.test_sqlite3.test_dbapi import memory_database - - -TIMEOUT = LOOPBACK_TIMEOUT / 10 +from .util import memory_database +from .util import MemoryDatabaseMixin +@unittest.skip("TODO: RUSTPYTHON timeout parameter does not accept int type") class TransactionTests(unittest.TestCase): def setUp(self): - self.con1 = sqlite.connect(TESTFN, timeout=TIMEOUT) + # We can disable the busy handlers, since we control + # the order of SQLite C API operations. + self.con1 = sqlite.connect(TESTFN, timeout=0) self.cur1 = self.con1.cursor() - self.con2 = sqlite.connect(TESTFN, timeout=TIMEOUT) + self.con2 = sqlite.connect(TESTFN, timeout=0) self.cur2 = self.con2.cursor() def tearDown(self): @@ -117,10 +119,8 @@ def test_raise_timeout(self): self.cur2.execute("insert into test(i) values (5)") def test_locking(self): - """ - This tests the improved concurrency with pysqlite 2.3.4. You needed - to roll back con2 before you could commit con1. - """ + # This tests the improved concurrency with pysqlite 2.3.4. You needed + # to roll back con2 before you could commit con1. self.cur1.execute("create table test(i)") self.cur1.execute("insert into test(i) values (5)") with self.assertRaises(sqlite.OperationalError): @@ -130,14 +130,14 @@ def test_locking(self): def test_rollback_cursor_consistency(self): """Check that cursors behave correctly after rollback.""" - con = sqlite.connect(":memory:") - cur = con.cursor() - cur.execute("create table test(x)") - cur.execute("insert into test(x) values (5)") - cur.execute("select 1 union select 2 union select 3") + with memory_database() as con: + cur = con.cursor() + cur.execute("create table test(x)") + cur.execute("insert into test(x) values (5)") + cur.execute("select 1 union select 2 union select 3") - con.rollback() - self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) + con.rollback() + self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) def test_multiple_cursors_and_iternext(self): # gh-94028: statements are cleared and reset in cursor iternext. @@ -216,10 +216,7 @@ def test_no_duplicate_rows_after_rollback_new_query(self): -class SpecialCommandTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - self.cur = self.con.cursor() +class SpecialCommandTests(MemoryDatabaseMixin, unittest.TestCase): def test_drop_table(self): self.cur.execute("create table test(i)") @@ -231,14 +228,8 @@ def test_pragma(self): self.cur.execute("insert into test(i) values (5)") self.cur.execute("pragma count_changes=1") - def tearDown(self): - self.cur.close() - self.con.close() - -class TransactionalDDL(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TransactionalDDL(MemoryDatabaseMixin, unittest.TestCase): def test_ddl_does_not_autostart_transaction(self): # For backwards compatibility reasons, DDL statements should not @@ -266,9 +257,6 @@ def test_transactional_ddl(self): with self.assertRaises(sqlite.OperationalError): self.con.execute("select * from test") - def tearDown(self): - self.con.close() - class IsolationLevelFromInit(unittest.TestCase): CREATE = "create table t(t)" @@ -366,5 +354,183 @@ def test_isolation_level_none(self): self.assertEqual(self.traced, [self.QUERY]) +class AutocommitAttribute(unittest.TestCase): + """Test PEP 249-compliant autocommit behaviour.""" + legacy = sqlite.LEGACY_TRANSACTION_CONTROL + + @contextmanager + def check_stmt_trace(self, cx, expected, reset=True): + try: + traced = [] + cx.set_trace_callback(lambda stmt: traced.append(stmt)) + yield + finally: + self.assertEqual(traced, expected) + if reset: + cx.set_trace_callback(None) + + def test_autocommit_default(self): + with memory_database() as cx: + self.assertEqual(cx.autocommit, + sqlite.LEGACY_TRANSACTION_CONTROL) + + def test_autocommit_setget(self): + dataset = ( + True, + False, + sqlite.LEGACY_TRANSACTION_CONTROL, + ) + for mode in dataset: + with self.subTest(mode=mode): + with memory_database(autocommit=mode) as cx: + self.assertEqual(cx.autocommit, mode) + with memory_database() as cx: + cx.autocommit = mode + self.assertEqual(cx.autocommit, mode) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit validation error messages differ + def test_autocommit_setget_invalid(self): + msg = "autocommit must be True, False, or.*LEGACY" + for mode in "a", 12, (), None: + with self.subTest(mode=mode): + with self.assertRaisesRegex(ValueError, msg): + sqlite.connect(":memory:", autocommit=mode) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled(self): + expected = [ + "SELECT 1", + "COMMIT", + "BEGIN", + "ROLLBACK", + "BEGIN", + ] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("SELECT 1") + cx.commit() + cx.rollback() + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_implicit_rollback(self): + expected = ["ROLLBACK"] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected, reset=False): + cx.close() + + def test_autocommit_enabled(self): + expected = ["CREATE TABLE t(t)", "INSERT INTO t VALUES(1)"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("CREATE TABLE t(t)") + cx.execute("INSERT INTO t VALUES(1)") + self.assertFalse(cx.in_transaction) + + def test_autocommit_enabled_txn_ctl(self): + for op in "commit", "rollback": + with self.subTest(op=op): + with memory_database(autocommit=True) as cx: + meth = getattr(cx, op) + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, []): + meth() # expect this to pass silently + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_then_enabled(self): + expected = ["COMMIT"] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.autocommit = True # should commit + self.assertFalse(cx.in_transaction) + + def test_autocommit_enabled_then_disabled(self): + expected = ["BEGIN"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.autocommit = False # should begin + self.assertTrue(cx.in_transaction) + + def test_autocommit_explicit_then_disabled(self): + expected = ["BEGIN DEFERRED"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("BEGIN DEFERRED") + cx.autocommit = False # should now be a no-op + self.assertTrue(cx.in_transaction) + + def test_autocommit_enabled_ctx_mgr(self): + with memory_database(autocommit=True) as cx: + # The context manager is a no-op if autocommit=True + with self.check_stmt_trace(cx, []): + with cx: + self.assertFalse(cx.in_transaction) + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_ctx_mgr(self): + expected = ["COMMIT", "BEGIN"] + with memory_database(autocommit=False) as cx: + with self.check_stmt_trace(cx, expected): + with cx: + self.assertTrue(cx.in_transaction) + self.assertTrue(cx.in_transaction) + + def test_autocommit_compat_ctx_mgr(self): + expected = ["BEGIN ", "INSERT INTO T VALUES(1)", "COMMIT"] + with memory_database(autocommit=self.legacy) as cx: + cx.execute("create table t(t)") + with self.check_stmt_trace(cx, expected): + with cx: + self.assertFalse(cx.in_transaction) + cx.execute("INSERT INTO T VALUES(1)") + self.assertTrue(cx.in_transaction) + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_enabled_executescript(self): + expected = ["BEGIN", "SELECT 1"] + with memory_database(autocommit=True) as cx: + with self.check_stmt_trace(cx, expected): + self.assertFalse(cx.in_transaction) + cx.execute("BEGIN") + cx.executescript("SELECT 1") + self.assertTrue(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_executescript(self): + expected = ["SELECT 1"] + with memory_database(autocommit=False) as cx: + with self.check_stmt_trace(cx, expected): + self.assertTrue(cx.in_transaction) + cx.executescript("SELECT 1") + self.assertTrue(cx.in_transaction) + + def test_autocommit_compat_executescript(self): + expected = ["BEGIN", "COMMIT", "SELECT 1"] + with memory_database(autocommit=self.legacy) as cx: + with self.check_stmt_trace(cx, expected): + self.assertFalse(cx.in_transaction) + cx.execute("BEGIN") + cx.executescript("SELECT 1") + self.assertFalse(cx.in_transaction) + + def test_autocommit_disabled_implicit_shutdown(self): + # The implicit ROLLBACK should not call back into Python during + # interpreter tear-down. + code = """if 1: + import sqlite3 + cx = sqlite3.connect(":memory:", autocommit=False) + cx.set_trace_callback(print) + """ + assert_python_ok("-c", code, PYTHONIOENCODING="utf-8") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_types.py b/Lib/test/test_sqlite3/test_types.py index 62318823510..66d27d21b8d 100644 --- a/Lib/test/test_sqlite3/test_types.py +++ b/Lib/test/test_sqlite3/test_types.py @@ -106,9 +106,9 @@ def test_string_with_surrogates(self): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=4, dry_run=False) def test_too_large_string(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() @@ -117,9 +117,9 @@ def test_too_large_string(self, maxsize): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=3, dry_run=False) def test_too_large_blob(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() @@ -371,7 +371,6 @@ def test_cursor_description_insert(self): self.assertIsNone(self.cur.description) -@unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "CTEs not supported") class CommonTableExpressionTests(unittest.TestCase): def setUp(self): @@ -517,7 +516,7 @@ def test_sqlite_timestamp(self): self.assertEqual(ts, ts2) def test_sql_timestamp(self): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(tz=datetime.UTC) self.cur.execute("insert into test(ts) values (current_timestamp)") self.cur.execute("select ts from test") with self.assertWarnsRegex(DeprecationWarning, "converter"): diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index e8b98a66a57..3fdde4a26cd 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -21,55 +21,15 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. -import contextlib -import functools -import io -import re import sys import unittest import sqlite3 as sqlite from unittest.mock import Mock, patch -from test.support import bigmemtest, catch_unraisable_exception, gc_collect - -from test.test_sqlite3.test_dbapi import cx_limit - - -def with_tracebacks(exc, regex="", name=""): - """Convenience decorator for testing callback tracebacks.""" - def decorator(func): - _regex = re.compile(regex) if regex else None - @functools.wraps(func) - def wrapper(self, *args, **kwargs): - with catch_unraisable_exception() as cm: - # First, run the test with traceback enabled. - with check_tracebacks(self, cm, exc, _regex, name): - func(self, *args, **kwargs) - - # Then run the test with traceback disabled. - func(self, *args, **kwargs) - return wrapper - return decorator - - -@contextlib.contextmanager -def check_tracebacks(self, cm, exc, regex, obj_name): - """Convenience context manager for testing callback tracebacks.""" - sqlite.enable_callback_tracebacks(True) - try: - buf = io.StringIO() - with contextlib.redirect_stderr(buf): - yield - - # TODO: RUSTPYTHON need unraisable exception - # self.assertEqual(cm.unraisable.exc_type, exc) - # if regex: - # msg = str(cm.unraisable.exc_value) - # self.assertIsNotNone(regex.search(msg)) - # if obj_name: - # self.assertEqual(cm.unraisable.object.__name__, obj_name) - finally: - sqlite.enable_callback_tracebacks(False) +from test.support import bigmemtest, gc_collect + +from .util import cx_limit, memory_database +from .util import with_tracebacks def func_returntext(): @@ -196,7 +156,6 @@ def setUp(self): self.con.create_function("returnblob", 0, func_returnblob) self.con.create_function("returnlonglong", 0, func_returnlonglong) self.con.create_function("returnnan", 0, lambda: float("nan")) - self.con.create_function("returntoolargeint", 0, lambda: 1 << 65) self.con.create_function("return_noncont_blob", 0, lambda: memoryview(b"blob")[::2]) self.con.create_function("raiseexception", 0, func_raiseexception) @@ -211,8 +170,9 @@ def setUp(self): def tearDown(self): self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_func_error_on_create(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, lambda x: 2*x) def test_func_too_many_args(self): @@ -295,12 +255,8 @@ def test_func_return_nan(self): cur.execute("select returnnan()") self.assertIsNone(cur.fetchone()[0]) - def test_func_return_too_large_int(self): - cur = self.con.cursor() - self.assertRaisesRegex(sqlite.DataError, "string or blob too big", - self.con.execute, "select returntoolargeint()") - - @with_tracebacks(ZeroDivisionError, name="func_raiseexception") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="func_raiseexception") def test_func_exception(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -308,14 +264,16 @@ def test_func_exception(self): cur.fetchone() self.assertEqual(str(cm.exception), 'user-defined function raised exception') - @with_tracebacks(MemoryError, name="func_memoryerror") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(MemoryError, msg_regex="func_memoryerror") def test_func_memory_error(self): cur = self.con.cursor() with self.assertRaises(MemoryError): cur.execute("select memoryerror()") cur.fetchone() - @with_tracebacks(OverflowError, name="func_overflowerror") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(OverflowError, msg_regex="func_overflowerror") def test_func_overflow_error(self): cur = self.con.cursor() with self.assertRaises(sqlite.DataError): @@ -348,6 +306,7 @@ def test_non_contiguous_blob(self): self.con.execute, "select spam(?)", (memoryview(b"blob")[::2],)) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BufferError, regex="buffer.*contiguous") def test_return_non_contiguous_blob(self): with self.assertRaises(sqlite.OperationalError): @@ -388,38 +347,22 @@ def append_result(arg): # Regarding deterministic functions: # # Between 3.8.3 and 3.15.0, deterministic functions were only used to - # optimize inner loops, so for those versions we can only test if the - # sqlite machinery has factored out a call or not. From 3.15.0 and onward, - # deterministic functions were permitted in WHERE clauses of partial - # indices, which allows testing based on syntax, iso. the query optimizer. - @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") + # optimize inner loops. From 3.15.0 and onward, deterministic functions + # were permitted in WHERE clauses of partial indices, which allows testing + # based on syntax, iso. the query optimizer. def test_func_non_deterministic(self): mock = Mock(return_value=None) self.con.create_function("nondeterministic", 0, mock, deterministic=False) - if sqlite.sqlite_version_info < (3, 15, 0): - self.con.execute("select nondeterministic() = nondeterministic()") - self.assertEqual(mock.call_count, 2) - else: - with self.assertRaises(sqlite.OperationalError): - self.con.execute("create index t on test(t) where nondeterministic() is not null") + with self.assertRaises(sqlite.OperationalError): + self.con.execute("create index t on test(t) where nondeterministic() is not null") - @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") def test_func_deterministic(self): mock = Mock(return_value=None) self.con.create_function("deterministic", 0, mock, deterministic=True) - if sqlite.sqlite_version_info < (3, 15, 0): - self.con.execute("select deterministic() = deterministic()") - self.assertEqual(mock.call_count, 1) - else: - try: - self.con.execute("create index t on test(t) where deterministic() is not null") - except sqlite.OperationalError: - self.fail("Unexpected failure while creating partial index") - - @unittest.skipIf(sqlite.sqlite_version_info >= (3, 8, 3), "SQLite < 3.8.3 needed") - def test_func_deterministic_not_supported(self): - with self.assertRaises(sqlite.NotSupportedError): - self.con.create_function("deterministic", 0, int, deterministic=True) + try: + self.con.execute("create index t on test(t) where deterministic() is not null") + except sqlite.OperationalError: + self.fail("Unexpected failure while creating partial index") def test_func_deterministic_keyword_only(self): with self.assertRaises(TypeError): @@ -428,29 +371,32 @@ def test_func_deterministic_keyword_only(self): def test_function_destructor_via_gc(self): # See bpo-44304: The destructor of the user function can # crash if is called without the GIL from the gc functions - dest = sqlite.connect(':memory:') def md5sum(t): return - dest.create_function("md5", 1, md5sum) - x = dest("create table lang (name, first_appeared)") - del md5sum, dest + with memory_database() as dest: + dest.create_function("md5", 1, md5sum) + x = dest("create table lang (name, first_appeared)") + del md5sum, dest - y = [x] - y.append(y) + y = [x] + y.append(y) - del x,y - gc_collect() + del x,y + gc_collect() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(OverflowError) def test_func_return_too_large_int(self): cur = self.con.cursor() + msg = "string or blob too big" for value in 2**63, -2**63-1, 2**64: self.con.create_function("largeint", 0, lambda value=value: value) - with self.assertRaises(sqlite.DataError): + with self.assertRaisesRegex(sqlite.DataError, msg): cur.execute("select largeint()") - @with_tracebacks(UnicodeEncodeError, "surrogates not allowed", "chr") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(UnicodeEncodeError, "surrogates not allowed") def test_func_return_text_with_surrogates(self): cur = self.con.cursor() self.con.create_function("pychr", 1, chr) @@ -482,6 +428,30 @@ def test_func_return_illegal_value(self): self.assertRaisesRegex(sqlite.OperationalError, msg, self.con.execute, "select badreturn()") + @unittest.expectedFailure # TODO: RUSTPYTHON deprecation warning not emitted for keyword args + def test_func_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'narg' and 'func' to " + r"_sqlite3.Connection.create_function\(\) is deprecated. " + r"Parameters 'name', 'narg' and 'func' will become " + r"positional-only in Python 3.15." + ) + + def noop(): + return None + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", 0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function(name="noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + class WindowSumInt: def __init__(self): @@ -536,15 +506,20 @@ def setUp(self): """ self.con.create_window_function("sumint", 1, WindowSumInt) + def tearDown(self): + self.cur.close() + self.con.close() + def test_win_sum_int(self): self.cur.execute(self.query % "sumint") self.assertEqual(self.cur.fetchall(), self.expected) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_win_error_on_create(self): - self.assertRaises(sqlite.ProgrammingError, - self.con.create_window_function, - "shouldfail", -100, WindowSumInt) + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): + self.con.create_window_function("shouldfail", -100, WindowSumInt) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BadWindow) def test_win_exception_in_method(self): for meth in "__init__", "step", "value", "inverse": @@ -557,17 +532,19 @@ def test_win_exception_in_method(self): self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BadWindow) def test_win_exception_in_finalize(self): # Note: SQLite does not (as of version 3.38.0) propagate finalize # callback errors to sqlite3_step(); this implies that OperationalError # is _not_ raised. with patch.object(WindowSumInt, "finalize", side_effect=BadWindow): - name = f"exception_in_finalize" + name = "exception_in_finalize" self.con.create_window_function(name, 1, WindowSumInt) self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(AttributeError) def test_win_missing_method(self): class MissingValue: @@ -599,6 +576,7 @@ def finalize(self): return 42 self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(AttributeError) def test_win_missing_finalize(self): # Note: SQLite does not (as of version 3.38.0) propagate finalize @@ -656,6 +634,7 @@ def setUp(self): """) cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", ("foo", 5, 3.14, None, memoryview(b"blob"),)) + cur.close() self.con.create_aggregate("nostep", 1, AggrNoStep) self.con.create_aggregate("nofinalize", 1, AggrNoFinalize) @@ -668,15 +647,15 @@ def setUp(self): self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): - #self.cur.close() - #self.con.close() - pass + self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_aggr_error_on_create(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, AggrSum) - @with_tracebacks(AttributeError, name="AggrNoStep") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(AttributeError, msg_regex="AggrNoStep") def test_aggr_no_step(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -691,7 +670,8 @@ def test_aggr_no_finalize(self): cur.execute("select nofinalize(t) from test") val = cur.fetchone()[0] - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInInit") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInInit") def test_aggr_exception_in_init(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -699,7 +679,8 @@ def test_aggr_exception_in_init(self): val = cur.fetchone()[0] self.assertEqual(str(cm.exception), "user-defined aggregate's '__init__' method raised error") - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInStep") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInStep") def test_aggr_exception_in_step(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -707,7 +688,8 @@ def test_aggr_exception_in_step(self): val = cur.fetchone()[0] self.assertEqual(str(cm.exception), "user-defined aggregate's 'step' method raised error") - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInFinalize") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInFinalize") def test_aggr_exception_in_finalize(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -772,6 +754,28 @@ def test_aggr_text(self): val = cur.fetchone()[0] self.assertEqual(val, txt) + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for create_aggregate + def test_agg_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' to " + r"_sqlite3.Connection.create_aggregate\(\) is deprecated. " + r"Parameters 'name', 'n_arg' and 'aggregate_class' will become " + r"positional-only in Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", 1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", n_arg=1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate(name="test", n_arg=0, + aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + class AuthorizerTests(unittest.TestCase): @staticmethod @@ -783,8 +787,6 @@ def authorizer_cb(action, arg1, arg2, dbname, source): return sqlite.SQLITE_OK def setUp(self): - # TODO: RUSTPYTHON difference 'prohibited' - self.prohibited = 'not authorized' self.con = sqlite.connect(":memory:") self.con.executescript(""" create table t1 (c1, c2); @@ -799,23 +801,38 @@ def setUp(self): self.con.set_authorizer(self.authorizer_cb) def tearDown(self): - pass + self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs def test_table_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select * from t2") - self.assertIn(self.prohibited, str(cm.exception)) + self.assertIn('prohibited', str(cm.exception)) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs def test_column_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select c2 from t1") - self.assertIn(self.prohibited, str(cm.exception)) + self.assertIn('prohibited', str(cm.exception)) def test_clear_authorizer(self): self.con.set_authorizer(None) self.con.execute("select * from t2") self.con.execute("select c2 from t1") + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_authorizer + def test_authorizer_keyword_args(self): + regex = ( + r"Passing keyword argument 'authorizer_callback' to " + r"_sqlite3.Connection.set_authorizer\(\) is deprecated. " + r"Parameter 'authorizer_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_authorizer(authorizer_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + class AuthorizerRaiseExceptionTests(AuthorizerTests): @staticmethod @@ -826,11 +843,13 @@ def authorizer_cb(action, arg1, arg2, dbname, source): raise ValueError return sqlite.SQLITE_OK - @with_tracebacks(ValueError, name="authorizer_cb") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ValueError, msg_regex="authorizer_cb") def test_table_access(self): super().test_table_access() - @with_tracebacks(ValueError, name="authorizer_cb") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ValueError, msg_regex="authorizer_cb") def test_column_access(self): super().test_table_access() diff --git a/Lib/test/test_sqlite3/util.py b/Lib/test/test_sqlite3/util.py new file mode 100644 index 00000000000..cccd062160f --- /dev/null +++ b/Lib/test/test_sqlite3/util.py @@ -0,0 +1,89 @@ +import contextlib +import functools +import io +import re +import sqlite3 +import test.support +import unittest + + +# Helper for temporary memory databases +def memory_database(*args, **kwargs): + cx = sqlite3.connect(":memory:", *args, **kwargs) + return contextlib.closing(cx) + + +# Temporarily limit a database connection parameter +@contextlib.contextmanager +def cx_limit(cx, category=sqlite3.SQLITE_LIMIT_SQL_LENGTH, limit=128): + try: + _prev = cx.setlimit(category, limit) + yield limit + finally: + cx.setlimit(category, _prev) + + +def with_tracebacks(exc, regex="", name="", msg_regex=""): + """Convenience decorator for testing callback tracebacks.""" + def decorator(func): + exc_regex = re.compile(regex) if regex else None + _msg_regex = re.compile(msg_regex) if msg_regex else None + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with test.support.catch_unraisable_exception() as cm: + # First, run the test with traceback enabled. + with check_tracebacks(self, cm, exc, exc_regex, _msg_regex, name): + func(self, *args, **kwargs) + + # Then run the test with traceback disabled. + func(self, *args, **kwargs) + return wrapper + return decorator + + +@contextlib.contextmanager +def check_tracebacks(self, cm, exc, exc_regex, msg_regex, obj_name): + """Convenience context manager for testing callback tracebacks.""" + sqlite3.enable_callback_tracebacks(True) + try: + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + yield + + self.assertEqual(cm.unraisable.exc_type, exc) + if exc_regex: + msg = str(cm.unraisable.exc_value) + self.assertIsNotNone(exc_regex.search(msg), (exc_regex, msg)) + if msg_regex: + msg = cm.unraisable.err_msg + self.assertIsNotNone(msg_regex.search(msg), (msg_regex, msg)) + if obj_name: + self.assertEqual(cm.unraisable.object.__name__, obj_name) + finally: + sqlite3.enable_callback_tracebacks(False) + + +class MemoryDatabaseMixin: + + def setUp(self): + self.con = sqlite3.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + + @property + def cx(self): + return self.con + + @property + def cu(self): + return self.cur + + +def requires_virtual_table(module): + with memory_database() as cx: + supported = (module,) in list(cx.execute("PRAGMA module_list")) + reason = f"Requires {module!r} virtual table support" + return unittest.skipUnless(supported, reason) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f073def5bc1..71b54e286a3 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2891,6 +2891,7 @@ def test_echo(self): 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', str(e.exception)) + @unittest.skip('TODO: RUSTPYTHON flaky') @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") def test_ssl_in_multiple_threads(self): # See GH-124984: OpenSSL is not thread safe. @@ -4595,7 +4596,7 @@ def server_callback(identity): with client_context.wrap_socket(socket.socket()) as s: s.connect((HOST, server.port)) - @unittest.skip("TODO: rustpython") + @unittest.skip("TODO: RUSTPYTHON; Hangs") def test_thread_recv_while_main_thread_sends(self): # GH-137583: Locking was added to calls to send() and recv() on SSL # socket objects. This seemed fine at the surface level because those @@ -5375,7 +5376,6 @@ def call_after_accept(conn_to_client): class TestEnumerations(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tlsversion(self): class CheckedTLSVersion(enum.IntEnum): MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED @@ -5387,7 +5387,6 @@ class CheckedTLSVersion(enum.IntEnum): MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED enum._test_simple_enum(CheckedTLSVersion, TLSVersion) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tlscontenttype(self): class Checked_TLSContentType(enum.IntEnum): """Content types (record layer) @@ -5403,7 +5402,6 @@ class Checked_TLSContentType(enum.IntEnum): INNER_CONTENT_TYPE = 0x101 enum._test_simple_enum(Checked_TLSContentType, _TLSContentType) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tlsalerttype(self): class Checked_TLSAlertType(enum.IntEnum): """Alert types for TLSContentType.ALERT messages @@ -5446,7 +5444,6 @@ class Checked_TLSAlertType(enum.IntEnum): NO_APPLICATION_PROTOCOL = 120 enum._test_simple_enum(Checked_TLSAlertType, _TLSAlertType) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tlsmessagetype(self): class Checked_TLSMessageType(enum.IntEnum): """Message types (handshake protocol) @@ -5477,7 +5474,6 @@ class Checked_TLSMessageType(enum.IntEnum): CHANGE_CIPHER_SPEC = 0x0101 enum._test_simple_enum(Checked_TLSMessageType, _TLSMessageType) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_sslmethod(self): Checked_SSLMethod = enum._old_convert_( enum.IntEnum, '_SSLMethod', 'ssl', @@ -5488,7 +5484,6 @@ def test_sslmethod(self): Checked_SSLMethod.PROTOCOL_SSLv23 = Checked_SSLMethod.PROTOCOL_TLS enum._test_simple_enum(Checked_SSLMethod, ssl._SSLMethod) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_options(self): CheckedOptions = enum._old_convert_( enum.IntFlag, 'Options', 'ssl', @@ -5497,7 +5492,6 @@ def test_options(self): ) enum._test_simple_enum(CheckedOptions, ssl.Options) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_alertdescription(self): CheckedAlertDescription = enum._old_convert_( enum.IntEnum, 'AlertDescription', 'ssl', @@ -5506,7 +5500,6 @@ def test_alertdescription(self): ) enum._test_simple_enum(CheckedAlertDescription, ssl.AlertDescription) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_sslerrornumber(self): Checked_SSLErrorNumber = enum._old_convert_( enum.IntEnum, 'SSLErrorNumber', 'ssl', @@ -5515,7 +5508,6 @@ def test_sslerrornumber(self): ) enum._test_simple_enum(Checked_SSLErrorNumber, ssl.SSLErrorNumber) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_verifyflags(self): CheckedVerifyFlags = enum._old_convert_( enum.IntFlag, 'VerifyFlags', 'ssl', @@ -5524,7 +5516,6 @@ def test_verifyflags(self): ) enum._test_simple_enum(CheckedVerifyFlags, ssl.VerifyFlags) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_verifymode(self): CheckedVerifyMode = enum._old_convert_( enum.IntEnum, 'VerifyMode', 'ssl', diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py new file mode 100644 index 00000000000..1e6f69d49e9 --- /dev/null +++ b/Lib/test/test_stable_abi_ctypes.py @@ -0,0 +1,1004 @@ +# Generated by Tools/build/stable_abi.py + +"""Test that all symbols of the Stable ABI are accessible using ctypes +""" + +import sys +import unittest +from test.support.import_helper import import_module +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") + +feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + +ctypes_test = import_module('ctypes') + +class TestStableABIAvailability(unittest.TestCase): + def test_available_symbols(self): + + for symbol_name in SYMBOL_NAMES: + with self.subTest(symbol_name): + ctypes_test.pythonapi[symbol_name] + + def test_feature_macros(self): + self.assertEqual( + set(get_feature_macros()), EXPECTED_FEATURE_MACROS) + + # The feature macros for Windows are used in creating the DLL + # definition, so they must be known on all platforms. + # If we are on Windows, we check that the hardcoded data matches + # the reality. + @unittest.skipIf(sys.platform != "win32", "Windows specific test") + def test_windows_feature_macros(self): + for name, value in WINDOWS_FEATURE_MACROS.items(): + if value != 'maybe': + with self.subTest(name): + self.assertEqual(feature_macros[name], value) + +SYMBOL_NAMES = ( + + "PyAIter_Check", + "PyArg_Parse", + "PyArg_ParseTuple", + "PyArg_ParseTupleAndKeywords", + "PyArg_UnpackTuple", + "PyArg_VaParse", + "PyArg_VaParseTupleAndKeywords", + "PyArg_ValidateKeywordArguments", + "PyBaseObject_Type", + "PyBool_FromLong", + "PyBool_Type", + "PyBuffer_FillContiguousStrides", + "PyBuffer_FillInfo", + "PyBuffer_FromContiguous", + "PyBuffer_GetPointer", + "PyBuffer_IsContiguous", + "PyBuffer_Release", + "PyBuffer_SizeFromFormat", + "PyBuffer_ToContiguous", + "PyByteArrayIter_Type", + "PyByteArray_AsString", + "PyByteArray_Concat", + "PyByteArray_FromObject", + "PyByteArray_FromStringAndSize", + "PyByteArray_Resize", + "PyByteArray_Size", + "PyByteArray_Type", + "PyBytesIter_Type", + "PyBytes_AsString", + "PyBytes_AsStringAndSize", + "PyBytes_Concat", + "PyBytes_ConcatAndDel", + "PyBytes_DecodeEscape", + "PyBytes_FromFormat", + "PyBytes_FromFormatV", + "PyBytes_FromObject", + "PyBytes_FromString", + "PyBytes_FromStringAndSize", + "PyBytes_Repr", + "PyBytes_Size", + "PyBytes_Type", + "PyCFunction_Call", + "PyCFunction_GetFlags", + "PyCFunction_GetFunction", + "PyCFunction_GetSelf", + "PyCFunction_New", + "PyCFunction_NewEx", + "PyCFunction_Type", + "PyCMethod_New", + "PyCallIter_New", + "PyCallIter_Type", + "PyCallable_Check", + "PyCapsule_GetContext", + "PyCapsule_GetDestructor", + "PyCapsule_GetName", + "PyCapsule_GetPointer", + "PyCapsule_Import", + "PyCapsule_IsValid", + "PyCapsule_New", + "PyCapsule_SetContext", + "PyCapsule_SetDestructor", + "PyCapsule_SetName", + "PyCapsule_SetPointer", + "PyCapsule_Type", + "PyClassMethodDescr_Type", + "PyCodec_BackslashReplaceErrors", + "PyCodec_Decode", + "PyCodec_Decoder", + "PyCodec_Encode", + "PyCodec_Encoder", + "PyCodec_IgnoreErrors", + "PyCodec_IncrementalDecoder", + "PyCodec_IncrementalEncoder", + "PyCodec_KnownEncoding", + "PyCodec_LookupError", + "PyCodec_NameReplaceErrors", + "PyCodec_Register", + "PyCodec_RegisterError", + "PyCodec_ReplaceErrors", + "PyCodec_StreamReader", + "PyCodec_StreamWriter", + "PyCodec_StrictErrors", + "PyCodec_Unregister", + "PyCodec_XMLCharRefReplaceErrors", + "PyComplex_FromDoubles", + "PyComplex_ImagAsDouble", + "PyComplex_RealAsDouble", + "PyComplex_Type", + "PyDescr_NewClassMethod", + "PyDescr_NewGetSet", + "PyDescr_NewMember", + "PyDescr_NewMethod", + "PyDictItems_Type", + "PyDictIterItem_Type", + "PyDictIterKey_Type", + "PyDictIterValue_Type", + "PyDictKeys_Type", + "PyDictProxy_New", + "PyDictProxy_Type", + "PyDictRevIterItem_Type", + "PyDictRevIterKey_Type", + "PyDictRevIterValue_Type", + "PyDictValues_Type", + "PyDict_Clear", + "PyDict_Contains", + "PyDict_Copy", + "PyDict_DelItem", + "PyDict_DelItemString", + "PyDict_GetItem", + "PyDict_GetItemRef", + "PyDict_GetItemString", + "PyDict_GetItemStringRef", + "PyDict_GetItemWithError", + "PyDict_Items", + "PyDict_Keys", + "PyDict_Merge", + "PyDict_MergeFromSeq2", + "PyDict_New", + "PyDict_Next", + "PyDict_SetItem", + "PyDict_SetItemString", + "PyDict_Size", + "PyDict_Type", + "PyDict_Update", + "PyDict_Values", + "PyEllipsis_Type", + "PyEnum_Type", + "PyErr_BadArgument", + "PyErr_BadInternalCall", + "PyErr_CheckSignals", + "PyErr_Clear", + "PyErr_Display", + "PyErr_DisplayException", + "PyErr_ExceptionMatches", + "PyErr_Fetch", + "PyErr_Format", + "PyErr_FormatV", + "PyErr_GetExcInfo", + "PyErr_GetHandledException", + "PyErr_GetRaisedException", + "PyErr_GivenExceptionMatches", + "PyErr_NewException", + "PyErr_NewExceptionWithDoc", + "PyErr_NoMemory", + "PyErr_NormalizeException", + "PyErr_Occurred", + "PyErr_Print", + "PyErr_PrintEx", + "PyErr_ProgramText", + "PyErr_ResourceWarning", + "PyErr_Restore", + "PyErr_SetExcInfo", + "PyErr_SetFromErrno", + "PyErr_SetFromErrnoWithFilename", + "PyErr_SetFromErrnoWithFilenameObject", + "PyErr_SetFromErrnoWithFilenameObjects", + "PyErr_SetHandledException", + "PyErr_SetImportError", + "PyErr_SetImportErrorSubclass", + "PyErr_SetInterrupt", + "PyErr_SetInterruptEx", + "PyErr_SetNone", + "PyErr_SetObject", + "PyErr_SetRaisedException", + "PyErr_SetString", + "PyErr_SyntaxLocation", + "PyErr_SyntaxLocationEx", + "PyErr_WarnEx", + "PyErr_WarnExplicit", + "PyErr_WarnFormat", + "PyErr_WriteUnraisable", + "PyEval_AcquireLock", + "PyEval_AcquireThread", + "PyEval_CallFunction", + "PyEval_CallMethod", + "PyEval_CallObjectWithKeywords", + "PyEval_EvalCode", + "PyEval_EvalCodeEx", + "PyEval_EvalFrame", + "PyEval_EvalFrameEx", + "PyEval_GetBuiltins", + "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", + "PyEval_GetFuncDesc", + "PyEval_GetFuncName", + "PyEval_GetGlobals", + "PyEval_GetLocals", + "PyEval_InitThreads", + "PyEval_ReleaseLock", + "PyEval_ReleaseThread", + "PyEval_RestoreThread", + "PyEval_SaveThread", + "PyEval_ThreadsInitialized", + "PyExc_ArithmeticError", + "PyExc_AssertionError", + "PyExc_AttributeError", + "PyExc_BaseException", + "PyExc_BaseExceptionGroup", + "PyExc_BlockingIOError", + "PyExc_BrokenPipeError", + "PyExc_BufferError", + "PyExc_BytesWarning", + "PyExc_ChildProcessError", + "PyExc_ConnectionAbortedError", + "PyExc_ConnectionError", + "PyExc_ConnectionRefusedError", + "PyExc_ConnectionResetError", + "PyExc_DeprecationWarning", + "PyExc_EOFError", + "PyExc_EncodingWarning", + "PyExc_EnvironmentError", + "PyExc_Exception", + "PyExc_FileExistsError", + "PyExc_FileNotFoundError", + "PyExc_FloatingPointError", + "PyExc_FutureWarning", + "PyExc_GeneratorExit", + "PyExc_IOError", + "PyExc_ImportError", + "PyExc_ImportWarning", + "PyExc_IndentationError", + "PyExc_IndexError", + "PyExc_InterruptedError", + "PyExc_IsADirectoryError", + "PyExc_KeyError", + "PyExc_KeyboardInterrupt", + "PyExc_LookupError", + "PyExc_MemoryError", + "PyExc_ModuleNotFoundError", + "PyExc_NameError", + "PyExc_NotADirectoryError", + "PyExc_NotImplementedError", + "PyExc_OSError", + "PyExc_OverflowError", + "PyExc_PendingDeprecationWarning", + "PyExc_PermissionError", + "PyExc_ProcessLookupError", + "PyExc_RecursionError", + "PyExc_ReferenceError", + "PyExc_ResourceWarning", + "PyExc_RuntimeError", + "PyExc_RuntimeWarning", + "PyExc_StopAsyncIteration", + "PyExc_StopIteration", + "PyExc_SyntaxError", + "PyExc_SyntaxWarning", + "PyExc_SystemError", + "PyExc_SystemExit", + "PyExc_TabError", + "PyExc_TimeoutError", + "PyExc_TypeError", + "PyExc_UnboundLocalError", + "PyExc_UnicodeDecodeError", + "PyExc_UnicodeEncodeError", + "PyExc_UnicodeError", + "PyExc_UnicodeTranslateError", + "PyExc_UnicodeWarning", + "PyExc_UserWarning", + "PyExc_ValueError", + "PyExc_Warning", + "PyExc_ZeroDivisionError", + "PyExceptionClass_Name", + "PyException_GetArgs", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyException_SetArgs", + "PyException_SetCause", + "PyException_SetContext", + "PyException_SetTraceback", + "PyFile_FromFd", + "PyFile_GetLine", + "PyFile_WriteObject", + "PyFile_WriteString", + "PyFilter_Type", + "PyFloat_AsDouble", + "PyFloat_FromDouble", + "PyFloat_FromString", + "PyFloat_GetInfo", + "PyFloat_GetMax", + "PyFloat_GetMin", + "PyFloat_Type", + "PyFrame_GetCode", + "PyFrame_GetLineNumber", + "PyFrozenSet_New", + "PyFrozenSet_Type", + "PyGC_Collect", + "PyGC_Disable", + "PyGC_Enable", + "PyGC_IsEnabled", + "PyGILState_Ensure", + "PyGILState_GetThisThreadState", + "PyGILState_Release", + "PyGetSetDescr_Type", + "PyImport_AddModule", + "PyImport_AddModuleObject", + "PyImport_AddModuleRef", + "PyImport_AppendInittab", + "PyImport_ExecCodeModule", + "PyImport_ExecCodeModuleEx", + "PyImport_ExecCodeModuleObject", + "PyImport_ExecCodeModuleWithPathnames", + "PyImport_GetImporter", + "PyImport_GetMagicNumber", + "PyImport_GetMagicTag", + "PyImport_GetModule", + "PyImport_GetModuleDict", + "PyImport_Import", + "PyImport_ImportFrozenModule", + "PyImport_ImportFrozenModuleObject", + "PyImport_ImportModule", + "PyImport_ImportModuleLevel", + "PyImport_ImportModuleLevelObject", + "PyImport_ImportModuleNoBlock", + "PyImport_ReloadModule", + "PyIndex_Check", + "PyInterpreterState_Clear", + "PyInterpreterState_Delete", + "PyInterpreterState_Get", + "PyInterpreterState_GetDict", + "PyInterpreterState_GetID", + "PyInterpreterState_New", + "PyIter_Check", + "PyIter_Next", + "PyIter_NextItem", + "PyIter_Send", + "PyListIter_Type", + "PyListRevIter_Type", + "PyList_Append", + "PyList_AsTuple", + "PyList_GetItem", + "PyList_GetItemRef", + "PyList_GetSlice", + "PyList_Insert", + "PyList_New", + "PyList_Reverse", + "PyList_SetItem", + "PyList_SetSlice", + "PyList_Size", + "PyList_Sort", + "PyList_Type", + "PyLongRangeIter_Type", + "PyLong_AsDouble", + "PyLong_AsInt", + "PyLong_AsInt32", + "PyLong_AsInt64", + "PyLong_AsLong", + "PyLong_AsLongAndOverflow", + "PyLong_AsLongLong", + "PyLong_AsLongLongAndOverflow", + "PyLong_AsNativeBytes", + "PyLong_AsSize_t", + "PyLong_AsSsize_t", + "PyLong_AsUInt32", + "PyLong_AsUInt64", + "PyLong_AsUnsignedLong", + "PyLong_AsUnsignedLongLong", + "PyLong_AsUnsignedLongLongMask", + "PyLong_AsUnsignedLongMask", + "PyLong_AsVoidPtr", + "PyLong_FromDouble", + "PyLong_FromInt32", + "PyLong_FromInt64", + "PyLong_FromLong", + "PyLong_FromLongLong", + "PyLong_FromNativeBytes", + "PyLong_FromSize_t", + "PyLong_FromSsize_t", + "PyLong_FromString", + "PyLong_FromUInt32", + "PyLong_FromUInt64", + "PyLong_FromUnsignedLong", + "PyLong_FromUnsignedLongLong", + "PyLong_FromUnsignedNativeBytes", + "PyLong_FromVoidPtr", + "PyLong_GetInfo", + "PyLong_Type", + "PyMap_Type", + "PyMapping_Check", + "PyMapping_GetItemString", + "PyMapping_GetOptionalItem", + "PyMapping_GetOptionalItemString", + "PyMapping_HasKey", + "PyMapping_HasKeyString", + "PyMapping_HasKeyStringWithError", + "PyMapping_HasKeyWithError", + "PyMapping_Items", + "PyMapping_Keys", + "PyMapping_Length", + "PyMapping_SetItemString", + "PyMapping_Size", + "PyMapping_Values", + "PyMarshal_ReadObjectFromString", + "PyMarshal_WriteObjectToString", + "PyMem_Calloc", + "PyMem_Free", + "PyMem_Malloc", + "PyMem_RawCalloc", + "PyMem_RawFree", + "PyMem_RawMalloc", + "PyMem_RawRealloc", + "PyMem_Realloc", + "PyMemberDescr_Type", + "PyMember_GetOne", + "PyMember_SetOne", + "PyMemoryView_FromBuffer", + "PyMemoryView_FromMemory", + "PyMemoryView_FromObject", + "PyMemoryView_GetContiguous", + "PyMemoryView_Type", + "PyMethodDescr_Type", + "PyModuleDef_Init", + "PyModuleDef_Type", + "PyModule_Add", + "PyModule_AddFunctions", + "PyModule_AddIntConstant", + "PyModule_AddObject", + "PyModule_AddObjectRef", + "PyModule_AddStringConstant", + "PyModule_AddType", + "PyModule_Create2", + "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", + "PyModule_GetDef", + "PyModule_GetDict", + "PyModule_GetFilename", + "PyModule_GetFilenameObject", + "PyModule_GetName", + "PyModule_GetNameObject", + "PyModule_GetState", + "PyModule_New", + "PyModule_NewObject", + "PyModule_SetDocString", + "PyModule_Type", + "PyNumber_Absolute", + "PyNumber_Add", + "PyNumber_And", + "PyNumber_AsSsize_t", + "PyNumber_Check", + "PyNumber_Divmod", + "PyNumber_Float", + "PyNumber_FloorDivide", + "PyNumber_InPlaceAdd", + "PyNumber_InPlaceAnd", + "PyNumber_InPlaceFloorDivide", + "PyNumber_InPlaceLshift", + "PyNumber_InPlaceMatrixMultiply", + "PyNumber_InPlaceMultiply", + "PyNumber_InPlaceOr", + "PyNumber_InPlacePower", + "PyNumber_InPlaceRemainder", + "PyNumber_InPlaceRshift", + "PyNumber_InPlaceSubtract", + "PyNumber_InPlaceTrueDivide", + "PyNumber_InPlaceXor", + "PyNumber_Index", + "PyNumber_Invert", + "PyNumber_Long", + "PyNumber_Lshift", + "PyNumber_MatrixMultiply", + "PyNumber_Multiply", + "PyNumber_Negative", + "PyNumber_Or", + "PyNumber_Positive", + "PyNumber_Power", + "PyNumber_Remainder", + "PyNumber_Rshift", + "PyNumber_Subtract", + "PyNumber_ToBase", + "PyNumber_TrueDivide", + "PyNumber_Xor", + "PyOS_FSPath", + "PyOS_InputHook", + "PyOS_InterruptOccurred", + "PyOS_double_to_string", + "PyOS_getsig", + "PyOS_mystricmp", + "PyOS_mystrnicmp", + "PyOS_setsig", + "PyOS_snprintf", + "PyOS_string_to_double", + "PyOS_strtol", + "PyOS_strtoul", + "PyOS_vsnprintf", + "PyObject_ASCII", + "PyObject_AsCharBuffer", + "PyObject_AsFileDescriptor", + "PyObject_AsReadBuffer", + "PyObject_AsWriteBuffer", + "PyObject_Bytes", + "PyObject_Call", + "PyObject_CallFunction", + "PyObject_CallFunctionObjArgs", + "PyObject_CallMethod", + "PyObject_CallMethodObjArgs", + "PyObject_CallNoArgs", + "PyObject_CallObject", + "PyObject_Calloc", + "PyObject_CheckBuffer", + "PyObject_CheckReadBuffer", + "PyObject_ClearWeakRefs", + "PyObject_CopyData", + "PyObject_DelAttr", + "PyObject_DelAttrString", + "PyObject_DelItem", + "PyObject_DelItemString", + "PyObject_Dir", + "PyObject_Format", + "PyObject_Free", + "PyObject_GC_Del", + "PyObject_GC_IsFinalized", + "PyObject_GC_IsTracked", + "PyObject_GC_Track", + "PyObject_GC_UnTrack", + "PyObject_GenericGetAttr", + "PyObject_GenericGetDict", + "PyObject_GenericSetAttr", + "PyObject_GenericSetDict", + "PyObject_GetAIter", + "PyObject_GetAttr", + "PyObject_GetAttrString", + "PyObject_GetBuffer", + "PyObject_GetItem", + "PyObject_GetIter", + "PyObject_GetOptionalAttr", + "PyObject_GetOptionalAttrString", + "PyObject_GetTypeData", + "PyObject_HasAttr", + "PyObject_HasAttrString", + "PyObject_HasAttrStringWithError", + "PyObject_HasAttrWithError", + "PyObject_Hash", + "PyObject_HashNotImplemented", + "PyObject_Init", + "PyObject_InitVar", + "PyObject_IsInstance", + "PyObject_IsSubclass", + "PyObject_IsTrue", + "PyObject_Length", + "PyObject_Malloc", + "PyObject_Not", + "PyObject_Realloc", + "PyObject_Repr", + "PyObject_RichCompare", + "PyObject_RichCompareBool", + "PyObject_SelfIter", + "PyObject_SetAttr", + "PyObject_SetAttrString", + "PyObject_SetItem", + "PyObject_Size", + "PyObject_Str", + "PyObject_Type", + "PyObject_Vectorcall", + "PyObject_VectorcallMethod", + "PyProperty_Type", + "PyRangeIter_Type", + "PyRange_Type", + "PyReversed_Type", + "PySeqIter_New", + "PySeqIter_Type", + "PySequence_Check", + "PySequence_Concat", + "PySequence_Contains", + "PySequence_Count", + "PySequence_DelItem", + "PySequence_DelSlice", + "PySequence_Fast", + "PySequence_GetItem", + "PySequence_GetSlice", + "PySequence_In", + "PySequence_InPlaceConcat", + "PySequence_InPlaceRepeat", + "PySequence_Index", + "PySequence_Length", + "PySequence_List", + "PySequence_Repeat", + "PySequence_SetItem", + "PySequence_SetSlice", + "PySequence_Size", + "PySequence_Tuple", + "PySetIter_Type", + "PySet_Add", + "PySet_Clear", + "PySet_Contains", + "PySet_Discard", + "PySet_New", + "PySet_Pop", + "PySet_Size", + "PySet_Type", + "PySlice_AdjustIndices", + "PySlice_GetIndices", + "PySlice_GetIndicesEx", + "PySlice_New", + "PySlice_Type", + "PySlice_Unpack", + "PyState_AddModule", + "PyState_FindModule", + "PyState_RemoveModule", + "PyStructSequence_GetItem", + "PyStructSequence_New", + "PyStructSequence_NewType", + "PyStructSequence_SetItem", + "PyStructSequence_UnnamedField", + "PySuper_Type", + "PySys_AddWarnOption", + "PySys_AddWarnOptionUnicode", + "PySys_AddXOption", + "PySys_Audit", + "PySys_AuditTuple", + "PySys_FormatStderr", + "PySys_FormatStdout", + "PySys_GetObject", + "PySys_GetXOptions", + "PySys_HasWarnOptions", + "PySys_ResetWarnOptions", + "PySys_SetArgv", + "PySys_SetArgvEx", + "PySys_SetObject", + "PySys_SetPath", + "PySys_WriteStderr", + "PySys_WriteStdout", + "PyThreadState_Clear", + "PyThreadState_Delete", + "PyThreadState_DeleteCurrent", + "PyThreadState_Get", + "PyThreadState_GetDict", + "PyThreadState_GetFrame", + "PyThreadState_GetID", + "PyThreadState_GetInterpreter", + "PyThreadState_New", + "PyThreadState_SetAsyncExc", + "PyThreadState_Swap", + "PyThread_GetInfo", + "PyThread_ReInitTLS", + "PyThread_acquire_lock", + "PyThread_acquire_lock_timed", + "PyThread_allocate_lock", + "PyThread_create_key", + "PyThread_delete_key", + "PyThread_delete_key_value", + "PyThread_exit_thread", + "PyThread_free_lock", + "PyThread_get_key_value", + "PyThread_get_stacksize", + "PyThread_get_thread_ident", + "PyThread_init_thread", + "PyThread_release_lock", + "PyThread_set_key_value", + "PyThread_set_stacksize", + "PyThread_start_new_thread", + "PyThread_tss_alloc", + "PyThread_tss_create", + "PyThread_tss_delete", + "PyThread_tss_free", + "PyThread_tss_get", + "PyThread_tss_is_created", + "PyThread_tss_set", + "PyTraceBack_Here", + "PyTraceBack_Print", + "PyTraceBack_Type", + "PyTupleIter_Type", + "PyTuple_GetItem", + "PyTuple_GetSlice", + "PyTuple_New", + "PyTuple_Pack", + "PyTuple_SetItem", + "PyTuple_Size", + "PyTuple_Type", + "PyType_ClearCache", + "PyType_Freeze", + "PyType_FromMetaclass", + "PyType_FromModuleAndSpec", + "PyType_FromSpec", + "PyType_FromSpecWithBases", + "PyType_GenericAlloc", + "PyType_GenericNew", + "PyType_GetBaseByToken", + "PyType_GetFlags", + "PyType_GetFullyQualifiedName", + "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", + "PyType_GetModuleState", + "PyType_GetName", + "PyType_GetQualName", + "PyType_GetSlot", + "PyType_GetTypeDataSize", + "PyType_IsSubtype", + "PyType_Modified", + "PyType_Ready", + "PyType_Type", + "PyUnicodeDecodeError_Create", + "PyUnicodeDecodeError_GetEncoding", + "PyUnicodeDecodeError_GetEnd", + "PyUnicodeDecodeError_GetObject", + "PyUnicodeDecodeError_GetReason", + "PyUnicodeDecodeError_GetStart", + "PyUnicodeDecodeError_SetEnd", + "PyUnicodeDecodeError_SetReason", + "PyUnicodeDecodeError_SetStart", + "PyUnicodeEncodeError_GetEncoding", + "PyUnicodeEncodeError_GetEnd", + "PyUnicodeEncodeError_GetObject", + "PyUnicodeEncodeError_GetReason", + "PyUnicodeEncodeError_GetStart", + "PyUnicodeEncodeError_SetEnd", + "PyUnicodeEncodeError_SetReason", + "PyUnicodeEncodeError_SetStart", + "PyUnicodeIter_Type", + "PyUnicodeTranslateError_GetEnd", + "PyUnicodeTranslateError_GetObject", + "PyUnicodeTranslateError_GetReason", + "PyUnicodeTranslateError_GetStart", + "PyUnicodeTranslateError_SetEnd", + "PyUnicodeTranslateError_SetReason", + "PyUnicodeTranslateError_SetStart", + "PyUnicode_Append", + "PyUnicode_AppendAndDel", + "PyUnicode_AsASCIIString", + "PyUnicode_AsCharmapString", + "PyUnicode_AsDecodedObject", + "PyUnicode_AsDecodedUnicode", + "PyUnicode_AsEncodedObject", + "PyUnicode_AsEncodedString", + "PyUnicode_AsEncodedUnicode", + "PyUnicode_AsLatin1String", + "PyUnicode_AsRawUnicodeEscapeString", + "PyUnicode_AsUCS4", + "PyUnicode_AsUCS4Copy", + "PyUnicode_AsUTF16String", + "PyUnicode_AsUTF32String", + "PyUnicode_AsUTF8AndSize", + "PyUnicode_AsUTF8String", + "PyUnicode_AsUnicodeEscapeString", + "PyUnicode_AsWideChar", + "PyUnicode_AsWideCharString", + "PyUnicode_BuildEncodingMap", + "PyUnicode_Compare", + "PyUnicode_CompareWithASCIIString", + "PyUnicode_Concat", + "PyUnicode_Contains", + "PyUnicode_Count", + "PyUnicode_Decode", + "PyUnicode_DecodeASCII", + "PyUnicode_DecodeCharmap", + "PyUnicode_DecodeFSDefault", + "PyUnicode_DecodeFSDefaultAndSize", + "PyUnicode_DecodeLatin1", + "PyUnicode_DecodeLocale", + "PyUnicode_DecodeLocaleAndSize", + "PyUnicode_DecodeRawUnicodeEscape", + "PyUnicode_DecodeUTF16", + "PyUnicode_DecodeUTF16Stateful", + "PyUnicode_DecodeUTF32", + "PyUnicode_DecodeUTF32Stateful", + "PyUnicode_DecodeUTF7", + "PyUnicode_DecodeUTF7Stateful", + "PyUnicode_DecodeUTF8", + "PyUnicode_DecodeUTF8Stateful", + "PyUnicode_DecodeUnicodeEscape", + "PyUnicode_EncodeFSDefault", + "PyUnicode_EncodeLocale", + "PyUnicode_Equal", + "PyUnicode_EqualToUTF8", + "PyUnicode_EqualToUTF8AndSize", + "PyUnicode_FSConverter", + "PyUnicode_FSDecoder", + "PyUnicode_Find", + "PyUnicode_FindChar", + "PyUnicode_Format", + "PyUnicode_FromEncodedObject", + "PyUnicode_FromFormat", + "PyUnicode_FromFormatV", + "PyUnicode_FromObject", + "PyUnicode_FromOrdinal", + "PyUnicode_FromString", + "PyUnicode_FromStringAndSize", + "PyUnicode_FromWideChar", + "PyUnicode_GetDefaultEncoding", + "PyUnicode_GetLength", + "PyUnicode_GetSize", + "PyUnicode_InternFromString", + "PyUnicode_InternImmortal", + "PyUnicode_InternInPlace", + "PyUnicode_IsIdentifier", + "PyUnicode_Join", + "PyUnicode_Partition", + "PyUnicode_RPartition", + "PyUnicode_RSplit", + "PyUnicode_ReadChar", + "PyUnicode_Replace", + "PyUnicode_Resize", + "PyUnicode_RichCompare", + "PyUnicode_Split", + "PyUnicode_Splitlines", + "PyUnicode_Substring", + "PyUnicode_Tailmatch", + "PyUnicode_Translate", + "PyUnicode_Type", + "PyUnicode_WriteChar", + "PyVectorcall_Call", + "PyVectorcall_NARGS", + "PyWeakref_GetObject", + "PyWeakref_GetRef", + "PyWeakref_NewProxy", + "PyWeakref_NewRef", + "PyWrapperDescr_Type", + "PyWrapper_New", + "PyZip_Type", + "Py_AddPendingCall", + "Py_AtExit", + "Py_BuildValue", + "Py_BytesMain", + "Py_CompileString", + "Py_DecRef", + "Py_DecodeLocale", + "Py_EncodeLocale", + "Py_EndInterpreter", + "Py_EnterRecursiveCall", + "Py_Exit", + "Py_FatalError", + "Py_FileSystemDefaultEncodeErrors", + "Py_FileSystemDefaultEncoding", + "Py_Finalize", + "Py_FinalizeEx", + "Py_GenericAlias", + "Py_GenericAliasType", + "Py_GetArgcArgv", + "Py_GetBuildInfo", + "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", + "Py_GetCopyright", + "Py_GetExecPrefix", + "Py_GetPath", + "Py_GetPlatform", + "Py_GetPrefix", + "Py_GetProgramFullPath", + "Py_GetProgramName", + "Py_GetPythonHome", + "Py_GetRecursionLimit", + "Py_GetVersion", + "Py_HasFileSystemDefaultEncoding", + "Py_IncRef", + "Py_Initialize", + "Py_InitializeEx", + "Py_Is", + "Py_IsFalse", + "Py_IsFinalizing", + "Py_IsInitialized", + "Py_IsNone", + "Py_IsTrue", + "Py_LeaveRecursiveCall", + "Py_Main", + "Py_MakePendingCalls", + "Py_NewInterpreter", + "Py_NewRef", + "Py_PACK_FULL_VERSION", + "Py_PACK_VERSION", + "Py_REFCNT", + "Py_ReprEnter", + "Py_ReprLeave", + "Py_SetPath", + "Py_SetProgramName", + "Py_SetPythonHome", + "Py_SetRecursionLimit", + "Py_TYPE", + "Py_UTF8Mode", + "Py_VaBuildValue", + "Py_Version", + "Py_XNewRef", + "_PyArg_ParseTupleAndKeywords_SizeT", + "_PyArg_ParseTuple_SizeT", + "_PyArg_Parse_SizeT", + "_PyArg_VaParseTupleAndKeywords_SizeT", + "_PyArg_VaParse_SizeT", + "_PyErr_BadInternalCall", + "_PyObject_CallFunction_SizeT", + "_PyObject_CallMethod_SizeT", + "_PyObject_GC_New", + "_PyObject_GC_NewVar", + "_PyObject_GC_Resize", + "_PyObject_New", + "_PyObject_NewVar", + "_PyState_AddModule", + "_PyThreadState_Init", + "_PyThreadState_Prealloc", + "_PyWeakref_CallableProxyType", + "_PyWeakref_ProxyType", + "_PyWeakref_RefType", + "_Py_BuildValue_SizeT", + "_Py_CheckRecursiveCall", + "_Py_Dealloc", + "_Py_DecRef", + "_Py_EllipsisObject", + "_Py_FalseStruct", + "_Py_IncRef", + "_Py_NoneStruct", + "_Py_NotImplementedStruct", + "_Py_SetRefcnt", + "_Py_SwappedOp", + "_Py_TrueStruct", + "_Py_VaBuildValue_SizeT", +) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) +if feature_macros['MS_WINDOWS']: + SYMBOL_NAMES += ( + 'PyErr_SetExcFromWindowsErr', + 'PyErr_SetExcFromWindowsErrWithFilename', + 'PyErr_SetExcFromWindowsErrWithFilenameObject', + 'PyErr_SetExcFromWindowsErrWithFilenameObjects', + 'PyErr_SetFromWindowsErr', + 'PyErr_SetFromWindowsErrWithFilename', + 'PyExc_WindowsError', + 'PyUnicode_AsMBCSString', + 'PyUnicode_DecodeCodePageStateful', + 'PyUnicode_DecodeMBCS', + 'PyUnicode_DecodeMBCSStateful', + 'PyUnicode_EncodeCodePage', + ) +if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: + SYMBOL_NAMES += ( + 'PyThread_get_thread_native_id', + ) +if feature_macros['Py_REF_DEBUG']: + SYMBOL_NAMES += ( + '_Py_NegativeRefcount', + '_Py_RefTotal', + ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) + +EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', + 'MS_WINDOWS', + 'PY_HAVE_THREAD_NATIVE_ID', + 'Py_REF_DEBUG', + 'Py_TRACE_REFS', + 'USE_STACKCHECK']) +WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, + 'MS_WINDOWS': True, + 'PY_HAVE_THREAD_NATIVE_ID': True, + 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', + 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 49013a4bcd8..a83f7d076f0 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -157,12 +157,17 @@ def test_mode(self): os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() - self.assertEqual(modestr[:3], '-rw') + self.assertStartsWith(modestr, '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) + def test_filemode_does_not_misclassify_random_bits(self): + # gh-144050 regression test + self.assertEqual(self.statmod.filemode(0o77777)[0], "?") + self.assertEqual(self.statmod.filemode(0o177777)[0], "?") + @os_helper.skip_unless_working_chmod def test_directory(self): os.mkdir(TESTFN) @@ -256,7 +261,7 @@ def test_flags_consistent(self): "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): for key, value in sorted(self.file_attributes.items()): - self.assertTrue(hasattr(self.statmod, key), key) + self.assertHasAttr(self.statmod, key) modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) @@ -314,7 +319,7 @@ def test_macosx_attribute_values(self): self.assertEqual(self.statmod.S_ISGID, 0o002000) self.assertEqual(self.statmod.S_ISVTX, 0o001000) - self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertNotHasAttr(self.statmod, "S_ISTXT") self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 9c2714e99d4..e0b74432c94 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -645,7 +645,7 @@ def do_test(self, args): def test_numerictestcase_is_testcase(self): # Ensure that NumericTestCase actually is a TestCase. - self.assertTrue(issubclass(NumericTestCase, unittest.TestCase)) + self.assertIsSubclass(NumericTestCase, unittest.TestCase) def test_error_msg_numeric(self): # Test the error message generated for numeric comparisons. @@ -683,32 +683,23 @@ class GlobalsTest(unittest.TestCase): def test_meta(self): # Test for the existence of metadata. for meta in self.expected_metadata: - self.assertTrue(hasattr(self.module, meta), - "%s not present" % meta) + self.assertHasAttr(self.module, meta) def test_check_all(self): # Check everything in __all__ exists and is public. module = self.module for name in module.__all__: # No private names in __all__: - self.assertFalse(name.startswith("_"), + self.assertNotStartsWith(name, "_", 'private name "%s" in __all__' % name) # And anything in __all__ must exist: - self.assertTrue(hasattr(module, name), - 'missing name "%s" in __all__' % name) + self.assertHasAttr(module, name) class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): - errmsg = ( - "Expected StatisticsError to be a ValueError, but got a" - " subclass of %r instead." - ) - self.assertTrue(hasattr(statistics, 'StatisticsError')) - self.assertTrue( - issubclass(statistics.StatisticsError, ValueError), - errmsg % statistics.StatisticsError.__base__ - ) + self.assertHasAttr(statistics, 'StatisticsError') + self.assertIsSubclass(statistics.StatisticsError, ValueError) # === Tests for private utility functions === @@ -2014,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2122,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): @@ -2435,17 +2433,22 @@ def integrate(func, low, high, steps=10_000): data.append(100) self.assertGreater(f_hat(100), 0.0) - def test_kde_kernel_invcdfs(self): - kernel_invcdfs = statistics._kernel_invcdfs - kde = statistics.kde + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() + kernel_specs = statistics._kernel_specs # Verify that cdf / invcdf will round trip xarr = [i/100 for i in range(-100, 101)] - for kernel, invcdf in kernel_invcdfs.items(): + parr = [i/1000 + 5/10000 for i in range(1000)] + for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] + invcdf = spec['invcdf'] with self.subTest(kernel=kernel): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) for x in xarr: - self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) @support.requires_resource('cpu') def test_kde_random(self): @@ -2797,7 +2800,7 @@ def test_sqrtprod_helper_function_fundamentals(self): @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "accuracy not guaranteed on machines with double rounding") - @support.cpython_only # Allow for a weaker sumprod() implmentation + @support.cpython_only # Allow for a weaker sumprod() implementation def test_sqrtprod_helper_function_improved_accuracy(self): # Test a known example where accuracy is improved x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 9d43a33cd9e..78a8dc24cce 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -112,7 +112,7 @@ def test_literals(self): # raw strings should not have unicode escapes self.assertNotEqual(r"\u0020", " ") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'str'> is not <class 'test.test_str.StrSubclass'> def test_ascii(self): self.assertEqual(ascii('abc'), "'abc'") self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") @@ -565,7 +565,6 @@ def __str__(self): return self.sval self.checkraises(TypeError, ' ', 'join', [1, 2, 3]) self.checkraises(TypeError, ' ', 'join', ['1', '2', 3]) - @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > 2**32, 'needs too much memory on a 64-bit platform') def test_join_overflow(self): @@ -794,7 +793,7 @@ def test_isdecimal(self): for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True def test_isdigit(self): super().test_isdigit() self.checkequalnofix(True, '\u2460', 'isdigit') @@ -940,7 +939,7 @@ def test_upper(self): self.assertEqual('\U0008fffe'.upper(), '\U0008fffe') self.assertEqual('\u2177'.upper(), '\u2167') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ def test_capitalize(self): string_tests.StringLikeTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') @@ -958,7 +957,7 @@ def test_capitalize(self): self.assertEqual('finnish'.capitalize(), 'Finnish') self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ def test_title(self): super().test_title() self.assertEqual('\U0001044F'.title(), '\U00010427') @@ -976,7 +975,7 @@ def test_title(self): self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy') self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + 𐐧 def test_swapcase(self): string_tests.StringLikeTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') @@ -1076,7 +1075,7 @@ def test_issue18183(self): '\U00100000'.ljust(3, '\U00010000') '\U00100000'.rjust(3, '\U00010000') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? + def test_format(self): self.assertEqual(''.format(), '') self.assertEqual('a'.format(), 'a') @@ -1460,19 +1459,18 @@ def __getitem__(self, key): self.assertRaises(TypeError, '{a}'.format_map, []) self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) - @unittest.skip('TODO: RUSTPYTHON; killed for chewing up RAM') def test_format_huge_precision(self): format_string = ".{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_format_huge_width(self): format_string = "{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: tuple index out of range def test_format_huge_item_number(self): format_string = "{{{}:.6f}}".format(sys.maxsize + 1) with self.assertRaises(ValueError): @@ -1508,7 +1506,7 @@ def __format__(self, spec): self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3') self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: %x format: an integer is required, not PseudoInt def test_formatting(self): string_tests.StringLikeTest.test_formatting(self) # Testing Unicode formatting strings... @@ -1757,7 +1755,7 @@ def __str__(self): 'character buffers are decoded to unicode' ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; Pass various keyword argument combinations to the constructor. def test_constructor_keyword_args(self): """Pass various keyword argument combinations to the constructor.""" # The object argument can be passed as a keyword. @@ -1767,7 +1765,7 @@ def test_constructor_keyword_args(self): self.assertEqual(str(b'foo', errors='strict'), 'foo') # not "b'foo'" self.assertEqual(str(object=b'foo', errors='strict'), 'foo') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; Check the constructor argument defaults. def test_constructor_defaults(self): """Check the constructor argument defaults.""" # The object argument defaults to '' or b''. @@ -1779,7 +1777,6 @@ def test_constructor_defaults(self): # The errors argument defaults to strict. self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs_utf7(self): utfTests = [ ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example @@ -2289,7 +2286,6 @@ def test_codecs_errors(self): self.assertRaises(ValueError, complex, "\ud800") self.assertRaises(ValueError, complex, "\udf00") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs(self): # Encoding self.assertEqual('hello'.encode('ascii'), b'hello') @@ -2419,7 +2415,7 @@ def test_ucs4(self): else: self.fail("Should have raised UnicodeDecodeError") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'str'> is not <class 'test.test_str.StrSubclass'> def test_conversion(self): # Make sure __str__() works properly class StrWithStr(str): @@ -2468,7 +2464,6 @@ def test_printable_repr(self): # This test only affects 32-bit platforms because expandtabs can only take # an int as the max value, not a 64-bit C long. If expandtabs is changed # to take a 64-bit long, this test should apply to all platforms. - @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') def test_expandtabs_overflows_gracefully(self): @@ -2479,7 +2474,7 @@ def test_expandtabs_optimization(self): s = 'abc' self.assertIs(s.expandtabs(), s) - @unittest.skip('TODO: RUSTPYTHON; aborted: memory allocation of 9223372036854775759 bytes failed') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raiseMemError(self): asciifields = "nnb" compactfields = asciifields + "nP" @@ -2619,12 +2614,12 @@ def test_compare(self): self.assertTrue(astral >= bmp2) self.assertFalse(astral >= astral2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, str) support.check_free_after_iterating(self, reversed, str) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') def test_check_encoding_errors(self): # bpo-37388: str(bytes) and str.decode() must check encoding and errors # arguments in dev mode @@ -2685,7 +2680,7 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "str expected at most 3 arguments, got 4" does not match "expected at most 3 arguments, got 4" def test_str_invalid_call(self): # too many args with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index be43c49e40a..375f6aaedd8 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -39,7 +39,21 @@ def _update_variables(self, now): if now[3] < 12: self.ampm='(AM|am)' else: self.ampm='(PM|pm)' - self.jan1 = time.localtime(time.mktime((now[0], 1, 1, 0, 0, 0, 0, 1, 0))) + jan1 = time.struct_time( + ( + now.tm_year, # Year + 1, # Month (January) + 1, # Day (1st) + 0, # Hour (0) + 0, # Minute (0) + 0, # Second (0) + -1, # tm_wday (will be determined) + 1, # tm_yday (day 1 of the year) + -1, # tm_isdst (let the system determine) + ) + ) + # use mktime to get the correct tm_wday and tm_isdst values + self.jan1 = time.localtime(time.mktime(jan1)) try: if now[8]: self.tz = time.tzname[1] @@ -54,14 +68,10 @@ def _update_variables(self, now): self.now = now def setUp(self): - try: - import java - java.util.Locale.setDefault(java.util.Locale.US) - except ImportError: - from locale import setlocale, LC_TIME - saved_locale = setlocale(LC_TIME) - setlocale(LC_TIME, 'C') - self.addCleanup(setlocale, LC_TIME, saved_locale) + from locale import setlocale, LC_TIME + saved_locale = setlocale(LC_TIME) + setlocale(LC_TIME, 'C') + self.addCleanup(setlocale, LC_TIME, saved_locale) def test_strftime(self): now = time.time() @@ -187,8 +197,7 @@ class Y1900Tests(unittest.TestCase): def test_y_before_1900(self): # Issue #13674, #19634 t = (1899, 1, 1, 0, 0, 0, 0, 0, 0) - if (sys.platform == "win32" - or sys.platform.startswith(("aix", "sunos", "solaris"))): + if sys.platform.startswith(("aix", "sunos", "solaris")): with self.assertRaises(ValueError): time.strftime("%y", t) else: diff --git a/Lib/test/test_string/__init__.py b/Lib/test/test_string/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_string/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py new file mode 100644 index 00000000000..cfead782b7d --- /dev/null +++ b/Lib/test/test_string/_support.py @@ -0,0 +1,67 @@ +import unittest +from string.templatelib import Interpolation + + +class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + + def assertTStringEqual(self, t, strings, interpolations): + """Test template string literal equality. + + The *strings* argument must be a tuple of strings equal to *t.strings*. + + The *interpolations* argument must be a sequence of tuples which are + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. + """ + self.assertEqual(t.strings, strings) + self.assertEqual(len(t.interpolations), len(interpolations)) + + for i, exp in zip(t.interpolations, interpolations, strict=True): + self.assertInterpolationEqual(i, exp) + + +def convert(value, conversion): + if conversion == "a": + return ascii(value) + elif conversion == "r": + return repr(value) + elif conversion == "s": + return str(value) + return value + + +def fstring(template): + parts = [] + for item in template: + match item: + case str() as s: + parts.append(s) + case Interpolation(value, _, conversion, format_spec): + value = convert(value, conversion) + value = format(value, format_spec) + parts.append(value) + return "".join(parts) diff --git a/Lib/test/test_string.py b/Lib/test/test_string/test_string.py similarity index 95% rename from Lib/test/test_string.py rename to Lib/test/test_string/test_string.py index 824b89ad517..5394fe4e12c 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -1,6 +1,15 @@ import unittest import string from string import Template +import types +from test.support import cpython_only +from test.support.import_helper import ensure_lazy_imports + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("base64", {"re", "collections"}) class ModuleTest(unittest.TestCase): @@ -101,6 +110,24 @@ def test_index_lookup(self): with self.assertRaises(KeyError): fmt.format("{0[2]}{0[0]}", {}) + def test_auto_numbering_lookup(self): + fmt = string.Formatter() + namespace = types.SimpleNamespace(foo=types.SimpleNamespace(bar='baz')) + widths = [None, types.SimpleNamespace(qux=4)] + self.assertEqual( + fmt.format("{.foo.bar:{[1].qux}}", namespace, widths), 'baz ') + + def test_auto_numbering_reenterability(self): + class ReenteringFormatter(string.Formatter): + def format_field(self, value, format_spec): + if format_spec.isdigit() and int(format_spec) > 0: + return self.format('{:{}}!', value, int(format_spec) - 1) + else: + return super().format_field(value, format_spec) + fmt = ReenteringFormatter() + x = types.SimpleNamespace(a='X') + self.assertEqual(fmt.format('{.a:{}}', x, 3), 'X!!!') + def test_override_get_value(self): class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py new file mode 100644 index 00000000000..1c86717155f --- /dev/null +++ b/Lib/test/test_string/test_templatelib.py @@ -0,0 +1,193 @@ +import pickle +import unittest +from collections.abc import Iterator, Iterable +from string.templatelib import Template, Interpolation, convert + +from test.test_string._support import TStringBaseCase, fstring + + +class TestTemplate(unittest.TestCase, TStringBaseCase): + + def test_common(self): + self.assertEqual(type(t'').__name__, 'Template') + self.assertEqual(type(t'').__qualname__, 'Template') + self.assertEqual(type(t'').__module__, 'string.templatelib') + + a = 'a' + i = t'{a}'.interpolations[0] + self.assertEqual(type(i).__name__, 'Interpolation') + self.assertEqual(type(i).__qualname__, 'Interpolation') + self.assertEqual(type(i).__module__, 'string.templatelib') + + def test_final_types(self): + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(Template): ... + + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(Interpolation): ... + + def test_basic_creation(self): + # Simple t-string creation + t = t'Hello, world' + self.assertIsInstance(t, Template) + self.assertTStringEqual(t, ('Hello, world',), ()) + self.assertEqual(fstring(t), 'Hello, world') + + # Empty t-string + t = t'' + self.assertTStringEqual(t, ('',), ()) + self.assertEqual(fstring(t), '') + + # Multi-line t-string + t = t"""Hello, +world""" + self.assertEqual(t.strings, ('Hello,\nworld',)) + self.assertEqual(len(t.interpolations), 0) + self.assertEqual(fstring(t), 'Hello,\nworld') + + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + + def test_creation_interleaving(self): + # Should add strings on either side + t = Template(Interpolation('Maria', 'name', None, '')) + self.assertTStringEqual(t, ('', ''), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Maria') + + # Should prepend empty string + t = Template(Interpolation('Maria', 'name', None, ''), ' is my name') + self.assertTStringEqual(t, ('', ' is my name'), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Maria is my name') + + # Should append empty string + t = Template('Hello, ', Interpolation('Maria', 'name', None, '')) + self.assertTStringEqual(t, ('Hello, ', ''), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Hello, Maria') + + # Should concatenate strings + t = Template('Hello', ', ', Interpolation('Maria', 'name', None, ''), + '!') + self.assertTStringEqual(t, ('Hello, ', '!'), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Hello, Maria!') + + # Should add strings on either side and in between + t = Template(Interpolation('Maria', 'name', None, ''), + Interpolation('Python', 'language', None, '')) + self.assertTStringEqual( + t, ('', '', ''), [('Maria', 'name'), ('Python', 'language')] + ) + self.assertEqual(fstring(t), 'MariaPython') + + def test_template_values(self): + t = t'Hello, world' + self.assertEqual(t.values, ()) + + name = "Lys" + t = t'Hello, {name}' + self.assertEqual(t.values, ("Lys",)) + + country = "GR" + age = 0 + t = t'Hello, {name}, {age} from {country}' + self.assertEqual(t.values, ("Lys", 0, "GR")) + + def test_pickle_template(self): + user = 'test' + for template in ( + t'', + t"No values", + t'With inter {user}', + t'With ! {user!r}', + t'With format {1 / 0.3:.2f}', + Template(), + Template('a'), + Template(Interpolation('Nikita', 'name', None, '')), + Template('a', Interpolation('Nikita', 'name', 'r', '')), + ): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto, template=template): + pickled = pickle.dumps(template, protocol=proto) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.values, template.values) + self.assertEqual(fstring(unpickled), fstring(template)) + + def test_pickle_interpolation(self): + for interpolation in ( + Interpolation('Nikita', 'name', None, ''), + Interpolation('Nikita', 'name', 'r', ''), + Interpolation(1/3, 'x', None, '.2f'), + ): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto, interpolation=interpolation): + pickled = pickle.dumps(interpolation, protocol=proto) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.value, interpolation.value) + self.assertEqual(unpickled.expression, interpolation.expression) + self.assertEqual(unpickled.conversion, interpolation.conversion) + self.assertEqual(unpickled.format_spec, interpolation.format_spec) + + +class TemplateIterTests(unittest.TestCase): + def test_abc(self): + self.assertIsInstance(iter(t''), Iterable) + self.assertIsInstance(iter(t''), Iterator) + + def test_final(self): + TemplateIter = type(iter(t'')) + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(TemplateIter): ... + + def test_iter(self): + x = 1 + res = list(iter(t'abc {x} yz')) + + self.assertEqual(res[0], 'abc ') + self.assertIsInstance(res[1], Interpolation) + self.assertEqual(res[1].value, 1) + self.assertEqual(res[1].expression, 'x') + self.assertEqual(res[1].conversion, None) + self.assertEqual(res[1].format_spec, '') + self.assertEqual(res[2], ' yz') + + def test_exhausted(self): + # See https://github.com/python/cpython/issues/134119. + template_iter = iter(t"{1}") + self.assertIsInstance(next(template_iter), Interpolation) + self.assertRaises(StopIteration, next, template_iter) + self.assertRaises(StopIteration, next, template_iter) + + +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Café', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index ef5602d083b..733006bb509 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,5 @@ from collections import abc +from itertools import combinations import array import gc import math @@ -9,14 +10,18 @@ import weakref from test import support -from test.support import import_helper, suppress_immortalization +from test.support import import_helper from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin ISBIGENDIAN = sys.byteorder == "big" integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +INF = float('inf') +NAN = float('nan') + def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: @@ -33,7 +38,7 @@ def bigendian_to_native(value): else: return string_reverse(value) -class StructTest(unittest.TestCase): +class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_isbigendian(self): self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN) @@ -360,8 +365,7 @@ def test_p_code(self): (got,) = struct.unpack(code, got) self.assertEqual(got, expectedback) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_705836(self): # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry # from the low-order discarded bits could propagate into the exponent @@ -430,56 +434,65 @@ def test_unpack_from(self): self.assertEqual(s.unpack_from(buffer=test_string, offset=2), (b'cd01',)) - def test_pack_into(self): + def _test_pack_into(self, pack_into): test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - s = struct.Struct(fmt) + writable_buf = memoryview(array.array('b', b' '*100)) # Test without offset - s.pack_into(writable_buf, 0, test_string) + pack_into(writable_buf, 0, test_string) from_buf = writable_buf.tobytes()[:len(test_string)] self.assertEqual(from_buf, test_string) # Test with offset. - s.pack_into(writable_buf, 10, test_string) + pack_into(writable_buf, 10, test_string) from_buf = writable_buf.tobytes()[:len(test_string)+10] self.assertEqual(from_buf, test_string[:10] + test_string) + # Test with negative offset. + pack_into(writable_buf, -30, test_string) + from_buf = writable_buf.tobytes()[-30:-30+len(test_string)] + self.assertEqual(from_buf, test_string) + # Go beyond boundaries. small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2, - test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(small_buf, 0, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 90, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -10, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 150, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -150, test_string) + + # Test invalid buffer. + self.assertRaises(TypeError, pack_into, b' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, ' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, [0]*100, 0, test_string) + self.assertRaises(TypeError, pack_into, None, 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::2], 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::-1], 0, test_string) + + # Test bogus offset (issue bpo-3694) + with self.assertRaises(TypeError): + pack_into(writable_buf, None, test_string) + with self.assertRaises(TypeError): + pack_into(writable_buf, 0.0, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, 2**1000, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, -2**1000, test_string) - # Test bogus offset (issue 3694) - sb = small_buf - self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb, - None) + @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object + def test_pack_into(self): + s = struct.Struct('21s') + self._test_pack_into(s.pack_into) + @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object def test_pack_into_fn(self): - test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - pack_into = lambda *args: struct.pack_into(fmt, *args) - - # Test without offset. - pack_into(writable_buf, 0, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)] - self.assertEqual(from_buf, test_string) - - # Test with offset. - pack_into(writable_buf, 10, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)+10] - self.assertEqual(from_buf, test_string[:10] + test_string) - - # Go beyond boundaries. - small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2, - test_string) + pack_into = lambda *args: struct.pack_into('21s', *args) + self._test_pack_into(pack_into) def test_unpack_with_buffer(self): # SF bug 1563759: struct.unpack doesn't support buffer protocol objects @@ -670,8 +683,7 @@ def test_format_attr(self): s2 = struct.Struct(s.format.encode()) self.assertEqual(s2.format, s.format) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_struct_cleans_up_at_runtime_shutdown(self): code = """if 1: import struct @@ -687,10 +699,9 @@ def __del__(self): rc, stdout, stderr = assert_python_ok("-c", code) self.assertEqual(rc, 0) self.assertEqual(stdout.rstrip(), b"") - self.assertIn(b"Exception ignored in:", stderr) + self.assertIn(b"Exception ignored while calling deallocator", stderr) self.assertIn(b"C.__del__", stderr) - @suppress_immortalization() def test__struct_reference_cycle_cleaned_up(self): # Regression test for python/cpython#94207. @@ -777,8 +788,7 @@ def test_error_propagation(fmt_str): test_error_propagation('N') test_error_propagation('n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_struct_subclass_instantiation(self): # Regression test for https://github.com/python/cpython/issues/112358 class MyStruct(struct.Struct): @@ -792,6 +802,47 @@ def test_repr(self): s = struct.Struct('=i2H') self.assertEqual(repr(s), f'Struct({s.format!r})') + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_complex_round_trip(self): + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for f in ['F', 'D', '>F', '>D', '<F', '<D']: + with self.subTest(z=z, format=f): + round_trip = struct.unpack(f, struct.pack(f, z))[0] + self.assertComplexesAreIdentical(z, round_trip) + + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at least 1 arguments, got 0 + def test_operations_on_half_initialized_Struct(self): + S = struct.Struct.__new__(struct.Struct) + + spam = array.array('b', b' ') + self.assertRaises(RuntimeError, S.iter_unpack, spam) + self.assertRaises(RuntimeError, S.pack, 1) + self.assertRaises(RuntimeError, S.pack_into, spam, 1) + self.assertRaises(RuntimeError, S.unpack, spam) + self.assertRaises(RuntimeError, S.unpack_from, spam) + self.assertRaises(RuntimeError, getattr, S, 'format') + self.assertEqual(S.size, -1) + + class UnpackIteratorTest(unittest.TestCase): """ Tests for iterative unpacking (struct.Struct.iter_unpack). @@ -865,6 +916,7 @@ def test_module_func(self): self.assertRaises(StopIteration, next, it) def test_half_float(self): + _testcapi = import_helper.import_module('_testcapi') # Little-endian examples from: # http://en.wikipedia.org/wiki/Half_precision_floating-point_format format_bits_float__cleanRoundtrip_list = [ @@ -909,10 +961,17 @@ def test_half_float(self): # Check that packing produces a bit pattern representing a quiet NaN: # all exponent bits and the msb of the fraction should all be 1. + if _testcapi.nan_msb_is_signaling: + # HP PA RISC and some MIPS CPUs use 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + expected = 0x7c + else: + expected = 0x7e + packed = struct.pack('<e', math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) packed = struct.pack('<e', -math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) # Checks for round-to-even behavior format_bits_float__rounding_list = [ @@ -970,4 +1029,4 @@ def test_half_float(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py index c007476e004..0d32aa509bd 100644 --- a/Lib/test/test_subclassinit.py +++ b/Lib/test/test_subclassinit.py @@ -134,30 +134,28 @@ class Descriptor: def __set_name__(self, owner, name): 1/0 - with self.assertRaises(RuntimeError) as cm: + with self.assertRaises(ZeroDivisionError) as cm: class NotGoingToWork: attr = Descriptor() - exc = cm.exception - self.assertRegex(str(exc), r'\bNotGoingToWork\b') - self.assertRegex(str(exc), r'\battr\b') - self.assertRegex(str(exc), r'\bDescriptor\b') - self.assertIsInstance(exc.__cause__, ZeroDivisionError) + notes = cm.exception.__notes__ + self.assertRegex(str(notes), r'\bNotGoingToWork\b') + self.assertRegex(str(notes), r'\battr\b') + self.assertRegex(str(notes), r'\bDescriptor\b') def test_set_name_wrong(self): class Descriptor: def __set_name__(self): pass - with self.assertRaises(RuntimeError) as cm: + with self.assertRaises(TypeError) as cm: class NotGoingToWork: attr = Descriptor() - exc = cm.exception - self.assertRegex(str(exc), r'\bNotGoingToWork\b') - self.assertRegex(str(exc), r'\battr\b') - self.assertRegex(str(exc), r'\bDescriptor\b') - self.assertIsInstance(exc.__cause__, TypeError) + notes = cm.exception.__notes__ + self.assertRegex(str(notes), r'\bNotGoingToWork\b') + self.assertRegex(str(notes), r'\battr\b') + self.assertRegex(str(notes), r'\bDescriptor\b') def test_set_name_lookup(self): resolved = [] @@ -232,7 +230,7 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) with self.assertRaises(TypeError): - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass class MyMeta(type): @@ -243,10 +241,10 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) self.otherarg = otherarg - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass3(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass3.otherarg, 1) def test_errors_changed_pep487(self): # These tests failed before Python 3.6, PEP 487 @@ -265,10 +263,10 @@ def __new__(cls, name, bases, namespace, otherarg): self.otherarg = otherarg return self - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass2.otherarg, 1) def test_type(self): t = type('NewClass', (object,), {}) @@ -281,4 +279,3 @@ def test_type(self): if __name__ == "__main__": unittest.main() - diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 6b20a5c00a5..aaac2447942 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -4,6 +4,7 @@ from test.support import check_sanitizer from test.support import import_helper from test.support import os_helper +from test.support import strace_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok import subprocess @@ -25,7 +26,6 @@ import gc import textwrap import json -import pathlib from test.support.os_helper import FakePath try: @@ -41,6 +41,10 @@ import grp except ImportError: grp = None +try: + import resource +except ImportError: + resource = None try: import fcntl @@ -158,6 +162,20 @@ def test_call_timeout(self): [sys.executable, "-c", "while True: pass"], timeout=0.1) + def test_timeout_exception(self): + try: + subprocess.run([sys.executable, '-c', 'import time;time.sleep(9)'], timeout = -1) + except subprocess.TimeoutExpired as e: + self.assertIn("-1 seconds", str(e)) + else: + self.fail("Expected TimeoutExpired exception not raised") + try: + subprocess.run([sys.executable, '-c', 'import time;time.sleep(9)'], timeout = 0) + except subprocess.TimeoutExpired as e: + self.assertIn("0 seconds", str(e)) + else: + self.fail("Expected TimeoutExpired exception not raised") + def test_check_call_zero(self): # check_call() function with zero return code rc = subprocess.check_call(ZERO_RETURN_CMD) @@ -269,21 +287,13 @@ def test_check_output_stdin_with_input_arg(self): self.assertIn('stdin', c.exception.args[0]) self.assertIn('input', c.exception.args[0]) - @support.requires_resource('walltime') def test_check_output_timeout(self): # check_output() function with timeout arg with self.assertRaises(subprocess.TimeoutExpired) as c: output = subprocess.check_output( [sys.executable, "-c", - "import sys, time\n" - "sys.stdout.write('BDFL')\n" - "sys.stdout.flush()\n" - "time.sleep(3600)"], - # Some heavily loaded buildbots (sparc Debian 3.x) require - # this much time to start and print. - timeout=3) - self.fail("Expected TimeoutExpired.") - self.assertEqual(c.exception.output, b'BDFL') + "import time; time.sleep(3600)"], + timeout=0.1) def test_call_kwargs(self): # call() function with keyword args @@ -791,8 +801,6 @@ def test_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.platform == "win32", "Windows only issue") def test_win32_duplicate_envs(self): newenv = os.environ.copy() @@ -949,6 +957,48 @@ def test_communicate(self): self.assertEqual(stdout, b"banana") self.assertEqual(stderr, b"pineapple") + def test_communicate_memoryview_input(self): + # Test memoryview input with byte elements + test_data = b"Hello, memoryview!" + mv = memoryview(test_data) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, test_data) + self.assertIsNone(stderr) + + def test_communicate_memoryview_input_nonbyte(self): + # Test memoryview input with non-byte elements (e.g., int32) + # This tests the fix for gh-134453 where non-byte memoryviews + # had incorrect length tracking on POSIX + import array + # Create an array of 32-bit integers that's large enough to trigger + # the chunked writing behavior (> PIPE_BUF) + pipe_buf = getattr(select, 'PIPE_BUF', 512) + # Each 'i' element is 4 bytes, so we need more than pipe_buf/4 elements + # Add some extra to ensure we exceed the buffer size + num_elements = pipe_buf + 1 + test_array = array.array('i', [0x64306f66 for _ in range(num_elements)]) + expected_bytes = test_array.tobytes() + mv = memoryview(test_array) + + p = subprocess.Popen([sys.executable, "-c", + 'import sys; ' + 'data = sys.stdin.buffer.read(); ' + 'sys.stdout.buffer.write(data)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, expected_bytes, + msg=f"{len(stdout)=} =? {len(expected_bytes)=}") + self.assertIsNone(stderr) + def test_communicate_timeout(self): p = subprocess.Popen([sys.executable, "-c", 'import sys,os,time;' @@ -984,6 +1034,62 @@ def test_communicate_timeout_large_output(self): (stdout, _) = p.communicate() self.assertEqual(len(stdout), 4 * 64 * 1024) + def test_communicate_timeout_large_input(self): + # Test that timeout is enforced when writing large input to a + # slow-to-read subprocess, and that partial input is preserved + # for continuation after timeout (gh-141473). + # + # This is a regression test for Windows matching POSIX behavior. + # On POSIX, select() is used to multiplex I/O with timeout checking. + # On Windows, stdin writing must also honor the timeout rather than + # blocking indefinitely when the pipe buffer fills. + + # Input larger than typical pipe buffer (4-64KB on Windows) + input_data = b"x" * (128 * 1024) + + p = subprocess.Popen( + [sys.executable, "-c", + "import sys, time; " + "time.sleep(30); " # Don't read stdin for a long time + "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + try: + timeout = 0.2 + start = time.monotonic() + try: + p.communicate(input_data, timeout=timeout) + # If we get here without TimeoutExpired, the timeout was ignored + elapsed = time.monotonic() - start + self.fail( + f"TimeoutExpired not raised. communicate() completed in " + f"{elapsed:.2f}s, but subprocess sleeps for 30s. " + "Stdin writing blocked without enforcing timeout.") + except subprocess.TimeoutExpired: + elapsed = time.monotonic() - start + + # Timeout should occur close to the specified timeout value, + # not after waiting for the subprocess to finish sleeping. + # Allow generous margin for slow CI, but must be well under + # the subprocess sleep time. + self.assertLess(elapsed, 5.0, + f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " + "Stdin writing blocked without checking timeout.") + + # After timeout, continue communication. The remaining input + # should be sent and we should receive all data back. + stdout, stderr = p.communicate() + + # Verify all input was eventually received by the subprocess + self.assertEqual(len(stdout), len(input_data), + f"Expected {len(input_data)} bytes output but got {len(stdout)}") + self.assertEqual(stdout, input_data) + finally: + p.kill() + p.wait() + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -1054,6 +1160,19 @@ def test_writes_before_communicate(self): self.assertEqual(stdout, b"bananasplit") self.assertEqual(stderr, b"") + def test_communicate_stdin_closed_before_call(self): + # gh-70560, gh-74389: stdin.close() before communicate() + # should not raise ValueError from stdin.flush() + with subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(0)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + p.stdin.close() # Close stdin before communicate + # This should not raise ValueError + (stdout, stderr) = p.communicate() + self.assertEqual(p.returncode, 0) + def test_universal_newlines_and_text(self): args = [ sys.executable, "-c", @@ -1170,10 +1289,8 @@ def test_universal_newlines_communicate_stdin_stdout_stderr(self): self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) # Python debug build push something like "[42442 refs]\n" # to stderr at exit of subprocess. - self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + self.assertStartsWith(stderr, "eline2\neline6\neline7\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines_communicate_encodings(self): # Check that universal newlines mode works for various encodings, # in particular for encodings in the UTF-16 and UTF-32 families. @@ -1223,6 +1340,16 @@ def test_no_leaking(self): max_handles = 1026 # too much for most UNIX systems else: max_handles = 2050 # too much for (at least some) Windows setups + if resource: + # And if it is not too much, try to make it too much. + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft > 1024: + resource.setrlimit(resource.RLIMIT_NOFILE, (1024, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + except (OSError, ValueError): + pass handles = [] tmpdir = tempfile.mkdtemp() try: @@ -1237,7 +1364,9 @@ def test_no_leaking(self): else: self.skipTest("failed to reach the file descriptor limit " "(tried %d)" % max_handles) - # Close a couple of them (should be enough for a subprocess) + # Close a couple of them (should be enough for a subprocess). + # Close lower file descriptors, so select() will work. + handles.reverse() for i in range(10): os.close(handles.pop()) # Loop creating some subprocesses. If one of them leaks some fds, @@ -1341,8 +1470,6 @@ def test_bufsize_equal_one_text_mode(self): line = "line\n" self._test_bufsize_equal_one(line, line, universal_newlines=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bufsize_equal_one_binary_mode(self): # line is not flushed in binary mode with bufsize=1. # we should get empty response @@ -1414,7 +1541,7 @@ def open_fds(): t = threading.Thread(target=open_fds) t.start() try: - with self.assertRaises(EnvironmentError): + with self.assertRaises(OSError): subprocess.Popen(NONEXISTING_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -1494,7 +1621,7 @@ def test_issue8780(self): "[sys.executable, '-c', 'print(\"Hello World!\")'])", 'assert retcode == 0')) output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + self.assertStartsWith(output, b'Hello World!') def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the @@ -1528,9 +1655,6 @@ def test_communicate_epipe(self): p.communicate(b"x" * 2**20) def test_repr(self): - path_cmd = pathlib.Path("my-tool.py") - pathlib_cls = path_cmd.__class__.__name__ - cases = [ ("ls", True, 123, "<Popen: returncode: 123 args: 'ls'>"), ('a' * 100, True, 0, @@ -1538,7 +1662,8 @@ def test_repr(self): (["ls"], False, None, "<Popen: returncode: None args: ['ls']>"), (["ls", '--my-opts', 'a' * 100], False, None, "<Popen: returncode: None args: ['ls', '--my-opts', 'aaaaaaaaaaaaaaaaaaaaaaaa...>"), - (path_cmd, False, 7, f"<Popen: returncode: 7 args: {pathlib_cls}('my-tool.py')>") + (os_helper.FakePath("my-tool.py"), False, 7, + "<Popen: returncode: 7 args: <FakePath 'my-tool.py'>>") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: @@ -1554,8 +1679,6 @@ def test_communicate_epipe_only_stdin(self): p.wait() p.communicate(b"x" * 2**20) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), "Requires signal.SIGUSR1") @unittest.skipUnless(hasattr(os, 'kill'), @@ -1615,21 +1738,6 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) - @unittest.skipUnless(hasattr(subprocess, '_winapi'), 'need subprocess._winapi') def test_wait_negative_timeout(self): @@ -1646,6 +1754,40 @@ def test_wait_negative_timeout(self): self.assertEqual(proc.wait(), 0) + def test_post_timeout_communicate_sends_input(self): + """GH-141473 regression test; the stdin pipe must close""" + with subprocess.Popen( + [sys.executable, "-uc", """\ +import sys +while c := sys.stdin.read(512): + sys.stdout.write(c) +print() +"""], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + data = f"spam{'#'*4096}beans" + proc.communicate( + input=data, + timeout=0, + ) + except subprocess.TimeoutExpired as exc: + pass + # Prior to the bugfix, this would hang as the stdin + # pipe to the child had not been closed. + try: + stdout, stderr = proc.communicate(timeout=15) + except subprocess.TimeoutExpired as exc: + self.fail("communicate() hung waiting on child process that should have seen its stdin pipe close and exit") + self.assertEqual( + proc.returncode, 0, + msg=f"STDERR:\n{stderr}\nSTDOUT:\n{stdout}") + self.assertStartsWith(stdout, "spam") + self.assertIn("beans", stdout) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1719,20 +1861,11 @@ def test_check_output_stdin_with_input_arg(self): self.assertIn('stdin', c.exception.args[0]) self.assertIn('input', c.exception.args[0]) - @support.requires_resource('walltime') def test_check_output_timeout(self): with self.assertRaises(subprocess.TimeoutExpired) as c: - cp = self.run_python(( - "import sys, time\n" - "sys.stdout.write('BDFL')\n" - "sys.stdout.flush()\n" - "time.sleep(3600)"), - # Some heavily loaded buildbots (sparc Debian 3.x) require - # this much time to start and print. - timeout=3, stdout=subprocess.PIPE) - self.assertEqual(c.exception.output, b'BDFL') - # output is aliased to stdout - self.assertEqual(c.exception.stdout, b'BDFL') + cp = self.run_python( + "import time; time.sleep(3600)", + timeout=0.1, stdout=subprocess.PIPE) def test_run_kwargs(self): newenv = os.environ.copy() @@ -1770,8 +1903,7 @@ def test_run_with_pathlike_path_and_arguments(self): res = subprocess.run(args) self.assertEqual(res.returncode, 57) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(mswindows, 'TODO: RUSTPYTHON; empty env block fails nondeterministically') @unittest.skipUnless(mswindows, "Maybe test trigger a leak on Ubuntu") def test_run_with_an_empty_env(self): # gh-105436: fix subprocess.run(..., env={}) broken on Windows @@ -1788,6 +1920,13 @@ def test_capture_output(self): self.assertIn(b'BDFL', cp.stdout) self.assertIn(b'FLUFL', cp.stderr) + def test_stdout_stdout(self): + # run() refuses to accept stdout=STDOUT + with self.assertRaises(ValueError, + msg=("STDOUT can only be used for stderr")): + self.run_python("print('will not be run')", + stdout=subprocess.STDOUT) + def test_stdout_with_capture_output_arg(self): # run() refuses to accept 'stdout' with 'capture_output' tf = tempfile.TemporaryFile() @@ -1842,8 +1981,8 @@ def test_encoding_warning(self): capture_output=True) lines = cp.stderr.splitlines() self.assertEqual(len(lines), 2, lines) - self.assertTrue(lines[0].startswith(b"<string>:2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b"<string>:3: EncodingWarning: ")) + self.assertStartsWith(lines[0], b"<string>:2: EncodingWarning: ") + self.assertStartsWith(lines[1], b"<string>:3: EncodingWarning: ") def _get_test_grp_name(): @@ -1972,8 +2111,6 @@ def bad_error(*args): self.assertIn(repr(error_data), str(e.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(not os.path.exists('/proc/self/status'), "need /proc/self/status") def test_restore_signals(self): @@ -2136,16 +2273,12 @@ def test_group_error(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, group=65535) - # TODO: RUSTPYTHON, observed gids do not match expected gids - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') def test_extra_groups(self): gid = os.getegid() group_list = [65534 if gid != 65534 else 65533] self._test_extra_groups_impl(gid=gid, group_list=group_list) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') def test_extra_groups_empty_list(self): self._test_extra_groups_impl(gid=os.getegid(), group_list=[]) @@ -2181,8 +2314,8 @@ def _test_extra_groups_impl(self, *, gid, group_list): subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[name_group]) - @unittest.skip("TODO: RUSTPYTHON; clarify failure condition") # No skip necessary, this test won't make it to a setgroup() call. + @unittest.skip("TODO: RUSTPYTHON; clarify failure condition") def test_extra_groups_invalid_gid_t_values(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) @@ -2192,8 +2325,6 @@ def test_extra_groups_invalid_gid_t_values(self): cwd=os.curdir, env=os.environ, extra_groups=[2**64]) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(mswindows or not hasattr(os, 'umask'), 'POSIX umask() is not available.') def test_umask(self): @@ -2245,8 +2376,6 @@ def test_CalledProcessError_str_non_zero(self): error_string = str(err) self.assertIn("non-zero exit status 2.", error_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec(self): # DISCLAIMER: Setting environment variables is *not* a good use # of a preexec_fn. This is merely a test. @@ -2258,8 +2387,6 @@ def test_preexec(self): with p: self.assertEqual(p.stdout.read(), b"apple") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_exception(self): def raise_it(): raise ValueError("What if two swallows carried a coconut?") @@ -2301,8 +2428,6 @@ def _execute_child(self, *args, **kwargs): for fd in devzero_fds: os.close(fd) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") def test_preexec_errpipe_does_not_double_close_pipes(self): """Issue16140: Don't double close pipes on preexec error.""" @@ -2317,8 +2442,6 @@ def raise_it(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=raise_it) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_gc_module_failure(self): # This tests the code that disables garbage collection if the child # process will execute any Python. @@ -2340,8 +2463,6 @@ def test_preexec_gc_module_failure(self): if not enabled: gc.disable() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') def test_preexec_fork_failure(self): @@ -2752,8 +2873,6 @@ def test_swap_std_fds_with_one_closed(self): for to_fds in itertools.permutations(range(3), 2): self._check_swap_std_fds_with_one_closed(from_fds, to_fds) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") @@ -2893,7 +3012,7 @@ def kill_p2(): p1.stdout.close() p2.stdout.close() - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_close_fds(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3021,11 +3140,11 @@ def test_close_fds_when_max_fd_is_lowered(self): msg="Some fds were left open.") - @unittest.skip("TODO: RUSTPYTHON, flaky test") # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file # descriptor of a pipe closed in the parent process is valid in the # child process according to fstat(), but the mode of the file # descriptor is invalid, and read or write raise an error. + @unittest.skip("TODO: RUSTPYTHON; flaky test") @support.requires_mac_ver(10, 5) def test_pass_fds(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3229,8 +3348,6 @@ def test_leak_fast_process_del_killed(self): else: self.assertNotIn(ident, [id(o) for o in subprocess._active]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_fds_after_preexec(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3324,7 +3441,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, True) + None) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: @@ -3440,8 +3557,7 @@ def test_communicate_repeated_call_after_stdout_close(self): except subprocess.TimeoutExpired: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'preexec_fn not supported at interpreter shutdown' not found in b"Exception ignored in: <function AtFinalization.__del__ at 0xa92f93840>\nAttributeError: 'NoneType' object has no attribute 'Popen'\n" def test_preexec_at_exit(self): code = f"""if 1: import atexit @@ -3461,6 +3577,62 @@ def __del__(self): self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @strace_helper.requires_strace() + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test_vfork_used_when_expected(self): + # This is a performance regression test to ensure we default to using + # vfork() when possible. + # Technically this test could pass when posix_spawn is used as well + # because libc tends to implement that internally using vfork. But + # that'd just be testing a libc+kernel implementation detail. + + # Are intersted in the system calls: + # clone,clone2,clone3,fork,vfork,exit,exit_group + # Unfortunately using `--trace` with that list to strace fails because + # not all are supported on all platforms (ex. clone2 is ia64 only...) + # So instead use `%process` which is recommended by strace, and contains + # the above. + true_binary = "/bin/true" + strace_args = ["--trace=%process"] + + with self.subTest(name="default_is_vfork"): + vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])""", + strace_args + ) + # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) + self.assertRegex(vfork_result.event_bytes, br"(?i)vfork") + # Do NOT check that fork() or other clones did not happen. + # If the OS denys the vfork it'll fallback to plain fork(). + + # Test that each individual thing that would disable the use of vfork + # actually disables it. + for sub_name, preamble, sp_kwarg, expect_permission_error in ( + ("preexec", "", "preexec_fn=lambda: None", False), + ("setgid", "", f"group={os.getgid()}", True), + ("setuid", "", f"user={os.getuid()}", True), + ("setgroups", "", "extra_groups=[]", True), + ): + with self.subTest(name=sub_name): + non_vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + {preamble} + try: + subprocess.check_call( + [{true_binary!r}], **dict({sp_kwarg})) + except PermissionError: + if not {expect_permission_error}: + raise""", + strace_args + ) + # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). + self.assertNotRegex(non_vfork_result.event_bytes, br"(?i)vfork") + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index f4a8d434ed1..d6d08ee53f8 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -18,10 +18,11 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import html.entities + import html.entities # noqa: F401 try: - import tty # Not available on Windows + # Not available on Windows + import tty # noqa: F401 except ImportError: if support.verbose: print("skipping tty") diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 8967dab8bdd..5548f4c71a2 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -344,12 +344,10 @@ def test_super_argcount(self): with self.assertRaisesRegex(TypeError, "expected at most"): super(int, int, int) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." def test_super_argtype(self): with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): super(1, int) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'test.support.import_helper' has no attribute 'ready_to_import' def test_shadowed_global(self): source = textwrap.dedent( """ @@ -410,7 +408,6 @@ def method(self): with self.assertRaisesRegex(AttributeError, "'super' object has no attribute 'msg'"): C().method() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." def test_bad_first_arg(self): class C: def method(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 7c8380498e3..37b5543badf 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,12 +1,16 @@ +import contextlib import errno import importlib import io +import logging import os import shutil +import signal import socket import stat import subprocess import sys +import sysconfig import tempfile import textwrap import unittest @@ -22,26 +26,51 @@ TESTFN = os_helper.TESTFN +class LogCaptureHandler(logging.StreamHandler): + # Inspired by pytest's caplog + def __init__(self): + super().__init__(io.StringIO()) + self.records = [] + + def emit(self, record) -> None: + self.records.append(record) + super().emit(record) + + def handleError(self, record): + raise + + +@contextlib.contextmanager +def _caplog(): + handler = LogCaptureHandler() + root_logger = logging.getLogger() + root_logger.addHandler(handler) + try: + yield handler + finally: + root_logger.removeHandler(handler) + + class TestSupport(unittest.TestCase): @classmethod def setUpClass(cls): - orig_filter_len = len(warnings.filters) + orig_filter_len = len(warnings._get_filters()) cls._warnings_helper_token = support.ignore_deprecations_from( "test.support.warnings_helper", like=".*used in test_support.*" ) cls._test_support_token = support.ignore_deprecations_from( __name__, like=".*You should NOT be seeing this.*" ) - assert len(warnings.filters) == orig_filter_len + 2 + assert len(warnings._get_filters()) == orig_filter_len + 2 @classmethod def tearDownClass(cls): - orig_filter_len = len(warnings.filters) + orig_filter_len = len(warnings._get_filters()) support.clear_ignored_deprecations( cls._warnings_helper_token, cls._test_support_token, ) - assert len(warnings.filters) == orig_filter_len - 2 + assert len(warnings._get_filters()) == orig_filter_len - 2 def test_ignored_deprecations_are_silent(self): """Test support.ignore_deprecations_from() silences warnings""" @@ -69,7 +98,7 @@ def test_get_original_stdout(self): self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): - import sched + import sched # noqa: F401 self.assertIn("sched", sys.modules) import_helper.unload("sched") self.assertNotIn("sched", sys.modules) @@ -185,7 +214,7 @@ def test_temp_dir__existing_dir__quiet_true(self): path = os.path.realpath(path) try: - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.temp_dir(path, quiet=True) as temp_path: self.assertEqual(path, temp_path) warnings = [str(w.message) for w in recorder.warnings] @@ -194,11 +223,14 @@ def test_temp_dir__existing_dir__quiet_true(self): finally: shutil.rmtree(path) - self.assertEqual(len(warnings), 1, warnings) - warn = warnings[0] - self.assertTrue(warn.startswith(f'tests may fail, unable to create ' - f'temporary directory {path!r}: '), - warn) + self.assertListEqual(warnings, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to create ' + f'temporary directory {path!r}: ' + ) @support.requires_fork() def test_temp_dir__forked_child(self): @@ -258,35 +290,41 @@ def test_change_cwd__non_existent_dir__quiet_true(self): with os_helper.temp_dir() as parent_dir: bad_dir = os.path.join(parent_dir, 'does_not_exist') - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.change_cwd(bad_dir, quiet=True) as new_cwd: self.assertEqual(new_cwd, original_cwd) self.assertEqual(os.getcwd(), new_cwd) warnings = [str(w.message) for w in recorder.warnings] - self.assertEqual(len(warnings), 1, warnings) - warn = warnings[0] - self.assertTrue(warn.startswith(f'tests may fail, unable to change ' - f'the current working directory ' - f'to {bad_dir!r}: '), - warn) + self.assertListEqual(warnings, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to change ' + f'the current working directory ' + f'to {bad_dir!r}: ' + ) # Tests for change_cwd() def test_change_cwd__chdir_warning(self): """Check the warning message when os.chdir() fails.""" path = TESTFN + '_does_not_exist' - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.change_cwd(path=path, quiet=True): pass messages = [str(w.message) for w in recorder.warnings] - self.assertEqual(len(messages), 1, messages) - msg = messages[0] - self.assertTrue(msg.startswith(f'tests may fail, unable to change ' - f'the current working directory ' - f'to {path!r}: '), - msg) + self.assertListEqual(messages, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to change ' + f'the current working directory ' + f'to {path!r}: ', + ) # Tests for temp_cwd() @@ -369,10 +407,10 @@ class Obj: with support.swap_attr(obj, "y", 5) as y: self.assertEqual(obj.y, 5) self.assertIsNone(y) - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') with support.swap_attr(obj, "y", 5): del obj.y - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') def test_swap_item(self): D = {"x":1} @@ -420,8 +458,7 @@ def test_detect_api_mismatch__ignore(self): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check__all__(self): extra = {'tempdir'} not_exported = {'template'} @@ -432,10 +469,7 @@ def test_check__all__(self): extra = { 'TextTestResult', - 'findTestCases', - 'getTestCaseNames', 'installHandler', - 'makeSuite', } not_exported = {'load_tests', "TestProgram", "BaseTestSuite"} support.check__all__(self, @@ -528,6 +562,7 @@ def test_args_from_interpreter_flags(self): ['-Wignore', '-X', 'dev'], ['-X', 'faulthandler'], ['-X', 'importtime'], + ['-X', 'importtime=2'], ['-X', 'showrefcount'], ['-X', 'tracemalloc'], ['-X', 'tracemalloc=3'], @@ -551,119 +586,13 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') - def test_match_test(self): - class Test: - def __init__(self, test_id): - self.test_id = test_id - - def id(self): - return self.test_id - - test_access = Test('test.test_os.FileTests.test_access') - test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') - - # Test acceptance - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests([]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests([test_access.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(['test_os'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(['test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(['filetests'], None) - self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) - self.assertTrue(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - support.set_match_tests(['test_access', 'DONTMATCH'], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests(None, [test_access.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(None, ['test_os']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(None, ['test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(None, ['filetests']) - self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) - self.assertFalse(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - support.set_match_tests(None, ['test_access', 'DONTMATCH']) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - @unittest.skipIf(sys.platform.startswith("win"), "TODO: RUSTPYTHON; os.dup on windows") - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) @@ -685,14 +614,13 @@ def test_print_warning(self): self.check_print_warning("a\nb", 'Warning -- a\nWarning -- b\n') - @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") def test_has_strftime_extensions(self): - if support.is_emscripten or sys.platform == "win32": + if sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - _testinternalcapi module not available def test_get_recursion_depth(self): # test support.get_recursion_depth() code = textwrap.dedent(""" @@ -736,13 +664,15 @@ def test_recursive(depth, limit): """) script_helper.assert_python_ok("-c", code) + @unittest.skip("TODO: RUSTPYTHON; - causes segfault in debug builds") + @support.skip_if_unlimited_stack_size def test_recursion(self): # Test infinite_recursion() and get_recursion_available() functions. def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() @@ -768,7 +698,107 @@ def recursive_function(depth): else: self.fail("RecursionError was not raised") - #self.assertEqual(available, 2) + def test_parse_memlimit(self): + parse = support._parse_memlimit + KiB = 1024 + MiB = KiB * 1024 + GiB = MiB * 1024 + TiB = GiB * 1024 + self.assertEqual(parse('0k'), 0) + self.assertEqual(parse('3k'), 3 * KiB) + self.assertEqual(parse('2.4m'), int(2.4 * MiB)) + self.assertEqual(parse('4g'), int(4 * GiB)) + self.assertEqual(parse('1t'), TiB) + + for limit in ('', '3', '3.5.10k', '10x'): + with self.subTest(limit=limit): + with self.assertRaises(ValueError): + parse(limit) + + def test_set_memlimit(self): + _4GiB = 4 * 1024 ** 3 + TiB = 1024 ** 4 + old_max_memuse = support.max_memuse + old_real_max_memuse = support.real_max_memuse + try: + if sys.maxsize > 2**32: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, _4GiB) + self.assertEqual(support.real_max_memuse, _4GiB) + + big = 2**100 // TiB + support.set_memlimit(f'{big}t') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, big * TiB) + else: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, _4GiB) + finally: + support.max_memuse = old_max_memuse + support.real_max_memuse = old_real_max_memuse + + def test_copy_python_src_ignore(self): + # Get source directory + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + src_dir = os.path.abspath(src_dir) + + # Check that the source code is available + if not os.path.exists(src_dir): + self.skipTest(f"cannot access Python source code directory:" + f" {src_dir!r}") + # Check that the landmark copy_python_src_ignore() expects is available + # (Previously we looked for 'Lib\os.py', which is always present on Windows.) + landmark = os.path.join(src_dir, 'Modules') + if not os.path.exists(landmark): + self.skipTest(f"cannot access Python source code directory:" + f" {landmark!r} landmark is missing") + + # Test support.copy_python_src_ignore() + + # Source code directory + ignored = {'.git', '__pycache__'} + names = os.listdir(src_dir) + self.assertEqual(support.copy_python_src_ignore(src_dir, names), + ignored | {'build'}) + + # Doc/ directory + path = os.path.join(src_dir, 'Doc') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored | {'build', 'venv'}) + + # Another directory + path = os.path.join(src_dir, 'Objects') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored) + + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(support.get_signal_name(exitcode), expected, + exitcode) + + def test_linked_to_musl(self): + linked = support.linked_to_musl() + self.assertIsNotNone(linked) + if support.is_wasm32: + self.assertTrue(linked) + # The value is cached, so make sure it returns the same value again. + self.assertIs(linked, support.linked_to_musl()) + # The musl version is either triple or just a major version number. + if linked: + self.assertIsInstance(linked, tuple) + self.assertIn(len(linked), (1, 3)) + for v in linked: + self.assertIsInstance(v, int) + # XXX -follows a list of untested API # make_legacy_pyc @@ -781,12 +811,10 @@ def recursive_function(depth): # EnvironmentVarGuard # transient_internet # run_with_locale - # set_memlimit # bigmemtest # precisionbigmemtest # bigaddrspacetest # requires_resource - # run_doctest # threading_cleanup # reap_threads # can_symlink diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 8f1a80a5242..ae93ee8d91f 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -2,6 +2,7 @@ Test the API of the symtable module. """ +import re import textwrap import symtable import unittest @@ -202,9 +203,7 @@ class SymtableTest(unittest.TestCase): # XXX: RUSTPYTHON # U = find_block(GenericMine, "U") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_type(self): self.assertEqual(self.top.get_type(), "module") self.assertEqual(self.Mine.get_type(), "class") @@ -222,8 +221,7 @@ def test_type(self): self.assertEqual(self.T.get_type(), "type variable") self.assertEqual(self.U.get_type(), "type variable") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_id(self): self.assertGreater(self.top.get_id(), 0) self.assertGreater(self.Mine.get_id(), 0) @@ -256,8 +254,7 @@ def test_lineno(self): self.assertEqual(self.top.get_lineno(), 0) self.assertEqual(self.spam.get_lineno(), 14) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_function_info(self): func = self.spam self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"]) @@ -266,8 +263,7 @@ def test_function_info(self): self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"]) self.assertEqual(self.internal.get_frees(), ("x",)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_globals(self): self.assertTrue(self.spam.lookup("glob").is_global()) self.assertFalse(self.spam.lookup("glob").is_declared_global()) @@ -280,16 +276,14 @@ def test_globals(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_global()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_global()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonlocal(self): self.assertFalse(self.spam.lookup("some_var").is_nonlocal()) self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal()) expected = ("some_var",) self.assertEqual(self.other_internal.get_nonlocals(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_local(self): self.assertTrue(self.spam.lookup("x").is_local()) self.assertFalse(self.spam.lookup("bar").is_local()) @@ -297,13 +291,11 @@ def test_local(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_local()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_local()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free(self): self.assertTrue(self.internal.lookup("x").is_free()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_referenced(self): self.assertTrue(self.internal.lookup("x").is_referenced()) self.assertTrue(self.spam.lookup("internal").is_referenced()) @@ -320,8 +312,7 @@ def test_symbol_lookup(self): self.assertRaises(KeyError, self.top.lookup, "not_here") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespaces(self): self.assertTrue(self.top.lookup("Mine").is_namespace()) self.assertTrue(self.Mine.lookup("a_method").is_namespace()) @@ -346,16 +337,17 @@ def test_assigned(self): self.assertTrue(self.Mine.lookup("a_method").is_assigned()) self.assertFalse(self.internal.lookup("x").is_assigned()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) @@ -373,8 +365,7 @@ def test_annotated(self): ' x: int', 'test', 'exec') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_imported(self): self.assertTrue(self.top.lookup("sys").is_imported()) @@ -384,28 +375,34 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_class_get_methods(self): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) + deprecation_mess = ( + re.escape('symtable.Class.get_methods() is deprecated ' + 'and will be removed in Python 3.16.') + ) + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") this = find_block(top, "ComplexClass") - self.assertEqual(this.get_methods(), ( - 'a_method', 'a_method_pep_695', - 'an_async_method', 'an_async_method_pep_695', - 'a_classmethod', 'a_classmethod_pep_695', - 'an_async_classmethod', 'an_async_classmethod_pep_695', - 'a_staticmethod', 'a_staticmethod_pep_695', - 'an_async_staticmethod', 'an_async_staticmethod_pep_695', - 'a_fakemethod', 'a_fakemethod_pep_695', - 'an_async_fakemethod', 'an_async_fakemethod_pep_695', - 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', - 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', - 'glob_assigned_meth', 'glob_assigned_meth_pep_695', - 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', - )) + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) # Test generator expressions that are of type TYPE_FUNCTION # but will not be reported by get_methods() since they are @@ -418,7 +415,8 @@ def check_body(body, expected_methods): indented = textwrap.indent(body, ' ' * 4) top = symtable.symtable(f"class A:\n{indented}", "?", "exec") this = find_block(top, "A") - self.assertEqual(this.get_methods(), expected_methods) + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), expected_methods) # statements with 'genexpr' inside it GENEXPRS = ( @@ -459,8 +457,7 @@ def check_body(body, expected_methods): check_body('\n'.join((gen, func)), ('genexpr',)) check_body('\n'.join((func, gen)), ('genexpr',)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised ### while parsing or building symbol table. @@ -492,8 +489,7 @@ def test_single(self): def test_exec(self): symbols = symtable.symtable("def f(x): return x", "?", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bytes(self): top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") self.assertIsNotNone(find_block(top, "Mine")) @@ -507,8 +503,7 @@ def test_symtable_repr(self): self.assertEqual(str(self.top), "<SymbolTable for module ?>") self.assertEqual(str(self.spam), "<Function SymbolTable for spam in ?>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_symbol_repr(self): self.assertEqual(repr(self.spam.lookup("glob")), "<symbol 'glob': GLOBAL_IMPLICIT, USE>") @@ -524,19 +519,68 @@ def test_symbol_repr(self): "<symbol 'x': FREE, USE>") self.assertEqual(repr(self.other_internal.lookup("some_var")), "<symbol 'some_var': FREE, USE|DEF_NONLOCAL|DEF_LOCAL>") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "<symbol 'T': LOCAL, DEF_LOCAL|DEF_TYPE_PARAM>") + + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "<symbol 'x': LOCAL, USE|DEF_LOCAL|DEF_COMP_ITER>") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "<symbol 'x': CELL, DEF_LOCAL|DEF_COMP_ITER|DEF_COMP_CELL>") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "<symbol 'x': LOCAL, DEF_LOCAL|DEF_FREE_CLASS>") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_symtable_entry_repr(self): expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" self.assertEqual(repr(self.top._table), expected) + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + + +class ComprehensionTests(unittest.TestCase): + def get_identifiers_recursive(self, st, res): + res.extend(st.get_identifiers()) + for ch in st.get_children(): + self.get_identifiers_recursive(ch, res) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 2 != 1 + def test_loopvar_in_only_one_scope(self): + # ensure that the loop variable appears only once in the symtable + comps = [ + "[x for x in [1]]", + "{x for x in [1]}", + "{x:x*x for x in [1]}", + ] + for comp in comps: + with self.subTest(comp=comp): + st = symtable.symtable(comp, "?", "exec") + ids = [] + self.get_identifiers_recursive(st, ids) + self.assertEqual(len([x for x in ids if x == 'x']), 1) + class CommandLineTest(unittest.TestCase): maxDiff = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_file(self): filename = os_helper.TESTFN self.addCleanup(os_helper.unlink, filename) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 29dd04995ec..08e69caae1f 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2293,7 +2293,6 @@ def test_expression_with_assignment(self): offset=7 ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_curly_brace_after_primary_raises_immediately(self): self._check_error("f{}", "invalid syntax", mode="single") @@ -2577,6 +2576,7 @@ async def bug(): with self.subTest(f"out of range: {n=}"): self._check_error(get_code(n), "too many statically nested blocks") + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1ce2e9fc0fe..9ebd6dd9cc1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1,27 +1,42 @@ import builtins import codecs +# import _datetime # TODO: RUSTPYTHON import gc +import io import locale import operator import os +import random +import socket import struct import subprocess import sys import sysconfig import test.support +from io import StringIO +from unittest import mock from test import support from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.socket_helper import find_unused_port from test.support import threading_helper from test.support import import_helper +from test.support import force_not_colorized +from test.support import SHORT_TIMEOUT +try: + from concurrent import interpreters +except ImportError: + interpreters = None import textwrap import unittest import warnings -# count the number of test runs, used to create unique -# strings to intern in test_intern() -INTERN_NUMRUNS = 0 +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) + DICT_KEY_STRUCT_FORMAT = 'n2BI2n' @@ -42,7 +57,7 @@ def test_original_displayhook(self): dh(None) self.assertEqual(out.getvalue(), "") - self.assertTrue(not hasattr(builtins, "_")) + self.assertNotHasAttr(builtins, "_") # sys.displayhook() requires arguments self.assertRaises(TypeError, dh) @@ -71,6 +86,18 @@ def baddisplayhook(obj): code = compile("42", "<string>", "single") self.assertRaises(ValueError, eval, code) + def test_gh130163(self): + class X: + def __repr__(self): + sys.stdout = io.StringIO() + support.gc_collect() + return 'foo' + + with support.swap_attr(sys, 'stdout', None): + sys.stdout = io.StringIO() # the only reference + sys.displayhook(X()) # should not crash + + class ActiveExceptionTests(unittest.TestCase): def test_exc_info_no_exception(self): self.assertEqual(sys.exc_info(), (None, None, None)) @@ -137,6 +164,7 @@ def f(): class ExceptHookTest(unittest.TestCase): + @force_not_colorized def test_original_excepthook(self): try: raise ValueError(42) @@ -144,12 +172,11 @@ def test_original_excepthook(self): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertEndsWith(err.getvalue(), "ValueError: 42\n") self.assertRaises(TypeError, sys.__excepthook__) - # TODO: RUSTPYTHON, SyntaxError formatting in arbitrary tracebacks - @unittest.expectedFailure + @force_not_colorized def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -165,11 +192,12 @@ def test_excepthook_bytes_filename(self): err = err.getvalue() self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) self.assertIn(""" text\n""", err) - self.assertTrue(err.endswith("SyntaxError: msg\n")) + self.assertEndsWith(err, "SyntaxError: msg\n") def test_excepthook(self): with test.support.captured_output("stderr") as stderr: - sys.excepthook(1, '1', 1) + with test.support.catch_unraisable_exception(): + sys.excepthook(1, '1', 1) self.assertTrue("TypeError: print_exception(): Exception expected for " \ "value, str found" in stderr.getvalue()) @@ -182,6 +210,7 @@ class SysModuleTest(unittest.TestCase): def tearDown(self): test.support.reap_children() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exit(self): # call with two arguments self.assertRaises(TypeError, sys.exit, 42, 42) @@ -196,6 +225,20 @@ def test_exit(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + # gh-125842: Windows uses 32-bit unsigned integers for exit codes + # so a -1 exit code is sometimes interpreted as 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(0xffff_ffff)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + # Overflow results in a -1 exit code, which may be converted to 0xff + # or 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(2**128)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + # call with integer argument with self.assertRaises(SystemExit) as cm: sys.exit(42) @@ -227,8 +270,7 @@ def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) self.assertEqual(rc, 1) self.assertEqual(out, b'') - self.assertTrue(err.startswith(expected), - "%s doesn't start with %s" % (ascii(err), ascii(expected))) + self.assertStartsWith(err, expected) # test that stderr buffer is flushed before the exit message is written # into stderr @@ -238,17 +280,36 @@ def check_exit_message(code, expected, **env_vars): # test that the exit message is written with backslashreplace error # handler to stderr - # TODO: RUSTPYTHON; allow surrogates in strings - # check_exit_message( - # r'import sys; sys.exit("surrogates:\uDCFF")', - # b"surrogates:\\udcff") + check_exit_message( + r'import sys; sys.exit("surrogates:\uDCFF")', + b"surrogates:\\udcff") # test that the unicode message is encoded to the stderr encoding # instead of the default encoding (utf8) - # TODO: RUSTPYTHON; handle PYTHONIOENCODING - # check_exit_message( - # r'import sys; sys.exit("h\xe9")', - # b"h\xe9", PYTHONIOENCODING='latin-1') + check_exit_message( + r'import sys; sys.exit("h\xe9")', + b"h\xe9", PYTHONIOENCODING='latin-1') + + @support.requires_subprocess() + def test_exit_codes_under_repl(self): + # GH-129900: SystemExit, or things that raised it, didn't + # get their return code propagated by the REPL + import tempfile + + exit_ways = [ + "exit", + "__import__('sys').exit", + "raise SystemExit" + ] + + for exitfunc in exit_ways: + for return_code in (0, 123): + with self.subTest(exitfunc=exitfunc, return_code=return_code): + with tempfile.TemporaryFile("w+") as stdin: + stdin.write(f"{exitfunc}({return_code})\n") + stdin.seek(0) + proc = subprocess.run([sys.executable], stdin=stdin) + self.assertEqual(proc.returncode, return_code) def test_getdefaultencoding(self): self.assertRaises(TypeError, sys.getdefaultencoding, 42) @@ -273,21 +334,30 @@ def test_switchinterval(self): finally: sys.setswitchinterval(orig) - def test_recursionlimit(self): + def test_getrecursionlimit(self): + limit = sys.getrecursionlimit() + self.assertIsInstance(limit, int) + self.assertGreater(limit, 1) + self.assertRaises(TypeError, sys.getrecursionlimit, 42) - oldlimit = sys.getrecursionlimit() - self.assertRaises(TypeError, sys.setrecursionlimit) - self.assertRaises(ValueError, sys.setrecursionlimit, -42) - sys.setrecursionlimit(10000) - self.assertEqual(sys.getrecursionlimit(), 10000) - sys.setrecursionlimit(oldlimit) - - @unittest.skipIf(getattr(sys, "_rustpython_debugbuild", False), "TODO: RUSTPYTHON, stack overflow on debug build") + + def test_setrecursionlimit(self): + old_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(10_005) + self.assertEqual(sys.getrecursionlimit(), 10_005) + + self.assertRaises(TypeError, sys.setrecursionlimit) + self.assertRaises(ValueError, sys.setrecursionlimit, -42) + finally: + sys.setrecursionlimit(old_limit) + + @unittest.skipIf(getattr(sys, '_rustpython_debugbuild', False), 'TODO: RUSTPYTHON; stack overflow on debug build') def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() def f(): f() try: @@ -306,38 +376,33 @@ def f(): with self.assertRaises(RecursionError): f() finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) @test.support.cpython_only - def test_setrecursionlimit_recursion_depth(self): + def test_setrecursionlimit_to_depth(self): # Issue #25274: Setting a low recursion limit must be blocked if the # current recursion depth is already higher than limit. - from _testinternalcapi import get_recursion_depth - - def set_recursion_limit_at_depth(depth, limit): - recursion_depth = get_recursion_depth() - if recursion_depth >= depth: - with self.assertRaises(RecursionError) as cm: - sys.setrecursionlimit(limit) - self.assertRegex(str(cm.exception), - "cannot set the recursion limit to [0-9]+ " - "at the recursion depth [0-9]+: " - "the limit is too low") - else: - set_recursion_limit_at_depth(depth, limit) - - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() try: - sys.setrecursionlimit(1000) - - for limit in (10, 25, 50, 75, 100, 150, 200): - set_recursion_limit_at_depth(limit, limit) + depth = support.get_recursion_depth() + with self.subTest(limit=sys.getrecursionlimit(), depth=depth): + # depth + 1 is OK + sys.setrecursionlimit(depth + 1) + + # reset the limit to be able to call self.assertRaises() + # context manager + sys.setrecursionlimit(old_limit) + with self.assertRaises(RecursionError) as cm: + sys.setrecursionlimit(depth) + self.assertRegex(str(cm.exception), + "cannot set the recursion limit to [0-9]+ " + "at the recursion depth [0-9]+: " + "the limit is too low") finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_getwindowsversion(self): # Raise SkipTest if sys doesn't have getwindowsversion attribute test.support.get_attribute(sys, "getwindowsversion") @@ -368,15 +433,14 @@ def test_getwindowsversion(self): # still has 5 elements maj, min, buildno, plat, csd = sys.getwindowsversion() - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute 'call_tracing' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'call_tracing' def test_call_tracing(self): self.assertRaises(TypeError, sys.call_tracing, type, 2) @unittest.skipUnless(hasattr(sys, "setdlopenflags"), 'test needs sys.setdlopenflags()') def test_dlopenflags(self): - self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertHasAttr(sys, "getdlopenflags") self.assertRaises(TypeError, sys.getdlopenflags, 42) oldflags = sys.getdlopenflags() self.assertRaises(TypeError, sys.setdlopenflags) @@ -386,15 +450,21 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) n = None - self.assertEqual(sys.getrefcount(None), c+1) + # Singleton refcnts don't change + self.assertEqual(sys.getrefcount(None), c) del n self.assertEqual(sys.getrefcount(None), c) if hasattr(sys, "gettotalrefcount"): @@ -408,6 +478,26 @@ def test_getframe(self): is sys._getframe().f_code ) + def test_getframemodulename(self): + # Default depth gets ourselves + self.assertEqual(__name__, sys._getframemodulename()) + self.assertEqual("unittest.case", sys._getframemodulename(1)) + i = 0 + f = sys._getframe(i) + while f: + self.assertEqual( + f.f_globals['__name__'], + sys._getframemodulename(i) or '__main__' + ) + i += 1 + f2 = f.f_back + try: + f = sys._getframe(i) + except ValueError: + break + self.assertIs(f, f2) + self.assertIsNone(sys._getframemodulename(i)) + # sys._current_frames() is a CPython-only gimmick. # XXX RUSTPYTHON: above comment is from original cpython test; not sure why the cpython_only decorator wasn't added @test.support.cpython_only @@ -436,49 +526,49 @@ def g456(): t.start() entered_g.wait() - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_frames() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - - # Verify that the captured main-thread frame is _this_ frame. - frame = d.pop(main_id) - self.assertTrue(frame is sys._getframe()) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - frame = d.pop(thread_id) - stack = traceback.extract_stack(frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") - - self.assertEqual(sourceline, "g456()") + try: + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_frames() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + + # Verify that the captured main-thread frame is _this_ frame. + frame = d.pop(main_id) + self.assertTrue(frame is sys._getframe()) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + frame = d.pop(thread_id) + stack = traceback.extract_stack(frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute '_current_exceptions' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute '_current_exceptions' @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_current_exceptions(self): @@ -488,7 +578,7 @@ def test_current_exceptions(self): # Spawn a thread that blocks at a known place. Then the main # thread does sys._current_frames(), and verifies that the frames # returned make sense. - entered_g = threading.Event() + g_raised = threading.Event() leave_g = threading.Event() thread_info = [] # the thread's id @@ -497,55 +587,53 @@ def f123(): def g456(): thread_info.append(threading.get_ident()) - entered_g.set() while True: try: raise ValueError("oops") except ValueError: + g_raised.set() if leave_g.wait(timeout=support.LONG_TIMEOUT): break t = threading.Thread(target=f123) t.start() - entered_g.wait() - - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_exceptions() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - self.assertEqual((None, None, None), d.pop(main_id)) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - exc_type, exc_value, exc_tb = d.pop(thread_id) - stack = traceback.extract_stack(exc_tb.tb_frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") + g_raised.wait(timeout=support.LONG_TIMEOUT) - self.assertEqual(sourceline, "g456()") + try: + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_exceptions() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + self.assertEqual(None, d.pop(main_id)) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + exc_value = d.pop(thread_id) + stack = traceback.extract_stack(exc_value.__traceback__.tb_frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertStartsWith(sourceline, ("if leave_g.wait(", "g_raised.set()")) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() def test_attributes(self): self.assertIsInstance(sys.api_version, int) @@ -641,13 +729,15 @@ def test_attributes(self): self.assertIn(sys.float_repr_style, ('short', 'legacy')) if not sys.platform.startswith('win'): self.assertIsInstance(sys.abiflags, str) + else: + self.assertFalse(hasattr(sys, 'abiflags')) def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -670,13 +760,23 @@ def test_43581(self): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): - global INTERN_NUMRUNS - INTERN_NUMRUNS += 1 + has_is_interned = (test.support.check_impl_detail(cpython=True) + or hasattr(sys, '_is_interned')) self.assertRaises(TypeError, sys.intern) - s = "never interned before" + str(INTERN_NUMRUNS) + self.assertRaises(TypeError, sys.intern, b'abc') + if has_is_interned: + self.assertRaises(TypeError, sys._is_interned) + self.assertRaises(TypeError, sys._is_interned, b'abc') + s = "never interned before" + str(random.randrange(0, 10**9)) self.assertTrue(sys.intern(s) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s), True) s2 = s.swapcase().swapcase() + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) self.assertTrue(sys.intern(s2) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) # Subclasses of string can't be interned, because they # provide too much opportunity for insane things to happen. @@ -688,7 +788,75 @@ def __hash__(self): return 123 self.assertRaises(TypeError, sys.intern, S("abc")) + if has_is_interned: + self.assertIs(sys._is_interned(S("abc")), False) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_dynamically_allocated(self): + # Implementation detail: Dynamically allocated strings + # are distinct between interpreters + s = "never interned before" + str(random.randrange(0, 10**9)) + t = sys.intern(s) + self.assertIs(t, s) + + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + + assert id(t) != {id(s)}, (id(t), {id(s)}) + assert id(t) != {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_statically_allocated(self): + # Implementation detail: Statically allocated strings are shared + # between interpreters. + # See Tools/build/generate_global_objects.py for the list + # of strings that are always statically allocated. + for s in ('__init__', 'CANCELLED', '<module>', 'utf-8', + '{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff', + ): + with self.subTest(s=s): + t = sys.intern(s) + + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) == {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_singleton(self): + # Implementation detail: singletons are used for 0- and 1-character + # latin1 strings. + for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff': + with self.subTest(s=s): + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + assert id(s) == {id(s)} + t = sys.intern(s) + ''')) + self.assertTrue(sys._is_interned(s)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; needs update for context_aware_warnings def test_sys_flags(self): self.assertTrue(sys.flags) attrs = ("debug", @@ -698,7 +866,7 @@ def test_sys_flags(self): "hash_randomization", "isolated", "dev_mode", "utf8_mode", "warn_default_encoding", "safe_path", "int_max_str_digits") for attr in attrs: - self.assertTrue(hasattr(sys.flags, attr), attr) + self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) @@ -709,12 +877,7 @@ def test_sys_flags(self): def assert_raise_on_new_sys_type(self, sys_attr): # Users are intentionally prevented from creating new instances of # sys.flags, sys.version_info, and sys.getwindowsversion. - arg = sys_attr - attr_type = type(sys_attr) - with self.assertRaises(TypeError): - attr_type(arg) - with self.assertRaises(TypeError): - attr_type.__new__(attr_type, arg) + support.check_disallow_instantiation(self, type(sys_attr), sys_attr) def test_sys_flags_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.flags) @@ -722,8 +885,7 @@ def test_sys_flags_no_instantiation(self): def test_sys_version_info_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.version_info) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sys_getwindowsversion_no_instantiation(self): # Skip if not being run on Windows. test.support.get_attribute(sys, "getwindowsversion") @@ -731,10 +893,12 @@ def test_sys_getwindowsversion_no_instantiation(self): @test.support.cpython_only def test_clear_type_cache(self): - sys._clear_type_cache() + with self.assertWarnsRegex(DeprecationWarning, + r"sys\._clear_type_cache\(\) is deprecated.*"): + sys._clear_type_cache() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip('TODO: RUSTPYTHON; cp424 encoding not supported, causes panic') + @force_not_colorized @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -898,14 +1062,12 @@ def check_locale_surrogateescape(self, locale): 'stdout: surrogateescape\n' 'stderr: backslashreplace\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_c_locale_surrogateescape(self): self.check_locale_surrogateescape('C') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_posix_locale_surrogateescape(self): self.check_locale_surrogateescape('POSIX') @@ -915,10 +1077,11 @@ def test_implementation(self): levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} - self.assertTrue(hasattr(sys.implementation, 'name')) - self.assertTrue(hasattr(sys.implementation, 'version')) - self.assertTrue(hasattr(sys.implementation, 'hexversion')) - self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + self.assertHasAttr(sys.implementation, 'name') + self.assertHasAttr(sys.implementation, 'version') + self.assertHasAttr(sys.implementation, 'hexversion') + self.assertHasAttr(sys.implementation, 'cache_tag') + self.assertHasAttr(sys.implementation, 'supports_isolated_interpreters') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -932,6 +1095,15 @@ def test_implementation(self): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + # https://peps.python.org/pep-0734 + sii = sys.implementation.supports_isolated_interpreters + self.assertIsInstance(sii, bool) + if test.support.check_impl_detail(cpython=True): + if test.support.is_emscripten or test.support.is_wasi: + self.assertFalse(sii) + else: + self.assertTrue(sii) + @test.support.cpython_only def test_debugmallocstats(self): # Test sys._debugmallocstats() @@ -942,14 +1114,10 @@ def test_debugmallocstats(self): # Output of sys._debugmallocstats() depends on configure flags. # The sysconfig vars are not available on Windows. if sys.platform != "win32": - with_freelists = sysconfig.get_config_var("WITH_FREELISTS") with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") - if with_freelists: - self.assertIn(b"free PyDictObjects", err) + self.assertIn(b"free PyDictObjects", err) if with_pymalloc: self.assertIn(b'Small block threshold', err) - if not with_freelists and not with_pymalloc: - self.assertFalse(err) # The function has no parameter self.assertRaises(TypeError, sys._debugmallocstats, True) @@ -958,12 +1126,12 @@ def test_debugmallocstats(self): "sys.getallocatedblocks unavailable on this build") def test_getallocatedblocks(self): try: - import _testcapi + import _testinternalcapi except ImportError: with_pymalloc = support.with_pymalloc() else: try: - alloc_name = _testcapi.pymem_getallocatorsname() + alloc_name = _testinternalcapi.pymem_getallocatorsname() except RuntimeError as exc: # "cannot get allocators name" (ex: tracemalloc is used) with_pymalloc = True @@ -980,23 +1148,29 @@ def test_getallocatedblocks(self): # about the underlying implementation: the function might # return 0 or something greater. self.assertGreaterEqual(a, 0) + gc.collect() + b = sys.getallocatedblocks() + self.assertLessEqual(b, a) try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - self.assertLess(a, sys.gettotalrefcount()) + # The reported blocks will include immortalized strings, but the + # total ref count will not. This will sanity check that among all + # other objects (those eligible for garbage collection) there + # are more references being tracked than allocated blocks. + interned_immortal = sys.getunicodeinternedsize(_only_immortal=True) + self.assertLess(a - interned_immortal, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass gc.collect() - b = sys.getallocatedblocks() - self.assertLessEqual(b, a) - gc.collect() c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) - # TODO: RUSTPYTHON, AtExit.__del__ is not invoked because module destruction is missing. - @unittest.expectedFailure + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1018,8 +1192,7 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) self.assertEqual(stdout.rstrip(), b'True') - # TODO: RUSTPYTHON, IndexError: list index out of range - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: list index out of range def test_issue20602(self): # sys.flags and sys.float_info were wiped during shutdown. code = """if 1: @@ -1052,15 +1225,14 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'getandroidapilevel' + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) self.assertGreater(level, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @force_not_colorized @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: @@ -1081,14 +1253,20 @@ def check(tracebacklimit, expected): traceback = [ b'Traceback (most recent call last):', b' File "<string>", line 8, in <module>', + b' f2()', + b' ~~^^', b' File "<string>", line 6, in f2', + b' f1()', + b' ~~^^', b' File "<string>", line 4, in f1', + b' 1 / 0', + b' ~~^~~', b'ZeroDivisionError: division by zero' ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[2:]) - check(1, traceback[:1] + traceback[3:]) + check(2, traceback[:1] + traceback[4:]) + check(1, traceback[:1] + traceback[7:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) @@ -1124,15 +1302,11 @@ def test_orig_argv(self): self.assertEqual(proc.stdout.rstrip().splitlines(), expected, proc) - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute 'stdlib_module_names' - @unittest.expectedFailure def test_module_names(self): self.assertIsInstance(sys.stdlib_module_names, frozenset) for name in sys.stdlib_module_names: self.assertIsInstance(name, str) - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute '_stdlib_dir' - @unittest.expectedFailure def test_stdlib_dir(self): os = import_helper.import_fresh_module('os') marker = getattr(os, '__file__', None) @@ -1142,41 +1316,76 @@ def test_stdlib_dir(self): self.assertEqual(os.path.normpath(sys._stdlib_dir), os.path.normpath(expected)) + @unittest.skipUnless(hasattr(sys, 'getobjects'), 'need sys.getobjects()') + def test_getobjects(self): + # sys.getobjects(0) + all_objects = sys.getobjects(0) + self.assertIsInstance(all_objects, list) + self.assertGreater(len(all_objects), 0) + + # sys.getobjects(0, MyType) + class MyType: + pass + size = 100 + my_objects = [MyType() for _ in range(size)] + get_objects = sys.getobjects(0, MyType) + self.assertEqual(len(get_objects), size) + for obj in get_objects: + self.assertIsInstance(obj, MyType) + + # sys.getobjects(3, MyType) + get_objects = sys.getobjects(3, MyType) + self.assertEqual(len(get_objects), 3) + + @unittest.skipUnless(hasattr(sys, '_stats_on'), 'need Py_STATS build') + def test_pystats(self): + # Call the functions, just check that they don't crash + # Cannot save/restore state. + sys._stats_on() + sys._stats_off() + sys._stats_clear() + sys._stats_dump() + + @test.support.cpython_only + @unittest.skipUnless(hasattr(sys, 'abiflags'), 'need sys.abiflags') + def test_disable_gil_abi(self): + self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): - def write_unraisable_exc(self, exc, err_msg, obj): - import _testcapi - import types - err_msg2 = f"Exception ignored {err_msg}" - try: - _testcapi.write_unraisable_exc(exc, err_msg, obj) - return types.SimpleNamespace(exc_type=type(exc), - exc_value=exc, - exc_traceback=exc.__traceback__, - err_msg=err_msg2, - object=obj) - finally: - # Explicitly break any reference cycle - exc = None - def test_original_unraisablehook(self): - for err_msg in (None, "original hook"): - with self.subTest(err_msg=err_msg): - obj = "an object" - - with test.support.captured_output("stderr") as stderr: - with test.support.swap_attr(sys, 'unraisablehook', - sys.__unraisablehook__): - self.write_unraisable_exc(ValueError(42), err_msg, obj) - - err = stderr.getvalue() - if err_msg is not None: - self.assertIn(f'Exception ignored {err_msg}: {obj!r}\n', err) - else: - self.assertIn(f'Exception ignored in: {obj!r}\n', err) - self.assertIn('Traceback (most recent call last):\n', err) - self.assertIn('ValueError: 42\n', err) + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable + obj = hex + + with support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Exception ignored in: {obj!r}') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), 'Error in %R', obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Error in {obj!r}:') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') def test_original_unraisablehook_err(self): # bpo-22836: PyErr_WriteUnraisable() should give sensible reports @@ -1216,13 +1425,15 @@ def __del__(self): else: self.assertIn("ValueError", report) self.assertIn("del is broken", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") def test_original_unraisablehook_exception_qualname(self): # See bpo-41031, bpo-45083. # Check that the exception is printed with its qualified name # rather than just classname, and the module names appears # unless it is one of the hard-coded exclusions. + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable class A: class B: class X(Exception): @@ -1234,9 +1445,7 @@ class X(Exception): with test.support.captured_stderr() as stderr, test.support.swap_attr( sys, 'unraisablehook', sys.__unraisablehook__ ): - expected = self.write_unraisable_exc( - A.B.X(), "msg", "obj" - ) + err_writeunraisable(A.B.X(), "obj") report = stderr.getvalue() self.assertIn(A.B.X.__qualname__, report) if moduleName in ['builtins', '__main__']: @@ -1252,34 +1461,45 @@ def test_original_unraisablehook_wrong_type(self): sys.unraisablehook(exc) def test_custom_unraisablehook(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable hook_args = None def hook_func(args): nonlocal hook_args hook_args = args - obj = object() + obj = hex try: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - expected = self.write_unraisable_exc(ValueError(42), - "custom hook", obj) - for attr in "exc_type exc_value exc_traceback err_msg object".split(): - self.assertEqual(getattr(hook_args, attr), - getattr(expected, attr), - (hook_args, expected)) + exc = ValueError(42) + err_writeunraisable(exc, obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertIsNone(hook_args.err_msg) + self.assertEqual(hook_args.object, obj) + + err_formatunraisable(exc, "custom hook %R", obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertEqual(hook_args.err_msg, f'custom hook {obj!r}') + self.assertIsNone(hook_args.object) finally: # expected and hook_args contain an exception: break reference cycle expected = None hook_args = None def test_custom_unraisablehook_fail(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable def hook_func(*args): raise Exception("hook_func failed") with test.support.captured_output("stderr") as stderr: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - self.write_unraisable_exc(ValueError(42), - "custom hook fail", None) + err_writeunraisable(ValueError(42), "custom hook fail") err = stderr.getvalue() self.assertIn(f'Exception ignored in sys.unraisablehook: ' @@ -1295,8 +1515,9 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1332,7 +1553,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1449,15 +1670,19 @@ class C(object): pass # float check(float(0), size('d')) # sys.floatinfo - check(sys.float_info, vsize('') + self.P * len(sys.float_info)) + check(sys.float_info, self.P + vsize('') + self.P * len(sys.float_info)) # frame def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + if support.Py_GIL_DISABLED: + INTERPRETER_FRAME = '9PihcP' + else: + INTERPRETER_FRAME = '9PhcP' + check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('14Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): @@ -1471,7 +1696,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('P2P4P4c7P2ic??P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator @@ -1499,7 +1724,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1514,9 +1742,10 @@ def delx(self): del self.__x x = property(getx, setx, delx, "") check(x, size('5Pi')) # PyCapsule - # XXX + check(_datetime.datetime_CAPI, size('6P')) # rangeiterator - check(iter(range(1)), size('4l')) + check(iter(range(1)), size('3l')) + check(iter(range(2**65)), size('3P')) # reverse check(reversed(''), size('nP')) # range @@ -1549,13 +1778,14 @@ def delx(self): del self.__x # super check(super(int), size('3P')) # tuple - check((), vsize('')) - check((1,2,3), vsize('') + 3*self.P) + check((), vsize('') + self.P) + check((1,2,3), vsize('') + self.P + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIP' - s = vsize('2P' + fmt) + fmt = 'P2nPI13Pl4Pn9Pn12PIPc' + s = vsize(fmt) check(int, s) + typeid = 'n' if support.Py_GIL_DISABLED else '' # class s = vsize(fmt + # PyTypeObject '4P' # PyAsyncMethods @@ -1563,8 +1793,9 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' - '1P' # Specializer cache + '7P' + '1PIP' # Specializer cache + + typeid # heap type id (free-threaded only) ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries @@ -1586,8 +1817,8 @@ class newstyleclass(object): pass '\u0100'*40, '\uffff'*100, '\U00010000'*30, '\U0010ffff'*100] # also update field definitions in test_unicode.test_raiseMemError - asciifields = "nnbP" - compactfields = asciifields + "nPn" + asciifields = "nnb" + compactfields = asciifields + "nP" unicodefields = compactfields + "P" for s in samples: maxchar = ord(max(s)) @@ -1611,11 +1842,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) @@ -1657,15 +1892,18 @@ def test_pythontypes(self): check(_ast.AST(), size('P')) try: raise TypeError - except TypeError: - tb = sys.exc_info()[2] + except TypeError as e: + tb = e.__traceback__ # traceback if tb is not None: check(tb, size('2P2i')) # symtable entry # XXX # sys.flags - check(sys.flags, vsize('') + self.P * len(sys.flags)) + # FIXME: The +3 is for the 'gil', 'thread_inherit_context' and + # 'context_aware_warnings' flags and will not be necessary once + # gh-122575 is fixed + check(sys.flags, vsize('') + self.P + self.P * (3 + len(sys.flags))) def test_asyncgen_hooks(self): old = sys.get_asyncgen_hooks() @@ -1673,6 +1911,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1680,7 +1933,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1709,5 +1961,318 @@ def write(self, s): self.assertEqual(out, b"") self.assertEqual(err, b"") +@test.support.support_remote_exec_only +@test.support.cpython_only +class TestRemoteExec(unittest.TestCase): + def tearDown(self): + test.support.reap_children() + + def _run_remote_exec_test(self, script_code, python_args=None, env=None, + prologue='', + script_path=os_helper.TESTFN + '_remote.py'): + # Create the script that will be remotely executed + self.addCleanup(os_helper.unlink, script_path) + + with open(script_path, 'w') as f: + f.write(script_code) + + # Create and run the target process + target = os_helper.TESTFN + '_target.py' + self.addCleanup(os_helper.unlink, target) + + port = find_unused_port() + + with open(target, 'w') as f: + f.write(f''' +import sys +import time +import socket + +# Connect to the test process +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(('localhost', {port})) + +{prologue} + +# Signal that the process is ready +sock.sendall(b"ready") + +print("Target process running...") + +# Wait for remote script to be executed +# (the execution will happen as the following +# code is processed as soon as the recv call +# unblocks) +sock.recv(1024) + +# Do a bunch of work to give the remote script time to run +x = 0 +for i in range(100): + x += i + +# Write confirmation back +sock.sendall(b"executed") +sock.close() +''') + + # Start the target process and capture its output + cmd = [sys.executable] + if python_args: + cmd.extend(python_args) + cmd.append(target) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.bind(('localhost', port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + with subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) as proc: + client_socket = None + try: + # Accept connection from target process + client_socket, _ = server_socket.accept() + server_socket.close() + + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + + # Try remote exec on the target process + sys.remote_exec(proc.pid, script_path) + + # Signal script to continue + client_socket.sendall(b"continue") + + # Wait for execution confirmation + response = client_socket.recv(1024) + self.assertEqual(response, b"executed") + + # Return output for test verification + stdout, stderr = proc.communicate(timeout=10.0) + return proc.returncode, stdout, stderr + except PermissionError: + self.skipTest("Insufficient permissions to execute code in remote process") + finally: + if client_socket is not None: + client_socket.close() + proc.kill() + proc.terminate() + proc.wait(timeout=SHORT_TIMEOUT) + + def test_remote_exec(self): + """Test basic remote exec functionality""" + script = 'print("Remote script executed successfully!")' + returncode, stdout, stderr = self._run_remote_exec_test(script) + # self.assertEqual(returncode, 0) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_bytes(self): + script = 'print("Remote script executed successfully!")' + script_path = os.fsencode(os_helper.TESTFN) + b'_bytes_remote.py' + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'requires undecodable path') + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable paths are not supported on macOS') + def test_remote_exec_undecodable(self): + script = 'print("Remote script executed successfully!")' + script_path = os_helper.TESTFN_UNDECODABLE + b'_undecodable_remote.py' + for script_path in [script_path, os.fsdecode(script_path)]: + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_with_self_process(self): + """Test remote exec with the target process being the same as the test process""" + + code = 'import sys;print("Remote script executed successfully!", file=sys.stderr)' + file = os_helper.TESTFN + '_remote_self.py' + with open(file, 'w') as f: + f.write(code) + self.addCleanup(os_helper.unlink, file) + with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + sys.remote_exec(os.getpid(), os.path.abspath(file)) + print("Done") + self.assertEqual(mock_stderr.getvalue(), "Remote script executed successfully!\n") + self.assertEqual(mock_stdout.getvalue(), "Done\n") + + def test_remote_exec_raises_audit_event(self): + """Test remote exec raises an audit event""" + prologue = '''\ +import sys +def audit_hook(event, arg): + print(f"Audit event: {event}, arg: {arg}".encode("ascii", errors="replace")) +sys.addaudithook(audit_hook) +''' + script = ''' +print("Remote script executed successfully!") +''' + returncode, stdout, stderr = self._run_remote_exec_test(script, prologue=prologue) + self.assertEqual(returncode, 0) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertIn(b"Audit event: cpython.remote_debugger_script, arg: ", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_with_exception(self): + """Test remote exec with an exception raised in the target process + + The exception should be raised in the main thread of the target process + but not crash the target process. + """ + script = ''' +raise Exception("Remote script exception") +''' + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertIn(b"Remote script exception", stderr) + self.assertEqual(stdout.strip(), b"Target process running...") + + def test_new_namespace_for_each_remote_exec(self): + """Test that each remote_exec call gets its own namespace.""" + script = textwrap.dedent( + """ + assert globals() is not __import__("__main__").__dict__ + print("Remote script executed successfully!") + """ + ) + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertEqual(stderr, b"") + self.assertIn(b"Remote script executed successfully", stdout) + + def test_remote_exec_disabled_by_env(self): + """Test remote exec is disabled when PYTHON_DISABLE_REMOTE_DEBUG is set""" + env = os.environ.copy() + env['PYTHON_DISABLE_REMOTE_DEBUG'] = '1' + with self.assertRaisesRegex(RuntimeError, "Remote debugging is not enabled in the remote process"): + self._run_remote_exec_test("print('should not run')", env=env) + + def test_remote_exec_disabled_by_xoption(self): + """Test remote exec is disabled with -Xdisable-remote-debug""" + with self.assertRaisesRegex(RuntimeError, "Remote debugging is not enabled in the remote process"): + self._run_remote_exec_test("print('should not run')", python_args=['-Xdisable-remote-debug']) + + def test_remote_exec_invalid_pid(self): + """Test remote exec with invalid process ID""" + with self.assertRaises(OSError): + sys.remote_exec(99999, "print('should not run')") + + def test_remote_exec_invalid_script(self): + """Test remote exec with invalid script type""" + with self.assertRaises(TypeError): + sys.remote_exec(0, None) + with self.assertRaises(TypeError): + sys.remote_exec(0, 123) + + def test_remote_exec_syntax_error(self): + """Test remote exec with syntax error in script""" + script = ''' +this is invalid python code +''' + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertIn(b"SyntaxError", stderr) + self.assertEqual(stdout.strip(), b"Target process running...") + + def test_remote_exec_invalid_script_path(self): + """Test remote exec with invalid script path""" + with self.assertRaises(OSError): + sys.remote_exec(os.getpid(), "invalid_script_path") + + def test_remote_exec_in_process_without_debug_fails_envvar(self): + """Test remote exec in a process without remote debugging enabled""" + script = os_helper.TESTFN + '_remote.py' + self.addCleanup(os_helper.unlink, script) + with open(script, 'w') as f: + f.write('print("Remote script executed successfully!")') + env = os.environ.copy() + env['PYTHON_DISABLE_REMOTE_DEBUG'] = '1' + + _, out, err = assert_python_failure('-c', f'import os, sys; sys.remote_exec(os.getpid(), "{script}")', **env) + self.assertIn(b"Remote debugging is not enabled", err) + self.assertEqual(out, b"") + + def test_remote_exec_in_process_without_debug_fails_xoption(self): + """Test remote exec in a process without remote debugging enabled""" + script = os_helper.TESTFN + '_remote.py' + self.addCleanup(os_helper.unlink, script) + with open(script, 'w') as f: + f.write('print("Remote script executed successfully!")') + + _, out, err = assert_python_failure('-Xdisable-remote-debug', '-c', f'import os, sys; sys.remote_exec(os.getpid(), "{script}")') + self.assertIn(b"Remote debugging is not enabled", err) + self.assertEqual(out, b"") + +class TestSysJIT(unittest.TestCase): + + def test_jit_is_available(self): + available = sys._jit.is_available() + script = f"import sys; assert sys._jit.is_available() is {available}" + assert_python_ok("-c", script, PYTHON_JIT="0") + assert_python_ok("-c", script, PYTHON_JIT="1") + + def test_jit_is_enabled(self): + available = sys._jit.is_available() + script = "import sys; assert sys._jit.is_enabled() is {enabled}" + assert_python_ok("-c", script.format(enabled=False), PYTHON_JIT="0") + assert_python_ok("-c", script.format(enabled=available), PYTHON_JIT="1") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_jit_is_active(self): + available = sys._jit.is_available() + script = textwrap.dedent( + """ + import _testcapi + import _testinternalcapi + import sys + + def frame_0_interpreter() -> None: + assert sys._jit.is_active() is False + + def frame_1_interpreter() -> None: + assert sys._jit.is_active() is False + frame_0_interpreter() + assert sys._jit.is_active() is False + + def frame_2_jit(expected: bool) -> None: + # Inlined into the last loop of frame_3_jit: + assert sys._jit.is_active() is expected + # Insert C frame: + _testcapi.pyobject_vectorcall(frame_1_interpreter, None, None) + assert sys._jit.is_active() is expected + + def frame_3_jit() -> None: + # JITs just before the last loop: + for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # Careful, doing this in the reverse order breaks tracing: + expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + assert sys._jit.is_active() is expected + frame_2_jit(expected) + assert sys._jit.is_active() is expected + + def frame_4_interpreter() -> None: + assert sys._jit.is_active() is False + frame_3_jit() + assert sys._jit.is_active() is False + + assert sys._jit.is_active() is False + frame_4_interpreter() + assert sys._jit.is_active() is False + """ + ) + assert_python_ok("-c", script.format(enabled=False), PYTHON_JIT="0") + assert_python_ok("-c", script.format(enabled=available), PYTHON_JIT="1") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index d5e3206a5ca..d23f5bf6d9b 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -100,8 +100,6 @@ class ProfileHookTestCase(TestCaseBase): def new_watcher(self): return HookWatcher() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): def f(p): pass @@ -110,8 +108,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (112, 'f'))] def test_exception(self): def f(p): 1/0 @@ -120,8 +117,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_exception(self): def f(p): try: 1/0 @@ -131,8 +126,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_nested_exception(self): def f(p): try: 1/0 @@ -142,8 +135,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (138, 'f'))] def test_nested_exception(self): def f(p): 1/0 @@ -155,8 +147,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (1, 'return', (151, 'g'))] def test_exception_in_except_clause(self): def f(p): 1/0 @@ -176,8 +167,7 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (1, 'falling through', (170, 'g'))] def test_exception_propagation(self): def f(p): 1/0 @@ -193,8 +183,7 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (183, 'f'))] def test_raise_twice(self): def f(p): try: 1/0 @@ -204,8 +193,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (192, 'f'))] def test_raise_reraise(self): def f(p): try: 1/0 @@ -215,8 +203,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (201, 'f'))] def test_raise(self): def f(p): raise Exception() @@ -225,8 +212,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (5, 'call', (209, 'f'))] def test_distant_exception(self): def f(): 1/0 @@ -255,8 +241,6 @@ def j(p): (1, 'return', j_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_generator(self): def f(): for i in range(2): @@ -279,8 +263,6 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_stop_iteration(self): def f(): for i in range(2): @@ -307,8 +289,6 @@ class ProfileSimulatorTestCase(TestCaseBase): def new_watcher(self): return ProfileSimulator(self) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): def f(p): pass @@ -317,8 +297,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (293, 'f'))] def test_basic_exception(self): def f(p): 1/0 @@ -327,8 +306,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_exception(self): def f(p): try: 1/0 @@ -338,8 +315,7 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (5, 'call', (310, 'f'))] def test_distant_exception(self): def f(): 1/0 @@ -368,8 +344,6 @@ def j(p): (1, 'return', j_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # bpo-34125: profiling method_descriptor with **kwargs def test_unbound_method(self): kwargs = {} @@ -379,9 +353,8 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34126) + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (348, 'f'))] def test_unbound_method_no_args(self): def f(p): dict.get() @@ -389,9 +362,8 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34126) + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (356, 'f'))] def test_unbound_method_invalid_args(self): def f(p): dict.get(print, 42) @@ -399,9 +371,8 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34125) + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (365, 'f'))] def test_unbound_method_no_keyword_args(self): kwargs = {} def f(p): @@ -410,9 +381,8 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34125) + @unittest.expectedFailure # TODO: RUSTPYTHON; [(1, 'call', (374, 'f'))] def test_unbound_method_invalid_keyword_args(self): kwargs = {} def f(p): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 35e62d54635..82c11bdf7e2 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -447,7 +447,6 @@ def test_main(self): _main() self.assertTrue(len(output.getvalue().split('\n')) > 0) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows") def test_ldshared_value(self): ldflags = sysconfig.get_config_var('LDFLAGS') @@ -599,7 +598,6 @@ def test_android_ext_suffix(self): self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), f"{machine=}, {suffix=}") - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index aa71166a380..372be9eb8c3 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -14,7 +14,7 @@ findfile) from test.support.os_helper import unlink -import unittest # TODO: RUSTPYTHON +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests SOURCE_CODES = { @@ -217,8 +217,7 @@ def test_when_tokenize_tokenerror(self): with self.assertRaises(SystemExit): self.verify_tabnanny_check(file_path, err=err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -232,8 +231,7 @@ def test_when_nannynag_error_verbose(self): tabnanny.verbose = 1 self.verify_tabnanny_check(file_path, out=out) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error(self): """A python source code file eligible for raising `tabnanny.NannyNag`.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: @@ -318,8 +316,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur self.assertListEqual(out.splitlines(), stdout.splitlines()) self.assertListEqual(err.splitlines(), stderr.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Should displays error when errored python file is given. def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: @@ -345,8 +342,7 @@ def test_quiet_flag(self): stdout = f"{file_path}\n" self.validate_cmd("-q", file_path, stdout=stdout) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_verbose_mode(self): """Should display more error information if verbose mode is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: @@ -355,8 +351,7 @@ def test_verbose_mode(self): ).strip() self.validate_cmd("-v", path, stdout=stdout, partial=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_double_verbose_mode(self): """Should display detailed error information if double verbose is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 635a1c1c85a..7a921d569a7 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -456,7 +456,6 @@ def test_premature_end_of_archive(self): with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): tar.extractfile(t).read() - @unittest.skip("TODO: RUSTPYTHON, infinite recursion") def test_length_zero_header(self): # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail # with an exception @@ -1050,19 +1049,15 @@ def _test_sparse_file(self, name): s = os.stat(filename) self.assertLess(s.st_blocks * 512, s.st_size) - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_old(self): self._test_sparse_file("gnu/sparse") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_00(self): self._test_sparse_file("gnu/sparse-0.0") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_01(self): self._test_sparse_file("gnu/sparse-0.1") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_10(self): self._test_sparse_file("gnu/sparse-1.0") @@ -1993,8 +1988,6 @@ class UnicodeTest: def test_iso8859_1_filename(self): self._test_unicode_filename("iso8859-1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utf7_filename(self): self._test_unicode_filename("utf7") @@ -2421,8 +2414,7 @@ def test__all__(self): 'SubsequentHeaderError', 'ExFileObject', 'main'} support.check__all__(self, tarfile, not_exported=not_exported) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; FileNotFoundError: [Errno 2] No such file or directory: '/Users/al03219714/Projects/RustPython3/crates/pylib/Lib/test/testtar.tar.xz' def test_useful_error_message_when_modules_missing(self): fname = os.path.join(os.path.dirname(__file__), 'testtar.tar.xz') with self.assertRaises(tarfile.ReadError) as excinfo: @@ -2498,8 +2490,6 @@ def test_test_command_invalid_file(self): finally: os_helper.unlink(tmpname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_command(self): for tar_name in testtarnames: with support.captured_stdout() as t: @@ -2511,8 +2501,6 @@ def test_list_command(self): PYTHONIOENCODING='ascii') self.assertEqual(out, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_command_verbose(self): for tar_name in testtarnames: with support.captured_stdout() as t: diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 5674839cd43..d389529ca3b 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -11,6 +11,9 @@ import stat import types import weakref +import gc +import shutil +import subprocess from unittest import mock import unittest @@ -60,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -90,14 +87,10 @@ class BaseTestCase(unittest.TestCase): b_check = re.compile(br"^[a-z0-9_-]{8}$") def setUp(self): - self._warnings_manager = warnings_helper.check_warnings() - self._warnings_manager.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.filterwarnings("ignore", category=RuntimeWarning, message="mktemp", module=__name__) - def tearDown(self): - self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] @@ -198,8 +191,7 @@ def supports_iter(self): if i == 20: break - @unittest.skipUnless(hasattr(os, 'fork'), - "os.fork is required for this test") + @support.requires_fork() def test_process_awareness(self): # ensure that the random source differs between # child and parent. @@ -290,19 +282,14 @@ def our_candidate_list(): def raise_OSError(*args, **kwargs): raise OSError() - with support.swap_attr(io, "open", raise_OSError): - # test again with failing io.open() + with support.swap_attr(os, "open", raise_OSError): + # test again with failing os.open() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - def bad_writer(*args, **kwargs): - fp = orig_open(*args, **kwargs) - fp.write = raise_OSError - return fp - - with support.swap_attr(io, "open", bad_writer) as orig_open: - # test again with failing write() + with support.swap_attr(os, "write", raise_OSError): + # test again with failing os.write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) @@ -342,6 +329,9 @@ def _mock_candidate_names(*names): class TestBadTempdir: + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot remove write bits." + ) def test_read_only_directory(self): with _inside_empty_temp_dir(): oldmode = mode = os.stat(tempfile.tempdir).st_mode @@ -447,11 +437,12 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) + @os_helper.skip_unless_working_chmod def test_file_mode(self): # _mkstemp_inner creates files with the proper mode @@ -465,8 +456,8 @@ def test_file_mode(self): expected = user * (1 + 8 + 64) self.assertEqual(mode, expected) - @support.requires_fork() @unittest.skipUnless(has_spawnl, 'os.spawnl not available') + @support.requires_subprocess() def test_noinherit(self): # _mkstemp_inner file handles are not inherited by child processes @@ -684,7 +675,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -785,10 +776,11 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) + @os_helper.skip_unless_working_chmod def test_mode(self): # mkdtemp creates directories with the proper mode @@ -806,6 +798,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists @@ -853,6 +872,15 @@ def test_for_tempdir_is_bytes_issue40701_api_warts(self): finally: tempfile.tempdir = orig_tempdir + def test_path_is_absolute(self): + # Test that the path returned by mkdtemp with a relative `dir` + # argument is absolute + try: + path = tempfile.mkdtemp(dir=".") + self.assertTrue(os.path.isabs(path)) + finally: + os.rmdir(path) + class TestMktemp(BaseTestCase): """Test mktemp().""" @@ -978,6 +1006,7 @@ def test_del_on_close(self): try: with tempfile.NamedTemporaryFile(dir=dir) as f: f.write(b'blat') + self.assertEqual(os.listdir(dir), []) self.assertFalse(os.path.exists(f.name), "NamedTemporaryFile %s exists after close" % f.name) finally: @@ -1017,18 +1046,101 @@ def use_closed(): pass self.assertRaises(ValueError, use_closed) - def test_no_leak_fd(self): - # Issue #21058: don't leak file descriptor when io.open() fails - closed = [] - os_close = os.close - def close(fd): - closed.append(fd) - os_close(fd) + def test_context_man_not_del_on_close_if_delete_on_close_false(self): + # Issue gh-58451: tempfile.NamedTemporaryFile is not particularly useful + # on Windows + # A NamedTemporaryFile is NOT deleted when closed if + # delete_on_close=False, but is deleted on context manager exit + dir = tempfile.mkdtemp() + try: + with tempfile.NamedTemporaryFile(dir=dir, + delete=True, + delete_on_close=False) as f: + f.write(b'blat') + f_name = f.name + f.close() + with self.subTest(): + # Testing that file is not deleted on close + self.assertTrue(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} is incorrectly " + f"deleted on closure when delete_on_close=False") + + with self.subTest(): + # Testing that file is deleted on context manager exit + self.assertFalse(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} exists " + f"after context manager exit") + + finally: + os.rmdir(dir) + + def test_context_man_ok_to_delete_manually(self): + # In the case of delete=True, a NamedTemporaryFile can be manually + # deleted in a with-statement context without causing an error. + dir = tempfile.mkdtemp() + try: + with tempfile.NamedTemporaryFile(dir=dir, + delete=True, + delete_on_close=False) as f: + f.write(b'blat') + f.close() + os.unlink(f.name) + + finally: + os.rmdir(dir) + + def test_context_man_not_del_if_delete_false(self): + # A NamedTemporaryFile is not deleted if delete = False + dir = tempfile.mkdtemp() + f_name = "" + try: + # Test that delete_on_close=True has no effect if delete=False. + with tempfile.NamedTemporaryFile(dir=dir, delete=False, + delete_on_close=True) as f: + f.write(b'blat') + f_name = f.name + self.assertTrue(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} exists after close") + finally: + os.unlink(f_name) + os.rmdir(dir) + + def test_del_by_finalizer(self): + # A NamedTemporaryFile is deleted when finalized in the case of + # delete=True, delete_on_close=False, and no with-statement is used. + def my_func(dir): + f = tempfile.NamedTemporaryFile(dir=dir, delete=True, + delete_on_close=False) + tmp_name = f.name + f.write(b'blat') + # Testing extreme case, where the file is not explicitly closed + # f.close() + return tmp_name + # Make sure that the garbage collector has finalized the file object. + gc.collect() + dir = tempfile.mkdtemp() + try: + tmp_name = my_func(dir) + self.assertFalse(os.path.exists(tmp_name), + f"NamedTemporaryFile {tmp_name!r} " + f"exists after finalizer ") + finally: + os.rmdir(dir) - with mock.patch('os.close', side_effect=close): - with mock.patch('io.open', side_effect=ValueError): - self.assertRaises(ValueError, tempfile.NamedTemporaryFile) - self.assertEqual(len(closed), 1) + def test_correct_finalizer_work_if_already_deleted(self): + # There should be no error in the case of delete=True, + # delete_on_close=False, no with-statement is used, and the file is + # deleted manually. + def my_func(dir)->str: + f = tempfile.NamedTemporaryFile(dir=dir, delete=True, + delete_on_close=False) + tmp_name = f.name + f.write(b'blat') + f.close() + os.unlink(tmp_name) + return tmp_name + # Make sure that the garbage collector has finalized the file object. + gc.collect() def test_bad_mode(self): dir = tempfile.mkdtemp() @@ -1039,6 +1151,24 @@ def test_bad_mode(self): tempfile.NamedTemporaryFile(mode=2, dir=dir) self.assertEqual(os.listdir(dir), []) + def test_bad_encoding(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(LookupError): + tempfile.NamedTemporaryFile('w', encoding='bad-encoding', dir=dir) + self.assertEqual(os.listdir(dir), []) + + def test_unexpected_error(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with mock.patch('tempfile._TemporaryFileWrapper') as mock_ntf, \ + mock.patch('io.open', mock.mock_open()) as mock_open: + mock_ntf.side_effect = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + tempfile.NamedTemporaryFile(dir=dir) + mock_open().close.assert_called() + self.assertEqual(os.listdir(dir), []) + # How to test the mode and bufsize parameters? class TestSpooledTemporaryFile(BaseTestCase): @@ -1059,6 +1189,31 @@ def test_basic(self): f = self.do_create(max_size=100, pre="a", suf=".txt") self.assertFalse(f._rolled) + def test_is_iobase(self): + # SpooledTemporaryFile should implement io.IOBase + self.assertIsInstance(self.do_create(), io.IOBase) + + def test_iobase_interface(self): + # SpooledTemporaryFile should implement the io.IOBase interface. + # Ensure it has all the required methods and properties. + iobase_attrs = { + # From IOBase + 'fileno', 'seek', 'truncate', 'close', 'closed', '__enter__', + '__exit__', 'flush', 'isatty', '__iter__', '__next__', 'readable', + 'readline', 'readlines', 'seekable', 'tell', 'writable', + 'writelines', + # From BufferedIOBase (binary mode) and TextIOBase (text mode) + 'detach', 'read', 'read1', 'write', 'readinto', 'readinto1', + 'encoding', 'errors', 'newlines', + } + spooledtempfile_attrs = set(dir(tempfile.SpooledTemporaryFile)) + missing_attrs = iobase_attrs - spooledtempfile_attrs + self.assertFalse( + missing_attrs, + 'SpooledTemporaryFile missing attributes from ' + 'IOBase/BufferedIOBase/TextIOBase' + ) + def test_del_on_close(self): # A SpooledTemporaryFile is deleted when closed dir = tempfile.mkdtemp() @@ -1069,11 +1224,40 @@ def test_del_on_close(self): self.assertTrue(f._rolled) filename = f.name f.close() - self.assertFalse(isinstance(filename, str) and os.path.exists(filename), - "SpooledTemporaryFile %s exists after close" % filename) + self.assertEqual(os.listdir(dir), []) + if not isinstance(filename, int): + self.assertFalse(os.path.exists(filename), + "SpooledTemporaryFile %s exists after close" % filename) finally: os.rmdir(dir) + def test_del_unrolled_file(self): + # The unrolled SpooledTemporaryFile should raise a ResourceWarning + # when deleted since the file was not explicitly closed. + f = self.do_create(max_size=10) + f.write(b'foo') + self.assertEqual(f.name, None) # Unrolled so no filename/fd + with self.assertWarns(ResourceWarning): + f.__del__() + + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot fstat renamed files." + ) + def test_del_rolled_file(self): + # The rolled file should be deleted when the SpooledTemporaryFile + # object is deleted. This should raise a ResourceWarning since the file + # was not explicitly closed. + f = self.do_create(max_size=2) + f.write(b'foo') + name = f.name # This is a fd on posix+cygwin, a filename everywhere else + self.assertTrue(os.path.exists(name)) + with self.assertWarns(ResourceWarning): + f.__del__() + self.assertFalse( + os.path.exists(name), + "Rolled SpooledTemporaryFile (name=%s) exists after delete" % name + ) + def test_rewrite_small(self): # A SpooledTemporaryFile can be written to multiple within the max_size f = self.do_create(max_size=30) @@ -1104,6 +1288,34 @@ def test_writelines(self): buf = f.read() self.assertEqual(buf, b'xyz') + def test_writelines_rollover(self): + # Verify writelines rolls over before exhausting the iterator + f = self.do_create(max_size=2) + + def it(): + yield b'xy' + self.assertFalse(f._rolled) + yield b'z' + self.assertTrue(f._rolled) + + f.writelines(it()) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'xyz') + + def test_writelines_fast_path(self): + f = self.do_create(max_size=2) + f.write(b'abc') + self.assertTrue(f._rolled) + + f.writelines([b'd', b'e', b'f']) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'abcdef') + + def test_writelines_sequential(self): # A SpooledTemporaryFile should hold exactly max_size bytes, and roll # over afterward @@ -1284,6 +1496,9 @@ def use_closed(): pass self.assertRaises(ValueError, use_closed) + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot fstat renamed files." + ) def test_truncate_with_size_parameter(self): # A SpooledTemporaryFile can be truncated to zero size f = tempfile.SpooledTemporaryFile(max_size=10) @@ -1357,19 +1572,34 @@ def roundtrip(input, *args, **kwargs): roundtrip("\u039B", "w+", encoding="utf-16") roundtrip("foo\r\n", "w+", newline="") - def test_no_leak_fd(self): - # Issue #21058: don't leak file descriptor when io.open() fails - closed = [] - os_close = os.close - def close(fd): - closed.append(fd) - os_close(fd) - - with mock.patch('os.close', side_effect=close): - with mock.patch('io.open', side_effect=ValueError): - self.assertRaises(ValueError, tempfile.TemporaryFile) - self.assertEqual(len(closed), 1) + def test_bad_mode(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(ValueError): + tempfile.TemporaryFile(mode='wr', dir=dir) + with self.assertRaises(TypeError): + tempfile.TemporaryFile(mode=2, dir=dir) + self.assertEqual(os.listdir(dir), []) + + def test_bad_encoding(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(LookupError): + tempfile.TemporaryFile('w', encoding='bad-encoding', dir=dir) + self.assertEqual(os.listdir(dir), []) + def test_unexpected_error(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with mock.patch('tempfile._O_TMPFILE_WORKS', False), \ + mock.patch('os.unlink') as mock_unlink, \ + mock.patch('os.open') as mock_open, \ + mock.patch('os.close') as mock_close: + mock_unlink.side_effect = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + tempfile.TemporaryFile(dir=dir) + mock_close.assert_called() + self.assertEqual(os.listdir(dir), []) # Helper for test_del_on_shutdown @@ -1437,7 +1667,7 @@ def test_explicit_cleanup(self): finally: os.rmdir(dir) - def test_explict_cleanup_ignore_errors(self): + def test_explicit_cleanup_ignore_errors(self): """Test that cleanup doesn't return an error when ignoring them.""" with tempfile.TemporaryDirectory() as working_dir: temp_dir = self.do_create( @@ -1461,6 +1691,28 @@ def test_explict_cleanup_ignore_errors(self): temp_path.exists(), f"TemporaryDirectory {temp_path!s} exists after cleanup") + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_explicit_cleanup_correct_error(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + with open(os.path.join(temp_dir.name, "example.txt"), 'wb'): + # Previously raised NotADirectoryError on some OSes + # (e.g. Windows). See bpo-43153. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_cleanup_with_used_directory(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + subdir = os.path.join(temp_dir.name, "subdir") + os.mkdir(subdir) + with os_helper.change_cwd(subdir): + # Previously raised RecursionError on some OSes + # (e.g. Windows). See bpo-35144. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + @os_helper.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) @@ -1482,6 +1734,104 @@ def test_cleanup_with_symlink_to_a_directory(self): "were deleted") d2.cleanup() + @os_helper.skip_unless_symlink + @unittest.skip('TODO: RUSTPYTHON; No such file or directory "..."') + def test_cleanup_with_symlink_modes(self): + # cleanup() should not follow symlinks when fixing mode bits (#91133) + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chmod(symlink, mode, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chmod(symlink, mode) + except FileNotFoundError: + pass + os.chmod(d1.name, mode) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chmod(file1, mode) + old_mode = os.stat(file1).st_mode + test(file1, target_is_directory=False) + new_mode = os.stat(file1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + with self.subTest('existing dir'): + os.chmod(dir1, mode) + old_mode = os.stat(dir1).st_mode + test(dir1, target_is_directory=True) + new_mode = os.stat(dir1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_flags(self): + # cleanup() should not follow symlinks when fixing flags (#91133) + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chflags(symlink, flags, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chflags(symlink, flags) + except FileNotFoundError: + pass + os.chflags(d1.name, flags) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chflags(file1, flags) + old_flags = os.stat(file1).st_flags + test(file1, target_is_directory=False) + new_flags = os.stat(file1).st_flags + self.assertEqual(new_flags, old_flags) + + with self.subTest('existing dir'): + os.chflags(dir1, flags) + old_flags = os.stat(dir1).st_flags + test(dir1, target_is_directory=True) + new_flags = os.stat(dir1).st_flags + self.assertEqual(new_flags, old_flags) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -1654,9 +2004,27 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) - @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.lchflags') + def check_flags(self, flags): + # skip the test if these flags are not supported (ex: FreeBSD 13) + filename = os_helper.TESTFN + try: + open(filename, "w").close() + try: + os.chflags(filename, flags) + except OSError as exc: + # "OSError: [Errno 45] Operation not supported" + self.skipTest(f"chflags() doesn't support flags " + f"{flags:#b}: {exc}") + else: + os.chflags(filename, 0) + finally: + os_helper.unlink(filename) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') def test_flags(self): flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + d = self.do_create(recurse=3, dirs=2, files=2) with d: # Change files and directories flags recursively. @@ -1667,6 +2035,11 @@ def test_flags(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) + def test_delete_false(self): + with tempfile.TemporaryDirectory(delete=False) as working_dir: + pass + self.assertTrue(os.path.exists(working_dir)) + shutil.rmtree(working_dir) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py new file mode 100644 index 00000000000..e7ebb20b120 --- /dev/null +++ b/Lib/test/test_termios.py @@ -0,0 +1,313 @@ +import errno +import os +import sys +import tempfile +import threading +import unittest +from test import support +from test.support import threading_helper +from test.support.import_helper import import_module + +termios = import_module('termios') + + +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class TestFunctions(unittest.TestCase): + + def setUp(self): + self.master_fd, self.fd = os.openpty() + self.addCleanup(os.close, self.master_fd) + self.stream = self.enterContext(open(self.fd, 'wb', buffering=0)) + tmp = self.enterContext(tempfile.TemporaryFile(mode='wb', buffering=0)) + self.bad_fd = tmp.fileno() + + def assertRaisesTermiosError(self, err, callable, *args): + # Some versions of Android return EACCES when calling termios functions + # on a regular file. + errs = [err] + if sys.platform == 'android' and err == errno.ENOTTY: + errs.append(errno.EACCES) + + with self.assertRaises(termios.error) as cm: + callable(*args) + self.assertIn(cm.exception.args[0], errs) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + def test_tcgetattr(self): + attrs = termios.tcgetattr(self.fd) + self.assertIsInstance(attrs, list) + self.assertEqual(len(attrs), 7) + for i in range(6): + self.assertIsInstance(attrs[i], int) + iflag, oflag, cflag, lflag, ispeed, ospeed, cc = attrs + self.assertIsInstance(cc, list) + self.assertEqual(len(cc), termios.NCCS) + for i, x in enumerate(cc): + if ((lflag & termios.ICANON) == 0 and + (i == termios.VMIN or i == termios.VTIME)): + self.assertIsInstance(x, int) + else: + self.assertIsInstance(x, bytes) + self.assertEqual(len(x), 1) + self.assertEqual(termios.tcgetattr(self.stream), attrs) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcgetattr_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcgetattr, self.bad_fd) + self.assertRaises(ValueError, termios.tcgetattr, -1) + self.assertRaises(OverflowError, termios.tcgetattr, 2**1000) + self.assertRaises(TypeError, termios.tcgetattr, object()) + self.assertRaises(TypeError, termios.tcgetattr) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + def test_tcsetattr(self): + attrs = termios.tcgetattr(self.fd) + termios.tcsetattr(self.fd, termios.TCSANOW, attrs) + termios.tcsetattr(self.fd, termios.TCSADRAIN, attrs) + termios.tcsetattr(self.fd, termios.TCSAFLUSH, attrs) + termios.tcsetattr(self.stream, termios.TCSANOW, attrs) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcsetattr_errors(self): + attrs = termios.tcgetattr(self.fd) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, tuple(attrs)) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1]) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs + [0]) + for i in range(6): + attrs2 = attrs[:] + attrs2[i] = 2**1000 + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[i] = object() + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1] + [attrs[-1][:-1]]) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1] + [attrs[-1] + [b'\0']]) + for i in range(len(attrs[-1])): + attrs2 = attrs[:] + attrs2[-1] = attrs2[-1][:] + attrs2[-1][i] = 2**1000 + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = object() + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = b'' + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = b'\0\0' + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, object()) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + self.assertRaisesTermiosError(errno.EINVAL, termios.tcsetattr, self.fd, -1, attrs) + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, 2**1000, attrs) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, object(), attrs) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsetattr, self.bad_fd, termios.TCSANOW, attrs) + self.assertRaises(ValueError, termios.tcsetattr, -1, termios.TCSANOW, attrs) + self.assertRaises(OverflowError, termios.tcsetattr, 2**1000, termios.TCSANOW, attrs) + self.assertRaises(TypeError, termios.tcsetattr, object(), termios.TCSANOW, attrs) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + @support.skip_android_selinux('tcsendbreak') + def test_tcsendbreak(self): + try: + termios.tcsendbreak(self.fd, 1) + except termios.error as exc: + if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): + self.skipTest('termios.tcsendbreak() is not supported ' + 'with pseudo-terminals (?) on this platform') + raise + termios.tcsendbreak(self.stream, 1) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcsendbreak') + def test_tcsendbreak_errors(self): + self.assertRaises(OverflowError, termios.tcsendbreak, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd, 0.0) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsendbreak, self.bad_fd, 0) + self.assertRaises(ValueError, termios.tcsendbreak, -1, 0) + self.assertRaises(OverflowError, termios.tcsendbreak, 2**1000, 0) + self.assertRaises(TypeError, termios.tcsendbreak, object(), 0) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + @support.skip_android_selinux('tcdrain') + def test_tcdrain(self): + termios.tcdrain(self.fd) + termios.tcdrain(self.stream) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcdrain') + def test_tcdrain_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcdrain, self.bad_fd) + self.assertRaises(ValueError, termios.tcdrain, -1) + self.assertRaises(OverflowError, termios.tcdrain, 2**1000) + self.assertRaises(TypeError, termios.tcdrain, object()) + self.assertRaises(TypeError, termios.tcdrain) + + def test_tcflush(self): + termios.tcflush(self.fd, termios.TCIFLUSH) + termios.tcflush(self.fd, termios.TCOFLUSH) + termios.tcflush(self.fd, termios.TCIOFLUSH) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcflush_errors(self): + self.assertRaisesTermiosError(errno.EINVAL, termios.tcflush, self.fd, -1) + self.assertRaises(OverflowError, termios.tcflush, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcflush, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcflush, self.bad_fd, termios.TCIFLUSH) + self.assertRaises(ValueError, termios.tcflush, -1, termios.TCIFLUSH) + self.assertRaises(OverflowError, termios.tcflush, 2**1000, termios.TCIFLUSH) + self.assertRaises(TypeError, termios.tcflush, object(), termios.TCIFLUSH) + self.assertRaises(TypeError, termios.tcflush, self.fd) + + def test_tcflush_clear_input_or_output(self): + wfd = self.fd + rfd = self.master_fd + # The data is buffered in the input buffer on Linux, and in + # the output buffer on other platforms. + inbuf = sys.platform in ('linux', 'android') + + os.write(wfd, b'abcdef') + self.assertEqual(os.read(rfd, 2), b'ab') + if inbuf: + # don't flush input + termios.tcflush(rfd, termios.TCOFLUSH) + else: + # don't flush output + termios.tcflush(wfd, termios.TCIFLUSH) + self.assertEqual(os.read(rfd, 2), b'cd') + if inbuf: + # flush input + termios.tcflush(rfd, termios.TCIFLUSH) + else: + # flush output + termios.tcflush(wfd, termios.TCOFLUSH) + os.write(wfd, b'ABCDEF') + self.assertEqual(os.read(rfd, 1024), b'ABCDEF') + + @support.skip_android_selinux('tcflow') + def test_tcflow(self): + termios.tcflow(self.fd, termios.TCOOFF) + termios.tcflow(self.fd, termios.TCOON) + termios.tcflow(self.fd, termios.TCIOFF) + termios.tcflow(self.fd, termios.TCION) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcflow') + def test_tcflow_errors(self): + self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) + self.assertRaises(OverflowError, termios.tcflow, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcflow, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcflow, self.bad_fd, termios.TCOON) + self.assertRaises(ValueError, termios.tcflow, -1, termios.TCOON) + self.assertRaises(OverflowError, termios.tcflow, 2**1000, termios.TCOON) + self.assertRaises(TypeError, termios.tcflow, object(), termios.TCOON) + self.assertRaises(TypeError, termios.tcflow, self.fd) + + @support.skip_android_selinux('tcflow') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'only works on Linux') + def test_tcflow_suspend_and_resume_output(self): + wfd = self.fd + rfd = self.master_fd + write_suspended = threading.Event() + write_finished = threading.Event() + + def writer(): + os.write(wfd, b'abc') + self.assertTrue(write_suspended.wait(support.SHORT_TIMEOUT)) + os.write(wfd, b'def') + write_finished.set() + + with threading_helper.start_threads([threading.Thread(target=writer)]): + self.assertEqual(os.read(rfd, 3), b'abc') + try: + try: + termios.tcflow(wfd, termios.TCOOFF) + finally: + write_suspended.set() + self.assertFalse(write_finished.wait(0.5), + 'output was not suspended') + finally: + termios.tcflow(wfd, termios.TCOON) + self.assertTrue(write_finished.wait(support.SHORT_TIMEOUT), + 'output was not resumed') + self.assertEqual(os.read(rfd, 1024), b'def') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcgetwinsize(self): + size = termios.tcgetwinsize(self.fd) + self.assertIsInstance(size, tuple) + self.assertEqual(len(size), 2) + self.assertIsInstance(size[0], int) + self.assertIsInstance(size[1], int) + self.assertEqual(termios.tcgetwinsize(self.stream), size) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcgetwinsize_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcgetwinsize, self.bad_fd) + self.assertRaises(ValueError, termios.tcgetwinsize, -1) + self.assertRaises(OverflowError, termios.tcgetwinsize, 2**1000) + self.assertRaises(TypeError, termios.tcgetwinsize, object()) + self.assertRaises(TypeError, termios.tcgetwinsize) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcsetwinsize(self): + size = termios.tcgetwinsize(self.fd) + termios.tcsetwinsize(self.fd, size) + termios.tcsetwinsize(self.fd, list(size)) + termios.tcsetwinsize(self.stream, size) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcsetwinsize_errors(self): + size = termios.tcgetwinsize(self.fd) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, size[:-1]) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, size + (0,)) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, object()) + self.assertRaises(OverflowError, termios.tcsetwinsize, self.fd, (size[0], 2**1000)) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (size[0], float(size[1]))) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (size[0], object())) + self.assertRaises(OverflowError, termios.tcsetwinsize, self.fd, (2**1000, size[1])) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (float(size[0]), size[1])) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (object(), size[1])) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsetwinsize, self.bad_fd, size) + self.assertRaises(ValueError, termios.tcsetwinsize, -1, size) + self.assertRaises(OverflowError, termios.tcsetwinsize, 2**1000, size) + self.assertRaises(TypeError, termios.tcsetwinsize, object(), size) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd) + + +class TestModule(unittest.TestCase): + def test_constants(self): + self.assertIsInstance(termios.B0, int) + self.assertIsInstance(termios.B38400, int) + self.assertIsInstance(termios.TCSANOW, int) + self.assertIsInstance(termios.TCSADRAIN, int) + self.assertIsInstance(termios.TCSAFLUSH, int) + self.assertIsInstance(termios.TCIFLUSH, int) + self.assertIsInstance(termios.TCOFLUSH, int) + self.assertIsInstance(termios.TCIOFLUSH, int) + self.assertIsInstance(termios.TCOOFF, int) + self.assertIsInstance(termios.TCOON, int) + self.assertIsInstance(termios.TCIOFF, int) + self.assertIsInstance(termios.TCION, int) + self.assertIsInstance(termios.VTIME, int) + self.assertIsInstance(termios.VMIN, int) + self.assertIsInstance(termios.NCCS, int) + self.assertLess(termios.VTIME, termios.NCCS) + self.assertLess(termios.VMIN, termios.NCCS) + + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'termios.error'> is a subclass of <class 'OSError'> + def test_exception(self): + self.assertIsSubclass(termios.error, Exception) + self.assertNotIsSubclass(termios.error, OSError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index dfbc2b93dfc..aca1f427656 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ def test_break_long(self): # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ def test_nobreak_long(self): def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], @@ -765,10 +765,67 @@ def test_subsequent_indent(self): # of IndentTestCase! class DedentTestCase(unittest.TestCase): + def test_type_error(self): + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(0) + + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(b'') + def assertUnchanged(self, text): """assert that dedent() has no effect on 'text'""" self.assertEqual(text, dedent(text)) + def test_dedent_whitespace(self): + # The empty string. + text = "" + self.assertUnchanged(text) + + # Only spaces. + text = " " + expect = "" + self.assertEqual(expect, dedent(text)) + + # Only tabs. + text = "\t\t\t\t" + expect = "" + self.assertEqual(expect, dedent(text)) + + # A mixture. + text = " \t \t\t \t " + expect = "" + self.assertEqual(expect, dedent(text)) + + # ASCII whitespace. + text = "\f\n\r\t\v " + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # One newline. + text = "\n" + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # Windows-style newlines. + text = "\r\n" * 5 + expect = "\n" * 5 + self.assertEqual(expect, dedent(text)) + + # Whitespace mixture. + text = " \n\t\n \n\t\t\n\n\n " + expect = "\n\n\n\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Lines consisting only of whitespace are always normalised + text = "a\n \n\t\n" + expect = "a\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Whitespace characters on non-empty lines are retained + text = "a\r\n\r\n\r\n" + expect = "a\r\n\n\n" + self.assertEqual(expect, dedent(text)) + def test_dedent_nomargin(self): # No lines indented. text = "Hello there.\nHow are you?\nOh good, I'm glad." diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index f55cf3656ea..4ae8a833b99 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -105,7 +105,6 @@ def test_nt_and_posix_stack_size(self): thread.stack_size(0) - @unittest.skip("TODO: RUSTPYTHON, weakref destructors") def test__count(self): # Test the _count() function. orig = thread._count() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 94f21d1c38f..4d799d968a8 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,11 @@ """ import test.support -from test.support import threading_helper +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import force_not_colorized import random import sys @@ -20,11 +21,18 @@ import signal import textwrap import traceback +import warnings from unittest import mock from test import lock_tests from test import support +try: + from test.support import interpreters +except ImportError: + interpreters = None + +threading_helper.requires_working_threading(module=True) # Between fork() and exec(), only async-safe functions are allowed (issues # #12316 and #11870), and fork() from a worker thread is known to trigger @@ -32,8 +40,23 @@ # on platforms known to behave badly. platforms_to_skip = ('netbsd5', 'hp-ux11') -# Is Python built with Py_DEBUG macro defined? -Py_DEBUG = hasattr(sys, 'gettotalrefcount') + +def skip_unless_reliable_fork(test): + if not support.has_fork_support: + return unittest.skip("requires working os.fork()")(test) + if sys.platform in platforms_to_skip: + return unittest.skip("due to known OS bug related to thread+fork")(test) + if support.HAVE_ASAN_FORK_BUG: + return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork")(test) + return test + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) def restore_default_excepthook(testcase): @@ -95,6 +118,7 @@ def tearDown(self): class ThreadTests(BaseTestCase): + maxDiff = 9999 @cpython_only def test_name(self): @@ -123,11 +147,48 @@ def func(): pass thread = threading.Thread(target=func) self.assertEqual(thread.name, "Thread-5 (func)") - @cpython_only - def test_disallow_instantiation(self): - # Ensure that the type disallows instantiation (bpo-43916) - lock = threading.Lock() - test.support.check_disallow_instantiation(self, type(lock)) + def test_args_argument(self): + # bpo-45735: Using list or tuple as *args* in constructor could + # achieve the same effect. + num_list = [1] + num_tuple = (1,) + + str_list = ["str"] + str_tuple = ("str",) + + list_in_tuple = ([1],) + tuple_in_list = [(1,)] + + test_cases = ( + (num_list, lambda arg: self.assertEqual(arg, 1)), + (num_tuple, lambda arg: self.assertEqual(arg, 1)), + (str_list, lambda arg: self.assertEqual(arg, "str")), + (str_tuple, lambda arg: self.assertEqual(arg, "str")), + (list_in_tuple, lambda arg: self.assertEqual(arg, [1])), + (tuple_in_list, lambda arg: self.assertEqual(arg, (1,))) + ) + + for args, target in test_cases: + with self.subTest(target=target, args=args): + t = threading.Thread(target=target, args=args) + t.start() + t.join() + + def test_lock_no_args(self): + threading.Lock() # works + self.assertRaises(TypeError, threading.Lock, 1) + self.assertRaises(TypeError, threading.Lock, a=1) + self.assertRaises(TypeError, threading.Lock, 1, 2, a=1, b=2) + + def test_lock_no_subclass(self): + # Intentionally disallow subclasses of threading.Lock because they have + # never been allowed, so why start now just because the type is public? + with self.assertRaises(TypeError): + class MyLock(threading.Lock): pass + + def test_lock_or_none(self): + import types + self.assertIsInstance(threading.Lock | None, types.UnionType) # Create a bunch of threads, let each do some work, wait until all are # done. @@ -179,8 +240,6 @@ def f(): tid = _thread.start_new_thread(f, ()) done.wait() self.assertEqual(ident[0], tid) - # Kill the "immortal" _DummyThread - del threading._active[ident[0]] # run with a small(ish) thread stack size (256 KiB) def test_various_ops_small_stack(self): @@ -208,11 +267,29 @@ def test_various_ops_large_stack(self): def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. + dummy_thread = None + error = None def f(mutex): - # Calling current_thread() forces an entry for the foreign - # thread to get made in the threading._active map. - threading.current_thread() - mutex.release() + try: + nonlocal dummy_thread + nonlocal error + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + dummy_thread = threading.current_thread() + tid = dummy_thread.ident + self.assertIn(tid, threading._active) + self.assertIsInstance(dummy_thread, threading._DummyThread) + self.assertIs(threading._active.get(tid), dummy_thread) + # gh-29376 + self.assertTrue( + dummy_thread.is_alive(), + 'Expected _DummyThread to be considered alive.' + ) + self.assertIn('_DummyThread', repr(dummy_thread)) + except BaseException as e: + error = e + finally: + mutex.release() mutex = threading.Lock() mutex.acquire() @@ -220,15 +297,29 @@ def f(mutex): tid = _thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() - self.assertIn(tid, threading._active) - self.assertIsInstance(threading._active[tid], threading._DummyThread) - #Issue 29376 - self.assertTrue(threading._active[tid].is_alive()) - self.assertRegex(repr(threading._active[tid]), '_DummyThread') - del threading._active[tid] + if error is not None: + raise error + self.assertEqual(tid, dummy_thread.ident) + # Issue gh-106236: + with self.assertRaises(RuntimeError): + dummy_thread.join() + dummy_thread._started.clear() + with self.assertRaises(RuntimeError): + dummy_thread.is_alive() + # Busy wait for the following condition: after the thread dies, the + # related dummy thread must be removed from threading._active. + timeout = 5 + timeout_at = time.monotonic() + timeout + while time.monotonic() < timeout_at: + if threading._active.get(dummy_thread.ident) is not dummy_thread: + break + time.sleep(.1) + else: + self.fail('It was expected that the created threading._DummyThread was removed from threading._active.') # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. + @cpython_only def test_PyThreadState_SetAsyncExc(self): ctypes = import_module("ctypes") @@ -317,12 +408,13 @@ def run(self): t.join() # else the thread is still running, and we have no way to kill it + @unittest.skip('TODO: RUSTPYTHON; threading._start_new_thread not exposed') def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() - _start_new_thread = threading._start_new_thread - threading._start_new_thread = fail_new_thread + _start_joinable_thread = threading._start_joinable_thread + threading._start_joinable_thread = fail_new_thread try: t = threading.Thread(target=lambda: None) self.assertRaises(threading.ThreadError, t.start) @@ -330,12 +422,17 @@ def fail_new_thread(*args): t in threading._limbo, "Failed to cleanup _limbo map on failure of Thread.start().") finally: - threading._start_new_thread = _start_new_thread + threading._start_joinable_thread = _start_joinable_thread + @unittest.expectedFailure # TODO: RUSTPYTHON; ctypes.pythonapi is not supported def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -368,6 +465,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -390,8 +492,6 @@ def func(frame, event, arg): sys.settrace(func) """) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_join_nondaemon_on_shutdown(self): # Issue 1722344 # Raising SystemExit skipped threading._shutdown @@ -419,7 +519,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() @@ -429,6 +529,48 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) + def test_join_from_multiple_threads(self, size): + # Thread.join() should be thread-safe + errors = [] + + def worker(): + time.sleep(0.005) + + def joiner(thread): + try: + thread.join() + except Exception as e: + errors.append(e) + + for N in range(2, 20): + threads = [threading.Thread(target=worker)] + for i in range(N): + threads.append(threading.Thread(target=joiner, + args=(threads[0],))) + for t in threads: + t.start() + time.sleep(0.01) + for t in threads: + t.join() + if errors: + raise errors[0] + + def test_join_with_timeout(self): + lock = _thread.allocate_lock() + lock.acquire() + + def worker(): + lock.acquire() + + thread = threading.Thread(target=worker) + thread.start() + thread.join(timeout=0.01) + assert thread.is_alive() + lock.release() + thread.join() + assert not thread.is_alive() + def test_no_refcycle_through_target(self): class RunSelfFunction(object): def __init__(self, should_raise): @@ -507,41 +649,12 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, exit_handler needs lock._at_fork_reinit') - @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') - def test_fork_at_exit(self): - # bpo-42350: Calling os.fork() after threading._shutdown() must - # not log an error. - code = textwrap.dedent(""" - import atexit - import os - import sys - from test.support import wait_process - - # Import the threading module to register its "at fork" callback - import threading - - def exit_handler(): - pid = os.fork() - if not pid: - print("child process ok", file=sys.stderr, flush=True) - # child process - else: - wait_process(pid, exitcode=0) - - # exit_handler() will be called after threading._shutdown() - atexit.register(exit_handler) - """) - _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') - self.assertEqual(err.rstrip(), b'child process ok') - - @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + @skip_unless_reliable_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. code = """if 1: - import _thread, threading, os, time + import _thread, threading, os, time, warnings def background_thread(evt): # Creates and registers the _DummyThread instance @@ -553,18 +666,23 @@ def background_thread(evt): _thread.start_new_thread(background_thread, (evt,)) evt.wait() assert threading.active_count() == 2, threading.active_count() - if os.fork() == 0: - assert threading.active_count() == 1, threading.active_count() - os._exit(0) - else: - os.wait() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + os.wait() """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, b'') self.assertEqual(err, b'') - @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "TODO: RUSTPYTHON, needs sys.getswitchinterval()") - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skip('TODO: RUSTPYTHON; flaky') + @skip_unless_reliable_fork def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on # threads that vanished after a fork. @@ -577,13 +695,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - pid = os.fork() - if pid == 0: - os._exit(11 if t.is_alive() else 10) - else: - t.join() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (pid := os.fork()) == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() - support.wait_process(pid, exitcode=10) + support.wait_process(pid, exitcode=10) def test_main_thread(self): main = threading.main_thread() @@ -598,48 +718,60 @@ def f(): th.start() th.join() - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): code = """if 1: import os, threading from test import support + ident = threading.get_ident() pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) else: support.wait_process(pid, exitcode=0) """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err, b"") - self.assertEqual(data, "MainThread\nTrue\nTrue\n") - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + self.assertEqual(data, + "current ident True\n" + "main MainThread\n" + "main ident True\n" + "current is main True\n") + + @unittest.skip("TODO: RUSTPYTHON flaky; process timeout after fork") + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") - @unittest.skipIf(os.name != 'posix', "test needs POSIX semantics") def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: - import os, threading, sys + import os, threading, sys, warnings from test import support def func(): - pid = os.fork() - if pid == 0: - main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) - # stdout is fully buffered because not a tty, - # we have to flush before exit. - sys.stdout.flush() - else: - support.wait_process(pid, exitcode=0) + ident = threading.get_ident() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + pid = os.fork() + if pid == 0: + print("current ident", threading.get_ident() == ident) + main = threading.main_thread() + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + support.wait_process(pid, exitcode=0) th = threading.Thread(target=func) th.start() @@ -647,11 +779,82 @@ def func(): """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') - self.assertEqual(err, b"") - self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") + self.assertEqual(err.decode('utf-8'), "") + self.assertEqual(data, + "current ident True\n" + "main Thread-1 (func) Thread\n" + "main ident True\n" + "current is main True\n" + ) + + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): + code = """if 1: + import os, threading, sys, traceback, _thread + from test import support - # TODO: RUSTPYTHON - @unittest.expectedFailure + def func(lock): + ident = threading.get_ident() + if %s: + # call current_thread() before fork to allocate DummyThread + current = threading.current_thread() + print("current", current.name, type(current).__name__) + print("ident in _active", ident in threading._active) + # flush before fork, so child won't flush it again + sys.stdout.flush() + pid = os.fork() + if pid == 0: + print("current ident", threading.get_ident() == ident) + main = threading.main_thread() + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + print("_dangling", [t.name for t in list(threading._dangling)]) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + try: + threading._shutdown() + os._exit(0) + except: + traceback.print_exc() + sys.stderr.flush() + os._exit(1) + else: + try: + support.wait_process(pid, exitcode=0) + except Exception: + # avoid 'could not acquire lock for + # <_io.BufferedWriter name='<stderr>'> at interpreter shutdown,' + traceback.print_exc() + sys.stderr.flush() + finally: + lock.release() + + join_lock = _thread.allocate_lock() + join_lock.acquire() + th = _thread.start_new_thread(func, (join_lock,)) + join_lock.acquire() + """ % create_dummy + # "DeprecationWarning: This process is multi-threaded, use of fork() + # may lead to deadlocks in the child" + _, out, err = assert_python_ok("-W", "ignore::DeprecationWarning", "-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err.decode(), "") + self.assertEqual(data, + ("current Dummy-1 _DummyThread\n" if create_dummy else "") + + f"ident in _active {create_dummy!s}\n" + + "current ident True\n" + "main MainThread _MainThread\n" + "main ident True\n" + "current is main True\n" + "_dangling ['MainThread']\n") + + def test_main_thread_after_fork_from_dummy_thread(self, create_dummy=False): + self.test_main_thread_after_fork_from_foreign_thread(create_dummy=True) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread # at shutdown @@ -716,41 +919,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -798,6 +966,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -825,7 +994,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -835,6 +1003,7 @@ def callback(): callback() finally: sys.settrace(old_trace) + threading.settrace(old_trace) def test_gettrace(self): def noop_trace(frame, event, arg): @@ -848,6 +1017,36 @@ def noop_trace(frame, event, arg): finally: threading.settrace(old_trace) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_gettrace_all_threads(self): + def fn(*args): pass + old_trace = threading.gettrace() + first_check = threading.Event() + second_check = threading.Event() + + trace_funcs = [] + def checker(): + trace_funcs.append(sys.gettrace()) + first_check.set() + second_check.wait() + trace_funcs.append(sys.gettrace()) + + try: + t = threading.Thread(target=checker) + t.start() + first_check.wait() + threading.settrace_all_threads(fn) + second_check.set() + t.join() + self.assertEqual(trace_funcs, [None, fn]) + self.assertEqual(threading.gettrace(), fn) + self.assertEqual(sys.gettrace(), fn) + finally: + threading.settrace_all_threads(old_trace) + + self.assertEqual(threading.gettrace(), old_trace) + self.assertEqual(sys.gettrace(), old_trace) + def test_getprofile(self): def fn(*args): pass old_profile = threading.getprofile() @@ -857,32 +1056,37 @@ def fn(*args): pass finally: threading.setprofile(old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_getprofile_all_threads(self): + def fn(*args): pass + old_profile = threading.getprofile() + first_check = threading.Event() + second_check = threading.Event() - # unblock the thread and join it - event.set() - thread.join() + profile_funcs = [] + def checker(): + profile_funcs.append(sys.getprofile()) + first_check.set() + second_check.wait() + profile_funcs.append(sys.getprofile()) - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) + try: + t = threading.Thread(target=checker) + t.start() + first_check.wait() + threading.setprofile_all_threads(fn) + second_check.set() + t.join() + self.assertEqual(profile_funcs, [None, fn]) + self.assertEqual(threading.getprofile(), fn) + self.assertEqual(sys.getprofile(), fn) + finally: + threading.setprofile_all_threads(old_profile) - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(threading.getprofile(), old_profile) + self.assertEqual(sys.getprofile(), old_profile) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called @@ -927,16 +1131,6 @@ def noop(): pass threading.Thread(target=noop).start() # Thread.join() is not called - @unittest.skipUnless(Py_DEBUG, 'need debug build (Py_DEBUG)') - def test_debug_deprecation(self): - # bpo-44584: The PYTHONTHREADDEBUG environment variable is deprecated - rc, out, err = assert_python_ok("-Wdefault", "-c", "pass", - PYTHONTHREADDEBUG="1") - msg = (b'DeprecationWarning: The threading debug ' - b'(PYTHONTHREADDEBUG environment variable) ' - b'is deprecated and will be removed in Python 3.12') - self.assertIn(msg, err) - def test_import_from_another_thread(self): # bpo-1596321: If the threading module is first import from a thread # different than the main thread, threading._shutdown() must handle @@ -970,6 +1164,164 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') + # TODO: RUSTPYTHON - __del__ not called during interpreter finalization (no cyclic GC) + @unittest.expectedFailure + def test_start_new_thread_at_finalization(self): + code = """if 1: + import _thread + + def f(): + print("shouldn't be printed") + + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out.strip(), b"OK") + self.assertIn(b"can't create new thread at interpreter shutdown", err) + + def test_start_new_thread_failed(self): + # gh-109746: if Python fails to start newly created thread + # due to failure of underlying PyThread_start_new_thread() call, + # its state should be removed from interpreter' thread states list + # to avoid its double cleanup + try: + from resource import setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + code = """if 1: + import resource + import _thread + + def f(): + print("shouldn't be printed") + + limits = resource.getrlimit(resource.RLIMIT_NPROC) + [_, hard] = limits + resource.setrlimit(resource.RLIMIT_NPROC, (0, hard)) + + try: + handle = _thread.start_joinable_thread(f) + except RuntimeError: + print('ok') + else: + print('!skip!') + handle.join() + """ + _, out, err = assert_python_ok("-u", "-c", code) + out = out.strip() + if b'!skip!' in out: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + self.assertEqual(out, b'ok') + self.assertEqual(err, b'') + + + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(threading, 'get_native_id'), "test needs threading.get_native_id()") + def test_native_id_after_fork(self): + script = """if True: + import threading + import os + from test import support + + parent_thread_native_id = threading.current_thread().native_id + print(parent_thread_native_id, flush=True) + assert parent_thread_native_id == threading.get_native_id() + childpid = os.fork() + if childpid == 0: + print(threading.current_thread().native_id, flush=True) + assert threading.current_thread().native_id == threading.get_native_id() + else: + try: + assert parent_thread_native_id == threading.current_thread().native_id + assert parent_thread_native_id == threading.get_native_id() + finally: + support.wait_process(childpid, exitcode=0) + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + native_ids = out.strip().splitlines() + self.assertEqual(len(native_ids), 2) + self.assertNotEqual(native_ids[0], native_ids[1]) + + @cpython_only + def test_finalize_daemon_thread_hang(self): + if support.check_sanitizer(thread=True, memory=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest( + "https://github.com/python/cpython/issues/124878 - Known" + " race condition that TSAN identifies.") + # gh-87135: tests that daemon threads hang during finalization + script = textwrap.dedent(''' + import os + import sys + import threading + import time + import _testcapi + + lock = threading.Lock() + lock.acquire() + thread_started_event = threading.Event() + def thread_func(): + try: + thread_started_event.set() + _testcapi.finalize_thread_hang(lock.acquire) + finally: + # Control must not reach here. + os._exit(2) + + t = threading.Thread(target=thread_func) + t.daemon = True + t.start() + thread_started_event.wait() + # Sleep to ensure daemon thread is blocked on `lock.acquire` + # + # Note: This test is designed so that in the unlikely case that + # `0.1` seconds is not sufficient time for the thread to become + # blocked on `lock.acquire`, the test will still pass, it just + # won't be properly testing the thread behavior during + # finalization. + time.sleep(0.1) + + def run_during_finalization(): + # Wake up daemon thread + lock.release() + # Sleep to give the daemon thread time to crash if it is going + # to. + # + # Note: If due to an exceptionally slow execution this delay is + # insufficient, the test will still pass but will simply be + # ineffective as a test. + time.sleep(0.1) + # If control reaches here, the test succeeded. + os._exit(0) + + # Replace sys.stderr.flush as a way to run code during finalization + orig_flush = sys.stderr.flush + def do_flush(*args, **kwargs): + orig_flush(*args, **kwargs) + if not sys.is_finalizing: + return + sys.stderr.flush = orig_flush + run_during_finalization() + + sys.stderr.flush = do_flush + + # If the follow exit code is retained, `run_during_finalization` + # did not run. + sys.exit(1) + ''') + assert_python_ok("-c", script) + + @unittest.skip('TODO: RUSTPYTHON; Thread._tstate_lock not implemented') + def test_tstate_lock(self): + return super().test_tstate_lock() + class ThreadJoinOnShutdown(BaseTestCase): @@ -990,8 +1342,6 @@ def joiningfunc(mainthread): data = out.decode().replace('\r', '') self.assertEqual(data, "end of main\nend of thread\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_1_join_on_shutdown(self): # The usual case: on exit, wait for a non-daemon thread script = """if 1: @@ -1004,10 +1354,7 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - # TODO: RUSTPYTHON need to fix test_1_join_on_shutdown then this might work - @unittest.expectedFailure + @skip_unless_reliable_fork def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: @@ -1027,9 +1374,8 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') + @skip_unless_reliable_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. @@ -1058,10 +1404,16 @@ def worker(): self._run_and_join(script) @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_daemon_threads(self): + @support.bigmemtest(size=40, memuse=70*2**20, dry_run=False) + def test_4_daemon_threads(self, size): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random @@ -1073,8 +1425,9 @@ def test_4_daemon_threads(self): def random_io(): '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + import test.test_threading as mod while True: - with open(os.__file__, 'rb') as in_f: + with open(mod.__file__, 'rb') as in_f: stuff = in_f.read(200) with open(os.devnull, 'wb') as null_f: null_f.write(stuff) @@ -1098,8 +1451,33 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + + # TODO: RUSTPYTHON - parking_lot mutex not fork-safe, child may SIGSEGV + @unittest.skip("TODO: RUSTPYTHON - flaky, parking_lot mutex not fork-safe") + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with # the ad-hoc TLS implementation. @@ -1112,18 +1490,20 @@ def do_fork_and_wait(): else: os._exit(50) - # start a bunch of threads that will fork() child processes - threads = [] - for i in range(16): - t = threading.Thread(target=do_fork_and_wait) - threads.append(t) - t.start() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() - for t in threads: - t.join() + for t in threads: + t.join() - @unittest.skipUnless(hasattr(sys, '_current_frames'), "TODO: RUSTPYTHON, needs sys._current_frames()") - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @skip_unless_reliable_fork def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() @@ -1134,18 +1514,22 @@ def test_clear_threads_states_after_fork(self): threads.append(t) t.start() - pid = os.fork() - if pid == 0: - # check that threads states have been cleared - if len(sys._current_frames()) == 1: - os._exit(51) - else: - os._exit(52) - else: - support.wait_process(pid, exitcode=51) - - for t in threads: - t.join() + try: + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(51) + else: + os._exit(52) + else: + support.wait_process(pid, exitcode=51) + finally: + for t in threads: + t.join() class SubinterpThreadingTests(BaseTestCase): @@ -1157,8 +1541,7 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join(self): # Non-daemon threads should be joined at subinterpreter shutdown # (issue #18808) @@ -1187,8 +1570,7 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join_2(self): # Same as above, but a delay gets introduced after the thread's # Python code returned but before the thread state is deleted. @@ -1226,8 +1608,47 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + @requires_subinterpreters + def test_threads_join_with_no_main(self): + r_interp, w_interp = self.pipe() + + INTERP = b'I' + FINI = b'F' + DONE = b'D' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + os.write({w_interp}, {FINI!r}) + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {DONE!r}) + t = threading.Thread(target=task) + t.start() + + os.write({w_interp}, {INTERP!r}) + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), INTERP) + self.assertEqual(os.read(r_interp, 1), FINI) + self.assertEqual(os.read(r_interp, 1), DONE) + @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1249,6 +1670,67 @@ def f(): self.assertIn("Fatal Python error: Py_EndInterpreter: " "not the last thread", err.decode()) + def _check_allowed(self, before_start='', *, + allowed=True, + daemon_allowed=True, + daemon=False, + ): + import_module("_testinternalcapi") + subinterp_code = textwrap.dedent(f""" + import test.support + import threading + def func(): + print('this should not have run!') + t = threading.Thread(target=func, daemon={daemon}) + {before_start} + t.start() + """) + check_multi_interp_extensions = bool(support.Py_GIL_DISABLED) + script = textwrap.dedent(f""" + import test.support + test.support.run_in_subinterp_with_config( + {subinterp_code!r}, + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads={allowed}, + allow_daemon_threads={daemon_allowed}, + check_multi_interp_extensions={check_multi_interp_extensions}, + own_gil=False, + ) + """) + with test.support.SuppressCrashReport(): + _, _, err = assert_python_ok("-c", script) + return err.decode() + + @cpython_only + def test_threads_not_allowed(self): + err = self._check_allowed( + allowed=False, + daemon_allowed=False, + daemon=False, + ) + self.assertIn('RuntimeError', err) + + @cpython_only + def test_daemon_threads_not_allowed(self): + with self.subTest('via Thread()'): + err = self._check_allowed( + allowed=True, + daemon_allowed=False, + daemon=True, + ) + self.assertIn('RuntimeError', err) + + with self.subTest('via Thread.daemon setter'): + err = self._check_allowed( + 't.daemon = True', + allowed=True, + daemon_allowed=False, + daemon=False, + ) + self.assertIn('RuntimeError', err) + class ThreadingExceptionTests(BaseTestCase): # A RuntimeError should be raised if Thread.start() is called @@ -1277,7 +1759,8 @@ def test_releasing_unacquired_lock(self): lock = threading.Lock() self.assertRaises(RuntimeError, lock.release) - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') + @requires_subprocess() def test_recursion_limit(self): # Issue 9670 # test that excessive recursion within a non-main thread causes @@ -1286,13 +1769,6 @@ def test_recursion_limit(self): # for threads script = """if True: import threading - # TODO: RUSTPYTHON - # Following lines set the recursion limit to previous default of 512 - # for the execution of this process. Without this, the test runners - # on Github fail. Ideally, at a future point this should be removed. - import os, sys - if os.getenv("CI"): - sys.setrecursionlimit(512) def recurse(): return recurse() @@ -1397,6 +1873,37 @@ def run(): self.assertEqual(out, b'') self.assertNotIn("Unhandled exception", err.decode()) + def test_print_exception_gh_102056(self): + # This used to crash. See gh-102056. + script = r"""if True: + import time + import threading + import _thread + + def f(): + try: + f() + except RecursionError: + f() + + def g(): + try: + raise ValueError() + except* ValueError: + f() + + def h(): + time.sleep(1) + _thread.interrupt_main() + + t = threading.Thread(target=h) + t.start() + g() + t.join() + """ + + assert_python_failure("-c", script) + def test_bare_raise_in_brand_new_thread(self): def bare_raise(): raise @@ -1434,6 +1941,23 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + class ThreadRunFail(threading.Thread): def run(self): @@ -1445,6 +1969,7 @@ def setUp(self): restore_default_excepthook(self) super().setUp() + @force_not_colorized def test_excepthook(self): with support.captured_output("stderr") as stderr: thread = ThreadRunFail(name="excepthook thread") @@ -1458,6 +1983,7 @@ def test_excepthook(self): self.assertIn('ValueError: run failed', stderr) @support.cpython_only + @force_not_colorized def test_excepthook_thread_None(self): # threading.excepthook called with thread=None: log the thread # identifier in this case. @@ -1593,32 +2119,51 @@ class PyRLockTests(lock_tests.RLockTests): class CRLockTests(lock_tests.RLockTests): locktype = staticmethod(threading._CRLock) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, flaky test") - def test_different_thread(self): - super().test_different_thread() + def test_signature(self): # gh-102029 + with warnings.catch_warnings(record=True) as warnings_log: + threading.RLock() + self.assertEqual(warnings_log, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + arg_types = [ + ((1,), {}), + ((), {'a': 1}), + ((1, 2), {'a': 1}), + ] + for args, kwargs in arg_types: + with self.subTest(args=args, kwargs=kwargs): + with self.assertWarns(DeprecationWarning): + threading.RLock(*args, **kwargs) + + # Subtypes with custom `__init__` are allowed (but, not recommended): + class CustomRLock(self.locktype): + def __init__(self, a, *, b) -> None: + super().__init__() + + with warnings.catch_warnings(record=True) as warnings_log: + CustomRLock(1, b=2) + self.assertEqual(warnings_log, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_release_save_unacquired(self): - super().test_release_save_unacquired() + return super().test_release_save_unacquired() + + @unittest.skip('TODO: RUSTPYTHON; flaky test') + def test_different_thread(self): + return super().test_different_thread() class EventTests(lock_tests.EventTests): eventtype = staticmethod(threading.Event) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reset_internal_locks(): # TODO: RUSTPYTHON; remove this when done - super().test_reset_internal_locks() - class ConditionAsRLockTests(lock_tests.RLockTests): # Condition uses an RLock by default and exports its API. locktype = staticmethod(threading.Condition) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, flaky test") + def test_recursion_count(self): + self.skipTest("Condition does not expose _recursion_count()") + + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_different_thread(self): - super().test_different_thread() + return super().test_different_thread() class ConditionTests(lock_tests.ConditionTests): condtype = staticmethod(threading.Condition) @@ -1634,8 +2179,6 @@ class BarrierTests(lock_tests.BarrierTests): class MiscTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test__all__(self): restore_default_excepthook(self) @@ -1669,7 +2212,8 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) - @unittest.skip("TODO: RUSTPYTHON; flaky") + @unittest.skip('TODO: RUSTPYTHON; flaky') + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. @@ -1728,8 +2272,6 @@ def worker(started, cont, interrupted): class AtexitTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_output(self): rc, out, err = assert_python_ok("-c", """if True: import threading @@ -1758,8 +2300,6 @@ def test_atexit_called_once(self): self.assertFalse(err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_after_shutdown(self): # The only way to do this is by registering an atexit within # an atexit, which is intended to raise an exception. diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 3443e3875d0..99052de4c7f 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -3,8 +3,8 @@ from doctest import DocTestSuite from test import support from test.support import threading_helper +from test.support.import_helper import import_module import weakref -import gc # Modules under test import _thread @@ -12,6 +12,9 @@ import _threading_local +threading_helper.requires_working_threading(module=True) + + class Weak(object): pass @@ -23,7 +26,7 @@ def target(local, weaklist): class BaseLocalTest: - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_local_refs(self): self._local_refs(20) self._local_refs(50) @@ -182,8 +185,7 @@ class LocalSubclass(self._local): """To test that subclasses behave properly.""" self._test_dict_attribute(LocalSubclass) - # TODO: RUSTPYTHON, cycle detection/collection - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cycle detection/collection def test_cycle_collection(self): class X: pass @@ -197,35 +199,57 @@ class X: self.assertIsNone(wr()) + def test_threading_local_clear_race(self): + # See https://github.com/python/cpython/issues/100892 + + _testcapi = import_module('_testcapi') + _testcapi.call_in_temporary_c_thread(lambda: None, False) + + for _ in range(1000): + _ = threading.local() + + _testcapi.join_temporary_c_thread() + + @support.cpython_only + def test_error(self): + class Loop(self._local): + attr = 1 + + # Trick the "if name == '__dict__':" test of __setattr__() + # to always be true + class NameCompareTrue: + def __eq__(self, other): + return True + + loop = Loop() + with self.assertRaisesRegex(AttributeError, 'Loop.*read-only'): + loop.__setattr__(NameCompareTrue(), 2) + + class ThreadLocalTest(unittest.TestCase, BaseLocalTest): _local = _thread._local - # TODO: RUSTPYTHON, __new__ vs __init__ cooperation - @unittest.expectedFailure - def test_arguments(): - super().test_arguments() - + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by _local + def test_arguments(self): + return super().test_arguments() class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): _local = _threading_local.local -def test_main(): - suite = unittest.TestSuite() - suite.addTest(DocTestSuite('_threading_local')) - suite.addTest(unittest.makeSuite(ThreadLocalTest)) - suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) +def load_tests(loader, tests, pattern): + tests.addTest(DocTestSuite('_threading_local')) local_orig = _threading_local.local def setUp(test): _threading_local.local = _thread._local def tearDown(test): _threading_local.local = local_orig - suite.addTest(DocTestSuite('_threading_local', - setUp=setUp, tearDown=tearDown) - ) + tests.addTests(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + return tests - support.run_unittest(suite) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py new file mode 100644 index 00000000000..bf241ada90e --- /dev/null +++ b/Lib/test/test_threadsignals.py @@ -0,0 +1,237 @@ +"""PyUnit testing that threads honor our signal semantics""" + +import unittest +import signal +import os +import sys +from test.support import threading_helper +import _thread as thread +import time + +if (sys.platform[:3] == 'win'): + raise unittest.SkipTest("Can't test signal on %s" % sys.platform) + +process_pid = os.getpid() +signalled_all=thread.allocate_lock() + +USING_PTHREAD_COND = (sys.thread_info.name == 'pthread' + and sys.thread_info.lock == 'mutex+cond') + +def registerSignals(for_usr1, for_usr2, for_alrm): + usr1 = signal.signal(signal.SIGUSR1, for_usr1) + usr2 = signal.signal(signal.SIGUSR2, for_usr2) + alrm = signal.signal(signal.SIGALRM, for_alrm) + return usr1, usr2, alrm + + +# The signal handler. Just note that the signal occurred and +# from who. +def handle_signals(sig,frame): + signal_blackboard[sig]['tripped'] += 1 + signal_blackboard[sig]['tripped_by'] = thread.get_ident() + +# a function that will be spawned as a separate thread. +def send_signals(): + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) + signalled_all.release() + + +@threading_helper.requires_working_threading() +class ThreadSignals(unittest.TestCase): + + def test_signals(self): + with threading_helper.wait_threads_exit(): + # Test signal handling semantics of threads. + # We spawn a thread, have the thread send itself two signals, and + # wait for it to finish. Check that we got both signals + # and that they were run by the main thread. + signalled_all.acquire() + self.spawnSignallingThread() + signalled_all.acquire() + + self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) + self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], + thread.get_ident()) + self.assertEqual( signal_blackboard[signal.SIGUSR2]['tripped'], 1) + self.assertEqual( signal_blackboard[signal.SIGUSR2]['tripped_by'], + thread.get_ident()) + signalled_all.release() + + def spawnSignallingThread(self): + thread.start_new_thread(send_signals, ()) + + def alarm_interrupt(self, sig, frame): + raise KeyboardInterrupt + + @unittest.skipIf(USING_PTHREAD_COND, + 'POSIX condition variables cannot be interrupted') + @unittest.skipIf(sys.platform.startswith('linux') and + not sys.thread_info.version, + 'Issue 34004: musl does not allow interruption of locks ' + 'by signals.') + # Issue #20564: sem_timedwait() cannot be interrupted on OpenBSD + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'lock cannot be interrupted on OpenBSD') + def test_lock_acquire_interruption(self): + # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck + # in a deadlock. + # XXX this test can fail when the legacy (non-semaphore) implementation + # of locks is used in thread_pthread.h, see issue #11223. + oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + try: + lock = thread.allocate_lock() + lock.acquire() + signal.alarm(1) + t1 = time.monotonic() + self.assertRaises(KeyboardInterrupt, lock.acquire, timeout=5) + dt = time.monotonic() - t1 + # Checking that KeyboardInterrupt was raised is not sufficient. + # We want to assert that lock.acquire() was interrupted because + # of the signal, not that the signal handler was called immediately + # after timeout return of lock.acquire() (which can fool assertRaises). + self.assertLess(dt, 3.0) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, oldalrm) + + @unittest.skipIf(USING_PTHREAD_COND, + 'POSIX condition variables cannot be interrupted') + @unittest.skipIf(sys.platform.startswith('linux') and + not sys.thread_info.version, + 'Issue 34004: musl does not allow interruption of locks ' + 'by signals.') + # Issue #20564: sem_timedwait() cannot be interrupted on OpenBSD + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'lock cannot be interrupted on OpenBSD') + def test_rlock_acquire_interruption(self): + # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck + # in a deadlock. + # XXX this test can fail when the legacy (non-semaphore) implementation + # of locks is used in thread_pthread.h, see issue #11223. + oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + try: + rlock = thread.RLock() + # For reentrant locks, the initial acquisition must be in another + # thread. + def other_thread(): + rlock.acquire() + + with threading_helper.wait_threads_exit(): + thread.start_new_thread(other_thread, ()) + # Wait until we can't acquire it without blocking... + while rlock.acquire(blocking=False): + rlock.release() + time.sleep(0.01) + signal.alarm(1) + t1 = time.monotonic() + self.assertRaises(KeyboardInterrupt, rlock.acquire, timeout=5) + dt = time.monotonic() - t1 + # See rationale above in test_lock_acquire_interruption + self.assertLess(dt, 3.0) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, oldalrm) + + def acquire_retries_on_intr(self, lock): + self.sig_recvd = False + def my_handler(signal, frame): + self.sig_recvd = True + + old_handler = signal.signal(signal.SIGUSR1, my_handler) + try: + def other_thread(): + # Acquire the lock in a non-main thread, so this test works for + # RLocks. + lock.acquire() + # Wait until the main thread is blocked in the lock acquire, and + # then wake it up with this. + time.sleep(0.5) + os.kill(process_pid, signal.SIGUSR1) + # Let the main thread take the interrupt, handle it, and retry + # the lock acquisition. Then we'll let it run. + time.sleep(0.5) + lock.release() + + with threading_helper.wait_threads_exit(): + thread.start_new_thread(other_thread, ()) + # Wait until we can't acquire it without blocking... + while lock.acquire(blocking=False): + lock.release() + time.sleep(0.01) + result = lock.acquire() # Block while we receive a signal. + self.assertTrue(self.sig_recvd) + self.assertTrue(result) + finally: + signal.signal(signal.SIGUSR1, old_handler) + + def test_lock_acquire_retries_on_intr(self): + self.acquire_retries_on_intr(thread.allocate_lock()) + + def test_rlock_acquire_retries_on_intr(self): + self.acquire_retries_on_intr(thread.RLock()) + + def test_interrupted_timed_acquire(self): + # Test to make sure we recompute lock acquisition timeouts when we + # receive a signal. Check this by repeatedly interrupting a lock + # acquire in the main thread, and make sure that the lock acquire times + # out after the right amount of time. + # NOTE: this test only behaves as expected if C signals get delivered + # to the main thread. Otherwise lock.acquire() itself doesn't get + # interrupted and the test trivially succeeds. + self.start = None + self.end = None + self.sigs_recvd = 0 + done = thread.allocate_lock() + done.acquire() + lock = thread.allocate_lock() + lock.acquire() + def my_handler(signum, frame): + self.sigs_recvd += 1 + old_handler = signal.signal(signal.SIGUSR1, my_handler) + try: + def timed_acquire(): + self.start = time.monotonic() + lock.acquire(timeout=0.5) + self.end = time.monotonic() + def send_signals(): + for _ in range(40): + time.sleep(0.02) + os.kill(process_pid, signal.SIGUSR1) + done.release() + + with threading_helper.wait_threads_exit(): + # Send the signals from the non-main thread, since the main thread + # is the only one that can process signals. + thread.start_new_thread(send_signals, ()) + timed_acquire() + # Wait for thread to finish + done.acquire() + # This allows for some timing and scheduling imprecision + self.assertLess(self.end - self.start, 2.0) + self.assertGreater(self.end - self.start, 0.3) + # If the signal is received several times before PyErr_CheckSignals() + # is called, the handler will get called less than 40 times. Just + # check it's been called at least once. + self.assertGreater(self.sigs_recvd, 0) + finally: + signal.signal(signal.SIGUSR1, old_handler) + + +def setUpModule(): + global signal_blackboard + + signal_blackboard = { signal.SIGUSR1 : {'tripped': 0, 'tripped_by': 0 }, + signal.SIGUSR2 : {'tripped': 0, 'tripped_by': 0 }, + signal.SIGALRM : {'tripped': 0, 'tripped_by': 0 } } + + oldsigs = registerSignals(handle_signals, handle_signals, handle_signals) + unittest.addModuleCleanup(registerSignals, *oldsigs) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 31a2a920d9e..f66a37edb4c 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -2,7 +2,6 @@ from test.support import warnings_helper import decimal import enum -import locale import math import platform import sys @@ -14,8 +13,12 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport # Max year is only limited by the size of C int. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 @@ -38,6 +41,10 @@ class _PyTime(enum.IntEnum): # Round away from zero ROUND_UP = 3 +# _PyTime_t is int64_t +PyTime_MIN = -2 ** 63 +PyTime_MAX = 2 ** 63 - 1 + # Rounding modes supported by PyTime ROUNDING_MODES = ( # (PyTime rounding method, decimal rounding method) @@ -53,8 +60,6 @@ class TimeTestCase(unittest.TestCase): def setUp(self): self.t = time.time() - # TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute 'altzone' - @unittest.expectedFailure def test_data_attributes(self): time.altzone time.daylight @@ -111,6 +116,7 @@ def test_clock_monotonic(self): 'need time.pthread_getcpuclockid()') @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') + @unittest.skipIf(support.is_emscripten, "Fails to find clock") def test_pthread_getcpuclockid(self): clk_id = time.pthread_getcpuclockid(threading.get_ident()) self.assertTrue(type(clk_id) is int) @@ -152,13 +158,32 @@ def test_conversions(self): self.assertEqual(int(time.mktime(time.localtime(self.t))), int(self.t)) - def test_sleep(self): + def test_sleep_exceptions(self): + self.assertRaises(TypeError, time.sleep, []) + self.assertRaises(TypeError, time.sleep, "a") + self.assertRaises(TypeError, time.sleep, complex(0, 0)) + self.assertRaises(ValueError, time.sleep, -2) self.assertRaises(ValueError, time.sleep, -1) - time.sleep(1.2) + self.assertRaises(ValueError, time.sleep, -0.1) + + # Improved exception #81267 + with self.assertRaises(TypeError) as errmsg: + time.sleep([]) + self.assertIn("integer or float", str(errmsg.exception)) + + def test_sleep(self): + for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: + with self.subTest(value=value): + time.sleep(value) + + def test_epoch(self): + # bpo-43869: Make sure that Python use the same Epoch on all platforms: + # January 1, 1970, 00:00:00 (UTC). + epoch = time.gmtime(0) + # Only test the date and time, ignore other gmtime() members + self.assertEqual(tuple(epoch)[:6], (1970, 1, 1, 0, 0, 0), epoch) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strftime(self): tt = time.gmtime(self.t) for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I', @@ -171,8 +196,44 @@ def test_strftime(self): self.fail('conversion specifier: %r failed.' % format) self.assertRaises(TypeError, time.strftime, b'%S', tt) - # embedded null character - self.assertRaises(ValueError, time.strftime, '%S\0', tt) + + def test_strftime_invalid_format(self): + tt = time.gmtime(self.t) + with SuppressCrashReport(): + for i in range(1, 128): + format = ' %' + chr(i) + with self.subTest(format=format): + try: + time.strftime(format, tt) + except ValueError as exc: + self.assertEqual(str(exc), 'Invalid format string') + + def test_strftime_special(self): + tt = time.gmtime(self.t) + s1 = time.strftime('%c', tt) + s2 = time.strftime('%B', tt) + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(time.strftime('\U0001f40d', tt), '\U0001f40d') + self.assertEqual(time.strftime('\U0001f4bb%c\U0001f40d%B', tt), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(time.strftime('%c\U0001f4bb%B\U0001f40d', tt), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(time.strftime('\ud83d', tt), '\ud83d') + self.assertEqual(time.strftime('\udc0d', tt), '\udc0d') + self.assertEqual(time.strftime('\ud83d%c\udc0d%B', tt), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(time.strftime('%c\ud83d%B\udc0d', tt), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(time.strftime('%c\udc0d%B\ud83d', tt), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(time.strftime('\ud83d\udc0d', tt), '\ud83d\udc0d') + self.assertEqual(time.strftime('%c\ud83d\udc0d%B', tt), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(time.strftime('\udcf0\udc9f\udc90\udc8d', tt), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(time.strftime('%c\udcf0\udc9f\udc90\udc8d%B', tt), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(time.strftime('\0', tt), '\0') + self.assertEqual(time.strftime('\0'*1000, tt), '\0'*1000) + self.assertEqual(time.strftime('\0%c\0%B', tt), f'\0{s1}\0{s2}') + self.assertEqual(time.strftime('%c\0%B\0', tt), f'{s1}\0{s2}\0') def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts @@ -231,8 +292,6 @@ def _bounds_checking(self, func): self.assertRaises(ValueError, func, (1900, 1, 1, 0, 0, 0, 0, 367, -1)) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_strftime_bounding_check(self): self._bounds_checking(lambda tup: time.strftime('', tup)) @@ -249,8 +308,6 @@ def test_strftime_format_check(self): except ValueError: pass - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_default_values_for_zero(self): # Make sure that using all zeros uses the proper default # values. No test for daylight savings since strftime() does @@ -261,8 +318,6 @@ def test_default_values_for_zero(self): result = time.strftime("%Y %m %d %H %M %S %w %j", (2000,)+(0,)*8) self.assertEqual(expected, result) - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_if_buggy_ucrt_strfptime def test_strptime(self): # Should be able to go round-trip from strftime to strptime without @@ -272,6 +327,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -288,14 +345,18 @@ def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: time.strptime('', '%D') - self.assertIs(e.exception.__suppress_context__, True) - # additional check for IndexError branch (issue #19545) + self.assertTrue(e.exception.__suppress_context__) + # additional check for stray % branch with self.assertRaises(ValueError) as e: - time.strptime('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + time.strptime('%', '%') + self.assertTrue(e.exception.__suppress_context__) + + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_asctime(self): time.asctime(time.gmtime(self.t)) @@ -311,13 +372,9 @@ def test_asctime(self): self.assertRaises(TypeError, time.asctime, ()) self.assertRaises(TypeError, time.asctime, (0,) * 10) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_asctime_bounding_check(self): self._bounds_checking(time.asctime) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ctime(self): t = time.mktime((1973, 9, 16, 1, 3, 52, 0, 0, -1)) self.assertEqual(time.ctime(t), 'Sun Sep 16 01:03:52 1973') @@ -417,8 +474,6 @@ def test_insane_timestamps(self): for unreasonable in -1e200, 1e200: self.assertRaises(OverflowError, func, unreasonable) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_ctime_without_arg(self): # Not sure how to check the values, since the clock could tick # at any time. Make sure these are at least accepted and @@ -426,8 +481,6 @@ def test_ctime_without_arg(self): time.ctime() time.ctime(None) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_gmtime_without_arg(self): gt0 = time.gmtime() gt1 = time.gmtime(None) @@ -435,8 +488,6 @@ def test_gmtime_without_arg(self): t1 = time.mktime(gt1) self.assertAlmostEqual(t1, t0, delta=0.2) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_localtime_without_arg(self): lt0 = time.localtime() lt1 = time.localtime(None) @@ -471,8 +522,6 @@ def test_mktime_error(self): pass self.assertEqual(time.strftime('%Z', tt), tzname) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "win32", "Implement get_clock_info for Windows.") def test_monotonic(self): # monotonic() should not go backward times = [time.monotonic() for n in range(100)] @@ -499,6 +548,12 @@ def test_monotonic(self): def test_perf_counter(self): time.perf_counter() + @unittest.skipIf( + support.is_wasi, "process_time not available on WASI" + ) + @unittest.skipIf( + support.is_emscripten, "process_time present but doesn't exclude sleep" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() @@ -514,7 +569,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: @@ -522,11 +577,10 @@ def test_thread_time(self): # thread_time() should not include time spend during a sleep start = time.thread_time() - time.sleep(0.100) + time.sleep(0.200) stop = time.thread_time() - # use 20 ms because thread_time() has usually a resolution of 15 ms - # on Windows - self.assertLess(stop - start, 0.020) + # gh-143528: use 100 ms to support slow CI + self.assertLess(stop - start, 0.100) info = time.get_clock_info('thread_time') self.assertTrue(info.monotonic) @@ -574,8 +628,9 @@ def test_get_clock_info(self): 'perf_counter', 'process_time', 'time', - 'thread_time', ] + if hasattr(time, 'thread_time'): + clocks.append('thread_time') for name in clocks: with self.subTest(name=name): @@ -594,17 +649,8 @@ def test_get_clock_info(self): class TestLocale(unittest.TestCase): - def setUp(self): - self.oldloc = locale.setlocale(locale.LC_ALL) - - def tearDown(self): - locale.setlocale(locale.LC_ALL, self.oldloc) - + @support.run_with_locale('LC_ALL', 'fr_FR', '') def test_bug_3061(self): - try: - tmp = locale.setlocale(locale.LC_ALL, "fr_FR") - except locale.Error: - self.skipTest('could not set locale.LC_ALL to fr_FR') # This should not cause an exception time.strftime("%B", (2009,2,1,0,0,0,0,0,0)) @@ -615,14 +661,11 @@ class _TestAsctimeYear: def yearstr(self, y): return time.asctime((y,) + (0,) * 8).split()[-1] - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_large_year(self): # Check that it doesn't crash for year > 9999 self.assertEqual(self.yearstr(12345), '12345') self.assertEqual(self.yearstr(123456789), '123456789') -@unittest.skip("TODO: RUSTPYTHON, ValueError: invalid struct_time parameter") class _TestStrftimeYear: # Issue 13305: For years < 1000, the value is not always @@ -630,15 +673,17 @@ class _TestStrftimeYear: # assumes year >= 1900, so it does not specify the number # of digits. - # TODO: RUSTPYTHON - # if time.strftime('%Y', (1,) + (0,) * 8) == '0001': - # _format = '%04d' - # else: - # _format = '%d' + if time.strftime('%Y', (1,) + (0,) * 8) == '0001': + _format = '%04d' + else: + _format = '%d' def yearstr(self, y): return time.strftime('%Y', (y,) + (0,) * 8) + @unittest.skipUnless( + support.has_strftime_extensions, "requires strftime extension" + ) def test_4dyear(self): # Check that we can return the zero padded value. if self._format == '%04d': @@ -649,8 +694,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: @@ -673,8 +717,6 @@ def test_negative(self): class _Test4dYear: _format = '%d' - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_year(self, fmt=None, func=None): fmt = fmt or self._format func = func or self.yearstr @@ -691,8 +733,6 @@ def test_large_year(self): self.assertEqual(self.yearstr(TIME_MAXYEAR).lstrip('+'), str(TIME_MAXYEAR)) self.assertRaises(OverflowError, self.yearstr, TIME_MAXYEAR + 1) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_negative(self): self.assertEqual(self.yearstr(-1), self._format % -1) self.assertEqual(self.yearstr(-1234), '-1234') @@ -709,30 +749,28 @@ def test_negative(self): class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear, unittest.TestCase): pass -# class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): -# pass +class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): + pass class TestPytime(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_if_buggy_ucrt_strfptime @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. lt = time.localtime() - self.assertTrue(hasattr(lt, "tm_gmtoff")) - self.assertTrue(hasattr(lt, "tm_zone")) + self.assertHasAttr(lt, "tm_gmtoff") + self.assertHasAttr(lt, "tm_zone") # See if the offset and zone are similar to the module # attributes. if lt.tm_gmtoff is None: - self.assertTrue(not hasattr(time, "timezone")) + self.assertNotHasAttr(time, "timezone") else: self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) if lt.tm_zone is None: - self.assertTrue(not hasattr(time, "tzname")) + self.assertNotHasAttr(time, "tzname") else: self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) @@ -751,18 +789,14 @@ def test_localtime_timezone(self): self.assertEqual(new_lt9, lt) self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) self.assertEqual(new_lt9.tm_zone, lt.tm_zone) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_strptime_timezone(self): t = time.strptime("UTC", "%Z") self.assertEqual(t.tm_zone, 'UTC') t = time.strptime("+0500", "%z") self.assertEqual(t.tm_gmtoff, 5 * 3600) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_short_times(self): @@ -775,7 +809,8 @@ def test_short_times(self): self.assertIs(lt.tm_zone, None) -@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +@unittest.skipIf(_testcapi is None, 'need the _testinternalcapi module') +@unittest.skipIf(_testinternalcapi is None, 'need the _testinternalcapi module') class CPyTimeTestCase: """ Base class to test the C _PyTime_t API. @@ -783,7 +818,7 @@ class CPyTimeTestCase: OVERFLOW_SECONDS = None def setUp(self): - from _testcapi import SIZEOF_TIME_T + from _testinternalcapi import SIZEOF_TIME_T bits = SIZEOF_TIME_T * 8 - 1 self.time_t_min = -2 ** bits self.time_t_max = 2 ** bits - 1 @@ -862,7 +897,7 @@ def convert_values(ns_timestamps): # test rounding ns_timestamps = self._rounding_values(use_float) valid_values = convert_values(ns_timestamps) - for time_rnd, decimal_rnd in ROUNDING_MODES : + for time_rnd, decimal_rnd in ROUNDING_MODES: with decimal.localcontext() as context: context.rounding = decimal_rnd @@ -911,36 +946,36 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) def test_FromSeconds(self): - from _testcapi import PyTime_FromSeconds + from _testinternalcapi import _PyTime_FromSeconds - # PyTime_FromSeconds() expects a C int, reject values out of range + # _PyTime_FromSeconds() expects a C int, reject values out of range def c_int_filter(secs): return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) - self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + self.check_int_rounding(lambda secs, rnd: _PyTime_FromSeconds(secs), lambda secs: secs * SEC_TO_NS, value_filter=c_int_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(TypeError): - PyTime_FromSeconds(float('nan')) + _PyTime_FromSeconds(float('nan')) def test_FromSecondsObject(self): - from _testcapi import PyTime_FromSecondsObject + from _testinternalcapi import _PyTime_FromSecondsObject self.check_int_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda secs: secs * SEC_TO_NS) self.check_float_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda ns: self.decimal_round(ns * SEC_TO_NS)) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - PyTime_FromSecondsObject(float('nan'), time_rnd) + _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble @@ -955,11 +990,6 @@ def float_converter(ns): float_converter, NS_TO_SEC) - # test nan - for time_rnd, _ in ROUNDING_MODES: - with self.assertRaises(TypeError): - PyTime_AsSecondsDouble(float('nan')) - def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -970,7 +1000,7 @@ def converter(value): return converter def test_AsTimeval(self): - from _testcapi import PyTime_AsTimeval + from _testinternalcapi import _PyTime_AsTimeval us_converter = self.create_decimal_converter(US_TO_NS) @@ -987,35 +1017,78 @@ def seconds_filter(secs): else: seconds_filter = self.time_t_filter - self.check_int_rounding(PyTime_AsTimeval, + self.check_int_rounding(_PyTime_AsTimeval, timeval_converter, NS_TO_SEC, value_filter=seconds_filter) - @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), - 'need _testcapi.PyTime_AsTimespec') + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec'), + 'need _testinternalcapi._PyTime_AsTimespec') def test_AsTimespec(self): - from _testcapi import PyTime_AsTimespec + from _testinternalcapi import _PyTime_AsTimespec def timespec_converter(ns): return divmod(ns, SEC_TO_NS) - self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + self.check_int_rounding(lambda ns, rnd: _PyTime_AsTimespec(ns), timespec_converter, NS_TO_SEC, value_filter=self.time_t_filter) + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimeval_clamp'), + 'need _testinternalcapi._PyTime_AsTimeval_clamp') + def test_AsTimeval_clamp(self): + from _testinternalcapi import _PyTime_AsTimeval_clamp + + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + tv_sec_max = LONG_MAX + tv_sec_min = LONG_MIN + else: + tv_sec_max = self.time_t_max + tv_sec_min = self.time_t_min + + for t in (PyTime_MIN, PyTime_MAX): + ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) + with decimal.localcontext() as context: + context.rounding = decimal.ROUND_CEILING + us = self.decimal_round(decimal.Decimal(t) / US_TO_NS) + tv_sec, tv_usec = divmod(us, SEC_TO_US) + if tv_sec_max < tv_sec: + tv_sec = tv_sec_max + tv_usec = 0 + elif tv_sec < tv_sec_min: + tv_sec = tv_sec_min + tv_usec = 0 + self.assertEqual(ts, (tv_sec, tv_usec)) + + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec_clamp'), + 'need _testinternalcapi._PyTime_AsTimespec_clamp') + def test_AsTimespec_clamp(self): + from _testinternalcapi import _PyTime_AsTimespec_clamp + + for t in (PyTime_MIN, PyTime_MAX): + ts = _PyTime_AsTimespec_clamp(t) + tv_sec, tv_nsec = divmod(t, NS_TO_SEC) + if self.time_t_max < tv_sec: + tv_sec = self.time_t_max + tv_nsec = 0 + elif tv_sec < self.time_t_min: + tv_sec = self.time_t_min + tv_nsec = 0 + self.assertEqual(ts, (tv_sec, tv_nsec)) + def test_AsMilliseconds(self): - from _testcapi import PyTime_AsMilliseconds + from _testinternalcapi import _PyTime_AsMilliseconds - self.check_int_rounding(PyTime_AsMilliseconds, + self.check_int_rounding(_PyTime_AsMilliseconds, self.create_decimal_converter(MS_TO_NS), NS_TO_SEC) def test_AsMicroseconds(self): - from _testcapi import PyTime_AsMicroseconds + from _testinternalcapi import _PyTime_AsMicroseconds - self.check_int_rounding(PyTime_AsMicroseconds, + self.check_int_rounding(_PyTime_AsMicroseconds, self.create_decimal_converter(US_TO_NS), NS_TO_SEC) @@ -1029,13 +1102,13 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = 2 ** 64 def test_object_to_time_t(self): - from _testcapi import pytime_object_to_time_t + from _testinternalcapi import _PyTime_ObjectToTime_t - self.check_int_rounding(pytime_object_to_time_t, + self.check_int_rounding(_PyTime_ObjectToTime_t, lambda secs: secs, value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_time_t, + self.check_float_rounding(_PyTime_ObjectToTime_t, self.decimal_round, value_filter=self.time_t_filter) @@ -1055,36 +1128,36 @@ def converter(secs): return converter def test_object_to_timeval(self): - from _testcapi import pytime_object_to_timeval + from _testinternalcapi import _PyTime_ObjectToTimeval - self.check_int_rounding(pytime_object_to_timeval, + self.check_int_rounding(_PyTime_ObjectToTimeval, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timeval, + self.check_float_rounding(_PyTime_ObjectToTimeval, self.create_converter(SEC_TO_US), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timeval(float('nan'), time_rnd) + _PyTime_ObjectToTimeval(float('nan'), time_rnd) def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec + from _testinternalcapi import _PyTime_ObjectToTimespec - self.check_int_rounding(pytime_object_to_timespec, + self.check_int_rounding(_PyTime_ObjectToTimespec, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timespec, + self.check_float_rounding(_PyTime_ObjectToTimespec, self.create_converter(SEC_TO_NS), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timespec(float('nan'), time_rnd) + _PyTime_ObjectToTimespec(float('nan'), time_rnd) @unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") class TestTimeWeaklinking(unittest.TestCase): @@ -1110,11 +1183,11 @@ def test_clock_functions(self): if mac_ver >= (10, 12): for name in clock_names: - self.assertTrue(hasattr(time, name), f"time.{name} is not available") + self.assertHasAttr(time, name) else: for name in clock_names: - self.assertFalse(hasattr(time, name), f"time.{name} is available") + self.assertNotHasAttr(time, name) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 72a104fc1a6..2aeebea9f93 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -222,8 +222,8 @@ def test_repeat_function_zero_iters(self): def assert_exc_string(self, exc_string, expected_exc_name): exc_lines = exc_string.splitlines() self.assertGreater(len(exc_lines), 2) - self.assertTrue(exc_lines[0].startswith('Traceback')) - self.assertTrue(exc_lines[-1].startswith(expected_exc_name)) + self.assertStartsWith(exc_lines[0], 'Traceback') + self.assertStartsWith(exc_lines[-1], expected_exc_name) def test_print_exc(self): s = io.StringIO() @@ -297,9 +297,7 @@ def test_main_negative_reps(self): @unittest.skipIf(sys.flags.optimize >= 2, "need __doc__") def test_main_help(self): s = self.run_main(switches=['-h']) - # Note: It's not clear that the trailing space was intended as part of - # the help text, but since it's there, check for it. - self.assertEqual(s, timeit.__doc__ + ' ') + self.assertEqual(s, timeit.__doc__) def test_main_verbose(self): s = self.run_main(switches=['-v']) diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py index f40c7ee48b0..70a0175d771 100644 --- a/Lib/test/test_timeout.py +++ b/Lib/test/test_timeout.py @@ -71,7 +71,6 @@ def testTypeCheck(self): self.assertRaises(TypeError, self.sock.settimeout, {}) self.assertRaises(TypeError, self.sock.settimeout, 0j) - @unittest.skip("TODO: RUSTPYTHON; crash") def testRangeCheck(self): # Test range checking by settimeout() self.assertRaises(ValueError, self.sock.settimeout, -1) diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f04..3a858749285 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,8 +39,19 @@ def test_invalid_char_quotes(self): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): - self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) + self.assertEqual( + tomllib.TOMLDecodeError("", "", 0).__module__, tomllib.__name__ + ) def test_invalid_parse_float(self): def dict_returner(s: str) -> dict: @@ -55,3 +66,33 @@ def list_returner(s: str) -> list: self.assertEqual( str(exc_info.exception), "parse_float must not return dicts or lists" ) + + def test_deprecated_tomldecodeerror(self): + for args in [ + (), + ("err msg",), + (None,), + (None, "doc"), + ("err msg", None), + (None, "doc", None), + ("err msg", "doc", None), + ("one", "two", "three", "four"), + ("one", "two", 3, "four", "five"), + ]: + with self.assertWarns(DeprecationWarning): + e = tomllib.TOMLDecodeError(*args) # type: ignore[arg-type] + self.assertEqual(e.args, args) + + def test_tomldecodeerror(self): + msg = "error parsing" + doc = "v=1\n[table]\nv='val'" + pos = 13 + formatted_msg = "error parsing (at line 3, column 2)" + e = tomllib.TOMLDecodeError(msg, doc, pos) + self.assertEqual(e.args, (formatted_msg,)) + self.assertEqual(str(e), formatted_msg) + self.assertEqual(e.msg, msg) + self.assertEqual(e.doc, doc) + self.assertEqual(e.pos, pos) + self.assertEqual(e.lineno, 3) + self.assertEqual(e.colno, 2) diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 9e677a337a2..118fde24d88 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -5,6 +5,7 @@ import copy import datetime from decimal import Decimal as D +import importlib from pathlib import Path import sys import tempfile @@ -92,6 +93,7 @@ def test_deepcopy(self): } self.assertEqual(obj_copy, expected_obj) + @support.skip_if_unlimited_stack_size def test_inline_array_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -103,6 +105,7 @@ def test_inline_array_recursion_limit(self): recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" tomllib.loads(recursive_array_toml) + @support.skip_if_unlimited_stack_size def test_inline_table_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -113,3 +116,11 @@ def test_inline_table_recursion_limit(self): nest_count=nest_count): recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) + + def test_types_import(self): + """Test that `_types` module runs. + + The module is for type annotations only, so it is otherwise + never imported by tests. + """ + importlib.import_module(f"{tomllib.__name__}._types") diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index a90c9d0baaf..e7cc4039750 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -130,8 +130,7 @@ def setUp(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) self.my_py_filename = fix_ext_py(__file__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 48): 1} def test_traced_func_linear(self): result = self.tracer.runfunc(traced_func_linear, 2, 5) self.assertEqual(result, 7) @@ -144,8 +143,7 @@ def test_traced_func_linear(self): self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 54): 1} def test_traced_func_loop(self): self.tracer.runfunc(traced_func_loop, 2, 3) @@ -158,8 +156,7 @@ def test_traced_func_loop(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/tracedmodules/testmod.py', 3): 1} def test_traced_func_importing(self): self.tracer.runfunc(traced_func_importing, 2, 5) @@ -172,8 +169,7 @@ def test_traced_func_importing(self): self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 76): 10} def test_trace_func_generator(self): self.tracer.runfunc(traced_func_calling_generator) @@ -189,8 +185,7 @@ def test_trace_func_generator(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 87): 1} def test_trace_list_comprehension(self): self.tracer.runfunc(traced_caller_list_comprehension) @@ -204,8 +199,7 @@ def test_trace_list_comprehension(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 996 characters long. Set self.maxDiff to None to see it. def test_traced_decorated_function(self): self.tracer.runfunc(traced_decorated_function) @@ -225,8 +219,7 @@ def test_traced_decorated_function(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + {('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 108): 1} def test_linear_methods(self): # XXX todo: later add 'static_method_linear' and 'class_method_linear' # here, once issue1764286 is resolved @@ -250,8 +243,7 @@ def setUp(self): self.my_py_filename = fix_ext_py(__file__) self.addCleanup(sys.settrace, sys.gettrace()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 51) def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) code = r'''traced_func_loop(2, 5)''' @@ -286,8 +278,6 @@ def tearDown(self): if self._saved_tracefunc is not None: sys.settrace(self._saved_tracefunc) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple_caller(self): self.tracer.runfunc(traced_func_simple_caller, 1) @@ -305,8 +295,6 @@ def test_arg_errors(self): with self.assertRaises(TypeError): self.tracer.runfunc() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -319,8 +307,6 @@ def test_loop_caller_importing(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") @@ -335,8 +321,6 @@ def test_inst_method_calling(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_traced_decorated_function(self): self.tracer.runfunc(traced_decorated_function) @@ -357,8 +341,6 @@ def setUp(self): self.tracer = Trace(count=0, trace=0, countcallers=1) self.filemod = my_file_and_modname() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") @@ -401,8 +383,7 @@ def _coverage(self, tracer, cmd=DEFAULT_SCRIPT): r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'pprint.py' not found in '' @requires_resource('cpu') def test_coverage(self): tracer = trace.Trace(trace=0, count=1) @@ -427,8 +408,7 @@ def test_coverage_ignore(self): files = os.listdir(TESTFN) self.assertEqual(files, ['_importlib.cover']) # Ignore __import__ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'test.tracedmodules.testmod' not found in {} def test_issue9936(self): tracer = trace.Trace(trace=0, count=1) modname = 'test.tracedmodules.testmod' @@ -493,8 +473,7 @@ def tearDown(self): unlink(self.codefile) unlink(self.coverfile) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- def test_cover_files_written_no_highlight(self): # Test also that the cover file for the trace module is not created # (issue #34171). @@ -515,8 +494,7 @@ def test_cover_files_written_no_highlight(self): " print('unreachable')\n" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- def test_cover_files_written_with_highlight(self): argv = '-m trace --count --missing'.split() + [self.codefile] status, stdout, stderr = assert_python_ok(*argv) @@ -544,8 +522,6 @@ def test_failures(self): *_, stderr = assert_python_failure('-m', 'trace', *args) self.assertIn(message, stderr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_listfuncs_flag_success(self): filename = TESTFN + '.py' modulename = os.path.basename(TESTFN) @@ -569,8 +545,7 @@ def test_sys_argv_list(self): PYTHONIOENCODING='utf-8') self.assertIn(direct_stdout.strip(), trace_stdout) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'lines cov% module (path)' not found in '' def test_count_and_summary(self): filename = f'{TESTFN}.py' coverfilename = f'{TESTFN}.cover' @@ -606,8 +581,7 @@ def setUp(self): self.tracer = Trace(count=0, trace=1) self.filemod = my_file_and_modname() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: list index out of range def test_no_source_file(self): filename = "<unknown>" co = traced_func_linear.__code__ diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 9d95903d526..8b7938e3283 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -18,8 +18,8 @@ from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ, requires_debug_ranges, has_no_debug_ranges, requires_subprocess) -from test.support.os_helper import TESTFN, unlink -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.os_helper import TESTFN, temp_dir, unlink +from test.support.script_helper import assert_python_ok, assert_python_failure, make_script from test.support.import_helper import forget from test.support import force_not_colorized, force_not_colorized_test_class @@ -37,6 +37,12 @@ test_frame = namedtuple('frame', ['f_code', 'f_globals', 'f_locals']) test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next', 'tb_lasti']) +color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E"} +colors = { + color_overrides.get(k, k[0].lower()): v + for k, v in _colorize.default_theme.traceback.items() +} + LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json' @@ -82,13 +88,11 @@ def syntax_error_bad_indentation2(self): def tokenizer_error_with_caret_range(self): compile("blech ( ", "?", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) self.assertEqual(len(err), 4) - self.assertTrue(err[1].strip() == "return x!") + self.assertEqual(err[1].strip(), "return x!") self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[1].find("!"), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), 1) @@ -196,8 +200,6 @@ def f(): finally: unlink(TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_indentation(self): err = self.get_exception_format(self.syntax_error_bad_indentation, IndentationError) @@ -218,8 +220,6 @@ def test_base_exception(self): lst = traceback.format_exception_only(e.__class__, e) self.assertEqual(lst, ['KeyboardInterrupt\n']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_only_bad__str__(self): class X(Exception): def __str__(self): @@ -238,8 +238,6 @@ def test_format_exception_group_without_show_group(self): err = traceback.format_exception_only(eg) self.assertEqual(err, ['ExceptionGroup: A (1 sub-exception)\n']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group(self): eg = ExceptionGroup('A', [ValueError('B')]) err = traceback.format_exception_only(eg, show_group=True) @@ -248,8 +246,6 @@ def test_format_exception_group(self): ' ValueError: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_base_exception_group(self): eg = BaseExceptionGroup('A', [BaseException('B')]) err = traceback.format_exception_only(eg, show_group=True) @@ -258,8 +254,6 @@ def test_format_base_exception_group(self): ' BaseException: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_note(self): exc = ValueError('B') exc.add_note('Note') @@ -271,8 +265,6 @@ def test_format_exception_group_with_note(self): ' Note\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_explicit_class(self): eg = ExceptionGroup('A', [ValueError('B')]) err = traceback.format_exception_only(ExceptionGroup, eg, show_group=True) @@ -281,8 +273,6 @@ def test_format_exception_group_explicit_class(self): ' ValueError: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiple_exceptions(self): eg = ExceptionGroup('A', [ValueError('B'), TypeError('C')]) err = traceback.format_exception_only(eg, show_group=True) @@ -292,8 +282,6 @@ def test_format_exception_group_multiple_exceptions(self): ' TypeError: C\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiline_messages(self): eg = ExceptionGroup('A\n1', [ValueError('B\n2')]) err = traceback.format_exception_only(eg, show_group=True) @@ -303,8 +291,6 @@ def test_format_exception_group_multiline_messages(self): ' 2\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiline2_messages(self): exc = ValueError('B\n\n2\n') exc.add_note('\nC\n\n3') @@ -323,8 +309,6 @@ def test_format_exception_group_multiline2_messages(self): ' IndexError: D\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_syntax_error(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) eg = ExceptionGroup('A\n1', [exc]) @@ -336,8 +320,6 @@ def test_format_exception_group_syntax_error(self): ' SyntaxError: error\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_nested_with_notes(self): exc = IndexError('D') exc.add_note('Note\nmultiline') @@ -358,8 +340,6 @@ def test_format_exception_group_nested_with_notes(self): ' TypeError: F\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_tracebacks(self): def f(): try: @@ -385,8 +365,6 @@ def g(): ' TypeError: g\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_cause(self): def f(): try: @@ -404,8 +382,6 @@ def f(): ' ValueError: 0\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_syntax_error_with_custom_values(self): # See https://github.com/python/cpython/issues/128894 for exc in [ @@ -430,8 +406,7 @@ def test_format_exception_group_syntax_error_with_custom_values(self): self.assertEqual(len(err), 1) self.assertEqual(err[-1], 'SyntaxError: error\n') - # TODO: RUSTPYTHON; IndexError: index out of range - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range @requires_subprocess() @force_not_colorized def test_encoded_file(self): @@ -475,16 +450,10 @@ def do_test(firstlines, message, charset, lineno): err_line = "raise RuntimeError('{0}')".format(message_ascii) err_msg = "RuntimeError: {0}".format(message_ascii) - self.assertIn(("line %s" % lineno), stdout[1], - "Invalid line number: {0!r} instead of {1}".format( - stdout[1], lineno)) - self.assertTrue(stdout[2].endswith(err_line), - "Invalid traceback line: {0!r} instead of {1!r}".format( - stdout[2], err_line)) + self.assertIn("line %s" % lineno, stdout[1]) + self.assertEndsWith(stdout[2], err_line) actual_err_msg = stdout[3] - self.assertTrue(actual_err_msg == err_msg, - "Invalid error message: {0!r} instead of {1!r}".format( - actual_err_msg, err_msg)) + self.assertEqual(actual_err_msg, err_msg) do_test("", "foo", "ascii", 3) for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"): @@ -503,8 +472,7 @@ def do_test(firstlines, message, charset, lineno): # Issue #18960: coding spec should have no effect do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + b'ZeroDivisionError: division by zero'] def test_print_traceback_at_exit(self): # Issue #22599: Ensure that it is possible to use the traceback module # to display an exception at Python exit @@ -538,6 +506,33 @@ def __del__(self): b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) + @cpython_only + def test_lost_io_open(self): + # GH-142737: Display the traceback even if io.open is lost + crasher = textwrap.dedent("""\ + import io + import traceback + # Trigger fallback mode + traceback._print_exception_bltin = None + del io.open + raise RuntimeError("should not crash") + """) + + # Create a temporary script to exercise _Py_FindSourceFile + with temp_dir() as script_dir: + script = make_script( + script_dir=script_dir, + script_basename='tb_test_no_io_open', + source=crasher) + rc, stdout, stderr = assert_python_failure(script) + + self.assertEqual(rc, 1) # Make sure it's not a crash + + expected = [b'Traceback (most recent call last):', + f' File "{script}", line 6, in <module>'.encode(), + b'RuntimeError: should not crash'] + self.assertEqual(stderr.splitlines(), expected) + def test_print_exception(self): output = StringIO() traceback.print_exception( @@ -550,16 +545,12 @@ def test_print_exception_exc(self): traceback.print_exception(Exception("projector"), file=output) self.assertEqual(output.getvalue(), "Exception: projector\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_print_last(self): with support.swap_attr(sys, 'last_exc', ValueError(42)): output = StringIO() traceback.print_last(file=output) self.assertEqual(output.getvalue(), "ValueError: 42\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_exc(self): e = Exception("projector") output = traceback.format_exception(e) @@ -598,8 +589,6 @@ def test_exception_is_None(self): self.assertEqual( traceback.format_exception_only(None, None), [NONE_EXC_STRING]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), @@ -646,13 +635,11 @@ def get_exception(self, callable, slice_start=0, slice_end=-1): class CAPIExceptionFormattingLegacyMixin(CAPIExceptionFormattingMixin): LEGACY = 1 -# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@requires_debug_ranges() class TracebackErrorLocationCaretTestBase: """ Tests for printing code error expressions as part of PEP 657 """ - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_caret(self): # NOTE: In caret tests, "if True:" is used as a way to force indicator # display, since the raising expression spans only part of the line. @@ -672,8 +659,6 @@ def f(): result_lines = self.get_exception(f) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_line_with_unicode(self): # Make sure that even if a line contains multi-byte unicode characters # the correct carets are printed. @@ -693,12 +678,11 @@ def f_with_unicode(): result_lines = self.get_exception(f_with_unicode) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -706,15 +690,15 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) result_lines = self.get_exception(f_with_type) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_multiline_expression(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -740,8 +724,6 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_multiline_expression_syntax_error(self): # Make sure an expression spanning multiple lines that has # a syntax error is correctly marked with carets. @@ -806,8 +788,6 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_multiline_expression_bin_op(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -832,8 +812,6 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators(self): def f_with_binary_operator(): divisor = 20 @@ -852,8 +830,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_with_unicode(self): def f_with_binary_operator(): áóí = 20 @@ -872,8 +848,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_two_char(self): def f_with_binary_operator(): divisor = 20 @@ -892,8 +866,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 @@ -913,8 +885,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_multiline(self): def f_with_binary_operator(): b = 1 @@ -941,8 +911,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_multiline_two_char(self): def f_with_binary_operator(): b = 1 @@ -980,8 +948,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_binary_operators_multiline_with_unicode(self): def f_with_binary_operator(): b = 1 @@ -1004,8 +970,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_subscript(self): def f_with_subscript(): some_dict = {'x': {'y': None}} @@ -1024,8 +988,6 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_subscript_unicode(self): def f_with_subscript(): some_dict = {'ó': {'á': {'í': {'theta': 1}}}} @@ -1044,8 +1006,6 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_subscript_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = [] @@ -1065,8 +1025,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_subscript_multiline(self): def f_with_subscript(): bbbbb = {} @@ -1103,8 +1061,6 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_call(self): def f_with_call(): def f1(a): @@ -1128,8 +1084,6 @@ def f2(b): result_lines = self.get_exception(f_with_call) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_call_unicode(self): def f_with_call(): def f1(a): @@ -1153,8 +1107,6 @@ def f2(b): result_lines = self.get_exception(f_with_call) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_call_with_spaces_and_parenthesis(self): def f_with_binary_operator(): def f(a): @@ -1176,8 +1128,6 @@ def f(a): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_for_call_multiline(self): def f_with_call(): class C: @@ -1211,8 +1161,6 @@ def g(x): result_lines = self.get_exception(f_with_call) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_lines(self): def f(): x = 1 @@ -1237,8 +1185,6 @@ def f(): result_lines = self.get_exception(f) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_lines_no_caret(self): def f(): x = 1 @@ -1261,8 +1207,6 @@ def f(): result_lines = self.get_exception(f) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_lines_binary_op(self): def f_with_binary_operator(): b = 1 @@ -1301,8 +1245,6 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") @@ -1326,8 +1268,6 @@ def test_traceback_specialization_with_syntax_error(self): ) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_traceback_very_long_line(self): source = "if True: " + "a" * 256 bytecode = compile(source, TESTFN, "exec") @@ -1351,8 +1291,6 @@ def test_traceback_very_long_line(self): ) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_secondary_caret_not_elided(self): # Always show a line's indicators if they include the secondary character. def f_with_subscript(): @@ -1372,8 +1310,6 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret_exception_group(self): # Notably, this covers whether indicators handle margin strings correctly. # (Exception groups use margin strings to display vertical indicators.) @@ -1404,8 +1340,6 @@ def assertSpecialized(self, func, expected_specialization): specialization_line = result_lines[-1] self.assertEqual(specialization_line.lstrip(), expected_specialization) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_specialization_variations(self): self.assertSpecialized(lambda: 1/0, "~^~") @@ -1438,8 +1372,6 @@ def test_specialization_variations(self): self.assertSpecialized(lambda: 1// 0, "~^^~~") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decorator_application_lineno_correct(self): def dec_error(func): raise TypeError @@ -1484,8 +1416,6 @@ class A: pass ) self.assertEqual(result_lines, expected_error.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiline_method_call_a(self): def f(): (None @@ -1503,8 +1433,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiline_method_call_b(self): def f(): (None. @@ -1521,8 +1449,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiline_method_call_c(self): def f(): (None @@ -1540,8 +1466,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_wide_characters_unicode_with_problematic_byte_offset(self): def f(): width @@ -1558,8 +1482,6 @@ def f(): self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_offset_with_wide_characters_middle(self): def f(): width = 1 @@ -1576,8 +1498,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_offset_multiline(self): def f(): www = 1 @@ -1600,8 +1520,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_offset_with_wide_characters_term_highlight(self): def f(): 说明说明 = 1 @@ -1620,8 +1538,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_offset_with_emojis_term_highlight(self): def f(): return "✨🐍" + func_说明说明("📗🚛", @@ -1639,8 +1555,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_offset_wide_chars_subscript(self): def f(): my_dct = { @@ -1664,8 +1578,6 @@ def f(): ] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_memory_error(self): def f(): raise MemoryError() @@ -1679,8 +1591,6 @@ def f(): ' raise MemoryError()'] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_anchors_for_simple_return_statements_are_elided(self): def g(): 1/0 @@ -1770,8 +1680,6 @@ def f(): ] self.assertEqual(result_lines, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_anchors_for_simple_assign_statements_are_elided(self): def g(): 1/0 @@ -1861,8 +1769,51 @@ def f(): ] self.assertEqual(result_lines, expected) - -# @requires_debug_ranges() # XXX: RUSTPYTHON patch +class TestKeywordTypoSuggestions(unittest.TestCase): + TYPO_CASES = [ + ("with block ad something:\n pass", "and"), + ("fur a in b:\n pass", "for"), + ("for a in b:\n pass\nelso:\n pass", "else"), + ("whille True:\n pass", "while"), + ("iff x > 5:\n pass", "if"), + ("if x:\n pass\nelseif y:\n pass", "elif"), + ("tyo:\n pass\nexcept y:\n pass", "try"), + ("classe MyClass:\n pass", "class"), + ("impor math", "import"), + ("form x import y", "from"), + ("defn calculate_sum(a, b):\n return a + b", "def"), + ("def foo():\n returm result", "return"), + ("lamda x: x ** 2", "lambda"), + ("def foo():\n yeld i", "yield"), + ("def foo():\n globel counter", "global"), + ("frum math import sqrt", "from"), + ("asynch def fetch_data():\n pass", "async"), + ("async def foo():\n awaid fetch_data()", "await"), + ('raisee ValueError("Error")', "raise"), + ("[x for x\nin range(3)\nof x]", "if"), + ("[123 fur x\nin range(3)\nif x]", "for"), + ("for x im n:\n pass", "in"), + ] + + def test_keyword_suggestions_from_file(self): + with tempfile.TemporaryDirectory() as script_dir: + for i, (code, expected_kw) in enumerate(self.TYPO_CASES): + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + script_name = make_script(script_dir, f"script_{i}", source) + rc, stdout, stderr = assert_python_failure(script_name) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + + def test_keyword_suggestions_from_command_string(self): + for code, expected_kw in self.TYPO_CASES: + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + +@requires_debug_ranges() @force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( PurePythonExceptionFormattingMixin, @@ -1874,9 +1825,121 @@ class PurePythonTracebackErrorCaretTests( traceback printing in traceback.py. """ + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~^^~~'] + def test_caret_for_binary_operators_two_char(self): + return super().test_caret_for_binary_operators_two_char() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~^~~'] + def test_caret_for_binary_operators(self): + return super().test_caret_for_binary_operators() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~^^^^^^^^^'] + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + return super().test_caret_for_subscript_with_spaces_and_parenthesis() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~^~~~~~~~~~~~'] + def test_byte_offset_with_wide_characters_term_highlight(self): + return super().test_byte_offset_with_wide_characters_term_highlight() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~^~'] + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + return super().test_caret_for_binary_operators_with_spaces_and_parenthesis() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~^^^^^'] + def test_caret_for_subscript(self): + return super().test_caret_for_subscript() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^'] + def test_byte_offset_wide_chars_subscript(self): + return super().test_byte_offset_wide_chars_subscript() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^'] + def test_caret_for_subscript_unicode(self): + return super().test_caret_for_subscript_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~'] + def test_caret_for_binary_operators_multiline(self): + return super().test_caret_for_binary_operators_multiline() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~^~~'] + def test_caret_for_binary_operators_multiline_with_unicode(self): + return super().test_caret_for_binary_operators_multiline_with_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ^^^^^'] + def test_traceback_specialization_with_syntax_error(self): + return super().test_traceback_specialization_with_syntax_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ^^^^^^^^^^^^^'] + def test_caret_multiline_expression_syntax_error(self): + return super().test_caret_multiline_expression_syntax_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~'] + def test_caret_multiline_expression_bin_op(self): + return super().test_caret_multiline_expression_bin_op() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~^^^^^'] + def test_secondary_caret_not_elided(self): + return super().test_secondary_caret_not_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ~^~ + def test_specialization_variations(self): + return super().test_specialization_variations() + + @unittest.expectedFailure # TODO: RUSTPYTHON; - ' ^^^^'] + def test_multiline_method_call_b(self): + return super().test_multiline_method_call_b() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^ ++ + def test_caret_for_binary_operators_with_unicode(self): + return super().test_caret_for_binary_operators_with_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++ + def test_multiline_method_call_a(self): + return super().test_multiline_method_call_a() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? +++ + def test_multiline_method_call_c(self): + return super().test_multiline_method_call_c() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_many_lines(self): + return super().test_many_lines() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_many_lines_no_caret(self): + return super().test_many_lines_no_caret() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + def test_anchors_for_simple_assign_statements_are_elided(self): + return super().test_anchors_for_simple_assign_statements_are_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + def test_anchors_for_simple_return_statements_are_elided(self): + return super().test_anchors_for_simple_return_statements_are_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: No exception thrown. + def test_caret_in_type_annotation(self): + return super().test_caret_in_type_annotation() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 652 characters long. Set self.maxDiff to None to see it. + def test_decorator_application_lineno_correct(self): + return super().test_decorator_application_lineno_correct() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 684 characters long. Set self.maxDiff to None to see it. + def test_many_lines_binary_op(self): + return super().test_many_lines_binary_op() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 726 characters long. Set self.maxDiff to None to see it. + def test_caret_for_binary_operators_multiline_two_char(self): + return super().test_caret_for_binary_operators_multiline_two_char() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 732 characters long. Set self.maxDiff to None to see it. + def test_caret_for_subscript_multiline(self): + return super().test_caret_for_subscript_multiline() + @cpython_only -# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@requires_debug_ranges() @force_not_colorized_test_class class CPythonTracebackErrorCaretTests( CAPIExceptionFormattingMixin, @@ -1888,7 +1951,7 @@ class CPythonTracebackErrorCaretTests( """ @cpython_only -# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@requires_debug_ranges() @force_not_colorized_test_class class CPythonTracebackLegacyErrorCaretTests( CAPIExceptionFormattingLegacyMixin, @@ -1960,9 +2023,9 @@ def check_traceback_format(self, cleanup_func=None): banner = tb_lines[0] self.assertEqual(len(tb_lines), 5) location, source_line = tb_lines[-2], tb_lines[-1] - self.assertTrue(banner.startswith('Traceback')) - self.assertTrue(location.startswith(' File')) - self.assertTrue(source_line.startswith(' raise')) + self.assertStartsWith(banner, 'Traceback') + self.assertStartsWith(location, ' File') + self.assertStartsWith(source_line, ' raise') def test_traceback_format(self): self.check_traceback_format() @@ -2195,9 +2258,7 @@ def h(count=10): actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure - # @requires_debug_ranges() # XXX: RUSTPYTHON patch + @requires_debug_ranges() def test_recursive_traceback(self): if self.DEBUG_RANGES: self._check_recursive_traceback_display(traceback.print_exc) @@ -2253,6 +2314,7 @@ def deep_eg(self): return e @cpython_only + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_capi(self): from _testcapi import exception_print LIMIT = 75 @@ -2264,6 +2326,7 @@ def test_exception_group_deep_recursion_capi(self): self.assertIn('ExceptionGroup', output) self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_traceback(self): LIMIT = 75 eg = self.deep_eg() @@ -2286,8 +2349,6 @@ def test_print_exception_bad_type_capi(self): 'Exception expected for value, int found\n') ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_print_exception_bad_type_python(self): msg = "Exception expected for value, int found" with self.assertRaisesRegex(TypeError, msg): @@ -2343,12 +2404,12 @@ def zero_div(self): def check_zero_div(self, msg): lines = msg.splitlines() if has_no_debug_ranges(): - self.assertTrue(lines[-3].startswith(' File')) + self.assertStartsWith(lines[-3], ' File') self.assertIn('1/0 # In zero_div', lines[-2]) else: - self.assertTrue(lines[-4].startswith(' File')) + self.assertStartsWith(lines[-4], ' File') self.assertIn('1/0 # In zero_div', lines[-3]) - self.assertTrue(lines[-1].startswith('ZeroDivisionError'), lines[-1]) + self.assertStartsWith(lines[-1], 'ZeroDivisionError') def test_simple(self): try: @@ -2358,16 +2419,14 @@ def test_simple(self): lines = self.get_report(e).splitlines() if has_no_debug_ranges(): self.assertEqual(len(lines), 4) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) + self.assertStartsWith(lines[3], 'ZeroDivisionError') else: self.assertEqual(len(lines), 5) - self.assertTrue(lines[4].startswith('ZeroDivisionError')) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + self.assertStartsWith(lines[4], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('1/0 # Marker', lines[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cause(self): def inner_raise(): try: @@ -2406,9 +2465,9 @@ def test_context_suppression(self): e = _ lines = self.get_report(e).splitlines() self.assertEqual(len(lines), 4) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + self.assertStartsWith(lines[3], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('ZeroDivisionError from None', lines[2]) def test_cause_and_context(self): @@ -2501,8 +2560,6 @@ def test_message_none(self): err = self.get_report(Exception('')) self.assertIn('Exception\n', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: @@ -2525,8 +2582,6 @@ def test_syntax_error_various_offsets(self): exp = "\n".join(expected) self.assertEqual(exp, err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_note(self): e = ValueError(123) vanilla = self.get_report(e) @@ -2545,8 +2600,6 @@ def test_exception_with_note(self): del e.__notes__ self.assertEqual(self.get_report(e), vanilla) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_invalid_notes(self): e = ValueError(123) vanilla = self.get_report(e) @@ -2604,8 +2657,6 @@ def __getattr__(self, name): self.get_report(e), vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_multiple_notes(self): for e in [ValueError(42), SyntaxError('bad syntax')]: with self.subTest(e=e): @@ -2685,8 +2736,6 @@ def __str__(self): exp = f'<unknown>.{X.__qualname__}: I am X\n' self.assertEqual(exp, err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad__str__(self): class X(Exception): def __str__(self): @@ -2699,8 +2748,6 @@ def __str__(self): # #### Exception Groups #### - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_basic(self): def exc(): raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) @@ -2722,8 +2769,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_cause(self): def exc(): EG = ExceptionGroup @@ -2760,8 +2805,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_context_with_context(self): def exc(): EG = ExceptionGroup @@ -2809,8 +2852,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_nested(self): def exc(): EG = ExceptionGroup @@ -2861,8 +2902,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_width_limit(self): excs = [] for i in range(1000): @@ -2907,8 +2946,6 @@ def test_exception_group_width_limit(self): report = self.get_report(eg) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_depth_limit(self): exc = TypeError('bad type') for i in range(1000): @@ -2991,8 +3028,6 @@ def test_exception_group_depth_limit(self): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_with_notes(self): def exc(): try: @@ -3043,8 +3078,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_with_multiple_notes(self): def exc(): try: @@ -3100,8 +3133,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - ''' def test_exception_group_wrapped_naked(self): # See gh-128799 @@ -3153,7 +3184,7 @@ def f(): # remove trailing writespace: report = '\n'.join([l.rstrip() for l in report.split('\n')]) self.assertEqual(report, expected) - ''' + @force_not_colorized_test_class class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): @@ -3171,6 +3202,10 @@ def get_report(self, e): self.assertEqual(sio.getvalue(), s) return s + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 1103 characters long. Set self.maxDiff to None to see it. + def test_exception_group_wrapped_naked(self): + return super().test_exception_group_wrapped_naked() + @force_not_colorized_test_class class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): @@ -3222,8 +3257,7 @@ def last_returns_frame4(self): def last_returns_frame5(self): return self.last_returns_frame4() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 not greater than 5 def test_extract_stack(self): frame = self.last_returns_frame5() def extract(**kwargs): @@ -3314,8 +3348,6 @@ class MiscTracebackCases(unittest.TestCase): # Check non-printing functions in traceback module # - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_clear(self): def outer(): middle() @@ -3371,8 +3403,6 @@ def test_basics(self): self.assertNotEqual(f, object()) self.assertEqual(f, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) @@ -3400,11 +3430,17 @@ class TestStack(unittest.TestCase): def test_walk_stack(self): def deeper(): return list(traceback.walk_stack(None)) - s1 = list(traceback.walk_stack(None)) - s2 = deeper() + s1, s2 = list(traceback.walk_stack(None)), deeper() self.assertEqual(len(s2) - len(s1), 1) self.assertEqual(s2[1:], s1) + def test_walk_innermost_frame(self): + def inner(): + return list(traceback.walk_stack(None)) + frames = inner() + innermost_frame, _ = frames[0] + self.assertEqual(innermost_frame.f_code.co_name, "inner") + def test_walk_tb(self): try: 1/0 @@ -3475,8 +3511,6 @@ def test_no_locals(self): s = traceback.StackSummary.extract(iter([(f, 6)])) self.assertEqual(s[0].locals, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_locals(self): def some_inner(k, v): a = 1 @@ -3493,8 +3527,6 @@ def some_inner(k, v): ' v = 4\n' % (__file__, some_inner.__code__.co_firstlineno + 3) ], s.format()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_custom_format_frame(self): class CustomStackSummary(traceback.StackSummary): def format_frame_summary(self, frame_summary, colorize=False): @@ -3509,8 +3541,6 @@ def some_inner(): s.format(), [f'{__file__}:{some_inner.__code__.co_firstlineno + 1}']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dropping_frames(self): def f(): 1/0 @@ -3539,8 +3569,6 @@ def format_frame_summary(self, frame_summary, colorize=False): f' File "{__file__}", line {lno}, in f\n 1/0\n' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_summary_should_show_carets(self): # See: https://github.com/python/cpython/issues/122353 @@ -3611,13 +3639,9 @@ def do_test_smoke(self, exc, expected_type_str): self.assertEqual(expected_type_str, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_smoke_builtin(self): self.do_test_smoke(ValueError(42), 'ValueError') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_smoke_user_exception(self): class MyException(Exception): pass @@ -3630,8 +3654,6 @@ class MyException(Exception): 'test_smoke_user_exception.<locals>.MyException') self.do_test_smoke(MyException('bad things happened'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -3657,8 +3679,6 @@ def foo(): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cause(self): try: try: @@ -3683,8 +3703,6 @@ def test_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context(self): try: try: @@ -3707,8 +3725,6 @@ def test_context(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_long_context_chain(self): def f(): try: @@ -3734,8 +3750,6 @@ def f(): self.assertIn( "RecursionError: maximum recursion depth exceeded", res[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compact_with_cause(self): try: try: @@ -3758,8 +3772,6 @@ def test_compact_with_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compact_no_cause(self): try: try: @@ -3782,8 +3794,6 @@ def test_compact_no_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_save_exc_type(self): try: 1/0 @@ -3796,6 +3806,7 @@ def test_no_save_exc_type(self): self.assertIsNone(te.exc_type) def test_no_refs_to_exception_and_traceback_objects(self): + exc_obj = None try: 1/0 except Exception as e: @@ -3819,8 +3830,6 @@ def test_comparison_basic(self): self.assertNotEqual(exc, object()) self.assertEqual(exc, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_comparison_params_variations(self): def raise_exc(): try: @@ -3912,8 +3921,6 @@ def test_lookup_lines(self): linecache.updatecache('/foo.py', globals()) self.assertEqual(exc.stack[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") @@ -3941,9 +3948,8 @@ def test_traceback_header(self): exc = traceback.TracebackException(Exception, Exception("haven"), None) self.assertEqual(list(exc.format()), ["Exception: haven\n"]) - # @requires_debug_ranges() # XXX: RUSTPYTHON patch - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + @requires_debug_ranges() def test_print(self): def f(): x = 12 @@ -3962,8 +3968,6 @@ def f(): 'ZeroDivisionError: division by zero', '']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dont_swallow_cause_or_context_of_falsey_exception(self): # see gh-132308: Ensure that __cause__ or __context__ attributes of exceptions # that evaluate as falsey are included in the output. For falsey term, @@ -4034,8 +4038,6 @@ def test_exception_group_format_exception_only(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_format_exception_onlyi_recursive(self): teg = traceback.TracebackException.from_exception(self.eg) formatted = ''.join(teg.format_exception_only(show_group=True)).split('\n') @@ -4050,8 +4052,6 @@ def test_exception_group_format_exception_onlyi_recursive(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_format(self): teg = traceback.TracebackException.from_exception(self.eg) @@ -4100,8 +4100,6 @@ def test_exception_group_format(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_group_width(self): excs1 = [] excs2 = [] @@ -4140,8 +4138,6 @@ def test_max_group_width(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_group_depth(self): exc = TypeError('bad type') for i in range(3): @@ -4191,8 +4187,6 @@ def test_comparison(self): self.assertNotEqual(exc, object()) self.assertEqual(exc, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): # see gh-132308: Ensure that subexceptions of exception groups # that evaluate as falsey are displayed in the output. For falsey term, @@ -4230,8 +4224,6 @@ def callable(): ) return result_lines[0] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions(self): class Substitution: noise = more_noise = a = bc = None @@ -4275,8 +4267,6 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_underscored(self): class A: bluch = None @@ -4330,8 +4320,6 @@ class A: actual = self.get_suggestion(A(), 'bluch') self.assertNotIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_no_args(self): class A: blech = None @@ -4349,8 +4337,6 @@ def __getattr__(self, attr): actual = self.get_suggestion(A(), 'bluch') self.assertIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_invalid_args(self): class NonStringifyClass: __str__ = None @@ -4392,8 +4378,6 @@ def __dir__(self): self.assertNotIn("blech", actual) self.assertNotIn("oh no!", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attribute_error_with_non_string_candidates(self): class T: bluch = 1 @@ -4412,8 +4396,6 @@ def raise_attribute_error_with_bad_name(): ) self.assertNotIn("?", result_lines[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attribute_error_inside_nested_getattr(self): class A: bluch = 1 @@ -4457,8 +4439,6 @@ def callable(): ) return result_lines[0] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import_from_suggestions(self): substitution = textwrap.dedent("""\ noise = more_noise = a = bc = None @@ -4509,8 +4489,6 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import_from_suggestions_underscored(self): code = "bluch = None" self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) @@ -4522,8 +4500,6 @@ def test_import_from_suggestions_underscored(self): self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import_from_suggestions_non_string(self): modWithNonStringAttr = textwrap.dedent("""\ globals()[0] = 1 @@ -4567,8 +4543,6 @@ def raise_attribute_error_with_bad_name(): ) self.assertNotIn("?", result_lines[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions(self): def Substitution(): noise = more_noise = a = bc = None @@ -4609,24 +4583,18 @@ def EliminationOverAddition(): actual = self.get_suggestion(func) self.assertIn(suggestion, actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_globals(self): def func(): print(global_for_suggestio) actual = self.get_suggestion(func) self.assertIn("'global_for_suggestions'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_builtins(self): def func(): print(ZeroDivisionErrrrr) actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_builtins_when_builtins_is_module(self): def func(): custom_globals = globals().copy() @@ -4635,8 +4603,6 @@ def func(): actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_with_non_string_candidates(self): def func(): abc = 1 @@ -4785,8 +4751,6 @@ def func(): actual = self.get_suggestion(func) self.assertNotIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_with_instance(self): class A: def __init__(self): @@ -4843,8 +4807,6 @@ def func(): actual = self.get_suggestion(func) self.assertNotIn("something", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_for_stdlib_modules(self): def func(): stream = io.StringIO() @@ -4852,8 +4814,6 @@ def func(): actual = self.get_suggestion(func) self.assertIn("forget to import 'io'", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_for_private_stdlib_modules(self): def func(): stream = _io.StringIO() @@ -4889,17 +4849,14 @@ class MiscTest(unittest.TestCase): def test_all(self): expected = set() - denylist = {'print_list'} for name in dir(traceback): - if name.startswith('_') or name in denylist: + if name.startswith('_'): continue module_object = getattr(traceback, name) if getattr(module_object, '__module__', None) == 'traceback': expected.add(name) self.assertCountEqual(traceback.__all__, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_levenshtein_distance(self): # copied from _testinternalcapi.test_edit_cost # to also exercise the Python implementation @@ -4929,8 +4886,7 @@ def CHECK(a, b, expected): CHECK("AttributeError", "AttributeErrorTests", 10) CHECK("ABA", "AAB", 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: /Users/al03219714/Projects/RustPython/crates/pylib/Lib/test/levenshtein_examples.json is missing. Run `make regen-test-levenshtein` @support.requires_resource('cpu') def test_levenshtein_distance_short_circuit(self): if not LEVENSHTEIN_DATA_FILE.is_file(): @@ -4987,8 +4943,8 @@ class MyList(list): class TestColorizedTraceback(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + maxDiff = None + def test_colorized_traceback(self): def foo(*args): x = {'a':{'b': None}} @@ -5011,9 +4967,9 @@ def bar(): e, capture_locals=True ) lines = "".join(exc.format(colorize=True)) - red = _colorize.ANSIColors.RED - boldr = _colorize.ANSIColors.BOLD_RED - reset = _colorize.ANSIColors.RESET + red = colors["e"] + boldr = colors["E"] + reset = colors["z"] self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) @@ -5021,8 +4977,6 @@ def bar(): self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_colorized_syntax_error(self): try: compile("a $ b", "<string>", "exec") @@ -5031,21 +4985,18 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - red = _colorize.ANSIColors.RED - magenta = _colorize.ANSIColors.MAGENTA - boldm = _colorize.ANSIColors.BOLD_MAGENTA - boldr = _colorize.ANSIColors.BOLD_RED - reset = _colorize.ANSIColors.RESET - expected = "".join([ - f' File {magenta}"<string>"{reset}, line {magenta}1{reset}\n', - f' a {boldr}${reset} b\n', - f' {boldr}^{reset}\n', - f'{boldm}SyntaxError{reset}: {magenta}invalid syntax{reset}\n'] - ) - self.assertIn(expected, actual) + def expected(t, m, fn, l, f, E, e, z): + return "".join( + [ + f' File {fn}"<string>"{z}, line {l}1{z}\n', + f' a {E}${z} b\n', + f' {E}^{z}\n', + f'{t}SyntaxError{z}: {m}invalid syntax{z}\n' + ] + ) + self.assertIn(expected(**colors), actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_testcapi' def test_colorized_traceback_is_the_default(self): def foo(): 1/0 @@ -5060,26 +5011,22 @@ def foo(): exception_print(e) actual = tbstderr.getvalue().splitlines() - red = _colorize.ANSIColors.RED - boldr = _colorize.ANSIColors.BOLD_RED - magenta = _colorize.ANSIColors.MAGENTA - boldm = _colorize.ANSIColors.BOLD_MAGENTA - reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno - expected = ['Traceback (most recent call last):', - f' File {magenta}"{__file__}"{reset}, ' - f'line {magenta}{lno_foo+5}{reset}, in {magenta}test_colorized_traceback_is_the_default{reset}', - f' {red}foo{reset+boldr}(){reset}', - f' {red}~~~{reset+boldr}^^{reset}', - f' File {magenta}"{__file__}"{reset}, ' - f'line {magenta}{lno_foo+1}{reset}, in {magenta}foo{reset}', - f' {red}1{reset+boldr}/{reset+red}0{reset}', - f' {red}~{reset+boldr}^{reset+red}~{reset}', - f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] - self.assertEqual(actual, expected) + def expected(t, m, fn, l, f, E, e, z): + return [ + 'Traceback (most recent call last):', + f' File {fn}"{__file__}"{z}, ' + f'line {l}{lno_foo+5}{z}, in {f}test_colorized_traceback_is_the_default{z}', + f' {e}foo{z}{E}(){z}', + f' {e}~~~{z}{E}^^{z}', + f' File {fn}"{__file__}"{z}, ' + f'line {l}{lno_foo+1}{z}, in {f}foo{z}', + f' {e}1{z}{E}/{z}{e}0{z}', + f' {e}~{z}{E}^{z}{e}~{z}', + f'{t}ZeroDivisionError{z}: {m}division by zero{z}', + ] + self.assertEqual(actual, expected(**colors)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_colorized_traceback_from_exception_group(self): def foo(): exceptions = [] @@ -5096,33 +5043,31 @@ def foo(): e, capture_locals=True ) - red = _colorize.ANSIColors.RED - boldr = _colorize.ANSIColors.BOLD_RED - magenta = _colorize.ANSIColors.MAGENTA - boldm = _colorize.ANSIColors.BOLD_MAGENTA - reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno actual = "".join(exc.format(colorize=True)).splitlines() - expected = [f" + Exception Group Traceback (most recent call last):", - f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+9}{reset}, in {magenta}test_colorized_traceback_from_exception_group{reset}', - f' | {red}foo{reset}{boldr}(){reset}', - f' | {red}~~~{reset}{boldr}^^{reset}', - f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", - f" | foo = {foo}", - f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', - f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+6}{reset}, in {magenta}foo{reset}', - f' | raise ExceptionGroup("test", exceptions)', - f" | exceptions = [ZeroDivisionError('division by zero')]", - f' | {boldm}ExceptionGroup{reset}: {magenta}test (1 sub-exception){reset}', - f' +-+---------------- 1 ----------------', - f' | Traceback (most recent call last):', - f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+3}{reset}, in {magenta}foo{reset}', - f' | {red}1 {reset}{boldr}/{reset}{red} 0{reset}', - f' | {red}~~{reset}{boldr}^{reset}{red}~~{reset}', - f" | exceptions = [ZeroDivisionError('division by zero')]", - f' | {boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}', - f' +------------------------------------'] - self.assertEqual(actual, expected) + def expected(t, m, fn, l, f, E, e, z): + return [ + f" + Exception Group Traceback (most recent call last):", + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+9}{z}, in {f}test_colorized_traceback_from_exception_group{z}', + f' | {e}foo{z}{E}(){z}', + f' | {e}~~~{z}{E}^^{z}', + f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", + f" | foo = {foo}", + f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+6}{z}, in {f}foo{z}', + f' | raise ExceptionGroup("test", exceptions)', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {t}ExceptionGroup{z}: {m}test (1 sub-exception){z}', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+3}{z}, in {f}foo{z}', + f' | {e}1 {z}{E}/{z}{e} 0{z}', + f' | {e}~~{z}{E}^{z}{e}~~{z}', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {t}ZeroDivisionError{z}: {m}division by zero{z}', + f' +------------------------------------', + ] + self.assertEqual(actual, expected(**colors)) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py new file mode 100644 index 00000000000..e91bf3f8b4e --- /dev/null +++ b/Lib/test/test_tstring.py @@ -0,0 +1,296 @@ +import unittest + +from test.test_string._support import TStringBaseCase, fstring + + +class TestTString(unittest.TestCase, TStringBaseCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; + Template(strings=('Hello',), interpolations=()) + def test_string_representation(self): + # Test __repr__ + t = t"Hello" + self.assertEqual(repr(t), "Template(strings=('Hello',), interpolations=())") + + name = "Python" + t = t"Hello, {name}" + self.assertEqual(repr(t), + "Template(strings=('Hello, ', ''), " + "interpolations=(Interpolation('Python', 'name', None, ''),))" + ) + + def test_interpolation_basics(self): + # Test basic interpolation + name = "Python" + t = t"Hello, {name}" + self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(t), "Hello, Python") + + # Multiple interpolations + first = "Python" + last = "Developer" + t = t"{first} {last}" + self.assertTStringEqual( + t, ("", " ", ""), [(first, 'first'), (last, 'last')] + ) + self.assertEqual(fstring(t), "Python Developer") + + # Interpolation with expressions + a = 10 + b = 20 + t = t"Sum: {a + b}" + self.assertTStringEqual(t, ("Sum: ", ""), [(a + b, "a + b")]) + self.assertEqual(fstring(t), "Sum: 30") + + # Interpolation with function + def square(x): + return x * x + t = t"Square: {square(5)}" + self.assertTStringEqual( + t, ("Square: ", ""), [(square(5), "square(5)")] + ) + self.assertEqual(fstring(t), "Square: 25") + + # Test attribute access in expressions + class Person: + def __init__(self, name): + self.name = name + + def upper(self): + return self.name.upper() + + person = Person("Alice") + t = t"Name: {person.name}" + self.assertTStringEqual( + t, ("Name: ", ""), [(person.name, "person.name")] + ) + self.assertEqual(fstring(t), "Name: Alice") + + # Test method calls + t = t"Name: {person.upper()}" + self.assertTStringEqual( + t, ("Name: ", ""), [(person.upper(), "person.upper()")] + ) + self.assertEqual(fstring(t), "Name: ALICE") + + # Test dictionary access + data = {"name": "Bob", "age": 30} + t = t"Name: {data['name']}, Age: {data['age']}" + self.assertTStringEqual( + t, ("Name: ", ", Age: ", ""), + [(data["name"], "data['name']"), (data["age"], "data['age']")], + ) + self.assertEqual(fstring(t), "Name: Bob, Age: 30") + + def test_format_specifiers(self): + # Test basic format specifiers + value = 3.14159 + t = t"Pi: {value:.2f}" + self.assertTStringEqual( + t, ("Pi: ", ""), [(value, "value", None, ".2f")] + ) + self.assertEqual(fstring(t), "Pi: 3.14") + + def test_conversions(self): + # Test !s conversion (str) + obj = object() + t = t"Object: {obj!s}" + self.assertTStringEqual(t, ("Object: ", ""), [(obj, "obj", "s")]) + self.assertEqual(fstring(t), f"Object: {str(obj)}") + + # Test !r conversion (repr) + t = t"Data: {obj!r}" + self.assertTStringEqual(t, ("Data: ", ""), [(obj, "obj", "r")]) + self.assertEqual(fstring(t), f"Data: {repr(obj)}") + + # Test !a conversion (ascii) + text = "Café" + t = t"ASCII: {text!a}" + self.assertTStringEqual(t, ("ASCII: ", ""), [(text, "text", "a")]) + self.assertEqual(fstring(t), f"ASCII: {ascii(text)}") + + # Test !z conversion (error) + num = 1 + with self.assertRaises(SyntaxError): + eval("t'{num!z}'") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++++ + def test_debug_specifier(self): + # Test debug specifier + value = 42 + t = t"Value: {value=}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", "r")] + ) + self.assertEqual(fstring(t), "Value: value=42") + + # Test debug specifier with format (conversion default to !r) + t = t"Value: {value=:.2f}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", None, ".2f")] + ) + self.assertEqual(fstring(t), "Value: value=42.00") + + # Test debug specifier with conversion + t = t"Value: {value=!s}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", "s")] + ) + + # Test white space in debug specifier + t = t"Value: {value = }" + self.assertTStringEqual( + t, ("Value: value = ", ""), [(value, "value", "r")] + ) + self.assertEqual(fstring(t), "Value: value = 42") + + def test_raw_tstrings(self): + path = r"C:\Users" + t = rt"{path}\Documents" + self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) + self.assertEqual(fstring(t), r"C:\Users\Documents") + + # Test alternative prefix + t = tr"{path}\Documents" + self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "can only concatenate string.templatelib.Template \(not "str"\) to string.templatelib.Template" does not match "can only concatenate Template (not 'str') to Template" + def test_template_concatenation(self): + # Test template + template + t1 = t"Hello, " + t2 = t"world" + combined = t1 + t2 + self.assertTStringEqual(combined, ("Hello, world",), ()) + self.assertEqual(fstring(combined), "Hello, world") + + # Test template + string + t1 = t"Hello" + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" + + # Test template + template with interpolation + name = "Python" + t1 = t"Hello, " + t2 = t"{name}" + combined = t1 + t2 + self.assertTStringEqual(combined, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(combined), "Hello, Python") + + # Test string + template + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" + + def test_nested_templates(self): + # Test a template inside another template expression + name = "Python" + inner = t"{name}" + t = t"Language: {inner}" + + t_interp = t.interpolations[0] + self.assertEqual(t.strings, ("Language: ", "")) + self.assertEqual(t_interp.value.strings, ("", "")) + self.assertEqual(t_interp.value.interpolations[0].value, name) + self.assertEqual(t_interp.value.interpolations[0].expression, "name") + self.assertEqual(t_interp.value.interpolations[0].conversion, None) + self.assertEqual(t_interp.value.interpolations[0].format_spec, "") + self.assertEqual(t_interp.expression, "inner") + self.assertEqual(t_interp.conversion, None) + self.assertEqual(t_interp.format_spec, "") + + @unittest.expectedFailure # TODO: RUSTPYTHON multiple instances of AssertionError + def test_syntax_errors(self): + for case, err in ( + ("t'", "unterminated t-string literal"), + ("t'''", "unterminated triple-quoted t-string literal"), + ("t''''", "unterminated triple-quoted t-string literal"), + ("t'{", "'{' was never closed"), + ("t'{'", "t-string: expecting '}'"), + ("t'{a'", "t-string: expecting '}'"), + ("t'}'", "t-string: single '}' is not allowed"), + ("t'{}'", "t-string: valid expression required before '}'"), + ("t'{=x}'", "t-string: valid expression required before '='"), + ("t'{!x}'", "t-string: valid expression required before '!'"), + ("t'{:x}'", "t-string: valid expression required before ':'"), + ("t'{x;y}'", "t-string: expecting '=', or '!', or ':', or '}'"), + ("t'{x=y}'", "t-string: expecting '!', or ':', or '}'"), + ("t'{x!s!}'", "t-string: expecting ':' or '}'"), + ("t'{x!s:'", "t-string: expecting '}', or format specs"), + ("t'{x!}'", "t-string: missing conversion character"), + ("t'{x=!}'", "t-string: missing conversion character"), + ("t'{x!z}'", "t-string: invalid conversion character 'z': " + "expected 's', 'r', or 'a'"), + ("t'{lambda:1}'", "t-string: lambda expressions are not allowed " + "without parentheses"), + ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"), + ("t'{1:d\n}'", "t-string: newlines are not allowed in format specifiers") + ): + with self.subTest(case), self.assertRaisesRegex(SyntaxError, err): + eval(case) + + def test_runtime_errors(self): + # Test missing variables + with self.assertRaises(NameError): + eval("t'Hello, {name}'") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_literal_concatenation(self): + # Test concatenation of t-string literals + t = t"Hello, " t"world" + self.assertTStringEqual(t, ("Hello, world",), ()) + self.assertEqual(fstring(t), "Hello, world") + + # Test concatenation with interpolation + name = "Python" + t = t"Hello, " t"{name}" + self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(t), "Hello, Python") + + # Test disallowed mix of t-string and string/f-string (incl. bytes) + what = 't' + expected_msg = 'cannot mix t-string literals with string or bytes literals' + for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", + "t'{what}-string literal' b'bytes literal'", + "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", + ): + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) + + def test_triple_quoted(self): + # Test triple-quoted t-strings + t = t""" + Hello, + world + """ + self.assertTStringEqual( + t, ("\n Hello,\n world\n ",), () + ) + self.assertEqual(fstring(t), "\n Hello,\n world\n ") + + # Test triple-quoted with interpolation + name = "Python" + t = t""" + Hello, + {name} + """ + self.assertTStringEqual( + t, ("\n Hello,\n ", "\n "), [(name, "name")] + ) + self.assertEqual(fstring(t), "\n Hello,\n Python\n ") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tty.py b/Lib/test/test_tty.py new file mode 100644 index 00000000000..681772fb519 --- /dev/null +++ b/Lib/test/test_tty.py @@ -0,0 +1,96 @@ +import os +import unittest +from test.support.import_helper import import_module + +termios = import_module('termios') +tty = import_module('tty') + + +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class TestTty(unittest.TestCase): + + def setUp(self): + master_fd, self.fd = os.openpty() + self.addCleanup(os.close, master_fd) + self.stream = self.enterContext(open(self.fd, 'wb', buffering=0)) + self.fd = self.stream.fileno() + self.mode = termios.tcgetattr(self.fd) + self.addCleanup(termios.tcsetattr, self.fd, termios.TCSANOW, self.mode) + self.addCleanup(termios.tcsetattr, self.fd, termios.TCSAFLUSH, self.mode) + + def check_cbreak(self, mode): + self.assertEqual(mode[3] & termios.ECHO, 0) + self.assertEqual(mode[3] & termios.ICANON, 0) + self.assertEqual(mode[6][termios.VMIN], 1) + self.assertEqual(mode[6][termios.VTIME], 0) + + def check_raw(self, mode): + self.check_cbreak(mode) + self.assertEqual(mode[0] & termios.ISTRIP, 0) + self.assertEqual(mode[0] & termios.ICRNL, 0) + self.assertEqual(mode[1] & termios.OPOST, 0) + self.assertEqual(mode[2] & termios.PARENB, termios.CS8 & termios.PARENB) + self.assertEqual(mode[2] & termios.CSIZE, termios.CS8 & termios.CSIZE) + self.assertEqual(mode[2] & termios.CS8, termios.CS8) + self.assertEqual(mode[3] & termios.ECHO, 0) + self.assertEqual(mode[3] & termios.ICANON, 0) + self.assertEqual(mode[3] & termios.ISIG, 0) + self.assertEqual(mode[6][termios.VMIN], 1) + self.assertEqual(mode[6][termios.VTIME], 0) + + def test_cfmakeraw(self): + mode = termios.tcgetattr(self.fd) + self.assertEqual(mode, self.mode) + tty.cfmakeraw(mode) + self.check_raw(mode) + self.assertEqual(mode[4], self.mode[4]) + self.assertEqual(mode[5], self.mode[5]) + + def test_cfmakecbreak(self): + mode = termios.tcgetattr(self.fd) + self.assertEqual(mode, self.mode) + tty.cfmakecbreak(mode) + self.check_cbreak(mode) + self.assertEqual(mode[1], self.mode[1]) + self.assertEqual(mode[2], self.mode[2]) + self.assertEqual(mode[4], self.mode[4]) + self.assertEqual(mode[5], self.mode[5]) + mode[tty.IFLAG] |= termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, termios.ICRNL, + msg="ICRNL should not be cleared by cbreak") + mode[tty.IFLAG] &= ~termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, 0, + msg="ICRNL should not be set by cbreak") + + @unittest.expectedFailure # TODO: RUSTPYTHON TypeError: Expected type "int" but "FileIO" found. + def test_setraw(self): + mode0 = termios.tcgetattr(self.fd) + mode1 = tty.setraw(self.fd) + self.assertEqual(mode1, mode0) + mode2 = termios.tcgetattr(self.fd) + self.check_raw(mode2) + mode3 = tty.setraw(self.fd, termios.TCSANOW) + self.assertEqual(mode3, mode2) + tty.setraw(self.stream) + tty.setraw(fd=self.fd, when=termios.TCSANOW) + + @unittest.expectedFailure # TODO: RUSTPYTHON TypeError: Expected type "int" but "FileIO" found. + def test_setcbreak(self): + mode0 = termios.tcgetattr(self.fd) + mode1 = tty.setcbreak(self.fd) + self.assertEqual(mode1, mode0) + mode2 = termios.tcgetattr(self.fd) + self.check_cbreak(mode2) + ICRNL = termios.ICRNL + self.assertEqual(mode2[tty.IFLAG] & ICRNL, mode0[tty.IFLAG] & ICRNL, + msg="ICRNL should not be altered by cbreak") + mode3 = tty.setcbreak(self.fd, termios.TCSANOW) + self.assertEqual(mode3, mode2) + tty.setcbreak(self.stream) + tty.setcbreak(fd=self.fd, when=termios.TCSANOW) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py new file mode 100644 index 00000000000..ee1791bc1d0 --- /dev/null +++ b/Lib/test/test_type_aliases.py @@ -0,0 +1,415 @@ +import pickle +import types +import unittest +from test.support import check_syntax_error, run_code +from test.typinganndata import mod_generics_cache + +from typing import ( + Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, Unpack, get_args, +) + + +class TypeParamsInvalidTest(unittest.TestCase): + def test_name_collisions(self): + check_syntax_error(self, 'type TA1[A, **A] = None', "duplicate type parameter 'A'") + check_syntax_error(self, 'type T[A, *A] = None', "duplicate type parameter 'A'") + check_syntax_error(self, 'type T[*A, **A] = None', "duplicate type parameter 'A'") + + def test_name_non_collision_02(self): + ns = run_code("""type TA1[A] = lambda A: A""") + self.assertIsInstance(ns["TA1"], TypeAliasType) + self.assertTrue(callable(ns["TA1"].__value__)) + self.assertEqual("arg", ns["TA1"].__value__("arg")) + + def test_name_non_collision_03(self): + ns = run_code(""" + class Outer[A]: + type TA1[A] = None + """ + ) + outer_A, = ns["Outer"].__type_params__ + inner_A, = ns["Outer"].TA1.__type_params__ + self.assertIsNot(outer_A, inner_A) + + +class TypeParamsAccessTest(unittest.TestCase): + def test_alias_access_01(self): + ns = run_code("type TA1[A, B] = dict[A, B]") + alias = ns["TA1"] + self.assertIsInstance(alias, TypeAliasType) + self.assertEqual(alias.__type_params__, get_args(alias.__value__)) + + def test_alias_access_02(self): + ns = run_code(""" + type TA1[A, B] = TA1[A, B] | int + """ + ) + alias = ns["TA1"] + self.assertIsInstance(alias, TypeAliasType) + A, B = alias.__type_params__ + self.assertEqual(alias.__value__, alias[A, B] | int) + + def test_alias_access_03(self): + ns = run_code(""" + class Outer[A]: + def inner[B](self): + type TA1[C] = TA1[A, B] | int + return TA1 + """ + ) + cls = ns["Outer"] + A, = cls.__type_params__ + B, = cls.inner.__type_params__ + alias = cls.inner(None) + self.assertIsInstance(alias, TypeAliasType) + alias2 = cls.inner(None) + self.assertIsNot(alias, alias2) + self.assertEqual(len(alias.__type_params__), 1) + + self.assertEqual(alias.__value__, alias[A, B] | int) + + +class TypeParamsAliasValueTest(unittest.TestCase): + def test_alias_value_01(self): + type TA1 = int + + self.assertIsInstance(TA1, TypeAliasType) + self.assertEqual(TA1.__value__, int) + self.assertEqual(TA1.__parameters__, ()) + self.assertEqual(TA1.__type_params__, ()) + + type TA2 = TA1 | str + + self.assertIsInstance(TA2, TypeAliasType) + a, b = TA2.__value__.__args__ + self.assertEqual(a, TA1) + self.assertEqual(b, str) + self.assertEqual(TA2.__parameters__, ()) + self.assertEqual(TA2.__type_params__, ()) + + def test_alias_value_02(self): + class Parent[A]: + type TA1[B] = dict[A, B] + + self.assertIsInstance(Parent.TA1, TypeAliasType) + self.assertEqual(len(Parent.TA1.__parameters__), 1) + self.assertEqual(len(Parent.__parameters__), 1) + a, = Parent.__parameters__ + b, = Parent.TA1.__parameters__ + self.assertEqual(Parent.__type_params__, (a,)) + self.assertEqual(Parent.TA1.__type_params__, (b,)) + self.assertEqual(Parent.TA1.__value__, dict[a, b]) + + def test_alias_value_03(self): + def outer[A](): + type TA1[B] = dict[A, B] + return TA1 + + o = outer() + self.assertIsInstance(o, TypeAliasType) + self.assertEqual(len(o.__parameters__), 1) + self.assertEqual(len(outer.__type_params__), 1) + b = o.__parameters__[0] + self.assertEqual(o.__type_params__, (b,)) + + def test_alias_value_04(self): + def more_generic[T, *Ts, **P](): + type TA[T2, *Ts2, **P2] = tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]] + return TA + + alias = more_generic() + self.assertIsInstance(alias, TypeAliasType) + T2, Ts2, P2 = alias.__type_params__ + self.assertEqual(alias.__parameters__, (T2, *Ts2, P2)) + T, Ts, P = more_generic.__type_params__ + self.assertEqual(alias.__value__, tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]]) + + def test_subscripting(self): + type NonGeneric = int + type Generic[A] = dict[A, A] + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + + with self.assertRaises(TypeError): + NonGeneric[int] + + specialized = Generic[int] + self.assertIsInstance(specialized, types.GenericAlias) + self.assertIs(specialized.__origin__, Generic) + self.assertEqual(specialized.__args__, (int,)) + + specialized2 = VeryGeneric[int, str, float, [bool, range]] + self.assertIsInstance(specialized2, types.GenericAlias) + self.assertIs(specialized2.__origin__, VeryGeneric) + self.assertEqual(specialized2.__args__, (int, str, float, [bool, range])) + + def test_repr(self): + type Simple = int + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + + self.assertEqual(repr(Simple), "Simple") + self.assertEqual(repr(VeryGeneric), "VeryGeneric") + self.assertEqual(repr(VeryGeneric[int, bytes, str, [float, object]]), + "VeryGeneric[int, bytes, str, [float, object]]") + self.assertEqual(repr(VeryGeneric[int, []]), + "VeryGeneric[int, []]") + self.assertEqual(repr(VeryGeneric[int, [VeryGeneric[int], list[str]]]), + "VeryGeneric[int, [VeryGeneric[int], list[str]]]") + + def test_recursive_repr(self): + type Recursive = Recursive + self.assertEqual(repr(Recursive), "Recursive") + + type X = list[Y] + type Y = list[X] + self.assertEqual(repr(X), "X") + self.assertEqual(repr(Y), "Y") + + type GenericRecursive[X] = list[X | GenericRecursive[X]] + self.assertEqual(repr(GenericRecursive), "GenericRecursive") + self.assertEqual(repr(GenericRecursive[int]), "GenericRecursive[int]") + self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), + "GenericRecursive[GenericRecursive[int]]") + + def test_raising(self): + type MissingName = list[_My_X] + with self.assertRaisesRegex( + NameError, + "cannot access free variable '_My_X' where it is not associated with a value", + ): + MissingName.__value__ + _My_X = int + self.assertEqual(MissingName.__value__, list[int]) + del _My_X + # Cache should still work: + self.assertEqual(MissingName.__value__, list[int]) + + # Explicit exception: + type ExprException = 1 / 0 + with self.assertRaises(ZeroDivisionError): + ExprException.__value__ + + +class TypeAliasConstructorTest(unittest.TestCase): + def test_basic(self): + TA = TypeAliasType("TA", int) + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + + def test_attributes_with_exec(self): + ns = {} + exec("type TA = int", ns, ns) + TA = ns["TA"] + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertIs(TA.__module__, None) + + def test_generic(self): + T = TypeVar("T") + TA = TypeAliasType("TA", list[T], type_params=(T,)) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[T]) + self.assertEqual(TA.__type_params__, (T,)) + self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] + + def test_type_params_order_with_defaults(self): + HasNoDefaultT = TypeVar("HasNoDefaultT") + WithDefaultT = TypeVar("WithDefaultT", default=int) + + HasNoDefaultP = ParamSpec("HasNoDefaultP") + WithDefaultP = ParamSpec("WithDefaultP", default=HasNoDefaultP) + + HasNoDefaultTT = TypeVarTuple("HasNoDefaultTT") + WithDefaultTT = TypeVarTuple("WithDefaultTT", default=HasNoDefaultTT) + + for type_params in [ + (HasNoDefaultT, WithDefaultT), + (HasNoDefaultP, WithDefaultP), + (HasNoDefaultTT, WithDefaultTT), + ]: + with self.subTest(type_params=type_params): + TypeAliasType("A", int, type_params=type_params) # ok + + msg = "follows default type parameter" + for type_params in [ + (WithDefaultT, HasNoDefaultT), + (WithDefaultP, HasNoDefaultP), + (WithDefaultTT, HasNoDefaultTT), + (WithDefaultT, HasNoDefaultP), # different types + ]: + with self.subTest(type_params=type_params): + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=type_params) + + def test_expects_type_like(self): + T = TypeVar("T") + + msg = "Expected a type param" + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1,)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1, 2)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(T, 2)) + + def test_keywords(self): + TA = TypeAliasType(name="TA", value=int) + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + + def test_errors(self): + with self.assertRaises(TypeError): + TypeAliasType() + with self.assertRaises(TypeError): + TypeAliasType("TA") + with self.assertRaises(TypeError): + TypeAliasType("TA", list, ()) + with self.assertRaises(TypeError): + TypeAliasType("TA", list, type_params=42) + + +class TypeAliasTypeTest(unittest.TestCase): + def test_immutable(self): + with self.assertRaises(TypeError): + TypeAliasType.whatever = "not allowed" + + def test_no_subclassing(self): + with self.assertRaisesRegex(TypeError, "not an acceptable base type"): + class MyAlias(TypeAliasType): + pass + + def test_union(self): + type Alias1 = int + type Alias2 = str + union = Alias1 | Alias2 + self.assertIsInstance(union, types.UnionType) + self.assertEqual(get_args(union), (Alias1, Alias2)) + union2 = Alias1 | list[float] + self.assertIsInstance(union2, types.UnionType) + self.assertEqual(get_args(union2), (Alias1, list[float])) + union3 = list[range] | Alias1 + self.assertIsInstance(union3, types.UnionType) + self.assertEqual(get_args(union3), (list[range], Alias1)) + + def test_module(self): + self.assertEqual(TypeAliasType.__module__, "typing") + type Alias = int + self.assertEqual(Alias.__module__, __name__) + self.assertEqual(mod_generics_cache.Alias.__module__, + mod_generics_cache.__name__) + self.assertEqual(mod_generics_cache.OldStyle.__module__, + mod_generics_cache.__name__) + + def test_unpack(self): + type Alias = tuple[int, int] + unpacked = (*Alias,)[0] + self.assertEqual(unpacked, Unpack[Alias]) + + class Foo[*Ts]: + pass + + x = Foo[str, *Alias] + self.assertEqual(x.__args__, (str, Unpack[Alias])) + + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +type SimpleAlias = int +type RecursiveAlias = dict[str, RecursiveAlias] +type GenericAlias[X] = list[X] +type GenericAliasMultipleTypes[X, Y] = dict[X, Y] +type RecursiveGenericAlias[X] = dict[str, RecursiveAlias[X]] +type BoundGenericAlias[X: int] = set[X] +type ConstrainedGenericAlias[LongName: (str, bytes)] = list[LongName] +type AllTypesAlias[A, *B, **C] = Callable[C, A] | tuple[*B] + + +class TypeAliasPickleTest(unittest.TestCase): + def test_pickling(self): + things_to_test = [ + SimpleAlias, + RecursiveAlias, + + GenericAlias, + GenericAlias[T], + GenericAlias[int], + + GenericAliasMultipleTypes, + GenericAliasMultipleTypes[str, T], + GenericAliasMultipleTypes[T, str], + GenericAliasMultipleTypes[int, str], + + RecursiveGenericAlias, + RecursiveGenericAlias[T], + RecursiveGenericAlias[int], + + BoundGenericAlias, + BoundGenericAlias[int], + BoundGenericAlias[T], + + ConstrainedGenericAlias, + ConstrainedGenericAlias[str], + ConstrainedGenericAlias[T], + + AllTypesAlias, + AllTypesAlias[int, str, T, [T, object]], + + # Other modules: + mod_generics_cache.Alias, + mod_generics_cache.OldStyle, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + type ClassLevel = str + + def test_pickling_local(self): + type A = int + things_to_test = [ + self.ClassLevel, + A, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py new file mode 100644 index 00000000000..a7b87bb2ee0 --- /dev/null +++ b/Lib/test/test_type_annotations.py @@ -0,0 +1,875 @@ +import annotationlib +import inspect +import textwrap +import types +import unittest +from test.support import run_code, check_syntax_error, import_helper, cpython_only +from test.test_inspect import inspect_stringized_annotations + + +class TypeAnnotationTests(unittest.TestCase): + + def test_lazy_create_annotations(self): + # type objects lazy create their __annotations__ dict on demand. + # the annotations dict is stored in type.__dict__ (as __annotations_cache__). + # a freshly created type shouldn't have an annotations dict yet. + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations_cache__" in foo.__dict__) + d = foo.__annotations__ + self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) + del foo.__annotations__ + + def test_setting_annotations(self): + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations_cache__" in foo.__dict__) + d = {'a': int} + foo.__annotations__ = d + self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) + del foo.__annotations__ + + def test_annotations_getset_raises(self): + # builtin types don't have __annotations__ (yet!) + with self.assertRaises(AttributeError): + print(float.__annotations__) + with self.assertRaises(TypeError): + float.__annotations__ = {} + with self.assertRaises(TypeError): + del float.__annotations__ + + # double delete + foo = type("Foo", (), {}) + foo.__annotations__ = {} + del foo.__annotations__ + with self.assertRaises(AttributeError): + del foo.__annotations__ + + def test_annotations_are_created_correctly(self): + class C: + a:int=3 + b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) + self.assertTrue("__annotations_cache__" in C.__dict__) + del C.__annotations__ + self.assertFalse("__annotations_cache__" in C.__dict__) + + def test_pep563_annotations(self): + isa = inspect_stringized_annotations + self.assertEqual( + isa.__annotations__, {"a": "int", "b": "str"}, + ) + self.assertEqual( + isa.MyClass.__annotations__, {"a": "int", "b": "str"}, + ) + + def test_explicitly_set_annotations(self): + class C: + __annotations__ = {"what": int} + self.assertEqual(C.__annotations__, {"what": int}) + + def test_explicitly_set_annotate(self): + class C: + __annotate__ = lambda format: {"what": int} + self.assertEqual(C.__annotations__, {"what": int}) + self.assertIsInstance(C.__annotate__, types.FunctionType) + self.assertEqual(C.__annotate__(annotationlib.Format.VALUE), {"what": int}) + + def test_del_annotations_and_annotate(self): + # gh-132285 + called = False + class A: + def __annotate__(format): + nonlocal called + called = True + return {'a': int} + + self.assertEqual(A.__annotations__, {'a': int}) + self.assertTrue(called) + self.assertTrue(A.__annotate__) + + del A.__annotations__ + called = False + + self.assertEqual(A.__annotations__, {}) + self.assertFalse(called) + self.assertIs(A.__annotate__, None) + + def test_descriptor_still_works(self): + class C: + def __init__(self, name=None, bases=None, d=None): + self.my_annotations = None + + @property + def __annotations__(self): + if not hasattr(self, 'my_annotations'): + self.my_annotations = {} + if not isinstance(self.my_annotations, dict): + self.my_annotations = {} + return self.my_annotations + + @__annotations__.setter + def __annotations__(self, value): + if not isinstance(value, dict): + raise ValueError("can only set __annotations__ to a dict") + self.my_annotations = value + + @__annotations__.deleter + def __annotations__(self): + if getattr(self, 'my_annotations', False) is None: + raise AttributeError('__annotations__') + self.my_annotations = None + + c = C() + self.assertEqual(c.__annotations__, {}) + d = {'a':'int'} + c.__annotations__ = d + self.assertEqual(c.__annotations__, d) + with self.assertRaises(ValueError): + c.__annotations__ = 123 + del c.__annotations__ + with self.assertRaises(AttributeError): + del c.__annotations__ + self.assertEqual(c.__annotations__, {}) + + + class D(metaclass=C): + pass + + self.assertEqual(D.__annotations__, {}) + d = {'a':'int'} + D.__annotations__ = d + self.assertEqual(D.__annotations__, d) + with self.assertRaises(ValueError): + D.__annotations__ = 123 + del D.__annotations__ + with self.assertRaises(AttributeError): + del D.__annotations__ + self.assertEqual(D.__annotations__, {}) + + def test_partially_executed_module(self): + partialexe = import_helper.import_fresh_module("test.typinganndata.partialexecution") + self.assertEqual( + partialexe.a.__annotations__, + {"v1": int, "v2": int}, + ) + self.assertEqual(partialexe.b.annos, {"v1": int}) + + @cpython_only + def test_no_cell(self): + # gh-130924: Test that uses of annotations in local scopes do not + # create cell variables. + def f(x): + a: x + return x + + self.assertEqual(f.__code__.co_cellvars, ()) + + +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + +class TestSetupAnnotations(unittest.TestCase): + def check(self, code: str): + code = textwrap.dedent(code) + for scope in ("module", "class"): + with self.subTest(scope=scope): + if scope == "class": + code = f"class C:\n{textwrap.indent(code, ' ')}" + ns = run_code(code) + annotations = ns["C"].__annotations__ + else: + annotations = build_module(code).__annotations__ + self.assertEqual(annotations, {"x": int}) + + def test_top_level(self): + self.check("x: int = 1") + + def test_blocks(self): + self.check("if True:\n x: int = 1") + self.check(""" + while True: + x: int = 1 + break + """) + self.check(""" + while False: + pass + else: + x: int = 1 + """) + self.check(""" + for i in range(1): + x: int = 1 + """) + self.check(""" + for i in range(1): + pass + else: + x: int = 1 + """) + + def test_try(self): + self.check(""" + try: + x: int = 1 + except: + pass + """) + self.check(""" + try: + pass + except: + pass + else: + x: int = 1 + """) + self.check(""" + try: + pass + except: + pass + finally: + x: int = 1 + """) + self.check(""" + try: + 1/0 + except: + x: int = 1 + """) + + def test_try_star(self): + self.check(""" + try: + x: int = 1 + except* Exception: + pass + """) + self.check(""" + try: + pass + except* Exception: + pass + else: + x: int = 1 + """) + self.check(""" + try: + pass + except* Exception: + pass + finally: + x: int = 1 + """) + self.check(""" + try: + 1/0 + except* Exception: + x: int = 1 + """) + + def test_match(self): + self.check(""" + match 0: + case 0: + x: int = 1 + """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) + + def test_user_defined_annotate(self): + class X: + a: int + + def __annotate__(format): + return {"a": str} + self.assertEqual(X.__annotate__(annotationlib.Format.VALUE), {"a": str}) + self.assertEqual(annotationlib.get_annotations(X), {"a": str}) + + mod = build_module( + """ + a: int + def __annotate__(format): + return {"a": str} + """ + ) + self.assertEqual(mod.__annotate__(annotationlib.Format.VALUE), {"a": str}) + self.assertEqual(annotationlib.get_annotations(mod), {"a": str}) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(3) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + preludes = [ + "", + "class X:\n ", + "def f():\n ", + "async def f():\n ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: [y async for y in x]): ...", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "def func(x: {y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "def func(x: {y: y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (__debug__ := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): [y async for y in x]", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "(x): {y async for y in x}", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "(x): {y: y async for y in x}", "asynchronous comprehension outside of an asynchronous function") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.STRING) + with self.assertRaises(TypeError): + annotate(None) + self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) + + sig = inspect.signature(annotate) + self.assertEqual(sig, inspect.Signature([ + inspect.Parameter("format", inspect.Parameter.POSITIONAL_ONLY) + ])) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_set_annotations(self): + function_code = textwrap.dedent(""" + def f(x: int): + pass + """) + class_code = textwrap.dedent(""" + class f: + x: int + """) + for future in (False, True): + for label, code in (("function", function_code), ("class", class_code)): + with self.subTest(future=future, label=label): + if future: + code = "from __future__ import annotations\n" + code + ns = run_code(code) + f = ns["f"] + anno = "int" if future else int + self.assertEqual(f.__annotations__, {"x": anno}) + + f.__annotations__ = {"x": str} + self.assertEqual(f.__annotations__, {"x": str}) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + # during symbol table construction + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) + + code = """ + class Outer: + class format: pass + + def meth(self, x: format): ... + """ + ns = run_code(code) + self.assertEqual(ns["Outer"].meth.__annotations__, {"x": ns["Outer"].format}) + + code = """ + def f(format): + def inner(x: format): pass + return inner + res = f("closure var") + """ + ns = run_code(code) + self.assertEqual(ns["res"].__annotations__, {"x": "closure var"}) + + code = """ + def f(x: format): + pass + """ + ns = run_code(code) + # picks up the format() builtin + self.assertEqual(ns["f"].__annotations__, {"x": format}) + + code = """ + def outer(): + def f(x: format): + pass + if False: + class format: pass + return f + f = outer() + """ + ns = run_code(code) + with self.assertRaisesRegex( + NameError, + "cannot access free variable 'format' where it is not associated with a value in enclosing scope", + ): + ns["f"].__annotations__ + + +class ConditionalAnnotationTests(unittest.TestCase): + def check_scopes(self, code, true_annos, false_annos): + for scope in ("class", "module"): + for (cond, expected) in ( + # Constants (so code might get optimized out) + (True, true_annos), (False, false_annos), + # Non-constant expressions + ("not not len", true_annos), ("not len", false_annos), + ): + with self.subTest(scope=scope, cond=cond): + code_to_run = code.format(cond=cond) + if scope == "class": + code_to_run = "class Cls:\n" + textwrap.indent(textwrap.dedent(code_to_run), " " * 4) + ns = run_code(code_to_run) + if scope == "class": + self.assertEqual(ns["Cls"].__annotations__, expected) + else: + self.assertEqual(ns["__annotate__"](annotationlib.Format.VALUE), + expected) + + def test_with(self): + code = """ + class Swallower: + def __enter__(self): + pass + + def __exit__(self, *args): + return True + + with Swallower(): + if {cond}: + about_to_raise: int + raise Exception + in_with: "with" + """ + self.check_scopes(code, {"about_to_raise": int}, {"in_with": "with"}) + + def test_simple_if(self): + code = """ + if {cond}: + in_if: "if" + else: + in_if: "else" + """ + self.check_scopes(code, {"in_if": "if"}, {"in_if": "else"}) + + def test_if_elif(self): + code = """ + if not len: + in_if: "if" + elif {cond}: + in_elif: "elif" + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_elif": "elif"}, + {"in_else": "else"} + ) + + def test_try(self): + code = """ + try: + if {cond}: + raise Exception + in_try: "try" + except Exception: + in_except: "except" + finally: + in_finally: "finally" + """ + self.check_scopes( + code, + {"in_except": "except", "in_finally": "finally"}, + {"in_try": "try", "in_finally": "finally"} + ) + + def test_try_star(self): + code = """ + try: + if {cond}: + raise Exception + in_try_star: "try" + except* Exception: + in_except_star: "except" + finally: + in_finally: "finally" + """ + self.check_scopes( + code, + {"in_except_star": "except", "in_finally": "finally"}, + {"in_try_star": "try", "in_finally": "finally"} + ) + + def test_while(self): + code = """ + while {cond}: + in_while: "while" + break + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_while": "while"}, + {"in_else": "else"} + ) + + def test_for(self): + code = """ + for _ in ([1] if {cond} else []): + in_for: "for" + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_for": "for", "in_else": "else"}, + {"in_else": "else"} + ) + + def test_match(self): + code = """ + match {cond}: + case True: + x: "true" + case False: + x: "false" + """ + self.check_scopes( + code, + {"x": "true"}, + {"x": "false"} + ) + + def test_nesting_override(self): + code = """ + if {cond}: + x: "foo" + if {cond}: + x: "bar" + """ + self.check_scopes( + code, + {"x": "bar"}, + {} + ) + + def test_nesting_outer(self): + code = """ + if {cond}: + outer_before: "outer_before" + if len: + inner_if: "inner_if" + else: + inner_else: "inner_else" + outer_after: "outer_after" + """ + self.check_scopes( + code, + {"outer_before": "outer_before", "inner_if": "inner_if", + "outer_after": "outer_after"}, + {} + ) + + def test_nesting_inner(self): + code = """ + if len: + outer_before: "outer_before" + if {cond}: + inner_if: "inner_if" + else: + inner_else: "inner_else" + outer_after: "outer_after" + """ + self.check_scopes( + code, + {"outer_before": "outer_before", "inner_if": "inner_if", + "outer_after": "outer_after"}, + {"outer_before": "outer_before", "inner_else": "inner_else", + "outer_after": "outer_after"}, + ) + + def test_non_name_annotations(self): + code = """ + before: "before" + if {cond}: + a = "x" + a[0]: int + else: + a = object() + a.b: str + after: "after" + """ + expected = {"before": "before", "after": "after"} + self.check_scopes(code, expected, expected) + + +class RegressionTests(unittest.TestCase): + # gh-132479 + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: the symbol 'unique_name_6' must be present in the symbol table + def test_complex_comprehension_inlining(self): + # Test that the various repro cases from the issue don't crash + cases = [ + """ + (unique_name_0): 0 + unique_name_1: ( + 0 + for ( + 0 + for unique_name_2 in 0 + for () in (0 for unique_name_3 in unique_name_4 for unique_name_5 in name_1) + ).name_3 in {0: 0 for name_1 in unique_name_8} + if name_1 + ) + """, + """ + unique_name_0: 0 + unique_name_1: { + 0: 0 + for unique_name_2 in [0 for name_0 in unique_name_4] + if { + 0: 0 + for unique_name_5 in 0 + if name_0 + if ((name_0 for unique_name_8 in unique_name_9) for [] in 0) + } + } + """, + """ + 0[0]: {0 for name_0 in unique_name_1} + unique_name_2: { + 0: (lambda: name_0 for unique_name_4 in unique_name_5) + for unique_name_6 in () + if name_0 + } + """, + ] + for case in cases: + case = textwrap.dedent(case) + compile(case, "<test>", "exec") + + def test_complex_comprehension_inlining_exec(self): + code = """ + unique_name_1 = unique_name_5 = [1] + name_0 = 42 + unique_name_7: {name_0 for name_0 in unique_name_1} + unique_name_2: { + 0: (lambda: name_0 for unique_name_4 in unique_name_5) + for unique_name_6 in [1] + if name_0 + } + """ + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos.keys(), {"unique_name_7", "unique_name_2"}) + self.assertEqual(annos["unique_name_7"], {True}) + genexp = annos["unique_name_2"][0] + lamb = list(genexp)[0] + self.assertEqual(lamb(), 42) + + # gh-138349 + def test_module_level_annotation_plus_listcomp(self): + cases = [ + """ + def report_error(): + pass + try: + [0 for name_2 in unique_name_0 if (lambda: name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + try: + [0 for name_2 in unique_name_0 if (0 for unique_name_1 in unique_name_2 for unique_name_3 in name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + annotated_name: 0 + try: + [0 for name_2 in [[0]] for unique_name_1 in unique_name_2 if (lambda: name_2)] + except: + pass + """, + ] + for code in cases: + with self.subTest(code=code): + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos, {"annotated_name": 0}) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 578c138767d..4f41171b2df 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -1,7 +1,6 @@ import ast import sys import unittest -from test import support funcdef = """\ @@ -67,6 +66,14 @@ def foo(): pass """ +parenthesized_withstmt = """\ +with (a as b): # type: int + pass + +with (a, b): # type: int + pass +""" + vardecl = """\ a = 0 # type: int """ @@ -244,8 +251,7 @@ def parse_all(self, source, minver=lowest, maxver=highest, expected_regex=""): def classic_parse(self, source): return ast.parse(source) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FunctionDef' object has no attribute 'type_comment' def test_funcdef(self): for tree in self.parse_all(funcdef): self.assertEqual(tree.body[0].type_comment, "() -> int") @@ -254,8 +260,7 @@ def test_funcdef(self): self.assertEqual(tree.body[0].type_comment, None) self.assertEqual(tree.body[1].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_asyncdef(self): for tree in self.parse_all(asyncdef, minver=5): self.assertEqual(tree.body[0].type_comment, "() -> int") @@ -264,75 +269,71 @@ def test_asyncdef(self): self.assertEqual(tree.body[0].type_comment, None) self.assertEqual(tree.body[1].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_asyncvar(self): - for tree in self.parse_all(asyncvar, maxver=6): - pass + with self.assertRaises(SyntaxError): + self.classic_parse(asyncvar) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_asynccomp(self): for tree in self.parse_all(asynccomp, minver=6): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_matmul(self): for tree in self.parse_all(matmul, minver=5): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fstring(self): - for tree in self.parse_all(fstring, minver=6): + for tree in self.parse_all(fstring): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_underscorednumber(self): for tree in self.parse_all(underscorednumber, minver=6): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_redundantdef(self): for tree in self.parse_all(redundantdef, maxver=0, expected_regex="^Cannot have two type comments on def"): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FunctionDef' object has no attribute 'type_comment' def test_nonasciidef(self): for tree in self.parse_all(nonasciidef): self.assertEqual(tree.body[0].type_comment, "() -> àçčéñt") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'For' object has no attribute 'type_comment' def test_forstmt(self): for tree in self.parse_all(forstmt): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(forstmt) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'With' object has no attribute 'type_comment' def test_withstmt(self): for tree in self.parse_all(withstmt): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(withstmt) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'With' object has no attribute 'type_comment' + def test_parenthesized_withstmt(self): + for tree in self.parse_all(parenthesized_withstmt): + self.assertEqual(tree.body[0].type_comment, "int") + self.assertEqual(tree.body[1].type_comment, "int") + tree = self.classic_parse(parenthesized_withstmt) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'int' def test_vardecl(self): for tree in self.parse_all(vardecl): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(vardecl) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + (11, ' whatever')] def test_ignores(self): for tree in self.parse_all(ignores): self.assertEqual( @@ -348,16 +349,15 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_longargs(self): - for tree in self.parse_all(longargs): + for tree in self.parse_all(longargs, minver=8): for t in tree.body: # The expected args are encoded in the function name todo = set(t.name[1:]) self.assertEqual(len(t.args.args) + len(t.args.posonlyargs), len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) - self.assertTrue(t.name.startswith('f'), t.name) + self.assertStartsWith(t.name, 'f') for index, c in enumerate(t.name[1:]): todo.remove(c) if c == 'v': @@ -380,8 +380,7 @@ def test_longargs(self): self.assertIsNone(arg.type_comment, "%s(%s:%r)" % (t.name, arg.arg, arg.type_comment)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Tests for inappropriately-placed type comments. def test_inappropriate_type_comments(self): """Tests for inappropriately-placed type comments. @@ -406,8 +405,7 @@ def check_both_ways(source): check_both_ways("pass # type: ignorewhatever\n") check_both_ways("pass # type: ignoreé\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: mode must be "exec", "eval", "ipython", or "single" def test_func_type_input(self): def parse_func_type_input(source): diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py new file mode 100644 index 00000000000..af26a6a0a01 --- /dev/null +++ b/Lib/test/test_type_params.py @@ -0,0 +1,1468 @@ +import annotationlib +import textwrap +import types +import unittest +import pickle +import weakref +from test.support import check_syntax_error, run_code, run_no_yield_async_fn + +from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args + + +class TypeParamsInvalidTest(unittest.TestCase): + def test_name_collisions(self): + check_syntax_error(self, 'def func[**A, A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'def func[A, *A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'def func[*A, **A](): ...', "duplicate type parameter 'A'") + + check_syntax_error(self, 'class C[**A, A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'class C[A, *A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'class C[*A, **A](): ...', "duplicate type parameter 'A'") + + def test_name_non_collision_02(self): + ns = run_code("""def func[A](A): return A""") + func = ns["func"] + self.assertEqual(func(1), 1) + A, = func.__type_params__ + self.assertEqual(A.__name__, "A") + + def test_name_non_collision_03(self): + ns = run_code("""def func[A](*A): return A""") + func = ns["func"] + self.assertEqual(func(1), (1,)) + A, = func.__type_params__ + self.assertEqual(A.__name__, "A") + + def test_name_non_collision_04(self): + # Mangled names should not cause a conflict. + ns = run_code(""" + class ClassA: + def func[__A](self, __A): return __A + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + A, = cls.func.__type_params__ + self.assertEqual(A.__name__, "__A") + + def test_name_non_collision_05(self): + ns = run_code(""" + class ClassA: + def func[_ClassA__A](self, __A): return __A + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + A, = cls.func.__type_params__ + self.assertEqual(A.__name__, "_ClassA__A") + + def test_name_non_collision_06(self): + ns = run_code(""" + class ClassA[X]: + def func(self, X): return X + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_07(self): + ns = run_code(""" + class ClassA[X]: + def func(self): + X = 1 + return X + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(), 1) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_08(self): + ns = run_code(""" + class ClassA[X]: + def func(self): + return [X for X in [1, 2]] + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(), [1, 2]) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_9(self): + ns = run_code(""" + class ClassA[X]: + def func[X](self): + ... + """ + ) + cls = ns["ClassA"] + outer_X, = cls.__type_params__ + inner_X, = cls.func.__type_params__ + self.assertEqual(outer_X.__name__, "X") + self.assertEqual(inner_X.__name__, "X") + self.assertIsNot(outer_X, inner_X) + + def test_name_non_collision_10(self): + ns = run_code(""" + class ClassA[X]: + X: int + """ + ) + cls = ns["ClassA"] + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + self.assertIs(cls.__annotations__["X"], int) + + def test_name_non_collision_13(self): + ns = run_code(""" + X = 1 + def outer(): + def inner[X](): + global X + X = 2 + return inner + """ + ) + self.assertEqual(ns["X"], 1) + outer = ns["outer"] + outer()() + self.assertEqual(ns["X"], 2) + + def test_disallowed_expressions(self): + check_syntax_error(self, "type X = (yield)") + check_syntax_error(self, "type X = (yield from x)") + check_syntax_error(self, "type X = (await 42)") + check_syntax_error(self, "async def f(): type X = (yield)") + check_syntax_error(self, "type X = (y := 3)") + check_syntax_error(self, "class X[T: (yield)]: pass") + check_syntax_error(self, "class X[T: (yield from x)]: pass") + check_syntax_error(self, "class X[T: (await 42)]: pass") + check_syntax_error(self, "class X[T: (y := 3)]: pass") + check_syntax_error(self, "class X[T](y := Sequence[T]): pass") + check_syntax_error(self, "def f[T](y: (x := Sequence[T])): pass") + check_syntax_error(self, "class X[T]([(x := 3) for _ in range(2)] and B): pass") + check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") + check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") + + def test_incorrect_mro_explicit_object(self): + with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): + class My[X](object): ... + + +class TypeParamsNonlocalTest(unittest.TestCase): + def test_nonlocal_disallowed_01(self): + code = """ + def outer(): + X = 1 + def inner[X](): + nonlocal X + return X + """ + check_syntax_error(self, code) + + def test_nonlocal_disallowed_02(self): + code = """ + def outer2[T](): + def inner1(): + nonlocal T + """ + check_syntax_error(self, textwrap.dedent(code)) + + def test_nonlocal_disallowed_03(self): + code = """ + class Cls[T]: + nonlocal T + """ + check_syntax_error(self, textwrap.dedent(code)) + + def test_nonlocal_allowed(self): + code = """ + def func[T](): + T = "func" + def inner(): + nonlocal T + T = "inner" + inner() + assert T == "inner" + """ + ns = run_code(code) + func = ns["func"] + T, = func.__type_params__ + self.assertEqual(T.__name__, "T") + + +class TypeParamsAccessTest(unittest.TestCase): + def test_class_access_01(self): + ns = run_code(""" + class ClassA[A, B](dict[A, B]): + ... + """ + ) + cls = ns["ClassA"] + A, B = cls.__type_params__ + self.assertEqual(types.get_original_bases(cls), (dict[A, B], Generic[A, B])) + + def test_class_access_02(self): + ns = run_code(""" + class MyMeta[A, B](type): ... + class ClassA[A, B](metaclass=MyMeta[A, B]): + ... + """ + ) + meta = ns["MyMeta"] + cls = ns["ClassA"] + A1, B1 = meta.__type_params__ + A2, B2 = cls.__type_params__ + self.assertIsNot(A1, A2) + self.assertIsNot(B1, B2) + self.assertIs(type(cls), meta) + + def test_class_access_03(self): + code = """ + def my_decorator(a): + ... + @my_decorator(A) + class ClassA[A, B](): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_function_access_01(self): + ns = run_code(""" + def func[A, B](a: dict[A, B]): + ... + """ + ) + func = ns["func"] + A, B = func.__type_params__ + self.assertEqual(func.__annotations__["a"], dict[A, B]) + + def test_function_access_02(self): + code = """ + def func[A](a = list[A]()): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_function_access_03(self): + code = """ + def my_decorator(a): + ... + @my_decorator(A) + def func[A](): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_method_access_01(self): + ns = run_code(""" + class ClassA: + x = int + def func[T](self, a: x, b: T): + ... + """ + ) + cls = ns["ClassA"] + self.assertIs(cls.func.__annotations__["a"], int) + T, = cls.func.__type_params__ + self.assertIs(cls.func.__annotations__["b"], T) + + def test_nested_access_01(self): + ns = run_code(""" + class ClassA[A]: + def funcB[B](self): + class ClassC[C]: + def funcD[D](self): + return lambda: (A, B, C, D) + return ClassC + """ + ) + cls = ns["ClassA"] + A, = cls.__type_params__ + B, = cls.funcB.__type_params__ + classC = cls().funcB() + C, = classC.__type_params__ + D, = classC.funcD.__type_params__ + self.assertEqual(classC().funcD()(), (A, B, C, D)) + + def test_out_of_scope_01(self): + code = """ + class ClassA[T]: ... + x = T + """ + + with self.assertRaisesRegex(NameError, "name 'T' is not defined"): + run_code(code) + + def test_out_of_scope_02(self): + code = """ + class ClassA[A]: + def funcB[B](self): ... + + x = B + """ + + with self.assertRaisesRegex(NameError, "name 'B' is not defined"): + run_code(code) + + def test_class_scope_interaction_01(self): + ns = run_code(""" + class C: + x = 1 + def method[T](self, arg: x): pass + """) + cls = ns["C"] + self.assertEqual(cls.method.__annotations__["arg"], 1) + + def test_class_scope_interaction_02(self): + ns = run_code(""" + class C: + class Base: pass + class Child[T](Base): pass + """) + cls = ns["C"] + self.assertEqual(cls.Child.__bases__, (cls.Base, Generic)) + T, = cls.Child.__type_params__ + self.assertEqual(types.get_original_bases(cls.Child), (cls.Base, Generic[T])) + + def test_class_deref(self): + ns = run_code(""" + class C[T]: + T = "class" + type Alias = T + """) + cls = ns["C"] + self.assertEqual(cls.Alias.__value__, "class") + + def test_shadowing_nonlocal(self): + ns = run_code(""" + def outer[T](): + T = "outer" + def inner(): + nonlocal T + T = "inner" + return T + return lambda: T, inner + """) + outer = ns["outer"] + T, = outer.__type_params__ + self.assertEqual(T.__name__, "T") + getter, inner = outer() + self.assertEqual(getter(), "outer") + self.assertEqual(inner(), "inner") + self.assertEqual(getter(), "inner") + + def test_reference_previous_typevar(self): + def func[S, T: Sequence[S]](): + pass + + S, T = func.__type_params__ + self.assertEqual(T.__bound__, Sequence[S]) + + def test_super(self): + class Base: + def meth(self): + return "base" + + class Child(Base): + # Having int in the annotation ensures the class gets cells for both + # __class__ and __classdict__ + def meth[T](self, arg: int) -> T: + return super().meth() + "child" + + c = Child() + self.assertEqual(c.meth(1), "basechild") + + def test_type_alias_containing_lambda(self): + type Alias[T] = lambda: T + T, = Alias.__type_params__ + self.assertIs(Alias.__value__(), T) + + def test_class_base_containing_lambda(self): + # Test that scopes nested inside hidden functions work correctly + outer_var = "outer" + class Base[T]: ... + class Child[T](Base[lambda: (int, outer_var, T)]): ... + base, _ = types.get_original_bases(Child) + func, = get_args(base) + T, = Child.__type_params__ + self.assertEqual(func(), (int, "outer", T)) + + def test_comprehension_01(self): + type Alias[T: ([T for T in (T, [1])[1]], T)] = [T for T in T.__name__] + self.assertEqual(Alias.__value__, ["T"]) + T, = Alias.__type_params__ + self.assertEqual(T.__constraints__, ([1], T)) + + def test_comprehension_02(self): + type Alias[T: [lambda: T for T in (T, [1])[1]]] = [lambda: T for T in T.__name__] + func, = Alias.__value__ + self.assertEqual(func(), "T") + T, = Alias.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + + def test_gen_exp_in_nested_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner(make_base(T for _ in (1,)), make_base(T)): + pass + """ + C = run_code(code)["C"] + T, = C.__type_params__ + base1, base2 = C.Inner.__bases__ + self.assertEqual(list(base1.__arg__), [T]) + self.assertEqual(base2.__arg__, "class") + + def test_gen_exp_in_nested_generic_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner[U](make_base(T for _ in (1,)), make_base(T)): + pass + """ + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") + + def test_listcomp_in_nested_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner(make_base([T for _ in (1,)]), make_base(T)): + pass + """ + C = run_code(code)["C"] + T, = C.__type_params__ + base1, base2 = C.Inner.__bases__ + self.assertEqual(base1.__arg__, [T]) + self.assertEqual(base2.__arg__, "class") + + def test_listcomp_in_nested_generic_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner[U](make_base([T for _ in (1,)]), make_base(T)): + pass + """ + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") + + def test_gen_exp_in_generic_method(self): + code = """ + class C[T]: + T = "class" + def meth[U](x: (T for _ in (1,)), y: T): + pass + """ + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") + + def test_nested_scope_in_generic_alias(self): + code = """ + T = "global" + class C: + T = "class" + {} + """ + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", + ] + for case in cases: + with self.subTest(case=case): + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") + + def test_lambda_in_alias_in_class(self): + code = """ + T = "global" + class C: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertEqual(C.Alias.__value__(), "global") + + def test_lambda_in_alias_in_generic_class(self): + code = """ + class C[T]: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertIs(C.Alias.__value__(), C.__type_params__[0]) + + def test_lambda_in_generic_alias_in_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + T = U = "global" + class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertEqual(U, "global") + + def test_lambda_in_generic_alias_in_generic_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + class C[T, U]: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertIs(U, C.__type_params__[1]) + + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) + + +def make_base(arg): + class Base: + __arg__ = arg + return Base + + +def global_generic_func[T](): + pass + +class GlobalGenericClass[T]: + pass + + +class TypeParamsLazyEvaluationTest(unittest.TestCase): + def test_qualname(self): + class Foo[T]: + pass + + def func[T](): + pass + + self.assertEqual(Foo.__qualname__, "TypeParamsLazyEvaluationTest.test_qualname.<locals>.Foo") + self.assertEqual(func.__qualname__, "TypeParamsLazyEvaluationTest.test_qualname.<locals>.func") + self.assertEqual(global_generic_func.__qualname__, "global_generic_func") + self.assertEqual(GlobalGenericClass.__qualname__, "GlobalGenericClass") + + def test_recursive_class(self): + class Foo[T: Foo, U: (Foo, Foo)]: + pass + + type_params = Foo.__type_params__ + self.assertEqual(len(type_params), 2) + self.assertEqual(type_params[0].__name__, "T") + self.assertIs(type_params[0].__bound__, Foo) + self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) + + self.assertEqual(type_params[1].__name__, "U") + self.assertIs(type_params[1].__bound__, None) + self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) + + def test_evaluation_error(self): + class Foo[T: Undefined, U: (Undefined,)]: + pass + + type_params = Foo.__type_params__ + with self.assertRaises(NameError): + type_params[0].__bound__ + self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) + with self.assertRaises(NameError): + type_params[1].__constraints__ + + Undefined = "defined" + self.assertEqual(type_params[0].__bound__, "defined") + self.assertEqual(type_params[0].__constraints__, ()) + + self.assertIs(type_params[1].__bound__, None) + self.assertEqual(type_params[1].__constraints__, ("defined",)) + + +class TypeParamsClassScopeTest(unittest.TestCase): + def test_alias(self): + class X: + T = int + type U = T + self.assertIs(X.U.__value__, int) + + ns = run_code(""" + glb = "global" + class X: + cls = "class" + type U = (glb, cls) + """) + cls = ns["X"] + self.assertEqual(cls.U.__value__, ("global", "class")) + + def test_bound(self): + class X: + T = int + def foo[U: T](self): ... + self.assertIs(X.foo.__type_params__[0].__bound__, int) + + ns = run_code(""" + glb = "global" + class X: + cls = "class" + def foo[T: glb, U: cls](self): ... + """) + cls = ns["X"] + T, U = cls.foo.__type_params__ + self.assertEqual(T.__bound__, "global") + self.assertEqual(U.__bound__, "class") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'int'> is not <class 'float'> + def test_modified_later(self): + class X: + T = int + def foo[U: T](self): ... + type Alias = T + X.T = float + self.assertIs(X.foo.__type_params__[0].__bound__, float) + self.assertIs(X.Alias.__value__, float) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global + def test_binding_uses_global(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + type Alias = x + val = Alias.__value__ + def meth[T: x](self, arg: x): ... + bound = meth.__type_params__[0].__bound__ + annotation = meth.__annotations__["arg"] + x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.val, "global") + self.assertEqual(cls.bound, "global") + self.assertEqual(cls.annotation, "global") + + def test_no_binding_uses_nonlocal(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + type Alias = x + val = Alias.__value__ + def meth[T: x](self, arg: x): ... + bound = meth.__type_params__[0].__bound__ + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.val, "nonlocal") + self.assertEqual(cls.bound, "nonlocal") + self.assertEqual(cls.meth.__annotations__["arg"], "nonlocal") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global + def test_explicit_global(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + global x + type Alias = x + Cls.x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global") + + def test_explicit_global_with_no_static_bound(self): + ns = run_code(""" + def outer(): + class Cls: + global x + type Alias = x + Cls.x = "class" + return Cls + """) + ns["x"] = "global" + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global from class + def test_explicit_global_with_assignment(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + global x + type Alias = x + x = "global from class" + Cls.x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global from class") + + def test_explicit_nonlocal(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + nonlocal x + type Alias = x + x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "class") + + def test_nested_free(self): + ns = run_code(""" + def f(): + T = str + class C: + T = int + class D[U](T): + x = T + return C + """) + C = ns["f"]() + self.assertIn(int, C.D.__bases__) + self.assertIs(C.D.x, str) + + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + +class TypeParamsManglingTest(unittest.TestCase): + def test_mangling(self): + class Foo[__T]: + param = __T + def meth[__U](self, arg: __T, arg2: __U): + return (__T, __U) + type Alias[__V] = (__T, __V) + + T = Foo.__type_params__[0] + self.assertEqual(T.__name__, "__T") + U = Foo.meth.__type_params__[0] + self.assertEqual(U.__name__, "__U") + V = Foo.Alias.__type_params__[0] + self.assertEqual(V.__name__, "__V") + + anno = Foo.meth.__annotations__ + self.assertIs(anno["arg"], T) + self.assertIs(anno["arg2"], U) + self.assertEqual(Foo().meth(1, 2), (T, U)) + + self.assertEqual(Foo.Alias.__value__, (T, V)) + + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + + +class TypeParamsComplexCallsTest(unittest.TestCase): + def test_defaults(self): + # Generic functions with both defaults and kwdefaults trigger a specific code path + # in the compiler. + def func[T](a: T = "a", *, b: T = "b"): + return (a, b) + + T, = func.__type_params__ + self.assertIs(func.__annotations__["a"], T) + self.assertIs(func.__annotations__["b"], T) + self.assertEqual(func(), ("a", "b")) + self.assertEqual(func(1), (1, "b")) + self.assertEqual(func(b=2), ("a", 2)) + + def test_complex_base(self): + class Base: + def __init_subclass__(cls, **kwargs) -> None: + cls.kwargs = kwargs + + kwargs = {"c": 3} + # Base classes with **kwargs trigger a different code path in the compiler. + class C[T](Base, a=1, b=2, **kwargs): + pass + + T, = C.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C.kwargs, {"a": 1, "b": 2, "c": 3}) + self.assertEqual(C.__bases__, (Base, Generic)) + + bases = (Base,) + class C2[T](*bases, **kwargs): + pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.kwargs, {"c": 3}) + self.assertEqual(C2.__bases__, (Base, Generic)) + + def test_starargs_base(self): + class C1[T](*()): pass + + T, = C1.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C1.__bases__, (Generic,)) + + class Base: pass + bases = [Base] + class C2[T](*bases): pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.__bases__, (Base, Generic)) + + +class TypeParamsTraditionalTypeVarsTest(unittest.TestCase): + def test_traditional_01(self): + code = """ + from typing import Generic + class ClassA[T](Generic[T]): ... + """ + + with self.assertRaisesRegex(TypeError, r"Cannot inherit from Generic\[...\] multiple times."): + run_code(code) + + def test_traditional_02(self): + from typing import TypeVar + S = TypeVar("S") + with self.assertRaises(TypeError): + class ClassA[T](dict[T, S]): ... + + def test_traditional_03(self): + # This does not generate a runtime error, but it should be + # flagged as an error by type checkers. + from typing import TypeVar + S = TypeVar("S") + def func[T](a: T, b: S) -> T | S: + return a + + +class TypeParamsTypeVarTest(unittest.TestCase): + def test_typevar_01(self): + def func1[A: str, B: str | int, C: (int, str)](): + return (A, B, C) + + a, b, c = func1() + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__bound__, str) + self.assertTrue(a.__infer_variance__) + self.assertFalse(a.__covariant__) + self.assertFalse(a.__contravariant__) + + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__bound__, str | int) + self.assertTrue(b.__infer_variance__) + self.assertFalse(b.__covariant__) + self.assertFalse(b.__contravariant__) + + self.assertIsInstance(c, TypeVar) + self.assertEqual(c.__bound__, None) + self.assertEqual(c.__constraints__, (int, str)) + self.assertTrue(c.__infer_variance__) + self.assertFalse(c.__covariant__) + self.assertFalse(c.__contravariant__) + + def test_typevar_generator(self): + def get_generator[A](): + def generator1[C](): + yield C + + def generator2[B](): + yield A + yield B + yield from generator1() + return generator2 + + gen = get_generator() + + a, b, c = [x for x in gen()] + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__name__, "A") + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__name__, "B") + self.assertIsInstance(c, TypeVar) + self.assertEqual(c.__name__, "C") + + def test_typevar_coroutine(self): + def get_coroutine[A](): + async def coroutine[B](): + return (A, B) + return coroutine + + co = get_coroutine() + + a, b = run_no_yield_async_fn(co) + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__name__, "A") + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__name__, "B") + + +class TypeParamsTypeVarTupleTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot use bound with TypeVarTuple" does not match "invalid syntax (<test string>, line 1)" + def test_typevartuple_01(self): + code = """def func1[*A: str](): pass""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """def func1[*A: (int, str)](): pass""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + code = """class X[*A: str]: pass""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """class X[*A: (int, str)]: pass""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + code = """type X[*A: str] = int""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """type X[*A: (int, str)] = int""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + + def test_typevartuple_02(self): + def func1[*A](): + return A + + a = func1() + self.assertIsInstance(a, TypeVarTuple) + + +class TypeParamsTypeVarParamSpecTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot use bound with ParamSpec" does not match "invalid syntax (<test string>, line 1)" + def test_paramspec_01(self): + code = """def func1[**A: str](): pass""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """def func1[**A: (int, str)](): pass""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + code = """class X[**A: str]: pass""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """class X[**A: (int, str)]: pass""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + code = """type X[**A: str] = int""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """type X[**A: (int, str)] = int""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + + def test_paramspec_02(self): + def func1[**A](): + return A + + a = func1() + self.assertIsInstance(a, ParamSpec) + self.assertTrue(a.__infer_variance__) + self.assertFalse(a.__covariant__) + self.assertFalse(a.__contravariant__) + + +class TypeParamsTypeParamsDunder(unittest.TestCase): + def test_typeparams_dunder_class_01(self): + class Outer[A, B]: + class Inner[C, D]: + @staticmethod + def get_typeparams(): + return A, B, C, D + + a, b, c, d = Outer.Inner.get_typeparams() + self.assertEqual(Outer.__type_params__, (a, b)) + self.assertEqual(Outer.Inner.__type_params__, (c, d)) + + self.assertEqual(Outer.__parameters__, (a, b)) + self.assertEqual(Outer.Inner.__parameters__, (c, d)) + + def test_typeparams_dunder_class_02(self): + class ClassA: + pass + + self.assertEqual(ClassA.__type_params__, ()) + + def test_typeparams_dunder_class_03(self): + code = """ + class ClassA[A](): + pass + ClassA.__type_params__ = () + params = ClassA.__type_params__ + """ + + ns = run_code(code) + self.assertEqual(ns["params"], ()) + + def test_typeparams_dunder_function_01(self): + def outer[A, B](): + def inner[C, D](): + return A, B, C, D + + return inner + + inner = outer() + a, b, c, d = inner() + self.assertEqual(outer.__type_params__, (a, b)) + self.assertEqual(inner.__type_params__, (c, d)) + + def test_typeparams_dunder_function_02(self): + def func1(): + pass + + self.assertEqual(func1.__type_params__, ()) + + def test_typeparams_dunder_function_03(self): + code = """ + def func[A](): + pass + func.__type_params__ = () + """ + + ns = run_code(code) + self.assertEqual(ns["func"].__type_params__, ()) + + + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +def func1[X](x: X) -> X: ... +def func2[X, Y](x: X | Y) -> X | Y: ... +def func3[X, *Y, **Z](x: X, y: tuple[*Y], z: Z) -> X: ... +def func4[X: int, Y: (bytes, str)](x: X, y: Y) -> X | Y: ... + +class Class1[X]: ... +class Class2[X, Y]: ... +class Class3[X, *Y, **Z]: ... +class Class4[X: int, Y: (bytes, str)]: ... + + +class TypeParamsPickleTest(unittest.TestCase): + def test_pickling_functions(self): + things_to_test = [ + func1, + func2, + func3, + func4, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + def test_pickling_classes(self): + things_to_test = [ + Class1, + Class1[int], + Class1[T], + + Class2, + Class2[int, T], + Class2[T, int], + Class2[int, str], + + Class3, + Class3[int, T, str, bytes, [float, object, T]], + + Class4, + Class4[int, bytes], + Class4[T, bytes], + Class4[int, T], + Class4[T, T], + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + for klass in things_to_test: + real_class = getattr(klass, '__origin__', klass) + thing = klass() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + # These instances are not equal, + # but class check is good enough: + self.assertIsInstance(pickle.loads(pickled), real_class) + + +class TypeParamsWeakRefTest(unittest.TestCase): + def test_weakrefs(self): + T = TypeVar('T') + P = ParamSpec('P') + class OldStyle(Generic[T]): + pass + + class NewStyle[T]: + pass + + cases = [ + T, + TypeVar('T', bound=int), + P, + P.args, + P.kwargs, + TypeVarTuple('Ts'), + OldStyle, + OldStyle[int], + OldStyle(), + NewStyle, + NewStyle[int], + NewStyle(), + Generic[T], + ] + for case in cases: + with self.subTest(case=case): + weakref.ref(case) + + +class TypeParamsRuntimeTest(unittest.TestCase): + def test_name_error(self): + # gh-109118: This crashed the interpreter due to a refcounting bug + code = """ + class name_2[name_5]: + class name_4[name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + # Crashed with a slightly different stack trace + code = """ + class name_2[name_5]: + class name_4[name_5: name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + def test_broken_class_namespace(self): + code = """ + class WeirdMapping(dict): + def __missing__(self, key): + if key == "T": + raise RuntimeError + raise KeyError(key) + + class Meta(type): + def __prepare__(name, bases): + return WeirdMapping() + + class MyClass[V](metaclass=Meta): + class Inner[U](T): + pass + """ + with self.assertRaises(RuntimeError): + run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") + + +class TestEvaluateFunctions(unittest.TestCase): + def test_general(self): + type Alias = int + Alias2 = TypeAliasType("Alias2", int) + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + T2 = TypeVar("T2", bound=int, default=int) + P2 = ParamSpec("P2", default=int) + Ts2 = TypeVarTuple("Ts2", default=int) + cases = [ + Alias.evaluate_value, + Alias2.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T2.evaluate_bound, + T2.evaluate_default, + P2.evaluate_default, + Ts2.evaluate_default, + ] + for case in cases: + with self.subTest(case=case): + self.assertIs(case(1), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + + def test_constraints(self): + def f[T: (int, str)](): pass + T, = f.__type_params__ + T2 = TypeVar("T2", int, str) + for case in [T, T2]: + with self.subTest(case=case): + self.assertEqual(case.evaluate_constraints(1), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') + + def test_const_evaluator(self): + T = TypeVar("T", bound=int) + self.assertEqual(repr(T.evaluate_bound), "<constevaluator <class 'int'>>") + + ConstEvaluator = type(T.evaluate_bound) + + with self.assertRaisesRegex(TypeError, r"cannot create '_typing\._ConstEvaluator' instances"): + ConstEvaluator() # This used to segfault. + with self.assertRaisesRegex(TypeError, r"cannot set 'attribute' attribute of immutable type '_typing\._ConstEvaluator'"): + ConstEvaluator.attribute = 1 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index fdcb9060e83..06099b87427 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,19 +1,35 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only +from test.support import ( + run_with_locale, cpython_only, no_rerun, + MISSING_C_DOCSTRINGS, EqualToForwardRef, check_disallow_instantiation, +) +from test.support.script_helper import assert_python_ok +from test.support.import_helper import import_fresh_module + import collections.abc -from collections import namedtuple +from collections import namedtuple, UserDict import copy +# XXX: RUSTPYTHON +try: + import _datetime +except ImportError: + _datetime = None import gc import inspect import pickle import locale import sys +import textwrap import types import unittest.mock import weakref import typing +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + +c_types = import_fresh_module('types', fresh=['_types']) +py_types = import_fresh_module('types', blocked=['_types']) T = typing.TypeVar("T") @@ -29,6 +45,29 @@ def clear_typing_caches(): class TypesTests(unittest.TestCase): + @unittest.skipUnless(c_types, "TODO: RUSTPYTHON; requires _types module") + def test_names(self): + c_only_names = {'CapsuleType'} + ignored = {'new_class', 'resolve_bases', 'prepare_class', + 'get_original_bases', 'DynamicClassAttribute', 'coroutine'} + + for name in c_types.__all__: + if name not in c_only_names | ignored: + self.assertIs(getattr(c_types, name), getattr(py_types, name)) + + all_names = ignored | { + 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', + 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', + 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', + 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', + 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', + 'ModuleType', 'NoneType', 'NotImplementedType', 'SimpleNamespace', + 'TracebackType', 'UnionType', 'WrapperDescriptorType', + } + self.assertEqual(all_names, set(c_types.__all__)) + self.assertEqual(all_names - c_only_names, set(py_types.__all__)) + def test_truth_values(self): if None: self.fail('None is true instead of false') if 0: self.fail('0 is true instead of false') @@ -226,8 +265,8 @@ def test_type_function(self): def test_int__format__(self): def test(i, format_spec, result): # just make sure we have the unified type for integers - assert type(i) == int - assert type(format_spec) == str + self.assertIs(type(i), int) + self.assertIs(type(format_spec), str) self.assertEqual(i.__format__(format_spec), result) test(123456789, 'd', '123456789') @@ -392,8 +431,8 @@ def test(i, format_spec, result): test(123456, "1=20", '11111111111111123456') test(123456, "*=20", '**************123456') - @unittest.expectedFailure - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @unittest.expectedFailure # TODO: RUSTPYTHON + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -402,7 +441,8 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @unittest.expectedFailure # TODO: RUSTPYTHON + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers @@ -420,9 +460,6 @@ def test_int__format__locale(self): self.assertEqual(len(format(0, rfmt)), len(format(x, rfmt))) self.assertEqual(len(format(0, lfmt)), len(format(x, lfmt))) self.assertEqual(len(format(0, cfmt)), len(format(x, cfmt))) - - if sys.platform != "darwin": - test_int__format__locale = unittest.expectedFailure(test_int__format__locale) def test_float__format__(self): def test(f, format_spec, result): @@ -489,8 +526,8 @@ def test(f, format_spec, result): # and a number after the decimal. This is tricky, because # a totally empty format specifier means something else. # So, just use a sign flag - test(1e200, '+g', '+1e+200') - test(1e200, '+', '+1e+200') + test(1.25e200, '+g', '+1.25e+200') + test(1.25e200, '+', '+1.25e+200') test(1.1e200, '+g', '+1.1e+200') test(1.1e200, '+', '+1.1e+200') @@ -591,26 +628,27 @@ def test_format_spec_errors(self): for code in 'xXobns': self.assertRaises(ValueError, format, 0, ',' + code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_internal_sizes(self): self.assertGreater(object.__basicsize__, 0) self.assertGreater(tuple.__itemsize__, 0) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slot_wrapper_types(self): self.assertIsInstance(object.__init__, types.WrapperDescriptorType) self.assertIsInstance(object.__str__, types.WrapperDescriptorType) self.assertIsInstance(object.__lt__, types.WrapperDescriptorType) self.assertIsInstance(int.__lt__, types.WrapperDescriptorType) - # TODO: RUSTPYTHON No signature found in builtin method __get__ of 'method_descriptor' objects. - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; No signature found in builtin method __get__ of 'method_descriptor' objects. + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_dunder_get_signature(self): sig = inspect.signature(object.__init__.__get__) self.assertEqual(list(sig.parameters), ["instance", "owner"]) # gh-93021: Second parameter is optional self.assertIs(sig.parameters["owner"].default, None) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_method_wrapper_types(self): self.assertIsInstance(object().__init__, types.MethodWrapperType) self.assertIsInstance(object().__str__, types.MethodWrapperType) @@ -627,6 +665,26 @@ def test_method_descriptor_types(self): self.assertIsInstance(int.from_bytes, types.BuiltinMethodType) self.assertIsInstance(int.__new__, types.BuiltinMethodType) + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_queue' + def test_method_descriptor_crash(self): + # gh-132747: The default __get__() implementation in C was unable + # to handle a second argument of None when called from Python + import _io + import io + import _queue + + to_check = [ + # (method, instance) + (_io._TextIOBase.read, io.StringIO()), + (_queue.SimpleQueue.put, _queue.SimpleQueue()), + (str.capitalize, "nobody expects the spanish inquisition") + ] + + for method, instance in to_check: + with self.subTest(method=method, instance=instance): + bound = method.__get__(instance) + self.assertIsInstance(bound, types.BuiltinMethodType) + def test_ellipsis_type(self): self.assertIsInstance(Ellipsis, types.EllipsisType) @@ -644,6 +702,29 @@ def test_traceback_and_frame_types(self): self.assertIsInstance(exc.__traceback__, types.TracebackType) self.assertIsInstance(exc.__traceback__.tb_frame, types.FrameType) + # XXX: RUSTPYTHON + @unittest.skipUnless(_datetime, "requires _datetime module") + def test_capsule_type(self): + self.assertIsInstance(_datetime.datetime_CAPI, types.CapsuleType) + + def test_call_unbound_crash(self): + # GH-131998: The specialized instruction would get tricked into dereferencing + # a bound "self" that didn't exist if subsequently called unbound. + code = """if True: + + def call(part): + [] + ([] + []) + part.pop() + + for _ in range(3): + call(['a']) + try: + call(list) + except TypeError: + pass + """ + assert_python_ok("-c", code) + class UnionTests(unittest.TestCase): @@ -706,15 +787,54 @@ def test_or_types_operator(self): y = int | bool with self.assertRaises(TypeError): x < y - # Check that we don't crash if typing.Union does not have a tuple in __args__ - y = typing.Union[str, int] - y.__args__ = [str, int] - self.assertEqual(x, y) def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) self.assertEqual(hash(int | str), hash(typing.Union[int, str])) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual((A | B).__args__, (A, B)) + union1 = A | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union1) + + union2 = int | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union2) + + union3 = A | int + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union3) + + def test_unhashable_becomes_hashable(self): + is_hashable = False + class UnhashableMeta(type): + def __hash__(self): + if is_hashable: + return 1 + else: + raise TypeError("not hashable") + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + union = A | B + self.assertEqual(union.__args__, (A, B)) + + with self.assertRaisesRegex(TypeError, "not hashable"): + hash(union) + + is_hashable = True + + with self.assertRaisesRegex(TypeError, "union contains 2 unhashable elements"): + hash(union) + def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): with self.subTest(x=x): @@ -722,15 +842,15 @@ def test_instancecheck_and_subclasscheck(self): self.assertIsInstance(True, x) self.assertIsInstance('a', x) self.assertNotIsInstance(None, x) - self.assertTrue(issubclass(int, x)) - self.assertTrue(issubclass(bool, x)) - self.assertTrue(issubclass(str, x)) - self.assertFalse(issubclass(type(None), x)) + self.assertIsSubclass(int, x) + self.assertIsSubclass(bool, x) + self.assertIsSubclass(str, x) + self.assertNotIsSubclass(type(None), x) for x in (int | None, typing.Union[int, None]): with self.subTest(x=x): self.assertIsInstance(None, x) - self.assertTrue(issubclass(type(None), x)) + self.assertIsSubclass(type(None), x) for x in ( int | collections.abc.Mapping, @@ -739,8 +859,8 @@ def test_instancecheck_and_subclasscheck(self): with self.subTest(x=x): self.assertIsInstance({}, x) self.assertNotIsInstance((), x) - self.assertTrue(issubclass(dict, x)) - self.assertFalse(issubclass(list, x)) + self.assertIsSubclass(dict, x) + self.assertNotIsSubclass(list, x) def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -752,7 +872,7 @@ def test_instancecheck_and_subclasscheck_order(self): for x in will_resolve: with self.subTest(x=x): self.assertIsInstance(1, x) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) wont_resolve = ( T | int, @@ -785,13 +905,13 @@ class BadMeta(type): def __subclasscheck__(cls, sub): 1/0 x = int | BadMeta('A', (), {}) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) self.assertRaises(ZeroDivisionError, issubclass, list, x) def test_or_type_operator_with_TypeVar(self): TV = typing.TypeVar('T') - assert TV | str == typing.Union[TV, str] - assert str | TV == typing.Union[str, TV] + self.assertEqual(TV | str, typing.Union[TV, str]) + self.assertEqual(str | TV, typing.Union[str, TV]) self.assertIs((int | TV)[int], int) self.assertIs((TV | int)[int], int) @@ -836,8 +956,6 @@ def test_union_parameter_chaining(self): self.assertEqual((list[T] | list[S])[int, T], list[int] | list[T]) self.assertEqual((list[T] | list[S])[int, int], list[int]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union_parameter_substitution(self): def eq(actual, expected, typed=True): self.assertEqual(actual, expected) @@ -897,54 +1015,83 @@ def test_or_type_operator_with_forward(self): ForwardBefore = 'Forward' | T def forward_after(x: ForwardAfter[int]) -> None: ... def forward_before(x: ForwardBefore[int]) -> None: ... - assert typing.get_args(typing.get_type_hints(forward_after)['x']) == (int, Forward) - assert typing.get_args(typing.get_type_hints(forward_before)['x']) == (int, Forward) + self.assertEqual(typing.get_args(typing.get_type_hints(forward_after)['x']), + (int, Forward)) + self.assertEqual(typing.get_args(typing.get_type_hints(forward_before)['x']), + (Forward, int)) def test_or_type_operator_with_Protocol(self): class Proto(typing.Protocol): def meth(self) -> int: ... - assert Proto | str == typing.Union[Proto, str] + self.assertEqual(Proto | str, typing.Union[Proto, str]) def test_or_type_operator_with_Alias(self): - assert list | str == typing.Union[list, str] - assert typing.List | str == typing.Union[typing.List, str] + self.assertEqual(list | str, typing.Union[list, str]) + self.assertEqual(typing.List | str, typing.Union[typing.List, str]) def test_or_type_operator_with_NamedTuple(self): - NT=namedtuple('A', ['B', 'C', 'D']) - assert NT | str == typing.Union[NT,str] + NT = namedtuple('A', ['B', 'C', 'D']) + self.assertEqual(NT | str, typing.Union[NT, str]) def test_or_type_operator_with_TypedDict(self): class Point2D(typing.TypedDict): x: int y: int label: str - assert Point2D | str == typing.Union[Point2D, str] + self.assertEqual(Point2D | str, typing.Union[Point2D, str]) def test_or_type_operator_with_NewType(self): UserId = typing.NewType('UserId', int) - assert UserId | str == typing.Union[UserId, str] + self.assertEqual(UserId | str, typing.Union[UserId, str]) def test_or_type_operator_with_IO(self): - assert typing.IO | str == typing.Union[typing.IO, str] + self.assertEqual(typing.IO | str, typing.Union[typing.IO, str]) def test_or_type_operator_with_SpecialForm(self): - assert typing.Any | str == typing.Union[typing.Any, str] - assert typing.NoReturn | str == typing.Union[typing.NoReturn, str] - assert typing.Optional[int] | str == typing.Union[typing.Optional[int], str] - assert typing.Optional[int] | str == typing.Union[int, str, None] - assert typing.Union[int, bool] | str == typing.Union[int, bool, str] + self.assertEqual(typing.Any | str, typing.Union[typing.Any, str]) + self.assertEqual(typing.NoReturn | str, typing.Union[typing.NoReturn, str]) + self.assertEqual(typing.Optional[int] | str, typing.Union[typing.Optional[int], str]) + self.assertEqual(typing.Optional[int] | str, typing.Union[int, str, None]) + self.assertEqual(typing.Union[int, bool] | str, typing.Union[int, bool, str]) + + def test_or_type_operator_with_Literal(self): + Literal = typing.Literal + self.assertEqual((Literal[1] | Literal[2]).__args__, + (Literal[1], Literal[2])) + + self.assertEqual((Literal[0] | Literal[False]).__args__, + (Literal[0], Literal[False])) + self.assertEqual((Literal[1] | Literal[True]).__args__, + (Literal[1], Literal[True])) + + self.assertEqual(Literal[1] | Literal[1], Literal[1]) + self.assertEqual(Literal['a'] | Literal['a'], Literal['a']) + + import enum + class Ints(enum.IntEnum): + A = 0 + B = 1 + + self.assertEqual(Literal[Ints.A] | Literal[Ints.A], Literal[Ints.A]) + self.assertEqual(Literal[Ints.B] | Literal[Ints.B], Literal[Ints.B]) + + self.assertEqual((Literal[Ints.B] | Literal[Ints.A]).__args__, + (Literal[Ints.B], Literal[Ints.A])) + + self.assertEqual((Literal[0] | Literal[Ints.A]).__args__, + (Literal[0], Literal[Ints.A])) + self.assertEqual((Literal[1] | Literal[Ints.B]).__args__, + (Literal[1], Literal[Ints.B])) def test_or_type_repr(self): - assert repr(int | str) == "int | str" - assert repr((int | str) | list) == "int | str | list" - assert repr(int | (str | list)) == "int | str | list" - assert repr(int | None) == "int | None" - assert repr(int | type(None)) == "int | None" - assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]" - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(repr(int | str), "int | str") + self.assertEqual(repr((int | str) | list), "int | str | list") + self.assertEqual(repr(int | (str | list)), "int | str | list") + self.assertEqual(repr(int | None), "int | None") + self.assertEqual(repr(int | type(None)), "int | None") + self.assertEqual(repr(int | typing.GenericAlias(list, int)), "int | list[int]") + def test_or_type_operator_with_genericalias(self): a = list[int] b = list[str] @@ -965,9 +1112,14 @@ def __eq__(self, other): return 1 / 0 bt = BadType('bt', (), {}) + bt2 = BadType('bt2', (), {}) # Comparison should fail and errors should propagate out for bad types. + union1 = int | bt + union2 = int | bt2 + with self.assertRaises(ZeroDivisionError): + union1 == union2 with self.assertRaises(ZeroDivisionError): - list[int] | list[bt] + bt | bt2 union_ga = (list[str] | int, collections.abc.Callable[..., str] | int, d | int) @@ -1010,6 +1162,19 @@ def test_or_type_operator_reference_cycle(self): self.assertLessEqual(sys.gettotalrefcount() - before, leeway, msg='Check for union reference leak.') + def test_instantiation(self): + check_disallow_instantiation(self, types.UnionType) + self.assertIs(int, types.UnionType[int]) + self.assertIs(int, types.UnionType[int, int]) + self.assertEqual(int | str, types.UnionType[int, str]) + + for obj in ( + int | typing.ForwardRef("str"), + typing.Union[int, "str"], + ): + self.assertIsInstance(obj, types.UnionType) + self.assertEqual(obj.__args__, (int, EqualToForwardRef("str"))) + class MappingProxyTests(unittest.TestCase): mappingproxy = types.MappingProxyType @@ -1199,8 +1364,7 @@ def test_copy(self): self.assertEqual(view['key1'], 70) self.assertEqual(copy['key1'], 27) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_union(self): mapping = {'a': 0, 'b': 1, 'c': 2} view = self.mappingproxy(mapping) @@ -1217,6 +1381,16 @@ def test_union(self): self.assertDictEqual(mapping, {'a': 0, 'b': 1, 'c': 2}) self.assertDictEqual(other, {'c': 3, 'p': 0}) + def test_hash(self): + class HashableDict(dict): + def __hash__(self): + return 3844817361 + view = self.mappingproxy({'a': 1, 'b': 2}) + self.assertRaises(TypeError, hash, view) + mapping = HashableDict({'a': 1, 'b': 2}) + view = self.mappingproxy(mapping) + self.assertEqual(hash(view), hash(mapping)) + class ClassCreationTests(unittest.TestCase): @@ -1240,7 +1414,7 @@ def test_new_class_basics(self): def test_new_class_subclass(self): C = types.new_class("C", (int,)) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) def test_new_class_meta(self): Meta = self.Meta @@ -1285,7 +1459,7 @@ def func(ns): bases=(int,), kwds=dict(metaclass=Meta, z=2), exec_body=func) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) self.assertIsInstance(C, Meta) self.assertEqual(C.x, 0) self.assertEqual(C.y, 1) @@ -1364,6 +1538,80 @@ class C: pass D = types.new_class('D', (A(), C, B()), {}) self.assertEqual(D.__bases__, (A1, A2, A3, C, B1, B2)) + def test_get_original_bases(self): + T = typing.TypeVar('T') + class A: pass + class B(typing.Generic[T]): pass + class C(B[int]): pass + class D(B[str], float): pass + + self.assertEqual(types.get_original_bases(A), (object,)) + self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) + self.assertEqual(types.get_original_bases(C), (B[int],)) + self.assertEqual(types.get_original_bases(int), (object,)) + self.assertEqual(types.get_original_bases(D), (B[str], float)) + + class E(list[T]): pass + class F(list[int]): pass + + self.assertEqual(types.get_original_bases(E), (list[T],)) + self.assertEqual(types.get_original_bases(F), (list[int],)) + + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + + class ClassBasedNamedTuple(typing.NamedTuple): + x: int + + class GenericNamedTuple(typing.NamedTuple, typing.Generic[T]): + x: T + + CallBasedNamedTuple = typing.NamedTuple("CallBasedNamedTuple", [("x", int)]) + + self.assertIs( + types.get_original_bases(ClassBasedNamedTuple)[0], typing.NamedTuple + ) + self.assertEqual( + types.get_original_bases(GenericNamedTuple), + (typing.NamedTuple, typing.Generic[T]) + ) + self.assertIs( + types.get_original_bases(CallBasedNamedTuple)[0], typing.NamedTuple + ) + + class ClassBasedTypedDict(typing.TypedDict): + x: int + + class GenericTypedDict(typing.TypedDict, typing.Generic[T]): + x: T + + CallBasedTypedDict = typing.TypedDict("CallBasedTypedDict", {"x": int}) + + self.assertIs( + types.get_original_bases(ClassBasedTypedDict)[0], + typing.TypedDict + ) + self.assertEqual( + types.get_original_bases(GenericTypedDict), + (typing.TypedDict, typing.Generic[T]) + ) + self.assertIs( + types.get_original_bases(CallBasedTypedDict)[0], + typing.TypedDict + ) + + with self.assertRaisesRegex(TypeError, "Expected an instance of type"): + types.get_original_bases(object()) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -1624,25 +1872,81 @@ class Model(metaclass=ModelBase): with self.assertRaises(RuntimeWarning): type("SouthPonies", (Model,), {}) + def test_subclass_inherited_slot_update(self): + # gh-132284: Make sure slot update still works after fix. + # Note that after assignment to D.__getitem__ the actual C slot will + # never go back to dict_subscript as it was on class type creation but + # rather be set to slot_mp_subscript, unfortunately there is no way to + # check that here. + + class D(dict): + pass + + d = D({None: None}) + self.assertIs(d[None], None) + D.__getitem__ = lambda self, item: 42 + self.assertEqual(d[None], 42) + D.__getitem__ = dict.__getitem__ + self.assertIs(d[None], None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'tuple'> != <class 'test.test_types.ClassCreationTests.test_tu[41 chars]ass'> + def test_tuple_subclass_as_bases(self): + # gh-132176: it used to crash on using + # tuple subclass for as base classes. + class TupleSubclass(tuple): pass + + typ = type("typ", TupleSubclass((int, object)), {}) + self.assertEqual(typ.__bases__, (int, object)) + self.assertEqual(type(typ.__bases__), TupleSubclass) + class SimpleNamespaceTests(unittest.TestCase): def test_constructor(self): - ns1 = types.SimpleNamespace() - ns2 = types.SimpleNamespace(x=1, y=2) - ns3 = types.SimpleNamespace(**dict(x=1, y=2)) + def check(ns, expected): + self.assertEqual(len(ns.__dict__), len(expected)) + self.assertEqual(vars(ns), expected) + # check order + self.assertEqual(list(vars(ns).items()), list(expected.items())) + for name in expected: + self.assertEqual(getattr(ns, name), expected[name]) + + check(types.SimpleNamespace(), {}) + check(types.SimpleNamespace(x=1, y=2), {'x': 1, 'y': 2}) + check(types.SimpleNamespace(**dict(x=1, y=2)), {'x': 1, 'y': 2}) + check(types.SimpleNamespace({'x': 1, 'y': 2}, x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace([['x', 1], ['y', 2]], x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace(UserDict({'x': 1, 'y': 2}), x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace({'x': 1, 'y': 2}), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([['x', 1], ['y', 2]]), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([], x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace({}, x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace([]), {}) + check(types.SimpleNamespace({}), {}) with self.assertRaises(TypeError): - types.SimpleNamespace(1, 2, 3) + types.SimpleNamespace([], []) # too many positional arguments with self.assertRaises(TypeError): - types.SimpleNamespace(**{1: 2}) - - self.assertEqual(len(ns1.__dict__), 0) - self.assertEqual(vars(ns1), {}) - self.assertEqual(len(ns2.__dict__), 2) - self.assertEqual(vars(ns2), {'y': 2, 'x': 1}) - self.assertEqual(len(ns3.__dict__), 2) - self.assertEqual(vars(ns3), {'y': 2, 'x': 1}) + types.SimpleNamespace(1) # not a mapping or iterable + with self.assertRaises(TypeError): + types.SimpleNamespace([1]) # non-iterable + with self.assertRaises(ValueError): + types.SimpleNamespace([['x']]) # not a pair + with self.assertRaises(ValueError): + types.SimpleNamespace([['x', 'y', 'z']]) + with self.assertRaises(TypeError): + types.SimpleNamespace(**{1: 2}) # non-string key + with self.assertRaises(TypeError): + types.SimpleNamespace({1: 2}) + with self.assertRaises(TypeError): + types.SimpleNamespace([[1, 2]]) + with self.assertRaises(TypeError): + types.SimpleNamespace(UserDict({1: 2})) + with self.assertRaises(TypeError): + types.SimpleNamespace([[[], 2]]) # non-hashable key def test_unbound(self): ns1 = vars(types.SimpleNamespace()) @@ -1799,6 +2103,33 @@ def test_pickle(self): self.assertEqual(ns, ns_roundtrip, pname) + def test_replace(self): + ns = types.SimpleNamespace(x=11, y=22) + + ns2 = copy.replace(ns) + self.assertEqual(ns2, ns) + self.assertIsNot(ns2, ns) + self.assertIs(type(ns2), types.SimpleNamespace) + self.assertEqual(vars(ns2), {'x': 11, 'y': 22}) + ns2.x = 3 + self.assertEqual(ns.x, 11) + ns.x = 4 + self.assertEqual(ns2.x, 3) + + self.assertEqual(vars(copy.replace(ns, x=1)), {'x': 1, 'y': 22}) + self.assertEqual(vars(copy.replace(ns, y=2)), {'x': 4, 'y': 2}) + self.assertEqual(vars(copy.replace(ns, x=1, y=2)), {'x': 1, 'y': 2}) + + def test_replace_subclass(self): + class Spam(types.SimpleNamespace): + pass + + spam = Spam(ham=8, eggs=9) + spam2 = copy.replace(spam, ham=5) + + self.assertIs(type(spam2), Spam) + self.assertEqual(vars(spam2), {'ham': 5, 'eggs': 9}) + def test_fake_namespace_compare(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # SystemError. @@ -1843,8 +2174,6 @@ def foo(): foo = types.coroutine(foo) self.assertIs(aw, foo()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_async_def(self): # Test that types.coroutine passes 'async def' coroutines # without modification @@ -2078,7 +2407,7 @@ def foo(): return gen wrapper = foo() wrapper.send(None) with self.assertRaisesRegex(Exception, 'ham'): - wrapper.throw(Exception, Exception('ham')) + wrapper.throw(Exception('ham')) # decorate foo second time foo = types.coroutine(foo) @@ -2101,8 +2430,6 @@ def foo(): foo = types.coroutine(foo) self.assertIs(foo(), gencoro) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_genfunc(self): def gen(): yield self.assertIs(types.coroutine(gen), gen) @@ -2133,5 +2460,125 @@ def coro(): 'close', 'throw'})) +class FunctionTests(unittest.TestCase): + def test_function_type_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + func = types.FunctionType( + ex.__code__, {}, "func", (1, 2), None, {'c': 3}, + ) + + self.assertEqual(func(), 6) + self.assertEqual(func.__defaults__, (1, 2)) + self.assertEqual(func.__kwdefaults__, {'c': 3}) + + func = types.FunctionType( + ex.__code__, {}, "func", None, None, None, + ) + self.assertEqual(func.__defaults__, None) + self.assertEqual(func.__kwdefaults__, None) + + def test_function_type_wrong_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + with self.assertRaisesRegex(TypeError, 'arg 4'): + types.FunctionType( + ex.__code__, {}, "func", 1, None, {'c': 3}, + ) + with self.assertRaisesRegex(TypeError, 'arg 6'): + types.FunctionType( + ex.__code__, {}, "func", None, None, 3, + ) + + +@unittest.skip("TODO: RUSTPYTHON; no subinterpreters yet") +class SubinterpreterTests(unittest.TestCase): + + NUMERIC_METHODS = { + '__abs__', + '__add__', + '__bool__', + '__divmod__', + '__float__', + '__floordiv__', + '__index__', + '__int__', + '__lshift__', + '__mod__', + '__mul__', + '__neg__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__sub__', + '__truediv__', + } + + @classmethod + def setUpClass(cls): + global interpreters + try: + from concurrent import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) + + @cpython_only + @no_rerun('channels (and queues) might have a refleak; see gh-122199') + def test_static_types_inherited_slots(self): + rch, sch = self.create_channel() + + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = tuple((repr(c), a, repr(w)) for c, a, w in results) + sch.send_nowait(results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + exec(script) + raw = rch.recv_nowait() + main_results = collate_results(raw) + + interp = interpreters.create() + interp.exec('from concurrent import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + raw = rch.recv_nowait() + interp_results = collate_results(raw) + + for key, expected in main_results.items(): + cls, attr = key + with self.subTest(cls=cls, slotattr=attr): + actual = interp_results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(interp_results, {}) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index db0dc916f1a..448d16f1f4a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,3 +1,4 @@ +import annotationlib import contextlib import collections import collections.abc @@ -5,8 +6,10 @@ from functools import lru_cache, wraps, reduce import gc import inspect +import io import itertools import operator +import os import pickle import re import sys @@ -43,13 +46,19 @@ import textwrap import typing import weakref +import warnings import types -from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper -from test.support.testcase import ExtraAssertions -from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper +from test.support import ( + captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code, + EqualToForwardRef, +) +from test.typinganndata import ( + ann_module695, mod_generics_cache, _typed_dict_helper, + ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ann_module8 +) -import unittest # XXX: RUSTPYTHON +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' @@ -57,7 +66,7 @@ CANNOT_SUBCLASS_INSTANCE = 'Cannot subclass an instance of %s' -class BaseTestCase(TestCase, ExtraAssertions): +class BaseTestCase(TestCase): def clear_caches(self): for f in typing._cleanups: @@ -115,18 +124,18 @@ def test_errors(self): def test_can_subclass(self): class Mock(Any): pass - self.assertTrue(issubclass(Mock, Any)) + self.assertIsSubclass(Mock, Any) self.assertIsInstance(Mock(), Mock) class Something: pass - self.assertFalse(issubclass(Something, Any)) + self.assertNotIsSubclass(Something, Any) self.assertNotIsInstance(Something(), Mock) class MockSomething(Something, Mock): pass - self.assertTrue(issubclass(MockSomething, Any)) - self.assertTrue(issubclass(MockSomething, MockSomething)) - self.assertTrue(issubclass(MockSomething, Something)) - self.assertTrue(issubclass(MockSomething, Mock)) + self.assertIsSubclass(MockSomething, Any) + self.assertIsSubclass(MockSomething, MockSomething) + self.assertIsSubclass(MockSomething, Something) + self.assertIsSubclass(MockSomething, Mock) ms = MockSomething() self.assertIsInstance(ms, MockSomething) self.assertIsInstance(ms, Something) @@ -373,6 +382,7 @@ def test_alias(self): self.assertEqual(get_args(alias_2), (LiteralString,)) self.assertEqual(get_args(alias_3), (LiteralString,)) + class TypeVarTests(BaseTestCase): def test_basic_plain(self): T = TypeVar('T') @@ -467,8 +477,8 @@ def test_or(self): self.assertEqual(X | "x", Union[X, "x"]) self.assertEqual("x" | X, Union["x", X]) # make sure the order is correct - self.assertEqual(get_args(X | "x"), (X, ForwardRef("x"))) - self.assertEqual(get_args("x" | X), (ForwardRef("x"), X)) + self.assertEqual(get_args(X | "x"), (X, EqualToForwardRef("x"))) + self.assertEqual(get_args("x" | X), (EqualToForwardRef("x"), X)) def test_union_constrained(self): A = TypeVar('A', str, bytes) @@ -502,7 +512,7 @@ def test_cannot_instantiate_vars(self): def test_bound_errors(self): with self.assertRaises(TypeError): - TypeVar('X', bound=Union) + TypeVar('X', bound=Optional) with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) with self.assertRaisesRegex(TypeError, @@ -542,7 +552,7 @@ def test_var_substitution(self): def test_bad_var_substitution(self): T = TypeVar('T') bad_args = ( - (), (int, str), Union, + (), (int, str), Optional, Generic, Generic[T], Protocol, Protocol[T], Final, Final[int], ClassVar, ClassVar[int], ) @@ -625,7 +635,7 @@ class TypeParameterDefaultsTests(BaseTestCase): def test_typevar(self): T = TypeVar('T', default=int) self.assertEqual(T.__default__, int) - self.assertTrue(T.has_default()) + self.assertIs(T.has_default(), True) self.assertIsInstance(T, TypeVar) class A(Generic[T]): ... @@ -635,19 +645,19 @@ def test_typevar_none(self): U = TypeVar('U') U_None = TypeVar('U_None', default=None) self.assertIs(U.__default__, NoDefault) - self.assertFalse(U.has_default()) + self.assertIs(U.has_default(), False) self.assertIs(U_None.__default__, None) - self.assertTrue(U_None.has_default()) + self.assertIs(U_None.has_default(), True) class X[T]: ... T, = X.__type_params__ self.assertIs(T.__default__, NoDefault) - self.assertFalse(T.has_default()) + self.assertIs(T.has_default(), False) def test_paramspec(self): P = ParamSpec('P', default=(str, int)) self.assertEqual(P.__default__, (str, int)) - self.assertTrue(P.has_default()) + self.assertIs(P.has_default(), True) self.assertIsInstance(P, ParamSpec) class A(Generic[P]): ... @@ -660,19 +670,19 @@ def test_paramspec_none(self): U = ParamSpec('U') U_None = ParamSpec('U_None', default=None) self.assertIs(U.__default__, NoDefault) - self.assertFalse(U.has_default()) + self.assertIs(U.has_default(), False) self.assertIs(U_None.__default__, None) - self.assertTrue(U_None.has_default()) + self.assertIs(U_None.has_default(), True) class X[**P]: ... P, = X.__type_params__ self.assertIs(P.__default__, NoDefault) - self.assertFalse(P.has_default()) + self.assertIs(P.has_default(), False) def test_typevartuple(self): Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) - self.assertTrue(Ts.has_default()) + self.assertIs(Ts.has_default(), True) self.assertIsInstance(Ts, TypeVarTuple) class A(Generic[Unpack[Ts]]): ... @@ -754,18 +764,28 @@ class A(Generic[T, P, U]): ... self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_paramspec_and_typevar_specialization_2(self): + T = TypeVar("T") + P = ParamSpec('P', default=...) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, ...) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, ..., float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_typevartuple_none(self): U = TypeVarTuple('U') U_None = TypeVarTuple('U_None', default=None) self.assertIs(U.__default__, NoDefault) - self.assertFalse(U.has_default()) + self.assertIs(U.has_default(), False) self.assertIs(U_None.__default__, None) - self.assertTrue(U_None.has_default()) + self.assertIs(U_None.has_default(), True) class X[**Ts]: ... Ts, = X.__type_params__ self.assertIs(Ts.__default__, NoDefault) - self.assertFalse(Ts.has_default()) + self.assertIs(Ts.has_default(), False) def test_no_default_after_non_default(self): DefaultStrT = TypeVar('DefaultStrT', default=str) @@ -966,7 +986,6 @@ class C(Generic[T]): pass ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_two_parameters(self): T1 = TypeVar('T1') T2 = TypeVar('T2') @@ -1064,7 +1083,7 @@ class C(Generic[T1, T2, T3]): pass eval(expected_str) ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_variadic_parameters(self): T1 = TypeVar('T1') T2 = TypeVar('T2') @@ -1168,7 +1187,6 @@ class C(Generic[*Ts]): pass ) - class UnpackTests(BaseTestCase): def test_accepts_single_type(self): @@ -1999,11 +2017,11 @@ def test_basics(self): self.assertNotEqual(u, Union) def test_union_isinstance(self): - self.assertTrue(isinstance(42, Union[int, str])) - self.assertTrue(isinstance('abc', Union[int, str])) - self.assertFalse(isinstance(3.14, Union[int, str])) - self.assertTrue(isinstance(42, Union[int, list[int]])) - self.assertTrue(isinstance(42, Union[int, Any])) + self.assertIsInstance(42, Union[int, str]) + self.assertIsInstance('abc', Union[int, str]) + self.assertNotIsInstance(3.14, Union[int, str]) + self.assertIsInstance(42, Union[int, list[int]]) + self.assertIsInstance(42, Union[int, Any]) def test_union_isinstance_type_error(self): with self.assertRaises(TypeError): @@ -2020,9 +2038,9 @@ def test_union_isinstance_type_error(self): isinstance(42, Union[Any, str]) def test_optional_isinstance(self): - self.assertTrue(isinstance(42, Optional[int])) - self.assertTrue(isinstance(None, Optional[int])) - self.assertFalse(isinstance('abc', Optional[int])) + self.assertIsInstance(42, Optional[int]) + self.assertIsInstance(None, Optional[int]) + self.assertNotIsInstance('abc', Optional[int]) def test_optional_isinstance_type_error(self): with self.assertRaises(TypeError): @@ -2035,20 +2053,16 @@ def test_optional_isinstance_type_error(self): isinstance(None, Optional[Any]) def test_union_issubclass(self): - self.assertTrue(issubclass(int, Union[int, str])) - self.assertTrue(issubclass(str, Union[int, str])) - self.assertFalse(issubclass(float, Union[int, str])) - self.assertTrue(issubclass(int, Union[int, list[int]])) - self.assertTrue(issubclass(int, Union[int, Any])) - self.assertFalse(issubclass(int, Union[str, Any])) - self.assertTrue(issubclass(int, Union[Any, int])) - self.assertFalse(issubclass(int, Union[Any, str])) + self.assertIsSubclass(int, Union[int, str]) + self.assertIsSubclass(str, Union[int, str]) + self.assertNotIsSubclass(float, Union[int, str]) + self.assertIsSubclass(int, Union[int, list[int]]) + self.assertIsSubclass(int, Union[int, Any]) + self.assertNotIsSubclass(int, Union[str, Any]) + self.assertIsSubclass(int, Union[Any, int]) + self.assertNotIsSubclass(int, Union[Any, str]) def test_union_issubclass_type_error(self): - with self.assertRaises(TypeError): - issubclass(int, Union) - with self.assertRaises(TypeError): - issubclass(Union, int) with self.assertRaises(TypeError): issubclass(Union[int, str], int) with self.assertRaises(TypeError): @@ -2059,12 +2073,12 @@ def test_union_issubclass_type_error(self): issubclass(int, Union[list[int], str]) def test_optional_issubclass(self): - self.assertTrue(issubclass(int, Optional[int])) - self.assertTrue(issubclass(type(None), Optional[int])) - self.assertFalse(issubclass(str, Optional[int])) - self.assertTrue(issubclass(Any, Optional[Any])) - self.assertTrue(issubclass(type(None), Optional[Any])) - self.assertFalse(issubclass(int, Optional[Any])) + self.assertIsSubclass(int, Optional[int]) + self.assertIsSubclass(type(None), Optional[int]) + self.assertNotIsSubclass(str, Optional[int]) + self.assertIsSubclass(Any, Optional[Any]) + self.assertIsSubclass(type(None), Optional[Any]) + self.assertNotIsSubclass(int, Optional[Any]) def test_optional_issubclass_type_error(self): with self.assertRaises(TypeError): @@ -2123,41 +2137,40 @@ class B(metaclass=UnhashableMeta): ... self.assertEqual(Union[A, B].__args__, (A, B)) union1 = Union[A, B] - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): hash(union1) union2 = Union[int, B] - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): hash(union2) union3 = Union[A, int] - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): hash(union3) def test_repr(self): - self.assertEqual(repr(Union), 'typing.Union') u = Union[Employee, int] - self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) + self.assertEqual(repr(u), f'{__name__}.Employee | int') u = Union[int, Employee] - self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + self.assertEqual(repr(u), f'int | {__name__}.Employee') T = TypeVar('T') u = Union[T, int][int] self.assertEqual(repr(u), repr(int)) u = Union[List[int], int] - self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') + self.assertEqual(repr(u), 'typing.List[int] | int') u = Union[list[int], dict[str, float]] - self.assertEqual(repr(u), 'typing.Union[list[int], dict[str, float]]') + self.assertEqual(repr(u), 'list[int] | dict[str, float]') u = Union[int | float] - self.assertEqual(repr(u), 'typing.Union[int, float]') + self.assertEqual(repr(u), 'int | float') u = Union[None, str] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'None | str') u = Union[str, None] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'str | None') u = Union[None, str, int] - self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') + self.assertEqual(repr(u), 'None | str | int') u = Optional[str] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'str | None') def test_dir(self): dir_items = set(dir(Union[str, int])) @@ -2169,14 +2182,11 @@ def test_dir(self): def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, - r'Cannot subclass typing\.Union'): + r"type 'typing\.Union' is not an acceptable base type"): class C(Union): pass - with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class D(type(Union)): - pass with self.assertRaisesRegex(TypeError, - r'Cannot subclass typing\.Union\[int, str\]'): + r'Cannot subclass int \| str'): class E(Union[int, str]): pass @@ -2192,8 +2202,8 @@ def test_cannot_instantiate(self): type(u)() def test_union_generalization(self): - self.assertFalse(Union[str, typing.Iterable[int]] == str) - self.assertFalse(Union[str, typing.Iterable[int]] == typing.Iterable[int]) + self.assertNotEqual(Union[str, typing.Iterable[int]], str) + self.assertNotEqual(Union[str, typing.Iterable[int]], typing.Iterable[int]) self.assertIn(str, Union[str, typing.Iterable[int]].__args__) self.assertIn(typing.Iterable[int], Union[str, typing.Iterable[int]].__args__) @@ -2222,7 +2232,7 @@ def f(x: u): ... def test_function_repr_union(self): def fun() -> int: ... - self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + self.assertEqual(repr(Union[fun, int]), f'{__name__}.{fun.__qualname__} | int') def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 @@ -2270,6 +2280,15 @@ class Ints(enum.IntEnum): self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, (Literal[1], Literal[Ints.B], Literal[True])) + def test_allow_non_types_in_or(self): + # gh-140348: Test that using | with a Union object allows things that are + # not allowed by is_unionable(). + U1 = Union[int, str] + self.assertEqual(U1 | float, Union[int, str, float]) + self.assertEqual(U1 | "float", Union[int, str, "float"]) + self.assertEqual(float | U1, Union[float, int, str]) + self.assertEqual("float" | U1, Union["float", int, str]) + class TupleTests(BaseTestCase): @@ -2557,7 +2576,7 @@ def test_concatenate(self): def test_nested_paramspec(self): # Since Callable has some special treatment, we want to be sure - # that substituion works correctly, see gh-103054 + # that substitution works correctly, see gh-103054 Callable = self.Callable P = ParamSpec('P') P2 = ParamSpec('P2') @@ -2609,6 +2628,7 @@ def test_errors(self): with self.assertRaisesRegex(TypeError, "few arguments for"): C1[int] + class TypingCallableTests(BaseCallableTests, BaseTestCase): Callable = typing.Callable @@ -2786,6 +2806,7 @@ class Coordinate(Protocol): x: int y: int + @runtime_checkable class Point(Coordinate, Protocol): label: str @@ -3155,6 +3176,21 @@ def x(self): ... with self.assertRaisesRegex(TypeError, only_classes_allowed): issubclass(1, BadPG) + def test_isinstance_against_superproto_doesnt_affect_subproto_instance(self): + @runtime_checkable + class Base(Protocol): + x: int + + @runtime_checkable + class Child(Base, Protocol): + y: str + + class Capybara: + x = 43 + + self.assertIsInstance(Capybara(), Base) + self.assertNotIsInstance(Capybara(), Child) + def test_implicit_issubclass_between_two_protocols(self): @runtime_checkable class CallableMembersProto(Protocol): @@ -3229,7 +3265,7 @@ def meth2(self, x, y): return True self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) - @unittest.skip('TODO: RUSTPYTHON; (no gc)') + @unittest.skip("TODO: RUSTPYTHON; (no gc)") def test_isinstance_checks_not_at_whim_of_gc(self): self.addCleanup(gc.enable) gc.disable() @@ -3848,7 +3884,8 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', + '__annotations_cache__', '__annotate_func__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( @@ -4070,8 +4107,8 @@ def test_generic_protocols_repr(self): class P(Protocol[T, S]): pass - self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) - self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + self.assertEndsWith(repr(P[T, S]), 'P[~T, ~S]') + self.assertEndsWith(repr(P[int, str]), 'P[int, str]') def test_generic_protocols_eq(self): T = TypeVar('T') @@ -4111,12 +4148,12 @@ class PG(Protocol[T]): def meth(self): pass - self.assertTrue(P._is_protocol) - self.assertTrue(PR._is_protocol) - self.assertTrue(PG._is_protocol) - self.assertFalse(P._is_runtime_protocol) - self.assertTrue(PR._is_runtime_protocol) - self.assertTrue(PG[int]._is_protocol) + self.assertIs(P._is_protocol, True) + self.assertIs(PR._is_protocol, True) + self.assertIs(PG._is_protocol, True) + self.assertIs(P._is_runtime_protocol, False) + self.assertIs(PR._is_runtime_protocol, True) + self.assertIs(PG[int]._is_protocol, True) self.assertEqual(typing._get_protocol_attrs(P), {'meth'}) self.assertEqual(typing._get_protocol_attrs(PR), {'x'}) self.assertEqual(frozenset(typing._get_protocol_attrs(PG)), @@ -4172,7 +4209,6 @@ class P(Protocol): Alias2 = typing.Union[P, typing.Iterable] self.assertEqual(Alias, Alias2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_protocols_pickleable(self): global P, CP # pickle wants to reference the class by name T = TypeVar('T') @@ -4323,11 +4359,48 @@ def __release_buffer__(self, mv: memoryview) -> None: self.assertNotIsSubclass(C, ReleasableBuffer) self.assertNotIsInstance(C(), ReleasableBuffer) + def test_io_reader_protocol_allowed(self): + @runtime_checkable + class CustomReader(io.Reader[bytes], Protocol): + def close(self): ... + + class A: pass + class B: + def read(self, sz=-1): + return b"" + def close(self): + pass + + self.assertIsSubclass(B, CustomReader) + self.assertIsInstance(B(), CustomReader) + self.assertNotIsSubclass(A, CustomReader) + self.assertNotIsInstance(A(), CustomReader) + + def test_io_writer_protocol_allowed(self): + @runtime_checkable + class CustomWriter(io.Writer[bytes], Protocol): + def close(self): ... + + class A: pass + class B: + def write(self, b): + pass + def close(self): + pass + + self.assertIsSubclass(B, CustomWriter) + self.assertIsInstance(B(), CustomWriter) + self.assertNotIsSubclass(A, CustomWriter) + self.assertNotIsInstance(A(), CustomWriter) + def test_builtin_protocol_allowlist(self): with self.assertRaises(TypeError): class CustomProtocol(TestCase, Protocol): pass + class CustomPathLikeProtocol(os.PathLike, Protocol): + pass + class CustomContextManager(typing.ContextManager, Protocol): pass @@ -4541,6 +4614,42 @@ class Commentable(Protocol): ) self.assertIs(type(exc.__cause__), CustomError) + def test_isinstance_with_deferred_evaluation_of_annotations(self): + @runtime_checkable + class P(Protocol): + def meth(self): + ... + + class DeferredClass: + x: undefined + + class DeferredClassImplementingP: + x: undefined | int + + def __init__(self): + self.x = 0 + + def meth(self): + ... + + # override meth with a non-method attribute to make it part of __annotations__ instead of __dict__ + class SubProtocol(P, Protocol): + meth: undefined + + + self.assertIsSubclass(SubProtocol, P) + self.assertNotIsInstance(DeferredClass(), P) + self.assertIsInstance(DeferredClassImplementingP(), P) + + def test_deferred_evaluation_of_annotations(self): + class DeferredProto(Protocol): + x: DoesNotExist + self.assertEqual(get_protocol_members(DeferredProto), {"x"}) + self.assertEqual( + annotationlib.get_annotations(DeferredProto, format=annotationlib.Format.STRING), + {'x': 'DoesNotExist'} + ) + class GenericTests(BaseTestCase): @@ -4588,6 +4697,35 @@ class D(Generic[T]): pass with self.assertRaises(TypeError): D[()] + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_generic_init_subclass_not_called_error(self): + notes = ["Note: this exception may have been caused by " + r"'GenericTests.test_generic_init_subclass_not_called_error.<locals>.Base.__init_subclass__' " + "(or the '__init_subclass__' method on a superclass) not calling 'super().__init_subclass__()'"] + + class Base: + def __init_subclass__(cls) -> None: + # Oops, I forgot super().__init_subclass__()! + pass + + with self.subTest(): + class Sub(Base, Generic[T]): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + with self.subTest(): + class Sub[U](Base): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + def test_generic_subclass_checks(self): for typ in [list[int], List[int], tuple[int, str], Tuple[int, str], @@ -4659,8 +4797,7 @@ class C(Generic[T]): self.assertNotEqual(Z, Y[int]) self.assertNotEqual(Z, Y[T]) - self.assertTrue(str(Z).endswith( - '.C[typing.Tuple[str, int]]')) + self.assertEndsWith(str(Z), '.C[typing.Tuple[str, int]]') def test_new_repr(self): T = TypeVar('T') @@ -4888,12 +5025,12 @@ class A(Generic[T]): self.assertNotEqual(typing.FrozenSet[A[str]], typing.FrozenSet[mod_generics_cache.B.A[str]]) - self.assertTrue(repr(Tuple[A[str]]).endswith('<locals>.A[str]]')) - self.assertTrue(repr(Tuple[B.A[str]]).endswith('<locals>.B.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) - .endswith('mod_generics_cache.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) - .endswith('mod_generics_cache.B.A[str]]')) + self.assertEndsWith(repr(Tuple[A[str]]), '<locals>.A[str]]') + self.assertEndsWith(repr(Tuple[B.A[str]]), '<locals>.B.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.A[str]]), + 'mod_generics_cache.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.B.A[str]]), + 'mod_generics_cache.B.A[str]]') def test_extended_generic_rules_eq(self): T = TypeVar('T') @@ -4914,11 +5051,11 @@ class Derived(Base): ... def test_extended_generic_rules_repr(self): T = TypeVar('T') self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), - 'Union[Tuple, Callable]') + 'Tuple | Callable') self.assertEqual(repr(Union[Tuple, Tuple[int]]).replace('typing.', ''), - 'Union[Tuple, Tuple[int]]') + 'Tuple | Tuple[int]') self.assertEqual(repr(Callable[..., Optional[T]][int]).replace('typing.', ''), - 'Callable[..., Optional[int]]') + 'Callable[..., int | None]') self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), 'Callable[[], List[int]]') @@ -4984,7 +5121,7 @@ class C3: def f(x: X): ... self.assertEqual( get_type_hints(f, globals(), locals()), - {'x': list[list[ForwardRef('X')]]} + {'x': list[list[EqualToForwardRef('X')]]} ) def test_pep695_generic_class_with_future_annotations(self): @@ -5098,9 +5235,9 @@ def __contains__(self, item): with self.assertRaises(TypeError): issubclass(Tuple[int, ...], typing.Iterable) - def test_fail_with_bare_union(self): + def test_fail_with_special_forms(self): with self.assertRaises(TypeError): - List[Union] + List[Final] with self.assertRaises(TypeError): Tuple[Optional] with self.assertRaises(TypeError): @@ -5147,7 +5284,6 @@ def test_all_repr_eq_any(self): self.assertNotEqual(repr(base), '') self.assertEqual(base, base) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): global C # pickle wants to reference the class by name T = TypeVar('T') @@ -5204,10 +5340,12 @@ class Node(Generic[T]): ... Tuple[Any, Any], Node[T], Node[int], Node[Any], typing.Iterable[T], typing.Iterable[Any], typing.Iterable[int], typing.Dict[int, str], typing.Dict[T, Any], ClassVar[int], ClassVar[List[T]], Tuple['T', 'T'], - Union['T', int], List['T'], typing.Mapping['T', int]] - for t in things + [Any]: - self.assertEqual(t, copy(t)) - self.assertEqual(t, deepcopy(t)) + Union['T', int], List['T'], typing.Mapping['T', int], + Union[b"x", b"y"], Any] + for t in things: + with self.subTest(thing=t): + self.assertEqual(t, copy(t)) + self.assertEqual(t, deepcopy(t)) def test_immutability_by_copy_and_pickle(self): # Special forms like Union, Any, etc., generic aliases to containers like List, @@ -5643,8 +5781,6 @@ def test_subclass_special_form(self): for obj in ( ClassVar[int], Final[int], - Union[int, float], - Optional[int], Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], @@ -5676,7 +5812,7 @@ class A: __parameters__ = (T,) # Bare classes should be skipped for a in (List, list): - for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, types.UnionType): + for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, Union): with self.subTest(generic=a, sub=b): with self.assertRaisesRegex(TypeError, '.* is not a generic class'): a[b][str] @@ -5695,7 +5831,7 @@ class A: for s in (int, G, A, List, list, TypeVar, TypeVarTuple, ParamSpec, - types.GenericAlias, types.UnionType): + types.GenericAlias, Union): for t in Tuple, tuple: with self.subTest(tuple=t, sub=s): @@ -5713,7 +5849,7 @@ class A: with self.assertRaises(TypeError): a[int] - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ".+__typing_subst__.+tuple.+int.*" does not match "'TypeAliasType' object is not subscriptable" + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ".+__typing_subst__.+tuple.+int.*" does not match "'TypeAliasType' object is not subscriptable" def test_return_non_tuple_while_unpacking(self): # GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually # returned a tuple @@ -5777,6 +5913,7 @@ def test_no_isinstance(self): with self.assertRaises(TypeError): issubclass(int, ClassVar) + class FinalTests(BaseTestCase): def test_basics(self): @@ -5833,7 +5970,6 @@ def test_final_unmodified(self): def func(x): ... self.assertIs(func, final(func)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dunder_final(self): @final def func(): ... @@ -5855,7 +5991,7 @@ def __call__(self, *args, **kwargs): @Wrapper def wrapped(): ... self.assertIsInstance(wrapped, Wrapper) - self.assertIs(False, hasattr(wrapped, "__final__")) + self.assertNotHasAttr(wrapped, "__final__") class Meta(type): @property @@ -5867,7 +6003,7 @@ class WithMeta(metaclass=Meta): ... # Builtin classes throw TypeError if you try to set an # attribute. final(int) - self.assertIs(False, hasattr(int, "__final__")) + self.assertNotHasAttr(int, "__final__") # Make sure it works with common builtin decorators class Methods: @@ -5948,19 +6084,19 @@ def static_method_bad_order(): self.assertEqual(Derived.class_method_good_order(), 42) self.assertIs(True, Derived.class_method_good_order.__override__) self.assertEqual(Derived.class_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.class_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.class_method_bad_order, "__override__") self.assertEqual(Derived.static_method_good_order(), 42) self.assertIs(True, Derived.static_method_good_order.__override__) self.assertEqual(Derived.static_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.static_method_bad_order, "__override__") # Base object is not changed: - self.assertIs(False, hasattr(Base.normal_method, "__override__")) - self.assertIs(False, hasattr(Base.class_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.class_method_bad_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Base.normal_method, "__override__") + self.assertNotHasAttr(Base.class_method_good_order, "__override__") + self.assertNotHasAttr(Base.class_method_bad_order, "__override__") + self.assertNotHasAttr(Base.static_method_good_order, "__override__") + self.assertNotHasAttr(Base.static_method_bad_order, "__override__") def test_property(self): class Base: @@ -5983,10 +6119,10 @@ def wrong(self) -> int: instance = Child() self.assertEqual(instance.correct, 2) - self.assertTrue(Child.correct.fget.__override__) + self.assertIs(Child.correct.fget.__override__, True) self.assertEqual(instance.wrong, 2) - self.assertFalse(hasattr(Child.wrong, "__override__")) - self.assertFalse(hasattr(Child.wrong.fset, "__override__")) + self.assertNotHasAttr(Child.wrong, "__override__") + self.assertNotHasAttr(Child.wrong.fset, "__override__") def test_silent_failure(self): class CustomProp: @@ -6003,7 +6139,7 @@ def some(self): return 1 self.assertEqual(WithOverride.some, 1) - self.assertFalse(hasattr(WithOverride.some, "__override__")) + self.assertNotHasAttr(WithOverride.some, "__override__") def test_multiple_decorators(self): def with_wraps(f): # similar to `lru_cache` definition @@ -6024,9 +6160,9 @@ def on_bottom(self, a: int) -> int: instance = WithOverride() self.assertEqual(instance.on_top(1), 2) - self.assertTrue(instance.on_top.__override__) + self.assertIs(instance.on_top.__override__, True) self.assertEqual(instance.on_bottom(1), 3) - self.assertTrue(instance.on_bottom.__override__) + self.assertIs(instance.on_bottom.__override__, True) class CastTests(BaseTestCase): @@ -6064,8 +6200,6 @@ def test_errors(self): # We need this to make sure that `@no_type_check` respects `__module__` attr: -from test.typinganndata import ann_module8 - @no_type_check class NoTypeCheck_Outer: Inner = ann_module8.NoTypeCheck_Outer.Inner @@ -6075,474 +6209,168 @@ class NoTypeCheck_WithFunction: NoTypeCheck_function = ann_module8.NoTypeCheck_function -class ForwardRefTests(BaseTestCase): - - def test_basics(self): +class NoTypeCheckTests(BaseTestCase): + def test_no_type_check(self): - class Node(Generic[T]): + @no_type_check + def foo(a: 'whatevers') -> {}: + pass - def __init__(self, label: T): - self.label = label - self.left = self.right = None + th = get_type_hints(foo) + self.assertEqual(th, {}) - def add_both(self, - left: 'Optional[Node[T]]', - right: 'Node[T]' = None, - stuff: int = None, - blah=None): - self.left = left - self.right = right + def test_no_type_check_class(self): - def add_left(self, node: Optional['Node[T]']): - self.add_both(node, None) + @no_type_check + class C: + def foo(a: 'whatevers') -> {}: + pass - def add_right(self, node: 'Node[T]' = None): - self.add_both(None, node) + cth = get_type_hints(C.foo) + self.assertEqual(cth, {}) + ith = get_type_hints(C().foo) + self.assertEqual(ith, {}) - t = Node[int] - both_hints = get_type_hints(t.add_both, globals(), locals()) - self.assertEqual(both_hints['left'], Optional[Node[T]]) - self.assertEqual(both_hints['right'], Node[T]) - self.assertEqual(both_hints['stuff'], int) - self.assertNotIn('blah', both_hints) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C - left_hints = get_type_hints(t.add_left, globals(), locals()) - self.assertEqual(left_hints['node'], Optional[Node[T]]) + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) - right_hints = get_type_hints(t.add_right, globals(), locals()) - self.assertEqual(right_hints['node'], Node[T]) + # and never child classes: + class Child(D): + def foo(self, x: int): ... - def test_forwardref_instance_type_error(self): - fr = typing.ForwardRef('int') - with self.assertRaises(TypeError): - isinstance(42, fr) + self.assertEqual(get_type_hints(Child.foo), {'x': int}) - def test_forwardref_subclass_type_error(self): - fr = typing.ForwardRef('int') - with self.assertRaises(TypeError): - issubclass(int, fr) + def test_no_type_check_nested_types(self): + # See https://bugs.python.org/issue46571 + class Other: + o: int + class B: # Has the same `__name__`` as `A.B` and different `__qualname__` + o: int + @no_type_check + class A: + a: int + class B: + b: int + class C: + c: int + class D: + d: int - def test_forwardref_only_str_arg(self): - with self.assertRaises(TypeError): - typing.ForwardRef(1) # only `str` type is allowed + Other = Other - def test_forward_equality(self): - fr = typing.ForwardRef('int') - self.assertEqual(fr, typing.ForwardRef('int')) - self.assertNotEqual(List['int'], List[int]) - self.assertNotEqual(fr, typing.ForwardRef('int', module=__name__)) - frm = typing.ForwardRef('int', module=__name__) - self.assertEqual(frm, typing.ForwardRef('int', module=__name__)) - self.assertNotEqual(frm, typing.ForwardRef('int', module='__other_name__')) + for klass in [A, A.B, A.B.C, A.D]: + with self.subTest(klass=klass): + self.assertIs(klass.__no_type_check__, True) + self.assertEqual(get_type_hints(klass), {}) - def test_forward_equality_gth(self): - c1 = typing.ForwardRef('C') - c1_gth = typing.ForwardRef('C') - c2 = typing.ForwardRef('C') - c2_gth = typing.ForwardRef('C') + for not_modified in [Other, B]: + with self.subTest(not_modified=not_modified): + with self.assertRaises(AttributeError): + not_modified.__no_type_check__ + self.assertNotEqual(get_type_hints(not_modified), {}) - class C: - pass - def foo(a: c1_gth, b: c2_gth): - pass + def test_no_type_check_class_and_static_methods(self): + @no_type_check + class Some: + @staticmethod + def st(x: int) -> int: ... + @classmethod + def cl(cls, y: int) -> int: ... - self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': C, 'b': C}) - self.assertEqual(c1, c2) - self.assertEqual(c1, c1_gth) - self.assertEqual(c1_gth, c2_gth) - self.assertEqual(List[c1], List[c1_gth]) - self.assertNotEqual(List[c1], List[C]) - self.assertNotEqual(List[c1_gth], List[C]) - self.assertEqual(Union[c1, c1_gth], Union[c1]) - self.assertEqual(Union[c1, c1_gth, int], Union[c1, int]) - - def test_forward_equality_hash(self): - c1 = typing.ForwardRef('int') - c1_gth = typing.ForwardRef('int') - c2 = typing.ForwardRef('int') - c2_gth = typing.ForwardRef('int') - - def foo(a: c1_gth, b: c2_gth): - pass - get_type_hints(foo, globals(), locals()) + self.assertIs(Some.st.__no_type_check__, True) + self.assertEqual(get_type_hints(Some.st), {}) + self.assertIs(Some.cl.__no_type_check__, True) + self.assertEqual(get_type_hints(Some.cl), {}) - self.assertEqual(hash(c1), hash(c2)) - self.assertEqual(hash(c1_gth), hash(c2_gth)) - self.assertEqual(hash(c1), hash(c1_gth)) + def test_no_type_check_other_module(self): + self.assertIs(NoTypeCheck_Outer.__no_type_check__, True) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.__no_type_check__ + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.Inner.__no_type_check__ - c3 = typing.ForwardRef('int', module=__name__) - c4 = typing.ForwardRef('int', module='__other_name__') + self.assertIs(NoTypeCheck_WithFunction.__no_type_check__, True) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_function.__no_type_check__ - self.assertNotEqual(hash(c3), hash(c1)) - self.assertNotEqual(hash(c3), hash(c1_gth)) - self.assertNotEqual(hash(c3), hash(c4)) - self.assertEqual(hash(c3), hash(typing.ForwardRef('int', module=__name__))) + def test_no_type_check_foreign_functions(self): + # We should not modify this function: + def some(*args: int) -> int: + ... - def test_forward_equality_namespace(self): + @no_type_check class A: - pass - def namespace1(): - a = typing.ForwardRef('A') - def fun(x: a): - pass - get_type_hints(fun, globals(), locals()) - return a + some_alias = some + some_class = classmethod(some) + some_static = staticmethod(some) - def namespace2(): - a = typing.ForwardRef('A') + with self.assertRaises(AttributeError): + some.__no_type_check__ + self.assertEqual(get_type_hints(some), {'args': int, 'return': int}) - class A: - pass - def fun(x: a): - pass + def test_no_type_check_lambda(self): + @no_type_check + class A: + # Corner case: `lambda` is both an assignment and a function: + bar: Callable[[int], int] = lambda arg: arg - get_type_hints(fun, globals(), locals()) - return a + self.assertIs(A.bar.__no_type_check__, True) + self.assertEqual(get_type_hints(A.bar), {}) - self.assertEqual(namespace1(), namespace1()) - self.assertNotEqual(namespace1(), namespace2()) + def test_no_type_check_TypeError(self): + # This simply should not fail with + # `TypeError: can't set attributes of built-in/extension type 'dict'` + no_type_check(dict) - def test_forward_repr(self): - self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]") - self.assertEqual(repr(List[ForwardRef('int', module='mod')]), - "typing.List[ForwardRef('int', module='mod')]") + def test_no_type_check_forward_ref_as_string(self): + class C: + foo: typing.ClassVar[int] = 7 + class D: + foo: ClassVar[int] = 7 + class E: + foo: 'typing.ClassVar[int]' = 7 + class F: + foo: 'ClassVar[int]' = 7 - def test_union_forward(self): + expected_result = {'foo': typing.ClassVar[int]} + for clazz in [C, D, E, F]: + self.assertEqual(get_type_hints(clazz), expected_result) - def foo(a: Union['T']): - pass + def test_meta_no_type_check(self): + depr_msg = ( + "'typing.no_type_check_decorator' is deprecated " + "and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, depr_msg): + @no_type_check_decorator + def magic_decorator(func): + return func - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Union[T]}) + self.assertEqual(magic_decorator.__name__, 'magic_decorator') - def foo(a: tuple[ForwardRef('T')] | int): + @magic_decorator + def foo(a: 'whatevers') -> {}: pass - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': tuple[T] | int}) + @magic_decorator + class C: + def foo(a: 'whatevers') -> {}: + pass - def test_tuple_forward(self): - - def foo(a: Tuple['T']): - pass - - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Tuple[T]}) - - def foo(a: tuple[ForwardRef('T')]): - pass - - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': tuple[T]}) - - def test_double_forward(self): - def foo(a: 'List[\'int\']'): - pass - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': List[int]}) - - def test_forward_recursion_actually(self): - def namespace1(): - a = typing.ForwardRef('A') - A = a - def fun(x: a): pass - - ret = get_type_hints(fun, globals(), locals()) - return a - - def namespace2(): - a = typing.ForwardRef('A') - A = a - def fun(x: a): pass - - ret = get_type_hints(fun, globals(), locals()) - return a - - def cmp(o1, o2): - return o1 == o2 - - with infinite_recursion(25): - r1 = namespace1() - r2 = namespace2() - self.assertIsNot(r1, r2) - self.assertRaises(RecursionError, cmp, r1, r2) - - def test_union_forward_recursion(self): - ValueList = List['Value'] - Value = Union[str, ValueList] - - class C: - foo: List[Value] - class D: - foo: Union[Value, ValueList] - class E: - foo: Union[List[Value], ValueList] - class F: - foo: Union[Value, List[Value], ValueList] - - self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals())) - self.assertEqual(get_type_hints(C, globals(), locals()), - {'foo': List[Union[str, List[Union[str, List['Value']]]]]}) - self.assertEqual(get_type_hints(D, globals(), locals()), - {'foo': Union[str, List[Union[str, List['Value']]]]}) - self.assertEqual(get_type_hints(E, globals(), locals()), - {'foo': Union[ - List[Union[str, List[Union[str, List['Value']]]]], - List[Union[str, List['Value']]] - ] - }) - self.assertEqual(get_type_hints(F, globals(), locals()), - {'foo': Union[ - str, - List[Union[str, List['Value']]], - List[Union[str, List[Union[str, List['Value']]]]] - ] - }) - - def test_callable_forward(self): - - def foo(a: Callable[['T'], 'T']): - pass - - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Callable[[T], T]}) - - def test_callable_with_ellipsis_forward(self): - - def foo(a: 'Callable[..., T]'): - pass - - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Callable[..., T]}) - - def test_special_forms_forward(self): - - class C: - a: Annotated['ClassVar[int]', (3, 5)] = 4 - b: Annotated['Final[int]', "const"] = 4 - x: 'ClassVar' = 4 - y: 'Final' = 4 - - class CF: - b: List['Final[int]'] = 4 - - self.assertEqual(get_type_hints(C, globals())['a'], ClassVar[int]) - self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) - self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) - self.assertEqual(get_type_hints(C, globals())['y'], Final) - with self.assertRaises(TypeError): - get_type_hints(CF, globals()), - - def test_syntax_error(self): - - with self.assertRaises(SyntaxError): - Generic['/T'] - - def test_delayed_syntax_error(self): - - def foo(a: 'Node[T'): - pass - - with self.assertRaises(SyntaxError): - get_type_hints(foo) - - def test_syntax_error_empty_string(self): - for form in [typing.List, typing.Set, typing.Type, typing.Deque]: - with self.subTest(form=form): - with self.assertRaises(SyntaxError): - form[''] - - def test_name_error(self): - - def foo(a: 'Noode[T]'): - pass - - with self.assertRaises(NameError): - get_type_hints(foo, locals()) - - def test_no_type_check(self): - - @no_type_check - def foo(a: 'whatevers') -> {}: - pass - - th = get_type_hints(foo) - self.assertEqual(th, {}) - - def test_no_type_check_class(self): - - @no_type_check - class C: - def foo(a: 'whatevers') -> {}: - pass - - cth = get_type_hints(C.foo) - self.assertEqual(cth, {}) - ith = get_type_hints(C().foo) - self.assertEqual(ith, {}) - - def test_no_type_check_no_bases(self): - class C: - def meth(self, x: int): ... - @no_type_check - class D(C): - c = C - - # verify that @no_type_check never affects bases - self.assertEqual(get_type_hints(C.meth), {'x': int}) - - # and never child classes: - class Child(D): - def foo(self, x: int): ... - - self.assertEqual(get_type_hints(Child.foo), {'x': int}) - - def test_no_type_check_nested_types(self): - # See https://bugs.python.org/issue46571 - class Other: - o: int - class B: # Has the same `__name__`` as `A.B` and different `__qualname__` - o: int - @no_type_check - class A: - a: int - class B: - b: int - class C: - c: int - class D: - d: int - - Other = Other - - for klass in [A, A.B, A.B.C, A.D]: - with self.subTest(klass=klass): - self.assertTrue(klass.__no_type_check__) - self.assertEqual(get_type_hints(klass), {}) - - for not_modified in [Other, B]: - with self.subTest(not_modified=not_modified): - with self.assertRaises(AttributeError): - not_modified.__no_type_check__ - self.assertNotEqual(get_type_hints(not_modified), {}) - - def test_no_type_check_class_and_static_methods(self): - @no_type_check - class Some: - @staticmethod - def st(x: int) -> int: ... - @classmethod - def cl(cls, y: int) -> int: ... - - self.assertTrue(Some.st.__no_type_check__) - self.assertEqual(get_type_hints(Some.st), {}) - self.assertTrue(Some.cl.__no_type_check__) - self.assertEqual(get_type_hints(Some.cl), {}) - - def test_no_type_check_other_module(self): - self.assertTrue(NoTypeCheck_Outer.__no_type_check__) - with self.assertRaises(AttributeError): - ann_module8.NoTypeCheck_Outer.__no_type_check__ - with self.assertRaises(AttributeError): - ann_module8.NoTypeCheck_Outer.Inner.__no_type_check__ - - self.assertTrue(NoTypeCheck_WithFunction.__no_type_check__) - with self.assertRaises(AttributeError): - ann_module8.NoTypeCheck_function.__no_type_check__ - - def test_no_type_check_foreign_functions(self): - # We should not modify this function: - def some(*args: int) -> int: - ... - - @no_type_check - class A: - some_alias = some - some_class = classmethod(some) - some_static = staticmethod(some) - - with self.assertRaises(AttributeError): - some.__no_type_check__ - self.assertEqual(get_type_hints(some), {'args': int, 'return': int}) - - def test_no_type_check_lambda(self): - @no_type_check - class A: - # Corner case: `lambda` is both an assignment and a function: - bar: Callable[[int], int] = lambda arg: arg - - self.assertTrue(A.bar.__no_type_check__) - self.assertEqual(get_type_hints(A.bar), {}) - - def test_no_type_check_TypeError(self): - # This simply should not fail with - # `TypeError: can't set attributes of built-in/extension type 'dict'` - no_type_check(dict) - - def test_no_type_check_forward_ref_as_string(self): - class C: - foo: typing.ClassVar[int] = 7 - class D: - foo: ClassVar[int] = 7 - class E: - foo: 'typing.ClassVar[int]' = 7 - class F: - foo: 'ClassVar[int]' = 7 - - expected_result = {'foo': typing.ClassVar[int]} - for clazz in [C, D, E, F]: - self.assertEqual(get_type_hints(clazz), expected_result) - - def test_meta_no_type_check(self): - depr_msg = ( - "'typing.no_type_check_decorator' is deprecated " - "and slated for removal in Python 3.15" - ) - with self.assertWarnsRegex(DeprecationWarning, depr_msg): - @no_type_check_decorator - def magic_decorator(func): - return func - - self.assertEqual(magic_decorator.__name__, 'magic_decorator') - - @magic_decorator - def foo(a: 'whatevers') -> {}: - pass - - @magic_decorator - class C: - def foo(a: 'whatevers') -> {}: - pass - - self.assertEqual(foo.__name__, 'foo') - th = get_type_hints(foo) - self.assertEqual(th, {}) - cth = get_type_hints(C.foo) - self.assertEqual(cth, {}) - ith = get_type_hints(C().foo) - self.assertEqual(ith, {}) - - def test_default_globals(self): - code = ("class C:\n" - " def foo(self, a: 'C') -> 'D': pass\n" - "class D:\n" - " def bar(self, b: 'D') -> C: pass\n" - ) - ns = {} - exec(code, ns) - hints = get_type_hints(ns['C'].foo) - self.assertEqual(hints, {'a': ns['C'], 'return': ns['D']}) - - def test_final_forward_ref(self): - self.assertEqual(gth(Loop, globals())['attr'], Final[Loop]) - self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) - self.assertNotEqual(gth(Loop, globals())['attr'], Final) - - def test_or(self): - X = ForwardRef('X') - # __or__/__ror__ itself - self.assertEqual(X | "x", Union[X, "x"]) - self.assertEqual("x" | X, Union["x", X]) + self.assertEqual(foo.__name__, 'foo') + th = get_type_hints(foo) + self.assertEqual(th, {}) + cth = get_type_hints(C.foo) + self.assertEqual(cth, {}) + ith = get_type_hints(C().foo) + self.assertEqual(ith, {}) class InternalsTests(BaseTestCase): @@ -6580,6 +6408,16 @@ def test_collect_parameters(self): typing._collect_parameters self.assertEqual(cm.filename, __file__) + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports("typing", { + "warnings", + "inspect", + "re", + "contextlib", + "annotationlib", + }) + @lru_cache() def cached_func(x, y): @@ -6686,10 +6524,6 @@ def test_overload_registry_repeated(self): self.assertEqual(list(get_overloads(impl)), overloads) -from test.typinganndata import ( - ann_module, ann_module2, ann_module3, ann_module5, ann_module6, -) - T_a = TypeVar('T_a') class AwaitableWrapper(typing.Awaitable[T_a]): @@ -6842,7 +6676,7 @@ def nested(self: 'ForRefExample'): pass -class GetTypeHintTests(BaseTestCase): +class GetTypeHintsTests(BaseTestCase): def test_get_type_hints_from_various_objects(self): # For invalid objects should fail with TypeError (not AttributeError etc). with self.assertRaises(TypeError): @@ -6852,9 +6686,8 @@ def test_get_type_hints_from_various_objects(self): with self.assertRaises(TypeError): gth(None) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6872,7 +6705,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) @@ -6903,8 +6736,8 @@ def test_respect_no_type_check(self): class NoTpCheck: class Inn: def __init__(self, x: 'not a type'): ... - self.assertTrue(NoTpCheck.__no_type_check__) - self.assertTrue(NoTpCheck.Inn.__init__.__no_type_check__) + self.assertIs(NoTpCheck.__no_type_check__, True) + self.assertIs(NoTpCheck.Inn.__init__.__no_type_check__, True) self.assertEqual(gth(ann_module2.NTC.meth), {}) class ABase(Generic[T]): def meth(x: int): ... @@ -7050,111 +6883,320 @@ def __iand__(self, other: Const["MySet[T]"]) -> "MySet[T]": {'other': MySet[T], 'return': MySet[T]} ) - def test_get_type_hints_annotated_with_none_default(self): - # See: https://bugs.python.org/issue46195 - def annotated_with_none_default(x: Annotated[int, 'data'] = None): ... - self.assertEqual( - get_type_hints(annotated_with_none_default), - {'x': int}, - ) - self.assertEqual( - get_type_hints(annotated_with_none_default, include_extras=True), - {'x': Annotated[int, 'data']}, - ) + def test_get_type_hints_annotated_with_none_default(self): + # See: https://bugs.python.org/issue46195 + def annotated_with_none_default(x: Annotated[int, 'data'] = None): ... + self.assertEqual( + get_type_hints(annotated_with_none_default), + {'x': int}, + ) + self.assertEqual( + get_type_hints(annotated_with_none_default, include_extras=True), + {'x': Annotated[int, 'data']}, + ) + + def test_get_type_hints_classes_str_annotations(self): + class Foo: + y = str + x: 'y' + # This previously raised an error under PEP 563. + self.assertEqual(get_type_hints(Foo), {'x': str}) + + def test_get_type_hints_bad_module(self): + # bpo-41515 + class BadModule: + pass + BadModule.__module__ = 'bad' # Something not in sys.modules + self.assertNotIn('bad', sys.modules) + self.assertEqual(get_type_hints(BadModule), {}) + + def test_get_type_hints_annotated_bad_module(self): + # See https://bugs.python.org/issue44468 + class BadBase: + foo: tuple + class BadType(BadBase): + bar: list + BadType.__module__ = BadBase.__module__ = 'bad' + self.assertNotIn('bad', sys.modules) + self.assertEqual(get_type_hints(BadType), {'foo': tuple, 'bar': list}) + + def test_forward_ref_and_final(self): + # https://bugs.python.org/issue45166 + hints = get_type_hints(ann_module5) + self.assertEqual(hints, {'name': Final[str]}) + + hints = get_type_hints(ann_module5.MyClass) + self.assertEqual(hints, {'value': Final}) + + def test_top_level_class_var(self): + # This is not meaningful but we don't raise for it. + # https://github.com/python/cpython/issues/133959 + hints = get_type_hints(ann_module6) + self.assertEqual(hints, {'wrong': ClassVar[int]}) + + def test_get_type_hints_typeddict(self): + self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { + 'title': str, + 'year': NotRequired[int], + }) + + self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { + 'a': Annotated[Required[int], "a", "b", "c"] + }) + + self.assertEqual(get_type_hints(ChildTotalMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildTotalMovie, include_extras=True), { + "title": Required[str], "year": NotRequired[int] + }) + + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie, include_extras=True), { + "title": Annotated[Required[str], "foobar", "another level"], + "year": NotRequired[Annotated[int, 2000]] + }) + + def test_get_type_hints_collections_abc_callable(self): + # https://github.com/python/cpython/issues/91621 + P = ParamSpec('P') + def f(x: collections.abc.Callable[[int], int]): ... + def g(x: collections.abc.Callable[..., int]): ... + def h(x: collections.abc.Callable[P, int]): ... + + self.assertEqual(get_type_hints(f), {'x': collections.abc.Callable[[int], int]}) + self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) + self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + + def test_get_type_hints_format(self): + class C: + x: undefined + + with self.assertRaises(NameError): + get_type_hints(C) + + with self.assertRaises(NameError): + get_type_hints(C, format=annotationlib.Format.VALUE) + + annos = get_type_hints(C, format=annotationlib.Format.FORWARDREF) + self.assertIsInstance(annos, dict) + self.assertEqual(list(annos), ['x']) + self.assertIsInstance(annos['x'], annotationlib.ForwardRef) + self.assertEqual(annos['x'].__arg__, 'undefined') + + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), + {'x': 'undefined'}) + # Make sure using an int as format also works: + self.assertEqual(get_type_hints(C, format=4), {'x': 'undefined'}) + + def test_get_type_hints_format_function(self): + def func(x: undefined) -> undefined: ... + + # VALUE + with self.assertRaises(NameError): + get_type_hints(func) + with self.assertRaises(NameError): + get_type_hints(func, format=annotationlib.Format.VALUE) + + # FORWARDREF + self.assertEqual( + get_type_hints(func, format=annotationlib.Format.FORWARDREF), + {'x': EqualToForwardRef('undefined', owner=func), + 'return': EqualToForwardRef('undefined', owner=func)}, + ) + + # STRING + self.assertEqual(get_type_hints(func, format=annotationlib.Format.STRING), + {'x': 'undefined', 'return': 'undefined'}) + + def test_callable_with_ellipsis_forward(self): + + def foo(a: 'Callable[..., T]'): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Callable[..., T]}) + + def test_special_forms_no_forward(self): + def f(x: ClassVar[int]): + pass + self.assertEqual(get_type_hints(f), {'x': ClassVar[int]}) + + def test_special_forms_forward(self): + + class C: + a: Annotated['ClassVar[int]', (3, 5)] = 4 + b: Annotated['Final[int]', "const"] = 4 + x: 'ClassVar' = 4 + y: 'Final' = 4 + + class CF: + b: List['Final[int]'] = 4 + + self.assertEqual(get_type_hints(C, globals())['a'], ClassVar[int]) + self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) + self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) + self.assertEqual(get_type_hints(C, globals())['y'], Final) + lfi = get_type_hints(CF, globals())['b'] + self.assertIs(get_origin(lfi), list) + self.assertEqual(get_args(lfi), (Final[int],)) + + def test_union_forward_recursion(self): + ValueList = List['Value'] + Value = Union[str, ValueList] + + class C: + foo: List[Value] + class D: + foo: Union[Value, ValueList] + class E: + foo: Union[List[Value], ValueList] + class F: + foo: Union[Value, List[Value], ValueList] + + self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals())) + self.assertEqual(get_type_hints(C, globals(), locals()), + {'foo': List[Union[str, List[Union[str, List['Value']]]]]}) + self.assertEqual(get_type_hints(D, globals(), locals()), + {'foo': Union[str, List[Union[str, List['Value']]]]}) + self.assertEqual(get_type_hints(E, globals(), locals()), + {'foo': Union[ + List[Union[str, List[Union[str, List['Value']]]]], + List[Union[str, List['Value']]] + ] + }) + self.assertEqual(get_type_hints(F, globals(), locals()), + {'foo': Union[ + str, + List[Union[str, List['Value']]], + List[Union[str, List[Union[str, List['Value']]]]] + ] + }) + + def test_tuple_forward(self): + + def foo(a: Tuple['T']): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Tuple[T]}) + + def foo(a: tuple[ForwardRef('T')]): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': tuple[T]}) + + def test_double_forward(self): + def foo(a: 'List[\'int\']'): + pass + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': List[int]}) + + def test_union_forward(self): - def test_get_type_hints_classes_str_annotations(self): - class Foo: - y = str - x: 'y' - # This previously raised an error under PEP 563. - self.assertEqual(get_type_hints(Foo), {'x': str}) + def foo(a: Union['T']): + pass - def test_get_type_hints_bad_module(self): - # bpo-41515 - class BadModule: + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Union[T]}) + + def foo(a: tuple[ForwardRef('T')] | int): pass - BadModule.__module__ = 'bad' # Something not in sys.modules - self.assertNotIn('bad', sys.modules) - self.assertEqual(get_type_hints(BadModule), {}) - def test_get_type_hints_annotated_bad_module(self): - # See https://bugs.python.org/issue44468 - class BadBase: - foo: tuple - class BadType(BadBase): - bar: list - BadType.__module__ = BadBase.__module__ = 'bad' - self.assertNotIn('bad', sys.modules) - self.assertEqual(get_type_hints(BadType), {'foo': tuple, 'bar': list}) + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': tuple[T] | int}) - def test_forward_ref_and_final(self): - # https://bugs.python.org/issue45166 - hints = get_type_hints(ann_module5) - self.assertEqual(hints, {'name': Final[str]}) + def test_default_globals(self): + code = ("class C:\n" + " def foo(self, a: 'C') -> 'D': pass\n" + "class D:\n" + " def bar(self, b: 'D') -> C: pass\n" + ) + ns = {} + exec(code, ns) + hints = get_type_hints(ns['C'].foo) + self.assertEqual(hints, {'a': ns['C'], 'return': ns['D']}) - hints = get_type_hints(ann_module5.MyClass) - self.assertEqual(hints, {'value': Final}) + def test_final_forward_ref(self): + gth = get_type_hints + self.assertEqual(gth(Loop, globals())['attr'], Final[Loop]) + self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) + self.assertNotEqual(gth(Loop, globals())['attr'], Final) - def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) + def test_name_error(self): - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_get_type_hints_typeddict(self): - self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) - self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { - 'title': str, - 'year': NotRequired[int], - }) + def foo(a: 'Noode[T]'): + pass - self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) - self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { - 'title': Annotated[Required[str], "foobar"], - 'year': NotRequired[Annotated[int, 2000]], - }) + with self.assertRaises(NameError): + get_type_hints(foo, locals()) - self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) - self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { - 'title': Annotated[Required[str], "foobar", "another level"], - 'year': NotRequired[Annotated[int, 2000]], - }) + def test_basics(self): - self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) - self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { - 'title': Annotated[Required[str], "foobar", "another level"], - 'year': NotRequired[Annotated[int, 2000]], - }) + class Node(Generic[T]): - self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) - self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { - 'a': Annotated[Required[int], "a", "b", "c"] - }) + def __init__(self, label: T): + self.label = label + self.left = self.right = None - self.assertEqual(get_type_hints(ChildTotalMovie), {"title": str, "year": int}) - self.assertEqual(get_type_hints(ChildTotalMovie, include_extras=True), { - "title": Required[str], "year": NotRequired[int] - }) + def add_both(self, + left: 'Optional[Node[T]]', + right: 'Node[T]' = None, + stuff: int = None, + blah=None): + self.left = left + self.right = right - self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie), {"title": str, "year": int}) - self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie, include_extras=True), { - "title": Annotated[Required[str], "foobar", "another level"], - "year": NotRequired[Annotated[int, 2000]] - }) + def add_left(self, node: Optional['Node[T]']): + self.add_both(node, None) - def test_get_type_hints_collections_abc_callable(self): - # https://github.com/python/cpython/issues/91621 - P = ParamSpec('P') - def f(x: collections.abc.Callable[[int], int]): ... - def g(x: collections.abc.Callable[..., int]): ... - def h(x: collections.abc.Callable[P, int]): ... + def add_right(self, node: 'Node[T]' = None): + self.add_both(None, node) - self.assertEqual(get_type_hints(f), {'x': collections.abc.Callable[[int], int]}) - self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) - self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + t = Node[int] + both_hints = get_type_hints(t.add_both, globals(), locals()) + self.assertEqual(both_hints['left'], Optional[Node[T]]) + self.assertEqual(both_hints['right'], Node[T]) + self.assertEqual(both_hints['stuff'], int) + self.assertNotIn('blah', both_hints) + + left_hints = get_type_hints(t.add_left, globals(), locals()) + self.assertEqual(left_hints['node'], Optional[Node[T]]) + + right_hints = get_type_hints(t.add_right, globals(), locals()) + self.assertEqual(right_hints['node'], Node[T]) + + def test_stringified_typeddict(self): + ns = run_code( + """ + from __future__ import annotations + from typing import TypedDict + class TD[UniqueT](TypedDict): + a: UniqueT + """ + ) + TD = ns['TD'] + self.assertEqual(TD.__annotations__, {'a': EqualToForwardRef('UniqueT', owner=TD, module=TD.__module__)}) + self.assertEqual(get_type_hints(TD), {'a': TD.__type_params__[0]}) class GetUtilitiesTestCase(TestCase): @@ -7179,7 +7221,7 @@ class C(Generic[T]): pass self.assertIs(get_origin(Callable), collections.abc.Callable) self.assertIs(get_origin(list[int]), list) self.assertIs(get_origin(list), None) - self.assertIs(get_origin(list | str), types.UnionType) + self.assertIs(get_origin(list | str), Union) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) self.assertIs(get_origin(Required[int]), Required) @@ -7258,6 +7300,124 @@ class C(Generic[T]): pass self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) +class EvaluateForwardRefTests(BaseTestCase): + def test_evaluate_forward_ref(self): + int_ref = ForwardRef('int') + self.assertIs(typing.evaluate_forward_ref(int_ref), int) + self.assertIs( + typing.evaluate_forward_ref(int_ref, type_params=()), + int, + ) + self.assertIs( + typing.evaluate_forward_ref(int_ref, format=annotationlib.Format.VALUE), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.FORWARDREF, + ), + int, + ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_undefined(self): + missing = ForwardRef('missing') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(missing) + self.assertIs( + typing.evaluate_forward_ref( + missing, format=annotationlib.Format.FORWARDREF, + ), + missing, + ) + self.assertEqual( + typing.evaluate_forward_ref( + missing, format=annotationlib.Format.STRING, + ), + "missing", + ) + + def test_evaluate_forward_ref_nested(self): + ref = ForwardRef("int | list['str']") + self.assertEqual( + typing.evaluate_forward_ref(ref), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "int | list['str']", + ) + + why = ForwardRef('"\'str\'"') + self.assertIs(typing.evaluate_forward_ref(why), str) + + def test_evaluate_forward_ref_none(self): + none_ref = ForwardRef('None') + self.assertIs(typing.evaluate_forward_ref(none_ref), None) + + def test_globals(self): + A = "str" + ref = ForwardRef('list[A]') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + self.assertEqual( + typing.evaluate_forward_ref(ref, globals={'A': A}), + list[str], + ) + + def test_owner(self): + ref = ForwardRef("A") + + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + # We default to the globals of `owner`, + # so it no longer raises `NameError` + self.assertIs( + typing.evaluate_forward_ref(ref, owner=Loop), A + ) + + def test_inherited_owner(self): + # owner passed to evaluate_forward_ref + ref = ForwardRef("list['A']") + self.assertEqual( + typing.evaluate_forward_ref(ref, owner=Loop), + list[A], + ) + + # owner set on the ForwardRef + ref = ForwardRef("list['A']", owner=Loop) + self.assertEqual( + typing.evaluate_forward_ref(ref), + list[A], + ) + + def test_partial_evaluation(self): + ref = ForwardRef("list[A]") + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + list[EqualToForwardRef('A')], + ) + + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) + + class CollectionsAbcTests(BaseTestCase): def test_hashable(self): @@ -7985,6 +8145,48 @@ class XMethBad2(NamedTuple): def _source(self): return 'no chance for this as well' + def test_annotation_type_check(self): + # These are rejected by _type_check + with self.assertRaises(TypeError): + class X(NamedTuple): + a: Final + with self.assertRaises(TypeError): + class Y(NamedTuple): + a: (1, 2) + + # Conversion by _type_convert + class Z(NamedTuple): + a: None + b: "str" + annos = {'a': type(None), 'b': EqualToForwardRef("str")} + self.assertEqual(Z.__annotations__, annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + from typing import NamedTuple + class X(NamedTuple): + a: int + b: None + """ + ns = run_code(textwrap.dedent(code)) + X = ns['X'] + self.assertEqual(X.__annotations__, {'a': EqualToForwardRef("int"), 'b': EqualToForwardRef("None")}) + + def test_deferred_annotations(self): + class X(NamedTuple): + y: undefined + + self.assertEqual(X._fields, ('y',)) + with self.assertRaises(NameError): + X.__annotations__ + + undefined = int + self.assertEqual(X.__annotations__, {'y': int}) + def test_multiple_inheritance(self): class A: pass @@ -8144,7 +8346,6 @@ class CNT(NamedTuple): self.assertEqual(struct.__annotations__, {}) self.assertIsInstance(struct(), struct) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() @@ -8232,7 +8433,6 @@ class Bar(NamedTuple): self.assertIsInstance(bar.attr, Vanilla) self.assertEqual(bar.attr.name, "attr") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_setname_raises_the_same_as_on_other_classes(self): class CustomException(BaseException): pass @@ -8287,6 +8487,23 @@ class VeryAnnoying(metaclass=Meta): pass class Foo(NamedTuple): attr = very_annoying + def test_super_explicitly_disallowed(self): + expected_message = ( + "uses of super() and __class__ are unsupported " + "in methods of NamedTuple subclasses" + ) + + with self.assertRaises(TypeError, msg=expected_message): + class ThisWontWork(NamedTuple): + def __repr__(self): + return super().__repr__() + + with self.assertRaises(TypeError, msg=expected_message): + class ThisWontWorkEither(NamedTuple): + @property + def name(self): + return __class__.__name__ + class TypedDictTests(BaseTestCase): def test_basics_functional_syntax(self): @@ -8301,7 +8518,11 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__name__, 'Emp') self.assertEqual(Emp.__module__, __name__) self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + annos = {'name': str, 'id': int} + self.assertEqual(Emp.__annotations__, annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8500,6 +8721,36 @@ class Child(Base1, Base2): self.assertEqual(Child.__required_keys__, frozenset(['a'])) self.assertEqual(Child.__optional_keys__, frozenset()) + def test_inheritance_pep563(self): + def _make_td(future, class_name, annos, base, extra_names=None): + lines = [] + if future: + lines.append('from __future__ import annotations') + lines.append('from typing import TypedDict') + lines.append(f'class {class_name}({base}):') + for name, anno in annos.items(): + lines.append(f' {name}: {anno}') + code = '\n'.join(lines) + ns = run_code(code, extra_names) + return ns[class_name] + + for base_future in (True, False): + for child_future in (True, False): + with self.subTest(base_future=base_future, child_future=child_future): + base = _make_td( + base_future, "Base", {"base": "int"}, "TypedDict" + ) + self.assertIsNotNone(base.__annotate__) + child = _make_td( + child_future, "Child", {"child": "int"}, "Base", {"Base": base} + ) + base_anno = ForwardRef("int", module="builtins", owner=base) if base_future else int + child_anno = ForwardRef("int", module="builtins", owner=child) if child_future else int + self.assertEqual(base.__annotations__, {'base': base_anno}) + self.assertEqual( + child.__annotations__, {'child': child_anno, 'base': base_anno} + ) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) @@ -8648,14 +8899,12 @@ class NewGeneric[T](TypedDict): # The TypedDict constructor is not itself a TypedDict self.assertIs(is_typeddict(TypedDict), False) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints(self): self.assertEqual( get_type_hints(Bar), {'a': typing.Optional[int], 'b': int} ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints_generic(self): self.assertEqual( get_type_hints(BarGeneric), @@ -8680,6 +8929,8 @@ class A[T](TypedDict): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8691,6 +8942,8 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8701,6 +8954,8 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__bases__, (Generic, dict)) self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__annotations__, {'a': T}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8711,6 +8966,8 @@ class B(A[KT], total=False): self.assertEqual(B.__bases__, (Generic, dict)) self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8735,6 +8992,11 @@ class C(B[int]): 'b': KT, 'c': int, }) + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): C[str] @@ -8754,6 +9016,11 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'T', + 'c': 'KT', + }) self.assertEqual(Point3D[int, str].__origin__, Point3D) with self.assertRaises(TypeError): @@ -8785,10 +9052,14 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): WithImplicitAny[str] - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_generic_subscript(self): # For backward compatibility, subscription works # on arbitrary TypedDict types. @@ -8916,7 +9187,6 @@ class Child(Base): self.assertEqual(Child.__readonly_keys__, frozenset()) self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_combine_qualifiers(self): class AllTheThings(TypedDict): a: Annotated[Required[ReadOnly[int]], "why not"] @@ -8943,6 +9213,54 @@ class AllTheThings(TypedDict): }, ) + def test_annotations(self): + # _type_check is applied + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + class X(TypedDict): + a: Final + + # _type_convert is applied + class Y(TypedDict): + a: None + b: "int" + fwdref = EqualToForwardRef('int', module=__name__) + self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) + self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) + + # _type_check is also applied later + class Z(TypedDict): + a: undefined + + with self.assertRaises(NameError): + Z.__annotations__ + + undefined = Final + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + Z.__annotations__ + + undefined = None + self.assertEqual(Z.__annotations__, {'a': type(None)}) + + def test_deferred_evaluation(self): + class A(TypedDict): + x: NotRequired[undefined] + y: ReadOnly[undefined] + z: Required[undefined] + + self.assertEqual(A.__required_keys__, frozenset({'y', 'z'})) + self.assertEqual(A.__optional_keys__, frozenset({'x'})) + self.assertEqual(A.__readonly_keys__, frozenset({'y'})) + self.assertEqual(A.__mutable_keys__, frozenset({'x', 'z'})) + + with self.assertRaises(NameError): + A.__annotations__ + + self.assertEqual( + A.__annotate__(annotationlib.Format.STRING), + {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', + 'z': 'Required[undefined]'}, + ) + class RequiredTests(BaseTestCase): @@ -9110,7 +9428,6 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cannot_subclass(self): with self.assertRaisesRegex( TypeError, @@ -9587,6 +9904,19 @@ class B(str): ... self.assertIs(type(field_c2.__metadata__[0]), float) self.assertIs(type(field_c3.__metadata__[0]), bool) + def test_forwardref_partial_evaluation(self): + # Test that Annotated partially evaluates if it contains a ForwardRef + # See: https://github.com/python/cpython/issues/137706 + def f(x: Annotated[undefined, '']): pass + + ann = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + + # Test that the attributes are retrievable from the partially evaluated annotation + x_ann = ann['x'] + self.assertIs(get_origin(x_ann), Annotated) + self.assertEqual(x_ann.__origin__, EqualToForwardRef('undefined', owner=f)) + self.assertEqual(x_ann.__metadata__, ('',)) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -10093,6 +10423,7 @@ def test_var_substitution(self): self.assertEqual(C[Concatenate[str, P2]], Concatenate[int, str, P2]) self.assertEqual(C[...], Concatenate[int, ...]) + class TypeGuardTests(BaseTestCase): def test_basics(self): TypeGuard[int] # OK @@ -10284,7 +10615,6 @@ def test_special_attrs(self): typing.ClassVar: 'ClassVar', typing.Concatenate: 'Concatenate', typing.Final: 'Final', - typing.ForwardRef: 'ForwardRef', typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', @@ -10294,9 +10624,8 @@ def test_special_attrs(self): typing.TypeGuard: 'TypeGuard', typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', - typing.Union: 'Union', typing.Self: 'Self', - # Subscribed special forms + # Subscripted special forms typing.Annotated[Any, "Annotation"]: 'Annotated', typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', @@ -10305,13 +10634,12 @@ def test_special_attrs(self): typing.Literal[Any]: 'Literal', typing.Literal[1, 2]: 'Literal', typing.Literal[True, 2]: 'Literal', - typing.Optional[Any]: 'Optional', + typing.Optional[Any]: 'Union', typing.TypeGuard[Any]: 'TypeGuard', typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) - # - typing.ForwardRef('set[Any]') # - typing.NewType('TypeName', Any) # - typing.ParamSpec('SpecialAttrsP') # - typing.TypeVar('T') @@ -10325,24 +10653,14 @@ def test_special_attrs(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(cls, proto) loaded = pickle.loads(s) - self.assertIs(cls, loaded) + if isinstance(cls, Union): + self.assertEqual(cls, loaded) + else: + self.assertIs(cls, loaded) TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_attrs2(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = typing.ForwardRef('set[Any]') - self.assertFalse(hasattr(fr, '__name__')) - self.assertFalse(hasattr(fr, '__qualname__')) - self.assertEqual(fr.__module__, 'typing') - # Forward refs are currently unpicklable. - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') self.assertEqual( SpecialAttrsTests.TypeName.__qualname__, @@ -10363,7 +10681,7 @@ def test_special_attrs2(self): # to the variable name to which it is assigned". Thus, providing # __qualname__ is unnecessary. self.assertEqual(SpecialAttrsT.__name__, 'SpecialAttrsT') - self.assertFalse(hasattr(SpecialAttrsT, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsT, '__qualname__') self.assertEqual(SpecialAttrsT.__module__, __name__) # Module-level type variables are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -10372,7 +10690,7 @@ def test_special_attrs2(self): self.assertIs(SpecialAttrsT, loaded) self.assertEqual(SpecialAttrsP.__name__, 'SpecialAttrsP') - self.assertFalse(hasattr(SpecialAttrsP, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsP, '__qualname__') self.assertEqual(SpecialAttrsP.__module__, __name__) # Module-level ParamSpecs are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -10496,7 +10814,6 @@ class CustomerModel(ModelBase, init=False): class NoDefaultTests(BaseTestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(NoDefault, proto) @@ -10522,7 +10839,6 @@ def test_no_call(self): with self.assertRaises(TypeError): NoDefault() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_attributes(self): with self.assertRaises(AttributeError): NoDefault.foo = 3 @@ -10598,7 +10914,6 @@ class TypeIterationTests(BaseTestCase): Annotated[T, ''], ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cannot_iterate(self): expected_error_regex = "object is not iterable" for test_type in self._UNITERABLE_TYPES: @@ -10616,6 +10931,37 @@ def test_is_not_instance_of_iterable(self): self.assertNotIsInstance(type_to_test, collections.abc.Iterable) +class UnionGenericAliasTests(BaseTestCase): + def test_constructor(self): + # Used e.g. in typer, pydantic + with self.assertWarns(DeprecationWarning): + inst = typing._UnionGenericAlias(typing.Union, (int, str)) + self.assertEqual(inst, int | str) + with self.assertWarns(DeprecationWarning): + # name is accepted but ignored + inst = typing._UnionGenericAlias(typing.Union, (int, None), name="Optional") + self.assertEqual(inst, int | None) + + def test_isinstance(self): + # Used e.g. in pydantic + with self.assertWarns(DeprecationWarning): + self.assertTrue(isinstance(Union[int, str], typing._UnionGenericAlias)) + with self.assertWarns(DeprecationWarning): + self.assertFalse(isinstance(int, typing._UnionGenericAlias)) + + def test_eq(self): + # type(t) == _UnionGenericAlias is used in vyos + with self.assertWarns(DeprecationWarning): + self.assertEqual(Union, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertEqual(typing._UnionGenericAlias, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertNotEqual(int, typing._UnionGenericAlias) + + def test_hashable(self): + self.assertEqual(hash(typing._UnionGenericAlias), hash(Union)) + + def load_tests(loader, tests, pattern): import doctest tests.addTests(doctest.DocTestSuite(typing)) diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 69b58da0202..10f262cff12 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -203,7 +203,6 @@ def check_version(testfile): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(seqname) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(TypeError, unicodedata.name) self.assertRaises(TypeError, unicodedata.name, 'xx') diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index e14123aaa65..ceae20e8cb2 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -24,10 +24,9 @@ class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '63aa77dcb36b0e1df082ee2a6071caeda7f0955e' + expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + 9e43ee3929471739680c0e705482b4ae1c4122e4 @requires_resource('cpu') def test_method_checksum(self): h = hashlib.sha1() @@ -79,10 +78,9 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '232affd2a50ec4bd69d2482aa0291385cbdefaba' + expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'digit' @requires_resource('cpu') def test_function_checksum(self): data = [] @@ -122,11 +120,9 @@ def test_no_names_in_pua(self): char = chr(i) self.assertRaises(ValueError, self.db.name, char) - # TODO: RUSTPYTHON; LookupError: undefined character name 'LATIN SMLL LETR A' - @unittest.expectedFailure def test_lookup_nonexistant(self): # just make sure that lookup can fail - for nonexistant in [ + for nonexistent in [ "LATIN SMLL LETR A", "OPEN HANDS SIGHS", "DREGS", @@ -134,10 +130,8 @@ def test_lookup_nonexistant(self): "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", ]: - self.assertRaises(KeyError, self.db.lookup, nonexistant) + self.assertRaises(KeyError, self.db.lookup, nonexistent) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_digit(self): self.assertEqual(self.db.digit('A', None), None) self.assertEqual(self.db.digit('9'), 9) @@ -150,8 +144,6 @@ def test_digit(self): self.assertRaises(TypeError, self.db.digit, 'xx') self.assertRaises(ValueError, self.db.digit, 'x') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_numeric(self): self.assertEqual(self.db.numeric('A',None), None) self.assertEqual(self.db.numeric('9'), 9) @@ -165,8 +157,6 @@ def test_numeric(self): self.assertRaises(TypeError, self.db.numeric, 'xx') self.assertRaises(ValueError, self.db.numeric, 'x') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decimal(self): self.assertEqual(self.db.decimal('A',None), None) self.assertEqual(self.db.decimal('9'), 9) @@ -189,8 +179,7 @@ def test_category(self): self.assertRaises(TypeError, self.db.category) self.assertRaises(TypeError, self.db.category, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - L def test_bidirectional(self): self.assertEqual(self.db.bidirectional('\uFFFE'), '') self.assertEqual(self.db.bidirectional(' '), 'WS') @@ -200,8 +189,6 @@ def test_bidirectional(self): self.assertRaises(TypeError, self.db.bidirectional) self.assertRaises(TypeError, self.db.bidirectional, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decomposition(self): self.assertEqual(self.db.decomposition('\uFFFE'),'') self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034') @@ -218,8 +205,6 @@ def test_mirrored(self): self.assertRaises(TypeError, self.db.mirrored) self.assertRaises(TypeError, self.db.mirrored, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_combining(self): self.assertEqual(self.db.combining('\uFFFE'), 0) self.assertEqual(self.db.combining('a'), 0) @@ -247,8 +232,7 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ? + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -275,8 +259,7 @@ def test_east_asian_width(self): self.assertEqual(eaw('\u2010'), 'A') self.assertEqual(eaw('\U00020000'), 'W') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + W def test_east_asian_width_unassigned(self): eaw = self.db.east_asian_width # unassigned @@ -294,8 +277,7 @@ def test_east_asian_width_unassigned(self): self.assertEqual(eaw(char), 'A') self.assertIs(self.db.name(char, None), None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + N def test_east_asian_width_9_0_changes(self): self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N') self.assertEqual(self.db.east_asian_width('\u231a'), 'W') @@ -307,8 +289,7 @@ def test_disallow_instantiation(self): # Ensure that the type disallows instantiation (bpo-43916) check_disallow_instantiation(self, unicodedata.UCD) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- @force_not_colorized def test_failed_import_during_compiling(self): # Issue 4367 @@ -326,8 +307,6 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, @@ -341,8 +320,6 @@ def test_decimal_numeric_consistent(self): count += 1 self.assertTrue(count >= 10) # should have tested at least the ASCII digits - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_digit_numeric_consistent(self): # Test that digit and numeric are consistent, # i.e. if a character has a digit value, @@ -359,8 +336,7 @@ def test_digit_numeric_consistent(self): def test_bug_1704793(self): self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_ucd_510(self): import unicodedata # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0 @@ -384,8 +360,7 @@ def test_bug_5828(self): [0] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + Dž def test_bug_4971(self): # LETTER DZ WITH CARON: DZ, Dz, dz self.assertEqual("\u01c4".title(), "\u01c5") @@ -414,7 +389,6 @@ def unistr(data): data = [int(x, 16) for x in data.split(" ")] return "".join([chr(x) for x in data]) - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_resource('network') @requires_resource('cpu') def test_normalization(self): @@ -502,6 +476,29 @@ def test_bug_834676(self): # Check for bug 834676 unicodedata.normalize('NFC', '\ud55c\uae00') + def test_normalize_return_type(self): + # gh-129569: normalize() return type must always be str + normalize = unicodedata.normalize + + class MyStr(str): + pass + + normalization_forms = ("NFC", "NFKC", "NFD", "NFKD") + input_strings = ( + # normalized strings + "", + "ascii", + # unnormalized strings + "\u1e0b\u0323", + "\u0071\u0307\u0323", + ) + + for form in normalization_forms: + for input_str in input_strings: + with self.subTest(form=form, input_str=input_str): + self.assertIs(type(normalize(form, input_str)), str) + self.assertIs(type(normalize(form, MyStr(input_str))), str) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unittest/__init__.py b/Lib/test/test_unittest/__init__.py index bc502ef32d2..365f26d6438 100644 --- a/Lib/test/test_unittest/__init__.py +++ b/Lib/test/test_unittest/__init__.py @@ -1,4 +1,5 @@ import os.path + from test.support import load_package_tests diff --git a/Lib/test/test_unittest/__main__.py b/Lib/test/test_unittest/__main__.py index 40a23a297ec..0d53bfab847 100644 --- a/Lib/test/test_unittest/__main__.py +++ b/Lib/test/test_unittest/__main__.py @@ -1,4 +1,5 @@ -from . import load_tests import unittest +from . import load_tests + unittest.main() diff --git a/Lib/test/test_unittest/_test_warnings.py b/Lib/test/test_unittest/_test_warnings.py index 08b846ee47e..d9f41a4144b 100644 --- a/Lib/test/test_unittest/_test_warnings.py +++ b/Lib/test/test_unittest/_test_warnings.py @@ -14,6 +14,7 @@ import unittest import warnings + def warnfun(): warnings.warn('rw', RuntimeWarning) diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 1dec947ea76..3d782573d7b 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -1,10 +1,11 @@ import datetime +import unittest import warnings import weakref -import unittest -from test.support import gc_collect from itertools import product +from test.support import gc_collect + class Test_Assertions(unittest.TestCase): def test_AlmostEqual(self): diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 1d2b87f161c..57228e78f8c 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -1,7 +1,9 @@ import asyncio import contextvars import unittest + from test import support +from test.support import force_not_colorized support.requires_working_socket(module=True) @@ -11,7 +13,7 @@ class MyException(Exception): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -48,8 +50,6 @@ def setUp(self): # starting a new event loop self.addCleanup(support.gc_collect) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_full_cycle(self): expected = ['setUp', 'asyncSetUp', @@ -254,6 +254,7 @@ async def on_cleanup(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup']) + @force_not_colorized def test_exception_in_tear_clean_up(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): @@ -296,8 +297,7 @@ async def on_cleanup2(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test class Nothing: @@ -316,18 +316,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] @@ -479,7 +482,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio.get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -488,10 +491,8 @@ async def test_demo1(self): result = test.run() self.assertTrue(result.wasSuccessful()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_loop_factory(self): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/test_break.py b/Lib/test/test_unittest/test_break.py index 1da98af3e74..8aa20008ac7 100644 --- a/Lib/test/test_unittest/test_break.py +++ b/Lib/test/test_unittest/test_break.py @@ -1,10 +1,10 @@ import gc import io import os -import sys import signal -import weakref +import sys import unittest +import weakref from test import support diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index ae6a2b94dba..6e77040c265 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1,26 +1,27 @@ import contextlib import difflib -import pprint +import inspect +import logging import pickle +import pprint import re import sys -import logging +import types +import unittest import warnings import weakref -import inspect -import types - +from collections import UserString from copy import deepcopy -from test import support - -import unittest +from test import support +from test.support import captured_stderr, gc_collect from test.test_unittest.support import ( - TestEquality, TestHashing, LoggingResult, LegacyLoggingResult, - ResultWithNoStartTestRunStopTestRun + LegacyLoggingResult, + LoggingResult, + ResultWithNoStartTestRunStopTestRun, + TestEquality, + TestHashing, ) -from test.support import captured_stderr, gc_collect - log_foo = logging.getLogger('foo') log_foobar = logging.getLogger('foo.bar') @@ -54,6 +55,10 @@ def tearDown(self): self.events.append('tearDown') +class List(list): + pass + + class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): ### Set up attributes used by inherited tests @@ -85,7 +90,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test().id()[-13:], '.Test.runTest') + self.assertEndsWith(Test().id(), '.Test.runTest') # test that TestCase can be instantiated with no args # primarily for use at the interactive interpreter @@ -106,7 +111,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test('test').id()[-10:], '.Test.test') + self.assertEndsWith(Test('test').id(), '.Test.test') # "class TestCase([methodName])" # ... @@ -325,18 +330,40 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + warnings.filterwarnings('ignore', + 'coroutine .* was never awaited', RuntimeWarning) + Foo('test1').run() + support.gc_collect() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): @@ -678,16 +705,136 @@ def testAssertIsNot(self): self.assertRaises(self.failureException, self.assertIsNot, thing, thing) def testAssertIsInstance(self): - thing = [] + thing = List() self.assertIsInstance(thing, list) - self.assertRaises(self.failureException, self.assertIsInstance, - thing, dict) + self.assertIsInstance(thing, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int) + self.assertEqual(str(cm.exception), + "[] is not an instance of <class 'int'>") + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, (int, float)) + self.assertEqual(str(cm.exception), + "[] is not an instance of any of (<class 'int'>, <class 'float'>)") + + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertNotIsInstance(self): - thing = [] - self.assertNotIsInstance(thing, dict) - self.assertRaises(self.failureException, self.assertNotIsInstance, - thing, list) + thing = List() + self.assertNotIsInstance(thing, int) + self.assertNotIsInstance(thing, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list) + self.assertEqual(str(cm.exception), + "[] is an instance of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, (int, list)) + self.assertEqual(str(cm.exception), + "[] is an instance of <class 'list'>") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertIsSubclass(self): + self.assertIsSubclass(List, list) + self.assertIsSubclass(List, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of <class 'int'>") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, (int, float)) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of any of (<class 'int'>, <class 'float'>)") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotIsSubclass(self): + self.assertNotIsSubclass(List, int) + self.assertNotIsSubclass(List, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, (int, list)) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertHasAttr(self): + a = List() + a.x = 1 + self.assertHasAttr(a, 'x') + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y') + self.assertEqual(str(cm.exception), + "'List' object has no attribute 'y'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(List, 'spam') + self.assertEqual(str(cm.exception), + "type object 'List' has no attribute 'spam'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(sys, 'nonexistent') + self.assertEqual(str(cm.exception), + "module 'sys' has no attribute 'nonexistent'") + + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotHasAttr(self): + a = List() + a.x = 1 + self.assertNotHasAttr(a, 'y') + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x') + self.assertEqual(str(cm.exception), + "'List' object has unexpected attribute 'x'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(List, 'append') + self.assertEqual(str(cm.exception), + "type object 'List' has unexpected attribute 'append'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(sys, 'modules') + self.assertEqual(str(cm.exception), + "module 'sys' has unexpected attribute 'modules'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertIn(self): animals = {'monkey': 'banana', 'cow': 'grass', 'seal': 'fish'} @@ -1842,6 +1989,186 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) + def testAssertStartsWith(self): + self.assertStartsWith('ababahalamaha', 'ababa') + self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertStartsWith(UserString('ababahalamaha'), 'ababa') + self.assertStartsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertStartsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertStartsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertStartsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotStartsWith(self): + self.assertNotStartsWith('ababahalamaha', 'amaha') + self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') + self.assertNotStartsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertNotStartsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertNotStartsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertNotStartsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertNotStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertEndsWith(self): + self.assertEndsWith('ababahalamaha', 'amaha') + self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEndsWith(UserString('ababahalamaha'), 'amaha') + self.assertEndsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertEndsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertEndsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertEndsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotEndsWith(self): + self.assertNotEndsWith('ababahalamaha', 'ababa') + self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') + self.assertNotEndsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertNotEndsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertNotEndsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertNotEndsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertNotEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + def testDeprecatedFailMethods(self): """Test that the deprecated fail* methods get removed in 3.12""" deprecated_names = [ @@ -1970,8 +2297,6 @@ def testNoCycles(self): del case self.assertFalse(wr()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_exception_leak(self): # Issue #19880: TestCase.run() should not keep a reference # to the exception diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 9e231ff8d2d..9ed3d04b1f8 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -1,15 +1,17 @@ import os.path -from os.path import abspath +import pickle import re import sys import types -import pickle -from test import support -from test.support import import_helper - import unittest import unittest.mock +from importlib._bootstrap_external import NamespaceLoader +from os.path import abspath + import test.test_unittest +from test import support +from test.support import import_helper +from test.test_importlib import util as test_util class TestableTestProgram(unittest.TestProgram): @@ -364,8 +366,6 @@ def __eq__(self, other): [(loader, [], 'test*.py'), (loader, [], 'test*.py')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_discover(self): loader = unittest.TestLoader() @@ -397,7 +397,7 @@ def restore_isdir(): self.addCleanup(restore_isdir) _find_tests_args = [] - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): _find_tests_args.append((start_dir, pattern)) return ['tests'] loader._find_tests = _find_tests @@ -412,8 +412,6 @@ def _find_tests(start_dir, pattern): self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_discover_should_not_persist_top_level_dir_between_calls(self): original_isfile = os.path.isfile original_isdir = os.path.isdir @@ -819,7 +817,7 @@ def test_discovery_from_dotted_path(self): expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): self.wasRun = True self.assertEqual(start_dir, expectedPath) return tests @@ -852,6 +850,55 @@ def restore(): 'Can not use builtin modules ' 'as dotted module names') + def test_discovery_from_dotted_namespace_packages(self): + loader = unittest.TestLoader() + + package = types.ModuleType('package') + package.__name__ = "tests" + package.__path__ = ['/a', '/b'] + package.__file__ = None + package.__spec__ = types.SimpleNamespace( + name=package.__name__, + loader=NamespaceLoader(package.__name__, package.__path__, None), + submodule_search_locations=['/a', '/b'] + ) + + def _import(packagename, *args, **kwargs): + sys.modules[packagename] = package + return package + + _find_tests_args = [] + def _find_tests(start_dir, pattern, namespace=None): + _find_tests_args.append((start_dir, pattern)) + return ['%s/tests' % start_dir] + + loader._find_tests = _find_tests + loader.suiteClass = list + + with unittest.mock.patch('builtins.__import__', _import): + # Since loader.discover() can modify sys.path, restore it when done. + with import_helper.DirsOnSysPath(): + # Make sure to remove 'package' from sys.modules when done. + with test_util.uncache('package'): + suite = loader.discover('package') + + self.assertEqual(suite, ['/a/tests', '/b/tests']) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_discovery_start_dir_is_namespace(self): + """Subdirectory discovery not affected if start_dir is a namespace pkg.""" + loader = unittest.TestLoader() + with ( + import_helper.DirsOnSysPath(os.path.join(os.path.dirname(__file__))), + test_util.uncache('namespace_test_pkg') + ): + suite = loader.discover('namespace_test_pkg') + self.assertEqual( + {list(suite)[0]._tests[0].__module__ for suite in suite._tests if list(suite)}, + # files under namespace_test_pkg.noop not discovered. + {'namespace_test_pkg.test_foo', 'namespace_test_pkg.bar.test_bar'}, + ) + def test_discovery_failed_discovery(self): from test.test_importlib import util diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index 9881335318e..0acefccf7f6 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,9 +1,9 @@ import functools import sys import types - import unittest + class Test_TestLoader(unittest.TestCase): ### Basic object tests @@ -76,7 +76,7 @@ def runTest(self): loader = unittest.TestLoader() # This has to be false for the test to succeed - self.assertFalse('runTest'.startswith(loader.testMethodPrefix)) + self.assertNotStartsWith('runTest', loader.testMethodPrefix) suite = loader.loadTestsFromTestCase(Foo) self.assertIsInstance(suite, loader.suiteClass) @@ -90,8 +90,6 @@ def test_loadTestsFromTestCase__from_TestCase(self): self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure # "Do not load any tests from `FunctionTestCase` class." def test_loadTestsFromTestCase__from_FunctionTestCase(self): loader = unittest.TestLoader() @@ -121,8 +119,6 @@ def test(self): expected = [loader.suiteClass([MyTestCase('test')])] self.assertEqual(list(suite), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure # "This test ensures that internal `TestCase` subclasses are not loaded" def test_loadTestsFromModule__TestCase_subclass_internals(self): # See https://github.com/python/cpython/issues/84867 @@ -187,8 +183,6 @@ class NotAModule(object): self.assertEqual(list(suite), reference) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Check that loadTestsFromModule honors a module # with a load_tests function. def test_loadTestsFromModule__load_tests(self): diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 8256aaeb8c3..99c5ec48b67 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -1,9 +1,10 @@ import os -import sys import subprocess -from test import support +import sys import unittest + import test.test_unittest +from test import support from test.test_unittest.test_result import BufferedWriter @@ -75,6 +76,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -127,14 +136,14 @@ def test_NonExit(self): argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), testLoader=self.TestLoader(self.FooBar)) - self.assertTrue(hasattr(program, 'result')) + self.assertHasAttr(program, 'result') out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_Exit(self): stream = BufferedWriter() @@ -151,7 +160,7 @@ def test_Exit(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitAsDefault(self): stream = BufferedWriter() @@ -166,7 +175,7 @@ def test_ExitAsDefault(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitSkippedSuite(self): stream = BufferedWriter() @@ -178,7 +187,7 @@ def test_ExitSkippedSuite(self): self.assertEqual(cm.exception.code, 0) out = stream.getvalue() expected = '\n\nOK (skipped=1)\n' - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitEmptySuite(self): stream = BufferedWriter() @@ -191,6 +200,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False @@ -485,8 +506,7 @@ def testParseArgsSelectedTestNames(self): self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIf(sys.platform != 'win32', "TODO: RUSTPYTHON") + @unittest.expectedFailureIf(sys.platform != 'win32', 'TODO: RUSTPYTHON') def testSelectedTestNamesFunctionalTest(self): def run_unittest(args): # Use -E to ignore PYTHONSAFEPATH env var diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 4d552d54e9a..c260f90bf03 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -4,8 +4,12 @@ import traceback import unittest from unittest.util import strclass -from test.support import warnings_helper -from test.support import captured_stdout, force_not_colorized_test_class + +from test.support import ( + captured_stdout, + force_not_colorized_test_class, + warnings_helper, +) from test.test_unittest.support import BufferedWriter @@ -13,7 +17,7 @@ class MockTraceback(object): class TracebackException: def __init__(self, *args, **kwargs): self.capture_locals = kwargs.get('capture_locals', False) - def format(self): + def format(self, **kwargs): result = ['A traceback'] if self.capture_locals: result.append('locals') @@ -186,7 +190,7 @@ def test_1(self): test = Foo('test_1') try: test.fail("foo") - except: + except AssertionError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -214,7 +218,7 @@ def test_1(self): def get_exc_info(): try: test.fail("foo") - except: + except AssertionError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -241,9 +245,9 @@ def get_exc_info(): try: try: test.fail("foo") - except: + except AssertionError: raise ValueError(42) - except: + except ValueError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -271,7 +275,7 @@ def get_exc_info(): loop.__cause__ = loop loop.__context__ = loop raise loop - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -300,7 +304,7 @@ def get_exc_info(): ex1.__cause__ = ex2 ex2.__context__ = ex1 raise C - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -345,7 +349,7 @@ def test_1(self): test = Foo('test_1') try: raise TypeError() - except: + except TypeError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -454,7 +458,7 @@ def test(result): self.assertTrue(result.failfast) result = runner.run(test) stream.flush() - self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) + self.assertEndsWith(stream.getvalue(), '\n\nOK\n') @force_not_colorized_test_class diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index a652c71513d..b215a3664d1 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1,13 +1,12 @@ import io import os -import sys import pickle import subprocess -from test import support - +import sys import unittest from unittest.case import _Outcome +from test import support from test.test_unittest.support import ( BufferedWriter, LoggingResult, @@ -1297,8 +1296,6 @@ def _makeResult(self): expected = ['startTestRun', 'stopTestRun'] self.assertEqual(events, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle_unpickle(self): # Issue #7197: a TextTestRunner should be (un)pickleable. This is # required by test_multiprocessing under Windows (in verbose mode). diff --git a/Lib/test/test_unittest/test_setups.py b/Lib/test/test_unittest/test_setups.py index 2df703ed934..2468681003b 100644 --- a/Lib/test/test_unittest/test_setups.py +++ b/Lib/test/test_unittest/test_setups.py @@ -1,6 +1,5 @@ import io import sys - import unittest diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index f146dcac18e..f5cb860c60b 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,5 +1,6 @@ import unittest +from test.support import force_not_colorized from test.test_unittest.support import LoggingResult @@ -293,6 +294,7 @@ def test_die(self): self.assertFalse(result.unexpectedSuccesses) self.assertTrue(result.wasSuccessful()) + @force_not_colorized def test_expected_failure_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -372,6 +374,7 @@ def test_die(self): self.assertEqual(result.unexpectedSuccesses, [test]) self.assertFalse(result.wasSuccessful()) + @force_not_colorized def test_unexpected_success_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure diff --git a/Lib/test/test_unittest/test_suite.py b/Lib/test/test_unittest/test_suite.py index ca52ee9d9c0..11c8c859f3d 100644 --- a/Lib/test/test_unittest/test_suite.py +++ b/Lib/test/test_unittest/test_suite.py @@ -1,10 +1,9 @@ -import unittest - import gc import sys +import unittest import weakref -from test.test_unittest.support import LoggingResult, TestEquality +from test.test_unittest.support import LoggingResult, TestEquality ### Support code for Test_TestSuite ################################################################ diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py index d590a333930..abadcb96601 100644 --- a/Lib/test/test_unittest/test_util.py +++ b/Lib/test/test_unittest/test_util.py @@ -1,5 +1,9 @@ import unittest -from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference +from unittest.util import ( + safe_repr, + sorted_list_difference, + unorderable_list_difference, +) class TestUtil(unittest.TestCase): diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 06ec7ea2045..81d9c9c55fd 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -340,8 +340,7 @@ def test_spec_async_attributes_instance(self): # only the shape of the spec at the time of mock construction matters self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -356,8 +355,7 @@ def inner_test(mock_type): with self.subTest(f"test spec kwarg with {mock_type}"): inner_test(mock_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_spec_mock_type_positional(self): def inner_test(mock_type): async_mock = mock_type(async_func) @@ -736,8 +734,6 @@ def __aiter__(self): pass async def __anext__(self): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_aiter_set_return_value(self): mock_iter = AsyncMock(name="tester") mock_iter.__aiter__.return_value = [1, 2, 3] @@ -763,8 +759,6 @@ def inner_test(mock_type): inner_test(mock_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mock_async_for(self): async def iterate(iterator): accumulator = [] @@ -810,8 +804,7 @@ async def _runnable_test(self, *args, **kwargs): async def _await_coroutine(self, coroutine): return await coroutine - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -852,8 +845,7 @@ def test_assert_called_and_awaited_at_same_time(self): self.mock.assert_called_once() self.mock.assert_awaited_once() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_twice_and_awaited_once(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -868,8 +860,7 @@ def test_assert_called_twice_and_awaited_once(self): mock.async_method.assert_awaited() mock.async_method.assert_awaited_once() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_once_and_awaited_twice(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -894,8 +885,7 @@ def test_assert_awaited_but_not_called(self): with self.assertRaises(AssertionError): self.mock.assert_called() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_calls_not_awaits(self): kalls = [call('foo')] with assertNeverAwaited(self): @@ -904,8 +894,7 @@ def test_assert_has_calls_not_awaits(self): with self.assertRaises(AssertionError): self.mock.assert_has_awaits(kalls) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_mock_calls_on_async_mock_no_spec(self): with assertNeverAwaited(self): self.mock() @@ -919,8 +908,7 @@ def test_assert_has_mock_calls_on_async_mock_no_spec(self): mock_kalls = ([call(), call('foo'), call('baz')]) self.assertEqual(self.mock.mock_calls, mock_kalls) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_mock_calls_on_async_mock_with_spec(self): a_class_mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -936,8 +924,7 @@ def test_assert_has_mock_calls_on_async_mock_with_spec(self): self.assertEqual(a_class_mock.async_method.mock_calls, method_kalls) self.assertEqual(a_class_mock.mock_calls, mock_kalls) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_async_method_calls_recorded(self): with assertNeverAwaited(self): self.mock.something(3, fish=None) @@ -953,8 +940,7 @@ def test_async_method_calls_recorded(self): [("something", (6,), {'cake': sentinel.Cake})], "method calls not recorded correctly") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_async_arg_lists(self): def assert_attrs(mock): names = ('call_args_list', 'method_calls', 'mock_calls') diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 5facac685fd..a19f04eb88f 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -43,8 +43,7 @@ def test_any_and_datetime(self): mock.assert_called_with(ANY, foo=ANY) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_any_mock_calls_comparison_order(self): mock = Mock() class Foo(object): @@ -884,8 +883,6 @@ def f(a, self): pass a.f.assert_called_with(self=10) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_autospec_data_descriptor(self): class Descriptor(object): def __init__(self, value): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index 62c6221f774..87424e07e4e 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -1778,8 +1778,6 @@ def test(mock): 'exception traceback not propagated') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_resolution_import_rebinding(self): # Currently mock.patch uses pkgutil.resolve_name(), but repeat # similar tests just for the case. @@ -1814,8 +1812,6 @@ def check_error(name): check('package3:submodule.B.attr') check_error('package3:submodule.A.attr') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_resolution_import_rebinding2(self): path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') def check(name): @@ -2016,8 +2012,7 @@ def test_patch_and_patch_dict_stopall(self): self.assertEqual(dic2, origdic2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_attrs(self): def foo(x=0): """TEST""" diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index 515ec128a08..305da05b7ce 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -18,6 +18,13 @@ >>> a == 4 and b == 5 and c == 6 True +Unpack dict + + >>> d = {4: 'four', 5: 'five', 6: 'six'} + >>> a, b, c = d + >>> a == 4 and b == 5 and c == 6 + True + Unpack implied tuple >>> a, b, c = 7, 8, 9 @@ -66,14 +73,14 @@ >>> a, b = t Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking tuple of wrong size >>> a, b = l Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking sequence too short @@ -140,14 +147,59 @@ >>> () = [42] Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 0) + ValueError: too many values to unpack (expected 0, got 1) + +Unpacking a larger iterable should raise ValuleError, but it +should not entirely consume the iterable + >>> it = iter(range(100)) + >>> x, y, z = it + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + >>> next(it) + 4 + +Unpacking unbalanced dict + + >>> d = {4: 'four', 5: 'five', 6: 'six', 7: 'seven'} + >>> a, b, c = d + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3, got 4) + +Ensure that custom `__len__()` is NOT called when showing the error message + + >>> class LengthTooLong: + ... def __len__(self): + ... return 5 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = LengthTooLong() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + +For evil cases like these as well, no actual count to be shown + + >>> class BadLength: + ... def __len__(self): + ... return 1 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = BadLength() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) """ __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # XXX: RUSTPYTHON return tests diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py new file mode 100644 index 00000000000..1496e3be93f --- /dev/null +++ b/Lib/test/test_unpack_ex.py @@ -0,0 +1,413 @@ +# Tests for extended unpacking, starred expressions. + +import doctest +import unittest + + +doctests = """ + +Unpack tuple + + >>> t = (1, 2, 3) + >>> a, *b, c = t + >>> a == 1 and b == [2] and c == 3 + True + +Unpack list + + >>> l = [4, 5, 6] + >>> a, *b = l + >>> a == 4 and b == [5, 6] + True + +Unpack implied tuple + + >>> *a, = 7, 8, 9 + >>> a == [7, 8, 9] + True + +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + +Unpack string... fun! + + >>> a, *b = 'one' + >>> a == 'o' and b == ['n', 'e'] + True + +Unpack long sequence + + >>> a, b, c, *d, e, f, g = range(10) + >>> (a, b, c, d, e, f, g) == (0, 1, 2, [3, 4, 5, 6], 7, 8, 9) + True + +Unpack short sequence + + >>> a, *b, c = (1, 2) + >>> a == 1 and c == 2 and b == [] + True + +Unpack generic sequence + + >>> class Seq: + ... def __getitem__(self, i): + ... if i >= 0 and i < 3: return i + ... raise IndexError + ... + >>> a, *b = Seq() + >>> a == 0 and b == [1, 2] + True + +Unpack in for statement + + >>> for a, *b, c in [(1,2,3), (4,5,6,7)]: + ... print(a, b, c) + ... + 1 [2] 3 + 4 [5, 6] 7 + +Unpack in list + + >>> [a, *b, c] = range(5) + >>> a == 0 and b == [1, 2, 3] and c == 4 + True + +Multiple targets + + >>> a, *b, c = *d, e = range(5) + >>> a == 0 and b == [1, 2, 3] and c == 4 and d == [0, 1, 2, 3] and e == 4 + True + +Assignment unpacking + + >>> a, b, *c = range(5) + >>> a, b, c + (0, 1, [2, 3, 4]) + >>> *a, b, c = a, b, *c + >>> a, b, c + ([0, 1, 2], 3, 4) + +Set display element unpacking + + >>> a = [1, 2, 3] + >>> sorted({1, *a, 0, 4}) + [0, 1, 2, 3, 4] + + >>> {1, *1, 0, 4} + Traceback (most recent call last): + ... + TypeError: 'int' object is not iterable + +Dict display element unpacking + + >>> kwds = {'z': 0, 'w': 12} + >>> sorted({'x': 1, 'y': 2, **kwds}.items()) + [('w', 12), ('x', 1), ('y', 2), ('z', 0)] + + >>> sorted({**{'x': 1}, 'y': 2, **{'z': 3}}.items()) + [('x', 1), ('y', 2), ('z', 3)] + + >>> sorted({**{'x': 1}, 'y': 2, **{'x': 3}}.items()) + [('x', 3), ('y', 2)] + + >>> sorted({**{'x': 1}, **{'x': 3}, 'x': 4}.items()) + [('x', 4)] + + >>> {**{}} + {} + + >>> a = {} + >>> {**a}[0] = 1 + >>> a + {} + + >>> {**1} + Traceback (most recent call last): + ... + TypeError: 'int' object is not a mapping + + >>> {**[]} + Traceback (most recent call last): + ... + TypeError: 'list' object is not a mapping + + >>> len(eval("{" + ", ".join("**{{{}: {}}}".format(i, i) + ... for i in range(1000)) + "}")) + 1000 + + >>> {0:1, **{0:2}, 0:3, 0:4} + {0: 4} + +List comprehension element unpacking + + >>> a, b, c = [0, 1, 2], 3, 4 + >>> [*a, b, c] + [0, 1, 2, 3, 4] + + >>> l = [a, (3, 4), {5}, {6: None}, (i for i in range(7, 10))] + >>> [*item for item in l] # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*[0, 1] for i in range(10)] # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*'a' for i in range(10)] # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*[] for i in range(10)] # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> {**{} for a in [1]} # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: dict unpacking cannot be used in dict comprehension + +# Pegen is better here. +# Generator expression in function arguments + +# >>> list(*x for x in (range(5) for i in range(3))) +# Traceback (most recent call last): +# ... +# list(*x for x in (range(5) for i in range(3))) +# ^ +# SyntaxError: invalid syntax + + >>> dict(**x for x in [{1:2}]) + Traceback (most recent call last): + ... + dict(**x for x in [{1:2}]) + ^ + SyntaxError: invalid syntax + +Iterable argument unpacking + + >>> print(*[1], *[2], 3) + 1 2 3 + +Make sure that they don't corrupt the passed-in dicts. + + >>> def f(x, y): + ... print(x, y) + ... + >>> original_dict = {'x': 1} + >>> f(**original_dict, y=2) + 1 2 + >>> original_dict + {'x': 1} + +Now for some failures + +Make sure the raised errors are right for keyword argument unpackings + + >>> from collections.abc import MutableMapping + >>> class CrazyDict(MutableMapping): + ... def __init__(self): + ... self.d = {} + ... + ... def __iter__(self): + ... for x in self.d.__iter__(): + ... if x == 'c': + ... self.d['z'] = 10 + ... yield x + ... + ... def __getitem__(self, k): + ... return self.d[k] + ... + ... def __len__(self): + ... return len(self.d) + ... + ... def __setitem__(self, k, v): + ... self.d[k] = v + ... + ... def __delitem__(self, k): + ... del self.d[k] + ... + >>> d = CrazyDict() + >>> d.d = {chr(ord('a') + x): x for x in range(5)} + >>> e = {**d} + Traceback (most recent call last): + ... + RuntimeError: dictionary changed size during iteration + + >>> d.d = {chr(ord('a') + x): x for x in range(5)} + >>> def f(**kwargs): print(kwargs) + >>> f(**d) + Traceback (most recent call last): + ... + RuntimeError: dictionary changed size during iteration + +Overridden parameters + + >>> f(x=5, **{'x': 3}, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{'x': 3}, x=5, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{'x': 3}, **{'x': 5}, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(x=5, **{'x': 3}, **{'x': 2}) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{1: 3}, **{1: 5}) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument '1' + +Unpacking non-sequence + + >>> a, *b = 7 + Traceback (most recent call last): + ... + TypeError: cannot unpack non-iterable int object + +Unpacking sequence too short + + >>> a, *b, c, d, e = Seq() + Traceback (most recent call last): + ... + ValueError: not enough values to unpack (expected at least 4, got 3) + +Unpacking sequence too short and target appears last + + >>> a, b, c, d, *e = Seq() + Traceback (most recent call last): + ... + ValueError: not enough values to unpack (expected at least 4, got 3) + +Unpacking a sequence where the test for too long raises a different kind of +error + + >>> class BozoError(Exception): + ... pass + ... + >>> class BadSeq: + ... def __getitem__(self, i): + ... if i >= 0 and i < 3: + ... return i + ... elif i == 3: + ... raise BozoError + ... else: + ... raise IndexError + ... + +Trigger code while not expecting an IndexError (unpack sequence too long, wrong +error) + + >>> a, *b, c, d, e = BadSeq() + Traceback (most recent call last): + ... + test.test_unpack_ex.BozoError + +Now some general starred expressions (all fail). + + >>> a, *b, c, *d, e = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> [*b, *c] = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> a,*b,*c,*d = range(4) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> *a = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: starred assignment target must be in a list or tuple + + >>> *a # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> *1 # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> x = *a # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> (*x),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> (((*x))),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> z,(*x),y = 1, 2, 4 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> z,(*x) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> ((*x),y) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + +Some size constraints (all fail.) + + >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range(1<<8 + 1)" + >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: too many expressions in star-unpacking assignment + + >>> s = ", ".join("a%d" % i for i in range(1<<8 + 1)) + ", *rest = range(1<<8 + 2)" + >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: too many expressions in star-unpacking assignment + +(there is an additional limit, on the number of expressions after the +'*rest', but it's 1<<24 and testing it takes too much memory.) + +""" + +__test__ = {'doctests' : doctests} + + +def load_tests(loader, tests, pattern): + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # XXX: RUSTPYTHON + return tests + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py new file mode 100644 index 00000000000..35e4652a87b --- /dev/null +++ b/Lib/test/test_unparse.py @@ -0,0 +1,1066 @@ +"""Tests for ast.unparse.""" + +import unittest +import test.support +import pathlib +import random +import tokenize +import warnings +import ast +from test.support.ast_helper import ASTTestMixin + + +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + with tokenize.open(filename) as stream: + return stream.read() + + +for_else = """\ +def f(): + for x in range(10): + break + else: + y = 2 + z = 3 +""" + +while_else = """\ +def g(): + while True: + break + else: + y = 2 + z = 3 +""" + +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +nonlocal_ex = """\ +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + y = 7 + def h(): + nonlocal x, y +""" + +# also acts as test for 'except ... as ...' +raise_from = """\ +try: + 1 / 0 +except ZeroDivisionError as e: + raise ArithmeticError from e +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +try_except_star_finally = """\ +try: + suite1 +except* ex1: + suite2 +except* ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +with_simple = """\ +with f(): + suite1 +""" + +with_as = """\ +with f() as x: + suite1 +""" + +with_two_items = """\ +with f() as x, g() as y: + suite1 +""" + +docstring_prefixes = ( + "", + "class foo:\n ", + "def foo():\n ", + "async def foo():\n ", +) + +class ASTTestCase(ASTTestMixin, unittest.TestCase): + def check_ast_roundtrip(self, code1, **kwargs): + with self.subTest(code1=code1, ast_parse_kwargs=kwargs): + ast1 = ast.parse(code1, **kwargs) + code2 = ast.unparse(ast1) + ast2 = ast.parse(code2, **kwargs) + self.assertASTEqual(ast1, ast2) + + def check_invalid(self, node, raises=ValueError): + with self.subTest(node=node): + self.assertRaises(raises, ast.unparse, node) + + def get_source(self, code1, code2=None, **kwargs): + code2 = code2 or code1 + code1 = ast.unparse(ast.parse(code1, **kwargs)) + return code1, code2 + + def check_src_roundtrip(self, code1, code2=None, **kwargs): + code1, code2 = self.get_source(code1, code2, **kwargs) + with self.subTest(code1=code1, code2=code2): + self.assertEqual(code2, code1) + + def check_src_dont_roundtrip(self, code1, code2=None): + code1, code2 = self.get_source(code1, code2) + with self.subTest(code1=code1, code2=code2): + self.assertNotEqual(code2, code1) + +class UnparseTestCase(ASTTestCase): + # Tests for specific bugs found in earlier versions of unparse + + def test_fstrings(self): + self.check_ast_roundtrip("f'a'") + self.check_ast_roundtrip("f'{{}}'") + self.check_ast_roundtrip("f'{{5}}'") + self.check_ast_roundtrip("f'{{5}}5'") + self.check_ast_roundtrip("f'X{{}}X'") + self.check_ast_roundtrip("f'{a}'") + self.check_ast_roundtrip("f'{ {1:2}}'") + self.check_ast_roundtrip("f'a{a}a'") + self.check_ast_roundtrip("f'a{a}{a}a'") + self.check_ast_roundtrip("f'a{a}a{a}a'") + self.check_ast_roundtrip("f'{a!r}x{a!s}12{{}}{a!a}'") + self.check_ast_roundtrip("f'{a:10}'") + self.check_ast_roundtrip("f'{a:100_000{10}}'") + self.check_ast_roundtrip("f'{a!r:10}'") + self.check_ast_roundtrip("f'{a:a{b}10}'") + self.check_ast_roundtrip( + "f'a{b}{c!s}{d!r}{e!a}{f:a}{g:a{b}}{h!s:a}" + "{j!s:{a}b}{k!s:a{b}c}{l!a:{b}c{d}}{x+y=}'" + ) + + def test_fstrings_special_chars(self): + # See issue 25180 + self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_ast_roundtrip("""f''""") + self.check_ast_roundtrip('''f"""'end' "quote\\""""''') + + def test_fstrings_complicated(self): + # See issue 28002 + self.check_ast_roundtrip("""f'''{"'"}'''""") + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') + self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""') + self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""') + self.check_ast_roundtrip('''f"a\\r\\nb"''') + self.check_ast_roundtrip('''f"\\u2028{'x'}"''') + + def test_fstrings_pep701(self): + self.check_ast_roundtrip('f" something { my_dict["key"] } something else "') + self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"') + + def test_tstrings(self): + self.check_ast_roundtrip("t'foo'") + self.check_ast_roundtrip("t'foo {bar}'") + self.check_ast_roundtrip("t'foo {bar!s:.2f}'") + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + self.check_ast_roundtrip('t""') + self.check_ast_roundtrip("t'{(lambda x: x)}'") + self.check_ast_roundtrip("t'{t'{x}'}'") + + def test_tstring_with_nonsensical_str_field(self): + # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.Name(id="test1", ctx=ast.Load()), str="test2", conversion=-1 + ) + ] + ) + ), + "t'{test2}'", + ) + + def test_tstring_with_none_str_field(self): + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="test1"), str=None, conversion=-1)] + ) + ), + "t'{test1}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ + ast.Interpolation( + value=ast.Lambda( + args=ast.arguments(args=[ast.arg(arg="x")]), + body=ast.Name(id="x"), + ), + str=None, + conversion=-1, + ) + ] + ) + ), + "t'{(lambda x: x)}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + # `str` field kept here + [ast.Interpolation(value=ast.Name(id="x"), str="y", conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{y}'}"''', + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="x"), str=None, conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{x}'}"''', + ) + self.assertEqual( + ast.unparse(ast.TemplateStr( + [ast.Interpolation(value=ast.Constant(value="foo"), str=None, conversion=114)] + )), + '''t"{'foo'!r}"''', + ) + + def test_strings(self): + self.check_ast_roundtrip("u'foo'") + self.check_ast_roundtrip("r'foo'") + self.check_ast_roundtrip("b'foo'") + + def test_del_statement(self): + self.check_ast_roundtrip("del x, y, z") + + def test_shifts(self): + self.check_ast_roundtrip("45 << 2") + self.check_ast_roundtrip("13 >> 7") + + def test_for_else(self): + self.check_ast_roundtrip(for_else) + + def test_while_else(self): + self.check_ast_roundtrip(while_else) + + def test_unary_parens(self): + self.check_ast_roundtrip("(-1)**7") + self.check_ast_roundtrip("(-1.)**8") + self.check_ast_roundtrip("(-1j)**6") + self.check_ast_roundtrip("not True or False") + self.check_ast_roundtrip("True or not False") + + def test_integer_parens(self): + self.check_ast_roundtrip("3 .__abs__()") + + def test_huge_float(self): + self.check_ast_roundtrip("1e1000") + self.check_ast_roundtrip("-1e1000") + self.check_ast_roundtrip("1e1000j") + self.check_ast_roundtrip("-1e1000j") + + def test_nan(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Constant(value=float('nan')))), + ast.parse('1e1000 - 1e1000') + ) + + def test_min_int(self): + self.check_ast_roundtrip(str(-(2 ** 31))) + self.check_ast_roundtrip(str(-(2 ** 63))) + + def test_imaginary_literals(self): + self.check_ast_roundtrip("7j") + self.check_ast_roundtrip("-7j") + self.check_ast_roundtrip("0j") + self.check_ast_roundtrip("-0j") + + def test_lambda_parentheses(self): + self.check_ast_roundtrip("(lambda: int)()") + + def test_chained_comparisons(self): + self.check_ast_roundtrip("1 < 4 <= 5") + self.check_ast_roundtrip("a is b is c is not d") + + def test_function_arguments(self): + self.check_ast_roundtrip("def f(): pass") + self.check_ast_roundtrip("def f(a): pass") + self.check_ast_roundtrip("def f(b = 2): pass") + self.check_ast_roundtrip("def f(a, b): pass") + self.check_ast_roundtrip("def f(a, b = 2): pass") + self.check_ast_roundtrip("def f(a = 5, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b): pass") + self.check_ast_roundtrip("def f(*, a, b = 2): pass") + self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") + self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") + self.check_ast_roundtrip("def f(*args, **kwargs): pass") + + def test_relative_import(self): + self.check_ast_roundtrip(relative_import) + + def test_nonlocal(self): + self.check_ast_roundtrip(nonlocal_ex) + + def test_raise_from(self): + self.check_ast_roundtrip(raise_from) + + def test_bytes(self): + self.check_ast_roundtrip("b'123'") + + def test_annotations(self): + self.check_ast_roundtrip("def f(a : int): pass") + self.check_ast_roundtrip("def f(a: int = 5): pass") + self.check_ast_roundtrip("def f(*args: [int]): pass") + self.check_ast_roundtrip("def f(**kwargs: dict): pass") + self.check_ast_roundtrip("def f() -> None: pass") + + def test_set_literal(self): + self.check_ast_roundtrip("{'a', 'b', 'c'}") + + def test_empty_set(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Set(elts=[]))), + ast.parse('{*()}') + ) + + def test_set_comprehension(self): + self.check_ast_roundtrip("{x for x in range(5)}") + + def test_dict_comprehension(self): + self.check_ast_roundtrip("{x: x*x for x in range(10)}") + + def test_class_decorators(self): + self.check_ast_roundtrip(class_decorator) + + def test_class_definition(self): + self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") + + def test_elifs(self): + self.check_ast_roundtrip(elif1) + self.check_ast_roundtrip(elif2) + + def test_try_except_finally(self): + self.check_ast_roundtrip(try_except_finally) + + def test_try_except_star_finally(self): + self.check_ast_roundtrip(try_except_star_finally) + + def test_starred_assignment(self): + self.check_ast_roundtrip("a, *b, c = seq") + self.check_ast_roundtrip("a, (*b, c) = seq") + self.check_ast_roundtrip("a, *b[0], c = seq") + self.check_ast_roundtrip("a, *(b, c) = seq") + + def test_with_simple(self): + self.check_ast_roundtrip(with_simple) + + def test_with_as(self): + self.check_ast_roundtrip(with_as) + + def test_with_two_items(self): + self.check_ast_roundtrip(with_two_items) + + def test_dict_unpacking_in_dict(self): + # See issue 26489 + self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""") + self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""") + + def test_slices(self): + self.check_ast_roundtrip("a[i]") + self.check_ast_roundtrip("a[i,]") + self.check_ast_roundtrip("a[i, j]") + # The AST for these next two both look like `a[(*a,)]` + self.check_ast_roundtrip("a[(*a,)]") + self.check_ast_roundtrip("a[*a]") + self.check_ast_roundtrip("a[b, *a]") + self.check_ast_roundtrip("a[*a, c]") + self.check_ast_roundtrip("a[b, *a, c]") + self.check_ast_roundtrip("a[*a, *a]") + self.check_ast_roundtrip("a[b, *a, *a]") + self.check_ast_roundtrip("a[*a, b, *a]") + self.check_ast_roundtrip("a[*a, *a, b]") + self.check_ast_roundtrip("a[b, *a, *a, c]") + self.check_ast_roundtrip("a[(a:=b)]") + self.check_ast_roundtrip("a[(a:=b,c)]") + self.check_ast_roundtrip("a[()]") + self.check_ast_roundtrip("a[i:j]") + self.check_ast_roundtrip("a[:j]") + self.check_ast_roundtrip("a[i:]") + self.check_ast_roundtrip("a[i:j:k]") + self.check_ast_roundtrip("a[:j:k]") + self.check_ast_roundtrip("a[i::k]") + self.check_ast_roundtrip("a[i:j,]") + self.check_ast_roundtrip("a[i:j, k]") + + def test_invalid_raise(self): + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) + + def test_invalid_fstring_value(self): + self.check_invalid( + ast.JoinedStr( + values=[ + ast.Name(id="test", ctx=ast.Load()), + ast.Constant(value="test") + ] + ) + ) + + def test_fstring_backslash(self): + # valid since Python 3.12 + self.assertEqual(ast.unparse( + ast.FormattedValue( + value=ast.Constant(value="\\\\"), + conversion=-1, + format_spec=None, + ) + ), "{'\\\\\\\\'}") + + def test_invalid_yield_from(self): + self.check_invalid(ast.YieldFrom(value=None)) + + def test_import_from_level_none(self): + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')]) + self.assertEqual(ast.unparse(tree), "from mod import x") + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')], level=None) + self.assertEqual(ast.unparse(tree), "from mod import x") + + def test_docstrings(self): + docstrings = ( + 'this ends with double quote"', + 'this includes a """triple quote"""', + '\r', + '\\r', + '\t', + '\\t', + '\n', + '\\n', + '\r\\r\t\\t\n\\n', + '""">>> content = \"\"\"blabla\"\"\" <<<"""', + r'foo\n\x00', + "' \\'\\'\\'\"\"\" \"\"\\'\\' \\'", + '🐍⛎𩸽üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' + ) + for docstring in docstrings: + # check as Module docstrings for easy testing + self.check_ast_roundtrip(f"'''{docstring}'''") + + def test_constant_tuples(self): + locs = ast.fix_missing_locations + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1,)))])), "(1,)") + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1, 2, 3)))])), "(1, 2, 3)" + ) + + def test_function_type(self): + for function_type in ( + "() -> int", + "(int, int) -> int", + "(Callable[complex], More[Complex(call.to_typevar())]) -> None" + ): + self.check_ast_roundtrip(function_type, mode="func_type") + + def test_type_comments(self): + for statement in ( + "a = 5 # type:", + "a = 5 # type: int", + "a = 5 # type: int and more", + "def x(): # type: () -> None\n\tpass", + "def x(y): # type: (int) -> None and more\n\tpass", + "async def x(): # type: () -> None\n\tpass", + "async def x(y): # type: (int) -> None and more\n\tpass", + "for x in y: # type: int\n\tpass", + "async for x in y: # type: int\n\tpass", + "with x(): # type: int\n\tpass", + "async with x(): # type: int\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + def test_type_ignore(self): + for statement in ( + "a = 5 # type: ignore", + "a = 5 # type: ignore and more", + "def x(): # type: ignore\n\tpass", + "def x(y): # type: ignore and more\n\tpass", + "async def x(): # type: ignore\n\tpass", + "async def x(y): # type: ignore and more\n\tpass", + "for x in y: # type: ignore\n\tpass", + "async for x in y: # type: ignore\n\tpass", + "with x(): # type: ignore\n\tpass", + "async with x(): # type: ignore\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + def test_unparse_interactive_semicolons(self): + # gh-129598: Fix ast.unparse() when ast.Interactive contains multiple statements + self.check_src_roundtrip("i = 1; 'expr'; raise Exception", mode='single') + self.check_src_roundtrip("i: int = 1; j: float = 0; k += l", mode='single') + combinable = ( + "'expr'", + "(i := 1)", + "import foo", + "from foo import bar", + "i = 1", + "i += 1", + "i: int = 1", + "return i", + "pass", + "break", + "continue", + "del i", + "assert i", + "global i", + "nonlocal j", + "await i", + "yield i", + "yield from i", + "raise i", + "type t[T] = ...", + "i", + ) + for a in combinable: + for b in combinable: + self.check_src_roundtrip(f"{a}; {b}", mode='single') + + def test_unparse_interactive_integrity_1(self): + # rest of unparse_interactive_integrity tests just make sure mode='single' parse and unparse didn't break + self.check_src_roundtrip( + "if i:\n 'expr'\nelse:\n raise Exception", + "if i:\n 'expr'\nelse:\n raise Exception", + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\ndef func():\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\ndef func():\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\nclass cls:\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\nclass cls:\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + + def test_unparse_interactive_integrity_2(self): + for statement in ( + "def x():\n pass", + "def x(y):\n pass", + "async def x():\n pass", + "async def x(y):\n pass", + "for x in y:\n pass", + "async for x in y:\n pass", + "with x():\n pass", + "async with x():\n pass", + "def f():\n pass", + "def f(a):\n pass", + "def f(b=2):\n pass", + "def f(a, b):\n pass", + "def f(a, b=2):\n pass", + "def f(a=5, b=2):\n pass", + "def f(*, a=1, b=2):\n pass", + "def f(*, a=1, b):\n pass", + "def f(*, a, b=2):\n pass", + "def f(a, b=None, *, c, **kwds):\n pass", + "def f(a=2, *args, c=5, d, **kwds):\n pass", + "def f(*args, **kwargs):\n pass", + "class cls:\n\n def f(self):\n pass", + "class cls:\n\n def f(self, a):\n pass", + "class cls:\n\n def f(self, b=2):\n pass", + "class cls:\n\n def f(self, a, b):\n pass", + "class cls:\n\n def f(self, a, b=2):\n pass", + "class cls:\n\n def f(self, a=5, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b):\n pass", + "class cls:\n\n def f(self, *, a, b=2):\n pass", + "class cls:\n\n def f(self, a, b=None, *, c, **kwds):\n pass", + "class cls:\n\n def f(self, a=2, *args, c=5, d, **kwds):\n pass", + "class cls:\n\n def f(self, *args, **kwargs):\n pass", + ): + self.check_src_roundtrip(statement, mode='single') + + def test_unparse_interactive_integrity_3(self): + for statement in ( + "def x():", + "def x(y):", + "async def x():", + "async def x(y):", + "for x in y:", + "async for x in y:", + "with x():", + "async with x():", + "def f():", + "def f(a):", + "def f(b=2):", + "def f(a, b):", + "def f(a, b=2):", + "def f(a=5, b=2):", + "def f(*, a=1, b=2):", + "def f(*, a=1, b):", + "def f(*, a, b=2):", + "def f(a, b=None, *, c, **kwds):", + "def f(a=2, *args, c=5, d, **kwds):", + "def f(*args, **kwargs):", + ): + src = statement + '\n i=1;j=2' + out = statement + '\n i = 1\n j = 2' + + self.check_src_roundtrip(src, out, mode='single') + + +class CosmeticTestCase(ASTTestCase): + """Test if there are cosmetic issues caused by unnecessary additions""" + + def test_simple_expressions_parens(self): + self.check_src_roundtrip("(a := b)") + self.check_src_roundtrip("await x") + self.check_src_roundtrip("x if x else y") + self.check_src_roundtrip("lambda x: x") + self.check_src_roundtrip("1 + 1") + self.check_src_roundtrip("1 + 2 / 3") + self.check_src_roundtrip("(1 + 2) / 3") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2") + self.check_src_roundtrip("~x") + self.check_src_roundtrip("x and y") + self.check_src_roundtrip("x and y and z") + self.check_src_roundtrip("x and (y and x)") + self.check_src_roundtrip("(x and y) and z") + self.check_src_roundtrip("(x ** y) ** z ** q") + self.check_src_roundtrip("x >> y") + self.check_src_roundtrip("x << y") + self.check_src_roundtrip("x >> y and x >> z") + self.check_src_roundtrip("x + y - z * q ^ t ** k") + self.check_src_roundtrip("P * V if P and V else n * R * T") + self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T") + self.check_src_roundtrip("flag & (other | foo)") + self.check_src_roundtrip("not x == y") + self.check_src_roundtrip("x == (not y)") + self.check_src_roundtrip("yield x") + self.check_src_roundtrip("yield from x") + self.check_src_roundtrip("call((yield x))") + self.check_src_roundtrip("return x + (yield x)") + + def test_class_bases_and_keywords(self): + self.check_src_roundtrip("class X:\n pass") + self.check_src_roundtrip("class X(A):\n pass") + self.check_src_roundtrip("class X(A, B, C, D):\n pass") + self.check_src_roundtrip("class X(x=y):\n pass") + self.check_src_roundtrip("class X(metaclass=z):\n pass") + self.check_src_roundtrip("class X(x=y, z=d):\n pass") + self.check_src_roundtrip("class X(A, x=y):\n pass") + self.check_src_roundtrip("class X(A, **kw):\n pass") + self.check_src_roundtrip("class X(*args):\n pass") + self.check_src_roundtrip("class X(*args, **kwargs):\n pass") + + def test_fstrings(self): + self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_src_roundtrip('''f\'-{f\'\'\'*{f"""+{f".{f'{x}'}."}+"""}*\'\'\'}-\'''') + self.check_src_roundtrip('''f\'-{f\'*{f\'\'\'+{f""".{f"{f'{x}'}"}."""}+\'\'\'}*\'}-\'''') + self.check_src_roundtrip('''f"\\u2028{'x'}"''') + self.check_src_roundtrip(r"f'{x}\n'") + self.check_src_roundtrip('''f"{'\\n'}\\n"''') + self.check_src_roundtrip('''f"{f'{x}\\n'}\\n"''') + + def test_docstrings(self): + docstrings = ( + '"""simple doc string"""', + '''"""A more complex one + with some newlines"""''', + '''"""Foo bar baz + + empty newline"""''', + '"""With some \t"""', + '"""Foo "bar" baz """', + '"""\\r"""', + '""""""', + '"""\'\'\'"""', + '"""\'\'\'\'\'\'"""', + '"""🐍⛎𩸽üéş^\\\\X\\\\BB⟿"""', + '"""end in single \'quote\'"""', + "'''end in double \"quote\"'''", + '"""almost end in double "quote"."""', + ) + + for prefix in docstring_prefixes: + for docstring in docstrings: + self.check_src_roundtrip(f"{prefix}{docstring}") + + def test_docstrings_negative_cases(self): + # Test some cases that involve strings in the children of the + # first node but aren't docstrings to make sure we don't have + # False positives. + docstrings_negative = ( + 'a = """false"""', + '"""false""" + """unless its optimized"""', + '1 + 1\n"""false"""', + 'f"""no, top level but f-fstring"""' + ) + for prefix in docstring_prefixes: + for negative in docstrings_negative: + # this cases should be result with single quote + # rather then triple quoted docstring + src = f"{prefix}{negative}" + self.check_ast_roundtrip(src) + self.check_src_dont_roundtrip(src) + + def test_unary_op_factor(self): + for prefix in ("+", "-", "~"): + self.check_src_roundtrip(f"{prefix}1") + for prefix in ("not",): + self.check_src_roundtrip(f"{prefix} 1") + + def test_slices(self): + self.check_src_roundtrip("a[()]") + self.check_src_roundtrip("a[1]") + self.check_src_roundtrip("a[1, 2]") + # Note that `a[*a]`, `a[*a,]`, and `a[(*a,)]` all evaluate to the same + # thing at runtime and have the same AST, but only `a[*a,]` passes + # this test, because that's what `ast.unparse` produces. + self.check_src_roundtrip("a[*a,]") + self.check_src_roundtrip("a[1, *a]") + self.check_src_roundtrip("a[*a, 2]") + self.check_src_roundtrip("a[1, *a, 2]") + self.check_src_roundtrip("a[*a, *a]") + self.check_src_roundtrip("a[1, *a, *a]") + self.check_src_roundtrip("a[*a, 1, *a]") + self.check_src_roundtrip("a[*a, *a, 1]") + self.check_src_roundtrip("a[1, *a, *a, 2]") + self.check_src_roundtrip("a[1:2, *a]") + self.check_src_roundtrip("a[*a, 1:2]") + + def test_lambda_parameters(self): + self.check_src_roundtrip("lambda: something") + self.check_src_roundtrip("four = lambda: 2 + 2") + self.check_src_roundtrip("lambda x: x * 2") + self.check_src_roundtrip("square = lambda n: n ** 2") + self.check_src_roundtrip("lambda x, y: x + y") + self.check_src_roundtrip("add = lambda x, y: x + y") + self.check_src_roundtrip("lambda x, y, /, z, q, *, u: None") + self.check_src_roundtrip("lambda x, *y, **z: None") + + def test_star_expr_assign_target(self): + for source_type, source in [ + ("single assignment", "{target} = foo"), + ("multiple assignment", "{target} = {target} = bar"), + ("for loop", "for {target} in foo:\n pass"), + ("async for loop", "async for {target} in foo:\n pass") + ]: + for target in [ + "a", + "a,", + "a, b", + "a, *b, c", + "a, (b, c), d", + "a, (b, c, d), *e", + "a, (b, *c, d), e", + "a, (b, *c, (d, e), f), g", + "[a]", + "[a, b]", + "[a, *b, c]", + "[a, [b, c], d]", + "[a, [b, c, d], *e]", + "[a, [b, *c, d], e]", + "[a, [b, *c, [d, e], f], g]", + "a, [b, c], d", + "[a, b, (c, d), (e, f)]", + "a, b, [*c], d, e" + ]: + with self.subTest(source_type=source_type, target=target): + self.check_src_roundtrip(source.format(target=target)) + + def test_star_expr_assign_target_multiple(self): + self.check_src_roundtrip("() = []") + self.check_src_roundtrip("[] = ()") + self.check_src_roundtrip("() = [a] = c, = [d] = e, f = () = g = h") + self.check_src_roundtrip("a = b = c = d") + self.check_src_roundtrip("a, b = c, d = e, f = g") + self.check_src_roundtrip("[a, b] = [c, d] = [e, f] = g") + self.check_src_roundtrip("a, b = [c, d] = e, f = g") + + def test_multiquote_joined_string(self): + self.check_ast_roundtrip("f\"'''{1}\\\"\\\"\\\"\" ") + self.check_ast_roundtrip("""f"'''{1}""\\"" """) + self.check_ast_roundtrip("""f'""\"{1}''' """) + self.check_ast_roundtrip("""f'""\"{1}""\\"' """) + + self.check_ast_roundtrip("""f"'''{"\\n"}""\\"" """) + self.check_ast_roundtrip("""f'""\"{"\\n"}''' """) + self.check_ast_roundtrip("""f'""\"{"\\n"}""\\"' """) + + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + + def test_backslash_in_format_spec(self): + import re + msg = re.escape('"\\ " is an invalid escape sequence. ' + 'Such sequences will not work in the future. ' + 'Did you mean "\\\\ "? A raw string is also an option.') + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + + +class ManualASTCreationTestCase(unittest.TestCase): + """Test that AST nodes created without a type_params field unparse correctly.""" + + def test_class(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X:\n pass") + + def test_class_with_type_params(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[], + type_params=[ast.TypeVar("T")]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X[T]:\n pass") + + def test_function(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f():\n pass") + + def test_function_with_type_params(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T]():\n pass") + + def test_function_with_type_params_and_bound(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + def test_async_function(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f():\n pass") + + def test_async_function_with_type_params(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + +class DirectoryTestCase(ASTTestCase): + """Test roundtrip behaviour on all files in Lib and Lib/test.""" + + lib_dir = pathlib.Path(__file__).parent / ".." + test_directories = (lib_dir, lib_dir / "test") + run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", + "test_ast.py", "test_asdl_parser.py", "test_fstring.py", + "test_patma.py", "test_type_alias.py", "test_type_params.py", + "test_tokenize.py", "test_tstring.py"} + + _files_to_test = None + + @classmethod + def files_to_test(cls): + + if cls._files_to_test is not None: + return cls._files_to_test + + items = [ + item.resolve() + for directory in cls.test_directories + for item in directory.glob("*.py") + if not item.name.startswith("bad") + ] + + # Test limited subset of files unless the 'cpu' resource is specified. + if not test.support.is_resource_enabled("cpu"): + + tests_to_run_always = {item for item in items if + item.name in cls.run_always_files} + + items = set(random.sample(items, 10)) + + # Make sure that at least tests that heavily use grammar features are + # always considered in order to reduce the chance of missing something. + items = list(items | tests_to_run_always) + + # bpo-31174: Store the names sample to always test the same files. + # It prevents false alarms when hunting reference leaks. + cls._files_to_test = items + + return items + + def test_files(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + + for item in self.files_to_test(): + if test.support.verbose: + print(f"Testing {item.absolute()}") + + with self.subTest(filename=item): + source = read_pyfile(item) + self.check_ast_roundtrip(source) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 82f1d9dc2e7..2dd739b77b8 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -7,23 +7,26 @@ import email.message import io import unittest -from unittest.mock import patch from test import support from test.support import os_helper -from test.support import warnings_helper +from test.support import socket_helper +from test.support import control_characters_c0 import os +import socket try: import ssl except ImportError: ssl = None import sys import tempfile -from nturl2path import url2pathname, pathname2url -from base64 import b64encode import collections +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + def hexescape(char): """Escape char as RFC 2396 specifies""" hex_repr = hex(ord(char))[2:].upper() @@ -31,32 +34,6 @@ def hexescape(char): hex_repr = "0%s" % hex_repr return "%" + hex_repr -# Shortcut for testing FancyURLopener -_urlopener = None - - -def urlopen(url, data=None, proxies=None): - """urlopen(url [, data]) -> open file-like object""" - global _urlopener - if proxies is not None: - opener = urllib.request.FancyURLopener(proxies=proxies) - elif not _urlopener: - opener = FancyURLopener() - _urlopener = opener - else: - opener = _urlopener - if data is None: - return opener.open(url) - else: - return opener.open(url, data) - - -def FancyURLopener(): - with warnings_helper.check_warnings( - ('FancyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - return urllib.request.FancyURLopener() - def fakehttp(fakedata, mock_close=False): class FakeSocket(io.BytesIO): @@ -115,26 +92,6 @@ def unfakehttp(self): http.client.HTTPConnection = self._connection_class -class FakeFTPMixin(object): - def fakeftp(self): - class FakeFtpWrapper(object): - def __init__(self, user, passwd, host, port, dirs, timeout=None, - persistent=True): - pass - - def retrfile(self, file, type): - return io.BytesIO(), 0 - - def close(self): - pass - - self._ftpwrapper_class = urllib.request.ftpwrapper - urllib.request.ftpwrapper = FakeFtpWrapper - - def unfakeftp(self): - urllib.request.ftpwrapper = self._ftpwrapper_class - - class urlopen_FileTests(unittest.TestCase): """Test urlopen() opening a temporary file. @@ -153,8 +110,8 @@ def setUp(self): finally: f.close() self.pathname = os_helper.TESTFN - self.quoted_pathname = urllib.parse.quote(self.pathname) - self.returned_obj = urlopen("file:%s" % self.quoted_pathname) + self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname)) + self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname) def tearDown(self): """Shut down the open object""" @@ -165,9 +122,7 @@ def test_interface(self): # Make sure object returned by urlopen() has the specified methods for attr in ("read", "readline", "readlines", "fileno", "close", "info", "geturl", "getcode", "__iter__"): - self.assertTrue(hasattr(self.returned_obj, attr), - "object returned by urlopen() lacks %s attribute" % - attr) + self.assertHasAttr(self.returned_obj, attr) def test_read(self): self.assertEqual(self.text, self.returned_obj.read()) @@ -201,7 +156,7 @@ def test_headers(self): self.assertIsInstance(self.returned_obj.headers, email.message.Message) def test_url(self): - self.assertEqual(self.returned_obj.url, self.quoted_pathname) + self.assertEqual(self.returned_obj.url, "file:" + self.quoted_pathname) def test_status(self): self.assertIsNone(self.returned_obj.status) @@ -210,7 +165,7 @@ def test_info(self): self.assertIsInstance(self.returned_obj.info(), email.message.Message) def test_geturl(self): - self.assertEqual(self.returned_obj.geturl(), self.quoted_pathname) + self.assertEqual(self.returned_obj.geturl(), "file:" + self.quoted_pathname) def test_getcode(self): self.assertIsNone(self.returned_obj.getcode()) @@ -227,22 +182,27 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + def test_remote_authority(self): + # Test for GH-90812. + url = 'file://pythontest.net/foo/bar' + with self.assertRaises(urllib.error.URLError) as e: + urllib.request.urlopen(url) + if os.name == 'nt': + self.assertEqual(e.exception.filename, r'\\pythontest.net\foo\bar') + else: + self.assertEqual(e.exception.reason, 'file:// scheme is supported only on localhost') + class ProxyTests(unittest.TestCase): def setUp(self): # Records changes to env vars - self.env = os_helper.EnvironmentVarGuard() + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) # Delete all proxy related env vars for k in list(os.environ): if 'proxy' in k.lower(): self.env.unset(k) - def tearDown(self): - # Restore all proxy related env vars - self.env.__exit__() - del self.env - def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.request.getproxies_environment() @@ -340,13 +300,13 @@ def test_getproxies_environment_prefer_lowercase(self): self.assertEqual('http://somewhere:3128', proxies['http']) -class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): """Test urlopen() opening a fake http connection.""" def check_read(self, ver): self.fakehttp(b"HTTP/" + ver + b" 200 OK\r\n\r\nHello!") try: - fp = urlopen("http://python.org/") + fp = urllib.request.urlopen("http://python.org/") self.assertEqual(fp.readline(), b"Hello!") self.assertEqual(fp.readline(), b"") self.assertEqual(fp.geturl(), 'http://python.org/') @@ -367,8 +327,8 @@ def test_url_fragment(self): def test_willclose(self): self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") try: - resp = urlopen("http://www.python.org") - self.assertTrue(resp.fp.will_close) + resp = urllib.request.urlopen("http://www.python.org") + self.assertTrue(resp.will_close) finally: self.unfakehttp() @@ -393,9 +353,6 @@ def test_url_path_with_control_char_rejected(self): with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(char, resp.geturl()) finally: self.unfakehttp() @@ -417,11 +374,6 @@ def test_url_path_with_newline_header_injection_rejected(self): urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(' ', resp.geturl()) - self.assertNotIn('\r', resp.geturl()) - self.assertNotIn('\n', resp.geturl()) finally: self.unfakehttp() @@ -436,9 +388,9 @@ def test_url_host_with_control_char_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -451,9 +403,9 @@ def test_url_host_with_newline_header_injection_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, r"contain control.*\\r"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -477,7 +429,9 @@ def test_read_bogus(self): Content-Type: text/html; charset=iso-8859-1 ''', mock_close=True) try: - self.assertRaises(OSError, urlopen, "http://python.org/") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() @@ -492,22 +446,24 @@ def test_invalid_redirect(self): ''', mock_close=True) try: msg = "Redirection to url 'file:" - with self.assertRaisesRegex(urllib.error.HTTPError, msg): - urlopen("http://python.org/") + with self.assertRaisesRegex(urllib.error.HTTPError, msg) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() def test_redirect_limit_independent(self): # Ticket #12923: make sure independent requests each use their # own retry limit. - for i in range(FancyURLopener().maxtries): + for i in range(urllib.request.HTTPRedirectHandler.max_redirections): self.fakehttp(b'''HTTP/1.1 302 Found Location: file://guidocomputer.athome.com:/python/license Connection: close ''', mock_close=True) try: - self.assertRaises(urllib.error.HTTPError, urlopen, - "http://something") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://something") + cm.exception.close() finally: self.unfakehttp() @@ -516,96 +472,47 @@ def test_empty_socket(self): # data. (#1680230) self.fakehttp(b'') try: - self.assertRaises(OSError, urlopen, "http://something") + self.assertRaises(OSError, urllib.request.urlopen, "http://something") finally: self.unfakehttp() def test_missing_localfile(self): # Test for #10836 with self.assertRaises(urllib.error.URLError) as e: - urlopen('file://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('file://localhost/a/file/which/doesnot/exists.py') self.assertTrue(e.exception.filename) self.assertTrue(e.exception.reason) def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() - tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + tmp_file_canon_url = urllib.request.pathname2url(tmp_file, add_scheme=True) + parsed = urllib.parse.urlsplit(tmp_file_canon_url) + tmp_fileurl = parsed._replace(netloc='localhost').geturl() try: self.assertTrue(os.path.exists(tmp_file)) - with urlopen(tmp_fileurl) as fobj: + with urllib.request.urlopen(tmp_fileurl) as fobj: self.assertTrue(fobj) + self.assertEqual(fobj.url, tmp_file_canon_url) finally: os.close(fd) os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) with self.assertRaises(urllib.error.URLError): - urlopen(tmp_fileurl) + urllib.request.urlopen(tmp_fileurl) def test_ftp_nohost(self): test_ftp_url = 'ftp:///path' with self.assertRaises(urllib.error.URLError) as e: - urlopen(test_ftp_url) + urllib.request.urlopen(test_ftp_url) self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) def test_ftp_nonexisting(self): with self.assertRaises(urllib.error.URLError) as e: - urlopen('ftp://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('ftp://localhost/a/file/which/doesnot/exists.py') self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) - @patch.object(urllib.request, 'MAXFTPCACHE', 0) - def test_ftp_cache_pruning(self): - self.fakeftp() - try: - urllib.request.ftpcache['test'] = urllib.request.ftpwrapper('user', 'pass', 'localhost', 21, []) - urlopen('ftp://localhost') - finally: - self.unfakeftp() - - def test_userpass_inurl(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - fp = urlopen("http://user:pass@python.org/") - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_userpass_inurl_w_spaces(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - userpass = "a b:c d" - url = "http://{}@python.org/".format(userpass) - fakehttp_wrapper = http.client.HTTPConnection - authorization = ("Authorization: Basic %s\r\n" % - b64encode(userpass.encode("ASCII")).decode("ASCII")) - fp = urlopen(url) - # The authorization header must be in place - self.assertIn(authorization, fakehttp_wrapper.buf.decode("UTF-8")) - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - # the spaces are quoted in URL so no match - self.assertNotEqual(fp.geturl(), url) - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_URLopener_deprecation(self): - with warnings_helper.check_warnings(('',DeprecationWarning)): - urllib.request.URLopener() - - @unittest.skipUnless(ssl, "ssl module required") - def test_cafile_and_context(self): - context = ssl.create_default_context() - with warnings_helper.check_warnings(('', DeprecationWarning)): - with self.assertRaises(ValueError): - urllib.request.urlopen( - "https://localhost", cafile="/nonexistent/path", context=context - ) - class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -636,18 +543,17 @@ def setUp(self): "QOjdAAAAAXNSR0IArs4c6QAAAA9JREFUCNdj%0AYGBg%2BP//PwAGAQL%2BCm8 " "vHgAAAABJRU5ErkJggg%3D%3D%0A%20") - self.text_url_resp = urllib.request.urlopen(self.text_url) - self.text_url_base64_resp = urllib.request.urlopen( - self.text_url_base64) - self.image_url_resp = urllib.request.urlopen(self.image_url) + self.text_url_resp = self.enterContext( + urllib.request.urlopen(self.text_url)) + self.text_url_base64_resp = self.enterContext( + urllib.request.urlopen(self.text_url_base64)) + self.image_url_resp = self.enterContext(urllib.request.urlopen(self.image_url)) def test_interface(self): # Make sure object returned by urlopen() has the specified methods for attr in ("read", "readline", "readlines", "close", "info", "geturl", "getcode", "__iter__"): - self.assertTrue(hasattr(self.text_url_resp, attr), - "object returned by urlopen() lacks %s attribute" % - attr) + self.assertHasAttr(self.text_url_resp, attr) def test_info(self): self.assertIsInstance(self.text_url_resp.info(), email.message.Message) @@ -655,8 +561,10 @@ def test_info(self): [('text/plain', ''), ('charset', 'ISO-8859-1')]) self.assertEqual(self.image_url_resp.info()['content-length'], str(len(self.image))) - self.assertEqual(urllib.request.urlopen("data:,").info().get_params(), + r = urllib.request.urlopen("data:,") + self.assertEqual(r.info().get_params(), [('text/plain', ''), ('charset', 'US-ASCII')]) + r.close() def test_geturl(self): self.assertEqual(self.text_url_resp.geturl(), self.text_url) @@ -683,6 +591,13 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + def test_invalid_mediatype(self): + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html;{c0},data') + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html{c0};base64,ZGF0YQ==') class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" @@ -719,11 +634,7 @@ def tearDown(self): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) - try: - filePath.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("filePath is not encodable to utf8") - return "file://%s" % urllib.request.pathname2url(filePath) + return urllib.request.pathname2url(filePath, add_scheme=True) def createNewTempFile(self, data=b""): """Creates a new temporary file containing the specified data, @@ -1104,6 +1015,8 @@ def test_unquoting(self): self.assertEqual(result.count('%'), 1, "using unquote(): not all characters escaped: " "%s" % result) + + def test_unquote_rejects_none_and_tuple(self): self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) @@ -1526,40 +1439,229 @@ def test_quoting(self): "url2pathname() failed; %s != %s" % (expect, result)) + def test_pathname2url(self): + # Test cases common to Windows and POSIX. + fn = urllib.request.pathname2url + sep = os.path.sep + self.assertEqual(fn(''), '') + self.assertEqual(fn(sep), '///') + self.assertEqual(fn('a'), 'a') + self.assertEqual(fn(f'a{sep}b.c'), 'a/b.c') + self.assertEqual(fn(f'{sep}a{sep}b.c'), '///a/b.c') + self.assertEqual(fn(f'{sep}a{sep}b%#c'), '///a/b%25%23c') + + def test_pathname2url_add_scheme(self): + sep = os.path.sep + subtests = [ + ('', 'file:'), + (sep, 'file:///'), + ('a', 'file:a'), + (f'a{sep}b.c', 'file:a/b.c'), + (f'{sep}a{sep}b.c', 'file:///a/b.c'), + (f'{sep}a{sep}b%#c', 'file:///a/b%25%23c'), + ] + for path, expected_url in subtests: + with self.subTest(path=path): + self.assertEqual( + urllib.request.pathname2url(path, add_scheme=True), expected_url) + @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the nturl2path functions.') - def test_prefixes(self): + 'test specific to Windows pathnames.') + def test_pathname2url_win(self): # Test special prefixes are correctly handled in pathname2url() - given = '\\\\?\\C:\\dir' - expect = '///C:/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - given = '\\\\?\\unc\\server\\share\\dir' - expect = '/server/share/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - + fn = urllib.request.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') + self.assertEqual(fn('c:\\a\\b.c'), '///c:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # NTFS alternate data streams + self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') + self.assertEqual(fn('foo:bar'), 'foo%3Abar') + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(urllib.request.url2pathname(url)), url) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_pathname2url_posix(self): + fn = urllib.request.pathname2url + self.assertEqual(fn('//a/b.c'), '////a/b.c') + self.assertEqual(fn('///a/b.c'), '/////a/b.c') + self.assertEqual(fn('////a/b.c'), '//////a/b.c') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_pathname2url_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = urllib.parse.quote(os_helper.FS_NONASCII, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.pathname2url(os_helper.FS_NONASCII), url) + + def test_url2pathname(self): + # Test cases common to Windows and POSIX. + fn = urllib.request.url2pathname + sep = os.path.sep + self.assertEqual(fn(''), '') + self.assertEqual(fn('/'), f'{sep}') + self.assertEqual(fn('///'), f'{sep}') + self.assertEqual(fn('////'), f'{sep}{sep}') + self.assertEqual(fn('foo'), 'foo') + self.assertEqual(fn('foo/bar'), f'foo{sep}bar') + self.assertEqual(fn('/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('//localhost/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('///foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') + self.assertEqual(fn('data:blah'), 'data:blah') + self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') + + def test_url2pathname_require_scheme(self): + sep = os.path.sep + subtests = [ + ('file:', ''), + ('FILE:', ''), + ('FiLe:', ''), + ('file:/', f'{sep}'), + ('file:///', f'{sep}'), + ('file:////', f'{sep}{sep}'), + ('file:foo', 'foo'), + ('file:foo/bar', f'foo{sep}bar'), + ('file:/foo/bar', f'{sep}foo{sep}bar'), + ('file://localhost/foo/bar', f'{sep}foo{sep}bar'), + ('file:///foo/bar', f'{sep}foo{sep}bar'), + ('file:////foo/bar', f'{sep}{sep}foo{sep}bar'), + ('file:data:blah', 'data:blah'), + ('file:data://blah', f'data:{sep}{sep}blah'), + ] + for url, expected_path in subtests: + with self.subTest(url=url): + self.assertEqual( + urllib.request.url2pathname(url, require_scheme=True), + expected_path) + + def test_url2pathname_require_scheme_errors(self): + subtests = [ + '', + ':', + 'foo', + 'http:foo', + 'localfile:foo', + 'data:foo', + 'data:file:foo', + 'data:file://foo', + ] + for url in subtests: + with self.subTest(url=url): + self.assertRaises( + urllib.error.URLError, + urllib.request.url2pathname, + url, require_scheme=True) + + @unittest.skipIf(support.is_emscripten, "Fixed by https://github.com/emscripten-core/emscripten/pull/24593") + def test_url2pathname_resolve_host(self): + fn = urllib.request.url2pathname + sep = os.path.sep + self.assertEqual(fn('//127.0.0.1/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the urllib.url2path function.') - def test_ntpath(self): - given = ('/C:/', '///C:/', '/C|//') - expect = 'C:\\' - for url in given: - result = urllib.request.url2pathname(url) - self.assertEqual(expect, result, - 'urllib.request..url2pathname() failed; %s != %s' % - (expect, result)) - given = '///C|/path' - expect = 'C:\\path' - result = urllib.request.url2pathname(given) - self.assertEqual(expect, result, - 'urllib.request.url2pathname() failed; %s != %s' % - (expect, result)) + 'test specific to Windows pathnames.') + def test_url2pathname_win(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn('//C:'), 'C:') + self.assertEqual(fn('//C:/'), 'C:\\') + self.assertEqual(fn('//C:\\'), 'C:\\') + self.assertEqual(fn('//C:80/'), 'C:80\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('c:/path/to/file'), 'c:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Colons in URI + self.assertEqual(fn('///\u00e8|/'), '\u00e8:\\') + self.assertEqual(fn('//host/share/spam.txt:eggs'), '\\\\host\\share\\spam.txt:eggs') + self.assertEqual(fn('///c:/spam.txt:eggs'), 'c:\\spam.txt:eggs') + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('//127.0.0.1/path/to/file'), '\\\\127.0.0.1\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/path/to/file'), '\\path\\to\\file') + self.assertEqual(fn('//localhost//server/path/to/file'), '\\\\server\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') + # Round-tripping + paths = ['C:', + r'\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(urllib.request.pathname2url(path)), path) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_url2pathname_posix(self): + fn = urllib.request.url2pathname + self.assertRaises(urllib.error.URLError, fn, '//foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//localhost:/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//:80/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//:/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//c:80/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//127.0.0.1/foo/bar') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_url2pathname_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = os_helper.FS_NONASCII + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) + url = urllib.parse.quote(url, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" @@ -1569,56 +1671,6 @@ def test_thishost(self): self.assertIsInstance(urllib.request.thishost(), tuple) -class URLopener_Tests(FakeHTTPMixin, unittest.TestCase): - """Testcase to test the open method of URLopener class.""" - - def test_quoted_open(self): - class DummyURLopener(urllib.request.URLopener): - def open_spam(self, url): - return url - with warnings_helper.check_warnings( - ('DummyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - self.assertEqual(DummyURLopener().open( - 'spam://example/ /'),'//example/%20/') - - # test the safe characters are not quoted by urlopen - self.assertEqual(DummyURLopener().open( - "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), - "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_file(self): - with os_helper.temp_dir() as tmpdir: - fd, tmpfile = tempfile.mkstemp(dir=tmpdir) - os.close(fd) - fileurl = "file:" + urllib.request.pathname2url(tmpfile) - filename, _ = urllib.request.URLopener().retrieve(fileurl) - # Some buildbots have TEMP folder that uses a lowercase drive letter. - self.assertEqual(os.path.normcase(filename), os.path.normcase(tmpfile)) - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_remote(self): - url = "http://www.python.org/file.txt" - self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") - self.addCleanup(self.unfakehttp) - filename, _ = urllib.request.URLopener().retrieve(url) - self.assertEqual(os.path.splitext(filename)[1], ".txt") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_local_file_open(self): - # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme - class DummyURLopener(urllib.request.URLopener): - def open_local_file(self, url): - return url - for url in ('local_file://example', 'local-file://example'): - self.assertRaises(OSError, urllib.request.urlopen, url) - self.assertRaises(OSError, urllib.request.URLopener().open, url) - self.assertRaises(OSError, urllib.request.URLopener().retrieve, url) - self.assertRaises(OSError, DummyURLopener().open, url) - self.assertRaises(OSError, DummyURLopener().retrieve, url) - - class RequestTests(unittest.TestCase): """Unit tests for urllib.request.Request.""" @@ -1643,60 +1695,5 @@ def test_with_method_arg(self): self.assertEqual(request.get_method(), 'HEAD') -class URL2PathNameTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(url2pathname("///C|"), 'C:') - self.assertEqual(url2pathname("///C:"), 'C:') - self.assertEqual(url2pathname("///C|/"), 'C:\\') - - def test_converting_when_no_drive_letter(self): - # cannot end a raw string in \ - self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\') - self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\') - - def test_simple_compare(self): - self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"), - r'C:\foo\bar\spam.foo') - - def test_non_ascii_drive_letter(self): - self.assertRaises(IOError, url2pathname, "///\u00e8|/") - - def test_roundtrip_url2pathname(self): - list_of_paths = ['C:', - r'\\\C\test\\', - r'C:\foo\bar\spam.foo' - ] - for path in list_of_paths: - self.assertEqual(url2pathname(pathname2url(path)), path) - -class PathName2URLTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(pathname2url("C:"), '///C:') - self.assertEqual(pathname2url("C:\\"), '///C:') - - def test_converting_when_no_drive_letter(self): - self.assertEqual(pathname2url(r"\\\folder\test" "\\"), - '/////folder/test/') - self.assertEqual(pathname2url(r"\\folder\test" "\\"), - '////folder/test/') - self.assertEqual(pathname2url(r"\folder\test" "\\"), - '/folder/test/') - - def test_simple_compare(self): - self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'), - "///C:/foo/bar/spam.foo" ) - - def test_long_drive_letter(self): - self.assertRaises(IOError, pathname2url, "XX:\\") - - def test_roundtrip_pathname2url(self): - list_of_paths = ['///C:', - '/////folder/test/', - '///C:/foo/bar/spam.foo'] - for path in list_of_paths: - self.assertEqual(pathname2url(url2pathname(path)), path) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 5a02b5db8e5..7d7f2fa00d3 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,12 +1,14 @@ import unittest from test import support from test.support import os_helper -from test.support import socket_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib +from unittest import mock import os import io +import ftplib import socket import array import sys @@ -14,16 +16,20 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) -from urllib.parse import urlparse +from urllib.parse import urlsplit import urllib.error import http.client + +support.requires_working_socket(module=True) + # XXX # Request # CacheFTPHandler (hard to write) @@ -38,10 +44,6 @@ def test___all__(self): context = {} exec('from urllib.%s import *' % module, context) del context['__builtins__'] - if module == 'request' and os.name == 'nt': - u, p = context.pop('url2pathname'), context.pop('pathname2url') - self.assertEqual(u.__module__, 'nturl2path') - self.assertEqual(p.__module__, 'nturl2path') for k, v in context.items(): self.assertEqual(v.__module__, 'urllib.%s' % module, "%r is exposed in 'urllib.%s' but defined in %r" % @@ -483,7 +485,18 @@ def build_test_opener(*handler_instances): return opener -class MockHTTPHandler(urllib.request.BaseHandler): +class MockHTTPHandler(urllib.request.HTTPHandler): + # Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection + + def __init__(self, debuglevel=None): + super(MockHTTPHandler, self).__init__(debuglevel=debuglevel) + self.httpconn = MockHTTPClass() + + def http_open(self, req): + return self.do_open(self.httpconn, req) + + +class MockHTTPHandlerRedirect(urllib.request.BaseHandler): # useful for testing redirections and auth # sends supplied headers and code as first response # sends 200 OK as second response @@ -511,16 +524,17 @@ def http_open(self, req): return MockResponse(200, "OK", msg, "", req.get_full_url()) -class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): - # Useful for testing the Proxy-Authorization request by verifying the - # properties of httpcon +if hasattr(http.client, 'HTTPSConnection'): + class MockHTTPSHandler(urllib.request.HTTPSHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon - def __init__(self, debuglevel=0): - urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) - self.httpconn = MockHTTPClass() + def __init__(self, debuglevel=None, context=None, check_hostname=None): + super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname) + self.httpconn = MockHTTPClass() - def https_open(self, req): - return self.do_open(self.httpconn, req) + def https_open(self, req): + return self.do_open(self.httpconn, req) class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): @@ -641,8 +655,6 @@ def test_raise(self): self.assertRaises(urllib.error.URLError, o.open, req) self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http_error(self): # XXX http_error_default # http errors are a special case @@ -666,8 +678,6 @@ def test_http_error(self): self.assertEqual((handler, method_name), got[:2]) self.assertEqual(args, got[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_processors(self): # *_request / *_response methods get called appropriately o = OpenerDirector() @@ -704,18 +714,6 @@ def test_processors(self): self.assertIsInstance(args[1], MockResponse) -def sanepathname2url(path): - try: - path.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("path is not encodable to utf8") - urlpath = urllib.request.pathname2url(path) - if os.name == "nt" and urlpath.startswith("///"): - urlpath = urlpath[2:] - # XXX don't ask me about the mac... - return urlpath - - class HandlerTests(unittest.TestCase): def test_ftp(self): @@ -742,7 +740,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -765,7 +762,7 @@ def connect_ftp(self, user, passwd, host, port, dirs, ["foo", "bar"], "", None), ("ftp://localhost/baz.gif;type=a", "localhost", ftplib.FTP_PORT, "", "", "A", - [], "baz.gif", None), # XXX really this should guess image/gif + [], "baz.gif", "image/gif"), ]: req = Request(url) req.timeout = None @@ -781,6 +778,29 @@ def connect_ftp(self, user, passwd, host, port, dirs, headers = r.info() self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + r.close() + + @support.requires_resource("network") + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") def test_file(self): import email.utils @@ -788,19 +808,22 @@ def test_file(self): o = h.parent = MockOpener() TESTFN = os_helper.TESTFN - urlpath = sanepathname2url(os.path.abspath(TESTFN)) towrite = b"hello, world\n" + canonurl = urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True) + parsed = urlsplit(canonurl) + if parsed.netloc: + raise unittest.SkipTest("non-local working directory") urls = [ - "file://localhost%s" % urlpath, - "file://%s" % urlpath, - "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + canonurl, + parsed._replace(netloc='localhost').geturl(), + parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), ] try: localaddr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: localaddr = '' if localaddr: - urls.append("file://%s%s" % (localaddr, urlpath)) + urls.append(parsed._replace(netloc=localaddr).geturl()) for url in urls: f = open(TESTFN, "wb") @@ -825,10 +848,10 @@ def test_file(self): self.assertEqual(headers["Content-type"], "text/plain") self.assertEqual(headers["Content-length"], "13") self.assertEqual(headers["Last-modified"], modified) - self.assertEqual(respurl, url) + self.assertEqual(respurl, canonurl) for url in [ - "file://localhost:80%s" % urlpath, + parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), @@ -874,8 +897,6 @@ def test_file(self): self.assertEqual(req.type, "ftp") self.assertEqual(req.type == "ftp", ftp) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http(self): h = urllib.request.AbstractHTTPHandler() @@ -990,6 +1011,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the @@ -1053,12 +1075,37 @@ def test_http_body_array(self): newreq = h.do_request_(req) self.assertEqual(int(newreq.get_header('Content-length')),16) - def test_http_handler_debuglevel(self): + def test_http_handler_global_debuglevel(self): + with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6): + o = OpenerDirector() + h = MockHTTPHandler() + o.add_handler(h) + o.open("http://www.example.com") + self.assertEqual(h._debuglevel, 6) + + def test_http_handler_local_debuglevel(self): + o = OpenerDirector() + h = MockHTTPHandler(debuglevel=5) + o.add_handler(h) + o.open("http://www.example.com") + self.assertEqual(h._debuglevel, 5) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_https_handler_global_debuglevel(self): + with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7): + o = OpenerDirector() + h = MockHTTPSHandler() + o.add_handler(h) + o.open("https://www.example.com") + self.assertEqual(h._debuglevel, 7) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_https_handler_local_debuglevel(self): o = OpenerDirector() - h = MockHTTPSHandler(debuglevel=1) + h = MockHTTPSHandler(debuglevel=4) o.add_handler(h) o.open("https://www.example.com") - self.assertEqual(h._debuglevel, 1) + self.assertEqual(h._debuglevel, 4) def test_http_doubleslash(self): # Checks the presence of any unnecessary double slash in url does not @@ -1102,13 +1149,13 @@ def test_full_url_setter(self): r = Request('http://example.com') for url in urls: r.full_url = url - parsed = urlparse(url) + parsed = urlsplit(url) self.assertEqual(r.get_full_url(), url) # full_url setter uses splittag to split into components. # splittag sets the fragment as None while urlparse sets it to '' self.assertEqual(r.fragment or '', parsed.fragment) - self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) def test_full_url_deleter(self): r = Request('http://www.example.com') @@ -1136,8 +1183,6 @@ def test_fixpath_in_weirdurls(self): self.assertEqual(newreq.host, 'www.python.org') self.assertEqual(newreq.selector, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): h = urllib.request.HTTPErrorProcessor() o = h.parent = MockOpener() @@ -1148,23 +1193,21 @@ def test_errors(self): r = MockResponse(200, "OK", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called r = MockResponse(202, "Accepted", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called r = MockResponse(206, "Partial content", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called # anything else calls o.error (and MockOpener returns None, here) r = MockResponse(502, "Bad gateway", {}, "", url) self.assertIsNone(h.http_response(req, r)) self.assertEqual(o.proto, "http") # o.error called self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookies(self): cj = MockCookieJar() h = urllib.request.HTTPCookieProcessor(cj) @@ -1189,7 +1232,7 @@ def test_redirect(self): o = h.parent = MockOpener() # ordinary redirect behaviour - for code in 301, 302, 303, 307: + for code in 301, 302, 303, 307, 308: for data in None, "blah\nblah\n": method = getattr(h, "http_error_%s" % code) req = Request(from_url, data) @@ -1201,10 +1244,11 @@ def test_redirect(self): try: method(req, MockFile(), code, "Blah", MockHeaders({"location": to_url})) - except urllib.error.HTTPError: - # 307 in response to POST requires user OK - self.assertEqual(code, 307) + except urllib.error.HTTPError as err: + # 307 and 308 in response to POST require user OK + self.assertIn(code, (307, 308)) self.assertIsNotNone(data) + err.close() self.assertEqual(o.req.get_full_url(), to_url) try: self.assertEqual(o.req.get_method(), "GET") @@ -1240,9 +1284,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/") count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: # don't stop until max_repeats, because cookies may introduce state self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) + err.close() # detect endless non-repeating chain of redirects req = Request(from_url, origin_req_host="example.com") @@ -1252,9 +1297,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/%d" % count) count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + err.close() def test_invalid_redirect(self): from_url = "http://example.com/a.html" @@ -1268,9 +1314,11 @@ def test_invalid_redirect(self): for scheme in invalid_schemes: invalid_url = scheme + '://' + schemeless_url - self.assertRaises(urllib.error.HTTPError, h.http_error_302, + with self.assertRaises(urllib.error.HTTPError) as cm: + h.http_error_302( req, MockFile(), 302, "Security Loophole", MockHeaders({"location": invalid_url})) + cm.exception.close() for scheme in valid_schemes: valid_url = scheme + '://' + schemeless_url @@ -1291,8 +1339,6 @@ def test_relative_redirect(self): MockHeaders({"location": valid_url})) self.assertEqual(o.req.get_full_url(), valid_url) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar @@ -1300,7 +1346,7 @@ def test_cookie_redirect(self): cj = CookieJar() interact_netscape(cj, "http://www.example.com/", "spam=eggs") - hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") + hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n") hdeh = urllib.request.HTTPDefaultErrorHandler() hrh = urllib.request.HTTPRedirectHandler() cp = urllib.request.HTTPCookieProcessor(cj) @@ -1308,11 +1354,9 @@ def test_cookie_redirect(self): o.open("http://www.example.com/") self.assertFalse(hh.req.has_header("Cookie")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_redirect_fragment(self): redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' - hh = MockHTTPHandler(302, 'Location: ' + redirected_url) + hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url) hdeh = urllib.request.HTTPDefaultErrorHandler() hrh = urllib.request.HTTPRedirectHandler() o = build_test_opener(hh, hdeh, hrh) @@ -1372,10 +1416,17 @@ def http_open(self, req): response = opener.open('http://example.com/') expected = b'GET ' + result + b' ' request = handler.last_buf - self.assertTrue(request.startswith(expected), repr(request)) + self.assertStartsWith(request, expected) + + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1395,7 +1446,8 @@ def test_proxy(self): [tup[0:2] for tup in o.calls]) def test_proxy_no_proxy(self): - os.environ['no_proxy'] = 'python.org' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['no_proxy'] = 'python.org' o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) o.add_handler(ph) @@ -1407,10 +1459,10 @@ def test_proxy_no_proxy(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] def test_proxy_no_proxy_all(self): - os.environ['no_proxy'] = '*' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['no_proxy'] = '*' o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) o.add_handler(ph) @@ -1418,10 +1470,7 @@ def test_proxy_no_proxy_all(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_https(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) @@ -1438,6 +1487,7 @@ def test_proxy_https(self): self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls]) + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') def test_proxy_https_proxy_authorization(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) @@ -1461,6 +1511,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; <local>" + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect <local> to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1501,7 +1575,7 @@ def check_basic_auth(self, headers, realm): password_manager = MockPasswordManager() auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) body = '\r\n'.join(headers) + '\r\n\r\n' - http_handler = MockHTTPHandler(401, body) + http_handler = MockHTTPHandlerRedirect(401, body) opener.add_handler(auth_handler) opener.add_handler(http_handler) self._test_basic_auth(opener, auth_handler, "Authorization", @@ -1509,8 +1583,6 @@ def check_basic_auth(self, headers, realm): "http://acme.example.com/protected", "http://acme.example.com/protected") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_auth(self): realm = "realm2@example.com" realm2 = "realm2@example.com" @@ -1556,8 +1628,6 @@ def test_basic_auth(self): for challenge in challenges] self.check_basic_auth(headers, realm) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_basic_auth(self): opener = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) @@ -1565,7 +1635,7 @@ def test_proxy_basic_auth(self): password_manager = MockPasswordManager() auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) realm = "ACME Networks" - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) opener.add_handler(auth_handler) opener.add_handler(http_handler) @@ -1575,15 +1645,13 @@ def test_proxy_basic_auth(self): "proxy.example.com:3128", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_and_digest_auth_handlers(self): # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* - # response (http://python.org/sf/1479302), where it should instead + # response (https://bugs.python.org/issue1479302), where it should instead # return None to allow another handler (especially # HTTPBasicAuthHandler) to handle the response. - # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must + # Also (https://bugs.python.org/issue14797027, RFC 2617 section 1.2), we must # try digest first (since it's the strongest auth scheme), so we record # order of calls here to check digest comes first: class RecordingOpenerDirector(OpenerDirector): @@ -1611,7 +1679,7 @@ def http_error_401(self, *args, **kwds): digest_handler = TestDigestAuthHandler(password_manager) basic_handler = TestBasicAuthHandler(password_manager) realm = "ACME Networks" - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) opener.add_handler(basic_handler) opener.add_handler(digest_handler) @@ -1631,7 +1699,7 @@ def test_unsupported_auth_digest_handler(self): opener = OpenerDirector() # While using DigestAuthHandler digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Kerberos\r\n\r\n') opener.add_handler(digest_auth_handler) opener.add_handler(http_handler) @@ -1641,7 +1709,7 @@ def test_unsupported_auth_basic_handler(self): # While using BasicAuthHandler opener = OpenerDirector() basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: NTLM\r\n\r\n') opener.add_handler(basic_auth_handler) opener.add_handler(http_handler) @@ -1684,8 +1752,6 @@ def _test_basic_auth(self, opener, auth_handler, auth_header, self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_auto_send(self): # Assume already authenticated if is_authenticated=True # for APIs like Github that don't return 401 @@ -1713,8 +1779,6 @@ def test_basic_prior_auth_auto_send(self): # expect request to be sent with auth header self.assertTrue(http_handler.has_auth_header) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_send_after_first_success(self): # Auto send auth header after authentication is successful once @@ -1732,7 +1796,7 @@ def test_basic_prior_auth_send_after_first_success(self): opener = OpenerDirector() opener.add_handler(auth_prior_handler) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) opener.add_handler(http_handler) @@ -1842,14 +1906,21 @@ def test_HTTPError_interface(self): url = code = fp = None hdrs = 'Content-Length: 42' err = urllib.error.HTTPError(url, code, msg, hdrs, fp) - self.assertTrue(hasattr(err, 'reason')) + self.assertHasAttr(err, 'reason') self.assertEqual(err.reason, 'something bad happened') - self.assertTrue(hasattr(err, 'headers')) + self.assertHasAttr(err, 'headers') self.assertEqual(err.headers, 'Content-Length: 42') expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) self.assertEqual(str(err), expected_errmsg) expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + err.close() + + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + self.assertIsInstance(x.fp.read(), bytes) + x.close() def test_parse_proxy(self): parse_proxy_test_cases = [ @@ -1896,10 +1967,38 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), - def test_unsupported_algorithm(self): - handler = AbstractDigestAuthHandler() + +skip_libssl_fips_mode = unittest.skipIf( + support.is_libssl_fips_mode(), + "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", +) + + +class TestDigestAuthAlgorithms(unittest.TestCase): + def setUp(self): + self.handler = AbstractDigestAuthHandler() + + @skip_libssl_fips_mode + def test_md5_algorithm(self): + H, KD = self.handler.get_algorithm_impls('MD5') + self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") + self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + + @skip_libssl_fips_mode + def test_sha_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA') + self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + + @skip_libssl_fips_mode + def test_sha256_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA-256') + self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") + + def test_invalid_algorithm(self): with self.assertRaises(ValueError) as exc: - handler.get_algorithm_impls('invalid') + self.handler.get_algorithm_impls('invalid') self.assertEqual( str(exc.exception), "Unsupported digest authentication algorithm 'invalid'" diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 2c54ef85b4b..87da54d5384 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -8,15 +8,17 @@ import unittest import hashlib +from test import support from test.support import hashlib_helper from test.support import threading_helper -from test.support import warnings_helper try: import ssl except ImportError: ssl = None +support.requires_working_socket(module=True) + here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem') @@ -314,7 +316,9 @@ def test_basic_auth_httperror(self): ah = urllib.request.HTTPBasicAuthHandler() ah.add_password(self.REALM, self.server_url, self.USER, self.INCORRECT_PASSWD) urllib.request.install_opener(urllib.request.build_opener(ah)) - self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen(self.server_url) + cm.exception.close() @hashlib_helper.requires_hashdigest("md5", openssl=True) @@ -356,23 +360,22 @@ def stop_server(self): self.server.stop() self.server = None - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_proxy_with_bad_password_raises_httperror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD+"bad") self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() + - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_proxy_with_no_password_raises_httperror(self): self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_proxy_qop_auth_works(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD) @@ -381,7 +384,7 @@ def test_proxy_qop_auth_works(self): while result.read(): pass - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_proxy_qop_auth_int_works_or_throws_urlerror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD) @@ -506,7 +509,7 @@ def start_https_server(self, responses=None, **kwargs): handler.port = server.port return handler - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_redirection(self): expected_response = b"We got here..." responses = [ @@ -520,7 +523,7 @@ def test_redirection(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/", "/somewhere_else"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_chunked(self): expected_response = b"hello world" chunked_start = ( @@ -535,7 +538,7 @@ def test_chunked(self): data = self.urlopen("http://localhost:%s/" % handler.port) self.assertEqual(data, expected_response) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_404(self): expected_response = b"Bad bad bad..." handler = self.start_server([(404, [], expected_response)]) @@ -551,7 +554,7 @@ def test_404(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/weeble"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_200(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -559,7 +562,7 @@ def test_200(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/bizarre"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_200_with_parameters(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -568,41 +571,14 @@ def test_200_with_parameters(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_https(self): handler = self.start_https_server() context = ssl.create_default_context(cafile=CERT_localhost) data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) self.assertEqual(data, b"we care a bit") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") - def test_https_with_cafile(self): - handler = self.start_https_server(certfile=CERT_localhost) - with warnings_helper.check_warnings(('', DeprecationWarning)): - # Good cert - data = self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_localhost) - self.assertEqual(data, b"we care a bit") - # Bad cert - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) - # Good cert, but mismatching hostname - handler = self.start_https_server(certfile=CERT_fakehostname) - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) - - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") - def test_https_with_cadefault(self): - handler = self.start_https_server(certfile=CERT_localhost) - # Self-signed cert should fail verification with system certificate store - with warnings_helper.check_warnings(('', DeprecationWarning)): - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cadefault=True) - - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_https_sni(self): if ssl is None: self.skipTest("ssl module required") @@ -619,7 +595,7 @@ def cb_sni(ssl_sock, server_name, initial_context): self.urlopen("https://localhost:%s" % handler.port, context=context) self.assertEqual(sni_name, "localhost") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_sending_headers(self): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, @@ -628,7 +604,7 @@ def test_sending_headers(self): pass self.assertEqual(handler.headers_received["Range"], "bytes=20-39") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_sending_headers_camel(self): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, @@ -638,16 +614,15 @@ def test_sending_headers_camel(self): self.assertIn("X-Some-Header", handler.headers_received.keys()) self.assertNotIn("X-SoMe-hEader", handler.headers_received.keys()) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_basic(self): handler = self.start_server() with urllib.request.urlopen("http://localhost:%s" % handler.port) as open_url: for attr in ("read", "close", "info", "geturl"): - self.assertTrue(hasattr(open_url, attr), "object returned from " - "urlopen lacks the %s attribute" % attr) + self.assertHasAttr(open_url, attr) self.assertTrue(open_url.read(), "calling 'read' failed") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_info(self): handler = self.start_server() open_url = urllib.request.urlopen( @@ -659,7 +634,7 @@ def test_info(self): "instance of email.message.Message") self.assertEqual(info_obj.get_content_subtype(), "plain") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_geturl(self): # Make sure same URL as opened is returned by geturl. handler = self.start_server() @@ -668,7 +643,7 @@ def test_geturl(self): url = open_url.geturl() self.assertEqual(url, "http://localhost:%s" % handler.port) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_iteration(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -676,7 +651,7 @@ def test_iteration(self): for line in data: self.assertEqual(line, expected_response) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_line_iteration(self): lines = [b"We\n", b"got\n", b"here\n", b"verylong " * 8192 + b"\n"] expected_response = b"".join(lines) @@ -689,7 +664,7 @@ def test_line_iteration(self): (index, len(lines[index]), len(line))) self.assertEqual(index + 1, len(lines)) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_issue16464(self): # See https://bugs.python.org/issue16464 # and https://bugs.python.org/issue46648 diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 41f170a6ad5..d015267cefd 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -7,7 +7,6 @@ from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied -from test.test_urllib2 import sanepathname2url from test.support.warnings_helper import check_no_resource_warning import os @@ -192,7 +191,7 @@ def test_file(self): f.write('hi there\n') f.close() urls = [ - 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True), ('file:///nonsensename/etc/passwd', None, urllib.error.URLError), ] diff --git a/Lib/test/test_urllib_response.py b/Lib/test/test_urllib_response.py index 73d2ef0424f..d949fa38bfc 100644 --- a/Lib/test/test_urllib_response.py +++ b/Lib/test/test_urllib_response.py @@ -4,6 +4,11 @@ import tempfile import urllib.response import unittest +from test import support + +if support.is_wasi: + raise unittest.SkipTest("Cannot create socket on WASI") + class TestResponse(unittest.TestCase): @@ -43,6 +48,7 @@ def test_addinfo(self): info = urllib.response.addinfo(self.fp, self.test_headers) self.assertEqual(info.info(), self.test_headers) self.assertEqual(info.headers, self.test_headers) + info.close() def test_addinfourl(self): url = "http://www.python.org" @@ -55,6 +61,7 @@ def test_addinfourl(self): self.assertEqual(infourl.headers, self.test_headers) self.assertEqual(infourl.url, url) self.assertEqual(infourl.status, code) + infourl.close() def tearDown(self): self.sock.close() diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 6733fe9c6ea..1a42c35dc49 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -2,10 +2,10 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support.testcase import ExtraAssertions import contextlib import socket +import urllib.error import urllib.parse import urllib.request import os @@ -35,7 +35,7 @@ def testURLread(self): f.read() -class urlopenNetworkTests(unittest.TestCase, ExtraAssertions): +class urlopenNetworkTests(unittest.TestCase): """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a @@ -101,13 +101,11 @@ def test_getcode(self): # test getcode() with the fancy opener to get 404 error codes URL = self.url + "XXXinvalidXXX" with socket_helper.transient_internet(URL): - with self.assertWarns(DeprecationWarning): - open_url = urllib.request.FancyURLopener().open(URL) - try: - code = open_url.getcode() - finally: - open_url.close() - self.assertEqual(code, 404) + with self.assertRaises(urllib.error.URLError) as e: + with urllib.request.urlopen(URL): + pass + self.assertEqual(e.exception.code, 404) + e.exception.close() @support.requires_resource('walltime') def test_bad_address(self): diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index af6fe99fb51..b2bde5a9b1d 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -2,6 +2,7 @@ import unicodedata import unittest import urllib.parse +from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" @@ -19,6 +20,10 @@ ("=a", [('', 'a')]), ("a", [('a', '')]), ("a=", [('a', '')]), + ("a=b=c", [('a', 'b=c')]), + ("a%3Db=c", [('a=b', 'c')]), + ("a=b&c=d", [('a', 'b'), ('c', 'd')]), + ("a=b%26c=d", [('a', 'b&c=d')]), ("&a=b", [('a', 'b')]), ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), ("a=1&a=2", [('a', '1'), ('a', '2')]), @@ -29,6 +34,10 @@ (b"=a", [(b'', b'a')]), (b"a", [(b'a', b'')]), (b"a=", [(b'a', b'')]), + (b"a=b=c", [(b'a', b'b=c')]), + (b"a%3Db=c", [(b'a=b', b'c')]), + (b"a=b&c=d", [(b'a', b'b'), (b'c', b'd')]), + (b"a=b%26c=d", [(b'a', b'b&c=d')]), (b"&a=b", [(b'a', b'b')]), (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), @@ -36,6 +45,14 @@ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), (b";a=b", [(b';a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), + + ("\u0141=\xE9", [('\u0141', '\xE9')]), + ("%C5%81=%C3%A9", [('\u0141', '\xE9')]), + ("%81=%A9", [('\ufffd', '\ufffd')]), + (b"\xc5\x81=\xc3\xa9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"%C5%81=%C3%A9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"\x81=\xA9", [(b'\x81', b'\xa9')]), + (b"%81=%A9", [(b'\x81', b'\xa9')]), ] # Each parse_qs testcase is a two-tuple that contains @@ -49,6 +66,10 @@ ("=a", {'': ['a']}), ("a", {'a': ['']}), ("a=", {'a': ['']}), + ("a=b=c", {'a': ['b=c']}), + ("a%3Db=c", {'a=b': ['c']}), + ("a=b&c=d", {'a': ['b'], 'c': ['d']}), + ("a=b%26c=d", {'a': ['b&c=d']}), ("&a=b", {'a': ['b']}), ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), ("a=1&a=2", {'a': ['1', '2']}), @@ -59,6 +80,10 @@ (b"=a", {b'': [b'a']}), (b"a", {b'a': [b'']}), (b"a=", {b'a': [b'']}), + (b"a=b=c", {b'a': [b'b=c']}), + (b"a%3Db=c", {b'a=b': [b'c']}), + (b"a=b&c=d", {b'a': [b'b'], b'c': [b'd']}), + (b"a=b%26c=d", {b'a': [b'b&c=d']}), (b"&a=b", {b'a': [b'b']}), (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1&a=2", {b'a': [b'1', b'2']}), @@ -66,26 +91,37 @@ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), (b";a=b", {b';a': [b'b']}), (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), + (b"a=a%E2%80%99b", {b'a': [b'a\xe2\x80\x99b']}), + + ("\u0141=\xE9", {'\u0141': ['\xE9']}), + ("%C5%81=%C3%A9", {'\u0141': ['\xE9']}), + ("%81=%A9", {'\ufffd': ['\ufffd']}), + (b"\xc5\x81=\xc3\xa9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"%C5%81=%C3%A9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"\x81=\xA9", {b'\x81': [b'\xa9']}), + (b"%81=%A9", {b'\x81': [b'\xa9']}), ] class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) - self.assertEqual(result, parsed) + self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, result.params, result.query, result.fragment) - self.assertEqual(t, parsed) + self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url2) + self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it # again to get the same result: result3 = urllib.parse.urlparse(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -99,18 +135,18 @@ def checkRoundtrips(self, url, parsed, split): # check the roundtrip using urlsplit() as well result = urllib.parse.urlsplit(url) - self.assertEqual(result, split) + self.assertSequenceEqual(result, split) t = (result.scheme, result.netloc, result.path, result.query, result.fragment) - self.assertEqual(t, split) + self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url2) + self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() result3 = urllib.parse.urlsplit(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -121,30 +157,79 @@ def checkRoundtrips(self, url, parsed, split): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = urllib.parse.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = [v for v in expect if len(v[1])] - result = urllib.parse.parse_qsl(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_qs(self): - for orig, expect in parse_qs_test_cases: - result = urllib.parse.parse_qs(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = {v: expect[v] - for v in expect if len(expect[v][0])} - result = urllib.parse.parse_qs(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_roundtrips(self): - str_cases = [ + @support.subTests('orig,expect', parse_qsl_test_cases) + def test_qsl(self, orig, expect): + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('orig,expect', parse_qs_test_cases) + def test_qs(self, orig, expect): + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,parsed,split', [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), + ('file:tmp/junk.txt', + ('file', '', 'tmp/junk.txt', '', '', ''), + ('file', '', 'tmp/junk.txt', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), + ('http:tmp/junk.txt', + ('http', '', 'tmp/junk.txt', '', '', ''), + ('http', '', 'tmp/junk.txt', '', '')), + ('http://example.com/tmp/junk.txt', + ('http', 'example.com', '/tmp/junk.txt', '', '', ''), + ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http:///example.com/tmp/junk.txt', + ('http', '', '/example.com/tmp/junk.txt', '', '', ''), + ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http:////example.com/tmp/junk.txt', + ('http', '', '//example.com/tmp/junk.txt', '', '', ''), + ('http', '', '//example.com/tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -162,24 +247,68 @@ def test_roundtrips(self): ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', '', '')), ('git+ssh://git@github.com/user/project.git', - ('git+ssh', 'git@github.com','/user/project.git', - '','',''), - ('git+ssh', 'git@github.com','/user/project.git', - '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - for url, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split) - - def test_http_roundtrips(self): - # urllib.parse.urlsplit treats 'http:' as an optimized special case, - # so we test both 'http:' and 'https:' in all the following. - # Three cheers for white box knowledge! - str_cases = [ + ('git+ssh', 'git@github.com','/user/project.git', + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), + ('itms-services://?action=download-manifest&url=https://example.com/app', + ('itms-services', '', '', '', + 'action=download-manifest&url=https://example.com/app', ''), + ('itms-services', '', '', + 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ]) + def test_roundtrips(self, bytes, url, parsed, split): + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,url2,parsed,split', [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ]) + def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): + if bytes: + url = str_encode(url) + url2 = str_encode(url2) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split, url2) + + @support.subTests('bytes', (False, True)) + @support.subTests('scheme', ('http', 'https')) + @support.subTests('url,parsed,split', [ ('://www.python.org', ('www.python.org', '', '', '', ''), ('www.python.org', '', '', '')), @@ -195,37 +324,42 @@ def test_http_roundtrips(self): ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_schemes = ('http', 'https') - bytes_schemes = (b'http', b'https') - str_tests = str_schemes, str_cases - bytes_tests = bytes_schemes, bytes_cases - for schemes, test_cases in (str_tests, bytes_tests): - for scheme in schemes: - for url, parsed, split in test_cases: - url = scheme + url - parsed = (scheme,) + parsed - split = (scheme,) + split - self.checkRoundtrips(url, parsed, split) - - def checkJoin(self, base, relurl, expected): - str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - bytes_components = baseb, relurlb, expectedb = [ - x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - - def test_unparse_parse(self): - str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] - bytes_cases = [x.encode('ascii') for x in str_cases] - for u in str_cases + bytes_cases: - self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) - self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + ]) + def test_http_roundtrips(self, bytes, scheme, url, parsed, split): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + if bytes: + scheme = str_encode(scheme) + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) + + def checkJoin(self, base, relurl, expected, *, relroundtrip=True): + with self.subTest(base=base, relurl=relurl): + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + baseb = base.encode('ascii') + relurlb = relurl.encode('ascii') + expectedb = expected.encode('ascii') + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + if relroundtrip: + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + @support.subTests('bytes', (False, True)) + @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) + def test_unparse_parse(self, bytes, u): + if bytes: + u = str_encode(u) + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) def test_RFC1808(self): # "normal" cases from RFC 1808: @@ -384,8 +518,6 @@ def test_RFC3986(self): def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g:h','g:h') - self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') @@ -406,8 +538,6 @@ def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') - self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') self.checkJoin('http:///', '..','http:///') @@ -437,8 +567,127 @@ def test_urljoins(self): # issue 23703: don't duplicate filename self.checkJoin('a', 'b', 'b') - def test_RFC2732(self): - str_cases = [ + # Test with empty (but defined) components. + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, ';', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, ';?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, ';#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, '/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '//#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '//?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, '//;x', 'http://;x') + self.checkJoin(RFC1808_BASE, '///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//v', 'http://v') + # For backward compatibility with RFC1630, the scheme name is allowed + # to be present in a relative reference if it is the same as the base + # URI scheme. + self.checkJoin(RFC1808_BASE, 'http:', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http:#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http:?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http:;', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, 'http:;?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, 'http:;#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, 'http:;x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, 'http:/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http://#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http://?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http://;x', 'http://;x') + self.checkJoin(RFC1808_BASE, 'http:///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://v', 'http://v') + # Different scheme is not ignored. + self.checkJoin(RFC1808_BASE, 'https:', 'https:', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#', 'https:#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#z', 'https:#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?', 'https:?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?y', 'https:?y', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') + self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + + def test_urljoins_relative_base(self): + # According to RFC 3986, Section 5.1, a base URI must conform to + # the absolute-URI syntax rule (Section 4.3). But urljoin() lacks + # a context to establish missed components of the relative base URI. + # It still has to return a sensible result for backwards compatibility. + # The following tests are figments of the imagination and artifacts + # of the current implementation that are not based on any standard. + self.checkJoin('', '', '') + self.checkJoin('', '//', '//', relroundtrip=False) + self.checkJoin('', '//v', '//v') + self.checkJoin('', '//v/w', '//v/w') + self.checkJoin('', '/w', '/w') + self.checkJoin('', '///w', '///w', relroundtrip=False) + self.checkJoin('', 'w', 'w') + + self.checkJoin('//', '', '//') + self.checkJoin('//', '//', '//') + self.checkJoin('//', '//v', '//v') + self.checkJoin('//', '//v/w', '//v/w') + self.checkJoin('//', '/w', '///w') + self.checkJoin('//', '///w', '///w') + self.checkJoin('//', 'w', '///w') + + self.checkJoin('//a', '', '//a') + self.checkJoin('//a', '//', '//a') + self.checkJoin('//a', '//v', '//v') + self.checkJoin('//a', '//v/w', '//v/w') + self.checkJoin('//a', '/w', '//a/w') + self.checkJoin('//a', '///w', '//a/w') + self.checkJoin('//a', 'w', '//a/w') + + for scheme in '', 'http:': + self.checkJoin('http:', scheme + '', 'http:') + self.checkJoin('http:', scheme + '//', 'http:') + self.checkJoin('http:', scheme + '//v', 'http://v') + self.checkJoin('http:', scheme + '//v/w', 'http://v/w') + self.checkJoin('http:', scheme + '/w', 'http:/w') + self.checkJoin('http:', scheme + '///w', 'http:/w') + self.checkJoin('http:', scheme + 'w', 'http:/w') + + self.checkJoin('http://', scheme + '', 'http://') + self.checkJoin('http://', scheme + '//', 'http://') + self.checkJoin('http://', scheme + '//v', 'http://v') + self.checkJoin('http://', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://', scheme + '/w', 'http:///w') + self.checkJoin('http://', scheme + '///w', 'http:///w') + self.checkJoin('http://', scheme + 'w', 'http:///w') + + self.checkJoin('http://a', scheme + '', 'http://a') + self.checkJoin('http://a', scheme + '//', 'http://a') + self.checkJoin('http://a', scheme + '//v', 'http://v') + self.checkJoin('http://a', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://a', scheme + '/w', 'http://a/w') + self.checkJoin('http://a', scheme + '///w', 'http://a/w') + self.checkJoin('http://a', scheme + 'w', 'http://a/w') + + self.checkJoin('/b/c', '', '/b/c') + self.checkJoin('/b/c', '//', '/b/c') + self.checkJoin('/b/c', '//v', '//v') + self.checkJoin('/b/c', '//v/w', '//v/w') + self.checkJoin('/b/c', '/w', '/w') + self.checkJoin('/b/c', '///w', '/w') + self.checkJoin('/b/c', 'w', '/b/w') + + self.checkJoin('///b/c', '', '///b/c') + self.checkJoin('///b/c', '//', '///b/c') + self.checkJoin('///b/c', '//v', '//v') + self.checkJoin('///b/c', '//v/w', '//v/w') + self.checkJoin('///b/c', '/w', '///w') + self.checkJoin('///b/c', '///w', '///w') + self.checkJoin('///b/c', 'w', '///b/w') + + @support.subTests('bytes', (False, True)) + @support.subTests('url,hostname,port', [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), @@ -469,26 +718,28 @@ def test_RFC2732(self): ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:12.34.56.78', None), - ] - def _encode(t): - return t[0].encode('ascii'), t[1].encode('ascii'), t[2] - bytes_cases = [_encode(x) for x in str_cases] - for url, hostname, port in str_cases + bytes_cases: - urlparsed = urllib.parse.urlparse(url) - self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) - - str_cases = [ + ]) + def test_RFC2732(self, bytes, url, hostname, port): + if bytes: + url = str_encode(url) + hostname = str_encode(hostname) + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port), (hostname, port)) + + @support.subTests('bytes', (False, True)) + @support.subTests('invalid_url', [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', - 'http://[::ffff:12.34.56.78'] - bytes_cases = [x.encode('ascii') for x in str_cases] - for invalid_url in str_cases + bytes_cases: - self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) - - def test_urldefrag(self): - str_cases = [ + 'http://[::ffff:12.34.56.78']) + def test_RFC2732_invalid(self, bytes, invalid_url): + if bytes: + invalid_url = str_encode(invalid_url) + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), ('http://python.org', 'http://python.org', ''), ('http://python.org/#frag', 'http://python.org/', 'frag'), @@ -499,16 +750,31 @@ def test_urldefrag(self): ('http://python.org/p?q', 'http://python.org/p?q', ''), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), - ] - def _encode(t): - return type(t)(x.encode('ascii') for x in t) - bytes_cases = [_encode(x) for x in str_cases] - for url, defrag, frag in str_cases + bytes_cases: - result = urllib.parse.urldefrag(url) - self.assertEqual(result.geturl(), url) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), + ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), + ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), + ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), + ('http://a/b/c?q#f', 'http://a/b/c?q', 'f'), + ('http:///b/c;p?q#f', 'http:///b/c;p?q', 'f'), + ('http:b/c;p?q#f', 'http:b/c;p?q', 'f'), + ('http:;?q#f', 'http:;?q', 'f'), + ('http:?q#f', 'http:?q', 'f'), + ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), + ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), + ]) + def test_urldefrag(self, bytes, url, defrag, frag): + if bytes: + url = str_encode(url) + defrag = str_encode(defrag) + frag = str_encode(frag) + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -649,21 +915,94 @@ def test_urlsplit_remove_unsafe_bytes(self): self.assertEqual(p.scheme, "http") self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment") - def test_attributes_bad_port(self): + def test_urlsplit_strip_url(self): + noise = bytes(range(0, 0x20 + 1)) + base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" + + url = noise.decode("utf-8") + base_url + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "User:Pass@www.python.org:080") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query=yes") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, "User") + self.assertEqual(p.password, "Pass") + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url) + + url = noise + base_url.encode("utf-8") + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"query=yes") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, b"User") + self.assertEqual(p.password, b"Pass") + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url.encode("utf-8")) + + # Test that trailing space is preserved as some applications rely on + # this within query strings. + query_spaces_url = "https://www.python.org:88/doc/?query= " + p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.netloc, "www.python.org:88") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query= ") + self.assertEqual(p.port, 88) + self.assertEqual(p.geturl(), query_spaces_url) + + p = urllib.parse.urlsplit("www.pypi.org ") + # That "hostname" gets considered a "path" due to the + # trailing space and our existing logic... YUCK... + # and re-assembles via geturl aka unurlsplit into the original. + # django.core.validators.URLValidator (at least through v3.2) relies on + # this, for better or worse, to catch it in a ValidationError via its + # regular expressions. + # Here we test the basic round trip concept of such a trailing space. + self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") + + # with scheme as cache-key + url = "//www.python.org/" + scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") + for _ in range(2): + p = urllib.parse.urlsplit(url, scheme=scheme) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.geturl(), "https://www.python.org/") + + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('port', ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६")) + def test_attributes_bad_port(self, bytes, parse, port): """Check handling of invalid ports.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port - url = "http://" + netloc - if bytes: - netloc = netloc.encode("ascii") - url = url.encode("ascii") - p = parse(url) - self.assertEqual(p.netloc, netloc) - with self.assertRaises(ValueError): - p.port + netloc = "www.example.net:" + port + url = "http://" + netloc + "/" + if bytes: + if not (netloc.isascii() and port.isascii()): + self.skipTest('non-ASCII bytes') + netloc = str_encode(netloc) + url = str_encode(url) + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port + + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('scheme', (".", "+", "-", "0", "http&", "६http")) + def test_attributes_bad_scheme(self, bytes, parse, scheme): + """Check handling of invalid schemes.""" + url = scheme + "://www.example.net" + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.scheme, b"" if bytes else "") def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it @@ -775,24 +1114,21 @@ def test_anyscheme(self): self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) - def test_default_scheme(self): + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_default_scheme(self, func): # Exercise the scheme parameter of urlparse() and urlsplit() - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - with self.subTest(function=func): - result = func("http://example.net/", "ftp") - self.assertEqual(result.scheme, "http") - result = func(b"http://example.net/", b"ftp") - self.assertEqual(result.scheme, b"http") - self.assertEqual(func("path", "ftp").scheme, "ftp") - self.assertEqual(func("path", scheme="ftp").scheme, "ftp") - self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") - self.assertEqual(func("path").scheme, "") - self.assertEqual(func(b"path").scheme, b"") - self.assertEqual(func(b"path", "").scheme, b"") - - def test_parse_fragments(self): - # Exercise the allow_fragments parameter of urlparse() and urlsplit() - tests = ( + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") + + @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), ("//example.net#frag", "path", "frag"), ("index.html#frag", "path", "frag"), @@ -803,25 +1139,24 @@ def test_parse_fragments(self): ("//abc#@frag", "path", "@frag"), ("//abc:80#@frag", "path", "@frag"), ("//abc#@frag:80", "path", "@frag:80"), - ) - for url, attr, expected_frag in tests: - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - if attr == "params" and func is urllib.parse.urlsplit: - attr = "path" - with self.subTest(url=url, function=func): - result = func(url, allow_fragments=False) - self.assertEqual(result.fragment, "") - self.assertTrue( - getattr(result, attr).endswith("#" + expected_frag)) - self.assertEqual(func(url, "", False).fragment, "") - - result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, expected_frag) - self.assertFalse( - getattr(result, attr).endswith(expected_frag)) - self.assertEqual(func(url, "", True).fragment, - expected_frag) - self.assertEqual(func(url).fragment, expected_frag) + )) + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_parse_fragments(self, url, attr, expected_frag, func): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertEndsWith(getattr(result, attr), + "#" + expected_frag) + self.assertEqual(func(url, "", False).fragment, "") + + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -847,7 +1182,14 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - def _check_result_type(self, str_type): + @support.subTests('result_type', [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ]) + def test_result_pairs(self, result_type): + # Check encoding and decoding between result pairs + str_type = result_type num_args = len(str_type._fields) bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -872,16 +1214,6 @@ def _check_result_type(self, str_type): self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) - def test_result_pairs(self): - # Check encoding and decoding between result pairs - result_types = [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ] - for result_type in result_types: - self._check_result_type(result_type) - def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") self.assertEqual(result, {'key': ['\u0141\xE9']}) @@ -910,11 +1242,10 @@ def test_parse_qsl_encoding(self): def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): - urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) - urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) - def test_parse_qs_separator(self): - parse_qs_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", {}), (";;", {}), (";a=b", {'a': ['b']}), @@ -925,17 +1256,14 @@ def test_parse_qs_separator(self): (b";a=b", {b'a': [b'b']}), (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1;a=2", {b'a': [b'1', b'2']}), - ] - for orig, expect in parse_qs_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qs(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qs(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - - - def test_parse_qsl_separator(self): - parse_qsl_semicolon_cases = [ + ]) + def test_parse_qs_separator(self, orig, expect): + result = urllib.parse.parse_qs(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qs(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + @support.subTests('orig,expect', [ (";", []), (";;", []), (";a=b", [('a', 'b')]), @@ -946,14 +1274,45 @@ def test_parse_qsl_separator(self): (b";a=b", [(b'a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), - ] - for orig, expect in parse_qsl_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qsl(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qsl(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - + ]) + def test_parse_qsl_separator(self, orig, expect): + result = urllib.parse.parse_qsl(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qsl(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + for x in 0, 0.0, [], {}: + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertEqual(cm.filename, __file__) + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qs(x, **kwargs), {}) + self.assertEqual(cm.filename, __file__) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: @@ -985,6 +1344,10 @@ def test_quote_from_bytes(self): self.assertEqual(result, 'archaeological%20arcana') result = urllib.parse.quote_from_bytes(b'') self.assertEqual(result, '') + result = urllib.parse.quote_from_bytes(b'A'*10_000) + self.assertEqual(result, 'A'*10_000) + result = urllib.parse.quote_from_bytes(b'z\x01/ '*253_183) + self.assertEqual(result, 'z%01/%20'*253_183) def test_unquote_to_bytes(self): result = urllib.parse.unquote_to_bytes('abc%20def') @@ -1012,6 +1375,67 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_invalid_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip[suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[suffix') + + def test_splitting_bracketed_hosts(self): + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]:1234/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + self.assertEqual(p1.port, 1234) + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + self.assertIs(p2.port, None) + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') @@ -1044,16 +1468,24 @@ def test_telurl_params(self): self.assertEqual(p1.params, 'phone-context=+1-914-555') def test_Quoter_repr(self): - quoter = urllib.parse.Quoter(urllib.parse._ALWAYS_SAFE) + quoter = urllib.parse._Quoter(urllib.parse._ALWAYS_SAFE) self.assertIn('Quoter', repr(quoter)) + def test_clear_cache_for_code_coverage(self): + urllib.parse.clear_cache() + + def test_urllib_parse_getattr_failure(self): + """Test that urllib.parse.__getattr__() fails correctly.""" + with self.assertRaises(AttributeError): + unused = urllib.parse.this_does_not_exist + def test_all(self): expected = [] undocumented = { 'splitattr', 'splithost', 'splitnport', 'splitpasswd', 'splitport', 'splitquery', 'splittag', 'splittype', 'splituser', 'splitvalue', - 'Quoter', 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap', + 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap', } for name in dir(urllib.parse): if name.startswith('_') or name in undocumented: @@ -1063,8 +1495,6 @@ def test_all(self): expected.append(name) self.assertCountEqual(urllib.parse.__all__, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_urlsplit_normalization(self): # Certain characters should never occur in the netloc, # including under normalization. @@ -1073,7 +1503,8 @@ def test_urlsplit_normalization(self): hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars} denorm_chars = [ c for c in map(chr, range(128, sys.maxunicode)) - if (hex_chars & set(unicodedata.decomposition(c).split())) + if unicodedata.decomposition(c) + and (hex_chars & set(unicodedata.decomposition(c).split())) and c not in illegal_chars ] # Sanity check that we found at least one such character @@ -1188,6 +1619,7 @@ def test_splitnport(self): self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + self.assertEqual(splitnport('parrot: +1_0 '), ('parrot', None)) def test_splitquery(self): # Normal cases are exercised by other tests; ensure that we also @@ -1238,15 +1670,15 @@ def test_to_bytes(self): self.assertRaises(UnicodeError, urllib.parse._to_bytes, 'http://www.python.org/medi\u00e6val') - def test_unwrap(self): - for wrapped_url in ('<URL:scheme://host/path>', '<scheme://host/path>', - 'URL:scheme://host/path', 'scheme://host/path'): - url = urllib.parse.unwrap(wrapped_url) - self.assertEqual(url, 'scheme://host/path') + @support.subTests('wrapped_url', + ('<URL:scheme://host/path>', '<scheme://host/path>', + 'URL:scheme://host/path', 'scheme://host/path')) + def test_unwrap(self, wrapped_url): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') class DeprecationTest(unittest.TestCase): - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') @@ -1324,5 +1756,11 @@ def test_to_bytes_deprecation(self): 'urllib.parse.to_bytes() is deprecated as of 3.8') +def str_encode(s): + return s.encode('ascii') + +def tuple_encode(t): + return tuple(str_encode(x) for x in t) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 61e79f553e8..75de9ea252d 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,6 +1,6 @@ # Check every path through every method of UserDict -from test import mapping_tests, support +from test import mapping_tests import unittest import collections @@ -166,7 +166,7 @@ def test_update(self): def test_missing(self): # Make sure UserDict doesn't have a __missing__ method - self.assertEqual(hasattr(collections.UserDict, "__missing__"), False) + self.assertNotHasAttr(collections.UserDict, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -213,11 +213,7 @@ class G(collections.UserDict): else: self.fail("g[42] didn't raise KeyError") - # Decorate existing test with recursion limit, because - # the test is for C structure, but `UserDict` is a Python structure. - test_repr_deep = support.infinite_recursion(25)( - mapping_tests.TestHashMappingProtocol.test_repr_deep, - ) + test_repr_deep = mapping_tests.TestHashMappingProtocol.test_repr_deep if __name__ == "__main__": diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 312702c8e39..da66ae694d5 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -3,7 +3,6 @@ from collections import UserList from test import list_tests import unittest -from test import support class UserListTest(list_tests.CommonTest): @@ -69,9 +68,8 @@ def test_userlist_copy(self): # Decorate existing test with recursion limit, because # the test is for C structure, but `UserList` is a Python structure. - test_repr_deep = support.infinite_recursion(25)( - list_tests.CommonTest.test_repr_deep, - ) + # test_repr_deep = list_tests.CommonTest.test_repr_deep + test_repr_deep = unittest.skip(list_tests.CommonTest.test_repr_deep) # TODO: RUSTPYTHON; Segfault if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index b3e3e0bb27f..176a2112718 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -46,8 +46,7 @@ def test_posix_locale(self): out = self.get_output('-c', code, LC_ALL=loc) self.assertEqual(out, '1') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON") def test_xoption(self): code = 'import sys; print(sys.flags.utf8_mode)' diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 4aa15f69932..0e1a723ce3a 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,7 +1,3 @@ -import unittest -from test import support -from test.support import import_helper -from test.support.script_helper import assert_python_ok import builtins import contextlib import copy @@ -9,10 +5,17 @@ import io import os import pickle +import random import sys +import unittest import weakref +from itertools import product from unittest import mock +from test import support +from test.support import import_helper +from test.support.script_helper import assert_python_ok + py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -33,7 +36,47 @@ def get_command_stdout(command, args): class BaseTestUUID: uuid = None - @unittest.expectedFailure # TODO: RUSTPYTHON + def test_nil_uuid(self): + nil_uuid = self.uuid.NIL + + s = '00000000-0000-0000-0000-000000000000' + i = 0 + self.assertEqual(nil_uuid, self.uuid.UUID(s)) + self.assertEqual(nil_uuid, self.uuid.UUID(int=i)) + self.assertEqual(nil_uuid.int, i) + self.assertEqual(str(nil_uuid), s) + # The Nil UUID falls within the range of the Apollo NCS variant as per + # RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.9-4 + self.assertEqual(nil_uuid.variant, self.uuid.RESERVED_NCS) + # A version field of all zeros is "Unused" in RFC 9562, but the version + # field also only applies to the 10xx variant, i.e. the variant + # specified in RFC 9562. As such, because the Nil UUID falls under a + # different variant, its version is considered undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(nil_uuid.version) + + def test_max_uuid(self): + max_uuid = self.uuid.MAX + + s = 'ffffffff-ffff-ffff-ffff-ffffffffffff' + i = (1 << 128) - 1 + self.assertEqual(max_uuid, self.uuid.UUID(s)) + self.assertEqual(max_uuid, self.uuid.UUID(int=i)) + self.assertEqual(max_uuid.int, i) + self.assertEqual(str(max_uuid), s) + # The Max UUID falls within the range of the "yet-to-be defined" future + # UUID variant as per RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.10-4 + self.assertEqual(max_uuid.variant, self.uuid.RESERVED_FUTURE) + # A version field of all ones is "Reserved for future definition" in + # RFC 9562, but the version field also only applies to the 10xx + # variant, i.e. the variant specified in RFC 9562. As such, because the + # Max UUID falls under a different variant, its version is considered + # undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(max_uuid.version) + def test_safe_uuid_enum(self): class CheckedSafeUUID(enum.Enum): safe = 0 @@ -269,7 +312,7 @@ def test_exceptions(self): # Version number out of range. badvalue(lambda: self.uuid.UUID('00'*16, version=0)) - badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + badvalue(lambda: self.uuid.UUID('00'*16, version=42)) # Integer value out of range. badvalue(lambda: self.uuid.UUID(int=-1)) @@ -683,6 +726,392 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + def test_uuid6(self): + equal = self.assertEqual + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + fake_nanoseconds = 0x1571_20a1_de1a_c533 + fake_node_value = 0x54e1_acf6_da7f + fake_clock_seq = 0x14c5 + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch.object(self.uuid, 'getnode', return_value=fake_node_value), + mock.patch('time.time_ns', return_value=fake_nanoseconds), + mock.patch('random.getrandbits', return_value=fake_clock_seq) + ): + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + # 32 (top) | 16 (mid) | 12 (low) == 60 (timestamp) + equal(u.time, 0x1e901fca_7a55_b92) + equal(u.fields[0], 0x1e901fca) # 32 top bits of time + equal(u.fields[1], 0x7a55) # 16 mid bits of time + # 4 bits of version + 12 low bits of time + equal((u.fields[2] >> 12) & 0xf, 6) + equal((u.fields[2] & 0xfff), 0xb92) + # 2 bits of variant + 6 high bits of clock_seq + equal((u.fields[3] >> 6) & 0xf, 2) + equal(u.fields[3] & 0x3f, fake_clock_seq >> 8) + # 8 low bits of clock_seq + equal(u.fields[4], fake_clock_seq & 0xff) + equal(u.fields[5], fake_node_value) + + def test_uuid6_uniqueness(self): + # Test that UUIDv6-generated values are unique. + + # Unlike UUIDv8, only 62 bits can be randomized for UUIDv6. + # In practice, however, it remains unlikely to generate two + # identical UUIDs for the same 60-bit timestamp if neither + # the node ID nor the clock sequence is specified. + uuids = {self.uuid.uuid6() for _ in range(1000)} + self.assertEqual(len(uuids), 1000) + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {6}) + + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + + with mock.patch('time.time_ns', return_value=fake_nanoseconds): + def gen(): + with mock.patch.object(self.uuid, '_last_timestamp_v6', None): + return self.uuid.uuid6(node=0, clock_seq=None) + + # By the birthday paradox, sampling N = 1024 UUIDs with identical + # node IDs and timestamps results in duplicates with probability + # close to 1 (not having a duplicate happens with probability of + # order 1E-15) since only the 14-bit clock sequence is randomized. + N = 1024 + uuids = {gen() for _ in range(N)} + self.assertSetEqual({u.node for u in uuids}, {0}) + self.assertSetEqual({u.time for u in uuids}, {timestamp}) + self.assertLess(len(uuids), N, 'collision property does not hold') + + def test_uuid6_node(self): + # Make sure the given node ID appears in the UUID. + # + # Note: when no node ID is specified, the same logic as for UUIDv1 + # is applied to UUIDv6. In particular, there is no need to test that + # getnode() correctly returns positive integers of exactly 48 bits + # since this is done in test_uuid1_eui64(). + self.assertLessEqual(self.uuid.uuid6().node.bit_length(), 48) + + self.assertEqual(self.uuid.uuid6(0).node, 0) + + # tests with explicit values + max_node = 0xffff_ffff_ffff + self.assertEqual(self.uuid.uuid6(max_node).node, max_node) + big_node = 0xE_1234_5678_ABCD # 52-bit node + res_node = 0x0_1234_5678_ABCD # truncated to 48 bits + self.assertEqual(self.uuid.uuid6(big_node).node, res_node) + + # randomized tests + for _ in range(10): + # node with > 48 bits is truncated + for b in [24, 48, 72]: + node = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, bitlen=b): + self.assertEqual(node.bit_length(), b) + u = self.uuid.uuid6(node=node) + self.assertEqual(u.node, node & 0xffff_ffff_ffff) + + def test_uuid6_clock_seq(self): + # Make sure the supplied clock sequence appears in the UUID. + # + # For UUIDv6, clock sequence bits are stored from bit 48 to bit 62, + # with the convention that the least significant bit is bit 0 and + # the most significant bit is bit 127. + get_clock_seq = lambda u: (u.int >> 48) & 0x3fff + + u = self.uuid.uuid6() + self.assertLessEqual(get_clock_seq(u).bit_length(), 14) + + # tests with explicit values + big_clock_seq = 0xffff # 16-bit clock sequence + res_clock_seq = 0x3fff # truncated to 14 bits + u = self.uuid.uuid6(clock_seq=big_clock_seq) + self.assertEqual(get_clock_seq(u), res_clock_seq) + + # some randomized tests + for _ in range(10): + # clock_seq with > 14 bits is truncated + for b in [7, 14, 28]: + node = random.getrandbits(48) + clock_seq = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, clock_seq=clock_seq, bitlen=b): + self.assertEqual(clock_seq.bit_length(), b) + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + self.assertEqual(get_clock_seq(u), clock_seq & 0x3fff) + + def test_uuid6_test_vectors(self): + equal = self.assertEqual + # https://www.rfc-editor.org/rfc/rfc9562#name-test-vectors + # (separators are put at the 12th and 28th bits) + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + # https://www.rfc-editor.org/rfc/rfc9562#name-example-of-a-uuidv6-value + node = 0x9f6bdeced846 + clock_seq = (3 << 12) | 0x3c8 + + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch('time.time_ns', return_value=fake_nanoseconds) + ): + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + equal(str(u).upper(), '1EC9414C-232A-6B00-B3C8-9F6BDECED846') + # 32 16 4 12 2 14 48 + # time_hi | time_mid | ver | time_lo | var | clock_seq | node + equal(u.time, timestamp) + equal(u.int & 0xffff_ffff_ffff, node) + equal((u.int >> 48) & 0x3fff, clock_seq) + equal((u.int >> 62) & 0x3, 0b10) + equal((u.int >> 64) & 0xfff, 0xb00) + equal((u.int >> 76) & 0xf, 0x6) + equal((u.int >> 80) & 0xffff, 0x232a) + equal((u.int >> 96) & 0xffff_ffff, 0x1ec9_414c) + + def test_uuid7(self): + equal = self.assertEqual + u = self.uuid.uuid7() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + for _ in range(100): + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(30) + counter = (counter_hi << 30) | counter_lo + + tail = random.getrandbits(32) + # effective number of bits is 32 + 30 + 11 = 73 + random_bits = counter << 32 | tail + + # set all remaining MSB of fake random bits to 1 to ensure that + # the implementation correctly removes them + random_bits = (((1 << 7) - 1) << 73) | random_bits + random_data = random_bits.to_bytes(10) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=None, + _last_counter_v7=0, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_once_with(10) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + + equal(self.uuid._last_timestamp_v7, timestamp_ms) + equal(self.uuid._last_counter_v7, counter) + + unix_ts_ms = timestamp_ms & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + + equal((u.int >> 75) & 1, 0) # check that the MSB is 0 + equal((u.int >> 64) & 0xfff, counter_hi) + equal((u.int >> 32) & 0x3fff_ffff, counter_lo) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid7_uniqueness(self): + # Test that UUIDv7-generated values are unique. + # + # While UUIDv8 has an entropy of 122 bits, those 122 bits may not + # necessarily be sampled from a PRNG. On the other hand, UUIDv7 + # uses os.urandom() as a PRNG which features better randomness. + N = 1000 + uuids = {self.uuid.uuid7() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {7}) + + def test_uuid7_monotonicity(self): + equal = self.assertEqual + + us = [self.uuid.uuid7() for _ in range(10_000)] + equal(us, sorted(us)) + + with mock.patch.multiple( + self.uuid, + _last_timestamp_v7=0, + _last_counter_v7=0, + ): + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + # counter_{hi,lo} are chosen so that "counter + 1" does not overflow + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(29) + counter = (counter_hi << 30) | counter_lo + self.assertLess(counter + 1, 0x3ff_ffff_ffff) + + tail = random.getrandbits(32) + random_bits = counter << 32 | tail + random_data = random_bits.to_bytes(10) + + with ( + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u1 = self.uuid.uuid7() + urand.assert_called_once_with(10) + equal(self.uuid._last_timestamp_v7, timestamp_ms) + equal(self.uuid._last_counter_v7, counter) + equal(u1.time, timestamp_ms) + equal((u1.int >> 64) & 0xfff, counter_hi) + equal((u1.int >> 32) & 0x3fff_ffff, counter_lo) + equal(u1.int & 0xffff_ffff, tail) + + # 1 Jan 2023 12:34:56.123_457_032 (same millisecond but not same ns) + next_timestamp_ns = 1672533296_123_457_032 + next_timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + equal(timestamp_ms, next_timestamp_ms) + + next_tail_bytes = os.urandom(4) + next_fail = int.from_bytes(next_tail_bytes) + + with ( + mock.patch('time.time_ns', return_value=next_timestamp_ns), + mock.patch('os.urandom', return_value=next_tail_bytes) as urand + ): + u2 = self.uuid.uuid7() + urand.assert_called_once_with(4) + # same milli-second + equal(self.uuid._last_timestamp_v7, timestamp_ms) + # 42-bit counter advanced by 1 + equal(self.uuid._last_counter_v7, counter + 1) + equal(u2.time, timestamp_ms) + equal((u2.int >> 64) & 0xfff, counter_hi) + equal((u2.int >> 32) & 0x3fff_ffff, counter_lo + 1) + equal(u2.int & 0xffff_ffff, next_fail) + + self.assertLess(u1, u2) + + def test_uuid7_timestamp_backwards(self): + equal = self.assertEqual + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + fake_last_timestamp_v7 = timestamp_ms + 1 + + # counter_{hi,lo} are chosen so that "counter + 1" does not overflow + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(29) + counter = (counter_hi << 30) | counter_lo + self.assertLess(counter + 1, 0x3ff_ffff_ffff) + + tail_bytes = os.urandom(4) + tail = int.from_bytes(tail_bytes) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=fake_last_timestamp_v7, + _last_counter_v7=counter, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=tail_bytes) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_once_with(4) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + equal(self.uuid._last_timestamp_v7, fake_last_timestamp_v7 + 1) + unix_ts_ms = (fake_last_timestamp_v7 + 1) & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + # 42-bit counter advanced by 1 + equal(self.uuid._last_counter_v7, counter + 1) + equal((u.int >> 64) & 0xfff, counter_hi) + # 42-bit counter advanced by 1 (counter_hi is untouched) + equal((u.int >> 32) & 0x3fff_ffff, counter_lo + 1) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid7_overflow_counter(self): + equal = self.assertEqual + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + new_counter_hi = random.getrandbits(11) + new_counter_lo = random.getrandbits(30) + new_counter = (new_counter_hi << 30) | new_counter_lo + + tail = random.getrandbits(32) + random_bits = (new_counter << 32) | tail + random_data = random_bits.to_bytes(10) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=timestamp_ms, + # same timestamp, but force an overflow on the counter + _last_counter_v7=0x3ff_ffff_ffff, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_with(10) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + # timestamp advanced due to overflow + equal(self.uuid._last_timestamp_v7, timestamp_ms + 1) + unix_ts_ms = (timestamp_ms + 1) & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + # counter overflowed, so we picked a new one + equal(self.uuid._last_counter_v7, new_counter) + equal((u.int >> 64) & 0xfff, new_counter_hi) + equal((u.int >> 32) & 0x3fff_ffff, new_counter_lo) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid8(self): + equal = self.assertEqual + u = self.uuid.uuid8() + + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + + for (_, hi, mid, lo) in product( + range(10), # repeat 10 times + [None, 0, random.getrandbits(48)], + [None, 0, random.getrandbits(12)], + [None, 0, random.getrandbits(62)], + ): + u = self.uuid.uuid8(hi, mid, lo) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + if hi is not None: + equal((u.int >> 80) & 0xffffffffffff, hi) + if mid is not None: + equal((u.int >> 64) & 0xfff, mid) + if lo is not None: + equal(u.int & 0x3fffffffffffffff, lo) + + def test_uuid8_uniqueness(self): + # Test that UUIDv8-generated values are unique (up to a negligible + # probability of failure). There are 122 bits of entropy and assuming + # that the underlying mt-19937-based random generator is sufficiently + # good, it is unlikely to have a collision of two UUIDs. + N = 1000 + uuids = {self.uuid.uuid8() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {8}) + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates @@ -711,6 +1140,23 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -743,6 +1189,20 @@ def test_cli_uuid4_outputted_with_no_args(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 4) + @mock.patch.object(sys, "argv", ["", "-C", "3"]) + def test_cli_uuid4_outputted_with_count(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip().splitlines() + + # Check that 3 UUIDs in the format of uuid4 have been generated + self.assertEqual(len(output), 3) + for o in output: + uuid_output = self.uuid.UUID(o) + self.assertEqual(uuid_output.version, 4) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): @@ -771,13 +1231,25 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid def check_has_stable_libuuid_extractable_node(self): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 94d626598ba..19b12070531 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -5,21 +5,31 @@ Licensed to the PSF under a contributor agreement. """ +import contextlib import ensurepip import os import os.path +import pathlib import re import shutil import struct import subprocess import sys +import sysconfig import tempfile -from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +import shlex +from test.support import (captured_stdout, captured_stderr, + skip_if_broken_multiprocessing_synchronize, verbose, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, + requires_resource, copy_python_src_ignore) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) +from test.support.testcase import ExtraAssertions import unittest import venv -from unittest.mock import patch +from unittest.mock import patch, Mock try: import ctypes @@ -33,18 +43,29 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") + +@requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err -class BaseTest(unittest.TestCase): +class BaseTest(unittest.TestCase, ExtraAssertions): """Base class for venv tests.""" maxDiff = 80 * 50 @@ -56,7 +77,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -70,6 +91,13 @@ def setUp(self): def tearDown(self): rmtree(self.env_dir) + def envpy(self, *, real_env_dir=False): + if real_env_dir: + env_dir = os.path.realpath(self.env_dir) + else: + env_dir = self.env_dir + return os.path.join(env_dir, self.bindir, self.exe) + def run_with_capture(self, func, *args, **kwargs): with captured_stdout() as output: with captured_stderr() as error: @@ -91,12 +119,27 @@ def isdir(self, *args): fn = self.get_env_file(*args) self.assertTrue(os.path.isdir(fn)) - def test_defaults(self): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_str_path(self): """ - Test the create function with default arguments. + Test the create function with default arguments and a str path. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) + self._check_output_of_default_create() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_pathlike(self): + """ + Test the create function with default arguments and a path-like path. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, FakePath(self.env_dir)) + self._check_output_of_default_create() + + def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) @@ -112,6 +155,12 @@ def test_defaults(self): executable = sys._base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) + self.assertIn('executable = %s' % + os.path.realpath(sys.executable), data) + copies = '' if os.name=='nt' else ' --copies' + cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' + f'--without-scm-ignore-files {self.env_dir}') + self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures bd = self.get_env_file(self.bindir) @@ -119,6 +168,39 @@ def test_defaults(self): print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + def test_config_file_command_key(self): + options = [ + (None, None, None), # Default case. + ('--copies', 'symlinks', False), + ('--without-pip', 'with_pip', False), + ('--system-site-packages', 'system_site_packages', True), + ('--clear', 'clear', True), + ('--upgrade', 'upgrade', True), + ('--upgrade-deps', 'upgrade_deps', True), + ('--prompt="foobar"', 'prompt', 'foobar'), + ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), + ] + for opt, attr, value in options: + with self.subTest(opt=opt, attr=attr, value=value): + rmtree(self.env_dir) + if not attr: + kwargs = {} + else: + kwargs = {attr: value} + b = venv.EnvBuilder(**kwargs) + b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps + b._setup_pip = Mock() # avoid pip setup + self.run_with_capture(b.create, self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + if not attr or opt.endswith('git'): + for opt in ('--system-site-packages', '--clear', '--upgrade', + '--upgrade-deps', '--prompt'): + self.assertNotRegex(data, rf'command = .* {opt}') + elif os.name=='nt' and attr=='symlinks': + pass + else: + self.assertRegex(data, rf'command = .* {opt}') + def test_prompt(self): env_name = os.path.split(self.env_dir)[1] @@ -127,7 +209,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertEqual(context.prompt, env_name) self.assertNotIn("prompt = ", data) rmtree(self.env_dir) @@ -135,7 +217,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(My prompt) ') + self.assertEqual(context.prompt, 'My prompt') self.assertIn("prompt = 'My prompt'\n", data) rmtree(self.env_dir) @@ -144,13 +226,19 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertEqual(context.prompt, cwd) self.assertIn("prompt = '%s'\n" % cwd, data) def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -158,7 +246,7 @@ def test_upgrade_dependencies(self): if sys.platform == 'win32': expect_exe = os.path.normcase(os.path.realpath(expect_exe)) - def pip_cmd_checker(cmd): + def pip_cmd_checker(cmd, **kwargs): cmd[0] = os.path.normcase(cmd[0]) self.assertEqual( cmd, @@ -169,12 +257,11 @@ def pip_cmd_checker(cmd): 'install', '--upgrade', 'pip', - 'setuptools' ] ) fake_context = builder.ensure_directories(fake_env_dir) - with patch('venv.subprocess.check_call', pip_cmd_checker): + with patch('venv.subprocess.check_output', pip_cmd_checker): builder.upgrade_dependencies(fake_context) @requireVenvCreate @@ -185,8 +272,7 @@ def test_prefixes(self): # check a venv's prefixes rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for prefix, expected in ( ('prefix', self.env_dir), ('exec_prefix', self.env_dir), @@ -194,7 +280,76 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) + + @requireVenvCreate + def test_sysconfig(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=False) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + + @requireVenvCreate + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_sysconfig_symlinks(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=True) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + # HACK: Test fails on POSIX with unversioned binary (PR gh-113033) + #('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( @@ -259,6 +414,8 @@ def test_unoverwritable_fails(self): self.assertRaises((ValueError, OSError), venv.create, self.env_dir) self.clear_directory(self.env_dir) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_upgrade(self): """ Test upgrading an existing environment directory. @@ -321,8 +478,7 @@ def test_executable(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) @@ -335,12 +491,89 @@ def test_executable_symlinks(self): rmtree(self.env_dir) builder = venv.EnvBuilder(clear=True, symlinks=True) builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ @@ -351,13 +584,27 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') - envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ @@ -370,15 +617,25 @@ def test_multiprocessing(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'from multiprocessing import Pool; ' 'pool = Pool(1); ' 'print(pool.apply_async("Python".lower).get(3)); ' 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) + @requireVenvCreate + def test_multiprocessing_recursion(self): + """ + Test that the multiprocessing is able to spawn itself + """ + skip_if_broken_multiprocessing_synchronize() + + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): bash = shutil.which("bash") @@ -404,19 +661,250 @@ def test_macos_env(self): builder = venv.EnvBuilder() builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) self.assertEqual(out.strip(), 'False'.encode()) + def test_pathsep_error(self): + """ + Test that venv creation fails when the target directory contains + the path separator. + """ + rmtree(self.env_dir) + bad_itempath = self.env_dir + os.pathsep + self.assertRaises(ValueError, venv.create, bad_itempath) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) + + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + @requireVenvCreate + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_zippath_from_non_installed_posix(self): + """ + Test that when create venv from non-installed python, the zip path + value is as expected. + """ + rmtree(self.env_dir) + # First try to create a non-installed python. It's not a real full + # functional non-installed python, but enough for this test. + platlibdir = sys.platlibdir + non_installed_dir = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(rmtree, non_installed_dir) + bindir = os.path.join(non_installed_dir, self.bindir) + os.mkdir(bindir) + shutil.copy2(sys.executable, bindir) + libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) + os.makedirs(libdir) + landmark = os.path.join(libdir, "os.py") + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" + zip_landmark = os.path.join(non_installed_dir, + platlibdir, + stdlib_zip) + additional_pythonpath_for_non_installed = [] + + # Copy stdlib files to the non-installed python so venv can + # correctly calculate the prefix. + for eachpath in sys.path: + if eachpath.endswith(".zip"): + if os.path.isfile(eachpath): + shutil.copyfile( + eachpath, + os.path.join(non_installed_dir, platlibdir)) + elif os.path.isfile(os.path.join(eachpath, "os.py")): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue + if name == "site-packages": + continue + fn = os.path.join(eachpath, name) + if os.path.isfile(fn): + shutil.copy(fn, libdir) + elif os.path.isdir(fn): + shutil.copytree(fn, os.path.join(libdir, name), + ignore=copy_python_src_ignore) + else: + additional_pythonpath_for_non_installed.append( + eachpath) + cmd = [os.path.join(non_installed_dir, self.bindir, self.exe), + "-m", + "venv", + "--without-pip", + "--without-scm-ignore-files", + self.env_dir] + # Our fake non-installed python is not fully functional because + # it cannot find the extensions. Set PYTHONPATH so it can run the + # venv module correctly. + pythonpath = os.pathsep.join( + additional_pythonpath_for_non_installed) + # For python built with shared enabled. We need to set + # LD_LIBRARY_PATH so the non-installed python can find and link + # libpython.so + ld_library_path = sysconfig.get_config_var("LIBDIR") + if not ld_library_path or sysconfig.is_python_build(): + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) + if sys.platform == 'darwin': + ld_library_path_env = "DYLD_LIBRARY_PATH" + else: + ld_library_path_env = "LD_LIBRARY_PATH" + child_env = { + "PYTHONPATH": pythonpath, + ld_library_path_env: ld_library_path, + } + if asan_options := os.environ.get("ASAN_OPTIONS"): + # prevent https://github.com/python/cpython/issues/104839 + child_env["ASAN_OPTIONS"] = asan_options + subprocess.check_call(cmd, env=child_env) + # Now check the venv created from the non-installed python has + # correct zip path in pythonpath. + cmd = [self.envpy(), '-S', '-c', 'import sys; print(sys.path)'] + out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + + @requireVenvCreate + def test_activate_shell_script_has_no_dos_newlines(self): + """ + Test that the `activate` shell script contains no CR LF. + This is relevant for Cygwin, as the Windows build might have + converted line endings accidentally. + """ + venv_dir = pathlib.Path(self.env_dir) + rmtree(venv_dir) + [[scripts_dir], *_] = self.ENV_SUBDIRS + script_path = venv_dir / scripts_dir / "activate" + venv.create(venv_dir) + with open(script_path, 'rb') as script: + for i, line in enumerate(script, 1): + error_message = f"CR LF found in line {i}" + self.assertFalse(line.endswith(b'\r\n'), error_message) + + @requireVenvCreate + def test_scm_ignore_files_git(self): + """ + Test that a .gitignore file is created when "git" is specified. + The file should contain a `*\n` line. + """ + self.run_with_capture(venv.create, self.env_dir, + scm_ignore_files={'git'}) + file_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', file_lines) + + @requireVenvCreate + def test_create_scm_ignore_files_multiple(self): + """ + Test that ``scm_ignore_files`` can work with multiple SCMs. + """ + bzrignore_name = ".bzrignore" + contents = "# For Bazaar.\n*\n" + + class BzrEnvBuilder(venv.EnvBuilder): + def create_bzr_ignore_file(self, context): + gitignore_path = os.path.join(context.env_dir, bzrignore_name) + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write(contents) + + builder = BzrEnvBuilder(scm_ignore_files={'git', 'bzr'}) + self.run_with_capture(builder.create, self.env_dir) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + bzrignore = self.get_text_file_contents(bzrignore_name) + self.assertEqual(bzrignore, contents) + + @requireVenvCreate + def test_create_scm_ignore_files_empty(self): + """ + Test that no default ignore files are created when ``scm_ignore_files`` + is empty. + """ + # scm_ignore_files is set to frozenset() by default. + self.run_with_capture(venv.create, self.env_dir) + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + self.assertIn("--without-scm-ignore-files", + self.get_text_file_contents('pyvenv.cfg')) + + @requireVenvCreate + def test_cli_with_scm_ignore_files(self): + """ + Test that default SCM ignore files are created by default via the CLI. + """ + self.run_with_capture(venv.main, ['--without-pip', self.env_dir]) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + @requireVenvCreate + def test_cli_without_scm_ignore_files(self): + """ + Test that ``--without-scm-ignore-files`` doesn't create SCM ignore files. + """ + args = ['--without-pip', '--without-scm-ignore-files', self.env_dir] + self.run_with_capture(venv.main, args) + + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + + # gh-126084: venvwlauncher should run pythonw, not python + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_venvwlauncher(self): + """ + Test that the GUI launcher runs the GUI python. + """ + rmtree(self.env_dir) + venv.create(self.env_dir) + exename = self.exe + # Retain the debug suffix if present + if "python" in exename and not "pythonw" in exename: + exename = exename.replace("python", "pythonw") + envpyw = os.path.join(self.env_dir, self.bindir, exename) + try: + subprocess.check_call([envpyw, "-c", "import sys; " + "assert sys._base_executable.endswith('%s')" % exename]) + except subprocess.CalledProcessError: + self.fail("venvwlauncher.exe did not run %s" % exename) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'try:\n import pip\nexcept ImportError:\n print("OK")']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results @@ -478,20 +966,14 @@ def do_test_with_pip(self, system_site_packages): # Actually run the create command with all that unhelpful # config in place to ensure we ignore it - try: + with self.nicer_error(): self.run_with_capture(venv.create, self.env_dir, system_site_packages=system_site_packages, with_pip=True) - except subprocess.CalledProcessError as exc: - # The output this produces can be a little hard to read, - # but at least it has all the details - details = exc.output.decode(errors="replace") - msg = "{}\n\n**Subprocess Output**\n{}" - self.fail(msg.format(exc, details)) # Ensure pip is available in the virtual environment - envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) # Ignore DeprecationWarning since pip code is not part of Python - out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', '-W', 'ignore::ImportWarning', '-I', '-m', 'pip', '--version']) # We force everything to text, so unittest gives the detailed diff @@ -508,13 +990,14 @@ def do_test_with_pip(self, system_site_packages): # Check the private uninstall command provided for the Windows # installers works (at least in a virtual environment) with EnvironmentVarGuard() as envvars: - # It seems ensurepip._uninstall calls subprocesses which do not - # inherit the interpreter settings. - envvars["PYTHONWARNINGS"] = "ignore" - out, err = check_output([envpy, - '-W', 'ignore::DeprecationWarning', - '-W', 'ignore::ImportWarning', '-I', - '-m', 'ensurepip._uninstall']) + with self.nicer_error(): + # It seems ensurepip._uninstall calls subprocesses which do not + # inherit the interpreter settings. + envvars["PYTHONWARNINGS"] = "ignore" + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', + '-W', 'ignore::ImportWarning', '-I', + '-m', 'ensurepip._uninstall']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results err = err.decode("latin-1") # Force to text, prevent decoding errors @@ -527,25 +1010,51 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in # future pip versions, this test can likely be relaxed further. out = out.decode("latin-1") # Force to text, prevent decoding errors self.assertIn("Successfully uninstalled pip", out) - self.assertIn("Successfully uninstalled setuptools", out) # Check pip is now gone from the virtual environment. This only # applies in the system_site_packages=False case, because in the # other case, pip may still be available in the system site-packages if not system_site_packages: self.assert_pip_not_installed() - # Issue #26610: pip/pep425tags.py requires ctypes - @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @contextlib.contextmanager + def nicer_error(self): + """ + Capture output from a failed subprocess for easier debugging. + + The output this handler produces can be a little hard to read, + but at least it has all the details. + """ + try: + yield + except subprocess.CalledProcessError as exc: + out = (exc.output or b'').decode(errors="replace") + err = (exc.stderr or b'').decode(errors="replace") + self.fail( + f"{exc}\n\n" + f"**Subprocess Output**\n{out}\n\n" + f"**Subprocess Error**\n{err}" + ) + + @requires_venv_with_pip() + @requires_resource('cpu') def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 123d51b77ea..53ac0363a3c 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -24,10 +24,13 @@ from warnings import deprecated -py_warnings = import_helper.import_fresh_module('warnings', - blocked=['_warnings']) -c_warnings = import_helper.import_fresh_module('warnings', - fresh=['_warnings']) +py_warnings = import_helper.import_fresh_module('_py_warnings') +py_warnings._set_module(py_warnings) + +c_warnings = import_helper.import_fresh_module( + "warnings", fresh=["_warnings", "_py_warnings"] +) +c_warnings._set_module(c_warnings) @contextmanager def warnings_state(module): @@ -43,15 +46,21 @@ def warnings_state(module): except NameError: pass original_warnings = warning_tests.warnings - original_filters = module.filters - try: + if module._use_context: + saved_context, context = module._new_context() + else: + original_filters = module.filters module.filters = original_filters[:] + try: module.simplefilter("once") warning_tests.warnings = module yield finally: warning_tests.warnings = original_warnings - module.filters = original_filters + if module._use_context: + module._set_context(saved_context) + else: + module.filters = original_filters class TestWarning(Warning): @@ -93,7 +102,7 @@ class PublicAPITests(BaseTest): """ def test_module_all_attribute(self): - self.assertTrue(hasattr(self.module, '__all__')) + self.assertHasAttr(self.module, '__all__') target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings", "deprecated"] @@ -111,14 +120,14 @@ class FilterTests(BaseTest): """Testing the filtering functionality.""" def test_error(self): - with original_warnings.catch_warnings(module=self.module) as w: + with self.module.catch_warnings() as w: self.module.resetwarnings() self.module.filterwarnings("error", category=UserWarning) self.assertRaises(UserWarning, self.module.warn, "FilterTests.test_error") def test_error_after_default(self): - with original_warnings.catch_warnings(module=self.module) as w: + with self.module.catch_warnings() as w: self.module.resetwarnings() message = "FilterTests.test_ignore_after_default" def f(): @@ -136,8 +145,7 @@ def f(): self.assertRaises(UserWarning, f) def test_ignore(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("ignore", category=UserWarning) self.module.warn("FilterTests.test_ignore", UserWarning) @@ -145,8 +153,7 @@ def test_ignore(self): self.assertEqual(list(__warningregistry__), ['version']) def test_ignore_after_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() message = "FilterTests.test_ignore_after_default" def f(): @@ -157,44 +164,43 @@ def f(): f() self.assertEqual(len(w), 1) - def test_always(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - self.module.filterwarnings("always", category=UserWarning) - message = "FilterTests.test_always" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all(self): + for mode in {"always", "all"}: + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings(mode, category=UserWarning) + message = "FilterTests.test_always_and_all" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) - def test_always_after_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - message = "FilterTests.test_always_after_ignore" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 1) - self.module.filterwarnings("always", category=UserWarning) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 3) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all_after_default(self): + for mode in {"always", "all"}: + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_and_all_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings(mode, category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) def test_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("default", category=UserWarning) message = UserWarning("FilterTests.test_default") @@ -209,8 +215,7 @@ def test_default(self): raise ValueError("loop variant unhandled") def test_module(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("module", category=UserWarning) message = UserWarning("FilterTests.test_module") @@ -221,8 +226,7 @@ def test_module(self): self.assertEqual(len(w), 0) def test_once(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("once", category=UserWarning) message = UserWarning("FilterTests.test_once") @@ -237,9 +241,87 @@ def test_once(self): 42) self.assertEqual(len(w), 0) + def test_filter_module(self): + MS_WINDOWS = (sys.platform == 'win32') + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'package\.module\z') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module='package') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='other.package.module') + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/otherpackage/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/module\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 2) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/PATH/TO/PACKAGE/MODULE', 42) + if MS_WINDOWS: + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module/__init__.py', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'\path\to\package\module', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/__init__\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__.py', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__', 42) + self.assertEqual(len(w), 2) + + if MS_WINDOWS: + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'C:\\path\\to\\package\\module\z') + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.py', 42) + self.assertEqual(len(w), 2) + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\PATH\TO\PACKAGE\MODULE', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__init__.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'<unknown>\z') + self.module.warn_explicit('msg', UserWarning, '', 42) + self.assertEqual(len(w), 1) + def test_module_globals(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter("always", UserWarning) # bpo-33509: module_globals=None must not crash @@ -259,15 +341,14 @@ def test_module_globals(self): self.assertEqual(len(w), 2) def test_inheritance(self): - with original_warnings.catch_warnings(module=self.module) as w: + with self.module.catch_warnings() as w: self.module.resetwarnings() self.module.filterwarnings("error", category=Warning) self.assertRaises(UserWarning, self.module.warn, "FilterTests.test_inheritance", UserWarning) def test_ordering(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("ignore", category=UserWarning) self.module.filterwarnings("error", category=UserWarning, @@ -282,8 +363,7 @@ def test_ordering(self): def test_filterwarnings(self): # Test filterwarnings(). # Implicitly also tests resetwarnings(). - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.filterwarnings("error", "", Warning, "", 0) self.assertRaises(UserWarning, self.module.warn, 'convert to error') @@ -307,8 +387,7 @@ def test_filterwarnings(self): self.assertIs(w[-1].category, UserWarning) def test_message_matching(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter("ignore", UserWarning) self.module.filterwarnings("error", "match", UserWarning) self.assertRaises(UserWarning, self.module.warn, "match") @@ -324,54 +403,52 @@ def match(self, a): L[:] = [] L = [("default",X(),UserWarning,X(),0) for i in range(2)] - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.filters = L self.module.warn_explicit(UserWarning("b"), None, "f.py", 42) self.assertEqual(str(w[-1].message), "b") def test_filterwarnings_duplicate_filters(self): - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.resetwarnings() self.module.filterwarnings("error", category=UserWarning) - self.assertEqual(len(self.module.filters), 1) + self.assertEqual(len(self.module._get_filters()), 1) self.module.filterwarnings("ignore", category=UserWarning) self.module.filterwarnings("error", category=UserWarning) self.assertEqual( - len(self.module.filters), 2, + len(self.module._get_filters()), 2, "filterwarnings inserted duplicate filter" ) self.assertEqual( - self.module.filters[0][0], "error", + self.module._get_filters()[0][0], "error", "filterwarnings did not promote filter to " "the beginning of list" ) def test_simplefilter_duplicate_filters(self): - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.resetwarnings() self.module.simplefilter("error", category=UserWarning) - self.assertEqual(len(self.module.filters), 1) + self.assertEqual(len(self.module._get_filters()), 1) self.module.simplefilter("ignore", category=UserWarning) self.module.simplefilter("error", category=UserWarning) self.assertEqual( - len(self.module.filters), 2, + len(self.module._get_filters()), 2, "simplefilter inserted duplicate filter" ) self.assertEqual( - self.module.filters[0][0], "error", + self.module._get_filters()[0][0], "error", "simplefilter did not promote filter to the beginning of list" ) def test_append_duplicate(self): - with original_warnings.catch_warnings(module=self.module, - record=True) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.simplefilter("ignore") self.module.simplefilter("error", append=True) self.module.simplefilter("ignore", append=True) self.module.warn("test_append_duplicate", category=UserWarning) - self.assertEqual(len(self.module.filters), 2, + self.assertEqual(len(self.module._get_filters()), 2, "simplefilter inserted duplicate filter" ) self.assertEqual(len(w), 0, @@ -401,19 +478,17 @@ def test_argument_validation(self): self.module.simplefilter('ignore', lineno=-1) def test_catchwarnings_with_simplefilter_ignore(self): - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(module=self.module): self.module.resetwarnings() self.module.simplefilter("error") - with self.module.catch_warnings( - module=self.module, action="ignore" - ): + with self.module.catch_warnings(action="ignore"): self.module.warn("This will be ignored") def test_catchwarnings_with_simplefilter_error(self): - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.resetwarnings() with self.module.catch_warnings( - module=self.module, action="error", category=FutureWarning + action="error", category=FutureWarning ): with support.captured_stderr() as stderr: error_msg = "Other types of warnings are not errors" @@ -435,8 +510,7 @@ class WarnTests(BaseTest): """Test warnings.warn() and warnings.warn_explicit().""" def test_message(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter("once") for i in range(4): text = 'multi %d' %i # Different text on each call. @@ -448,8 +522,7 @@ def test_message(self): def test_warn_nonstandard_types(self): # warn() should handle non-standard types without issue. for ob in (Warning, None, 42): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter("once") self.module.warn(ob) # Don't directly compare objects since @@ -458,8 +531,7 @@ def test_warn_nonstandard_types(self): def test_filename(self): with warnings_state(self.module): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: warning_tests.inner("spam1") self.assertEqual(os.path.basename(w[-1].filename), "stacklevel.py") @@ -471,8 +543,7 @@ def test_stacklevel(self): # Test stacklevel argument # make sure all messages are different, so the warning won't be skipped with warnings_state(self.module): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: warning_tests.inner("spam3", stacklevel=1) self.assertEqual(os.path.basename(w[-1].filename), "stacklevel.py") @@ -494,23 +565,20 @@ def test_stacklevel(self): self.assertEqual(os.path.basename(w[-1].filename), "<sys>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + /Users/al03219714/Projects/RustPython1/crates/pylib/Lib/test/test_warnings/__init__.py def test_stacklevel_import(self): # Issue #24305: With stacklevel=2, module-level warnings should work. import_helper.unload('test.test_warnings.data.import_warning') with warnings_state(self.module): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter('always') - import test.test_warnings.data.import_warning + import test.test_warnings.data.import_warning # noqa: F401 self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, __file__) def test_skip_file_prefixes(self): with warnings_state(self.module): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter('always') # Warning never attributed to the data/ package. @@ -533,6 +601,16 @@ def test_skip_file_prefixes(self): warning_tests.package("prefix02", stacklevel=3) self.assertIn("unittest", w[-1].filename) + def test_skip_file_prefixes_file_path(self): + # see: gh-126209 + with warnings_state(self.module): + skipped = warning_tests.__file__ + with self.module.catch_warnings(record=True) as w: + warning_tests.outer("msg", skip_file_prefixes=(skipped,)) + + self.assertEqual(len(w), 1) + self.assertNotEqual(w[-1].filename, skipped) + def test_skip_file_prefixes_type_errors(self): with warnings_state(self.module): warn = warning_tests.warnings.warn @@ -548,23 +626,16 @@ def test_exec_filename(self): codeobj = compile(("import warnings\n" "warnings.warn('hello', UserWarning)"), filename, "exec") - with original_warnings.catch_warnings(record=True) as w: + with self.module.catch_warnings(record=True) as w: self.module.simplefilter("always", category=UserWarning) exec(codeobj) self.assertEqual(w[0].filename, filename) def test_warn_explicit_non_ascii_filename(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - filenames = ["nonascii\xe9\u20ac"] - if not support.is_emscripten: - # JavaScript does not like surrogates. - # Invalid UTF-8 leading byte 0x80 encountered when - # deserializing a UTF-8 string in wasm memory to a JS - # string! - filenames.append("surrogate\udc80") + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] for filename in filenames: try: os.fsencode(filename) @@ -625,7 +696,7 @@ class NonWarningSubclass: self.assertIn('category must be a Warning subclass, not ', str(cm.exception)) - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.resetwarnings() self.module.filterwarnings('default') with self.assertWarns(MyWarningClass) as cm: @@ -641,7 +712,7 @@ class NonWarningSubclass: self.assertIsInstance(cm.warning, Warning) def check_module_globals(self, module_globals): - with original_warnings.catch_warnings(module=self.module, record=True) as w: + with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('default') self.module.warn_explicit( 'eggs', UserWarning, 'bar', 1, @@ -654,7 +725,7 @@ def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError) if self.module is py_warnings: self.check_module_globals(module_globals) return - with original_warnings.catch_warnings(module=self.module, record=True) as w: + with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') with self.assertRaisesRegex(errtype, re.escape(errmsg)): self.module.warn_explicit( @@ -666,7 +737,7 @@ def check_module_globals_deprecated(self, module_globals, msg): if self.module is py_warnings: self.check_module_globals(module_globals) return - with original_warnings.catch_warnings(module=self.module, record=True) as w: + with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') self.module.warn_explicit( 'eggs', UserWarning, 'bar', 1, @@ -734,54 +805,44 @@ def test_gh86298_no_loader_with_spec_loader_okay(self): class CWarnTests(WarnTests, unittest.TestCase): module = c_warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure # As an early adopter, we sanity check the # test.import_helper.import_fresh_module utility function def test_accelerated(self): self.assertIsNot(original_warnings, self.module) - self.assertFalse(hasattr(self.module.warn, '__code__')) + self.assertNotHasAttr(self.module.warn, '__code__') - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gh86298_no_loader_and_spec_is_none(self): - return super().test_gh86298_no_loader_and_spec_is_none() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gh86298_loader_is_none_and_spec_is_none(self): - return super().test_gh86298_loader_is_none_and_spec_is_none() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_loader_and_spec_loader_disagree(self): + return super().test_gh86298_loader_and_spec_loader_disagree() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gh86298_loader_is_none_and_spec_loader_is_none(self): - return super().test_gh86298_loader_is_none_and_spec_loader_is_none() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 def test_gh86298_no_spec(self): return super().test_gh86298_no_spec() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gh86298_spec_is_none(self): - return super().test_gh86298_spec_is_none() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 def test_gh86298_no_spec_loader(self): return super().test_gh86298_no_spec_loader() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gh86298_loader_and_spec_loader_disagree(self): - return super().test_gh86298_loader_and_spec_loader_disagree() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_spec_is_none(self): + return super().test_gh86298_spec_is_none() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised def test_gh86298_no_loader_and_no_spec_loader(self): return super().test_gh86298_no_loader_and_no_spec_loader() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_loader_is_none_and_spec_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_is_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_loader_is_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_no_loader_and_spec_is_none(self): + return super().test_gh86298_no_loader_and_spec_is_none() + class PyWarnTests(WarnTests, unittest.TestCase): module = py_warnings @@ -789,7 +850,7 @@ class PyWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_pure_python(self): self.assertIsNot(original_warnings, self.module) - self.assertTrue(hasattr(self.module.warn, '__code__')) + self.assertHasAttr(self.module.warn, '__code__') class WCmdLineTests(BaseTest): @@ -797,7 +858,7 @@ class WCmdLineTests(BaseTest): def test_improper_input(self): # Uses the private _setoption() function to test the parsing # of command-line warning arguments - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.assertRaises(self.module._OptionError, self.module._setoption, '1:2:3:4:5:6') self.assertRaises(self.module._OptionError, @@ -816,7 +877,7 @@ def test_improper_input(self): self.assertRaises(UserWarning, self.module.warn, 'convert to error') def test_import_from_module(self): - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module._setoption('ignore::Warning') with self.assertRaises(self.module._OptionError): self.module._setoption('ignore::TestWarning') @@ -834,8 +895,6 @@ class CWCmdLineTests(WCmdLineTests, unittest.TestCase): class PyWCmdLineTests(WCmdLineTests, unittest.TestCase): module = py_warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_improper_option(self): # Same as above, but check that the message is printed out when # the interpreter is executed. This also checks that options are @@ -843,8 +902,6 @@ def test_improper_option(self): rc, out, err = assert_python_ok("-Wxxx", "-c", "pass") self.assertIn(b"Invalid -W option ignored: invalid action: 'xxx'", err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warnings_bootstrap(self): # Check that the warnings module does get loaded when -W<some option> # is used (see issue #10372 for an example of silent bootstrap failure). @@ -861,11 +918,10 @@ class _WarningsTests(BaseTest, unittest.TestCase): module = c_warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UserWarning not raised by warn def test_filter(self): # Everything should function even if 'filters' is not in warnings. - with original_warnings.catch_warnings(module=self.module) as w: + with self.module.catch_warnings() as w: self.module.filterwarnings("error", "", Warning, "", 0) self.assertRaises(UserWarning, self.module.warn, 'convert to error') @@ -873,8 +929,6 @@ def test_filter(self): self.assertRaises(UserWarning, self.module.warn, 'convert to error') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_onceregistry(self): # Replacing or removing the onceregistry should be okay. global __warningregistry__ @@ -882,8 +936,7 @@ def test_onceregistry(self): try: original_registry = self.module.onceregistry __warningregistry__ = {} - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("once", category=UserWarning) self.module.warn_explicit(message, UserWarning, "file", 42) @@ -905,15 +958,12 @@ def test_onceregistry(self): finally: self.module.onceregistry = original_registry - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_default_action(self): # Replacing or removing defaultaction should be okay. message = UserWarning("defaultaction test") original = self.module.defaultaction try: - with original_warnings.catch_warnings(record=True, - module=self.module) as w: + with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() registry = {} self.module.warn_explicit(message, UserWarning, "<test>", 42, @@ -946,8 +996,12 @@ def test_default_action(self): def test_showwarning_missing(self): # Test that showwarning() missing is okay. + if self.module._use_context: + # If _use_context is true, the warnings module does not + # override/restore showwarning() + return text = 'del showwarning test' - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.filterwarnings("always", category=UserWarning) del self.module.showwarning with support.captured_output('stderr') as stream: @@ -955,12 +1009,10 @@ def test_showwarning_missing(self): result = stream.getvalue() self.assertIn(text, result) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_showwarnmsg_missing(self): # Test that _showwarnmsg() missing is okay. text = 'del _showwarnmsg test' - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.filterwarnings("always", category=UserWarning) show = self.module._showwarnmsg @@ -974,50 +1026,55 @@ def test_showwarnmsg_missing(self): self.assertIn(text, result) def test_showwarning_not_callable(self): - with original_warnings.catch_warnings(module=self.module): - self.module.filterwarnings("always", category=UserWarning) - self.module.showwarning = print - with support.captured_output('stdout'): - self.module.warn('Warning!') - self.module.showwarning = 23 - self.assertRaises(TypeError, self.module.warn, "Warning!") + orig = self.module.showwarning + try: + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + self.module.showwarning = print + with support.captured_output('stdout'): + self.module.warn('Warning!') + self.module.showwarning = 23 + self.assertRaises(TypeError, self.module.warn, "Warning!") + finally: + self.module.showwarning = orig def test_show_warning_output(self): # With showwarning() missing, make sure that output is okay. - text = 'test show_warning' - with original_warnings.catch_warnings(module=self.module): - self.module.filterwarnings("always", category=UserWarning) - del self.module.showwarning - with support.captured_output('stderr') as stream: - warning_tests.inner(text) - result = stream.getvalue() - self.assertEqual(result.count('\n'), 2, - "Too many newlines in %r" % result) - first_line, second_line = result.split('\n', 1) - expected_file = os.path.splitext(warning_tests.__file__)[0] + '.py' - first_line_parts = first_line.rsplit(':', 3) - path, line, warning_class, message = first_line_parts - line = int(line) - self.assertEqual(expected_file, path) - self.assertEqual(warning_class, ' ' + UserWarning.__name__) - self.assertEqual(message, ' ' + text) - expected_line = ' ' + linecache.getline(path, line).strip() + '\n' - assert expected_line - self.assertEqual(second_line, expected_line) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + orig = self.module.showwarning + try: + text = 'test show_warning' + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + del self.module.showwarning + with support.captured_output('stderr') as stream: + warning_tests.inner(text) + result = stream.getvalue() + self.assertEqual(result.count('\n'), 2, + "Too many newlines in %r" % result) + first_line, second_line = result.split('\n', 1) + expected_file = os.path.splitext(warning_tests.__file__)[0] + '.py' + first_line_parts = first_line.rsplit(':', 3) + path, line, warning_class, message = first_line_parts + line = int(line) + self.assertEqual(expected_file, path) + self.assertEqual(warning_class, ' ' + UserWarning.__name__) + self.assertEqual(message, ' ' + text) + expected_line = ' ' + linecache.getline(path, line).strip() + '\n' + assert expected_line + self.assertEqual(second_line, expected_line) + finally: + self.module.showwarning = orig + def test_filename_none(self): # issue #12467: race condition if a warning is emitted at shutdown globals_dict = globals() oldfile = globals_dict['__file__'] try: - catch = original_warnings.catch_warnings(record=True, - module=self.module) + catch = self.module.catch_warnings(record=True) with catch as w: self.module.filterwarnings("always", category=UserWarning) globals_dict['__file__'] = None - original_warnings.warn('test', UserWarning) + self.module.warn('test', UserWarning) self.assertTrue(len(w)) finally: globals_dict['__file__'] = oldfile @@ -1031,8 +1088,7 @@ def test_stderr_none(self): self.assertNotIn(b'Warning!', stderr) self.assertNotIn(b'Error', stderr) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'int' object is not iterable def test_issue31285(self): # warn_explicit() should neither raise a SystemError nor cause an # assertion failure, in case the return value of get_source() has a @@ -1056,7 +1112,7 @@ def get_source(self, fullname): wmod = self.module - with original_warnings.catch_warnings(module=wmod): + with wmod.catch_warnings(): wmod.filterwarnings('default', category=UserWarning) linecache.clearcache() @@ -1083,7 +1139,7 @@ def test_issue31411(self): # warn_explicit() shouldn't raise a SystemError in case # warnings.onceregistry isn't a dictionary. wmod = self.module - with original_warnings.catch_warnings(module=wmod): + with wmod.catch_warnings(): wmod.filterwarnings('once') with support.swap_attr(wmod, 'onceregistry', None): with self.assertRaises(TypeError): @@ -1094,12 +1150,12 @@ def test_issue31416(self): # warn_explicit() shouldn't cause an assertion failure in case of a # bad warnings.filters or warnings.defaultaction. wmod = self.module - with original_warnings.catch_warnings(module=wmod): - wmod.filters = [(None, None, Warning, None, 0)] + with wmod.catch_warnings(): + wmod._get_filters()[:] = [(None, None, Warning, None, 0)] with self.assertRaises(TypeError): wmod.warn_explicit('foo', Warning, 'bar', 1) - wmod.filters = [] + wmod._get_filters()[:] = [] with support.swap_attr(wmod, 'defaultaction', None), \ self.assertRaises(TypeError): wmod.warn_explicit('foo', Warning, 'bar', 1) @@ -1108,7 +1164,7 @@ def test_issue31416(self): def test_issue31566(self): # warn() shouldn't cause an assertion failure in case of a bad # __name__ global. - with original_warnings.catch_warnings(module=self.module): + with self.module.catch_warnings(): self.module.filterwarnings('error', category=UserWarning) with support.swap_item(globals(), '__name__', b'foo'), \ support.swap_item(globals(), '__file__', None): @@ -1185,8 +1241,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ResourceWarning: Enable tracemalloc to get the object allocation traceback def test_tracemalloc(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) @@ -1238,16 +1293,18 @@ class CatchWarningTests(BaseTest): """Test catch_warnings().""" def test_catch_warnings_restore(self): + if self.module._use_context: + return # test disabled if using context vars wmod = self.module orig_filters = wmod.filters orig_showwarning = wmod.showwarning # Ensure both showwarning and filters are restored when recording - with wmod.catch_warnings(module=wmod, record=True): + with wmod.catch_warnings(record=True): wmod.filters = wmod.showwarning = object() self.assertIs(wmod.filters, orig_filters) self.assertIs(wmod.showwarning, orig_showwarning) # Same test, but with recording disabled - with wmod.catch_warnings(module=wmod, record=False): + with wmod.catch_warnings(record=False): wmod.filters = wmod.showwarning = object() self.assertIs(wmod.filters, orig_filters) self.assertIs(wmod.showwarning, orig_showwarning) @@ -1255,7 +1312,7 @@ def test_catch_warnings_restore(self): def test_catch_warnings_recording(self): wmod = self.module # Ensure warnings are recorded when requested - with wmod.catch_warnings(module=wmod, record=True) as w: + with wmod.catch_warnings(record=True) as w: self.assertEqual(w, []) self.assertIs(type(w), list) wmod.simplefilter("always") @@ -1269,44 +1326,48 @@ def test_catch_warnings_recording(self): self.assertEqual(w, []) # Ensure warnings are not recorded when not requested orig_showwarning = wmod.showwarning - with wmod.catch_warnings(module=wmod, record=False) as w: + with wmod.catch_warnings(record=False) as w: self.assertIsNone(w) self.assertIs(wmod.showwarning, orig_showwarning) def test_catch_warnings_reentry_guard(self): wmod = self.module # Ensure catch_warnings is protected against incorrect usage - x = wmod.catch_warnings(module=wmod, record=True) + x = wmod.catch_warnings(record=True) self.assertRaises(RuntimeError, x.__exit__) with x: self.assertRaises(RuntimeError, x.__enter__) # Same test, but with recording disabled - x = wmod.catch_warnings(module=wmod, record=False) + x = wmod.catch_warnings(record=False) self.assertRaises(RuntimeError, x.__exit__) with x: self.assertRaises(RuntimeError, x.__enter__) def test_catch_warnings_defaults(self): wmod = self.module - orig_filters = wmod.filters + orig_filters = wmod._get_filters() orig_showwarning = wmod.showwarning # Ensure default behaviour is not to record warnings - with wmod.catch_warnings(module=wmod) as w: + with wmod.catch_warnings() as w: self.assertIsNone(w) self.assertIs(wmod.showwarning, orig_showwarning) - self.assertIsNot(wmod.filters, orig_filters) - self.assertIs(wmod.filters, orig_filters) + self.assertIsNot(wmod._get_filters(), orig_filters) + self.assertIs(wmod._get_filters(), orig_filters) if wmod is sys.modules['warnings']: # Ensure the default module is this one with wmod.catch_warnings() as w: self.assertIsNone(w) self.assertIs(wmod.showwarning, orig_showwarning) - self.assertIsNot(wmod.filters, orig_filters) - self.assertIs(wmod.filters, orig_filters) + self.assertIsNot(wmod._get_filters(), orig_filters) + self.assertIs(wmod._get_filters(), orig_filters) def test_record_override_showwarning_before(self): # Issue #28835: If warnings.showwarning() was overridden, make sure # that catch_warnings(record=True) overrides it again. + if self.module._use_context: + # If _use_context is true, the warnings module does not restore + # showwarning() + return text = "This is a warning" wmod = self.module my_log = [] @@ -1317,7 +1378,7 @@ def my_logger(message, category, filename, lineno, file=None, line=None): # Override warnings.showwarning() before calling catch_warnings() with support.swap_attr(wmod, 'showwarning', my_logger): - with wmod.catch_warnings(module=wmod, record=True) as log: + with wmod.catch_warnings(record=True) as log: self.assertIsNot(wmod.showwarning, my_logger) wmod.simplefilter("always") @@ -1332,6 +1393,10 @@ def my_logger(message, category, filename, lineno, file=None, line=None): def test_record_override_showwarning_inside(self): # Issue #28835: It is possible to override warnings.showwarning() # in the catch_warnings(record=True) context manager. + if self.module._use_context: + # If _use_context is true, the warnings module does not restore + # showwarning() + return text = "This is a warning" wmod = self.module my_log = [] @@ -1340,7 +1405,7 @@ def my_logger(message, category, filename, lineno, file=None, line=None): nonlocal my_log my_log.append(message) - with wmod.catch_warnings(module=wmod, record=True) as log: + with wmod.catch_warnings(record=True) as log: wmod.simplefilter("always") wmod.showwarning = my_logger wmod.warn(text) @@ -1389,8 +1454,6 @@ class PyCatchWarningTests(CatchWarningTests, unittest.TestCase): class EnvironmentVariableTests(BaseTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_single_warning(self): rc, stdout, stderr = assert_python_ok("-c", "import sys; sys.stdout.write(str(sys.warnoptions))", @@ -1398,8 +1461,6 @@ def test_single_warning(self): PYTHONDEVMODE="") self.assertEqual(stdout, b"['ignore::DeprecationWarning']") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_comma_separated_warnings(self): rc, stdout, stderr = assert_python_ok("-c", "import sys; sys.stdout.write(str(sys.warnoptions))", @@ -1408,8 +1469,6 @@ def test_comma_separated_warnings(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") - # TODO: RUSTPYTHON - @unittest.expectedFailure @force_not_colorized def test_envvar_and_command_line(self): rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", @@ -1419,8 +1478,6 @@ def test_envvar_and_command_line(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") - # TODO: RUSTPYTHON - @unittest.expectedFailure @force_not_colorized def test_conflicting_envvar_and_command_line(self): rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", @@ -1462,7 +1519,7 @@ def test_default_filter_configuration(self): code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; " else: code = "" - code += "import warnings; [print(f) for f in warnings.filters]" + code += "import warnings; [print(f) for f in warnings._get_filters()]" rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True) stdout_lines = [line.strip() for line in stdout.splitlines()] @@ -1470,8 +1527,6 @@ def test_default_filter_configuration(self): self.assertEqual(stdout_lines, expected_output) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', 'requires non-ascii filesystemencoding') def test_nonascii(self): @@ -1486,17 +1541,21 @@ def test_nonascii(self): class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): module = c_warnings - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_default_filter_configuration(self): - # XXX: RUSTPYHTON; remove the entire function when fixed - super().test_default_filter_configuration() - - class PyEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): module = py_warnings +class LocksTest(unittest.TestCase): + @support.cpython_only + @unittest.skipUnless(c_warnings, 'C module is required') + def test_release_lock_no_lock(self): + with self.assertRaisesRegex( + RuntimeError, + 'cannot release un-acquired lock', + ): + c_warnings._release_lock() + + class _DeprecatedTest(BaseTest, unittest.TestCase): """Test _deprecated().""" @@ -1551,8 +1610,7 @@ def test_issue_8766(self): class FinalizationTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - TypeError: 'NoneType' object is not callable def test_finalization(self): # Issue #19421: warnings.warn() should not crash # during Python finalization @@ -1570,8 +1628,6 @@ def __del__(self): self.assertEqual(err.decode().rstrip(), '<string>:7: UserWarning: test') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_late_resource_warning(self): # Issue #21925: Emitting a ResourceWarning late during the Python # shutdown must be logged. @@ -1582,15 +1638,178 @@ def test_late_resource_warning(self): # (_warnings will try to import it) code = "f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) # import the warnings module code = "import warnings; f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) + + +class AsyncTests(BaseTest): + """Verifies that the catch_warnings() context manager behaves + as expected when used inside async co-routines. This requires + that the context_aware_warnings flag is enabled, so that + the context manager uses a context variable. + """ + + def setUp(self): + super().setUp() + self.module.resetwarnings() + + @unittest.skipIf(not sys.flags.context_aware_warnings, + "requires context aware warnings") + def test_async_context(self): + import asyncio + + # Events to force the execution interleaving we want. + step_a1 = asyncio.Event() + step_a2 = asyncio.Event() + step_b1 = asyncio.Event() + step_b2 = asyncio.Event() + + async def run_a(): + with self.module.catch_warnings(record=True) as w: + await step_a1.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_a warning', UserWarning) + step_b1.set() + await step_a2.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_a warning') + step_b2.set() + + async def run_b(): + with self.module.catch_warnings(record=True) as w: + step_a1.set() + await step_b1.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_b warning', UserWarning) + step_a2.set() + await step_b2.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_b warning') + async def run_tasks(): + await asyncio.gather(run_a(), run_b()) -class DeprecatedTests(unittest.TestCase): + asyncio.run(run_tasks()) + + @unittest.skipIf(not sys.flags.context_aware_warnings, + "requires context aware warnings") + def test_async_task_inherit(self): + """Check that a new asyncio task inherits warnings context from the + coroutine that spawns it. + """ + import asyncio + + step1 = asyncio.Event() + step2 = asyncio.Event() + + async def run_child1(): + await step1.wait() + # This should be recorded by the run_parent() catch_warnings + # context. + self.module.warn('child warning', UserWarning) + step2.set() + + async def run_child2(): + # This establishes a new catch_warnings() context. The + # run_child1() task should still be using the context from + # run_parent() if context-aware warnings are enabled. + with self.module.catch_warnings(record=True) as w: + step1.set() + await step2.wait() + + async def run_parent(): + with self.module.catch_warnings(record=True) as w: + await asyncio.gather(run_child1(), run_child2()) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'child warning') + + asyncio.run(run_parent()) + + +class CAsyncTests(AsyncTests, unittest.TestCase): + module = c_warnings + + +class PyAsyncTests(AsyncTests, unittest.TestCase): + module = py_warnings + + +class ThreadTests(BaseTest): + """Verifies that the catch_warnings() context manager behaves as + expected when used within threads. This requires that both the + context_aware_warnings flag and thread_inherit_context flags are enabled. + """ + + ENABLE_THREAD_TESTS = (sys.flags.context_aware_warnings and + sys.flags.thread_inherit_context) + + def setUp(self): + super().setUp() + self.module.resetwarnings() + + @unittest.skipIf(not ENABLE_THREAD_TESTS, + "requires thread-safe warnings flags") + def test_threaded_context(self): + import threading + + barrier = threading.Barrier(2, timeout=2) + + def run_a(): + with self.module.catch_warnings(record=True) as w: + barrier.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_a warning', UserWarning) + barrier.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_a warning') + # Should be caught be the catch_warnings() context manager of run_threads() + self.module.warn('main warning', UserWarning) + + def run_b(): + with self.module.catch_warnings(record=True) as w: + barrier.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + barrier.wait() + self.module.warn('run_b warning', UserWarning) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_b warning') + # Should be caught be the catch_warnings() context manager of run_threads() + self.module.warn('main warning', UserWarning) + + def run_threads(): + threads = [ + threading.Thread(target=run_a), + threading.Thread(target=run_b), + ] + with self.module.catch_warnings(record=True) as w: + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(len(w), 2) + self.assertEqual(w[0].message.args[0], 'main warning') + self.assertEqual(w[1].message.args[0], 'main warning') + + run_threads() + + +class CThreadTests(ThreadTests, unittest.TestCase): + module = c_warnings + + +class PyThreadTests(ThreadTests, unittest.TestCase): + module = py_warnings + + +class DeprecatedTests(PyPublicAPITests): def test_dunder_deprecated(self): @deprecated("A will go away soon") class A: @@ -1766,6 +1985,25 @@ class D(C, x=3): self.assertEqual(D.inited, 3) + def test_existing_init_subclass_in_sibling_base(self): + @deprecated("A will go away soon") + class A: + pass + class B: + def __init_subclass__(cls, x): + super().__init_subclass__() + cls.inited = x + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class C(A, B, x=42): + pass + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class D(B, A, x=42): + pass + self.assertEqual(D.inited, 42) + def test_init_subclass_has_correct_cls(self): init_subclass_saw = None diff --git a/Lib/test/test_warnings/data/stacklevel.py b/Lib/test/test_warnings/data/stacklevel.py index c6dd24733b3..fe36242d3d2 100644 --- a/Lib/test/test_warnings/data/stacklevel.py +++ b/Lib/test/test_warnings/data/stacklevel.py @@ -4,11 +4,13 @@ import warnings from test.test_warnings.data import package_helper -def outer(message, stacklevel=1): - inner(message, stacklevel) -def inner(message, stacklevel=1): - warnings.warn(message, stacklevel=stacklevel) +def outer(message, stacklevel=1, skip_file_prefixes=()): + inner(message, stacklevel, skip_file_prefixes) + +def inner(message, stacklevel=1, skip_file_prefixes=()): + warnings.warn(message, stacklevel=stacklevel, + skip_file_prefixes=skip_file_prefixes) def package(message, *, stacklevel): package_helper.inner_api(message, stacklevel=stacklevel, diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 5e771c8de96..346a343761a 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,7 @@ from test import audiotests from test import support import io +import os import struct import sys import wave @@ -222,6 +223,14 @@ def test_read_wrong_sample_width(self): with self.assertRaisesRegex(wave.Error, 'bad sample width'): wave.open(io.BytesIO(b)) + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index e7cd5962cf9..ac4e8f82b9c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -13,7 +13,7 @@ import textwrap from test import support -from test.support import script_helper, ALWAYS_EQ, suppress_immortalization +from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect from test.support import import_helper from test.support import threading_helper @@ -289,8 +289,7 @@ def test_ref_reuse(self): self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_reuse(self): o = C() proxy1 = weakref.proxy(o) @@ -338,8 +337,7 @@ def __bytes__(self): self.assertIn("__bytes__", dir(weakref.proxy(instance))) self.assertEqual(bytes(weakref.proxy(instance)), b"bytes") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_index(self): class C: def __index__(self): @@ -348,8 +346,7 @@ def __index__(self): p = weakref.proxy(o) self.assertEqual(operator.index(p), 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_div(self): class C: def __floordiv__(self, other): @@ -362,8 +359,7 @@ def __ifloordiv__(self, other): p //= 5 self.assertEqual(p, 21) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_matmul(self): class C: def __matmul__(self, other): @@ -387,13 +383,11 @@ def __imatmul__(self, other): # was not honored, and was broken in different ways for # PyWeakref_NewRef() and PyWeakref_NewProxy(). (Two tests.) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_ref_without_callback(self): self.check_shared_without_callback(weakref.ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_proxy_without_callback(self): self.check_shared_without_callback(weakref.proxy) @@ -415,8 +409,7 @@ def check_shared_without_callback(self, makeref): p2 = makeref(o) self.assertIs(p1, p2, "callbacks were None, NULL in the C API") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callable_proxy(self): o = Callable() ref1 = weakref.proxy(o) @@ -446,7 +439,7 @@ def check_proxy(self, o, proxy): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -456,7 +449,7 @@ def check_proxy(self, o, proxy): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -511,8 +504,7 @@ def __iter__(self): # Calls proxy.__next__ self.assertEqual(list(weak_it), [4, 5, 6]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_bad_next(self): # bpo-44720: PyIter_Next() shouldn't be called if the reference # isn't an iterator. @@ -602,8 +594,7 @@ def test_getweakrefs(self): self.assertEqual(weakref.getweakrefs(1), [], "list of refs does not match for int") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newstyle_number_ops(self): class F(float): pass @@ -677,7 +668,6 @@ class C(object): # deallocation of c2. del c2 - @suppress_immortalization() def test_callback_in_cycle(self): import gc @@ -770,9 +760,7 @@ class D: del c1, c2, C, D gc.collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure - @suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_in_cycle_resurrection(self): import gc @@ -819,8 +807,7 @@ def C_went_away(ignore): gc.collect() self.assertEqual(alist, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callbacks_on_callback(self): import gc @@ -859,13 +846,9 @@ def cb(self, ignore): gc.collect() self.assertEqual(alist, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_ref_creation(self): self.check_gc_during_creation(weakref.ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_proxy_creation(self): self.check_gc_during_creation(weakref.proxy) @@ -906,8 +889,6 @@ def __del__(self): w = Target() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_init(self): # Issue 3634 # <weakref to class>.__init__() doesn't check errors correctly @@ -916,7 +897,7 @@ def test_init(self): # No exception should be raised here gc.collect() - @suppress_immortalization() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'test.test_weakref.ReferencesTestCase.test_classes.<locals>.A'> != None def test_classes(self): # Check that classes are weakrefable. class A(object): @@ -1018,8 +999,7 @@ def cb(wparent): del root gc.collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute(self): x = Object(1) callback = lambda ref: None @@ -1029,8 +1009,7 @@ def test_callback_attribute(self): ref2 = weakref.ref(x) self.assertIsNone(ref2.__callback__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute_after_deletion(self): x = Object(1) ref = weakref.ref(x, self.callback) @@ -1082,8 +1061,6 @@ def callback(obj): class SubclassableWeakrefTestCase(TestBase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs(self): class MyRef(weakref.ref): def __init__(self, ob, callback=None, value=42): @@ -1102,8 +1079,6 @@ def __call__(self): self.assertIsNone(mr()) self.assertTrue(mr.called) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs_dont_replace_standard_refs(self): class MyRef(weakref.ref): pass @@ -1147,7 +1122,7 @@ def meth(self): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" @@ -1355,13 +1330,11 @@ def check_len_cycles(self, dict_type, cons): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_keyed_len_cycles(self): self.check_len_cycles(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_valued_len_cycles(self): self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1389,13 +1362,9 @@ def check_len_race(self, dict_type, cons): self.assertGreaterEqual(n2, 0) self.assertLessEqual(n2, n1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_len_race(self): self.check_len_race(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_valued_len_race(self): self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1896,8 +1865,7 @@ def test_weak_valued_delitem(self): self.assertEqual(len(d), 1) self.assertEqual(list(d.items()), [('something else', o2)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weak_keyed_bad_delitem(self): d = weakref.WeakKeyDictionary() o = Object('1') @@ -2077,12 +2045,15 @@ def pop_and_collect(lst): if exc: raise exc[0] + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threaded_weak_key_dict_deepcopy(self): @@ -2090,13 +2061,15 @@ def test_threaded_weak_key_dict_deepcopy(self): # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) - @unittest.skip("TODO: RUSTPYTHON; occasionally crash (Exit code -6)") + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threaded_weak_value_dict_deepcopy(self): @@ -2281,7 +2254,6 @@ def error(): assert f3.atexit == True assert f4.atexit == True - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_atexit(self): prog = ('from test.test_weakref import FinalizeTestCase;'+ 'FinalizeTestCase.run_in_child()') @@ -2292,8 +2264,7 @@ def test_atexit(self): class ModuleTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_names(self): for name in ('ReferenceType', 'ProxyType', 'CallableProxyType', 'WeakMethod', 'WeakSet', 'WeakKeyDictionary', 'WeakValueDictionary'): @@ -2393,8 +2364,7 @@ def test_names(self): __test__ = {'libreftest' : libreftest} def load_tests(loader, tests, pattern): - # TODO: RUSTPYTHON - # tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 5e8cacc09dc..af9bbe7cd41 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -425,8 +425,6 @@ def test_len_cycles(self): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_len_race(self): # Extended sanity checks for len() in the face of cyclic collection self.addCleanup(gc.set_threshold, *gc.get_threshold()) diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index 7a33f906986..e64208330ad 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -2,10 +2,7 @@ import os import pathlib -import random import re -import threading -import time import unittest from test.support import import_helper, os_helper @@ -77,41 +74,27 @@ def _events_waitany_test(self, n): evts[i] = old_evt - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_few_events_waitall(self): self._events_waitall_test(16) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_events_waitall(self): self._events_waitall_test(256) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_events_waitall(self): self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_few_events_waitany(self): self._events_waitany_test(16) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_events_waitany(self): self._events_waitany_test(256) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_events_waitany(self): self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) class WinAPITests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getlongpathname(self): testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" if not os.path.isdir(testfn): @@ -128,8 +111,6 @@ def test_getlongpathname(self): candidates = set(testfn.parent.glob("Progra*")) self.assertIn(pathlib.Path(actual), candidates) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getshortpathname(self): testfn = pathlib.Path(os.getenv("ProgramFiles")) if not os.path.isdir(testfn): @@ -144,8 +125,6 @@ def test_getshortpathname(self): # Should contain "PROGRA~" but we can't predict the number self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_namedpipe(self): pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" @@ -176,4 +155,4 @@ def test_namedpipe(self): self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) pipe2.write(b'testdata') pipe2.flush() - self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) \ No newline at end of file + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py new file mode 100644 index 00000000000..1bae884ed9a --- /dev/null +++ b/Lib/test/test_winconsoleio.py @@ -0,0 +1,244 @@ +'''Tests for WindowsConsoleIO +''' + +import io +import os +import sys +import tempfile +import unittest +from test.support import os_helper, requires_resource + +if sys.platform != 'win32': + raise unittest.SkipTest("test only relevant on win32") + +from _testconsole import write_input + +ConIO = io._WindowsConsoleIO + +class WindowsConsoleIOTests(unittest.TestCase): + def test_abc(self): + self.assertIsSubclass(ConIO, io.RawIOBase) + self.assertNotIsSubclass(ConIO, io.BufferedIOBase) + self.assertNotIsSubclass(ConIO, io.TextIOBase) + + def test_open_fd(self): + self.assertRaisesRegex(ValueError, + "negative file descriptor", ConIO, -1) + + with tempfile.TemporaryFile() as tmpfile: + fd = tmpfile.fileno() + # Windows 10: "Cannot open non-console file" + # Earlier: "Cannot open console output buffer for reading" + self.assertRaisesRegex(ValueError, + "Cannot open (console|non-console file)", ConIO, fd) + + try: + f = ConIO(0) + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertEqual(0, f.fileno()) + f.close() # multiple close should not crash + f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass + + try: + f = ConIO(1, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(1, f.fileno()) + f.close() + f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass + + try: + f = ConIO(2, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(2, f.fileno()) + f.close() + f.close() + + def test_open_name(self): + self.assertRaises(ValueError, ConIO, sys.executable) + + f = ConIO("CON") + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() # multiple close should not crash + f.close() + + f = ConIO('CONIN$') + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + f = ConIO('CONOUT$', 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + # bpo-45354: Windows 11 changed MS-DOS device name handling + if sys.getwindowsversion()[:3] < (10, 0, 22000): + f = open('C:/con', 'rb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + def test_subclass_repr(self): + class TestSubclass(ConIO): + pass + + f = TestSubclass("CON") + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + + @unittest.skipIf(sys.getwindowsversion()[:2] <= (6, 1), + "test does not work on Windows 7 and earlier") + def test_conin_conout_names(self): + f = open(r'\\.\conin$', 'rb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + f = open('//?/conout$', 'wb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + def test_conout_path(self): + temp_path = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_path) + + conout_path = os.path.join(temp_path, 'CONOUT$') + + with open(conout_path, 'wb', buffering=0) as f: + # bpo-45354: Windows 11 changed MS-DOS device name handling + if (6, 1) < sys.getwindowsversion()[:3] < (10, 0, 22000): + self.assertIsInstance(f, ConIO) + else: + self.assertNotIsInstance(f, ConIO) + + def test_write_empty_data(self): + with ConIO('CONOUT$', 'w') as f: + self.assertEqual(f.write(b''), 0) + + @requires_resource('console') + def test_write(self): + testcases = [] + with ConIO('CONOUT$', 'w') as f: + for a in [ + b'', + b'abc', + b'\xc2\xa7\xe2\x98\x83\xf0\x9f\x90\x8d', + b'\xff'*10, + ]: + for b in b'\xc2\xa7', b'\xe2\x98\x83', b'\xf0\x9f\x90\x8d': + testcases.append(a + b) + for i in range(1, len(b)): + data = a + b[:i] + testcases.append(data + b'z') + testcases.append(data + b'\xff') + # incomplete multibyte sequence + with self.subTest(data=data): + self.assertEqual(f.write(data), len(a)) + for data in testcases: + with self.subTest(data=data): + self.assertEqual(f.write(data), len(data)) + + def assertStdinRoundTrip(self, text): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input( + stdin.buffer.raw, + (text + '\r\n').encode('utf-16-le', 'surrogatepass') + ) + actual = input() + finally: + sys.stdin = old_stdin + self.assertEqual(actual, text) + + @requires_resource('console') + def test_input(self): + # ASCII + self.assertStdinRoundTrip('abc123') + # Non-ASCII + self.assertStdinRoundTrip('ϼўТλФЙ') + # Combining characters + self.assertStdinRoundTrip('A͏B ﬖ̳AA̝') + + # bpo-38325 + @unittest.skipIf(True, "Handling Non-BMP characters is broken") + def test_input_nonbmp(self): + # Non-BMP + self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + + @requires_resource('console') + def test_partial_reads(self): + # Test that reading less than 1 full character works when stdin + # contains multibyte UTF-8 sequences + source = 'ϼўТλФЙ\r\n'.encode('utf-16-le') + expected = 'ϼўТλФЙ\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + # bpo-38325 + @unittest.skipIf(True, "Handling Non-BMP characters is broken") + def test_partial_surrogate_reads(self): + # Test that reading less than 1 full character works when stdin + # contains surrogate pairs that cannot be decoded to UTF-8 without + # reading an extra character. + source = '\U00101FFF\U00101001\r\n'.encode('utf-16-le') + expected = '\U00101FFF\U00101001\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + @requires_resource('console') + def test_ctrl_z(self): + with open('CONIN$', 'rb', buffering=0) as stdin: + source = '\xC4\x1A\r\n'.encode('utf-16-le') + expected = '\xC4'.encode('utf-8') + write_input(stdin, source) + a, b = stdin.read(1), stdin.readall() + self.assertEqual(expected[0:1], a) + self.assertEqual(expected[1:], b) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 924a962781a..1bc830c02c3 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -3,6 +3,7 @@ import gc import os, sys, errno +import itertools import threading import unittest from platform import machine, win32_edition @@ -291,6 +292,37 @@ def run(self): DeleteKey(HKEY_CURRENT_USER, test_key_name+'\\changing_value') DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_race_condition(self): + # gh-142282: QueryValueEx could read garbage buffer under race + # condition when another thread changes the value size + done = False + ready = threading.Event() + values = [b'ham', b'spam'] + + class WriterThread(threading.Thread): + def run(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + values_iter = itertools.cycle(values) + while not done: + val = next(values_iter) + SetValueEx(key, 'test_value', 0, REG_BINARY, val) + ready.set() + + thread = WriterThread() + thread.start() + try: + ready.wait() + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + for _ in range(1000): + result, typ = QueryValueEx(key, 'test_value') + # The result must be one of the written values, + # not garbage data from uninitialized buffer + self.assertIn(result, values) + finally: + done = True + thread.join() + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_long_key(self): # Issue2810, in 2.6 and 3.1 when the key name was exactly 256 # characters, EnumKey raised "WindowsError: More data is diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py new file mode 100644 index 00000000000..9724d830ade --- /dev/null +++ b/Lib/test/test_winsound.py @@ -0,0 +1,187 @@ +# Ridiculously simple test of the winsound module for Windows. + +import functools +import os +import time +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper + + +support.requires('audio') +winsound = import_helper.import_module('winsound') + + +# Unless we actually have an ear in the room, we have no idea whether a sound +# actually plays, and it's incredibly flaky trying to figure out if a sound +# even *should* play. Instead of guessing, just call the function and assume +# it either passed or raised the RuntimeError we expect in case of failure. +def sound_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + ret = func(*args, **kwargs) + except RuntimeError as e: + if support.verbose: + print(func.__name__, 'failed:', e) + else: + if support.verbose: + print(func.__name__, 'returned') + return ret + return wrapper + + +safe_Beep = sound_func(winsound.Beep) +safe_MessageBeep = sound_func(winsound.MessageBeep) +safe_PlaySound = sound_func(winsound.PlaySound) + + +class BeepTest(unittest.TestCase): + + def test_errors(self): + self.assertRaises(TypeError, winsound.Beep) + self.assertRaises(ValueError, winsound.Beep, 36, 75) + self.assertRaises(ValueError, winsound.Beep, 32768, 75) + + def test_extremes(self): + safe_Beep(37, 75) + safe_Beep(32767, 75) + + def test_increasingfrequency(self): + for i in range(100, 2000, 100): + safe_Beep(i, 75) + + def test_keyword_args(self): + safe_Beep(duration=75, frequency=2000) + + +class MessageBeepTest(unittest.TestCase): + + def tearDown(self): + time.sleep(0.5) + + def test_default(self): + self.assertRaises(TypeError, winsound.MessageBeep, "bad") + self.assertRaises(TypeError, winsound.MessageBeep, 42, 42) + safe_MessageBeep() + + def test_ok(self): + safe_MessageBeep(winsound.MB_OK) + + def test_asterisk(self): + safe_MessageBeep(winsound.MB_ICONASTERISK) + + def test_exclamation(self): + safe_MessageBeep(winsound.MB_ICONEXCLAMATION) + + def test_hand(self): + safe_MessageBeep(winsound.MB_ICONHAND) + + def test_question(self): + safe_MessageBeep(winsound.MB_ICONQUESTION) + + def test_error(self): + safe_MessageBeep(winsound.MB_ICONERROR) + + def test_information(self): + safe_MessageBeep(winsound.MB_ICONINFORMATION) + + def test_stop(self): + safe_MessageBeep(winsound.MB_ICONSTOP) + + def test_warning(self): + safe_MessageBeep(winsound.MB_ICONWARNING) + + def test_keyword_args(self): + safe_MessageBeep(type=winsound.MB_OK) + + +class PlaySoundTest(unittest.TestCase): + + def test_errors(self): + self.assertRaises(TypeError, winsound.PlaySound) + self.assertRaises(TypeError, winsound.PlaySound, "bad", "bad") + self.assertRaises( + RuntimeError, + winsound.PlaySound, + "none", winsound.SND_ASYNC | winsound.SND_MEMORY + ) + self.assertRaises(TypeError, winsound.PlaySound, b"bad", 0) + self.assertRaises(TypeError, winsound.PlaySound, "bad", + winsound.SND_MEMORY) + self.assertRaises(TypeError, winsound.PlaySound, 1, 0) + # embedded null character + self.assertRaises(ValueError, winsound.PlaySound, 'bad\0', 0) + + def test_keyword_args(self): + safe_PlaySound(flags=winsound.SND_ALIAS, sound="SystemExit") + + def test_snd_memory(self): + with open(support.findfile('pluck-pcm8.wav', + subdir='audiodata'), 'rb') as f: + audio_data = f.read() + safe_PlaySound(audio_data, winsound.SND_MEMORY) + audio_data = bytearray(audio_data) + safe_PlaySound(audio_data, winsound.SND_MEMORY) + + def test_snd_filename(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + safe_PlaySound(fn, winsound.SND_FILENAME | winsound.SND_NODEFAULT) + + def test_snd_filepath(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + path = os_helper.FakePath(fn) + safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) + + def test_snd_filepath_as_bytes(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + self.assertRaises( + TypeError, + winsound.PlaySound, + os_helper.FakePath(os.fsencode(fn)), + winsound.SND_FILENAME | winsound.SND_NODEFAULT + ) + + def test_aliases(self): + aliases = [ + "SystemAsterisk", + "SystemExclamation", + "SystemExit", + "SystemHand", + "SystemQuestion", + ] + for alias in aliases: + with self.subTest(alias=alias): + safe_PlaySound(alias, winsound.SND_ALIAS) + + def test_alias_fallback(self): + safe_PlaySound('!"$%&/(#+*', winsound.SND_ALIAS) + + def test_alias_nofallback(self): + safe_PlaySound('!"$%&/(#+*', winsound.SND_ALIAS | winsound.SND_NODEFAULT) + + def test_stopasync(self): + safe_PlaySound( + 'SystemQuestion', + winsound.SND_ALIAS | winsound.SND_ASYNC | winsound.SND_LOOP + ) + time.sleep(0.5) + safe_PlaySound('SystemQuestion', winsound.SND_ALIAS | winsound.SND_NOSTOP) + # Issue 8367: PlaySound(None, winsound.SND_PURGE) + # does not raise on systems without a sound card. + winsound.PlaySound(None, winsound.SND_PURGE) + + def test_sound_sentry(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SENTRY) + + def test_sound_sync(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SYNC) + + def test_sound_system(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SYSTEM) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index b321dac6c66..68ee3725088 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -1,9 +1,10 @@ -"""Unit tests for the with statement specified in PEP 343.""" +"""Unit tests for the 'with/async with' statements specified in PEP 343/492.""" __author__ = "Mike Bland" __email__ = "mbland at acm dot org" +import re import sys import traceback import unittest @@ -11,6 +12,16 @@ from contextlib import _GeneratorContextManager, contextmanager, nullcontext +def do_with(obj): + with obj: + pass + + +async def do_async_with(obj): + async with obj: + pass + + class MockContextManager(_GeneratorContextManager): def __init__(self, *args): super().__init__(*args) @@ -110,34 +121,77 @@ def fooNotDeclared(): with foo: pass self.assertRaises(NameError, fooNotDeclared) - def testEnterAttributeError1(self): - class LacksEnter(object): - def __exit__(self, type, value, traceback): - pass - - def fooLacksEnter(): - foo = LacksEnter() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager', fooLacksEnter) - - def testEnterAttributeError2(self): - class LacksEnterAndExit(object): - pass + def testEnterAttributeError(self): + class LacksEnter: + def __exit__(self, type, value, traceback): ... - def fooLacksEnterAndExit(): - foo = LacksEnterAndExit() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager', fooLacksEnterAndExit) + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the context manager protocol " + "(missed __enter__ method)" + ))): + do_with(LacksEnter()) def testExitAttributeError(self): - class LacksExit(object): - def __enter__(self): - pass - - def fooLacksExit(): - foo = LacksExit() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager.*__exit__', fooLacksExit) + class LacksExit: + def __enter__(self): ... + + msg = re.escape(( + "object does not support the context manager protocol " + "(missed __exit__ method)" + )) + # a missing __exit__ is reported missing before a missing __enter__ + with self.assertRaisesRegex(TypeError, msg): + do_with(object()) + with self.assertRaisesRegex(TypeError, msg): + do_with(LacksExit()) + + def testWithForAsyncManager(self): + class AsyncManager: + async def __aenter__(self): ... + async def __aexit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the context manager protocol " + "(missed __exit__ method) but it supports the asynchronous " + "context manager protocol. Did you mean to use 'async with'?" + ))): + do_with(AsyncManager()) + + def testAsyncEnterAttributeError(self): + class LacksAsyncEnter: + async def __aexit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aenter__ method)" + ))): + do_async_with(LacksAsyncEnter()).send(None) + + def testAsyncExitAttributeError(self): + class LacksAsyncExit: + async def __aenter__(self): ... + + msg = re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aexit__ method)" + )) + # a missing __aexit__ is reported missing before a missing __aenter__ + with self.assertRaisesRegex(TypeError, msg): + do_async_with(object()).send(None) + with self.assertRaisesRegex(TypeError, msg): + do_async_with(LacksAsyncExit()).send(None) + + def testAsyncWithForSyncManager(self): + class SyncManager: + def __enter__(self): ... + def __exit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aexit__ method) but it supports the context manager " + "protocol. Did you mean to use 'with'?" + ))): + do_async_with(SyncManager()).send(None) def assertRaisesSyntaxError(self, codestr): def shouldRaiseSyntaxError(s): @@ -190,6 +244,7 @@ def shouldThrow(): pass self.assertRaises(RuntimeError, shouldThrow) + class ContextmanagerAssertionMixin(object): def setUp(self): @@ -624,7 +679,7 @@ def testSingleComplexTarget(self): class C: pass blah = C() with mock_contextmanager_generator() as blah.foo: - self.assertEqual(hasattr(blah, "foo"), True) + self.assertHasAttr(blah, "foo") def testMultipleComplexTargets(self): class C: @@ -719,7 +774,7 @@ def testExceptionInExprList(self): try: with self.Dummy() as a, self.InitRaises(): pass - except: + except RuntimeError: pass self.assertTrue(a.enter_called) self.assertTrue(a.exit_called) @@ -752,7 +807,7 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' def testExceptionLocation(self): # The location of an exception raised from # __init__, __enter__ or __exit__ of a context diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py new file mode 100644 index 00000000000..90eb40439d4 --- /dev/null +++ b/Lib/test/test_wmi.py @@ -0,0 +1,89 @@ +# Test the internal _wmi module on Windows +# This is used by the platform module, and potentially others + +import unittest +from test import support +from test.support import import_helper + + +# Do this first so test will be skipped if module doesn't exist +_wmi = import_helper.import_module('_wmi', required_on=['win']) + + +def wmi_exec_query(query): + # gh-112278: WMI maybe slow response when first call. + for _ in support.sleeping_retry(support.LONG_TIMEOUT): + try: + return _wmi.exec_query(query) + except BrokenPipeError: + pass + # retry on pipe error + except WindowsError as exc: + if exc.winerror != 258: + raise + # retry on timeout + + +class WmiTests(unittest.TestCase): + def test_wmi_query_os_version(self): + r = wmi_exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") + self.assertEqual(1, len(r)) + k, eq, v = r[0].partition("=") + self.assertEqual("=", eq, r[0]) + self.assertEqual("Version", k, r[0]) + # Best we can check for the version is that it's digits, dot, digits, anything + # Otherwise, we are likely checking the result of the query against itself + self.assertRegex(v, r"\d+\.\d+.+$", r[0]) + + def test_wmi_query_repeated(self): + # Repeated queries should not break + for _ in range(10): + self.test_wmi_query_os_version() + + def test_wmi_query_error(self): + # Invalid queries fail with OSError + try: + wmi_exec_query("SELECT InvalidColumnName FROM InvalidTableName") + except OSError as ex: + if ex.winerror & 0xFFFFFFFF == 0x80041010: + # This is the expected error code. All others should fail the test + return + self.fail("Expected OSError") + + def test_wmi_query_repeated_error(self): + for _ in range(10): + self.test_wmi_query_error() + + def test_wmi_query_not_select(self): + # Queries other than SELECT are blocked to avoid potential exploits + with self.assertRaises(ValueError): + wmi_exec_query("not select, just in case someone tries something") + + @support.requires_resource('cpu') + def test_wmi_query_overflow(self): + # Ensure very big queries fail + # Test multiple times to ensure consistency + for _ in range(2): + with self.assertRaises(OSError): + wmi_exec_query("SELECT * FROM CIM_DataFile") + + def test_wmi_query_multiple_rows(self): + # Multiple instances should have an extra null separator + r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") + self.assertNotStartsWith(r, "\0") + self.assertNotEndsWith(r, "\0") + it = iter(r.split("\0")) + try: + while True: + self.assertRegex(next(it), r"ProcessId=\d+") + self.assertEqual("", next(it)) + except StopIteration: + pass + + def test_wmi_query_threads(self): + from concurrent.futures import ThreadPoolExecutor + query = "SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000" + with ThreadPoolExecutor(4) as pool: + task = [pool.submit(wmi_exec_query, query) for _ in range(32)] + for t in task: + self.assertRegex(t.result(), "ProcessId=") diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index b89e181f9f8..d546e3ef219 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -1,6 +1,6 @@ from unittest import mock from test import support -from test.support import warnings_helper +from test.support import socket_helper from test.test_httpservers import NoLogRequestHandler from unittest import TestCase from wsgiref.util import setup_testing_defaults @@ -80,41 +80,26 @@ def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): return out.getvalue(), err.getvalue() -def compare_generic_iter(make_it,match): - """Utility to compare a generic 2.1/2.2+ iterator with an iterable - If running under Python 2.2+, this tests the iterator using iter()/next(), - as well as __getitem__. 'make_it' must be a function returning a fresh +def compare_generic_iter(make_it, match): + """Utility to compare a generic iterator with an iterable + + This tests the iterator using iter()/next(). + 'make_it' must be a function returning a fresh iterator to be tested (since this may test the iterator twice).""" it = make_it() - n = 0 + if not iter(it) is it: + raise AssertionError for item in match: - if not it[n]==item: raise AssertionError - n+=1 - try: - it[n] - except IndexError: - pass - else: - raise AssertionError("Too many items from __getitem__",it) - + if not next(it) == item: + raise AssertionError try: - iter, StopIteration - except NameError: + next(it) + except StopIteration: pass else: - # Only test iter mode under 2.2+ - it = make_it() - if not iter(it) is it: raise AssertionError - for item in match: - if not next(it) == item: raise AssertionError - try: - next(it) - except StopIteration: - pass - else: - raise AssertionError("Too many items from .__next__()", it) + raise AssertionError("Too many items from .__next__()", it) class IntegrationTests(TestCase): @@ -152,7 +137,7 @@ def test_environ(self): def test_request_length(self): out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") self.assertEqual(out.splitlines()[0], - b"HTTP/1.0 414 Request-URI Too Long") + b"HTTP/1.0 414 URI Too Long") def test_validated_hello(self): out, err = run_amock(validator(hello_app)) @@ -264,7 +249,7 @@ def app(environ, start_response): class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): pass - server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + server = make_server(socket_helper.HOST, 0, app, handler_class=WsgiHandler) self.addCleanup(server.server_close) interrupted = threading.Event() @@ -339,7 +324,6 @@ def checkReqURI(self,uri,query=1,**kw): util.setup_testing_defaults(kw) self.assertEqual(util.request_uri(kw,query),uri) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def checkFW(self,text,size,match): def make_it(text=text,size=size): @@ -358,15 +342,6 @@ def make_it(text=text,size=size): it.close() self.assertTrue(it.filelike.closed) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_filewrapper_getitem_deprecation(self): - wrapper = util.FileWrapper(StringIO('foobar'), 3) - with self.assertWarnsRegex(DeprecationWarning, - r'Use iterator protocol instead'): - # This should have returned 'bar'. - self.assertEqual(wrapper[1], 'foo') - def testSimpleShifts(self): self.checkShift('','/', '', '/', '') self.checkShift('','/x', 'x', '/x', '') @@ -473,6 +448,10 @@ def testHopByHop(self): for alt in hop, hop.title(), hop.upper(), hop.lower(): self.assertFalse(util.is_hop_by_hop(alt)) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_filewrapper_getitem_deprecation(self): + return super().test_filewrapper_getitem_deprecation() + class HeaderTests(TestCase): def testMappingInterface(self): @@ -581,7 +560,7 @@ def testEnviron(self): # Test handler.environ as a dict expected = {} setup_testing_defaults(expected) - # Handler inherits os_environ variables which are not overriden + # Handler inherits os_environ variables which are not overridden # by SimpleHandler.add_cgi_vars() (SimpleHandler.base_env) for key, value in os_environ.items(): if key not in expected: @@ -821,8 +800,6 @@ def flush(self): b"Hello, world!", written) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testClientConnectionTerminations(self): environ = {"SERVER_PROTOCOL": "HTTP/1.0"} for exception in ( @@ -841,8 +818,6 @@ def write(self, b): self.assertFalse(stderr.getvalue()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testDontResetInternalStateOnException(self): class CustomException(ValueError): pass diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py index 5282e806e40..5f5f2eb328d 100644 --- a/Lib/test/test_xml_dom_xmlbuilder.py +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -50,8 +50,6 @@ def test_builder(self): builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) self.assertIsInstance(builder, xmlbuilder.DOMBuilder) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_uri(self): body = ( b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" @@ -74,8 +72,6 @@ def test_parse_uri(self): self.assertIsInstance(document, minidom.Document) self.assertEqual(len(document.childNodes), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_with_systemId(self): response = io.BytesIO(SMALL_SAMPLE) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 11c617b5f34..9d6d39307ff 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -384,7 +384,6 @@ def test_simpleops(self): self.serialize_check(element, '<tag key="value"><subtag /><subtag /></tag>') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cdata(self): # Test CDATA handling (etc). @@ -407,7 +406,6 @@ def test_file_init(self): self.assertEqual(tree.find("element/../empty-element").tag, 'empty-element') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_path_cache(self): # Check that the path cache behaves sanely. @@ -424,7 +422,6 @@ def test_path_cache(self): for i in range(600): ET.ElementTree(elem).find('./'+str(i)) self.assertLess(len(ElementPath._cache), 500) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_copy(self): # Test copy handling (etc). @@ -888,7 +885,6 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialize_parser_without_target(self): # Explicit None parser = ET.XMLParser(target=None) @@ -936,14 +932,12 @@ def test_children(self): elem.clear() self.assertEqual(list(elem), []) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_writestring(self): elem = ET.XML("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') elem = ET.fromstring("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent(self): elem = ET.XML("<root></root>") ET.indent(elem) @@ -988,7 +982,6 @@ def test_indent(self): b'</html>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") ET.indent(elem, space='\t') @@ -1014,7 +1007,6 @@ def test_indent_space(self): b'</html>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space_caching(self): elem = ET.XML("<html><body><p>par</p><p>text</p><p><br/></p><p /></body></html>") ET.indent(elem) @@ -1031,7 +1023,6 @@ def test_indent_space_caching(self): len({id(el.tail) for el in elem.iter()}), ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_level(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") with self.assertRaises(ValueError): @@ -1064,7 +1055,6 @@ def test_indent_level(self): b' </html>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1076,7 +1066,6 @@ def test_tostring_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_different_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1084,14 +1073,12 @@ def test_tostring_default_namespace_different_namespace(self): '<ns1:body xmlns="foobar" xmlns:ns1="http://effbot.org/ns"><ns1:tag /></ns1:body>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_original_no_namespace(self): elem = ET.XML('<body><tag/></body>') EXPECTED_MSG = '^cannot use non-qualified names with default_namespace option$' with self.assertRaisesRegex(ValueError, EXPECTED_MSG): ET.tostring(elem, encoding='unicode', default_namespace='foobar') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_no_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1099,7 +1086,6 @@ def test_tostring_no_xml_declaration(self): '<body><tag /></body>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1107,7 +1093,6 @@ def test_tostring_xml_declaration(self): b"<?xml version='1.0' encoding='utf8'?>\n<body><tag /></body>" ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1115,7 +1100,6 @@ def test_tostring_xml_declaration_unicode_encoding(self): "<?xml version='1.0' encoding='utf-8'?>\n<body><tag /></body>" ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_cases(self): elem = ET.XML('<body><tag>ø</tag></body>') TESTCASES = [ @@ -1160,7 +1144,6 @@ def test_tostring_xml_declaration_cases(self): expected_retval ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1172,7 +1155,6 @@ def test_tostringlist_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1252,7 +1234,6 @@ def bxml(encoding): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_methods(self): # Test serialization methods. @@ -1268,7 +1249,6 @@ def test_methods(self): '<html><link><script>1 < 2</script></html>\n') self.assertEqual(serialize(e, method="text"), '1 < 2\n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue18347(self): e = ET.XML('<html><CamelCase>text</CamelCase></html>') self.assertEqual(serialize(e), @@ -1413,7 +1393,6 @@ def test_qname(self): self.assertNotEqual(q1, 'ns:tag') self.assertEqual(q1, '{ns}tag') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_public(self): # Test PUBLIC doctype. @@ -1496,7 +1475,6 @@ def test_processinginstruction(self): b"<?xml version='1.0' encoding='latin-1'?>\n" b"<?test <testing&>\xe3?>") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html @@ -1775,7 +1753,6 @@ def test_events_pi(self): self._feed(parser, "<?pitarget some text ?>\n") self.assert_events(parser, [('pi', (ET.PI, 'pitarget some text '))]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_sequence(self): # Test that events can be some sequence that's not just a tuple or list eventset = {'end', 'start'} @@ -1795,7 +1772,6 @@ def __next__(self): self._feed(parser, "<foo>bar</foo>") self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) @@ -2210,7 +2186,6 @@ def test_bug_xmltoolkit25(self): self.assertEqual(tree.findtext("tag"), 'text') self.assertEqual(tree.findtext("section/tag"), 'subtext') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit28(self): # .//tag causes exceptions @@ -2218,7 +2193,6 @@ def test_bug_xmltoolkit28(self): self.assertEqual(summarize_list(tree.findall(".//thead")), []) self.assertEqual(summarize_list(tree.findall(".//tbody")), ['tbody']) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkitX1(self): # dump() doesn't flush the output buffer @@ -2253,7 +2227,6 @@ def test_bug_xmltoolkit39(self): self.assertEqual(ET.tostring(tree, "utf-8"), b'<tag \xc3\xa4ttr="v\xc3\xa4lue" />') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit54(self): # problems handling internally defined entities @@ -2273,7 +2246,6 @@ def test_bug_xmltoolkit55(self): self.assertEqual(str(cm.exception), 'undefined entity &ldots;: line 1, column 36') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit60(self): # Handle crash in stream source. @@ -2301,7 +2273,6 @@ def test_bug_xmltoolkit62(self): self.assertEqual(t.find('.//paragraph').text, 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.gettrace(), "Skips under coverage.") def test_bug_xmltoolkit63(self): # Check reference leak. @@ -2329,7 +2300,6 @@ def test_bug_200708_newline(self): self.assertEqual(ET.tostring(ET.XML(ET.tostring(e))), b'<SomeTag text="def _f():&#10; return 3&#10;" />') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_close(self): # Test default builder. parser = ET.XMLParser() # default @@ -2422,7 +2392,6 @@ def test_bug_1534630(self): e = bob.close() self.assertEqual(serialize(e), '<tag />') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6233(self): e = ET.XML(b"<?xml version='1.0' encoding='utf-8'?>" b'<body>t\xc3\xa3g</body>') @@ -2743,7 +2712,6 @@ def test_pickle(self): self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle_issue18997(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): for dumper, loader in product(self.modules, repeat=2): @@ -3427,7 +3395,6 @@ def test_findall(self): self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), ['tag', 'tag']) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_test_find_with_ns(self): e = ET.XML(SAMPLE_XML_NS) self.assertEqual(summarize_list(e.findall('tag')), []) @@ -3438,7 +3405,6 @@ def test_test_find_with_ns(self): summarize_list(e.findall(".//{http://effbot.org/ns}tag")), ['{http://effbot.org/ns}tag'] * 3) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_different_nsmaps(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3456,7 +3422,6 @@ def test_findall_different_nsmaps(self): self.assertEqual(len(root.findall(".//xx:b", namespaces=nsmap)), 2) self.assertEqual(len(root.findall(".//b", namespaces=nsmap)), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_wildcard(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3501,7 +3466,6 @@ def test_findall_wildcard(self): self.assertEqual(summarize_list(root.findall(".//{}b")), summarize_list(root.findall(".//b"))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_find(self): e = ET.XML(SAMPLE_XML) with self.assertRaisesRegex(SyntaxError, 'cannot use absolute path'): @@ -3527,7 +3491,6 @@ class ElementIterTest(unittest.TestCase): def _ilist(self, elem, tag=None): return summarize_list(elem.iter(tag)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): doc = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>") self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) @@ -3576,7 +3539,6 @@ def test_corners(self): del a[1] self.assertEqual(self._ilist(a), ['a', 'd']) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_by_tag(self): doc = ET.XML(''' <document> @@ -3641,7 +3603,6 @@ def _check_sample1_element(self, e): self.assertEqual(child.tail, 'tail') self.assertEqual(child.attrib, {}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dummy_builder(self): class BaseDummyBuilder: def close(self): @@ -3689,7 +3650,6 @@ def test_treebuilder_pi(self): self.assertEqual(b.pi('target'), (len('target'), None)) self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text ')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. class TreeBuilderSubclass(ET.TreeBuilder): @@ -3751,14 +3711,12 @@ class TreeBuilderSubclass(ET.TreeBuilder): self.assertEqual(a[0].tail, 'tail') self.assertEqual(a.text, "text\n") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_treebuilder_elementfactory_none(self): parser = ET.XMLParser(target=ET.TreeBuilder(element_factory=None)) parser.feed(self.sample1) e = parser.close() self._check_sample1_element(e) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3773,7 +3731,6 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_comment_pi(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3789,7 +3746,6 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory(self): lst = [] def myfactory(tag, attrib): @@ -3813,13 +3769,11 @@ def _check_element_factory_class(self, cls): self.assertIsInstance(e, cls) self._check_sample1_element(e) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_subclass(self): class MyElement(ET.Element): pass self._check_element_factory_class(MyElement) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_pure_python_subclass(self): # Mimic SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the @@ -3851,7 +3805,6 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_builder_lookup_errors(self): class RaisingBuilder: def __init__(self, raise_in=None, what=ValueError): @@ -3892,14 +3845,12 @@ def _check_sample_element(self, e): self.assertEqual(e[0].tag, 'line') self.assertEqual(e[0].text, '22') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_args(self): parser2 = ET.XMLParser(encoding='utf-8', target=ET.TreeBuilder()) parser2.feed(self.sample1) self._check_sample_element(parser2.close()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyParser(ET.XMLParser): pass @@ -3907,7 +3858,6 @@ class MyParser(ET.XMLParser): parser.feed(self.sample1) self._check_sample_element(parser.close()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_warning(self): with warnings.catch_warnings(): warnings.simplefilter('error', DeprecationWarning) @@ -3946,7 +3896,6 @@ def doctype(self, name, pubid, system): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_inherited_doctype(self): '''Ensure that ordinary usage is not deprecated (Issue 19176)''' with warnings.catch_warnings(): @@ -3969,7 +3918,6 @@ def test_parse_string(self): class NamespaceParseTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_with_namespace(self): nsmap = {'h': 'hello', 'f': 'foo'} doc = ET.fromstring(SAMPLE_XML_NS_ELEMS) @@ -4146,7 +4094,6 @@ def f(): e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): # Test encoding issues. elem = ET.Element("tag") @@ -4216,7 +4163,6 @@ def test_encoding(self): ("<?xml version='1.0' encoding='%s'?>\n" "<tag key=\"åöö&lt;&gt;\" />" % enc).encode(enc)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4224,7 +4170,6 @@ def test_write_to_filename(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>&#248;</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4238,7 +4183,6 @@ def test_write_to_filename_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_as_unicode(self): self.addCleanup(os_helper.unlink, TESTFN) with open(TESTFN, 'w') as f: @@ -4250,7 +4194,6 @@ def test_write_to_filename_as_unicode(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b"<site>\xc3\xb8</site>") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_text_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4272,7 +4215,6 @@ def test_write_to_text_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>\xf8</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4282,7 +4224,6 @@ def test_write_to_binary_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>&#248;</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4300,7 +4241,6 @@ def test_write_to_binary_file_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_bom(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4321,28 +4261,24 @@ def test_write_to_binary_file_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site>\xf8</site>'''.encode("utf-16")) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_stringio(self): tree = ET.ElementTree() stream = io.StringIO('''<?xml version="1.0"?><site></site>''') tree.parse(stream) self.assertEqual(tree.getroot().tag, 'site') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_stringio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() tree.write(stream, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_bytesio(self): tree = ET.ElementTree() raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') tree.parse(raw) self.assertEqual(tree.getroot().tag, 'site') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_bytesio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4352,7 +4288,6 @@ def test_write_to_bytesio(self): class dummy: pass - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_text_reader(self): stream = io.StringIO('''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4361,7 +4296,6 @@ def test_read_from_user_text_reader(self): tree.parse(reader) self.assertEqual(tree.getroot().tag, 'site') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_text_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() @@ -4370,7 +4304,6 @@ def test_write_to_user_text_writer(self): tree.write(writer, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_binary_reader(self): raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4380,7 +4313,6 @@ def test_read_from_user_binary_reader(self): self.assertEqual(tree.getroot().tag, 'site') tree = ET.ElementTree() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4389,7 +4321,6 @@ def test_write_to_user_binary_writer(self): tree.write(writer) self.assertEqual(raw.getvalue(), b'''<site>&#248;</site>''') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer_with_bom(self): tree = ET.ElementTree(ET.XML('''<site />''')) raw = io.BytesIO() @@ -4402,7 +4333,6 @@ def test_write_to_user_binary_writer_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site />'''.encode("utf-16")) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_invariant(self): root = ET.fromstring('<tag>foo</tag>') self.assertEqual( @@ -4412,7 +4342,6 @@ def test_tostringlist_invariant(self): ET.tostring(root, 'utf-16'), b''.join(ET.tostringlist(root, 'utf-16'))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_short_empty_elements(self): root = ET.fromstring('<tag>a<x />b<y></y>c</tag>') self.assertEqual( @@ -4452,7 +4381,6 @@ def test_error_code(self): class KeywordArgsTest(unittest.TestCase): # Test various issues with keyword arguments passed to ET.Element # constructor and methods - @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue14818(self): x = ET.XML("<a>foo</a>") self.assertEqual(x.find('a', None), diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 8684e042bee..bb2a1ef0904 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -47,13 +47,11 @@ class XMLRPCTestCase(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_load(self): dump = xmlrpclib.dumps((alist,)) load = xmlrpclib.loads(dump) self.assertEqual(alist, load[0][0]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_bare_datetime(self): # This checks that an unwrapped datetime.date object can be handled # by the marshalling code. This can't be done via test_dump_load() @@ -88,7 +86,6 @@ def test_dump_bare_datetime(self): self.assertIsNone(m) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_datetime_before_1900(self): # same as before but with a date before 1900 dt = datetime.datetime(1, 2, 10, 11, 41, 23) @@ -107,7 +104,6 @@ def test_datetime_before_1900(self): self.assertIs(type(newdt), xmlrpclib.DateTime) self.assertIsNone(m) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_1164912 (self): d = xmlrpclib.DateTime() ((new_d,), dummy) = xmlrpclib.loads(xmlrpclib.dumps((d,), @@ -118,7 +114,6 @@ def test_bug_1164912 (self): s = xmlrpclib.dumps((new_d,), methodresponse=True) self.assertIsInstance(s, str) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_newstyle_class(self): class T(object): pass @@ -184,7 +179,6 @@ def dummy_write(s): m.dump_double(xmlrpclib.MAXINT + 42, dummy_write) m.dump_double(xmlrpclib.MININT - 42, dummy_write) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_none(self): value = alist + [None] arg1 = (alist + [None],) @@ -193,7 +187,7 @@ def test_dump_none(self): xmlrpclib.loads(strg)[0][0]) self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_encoding(self): value = {'key\u20ac\xa4': 'value\u20ac\xa4'} @@ -215,7 +209,6 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) @@ -235,7 +228,7 @@ def test_dump_bytes(self): self.assertIs(type(newvalue), xmlrpclib.Binary) self.assertIsNone(m) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_loads_unsupported(self): ResponseError = xmlrpclib.ResponseError data = '<params><param><value><spam/></value></param></params>' @@ -258,7 +251,6 @@ def check_loads(self, s, value, **kwargs): self.assertIs(type(newvalue), type(value)) self.assertIsNone(m) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_load_standard_types(self): check = self.check_loads check('string', 'string') @@ -286,7 +278,7 @@ def test_load_standard_types(self): '<member><name>a</name><value><int>1</int></value></member>' '</struct>', {'a': 1, 'b': 2}) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_load_extension_types(self): check = self.check_loads check('<nil/>', None) @@ -300,7 +292,6 @@ def test_load_extension_types(self): check('<bigdecimal>9876543210.0123456789</bigdecimal>', decimal.Decimal('9876543210.0123456789')) - @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name 'expat' is not defined def test_limit_int(self): check = self.check_loads maxdigits = 5000 @@ -320,7 +311,7 @@ def test_get_host_info(self): def test_ssl_presence(self): try: - import ssl + import ssl # noqa: F401 except ImportError: has_ssl = False else: @@ -332,7 +323,6 @@ def test_ssl_presence(self): except OSError: self.assertTrue(has_ssl) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_keepalive_disconnect(self): class RequestHandler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -472,7 +462,6 @@ def test_repr(self): self.assertEqual(repr(f), "<Fault 42: 'Test Fault'>") self.assertEqual(repr(f), str(f)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_fault(self): f = xmlrpclib.Fault(42, 'Test Fault') s = xmlrpclib.dumps((f,)) @@ -820,7 +809,6 @@ def tearDown(self): xmlrpc.server.SimpleXMLRPCServer._send_traceback_header = False class SimpleServerTestCase(BaseServerTestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple1(self): try: p = xmlrpclib.ServerProxy(URL) @@ -831,7 +819,6 @@ def test_simple1(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonascii(self): start_string = 'P\N{LATIN SMALL LETTER Y WITH CIRCUMFLEX}t' end_string = 'h\N{LATIN SMALL LETTER O WITH HORN}n' @@ -845,7 +832,7 @@ def test_nonascii(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_client_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -860,7 +847,6 @@ def test_client_encoding(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonascii_methodname(self): try: p = xmlrpclib.ServerProxy(URL, encoding='ascii') @@ -881,7 +867,6 @@ def test_404(self): self.assertEqual(response.status, 404) self.assertEqual(response.reason, 'Not Found') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection1(self): expected_methods = set(['pow', 'div', 'my_function', 'add', 'têšt', 'system.listMethods', 'system.methodHelp', @@ -898,7 +883,6 @@ def test_introspection1(self): self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection2(self): try: # test _methodHelp() @@ -911,7 +895,6 @@ def test_introspection2(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON @make_request_and_skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_introspection3(self): @@ -926,7 +909,6 @@ def test_introspection3(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection4(self): # the SimpleXMLRPCServer doesn't support signatures, but # at least check that we can try making the call @@ -940,7 +922,6 @@ def test_introspection4(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -958,7 +939,6 @@ def test_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_existing_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -980,7 +960,6 @@ def test_non_existing_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dotted_attribute(self): # Raises an AttributeError because private methods are not allowed. self.assertRaises(AttributeError, @@ -991,14 +970,12 @@ def test_dotted_attribute(self): # This avoids waiting for the socket timeout. self.test_simple1() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_allow_dotted_names_true(self): # XXX also need allow_dotted_names_false test. server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) data = server.Fixture.getData() self.assertEqual(data, '42') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_host(self): server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) self.assertEqual(server.add("a", "\xe9"), "a\xe9") @@ -1013,7 +990,6 @@ def test_partial_post(self): 'Accept-Encoding: identity\r\n' 'Content-Length: 0\r\n\r\n'.encode('ascii')) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_context_manager(self): with xmlrpclib.ServerProxy(URL) as server: server.add(2, 3) @@ -1022,7 +998,6 @@ def test_context_manager(self): self.assertEqual(server('transport')._connection, (None, None)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_context_manager_method_error(self): try: with xmlrpclib.ServerProxy(URL) as server: @@ -1038,7 +1013,7 @@ class SimpleServerEncodingTestCase(BaseServerTestCase): def threadFunc(evt, numrequests, requestHandler=None, encoding=None): http_server(evt, numrequests, requestHandler, 'iso-8859-15') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_server_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -1057,67 +1032,56 @@ def test_server_encoding(self): class MultiPathServerTestCase(BaseServerTestCase): threadFunc = staticmethod(http_multi_server) request_count = 2 - @unittest.expectedFailure # TODO: RUSTPYTHON def test_path1(self): p = xmlrpclib.ServerProxy(URL+"/foo") self.assertEqual(p.pow(6,8), 6**8) self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_path2(self): p = xmlrpclib.ServerProxy(URL+"/foo/bar") self.assertEqual(p.add(6,8), 6+8) self.assertRaises(xmlrpclib.Fault, p.pow, 6, 8) - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_path3(self): p = xmlrpclib.ServerProxy(URL+"/is/broken") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_invalid_path(self): p = xmlrpclib.ServerProxy(URL+"/invalid") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_path_query_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v#frag") self.assertEqual(p.test(), "/foo?k=v#frag") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_path_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo#frag") self.assertEqual(p.test(), "/foo#frag") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_path_query(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v") self.assertEqual(p.test(), "/foo?k=v") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_empty_path(self): p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.test(), "/RPC2") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_root_path(self): p = xmlrpclib.ServerProxy(URL + "/") self.assertEqual(p.test(), "/") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_empty_path_query(self): p = xmlrpclib.ServerProxy(URL + "?k=v") self.assertEqual(p.test(), "?k=v") - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('walltime') def test_empty_path_fragment(self): p = xmlrpclib.ServerProxy(URL + "#frag") @@ -1151,7 +1115,6 @@ def setUp(self): #A test case that verifies that a server using the HTTP/1.1 keep-alive mechanism #does indeed serve subsequent requests on the same connection class KeepaliveServerTestCase1(BaseKeepaliveServerTestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_two(self): p = xmlrpclib.ServerProxy(URL) #do three requests. @@ -1170,7 +1133,6 @@ def test_two(self): #test special attribute access on the serverproxy, through the __call__ #function. -@unittest.skip("TODO: RUSTPYTHON, appears to hang") class KeepaliveServerTestCase2(BaseKeepaliveServerTestCase): #ask for two keepalive requests to be handled. request_count=2 @@ -1235,7 +1197,6 @@ def send_content(self, connection, body): def setUp(self): BaseServerTestCase.setUp(self) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_gzip_request(self): t = self.Transport() t.encode_threshold = None @@ -1259,7 +1220,6 @@ def test_bad_gzip_request(self): p.pow(6, 8) p("close")() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_gzip_response(self): t = self.Transport() p = xmlrpclib.ServerProxy(URL, transport=t) @@ -1318,7 +1278,6 @@ def assertContainsAdditionalHeaders(self, headers, additional): for key, value in additional.items(): self.assertEqual(headers.get(key), value) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_header(self): p = xmlrpclib.ServerProxy(URL, headers=[('X-Test', 'foo')]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1326,7 +1285,6 @@ def test_header(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_many(self): p = xmlrpclib.ServerProxy( URL, headers=[('X-Test', 'foo'), ('X-Test-Second', 'bar')]) @@ -1336,7 +1294,6 @@ def test_header_many(self): self.assertContainsAdditionalHeaders( headers, {'X-Test': 'foo', 'X-Test-Second': 'bar'}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_empty(self): p = xmlrpclib.ServerProxy(URL, headers=[]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1344,7 +1301,6 @@ def test_header_empty(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_tuple(self): p = xmlrpclib.ServerProxy(URL, headers=(('X-Test', 'foo'),)) self.assertEqual(p.pow(6, 8), 6**8) @@ -1352,7 +1308,6 @@ def test_header_tuple(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_items(self): p = xmlrpclib.ServerProxy(URL, headers={'X-Test': 'foo'}.items()) self.assertEqual(p.pow(6, 8), 6**8) @@ -1411,7 +1366,6 @@ def tearDown(self): default_class = http.client.HTTPMessage xmlrpc.server.SimpleXMLRPCRequestHandler.MessageClass = default_class - @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): # check that flag is false by default flagval = xmlrpc.server.SimpleXMLRPCServer._send_traceback_header @@ -1506,7 +1460,6 @@ def test_cgi_get(self): self.assertEqual(message, 'Bad Request') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cgi_xmlrpc_response(self): data = """<?xml version='1.0'?> <methodCall> @@ -1552,7 +1505,6 @@ def test_cgi_xmlrpc_response(self): class UseBuiltinTypesTestCase(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_use_builtin_types(self): # SimpleXMLRPCDispatcher.__init__ accepts use_builtin_types, which # makes all dispatch of binary data as bytes instances, and all diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 88fa1b88c90..e0e3db0839e 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -786,7 +786,6 @@ def outer(): repr(value), ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgen_that_returns(self): """ Test throwing GeneratorExit into a subgenerator that @@ -817,7 +816,6 @@ def g(): "Enter f", ]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgenerator_that_yields(self): """ Test throwing GeneratorExit into a subgenerator that @@ -1135,7 +1133,6 @@ def outer(): self.assertIsNone(caught.exception.__context__) self.assert_stop_iteration(g) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: GeneratorExit() is not GeneratorExit() def test_close_and_throw_raise_generator_exit(self): yielded_first = object() @@ -1524,7 +1521,6 @@ def outer(): self.assertIsNone(caught.exception.__context__) self.assert_stop_iteration(g) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_close_and_throw_return(self): yielded_first = object() diff --git a/Lib/test/test_zipfile/__init__.py b/Lib/test/test_zipfile/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_zipfile/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zipfile/__main__.py b/Lib/test/test_zipfile/__main__.py new file mode 100644 index 00000000000..e25ac946edf --- /dev/null +++ b/Lib/test/test_zipfile/__main__.py @@ -0,0 +1,7 @@ +import unittest + +from . import load_tests # noqa: F401 + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_zipfile/_path/__init__.py b/Lib/test/test_zipfile/_path/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_zipfile/_path/_functools.py b/Lib/test/test_zipfile/_path/_functools.py new file mode 100644 index 00000000000..75f2b20e06d --- /dev/null +++ b/Lib/test/test_zipfile/_path/_functools.py @@ -0,0 +1,9 @@ +import functools + + +# from jaraco.functools 3.5.2 +def compose(*funcs): + def compose_two(f1, f2): + return lambda *args, **kwargs: f1(f2(*args, **kwargs)) + + return functools.reduce(compose_two, funcs) diff --git a/Lib/test/test_zipfile/_path/_itertools.py b/Lib/test/test_zipfile/_path/_itertools.py new file mode 100644 index 00000000000..f735dd21733 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_itertools.py @@ -0,0 +1,79 @@ +import itertools +from collections import deque +from itertools import islice + + +# from jaraco.itertools 6.3.0 +class Counter: + """ + Wrap an iterable in an object that stores the count of items + that pass through it. + + >>> items = Counter(range(20)) + >>> items.count + 0 + >>> values = list(items) + >>> items.count + 20 + """ + + def __init__(self, i): + self.count = 0 + self.iter = zip(itertools.count(1), i) + + def __iter__(self): + return self + + def __next__(self): + self.count, result = next(self.iter) + return result + + +# from more_itertools v8.13.0 +def always_iterable(obj, base_type=(str, bytes)): + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) + + +# from more_itertools v9.0.0 +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) diff --git a/Lib/test/test_zipfile/_path/_support.py b/Lib/test/test_zipfile/_path/_support.py new file mode 100644 index 00000000000..1afdf3b3a77 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_support.py @@ -0,0 +1,9 @@ +import importlib +import unittest + + +def import_or_skip(name): + try: + return importlib.import_module(name) + except ImportError: # pragma: no cover + raise unittest.SkipTest(f'Unable to import {name}') diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py new file mode 100644 index 00000000000..00a9eaf2f99 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -0,0 +1,39 @@ +import functools +import types + +from ._itertools import always_iterable + + +def parameterize(names, value_groups): + """ + Decorate a test method to run it as a set of subtests. + + Modeled after pytest.parametrize. + """ + + def decorator(func): + @functools.wraps(func) + def wrapped(self): + for values in value_groups: + resolved = map(Invoked.eval, always_iterable(values)) + params = dict(zip(always_iterable(names), resolved)) + with self.subTest(**params): + func(self, **params) + + return wrapped + + return decorator + + +class Invoked(types.SimpleNamespace): + """ + Wrap a function to be invoked for each usage. + """ + + @classmethod + def wrap(cls, func): + return cls(func=func) + + @classmethod + def eval(cls, cand): + return cand.func() if isinstance(cand, cls) else cand diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py new file mode 100644 index 00000000000..7c108fc6ab8 --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -0,0 +1,105 @@ +import io +import itertools +import math +import re +import string +import unittest +import zipfile + +from ._functools import compose +from ._itertools import consume +from ._support import import_or_skip + +big_o = import_or_skip('big_o') +pytest = import_or_skip('pytest') + + +class TestComplexity(unittest.TestCase): + @pytest.mark.flaky + def test_implied_dirs_performance(self): + best, others = big_o.big_o( + compose(consume, zipfile._path.CompleteDirs._implied_dirs), + lambda size: [ + '/'.join(string.ascii_lowercase + str(n)) for n in range(size) + ], + max_n=1000, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + def make_zip_path(self, depth=1, width=1) -> zipfile.Path: + """ + Construct a Path with width files at every level of depth. + """ + zf = zipfile.ZipFile(io.BytesIO(), mode='w') + pairs = itertools.product(self.make_deep_paths(depth), self.make_names(width)) + for path, name in pairs: + zf.writestr(f"{path}{name}.txt", b'') + zf.filename = "big un.zip" + return zipfile.Path(zf) + + @classmethod + def make_names(cls, width, letters=string.ascii_lowercase): + """ + >>> list(TestComplexity.make_names(1)) + ['a'] + >>> list(TestComplexity.make_names(2)) + ['a', 'b'] + >>> list(TestComplexity.make_names(30)) + ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] + """ + # determine how many products are needed to produce width + n_products = max(1, math.ceil(math.log(width, len(letters)))) + inputs = (letters,) * n_products + combinations = itertools.product(*inputs) + names = map(''.join, combinations) + return itertools.islice(names, width) + + @classmethod + def make_deep_paths(cls, depth): + return map(cls.make_deep_path, range(depth)) + + @classmethod + def make_deep_path(cls, depth): + return ''.join(('d/',) * depth) + + def test_baseline_regex_complexity(self): + best, others = big_o.big_o( + lambda path: re.fullmatch(r'[^/]*\\.txt', path), + self.make_deep_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Constant + + @pytest.mark.flaky + def test_glob_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + self.make_zip_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(width=size), + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width_and_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(depth=size, width=size), + max_n=10, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py new file mode 100644 index 00000000000..f34251bc93c --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -0,0 +1,691 @@ +import contextlib +import io +import itertools +import pathlib +import pickle +import stat +import sys +import time +import unittest +import zipfile +import zipfile._path + +from test.support.os_helper import FakePath, temp_dir + +from ._functools import compose +from ._itertools import Counter +from ._test_params import Invoked, parameterize + + +class jaraco: + class itertools: + Counter = Counter + + +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 + + +def build_alpharep_fixture(): + """ + Create a zip file with this structure: + + . + ├── a.txt + ├── n.txt (-> a.txt) + ├── b + │ ├── c.txt + │ ├── d + │ │ └── e.txt + │ └── f.txt + ├── g + │ └── h + │ └── i.txt + └── j + ├── k.bin + ├── l.baz + └── m.bar + + This fixture has the following key characteristics: + + - a file at the root (a) + - a file two levels deep (b/d/e) + - multiple files in a directory (b/c, b/f) + - a directory containing only a directory (g/h) + - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) + + "alpha" because it uses alphabet + "rep" because it's a representative example + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("a.txt", b"content of a") + zf.writestr("b/c.txt", b"content of c") + zf.writestr("b/d/e.txt", b"content of e") + zf.writestr("b/f.txt", b"content of f") + zf.writestr("g/h/i.txt", b"content of i") + zf.writestr("j/k.bin", b"content of k") + zf.writestr("j/l.baz", b"content of l") + zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + + zf.filename = "alpharep.zip" + return zf + + +alpharep_generators = [ + Invoked.wrap(build_alpharep_fixture), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), +] + +pass_alpharep = parameterize(['alpharep'], alpharep_generators) + + +class TestPath(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def zipfile_ondisk(self, alpharep): + tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) + buffer = alpharep.fp + alpharep.close() + path = tmpdir / alpharep.filename + with path.open("wb") as strm: + strm.write(buffer.getvalue()) + return path + + @pass_alpharep + def test_iterdir_and_types(self, alpharep): + root = zipfile.Path(alpharep) + assert root.is_dir() + a, n, b, g, j = root.iterdir() + assert a.is_file() + assert b.is_dir() + assert g.is_dir() + c, f, d = b.iterdir() + assert c.is_file() and f.is_file() + (e,) = d.iterdir() + assert e.is_file() + (h,) = g.iterdir() + (i,) = h.iterdir() + assert i.is_file() + + @pass_alpharep + def test_is_file_missing(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('missing.txt').is_file() + + @pass_alpharep + def test_iterdir_on_file(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with self.assertRaises(ValueError): + a.iterdir() + + @pass_alpharep + def test_subdir_is_dir(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + + @pass_alpharep + def test_open(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with a.open(encoding="utf-8") as strm: + data = strm.read() + self.assertEqual(data, "content of a") + with a.open('r', "utf-8") as strm: # not a kw, no gh-101144 TypeError + data = strm.read() + self.assertEqual(data, "content of a") + + def test_open_encoding_utf16(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/16.txt", "This was utf-16".encode("utf-16")) + zf.filename = "test_open_utf16.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("16.txt") + with u16.open('r', "utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + with u16.open(encoding="utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + + def test_open_encoding_errors(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/bad-utf8.bin", b"invalid utf-8: \xff\xff.") + zf.filename = "test_read_text_encoding_errors.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("bad-utf8.bin") + + # encoding= as a positional argument for gh-101144. + data = u16.read_text("utf-8", errors="ignore") + assert data == "invalid utf-8: ." + with u16.open("r", "utf-8", errors="surrogateescape") as f: + assert f.read() == "invalid utf-8: \udcff\udcff." + + # encoding= both positional and keyword is an error; gh-101144. + with self.assertRaisesRegex(TypeError, "encoding"): + data = u16.read_text("utf-8", encoding="utf-8") + + # both keyword arguments work. + with u16.open("r", encoding="utf-8", errors="strict") as f: + # error during decoding with wrong codec. + with self.assertRaises(UnicodeDecodeError): + f.read() + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + @pass_alpharep + def test_encoding_warnings(self, alpharep): + """EncodingWarning must blame the read_text and open calls.""" + assert sys.flags.warn_default_encoding + root = zipfile.Path(alpharep) + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").read_text() + assert __file__ == wc.filename + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").open("r").close() + assert __file__ == wc.filename + + def test_open_write(self): + """ + If the zipfile is open for write, it should be possible to + write bytes or text to it. + """ + zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) + with zf.joinpath('file.bin').open('wb') as strm: + strm.write(b'binary contents') + with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: + strm.write('text file') + + @pass_alpharep + def test_open_extant_directory(self, alpharep): + """ + Attempting to open a directory raises IsADirectoryError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(IsADirectoryError): + zf.joinpath('b').open() + + @pass_alpharep + def test_open_binary_invalid_args(self, alpharep): + root = zipfile.Path(alpharep) + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', encoding='utf-8') + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', 'utf-8') + + @pass_alpharep + def test_open_missing_directory(self, alpharep): + """ + Attempting to open a missing directory raises FileNotFoundError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(FileNotFoundError): + zf.joinpath('z').open() + + @pass_alpharep + def test_read(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + assert a.read_text(encoding="utf-8") == "content of a" + # Also check positional encoding arg (gh-101144). + assert a.read_text("utf-8") == "content of a" + assert a.read_bytes() == b"content of a" + + @pass_alpharep + def test_joinpath(self, alpharep): + root = zipfile.Path(alpharep) + a = root.joinpath("a.txt") + assert a.is_file() + e = root.joinpath("b").joinpath("d").joinpath("e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_joinpath_multiple(self, alpharep): + root = zipfile.Path(alpharep) + e = root.joinpath("b", "d", "e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_traverse_truediv(self, alpharep): + root = zipfile.Path(alpharep) + a = root / "a.txt" + assert a.is_file() + e = root / "b" / "d" / "e.txt" + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_pathlike_construction(self, alpharep): + """ + zipfile.Path should be constructable from a path-like object + """ + zipfile_ondisk = self.zipfile_ondisk(alpharep) + pathlike = FakePath(str(zipfile_ondisk)) + root = zipfile.Path(pathlike) + root.root.close() + + @pass_alpharep + def test_traverse_pathlike(self, alpharep): + root = zipfile.Path(alpharep) + root / FakePath("a") + + @pass_alpharep + def test_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'a').parent.at == '' + assert (root / 'a' / 'b').parent.at == 'a/' + + @pass_alpharep + def test_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').parent.at == '' + assert (root / 'b/').parent.at == '' + + @pass_alpharep + def test_missing_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'missing dir/').parent.at == '' + + @pass_alpharep + def test_mutability(self, alpharep): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any(child.name == 'foo.txt' for child in root.iterdir()) + assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' + (baz,) = (root / 'bar').iterdir() + assert baz.read_text(encoding="utf-8") == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2**13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + + @pass_alpharep + def test_read_does_not_close(self, alpharep): + alpharep = self.zipfile_ondisk(alpharep) + with zipfile.ZipFile(alpharep) as file: + for rep in range(2): + zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") + + @pass_alpharep + def test_subclass(self, alpharep): + class Subclass(zipfile.Path): + pass + + root = Subclass(alpharep) + assert isinstance(root / 'b', Subclass) + + @pass_alpharep + def test_filename(self, alpharep): + root = zipfile.Path(alpharep) + assert root.filename == pathlib.Path('alpharep.zip') + + @pass_alpharep + def test_root_name(self, alpharep): + """ + The name of the root should be the name of the zipfile + """ + root = zipfile.Path(alpharep) + assert root.name == 'alpharep.zip' == root.filename.name + + @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + root.root.close() + + @pass_alpharep + def test_suffix(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffix == '.zip' == root.filename.suffix + + b = root / "b.txt" + assert b.suffix == ".txt" + + c = root / "c" / "filename.tar.gz" + assert c.suffix == ".gz" + + d = root / "d" + assert d.suffix == "" + + @pass_alpharep + def test_suffixes(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffixes == ['.zip'] == root.filename.suffixes + + b = root / 'b.txt' + assert b.suffixes == ['.txt'] + + c = root / 'c' / 'filename.tar.gz' + assert c.suffixes == ['.tar', '.gz'] + + d = root / 'd' + assert d.suffixes == [] + + e = root / '.hgrc' + assert e.suffixes == [] + + @pass_alpharep + def test_suffix_no_filename(self, alpharep): + alpharep.filename = None + root = zipfile.Path(alpharep) + assert root.joinpath('example').suffix == "" + assert root.joinpath('example').suffixes == [] + + @pass_alpharep + def test_stem(self, alpharep): + """ + The final path component, without its suffix + """ + root = zipfile.Path(alpharep) + assert root.stem == 'alpharep' == root.filename.stem + + b = root / "b.txt" + assert b.stem == "b" + + c = root / "c" / "filename.tar.gz" + assert c.stem == "filename.tar" + + d = root / "d" + assert d.stem == "d" + + assert (root / ".gitignore").stem == ".gitignore" + + @pass_alpharep + def test_root_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert root.parent == pathlib.Path('.') + root.root.filename = 'foo/bar.zip' + assert root.parent == pathlib.Path('foo') + + @pass_alpharep + def test_root_unnamed(self, alpharep): + """ + It is an error to attempt to get the name + or parent of an unnamed zipfile. + """ + alpharep.filename = None + root = zipfile.Path(alpharep) + with self.assertRaises(TypeError): + root.name + with self.assertRaises(TypeError): + root.parent + + # .name and .parent should still work on subs + sub = root / "b" + assert sub.name == "b" + assert sub.parent + + @pass_alpharep + def test_match_and_glob(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.match("*.txt") + + assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + assert list(root.glob("b/*.txt")) == [ + zipfile.Path(alpharep, "b/c.txt"), + zipfile.Path(alpharep, "b/f.txt"), + ] + + @pass_alpharep + def test_glob_recursive(self, alpharep): + root = zipfile.Path(alpharep) + files = root.glob("**/*.txt") + assert all(each.match("*.txt") for each in files) + + assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + + @pass_alpharep + def test_glob_dirs(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('b')) == [zipfile.Path(alpharep, "b/")] + assert list(root.glob('b*')) == [zipfile.Path(alpharep, "b/")] + + @pass_alpharep + def test_glob_subdir(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('g/h')) == [zipfile.Path(alpharep, "g/h/")] + assert list(root.glob('g*/h*')) == [zipfile.Path(alpharep, "g/h/")] + + @pass_alpharep + def test_glob_subdirs(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*/i.txt")) == [] + assert list(root.rglob("*/i.txt")) == [zipfile.Path(alpharep, "g/h/i.txt")] + + @pass_alpharep + def test_glob_does_not_overmatch_dot(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*.xt")) == [] + + @pass_alpharep + def test_glob_single_char(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("a?txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[.]txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[?]txt")) == [] + + @pass_alpharep + def test_glob_chars(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("j/?.b[ai][nz]")) == [ + zipfile.Path(alpharep, "j/k.bin"), + zipfile.Path(alpharep, "j/l.baz"), + ] + + def test_glob_empty(self): + root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) + with self.assertRaises(ValueError): + root.glob('') + + @pass_alpharep + def test_eq_hash(self, alpharep): + root = zipfile.Path(alpharep) + assert root == zipfile.Path(alpharep) + + assert root != (root / "a.txt") + assert (root / "a.txt") == (root / "a.txt") + + root = zipfile.Path(alpharep) + assert root in {root} + + @pass_alpharep + def test_is_symlink(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() + + @pass_alpharep + def test_relative_to(self, alpharep): + root = zipfile.Path(alpharep) + relative = root.joinpath("b", "c.txt").relative_to(root / "b") + assert str(relative) == "c.txt" + + relative = root.joinpath("b", "d", "e.txt").relative_to(root / "b") + assert str(relative) == "d/e.txt" + + @pass_alpharep + def test_inheritance(self, alpharep): + cls = type('PathChild', (zipfile.Path,), {}) + file = cls(alpharep).joinpath('some dir').parent + assert isinstance(file, cls) + + @parameterize( + ['alpharep', 'path_type', 'subpath'], + itertools.product( + alpharep_generators, + [str, FakePath], + ['', 'b/'], + ), + ) + def test_pickle(self, alpharep, path_type, subpath): + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() + restored_1 = pickle.loads(saved_1) + first, *rest = restored_1.iterdir() + assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() + + @pass_alpharep + def test_extract_orig_with_implied_dirs(self, alpharep): + """ + A zip file wrapped in a Path should extract even with implied dirs. + """ + source_path = self.zipfile_ondisk(alpharep) + zf = zipfile.ZipFile(source_path) + # wrap the zipfile for its side effect + zipfile.Path(zf) + zf.extractall(source_path.parent) + zf.close() + + @pass_alpharep + def test_getinfo_missing(self, alpharep): + """ + Validate behavior of getinfo on original zipfile after wrapping. + """ + zipfile.Path(alpharep) + with self.assertRaises(KeyError): + alpharep.getinfo('does-not-exist') + + def test_malformed_paths(self): + """ + Path should handle malformed paths gracefully. + + Paths with leading slashes are not visible. + + Paths with dots are treated like regular files. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("/one-slash.txt", b"content") + zf.writestr("//two-slash.txt", b"content") + zf.writestr("../parent.txt", b"content") + zf.filename = '' + root = zipfile.Path(zf) + assert list(map(str, root.iterdir())) == ['../'] + assert root.joinpath('..').joinpath('parent.txt').read_bytes() == b'content' + + def test_unsupported_names(self): + """ + Path segments with special characters are readable. + + On some platforms or file systems, characters like + ``:`` and ``?`` are not allowed, but they are valid + in the zip file. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("path?", b"content") + zf.writestr("V: NMS.flac", b"fLaC...") + zf.filename = '' + root = zipfile.Path(zf) + contents = root.iterdir() + assert next(contents).name == 'path?' + assert next(contents).name == 'V: NMS.flac' + assert root.joinpath('V: NMS.flac').read_bytes() == b"fLaC..." + + def test_backslash_not_separator(self): + """ + In a zip file, backslashes are not separators. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.filename = '' + root = zipfile.Path(zf) + (first,) = root.iterdir() + assert not first.is_dir() + assert first.name == 'foo\\bar' + + @pass_alpharep + def test_interface(self, alpharep): + from importlib.resources.abc import Traversable + + zf = zipfile.Path(alpharep) + assert isinstance(zf, Traversable) + + +class DirtyZipInfo(zipfile.ZipInfo): + """ + Bypass name sanitization. + """ + + def __init__(self, filename, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.filename = filename + + @classmethod + def for_name(cls, name, archive): + """ + Construct the same way that ZipFile.writestr does. + + TODO: extract this functionality and re-use + """ + self = cls(filename=name, date_time=time.localtime(time.time())[:6]) + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py new file mode 100644 index 00000000000..7418391abad --- /dev/null +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -0,0 +1,3 @@ +from . import test_path + +__name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile/test_core.py similarity index 69% rename from Lib/test/test_zipfile.py rename to Lib/test/test_zipfile/test_core.py index 0a50f036046..63413d7b944 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1,12 +1,11 @@ +import _pyio import array import contextlib import importlib.util import io import itertools import os -import pathlib import posixpath -import string import struct import subprocess import sys @@ -14,16 +13,20 @@ import unittest import unittest.mock as mock import zipfile -import functools from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper -from test.support import (findfile, requires_zlib, requires_bz2, - requires_lzma, captured_stdout) -from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd +from test.support import ( + findfile, requires_zlib, requires_bz2, requires_lzma, + captured_stdout, captured_stderr, requires_subprocess +) +from test.support.os_helper import ( + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath +) TESTFN2 = TESTFN + "2" @@ -120,8 +123,9 @@ def zip_test(self, f, compression, compresslevel=None): self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) - # Check that testzip doesn't raise an exception - zipfp.testzip() + # Check that testzip thinks the archive is ok + # (it returns None if all contents could be read properly) + self.assertIsNone(zipfp.testzip()) def test_basic(self): for f in get_files(self): @@ -156,7 +160,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -298,26 +302,26 @@ def test_low_compression(self): self.assertEqual(openobj.read(1), b'2') def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compresslevel, 1) + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -386,7 +390,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -405,7 +408,7 @@ def test_per_file_compresslevel(self): one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') self.assertEqual(one_info._compresslevel, 1) - self.assertEqual(nine_info._compresslevel, 9) + self.assertEqual(nine_info.compress_level, 9) def test_writing_errors(self): class BrokenFile(io.BytesIO): @@ -443,6 +446,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'rb') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'rb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -574,17 +598,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -744,8 +767,8 @@ def zip_test(self, f, compression): self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) - # Check that testzip doesn't raise an exception - zipfp.testzip() + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) def test_basic(self): for f in get_files(self): @@ -861,6 +884,8 @@ def make_zip64_file( self, file_size_64_set=False, file_size_extra=False, compress_size_64_set=False, compress_size_extra=False, header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, ): """Generate bytes sequence for a zip with (incomplete) zip64 data. @@ -914,6 +939,12 @@ def make_zip64_file( central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) + if end_of_central_dir_size is None: + end_of_central_dir_size = 44 + len(extensible_data) + if offset_to_end_of_central_dir is None: + offset_to_end_of_central_dir = (108 + + 8 * len(local_zip64_fields) + + 8 * len(central_zip64_fields)) local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) @@ -942,14 +973,17 @@ def make_zip64_file( + filename + central_extra # Zip64 end of central directory - + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-" - + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + + b"PK\x06\x06" + + struct.pack('<Q', end_of_central_dir_size) + + b"-\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" + central_dir_size + offset_to_central_dir + + extensible_data # Zip64 end of central directory locator - + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01" - + b"\x00\x00\x00" + + b"PK\x06\x07\x00\x00\x00\x00" + + struct.pack('<Q', offset_to_end_of_central_dir) + + b"\x01\x00\x00\x00" # end of central directory + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" + b"\x00\x00\x00\x00" @@ -980,6 +1014,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) self.assertIn('file size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_file_size_extra))) # zip64 file size present, zip64 compress size present, one field in # extra, expecting two, equals missing compress size. @@ -991,6 +1026,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 compress size present, no fields in extra, expecting one, # equals missing compress size. @@ -1000,6 +1036,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 file size present, zip64 compress size present, zip64 header # offset present, two fields in extra, expecting three, equals missing @@ -1014,6 +1051,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 compress size present, zip64 header offset present, one field # in extra, expecting two, equals missing header offset @@ -1026,6 +1064,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 file size present, zip64 header offset present, one field in # extra, expecting two, equals missing header offset @@ -1038,6 +1077,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 header offset present, no fields in extra, expecting one, # equals missing header offset @@ -1049,6 +1089,63 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + def test_bad_zip64_end_of_central_dir(self): + zipdata = self.make_zip64_file(end_of_central_dir_size=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(end_of_central_dir_size=100) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=1000) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*locator'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_end_of_central_dir_record_not_found(self): + zipdata = self.make_zip64_file() + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_extensible_data(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + with zipfile.ZipFile(io.BytesIO(zipdata)) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(zipdata))) + + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(b'prepended' + zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(b'prepended' + zipdata))) def test_generated_valid_zip64_extra(self): # These values are what is set in the make_zip64_file method. @@ -1077,6 +1174,159 @@ def test_generated_valid_zip64_extra(self): self.assertEqual(zinfo.header_offset, expected_header_offset) self.assertEqual(zf.read(zinfo), expected_content) + def test_force_zip64(self): + """Test that forcing zip64 extensions correctly notes this in the zip file""" + + # GH-103861 describes an issue where forcing a small file to use zip64 + # extensions would add a zip64 extra record, but not change the data + # sizes to 0xFFFFFFFF to indicate to the extractor that the zip64 + # record should be read. Additionally, it would not set the required + # version to indicate that zip64 extensions are required to extract it. + # This test replicates the situation and reads the raw data to specifically ensure: + # - The required extract version is always >= ZIP64_VERSION + # - The compressed and uncompressed size in the file headers are both + # 0xFFFFFFFF (ie. point to zip64 record) + # - The zip64 record is provided and has the correct sizes in it + # Other aspects of the zip are checked as well, but verifying the above is the main goal. + # Because this is hard to verify by parsing the data as a zip, the raw + # bytes are checked to ensure that they line up with the zip spec. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.5.3 for zip64 extra field + + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=True) as zi: + zi.write(b"_") + + zipdata = data.getvalue() + + # pull out and check zip information + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQx4s", zipdata[:63]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 1) # uncompressed size + self.assertEqual(ex_csize, 1) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + z = zipfile.ZipFile(io.BytesIO(zipdata)) + zinfos = z.infolist() + self.assertEqual(len(zinfos), 1) + self.assertGreaterEqual(zinfos[0].extract_version, zipfile.ZIP64_VERSION) # requires zip64 to extract + + def test_unseekable_zip_unknown_filesize(self): + """Test that creating a zip with/without seeking will raise a RuntimeError if zip64 was required but not used""" + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=False) as zi: + zi.write(b"_" * (zipfile.ZIP64_LIMIT + 1)) + + self.assertRaises(RuntimeError, make_zip, io.BytesIO()) + self.assertRaises(RuntimeError, make_zip, Unseekable(io.BytesIO())) + + def test_zip64_required_not_allowed_fail(self): + """Test that trying to add a large file to a zip that doesn't allow zip64 extensions fails on add""" + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=False) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = zipfile.ZIP64_LIMIT + 1 + zf.open(info, mode="w") + + self.assertRaises(zipfile.LargeZipFile, make_zip, io.BytesIO()) + self.assertRaises(zipfile.LargeZipFile, make_zip, Unseekable(io.BytesIO())) + + def test_unseekable_zip_known_filesize(self): + """Test that creating a zip without seeking will use zip64 extensions if the file size is provided up-front""" + + # This test ensures that the zip will use a zip64 data descriptor (same + # as a regular data descriptor except the sizes are 8 bytes instead of + # 4) record to communicate the size of a file if the zip is being + # written to an unseekable stream. + # Because this sort of thing is hard to verify by parsing the data back + # in as a zip, this test looks at the raw bytes created to ensure that + # the correct data has been generated. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.3.9 for the data descriptor + # - 4.5.3 for zip64 extra field + + file_size = zipfile.ZIP64_LIMIT + 1 + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = file_size + with zf.open(info, mode="w", force_zip64=False) as zi: + zi.write(b"_" * file_size) + return fp + + # check seekable file information + seekable_data = make_zip(io.BytesIO()).getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s".format(file_size), seekable_data[:62 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, file_size) # uncompressed size + self.assertEqual(ex_csize, file_size) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + # check unseekable file information + unseekable_data = make_zip(Unseekable(io.BytesIO())).fp.getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + dd_header, dd_usize, dd_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s4xQQ4s".format(file_size), unseekable_data[:86 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual("{:b}".format(flags), "1000") # streaming flag set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 0) # uncompressed size - 0 to defer to data descriptor + self.assertEqual(ex_csize, 0) # compressed size - 0 to defer to data descriptor + self.assertEqual(dd_header, b"PK\07\x08") # data descriptor + self.assertEqual(dd_usize, file_size) # file size (8 bytes because zip64) + self.assertEqual(dd_csize, file_size) # compressed size (8 bytes because zip64) + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + @requires_zlib() class DeflateTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, @@ -1128,6 +1378,25 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1162,8 +1431,7 @@ def requiresWriteAccess(self, path): self.skipTest('requires write access to the installed location') unlink(filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_pyfile(self): self.requiresWriteAccess(os.path.dirname(__file__)) with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: @@ -1194,8 +1462,7 @@ def test_write_pyfile(self): self.assertNotIn(bn, zipfp.namelist()) self.assertCompiledIn(bn, zipfp.namelist()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_python_package(self): import email packagedir = os.path.dirname(email.__file__) @@ -1210,8 +1477,7 @@ def test_write_python_package(self): self.assertCompiledIn('email/__init__.py', names) self.assertCompiledIn('email/mime/text.py', names) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: module 'os' has no attribute 'supports_effective_ids' def test_write_filtered_python_package(self): import test packagedir = os.path.dirname(test.__file__) @@ -1242,8 +1508,7 @@ def filter(path): print(reportStr) self.assertTrue('SyntaxError' not in reportStr) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_with_optimization(self): import email packagedir = os.path.dirname(email.__file__) @@ -1338,7 +1603,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1396,7 +1661,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1431,7 +1696,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1444,6 +1709,8 @@ def test_sanitize_windows_name(self): self.assertEqual(san(r',,?,C:,foo,bar/z', ','), r'_,C_,foo,bar/z') self.assertEqual(san(r'a\b,c<d>e|f"g?h*i', ','), r'a\b,c_d_e_f_g_h_i') self.assertEqual(san('../../foo../../ba..r', '/'), r'foo/ba..r') + self.assertEqual(san(' / /foo / /ba r', '/'), r'foo/ba r') + self.assertEqual(san(' . /. /foo ./ . /. ./ba .r', '/'), r'foo/ba .r') def test_extract_hackers_arcnames_common_cases(self): common_hacknames = [ @@ -1458,8 +1725,6 @@ def test_extract_hackers_arcnames_common_cases(self): ] self._test_extract_hackers_arcnames(common_hacknames) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.path.sep != '\\', 'Requires \\ as path separator.') def test_extract_hackers_arcnames_windows_only(self): """Test combination of path fixing and windows name sanitization.""" @@ -1474,10 +1739,10 @@ def test_extract_hackers_arcnames_windows_only(self): (r'C:\foo\bar', 'foo/bar'), (r'//conky/mountpoint/foo/bar', 'foo/bar'), (r'\\conky\mountpoint\foo\bar', 'foo/bar'), - (r'///conky/mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\\conky\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), - (r'//conky//mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\conky\\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), + (r'///conky/mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\\conky\mountpoint\foo\bar', 'mountpoint/foo/bar'), + (r'//conky//mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\conky\\mountpoint\foo\bar', 'mountpoint/foo/bar'), (r'//?/C:/foo/bar', 'foo/bar'), (r'\\?\C:\foo\bar', 'foo/bar'), (r'C:/../C:/foo/bar', 'C_/foo/bar'), @@ -1539,6 +1804,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -1564,7 +1856,7 @@ def test_writestr_extended_local_header_issue1202(self): with zipfile.ZipFile(TESTFN2, 'w') as orig_zip: for data in 'abcdefghijklmnop': zinfo = zipfile.ZipInfo(data) - zinfo.flag_bits |= 0x08 # Include an extended local header. + zinfo.flag_bits |= zipfile._MASK_USE_DATA_DESCRIPTOR # Include an extended local header. orig_zip.writestr(zinfo, data) def test_close(self): @@ -1606,7 +1898,7 @@ def test_unsupported_version(self): @requires_zlib() def test_read_unicode_filenames(self): # bug #10801 - fname = findfile('zip_cp437_header.zip') + fname = findfile('zip_cp437_header.zip', subdir='archivetestdata') with zipfile.ZipFile(fname) as zipfp: for name in zipfp.namelist(): zipfp.open(name).close() @@ -1621,6 +1913,44 @@ def test_write_unicode_filenames(self): self.assertEqual(zf.filelist[0].filename, "foo.txt") self.assertEqual(zf.filelist[1].filename, "\xf6.txt") + def create_zipfile_with_extra_data(self, filename, extra_data_name): + with zipfile.ZipFile(TESTFN, mode='w') as zf: + filename_encoded = filename.encode("utf-8") + # create a ZipInfo object with Unicode path extra field + zip_info = zipfile.ZipInfo(filename) + + tag_for_unicode_path = b'\x75\x70' + version_of_unicode_path = b'\x01' + + import zlib + filename_crc = struct.pack('<L', zlib.crc32(filename_encoded)) + + extra_data = version_of_unicode_path + filename_crc + extra_data_name + tsize = len(extra_data).to_bytes(2, 'little') + + zip_info.extra = tag_for_unicode_path + tsize + extra_data + + # add the file to the ZIP archive + zf.writestr(zip_info, b'Hello World!') + + @requires_zlib() + def test_read_zipfile_containing_unicode_path_extra_field(self): + self.create_zipfile_with_extra_data("이름.txt", "이름.txt".encode("utf-8")) + with zipfile.ZipFile(TESTFN, "r") as zf: + self.assertEqual(zf.filelist[0].filename, "이름.txt") + + @requires_zlib() + def test_read_zipfile_warning(self): + self.create_zipfile_with_extra_data("이름.txt", b"") + with self.assertWarns(UserWarning): + zipfile.ZipFile(TESTFN, "r").close() + + @requires_zlib() + def test_read_zipfile_error(self): + self.create_zipfile_with_extra_data("이름.txt", b"\xff") + with self.assertRaises(zipfile.BadZipfile): + zipfile.ZipFile(TESTFN, "r").close() + def test_read_after_write_unicode_filenames(self): with zipfile.ZipFile(TESTFN2, 'w') as zipfp: zipfp.writestr('приклад', b'sample') @@ -1679,7 +2009,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -1746,8 +2076,6 @@ def test_empty_file_raises_BadZipFile(self): fp.write("short file") self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_negative_central_directory_offset_raises_BadZipFile(self): # Zip file containing an empty EOCD record buffer = bytearray(b'PK\x05\x06' + b'\0'*18) @@ -1931,6 +2259,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except zipfile.BadZipFile: self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() zipf = zipfile.ZipFile(TESTFN, mode="a") zipf.close() @@ -1938,6 +2267,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except: self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes @@ -2038,6 +2368,7 @@ def test_seek_tell(self): fp.seek(bloc, os.SEEK_CUR) self.assertEqual(fp.tell(), bloc) self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + self.assertEqual(fp.tell(), bloc + 5) fp.seek(0, os.SEEK_END) self.assertEqual(fp.tell(), len(txt)) fp.seek(0, os.SEEK_SET) @@ -2055,11 +2386,40 @@ def test_seek_tell(self): fp.seek(bloc, os.SEEK_CUR) self.assertEqual(fp.tell(), bloc) self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + self.assertEqual(fp.tell(), bloc + 5) fp.seek(0, os.SEEK_END) self.assertEqual(fp.tell(), len(txt)) fp.seek(0, os.SEEK_SET) self.assertEqual(fp.tell(), 0) + def test_read_after_seek(self): + # Issue 102956: Make sure seek(x, os.SEEK_CUR) doesn't break read() + txt = b"Charge men!" + bloc = txt.find(b"men") + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.read(6) + fp.seek(1, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + + def test_uncompressed_interleaved_seek_read(self): + # gh-127847: Make sure the position in the archive is correct + # in the special case of seeking in a ZIP_STORED entry. + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("a.txt", "123") + zipf.writestr("b.txt", "456") + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("a.txt", "r") as a, zipf.open("b.txt", "r") as b: + self.assertEqual(a.read(1), b"1") + self.assertEqual(b.seek(1), 1) + self.assertEqual(b.read(1), b"5") + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' @@ -2070,6 +2430,170 @@ def test_decompress_without_3rd_party_library(self): with zipfile.ZipFile(zip_file) as zf: self.assertRaises(RuntimeError, zf.extract, 'a.txt') + @requires_zlib() + def test_full_overlap_different_names(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('b')), 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('a') + + @requires_zlib() + def test_full_overlap_different_names2(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('b') + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + self.assertEqual(len(zipf.read('a')), 1033) + self.assertEqual(cm.filename, __file__) + + @requires_zlib() + def test_full_overlap_same_name(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'a']) + self.assertEqual(len(zipf.infolist()), 2) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('a')), 1033) + self.assertEqual(len(zipf.read(zi)), 1033) + self.assertEqual(len(zipf.read(zipf.infolist()[1])), 1033) + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + self.assertEqual(len(zipf.read(zipf.infolist()[0])), 1033) + self.assertEqual(cm.filename, __file__) + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + zipf.open(zipf.infolist()[0]).close() + self.assertEqual(cm.filename, __file__) + + @requires_zlib() + def test_quoted_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 52) + self.assertEqual(zi.file_size, 1064) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 36) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + self.assertEqual(len(zipf.read('b')), 1033) + + @requires_zlib() + def test_overlap_with_central_dir(self): + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81\x00\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a']) + self.assertEqual(len(zipf.infolist()), 1) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 11) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): + zipf.read('a') + + @requires_zlib() + def test_overlap_with_archive_comment(self): + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81E\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00*\x00' + b'PK\x03\x04\x14\x00\x00\x00\x08\x00G_|Z\xe2\x1e' + b'8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00aK' + b'L\x1c\x05\xa3`\x14\x8cx\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a']) + self.assertEqual(len(zipf.infolist()), 1) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 69) + self.assertEqual(zi.compress_size, 11) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) @@ -2222,10 +2746,23 @@ def test_good_password(self): self.assertEqual(self.zip2.read("zero"), self.plain2) def test_unicode_password(self): - self.assertRaises(TypeError, self.zip.setpassword, "unicode") - self.assertRaises(TypeError, self.zip.read, "test.txt", "python") - self.assertRaises(TypeError, self.zip.open, "test.txt", pwd="python") - self.assertRaises(TypeError, self.zip.extract, "test.txt", pwd="python") + expected_msg = "pwd: expected bytes, got str" + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.setpassword("unicode") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.read("test.txt", "python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.open("test.txt", pwd="python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.extract("test.txt", pwd="python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.pwd = "python" + self.zip.open("test.txt") def test_seek_tell(self): self.zip.setpassword(b"python") @@ -2555,22 +3092,17 @@ def test_write_after_read(self): self.assertEqual(data1, self.data1) self.assertEqual(data2, self.data2) - # TODO: RUSTPYTHON other tests can impact the file descriptor incrementor - # by leaving file handles unclosed. If there are more than 100 files in - # TESTFN and references to them are left unclosed and ungarbage collected - # in another test, then fileno() will always be too high for this test to - # pass. The solution is to increase the number of files from 100 to 200 def test_many_opens(self): # Verify that read() and open() promptly close the file descriptor, # and don't rely on the garbage collector to free resources. + startcount = fd_count() self.make_test_archive(TESTFN2) with zipfile.ZipFile(TESTFN2, mode="r") as zipf: - for x in range(200): + for x in range(100): zipf.read('ones') with zipf.open('ones') as zopen1: pass - with open(os.devnull, "rb") as f: - self.assertLess(f.fileno(), 200) + self.assertEqual(startcount, fd_count()) def test_write_while_reading(self): with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: @@ -2594,7 +3126,7 @@ def setUp(self): os.mkdir(TESTFN2) def test_extract_dir(self): - with zipfile.ZipFile(findfile("zipdir.zip")) as zipf: + with zipfile.ZipFile(findfile("zipdir.zip", subdir="archivetestdata")) as zipf: zipf.extractall(TESTFN2) self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) @@ -2605,6 +3137,22 @@ def test_bug_6050(self): os.mkdir(os.path.join(TESTFN2, "a")) self.test_extract_dir() + def test_extract_dir_backslash(self): + zfname = findfile("zipdir_backslash.zip", subdir="archivetestdata") + with zipfile.ZipFile(zfname) as zipf: + zipf.extractall(TESTFN2) + if os.name == 'nt': + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a", "b", "c"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d", "e"))) + else: + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a\\b\\c"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "d\\e\\"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "a"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "d"))) + def test_write_dir(self): dirpath = os.path.join(TESTFN2, "x") os.mkdir(dirpath) @@ -2648,6 +3196,70 @@ def test_writestr_dir(self): self.assertTrue(os.path.isdir(os.path.join(target, "x"))) self.assertEqual(os.listdir(target), ["x"]) + def test_mkdir(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.mkdir("directory") + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory3", mode=0o777) + zinfo = zf.filelist[2] + self.assertEqual(zinfo.filename, "directory3/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + old_zinfo = zipfile.ZipInfo("directory4/") + old_zinfo.external_attr = (0o40777 << 16) | 0x10 + old_zinfo.CRC = 0 + old_zinfo.file_size = 0 + old_zinfo.compress_size = 0 + zf.mkdir(old_zinfo) + new_zinfo = zf.filelist[3] + self.assertEqual(old_zinfo.filename, "directory4/") + self.assertEqual(old_zinfo.external_attr, new_zinfo.external_attr) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + self.assertEqual(set(os.listdir(target)), {"directory", "directory2", "directory3", "directory4"}) + + def test_create_directory_with_write(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr(zipfile.ZipInfo('directory/'), '') + + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + + directory = os.path.join(TESTFN2, "directory2") + os.mkdir(directory) + mode = os.stat(directory).st_mode & 0xFFFF + zf.write(directory, arcname="directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + + self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): @@ -2657,13 +3269,13 @@ def tearDown(self): class ZipInfoTests(unittest.TestCase): def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) @@ -2688,6 +3300,17 @@ def test_from_dir(self): self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) self.assertEqual(zi.file_size, 0) + def test_compresslevel_property(self): + zinfo = zipfile.ZipInfo("xxx") + self.assertFalse(zinfo._compresslevel) + self.assertFalse(zinfo.compress_level) + zinfo._compresslevel = 99 # test the legacy @property.setter + self.assertEqual(zinfo.compress_level, 99) + self.assertEqual(zinfo._compresslevel, 99) + zinfo.compress_level = 8 + self.assertEqual(zinfo.compress_level, 8) + self.assertEqual(zinfo._compresslevel, 8) + class CommandLineTest(unittest.TestCase): @@ -2710,7 +3333,7 @@ def test_bad_use(self): self.assertNotEqual(err.strip(), b'') def test_test_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') for opt in '-t', '--test': out = self.zipfilecmd(opt, zip_name) self.assertEqual(out.rstrip(), b'Done testing') @@ -2719,7 +3342,7 @@ def test_test_command(self): self.assertEqual(out, b'') def test_list_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') t = io.StringIO() with zipfile.ZipFile(zip_name, 'r') as tf: tf.printdir(t) @@ -2752,7 +3375,7 @@ def test_create_command(self): unlink(TESTFN2) def test_extract_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') for opt in '-e', '--extract': with temp_dir() as extdir: out = self.zipfilecmd(opt, zip_name, extdir) @@ -2773,8 +3396,8 @@ class TestExecutablePrependedZip(unittest.TestCase): """Test our ability to open zip files with an executable prepended.""" def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') + self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') + self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') def _test_zip_works(self, name): # bpo28494 sanity check: ensure is_zipfile works on these. @@ -2792,379 +3415,323 @@ def test_read_zip_with_exe_prepended(self): def test_read_zip64_with_exe_prepended(self): self._test_zip_works(self.exe_zip64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.executable, 'sys.executable required.') @unittest.skipUnless(os.access('/bin/bash', os.X_OK), 'Test relies on #!/bin/bash working.') + @requires_subprocess() def test_execute_zip2(self): output = subprocess.check_output([self.exe_zip, sys.executable]) self.assertIn(b'number in executable: 5', output) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.executable, 'sys.executable required.') @unittest.skipUnless(os.access('/bin/bash', os.X_OK), 'Test relies on #!/bin/bash working.') + @requires_subprocess() def test_execute_zip64(self): output = subprocess.check_output([self.exe_zip64, sys.executable]) self.assertIn(b'number in executable: 5', output) -# Poor man's technique to consume a (smallish) iterable. -consume = tuple - - -# from jaraco.itertools 5.0 -class jaraco: - class itertools: - class Counter: - def __init__(self, i): - self.count = 0 - self._orig_iter = iter(i) - - def __iter__(self): - return self - - def __next__(self): - result = next(self._orig_iter) - self.count += 1 - return result - - -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf - - -def build_alpharep_fixture(): - """ - Create a zip file with this structure: - - . - ├── a.txt - ├── b - │ ├── c.txt - │ ├── d - │ │ └── e.txt - │ └── f.txt - └── g - └── h - └── i.txt - - This fixture has the following key characteristics: - - - a file at the root (a) - - a file two levels deep (b/d/e) - - multiple files in a directory (b/c, b/f) - - a directory containing only a directory (g/h) - - "alpha" because it uses alphabet - "rep" because it's a representative example - """ - data = io.BytesIO() - zf = zipfile.ZipFile(data, "w") - zf.writestr("a.txt", b"content of a") - zf.writestr("b/c.txt", b"content of c") - zf.writestr("b/d/e.txt", b"content of e") - zf.writestr("b/f.txt", b"content of f") - zf.writestr("g/h/i.txt", b"content of i") - zf.filename = "alpharep.zip" - return zf - - -def pass_alpharep(meth): - """ - Given a method, wrap it in a for loop that invokes method - with each subtest. - """ - - @functools.wraps(meth) - def wrapper(self): - for alpharep in self.zipfile_alpharep(): - meth(self, alpharep=alpharep) - - return wrapper - - -class TestPath(unittest.TestCase): - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - def zipfile_alpharep(self): - with self.subTest(): - yield build_alpharep_fixture() - with self.subTest(): - yield add_dirs(build_alpharep_fixture()) - - def zipfile_ondisk(self, alpharep): - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - buffer = alpharep.fp - alpharep.close() - path = tmpdir / alpharep.filename - with path.open("wb") as strm: - strm.write(buffer.getvalue()) - return path - - @pass_alpharep - def test_iterdir_and_types(self, alpharep): - root = zipfile.Path(alpharep) - assert root.is_dir() - a, b, g = root.iterdir() - assert a.is_file() - assert b.is_dir() - assert g.is_dir() - c, f, d = b.iterdir() - assert c.is_file() and f.is_file() - (e,) = d.iterdir() - assert e.is_file() - (h,) = g.iterdir() - (i,) = h.iterdir() - assert i.is_file() - - @pass_alpharep - def test_is_file_missing(self, alpharep): - root = zipfile.Path(alpharep) - assert not root.joinpath('missing.txt').is_file() - - @pass_alpharep - def test_iterdir_on_file(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with self.assertRaises(ValueError): - a.iterdir() - - @pass_alpharep - def test_subdir_is_dir(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').is_dir() - assert (root / 'b/').is_dir() - assert (root / 'g').is_dir() - assert (root / 'g/').is_dir() - - @pass_alpharep - def test_open(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with a.open(encoding="utf-8") as strm: - data = strm.read() - assert data == "content of a" - - def test_open_write(self): - """ - If the zipfile is open for write, it should be possible to - write bytes or text to it. - """ - zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) - with zf.joinpath('file.bin').open('wb') as strm: - strm.write(b'binary contents') - with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: - strm.write('text file') - - def test_open_extant_directory(self): - """ - Attempting to open a directory raises IsADirectoryError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(IsADirectoryError): - zf.joinpath('b').open() - - @pass_alpharep - def test_open_binary_invalid_args(self, alpharep): - root = zipfile.Path(alpharep) - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', encoding='utf-8') - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', 'utf-8') - - def test_open_missing_directory(self): - """ - Attempting to open a missing directory raises FileNotFoundError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(FileNotFoundError): - zf.joinpath('z').open() - - @pass_alpharep - def test_read(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - assert a.read_text(encoding="utf-8") == "content of a" - assert a.read_bytes() == b"content of a" - - @pass_alpharep - def test_joinpath(self, alpharep): - root = zipfile.Path(alpharep) - a = root.joinpath("a.txt") - assert a.is_file() - e = root.joinpath("b").joinpath("d").joinpath("e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_joinpath_multiple(self, alpharep): - root = zipfile.Path(alpharep) - e = root.joinpath("b", "d", "e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_truediv(self, alpharep): - root = zipfile.Path(alpharep) - a = root / "a.txt" - assert a.is_file() - e = root / "b" / "d" / "e.txt" - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_simplediv(self, alpharep): - """ - Disable the __future__.division when testing traversal. - """ - code = compile( - source="zipfile.Path(alpharep) / 'a'", - filename="(test)", - mode="eval", - dont_inherit=True, - ) - eval(code) - - @pass_alpharep - def test_pathlike_construction(self, alpharep): - """ - zipfile.Path should be constructable from a path-like object - """ - zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) - zipfile.Path(pathlike) - - @pass_alpharep - def test_traverse_pathlike(self, alpharep): - root = zipfile.Path(alpharep) - root / pathlib.Path("a") - - @pass_alpharep - def test_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'a').parent.at == '' - assert (root / 'a' / 'b').parent.at == 'a/' - - @pass_alpharep - def test_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').parent.at == '' - assert (root / 'b/').parent.at == '' - - @pass_alpharep - def test_missing_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'missing dir/').parent.at == '' - - @pass_alpharep - def test_mutability(self, alpharep): - """ - If the underlying zipfile is changed, the Path object should - reflect that change. - """ - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - alpharep.writestr('foo.txt', 'foo') - alpharep.writestr('bar/baz.txt', 'baz') - assert any(child.name == 'foo.txt' for child in root.iterdir()) - assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' - (baz,) = (root / 'bar').iterdir() - assert baz.read_text(encoding="utf-8") == 'baz' - - HUGE_ZIPFILE_NUM_ENTRIES = 2 ** 13 - - def huge_zipfile(self): - """Create a read-only zipfile with a huge number of entries entries.""" - strm = io.BytesIO() - zf = zipfile.ZipFile(strm, "w") - for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): - zf.writestr(entry, entry) - zf.mode = 'r' - return zf - - def test_joinpath_constant_time(self): - """ - Ensure joinpath on items in zipfile is linear time. - """ - root = zipfile.Path(self.huge_zipfile()) - entries = jaraco.itertools.Counter(root.iterdir()) - for entry in entries: - entry.joinpath('suffix') - # Check the file iterated all items - assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES - - # @func_timeout.func_set_timeout(3) - def test_implied_dirs_performance(self): - data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] - zipfile.CompleteDirs._implied_dirs(data) - - @pass_alpharep - def test_read_does_not_close(self, alpharep): - alpharep = self.zipfile_ondisk(alpharep) - with zipfile.ZipFile(alpharep) as file: - for rep in range(2): - zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") - - @pass_alpharep - def test_subclass(self, alpharep): - class Subclass(zipfile.Path): - pass - - root = Subclass(alpharep) - assert isinstance(root / 'b', Subclass) +@unittest.skip("TODO: RUSTPYTHON shift_jis encoding unsupported") +class EncodedMetadataTests(unittest.TestCase): + file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' + file_content = [ + "This is pure ASCII.\n".encode('ascii'), + # This is modern Japanese. (UTF-8) + "\u3053\u308c\u306f\u73fe\u4ee3\u7684\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('utf-8'), + # TODO RUSTPYTHON + # Uncomment when Shift JIS is supported + # This is obsolete Japanese. (Shift JIS) + # "\u3053\u308c\u306f\u53e4\u3044\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('shift_jis'), + ] - @pass_alpharep - def test_filename(self, alpharep): - root = zipfile.Path(alpharep) - assert root.filename == pathlib.Path('alpharep.zip') + def setUp(self): + self.addCleanup(unlink, TESTFN) + # Create .zip of 3 members with Han names encoded in Shift JIS. + # Each name is 1 Han character encoding to 2 bytes in Shift JIS. + # The ASCII names are arbitrary as long as they are length 2 and + # not otherwise contained in the zip file. + # Data elements are encoded bytes (ascii, utf-8, shift_jis). + placeholders = ["n1", "n2"] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, mode="w") as tf: + for temp, content in zip(placeholders, self.file_content): + tf.writestr(temp, content, zipfile.ZIP_STORED) + # Hack in the Shift JIS names with flag bit 11 (UTF-8) unset. + with open(TESTFN, "rb") as tf: + data = tf.read() + for name, temp in zip(self.file_names, placeholders[:2]): + data = data.replace(temp.encode('ascii'), + name.encode('shift_jis')) + with open(TESTFN, "wb") as tf: + tf.write(data) + + def _test_read(self, zipfp, expected_names, expected_content): + # Check the namelist + names = zipfp.namelist() + self.assertEqual(sorted(names), sorted(expected_names)) + + # Check infolist + infos = zipfp.infolist() + names = [zi.filename for zi in infos] + self.assertEqual(sorted(names), sorted(expected_names)) + + # check getinfo + for name, content in zip(expected_names, expected_content): + info = zipfp.getinfo(name) + self.assertEqual(info.filename, name) + self.assertEqual(info.file_size, len(content)) + self.assertEqual(zipfp.read(name), content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_metadata_encoding(self): + # Read the ZIP archive with correct metadata_encoding + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_without_metadata_encoding(self): + # Read the ZIP archive without metadata_encoding + expected_names = [name.encode('shift_jis').decode('cp437') + for name in self.file_names[:2]] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, expected_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_incorrect_metadata_encoding(self): + # Read the ZIP archive with incorrect metadata_encoding + expected_names = [name.encode('shift_jis').decode('koi8-u') + for name in self.file_names[:2]] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='koi8-u') as zipfp: + self._test_read(zipfp, expected_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_unsuitable_metadata_encoding(self): + # Read the ZIP archive with metadata_encoding unsuitable for + # decoding metadata + with self.assertRaises(UnicodeDecodeError): + zipfile.ZipFile(TESTFN, "r", metadata_encoding='ascii') + with self.assertRaises(UnicodeDecodeError): + zipfile.ZipFile(TESTFN, "r", metadata_encoding='utf-8') + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_after_append(self): + newname = '\u56db' # Han 'four' + expected_names = [name.encode('shift_jis').decode('cp437') + for name in self.file_names[:2]] + self.file_names[2:] + expected_names.append(newname) + expected_content = (*self.file_content, b"newcontent") + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.writestr(newname, "newcontent") + self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, expected_names, expected_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + for i, (name, content) in enumerate(zip(expected_names, expected_content)): + info = zipfp.getinfo(name) + self.assertEqual(info.filename, name) + self.assertEqual(info.file_size, len(content)) + if i < 2: + with self.assertRaises(zipfile.BadZipFile): + zipfp.read(name) + else: + self.assertEqual(zipfp.read(name), content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_write_with_metadata_encoding(self): + ZF = zipfile.ZipFile + for mode in ("w", "x", "a"): + with self.assertRaisesRegex(ValueError, + "^metadata_encoding is only"): + ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_cli_with_metadata_encoding(self): + errmsg = "Non-conforming encodings not supported with -c." + args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] + with captured_stdout() as stdout: + with captured_stderr() as stderr: + self.assertRaises(SystemExit, zipfile.main, args) + self.assertEqual(stdout.getvalue(), "") + self.assertIn(errmsg, stderr.getvalue()) + + with captured_stdout() as stdout: + zipfile.main(["--metadata-encoding=shift_jis", "-t", TESTFN]) + listing = stdout.getvalue() + + with captured_stdout() as stdout: + zipfile.main(["--metadata-encoding=shift_jis", "-l", TESTFN]) + listing = stdout.getvalue() + for name in self.file_names: + self.assertIn(name, listing) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_cli_with_metadata_encoding_extract(self): + os.mkdir(TESTFN2) + self.addCleanup(rmtree, TESTFN2) + # Depending on locale, extracted file names can be not encodable + # with the filesystem encoding. + for fn in self.file_names: + try: + os.stat(os.path.join(TESTFN2, fn)) + except OSError: + pass + except UnicodeEncodeError: + self.skipTest(f'cannot encode file name {fn!a}') + + zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2]) + listing = os.listdir(TESTFN2) + for name in self.file_names: + self.assertIn(name, listing) + + +class StripExtraTests(unittest.TestCase): + # Note: all of the "z" characters are technically invalid, but up + # to 3 bytes at the end of the extra will be passed through as they + # are too short to encode a valid extra. + + ZIP64_EXTRA = 1 + + def test_no_data(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 0) + b = s.pack(2, 0) + c = s.pack(3, 0) + + self.assertEqual(b'', zipfile._Extra.strip(a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b, (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b+c, zipfile._Extra.strip(a+b+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+a+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+c+a, (self.ZIP64_EXTRA,))) + + def test_with_data(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 1) + b"a" + b = s.pack(2, 2) + b"bb" + c = s.pack(3, 3) + b"ccc" + + self.assertEqual(b"", zipfile._Extra.strip(a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b, (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b+c, zipfile._Extra.strip(a+b+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+a+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+c+a, (self.ZIP64_EXTRA,))) + + def test_multiples(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 1) + b"a" + b = s.pack(2, 2) + b"bb" + + self.assertEqual(b"", zipfile._Extra.strip(a+a, (self.ZIP64_EXTRA,))) + self.assertEqual(b"", zipfile._Extra.strip(a+a+a, (self.ZIP64_EXTRA,))) + self.assertEqual( + b"z", zipfile._Extra.strip(a+a+b"z", (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(a+a+b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b, zipfile._Extra.strip(a+a+b, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(a+b+a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b+a+a, (self.ZIP64_EXTRA,))) + + def test_too_short(self): + self.assertEqual(b"", zipfile._Extra.strip(b"", (self.ZIP64_EXTRA,))) + self.assertEqual(b"z", zipfile._Extra.strip(b"z", (self.ZIP64_EXTRA,))) + self.assertEqual( + b"zz", zipfile._Extra.strip(b"zz", (self.ZIP64_EXTRA,))) + self.assertEqual( + b"zzz", zipfile._Extra.strip(b"zzz", (self.ZIP64_EXTRA,))) + + +class StatIO(_pyio.BytesIO): + """Buffer which remembers the number of bytes that were read.""" + + def __init__(self): + super().__init__() + self.bytes_read = 0 + + def read(self, size=-1): + bs = super().read(size) + self.bytes_read += len(bs) + return bs + + +class StoredZipExtFileRandomReadTest(unittest.TestCase): + """Tests whether an uncompressed, unencrypted zip entry can be randomly + seek and read without reading redundant bytes.""" + def test_stored_seek_and_read(self): + + sio = StatIO() + # 20000 bytes + txt = b'0123456789' * 2000 + + # The seek length must be greater than ZipExtFile.MIN_READ_SIZE + # as `ZipExtFile._read2()` reads in blocks of this size and we + # need to seek out of the buffered data + read_buffer_size = zipfile.ZipExtFile.MIN_READ_SIZE + self.assertGreaterEqual(10002, read_buffer_size) # for forward seek test + self.assertGreaterEqual(5003, read_buffer_size) # for backward seek test + # The read length must be less than MIN_READ_SIZE, since we assume that + # only 1 block is read in the test. + read_length = 100 + self.assertGreaterEqual(read_buffer_size, read_length) # for read() calls + + with zipfile.ZipFile(sio, "w", compression=zipfile.ZIP_STORED) as zipf: + zipf.writestr("foo.txt", txt) - @pass_alpharep - def test_root_name(self, alpharep): - """ - The name of the root should be the name of the zipfile - """ - root = zipfile.Path(alpharep) - assert root.name == 'alpharep.zip' == root.filename.name - - @pass_alpharep - def test_root_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert root.parent == pathlib.Path('.') - root.root.filename = 'foo/bar.zip' - assert root.parent == pathlib.Path('foo') - - @pass_alpharep - def test_root_unnamed(self, alpharep): - """ - It is an error to attempt to get the name - or parent of an unnamed zipfile. - """ - alpharep.filename = None - root = zipfile.Path(alpharep) - with self.assertRaises(TypeError): - root.name - with self.assertRaises(TypeError): - root.parent - - # .name and .parent should still work on subs - sub = root / "b" - assert sub.name == "b" - assert sub.parent - - @pass_alpharep - def test_inheritance(self, alpharep): - cls = type('PathChild', (zipfile.Path,), {}) - for alpharep in self.zipfile_alpharep(): - file = cls(alpharep).joinpath('some dir').parent - assert isinstance(file, cls) + # check random seek and read on a file + with zipfile.ZipFile(sio, "r") as zipf: + with zipf.open("foo.txt", "r") as fp: + # Test this optimized read hasn't rewound and read from the + # start of the file (as in the case of the unoptimized path) + + # forward seek + old_count = sio.bytes_read + forward_seek_len = 10002 + current_pos = 0 + fp.seek(forward_seek_len, os.SEEK_CUR) + current_pos += forward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # backward seek + old_count = sio.bytes_read + backward_seek_len = 5003 + fp.seek(-backward_seek_len, os.SEEK_CUR) + current_pos -= backward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # eof flags test + fp.seek(0, os.SEEK_END) + fp.seek(12345, os.SEEK_SET) + current_pos = 12345 + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) if __name__ == "__main__": diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 0947013afbc..2e1affe0252 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys @@ -32,10 +32,6 @@ def setUp(self): line_gen = ("Test of zipfile line %d." % i for i in range(1000000)) self.data = '\n'.join(line_gen).encode('ascii') - # And write it to a file. - with open(TESTFN, "wb") as fp: - fp.write(self.data) - def zipTest(self, f, compression): # Create the ZIP archive. with zipfile.ZipFile(f, "w", compression) as zipfp: @@ -67,6 +63,9 @@ def zipTest(self, f, compression): (num, filecount)), file=sys.__stdout__) sys.__stdout__.flush() + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) + def testStored(self): # Try the temp file first. If we do TESTFN2 first, then it hogs # gigabytes of disk space for the duration of the test. @@ -85,9 +84,7 @@ def testDeflated(self): self.zipTest(TESTFN2, zipfile.ZIP_DEFLATED) def tearDown(self): - for fname in TESTFN, TESTFN2: - if os.path.exists(fname): - os.remove(fname) + os_helper.unlink(TESTFN2) class OtherTests(unittest.TestCase): diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index b291d530169..d448e3df5d7 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -50,10 +52,14 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -92,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -104,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -128,20 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + + def getZip64Files(self): + # This is the simplest way to make zipfile generate the zip64 EOCD block + return {f"f{n}.py": test_src for n in range(65537)} + def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -155,7 +186,8 @@ def testAFakeZlib(self): # zlib.decompress function object, after which the problem being # tested here wouldn't be a problem anymore... # (Hence the 'A' in the test method name: to make it the first - # item in a list sorted by name, like unittest.makeSuite() does.) + # item in a list sorted by name, like + # unittest.TestLoader.getTestCaseNames() does.) # # This test fails on platforms on which the zlib module is # statically linked, but the problem it tests for can't @@ -166,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -177,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -219,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -247,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -270,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -281,29 +313,104 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) + def testPackageExplicitDirectories(self): + # Test explicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.writestr('a/__init__.py', test_src) + z.mkdir('a/b') + z.writestr('a/b/__init__.py', test_src) + z.mkdir('a/b/c') + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testPackageImplicitDirectories(self): + # Test explicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/__init__.py', test_src) + z.writestr('a/b/__init__.py', test_src) + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testNamespacePackageExplicitDirectories(self): + # Test implicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def testNamespacePackageImplicitDirectories(self): + # Test implicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def _testPackage(self, initfile): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a')) + if initfile is None: + # XXX Should it work? + self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b') + else: + self.assertTrue(zi.is_package('b')) + self.assertEqual(zi.get_source('b'), test_src) + self.assertEqual(zi.get_code('b').co_filename, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + + sys.path.insert(0, TEMP_ZIP) + self.assertNotIn('a', sys.modules) + + mod = importlib.import_module(f'a.b') + self.assertIn('a', sys.modules) + self.assertIs(sys.modules['a.b'], mod) + if initfile is None: + self.assertIsNone(mod.__file__) + else: + self.assertEqual(mod.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + self.assertEqual(len(mod.__path__), 1, mod.__path__) + self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b')) + + mod2 = importlib.import_module(f'a.b.c.d') + self.assertIn('a.b.c', sys.modules) + self.assertIn('a.b.c.d', sys.modules) + self.assertIs(sys.modules['a.b.c.d'], mod2) + self.assertIs(mod.c.d, mod2) + self.assertEqual(mod2.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py')) + def testMixedNamespacePackage(self): # Test implicit namespace packages spread between a # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -336,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -354,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -368,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -412,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -428,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -459,12 +559,6 @@ def testZipImporterMethods(self): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - find_mod = zi.find_module('spam') - self.assertIsNotNone(find_mod) - self.assertIsInstance(find_mod, zipimport.zipimporter) - self.assertFalse(find_mod.is_package('spam')) - load_mod = find_mod.load_module('spam') - self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) mod = zi.load_module(TESTPACK) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) @@ -512,58 +606,70 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Check that the file information remains accurate after reloading zi.invalidate_caches() - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) spec = zi.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) # Check that the cached data is removed if the file is deleted os.remove(TEMP_ZIP) zi.invalidate_caches() - self.assertFalse(zi._files) + self.assertFalse(zi._get_files()) self.assertIsNone(zipimport._zip_directory_cache.get(zi.archive)) self.assertIsNone(zi.find_spec("name_does_not_matter")) - def testZipImporterMethodsInSubDirectory(self): + def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + zi = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) + # Zipimporter for the same path. + zi2 = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) + # Add a new file to the ZIP archive to make the cache wrong. + newfile = {"spam2" + pyc_ext: test_pyc} + files.update(newfile) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") + # Invalidate the cache of the first zipimporter. + zi.invalidate_caches() + # Check that the second zipimporter detects the new file and isn't using a stale cache. + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) + spec = zi2.find_spec('spam2') + self.assertIsNotNone(spec) + self.assertIsInstance(spec.loader, zipimport.zipimporter) + + def testZipImporterMethodsInSubDirectory(self): + packdir = TESTPACK + os.sep + packdir2 = packdir + TESTPACK2 + os.sep + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -585,16 +691,6 @@ def testZipImporterMethodsInSubDirectory(self): pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2 zi2 = zipimport.zipimporter(pkg_path) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - find_mod_dotted = zi2.find_module(TESTMOD) - self.assertIsNotNone(find_mod_dotted) - self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) - self.assertFalse(zi2.is_package(TESTMOD)) - load_mod = find_mod_dotted.load_module(TESTMOD) - self.assertEqual( - find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) # PEP 451 spec = zi2.find_spec(TESTMOD) @@ -619,17 +715,33 @@ def testZipImporterMethodsInSubDirectory(self): self.assertIsNone(loader.get_source(mod_name)) self.assertEqual(loader.get_filename(mod_name), mod.__file__) - def testGetData(self): + def testGetDataExplicitDirectories(self): self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - z.compression = self.compression - name = "testdata.dat" - data = bytes(x for x in range(256)) - z.writestr(name, data) - - zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(data, zi.get_data(name)) - self.assertIn('zipimporter object', repr(zi)) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def testGetDataImplicitDirectories(self): + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def _testGetData(self): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored')) + pathname = os.path.join('a', 'b', 'c', 'testdata.dat') + data = bytes(range(256)) + self.assertEqual(zi.get_data(pathname), data) + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data) + self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'') + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'') + self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b')) + self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b')) def testImporterAttr(self): src = """if 1: # indent hack @@ -638,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "<???>", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -648,17 +760,20 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) self.assertEqual(mod.test(1), 1) - self.assertRaises(AssertionError, mod.test, False) + if __debug__: + self.assertRaises(AssertionError, mod.test, False) + else: + self.assertEqual(mod.test(0), 0) def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -666,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "<???>", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -720,54 +835,179 @@ def doTraceback(self, module): s = io.StringIO() print_tb(tb, 1, s) - self.assertTrue(s.getvalue().endswith(raise_src)) + self.assertEndsWith(s.getvalue(), + ' def do_raise(): raise TypeError\n' + '' if support.has_no_debug_ranges() else + ' ^^^^^^^^^^^^^^^\n' + ) else: raise AssertionError("This ought to be impossible") + @unittest.expectedFailure # TODO: RUSTPYTHON; empty caret lines from equal col/end_col def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) - zipimport.zipimporter(os.fsencode(filename)) + with self.assertRaises(TypeError): + zipimport.zipimporter(os.fsencode(filename)) with self.assertRaises(TypeError): zipimport.zipimporter(bytearray(os.fsencode(filename))) with self.assertRaises(TypeError): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + @support.requires_resource('cpu') + def testZip64(self): + files = self.getZip64Files() + self.doTest(".py", files, "f6") + + @support.requires_resource('cpu') + def testZip64CruftAndComment(self): + files = self.getZip64Files() + self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + + @unittest.skip("TODO: RUSTPYTHON; (intermittent success/failures); ValueError: name=\"RustPython/crates/pylib/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part\" does not fit expected pattern.") + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do a lot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): @@ -799,6 +1039,7 @@ def testEmptyFile(self): os_helper.create_empty_file(TESTMOD) self.assertZipFailure(TESTMOD) + @unittest.skipIf(support.is_wasi, "mode 000 not supported.") def testFileUnreadable(self): os_helper.unlink(TESTMOD) fd = os.open(TESTMOD, os.O_CREAT, 000) @@ -842,7 +1083,6 @@ def _testBogusZipFile(self): self.assertRaises(TypeError, z.get_source, None) error = zipimport.ZipImportError - self.assertIsNone(z.find_module('abc')) self.assertIsNone(z.find_spec('abc')) with warnings.catch_warnings(): diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 0a75457ad85..bb1366cb21c 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -3,7 +3,6 @@ from test.support import import_helper import binascii import copy -import os import pickle import random import sys @@ -13,11 +12,11 @@ zlib = import_helper.import_module('zlib') requires_Compress_copy = unittest.skipUnless( - hasattr(zlib.compressobj(), "copy"), - 'requires Compress.copy()') + hasattr(zlib.compressobj(), "copy"), + 'requires Compress.copy()') requires_Decompress_copy = unittest.skipUnless( - hasattr(zlib.decompressobj(), "copy"), - 'requires Decompress.copy()') + hasattr(zlib.decompressobj(), "copy"), + 'requires Decompress.copy()') def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): @@ -154,7 +153,7 @@ def test_badcompressobj(self): self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0) # specifying total bits too large causes an error self.assertRaises(ValueError, - zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1) + zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1) def test_baddecompressobj(self): # verify failure on building decompress object with bad params @@ -242,8 +241,8 @@ def test_incomplete_stream(self): # A useful error message is given x = zlib.compress(HAMLET_SCENE) self.assertRaisesRegex(zlib.error, - "Error -5 while decompressing data: incomplete or truncated stream", - zlib.decompress, x[:-1]) + "Error -5 while decompressing data: incomplete or truncated stream", + zlib.decompress, x[:-1]) # Memory use of the following functions takes into account overallocation @@ -377,7 +376,7 @@ def test_decompinc(self, flush=False, source=None, cx=256, dcx=64): bufs.append(dco.decompress(combuf[i:i+dcx])) self.assertEqual(b'', dco.unconsumed_tail, ######## "(A) uct should be b'': not %d long" % - len(dco.unconsumed_tail)) + len(dco.unconsumed_tail)) self.assertEqual(b'', dco.unused_data) if flush: bufs.append(dco.flush()) @@ -390,7 +389,7 @@ def test_decompinc(self, flush=False, source=None, cx=256, dcx=64): break self.assertEqual(b'', dco.unconsumed_tail, ######## "(B) uct should be b'': not %d long" % - len(dco.unconsumed_tail)) + len(dco.unconsumed_tail)) self.assertEqual(b'', dco.unused_data) self.assertEqual(data, b''.join(bufs)) # Failure means: "decompressobj with init options failed" @@ -419,7 +418,7 @@ def test_decompimax(self, source=None, cx=256, dcx=64): #max_length = 1 + len(cb)//10 chunk = dco.decompress(cb, dcx) self.assertFalse(len(chunk) > dcx, - 'chunk too big (%d>%d)' % (len(chunk), dcx)) + 'chunk too big (%d>%d)' % (len(chunk), dcx)) bufs.append(chunk) cb = dco.unconsumed_tail bufs.append(dco.flush()) @@ -444,7 +443,7 @@ def test_decompressmaxlen(self, flush=False): max_length = 1 + len(cb)//10 chunk = dco.decompress(cb, max_length) self.assertFalse(len(chunk) > max_length, - 'chunk too big (%d>%d)' % (len(chunk),max_length)) + 'chunk too big (%d>%d)' % (len(chunk),max_length)) bufs.append(chunk) cb = dco.unconsumed_tail if flush: @@ -453,7 +452,7 @@ def test_decompressmaxlen(self, flush=False): while chunk: chunk = dco.decompress(b'', max_length) self.assertFalse(len(chunk) > max_length, - 'chunk too big (%d>%d)' % (len(chunk),max_length)) + 'chunk too big (%d>%d)' % (len(chunk),max_length)) bufs.append(chunk) self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') @@ -490,8 +489,7 @@ def test_clear_unconsumed_tail(self): ddata += dco.decompress(dco.unconsumed_tail) self.assertEqual(dco.unconsumed_tail, b"") - # TODO: RUSTPYTHON: Z_BLOCK support in flate2 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Z_BLOCK support in flate2 def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. @@ -633,7 +631,7 @@ def test_decompress_unused_data(self): self.assertEqual(dco.unconsumed_tail, b'') else: data += dco.decompress( - dco.unconsumed_tail + x[i : i + step], maxlen) + dco.unconsumed_tail + x[i : i + step], maxlen) data += dco.flush() self.assertTrue(dco.eof) self.assertEqual(data, source) @@ -747,15 +745,11 @@ def test_baddecompresscopy(self): self.assertRaises(ValueError, copy.copy, d) self.assertRaises(ValueError, copy.deepcopy, d) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): @@ -815,8 +809,7 @@ def test_large_unconsumed_tail(self, size): finally: comp = uncomp = data = None - # TODO: RUSTPYTHON: wbits=0 support in flate2 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; wbits=0 support in flate2 def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5) @@ -945,6 +938,7 @@ def choose_lines(source, number, seed=None, generator=random): Farewell. """ + class ZlibDecompressorTest(unittest.TestCase): # Test adopted from test_bz2.py TEXT = HAMLET_SCENE @@ -1006,8 +1000,6 @@ def testDecompress4G(self, size): compressed = None decompressed = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def testPickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): @@ -1021,7 +1013,7 @@ def testDecompressorChunksMaxsize(self): # Feed some input len_ = len(self.BIG_DATA) - 64 out.append(zlibd.decompress(self.BIG_DATA[:len_], - max_length=max_length)) + max_length=max_length)) self.assertFalse(zlibd.needs_input) self.assertEqual(len(out[-1]), max_length) @@ -1032,7 +1024,7 @@ def testDecompressorChunksMaxsize(self): # Retrieve more data while providing more input out.append(zlibd.decompress(self.BIG_DATA[len_:], - max_length=max_length)) + max_length=max_length)) self.assertLessEqual(len(out[-1]), max_length) # Retrieve remaining uncompressed data @@ -1052,7 +1044,7 @@ def test_decompressor_inputbuf_1(self): # Create input buffer and fill it self.assertEqual(zlibd.decompress(self.DATA[:100], - max_length=0), b'') + max_length=0), b'') # Retrieve some results, freeing capacity at beginning # of input buffer @@ -1074,7 +1066,7 @@ def test_decompressor_inputbuf_2(self): # Create input buffer and empty it self.assertEqual(zlibd.decompress(self.DATA[:200], - max_length=0), b'') + max_length=0), b'') out.append(zlibd.decompress(b'')) # Fill buffer with new data @@ -1118,6 +1110,7 @@ def test_refleaks_in___init__(self): zlibd.__init__() self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + class CustomInt: def __index__(self): return 100 diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index e05bd046e83..46aa42063a4 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1937,8 +1937,6 @@ def test_cache_location(self): self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_tracked(self): import gc diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py new file mode 100644 index 00000000000..cf618534add --- /dev/null +++ b/Lib/test/test_zstd.py @@ -0,0 +1,2802 @@ +import array +import gc +import io +import pathlib +import random +import re +import os +import unittest +import tempfile +import threading + +from test.support.import_helper import import_module +from test.support import threading_helper +from test.support import _1M +from test.support import Py_GIL_DISABLED + +_zstd = import_module("_zstd") +zstd = import_module("compression.zstd") + +from compression.zstd import ( + open, + compress, + decompress, + ZstdCompressor, + ZstdDecompressor, + ZstdDict, + ZstdError, + zstd_version, + zstd_version_info, + COMPRESSION_LEVEL_DEFAULT, + get_frame_info, + get_frame_size, + finalize_dict, + train_dict, + CompressionParameter, + DecompressionParameter, + Strategy, + ZstdFile, +) + +_1K = 1024 +_130_1K = 130 * _1K +DICT_SIZE1 = 3*_1K + +DAT_130K_D = None +DAT_130K_C = None + +DECOMPRESSED_DAT = None +COMPRESSED_DAT = None + +DECOMPRESSED_100_PLUS_32KB = None +COMPRESSED_100_PLUS_32KB = None + +SKIPPABLE_FRAME = None + +THIS_FILE_BYTES = None +THIS_FILE_STR = None +COMPRESSED_THIS_FILE = None + +COMPRESSED_BOGUS = None + +SAMPLES = None + +TRAINED_DICT = None + +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False + +C_INT_MIN = -(2**31) +C_INT_MAX = (2**31) - 1 + + +def setUpModule(): + # uncompressed size 130KB, more than a zstd block. + # with a frame epilogue, 4 bytes checksum. + global DAT_130K_D + DAT_130K_D = bytes([random.randint(0, 127) for _ in range(130*_1K)]) + + global DAT_130K_C + DAT_130K_C = compress(DAT_130K_D, options={CompressionParameter.checksum_flag:1}) + + global DECOMPRESSED_DAT + DECOMPRESSED_DAT = b'abcdefg123456' * 1000 + + global COMPRESSED_DAT + COMPRESSED_DAT = compress(DECOMPRESSED_DAT) + + global DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_100_PLUS_32KB = b'a' * (100 + 32*_1K) + + global COMPRESSED_100_PLUS_32KB + COMPRESSED_100_PLUS_32KB = compress(DECOMPRESSED_100_PLUS_32KB) + + global SKIPPABLE_FRAME + SKIPPABLE_FRAME = (0x184D2A50).to_bytes(4, byteorder='little') + \ + (32*_1K).to_bytes(4, byteorder='little') + \ + b'a' * (32*_1K) + + global THIS_FILE_BYTES, THIS_FILE_STR + with io.open(os.path.abspath(__file__), 'rb') as f: + THIS_FILE_BYTES = f.read() + THIS_FILE_BYTES = re.sub(rb'\r?\n', rb'\n', THIS_FILE_BYTES) + THIS_FILE_STR = THIS_FILE_BYTES.decode('utf-8') + + global COMPRESSED_THIS_FILE + COMPRESSED_THIS_FILE = compress(THIS_FILE_BYTES) + + global COMPRESSED_BOGUS + COMPRESSED_BOGUS = DECOMPRESSED_DAT + + # dict data + words = [b'red', b'green', b'yellow', b'black', b'withe', b'blue', + b'lilac', b'purple', b'navy', b'glod', b'silver', b'olive', + b'dog', b'cat', b'tiger', b'lion', b'fish', b'bird'] + lst = [] + for i in range(300): + sample = [b'%s = %d' % (random.choice(words), random.randrange(100)) + for j in range(20)] + sample = b'\n'.join(sample) + + lst.append(sample) + global SAMPLES + SAMPLES = lst + assert len(SAMPLES) > 10 + + global TRAINED_DICT + TRAINED_DICT = train_dict(SAMPLES, 3*_1K) + assert len(TRAINED_DICT.dict_content) <= 3*_1K + + +class FunctionsTestCase(unittest.TestCase): + + def test_version(self): + s = ".".join((str(i) for i in zstd_version_info)) + self.assertEqual(s, zstd_version) + + def test_compressionLevel_values(self): + min, max = CompressionParameter.compression_level.bounds() + self.assertIs(type(COMPRESSION_LEVEL_DEFAULT), int) + self.assertIs(type(min), int) + self.assertIs(type(max), int) + self.assertLess(min, max) + + def test_roundtrip_default(self): + raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + dat1 = compress(raw_dat) + dat2 = decompress(dat1) + self.assertEqual(dat2, raw_dat) + + def test_roundtrip_level(self): + raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + level_min, level_max = CompressionParameter.compression_level.bounds() + + for level in range(max(-20, level_min), level_max + 1): + dat1 = compress(raw_dat, level) + dat2 = decompress(dat1) + self.assertEqual(dat2, raw_dat) + + def test_get_frame_info(self): + # no dict + info = get_frame_info(COMPRESSED_100_PLUS_32KB[:20]) + self.assertEqual(info.decompressed_size, 32 * _1K + 100) + self.assertEqual(info.dictionary_id, 0) + + # use dict + dat = compress(b"a" * 345, zstd_dict=TRAINED_DICT) + info = get_frame_info(dat) + self.assertEqual(info.decompressed_size, 345) + self.assertEqual(info.dictionary_id, TRAINED_DICT.dict_id) + + with self.assertRaisesRegex(ZstdError, "not less than the frame header"): + get_frame_info(b"aaaaaaaaaaaaaa") + + def test_get_frame_size(self): + size = get_frame_size(COMPRESSED_100_PLUS_32KB) + self.assertEqual(size, len(COMPRESSED_100_PLUS_32KB)) + + with self.assertRaisesRegex(ZstdError, "not less than this complete frame"): + get_frame_size(b"aaaaaaaaaaaaaa") + + def test_decompress_2x130_1K(self): + decompressed_size = get_frame_info(DAT_130K_C).decompressed_size + self.assertEqual(decompressed_size, _130_1K) + + dat = decompress(DAT_130K_C + DAT_130K_C) + self.assertEqual(len(dat), 2 * _130_1K) + + +class CompressorTestCase(unittest.TestCase): + + def test_simple_compress_bad_args(self): + # ZstdCompressor + self.assertRaises(TypeError, ZstdCompressor, []) + self.assertRaises(TypeError, ZstdCompressor, level=3.14) + self.assertRaises(TypeError, ZstdCompressor, level="abc") + self.assertRaises(TypeError, ZstdCompressor, options=b"abc") + + self.assertRaises(TypeError, ZstdCompressor, zstd_dict=123) + self.assertRaises(TypeError, ZstdCompressor, zstd_dict=b"abcd1234") + self.assertRaises(TypeError, ZstdCompressor, zstd_dict={1: 2, 3: 4}) + + # valid range for compression level is [-(1<<17), 22] + msg = r'illegal compression level {}; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MAX)): + ZstdCompressor(C_INT_MAX) + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MIN)): + ZstdCompressor(C_INT_MIN) + msg = r'illegal compression level; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=-(2**1000)) + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=2**1000) + + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.window_log: 100}) + with self.assertRaises(ValueError): + ZstdCompressor(options={3333: 100}) + + # Method bad arguments + zc = ZstdCompressor() + self.assertRaises(TypeError, zc.compress) + self.assertRaises((TypeError, ValueError), zc.compress, b"foo", b"bar") + self.assertRaises(TypeError, zc.compress, "str") + self.assertRaises((TypeError, ValueError), zc.flush, b"foo") + self.assertRaises(TypeError, zc.flush, b"blah", 1) + + self.assertRaises(ValueError, zc.compress, b'', -1) + self.assertRaises(ValueError, zc.compress, b'', 3) + self.assertRaises(ValueError, zc.flush, zc.CONTINUE) # 0 + self.assertRaises(ValueError, zc.flush, 3) + + zc.compress(b'') + zc.compress(b'', zc.CONTINUE) + zc.compress(b'', zc.FLUSH_BLOCK) + zc.compress(b'', zc.FLUSH_FRAME) + empty = zc.flush() + zc.flush(zc.FLUSH_BLOCK) + zc.flush(zc.FLUSH_FRAME) + + def test_compress_parameters(self): + d = {CompressionParameter.compression_level : 10, + + CompressionParameter.window_log : 12, + CompressionParameter.hash_log : 10, + CompressionParameter.chain_log : 12, + CompressionParameter.search_log : 12, + CompressionParameter.min_match : 4, + CompressionParameter.target_length : 12, + CompressionParameter.strategy : Strategy.lazy, + + CompressionParameter.enable_long_distance_matching : 1, + CompressionParameter.ldm_hash_log : 12, + CompressionParameter.ldm_min_match : 11, + CompressionParameter.ldm_bucket_size_log : 5, + CompressionParameter.ldm_hash_rate_log : 12, + + CompressionParameter.content_size_flag : 1, + CompressionParameter.checksum_flag : 1, + CompressionParameter.dict_id_flag : 0, + + CompressionParameter.nb_workers : 2 if SUPPORT_MULTITHREADING else 0, + CompressionParameter.job_size : 5*_1M if SUPPORT_MULTITHREADING else 0, + CompressionParameter.overlap_log : 9 if SUPPORT_MULTITHREADING else 0, + } + ZstdCompressor(options=d) + + d1 = d.copy() + # larger than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + # smaller than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + + # out of bounds compression level + level_min, level_max = CompressionParameter.compression_level.bounds() + with self.assertRaises(ValueError): + compress(b'', level_max+1) + with self.assertRaises(ValueError): + compress(b'', level_min-1) + with self.assertRaises(ValueError): + compress(b'', 2**1000) + with self.assertRaises(ValueError): + compress(b'', -(2**1000)) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_max+1}) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_min-1}) + + # zstd lib doesn't support MT compression + if not SUPPORT_MULTITHREADING: + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.nb_workers:4}) + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.job_size:4}) + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.overlap_log:4}) + + # out of bounds error msg + option = {CompressionParameter.window_log:100} + with self.assertRaisesRegex( + ValueError, + "compression parameter 'window_log' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + compress(b'', options=option) + + def test_unknown_compression_parameter(self): + KEY = 100001234 + option = {CompressionParameter.compression_level: 10, + KEY: 200000000} + pattern = rf"invalid compression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): + ZstdCompressor(options=option) + + @unittest.skipIf(not SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + def test_zstd_multithread_compress(self): + size = 40*_1M + b = THIS_FILE_BYTES * (size // len(THIS_FILE_BYTES)) + + options = {CompressionParameter.compression_level : 4, + CompressionParameter.nb_workers : 2} + + # compress() + dat1 = compress(b, options=options) + dat2 = decompress(dat1) + self.assertEqual(dat2, b) + + # ZstdCompressor + c = ZstdCompressor(options=options) + dat1 = c.compress(b, c.CONTINUE) + dat2 = c.compress(b, c.FLUSH_BLOCK) + dat3 = c.compress(b, c.FLUSH_FRAME) + dat4 = decompress(dat1+dat2+dat3) + self.assertEqual(dat4, b * 3) + + # ZstdFile + with ZstdFile(io.BytesIO(), 'w', options=options) as f: + f.write(b) + + def test_compress_flushblock(self): + point = len(THIS_FILE_BYTES) // 2 + + c = ZstdCompressor() + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + dat1 = c.compress(THIS_FILE_BYTES[:point]) + self.assertEqual(c.last_mode, c.CONTINUE) + dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_BLOCK) + self.assertEqual(c.last_mode, c.FLUSH_BLOCK) + dat2 = c.flush() + pattern = "Compressed data ended before the end-of-stream marker" + with self.assertRaisesRegex(ZstdError, pattern): + decompress(dat1) + + dat3 = decompress(dat1 + dat2) + + self.assertEqual(dat3, THIS_FILE_BYTES) + + def test_compress_flushframe(self): + # test compress & decompress + point = len(THIS_FILE_BYTES) // 2 + + c = ZstdCompressor() + + dat1 = c.compress(THIS_FILE_BYTES[:point]) + self.assertEqual(c.last_mode, c.CONTINUE) + + dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_FRAME) + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + + nt = get_frame_info(dat1) + self.assertEqual(nt.decompressed_size, None) # no content size + + dat2 = decompress(dat1) + + self.assertEqual(dat2, THIS_FILE_BYTES) + + # single .FLUSH_FRAME mode has content size + c = ZstdCompressor() + dat = c.compress(THIS_FILE_BYTES, mode=c.FLUSH_FRAME) + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + + nt = get_frame_info(dat) + self.assertEqual(nt.decompressed_size, len(THIS_FILE_BYTES)) + + def test_compress_empty(self): + # output empty content frame + self.assertNotEqual(compress(b''), b'') + + c = ZstdCompressor() + self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'') + + def test_set_pledged_input_size(self): + DAT = DECOMPRESSED_100_PLUS_32KB + CHUNK_SIZE = len(DAT) // 3 + + # wrong value + c = ZstdCompressor() + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(-300) + # overflow + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64) + # ZSTD_CONTENTSIZE_ERROR is invalid + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-2) + # ZSTD_CONTENTSIZE_UNKNOWN should use None + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-1) + + # check valid values are settable + c.set_pledged_input_size(2**63) + c.set_pledged_input_size(2**64-3) + + # check that zero means empty frame + c = ZstdCompressor(level=1) + c.set_pledged_input_size(0) + c.compress(b'') + dat = c.flush() + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, 0) + + + # wrong mode + c = ZstdCompressor(level=1) + c.compress(b'123456') + self.assertEqual(c.last_mode, c.CONTINUE) + with self.assertRaisesRegex(ValueError, + r'last_mode == FLUSH_FRAME'): + c.set_pledged_input_size(300) + + # None value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(None) + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, None) + + # correct value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + chunks = [] + posi = 0 + while posi < len(DAT): + dat = c.compress(DAT[posi:posi+CHUNK_SIZE]) + posi += CHUNK_SIZE + chunks.append(dat) + + dat = c.flush() + chunks.append(dat) + chunks = b''.join(chunks) + + ret = get_frame_info(chunks) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(chunks), DAT) + + c.set_pledged_input_size(len(DAT)) # the second frame + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(dat), DAT) + + # not enough data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)+1) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.flush() + + # too much data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.compress(b'extra', ZstdCompressor.FLUSH_FRAME) + + # content size not set if content_size_flag == 0 + c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0}) + c.set_pledged_input_size(10) + dat1 = c.compress(b"hello") + dat2 = c.compress(b"world") + dat3 = c.flush() + frame_data = get_frame_info(dat1 + dat2 + dat3) + self.assertIsNone(frame_data.decompressed_size) + + +class DecompressorTestCase(unittest.TestCase): + + def test_simple_decompress_bad_args(self): + # ZstdDecompressor + self.assertRaises(TypeError, ZstdDecompressor, ()) + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=123) + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=b'abc') + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict={1:2, 3:4}) + + self.assertRaises(TypeError, ZstdDecompressor, options=123) + self.assertRaises(TypeError, ZstdDecompressor, options='abc') + self.assertRaises(TypeError, ZstdDecompressor, options=b'abc') + + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MAX: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MIN: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={0: C_INT_MAX}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={2**1000: 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={-(2**1000): 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={0: -(2**1000)}) + + with self.assertRaises(ValueError): + ZstdDecompressor(options={DecompressionParameter.window_log_max: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={3333: 100}) + + empty = compress(b'') + lzd = ZstdDecompressor() + self.assertRaises(TypeError, lzd.decompress) + self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar") + self.assertRaises(TypeError, lzd.decompress, "str") + lzd.decompress(empty) + + def test_decompress_parameters(self): + d = {DecompressionParameter.window_log_max : 15} + ZstdDecompressor(options=d) + + d1 = d.copy() + # larger than signed int + d1[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + # smaller than signed int + d1[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + + d1[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + d1[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + + # out of bounds error msg + options = {DecompressionParameter.window_log_max:100} + with self.assertRaisesRegex( + ValueError, + "decompression parameter 'window_log_max' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + decompress(b'', options=options) + + # out of bounds deecompression parameter + options[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + decompress(b'', options=options) + + def test_unknown_decompression_parameter(self): + KEY = 100001234 + options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], + KEY: 200000000} + pattern = rf"invalid decompression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): + ZstdDecompressor(options=options) + + def test_decompress_epilogue_flags(self): + # DAT_130K_C has a 4 bytes checksum at frame epilogue + + # full unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + with self.assertRaises(EOFError): + dat = d.decompress(b'') + + # full limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C, _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + with self.assertRaises(EOFError): + dat = d.decompress(b'', 0) + + # [:-4] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-4] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + # [:-3] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-3]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-3] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-3], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + # [:-1] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-1]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-1] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-1], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + def test_decompressor_arg(self): + zd = ZstdDict(b'12345678', is_raw=True) + + with self.assertRaises(TypeError): + d = ZstdDecompressor(zstd_dict={}) + + with self.assertRaises(TypeError): + d = ZstdDecompressor(options=zd) + + ZstdDecompressor() + ZstdDecompressor(zd, {}) + ZstdDecompressor(zstd_dict=zd, options={DecompressionParameter.window_log_max:25}) + + def test_decompressor_1(self): + # empty + d = ZstdDecompressor() + dat = d.decompress(b'') + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + + # 130_1K full + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + + # 130_1K full, limit output + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C, _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + + # 130_1K, without 4 bytes checksum + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4]) + + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + + # above, limit output + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4], _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + + # full, unused_data + TRAIL = b'89234893abcd' + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C + TRAIL, _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, TRAIL) + + def test_decompressor_chunks_read_300(self): + TRAIL = b'89234893abcd' + DAT = DAT_130K_C + TRAIL + d = ZstdDecompressor() + + bi = io.BytesIO(DAT) + lst = [] + while True: + if d.needs_input: + dat = bi.read(300) + if not dat: + break + else: + raise Exception('should not get here') + + ret = d.decompress(dat) + lst.append(ret) + if d.eof: + break + + ret = b''.join(lst) + + self.assertEqual(len(ret), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data + bi.read(), TRAIL) + + def test_decompressor_chunks_read_3(self): + TRAIL = b'89234893' + DAT = DAT_130K_C + TRAIL + d = ZstdDecompressor() + + bi = io.BytesIO(DAT) + lst = [] + while True: + if d.needs_input: + dat = bi.read(3) + if not dat: + break + else: + dat = b'' + + ret = d.decompress(dat, 1) + lst.append(ret) + if d.eof: + break + + ret = b''.join(lst) + + self.assertEqual(len(ret), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data + bi.read(), TRAIL) + + + def test_decompress_empty(self): + with self.assertRaises(ZstdError): + decompress(b'') + + d = ZstdDecompressor() + self.assertEqual(d.decompress(b''), b'') + self.assertFalse(d.eof) + + def test_decompress_empty_content_frame(self): + DAT = compress(b'') + # decompress + self.assertGreaterEqual(len(DAT), 4) + self.assertEqual(decompress(DAT), b'') + + with self.assertRaises(ZstdError): + decompress(DAT[:-1]) + + # ZstdDecompressor + d = ZstdDecompressor() + dat = d.decompress(DAT) + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + d = ZstdDecompressor() + dat = d.decompress(DAT[:-1]) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + +class DecompressorFlagsTestCase(unittest.TestCase): + + @classmethod + def setUpClass(cls): + options = {CompressionParameter.checksum_flag:1} + c = ZstdCompressor(options=options) + + cls.DECOMPRESSED_42 = b'a'*42 + cls.FRAME_42 = c.compress(cls.DECOMPRESSED_42, c.FLUSH_FRAME) + + cls.DECOMPRESSED_60 = b'a'*60 + cls.FRAME_60 = c.compress(cls.DECOMPRESSED_60, c.FLUSH_FRAME) + + cls.FRAME_42_60 = cls.FRAME_42 + cls.FRAME_60 + cls.DECOMPRESSED_42_60 = cls.DECOMPRESSED_42 + cls.DECOMPRESSED_60 + + cls._130_1K = 130*_1K + + c = ZstdCompressor() + cls.UNKNOWN_FRAME_42 = c.compress(cls.DECOMPRESSED_42) + c.flush() + cls.UNKNOWN_FRAME_60 = c.compress(cls.DECOMPRESSED_60) + c.flush() + cls.UNKNOWN_FRAME_42_60 = cls.UNKNOWN_FRAME_42 + cls.UNKNOWN_FRAME_60 + + cls.TRAIL = b'12345678abcdefg!@#$%^&*()_+|' + + def test_function_decompress(self): + + self.assertEqual(len(decompress(COMPRESSED_100_PLUS_32KB)), 100+32*_1K) + + # 1 frame + self.assertEqual(decompress(self.FRAME_42), self.DECOMPRESSED_42) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42), self.DECOMPRESSED_42) + + pattern = r"Compressed data ended before the end-of-stream marker" + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:1]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:-1]) + + # 2 frames + self.assertEqual(decompress(self.FRAME_42_60), self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60), self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.FRAME_42 + self.UNKNOWN_FRAME_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42 + self.FRAME_60), + self.DECOMPRESSED_42_60) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42_60[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.UNKNOWN_FRAME_42_60[:-1]) + + # 130_1K + self.assertEqual(decompress(DAT_130K_C), DAT_130K_D) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(DAT_130K_C[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(DAT_130K_C[:-1]) + + # Unknown frame descriptor + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(self.FRAME_42 + b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(self.UNKNOWN_FRAME_42_60 + b'aaaaaaaaa') + + # doesn't match checksum + checksum = DAT_130K_C[-4:] + if checksum[0] == 255: + wrong_checksum = bytes([254]) + checksum[1:] + else: + wrong_checksum = bytes([checksum[0]+1]) + checksum[1:] + + dat = DAT_130K_C[:-4] + wrong_checksum + + with self.assertRaisesRegex(ZstdError, "doesn't match checksum"): + decompress(dat) + + def test_function_skippable(self): + self.assertEqual(decompress(SKIPPABLE_FRAME), b'') + self.assertEqual(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME), b'') + + # 1 frame + 2 skippable + self.assertEqual(len(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + DAT_130K_C)), + self._130_1K) + + self.assertEqual(len(decompress(DAT_130K_C + SKIPPABLE_FRAME + SKIPPABLE_FRAME)), + self._130_1K) + + self.assertEqual(len(decompress(SKIPPABLE_FRAME + DAT_130K_C + SKIPPABLE_FRAME)), + self._130_1K) + + # unknown size + self.assertEqual(decompress(SKIPPABLE_FRAME + self.UNKNOWN_FRAME_60), + self.DECOMPRESSED_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_60 + SKIPPABLE_FRAME), + self.DECOMPRESSED_60) + + # 2 frames + 1 skippable + self.assertEqual(decompress(self.FRAME_42 + SKIPPABLE_FRAME + self.FRAME_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(SKIPPABLE_FRAME + self.FRAME_42_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60 + SKIPPABLE_FRAME), + self.DECOMPRESSED_42_60) + + # incomplete + with self.assertRaises(ZstdError): + decompress(SKIPPABLE_FRAME[:1]) + + with self.assertRaises(ZstdError): + decompress(SKIPPABLE_FRAME[:-1]) + + with self.assertRaises(ZstdError): + decompress(self.FRAME_42 + SKIPPABLE_FRAME[:-1]) + + # Unknown frame descriptor + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(b'aaaaaaaaa' + SKIPPABLE_FRAME) + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(SKIPPABLE_FRAME + b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + b'aaaaaaaaa') + + def test_decompressor_1(self): + # empty 1 + d = ZstdDecompressor() + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'', 0) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a') + self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'a') + self.assertEqual(d.unused_data, b'a') # twice + + # empty 2 + d = ZstdDecompressor() + + dat = d.decompress(b'', 0) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a') + self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'a') + self.assertEqual(d.unused_data, b'a') # twice + + # 1 frame + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_42) + + self.assertEqual(dat, self.DECOMPRESSED_42) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # 1 frame, trail + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_42 + self.TRAIL) + + self.assertEqual(dat, self.DECOMPRESSED_42) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + # 1 frame, 32_1K + temp = compress(b'a'*(32*_1K)) + d = ZstdDecompressor() + dat = d.decompress(temp, 32*_1K) + + self.assertEqual(dat, b'a'*(32*_1K)) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # 1 frame, 32_1K+100, trail + d = ZstdDecompressor() + dat = d.decompress(COMPRESSED_100_PLUS_32KB+self.TRAIL, 100) # 100 bytes + + self.assertEqual(len(dat), 100) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + + dat = d.decompress(b'') # 32_1K + + self.assertEqual(len(dat), 32*_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # incomplete 1 + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_60[:1]) + + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete 2 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-4]) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete 3 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-1]) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + + # incomplete 4 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-4], 60) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # Unknown frame descriptor + d = ZstdDecompressor() + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + d.decompress(b'aaaaaaaaa') + + def test_decompressor_skippable(self): + # 1 skippable + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # 1 skippable, max_length=0 + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME, 0) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # 1 skippable, trail + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME + self.TRAIL) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + # incomplete + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME[:-1]) + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME[:-1], 0) + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + + +class ZstdDictTestCase(unittest.TestCase): + + def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + + # content < 8 + b = b'1234567' + with self.assertRaises(ValueError): + ZstdDict(b) + + # content == 8 + b = b'12345678' + zd = ZstdDict(b, is_raw=True) + self.assertEqual(zd.dict_id, 0) + + temp = compress(b'aaa12345678', level=3, zstd_dict=zd) + self.assertEqual(b'aaa12345678', decompress(temp, zd)) + + # is_raw == False + b = b'12345678abcd' + with self.assertRaises(ValueError): + ZstdDict(b) + + # read only attributes + with self.assertRaises(AttributeError): + zd.dict_content = b + + with self.assertRaises(AttributeError): + zd.dict_id = 10000 + + # ZstdDict arguments + zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=False) + self.assertNotEqual(zd.dict_id, 0) + + zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=True) + self.assertNotEqual(zd.dict_id, 0) # note this assertion + + with self.assertRaises(TypeError): + ZstdDict("12345678abcdef", is_raw=True) + with self.assertRaises(TypeError): + ZstdDict(TRAINED_DICT) + + # invalid parameter + with self.assertRaises(TypeError): + ZstdDict(desk333=345) + + def test_invalid_dict(self): + DICT_MAGIC = 0xEC30A437.to_bytes(4, byteorder='little') + dict_content = DICT_MAGIC + b'abcdefghighlmnopqrstuvwxyz' + + # corrupted + zd = ZstdDict(dict_content, is_raw=False) + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): + ZstdCompressor(zstd_dict=zd.as_digested_dict) + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): + ZstdDecompressor(zd) + + # wrong type + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1, 2)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, -1)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 3)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, -2**1000)) + + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, 1, 2)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, -1)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, 3)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, -2**1000)) + + def test_train_dict(self): + TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) + + self.assertNotEqual(TRAINED_DICT.dict_id, 0) + self.assertGreater(len(TRAINED_DICT.dict_content), 0) + self.assertLessEqual(len(TRAINED_DICT.dict_content), DICT_SIZE1) + self.assertTrue(re.match(r'^<ZstdDict dict_id=\d+ dict_size=\d+>$', str(TRAINED_DICT))) + + # compress/decompress + c = ZstdCompressor(zstd_dict=TRAINED_DICT) + for sample in SAMPLES: + dat1 = compress(sample, zstd_dict=TRAINED_DICT) + dat2 = decompress(dat1, TRAINED_DICT) + self.assertEqual(sample, dat2) + + dat1 = c.compress(sample) + dat1 += c.flush() + dat2 = decompress(dat1, TRAINED_DICT) + self.assertEqual(sample, dat2) + + def test_finalize_dict(self): + DICT_SIZE2 = 200*_1K + C_LEVEL = 6 + + try: + dic2 = finalize_dict(TRAINED_DICT, SAMPLES, DICT_SIZE2, C_LEVEL) + except NotImplementedError: + # < v1.4.5 at compile-time, >= v.1.4.5 at run-time + return + + self.assertNotEqual(dic2.dict_id, 0) + self.assertGreater(len(dic2.dict_content), 0) + self.assertLessEqual(len(dic2.dict_content), DICT_SIZE2) + + # compress/decompress + c = ZstdCompressor(C_LEVEL, zstd_dict=dic2) + for sample in SAMPLES: + dat1 = compress(sample, C_LEVEL, zstd_dict=dic2) + dat2 = decompress(dat1, dic2) + self.assertEqual(sample, dat2) + + dat1 = c.compress(sample) + dat1 += c.flush() + dat2 = decompress(dat1, dic2) + self.assertEqual(sample, dat2) + + # dict mismatch + self.assertNotEqual(TRAINED_DICT.dict_id, dic2.dict_id) + + dat1 = compress(SAMPLES[0], zstd_dict=TRAINED_DICT) + with self.assertRaises(ZstdError): + decompress(dat1, dic2) + + def test_train_dict_arguments(self): + with self.assertRaises(ValueError): + train_dict([], 100*_1K) + + with self.assertRaises(ValueError): + train_dict(SAMPLES, -100) + + with self.assertRaises(ValueError): + train_dict(SAMPLES, 0) + + def test_finalize_dict_arguments(self): + with self.assertRaises(TypeError): + finalize_dict({1:2}, (b'aaa', b'bbb'), 100*_1K, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, [], 100*_1K, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, SAMPLES, -100, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, SAMPLES, 0, 2) + + def test_train_dict_c(self): + # argument wrong type + with self.assertRaises(TypeError): + _zstd.train_dict({}, (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(bytearray(), (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', 99, 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', [], 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (), 100.1) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (99.1,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (4, -1), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (2,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (99,), 100) + + # size > size_t + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (2**1000,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (-2**1000,), 100) + + # dict_size <= 0 + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), 0) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), -1) + + with self.assertRaises(ZstdError): + _zstd.train_dict(b'', (), 1) + + def test_finalize_dict_c(self): + with self.assertRaises(TypeError): + _zstd.finalize_dict(1, 2, 3, 4, 5) + + # argument wrong type + with self.assertRaises(TypeError): + _zstd.finalize_dict({}, b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5) + + # size > size_t + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5) + + # dict_size <= 0 + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5) + + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000) + + with self.assertRaises(ZstdError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5) + + def test_train_buffer_protocol_samples(self): + def _nbytes(dat): + if isinstance(dat, (bytes, bytearray)): + return len(dat) + return memoryview(dat).nbytes + + # prepare samples + chunk_lst = [] + wrong_size_lst = [] + correct_size_lst = [] + for _ in range(300): + arr = array.array('Q', [random.randint(0, 20) for i in range(20)]) + chunk_lst.append(arr) + correct_size_lst.append(_nbytes(arr)) + wrong_size_lst.append(len(arr)) + concatenation = b''.join(chunk_lst) + + # wrong size list + with self.assertRaisesRegex(ValueError, + "The samples size tuple doesn't match the concatenation's size"): + _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) + + # correct size list + _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K) + + # wrong size list + with self.assertRaisesRegex(ValueError, + "The samples size tuple doesn't match the concatenation's size"): + _zstd.finalize_dict(TRAINED_DICT.dict_content, + concatenation, tuple(wrong_size_lst), 300*_1K, 5) + + # correct size list + _zstd.finalize_dict(TRAINED_DICT.dict_content, + concatenation, tuple(correct_size_lst), 300*_1K, 5) + + def test_as_prefix(self): + # V1 + V1 = THIS_FILE_BYTES + zd = ZstdDict(V1, is_raw=True) + + # V2 + mid = len(V1) // 2 + V2 = V1[:mid] + \ + (b'a' if V1[mid] != int.from_bytes(b'a') else b'b') + \ + V1[mid+1:] + + # compress + dat = compress(V2, zstd_dict=zd.as_prefix) + self.assertEqual(get_frame_info(dat).dictionary_id, 0) + + # decompress + self.assertEqual(decompress(dat, zd.as_prefix), V2) + + # use wrong prefix + zd2 = ZstdDict(SAMPLES[0], is_raw=True) + try: + decompressed = decompress(dat, zd2.as_prefix) + except ZstdError: # expected + pass + else: + self.assertNotEqual(decompressed, V2) + + # read only attribute + with self.assertRaises(AttributeError): + zd.as_prefix = b'1234' + + def test_as_digested_dict(self): + zd = TRAINED_DICT + + # test .as_digested_dict + dat = compress(SAMPLES[0], zstd_dict=zd.as_digested_dict) + self.assertEqual(decompress(dat, zd.as_digested_dict), SAMPLES[0]) + with self.assertRaises(AttributeError): + zd.as_digested_dict = b'1234' + + # test .as_undigested_dict + dat = compress(SAMPLES[0], zstd_dict=zd.as_undigested_dict) + self.assertEqual(decompress(dat, zd.as_undigested_dict), SAMPLES[0]) + with self.assertRaises(AttributeError): + zd.as_undigested_dict = b'1234' + + def test_advanced_compression_parameters(self): + options = {CompressionParameter.compression_level: 6, + CompressionParameter.window_log: 20, + CompressionParameter.enable_long_distance_matching: 1} + + # automatically select + dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT) + self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0]) + + # explicitly select + dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT.as_digested_dict) + self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0]) + + def test_len(self): + self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) + self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + +class FileTestCase(unittest.TestCase): + def setUp(self): + self.DECOMPRESSED_42 = b'a'*42 + self.FRAME_42 = compress(self.DECOMPRESSED_42) + + def test_init(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + pass + with ZstdFile(io.BytesIO(), "w") as f: + pass + with ZstdFile(io.BytesIO(), "x") as f: + pass + with ZstdFile(io.BytesIO(), "a") as f: + pass + + with ZstdFile(io.BytesIO(), "w", level=12) as f: + pass + with ZstdFile(io.BytesIO(), "w", options={CompressionParameter.checksum_flag:1}) as f: + pass + with ZstdFile(io.BytesIO(), "w", options={}) as f: + pass + with ZstdFile(io.BytesIO(), "w", level=20, zstd_dict=TRAINED_DICT) as f: + pass + + with ZstdFile(io.BytesIO(), "r", options={DecompressionParameter.window_log_max:25}) as f: + pass + with ZstdFile(io.BytesIO(), "r", options={}, zstd_dict=TRAINED_DICT) as f: + pass + + def test_init_with_PathLike_filename(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + with ZstdFile(filename, "a") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(filename) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + with ZstdFile(filename, "a") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(filename) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 2) + + os.remove(filename) + + def test_init_with_filename(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + with ZstdFile(filename) as f: + pass + with ZstdFile(filename, "w") as f: + pass + with ZstdFile(filename, "a") as f: + pass + + os.remove(filename) + + def test_init_mode(self): + bi = io.BytesIO() + + with ZstdFile(bi, "r"): + pass + with ZstdFile(bi, "rb"): + pass + with ZstdFile(bi, "w"): + pass + with ZstdFile(bi, "wb"): + pass + with ZstdFile(bi, "a"): + pass + with ZstdFile(bi, "ab"): + pass + + def test_init_with_x_mode(self): + with tempfile.NamedTemporaryFile() as tmp_f: + filename = pathlib.Path(tmp_f.name) + + for mode in ("x", "xb"): + with ZstdFile(filename, mode): + pass + with self.assertRaises(FileExistsError): + with ZstdFile(filename, mode): + pass + os.remove(filename) + + def test_init_bad_mode(self): + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), (3, "x")) + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "xt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "x+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rx") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wx") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "w+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw") + + with self.assertRaisesRegex(TypeError, + r"not be a CompressionParameter"): + ZstdFile(io.BytesIO(), 'rb', + options={CompressionParameter.compression_level:5}) + with self.assertRaisesRegex(TypeError, + r"not be a DecompressionParameter"): + ZstdFile(io.BytesIO(), 'wb', + options={DecompressionParameter.window_log_max:21}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", level=12) + + def test_init_bad_check(self): + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(), "w", level='asd') + # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(), "w", options={999:9999}) + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(), "w", options={CompressionParameter.window_log:99}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", options=33) + + with self.assertRaises(OverflowError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={DecompressionParameter.window_log_max:2**31}) + + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={444:333}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict={1:2}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict=b'dict123456') + + def test_init_close_fp(self): + # get a temp file name + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + tmp_f.write(DAT_130K_C) + filename = tmp_f.name + + with self.assertRaises(TypeError): + ZstdFile(filename, options={'a':'b'}) + + # for PyPy + gc.collect() + + os.remove(filename) + + def test_close(self): + with io.BytesIO(COMPRESSED_100_PLUS_32KB) as src: + f = ZstdFile(src) + f.close() + # ZstdFile.close() should not close the underlying file object. + self.assertFalse(src.closed) + # Try closing an already-closed ZstdFile. + f.close() + self.assertFalse(src.closed) + + # Test with a real file on disk, opened directly by ZstdFile. + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + fp = f._fp + f.close() + # Here, ZstdFile.close() *should* close the underlying file object. + self.assertTrue(fp.closed) + # Try closing an already-closed ZstdFile. + f.close() + + os.remove(filename) + + def test_closed(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertFalse(f.closed) + f.read() + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + def test_fileno(self): + # 1 + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertRaises(io.UnsupportedOperation, f.fileno) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + # 2 + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + try: + self.assertEqual(f.fileno(), f._fp.fileno()) + self.assertIsInstance(f.fileno(), int) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + os.remove(filename) + + # 3, no .fileno() method + class C: + def read(self, size=-1): + return b'123' + with ZstdFile(C(), 'rb') as f: + with self.assertRaisesRegex(AttributeError, r'fileno'): + f.fileno() + + def test_name(self): + # 1 + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + with self.assertRaises(AttributeError): + f.name + finally: + f.close() + with self.assertRaises(ValueError): + f.name + + # 2 + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + try: + self.assertEqual(f.name, f._fp.name) + self.assertIsInstance(f.name, str) + finally: + f.close() + with self.assertRaises(ValueError): + f.name + + os.remove(filename) + + # 3, no .filename property + class C: + def read(self, size=-1): + return b'123' + with ZstdFile(C(), 'rb') as f: + with self.assertRaisesRegex(AttributeError, r'name'): + f.name + + def test_seekable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertTrue(f.seekable()) + f.read() + self.assertTrue(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + src = io.BytesIO(COMPRESSED_100_PLUS_32KB) + src.seekable = lambda: False + f = ZstdFile(src) + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + def test_readable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertTrue(f.readable()) + f.read() + self.assertTrue(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + def test_writable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertFalse(f.writable()) + f.read() + self.assertFalse(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertTrue(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + def test_read_0(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertEqual(f.read(0), b"") + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={DecompressionParameter.window_log_max:20}) as f: + self.assertEqual(f.read(0), b"") + + # empty file + with ZstdFile(io.BytesIO(b'')) as f: + self.assertEqual(f.read(0), b"") + with self.assertRaises(EOFError): + f.read(10) + + with ZstdFile(io.BytesIO(b'')) as f: + with self.assertRaises(EOFError): + f.read(10) + + def test_read_10(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + chunks = [] + while True: + result = f.read(10) + if not result: + break + self.assertLessEqual(len(result), 10) + chunks.append(result) + self.assertEqual(b"".join(chunks), DECOMPRESSED_100_PLUS_32KB) + + def test_read_multistream(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 5) + + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + SKIPPABLE_FRAME)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + COMPRESSED_DAT)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_DAT) + + def test_read_incomplete(self): + with ZstdFile(io.BytesIO(DAT_130K_C[:-200])) as f: + self.assertRaises(EOFError, f.read) + + # Trailing data isn't a valid compressed stream + with ZstdFile(io.BytesIO(self.FRAME_42 + b'12345')) as f: + self.assertRaises(ZstdError, f.read) + + with ZstdFile(io.BytesIO(SKIPPABLE_FRAME + b'12345')) as f: + self.assertRaises(ZstdError, f.read) + + def test_read_truncated(self): + # Drop stream epilogue: 4 bytes checksum + truncated = DAT_130K_C[:-4] + with ZstdFile(io.BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + + with ZstdFile(io.BytesIO(truncated)) as f: + # this is an important test, make sure it doesn't raise EOFError. + self.assertEqual(f.read(130*_1K), DAT_130K_D) + with self.assertRaises(EOFError): + f.read(1) + + # Incomplete header + for i in range(1, 20): + with ZstdFile(io.BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + + def test_read_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_DAT)) + f.close() + self.assertRaises(ValueError, f.read) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read) + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + self.assertRaises(TypeError, f.read, float()) + + def test_read_bad_data(self): + with ZstdFile(io.BytesIO(COMPRESSED_BOGUS)) as f: + self.assertRaises(ZstdError, f.read) + + def test_read_exception(self): + class C: + def read(self, size=-1): + raise OSError + with ZstdFile(C()) as f: + with self.assertRaises(OSError): + f.read(10) + + def test_read1(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + blocks = [] + while True: + result = f.read1() + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DAT_130K_D) + self.assertEqual(f.read1(), b"") + + def test_read1_0(self): + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + self.assertEqual(f.read1(0), b"") + + def test_read1_10(self): + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + blocks = [] + while True: + result = f.read1(10) + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DECOMPRESSED_DAT) + self.assertEqual(f.read1(), b"") + + def test_read1_multistream(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f: + blocks = [] + while True: + result = f.read1() + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DECOMPRESSED_100_PLUS_32KB * 5) + self.assertEqual(f.read1(), b"") + + def test_read1_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.read1) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read1) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertRaises(TypeError, f.read1, None) + + def test_readinto(self): + arr = array.array("I", range(100)) + self.assertEqual(len(arr), 100) + self.assertEqual(len(arr) * arr.itemsize, 400) + ba = bytearray(300) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + # 0 length output buffer + self.assertEqual(f.readinto(ba[0:0]), 0) + + # use correct length for buffer protocol object + self.assertEqual(f.readinto(arr), 400) + self.assertEqual(arr.tobytes(), DECOMPRESSED_100_PLUS_32KB[:400]) + + # normal readinto + self.assertEqual(f.readinto(ba), 300) + self.assertEqual(ba, DECOMPRESSED_100_PLUS_32KB[400:700]) + + def test_peek(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + result = f.peek() + self.assertGreater(len(result), 0) + self.assertTrue(DAT_130K_D.startswith(result)) + self.assertEqual(f.read(), DAT_130K_D) + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + result = f.peek(10) + self.assertGreater(len(result), 0) + self.assertTrue(DAT_130K_D.startswith(result)) + self.assertEqual(f.read(), DAT_130K_D) + + def test_peek_bad_args(self): + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.peek) + + def test_iterator(self): + with io.BytesIO(THIS_FILE_BYTES) as f: + lines = f.readlines() + compressed = compress(THIS_FILE_BYTES) + + # iter + with ZstdFile(io.BytesIO(compressed)) as f: + self.assertListEqual(list(iter(f)), lines) + + # readline + with ZstdFile(io.BytesIO(compressed)) as f: + for line in lines: + self.assertEqual(f.readline(), line) + self.assertEqual(f.readline(), b'') + self.assertEqual(f.readline(), b'') + + # readlines + with ZstdFile(io.BytesIO(compressed)) as f: + self.assertListEqual(f.readlines(), lines) + + def test_decompress_limited(self): + _ZSTD_DStreamInSize = 128*_1K + 3 + + bomb = compress(b'\0' * int(2e6), level=10) + self.assertLess(len(bomb), _ZSTD_DStreamInSize) + + decomp = ZstdFile(io.BytesIO(bomb)) + self.assertEqual(decomp.read(1), b'\0') + + # BufferedReader uses 128 KiB buffer in __init__.py + max_decomp = 128*_1K + self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, + "Excessive amount of data was decompressed") + + def test_write(self): + raw_data = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.write(raw_data) + + comp = ZstdCompressor() + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + with ZstdFile(dst, "w", level=12) as f: + f.write(raw_data) + + comp = ZstdCompressor(12) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + with ZstdFile(dst, "w", options={CompressionParameter.checksum_flag:1}) as f: + f.write(raw_data) + + comp = ZstdCompressor(options={CompressionParameter.checksum_flag:1}) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + options = {CompressionParameter.compression_level:-5, + CompressionParameter.checksum_flag:1} + with ZstdFile(dst, "w", + options=options) as f: + f.write(raw_data) + + comp = ZstdCompressor(options=options) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + def test_write_empty_frame(self): + # .FLUSH_FRAME generates an empty content frame + c = ZstdCompressor() + self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'') + self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'') + + # don't generate empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + pass + self.assertEqual(bo.getvalue(), b'') + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_FRAME) + self.assertEqual(bo.getvalue(), b'') + + # if .write(b''), generate empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'') + self.assertNotEqual(bo.getvalue(), b'') + + # has an empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_BLOCK) + self.assertNotEqual(bo.getvalue(), b'') + + def test_write_empty_block(self): + # If no internal data, .FLUSH_BLOCK return b''. + c = ZstdCompressor() + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + self.assertNotEqual(c.compress(b'123', c.FLUSH_BLOCK), + b'') + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + self.assertEqual(c.compress(b''), b'') + self.assertEqual(c.compress(b''), b'') + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + + # mode = .last_mode + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'123') + f.flush(f.FLUSH_BLOCK) + fp_pos = f._fp.tell() + self.assertNotEqual(fp_pos, 0) + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), fp_pos) + + # mode != .last_mode + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), 0) + f.write(b'') + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), 0) + + def test_write_101(self): + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + for start in range(0, len(THIS_FILE_BYTES), 101): + f.write(THIS_FILE_BYTES[start:start+101]) + + comp = ZstdCompressor() + expected = comp.compress(THIS_FILE_BYTES) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + def test_write_append(self): + def comp(data): + comp = ZstdCompressor() + return comp.compress(data) + comp.flush() + + part1 = THIS_FILE_BYTES[:_1K] + part2 = THIS_FILE_BYTES[_1K:1536] + part3 = THIS_FILE_BYTES[1536:] + expected = b"".join(comp(x) for x in (part1, part2, part3)) + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.write(part1) + with ZstdFile(dst, "a") as f: + f.write(part2) + with ZstdFile(dst, "a") as f: + f.write(part3) + self.assertEqual(dst.getvalue(), expected) + + def test_write_bad_args(self): + f = ZstdFile(io.BytesIO(), "w") + f.close() + self.assertRaises(ValueError, f.write, b"foo") + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r") as f: + self.assertRaises(ValueError, f.write, b"bar") + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(TypeError, f.write, None) + self.assertRaises(TypeError, f.write, "text") + self.assertRaises(TypeError, f.write, 789) + + def test_writelines(self): + def comp(data): + comp = ZstdCompressor() + return comp.compress(data) + comp.flush() + + with io.BytesIO(THIS_FILE_BYTES) as f: + lines = f.readlines() + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.writelines(lines) + expected = comp(THIS_FILE_BYTES) + self.assertEqual(dst.getvalue(), expected) + + def test_seek_forward(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(555) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[555:]) + + def test_seek_forward_across_streams(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f: + f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 123) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[123:]) + + def test_seek_forward_relative_to_current(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.read(100) + f.seek(1236, 1) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[1336:]) + + def test_seek_forward_relative_to_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-555, 2) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-555:]) + + def test_seek_backward(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.read(1001) + f.seek(211) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[211:]) + + def test_seek_backward_across_streams(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f: + f.read(len(DECOMPRESSED_100_PLUS_32KB) + 333) + f.seek(737) + self.assertEqual(f.read(), + DECOMPRESSED_100_PLUS_32KB[737:] + DECOMPRESSED_100_PLUS_32KB) + + def test_seek_backward_relative_to_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-150, 2) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-150:]) + + def test_seek_past_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 9001) + self.assertEqual(f.tell(), len(DECOMPRESSED_100_PLUS_32KB)) + self.assertEqual(f.read(), b"") + + def test_seek_past_start(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-88) + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + def test_seek_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.seek, 0) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.seek, 0) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertRaises(ValueError, f.seek, 0, 3) + # io.BufferedReader raises TypeError instead of ValueError + self.assertRaises((TypeError, ValueError), f.seek, 9, ()) + self.assertRaises(TypeError, f.seek, None) + self.assertRaises(TypeError, f.seek, b"derp") + + def test_seek_not_seekable(self): + class C(io.BytesIO): + def seekable(self): + return False + obj = C(COMPRESSED_100_PLUS_32KB) + with ZstdFile(obj, 'r') as f: + d = f.read(1) + self.assertFalse(f.seekable()) + with self.assertRaisesRegex(io.UnsupportedOperation, + 'File or stream is not seekable'): + f.seek(0) + d += f.read() + self.assertEqual(d, DECOMPRESSED_100_PLUS_32KB) + + def test_tell(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + pos = 0 + while True: + self.assertEqual(f.tell(), pos) + result = f.read(random.randint(171, 189)) + if not result: + break + pos += len(result) + self.assertEqual(f.tell(), len(DAT_130K_D)) + with ZstdFile(io.BytesIO(), "w") as f: + for pos in range(0, len(DAT_130K_D), 143): + self.assertEqual(f.tell(), pos) + f.write(DAT_130K_D[pos:pos+143]) + self.assertEqual(f.tell(), len(DAT_130K_D)) + + def test_tell_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.tell) + + def test_file_dict(self): + # default + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # .as_(un)digested_dict + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_file_prefix(self): + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_prefix) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_UnsupportedOperation(self): + # 1 + with ZstdFile(io.BytesIO(), 'r') as f: + with self.assertRaises(io.UnsupportedOperation): + f.write(b'1234') + + # 2 + class T: + def read(self, size): + return b'a' * size + + with self.assertRaises(TypeError): # on creation + with ZstdFile(T(), 'w') as f: + pass + + # 3 + with ZstdFile(io.BytesIO(), 'w') as f: + with self.assertRaises(io.UnsupportedOperation): + f.read(100) + with self.assertRaises(io.UnsupportedOperation): + f.seek(100) + self.assertEqual(f.closed, True) + with self.assertRaises(ValueError): + f.readable() + with self.assertRaises(ValueError): + f.tell() + with self.assertRaises(ValueError): + f.read(100) + + def test_read_readinto_readinto1(self): + lst = [] + with ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE*5)) as f: + while True: + method = random.randint(0, 2) + size = random.randint(0, 300) + + if method == 0: + dat = f.read(size) + if not dat and size: + break + lst.append(dat) + elif method == 1: + ba = bytearray(size) + read_size = f.readinto(ba) + if read_size == 0 and size: + break + lst.append(bytes(ba[:read_size])) + elif method == 2: + ba = bytearray(size) + read_size = f.readinto1(ba) + if read_size == 0 and size: + break + lst.append(bytes(ba[:read_size])) + self.assertEqual(b''.join(lst), THIS_FILE_BYTES*5) + + def test_zstdfile_flush(self): + # closed + f = ZstdFile(io.BytesIO(), 'w') + f.close() + with self.assertRaises(ValueError): + f.flush() + + # read + with ZstdFile(io.BytesIO(), 'r') as f: + # does nothing for read-only stream + f.flush() + + # write + DAT = b'abcd' + bi = io.BytesIO() + with ZstdFile(bi, 'w') as f: + self.assertEqual(f.write(DAT), len(DAT)) + self.assertEqual(f.tell(), len(DAT)) + self.assertEqual(bi.tell(), 0) # not enough for a block + + self.assertEqual(f.flush(), None) + self.assertEqual(f.tell(), len(DAT)) + self.assertGreater(bi.tell(), 0) # flushed + + # write, no .flush() method + class C: + def write(self, b): + return len(b) + with ZstdFile(C(), 'w') as f: + self.assertEqual(f.write(DAT), len(DAT)) + self.assertEqual(f.tell(), len(DAT)) + + self.assertEqual(f.flush(), None) + self.assertEqual(f.tell(), len(DAT)) + + def test_zstdfile_flush_mode(self): + self.assertEqual(ZstdFile.FLUSH_BLOCK, ZstdCompressor.FLUSH_BLOCK) + self.assertEqual(ZstdFile.FLUSH_FRAME, ZstdCompressor.FLUSH_FRAME) + with self.assertRaises(AttributeError): + ZstdFile.CONTINUE + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + # flush block + self.assertEqual(f.write(b'123'), 3) + self.assertIsNone(f.flush(f.FLUSH_BLOCK)) + p1 = bo.tell() + # mode == .last_mode, should return + self.assertIsNone(f.flush()) + p2 = bo.tell() + self.assertEqual(p1, p2) + # flush frame + self.assertEqual(f.write(b'456'), 3) + self.assertIsNone(f.flush(mode=f.FLUSH_FRAME)) + # flush frame + self.assertEqual(f.write(b'789'), 3) + self.assertIsNone(f.flush(f.FLUSH_FRAME)) + p1 = bo.tell() + # mode == .last_mode, should return + self.assertIsNone(f.flush(f.FLUSH_FRAME)) + p2 = bo.tell() + self.assertEqual(p1, p2) + self.assertEqual(decompress(bo.getvalue()), b'123456789') + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'123') + with self.assertRaisesRegex(ValueError, r'\.FLUSH_.*?\.FLUSH_'): + f.flush(ZstdCompressor.CONTINUE) + with self.assertRaises(ValueError): + f.flush(-1) + with self.assertRaises(ValueError): + f.flush(123456) + with self.assertRaises(TypeError): + f.flush(node=ZstdCompressor.CONTINUE) + with self.assertRaises((TypeError, ValueError)): + f.flush('FLUSH_FRAME') + with self.assertRaises(TypeError): + f.flush(b'456', f.FLUSH_BLOCK) + + def test_zstdfile_truncate(self): + with ZstdFile(io.BytesIO(), 'w') as f: + with self.assertRaises(io.UnsupportedOperation): + f.truncate(200) + + def test_zstdfile_iter_issue45475(self): + lines = [l for l in ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE))] + self.assertGreater(len(lines), 0) + + def test_append_new_file(self): + with tempfile.NamedTemporaryFile(delete=True) as tmp_f: + filename = tmp_f.name + + with ZstdFile(filename, 'a') as f: + pass + self.assertTrue(os.path.isfile(filename)) + + os.remove(filename) + +class OpenTestCase(unittest.TestCase): + + def test_binary_modes(self): + with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb") as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + with io.BytesIO() as bio: + with open(bio, "wb") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB) + with open(bio, "ab") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB * 2) + + def test_text_modes(self): + # empty input + with self.assertRaises(EOFError): + with open(io.BytesIO(b''), "rt", encoding="utf-8", newline='\n') as reader: + for _ in reader: + pass + + # read + uncompressed = THIS_FILE_STR.replace(os.linesep, "\n") + with open(io.BytesIO(COMPRESSED_THIS_FILE), "rt", encoding="utf-8") as f: + self.assertEqual(f.read(), uncompressed) + + with io.BytesIO() as bio: + # write + with open(bio, "wt", encoding="utf-8") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-8") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed) + # append + with open(bio, "at", encoding="utf-8") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-8") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed * 2) + + def test_bad_params(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + TESTFN = pathlib.Path(tmp_f.name) + + with self.assertRaises(ValueError): + open(TESTFN, "") + with self.assertRaises(ValueError): + open(TESTFN, "rbt") + with self.assertRaises(ValueError): + open(TESTFN, "rb", encoding="utf-8") + with self.assertRaises(ValueError): + open(TESTFN, "rb", errors="ignore") + with self.assertRaises(ValueError): + open(TESTFN, "rb", newline="\n") + + os.remove(TESTFN) + + def test_option(self): + options = {DecompressionParameter.window_log_max:25} + with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb", options=options) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + options = {CompressionParameter.compression_level:12} + with io.BytesIO() as bio: + with open(bio, "wb", options=options) as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB) + + def test_encoding(self): + uncompressed = THIS_FILE_STR.replace(os.linesep, "\n") + + with io.BytesIO() as bio: + with open(bio, "wt", encoding="utf-16-le") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-16-le") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed) + bio.seek(0) + with open(bio, "rt", encoding="utf-16-le") as f: + self.assertEqual(f.read().replace(os.linesep, "\n"), uncompressed) + + def test_encoding_error_handler(self): + with io.BytesIO(compress(b"foo\xffbar")) as bio: + with open(bio, "rt", encoding="ascii", errors="ignore") as f: + self.assertEqual(f.read(), "foobar") + + def test_newline(self): + # Test with explicit newline (universal newline mode disabled). + text = THIS_FILE_STR.replace(os.linesep, "\n") + with io.BytesIO() as bio: + with open(bio, "wt", encoding="utf-8", newline="\n") as f: + f.write(text) + bio.seek(0) + with open(bio, "rt", encoding="utf-8", newline="\r") as f: + self.assertEqual(f.readlines(), [text]) + + def test_x_mode(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + TESTFN = pathlib.Path(tmp_f.name) + + for mode in ("x", "xb", "xt"): + os.remove(TESTFN) + + if mode == "xt": + encoding = "utf-8" + else: + encoding = None + with open(TESTFN, mode, encoding=encoding): + pass + with self.assertRaises(FileExistsError): + with open(TESTFN, mode): + pass + + os.remove(TESTFN) + + def test_open_dict(self): + # default + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # .as_(un)digested_dict + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # invalid dictionary + bi = io.BytesIO() + with self.assertRaisesRegex(TypeError, 'zstd_dict'): + open(bi, 'w', zstd_dict={1:2, 2:3}) + + with self.assertRaisesRegex(TypeError, 'zstd_dict'): + open(bi, 'w', zstd_dict=b'1234567890') + + def test_open_prefix(self): + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT.as_prefix) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_buffer_protocol(self): + # don't use len() for buffer protocol objects + arr = array.array("i", range(1000)) + LENGTH = len(arr) * arr.itemsize + + with open(io.BytesIO(), "wb") as f: + self.assertEqual(f.write(arr), LENGTH) + self.assertEqual(f.tell(), LENGTH) + +class FreeThreadingMethodTests(unittest.TestCase): + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_locking(self): + input = b'a'* (16*_1K) + num_threads = 8 + + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + + comp = ZstdCompressor() + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] + for _ in range(num_threads): + res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) + if res: + parts.append(res) + rest1 = comp.flush() + expected = b''.join(parts) + rest1 + + comp = ZstdCompressor() + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] + def run_method(method, input_data, output_data): + res = method(input_data, ZstdCompressor.FLUSH_BLOCK) + if res: + output_data.append(res) + threads = [] + + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(comp.compress, input, output)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + rest2 = comp.flush() + self.assertEqual(rest1, rest2) + actual = b''.join(output) + rest2 + self.assertEqual(expected, actual) + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_locking(self): + input = compress(b'a'* (16*_1K)) + num_threads = 8 + # to ensure we decompress over multiple calls, set maxsize + window_size = _1K * 16//num_threads + + decomp = ZstdDecompressor() + parts = [] + for _ in range(num_threads): + res = decomp.decompress(input, window_size) + if res: + parts.append(res) + expected = b''.join(parts) + + comp = ZstdDecompressor() + output = [] + def run_method(method, input_data, output_data): + res = method(input_data, window_size) + if res: + output_data.append(res) + threads = [] + + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(comp.decompress, input, output)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + actual = b''.join(output) + self.assertEqual(expected, actual) + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_shared_dict(self): + num_threads = 8 + + def run_method(b): + level = threading.get_ident() % 4 + # sync threads to increase chance of contention on + # capsule storing dictionary levels + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_shared_dict(self): + num_threads = 8 + + def run_method(b): + # sync threads to increase chance of contention on + # decompression dictionary + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/tokenizedata/__init__.py b/Lib/test/tokenizedata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/tokenizedata/bad_coding.py b/Lib/test/tokenizedata/bad_coding.py new file mode 100644 index 00000000000..971b0a8f3d6 --- /dev/null +++ b/Lib/test/tokenizedata/bad_coding.py @@ -0,0 +1 @@ +# -*- coding: uft-8 -*- diff --git a/Lib/test/tokenizedata/bad_coding2.py b/Lib/test/tokenizedata/bad_coding2.py new file mode 100644 index 00000000000..bb2bb7e1e75 --- /dev/null +++ b/Lib/test/tokenizedata/bad_coding2.py @@ -0,0 +1,2 @@ +#coding: utf8 +print('我') diff --git a/Lib/test/tokenizedata/badsyntax_3131.py b/Lib/test/tokenizedata/badsyntax_3131.py new file mode 100644 index 00000000000..901d3744ca0 --- /dev/null +++ b/Lib/test/tokenizedata/badsyntax_3131.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +€ = 2 diff --git a/Lib/test/tokenizedata/badsyntax_pep3120.py b/Lib/test/tokenizedata/badsyntax_pep3120.py new file mode 100644 index 00000000000..d14b4c96ede --- /dev/null +++ b/Lib/test/tokenizedata/badsyntax_pep3120.py @@ -0,0 +1 @@ +print("bse") diff --git a/Lib/test/tokenizedata/coding20731.py b/Lib/test/tokenizedata/coding20731.py new file mode 100644 index 00000000000..b0e227ad110 --- /dev/null +++ b/Lib/test/tokenizedata/coding20731.py @@ -0,0 +1,4 @@ +#coding:latin1 + + + diff --git a/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt new file mode 100644 index 00000000000..1b5335b64ed --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt @@ -0,0 +1,13 @@ +# -*- coding: latin1 -*- +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! Also note that the coding cookie above conflicts with +# the presence of a utf-8 BOM signature -- this is intended. + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt b/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt new file mode 100644 index 00000000000..23fd2168ae5 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt @@ -0,0 +1,11 @@ +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt new file mode 100644 index 00000000000..04561e48472 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# IMPORTANT: unlike the other test_tokenize-*.txt files, this file +# does NOT have the utf-8 BOM signature '\xef\xbb\xbf' at the start +# of it. Make sure this is not added inadvertently by your editor +# if any changes are made to this file! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt new file mode 100644 index 00000000000..4b20ff6ad6d --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests.txt b/Lib/test/tokenizedata/tokenize_tests.txt new file mode 100644 index 00000000000..c4f5a58a946 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests.txt @@ -0,0 +1,189 @@ +# Tests for the 'tokenize' module. +# Large bits stolen from test_grammar.py. + +# Comments +"#" +#' +#" +#\ + # + # abc +'''# +#''' + +x = 1 # + +# Balancing continuation + +a = (3, 4, + 5, 6) +y = [3, 4, + 5] +z = {'a':5, + 'b':6} +x = (len(repr(y)) + 5*x - a[ + 3 ] + - x + len({ + } + ) + ) + +# Backslash means line continuation: +x = 1 \ ++ 1 + +# Backslash does not means continuation in comments :\ +x = 0 + +# Ordinary integers +0xff != 255 +0o377 != 255 +2147483647 != 0o17777777777 +-2147483647-1 != 0o20000000000 +0o37777777777 != -1 +0xffffffff != -1; 0o37777777777 != -1; -0o1234567 == 0O001234567; 0b10101 == 0B00010101 + +# Long integers +x = 0 +x = 0 +x = 0xffffffffffffffff +x = 0xffffffffffffffff +x = 0o77777777777777777 +x = 0B11101010111111111 +x = 123456789012345678901234567890 +x = 123456789012345678901234567890 + +# Floating-point numbers +x = 3.14 +x = 314. +x = 0.314 +# XXX x = 000.314 +x = .314 +x = 3e14 +x = 3E14 +x = 3e-14 +x = 3e+14 +x = 3.e14 +x = .3e14 +x = 3.1e4 + +# String literals +x = ''; y = ""; +x = '\''; y = "'"; +x = '"'; y = "\""; +x = "doesn't \"shrink\" does it" +y = 'doesn\'t "shrink" does it' +x = "does \"shrink\" doesn't it" +y = 'does "shrink" doesn\'t it' +x = """ +The "quick" +brown fox +jumps over +the 'lazy' dog. +""" +y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' +y = ''' +The "quick" +brown fox +jumps over +the 'lazy' dog. +'''; +y = "\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the 'lazy' dog.\n\ +"; +y = '\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the \'lazy\' dog.\n\ +'; +x = r'\\' + R'\\' +x = r'\'' + '' +y = r''' +foo bar \\ +baz''' + R''' +foo''' +y = r"""foo +bar \\ baz +""" + R'''spam +''' +x = b'abc' + B'ABC' +y = b"abc" + B"ABC" +x = br'abc' + Br'ABC' + bR'ABC' + BR'ABC' +y = br"abc" + Br"ABC" + bR"ABC" + BR"ABC" +x = rb'abc' + rB'ABC' + Rb'ABC' + RB'ABC' +y = rb"abc" + rB"ABC" + Rb"ABC" + RB"ABC" +x = br'\\' + BR'\\' +x = rb'\\' + RB'\\' +x = br'\'' + '' +x = rb'\'' + '' +y = br''' +foo bar \\ +baz''' + BR''' +foo''' +y = Br"""foo +bar \\ baz +""" + bR'''spam +''' +y = rB"""foo +bar \\ baz +""" + Rb'''spam +''' + +# Indentation +if 1: + x = 2 +if 1: + x = 2 +if 1: + while 0: + if 0: + x = 2 + x = 2 +if 0: + if 2: + while 0: + if 1: + x = 2 + +# Operators + +def d22(a, b, c=1, d=2): pass +def d01v(a=1, *restt, **restd): pass + +(x, y) != ({'a':1}, {'b':2}) + +# comparison +if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 != 1 in 1 not in 1 is 1 is not 1: pass + +# binary +x = 1 & 1 +x = 1 ^ 1 +x = 1 | 1 + +# shift +x = 1 << 1 >> 1 + +# additive +x = 1 - 1 + 1 - 1 + 1 + +# multiplicative +x = 1 / 1 * 1 % 1 + +# unary +x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 +x = -1*1/1 + 1*1 - ---1*1 + +# selector +import sys, time +x = sys.modules['time'].time() + +@staticmethod +def foo(): pass + +@staticmethod +def foo(x:1)->1: pass + diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b5834..e1a1792cb4a 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 00000000000..7347a7a4245 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/test/typinganndata/partialexecution/__init__.py b/Lib/test/typinganndata/partialexecution/__init__.py new file mode 100644 index 00000000000..c39074ea84b --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/__init__.py @@ -0,0 +1 @@ +from . import a diff --git a/Lib/test/typinganndata/partialexecution/a.py b/Lib/test/typinganndata/partialexecution/a.py new file mode 100644 index 00000000000..ed0b8dcbd55 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/a.py @@ -0,0 +1,5 @@ +v1: int + +from . import b + +v2: int diff --git a/Lib/test/typinganndata/partialexecution/b.py b/Lib/test/typinganndata/partialexecution/b.py new file mode 100644 index 00000000000..36b8d2e52a3 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/b.py @@ -0,0 +1,3 @@ +from . import a + +annos = a.__annotations__ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000..c6beae8e255 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000..74ab03b4648 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000..9769a404f67 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 7ca393d1c37..41366fbf443 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -2,7 +2,7 @@ """ # Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. +# Copyright (C) 2002 Python Software Foundation. # Written by Greg Ward <gward@python.net> import re @@ -86,7 +86,7 @@ class TextWrapper: -(?: (?<=%(lt)s{2}-) | (?<=%(lt)s-%(lt)s-)) (?= %(lt)s -? %(lt)s) | # end of word - (?=%(ws)s|\Z) + (?=%(ws)s|\z) | # em-dash (?<=%(wp)s) (?=-{2,}\w) ) @@ -107,7 +107,7 @@ class TextWrapper: sentence_end_re = re.compile(r'[a-z]' # lowercase letter r'[\.\!\?]' # sentence-ending punct. r'[\"\']?' # optional end-of-quote - r'\Z') # end of chunk + r'\z') # end of chunk def __init__(self, width=70, @@ -211,7 +211,7 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: @@ -413,9 +413,6 @@ def shorten(text, width, **kwargs): # -- Loosely related functionality ------------------------------------- -_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) -_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) - def dedent(text): """Remove any common leading whitespace from every line in `text`. @@ -429,42 +426,22 @@ def dedent(text): Entirely blank lines are normalized to a newline character. """ - # Look for the longest leading string of spaces and tabs common to - # all lines. - margin = None - text = _whitespace_only_re.sub('', text) - indents = _leading_whitespace_re.findall(text) - for indent in indents: - if margin is None: - margin = indent - - # Current line more deeply indented than previous winner: - # no change (previous winner is still on top). - elif indent.startswith(margin): - pass - - # Current line consistent with and no deeper than previous winner: - # it's the new winner. - elif margin.startswith(indent): - margin = indent - - # Find the largest common whitespace between current line and previous - # winner. - else: - for i, (x, y) in enumerate(zip(margin, indent)): - if x != y: - margin = margin[:i] - break + try: + lines = text.split('\n') + except (AttributeError, TypeError): + msg = f'expected str object, not {type(text).__qualname__!r}' + raise TypeError(msg) from None - # sanity check (testing/debugging only) - if 0 and margin: - for line in text.split("\n"): - assert not line or line.startswith(margin), \ - "line = %r, margin = %r" % (line, margin) + # Get length of leading whitespace, inspired by ``os.path.commonprefix()``. + non_blank_lines = [l for l in lines if l and not l.isspace()] + l1 = min(non_blank_lines, default='') + l2 = max(non_blank_lines, default='') + margin = 0 + for margin, c in enumerate(l1): + if c != l2[margin] or c not in ' \t': + break - if margin: - text = re.sub(r'(?m)^' + margin, '', text) - return text + return '\n'.join([l[margin:] if not l.isspace() else '' for l in lines]) def indent(text, prefix, predicate=None): @@ -475,19 +452,20 @@ def indent(text, prefix, predicate=None): it will default to adding 'prefix' to all non-empty lines that do not consist solely of whitespace characters. """ - if predicate is None: - # str.splitlines(True) doesn't produce empty string. - # ''.splitlines(True) => [] - # 'foo\n'.splitlines(True) => ['foo\n'] - # So we can use just `not s.isspace()` here. - predicate = lambda s: not s.isspace() - prefixed_lines = [] - for line in text.splitlines(True): - if predicate(line): - prefixed_lines.append(prefix) - prefixed_lines.append(line) - + if predicate is None: + # str.splitlines(keepends=True) doesn't produce the empty string, + # so we need to use `str.isspace()` rather than a truth test. + # Inlining the predicate leads to a ~30% performance improvement. + for line in text.splitlines(True): + if not line.isspace(): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + else: + for line in text.splitlines(True): + if predicate(line): + prefixed_lines.append(prefix) + prefixed_lines.append(line) return ''.join(prefixed_lines) diff --git a/Lib/threading.py b/Lib/threading.py index 668126523d5..c03b0b5370c 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,11 +3,11 @@ import os as _os import sys as _sys import _thread -import functools +import _contextvars from time import monotonic as _time from _weakrefset import WeakSet -from itertools import islice as _islice, count as _count +from itertools import count as _count try: from _collections import deque as _deque except ImportError: @@ -28,19 +28,30 @@ 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size', - 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile'] + 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile', + 'setprofile_all_threads','settrace_all_threads'] # Rename some stuff so "from threading import *" is safe -_start_new_thread = _thread.start_new_thread +_start_joinable_thread = _thread.start_joinable_thread +_daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock -_set_sentinel = _thread._set_sentinel +_LockType = _thread.LockType +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident +_is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id _HAVE_THREAD_NATIVE_ID = True __all__.append('get_native_id') except AttributeError: _HAVE_THREAD_NATIVE_ID = False +try: + _set_name = _thread.set_name +except AttributeError: + _set_name = None ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -49,6 +60,13 @@ TIMEOUT_MAX = _thread.TIMEOUT_MAX del _thread +# get thread-local implementation, either from the thread +# module, or from the python fallback + +try: + from _thread import _local as local +except ImportError: + from _threading_local import local # Support for profile and trace hooks @@ -60,11 +78,20 @@ def setprofile(func): The func will be passed to sys.setprofile() for each thread, before its run() method is called. - """ global _profile_hook _profile_hook = func +def setprofile_all_threads(func): + """Set a profile function for all threads started from the threading module + and all Python threads that are currently executing. + + The func will be passed to sys.setprofile() for each thread, before its + run() method is called. + """ + setprofile(func) + _sys._setprofileallthreads(func) + def getprofile(): """Get the profiler function as set by threading.setprofile().""" return _profile_hook @@ -74,18 +101,27 @@ def settrace(func): The func will be passed to sys.settrace() for each thread, before its run() method is called. - """ global _trace_hook _trace_hook = func +def settrace_all_threads(func): + """Set a trace function for all threads started from the threading module + and all Python threads that are currently executing. + + The func will be passed to sys.settrace() for each thread, before its run() + method is called. + """ + settrace(func) + _sys._settraceallthreads(func) + def gettrace(): """Get the trace function as set by threading.settrace().""" return _trace_hook # Synchronization classes -Lock = _allocate_lock +Lock = _LockType def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. @@ -96,6 +132,13 @@ def RLock(*args, **kwargs): acquired it. """ + if args or kwargs: + import warnings + warnings.warn( + 'Passing arguments to RLock is deprecated and will be removed in 3.15', + DeprecationWarning, + stacklevel=2, + ) if _CRLock is None: return _PyRLock(*args, **kwargs) return _CRLock(*args, **kwargs) @@ -122,7 +165,7 @@ def __repr__(self): except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( - "locked" if self._block.locked() else "unlocked", + "locked" if self.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, @@ -199,6 +242,10 @@ def release(self): def __exit__(self, t, v, tb): self.release() + def locked(self): + """Return whether this object is locked.""" + return self._block.locked() + # Internal methods used by condition variables def _acquire_restore(self, state): @@ -218,6 +265,13 @@ def _release_save(self): def _is_owned(self): return self._owner == get_ident() + # Internal method used for reentrancy checks + + def _recursion_count(self): + if self._owner != get_ident(): + return 0 + return self._count + _PyRLock = _RLock @@ -237,24 +291,19 @@ def __init__(self, lock=None): if lock is None: lock = RLock() self._lock = lock - # Export the lock's acquire() and release() methods + # Export the lock's acquire(), release(), and locked() methods self.acquire = lock.acquire self.release = lock.release + self.locked = lock.locked # If the lock defines _release_save() and/or _acquire_restore(), # these override the default implementations (which just call # release() and acquire() on the lock). Ditto for _is_owned(). - try: + if hasattr(lock, '_release_save'): self._release_save = lock._release_save - except AttributeError: - pass - try: + if hasattr(lock, '_acquire_restore'): self._acquire_restore = lock._acquire_restore - except AttributeError: - pass - try: + if hasattr(lock, '_is_owned'): self._is_owned = lock._is_owned - except AttributeError: - pass self._waiters = _deque() def _at_fork_reinit(self): @@ -297,7 +346,7 @@ def wait(self, timeout=None): awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its @@ -425,6 +474,11 @@ def __init__(self, value=1): self._cond = Condition(Lock()) self._value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}>") + def acquire(self, blocking=True, timeout=None): """Acquire a semaphore, decrementing the internal counter by one. @@ -483,8 +537,7 @@ def release(self, n=1): raise ValueError('n must be one or more') with self._cond: self._value += n - for i in range(n): - self._cond.notify() + self._cond.notify(n) def __exit__(self, t, v, tb): self.release() @@ -508,9 +561,14 @@ class BoundedSemaphore(Semaphore): """ def __init__(self, value=1): - Semaphore.__init__(self, value) + super().__init__(value) self._initial_value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}/{self._initial_value}>") + def release(self, n=1): """Release a semaphore, incrementing the internal counter by one or more. @@ -527,8 +585,7 @@ def release(self, n=1): if self._value + n > self._initial_value: raise ValueError("Semaphore released too many times") self._value += n - for i in range(n): - self._cond.notify() + self._cond.notify(n) class Event: @@ -546,8 +603,13 @@ def __init__(self): self._cond = Condition(Lock()) self._flag = False + def __repr__(self): + cls = self.__class__ + status = 'set' if self._flag else 'unset' + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: {status}>" + def _at_fork_reinit(self): - # Private method called by Thread._reset_internal_locks() + # Private method called by Thread._after_fork() self._cond._at_fork_reinit() def is_set(self): @@ -557,7 +619,7 @@ def is_set(self): def isSet(self): """Return true if and only if the internal flag is true. - This method is deprecated, use notify_all() instead. + This method is deprecated, use is_set() instead. """ import warnings @@ -594,11 +656,12 @@ def wait(self, timeout=None): the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return - True except if a timeout is given and the operation times out. + ``True`` except if a timeout is given and the operation times out, when + it will return ``False``. """ with self._cond: @@ -637,6 +700,8 @@ def __init__(self, parties, action=None, timeout=None): default for all subsequent 'wait()' calls. """ + if parties < 1: + raise ValueError("parties must be >= 1") self._cond = Condition(Lock()) self._action = action self._timeout = timeout @@ -644,6 +709,13 @@ def __init__(self, parties, action=None, timeout=None): self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken self._count = 0 + def __repr__(self): + cls = self.__class__ + if self.broken: + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: broken>" + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" waiters={self.n_waiting}/{self.parties}>") + def wait(self, timeout=None): """Wait for the barrier. @@ -791,25 +863,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -825,7 +878,7 @@ class Thread: _initialized = False def __init__(self, group=None, target=None, name=None, - args=(), kwargs=None, *, daemon=None): + args=(), kwargs=None, *, daemon=None, context=None): """This constructor should always be called with keyword arguments. Arguments are: *group* should be None; reserved for future extension when a ThreadGroup @@ -837,11 +890,19 @@ class is implemented. *name* is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number. - *args* is the argument tuple for the target invocation. Defaults to (). + *args* is a list or tuple of arguments for the target invocation. Defaults to (). *kwargs* is a dictionary of keyword arguments for the target invocation. Defaults to {}. + *context* is the contextvars.Context value to use for the thread. + The default value is None, which means to check + sys.flags.thread_inherit_context. If that flag is true, use a copy + of the context of the caller. If false, use an empty context. To + explicitly start with an empty context, pass a new instance of + contextvars.Context(). To explicitly start with a copy of the current + context, pass the value from contextvars.copy_context(). + If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread. @@ -866,15 +927,17 @@ class is implemented. self._args = args self._kwargs = kwargs if daemon is not None: + if daemon and not _daemon_threads_allowed(): + raise RuntimeError('daemon threads are disabled in this (sub)interpreter') self._daemonic = daemon else: self._daemonic = current_thread().daemon + self._context = context self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None + self._os_thread_handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -882,30 +945,26 @@ class is implemented. # For debugging and _after_fork() _dangling.add(self) - def _reset_internal_locks(self, is_alive): - # private! Called by _after_fork() to reset our internal locks as - # they may be in an invalid state leading to a deadlock or crash. + def _after_fork(self, new_ident=None): + # Private! Called by threading._after_fork(). self._started._at_fork_reinit() - if is_alive: - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() + if new_ident is not None: + # This thread is alive. + self._ident = new_ident + assert self._os_thread_handle.ident == new_ident + if _HAVE_THREAD_NATIVE_ID: + self._set_native_id() else: - # The thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._os_thread_handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -931,13 +990,25 @@ def start(self): with _active_limbo_lock: _limbo[self] = self + + if self._context is None: + # No context provided + if _sys.flags.thread_inherit_context: + # start with a copy of the context of the caller + self._context = _contextvars.copy_context() + else: + # start with an empty context + self._context = _contextvars.Context() + try: - _start_new_thread(self._bootstrap, ()) + # Start joinable thread + _start_joinable_thread(self._bootstrap, handle=self._os_thread_handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] raise - self._started.wait() + self._started.wait() # Will set ident and native_id def run(self): """Method representing the thread's activity. @@ -983,25 +1054,20 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) + def _set_os_name(self): + if _set_name is None or not self._name: + return + try: + _set_name(self._name) + except OSError: + pass def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() + self._set_os_name() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -1013,44 +1079,11 @@ def _bootstrap_inner(self): _sys.setprofile(_profile_hook) try: - self.run() + self._context.run(self.run) except: self._invoke_excepthook(self) finally: - with _active_limbo_lock: - try: - # We don't call self._delete() because it also - # grabs _active_limbo_lock. - del _active[get_ident()] - except: - pass - - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() + self._delete() def _delete(self): "Remove current thread from the dict of currently running threads." @@ -1069,7 +1102,7 @@ def join(self, timeout=None): or until the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. @@ -1092,39 +1125,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._os_thread_handle.join(timeout) @property def name(self): @@ -1141,6 +1147,8 @@ def name(self): def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + if get_ident() == self._ident: + self._set_os_name() @property def ident(self): @@ -1175,10 +1183,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - return not self._is_stopped + return self._started.is_set() and not self._os_thread_handle.is_done() @property def daemon(self): @@ -1199,6 +1204,8 @@ def daemon(self): def daemon(self, daemonic): if not self._initialized: raise RuntimeError("Thread.__init__() not called") + if daemonic and not _daemon_threads_allowed(): + raise RuntimeError('daemon threads are disabled in this interpreter') if self._started.is_set(): raise RuntimeError("cannot set daemon status of active thread") self._daemonic = daemonic @@ -1385,19 +1392,45 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._os_thread_handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self +# Helper thread-local instance to detect when a _DummyThread +# is collected. Not a part of the public API. +_thread_local_info = local() + + +class _DeleteDummyThreadOnDel: + ''' + Helper class to remove a dummy thread from threading._active on __del__. + ''' + + def __init__(self, dummy_thread): + self._dummy_thread = dummy_thread + self._tident = dummy_thread.ident + # Put the thread on a thread local variable so that when + # the related thread finishes this instance is collected. + # + # Note: no other references to this instance may be created. + # If any client code creates a reference to this instance, + # the related _DummyThread will be kept forever! + _thread_local_info._track_dummy_thread_ref = self + + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): + with _active_limbo_lock: + if _active.get(self._tident) is self._dummy_thread: + _active.pop(self._tident, None) + + # Dummy thread class to represent threads not started here. -# These aren't garbage collected when they die, nor can they be waited for. -# If they invoke anything in threading.py that calls current_thread(), they -# leave an entry in the _active dict forever after. +# These should be added to `_active` and removed automatically +# when they die, although they can't be waited for. # Their purpose is to return *something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). @@ -1405,24 +1438,31 @@ def __init__(self): class _DummyThread(Thread): def __init__(self): - Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) - + Thread.__init__(self, name=_newname("Dummy-%d"), + daemon=_daemon_threads_allowed()) self._started.set() self._set_ident() + self._os_thread_handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self - - def _stop(self): - pass + _DeleteDummyThreadOnDel(self) def is_alive(self): - assert not self._is_stopped and self._started.is_set() - return True + if not self._os_thread_handle.is_done() and self._started.is_set(): + return True + raise RuntimeError("thread is not alive") def join(self, timeout=None): - assert False, "cannot join a dummy thread" + raise RuntimeError("cannot join a dummy thread") + + def _after_fork(self, new_ident=None): + if new_ident is not None: + self.__class__ = _MainThread + self._name = 'MainThread' + self._daemonic = False + Thread._after_fork(self, new_ident=new_ident) # Global API functions @@ -1457,6 +1497,8 @@ def active_count(): enumerate(). """ + # NOTE: if the logic in here ever changes, update Modules/posixmodule.c + # warn_about_fork_with_threads() to match. with _active_limbo_lock: return len(_active) + len(_limbo) @@ -1503,8 +1545,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size @@ -1519,12 +1560,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped: + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._os_thread_handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1536,39 +1576,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - assert tlock.locked() - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._os_thread_handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1577,16 +1589,9 @@ def main_thread(): In normal conditions, the main thread is the thread from which the Python interpreter was started. """ + # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread -# get thread-local implementation, either from the thread -# module, or from the python fallback - -try: - from _thread import _local as local -except ImportError: - from _threading_local import local - def _after_fork(): """ @@ -1595,7 +1600,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1611,10 +1615,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1624,16 +1624,13 @@ def _after_fork(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: - # There is only one active thread. We reset the ident to - # its new value since it can have changed. - thread._reset_internal_locks(True) + # This is the one and only active thread. ident = get_ident() - thread._ident = ident + thread._after_fork(new_ident=ident) new_active[ident] = thread else: # All the others are already stopped. - thread._reset_internal_locks(False) - thread._stop() + thread._after_fork() _limbo.clear() _active.clear() diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index 258dedccd08..e767f018782 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution @@ -46,7 +44,6 @@ timeit(string, string) -> float repeat(string, string) -> list default_timer() -> float - """ import gc @@ -174,16 +171,14 @@ def timeit(self, number=default_number): the timer function to be used are passed to the constructor. """ it = itertools.repeat(None, number) - # XXX RUSTPYTHON TODO: gc module implementation - # gcold = gc.isenabled() - # gc.disable() - # try: - # timing = self.inner(it, self.timer) - # finally: - # if gcold: - # gc.enable() - # return timing - return self.inner(it, self.timer) + gcold = gc.isenabled() + gc.disable() + try: + timing = self.inner(it, self.timer) + finally: + if gcold: + gc.enable() + return timing def repeat(self, repeat=default_repeat, number=default_number): """Call timeit() a few times. @@ -306,7 +301,7 @@ def main(args=None, *, _wrap_timer=None): precision += 1 verbose += 1 if o in ("-h", "--help"): - print(__doc__, end=' ') + print(__doc__, end="") return 0 setup = "\n".join(setup) or "pass" diff --git a/Lib/token.py b/Lib/token.py index 493bf042650..f61723cc09d 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" -# Auto-generated by Tools/scripts/generate_token.py +# Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 @@ -57,17 +58,23 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 -OP = 54 -AWAIT = 55 -ASYNC = 56 -TYPE_IGNORE = 57 -TYPE_COMMENT = 58 +EXCLAMATION = 54 +OP = 55 +TYPE_IGNORE = 56 +TYPE_COMMENT = 57 +SOFT_KEYWORD = 58 +FSTRING_START = 59 +FSTRING_MIDDLE = 60 +FSTRING_END = 61 +TSTRING_START = 62 +TSTRING_MIDDLE = 63 +TSTRING_END = 64 +COMMENT = 65 +NL = 66 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 59 -COMMENT = 60 -NL = 61 -ENCODING = 62 -N_TOKENS = 63 +ERRORTOKEN = 67 +ENCODING = 68 +N_TOKENS = 69 # Special definitions for cooperation with parser NT_OFFSET = 256 @@ -77,6 +84,7 @@ __all__.extend(tok_name.values()) EXACT_TOKEN_TYPES = { + '!': EXCLAMATION, '!=': NOTEQUAL, '%': PERCENT, '%=': PERCENTEQUAL, @@ -126,11 +134,11 @@ '~': TILDE, } -def ISTERMINAL(x): +def ISTERMINAL(x: int) -> bool: return x < NT_OFFSET -def ISNONTERMINAL(x): +def ISNONTERMINAL(x: int) -> bool: return x >= NT_OFFSET -def ISEOF(x): +def ISEOF(x: int) -> bool: return x == ENDMARKER diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 9c80a6a547d..3ee47aa9e0a 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,10 +4,7 @@ from __future__ import annotations -from collections.abc import Iterable -import string from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple from ._re import ( RE_DATETIME, @@ -17,7 +14,13 @@ match_to_localtime, match_to_number, ) -from ._types import Key, ParseFloat, Pos + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import IO, Any + + from ._types import Key, ParseFloat, Pos ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) @@ -33,9 +36,11 @@ TOML_WS = frozenset(" \t") TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +BARE_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_" +) KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) +HEXDIGIT_CHARS = frozenset("abcdef" "ABCDEF" "0123456789") BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( { @@ -50,11 +55,73 @@ ) +class DEPRECATED_DEFAULT: + """Sentinel to be used as default arg during deprecation + period of TOMLDecodeError's free-form arguments.""" + + class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" + """An error raised if a document is not valid TOML. + + Adds the following attributes to ValueError: + msg: The unformatted error message + doc: The TOML document being parsed + pos: The index of doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__( + self, + msg: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + doc: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + pos: Pos = DEPRECATED_DEFAULT, # type: ignore[assignment] + *args: Any, + ): + if ( + args + or not isinstance(msg, str) + or not isinstance(doc, str) + or not isinstance(pos, int) + ): + import warnings + + warnings.warn( + "Free-form arguments for TOMLDecodeError are deprecated. " + "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.", + DeprecationWarning, + stacklevel=2, + ) + if pos is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = pos, *args + if doc is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = doc, *args + if msg is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = msg, *args + ValueError.__init__(self, *args) + return + + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + + if pos >= len(doc): + coord_repr = "end of document" + else: + coord_repr = f"line {lineno}, column {colno}" + errmsg = f"{msg} (at {coord_repr})" + ValueError.__init__(self, errmsg) + + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno -def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: +def load(fp: IO[bytes], /, *, parse_float: ParseFloat = float) -> dict[str, Any]: """Parse TOML from a binary file object.""" b = fp.read() try: @@ -71,9 +138,14 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 - out = Output(NestedDict(), Flags()) + out = Output() header: Key = () parse_float = make_safe_parse_float(parse_float) @@ -113,7 +185,7 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n pos, header = create_dict_rule(src, pos, out) pos = skip_chars(src, pos, TOML_WS) elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") + raise TOMLDecodeError("Invalid statement", src, pos) # 3. Skip comment pos = skip_comment(src, pos) @@ -124,8 +196,8 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n except IndexError: break if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" + raise TOMLDecodeError( + "Expected newline or end of document after a statement", src, pos ) pos += 1 @@ -224,9 +296,10 @@ def append_nest_to_list(self, key: Key) -> None: cont[last_key] = [{}] -class Output(NamedTuple): - data: NestedDict - flags: Flags +class Output: + def __init__(self) -> None: + self.data = NestedDict() + self.flags = Flags() def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: @@ -251,12 +324,12 @@ def skip_until( except ValueError: new_pos = len(src) if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None if not error_on.isdisjoint(src[pos:new_pos]): while src[pos] not in error_on: pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos) return new_pos @@ -287,15 +360,17 @@ def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") + raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos) out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) try: out.data.get_or_create_nest(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + raise TOMLDecodeError( + "Expected ']' at the end of a table declaration", src, pos + ) return pos + 1, key @@ -305,7 +380,7 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) # Free the namespace now that it points to another empty list item... out.flags.unset_all(key) # ...but this key precisely is still prohibited from table declaration @@ -313,10 +388,12 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: try: out.data.append_nest_to_list(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + raise TOMLDecodeError( + "Expected ']]' at the end of an array declaration", src, pos + ) return pos + 2, key @@ -331,22 +408,22 @@ def key_value_rule( for cont_key in relative_path_cont_keys: # Check that dotted key syntax does not redefine an existing table if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos) # Containers in the relative path can't be opened with the table syntax or # dotted key/value syntax in following table sections. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + raise TOMLDecodeError( + f"Cannot mutate immutable namespace {abs_key_parent}", src, pos ) try: nest = out.data.get_or_create_nest(abs_key_parent) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") + raise TOMLDecodeError("Cannot overwrite a value", src, pos) # Mark inline table and array namespaces recursively immutable if isinstance(value, (dict, list)): out.flags.set(header + key, Flags.FROZEN, recursive=True) @@ -363,7 +440,7 @@ def parse_key_value_pair( except IndexError: char = None if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS) pos, value = parse_value(src, pos, parse_float) @@ -401,7 +478,7 @@ def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: return parse_literal_str(src, pos) if char == '"': return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") + raise TOMLDecodeError("Invalid initial character for a key part", src, pos) def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: @@ -425,7 +502,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list[ if c == "]": return pos + 1, array if c != ",": - raise suffixed_err(src, pos, "Unclosed array") + raise TOMLDecodeError("Unclosed array", src, pos) pos += 1 pos = skip_comments_and_array_ws(src, pos) @@ -445,20 +522,20 @@ def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos pos, key, value = parse_key_value_pair(src, pos, parse_float) key_parent, key_stem = key[:-1], key[-1] if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) try: nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos) nest[key_stem] = value pos = skip_chars(src, pos, TOML_WS) c = src[pos : pos + 1] if c == "}": return pos + 1, nested_dict.dict if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") + raise TOMLDecodeError("Unclosed inline table", src, pos) if isinstance(value, (dict, list)): flags.set(key, Flags.FROZEN, recursive=True) pos += 1 @@ -480,7 +557,7 @@ def parse_basic_str_escape( except IndexError: return pos, "" if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) return pos, "" @@ -491,7 +568,7 @@ def parse_basic_str_escape( try: return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: @@ -501,11 +578,13 @@ def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: hex_str = src[pos : pos + hex_len] if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") + raise TOMLDecodeError("Invalid hex value", src, pos) pos += hex_len hex_int = int(hex_str, 16) if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + raise TOMLDecodeError( + "Escaped character is not a Unicode scalar value", src, pos + ) return pos, chr(hex_int) @@ -562,7 +641,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: try: char = src[pos] except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None + raise TOMLDecodeError("Unterminated string", src, pos) from None if char == '"': if not multiline: return pos + 1, result + src[start_pos:pos] @@ -577,7 +656,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: start_pos = pos continue if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") + raise TOMLDecodeError(f"Illegal character {char!r}", src, pos) pos += 1 @@ -625,7 +704,7 @@ def parse_value( # noqa: C901 try: datetime_obj = match_to_datetime(datetime_match) except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e + raise TOMLDecodeError("Invalid date or datetime", src, pos) from e return datetime_match.end(), datetime_obj localtime_match = RE_LOCALTIME.match(src, pos) if localtime_match: @@ -646,24 +725,7 @@ def parse_value( # noqa: C901 if first_four in {"-inf", "+inf", "-nan", "+nan"}: return pos + 4, parse_float(first_four) - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + raise TOMLDecodeError("Invalid value", src, pos) def is_unicode_scalar_value(codepoint: int) -> bool: diff --git a/Lib/tomllib/_re.py b/Lib/tomllib/_re.py index a97cab2f9db..eb8beb19747 100644 --- a/Lib/tomllib/_re.py +++ b/Lib/tomllib/_re.py @@ -7,9 +7,12 @@ from datetime import date, datetime, time, timedelta, timezone, tzinfo from functools import lru_cache import re -from typing import Any -from ._types import ParseFloat +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any + + from ._types import ParseFloat # E.g. # - 00:32:00.999999 @@ -84,6 +87,9 @@ def match_to_datetime(match: re.Match[str]) -> datetime | date: return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) +# No need to limit cache size. This is only ever called on input +# that matched RE_DATETIME, so there is an implicit bound of +# 24 (hours) * 60 (minutes) * 2 (offset direction) = 2880. @lru_cache(maxsize=None) def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: sign = 1 if sign_str == "+" else -1 diff --git a/Lib/traceback.py b/Lib/traceback.py index d6a010f4157..5a34a2b87b6 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,21 +1,31 @@ """Extract, format and print information about Python stack traces.""" -import collections +import collections.abc import itertools import linecache import sys +import textwrap +import warnings +import codeop +import keyword +import tokenize +import io +import _colorize + +from contextlib import suppress __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', - 'walk_stack', 'walk_tb'] + 'walk_stack', 'walk_tb', 'print_list'] # # Formatting and printing lists of traceback lines. # + def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" @@ -69,7 +79,8 @@ def extract_tb(tb, limit=None): trace. The line is a string with leading and trailing whitespace stripped; if the source is not available it is None. """ - return StackSummary.extract(walk_tb(tb), limit=limit) + return StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(tb), limit=limit) # # Exception formatting and output. @@ -95,14 +106,18 @@ def _parse_value_tb(exc, value, tb): raise ValueError("Both or neither of value and tb must be given") if value is tb is _sentinel: if exc is not None: - return exc, exc.__traceback__ + if isinstance(exc, BaseException): + return exc, exc.__traceback__ + + raise TypeError(f'Exception expected for value, ' + f'{type(exc).__name__} found') else: return None, None return value, tb def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - file=None, chain=True): + file=None, chain=True, **kwargs): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -113,16 +128,23 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ occurred with a caret on the next line indicating the approximate position of the error. """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) - if file is None: - file = sys.stderr te = TracebackException(type(value), value, tb, limit=limit, compact=True) - for line in te.format(chain=chain): - print(line, file=file, end="") + te.print(file=file, chain=chain, colorize=colorize) + + +BUILTIN_EXCEPTION_LIMIT = object() + + +def _print_exception_bltin(exc, /): + file = sys.stderr if sys.stderr is not None else sys.__stderr__ + colorize = _colorize.can_colorize(file=file) + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -131,64 +153,77 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only()) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. -def _format_final_exc_line(etype, value): - valuestr = _some_str(value) +def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize=False): + valuestr = _safe_string(value, 'exception') + end_char = "\n" if insert_final_newline else "" + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback if value is None or not valuestr: - line = "%s\n" % etype + line = f"{theme.type}{etype}{theme.reset}{end_char}" else: - line = "%s: %s\n" % (etype, valuestr) + line = f"{theme.type}{etype}{theme.reset}: {theme.message}{valuestr}{theme.reset}{end_char}" return line -def _some_str(value): + +def _safe_string(value, what, func=str): try: - return str(value) + return func(value) except: - return '<unprintable %s object>' % type(value).__name__ + return f'<{what} {func.__name__}() failed>' # -- def print_exc(limit=None, file=None, chain=True): - """Shorthand for 'print_exception(*sys.exc_info(), limit, file)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + """Shorthand for 'print_exception(sys.exception(), limit=limit, file=file, chain=chain)'.""" + print_exception(sys.exception(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception(sys.exception(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): - """This is a shorthand for 'print_exception(sys.last_type, - sys.last_value, sys.last_traceback, limit, file)'.""" - if not hasattr(sys, "last_type"): + """This is a shorthand for 'print_exception(sys.last_exc, limit=limit, file=file, chain=chain)'.""" + if not hasattr(sys, "last_exc") and not hasattr(sys, "last_type"): raise ValueError("no last exception") - print_exception(sys.last_type, sys.last_value, sys.last_traceback, - limit, file, chain) + + if hasattr(sys, "last_exc"): + print_exception(sys.last_exc, limit=limit, file=file, chain=chain) + else: + print_exception(sys.last_type, sys.last_value, sys.last_traceback, + limit=limit, file=file, chain=chain) + # # Printing and Extracting Stacks. @@ -241,7 +276,7 @@ def clear_frames(tb): class FrameSummary: - """A single frame from a traceback. + """Information about a single frame from a traceback. - :attr:`filename` The filename for the frame. - :attr:`lineno` The line within filename for the frame that was @@ -254,10 +289,12 @@ class FrameSummary: mapping the name to the repr() of the variable. """ - __slots__ = ('filename', 'lineno', 'name', '_line', 'locals') + __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', + 'name', '_lines', '_lines_dedented', 'locals', '_code') def __init__(self, filename, lineno, name, *, lookup_line=True, - locals=None, line=None): + locals=None, line=None, + end_lineno=None, colno=None, end_colno=None, **kwargs): """Construct a FrameSummary. :param lookup_line: If True, `linecache` is consulted for the source @@ -269,11 +306,17 @@ def __init__(self, filename, lineno, name, *, lookup_line=True, """ self.filename = filename self.lineno = lineno + self.end_lineno = lineno if end_lineno is None else end_lineno + self.colno = colno + self.end_colno = end_colno self.name = name - self._line = line + self._code = kwargs.get("_code") + self._lines = line + self._lines_dedented = None if lookup_line: self.line - self.locals = {k: repr(v) for k, v in locals.items()} if locals else None + self.locals = {k: _safe_string(v, 'local', func=repr) + for k, v in locals.items()} if locals else None def __eq__(self, other): if isinstance(other, FrameSummary): @@ -298,13 +341,43 @@ def __repr__(self): def __len__(self): return 4 + def _set_lines(self): + if ( + self._lines is None + and self.lineno is not None + and self.end_lineno is not None + ): + lines = [] + for lineno in range(self.lineno, self.end_lineno + 1): + # treat errors (empty string) and empty lines (newline) as the same + line = linecache.getline(self.filename, lineno).rstrip() + if not line and self._code is not None and self.filename.startswith("<"): + line = linecache._getline_from_code(self._code, lineno).rstrip() + lines.append(line) + self._lines = "\n".join(lines) + "\n" + + @property + def _original_lines(self): + # Returns the line as-is from the source, without modifying whitespace. + self._set_lines() + return self._lines + + @property + def _dedented_lines(self): + # Returns _original_lines, but dedented + self._set_lines() + if self._lines_dedented is None and self._lines is not None: + self._lines_dedented = textwrap.dedent(self._lines) + return self._lines_dedented + @property def line(self): - if self._line is None: - if self.lineno is None: - return None - self._line = linecache.getline(self.filename, self.lineno) - return self._line.strip() + self._set_lines() + if self._lines is None: + return None + # return only the first line, stripped + return self._lines.partition("\n")[0].strip() + def walk_stack(f): """Walk a stack yielding the frame and line number for each frame. @@ -313,10 +386,14 @@ def walk_stack(f): current stack is used. Usually used with StackSummary.extract. """ if f is None: - f = sys._getframe().f_back.f_back - while f is not None: - yield f, f.f_lineno - f = f.f_back + f = sys._getframe().f_back + + def walk_stack_generator(frame): + while frame is not None: + yield frame, frame.f_lineno + frame = frame.f_back + + return walk_stack_generator(f) def walk_tb(tb): @@ -330,18 +407,40 @@ def walk_tb(tb): tb = tb.tb_next +def _walk_tb_with_full_positions(tb): + # Internal version of walk_tb that yields full code positions including + # end line and column information. + while tb is not None: + positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti) + # Yield tb_lineno when co_positions does not have a line number to + # maintain behavior with walk_tb. + if positions[0] is None: + yield tb.tb_frame, (tb.tb_lineno, ) + positions[1:] + else: + yield tb.tb_frame, positions + tb = tb.tb_next + + +def _get_code_position(code, instruction_index): + if instruction_index < 0: + return (None, None, None, None) + positions_gen = code.co_positions() + return next(itertools.islice(positions_gen, instruction_index // 2, None)) + + _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. + class StackSummary(list): - """A stack of frames.""" + """A list of FrameSummary objects, representing a stack of frames.""" @classmethod def extract(klass, frame_gen, *, limit=None, lookup_lines=True, capture_locals=False): """Create a StackSummary from a traceback or stack object. - :param frame_gen: A generator that yields (frame, lineno) tuples to - include in the stack. + :param frame_gen: A generator that yields (frame, lineno) tuples + whose summaries are to be included in the stack. :param limit: None to include all frames or the number of frames to include. :param lookup_lines: If True, lookup lines for each frame immediately, @@ -349,23 +448,41 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, :param capture_locals: If True, the local variables from each frame will be captured as object representations into the FrameSummary. """ - if limit is None: + def extended_frame_gen(): + for f, lineno in frame_gen: + yield f, (lineno, None, None, None) + + return klass._extract_from_extended_frame_gen( + extended_frame_gen(), limit=limit, lookup_lines=lookup_lines, + capture_locals=capture_locals) + + @classmethod + def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, + lookup_lines=True, capture_locals=False): + # Same as extract but operates on a frame generator that yields + # (frame, (lineno, end_lineno, colno, end_colno)) in the stack. + # Only lineno is required, the remaining fields can be None if the + # information is not available. + builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT + if limit is None or builtin_limit: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 if limit is not None: - if limit >= 0: + if builtin_limit: + frame_gen = tuple(frame_gen) + frame_gen = frame_gen[len(frame_gen) - limit:] + elif limit >= 0: frame_gen = itertools.islice(frame_gen, limit) else: frame_gen = collections.deque(frame_gen, maxlen=-limit) result = klass() fnames = set() - for f, lineno in frame_gen: + for f, (lineno, end_lineno, colno, end_colno) in frame_gen: co = f.f_code filename = co.co_filename name = co.co_name - fnames.add(filename) linecache.lazycache(filename, f.f_globals) # Must defer line lookups until we have called checkcache. @@ -373,10 +490,16 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, f_locals = f.f_locals else: f_locals = None - result.append(FrameSummary( - filename, lineno, name, lookup_line=False, locals=f_locals)) + result.append( + FrameSummary(filename, lineno, name, + lookup_line=False, locals=f_locals, + end_lineno=end_lineno, colno=colno, end_colno=end_colno, + _code=f.f_code, + ) + ) for filename in fnames: linecache.checkcache(filename) + # If immediate lookup was desired, trigger lookups now. if lookup_lines: for f in result: @@ -402,7 +525,224 @@ def from_list(klass, a_list): result.append(FrameSummary(filename, lineno, name, line=line)) return result - def format(self): + def format_frame_summary(self, frame_summary, **kwargs): + """Format the lines for a single FrameSummary. + + Returns a string representing one frame involved in the stack. This + gets called for every frame to be printed in the stack summary. + """ + colorize = kwargs.get("colorize", False) + row = [] + filename = frame_summary.filename + if frame_summary.filename.startswith("<stdin-") and frame_summary.filename.endswith('>'): + filename = "<stdin>" + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback + row.append( + ' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( + theme.filename, + filename, + theme.reset, + theme.line_no, + frame_summary.lineno, + theme.reset, + theme.frame, + frame_summary.name, + theme.reset, + ) + ) + if frame_summary._dedented_lines and frame_summary._dedented_lines.strip(): + if ( + frame_summary.colno is None or + frame_summary.end_colno is None + ): + # only output first line if column information is missing + row.append(textwrap.indent(frame_summary.line, ' ') + "\n") + else: + # get first and last line + all_lines_original = frame_summary._original_lines.splitlines() + first_line = all_lines_original[0] + # assume all_lines_original has enough lines (since we constructed it) + last_line = all_lines_original[frame_summary.end_lineno - frame_summary.lineno] + + # character index of the start/end of the instruction + start_offset = _byte_offset_to_character_offset(first_line, frame_summary.colno) + end_offset = _byte_offset_to_character_offset(last_line, frame_summary.end_colno) + + all_lines = frame_summary._dedented_lines.splitlines()[ + :frame_summary.end_lineno - frame_summary.lineno + 1 + ] + + # adjust start/end offset based on dedent + dedent_characters = len(first_line) - len(all_lines[0]) + start_offset = max(0, start_offset - dedent_characters) + end_offset = max(0, end_offset - dedent_characters) + + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(all_lines[0], offset=start_offset) + dp_end_offset = _display_width(all_lines[-1], offset=end_offset) + + # get exact code segment corresponding to the instruction + segment = "\n".join(all_lines) + segment = segment[start_offset:len(segment) - (len(all_lines[-1]) - end_offset)] + + # attempt to parse for anchors + anchors = None + show_carets = False + with suppress(Exception): + anchors = _extract_caret_anchors_from_line_segment(segment) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) + + result = [] + + # only display first line, last line, and lines around anchor start/end + significant_lines = {0, len(all_lines) - 1} + + anchors_left_end_offset = 0 + anchors_right_start_offset = 0 + primary_char = "^" + secondary_char = "^" + if anchors: + anchors_left_end_offset = anchors.left_end_offset + anchors_right_start_offset = anchors.right_start_offset + # computed anchor positions do not take start_offset into account, + # so account for it here + if anchors.left_end_lineno == 0: + anchors_left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors_right_start_offset += start_offset + + # account for display width + anchors_left_end_offset = _display_width( + all_lines[anchors.left_end_lineno], offset=anchors_left_end_offset + ) + anchors_right_start_offset = _display_width( + all_lines[anchors.right_start_lineno], offset=anchors_right_start_offset + ) + + primary_char = anchors.primary_char + secondary_char = anchors.secondary_char + significant_lines.update( + range(anchors.left_end_lineno - 1, anchors.left_end_lineno + 2) + ) + significant_lines.update( + range(anchors.right_start_lineno - 1, anchors.right_start_lineno + 2) + ) + + # remove bad line numbers + significant_lines.discard(-1) + significant_lines.discard(len(all_lines)) + + def output_line(lineno): + """output all_lines[lineno] along with carets""" + result.append(all_lines[lineno] + "\n") + if not show_carets: + return + num_spaces = len(all_lines[lineno]) - len(all_lines[lineno].lstrip()) + carets = [] + num_carets = dp_end_offset if lineno == len(all_lines) - 1 else _display_width(all_lines[lineno]) + # compute caret character for each position + for col in range(num_carets): + if col < num_spaces or (lineno == 0 and col < dp_start_offset): + # before first non-ws char of the line, or before start of instruction + carets.append(' ') + elif anchors and ( + lineno > anchors.left_end_lineno or + (lineno == anchors.left_end_lineno and col >= anchors_left_end_offset) + ) and ( + lineno < anchors.right_start_lineno or + (lineno == anchors.right_start_lineno and col < anchors_right_start_offset) + ): + # within anchors + carets.append(secondary_char) + else: + carets.append(primary_char) + if colorize: + # Replace the previous line with a red version of it only in the parts covered + # by the carets. + line = result[-1] + colorized_line_parts = [] + colorized_carets_parts = [] + + for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + caret_group = list(group) + if color == "^": + colorized_line_parts.append(theme.error_highlight + "".join(char for char, _ in caret_group) + theme.reset) + colorized_carets_parts.append(theme.error_highlight + "".join(caret for _, caret in caret_group) + theme.reset) + elif color == "~": + colorized_line_parts.append(theme.error_range + "".join(char for char, _ in caret_group) + theme.reset) + colorized_carets_parts.append(theme.error_range + "".join(caret for _, caret in caret_group) + theme.reset) + else: + colorized_line_parts.append("".join(char for char, _ in caret_group)) + colorized_carets_parts.append("".join(caret for _, caret in caret_group)) + + colorized_line = "".join(colorized_line_parts) + colorized_carets = "".join(colorized_carets_parts) + result[-1] = colorized_line + result.append(colorized_carets + "\n") + else: + result.append("".join(carets) + "\n") + + # display significant lines + sig_lines_list = sorted(significant_lines) + for i, lineno in enumerate(sig_lines_list): + if i: + linediff = lineno - sig_lines_list[i - 1] + if linediff == 2: + # 1 line in between - just output it + output_line(lineno - 1) + elif linediff > 2: + # > 1 line in between - abbreviate + result.append(f"...<{linediff - 1} lines>...\n") + output_line(lineno) + + row.append( + textwrap.indent(textwrap.dedent("".join(result)), ' ', lambda line: True) + ) + if frame_summary.locals: + for name, value in sorted(frame_summary.locals.items()): + row.append(' {name} = {value}\n'.format(name=name, value=value)) + + return ''.join(row) + + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + + def format(self, **kwargs): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the @@ -414,37 +754,34 @@ def format(self): repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ + colorize = kwargs.get("colorize", False) result = [] last_file = None last_line = None last_name = None count = 0 - for frame in self: - if (last_file is None or last_file != frame.filename or - last_line is None or last_line != frame.lineno or - last_name is None or last_name != frame.name): + for frame_summary in self: + formatted_frame = self.format_frame_summary(frame_summary, colorize=colorize) + if formatted_frame is None: + continue + if (last_file is None or last_file != frame_summary.filename or + last_line is None or last_line != frame_summary.lineno or + last_name is None or last_name != frame_summary.name): if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( f' [Previous line repeated {count} more ' f'time{"s" if count > 1 else ""}]\n' ) - last_file = frame.filename - last_line = frame.lineno - last_name = frame.name + last_file = frame_summary.filename + last_line = frame_summary.lineno + last_name = frame_summary.name count = 0 count += 1 if count > _RECURSIVE_CUTOFF: continue - row = [] - row.append(' File "{}", line {}, in {}\n'.format( - frame.filename, frame.lineno, frame.name)) - if frame.line: - row.append(' {}\n'.format(frame.line.strip())) - if frame.locals: - for name, value in sorted(frame.locals.items()): - row.append(' {name} = {value}\n'.format(name=name, value=value)) - result.append(''.join(row)) + result.append(formatted_frame) + if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( @@ -454,6 +791,216 @@ def format(self): return result +def _byte_offset_to_character_offset(str, offset): + as_utf8 = str.encode('utf-8') + return len(as_utf8[:offset].decode("utf-8", errors="replace")) + + +_Anchors = collections.namedtuple( + "_Anchors", + [ + "left_end_lineno", + "left_end_offset", + "right_start_lineno", + "right_start_offset", + "primary_char", + "secondary_char", + ], + defaults=["~", "^"] +) + +def _extract_caret_anchors_from_line_segment(segment): + """ + Given source code `segment` corresponding to a FrameSummary, determine: + - for binary ops, the location of the binary op + - for indexing and function calls, the location of the brackets. + `segment` is expected to be a valid Python expression. + """ + import ast + + try: + # Without parentheses, `segment` is parsed as a statement. + # Binary ops, subscripts, and calls are expressions, so + # we can wrap them with parentheses to parse them as + # (possibly multi-line) expressions. + # e.g. if we try to highlight the addition in + # x = ( + # a + + # b + # ) + # then we would ast.parse + # a + + # b + # which is not a valid statement because of the newline. + # Adding brackets makes it a valid expression. + # ( + # a + + # b + # ) + # Line locations will be different than the original, + # which is taken into account later on. + tree = ast.parse(f"(\n{segment}\n)") + except SyntaxError: + return None + + if len(tree.body) != 1: + return None + + lines = segment.splitlines() + + def normalize(lineno, offset): + """Get character index given byte offset""" + return _byte_offset_to_character_offset(lines[lineno], offset) + + def next_valid_char(lineno, col): + """Gets the next valid character index in `lines`, if + the current location is not valid. Handles empty lines. + """ + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + def increment(lineno, col): + """Get the next valid character index in `lines`.""" + col += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def nextline(lineno, col): + """Get the next valid character at least on the next line""" + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def increment_until(lineno, col, stop): + """Get the next valid non-"\\#" character that satisfies the `stop` predicate""" + while True: + ch = lines[lineno][col] + if ch in "\\#": + lineno, col = nextline(lineno, col) + elif not stop(ch): + lineno, col = increment(lineno, col) + else: + break + return lineno, col + + def setup_positions(expr, force_valid=True): + """Get the lineno/col position of the end of `expr`. If `force_valid` is True, + forces the position to be a valid character (e.g. if the position is beyond the + end of the line, move to the next line) + """ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket + newline to `segment` when calling ast.parse + lineno = expr.end_lineno - 2 + col = normalize(lineno, expr.end_col_offset) + return next_valid_char(lineno, col) if force_valid else (lineno, col) + + statement = tree.body[0] + match statement: + case ast.Expr(expr): + match expr: + case ast.BinOp(): + # ast gives these locations for BinOp subexpressions + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + lineno, col = setup_positions(expr.left) + + # First operator character is the first non-space/')' character + lineno, col = increment_until(lineno, col, lambda x: not x.isspace() and x != ')') + + # binary op is 1 or 2 characters long, on the same line, + # before the right subexpression + right_col = col + 1 + if ( + right_col < len(lines[lineno]) + and ( + # operator char should not be in the right subexpression + expr.right.lineno - 2 > lineno or + right_col < normalize(expr.right.lineno - 2, expr.right.col_offset) + ) + and not (ch := lines[lineno][right_col]).isspace() + and ch not in "\\#" + ): + right_col += 1 + + # right_col can be invalid since it is exclusive + return _Anchors(lineno, col, lineno, right_col) + case ast.Subscript(): + # ast gives these locations for value and slice subexpressions + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.value) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '[') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + case ast.Call(): + # ast gives these locations for function call expressions + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.func) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '(') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + + return None + +_WIDE_CHAR_SPECIFIERS = "WF" + +def _display_width(line, offset=None): + """Calculate the extra amount of width space the given source + code segment might take if it were to be displayed on a fixed + width output device. Supports wide unicode characters and emojis.""" + + if offset is None: + offset = len(line) + + # Fast track for ASCII-only strings + if line.isascii(): + return offset + + import unicodedata + + return sum( + 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 + for char in line[:offset] + ) + + + +class _ExceptionPrintContext: + def __init__(self): + self.seen = set() + self.exception_group_depth = 0 + self.need_close = False + + def indent(self): + return ' ' * (2 * self.exception_group_depth) + + def emit(self, text_gen, margin_char=None): + if margin_char is None: + margin_char = '|' + indent_str = self.indent() + if self.exception_group_depth: + indent_str += margin_char + ' ' + + if isinstance(text_gen, str): + yield textwrap.indent(text_gen, indent_str, lambda line: True) + else: + for text in text_gen: + yield textwrap.indent(text, indent_str, lambda line: True) + + class TracebackException: """An exception ready for rendering. @@ -461,16 +1008,24 @@ class TracebackException: to this intermediary form to ensure that no references are held, while still being able to fully print or format it. + max_group_width and max_group_depth control the formatting of exception + groups. The depth refers to the nesting level of the group, and the width + refers to the size of a single exception group's exceptions array. The + formatted output is truncated when either limit is exceeded. + Use `from_exception` to create TracebackException instances from exception objects, or the constructor to create TracebackException instances from individual components. - :attr:`__cause__` A TracebackException of the original *__cause__*. - :attr:`__context__` A TracebackException of the original *__context__*. + - :attr:`exceptions` For exception groups - a list of TracebackException + instances for the nested *exceptions*. ``None`` for other exceptions. - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -481,14 +1036,14 @@ class TracebackException: occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. - - :attr:`end_offset` For syntax errors - the offset into the text where the - error occurred. Can be `None` if not present. + - :attr:`end_offset` For syntax errors - the end offset into the text where + the error occurred. Can be `None` if not present. - :attr:`msg` For syntax errors - the compiler error message. """ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -498,14 +1053,34 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen = set() _seen.add(id(exc_value)) - # TODO: locals. - self.stack = StackSummary.extract( - walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, + self.max_group_width = max_group_width + self.max_group_depth = max_group_depth + + self.stack = StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(exc_traceback), + limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line - self._str = _some_str(exc_value) + self._str = _safe_string(exc_value, 'exception') + try: + self.__notes__ = getattr(exc_value, '__notes__', None) + except Exception as e: + self.__notes__ = [ + f'Ignored error getting __notes__: {_safe_string(e, '__notes__', repr)}'] + + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None + if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename @@ -517,6 +1092,27 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True + self._exc_metadata = getattr(exc_value, "_metadata", None) + elif exc_type and issubclass(exc_type, ImportError) and \ + getattr(exc_value, "name_from", None) is not None: + wrong_name = getattr(exc_value, "name_from", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ + getattr(exc_value, "name", None) is not None: + wrong_name = getattr(exc_value, "name", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + if issubclass(exc_type, NameError): + wrong_name = getattr(exc_value, "name", None) + if wrong_name is not None and wrong_name in sys.stdlib_module_names: + if suggestion: + self._str += f" Or did you forget to import '{wrong_name}'?" + else: + self._str += f". Did you forget to import '{wrong_name}'?" if lookup_lines: self._load_lines() self.__suppress_context__ = \ @@ -528,7 +1124,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, queue = [(self, exc_value)] while queue: te, e = queue.pop() - if (e and e.__cause__ is not None + if (e is not None and e.__cause__ is not None and id(e.__cause__) not in _seen): cause = TracebackException( type(e.__cause__), @@ -537,6 +1133,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: cause = None @@ -547,7 +1145,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, not e.__suppress_context__) else: need_context = True - if (e and e.__context__ is not None + if (e is not None and e.__context__ is not None and need_context and id(e.__context__) not in _seen): context = TracebackException( type(e.__context__), @@ -556,21 +1154,62 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: context = None + + if e is not None and isinstance(e, BaseExceptionGroup): + exceptions = [] + for exc in e.exceptions: + texc = TracebackException( + type(exc), + exc, + exc.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, + _seen=_seen) + exceptions.append(texc) + else: + exceptions = None + te.__cause__ = cause te.__context__ = context + te.exceptions = exceptions if cause: queue.append((te.__cause__, e.__cause__)) if context: queue.append((te.__context__, e.__context__)) + if exceptions: + queue.extend(zip(te.exceptions, e.exceptions)) @classmethod def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "<unknown>" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -584,72 +1223,249 @@ def __eq__(self, other): def __str__(self): return self._str - def format_exception_only(self): + def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. + + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ - if self.exc_type is None: - yield _format_final_exc_line(None, self._str) + colorize = kwargs.get("colorize", False) + + indent = 3 * _depth * ' ' + if not self._have_exc_type: + yield indent + _format_final_exc_line(None, self._str, colorize=colorize) return - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "<unknown>" - stype = smod + '.' + stype + stype = self.exc_type_str + if not self._is_syntax_error: + if _depth > 0: + # Nested exceptions needs correct handling of multiline messages. + formatted = _format_final_exc_line( + stype, self._str, insert_final_newline=False, colorize=colorize + ).split('\n') + yield from [ + indent + l + '\n' + for l in formatted + ] + else: + yield _format_final_exc_line(stype, self._str, colorize=colorize) + else: + yield from [indent + l for l in self._format_syntax_error(stype, colorize=colorize)] + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): + for note in self.__notes__: + note = _safe_string(note, 'note') + yield from [indent + l + '\n' for l in note.split('\n')] + elif self.__notes__ is not None: + yield indent + "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) + + if self.exceptions and show_group: + for ex in self.exceptions: + yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) + + def _find_keyword_typos(self): + assert self._is_syntax_error + try: + import _suggestions + except ImportError: + _suggestions = None + + # Only try to find keyword typos if there is no custom message + if self.msg != "invalid syntax" and "Perhaps you forgot a comma" not in self.msg: + return + + if not self._exc_metadata: + return - if not issubclass(self.exc_type, SyntaxError): - yield _format_final_exc_line(stype, self._str) + line, offset, source = self._exc_metadata + end_line = int(self.lineno) if self.lineno is not None else 0 + lines = None + from_filename = False + + if source is None: + if self.filename: + try: + with open(self.filename) as f: + lines = f.read().splitlines() + except Exception: + line, end_line, offset = 0,1,0 + else: + from_filename = True + lines = lines if lines is not None else self.text.splitlines() else: - yield from self._format_syntax_error(stype) + lines = source.splitlines() + + error_code = lines[line -1 if line > 0 else 0:end_line] + error_code = textwrap.dedent('\n'.join(error_code)) + + # Do not continue if the source is too large + if len(error_code) > 1024: + return + + error_lines = error_code.splitlines() + tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) + tokens_left_to_process = 10 + import difflib + for token in tokens: + start, end = token.start, token.end + if token.type != tokenize.NAME: + continue + # Only consider NAME tokens on the same line as the error + the_end = end_line if line == 0 else end_line + 1 + if from_filename and token.start[0]+line != the_end: + continue + wrong_name = token.string + if wrong_name in keyword.kwlist: + continue - def _format_syntax_error(self, stype): + # Limit the number of valid tokens to consider to not spend + # to much time in this function + tokens_left_to_process -= 1 + if tokens_left_to_process < 0: + break + # Limit the number of possible matches to try + max_matches = 3 + matches = [] + if _suggestions is not None: + suggestion = _suggestions._generate_suggestions(keyword.kwlist, wrong_name) + if suggestion: + matches.append(suggestion) + matches.extend(difflib.get_close_matches(wrong_name, keyword.kwlist, n=max_matches, cutoff=0.5)) + matches = matches[:max_matches] + for suggestion in matches: + if not suggestion or suggestion == wrong_name: + continue + # Try to replace the token with the keyword + the_lines = error_lines.copy() + the_line = the_lines[start[0] - 1][:] + chars = list(the_line) + chars[token.start[1]:token.end[1]] = suggestion + the_lines[start[0] - 1] = ''.join(chars) + code = '\n'.join(the_lines) + + # Check if it works + try: + codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST) + except SyntaxError: + continue + + # Keep token.line but handle offsets correctly + self.text = token.line + self.offset = token.start[1] + 1 + self.end_offset = token.end[1] + 1 + self.lineno = start[0] + self.end_lineno = end[0] + self.msg = f"invalid syntax. Did you mean '{suggestion}'?" + return + + + def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. + colorize = kwargs.get("colorize", False) + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback filename_suffix = '' if self.lineno is not None: - yield ' File "{}", line {}\n'.format( - self.filename or "<string>", self.lineno) + yield ' File {}"{}"{}, line {}{}{}\n'.format( + theme.filename, + self.filename or "<string>", + theme.reset, + theme.line_no, + self.lineno, + theme.reset, + ) elif self.filename is not None: filename_suffix = ' ({})'.format(self.filename) text = self.text - if text is not None: + if isinstance(text, str): # text = " foo\n" # rtext = " foo" # ltext = "foo" + with suppress(Exception): + self._find_keyword_typos() + text = self.text rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) - yield ' {}\n'.format(ltext) - - if self.offset is not None: + if self.offset is None: + yield ' {}\n'.format(ltext) + elif isinstance(self.offset, int): offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset - if offset == end_offset or end_offset == -1: + if self.lineno == self.end_lineno: + end_offset = ( + self.end_offset + if ( + isinstance(self.end_offset, int) + and self.end_offset != 0 + ) + else offset + ) + else: + end_offset = len(rtext) + 1 + + if self.text and offset > len(self.text): + offset = len(rtext) + 1 + if self.text and end_offset > len(self.text): + end_offset = len(rtext) + 1 + if offset >= end_offset or end_offset < 0: end_offset = offset + 1 # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces + caretspace = ' ' if colno >= 0: # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) - yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) + start_color = end_color = "" + if colorize: + # colorize from colno to end_colno + ltext = ( + ltext[:colno] + + theme.error_highlight + ltext[colno:end_colno] + theme.reset + + ltext[end_colno:] + ) + start_color = theme.error_highlight + end_color = theme.reset + yield ' {}\n'.format(ltext) + yield ' {}{}{}{}\n'.format( + "".join(caretspace), + start_color, + ('^' * (end_colno - colno)), + end_color, + ) + else: + yield ' {}\n'.format(ltext) msg = self.msg or "<no detail available>" - yield "{}: {}{}\n".format(stype, msg, filename_suffix) - - def format(self, *, chain=True): + yield "{}{}{}: {}{}{}{}\n".format( + theme.type, + stype, + theme.reset, + theme.message, + msg, + theme.reset, + filename_suffix, + ) + + def format(self, *, chain=True, _ctx=None, **kwargs): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -661,11 +1477,14 @@ def format(self, *, chain=True): The message indicating which exception occurred is always the last string in the output. """ + colorize = kwargs.get("colorize", False) + if _ctx is None: + _ctx = _ExceptionPrintContext() output = [] exc = self - while exc: - if chain: + if chain: + while exc: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ @@ -679,14 +1498,246 @@ def format(self, *, chain=True): output.append((chained_msg, exc)) exc = chained_exc - else: - output.append((None, exc)) - exc = None + else: + output.append((None, exc)) for msg, exc in reversed(output): if msg is not None: - yield msg - if exc.stack: - yield 'Traceback (most recent call last):\n' - yield from exc.stack.format() - yield from exc.format_exception_only() + yield from _ctx.emit(msg) + if exc.exceptions is None: + if exc.stack: + yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + elif _ctx.exception_group_depth > self.max_group_depth: + # exception group, but depth exceeds limit + yield from _ctx.emit( + f"... (max_group_depth is {self.max_group_depth})\n") + else: + # format exception group + is_toplevel = (_ctx.exception_group_depth == 0) + if is_toplevel: + _ctx.exception_group_depth += 1 + + if exc.stack: + yield from _ctx.emit( + 'Exception Group Traceback (most recent call last):\n', + margin_char = '+' if is_toplevel else None) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + num_excs = len(exc.exceptions) + if num_excs <= self.max_group_width: + n = num_excs + else: + n = self.max_group_width + 1 + _ctx.need_close = False + for i in range(n): + last_exc = (i == n-1) + if last_exc: + # The closing frame may be added by a recursive call + _ctx.need_close = True + + if self.max_group_width is not None: + truncated = (i >= self.max_group_width) + else: + truncated = False + title = f'{i+1}' if not truncated else '...' + yield (_ctx.indent() + + ('+-' if i==0 else ' ') + + f'+---------------- {title} ----------------\n') + _ctx.exception_group_depth += 1 + if not truncated: + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx, colorize=colorize) + else: + remaining = num_excs - self.max_group_width + plural = 's' if remaining > 1 else '' + yield from _ctx.emit( + f"and {remaining} more exception{plural}\n") + + if last_exc and _ctx.need_close: + yield (_ctx.indent() + + "+------------------------------------\n") + _ctx.need_close = False + _ctx.exception_group_depth -= 1 + + if is_toplevel: + assert _ctx.exception_group_depth == 1 + _ctx.exception_group_depth = 0 + + + def print(self, *, file=None, chain=True, **kwargs): + """Print the result of self.format(chain=chain) to 'file'.""" + colorize = kwargs.get("colorize", False) + if file is None: + file = sys.stderr + for line in self.format(chain=chain, colorize=colorize): + print(line, file=file, end="") + + +_MAX_CANDIDATE_ITEMS = 750 +_MAX_STRING_SIZE = 40 +_MOVE_COST = 2 +_CASE_COST = 1 + + +def _substitution_cost(ch_a, ch_b): + if ch_a == ch_b: + return 0 + if ch_a.lower() == ch_b.lower(): + return _CASE_COST + return _MOVE_COST + + +def _compute_suggestion_error(exc_value, tb, wrong_name): + if wrong_name is None or not isinstance(wrong_name, str): + return None + if isinstance(exc_value, AttributeError): + obj = exc_value.obj + try: + try: + d = dir(obj) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + elif isinstance(exc_value, ImportError): + try: + mod = __import__(exc_value.name) + try: + d = dir(mod) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(mod.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + else: + assert isinstance(exc_value, NameError) + # find most recent frame + if tb is None: + return None + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + d = ( + list(frame.f_locals) + + list(frame.f_globals) + + list(frame.f_builtins) + ) + d = [x for x in d if isinstance(x, str)] + + # Check first if we are in a method and the instance + # has the wrong name as attribute + if 'self' in frame.f_locals: + self = frame.f_locals['self'] + try: + has_wrong_name = hasattr(self, wrong_name) + except Exception: + has_wrong_name = False + if has_wrong_name: + return f"self.{wrong_name}" + + try: + import _suggestions + except ImportError: + pass + else: + return _suggestions._generate_suggestions(d, wrong_name) + + # Compute closest match + + if len(d) > _MAX_CANDIDATE_ITEMS: + return None + wrong_name_len = len(wrong_name) + if wrong_name_len > _MAX_STRING_SIZE: + return None + best_distance = wrong_name_len + suggestion = None + for possible_name in d: + if possible_name == wrong_name: + # A missing attribute is "found". Don't suggest it (see GH-88821). + continue + # No more than 1/3 of the involved characters should need changed. + max_distance = (len(possible_name) + wrong_name_len + 3) * _MOVE_COST // 6 + # Don't take matches we've already beaten. + max_distance = min(max_distance, best_distance - 1) + current_distance = _levenshtein_distance(wrong_name, possible_name, max_distance) + if current_distance > max_distance: + continue + if not suggestion or current_distance < best_distance: + suggestion = possible_name + best_distance = current_distance + return suggestion + + +def _levenshtein_distance(a, b, max_cost): + # A Python implementation of Python/suggestions.c:levenshtein_distance. + + # Both strings are the same + if a == b: + return 0 + + # Trim away common affixes + pre = 0 + while a[pre:] and b[pre:] and a[pre] == b[pre]: + pre += 1 + a = a[pre:] + b = b[pre:] + post = 0 + while a[:post or None] and b[:post or None] and a[post-1] == b[post-1]: + post -= 1 + a = a[:post or None] + b = b[:post or None] + if not a or not b: + return _MOVE_COST * (len(a) + len(b)) + if len(a) > _MAX_STRING_SIZE or len(b) > _MAX_STRING_SIZE: + return max_cost + 1 + + # Prefer shorter buffer + if len(b) < len(a): + a, b = b, a + + # Quick fail when a match is impossible + if (len(b) - len(a)) * _MOVE_COST > max_cost: + return max_cost + 1 + + # Instead of producing the whole traditional len(a)-by-len(b) + # matrix, we can update just one row in place. + # Initialize the buffer row + row = list(range(_MOVE_COST, _MOVE_COST * (len(a) + 1), _MOVE_COST)) + + result = 0 + for bindex in range(len(b)): + bchar = b[bindex] + distance = result = bindex * _MOVE_COST + minimum = sys.maxsize + for index in range(len(a)): + # 1) Previous distance in this row is cost(b[:b_index], a[:index]) + substitute = distance + _substitution_cost(bchar, a[index]) + # 2) cost(b[:b_index], a[:index+1]) from previous row + distance = row[index] + # 3) existing result is cost(b[:b_index+1], a[index]) + + insert_delete = min(result, distance) + _MOVE_COST + result = min(insert_delete, substitute) + + # cost(b[:b_index+1], a[:index+1]) + row[index] = result + if result < minimum: + minimum = result + if minimum > max_cost: + # Everything in this row is too big, so bail early. + return max_cost + 1 + return result diff --git a/Lib/tty.py b/Lib/tty.py index a72eb675545..5a49e040042 100644 --- a/Lib/tty.py +++ b/Lib/tty.py @@ -4,9 +4,9 @@ from termios import * -__all__ = ["setraw", "setcbreak"] +__all__ = ["cfmakeraw", "cfmakecbreak", "setraw", "setcbreak"] -# Indexes for termios list. +# Indices for termios list. IFLAG = 0 OFLAG = 1 CFLAG = 2 @@ -15,22 +15,59 @@ OSPEED = 5 CC = 6 -def setraw(fd, when=TCSAFLUSH): - """Put terminal into a raw mode.""" - mode = tcgetattr(fd) - mode[IFLAG] = mode[IFLAG] & ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON) - mode[OFLAG] = mode[OFLAG] & ~(OPOST) - mode[CFLAG] = mode[CFLAG] & ~(CSIZE | PARENB) - mode[CFLAG] = mode[CFLAG] | CS8 - mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON | IEXTEN | ISIG) +def cfmakeraw(mode): + """Make termios mode raw.""" + # Clear all POSIX.1-2017 input mode flags. + # See chapter 11 "General Terminal Interface" + # of POSIX.1-2017 Base Definitions. + mode[IFLAG] &= ~(IGNBRK | BRKINT | IGNPAR | PARMRK | INPCK | ISTRIP | + INLCR | IGNCR | ICRNL | IXON | IXANY | IXOFF) + + # Do not post-process output. + mode[OFLAG] &= ~OPOST + + # Disable parity generation and detection; clear character size mask; + # let character size be 8 bits. + mode[CFLAG] &= ~(PARENB | CSIZE) + mode[CFLAG] |= CS8 + + # Clear all POSIX.1-2017 local mode flags. + mode[LFLAG] &= ~(ECHO | ECHOE | ECHOK | ECHONL | ICANON | + IEXTEN | ISIG | NOFLSH | TOSTOP) + + # POSIX.1-2017, 11.1.7 Non-Canonical Mode Input Processing, + # Case B: MIN>0, TIME=0 + # A pending read shall block until MIN (here 1) bytes are received, + # or a signal is received. + mode[CC] = list(mode[CC]) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 - tcsetattr(fd, when, mode) -def setcbreak(fd, when=TCSAFLUSH): - """Put terminal into a cbreak mode.""" - mode = tcgetattr(fd) - mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON) +def cfmakecbreak(mode): + """Make termios mode cbreak.""" + # Do not echo characters; disable canonical input. + mode[LFLAG] &= ~(ECHO | ICANON) + + # POSIX.1-2017, 11.1.7 Non-Canonical Mode Input Processing, + # Case B: MIN>0, TIME=0 + # A pending read shall block until MIN (here 1) bytes are received, + # or a signal is received. + mode[CC] = list(mode[CC]) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 - tcsetattr(fd, when, mode) + +def setraw(fd, when=TCSAFLUSH): + """Put terminal into raw mode.""" + mode = tcgetattr(fd) + new = list(mode) + cfmakeraw(new) + tcsetattr(fd, when, new) + return mode + +def setcbreak(fd, when=TCSAFLUSH): + """Put terminal into cbreak mode.""" + mode = tcgetattr(fd) + new = list(mode) + cfmakecbreak(new) + tcsetattr(fd, when, new) + return mode diff --git a/Lib/types.py b/Lib/types.py index b036a850687..6efac339434 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -2,65 +2,78 @@ Define names for built-in types that aren't directly accessible as a builtin. """ -import sys - # Iterators in Python aren't a matter of type but of protocol. A large # and changing number of builtin types implement *some* flavor of # iterator. Don't check the type! Use hasattr to check for both # "__iter__" and "__next__" attributes instead. -def _f(): pass -FunctionType = type(_f) -LambdaType = type(lambda: None) # Same as FunctionType -CodeType = type(_f.__code__) -MappingProxyType = type(type.__dict__) -SimpleNamespace = type(sys.implementation) - -def _cell_factory(): - a = 1 - def f(): - nonlocal a - return f.__closure__[0] -CellType = type(_cell_factory()) - -def _g(): - yield 1 -GeneratorType = type(_g()) - -async def _c(): pass -_c = _c() -CoroutineType = type(_c) -_c.close() # Prevent ResourceWarning - -async def _ag(): - yield -_ag = _ag() -AsyncGeneratorType = type(_ag) - -class _C: - def _m(self): pass -MethodType = type(_C()._m) +try: + from _types import * +except ImportError: + import sys + + def _f(): pass + FunctionType = type(_f) + LambdaType = type(lambda: None) # Same as FunctionType + CodeType = type(_f.__code__) + MappingProxyType = type(type.__dict__) + SimpleNamespace = type(sys.implementation) + + def _cell_factory(): + a = 1 + def f(): + nonlocal a + return f.__closure__[0] + CellType = type(_cell_factory()) + + def _g(): + yield 1 + GeneratorType = type(_g()) + + async def _c(): pass + _c = _c() + CoroutineType = type(_c) + _c.close() # Prevent ResourceWarning + + async def _ag(): + yield + _ag = _ag() + AsyncGeneratorType = type(_ag) + + class _C: + def _m(self): pass + MethodType = type(_C()._m) + + BuiltinFunctionType = type(len) + BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) + ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + + ModuleType = type(sys) -BuiltinFunctionType = type(len) -BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + try: + raise TypeError + except TypeError as exc: + TracebackType = type(exc.__traceback__) + FrameType = type(exc.__traceback__.tb_frame) -WrapperDescriptorType = type(object.__init__) -MethodWrapperType = type(object().__str__) -MethodDescriptorType = type(str.join) -ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + GetSetDescriptorType = type(FunctionType.__code__) + MemberDescriptorType = type(FunctionType.__globals__) -ModuleType = type(sys) + GenericAlias = type(list[int]) + UnionType = type(int | str) -try: - raise TypeError -except TypeError as exc: - TracebackType = type(exc.__traceback__) - FrameType = type(exc.__traceback__.tb_frame) + EllipsisType = type(Ellipsis) + NoneType = type(None) + NotImplementedType = type(NotImplemented) -GetSetDescriptorType = type(FunctionType.__code__) -MemberDescriptorType = type(FunctionType.__globals__) + # CapsuleType cannot be accessed from pure Python, + # so there is no fallback definition. -del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export + del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export # Provide a PEP 3115 compliant mechanism for class creation @@ -279,8 +292,7 @@ def coroutine(func): if not callable(func): raise TypeError('types.coroutine() expects a callable') - # XXX RUSTPYTHON TODO: iterable coroutine - if (False and func.__class__ is FunctionType and + if (func.__class__ is FunctionType and getattr(func, '__code__', None).__class__ is CodeType): co_flags = func.__code__.co_flags @@ -325,18 +337,4 @@ def wrapped(*args, **kwargs): return wrapped -GenericAlias = type(list[int]) -UnionType = type(int | str) - -EllipsisType = type(Ellipsis) -NoneType = type(None) -NotImplementedType = type(NotImplemented) - -def __getattr__(name): - if name == 'CapsuleType': - import _socket - return type(_socket.CAPI) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - -__all__ = [n for n in globals() if n[:1] != '_'] -__all__ += ['CapsuleType'] +__all__ = [n for n in globals() if not n.startswith('_')] # for pydoc diff --git a/Lib/typing.py b/Lib/typing.py index a7397356d65..380211183a4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -27,7 +27,7 @@ import operator import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from types import GenericAlias from _typing import ( _idfunc, @@ -38,6 +38,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + Union, NoDefault, ) @@ -126,6 +127,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'evaluate_forward_ref', 'final', 'get_args', 'get_origin', @@ -160,17 +162,26 @@ 'Unpack', ] +class _LazyAnnotationLib: + def __getattr__(self, attr): + global _lazy_annotationlib + import annotationlib + _lazy_annotationlib = annotationlib + return getattr(annotationlib, attr) + +_lazy_annotationlib = _LazyAnnotationLib() -def _type_convert(arg, module=None, *, allow_special_forms=False): + +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) if isinstance(arg, str): - return ForwardRef(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms, owner=owner) return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -188,7 +199,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -240,21 +251,10 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - # When changing this function, don't forget about - # `_collections_abc._type_repr`, which does the same thing - # and must be consistent with this one. - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return '...' - if isinstance(obj, types.FunctionType): - return obj.__name__ if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return repr(obj) + return _lazy_annotationlib.type_repr(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -356,41 +356,11 @@ def _deduplicate(params, *, unhashable_fallback=False): if not unhashable_fallback: raise # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` - return _deduplicate_unhashable(params) - -def _deduplicate_unhashable(unhashable_params): - new_unhashable = [] - for t in unhashable_params: - if t not in new_unhashable: - new_unhashable.append(t) - return new_unhashable - -def _compare_args_orderless(first_args, second_args): - first_unhashable = _deduplicate_unhashable(first_args) - second_unhashable = _deduplicate_unhashable(second_args) - t = list(second_unhashable) - try: - for elem in first_unhashable: - t.remove(elem) - except ValueError: - return False - return not t - -def _remove_dups_flatten(parameters): - """Internal helper for Union creation and substitution. - - Flatten Unions among parameters, then remove duplicates. - """ - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, (_UnionGenericAlias, types.UnionType)): - params.extend(p.__args__) - else: - params.append(p) - - return tuple(_deduplicate(params, unhashable_fallback=True)) - + new_unhashable = [] + for t in params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable def _flatten_literal_params(parameters): """Internal helper for Literal creation: flatten Literals among parameters.""" @@ -460,7 +430,8 @@ def __repr__(self): _sentinel = _Sentinel() -def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), + format=None, owner=None, parent_fwdref=None, prefer_fwd_module=False): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -470,12 +441,30 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f if type_params is _sentinel: _deprecation_warning_for_no_type_params_passed("typing._eval_type") type_params = () - if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): + if isinstance(t, _lazy_annotationlib.ForwardRef): + # If the forward_ref has __forward_module__ set, evaluate() infers the globals + # from the module, and it will probably pick better than the globals we have here. + # We do this only for calls from get_type_hints() (which opts in through the + # prefer_fwd_module flag), so that the default behavior remains more straightforward. + if prefer_fwd_module and t.__forward_module__ is not None: + globalns = None + # If there are type params on the owner, we need to add them back, because + # annotationlib won't. + if owner_type_params := getattr(owner, "__type_params__", None): + globalns = getattr( + sys.modules.get(t.__forward_module__, None), "__dict__", None + ) + if globalns is not None: + globalns = dict(globalns) + for type_param in owner_type_params: + globalns[type_param.__name__] = type_param + return evaluate_forward_ref(t, globals=globalns, locals=localns, + type_params=type_params, owner=owner, + _recursive_guard=recursive_guard, format=format) + if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( - ForwardRef(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -488,7 +477,8 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f ev_args = tuple( _eval_type( - a, globalns, localns, type_params, recursive_guard=recursive_guard + a, globalns, localns, type_params, recursive_guard=recursive_guard, + format=format, owner=owner, prefer_fwd_module=prefer_fwd_module, ) for a in t.__args__ ) @@ -496,7 +486,7 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f return t if isinstance(t, GenericAlias): return GenericAlias(t.__origin__, ev_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): return functools.reduce(operator.or_, ev_args) else: return t.copy_with(ev_args) @@ -750,59 +740,6 @@ class FastConnector(Connection): item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) -@_SpecialForm -def Union(self, parameters): - """Union type; Union[X, Y] means either X or Y. - - On Python 3.10 and higher, the | operator - can also be used to denote unions; - X | Y means the same thing to the type checker as Union[X, Y]. - - To define a union, use e.g. Union[int, str]. Details: - - The arguments must be types and there must be at least one. - - None as an argument is a special case and is replaced by - type(None). - - Unions of unions are flattened, e.g.:: - - assert Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - assert Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - assert Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - assert Union[int, str] == Union[str, int] - - - You cannot subclass or instantiate a union. - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - parameters = _remove_dups_flatten(parameters) - if len(parameters) == 1: - return parameters[0] - if len(parameters) == 2 and type(None) in parameters: - return _UnionGenericAlias(self, parameters, name="Optional") - return _UnionGenericAlias(self, parameters) - -def _make_union(left, right): - """Used from the C implementation of TypeVar. - - TypeVar.__or__ calls this instead of returning types.UnionType - because we want to allow unions between TypeVars and strings - (forward references). - """ - return Union[left, right] - @_SpecialForm def Optional(self, parameters): """Optional[X] is equivalent to Union[X, None].""" @@ -1012,116 +949,85 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -class ForwardRef(_Final, _root=True): - """Internal wrapper to hold a forward reference.""" - - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__', - '__forward_is_argument__', '__forward_is_class__', - '__forward_module__') - - def __init__(self, arg, is_argument=True, module=None, *, is_class=False): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ + forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) + # For compatibility, eagerly compile the forwardref's code. + forward_ref.__forward_code__ + return forward_ref - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith('*'): - arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg - try: - code = compile(arg_to_compile, '<string>', 'eval') - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - - def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): - if type_params is _sentinel: - _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") - type_params = () - if self.__forward_arg__ in recursive_guard: - return self - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - if self.__forward_module__ is not None: - globalns = getattr( - sys.modules.get(self.__forward_module__, None), '__dict__', globalns - ) - - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) - if type_params: - globalns, localns = dict(globalns), dict(localns) - for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globalns: - globalns[param_name] = param - localns.pop(param_name, None) - - type_ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.", - is_argument=self.__forward_is_argument__, - allow_special_forms=self.__forward_is_class__, - ) - self.__forward_value__ = _eval_type( - type_, - globalns, - localns, - type_params, - recursive_guard=(recursive_guard | {self.__forward_arg__}), - ) - self.__forward_evaluated__ = True - return self.__forward_value__ - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - if self.__forward_evaluated__ and other.__forward_evaluated__: - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_module__ == other.__forward_module__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_module__)) - - def __or__(self, other): - return Union[self, other] - - def __ror__(self, other): - return Union[other, self] +def evaluate_forward_ref( + forward_ref, + *, + owner=None, + globals=None, + locals=None, + type_params=None, + format=None, + _recursive_guard=frozenset(), +): + """Evaluate a forward reference as a type hint. + + This is similar to calling the ForwardRef.evaluate() method, + but unlike that method, evaluate_forward_ref() also + recursively evaluates forward references nested within the type hint. + + *forward_ref* must be an instance of ForwardRef. *owner*, if given, + should be the object that holds the annotations that the forward reference + derived from, such as a module, class object, or function. It is used to + infer the namespaces to use for looking up names. *globals* and *locals* + can also be explicitly given to provide the global and local namespaces. + *type_params* is a tuple of type parameters that are in scope when + evaluating the forward reference. This parameter should be provided (though + it may be an empty tuple) if *owner* is not given and the forward reference + does not already have an owner set. *format* specifies the format of the + annotation and is a member of the annotationlib.Format enum, defaulting to + VALUE. - def __repr__(self): - if self.__forward_module__ is None: - module_repr = '' - else: - module_repr = f', module={self.__forward_module__!r}' - return f'ForwardRef({self.__forward_arg__!r}{module_repr})' + """ + if format == _lazy_annotationlib.Format.STRING: + return forward_ref.__forward_arg__ + if forward_ref.__forward_arg__ in _recursive_guard: + return forward_ref + + if format is None: + format = _lazy_annotationlib.Format.VALUE + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner, format=format) + + if (isinstance(value, _lazy_annotationlib.ForwardRef) + and format == _lazy_annotationlib.Format.FORWARDREF): + return value + + if isinstance(value, str): + value = _make_forward_ref(value, module=forward_ref.__forward_module__, + owner=owner or forward_ref.__owner__, + is_argument=forward_ref.__forward_is_argument__, + is_class=forward_ref.__forward_is_class__) + if owner is None: + owner = forward_ref.__owner__ + return _eval_type( + value, + globals, + locals, + type_params, + recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, + format=format, + owner=owner, + parent_fwdref=forward_ref, + ) def _is_unpacked_typevartuple(x: Any) -> bool: + # Need to check 'is True' here + # See: https://github.com/python/cpython/issues/137706 return ((not isinstance(x, type)) and - getattr(x, '__typing_is_unpacked_typevartuple__', False)) + getattr(x, '__typing_is_unpacked_typevartuple__', False) is True) def _is_typevar_like(x: Any) -> bool: @@ -1191,7 +1097,7 @@ def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) if i == len(args) and self.has_default(): - args = [*args, self.__default__] + args = (*args, self.__default__) if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1236,14 +1142,26 @@ def _generic_class_getitem(cls, args): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - for param in cls.__parameters__: + try: + parameters = cls.__parameters__ + except AttributeError as e: + init_subclass = getattr(cls, '__init_subclass__', None) + if init_subclass not in {None, Generic.__init_subclass__}: + e.add_note( + f"Note: this exception may have been caused by " + f"{init_subclass.__qualname__!r} (or the " + f"'__init_subclass__' method on a superclass) not " + f"calling 'super().__init_subclass__()'" + ) + raise + for param in parameters: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: args = prepare(cls, args) _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, args): + for param, new_arg in zip(parameters, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: @@ -1627,9 +1545,9 @@ def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): self._nparams = nparams self._defaults = defaults if origin.__module__ == 'builtins': - self.__doc__ = f'A generic version of {origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__qualname__}.' else: - self.__doc__ = f'A generic version of {origin.__module__}.{origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__module__}.{origin.__qualname__}.' @_tp_cache def __getitem__(self, params): @@ -1758,45 +1676,41 @@ def __getitem__(self, params): return self.copy_with(params) -class _UnionGenericAlias(_NotIterable, _GenericAlias, _root=True): - def copy_with(self, params): - return Union[params] +class _UnionGenericAliasMeta(type): + def __instancecheck__(self, inst: object) -> bool: + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return isinstance(inst, Union) + + def __subclasscheck__(self, inst: type) -> bool: + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return issubclass(inst, Union) def __eq__(self, other): - if not isinstance(other, (_UnionGenericAlias, types.UnionType)): - return NotImplemented - try: # fast path - return set(self.__args__) == set(other.__args__) - except TypeError: # not hashable, slow path - return _compare_args_orderless(self.__args__, other.__args__) + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + if other is _UnionGenericAlias or other is Union: + return True + return NotImplemented def __hash__(self): - return hash(frozenset(self.__args__)) + return hash(Union) - def __repr__(self): - args = self.__args__ - if len(args) == 2: - if args[0] is type(None): - return f'typing.Optional[{_type_repr(args[1])}]' - elif args[1] is type(None): - return f'typing.Optional[{_type_repr(args[0])}]' - return super().__repr__() - def __instancecheck__(self, obj): - for arg in self.__args__: - if isinstance(obj, arg): - return True - return False +class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): + """Compatibility hack. - def __subclasscheck__(self, cls): - for arg in self.__args__: - if issubclass(cls, arg): - return True - return False + A class named _UnionGenericAlias used to be used to implement + typing.Union. This class exists to serve as a shim to preserve + the meaning of some code that used to use _UnionGenericAlias + directly. - def __reduce__(self): - func, (origin, args) = super().__reduce__() - return func, (Union, args) + """ + def __new__(cls, self_cls, parameters, /, *, name=None): + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return Union[parameters] def _value_and_type_iter(parameters): @@ -1918,6 +1832,7 @@ class _TypingEllipsis: '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', '__annotate_func__', '__annotations_cache__', }) # These special attributes will be not collected as protocol members. @@ -1934,7 +1849,13 @@ def _get_protocol_attrs(cls): for base in cls.__mro__[:-1]: # without object if base.__name__ in {'Protocol', 'Generic'}: continue - annotations = getattr(base, '__annotations__', {}) + try: + annotations = base.__annotations__ + except Exception: + # Only go through annotationlib to handle deferred annotations if we need to + annotations = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: attrs.add(attr) @@ -1987,8 +1908,7 @@ def _allow_reckless_class_checks(depth=2): The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ - # XXX: RUSTPYTHON; https://github.com/python/cpython/pull/136115 - return _caller(depth) in {'abc', '_py_abc', 'functools', None} + return _caller(depth) in {'abc', 'functools', None} _PROTO_ALLOWLIST = { @@ -1998,6 +1918,8 @@ def _allow_reckless_class_checks(depth=2): 'Reversible', 'Buffer', ], 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], + 'io': ['Reader', 'Writer'], + 'os': ['PathLike'], } @@ -2150,11 +2072,17 @@ def _proto_hook(cls, other): break # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if (isinstance(annotations, collections.abc.Mapping) and - attr in annotations and - issubclass(other, Generic) and getattr(other, '_is_protocol', False)): - break + if issubclass(other, Generic) and getattr(other, "_is_protocol", False): + # We avoid the slower path through annotationlib here because in most + # cases it should be unnecessary. + try: + annos = base.__annotations__ + except Exception: + annos = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) + if attr in annos: + break else: return NotImplemented return True @@ -2217,7 +2145,7 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias. + with extra metadata. The alias behaves like a normal typing alias. Instantiating is the same as instantiating the underlying type; binding it to types is also the same. @@ -2396,12 +2324,8 @@ def greet(name: str) -> None: return val -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) - - -def get_type_hints(obj, globalns=None, localns=None, include_extras=False): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False, + *, format=None): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles @@ -2434,17 +2358,21 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): """ if getattr(obj, '__no_type_check__', None): return {} + Format = _lazy_annotationlib.Format + if format is None: + format = Format.VALUE # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + ann = _lazy_annotationlib.get_annotations(base, format=format) + if format == Format.STRING: + hints.update(ann) + continue if globalns is None: base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, @@ -2454,14 +2382,33 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): # *base_globals* first rather than *base_locals*. # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals + type_params = base.__type_params__ + base_globals, base_locals = _add_type_params_to_scope( + type_params, base_globals, base_locals, True) for name, value in ann.items(): + if isinstance(value, str): + value = _make_forward_ref(value, is_argument=False, is_class=True) + value = _eval_type(value, base_globals, base_locals, (), + format=format, owner=obj, prefer_fwd_module=True) if value is None: value = type(None) - if isinstance(value, str): - value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals, base.__type_params__) hints[name] = value - return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + if include_extras or format == Format.STRING: + return hints + else: + return {k: _strip_annotations(t) for k, t in hints.items()} + + hints = _lazy_annotationlib.get_annotations(obj, format=format) + if ( + not hints + and not isinstance(obj, types.ModuleType) + and not callable(obj) + and not hasattr(obj, '__annotations__') + and not hasattr(obj, '__annotate__') + ): + raise TypeError(f"{obj!r} is not a module, class, or callable.") + if format == Format.STRING: + return hints if globalns is None: if isinstance(obj, types.ModuleType): @@ -2476,31 +2423,38 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - hints = dict(hints) type_params = getattr(obj, "__type_params__", ()) + globalns, localns = _add_type_params_to_scope(type_params, globalns, localns, False) for name, value in hints.items(): - if value is None: - value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation - value = ForwardRef( + value = _make_forward_ref( value, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns, type_params) + value = _eval_type(value, globalns, localns, (), format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) + hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} +# Add type parameters to the globals and locals scope. This is needed for +# compatibility. +def _add_type_params_to_scope(type_params, globalns, localns, is_class): + if not type_params: + return globalns, localns + globalns = dict(globalns) + localns = dict(localns) + for param in type_params: + if not is_class or param.__name__ not in globalns: + globalns[param.__name__] = param + localns.pop(param.__name__, None) + return globalns, localns + + def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): @@ -2517,7 +2471,7 @@ def _strip_annotations(t): if stripped_args == t.__args__: return t return GenericAlias(t.__origin__, stripped_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: return t @@ -2551,8 +2505,8 @@ def get_origin(tp): return tp.__origin__ if tp is Generic: return Generic - if isinstance(tp, types.UnionType): - return types.UnionType + if isinstance(tp, Union): + return Union return None @@ -2577,7 +2531,7 @@ def get_args(tp): if _should_unflatten_callable_args(tp, res): res = (list(res[:-1]), res[-1]) return res - if isinstance(tp, types.UnionType): + if isinstance(tp, Union): return tp.__args__ return () @@ -2844,7 +2798,7 @@ class Other(Leaf): # Error reported by type checker Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) ByteString = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. + collections.abc.ByteString, 0, removal_version=(3, 17) # Not generic. ) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') @@ -2977,35 +2931,73 @@ def __round__(self, ndigits: int = 0) -> T: pass -def _make_nmtuple(name, types, module, defaults = ()): - fields = [n for n, t in types] - types = {n: _type_check(t, f"field {n} annotation must be a type") - for n, t in types} +def _make_nmtuple(name, fields, annotate_func, module, defaults = ()): nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = types + nm_tpl.__annotate__ = nm_tpl.__new__.__annotate__ = annotate_func return nm_tpl +def _make_eager_annotate(types): + checked_types = {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in types.items()} + def annotate(format): + match format: + case _lazy_annotationlib.Format.VALUE | _lazy_annotationlib.Format.FORWARDREF: + return checked_types + case _lazy_annotationlib.Format.STRING: + return _lazy_annotationlib.annotations_to_string(types) + case _: + raise NotImplementedError(format) + return annotate + + # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__', + '__annotate_func__', '__annotations_cache__'}) class NamedTupleMeta(type): def __new__(cls, typename, bases, ns): assert _NamedTuple in bases + if "__classcell__" in ns: + raise TypeError( + "uses of super() and __class__ are unsupported in methods of NamedTuple subclasses") for base in bases: if base is not _NamedTuple and base is not Generic: raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + field_names = list(types) + annotate = _make_eager_annotate(types) + elif (original_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = _lazy_annotationlib.call_annotate_function( + original_annotate, _lazy_annotationlib.Format.FORWARDREF) + field_names = list(types) + + # For backward compatibility, type-check all the types at creation time + for typ in types.values(): + _type_check(typ, "field annotation must be a type") + + def annotate(format): + annos = _lazy_annotationlib.call_annotate_function( + original_annotate, format) + if format != _lazy_annotationlib.Format.STRING: + return {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in annos.items()} + return annos + else: + # Empty NamedTuple + field_names = [] + annotate = lambda format: {} default_names = [] - for field_name in types: + for field_name in field_names: if field_name in ns: default_names.append(field_name) elif default_names: @@ -3013,7 +3005,7 @@ def __new__(cls, typename, bases, ns): f"cannot follow default field" f"{'s' if len(default_names) > 1 else ''} " f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple(typename, types.items(), + nm_tpl = _make_nmtuple(typename, field_names, annotate, defaults=[ns[n] for n in default_names], module=ns['__module__']) nm_tpl.__bases__ = bases @@ -3104,7 +3096,11 @@ class Employee(NamedTuple): import warnings warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) fields = kwargs.items() - nt = _make_nmtuple(typename, fields, module=_caller()) + types = {n: _type_check(t, f"field {n} annotation must be a type") + for n, t in fields} + field_names = [n for n, _ in fields] + + nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) nt.__orig_bases__ = (NamedTuple,) return nt @@ -3158,16 +3154,26 @@ def __new__(cls, name, bases, ns, total=True): else: generic_base = () + ns_annotations = ns.pop('__annotations__', None) + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} - own_annotations = ns.get('__annotations__', {}) + if ns_annotations is not None: + own_annotate = None + own_annotations = ns_annotations + elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = _lazy_annotationlib.call_annotate_function( + own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + own_checked_annotations = { + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() @@ -3176,8 +3182,6 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -3189,8 +3193,7 @@ def __new__(cls, name, bases, ns, total=True): readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): + for annotation_key, annotation_type in own_checked_annotations.items(): qualifiers = set(_get_typeddict_qualifiers(annotation_type)) if Required in qualifiers: is_required = True @@ -3221,7 +3224,36 @@ def __new__(cls, name, bases, ns, total=True): f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = _lazy_annotationlib.call_annotate_function( + base_annotate, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = _lazy_annotationlib.call_annotate_function( + own_annotate, format, owner=tp_dict) + if format != _lazy_annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == _lazy_annotationlib.Format.STRING: + own = _lazy_annotationlib.annotations_to_string(own_annotations) + elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + own = own_checked_annotations + else: + raise NotImplementedError(format) + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) tp_dict.__readonly_keys__ = frozenset(readonly_keys) @@ -3517,7 +3549,7 @@ def readline(self, limit: int = -1) -> AnyStr: pass @abstractmethod - def readlines(self, hint: int = -1) -> List[AnyStr]: + def readlines(self, hint: int = -1) -> list[AnyStr]: pass @abstractmethod @@ -3533,7 +3565,7 @@ def tell(self) -> int: pass @abstractmethod - def truncate(self, size: int = None) -> int: + def truncate(self, size: int | None = None) -> int: pass @abstractmethod @@ -3545,11 +3577,11 @@ def write(self, s: AnyStr) -> int: pass @abstractmethod - def writelines(self, lines: List[AnyStr]) -> None: + def writelines(self, lines: list[AnyStr]) -> None: pass @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': + def __enter__(self) -> IO[AnyStr]: pass @abstractmethod @@ -3563,11 +3595,11 @@ class BinaryIO(IO[bytes]): __slots__ = () @abstractmethod - def write(self, s: Union[bytes, bytearray]) -> int: + def write(self, s: bytes | bytearray) -> int: pass @abstractmethod - def __enter__(self) -> 'BinaryIO': + def __enter__(self) -> BinaryIO: pass @@ -3588,7 +3620,7 @@ def encoding(self) -> str: @property @abstractmethod - def errors(self) -> Optional[str]: + def errors(self) -> str | None: pass @property @@ -3602,7 +3634,7 @@ def newlines(self) -> Any: pass @abstractmethod - def __enter__(self) -> 'TextIO': + def __enter__(self) -> TextIO: pass @@ -3798,7 +3830,9 @@ def __getattr__(attr): Soft-deprecated objects which are costly to create are only created on-demand here. """ - if attr in {"Pattern", "Match"}: + if attr == "ForwardRef": + obj = _lazy_annotationlib.ForwardRef + elif attr in {"Pattern", "Match"}: import re obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 7ab3f5e87b4..b049402eed7 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -27,7 +27,7 @@ def testMultiply(self): http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell -Copyright (c) 2003-2010 Python Software Foundation +Copyright (c) 2003 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. @@ -51,36 +51,33 @@ def testMultiply(self): 'registerResult', 'removeResult', 'removeHandler', 'addModuleCleanup', 'doModuleCleanups', 'enterModuleContext'] -# Expose obsolete functions for backwards compatibility -# bpo-5846: Deprecated in Python 3.11, scheduled for removal in Python 3.13. -__all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases']) - __unittest = True -from .result import TestResult -from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, - skipIf, skipUnless, expectedFailure, doModuleCleanups, - enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .case import ( + FunctionTestCase, + SkipTest, + TestCase, + addModuleCleanup, + doModuleCleanups, + enterModuleContext, + expectedFailure, + skip, + skipIf, + skipUnless, +) from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main -from .runner import TextTestRunner, TextTestResult -from .signals import installHandler, registerResult, removeResult, removeHandler -# IsolatedAsyncioTestCase will be imported lazily. -from .loader import makeSuite, getTestCaseNames, findTestCases - -# deprecated -_TextTestResult = TextTestResult - +from .main import TestProgram, main # noqa: F401 +from .result import TestResult +from .runner import TextTestResult, TextTestRunner +from .signals import ( + installHandler, + registerResult, + removeHandler, + removeResult, +) +from .suite import BaseTestSuite, TestSuite # noqa: F401 -# There are no tests here, so don't try to run anything discovered from -# introspecting the symbols (e.g. FunctionTestCase). Instead, all our -# tests come from within unittest.test. -def load_tests(loader, tests, pattern): - import os.path - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - return loader.discover(start_dir=this_dir, pattern=pattern) +# IsolatedAsyncioTestCase will be imported lazily. # Lazy import of IsolatedAsyncioTestCase from .async_case @@ -98,6 +95,7 @@ def __getattr__(name): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + # XXX: RUSTPYTHON # This is very useful to reduce platform difference boilerplates in tests. def expectedFailureIf(condition, reason): @@ -111,4 +109,4 @@ def expectedFailureIf(condition, reason): # Even more useful because most of them are windows only. def expectedFailureIfWindows(reason): import sys - return expectedFailureIf(sys.platform == 'win32', reason) + return expectedFailureIf(sys.platform == 'win32', reason) \ No newline at end of file diff --git a/Lib/unittest/__main__.py b/Lib/unittest/__main__.py index e5876f569b5..50111190eee 100644 --- a/Lib/unittest/__main__.py +++ b/Lib/unittest/__main__.py @@ -1,6 +1,7 @@ """Main entry point""" import sys + if sys.argv[0].endswith("__main__.py"): import os.path # We change sys.argv[0] to make help message more useful diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 94868e5bb95..c61abb15745 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -1,9 +1,8 @@ -import logging import collections +import logging from .case import _BaseTestCaseContext - _LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index bd2a4711560..a1c0d6c368c 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -5,6 +5,7 @@ from .case import TestCase +__unittest = True class IsolatedAsyncioTestCase(TestCase): # Names intentionally have a long prefix @@ -25,12 +26,15 @@ class IsolatedAsyncioTestCase(TestCase): # them inside the same task. # Note: the test case modifies event loop policy if the policy was not instantiated - # yet. + # yet, unless loop_factory=asyncio.EventLoop is set. # asyncio.get_event_loop_policy() creates a default policy on demand but never # returns None # I believe this is not an issue in user level tests but python itself for testing # should reset a policy in every test module # by calling asyncio.set_event_loop_policy(None) in tearDownModule() + # or set loop_factory=asyncio.EventLoop + + loop_factory = None def __init__(self, methodName='runTest'): super().__init__(methodName) @@ -71,9 +75,17 @@ async def enterAsyncContext(self, cm): enter = cls.__aenter__ exit = cls.__aexit__ except AttributeError: - raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the asynchronous context manager protocol" - ) from None + msg = (f"'{cls.__module__}.{cls.__qualname__}' object does " + "not support the asynchronous context manager protocol") + try: + cls.__enter__ + cls.__exit__ + except AttributeError: + pass + else: + msg += (" but it supports the context manager protocol. " + "Did you mean to use enterContext()?") + raise TypeError(msg) from None result = await enter(cm) self.addAsyncCleanup(exit, cm, None, None, None) return result @@ -87,9 +99,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) @@ -118,7 +134,7 @@ def _callMaybeAsync(self, func, /, *args, **kwargs): def _setupAsyncioRunner(self): assert self._asyncioRunner is None, 'asyncio runner is already initialized' - runner = asyncio.Runner(debug=True) + runner = asyncio.Runner(debug=True, loop_factory=self.loop_factory) self._asyncioRunner = runner def _tearDownAsyncioRunner(self): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 36daa61fa31..b09836d6747 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1,20 +1,25 @@ """Test case implementation""" -import sys -import functools +import collections +import contextlib import difflib +import functools import pprint import re -import warnings -import collections -import contextlib -import traceback +import sys import time +import traceback import types +import warnings from . import result -from .util import (strclass, safe_repr, _count_diff_all_purpose, - _count_diff_hashable, _common_shorten_repr) +from .util import ( + _common_shorten_repr, + _count_diff_all_purpose, + _count_diff_hashable, + safe_repr, + strclass, +) __unittest = True @@ -111,8 +116,17 @@ def _enter_context(cm, addcleanup): enter = cls.__enter__ exit = cls.__exit__ except AttributeError: - raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the context manager protocol") from None + msg = (f"'{cls.__module__}.{cls.__qualname__}' object does " + "not support the context manager protocol") + try: + cls.__aenter__ + cls.__aexit__ + except AttributeError: + pass + else: + msg += (" but it supports the asynchronous context manager " + "protocol. Did you mean to use enterAsyncContext()?") + raise TypeError(msg) from None result = enter(cm) addcleanup(exit, cm, None, None, None) return result @@ -603,9 +617,18 @@ def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() @@ -1312,13 +1335,71 @@ def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + standardMsg = f'{safe_repr(obj)} is not an instance of any of {cls!r}' + else: + standardMsg = f'{safe_repr(obj)} is not an instance of {cls!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): - standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + for x in cls: + if isinstance(obj, x): + cls = x + break + standardMsg = f'{safe_repr(obj)} is an instance of {cls!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertIsSubclass(self, cls, superclass, msg=None): + try: + if issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + standardMsg = f'{cls!r} is not a subclass of any of {superclass!r}' + else: + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + try: + if not issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + for x in superclass: + if issubclass(cls, x): + superclass = x + break + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has unexpected attribute {name!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegex(self, expected_exception, expected_regex, @@ -1382,6 +1463,80 @@ def assertNotRegex(self, text, unexpected_regex, msg=None): msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) + def _tail_type_check(self, s, tails, msg): + if not isinstance(tails, tuple): + tails = (tails,) + for tail in tails: + if isinstance(tail, str): + if not isinstance(s, str): + self.fail(self._formatMessage(msg, + f'Expected str, not {type(s).__name__}')) + elif isinstance(tail, (bytes, bytearray)): + if not isinstance(s, (bytes, bytearray)): + self.fail(self._formatMessage(msg, + f'Expected bytes, not {type(s).__name__}')) + + def assertStartsWith(self, s, prefix, msg=None): + try: + if s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(prefix) + if isinstance(prefix, tuple): + standardMsg = f"{a} doesn't start with any of {b}" + else: + standardMsg = f"{a} doesn't start with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + try: + if not s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + if isinstance(prefix, tuple): + for x in prefix: + if s.startswith(x): + prefix = x + break + a = safe_repr(s, short=True) + b = safe_repr(prefix) + self.fail(self._formatMessage(msg, f"{a} starts with {b}")) + + def assertEndsWith(self, s, suffix, msg=None): + try: + if s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(suffix) + if isinstance(suffix, tuple): + standardMsg = f"{a} doesn't end with any of {b}" + else: + standardMsg = f"{a} doesn't end with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotEndsWith(self, s, suffix, msg=None): + try: + if not s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + if isinstance(suffix, tuple): + for x in suffix: + if s.endswith(x): + suffix = x + break + a = safe_repr(s, short=True) + b = safe_repr(suffix) + self.fail(self._formatMessage(msg, f"{a} ends with {b}")) class FunctionTestCase(TestCase): diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 7e6ce2f224b..fa8d647ad8a 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -1,13 +1,11 @@ """Loading unittests.""" +import functools import os import re import sys import traceback import types -import functools -import warnings - from fnmatch import fnmatch, fnmatchcase from . import case, suite, util @@ -57,9 +55,7 @@ def testSkipped(self): TestClass = type("ModuleSkipped", (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) -def _jython_aware_splitext(path): - if path.lower().endswith('$py.class'): - return path[:-9] +def _splitext(path): return os.path.splitext(path)[0] @@ -87,40 +83,26 @@ def loadTestsFromTestCase(self, testCaseClass): raise TypeError("Test cases should not be derived from " "TestSuite. Maybe you meant to derive from " "TestCase?") - testCaseNames = self.getTestCaseNames(testCaseClass) - if not testCaseNames and hasattr(testCaseClass, 'runTest'): - testCaseNames = ['runTest'] + if testCaseClass in (case.TestCase, case.FunctionTestCase): + # We don't load any tests from base types that should not be loaded. + testCaseNames = [] + else: + testCaseNames = self.getTestCaseNames(testCaseClass) + if not testCaseNames and hasattr(testCaseClass, 'runTest'): + testCaseNames = ['runTest'] loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) return loaded_suite - # XXX After Python 3.5, remove backward compatibility hacks for - # use_load_tests deprecation via *args and **kws. See issue 16662. - def loadTestsFromModule(self, module, *args, pattern=None, **kws): + def loadTestsFromModule(self, module, *, pattern=None): """Return a suite of all test cases contained in the given module""" - # This method used to take an undocumented and unofficial - # use_load_tests argument. For backward compatibility, we still - # accept the argument (which can also be the first position) but we - # ignore it and issue a deprecation warning if it's present. - if len(args) > 0 or 'use_load_tests' in kws: - warnings.warn('use_load_tests is deprecated and ignored', - DeprecationWarning) - kws.pop('use_load_tests', None) - if len(args) > 1: - # Complain about the number of arguments, but don't forget the - # required `module` argument. - complaint = len(args) + 1 - raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(complaint)) - if len(kws) != 0: - # Since the keyword arguments are unsorted (see PEP 468), just - # pick the alphabetically sorted first argument to complain about, - # if multiple were given. At least the error message will be - # predictable. - complaint = sorted(kws)[0] - raise TypeError("loadTestsFromModule() got an unexpected keyword argument '{}'".format(complaint)) tests = [] for name in dir(module): obj = getattr(module, name) - if isinstance(obj, type) and issubclass(obj, case.TestCase): + if ( + isinstance(obj, type) + and issubclass(obj, case.TestCase) + and obj not in (case.TestCase, case.FunctionTestCase) + ): tests.append(self.loadTestsFromTestCase(obj)) load_tests = getattr(module, 'load_tests', None) @@ -189,7 +171,11 @@ def loadTestsFromName(self, name, module=None): if isinstance(obj, types.ModuleType): return self.loadTestsFromModule(obj) - elif isinstance(obj, type) and issubclass(obj, case.TestCase): + elif ( + isinstance(obj, type) + and issubclass(obj, case.TestCase) + and obj not in (case.TestCase, case.FunctionTestCase) + ): return self.loadTestsFromTestCase(obj) elif (isinstance(obj, types.FunctionType) and isinstance(parent, type) and @@ -267,6 +253,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -286,6 +273,8 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): self._top_level_dir = top_level_dir is_not_importable = False + is_namespace = False + tests = [] if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: @@ -298,12 +287,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): is_not_importable = True else: the_module = sys.modules[start_dir] - top_part = start_dir.split('.')[0] - try: - start_dir = os.path.abspath( - os.path.dirname((the_module.__file__))) - except AttributeError: - if the_module.__name__ in sys.builtin_module_names: + if not hasattr(the_module, "__file__") or the_module.__file__ is None: + # look for namespace packages + try: + spec = the_module.__spec__ + except AttributeError: + spec = None + + if spec and spec.submodule_search_locations is not None: + is_namespace = True + + for path in the_module.__path__: + if (not set_implicit_top and + not path.startswith(top_level_dir)): + continue + self._top_level_dir = \ + (path.split(the_module.__name__ + .replace(".", os.path.sep))[0]) + tests.extend(self._find_tests(path, pattern, namespace=True)) + elif the_module.__name__ in sys.builtin_module_names: # builtin module raise TypeError('Can not use builtin modules ' 'as dotted module names') from None @@ -312,14 +314,28 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): f"don't know how to discover from {the_module!r}" ) from None + else: + top_part = start_dir.split('.')[0] + start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) + if set_implicit_top: - self._top_level_dir = self._get_directory_containing_module(top_part) + if not is_namespace: + if sys.modules[top_part].__file__ is None: + self._top_level_dir = os.path.dirname(the_module.__file__) + if self._top_level_dir not in sys.path: + sys.path.insert(0, self._top_level_dir) + else: + self._top_level_dir = \ + self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) - tests = list(self._find_tests(start_dir, pattern)) + if not is_namespace: + tests = list(self._find_tests(start_dir, pattern)) + + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): @@ -337,7 +353,7 @@ def _get_directory_containing_module(self, module_name): def _get_name_from_path(self, path): if path == self._top_level_dir: return '.' - path = _jython_aware_splitext(os.path.normpath(path)) + path = _splitext(os.path.normpath(path)) _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" @@ -354,7 +370,7 @@ def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) - def _find_tests(self, start_dir, pattern): + def _find_tests(self, start_dir, pattern, namespace=False): """Used by discovery. Yields test suites it loads.""" # Handle the __init__ in this package name = self._get_name_from_path(start_dir) @@ -363,7 +379,8 @@ def _find_tests(self, start_dir, pattern): if name != '.' and name not in self._loading_packages: # name is in self._loading_packages while we have called into # loadTestsFromModule with name. - tests, should_recurse = self._find_test_path(start_dir, pattern) + tests, should_recurse = self._find_test_path( + start_dir, pattern, namespace) if tests is not None: yield tests if not should_recurse: @@ -374,7 +391,8 @@ def _find_tests(self, start_dir, pattern): paths = sorted(os.listdir(start_dir)) for path in paths: full_path = os.path.join(start_dir, path) - tests, should_recurse = self._find_test_path(full_path, pattern) + tests, should_recurse = self._find_test_path( + full_path, pattern, False) if tests is not None: yield tests if should_recurse: @@ -382,11 +400,11 @@ def _find_tests(self, start_dir, pattern): name = self._get_name_from_path(full_path) self._loading_packages.add(name) try: - yield from self._find_tests(full_path, pattern) + yield from self._find_tests(full_path, pattern, False) finally: self._loading_packages.discard(name) - def _find_test_path(self, full_path, pattern): + def _find_test_path(self, full_path, pattern, namespace=False): """Used by discovery. Loads tests from a single file, or a directories' __init__.py when @@ -415,13 +433,13 @@ def _find_test_path(self, full_path, pattern): else: mod_file = os.path.abspath( getattr(module, '__file__', full_path)) - realpath = _jython_aware_splitext( + realpath = _splitext( os.path.realpath(mod_file)) - fullpath_noext = _jython_aware_splitext( + fullpath_noext = _splitext( os.path.realpath(full_path)) if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) - mod_name = _jython_aware_splitext( + mod_name = _splitext( os.path.basename(full_path)) expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected " @@ -430,7 +448,8 @@ def _find_test_path(self, full_path, pattern): msg % (mod_name, module_dir, expected_dir)) return self.loadTestsFromModule(module, pattern=pattern), False elif os.path.isdir(full_path): - if not os.path.isfile(os.path.join(full_path, '__init__.py')): + if (not namespace and + not os.path.isfile(os.path.join(full_path, '__init__.py'))): return None, False load_tests = None @@ -462,47 +481,3 @@ def _find_test_path(self, full_path, pattern): defaultTestLoader = TestLoader() - - -# These functions are considered obsolete for long time. -# They will be removed in Python 3.13. - -def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): - loader = TestLoader() - loader.sortTestMethodsUsing = sortUsing - loader.testMethodPrefix = prefix - loader.testNamePatterns = testNamePatterns - if suiteClass: - loader.suiteClass = suiteClass - return loader - -def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): - import warnings - warnings.warn( - "unittest.getTestCaseNames() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.getTestCaseNames() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) - -def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, - suiteClass=suite.TestSuite): - import warnings - warnings.warn( - "unittest.makeSuite() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.loadTestsFromTestCase() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase( - testCaseClass) - -def findTestCases(module, prefix='test', sortUsing=util.three_way_cmp, - suiteClass=suite.TestSuite): - import warnings - warnings.warn( - "unittest.findTestCases() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.loadTestsFromModule() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(\ - module) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index c3869de3f6f..1855fccf336 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -1,8 +1,8 @@ """Unittest main program""" -import sys import argparse import os +import sys from . import loader, runner from .signals import installHandler @@ -197,7 +197,7 @@ def _getParentArgParser(self): return parser def _getMainArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent]) + parser = argparse.ArgumentParser(parents=[parent], color=True) parser.prog = self.progName parser.print_help = self._print_help @@ -208,7 +208,7 @@ def _getMainArgParser(self, parent): return parser def _getDiscoveryArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent]) + parser = argparse.ArgumentParser(parents=[parent], color=True) parser.prog = '%s discover' % self.progName parser.epilog = ('For test discovery all test modules must be ' 'importable from the top level directory of the ' @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: + if not self.result.wasSuccessful(): + sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) else: - sys.exit(1) + sys.exit(0) main = TestProgram diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 6cec61ff35c..1089dcb11f1 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -25,19 +25,20 @@ import asyncio +import builtins import contextlib -import io import inspect +import io +import pkgutil import pprint import sys -import builtins -import pkgutil -from asyncio import iscoroutinefunction import threading -from types import CodeType, ModuleType, MethodType -from unittest.util import safe_repr -from functools import wraps, partial +from dataclasses import fields, is_dataclass +from functools import partial, wraps +from inspect import iscoroutinefunction from threading import RLock +from types import CodeType, MethodType, ModuleType +from unittest.util import safe_repr class InvalidSpecError(Exception): @@ -568,6 +569,11 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__['_mock_methods'] = spec __dict__['_spec_asyncs'] = _spec_asyncs + def _mock_extend_spec_methods(self, spec_methods): + methods = self.__dict__.get('_mock_methods') or [] + methods.extend(spec_methods) + self.__dict__['_mock_methods'] = methods + def __get_return_value(self): ret = self._mock_return_value if self._mock_delegate is not None: @@ -1766,7 +1772,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created @@ -1840,7 +1846,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -2176,8 +2183,6 @@ def _mock_set_magics(self): if getattr(self, "_mock_methods", None) is not None: these_magics = orig_magics.intersection(self._mock_methods) - - remove_magics = set() remove_magics = orig_magics - these_magics for entry in remove_magics: @@ -2477,7 +2482,7 @@ class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): recognized as an async function, and the result of a call is an awaitable: >>> mock = AsyncMock() - >>> iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) True @@ -2767,6 +2772,16 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, f'[object={spec!r}]') is_async_func = _is_async_func(spec) _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + is_dataclass_spec = True + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + dataclass_spec_list = [f.name for f in dataclass_fields] + else: + is_dataclass_spec = False + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2802,6 +2817,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) + if is_dataclass_spec: + mock._mock_extend_spec_methods(dataclass_spec_list) if isinstance(spec, FunctionTypes): # should only happen at the top level because we don't @@ -2823,7 +2840,7 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _name='()', _parent=mock, wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2837,10 +2854,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue child_kwargs = {'spec': original} # Wrap child attributes also. diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3ace0a5b7bf..8eafb3891c9 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -3,9 +3,9 @@ import io import sys import traceback +from functools import wraps from . import util -from functools import wraps __unittest = True @@ -189,7 +189,10 @@ def _exc_info_to_string(self, err, test): tb_e = traceback.TracebackException( exctype, value, tb, capture_locals=self.tb_locals, compact=True) - msgLines = list(tb_e.format()) + from _colorize import can_colorize + + colorize = hasattr(self, "stream") and can_colorize(file=self.stream) + msgLines = list(tb_e.format(colorize=colorize)) if self.buffer: output = sys.stdout.getvalue() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 2bcadf0c998..5f22d91aebd 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,6 +4,8 @@ import time import warnings +from _colorize import get_theme + from . import result from .case import _SubTest from .signals import registerResult @@ -13,18 +15,18 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): @@ -43,6 +45,7 @@ def __init__(self, stream, descriptions, verbosity, *, durations=None): self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions + self._theme = get_theme(tty_file=stream).unittest self._newline = True self.durations = durations @@ -76,86 +79,100 @@ def _write_status(self, test, status): def addSubTest(self, test, subtest, err): if err is not None: + t = self._theme if self.showAll: if issubclass(err[0], subtest.failureException): - self._write_status(subtest, "FAIL") + self._write_status(subtest, f"{t.fail}FAIL{t.reset}") else: - self._write_status(subtest, "ERROR") + self._write_status(subtest, f"{t.fail}ERROR{t.reset}") elif self.dots: if issubclass(err[0], subtest.failureException): - self.stream.write('F') + self.stream.write(f"{t.fail}F{t.reset}") else: - self.stream.write('E') + self.stream.write(f"{t.fail}E{t.reset}") self.stream.flush() super(TextTestResult, self).addSubTest(test, subtest, err) def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) + t = self._theme if self.showAll: - self._write_status(test, "ok") + self._write_status(test, f"{t.passed}ok{t.reset}") elif self.dots: - self.stream.write('.') + self.stream.write(f"{t.passed}.{t.reset}") self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) + t = self._theme if self.showAll: - self._write_status(test, "ERROR") + self._write_status(test, f"{t.fail}ERROR{t.reset}") elif self.dots: - self.stream.write('E') + self.stream.write(f"{t.fail}E{t.reset}") self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) + t = self._theme if self.showAll: - self._write_status(test, "FAIL") + self._write_status(test, f"{t.fail}FAIL{t.reset}") elif self.dots: - self.stream.write('F') + self.stream.write(f"{t.fail}F{t.reset}") self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) + t = self._theme if self.showAll: - self._write_status(test, "skipped {0!r}".format(reason)) + self._write_status(test, f"{t.warn}skipped{t.reset} {reason!r}") elif self.dots: - self.stream.write("s") + self.stream.write(f"{t.warn}s{t.reset}") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) + t = self._theme if self.showAll: - self.stream.writeln("expected failure") + self.stream.writeln(f"{t.warn}expected failure{t.reset}") self.stream.flush() elif self.dots: - self.stream.write("x") + self.stream.write(f"{t.warn}x{t.reset}") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) + t = self._theme if self.showAll: - self.stream.writeln("unexpected success") + self.stream.writeln(f"{t.fail}unexpected success{t.reset}") self.stream.flush() elif self.dots: - self.stream.write("u") + self.stream.write(f"{t.fail}u{t.reset}") self.stream.flush() def printErrors(self): + t = self._theme if self.dots or self.showAll: self.stream.writeln() self.stream.flush() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ()) + self.printErrorList(f"{t.fail}ERROR{t.reset}", self.errors) + self.printErrorList(f"{t.fail}FAIL{t.reset}", self.failures) + unexpectedSuccesses = getattr(self, "unexpectedSuccesses", ()) if unexpectedSuccesses: self.stream.writeln(self.separator1) for test in unexpectedSuccesses: - self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}") + self.stream.writeln( + f"{t.fail}UNEXPECTED SUCCESS{t.fail_info}: " + f"{self.getDescription(test)}{t.reset}" + ) self.stream.flush() def printErrorList(self, flavour, errors): + t = self._theme for test, err in errors: self.stream.writeln(self.separator1) - self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) + self.stream.writeln( + f"{flavour}{t.fail_info}: {self.getDescription(test)}{t.reset}" + ) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) self.stream.flush() @@ -232,7 +249,7 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - startTime = time.perf_counter() + start_time = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -242,8 +259,8 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.perf_counter() - timeTaken = stopTime - startTime + stop_time = time.perf_counter() + time_taken = stop_time - start_time result.printErrors() if self.durations is not None: self._printDurations(result) @@ -253,10 +270,10 @@ def run(self, test): run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) + (run, run != 1 and "s" or "", time_taken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expected_fails = unexpected_successes = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, @@ -264,26 +281,30 @@ def run(self, test): except AttributeError: pass else: - expectedFails, unexpectedSuccesses, skipped = results + expected_fails, unexpected_successes, skipped = results infos = [] + t = get_theme(tty_file=self.stream).unittest + if not result.wasSuccessful(): - self.stream.write("FAILED") + self.stream.write(f"{t.fail_info}FAILED{t.reset}") failed, errored = len(result.failures), len(result.errors) if failed: - infos.append("failures=%d" % failed) + infos.append(f"{t.fail_info}failures={failed}{t.reset}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"{t.fail_info}errors={errored}{t.reset}") elif run == 0 and not skipped: - self.stream.write("NO TESTS RAN") + self.stream.write(f"{t.warn}NO TESTS RAN{t.reset}") else: - self.stream.write("OK") + self.stream.write(f"{t.passed}OK{t.reset}") if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) + infos.append(f"{t.warn}skipped={skipped}{t.reset}") + if expected_fails: + infos.append(f"{t.warn}expected failures={expected_fails}{t.reset}") + if unexpected_successes: + infos.append( + f"{t.fail}unexpected successes={unexpected_successes}{t.reset}" + ) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: diff --git a/Lib/unittest/signals.py b/Lib/unittest/signals.py index e6a5fc52439..4e654c2c5db 100644 --- a/Lib/unittest/signals.py +++ b/Lib/unittest/signals.py @@ -1,6 +1,5 @@ import signal import weakref - from functools import wraps __unittest = True diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6..3c40176f070 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -2,8 +2,7 @@ import sys -from . import case -from . import util +from . import case, util __unittest = True diff --git a/Lib/unittest/util.py b/Lib/unittest/util.py index 050eaed0b3f..b81b6a4219b 100644 --- a/Lib/unittest/util.py +++ b/Lib/unittest/util.py @@ -1,6 +1,6 @@ """Various utility functions.""" -from collections import namedtuple, Counter +from collections import Counter, namedtuple from os.path import commonprefix __unittest = True diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8..a9cd1ecadd6 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.BytesIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index b35997bc00c..67d9bbea0d3 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -25,13 +25,19 @@ scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in test_urlparse.py provides a good indicator of parsing behavior. + +The WHATWG URL Parser spec should also be considered. We are not compliant with +it either due to existing user code API behavior expectations (Hyrum's Law). +It serves as a useful guide when making changes. """ +from collections import namedtuple +import functools +import math import re -import sys import types -import collections import warnings +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -46,18 +52,18 @@ uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', 'sftp', + 'prospero', 'rtsp', 'rtsps', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', - 'ws', 'wss'] + 'ws', 'wss', 'itms-services'] uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', - 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', 'sftp', 'tel'] + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', + 'sips', 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) @@ -66,7 +72,7 @@ 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + 'gopher', 'rtsp', 'rtsps', 'rtspu', 'sip', 'sips'] uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', @@ -78,18 +84,17 @@ '0123456789' '+-.') +# Leading and trailing C0 control and space to be stripped per WHATWG spec. +# == "".join([chr(i) for i in range(0, 0x20 + 1)]) +_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' + # Unsafe bytes to be removed per WHATWG spec _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] -# XXX: Consider replacing with functools.lru_cache -MAX_CACHE_SIZE = 20 -_parse_cache = {} - def clear_cache(): - """Clear the parse cache and the quoters cache.""" - _parse_cache.clear() - _safe_quoters.clear() - + """Clear internal performance caches. Undocumented; some tests want it.""" + urlsplit.cache_clear() + _byte_quoter_factory.cache_clear() # Helpers for bytes handling # For 3.2, we deliberately require applications that @@ -171,12 +176,11 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - try: - port = int(port, 10) - except ValueError: - message = f'Port could not be cast to integer value as {port!r}' - raise ValueError(message) from None - if not ( 0 <= port <= 65535): + if port.isdigit() and port.isascii(): + port = int(port) + else: + raise ValueError(f"Port could not be cast to integer value as {port!r}") + if not (0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port @@ -243,13 +247,11 @@ def _hostinfo(self): return hostname, port -from collections import namedtuple - -_DefragResultBase = namedtuple('DefragResult', 'url fragment') +_DefragResultBase = namedtuple('_DefragResultBase', 'url fragment') _SplitResultBase = namedtuple( - 'SplitResult', 'scheme netloc path query fragment') + '_SplitResultBase', 'scheme netloc path query fragment') _ParseResultBase = namedtuple( - 'ParseResult', 'scheme netloc path params query fragment') + '_ParseResultBase', 'scheme netloc path params query fragment') _DefragResultBase.__doc__ = """ DefragResult(url, fragment) @@ -390,20 +392,23 @@ def urlparse(url, scheme='', allow_fragments=True): Note that % escapes are not expanded. """ url, scheme, _coerce_result = _coerce_args(url, scheme) - splitresult = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) + scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) + result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') return _coerce_result(result) -def _splitparams(url): +def _urlparse(url, scheme=None, allow_fragments=True): + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if (scheme or '') in uses_params and ';' in url: + url, params = _splitparams(url, allow_none=True) + else: + params = None + return (scheme, netloc, url, params, query, fragment) + +def _splitparams(url, allow_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, '' + return url, None if allow_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -434,6 +439,37 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +def _check_bracketed_netloc(netloc): + # Note that this function must mirror the splitting + # done in NetlocResultMixins._hostinfo(). + hostname_and_port = netloc.rpartition('@')[2] + before_bracket, have_open_br, bracketed = hostname_and_port.partition('[') + if have_open_br: + # No data is allowed before a bracket. + if before_bracket: + raise ValueError("Invalid IPv6 URL") + hostname, _, port = bracketed.partition(']') + # No data is allowed after the bracket but before the port delimiter. + if port and not port.startswith(":"): + raise ValueError("Invalid IPv6 URL") + else: + hostname, _, port = hostname_and_port.partition(':') + _check_bracketed_host(hostname) + +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") + +# typed=True avoids BytesWarnings being emitted during cache key +# comparison since this API supports both bytes and str input. +@functools.lru_cache(typed=True) def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: <scheme>://<netloc>/<path>?<query>#<fragment> @@ -456,40 +492,43 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') + return _coerce_result(v) +def _urlsplit(url, scheme=None, allow_fragments=True): + # Only lstrip url as some applications rely on preserving trailing space. + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) for b in _UNSAFE_URL_BYTES_TO_REMOVE: url = url.replace(b, "") - scheme = scheme.replace(b, "") + if scheme is not None: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + scheme = scheme.replace(b, "") allow_fragments = bool(allow_fragments) - key = url, scheme, allow_fragments, type(url), type(scheme) - cached = _parse_cache.get(key, None) - if cached: - return _coerce_result(cached) - if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth - clear_cache() - netloc = query = fragment = '' + netloc = query = fragment = None i = url.find(':') - if i > 0: + if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: if c not in scheme_chars: break else: scheme, url = url[:i].lower(), url[i+1:] - if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + _check_bracketed_netloc(netloc) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) _checknetloc(netloc) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return _coerce_result(v) + return (scheme, netloc, url, query, fragment) def urlunparse(components): """Put a parsed URL back together again. This may result in a @@ -498,9 +537,15 @@ def urlunparse(components): (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None if params: url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) def urlunsplit(components): """Combine the elements of a tuple as returned by urlsplit() into a @@ -510,16 +555,27 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc is not None: if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url + url = '//' + netloc + url + elif url[:2] == '//': + url = '//' + url if scheme: url = scheme + ':' + url - if query: + if query is not None: url = url + '?' + query - if fragment: + if fragment is not None: url = url + '#' + fragment - return _coerce_result(url) + return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute @@ -530,26 +586,29 @@ def urljoin(base, url, allow_fragments=True): return base base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) - - if scheme != bscheme or scheme not in uses_relative: + bscheme, bnetloc, bpath, bquery, bfragment = \ + _urlsplit(base, None, allow_fragments) + scheme, netloc, path, query, fragment = \ + _urlsplit(url, None, allow_fragments) + + if scheme is None: + scheme = bscheme + if scheme != bscheme or (scheme and scheme not in uses_relative): return _coerce_result(url) - if scheme in uses_netloc: + if not scheme or scheme in uses_netloc: if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) netloc = bnetloc - if not path and not params: + if not path: path = bpath - params = bparams - if not query: + if query is None: query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + if fragment is None: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) base_parts = bpath.split('/') if base_parts[-1] != '': @@ -586,8 +645,8 @@ def urljoin(base, url, allow_fragments=True): # then we need to append the trailing '/' resolved_path.append('') - return _coerce_result(urlunparse((scheme, netloc, '/'.join( - resolved_path) or '/', params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( + resolved_path) or '/', query, fragment)) def urldefrag(url): @@ -599,18 +658,21 @@ def urldefrag(url): """ url, _coerce_result = _coerce_args(url) if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) + s, n, p, q, frag = _urlsplit(url) + defrag = _urlunsplit(s, n, p, q, None) else: frag = '' defrag = url - return _coerce_result(DefragResult(defrag, frag)) + return _coerce_result(DefragResult(defrag, frag or '')) _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None def unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" + return bytes(_unquote_impl(string)) + +def _unquote_impl(string: bytes | bytearray | str) -> bytes | bytearray: # Note: strings are encoded as UTF-8. This is only an issue if it contains # unescaped non-ASCII characters, which URIs should not. if not string: @@ -622,8 +684,8 @@ def unquote_to_bytes(string): bits = string.split(b'%') if len(bits) == 1: return string - res = [bits[0]] - append = res.append + res = bytearray(bits[0]) + append = res.extend # Delay the initialization of the table to not waste memory # if the function is never called global _hextobyte @@ -637,10 +699,20 @@ def unquote_to_bytes(string): except KeyError: append(b'%') append(item) - return b''.join(res) + return res _asciire = re.compile('([\x00-\x7f]+)') +def _generate_unquoted_parts(string, encoding, errors): + previous_match_end = 0 + for ascii_match in _asciire.finditer(string): + start, end = ascii_match.span() + yield string[previous_match_end:start] # Non-ASCII + # The ascii_match[1] group == string[start:end]. + yield _unquote_impl(ascii_match[1]).decode(encoding, errors) + previous_match_end = end + yield string[previous_match_end:] # Non-ASCII tail + def unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded @@ -652,21 +724,16 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ if isinstance(string, bytes): - return unquote_to_bytes(string).decode(encoding, errors) + return _unquote_impl(string).decode(encoding, errors) if '%' not in string: + # Is it a string-like object? string.split return string if encoding is None: encoding = 'utf-8' if errors is None: errors = 'replace' - bits = _asciire.split(string) - res = [bits[0]] - append = res.append - for i in range(1, len(bits), 2): - append(unquote_to_bytes(bits[i]).decode(encoding, errors)) - append(bits[i + 1]) - return ''.join(res) + return ''.join(_generate_unquoted_parts(string, encoding, errors)) def parse_qs(qs, keep_blank_values=False, strict_parsing=False, @@ -702,7 +769,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, - max_num_fields=max_num_fields, separator=separator) + max_num_fields=max_num_fields, separator=separator, + _stacklevel=2) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -712,7 +780,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&', *, _stacklevel=1): """Parse a query given as a string argument. Arguments: @@ -740,11 +808,37 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - qs, _coerce_result = _coerce_args(qs) - separator, _ = _coerce_args(separator) - - if not separator or (not isinstance(separator, (str, bytes))): + if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") + if isinstance(qs, str): + if not isinstance(separator, str): + separator = str(separator, 'ascii') + eq = '=' + def _unquote(s): + return unquote_plus(s, encoding=encoding, errors=errors) + elif qs is None: + return [] + else: + try: + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + except TypeError: + if not qs: + warnings.warn(f"Accepting {type(qs).__name__} objects with " + f"false value in urllib.parse.parse_qsl() is " + f"deprecated as of 3.14", + DeprecationWarning, stacklevel=_stacklevel + 1) + return [] + raise + if isinstance(separator, str): + separator = bytes(separator, 'ascii') + eq = b'=' + def _unquote(s): + return unquote_to_bytes(s.replace(b'+', b' ')) + + if not qs: + return [] # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS @@ -756,25 +850,14 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, r = [] for name_value in qs.split(separator): - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: + if name_value or strict_parsing: + name, has_eq, value = name_value.partition(eq) + if not has_eq and strict_parsing: raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) + if value or keep_blank_values: + name = _unquote(name) + value = _unquote(value) + r.append((name, value)) return r def unquote_plus(string, encoding='utf-8', errors='replace'): @@ -791,23 +874,22 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'0123456789' b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -_safe_quoters = {} -class Quoter(collections.defaultdict): - """A mapping from bytes (in range(0,256)) to strings. + +class _Quoter(dict): + """A mapping from bytes numbers (in range(0,256)) to strings. String values are percent-encoded byte values, unless the key < 128, and - in the "safe" set (either the specified safe set, or default set). + in either of the specified safe set, or the always safe set. """ - # Keeps a cache internally, using defaultdict, for efficiency (lookups + # Keeps a cache internally, via __missing__, for efficiency (lookups # of cached keys don't call Python code at all). def __init__(self, safe): """safe: bytes object.""" self.safe = _ALWAYS_SAFE.union(safe) def __repr__(self): - # Without this, will just display as a defaultdict - return "<%s %r>" % (self.__class__.__name__, dict(self)) + return f"<Quoter {dict(self)!r}>" def __missing__(self, b): # Handle a cache miss. Store quoted string in cache and return. @@ -886,6 +968,11 @@ def quote_plus(string, safe='', encoding=None, errors=None): string = quote(string, safe + space, encoding, errors) return string.replace(' ', '+') +# Expectation: A typical program is unlikely to create more than 5 of these. +@functools.lru_cache +def _byte_quoter_factory(safe): + return _Quoter(safe).__getitem__ + def quote_from_bytes(bs, safe='/'): """Like quote(), but accepts a bytes object rather than a str, and does not perform string-to-bytes encoding. It always returns an ASCII string. @@ -899,14 +986,19 @@ def quote_from_bytes(bs, safe='/'): # Normalize 'safe' by converting to bytes and removing non-ASCII chars safe = safe.encode('ascii', 'ignore') else: + # List comprehensions are faster than generator expressions. safe = bytes([c for c in safe if c < 128]) if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe): return bs.decode() - try: - quoter = _safe_quoters[safe] - except KeyError: - _safe_quoters[safe] = quoter = Quoter(safe).__getitem__ - return ''.join([quoter(char) for char in bs]) + quoter = _byte_quoter_factory(safe) + if (bs_len := len(bs)) < 200_000: + return ''.join(map(quoter, bs)) + else: + # This saves memory - https://github.com/python/cpython/issues/95865 + chunk_size = math.isqrt(bs_len) + chunks = [''.join(map(quoter, bs[i:i+chunk_size])) + for i in range(0, bs_len, chunk_size)] + return ''.join(chunks) def urlencode(query, doseq=False, safe='', encoding=None, errors=None, quote_via=quote_plus): @@ -939,10 +1031,9 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None, # but that's a minor nit. Since the original implementation # allowed empty dicts that type of behavior probably should be # preserved for consistency - except TypeError: - ty, va, tb = sys.exc_info() + except TypeError as err: raise TypeError("not a valid non-string sequence " - "or mapping object").with_traceback(tb) + "or mapping object") from err l = [] if not doseq: @@ -1125,15 +1216,15 @@ def splitnport(host, defport=-1): def _splitnport(host, defport=-1): """Split host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. - Return numerical port if a valid number are found after ':'. + Return numerical port if a valid number is found after ':'. Return None if ':' but not a valid number.""" host, delim, port = host.rpartition(':') if not delim: host = port elif port: - try: + if port.isdigit() and port.isascii(): nport = int(port) - except ValueError: + else: nport = None return host, nport return host, defport diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a0ef60b30de..8d7470a2273 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -11,8 +11,8 @@ Handlers needed to open the requested URL. For example, the HTTPHandler performs HTTP GET and POST requests and deals with non-error returns. The HTTPRedirectHandler automatically deals with -HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler -deals with digest authentication. +HTTP 301, 302, 303, 307, and 308 redirect errors, and the +HTTPDigestAuthHandler deals with digest authentication. urlopen(url, data=None) -- Basic usage is the same as original urllib. pass the url and optionally data to post to an HTTP URL, and @@ -83,33 +83,31 @@ import base64 import bisect +import contextlib import email import hashlib import http.client import io import os -import posixpath import re import socket import string import sys import time import tempfile -import contextlib -import warnings from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, _splittype, _splithost, _splitport, _splituser, _splitpasswd, - _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes, + _splitattr, _splitvalue, _splittag, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: @@ -129,7 +127,7 @@ 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', # Legacy interface - 'urlretrieve', 'urlcleanup', 'URLopener', 'FancyURLopener', + 'urlretrieve', 'urlcleanup', ] # used in User-Agent header sent @@ -137,7 +135,7 @@ _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - *, cafile=None, capath=None, cadefault=False, context=None): + *, context=None): '''Open the URL url, which can be either a string or a Request object. *data* must be an object specifying additional data to be sent to @@ -155,14 +153,6 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, If *context* is specified, it must be a ssl.SSLContext instance describing the various SSL options. See HTTPSConnection for more details. - The optional *cafile* and *capath* parameters specify a set of trusted CA - certificates for HTTPS requests. cafile should point to a single file - containing a bundle of CA certificates, whereas capath should point to a - directory of hashed certificate files. More information can be found in - ssl.SSLContext.load_verify_locations(). - - The *cadefault* parameter is ignored. - This function always returns an object which can work as a context manager and has the properties url, headers, and status. @@ -174,8 +164,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, the reason phrase returned by the server --- instead of the response headers as it is specified in the documentation for HTTPResponse. - For FTP, file, and data URLs and requests explicitly handled by legacy - URLopener and FancyURLopener classes, this function returns a + For FTP, file, and data URLs, this function returns a urllib.response.addinfourl object. Note that None may be returned if no handler handles the request (though @@ -188,25 +177,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ''' global _opener - if cafile or capath or cadefault: - import warnings - warnings.warn("cafile, capath and cadefault are deprecated, use a " - "custom context instead.", DeprecationWarning, 2) - if context is not None: - raise ValueError( - "You can't pass both context and any of cafile, capath, and " - "cadefault" - ) - if not _have_ssl: - raise ValueError('SSL support not available') - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, - cafile=cafile, - capath=capath) - # send ALPN extension to indicate HTTP/1.1 protocol - context.set_alpn_protocols(['http/1.1']) - https_handler = HTTPSHandler(context=context) - opener = build_opener(https_handler) - elif context: + if context: https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif _opener is None: @@ -266,10 +237,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None): if reporthook: reporthook(blocknum, bs, size) - while True: - block = fp.read(bs) - if not block: - break + while block := fp.read(bs): read += len(block) tfp.write(block) blocknum += 1 @@ -661,7 +629,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): but another Handler might. """ m = req.get_method() - if (not (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD") or code in (301, 302, 303) and m == "POST")): raise HTTPError(req.full_url, code, msg, headers, fp) @@ -680,6 +648,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) @@ -748,7 +717,7 @@ def http_error_302(self, req, fp, code, msg, headers): return self.parent.open(new, timeout=req.timeout) - http_error_301 = http_error_303 = http_error_307 = http_error_302 + http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302 inf_msg = "The HTTP server returned a redirect error that would " \ "lead to an infinite loop.\n" \ @@ -907,9 +876,9 @@ def find_user_password(self, realm, authuri): class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): - def __init__(self, *args, **kwargs): + def __init__(self): self.authenticated = {} - super().__init__(*args, **kwargs) + super().__init__() def add_password(self, realm, uri, user, passwd, is_authenticated=False): self.update_authenticated(uri, is_authenticated) @@ -969,6 +938,7 @@ def _parse_realm(self, header): for mo in AbstractBasicAuthHandler.rx.finditer(header): scheme, quote, realm = mo.groups() if quote not in ['"', "'"]: + import warnings warnings.warn("Basic Auth Realm was unquoted", UserWarning, 3) @@ -1078,7 +1048,7 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractDigestAuthHandler: - # Digest authentication is specified in RFC 2617. + # Digest authentication is specified in RFC 2617/7616. # XXX The client does not inspect the Authentication-Info header # in a successful response. @@ -1206,11 +1176,14 @@ def get_authorization(self, req, chal): return base def get_algorithm_impls(self, algorithm): + # algorithm names taken from RFC 7616 Section 6.1 # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest() - elif algorithm == 'SHA': + elif algorithm == 'SHA': # non-standard, retained for compatibility. H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest() + elif algorithm == 'SHA-256': + H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest() # XXX MD5-sess else: raise ValueError("Unsupported digest authentication " @@ -1255,8 +1228,8 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractHTTPHandler(BaseHandler): - def __init__(self, debuglevel=0): - self._debuglevel = debuglevel + def __init__(self, debuglevel=None): + self._debuglevel = debuglevel if debuglevel is not None else http.client.HTTPConnection.debuglevel def set_http_debuglevel(self, level): self._debuglevel = level @@ -1382,14 +1355,19 @@ def http_open(self, req): class HTTPSHandler(AbstractHTTPHandler): - def __init__(self, debuglevel=0, context=None, check_hostname=None): + def __init__(self, debuglevel=None, context=None, check_hostname=None): + debuglevel = debuglevel if debuglevel is not None else http.client.HTTPSConnection.debuglevel AbstractHTTPHandler.__init__(self, debuglevel) + if context is None: + http_version = http.client.HTTPSConnection._http_vsn + context = http.client._create_https_context(http_version) + if check_hostname is not None: + context.check_hostname = check_hostname self._context = context - self._check_hostname = check_hostname def https_open(self, req): return self.do_open(http.client.HTTPSConnection, req, - context=self._context, check_hostname=self._check_hostname) + context=self._context) https_request = AbstractHTTPHandler.do_request_ @@ -1472,16 +1450,6 @@ def parse_http_list(s): return [part.strip() for part in res] class FileHandler(BaseHandler): - # Use local file or FTP depending on form of URL - def file_open(self, req): - url = req.selector - if url[:2] == '//' and url[2:3] != '/' and (req.host and - req.host != 'localhost'): - if not req.host in self.get_names(): - raise URLError("file:// scheme is supported only on localhost") - else: - return self.open_local_file(req) - # names for the localhost names = None def get_names(self): @@ -1498,35 +1466,41 @@ def get_names(self): def open_local_file(self, req): import email.utils import mimetypes - host = req.host - filename = req.selector - localfile = url2pathname(filename) + localfile = url2pathname(req.full_url, require_scheme=True, resolve_host=True) try: stats = os.stat(localfile) size = stats.st_size modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - mtype = mimetypes.guess_type(filename)[0] + mtype = mimetypes.guess_file_type(localfile)[0] headers = email.message_from_string( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified)) - if host: - host, port = _splitport(host) - if not host or \ - (not port and _safe_gethostbyname(host) in self.get_names()): - if host: - origurl = 'file://' + host + filename - else: - origurl = 'file://' + filename - return addinfourl(open(localfile, 'rb'), headers, origurl) + origurl = pathname2url(localfile, add_scheme=True) + return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - raise URLError(exp) - raise URLError('file not on local host') + raise URLError(exp, exp.filename) + + file_open = open_local_file -def _safe_gethostbyname(host): +def _is_local_authority(authority, resolve): + # Compare hostnames + if not authority or authority == 'localhost': + return True try: - return socket.gethostbyname(host) - except socket.gaierror: - return None + hostname = socket.gethostname() + except (socket.gaierror, AttributeError): + pass + else: + if authority == hostname: + return True + # Compare IP addresses + if not resolve: + return False + try: + address = socket.gethostbyname(authority) + except (socket.gaierror, AttributeError, UnicodeEncodeError): + return False + return address in FileHandler().get_names() class FTPHandler(BaseHandler): def ftp_open(self, req): @@ -1561,6 +1535,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1578,9 +1553,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - exc = URLError('ftp error: %r' % exp) - raise exc.with_traceback(sys.exc_info()[2]) + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1604,14 +1582,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones @@ -1655,6 +1634,11 @@ def data_open(self, req): scheme, data = url.split(":",1) mediatype, data = data.split(",",1) + # Disallow control characters within mediatype. + if re.search(r"[\x00-\x1F\x7F]", mediatype): + raise ValueError( + "Control characters not allowed in data: mediatype") + # even base64 encoded data URLs might be quoted so unquote in any case: data = unquote_to_bytes(data) if mediatype.endswith(";base64"): @@ -1670,683 +1654,80 @@ def data_open(self, req): return addinfourl(io.BytesIO(data), headers, url) -# Code move from the old urllib module - -MAXFTPCACHE = 10 # Trim the ftp cache beyond this size - -# Helper for non-unix systems -if os.name == 'nt': - from nturl2path import url2pathname, pathname2url -else: - def url2pathname(pathname): - """OS-specific conversion from a relative URL of the 'file' scheme - to a file system path; not recommended for general use.""" - return unquote(pathname) - - def pathname2url(pathname): - """OS-specific conversion from a file system path to a relative URL - of the 'file' scheme; not recommended for general use.""" - return quote(pathname) - - -ftpcache = {} - +# Code moved from the old urllib module -class URLopener: - """Class to open URLs. - This is a class rather than just a subroutine because we may need - more than one set of global protocol-specific options. - Note -- this is a base class for those who don't want the - automatic handling of errors type 302 (relocated) and 401 - (authorization needed).""" +def url2pathname(url, *, require_scheme=False, resolve_host=False): + """Convert the given file URL to a local file system path. - __tempfiles = None + The 'file:' scheme prefix must be omitted unless *require_scheme* + is set to true. - version = "Python-urllib/%s" % __version__ - - # Constructor - def __init__(self, proxies=None, **x509): - msg = "%(class)s style of invoking requests is deprecated. " \ - "Use newer urlopen functions/methods" % {'class': self.__class__.__name__} - warnings.warn(msg, DeprecationWarning, stacklevel=3) - if proxies is None: - proxies = getproxies() - assert hasattr(proxies, 'keys'), "proxies must be a mapping" - self.proxies = proxies - self.key_file = x509.get('key_file') - self.cert_file = x509.get('cert_file') - self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] - self.__tempfiles = [] - self.__unlink = os.unlink # See cleanup() - self.tempcache = None - # Undocumented feature: if you assign {} to tempcache, - # it is used to cache files retrieved with - # self.retrieve(). This is not enabled by default - # since it does not work for changing documents (and I - # haven't got the logic to check expiration headers - # yet). - self.ftpcache = ftpcache - # Undocumented feature: you can use a different - # ftp cache by assigning to the .ftpcache member; - # in case you want logically independent URL openers - # XXX This is not threadsafe. Bah. - - def __del__(self): - self.close() - - def close(self): - self.cleanup() - - def cleanup(self): - # This code sometimes runs when the rest of this module - # has already been deleted, so it can't use any globals - # or import anything. - if self.__tempfiles: - for file in self.__tempfiles: - try: - self.__unlink(file) - except OSError: - pass - del self.__tempfiles[:] - if self.tempcache: - self.tempcache.clear() - - def addheader(self, *args): - """Add a header to be used by the HTTP interface only - e.g. u.addheader('Accept', 'sound/basic')""" - self.addheaders.append(args) - - # External interface - def open(self, fullurl, data=None): - """Use URLopener().open(file) instead of open(file, 'r').""" - fullurl = unwrap(_to_bytes(fullurl)) - fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|") - if self.tempcache and fullurl in self.tempcache: - filename, headers = self.tempcache[fullurl] - fp = open(filename, 'rb') - return addinfourl(fp, headers, fullurl) - urltype, url = _splittype(fullurl) - if not urltype: - urltype = 'file' - if urltype in self.proxies: - proxy = self.proxies[urltype] - urltype, proxyhost = _splittype(proxy) - host, selector = _splithost(proxyhost) - url = (host, fullurl) # Signal special case to open_*() - else: - proxy = None - name = 'open_' + urltype - self.type = urltype - name = name.replace('-', '_') - if not hasattr(self, name) or name == 'open_local_file': - if proxy: - return self.open_unknown_proxy(proxy, fullurl, data) - else: - return self.open_unknown(fullurl, data) - try: - if data is None: - return getattr(self, name)(url) - else: - return getattr(self, name)(url, data) - except (HTTPError, URLError): - raise - except OSError as msg: - raise OSError('socket error', msg).with_traceback(sys.exc_info()[2]) - - def open_unknown(self, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'unknown url type', type) - - def open_unknown_proxy(self, proxy, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'invalid proxy for %s' % type, proxy) - - # External interface - def retrieve(self, url, filename=None, reporthook=None, data=None): - """retrieve(url) returns (filename, headers) for a local object - or (tempfilename, headers) for a remote object.""" - url = unwrap(_to_bytes(url)) - if self.tempcache and url in self.tempcache: - return self.tempcache[url] - type, url1 = _splittype(url) - if filename is None and (not type or type == 'file'): - try: - fp = self.open_local_file(url1) - hdrs = fp.info() - fp.close() - return url2pathname(_splithost(url1)[1]), hdrs - except OSError: - pass - fp = self.open(url, data) - try: - headers = fp.info() - if filename: - tfp = open(filename, 'wb') - else: - garbage, path = _splittype(url) - garbage, path = _splithost(path or "") - path, garbage = _splitquery(path or "") - path, garbage = _splitattr(path or "") - suffix = os.path.splitext(path)[1] - (fd, filename) = tempfile.mkstemp(suffix) - self.__tempfiles.append(filename) - tfp = os.fdopen(fd, 'wb') - try: - result = filename, headers - if self.tempcache is not None: - self.tempcache[url] = result - bs = 1024*8 - size = -1 - read = 0 - blocknum = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - if reporthook: - reporthook(blocknum, bs, size) - while 1: - block = fp.read(bs) - if not block: - break - read += len(block) - tfp.write(block) - blocknum += 1 - if reporthook: - reporthook(blocknum, bs, size) - finally: - tfp.close() - finally: - fp.close() - - # raise exception if actual size does not match content-length header - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size), result) - - return result - - # Each method named open_<type> knows how to open that type of URL - - def _open_generic_http(self, connection_factory, url, data): - """Make an HTTP connection using connection_class. - - This is an internal method that should be called from - open_http() or open_https(). - - Arguments: - - connection_factory should take a host name and return an - HTTPConnection instance. - - url is the url to retrieval or a host, relative-path pair. - - data is payload for a POST request or None. - """ - - user_passwd = None - proxy_passwd= None - if isinstance(url, str): - host, selector = _splithost(url) - if host: - user_passwd, host = _splituser(host) - host = unquote(host) - realhost = host - else: - host, selector = url - # check whether the proxy contains authorization information - proxy_passwd, host = _splituser(host) - # now we proceed with the url we want to obtain - urltype, rest = _splittype(selector) - url = rest - user_passwd = None - if urltype.lower() != 'http': - realhost = None - else: - realhost, rest = _splithost(rest) - if realhost: - user_passwd, realhost = _splituser(realhost) - if user_passwd: - selector = "%s://%s%s" % (urltype, realhost, rest) - if proxy_bypass(realhost): - host = realhost - - if not host: raise OSError('http error', 'no host given') - - if proxy_passwd: - proxy_passwd = unquote(proxy_passwd) - proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('ascii') - else: - proxy_auth = None - - if user_passwd: - user_passwd = unquote(user_passwd) - auth = base64.b64encode(user_passwd.encode()).decode('ascii') - else: - auth = None - http_conn = connection_factory(host) - headers = {} - if proxy_auth: - headers["Proxy-Authorization"] = "Basic %s" % proxy_auth - if auth: - headers["Authorization"] = "Basic %s" % auth - if realhost: - headers["Host"] = realhost - - # Add Connection:close as we don't support persistent connections yet. - # This helps in closing the socket and avoiding ResourceWarning - - headers["Connection"] = "close" - - for header, value in self.addheaders: - headers[header] = value - - if data is not None: - headers["Content-Type"] = "application/x-www-form-urlencoded" - http_conn.request("POST", selector, data, headers) - else: - http_conn.request("GET", selector, headers=headers) - - try: - response = http_conn.getresponse() - except http.client.BadStatusLine: - # something went wrong with the HTTP status line - raise URLError("http protocol error: bad status line") - - # According to RFC 2616, "2xx" code indicates that the client's - # request was successfully received, understood, and accepted. - if 200 <= response.status < 300: - return addinfourl(response, response.msg, "http:" + url, - response.status) - else: - return self.http_error( - url, response.fp, - response.status, response.reason, response.msg, data) - - def open_http(self, url, data=None): - """Use HTTP protocol.""" - return self._open_generic_http(http.client.HTTPConnection, url, data) - - def http_error(self, url, fp, errcode, errmsg, headers, data=None): - """Handle http errors. - - Derived class can override this, or provide specific handlers - named http_error_DDD where DDD is the 3-digit error code.""" - # First check if there's a specific handler for this error - name = 'http_error_%d' % errcode - if hasattr(self, name): - method = getattr(self, name) - if data is None: - result = method(url, fp, errcode, errmsg, headers) - else: - result = method(url, fp, errcode, errmsg, headers, data) - if result: return result - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handler: close the connection and raise OSError.""" - fp.close() - raise HTTPError(url, errcode, errmsg, headers, None) - - if _have_ssl: - def _https_connection(self, host): - return http.client.HTTPSConnection(host, - key_file=self.key_file, - cert_file=self.cert_file) - - def open_https(self, url, data=None): - """Use HTTPS protocol.""" - return self._open_generic_http(self._https_connection, url, data) - - def open_file(self, url): - """Use local file or FTP depending on form of URL.""" - if not isinstance(url, str): - raise URLError('file error: proxy support for file protocol currently not implemented') - if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/': - raise ValueError("file:// scheme is supported only on localhost") - else: - return self.open_local_file(url) - - def open_local_file(self, url): - """Use local file.""" - import email.utils - import mimetypes - host, file = _splithost(url) - localname = url2pathname(file) - try: - stats = os.stat(localname) - except OSError as e: - raise URLError(e.strerror, e.filename) - size = stats.st_size - modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - mtype = mimetypes.guess_type(url)[0] - headers = email.message_from_string( - 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' % - (mtype or 'text/plain', size, modified)) - if not host: - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - return addinfourl(open(localname, 'rb'), headers, urlfile) - host, port = _splitport(host) - if (not port - and socket.gethostbyname(host) in ((localhost(),) + thishost())): - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - elif file[:2] == './': - raise ValueError("local file url may start with / or file:. Unknown url of type: %s" % url) - return addinfourl(open(localname, 'rb'), headers, urlfile) - raise URLError('local file error: not on local host') - - def open_ftp(self, url): - """Use FTP protocol.""" - if not isinstance(url, str): - raise URLError('ftp error: proxy support for ftp protocol currently not implemented') - import mimetypes - host, path = _splithost(url) - if not host: raise URLError('ftp error: no host given') - host, port = _splitport(host) - user, host = _splituser(host) - if user: user, passwd = _splitpasswd(user) - else: passwd = None - host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') - host = socket.gethostbyname(host) - if not port: - import ftplib - port = ftplib.FTP_PORT - else: - port = int(port) - path, attrs = _splitattr(path) - path = unquote(path) - dirs = path.split('/') - dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: dirs = dirs[1:] - if dirs and not dirs[0]: dirs[0] = '/' - key = user, host, port, '/'.join(dirs) - # XXX thread unsafe! - if len(self.ftpcache) > MAXFTPCACHE: - # Prune the cache, rather arbitrarily - for k in list(self.ftpcache): - if k != key: - v = self.ftpcache[k] - del self.ftpcache[k] - v.close() - try: - if key not in self.ftpcache: - self.ftpcache[key] = \ - ftpwrapper(user, passwd, host, port, dirs) - if not file: type = 'D' - else: type = 'I' - for attr in attrs: - attr, value = _splitvalue(attr) - if attr.lower() == 'type' and \ - value in ('a', 'A', 'i', 'I', 'd', 'D'): - type = value.upper() - (fp, retrlen) = self.ftpcache[key].retrfile(file, type) - mtype = mimetypes.guess_type("ftp:" + url)[0] - headers = "" - if mtype: - headers += "Content-Type: %s\n" % mtype - if retrlen is not None and retrlen >= 0: - headers += "Content-Length: %d\n" % retrlen - headers = email.message_from_string(headers) - return addinfourl(fp, headers, "ftp:" + url) - except ftperrors() as exp: - raise URLError('ftp error %r' % exp).with_traceback(sys.exc_info()[2]) - - def open_data(self, url, data=None): - """Use "data" URL.""" - if not isinstance(url, str): - raise URLError('data error: proxy support for data protocol currently not implemented') - # ignore POSTed data - # - # syntax of data URLs: - # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data - # mediatype := [ type "/" subtype ] *( ";" parameter ) - # data := *urlchar - # parameter := attribute "=" value - try: - [type, data] = url.split(',', 1) - except ValueError: - raise OSError('data error', 'bad data URL') - if not type: - type = 'text/plain;charset=US-ASCII' - semi = type.rfind(';') - if semi >= 0 and '=' not in type[semi:]: - encoding = type[semi+1:] - type = type[:semi] - else: - encoding = '' - msg = [] - msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT', - time.gmtime(time.time()))) - msg.append('Content-type: %s' % type) - if encoding == 'base64': - # XXX is this encoding/decoding ok? - data = base64.decodebytes(data.encode('ascii')).decode('latin-1') - else: - data = unquote(data) - msg.append('Content-Length: %d' % len(data)) - msg.append('') - msg.append(data) - msg = '\n'.join(msg) - headers = email.message_from_string(msg) - f = io.StringIO(msg) - #f.fileno = None # needed for addinfourl - return addinfourl(f, headers, url) - - -class FancyURLopener(URLopener): - """Derived class with handlers for errors we can handle (perhaps).""" - - def __init__(self, *args, **kwargs): - URLopener.__init__(self, *args, **kwargs) - self.auth_cache = {} - self.tries = 0 - self.maxtries = 10 - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handling -- don't raise an exception.""" - return addinfourl(fp, headers, "http:" + url, errcode) - - def http_error_302(self, url, fp, errcode, errmsg, headers, data=None): - """Error 302 -- relocated (temporarily).""" - self.tries += 1 - try: - if self.maxtries and self.tries >= self.maxtries: - if hasattr(self, "http_error_500"): - meth = self.http_error_500 - else: - meth = self.http_error_default - return meth(url, fp, 500, - "Internal Server Error: Redirect Recursion", - headers) - result = self.redirect_internal(url, fp, errcode, errmsg, - headers, data) - return result - finally: - self.tries = 0 - - def redirect_internal(self, url, fp, errcode, errmsg, headers, data): - if 'location' in headers: - newurl = headers['location'] - elif 'uri' in headers: - newurl = headers['uri'] - else: - return - fp.close() - - # In case the server sent a relative URL, join with original: - newurl = urljoin(self.type + ":" + url, newurl) - - urlparts = urlparse(newurl) - - # For security reasons, we don't allow redirection to anything other - # than http, https and ftp. - - # We are using newer HTTPError with older redirect_internal method - # This older method will get deprecated in 3.3 - - if urlparts.scheme not in ('http', 'https', 'ftp', ''): - raise HTTPError(newurl, errcode, - errmsg + - " Redirection to url '%s' is not allowed." % newurl, - headers, fp) - - return self.open(newurl) - - def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): - """Error 301 -- also relocated (permanently).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_303(self, url, fp, errcode, errmsg, headers, data=None): - """Error 303 -- also relocated (essentially identical to 302).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_307(self, url, fp, errcode, errmsg, headers, data=None): - """Error 307 -- relocated, but turn POST into error.""" - if data is None: - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - else: - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_401(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 401 -- authentication required. - This function supports Basic authentication only.""" - if 'www-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['www-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def http_error_407(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 407 -- proxy authentication required. - This function supports Basic authentication only.""" - if 'proxy-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['proxy-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_proxy_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def retry_proxy_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'http://' + host + selector - proxy = self.proxies['http'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['http'] = 'http://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_proxy_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'https://' + host + selector - proxy = self.proxies['https'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['https'] = 'https://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'http://' + host + selector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'https://' + host + selector - if data is None: - return self.open(newurl) + The URL authority may be resolved with gethostbyname() if + *resolve_host* is set to true. + """ + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") + if os.name == 'nt': + if authority[1:2] == ':': + # e.g. file://c:/file.txt + url = authority + url + elif not _is_local_authority(authority, resolve_host): + # e.g. file://server/share/file.txt + url = '//' + authority + url + elif url[:3] == '///': + # e.g. file://///server/share/file.txt + url = url[1:] else: - return self.open(newurl, data) - - def get_user_passwd(self, host, realm, clear_cache=0): - key = realm + '@' + host.lower() - if key in self.auth_cache: - if clear_cache: - del self.auth_cache[key] - else: - return self.auth_cache[key] - user, passwd = self.prompt_user_passwd(host, realm) - if user or passwd: self.auth_cache[key] = (user, passwd) - return user, passwd - - def prompt_user_passwd(self, host, realm): - """Override this in a GUI environment!""" - import getpass - try: - user = input("Enter username for %s at %s: " % (realm, host)) - passwd = getpass.getpass("Enter password for %s in %s at %s: " % - (user, realm, host)) - return user, passwd - except KeyboardInterrupt: - print() - return None, None + if url[:1] == '/' and url[2:3] in (':', '|'): + # Skip past extra slash before DOS drive in URL path. + url = url[1:] + if url[1:2] == '|': + # Older URLs use a pipe after a drive letter + url = url[:1] + ':' + url[2:] + url = url.replace('/', '\\') + elif not _is_local_authority(authority, resolve_host): + raise URLError("file:// scheme is supported only on localhost") + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + return unquote(url, encoding=encoding, errors=errors) + + +def pathname2url(pathname, *, add_scheme=False): + """Convert the given local file system path to a file URL. + + The 'file:' scheme prefix is omitted unless *add_scheme* + is set to true. + """ + if os.name == 'nt': + pathname = pathname.replace('\\', '/') + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + scheme = 'file:' if add_scheme else '' + drive, root, tail = os.path.splitroot(pathname) + if drive: + # First, clean up some special forms. We are going to sacrifice the + # additional information anyway + if drive[:4] == '//?/': + drive = drive[4:] + if drive[:4].upper() == 'UNC/': + drive = '//' + drive[4:] + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = '///' + drive + drive = quote(drive, encoding=encoding, errors=errors, safe='/:') + elif root: + # Add explicitly empty authority to absolute path. If the path + # starts with exactly one slash then this change is mostly + # cosmetic, but if it begins with two or more slashes then this + # avoids interpreting the path as a URL authority. + root = '//' + root + tail = quote(tail, encoding=encoding, errors=errors) + return scheme + drive + root + tail # Utility functions @@ -2436,8 +1817,7 @@ def retrfile(self, file, type): conn, retrlen = self.ftp.ntransfercmd(cmd) except ftplib.error_perm as reason: if str(reason)[:3] != '550': - raise URLError('ftp error: %r' % reason).with_traceback( - sys.exc_info()[2]) + raise URLError(f'ftp error: {reason}') from reason if not conn: # Set transfer mode to ASCII! self.ftp.voidcmd('TYPE A') @@ -2464,7 +1844,13 @@ def retrfile(self, file, type): return (ftpobj, retrlen) def endtransfer(self): + if not self.busy: + return self.busy = 0 + try: + self.ftp.voidresp() + except ftperrors(): + pass def close(self): self.keepalive = False @@ -2489,31 +1875,35 @@ def getproxies_environment(): """Return a dictionary of scheme -> proxy server URL mappings. Scan the environment for variables named <scheme>_proxy; - this seems to be the standard convention. If you need a - different way, you can pass a proxies dictionary to the - [Fancy]URLopener constructor. - + this seems to be the standard convention. """ - proxies = {} # in order to prefer lowercase variables, process environment in # two passes: first matches any, second pass matches lowercase only - for name, value in os.environ.items(): - name = name.lower() - if value and name[-6:] == '_proxy': - proxies[name[:-6]] = value + + # select only environment variables which end in (after making lowercase) _proxy + proxies = {} + environment = [] + for name in os.environ: + # fast screen underscore position before more expensive case-folding + if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy": + value = os.environ[name] + proxy_name = name[:-6].lower() + environment.append((name, value, proxy_name)) + if value: + proxies[proxy_name] = value # CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY # (non-all-lowercase) as it may be set from the web server by a "Proxy:" # header from the client # If "proxy" is lowercase, it will still be used thanks to the next block if 'REQUEST_METHOD' in os.environ: proxies.pop('http', None) - for name, value in os.environ.items(): + for name, value, proxy_name in environment: + # not case-folded, checking here for lower-case env vars only if name[-6:] == '_proxy': - name = name.lower() if value: - proxies[name[:-6]] = value + proxies[proxy_name] = value else: - proxies.pop(name[:-6], None) + proxies.pop(proxy_name, None) return proxies def proxy_bypass_environment(host, proxies=None): @@ -2566,6 +1956,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2582,20 +1973,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2618,6 +2006,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "<local>" should bypass the proxy server for all intranet addresses + if test == '<local>': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2716,7 +2129,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2726,40 +2139,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '<local>' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '<local>': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index c58565e3945..4009fd6b58f 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -11,6 +11,8 @@ """ import collections +import re +import urllib.error import urllib.parse import urllib.request @@ -19,6 +21,19 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_path(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -54,7 +69,7 @@ def modified(self): def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url - self.host, self.path = urllib.parse.urlparse(url)[1:3] + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" @@ -65,9 +80,10 @@ def read(self): self.disallow_all = True elif err.code >= 400 and err.code < 500: self.allow_all = True + err.close() else: raw = f.read() - self.parse(raw.decode("utf-8").splitlines()) + self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): if "*" in entry.useragents: @@ -111,7 +127,7 @@ def parse(self, lines): line = line.split(':', 1) if len(line) == 2: line[0] = line[0].strip().lower() - line[1] = urllib.parse.unquote(line[1].strip()) + line[1] = line[1].strip() if line[0] == "user-agent": if state == 2: self._add_entry(entry) @@ -165,10 +181,11 @@ def can_fetch(self, useragent, url): return False # search for given user agent matches # the first match counts - parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) - url = urllib.parse.urlunparse(('','',parsed_url.path, - parsed_url.params,parsed_url.query, parsed_url.fragment)) - url = urllib.parse.quote(url) + # TODO: The private API is used in order to preserve an empty query. + # This is temporary until the public API starts supporting this feature. + parsed_url = urllib.parse._urlsplit(url, '') + url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) + url = normalize_path(url) if not url: url = "/" for entry in self.entries: @@ -211,7 +228,6 @@ def __str__(self): entries = entries + [self.default_entry] return '\n\n'.join(map(str, entries)) - class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.""" @@ -219,8 +235,7 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) - self.path = urllib.parse.quote(path) + self.path = normalize_path(path) self.allowance = allowance def applies_to(self, filename): @@ -266,7 +281,7 @@ def applies_to(self, useragent): def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL decoded""" + - filename is URL encoded""" for line in self.rulelines: if line.applies_to(filename): return line.allowance diff --git a/Lib/uuid.py b/Lib/uuid.py index 55f46eb5106..313f2fc46cb 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -1,8 +1,12 @@ -r"""UUID objects (universally unique identifiers) according to RFC 4122. +r"""UUID objects (universally unique identifiers) according to RFC 4122/9562. -This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 -UUIDs as specified in RFC 4122. +This module provides immutable UUID objects (class UUID) and functions for +generating UUIDs corresponding to a specific UUID version as specified in +RFC 4122/9562, e.g., uuid1() for UUID version 1, uuid3() for UUID version 3, +and so on. + +Note that UUID version 2 is deliberately omitted as it is outside the scope +of the RFC. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing @@ -42,10 +46,19 @@ # make a UUID from a 16-byte string >>> uuid.UUID(bytes=x.bytes) UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') + + # get the Nil UUID + >>> uuid.NIL + UUID('00000000-0000-0000-0000-000000000000') + + # get the Max UUID + >>> uuid.MAX + UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') """ import os import sys +import time from enum import Enum, _simple_enum @@ -85,6 +98,19 @@ class SafeUUID: unknown = None +_UINT_128_MAX = (1 << 128) - 1 +# 128-bit mask to clear the variant and version bits of a UUID integral value +_RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48)) +# RFC 4122 variant bits and version bits to activate on a UUID integral value. +_RFC_4122_VERSION_1_FLAGS = ((1 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_6_FLAGS = ((6 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_7_FLAGS = ((7 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) + + class UUID: """Instances of the UUID class represent UUIDs as specified in RFC 4122. UUID objects are immutable, hashable, and usable as dictionary keys. @@ -108,7 +134,16 @@ class UUID: fields a tuple of the six integer fields of the UUID, which are also available as six individual attributes - and two derived attributes: + and two derived attributes. Those attributes are not + always relevant to all UUID versions: + + The 'time_*' attributes are only relevant to version 1. + + The 'clock_seq*' and 'node' attributes are only relevant + to versions 1 and 6. + + The 'time' attribute is only relevant to versions 1, 6 + and 7. time_low the first 32 bits of the UUID time_mid the next 16 bits of the UUID @@ -117,19 +152,20 @@ class UUID: clock_seq_low the next 8 bits of the UUID node the last 48 bits of the UUID - time the 60-bit timestamp + time the 60-bit timestamp for UUIDv1/v6, + or the 48-bit timestamp for UUIDv7 clock_seq the 14-bit sequence number hex the UUID as a 32-character hexadecimal string int the UUID as a 128-bit integer - urn the UUID as a URN as specified in RFC 4122 + urn the UUID as a URN as specified in RFC 4122/9562 variant the UUID variant (one of the constants RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) - version the UUID version number (1 through 5, meaningful only + version the UUID version number (1 through 8, meaningful only when the variant is RFC_4122) is_safe An enum indicating whether the UUID has been generated in @@ -174,57 +210,69 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, if [hex, bytes, bytes_le, fields, int].count(None) != 4: raise TypeError('one of the hex, bytes, bytes_le, fields, ' 'or int arguments must be given') - if hex is not None: + if int is not None: + pass + elif hex is not None: hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = int_(hex, 16) - if bytes_le is not None: + elif bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') + assert isinstance(bytes_le, bytes_), repr(bytes_le) bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + bytes_le[8-1:6-1:-1] + bytes_le[8:]) - if bytes is not None: + int = int_.from_bytes(bytes) # big endian + elif bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') assert isinstance(bytes, bytes_), repr(bytes) int = int_.from_bytes(bytes) # big endian - if fields is not None: + elif fields is not None: if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields - if not 0 <= time_low < 1<<32: + if not 0 <= time_low < (1 << 32): raise ValueError('field 1 out of range (need a 32-bit value)') - if not 0 <= time_mid < 1<<16: + if not 0 <= time_mid < (1 << 16): raise ValueError('field 2 out of range (need a 16-bit value)') - if not 0 <= time_hi_version < 1<<16: + if not 0 <= time_hi_version < (1 << 16): raise ValueError('field 3 out of range (need a 16-bit value)') - if not 0 <= clock_seq_hi_variant < 1<<8: + if not 0 <= clock_seq_hi_variant < (1 << 8): raise ValueError('field 4 out of range (need an 8-bit value)') - if not 0 <= clock_seq_low < 1<<8: + if not 0 <= clock_seq_low < (1 << 8): raise ValueError('field 5 out of range (need an 8-bit value)') - if not 0 <= node < 1<<48: + if not 0 <= node < (1 << 48): raise ValueError('field 6 out of range (need a 48-bit value)') clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low int = ((time_low << 96) | (time_mid << 80) | (time_hi_version << 64) | (clock_seq << 48) | node) - if int is not None: - if not 0 <= int < 1<<128: - raise ValueError('int is out of range (need a 128-bit value)') + if not 0 <= int <= _UINT_128_MAX: + raise ValueError('int is out of range (need a 128-bit value)') if version is not None: - if not 1 <= version <= 5: + if not 1 <= version <= 8: raise ValueError('illegal version number') - # Set the variant to RFC 4122. - int &= ~(0xc000 << 48) - int |= 0x8000 << 48 + # clear the variant and the version number bits + int &= _RFC_4122_CLEARFLAGS_MASK + # Set the variant to RFC 4122/9562. + int |= 0x8000_0000_0000_0000 # (0x8000 << 48) # Set the version number. - int &= ~(0xf000 << 64) int |= version << 76 object.__setattr__(self, 'int', int) object.__setattr__(self, 'is_safe', is_safe) + @classmethod + def _from_int(cls, value): + """Create a UUID from an integer *value*. Internal use only.""" + assert 0 <= value <= _UINT_128_MAX, repr(value) + self = object.__new__(cls) + object.__setattr__(self, 'int', value) + object.__setattr__(self, 'is_safe', SafeUUID.unknown) + return self + def __getstate__(self): d = {'int': self.int} if self.is_safe != SafeUUID.unknown: @@ -281,9 +329,8 @@ def __setattr__(self, name, value): raise TypeError('UUID objects are immutable') def __str__(self): - hex = '%032x' % self.int - return '%s-%s-%s-%s-%s' % ( - hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:]) + x = self.hex + return f'{x[:8]}-{x[8:12]}-{x[12:16]}-{x[16:20]}-{x[20:]}' @property def bytes(self): @@ -322,8 +369,22 @@ def clock_seq_low(self): @property def time(self): - return (((self.time_hi_version & 0x0fff) << 48) | - (self.time_mid << 32) | self.time_low) + if self.version == 6: + # time_hi (32) | time_mid (16) | ver (4) | time_lo (12) | ... (64) + time_hi = self.int >> 96 + time_lo = (self.int >> 64) & 0x0fff + return time_hi << 28 | (self.time_mid << 12) | time_lo + elif self.version == 7: + # unix_ts_ms (48) | ... (80) + return self.int >> 80 + else: + # time_lo (32) | time_mid (16) | ver (4) | time_hi (12) | ... (64) + # + # For compatibility purposes, we do not warn or raise when the + # version is not 1 (timestamp is irrelevant to other versions). + time_hi = (self.int >> 64) & 0x0fff + time_lo = self.int >> 96 + return time_hi << 48 | (self.time_mid << 32) | time_lo @property def clock_seq(self): @@ -336,7 +397,7 @@ def node(self): @property def hex(self): - return '%032x' % self.int + return self.bytes.hex() @property def urn(self): @@ -355,7 +416,7 @@ def variant(self): @property def version(self): - # The version bits are only meaningful for RFC 4122 UUIDs. + # The version bits are only meaningful for RFC 4122/9562 UUIDs. if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) @@ -374,7 +435,7 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - # Empty strings will be quoted by popen so we should just ommit it + # Empty strings will be quoted by popen so we should just omit it if args != ('',): command = (executable, *args) else: @@ -572,7 +633,7 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) - _has_stable_extractable_node = getattr(_uuid, "has_stable_extractable_node", False) + _has_stable_extractable_node = _uuid.has_stable_extractable_node _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None @@ -679,7 +740,6 @@ def uuid1(node=None, clock_seq=None): return UUID(bytes=uuid_time, is_safe=is_safe) global _last_timestamp - import time nanoseconds = time.time_ns() # 0x01b21dd213814000 is the number of 100-ns intervals between the # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. @@ -704,24 +764,171 @@ def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import md5 - digest = md5( - namespace.bytes + name, - usedforsecurity=False - ).digest() - return UUID(bytes=digest[:16], version=3) + import hashlib + h = hashlib.md5(namespace.bytes + name, usedforsecurity=False) + int_uuid_3 = int.from_bytes(h.digest()) + int_uuid_3 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_3 |= _RFC_4122_VERSION_3_FLAGS + return UUID._from_int(int_uuid_3) def uuid4(): """Generate a random UUID.""" - return UUID(bytes=os.urandom(16), version=4) + int_uuid_4 = int.from_bytes(os.urandom(16)) + int_uuid_4 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_4 |= _RFC_4122_VERSION_4_FLAGS + return UUID._from_int(int_uuid_4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import sha1 - hash = sha1(namespace.bytes + name).digest() - return UUID(bytes=hash[:16], version=5) + import hashlib + h = hashlib.sha1(namespace.bytes + name, usedforsecurity=False) + int_uuid_5 = int.from_bytes(h.digest()[:16]) + int_uuid_5 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS + return UUID._from_int(int_uuid_5) + + +_last_timestamp_v6 = None + +def uuid6(node=None, clock_seq=None): + """Similar to :func:`uuid1` but where fields are ordered differently + for improved DB locality. + + More precisely, given a 60-bit timestamp value as specified for UUIDv1, + for UUIDv6 the first 48 most significant bits are stored first, followed + by the 4-bit version (same position), followed by the remaining 12 bits + of the original 60-bit timestamp. + """ + global _last_timestamp_v6 + import time + nanoseconds = time.time_ns() + # 0x01b21dd213814000 is the number of 100-ns intervals between the + # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + timestamp = nanoseconds // 100 + 0x01b21dd213814000 + if _last_timestamp_v6 is not None and timestamp <= _last_timestamp_v6: + timestamp = _last_timestamp_v6 + 1 + _last_timestamp_v6 = timestamp + if clock_seq is None: + import random + clock_seq = random.getrandbits(14) # instead of stable storage + time_hi_and_mid = (timestamp >> 12) & 0xffff_ffff_ffff + time_lo = timestamp & 0x0fff # keep 12 bits and clear version bits + clock_s = clock_seq & 0x3fff # keep 14 bits and clear variant bits + if node is None: + node = getnode() + # --- 32 + 16 --- -- 4 -- -- 12 -- -- 2 -- -- 14 --- 48 + # time_hi_and_mid | version | time_lo | variant | clock_seq | node + int_uuid_6 = time_hi_and_mid << 80 + int_uuid_6 |= time_lo << 64 + int_uuid_6 |= clock_s << 48 + int_uuid_6 |= node & 0xffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_6 |= _RFC_4122_VERSION_6_FLAGS + return UUID._from_int(int_uuid_6) + + +_last_timestamp_v7 = None +_last_counter_v7 = 0 # 42-bit counter + +def _uuid7_get_counter_and_tail(): + rand = int.from_bytes(os.urandom(10)) + # 42-bit counter with MSB set to 0 + counter = (rand >> 32) & 0x1ff_ffff_ffff + # 32-bit random data + tail = rand & 0xffff_ffff + return counter, tail + + +def uuid7(): + """Generate a UUID from a Unix timestamp in milliseconds and random bits. + + UUIDv7 objects feature monotonicity within a millisecond. + """ + # --- 48 --- -- 4 -- --- 12 --- -- 2 -- --- 30 --- - 32 - + # unix_ts_ms | version | counter_hi | variant | counter_lo | random + # + # 'counter = counter_hi | counter_lo' is a 42-bit counter constructed + # with Method 1 of RFC 9562, §6.2, and its MSB is set to 0. + # + # 'random' is a 32-bit random value regenerated for every new UUID. + # + # If multiple UUIDs are generated within the same millisecond, the LSB + # of 'counter' is incremented by 1. When overflowing, the timestamp is + # advanced and the counter is reset to a random 42-bit integer with MSB + # set to 0. + + global _last_timestamp_v7 + global _last_counter_v7 + + nanoseconds = time.time_ns() + timestamp_ms = nanoseconds // 1_000_000 + + if _last_timestamp_v7 is None or timestamp_ms > _last_timestamp_v7: + counter, tail = _uuid7_get_counter_and_tail() + else: + if timestamp_ms < _last_timestamp_v7: + timestamp_ms = _last_timestamp_v7 + 1 + # advance the 42-bit counter + counter = _last_counter_v7 + 1 + if counter > 0x3ff_ffff_ffff: + # advance the 48-bit timestamp + timestamp_ms += 1 + counter, tail = _uuid7_get_counter_and_tail() + else: + # 32-bit random data + tail = int.from_bytes(os.urandom(4)) + + unix_ts_ms = timestamp_ms & 0xffff_ffff_ffff + counter_msbs = counter >> 30 + # keep 12 counter's MSBs and clear variant bits + counter_hi = counter_msbs & 0x0fff + # keep 30 counter's LSBs and clear version bits + counter_lo = counter & 0x3fff_ffff + # ensure that the tail is always a 32-bit integer (by construction, + # it is already the case, but future interfaces may allow the user + # to specify the random tail) + tail &= 0xffff_ffff + + int_uuid_7 = unix_ts_ms << 80 + int_uuid_7 |= counter_hi << 64 + int_uuid_7 |= counter_lo << 32 + int_uuid_7 |= tail + # by construction, the variant and version bits are already cleared + int_uuid_7 |= _RFC_4122_VERSION_7_FLAGS + res = UUID._from_int(int_uuid_7) + + # defer global update until all computations are done + _last_timestamp_v7 = timestamp_ms + _last_counter_v7 = counter + return res + + +def uuid8(a=None, b=None, c=None): + """Generate a UUID from three custom blocks. + + * 'a' is the first 48-bit chunk of the UUID (octets 0-5); + * 'b' is the mid 12-bit chunk (octets 6-7); + * 'c' is the last 62-bit chunk (octets 8-15). + + When a value is not specified, a pseudo-random value is generated. + """ + if a is None: + import random + a = random.getrandbits(48) + if b is None: + import random + b = random.getrandbits(12) + if c is None: + import random + c = random.getrandbits(62) + int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 + int_uuid_8 |= (b & 0xfff) << 64 + int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_8 |= _RFC_4122_VERSION_8_FLAGS + return UUID._from_int(int_uuid_8) def main(): @@ -730,7 +937,10 @@ def main(): "uuid1": uuid1, "uuid3": uuid3, "uuid4": uuid4, - "uuid5": uuid5 + "uuid5": uuid5, + "uuid6": uuid6, + "uuid7": uuid7, + "uuid8": uuid8, } uuid_namespace_funcs = ("uuid3", "uuid5") namespaces = { @@ -742,18 +952,24 @@ def main(): import argparse parser = argparse.ArgumentParser( - description="Generates a uuid using the selected uuid function.") - parser.add_argument("-u", "--uuid", choices=uuid_funcs.keys(), default="uuid4", - help="The function to use to generate the uuid. " - "By default uuid4 function is used.") + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Generate a UUID using the selected UUID function.", + color=True, + ) + parser.add_argument("-u", "--uuid", + choices=uuid_funcs.keys(), + default="uuid4", + help="function to generate the UUID") parser.add_argument("-n", "--namespace", - help="The namespace is a UUID, or '@ns' where 'ns' is a " - "well-known predefined UUID addressed by namespace name. " - "Such as @dns, @url, @oid, and @x500. " - "Only required for uuid3/uuid5 functions.") + choices=["any UUID", *namespaces.keys()], + help="uuid3/uuid5 only: " + "a UUID, or a well-known predefined UUID addressed " + "by namespace name") parser.add_argument("-N", "--name", - help="The name used as part of generating the uuid. " - "Only required for uuid3/uuid5 functions.") + help="uuid3/uuid5 only: " + "name used as part of generating the UUID") + parser.add_argument("-C", "--count", metavar="NUM", type=int, default=1, + help="generate NUM fresh UUIDs") args = parser.parse_args() uuid_func = uuid_funcs[args.uuid] @@ -768,9 +984,11 @@ def main(): "Run 'python -m uuid -h' for more information." ) namespace = namespaces[namespace] if namespace in namespaces else UUID(namespace) - print(uuid_func(namespace, name)) + for _ in range(args.count): + print(uuid_func(namespace, name)) else: - print(uuid_func()) + for _ in range(args.count): + print(uuid_func()) # The following standard UUIDs are for use with uuid3() or uuid5(). @@ -780,5 +998,10 @@ def main(): NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8') +# RFC 9562 Sections 5.9 and 5.10 define the special Nil and Max UUID formats. + +NIL = UUID('00000000-0000-0000-0000-000000000000') +MAX = UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') + if __name__ == "__main__": main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6f1af294ae6..f7a6d261401 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,9 +11,10 @@ import sys import sysconfig import types +import shlex -CORE_VENV_DEPS = ('pip', 'setuptools') +CORE_VENV_DEPS = ('pip',) logger = logging.getLogger(__name__) @@ -41,20 +42,24 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI + :param scm_ignore_files: Create ignore files for the SCMs specified by the + iterable. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False): + upgrade_deps=False, *, scm_ignore_files=frozenset()): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks self.upgrade = upgrade self.with_pip = with_pip + self.orig_prompt = prompt if prompt == '.': # see bpo-38901 prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps + self.scm_ignore_files = frozenset(map(str.lower, scm_ignore_files)) def create(self, env_dir): """ @@ -65,6 +70,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) + for scm in self.scm_ignore_files: + getattr(self, f"create_{scm}_ignore_file")(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -92,6 +99,42 @@ def clear_directory(self, path): elif os.path.isdir(fn): shutil.rmtree(fn) + def _venv_path(self, env_dir, name): + vars = { + 'base': env_dir, + 'platbase': env_dir, + 'installed_base': env_dir, + 'installed_platbase': env_dir, + } + return sysconfig.get_path(name, scheme='venv', vars=vars) + + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -106,31 +149,38 @@ def create_if_needed(d): elif os.path.islink(d) or os.path.isfile(d): raise ValueError('Unable to create directory %r' % d) + if os.pathsep in os.fspath(env_dir): + raise ValueError(f'Refusing to create a venv in {env_dir} because ' + f'it contains the PATH separator {os.pathsep}.') if os.path.exists(env_dir) and self.clear: self.clear_directory(env_dir) context = types.SimpleNamespace() context.env_dir = env_dir context.env_name = os.path.split(env_dir)[1] - prompt = self.prompt if self.prompt is not None else context.env_name - context.prompt = '(%s) ' % prompt + context.prompt = self.prompt if self.prompt is not None else context.env_name create_if_needed(env_dir) executable = sys._base_executable + if not executable: # see gh-96861 + raise ValueError('Unable to determine path to the running ' + 'Python interpreter. Provide an explicit path or ' + 'check that your PATH environment variable is ' + 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename - if sys.platform == 'win32': - binname = 'Scripts' - incpath = 'Include' - libpath = os.path.join(env_dir, 'Lib', 'site-packages') - else: - binname = 'bin' - incpath = 'include' - libpath = os.path.join(env_dir, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') - context.inc_path = path = os.path.join(env_dir, incpath) - create_if_needed(path) + binpath = self._venv_path(env_dir, 'scripts') + incpath = self._venv_path(env_dir, 'include') + libpath = self._venv_path(env_dir, 'purelib') + + context.inc_path = incpath + create_if_needed(incpath) + context.lib_path = libpath create_if_needed(libpath) # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX if ((sys.maxsize > 2**32) and (os.name == 'posix') and @@ -138,8 +188,8 @@ def create_if_needed(d): link_path = os.path.join(env_dir, 'lib64') if not os.path.exists(link_path): # Issue #21643 os.symlink('lib', link_path) - context.bin_path = binpath = os.path.join(env_dir, binname) - context.bin_name = binname + context.bin_path = binpath + context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) # Assign and update the command to use when launching the newly created @@ -149,7 +199,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -178,86 +228,84 @@ def create_configuration(self, context): f.write('version = %d.%d.%d\n' % sys.version_info[:3]) if self.prompt is not None: f.write(f'prompt = {self.prompt!r}\n') - - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + f.write('executable = %s\n' % os.path.realpath(sys.executable)) + args = [] + nt = os.name == 'nt' + if nt and self.symlinks: + args.append('--symlinks') + if not nt and not self.symlinks: + args.append('--copies') + if not self.with_pip: + args.append('--without-pip') + if self.system_site_packages: + args.append('--system-site-packages') + if self.clear: + args.append('--clear') + if self.upgrade: + args.append('--upgrade') + if self.upgrade_deps: + args.append('--upgrade-deps') + if self.orig_prompt is not None: + args.append(f'--prompt="{self.orig_prompt}"') + if not self.scm_ignore_files: + args.append('--without-scm-ignore-files') + + args.append(context.env_dir) + args = ' '.join(args) + f.write(f'command = {sys.executable} -m venv {args}\n') + + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) - def setup_python(self, context): + def create_git_ignore_file(self, context): """ - Set up a Python executable in the environment. + Create a .gitignore file in the environment directory. - :param context: The information for the environment creation request - being processed. + The contents of the file cause the entire environment directory to be + ignored by git. """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + gitignore_path = os.path.join(context.env_dir, '.gitignore') + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write('# Created by venv; ' + 'see https://docs.python.org/3/library/venv.html\n') + file.write('*\n') + + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -265,32 +313,107 @@ def setup_python(self, context): copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(True): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + else: + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dest) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: @@ -303,14 +426,25 @@ def setup_python(self, context): shutil.copyfile(src, dst) break + def _call_new_python(self, context, *py_args, **kwargs): + """Executes the newly created Python using safe-ish options""" + # gh-98251: We do not want to just use '-I' because that masks + # legitimate user preferences (such as not writing bytecode). All we + # really need is to ensure that the path variables do not overrule + # normal venv handling. + args = [context.env_exec_cmd, *py_args] + kwargs['env'] = env = os.environ.copy() + env['VIRTUAL_ENV'] = context.env_dir + env.pop('PYTHONHOME', None) + env.pop('PYTHONPATH', None) + kwargs['cwd'] = context.env_dir + kwargs['executable'] = context.env_exec_cmd + subprocess.check_output(args, **kwargs) + def _setup_pip(self, context): """Installs or upgrades pip in a virtual environment""" - # We run ensurepip in isolated mode to avoid side effects from - # environment vars, the current directory and anything else - # intended for the global Python environment - cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) + self._call_new_python(context, '-m', 'ensurepip', '--upgrade', + '--default-pip', stderr=subprocess.STDOUT) def setup_scripts(self, context): """ @@ -348,11 +482,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -370,15 +534,22 @@ def install_scripts(self, context, path): """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -389,116 +560,122 @@ def install_scripts(self, context, path): if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + context.script_path = srcfile + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] - cmd.extend(CORE_VENV_DEPS) - subprocess.check_call(cmd) + self._call_new_python(context, '-m', 'pip', 'install', '--upgrade', + *CORE_VENV_DEPS) def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, + *, scm_ignore_files=frozenset()): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, - prompt=prompt, upgrade_deps=upgrade_deps) + prompt=prompt, upgrade_deps=upgrade_deps, + scm_ignore_files=scm_ignore_files) builder.create(env_dir) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with Python >= 3.3') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.', - epilog='Once an environment has been ' - 'created, you may wish to ' - 'activate it, e.g. by ' - 'sourcing an activate script ' - 'in its bin directory.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - group = parser.add_mutually_exclusive_group() - group.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - group.add_argument('--copies', default=not use_symlinks, - action='store_false', dest='symlinks', - help='Try to use copies rather than symlinks, ' - 'even when symlinks are the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--without-pip', dest='with_pip', - default=True, action='store_false', - help='Skips installing or upgrading pip in the ' - 'virtual environment (pip is bootstrapped ' - 'by default)') - parser.add_argument('--prompt', - help='Provides an alternative prompt prefix for ' - 'this environment.') - parser.add_argument('--upgrade-deps', default=False, action='store_true', - dest='upgrade_deps', - help='Upgrade core dependencies: {} to the latest ' - 'version in PyPI'.format( - ' '.join(CORE_VENV_DEPS))) - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = EnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - with_pip=options.with_pip, - prompt=options.prompt, - upgrade_deps=options.upgrade_deps) - for d in options.dirs: - builder.create(d) + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' + 'to the latest version in PyPI') + parser.add_argument('--without-scm-ignore-files', dest='scm_ignore_files', + action='store_const', const=frozenset(), + default=frozenset(['git']), + help='Skips adding SCM ignore files to the environment ' + 'directory (Git is supported by default).') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps, + scm_ignore_files=options.scm_ignore_files) + for d in options.dirs: + builder.create(d) + if __name__ == '__main__': rc = 1 diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py index 912423e4a78..88f55439dc2 100644 --- a/Lib/venv/__main__.py +++ b/Lib/venv/__main__.py @@ -6,5 +6,5 @@ main() rc = 0 except Exception as e: - print('Error: %s' % e, file=sys.stderr) + print('Error:', e, file=sys.stderr) sys.exit(rc) diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index b49d77ba44b..16ba5290fae 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -219,6 +219,8 @@ deactivate -nondestructive # that there is an activated venv. $env:VIRTUAL_ENV = $VenvDir +$env:VIRTUAL_ENV_PROMPT = $Prompt + if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Verbose "Setting prompt to '$Prompt'" @@ -233,7 +235,6 @@ if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " _OLD_VIRTUAL_PROMPT } - $env:VIRTUAL_ENV_PROMPT = $Prompt } # Clear PYTHONHOME diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fbc2b8801d..70673a265d4 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -1,5 +1,5 @@ # This file must be used with "source bin/activate" *from bash* -# you cannot run it directly +# You cannot run it directly deactivate () { # reset old environment variables @@ -14,12 +14,10 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -38,13 +36,27 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" -export VIRTUAL_ENV +# on Windows, a path can contain colons and backslashes and has to be converted: +case "$(uname)" in + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath __VENV_DIR__) + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV=__VENV_DIR__ + ;; +esac _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH +VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +export VIRTUAL_ENV_PROMPT + # unset PYTHONHOME if set # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) # could use `if (set -u; : $PYTHONHOME) ;` in bash @@ -55,15 +67,10 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1="("__VENV_PROMPT__") ${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" - export VIRTUAL_ENV_PROMPT fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 76% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index e40a1d71489..284a7469c99 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -1,5 +1,5 @@ # This file must be used with "source <venv>/bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. +# (https://fishshell.com/). You cannot run it directly. function deactivate -d "Exit virtual environment and return to normal shell environment" # reset old environment variables @@ -13,10 +13,13 @@ function deactivate -d "Exit virtual environment and return to normal shell env end if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end end set -e VIRTUAL_ENV @@ -30,10 +33,11 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH +set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -53,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . @@ -62,5 +66,4 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" end diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 5daa45afc9f..9ac5c20b477 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,15 +8,15 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/nt/venvlauncher.exe b/Lib/venv/scripts/nt/venvlauncher.exe new file mode 100644 index 00000000000..c6863b56e57 Binary files /dev/null and b/Lib/venv/scripts/nt/venvlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvlaunchert.exe b/Lib/venv/scripts/nt/venvlaunchert.exe new file mode 100644 index 00000000000..c12a7a869f4 Binary files /dev/null and b/Lib/venv/scripts/nt/venvlaunchert.exe differ diff --git a/Lib/venv/scripts/nt/venvwlauncher.exe b/Lib/venv/scripts/nt/venvwlauncher.exe new file mode 100644 index 00000000000..d0d3733266f Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvwlaunchert.exe b/Lib/venv/scripts/nt/venvwlaunchert.exe new file mode 100644 index 00000000000..9456a9e9b4a Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlaunchert.exe differ diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index d6f697c55ed..2a3fa835476 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -1,5 +1,6 @@ # This file must be used with "source bin/activate.csh" *from csh*. # You cannot run it directly. + # Created by Davide Di Blasi <davidedb@gmail.com>. # Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com> @@ -8,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" +setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set prompt = "("__VENV_PROMPT__") $prompt:q" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index f83aaf231ea..6759857d909 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -1,735 +1,99 @@ -"""Python part of the warnings subsystem.""" - import sys +__all__ = [ + "warn", + "warn_explicit", + "showwarning", + "formatwarning", + "filterwarnings", + "simplefilter", + "resetwarnings", + "catch_warnings", + "deprecated", +] + +from _py_warnings import ( + WarningMessage, + _DEPRECATED_MSG, + _OptionError, + _add_filter, + _deprecated, + _filters_mutated, + _filters_mutated_lock_held, + _filters_version, + _formatwarning_orig, + _formatwarnmsg, + _formatwarnmsg_impl, + _get_context, + _get_filters, + _getaction, + _getcategory, + _is_filename_to_skip, + _is_internal_filename, + _is_internal_frame, + _lock, + _new_context, + _next_external_frame, + _processoptions, + _set_context, + _set_module, + _setoption, + _setup_defaults, + _showwarning_orig, + _showwarnmsg, + _showwarnmsg_impl, + _use_context, + _warn_unawaited_coroutine, + _warnings_context, + catch_warnings, + defaultaction, + deprecated, + filters, + filterwarnings, + formatwarning, + onceregistry, + resetwarnings, + showwarning, + simplefilter, + warn, + warn_explicit, +) -__all__ = ["warn", "warn_explicit", "showwarning", - "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings", "deprecated"] - -def showwarning(message, category, filename, lineno, file=None, line=None): - """Hook to write a warning to a file; replace if you like.""" - msg = WarningMessage(message, category, filename, lineno, file, line) - _showwarnmsg_impl(msg) - -def formatwarning(message, category, filename, lineno, line=None): - """Function to format a warning the standard way.""" - msg = WarningMessage(message, category, filename, lineno, None, line) - return _formatwarnmsg_impl(msg) - -def _showwarnmsg_impl(msg): - file = msg.file - if file is None: - file = sys.stderr - if file is None: - # sys.stderr is None when run with pythonw.exe: - # warnings get lost - return - text = _formatwarnmsg(msg) - try: - file.write(text) - except OSError: - # the file (probably stderr) is invalid - this warning gets lost. - pass - -def _formatwarnmsg_impl(msg): - category = msg.category.__name__ - s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" - - if msg.line is None: - try: - import linecache - line = linecache.getline(msg.filename, msg.lineno) - except Exception: - # When a warning is logged during Python shutdown, linecache - # and the import machinery don't work anymore - line = None - linecache = None - else: - line = msg.line - if line: - line = line.strip() - s += " %s\n" % line - - if msg.source is not None: - try: - import tracemalloc - # Logging a warning should not raise a new exception: - # catch Exception, not only ImportError and RecursionError. - except Exception: - # don't suggest to enable tracemalloc if it's not available - suggest_tracemalloc = False - tb = None - else: - try: - suggest_tracemalloc = not tracemalloc.is_tracing() - tb = tracemalloc.get_object_traceback(msg.source) - except Exception: - # When a warning is logged during Python shutdown, tracemalloc - # and the import machinery don't work anymore - suggest_tracemalloc = False - tb = None - - if tb is not None: - s += 'Object allocated at (most recent call last):\n' - for frame in tb: - s += (' File "%s", lineno %s\n' - % (frame.filename, frame.lineno)) - - try: - if linecache is not None: - line = linecache.getline(frame.filename, frame.lineno) - else: - line = None - except Exception: - line = None - if line: - line = line.strip() - s += ' %s\n' % line - elif suggest_tracemalloc: - s += (f'{category}: Enable tracemalloc to get the object ' - f'allocation traceback\n') - return s - -# Keep a reference to check if the function was replaced -_showwarning_orig = showwarning - -def _showwarnmsg(msg): - """Hook to write a warning to a file; replace if you like.""" - try: - sw = showwarning - except NameError: - pass - else: - if sw is not _showwarning_orig: - # warnings.showwarning() was replaced - if not callable(sw): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") - - sw(msg.message, msg.category, msg.filename, msg.lineno, - msg.file, msg.line) - return - _showwarnmsg_impl(msg) - -# Keep a reference to check if the function was replaced -_formatwarning_orig = formatwarning - -def _formatwarnmsg(msg): - """Function to format a warning the standard way.""" - try: - fw = formatwarning - except NameError: - pass - else: - if fw is not _formatwarning_orig: - # warnings.formatwarning() was replaced - return fw(msg.message, msg.category, - msg.filename, msg.lineno, msg.line) - return _formatwarnmsg_impl(msg) - -def filterwarnings(action, message="", category=Warning, module="", lineno=0, - append=False): - """Insert an entry into the list of warnings filters (at the front). - - 'action' -- one of "error", "ignore", "always", "default", "module", - or "once" - 'message' -- a regex that the warning message must match - 'category' -- a class that the warning must be a subclass of - 'module' -- a regex that the module name must match - 'lineno' -- an integer line number, 0 matches all warnings - 'append' -- if true, append to the list of filters - """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: - raise ValueError(f"invalid action: {action!r}") - if not isinstance(message, str): - raise TypeError("message must be a string") - if not isinstance(category, type) or not issubclass(category, Warning): - raise TypeError("category must be a Warning subclass") - if not isinstance(module, str): - raise TypeError("module must be a string") - if not isinstance(lineno, int): - raise TypeError("lineno must be an int") - if lineno < 0: - raise ValueError("lineno must be an int >= 0") - - if message or module: - import re - - if message: - message = re.compile(message, re.I) - else: - message = None - if module: - module = re.compile(module) - else: - module = None - - _add_filter(action, message, category, module, lineno, append=append) - -def simplefilter(action, category=Warning, lineno=0, append=False): - """Insert a simple entry into the list of warnings filters (at the front). - - A simple filter matches all modules and messages. - 'action' -- one of "error", "ignore", "always", "default", "module", - or "once" - 'category' -- a class that the warning must be a subclass of - 'lineno' -- an integer line number, 0 matches all warnings - 'append' -- if true, append to the list of filters - """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: - raise ValueError(f"invalid action: {action!r}") - if not isinstance(lineno, int): - raise TypeError("lineno must be an int") - if lineno < 0: - raise ValueError("lineno must be an int >= 0") - _add_filter(action, None, category, None, lineno, append=append) - -def _add_filter(*item, append): - # Remove possible duplicate filters, so new one will be placed - # in correct place. If append=True and duplicate exists, do nothing. - if not append: - try: - filters.remove(item) - except ValueError: - pass - filters.insert(0, item) - else: - if item not in filters: - filters.append(item) - _filters_mutated() - -def resetwarnings(): - """Clear the list of warning filters, so that no filters are active.""" - filters[:] = [] - _filters_mutated() - -class _OptionError(Exception): - """Exception used by option processing helpers.""" - pass - -# Helper to process -W options passed via sys.warnoptions -def _processoptions(args): - for arg in args: - try: - _setoption(arg) - except _OptionError as msg: - print("Invalid -W option ignored:", msg, file=sys.stderr) - -# Helper for _processoptions() -def _setoption(arg): - parts = arg.split(':') - if len(parts) > 5: - raise _OptionError("too many fields (max 5): %r" % (arg,)) - while len(parts) < 5: - parts.append('') - action, message, category, module, lineno = [s.strip() - for s in parts] - action = _getaction(action) - category = _getcategory(category) - if message or module: - import re - if message: - message = re.escape(message) - if module: - module = re.escape(module) + r'\Z' - if lineno: - try: - lineno = int(lineno) - if lineno < 0: - raise ValueError - except (ValueError, OverflowError): - raise _OptionError("invalid lineno %r" % (lineno,)) from None - else: - lineno = 0 - filterwarnings(action, message, category, module, lineno) - -# Helper for _setoption() -def _getaction(action): - if not action: - return "default" - if action == "all": return "always" # Alias - for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): - if a.startswith(action): - return a - raise _OptionError("invalid action: %r" % (action,)) - -# Helper for _setoption() -def _getcategory(category): - if not category: - return Warning - if '.' not in category: - import builtins as m - klass = category - else: - module, _, klass = category.rpartition('.') - try: - m = __import__(module, None, None, [klass]) - except ImportError: - raise _OptionError("invalid module name: %r" % (module,)) from None - try: - cat = getattr(m, klass) - except AttributeError: - raise _OptionError("unknown warning category: %r" % (category,)) from None - if not issubclass(cat, Warning): - raise _OptionError("invalid warning category: %r" % (category,)) - return cat - - -def _is_internal_filename(filename): - return 'importlib' in filename and '_bootstrap' in filename - - -def _is_filename_to_skip(filename, skip_file_prefixes): - return any(filename.startswith(prefix) for prefix in skip_file_prefixes) - - -def _is_internal_frame(frame): - """Signal whether the frame is an internal CPython implementation detail.""" - return _is_internal_filename(frame.f_code.co_filename) - - -def _next_external_frame(frame, skip_file_prefixes): - """Find the next frame that doesn't involve Python or user internals.""" - frame = frame.f_back - while frame is not None and ( - _is_internal_filename(filename := frame.f_code.co_filename) or - _is_filename_to_skip(filename, skip_file_prefixes)): - frame = frame.f_back - return frame - - -# Code typically replaced by _warnings -def warn(message, category=None, stacklevel=1, source=None, - *, skip_file_prefixes=()): - """Issue a warning, or maybe ignore it or raise an exception.""" - # Check if message is already a Warning object - if isinstance(message, Warning): - category = message.__class__ - # Check category argument - if category is None: - category = UserWarning - if not (isinstance(category, type) and issubclass(category, Warning)): - raise TypeError("category must be a Warning subclass, " - "not '{:s}'".format(type(category).__name__)) - if not isinstance(skip_file_prefixes, tuple): - # The C version demands a tuple for implementation performance. - raise TypeError('skip_file_prefixes must be a tuple of strs.') - if skip_file_prefixes: - stacklevel = max(2, stacklevel) - # Get context information - try: - if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): - # If frame is too small to care or if the warning originated in - # internal code, then do not try to hide any frames. - frame = sys._getframe(stacklevel) - else: - frame = sys._getframe(1) - # Look for one frame less since the above line starts us off. - for x in range(stacklevel-1): - frame = _next_external_frame(frame, skip_file_prefixes) - if frame is None: - raise ValueError - except ValueError: - globals = sys.__dict__ - filename = "<sys>" - lineno = 0 - else: - globals = frame.f_globals - filename = frame.f_code.co_filename - lineno = frame.f_lineno - if '__name__' in globals: - module = globals['__name__'] - else: - module = "<string>" - registry = globals.setdefault("__warningregistry__", {}) - warn_explicit(message, category, filename, lineno, module, registry, - globals, source) - -def warn_explicit(message, category, filename, lineno, - module=None, registry=None, module_globals=None, - source=None): - lineno = int(lineno) - if module is None: - module = filename or "<unknown>" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? - if registry is None: - registry = {} - if registry.get('version', 0) != _filters_version: - registry.clear() - registry['version'] = _filters_version - if isinstance(message, Warning): - text = str(message) - category = message.__class__ - else: - text = message - message = category(message) - key = (text, category, lineno) - # Quick test for common case - if registry.get(key): - return - # Search the filters - for item in filters: - action, msg, cat, mod, ln = item - if ((msg is None or msg.match(text)) and - issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break - else: - action = defaultaction - # Early exit actions - if action == "ignore": - return - - # Prime the linecache for formatting, in case the - # "file" is actually in a zipfile or something. - import linecache - linecache.getlines(filename, module_globals) - - if action == "error": - raise message - # Other actions - if action == "once": - registry[key] = 1 - oncekey = (text, category) - if onceregistry.get(oncekey): - return - onceregistry[oncekey] = 1 - elif action == "always": - pass - elif action == "module": - registry[key] = 1 - altkey = (text, category, 0) - if registry.get(altkey): - return - registry[altkey] = 1 - elif action == "default": - registry[key] = 1 - else: - # Unrecognized actions are errors - raise RuntimeError( - "Unrecognized action (%r) in warnings.filters:\n %s" % - (action, item)) - # Print message and context - msg = WarningMessage(message, category, filename, lineno, source=source) - _showwarnmsg(msg) - - -class WarningMessage(object): - - _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line", "source") - - def __init__(self, message, category, filename, lineno, file=None, - line=None, source=None): - self.message = message - self.category = category - self.filename = filename - self.lineno = lineno - self.file = file - self.line = line - self.source = source - self._category_name = category.__name__ if category else None - - def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) - - -class catch_warnings(object): - - """A context manager that copies and restores the warnings filter upon - exiting the context. - - The 'record' argument specifies whether warnings should be captured by a - custom implementation of warnings.showwarning() and be appended to a list - returned by the context manager. Otherwise None is returned by the context - manager. The objects appended to the list are arguments whose attributes - mirror the arguments to showwarning(). - - The 'module' argument is to specify an alternative module to the module - named 'warnings' and imported under that name. This argument is only useful - when testing the warnings module itself. - - If the 'action' argument is not None, the remaining arguments are passed - to warnings.simplefilter() as if it were called immediately on entering the - context. - """ - - def __init__(self, *, record=False, module=None, - action=None, category=Warning, lineno=0, append=False): - """Specify whether to record warnings and if an alternative module - should be used other than sys.modules['warnings']. - - For compatibility with Python 3.0, please consider all arguments to be - keyword-only. - - """ - self._record = record - self._module = sys.modules['warnings'] if module is None else module - self._entered = False - if action is None: - self._filter = None - else: - self._filter = (action, category, lineno, append) - - def __repr__(self): - args = [] - if self._record: - args.append("record=True") - if self._module is not sys.modules['warnings']: - args.append("module=%r" % self._module) - name = type(self).__name__ - return "%s(%s)" % (name, ", ".join(args)) - - def __enter__(self): - if self._entered: - raise RuntimeError("Cannot enter %r twice" % self) - self._entered = True - self._filters = self._module.filters - self._module.filters = self._filters[:] - self._module._filters_mutated() - self._showwarning = self._module.showwarning - self._showwarnmsg_impl = self._module._showwarnmsg_impl - if self._filter is not None: - simplefilter(*self._filter) - if self._record: - log = [] - self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig - return log - else: - return None - - def __exit__(self, *exc_info): - if not self._entered: - raise RuntimeError("Cannot exit %r without entering first" % self) - self._module.filters = self._filters - self._module._filters_mutated() - self._module.showwarning = self._showwarning - self._module._showwarnmsg_impl = self._showwarnmsg_impl - - -class deprecated: - """Indicate that a class, function or overload is deprecated. - - When this decorator is applied to an object, the type checker - will generate a diagnostic on usage of the deprecated object. - - Usage: - - @deprecated("Use B instead") - class A: - pass - - @deprecated("Use g instead") - def f(): - pass - - @overload - @deprecated("int support is deprecated") - def g(x: int) -> int: ... - @overload - def g(x: str) -> int: ... - - The warning specified by *category* will be emitted at runtime - on use of deprecated objects. For functions, that happens on calls; - for classes, on instantiation and on creation of subclasses. - If the *category* is ``None``, no warning is emitted at runtime. - The *stacklevel* determines where the - warning is emitted. If it is ``1`` (the default), the warning - is emitted at the direct caller of the deprecated object; if it - is higher, it is emitted further up the stack. - Static type checker behavior is not affected by the *category* - and *stacklevel* arguments. - - The deprecation message passed to the decorator is saved in the - ``__deprecated__`` attribute on the decorated object. - If applied to an overload, the decorator - must be after the ``@overload`` decorator for the attribute to - exist on the overload as returned by ``get_overloads()``. - - See PEP 702 for details. - - """ - def __init__( - self, - message: str, - /, - *, - category: type[Warning] | None = DeprecationWarning, - stacklevel: int = 1, - ) -> None: - if not isinstance(message, str): - raise TypeError( - f"Expected an object of type str for 'message', not {type(message).__name__!r}" - ) - self.message = message - self.category = category - self.stacklevel = stacklevel - - def __call__(self, arg, /): - # Make sure the inner functions created below don't - # retain a reference to self. - msg = self.message - category = self.category - stacklevel = self.stacklevel - if category is None: - arg.__deprecated__ = msg - return arg - elif isinstance(arg, type): - import functools - from types import MethodType - - original_new = arg.__new__ - - @functools.wraps(original_new) - def __new__(cls, /, *args, **kwargs): - if cls is arg: - warn(msg, category=category, stacklevel=stacklevel + 1) - if original_new is not object.__new__: - return original_new(cls, *args, **kwargs) - # Mirrors a similar check in object.__new__. - elif cls.__init__ is object.__init__ and (args or kwargs): - raise TypeError(f"{cls.__name__}() takes no arguments") - else: - return original_new(cls) - - arg.__new__ = staticmethod(__new__) - - original_init_subclass = arg.__init_subclass__ - # We need slightly different behavior if __init_subclass__ - # is a bound method (likely if it was implemented in Python) - if isinstance(original_init_subclass, MethodType): - original_init_subclass = original_init_subclass.__func__ - - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): - warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = classmethod(__init_subclass__) - # Or otherwise, which likely means it's a builtin such as - # object's implementation of __init_subclass__. - else: - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): - warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = __init_subclass__ - - arg.__deprecated__ = __new__.__deprecated__ = msg - __init_subclass__.__deprecated__ = msg - return arg - elif callable(arg): - import functools - import inspect - - @functools.wraps(arg) - def wrapper(*args, **kwargs): - warn(msg, category=category, stacklevel=stacklevel + 1) - return arg(*args, **kwargs) - - if inspect.iscoroutinefunction(arg): - wrapper = inspect.markcoroutinefunction(wrapper) - - arg.__deprecated__ = wrapper.__deprecated__ = msg - return wrapper - else: - raise TypeError( - "@deprecated decorator with non-None category must be applied to " - f"a class or callable, not {arg!r}" - ) - - -_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" - -def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): - """Warn that *name* is deprecated or should be removed. - - RuntimeError is raised if *remove* specifies a major/minor tuple older than - the current Python version or the same version but past the alpha. - - The *message* argument is formatted with *name* and *remove* as a Python - version tuple (e.g. (3, 11)). - - """ - remove_formatted = f"{remove[0]}.{remove[1]}" - if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): - msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha" - raise RuntimeError(msg) - else: - msg = message.format(name=name, remove=remove_formatted) - warn(msg, DeprecationWarning, stacklevel=3) - - -# Private utility function called by _PyErr_WarnUnawaitedCoroutine -def _warn_unawaited_coroutine(coro): - msg_lines = [ - f"coroutine '{coro.__qualname__}' was never awaited\n" - ] - if coro.cr_origin is not None: - import linecache, traceback - def extract(): - for filename, lineno, funcname in reversed(coro.cr_origin): - line = linecache.getline(filename, lineno) - yield (filename, lineno, funcname, line) - msg_lines.append("Coroutine created at (most recent call last)\n") - msg_lines += traceback.format_list(list(extract())) - msg = "".join(msg_lines).rstrip("\n") - # Passing source= here means that if the user happens to have tracemalloc - # enabled and tracking where the coroutine was created, the warning will - # contain that traceback. This does mean that if they have *both* - # coroutine origin tracking *and* tracemalloc enabled, they'll get two - # partially-redundant tracebacks. If we wanted to be clever we could - # probably detect this case and avoid it, but for now we don't bother. - warn(msg, category=RuntimeWarning, stacklevel=2, source=coro) - - -# filters contains a sequence of filter 5-tuples -# The components of the 5-tuple are: -# - an action: error, ignore, always, default, module, or once -# - a compiled regex that must match the warning message -# - a class representing the warning category -# - a compiled regex that must match the module that is being warned -# - a line number for the line being warning, or 0 to mean any line -# If either if the compiled regexs are None, match anything. try: - from _warnings import (filters, _defaultaction, _onceregistry, - warn, warn_explicit, _filters_mutated) - defaultaction = _defaultaction - onceregistry = _onceregistry + # Try to use the C extension, this will replace some parts of the + # _py_warnings implementation imported above. + from _warnings import ( + _acquire_lock, + _defaultaction as defaultaction, + _filters_mutated_lock_held, + _onceregistry as onceregistry, + _release_lock, + _warnings_context, + filters, + warn, + warn_explicit, + ) + _warnings_defaults = True -except ImportError: - filters = [] - defaultaction = "default" - onceregistry = {} - _filters_version = 1 + class _Lock: + def __enter__(self): + _acquire_lock() + return self - def _filters_mutated(): - global _filters_version - _filters_version += 1 + def __exit__(self, *args): + _release_lock() + _lock = _Lock() +except ImportError: _warnings_defaults = False # Module initialization +_set_module(sys.modules[__name__]) _processoptions(sys.warnoptions) if not _warnings_defaults: - # Several warning categories are ignored by default in regular builds - if not hasattr(sys, 'gettotalrefcount'): - filterwarnings("default", category=DeprecationWarning, - module="__main__", append=1) - simplefilter("ignore", category=DeprecationWarning, append=1) - simplefilter("ignore", category=PendingDeprecationWarning, append=1) - simplefilter("ignore", category=ImportWarning, append=1) - simplefilter("ignore", category=ResourceWarning, append=1) + _setup_defaults() del _warnings_defaults +del _setup_defaults diff --git a/Lib/wave.py b/Lib/wave.py index a34af244c3e..b8476e26486 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -441,6 +441,8 @@ class Wave_write: _datawritten -- the size of the audio samples actually written """ + _file = None + def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): diff --git a/Lib/weakref.py b/Lib/weakref.py index 994ea8aa37d..94e4278143c 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -2,7 +2,7 @@ This module is an implementation of PEP 205: -https://www.python.org/dev/peps/pep-0205/ +https://peps.python.org/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; @@ -19,7 +19,7 @@ ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -33,7 +33,6 @@ "WeakSet", "WeakMethod", "finalize"] -_collections_abc.Set.register(WeakSet) _collections_abc.MutableSet.register(WeakSet) class WeakMethod(ref): @@ -106,34 +105,14 @@ def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -141,18 +120,12 @@ def __getitem__(self, key): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -163,38 +136,28 @@ def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -208,21 +171,15 @@ def get(self, key, default=None): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -236,23 +193,15 @@ def itervaluerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -260,8 +209,6 @@ def popitem(self): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -280,16 +227,12 @@ def setdefault(self, key, default=None): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -309,9 +252,7 @@ def valuerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -370,57 +311,22 @@ def __init__(self, dict=None): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -430,11 +336,10 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -442,11 +347,10 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -460,26 +364,23 @@ def __contains__(self, key): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -494,7 +395,6 @@ def keyrefs(self): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -502,7 +402,6 @@ def popitem(self): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): diff --git a/Lib/wsgiref/__init__.py b/Lib/wsgiref/__init__.py index 1efbba01a30..59ee48fddec 100644 --- a/Lib/wsgiref/__init__.py +++ b/Lib/wsgiref/__init__.py @@ -13,6 +13,8 @@ * validate -- validation wrapper that sits between an app and a server to detect errors in either +* types -- collection of WSGI-related types for static type checking + To-Do: * cgi_gateway -- Run WSGI apps under CGI (pending a deployment standard) diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index f4300b831a4..cafe872c7aa 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -136,6 +136,10 @@ def run(self, application): self.setup_environ() self.result = application(self.environ, self.start_response) self.finish_response() + except (ConnectionAbortedError, BrokenPipeError, ConnectionResetError): + # We expect the client to close the connection abruptly from time + # to time. + return except: try: self.handle_error() @@ -179,7 +183,16 @@ def finish_response(self): for data in self.result: self.write(data) self.finish_content() - finally: + except: + # Call close() on the iterable returned by the WSGI application + # in case of an exception. + if hasattr(self.result, 'close'): + self.result.close() + raise + else: + # We only call close() when no exception is raised, because it + # will set status, result, headers, and environ fields to None. + # See bpo-29183 for more details. self.close() @@ -215,8 +228,7 @@ def start_response(self, status, headers,exc_info=None): if exc_info: try: if self.headers_sent: - # Re-raise original exception if headers sent - raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) + raise finally: exc_info = None # avoid dangling circular ref elif self.headers is not None: @@ -225,18 +237,25 @@ def start_response(self, status, headers,exc_info=None): self.status = status self.headers = self.headers_class(headers) status = self._convert_string_type(status, "Status") - assert len(status)>=4,"Status must be at least 4 characters" - assert status[:3].isdigit(), "Status message must begin w/3-digit code" - assert status[3]==" ", "Status message must have a space after code" + self._validate_status(status) if __debug__: for name, val in headers: name = self._convert_string_type(name, "Header name") val = self._convert_string_type(val, "Header value") - assert not is_hop_by_hop(name),"Hop-by-hop headers not allowed" + assert not is_hop_by_hop(name),\ + f"Hop-by-hop header, '{name}: {val}', not allowed" return self.write + def _validate_status(self, status): + if len(status) < 4: + raise AssertionError("Status must be at least 4 characters") + if not status[:3].isdigit(): + raise AssertionError("Status message must begin w/3-digit code") + if status[3] != " ": + raise AssertionError("Status message must have a space after code") + def _convert_string_type(self, value, title): """Convert/check value type.""" if type(value) is str: @@ -456,10 +475,7 @@ def _write(self,data): from warnings import warn warn("SimpleHandler.stdout.write() should not do partial writes", DeprecationWarning) - while True: - data = data[result:] - if not data: - break + while data := data[result:]: result = self.stdout.write(data) def _flush(self): diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index f71563a5ae0..a0f2397fcf0 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -84,10 +84,6 @@ def get_environ(self): env['PATH_INFO'] = urllib.parse.unquote(path, 'iso-8859-1') env['QUERY_STRING'] = query - - host = self.address_string() - if host != self.client_address[0]: - env['REMOTE_HOST'] = host env['REMOTE_ADDR'] = self.client_address[0] if self.headers.get('content-type') is None: @@ -127,7 +123,8 @@ def handle(self): return handler = ServerHandler( - self.rfile, self.wfile, self.get_stderr(), self.get_environ() + self.rfile, self.wfile, self.get_stderr(), self.get_environ(), + multithread=False, ) handler.request_handler = self # backpointer for logging handler.run(self.server.get_app()) diff --git a/Lib/wsgiref/types.py b/Lib/wsgiref/types.py new file mode 100644 index 00000000000..ef0aead5b28 --- /dev/null +++ b/Lib/wsgiref/types.py @@ -0,0 +1,54 @@ +"""WSGI-related types for static type checking""" + +from collections.abc import Callable, Iterable, Iterator +from types import TracebackType +from typing import Any, Protocol, TypeAlias + +__all__ = [ + "StartResponse", + "WSGIEnvironment", + "WSGIApplication", + "InputStream", + "ErrorStream", + "FileWrapper", +] + +_ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType] +_OptExcInfo: TypeAlias = _ExcInfo | tuple[None, None, None] + +class StartResponse(Protocol): + """start_response() callable as defined in PEP 3333""" + def __call__( + self, + status: str, + headers: list[tuple[str, str]], + exc_info: _OptExcInfo | None = ..., + /, + ) -> Callable[[bytes], object]: ... + +WSGIEnvironment: TypeAlias = dict[str, Any] +WSGIApplication: TypeAlias = Callable[[WSGIEnvironment, StartResponse], + Iterable[bytes]] + +class InputStream(Protocol): + """WSGI input stream as defined in PEP 3333""" + def read(self, size: int = ..., /) -> bytes: ... + def readline(self, size: int = ..., /) -> bytes: ... + def readlines(self, hint: int = ..., /) -> list[bytes]: ... + def __iter__(self) -> Iterator[bytes]: ... + +class ErrorStream(Protocol): + """WSGI error stream as defined in PEP 3333""" + def flush(self) -> object: ... + def write(self, s: str, /) -> object: ... + def writelines(self, seq: list[str], /) -> object: ... + +class _Readable(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + # Optional: def close(self) -> object: ... + +class FileWrapper(Protocol): + """WSGI file wrapper as defined in PEP 3333""" + def __call__( + self, file: _Readable, block_size: int = ..., /, + ) -> Iterable[bytes]: ... diff --git a/Lib/wsgiref/util.py b/Lib/wsgiref/util.py index 516fe898d01..63b92331737 100644 --- a/Lib/wsgiref/util.py +++ b/Lib/wsgiref/util.py @@ -4,7 +4,7 @@ __all__ = [ 'FileWrapper', 'guess_scheme', 'application_uri', 'request_uri', - 'shift_path_info', 'setup_testing_defaults', + 'shift_path_info', 'setup_testing_defaults', 'is_hop_by_hop', ] @@ -17,12 +17,6 @@ def __init__(self, filelike, blksize=8192): if hasattr(filelike,'close'): self.close = filelike.close - def __getitem__(self,key): - data = self.filelike.read(self.blksize) - if data: - return data - raise IndexError - def __iter__(self): return self @@ -155,9 +149,9 @@ def setup_testing_defaults(environ): _hoppish = { - 'connection':1, 'keep-alive':1, 'proxy-authenticate':1, - 'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1, - 'upgrade':1 + 'connection', 'keep-alive', 'proxy-authenticate', + 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', + 'upgrade' }.__contains__ def is_hop_by_hop(header_name): diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 6107dcd7a4d..1a1853cd63a 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -1,6 +1,6 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -# Also licenced under the Apache License, 2.0: http://opensource.org/licenses/apache2.0.php +# Licensed under the MIT license: https://opensource.org/licenses/mit-license.php +# Also licenced under the Apache License, 2.0: https://opensource.org/licenses/apache2.0.php # Licensed to PSF under a Contributor Agreement """ Middleware to check for obedience to the WSGI specification. @@ -77,7 +77,7 @@ * That wsgi.input is used properly: - - .read() is called with zero or one argument + - .read() is called with exactly one argument - That it returns a string @@ -137,7 +137,7 @@ def validator(application): """ When applied between a WSGI server and a WSGI application, this - middleware will check for WSGI compliancy on a number of levels. + middleware will check for WSGI compliance on a number of levels. This middleware does not modify the request or response in any way, but will raise an AssertionError if anything seems off (except for a failure to close the application iterator, which @@ -214,10 +214,7 @@ def readlines(self, *args): return lines def __iter__(self): - while 1: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def close(self): @@ -390,7 +387,6 @@ def check_headers(headers): assert_(type(headers) is list, "Headers (%r) must be of type list: %r" % (headers, type(headers))) - header_names = {} for item in headers: assert_(type(item) is tuple, "Individual headers (%r) must be of type tuple: %r" @@ -403,7 +399,6 @@ def check_headers(headers): "The Status header cannot be used; it conflicts with CGI " "script, and HTTP status is not given through headers " "(value: %r)." % value) - header_names[name.lower()] = None assert_('\n' not in name and ':' not in name, "Header names may not contain ':' or '\\n': %r" % name) assert_(header_re.search(name), "Bad header name: %r" % name) diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index 121e44023c1..f441376d09c 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -135,8 +135,7 @@ from decimal import Decimal import http.client import urllib.parse -# XXX RUSTPYTHON TODO: pyexpat -# from xml.parsers import expat +from xml.parsers import expat import errno from io import BytesIO try: @@ -246,41 +245,15 @@ def __repr__(self): ## # Backwards compatibility - boolean = Boolean = bool -## -# Wrapper for XML-RPC DateTime values. This converts a time value to -# the format used by XML-RPC. -# <p> -# The value can be given as a datetime object, as a string in the -# format "yyyymmddThh:mm:ss", as a 9-item time tuple (as returned by -# time.localtime()), or an integer value (as returned by time.time()). -# The wrapper uses time.localtime() to convert an integer to a time -# tuple. -# -# @param value The time, given as a datetime object, an ISO 8601 string, -# a time tuple, or an integer time value. - -# Issue #13305: different format codes across platforms -_day0 = datetime(1, 1, 1) -def _try(fmt): - try: - return _day0.strftime(fmt) == '0001' - except ValueError: - return False -if _try('%Y'): # Mac OS X - def _iso8601_format(value): - return value.strftime("%Y%m%dT%H:%M:%S") -elif _try('%4Y'): # Linux - def _iso8601_format(value): - return value.strftime("%4Y%m%dT%H:%M:%S") -else: - def _iso8601_format(value): - return value.strftime("%Y%m%dT%H:%M:%S").zfill(17) -del _day0 -del _try +def _iso8601_format(value): + if value.tzinfo is not None: + # XML-RPC only uses the naive portion of the datetime + value = value.replace(tzinfo=None) + # XML-RPC doesn't use '-' separator in the date part + return value.isoformat(timespec='seconds').replace('-', '') def _strftime(value): @@ -851,9 +824,9 @@ def __init__(self, results): def __getitem__(self, i): item = self.results[i] - if type(item) == type({}): + if isinstance(item, dict): raise Fault(item['faultCode'], item['faultString']) - elif type(item) == type([]): + elif isinstance(item, list): return item[0] else: raise ValueError("unexpected type in multicall result") @@ -1340,10 +1313,7 @@ def parse_response(self, response): p, u = self.getparser() - while 1: - data = stream.read(1024) - if not data: - break + while data := stream.read(1024): if self.verbose: print("body:", repr(data)) p.feed(data) diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 69a260f5b12..3e6871157d0 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. @@ -268,17 +268,11 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): except Fault as fault: response = dumps(fault, allow_none=self.allow_none, encoding=self.encoding) - except: - # report exception back to server - exc_type, exc_value, exc_tb = sys.exc_info() - try: - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none, - ) - finally: - # Break reference cycle - exc_type = exc_value = exc_tb = None + except BaseException as exc: + response = dumps( + Fault(1, "%s:%s" % (type(exc), exc)), + encoding=self.encoding, allow_none=self.allow_none, + ) return response.encode(self.encoding, 'xmlcharrefreplace') @@ -368,16 +362,11 @@ def system_multicall(self, call_list): {'faultCode' : fault.faultCode, 'faultString' : fault.faultString} ) - except: - exc_type, exc_value, exc_tb = sys.exc_info() - try: - results.append( - {'faultCode' : 1, - 'faultString' : "%s:%s" % (exc_type, exc_value)} - ) - finally: - # Break reference cycle - exc_type = exc_value = exc_tb = None + except BaseException as exc: + results.append( + {'faultCode' : 1, + 'faultString' : "%s:%s" % (type(exc), exc)} + ) return results def _dispatch(self, method, params): @@ -440,7 +429,7 @@ class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): # Class attribute listing the accessible path components; # paths not on this list will result in a 404 error. - rpc_paths = ('/', '/RPC2') + rpc_paths = ('/', '/RPC2', '/pydoc.css') #if not None, encode responses larger than this, if possible encode_threshold = 1400 #a common MTU @@ -589,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = False # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception @@ -634,19 +624,14 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): try: response = self.dispatchers[path]._marshaled_dispatch( data, dispatch_method, path) - except: + except BaseException as exc: # report low level exception back to server # (each dispatcher should have handled their own # exceptions) - exc_type, exc_value = sys.exc_info()[:2] - try: - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none) - response = response.encode(self.encoding, 'xmlcharrefreplace') - finally: - # Break reference cycle - exc_type = exc_value = None + response = dumps( + Fault(1, "%s:%s" % (type(exc), exc)), + encoding=self.encoding, allow_none=self.allow_none) + response = response.encode(self.encoding, 'xmlcharrefreplace') return response class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher): @@ -736,9 +721,7 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?((?:\w|\.)+))\b') - while 1: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) @@ -747,10 +730,10 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): url = escape(all).replace('"', '&quot;') results.append('<a href="%s">%s</a>' % (url, url)) elif rfc: - url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif pep: - url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://peps.python.org/pep-%04d/' % int(pep) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif text[end:end+1] == '(': results.append(self.namelink(name, methods, funcs, classes)) @@ -801,7 +784,7 @@ def docserver(self, server_name, package_documentation, methods): server_name = self.escape(server_name) head = '<big><big><strong>%s</strong></big></big>' % server_name - result = self.heading(head, '#ffffff', '#7799ee') + result = self.heading(head) doc = self.markup(package_documentation, self.preformat, fdict) doc = doc and '<tt>%s</tt>' % doc @@ -812,10 +795,25 @@ def docserver(self, server_name, package_documentation, methods): for key, value in method_items: contents.append(self.docroutine(value, key, funcs=fdict)) result = result + self.bigsection( - 'Methods', '#ffffff', '#eeaa77', ''.join(contents)) + 'Methods', 'functions', ''.join(contents)) return result + + def page(self, title, contents): + """Format an HTML page.""" + css_path = "/pydoc.css" + css_link = ( + '<link rel="stylesheet" type="text/css" href="%s">' % + css_path) + return '''\ +<!DOCTYPE> +<html lang="en"> +<head> +<meta charset="utf-8"> +<title>Python: %s</title> +%s</head><body>%s</body></html>''' % (title, css_link, contents) + class XMLRPCDocGenerator: """Generates documentation for an XML-RPC server. @@ -907,6 +905,12 @@ class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): for documentation. """ + def _get_css(self, url): + path_here = os.path.dirname(os.path.realpath(__file__)) + css_path = os.path.join(path_here, "..", "pydoc_data", "_pydoc.css") + with open(css_path, mode="rb") as fp: + return fp.read() + def do_GET(self): """Handles the HTTP GET request. @@ -918,9 +922,15 @@ def do_GET(self): self.report_404() return - response = self.server.generate_html_documentation().encode('utf-8') + if self.path.endswith('.css'): + content_type = 'text/css' + response = self._get_css(self.path) + else: + content_type = 'text/html' + response = self.server.generate_html_documentation().encode('utf-8') + self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) diff --git a/Lib/zipfile.py b/Lib/zipfile/__init__.py similarity index 83% rename from Lib/zipfile.py rename to Lib/zipfile/__init__.py index ef1eb47f9f1..c01f13729e1 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile/__init__.py @@ -6,26 +6,13 @@ import binascii import importlib.util import io -import itertools -try: - import os -except ImportError: - import _dummy_os as os -import posixpath -try: - import shutil -except ImportError: - pass +import os +import shutil import stat import struct import sys -try: - import threading -except ImportError: - import dummy_threading as threading +import threading import time -import contextlib -import pathlib try: import zlib # We may need its compression method @@ -132,6 +119,32 @@ class LargeZipFile(Exception): _CD_EXTERNAL_FILE_ATTRIBUTES = 17 _CD_LOCAL_HEADER_OFFSET = 18 +# General purpose bit flags +# Zip Appnote: 4.4.4 general purpose bit flag: (2 bytes) +_MASK_ENCRYPTED = 1 << 0 +# Bits 1 and 2 have different meanings depending on the compression used. +_MASK_COMPRESS_OPTION_1 = 1 << 1 +# _MASK_COMPRESS_OPTION_2 = 1 << 2 +# _MASK_USE_DATA_DESCRIPTOR: If set, crc-32, compressed size and uncompressed +# size are zero in the local header and the real values are written in the data +# descriptor immediately following the compressed data. +_MASK_USE_DATA_DESCRIPTOR = 1 << 3 +# Bit 4: Reserved for use with compression method 8, for enhanced deflating. +# _MASK_RESERVED_BIT_4 = 1 << 4 +_MASK_COMPRESSED_PATCH = 1 << 5 +_MASK_STRONG_ENCRYPTION = 1 << 6 +# _MASK_UNUSED_BIT_7 = 1 << 7 +# _MASK_UNUSED_BIT_8 = 1 << 8 +# _MASK_UNUSED_BIT_9 = 1 << 9 +# _MASK_UNUSED_BIT_10 = 1 << 10 +_MASK_UTF_FILENAME = 1 << 11 +# Bit 12: Reserved by PKWARE for enhanced compression. +# _MASK_RESERVED_BIT_12 = 1 << 12 +# _MASK_ENCRYPTED_CENTRAL_DIR = 1 << 13 +# Bit 14, 15: Reserved by PKWARE +# _MASK_RESERVED_BIT_14 = 1 << 14 +# _MASK_RESERVED_BIT_15 = 1 << 15 + # The "local file header" structure, magic number, size, and indices # (section V.A in the format document) structFileHeader = "<4s2B4HL2L2H" @@ -175,26 +188,42 @@ class LargeZipFile(Exception): _DD_SIGNATURE = 0x08074b50 -_EXTRA_FIELD_STRUCT = struct.Struct('<HH') - -def _strip_extra(extra, xids): - # Remove Extra Fields with specified IDs. - unpack = _EXTRA_FIELD_STRUCT.unpack - modified = False - buffer = [] - start = i = 0 - while i + 4 <= len(extra): - xid, xlen = unpack(extra[i : i + 4]) - j = i + 4 + xlen - if xid in xids: - if i != start: - buffer.append(extra[start : i]) - start = j - modified = True - i = j - if not modified: - return extra - return b''.join(buffer) + +class _Extra(bytes): + FIELD_STRUCT = struct.Struct('<HH') + + def __new__(cls, val, id=None): + return super().__new__(cls, val) + + def __init__(self, val, id=None): + self.id = id + + @classmethod + def read_one(cls, raw): + try: + xid, xlen = cls.FIELD_STRUCT.unpack(raw[:4]) + except struct.error: + xid = None + xlen = 0 + return cls(raw[:4+xlen], xid), raw[4+xlen:] + + @classmethod + def split(cls, data): + # use memoryview for zero-copy slices + rest = memoryview(data) + while rest: + extra, rest = _Extra.read_one(rest) + yield extra + + @classmethod + def strip(cls, data, xids): + """Remove Extra fields with specified IDs.""" + return b''.join( + ex + for ex in cls.split(data) + if ex.id not in xids + ) + def _check_zipfile(fp): try: @@ -216,7 +245,7 @@ def is_zipfile(filename): else: with open(filename, "rb") as fp: result = _check_zipfile(fp) - except OSError: + except (OSError, BadZipFile): pass return result @@ -224,16 +253,15 @@ def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 + offset -= sizeEndCentDir64Locator + if offset < 0: + # The file is not large enough to contain a ZIP64 # end-of-archive record, so just return the end record we were given. return endrec - + fpin.seek(offset) data = fpin.read(sizeEndCentDir64Locator) if len(data) != sizeEndCentDir64Locator: - return endrec + raise OSError("Unknown I/O error") sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) if sig != stringEndArchive64Locator: return endrec @@ -241,16 +269,33 @@ def _EndRecData64(fpin, offset, endrec): if diskno != 0 or disks > 1: raise BadZipFile("zipfiles that span multiple disks are not supported") - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: - return endrec + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig @@ -260,6 +305,7 @@ def _EndRecData64(fpin, offset, endrec): endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz return endrec @@ -280,7 +326,7 @@ def _EndRecData(fpin): fpin.seek(-sizeEndCentDir, 2) except OSError: return None - data = fpin.read() + data = fpin.read(sizeEndCentDir) if (len(data) == sizeEndCentDir and data[0:4] == stringEndArchive and data[-2:] == b"\000\000"): @@ -293,16 +339,16 @@ def _EndRecData(fpin): endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" # record signature. The comment is the last item in the ZIP file and may be # up to 64K long. It is assumed that the "end of central directory" magic # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) fpin.seek(maxCommentStart, 0) - data = fpin.read() + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) start = data.rfind(stringEndArchive) if start >= 0: # found the magic number; attempt to unpack and interpret @@ -317,14 +363,31 @@ def _EndRecData(fpin): endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) + return _EndRecData64(fpin, maxCommentStart + start, endrec) # Unable to find a valid end of central directory structure return None - -class ZipInfo (object): +def _sanitize_filename(filename): + """Terminate the file name at the first null byte and + ensure paths always use forward slashes as the directory separator.""" + + # Terminate the file name at the first null byte. Null bytes in file + # names are used as tricks by viruses in archives. + null_byte = filename.find(chr(0)) + if null_byte >= 0: + filename = filename[0:null_byte] + # This is used to ensure paths in generated ZIP files always use + # forward slashes as the directory separator, as required by the + # ZIP format specification. + if os.sep != "/" and os.sep in filename: + filename = filename.replace(os.sep, "/") + if os.altsep and os.altsep != "/" and os.altsep in filename: + filename = filename.replace(os.altsep, "/") + return filename + + +class ZipInfo: """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( @@ -332,7 +395,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - '_compresslevel', + 'compress_level', 'comment', 'extra', 'create_system', @@ -348,21 +411,15 @@ class ZipInfo (object): 'compress_size', 'file_size', '_raw_time', + '_end_offset', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.orig_filename = filename # Original file name in archive - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") + # Terminate the file name at the first null byte and + # ensure paths always use forward slashes as the directory separator. + filename = _sanitize_filename(filename) self.filename = filename # Normalized file name self.date_time = date_time # year, month, day, hour, min, sec @@ -372,7 +429,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self._compresslevel = None # Level for the compressor + self.compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -389,10 +446,20 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.external_attr = 0 # External file attributes self.compress_size = 0 # Size of the compressed file self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file + # Maintain backward compatibility with the old protected attribute name. + @property + def _compresslevel(self): + return self.compress_level + + @_compresslevel.setter + def _compresslevel(self, value): + self.compress_level = value + def __repr__(self): result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] if self.compress_type != ZIP_STORED: @@ -416,11 +483,16 @@ def __repr__(self): return ''.join(result) def FileHeader(self, zip64=None): - """Return the per-file header as a bytes object.""" + """Return the per-file header as a bytes object. + + When the optional zip64 arg is None rather than a bool, we will + decide based upon the file_size and compress_size, if known, + False otherwise. + """ dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: + if self.flag_bits & _MASK_USE_DATA_DESCRIPTOR: # Set these to zero because we write them after the file data CRC = compress_size = file_size = 0 else: @@ -432,16 +504,13 @@ def FileHeader(self, zip64=None): min_version = 0 if zip64 is None: + # We always explicitly pass zip64 within this module.... This + # remains for anyone using ZipInfo.FileHeader as a public API. zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT if zip64: fmt = '<HHQQ' extra = extra + struct.pack(fmt, 1, struct.calcsize(fmt)-4, file_size, compress_size) - if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension file_size = 0xffffffff compress_size = 0xffffffff min_version = ZIP64_VERSION @@ -465,9 +534,9 @@ def _encodeFilenameFlags(self): try: return self.filename.encode('ascii'), self.flag_bits except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 + return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME - def _decodeExtra(self): + def _decodeExtra(self, filename_crc): # Try to decode the extra field. extra = self.extra unpack = struct.unpack @@ -493,6 +562,22 @@ def _decodeExtra(self): except struct.error: raise BadZipFile(f"Corrupt zip64 extra field. " f"{field} not found.") from None + elif tp == 0x7075: + data = extra[4:ln+4] + # Unicode Path Extra Field + try: + up_version, up_name_crc = unpack('<BL', data[:5]) + if up_version == 1 and up_name_crc == filename_crc: + up_unicode_name = data[5:].decode('utf-8') + if up_unicode_name: + self.filename = _sanitize_filename(up_unicode_name) + else: + import warnings + warnings.warn("Empty unicode path extra field (0x7075)", stacklevel=2) + except struct.error as e: + raise BadZipFile("Corrupt unicode path extra field (0x7075)") from e + except UnicodeDecodeError as e: + raise BadZipFile('Corrupt unicode path extra field (0x7075): invalid utf-8 bytes') from e extra = extra[ln+4:] @@ -536,7 +621,15 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): def is_dir(self): """Return True if this archive member is a directory.""" - return self.filename[-1] == '/' + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False # ZIP encryption uses the CRC32 one-byte primitive for scrambling some @@ -740,7 +833,10 @@ def seek(self, offset, whence=0): raise ValueError("Can't reposition in the ZIP file while " "there is an open writing handle on it. " "Close the writing handle before trying to read.") - self._file.seek(offset, whence) + if whence == os.SEEK_CUR: + self._file.seek(self._pos + offset) + else: + self._file.seek(offset, whence) self._pos = self._file.tell() return self._pos @@ -830,13 +926,14 @@ def __init__(self, fileobj, mode, zipinfo, pwd=None, self._orig_compress_size = zipinfo.compress_size self._orig_file_size = zipinfo.file_size self._orig_start_crc = self._running_crc + self._orig_crc = self._expected_crc self._seekable = True except AttributeError: pass self._decrypter = None if pwd: - if zipinfo.flag_bits & 0x8: + if zipinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: # compare against the file type from extended local headers check_byte = (zipinfo._raw_time >> 8) & 0xff else: @@ -862,7 +959,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1052,17 +1149,17 @@ def seekable(self): raise ValueError("I/O operation on closed file.") return self._seekable - def seek(self, offset, whence=0): + def seek(self, offset, whence=os.SEEK_SET): if self.closed: raise ValueError("seek on closed file.") if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") curr_pos = self.tell() - if whence == 0: # Seek from start of file + if whence == os.SEEK_SET: new_pos = offset - elif whence == 1: # Seek from current position + elif whence == os.SEEK_CUR: new_pos = curr_pos + offset - elif whence == 2: # Seek from EOF + elif whence == os.SEEK_END: new_pos = self._orig_file_size + offset else: raise ValueError("whence must be os.SEEK_SET (0), " @@ -1081,10 +1178,25 @@ def seek(self, offset, whence=0): # Just move the _offset index if the new position is in the _readbuffer self._offset = buff_offset read_offset = 0 + # Fast seek uncompressed unencrypted file + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset != 0: + # disable CRC checking after first seeking - it would be invalid + self._expected_crc = None + # seek actual file taking already buffered data into account + read_offset -= len(self._readbuffer) - self._offset + self._fileobj.seek(read_offset, os.SEEK_CUR) + self._left -= read_offset + self._compress_left -= read_offset + self._eof = self._left <= 0 + read_offset = 0 + # flush read buffer + self._readbuffer = b'' + self._offset = 0 elif read_offset < 0: # Position is before the current position. Reset the ZipExtFile self._fileobj.seek(self._orig_compress_start) self._running_crc = self._orig_start_crc + self._expected_crc = self._orig_crc self._compress_left = self._orig_compress_size self._left = self._orig_file_size self._readbuffer = b'' @@ -1117,7 +1229,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compresslevel) + zinfo.compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1126,6 +1238,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1164,21 +1284,20 @@ def close(self): self._zinfo.CRC = self._crc self._zinfo.file_size = self._file_size + if not self._zip64: + if self._file_size > ZIP64_LIMIT: + raise RuntimeError("File size too large, try using force_zip64") + if self._compress_size > ZIP64_LIMIT: + raise RuntimeError("Compressed size too large, try using force_zip64") + # Write updated header info - if self._zinfo.flag_bits & 0x08: + if self._zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: # Write CRC and file sizes after the file data fmt = '<LLQQ' if self._zip64 else '<LLLL' self._fileobj.write(struct.pack(fmt, _DD_SIGNATURE, self._zinfo.CRC, self._zinfo.compress_size, self._zinfo.file_size)) self._zipfile.start_dir = self._fileobj.tell() else: - if not self._zip64: - if self._file_size > ZIP64_LIMIT: - raise RuntimeError( - 'File size unexpectedly exceeded ZIP64 limit') - if self._compress_size > ZIP64_LIMIT: - raise RuntimeError( - 'Compressed size unexpectedly exceeded ZIP64 limit') # Seek backwards and write file header (which will now include # correct CRC and file sizes) @@ -1223,7 +1342,7 @@ class ZipFile: _windows_illegal_name_trans_table = None def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, - compresslevel=None, *, strict_timestamps=True): + compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): @@ -1242,6 +1361,12 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self.pwd = None self._comment = b'' self._strict_timestamps = strict_timestamps + self.metadata_encoding = metadata_encoding + + # Check that we don't try to write with nonconforming codecs + if self.metadata_encoding and mode != 'r': + raise ValueError( + "metadata_encoding is only supported for reading files") # Check if we were passed a file-like object if isinstance(file, os.PathLike): @@ -1349,9 +1474,6 @@ def _RealGetContents(self): # "concat" is zero, unless zip was concatenated to another file concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: inferred = concat + offset_cd @@ -1374,13 +1496,14 @@ def _RealGetContents(self): if self.debug > 2: print(centdir) filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - flags = centdir[5] - if flags & 0x800: + orig_filename_crc = crc32(filename) + flags = centdir[_CD_FLAG_BITS] + if flags & _MASK_UTF_FILENAME: # UTF-8 file names extension filename = filename.decode('utf-8') else: # Historical ZIP filename encoding - filename = filename.decode('cp437') + filename = filename.decode(self.metadata_encoding or 'cp437') # Create ZipInfo instance to store file information x = ZipInfo(filename) x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) @@ -1397,8 +1520,7 @@ def _RealGetContents(self): x._raw_time = t x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() + x._decodeExtra(orig_filename_crc) x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1411,6 +1533,11 @@ def _RealGetContents(self): if self.debug > 2: print("total", total) + end_offset = self.start_dir + for zinfo in reversed(sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset)): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset def namelist(self): """Return a list of file names in the archive.""" @@ -1431,7 +1558,10 @@ def printdir(self, file=None): file=file) def testzip(self): - """Read all the files and check the CRC.""" + """Read all the files and check the CRC. + + Return None if all files could be read successfully, or the name + of the offending file otherwise.""" chunk_size = 2 ** 20 for zinfo in self.filelist: try: @@ -1480,7 +1610,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1502,8 +1633,6 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if pwd and (mode == "w"): raise ValueError("pwd is only supported for reading files") if not self.fp: @@ -1517,7 +1646,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1545,39 +1674,54 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) + zef_file.seek(fheader[_FH_EXTRA_FIELD_LENGTH], whence=1) - if zinfo.flag_bits & 0x20: + if zinfo.flag_bits & _MASK_COMPRESSED_PATCH: # Zip 2.7: compressed patched data raise NotImplementedError("compressed patched data (flag bit 5)") - if zinfo.flag_bits & 0x40: + if zinfo.flag_bits & _MASK_STRONG_ENCRYPTION: # strong encryption raise NotImplementedError("strong encryption (flag bit 6)") - if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800: + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & _MASK_UTF_FILENAME: # UTF-8 filename fname_str = fname.decode("utf-8") else: - fname_str = fname.decode("cp437") + fname_str = fname.decode(self.metadata_encoding or "cp437") if fname_str != zinfo.orig_filename: raise BadZipFile( 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + if zinfo._end_offset == zinfo.header_offset: + import warnings + warnings.warn( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)", + skip_file_prefixes=(os.path.dirname(__file__),)) + else: + raise BadZipFile( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)") + # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 + is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED if is_encrypted: if not pwd: pwd = self.pwd + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if not pwd: raise RuntimeError("File %r is encrypted, password " "required for extraction" % name) else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise @@ -1600,16 +1744,17 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.flag_bits = 0x00 if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 if not self._seekable: - zinfo.flag_bits |= 0x08 + zinfo.flag_bits |= _MASK_USE_DATA_DESCRIPTOR if not zinfo.external_attr: zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) + zip64 = force_zip64 or (zinfo.file_size * 1.05 > ZIP64_LIMIT) + if not self._allowZip64 and zip64: + raise LargeZipFile("Filesize would require ZIP64 extensions") if self._seekable: self.fp.seek(self.start_dir) @@ -1627,7 +1772,8 @@ def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + specify a different directory using `path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1640,7 +1786,8 @@ def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working directory. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned - by namelist(). + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() @@ -1662,8 +1809,8 @@ def _sanitize_windows_name(cls, arcname, pathsep): table = str.maketrans(illegal, '_' * len(illegal)) cls._windows_illegal_name_trans_table = table arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(pathsep)) + # remove trailing dots and spaces + arcname = (x.rstrip(' .') for x in arcname.split(pathsep)) # rejoin, removing empty parts. arcname = pathsep.join(x for x in arcname if x) return arcname @@ -1691,17 +1838,24 @@ def _extract_member(self, member, targetpath, pwd): # filter illegal characters on Windows arcname = self._sanitize_windows_name(arcname, os.path.sep) + if not arcname and not member.is_dir(): + raise ValueError("Empty filename.") + targetpath = os.path.join(targetpath, arcname) targetpath = os.path.normpath(targetpath) # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if member.is_dir(): if not os.path.isdir(targetpath): - os.mkdir(targetpath) + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise return targetpath with self.open(member, pwd=pwd) as source, \ @@ -1751,6 +1905,7 @@ def write(self, filename, arcname=None, if zinfo.is_dir(): zinfo.compress_size = 0 zinfo.CRC = 0 + self.mkdir(zinfo) else: if compress_type is not None: zinfo.compress_type = compress_type @@ -1758,27 +1913,10 @@ def write(self, filename, arcname=None, zinfo.compress_type = self.compression if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel else: - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel - if zinfo.is_dir(): - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - - self._writecheck(zinfo) - self._didModify = True - - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - self.start_dir = self.fp.tell() - else: with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) @@ -1795,8 +1933,8 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel - if zinfo.filename[-1] == '/': + zinfo.compress_level = self.compresslevel + if zinfo.filename.endswith('/'): zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag else: @@ -1816,13 +1954,48 @@ def writestr(self, zinfo_or_arcname, data, zinfo.compress_type = compress_type if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: with self.open(zinfo, mode='w') as dest: dest.write(data) + def mkdir(self, zinfo_or_directory_name, mode=511): + """Creates a directory inside the zip archive.""" + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + if not zinfo.is_dir(): + raise ValueError("The given ZipInfo does not describe a directory") + elif isinstance(zinfo_or_directory_name, str): + directory_name = zinfo_or_directory_name + if not directory_name.endswith("/"): + directory_name += "/" + zinfo = ZipInfo(directory_name) + zinfo.compress_size = 0 + zinfo.CRC = 0 + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + raise TypeError("Expected type str or ZipInfo") + + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + + self._writecheck(zinfo) + self._didModify = True + + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self.fp.write(zinfo.FileHeader(False)) + self.start_dir = self.fp.tell() + def __del__(self): """Call the "close()" method in case the user forgot.""" self.close() @@ -1875,7 +2048,7 @@ def _write_end_record(self): min_version = 0 if extra: # Append a ZIP64 field to the extra's - extra_data = _strip_extra(extra_data, (1,)) + extra_data = _Extra.strip(extra_data, (1,)) extra_data = struct.pack( '<HH' + 'Q'*len(extra), 1, 8*len(extra), *extra) + extra_data @@ -1922,7 +2095,7 @@ def _write_end_record(self): " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) @@ -2124,300 +2297,6 @@ def _compile(file, optimize=-1): return (fname, archivename) -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = dict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, ZipFile): - return cls(source) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - else: - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *other) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) - - def main(args=None): import argparse @@ -2434,11 +2313,15 @@ def main(args=None): help='Create zipfile from sources') group.add_argument('-t', '--test', metavar='<zipfile>', help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='<encoding>', + help='Specify encoding of member names for -l, -e and -t') args = parser.parse_args(args) + encoding = args.metadata_encoding + if args.test is not None: src = args.test - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: badfile = zf.testzip() if badfile: print("The following enclosed file is corrupted: {!r}".format(badfile)) @@ -2446,15 +2329,20 @@ def main(args=None): elif args.list is not None: src = args.list - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: zf.printdir() elif args.extract is not None: src, curdir = args.extract - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: zf.extractall(curdir) elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + zip_name = args.create.pop(0) files = args.create @@ -2479,5 +2367,9 @@ def addToZip(zf, path, zippath): addToZip(zf, path, zippath) -if __name__ == "__main__": - main() +from ._path import ( # noqa: E402 + Path, + + # used privately for tests + CompleteDirs, # noqa: F401 +) diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py new file mode 100644 index 00000000000..868d99efc3c --- /dev/null +++ b/Lib/zipfile/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py new file mode 100644 index 00000000000..02f81171b4f --- /dev/null +++ b/Lib/zipfile/_path/__init__.py @@ -0,0 +1,452 @@ +""" +A Path-like interface for zipfiles. + +This codebase is shared between zipfile.Path in the stdlib +and zipp in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + +import contextlib +import io +import itertools +import pathlib +import posixpath +import re +import stat +import sys +import zipfile + +from .glob import Translator + +__all__ = ['Path'] + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path. + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + + Multiple separators are treated like a single. + + >>> list(_ancestry('//b//d///f//')) + ['//b//d///f', '//b//d', '//b'] + """ + path = path.rstrip(posixpath.sep) + while path.rstrip(posixpath.sep): + yield path + path, tail = posixpath.split(path) + + +_dedupe = dict.fromkeys +"""Deduplicate an iterable in original order""" + + +def _difference(minuend, subtrahend): + """ + Return items in minuend not in subtrahend, retaining order + with O(1) lookup. + """ + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + +class InitializedState: + """ + Mix-in to save the initialization state for pickling. + """ + + def __init__(self, *args, **kwargs): + self.__args = args + self.__kwargs = kwargs + super().__init__(*args, **kwargs) + + def __getstate__(self): + return self.__args, self.__kwargs + + def __setstate__(self, state): + args, kwargs = state + super().__init__(*args, **kwargs) + + +class CompleteDirs(InitializedState, zipfile.ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt'])) + ['foo/', 'foo/bar/'] + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt', 'foo/bar/'])) + ['foo/'] + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + as_dirs = (p + posixpath.sep for p in parents) + return _dedupe(_difference(as_dirs, names)) + + def namelist(self): + names = super().namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + def getinfo(self, name): + """ + Supplement getinfo for implied dirs. + """ + try: + return super().getinfo(name) + except KeyError: + if not name.endswith('/') or name not in self._name_set(): + raise + return zipfile.ZipInfo(filename=name) + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, zipfile.ZipFile): + return cls(source) + + # Only allow for FastLookup when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + source.__class__ = cls + return source + + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super().namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super()._name_set() + return self.__lookup + +def _extract_text_encoding(encoding=None, *args, **kwargs): + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi + return io.text_encoding(encoding, stack_level), args, kwargs + + +class Path: + """ + A :class:`importlib.resources.abc.Traversable` interface for zip files. + + Implements many of the features users enjoy from + :class:`pathlib.Path`. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'mem/abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> path = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = path.iterdir() + >>> a + Path('mem/abcde.zip', 'a.txt') + >>> b + Path('mem/abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('mem/abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text(encoding='utf-8') + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> import os + >>> str(c).replace(os.sep, posixpath.sep) + 'mem/abcde.zip/b/c.txt' + + At the root, ``name``, ``filename``, and ``parent`` + resolve to the zipfile. + + >>> str(path) + 'mem/abcde.zip/' + >>> path.name + 'abcde.zip' + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) + 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + """ + Construct a Path from a ZipFile or filename. + + Note: When the source is an existing ZipFile object, + its type (__class__) will be mutated to a + specialized type. If the caller wishes to retain the + original type, the caller should either create a + separate ZipFile object or pass a filename. + """ + self.root = FastLookup.make(root) + self.at = at + + def __eq__(self, other): + """ + >>> Path(zipfile.ZipFile(io.BytesIO(), 'w')) == 'foo' + False + """ + if self.__class__ is not other.__class__: + return NotImplemented + return (self.root, self.at) == (other.root, other.at) + + def __hash__(self): + return hash((self.root, self.at)) + + def open(self, mode='r', *args, pwd=None, **kwargs): + """ + Open this entry as text or binary following the semantics + of ``pathlib.Path.open()`` by passing arguments through + to io.TextIOWrapper(). + """ + if self.is_dir(): + raise IsADirectoryError(self) + zip_mode = mode[0] + if zip_mode == 'r' and not self.exists(): + raise FileNotFoundError(self) + stream = self.root.open(self.at, zip_mode, pwd=pwd) + if 'b' in mode: + if args or kwargs: + raise ValueError("encoding args invalid for binary operation") + return stream + # Text mode: + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + return io.TextIOWrapper(stream, encoding, *args, **kwargs) + + def _base(self): + return pathlib.PurePosixPath(self.at) if self.at else self.filename + + @property + def name(self): + return self._base().name + + @property + def suffix(self): + return self._base().suffix + + @property + def suffixes(self): + return self._base().suffixes + + @property + def stem(self): + return self._base().stem + + @property + def filename(self): + return pathlib.Path(self.root.filename).joinpath(self.at) + + def read_text(self, *args, **kwargs): + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + with self.open('r', encoding, *args, **kwargs) as strm: + return strm.read() + + def read_bytes(self): + with self.open('rb') as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return self.__class__(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return self.exists() and not self.is_dir() + + def exists(self): + return self.at in self.root._name_set() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self.root.namelist()) + return filter(self._is_child, subs) + + def match(self, path_pattern): + return pathlib.PurePosixPath(self.at).match(path_pattern) + + def is_symlink(self): + """ + Return whether this path is a symlink. + """ + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) + + def glob(self, pattern): + if not pattern: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + + prefix = re.escape(self.at) + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch + return map(self._next, filter(matches, self.root.namelist())) + + def rglob(self, pattern): + return self.glob(f'**/{pattern}') + + def relative_to(self, other, *extra): + return posixpath.relpath(str(self), str(other.joinpath(*extra))) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, *other): + next = posixpath.join(self.at, *other) + return self._next(self.root.resolve_dir(next)) + + __truediv__ = joinpath + + @property + def parent(self): + if not self.at: + return self.filename.parent + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py new file mode 100644 index 00000000000..4ed74cc48d9 --- /dev/null +++ b/Lib/zipfile/_path/glob.py @@ -0,0 +1,113 @@ +import os +import re + +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) + + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + """ + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.match_dirs(self.translate_core(pattern))) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def match_dirs(self, pattern): + """ + Ensure that zipfile.Path directory names are matched. + + zipfile.Path directory names always end in a slash. + """ + return rf'{pattern}[/]?' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) + + +def separate(pattern): + """ + Separate out character sets to avoid translating their contents. + + >>> [m.group(0) for m in separate('*.txt')] + ['*.txt'] + >>> [m.group(0) for m in separate('a[?]txt')] + ['a', '[?]', 'txt'] + """ + return re.finditer(r'([^\[]+)|(?P<set>[\[].*?[\]])|([\[][^\]]*$)', pattern) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 25eaee9c0f2..444c9dd11d8 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths @@ -15,14 +13,13 @@ #from importlib import _bootstrap_external #from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external -from _frozen_importlib_external import _unpack_uint16, _unpack_uint32 +from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open import marshal # for loads import sys # for modules import time # for mktime -import _warnings # For warn() __all__ = ['ZipImportError', 'zipimporter'] @@ -40,8 +37,14 @@ class ZipImportError(ImportError): _module_type = type(sys) END_CENTRAL_DIR_SIZE = 22 -STRING_END_ARCHIVE = b'PK\x05\x06' +END_CENTRAL_DIR_SIZE_64 = 56 +END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 +STRING_END_ARCHIVE = b'PK\x05\x06' # standard EOCD signature +STRING_END_LOCATOR_64 = b'PK\x06\x07' # Zip64 EOCD Locator signature +STRING_END_ZIP_64 = b'PK\x06\x06' # Zip64 EOCD signature MAX_COMMENT_LEN = (1 << 16) - 1 +MAX_UINT32 = 0xffffffff +ZIP64_EXTRA_TAG = 0x1 class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object @@ -63,8 +66,7 @@ class zipimporter(_bootstrap_external._LoaderBasics): # if found, or else read it from the archive. def __init__(self, path): if not isinstance(path, str): - import os - path = os.fsdecode(path) + raise TypeError(f"expected str, not {type(path)!r}") if not path: raise ZipImportError('archive path is empty', path=path) if alt_path_sep: @@ -89,12 +91,8 @@ def __init__(self, path): raise ZipImportError('not a Zip file', path=path) break - try: - files = _zip_directory_cache[path] - except KeyError: - files = _read_directory(path) - _zip_directory_cache[path] = files - self._files = files + if path not in _zip_directory_cache: + _zip_directory_cache[path] = _read_directory(path) self.archive = path # a prefix directory following the ZIP file path. self.prefix = _bootstrap_external._path_join(*prefix[::-1]) @@ -102,64 +100,6 @@ def __init__(self, path): self.prefix += path_sep - # Check whether we can satisfy the import of the module named by - # 'fullname', or whether it could be a portion of a namespace - # package. Return self if we can load it, a string containing the - # full path if it's a possible namespace portion, None if we - # can't load it. - def find_loader(self, fullname, path=None): - """find_loader(fullname, path=None) -> self, str or None. - - Search for a module specified by 'fullname'. 'fullname' must be the - fully qualified (dotted) module name. It returns the zipimporter - instance itself if the module was found, a string containing the - full path name if it's possibly a portion of a namespace package, - or None otherwise. The optional 'path' argument is ignored -- it's - there for compatibility with the importer protocol. - - Deprecated since Python 3.10. Use find_spec() instead. - """ - _warnings.warn("zipimporter.find_loader() is deprecated and slated for " - "removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - mi = _get_module_info(self, fullname) - if mi is not None: - # This is a module or package. - return self, [] - - # Not a module or regular package. See if this is a directory, and - # therefore possibly a portion of a namespace package. - - # We're only interested in the last path component of fullname - # earlier components are recorded in self.prefix. - modpath = _get_module_path(self, fullname) - if _is_dir(self, modpath): - # This is possibly a portion of a namespace - # package. Return the string representing its path, - # without a trailing separator. - return None, [f'{self.archive}{path_sep}{modpath}'] - - return None, [] - - - # Check whether we can satisfy the import of the module named by - # 'fullname'. Return self if we can, None if we can't. - def find_module(self, fullname, path=None): - """find_module(fullname, path=None) -> self or None. - - Search for a module specified by 'fullname'. 'fullname' must be the - fully qualified (dotted) module name. It returns the zipimporter - instance itself if the module was found, or None if it wasn't. - The optional 'path' argument is ignored -- it's there for compatibility - with the importer protocol. - - Deprecated since Python 3.10. Use find_spec() instead. - """ - _warnings.warn("zipimporter.find_module() is deprecated and slated for " - "removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - return self.find_loader(fullname, path)[0] - def find_spec(self, fullname, target=None): """Create a ModuleSpec for the specified module. @@ -211,9 +151,11 @@ def get_data(self, pathname): key = pathname[len(self.archive + path_sep):] try: - toc_entry = self._files[key] + toc_entry = self._get_files()[key] except KeyError: raise OSError(0, '', key) + if toc_entry is None: + return b'' return _get_data(self.archive, toc_entry) @@ -248,7 +190,7 @@ def get_source(self, fullname): fullpath = f'{path}.py' try: - toc_entry = self._files[fullpath] + toc_entry = self._get_files()[fullpath] except KeyError: # we have the module, but no source return None @@ -278,9 +220,11 @@ def load_module(self, fullname): Deprecated since Python 3.10. Use exec_module() instead. """ - msg = ("zipimport.zipimporter.load_module() is deprecated and slated for " - "removal in Python 3.12; use exec_module() instead") - _warnings.warn(msg, DeprecationWarning) + import warnings + warnings._deprecated("zipimport.zipimporter.load_module", + f"{warnings._DEPRECATED_MSG}; " + "use zipimport.zipimporter.exec_module() instead", + remove=(3, 15)) code, ispackage, modpath = _get_module_code(self, fullname) mod = sys.modules.get(fullname) if mod is None or not isinstance(mod, _module_type): @@ -313,28 +257,28 @@ def load_module(self, fullname): def get_resource_reader(self, fullname): - """Return the ResourceReader for a package in a zip file. - - If 'fullname' is a package within the zip file, return the - 'ResourceReader' object for the package. Otherwise return None. - """ - try: - if not self.is_package(fullname): - return None - except ZipImportError: - return None + """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader + return ZipReader(self, fullname) - def invalidate_caches(self): - """Reload the file data of the archive path.""" + def _get_files(self): + """Return the files within the archive path.""" try: - self._files = _read_directory(self.archive) - _zip_directory_cache[self.archive] = self._files - except ZipImportError: - _zip_directory_cache.pop(self.archive, None) - self._files = {} + files = _zip_directory_cache[self.archive] + except KeyError: + try: + files = _zip_directory_cache[self.archive] = _read_directory(self.archive) + except ZipImportError: + files = {} + + return files + + + def invalidate_caches(self): + """Invalidates the cache of file data of the archive path.""" + _zip_directory_cache.pop(self.archive, None) def __repr__(self): @@ -364,15 +308,15 @@ def _is_dir(self, path): # of a namespace package. We test by seeing if the name, with an # appended path separator, exists. dirpath = path + path_sep - # If dirpath is present in self._files, we have a directory. - return dirpath in self._files + # If dirpath is present in self._get_files(), we have a directory. + return dirpath in self._get_files() # Return some information about a module. def _get_module_info(self, fullname): path = _get_module_path(self, fullname) for suffix, isbytecode, ispackage in _zip_searchorder: fullpath = path + suffix - if fullpath in self._files: + if fullpath in self._get_files(): return ispackage return None @@ -406,16 +350,11 @@ def _read_directory(archive): raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive) with fp: + # GH-87235: On macOS all file descriptors for /dev/fd/N share the same + # file offset, reset the file offset after scanning the zipfile directory + # to not cause problems when some runs 'python3 /dev/fd/9 9<some_script' + start_offset = fp.tell() try: - fp.seek(-END_CENTRAL_DIR_SIZE, 2) - header_position = fp.tell() - buffer = fp.read(END_CENTRAL_DIR_SIZE) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(buffer) != END_CENTRAL_DIR_SIZE: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if buffer[:4] != STRING_END_ARCHIVE: - # Bad: End of Central Dir signature # Check if there's a comment. try: fp.seek(0, 2) @@ -423,98 +362,209 @@ def _read_directory(archive): except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - max_comment_start = max(file_size - MAX_COMMENT_LEN - - END_CENTRAL_DIR_SIZE, 0) + max_comment_plus_dirs_size = ( + MAX_COMMENT_LEN + END_CENTRAL_DIR_SIZE + + END_CENTRAL_DIR_SIZE_64 + END_CENTRAL_DIR_LOCATOR_SIZE_64) + max_comment_start = max(file_size - max_comment_plus_dirs_size, 0) try: fp.seek(max_comment_start) - data = fp.read() + data = fp.read(max_comment_plus_dirs_size) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) pos = data.rfind(STRING_END_ARCHIVE) - if pos < 0: + pos64 = data.rfind(STRING_END_ZIP_64) + + if (pos64 >= 0 and pos64+END_CENTRAL_DIR_SIZE_64+END_CENTRAL_DIR_LOCATOR_SIZE_64==pos): + # Zip64 at "correct" offset from standard EOCD + buffer = data[pos64:pos64 + END_CENTRAL_DIR_SIZE_64] + if len(buffer) != END_CENTRAL_DIR_SIZE_64: + raise ZipImportError( + f"corrupt Zip64 file: Expected {END_CENTRAL_DIR_SIZE_64} byte " + f"zip64 central directory, but read {len(buffer)} bytes.", + path=archive) + header_position = file_size - len(data) + pos64 + + central_directory_size = _unpack_uint64(buffer[40:48]) + central_directory_position = _unpack_uint64(buffer[48:56]) + num_entries = _unpack_uint64(buffer[24:32]) + elif pos >= 0: + buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] + if len(buffer) != END_CENTRAL_DIR_SIZE: + raise ZipImportError(f"corrupt Zip file: {archive!r}", + path=archive) + + header_position = file_size - len(data) + pos + + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + central_directory_size = _unpack_uint32(buffer[12:16]) + central_directory_position = _unpack_uint32(buffer[16:20]) + num_entries = _unpack_uint16(buffer[8:10]) + + # N.b. if someday you want to prefer the standard (non-zip64) EOCD, + # you need to adjust position by 76 for arc to be 0. + else: raise ZipImportError(f'not a Zip file: {archive!r}', path=archive) - buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] - if len(buffer) != END_CENTRAL_DIR_SIZE: - raise ZipImportError(f"corrupt Zip file: {archive!r}", - path=archive) - header_position = file_size - len(data) + pos - - header_size = _unpack_uint32(buffer[12:16]) - header_offset = _unpack_uint32(buffer[16:20]) - if header_position < header_size: - raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) - if header_position < header_offset: - raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) - header_position -= header_size - arc_offset = header_position - header_offset - if arc_offset < 0: - raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) - - files = {} - # Start of Central Directory - count = 0 - try: - fp.seek(header_position) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - while True: - buffer = fp.read(46) - if len(buffer) < 4: - raise EOFError('EOF read where not expected') - # Start of file header - if buffer[:4] != b'PK\x01\x02': - break # Bad: Central Dir File Header - if len(buffer) != 46: - raise EOFError('EOF read where not expected') - flags = _unpack_uint16(buffer[8:10]) - compress = _unpack_uint16(buffer[10:12]) - time = _unpack_uint16(buffer[12:14]) - date = _unpack_uint16(buffer[14:16]) - crc = _unpack_uint32(buffer[16:20]) - data_size = _unpack_uint32(buffer[20:24]) - file_size = _unpack_uint32(buffer[24:28]) - name_size = _unpack_uint16(buffer[28:30]) - extra_size = _unpack_uint16(buffer[30:32]) - comment_size = _unpack_uint16(buffer[32:34]) - file_offset = _unpack_uint32(buffer[42:46]) - header_size = name_size + extra_size + comment_size - if file_offset > header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + # XXX: These are cursory checks but are not as exact or strict as they + # could be. Checking the arc-adjusted value is probably good too. + if header_position < central_directory_size: + raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) + if header_position < central_directory_position: + raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) + header_position -= central_directory_size + # On just-a-zipfile these values are the same and arc_offset is zero; if + # the file has some bytes prepended, `arc_offset` is the number of such + # bytes. This is used for pex as well as self-extracting .exe. + arc_offset = header_position - central_directory_position + if arc_offset < 0: + raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) + + files = {} + # Start of Central Directory + count = 0 try: - name = fp.read(name_size) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(name) != name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - # On Windows, calling fseek to skip over the fields we don't use is - # slower than reading the data because fseek flushes stdio's - # internal buffers. See issue #8745. - try: - if len(fp.read(header_size - name_size)) != header_size - name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + fp.seek(header_position) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + while True: + buffer = fp.read(46) + if len(buffer) < 4: + raise EOFError('EOF read where not expected') + # Start of file header + if buffer[:4] != b'PK\x01\x02': + if count != num_entries: + raise ZipImportError( + f"mismatched num_entries: {count} should be {num_entries} in {archive!r}", + path=archive, + ) + break # Bad: Central Dir File Header + if len(buffer) != 46: + raise EOFError('EOF read where not expected') + flags = _unpack_uint16(buffer[8:10]) + compress = _unpack_uint16(buffer[10:12]) + time = _unpack_uint16(buffer[12:14]) + date = _unpack_uint16(buffer[14:16]) + crc = _unpack_uint32(buffer[16:20]) + data_size = _unpack_uint32(buffer[20:24]) + file_size = _unpack_uint32(buffer[24:28]) + name_size = _unpack_uint16(buffer[28:30]) + extra_size = _unpack_uint16(buffer[30:32]) + comment_size = _unpack_uint16(buffer[32:34]) + file_offset = _unpack_uint32(buffer[42:46]) + header_size = name_size + extra_size + comment_size - if flags & 0x800: - # UTF-8 file names extension - name = name.decode() - else: - # Historical ZIP filename encoding try: - name = name.decode('ascii') - except UnicodeDecodeError: - name = name.decode('latin1').translate(cp437_table) - - name = name.replace('/', path_sep) - path = _bootstrap_external._path_join(archive, name) - t = (path, compress, data_size, file_size, file_offset, time, date, crc) - files[name] = t - count += 1 + name = fp.read(name_size) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + if len(name) != name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + # On Windows, calling fseek to skip over the fields we don't use is + # slower than reading the data because fseek flushes stdio's + # internal buffers. See issue #8745. + try: + extra_data_len = header_size - name_size + extra_data = memoryview(fp.read(extra_data_len)) + + if len(extra_data) != extra_data_len: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + + if flags & 0x800: + # UTF-8 file names extension + name = name.decode() + else: + # Historical ZIP filename encoding + try: + name = name.decode('ascii') + except UnicodeDecodeError: + name = name.decode('latin1').translate(cp437_table) + + name = name.replace('/', path_sep) + path = _bootstrap_external._path_join(archive, name) + + # Ordering matches unpacking below. + if ( + file_size == MAX_UINT32 or + data_size == MAX_UINT32 or + file_offset == MAX_UINT32 + ): + # need to decode extra_data looking for a zip64 extra (which might not + # be present) + while extra_data: + if len(extra_data) < 4: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + tag = _unpack_uint16(extra_data[:2]) + size = _unpack_uint16(extra_data[2:4]) + if len(extra_data) < 4 + size: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + if tag == ZIP64_EXTRA_TAG: + if (len(extra_data) - 4) % 8 != 0: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + num_extra_values = (len(extra_data) - 4) // 8 + if num_extra_values > 3: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) + + # N.b. Here be dragons: the ordering of these is different than + # the header fields, and it's really easy to get it wrong since + # naturally-occurring zips that use all 3 are >4GB + if file_size == MAX_UINT32: + file_size = values.pop(0) + if data_size == MAX_UINT32: + data_size = values.pop(0) + if file_offset == MAX_UINT32: + file_offset = values.pop(0) + + break + + # For a typical zip, this bytes-slicing only happens 2-3 times, on + # small data like timestamps and filesizes. + extra_data = extra_data[4+size:] + else: + _bootstrap._verbose_message( + "zipimport: suspected zip64 but no zip64 extra for {!r}", + path, + ) + # XXX These two statements seem swapped because `central_directory_position` + # is a position within the actual file, but `file_offset` (when compared) is + # as encoded in the entry, not adjusted for this file. + # N.b. this must be after we've potentially read the zip64 extra which can + # change `file_offset`. + if file_offset > central_directory_position: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset + + t = (path, compress, data_size, file_size, file_offset, time, date, crc) + files[name] = t + count += 1 + finally: + fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) + + # Add implicit directories. + count = 0 + for name in list(files): + while True: + i = name.rstrip(path_sep).rfind(path_sep) + if i < 0: + break + name = name[:i + 1] + if name in files: + break + files[name] = None + count += 1 + if count: + _bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}', + count, archive) return files # During bootstrap, we may need to load the encodings @@ -648,7 +698,7 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes = _get_pyc_source(self, fullpath) if source_bytes is not None: source_hash = _imp.source_hash( - _bootstrap_external._RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) @@ -708,7 +758,7 @@ def _get_mtime_and_size_of_source(self, path): # strip 'c' or 'o' from *.py[co] assert path[-1:] in ('c', 'o') path = path[:-1] - toc_entry = self._files[path] + toc_entry = self._get_files()[path] # fetch the time stamp of the .py file for comparison # with an embedded pyc time stamp time = toc_entry[5] @@ -728,7 +778,7 @@ def _get_pyc_source(self, path): path = path[:-1] try: - toc_entry = self._files[path] + toc_entry = self._get_files()[path] except KeyError: return None else: @@ -744,7 +794,7 @@ def _get_module_code(self, fullname): fullpath = path + suffix _bootstrap._verbose_message('trying {}{}{}', self.archive, path_sep, fullpath, verbosity=2) try: - toc_entry = self._files[fullpath] + toc_entry = self._get_files()[fullpath] except KeyError: pass else: diff --git a/README.md b/README.md index 86d0738ec8e..6949c6e66e2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.13.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) @@ -66,7 +66,25 @@ Welcome to the magnificent Rust Python interpreter >>>>> ``` -You can install pip by +### venv + +Because RustPython currently doesn't provide a well-packaged installation, using venv helps to use pip easier. + +```sh +$ rustpython -m venv <your_env_name> +$ . <your_env_name>/bin/activate +$ python # now `python` is the alias of the RustPython for the new env +``` + +### PIP + +If you'd like to make https requests, you can enable the `ssl` feature, which +also lets you install the `pip` package manager. Note that on Windows, you may +need to install OpenSSL, or you can enable the `ssl-vendor` feature instead, +which compiles OpenSSL for you but requires a C compiler, perl, and `make`. +OpenSSL version 3 is expected and tested in CI. Older versions may not work. + +Once you've installed rustpython with SSL support, you can install pip by running: ```bash @@ -227,7 +245,7 @@ To enhance CPython compatibility, try to increase unittest coverage by checking Another approach is to checkout the source code: builtin functions and object methods are often the simplest and easiest way to contribute. -You can also simply run `python -I whats_left.py` to assist in finding any unimplemented +You can also simply run `python -I scripts/whats_left.py` to assist in finding any unimplemented method. ## Compiling to WebAssembly diff --git a/benches/_data/pypi_org__simple__psutil.json b/benches/_data/pypi_org__simple__psutil.json new file mode 100644 index 00000000000..91e2ff6b39e --- /dev/null +++ b/benches/_data/pypi_org__simple__psutil.json @@ -0,0 +1 @@ +{"alternate-locations":[],"files":[{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.1.tar.gz","hashes":{"sha256":"25c6caffbf00d8be77489391a784654e99fcbaf2a5278e80f748be4112ee0188"},"provenance":null,"requires-python":null,"size":44485,"upload-time":"2014-02-06T02:06:57.249874Z","url":"https://files.pythonhosted.org/packages/69/e4/7e36e3e6cbc83b76f1c93a63d4c053a03ca99f1c99b106835cb175b5932a/psutil-0.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.2.tar.gz","hashes":{"sha256":"4a13d7f760b043b263346e48823b1dfd4c202e97b23483e481e5ff696e74509e"},"provenance":null,"requires-python":null,"size":61640,"upload-time":"2014-02-06T02:06:51.674389Z","url":"https://files.pythonhosted.org/packages/6e/51/56198d83577106bf89cb23bffcb273f923aea8d5ffe03e3fce55f830c323/psutil-0.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.3.tar.gz","hashes":{"sha256":"43e327934b4a273da20d3a5797d6abcaab37914f61499d96fcf9e8e1ae75442b"},"provenance":null,"requires-python":null,"size":85749,"upload-time":"2014-02-06T02:06:45.294070Z","url":"https://files.pythonhosted.org/packages/1d/4f/dcfe500fd43e3d6b26d253cb0d7e6e3a7d80224b5059bd50c482aff62eef/psutil-0.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.tar.gz","hashes":{"sha256":"8de8efa92162c94f623297f522e440e818dc7b832f421f9f490324bc7d5d0d92"},"provenance":null,"requires-python":null,"size":129382,"upload-time":"2014-02-06T02:03:03.376283Z","url":"https://files.pythonhosted.org/packages/58/20/3457e441edc1625c6e1dbfcf780d2b22f2e9caa8606c3fd8ce6c48104e87/psutil-0.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"4ce48cbbe3c915a9b720ce6465da34ce79465ffde63de8254cc9c8235fef824d"},"provenance":null,"requires-python":null,"size":291186,"upload-time":"2014-02-06T16:48:27.727869Z","url":"https://files.pythonhosted.org/packages/e6/1d/9a90eec0aec7e015d16f3922328336b4f8cd783f782e9ab81146b47ffee3/psutil-0.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"eeba1fa1f2455219a05775044f7a76741252ea0bd4c762e7b652d281843c79ff"},"provenance":null,"requires-python":null,"size":289973,"upload-time":"2014-02-06T16:48:36.482056Z","url":"https://files.pythonhosted.org/packages/1d/75/7c67bc2c8304b137a8ff709d21cb2dd7f600bc5ee76ecc88f77ec008e69e/psutil-0.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"919dabb5d47dc769a198a5d015c6765ef39b3b920c2ab6d04d05d9aa3ae6bb67"},"provenance":null,"requires-python":null,"size":289890,"upload-time":"2014-02-06T16:48:46.485205Z","url":"https://files.pythonhosted.org/packages/e0/e4/2ec24cecccf111a4dbda892b484bdfd8c00d3da5f99c1a0a79c469c62fb2/psutil-0.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.5.exe","hashes":{"sha256":"92f51445439f1082dab7733b9150b594b3797f4c96578457768c3e761f86c240"},"provenance":null,"requires-python":null,"size":129555,"upload-time":"2014-02-06T16:47:42.630117Z","url":"https://files.pythonhosted.org/packages/04/4f/478408899102af1d0d877be0b67e302fada1bdd8c8faaa4adc0e19550e53/psutil-0.2.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.6.exe","hashes":{"sha256":"c3e3dc863834d03485f6c48a735f76ab9a7cff2911d101c8cbd1e553c03f0fae"},"provenance":null,"requires-python":null,"size":262087,"upload-time":"2014-02-06T16:47:50.995531Z","url":"https://files.pythonhosted.org/packages/d7/f9/984326888f6c519cc813a96640cc9a2900dad85e15b9d368fe1f12a9bd11/psutil-0.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.7.exe","hashes":{"sha256":"08c6f16ac00babe2c54067dbc305c0587cb896a8eb64383290cdafdba74ea123"},"provenance":null,"requires-python":null,"size":261605,"upload-time":"2014-02-06T16:47:58.286157Z","url":"https://files.pythonhosted.org/packages/69/5c/be56c645a254ad83a9aa3055564aae543fdf23186aa5ec9c495a3937300b/psutil-0.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py3.3.exe","hashes":{"sha256":"e1f7cf0158a94e6b50f9a4b6f4258dec1bd89c9485c65d54b89ba2a09f0d51a3"},"provenance":null,"requires-python":null,"size":256763,"upload-time":"2014-02-06T16:48:06.663656Z","url":"https://files.pythonhosted.org/packages/76/42/d6813ce55af42553b2a3136e5159bd71e544bda5fdaed04c72759514e375/psutil-0.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py3.4.exe","hashes":{"sha256":"e49d818a4a49fd1ced40dac31076af622a40407234f717bc6a5c39b3f2335e81"},"provenance":null,"requires-python":null,"size":256718,"upload-time":"2014-02-06T16:48:16.971617Z","url":"https://files.pythonhosted.org/packages/f5/83/45b721a52001c1ba1f196e6b2ba3b12d99cbc03bf3f508f1fe56b4e25c66/psutil-0.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.tar.gz","hashes":{"sha256":"bd33f5f9e04b7677a932cc90d541ceec0050080d1b053ed39488ef39cb0fb4f4"},"provenance":null,"requires-python":null,"size":144657,"upload-time":"2014-02-06T02:06:37.338565Z","url":"https://files.pythonhosted.org/packages/9b/62/03133a1b4d1439227bc9b27389fc7d1137d111cbb15197094225967e21cf/psutil-0.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"ab9d06c0c2a7baadebf66fbc74a2532520b3b6d4a58ba3c1f04311a175ed9e7f"},"provenance":null,"requires-python":null,"size":294651,"upload-time":"2014-02-06T16:46:46.406386Z","url":"https://files.pythonhosted.org/packages/55/3e/e49b7929b9daf9c3dcf31668a43ba669d22bb8606deca52e6f92cb67324d/psutil-0.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"076c9914d8aadccd459e3efedc49c10f14d16918d0af5735c22a2bdcc3f90cac"},"provenance":null,"requires-python":null,"size":293431,"upload-time":"2014-02-06T16:46:54.986513Z","url":"https://files.pythonhosted.org/packages/34/f5/f471c56be91d10c9b83d22aa9f6f29006437f2196c89156f00f56f6e4661/psutil-0.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"2679f8f9f20e61d85177f3fc6ee10bfbd6c4bd122b3f5774ba36d0035cac4f7a"},"provenance":null,"requires-python":null,"size":293352,"upload-time":"2014-02-06T16:47:04.453988Z","url":"https://files.pythonhosted.org/packages/68/ee/fc85a707394d1b374da61c071dee12299e779d87c09a86c727b6f30ad9a6/psutil-0.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.5.exe","hashes":{"sha256":"a8345934d7585c75d500098491f2a434dbba9e378fd908d76fbeb0bc20b8a3d3"},"provenance":null,"requires-python":null,"size":132806,"upload-time":"2014-02-06T16:46:02.328109Z","url":"https://files.pythonhosted.org/packages/ca/6e/5c0a87e2c1bac5e32471eb84b349f0808f6d6ed49b9118784af9d0e71baa/psutil-0.2.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.6.exe","hashes":{"sha256":"9b6294c3327625839fdbe92dca4093ab55e422fb348790b2ffc5c6cde163fe5b"},"provenance":null,"requires-python":null,"size":265263,"upload-time":"2014-02-06T16:46:11.006069Z","url":"https://files.pythonhosted.org/packages/e5/13/b358b509d4996df82ef77758c14c0999c443df537eefacc7bb24b6339758/psutil-0.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.7.exe","hashes":{"sha256":"ac4d86abc4c850141cd1e054a64f1fcb6977c4ccbbd65c5bac9bf9c8c5950cb1"},"provenance":null,"requires-python":null,"size":264780,"upload-time":"2014-02-06T16:46:19.079623Z","url":"https://files.pythonhosted.org/packages/a9/1e/8ad399f44f63de6de262fc503b1d3f091f6a9b22feaf3c273ee8bb7a7c38/psutil-0.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py3.3.exe","hashes":{"sha256":"60048c47698aa20936b7593636dd88454bae6c436852aef95fb9774b0551d06c"},"provenance":null,"requires-python":null,"size":259950,"upload-time":"2014-02-06T16:46:28.163437Z","url":"https://files.pythonhosted.org/packages/75/f3/296f43c033c0453ebfdf125dc9b8907193e31ffb92256e4b702d16ef8ac1/psutil-0.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py3.4.exe","hashes":{"sha256":"4aad401c1a45848107d395f97fc9cdb783acf120553ca1761dd77a7d52adb585"},"provenance":null,"requires-python":null,"size":259910,"upload-time":"2014-02-06T16:46:37.546918Z","url":"https://files.pythonhosted.org/packages/ae/8c/751f1e2fdcaa0ea817a8529468dca052b5de55033ee8539b996497b5be0e/psutil-0.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.tar.gz","hashes":{"sha256":"d70e16e60a575637f9d75a1005a8987d239700c0955134d4c8666a5aefbe16b8"},"provenance":null,"requires-python":null,"size":153990,"upload-time":"2014-02-06T02:06:31.022178Z","url":"https://files.pythonhosted.org/packages/f2/5c/4e74b08905dab9474a53534507e463fe8eec98b2fc0d29964b0c6a7f959d/psutil-0.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"a554abcd10534ff32e3cbc4bd80b3c76a73d0bbb50739ee59482f0151703828b"},"provenance":null,"requires-python":null,"size":297305,"upload-time":"2014-02-06T16:45:05.336122Z","url":"https://files.pythonhosted.org/packages/78/91/21944fbddafa1bdf7c0c0ed7d9f4043cd8470e36a1f7a9f6e8b4406a8e39/psutil-0.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"b33444731c63361ac316657bb305642e562097e870793bc57203cd9a4e4819a4"},"provenance":null,"requires-python":null,"size":296059,"upload-time":"2014-02-06T16:45:13.056233Z","url":"https://files.pythonhosted.org/packages/88/bd/5e9d566e941e165987b8c9bd61583c58242ef4da12b7d9fbb07494580aff/psutil-0.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"89419f0cab4dd3ef4b32933237e34f58799069930fd25ed7f69deb9c64106c85"},"provenance":null,"requires-python":null,"size":296960,"upload-time":"2014-02-06T16:45:22.608830Z","url":"https://files.pythonhosted.org/packages/1f/06/c827177a80996d00f8f46ec99292e9d03fdf230a6e4fcea25ed9d19f3dcf/psutil-0.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.5.exe","hashes":{"sha256":"515b12e23581730c47e181fa163b457b90b6e848de0637d78b7fa8ca84ec2bc6"},"provenance":null,"requires-python":null,"size":135440,"upload-time":"2014-02-06T16:44:20.300778Z","url":"https://files.pythonhosted.org/packages/e5/55/6a353c646e6d15ee618348ed68cf79800f1275ade127f665779b6be0ed19/psutil-0.3.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.6.exe","hashes":{"sha256":"5bc91423ff25ad8dffb3c9979abdafb1ecd2e9a7ef4b9287bb731e76ea2c4b91"},"provenance":null,"requires-python":null,"size":267782,"upload-time":"2014-02-06T16:44:30.319267Z","url":"https://files.pythonhosted.org/packages/61/04/c447b4d468402bfb10d21995b0eae219c8397c00d9e8a2b209fb032ac1aa/psutil-0.3.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.7.exe","hashes":{"sha256":"4b7fbaca98b72f69561d593ef18b5254a8f98482b561db3d906aac4dd307bbcc"},"provenance":null,"requires-python":null,"size":267308,"upload-time":"2014-02-06T16:44:39.895580Z","url":"https://files.pythonhosted.org/packages/e4/2f/b060e631686e6455ff483ea5b6333382c6dddb5727edbfc9dcca1a8f024c/psutil-0.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py3.3.exe","hashes":{"sha256":"005217c21551618085e3323d653d75f9ad8f7418bff4fd5109850fba11950d6e"},"provenance":null,"requires-python":null,"size":262514,"upload-time":"2014-02-06T16:44:47.866655Z","url":"https://files.pythonhosted.org/packages/50/34/7bfc452cb2cd0069a81064db51b484ef135125cf1d246d0ff82c65653c81/psutil-0.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py3.4.exe","hashes":{"sha256":"331ac97b50def25e41bae637642d0131d5c4223b52ab55281d002549632ed5e0"},"provenance":null,"requires-python":null,"size":262479,"upload-time":"2014-02-06T16:44:56.381135Z","url":"https://files.pythonhosted.org/packages/b0/08/1944d2281986237cdaeabfa36195f44dc4c847d5c4fda3523b44c6823e19/psutil-0.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.tar.gz","hashes":{"sha256":"4b0ecc77d6c503449af3d2f0a41ad4cb8338e173f5d655a8239e41b1a49bc278"},"provenance":null,"requires-python":null,"size":167796,"upload-time":"2014-02-06T02:06:24.041544Z","url":"https://files.pythonhosted.org/packages/88/46/a933ab20c6d9b0ca5704b60307b9e80bdc119b759e89b74a2609b4c10eb6/psutil-0.4.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py2.7.exe","hashes":{"sha256":"9689512e5a32508216bd6cc7cb2fbe938e4a47b483cd345b5bec6dda790a9210"},"provenance":null,"requires-python":null,"size":302844,"upload-time":"2014-02-06T16:43:18.853340Z","url":"https://files.pythonhosted.org/packages/a5/06/3ca1c85b733ceaced144cb211c1fc40a6b3b1c4f7a109bbb8e19fe09eac0/psutil-0.4.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py3.3.exe","hashes":{"sha256":"9ee193dec84c2c91ac8712760eaaae0488dee92db37134dfcddc889ecd4f578d"},"provenance":null,"requires-python":null,"size":301654,"upload-time":"2014-02-06T16:43:28.863005Z","url":"https://files.pythonhosted.org/packages/ac/eb/f6923ea1b46803251f2bb42ad9b8ed90f4b8d04cdb2384c9ea535193e4f7/psutil-0.4.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py3.4.exe","hashes":{"sha256":"94d2d0a24fa62f5655e9e21dd558592bf1d3d515d944eee70e9cd2e2bf2bf244"},"provenance":null,"requires-python":null,"size":302558,"upload-time":"2014-02-06T16:43:38.802980Z","url":"https://files.pythonhosted.org/packages/7c/5b/d9057a158b09c377eab80dae60928f8fe29bdd11635ca725098587981eeb/psutil-0.4.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.5.exe","hashes":{"sha256":"fab822fb8968b45695174a2c9b4d40e9ae81126343e24d38b627036579f1ee5e"},"provenance":null,"requires-python":null,"size":140768,"upload-time":"2014-02-06T16:42:35.663297Z","url":"https://files.pythonhosted.org/packages/07/1e/7827d1271248ddef694681747297aeb3f77dafe5c93ff707a625e3947082/psutil-0.4.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.6.exe","hashes":{"sha256":"ca511f56db8f1586612851a2eedbf80e38cf29f4a98f133410a74da284822747"},"provenance":null,"requires-python":null,"size":273280,"upload-time":"2014-02-06T16:42:44.697290Z","url":"https://files.pythonhosted.org/packages/b4/56/78f182bf9c81a978067ba5447eca2ea70ef2bc9ac94228adfae5428cd108/psutil-0.4.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.7.exe","hashes":{"sha256":"5b6cbf5aaeab55db1d9657fc75e4b54fa94d17d03ca97c94aa150e1bfdc8fb2e"},"provenance":null,"requires-python":null,"size":272807,"upload-time":"2014-02-06T16:42:52.281617Z","url":"https://files.pythonhosted.org/packages/7d/38/b719c8867699a71a98e92e61fd1b0f12bad997a9de60bf8f6215184692e8/psutil-0.4.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py3.3.exe","hashes":{"sha256":"dfe18b8eb9ee6dbbd49e28f245794c2fc22b56d770038668cb6e112935c49927"},"provenance":null,"requires-python":null,"size":268022,"upload-time":"2014-02-06T16:43:01.185161Z","url":"https://files.pythonhosted.org/packages/f4/2b/13be095a72c83fdbe11d519310cb34ff8e442d72ab52691218a83c216ac4/psutil-0.4.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py3.4.exe","hashes":{"sha256":"5b63e0e83dde30d62206497320c06b27c489ffb261e95ac1267b7a1b8d4ce72d"},"provenance":null,"requires-python":null,"size":267989,"upload-time":"2014-02-06T16:43:09.528893Z","url":"https://files.pythonhosted.org/packages/b4/8e/028b849600447c45c20582d7ddae1e39d31b83799289f54f1b79af664295/psutil-0.4.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.tar.gz","hashes":{"sha256":"33002c38f916835c949ae39a84f3f6d09ce01818ed805dfd61f8f3844c395c9d"},"provenance":null,"requires-python":null,"size":171549,"upload-time":"2014-02-06T02:06:17.394021Z","url":"https://files.pythonhosted.org/packages/a0/cd/00550e16a2a0357a9c24946160e60fc947bbe23bc276aff1b4d3e7b90345/psutil-0.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"3e408028898b4ad09c207659c1b6cb8e6a74fb39ab3157b761e057a629118fec"},"provenance":null,"requires-python":null,"size":310414,"upload-time":"2014-02-06T16:41:41.640755Z","url":"https://files.pythonhosted.org/packages/06/75/3dc33773b1a1f7055b24245a0c8bec4d5776f1e1125329a13e82449007be/psutil-0.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"457cfdecbf63f813bf86e2f736ab3b20447e8d8c2c516dd2a13a7463177f8bc6"},"provenance":null,"requires-python":null,"size":309169,"upload-time":"2014-02-06T16:41:50.926907Z","url":"https://files.pythonhosted.org/packages/6e/9e/6c7f5baf2529d9bba563cdff0b81413b5870b0681739318112adecd99ad0/psutil-0.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"fc75e094cf573a7b9977be2a9d3a9d94671bc0ab098d52c391dd63227309cbea"},"provenance":null,"requires-python":null,"size":310087,"upload-time":"2014-02-06T16:42:00.068444Z","url":"https://files.pythonhosted.org/packages/cd/e6/9f963b32ab254b8c2bbedf5ce543cace8b08054924fd26fa233f99863ecc/psutil-0.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.5.exe","hashes":{"sha256":"ef4ed886b4d2d664fab5c369543690cda04e04a3b7496d72e369ef5240a8af6b"},"provenance":null,"requires-python":null,"size":148474,"upload-time":"2014-02-06T16:40:57.772154Z","url":"https://files.pythonhosted.org/packages/65/ad/e7828797558dd5f209694b02ce079cd3b6beacf6a5175f38c1973c688494/psutil-0.4.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.6.exe","hashes":{"sha256":"aafe9e328ffc173b26c78897c168858da335c85d4a02d666ad68fe2fe14601d1"},"provenance":null,"requires-python":null,"size":280897,"upload-time":"2014-02-06T16:41:07.265101Z","url":"https://files.pythonhosted.org/packages/1e/a1/594bf54e7c4056bc5284023be97f67c930175b3329e086a4ed8966cb067a/psutil-0.4.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.7.exe","hashes":{"sha256":"cefad502010c78425190498efc54541ae7bbe9eaa73ef4000cf11b032d32a8bb"},"provenance":null,"requires-python":null,"size":280445,"upload-time":"2014-02-06T16:41:15.492817Z","url":"https://files.pythonhosted.org/packages/c6/c4/14807de009a1beab2426b537379a8b05b1d69fef1fde7e23581cc332cdb3/psutil-0.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py3.3.exe","hashes":{"sha256":"a35ce124b2dec01d7632627c9370f5202978fce89ae59235cf4d5e05bbd0e02b"},"provenance":null,"requires-python":null,"size":275668,"upload-time":"2014-02-06T16:41:23.498162Z","url":"https://files.pythonhosted.org/packages/38/7d/407f7586bccdeb80f5da82d8ebb98712bfcc7217e3d7c3fc61b3bba893f2/psutil-0.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py3.4.exe","hashes":{"sha256":"860a039b67ee015a6304e8d05e6dc5c2a447eca331b7d6c9d04f41d7f2fc3a66"},"provenance":null,"requires-python":null,"size":275635,"upload-time":"2014-02-06T16:41:33.515203Z","url":"https://files.pythonhosted.org/packages/c5/b7/d2662ebf114961766c9eb3d9b737cb5541060d35e65b542b0acef470221a/psutil-0.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.tar.gz","hashes":{"sha256":"b10d1c19b9d334bea9f025f3b82b5664d642d26ba32ae0e71e7e20c5a6b4164f"},"provenance":null,"requires-python":null,"size":118411,"upload-time":"2014-02-06T02:06:09.471218Z","url":"https://files.pythonhosted.org/packages/3f/eb/e4115c3ecc189fd345b9a50d521e2ff76990bfb12a91921e93ea6398feef/psutil-0.5.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py2.7.exe","hashes":{"sha256":"5bf7a91fd57be3a63c6b9fa78da63c185b9be909ca49032f3ceda662628be74d"},"provenance":null,"requires-python":null,"size":319335,"upload-time":"2014-02-06T15:53:45.976879Z","url":"https://files.pythonhosted.org/packages/c5/cc/9bb29fdd60e347617c3a09f51a595941ddb4777fc1608068d174ced5bfdd/psutil-0.5.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py3.3.exe","hashes":{"sha256":"0cd9872d0d675b0314d2d15513e8ff3693a417e4ebaa158fc1f8780cc611574c"},"provenance":null,"requires-python":null,"size":317930,"upload-time":"2014-02-06T15:53:57.230392Z","url":"https://files.pythonhosted.org/packages/37/d8/276629523e0477639b33a606f93c2fc2e9c4cafba76b3a4cd1ddc42dd307/psutil-0.5.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py3.4.exe","hashes":{"sha256":"13f71acf1d2fe08db072d890280e985e8414080bcd525c13c66e0e8fbc917710"},"provenance":null,"requires-python":null,"size":318790,"upload-time":"2014-02-06T15:54:08.039630Z","url":"https://files.pythonhosted.org/packages/5f/bc/a026bbb10a4df4cf458b5d6feabc72a0017e6a4f472898280442ff3f9393/psutil-0.5.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.5.exe","hashes":{"sha256":"d6839532038b97792d90341465430520e58ea3d0956a1df710b3fdd733dde587"},"provenance":null,"requires-python":null,"size":156923,"upload-time":"2014-02-06T16:39:37.158716Z","url":"https://files.pythonhosted.org/packages/26/83/d1aed38bad84576e48b9ca6501edcf829ca3b48631c823736c36fa2b24de/psutil-0.5.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.6.exe","hashes":{"sha256":"be79349ecf091470ea4920e45b78914b67f17bec9c23e4bea4ec0ca5fee1feab"},"provenance":null,"requires-python":null,"size":289457,"upload-time":"2014-02-06T15:53:07.782148Z","url":"https://files.pythonhosted.org/packages/4e/b2/bfdeae58283e0fb09e2c6725c01b9429acbd15917c7ead91c96f2df37d05/psutil-0.5.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.7.exe","hashes":{"sha256":"e4fa8e965eb58a885e65a360f1a7ffc7ed476e9227a2de3726091d7560c694a8"},"provenance":null,"requires-python":null,"size":289002,"upload-time":"2014-02-06T15:53:16.212429Z","url":"https://files.pythonhosted.org/packages/2d/62/ed3c23fb8648a916460e95306d58f5ba42b3b20c68f2a55d1fc6396190a5/psutil-0.5.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py3.3.exe","hashes":{"sha256":"0380ece3eeb6f170ba156c58eab2fa36508600b1c7e10bc9b7c651f721236f38"},"provenance":null,"requires-python":null,"size":284089,"upload-time":"2014-02-06T15:53:25.963338Z","url":"https://files.pythonhosted.org/packages/59/3d/25912eb67f01af306d8ec035041c07daa7f783aafa5f4e1dd90de91c4849/psutil-0.5.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py3.4.exe","hashes":{"sha256":"d6a9e67de81fc0fc58c790e1355ddfbb8a0760963868a538f217ac2325a81581"},"provenance":null,"requires-python":null,"size":284045,"upload-time":"2014-02-06T15:53:35.675392Z","url":"https://files.pythonhosted.org/packages/38/74/e6c5bfb482f011d19bb1366cecf7d4f0b5c9c9b7664ef31e951cc4d0757b/psutil-0.5.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.tar.gz","hashes":{"sha256":"f660d244a08373f5e89633650970819a59463b37af1c9d205699fb0d0608986d"},"provenance":null,"requires-python":null,"size":118781,"upload-time":"2014-02-06T02:06:02.078066Z","url":"https://files.pythonhosted.org/packages/aa/06/6ebc13a14c0961d7bb8184da530448d8fc198465eb5ecd1ad36c761807d2/psutil-0.5.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py2.7.exe","hashes":{"sha256":"eff5ed677b6bdb5d75f1de925df774d8029582eea17abc3bbd31b9885d3c649d"},"provenance":null,"requires-python":null,"size":319512,"upload-time":"2014-02-06T16:36:10.664265Z","url":"https://files.pythonhosted.org/packages/8a/26/15751b500afdd0aea141905d5ba5319c38be41e0ca55a374c02827d1a79c/psutil-0.5.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py3.3.exe","hashes":{"sha256":"8a12d8e368bfccfe2074616cae2472dfb1e136a2aaaea5b7dbafdb390166b202"},"provenance":null,"requires-python":null,"size":318245,"upload-time":"2014-02-06T16:36:21.268821Z","url":"https://files.pythonhosted.org/packages/13/9d/49f6fb5e1f75b3d6815bbdbbeb1630f7299d6ff96f85836dbd27989780ae/psutil-0.5.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py3.4.exe","hashes":{"sha256":"9d991aae6b993ac58b540f27b2dff71137db62253f5d7acc5ede1ec845fd6b0b"},"provenance":null,"requires-python":null,"size":319083,"upload-time":"2014-02-06T16:36:31.287414Z","url":"https://files.pythonhosted.org/packages/2c/fa/a0469c75acedccc5006b655434e96a8a6a490725a9ae8208faaf39c043f6/psutil-0.5.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.5.exe","hashes":{"sha256":"ee32be6667bb69ebefc2fb5e23d115d98e307b5246c01ef50e45cfd8ad824d07"},"provenance":null,"requires-python":null,"size":157149,"upload-time":"2014-02-06T16:35:24.026975Z","url":"https://files.pythonhosted.org/packages/2e/30/9ed6283c7f1a716a3bf76a063b278ace2f89d1a8b2af9d35080dc2a0319b/psutil-0.5.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.6.exe","hashes":{"sha256":"f434f546d73366bff6283db907105b2607976ff9be67f2b5a7b35533e5f27219"},"provenance":null,"requires-python":null,"size":289555,"upload-time":"2014-02-06T16:35:32.000842Z","url":"https://files.pythonhosted.org/packages/eb/d9/55f5d02e42dc6c73f66c7cfc99bc3bf97cf10e9f259dbea69e123e0ef459/psutil-0.5.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.7.exe","hashes":{"sha256":"45acf090d65f2f47f3b1444b056cafb1b1ca70a8e266ddf10422734fb0ada897"},"provenance":null,"requires-python":null,"size":289095,"upload-time":"2014-02-06T16:35:40.182115Z","url":"https://files.pythonhosted.org/packages/81/25/cbbb80d1957c28dffb9f20c1eaf349d5642215d7ca15ec5b7015b6a510c8/psutil-0.5.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py3.3.exe","hashes":{"sha256":"9a20a99b6727334c8cc3a10ecc556d2f15a1e92efef97670ed022b86e09df36b"},"provenance":null,"requires-python":null,"size":284290,"upload-time":"2014-02-06T16:35:50.441013Z","url":"https://files.pythonhosted.org/packages/73/a2/ab4f2fa7ba6f8fc51d9cc9a0bb5ec269860a34574d391e39d20c2dca7b56/psutil-0.5.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py3.4.exe","hashes":{"sha256":"d547e164013a4d78909cbac0c176d65aa51c06de2fb888c63b738906f5ccc105"},"provenance":null,"requires-python":null,"size":284247,"upload-time":"2014-02-06T16:36:00.155604Z","url":"https://files.pythonhosted.org/packages/7f/8b/c9abd21cdc79b70744e169b98be06f0828687dfcac6468ed87455099f88d/psutil-0.5.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.tar.gz","hashes":{"sha256":"e918763243371f69bc98b601769df7337e196029dcdb797224f0ede474e17b94"},"provenance":null,"requires-python":null,"size":130803,"upload-time":"2014-02-06T02:05:55.032752Z","url":"https://files.pythonhosted.org/packages/3c/9f/0de622fc62e74f4c154656440c02c8a24a01a997d605a744272eb0d93742/psutil-0.6.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py2.7.exe","hashes":{"sha256":"1792be8fca4b3f88553e1c5cc197e18932c3da9b969ce7f545b741913597d600"},"provenance":null,"requires-python":null,"size":322153,"upload-time":"2014-02-06T16:34:24.006415Z","url":"https://files.pythonhosted.org/packages/5f/fd/fc06aaf69ad438d1354312c728791a1f571b2c59a49b411128911abf647e/psutil-0.6.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py3.3.exe","hashes":{"sha256":"084a1cbeca8e10f0cbba4b9d3f7d3136cb1d03997a868eea334401f82105207a"},"provenance":null,"requires-python":null,"size":320646,"upload-time":"2014-02-06T16:34:33.535927Z","url":"https://files.pythonhosted.org/packages/70/5e/b1d4a1d5238497d48112bc27e3448d05a0acb6c8fcd537565724d916e7c2/psutil-0.6.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py3.4.exe","hashes":{"sha256":"e7233e45ed7505efe46309978edd3e8a1cb4160400c3535512be2e4114615bb9"},"provenance":null,"requires-python":null,"size":321547,"upload-time":"2014-02-06T16:34:44.543150Z","url":"https://files.pythonhosted.org/packages/47/ad/4139c3eaa2ee1da71d071c9b49e8a04ff23f2c71f6fdecc11850b0e7343e/psutil-0.6.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.5.exe","hashes":{"sha256":"469628dea02a7d91837e05777cb66bfcdc2e1639cc8607ed93f19679c4e42cef"},"provenance":null,"requires-python":null,"size":160485,"upload-time":"2014-02-06T16:33:39.286987Z","url":"https://files.pythonhosted.org/packages/10/3a/a0136c299b74224be3b0735812a1544e6962d7c61af41adfcc400791c684/psutil-0.6.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.6.exe","hashes":{"sha256":"a628ad11fefc16d6dd144a8d29a03ac379debcf6c2e4b576e6ee2b2677c15836"},"provenance":null,"requires-python":null,"size":292261,"upload-time":"2014-02-06T16:33:47.696744Z","url":"https://files.pythonhosted.org/packages/2b/79/eec1c63d3ea968cc9754871a1b9d17d50bfc24136ecac415f701b1d240ea/psutil-0.6.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.7.exe","hashes":{"sha256":"71197523bf8acb49e3e909ea945cc6e77a21693428db2f169e33348988c1bb11"},"provenance":null,"requires-python":null,"size":291811,"upload-time":"2014-02-06T16:33:56.338977Z","url":"https://files.pythonhosted.org/packages/e9/64/fe90d0f1ba2dff8ad2511423720aed3569cfe7b5dbb95ede64af25b77e84/psutil-0.6.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py3.3.exe","hashes":{"sha256":"1d7df98f4532e76685fabbd82e9502c91be9a60e4dfdfd5a19c2152936b7d198"},"provenance":null,"requires-python":null,"size":286963,"upload-time":"2014-02-06T16:34:04.390605Z","url":"https://files.pythonhosted.org/packages/d6/d7/91226f8635d3850917b64e94ca4903910a60739ee7c930ef9678d93638eb/psutil-0.6.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py3.4.exe","hashes":{"sha256":"da568edc3b8ec4be5b8cee54851289eb4c24cbaa8ea165858e9bfc279269bf22"},"provenance":null,"requires-python":null,"size":286905,"upload-time":"2014-02-06T16:34:14.322283Z","url":"https://files.pythonhosted.org/packages/56/67/533833832596e5f8c1fb51d3f94cd110d688579be7af46996cf96ad890e7/psutil-0.6.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.tar.gz","hashes":{"sha256":"52eba795281cdd1079f13ded6a851f6d029551ddf552eadc9a2ee3eb26fe994d"},"provenance":null,"requires-python":null,"size":131473,"upload-time":"2014-02-06T02:05:47.971617Z","url":"https://files.pythonhosted.org/packages/4f/e6/989cb0b2f7f0ebe3ab0e7144b78db17387810f1526e98498be96fb755fa9/psutil-0.6.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py2.7.exe","hashes":{"sha256":"566bcec389a77279e10335eb5d0021eb968445720496b70b46d208dbbf8e49a1"},"provenance":null,"requires-python":null,"size":322415,"upload-time":"2014-02-06T16:32:41.593785Z","url":"https://files.pythonhosted.org/packages/9a/bb/cb19a41fa75a2205ac2d337be45183f879aa77e60bbb19387c872c81d451/psutil-0.6.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py3.3.exe","hashes":{"sha256":"fde5e57f9b51057998cce5a030ad481f826832b26cc2a6490d60317c9024724c"},"provenance":null,"requires-python":null,"size":320911,"upload-time":"2014-02-06T16:32:50.930654Z","url":"https://files.pythonhosted.org/packages/7d/d1/9e75848da16a5d165e3918540418f3052e6409357bcce91a7342b22d674f/psutil-0.6.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py3.4.exe","hashes":{"sha256":"402475d1a93306ac025070da9628a0ca0e0d253b0755bf13131318a39aaeea8e"},"provenance":null,"requires-python":null,"size":321810,"upload-time":"2014-02-06T16:33:01.995635Z","url":"https://files.pythonhosted.org/packages/95/82/1f52f8aaa37e07da54da4468c031f59717d3f40cd82b6faaf8f203d84bd7/psutil-0.6.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.5.exe","hashes":{"sha256":"3f5b8afde564c563cd6fa28ac14ed020d91d654ef1efd6c0e1672282fcde8e65"},"provenance":null,"requires-python":null,"size":160754,"upload-time":"2014-02-06T16:31:55.533884Z","url":"https://files.pythonhosted.org/packages/5b/3f/10449a8a3dfb809fbeae3d853453aafb36376d51f33c46e02b5f0723c620/psutil-0.6.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.6.exe","hashes":{"sha256":"1c1763ff214042e7be75fd6eb336322a47193342cddfc70b8547f086968e024c"},"provenance":null,"requires-python":null,"size":292530,"upload-time":"2014-02-06T16:32:04.859349Z","url":"https://files.pythonhosted.org/packages/3e/7a/5d53c99ec66ea86548d95acaf98b32f1cbb761cf2e1fc8ee62d0838f8e24/psutil-0.6.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.7.exe","hashes":{"sha256":"e411f2374994663ad35bc21afe4ff9003e3675e23b07a2a60b57c760da0c5f3f"},"provenance":null,"requires-python":null,"size":292082,"upload-time":"2014-02-06T16:32:13.267526Z","url":"https://files.pythonhosted.org/packages/4a/40/e43e412c55f66b57aee3a418f47b05e7bb92db83df93c0c93b00f11f1357/psutil-0.6.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py3.3.exe","hashes":{"sha256":"03ea1f851920d6ddc286c1940ebd167ed92c1cb7a870f1789510c21c6fa6b7bc"},"provenance":null,"requires-python":null,"size":287227,"upload-time":"2014-02-06T16:32:22.314264Z","url":"https://files.pythonhosted.org/packages/07/12/1ce6196592e01965a375b307a57f68e90488a49a56a647f17ba027382448/psutil-0.6.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py3.4.exe","hashes":{"sha256":"095107d63b176fed2f4a44de3a4d19e8aa6291de0c3ee5e398272c9b72bf2254"},"provenance":null,"requires-python":null,"size":287169,"upload-time":"2014-02-06T16:32:32.819904Z","url":"https://files.pythonhosted.org/packages/f4/88/d20f7eefa6b8cc59f9320c52bf561ba52baea2b9b36568b198bca0b3548d/psutil-0.6.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.tar.gz","hashes":{"sha256":"95089802017ee629b84332deeb367f7f775b42e827ee283d46b7e99d05120d71"},"provenance":null,"requires-python":null,"size":138681,"upload-time":"2014-02-06T02:05:40.485455Z","url":"https://files.pythonhosted.org/packages/0b/8c/aeb6acf5a4610f8d5bb29ade04081d8672c2294c3cefa1f0422c6398bc66/psutil-0.7.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py2.7.exe","hashes":{"sha256":"add60c89c13443617c83116fab34a3c6e4f32ee8b102ff6ffb4cd20ef6f511c3"},"provenance":null,"requires-python":null,"size":325666,"upload-time":"2014-02-06T16:30:51.588803Z","url":"https://files.pythonhosted.org/packages/38/8e/f3023be1a2268b5ee3a20e05ec8997a95c987f6f2b7dadad837674d562e8/psutil-0.7.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py3.3.exe","hashes":{"sha256":"520a43be7fcd21d63c02039cdd7ebfe959b9fa9a865ba9aab6b0194a85184250"},"provenance":null,"requires-python":null,"size":324143,"upload-time":"2014-02-06T16:31:01.226228Z","url":"https://files.pythonhosted.org/packages/7b/03/d6bff7ff8570f3521a1d7949f03741210f11ba5ba807789d37e185a0a780/psutil-0.7.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py3.4.exe","hashes":{"sha256":"0e78c6b441e659d922c5fddd5d5a36a368a61192873a0600ba23171c14dc2a6d"},"provenance":null,"requires-python":null,"size":325001,"upload-time":"2014-02-06T16:31:12.826305Z","url":"https://files.pythonhosted.org/packages/ce/8a/95e7c8b343a92137d82e4635fcb3f5c69c29e2a84ce4d51f2d5d86020b1e/psutil-0.7.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.5.exe","hashes":{"sha256":"532b918d8d2df60c20c69c1f764b9b16373ea22c32595fe801e3cc9b1aa5adfb"},"provenance":null,"requires-python":null,"size":163763,"upload-time":"2014-02-06T16:30:02.815062Z","url":"https://files.pythonhosted.org/packages/9e/6f/a946eeb3c6b8848fcf6d21c52884e2639e7d4393b51f330d4555ec177e83/psutil-0.7.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.6.exe","hashes":{"sha256":"21e1d7880579ef7eea28547c737923eaeee53389d51fa971d9f70bf89e9759ee"},"provenance":null,"requires-python":null,"size":295425,"upload-time":"2014-02-06T16:30:11.405843Z","url":"https://files.pythonhosted.org/packages/14/e9/ac4865772c55a3c09c85d987ac2a0e679c31e897d536f389485019c29450/psutil-0.7.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.7.exe","hashes":{"sha256":"bc36a7350b89e2ada06691583f4c4c2f99988fe6fa710d4f8e4143bed9f9f8bf"},"provenance":null,"requires-python":null,"size":294933,"upload-time":"2014-02-06T16:30:20.866887Z","url":"https://files.pythonhosted.org/packages/12/1c/053e0d5485a39c055e92b845b6311ab54887a091836a3985369cfffa9e53/psutil-0.7.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py3.3.exe","hashes":{"sha256":"9ce0d7a8cedde996eaae1eda3db9b473d441bd6c797b36a166d7b7a70dd4650e"},"provenance":null,"requires-python":null,"size":290006,"upload-time":"2014-02-06T16:30:29.860768Z","url":"https://files.pythonhosted.org/packages/73/19/8009828f3cfcf529570822e52988b95ac1c25c90ba4fb96ae570b5c25e66/psutil-0.7.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py3.4.exe","hashes":{"sha256":"6fdeca414a470d156ebdf1b6e4e9ca0fd9eb1b53e8d9a0cbb7d4a69d31f68bdc"},"provenance":null,"requires-python":null,"size":289986,"upload-time":"2014-02-06T16:30:39.817575Z","url":"https://files.pythonhosted.org/packages/42/7d/8e159acbf98eed98b778c244287b295f06411ac9e1903d2d358ba48b1d36/psutil-0.7.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.tar.gz","hashes":{"sha256":"5236f649318a06dcff8b86947c888d4510abce1783923aa5455b2d62df7204c7"},"provenance":null,"requires-python":null,"size":138525,"upload-time":"2014-02-06T02:05:32.811951Z","url":"https://files.pythonhosted.org/packages/8c/75/1eeb93df943b70c2e18bf412b32d18af3577da831f2bfe8c7d29f8853a67/psutil-0.7.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py2.7.exe","hashes":{"sha256":"47fc6887ec86bd7a422326dfc77e2b43fcbc71e498194f86088bb2268c2372b7"},"provenance":null,"requires-python":null,"size":325663,"upload-time":"2014-02-06T16:29:31.935519Z","url":"https://files.pythonhosted.org/packages/2d/ab/8fde8a7358a21bc2488a5d21a24879ce3cc162d2a99c56f9599c7d759c75/psutil-0.7.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py3.3.exe","hashes":{"sha256":"c00ec6a0e061e31687851c06d542500fd47b0ed415818e0da61c294f2796572b"},"provenance":null,"requires-python":null,"size":324140,"upload-time":"2014-02-06T16:29:41.403650Z","url":"https://files.pythonhosted.org/packages/35/b1/8506f2e78974da833b20b162d70b7e38ac3ab722adf18c03a3e428c9d9c4/psutil-0.7.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py3.4.exe","hashes":{"sha256":"a5238a16cd85d4f3240dfca8ad94cb0bfe8643acb5c21d839a53caa86d8a23ab"},"provenance":null,"requires-python":null,"size":324998,"upload-time":"2014-02-06T16:29:50.929853Z","url":"https://files.pythonhosted.org/packages/cc/29/d00e1011e6db097ffac31c2876fdba51fd1fc251ee472ad6219095739a86/psutil-0.7.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.5.exe","hashes":{"sha256":"facf21db78120afb79ec51685043a5f84e897a84a7678db4381c0e377a481364"},"provenance":null,"requires-python":null,"size":163764,"upload-time":"2014-02-06T16:28:44.337888Z","url":"https://files.pythonhosted.org/packages/06/d8/a34c090687fa4c09bee5b5a06a762eb1af134d847b907d3e63c58e5d2cba/psutil-0.7.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.6.exe","hashes":{"sha256":"709d2e851c161f1ffad6bd7ed925ee90229dbd2067fbc26126c11d473075d7b7"},"provenance":null,"requires-python":null,"size":295427,"upload-time":"2014-02-06T16:28:53.964081Z","url":"https://files.pythonhosted.org/packages/4d/8a/d51e550aa0928ab02e4f6b2531453b2e63c8877e30543e342c8a35c90d4d/psutil-0.7.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.7.exe","hashes":{"sha256":"ee218aa12ee3af563833cf0bf109b15382261a0760a1ce636bf9e26e630f833e"},"provenance":null,"requires-python":null,"size":294930,"upload-time":"2014-02-06T16:29:03.098211Z","url":"https://files.pythonhosted.org/packages/3c/4c/d46a8bc865e58b30dcb160772667abf42f854ec4910ee1100e79961035ce/psutil-0.7.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py3.3.exe","hashes":{"sha256":"fdf22a72f891a45b8a9ebca85b50a055a4ba7a8c05b98f93e1387b8ee88a4562"},"provenance":null,"requires-python":null,"size":290003,"upload-time":"2014-02-06T16:29:12.556491Z","url":"https://files.pythonhosted.org/packages/1a/cf/98fdaf08279cd754414ab6723bbb6bd55d890624b766d64b21c35379d865/psutil-0.7.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py3.4.exe","hashes":{"sha256":"f675a6346e5bd02469b043afff2db1b8e0b180e8a6d1ca94d44d23f26f60e68f"},"provenance":null,"requires-python":null,"size":289983,"upload-time":"2014-02-06T16:29:21.732021Z","url":"https://files.pythonhosted.org/packages/f4/f9/79ac18809795f53197fb5b6bd452d2b52f62e5bff6a66d9b813077a85eb9/psutil-0.7.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.tar.gz","hashes":{"sha256":"af776ebeaf2420709b1ab741f664c048dcfc2890e3e7d151ae868fbdc3d47920"},"provenance":null,"requires-python":null,"size":156516,"upload-time":"2014-02-06T02:05:25.754044Z","url":"https://files.pythonhosted.org/packages/ed/7e/dd4062bfbe9b735793a5e63a38d83ba37292c4ada3615fe54403c54e3260/psutil-1.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"7868ef2ecc1d847620d99b40f8105b6d5303784abbde34da2388b2c18e73a3d2"},"provenance":null,"requires-python":null,"size":327529,"upload-time":"2014-02-06T16:25:06.308578Z","url":"https://files.pythonhosted.org/packages/11/46/8a6581c14d644f747ea66a12c60ffebebd4f8a77af62192aeb20cc16ae17/psutil-1.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"cdced3409a241b9d357b82c72d00ece49602cdc3f694ed988a952761367df560"},"provenance":null,"requires-python":null,"size":326023,"upload-time":"2014-02-06T16:25:15.281576Z","url":"https://files.pythonhosted.org/packages/03/a4/aa6592e676f75705b77df8e319f6aa35cf08a662fb0cfaa34434d273d7c9/psutil-1.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"ef6974ac18407fb15430cd7b81aecc2f3666e75ca83fe3a66848c4c5c086e0a3"},"provenance":null,"requires-python":null,"size":326915,"upload-time":"2014-02-06T16:25:27.390929Z","url":"https://files.pythonhosted.org/packages/5b/c1/003f0071d1e2a4dfe12ac2d66ac14f5a5a93049544c11ac9c78d83d85318/psutil-1.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.5.exe","hashes":{"sha256":"d8213feaa4c560c61514cea4e6a7eec83f6df83694bb97b06d009e919f60c48f"},"provenance":null,"requires-python":null,"size":165576,"upload-time":"2014-02-06T16:24:22.454259Z","url":"https://files.pythonhosted.org/packages/d6/28/5753ba13863c26d8a8fef685690b539a4652f1c4929d13d482eb4d691d99/psutil-1.0.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.6.exe","hashes":{"sha256":"7571d74d351bba723b28666a13da43c07bb3568543be3f9f7404af2b8c1ecf3a"},"provenance":null,"requires-python":null,"size":297205,"upload-time":"2014-02-06T16:24:31.483662Z","url":"https://files.pythonhosted.org/packages/ad/8c/37599876ea078c7c2db10d746f0e0aa805abcacbab96d1cf5dfb0465c40b/psutil-1.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.7.exe","hashes":{"sha256":"3f311d4f060b83d7cd0068dbad867ad17e699d3c335e494dc31db5380154fdf4"},"provenance":null,"requires-python":null,"size":296787,"upload-time":"2014-02-06T16:24:39.921427Z","url":"https://files.pythonhosted.org/packages/99/4e/8c1f5db3e90dbdbde891c34fd2496805a93aa279433bac716098efad1b4c/psutil-1.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py3.3.exe","hashes":{"sha256":"acf6cbc2408869b2843d406a802b6895969a358c762b5ab3e6f5afdd0c918faf"},"provenance":null,"requires-python":null,"size":291856,"upload-time":"2014-02-06T16:24:48.963293Z","url":"https://files.pythonhosted.org/packages/5b/f7/0bad3fc8ff5f226fa933a847b5fb3355562421f0456348747cf18f7137f5/psutil-1.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py3.4.exe","hashes":{"sha256":"191c2e6953426c861d7e100d48a5b522307ffc3de7d9e7bc8509aa45fb3de60c"},"provenance":null,"requires-python":null,"size":291810,"upload-time":"2014-02-06T16:24:57.589702Z","url":"https://files.pythonhosted.org/packages/0a/50/18ebd254b6662c98a8310983fff6795c7ac303d25188fd727888ca1c5211/psutil-1.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.tar.gz","hashes":{"sha256":"ba4c81622434836f6645e8d04e221ca5b22a9bd508c29989407f116b917be5b3"},"provenance":null,"requires-python":null,"size":156516,"upload-time":"2014-02-06T02:05:16.665325Z","url":"https://files.pythonhosted.org/packages/94/50/7c9e94cf6cdbf4e4e41d2e318094c2b0d58d3bb9196017fb6e4897adf277/psutil-1.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"2d6705cf53a7474b992d0d5beac4697cdd37f2c24098d4e1b5bb35a83b70b27e"},"provenance":null,"requires-python":null,"size":327529,"upload-time":"2014-02-06T16:20:20.120751Z","url":"https://files.pythonhosted.org/packages/a9/28/ac3c9da11fbe1ae3ddefb2cb6b410c8ad701f39d86489931e87c871cc4d1/psutil-1.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"6194670fcbddec715846fa9beca4450b4419423b2d466894da2fa1915fc7b712"},"provenance":null,"requires-python":null,"size":326023,"upload-time":"2014-02-06T16:20:43.401705Z","url":"https://files.pythonhosted.org/packages/97/30/094bb08295e15e05a266fcb15c116912f643fbb4dd8090ec64aca801ea72/psutil-1.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"5cf4ddea8dd21b99b90656dafa2d28c32fa9c159ab30580782b65b3b615c62a7"},"provenance":null,"requires-python":null,"size":326915,"upload-time":"2014-02-06T16:20:52.673334Z","url":"https://files.pythonhosted.org/packages/e8/86/e72c22699491573508e85b226f1782eab840b713cd6b8d42655c5329324c/psutil-1.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.5.exe","hashes":{"sha256":"fadf7625c309bc5f6e6d3b9d42836491efabbfda1834db7166775d705db420bf"},"provenance":null,"requires-python":null,"size":165580,"upload-time":"2014-02-06T16:04:23.085098Z","url":"https://files.pythonhosted.org/packages/a5/40/d307abd2ee3015e38cbeeac18c6b0f091cdcac496688f4d2c286145fdc5f/psutil-1.0.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.6.exe","hashes":{"sha256":"84a257b4adea431473089ea4b798bea98ce64452587ba7b71856e150d88e9213"},"provenance":null,"requires-python":null,"size":297208,"upload-time":"2014-02-06T15:59:31.669293Z","url":"https://files.pythonhosted.org/packages/75/3f/c10ca801aff50ce6ea84f71ce69a32de1b4a1f0439ada94adeccd36e1afd/psutil-1.0.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.7.exe","hashes":{"sha256":"4b5047bb4055d48f718bbb5a457c9b1c1c06776ca6353cb310b471c38d51324d"},"provenance":null,"requires-python":null,"size":296787,"upload-time":"2014-02-06T15:59:41.621679Z","url":"https://files.pythonhosted.org/packages/15/6c/944089b8dd730314a8eec9faa4d7d383f31490130e9e0534c882173e2eb6/psutil-1.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py3.3.exe","hashes":{"sha256":"3553a27b4d0e7552d0d863c017e152a88cb689d2090b39e2bca68db7fc27e8b2"},"provenance":null,"requires-python":null,"size":291856,"upload-time":"2014-02-06T15:59:51.241604Z","url":"https://files.pythonhosted.org/packages/4c/51/eb8b12a0124050492faad7fe3e695cb44e4ec5495a64a8570abf6e7cddc3/psutil-1.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py3.4.exe","hashes":{"sha256":"d3ab1b882647d66099adefa8ff780c4144bf8df1bb97bc31c55cbd999807cae8"},"provenance":null,"requires-python":null,"size":291810,"upload-time":"2014-02-06T16:00:00.814707Z","url":"https://files.pythonhosted.org/packages/4d/2a/33063e4a674e3be38d77fbc6e5e988f0e0323cd000cb7eb60f9300493631/psutil-1.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.tar.gz","hashes":{"sha256":"31b4b411d3f6960d26dac1f075709ff6d36f60216bf1f2b47fc4dc62a2d0aa6f"},"provenance":null,"requires-python":null,"size":163785,"upload-time":"2013-09-28T09:48:01.746815Z","url":"https://files.pythonhosted.org/packages/f6/71/1f9049fc7936f7f48f0553abd8bf8394f4ad2d9fb63a881b3a653b67abd6/psutil-1.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"ef250fbe783c0b31d9574d06324abd78ebf23c75981a8d0f9319d080435aa056"},"provenance":null,"requires-python":null,"size":305459,"upload-time":"2013-09-28T09:55:10.368138Z","url":"https://files.pythonhosted.org/packages/28/ff/e0dcc8e0194817191817ff02926a734a51d24a91e1cfc5c47be8c1e8508c/psutil-1.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win-amd64-py3.2.exe","hashes":{"sha256":"a95bf68f2189b7cd3f22558b12a989f368335c905cbd3045ca315514538b3115"},"provenance":null,"requires-python":null,"size":306085,"upload-time":"2013-09-28T09:55:36.198407Z","url":"https://files.pythonhosted.org/packages/a5/be/6f3b8f8ff89eaf480f59bbcd3d18c10ee0206891aa5fb4a25280c031801c/psutil-1.1.0.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.4.exe","hashes":{"sha256":"bc53be7f65ccb8a530e0c496a8db3ecd78b57b1faa443e43e228df327cf26899"},"provenance":null,"requires-python":null,"size":142269,"upload-time":"2013-09-28T09:51:43.685077Z","url":"https://files.pythonhosted.org/packages/ec/60/0a9a96611c77866a2c4140159a57c2702523ebc5371eaa24e0353d88ac2c/psutil-1.1.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.5.exe","hashes":{"sha256":"4a587008d6aae91fd1d3fee21141a27b4a2978fff75c80e57a37678a63a1cfcc"},"provenance":null,"requires-python":null,"size":142249,"upload-time":"2013-09-28T09:52:12.373832Z","url":"https://files.pythonhosted.org/packages/b5/16/dd5a768773cb017c4a66d69ec49abdd33df831215418e4c116639ed41afb/psutil-1.1.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.6.exe","hashes":{"sha256":"be853a268f0f3c3336c9bb648253db85ef2024213b6fc199f84b89643b74b130"},"provenance":null,"requires-python":null,"size":276570,"upload-time":"2013-09-28T09:53:51.099422Z","url":"https://files.pythonhosted.org/packages/cc/e0/adb41ece0bb2c27d75a41d98bd8f9802475d5d488df7d4905229596c0a70/psutil-1.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py3.2.exe","hashes":{"sha256":"dfdce701cfc7799dc1964debb486907b447fbdb361e06be786a9a5e11a1d5309"},"provenance":null,"requires-python":null,"size":275628,"upload-time":"2013-09-28T09:54:16.430842Z","url":"https://files.pythonhosted.org/packages/8f/c9/231df338b0cef518dbde2f98c1e0bb81142dc7dbec4a39267bb60dde1462/psutil-1.1.0.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py3.3.exe","hashes":{"sha256":"b4106508ea96ad24e1102de2a31a99c557ee608abab15d978bfe8d0f626e518c"},"provenance":null,"requires-python":null,"size":270259,"upload-time":"2013-09-28T09:54:43.666687Z","url":"https://files.pythonhosted.org/packages/a7/03/80f426e186dfdb202128c78b421b3ba34de461c32f6ed2c04041fc548d7b/psutil-1.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.tar.gz","hashes":{"sha256":"a5201e4c2a9b57e9e5d8de92b3a4006d093eedd9b56915b8279f365aaedd0f48"},"provenance":null,"requires-python":null,"size":165467,"upload-time":"2013-10-07T22:38:06.357263Z","url":"https://files.pythonhosted.org/packages/8d/0d/1a4bfc94f9cc783a510b3fc7efd5a2ef39858b3a2b6ab40a094a1ca8a54d/psutil-1.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"b8ccaef0a96d7ef40eda7493ffcaa3c5e2d63f531e090c3567e4198b38ca8e33"},"provenance":null,"requires-python":null,"size":305937,"upload-time":"2013-10-07T22:43:47.570605Z","url":"https://files.pythonhosted.org/packages/5a/b8/66489689c8751acd67bdbc8f37534176a14b63f74e74a229c30b230c8a18/psutil-1.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win-amd64-py3.2.exe","hashes":{"sha256":"fdabf11b316b0c406709aabb76f6c3cfdeb448c10ed38193786110f13e5c1248"},"provenance":null,"requires-python":null,"size":306563,"upload-time":"2013-10-07T22:44:09.209012Z","url":"https://files.pythonhosted.org/packages/e3/2d/cd5c620f9e5cb090eea573b2dc16574e746529347b6133b2f0b6e686d917/psutil-1.1.1.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.4.exe","hashes":{"sha256":"74bb1ccf0b28a914bb9d303642c08450ea6c5876851eb71e17f24b3d8672ca7d"},"provenance":null,"requires-python":null,"size":142751,"upload-time":"2013-10-07T22:45:21.036212Z","url":"https://files.pythonhosted.org/packages/28/9a/b83f884add09296894a1223c9c404cb57155c1c4317d318abf8c170e07b5/psutil-1.1.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.5.exe","hashes":{"sha256":"8e52d5b6af64e3eefbda30e0b16c9f29434ef6f79ea10b6fbd4520a6fbdb2481"},"provenance":null,"requires-python":null,"size":142731,"upload-time":"2013-10-07T22:41:32.428090Z","url":"https://files.pythonhosted.org/packages/8b/80/c41382a4f650f47a37300411169b7b248acf0b0925eb92cb22286362c3df/psutil-1.1.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.6.exe","hashes":{"sha256":"1e3a17b2a2f2bc138774f9b4b5ff52767f65ffb6349f9e05eb24f243c550ce1b"},"provenance":null,"requires-python":null,"size":277048,"upload-time":"2013-10-07T22:41:54.149373Z","url":"https://files.pythonhosted.org/packages/16/cd/25a3b9af88d130dd1084acab467b30996884219afc0a1e989d2a015ea54b/psutil-1.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.7.exe","hashes":{"sha256":"2e7691ddbde94b1ec7bb5b4cd986b2f763efc6be5b122dd499e156c9802a195b"},"provenance":null,"requires-python":null,"size":276822,"upload-time":"2013-10-07T22:42:18.407145Z","url":"https://files.pythonhosted.org/packages/42/78/eeacb1210abbe15cf06b9810e84afeabae1f9362abe389e8d5ca2c19df43/psutil-1.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py3.2.exe","hashes":{"sha256":"01276bd695b5cebc7bea7c97814713b0f3030861e30be88a60d40c9daf3d529c"},"provenance":null,"requires-python":null,"size":276106,"upload-time":"2013-10-07T22:42:41.551135Z","url":"https://files.pythonhosted.org/packages/92/14/90b9a4690f04ef1aab89a97a7b5407708f56785ccc264d9f9ce372feaea4/psutil-1.1.1.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py3.3.exe","hashes":{"sha256":"86ce0cfa7a95c7c1bc3e2f5358e446b01fc2be5f0c879c8def21e7ede9dc77de"},"provenance":null,"requires-python":null,"size":270733,"upload-time":"2013-10-07T22:43:07.069809Z","url":"https://files.pythonhosted.org/packages/a7/64/ba4601de7df6130c27f42bcec9f11da4ea905eda26d2f5a41efdb481f377/psutil-1.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.tar.gz","hashes":{"sha256":"adeb1afcb46327bed6603aa8981dce863f052043a52f003e2742ec7c3739677a"},"provenance":null,"requires-python":null,"size":165709,"upload-time":"2013-10-22T18:13:09.038583Z","url":"https://files.pythonhosted.org/packages/e4/b1/34a4bd75027d08c8db4f6301d6562e333c8d9131dca08b7f76f05aeae00a/psutil-1.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"4c9e952a7faf50f11fb8bcd6c12c952b063664d9327957c9f6abd498e6ef3bc8"},"provenance":null,"requires-python":null,"size":306157,"upload-time":"2013-10-22T18:16:38.181306Z","url":"https://files.pythonhosted.org/packages/f9/df/437db01296118d668cf654f097ad2b1c341291ba5dc4b5eb80f0a0a40c52/psutil-1.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win-amd64-py3.2.exe","hashes":{"sha256":"6e8ed376b63b15b09a94eff7998dae1f8506bd174d795e509cc735f875fddabe"},"provenance":null,"requires-python":null,"size":306782,"upload-time":"2013-10-22T18:17:05.663884Z","url":"https://files.pythonhosted.org/packages/c9/d0/1e413f0258d02bf77bc1d94002d041b8853584369e4af039cd5cf89e3270/psutil-1.1.2.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.4.exe","hashes":{"sha256":"ecc5ab4537259db7254c6f6dc7b7cb5a7f398c322a01612c272a8222696334a8"},"provenance":null,"requires-python":null,"size":142969,"upload-time":"2013-10-22T18:13:39.868851Z","url":"https://files.pythonhosted.org/packages/a7/63/fd5770ec4fe87d30bd836989d314b85662c775a52dbd017747fc69fe8f0e/psutil-1.1.2.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.5.exe","hashes":{"sha256":"e05d83d6b53ea24333c8b04d329e28ff11ecad75945f371ff5ce7f785df36aee"},"provenance":null,"requires-python":null,"size":142950,"upload-time":"2013-10-22T18:14:00.511107Z","url":"https://files.pythonhosted.org/packages/8d/e1/22650079452725e44ec790c3e75282f4d341f359b213b2afc7f2ada46930/psutil-1.1.2.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.6.exe","hashes":{"sha256":"f06ee6e1508a12afcfed04a4022ded9f872e2a964a62bd86617ece943d89ab01"},"provenance":null,"requires-python":null,"size":277266,"upload-time":"2013-10-22T18:14:33.903116Z","url":"https://files.pythonhosted.org/packages/3e/a4/5177488368f230acd4708a117c0820fb16843521e2a7a492078a2335bb9f/psutil-1.1.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.7.exe","hashes":{"sha256":"6c5be5538202ed7419911178ded41c65e118104fa634109f528a6d2d3e50a7d0"},"provenance":null,"requires-python":null,"size":277039,"upload-time":"2013-10-22T18:14:56.072979Z","url":"https://files.pythonhosted.org/packages/5e/d6/56f2891f6dd56f950866cc39892e5a56e85331d97c39e2634cfc4014f0df/psutil-1.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py3.2.exe","hashes":{"sha256":"5cfa97b52fb48dbb8255c120b567be4f06802afa7d2fe71b8fe7c7c4ee53ad88"},"provenance":null,"requires-python":null,"size":276327,"upload-time":"2013-10-22T18:15:25.393310Z","url":"https://files.pythonhosted.org/packages/dd/be/1aea1e7a1a3fb44f4c8d887a1d55e960de283d86875f15457a284268e197/psutil-1.1.2.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py3.3.exe","hashes":{"sha256":"f7dc7507fb9d4edb42709b356eb2e4b3da356efa54d83900e4cef59f3adebfbf"},"provenance":null,"requires-python":null,"size":270952,"upload-time":"2013-10-22T18:15:51.874969Z","url":"https://files.pythonhosted.org/packages/f5/7c/1a33b78a66a96e740e197ae55719496ba57bb9cee32f710a5a6affa68cc8/psutil-1.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.tar.gz","hashes":{"sha256":"5e1164086a7ed3b863ebd12315d35086e22252b328401fce901a0862050ef98c"},"provenance":null,"requires-python":null,"size":165550,"upload-time":"2013-11-07T20:47:17.039714Z","url":"https://files.pythonhosted.org/packages/93/fa/1f70b7fcdff77348f4e79d84cc9b568596874ca34940ede78c63d503a095/psutil-1.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"43cbf089cbe160d193e1658fe4eec4a719430432e866334b25dc3acef73c3e61"},"provenance":null,"requires-python":null,"size":306145,"upload-time":"2013-11-07T21:05:00.986042Z","url":"https://files.pythonhosted.org/packages/03/ec/f05db404504d67a19397e17e64f0276cc610a9dc450eb51ed70436f37c43/psutil-1.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.4.exe","hashes":{"sha256":"ef7445e1b0449af34ff51a9e1d3f030b8794c9fcd2b99d072b2e815d00e8a783"},"provenance":null,"requires-python":null,"size":142958,"upload-time":"2013-11-07T21:16:11.539460Z","url":"https://files.pythonhosted.org/packages/37/c2/5d40dd0a36f0280c1dea0f651cfde79b5e99a0e6fab92273fa3ac41055f0/psutil-1.1.3.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.5.exe","hashes":{"sha256":"b7f4103b1058d2ee597f88902ff50f19421b95c18ac9b82ea9e8a00091786131"},"provenance":null,"requires-python":null,"size":142940,"upload-time":"2013-11-07T21:15:50.256574Z","url":"https://files.pythonhosted.org/packages/ee/cb/7d42052f4057c6233ba3f4b7afade92117f58c5d7544ee6ab16e82c515c7/psutil-1.1.3.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.6.exe","hashes":{"sha256":"e21b9c5a26ad3328b6f5b629e30944a9b7e56ea6ac0aa051455c8a352e67fbca"},"provenance":null,"requires-python":null,"size":277254,"upload-time":"2013-11-07T20:58:49.794773Z","url":"https://files.pythonhosted.org/packages/b2/7e/c42d752b333c5846d88a8b56bbab23325b247766c100dc6f68c6bc56019d/psutil-1.1.3.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.7.exe","hashes":{"sha256":"fc9732a4dea2a3f73f9320472aa54e1cfae45d324bea24bd207d88d3a0a281b0"},"provenance":null,"requires-python":null,"size":277028,"upload-time":"2013-11-07T21:16:56.712839Z","url":"https://files.pythonhosted.org/packages/2b/76/eec917b6f74ea9bd20a55dd8b4b01e69689278235335dbedc2b9be212815/psutil-1.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py3.2.exe","hashes":{"sha256":"a141d59c03a28bdd32dfce11a5d51b9a63a8e6c6c4245e416eeffcc1c43f1de9"},"provenance":null,"requires-python":null,"size":276315,"upload-time":"2013-11-07T21:02:04.504719Z","url":"https://files.pythonhosted.org/packages/3a/b9/748bbd0c53c74c682051137830048abd0126ae50e3bf4d5854a0188da143/psutil-1.1.3.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py3.3.exe","hashes":{"sha256":"36e855ea7c872beb04f22d968c55f1ef897df0261ea993ecf5ae6e8216939a55"},"provenance":null,"requires-python":null,"size":270942,"upload-time":"2013-11-07T21:04:24.025560Z","url":"https://files.pythonhosted.org/packages/e6/7b/655e08abdf19d06c8a3fd6fba35932867c80fcff05c97cf909b5d364603c/psutil-1.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.tar.gz","hashes":{"sha256":"c3a2b02e92363837499680760b674b3bf3bd03dd9528a5dc41392625a61b162a"},"provenance":null,"requires-python":null,"size":166747,"upload-time":"2013-11-20T20:02:56.854747Z","url":"https://files.pythonhosted.org/packages/77/89/8ee72ae2b5e3c749e43a9c1e95d61eceff022ab7c929cdde8a3b7539a707/psutil-1.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"2bedcda55618f71bc98d6669bd190e2b51b940e3a3af2512dd77745cb1ebc101"},"provenance":null,"requires-python":null,"size":307021,"upload-time":"2013-11-20T20:08:29.733072Z","url":"https://files.pythonhosted.org/packages/9c/d1/598bcaa9701e06561cb8823a7a04ee44e702b0e327ef0a65bdd97b019613/psutil-1.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.4.exe","hashes":{"sha256":"417695f66562a22ac4f92bf7df1e9dda6264579eb308badd2c3e85df88ab9436"},"provenance":null,"requires-python":null,"size":143831,"upload-time":"2013-11-20T20:18:44.725428Z","url":"https://files.pythonhosted.org/packages/a4/ba/4b54baace7b49b73df74b54815337878137d25fda506e518f2d8dd2472fc/psutil-1.2.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.5.exe","hashes":{"sha256":"40747c59cf92cef26d595a9cefaac9c1a4bc0290abff6004bb092ee2ddec7d7b"},"provenance":null,"requires-python":null,"size":143812,"upload-time":"2013-11-20T20:18:35.170091Z","url":"https://files.pythonhosted.org/packages/88/17/3ba1b45ec63d666c2631f16679adb819564feb775cfd9e774d23e5774a44/psutil-1.2.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.6.exe","hashes":{"sha256":"fb0c3942a48fde9c52828dae4074381556eff8eb5651bdc9c4c107b40dc8318e"},"provenance":null,"requires-python":null,"size":278130,"upload-time":"2013-11-20T20:07:34.211552Z","url":"https://files.pythonhosted.org/packages/ae/4a/e62000dbe462270c30f1e2f2dcc913596f70e264d9a656f881bb9f487283/psutil-1.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.7.exe","hashes":{"sha256":"1024eff34ff14f2899186549ca8f8e461403e5700910e52eb013ffffb078cda3"},"provenance":null,"requires-python":null,"size":277905,"upload-time":"2013-11-20T20:18:07.310116Z","url":"https://files.pythonhosted.org/packages/9d/1a/b8e679a7e47229d07e41439a43fc1be48c0d34774e5e35ad6730398485bd/psutil-1.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py3.2.exe","hashes":{"sha256":"2b588673530d67f45287c31b6dfef02ee4e753aee7e7216448e6dbcb95b482e1"},"provenance":null,"requires-python":null,"size":277189,"upload-time":"2013-11-20T20:05:01.902941Z","url":"https://files.pythonhosted.org/packages/05/10/20a2364e4e69206d66da106288b65bf5bbde0d648aad5bb829f3eb08fabb/psutil-1.2.0.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py3.3.exe","hashes":{"sha256":"3e23d3b1ef5f20a458092887d1df32f55eaf4fc86f2ed49cc68cf164537128d6"},"provenance":null,"requires-python":null,"size":271857,"upload-time":"2013-11-20T20:05:28.007235Z","url":"https://files.pythonhosted.org/packages/24/39/4630dff3d0fa4896db905a118877fce5b72ced6629ca9958e207f7a0b198/psutil-1.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.tar.gz","hashes":{"sha256":"508e4a44c8253a386a0f86d9c9bd4a1b4cbb2f94e88d49a19c1513653ca66c45"},"provenance":null,"requires-python":null,"size":167397,"upload-time":"2013-11-25T20:06:20.566209Z","url":"https://files.pythonhosted.org/packages/8a/45/3b9dbd7a58482018927f756de098388ee252dd230143ddf486b3017117b1/psutil-1.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"f3af7b44925554531dff038e0401976a6b92b089ecca51d50be903420d7a262d"},"provenance":null,"requires-python":null,"size":307074,"upload-time":"2013-11-25T20:09:25.387664Z","url":"https://files.pythonhosted.org/packages/40/47/665755b95ad75e6223af96f2d7c04667f663a53dede0315df9832c38b60d/psutil-1.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"26fd84571ac026861d806a9f64e3bbfd38d619a8195c1edfd31c0a9ee2295b03"},"provenance":null,"requires-python":null,"size":329716,"upload-time":"2014-02-06T00:57:26.931878Z","url":"https://files.pythonhosted.org/packages/d8/65/2e9941492b3d001a87d87b5e5827b1f3cec42e30b7110fa82d24be8c4526/psutil-1.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.4.exe","hashes":{"sha256":"84a4e7e2de1ca6f45359cfd7fd60c807f494385a6d97804bd58759a94b9c5e2d"},"provenance":null,"requires-python":null,"size":143888,"upload-time":"2013-11-25T20:07:25.331363Z","url":"https://files.pythonhosted.org/packages/5c/99/d9147b76eea8c185b6cefbb46de73ae880e5ef0ff36d93cb3f6084e50d59/psutil-1.2.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.5.exe","hashes":{"sha256":"aad7d81607b3ad740fee47b27a9f1434a05fe35dc68abefb1f961f74bae3c3f9"},"provenance":null,"requires-python":null,"size":143868,"upload-time":"2013-11-25T20:07:40.595942Z","url":"https://files.pythonhosted.org/packages/bd/a0/5087d4a5145326a5a07a53ed4f9cd5c09bf5dad4f8d7b9850e6aaa13caa2/psutil-1.2.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.6.exe","hashes":{"sha256":"3ad3a40afd859cf0217a2d643c74be3a12ed2f54ebd4a91c40fa7b13084573c6"},"provenance":null,"requires-python":null,"size":278185,"upload-time":"2013-11-25T20:07:59.926444Z","url":"https://files.pythonhosted.org/packages/c6/e0/67810b602a598488d1f2982451655427effe7c7062184fe036c2b5bc928f/psutil-1.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.7.exe","hashes":{"sha256":"02fb79b9e5336ff179c44ce2017308cf46316e19bea70abb8855afd808db2a0f"},"provenance":null,"requires-python":null,"size":277955,"upload-time":"2013-11-25T20:08:20.641779Z","url":"https://files.pythonhosted.org/packages/5b/7f/9334b57597acabaaf2261c93bb9b1f9f02cdfef5c1b1aa808b262f770adb/psutil-1.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py3.2.exe","hashes":{"sha256":"7e64d065f12e8f941f2dbb2f3df0887b2677fee7b2b4c50ed91e490e094c7273"},"provenance":null,"requires-python":null,"size":277243,"upload-time":"2013-11-25T20:08:40.628203Z","url":"https://files.pythonhosted.org/packages/ae/c5/2842c69c67ae171f219efa8bb11bddc2fcec2cea059721e716fe4d48b50c/psutil-1.2.1.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py3.3.exe","hashes":{"sha256":"1a1c8e6635949a698b6ade3d5d2a8368daff916d8122cf13286c79a52ec8d7a1"},"provenance":null,"requires-python":null,"size":271900,"upload-time":"2013-11-25T20:09:05.464640Z","url":"https://files.pythonhosted.org/packages/86/df/007ca575da6ee7cbb015dc00122028ee0c97fc6a0c9e8bc02333753bfd2f/psutil-1.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.tar.gz","hashes":{"sha256":"38af34b0f40a4f50988a7401b7111ae4468beb5bcce0fbae409504dd3d5f2e8d"},"provenance":null,"requires-python":null,"size":207168,"upload-time":"2014-03-10T11:23:38.510743Z","url":"https://files.pythonhosted.org/packages/9c/2c/d4380234ddc21ecfb03691a982f5f26b03061e165658ac455b61886fe3ff/psutil-2.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"32f27f9be7c03f60f61dc27af2464cfb1edd64525d05b81478e80dc12913fe3b"},"provenance":null,"requires-python":null,"size":310569,"upload-time":"2014-03-10T11:16:14.555076Z","url":"https://files.pythonhosted.org/packages/c3/10/c0e1b505d7d2b4a7f3294c1b4b2bc2644a4629462d777fe2cdcd57b1debe/psutil-2.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"219cf2e5832cf68798e522e815d79a7307dea1f6b1d9b2372704f6e7fea085f3"},"provenance":null,"requires-python":null,"size":309024,"upload-time":"2014-03-10T11:16:29.122028Z","url":"https://files.pythonhosted.org/packages/eb/db/ab023b5ce09f314ee58ee4b9e73e85172dd06272501570a34a1afe6115c2/psutil-2.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"12b9c02e108e887a43e0ed44e3a8e9968e65236d6d0b79c45891471ea2b9e14d"},"provenance":null,"requires-python":null,"size":310130,"upload-time":"2014-03-10T11:16:49.562585Z","url":"https://files.pythonhosted.org/packages/67/94/0dded4aab9c4992bddb311d2ae8fd9a638df5f6039d12a4fe66481f3ea1c/psutil-2.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.4.exe","hashes":{"sha256":"54d8636623a4f676a9a38b0afe3dfad5f1b90f710f2cb6c55e1a0803813d76a5"},"provenance":null,"requires-python":null,"size":143691,"upload-time":"2014-03-10T11:20:46.107824Z","url":"https://files.pythonhosted.org/packages/76/a3/48c0984c0b65be53a9e3e090df0cd5f3e6bddd767c3f8e62cf286be240e1/psutil-2.0.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.5.exe","hashes":{"sha256":"f669fe7e7cab107fb738362366cc9e0ecda532269ac3a9815a28930b474edf0b"},"provenance":null,"requires-python":null,"size":147735,"upload-time":"2014-03-10T11:15:02.487295Z","url":"https://files.pythonhosted.org/packages/f8/ba/346cc719249b9a5281dab059cb8796aff6faf487142f50966fc08330ad79/psutil-2.0.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.6.exe","hashes":{"sha256":"33b580008e0c65073198472da81d0d0c50d2a30e3e82c269713b1e3bdf14c2c6"},"provenance":null,"requires-python":null,"size":280920,"upload-time":"2014-03-10T11:15:17.132174Z","url":"https://files.pythonhosted.org/packages/2f/6f/7326e900c5333d59aa96a574d13321c94a9357ab56b0dd489b8f24ebab78/psutil-2.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.7.exe","hashes":{"sha256":"4f9c78cdd57e1c83242096f8343617ae038efd1c23af3864c25992335eabed3f"},"provenance":null,"requires-python":null,"size":280693,"upload-time":"2014-03-10T11:15:31.704968Z","url":"https://files.pythonhosted.org/packages/18/dd/c81485b54894c35fd8b62822563293db7c4dd17a05ea4eade169cf383266/psutil-2.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py3.3.exe","hashes":{"sha256":"380f8ff680ce3c8fdd7b31a3fe42d697b15a0c8677559691ed90b755051a5acf"},"provenance":null,"requires-python":null,"size":275631,"upload-time":"2014-03-10T11:15:45.735326Z","url":"https://files.pythonhosted.org/packages/77/5a/e87efff3e46862421a9c87847e63ebecd3bb332031305b476399918fea4f/psutil-2.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py3.4.exe","hashes":{"sha256":"4239b2431c825db2ad98e404a96320f0c78fb1c7d5bdf52a49a00d36bedfa0df"},"provenance":null,"requires-python":null,"size":275638,"upload-time":"2014-03-10T11:16:00.263121Z","url":"https://files.pythonhosted.org/packages/c4/80/35eb7f189482d25e3669871e7dcd295ec38f792dc4670b8635d72b4f949a/psutil-2.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.tar.gz","hashes":{"sha256":"d1e3ce46736d86164b6d72070f2cbaf86dcf9db03066b7f36a7b302e334a8d01"},"provenance":null,"requires-python":null,"size":211640,"upload-time":"2014-04-08T14:59:37.598513Z","url":"https://files.pythonhosted.org/packages/6c/d1/69431c4fab9b5cecaf28f2f2e0abee21805c5783c47143db5f0f7d42dbec/psutil-2.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"9dc35f899946d58baf5b805ebed2f723f75d93f6b6bce212f79581d40f7276db"},"provenance":null,"requires-python":null,"size":312139,"upload-time":"2014-04-08T15:13:59.193442Z","url":"https://files.pythonhosted.org/packages/7a/3c/ce6a447030cdb50cf68a3988337df0c42e52abf45c3adfea3b225760eb70/psutil-2.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"46232b7b0eb48c6ec1f12d17b2cc15ec3cc70ed952f2c98edb50aaa405dafa5d"},"provenance":null,"requires-python":null,"size":310591,"upload-time":"2014-04-08T15:14:05.019441Z","url":"https://files.pythonhosted.org/packages/cd/60/7134a7f812ef1eba9373c86b95ce6254f5f58928baba04af161f0066629f/psutil-2.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"fa770a2c9e384924df021aed9aa5a6f92db8062a7f700113ba3943e262028454"},"provenance":null,"requires-python":null,"size":311682,"upload-time":"2014-04-08T15:14:24.701683Z","url":"https://files.pythonhosted.org/packages/45/56/3ac0a63799d54cb9b1914214944aed71e49297fb90de92b8d1fe20de3bd8/psutil-2.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.4.exe","hashes":{"sha256":"15d0d10bbd60e461690188cdd209d6688a1112648589bf291b17cc84e99cb6e7"},"provenance":null,"requires-python":null,"size":145275,"upload-time":"2014-04-08T15:12:04.809516Z","url":"https://files.pythonhosted.org/packages/09/e8/f6e4209b3d6373ea11fa340c2e6e4640a7ee53ef4697c49d610ffdf86674/psutil-2.1.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.5.exe","hashes":{"sha256":"381ac1027a270c04cf6decdc011a28a4270105d009f213d62daec4b3116b92b6"},"provenance":null,"requires-python":null,"size":149378,"upload-time":"2014-04-08T15:12:25.638187Z","url":"https://files.pythonhosted.org/packages/6d/05/0e7213e6f0dc71490a523a70e6ab5e7cd5140d87dc93a4447da58c440d6b/psutil-2.1.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.6.exe","hashes":{"sha256":"7eda949fbdf89548b0e52e7d666e096ea66b30b65bcbe04d305e036a24a76d11"},"provenance":null,"requires-python":null,"size":282509,"upload-time":"2014-04-08T15:12:57.055047Z","url":"https://files.pythonhosted.org/packages/f2/88/a856c5ed36d15b8ad74597f380baa29891ec284e7a1be4cb2f91f8453bd8/psutil-2.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.7.exe","hashes":{"sha256":"f1ec06db1b0d27681a1bea4c9f0b33705bc5a00035c32f168da0ea193883bb91"},"provenance":null,"requires-python":null,"size":282317,"upload-time":"2014-04-08T15:13:11.798772Z","url":"https://files.pythonhosted.org/packages/d0/17/ec99e9252dae834f4a012e13c804c86907fcb1cb474f7b1bc767562bfa7b/psutil-2.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py3.3.exe","hashes":{"sha256":"feccef3ccf09785c6f81e67223cf2d8736a90c8994623dd75f683dd2bf849235"},"provenance":null,"requires-python":null,"size":277268,"upload-time":"2014-04-08T15:13:26.711424Z","url":"https://files.pythonhosted.org/packages/f0/0a/32abfd9b965c9a433b5011574c904372b954330c170c6f92b637d661ecd2/psutil-2.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py3.4.exe","hashes":{"sha256":"4719134be984b0f2ba072ff761f334c2f3dbb8ca6af70ba43d8ca31b7e13c3db"},"provenance":null,"requires-python":null,"size":277279,"upload-time":"2014-04-08T15:13:41.834397Z","url":"https://files.pythonhosted.org/packages/f5/3b/eab6a8d832d805c7a00d0f2398c12a32bea7e8b6eb7d5fbdf869e4bcc9e0/psutil-2.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.tar.gz","hashes":{"sha256":"bf812a4aa6a41147d0e96e63d826eb7582fda6b54ad8f22534354b7f8ac45593"},"provenance":null,"requires-python":null,"size":216796,"upload-time":"2014-04-30T14:27:01.651394Z","url":"https://files.pythonhosted.org/packages/64/4b/70601d39b8e445265ed148affc49f7bfbd246940637785be5c80e007fa6e/psutil-2.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"1b5ee64306535a625b77648e32f1b064c31bf58bfc7a37cde6c7bb3fa4abb6bd"},"provenance":null,"requires-python":null,"size":312763,"upload-time":"2014-04-30T14:31:09.411365Z","url":"https://files.pythonhosted.org/packages/e6/50/df05e0cbfcf20f022756d5e2da32b4f4f37d5bca6f5bd6965b4ef0460e8b/psutil-2.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"9e28fe35e1185d59ba55313d90efc122d45b7a2e3fa304869024d26b0a809bc5"},"provenance":null,"requires-python":null,"size":311262,"upload-time":"2014-04-30T14:31:28.733500Z","url":"https://files.pythonhosted.org/packages/a3/a9/b3f114141a49244739321f221f35c300ac7f34ec9e3a352ea70c9fae41f8/psutil-2.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"3afa327284a218d6a25b2f3520135337cfa4a47eaea030273ad8eb02894d60fe"},"provenance":null,"requires-python":null,"size":312365,"upload-time":"2014-04-30T14:31:47.981590Z","url":"https://files.pythonhosted.org/packages/c7/67/2aae3f66090c2e06fa60c29a2e554fd2a718949aca53bca78d640212cb34/psutil-2.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.4.exe","hashes":{"sha256":"7c76d99bfaeafbcf66096c69b8fca1f7269603d66cad1b14cd8dd93b14bceeb0"},"provenance":null,"requires-python":null,"size":145848,"upload-time":"2014-04-30T14:37:26.909225Z","url":"https://files.pythonhosted.org/packages/27/4e/c9b4802420d6b5d0a844208c9b8a4e25b3c37305428e40bc1da6f5076891/psutil-2.1.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.5.exe","hashes":{"sha256":"6aeb358b66cc4367378edadb928c77e82646f36858b2a516298d1917aa6aca25"},"provenance":null,"requires-python":null,"size":149998,"upload-time":"2014-04-30T14:28:49.630619Z","url":"https://files.pythonhosted.org/packages/5d/e9/69dee6454940bb102fbbcaa1e44b46bfefc4c0bf53e5e3835d3de3ebc9ae/psutil-2.1.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.6.exe","hashes":{"sha256":"a53546550067773920e1f7e6e3f1fad2f2befe45c6ff6c94e22b67f4b54c321a"},"provenance":null,"requires-python":null,"size":283143,"upload-time":"2014-04-30T14:29:06.869445Z","url":"https://files.pythonhosted.org/packages/9f/45/d2dfa6bdd9b64bfd46ad35af774c3819e0d5abf23a99d51adf11984ca658/psutil-2.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.7.exe","hashes":{"sha256":"37ce747f9042c375f62e861d627b5eb7bace24767303f5d0c4c03d17173a551c"},"provenance":null,"requires-python":null,"size":282941,"upload-time":"2014-04-30T14:29:27.059188Z","url":"https://files.pythonhosted.org/packages/78/47/14db8651f9863d301c0673d25fa22b87d13fde2974f94854502886a21fd1/psutil-2.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py3.3.exe","hashes":{"sha256":"8f0f88752e1e9bfeced78daf29d90e0028d17f39b805bb0acf70fefe77ba5ccb"},"provenance":null,"requires-python":null,"size":277944,"upload-time":"2014-04-30T14:29:49.405456Z","url":"https://files.pythonhosted.org/packages/f1/63/2fcaa58b101dce55a12f768508a7f0a0028ccc5a90633d86dd4cc0bcdb52/psutil-2.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py3.4.exe","hashes":{"sha256":"bc353df13a6ea40651cba82810a38590e2439c158cf6f130cd0876d0dda53118"},"provenance":null,"requires-python":null,"size":279092,"upload-time":"2014-04-30T14:30:08.449338Z","url":"https://files.pythonhosted.org/packages/5f/26/0be35c7f3dc9e78d405ed6be3aa76a9ce97b51e0076db98408a6f2c288fb/psutil-2.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp26-none-win32.whl","hashes":{"sha256":"ccb5e28357a4b6c572b97e710a070f349e9b45172315eaed6e0b72e86d333b68"},"provenance":null,"requires-python":null,"size":83869,"upload-time":"2014-09-21T13:47:09.164866Z","url":"https://files.pythonhosted.org/packages/c7/22/811ac7c641191e3b65c053c95eb34f6567bbc5155912630464271ab6f3df/psutil-2.1.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp27-none-win32.whl","hashes":{"sha256":"41992126c0281c2f5f279a0e8583382a3b840bd0d48262dfb7bc3cb67bdc6587"},"provenance":null,"requires-python":null,"size":83672,"upload-time":"2014-09-21T13:47:14.034296Z","url":"https://files.pythonhosted.org/packages/79/9d/154b179a73695ae605856c2d77ab5da2a66ef4819c3c4f97e4ab297a2902/psutil-2.1.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp27-none-win_amd64.whl","hashes":{"sha256":"79cb57bba4cbeebb7e445d19c531107493458a47d0f4c888ce49fc8dec670c32"},"provenance":null,"requires-python":null,"size":85848,"upload-time":"2014-09-21T13:47:30.044802Z","url":"https://files.pythonhosted.org/packages/00/ae/567c30ff44c263cc78dfe50197184e58b62ca9fcfebad144cd235f5d8d2d/psutil-2.1.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp33-none-win32.whl","hashes":{"sha256":"c5e6424833620f0d0a70ebc1305260bfad4b5c73989b8a2e8bd01bf39b4b600c"},"provenance":null,"requires-python":null,"size":83742,"upload-time":"2014-09-21T13:47:18.481401Z","url":"https://files.pythonhosted.org/packages/62/4d/be87c318274ade9d6589e8545dd5d5bb4bcc42e92334d88d948cf8daa36b/psutil-2.1.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp33-none-win_amd64.whl","hashes":{"sha256":"b4c69627c245025b9209acbffc066ff8ac1237d13e2eddd0461f3149969b7da3"},"provenance":null,"requires-python":null,"size":85822,"upload-time":"2014-09-21T13:47:33.681070Z","url":"https://files.pythonhosted.org/packages/bc/52/ab3c88a0574275ec33d9de0fe7dc9b1a32c573de0e468502876bc1df6f84/psutil-2.1.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp34-none-win32.whl","hashes":{"sha256":"97a1d0ddbb028a20feffaf7fa3d0c451004abc74ef7ea1e492d039f8b7278399"},"provenance":null,"requires-python":null,"size":83748,"upload-time":"2014-09-21T13:47:25.367914Z","url":"https://files.pythonhosted.org/packages/ef/4a/c956675314e4a50b319d9894c2ee2d48ce83df4d639d9c2fc06a99415dec/psutil-2.1.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp34-none-win_amd64.whl","hashes":{"sha256":"a78226f9236c674d43b206e7da06478cf2bcf10e5aee647f671878ea2434805f"},"provenance":null,"requires-python":null,"size":85789,"upload-time":"2014-09-21T13:47:38.760178Z","url":"https://files.pythonhosted.org/packages/b9/41/68b5fc38b97e037ef55e6475d618c079fe2b5d148f5e3fda795c21d888a7/psutil-2.1.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.tar.gz","hashes":{"sha256":"897e5163e0669001bf8bcb0557362f14703356336519082a93c38d54e5b392e4"},"provenance":null,"requires-python":null,"size":223595,"upload-time":"2014-09-21T13:50:56.650693Z","url":"https://files.pythonhosted.org/packages/53/6a/8051b913b2f94eb00fd045fe9e14a7182b6e7f088b12c308edd7616a559b/psutil-2.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"3ed7ef4ab59894e6eb94adae0b656733c91906af169b55443329315322cd63b3"},"provenance":null,"requires-python":null,"size":319335,"upload-time":"2014-09-21T13:48:15.057062Z","url":"https://files.pythonhosted.org/packages/a6/88/cc912d38640ddf3441db1f85f8ff8a87f906056554239a4a211970cf6446/psutil-2.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py3.3.exe","hashes":{"sha256":"e20f317187a7186e13dc6fdd6960083c227db86fffbb145fe000e8c167a854a6"},"provenance":null,"requires-python":null,"size":317817,"upload-time":"2014-09-21T13:48:20.665047Z","url":"https://files.pythonhosted.org/packages/d2/b4/887323eb2b0b5becb5a7b77ab04167741346fddffe27bc6ae5deeffcc3c1/psutil-2.1.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py3.4.exe","hashes":{"sha256":"f7f2a61be7a828656dceffdf0d2fa304a144db6ab5ec4e2b033108f3822df6ff"},"provenance":null,"requires-python":null,"size":317804,"upload-time":"2014-09-21T13:48:33.226186Z","url":"https://files.pythonhosted.org/packages/57/e5/c041b08bea32246f53b7cf27b897e882a695f3ea95eb632b384dd35cf41f/psutil-2.1.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.4.exe","hashes":{"sha256":"77aac0179fe0e8b39c9b25ea732285d307bb941e715075366dc467b8609ebf09"},"provenance":null,"requires-python":null,"size":155426,"upload-time":"2014-09-21T13:54:03.017765Z","url":"https://files.pythonhosted.org/packages/ce/3f/7c434baec7ca47e65fb1e3bcbc8fe1c9f504d93c750a7fa4f98b16635920/psutil-2.1.2.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.5.exe","hashes":{"sha256":"982f4568876d10881f9c90fe992d1d9af353a59b5de7771cc4766d7432aee7ab"},"provenance":null,"requires-python":null,"size":155414,"upload-time":"2014-09-21T13:47:47.970150Z","url":"https://files.pythonhosted.org/packages/46/86/2971b31e2637ddc53adce2d997472cee8d4dec366d5a1e140945b95860d5/psutil-2.1.2.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.6.exe","hashes":{"sha256":"f35d91b5fc2b52472c0766fa411ac22f7a5ed8b1ececd96c1486380bdad0bb41"},"provenance":null,"requires-python":null,"size":289714,"upload-time":"2014-09-21T13:47:52.391726Z","url":"https://files.pythonhosted.org/packages/dd/3e/cdc3d4f343e6edc583d0686d1d20d98f9e11de35c51dc7990525ab2ca332/psutil-2.1.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.7.exe","hashes":{"sha256":"dbd9e4fbe08c0ff6d4a87a4d6dfae16fccd03346f980cffa105941de8801d00b"},"provenance":null,"requires-python":null,"size":289515,"upload-time":"2014-09-21T13:47:57.624310Z","url":"https://files.pythonhosted.org/packages/fb/01/73753147113f6db19734db6e7ac994cee5cce0f0935e12320d7aa1b56a14/psutil-2.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py3.3.exe","hashes":{"sha256":"73428e7b695e5889f07f1b37c5ec0cc0f49d3808dc40986050f1b13fd7c4c71e"},"provenance":null,"requires-python":null,"size":284506,"upload-time":"2014-09-21T13:48:02.428235Z","url":"https://files.pythonhosted.org/packages/b0/97/ba13c3915aba7776bb0d23819c04255230c46df1f8582752f7e0382c0b67/psutil-2.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py3.4.exe","hashes":{"sha256":"e2326c7b9f64a0f22891d3b362efcd92389e641023b07bc54567280c0c2e160d"},"provenance":null,"requires-python":null,"size":284538,"upload-time":"2014-09-21T13:48:08.358281Z","url":"https://files.pythonhosted.org/packages/88/bd/843fadc578d62f2888d5a7b4e2a418a28140ac239a5a5039c0d3678df647/psutil-2.1.2.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp26-none-win32.whl","hashes":{"sha256":"331a3b0b6688f95acebe87c02efca53003e613f1c7892a09063a9d3c0af33656"},"provenance":null,"requires-python":null,"size":83927,"upload-time":"2014-09-26T20:26:46.078909Z","url":"https://files.pythonhosted.org/packages/72/4c/f93448dfe2dec286b6eae91a00c32f1fbf65506e089354c51db76f4efdaf/psutil-2.1.3-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp27-none-win32.whl","hashes":{"sha256":"5ee9e17f6e92aaec7c53f6483c6708eec22124060a52ac339d104effcb4af37f"},"provenance":null,"requires-python":null,"size":83731,"upload-time":"2014-09-26T20:26:51.215134Z","url":"https://files.pythonhosted.org/packages/27/c5/a644b5df545c467569c7b8e768717fad6758eab8e2544b7d412d07c30ffe/psutil-2.1.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp27-none-win_amd64.whl","hashes":{"sha256":"47ea728de15c16a7cb476eb4f348be57e9daebfa730b9a4a104e9086da2cf6cd"},"provenance":null,"requires-python":null,"size":85906,"upload-time":"2014-09-26T20:27:07.470817Z","url":"https://files.pythonhosted.org/packages/89/82/1070ecb59d83967ddb2ea58b41ba2b9ffa8f3f686a3f49b8d33dc5684964/psutil-2.1.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp33-none-win32.whl","hashes":{"sha256":"ae0aac37af1d7d6821bb11dcffb12bb885b44773858bd40703e9dcb14325369c"},"provenance":null,"requires-python":null,"size":83804,"upload-time":"2014-09-26T20:26:56.233644Z","url":"https://files.pythonhosted.org/packages/b6/fc/7f2ceac523bb5ce9ea93bd85806ebb2d5327f0e430cbb6e70d32e4306e1d/psutil-2.1.3-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp33-none-win_amd64.whl","hashes":{"sha256":"45ebaf679050a6d64a32a91d901791868a29574f6961629997dcd4661edb4ece"},"provenance":null,"requires-python":null,"size":85884,"upload-time":"2014-09-26T20:27:12.800416Z","url":"https://files.pythonhosted.org/packages/45/33/a50ed3def836cd8fc53ce04d09e50df67a6816fd24a819e8e9c45b93fc74/psutil-2.1.3-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp34-none-win32.whl","hashes":{"sha256":"0ae7d503fc458181af7dd9a2bbf43c4b5fc7cd6e6797ef4a6d58bb8046026d76"},"provenance":null,"requires-python":null,"size":83804,"upload-time":"2014-09-26T20:27:02.155242Z","url":"https://files.pythonhosted.org/packages/55/e9/c122a9d2528cc36ff4f2824a714ee6d5da5462e4b6725e6b1c7098142c4a/psutil-2.1.3-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp34-none-win_amd64.whl","hashes":{"sha256":"da0961ba9a6d97c137381cd59b7d330894dd4f7deb5a8291bc3251375fd6d6ec"},"provenance":null,"requires-python":null,"size":85840,"upload-time":"2014-09-26T20:27:18.296315Z","url":"https://files.pythonhosted.org/packages/6f/97/5e01561cde882306c28e462b427f67d549add65f5fca324bf0bbdf831d21/psutil-2.1.3-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.tar.gz","hashes":{"sha256":"b434c75f01715777391f10f456002e33d0ca14633f96fdbd9ff9139b42d9452c"},"provenance":null,"requires-python":null,"size":224008,"upload-time":"2014-09-30T18:10:37.283557Z","url":"https://files.pythonhosted.org/packages/fe/a3/7cf43f28bbb52c4d680378f99e900ced201ade5073ee3a7b30e7f09e3c66/psutil-2.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"449d07df8f8b9700cfae4ee67f0a73e4f96b55697428ae92cab29e33db4c3102"},"provenance":null,"requires-python":null,"size":319620,"upload-time":"2014-09-26T20:24:42.575161Z","url":"https://files.pythonhosted.org/packages/0e/66/bf4346c9ada08acee7e8f87d270fb4d4e6c86a477630adcfa3caa69aa5bb/psutil-2.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py3.3.exe","hashes":{"sha256":"5b444e5c1f3d3ee7a19a5720c62d6462fe81dd1d1bbb8aa955546ead509b3c4a"},"provenance":null,"requires-python":null,"size":318101,"upload-time":"2014-09-26T20:24:47.973672Z","url":"https://files.pythonhosted.org/packages/bd/a7/888e9fa42c8d475de7b467b9635bb18ed360cb1c261bcf59f5f932d624ea/psutil-2.1.3.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py3.4.exe","hashes":{"sha256":"3d25b4b20aabd360b7eda3dcbf4a14d2b256c2f61a8a569028e1c4b65b4d585a"},"provenance":null,"requires-python":null,"size":318089,"upload-time":"2014-09-26T20:24:55.536742Z","url":"https://files.pythonhosted.org/packages/b1/15/3f657d395213ad90efed423c88f1cb2f3b0a429d12bdea98a28c26de7ff4/psutil-2.1.3.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.4.exe","hashes":{"sha256":"36424b4e683d640291a1723f46a3949ebea37e43492f61027c168fb9dfe2055f"},"provenance":null,"requires-python":null,"size":155710,"upload-time":"2014-09-26T20:29:01.901009Z","url":"https://files.pythonhosted.org/packages/31/2e/d5448240fed09e88bbb59de194e1f2d3ba29f936a3231d718e5373736299/psutil-2.1.3.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.5.exe","hashes":{"sha256":"baa1803aaa4505fcf48bbd50d038d1f686d9e290defeecde41770d8fe876812b"},"provenance":null,"requires-python":null,"size":155699,"upload-time":"2014-09-26T20:24:13.810499Z","url":"https://files.pythonhosted.org/packages/2a/45/10d7b5057b3f941f803ca8ee690e651d2778e731f15ef3eee83c3f05f82f/psutil-2.1.3.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.6.exe","hashes":{"sha256":"55811a125d8a79bed0a65ed855fafc34f7d59cde162542a30cfa518da9e015bc"},"provenance":null,"requires-python":null,"size":289998,"upload-time":"2014-09-26T20:24:18.935804Z","url":"https://files.pythonhosted.org/packages/f3/4d/7f105269ece54ad7344c2a24b42ecb216b2746460f349c0ed1577b5ab8fa/psutil-2.1.3.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.7.exe","hashes":{"sha256":"6a1c8f1884f1983f8b74fda5dc89da07a69e3972fc022c3205f4964e1b01d235"},"provenance":null,"requires-python":null,"size":289802,"upload-time":"2014-09-26T20:24:24.112924Z","url":"https://files.pythonhosted.org/packages/93/64/1432ca27dfd7102a6726161b79b15a2997f461d3835867271a6c1e3353f7/psutil-2.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py3.3.exe","hashes":{"sha256":"8df72fcd30436e78bf8c5c6c796bb6815966511fa0bc8e3065e1aabbe4a2cf3d"},"provenance":null,"requires-python":null,"size":284794,"upload-time":"2014-09-26T20:24:30.110864Z","url":"https://files.pythonhosted.org/packages/0a/bb/d6cc7133624e3532e1d99f6cece35be7bd8d95ff7c82ca28cd502388c225/psutil-2.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py3.4.exe","hashes":{"sha256":"1f353ab0bbe0216e3a1636bfefb70ed366a2b1d95c95689f97e95d603626ef70"},"provenance":null,"requires-python":null,"size":284824,"upload-time":"2014-09-26T20:24:36.842776Z","url":"https://files.pythonhosted.org/packages/4a/66/9b1acf05dba9b9cb012ab3b8ed3d0fd7b01e1620808977fa6f1efe05dee2/psutil-2.1.3.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp26-none-win32.whl","hashes":{"sha256":"c92d2853ac52d5aaf8c2366435fb74892f7503daf2cf56b785f1ecadbf68e712"},"provenance":null,"requires-python":null,"size":82092,"upload-time":"2015-01-06T15:36:18.243917Z","url":"https://files.pythonhosted.org/packages/3c/65/f5bd2f8b54f14d950596c0c63d7b0f98b6dc779f7c61ac0dd2e9fc7e5e74/psutil-2.2.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp27-none-win32.whl","hashes":{"sha256":"6ad0e79b95f57a20f0cace08a063b0fc33fd83da1e5e501bc800bc69329a4501"},"provenance":null,"requires-python":null,"size":81891,"upload-time":"2015-01-06T15:36:28.466138Z","url":"https://files.pythonhosted.org/packages/6a/e6/96a4b4976f0eca169715d975b8c669b78b9afca58f9aadf261915694b73e/psutil-2.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"bbe719046986568ed3e84ce66ed617eebc46251ea8193566b3707abdf635afe6"},"provenance":null,"requires-python":null,"size":84072,"upload-time":"2015-01-06T15:36:58.206788Z","url":"https://files.pythonhosted.org/packages/28/28/4e4f94c9778ed16163f6c3dd696ce5e331121ac1600308830a3e98fa979a/psutil-2.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp33-none-win32.whl","hashes":{"sha256":"73013822a953fc4e3fb269256ec01b261752c590e23851e666201d1bfd32a3a9"},"provenance":null,"requires-python":null,"size":81885,"upload-time":"2015-01-06T15:36:38.776238Z","url":"https://files.pythonhosted.org/packages/66/8d/3143623c2f5bc264197727854040fdc02e3175b4ad991490586db7a512ed/psutil-2.2.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp33-none-win_amd64.whl","hashes":{"sha256":"d664a896feb10ec5bf4c43f4df8b6c7fceeb94677004cc9cf8c9f35b52c0e4fc"},"provenance":null,"requires-python":null,"size":84002,"upload-time":"2015-01-06T15:37:08.775842Z","url":"https://files.pythonhosted.org/packages/bd/08/112807380b8e7b76de8c84f920234a0ebed35e6511271f93f900f691a10c/psutil-2.2.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp34-none-win32.whl","hashes":{"sha256":"84299c41b251bef2a8b0812d651f4715209b3c4dfebe4a5df0f103bbdec78221"},"provenance":null,"requires-python":null,"size":81886,"upload-time":"2015-01-06T15:36:49.808417Z","url":"https://files.pythonhosted.org/packages/ef/0e/0cf2fea8f6e2e5ef84797eefc2c5ce561123e2417d7b931a6c54f9f8d413/psutil-2.2.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp34-none-win_amd64.whl","hashes":{"sha256":"3636879fcbde2b0b63db08abd0e3673c2cc72bb14075e46e15f98774b0c78236"},"provenance":null,"requires-python":null,"size":83954,"upload-time":"2015-01-06T15:37:18.558007Z","url":"https://files.pythonhosted.org/packages/82/ed/ce9c4281fd8a944a725c0f9a5d77f32295777ab22f54b4f706a51d59edd3/psutil-2.2.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.tar.gz","hashes":{"sha256":"b15cc9e7cad0991bd1cb806fa90ea85ba3a95d0f1226625ecef993294ad61521"},"provenance":null,"requires-python":null,"size":223676,"upload-time":"2015-01-06T15:38:45.979998Z","url":"https://files.pythonhosted.org/packages/ba/e4/760f64a8a5a5f1b95f3ce17c0d51134952f930caf1218e6ce21902f6c4ab/psutil-2.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"623baf2a213adc99cbc067e1db04b9e578eb4c38d9535c903e04b6d1ded10cab"},"provenance":null,"requires-python":null,"size":317990,"upload-time":"2015-01-06T15:34:57.187611Z","url":"https://files.pythonhosted.org/packages/30/a8/d1754e9e5492717d1c1afb30970f0677056052859f1af935b48c72c6cd68/psutil-2.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"fc6d348f603ae8465992a7edbc7c62cc16f0493bdd43aa609dcb7437992cac96"},"provenance":null,"requires-python":null,"size":316424,"upload-time":"2015-01-06T15:35:20.860940Z","url":"https://files.pythonhosted.org/packages/32/f1/e98caa1f6be7ba49dfafa0fbb63f50ffa191d316a1b2b3ec431d97ebf494/psutil-2.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"5e4508ca6822839f25310bfc14c050e60362dc29ddea6f5eac91de2c7423a471"},"provenance":null,"requires-python":null,"size":316403,"upload-time":"2015-01-06T15:35:42.094623Z","url":"https://files.pythonhosted.org/packages/dd/46/31505418bd950b09d28c6c21be76a4a51d593ec06b412539797080c0aa6b/psutil-2.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py2.6.exe","hashes":{"sha256":"1f93ccdf415da40f15a84ab6d9d32ddda61bb1b20079dae602356e087f408d28"},"provenance":null,"requires-python":null,"size":288364,"upload-time":"2015-01-06T15:33:25.830661Z","url":"https://files.pythonhosted.org/packages/9c/0d/f360da29c906dafd825edccc219e70dab73370ad2c287fa5baa9f7fa370b/psutil-2.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py2.7.exe","hashes":{"sha256":"83eb1739b7c87a21a65224d77217325e582199bedd0141b29dac81b4e4144c62"},"provenance":null,"requires-python":null,"size":288166,"upload-time":"2015-01-06T15:33:46.994076Z","url":"https://files.pythonhosted.org/packages/7a/30/5bb6644318f2279caf5d334f32df19fe4b2bce11ef418af32124e16f8e98/psutil-2.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py3.3.exe","hashes":{"sha256":"9db585f37d56381c37738e03aa8beb0b501b26adc7c70660ff182e0473f2cb0a"},"provenance":null,"requires-python":null,"size":283089,"upload-time":"2015-01-06T15:34:11.431221Z","url":"https://files.pythonhosted.org/packages/12/c4/cb75d51edb425ff77e2af5bb32e405ed41beb124ad062fb927bbe135c709/psutil-2.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py3.4.exe","hashes":{"sha256":"4a0d8a192d1523a3f02a4028bf4ac296f7da92c935464d302618bd639c03a2c6"},"provenance":null,"requires-python":null,"size":283110,"upload-time":"2015-01-06T15:34:33.472476Z","url":"https://files.pythonhosted.org/packages/17/fb/ae589efdfd076d1961e1f858c969d234a63e26f648a79b3fac0409e95c2f/psutil-2.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp26-none-win32.whl","hashes":{"sha256":"610f08fc0df646e4997b52f2bf2e40118a560738422d540883297c6f4e38be8c"},"provenance":null,"requires-python":null,"size":82104,"upload-time":"2015-02-02T13:08:03.993201Z","url":"https://files.pythonhosted.org/packages/0a/a7/b6323d30a8dde3f6a139809898e4bdcabcae3129314621aa24eb5ca4468e/psutil-2.2.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp27-none-win32.whl","hashes":{"sha256":"84ec9e77cab1d9ecf2d93bf43eba255a11b26d480966ee542bc846be1e272ea5"},"provenance":null,"requires-python":null,"size":81907,"upload-time":"2015-02-02T13:08:14.963426Z","url":"https://files.pythonhosted.org/packages/40/b6/a94c6d00ac18f779c72973639b78519de0c488e8e8e8d00b65cf4287bec3/psutil-2.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"81a6ab277886f233f230c46073fa6fc97ca2a95a44a14b80c6f6054acb7a644b"},"provenance":null,"requires-python":null,"size":84086,"upload-time":"2015-02-02T13:08:42.917519Z","url":"https://files.pythonhosted.org/packages/1c/ff/6b00405e4eeb3c40d4fc07142ef7bdc3a7a7aa2a1640b65e654bdb7682d1/psutil-2.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp33-none-win32.whl","hashes":{"sha256":"f5e0d247b09c9460b896ff89098015b7687a2934d4ed6165a5c3a662fad7ab6b"},"provenance":null,"requires-python":null,"size":81896,"upload-time":"2015-02-02T13:08:24.437138Z","url":"https://files.pythonhosted.org/packages/a2/fe/40b7d42057c5e68ff4c733b5689e9ae9d8785f13576326ba44371689eff7/psutil-2.2.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp33-none-win_amd64.whl","hashes":{"sha256":"07f31cbe5fc1183c8efb52bab9a5382ebd001a6e11b5a5df827f69456219d514"},"provenance":null,"requires-python":null,"size":84011,"upload-time":"2015-02-02T13:08:53.420801Z","url":"https://files.pythonhosted.org/packages/fe/8d/ffc94f092a12fc3cb837b9bda93abd88fb12219bcc6684b9bffea5e1f385/psutil-2.2.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp34-none-win32.whl","hashes":{"sha256":"a98b350251df2658c9be6bf0b6ebbab10fe88ccb513079c2c56fb53330530ee6"},"provenance":null,"requires-python":null,"size":81896,"upload-time":"2015-02-02T13:08:34.116542Z","url":"https://files.pythonhosted.org/packages/7f/3a/4bb24092c3de0a44f2fc1afb7bcaf898f1daba67cba5b9188b0ae6527102/psutil-2.2.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp34-none-win_amd64.whl","hashes":{"sha256":"f49d3ca9150f20269eae79d6efac98f0bbe4e5ceac46fe418d881560a04b6d8e"},"provenance":null,"requires-python":null,"size":83963,"upload-time":"2015-02-02T13:09:03.990685Z","url":"https://files.pythonhosted.org/packages/64/f7/c385e9350abbbb082364596fd4b0066fcb0c51ebc103a7ebf6eb9bc837e9/psutil-2.2.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.tar.gz","hashes":{"sha256":"a0e9b96f1946975064724e242ac159f3260db24ffa591c3da0a355361a3a337f"},"provenance":null,"requires-python":null,"size":223688,"upload-time":"2015-02-02T13:10:14.673714Z","url":"https://files.pythonhosted.org/packages/df/47/ee54ef14dd40f8ce831a7581001a5096494dc99fe71586260ca6b531fe86/psutil-2.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"552eaba2dbc9a49af2da64fc00cedf8847c7c9c2559cc620d9c8855583105764"},"provenance":null,"requires-python":null,"size":317995,"upload-time":"2015-02-02T13:06:42.422390Z","url":"https://files.pythonhosted.org/packages/97/27/e64014166df9efc3d00f987a251938d534d3897d62ef21486559a697a770/psutil-2.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"d25746746f2680d6174c9eb0e2793cb73304ca200d5ec7334000fdb575cd6577"},"provenance":null,"requires-python":null,"size":316429,"upload-time":"2015-02-02T13:07:07.458895Z","url":"https://files.pythonhosted.org/packages/c2/dd/5fdd5fd6102ae546452a5f32a6f75ec44c11d6b84c188dd33cfe1e2809e1/psutil-2.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"298ed760a365a846337750c5089adcec2a87014c61132ae39db671982750f35a"},"provenance":null,"requires-python":null,"size":316409,"upload-time":"2015-02-02T13:07:29.047577Z","url":"https://files.pythonhosted.org/packages/d9/59/0f14daf5797c61db2fec25bca49b49aa0a5d57dec57b4e58ae4652dadb95/psutil-2.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py2.6.exe","hashes":{"sha256":"6e49f72a1cd57918b275fdf7dd041618696fc39d5705ac722fdfccaf0b784a93"},"provenance":null,"requires-python":null,"size":288372,"upload-time":"2015-02-02T13:05:10.073479Z","url":"https://files.pythonhosted.org/packages/0c/a2/63967c61fbfc3f60c1b17e652a52d1afeb3c56403990c5a8f2fb801e2aa1/psutil-2.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py2.7.exe","hashes":{"sha256":"9227949290983df92bb3bb57c5b25605ebbc0e61a3f9baa394f752ed7abc914c"},"provenance":null,"requires-python":null,"size":288171,"upload-time":"2015-02-02T13:05:31.604222Z","url":"https://files.pythonhosted.org/packages/46/f3/d50f059d7d297db7335e340333124eac9441c5897bb29223c85ebaa64e7d/psutil-2.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py3.3.exe","hashes":{"sha256":"f8d94c8228011d1b658c39783596d95244e01950413bfc41fd44e78129ef7075"},"provenance":null,"requires-python":null,"size":283095,"upload-time":"2015-02-02T13:05:54.134208Z","url":"https://files.pythonhosted.org/packages/4a/d6/341767cdc6890adfb19ad60683e66dab3cdde605bc226427c8ec17dbd3f5/psutil-2.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py3.4.exe","hashes":{"sha256":"1e1df017bfa14c0edc87a124b11ca6e7fd8b6b5c9692eb06c1b72077bcca3845"},"provenance":null,"requires-python":null,"size":283115,"upload-time":"2015-02-02T13:06:12.827127Z","url":"https://files.pythonhosted.org/packages/8c/c1/24179e79ab74bde86ee766c4c0d2bb90b98e4e4d14017efeec024caba268/psutil-2.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"data-dist-info-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"filename":"psutil-3.0.0-cp27-none-win32.whl","hashes":{"sha256":"9641677de91769127c82aa032b70a8d9aa75369d0a9965aba880ac335b02eccb"},"provenance":null,"requires-python":null,"size":85709,"upload-time":"2015-06-13T19:16:33.857911Z","url":"https://files.pythonhosted.org/packages/bb/cf/f893fa2fb0888384b5d668e22b6e3652c9ce187e749604027a64a6bd6646/psutil-3.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"data-dist-info-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"filename":"psutil-3.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"11c6b17b37ab1ea04f6a1365bc4598637490819d6b744f3947fd3b8b425366b3"},"provenance":null,"requires-python":null,"size":88043,"upload-time":"2015-06-13T19:16:52.139563Z","url":"https://files.pythonhosted.org/packages/f1/26/a0904455b550f7fd5baa069ca10889825cc22440147813713998cd0676a0/psutil-3.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp33-none-win32.whl","hashes":{"sha256":"8333d2a496fe5d24171360ab1f78468db7a9691d61fd26bf15ca62336c699bac"},"provenance":null,"requires-python":null,"size":85737,"upload-time":"2015-06-13T19:16:39.871421Z","url":"https://files.pythonhosted.org/packages/ea/45/defb71d5009e178ef13e2474627a1070ae291b2dee64ae00e8f2b1a2aec2/psutil-3.0.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp33-none-win_amd64.whl","hashes":{"sha256":"44711a1a683369056d99fd08a05789d217205d72b6dbbe75c0b3743c5e1e3768"},"provenance":null,"requires-python":null,"size":87943,"upload-time":"2015-06-13T19:16:58.439865Z","url":"https://files.pythonhosted.org/packages/3c/0b/648fbaef2a037cc0c05388c0416ba3ca9244c22c4156dba73e4a900540a6/psutil-3.0.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp34-none-win32.whl","hashes":{"sha256":"9036f9c57a8c8a571f36c7d3b2c9a15a22dc9a95137fa60191e2eaa9527209d5"},"provenance":null,"requires-python":null,"size":85695,"upload-time":"2015-06-13T19:16:46.286646Z","url":"https://files.pythonhosted.org/packages/6d/ca/1d5dafae4d7396a825531296b6837b45129b63463b4eb221ac3429eb157d/psutil-3.0.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp34-none-win_amd64.whl","hashes":{"sha256":"18e53256390b299ff03bd58dc134d070bde4871d8c93baf08faea316e87544c8"},"provenance":null,"requires-python":null,"size":87893,"upload-time":"2015-06-13T19:17:05.116909Z","url":"https://files.pythonhosted.org/packages/59/82/51ab7cecaf03c8cbe7eff3b341f6564c3fa80baaa5a37643096cb526cae2/psutil-3.0.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.tar.gz","hashes":{"sha256":"a43cc84c6a2406e8f23785c68a52b792d95eb91e2b43be40f7c814bf80dc5979"},"provenance":null,"requires-python":null,"size":240872,"upload-time":"2015-06-13T19:14:46.063288Z","url":"https://files.pythonhosted.org/packages/ba/18/c5bb52abb67194aabadfd61286b538ee8856c7cb9c0e92dd64c1b132bf5e/psutil-3.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"26b2cf4456e039f490f79457100564ff34718f4257b68dee53b42d17493f0187"},"provenance":null,"requires-python":null,"size":323299,"upload-time":"2015-06-13T19:15:53.089846Z","url":"https://files.pythonhosted.org/packages/0c/b1/fa44dd6ed19e8be6f9f05f776e824ce202822f5d6237093c54d6c5102888/psutil-3.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"32808b29473e788487ee28a65caaab1c9fbd44c2d654ce6d563a89f20e106996"},"provenance":null,"requires-python":null,"size":321705,"upload-time":"2015-06-13T19:16:01.370127Z","url":"https://files.pythonhosted.org/packages/62/4f/89c6ac53e424109639f42033d0e87c4de4ddb610060007c6d5f6081a0fba/psutil-3.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"37a9869ef17680e768bf32d7556252a04f426e5b1e6fa51d5af16305c8a701b6"},"provenance":null,"requires-python":null,"size":321688,"upload-time":"2015-06-13T19:16:11.540515Z","url":"https://files.pythonhosted.org/packages/d2/67/9c988331d40c445ebc9d29d95534ce3908f3949f6a84d640a6c125bc56e2/psutil-3.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py2.6.exe","hashes":{"sha256":"f29ee1aafa452cce77667e75f5e0345b678b8c091b288d0118840df30f303bed"},"provenance":null,"requires-python":null,"size":293522,"upload-time":"2015-06-13T19:15:19.130222Z","url":"https://files.pythonhosted.org/packages/10/19/9e2f8a305f5a6c01feee59dfeb38a73dc1ea4f4af0133cef9a36708f4111/psutil-3.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py2.7.exe","hashes":{"sha256":"7cbcf866faa23e5229ecfe44ff562f406b4806caae66cb255dbb327247e540aa"},"provenance":null,"requires-python":null,"size":293319,"upload-time":"2015-06-13T19:15:27.353152Z","url":"https://files.pythonhosted.org/packages/fc/0f/92e595cd2f2a80cead241d45b4ce4961e2515deff101644f3812c75e9bc7/psutil-3.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py3.3.exe","hashes":{"sha256":"b968771d3db5eef0c17ef6bed2d7756b8ac3cce40e11d5493824863d195dd8e3"},"provenance":null,"requires-python":null,"size":288278,"upload-time":"2015-06-13T19:15:35.880360Z","url":"https://files.pythonhosted.org/packages/fc/29/387c555f9dc38c6bb084a8df8936e607848de9e2e984dbf7eb2a298c1ceb/psutil-3.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py3.4.exe","hashes":{"sha256":"81ac6f54a27d091c75333f7c90082d028db45a1e78de59d28ca9f38cf6186395"},"provenance":null,"requires-python":null,"size":288253,"upload-time":"2015-06-13T19:15:44.871254Z","url":"https://files.pythonhosted.org/packages/fb/29/25ac80b589c0e56214ac64fdd8216992be162a2b2290f9b88b9a5c517cfd/psutil-3.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"data-dist-info-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"filename":"psutil-3.0.1-cp27-none-win32.whl","hashes":{"sha256":"bcb8d23121848953ed295f7e3c0875b0164ee98d3245060beb3623c59ff2a1bc"},"provenance":null,"requires-python":null,"size":85793,"upload-time":"2015-06-18T02:36:50.974286Z","url":"https://files.pythonhosted.org/packages/17/05/b2c807b470464fbe3d357734b68199451574fcd75431fd3e5c77be24b6e0/psutil-3.0.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"data-dist-info-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"filename":"psutil-3.0.1-cp27-none-win_amd64.whl","hashes":{"sha256":"ba56ec5c052489b7a7015c26ed3f917f2df4ffa9799266e86be41815bc358b80"},"provenance":null,"requires-python":null,"size":88128,"upload-time":"2015-06-18T02:37:08.778852Z","url":"https://files.pythonhosted.org/packages/38/bf/0b743c8a07265f2ecb203f8e60310571dcae33036aa6ba7aa16e2641ac7a/psutil-3.0.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp33-none-win32.whl","hashes":{"sha256":"23606e9b42760a8fdeada33281d0c3ce88df220949746e0ddf54d5db7974b4c6"},"provenance":null,"requires-python":null,"size":85819,"upload-time":"2015-06-18T02:36:56.905878Z","url":"https://files.pythonhosted.org/packages/b2/9c/b2c4373b9406eaf33654c4be828d9316ee780f4b3c19d0fa56f55eb64d61/psutil-3.0.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp33-none-win_amd64.whl","hashes":{"sha256":"ae4a7f51f40154d02ab1576e94377171a28cc83fc89c077c152f16ba3dae72f3"},"provenance":null,"requires-python":null,"size":88028,"upload-time":"2015-06-18T02:37:14.406512Z","url":"https://files.pythonhosted.org/packages/7a/59/1a9a10238226dbaed24b8d978e9cc743ac143504b3f4ad8f5e3a169e263a/psutil-3.0.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp34-none-win32.whl","hashes":{"sha256":"b7520db52c7c4e38cdd50bb11be02a826372d8002bf92bbe955823cac32c7f9d"},"provenance":null,"requires-python":null,"size":85783,"upload-time":"2015-06-18T02:37:03.210308Z","url":"https://files.pythonhosted.org/packages/ed/85/1d4d97ce4c9a8c6dd479f5f717694016651170d3731cdecf740db0e4eae3/psutil-3.0.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp34-none-win_amd64.whl","hashes":{"sha256":"1754d4118eaab16f299a37dafaf7d34111e9a8e5ac2a799e2bd9b1a5d9d1122b"},"provenance":null,"requires-python":null,"size":87982,"upload-time":"2015-06-18T02:37:20.490529Z","url":"https://files.pythonhosted.org/packages/1e/e6/0af7e190f74d6f959dfb87f2b56f4711271729952691f843fb91c5e06712/psutil-3.0.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.tar.gz","hashes":{"sha256":"3f213b9ceed3c3068a973e04d7a8b2a29d1076abcb5ef45382517bfc6b808801"},"provenance":null,"requires-python":null,"size":241539,"upload-time":"2015-06-18T02:33:52.119548Z","url":"https://files.pythonhosted.org/packages/aa/5d/cbd3b7227fe7a4c2c77e4031b6c43961563a3ecde2981190e5afe959be51/psutil-3.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"b0a2ed567b31f71ae2e893768f0da0d51d51d12714471d4b7431e70ff5e36577"},"provenance":null,"requires-python":null,"size":323471,"upload-time":"2015-06-18T02:36:04.537383Z","url":"https://files.pythonhosted.org/packages/9d/a4/815181cacd33f0ce62a3c3aa188af65bd3945888aa2f6c16a925837ca517/psutil-3.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"99c593cb459b54209cdb4aed4a607fa8b2920fbd4f3c5a9219a0ede114975758"},"provenance":null,"requires-python":null,"size":321879,"upload-time":"2015-06-18T02:36:14.693044Z","url":"https://files.pythonhosted.org/packages/08/b2/278ac09b03db15b9e51c3ae7f678b3fbf050e895a2eccab66d05017bdef9/psutil-3.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"8f803bfe00a763254581bb6a3d788b3332492fbc67dbf46ad5068b663e44f309"},"provenance":null,"requires-python":null,"size":321861,"upload-time":"2015-06-18T02:36:23.687685Z","url":"https://files.pythonhosted.org/packages/d8/66/d21041db114938ae22d4994ea31f2d8f1353aa61e5f0d0c6c4e185b016a7/psutil-3.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py2.6.exe","hashes":{"sha256":"4a19475c1d6071c685b38f85837f6e6daa2c6ccd4d0132ab840123deb8ea2372"},"provenance":null,"requires-python":null,"size":293700,"upload-time":"2015-06-18T02:35:31.083110Z","url":"https://files.pythonhosted.org/packages/99/67/980b4a9257abaa3f53a60a3441f759301a0aca2922d60e18a05d23fb1b0e/psutil-3.0.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py2.7.exe","hashes":{"sha256":"363d3dbd610ce7bcf7f13b0a31133ee231d0990e99315142ed37f6ba2c1a84e6"},"provenance":null,"requires-python":null,"size":293492,"upload-time":"2015-06-18T02:35:39.030138Z","url":"https://files.pythonhosted.org/packages/5f/64/b994ed73ab49d5c847d97c47600b539603ebf0d6cfd8d3575b80db7aefb5/psutil-3.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py3.3.exe","hashes":{"sha256":"fc7cfe1d6919cb67f4144947acfcefc099d7d8299dd88bb4d863e62c44d041b4"},"provenance":null,"requires-python":null,"size":288451,"upload-time":"2015-06-18T02:35:47.145538Z","url":"https://files.pythonhosted.org/packages/1f/11/4871e823ff0d5a302a7f8ece60358d20ba7fc1620c4c9e2d94e826e1ff0e/psutil-3.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py3.4.exe","hashes":{"sha256":"73091a80ed295f990ab377dfd8cd4f7f00e3abfffb5e500192f1ea9fa58de158"},"provenance":null,"requires-python":null,"size":288427,"upload-time":"2015-06-18T02:35:55.768442Z","url":"https://files.pythonhosted.org/packages/08/af/c78ef8dc09ac61a479968b0c912677d0a0cb138c5ea7813b2a196fa32c53/psutil-3.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp26-none-win32.whl","hashes":{"sha256":"399916a016503c9ae99fd6aafbba4628b86ebdee52d67034e8c3f26e88a3504b"},"provenance":null,"requires-python":null,"size":87753,"upload-time":"2015-07-15T00:41:13.293562Z","url":"https://files.pythonhosted.org/packages/4e/0a/ab710541d02ff3ce4169a0922e4412952332991c4c39a0ea9df20e9279b0/psutil-3.1.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp27-none-win32.whl","hashes":{"sha256":"71af7a30eb6ed694a3624330334bd12da28bffdbe813f4b1080fadeb86f5970f"},"provenance":null,"requires-python":null,"size":87559,"upload-time":"2015-07-15T00:41:27.810047Z","url":"https://files.pythonhosted.org/packages/db/87/34b52811b755db94a7dd123e5c0f1d257885535f08f5f185c17810214d55/psutil-3.1.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp27-none-win_amd64.whl","hashes":{"sha256":"ada1cf324e6aba0affcb23c6fd959dae9f72de6ec135530788cbf17153d4fd3c"},"provenance":null,"requires-python":null,"size":90071,"upload-time":"2015-07-15T00:42:15.297263Z","url":"https://files.pythonhosted.org/packages/0b/7c/90869233a3e4056ddfdd1040d0e7722d3bb023c74b48bf10c09380c26eae/psutil-3.1.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp33-none-win32.whl","hashes":{"sha256":"0a313ebe14b9e277dfd151f4ad021012fb344dd51248e6de2aa1e7062d678541"},"provenance":null,"requires-python":null,"size":87569,"upload-time":"2015-07-15T00:41:43.765697Z","url":"https://files.pythonhosted.org/packages/f4/7c/56b718693e4c41b32af8bbe39160e8a3ea0ca12d3eece3dbbb8d4c046855/psutil-3.1.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp33-none-win_amd64.whl","hashes":{"sha256":"92a7f420bc97f899b5abab30392c23ba652304aec18415f2d1167da04dae9913"},"provenance":null,"requires-python":null,"size":89918,"upload-time":"2015-07-15T00:42:29.864336Z","url":"https://files.pythonhosted.org/packages/c7/5a/4046949e207b72b93540f3e19d699813fd35e290ccdf48080f332226b912/psutil-3.1.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp34-none-win32.whl","hashes":{"sha256":"8f5a0e859ae6dcc349914fb9ea0acc21cfd82a321d1c1b02d3d92c195f523ccd"},"provenance":null,"requires-python":null,"size":87592,"upload-time":"2015-07-15T00:42:00.678342Z","url":"https://files.pythonhosted.org/packages/09/34/09d53d29318a5fea88bd30d629595805064a0e3776e706eca2d79ceaebac/psutil-3.1.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp34-none-win_amd64.whl","hashes":{"sha256":"dcb4f208ec28fb72b35d1edf49aa51f2cc116b439aa40c4c415cbfe1fee54078"},"provenance":null,"requires-python":null,"size":89877,"upload-time":"2015-07-15T00:42:45.261251Z","url":"https://files.pythonhosted.org/packages/0b/f6/62592864eb064763989aa5830706034f9ad3c6ae2255fb7cae0c66b336a1/psutil-3.1.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.tar.gz","hashes":{"sha256":"4cdfeb2a328b6f8a2937f9b21f513c8aeda96dc076ecafda424f5c401dbad876"},"provenance":null,"requires-python":null,"size":246767,"upload-time":"2015-07-15T00:40:47.134419Z","url":"https://files.pythonhosted.org/packages/ce/d2/ab7f80718b4eafb2e474b8b410274d2c0d65341b963d730e653be9ed0ec8/psutil-3.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"6570fb3ddde83597e11c062e20ab86210ff84a1fa97e54bc8bda05e4cd34670a"},"provenance":null,"requires-python":null,"size":326179,"upload-time":"2015-07-15T00:42:09.118162Z","url":"https://files.pythonhosted.org/packages/26/f5/c76bf7ef62736913146b0482879705d1d877c9334092a0739f2b3bbae162/psutil-3.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"578a52f4b108857273d1e32de4d9bebf9b8f842f8c53ab3e242252cfc9bde295"},"provenance":null,"requires-python":null,"size":324540,"upload-time":"2015-07-15T00:42:24.223021Z","url":"https://files.pythonhosted.org/packages/00/51/2dc07e5618adb4a3676ab6c9c1759a3f268eda91808d58f45eb4dfd3d2c5/psutil-3.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"2aea29ca2a5ea318155fb856f24e6c7563c8741ccfeec862f0fd9af0f2d8ae87"},"provenance":null,"requires-python":null,"size":324490,"upload-time":"2015-07-15T00:42:38.312454Z","url":"https://files.pythonhosted.org/packages/da/76/dcefdcf88fd51becaff6c1ec0ba7566ca654a207fe6d69662723c645e3b0/psutil-3.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py2.6.exe","hashes":{"sha256":"271f1b6fcb4861d1b0fc7f612b2abaacb36c0f878fcc2908f1cf673337c3472c"},"provenance":null,"requires-python":null,"size":296215,"upload-time":"2015-07-15T00:41:08.168047Z","url":"https://files.pythonhosted.org/packages/3f/c2/f7ec0a70bc58c1918f814001682cca30f4b168f5b46dce913220e625dee6/psutil-3.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py2.7.exe","hashes":{"sha256":"1bde1cea6b7f9bd66202feee289c282b076c00419dc6404db357b05125f4d692"},"provenance":null,"requires-python":null,"size":296024,"upload-time":"2015-07-15T00:41:21.733811Z","url":"https://files.pythonhosted.org/packages/56/2f/c97adcba8f119a23d3580a3a95939c1a37d37d514b304d90912585a85521/psutil-3.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py3.3.exe","hashes":{"sha256":"667fa795ca5ccde216b769fc8572398598f1e0e2619f78605df1a4c75a475174"},"provenance":null,"requires-python":null,"size":290958,"upload-time":"2015-07-15T00:41:37.038168Z","url":"https://files.pythonhosted.org/packages/37/89/a949b02d66d600c45230a3a622f0d5f491182dbbc96fff9ffbcd869431bd/psutil-3.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py3.4.exe","hashes":{"sha256":"8e637fe2a23fad4f4ea8ae11402b593e09fc587b86a6ced40b6bc1017be8d978"},"provenance":null,"requires-python":null,"size":290975,"upload-time":"2015-07-15T00:41:53.144204Z","url":"https://files.pythonhosted.org/packages/53/3f/249ff2e418313f30a6dcc4995966725f72b00493848ffddeb72b506e8e50/psutil-3.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp26-none-win32.whl","hashes":{"sha256":"13a6377cc8d2859f846058170830127822877e05229c4a43aea893cdcb504d65"},"provenance":null,"requires-python":null,"size":87749,"upload-time":"2015-07-15T12:34:56.810625Z","url":"https://files.pythonhosted.org/packages/e1/9e/721afc99b6fe467b47fa2cad6899acc19b45dee32d30b498dc731b6c09ef/psutil-3.1.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp27-none-win32.whl","hashes":{"sha256":"5b7228cb69fdaea5aeb901704f5ecd21b7846aa60c2c8d408f22573fcbaa7e6f"},"provenance":null,"requires-python":null,"size":87554,"upload-time":"2015-07-15T12:35:15.647007Z","url":"https://files.pythonhosted.org/packages/07/60/c88366202816ba42b3d8e93e793c14d1ac5e71be30dd53c2d0117c106eec/psutil-3.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"da7650e2f3fcf06419d5ad75123e6c68b9bf5ff2a6c91d4c77aaed8e6f444fc4"},"provenance":null,"requires-python":null,"size":90065,"upload-time":"2015-07-15T12:36:14.571710Z","url":"https://files.pythonhosted.org/packages/fa/5b/8834e22cc22b6b0e9c2c68e240ab69754bed7c4c5388fb65abfa716f4a67/psutil-3.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp33-none-win32.whl","hashes":{"sha256":"f3d68eb44ba49e24a18d6f7934463478294a49152f97fea2eefe1e1e1ee957f3"},"provenance":null,"requires-python":null,"size":87562,"upload-time":"2015-07-15T12:35:34.924344Z","url":"https://files.pythonhosted.org/packages/cb/96/0eb8eb289681364d2cda2a22a7d1abeb0196b321ab95694335dd178a5b35/psutil-3.1.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp33-none-win_amd64.whl","hashes":{"sha256":"f9be0ae975b55a3b5d5a8b769560096d76184b60a56c6e88ff6b7ebecf1bc684"},"provenance":null,"requires-python":null,"size":89916,"upload-time":"2015-07-15T12:36:34.374903Z","url":"https://files.pythonhosted.org/packages/eb/13/a38bc1e0ac6f7c42dddd9c17a206877befb822ba3af9c3b0dab9e85911a6/psutil-3.1.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp34-none-win32.whl","hashes":{"sha256":"8f25aad572bde88d5ee0b3a11a75ff2ae3c8b0a334c4128d6f8eb4fc95172734"},"provenance":null,"requires-python":null,"size":87574,"upload-time":"2015-07-15T12:35:55.507642Z","url":"https://files.pythonhosted.org/packages/02/10/439ec497e3e38a8c493d0c67c56e23d106b85c8b9f616e8df9ec6ce1e606/psutil-3.1.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp34-none-win_amd64.whl","hashes":{"sha256":"4be182c273758dcdbd30827fdeecd889e27cb6a30238798e91bddeebc29cdc4f"},"provenance":null,"requires-python":null,"size":89865,"upload-time":"2015-07-15T12:36:55.016343Z","url":"https://files.pythonhosted.org/packages/ec/2e/8d98579399bc1979904455df182a063dd584b285ee8c141f3c94e7814c47/psutil-3.1.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.tar.gz","hashes":{"sha256":"d3290bd4a027fa0b3a2e2ee87728056fe49d4112640e2b8c2ea4dd94ba0cf057"},"provenance":null,"requires-python":null,"size":247284,"upload-time":"2015-07-15T12:33:47.020532Z","url":"https://files.pythonhosted.org/packages/8d/b3/954de176aa8e3a7782bae52ce938f24726c2c68d0f4c60d159271b6b293d/psutil-3.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"9c4fd3cc19bbc04eaa7ef3c61e3db26a41ac5e056f770977211d4569d0bf0086"},"provenance":null,"requires-python":null,"size":326261,"upload-time":"2015-07-15T12:36:07.064169Z","url":"https://files.pythonhosted.org/packages/e8/ac/7fb95ccc69ced76d0920e411b2fdfd3d38398c4bce53ec1dae92800df88a/psutil-3.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"e7cc26f661c9eaa9b32d0543dd7838daea72aad6e9f02fe73715ffd0dcb65170"},"provenance":null,"requires-python":null,"size":324622,"upload-time":"2015-07-15T12:36:27.201153Z","url":"https://files.pythonhosted.org/packages/2f/84/ae41f6bb61d4a93399c621218f99b761171a69a0c9163b9a72db1d53d62a/psutil-3.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"46cbfd86d6762e63c7df4ab0df889f6f2fffa9b5781ea3fc0431237f2a408382"},"provenance":null,"requires-python":null,"size":324571,"upload-time":"2015-07-15T12:36:47.165677Z","url":"https://files.pythonhosted.org/packages/88/d4/ca15a913ab43222e308774845317544e765718a9e56bd4efe5b3cedf1fbd/psutil-3.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py2.6.exe","hashes":{"sha256":"9efbd578d2f400dfe0ecab123b58d8af105854fdbb6222f841151e010e820b75"},"provenance":null,"requires-python":null,"size":296306,"upload-time":"2015-07-15T12:34:49.442980Z","url":"https://files.pythonhosted.org/packages/0c/f6/e81385c7ec989157eb68688a64a69c5a7477ff93d544893a9e1f251588b1/psutil-3.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py2.7.exe","hashes":{"sha256":"e0065e7cade4ac5ac70411674bc32326dee8d11c44469012a2b5164bf6dea97a"},"provenance":null,"requires-python":null,"size":296106,"upload-time":"2015-07-15T12:35:08.277861Z","url":"https://files.pythonhosted.org/packages/6b/fe/51596968f5a6a0970d9424021989f542d5aa715fe21d1a9c6bbbb0e377a9/psutil-3.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py3.3.exe","hashes":{"sha256":"c8ab17e07ea4907d2f9129254e82b6765ae08e61f0ce6dc8e2fc1faf145b166c"},"provenance":null,"requires-python":null,"size":291039,"upload-time":"2015-07-15T12:35:27.250487Z","url":"https://files.pythonhosted.org/packages/83/49/ff116fb9981ef04a5aed1c091ace117c214ed752d37be267ea4e2f28efad/psutil-3.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py3.4.exe","hashes":{"sha256":"3003d8be6e86eb6beb990863a88950f9b9fe53ccaae92edcd8efcd152d7451ea"},"provenance":null,"requires-python":null,"size":291056,"upload-time":"2015-07-15T12:35:47.023807Z","url":"https://files.pythonhosted.org/packages/7b/cd/accf3d7e37006bffe7a569e4fc587eb686d275a19a4e8a37a12930a1e2db/psutil-3.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"data-dist-info-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"filename":"psutil-3.2.0-cp27-none-win32.whl","hashes":{"sha256":"1493041336a591f22c77bcb815a399faf9bdac32f79f4de354eda3507a0d6d6b"},"provenance":null,"requires-python":null,"size":88077,"upload-time":"2015-09-02T11:56:33.437905Z","url":"https://files.pythonhosted.org/packages/5f/fc/5f317fd548909b1bbb111d462c072faf8af3938268f3e7dd3ab2f9181461/psutil-3.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"data-dist-info-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"filename":"psutil-3.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"3744ee760dff697f45731a71e7902514aa043c99800cc8fabeb6bebc9dad973d"},"provenance":null,"requires-python":null,"size":90526,"upload-time":"2015-09-02T11:57:23.745435Z","url":"https://files.pythonhosted.org/packages/84/6c/7efbe64b42748125e7113a90e48c0da9859b7f0363ac85ca5617decbafee/psutil-3.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp33-none-win32.whl","hashes":{"sha256":"8836f77d2c4ae2935431ca66e445435b87b53b4db637fcceb438b78843239210"},"provenance":null,"requires-python":null,"size":88080,"upload-time":"2015-09-02T11:56:48.988091Z","url":"https://files.pythonhosted.org/packages/64/8e/0a06028a1ac093402885febf2aeb18093f1d28ae2110c7eb10b43e7554c1/psutil-3.2.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp33-none-win_amd64.whl","hashes":{"sha256":"b16eb62d9c21efaa2c9ac8a9f8b23bb7a695cb799b597edf4b1289ce8e6973ac"},"provenance":null,"requires-python":null,"size":90372,"upload-time":"2015-09-02T12:03:01.254324Z","url":"https://files.pythonhosted.org/packages/cd/29/a8383040200a3ebe0e985f54f35691cc078a1deb632abb5340d3deb5b7b7/psutil-3.2.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp34-none-win32.whl","hashes":{"sha256":"0ac1d68ab3c5a65641cbbb23d19deda466f73226f9d967f91436851995281777"},"provenance":null,"requires-python":null,"size":88096,"upload-time":"2015-09-02T11:57:08.035697Z","url":"https://files.pythonhosted.org/packages/07/c3/76a50982a82c0e9d93d9614a0cd06644c1d3406c9bb80a43f95abdd4ab97/psutil-3.2.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp34-none-win_amd64.whl","hashes":{"sha256":"0b26ef262fe2d10185ab562cd0530af7f6d9a6744c631c44e64be94796f4ba2d"},"provenance":null,"requires-python":null,"size":90354,"upload-time":"2015-09-02T12:07:12.010031Z","url":"https://files.pythonhosted.org/packages/58/2d/8b7abb9b6f8956d9a6dfc3b1dffce27efab8c7c0497a6366e7fee444ae53/psutil-3.2.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.tar.gz","hashes":{"sha256":"06f9d255f8b12a6a04aa2b468ec453c539f54a464d110b3458c32b0152a5c943"},"provenance":null,"requires-python":null,"size":251988,"upload-time":"2015-09-02T11:59:23.849318Z","url":"https://files.pythonhosted.org/packages/1d/3a/d396274e6f086e342dd43401d4012973af98c00b3aabdb5cc4a432df660e/psutil-3.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"71fd8712715f8e6acc5bee5719a83a61a396067cf2bfb15b4d8f1f2955648637"},"provenance":null,"requires-python":null,"size":326895,"upload-time":"2015-09-02T11:57:17.538160Z","url":"https://files.pythonhosted.org/packages/e1/49/990073ab7e010965a7d0df5e48181d07c212edd7fafb890feda664ea9b3c/psutil-3.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py2.7.exe","hashes":{"sha256":"2a8b5878d4e787d81a1eeddcc09ff28d501a3ceb320c7fffa7e207da5d61d01c"},"provenance":null,"requires-python":null,"size":296802,"upload-time":"2015-09-02T11:56:26.879142Z","url":"https://files.pythonhosted.org/packages/0b/c5/d6ad511c3c17afa9837d08fc26d76e85dc83ebc304c6c7bec3970e74f240/psutil-3.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py3.3.exe","hashes":{"sha256":"adfc63ceede4e8f6bf21e4bdf6fc91f70f9612ec2b1bf9ad306828909bb71c52"},"provenance":null,"requires-python":null,"size":291738,"upload-time":"2015-09-02T11:56:41.826609Z","url":"https://files.pythonhosted.org/packages/e3/35/81842c6c4366d19c87d1a1fb4ad4e4d22a18aa9facaea8b6f12ccd4c1212/psutil-3.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py3.4.exe","hashes":{"sha256":"4098e0ed7930003ef15feb852e64f73180c17a651c4170fb5573f8c44622d068"},"provenance":null,"requires-python":null,"size":291748,"upload-time":"2015-09-02T11:56:58.479934Z","url":"https://files.pythonhosted.org/packages/e9/e8/5b432a0490328cfff86605b574d2aa31b1fac4e61587dff5a7f76d4cb95e/psutil-3.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp26-none-win32.whl","hashes":{"sha256":"a77230ecd6f42d0b549f8eb6aa105f14e4bc5908c754d6e10ff979c900934481"},"provenance":null,"requires-python":null,"size":88306,"upload-time":"2015-09-03T15:37:28.878155Z","url":"https://files.pythonhosted.org/packages/37/46/f348f7728dea66436abdfc9fa14ef017e0148c6bca08a822ee4dd7cb6d75/psutil-3.2.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp27-none-win32.whl","hashes":{"sha256":"9453b8ceb249d4d9ddc69153729761be340dfef9c99509390e4fb0f1fcbb3853"},"provenance":null,"requires-python":null,"size":88111,"upload-time":"2015-09-03T15:30:11.278302Z","url":"https://files.pythonhosted.org/packages/ba/27/f55ca7d15af50e731e9bbbff9b22fc31a40b786c02f85d173568e5084152/psutil-3.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"014714beed46a66370834cebe0bbb53799bddc164f7f0149a4a70e2051f7bc1a"},"provenance":null,"requires-python":null,"size":90561,"upload-time":"2015-09-03T15:35:05.710499Z","url":"https://files.pythonhosted.org/packages/6b/ac/5da840018ce300a258925d4535a55a32b75236d5d777a8de6c3de18e71f3/psutil-3.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp33-none-win32.whl","hashes":{"sha256":"5c0daf045fd7d7f105863a5f9508d1698559ebbdfd70d7d8b6fe6fedde575735"},"provenance":null,"requires-python":null,"size":88117,"upload-time":"2015-09-03T15:32:19.746087Z","url":"https://files.pythonhosted.org/packages/c7/96/3ae14e4bf81f18f404bb5285fcda28e50ae6df87e91ad6bf45a9a4f51ac3/psutil-3.2.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp33-none-win_amd64.whl","hashes":{"sha256":"452622592564cd67f86808c8176720c1443d43e248cfd242d71cff559ed1424c"},"provenance":null,"requires-python":null,"size":90410,"upload-time":"2015-09-03T15:36:13.351055Z","url":"https://files.pythonhosted.org/packages/76/62/fe6f705cb331be5fcc97b268987527dcdb3f3aa104bf830b0ec8bf1e2ad4/psutil-3.2.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp34-none-win32.whl","hashes":{"sha256":"96379bee09d4c6b4d57d72cb7347dbc51b6847977f2fad01cdfefef3b53e44e3"},"provenance":null,"requires-python":null,"size":88131,"upload-time":"2015-09-03T15:33:36.571683Z","url":"https://files.pythonhosted.org/packages/31/ec/1a54f23c767e27dac09d2372f3522f88ef34f3a0ddd44c16122970259f6f/psutil-3.2.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp34-none-win_amd64.whl","hashes":{"sha256":"7a7f4f2ed6d2835c48c24a81b251ba4f9b21f6bba2323291f8205c9ecb6f659d"},"provenance":null,"requires-python":null,"size":90387,"upload-time":"2015-09-03T15:36:29.262691Z","url":"https://files.pythonhosted.org/packages/2e/10/f1590ae942a6b8dd2bdeef6088e30e89b30161a264881b14134f3c4a3a0e/psutil-3.2.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.tar.gz","hashes":{"sha256":"7f6bea8bfe2e5cfffd0f411aa316e837daadced1893b44254bb9a38a654340f7"},"provenance":null,"requires-python":null,"size":251653,"upload-time":"2015-09-03T15:30:34.118573Z","url":"https://files.pythonhosted.org/packages/cd/5f/4fae1036903c01929c48ded6800a8705106ee20f9e39e3f2ad5d1824e210/psutil-3.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"e9a8a44f3847a0e20a54d321ed62de4e9cee5bc4e880e25fe88ae20cfa4e32b2"},"provenance":null,"requires-python":null,"size":327122,"upload-time":"2015-09-03T15:34:58.714500Z","url":"https://files.pythonhosted.org/packages/c5/af/4ab069ba93a037a4acf9bb84248daa44204a46687abc6a9f3a82ad8c5ee2/psutil-3.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"5a62c38852de4513f1816b9c431a94f02531619c1edc60a2cc163c8754f51c50"},"provenance":null,"requires-python":null,"size":325481,"upload-time":"2015-09-03T15:36:01.424492Z","url":"https://files.pythonhosted.org/packages/32/87/82e449ff9573dde3c78685b64ac3d17b5d19db11e976eab27c4dc5dca942/psutil-3.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"da898c0708b99b3892bfb7d5caebb447d14d03c7a655c55e484eb5fcc741c3ca"},"provenance":null,"requires-python":null,"size":325465,"upload-time":"2015-09-03T15:36:22.916532Z","url":"https://files.pythonhosted.org/packages/4b/11/f18b29033a0b383e67f664576eca59fbe8552a9fd97f9f22d6d0ff1c4951/psutil-3.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py2.6.exe","hashes":{"sha256":"0e5fe3d50f9f8d9a5216cfa23f56890aa0c6a6163869434001f4f2ba463dace5"},"provenance":null,"requires-python":null,"size":297224,"upload-time":"2015-09-03T15:37:22.791161Z","url":"https://files.pythonhosted.org/packages/c0/bb/ed28c191c4d9f27b60d9ea6bd7774b44d78778f0b1fb507ee1e789a490d7/psutil-3.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py2.7.exe","hashes":{"sha256":"9fb6f11bdd3fdbe1e611ae02b3ad3dff8f70ef6eaa694d13e8ad0906fd7a7261"},"provenance":null,"requires-python":null,"size":297025,"upload-time":"2015-09-03T15:30:14.327211Z","url":"https://files.pythonhosted.org/packages/9a/c1/075598067efefe25f6a2b0cf1b3eb896322597ee64ba097cc6611b89ada7/psutil-3.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py3.3.exe","hashes":{"sha256":"605cc7dbe2170e89f2f6709cf1577c8a02f89951fe4a0eb48e72530f605141ca"},"provenance":null,"requires-python":null,"size":291964,"upload-time":"2015-09-03T15:32:12.890526Z","url":"https://files.pythonhosted.org/packages/e0/ce/ff1db37cbdf6b3071e89afe4fced6f73d2cdf9c3a87696e7754d207e0ec5/psutil-3.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py3.4.exe","hashes":{"sha256":"0cd127239f527eae6c0f778dd41bb3ced84e6049b919713022c9b72e5f22a1c1"},"provenance":null,"requires-python":null,"size":291976,"upload-time":"2015-09-03T15:33:29.509014Z","url":"https://files.pythonhosted.org/packages/5d/a0/d624d4f5660a476821fe0d920a4c2e995a23151928b15cc3383379228f15/psutil-3.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp26-none-win32.whl","hashes":{"sha256":"5a8ce70327c0da578a31ebbf0042671ed9be6f4b6b022c02f03302b690074966"},"provenance":null,"requires-python":null,"size":88377,"upload-time":"2015-10-04T16:38:11.705751Z","url":"https://files.pythonhosted.org/packages/4d/af/5b8c2471ea942a4b6ee85706e9279284ae9dc86ee30b6f97db2d84a95433/psutil-3.2.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp27-none-win32.whl","hashes":{"sha256":"6c5809582d3d165511d2319401bd0f6c0e825d7853e49da59027c1fb8aa8f897"},"provenance":null,"requires-python":null,"size":88183,"upload-time":"2015-10-04T16:38:39.878475Z","url":"https://files.pythonhosted.org/packages/63/1e/a510f3f310b5f530336fbc708fb1456bf3e49e3b3d85c31d151b6e389c4f/psutil-3.2.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp27-none-win_amd64.whl","hashes":{"sha256":"37f1cc8fc7586cc930ea3737533d6d79c1f761d577fd1bb1bb5798ccd1543b53"},"provenance":null,"requires-python":null,"size":90637,"upload-time":"2015-10-04T16:40:05.379646Z","url":"https://files.pythonhosted.org/packages/e7/b7/f04d64a692159733ed383b4638abd9d3dc4538d4aacb5e193af02a3840a2/psutil-3.2.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp33-none-win32.whl","hashes":{"sha256":"f8fb145f8fa9e223696ff2f99924ea42538f3ad6b9738707292d840acbde528f"},"provenance":null,"requires-python":null,"size":88189,"upload-time":"2015-10-04T16:39:00.887284Z","url":"https://files.pythonhosted.org/packages/77/f3/6b3742040b634692393faf3a81e6c0e40366c22bc338ad3fc62ed21b157a/psutil-3.2.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp33-none-win_amd64.whl","hashes":{"sha256":"65c78ba625cf9761d5966603838cc959f396bd03536c480db69f8cf37bdf9994"},"provenance":null,"requires-python":null,"size":90471,"upload-time":"2015-10-04T16:40:27.009383Z","url":"https://files.pythonhosted.org/packages/59/33/3ccdbec4ef1452758ba80f711af46736717f63d73786744d6251afb68624/psutil-3.2.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp34-none-win32.whl","hashes":{"sha256":"3df8d3e32e2b4f7c2ea91014294844670eddb125ba76c24152c0a155a1f73b5b"},"provenance":null,"requires-python":null,"size":88202,"upload-time":"2015-10-04T16:39:39.989654Z","url":"https://files.pythonhosted.org/packages/62/6e/ee3597f32c650f744359e57fd18bcede773dd7465d392dabbb008bc79b48/psutil-3.2.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp34-none-win_amd64.whl","hashes":{"sha256":"e321d3f029268bc8442a7ff214da43fe91041924898f5e23d88bfda7ecb81acc"},"provenance":null,"requires-python":null,"size":90463,"upload-time":"2015-10-04T16:40:56.120875Z","url":"https://files.pythonhosted.org/packages/ed/fe/f31bb708dfdecfbc59b946ecb9ee3379fe7a8183c37ea6c43d6f4da5117d/psutil-3.2.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"data-dist-info-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"filename":"psutil-3.2.2-cp35-none-win32.whl","hashes":{"sha256":"76c68c9005a2aa983fce440ef98b66e6f200f740f52064a90fdcc30d11771bc2"},"provenance":null,"requires-python":null,"size":90363,"upload-time":"2015-11-06T10:48:59.454097Z","url":"https://files.pythonhosted.org/packages/cb/79/fcedcf009ab9f8c605f2f345b1797b72134ecc6c9c9f786575e34b3471bc/psutil-3.2.2-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"data-dist-info-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"filename":"psutil-3.2.2-cp35-none-win_amd64.whl","hashes":{"sha256":"d3ac8ad04a509819d7b5d58e453749a3ceb37253267dac6e8856ea7953e22ca0"},"provenance":null,"requires-python":null,"size":93288,"upload-time":"2015-11-06T10:50:14.699381Z","url":"https://files.pythonhosted.org/packages/03/c5/15e44d590afc788228e93cdacf55f98828f326de985242bbf03b3545b129/psutil-3.2.2-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.tar.gz","hashes":{"sha256":"f9d848e5bd475ffe7fa3ab1c20d249807e648568af64bb0058412296ec990a0c"},"provenance":null,"requires-python":null,"size":253502,"upload-time":"2015-10-04T16:39:43.138939Z","url":"https://files.pythonhosted.org/packages/dc/b2/ab65a2209b996c891209b8a7444a0c825125fba850efaec07b95bccb3ff5/psutil-3.2.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py2.7.exe","hashes":{"sha256":"1fca15005063b401cbf94cebe3c01ef6ba3d86ba563730d5d5d6be962a637cf4"},"provenance":null,"requires-python":null,"size":327220,"upload-time":"2015-10-04T16:39:56.198669Z","url":"https://files.pythonhosted.org/packages/01/7c/47b7ac498c9dd6ac9f9b4489d3bd9cea8158e774592661a7c956a815dc78/psutil-3.2.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py3.3.exe","hashes":{"sha256":"01a1f55819019ad13c288c41cb233e17a6ee648baf19591a70f6c2c2295dde6c"},"provenance":null,"requires-python":null,"size":325577,"upload-time":"2015-10-04T16:40:17.960471Z","url":"https://files.pythonhosted.org/packages/5d/89/b3bfca24d038b16af09055912e3a5bc35347ebeb0e6af322959b68dbf237/psutil-3.2.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py3.5.exe","hashes":{"sha256":"9afc68c02717fb4416f91b3c2da4c407756f683804383d1499cdc9e1512e7942"},"provenance":null,"requires-python":null,"size":242452,"upload-time":"2015-11-06T10:49:50.288504Z","url":"https://files.pythonhosted.org/packages/1d/0b/9582cadcba005f4eb0207107baeae7eb2382431fea5e4bc0ea7ce683430f/psutil-3.2.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py2.6.exe","hashes":{"sha256":"c7c516a83d072d1a375ed3a0b5a1b1b9307ad839011e8d30aa16b0f932f6c481"},"provenance":null,"requires-python":null,"size":297317,"upload-time":"2015-10-04T16:38:04.476936Z","url":"https://files.pythonhosted.org/packages/66/e4/bb85391bb46b607be0578e0a091bc064daeb2d1c2e80aa2dab89260dff00/psutil-3.2.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.3.exe","hashes":{"sha256":"5168f99f065f6116ad1e8529bd5dd5309815198c6250c9180ff6058c6a3641d9"},"provenance":null,"requires-python":null,"size":292060,"upload-time":"2015-10-04T16:38:46.104177Z","url":"https://files.pythonhosted.org/packages/8e/86/8f1e1c0ffc0530dca5e71777f04fb90833c92d3ffc1d075b8b546874eae5/psutil-3.2.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.4.exe","hashes":{"sha256":"d5c96aa591ba711dfdadb1ab7a0adf08c5e637644a21437f3a39a9e427aa969b"},"provenance":null,"requires-python":null,"size":292515,"upload-time":"2015-11-06T01:39:51.263901Z","url":"https://files.pythonhosted.org/packages/28/fc/08f1098976de5416cd15967d88e03297569e9a34ca875e7dee38ff9150f0/psutil-3.2.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.5.exe","hashes":{"sha256":"f0e88f94f9822fd34bcd927aba4bb606bcb0ad0dda1543c9333e8175d5c05822"},"provenance":null,"requires-python":null,"size":232367,"upload-time":"2015-11-06T10:48:42.065794Z","url":"https://files.pythonhosted.org/packages/1d/b8/b725f9bd884f75ce141a9e871a79b2bdd1b5f31e814fc4e396d9ff7c98a2/psutil-3.2.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp26-none-win32.whl","hashes":{"sha256":"584f0b29fcc5d523b433cb8918b2fc74d67e30ee0b44a95baf031528f424619f"},"provenance":null,"requires-python":null,"size":90099,"upload-time":"2015-11-25T18:49:50.211423Z","url":"https://files.pythonhosted.org/packages/91/75/c20c3b9f4d3feb3436d607f498744e46dd28b265b8a72509812322198c7c/psutil-3.3.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp26-none-win_amd64.whl","hashes":{"sha256":"28ca0b6e9d99aa8dc286e8747a4471362b69812a25291de29b6a8d70a1545a0d"},"provenance":null,"requires-python":null,"size":92645,"upload-time":"2015-11-25T18:50:32.375134Z","url":"https://files.pythonhosted.org/packages/6a/d1/0ce316e4346bcae9dd23911366d894eda65875b88ff447ec8f0402ce556b/psutil-3.3.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp27-none-win32.whl","hashes":{"sha256":"167ad5fff52a672c4ddc1c1a0b25146d6813ebb08a9aab0a3ac45f8a5b669c3b"},"provenance":null,"requires-python":null,"size":90131,"upload-time":"2015-11-25T01:20:43.015358Z","url":"https://files.pythonhosted.org/packages/91/73/1f55b4a19db535759fec5fdbdd0653d7192336557078e3ac9085d7d77cd1/psutil-3.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"e6dea6173a988727bb223d3497349ad5cdef5c0b282eff2d83e5f9065c53f85f"},"provenance":null,"requires-python":null,"size":92586,"upload-time":"2015-11-25T01:22:19.444688Z","url":"https://files.pythonhosted.org/packages/a2/ab/d15a34c6b9090d58601541f8f5564f5b48d01e82f56e07593be969d529e7/psutil-3.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp33-none-win32.whl","hashes":{"sha256":"2af5e0a4aad66049955d0734aa4e3dc8caa17a9eaf8b4c1a27a5f1ee6e40f6fc"},"provenance":null,"requires-python":null,"size":90141,"upload-time":"2015-11-25T01:21:10.374023Z","url":"https://files.pythonhosted.org/packages/c5/1f/5038a2567f5853ea1e0fb55f795c30b339a318717573c5b0c85b8814d733/psutil-3.3.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp33-none-win_amd64.whl","hashes":{"sha256":"d9884dc0dc2e55e2448e495778dc9899c1c8bf37aeb2f434c1bea74af93c2683"},"provenance":null,"requires-python":null,"size":92432,"upload-time":"2015-11-25T01:22:39.035467Z","url":"https://files.pythonhosted.org/packages/b1/9c/a9cd75c8cfbac44397a2ca76430229c5496b21e0ab93cba5987d80e3f262/psutil-3.3.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp34-none-win32.whl","hashes":{"sha256":"e27c2fe6dfcc8738be3d2c5a022f785eb72971057e1a9e1e34fba73bce8a71a6"},"provenance":null,"requires-python":null,"size":90151,"upload-time":"2015-11-25T01:21:33.616930Z","url":"https://files.pythonhosted.org/packages/23/57/6a7c3ab4d04d055cada3b5511c40e0e699d8dd5d8217cae6fb68ae61dff6/psutil-3.3.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp34-none-win_amd64.whl","hashes":{"sha256":"65afd6fecc8f3aed09ee4be63583bc8eb472f06ceaa4fe24c4d1d5a1a3c0e13f"},"provenance":null,"requires-python":null,"size":92416,"upload-time":"2015-11-25T01:23:04.575877Z","url":"https://files.pythonhosted.org/packages/5d/a8/e62ec8105350c1e615ac84b084c7c8799d09e0d1b4530d3e68291dca8976/psutil-3.3.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp35-none-win32.whl","hashes":{"sha256":"ba1c558fbfcdf94515c2394b1155c1dc56e2bc2a9c17d30349827c9ed8a67e46"},"provenance":null,"requires-python":null,"size":91965,"upload-time":"2015-11-25T01:21:52.452732Z","url":"https://files.pythonhosted.org/packages/6e/6d/cf51e672eef1f1fbf9efce429d5411d4a2f3aa239e079b82531389562cd2/psutil-3.3.0-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp35-none-win_amd64.whl","hashes":{"sha256":"ba95ea0022dcb64d36f0c1335c0605fae35bdf3e0fea8d92f5d0f6456a35e55b"},"provenance":null,"requires-python":null,"size":94891,"upload-time":"2015-11-25T01:23:25.107121Z","url":"https://files.pythonhosted.org/packages/90/49/3726db12f0fa7ff8f7e5493cc128ee6b40f5720f7397a4ef01db9e28dd7b/psutil-3.3.0-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.tar.gz","hashes":{"sha256":"421b6591d16b509aaa8d8c15821d66bb94cb4a8dc4385cad5c51b85d4a096d85"},"provenance":null,"requires-python":null,"size":261983,"upload-time":"2015-11-25T01:20:55.681846Z","url":"https://files.pythonhosted.org/packages/fe/69/c0d8e9b9f8a58cbf71aa4cf7f27c27ee0ab05abe32d9157ec22e223edef4/psutil-3.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py2.6.exe","hashes":{"sha256":"326b305cbdb6f94dafbfe2c26b11da88b0ab07b8a07f8188ab9d75ff0c6e841a"},"provenance":null,"requires-python":null,"size":329214,"upload-time":"2015-11-25T18:50:19.881553Z","url":"https://files.pythonhosted.org/packages/a9/c1/7642d44312cffaa1b3efc6ac5252b7f1ab1c528903b55d56d7bd46805d92/psutil-3.3.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"9aede5b2b6fe46b3748ea8e5214443890d1634027bef3d33b7dad16556830278"},"provenance":null,"requires-python":null,"size":329153,"upload-time":"2015-11-25T01:22:03.853435Z","url":"https://files.pythonhosted.org/packages/bb/e0/f8e4e286bf9c075f0e9fb3c0b17cecef04cda5e91f4c54982b91b3baf338/psutil-3.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"73bed1db894d1aa9c3c7e611d302cdeab7ae8a0dc0eeaf76727878db1ac5cd87"},"provenance":null,"requires-python":null,"size":327512,"upload-time":"2015-11-25T01:22:31.545622Z","url":"https://files.pythonhosted.org/packages/fb/20/9438b78a3155b1eb480a4ea09dab6370f06e0a003cf43c3975743e0c9e8d/psutil-3.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"935b5dd6d558af512f42501a7c08f41d7aff139af1bb3959daa3abb859234d6c"},"provenance":null,"requires-python":null,"size":327497,"upload-time":"2015-11-25T01:22:51.620885Z","url":"https://files.pythonhosted.org/packages/ae/71/c68af8e9b05144de969da58e1bf5ebfe0859b1c83b827e05ae3116178bb1/psutil-3.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"4ca0111cf157dcc0f2f69a323c5b5478718d68d45fc9435d84be0ec0f186215b"},"provenance":null,"requires-python":null,"size":243943,"upload-time":"2015-11-25T01:23:16.182734Z","url":"https://files.pythonhosted.org/packages/bd/14/a67db75c827761bf55a50c6ce455cdf0fd7e75d1c7c395b7283359676288/psutil-3.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py2.6.exe","hashes":{"sha256":"b6f13c95398a3fcf0226c4dcfa448560ba5865259cd96ec2810658651e932189"},"provenance":null,"requires-python":null,"size":299025,"upload-time":"2015-11-25T18:49:29.675333Z","url":"https://files.pythonhosted.org/packages/ad/ea/d7c41ad9fab6e89263225c66971f9807a0396925dddf7c20901b637b99e2/psutil-3.3.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py2.7.exe","hashes":{"sha256":"ee6be30d1635bbdea4c4325d507dc8a0dbbde7e1c198bd62ddb9f43198b9e214"},"provenance":null,"requires-python":null,"size":299056,"upload-time":"2015-11-25T01:20:34.764583Z","url":"https://files.pythonhosted.org/packages/15/f7/a34370848c11d7d7933c0c107763ee470b54a7e48aa90a301919a3ad6757/psutil-3.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.3.exe","hashes":{"sha256":"dfa786858c268d7fbbe1b6175e001ec02738d7cfae0a7ce77bf9b651af676729"},"provenance":null,"requires-python":null,"size":293994,"upload-time":"2015-11-25T01:20:57.318743Z","url":"https://files.pythonhosted.org/packages/71/1e/af5675d52b426857441c29ad88d4fccfd55d300867ad02531d77991ab661/psutil-3.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.4.exe","hashes":{"sha256":"aa77f9de72af9c16cc288cd4a24cf58824388f57d7a81e400c4616457629870e"},"provenance":null,"requires-python":null,"size":294006,"upload-time":"2015-11-25T01:21:23.806352Z","url":"https://files.pythonhosted.org/packages/6d/41/cf5b54535ea052a32a76a8e8e56af817deb95f4ffde49277a52ded29763b/psutil-3.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.5.exe","hashes":{"sha256":"f500093357d04da8140d87932cac2e54ef592a54ca8a743abb2850f60c2c22eb"},"provenance":null,"requires-python":null,"size":233858,"upload-time":"2015-11-25T01:21:44.041656Z","url":"https://files.pythonhosted.org/packages/1e/1d/151535e51338efebe453a28d2f14d4d5b1e1f3ce54ccc63866c96dc7e1bd/psutil-3.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp26-none-win32.whl","hashes":{"sha256":"0b1382db1cf76d53fb1d6e5619b5f3c86126e11a933b200c21ed4fa7fe5037aa"},"provenance":null,"requires-python":null,"size":91763,"upload-time":"2016-01-15T12:34:15.511699Z","url":"https://files.pythonhosted.org/packages/8a/a4/6dfd46e45d06da1a4d42814dbbdcffe3a4fc7f9b655e1c2919ac960512c2/psutil-3.4.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp26-none-win_amd64.whl","hashes":{"sha256":"bcf212a926e8cffd3bec2acaeb584bf59a536e569d404bd8ea306f1752fbfc41"},"provenance":null,"requires-python":null,"size":94311,"upload-time":"2016-01-15T12:34:35.252561Z","url":"https://files.pythonhosted.org/packages/85/38/d9882d4e37f4b791bd949a1f45c620e0f2573bb4048eb16d59d469e97ec6/psutil-3.4.1-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp27-none-win32.whl","hashes":{"sha256":"1b8424eaa712fef7da41fc7f391b452e8991a641a54e49c4f46eb72ca2585577"},"provenance":null,"requires-python":null,"size":91570,"upload-time":"2016-01-15T12:25:47.323665Z","url":"https://files.pythonhosted.org/packages/cd/2d/760f774b1325037ea4ef85972f45fc9dee417da33ba225b21a0a8e512f5d/psutil-3.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"46d7429bae3703a0f2980c0299d4d49ada733c7ebd2cfa4e29fa3e31b5b16014"},"provenance":null,"requires-python":null,"size":94024,"upload-time":"2016-01-15T12:29:47.181880Z","url":"https://files.pythonhosted.org/packages/13/06/0104f224dd52bf9e3fb3ef14f6b6b93e9fac72f562842d54445af041f3f0/psutil-3.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp33-none-win32.whl","hashes":{"sha256":"08f4ab9b720310890fa9337321a6e1e8aa525538636526be77e82653588df46b"},"provenance":null,"requires-python":null,"size":91580,"upload-time":"2016-01-15T12:26:59.466185Z","url":"https://files.pythonhosted.org/packages/65/22/f7121341bc75bff65000ecc0c5aad4f2a6d129506c26d5533ade2ca67349/psutil-3.4.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp33-none-win_amd64.whl","hashes":{"sha256":"9e52230373076d0ecdb4aec373afd342c576ab52e11c382e058ed0188181a352"},"provenance":null,"requires-python":null,"size":93869,"upload-time":"2016-01-15T12:30:39.415707Z","url":"https://files.pythonhosted.org/packages/b0/84/a9edadc49ef3dbb89298855ae069b18ec534ca9f79a9294de417b8e46571/psutil-3.4.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp34-none-win32.whl","hashes":{"sha256":"e658cd0e0ad7a2971b2eeb6ee4b1a0ad14245003ea47425846bc8c3e892fd567"},"provenance":null,"requires-python":null,"size":91584,"upload-time":"2016-01-15T12:28:02.007581Z","url":"https://files.pythonhosted.org/packages/b6/cd/59a87e4f10181ee228c4edc7d4927e3d62f652cff9f25f95a7e7e9ab3df0/psutil-3.4.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp34-none-win_amd64.whl","hashes":{"sha256":"9c3e1146003df43aec9274be4741371a06896d70d7d590eb882ad59de2c06120"},"provenance":null,"requires-python":null,"size":93858,"upload-time":"2016-01-15T12:31:25.406165Z","url":"https://files.pythonhosted.org/packages/30/06/cf0559d12ca5ded37e6a32b1671be57fad3bade7f24536b943851aa6393e/psutil-3.4.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp35-none-win32.whl","hashes":{"sha256":"3d3b2df184a31646a7e66cc48304f900a82c18ab3dc69d2d5f693ea97fca0572"},"provenance":null,"requires-python":null,"size":93406,"upload-time":"2016-01-15T12:28:48.357070Z","url":"https://files.pythonhosted.org/packages/c9/57/6e65ff27fa567cd9a7bfbc0a435e33293451b80865bc3a3a7b13c9bf7799/psutil-3.4.1-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp35-none-win_amd64.whl","hashes":{"sha256":"820ed01d84ffcda1c613be80c09318d7560dd3505299c65bb99f101963bfc3dd"},"provenance":null,"requires-python":null,"size":96333,"upload-time":"2016-01-15T12:31:50.145506Z","url":"https://files.pythonhosted.org/packages/f0/f6/ccf16168a627d10ffbd80120cd2c521c4c9ecdb4545e402b7deca79f93ac/psutil-3.4.1-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.tar.gz","hashes":{"sha256":"c7443659674c87d1f9feecee0dfeea765da02181c58d532e0633337e42180c89"},"provenance":null,"requires-python":null,"size":271657,"upload-time":"2016-01-15T12:22:39.420430Z","url":"https://files.pythonhosted.org/packages/a5/56/c64187a9a6889e622f7ec687254cdb3cc3c706e11bba9244e6ac781ecf38/psutil-3.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py2.6.exe","hashes":{"sha256":"e88e43423af984d7f2ecf8babf9d861ff59436794b0fdd2f85e9ea6bf7af6627"},"provenance":null,"requires-python":null,"size":331136,"upload-time":"2016-01-15T12:34:26.229804Z","url":"https://files.pythonhosted.org/packages/f7/90/53adfe2804c9cde062eb5014d88f0d067690fe1457ad2f49a2a553767689/psutil-3.4.1.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"3c08e2c200b222a92a4ecaa8055a48e27e7cfe82d9bf6402b52dd82413a786ed"},"provenance":null,"requires-python":null,"size":330798,"upload-time":"2016-01-15T12:29:26.473954Z","url":"https://files.pythonhosted.org/packages/66/9b/2b58fdab300e5f2a20c3999c485692cfa73cc9d4e50770a19cc871f92743/psutil-3.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"d03b8d081a281ebaa2122f259f7c0b3a464b2b98a3c221b9a54bfb0840355a9f"},"provenance":null,"requires-python":null,"size":329156,"upload-time":"2016-01-15T12:30:04.062459Z","url":"https://files.pythonhosted.org/packages/4c/26/695fa5b3578248f424d9a8e5bf2aafc6f706aeb7ec21ee31a5ebc2f79660/psutil-3.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"66a243c4b9ad93059be391a18e3f75e015ad70b220df4f7f30f9f578b89f27ad"},"provenance":null,"requires-python":null,"size":329140,"upload-time":"2016-01-15T12:31:00.945171Z","url":"https://files.pythonhosted.org/packages/25/63/54f2ba7cf31bb936b9c2cd7a77fd40a698fb232cd7c95c1ce997295a5954/psutil-3.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.5.exe","hashes":{"sha256":"4ee8641803d68a2e48952951336f9474a8914854da088fca673d67a91da7f9a4"},"provenance":null,"requires-python":null,"size":245586,"upload-time":"2016-01-15T12:31:39.398299Z","url":"https://files.pythonhosted.org/packages/43/54/f2d3b8845105fe5f55d5f0fde36773ab94fe1e35a0f4a5219adc818d586b/psutil-3.4.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py2.6.exe","hashes":{"sha256":"a4de0daf0dc7aeff6d45c6a1c782ef30d2b4fc6495196acabcb5cde2fb9b5a74"},"provenance":null,"requires-python":null,"size":300946,"upload-time":"2016-01-15T12:33:59.097771Z","url":"https://files.pythonhosted.org/packages/9e/34/53e1e85df97508df1b3eea711ab2809cc8f01b20e3b2341645673eb5d835/psutil-3.4.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py2.7.exe","hashes":{"sha256":"0efcecb6fcc21d83e9d4354754c6b8a8deb47a5fa06ec5d09fcf9799719eeac2"},"provenance":null,"requires-python":null,"size":300700,"upload-time":"2016-01-15T12:25:12.391548Z","url":"https://files.pythonhosted.org/packages/2f/4c/a07a53ff938e3bbc2ba73e4a484af8d1e02054b0bfcaf0f6d30117187d9f/psutil-3.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.3.exe","hashes":{"sha256":"62d0b529e40262293f39d7455db24f7ee297a1a3fe7f0e3e5923ae8168bd865c"},"provenance":null,"requires-python":null,"size":295638,"upload-time":"2016-01-15T12:26:24.412799Z","url":"https://files.pythonhosted.org/packages/39/6c/03ade7ba131b3952d916ed26c277418234bec0c9a5dfad513b9a5bb51046/psutil-3.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.4.exe","hashes":{"sha256":"6102294d6150f2c072dbc0166348389e8fa5d14a769ad118b697cda5b31c3381"},"provenance":null,"requires-python":null,"size":295651,"upload-time":"2016-01-15T12:27:36.577070Z","url":"https://files.pythonhosted.org/packages/52/d7/c2e9e0cb21482304e39a7681066c32c50e984f109bcda5929a84af926d70/psutil-3.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.5.exe","hashes":{"sha256":"0ffd99167272bb80c6ecf68f4c3d3176bef0f8c2a68f7e2787cab32413830023"},"provenance":null,"requires-python":null,"size":235502,"upload-time":"2016-01-15T12:28:24.463783Z","url":"https://files.pythonhosted.org/packages/1c/06/4d0ec9a6427db9c3b9885c4d724ca299746519b2ee61b724665d49e352c6/psutil-3.4.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp26-none-win32.whl","hashes":{"sha256":"2ac75c13657ab18eac0014e3f4c80def16978507b30e7719e46042ec93316bb0"},"provenance":null,"requires-python":null,"size":91920,"upload-time":"2016-01-20T16:25:20.303966Z","url":"https://files.pythonhosted.org/packages/be/b7/999dcbee8cc5ae32e64b2d0c9d588a3f5a441a07404772af83e86f3c8bc7/psutil-3.4.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp26-none-win_amd64.whl","hashes":{"sha256":"162f76140ca09490b9d218840bd641cbd1439245dcc2a9dd41f86224ed19490c"},"provenance":null,"requires-python":null,"size":94467,"upload-time":"2016-01-20T16:27:40.461506Z","url":"https://files.pythonhosted.org/packages/60/97/f9ea4fa7a4914350d15347a6a583c8a185643bb6bd5dc76d13e9d7dfc150/psutil-3.4.2-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp27-none-win32.whl","hashes":{"sha256":"b5f4bfdaa6389552501253b13b6022b7e3d715e4dca4b5cc1808f58cca181359"},"provenance":null,"requires-python":null,"size":91721,"upload-time":"2016-01-20T16:25:38.512050Z","url":"https://files.pythonhosted.org/packages/d4/19/4e5c376587076c969784762de8024bb30168a548b402e1b432221c5b97b1/psutil-3.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"9267e9bccb5c8b1c5ca872eb1caf88ba0ae47e336eb200be138f51d9f75e1113"},"provenance":null,"requires-python":null,"size":94177,"upload-time":"2016-01-20T16:27:58.779061Z","url":"https://files.pythonhosted.org/packages/d3/19/39f42cdfba58ab593d24f49ffc073c07b9b34ff7d5ba079b975018002e51/psutil-3.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp33-none-win32.whl","hashes":{"sha256":"413800a94815e6bf3e3227823e4d46b06c63bd22ab9e5af112b9220af9a9c9d8"},"provenance":null,"requires-python":null,"size":91738,"upload-time":"2016-01-20T16:25:59.987045Z","url":"https://files.pythonhosted.org/packages/8e/7a/8ba0c9da039b5733edbe17321f0546f08a7066861ffcdb83c31eb061e8f1/psutil-3.4.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp33-none-win_amd64.whl","hashes":{"sha256":"45535c18a0f261f90ff1ebb7d74c5e88d582cfb2006e4588498b9c0c9da5acb3"},"provenance":null,"requires-python":null,"size":94024,"upload-time":"2016-01-20T16:28:42.605657Z","url":"https://files.pythonhosted.org/packages/34/a7/94fc00a023ede02c3a7f525c63997a23c22434cbed65738ee3ef8939e084/psutil-3.4.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp34-none-win32.whl","hashes":{"sha256":"c9ef9a08254c251858cf747703e6fd75fe6e9549b1e040bb4a501feaf44a5a75"},"provenance":null,"requires-python":null,"size":91749,"upload-time":"2016-01-20T16:26:31.788083Z","url":"https://files.pythonhosted.org/packages/dc/8b/df5d7dcfe8fc5db0c303e518ce12b7117cb70e1cbb29c0396ea6e36fc7a2/psutil-3.4.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp34-none-win_amd64.whl","hashes":{"sha256":"0bf8925c3d252178c47bd8f29aff99c57a56f94513354b60069b457ca04bc25b"},"provenance":null,"requires-python":null,"size":94004,"upload-time":"2016-01-20T16:29:27.043903Z","url":"https://files.pythonhosted.org/packages/1c/cf/c9ce0014f43f74b1ce72c004c8f2eda68339cbc19117d9f090ee14afce3e/psutil-3.4.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp35-none-win32.whl","hashes":{"sha256":"461d1431a14e4da5e687cfdc2a8576b1f0e3bc658694ab9c6ef2fa1e4c1a4871"},"provenance":null,"requires-python":null,"size":93563,"upload-time":"2016-01-20T16:27:20.576920Z","url":"https://files.pythonhosted.org/packages/de/51/d1ab564dfe98d5fcbccfcab0afdedb269f7266192721e94688ab3956b123/psutil-3.4.2-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp35-none-win_amd64.whl","hashes":{"sha256":"23d4ea79fea3de81daf9460662e49ff718555779b2f5e5e3610648c0a8cafecc"},"provenance":null,"requires-python":null,"size":96484,"upload-time":"2016-01-20T16:29:53.151921Z","url":"https://files.pythonhosted.org/packages/89/d8/dc9ce7b8862ab2d86975dd5199791b0ab2b2168fc2389223a216c2da1d45/psutil-3.4.2-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.tar.gz","hashes":{"sha256":"b17fa01aa766daa388362d0eda5c215d77e03a8d37676b68971f37bf3913b725"},"provenance":null,"requires-python":null,"size":274361,"upload-time":"2016-01-20T16:26:46.533423Z","url":"https://files.pythonhosted.org/packages/7b/58/2675697b6831e6ac4b7b7bc4e5dcdb24a2f39f8411186573eb0de16eb6d5/psutil-3.4.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py2.6.exe","hashes":{"sha256":"3716cb36373ecfd033c148c8e8e22d815a9b682c87538c5bde2a3faca1a44705"},"provenance":null,"requires-python":null,"size":331279,"upload-time":"2016-01-20T16:27:30.129391Z","url":"https://files.pythonhosted.org/packages/6f/52/26cefb84d714ecdf39f6d4ea62af49af731cf55d8937a252226de41d3fb0/psutil-3.4.2.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py2.7.exe","hashes":{"sha256":"9eba153441fabd6677f9dec95eedbfdcf4fe832a43b91c18f2c15bfd0a12b6c0"},"provenance":null,"requires-python":null,"size":330991,"upload-time":"2016-01-20T16:27:49.025596Z","url":"https://files.pythonhosted.org/packages/1f/85/817b298f6865d7a140897882015096ab25514e113c98ba3896b2e2c0425c/psutil-3.4.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.3.exe","hashes":{"sha256":"2ffc77ec7452675db45174f77e65bfc9abd5780e696c4dd486fff89e18ef104a"},"provenance":null,"requires-python":null,"size":329349,"upload-time":"2016-01-20T16:28:23.729829Z","url":"https://files.pythonhosted.org/packages/2b/05/ccb8e8dc272a6aa126a6066583102b78df8939aeffd910a3ea28a51d48af/psutil-3.4.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.4.exe","hashes":{"sha256":"6b576ea8faa312700953de30b92ff49dcd966dcdbf2e039c3655077826b59812"},"provenance":null,"requires-python":null,"size":329334,"upload-time":"2016-01-20T16:29:12.322779Z","url":"https://files.pythonhosted.org/packages/8f/0e/9b3eedad9ea2aa8e51c3ca6aa6485c0ec85bfc73925fbd4d82bbe03ead18/psutil-3.4.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.5.exe","hashes":{"sha256":"2a987c57ddb06a1e67f75a4dd34d2962f8675c3d60b2104da2d60fdaa378b50f"},"provenance":null,"requires-python":null,"size":245779,"upload-time":"2016-01-20T16:29:45.537968Z","url":"https://files.pythonhosted.org/packages/2f/bb/5483a7a54dfaedcd5bc6d0f9f8beef21d96785589e10d09e246f7092cfe1/psutil-3.4.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py2.6.exe","hashes":{"sha256":"87b657f1021ab4155669f77baf9557657a015b0762854702d64ee7cfa19d5ae2"},"provenance":null,"requires-python":null,"size":301089,"upload-time":"2016-01-20T16:25:11.034218Z","url":"https://files.pythonhosted.org/packages/2e/9b/2bb0317a5113b4a3d597a9bcb94cafacb52f15a10631b1f0eb781ceb8e7a/psutil-3.4.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py2.7.exe","hashes":{"sha256":"d837d654a78fcc6cf7338fc3c3f025e5a43cf646d4d6cf180f0f3573ef255844"},"provenance":null,"requires-python":null,"size":300894,"upload-time":"2016-01-20T16:25:29.293012Z","url":"https://files.pythonhosted.org/packages/8c/3b/bf4d0698153784231768aa79255f1641efde680c4d178ba327546eba69df/psutil-3.4.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.3.exe","hashes":{"sha256":"ef756512c86cf24916f47b2209ff5dc69ef4d5ff8b3b0229863aab3537af58a1"},"provenance":null,"requires-python":null,"size":295833,"upload-time":"2016-01-20T16:25:50.417672Z","url":"https://files.pythonhosted.org/packages/50/be/c4911ae27c944e12183d9ba844ff0eee706b5208f92e4929d8120b79448e/psutil-3.4.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.4.exe","hashes":{"sha256":"13917c61de33518fcdca94a8f1005c4bb0be1f106af773de8c12d6a5a3f349ae"},"provenance":null,"requires-python":null,"size":295845,"upload-time":"2016-01-20T16:26:17.574774Z","url":"https://files.pythonhosted.org/packages/b5/d9/9a15af2703d8a0d7b685511df811fc4930b12d7b85b96573e843a0ba1067/psutil-3.4.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.5.exe","hashes":{"sha256":"75c4484f0e1038c1a37ae37fc80f5b59b456e363c696ed889027e79a831853ec"},"provenance":null,"requires-python":null,"size":235696,"upload-time":"2016-01-20T16:27:12.935221Z","url":"https://files.pythonhosted.org/packages/2c/74/46cf554a4a8d7d2367f058e6c90ba26a66e52e818812ec6de53a5013bd87/psutil-3.4.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp26-none-win32.whl","hashes":{"sha256":"0661261b634f01ec2568136fedf29382f5c94678c34f56b4137b1d019085ca6f"},"provenance":null,"requires-python":null,"size":154479,"upload-time":"2016-02-17T16:42:58.173240Z","url":"https://files.pythonhosted.org/packages/72/e2/24700d1a099dcd824ca7305cf439625a457221459494e677e7649a2f228b/psutil-4.0.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp26-none-win_amd64.whl","hashes":{"sha256":"b613a9fb5c3d2b16c65df3121aa369a28caed83391883bc24918cf16c5de495b"},"provenance":null,"requires-python":null,"size":156749,"upload-time":"2016-02-17T16:45:09.428253Z","url":"https://files.pythonhosted.org/packages/62/c0/1adb11a832d5aab8a984702a4f6fc08e5698fa4bbc8dc6eddac1c711c7ab/psutil-4.0.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-cp27m-win32.whl","hashes":{"sha256":"3eb6bc7e8d92777deb4288178c25455e21109033bb54ec475485b611e92d3b42"},"provenance":null,"requires-python":null,"size":154302,"upload-time":"2016-02-17T16:43:23.863441Z","url":"https://files.pythonhosted.org/packages/e8/c3/542bc833b743e952cbf99017ecb60add0ad3725a82e942b25aa5de523f8c/psutil-4.0.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"c94193f38aa3bc35fd5dbcd24653d1f683c88ec8030997d1d56f92207ba7c523"},"provenance":null,"requires-python":null,"size":156487,"upload-time":"2016-02-24T17:23:02.172281Z","url":"https://files.pythonhosted.org/packages/32/1b/5f3cc96374c4eac441e96bb8698556c6c48eacfdcf843093bebcfd8ce56b/psutil-4.0.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"cb969d3c77db8810aba45e8a04e0b2851cd088e338be2430e1ff452f4e06007c"},"provenance":null,"requires-python":null,"size":156486,"upload-time":"2016-02-17T16:45:40.145586Z","url":"https://files.pythonhosted.org/packages/94/37/dc09e24aa80016ddeaff235d2f724d8aac9813b73cc3bf8a7fe3d1878315/psutil-4.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-cp33m-win32.whl","hashes":{"sha256":"6a372681382b523bc837ee2eff6a84ded0f85b013b7c29ea6211bc928c7cc656"},"provenance":null,"requires-python":null,"size":154244,"upload-time":"2016-02-24T17:21:40.946801Z","url":"https://files.pythonhosted.org/packages/a0/8a/b9352e0daf69b501296715e0fca1b49d861130eb66156ce3b12aeeb039e4/psutil-4.0.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"2bbb75fc2549965b457f313cbdfb98a00624f25fcb36e075322bb8b8912d83b5"},"provenance":null,"requires-python":null,"size":156393,"upload-time":"2016-02-24T17:23:25.807132Z","url":"https://files.pythonhosted.org/packages/4b/77/0fefa732947da69cba7f2580285eff553fe4a416314234f809901bede361/psutil-4.0.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-none-win32.whl","hashes":{"sha256":"d6219c89940d745b614716be7c906660f2108a1d84b8ffc720922596b8306e23"},"provenance":null,"requires-python":null,"size":154245,"upload-time":"2016-02-17T16:43:48.027623Z","url":"https://files.pythonhosted.org/packages/1f/d9/34fb4fab5f1bbfeddc76675b1b5ca00b45ef490e63295af33542cedcd26b/psutil-4.0.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-none-win_amd64.whl","hashes":{"sha256":"a64bb22e264f91a6d80cf8fdd813bd4fdd349dc367b363d517cf8ae1bc2c5db0"},"provenance":null,"requires-python":null,"size":156399,"upload-time":"2016-02-17T16:46:09.673427Z","url":"https://files.pythonhosted.org/packages/97/03/b9485635cb38dfad854754625422a49f434f53f214bff4885580f0fb21e6/psutil-4.0.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-cp34m-win32.whl","hashes":{"sha256":"a521266ac13485772987f00342b53cb230cde98ce91d61154860ba4109fe2ebe"},"provenance":null,"requires-python":null,"size":154269,"upload-time":"2016-02-17T16:44:14.936699Z","url":"https://files.pythonhosted.org/packages/73/32/6399071b097f1251f6fa12770b30a67d5b3c9c0c76e81eacbb6139e1bf6d/psutil-4.0.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"b559b8e8a85cde929e01e94e9635649e8641a88b2d077714933dc7723a967020"},"provenance":null,"requires-python":null,"size":156390,"upload-time":"2016-02-24T17:23:50.194021Z","url":"https://files.pythonhosted.org/packages/71/d7/878b77bad61bd94f4454536e823b6a48cd0af0f23b1506a2c8a49b2578cd/psutil-4.0.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-none-win_amd64.whl","hashes":{"sha256":"02d7291f81e78c506ac2b5481aa9dc6d3888195484ac114ac984b37477f60929"},"provenance":null,"requires-python":null,"size":156397,"upload-time":"2016-02-17T16:46:44.626238Z","url":"https://files.pythonhosted.org/packages/ea/22/5f44e6eaa1e82f5a1497f3dfcf045e1998fca36d70de8a370ec96ce0f789/psutil-4.0.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-cp35m-win32.whl","hashes":{"sha256":"7906302696960a6a788bb8fe1165b4ccd0156553b8a2f61640fd45a836d39024"},"provenance":null,"requires-python":null,"size":156177,"upload-time":"2016-02-24T17:22:25.142667Z","url":"https://files.pythonhosted.org/packages/e2/fe/a5ec73e62878cc2d0451b7029f4406647435dd8036ab15d6ed2fd42558bf/psutil-4.0.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f4214bdb2e96374b4c4a3a818bd8c7867f94571d33b91867b6dfd5f9b328c8ac"},"provenance":null,"requires-python":null,"size":158783,"upload-time":"2016-02-24T17:24:26.439061Z","url":"https://files.pythonhosted.org/packages/1d/a7/9300ad3d4071c191894073a94217ed5c0ca9604c782bdbf083bbedfa9cb1/psutil-4.0.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-none-win32.whl","hashes":{"sha256":"4a1631cb8c4de2b6c9b4b16f8800d43de23c683805f7b6a5aec1c268a73df270"},"provenance":null,"requires-python":null,"size":156183,"upload-time":"2016-02-17T16:44:42.458015Z","url":"https://files.pythonhosted.org/packages/14/f0/a2436cb642ecfec0bfb6338e5fa26581d4dbcf1a00f7d9fe99380eb6779f/psutil-4.0.0-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-none-win_amd64.whl","hashes":{"sha256":"994839b6d99acbf90914fddf2e2817aaffb67ceca5d10134319267e3ffe97258"},"provenance":null,"requires-python":null,"size":158787,"upload-time":"2016-02-17T16:47:18.920483Z","url":"https://files.pythonhosted.org/packages/e8/2a/e215824c785d77119af61802bbb4d16dacc26ec0687709274afa3ac039fa/psutil-4.0.0-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.tar.gz","hashes":{"sha256":"1a7c672f9ee79c84ff16b8de6f6040080f0e25002ac47f115f4a54aa88e5cfcd"},"provenance":null,"requires-python":null,"size":293800,"upload-time":"2016-02-17T16:41:45.938066Z","url":"https://files.pythonhosted.org/packages/c4/3b/44bcae6c0fc53362bb7325fde25a73b7fd46541b57c89b7556ca81b08e7e/psutil-4.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py2.6.exe","hashes":{"sha256":"61bf81cbe84e679a5b619e65775b0674b2c463885e49ddab73778198608198c5"},"provenance":null,"requires-python":null,"size":394157,"upload-time":"2016-02-17T16:45:00.666953Z","url":"https://files.pythonhosted.org/packages/86/50/6303a28a4ab5c9b6b9ef74eef70b141d5bd743ab096c58d225b6212fa057/psutil-4.0.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"23618cbc04e2431d9c4d97f56ab8b4e2e35366c9a9a6e1ef89a3a7287d359864"},"provenance":null,"requires-python":null,"size":393901,"upload-time":"2016-02-17T16:45:21.822412Z","url":"https://files.pythonhosted.org/packages/59/82/93052d6359addea338c528ebd50254806d62bc2b2d1ad1303c49d85162f9/psutil-4.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"b6f16c71be03495eeb4772c1f3f926213e3ea82ea7779bd1143229e6b419760b"},"provenance":null,"requires-python":null,"size":392318,"upload-time":"2016-02-17T16:45:57.189438Z","url":"https://files.pythonhosted.org/packages/e6/14/9ac37705e0753732c7707b000d1e076daac95ee02f35fd43ce906235ea1f/psutil-4.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"24b41e436afbcecb07e485e58a52effdbd7b8065ad8a2e4d555b6d88907f19b7"},"provenance":null,"requires-python":null,"size":392315,"upload-time":"2016-02-17T16:46:29.967466Z","url":"https://files.pythonhosted.org/packages/77/a9/ff7c29d2e244f5bdc7654a626cbfcccd401e78df6e9388713f322c7aa7c7/psutil-4.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.5.exe","hashes":{"sha256":"88edc0e7bfa672c245df74b1ac3b59db432cd75e5704beccc268e177ac2ffbbc"},"provenance":null,"requires-python":null,"size":308678,"upload-time":"2016-02-17T16:47:07.280714Z","url":"https://files.pythonhosted.org/packages/cc/1b/863bee07da70fe61cae804333d64242d9001b54288e8ff54e770225bbc0a/psutil-4.0.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py2.6.exe","hashes":{"sha256":"882a8ac29b63f256f76465d8fcd6e9eaeb9c929acdac26af102da97d66b2b619"},"provenance":null,"requires-python":null,"size":364245,"upload-time":"2016-02-17T16:42:49.267819Z","url":"https://files.pythonhosted.org/packages/41/39/6ea85cb0c748aae2943144118f1696004f5a99c54dab1fc635cffbb0d06c/psutil-4.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py2.7.exe","hashes":{"sha256":"bf3b2e305ca7408df40156c9aa6261c7baaff831441f0c018d0682bd820286f2"},"provenance":null,"requires-python":null,"size":364072,"upload-time":"2016-02-17T16:43:14.467612Z","url":"https://files.pythonhosted.org/packages/8f/0d/3e9cf8abb62d7241531019d78abaa87a19f3fcc017bfb9c2058ba61e8cf1/psutil-4.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.3.exe","hashes":{"sha256":"38d38211bba35c705a007e62b9dcc9be1d222acfcbee812612d4a48f9d8f0230"},"provenance":null,"requires-python":null,"size":358940,"upload-time":"2016-02-17T16:43:38.439349Z","url":"https://files.pythonhosted.org/packages/57/eb/514a71eab624b381473a3df9c3e3a02f5bb15707b12daf02c137271dfd26/psutil-4.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.4.exe","hashes":{"sha256":"bc868653a6502c3a01da32b3a598a8575674975f5586ac0bf9181349588925b9"},"provenance":null,"requires-python":null,"size":358966,"upload-time":"2016-02-17T16:44:02.873748Z","url":"https://files.pythonhosted.org/packages/25/f6/ef4b802658c21d5b79a0e038db941f4b08a7cc2de78df1949ad709542682/psutil-4.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.5.exe","hashes":{"sha256":"38244b0d07d3bece481a6f1c049e6101fdd26f2ee63dadcb63ce993283032fdc"},"provenance":null,"requires-python":null,"size":298915,"upload-time":"2016-02-17T16:44:31.772075Z","url":"https://files.pythonhosted.org/packages/b9/be/b8938c409231dab07d2954ff7b4d1129e725e0e8ab1b016d7f471f3285e9/psutil-4.0.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp26-none-win32.whl","hashes":{"sha256":"13aed96ad945db5c6b3d5fbe92be65330a3f2f757a300c7d1578a16efa0ece7f"},"provenance":null,"requires-python":null,"size":159498,"upload-time":"2016-03-12T17:13:58.069685Z","url":"https://files.pythonhosted.org/packages/77/04/d5a92cb5c0e79b84294f6c99b9725806921d1d88032e9d056ca8a7ba31c1/psutil-4.1.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp26-none-win_amd64.whl","hashes":{"sha256":"90b58cf88e80a4af52b79678df474679d231ed22200e6c25605a42ca71708a47"},"provenance":null,"requires-python":null,"size":161924,"upload-time":"2016-03-12T17:17:01.346400Z","url":"https://files.pythonhosted.org/packages/b5/a5/cf96f9f13f9e20bdb4cd2ca1af2ddd74f76fea4bbfb8505c31a5900b38d2/psutil-4.1.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp27-cp27m-win32.whl","hashes":{"sha256":"ac141a44a5c145e9006bc7081c714b2c317077d158b65fe4624c9cbf2b8ac7bf"},"provenance":null,"requires-python":null,"size":159318,"upload-time":"2016-03-12T17:14:48.199436Z","url":"https://files.pythonhosted.org/packages/8a/31/439614cc2ccd6f3ce1d173c0d7c7a9e45be17cd2bf3ae1f8feaaf0a90cee/psutil-4.1.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"3605b6b9f23f3e186b157b03a95e0158559eb74bcef5d51920b8ddb48cc3a7e7"},"provenance":null,"requires-python":null,"size":161579,"upload-time":"2016-03-12T17:17:24.047978Z","url":"https://files.pythonhosted.org/packages/90/97/0a34c0e98bb794f0fc19f0eae13d26fbf39583c768a9a6c614c917135c00/psutil-4.1.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp33-cp33m-win32.whl","hashes":{"sha256":"5568e21c8eb9de0e56c8a4a38982b725bf42117bca7ac75c7b079e5214aea5c4"},"provenance":null,"requires-python":null,"size":159241,"upload-time":"2016-03-12T17:15:16.355951Z","url":"https://files.pythonhosted.org/packages/bb/de/8e1f8c4ea6035d08e3c87a0cfc8af6f2862da21697c16d1d17311e095117/psutil-4.1.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"3d3f8ae20b04b68e65b46dc2eedf15f32925655dacbb11cb7afe56ac562e112a"},"provenance":null,"requires-python":null,"size":161458,"upload-time":"2016-03-12T17:17:51.825112Z","url":"https://files.pythonhosted.org/packages/ac/cf/6241dd597ef4f995ab8e29746c54890c1acbb322484afed05aa8988118e1/psutil-4.1.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp34-cp34m-win32.whl","hashes":{"sha256":"faafb81bf7717fa8c44bb0f2e826768f561c0311fd0568090c59c9b253b65238"},"provenance":null,"requires-python":null,"size":159266,"upload-time":"2016-03-12T17:15:48.577644Z","url":"https://files.pythonhosted.org/packages/48/2d/46ba91df965d4f0af5fd4252ac249ff408f6cb966fe1208396933275246f/psutil-4.1.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"4ab1ee4152dbb790a37291149b73b1918ab4398c8edb3af7847fa6c884024c93"},"provenance":null,"requires-python":null,"size":161466,"upload-time":"2016-03-12T17:18:23.409579Z","url":"https://files.pythonhosted.org/packages/c0/b1/70868328ddf2cfcde201136bdaf4c9f7fabf868890bc91694fd5fa0fbc19/psutil-4.1.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp35-cp35m-win32.whl","hashes":{"sha256":"2fa06b7ba58e870fdaa1427e82ed427a785493c7a998e059e0806b2c48bdbfaf"},"provenance":null,"requires-python":null,"size":161345,"upload-time":"2016-03-12T17:16:33.062364Z","url":"https://files.pythonhosted.org/packages/dc/f7/5d3f84507c057af85bc70da10b51a827766999f221b42cdf9621ca756e80/psutil-4.1.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"1ca460fea3822d04f332f5dde144accc5ca4610e5bbce53f15d09cd985f30385"},"provenance":null,"requires-python":null,"size":164147,"upload-time":"2016-03-12T17:18:49.483687Z","url":"https://files.pythonhosted.org/packages/f1/65/040624aab6ca646af0c8b68ac54d08e0a33a672feb9405581cd509741367/psutil-4.1.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.tar.gz","hashes":{"sha256":"c6abebec9c8833baaf1c51dd1b0259246d1d50b9b50e9a4aa66f33b1e98b8d17"},"provenance":null,"requires-python":null,"size":301330,"upload-time":"2016-03-12T17:12:53.032151Z","url":"https://files.pythonhosted.org/packages/71/9b/6b6f630ad4262572839033b69905d415ef152d7701ef40aa98941ba75b38/psutil-4.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py2.6.exe","hashes":{"sha256":"30a97a9c3ace92001e419a6bb039b2e899c5ab24cab6ad8bc249506475c84a0c"},"provenance":null,"requires-python":null,"size":399511,"upload-time":"2016-03-12T17:16:48.833953Z","url":"https://files.pythonhosted.org/packages/28/bd/c389af84b684d36010a634834f76932ff60f33505c54413c50eceb720a5d/psutil-4.1.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"3ba9236c38fe85088b9d79bd5871e07f253d225b357ce82db9240e5807f147b6"},"provenance":null,"requires-python":null,"size":399168,"upload-time":"2016-03-12T17:17:14.518227Z","url":"https://files.pythonhosted.org/packages/6c/e8/49ff1b33e9fa6f7ea1232d40bbef9a453424fbf0421a7c53d9ecb9a896e7/psutil-4.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"99a902d1bf5beb13cca4d7a3dc82efb6eaf40aafe916a5b632d47393313bfcfd"},"provenance":null,"requires-python":null,"size":397557,"upload-time":"2016-03-12T17:17:40.792655Z","url":"https://files.pythonhosted.org/packages/c5/3e/47adda552a79480e2a5d38a2f90144ab4e7ea34eba2b707523cdd1c65fc6/psutil-4.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"ecf8b83e038acdda9102aebadb1525f89231463772cbe89e3e8a2cf5a5c6065d"},"provenance":null,"requires-python":null,"size":397564,"upload-time":"2016-03-12T17:18:11.514028Z","url":"https://files.pythonhosted.org/packages/52/9f/5874f391a300feead5519872395cbb4d4588eace24962150a03cd9b6ffdf/psutil-4.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.5.exe","hashes":{"sha256":"448db9bf9db5d162f0282af70688e03d67d756b93d5bed94b0790a27a96af75b"},"provenance":null,"requires-python":null,"size":314216,"upload-time":"2016-03-12T17:18:36.745340Z","url":"https://files.pythonhosted.org/packages/32/34/6588580a1775a0741e14946003bf2722c493c7956c7dcc9a40c85dbe19f5/psutil-4.1.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py2.6.exe","hashes":{"sha256":"11b48d7ec00061960ffbef33a42e919f188e10a6a54c4161692d77a3ac37f1e2"},"provenance":null,"requires-python":null,"size":369441,"upload-time":"2016-03-12T17:13:47.639393Z","url":"https://files.pythonhosted.org/packages/5d/cc/7bf8593a60ed54b47def9a49a60b8bc3517d6fef6ee52a229583a5bc9046/psutil-4.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py2.7.exe","hashes":{"sha256":"c15ddba9f4c278f7d1bbbadc34df89d993171bb328fa1117cbecf68bcc1a01e5"},"provenance":null,"requires-python":null,"size":369263,"upload-time":"2016-03-12T17:14:25.482243Z","url":"https://files.pythonhosted.org/packages/bf/fa/7f0eda490dd5480bb8e6358f9e893fe7f824ec3a5b275b069708272d2260/psutil-4.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.3.exe","hashes":{"sha256":"8f97a836b89b09718b00aedbfd11d8468bbfcb821feadd65e338bd0b972dac54"},"provenance":null,"requires-python":null,"size":364112,"upload-time":"2016-03-12T17:15:06.252090Z","url":"https://files.pythonhosted.org/packages/e8/1c/e7d17500c0f5899f0fd3fd3b6d93c92254aff6de00cabc9c08de5a3803a2/psutil-4.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.4.exe","hashes":{"sha256":"fef37f4e2964043a58b6aa3b3be05ef9b3c7a58feb806d7e49dc91702bac52fa"},"provenance":null,"requires-python":null,"size":364136,"upload-time":"2016-03-12T17:15:29.468205Z","url":"https://files.pythonhosted.org/packages/6d/8e/7acd4079567fc64e1df12be6faeddf0ee19432445210293594444ea980a6/psutil-4.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.5.exe","hashes":{"sha256":"1a9409d2204d397b1ed10528065e45020e0b1c2ee5204bf5a1bc6fdff3f6ab91"},"provenance":null,"requires-python":null,"size":304255,"upload-time":"2016-03-12T17:16:06.024724Z","url":"https://files.pythonhosted.org/packages/5b/30/10f3bef7fa284167a3f9bd3862bcba5cc14fb8509f146a2f696ea5265b93/psutil-4.1.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"data-dist-info-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"filename":"psutil-4.2.0-cp27-cp27m-win32.whl","hashes":{"sha256":"19f6c8bd30d7827ce4d4bbcfe23fe7158fea3d72f59505850c5afa12985184bb"},"provenance":null,"requires-python":null,"size":165248,"upload-time":"2016-05-15T06:36:58.089018Z","url":"https://files.pythonhosted.org/packages/58/a5/2ccc9f6180ea769005405381f6b0d01fe1268f20cc85877b02c04c27d306/psutil-4.2.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"data-dist-info-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"filename":"psutil-4.2.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"92bc2351bb4bc7672b3d0e251a449ac2234bbe4fac11f708614bdc0a8ebffe3b"},"provenance":null,"requires-python":null,"size":167782,"upload-time":"2016-05-15T06:37:06.221707Z","url":"https://files.pythonhosted.org/packages/c8/e5/5d0a1b2e182e41888fc4e9f4f657f37f126f9fdcd431b592442311c2db98/psutil-4.2.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp33-cp33m-win32.whl","hashes":{"sha256":"2e16f792deceb1d33320981aaff7f139561cf6195ee3f1b21256d7f214162517"},"provenance":null,"requires-python":null,"size":165259,"upload-time":"2016-05-15T06:37:12.236981Z","url":"https://files.pythonhosted.org/packages/dd/94/8aeb332d07530b552099eaf207db13d859e09facfa8162892b4f9ef302dd/psutil-4.2.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"3c57a6731b3bd4c4af834b0137493a388b76192f5adc2399825015b777e0b02b"},"provenance":null,"requires-python":null,"size":167667,"upload-time":"2016-05-15T06:37:17.736147Z","url":"https://files.pythonhosted.org/packages/d5/6b/c10a228ef2cdbc077171be3b273cd2f49e4f814bf7dc2deb3a464cc126de/psutil-4.2.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp34-cp34m-win32.whl","hashes":{"sha256":"0cda72a1efacd2b028c9dbf0731111041e6cf9e7be938162811ab32ab3e88254"},"provenance":null,"requires-python":null,"size":165272,"upload-time":"2016-05-15T06:37:23.488252Z","url":"https://files.pythonhosted.org/packages/35/3e/3db756e014fe3e6e22e35c8394057dcf1eef58076d84fdf83ac00a053182/psutil-4.2.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"ce208e1c416e143697a1ee9dd86ae9720c740c11764a1fda88eb28a2ecc0b510"},"provenance":null,"requires-python":null,"size":167627,"upload-time":"2016-05-15T06:37:28.596251Z","url":"https://files.pythonhosted.org/packages/c7/7d/cfb299960cf6923cca782f331b034c09239e9015dedf530dd206177dd6e4/psutil-4.2.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp35-cp35m-win32.whl","hashes":{"sha256":"375b0acad448e49c8bc62e036f948af610b4e0cbe2a9a28eebc06357f20f67ea"},"provenance":null,"requires-python":null,"size":167370,"upload-time":"2016-05-15T06:37:34.104073Z","url":"https://files.pythonhosted.org/packages/4d/bc/f49882e8935f147b8922fc8bb0f430fe0e7b0d3231a601cd12e1c0272f77/psutil-4.2.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bd4b535996d06728b50bc7cd8777c402bf7294ad05229c843701bd1e63583c2c"},"provenance":null,"requires-python":null,"size":170689,"upload-time":"2016-05-15T06:37:39.395642Z","url":"https://files.pythonhosted.org/packages/7b/e2/2e1078a38189d51409f50af50b598309a2bd84ebe8ca71b79515da915c82/psutil-4.2.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.tar.gz","hashes":{"sha256":"544f013a0aea7199e07e3efe5627f5d4165179a04c66050b234cc3be2eca1ace"},"provenance":null,"requires-python":null,"size":311767,"upload-time":"2016-05-15T06:35:49.367304Z","url":"https://files.pythonhosted.org/packages/a6/bf/5ce23dc9f50de662af3b4bf54812438c298634224924c4e18b7c3b57a2aa/psutil-4.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"1329160e09a86029ef4e07f47dbcc39d511c343257a53acf1af429c537caae57"},"provenance":null,"requires-python":null,"size":406095,"upload-time":"2016-05-15T06:37:44.994723Z","url":"https://files.pythonhosted.org/packages/6b/af/9e43a4a4976f1d1291de8be40c848c591c6e48d7e4053a7b26ad88ba750c/psutil-4.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"12623a1e2e264eac8c899b89d78648e241c12eec754a879453b2e0a4a78b10dd"},"provenance":null,"requires-python":null,"size":404495,"upload-time":"2016-05-15T06:37:50.698273Z","url":"https://files.pythonhosted.org/packages/36/85/64244b1e930aa276205f079ba3e2996e8492bd173af019bbdaee47336a6a/psutil-4.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"66a4a7793dc543a3c7413cda3187e3ced45acf302f95c4d596ebcfc663c01b40"},"provenance":null,"requires-python":null,"size":404456,"upload-time":"2016-05-15T06:37:56.103096Z","url":"https://files.pythonhosted.org/packages/30/fa/a734058699f351ef90b757e0fd8d67a6145c8272bbed85c498276acacd2e/psutil-4.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.5.exe","hashes":{"sha256":"c2b7aa0a99b06967fb76e83e7e9c7153a2d9a5df073986a99a4e9656cfaabe28"},"provenance":null,"requires-python":null,"size":321489,"upload-time":"2016-05-15T06:38:02.113109Z","url":"https://files.pythonhosted.org/packages/a9/1b/c90c802a6db438aeebac412ac3ecf389022f4fd93abef9c0441358f46a71/psutil-4.2.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py2.7.exe","hashes":{"sha256":"856480ce003ecd1601bcb83d97e25bfe79f5b08c430ee9f139a5e768173b06ef"},"provenance":null,"requires-python":null,"size":375917,"upload-time":"2016-05-15T06:38:09.945181Z","url":"https://files.pythonhosted.org/packages/e4/63/267b0977027c8a4a2f98a1ffbc2ecc7c0689d12adabee591a1ac99b4c14e/psutil-4.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.3.exe","hashes":{"sha256":"ab83fefffa495813d36300cd3ad3f232cf7c86a5e5a02d8e8ea7ab7dba5a1a90"},"provenance":null,"requires-python":null,"size":370860,"upload-time":"2016-05-15T06:38:16.360218Z","url":"https://files.pythonhosted.org/packages/c0/96/8197557cbebb16be1cfd3c87f1d0972bd2e5b0733b21d0e5d890541634e8/psutil-4.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.4.exe","hashes":{"sha256":"44c9f0e26b93c2cc9437eb88c31df32bd4337c394a959e0c31bf006da6e0f073"},"provenance":null,"requires-python":null,"size":370872,"upload-time":"2016-05-15T06:38:22.312293Z","url":"https://files.pythonhosted.org/packages/22/c9/01646a50e3c52dda4b591aae411e85afc83952107f9906ec8a5806c9fcc0/psutil-4.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.5.exe","hashes":{"sha256":"c0013a6663b794fbe18284e06d4d553a9e2135b5489a2ac6982ad53641966a55"},"provenance":null,"requires-python":null,"size":311007,"upload-time":"2016-05-15T06:38:28.533757Z","url":"https://files.pythonhosted.org/packages/48/6f/2259000fa07a4efcdc843034810eb2f8675ef5b97845912f2265589483f4/psutil-4.2.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"data-dist-info-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"filename":"psutil-4.3.0-cp27-none-win32.whl","hashes":{"sha256":"99c2ab6c8f0d60e0c86775f8e5844e266af48cc1d9ecd1be209cd407a3e9c9a1"},"provenance":null,"requires-python":null,"size":167400,"upload-time":"2016-06-18T17:56:45.275403Z","url":"https://files.pythonhosted.org/packages/7a/a5/002caeac2ff88526cf0788315bad93be61e66477acd54209fb01cc874745/psutil-4.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"data-dist-info-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"filename":"psutil-4.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"a91474d34bf1bc86a0d95e2c198a70723208f9dc9e50258c2060a1bab3796f81"},"provenance":null,"requires-python":null,"size":169895,"upload-time":"2016-06-18T17:56:51.386191Z","url":"https://files.pythonhosted.org/packages/e1/77/fae92fff4ca7092555c7c8fc6e02c5dfc2a9af7e15762b7354d436adeb06/psutil-4.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp33-cp33m-win32.whl","hashes":{"sha256":"c987f0691c01cbe81813b0c895208c474240c96e26f7d1e945e8dabee5c85437"},"provenance":null,"requires-python":null,"size":167414,"upload-time":"2016-06-18T17:56:57.419709Z","url":"https://files.pythonhosted.org/packages/73/46/224a6a3c05df4e282240826f6bef0bd51da5ca283ffe34ed53f5601fbca1/psutil-4.3.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"233a943d3e6636d648f05515a4d21b4dda63499d8ca38d6890a57a3f78a9cceb"},"provenance":null,"requires-python":null,"size":169791,"upload-time":"2016-06-18T17:57:03.096187Z","url":"https://files.pythonhosted.org/packages/6b/4d/29b3d73c27cd8f348bb313376dd98a2e97cc8a075862cf483dea4c27e4bf/psutil-4.3.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp34-cp34m-win32.whl","hashes":{"sha256":"1893fe42b0fb5f11bf84ffe770be5b2e27fb7ec959ba8bf620b704552b738c72"},"provenance":null,"requires-python":null,"size":167396,"upload-time":"2016-06-18T17:57:09.859528Z","url":"https://files.pythonhosted.org/packages/dc/f5/34070d328a578c38131d96c4fe539ebbabf1c31128012755e344c030bb34/psutil-4.3.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"1345217075dd5bb4fecbf7cb0fe4c0c170e93ec57d48494756f4b617cd21b449"},"provenance":null,"requires-python":null,"size":169759,"upload-time":"2016-06-18T17:57:15.923623Z","url":"https://files.pythonhosted.org/packages/8d/91/7ae0835ae1a4bc1043b565dec9c6468d56c80a4f199472a7d005ddcd48e1/psutil-4.3.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp35-cp35m-win32.whl","hashes":{"sha256":"c9c4274f5f95a171437c90f65c3e9b71a871753f0a827f930e1b14aa43041eab"},"provenance":null,"requires-python":null,"size":169508,"upload-time":"2016-06-18T17:57:21.072109Z","url":"https://files.pythonhosted.org/packages/e4/40/801cd906da337a5e7a0afaaa1ce5919d9834c50d804d31ee4a1d2120a51c/psutil-4.3.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"5984ee7b2880abcdaa0819315f69a5f37da963863495c2294392cb3e98141a95"},"provenance":null,"requires-python":null,"size":172803,"upload-time":"2016-06-18T17:57:26.746663Z","url":"https://files.pythonhosted.org/packages/35/54/ddb6e8e583abf2f5a2be52a2223ba4935b382214b696fe334af54fb03dad/psutil-4.3.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.tar.gz","hashes":{"sha256":"86197ae5978f216d33bfff4383d5cc0b80f079d09cf45a2a406d1abb5d0299f0"},"provenance":null,"requires-python":null,"size":316470,"upload-time":"2016-06-18T17:54:55.929749Z","url":"https://files.pythonhosted.org/packages/22/a8/6ab3f0b3b74a36104785808ec874d24203c6a511ffd2732dd215cf32d689/psutil-4.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"1ad0075b6c86c0ea5076149ec39dcecf0c692711c34317d43d73a4d8c4d4ec30"},"provenance":null,"requires-python":null,"size":408495,"upload-time":"2016-06-18T17:57:33.655137Z","url":"https://files.pythonhosted.org/packages/d5/1f/638b17eab913d19203ebd721c4e5c726b57bc50def33ab1ec0070fe2ddc2/psutil-4.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"02c042e1f6c68c807de5caf45547971cf02977abf4cd92c8961186af9c91c488"},"provenance":null,"requires-python":null,"size":406900,"upload-time":"2016-06-18T17:57:40.580278Z","url":"https://files.pythonhosted.org/packages/a6/9b/94598ec4041ee2834f7ca21c9f1ff67e8f8cef4376ee9d9b9e3ff6950d05/psutil-4.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"2bcfeff969f3718bb31effea752d92511f375547d337687db1bd99ccd85b7ad7"},"provenance":null,"requires-python":null,"size":406871,"upload-time":"2016-06-18T17:57:47.827133Z","url":"https://files.pythonhosted.org/packages/79/2a/abe407e5b594b2a7c0432f5605579d33ce21e0049e3dc2e5c4f37402f760/psutil-4.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"34736b91c9785ede9d859a79e28129390f609339014f904c276ad46d0a440730"},"provenance":null,"requires-python":null,"size":323883,"upload-time":"2016-06-18T17:57:54.043353Z","url":"https://files.pythonhosted.org/packages/14/5e/072b19d913b3fc1f86b871c2869d19db3b5fa50c2e9f4980ae8646e189e0/psutil-4.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py2.7.exe","hashes":{"sha256":"da6ee62fb5ffda188c39aacb0499d401c131046e922ba53fb4b908937e771f94"},"provenance":null,"requires-python":null,"size":378354,"upload-time":"2016-06-18T17:58:00.735354Z","url":"https://files.pythonhosted.org/packages/7e/85/a60111c14eb80aa7a74e1a0086c1a1bbc62282df0c913730e558d16e6a8c/psutil-4.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.3.exe","hashes":{"sha256":"6f36fa29aa9ca935405a3cebe03cfbc6dde27088f9ad3d9b2d3baa47d4b89914"},"provenance":null,"requires-python":null,"size":373297,"upload-time":"2016-06-18T17:58:08.504160Z","url":"https://files.pythonhosted.org/packages/8e/6e/744b98493947a11625a4c63adb9a148c69c2a6e0cf7ef2afd6212804df1d/psutil-4.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.4.exe","hashes":{"sha256":"63b75c5374fafdaf3f389229b592f19b88a8c7951d1b973b9113df649ade5cb9"},"provenance":null,"requires-python":null,"size":373279,"upload-time":"2016-06-18T17:58:15.187200Z","url":"https://files.pythonhosted.org/packages/87/ce/a3cfd0e1b7d34fccff488a6a2283e5b947841465f81960bd87eee5f78828/psutil-4.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.5.exe","hashes":{"sha256":"6d0e1e9bbdab5a7d40b57d8f0841eb9e11c2395ac7ec0ddd396116240d733f46"},"provenance":null,"requires-python":null,"size":313430,"upload-time":"2016-06-18T17:58:22.261628Z","url":"https://files.pythonhosted.org/packages/eb/23/d01e7eccd76a096b0675ab9d07c8fd2de9fd31bc8f4c92f1a150bc612ad4/psutil-4.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"data-dist-info-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"filename":"psutil-4.3.1-cp27-none-win32.whl","hashes":{"sha256":"b0c5bf0d2a29a6f18ac22e2d24210730dca458c9f961914289c9e027ccb5ae43"},"provenance":null,"requires-python":null,"size":168476,"upload-time":"2016-09-02T13:58:43.429910Z","url":"https://files.pythonhosted.org/packages/2d/70/0b24c7272efbb1d8cac1be1768aabfb8ddb37bdc9ab8a176f6afc7e52b0d/psutil-4.3.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"data-dist-info-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"filename":"psutil-4.3.1-cp27-none-win_amd64.whl","hashes":{"sha256":"fc78c29075e623b6ea1c4a1620a120a1534ee05370b76c0ec96f6d161d79e7a1"},"provenance":null,"requires-python":null,"size":170725,"upload-time":"2016-09-02T13:58:47.944701Z","url":"https://files.pythonhosted.org/packages/67/d4/0403e6bd1cf78bd597ac960a3a6ad36cea6c12e3b413c0a1d43361128fb5/psutil-4.3.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp33-cp33m-win32.whl","hashes":{"sha256":"aa05f44a77ef83773af39446f99e461aa3b6edb7fdabeefdcf06e913d8884d3a"},"provenance":null,"requires-python":null,"size":168384,"upload-time":"2016-09-02T13:58:52.479199Z","url":"https://files.pythonhosted.org/packages/9c/ec/5f3f06012c54de3b4443a6948fc75fe1e348ca3de408b00815b5976b8877/psutil-4.3.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"6b3882eb16f2f40f1da6208a051800abadb1f82a675d9ef6ca7386e1a208b1ad"},"provenance":null,"requires-python":null,"size":170566,"upload-time":"2016-09-02T13:58:56.870812Z","url":"https://files.pythonhosted.org/packages/de/c6/74f8b4d460d89811b4c8fd426b63530222456bad767fe372ebb4f5f207be/psutil-4.3.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp34-cp34m-win32.whl","hashes":{"sha256":"cf1be0b16b38f0e2081ff0c81a1a4321c206a824ba6bd51903fdd440abb370b6"},"provenance":null,"requires-python":null,"size":168375,"upload-time":"2016-09-02T13:59:01.079445Z","url":"https://files.pythonhosted.org/packages/75/ff/d02c907869d5e4cc260ce72eb253f5007a7cdf0b47326d19693f8f937eb0/psutil-4.3.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"afa94bed972722882264a4df06176f6b6e6acc6bcebcc3f1db5428c7271dacba"},"provenance":null,"requires-python":null,"size":170529,"upload-time":"2016-09-02T13:59:05.507453Z","url":"https://files.pythonhosted.org/packages/c9/72/07da416b1dcf258a0cb0587c823e3611392fe29b1fcae6e078b1c254dce5/psutil-4.3.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp35-cp35m-win32.whl","hashes":{"sha256":"d2254f518624e6b2262f0f878931faa4bdbe8a77d1f8826564bc4576c6a4f85e"},"provenance":null,"requires-python":null,"size":170124,"upload-time":"2016-09-02T13:59:09.837050Z","url":"https://files.pythonhosted.org/packages/cd/b0/07b7083a134c43b58515d59f271734034f8ba06840b1f371eaa6b3ab85b2/psutil-4.3.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"3b377bc8ba5e62adbc709a90ea07dce2d4addbd6e1cc7acede61ddfa1c66e00a"},"provenance":null,"requires-python":null,"size":173545,"upload-time":"2016-09-02T13:59:14.368908Z","url":"https://files.pythonhosted.org/packages/7e/7e/17c1467158ccac5dc54986a657420fc194686653cfb6feddb6717a60d17f/psutil-4.3.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.tar.gz","hashes":{"sha256":"38f74182fb9e15cafd0cdf0821098a95cc17301807aed25634a18b66537ba51b"},"provenance":null,"requires-python":null,"size":315878,"upload-time":"2016-09-01T20:56:06.777431Z","url":"https://files.pythonhosted.org/packages/78/cc/f267a1371f229bf16db6a4e604428c3b032b823b83155bd33cef45e49a53/psutil-4.3.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py2.7.exe","hashes":{"sha256":"733210f39e95744da26f2256bc36035fc463b0ae88e91496e97486ba21c63cab"},"provenance":null,"requires-python":null,"size":408715,"upload-time":"2016-09-01T21:31:30.452539Z","url":"https://files.pythonhosted.org/packages/57/44/09ec7b6a3fc1216ecc2182a64f8b6c2eaf8dd0d983fb98ecbf9cecbf54b4/psutil-4.3.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.3.exe","hashes":{"sha256":"4690f720054beff4fc66551a6a34512faff328588dca8e2dbed94398b6941112"},"provenance":null,"requires-python":null,"size":407063,"upload-time":"2016-09-01T21:31:37.405818Z","url":"https://files.pythonhosted.org/packages/f2/b5/2921abbf7779d2f7bb210aa819dd2d86ecd004430c59aef9cc52c2d4057b/psutil-4.3.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.4.exe","hashes":{"sha256":"fd9b66edb9f8943eda6b39e7bb9bff8b14aa8d785f5b417d7a0bfa53d4781a7a"},"provenance":null,"requires-python":null,"size":407028,"upload-time":"2016-09-01T21:31:44.758391Z","url":"https://files.pythonhosted.org/packages/91/ea/d5fb1ef4615c0febab3a347ffbc98d50177878546c22b3291892f41de8f4/psutil-4.3.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.5.exe","hashes":{"sha256":"9ab5b62c6571ce545b1c40b9740af81276bd5d94439fd54de07ed59be0ce3f4f"},"provenance":null,"requires-python":null,"size":777648,"upload-time":"2016-09-01T21:31:56.074229Z","url":"https://files.pythonhosted.org/packages/0d/d9/626030b223140d88176b5dda42ea081b843249af11fa71d9e36a465b4b18/psutil-4.3.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py2.7.exe","hashes":{"sha256":"ad8857923e9bc5802d5559ab5d70c1abc1a7be8e74e779adde883c5391e2061c"},"provenance":null,"requires-python":null,"size":378819,"upload-time":"2016-09-01T21:30:59.795815Z","url":"https://files.pythonhosted.org/packages/02/99/2cc2981b30b2b1fb5b393c921ad17e4baaa1f95d7de527fe54ee222e5663/psutil-4.3.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.3.exe","hashes":{"sha256":"ae20b76cddb3391ea37de5d2aaa1656d6373161bbc8fd868a0ca055194a46e45"},"provenance":null,"requires-python":null,"size":373653,"upload-time":"2016-09-01T21:31:06.076800Z","url":"https://files.pythonhosted.org/packages/06/2f/bbcae933425945d2c9ca3e30e2f35728827e87cedad113808d3f527fe2ea/psutil-4.3.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.4.exe","hashes":{"sha256":"0613437cc28b8721de92c582d5baf742dfa6dd824c84b578f8c49a60077e969a"},"provenance":null,"requires-python":null,"size":373650,"upload-time":"2016-09-01T21:31:12.717399Z","url":"https://files.pythonhosted.org/packages/fb/a6/28bca55c499426202341f8224f66fcd69e6577211201b02605be779f44b1/psutil-4.3.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.5.exe","hashes":{"sha256":"c2031732cd0fb7536af491bb8d8119c9263020a52450f9999c884fd49d346b26"},"provenance":null,"requires-python":null,"size":644701,"upload-time":"2016-09-01T21:31:22.237380Z","url":"https://files.pythonhosted.org/packages/f3/c7/f5da6e76f69c97ecfabe684f4d9e179ceaf9f7c1a74d6148480b1a6c41a8/psutil-4.3.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"data-dist-info-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"filename":"psutil-4.4.0-cp27-none-win32.whl","hashes":{"sha256":"4b907a0bed62a76422eae4e1ed8c8eca25fc21e57a31fc080158b8a300e21dad"},"provenance":null,"requires-python":null,"size":172746,"upload-time":"2016-10-23T14:08:37.739179Z","url":"https://files.pythonhosted.org/packages/8b/00/f7203827a3b2576bf8162ea3ba41e39c3307651218ab08e22e1433d8dc36/psutil-4.4.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"data-dist-info-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"filename":"psutil-4.4.0-cp27-none-win_amd64.whl","hashes":{"sha256":"f60ab95f5e65c420743d5dd5285bda2a6bba6712e9380fb9a5903ea539507326"},"provenance":null,"requires-python":null,"size":175102,"upload-time":"2016-10-23T14:08:40.758084Z","url":"https://files.pythonhosted.org/packages/61/5c/d1f89100973829813e485e7bfc2f203e0d11937f958109819b62e8995b51/psutil-4.4.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp33-cp33m-win32.whl","hashes":{"sha256":"d45919d8b900a9ae03f3d43c489323842d4051cf7a728169f01a3889f50d24ad"},"provenance":null,"requires-python":null,"size":172612,"upload-time":"2016-10-23T14:08:44.422307Z","url":"https://files.pythonhosted.org/packages/34/16/ecc7366a7d023731b8b051cff64d3bb4bed2efa4947b125a3c9031e3d799/psutil-4.4.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"df706e4a8533b43c1a083c2c94e816c7a605487db49e5d49fc64329e0147c0f5"},"provenance":null,"requires-python":null,"size":174980,"upload-time":"2016-10-23T14:08:48.077823Z","url":"https://files.pythonhosted.org/packages/f7/17/6fc596007c243a12cf8c9e81b150f1e7a49c7c85c0544771398f257d5608/psutil-4.4.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp34-cp34m-win32.whl","hashes":{"sha256":"d119281d253bbcc44b491b7b7e5c38802f0933179af97ab228cfd8d072ad1503"},"provenance":null,"requires-python":null,"size":172623,"upload-time":"2016-10-23T14:08:50.867277Z","url":"https://files.pythonhosted.org/packages/50/8f/e8fcfb1bacd02c743545d58514004978318d8476c8bffeee9fecfc9f1f79/psutil-4.4.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"ff8a22a40bf884f52cf8dd86872b5199e7ed58eef1575e837d1d9b668fb2416a"},"provenance":null,"requires-python":null,"size":174974,"upload-time":"2016-10-23T14:08:53.576968Z","url":"https://files.pythonhosted.org/packages/41/8f/31f1fb4bb639ccaff2192b0b2a760118a05fb108e156d4966c81a246071b/psutil-4.4.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp35-cp35m-win32.whl","hashes":{"sha256":"7b7c850b3afe6c7895cfd3ee630492c4aabe644a6723a583cd56bac0222d7cd8"},"provenance":null,"requires-python":null,"size":174516,"upload-time":"2016-10-23T14:08:56.521588Z","url":"https://files.pythonhosted.org/packages/18/9e/702b1da450a13448f9d96f2d9e2f69eed901c56adb01e173d8307278c883/psutil-4.4.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"321d09e39bb4641c98544e51fba598f894d09355a18d14367468723632cb149e"},"provenance":null,"requires-python":null,"size":178046,"upload-time":"2016-10-23T14:08:59.346105Z","url":"https://files.pythonhosted.org/packages/8a/87/a22b95d91fb1a07591ebf373d1c286a16037bf2e321577a083156d7f6a13/psutil-4.4.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.tar.gz","hashes":{"sha256":"f4da111f473dbf7e813e6610aec1329000536aea5e7d7e73ed20bc42cfda7ecc"},"provenance":null,"requires-python":null,"size":1831734,"upload-time":"2016-10-23T14:09:04.179623Z","url":"https://files.pythonhosted.org/packages/fc/63/af9c6a4f2ab48293f60ec204cc1336f6f17c1cb782ffb0275982ac08d663/psutil-4.4.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py2.7.exe","hashes":{"sha256":"184132916ceb845f12f7cced3a2cf5273097d314f44be8357bdc435a6dee49cd"},"provenance":null,"requires-python":null,"size":413868,"upload-time":"2016-10-23T14:09:10.236158Z","url":"https://files.pythonhosted.org/packages/98/d9/407ee7d17b4e78c45b92639bfbd9b376b6a6022b0c63965946bfd74af6f4/psutil-4.4.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.3.exe","hashes":{"sha256":"7bb1c69a062aaff4a1773cca6bbe33b261293318d72246b9519357260b67be1e"},"provenance":null,"requires-python":null,"size":412259,"upload-time":"2016-10-23T14:09:15.072697Z","url":"https://files.pythonhosted.org/packages/3f/cc/b31fbc28247722ae7ece86f8911459e29a08f5075e3d4e794cef20f920b2/psutil-4.4.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.4.exe","hashes":{"sha256":"887eca39665d3de362693d66db9c15bf93313cde261de93908452241ee439e56"},"provenance":null,"requires-python":null,"size":412252,"upload-time":"2016-10-23T14:09:19.006281Z","url":"https://files.pythonhosted.org/packages/32/72/88e9e57964144ac3868f817f77e269f987c74fa51e503eb0fe8040523d28/psutil-4.4.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.5.exe","hashes":{"sha256":"a289ed4f7be6c535aa4c59c997baf327788d564881c8bf08fee502fab0cdc7cb"},"provenance":null,"requires-python":null,"size":782926,"upload-time":"2016-10-23T14:09:24.493051Z","url":"https://files.pythonhosted.org/packages/45/09/8908f5931a78c90bf2ef7f91a37a752a13d4017347ac2765d0cbb89e82f6/psutil-4.4.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py2.7.exe","hashes":{"sha256":"8a18d0527ac339f5501f0bd5471040d5fb83052453eedf13106feaf99ddef6cb"},"provenance":null,"requires-python":null,"size":383869,"upload-time":"2016-10-23T14:09:28.382095Z","url":"https://files.pythonhosted.org/packages/4a/eb/a131c438621822833e3131c4998741c3b68fb6ff67786d25e9398439a640/psutil-4.4.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.3.exe","hashes":{"sha256":"8700a0373fd0c95c79ad79bc08e0a7b76cabc0c463658cbe398600c69d57293d"},"provenance":null,"requires-python":null,"size":378663,"upload-time":"2016-10-23T14:09:32.358639Z","url":"https://files.pythonhosted.org/packages/be/c8/fd76b7ba3e1cc61d586c5779d9b0a45d1ad020e1de677eb344d2583bd0f1/psutil-4.4.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.4.exe","hashes":{"sha256":"ddd3f76486a366f1dc7adc8a9e8a285ec24b3b213912b8dce0cbbb629f954b8c"},"provenance":null,"requires-python":null,"size":378674,"upload-time":"2016-10-23T14:09:35.849351Z","url":"https://files.pythonhosted.org/packages/ce/0c/efbe5aefdd136bb3e2fc292b0506761a9b04c215d295be07e11263a90923/psutil-4.4.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.5.exe","hashes":{"sha256":"28963853709f6e0958edd8d6d7469152098680800deb98a759a67745d65b164b"},"provenance":null,"requires-python":null,"size":649869,"upload-time":"2016-10-23T14:09:40.045794Z","url":"https://files.pythonhosted.org/packages/fa/b6/b97b6d4dcbc87b91db9fd2753a0c3b999b734595a97155a38003be475a6d/psutil-4.4.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"data-dist-info-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"filename":"psutil-4.4.1-cp27-none-win32.whl","hashes":{"sha256":"7e77ec1a9c75a858781c1fb46fe81c999e1ae0e711198b4aaf59e5f5bd373b11"},"provenance":null,"requires-python":null,"size":172150,"upload-time":"2016-10-25T15:16:10.656877Z","url":"https://files.pythonhosted.org/packages/a9/aa/ff4e5602420cda42e9ff12949eae95abf7d6838fc79d7892b29af416f4c2/psutil-4.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"data-dist-info-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"filename":"psutil-4.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"0f7f830db35c1baeb0131b2bba458b77f7db98944b2fedafc34922168e467d09"},"provenance":null,"requires-python":null,"size":174507,"upload-time":"2016-10-25T15:16:16.530462Z","url":"https://files.pythonhosted.org/packages/62/e8/c3d3e4161e29bd86d6b06d16456defcb114fd74693b15e5692df9e2b611e/psutil-4.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp33-cp33m-win32.whl","hashes":{"sha256":"fa4ad0533adef033bbcbac5e20d06e77f9aadf5d9c1596317d1b668f94b01b99"},"provenance":null,"requires-python":null,"size":172015,"upload-time":"2016-10-25T15:16:19.320410Z","url":"https://files.pythonhosted.org/packages/2c/da/0428f61f2c5eda7c5053ae35a2d57b097e4eb15e48f9d222319df7f4cbd3/psutil-4.4.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"af337d186b07249b86f00a71b4cf6fcfa1964484fe5fb8a7b623f4559c2859c9"},"provenance":null,"requires-python":null,"size":174384,"upload-time":"2016-10-25T15:16:22.634310Z","url":"https://files.pythonhosted.org/packages/da/ce/d36c39da6d387fbfaea68f382319887315e9201e885ef93c418b89209b31/psutil-4.4.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp34-cp34m-win32.whl","hashes":{"sha256":"4e5cb45c9616dd855c07e538f523c838705ce7c24e021e645cdce4c7894e7209"},"provenance":null,"requires-python":null,"size":172026,"upload-time":"2016-10-25T15:16:28.801885Z","url":"https://files.pythonhosted.org/packages/64/63/49aeca2d1a20f5c5203302a25c825be9262f81d8bf74d7d9e0bf0789b189/psutil-4.4.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"0876646748f3db5e1678a94ae3f68dcef3bd51e82b34f06109e6a28bcddc266c"},"provenance":null,"requires-python":null,"size":174374,"upload-time":"2016-10-25T15:16:32.462229Z","url":"https://files.pythonhosted.org/packages/e0/b2/48a62d3204a714b6354105ed540d54d1e272d5b230bc35eddc14a1494dd1/psutil-4.4.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp35-cp35m-win32.whl","hashes":{"sha256":"e4e9033ef5d775ef8a522750688161241e79c7d4669a05784a0a1a8d37dc8c3c"},"provenance":null,"requires-python":null,"size":173920,"upload-time":"2016-10-25T15:16:38.218083Z","url":"https://files.pythonhosted.org/packages/95/e6/d6bad966efeb674bb3357c94f3e89bbd3477a24f2635c3cadc1d0f2aea76/psutil-4.4.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d8464100c62932eeeacce2be0a1041f68b3bfcc7be261cf9486c11ee98eaedd2"},"provenance":null,"requires-python":null,"size":177448,"upload-time":"2016-10-25T15:16:42.121191Z","url":"https://files.pythonhosted.org/packages/00/58/6845a2a2f6fbbc56a58f0605744368164bf68889299f134684cc1478baf6/psutil-4.4.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.tar.gz","hashes":{"sha256":"9da43dbf7c08f5c2a8e5e2c8792f5c438f52435677b1334e9653d23ea028f4f7"},"provenance":null,"requires-python":null,"size":1831794,"upload-time":"2016-10-25T15:16:47.508715Z","url":"https://files.pythonhosted.org/packages/e5/f3/b816daefa9a6757f867f81903f40849dcf0887f588793236b476e6a30ded/psutil-4.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"23ac3e7b6784751ceeb4338d54b0e683c955cd22b86acbc089696aeb0717ab75"},"provenance":null,"requires-python":null,"size":408863,"upload-time":"2016-10-25T15:16:52.293269Z","url":"https://files.pythonhosted.org/packages/88/93/3c8434bd64a5f89f6b0cc58e1394e88dc3c272f83f1877afac9ede222c63/psutil-4.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"8b97ece77d2dce49dd6adbfc0c9bfb7820be4460d00732bb8cf18b77b9ffb07f"},"provenance":null,"requires-python":null,"size":407253,"upload-time":"2016-10-25T15:16:57.006499Z","url":"https://files.pythonhosted.org/packages/50/21/6b462342ab2f091bed70898894322c107493d69e24a834b9f2ab93a1e876/psutil-4.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"a0b5976a5b5a781754b6cebe89f4e21413b04b7f4005f5713f65257bee29be5f"},"provenance":null,"requires-python":null,"size":407244,"upload-time":"2016-10-25T15:17:01.612657Z","url":"https://files.pythonhosted.org/packages/09/94/a7c3c875884adef49f52c19faff68bfd2b04160a5eb0489dbffa60b5c2dc/psutil-4.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.5.exe","hashes":{"sha256":"b0bfe16b9bd095e56b8ec481328ba64094ccd615ee1f789a12a13cfd7bc5e34a"},"provenance":null,"requires-python":null,"size":777920,"upload-time":"2016-10-25T15:17:05.774786Z","url":"https://files.pythonhosted.org/packages/09/bf/3d850244db8e6aa94116db6787edee352692bbb9647e31c056189e6742cc/psutil-4.4.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py2.7.exe","hashes":{"sha256":"392877b5f86977b4a747fb61968e3b44df62c1d6d87536f021c4979b601c68f5"},"provenance":null,"requires-python":null,"size":378863,"upload-time":"2016-10-25T15:17:10.258103Z","url":"https://files.pythonhosted.org/packages/ed/f9/ed4cc19c84086149067d03de7cfb9f3213cf02d58b6f75c2ac16f85e0baf/psutil-4.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.3.exe","hashes":{"sha256":"a571f179f29215f0aa710ec495573a3522f24e8a8ba0be67d094ad6f593fba05"},"provenance":null,"requires-python":null,"size":373657,"upload-time":"2016-10-25T15:17:17.117641Z","url":"https://files.pythonhosted.org/packages/e6/ad/d8ec058821191334cf17db08c32c0308252d7b3c7490417690061dd12096/psutil-4.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.4.exe","hashes":{"sha256":"c2614dc6194cafd74c147e4b95febe8778430c2ecb91197ad140bf45e3e3ada7"},"provenance":null,"requires-python":null,"size":373668,"upload-time":"2016-10-25T15:17:20.392896Z","url":"https://files.pythonhosted.org/packages/26/15/6d03a2171262a1be565d06c87d584c10daa51e787c7cdd2ff932a433419e/psutil-4.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.5.exe","hashes":{"sha256":"5915da802a1648d0baeeee06b0d1a87eb0e4e20654f65b8428a6f4cc8cd32caf"},"provenance":null,"requires-python":null,"size":644863,"upload-time":"2016-10-25T15:17:24.141533Z","url":"https://files.pythonhosted.org/packages/d1/91/89db11eaa91c1de34d77600c47d657fce7b60e847bcf6457dddfc23b387c/psutil-4.4.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"data-dist-info-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"filename":"psutil-4.4.2-cp27-none-win32.whl","hashes":{"sha256":"15aba78f0262d7839702913f5d2ce1e97c89e31456bb26da1a5f9f7d7fe6d336"},"provenance":null,"requires-python":null,"size":172169,"upload-time":"2016-10-26T11:01:25.916580Z","url":"https://files.pythonhosted.org/packages/d7/da/b7895f01868977c9205bf1c8ff6a88290ec443535084e206b7db7f33918f/psutil-4.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"data-dist-info-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"filename":"psutil-4.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"69e30d789c495b781f7cd47c13ee64452c58abfc7132d6dd1b389af312a78239"},"provenance":null,"requires-python":null,"size":174525,"upload-time":"2016-10-26T11:01:29.001553Z","url":"https://files.pythonhosted.org/packages/0c/76/f50742570195a9a13e26ee3e3e32575a9315df90408056c991af85d23792/psutil-4.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp33-cp33m-win32.whl","hashes":{"sha256":"e44d6b758a96539e3e02336430d3f85263d43c470c5bad93572e9b6a86c67f76"},"provenance":null,"requires-python":null,"size":172034,"upload-time":"2016-10-26T11:01:31.931360Z","url":"https://files.pythonhosted.org/packages/f1/58/64db332e3a665d0d0260fcfe39ff8609871a1203ed11afcff0e5991db724/psutil-4.4.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"c2b0d8d1d8b5669b9884d0dd49ccb4094d163858d672d3d13a3fa817bc8a3197"},"provenance":null,"requires-python":null,"size":174401,"upload-time":"2016-10-26T11:01:35.879864Z","url":"https://files.pythonhosted.org/packages/39/31/4b5a2ab9d20d14fa6c821b7fbbf69ae93708e1a4ba335d4e2ead549b2f59/psutil-4.4.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp34-cp34m-win32.whl","hashes":{"sha256":"10fbb631142a3200623f4ab49f8bf82c32b79b8fe179f6056d01da3dfc589da1"},"provenance":null,"requires-python":null,"size":172044,"upload-time":"2016-10-26T11:01:38.950793Z","url":"https://files.pythonhosted.org/packages/67/f6/f034ad76faa7b9bd24d70a663b5dca8d71b6b7ddb4a3ceecd6a0a63afe92/psutil-4.4.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"e423dd9cb12256c742d1d56ec38bc7d2a7fa09287c82c41e475e68b9f932c2af"},"provenance":null,"requires-python":null,"size":174393,"upload-time":"2016-10-26T11:01:41.933150Z","url":"https://files.pythonhosted.org/packages/f8/f2/df2d8de978236067d0fce847e3734c6e78bfd57df9db5a2786cc544ca85f/psutil-4.4.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp35-cp35m-win32.whl","hashes":{"sha256":"7481f299ae0e966a10cb8dd93a327efd8f51995d9bdc8810dcc65d3b12d856ee"},"provenance":null,"requires-python":null,"size":173938,"upload-time":"2016-10-26T11:01:44.870925Z","url":"https://files.pythonhosted.org/packages/57/2d/0cdc3fdb9853ec1d815e0948104f1c51894f47c8ee2152c019ba754ce678/psutil-4.4.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d96d31d83781c7f3d0df8ccb1cc50650ca84d4722c5070b71ce8f1cc112e02e0"},"provenance":null,"requires-python":null,"size":177465,"upload-time":"2016-10-26T11:01:47.925426Z","url":"https://files.pythonhosted.org/packages/cd/50/3c238d67e025701fa18cc37ed445327efcf14ea4d06e357eef92d1a250bf/psutil-4.4.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.tar.gz","hashes":{"sha256":"1c37e6428f7fe3aeea607f9249986d9bb933bb98133c7919837fd9aac4996b07"},"provenance":null,"requires-python":null,"size":1832052,"upload-time":"2016-10-26T11:01:53.029413Z","url":"https://files.pythonhosted.org/packages/6c/49/0f784a247868e167389f6ac76b8699b2f3d6f4e8e85685dfec43e58d1ed1/psutil-4.4.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py2.7.exe","hashes":{"sha256":"11a20c0328206dce68f8da771461aeaef9c44811e639216fd935837e758632dc"},"provenance":null,"requires-python":null,"size":408884,"upload-time":"2016-10-26T11:01:57.830427Z","url":"https://files.pythonhosted.org/packages/58/bc/ac2a052035e945153f0bdc23d7c169e7e64e0729af6ff96f85d960962def/psutil-4.4.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.3.exe","hashes":{"sha256":"642194ebefa573de62406883eb33868917bab2cc2e21b68d551248e194dd0b0a"},"provenance":null,"requires-python":null,"size":407275,"upload-time":"2016-10-26T11:02:00.968973Z","url":"https://files.pythonhosted.org/packages/38/b3/c8244a4858de5f9525f176d72cb06584433446a5b9d7272d5eec7a435f81/psutil-4.4.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.4.exe","hashes":{"sha256":"c02b9fb5f1f3c857938b26a73b1ca92007e8b0b2fd64693b29300fae0ceaf679"},"provenance":null,"requires-python":null,"size":407267,"upload-time":"2016-10-26T11:02:04.291434Z","url":"https://files.pythonhosted.org/packages/cd/4d/8f447ecba1a3d5cae9f3bdd8395b45e2e6d07834a05fa3ce58c766ffcea6/psutil-4.4.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.5.exe","hashes":{"sha256":"6c40dc16b579f645e1804341322364203d0b21045747e62e360fae843d945e20"},"provenance":null,"requires-python":null,"size":777940,"upload-time":"2016-10-26T11:02:10.124058Z","url":"https://files.pythonhosted.org/packages/73/6e/1c58cbdedcf4e2226f8ca182c947e29f29dba6965fdae0aa8e6a361365ce/psutil-4.4.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py2.7.exe","hashes":{"sha256":"c353ecc62e67bf7c7051c087670d49eae9472f1b30bb1623d667b0cd137e8934"},"provenance":null,"requires-python":null,"size":378885,"upload-time":"2016-10-26T11:02:16.586357Z","url":"https://files.pythonhosted.org/packages/37/27/9eba3e29f8291be103a8e5e44c06eefe5357217e9d8d294ff154dc3bff90/psutil-4.4.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.3.exe","hashes":{"sha256":"7106cb3722235ccb6fe4b18c51f60a548d4b111ec2d209abdcd3998661f4593a"},"provenance":null,"requires-python":null,"size":373678,"upload-time":"2016-10-26T11:02:20.035049Z","url":"https://files.pythonhosted.org/packages/62/25/189e73856fa3172d5e785404eb9cf4561b01094363cc39ec267399e401c4/psutil-4.4.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.4.exe","hashes":{"sha256":"de1f53fe955dfba562f7791f72517935010a2e88f9caad36917e8c5c03de9051"},"provenance":null,"requires-python":null,"size":373689,"upload-time":"2016-10-26T11:02:23.073976Z","url":"https://files.pythonhosted.org/packages/10/4a/e50ca35162538645c4d98816757763e4aaaac1ccaf055b39265acb8739e1/psutil-4.4.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.5.exe","hashes":{"sha256":"2eb123ca86057ed4f31cfc9880e098ee7a8e19c7ec02b068c45e7559ae7539a6"},"provenance":null,"requires-python":null,"size":644886,"upload-time":"2016-10-26T11:02:26.650470Z","url":"https://files.pythonhosted.org/packages/c5/97/018f0580bc3d3d85f1e084e683e904bc33cd58ca96e0ecb5ae6f346d6452/psutil-4.4.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"data-dist-info-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"filename":"psutil-5.0.0-cp27-none-win32.whl","hashes":{"sha256":"cc2560b527cd88a9bc062ee4bd055c40b9fc107e37db01997422c75a3f94efe9"},"provenance":null,"requires-python":null,"size":175039,"upload-time":"2016-11-06T18:28:50.896397Z","url":"https://files.pythonhosted.org/packages/bb/82/efee0c83eab6ad3dd4a70b2b91013a50776ff6253f9b71d825914e693dfa/psutil-5.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"data-dist-info-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"filename":"psutil-5.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"8a6cbc7165a476d08a89ee3078a74d111729cf515fd831db9f635012e56f9759"},"provenance":null,"requires-python":null,"size":177395,"upload-time":"2016-11-06T18:28:54.500906Z","url":"https://files.pythonhosted.org/packages/16/bf/09cb9f5286e5037eba1d5fe347c0f622b325e068757f1072c56f9b130cc3/psutil-5.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp33-cp33m-win32.whl","hashes":{"sha256":"9b0f13e325f007a0fd04e9d44cfdb5187c0b3e144f89533324dd9f74c25bd9ec"},"provenance":null,"requires-python":null,"size":174954,"upload-time":"2016-11-06T18:28:57.539707Z","url":"https://files.pythonhosted.org/packages/a2/5d/82b85eb4c24c82064c5c216f52df92f7861ff0a02d3cbcfebfacadc2b28f/psutil-5.0.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"ade8924028b2c23cc9ffe4a0737de38c668d50be5ce19495790154f530ce5389"},"provenance":null,"requires-python":null,"size":177229,"upload-time":"2016-11-06T18:29:00.558809Z","url":"https://files.pythonhosted.org/packages/14/5b/00b74d0ada625090dfaba6244860a9ec4df2e79a6c8440dd490e46c65543/psutil-5.0.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp34-cp34m-win32.whl","hashes":{"sha256":"af01b73fd66f138e06f804508fd33118823fd2abb89f105ae2b99efa4c8fd1a3"},"provenance":null,"requires-python":null,"size":174957,"upload-time":"2016-11-06T18:29:04.527838Z","url":"https://files.pythonhosted.org/packages/fd/e6/5a08cd23cefadd22b8e74d0bd1c525897c9db4dc1d2f44716dee848b8fe3/psutil-5.0.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"846925435e69cc7b802cd7d1a4bd640e180d0db15277c24e196d3a5799bf6760"},"provenance":null,"requires-python":null,"size":177189,"upload-time":"2016-11-06T18:29:08.003063Z","url":"https://files.pythonhosted.org/packages/b4/11/f68bad0d5fc08800006d2355c941e186315987456158a9b06f40e483d4e7/psutil-5.0.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp35-cp35m-win32.whl","hashes":{"sha256":"d7885ff254425c64bcc2dbff256ec1367515c15218bfda0fd3d799092437d908"},"provenance":null,"requires-python":null,"size":176812,"upload-time":"2016-11-06T18:29:11.700563Z","url":"https://files.pythonhosted.org/packages/c4/16/84213b90c78e437eff09285138947d12105ea982cb6f8fda4ab2855014e6/psutil-5.0.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e35cb38037973ff05bc52dac4f382d17028104d77f0bb51792de93f359046902"},"provenance":null,"requires-python":null,"size":180171,"upload-time":"2016-11-06T18:29:15.915700Z","url":"https://files.pythonhosted.org/packages/98/ef/40582d2d3e39fdcc202a33ba6aab15f6ccb36cdcd04f6756cc9afe30df30/psutil-5.0.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"964e9db2641e3be6a80b5a3135f7a9425f87d8416342b4d967202e4854f3eeba"},"provenance":null,"requires-python":null,"size":411778,"upload-time":"2016-11-06T18:29:32.458135Z","url":"https://files.pythonhosted.org/packages/74/f0/e8964d58e12c7716775157821de2e758fece2581bc9f3b7c333a4de29b90/psutil-5.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"d7b933193322523314b0b2d699a43e43a70f43016f73782dfd892bc7ee95ecd1"},"provenance":null,"requires-python":null,"size":410125,"upload-time":"2016-11-06T18:29:37.513499Z","url":"https://files.pythonhosted.org/packages/6e/a9/4438ec3289e3925a638cb079039b748ee4fb7e3ba18a93baed39f4fcd11e/psutil-5.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"8709a5057d42d5e55f3940bb1e70370316defb3da03ad869342755b5a2c17c78"},"provenance":null,"requires-python":null,"size":410086,"upload-time":"2016-11-06T18:29:41.067409Z","url":"https://files.pythonhosted.org/packages/db/85/c2e27aab6db3c4d404f0e81762f5a962b7213caed0a789cb5df2990ac489/psutil-5.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.5.exe","hashes":{"sha256":"c065eaf76a5423341f511e732f1a17e75a55dc4aceee9a321a619a5892aec18f"},"provenance":null,"requires-python":null,"size":780670,"upload-time":"2016-11-06T18:29:45.597507Z","url":"https://files.pythonhosted.org/packages/27/fd/12a4ef4a9940331a5521216e839444dcab0addd3a49fb90c940bdc40e46e/psutil-5.0.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py2.7.exe","hashes":{"sha256":"8eca28511d493209f59fe99cebfb8ecc65b8d6691f8a80fa3ab50dbb4994c81b"},"provenance":null,"requires-python":null,"size":381779,"upload-time":"2016-11-06T18:29:49.728094Z","url":"https://files.pythonhosted.org/packages/bc/77/457ddeac355fe88c8d4ee8062ce66ab5dd142f5c9423d1bfd10a568c7884/psutil-5.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.3.exe","hashes":{"sha256":"7a8e7654789c468d2a6c32508638563741046f138405fea2a4427a9228ac86f4"},"provenance":null,"requires-python":null,"size":376623,"upload-time":"2016-11-06T18:29:53.256127Z","url":"https://files.pythonhosted.org/packages/8d/1a/2f61de7f89602dfd975f961bf057522ff53ef3bdc48610dbdcf306b542df/psutil-5.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.4.exe","hashes":{"sha256":"4db1f1d8655a63832c9ab85d77c969ab95b740128ca9053b5c108a1e5efe6a7c"},"provenance":null,"requires-python":null,"size":376624,"upload-time":"2016-11-06T18:29:56.669027Z","url":"https://files.pythonhosted.org/packages/c1/3f/d8921d1a8672545a390fd874b77c9a30444b64ab5f8c48e2e9d971f4e98f/psutil-5.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.5.exe","hashes":{"sha256":"10f05841e3bf7b060b3779367b9cd953a6c97303c08a0e2630a9e461ce124412"},"provenance":null,"requires-python":null,"size":647782,"upload-time":"2016-11-06T18:30:00.475783Z","url":"https://files.pythonhosted.org/packages/ec/53/dc7d9e33e77efd26f7c4a9ba8d1e23ba9ce432077e16a55f1f755970e7f7/psutil-5.0.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.zip","hashes":{"sha256":"5411e22c63168220f4b8cc42fd05ea96f5b5e65e08b93b675ca50653aea482f8"},"provenance":null,"requires-python":null,"size":374074,"upload-time":"2016-11-06T18:41:47.892123Z","url":"https://files.pythonhosted.org/packages/93/7f/347309562d30c688299727e65f4d76ef34180c406dfb6f2c7b6c8d746e13/psutil-5.0.0.zip","yanked":false},{"core-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"data-dist-info-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"filename":"psutil-5.0.1-cp27-none-win32.whl","hashes":{"sha256":"1f2379809f2182652fc740faefa511f78b5975e6471e5fa419882dd9e082f245"},"provenance":null,"requires-python":null,"size":176737,"upload-time":"2016-12-21T01:35:18.007609Z","url":"https://files.pythonhosted.org/packages/cd/2d/ecb32ce765c52780768e4078d4d2916bdaa790b313c454f52aebe27d559d/psutil-5.0.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"data-dist-info-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"filename":"psutil-5.0.1-cp27-none-win_amd64.whl","hashes":{"sha256":"86d67da7bfed474b5b0d545b29211f16d622e87c4089de5ea41a3fbcfc4872c7"},"provenance":null,"requires-python":null,"size":179100,"upload-time":"2016-12-21T01:35:20.364405Z","url":"https://files.pythonhosted.org/packages/6b/7e/68835630dcc765452b76b61936f75c1ef71fceae8ebaa7ecb27e42bf3f28/psutil-5.0.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp33-cp33m-win32.whl","hashes":{"sha256":"6a95283fd048810811cf971bec5cec3998e1e62e66237ef7a41a42dd0da29f8c"},"provenance":null,"requires-python":null,"size":176612,"upload-time":"2016-12-21T01:35:22.141278Z","url":"https://files.pythonhosted.org/packages/43/46/3fad73a8c581f63f08fb1f93ef5651e939caa5601a50baca7f2af3c54e2b/psutil-5.0.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"8f492b531c4321c7c43ef82b60421f4bcf5ded4ba4e13f534c064ad6c2d910ed"},"provenance":null,"requires-python":null,"size":178930,"upload-time":"2016-12-21T01:35:23.896324Z","url":"https://files.pythonhosted.org/packages/b0/d0/6d9e39115aaab0d8d52422b909e365938a6dcebe5fa04fac3a6b70398aee/psutil-5.0.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp34-cp34m-win32.whl","hashes":{"sha256":"5cc1b91d4848453b74ad8e63275a19e784ef3acd943c3627134a607b602bc31d"},"provenance":null,"requires-python":null,"size":176623,"upload-time":"2016-12-21T01:35:26.391755Z","url":"https://files.pythonhosted.org/packages/31/21/d836f37c8fde1760e889b0ace67e151c6e56f2ce819b227715a5f291452f/psutil-5.0.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"55874a1814faceaa090b7fa7addbf350603fd7042562adaae13eb6e46d3ec907"},"provenance":null,"requires-python":null,"size":178917,"upload-time":"2016-12-21T01:35:28.280103Z","url":"https://files.pythonhosted.org/packages/a1/9c/cb3f68be56ab366dbf4c901002d281e30c50cf951377ca8155c0ad304394/psutil-5.0.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp35-cp35m-win32.whl","hashes":{"sha256":"7f2d7b97faa524f75736dfde418680eff332f4be66d6217b67d09c630c90d02e"},"provenance":null,"requires-python":null,"size":178514,"upload-time":"2016-12-21T01:35:30.152277Z","url":"https://files.pythonhosted.org/packages/f7/f1/dd823eab436db1eac1ca2c8364918786584db996dc4d2bef5a3b0ebd7619/psutil-5.0.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bf4072d4d188802505b9229ec00e141083c127bb19a6c5636c62b0daabda4bd5"},"provenance":null,"requires-python":null,"size":181835,"upload-time":"2016-12-21T01:35:32.091445Z","url":"https://files.pythonhosted.org/packages/2a/3a/107a964dc66cb4f22af9e919dde846b73c0b4ff735b10ede680895ed296c/psutil-5.0.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp36-cp36m-win32.whl","hashes":{"sha256":"786bbbeb3ea98d82ff5cedc86b640bad97bff435c819f26bddaa388da58d47da"},"provenance":null,"requires-python":null,"size":178650,"upload-time":"2017-01-15T18:27:34.549244Z","url":"https://files.pythonhosted.org/packages/c0/a1/c5d0aa766323b150dce5e51a5360542ccb75c109723d2d95e70d8cb248b8/psutil-5.0.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"544c0803760995fe42a2b7050cfd6ed32379b09ce6cd7a1eaf89c221d8669cc3"},"provenance":null,"requires-python":null,"size":181970,"upload-time":"2017-01-15T18:27:39.376642Z","url":"https://files.pythonhosted.org/packages/c0/e4/bf1059c5dd55abf65fe2ac92a0b24a92d09644a9bbc17f1ad2b497ae9d7d/psutil-5.0.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.tar.gz","hashes":{"sha256":"9d8b7f8353a2b2eb6eb7271d42ec99d0d264a9338a37be46424d56b4e473b39e"},"provenance":null,"requires-python":null,"size":326693,"upload-time":"2016-12-21T01:35:34.145966Z","url":"https://files.pythonhosted.org/packages/d9/c8/8c7a2ab8ec108ba9ab9a4762c5a0d67c283d41b13b5ce46be81fdcae3656/psutil-5.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"0c54e3b7cdc0dbb8a19b58c3eb3845a5f9f48d3be2b06ed9aa1e553db8f9db74"},"provenance":null,"requires-python":null,"size":414521,"upload-time":"2016-12-21T01:35:36.513939Z","url":"https://files.pythonhosted.org/packages/3d/90/61f2860cd2c39a4381e415897b7ff94aab21c9aca38690d0c99c1c83b85f/psutil-5.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"0fccb19631d555998fc8c98840c83678244f873486d20d5f24ebb0ac8e19d2f1"},"provenance":null,"requires-python":null,"size":412871,"upload-time":"2016-12-21T01:35:38.959549Z","url":"https://files.pythonhosted.org/packages/05/f7/6ba6efe79620bcaa9af41e0a8501e477815edccc0cad4bd379a8e8bea89c/psutil-5.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"77fde4936f26080aa14b89d292b3ebefabb80be69ef407352cbad6d2ff6882d4"},"provenance":null,"requires-python":null,"size":412858,"upload-time":"2016-12-21T01:35:42.062765Z","url":"https://files.pythonhosted.org/packages/c7/b3/3b83e69b5d96ed052b45f1fe88882b3976a3a066c0497597d3ce1407f17f/psutil-5.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.5.exe","hashes":{"sha256":"a9125b7bc12127174cf7974444ca2b39a3722f59ead1d985053e7358a3d29acd"},"provenance":null,"requires-python":null,"size":783380,"upload-time":"2016-12-21T01:35:44.660601Z","url":"https://files.pythonhosted.org/packages/4e/cf/d1004fa0617f6d6e39adf48d242f76d5d43ecd497f46add2abb513b6ffcd/psutil-5.0.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.6.exe","hashes":{"sha256":"9a3f1413b24b6751e97e51284761e1778ec0cd0a456595edad6c2f7c115b3368"},"provenance":null,"requires-python":null,"size":783511,"upload-time":"2017-01-15T18:25:42.785800Z","url":"https://files.pythonhosted.org/packages/a0/94/1f29e03230504dafeb128a4c6e381606b038ce5d4dc9b8731632eb6b0928/psutil-5.0.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py2.7.exe","hashes":{"sha256":"671c7b2d3fa8deffb879e9cb6cc00d83d1d990bc81f0c487576b70e811f102bf"},"provenance":null,"requires-python":null,"size":384515,"upload-time":"2016-12-21T01:35:47.208014Z","url":"https://files.pythonhosted.org/packages/2d/7c/85aa8efb36c4fcac4a845b9cfc0744c502620394cdb81db574dd20f7dda0/psutil-5.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.3.exe","hashes":{"sha256":"603bd19426b59762ad55ba5a0b8237282868addf2d0930e21b4dca79fc188787"},"provenance":null,"requires-python":null,"size":379325,"upload-time":"2016-12-21T01:35:49.642792Z","url":"https://files.pythonhosted.org/packages/1c/06/201f913389b920c24153fbfdc07c7a451653b00688d6c2ba92428b9a2c23/psutil-5.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.4.exe","hashes":{"sha256":"08e28b4dba54dd6b0d84718c988239c305e58c7f93160496661955de4f6cfe13"},"provenance":null,"requires-python":null,"size":379338,"upload-time":"2016-12-21T01:35:52.592934Z","url":"https://files.pythonhosted.org/packages/58/27/a42b1d12c201880822fd92588321e489c869f4b72eb906f8f79745bee6cb/psutil-5.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.5.exe","hashes":{"sha256":"1e71dd9aa041a7fa55f3bcc78b578e84f31004e6ce9df97229f05c60529fedb1"},"provenance":null,"requires-python":null,"size":650529,"upload-time":"2016-12-21T01:35:55.051616Z","url":"https://files.pythonhosted.org/packages/fb/c6/c60717f25c8ccfddabd57109d03580d24542eedde8d23a673328876b4137/psutil-5.0.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.6.exe","hashes":{"sha256":"c879310328c0c248331dffeea4adbe691fad7b7095cf9c2ac0a4d78a09cd8a17"},"provenance":null,"requires-python":null,"size":650663,"upload-time":"2017-01-15T18:25:51.768008Z","url":"https://files.pythonhosted.org/packages/00/09/04898f2ce604af89d7ca6866632c05a1fb0fbd7b5ea0c7e84a62570c2678/psutil-5.0.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"data-dist-info-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"filename":"psutil-5.1.0-cp27-cp27m-win32.whl","hashes":{"sha256":"85524e46a1c0c7f5274640809deb96c7cdb5133a0eb805bbed0a1f825c8de77e"},"provenance":null,"requires-python":null,"size":183880,"upload-time":"2017-02-01T18:27:27.314031Z","url":"https://files.pythonhosted.org/packages/d9/bf/eae1c0aa2a5a01456e78d5df4c7cefc512854a24f43103034180ff5ea92f/psutil-5.1.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"data-dist-info-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"filename":"psutil-5.1.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"0a64f395295aafe7c22d6cde81076151bf80f26e180e50c00c1cde5857cad224"},"provenance":null,"requires-python":null,"size":186335,"upload-time":"2017-02-01T18:27:35.043588Z","url":"https://files.pythonhosted.org/packages/f6/5d/67120bb4cea2432d333cdf0ab406336ef62122caacb2379e73a88a9cde82/psutil-5.1.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp33-cp33m-win32.whl","hashes":{"sha256":"bd432606d09722220f8f492223e8b69343d30f21e7ed193712632e4fdcf87af2"},"provenance":null,"requires-python":null,"size":183793,"upload-time":"2017-02-01T18:27:43.056910Z","url":"https://files.pythonhosted.org/packages/26/35/ec223413c9301d8eeecf3d319d71a117dbd2847b628b079f3757d5ec693d/psutil-5.1.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"f98c37506e5abfeb2c857794ccfbce86b376cf6344210ea55f41b0ced5fd8d98"},"provenance":null,"requires-python":null,"size":186185,"upload-time":"2017-02-01T18:27:49.896261Z","url":"https://files.pythonhosted.org/packages/9f/20/bb95f30c99837f3c41a8a595752d9cc1afe595f160d0612eea6e4a905de9/psutil-5.1.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp34-cp34m-win32.whl","hashes":{"sha256":"8760214ce01f71cd6766527396607cf2b3b41e6f8a34aaa9f716090750fd9925"},"provenance":null,"requires-python":null,"size":183803,"upload-time":"2017-02-01T18:27:56.955786Z","url":"https://files.pythonhosted.org/packages/0b/f2/90a7288b36a18fb522e00116f7f5c657df9dba3a12636d2fc9bcef97f524/psutil-5.1.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"2309cf46f5db0a78f52e56e06cbb258fe76d3d54cf8ab9fa067b4c26cd722541"},"provenance":null,"requires-python":null,"size":186156,"upload-time":"2017-02-01T18:28:03.790958Z","url":"https://files.pythonhosted.org/packages/51/c6/3757b2bea37edc9889fe85e2ca9b8a2c47b3b758d35f65704f2a5ac5c13e/psutil-5.1.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp35-cp35m-win32.whl","hashes":{"sha256":"6d89d5f68433afe6755bd26ae3ea4587f395c7613d7be5bc4f0f97b1e299228a"},"provenance":null,"requires-python":null,"size":185696,"upload-time":"2017-02-01T18:28:10.369149Z","url":"https://files.pythonhosted.org/packages/30/9c/322df29c490ceb97d8cbbc663424fad94e900a1921d03cc9c61e36f17444/psutil-5.1.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b3c15965b2dfad34ea79494f82007e71b02e1c5352b551d00647fb2be9deabe1"},"provenance":null,"requires-python":null,"size":189130,"upload-time":"2017-02-01T18:28:17.197543Z","url":"https://files.pythonhosted.org/packages/b3/3a/37d6bf7dedf263e1119b03aaee8565182bf2577ed5ecd7975e253e242bd9/psutil-5.1.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp36-cp36m-win32.whl","hashes":{"sha256":"d32551c242c041b0506d13cfd271db56ba57323283b1f952b858505824f3e82a"},"provenance":null,"requires-python":null,"size":185695,"upload-time":"2017-02-01T18:28:25.059542Z","url":"https://files.pythonhosted.org/packages/42/0b/5ee6feda28c2895b0e8cae4d828d356e13e29bf88bc614065e490bfc8b51/psutil-5.1.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"3b821bb59911afdba5c0c28b1e6e7511cfb0869dd8827c3ab6916ead508c9155"},"provenance":null,"requires-python":null,"size":189127,"upload-time":"2017-02-01T18:28:32.228516Z","url":"https://files.pythonhosted.org/packages/af/6f/ef0799221597f367171c0bb8b3f1b22865e6b7507ee26e1a6a16f408a058/psutil-5.1.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.tar.gz","hashes":{"sha256":"7570e1d82345fab3a0adce24baf993adbca4c87a1be2fa6ee79babdaafa817fb"},"provenance":null,"requires-python":null,"size":339603,"upload-time":"2017-02-01T18:29:04.029583Z","url":"https://files.pythonhosted.org/packages/ba/5f/87b151dd53f8790408adf5096fc81c3061313c36a089d9f7ec9e916da0c1/psutil-5.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"07aab3a12c00315a144b2a61d384364e64077c08c0a79966f18b744d867f9727"},"provenance":null,"requires-python":null,"size":422485,"upload-time":"2017-02-01T18:24:47.444024Z","url":"https://files.pythonhosted.org/packages/8c/1c/8bf1c2f6f2b4f012449bd9781d55397e85e30487c2879ed35569c7f616de/psutil-5.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"b78de627c157b4889e055fd00a635ad0fbe107ae4b3059f419738edbfb15b3f4"},"provenance":null,"requires-python":null,"size":420852,"upload-time":"2017-02-01T18:25:00.582537Z","url":"https://files.pythonhosted.org/packages/de/5a/3041f7d07d9699ef148291721ee167608761e223bc26bbe33483cbe90517/psutil-5.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"48fc3945219f6577b61a76f81fbbaa3ce6bdccf0e87650597c88aac38e63904b"},"provenance":null,"requires-python":null,"size":420825,"upload-time":"2017-02-01T18:25:14.404878Z","url":"https://files.pythonhosted.org/packages/d0/83/4e03c76cd202d4a4ead9b468cf2afa611d689b1ae17e4f152b67c05d2a27/psutil-5.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.5.exe","hashes":{"sha256":"98180376d2f0a3b0d24dbd8ef64da4aac46f98ccc84d2e3ebc3a486b97a75915"},"provenance":null,"requires-python":null,"size":791399,"upload-time":"2017-02-01T18:25:37.374511Z","url":"https://files.pythonhosted.org/packages/ff/86/328005e07308a87f4d86108a51f940fba1fb1c4f60d4676f80fc66a9dd7a/psutil-5.1.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.6.exe","hashes":{"sha256":"a574d3fb56514ae98f2836a218b0dcf6daeb31b9945cec9b7daa9b619ad3005a"},"provenance":null,"requires-python":null,"size":791396,"upload-time":"2017-02-01T18:26:00.310591Z","url":"https://files.pythonhosted.org/packages/83/86/88c20e9ebb1565598401b23ece21367bd059d569cb0ce797509ebdc429be/psutil-5.1.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py2.7.exe","hashes":{"sha256":"b014a8df14f00054762a2ef6f9a449ccde18d9c4de48dc8cb1809343f328f83d"},"provenance":null,"requires-python":null,"size":392388,"upload-time":"2017-02-01T18:26:13.878537Z","url":"https://files.pythonhosted.org/packages/a5/e8/1032177e5af2300b39aa50623e0df4d9a91a9b316803aab8f597e6c855e2/psutil-5.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.3.exe","hashes":{"sha256":"01e68d047f5023d4cb55b3a2653e296d86f53abc7e63e74c8abeefa627d9307f"},"provenance":null,"requires-python":null,"size":387232,"upload-time":"2017-02-01T18:26:25.910876Z","url":"https://files.pythonhosted.org/packages/cc/b8/24bf07e83d063f552132e7808d7029663466ebed945c532ba05068c65ae3/psutil-5.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.4.exe","hashes":{"sha256":"bfd279124512ee1b744eb7efa269be2096e0ce7075204daccd6f4b73c359316c"},"provenance":null,"requires-python":null,"size":387243,"upload-time":"2017-02-01T18:26:38.999172Z","url":"https://files.pythonhosted.org/packages/2a/ad/40d39c8911ca0783df9684649038241913fa3119c74030b9d1a02eb28da3/psutil-5.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.5.exe","hashes":{"sha256":"518356f5384a8996eb4056b6fd01a28bb502476e5d0ab9bd112d8e1dc9ce041a"},"provenance":null,"requires-python":null,"size":658437,"upload-time":"2017-02-01T18:26:58.991042Z","url":"https://files.pythonhosted.org/packages/db/b1/023fa6a995356c4a4ae68ba53e1269d693ad66783ee2c0c4ac1d6e556226/psutil-5.1.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.6.exe","hashes":{"sha256":"23670901cfa4308cc6c442e08efcd93c6a2adb3bfdbbf51d4c120fc2a1595899"},"provenance":null,"requires-python":null,"size":658436,"upload-time":"2017-02-01T18:27:18.934060Z","url":"https://files.pythonhosted.org/packages/f8/ee/be644cdded3a5761f22cd0255534f04ad89a6fabdfb85f58100d25b6bf1a/psutil-5.1.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"data-dist-info-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"filename":"psutil-5.1.1-cp27-none-win32.whl","hashes":{"sha256":"b9cc631c31794f8150a034a15448ecbde6c65ab078437eb01e0a71103bced297"},"provenance":null,"requires-python":null,"size":184444,"upload-time":"2017-02-03T12:00:02.925645Z","url":"https://files.pythonhosted.org/packages/e3/6f/3546783d70949aff150555cafb1cdd51cfddbb2fea042ed3d5b930232233/psutil-5.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"data-dist-info-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"filename":"psutil-5.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"d723be81a8db76ad6d78b04effd33e73813567eb1b1b7669c6926da033cb9774"},"provenance":null,"requires-python":null,"size":186898,"upload-time":"2017-02-03T12:00:10.481992Z","url":"https://files.pythonhosted.org/packages/48/98/9f3277e1d4a6d6fbc501146b9f5e8547c793df038dfa7037816edbf90325/psutil-5.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp33-cp33m-win32.whl","hashes":{"sha256":"5fc6d4fe04f014ea68392d1e7ab7103637b9dcbfd7bf3bc6d9d482177bf82777"},"provenance":null,"requires-python":null,"size":184356,"upload-time":"2017-02-03T12:00:18.015691Z","url":"https://files.pythonhosted.org/packages/98/fa/c43ce8992690d3b674259f21c897dab254b3fbbb2758254787e4b0f5aee8/psutil-5.1.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"7d0c6f77ebeb248ee62383340a8bd5a9b067e64618c9056d701eefdccf27f9f4"},"provenance":null,"requires-python":null,"size":186735,"upload-time":"2017-02-03T12:00:25.248665Z","url":"https://files.pythonhosted.org/packages/e1/4f/7d16da2b82d615a7fc57f159e5e7c8776c7b794ca6330a9318ca93eb735b/psutil-5.1.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp34-cp34m-win32.whl","hashes":{"sha256":"d31859dae480bc1a0be48f239bcf3caa26447fae177549a30c4b1a2a2776f299"},"provenance":null,"requires-python":null,"size":184362,"upload-time":"2017-02-03T12:00:31.866390Z","url":"https://files.pythonhosted.org/packages/bc/a8/3f3e69227217d1e7355e6de1980f357f6151ce1543bd520652b47823f93e/psutil-5.1.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"857a3620b12a33ed4169aee959e1640e7323f18dd7726035cc057e1d39639df2"},"provenance":null,"requires-python":null,"size":186740,"upload-time":"2017-02-03T12:00:39.950524Z","url":"https://files.pythonhosted.org/packages/d3/2c/974ced45441b7adf476f1b32ac3f7840f0d7ae295dd4512de21997e2049b/psutil-5.1.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp35-cp35m-win32.whl","hashes":{"sha256":"4ed11e8caff64c452a2eee8c1bea614b717a8e66e97fc88ce272d98a6499cb9a"},"provenance":null,"requires-python":null,"size":186257,"upload-time":"2017-02-03T12:00:46.799733Z","url":"https://files.pythonhosted.org/packages/2f/17/cb34ac2f50ff6499e6b64837a2d34b92696ff1dc14e57933363d94b7301c/psutil-5.1.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"3115d7acbf3cc81345fd7252946a59c3730f7baba546361a535f0a8f00d862c9"},"provenance":null,"requires-python":null,"size":189715,"upload-time":"2017-02-03T12:00:55.165186Z","url":"https://files.pythonhosted.org/packages/09/da/f9f435d53711859d41ee8a0fb87177e8b30b5dbca5bd06dd05b5c4b46ef3/psutil-5.1.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp36-cp36m-win32.whl","hashes":{"sha256":"9f6cb84d0f8e0c993c91d10ef86c637b7f1c1d4d4ca63ec0a73545cc13e9656a"},"provenance":null,"requires-python":null,"size":186256,"upload-time":"2017-02-03T12:01:01.728292Z","url":"https://files.pythonhosted.org/packages/24/19/20a4a33db5d005959458f68816fbb725791ee7843ba4f8a40dfd38e8e840/psutil-5.1.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"35311e26f7138276fa3e7af86bcb8ecbaee945c3549e690a481379075de386ba"},"provenance":null,"requires-python":null,"size":189711,"upload-time":"2017-02-03T12:01:08.476075Z","url":"https://files.pythonhosted.org/packages/6e/2b/44aea26dc3a304285e8eb91af628c9722ba4e1f44f7969cc24fded4aa744/psutil-5.1.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.tar.gz","hashes":{"sha256":"ece06401d719050a84cca97764ff5b0e41aafe6b6a2ec8a1d0bb89ca5e206d0f"},"provenance":null,"requires-python":null,"size":341006,"upload-time":"2017-02-03T12:01:19.347630Z","url":"https://files.pythonhosted.org/packages/49/ed/2a0b13f890e798b6f1f3625f0e87e5b712471d2c1c625bdcd396d36c56dc/psutil-5.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"e75edc462005475da019f82c8a13b215e2e48db8f284d8be14308b611505d185"},"provenance":null,"requires-python":null,"size":423004,"upload-time":"2017-02-03T12:01:32.247678Z","url":"https://files.pythonhosted.org/packages/99/35/96acf109f463ce31cd29ec327d9b3b1c3eb13afbc5635e54c75e1e22c924/psutil-5.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"e18274b77e61b3774bcf11a4aa368032fe3bbc21e4217ca903c77541fbe00eef"},"provenance":null,"requires-python":null,"size":421360,"upload-time":"2017-02-03T12:01:46.775449Z","url":"https://files.pythonhosted.org/packages/b7/56/70fc2173d34f0ee73215a0e88e2702933cf1549e3ff3d8f2d757b9e1351c/psutil-5.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"76307f8ac94adc87509df43bba28706c27d6c5e4b7429fb658dd5905adae4dc3"},"provenance":null,"requires-python":null,"size":421366,"upload-time":"2017-02-03T12:02:01.021939Z","url":"https://files.pythonhosted.org/packages/f1/9b/961442950e039c65938665dcbe6e72bff1a2dbbd6c6c08c55e3e10db38ed/psutil-5.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.5.exe","hashes":{"sha256":"8101a2b83fa3b93fbf5d5edca7169a4289f34ace2ee25d0f758cec5ae553190f"},"provenance":null,"requires-python":null,"size":791943,"upload-time":"2017-02-03T12:02:23.633303Z","url":"https://files.pythonhosted.org/packages/bb/69/95fdd45b2c1cc4735760b81d826a62433d6726ab6e8e6e2e982fb1264c20/psutil-5.1.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.6.exe","hashes":{"sha256":"b16b48868a58322edd240cf55a0855e1b6fead3e5f02a41b73503d5c47acf330"},"provenance":null,"requires-python":null,"size":791938,"upload-time":"2017-02-03T12:02:48.386069Z","url":"https://files.pythonhosted.org/packages/15/b4/1ba8944aaae568f8f4acd7fd3ca46a95077f07b005774f212226c99a82d8/psutil-5.1.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py2.7.exe","hashes":{"sha256":"12aeeaf5269bc75ed605ffc4979cb95c889989b40d307adf067f04f93f6d3365"},"provenance":null,"requires-python":null,"size":392908,"upload-time":"2017-02-03T12:03:01.259012Z","url":"https://files.pythonhosted.org/packages/d0/38/c274e67564aed7475fba4e0f9eaac3235797226df0b68abce7f3fc96bffa/psutil-5.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.3.exe","hashes":{"sha256":"2f51c669bd528982fc5397d9c84b8b389a56611c111159b1710570db33fc9750"},"provenance":null,"requires-python":null,"size":387752,"upload-time":"2017-02-03T12:03:13.897990Z","url":"https://files.pythonhosted.org/packages/d9/51/52e109c7884a6fa931882081e94dfd0e1b6adb054d4fc82943e1c9b691d2/psutil-5.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.4.exe","hashes":{"sha256":"f68ce5d80db909ee655ac8a322cda3abf47f41c134e7cf61b25565f116fce33f"},"provenance":null,"requires-python":null,"size":387758,"upload-time":"2017-02-03T12:03:26.769319Z","url":"https://files.pythonhosted.org/packages/5e/ee/ff6d626da64a799db055bab9f69ff9a50612a645781d14c24078bca418cb/psutil-5.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.5.exe","hashes":{"sha256":"380aa69aa529e4a4e35579d3e0320617e112473240c95b815d3b421c86e2ab6c"},"provenance":null,"requires-python":null,"size":658955,"upload-time":"2017-02-03T12:03:45.921701Z","url":"https://files.pythonhosted.org/packages/62/9b/847c9c6b1c053a15b44ccf809fce54bbd71e466fd8c1b8eecb0f8bbeb2ce/psutil-5.1.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.6.exe","hashes":{"sha256":"bd3b881faa071a5f6f999d036cfc0d744eed223390fde05ae2a74f0f514f8bd0"},"provenance":null,"requires-python":null,"size":658953,"upload-time":"2017-02-03T12:04:04.770301Z","url":"https://files.pythonhosted.org/packages/80/ef/c23e653bbc1a523217d676fac3ac4050e9fe4d673c84680498c02ff89d23/psutil-5.1.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"data-dist-info-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"filename":"psutil-5.1.2-cp27-none-win32.whl","hashes":{"sha256":"caa870244015bb547eeab7377d9fe41c44319fda6862aa56974108a224c04b1a"},"provenance":null,"requires-python":null,"size":185018,"upload-time":"2017-02-03T19:13:24.939112Z","url":"https://files.pythonhosted.org/packages/57/bc/e0a5e35a9b7f407e229db75b24c46bef012609450d7eac6b5e596a604acb/psutil-5.1.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"data-dist-info-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"filename":"psutil-5.1.2-cp27-none-win_amd64.whl","hashes":{"sha256":"d60d978f5a9b4bc9bb22c5c4bbeb50043db6fb70e3270c08f51e81357f8ca556"},"provenance":null,"requires-python":null,"size":187473,"upload-time":"2017-02-03T19:13:32.487659Z","url":"https://files.pythonhosted.org/packages/84/bd/49681360ca11a1aeba5e48b80b6bd576695a67fe15baca144c6bfcbc9285/psutil-5.1.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp33-cp33m-win32.whl","hashes":{"sha256":"61b9104157df03790fae3b138afb64ad3ecb663669ee3548e609306e0b35ca61"},"provenance":null,"requires-python":null,"size":184928,"upload-time":"2017-02-03T19:13:39.838419Z","url":"https://files.pythonhosted.org/packages/59/c7/7f31631d96ba58e81ad03b4f7cee234a35c67edc44b2fe5936e41e7d1e9e/psutil-5.1.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"11bfc49cd680dec42c9a7200f8b0cc4d89a9d5dbad7f3b027cfac3a305343a2a"},"provenance":null,"requires-python":null,"size":187306,"upload-time":"2017-02-03T19:13:47.199966Z","url":"https://files.pythonhosted.org/packages/a3/dc/0c87a42dbff252ab68263f88e3bd8ffaa02d766e56f743cc9074bb28c0e1/psutil-5.1.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp34-cp34m-win32.whl","hashes":{"sha256":"75b27bac9e91e9868723d8964e73a157799c26190978029b0ef294ad7727d91d"},"provenance":null,"requires-python":null,"size":184936,"upload-time":"2017-02-03T19:13:54.487932Z","url":"https://files.pythonhosted.org/packages/e6/2c/fa2d3770ea68f7f7577bfcf3fc00a2a68760e7d78b8ccb4a323fb44b27fa/psutil-5.1.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"abcfbf43eb47372e961278ee7da9a4757d59999462225b358f49c8e69a393f32"},"provenance":null,"requires-python":null,"size":187315,"upload-time":"2017-02-03T19:14:02.683013Z","url":"https://files.pythonhosted.org/packages/ed/bf/cc8cdbfc7e10518e2e1141e0724623f52bc2f41e7bdd327f8ed63e0edc01/psutil-5.1.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp35-cp35m-win32.whl","hashes":{"sha256":"3fd346acebeb84d9d351cabc02ea1bee1536fb7e165e7e5ead0e0912ce40cbb1"},"provenance":null,"requires-python":null,"size":186829,"upload-time":"2017-02-03T19:14:10.158362Z","url":"https://files.pythonhosted.org/packages/bf/60/e78e3455085393ab4902ef9b1c82e52d89f7b212582c58b76407a7078ad6/psutil-5.1.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"69644ed20c08bd257039733c71a47d871f5bdd481d63f8408e28f03f491e2a03"},"provenance":null,"requires-python":null,"size":190288,"upload-time":"2017-02-03T19:14:17.352480Z","url":"https://files.pythonhosted.org/packages/55/0a/46eba44208248da5aaeadeeab57aac3ee172df8a7291ad5d62efbed566ee/psutil-5.1.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp36-cp36m-win32.whl","hashes":{"sha256":"006ea083c66aa2a2be18bce84e35d0f3301d4ee3c869cb9d475fecce68470a71"},"provenance":null,"requires-python":null,"size":186826,"upload-time":"2017-02-03T19:14:24.235507Z","url":"https://files.pythonhosted.org/packages/26/e4/f5316795a709397e911eca478d547a3bb51b8dca447b7ed62d328cafa570/psutil-5.1.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"b64fa3a1ec7e74689b093ee6b3a487979157f81915677b9145585a2babe1b9f5"},"provenance":null,"requires-python":null,"size":190282,"upload-time":"2017-02-03T19:14:31.621854Z","url":"https://files.pythonhosted.org/packages/c0/e2/2c758b825b48eb9ae30676499a862b8475efdf683952efbb63f531413489/psutil-5.1.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.tar.gz","hashes":{"sha256":"43f32b0a392c80cff0f480bd0792763333e46d7062285dd1226b70473c55e8ac"},"provenance":null,"requires-python":null,"size":341325,"upload-time":"2017-02-03T19:14:42.976980Z","url":"https://files.pythonhosted.org/packages/19/2c/41c601cdd5586f601663d6985ff2cf1c5322f1ffd32d67d3001035d9f81d/psutil-5.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"a44f1735f8464b5cde862d76a78843869da02e1454278a38b1026c9cfa172daf"},"provenance":null,"requires-python":null,"size":423677,"upload-time":"2017-02-03T19:14:56.817144Z","url":"https://files.pythonhosted.org/packages/f2/0e/f26bd5b5e0293f715e18b1de0e46eb3378c8b2a2f54ce67bb48ee47dff44/psutil-5.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.3.exe","hashes":{"sha256":"e8bb29ba0e1526de8932025e07c13f6e26ab43a4cc2861b849ab2daf83ef4c3a"},"provenance":null,"requires-python":null,"size":422033,"upload-time":"2017-02-03T19:15:12.278688Z","url":"https://files.pythonhosted.org/packages/82/d1/e86d5892bfeb5b8439a96b39b2acc892aa54b9df9feb75ef7384673fe883/psutil-5.1.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.4.exe","hashes":{"sha256":"71cd26331eb0c26ba19d5acb67716741666a581f90bce35cc5cb733eb6bbb087"},"provenance":null,"requires-python":null,"size":422041,"upload-time":"2017-02-03T19:15:27.020838Z","url":"https://files.pythonhosted.org/packages/9f/75/f9232734ab6cdf3d656a4ef65fbe5440948c38db910002a4570b01e233d2/psutil-5.1.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.5.exe","hashes":{"sha256":"e3ea19ac2c6e1d54cb3de5919040018af2b5a0d846f7f9dc0cc4e2a125725015"},"provenance":null,"requires-python":null,"size":792618,"upload-time":"2017-02-03T19:15:52.787383Z","url":"https://files.pythonhosted.org/packages/e9/3d/cb2444851956cc4c1fe62bc4b266881e07f16b4be3cdc7b5c1ead5d99b76/psutil-5.1.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.6.exe","hashes":{"sha256":"18ad3e6d3b46dc56eda32674e9da77527bb4ac98503e45c89837fba641a2cc16"},"provenance":null,"requires-python":null,"size":792613,"upload-time":"2017-02-03T19:16:17.350579Z","url":"https://files.pythonhosted.org/packages/36/a5/803a7fdb45924ee69221259184bb2766a1a2819d91e806912f53c6dccc6d/psutil-5.1.2.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py2.7.exe","hashes":{"sha256":"1f8e54923b5d80b880d0dbc2ec5bcf51cbf1db7d54e4d2acdeeb02f42a21735a"},"provenance":null,"requires-python":null,"size":393580,"upload-time":"2017-02-03T19:16:30.238524Z","url":"https://files.pythonhosted.org/packages/b6/89/191e13e1ba569fd143c66fe340e820b909fef8599f78237cc505dd59552b/psutil-5.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.3.exe","hashes":{"sha256":"99a77884670999cf6c589539f1af3af66d8f59d9b8a9697b60398434933f56a8"},"provenance":null,"requires-python":null,"size":388425,"upload-time":"2017-02-03T19:16:43.231178Z","url":"https://files.pythonhosted.org/packages/45/ba/764a926681d90302a54494d59665ea906676b974c2ae0af7b44d5d6d9f24/psutil-5.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.4.exe","hashes":{"sha256":"2a7874d2d2718a4648cfaa4ab731f7de4867230bdcd543bf1c1fda05bc34e068"},"provenance":null,"requires-python":null,"size":388433,"upload-time":"2017-02-03T19:16:58.697184Z","url":"https://files.pythonhosted.org/packages/eb/91/98584bf6f6934c5d02b116634d6e00eddb2ec6b057556d77fdda2fe72ab0/psutil-5.1.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.5.exe","hashes":{"sha256":"42cbf7db0ce76431676da30e792b80e1228857e50afe859518b999125b5da673"},"provenance":null,"requires-python":null,"size":659629,"upload-time":"2017-02-03T19:17:19.107384Z","url":"https://files.pythonhosted.org/packages/80/b9/b17a77e77be75640594fba23837490a1dc425c9d6a9fc0b4997f5bfce984/psutil-5.1.2.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.6.exe","hashes":{"sha256":"cfd6bddd0db750b606454791432023a4260184204f4462b2a4206b7802546ead"},"provenance":null,"requires-python":null,"size":659626,"upload-time":"2017-02-03T19:17:39.500599Z","url":"https://files.pythonhosted.org/packages/8b/bd/af0e5d889818caa35fc8dc5157363c38bdb6fcb7fa1389c5daec24857a63/psutil-5.1.2.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"data-dist-info-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"filename":"psutil-5.1.3-cp27-none-win32.whl","hashes":{"sha256":"359a66879068ce609f8c034b3c575e357a92c033357f398490fc77cf8af46bf7"},"provenance":null,"requires-python":null,"size":185714,"upload-time":"2017-02-07T21:33:17.748014Z","url":"https://files.pythonhosted.org/packages/6f/c0/82c15d73633fdee59b5bd064396038a63a8920359c86460cd01d6ddcedfc/psutil-5.1.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"data-dist-info-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"filename":"psutil-5.1.3-cp27-none-win_amd64.whl","hashes":{"sha256":"a0becbbe09bed44f8f5dc3909c7eb383315f932faeb0029abe8d5c737e8dcc7e"},"provenance":null,"requires-python":null,"size":188166,"upload-time":"2017-02-07T21:33:23.089530Z","url":"https://files.pythonhosted.org/packages/80/92/c5136bbade8ba85d0aa2f5d5cbe8af80bac6ea1ef77c5445aa625a8caefb/psutil-5.1.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp33-cp33m-win32.whl","hashes":{"sha256":"5b2cc379287ded7f9a22521318bf010429234c2864b4146fe518f11729821771"},"provenance":null,"requires-python":null,"size":185622,"upload-time":"2017-02-07T21:33:27.925426Z","url":"https://files.pythonhosted.org/packages/b2/fb/8ad6ff1a9169b35a001ea089ab8657156df1b9b9903e218f1839fba0aae3/psutil-5.1.3-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"d0e88d2e8ac9ede745f589049a74ac1e3e614c4e5eed69e507d58bda8fa3c958"},"provenance":null,"requires-python":null,"size":188003,"upload-time":"2017-02-07T21:33:33.107156Z","url":"https://files.pythonhosted.org/packages/a7/8c/8a8bca010487f008ca026e69e76c53a025905a7bd1759567ec70d85d89a0/psutil-5.1.3-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp34-cp34m-win32.whl","hashes":{"sha256":"72b67b988c0a42825a8ca76000fc385dde85652310278cca807db7dfbcba5e7e"},"provenance":null,"requires-python":null,"size":185627,"upload-time":"2017-02-07T21:33:37.199567Z","url":"https://files.pythonhosted.org/packages/81/69/0ea7d353a82df8d8251842c71131f24b8a5cfa6982f7b89e809c37819c0b/psutil-5.1.3-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"474ab9a6abc05fcd7bb5c32cb828f3f9fc54a2cd349d63c94dff0af3b3ba7e64"},"provenance":null,"requires-python":null,"size":188011,"upload-time":"2017-02-07T21:33:41.528046Z","url":"https://files.pythonhosted.org/packages/93/82/2172e0a319e4c1387898b588e4dc9b8f2283fd9eda0a486ebe03ece2bff7/psutil-5.1.3-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp35-cp35m-win32.whl","hashes":{"sha256":"92bfc1f1929593ab7793ddce512295336e3e788b86a1bbf32701aa67c5ce27f4"},"provenance":null,"requires-python":null,"size":187522,"upload-time":"2017-02-07T21:33:45.934959Z","url":"https://files.pythonhosted.org/packages/43/74/08fc07b34eeb8e357dbe6bca02b844cf76cef15b36098611a50720bf7786/psutil-5.1.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7be50561ed0060c86385c2ef4dd8a383298f29728eb6e30955ae2ebbd4554e1a"},"provenance":null,"requires-python":null,"size":190983,"upload-time":"2017-02-07T21:33:51.074513Z","url":"https://files.pythonhosted.org/packages/fa/42/42e547764bf65617077b696c4c49bed6109b00696882a196008de5f8917b/psutil-5.1.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp36-cp36m-win32.whl","hashes":{"sha256":"4de5566d9d8c3695726f9ec3324cd56d3eb363365508ea39854d2ebe5d57b945"},"provenance":null,"requires-python":null,"size":187516,"upload-time":"2017-02-07T21:33:56.040728Z","url":"https://files.pythonhosted.org/packages/c5/5f/71b89c9dede1da356dbbe321ae410786b94e0b516791191c008864844733/psutil-5.1.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"678ef7b4e38281ff16dbdac98fc1d0679d46fed3fadd5d4648096fbb6d6b1b95"},"provenance":null,"requires-python":null,"size":190979,"upload-time":"2017-02-07T21:34:01.058372Z","url":"https://files.pythonhosted.org/packages/bc/21/1823e2349b1f6ec526a55d21497e5d627ac26a6c5a3d49a07b4afad45547/psutil-5.1.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.tar.gz","hashes":{"sha256":"959bd58bdc8152b0a143cb3bd822d4a1b8f7230617b0e3eb2ff6e63812120f2b"},"provenance":null,"requires-python":null,"size":341980,"upload-time":"2017-02-07T21:34:07.636785Z","url":"https://files.pythonhosted.org/packages/78/0a/aa90434c6337dd50d182a81fe4ae4822c953e166a163d1bf5f06abb1ac0b/psutil-5.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"6c288b7a639f341391ba474d4c8fb495a19220015284b46e7b23f626afafc810"},"provenance":null,"requires-python":null,"size":424369,"upload-time":"2017-02-07T21:34:15.354723Z","url":"https://files.pythonhosted.org/packages/07/bb/aac12b9c56722cf8b6ed0c89eccf1e3db75795576b7e3575001248802c0d/psutil-5.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.3.exe","hashes":{"sha256":"c4b53e0630b83c784f807170ae2d12f1cf1e45e3913f35f9784e5556ba4a0786"},"provenance":null,"requires-python":null,"size":422726,"upload-time":"2017-02-07T21:34:24.113585Z","url":"https://files.pythonhosted.org/packages/1e/5d/4804b1d23e0e8c442876c02438c4f89b3512806b034dea5807ca1ddd6535/psutil-5.1.3.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.4.exe","hashes":{"sha256":"57a4e51a0f2fd8f361fcf545eeff54932a29b716ad01e60247d1abaffbc1b954"},"provenance":null,"requires-python":null,"size":422733,"upload-time":"2017-02-07T21:34:32.211085Z","url":"https://files.pythonhosted.org/packages/51/70/34ea430cf5c21540e30b805f0740d81f75c5766c2f68af9207ae18e147dc/psutil-5.1.3.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.5.exe","hashes":{"sha256":"4b26f56f09ad206d9fb8b2fa29926a696419b26e2c5d461afe477481cec1105c"},"provenance":null,"requires-python":null,"size":793310,"upload-time":"2017-02-07T21:34:46.367507Z","url":"https://files.pythonhosted.org/packages/a2/8c/004fabd6406879fd1caa1ed4ea7ab850afad6e27e3c5b8e8a4d0d134de3e/psutil-5.1.3.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.6.exe","hashes":{"sha256":"94ed102897b8c7103ff51e2b2953caf56bb80c3343523fd3013db3ec91bd8c4b"},"provenance":null,"requires-python":null,"size":793304,"upload-time":"2017-02-07T21:35:00.329688Z","url":"https://files.pythonhosted.org/packages/55/18/6a48a9b9dad56c54236260c1e0e2313497b3af176b053a110ea134b9bb9f/psutil-5.1.3.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py2.7.exe","hashes":{"sha256":"9fcac25e01c0f9f1b6d86c860c6d4da627e458f277f24415f15b1b29cce35f60"},"provenance":null,"requires-python":null,"size":394272,"upload-time":"2017-02-07T21:35:07.791551Z","url":"https://files.pythonhosted.org/packages/33/97/442e6eefe2a12cd00d09721fb24ddf726dd62c1073579a860682919cc640/psutil-5.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.3.exe","hashes":{"sha256":"8349494ee9405a31f4f9d9d3564663c870fed5dd62efd2edfdf64c5841bb838f"},"provenance":null,"requires-python":null,"size":389117,"upload-time":"2017-02-07T21:35:16.679190Z","url":"https://files.pythonhosted.org/packages/ca/46/6d9a5c657298d1363cb37ab0f84eb1fd54639fa4b2729523a68cd6a1b043/psutil-5.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.4.exe","hashes":{"sha256":"c8dc71de8ba61604a5cae5dee5330229dc71538c82ef13458cee838b6c0f6435"},"provenance":null,"requires-python":null,"size":389124,"upload-time":"2017-02-07T21:35:24.590876Z","url":"https://files.pythonhosted.org/packages/5b/c4/7056b6c602ff5be0095fe403617cded940a75a80db49bb51846bc235a0bb/psutil-5.1.3.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.5.exe","hashes":{"sha256":"38c1e88f3a8a548d9caa7f56db1cc7d508eda48eb2c4aa484a908bc5d06f87bd"},"provenance":null,"requires-python":null,"size":660318,"upload-time":"2017-02-07T21:35:36.604103Z","url":"https://files.pythonhosted.org/packages/5f/b3/966c2979172a46f9fe42f34ce7321a59102054e26cdf9b26e3d604807953/psutil-5.1.3.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.6.exe","hashes":{"sha256":"0961ebc2ba4b1c811ef164612d0d963532ad0a9af1755e022a99648a9027b065"},"provenance":null,"requires-python":null,"size":660315,"upload-time":"2017-02-07T21:35:48.440270Z","url":"https://files.pythonhosted.org/packages/be/43/8f0099425146c01c2d77e3ac90b28a7f42d69ccb2af6e161f059db132d99/psutil-5.1.3.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"data-dist-info-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"filename":"psutil-5.2.0-cp27-none-win32.whl","hashes":{"sha256":"6eb2f6fb976152f320ee48a90ab732d694b2ae0c835260ce4f5af3907584448a"},"provenance":null,"requires-python":null,"size":187506,"upload-time":"2017-03-05T04:50:46.045903Z","url":"https://files.pythonhosted.org/packages/d2/56/56a15e285c7cf0104ed9fc569b2c75a24f97e7ab5c34567956b266d23ba3/psutil-5.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"data-dist-info-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"filename":"psutil-5.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"35898b80a3f393a7ace8ad5da9a26800676b7fc40628a3a334902b9d0e444c8d"},"provenance":null,"requires-python":null,"size":189973,"upload-time":"2017-03-05T04:50:49.198757Z","url":"https://files.pythonhosted.org/packages/a5/a5/1039829542b856ca4d3d40bb4978fbb679b7f0bb684ece6340ce655aedc9/psutil-5.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp33-cp33m-win32.whl","hashes":{"sha256":"5626533fc459ce1ac4bd017f7a38b99947c039d79175a10a2a6b6246e3a82fc8"},"provenance":null,"requires-python":null,"size":187442,"upload-time":"2017-03-05T04:50:52.517268Z","url":"https://files.pythonhosted.org/packages/8c/1e/7e6ac521b3c393b2f312f1c3795d702f3267dca23d603827d673b8170920/psutil-5.2.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"d34cc4d48245873492e4befc5c58a146f0f6c98038ffa2430e191a6752717c61"},"provenance":null,"requires-python":null,"size":189849,"upload-time":"2017-03-05T04:50:55.188058Z","url":"https://files.pythonhosted.org/packages/b6/d0/6edd271e3ca150104c818ec0f4b2affc447fe79ec1504506cecb2900d391/psutil-5.2.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp34-cp34m-win32.whl","hashes":{"sha256":"5834168071a92037736142616b33691ec4786f8806e28355e74b2e1a037cad4c"},"provenance":null,"requires-python":null,"size":187450,"upload-time":"2017-03-05T04:50:58.524346Z","url":"https://files.pythonhosted.org/packages/89/88/8fb4ce470a2022c33ab3cd16b3f2152f544e264c9db0f2f7159a93e0d2a3/psutil-5.2.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"d29c24bc7c14ecb4e64b3b748814ebe0e3ac049802ea7f129edbfcb068e75c16"},"provenance":null,"requires-python":null,"size":189841,"upload-time":"2017-03-05T04:51:03.287652Z","url":"https://files.pythonhosted.org/packages/43/9b/35cae8c56d3ee2e9a02599fba6a2e1f3fcf3553fe55f70c0ea723f9a9522/psutil-5.2.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp35-cp35m-win32.whl","hashes":{"sha256":"8353692da46bc6024b4001a9ed8849beb863fbb1d022553dd4ed8348745540bb"},"provenance":null,"requires-python":null,"size":189373,"upload-time":"2017-03-05T04:51:07.095988Z","url":"https://files.pythonhosted.org/packages/65/35/fff62f84dc6c165e8a9f7646e2c106bd223a3967a0a3f471979b38b5a5c0/psutil-5.2.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"1e00f5684fb335dacfa750e5e01f83bb79d521eb5f0805b798de0a29a1fb25d4"},"provenance":null,"requires-python":null,"size":192813,"upload-time":"2017-03-05T04:51:10.595180Z","url":"https://files.pythonhosted.org/packages/94/d2/f78b5a0ded0993f4c5127bf17427e4bc10b183dc102a5e469d7f6725ecb9/psutil-5.2.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp36-cp36m-win32.whl","hashes":{"sha256":"55d546333f1423ad219a0798867a9bbf9a90e1912c3336ad275476473624c071"},"provenance":null,"requires-python":null,"size":189376,"upload-time":"2017-03-05T04:51:15.088897Z","url":"https://files.pythonhosted.org/packages/91/8e/bd4f794b9f092d82a5b63b17da95ebd864f544ff62fb70bb1bce0687b013/psutil-5.2.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"548f14e3e21225884904e3ab228a769a73f886a3394399c591ec5f31fedc48ac"},"provenance":null,"requires-python":null,"size":192808,"upload-time":"2017-03-05T04:51:18.018177Z","url":"https://files.pythonhosted.org/packages/75/65/8499f256dc203b94f8a439f52b092742247668365dcb0997aee7349d530d/psutil-5.2.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.tar.gz","hashes":{"sha256":"2fc91d068faa5613c093335f0e758673ef8c722ad4bfa4aded64c13ae69089eb"},"provenance":null,"requires-python":null,"size":345519,"upload-time":"2017-03-05T04:51:23.230758Z","url":"https://files.pythonhosted.org/packages/3c/2f/f3ab91349c666f009077157b12057e613a3152a46a6c3be883777546b6de/psutil-5.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"7cd5dd38e08d74112c68a8f4e9b9e12fac6c6f6270792604c79a9bbd574053fa"},"provenance":null,"requires-python":null,"size":426460,"upload-time":"2017-03-05T04:51:28.726871Z","url":"https://files.pythonhosted.org/packages/ca/6f/6289db524b6aae542fa36d539524e74f25d7f9296aabadb3b5a9f17746e8/psutil-5.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"e5f1688d9bfd9e122edd35adcd8a0050430397094d08d95c380bd9c7dae48da3"},"provenance":null,"requires-python":null,"size":424854,"upload-time":"2017-03-05T04:51:34.189840Z","url":"https://files.pythonhosted.org/packages/93/7c/e92a80f5803be3febbcb40807d5d2bfe66dfe20b256c07616599c14ba2aa/psutil-5.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"11e684bf163821bd73843ebf9a27b7cb6f8a8325b943954b49b1f622264b6e80"},"provenance":null,"requires-python":null,"size":424846,"upload-time":"2017-03-05T04:51:38.965015Z","url":"https://files.pythonhosted.org/packages/4a/82/5a78b9d40c17dc4d01a06345596ac1e3f7ba590f7329d83e80b817d47f9b/psutil-5.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.5.exe","hashes":{"sha256":"9eeae9bef0875432b9aea9504ed4f7c72f8ee3e8d7ded63e484a453ee82d4a98"},"provenance":null,"requires-python":null,"size":793372,"upload-time":"2017-03-05T04:51:43.517651Z","url":"https://files.pythonhosted.org/packages/dd/f1/4bf05d2b34198954b5fd6455ebe06dd08cae5357354f6f142ef4321e41ab/psutil-5.2.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.6.exe","hashes":{"sha256":"461445afc35f98d4b7781ea2f67d0ca4ae6cf36c065ee0f4be61ace59045a2a5"},"provenance":null,"requires-python":null,"size":795416,"upload-time":"2017-03-05T04:51:53.165366Z","url":"https://files.pythonhosted.org/packages/94/59/c54e7f853586561ae42c9bcf1aa0644e8c0298479e4654b4ca36fa9eafe6/psutil-5.2.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py2.7.exe","hashes":{"sha256":"8bbbd02eb474045d201f6617d16dc8ee1d9903d5cec94f7f39cce610fc1e924b"},"provenance":null,"requires-python":null,"size":396349,"upload-time":"2017-03-05T04:51:59.453206Z","url":"https://files.pythonhosted.org/packages/21/79/40ea4e11ef6ca2b044d0aeb28d829a716a565b34c7786f779e99c005b80a/psutil-5.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.3.exe","hashes":{"sha256":"63d4320b0f3498da3551028a6ab9ee1c5aebabe0d23a7c38600c35333953ef6c"},"provenance":null,"requires-python":null,"size":391219,"upload-time":"2017-03-05T04:52:03.289639Z","url":"https://files.pythonhosted.org/packages/2e/9d/def6a3fb8150adfd71889f3e3d48160a1ba6210911baf12ce3ebd294307c/psutil-5.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.4.exe","hashes":{"sha256":"3c06b0162192db85e04846674a55915fca80f728cf626228a6b31684fc6930da"},"provenance":null,"requires-python":null,"size":391229,"upload-time":"2017-03-05T04:52:10.103932Z","url":"https://files.pythonhosted.org/packages/2c/ae/8616ac1eb00a7770d837b15ebb9ae759c43623c182f32fd43d2e6fed8649/psutil-5.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.5.exe","hashes":{"sha256":"0ba082468d6b45fb15cc1c4488aaf3ffcf0616a674c46393bf04eccc8d7c2196"},"provenance":null,"requires-python":null,"size":660405,"upload-time":"2017-03-05T04:52:18.130601Z","url":"https://files.pythonhosted.org/packages/f7/b6/c8cb94fd6696414a66021aa2229747d71612551eade262e9ab52eeb54ee2/psutil-5.2.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.6.exe","hashes":{"sha256":"6ed1cb1c9339493e1f3c379de0155c543a4c8de18224bda894190843f9509cad"},"provenance":null,"requires-python":null,"size":662455,"upload-time":"2017-03-05T04:52:27.540851Z","url":"https://files.pythonhosted.org/packages/5d/9d/8b552e9d4c2a5c3baa00d1baa1468f2a8128acd3eba79ef39e59c182676a/psutil-5.2.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"data-dist-info-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"filename":"psutil-5.2.1-cp27-none-win32.whl","hashes":{"sha256":"4e236c4ec6b0b20171c2477ded7a5b4402e4a877530640f814df839af0a40e30"},"provenance":null,"requires-python":null,"size":187855,"upload-time":"2017-03-24T15:42:20.835636Z","url":"https://files.pythonhosted.org/packages/3d/14/1242a70873873e92732dc35162317df448503a7a32e29c8bdbe30d4fa175/psutil-5.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"data-dist-info-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"filename":"psutil-5.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"e88fe0d0ca5a9623f0d8d6be05a82e33984f27b067f08806bf8a548ba4361b40"},"provenance":null,"requires-python":null,"size":190301,"upload-time":"2017-03-24T15:42:26.623783Z","url":"https://files.pythonhosted.org/packages/4c/03/fffda9f6e1ca56ce989362969b709bf7a7ade16abf7d82661bbec96580f5/psutil-5.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp33-cp33m-win32.whl","hashes":{"sha256":"54275bdbfbd20909d37ed7a2570cf9dd373ac702a89bac4814249cbc10503c03"},"provenance":null,"requires-python":null,"size":187792,"upload-time":"2017-03-24T15:42:30.952153Z","url":"https://files.pythonhosted.org/packages/d7/e0/4fde7667fad4271c06ed5e533a156bd600cdad1b69d8e6f278fe425452d2/psutil-5.2.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"316c3e334b046dc12b4f0a3dafa1d1c394e38106ac519003694fc8aeb672eafd"},"provenance":null,"requires-python":null,"size":190190,"upload-time":"2017-03-24T15:42:35.956721Z","url":"https://files.pythonhosted.org/packages/a8/c5/63453c20ac576ccb58ee56f88388434380f5e2a729aa08885d2655eb83b7/psutil-5.2.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp34-cp34m-win32.whl","hashes":{"sha256":"2249c687088145dcce87ecb90221258f9c0e7b7cea830886656cf07351e50e1b"},"provenance":null,"requires-python":null,"size":187785,"upload-time":"2017-03-24T15:42:40.746327Z","url":"https://files.pythonhosted.org/packages/59/8b/8ebb86ae5c0ba81e95bae8263de81038d3d7ee8a050f31b2b58f1a330198/psutil-5.2.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"f74532c2037fac87b76737798c74102e17f8594ea9de07aa3cb19027a630bdb0"},"provenance":null,"requires-python":null,"size":190213,"upload-time":"2017-03-24T15:42:45.196850Z","url":"https://files.pythonhosted.org/packages/e7/81/c4dd47453864984d1bd5ad0c387efc11aa6791b5abb5b369ebe2e81f7ada/psutil-5.2.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp35-cp35m-win32.whl","hashes":{"sha256":"c7c8ed864a9ef04d4736a998273e3ba0f95f22300f1e082c13a7c824b514f411"},"provenance":null,"requires-python":null,"size":189737,"upload-time":"2017-03-24T15:42:50.097781Z","url":"https://files.pythonhosted.org/packages/76/3b/2e6b3306dd2927fef9c81fdc29bc450beeb6f4bfe4cddec80260ab042900/psutil-5.2.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7a21b9d908a3bf381cc160c157a06bfcea3c6402362b26a2489566914cea9cc5"},"provenance":null,"requires-python":null,"size":193175,"upload-time":"2017-03-24T15:42:55.344665Z","url":"https://files.pythonhosted.org/packages/de/ee/cf9ecf7cea0a984a360bc889bb0bf11335755d5b7d2be9d8399fe5dc01fb/psutil-5.2.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp36-cp36m-win32.whl","hashes":{"sha256":"d1efbfc743555e7fd366956d8fe39690a3ae87e8e9e9ac06cc80bd7e2ca3059b"},"provenance":null,"requires-python":null,"size":189738,"upload-time":"2017-03-24T15:43:00.448274Z","url":"https://files.pythonhosted.org/packages/15/18/e6b1b4288d885218c845f9a340e236f03352358fc83675b9b8ef96e26227/psutil-5.2.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"cf40e944f47000375320ce0e712585321ec624a0ef67e8259f522e51bcb35a35"},"provenance":null,"requires-python":null,"size":193175,"upload-time":"2017-03-24T15:43:06.043078Z","url":"https://files.pythonhosted.org/packages/e8/7c/240fd3dfcec8d839a9a48dd2f88ba5f6e687263adc8b2452ed973b66b862/psutil-5.2.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.tar.gz","hashes":{"sha256":"fe0ea53b302f68fca1c2a3bac289e11344456786141b73391ed4022b412d5455"},"provenance":null,"requires-python":null,"size":347241,"upload-time":"2017-03-24T15:43:12.551784Z","url":"https://files.pythonhosted.org/packages/b8/47/c85fbcd23f40892db6ecc88782beb6ee66d22008c2f9821d777cb1984240/psutil-5.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"60e9bd558d640eaf9c7a4fbb0627b423b1e58fce95b41b8a24fda9145b753471"},"provenance":null,"requires-python":null,"size":426778,"upload-time":"2017-03-24T15:43:20.982480Z","url":"https://files.pythonhosted.org/packages/88/e8/40e20ea582157c81e55e1765139a5f6e969d8c01e47c016d90946b495531/psutil-5.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"318cf7bf546a23564fe4f049eae0bf205895a0524120bd549de3e46599a7f265"},"provenance":null,"requires-python":null,"size":425186,"upload-time":"2017-03-24T15:43:29.554448Z","url":"https://files.pythonhosted.org/packages/dd/55/2a74e973eb217fa5006c910a24abbd720efb7720beae9659be14fe96a413/psutil-5.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"3bcfbe8b8141c8145f1d54c3f9c2c86597508bb7cc2552e333de770a3c9b9368"},"provenance":null,"requires-python":null,"size":425211,"upload-time":"2017-03-24T15:43:39.017569Z","url":"https://files.pythonhosted.org/packages/9d/12/92575d652d33d28e6f8b0f858f3db326db5ffc4c8d55b09ac411b021d86d/psutil-5.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.5.exe","hashes":{"sha256":"979a5804366b47acd0ebf28923ee645e9fc29f4c54cbc44c41d112a1cd36e9ba"},"provenance":null,"requires-python":null,"size":793725,"upload-time":"2017-03-24T15:43:54.421574Z","url":"https://files.pythonhosted.org/packages/7f/58/de0b10442e2f277de4de0ecfae277576a6bfac1a3137fe547a4085dafa32/psutil-5.2.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.6.exe","hashes":{"sha256":"bf7d2cff21e3262d2b3e33a4b9dc27bbae81e851d694667d68dc7405c67ff31f"},"provenance":null,"requires-python":null,"size":795772,"upload-time":"2017-03-24T15:44:09.051326Z","url":"https://files.pythonhosted.org/packages/33/c0/7094de6644330b8dcdfefb0bae0a00379238588a6cf6cc9cd71c69e0cdce/psutil-5.2.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py2.7.exe","hashes":{"sha256":"ad8b603d7cc6d070cf07d39276869683474dace4da51d8050f29893ac2e22baf"},"provenance":null,"requires-python":null,"size":396688,"upload-time":"2017-03-24T15:44:17.148742Z","url":"https://files.pythonhosted.org/packages/a9/7a/5d19102362c28b6a478f9a7f3262f3ca301f8c5fed12e8d0af9e9e82e6a2/psutil-5.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.3.exe","hashes":{"sha256":"71fbaa3649aa8fa92edb1ad2b45de1e9caa7ffc63f448be951d43d6b5c6263b1"},"provenance":null,"requires-python":null,"size":391560,"upload-time":"2017-03-24T15:44:25.069190Z","url":"https://files.pythonhosted.org/packages/0b/01/f2963d84b439b0802c2354d0f777b5ed4bd0c2c11161ba81e7057a0d0523/psutil-5.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.4.exe","hashes":{"sha256":"cab6e8cfab49511f34e7ae40885792d7e655bb107f6f3c89440d5061cb19ad2f"},"provenance":null,"requires-python":null,"size":391554,"upload-time":"2017-03-24T15:44:33.671369Z","url":"https://files.pythonhosted.org/packages/d5/46/b36ff70ba0ba3b92bb5088be595fdb5641ffd982bac8e206e7c4936b2dc5/psutil-5.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.5.exe","hashes":{"sha256":"0e9e3d74f6ee1a6cac503c0bba08563dc3954e723b8392a4c74ce36f46e119ea"},"provenance":null,"requires-python":null,"size":660759,"upload-time":"2017-03-24T15:44:45.335659Z","url":"https://files.pythonhosted.org/packages/62/81/e7431ad75f9d9ae1524ee886c1aff25ec3714058de6568d305de2e0c8373/psutil-5.2.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.6.exe","hashes":{"sha256":"03e419618c3c715489ca5073cbdac6a0b12da41def69d3e4ee83f18fbb5798e5"},"provenance":null,"requires-python":null,"size":662807,"upload-time":"2017-03-24T15:44:56.446939Z","url":"https://files.pythonhosted.org/packages/77/c8/e256a28a63d06fe028f8837b860b7f6440c6ef9a475fb8c4490e1e08498b/psutil-5.2.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"data-dist-info-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"filename":"psutil-5.2.2-cp27-none-win32.whl","hashes":{"sha256":"db473f0d45a56d422502043f3755385fcfd83f5bb0947bc807fcad689230f37f"},"provenance":null,"requires-python":null,"size":187988,"upload-time":"2017-04-10T17:19:35.132698Z","url":"https://files.pythonhosted.org/packages/9c/31/c651e4c475a4d0df9609024a86fcb358a21b7a01872f7c69c7cf501a2896/psutil-5.2.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"data-dist-info-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"filename":"psutil-5.2.2-cp27-none-win_amd64.whl","hashes":{"sha256":"dcd9d3131f83480648da40d2c39403657c63a81e56e4e8d8e905bf65c133d59c"},"provenance":null,"requires-python":null,"size":190432,"upload-time":"2017-04-10T17:19:39.867178Z","url":"https://files.pythonhosted.org/packages/6e/5c/15f41041a321ffd4058ae67aade067924489acba6d277e10571b59b3127c/psutil-5.2.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp33-cp33m-win32.whl","hashes":{"sha256":"3f79a044db0aae96592ef42be459e37095d0c2cebcae4fd7baf486d37a85a8cd"},"provenance":null,"requires-python":null,"size":187924,"upload-time":"2017-04-10T17:19:44.464637Z","url":"https://files.pythonhosted.org/packages/04/a5/a027a8584208fa2cb6a88e6337b06b11388edf7d39feb0a897c9c2024639/psutil-5.2.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"838c66c123cb024bf8c8d2fec902b38c51f75b27988f4487d81383d1d3d8a8ce"},"provenance":null,"requires-python":null,"size":190325,"upload-time":"2017-04-10T17:19:49.563017Z","url":"https://files.pythonhosted.org/packages/bc/95/385e0f7e0299295401d41dd4cb6e568bf50c884af336b92a69d16981f71c/psutil-5.2.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp34-cp34m-win32.whl","hashes":{"sha256":"a155875d2fedb614c2cd687fe47953d03a47f76eb39bd5756931b288b685655f"},"provenance":null,"requires-python":null,"size":187918,"upload-time":"2017-04-10T17:19:54.252736Z","url":"https://files.pythonhosted.org/packages/12/ad/aca0f4f146b25fb2b7e9e0735287ba3ebcc02eb2bf84d49916aef730d860/psutil-5.2.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"a989876ac0cc7942ef9481b96d3bfc02777dc798d4a7a1b4e8f0f284228f3434"},"provenance":null,"requires-python":null,"size":190345,"upload-time":"2017-04-10T17:19:59.043821Z","url":"https://files.pythonhosted.org/packages/f4/45/6cbf2b7a55375f6aafc68f33581aa143f86ae1be9112546f04d8e9ee34da/psutil-5.2.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp35-cp35m-win32.whl","hashes":{"sha256":"32616c5736f1de446e77865305e7f56905c718991f820c8286436adea8192f32"},"provenance":null,"requires-python":null,"size":189869,"upload-time":"2017-04-10T17:20:03.832890Z","url":"https://files.pythonhosted.org/packages/7b/1d/8cef4ee6c1a49b1204dcdca1231ac773e27f2ed0abbbf42deb14aaf2b5cc/psutil-5.2.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"50c8ddc3a6d1cda1de6d7aaf1af10896832c6d686fc7d0fe3d01c1eb51e6f521"},"provenance":null,"requires-python":null,"size":193304,"upload-time":"2017-04-10T17:20:09.444679Z","url":"https://files.pythonhosted.org/packages/82/f4/9d4cb35c5e1c84f93718d1851adf0b4147b253111cb89f1996a08d14dba5/psutil-5.2.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp36-cp36m-win32.whl","hashes":{"sha256":"e8b65a80e978af9bf10be423442155032c589b7042b4a26edc410dc36819d65e"},"provenance":null,"requires-python":null,"size":189868,"upload-time":"2017-04-10T17:20:16.876191Z","url":"https://files.pythonhosted.org/packages/72/9f/5ff6e45db392bc9dad642dffcca44eeee552289595c087a4f1d245fdb4f9/psutil-5.2.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"7a5c0973bd4c1de98d9b225bd4303a0718d31e31d6e2342e825c3e656f7056df"},"provenance":null,"requires-python":null,"size":193305,"upload-time":"2017-04-10T17:20:21.983057Z","url":"https://files.pythonhosted.org/packages/eb/c6/29c695be774c52cca9bb68b94ae4dc866a42ddf29dcd19b7ab4c0d97bdda/psutil-5.2.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.tar.gz","hashes":{"sha256":"44746540c0fab5b95401520d29eb9ffe84b3b4a235bd1d1971cbe36e1f38dd13"},"provenance":null,"requires-python":null,"size":348413,"upload-time":"2017-04-10T17:20:29.132011Z","url":"https://files.pythonhosted.org/packages/57/93/47a2e3befaf194ccc3d05ffbcba2cdcdd22a231100ef7e4cf63f085c900b/psutil-5.2.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py2.7.exe","hashes":{"sha256":"70732850abd11f4d9fa46f0e110af21030e0a6088204f332d335921b36e66305"},"provenance":null,"requires-python":null,"size":427159,"upload-time":"2017-04-10T17:20:36.896101Z","url":"https://files.pythonhosted.org/packages/56/bb/d03fa2260839abbcdd4d323e83c2e91ffaedcb1975b88d5d0bc71c95c1fb/psutil-5.2.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.3.exe","hashes":{"sha256":"5d2f076788d71d2e1c7276f1e5a1bc255f29c2e80eb8879a9ffc633c5bf69481"},"provenance":null,"requires-python":null,"size":425568,"upload-time":"2017-04-10T17:20:45.676662Z","url":"https://files.pythonhosted.org/packages/79/fd/3d2626e6a9fd4d99859fd8eac7d52ff9850d5e4ea62611a1e3ffe6f3d257/psutil-5.2.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.4.exe","hashes":{"sha256":"fecda42b274dc618278bd9139e8493c9459d2174376f82b65ba929557f10e880"},"provenance":null,"requires-python":null,"size":425592,"upload-time":"2017-04-10T17:20:54.334109Z","url":"https://files.pythonhosted.org/packages/24/56/637ef0dfac83cd3e51096436faf7ea030f780aff3da98a79b7e7ac98a8bb/psutil-5.2.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.5.exe","hashes":{"sha256":"92e3500dfaf7a5502ebaf4a7472e2afb9ff0cb36b4e5dc1977b3c774f58332db"},"provenance":null,"requires-python":null,"size":794103,"upload-time":"2017-04-10T17:21:08.107023Z","url":"https://files.pythonhosted.org/packages/09/4d/9cf34797696c0a75fd76606c362ddfbbc0f87d2c19c95ae26e61120241ad/psutil-5.2.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.6.exe","hashes":{"sha256":"ed09521d49ee177f1205ed9791ad62263feacd2fe1cc20d1d33cf37923f240ea"},"provenance":null,"requires-python":null,"size":796151,"upload-time":"2017-04-10T17:21:22.493104Z","url":"https://files.pythonhosted.org/packages/04/74/5ea4412f31c9652335c03537ff4608ff6a12895fb3a34a187578d693b865/psutil-5.2.2.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py2.7.exe","hashes":{"sha256":"147093b75b8874e55e6b26c540544d40e98845bc4ee74dc6054c881fd2a3eed9"},"provenance":null,"requires-python":null,"size":397069,"upload-time":"2017-04-10T17:21:30.834225Z","url":"https://files.pythonhosted.org/packages/b9/aa/b779310ee8a120b5bb90880d22e7b3869f98f7a30381a71188c6fb7ff4a6/psutil-5.2.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.3.exe","hashes":{"sha256":"3d3c5c117e55c486a53ef796cc715035bf4f56419cc32dbd124fe26e9289ad1e"},"provenance":null,"requires-python":null,"size":391941,"upload-time":"2017-04-10T17:21:38.680258Z","url":"https://files.pythonhosted.org/packages/e0/3f/fd40d4edbfac1d7996b6ecb3bfa5f923662e21686727eb070490ab6dcca8/psutil-5.2.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.4.exe","hashes":{"sha256":"0c74c6a494b650966b88da256cab4e507f483c53e85b9b10d3ff9c38f059330b"},"provenance":null,"requires-python":null,"size":391936,"upload-time":"2017-04-10T17:21:46.576288Z","url":"https://files.pythonhosted.org/packages/c7/59/f7aa53e3f72d6dcfce7c60d80e74853a1444a8a4d7fef788441435c645bf/psutil-5.2.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.5.exe","hashes":{"sha256":"b5583d1c2c858056d39bd148ed25839c4f1b76fec8fb2cb9b564c82997a21266"},"provenance":null,"requires-python":null,"size":661141,"upload-time":"2017-04-10T17:21:57.551743Z","url":"https://files.pythonhosted.org/packages/f2/62/0fc0c5459bcc04171ea3bf5603622db6815ddd92a472db06bcd69612177d/psutil-5.2.2.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.6.exe","hashes":{"sha256":"1da0aa70d66612588d77daed7784e623aac1fd038681c3acd0e1c76b2b2f0819"},"provenance":null,"requires-python":null,"size":663188,"upload-time":"2017-04-10T17:22:09.360293Z","url":"https://files.pythonhosted.org/packages/a0/7f/494b600e45a8a25a76658a45747f232d38538d1c177f5b80123902d2b8da/psutil-5.2.2.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"data-dist-info-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"filename":"psutil-5.3.0-cp27-none-win32.whl","hashes":{"sha256":"6f8f858cdb79397509ee067ae9d25bee8f4b4902453ac8d155fa1629f03aa39d"},"provenance":null,"requires-python":null,"size":210071,"upload-time":"2017-09-01T10:50:37.295614Z","url":"https://files.pythonhosted.org/packages/db/36/71afb537f3718a00a9e63b75e8534bf14fee38b67fb7cb72eb60a3378162/psutil-5.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"data-dist-info-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"filename":"psutil-5.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"b31d6d19e445b56559abaa21703a6bc4b162aaf9ab99867b6f2bbbdb2c7fce66"},"provenance":null,"requires-python":null,"size":212901,"upload-time":"2017-09-01T10:50:41.892260Z","url":"https://files.pythonhosted.org/packages/c8/2b/9bc89bb0d8a2ac49ab29954017e65a9c28b83b13453418c8166938281458/psutil-5.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"data-dist-info-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"filename":"psutil-5.3.0-cp33-cp33m-win32.whl","hashes":{"sha256":"7f1ba5011095e39b3f543e9c87008409dd8a57a3e48ea1022c348244b5af77bf"},"provenance":null,"requires-python":null,"size":209934,"upload-time":"2017-09-01T10:50:46.901965Z","url":"https://files.pythonhosted.org/packages/3c/5b/b020c5f5b6fbe69bfe77072a6d35a6c6d68f7ec7fe8b148223d9365bf8b4/psutil-5.3.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"data-dist-info-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"filename":"psutil-5.3.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"853f68a85cec0137acf0504d8ca6d40d899e48ecbe931130f593a072a35b812e"},"provenance":null,"requires-python":null,"size":212722,"upload-time":"2017-09-01T10:50:51.125180Z","url":"https://files.pythonhosted.org/packages/ad/f8/4d7f713241f1786097faf48afb51e6cb9f7966d2fc36656098e182056704/psutil-5.3.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp34-cp34m-win32.whl","hashes":{"sha256":"01d9cb9473eee0e7e88319f9a5205a69e6e160b3ab2bd430a05b93bfae1528c2"},"provenance":null,"requires-python":null,"size":209879,"upload-time":"2017-09-01T10:50:55.651560Z","url":"https://files.pythonhosted.org/packages/d6/77/01a752e6d05061decf570acc800cd490766ea4534eccbe8c523b84fe5cc1/psutil-5.3.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"91d37262095c1a0f97a78f5034e10e0108e3fa326c85baa17f8cdd63fa5f81b9"},"provenance":null,"requires-python":null,"size":212614,"upload-time":"2017-09-01T10:50:59.871475Z","url":"https://files.pythonhosted.org/packages/0f/cf/fd1d752d428c5845fed4904e7bcdbb89ea3327aa063247fddcdca319c615/psutil-5.3.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp35-cp35m-win32.whl","hashes":{"sha256":"bd1776dc14b197388d728db72c103c0ebec834690ef1ce138035abf0123e2268"},"provenance":null,"requires-python":null,"size":212062,"upload-time":"2017-09-01T10:51:04.687879Z","url":"https://files.pythonhosted.org/packages/68/05/6d097706fd9cb43eda36a02a5feaee085aeac21f5bc6ea0b557109bc2eca/psutil-5.3.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7fadb1b1357ef58821b3f1fc2afb6e1601609b0daa3b55c2fabf765e0ea98901"},"provenance":null,"requires-python":null,"size":215734,"upload-time":"2017-09-01T10:51:09.110127Z","url":"https://files.pythonhosted.org/packages/6e/5a/7c688b472ff5b6cb5413acfa5a178b5e8140ffbdf0011b6d0469e97af3b1/psutil-5.3.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp36-cp36m-win32.whl","hashes":{"sha256":"d5f4634a19e7d4692f37d8d67f8418f85f2bc1e2129914ec0e4208bf7838bf63"},"provenance":null,"requires-python":null,"size":212060,"upload-time":"2017-09-01T10:51:14.595884Z","url":"https://files.pythonhosted.org/packages/28/30/4ab277d7e37cd5ee1c47a89d21465c3eec3435b973ca86cd986efdd0aeac/psutil-5.3.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"31505ee459913ef63fa4c1c0d9a11a4da60b5c5ec6a92d6d7f5d12b9653fc61b"},"provenance":null,"requires-python":null,"size":215731,"upload-time":"2017-09-01T10:51:18.736362Z","url":"https://files.pythonhosted.org/packages/c3/e6/98fd6259d8ed834d9a81567d4f41dd3645e12a9aea9d38563efaf245610a/psutil-5.3.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.tar.gz","hashes":{"sha256":"a3940e06e92c84ab6e82b95dad056241beea93c3c9b1d07ddf96485079855185"},"provenance":null,"requires-python":null,"size":397265,"upload-time":"2017-09-01T12:31:14.428985Z","url":"https://files.pythonhosted.org/packages/1c/da/555e3ad3cad30f30bcf0d539cdeae5c8e7ef9e2a6078af645c70aa81e418/psutil-5.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"ba94f021942d6cc27e18dcdccd2c1a0976f0596765ef412316ecb887d4fd3db2"},"provenance":null,"requires-python":null,"size":450426,"upload-time":"2017-09-01T10:51:26.068723Z","url":"https://files.pythonhosted.org/packages/e3/d6/238a22e898d0a3703d5fd486487108bf57d8fab1137bf085d3602d04894a/psutil-5.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"0f2fccf98bc25e8d6d61e24b2cc6350b8dfe8fa7f5251c817e977d8c61146e5d"},"provenance":null,"requires-python":null,"size":448781,"upload-time":"2017-09-01T10:51:33.193333Z","url":"https://files.pythonhosted.org/packages/91/d6/4ccff87a9f93f4837c97c3eb105c4cd1ccd544091658ba1938feca366b59/psutil-5.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"d06f02c53260d16fb445e426410263b2d271cea19136b1bb715cf10b76960359"},"provenance":null,"requires-python":null,"size":448549,"upload-time":"2017-09-01T10:51:40.339200Z","url":"https://files.pythonhosted.org/packages/74/29/91f18e7798a150c922c7da5c2153edecd7af0292332ef4f46e8ebd7184d3/psutil-5.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"724439fb20d083c943a2c62db1aa240fa15fe23644c4d4a1e9f573ffaf0bbddd"},"provenance":null,"requires-python":null,"size":817222,"upload-time":"2017-09-01T10:51:52.173626Z","url":"https://files.pythonhosted.org/packages/41/fe/5081186ce35c0def7db2f8531ccac908b83edf735c2b8b78241633853b34/psutil-5.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.6.exe","hashes":{"sha256":"a58708f3f6f74897450babb012cd8067f8911e7c8a1f2991643ec9937a8f6c15"},"provenance":null,"requires-python":null,"size":817218,"upload-time":"2017-09-01T10:52:03.695382Z","url":"https://files.pythonhosted.org/packages/aa/49/2076044f5f8554232eb5f8fb69b4a59c8f96da6d107d6f6a35aea2fb0344/psutil-5.3.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py2.7.exe","hashes":{"sha256":"108dae5ecb68f6e6212bf0553be055a2a0eec210227d8e14c3a26368b118624a"},"provenance":null,"requires-python":null,"size":419952,"upload-time":"2017-09-01T10:52:12.411324Z","url":"https://files.pythonhosted.org/packages/d1/18/7e1a418aff3f65c3072eb765b0e51445df6309be3da25c5a4cb9f1d1d18b/psutil-5.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.3.exe","hashes":{"sha256":"9832124af1e9ec0f298f17ab11c3bb91164f8068ec9429c39a7f7a0eae637a94"},"provenance":null,"requires-python":null,"size":414764,"upload-time":"2017-09-01T10:52:32.286415Z","url":"https://files.pythonhosted.org/packages/de/46/0ec33721564e235fab9112fecb836eb084d0475ab97d5a6d5c462656e715/psutil-5.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.4.exe","hashes":{"sha256":"7b8d10e7d72862d1e97caba546b60ce263b3fcecd6176e4c94efebef87ee68d3"},"provenance":null,"requires-python":null,"size":414582,"upload-time":"2017-09-01T10:52:41.243220Z","url":"https://files.pythonhosted.org/packages/86/82/254439b29eea5670633a947ec3f67c5ec33b88322a1a443f121d21dc2714/psutil-5.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.5.exe","hashes":{"sha256":"ed1f7cbbbf778a6ed98e25d48fdbdc098e66b360427661712610d72c1b4cf5f5"},"provenance":null,"requires-python":null,"size":684021,"upload-time":"2017-09-01T10:52:51.603711Z","url":"https://files.pythonhosted.org/packages/20/9c/2d84e2926e1a89c4d1ea8fc315e05145e6b10af79459727ebb688b22dab8/psutil-5.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.6.exe","hashes":{"sha256":"3d8d62f3da0b38dbfaf4756a32e18c866530b9066c298da3fc293cfefae22f0a"},"provenance":null,"requires-python":null,"size":684018,"upload-time":"2017-09-01T10:53:02.878374Z","url":"https://files.pythonhosted.org/packages/61/c7/bbcc29ba03d59f1add008245edf0d56d45434b137dde0c0d6b8441ae3be6/psutil-5.3.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"data-dist-info-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"filename":"psutil-5.3.1-cp27-none-win32.whl","hashes":{"sha256":"7a669b1897b8cdce1cea79defdf3a10fd6e4f0a8e42ac2a971dfe74bc1ce5679"},"provenance":null,"requires-python":null,"size":210168,"upload-time":"2017-09-10T05:26:42.273181Z","url":"https://files.pythonhosted.org/packages/34/90/145ff234428b4bd519c20d460d6d51db7820e8120879ca9cdc88602c57f4/psutil-5.3.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"data-dist-info-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"filename":"psutil-5.3.1-cp27-none-win_amd64.whl","hashes":{"sha256":"57be53c045f2085e28d5371eedfce804f5e49e7b35fa79bcf63e271046058002"},"provenance":null,"requires-python":null,"size":212998,"upload-time":"2017-09-10T05:27:13.646528Z","url":"https://files.pythonhosted.org/packages/c8/c9/d8cbfc3844e1a3e8b648fcca317ad8589283a7cbbc232c2c5d29cae88352/psutil-5.3.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"data-dist-info-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"filename":"psutil-5.3.1-cp33-cp33m-win32.whl","hashes":{"sha256":"27d4c5ff3ab97389a9372d246e1aa27e5f02e4709fede48a0599f89d2873ca88"},"provenance":null,"requires-python":null,"size":210025,"upload-time":"2017-09-10T05:27:19.363106Z","url":"https://files.pythonhosted.org/packages/c4/82/7e412884fcf9ae538b1d96e31688b804377803e34f4e3e86bba869eaa9f7/psutil-5.3.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"data-dist-info-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"filename":"psutil-5.3.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"72ba7e4c82879b3781ccced1eeb901f07725a36fab66270e7555e484a460760d"},"provenance":null,"requires-python":null,"size":212814,"upload-time":"2017-09-10T05:27:24.067035Z","url":"https://files.pythonhosted.org/packages/8a/1b/e185f211c9a959739c6789872458b78f6424466819afc5c420af09b222af/psutil-5.3.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp34-cp34m-win32.whl","hashes":{"sha256":"fc11c3a52990ec44064cbe026338dedcfff0e0027ca7516416eaa7d4f206c5af"},"provenance":null,"requires-python":null,"size":209976,"upload-time":"2017-09-10T05:27:29.519738Z","url":"https://files.pythonhosted.org/packages/ae/a9/ba609de04d2350878c6c3d641997dd37fa362775bf79aca3e6d542aae89e/psutil-5.3.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"7b1f9856c2fc9503a8a687db85e4f419ad1a10bfcab92ba786a7d43a6aa8cea0"},"provenance":null,"requires-python":null,"size":212710,"upload-time":"2017-09-10T05:27:34.832459Z","url":"https://files.pythonhosted.org/packages/9e/4e/e35f4e9b3f5dfb8eb88be75ccbc6e6a6428443afa4d641ff5e9e29a8991f/psutil-5.3.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp35-cp35m-win32.whl","hashes":{"sha256":"54781e463d9b9aa8c143033ee0d6a3149f9f143e6cc63099a95d4078f433dd56"},"provenance":null,"requires-python":null,"size":212159,"upload-time":"2017-09-10T05:27:39.270103Z","url":"https://files.pythonhosted.org/packages/79/4b/4531d21a7e428f3f25dc1b05be7e2024d9c2d45845ba005193dd9420e6b7/psutil-5.3.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e9ef8d265298268cad784dfece103ab06bd726512d57fc6ed9f94b55452e4571"},"provenance":null,"requires-python":null,"size":215831,"upload-time":"2017-09-10T05:27:43.869238Z","url":"https://files.pythonhosted.org/packages/aa/a7/1bc0baaea0798c1b29bfb6cec05f18f6a4cbaaf96646818429d998feb2f5/psutil-5.3.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp36-cp36m-win32.whl","hashes":{"sha256":"f5d55618cd5b9270355fb52c0430ff30c4c84c5caf5b1254eec27f80d48e7a12"},"provenance":null,"requires-python":null,"size":212160,"upload-time":"2017-09-10T05:27:51.405015Z","url":"https://files.pythonhosted.org/packages/ba/03/1946dc720fec2083148b1d7b579e789bde85cebad67b22e05d5189871114/psutil-5.3.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"773ba33fe365cb8b0998eedcbe494dc92ce7428998f07dca652a1360a9e2bce8"},"provenance":null,"requires-python":null,"size":215831,"upload-time":"2017-09-10T05:27:56.438641Z","url":"https://files.pythonhosted.org/packages/9b/de/5bd7038f8bb68516eedafc8bba9daf6740011db8afc1bb25cdcc8d654771/psutil-5.3.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.1.tar.gz","hashes":{"sha256":"12dd9c8abbad15f055e9579130035b38617020ce176f4a498b7870e6321ffa67"},"provenance":null,"requires-python":null,"size":397075,"upload-time":"2017-09-10T05:28:34.436466Z","url":"https://files.pythonhosted.org/packages/d3/0a/74dcbb162554909b208e5dbe9f4e7278d78cc27470993e05177005e627d0/psutil-5.3.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"data-dist-info-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"filename":"psutil-5.4.0-cp27-none-win32.whl","hashes":{"sha256":"8121039d2280275ac82f99a0a48110450cbf5b356a11c842c8f5cdeafdf105e1"},"provenance":null,"requires-python":null,"size":218177,"upload-time":"2017-10-12T07:26:59.429237Z","url":"https://files.pythonhosted.org/packages/4c/89/08a536124b4ee1fd850982f59ab7268a359e4160a6bcc1d473f6971fbdd4/psutil-5.4.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"data-dist-info-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"filename":"psutil-5.4.0-cp27-none-win_amd64.whl","hashes":{"sha256":"fcd93acb2602d01b86e0cfa4c2db689a81badae98d9c572348c94f1b2ea4b30d"},"provenance":null,"requires-python":null,"size":220988,"upload-time":"2017-10-12T07:27:04.404602Z","url":"https://files.pythonhosted.org/packages/c9/3d/0cd95044d1245166ddf24144e1b3e7a6b6a81933de3ff48c1851664109fc/psutil-5.4.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"data-dist-info-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"filename":"psutil-5.4.0-cp33-cp33m-win32.whl","hashes":{"sha256":"60a58bfdda1fc6e86ecf95c6eef71252d9049694df9aa0a16c2841a425fc9deb"},"provenance":null,"requires-python":null,"size":218040,"upload-time":"2017-10-12T07:27:13.087638Z","url":"https://files.pythonhosted.org/packages/e5/1a/0f27898ad585d924e768b0c7029c7d7ac429994a3a032419c51f1c1f3e41/psutil-5.4.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"data-dist-info-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"filename":"psutil-5.4.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"9956b370243005d5561a94efa44b0cddb826d1f14d21958003925b008d3b9eb1"},"provenance":null,"requires-python":null,"size":220796,"upload-time":"2017-10-12T07:27:19.356245Z","url":"https://files.pythonhosted.org/packages/19/8e/b42236e04fbd03ffa2a08a393c8800fd7d51b11823c0e18723f0780d9b6a/psutil-5.4.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp34-cp34m-win32.whl","hashes":{"sha256":"104fec73d9ed573351f3efbf1b7ee19eb3b4097e2b3d9ff26b1ac5bca52b6f9e"},"provenance":null,"requires-python":null,"size":217969,"upload-time":"2017-10-12T07:27:24.486998Z","url":"https://files.pythonhosted.org/packages/11/42/0711b59b7f2f2f7de7912d30bc599950011d25401bda8a3330f878ff1e56/psutil-5.4.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"64b2814b30452d854d5f7a7c9c0d77423388b44eb2a8bcab3b84feeceaba8ffb"},"provenance":null,"requires-python":null,"size":220696,"upload-time":"2017-10-12T07:27:29.623184Z","url":"https://files.pythonhosted.org/packages/e6/67/e19ccc78646810cbc432c429e2993315cf50a126379b83b0f8b3eb9172b9/psutil-5.4.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp35-cp35m-win32.whl","hashes":{"sha256":"1a89ba967d4b9a3d5f19ea2c63b09e5ffb3a81de3116ead7bfb67b9c308e8dba"},"provenance":null,"requires-python":null,"size":220150,"upload-time":"2017-10-12T07:27:34.351564Z","url":"https://files.pythonhosted.org/packages/cc/c1/47edb3fccbb1354499297702ca55aec41ff7aab67b4df69aba9db4e52a7c/psutil-5.4.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c8362902d4d94640c61960c18d8aa1af074822a549c3d4f137be3aa62c17f4b9"},"provenance":null,"requires-python":null,"size":223827,"upload-time":"2017-10-12T07:27:46.430105Z","url":"https://files.pythonhosted.org/packages/a9/e4/d399d060bbf40c1480ce2bfbc429edbd42769b048c4c7fdc51a49d74c9cf/psutil-5.4.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp36-cp36m-win32.whl","hashes":{"sha256":"04ed7548dfe61ab2561a94ac848a5b79239bb23e9015596ffd0644efd22461ba"},"provenance":null,"requires-python":null,"size":220151,"upload-time":"2017-10-12T07:27:51.153048Z","url":"https://files.pythonhosted.org/packages/2b/46/88fdd6206f01b04547bb89a6d2ff7eec380e5c59bda56ee03a0759ed397c/psutil-5.4.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"346555603ea903a5524bb37a49b0e0f90960c6c9973ebf794ae0802a4aa875eb"},"provenance":null,"requires-python":null,"size":223825,"upload-time":"2017-10-12T07:27:57.513929Z","url":"https://files.pythonhosted.org/packages/34/e2/8421ca5b99209fa43e48d2f76a21d5d86d7346838053abf7a4d2aa37255e/psutil-5.4.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.0.tar.gz","hashes":{"sha256":"8e6397ec24a2ec09751447d9f169486b68b37ac7a8d794dca003ace4efaafc6a"},"provenance":null,"requires-python":null,"size":406945,"upload-time":"2017-10-12T07:22:51.321681Z","url":"https://files.pythonhosted.org/packages/8d/96/1fc6468be91521192861966c40bd73fdf8b065eae6d82dd0f870b9825a65/psutil-5.4.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"data-dist-info-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"filename":"psutil-5.4.1-cp27-none-win32.whl","hashes":{"sha256":"7ef26ebe728ac821de17df23820e6ffcfd37c409fc865380e4d5ae1388f274a1"},"provenance":null,"requires-python":null,"size":218627,"upload-time":"2017-11-08T13:50:32.260924Z","url":"https://files.pythonhosted.org/packages/dd/dd/1811a99faefd2b5947b4e68bd70767b525fdbad65481a4bd2ee7e6408749/psutil-5.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"data-dist-info-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"filename":"psutil-5.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"692dc72817d157aae522231dd334ea2524c6b07d844db0e7a2d6897820083427"},"provenance":null,"requires-python":null,"size":221450,"upload-time":"2017-11-08T13:50:37.542045Z","url":"https://files.pythonhosted.org/packages/51/c1/eec6a42a9f5fcd564c3fe3c6435c2c00e4e951a37f3ea3d324b04503ca6f/psutil-5.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp34-cp34m-win32.whl","hashes":{"sha256":"92342777d46e4630cf17d437412dc7fce0a8561217e074d36a35eb911ffd570e"},"provenance":null,"requires-python":null,"size":218421,"upload-time":"2017-11-08T13:50:42.728570Z","url":"https://files.pythonhosted.org/packages/e1/bd/d34935cd39f893d6e9ed46df5245aa71b29d2408b98f23755f234d517f80/psutil-5.4.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"f8f2f47a987c32ed3ca2068f3dfa9060dc9ff6cbed023d627d3f27060f4e59c4"},"provenance":null,"requires-python":null,"size":221144,"upload-time":"2017-11-08T13:50:47.928399Z","url":"https://files.pythonhosted.org/packages/24/ae/dcb7394e75b23b71f46742ffdab21d864a7ae74124b7930e5ea4f47b9049/psutil-5.4.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp35-cp35m-win32.whl","hashes":{"sha256":"1fce45549618d1930afefe322834ba91758331725bfdaec73ba6abcc83f6dc11"},"provenance":null,"requires-python":null,"size":220611,"upload-time":"2017-11-08T13:50:52.754745Z","url":"https://files.pythonhosted.org/packages/e8/58/c60fbf66c58d1e4b18902d601a294cb6ee993f3d051b44fdf397b6166852/psutil-5.4.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f8a88553b2b5916f3bd814a91942215822a1dabae6db033cbb019095d6a24bc2"},"provenance":null,"requires-python":null,"size":224288,"upload-time":"2017-11-08T13:50:58.405900Z","url":"https://files.pythonhosted.org/packages/4f/f8/0e4e80114bc58267199c932f9227d09a00dea952b37400f76aa2a3bb9492/psutil-5.4.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp36-cp36m-win32.whl","hashes":{"sha256":"4139f76baa59142b907dd581d7ff3506a5163cb8ef69e8e92060df330bbf5788"},"provenance":null,"requires-python":null,"size":220610,"upload-time":"2017-11-08T13:51:03.361290Z","url":"https://files.pythonhosted.org/packages/8e/16/02eb53ea087776d9f219973e7b52c7d729929a2727c15894842f9b3629e6/psutil-5.4.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d61bc04401ce938576e4c6ec201e812ed4114bfb9712202b87003619116c90c6"},"provenance":null,"requires-python":null,"size":224287,"upload-time":"2017-11-08T13:51:07.991086Z","url":"https://files.pythonhosted.org/packages/4a/89/0610a20ab3d1546fbe288001fb79ec4818ef6de29f89259c39daea85984f/psutil-5.4.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.1.tar.gz","hashes":{"sha256":"42e2de159e3c987435cb3b47d6f37035db190a1499f3af714ba7af5c379b6ba2"},"provenance":null,"requires-python":null,"size":408489,"upload-time":"2017-11-08T13:51:15.716367Z","url":"https://files.pythonhosted.org/packages/fe/17/0f0bf5792b2dfe6003efc5175c76225f7d3426f88e2bf8d360cfab870cd8/psutil-5.4.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"data-dist-info-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"filename":"psutil-5.4.2-cp27-none-win32.whl","hashes":{"sha256":"2fbbc7dce43c5240b9dc6d56302d57412f1c5a0d665d1f04eb05a6b7279f4e9b"},"provenance":null,"requires-python":null,"size":221114,"upload-time":"2017-12-07T12:03:17.419322Z","url":"https://files.pythonhosted.org/packages/e6/e2/09d55a6e899cf1a7b6a22d0cc2a75d45553df5b63d7c9f2eb2553c7207bc/psutil-5.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"data-dist-info-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"filename":"psutil-5.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"259ec8578d19643179eb2377348c63b650b51ba40f58f2620a3d9732b8a0b557"},"provenance":null,"requires-python":null,"size":223996,"upload-time":"2017-12-07T12:03:22.758210Z","url":"https://files.pythonhosted.org/packages/47/fc/e2199322f422e4bb6e25808a335132235fb0f3fabb0ea71ec3442719fdaf/psutil-5.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp34-cp34m-win32.whl","hashes":{"sha256":"d3808be8241433db17fa955566c3b8be61dac8ba8f221dcbb202a9daba918db5"},"provenance":null,"requires-python":null,"size":220935,"upload-time":"2017-12-07T12:03:27.058880Z","url":"https://files.pythonhosted.org/packages/d5/50/044ad4b47bf0e992a11ae7cb060e8c7dd52eb6983c2c4b6fdd5314fcc3b2/psutil-5.4.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"449747f638c221f8ce6ca3548aefef13339aa05b453cc1f233f4d6c31c206198"},"provenance":null,"requires-python":null,"size":223749,"upload-time":"2017-12-07T12:03:31.671187Z","url":"https://files.pythonhosted.org/packages/b5/77/6f8b4c7c6e9e8fa60b0e0627853ba7182f1a2459ad0b557fe3257bbe014b/psutil-5.4.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp35-cp35m-win32.whl","hashes":{"sha256":"f6c2d54abd59ed8691882de7fd6b248f5808a567885f20f50b3b4b9eedaebb1f"},"provenance":null,"requires-python":null,"size":223260,"upload-time":"2017-12-07T12:03:36.139696Z","url":"https://files.pythonhosted.org/packages/ad/fa/ac5b6bc2b25817509fc2e44f910f39b3385d1575262f7802b9c113ab786a/psutil-5.4.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e3d00d8fc3d4217f05d07af45390f072c04cb7c7dddd70b86b728e5fbe485c81"},"provenance":null,"requires-python":null,"size":226950,"upload-time":"2017-12-07T12:03:40.889125Z","url":"https://files.pythonhosted.org/packages/0e/5c/67b33328d4f307608006c3630838b272719125b36e5456a4dd8f4e76eca9/psutil-5.4.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp36-cp36m-win32.whl","hashes":{"sha256":"3473d6abad9d6ec7b8a97f4dc55f0b3483ecf470d85f08f5e23c1c07592b914f"},"provenance":null,"requires-python":null,"size":223259,"upload-time":"2017-12-07T12:03:45.502272Z","url":"https://files.pythonhosted.org/packages/59/75/97862069d3fa20f1f2897fecd5b6822a3c06ee2af1161f1beb320cc4f5f8/psutil-5.4.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"7dc6c3bbb5d28487f791f195d6abfdef295d34c44ce6cb5f2d178613fb3338ab"},"provenance":null,"requires-python":null,"size":226952,"upload-time":"2017-12-07T12:03:51.257182Z","url":"https://files.pythonhosted.org/packages/8f/9a/5dea138e49addd68d8e65d08103dd28428f1ea0b8d4e8beef9b24f069a16/psutil-5.4.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.2.tar.gz","hashes":{"sha256":"00a1f9ff8d1e035fba7bfdd6977fa8ea7937afdb4477339e5df3dba78194fe11"},"provenance":null,"requires-python":null,"size":411888,"upload-time":"2017-12-07T12:03:58.583601Z","url":"https://files.pythonhosted.org/packages/54/24/aa854703715fa161110daa001afce75d21d1840e9ab5eb28708d6a5058b0/psutil-5.4.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"data-dist-info-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"filename":"psutil-5.4.3-cp27-none-win32.whl","hashes":{"sha256":"82a06785db8eeb637b349006cc28a92e40cd190fefae9875246d18d0de7ccac8"},"provenance":null,"requires-python":null,"size":220984,"upload-time":"2018-01-01T20:33:16.515515Z","url":"https://files.pythonhosted.org/packages/e5/cc/6dd427e738a8db6d0b66525856da43d2ef12c4c19269863927f7cf0e2aaf/psutil-5.4.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"data-dist-info-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"filename":"psutil-5.4.3-cp27-none-win_amd64.whl","hashes":{"sha256":"4152ae231709e3e8b80e26b6da20dc965a1a589959c48af1ed024eca6473f60d"},"provenance":null,"requires-python":null,"size":223871,"upload-time":"2018-01-01T20:33:24.887595Z","url":"https://files.pythonhosted.org/packages/b9/e4/6867765edcab8d12a52c84c9b0af492ecb99f8cc565ad552341bcf73ebd9/psutil-5.4.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp34-cp34m-win32.whl","hashes":{"sha256":"230eeb3aeb077814f3a2cd036ddb6e0f571960d327298cc914c02385c3e02a63"},"provenance":null,"requires-python":null,"size":220823,"upload-time":"2018-01-01T20:33:31.019909Z","url":"https://files.pythonhosted.org/packages/5b/fc/745a864190a4221cdb984e666f4218e98d9a53a64b4dcf2eb7a71c1bf693/psutil-5.4.3-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"a3286556d4d2f341108db65d8e20d0cd3fcb9a91741cb5eb496832d7daf2a97c"},"provenance":null,"requires-python":null,"size":223605,"upload-time":"2018-01-01T20:33:38.392793Z","url":"https://files.pythonhosted.org/packages/b0/25/414738d5e8e75418e560a36651d1e1b09c9df05440a2a808d999a5548b1e/psutil-5.4.3-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp35-cp35m-win32.whl","hashes":{"sha256":"94d4e63189f2593960e73acaaf96be235dd8a455fe2bcb37d8ad6f0e87f61556"},"provenance":null,"requires-python":null,"size":223167,"upload-time":"2018-01-01T20:33:43.796884Z","url":"https://files.pythonhosted.org/packages/e9/80/8da216f42050220f37f7133d2accfcd001a1bd0f31d7cdb8660acb46b8fe/psutil-5.4.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c91eee73eea00df5e62c741b380b7e5b6fdd553891bee5669817a3a38d036f13"},"provenance":null,"requires-python":null,"size":226810,"upload-time":"2018-01-01T20:33:50.884798Z","url":"https://files.pythonhosted.org/packages/23/34/b3de39502c2c34899f9e7ae3c8d1050c9317997ab1fe6c647e7a789571a8/psutil-5.4.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp36-cp36m-win32.whl","hashes":{"sha256":"779ec7e7621758ca11a8d99a1064996454b3570154277cc21342a01148a49c28"},"provenance":null,"requires-python":null,"size":223168,"upload-time":"2018-01-01T20:33:56.458784Z","url":"https://files.pythonhosted.org/packages/da/c1/caadba7c64f72118b02f019c60ad85a5668ddf0a32836230b71692b0cbfa/psutil-5.4.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"8a15d773203a1277e57b1d11a7ccdf70804744ef4a9518a87ab8436995c31a4b"},"provenance":null,"requires-python":null,"size":226804,"upload-time":"2018-01-01T20:34:03.130620Z","url":"https://files.pythonhosted.org/packages/71/80/90799d3dc6e33e650ee03f96fa18157faed885593eabea3a6560ebff7de0/psutil-5.4.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.3.tar.gz","hashes":{"sha256":"e2467e9312c2fa191687b89ff4bc2ad8843be4af6fb4dc95a7cc5f7d7a327b18"},"provenance":null,"requires-python":null,"size":412550,"upload-time":"2018-01-01T20:34:13.285899Z","url":"https://files.pythonhosted.org/packages/e2/e1/600326635f97fee89bf8426fef14c5c29f4849c79f68fd79f433d8c1bd96/psutil-5.4.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"data-dist-info-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"filename":"psutil-5.4.4-cp27-none-win32.whl","hashes":{"sha256":"8f208867d41eb3b6de416df098a9a28d08d40b432467d821b8ef5bb589a394ce"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216647,"upload-time":"2018-04-13T09:10:24.220147Z","url":"https://files.pythonhosted.org/packages/e8/cd/dbf537e32de1c9f06a0069bf0ef13c8707f653e9af2b0ea0ed7040b73083/psutil-5.4.4-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"data-dist-info-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"filename":"psutil-5.4.4-cp27-none-win_amd64.whl","hashes":{"sha256":"77b5e310de17085346ef2c4c21b64d5e39616ab4559b8ef6fea9f6f2ab0de66f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219668,"upload-time":"2018-04-13T09:10:29.328807Z","url":"https://files.pythonhosted.org/packages/7f/37/538bb4275d8a26ec369944878a681000e827e72cab9b4f27f4b1b5932446/psutil-5.4.4-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp34-cp34m-win32.whl","hashes":{"sha256":"fec0e59dacbe91db7e063f038301f49da7e9361732fc31d28338ecaa4719520e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216642,"upload-time":"2018-04-13T09:10:33.842941Z","url":"https://files.pythonhosted.org/packages/92/e9/c9c4ec1a0ac55ee1514c1a249d017dd2f7a89e727d236ed6862b493de154/psutil-5.4.4-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"1268fb6959cd8d761c30e13e79908ae73ba5a69c3c3a5d09a7a27278446f9800"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219513,"upload-time":"2018-04-13T09:10:38.041575Z","url":"https://files.pythonhosted.org/packages/83/49/c903f446d28bfb6e92fa08b710f68cd5d17cc2ccfc4a13fe607f8b20f6dd/psutil-5.4.4-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp35-cp35m-win32.whl","hashes":{"sha256":"7eb2d80ef79d90474a03eead13b32e541d1fdeb47468cf04c881f0a7392ddbc5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219220,"upload-time":"2018-04-13T09:10:42.685382Z","url":"https://files.pythonhosted.org/packages/bb/63/c9b3e9ff8d23409896a7e9e7356a730fdfb5a45a1edc0e6d4ca5ce655f29/psutil-5.4.4-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"69f1db4d13f362ce11a6246b20c752c31b87a6fd77452170fd03c26a8a20a4f2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222757,"upload-time":"2018-04-13T09:10:47.950981Z","url":"https://files.pythonhosted.org/packages/72/22/a5ce34af1285679e02d7fd701ff6389f579a17e623dd89236ea1873ce12b/psutil-5.4.4-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp36-cp36m-win32.whl","hashes":{"sha256":"6eb59bcfd48eade8889bae67a16e0d8c7b18af0732ba64dead61206fd7cb4e45"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219224,"upload-time":"2018-04-13T09:10:52.291473Z","url":"https://files.pythonhosted.org/packages/3c/6d/5e9a3683d4532997525aa20d1d9ce0ca1201271d30aad9e5f18c34459478/psutil-5.4.4-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"a4af5d4fcf6022886a30fb3b4fff71ff25f645865a68506680d43a3e634764af"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222759,"upload-time":"2018-04-13T09:10:56.584621Z","url":"https://files.pythonhosted.org/packages/b6/61/eeeab30fa737b8b95b790d3eb8f49ebedeb783e43aef2d8d851687592d6c/psutil-5.4.4-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.4.tar.gz","hashes":{"sha256":"5959e33e0fc69742dd22e88bfc7789a1f2e1fc2297794b543119e10cdac8dfb1"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":417890,"upload-time":"2018-04-13T09:11:03.199870Z","url":"https://files.pythonhosted.org/packages/35/35/7da482448cd9ee3555faa9c5e541e37b18a849fb961e55d6fda6ca936ddb/psutil-5.4.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"data-dist-info-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"filename":"psutil-5.4.5-cp27-none-win32.whl","hashes":{"sha256":"33384065f0014351fa70187548e3e95952c4df4bc5c38648bd0e647d21eaaf01"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216630,"upload-time":"2018-04-13T17:59:43.195295Z","url":"https://files.pythonhosted.org/packages/63/9f/529a599db3057602114a30aa5e3641e78bce6c6e195adb75309c9286cb88/psutil-5.4.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"data-dist-info-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"filename":"psutil-5.4.5-cp27-none-win_amd64.whl","hashes":{"sha256":"f24cd52bafa06917935fe1b68c5a45593abe1f3097dc35b2dfc4718236795890"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219650,"upload-time":"2018-04-13T17:59:53.389911Z","url":"https://files.pythonhosted.org/packages/b6/ca/2d23b37e9b30908174d2cb596f60f06b3858856a2e595c931f7d4d640c03/psutil-5.4.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp34-cp34m-win32.whl","hashes":{"sha256":"99029b6af386b22882f0b6d537ffed5a9c3d5ff31782974aeaa1d683262d8543"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216589,"upload-time":"2018-04-13T18:00:04.166075Z","url":"https://files.pythonhosted.org/packages/bf/bc/f687dfa4679aad782fb78c43fed2626cb0157567a5b06790997e5aa0f166/psutil-5.4.5-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"51e12aa74509832443862373a2655052b20c83cad7322f49d217452500b9a405"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219462,"upload-time":"2018-04-13T18:00:14.637120Z","url":"https://files.pythonhosted.org/packages/32/b5/545953316dd9cb053e4c7d4f70d88aba5362dbbe58422ca6bbec1bbf8956/psutil-5.4.5-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp35-cp35m-win32.whl","hashes":{"sha256":"325c334596ad2d8a178d0e7b4eecc91748096a87489b3701ee16986173000aaa"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219172,"upload-time":"2018-04-13T18:00:28.230677Z","url":"https://files.pythonhosted.org/packages/00/ed/fdf2930c41e76e3a8bc59bf998062ee5ad0c393170a7d2c273dd3b259794/psutil-5.4.5-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"52a91ba928a5e86e0249b4932d6e36972a72d1ad8dcc5b7f753a2ae14825a4ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222706,"upload-time":"2018-04-13T18:00:38.180085Z","url":"https://files.pythonhosted.org/packages/c6/bf/09b13c17f54f0004ccb43cc1c2d36bab2eb75f471564b7856749dcaf62c3/psutil-5.4.5-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp36-cp36m-win32.whl","hashes":{"sha256":"b10703a109cc9225cd588c207f7f93480a420ade35c13515ea8f20063b42a392"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219171,"upload-time":"2018-04-13T18:00:52.610707Z","url":"https://files.pythonhosted.org/packages/3c/ae/34952007b4d64f88a03510866b9cd90207e391f6b2b59b6301ad96fa0fb5/psutil-5.4.5-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ddba952ed256151844d82fb13c8fb1019fe11ecaeacbd659d67ba5661ae73d0d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222710,"upload-time":"2018-04-13T18:01:02.457641Z","url":"https://files.pythonhosted.org/packages/4c/bb/303f15f4a47b96ff0ae5025d89b330e2be314085c418c0b726877476e937/psutil-5.4.5-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.5.tar.gz","hashes":{"sha256":"ebe293be36bb24b95cdefc5131635496e88b17fabbcf1e4bc9b5c01f5e489cfe"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":418003,"upload-time":"2018-04-13T18:01:19.381491Z","url":"https://files.pythonhosted.org/packages/14/a2/8ac7dda36eac03950ec2668ab1b466314403031c83a95c5efc81d2acf163/psutil-5.4.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"data-dist-info-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"filename":"psutil-5.4.6-cp27-none-win32.whl","hashes":{"sha256":"319e12f6bae4d4d988fbff3bed792953fa3b44c791f085b0a1a230f755671ef7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216468,"upload-time":"2018-06-07T15:39:23.967375Z","url":"https://files.pythonhosted.org/packages/b4/00/82c9fb4ffca22f2f6d0d883469584cd0cff71a604a19809015045b1fbab6/psutil-5.4.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"data-dist-info-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"filename":"psutil-5.4.6-cp27-none-win_amd64.whl","hashes":{"sha256":"7789885a72aa3075d28d028236eb3f2b84d908f81d38ad41769a6ddc2fd81b7c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219532,"upload-time":"2018-06-07T15:39:26.297791Z","url":"https://files.pythonhosted.org/packages/c2/23/22df1d36dc8ae002e9f646f9ed06b4f6bfbc7a22b67804c3a497be21d002/psutil-5.4.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp34-cp34m-win32.whl","hashes":{"sha256":"0ff2b16e9045d01edb1dd10d7fbcc184012e37f6cd38029e959f2be9c6223f50"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216464,"upload-time":"2018-06-07T15:39:28.563215Z","url":"https://files.pythonhosted.org/packages/f9/fa/966988e350306e1a1a9024e77ad5f118cbfe11318e8bdfc258c3d5a1c68b/psutil-5.4.6-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"dc85fad15ef98103ecc047a0d81b55bbf5fe1b03313b96e883acc2e2fa87ed5c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219331,"upload-time":"2018-06-07T15:39:31.250304Z","url":"https://files.pythonhosted.org/packages/a5/0d/40b552c2c089523df1f7ab5a0249fbf90cb2e80e89177b0189e41e367adc/psutil-5.4.6-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp35-cp35m-win32.whl","hashes":{"sha256":"7f4616bcb44a6afda930cfc40215e5e9fa7c6896e683b287c771c937712fbe2f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219052,"upload-time":"2018-06-07T15:39:33.732713Z","url":"https://files.pythonhosted.org/packages/00/4d/194ec701de80c704f679bf78495c054994cc403884ffec816787813c4fde/psutil-5.4.6-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"529ae235896efb99a6f77653a7138273ab701ec9f0343a1f5030945108dee3c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222533,"upload-time":"2018-06-07T15:39:36.224840Z","url":"https://files.pythonhosted.org/packages/e3/21/f82f270326e098f211bcc36cbb2ae7100732dcad03bd324e6af8c9d7e407/psutil-5.4.6-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp36-cp36m-win32.whl","hashes":{"sha256":"254adb6a27c888f141d2a6032ae231d8ed4fc5f7583b4c825e5f7d7c78d26d2e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219048,"upload-time":"2018-06-07T15:39:38.273913Z","url":"https://files.pythonhosted.org/packages/d6/e0/0f1b4f61246c4e2b540898b1ca0fa51ee2f52f0366956974f1039e00ed67/psutil-5.4.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"a9b85b335b40a528a8e2a6b549592138de8429c6296e7361892958956e6a73cf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222534,"upload-time":"2018-06-07T15:39:40.675791Z","url":"https://files.pythonhosted.org/packages/36/4b/80d9eb5d39ec4b4d8aec8b098b5097a7291de20bbbe6c2ab233b9d8fe245/psutil-5.4.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp37-cp37m-win32.whl","hashes":{"sha256":"6d981b4d863b20c8ceed98b8ac3d1ca7f96d28707a80845d360fa69c8fc2c44b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219860,"upload-time":"2018-06-28T22:47:57.979619Z","url":"https://files.pythonhosted.org/packages/7a/62/28923c44954b6cf8aee637f3a2f30e0e1ff39ec0f74a4f98069d37f00751/psutil-5.4.6-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"7fdb3d02bfd68f508e6745021311a4a4dbfec53fca03721474e985f310e249ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224296,"upload-time":"2018-06-28T22:48:01.565801Z","url":"https://files.pythonhosted.org/packages/b1/be/78f9d786bddc190c4b394a01531741a11b95f1522cf2759958f13b46407f/psutil-5.4.6-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.6.tar.gz","hashes":{"sha256":"686e5a35fe4c0acc25f3466c32e716f2d498aaae7b7edc03e2305b682226bcf6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":418059,"upload-time":"2018-06-07T15:39:42.364107Z","url":"https://files.pythonhosted.org/packages/51/9e/0f8f5423ce28c9109807024f7bdde776ed0b1161de20b408875de7e030c3/psutil-5.4.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"data-dist-info-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"filename":"psutil-5.4.7-cp27-none-win32.whl","hashes":{"sha256":"b34611280a2d0697f1c499e15e936d88109170194b390599c98bab8072a71f05"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":217777,"upload-time":"2018-08-14T21:01:10.586697Z","url":"https://files.pythonhosted.org/packages/b7/e9/bedbdfecef9d708489cfcd8b9aeada8d8f014fc14644c7129c7177e80d32/psutil-5.4.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"data-dist-info-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"filename":"psutil-5.4.7-cp27-none-win_amd64.whl","hashes":{"sha256":"a890c3e490493f21da2817ffc92822693bc0d6bcac9999caa04ffce8dd4e7132"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220989,"upload-time":"2018-08-14T21:01:13.273922Z","url":"https://files.pythonhosted.org/packages/50/6a/34525bc4e6e153bf6e849a4c4e936742b365f6819c0462cebfa4f082a3c4/psutil-5.4.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp35-cp35m-win32.whl","hashes":{"sha256":"1914bacbd2fc2af8f795daa44b9d2e0649a147460cfd21b1a70a124472f66d40"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220344,"upload-time":"2018-08-14T21:01:15.659359Z","url":"https://files.pythonhosted.org/packages/36/32/5b10d7c3940c64fe92620c368ede8a10016d51aa36079a5cd69944da5a74/psutil-5.4.7-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d081707ef0081920533db30200a2d30d5c0ea9cf6afa7cf8881ae4516cc69c48"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224081,"upload-time":"2018-08-14T21:01:18.052521Z","url":"https://files.pythonhosted.org/packages/ef/fc/8dc7731df7de2f4c65378a7147ecc977221093eee90d9777ca501c2790c5/psutil-5.4.7-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp36-cp36m-win32.whl","hashes":{"sha256":"0d8da7333549a998556c18eb2af3ce902c28d66ceb947505c008f91e9f988abd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220342,"upload-time":"2018-08-14T21:01:20.870660Z","url":"https://files.pythonhosted.org/packages/20/6e/a9a0f84bc3efe970b4c8688b7e7f14ee8342497de8a88cffd35bb485cdcc/psutil-5.4.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"cea2557ee6a9faa2c100947637ded68414e12b851633c4ce26e0311b2a2ed539"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224085,"upload-time":"2018-08-14T21:01:22.891149Z","url":"https://files.pythonhosted.org/packages/97/9e/c056abafcf0fc7ca5bddbc21ad1bb7c67889e16c088b9759f00b95fefcb4/psutil-5.4.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp37-cp37m-win32.whl","hashes":{"sha256":"215d61a901e67b1a35e14c6aedef317f7fa7e6075a20c150fd11bd2c906d2c83"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220347,"upload-time":"2018-08-14T21:01:25.157286Z","url":"https://files.pythonhosted.org/packages/2e/33/5cef36162d94cf0adce428729adeb18b8548ff060781854f3aca71e6b0f0/psutil-5.4.7-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"51057c03aea251ad6667c2bba259bc7ed3210222d3a74152c84e3ab06e1da0ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224087,"upload-time":"2018-08-14T21:01:27.563142Z","url":"https://files.pythonhosted.org/packages/bb/15/aa3d11ae8bf04b7683224f7d3b8f2dd4d3f8a918dcce59bb1f987fca9c6e/psutil-5.4.7-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.7.tar.gz","hashes":{"sha256":"5b6322b167a5ba0c5463b4d30dfd379cd4ce245a1162ebf8fc7ab5c5ffae4f3b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":420300,"upload-time":"2018-08-14T21:01:30.063470Z","url":"https://files.pythonhosted.org/packages/7d/9a/1e93d41708f8ed2b564395edfa3389f0fd6d567597401c2e5e2775118d8b/psutil-5.4.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"data-dist-info-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"filename":"psutil-5.4.8-cp27-none-win32.whl","hashes":{"sha256":"809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220106,"upload-time":"2018-10-30T09:57:34.685390Z","url":"https://files.pythonhosted.org/packages/5a/3f/3f0920df352dae7f824e0e612ff02591378f78405d6c7663dcac023005c4/psutil-5.4.8-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"data-dist-info-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"filename":"psutil-5.4.8-cp27-none-win_amd64.whl","hashes":{"sha256":"3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":223347,"upload-time":"2018-10-30T09:57:37.065973Z","url":"https://files.pythonhosted.org/packages/0f/fb/6aecd2c8c9d0ac83d789eaf9f9ec052dd61dd5aea2b47ffa4704175d7a2a/psutil-5.4.8-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp35-cp35m-win32.whl","hashes":{"sha256":"bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222713,"upload-time":"2018-10-30T09:57:39.214229Z","url":"https://files.pythonhosted.org/packages/46/2e/ce4ec4b60decc23e0e4d148b6f44c7ddd06ba0ab207dfaee21958bd669df/psutil-5.4.8-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226441,"upload-time":"2018-10-30T09:57:41.411069Z","url":"https://files.pythonhosted.org/packages/7f/28/5ccdb98eff12e7741cc2a6d9dcfdd5d9e06f6d363c2c019d5bfa0e0c1282/psutil-5.4.8-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp36-cp36m-win32.whl","hashes":{"sha256":"3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222711,"upload-time":"2018-10-30T09:57:43.901154Z","url":"https://files.pythonhosted.org/packages/b5/31/8ac896ca77a6aa75ee900698f96ddce46e96bb2484a92457c359a4e4bae6/psutil-5.4.8-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226440,"upload-time":"2018-10-30T09:57:46.331527Z","url":"https://files.pythonhosted.org/packages/3b/15/62d1eeb4c015e20295e0197f7de0202bd9e5bcb5529b9503932decde2505/psutil-5.4.8-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp37-cp37m-win32.whl","hashes":{"sha256":"bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222709,"upload-time":"2018-10-30T09:57:48.860424Z","url":"https://files.pythonhosted.org/packages/21/1e/fe6731e5f03ddf2e57d5b307f25bba294262bc88e27a0fbefdb3515d1727/psutil-5.4.8-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226447,"upload-time":"2018-10-30T09:57:50.676641Z","url":"https://files.pythonhosted.org/packages/50/00/ae52663b879333aa5c65fc9a87ddc24169f8fdd1831762a1ba9c9be7740d/psutil-5.4.8-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.8.tar.gz","hashes":{"sha256":"6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":422742,"upload-time":"2018-10-30T09:57:52.770639Z","url":"https://files.pythonhosted.org/packages/e3/58/0eae6e4466e5abf779d7e2b71fac7fba5f59e00ea36ddb3ed690419ccb0f/psutil-5.4.8.tar.gz","yanked":false},{"core-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"data-dist-info-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"filename":"psutil-5.5.0-cp27-none-win32.whl","hashes":{"sha256":"96f3fdb4ef7467854d46ad5a7e28eb4c6dc6d455d751ddf9640cd6d52bdb03d7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":221314,"upload-time":"2019-01-23T18:23:33.526811Z","url":"https://files.pythonhosted.org/packages/fa/53/53f8c4f1af6f81b169ce76bfd0f56698bc1705da498a47d2ce701f7d7fe3/psutil-5.5.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"data-dist-info-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"filename":"psutil-5.5.0-cp27-none-win_amd64.whl","hashes":{"sha256":"d23f7025bac9b3e38adc6bd032cdaac648ac0074d18e36950a04af35458342e8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224546,"upload-time":"2019-01-23T18:23:36.443393Z","url":"https://files.pythonhosted.org/packages/86/96/a7bfcc3aebedd7112ff353204901db6a1a0c1f3555b2788c68842bb78005/psutil-5.5.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp35-cp35m-win32.whl","hashes":{"sha256":"04d2071100aaad59f9bcbb801be2125d53b2e03b1517d9fed90b45eea51d297e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224084,"upload-time":"2019-01-23T18:23:39.132334Z","url":"https://files.pythonhosted.org/packages/c6/ca/f5d3841ca35e3e3607ed64fe61d2c392054692f05f35e807335299a7952b/psutil-5.5.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d0c4230d60376aee0757d934020b14899f6020cd70ef8d2cb4f228b6ffc43e8f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227536,"upload-time":"2019-01-23T18:23:42.027389Z","url":"https://files.pythonhosted.org/packages/86/0d/a13c15ddccd8e2ccabe63f6d4f139f5a94150b758026e030310e87dded80/psutil-5.5.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp36-cp36m-win32.whl","hashes":{"sha256":"3ac48568f5b85fee44cd8002a15a7733deca056a191d313dbf24c11519c0c4a8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224085,"upload-time":"2019-01-23T18:23:44.450649Z","url":"https://files.pythonhosted.org/packages/45/00/7cdd50ded02e18e50667e2f76ceb645ecdfce59deb39422485f198c7be37/psutil-5.5.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"f0fcb7d3006dd4d9ccf3ccd0595d44c6abbfd433ec31b6ca177300ee3f19e54e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227537,"upload-time":"2019-01-23T18:23:46.937101Z","url":"https://files.pythonhosted.org/packages/48/d1/c9105512328c7f9800c51992b912df6f945eac696dfcd850f719541f67f3/psutil-5.5.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp37-cp37m-win32.whl","hashes":{"sha256":"c8ee08ad1b716911c86f12dc753eb1879006224fd51509f077987bb6493be615"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224078,"upload-time":"2019-01-23T18:23:49.451606Z","url":"https://files.pythonhosted.org/packages/3f/14/5adcad73f22ae0c8fba8b054c0bb7c33c906121b588cc6cbdd686f098947/psutil-5.5.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"b755be689d6fc8ebc401e1d5ce5bac867e35788f10229e166338484eead51b12"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227542,"upload-time":"2019-01-23T18:23:52.035332Z","url":"https://files.pythonhosted.org/packages/38/f1/a822d2b3d973c1ddd9d8a81d269e36987bab20e7bb28ecaa55aef66e8df5/psutil-5.5.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.5.0.tar.gz","hashes":{"sha256":"1aba93430050270750d046a179c5f3d6e1f5f8b96c20399ba38c596b28fc4d37"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":425058,"upload-time":"2019-01-23T18:23:54.951599Z","url":"https://files.pythonhosted.org/packages/6e/a0/833bcbcede5141cc5615e50c7cc5b960ce93d9c9b885fbe3b7d36e48a2d4/psutil-5.5.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"data-dist-info-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"filename":"psutil-5.5.1-cp27-none-win32.whl","hashes":{"sha256":"77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":221914,"upload-time":"2019-02-15T19:34:04.528250Z","url":"https://files.pythonhosted.org/packages/cc/cd/64aaf20c945662260026a128a08e46b93a49953224c0dccfdc6f37495d45/psutil-5.5.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"data-dist-info-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"filename":"psutil-5.5.1-cp27-none-win_amd64.whl","hashes":{"sha256":"5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":225325,"upload-time":"2019-02-15T19:34:07.193210Z","url":"https://files.pythonhosted.org/packages/08/92/97b011d665ade1caf05dd02a3af4ede751c7b80f34812bc81479ec867d85/psutil-5.5.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp35-cp35m-win32.whl","hashes":{"sha256":"a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224822,"upload-time":"2019-02-15T19:34:09.528347Z","url":"https://files.pythonhosted.org/packages/6f/05/70f033e35cd34bc23a08793eaf713347c615674585ccfc7628b40eac4094/psutil-5.5.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228328,"upload-time":"2019-02-15T19:34:12.035704Z","url":"https://files.pythonhosted.org/packages/8e/8d/1854ed30b9f69f4b2cc04ef4364ae8a52ad2988a3223bf6314d2d47f0f04/psutil-5.5.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp36-cp36m-win32.whl","hashes":{"sha256":"ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224831,"upload-time":"2019-02-15T19:34:15.031823Z","url":"https://files.pythonhosted.org/packages/e8/c1/32fe16cb90192a9413f3ba303021047c158cbdde5aabd7e26ace8b54f69e/psutil-5.5.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228328,"upload-time":"2019-02-15T19:34:17.494390Z","url":"https://files.pythonhosted.org/packages/3e/6e/c0af4900f18811f09b93064588e53f3997abc051ae43f717d1ba610de3b7/psutil-5.5.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp37-cp37m-win32.whl","hashes":{"sha256":"c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224832,"upload-time":"2019-02-15T19:34:19.685175Z","url":"https://files.pythonhosted.org/packages/00/e6/561fed27453add44af41a52e13e1dfca4d1e35705d698769edea6292339a/psutil-5.5.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228327,"upload-time":"2019-02-15T19:34:22.218113Z","url":"https://files.pythonhosted.org/packages/3d/22/ed4fa46c5bfd95b4dc57d6544c3fe6568abe398aef3990f6011777f1a3f3/psutil-5.5.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.5.1.tar.gz","hashes":{"sha256":"72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":426750,"upload-time":"2019-02-15T19:34:24.841527Z","url":"https://files.pythonhosted.org/packages/c7/01/7c30b247cdc5ba29623faa5c8cf1f1bbf7e041783c340414b0ed7e067c64/psutil-5.5.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"data-dist-info-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"filename":"psutil-5.6.0-cp27-none-win32.whl","hashes":{"sha256":"1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222534,"upload-time":"2019-03-05T12:00:59.141291Z","url":"https://files.pythonhosted.org/packages/04/f5/11b1c93a8882615fdaf6222aaf9d3197f250ab3036d7ecf6b6c8594ddf61/psutil-5.6.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"data-dist-info-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"filename":"psutil-5.6.0-cp27-none-win_amd64.whl","hashes":{"sha256":"d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":225803,"upload-time":"2019-03-05T12:01:01.831664Z","url":"https://files.pythonhosted.org/packages/11/88/ed94a7c091fb6ad8fbf545f0f20e140c47286712d6d85dd8cfc40b34fe72/psutil-5.6.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp35-cp35m-win32.whl","hashes":{"sha256":"c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226153,"upload-time":"2019-03-05T12:01:04.695139Z","url":"https://files.pythonhosted.org/packages/ab/5c/dacbb4b6623dc390939e1785a818e6e816eb42c8c96c1f2d30a2fc22a773/psutil-5.6.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230078,"upload-time":"2019-03-05T12:01:07.429607Z","url":"https://files.pythonhosted.org/packages/41/27/9e6d39ae822387ced9a77da904e24e8796c1fa55c5f637cd35221b170980/psutil-5.6.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp36-cp36m-win32.whl","hashes":{"sha256":"1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226154,"upload-time":"2019-03-05T12:01:10.226359Z","url":"https://files.pythonhosted.org/packages/fb/d0/14b83939ed41c5a80dca6fa072b1a99cd576e33810f691e73a08a7c045b4/psutil-5.6.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230083,"upload-time":"2019-03-05T12:01:12.681476Z","url":"https://files.pythonhosted.org/packages/ce/3a/ff53c0ee59a864c3614fcaea45f4246e670934ea0d30b632c6e3905533c9/psutil-5.6.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp37-cp37m-win32.whl","hashes":{"sha256":"86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226154,"upload-time":"2019-03-05T12:01:15.084497Z","url":"https://files.pythonhosted.org/packages/17/15/bf444a07aae2200f7b4e786c73bfe959201cb9ab63f737be12d21b5f252e/psutil-5.6.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230081,"upload-time":"2019-03-05T12:01:17.936041Z","url":"https://files.pythonhosted.org/packages/88/fd/a32491e77b37ffc00faf79c864975cfd720ae0ac867d93c90640d44adf43/psutil-5.6.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.0.tar.gz","hashes":{"sha256":"dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":426596,"upload-time":"2019-03-05T12:01:20.833448Z","url":"https://files.pythonhosted.org/packages/79/e6/a4e3c92fe19d386dcc6149dbf0b76f1c93c5491ae9d9ecf866f6769b45a4/psutil-5.6.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"data-dist-info-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"filename":"psutil-5.6.1-cp27-none-win32.whl","hashes":{"sha256":"23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":223000,"upload-time":"2019-03-11T17:24:48.368476Z","url":"https://files.pythonhosted.org/packages/9d/f2/a84e650af7f04940709466384e94a0894cfe736e7cf43f48fb3bfb01be1b/psutil-5.6.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"data-dist-info-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"filename":"psutil-5.6.1-cp27-none-win_amd64.whl","hashes":{"sha256":"e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226302,"upload-time":"2019-03-11T17:24:51.027456Z","url":"https://files.pythonhosted.org/packages/46/51/4007e6188b6d6b68c8b4195d6755bd76585cffd4d286d9f1815ff9f1af01/psutil-5.6.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp35-cp35m-win32.whl","hashes":{"sha256":"ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226675,"upload-time":"2019-03-11T17:24:53.625833Z","url":"https://files.pythonhosted.org/packages/55/f4/2dc147b66111116f5cb6a85fe42519b2cbfdbf6138562f8b0427bc754fc8/psutil-5.6.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230613,"upload-time":"2019-03-11T17:24:56.387303Z","url":"https://files.pythonhosted.org/packages/af/dd/f3bdafb37af7bf417f984db5381a5a27899f93fd2f87d5fd10fb6f3f4087/psutil-5.6.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp36-cp36m-win32.whl","hashes":{"sha256":"d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226672,"upload-time":"2019-03-11T17:24:58.629155Z","url":"https://files.pythonhosted.org/packages/d5/cc/e3fe388fe3c987973e929cb17cceb33bcd2ff7f8b754cd354826fbb7dfe7/psutil-5.6.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230621,"upload-time":"2019-03-11T17:25:01.330880Z","url":"https://files.pythonhosted.org/packages/16/6a/cc5ba8d7e3ada0d4621d493dbdcb43ed38f3549642916a14c9e070add21a/psutil-5.6.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp37-cp37m-win32.whl","hashes":{"sha256":"27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226673,"upload-time":"2019-03-11T17:25:03.865746Z","url":"https://files.pythonhosted.org/packages/a8/8e/3e04eefe955ed94f93c3cde5dc1f1ccd99e2b9a56697fa804ea9f54f7baa/psutil-5.6.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230621,"upload-time":"2019-03-11T17:25:06.996226Z","url":"https://files.pythonhosted.org/packages/6a/48/dbcda6d136da319e8bee8196e6c52ff7febf56bd241435cf6a516341a4b1/psutil-5.6.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.1.tar.gz","hashes":{"sha256":"fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":427472,"upload-time":"2019-03-11T17:25:09.802408Z","url":"https://files.pythonhosted.org/packages/2f/b8/11ec5006d2ec2998cb68349b8d1317c24c284cf918ecd6729739388e4c56/psutil-5.6.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"data-dist-info-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"filename":"psutil-5.6.2-cp27-none-win32.whl","hashes":{"sha256":"76fb0956d6d50e68e3f22e7cc983acf4e243dc0fcc32fd693d398cb21c928802"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226692,"upload-time":"2019-04-26T02:43:03.614779Z","url":"https://files.pythonhosted.org/packages/56/8f/1bfb7e563e413f110ee06ac0cf12bb4b27f78e9cf277892d97ba08de4eac/psutil-5.6.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"data-dist-info-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"filename":"psutil-5.6.2-cp27-none-win_amd64.whl","hashes":{"sha256":"753c5988edc07da00dafd6d3d279d41f98c62cd4d3a548c4d05741a023b0c2e7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230225,"upload-time":"2019-04-26T02:43:08.318806Z","url":"https://files.pythonhosted.org/packages/91/3f/2ae9cd04b2ccc5340838383ba638839a498d2613936b7830079f77de2bf1/psutil-5.6.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp35-cp35m-win32.whl","hashes":{"sha256":"a4c62319ec6bf2b3570487dd72d471307ae5495ce3802c1be81b8a22e438b4bc"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230751,"upload-time":"2019-04-26T02:43:12.014098Z","url":"https://files.pythonhosted.org/packages/ec/39/33a7d6c4e347ffff7784ec4967c04d6319886c5a77c208d29a0980045bc8/psutil-5.6.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"ef342cb7d9b60e6100364f50c57fa3a77d02ff8665d5b956746ac01901247ac4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234724,"upload-time":"2019-04-26T02:43:15.850786Z","url":"https://files.pythonhosted.org/packages/a9/43/87e610adacc3f8e51d61c27a9d48db2c0e7fbac783764fefca4d4ec71dbe/psutil-5.6.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp36-cp36m-win32.whl","hashes":{"sha256":"acba1df9da3983ec3c9c963adaaf530fcb4be0cd400a8294f1ecc2db56499ddd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230749,"upload-time":"2019-04-26T02:43:19.804035Z","url":"https://files.pythonhosted.org/packages/23/2a/0f14e964507adc1732ade6eea3abd92afe34305614515ab856cbccca489a/psutil-5.6.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"206eb909aa8878101d0eca07f4b31889c748f34ed6820a12eb3168c7aa17478e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234728,"upload-time":"2019-04-26T02:43:23.354778Z","url":"https://files.pythonhosted.org/packages/62/b0/54effe77128bdd8b62ca10edf38c22dbe5594ad8b34ce31836011949ac0a/psutil-5.6.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp37-cp37m-win32.whl","hashes":{"sha256":"649f7ffc02114dced8fbd08afcd021af75f5f5b2311bc0e69e53e8f100fe296f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230750,"upload-time":"2019-04-26T02:43:26.831308Z","url":"https://files.pythonhosted.org/packages/86/90/ea3d046bb26aebcbf25fba32cf9ec7d0954ea6b8e4e9d9c87a31633dd96b/psutil-5.6.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"6ebf2b9c996bb8c7198b385bade468ac8068ad8b78c54a58ff288cd5f61992c7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234731,"upload-time":"2019-04-26T02:43:30.698781Z","url":"https://files.pythonhosted.org/packages/98/3c/65f28f78848730c18dfa3ff3a107e3911b6d51d1442cfce8db53356179c3/psutil-5.6.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.2.tar.gz","hashes":{"sha256":"828e1c3ca6756c54ac00f1427fdac8b12e21b8a068c3bb9b631a1734cada25ed"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":432907,"upload-time":"2019-04-26T02:43:34.455685Z","url":"https://files.pythonhosted.org/packages/c6/c1/beed5e4eaa1345901b595048fab1c85aee647ea0fc02d9e8bf9aceb81078/psutil-5.6.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"data-dist-info-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"filename":"psutil-5.6.3-cp27-none-win32.whl","hashes":{"sha256":"d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226847,"upload-time":"2019-06-11T04:24:07.412816Z","url":"https://files.pythonhosted.org/packages/da/76/7e445566f2c4363691a98f16df0072c6ba92c7c29b4410ef2df6514c9861/psutil-5.6.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"data-dist-info-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"filename":"psutil-5.6.3-cp27-none-win_amd64.whl","hashes":{"sha256":"b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230406,"upload-time":"2019-06-11T04:24:10.852056Z","url":"https://files.pythonhosted.org/packages/72/75/43047d7df3ea2af2bcd072e63420b8fa240b729c052295f8c4b964335d36/psutil-5.6.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp35-cp35m-win32.whl","hashes":{"sha256":"cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230912,"upload-time":"2019-06-11T04:24:14.533883Z","url":"https://files.pythonhosted.org/packages/c2/f2/d99eaeefb2c0b1f7f9aca679db17f4072d4cc362f40f809157d4e2d273dd/psutil-5.6.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234942,"upload-time":"2019-06-11T04:24:18.128893Z","url":"https://files.pythonhosted.org/packages/90/86/d5ae0eb79cab6acc00d3640a45243e3e0602dc2f7abca29fc2fe6b4819ca/psutil-5.6.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp36-cp36m-win32.whl","hashes":{"sha256":"eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230914,"upload-time":"2019-06-11T04:24:21.735106Z","url":"https://files.pythonhosted.org/packages/28/0c/e41fd3020662487cf92f0c47b09285d7e53c42ee56cdc3ddb03a147cfa5d/psutil-5.6.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234945,"upload-time":"2019-06-11T04:24:26.649579Z","url":"https://files.pythonhosted.org/packages/86/91/f15a3aae2af13f008ed95e02292d1a2e84615ff42b7203357c1c0bbe0651/psutil-5.6.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp37-cp37m-win32.whl","hashes":{"sha256":"503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230914,"upload-time":"2019-06-11T04:24:30.881936Z","url":"https://files.pythonhosted.org/packages/3b/92/2a7fb18054ac12483fb72f281c285d21642ca3d29fc6a06f0e44d4b36d83/psutil-5.6.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234949,"upload-time":"2019-06-11T04:24:34.722512Z","url":"https://files.pythonhosted.org/packages/7c/58/f5d68ddca37480d8557b8566a20bf6108d7e1c6c9b9208ee0786e0cd012b/psutil-5.6.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"991ea9c126c59fb12521c83a1676786ad9a7ea2af4f3985b1e8086af3eda4156"},"data-dist-info-metadata":{"sha256":"991ea9c126c59fb12521c83a1676786ad9a7ea2af4f3985b1e8086af3eda4156"},"filename":"psutil-5.6.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"12542c3642909f4cd1928a2fba59e16fa27e47cbeea60928ebb62a8cbd1ce123"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238041,"upload-time":"2019-10-24T10:11:31.818649Z","url":"https://files.pythonhosted.org/packages/a7/7f/0761489b5467af4e97ae3c5a25f24f8662d69a25692d85490c8ea5d52e45/psutil-5.6.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.3.tar.gz","hashes":{"sha256":"863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":435374,"upload-time":"2019-06-11T04:24:39.293855Z","url":"https://files.pythonhosted.org/packages/1c/ca/5b8c1fe032a458c2c4bcbe509d1401dca9dda35c7fc46b36bb81c2834740/psutil-5.6.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"data-dist-info-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"filename":"psutil-5.6.4-cp27-none-win32.whl","hashes":{"sha256":"75d50d1138b2476a11dca33ab1ad2b78707d428418b581966ccedac768358f72"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228290,"upload-time":"2019-11-04T08:38:28.711265Z","url":"https://files.pythonhosted.org/packages/8a/63/0273163f0197ff2d8f026ae0f9c7cac371f8819f6451e6cfd2e1e7132b90/psutil-5.6.4-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"data-dist-info-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"filename":"psutil-5.6.4-cp27-none-win_amd64.whl","hashes":{"sha256":"0ff1f630ee0df7c048ef53e50196437d2c9cebab8ccca0e3078d9300c4b7da47"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231797,"upload-time":"2019-11-04T08:38:33.155488Z","url":"https://files.pythonhosted.org/packages/db/d0/17a47a1876cf408d0dd679f78ba18a206228fdb95928137dd85538a51298/psutil-5.6.4-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp35-cp35m-win32.whl","hashes":{"sha256":"10175ea15b7e4a1bf1a0863da7e17042862b3ea3e7d24285c96fa4cc65ab9788"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232325,"upload-time":"2019-11-04T08:38:37.308918Z","url":"https://files.pythonhosted.org/packages/cb/22/e2ce6539342fb0ebd65fb9e2a0a168585eb5cae9bd5dca7977735f8d430e/psutil-5.6.4-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f6b66a5663700b71bac3d8ecf6533a1550a679823e63b2c92dc4c3c8c244c52e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236342,"upload-time":"2019-11-04T08:38:41.583480Z","url":"https://files.pythonhosted.org/packages/32/9a/beaf5df1f8b38d3ec7ec737f873515813586aa26926738896957c3af0a9d/psutil-5.6.4-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp36-cp36m-win32.whl","hashes":{"sha256":"4f637dd25d3bce4879d0b4032d13f4120ba18ed2d028e85d911d429f447c251c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232330,"upload-time":"2019-11-04T08:38:46.049782Z","url":"https://files.pythonhosted.org/packages/74/3c/61d46eccf9fca27fc11f30d27af11c3886609d5eb52475926a1ad12d6ebd/psutil-5.6.4-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"43f0d7536a98c20a538242ce2bd8c64dbc1f6c396e97f2bdceb496d7583b9b80"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236347,"upload-time":"2019-11-04T08:38:50.265170Z","url":"https://files.pythonhosted.org/packages/2c/59/beae1392ad5188b419709d3e04641cbe93e36184b7b8686af825a2232b2b/psutil-5.6.4-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp37-cp37m-win32.whl","hashes":{"sha256":"f0ec1a3ea56503f4facc1dca364cf3dd66dc39169c4603000d3d34270e05fbb3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-04T08:38:54.615037Z","url":"https://files.pythonhosted.org/packages/e6/2e/e0c14d0fa46afb15b1467daa792f143e675034f8c544d03a2b7365d4926c/psutil-5.6.4-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"512e77ac987105e2d7aa2386d9f260434ad8b71e41484f8d84bfecd4ae3764ca"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236347,"upload-time":"2019-11-04T08:38:58.188759Z","url":"https://files.pythonhosted.org/packages/a3/59/fe32a3ec677990a5ed6b1a44dc2e372c9ee40693890814e8b6d96c290c4d/psutil-5.6.4-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp38-cp38-win32.whl","hashes":{"sha256":"41d645f100c6b4c995ff342ef7d79a936f3f48e9a816d7d655c69b352460341d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234629,"upload-time":"2019-11-04T08:46:40.809419Z","url":"https://files.pythonhosted.org/packages/50/0c/7b2fa0baedd36147beb820f662844b1079e168c06e9fe62d78a4d2666375/psutil-5.6.4-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp38-cp38-win_amd64.whl","hashes":{"sha256":"fb58e87c29ec0fb99937b95c5d473bb786d263aaa767d017a6bd4ad52d694e79"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239057,"upload-time":"2019-11-04T08:46:45.192939Z","url":"https://files.pythonhosted.org/packages/e3/28/fa3a597ed798a5a2cb9c9e53c2d8b6bc1310c4a147c2aaa65c5d3afdb6ed/psutil-5.6.4-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.4.tar.gz","hashes":{"sha256":"512e854d68f8b42f79b2c7864d997b39125baff9bcff00028ce43543867de7c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447564,"upload-time":"2019-11-04T08:39:02.419203Z","url":"https://files.pythonhosted.org/packages/47/ea/d3b6d6fd0b4a6c12984df652525f394e68c8678d2b05075219144eb3a1cf/psutil-5.6.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"data-dist-info-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"filename":"psutil-5.6.5-cp27-none-win32.whl","hashes":{"sha256":"145e0f3ab9138165f9e156c307100905fd5d9b7227504b8a9d3417351052dc3d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228289,"upload-time":"2019-11-06T10:07:27.672551Z","url":"https://files.pythonhosted.org/packages/97/52/9a44db00d4400814384d574f3a2c2b588259c212e457f542c2589dbdac58/psutil-5.6.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"data-dist-info-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"filename":"psutil-5.6.5-cp27-none-win_amd64.whl","hashes":{"sha256":"3feea46fbd634a93437b718518d15b5dd49599dfb59a30c739e201cc79bb759d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231796,"upload-time":"2019-11-06T10:07:31.973662Z","url":"https://files.pythonhosted.org/packages/c4/35/4422a966d304faa401d55dc45caf342722a14c0a9b56d57ecf208f9bb6a3/psutil-5.6.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp35-cp35m-win32.whl","hashes":{"sha256":"348ad4179938c965a27d29cbda4a81a1b2c778ecd330a221aadc7bd33681afbd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232325,"upload-time":"2019-11-06T10:07:36.463607Z","url":"https://files.pythonhosted.org/packages/4c/2f/93ac7f065f8c4c361e422dffff21dea1803cb40f2449e6ea52800caddf9f/psutil-5.6.5-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"474e10a92eeb4100c276d4cc67687adeb9d280bbca01031a3e41fb35dfc1d131"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236340,"upload-time":"2019-11-06T10:07:41.723247Z","url":"https://files.pythonhosted.org/packages/9f/49/fddf3b6138a5d47668e2daf98ac9a6eeea4c28e3eb68b56580f18e8fe6af/psutil-5.6.5-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp36-cp36m-win32.whl","hashes":{"sha256":"e3f5f9278867e95970854e92d0f5fe53af742a7fc4f2eba986943345bcaed05d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-06T10:07:45.647021Z","url":"https://files.pythonhosted.org/packages/ae/5d/11bb7fb7cc004bdf1325c0b40827e67e479ababe47c553ee871494353acf/psutil-5.6.5-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"dfb8c5c78579c226841908b539c2374da54da648ee5a837a731aa6a105a54c00"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236344,"upload-time":"2019-11-06T10:07:49.683064Z","url":"https://files.pythonhosted.org/packages/c6/18/221e8a5084585c6a2550894fb0617dc4691b5216f4aa6bb82c330aa5d99c/psutil-5.6.5-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp37-cp37m-win32.whl","hashes":{"sha256":"021d361439586a0fd8e64f8392eb7da27135db980f249329f1a347b9de99c695"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-06T10:07:53.427002Z","url":"https://files.pythonhosted.org/packages/31/f8/d612f18fed1422a016bb641d3ce1922904a2aa0cc3472ce4abf2716cf542/psutil-5.6.5-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"e9649bb8fc5cea1f7723af53e4212056a6f984ee31784c10632607f472dec5ee"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236346,"upload-time":"2019-11-06T10:07:57.218779Z","url":"https://files.pythonhosted.org/packages/97/c7/9b18c3b429c987796d74647c61030c7029518ac223d1a664951417781fe4/psutil-5.6.5-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp38-cp38-win32.whl","hashes":{"sha256":"47aeb4280e80f27878caae4b572b29f0ec7967554b701ba33cd3720b17ba1b07"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234627,"upload-time":"2019-11-06T10:08:50.811061Z","url":"https://files.pythonhosted.org/packages/03/94/e4ee514cfbc4cca176fcc6b4b1118a724848b570941e90f0b98a9bd234e1/psutil-5.6.5-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp38-cp38-win_amd64.whl","hashes":{"sha256":"73a7e002781bc42fd014dfebb3fc0e45f8d92a4fb9da18baea6fb279fbc1d966"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239056,"upload-time":"2019-11-06T10:08:55.334494Z","url":"https://files.pythonhosted.org/packages/cd/3b/de8a1f1692de2f16716a108a63366aa66692e5a087b6e0458eef9739e652/psutil-5.6.5-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.5.tar.gz","hashes":{"sha256":"d051532ac944f1be0179e0506f6889833cf96e466262523e57a871de65a15147"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447489,"upload-time":"2019-11-06T10:08:01.639000Z","url":"https://files.pythonhosted.org/packages/03/9a/95c4b3d0424426e5fd94b5302ff74cea44d5d4f53466e1228ac8e73e14b4/psutil-5.6.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"data-dist-info-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"filename":"psutil-5.6.6-cp27-none-win32.whl","hashes":{"sha256":"06660136ab88762309775fd47290d7da14094422d915f0466e0adf8e4b22214e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228564,"upload-time":"2019-11-25T12:30:48.823010Z","url":"https://files.pythonhosted.org/packages/c9/c2/4d703fcb5aacd4cb8d472d3c1d0e7d8c21e17f37b515016a7ee34ff3f4a0/psutil-5.6.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"data-dist-info-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"filename":"psutil-5.6.6-cp27-none-win_amd64.whl","hashes":{"sha256":"f21a7bb4b207e4e7c60b3c40ffa89d790997619f04bbecec9db8e3696122bc78"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232093,"upload-time":"2019-11-25T12:30:53.306595Z","url":"https://files.pythonhosted.org/packages/76/4d/6452c4791f9d95b48cca084b7cc6aa8a72b2f4c8d3d8bd38e7f3abfaf364/psutil-5.6.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp35-cp35m-win32.whl","hashes":{"sha256":"5e8dbf31871b0072bcba8d1f2861c0ec6c84c78f13c723bb6e981bce51b58f12"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232818,"upload-time":"2019-11-25T12:30:58.414546Z","url":"https://files.pythonhosted.org/packages/f3/b3/e585b9b0c5a40e6a778e32e8d3040ab2363433990417202bf6cc0261ed77/psutil-5.6.6-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"724390895cff80add7a1c4e7e0a04d9c94f3ee61423a2dcafd83784fabbd1ee9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236751,"upload-time":"2019-11-25T12:31:03.059024Z","url":"https://files.pythonhosted.org/packages/73/38/a8ebf6dc6ada2257591284be45a52dcd19479163b8d3575186333a79a18e/psutil-5.6.6-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp36-cp36m-win32.whl","hashes":{"sha256":"6d81b9714791ef9a3a00b2ca846ee547fc5e53d259e2a6258c3d2054928039ff"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232821,"upload-time":"2019-11-25T12:31:08.161481Z","url":"https://files.pythonhosted.org/packages/5b/04/2223b4fe61d3e5962c08ce5062b09633fcfdd8c3bb08c31b76306f748431/psutil-5.6.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"3004361c6b93dbad71330d992c1ae409cb8314a6041a0b67507cc882357f583e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236759,"upload-time":"2019-11-25T12:31:13.077581Z","url":"https://files.pythonhosted.org/packages/33/6c/6eb959bca82064d42e725dddd3aeeb39d9bed34eed7b513880bcfd8a3d59/psutil-5.6.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp37-cp37m-win32.whl","hashes":{"sha256":"0fc7a5619b47f74331add476fbc6022d7ca801c22865c7069ec0867920858963"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232820,"upload-time":"2019-11-25T12:31:17.411233Z","url":"https://files.pythonhosted.org/packages/76/46/3a8dc20eb9d7d2c44178e71c2412dcc2c6476ab71c05a2cf2f77247c6e53/psutil-5.6.6-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"f60042bef7dc50a78c06334ca8e25580455948ba2fa98f240d034a4fed9141a5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236757,"upload-time":"2019-11-25T12:31:21.816790Z","url":"https://files.pythonhosted.org/packages/40/48/5debf9783077ac71f0d715c1171fde9ef287909f61672ed9a3e5fdca63cc/psutil-5.6.6-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp38-cp38-win32.whl","hashes":{"sha256":"0c11adde31011a286197630ba2671e34651f004cc418d30ae06d2033a43c9e20"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233147,"upload-time":"2019-11-25T12:31:26.294340Z","url":"https://files.pythonhosted.org/packages/c6/10/035c432a15bec90d51f3625d4b70b7d12f1b363062d0d8815213229f69ca/psutil-5.6.6-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp38-cp38-win_amd64.whl","hashes":{"sha256":"0c211eec4185725847cb6c28409646c7cfa56fdb531014b35f97b5dc7fe04ff9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":237165,"upload-time":"2019-11-25T12:31:30.835876Z","url":"https://files.pythonhosted.org/packages/95/81/fb02ea8de73eca26cfa347f5ce81a7963b9dee6e038e0a7389ccbc971093/psutil-5.6.6-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.6.tar.gz","hashes":{"sha256":"ad21281f7bd6c57578dd53913d2d44218e9e29fd25128d10ff7819ef16fa46e7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447805,"upload-time":"2019-11-25T12:31:36.347718Z","url":"https://files.pythonhosted.org/packages/5f/dc/edf6758183afc7591a16bd4b8a44d8eea80aca1327ea60161dd3bad9ad22/psutil-5.6.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"data-dist-info-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"filename":"psutil-5.6.7-cp27-none-win32.whl","hashes":{"sha256":"1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228449,"upload-time":"2019-11-26T07:25:43.864472Z","url":"https://files.pythonhosted.org/packages/13/a7/626f257d22168c954fd3ad69760c02bdec27c0648a62f6ea5060c4d40672/psutil-5.6.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"data-dist-info-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"filename":"psutil-5.6.7-cp27-none-win_amd64.whl","hashes":{"sha256":"28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231969,"upload-time":"2019-11-26T07:25:50.426501Z","url":"https://files.pythonhosted.org/packages/52/44/e1e1954da522ea8640e035c8b101c116a9f8a0e94e04e108e56911064de5/psutil-5.6.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp35-cp35m-win32.whl","hashes":{"sha256":"21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232644,"upload-time":"2019-11-26T07:25:56.230501Z","url":"https://files.pythonhosted.org/packages/d8/39/bf74da6282d9521fe3987b2d67f581b3464e635e6cb56660d0315d1bf1ed/psutil-5.6.7-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236538,"upload-time":"2019-11-26T07:26:01.396655Z","url":"https://files.pythonhosted.org/packages/b3/84/0a899c6fac13aedfb4734413a2357a504c4f21cbf8acd7ad4caa6712d8cf/psutil-5.6.7-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp36-cp36m-win32.whl","hashes":{"sha256":"e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232649,"upload-time":"2019-11-26T07:26:06.727306Z","url":"https://files.pythonhosted.org/packages/7c/d7/be2b607abfab4a98f04dd2155d6a7a40a666618d69c079897f09ce776a34/psutil-5.6.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236541,"upload-time":"2019-11-26T07:26:11.229459Z","url":"https://files.pythonhosted.org/packages/78/e6/ce8a91afd605f254342f1294790f2a77c76202386d6927eb5ff0e36e4449/psutil-5.6.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp37-cp37m-win32.whl","hashes":{"sha256":"094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232645,"upload-time":"2019-11-26T07:26:15.814275Z","url":"https://files.pythonhosted.org/packages/c4/e1/80c7840db569ad5b1b60987893e066a5536779c0d2402363cbf1230613a2/psutil-5.6.7-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236539,"upload-time":"2019-11-26T07:26:20.217732Z","url":"https://files.pythonhosted.org/packages/9d/84/0a2006cc263e9f5b6dfbb2301fbcce5558f0d6d17d0c11c7c6749a45c79e/psutil-5.6.7-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp38-cp38-win32.whl","hashes":{"sha256":"70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232983,"upload-time":"2019-11-26T07:26:24.615122Z","url":"https://files.pythonhosted.org/packages/55/9d/9a6df5f730a1e2a3938fad0ccf541b30fad34706128b43ed3f965eaf7550/psutil-5.6.7-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp38-cp38-win_amd64.whl","hashes":{"sha256":"10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236965,"upload-time":"2019-11-26T07:26:28.827408Z","url":"https://files.pythonhosted.org/packages/8a/fa/b573850e912d6ffdad4aef3f5f705f94a64d098a83eec15d1cd3e1223f5e/psutil-5.6.7-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.7.tar.gz","hashes":{"sha256":"ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":448321,"upload-time":"2019-11-26T07:26:34.515073Z","url":"https://files.pythonhosted.org/packages/73/93/4f8213fbe66fc20cb904f35e6e04e20b47b85bee39845cc66a0bcf5ccdcb/psutil-5.6.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"data-dist-info-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"filename":"psutil-5.7.0-cp27-none-win32.whl","hashes":{"sha256":"298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227641,"upload-time":"2020-02-18T18:02:31.085536Z","url":"https://files.pythonhosted.org/packages/0b/6b/f613593812c5f379c6d609bf5eca36a409812f508e13c704acd25712a73e/psutil-5.7.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"data-dist-info-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"filename":"psutil-5.7.0-cp27-none-win_amd64.whl","hashes":{"sha256":"75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230741,"upload-time":"2020-02-18T18:02:35.453910Z","url":"https://files.pythonhosted.org/packages/79/b1/377fa0f28630d855cb6b5bfb2ee4c1bf0df3bc2603c691ceefce59a95181/psutil-5.7.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp35-cp35m-win32.whl","hashes":{"sha256":"f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231151,"upload-time":"2020-02-18T18:02:39.410543Z","url":"https://files.pythonhosted.org/packages/74/e6/4a0ef10b1a4ca43954cd8fd9eac02cc8606f9d2a5a66859a283f5f95452b/psutil-5.7.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235435,"upload-time":"2020-02-18T18:02:43.014979Z","url":"https://files.pythonhosted.org/packages/65/c2/0aeb9f0cc7e4be2807aa052b3fd017e59439ed6d830b461f8ecb35b2f367/psutil-5.7.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp36-cp36m-win32.whl","hashes":{"sha256":"a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231156,"upload-time":"2020-02-18T18:02:46.432233Z","url":"https://files.pythonhosted.org/packages/c9/37/b94930ae428b2d67d505aecc5ba84c53a0b75479a8a87cd35cc9a2c6eb7e/psutil-5.7.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235439,"upload-time":"2020-02-18T18:02:49.890797Z","url":"https://files.pythonhosted.org/packages/4f/3c/205850b172a14a8b9fdc9b1e84a2c055d6b9aea226431da7685bea644f04/psutil-5.7.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp37-cp37m-win32.whl","hashes":{"sha256":"d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231153,"upload-time":"2020-02-18T18:02:53.653943Z","url":"https://files.pythonhosted.org/packages/54/25/7825fefd62635f7ca556c8e0d44369ce4674aa2ca0eca50b8ae4ff49954b/psutil-5.7.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235439,"upload-time":"2020-02-18T18:02:57.139440Z","url":"https://files.pythonhosted.org/packages/86/f7/385040b90dd190edc28908c4a26af99b00ae37564ee5f5c4526dc1d80c27/psutil-5.7.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp38-cp38-win32.whl","hashes":{"sha256":"60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231789,"upload-time":"2020-02-18T18:03:00.808478Z","url":"https://files.pythonhosted.org/packages/63/d5/f34a9433a0299d944605fb5a970306a89e076f5412164179dc59ebf70fa9/psutil-5.7.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235952,"upload-time":"2020-02-18T18:03:04.130414Z","url":"https://files.pythonhosted.org/packages/86/fe/9f1d1f8c1c8138d42fc0e7c06ca5004e01f38e86e61342374d8e0fa919e4/psutil-5.7.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.0.tar.gz","hashes":{"sha256":"685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":449628,"upload-time":"2020-02-18T18:03:07.566242Z","url":"https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"2dfb7b5638ffaa33602a86b39cca60cded2324dabbe2617b1b5e65250e448769"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233646,"upload-time":"2020-07-15T11:14:11.635539Z","url":"https://files.pythonhosted.org/packages/79/e4/cbaa3ecc458c2dd8da64073de983473543b8b6ef4ca21159cea9069d53dd/psutil-5.7.1-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-macosx_10_9_x86_64.whl","hashes":{"sha256":"36c5e6882caf3d385c6c3a0d2f3b302b4cc337c808ea589d9a8c563b545beb8b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233862,"upload-time":"2020-07-15T11:14:27.607855Z","url":"https://files.pythonhosted.org/packages/4d/d9/48c3d16c1dfbbf528bd69254b5a604c9f6860f12169b1b73a5005723c6bf/psutil-5.7.1-cp35-cp35m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-win32.whl","hashes":{"sha256":"3c5ffd00bc1ee809350dca97613985d387a7e13dff61d62fc1bdf4dc10892ddd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238370,"upload-time":"2020-07-15T11:14:34.746888Z","url":"https://files.pythonhosted.org/packages/d0/e2/d4cdadda6a9fba79026ab628fc2b4da5e2e48dcfc6beada0a39363732ba1/psutil-5.7.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"4975c33aebe7de191d745ee3c545e907edd14d65c850a0b185c05024aa77cbcd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242862,"upload-time":"2020-07-15T11:14:37.084757Z","url":"https://files.pythonhosted.org/packages/a0/6b/cdb41805a6bb62c051cfbb1b65a9cb40767e0144b3d40fdd7082d8271701/psutil-5.7.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"436a6e99098eba14b54a149f921c9d4e1df729f02645876af0c828396d36c46a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233861,"upload-time":"2020-07-15T11:14:39.242780Z","url":"https://files.pythonhosted.org/packages/1f/fb/097aeed40c361225cb69d6d04202421f2c172d7e42753130d1b619f68956/psutil-5.7.1-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-win32.whl","hashes":{"sha256":"630ceda48c16b24ffd981fe06ae1a43684af1a3a837d6a3496a1be3dd3c7d332"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238378,"upload-time":"2020-07-15T11:14:45.606142Z","url":"https://files.pythonhosted.org/packages/ae/49/cba9353fd9946eac95031c85763daaf7904d3c3e8b0b4f2801199586b413/psutil-5.7.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d3bb7f65199595a72a3ec53e4d05c159857ab832fadaae9d85e68db467d2d191"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242864,"upload-time":"2020-07-15T11:14:47.861971Z","url":"https://files.pythonhosted.org/packages/7b/38/f27fc6a30f81be1ee657bd4c355c2dc03a5fbb49f304a37c79c0bed05821/psutil-5.7.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"66d085317599684f70d995dd4a770894f518fb34d027d7f742b579bf47732858"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233863,"upload-time":"2020-07-15T11:14:49.991128Z","url":"https://files.pythonhosted.org/packages/23/cb/410a516385c8cd69f090f98c8014636c51d124c96e4d6ab51e1bb2d04232/psutil-5.7.1-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-win32.whl","hashes":{"sha256":"fb442b912fe28d80e0f966adcc3df4e394fbb7ef7575ae21fd171aeb06c8b0df"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238369,"upload-time":"2020-07-15T11:14:57.383010Z","url":"https://files.pythonhosted.org/packages/59/44/e9cfa470dd2790b5475ceb590949842a5f2feb52445e898576b721033f04/psutil-5.7.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"425d6c95ca3ece7ff4da7e67af2954b8eb56b0f15743b237dc84ad975f51c2a4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242865,"upload-time":"2020-07-15T11:14:59.839217Z","url":"https://files.pythonhosted.org/packages/06/76/b4607e0eaf36369ad86f7ac73bde19aeaf32c82fb22675cb8f8dd975c692/psutil-5.7.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"f2817a763c33c19fdefbb832c790bc85b3de90b51fb69dae43097a9885be0332"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234129,"upload-time":"2020-07-15T11:15:01.982258Z","url":"https://files.pythonhosted.org/packages/ba/f7/64bf7fd7a12a40c50408b7d90cdf3addc28071e5463af1dbb7f3884a32d2/psutil-5.7.1-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-win32.whl","hashes":{"sha256":"3cf43d2265ee03fcf70f0f574487ed19435c92a330e15a3e773144811c1275f0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239004,"upload-time":"2020-07-15T11:15:08.125783Z","url":"https://files.pythonhosted.org/packages/7e/64/c3bd24d53f6056ded095e8d147c0ca269bb6d858aea903561bd660d67035/psutil-5.7.1-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-win_amd64.whl","hashes":{"sha256":"006b720a67881037c8b02b1de012a39a2f007bd2b1b244b58fabef8eff0ad6d2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243418,"upload-time":"2020-07-15T11:15:10.044651Z","url":"https://files.pythonhosted.org/packages/ff/5a/1e990cf86f47721225143ed4a903a226685fa1ba0b43500f52d71500b1be/psutil-5.7.1-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"621e2f3b573aa563717ee39c3f052b4cd3cda5e1f25e5973fa77e727433c6ce1"},"data-dist-info-metadata":{"sha256":"621e2f3b573aa563717ee39c3f052b4cd3cda5e1f25e5973fa77e727433c6ce1"},"filename":"psutil-5.7.1-pp27-pypy_73-macosx_10_9_x86_64.whl","hashes":{"sha256":"0c9187ec0c314a128362c3409afea2b80c6d6d2c2cb1d661fe20631a2ff8ad77"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232076,"upload-time":"2020-07-15T11:15:12.408818Z","url":"https://files.pythonhosted.org/packages/80/fd/d91ff7582513d093097678eedd141a4879698da26e6163fbae16905aa75b/psutil-5.7.1-pp27-pypy_73-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.1.tar.gz","hashes":{"sha256":"4ef6845b35e152e6937d4f28388c2440ca89a0089ced0a30a116fa3ceefdfa3a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":460148,"upload-time":"2020-07-15T11:15:21.985314Z","url":"https://files.pythonhosted.org/packages/2c/5c/cb95a715fb635e1ca858ffb8c50a523a16e2dc06aa3e207ab73cb93516af/psutil-5.7.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"data-dist-info-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"filename":"psutil-5.7.2-cp27-none-win32.whl","hashes":{"sha256":"f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234782,"upload-time":"2020-07-15T13:19:05.321836Z","url":"https://files.pythonhosted.org/packages/d0/da/d7da0365f690e7555f6dda34bcb5bde10266379c9a23ee6a0735c3a7fdfd/psutil-5.7.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"data-dist-info-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"filename":"psutil-5.7.2-cp27-none-win_amd64.whl","hashes":{"sha256":"66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238050,"upload-time":"2020-07-15T13:19:07.922423Z","url":"https://files.pythonhosted.org/packages/7d/9d/30a053a06d598cee4bdbc6ba69df44ced9e6d2ebb16e2de401a2a3bc6d63/psutil-5.7.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp35-cp35m-win32.whl","hashes":{"sha256":"5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238373,"upload-time":"2020-07-15T13:19:10.146779Z","url":"https://files.pythonhosted.org/packages/df/27/e5cf14b0894b4f06c23dc4f58288c60b17e71d8bef9af463f0b32ee46773/psutil-5.7.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242862,"upload-time":"2020-07-15T13:19:12.906781Z","url":"https://files.pythonhosted.org/packages/0a/4c/d31d58992314664e69bda6d575c1fd47b86ed5d67e00e300fc909040a9aa/psutil-5.7.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp36-cp36m-win32.whl","hashes":{"sha256":"d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238380,"upload-time":"2020-07-15T13:19:15.247084Z","url":"https://files.pythonhosted.org/packages/57/c5/0aa3b1513b914a417db7ee149b60579a139111f81f79f5f1d38ae440cebf/psutil-5.7.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242866,"upload-time":"2020-07-15T13:19:17.782781Z","url":"https://files.pythonhosted.org/packages/22/f8/7be159475303a508347efc82c0d5858b1786fe73fc2a6b21d82891791920/psutil-5.7.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp37-cp37m-win32.whl","hashes":{"sha256":"ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238375,"upload-time":"2020-07-15T13:19:20.591814Z","url":"https://files.pythonhosted.org/packages/56/de/6f07749f275d0ba7f9b985cd6a4526e2fa47ad63b5179948c6650117f7d9/psutil-5.7.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242863,"upload-time":"2020-07-15T13:19:23.022907Z","url":"https://files.pythonhosted.org/packages/f8/9b/1d7df5e1747e047abef4ec877d895b642f3a796ab8bd2e0f682516740dfe/psutil-5.7.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp38-cp38-win32.whl","hashes":{"sha256":"10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239003,"upload-time":"2020-07-15T13:19:25.770960Z","url":"https://files.pythonhosted.org/packages/da/d6/f66bbdbc8831a5cc78ba0e9bf69d924e68eac1a7b4191de93cf4e3643c54/psutil-5.7.2-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp38-cp38-win_amd64.whl","hashes":{"sha256":"68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243419,"upload-time":"2020-07-15T13:19:28.240707Z","url":"https://files.pythonhosted.org/packages/6c/e6/f963547a36a96f74244cbe5e4046a02f140e3b7cbc5e5176035b38e2deb2/psutil-5.7.2-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.2.tar.gz","hashes":{"sha256":"90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":460198,"upload-time":"2020-07-15T13:19:30.438785Z","url":"https://files.pythonhosted.org/packages/aa/3e/d18f2c04cf2b528e18515999b0c8e698c136db78f62df34eee89cee205f1/psutil-5.7.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"data-dist-info-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"filename":"psutil-5.7.3-cp27-none-win32.whl","hashes":{"sha256":"1cd6a0c9fb35ece2ccf2d1dd733c1e165b342604c67454fd56a4c12e0a106787"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235225,"upload-time":"2020-10-24T14:00:54.139761Z","url":"https://files.pythonhosted.org/packages/1d/a2/b732590561ef9d7dbc078ed0e2635e282115604a478911fef97ddaa3ad43/psutil-5.7.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"data-dist-info-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"filename":"psutil-5.7.3-cp27-none-win_amd64.whl","hashes":{"sha256":"e02c31b2990dcd2431f4524b93491941df39f99619b0d312dfe1d4d530b08b4b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238499,"upload-time":"2020-10-24T14:00:57.724543Z","url":"https://files.pythonhosted.org/packages/b4/4c/c14a9485957b00c20f70e208a03663e81ddc8dafdf5137fee2d50aa1ee5e/psutil-5.7.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp35-cp35m-win32.whl","hashes":{"sha256":"56c85120fa173a5d2ad1d15a0c6e0ae62b388bfb956bb036ac231fbdaf9e4c22"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238813,"upload-time":"2020-10-24T15:02:21.678008Z","url":"https://files.pythonhosted.org/packages/e0/ba/a7c1096470e11c449019690ee9e7fd3adca1a4b9cfa6e5a13b60db3187b4/psutil-5.7.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"fa38ac15dbf161ab1e941ff4ce39abd64b53fec5ddf60c23290daed2bc7d1157"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243298,"upload-time":"2020-10-24T15:02:24.823245Z","url":"https://files.pythonhosted.org/packages/3f/7c/98aada1208462c841788712383f4288b3c31e45504570a818c0c303d78e7/psutil-5.7.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp36-cp36m-win32.whl","hashes":{"sha256":"01bc82813fbc3ea304914581954979e637bcc7084e59ac904d870d6eb8bb2bc7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238821,"upload-time":"2020-10-24T15:02:27.844759Z","url":"https://files.pythonhosted.org/packages/44/a8/ebfcbb4967e74a27049ea6e13b3027ae05c0cb73d1a2b71c2f0519c6d5f2/psutil-5.7.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"6a3e1fd2800ca45083d976b5478a2402dd62afdfb719b30ca46cd28bb25a2eb4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243300,"upload-time":"2020-10-24T15:02:31.552663Z","url":"https://files.pythonhosted.org/packages/9a/c3/3b0023b46fc038eff02fbb69a0e6e50d15a7dce25e717d8469e8eaa837a7/psutil-5.7.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp37-cp37m-win32.whl","hashes":{"sha256":"fbcac492cb082fa38d88587d75feb90785d05d7e12d4565cbf1ecc727aff71b7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238814,"upload-time":"2020-10-24T15:02:34.703606Z","url":"https://files.pythonhosted.org/packages/e1/f0/d4f58ddf077d970440b82b92e909e8e9b2f50e39a2dc2aa716b1e2fde5ef/psutil-5.7.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"5d9106ff5ec2712e2f659ebbd112967f44e7d33f40ba40530c485cc5904360b8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243302,"upload-time":"2020-10-24T15:02:38.016752Z","url":"https://files.pythonhosted.org/packages/f1/a0/094a6e32185bd1288a4681d91ebe362d5b41aa64413bbbd96ed547051f17/psutil-5.7.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp38-cp38-win32.whl","hashes":{"sha256":"ade6af32eb80a536eff162d799e31b7ef92ddcda707c27bbd077238065018df4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239426,"upload-time":"2020-10-24T15:02:41.250786Z","url":"https://files.pythonhosted.org/packages/bd/95/394485321a128f5ddb23f0a559f940309280f57bd6117580868fb2d5a246/psutil-5.7.3-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"2cb55ef9591b03ef0104bedf67cc4edb38a3edf015cf8cf24007b99cb8497542"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243849,"upload-time":"2020-10-24T15:02:44.162783Z","url":"https://files.pythonhosted.org/packages/df/64/8d7b55ac87e67398ffc260d43a5fb327f1e230b09758b7d8caaecf917dd6/psutil-5.7.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.3.tar.gz","hashes":{"sha256":"af73f7bcebdc538eda9cc81d19db1db7bf26f103f91081d780bbacfcb620dee2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":465556,"upload-time":"2020-10-24T14:02:15.604830Z","url":"https://files.pythonhosted.org/packages/33/e0/82d459af36bda999f82c7ea86c67610591cf5556168f48fd6509e5fa154d/psutil-5.7.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235772,"upload-time":"2020-12-19T01:19:27.017908Z","url":"https://files.pythonhosted.org/packages/f5/7f/a2559a514bdeb2a33e4bf3dc3d2bb17d5acded718893869a82536130cfb3/psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284461,"upload-time":"2020-12-19T01:19:30.158384Z","url":"https://files.pythonhosted.org/packages/19/2c/9f1bad783faee4e9704868f381913e68dbb69f0de3fcdc71ee7071c47847/psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287768,"upload-time":"2020-12-19T01:19:33.394809Z","url":"https://files.pythonhosted.org/packages/82/0a/eddb9a51ba5055cc7c242da07c1643a6b146070740c5eb5540277a0f01f4/psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284476,"upload-time":"2020-12-19T01:19:36.479406Z","url":"https://files.pythonhosted.org/packages/15/28/47c28171fd7eeb83df74f78ccac090211f4a49408f376eb8e78a7bb47dc0/psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287754,"upload-time":"2020-12-19T01:19:39.219693Z","url":"https://files.pythonhosted.org/packages/26/ef/461e9eec56fba7fa66692c4af00cbd6547b788a7ca818d9b8b5f1951f228/psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"data-dist-info-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"filename":"psutil-5.8.0-cp27-none-win32.whl","hashes":{"sha256":"ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236561,"upload-time":"2020-12-19T01:19:41.916204Z","url":"https://files.pythonhosted.org/packages/a8/b3/6a21c5b7e4f600bd6eaaecd4a5e76230fa34876e48cbc87b2cef0ab91c0a/psutil-5.8.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"data-dist-info-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"filename":"psutil-5.8.0-cp27-none-win_amd64.whl","hashes":{"sha256":"5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239900,"upload-time":"2020-12-19T01:19:45.046259Z","url":"https://files.pythonhosted.org/packages/b2/3d/01ef1f4bf71413078bf2ce2aae04d47bc132cfede58738183a9de41aa122/psutil-5.8.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236013,"upload-time":"2020-12-19T01:19:47.734642Z","url":"https://files.pythonhosted.org/packages/30/81/37ebe0ba2840b76681072e786bae3319cade8a6861029d0ae885c274fa0b/psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl","hashes":{"sha256":"74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289247,"upload-time":"2020-12-19T01:19:50.429849Z","url":"https://files.pythonhosted.org/packages/2e/7c/13a6c3f068aa39ffafd99ae159c1a345521e7dd0074ccadb917e5670dbdc/psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl","hashes":{"sha256":"99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291875,"upload-time":"2020-12-19T01:19:53.450222Z","url":"https://files.pythonhosted.org/packages/da/82/56cd16a4c5f53e3e5dd7b2c30d5c803e124f218ebb644ca9c30bc907eadd/psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-win32.whl","hashes":{"sha256":"36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240337,"upload-time":"2020-12-19T01:19:56.748668Z","url":"https://files.pythonhosted.org/packages/19/29/f7a38ee30083f2caa14cc77a6d34c4d5cfd1a69641e87bf1b3d6ba90d0ba/psutil-5.8.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244835,"upload-time":"2020-12-19T01:19:59.102544Z","url":"https://files.pythonhosted.org/packages/44/ed/49d75a29007727d44937ed4d233f116be346bc4657a83b5a9e2f423bca57/psutil-5.8.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236013,"upload-time":"2020-12-19T01:20:02.260367Z","url":"https://files.pythonhosted.org/packages/fe/19/83ab423a7b69cafe4078dea751acdff7377e4b59c71e3718125ba3c341f9/psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl","hashes":{"sha256":"61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":290299,"upload-time":"2020-12-19T01:20:04.653817Z","url":"https://files.pythonhosted.org/packages/cc/5f/2a1967092086acc647962168d0e6fd1c22e14a973f03e3ffb1e2f0da5de9/psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl","hashes":{"sha256":"0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296329,"upload-time":"2020-12-19T01:20:07.174230Z","url":"https://files.pythonhosted.org/packages/84/da/f7efdcf012b51506938553dbe302aecc22f3f43abd5cffa8320e8e0588d5/psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-win32.whl","hashes":{"sha256":"1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240337,"upload-time":"2020-12-19T01:20:10.138302Z","url":"https://files.pythonhosted.org/packages/18/c9/1db6aa0d28831f60408a6aab9d108c2edbd5a9ed11e5957a91d9d8023898/psutil-5.8.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244834,"upload-time":"2020-12-19T01:20:12.840200Z","url":"https://files.pythonhosted.org/packages/71/ce/35107e81e7eae55c847313f872d4258a71d2640fa04f57c5520fc81473ce/psutil-5.8.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236297,"upload-time":"2020-12-19T01:20:15.489742Z","url":"https://files.pythonhosted.org/packages/10/d6/c5c19e40bb05e2cb5f053f480dfe47e9543a8322f1a5985d7352bf689611/psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl","hashes":{"sha256":"d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293809,"upload-time":"2020-12-19T01:20:18.539919Z","url":"https://files.pythonhosted.org/packages/e9/d6/7d0bcf272923f6b3433e22effd31860b63ab580d65fb2d8f5cb443a9e6fc/psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl","hashes":{"sha256":"28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296040,"upload-time":"2020-12-19T01:20:21.423284Z","url":"https://files.pythonhosted.org/packages/3b/c2/78109a12da9febb2f965abf29da6f81b0a3f2b89a7b59d88b759e68dc6db/psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-win32.whl","hashes":{"sha256":"ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240950,"upload-time":"2020-12-19T01:20:24.359373Z","url":"https://files.pythonhosted.org/packages/87/be/6511e1341c203608fe2553249216c40b92cd8a72d8b35fa3c1decee9a616/psutil-5.8.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245386,"upload-time":"2020-12-19T01:20:26.799194Z","url":"https://files.pythonhosted.org/packages/8e/5c/c4b32c2024daeac35e126b90a1ff7a0209ef8b32675d1d50e55d58e78c81/psutil-5.8.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236274,"upload-time":"2020-12-19T01:20:29.615503Z","url":"https://files.pythonhosted.org/packages/12/80/8d09c345f19af2b29a309f8f9284e3ba1ae1ebd9438419080c14630f743a/psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl","hashes":{"sha256":"245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291032,"upload-time":"2020-12-19T01:20:32.422979Z","url":"https://files.pythonhosted.org/packages/b6/2f/118e23a8f4e59d2c4ffe03a921cc72f364966e25548dc6c5a3011a334dc5/psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl","hashes":{"sha256":"90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293491,"upload-time":"2020-12-19T01:20:34.915141Z","url":"https://files.pythonhosted.org/packages/91/4d/033cc02ae3a47197d0ced818814e4bb8d9d29ebed4f1eb55badedec160f7/psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-win32.whl","hashes":{"sha256":"ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241508,"upload-time":"2020-12-19T01:20:37.527554Z","url":"https://files.pythonhosted.org/packages/a7/13/7285b74e061da21dfc4f15c8307eb2da1d2137367502d6598f03f4a5b5e7/psutil-5.8.0-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-win_amd64.whl","hashes":{"sha256":"f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246139,"upload-time":"2020-12-19T01:20:39.890196Z","url":"https://files.pythonhosted.org/packages/21/71/33cb528381c443df1ee25cbb451da975421bddb5099b11e7f2eb3fc90d6d/psutil-5.8.0-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.8.0.tar.gz","hashes":{"sha256":"0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":470886,"upload-time":"2020-12-19T01:20:42.916847Z","url":"https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285348,"upload-time":"2021-12-29T21:26:19.591062Z","url":"https://files.pythonhosted.org/packages/c9/62/5cfcb69c256d469236d4bddeb7ad4ee6a8b37d604dcfc82b7c938fd8ee37/psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288125,"upload-time":"2021-12-29T21:26:24.644701Z","url":"https://files.pythonhosted.org/packages/eb/0d/c19872c9121208bbbb4335bb13a4a2f2b95661fd69d24f26e32f94e5a8a1/psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285364,"upload-time":"2021-12-29T21:26:28.242468Z","url":"https://files.pythonhosted.org/packages/1a/3e/ff287d01bca130b72cf53a9b20bbc31bf566d503ee63adf8c7dcfd9315e2/psutil-5.9.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288110,"upload-time":"2021-12-29T21:26:31.963660Z","url":"https://files.pythonhosted.org/packages/d8/49/fbce284331d482703decdc8dec9bfd910fa00a3acd5b974e8efa8c30104a/psutil-5.9.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"data-dist-info-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"filename":"psutil-5.9.0-cp27-none-win32.whl","hashes":{"sha256":"ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239056,"upload-time":"2021-12-29T21:26:35.738857Z","url":"https://files.pythonhosted.org/packages/ab/d7/a8b076603943ebce7872ca7d4e012f6dcdc33e86eabb117921a6fe6e1f8a/psutil-5.9.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"data-dist-info-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"filename":"psutil-5.9.0-cp27-none-win_amd64.whl","hashes":{"sha256":"ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242517,"upload-time":"2021-12-29T21:26:39.603955Z","url":"https://files.pythonhosted.org/packages/2d/7a/ee32fa2c5712fa0bc6a9f376ffe9d2e1dc856e2e011d2bab4e12293dcd88/psutil-5.9.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238624,"upload-time":"2021-12-29T21:26:42.964503Z","url":"https://files.pythonhosted.org/packages/89/48/2c6f566d35a38fb9f882e51d75425a6f1d097cb946e05b6aff98d450a151/psutil-5.9.0-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279343,"upload-time":"2021-12-29T21:26:46.859457Z","url":"https://files.pythonhosted.org/packages/11/46/e790221e8281af5163517a17a20c88b10a75a5642d9c5106a868f2879edd/psutil-5.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281400,"upload-time":"2021-12-29T21:26:51.801761Z","url":"https://files.pythonhosted.org/packages/6f/8a/d1810472a4950a31df385eafbc9bd20cde971814ff6533021dc565bf14ae/psutil-5.9.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-win32.whl","hashes":{"sha256":"8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241383,"upload-time":"2021-12-29T21:26:55.364799Z","url":"https://files.pythonhosted.org/packages/61/93/4251cfa58e5bbd7f92e1bfb965a0c41376cbcbc83c524a8b60d2678f0edd/psutil-5.9.0-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-win_amd64.whl","hashes":{"sha256":"9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245540,"upload-time":"2021-12-29T21:26:59.088538Z","url":"https://files.pythonhosted.org/packages/9f/c9/7fb339d6a04db3b4ab94671536d11e03b23c056d1604e50e564075a96cd8/psutil-5.9.0-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238331,"upload-time":"2022-01-07T14:28:07.566595Z","url":"https://files.pythonhosted.org/packages/48/cb/6841d4f39b5711652a93359748879f2977ede55c1020f69d038891073592/psutil-5.9.0-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":277267,"upload-time":"2022-01-07T14:28:12.318473Z","url":"https://files.pythonhosted.org/packages/1f/2d/e6640979580db1b51220d3165e256a1d0a31847944a3e2622800a737fe86/psutil-5.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279702,"upload-time":"2022-01-07T14:28:16.706587Z","url":"https://files.pythonhosted.org/packages/64/87/461555057b080e1996427098a6c51c64a8a9025ec18571dabfe5be07eeec/psutil-5.9.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-win32.whl","hashes":{"sha256":"e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243193,"upload-time":"2022-01-07T14:28:20.695929Z","url":"https://files.pythonhosted.org/packages/2b/a3/24d36239a7bfa30b0eb4302b045417796e9f2c7c21b296d2405735e8949e/psutil-5.9.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247862,"upload-time":"2022-01-07T14:28:23.774803Z","url":"https://files.pythonhosted.org/packages/14/c9/f0bccd60a25197d63a02688ea8f5c5cd5a8b1baf1a7d6bf493d3291132d2/psutil-5.9.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238334,"upload-time":"2021-12-29T21:27:03.461413Z","url":"https://files.pythonhosted.org/packages/70/40/0a6ca5641f7574b6ea38cdb561c30065659734755a1779db67b56e225f84/psutil-5.9.0-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278077,"upload-time":"2021-12-29T21:27:07.864433Z","url":"https://files.pythonhosted.org/packages/6b/c0/0f233f87e816c20e5489bca749798255a464282cdd5911d62bb8344c4b5a/psutil-5.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280736,"upload-time":"2021-12-29T21:27:11.252493Z","url":"https://files.pythonhosted.org/packages/60/f9/b78291ed21146ece2417bd1ba715564c6d3bdf2f1e9297ed67709bb36eeb/psutil-5.9.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-win32.whl","hashes":{"sha256":"df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241810,"upload-time":"2021-12-29T21:27:14.020326Z","url":"https://files.pythonhosted.org/packages/47/3f/0475146306d02270243e55cad8167d5185c8918933953c90eda846d72ff3/psutil-5.9.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246380,"upload-time":"2021-12-29T21:27:17.444889Z","url":"https://files.pythonhosted.org/packages/7c/d6/4ade7cebfe04710a89e2dc5638f712f09dc5e402a8fea95c3d16dc7f64bf/psutil-5.9.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238620,"upload-time":"2021-12-29T21:27:21.799788Z","url":"https://files.pythonhosted.org/packages/89/8e/2a8814f903bc06471621f6e0cd3fc1a7085868656106f31aacf2f844eea2/psutil-5.9.0-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281246,"upload-time":"2021-12-29T21:27:24.591382Z","url":"https://files.pythonhosted.org/packages/4c/95/3c0858c62ec02106cf5f3e79d74223264a6269a16996f31d5ab43abcec86/psutil-5.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":283823,"upload-time":"2021-12-29T21:27:27.809766Z","url":"https://files.pythonhosted.org/packages/0a/66/b2188d8e738ee52206a4ee804907f6eab5bcc9fc0e8486e7ab973a8323b7/psutil-5.9.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-win32.whl","hashes":{"sha256":"76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242160,"upload-time":"2021-12-29T21:27:31.512894Z","url":"https://files.pythonhosted.org/packages/d0/cf/7a86fc08f821d66c528939f155079df7d0945678fc474c6a6455c909f6eb/psutil-5.9.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246488,"upload-time":"2021-12-29T21:27:36.422809Z","url":"https://files.pythonhosted.org/packages/62/d4/72fc44dfd9939851bd672e94e43d12848a98b1d2c3f6f794d54a220fe4a7/psutil-5.9.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238613,"upload-time":"2021-12-29T21:27:39.813718Z","url":"https://files.pythonhosted.org/packages/48/6a/c6e88a5584544033dbb8318c380e7e1e3796e5ac336577eb91dc75bdecd7/psutil-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278536,"upload-time":"2021-12-29T21:27:43.204962Z","url":"https://files.pythonhosted.org/packages/f7/b1/82e95f6368dbde6b7e54ea6b18cf8ac3958223540d0bcbde23ba7be19478/psutil-5.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280415,"upload-time":"2021-12-29T21:27:47.616041Z","url":"https://files.pythonhosted.org/packages/c4/35/7cec9647be077784d20913404f914fffd8fe6dfd0673e29f7bd822ac1331/psutil-5.9.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-win32.whl","hashes":{"sha256":"4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241428,"upload-time":"2021-12-29T21:27:52.616449Z","url":"https://files.pythonhosted.org/packages/5a/c6/923aed22f6c9c5197998fa6907c983e884975a0ae3430ccd8514f5fd0d6a/psutil-5.9.0-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-win_amd64.whl","hashes":{"sha256":"7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245606,"upload-time":"2021-12-29T21:27:56.202709Z","url":"https://files.pythonhosted.org/packages/9e/9e/3a48f15a1539505e2f3058a709eee56acfb379f2b0ff409d6291099e2a7e/psutil-5.9.0-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.0.tar.gz","hashes":{"sha256":"869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":478322,"upload-time":"2021-12-29T21:27:59.163343Z","url":"https://files.pythonhosted.org/packages/47/b6/ea8a7728f096a597f0032564e8013b705aa992a0990becd773dcc4d7b4a7/psutil-5.9.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286186,"upload-time":"2022-05-20T20:09:47.558524Z","url":"https://files.pythonhosted.org/packages/77/06/f9fd79449440d7217d6bf2c90998d540e125cfeffe39d214a328dadc46f4/psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288858,"upload-time":"2022-05-20T20:09:52.956913Z","url":"https://files.pythonhosted.org/packages/cf/29/ad704a45960bfb52ef8bf0beb9c41c09ce92d61c40333f03e9a03f246c22/psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"data-dist-info-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"filename":"psutil-5.9.1-cp27-cp27m-win32.whl","hashes":{"sha256":"0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239479,"upload-time":"2022-05-20T20:09:57.166474Z","url":"https://files.pythonhosted.org/packages/7e/8d/e0a66123fa98e309597815de518b47a7a6c571a8f886fc8d4db2331fd2ab/psutil-5.9.1-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"data-dist-info-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"filename":"psutil-5.9.1-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242812,"upload-time":"2022-05-20T20:10:01.868774Z","url":"https://files.pythonhosted.org/packages/1b/53/8f0772df0a6d593bc2fcdf12f4f790bab5c4f6a77bb61a8ddaad2cbba7f8/psutil-5.9.1-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286184,"upload-time":"2022-05-20T20:10:05.575313Z","url":"https://files.pythonhosted.org/packages/2d/56/54b4ed8102ce5a2f5367b4e766c1873c18f9c32cde321435d0e0ee2abcc5/psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288863,"upload-time":"2022-05-20T20:10:09.032899Z","url":"https://files.pythonhosted.org/packages/2c/9d/dc329b7da284677ea843f3ff4b35b8ab3b96b65a58a544b3c3f86d9d032f/psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239266,"upload-time":"2022-05-20T20:10:12.541628Z","url":"https://files.pythonhosted.org/packages/d1/16/6239e76ab5d990dc7866bc22a80585f73421588d63b42884d607f5f815e2/psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280127,"upload-time":"2022-05-20T20:10:16.449769Z","url":"https://files.pythonhosted.org/packages/14/06/39d7e963a6a8bbf26519de208593cdb0ddfe22918b8989f4b2363d4ab49f/psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282049,"upload-time":"2022-05-20T20:10:19.914788Z","url":"https://files.pythonhosted.org/packages/6d/c6/6a4e46802e8690d50ba6a56c7f79ac283e703fcfa0fdae8e41909c8cef1f/psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp310-cp310-win32.whl","hashes":{"sha256":"20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241699,"upload-time":"2022-05-20T20:10:23.278390Z","url":"https://files.pythonhosted.org/packages/26/b4/a58cf15ea649faa92c54f00c627aef1d50b9f1abf207485f10c967a50c95/psutil-5.9.1-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp310-cp310-win_amd64.whl","hashes":{"sha256":"58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245843,"upload-time":"2022-05-20T20:10:26.856840Z","url":"https://files.pythonhosted.org/packages/c0/5a/2ac88d5265b711c8aa4e786825b38d5d0b1e5ecbdd0ce78e9b04a820d247/psutil-5.9.1-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239007,"upload-time":"2022-05-20T20:10:30.191299Z","url":"https://files.pythonhosted.org/packages/65/1d/6a112f146faee6292a6c3ee2a7f24a8e572697adb7e1c5de3d8508f647cc/psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278074,"upload-time":"2022-05-20T20:10:33.843171Z","url":"https://files.pythonhosted.org/packages/7e/52/a02dc53e26714a339c8b4972d8e3f268e4db8905f5d1a3a100f1e40b6fa7/psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280402,"upload-time":"2022-05-20T20:10:38.665061Z","url":"https://files.pythonhosted.org/packages/6b/76/a8cb69ed3566877dcbccf408f5f9d6055227ad4fed694e88809fa8506b0b/psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-win32.whl","hashes":{"sha256":"0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243501,"upload-time":"2022-05-20T20:10:42.164047Z","url":"https://files.pythonhosted.org/packages/85/4d/78173e3dffb74c5fa87914908f143473d0b8b9183f9d275333679a4e4649/psutil-5.9.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248156,"upload-time":"2022-05-20T20:10:45.327961Z","url":"https://files.pythonhosted.org/packages/73/1a/d78f2f2de2aad6628415d2a48917cabc2c7fb0c3a31c7cdf187cffa4eb36/psutil-5.9.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238960,"upload-time":"2022-05-20T20:10:48.678116Z","url":"https://files.pythonhosted.org/packages/d6/ef/fd4dc9085e3879c3af63fe60667dd3b71adf50d030b5549315f4a619271b/psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278553,"upload-time":"2022-05-20T20:10:52.019471Z","url":"https://files.pythonhosted.org/packages/97/f6/0180e58dd1359da7d6fbc27d04dac6fb500dc758b6f4b65407608bb13170/psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281356,"upload-time":"2022-05-20T20:10:55.963864Z","url":"https://files.pythonhosted.org/packages/13/71/c25adbd9b33a2e27edbe1fc84b3111a5ad97611885d7abcbdd8d1f2bb7ca/psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp37-cp37m-win32.whl","hashes":{"sha256":"d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242105,"upload-time":"2022-05-20T20:10:59.100543Z","url":"https://files.pythonhosted.org/packages/2a/32/136cd5bf55728ea64a22b1d817890e35fc17314c46a24ee3268b65f9076f/psutil-5.9.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246671,"upload-time":"2022-05-20T20:11:02.513602Z","url":"https://files.pythonhosted.org/packages/df/88/427f3959855fcb3ab04891e00c026a246892feb11b20433db814b7a24405/psutil-5.9.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239250,"upload-time":"2022-05-20T20:11:06.292977Z","url":"https://files.pythonhosted.org/packages/46/80/1de3a9bac336b5c8e4f7b0ff2e80c85ba237f18f2703be68884ee6798432/psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281678,"upload-time":"2022-05-20T20:11:09.990719Z","url":"https://files.pythonhosted.org/packages/fd/ba/c5a3f46f351ab609cc0be6a563e492900c57e3d5c9bda0b79b84d8c3eae9/psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284665,"upload-time":"2022-05-20T20:11:13.584309Z","url":"https://files.pythonhosted.org/packages/9d/41/d5f2db2ab7f5dff2fa795993a0cd6fa8a8f39ca197c3a86857875333ec10/psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp38-cp38-win32.whl","hashes":{"sha256":"a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242461,"upload-time":"2022-05-20T20:11:16.942263Z","url":"https://files.pythonhosted.org/packages/41/ec/5fd3e9388d0ed1edfdeae71799df374f4a117932646a63413fa95a121e9f/psutil-5.9.1-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp38-cp38-win_amd64.whl","hashes":{"sha256":"79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246804,"upload-time":"2022-05-20T20:11:20.322459Z","url":"https://files.pythonhosted.org/packages/b2/ad/65e2b2b97677f98d718388dc11b2a9d7f177ebbae5eef72547a32bc28911/psutil-5.9.1-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239248,"upload-time":"2022-05-20T20:11:23.352312Z","url":"https://files.pythonhosted.org/packages/9f/ca/84ce3e48b3ca2f0f74314d89929b3a523220f3f4a8dff395d6ef74dadef3/psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279320,"upload-time":"2022-05-20T20:11:27.310813Z","url":"https://files.pythonhosted.org/packages/a9/97/b7e3532d97d527349701d2143c3f868733b94e2db6f531b07811b698f549/psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281065,"upload-time":"2022-05-20T20:11:30.617653Z","url":"https://files.pythonhosted.org/packages/62/1f/f14225bda76417ab9bd808ff21d5cd59d5435a9796ca09b34d4cb0edcd88/psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp39-cp39-win32.whl","hashes":{"sha256":"32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241748,"upload-time":"2022-05-20T20:11:34.374569Z","url":"https://files.pythonhosted.org/packages/b1/d2/c5374a784567c1e42ee8a589b1b42e2bd6e14c7be3c234d84360ab3a0a39/psutil-5.9.1-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp39-cp39-win_amd64.whl","hashes":{"sha256":"f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245883,"upload-time":"2022-05-20T20:11:37.840887Z","url":"https://files.pythonhosted.org/packages/e0/ac/fd6f098969d49f046083ac032e6788d9f861903596fb9555a02bf50a1238/psutil-5.9.1-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.1.tar.gz","hashes":{"sha256":"57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":479090,"upload-time":"2022-05-20T20:11:41.043143Z","url":"https://files.pythonhosted.org/packages/d6/de/0999ea2562b96d7165812606b18f7169307b60cd378bc29cf3673322c7e9/psutil-5.9.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285739,"upload-time":"2022-09-04T20:15:28.668835Z","url":"https://files.pythonhosted.org/packages/37/a4/cb10e4c0faa3091de22eb78fa1c332566e60b9b59001bef326a4c1070417/psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289461,"upload-time":"2022-09-04T20:15:32.802519Z","url":"https://files.pythonhosted.org/packages/93/40/58dfcab15435b6fedf5385bc7e88a4c162cc6af0056f5d9d97f5ebfd7fa0/psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"data-dist-info-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"filename":"psutil-5.9.2-cp27-cp27m-win32.whl","hashes":{"sha256":"b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240036,"upload-time":"2022-09-04T20:15:36.698093Z","url":"https://files.pythonhosted.org/packages/42/eb/83470960f2c13a026b07051456ad834f5fea0c80e8cb83fc65005f5f18d5/psutil-5.9.2-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"data-dist-info-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"filename":"psutil-5.9.2-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243378,"upload-time":"2022-09-04T20:15:40.746927Z","url":"https://files.pythonhosted.org/packages/d1/5b/b9d6ac192d3108e1dc7875ab1579b7f65eb7bf0ef799dadd3f3798d0af2e/psutil-5.9.2-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285740,"upload-time":"2022-09-04T20:15:44.420732Z","url":"https://files.pythonhosted.org/packages/b6/96/ddf877440f2686eb17933531507fe4822ff1ed76d85df4a093a605b91db8/psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289451,"upload-time":"2022-09-04T20:15:47.573981Z","url":"https://files.pythonhosted.org/packages/d7/df/ff5c766b50350f2a4555d5068127d372bb26201a2a5eeda9efc8dbf570b4/psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239273,"upload-time":"2022-09-04T20:15:52.215984Z","url":"https://files.pythonhosted.org/packages/04/5d/d52473097582db5d3094bc34acf9874de726327a3166426e22ed0806de6a/psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280571,"upload-time":"2022-09-04T20:15:55.608887Z","url":"https://files.pythonhosted.org/packages/47/2b/bd12c4f2d1bd3024fe7c5d8388f8a5627cc02fbe11d62bd451aff356415d/psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282777,"upload-time":"2022-09-04T20:15:59.840541Z","url":"https://files.pythonhosted.org/packages/4c/85/7a112fb6a8c598a6f5d079228bbc03ae84c472397be79c075e7514b6ed36/psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-win32.whl","hashes":{"sha256":"e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241713,"upload-time":"2022-09-05T14:16:44.931542Z","url":"https://files.pythonhosted.org/packages/39/07/5cbcf3322031fcf8dcbfa431b1c145f193c96b18964ef374a88d6a83f2c9/psutil-5.9.2-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-win_amd64.whl","hashes":{"sha256":"f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245853,"upload-time":"2022-09-05T14:16:48.388734Z","url":"https://files.pythonhosted.org/packages/ae/9c/d29dd82d5fda2c6c6d959d57101c78ddbac8325defe94e1b9f983e7cfff3/psutil-5.9.2-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238998,"upload-time":"2022-09-04T20:16:03.484887Z","url":"https://files.pythonhosted.org/packages/df/aa/8268eee572fb9bdf3486d384e3973ad9d635403841c6e7f2af7781e5525b/psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278226,"upload-time":"2022-09-04T20:16:07.134877Z","url":"https://files.pythonhosted.org/packages/f0/43/bcb92221f5dd45e155337aae37e412fe02a3e5d99e936156a4dcff89fa55/psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280489,"upload-time":"2022-09-04T20:16:10.463113Z","url":"https://files.pythonhosted.org/packages/a4/eb/d841d5bc526641aad65373b0a4850e98284580df967daff5288779090ea3/psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-win32.whl","hashes":{"sha256":"f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243511,"upload-time":"2022-09-05T14:16:51.624881Z","url":"https://files.pythonhosted.org/packages/54/5f/3619e7d22ded096fa6dbd329fc057bfcf53e998b1e2c1ecc07a4155175b1/psutil-5.9.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248164,"upload-time":"2022-09-05T14:16:55.091982Z","url":"https://files.pythonhosted.org/packages/53/ac/7c4ff994b1ea7d46a84932f0c8d49e28e36a668173975876353f4ea38588/psutil-5.9.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238970,"upload-time":"2022-09-04T20:16:14.127339Z","url":"https://files.pythonhosted.org/packages/55/c5/fd2c45a0845e7bae07c8112ed67c21163742cc116732ac2702d9139a9a92/psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279452,"upload-time":"2022-09-04T20:16:18.609472Z","url":"https://files.pythonhosted.org/packages/89/cf/b228a7554eda5e72fd8c33b89c628a86336e5cdbd62fe8b8d2a61a099b2d/psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281281,"upload-time":"2022-09-04T20:16:22.476894Z","url":"https://files.pythonhosted.org/packages/3d/73/d8c87b5612c58d1e6c6d91997c1590771d34e4ee27d9c11eb1e64ecbf365/psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-win32.whl","hashes":{"sha256":"7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242115,"upload-time":"2022-09-05T14:16:58.770138Z","url":"https://files.pythonhosted.org/packages/98/42/62470fae4e1e9c0f4336acf74af9d4a6d5c6b5788c8435ec387e987a7ebe/psutil-5.9.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246680,"upload-time":"2022-09-05T14:17:03.836879Z","url":"https://files.pythonhosted.org/packages/5e/a2/4025f29069010f118eba4bcd681167d547525d40d2c45029db2f64606f86/psutil-5.9.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239258,"upload-time":"2022-09-04T20:16:26.490949Z","url":"https://files.pythonhosted.org/packages/2b/52/c69f5d0acc4bbd3cf44178f025e498666d2eebc216f5f5725d9142244365/psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282623,"upload-time":"2022-09-04T20:16:29.707162Z","url":"https://files.pythonhosted.org/packages/c4/02/5fc4419f47f141ec0dd28db36fb8bcf1eb6e9df332690617b052c8bec76d/psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284813,"upload-time":"2022-09-04T20:16:33.456488Z","url":"https://files.pythonhosted.org/packages/79/61/a8d6d649996494672d8a86fe8be6c81b2880ee30881709d84435f2505b47/psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-win32.whl","hashes":{"sha256":"561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242471,"upload-time":"2022-09-05T14:17:07.677347Z","url":"https://files.pythonhosted.org/packages/6f/8d/41c402ae33b1ce3f8e37a0dec691d753cbe66e6784e7fd26ed0cd16d99ab/psutil-5.9.2-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-win_amd64.whl","hashes":{"sha256":"67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246816,"upload-time":"2022-09-05T14:17:11.540699Z","url":"https://files.pythonhosted.org/packages/29/07/a35c4127942cce6899d447cb54f9926d33cf1800a37c09192dd9b5a08744/psutil-5.9.2-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239256,"upload-time":"2022-09-04T20:16:37.181485Z","url":"https://files.pythonhosted.org/packages/65/74/0ad485d753b2f0d00ee4ec933da1e169bc4c8f4f58db88132e886efed14b/psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279700,"upload-time":"2022-09-04T20:16:40.816901Z","url":"https://files.pythonhosted.org/packages/bb/df/0819b9aed416b0dedf668cc6b3f291899c276cb2b566c4aa0dc212a03d55/psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281896,"upload-time":"2022-09-04T20:16:45.773716Z","url":"https://files.pythonhosted.org/packages/b3/61/54822666fbbdd4ae1825f7a0b0cf8925a96fac1f778b4a0d5c9c066cf4b2/psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-win32.whl","hashes":{"sha256":"ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241759,"upload-time":"2022-09-05T14:17:15.034289Z","url":"https://files.pythonhosted.org/packages/67/cf/f620f740da5bb5895b441248e08b0cd167fb545ecaa3e74ea06f3551975e/psutil-5.9.2-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-win_amd64.whl","hashes":{"sha256":"68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245892,"upload-time":"2022-09-05T14:17:18.376384Z","url":"https://files.pythonhosted.org/packages/10/cf/7595896a7487937c171f53bae2eeb0adcc1690ebeef684ac180a77910639/psutil-5.9.2-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.2.tar.gz","hashes":{"sha256":"feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":479757,"upload-time":"2022-09-04T20:16:49.093359Z","url":"https://files.pythonhosted.org/packages/8f/57/828ac1f70badc691a716e77bfae258ef5db76bb7830109bf4bcf882de020/psutil-5.9.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241919,"upload-time":"2022-10-18T20:12:47.255096Z","url":"https://files.pythonhosted.org/packages/74/42/6268344958236744962c711664de259598fe2005e5818c7d6bc77ae12690/psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293266,"upload-time":"2022-10-18T20:12:51.216028Z","url":"https://files.pythonhosted.org/packages/79/6a/7bb45dddeb348cdb9d91d7bc78e903026870ef7f257c35de250392719cf8/psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":297585,"upload-time":"2022-10-18T20:12:54.920684Z","url":"https://files.pythonhosted.org/packages/02/c7/d5a6106cf31cc58f4a8a9d88b1ab8405b645b02c482353dd59f5ef19926f/psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"data-dist-info-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"filename":"psutil-5.9.3-cp27-cp27m-win32.whl","hashes":{"sha256":"9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240677,"upload-time":"2022-10-18T20:12:58.145019Z","url":"https://files.pythonhosted.org/packages/62/0a/27aa8d95995fe97a944939f8fff7183f151814a1052b76d125812bed4800/psutil-5.9.3-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"data-dist-info-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"filename":"psutil-5.9.3-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244019,"upload-time":"2022-10-18T20:13:01.999625Z","url":"https://files.pythonhosted.org/packages/2b/0a/36951d279e1d716ab264b04e8ddb12e0c08cc1c7cbd44f2d22c84dc61e33/psutil-5.9.3-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293262,"upload-time":"2022-10-18T20:13:05.579750Z","url":"https://files.pythonhosted.org/packages/5b/9c/5412473100e3213d970c8b9291371816e57f1e4a74296b2e3b8a5c8ebb47/psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":297584,"upload-time":"2022-10-18T20:13:09.701506Z","url":"https://files.pythonhosted.org/packages/8f/5a/e9e98bb3ade26bc7847d5722d0e4a6d437621fa8fc02269d9cba78f6f241/psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242575,"upload-time":"2022-10-18T20:13:13.343655Z","url":"https://files.pythonhosted.org/packages/95/90/822c926e170e8a5769ff11edb92ac59dd523df505b5d56cad0ef3f15c325/psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl","hashes":{"sha256":"71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243240,"upload-time":"2022-10-18T20:13:17.345095Z","url":"https://files.pythonhosted.org/packages/42/9e/243aa51c3d71355913dafc27c5cb7ffdbe9a42c939a5aace526906bfc721/psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289285,"upload-time":"2022-10-18T20:13:20.535587Z","url":"https://files.pythonhosted.org/packages/f7/b0/6925fbfac4c342cb2f8bad1571b48e12802ac8031e1d4453a31e9a12b64d/psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":292339,"upload-time":"2022-10-18T20:13:26.635784Z","url":"https://files.pythonhosted.org/packages/ed/2c/483ed7332d74b3fef0f5ba13c192d33f21fe95df5468a7ca040f02bd7af9/psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp310-cp310-win32.whl","hashes":{"sha256":"1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242906,"upload-time":"2022-10-18T20:13:30.051727Z","url":"https://files.pythonhosted.org/packages/55/07/94730401200098b1119dc9f5d3a271e3bf865b31bfa64a2b58a0bbd9d222/psutil-5.9.3-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp310-cp310-win_amd64.whl","hashes":{"sha256":"547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247045,"upload-time":"2022-10-18T20:13:34.024950Z","url":"https://files.pythonhosted.org/packages/37/c0/8a102d4ce45dbc5d04932b52327c4385b88023635e57af9d457ca5ea6bb3/psutil-5.9.3-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242147,"upload-time":"2022-10-18T20:13:37.172733Z","url":"https://files.pythonhosted.org/packages/61/f2/74908ddbe57863007e3b3a76f39b509bbab9892d0949f1e9d5a888f8ec60/psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286758,"upload-time":"2022-10-18T20:13:40.932526Z","url":"https://files.pythonhosted.org/packages/30/2f/696c4459864385cc5c63a21f30584dfd99d2130c21c8b3084ffbaa0edd82/psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":290363,"upload-time":"2022-10-18T20:13:43.847649Z","url":"https://files.pythonhosted.org/packages/2c/80/2f3072492a7f14faf4f4565dd26fe1baf4b3fd28557f1427b6708064a622/psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"data-dist-info-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"filename":"psutil-5.9.3-cp36-cp36m-win32.whl","hashes":{"sha256":"828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244701,"upload-time":"2022-10-18T20:13:47.431806Z","url":"https://files.pythonhosted.org/packages/db/e3/10363d747d900f89f7920b8e4060b42cd862b580a69a2b9c9788c4de9035/psutil-5.9.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"data-dist-info-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"filename":"psutil-5.9.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":249356,"upload-time":"2022-10-18T20:13:51.794050Z","url":"https://files.pythonhosted.org/packages/ac/cc/092ca7ae0c5f270bb14720cd8ac86a3fafda25fae31d08d2465eed4498b3/psutil-5.9.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242154,"upload-time":"2022-10-18T20:14:12.425822Z","url":"https://files.pythonhosted.org/packages/16/51/d431f7db3a3a44d9c03ec1681835a5de52d2f0bb7e28f29ecd806ccc46ec/psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287383,"upload-time":"2022-10-18T20:14:15.962130Z","url":"https://files.pythonhosted.org/packages/94/b0/cd3be14dc74a6f262b1de296841a5141a794cc485d4e3af5c1c0ffc9b886/psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291396,"upload-time":"2022-10-18T20:14:19.855473Z","url":"https://files.pythonhosted.org/packages/5e/86/856aa554ec7eb843fb006ef125cf4543ee9058cb39ad09d131dd820c71f7/psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp37-cp37m-win32.whl","hashes":{"sha256":"beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243306,"upload-time":"2022-10-18T20:14:23.302434Z","url":"https://files.pythonhosted.org/packages/ab/10/547feeec01275dd544a389ba05ecb3c316015d4b402cc7b440ca2d98ebcd/psutil-5.9.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247870,"upload-time":"2022-10-18T20:14:26.971453Z","url":"https://files.pythonhosted.org/packages/0c/f1/50e71c11ef14c592686dfc60e2b42a381fe57af2d22713e66a72c07cf9d1/psutil-5.9.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242611,"upload-time":"2022-10-18T20:14:30.842925Z","url":"https://files.pythonhosted.org/packages/01/d6/9ca99b416dddf4a49855a9ebf4af3a2db9526e94e9693da169fa5ed61788/psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl","hashes":{"sha256":"46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243235,"upload-time":"2022-10-18T20:14:34.363643Z","url":"https://files.pythonhosted.org/packages/2c/ce/daf28e50305fdbba0754ba58ab0346ec6cfa41293110412f4c6bf74738bb/psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291620,"upload-time":"2022-10-18T20:14:38.179697Z","url":"https://files.pythonhosted.org/packages/b9/cf/56278ae450741b6390491aecaa5f6152ff491bf00544799830e98340ff48/psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":295598,"upload-time":"2022-10-18T20:14:41.691453Z","url":"https://files.pythonhosted.org/packages/af/5d/9c03a47af929fc12699fcf5174313744eef33a7b9e106e8111f57427b7d7/psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp38-cp38-win32.whl","hashes":{"sha256":"6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243663,"upload-time":"2022-10-18T20:14:45.084651Z","url":"https://files.pythonhosted.org/packages/69/3d/e1a12f505eb0171912b94e4689453639bb0deeb70ab4eddbc7b9266f819e/psutil-5.9.3-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248005,"upload-time":"2022-10-18T20:14:48.240035Z","url":"https://files.pythonhosted.org/packages/69/cf/47a028bbb4589fdc0494bc60f134c73e319ec78c86c37e2dc66fd118e4db/psutil-5.9.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242570,"upload-time":"2022-10-18T20:14:51.709705Z","url":"https://files.pythonhosted.org/packages/e5/64/ced1461fd5ebc944d90f9e471149991893bd7ede05b5a88069c1953738dc/psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl","hashes":{"sha256":"a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243242,"upload-time":"2022-10-18T20:14:54.641746Z","url":"https://files.pythonhosted.org/packages/ac/55/c108e74f22905382aeeef56110bd6c4b89b5fc64944d21cb83acb66faa4c/psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288533,"upload-time":"2022-10-18T20:14:57.902650Z","url":"https://files.pythonhosted.org/packages/db/6f/2441388c48306f9b9d561080c6ba652b4ebd1199faac237069ec8983c8ef/psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291401,"upload-time":"2022-10-18T20:15:01.503982Z","url":"https://files.pythonhosted.org/packages/03/47/15604dd812b1b860e81cabaf8c930474c549773389170cd03a093ecf54b6/psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp39-cp39-win32.whl","hashes":{"sha256":"9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242954,"upload-time":"2022-10-18T20:15:04.992990Z","url":"https://files.pythonhosted.org/packages/2f/5e/c74dab9858ca67a68a543ad8fefac2aec107383c171019b45ba9ac5223c1/psutil-5.9.3-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp39-cp39-win_amd64.whl","hashes":{"sha256":"4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247081,"upload-time":"2022-10-18T20:15:08.131670Z","url":"https://files.pythonhosted.org/packages/34/31/9aa19bf0fb0cecae904c9e1ac400c5704d935252515da605aa08fca2be86/psutil-5.9.3-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.3.tar.gz","hashes":{"sha256":"7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":483579,"upload-time":"2022-10-18T20:15:11.635566Z","url":"https://files.pythonhosted.org/packages/de/eb/1c01a34c86ee3b058c556e407ce5b07cb7d186ebe47b3e69d6f152ca5cc5/psutil-5.9.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242993,"upload-time":"2022-11-07T18:44:23.237667Z","url":"https://files.pythonhosted.org/packages/60/f8/b92fecd5297edcecda825a04dfde7cb0a2ecd178eb976cb5a7956e375c6a/psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":294126,"upload-time":"2022-11-07T18:44:28.809923Z","url":"https://files.pythonhosted.org/packages/8e/6b/9a3a5471b74d92dc85bfd71a7f7a55e013b258d86b4c3826ace9d49f7b8c/psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":298394,"upload-time":"2022-11-07T18:44:34.503099Z","url":"https://files.pythonhosted.org/packages/1d/80/e1502ba4ff65390bd17b4612010762075f64f5a0e7c28e889c4820bd95a9/psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"data-dist-info-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"filename":"psutil-5.9.4-cp27-cp27m-win32.whl","hashes":{"sha256":"852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242001,"upload-time":"2022-11-07T18:44:39.844844Z","url":"https://files.pythonhosted.org/packages/53/ae/536719016fe9399187dbf52cdc65aef942f82b75924495918a2f701bcb77/psutil-5.9.4-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"data-dist-info-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"filename":"psutil-5.9.4-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245334,"upload-time":"2022-11-07T18:44:44.877461Z","url":"https://files.pythonhosted.org/packages/99/9c/7a5761f9d2e79e6f781db5b25eeb9e74c2dc533bc52ee4749cb055a32ce9/psutil-5.9.4-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":294138,"upload-time":"2022-11-07T18:44:49.025560Z","url":"https://files.pythonhosted.org/packages/ec/be/b8df2071eda861e65a1b2cec35770bb1f4523737e84a10aa41c53e39e9bc/psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":298409,"upload-time":"2022-11-07T18:44:56.505271Z","url":"https://files.pythonhosted.org/packages/89/a8/dd2f0866a7e87de751fb5f7c6eca99cbb953c81be76e1814ab3c8c3b0908/psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243468,"upload-time":"2022-11-07T18:45:00.474371Z","url":"https://files.pythonhosted.org/packages/a5/73/35cea01aad1baf901c915dc95ea33a2f271c8ff8cf2f1c73b7f591f1bdf1/psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":277515,"upload-time":"2022-11-07T18:45:05.428752Z","url":"https://files.pythonhosted.org/packages/5a/37/ef88eed265d93bc28c681316f68762c5e04167519e5627a0187c8878b409/psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280218,"upload-time":"2022-11-07T18:45:11.831087Z","url":"https://files.pythonhosted.org/packages/6e/c8/784968329c1c67c28cce91991ef9af8a8913aa5a3399a6a8954b1380572f/psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"data-dist-info-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"filename":"psutil-5.9.4-cp36-abi3-win32.whl","hashes":{"sha256":"149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247217,"upload-time":"2022-11-08T11:50:03.989781Z","url":"https://files.pythonhosted.org/packages/3e/af/fe14b984e8b0f778d502d387b789d846cb2fcc3989f63be942741266d8c8/psutil-5.9.4-cp36-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"data-dist-info-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"filename":"psutil-5.9.4-cp36-abi3-win_amd64.whl","hashes":{"sha256":"fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":252462,"upload-time":"2022-11-08T11:50:07.829565Z","url":"https://files.pythonhosted.org/packages/25/6e/ba97809175c90cbdcd33b470e466ebf0854d15d1506e605cc0ddd284d5b6/psutil-5.9.4-cp36-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"3c3c1d22a00986e9736611ca6121b0926aba1c13b75f0b3509aebf5f95b0d409"},"data-dist-info-metadata":{"sha256":"3c3c1d22a00986e9736611ca6121b0926aba1c13b75f0b3509aebf5f95b0d409"},"filename":"psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244234,"upload-time":"2022-11-07T19:53:20.872251Z","url":"https://files.pythonhosted.org/packages/79/26/f026804298b933b11640cc2d15155a545805df732e5ead3a2ad7cf45a38b/psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.4.tar.gz","hashes":{"sha256":"3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":485825,"upload-time":"2022-11-07T19:53:36.245577Z","url":"https://files.pythonhosted.org/packages/3d/7d/d05864a69e452f003c0d77e728e155a89a2a26b09e64860ddd70ad64fb26/psutil-5.9.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244852,"upload-time":"2023-04-17T18:24:26.646150Z","url":"https://files.pythonhosted.org/packages/3b/e4/fee119c206545fd37be1e5fa4eeb0c729a52ec2ade4f728ae1fd1acb2a3a/psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296014,"upload-time":"2023-04-17T18:24:31.346064Z","url":"https://files.pythonhosted.org/packages/8d/24/ed6b6506f187def39887a91a68e58336eff4cf3e3d5a163ded58bee98624/psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":300274,"upload-time":"2023-04-17T18:24:35.244779Z","url":"https://files.pythonhosted.org/packages/89/fa/ab117fa86195050802207639f5daee857791daaabe9a996935b5b77dbe10/psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":295979,"upload-time":"2023-04-17T18:24:38.850555Z","url":"https://files.pythonhosted.org/packages/99/f5/ec768e107445f18baa907509aaa0562a4d148a602bd97e8114d79bd6c84d/psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":300257,"upload-time":"2023-04-17T18:24:42.928863Z","url":"https://files.pythonhosted.org/packages/5f/da/de9d2342db0b7a96863ef84ab94ef1022eec78ece05aac253cddc494e1a7/psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"data-dist-info-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"filename":"psutil-5.9.5-cp27-none-win32.whl","hashes":{"sha256":"5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243956,"upload-time":"2023-04-17T18:24:46.410452Z","url":"https://files.pythonhosted.org/packages/cf/e3/6af6ec0cbe72f63e9a16d8b53590489e40ed0ff0c99b6a6f05d6af3bb80e/psutil-5.9.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"data-dist-info-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"filename":"psutil-5.9.5-cp27-none-win_amd64.whl","hashes":{"sha256":"8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247338,"upload-time":"2023-04-17T18:24:49.089552Z","url":"https://files.pythonhosted.org/packages/26/f2/dcd8a3cc9c9b1fcd7576a54e3603ce4d1f85672f2687a44050340f7d47b0/psutil-5.9.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245316,"upload-time":"2023-04-17T18:24:52.864585Z","url":"https://files.pythonhosted.org/packages/9a/76/c0195c3443a725c24b3a479f57636dec89efe53d19d435d1752c5188f7de/psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279398,"upload-time":"2023-04-17T18:24:56.977087Z","url":"https://files.pythonhosted.org/packages/e5/2e/56db2b45508ad484b3f22888b3e1adaaf09b8766eaa058ed0e4486c1abae/psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282082,"upload-time":"2023-04-17T18:25:00.863664Z","url":"https://files.pythonhosted.org/packages/af/4d/389441079ecef400e2551a3933224885a7bde6b8a4810091d628cdd75afe/psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-win32.whl","hashes":{"sha256":"104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":249834,"upload-time":"2023-04-17T18:25:05.571829Z","url":"https://files.pythonhosted.org/packages/fa/e0/e91277b1cabf5c3f2995c22314553f1be68b17444260101f365c5a5b6ba1/psutil-5.9.5-cp36-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-win_amd64.whl","hashes":{"sha256":"b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":255148,"upload-time":"2023-04-17T18:25:09.779955Z","url":"https://files.pythonhosted.org/packages/86/f3/23e4e4e7ec7855d506ed928756b04735c246b14d9f778ed7ffaae18d8043/psutil-5.9.5-cp36-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2ae643bfca9fa3b942bf775226368a8ef859018ea312be94e137a8511ba0da07"},"data-dist-info-metadata":{"sha256":"2ae643bfca9fa3b942bf775226368a8ef859018ea312be94e137a8511ba0da07"},"filename":"psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246094,"upload-time":"2023-04-17T18:25:14.584295Z","url":"https://files.pythonhosted.org/packages/ed/98/2624954f83489ab13fde2b544baa337d5578c07eee304d320d9ba56e1b1f/psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.5.tar.gz","hashes":{"sha256":"5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":493489,"upload-time":"2023-04-17T18:25:18.787463Z","url":"https://files.pythonhosted.org/packages/d6/0f/96b7309212a926c1448366e9ce69b081ea79d63265bde33f11cc9cfc2c07/psutil-5.9.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245665,"upload-time":"2023-10-15T09:08:50.362285Z","url":"https://files.pythonhosted.org/packages/84/d6/7e23b2b208db3953f630934bc0e9c1736a0a831a781acf8c5891c27b29cf/psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":297167,"upload-time":"2023-10-15T09:08:52.370112Z","url":"https://files.pythonhosted.org/packages/d2/76/f154e5169756f3d18da160359a404f49f476756809ef21a79afdd0d5b552/psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":301338,"upload-time":"2023-10-15T09:08:55.187829Z","url":"https://files.pythonhosted.org/packages/35/e8/5cc0e149ec32a91d459fbe51d0ce3c2dd7f8d67bc1400803ff810247d6dc/psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":297167,"upload-time":"2023-10-15T09:08:57.863175Z","url":"https://files.pythonhosted.org/packages/8d/f7/074071fa91dab747c8d1fe2eb74da439b3712248d6b254ba0136ada8694f/psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":301350,"upload-time":"2023-10-15T09:09:00.411072Z","url":"https://files.pythonhosted.org/packages/4a/65/557545149422a7845248641c1c35a0c8ea940c838896320f774072e16523/psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"data-dist-info-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"filename":"psutil-5.9.6-cp27-none-win32.whl","hashes":{"sha256":"70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":244900,"upload-time":"2023-10-15T09:09:03.051464Z","url":"https://files.pythonhosted.org/packages/b8/23/d5d9e20c4ae7374abe1f826c69ecf2ab52f93827ca2b92c2c51f9aeb9226/psutil-5.9.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"data-dist-info-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"filename":"psutil-5.9.6-cp27-none-win_amd64.whl","hashes":{"sha256":"51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248205,"upload-time":"2023-10-15T09:09:05.456917Z","url":"https://files.pythonhosted.org/packages/7a/5e/db765b94cb620c04aaea0cb03d8b589905e50ec278130d25646eead8dff0/psutil-5.9.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246101,"upload-time":"2023-10-15T09:09:08.012635Z","url":"https://files.pythonhosted.org/packages/f8/36/35b12441ba1bc6684c9215191f955415196ca57ca85d88e313bec7f2cf8e/psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":280854,"upload-time":"2023-10-15T09:09:09.832401Z","url":"https://files.pythonhosted.org/packages/61/c8/e684dea1912943347922ab5c05efc94b4ff3d7470038e8afbe3941ef9efe/psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":283614,"upload-time":"2023-10-15T09:09:12.314910Z","url":"https://files.pythonhosted.org/packages/19/06/4e3fa3c1b79271e933c5ddbad3a48aa2c3d5f592a0fb7c037f3e0f619f4d/psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-cp36m-win32.whl","hashes":{"sha256":"3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":250445,"upload-time":"2023-10-15T09:09:14.942143Z","url":"https://files.pythonhosted.org/packages/3f/63/d4a8dace1756b9c84b94683aa80ed0ba8fc7a4421904933b472d59268976/psutil-5.9.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255893,"upload-time":"2023-10-15T09:09:17.467925Z","url":"https://files.pythonhosted.org/packages/ad/00/c87d449746f8962eb9203554b46ab7dcf243be236dcf007372902791b374/psutil-5.9.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"data-dist-info-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"filename":"psutil-5.9.6-cp37-abi3-win32.whl","hashes":{"sha256":"a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248489,"upload-time":"2023-10-15T09:09:19.912001Z","url":"https://files.pythonhosted.org/packages/06/ac/f31a0faf98267e63fc6ed046ad2aca68bd79521380026e92fd4921c869aa/psutil-5.9.6-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"data-dist-info-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"filename":"psutil-5.9.6-cp37-abi3-win_amd64.whl","hashes":{"sha256":"6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":252327,"upload-time":"2023-10-15T09:09:32.052033Z","url":"https://files.pythonhosted.org/packages/c5/b2/699c50fe0b0402a1ccb64ad71313bcb740e735008dd3ab9abeddbe148e45/psutil-5.9.6-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7298b90db12439da117e2817e35cbbb00312edeb4f885274cd1135a533903d6c"},"data-dist-info-metadata":{"sha256":"7298b90db12439da117e2817e35cbbb00312edeb4f885274cd1135a533903d6c"},"filename":"psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246859,"upload-time":"2023-10-15T09:09:34.494297Z","url":"https://files.pythonhosted.org/packages/9e/cb/e4b83c27eea66bc255effc967053f6fce7c14906dd9b43a348ead9f0cfea/psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.6.tar.gz","hashes":{"sha256":"e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":496866,"upload-time":"2023-10-15T09:08:46.623978Z","url":"https://files.pythonhosted.org/packages/2d/01/beb7331fc6c8d1c49dd051e3611379bfe379e915c808e1301506027fce9d/psutil-5.9.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245542,"upload-time":"2023-12-17T11:25:25.875219Z","url":"https://files.pythonhosted.org/packages/3e/16/c86fcf73f02bd0a3d49b0dcabc8ebd4020647be2ea40ff668f717587af97/psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312042,"upload-time":"2023-12-17T11:25:29.439835Z","url":"https://files.pythonhosted.org/packages/93/fc/e45a8e9b2acd54fe80ededa2f7b19de21e776f64e00437417c16c3e139d9/psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312681,"upload-time":"2023-12-17T11:25:33.024609Z","url":"https://files.pythonhosted.org/packages/d7/43/dd7034a3a3a900e95b9dcf47ee710680cfd11a224ab18b31c34370da36a8/psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312066,"upload-time":"2023-12-17T11:25:37.010306Z","url":"https://files.pythonhosted.org/packages/ff/ea/a47eecddcd97d65b496ac655c9f9ba8af270c203d5ea1630273cfc5ec740/psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312684,"upload-time":"2023-12-17T11:25:39.891899Z","url":"https://files.pythonhosted.org/packages/cd/ee/d946d0b758120e724d9cdd9607c304ff1eedb9380bf60597c295dc7def6b/psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"data-dist-info-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"filename":"psutil-5.9.7-cp27-none-win32.whl","hashes":{"sha256":"1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":244785,"upload-time":"2023-12-17T11:25:42.761256Z","url":"https://files.pythonhosted.org/packages/98/c5/6773a3f1c384ac4863665e167cd4da72433b3020580c0b7c6a7b497e11e2/psutil-5.9.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"data-dist-info-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"filename":"psutil-5.9.7-cp27-none-win_amd64.whl","hashes":{"sha256":"4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248087,"upload-time":"2023-12-17T11:25:45.316036Z","url":"https://files.pythonhosted.org/packages/2d/91/40ac017db38c9f7f325385dd0dab1be3d4c65e3291100e74d5d7b6a213e8/psutil-5.9.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245972,"upload-time":"2023-12-17T11:25:48.202730Z","url":"https://files.pythonhosted.org/packages/6c/63/86a4ccc640b4ee1193800f57bbd20b766853c0cdbdbb248a27cdfafe6cbf/psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":282514,"upload-time":"2023-12-17T11:25:51.371460Z","url":"https://files.pythonhosted.org/packages/58/80/cc6666b3968646f2d94de66bbc63d701d501f4aa04de43dd7d1f5dc477dd/psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":285469,"upload-time":"2023-12-17T11:25:54.250669Z","url":"https://files.pythonhosted.org/packages/be/fa/f1f626620e3b47e6237dcc64cb8cc1472f139e99422e5b9fa5bbcf457f48/psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-cp36m-win32.whl","hashes":{"sha256":"b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":250357,"upload-time":"2023-12-17T12:38:23.681291Z","url":"https://files.pythonhosted.org/packages/63/16/11dfb52cdccd561da711ee2c127b4c0bd2baf4736d10828c707694f31b90/psutil-5.9.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255808,"upload-time":"2023-12-17T12:38:34.170079Z","url":"https://files.pythonhosted.org/packages/0e/88/9b74b25c63b91ff0403a1b89e258238380b4a88e4116cbae4eaadbb4c17a/psutil-5.9.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"data-dist-info-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"filename":"psutil-5.9.7-cp37-abi3-win32.whl","hashes":{"sha256":"c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248406,"upload-time":"2023-12-17T12:38:50.326952Z","url":"https://files.pythonhosted.org/packages/7c/b8/dc6ebfc030b47cccc5f5229eeb15e64142b4782796c3ce169ccd60b4d511/psutil-5.9.7-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"data-dist-info-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"filename":"psutil-5.9.7-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":252245,"upload-time":"2023-12-17T12:39:00.686632Z","url":"https://files.pythonhosted.org/packages/50/28/92b74d95dd991c837813ffac0c79a581a3d129eb0fa7c1dd616d9901e0f3/psutil-5.9.7-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c6272489f71a5b9474dff03296e3f8e100ef6f8ed990f7e4ef444ac6e6a3d6fb"},"data-dist-info-metadata":{"sha256":"c6272489f71a5b9474dff03296e3f8e100ef6f8ed990f7e4ef444ac6e6a3d6fb"},"filename":"psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246739,"upload-time":"2023-12-17T11:25:57.305436Z","url":"https://files.pythonhosted.org/packages/ba/8a/000d0e80156f0b96c55bda6c60f5ed6543d7b5e893ccab83117e50de1400/psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.7.tar.gz","hashes":{"sha256":"3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":498429,"upload-time":"2023-12-17T11:25:21.220127Z","url":"https://files.pythonhosted.org/packages/a0/d0/c9ae661a302931735237791f04cb7086ac244377f78692ba3b3eae3a9619/psutil-5.9.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248274,"upload-time":"2024-01-19T20:47:14.006890Z","url":"https://files.pythonhosted.org/packages/15/9a/c3e2922e2d672bafd37cf3b9681097c350463cdcf0e286e907ddd6cfb014/psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":314796,"upload-time":"2024-01-19T20:47:17.872998Z","url":"https://files.pythonhosted.org/packages/62/e6/6d62285989d53a83def28ea49b46d3e00462d1273c7c47d9678ee28a0a39/psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":315422,"upload-time":"2024-01-19T20:47:21.442877Z","url":"https://files.pythonhosted.org/packages/38/ba/41815f353f79374c1ad82aba998c666c7209793daf12f4799cfaa7302f29/psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":314802,"upload-time":"2024-01-19T20:47:24.219052Z","url":"https://files.pythonhosted.org/packages/a8/2f/ad80cc502c452e1f207307a7d53533505ca47c503ec6e9f7e2c9fbb367e8/psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":315420,"upload-time":"2024-01-19T20:47:26.828052Z","url":"https://files.pythonhosted.org/packages/e4/c3/357a292dee683282f7a46b752a76c5d56c78bf8f5d9def0ca0d39073344a/psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"data-dist-info-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"filename":"psutil-5.9.8-cp27-none-win32.whl","hashes":{"sha256":"36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248660,"upload-time":"2024-01-19T20:47:29.706532Z","url":"https://files.pythonhosted.org/packages/fe/5f/c26deb822fd3daf8fde4bdb658bf87d9ab1ffd3fca483816e89a9a9a9084/psutil-5.9.8-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"data-dist-info-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"filename":"psutil-5.9.8-cp27-none-win_amd64.whl","hashes":{"sha256":"bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":251966,"upload-time":"2024-01-19T20:47:33.134054Z","url":"https://files.pythonhosted.org/packages/32/1d/cf66073d74d6146187e2d0081a7616df4437214afa294ee4f16f80a2f96a/psutil-5.9.8-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248702,"upload-time":"2024-01-19T20:47:36.303498Z","url":"https://files.pythonhosted.org/packages/e7/e3/07ae864a636d70a8a6f58da27cb1179192f1140d5d1da10886ade9405797/psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":285242,"upload-time":"2024-01-19T20:47:39.650099Z","url":"https://files.pythonhosted.org/packages/b3/bd/28c5f553667116b2598b9cc55908ec435cb7f77a34f2bff3e3ca765b0f78/psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":288191,"upload-time":"2024-01-19T20:47:43.078208Z","url":"https://files.pythonhosted.org/packages/c5/4f/0e22aaa246f96d6ac87fe5ebb9c5a693fbe8877f537a1022527c47ca43c5/psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-cp36m-win32.whl","hashes":{"sha256":"7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":253203,"upload-time":"2024-01-19T20:47:46.133427Z","url":"https://files.pythonhosted.org/packages/dd/9e/85c3bd5b466d96c091bbd6339881e99106adb43d5d60bde32ac181ab6fef/psutil-5.9.8-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":258655,"upload-time":"2024-01-19T20:47:48.804624Z","url":"https://files.pythonhosted.org/packages/0b/58/bcffb5ab03ec558e565d2871c01215dde74e11f583fb71e7d2b107200caa/psutil-5.9.8-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"data-dist-info-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"filename":"psutil-5.9.8-cp37-abi3-win32.whl","hashes":{"sha256":"bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":251252,"upload-time":"2024-01-19T20:47:52.880124Z","url":"https://files.pythonhosted.org/packages/6e/f5/2aa3a4acdc1e5940b59d421742356f133185667dd190b166dbcfcf5d7b43/psutil-5.9.8-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"data-dist-info-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"filename":"psutil-5.9.8-cp37-abi3-win_amd64.whl","hashes":{"sha256":"8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255090,"upload-time":"2024-01-19T20:47:56.019799Z","url":"https://files.pythonhosted.org/packages/93/52/3e39d26feae7df0aa0fd510b14012c3678b36ed068f7d78b8d8784d61f0e/psutil-5.9.8-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1526b4912f0f04a9c6523b41c128f4d1c6c666fbbdccb62c6705d7e5747c95cc"},"data-dist-info-metadata":{"sha256":"1526b4912f0f04a9c6523b41c128f4d1c6c666fbbdccb62c6705d7e5747c95cc"},"filename":"psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":249898,"upload-time":"2024-01-19T20:47:59.238740Z","url":"https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.8.tar.gz","hashes":{"sha256":"6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":503247,"upload-time":"2024-01-19T20:47:09.517227Z","url":"https://files.pythonhosted.org/packages/90/c7/6dc0a455d111f68ee43f27793971cf03fe29b6ef972042549db29eec39a2/psutil-5.9.8.tar.gz","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250527,"upload-time":"2024-06-18T21:40:17.061973Z","url":"https://files.pythonhosted.org/packages/13/e5/35ebd7169008752be5561cafdba3f1634be98193b85fe3d22e883f9fe2e1/psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":316838,"upload-time":"2024-06-18T21:40:32.679396Z","url":"https://files.pythonhosted.org/packages/92/a7/083388ef0964a6d74df51c677b3d761e0866d823d37e3a8823551c0d375d/psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":317493,"upload-time":"2024-06-18T21:40:41.710402Z","url":"https://files.pythonhosted.org/packages/52/2f/44b7005f306ea8bfd24aa662b5d0ba6ea1daf29dbd0b6c7bbcd3606373ad/psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":316855,"upload-time":"2024-06-18T21:40:47.752750Z","url":"https://files.pythonhosted.org/packages/81/c9/8cb36769b6636d817be3414ebbb27a9ab3fbe6d13835d00f31e77e1fccce/psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":317519,"upload-time":"2024-06-18T21:40:53.708954Z","url":"https://files.pythonhosted.org/packages/14/c0/024ac5369ca160e9ed45ed09247d9d779c460017fbd9aa801fd6eb0f060c/psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"data-dist-info-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"filename":"psutil-6.0.0-cp27-none-win32.whl","hashes":{"sha256":"02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":249766,"upload-time":"2024-06-18T21:40:58.381272Z","url":"https://files.pythonhosted.org/packages/c5/66/78c9c3020f573c58101dc43a44f6855d01bbbd747e24da2f0c4491200ea3/psutil-6.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"data-dist-info-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"filename":"psutil-6.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":253024,"upload-time":"2024-06-18T21:41:04.548455Z","url":"https://files.pythonhosted.org/packages/e1/3f/2403aa9558bea4d3854b0e5e567bc3dd8e9fbc1fc4453c0aa9aafeb75467/psutil-6.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250961,"upload-time":"2024-06-18T21:41:11.662513Z","url":"https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287478,"upload-time":"2024-06-18T21:41:16.180526Z","url":"https://files.pythonhosted.org/packages/35/56/72f86175e81c656a01c4401cd3b1c923f891b31fbcebe98985894176d7c9/psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":290455,"upload-time":"2024-06-18T21:41:29.048203Z","url":"https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":292046,"upload-time":"2024-06-18T21:41:33.530555Z","url":"https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-cp36m-win32.whl","hashes":{"sha256":"fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":255537,"upload-time":"2024-06-18T21:41:38.034852Z","url":"https://files.pythonhosted.org/packages/cd/ff/39c38910cdb8f02fc9965afb520967a1e9307d53d14879dddd0a4f41f6f8/psutil-6.0.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":260973,"upload-time":"2024-06-18T21:41:41.566213Z","url":"https://files.pythonhosted.org/packages/08/88/16dd53af4a84e719e27a5ad7db040231415d8caeb48f019bacafbb4d0002/psutil-6.0.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"data-dist-info-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"filename":"psutil-6.0.0-cp37-abi3-win32.whl","hashes":{"sha256":"a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":253560,"upload-time":"2024-06-18T21:41:46.067057Z","url":"https://files.pythonhosted.org/packages/8b/20/2ff69ad9c35c3df1858ac4e094f20bd2374d33c8643cf41da8fd7cdcb78b/psutil-6.0.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"data-dist-info-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"filename":"psutil-6.0.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":257399,"upload-time":"2024-06-18T21:41:52.100137Z","url":"https://files.pythonhosted.org/packages/73/44/561092313ae925f3acfaace6f9ddc4f6a9c748704317bad9c8c8f8a36a79/psutil-6.0.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c0710bf31d8e9160c90747acceb7692596b32d632efd784e22990fc01fad42dd"},"data-dist-info-metadata":{"sha256":"c0710bf31d8e9160c90747acceb7692596b32d632efd784e22990fc01fad42dd"},"filename":"psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":251988,"upload-time":"2024-06-18T21:41:57.337231Z","url":"https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.0.0.tar.gz","hashes":{"sha256":"8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508067,"upload-time":"2024-06-18T21:40:10.559591Z","url":"https://files.pythonhosted.org/packages/18/c7/8c6872f7372eb6a6b2e4708b88419fb46b857f7a2e1892966b851cc79fc9/psutil-6.0.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247385,"upload-time":"2024-10-17T21:31:49.162372Z","url":"https://files.pythonhosted.org/packages/cd/8e/87b51bedb52f0fa02a6c9399702912a5059b24c7242fa8ea4fd027cb5238/psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312067,"upload-time":"2024-10-17T21:31:51.572046Z","url":"https://files.pythonhosted.org/packages/2c/56/99304ecbf1f25a2aa336c66e43a8f9462de70d089d3fbb487991dfd96b37/psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312345,"upload-time":"2024-10-17T21:31:53.950865Z","url":"https://files.pythonhosted.org/packages/24/87/7c1eeb2fd86a8eb792b15438a3d25eda05c970924df3457669b50e0c022b/psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312058,"upload-time":"2024-10-17T21:31:55.843636Z","url":"https://files.pythonhosted.org/packages/aa/fe/c94a914040c74b2bbe2ddb2c82b3f9a74d8a40401bb1239b0e949331c957/psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312330,"upload-time":"2024-10-17T21:31:57.551704Z","url":"https://files.pythonhosted.org/packages/44/8c/624823d5a5a9ec8635d63b273c3ab1554a4fcc3513f4d0236ff9706f1025/psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"data-dist-info-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"filename":"psutil-6.1.0-cp27-none-win32.whl","hashes":{"sha256":"9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":246648,"upload-time":"2024-10-17T21:31:59.369185Z","url":"https://files.pythonhosted.org/packages/da/2b/f4dea5d993d9cd22ad958eea828a41d5d225556123d372f02547c29c4f97/psutil-6.1.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"data-dist-info-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"filename":"psutil-6.1.0-cp27-none-win_amd64.whl","hashes":{"sha256":"a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":249905,"upload-time":"2024-10-17T21:32:01.974050Z","url":"https://files.pythonhosted.org/packages/9f/14/4aa97a7f2e0ac33a050d990ab31686d651ae4ef8c86661fef067f00437b9/psutil-6.1.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247762,"upload-time":"2024-10-17T21:32:05.991637Z","url":"https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"31aa42573bd48a390acf1e517f08ce9f05a4954542dc8afb9a8805655df7aa18"},"data-dist-info-metadata":{"sha256":"31aa42573bd48a390acf1e517f08ce9f05a4954542dc8afb9a8805655df7aa18"},"filename":"psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":248777,"upload-time":"2024-10-17T21:32:07.872442Z","url":"https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":284259,"upload-time":"2024-10-17T21:32:10.177301Z","url":"https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287255,"upload-time":"2024-10-17T21:32:11.964687Z","url":"https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":288804,"upload-time":"2024-10-17T21:32:13.785068Z","url":"https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"data-dist-info-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"filename":"psutil-6.1.0-cp36-cp36m-win32.whl","hashes":{"sha256":"6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":252360,"upload-time":"2024-10-17T21:32:16.434334Z","url":"https://files.pythonhosted.org/packages/43/39/414d7b67f4df35bb9c373d0fb9a75dd40b223d9bd6d02ebdc7658fd461a3/psutil-6.1.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"data-dist-info-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"filename":"psutil-6.1.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":257797,"upload-time":"2024-10-17T21:32:18.946615Z","url":"https://files.pythonhosted.org/packages/ca/da/ef86c99e33be4aa888570e79350caca8c4819b62f84a6d9274c88c40e331/psutil-6.1.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"data-dist-info-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"filename":"psutil-6.1.0-cp37-abi3-win32.whl","hashes":{"sha256":"1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250386,"upload-time":"2024-10-17T21:32:21.399329Z","url":"https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"data-dist-info-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"filename":"psutil-6.1.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":254228,"upload-time":"2024-10-17T21:32:23.880601Z","url":"https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.1.0.tar.gz","hashes":{"sha256":"353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508565,"upload-time":"2024-10-17T21:31:45.680545Z","url":"https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247226,"upload-time":"2024-12-19T18:21:25.276122Z","url":"https://files.pythonhosted.org/packages/09/ea/f8844afff4c8c11d1d0586b737d8d579fd7cb13f1fa3eea599c71877b526/psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312292,"upload-time":"2024-12-19T18:21:30.930117Z","url":"https://files.pythonhosted.org/packages/51/f8/e376f9410beb915bbf64cb4ae8ce5cf2d03e9a661a2519ebc6a63045a1ca/psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312542,"upload-time":"2024-12-19T18:21:34.735400Z","url":"https://files.pythonhosted.org/packages/a7/3a/069d6c1e4a7af3cdb162c9ba0737ff9baed1d05cbab6f082f49e3b9ab0a5/psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312279,"upload-time":"2024-12-19T18:21:37.897094Z","url":"https://files.pythonhosted.org/packages/81/d5/ee5de2cb8d0c938bb07dcccd4ff7e950359bd6ddbd2fe3118552f863bb52/psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312521,"upload-time":"2024-12-19T18:21:40.651860Z","url":"https://files.pythonhosted.org/packages/37/98/443eff82762b3f2c6a4bd0cdf3bc5c9f62245376c5486b39ee194e920794/psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"data-dist-info-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"filename":"psutil-6.1.1-cp27-none-win32.whl","hashes":{"sha256":"6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":246855,"upload-time":"2024-12-19T18:54:12.657947Z","url":"https://files.pythonhosted.org/packages/d2/d4/8095b53c4950f44dc99b8d983b796f405ae1f58d80978fcc0421491b4201/psutil-6.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"data-dist-info-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"filename":"psutil-6.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250110,"upload-time":"2024-12-19T18:54:16.635901Z","url":"https://files.pythonhosted.org/packages/b1/63/0b6425ea4f2375988209a9934c90d6079cc7537847ed58a28fbe30f4277e/psutil-6.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247511,"upload-time":"2024-12-19T18:21:45.163741Z","url":"https://files.pythonhosted.org/packages/61/99/ca79d302be46f7bdd8321089762dd4476ee725fce16fc2b2e1dbba8cac17/psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"f53aa914fe0e63af2de701b5d7b1c28a494b1852f4eddac6bec1c1d8eecc00d7"},"data-dist-info-metadata":{"sha256":"f53aa914fe0e63af2de701b5d7b1c28a494b1852f4eddac6bec1c1d8eecc00d7"},"filename":"psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":248985,"upload-time":"2024-12-19T18:21:49.254078Z","url":"https://files.pythonhosted.org/packages/0b/6b/73dbde0dd38f3782905d4587049b9be64d76671042fdcaf60e2430c6796d/psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":284488,"upload-time":"2024-12-19T18:21:51.638630Z","url":"https://files.pythonhosted.org/packages/17/38/c319d31a1d3f88c5b79c68b3116c129e5133f1822157dd6da34043e32ed6/psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287477,"upload-time":"2024-12-19T18:21:55.306984Z","url":"https://files.pythonhosted.org/packages/9c/39/0f88a830a1c8a3aba27fededc642da37613c57cbff143412e3536f89784f/psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":289017,"upload-time":"2024-12-19T18:21:57.875754Z","url":"https://files.pythonhosted.org/packages/47/da/99f4345d4ddf2845cb5b5bd0d93d554e84542d116934fde07a0c50bd4e9f/psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"data-dist-info-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"filename":"psutil-6.1.1-cp36-cp36m-win32.whl","hashes":{"sha256":"384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":252576,"upload-time":"2024-12-19T18:22:01.852822Z","url":"https://files.pythonhosted.org/packages/8e/1f/1aebe4dd5914ccba6f7d6cc6d11fb79f6f23f95b858a7f631446bdc5d67f/psutil-6.1.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"data-dist-info-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"filename":"psutil-6.1.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":258012,"upload-time":"2024-12-19T18:22:04.204308Z","url":"https://files.pythonhosted.org/packages/f4/de/fb4561e59611c19a2d7377c2b2534d11274b8a7df9bb7b7e7f1de5be3641/psutil-6.1.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"data-dist-info-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"filename":"psutil-6.1.1-cp37-abi3-win32.whl","hashes":{"sha256":"eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250602,"upload-time":"2024-12-19T18:22:08.808295Z","url":"https://files.pythonhosted.org/packages/38/53/bd755c2896f4461fd4f36fa6a6dcb66a88a9e4b9fd4e5b66a77cf9d4a584/psutil-6.1.1-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"data-dist-info-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"filename":"psutil-6.1.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":254444,"upload-time":"2024-12-19T18:22:11.335598Z","url":"https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.1.1.tar.gz","hashes":{"sha256":"cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508502,"upload-time":"2024-12-19T18:21:20.568966Z","url":"https://files.pythonhosted.org/packages/1f/5a/07871137bb752428aa4b659f910b399ba6f291156bdea939be3e96cae7cb/psutil-6.1.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"},"provenance":null,"requires-python":">=3.6","size":238051,"upload-time":"2025-02-13T21:54:12.360451Z","url":"https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"65f64bcd26d7284f4f07568c343ced6641bf8235c8c5a09a4986dd2ec5f68de7"},"data-dist-info-metadata":{"sha256":"65f64bcd26d7284f4f07568c343ced6641bf8235c8c5a09a4986dd2ec5f68de7"},"filename":"psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"},"provenance":null,"requires-python":">=3.6","size":239535,"upload-time":"2025-02-13T21:54:16.070769Z","url":"https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"},"provenance":null,"requires-python":">=3.6","size":275004,"upload-time":"2025-02-13T21:54:18.662603Z","url":"https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"},"provenance":null,"requires-python":">=3.6","size":277986,"upload-time":"2025-02-13T21:54:21.811145Z","url":"https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"},"provenance":null,"requires-python":">=3.6","size":279544,"upload-time":"2025-02-13T21:54:24.680762Z","url":"https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"data-dist-info-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"filename":"psutil-7.0.0-cp36-cp36m-win32.whl","hashes":{"sha256":"84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"},"provenance":null,"requires-python":">=3.6","size":243024,"upload-time":"2025-02-13T21:54:27.767214Z","url":"https://files.pythonhosted.org/packages/98/04/9e7b8afdad85824dec17de92c121d0fb1907ded624f486b86cd5e8189ebe/psutil-7.0.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"data-dist-info-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"filename":"psutil-7.0.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"},"provenance":null,"requires-python":">=3.6","size":248462,"upload-time":"2025-02-13T21:54:31.148496Z","url":"https://files.pythonhosted.org/packages/25/9b/43f2c5f7794a3eba3fc0bb47020d1da44d43ff41c95637c5d760c3ef33eb/psutil-7.0.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"data-dist-info-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"filename":"psutil-7.0.0-cp37-abi3-win32.whl","hashes":{"sha256":"ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"},"provenance":null,"requires-python":">=3.6","size":241053,"upload-time":"2025-02-13T21:54:34.310916Z","url":"https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"data-dist-info-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"filename":"psutil-7.0.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"},"provenance":null,"requires-python":">=3.6","size":244885,"upload-time":"2025-02-13T21:54:37.486453Z","url":"https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.0.0.tar.gz","hashes":{"sha256":"7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"},"provenance":null,"requires-python":">=3.6","size":497003,"upload-time":"2025-02-13T21:54:07.946974Z","url":"https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13"},"provenance":null,"requires-python":">=3.6","size":245242,"upload-time":"2025-09-17T20:14:56.126572Z","url":"https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5"},"provenance":null,"requires-python":">=3.6","size":246682,"upload-time":"2025-09-17T20:14:58.250040Z","url":"https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3"},"provenance":null,"requires-python":">=3.6","size":287994,"upload-time":"2025-09-17T20:14:59.901485Z","url":"https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3"},"provenance":null,"requires-python":">=3.6","size":291163,"upload-time":"2025-09-17T20:15:01.481447Z","url":"https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d"},"provenance":null,"requires-python":">=3.6","size":293625,"upload-time":"2025-09-17T20:15:04.492789Z","url":"https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"data-dist-info-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"filename":"psutil-7.1.0-cp37-abi3-win32.whl","hashes":{"sha256":"09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca"},"provenance":null,"requires-python":">=3.6","size":244812,"upload-time":"2025-09-17T20:15:07.462276Z","url":"https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"data-dist-info-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"filename":"psutil-7.1.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d"},"provenance":null,"requires-python":">=3.6","size":247965,"upload-time":"2025-09-17T20:15:09.673366Z","url":"https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"24a51389a00af9d96fada7f4a79aed176d3a74f0ae399a5305d12ce9784f2635"},"data-dist-info-metadata":{"sha256":"24a51389a00af9d96fada7f4a79aed176d3a74f0ae399a5305d12ce9784f2635"},"filename":"psutil-7.1.0-cp37-abi3-win_arm64.whl","hashes":{"sha256":"6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07"},"provenance":null,"requires-python":">=3.6","size":244971,"upload-time":"2025-09-17T20:15:12.262753Z","url":"https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.0.tar.gz","hashes":{"sha256":"655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2"},"provenance":null,"requires-python":">=3.6","size":497660,"upload-time":"2025-09-17T20:14:52.902036Z","url":"https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"8fa59d7b1f01f0337f12cd10dbd76e4312a4d3c730a4fedcbdd4e5447a8b8460"},"provenance":null,"requires-python":">=3.6","size":244221,"upload-time":"2025-10-19T15:44:03.145914Z","url":"https://files.pythonhosted.org/packages/51/30/f97f8fb1f9ecfbeae4b5ca738dcae66ab28323b5cfbc96cb5565f3754056/psutil-7.1.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"2a95104eae85d088891716db676f780c1404fc15d47fde48a46a5d61e8f5ad2c"},"provenance":null,"requires-python":">=3.6","size":245660,"upload-time":"2025-10-19T15:44:05.657308Z","url":"https://files.pythonhosted.org/packages/7b/98/b8d1f61ebf35f4dbdbaabadf9208282d8adc820562f0257e5e6e79e67bf2/psutil-7.1.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"98629cd8567acefcc45afe2f4ba1e9290f579eacf490a917967decce4b74ee9b"},"provenance":null,"requires-python":">=3.6","size":286963,"upload-time":"2025-10-19T15:44:08.877299Z","url":"https://files.pythonhosted.org/packages/f0/4a/b8015d7357fefdfe34bc4a3db48a107bae4bad0b94fb6eb0613f09a08ada/psutil-7.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"92ebc58030fb054fa0f26c3206ef01c31c29d67aee1367e3483c16665c25c8d2"},"provenance":null,"requires-python":">=3.6","size":290118,"upload-time":"2025-10-19T15:44:11.897889Z","url":"https://files.pythonhosted.org/packages/3d/3c/b56076bb35303d0733fc47b110a1c9cce081a05ae2e886575a3587c1ee76/psutil-7.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"146a704f224fb2ded2be3da5ac67fc32b9ea90c45b51676f9114a6ac45616967"},"provenance":null,"requires-python":">=3.6","size":292587,"upload-time":"2025-10-19T15:44:14.670833Z","url":"https://files.pythonhosted.org/packages/dc/af/c13d360c0adc6f6218bf9e2873480393d0f729c8dd0507d171f53061c0d3/psutil-7.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"data-dist-info-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"filename":"psutil-7.1.1-cp37-abi3-win32.whl","hashes":{"sha256":"295c4025b5cd880f7445e4379e6826f7307e3d488947bf9834e865e7847dc5f7"},"provenance":null,"requires-python":">=3.6","size":243772,"upload-time":"2025-10-19T15:44:16.938205Z","url":"https://files.pythonhosted.org/packages/90/2d/c933e7071ba60c7862813f2c7108ec4cf8304f1c79660efeefd0de982258/psutil-7.1.1-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"data-dist-info-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"filename":"psutil-7.1.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"9b4f17c5f65e44f69bd3a3406071a47b79df45cf2236d1f717970afcb526bcd3"},"provenance":null,"requires-python":">=3.6","size":246936,"upload-time":"2025-10-19T15:44:18.663465Z","url":"https://files.pythonhosted.org/packages/be/f3/11fd213fff15427bc2853552138760c720fd65032d99edfb161910d04127/psutil-7.1.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"41f2736f936aaaa81486aad04717acc54669b33ae5ad93df72d7a7b26aa50fa7"},"data-dist-info-metadata":{"sha256":"41f2736f936aaaa81486aad04717acc54669b33ae5ad93df72d7a7b26aa50fa7"},"filename":"psutil-7.1.1-cp37-abi3-win_arm64.whl","hashes":{"sha256":"5457cf741ca13da54624126cd5d333871b454ab133999a9a103fb097a7d7d21a"},"provenance":null,"requires-python":">=3.6","size":243944,"upload-time":"2025-10-19T15:44:20.666512Z","url":"https://files.pythonhosted.org/packages/0a/8d/8a9a45c8b655851f216c1d44f68e3533dc8d2c752ccd0f61f1aa73be4893/psutil-7.1.1-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.1.tar.gz","hashes":{"sha256":"092b6350145007389c1cfe5716050f02030a05219d90057ea867d18fe8d372fc"},"provenance":null,"requires-python":">=3.6","size":487067,"upload-time":"2025-10-19T15:43:59.373160Z","url":"https://files.pythonhosted.org/packages/89/fc/889242351a932d6183eec5df1fc6539b6f36b6a88444f1e63f18668253aa/psutil-7.1.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"0cc5c6889b9871f231ed5455a9a02149e388fffcb30b607fb7a8896a6d95f22e"},"provenance":null,"requires-python":">=3.6","size":238575,"upload-time":"2025-10-25T10:46:38.728747Z","url":"https://files.pythonhosted.org/packages/b8/d9/b56cc9f883140ac10021a8c9b0f4e16eed1ba675c22513cdcbce3ba64014/psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"8e9e77a977208d84aa363a4a12e0f72189d58bbf4e46b49aae29a2c6e93ef206"},"provenance":null,"requires-python":">=3.6","size":239297,"upload-time":"2025-10-25T10:46:41.347184Z","url":"https://files.pythonhosted.org/packages/36/eb/28d22de383888deb252c818622196e709da98816e296ef95afda33f1c0a2/psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"7d9623a5e4164d2220ecceb071f4b333b3c78866141e8887c072129185f41278"},"provenance":null,"requires-python":">=3.6","size":280420,"upload-time":"2025-10-25T10:46:44.122205Z","url":"https://files.pythonhosted.org/packages/89/5d/220039e2f28cc129626e54d63892ab05c0d56a29818bfe7268dcb5008932/psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"364b1c10fe4ed59c89ec49e5f1a70da353b27986fa8233b4b999df4742a5ee2f"},"provenance":null,"requires-python":">=3.6","size":283049,"upload-time":"2025-10-25T10:46:47.095973Z","url":"https://files.pythonhosted.org/packages/ba/7a/286f0e1c167445b2ef4a6cbdfc8c59fdb45a5a493788950cf8467201dc73/psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"f101ef84de7e05d41310e3ccbdd65a6dd1d9eed85e8aaf0758405d022308e204"},"provenance":null,"requires-python":">=3.6","size":248713,"upload-time":"2025-10-25T10:46:49.573269Z","url":"https://files.pythonhosted.org/packages/aa/cc/7eb93260794a42e39b976f3a4dde89725800b9f573b014fac142002a5c98/psutil-7.1.2-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"20c00824048a95de67f00afedc7b08b282aa08638585b0206a9fb51f28f1a165"},"provenance":null,"requires-python":">=3.6","size":244644,"upload-time":"2025-10-25T10:46:51.924062Z","url":"https://files.pythonhosted.org/packages/ab/1a/0681a92b53366e01f0a099f5237d0c8a2f79d322ac589cccde5e30c8a4e2/psutil-7.1.2-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"e09cfe92aa8e22b1ec5e2d394820cf86c5dff6367ac3242366485dfa874d43bc"},"provenance":null,"requires-python":">=3.6","size":238640,"upload-time":"2025-10-25T10:46:54.089898Z","url":"https://files.pythonhosted.org/packages/56/9e/f1c5c746b4ed5320952acd3002d3962fe36f30524c00ea79fdf954cc6779/psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"fa6342cf859c48b19df3e4aa170e4cfb64aadc50b11e06bb569c6c777b089c9e"},"provenance":null,"requires-python":">=3.6","size":239303,"upload-time":"2025-10-25T10:46:56.932071Z","url":"https://files.pythonhosted.org/packages/32/ee/fd26216a735395cc25c3899634e34aeb41fb1f3dbb44acc67d9e594be562/psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"625977443498ee7d6c1e63e93bacca893fd759a66c5f635d05e05811d23fb5ee"},"provenance":null,"requires-python":">=3.6","size":281717,"upload-time":"2025-10-25T10:46:59.116890Z","url":"https://files.pythonhosted.org/packages/3c/cd/7d96eaec4ef7742b845a9ce2759a2769ecce4ab7a99133da24abacbc9e41/psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"4a24bcd7b7f2918d934af0fb91859f621b873d6aa81267575e3655cd387572a7"},"provenance":null,"requires-python":">=3.6","size":284575,"upload-time":"2025-10-25T10:47:00.944625Z","url":"https://files.pythonhosted.org/packages/bc/1a/7f0b84bdb067d35fe7fade5fff888408688caf989806ce2d6dae08c72dd5/psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"329f05610da6380982e6078b9d0881d9ab1e9a7eb7c02d833bfb7340aa634e31"},"provenance":null,"requires-python":">=3.6","size":249491,"upload-time":"2025-10-25T10:47:03.174087Z","url":"https://files.pythonhosted.org/packages/de/05/7820ef8f7b275268917e0c750eada5834581206d9024ca88edce93c4b762/psutil-7.1.2-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"7b04c29e3c0c888e83ed4762b70f31e65c42673ea956cefa8ced0e31e185f582"},"provenance":null,"requires-python":">=3.6","size":244880,"upload-time":"2025-10-25T10:47:05.228789Z","url":"https://files.pythonhosted.org/packages/db/9a/58de399c7cb58489f08498459ff096cd76b3f1ddc4f224ec2c5ef729c7d0/psutil-7.1.2-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c9ba5c19f2d46203ee8c152c7b01df6eec87d883cfd8ee1af2ef2727f6b0f814"},"provenance":null,"requires-python":">=3.6","size":237244,"upload-time":"2025-10-25T10:47:07.086631Z","url":"https://files.pythonhosted.org/packages/ae/89/b9f8d47ddbc52d7301fc868e8224e5f44ed3c7f55e6d0f54ecaf5dd9ff5e/psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"2a486030d2fe81bec023f703d3d155f4823a10a47c36784c84f1cc7f8d39bedb"},"provenance":null,"requires-python":">=3.6","size":238101,"upload-time":"2025-10-25T10:47:09.523680Z","url":"https://files.pythonhosted.org/packages/c8/7a/8628c2f6b240680a67d73d8742bb9ff39b1820a693740e43096d5dcb01e5/psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"3efd8fc791492e7808a51cb2b94889db7578bfaea22df931424f874468e389e3"},"provenance":null,"requires-python":">=3.6","size":258675,"upload-time":"2025-10-25T10:47:11.082823Z","url":"https://files.pythonhosted.org/packages/30/28/5e27f4d5a0e347f8e3cc16cd7d35533dbce086c95807f1f0e9cd77e26c10/psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"e2aeb9b64f481b8eabfc633bd39e0016d4d8bbcd590d984af764d80bf0851b8a"},"provenance":null,"requires-python":">=3.6","size":260203,"upload-time":"2025-10-25T10:47:13.226564Z","url":"https://files.pythonhosted.org/packages/e5/5c/79cf60c9acf36d087f0db0f82066fca4a780e97e5b3a2e4c38209c03d170/psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4e0958cd1551f981a3d0063faf8314b741cd97cb43d1e58934800cb4da537f64"},"data-dist-info-metadata":{"sha256":"4e0958cd1551f981a3d0063faf8314b741cd97cb43d1e58934800cb4da537f64"},"filename":"psutil-7.1.2-cp37-abi3-win_amd64.whl","hashes":{"sha256":"8e17852114c4e7996fe9da4745c2bdef001ebbf2f260dec406290e66628bdb91"},"provenance":null,"requires-python":">=3.6","size":246714,"upload-time":"2025-10-25T10:47:15.093571Z","url":"https://files.pythonhosted.org/packages/f7/03/0a464404c51685dcb9329fdd660b1721e076ccd7b3d97dee066bcc9ffb15/psutil-7.1.2-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp37-abi3-win_arm64.whl","hashes":{"sha256":"3e988455e61c240cc879cb62a008c2699231bf3e3d061d7fce4234463fd2abb4"},"provenance":null,"requires-python":">=3.6","size":243742,"upload-time":"2025-10-25T10:47:17.302139Z","url":"https://files.pythonhosted.org/packages/6a/32/97ca2090f2f1b45b01b6aa7ae161cfe50671de097311975ca6eea3e7aabc/psutil-7.1.2-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.2.tar.gz","hashes":{"sha256":"aa225cdde1335ff9684708ee8c72650f6598d5ed2114b9a7c5802030b1785018"},"provenance":null,"requires-python":">=3.6","size":487424,"upload-time":"2025-10-25T10:46:34.931002Z","url":"https://files.pythonhosted.org/packages/cd/ec/7b8e6b9b1d22708138630ef34c53ab2b61032c04f16adfdbb96791c8c70c/psutil-7.1.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc"},"provenance":null,"requires-python":">=3.6","size":239751,"upload-time":"2025-11-02T12:25:58.161404Z","url":"https://files.pythonhosted.org/packages/bd/93/0c49e776b8734fef56ec9c5c57f923922f2cf0497d62e0f419465f28f3d0/psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0"},"provenance":null,"requires-python":">=3.6","size":240368,"upload-time":"2025-11-02T12:26:00.491685Z","url":"https://files.pythonhosted.org/packages/6f/8d/b31e39c769e70780f007969815195a55c81a63efebdd4dbe9e7a113adb2f/psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7"},"provenance":null,"requires-python":">=3.6","size":287134,"upload-time":"2025-11-02T12:26:02.613574Z","url":"https://files.pythonhosted.org/packages/62/61/23fd4acc3c9eebbf6b6c78bcd89e5d020cfde4acf0a9233e9d4e3fa698b4/psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251"},"provenance":null,"requires-python":">=3.6","size":289904,"upload-time":"2025-11-02T12:26:05.207933Z","url":"https://files.pythonhosted.org/packages/30/1c/f921a009ea9ceb51aa355cb0cc118f68d354db36eae18174bab63affb3e6/psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa"},"provenance":null,"requires-python":">=3.6","size":249642,"upload-time":"2025-11-02T12:26:07.447774Z","url":"https://files.pythonhosted.org/packages/a6/82/62d68066e13e46a5116df187d319d1724b3f437ddd0f958756fc052677f4/psutil-7.1.3-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee"},"provenance":null,"requires-python":">=3.6","size":245518,"upload-time":"2025-11-02T12:26:09.719155Z","url":"https://files.pythonhosted.org/packages/df/ad/c1cd5fe965c14a0392112f68362cfceb5230819dbb5b1888950d18a11d9f/psutil-7.1.3-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353"},"provenance":null,"requires-python":">=3.6","size":239843,"upload-time":"2025-11-02T12:26:11.968073Z","url":"https://files.pythonhosted.org/packages/2e/bb/6670bded3e3236eb4287c7bcdc167e9fae6e1e9286e437f7111caed2f909/psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b"},"provenance":null,"requires-python":">=3.6","size":240369,"upload-time":"2025-11-02T12:26:14.358801Z","url":"https://files.pythonhosted.org/packages/b8/66/853d50e75a38c9a7370ddbeefabdd3d3116b9c31ef94dc92c6729bc36bec/psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9"},"provenance":null,"requires-python":">=3.6","size":288210,"upload-time":"2025-11-02T12:26:16.699739Z","url":"https://files.pythonhosted.org/packages/41/bd/313aba97cb5bfb26916dc29cf0646cbe4dd6a89ca69e8c6edce654876d39/psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f"},"provenance":null,"requires-python":">=3.6","size":291182,"upload-time":"2025-11-02T12:26:18.848963Z","url":"https://files.pythonhosted.org/packages/c2/fa/76e3c06e760927a0cfb5705eb38164254de34e9bd86db656d4dbaa228b04/psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7"},"provenance":null,"requires-python":">=3.6","size":250466,"upload-time":"2025-11-02T12:26:21.183069Z","url":"https://files.pythonhosted.org/packages/0f/1d/5774a91607035ee5078b8fd747686ebec28a962f178712de100d00b78a32/psutil-7.1.3-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264"},"provenance":null,"requires-python":">=3.6","size":245756,"upload-time":"2025-11-02T12:26:23.148427Z","url":"https://files.pythonhosted.org/packages/00/ca/e426584bacb43a5cb1ac91fae1937f478cd8fbe5e4ff96574e698a2c77cd/psutil-7.1.3-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab"},"provenance":null,"requires-python":">=3.6","size":238359,"upload-time":"2025-11-02T12:26:25.284599Z","url":"https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880"},"provenance":null,"requires-python":">=3.6","size":239171,"upload-time":"2025-11-02T12:26:27.230751Z","url":"https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3"},"provenance":null,"requires-python":">=3.6","size":263261,"upload-time":"2025-11-02T12:26:29.480632Z","url":"https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b"},"provenance":null,"requires-python":">=3.6","size":264635,"upload-time":"2025-11-02T12:26:31.740762Z","url":"https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"89db7acafbe66cd8b6a060ae993afda9c8f50aa4342c34c92a284be2c8c8534f"},"data-dist-info-metadata":{"sha256":"89db7acafbe66cd8b6a060ae993afda9c8f50aa4342c34c92a284be2c8c8534f"},"filename":"psutil-7.1.3-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd"},"provenance":null,"requires-python":">=3.6","size":247633,"upload-time":"2025-11-02T12:26:33.887174Z","url":"https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp37-abi3-win_arm64.whl","hashes":{"sha256":"bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1"},"provenance":null,"requires-python":">=3.6","size":244608,"upload-time":"2025-11-02T12:26:36.136434Z","url":"https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.3.tar.gz","hashes":{"sha256":"6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74"},"provenance":null,"requires-python":">=3.6","size":489059,"upload-time":"2025-11-02T12:25:54.619294Z","url":"https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"c31e927555539132a00380c971816ea43d089bf4bd5f3e918ed8c16776d68474"},"provenance":null,"requires-python":">=3.6","size":129593,"upload-time":"2025-12-23T20:26:28.019569Z","url":"https://files.pythonhosted.org/packages/a8/8e/b35aae6ed19bc4e2286cac4832e4d522fcf00571867b0a85a3f77ef96a80/psutil-7.2.0-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"db8e44e766cef86dea47d9a1fa535d38dc76449e5878a92f33683b7dba5bfcb2"},"provenance":null,"requires-python":">=3.6","size":130104,"upload-time":"2025-12-23T20:26:30.270800Z","url":"https://files.pythonhosted.org/packages/61/a2/773d17d74e122bbffe08b97f73f2d4a01ef53fb03b98e61b8e4f64a9c6b9/psutil-7.2.0-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"85ef849ac92169dedc59a7ac2fb565f47b3468fbe1524bf748746bc21afb94c7"},"provenance":null,"requires-python":">=3.6","size":180579,"upload-time":"2025-12-23T20:26:32.628357Z","url":"https://files.pythonhosted.org/packages/0d/e3/d3a9b3f4bd231abbd70a988beb2e3edd15306051bccbfc4472bd34a56e01/psutil-7.2.0-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"26782bdbae2f5c14ce9ebe8ad2411dc2ca870495e0cd90f8910ede7fa5e27117"},"provenance":null,"requires-python":">=3.6","size":183171,"upload-time":"2025-12-23T20:26:34.972726Z","url":"https://files.pythonhosted.org/packages/66/f8/6c73044424aabe1b7824d4d4504029d406648286d8fe7ba8c4682e0d3042/psutil-7.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"b7665f612d3b38a583391b95969667a53aaf6c5706dc27a602c9a4874fbf09e4"},"provenance":null,"requires-python":">=3.6","size":139055,"upload-time":"2025-12-23T20:26:36.848832Z","url":"https://files.pythonhosted.org/packages/48/7d/76d7a863340885d41826562225a566683e653ee6c9ba03c9f3856afa7d80/psutil-7.2.0-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"4413373c174520ae28a24a8974ad8ce6b21f060d27dde94e25f8c73a7effe57a"},"provenance":null,"requires-python":">=3.6","size":134737,"upload-time":"2025-12-23T20:26:38.784066Z","url":"https://files.pythonhosted.org/packages/a0/48/200054ada0ae4872c8a71db54f3eb6a9af4101680ee6830d373b7fda526b/psutil-7.2.0-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"2f2f53fd114e7946dfba3afb98c9b7c7f376009447360ca15bfb73f2066f84c7"},"provenance":null,"requires-python":">=3.6","size":129692,"upload-time":"2025-12-23T20:26:40.623176Z","url":"https://files.pythonhosted.org/packages/44/86/98da45dff471b93ef5ce5bcaefa00e3038295a7880a77cf74018243d37fb/psutil-7.2.0-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"e65c41d7e60068f60ce43b31a3a7fc90deb0dfd34ffc824a2574c2e5279b377e"},"provenance":null,"requires-python":">=3.6","size":130110,"upload-time":"2025-12-23T20:26:42.569228Z","url":"https://files.pythonhosted.org/packages/50/ee/10eae91ba4ad071c92db3c178ba861f30406342de9f0ddbe6d51fd741236/psutil-7.2.0-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"cc66d21366850a4261412ce994ae9976bba9852dafb4f2fa60db68ed17ff5281"},"provenance":null,"requires-python":">=3.6","size":181487,"upload-time":"2025-12-23T20:26:44.633020Z","url":"https://files.pythonhosted.org/packages/87/3a/2b2897443d56fedbbc34ac68a0dc7d55faa05d555372a2f989109052f86d/psutil-7.2.0-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"e025d67b42b8f22b096d5d20f5171de0e0fefb2f0ce983a13c5a1b5ed9872706"},"provenance":null,"requires-python":">=3.6","size":184320,"upload-time":"2025-12-23T20:26:46.830615Z","url":"https://files.pythonhosted.org/packages/11/66/44308428f7333db42c5ea7390c52af1b38f59b80b80c437291f58b5dfdad/psutil-7.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"45f6b91f7ad63414d6454fd609e5e3556d0e1038d5d9c75a1368513bdf763f57"},"provenance":null,"requires-python":">=3.6","size":140372,"upload-time":"2025-12-23T20:26:49.334377Z","url":"https://files.pythonhosted.org/packages/18/28/d2feadc7f18e501c5ce687c377db7dca924585418fd694272b8e488ea99f/psutil-7.2.0-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"87b18a19574139d60a546e88b5f5b9cbad598e26cdc790d204ab95d7024f03ee"},"provenance":null,"requires-python":">=3.6","size":135400,"upload-time":"2025-12-23T20:26:51.585604Z","url":"https://files.pythonhosted.org/packages/b2/1d/48381f5fd0425aa054c4ee3de24f50de3d6c347019f3aec75f357377d447/psutil-7.2.0-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"977a2fcd132d15cb05b32b2d85b98d087cad039b0ce435731670ba74da9e6133"},"provenance":null,"requires-python":">=3.6","size":128116,"upload-time":"2025-12-23T20:26:53.516520Z","url":"https://files.pythonhosted.org/packages/40/c5/a49160bf3e165b7b93a60579a353cf5d939d7f878fe5fd369110f1d18043/psutil-7.2.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"24151011c21fadd94214d7139d7c6c54569290d7e553989bdf0eab73b13beb8c"},"provenance":null,"requires-python":">=3.6","size":128925,"upload-time":"2025-12-23T20:26:55.573324Z","url":"https://files.pythonhosted.org/packages/10/a1/c75feb480f60cd768fb6ed00ac362a16a33e5076ec8475a22d8162fb2659/psutil-7.2.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"91f211ba9279e7c61d9d8f84b713cfc38fa161cb0597d5cb3f1ca742f6848254"},"provenance":null,"requires-python":">=3.6","size":154666,"upload-time":"2025-12-23T20:26:57.312776Z","url":"https://files.pythonhosted.org/packages/12/ff/e93136587c00a543f4bc768b157fac2c47cd77b180d4f4e5c6efb6ea53a2/psutil-7.2.0-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"f37415188b7ea98faf90fed51131181646c59098b077550246e2e092e127418b"},"provenance":null,"requires-python":">=3.6","size":156109,"upload-time":"2025-12-23T20:26:58.851353Z","url":"https://files.pythonhosted.org/packages/b8/dd/4c2de9c3827c892599d277a69d2224136800870a8a88a80981de905de28d/psutil-7.2.0-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-musllinux_1_2_aarch64.whl","hashes":{"sha256":"0d12c7ce6ed1128cd81fd54606afa054ac7dbb9773469ebb58cf2f171c49f2ac"},"provenance":null,"requires-python":">=3.6","size":148081,"upload-time":"2025-12-23T20:27:01.318270Z","url":"https://files.pythonhosted.org/packages/81/3f/090943c682d3629968dd0b04826ddcbc760ee1379021dbe316e2ddfcd01b/psutil-7.2.0-cp36-abi3-musllinux_1_2_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-musllinux_1_2_x86_64.whl","hashes":{"sha256":"ca0faef7976530940dcd39bc5382d0d0d5eb023b186a4901ca341bd8d8684151"},"provenance":null,"requires-python":">=3.6","size":147376,"upload-time":"2025-12-23T20:27:03.347816Z","url":"https://files.pythonhosted.org/packages/c4/88/c39648ebb8ec182d0364af53cdefe6eddb5f3872ba718b5855a8ff65d6d4/psutil-7.2.0-cp36-abi3-musllinux_1_2_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"758019d458979edff64acde45968217765d5828ba125593a164a985cdc0754bd"},"data-dist-info-metadata":{"sha256":"758019d458979edff64acde45968217765d5828ba125593a164a985cdc0754bd"},"filename":"psutil-7.2.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"abdb74137ca232d20250e9ad471f58d500e7743bc8253ba0bfbf26e570c0e437"},"provenance":null,"requires-python":">=3.6","size":136910,"upload-time":"2025-12-23T20:27:05.289344Z","url":"https://files.pythonhosted.org/packages/01/a2/5b39e08bd9b27476bc7cce7e21c71a481ad60b81ffac49baf02687a50d7f/psutil-7.2.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp37-abi3-win_arm64.whl","hashes":{"sha256":"284e71038b3139e7ab3834b63b3eb5aa5565fcd61a681ec746ef9a0a8c457fd2"},"provenance":null,"requires-python":">=3.6","size":133807,"upload-time":"2025-12-23T20:27:06.825130Z","url":"https://files.pythonhosted.org/packages/59/54/53839db1258c1eaeb4ded57ff202144ebc75b23facc05a74fd98d338b0c6/psutil-7.2.0-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.2.0.tar.gz","hashes":{"sha256":"2e4f8e1552f77d14dc96fb0f6240c5b34a37081c0889f0853b3b29a496e5ef64"},"provenance":null,"requires-python":">=3.6","size":489863,"upload-time":"2025-12-23T20:26:24.616214Z","url":"https://files.pythonhosted.org/packages/be/7c/31d1c3ceb1260301f87565f50689dc6da3db427ece1e1e012af22abca54e/psutil-7.2.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d"},"provenance":null,"requires-python":">=3.6","size":129624,"upload-time":"2025-12-29T08:26:04.255517Z","url":"https://files.pythonhosted.org/packages/77/8e/f0c242053a368c2aa89584ecd1b054a18683f13d6e5a318fc9ec36582c94/psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49"},"provenance":null,"requires-python":">=3.6","size":130132,"upload-time":"2025-12-29T08:26:06.228150Z","url":"https://files.pythonhosted.org/packages/26/97/a58a4968f8990617decee234258a2b4fc7cd9e35668387646c1963e69f26/psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc"},"provenance":null,"requires-python":">=3.6","size":180612,"upload-time":"2025-12-29T08:26:08.276538Z","url":"https://files.pythonhosted.org/packages/db/6d/ed44901e830739af5f72a85fa7ec5ff1edea7f81bfbf4875e409007149bd/psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf"},"provenance":null,"requires-python":">=3.6","size":183201,"upload-time":"2025-12-29T08:26:10.622093Z","url":"https://files.pythonhosted.org/packages/c7/65/b628f8459bca4efbfae50d4bf3feaab803de9a160b9d5f3bd9295a33f0c2/psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f"},"provenance":null,"requires-python":">=3.6","size":139081,"upload-time":"2025-12-29T08:26:12.483257Z","url":"https://files.pythonhosted.org/packages/fb/23/851cadc9764edcc18f0effe7d0bf69f727d4cf2442deb4a9f78d4e4f30f2/psutil-7.2.1-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672"},"provenance":null,"requires-python":">=3.6","size":134767,"upload-time":"2025-12-29T08:26:14.528248Z","url":"https://files.pythonhosted.org/packages/59/82/d63e8494ec5758029f31c6cb06d7d161175d8281e91d011a4a441c8a43b5/psutil-7.2.1-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679"},"provenance":null,"requires-python":">=3.6","size":129716,"upload-time":"2025-12-29T08:26:16.017500Z","url":"https://files.pythonhosted.org/packages/05/c2/5fb764bd61e40e1fe756a44bd4c21827228394c17414ade348e28f83cd79/psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f"},"provenance":null,"requires-python":">=3.6","size":130133,"upload-time":"2025-12-29T08:26:18.009017Z","url":"https://files.pythonhosted.org/packages/c9/d2/935039c20e06f615d9ca6ca0ab756cf8408a19d298ffaa08666bc18dc805/psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129"},"provenance":null,"requires-python":">=3.6","size":181518,"upload-time":"2025-12-29T08:26:20.241501Z","url":"https://files.pythonhosted.org/packages/77/69/19f1eb0e01d24c2b3eacbc2f78d3b5add8a89bf0bb69465bc8d563cc33de/psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a"},"provenance":null,"requires-python":">=3.6","size":184348,"upload-time":"2025-12-29T08:26:22.215886Z","url":"https://files.pythonhosted.org/packages/e1/6d/7e18b1b4fa13ad370787626c95887b027656ad4829c156bb6569d02f3262/psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79"},"provenance":null,"requires-python":">=3.6","size":140400,"upload-time":"2025-12-29T08:26:23.993885Z","url":"https://files.pythonhosted.org/packages/98/60/1672114392dd879586d60dd97896325df47d9a130ac7401318005aab28ec/psutil-7.2.1-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266"},"provenance":null,"requires-python":">=3.6","size":135430,"upload-time":"2025-12-29T08:26:25.999852Z","url":"https://files.pythonhosted.org/packages/fb/7b/d0e9d4513c46e46897b46bcfc410d51fc65735837ea57a25170f298326e6/psutil-7.2.1-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42"},"provenance":null,"requires-python":">=3.6","size":128137,"upload-time":"2025-12-29T08:26:27.759659Z","url":"https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1"},"provenance":null,"requires-python":">=3.6","size":128947,"upload-time":"2025-12-29T08:26:29.548034Z","url":"https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8"},"provenance":null,"requires-python":">=3.6","size":154694,"upload-time":"2025-12-29T08:26:32.147774Z","url":"https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6"},"provenance":null,"requires-python":">=3.6","size":156136,"upload-time":"2025-12-29T08:26:34.079911Z","url":"https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl","hashes":{"sha256":"f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8"},"provenance":null,"requires-python":">=3.6","size":148108,"upload-time":"2025-12-29T08:26:36.225796Z","url":"https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl","hashes":{"sha256":"99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67"},"provenance":null,"requires-python":">=3.6","size":147402,"upload-time":"2025-12-29T08:26:39.210698Z","url":"https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"54955c981ab8a8b2b87efc900b6916c908a315660b0c3c5d205c910565bb3261"},"data-dist-info-metadata":{"sha256":"54955c981ab8a8b2b87efc900b6916c908a315660b0c3c5d205c910565bb3261"},"filename":"psutil-7.2.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17"},"provenance":null,"requires-python":">=3.6","size":136938,"upload-time":"2025-12-29T08:26:41.036253Z","url":"https://files.pythonhosted.org/packages/34/68/d9317542e3f2b180c4306e3f45d3c922d7e86d8ce39f941bb9e2e9d8599e/psutil-7.2.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp37-abi3-win_arm64.whl","hashes":{"sha256":"0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442"},"provenance":null,"requires-python":">=3.6","size":133836,"upload-time":"2025-12-29T08:26:43.086974Z","url":"https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.2.1.tar.gz","hashes":{"sha256":"f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3"},"provenance":null,"requires-python":">=3.6","size":490253,"upload-time":"2025-12-29T08:26:00.169622Z","url":"https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz","yanked":false}],"meta":{"_last-serial":33241970,"api-version":"1.4"},"name":"psutil","project-status":{"status":"active"},"versions":["0.1.1","0.1.2","0.1.3","0.2.0","0.2.1","0.3.0","0.4.0","0.4.1","0.5.0","0.5.1","0.6.0","0.6.1","0.7.0","0.7.1","1.0.0","1.0.1","1.1.0","1.1.1","1.1.2","1.1.3","1.2.0","1.2.1","2.0.0","2.1.0","2.1.1","2.1.2","2.1.3","2.2.0","2.2.1","3.0.0","3.0.1","3.1.0","3.1.1","3.2.0","3.2.1","3.2.2","3.3.0","3.4.1","3.4.2","4.0.0","4.1.0","4.2.0","4.3.0","4.3.1","4.4.0","4.4.1","4.4.2","5.0.0","5.0.1","5.1.0","5.1.1","5.1.2","5.1.3","5.2.0","5.2.1","5.2.2","5.3.0","5.3.1","5.4.0","5.4.1","5.4.2","5.4.3","5.4.4","5.4.5","5.4.6","5.4.7","5.4.8","5.5.0","5.5.1","5.6.0","5.6.1","5.6.2","5.6.3","5.6.4","5.6.5","5.6.6","5.6.7","5.7.0","5.7.1","5.7.2","5.7.3","5.8.0","5.9.0","5.9.1","5.9.2","5.9.3","5.9.4","5.9.5","5.9.6","5.9.7","5.9.8","6.0.0","6.1.0","6.1.1","7.0.0","7.1.0","7.1.1","7.1.2","7.1.3","7.2.0","7.2.1"]} diff --git a/benches/benchmarks/json_loads.py b/benches/benchmarks/json_loads.py new file mode 100644 index 00000000000..f67f14d2d9e --- /dev/null +++ b/benches/benchmarks/json_loads.py @@ -0,0 +1,7 @@ +import json + +with open('benches/_data/pypi_org__simple__psutil.json') as f: + data = f.read() + + +loaded = json.loads(data) diff --git a/benches/execution.rs b/benches/execution.rs index c2239b59d12..2816f0e8201 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -24,7 +24,9 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) { settings.path_list.push("Lib/".to_string()); settings.write_bytecode = false; settings.user_site_directory = false; - Interpreter::without_stdlib(settings).enter(|vm| { + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder.add_native_modules(&defs).build().enter(|vm| { // Note: bench_cpython is both compiling and executing the code. // As such we compile the code in the benchmark loop as well. b.iter(|| { diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 98993b41543..ba5dcd6c2ec 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -113,12 +113,10 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc settings.write_bytecode = false; settings.user_site_directory = false; - Interpreter::with_init(settings, |vm| { - for (name, init) in rustpython_stdlib::get_module_inits() { - vm.add_native_module(name, init); - } - }) - .enter(|vm| { + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); + interp.enter(|vm| { let setup_code = vm .compile(&bench.setup, Mode::Exec, bench.name.to_owned()) .expect("Error compiling setup code"); diff --git a/crates/codegen/Cargo.toml b/crates/codegen/Cargo.toml index ce7e8d74f59..78065962fff 100644 --- a/crates/codegen/Cargo.toml +++ b/crates/codegen/Cargo.toml @@ -8,6 +8,10 @@ rust-version.workspace = true repository.workspace = true license.workspace = true +[features] +default = ["std"] +std = ["thiserror/std", "itertools/use_std"] + [dependencies] rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 9620edbd107..205facd65bd 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -16,32 +16,68 @@ use crate::{ symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, unparse::UnparseExpr, }; +use alloc::borrow::Cow; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex; use num_traits::{Num, ToPrimitive}; -use ruff_python_ast::{ - Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, - ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext, - ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, - FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedElement, - InterpolatedStringElement, InterpolatedStringElements, Keyword, MatchCase, ModExpression, - ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchClass, - PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, - PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, - TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, -}; -use ruff_text_size::{Ranged, TextRange}; +use ruff_python_ast as ast; +use ruff_text_size::{Ranged, TextRange, TextSize}; use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ - self, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, - ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, Invert, OpArg, OpArgType, + self, AnyInstruction, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, + ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, IntrinsicFunction1, + Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, SpecialMethod, UnpackExArgs, }, }; use rustpython_wtf8::Wtf8Buf; -use std::{borrow::Cow, collections::HashSet}; + +/// Extension trait for `ast::Expr` to add constant checking methods +trait ExprExt { + /// Check if an expression is a constant literal + fn is_constant(&self) -> bool; + + /// Check if a slice expression has all constant elements + fn is_constant_slice(&self) -> bool; + + /// Check if we should use BINARY_SLICE/STORE_SLICE optimization + fn should_use_slice_optimization(&self) -> bool; +} + +impl ExprExt for ast::Expr { + fn is_constant(&self) -> bool { + matches!( + self, + ast::Expr::NumberLiteral(_) + | ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::EllipsisLiteral(_) + ) + } + + fn is_constant_slice(&self) -> bool { + match self { + ast::Expr::Slice(s) => { + let lower_const = + s.lower.is_none() || s.lower.as_deref().is_some_and(|e| e.is_constant()); + let upper_const = + s.upper.is_none() || s.upper.as_deref().is_some_and(|e| e.is_constant()); + let step_const = + s.step.is_none() || s.step.as_deref().is_some_and(|e| e.is_constant()); + lower_const && upper_const && step_const + } + _ => false, + } + } + + fn should_use_slice_optimization(&self) -> bool { + !self.is_constant_slice() && matches!(self, ast::Expr::Slice(s) if s.step.is_none()) + } +} const MAXBLOCKS: usize = 20; @@ -62,12 +98,36 @@ pub enum FBlockType { StopIteration, } +/// Stores additional data for fblock unwinding +// fb_datum +#[derive(Debug, Clone)] +pub enum FBlockDatum { + None, + /// For FinallyTry: stores the finally body statements to compile during unwind + FinallyBody(Vec<ast::Stmt>), + /// For HandlerCleanup: stores the exception variable name (e.g., "e" in "except X as e") + ExceptionName(String), +} + +/// Type of super() call optimization detected by can_optimize_super_call() +#[derive(Debug, Clone)] +enum SuperCallType<'a> { + /// super(class, self) - explicit 2-argument form + TwoArg { + class_arg: &'a ast::Expr, + self_arg: &'a ast::Expr, + }, + /// super() - implicit 0-argument form (uses __class__ cell) + ZeroArg, +} + #[derive(Debug, Clone)] pub struct FBlockInfo { pub fb_type: FBlockType, pub fb_block: BlockIdx, pub fb_exit: BlockIdx, - // fb_datum is not needed in RustPython + // additional data for fblock unwinding + pub fb_datum: FBlockDatum, } pub(crate) type InternalResult<T> = Result<T, InternalError>; @@ -79,20 +139,6 @@ enum NameUsage { Store, Delete, } - -enum CallType { - Positional { nargs: u32 }, - Keyword { nargs: u32 }, - Ex { has_kwargs: bool }, -} - -fn is_forbidden_name(name: &str) -> bool { - // See https://docs.python.org/3/library/constants.html#built-in-constants - const BUILTIN_CONSTANTS: &[&str] = &["__debug__"]; - - BUILTIN_CONSTANTS.contains(&name) -} - /// Main structure holding the state of compilation. struct Compiler { code_stack: Vec<ir::CodeInfo>, @@ -105,19 +151,34 @@ struct Compiler { ctx: CompileContext, opts: CompileOpts, in_annotation: bool, + /// True when compiling in "single" (interactive) mode. + /// Expression statements at module scope emit CALL_INTRINSIC_1(Print). + interactive: bool, } +#[derive(Clone, Copy)] enum DoneWithFuture { No, DoneWithDoc, Yes, } -#[derive(Debug, Clone, Default)] +#[derive(Clone, Copy, Debug)] pub struct CompileOpts { /// How optimized the bytecode output should be; any optimize > 0 does /// not emit assert statements pub optimize: u8, + /// Include column info in bytecode (-X no_debug_ranges disables) + pub debug_ranges: bool, +} + +impl Default for CompileOpts { + fn default() -> Self { + Self { + optimize: 0, + debug_ranges: true, + } + } } #[derive(Debug, Clone, Copy)] @@ -125,6 +186,8 @@ struct CompileContext { loop_data: Option<(BlockIdx, BlockIdx)>, in_class: bool, func: FunctionContext, + /// True if we're anywhere inside an async function (even inside nested comprehensions) + in_async_scope: bool, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -148,8 +211,8 @@ enum ComprehensionType { Dict, } -fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { - let mut seen_params = HashSet::new(); +fn validate_duplicate_params(params: &ast::Parameters) -> Result<(), CodegenErrorType> { + let mut seen_params = IndexSet::default(); for param in params { let param_name = param.name().as_str(); if !seen_params.insert(param_name) { @@ -181,7 +244,7 @@ pub fn compile_top( /// Compile a standard Python program to bytecode pub fn compile_program( - ast: &ModModule, + ast: &ast::ModModule, source_file: SourceFile, opts: CompileOpts, ) -> CompileResult<CodeObject> { @@ -196,7 +259,7 @@ pub fn compile_program( /// Compile a Python program to bytecode for the context of a REPL pub fn compile_program_single( - ast: &ModModule, + ast: &ast::ModModule, source_file: SourceFile, opts: CompileOpts, ) -> CompileResult<CodeObject> { @@ -210,7 +273,7 @@ pub fn compile_program_single( } pub fn compile_block_expression( - ast: &ModModule, + ast: &ast::ModModule, source_file: SourceFile, opts: CompileOpts, ) -> CompileResult<CodeObject> { @@ -224,7 +287,7 @@ pub fn compile_block_expression( } pub fn compile_expression( - ast: &ModExpression, + ast: &ast::ModExpression, source_file: SourceFile, opts: CompileOpts, ) -> CompileResult<CodeObject> { @@ -237,17 +300,24 @@ pub fn compile_expression( } macro_rules! emit { - ($c:expr, Instruction::$op:ident { $arg:ident$(,)? }$(,)?) => { - $c.emit_arg($arg, |x| Instruction::$op { $arg: x }) + // Struct variant with single identifier (e.g., Foo::A { arg }) + ($c:expr, $enum:ident :: $op:ident { $arg:ident $(,)? } $(,)?) => { + $c.emit_arg($arg, |x| $enum::$op { $arg: x }) }; - ($c:expr, Instruction::$op:ident { $arg:ident : $arg_val:expr $(,)? }$(,)?) => { - $c.emit_arg($arg_val, |x| Instruction::$op { $arg: x }) + + // Struct variant with explicit value (e.g., Foo::A { arg: 42 }) + ($c:expr, $enum:ident :: $op:ident { $arg:ident : $arg_val:expr $(,)? } $(,)?) => { + $c.emit_arg($arg_val, |x| $enum::$op { $arg: x }) }; - ($c:expr, Instruction::$op:ident( $arg_val:expr $(,)? )$(,)?) => { - $c.emit_arg($arg_val, Instruction::$op) + + // Tuple variant (e.g., Foo::B(42)) + ($c:expr, $enum:ident :: $op:ident($arg_val:expr $(,)? ) $(,)?) => { + $c.emit_arg($arg_val, $enum::$op) }; - ($c:expr, Instruction::$op:ident$(,)?) => { - $c.emit_no_arg(Instruction::$op) + + // No-arg variant (e.g., Foo::C) + ($c:expr, $enum:ident :: $op:ident $(,)?) => { + $c.emit_no_arg($enum::$op) }; } @@ -292,7 +362,7 @@ fn compiler_unwrap_option<T>(zelf: &Compiler, o: Option<T>) -> T { o.unwrap() } -// fn compiler_result_unwrap<T, E: std::fmt::Debug>(zelf: &Compiler, result: Result<T, E>) -> T { +// fn compiler_result_unwrap<T, E: core::fmt::Debug>(zelf: &Compiler, result: Result<T, E>) -> T { // if result.is_err() { // eprintln!("=== CODEGEN PANIC INFO ==="); // eprintln!("This IS an internal error, an result was unwrapped during codegen"); @@ -352,7 +422,7 @@ enum CollectionType { impl Compiler { fn new(opts: CompileOpts, source_file: SourceFile, code_name: String) -> Self { let module_code = ir::CodeInfo { - flags: bytecode::CodeFlags::NEW_LOCALS, + flags: bytecode::CodeFlags::NEWLOCALS, source_path: source_file.name().to_owned(), private: None, blocks: vec![ir::Block::default()], @@ -375,6 +445,8 @@ impl Compiler { in_inlined_comp: false, fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: 0, // Module is always the first symbol table + in_conditional_block: 0, + next_conditional_annotation_index: 0, }; Self { code_stack: vec![module_code], @@ -388,100 +460,81 @@ impl Compiler { loop_data: None, in_class: false, func: FunctionContext::NoFunction, + in_async_scope: false, }, opts, in_annotation: false, + interactive: false, } } - /// Check if the slice is a two-element slice (no step) - // = is_two_element_slice - const fn is_two_element_slice(slice: &Expr) -> bool { - matches!(slice, Expr::Slice(s) if s.step.is_none()) - } - - /// Compile a slice expression - // = compiler_slice - fn compile_slice(&mut self, s: &ExprSlice) -> CompileResult<BuildSliceArgCount> { - // Compile lower + /// Compile just start and stop of a slice (for BINARY_SLICE/STORE_SLICE) + // = codegen_slice_two_parts + fn compile_slice_two_parts(&mut self, s: &ast::ExprSlice) -> CompileResult<()> { + // Compile lower (or None) if let Some(lower) = &s.lower { self.compile_expression(lower)?; } else { self.emit_load_const(ConstantData::None); } - // Compile upper + // Compile upper (or None) if let Some(upper) = &s.upper { self.compile_expression(upper)?; } else { self.emit_load_const(ConstantData::None); } - Ok(match &s.step { - Some(step) => { - // Compile step if present - self.compile_expression(step)?; - BuildSliceArgCount::Three - } - None => BuildSliceArgCount::Two, - }) + Ok(()) } - /// Compile a subscript expression // = compiler_subscript fn compile_subscript( &mut self, - value: &Expr, - slice: &Expr, - ctx: ExprContext, + value: &ast::Expr, + slice: &ast::Expr, + ctx: ast::ExprContext, ) -> CompileResult<()> { - // 1. Check subscripter and index for Load context - // 2. VISIT value - // 3. Handle two-element slice specially - // 4. Otherwise VISIT slice and emit appropriate instruction - - // For Load context, CPython does some checks (we skip for now) - // if ctx == ExprContext::Load { - // check_subscripter(value); - // check_index(value, slice); - // } + // Save full subscript expression range (set by compile_expression before this call) + let subscript_range = self.current_source_range; // VISIT(c, expr, e->v.Subscript.value) self.compile_expression(value)?; - // Handle two-element slice (for Load/Store, not Del) - if Self::is_two_element_slice(slice) && !matches!(ctx, ExprContext::Del) { - let argc = match slice { - Expr::Slice(s) => self.compile_slice(s)?, - _ => unreachable!("is_two_element_slice should only return true for Expr::Slice"), + // Handle two-element non-constant slice with BINARY_SLICE/STORE_SLICE + let use_slice_opt = matches!(ctx, ast::ExprContext::Load | ast::ExprContext::Store) + && slice.should_use_slice_optimization(); + if use_slice_opt { + match slice { + ast::Expr::Slice(s) => self.compile_slice_two_parts(s)?, + _ => unreachable!( + "should_use_slice_optimization should only return true for ast::Expr::Slice" + ), }; - match ctx { - ExprContext::Load => { - // CPython uses BINARY_SLICE - emit!(self, Instruction::BuildSlice { argc }); - emit!(self, Instruction::Subscript); - } - ExprContext::Store => { - // CPython uses STORE_SLICE - emit!(self, Instruction::BuildSlice { argc }); - emit!(self, Instruction::StoreSubscript); - } - _ => unreachable!(), - } } else { // VISIT(c, expr, e->v.Subscript.slice) self.compile_expression(slice)?; + } - // Emit appropriate instruction based on context - match ctx { - ExprContext::Load => emit!(self, Instruction::Subscript), - ExprContext::Store => emit!(self, Instruction::StoreSubscript), - ExprContext::Del => emit!(self, Instruction::DeleteSubscript), - ExprContext::Invalid => { - return Err(self.error(CodegenErrorType::SyntaxError( - "Invalid expression context".to_owned(), - ))); + // Restore full subscript expression range before emitting + self.set_source_range(subscript_range); + + match (use_slice_opt, ctx) { + (true, ast::ExprContext::Load) => emit!(self, Instruction::BinarySlice), + (true, ast::ExprContext::Store) => emit!(self, Instruction::StoreSlice), + (true, _) => unreachable!(), + (false, ast::ExprContext::Load) => emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr } + ), + (false, ast::ExprContext::Store) => emit!(self, Instruction::StoreSubscr), + (false, ast::ExprContext::Del) => emit!(self, Instruction::DeleteSubscr), + (false, ast::ExprContext::Invalid) => { + return Err(self.error(CodegenErrorType::SyntaxError( + "Invalid expression context".to_owned(), + ))); } } @@ -490,7 +543,7 @@ impl Compiler { /// Helper function for compiling tuples/lists/sets with starred expressions /// - /// Parameters: + /// ast::Parameters: /// - elts: The elements to compile /// - pushed: Number of items already on the stack /// - collection_type: What type of collection to build (tuple, list, set) @@ -498,42 +551,119 @@ impl Compiler { // = starunpack_helper in compile.c fn starunpack_helper( &mut self, - elts: &[Expr], + elts: &[ast::Expr], pushed: u32, collection_type: CollectionType, ) -> CompileResult<()> { - // Use RustPython's existing approach with BuildXFromTuples - let (size, unpack) = self.gather_elements(pushed, elts)?; + let n = elts.len().to_u32(); + let seen_star = elts.iter().any(|e| matches!(e, ast::Expr::Starred(_))); - if unpack { - // Has starred elements + // Determine collection size threshold for optimization + let big = match collection_type { + CollectionType::Set => n > 8, + _ => n > 4, + }; + + // If no stars and not too big, compile all elements and build once + if !seen_star && !big { + for elt in elts { + self.compile_expression(elt)?; + } + let total_size = n + pushed; match collection_type { + CollectionType::List => { + emit!(self, Instruction::BuildList { size: total_size }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { size: total_size }); + } CollectionType::Tuple => { - if size > 1 || pushed > 0 { - emit!(self, Instruction::BuildTupleFromTuples { size }); + emit!(self, Instruction::BuildTuple { size: total_size }); + } + } + return Ok(()); + } + + // Has stars or too big: use streaming approach + let mut sequence_built = false; + let mut i = 0u32; + + for elt in elts.iter() { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = elt { + // When we hit first star, build sequence with elements so far + if !sequence_built { + match collection_type { + CollectionType::List => { + emit!(self, Instruction::BuildList { size: i + pushed }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { size: i + pushed }); + } + CollectionType::Tuple => { + emit!(self, Instruction::BuildList { size: i + pushed }); + } } - // If size == 1 and pushed == 0, the single tuple is already on the stack + sequence_built = true; } - CollectionType::List => { - emit!(self, Instruction::BuildListFromTuples { size }); + + // Compile the starred expression and extend + self.compile_expression(value)?; + match collection_type { + CollectionType::List => { + emit!(self, Instruction::ListExtend { i: 0 }); + } + CollectionType::Set => { + emit!(self, Instruction::SetUpdate { i: 0 }); + } + CollectionType::Tuple => { + emit!(self, Instruction::ListExtend { i: 0 }); + } } - CollectionType::Set => { - emit!(self, Instruction::BuildSetFromTuples { size }); + } else { + // Non-starred element + self.compile_expression(elt)?; + + if sequence_built { + // Sequence already exists, append to it + match collection_type { + CollectionType::List => { + emit!(self, Instruction::ListAppend { i: 0 }); + } + CollectionType::Set => { + emit!(self, Instruction::SetAdd { i: 0 }); + } + CollectionType::Tuple => { + emit!(self, Instruction::ListAppend { i: 0 }); + } + } + } else { + // Still collecting elements before first star + i += 1; } } - } else { - // No starred elements + } + + // If we never built sequence (all non-starred), build it now + if !sequence_built { match collection_type { - CollectionType::Tuple => { - emit!(self, Instruction::BuildTuple { size }); - } CollectionType::List => { - emit!(self, Instruction::BuildList { size }); + emit!(self, Instruction::BuildList { size: i + pushed }); } CollectionType::Set => { - emit!(self, Instruction::BuildSet { size }); + emit!(self, Instruction::BuildSet { size: i + pushed }); + } + CollectionType::Tuple => { + emit!(self, Instruction::BuildTuple { size: i + pushed }); } } + } else if collection_type == CollectionType::Tuple { + // For tuples, convert the list to tuple + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); } Ok(()) @@ -607,25 +737,85 @@ impl Compiler { } /// Push the next symbol table on to the stack - fn push_symbol_table(&mut self) -> &SymbolTable { + fn push_symbol_table(&mut self) -> CompileResult<&SymbolTable> { // Look up the next table contained in the scope of the current table let current_table = self .symbol_table_stack .last_mut() .expect("no current symbol table"); - if current_table.sub_tables.is_empty() { - panic!( - "push_symbol_table: no sub_tables available in {} (type: {:?})", - current_table.name, current_table.typ - ); + if current_table.next_sub_table >= current_table.sub_tables.len() { + let name = current_table.name.clone(); + let typ = current_table.typ; + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "no symbol table available in {} (type: {:?})", + name, typ + )))); } - let table = current_table.sub_tables.remove(0); + let idx = current_table.next_sub_table; + current_table.next_sub_table += 1; + let table = current_table.sub_tables[idx].clone(); // Push the next table onto the stack self.symbol_table_stack.push(table); - self.current_symbol_table() + Ok(self.current_symbol_table()) + } + + /// Push the annotation symbol table from the next sub_table's annotation_block + /// The annotation_block is stored in the function's scope, which is the next sub_table + /// Returns true if annotation_block exists, false otherwise + fn push_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // The annotation_block is in the next sub_table (function scope) + let next_idx = current_table.next_sub_table; + if next_idx >= current_table.sub_tables.len() { + return false; + } + + let next_table = &mut current_table.sub_tables[next_idx]; + if let Some(annotation_block) = next_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Push the annotation symbol table for module/class level annotations + /// This takes annotation_block from the current symbol table (not sub_tables) + fn push_current_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // For modules/classes, annotation_block is directly in the current table + if let Some(annotation_block) = current_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Pop the annotation symbol table and restore it to the function scope's annotation_block + fn pop_annotation_symbol_table(&mut self) { + let annotation_table = self.symbol_table_stack.pop().expect("compiler bug"); + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // Restore to the next sub_table (function scope) where it came from + let next_idx = current_table.next_sub_table; + if next_idx < current_table.sub_tables.len() { + current_table.sub_tables[next_idx].annotation_block = Some(Box::new(annotation_table)); + } } /// Pop the current symbol table off the stack @@ -633,6 +823,159 @@ impl Compiler { self.symbol_table_stack.pop().expect("compiler bug") } + /// Check if a super() call can be optimized + /// Returns Some(SuperCallType) if optimization is possible, None otherwise + fn can_optimize_super_call<'a>( + &self, + value: &'a ast::Expr, + attr: &str, + ) -> Option<SuperCallType<'a>> { + // 1. value must be a Call expression + let ast::Expr::Call(ast::ExprCall { + func, arguments, .. + }) = value + else { + return None; + }; + + // 2. func must be Name("super") + let ast::Expr::Name(ast::ExprName { id, .. }) = func.as_ref() else { + return None; + }; + if id.as_str() != "super" { + return None; + } + + // 3. attr must not be "__class__" + if attr == "__class__" { + return None; + } + + // 4. No keyword arguments + if !arguments.keywords.is_empty() { + return None; + } + + // 5. Must be inside a function (not at module level or class body) + if !self.ctx.in_func() { + return None; + } + + // 6. "super" must be GlobalImplicit (not redefined locally or at module level) + let table = self.current_symbol_table(); + if let Some(symbol) = table.lookup("super") + && symbol.scope != SymbolScope::GlobalImplicit + { + return None; + } + // Also check top-level scope to detect module-level shadowing. + // Only block if super is actually *bound* at module level (not just used). + if let Some(top_table) = self.symbol_table_stack.first() + && let Some(sym) = top_table.lookup("super") + && sym.scope != SymbolScope::GlobalImplicit + { + return None; + } + + // 7. Check argument pattern + let args = &arguments.args; + + // No starred expressions allowed + if args.iter().any(|arg| matches!(arg, ast::Expr::Starred(_))) { + return None; + } + + match args.len() { + 2 => { + // 2-arg: super(class, self) + Some(SuperCallType::TwoArg { + class_arg: &args[0], + self_arg: &args[1], + }) + } + 0 => { + // 0-arg: super() - need __class__ cell and first parameter + // Enclosing function should have at least one positional argument + let info = self.code_stack.last()?; + if info.metadata.argcount == 0 && info.metadata.posonlyargcount == 0 { + return None; + } + + // Check if __class__ is available as a cell/free variable + // The scope must be Free (from enclosing class) or have FREE_CLASS flag + if let Some(symbol) = table.lookup("__class__") { + if symbol.scope != SymbolScope::Free + && !symbol.flags.contains(SymbolFlags::FREE_CLASS) + { + return None; + } + } else { + // __class__ not in symbol table, optimization not possible + return None; + } + + Some(SuperCallType::ZeroArg) + } + _ => None, // 1 or 3+ args - not optimizable + } + } + + /// Load arguments for super() optimization onto the stack + /// Stack result: [global_super, class, self] + fn load_args_for_super(&mut self, super_type: &SuperCallType<'_>) -> CompileResult<()> { + // 1. Load global super + self.compile_name("super", NameUsage::Load)?; + + match super_type { + SuperCallType::TwoArg { + class_arg, + self_arg, + } => { + // 2-arg: load provided arguments + self.compile_expression(class_arg)?; + self.compile_expression(self_arg)?; + } + SuperCallType::ZeroArg => { + // 0-arg: load __class__ cell and first parameter + // Load __class__ from cell/free variable + let scope = self.get_ref_type("__class__").map_err(|e| self.error(e))?; + let idx = match scope { + SymbolScope::Cell => self.get_cell_var_index("__class__")?, + SymbolScope::Free => self.get_free_var_index("__class__")?, + _ => { + return Err(self.error(CodegenErrorType::SyntaxError( + "super(): __class__ cell not found".to_owned(), + ))); + } + }; + self.emit_arg(idx, Instruction::LoadDeref); + + // Load first parameter (typically 'self'). + // Safety: can_optimize_super_call() ensures argcount > 0, and + // parameters are always added to varnames first (see symboltable.rs). + let first_param = { + let info = self.code_stack.last().unwrap(); + info.metadata.varnames.first().cloned() + }; + let first_param = first_param.ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError( + "super(): no arguments and no first parameter".to_owned(), + )) + })?; + self.compile_name(&first_param, NameUsage::Load)?; + } + } + Ok(()) + } + + /// Check if this is an inlined comprehension context (PEP 709) + /// Currently disabled - always returns false to avoid stack issues + fn is_inlined_comprehension_context(&self, _comprehension_type: ComprehensionType) -> bool { + // TODO: Implement PEP 709 inlined comprehensions properly + // For now, disabled to avoid stack underflow issues + false + } + /// Enter a new scope // = compiler_enter_scope fn enter_scope( @@ -642,12 +985,6 @@ impl Compiler { key: usize, // In RustPython, we use the index in symbol_table_stack as key lineno: u32, ) -> CompileResult<()> { - // Create location - let location = SourceLocation { - line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), - character_offset: OneIndexed::MIN, - }; - // Allocate a new compiler unit // In Rust, we'll create the structure directly @@ -693,6 +1030,12 @@ impl Compiler { cellvar_cache.insert("__classdict__".to_string()); } + // Handle implicit __conditional_annotations__ cell if needed + // Only for class scope - module scope uses NAME operations, not DEREF + if ste.has_conditional_annotations && scope_type == CompilerScope::Class { + cellvar_cache.insert("__conditional_annotations__".to_string()); + } + // Build freevars using dictbytype (FREE scope, offset by cellvars size) let mut freevar_cache = IndexSet::default(); let mut free_names: Vec<_> = ste @@ -713,21 +1056,27 @@ impl Compiler { CompilerScope::Module => (bytecode::CodeFlags::empty(), 0, 0, 0), CompilerScope::Class => (bytecode::CodeFlags::empty(), 0, 0, 0), CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => ( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, 0, // Will be set later in enter_function 0, // Will be set later in enter_function 0, // Will be set later in enter_function ), CompilerScope::Comprehension => ( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, 0, 1, // comprehensions take one argument (.0) 0, ), CompilerScope::TypeParams => ( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, 0, 0, + ), + CompilerScope::Annotation => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 1, // format is positional-only + 1, // annotation scope takes one argument (format) 0, ), }; @@ -768,6 +1117,8 @@ impl Compiler { in_inlined_comp: false, fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: key, + in_conditional_block: 0, + next_conditional_annotation_index: 0, }; // Push the old compiler unit on the stack (like PyCapsule) @@ -779,33 +1130,50 @@ impl Compiler { self.set_qualname(); } - // Emit RESUME instruction - let _resume_loc = if scope_type == CompilerScope::Module { - // Module scope starts with lineno 0 - SourceLocation { - line: OneIndexed::MIN, - character_offset: OneIndexed::MIN, - } - } else { - location - }; + // Emit RESUME (handles async preamble and module lineno 0) + // CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module + self.emit_resume_for_scope(scope_type, lineno); - // Set the source range for the RESUME instruction - // For now, just use an empty range at the beginning - self.current_source_range = TextRange::default(); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AtFuncStart as u32 - } - ); + Ok(()) + } - if scope_type == CompilerScope::Module { - // This would be loc.lineno = -1 in CPython - // We handle this differently in RustPython + /// Emit RESUME instruction with proper handling for async preamble and module lineno. + /// codegen_enter_scope equivalent for RESUME emission. + fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) { + // For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME + if scope_type == CompilerScope::AsyncFunction { + emit!(self, Instruction::ReturnGenerator); + emit!(self, Instruction::PopTop); } - Ok(()) + // CPython: LOCATION(lineno, lineno, 0, 0) + // Module scope: loc.lineno = 0 (before the first line) + let lineno_override = if scope_type == CompilerScope::Module { + Some(0) + } else { + None + }; + + // Use lineno for location (col = 0 as in CPython) + let location = SourceLocation { + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::MIN, // col = 0 + }; + let end_location = location; // end_lineno = lineno, end_col = 0 + let except_handler = None; + + self.current_block().instructions.push(ir::InstructionInfo { + instr: Instruction::Resume { + arg: OpArgMarker::marker(), + } + .into(), + arg: OpArg::new(u32::from(bytecode::ResumeType::AtFuncStart)), + target: BlockIdx::NULL, + location, + end_location, + except_handler, + lineno_override, + }); } fn push_output( @@ -815,9 +1183,9 @@ impl Compiler { arg_count: u32, kwonlyarg_count: u32, obj_name: String, - ) { + ) -> CompileResult<()> { // First push the symbol table - let table = self.push_symbol_table(); + let table = self.push_symbol_table()?; let scope_type = table.typ; // The key is the current position in the symbol table stack @@ -827,11 +1195,7 @@ impl Compiler { let lineno = self.get_source_line_number().get(); // Call enter_scope which does most of the work - if let Err(e) = self.enter_scope(&obj_name, scope_type, key, lineno.to_u32()) { - // In the current implementation, push_output doesn't return an error, - // so we panic here. This maintains the same behavior. - panic!("enter_scope failed: {e:?}"); - } + self.enter_scope(&obj_name, scope_type, key, lineno.to_u32())?; // Override the values that push_output sets explicitly // enter_scope sets default values based on scope_type, but push_output @@ -842,6 +1206,7 @@ impl Compiler { info.metadata.posonlyargcount = posonlyarg_count; info.metadata.kwonlyargcount = kwonlyarg_count; } + Ok(()) } // compiler_exit_scope @@ -849,7 +1214,7 @@ impl Compiler { let _table = self.pop_symbol_table(); // Various scopes can have sub_tables: - // - TypeParams scope can have sub_tables (the function body's symbol table) + // - ast::TypeParams scope can have sub_tables (the function body's symbol table) // - Module scope can have sub_tables (for TypeAlias scopes, nested functions, classes) // - Function scope can have sub_tables (for nested functions, classes) // - Class scope can have sub_tables (for nested classes, methods) @@ -857,7 +1222,101 @@ impl Compiler { let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); // No parent scope stack to maintain - unwrap_internal(self, stack_top.finalize_code(self.opts.optimize)) + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Exit annotation scope - similar to exit_scope but restores annotation_block to parent + fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { + self.pop_annotation_symbol_table(); + self.ctx = saved_ctx; + + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Enter annotation scope using the symbol table's annotation_block. + /// Returns None if no annotation_block exists. + /// On success, returns the saved CompileContext to pass to exit_annotation_scope. + fn enter_annotation_scope( + &mut self, + _func_name: &str, + ) -> CompileResult<Option<CompileContext>> { + if !self.push_annotation_symbol_table() { + return Ok(None); + } + + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope( + "__annotate__", + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Override arg_count since enter_scope sets it to 1 but we need the varnames + // setup to be correct too + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) + self.emit_format_validation()?; + + Ok(Some(saved_ctx)) + } + + /// Emit format parameter validation for annotation scope + /// if format > VALUE_WITH_FAKE_GLOBALS (2): raise NotImplementedError + fn emit_format_validation(&mut self) -> CompileResult<()> { + // Load format parameter (first local variable, index 0) + emit!(self, Instruction::LoadFast(0)); + + // Load VALUE_WITH_FAKE_GLOBALS constant (2) + self.emit_load_const(ConstantData::Integer { value: 2.into() }); + + // Compare: format > 2 + emit!( + self, + Instruction::CompareOp { + op: ComparisonOperator::Greater + } + ); + + // Jump to body if format <= 2 (comparison is false) + let body_block = self.new_block(); + emit!(self, Instruction::PopJumpIfFalse { target: body_block }); + + // Raise NotImplementedError + emit!( + self, + Instruction::LoadCommonConstant { + idx: bytecode::CommonConstant::NotImplementedError + } + ); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::Raise + } + ); + + // Body label - continue with annotation evaluation + self.switch_to_block(body_block); + + Ok(()) } /// Push a new fblock @@ -868,8 +1327,19 @@ impl Compiler { fb_block: BlockIdx, fb_exit: BlockIdx, ) -> CompileResult<()> { - let code = self.current_code_info(); - if code.fblock.len() >= MAXBLOCKS { + self.push_fblock_full(fb_type, fb_block, fb_exit, FBlockDatum::None) + } + + /// Push an fblock with all parameters including fb_datum + fn push_fblock_full( + &mut self, + fb_type: FBlockType, + fb_block: BlockIdx, + fb_exit: BlockIdx, + fb_datum: FBlockDatum, + ) -> CompileResult<()> { + let code = self.current_code_info(); + if code.fblock.len() >= MAXBLOCKS { return Err(self.error(CodegenErrorType::SyntaxError( "too many statically nested blocks".to_owned(), ))); @@ -878,6 +1348,7 @@ impl Compiler { fb_type, fb_block, fb_exit, + fb_datum, }); Ok(()) } @@ -891,17 +1362,207 @@ impl Compiler { code.fblock.pop().expect("fblock stack underflow") } + /// Unwind a single fblock, emitting cleanup code + /// preserve_tos: if true, preserve the top of stack (e.g., return value) + fn unwind_fblock(&mut self, info: &FBlockInfo, preserve_tos: bool) -> CompileResult<()> { + match info.fb_type { + FBlockType::WhileLoop + | FBlockType::ExceptionHandler + | FBlockType::ExceptionGroupHandler + | FBlockType::AsyncComprehensionGenerator + | FBlockType::StopIteration => { + // No cleanup needed + } + + FBlockType::ForLoop => { + // Pop the iterator + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + emit!(self, Instruction::PopIter); + } + + FBlockType::TryExcept => { + emit!(self, PseudoInstruction::PopBlock); + } + + FBlockType::FinallyTry => { + // FinallyTry is now handled specially in unwind_fblock_stack + // to avoid infinite recursion when the finally body contains return/break/continue. + // This branch should not be reached. + unreachable!("FinallyTry should be handled by unwind_fblock_stack"); + } + + FBlockType::FinallyEnd => { + // codegen_unwind_fblock(FINALLY_END) + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + emit!(self, Instruction::PopTop); // exc_value + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + } + + FBlockType::With | FBlockType::AsyncWith => { + // Stack when entering: [..., __exit__, return_value (if preserve_tos)] + // Need to call __exit__(None, None, None) + + emit!(self, PseudoInstruction::PopBlock); + + // If preserving return value, swap it below __exit__ + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + // Stack after swap: [..., return_value, __exit__] or [..., __exit__] + + // Call __exit__(None, None, None) + // Call protocol: [callable, self_or_null, arg1, arg2, arg3] + emit!(self, Instruction::PushNull); + // Stack: [..., __exit__, NULL] + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + // Stack: [..., __exit__, NULL, None, None, None] + emit!(self, Instruction::Call { nargs: 3 }); + + // For async with, await the result + if matches!(info.fb_type, FBlockType::AsyncWith) { + emit!(self, Instruction::GetAwaitable { arg: 2 }); + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + } + + // Pop the __exit__ result + emit!(self, Instruction::PopTop); + } + + FBlockType::HandlerCleanup => { + // codegen_unwind_fblock(HANDLER_CLEANUP) + if let FBlockDatum::ExceptionName(_) = info.fb_datum { + // Named handler: PopBlock for inner SETUP_CLEANUP + emit!(self, PseudoInstruction::PopBlock); + } + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + + // If there's an exception name, clean it up + if let FBlockDatum::ExceptionName(ref name) = info.fb_datum { + self.emit_load_const(ConstantData::None); + self.store_name(name)?; + self.compile_name(name, NameUsage::Delete)?; + } + } + + FBlockType::PopValue => { + if preserve_tos { + emit!(self, Instruction::Swap { index: 2 }); + } + emit!(self, Instruction::PopTop); + } + } + Ok(()) + } + + /// Unwind the fblock stack, emitting cleanup code for each block + /// preserve_tos: if true, preserve the top of stack (e.g., return value) + /// stop_at_loop: if true, stop when encountering a loop (for break/continue) + fn unwind_fblock_stack(&mut self, preserve_tos: bool, stop_at_loop: bool) -> CompileResult<()> { + // Collect the info we need, with indices for FinallyTry blocks + #[derive(Clone)] + enum UnwindInfo { + Normal(FBlockInfo), + FinallyTry { + body: Vec<ruff_python_ast::Stmt>, + fblock_idx: usize, + }, + } + let mut unwind_infos = Vec::new(); + + { + let code = self.current_code_info(); + for i in (0..code.fblock.len()).rev() { + // Check for exception group handler (forbidden) + if matches!(code.fblock[i].fb_type, FBlockType::ExceptionGroupHandler) { + return Err(self.error(CodegenErrorType::BreakContinueReturnInExceptStar)); + } + + // Stop at loop if requested + if stop_at_loop + && matches!( + code.fblock[i].fb_type, + FBlockType::WhileLoop | FBlockType::ForLoop + ) + { + break; + } + + if matches!(code.fblock[i].fb_type, FBlockType::FinallyTry) { + if let FBlockDatum::FinallyBody(ref body) = code.fblock[i].fb_datum { + unwind_infos.push(UnwindInfo::FinallyTry { + body: body.clone(), + fblock_idx: i, + }); + } + } else { + unwind_infos.push(UnwindInfo::Normal(code.fblock[i].clone())); + } + } + } + + // Process each fblock + for info in unwind_infos { + match info { + UnwindInfo::Normal(fblock_info) => { + self.unwind_fblock(&fblock_info, preserve_tos)?; + } + UnwindInfo::FinallyTry { body, fblock_idx } => { + // codegen_unwind_fblock(FINALLY_TRY) + emit!(self, PseudoInstruction::PopBlock); + + // Temporarily remove the FinallyTry fblock so nested return/break/continue + // in the finally body won't see it again + let code = self.current_code_info(); + let saved_fblock = code.fblock.remove(fblock_idx); + + // Push PopValue fblock if preserving tos + if preserve_tos { + self.push_fblock( + FBlockType::PopValue, + saved_fblock.fb_block, + saved_fblock.fb_block, + )?; + } + + self.compile_statements(&body)?; + + if preserve_tos { + self.pop_fblock(FBlockType::PopValue); + } + + // Restore the fblock + let code = self.current_code_info(); + code.fblock.insert(fblock_idx, saved_fblock); + } + } + } + + Ok(()) + } + // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never // actually have a `String` to pass fn name(&mut self, name: &str) -> bytecode::NameIdx { self._name_inner(name, |i| &mut i.metadata.names) } fn varname(&mut self, name: &str) -> CompileResult<bytecode::NameIdx> { - if Self::is_forbidden_arg_name(name) { - return Err(self.error(CodegenErrorType::SyntaxError(format!( - "cannot assign to {name}", - )))); - } + // Note: __debug__ checks are now handled in symboltable phase Ok(self._name_inner(name, |i| &mut i.metadata.varnames)) } fn _name_inner( @@ -939,7 +1600,7 @@ impl Compiler { let mut parent_idx = stack_size - 2; let mut parent = &self.code_stack[parent_idx]; - // If parent is TypeParams scope, look at grandparent + // If parent is ast::TypeParams scope, look at grandparent // Check if parent is a type params scope by name pattern if parent.metadata.name.starts_with("<generic parameters of ") { if stack_size == 2 { @@ -986,7 +1647,7 @@ impl Compiler { let parent_obj_name = &parent.metadata.name; // Determine if parent is a function-like scope - let is_function_parent = parent.flags.contains(bytecode::CodeFlags::IS_OPTIMIZED) + let is_function_parent = parent.flags.contains(bytecode::CodeFlags::OPTIMIZED) && !parent_obj_name.starts_with("<") // Not a special scope like <lambda>, <listcomp>, etc. && parent_obj_name != "<module>"; // Not the module scope @@ -1012,12 +1673,16 @@ impl Compiler { fn compile_program( &mut self, - body: &ModModule, + body: &ast::ModModule, symbol_table: SymbolTable, ) -> CompileResult<()> { let size_before = self.code_stack.len(); + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + let (doc, statements) = split_doc(&body.body, &self.opts); if let Some(value) = doc { self.emit_load_const(ConstantData::Str { @@ -1027,10 +1692,24 @@ impl Compiler { emit!(self, Instruction::StoreGlobal(doc)) } + // Handle annotations based on future_annotations flag if Self::find_ann(statements) { - emit!(self, Instruction::SetupAnnotation); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(statements)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } + // Compile all statements self.compile_statements(statements)?; assert_eq!(self.code_stack.len(), size_before); @@ -1042,18 +1721,36 @@ impl Compiler { fn compile_program_single( &mut self, - body: &[Stmt], + body: &[ast::Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { + self.interactive = true; + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + + // Handle annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotation); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(body)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } if let Some((last, body)) = body.split_last() { for statement in body { - if let Stmt::Expr(StmtExpr { value, .. }) = &statement { + if let ast::Stmt::Expr(ast::StmtExpr { value, .. }) = &statement { self.compile_expression(value)?; emit!( self, @@ -1062,15 +1759,15 @@ impl Compiler { } ); - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } else { self.compile_statement(statement)?; } } - if let Stmt::Expr(StmtExpr { value, .. }) = &last { + if let ast::Stmt::Expr(ast::StmtExpr { value, .. }) = &last { self.compile_expression(value)?; - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); emit!( self, Instruction::CallIntrinsic1 { @@ -1078,7 +1775,7 @@ impl Compiler { } ); - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } else { self.compile_statement(last)?; self.emit_load_const(ConstantData::None); @@ -1093,22 +1790,23 @@ impl Compiler { fn compile_block_expr( &mut self, - body: &[Stmt], + body: &[ast::Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); self.compile_statements(body)?; if let Some(last_statement) = body.last() { match last_statement { - Stmt::Expr(_) => { - self.current_block().instructions.pop(); // pop Instruction::Pop + ast::Stmt::Expr(_) => { + self.current_block().instructions.pop(); // pop Instruction::PopTop } - Stmt::FunctionDef(_) | Stmt::ClassDef(_) => { + ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => { let pop_instructions = self.current_block().instructions.pop(); let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.current_block().instructions.push(store_inst); } _ => self.emit_load_const(ConstantData::None), @@ -1122,16 +1820,18 @@ impl Compiler { // Compile statement in eval mode: fn compile_eval( &mut self, - expression: &ModExpression, + expression: &ast::ModExpression, symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + self.compile_expression(&expression.body)?; self.emit_return_value(); Ok(()) } - fn compile_statements(&mut self, statements: &[Stmt]) -> CompileResult<()> { + fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { for statement in statements { self.compile_statement(statement)? } @@ -1152,16 +1852,8 @@ impl Compiler { .code_stack .last() .and_then(|info| info.private.as_deref()); - symboltable::mangle_name(private, name) - } - - fn check_forbidden_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { - let msg = match usage { - NameUsage::Store if is_forbidden_name(name) => "cannot assign to", - NameUsage::Delete if is_forbidden_name(name) => "cannot delete", - _ => return Ok(()), - }; - Err(self.error(CodegenErrorType::SyntaxError(format!("{msg} {name}")))) + let mangled_names = self.current_symbol_table().mangled_names.as_ref(); + symboltable::maybe_mangle_name(private, mangled_names, name) } // = compiler_nameop @@ -1171,10 +1863,10 @@ impl Compiler { Global, Deref, Name, + DictOrGlobals, // PEP 649: can_see_class_scope } let name = self.mangle(name); - self.check_forbidden_name(&name, usage)?; // Special handling for __debug__ if NameUsage::Load == usage && name == "__debug__" { @@ -1187,16 +1879,18 @@ impl Compiler { // Determine the operation type based on symbol scope let is_function_like = self.ctx.in_func(); - // Look up the symbol, handling TypeParams scope specially - let (symbol_scope, _is_typeparams) = { + // Look up the symbol, handling ast::TypeParams and Annotation scopes specially + let (symbol_scope, can_see_class_scope) = { let current_table = self.current_symbol_table(); let is_typeparams = current_table.typ == CompilerScope::TypeParams; + let is_annotation = current_table.typ == CompilerScope::Annotation; + let can_see_class = current_table.can_see_class_scope; // First try to find in current table let symbol = current_table.lookup(name.as_ref()); - // If not found and we're in TypeParams scope, try parent scope - let symbol = if symbol.is_none() && is_typeparams { + // If not found and we're in ast::TypeParams or Annotation scope, try parent scope + let symbol = if symbol.is_none() && (is_typeparams || is_annotation) { self.symbol_table_stack .get(self.symbol_table_stack.len() - 2) // Try to get parent index .expect("Symbol has no parent! This is a compiler bug.") @@ -1205,14 +1899,48 @@ impl Compiler { symbol }; - (symbol.map(|s| s.scope), is_typeparams) + (symbol.map(|s| s.scope), can_see_class) + }; + + // Special handling for class scope implicit cell variables + // These are treated as Cell even if not explicitly marked in symbol table + // __class__ and __classdict__: only LOAD uses Cell (stores go to class namespace) + // __conditional_annotations__: both LOAD and STORE use Cell (it's a mutable set + // that the annotation scope accesses through the closure) + let symbol_scope = { + let current_table = self.current_symbol_table(); + if current_table.typ == CompilerScope::Class + && ((usage == NameUsage::Load + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__")) + || (name == "__conditional_annotations__" && usage == NameUsage::Store)) + { + Some(SymbolScope::Cell) + } else { + symbol_scope + } }; - let actual_scope = symbol_scope.ok_or_else(|| { - self.error(CodegenErrorType::SyntaxError(format!( - "The symbol '{name}' must be present in the symbol table" - ))) - })?; + // In annotation or type params scope, missing symbols are treated as global implicit + // This allows referencing global names like Union, Optional, etc. that are imported + // at module level but not explicitly bound in the function scope + let actual_scope = match symbol_scope { + Some(scope) => scope, + None => { + let current_table = self.current_symbol_table(); + if matches!( + current_table.typ, + CompilerScope::Annotation | CompilerScope::TypeParams + ) { + SymbolScope::GlobalImplicit + } else { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "the symbol '{name}' must be present in the symbol table" + )))); + } + } + }; // Determine operation type based on scope let op_type = match actual_scope { @@ -1226,7 +1954,11 @@ impl Compiler { } } SymbolScope::GlobalImplicit => { - if is_function_like { + // PEP 649: In annotation scope with class visibility, use DictOrGlobals + // to check classdict first before globals + if can_see_class_scope { + NameOp::DictOrGlobals + } else if is_function_like { NameOp::Global } else { NameOp::Name @@ -1247,9 +1979,15 @@ impl Compiler { let op = match usage { NameUsage::Load => { - // Special case for class scope + // ClassBlock (not inlined comp): LOAD_LOCALS first, then LOAD_FROM_DICT_OR_DEREF if self.ctx.in_class && !self.ctx.in_func() { - Instruction::LoadClassDeref + emit!(self, Instruction::LoadLocals); + Instruction::LoadFromDictOrDeref + // can_see_class_scope: LOAD_DEREF(__classdict__) first + } else if can_see_class_scope { + let classdict_idx = self.get_free_var_index("__classdict__")?; + self.emit_arg(classdict_idx, Instruction::LoadDeref); + Instruction::LoadFromDictOrDeref } else { Instruction::LoadDeref } @@ -1280,33 +2018,51 @@ impl Compiler { NameOp::Name => { let idx = self.get_global_name_index(&name); let op = match usage { - NameUsage::Load => Instruction::LoadNameAny, - NameUsage::Store => Instruction::StoreLocal, - NameUsage::Delete => Instruction::DeleteLocal, + NameUsage::Load => Instruction::LoadName, + NameUsage::Store => Instruction::StoreName, + NameUsage::Delete => Instruction::DeleteName, }; self.emit_arg(idx, op); } + NameOp::DictOrGlobals => { + // PEP 649: First check classdict (from __classdict__ freevar), then globals + let idx = self.get_global_name_index(&name); + match usage { + NameUsage::Load => { + // Load __classdict__ first (it's a free variable in annotation scope) + let classdict_idx = self.get_free_var_index("__classdict__")?; + self.emit_arg(classdict_idx, Instruction::LoadDeref); + self.emit_arg(idx, Instruction::LoadFromDictOrGlobals); + } + // Store/Delete in annotation scope should use Name ops + NameUsage::Store => { + self.emit_arg(idx, Instruction::StoreName); + } + NameUsage::Delete => { + self.emit_arg(idx, Instruction::DeleteName); + } + } + } } Ok(()) } - fn compile_statement(&mut self, statement: &Stmt) -> CompileResult<()> { - use ruff_python_ast::*; + fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> { trace!("Compiling {statement:?}"); self.set_source_range(statement.range()); match &statement { // we do this here because `from __future__` still executes that `from` statement at runtime, // we still need to compile the ImportFrom down below - Stmt::ImportFrom(StmtImportFrom { module, names, .. }) + ast::Stmt::ImportFrom(ast::StmtImportFrom { module, names, .. }) if module.as_ref().map(|id| id.as_str()) == Some("__future__") => { self.compile_future_features(names)? } // ignore module-level doc comments - Stmt::Expr(StmtExpr { value, .. }) - if matches!(&**value, Expr::StringLiteral(..)) + ast::Stmt::Expr(ast::StmtExpr { value, .. }) + if matches!(&**value, ast::Expr::StringLiteral(..)) && matches!(self.done_with_future_stmts, DoneWithFuture::No) => { self.done_with_future_stmts = DoneWithFuture::DoneWithDoc @@ -1316,7 +2072,7 @@ impl Compiler { } match &statement { - Stmt::Import(StmtImport { names, .. }) => { + ast::Stmt::Import(ast::StmtImport { names, .. }) => { // import a, b, c as d for name in names { let name = &name; @@ -1327,17 +2083,25 @@ impl Compiler { let idx = self.name(&name.name); emit!(self, Instruction::ImportName { idx }); if let Some(alias) = &name.asname { - for part in name.name.split('.').skip(1) { + let parts: Vec<&str> = name.name.split('.').skip(1).collect(); + for (i, part) in parts.iter().enumerate() { let idx = self.name(part); - emit!(self, Instruction::LoadAttr { idx }); + emit!(self, Instruction::ImportFrom { idx }); + if i < parts.len() - 1 { + emit!(self, Instruction::Swap { index: 2 }); + emit!(self, Instruction::PopTop); + } + } + self.store_name(alias.as_str())?; + if !parts.is_empty() { + emit!(self, Instruction::PopTop); } - self.store_name(alias.as_str())? } else { self.store_name(name.name.split('.').next().unwrap())? } } } - Stmt::ImportFrom(StmtImportFrom { + ast::Stmt::ImportFrom(ast::StmtImportFrom { level, module, names, @@ -1382,6 +2146,7 @@ impl Compiler { func: bytecode::IntrinsicFunction1::ImportStar } ); + emit!(self, Instruction::PopTop); } else { // from mod import a, b as c @@ -1400,24 +2165,33 @@ impl Compiler { } // Pop module from stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } } - Stmt::Expr(StmtExpr { value, .. }) => { + ast::Stmt::Expr(ast::StmtExpr { value, .. }) => { self.compile_expression(value)?; - // Pop result of stack, since we not use it: - emit!(self, Instruction::Pop); + if self.interactive && !self.ctx.in_func() && !self.ctx.in_class { + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + } + + emit!(self, Instruction::PopTop); } - Stmt::Global(_) | Stmt::Nonlocal(_) => { + ast::Stmt::Global(_) | ast::Stmt::Nonlocal(_) => { // Handled during symbol table construction. } - Stmt::If(StmtIf { + ast::Stmt::If(ast::StmtIf { test, body, elif_else_clauses, .. }) => { + self.enter_conditional_block(); match elif_else_clauses.as_slice() { // Only if [] => { @@ -1435,7 +2209,7 @@ impl Compiler { self.compile_statements(body)?; emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: after_block } ); @@ -1451,7 +2225,7 @@ impl Compiler { self.compile_statements(&clause.body)?; emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: after_block } ); @@ -1465,17 +2239,18 @@ impl Compiler { self.switch_to_block(after_block); } } + self.leave_conditional_block(); } - Stmt::While(StmtWhile { + ast::Stmt::While(ast::StmtWhile { test, body, orelse, .. }) => self.compile_while(test, body, orelse)?, - Stmt::With(StmtWith { + ast::Stmt::With(ast::StmtWith { items, body, is_async, .. }) => self.compile_with(items, body, *is_async)?, - Stmt::For(StmtFor { + ast::Stmt::For(ast::StmtFor { target, iter, body, @@ -1483,8 +2258,12 @@ impl Compiler { is_async, .. }) => self.compile_for(target, iter, body, orelse, *is_async)?, - Stmt::Match(StmtMatch { subject, cases, .. }) => self.compile_match(subject, cases)?, - Stmt::Raise(StmtRaise { exc, cause, .. }) => { + ast::Stmt::Match(ast::StmtMatch { subject, cases, .. }) => { + self.compile_match(subject, cases)? + } + ast::Stmt::Raise(ast::StmtRaise { + exc, cause, range, .. + }) => { let kind = match exc { Some(value) => { self.compile_expression(value)?; @@ -1496,11 +2275,16 @@ impl Compiler { None => bytecode::RaiseKind::Raise, } } - None => bytecode::RaiseKind::Reraise, + None => bytecode::RaiseKind::BareRaise, }; - emit!(self, Instruction::Raise { kind }); - } - Stmt::Try(StmtTry { + self.set_source_range(*range); + emit!(self, Instruction::RaiseVarargs { kind }); + // Start a new block so dead code after raise doesn't + // corrupt the except stack in label_exception_targets + let dead = self.new_block(); + self.switch_to_block(dead); + } + ast::Stmt::Try(ast::StmtTry { body, handlers, orelse, @@ -1508,13 +2292,15 @@ impl Compiler { is_star, .. }) => { + self.enter_conditional_block(); if *is_star { - self.compile_try_star_statement(body, handlers, orelse, finalbody)? + self.compile_try_star_except(body, handlers, orelse, finalbody)? } else { self.compile_try_statement(body, handlers, orelse, finalbody)? } + self.leave_conditional_block(); } - Stmt::FunctionDef(StmtFunctionDef { + ast::Stmt::FunctionDef(ast::StmtFunctionDef { name, parameters, body, @@ -1536,7 +2322,7 @@ impl Compiler { type_params.as_deref(), )? } - Stmt::ClassDef(StmtClassDef { + ast::Stmt::ClassDef(ast::StmtClassDef { name, body, decorator_list, @@ -1550,26 +2336,31 @@ impl Compiler { type_params.as_deref(), arguments.as_deref(), )?, - Stmt::Assert(StmtAssert { test, msg, .. }) => { + ast::Stmt::Assert(ast::StmtAssert { test, msg, .. }) => { // if some flag, ignore all assert statements! if self.opts.optimize == 0 { let after_block = self.new_block(); self.compile_jump_if(test, true, after_block)?; - let assertion_error = self.name("AssertionError"); - emit!(self, Instruction::LoadGlobal(assertion_error)); + emit!( + self, + Instruction::LoadCommonConstant { + idx: bytecode::CommonConstant::AssertionError + } + ); + emit!(self, Instruction::PushNull); match msg { Some(e) => { self.compile_expression(e)?; - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); + emit!(self, Instruction::Call { nargs: 1 }); } None => { - emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); + emit!(self, Instruction::Call { nargs: 0 }); } } emit!( self, - Instruction::Raise { + Instruction::RaiseVarargs { kind: bytecode::RaiseKind::Raise, } ); @@ -1577,75 +2368,32 @@ impl Compiler { self.switch_to_block(after_block); } } - Stmt::Break(_) => { - // Find the innermost loop in fblock stack - let found_loop = { - let code = self.current_code_info(); - let mut result = None; - for i in (0..code.fblock.len()).rev() { - match code.fblock[i].fb_type { - FBlockType::WhileLoop | FBlockType::ForLoop => { - result = Some(code.fblock[i].fb_exit); - break; - } - _ => continue, - } - } - result - }; - - match found_loop { - Some(exit_block) => { - emit!(self, Instruction::Break { target: exit_block }); - } - None => { - return Err( - self.error_ranged(CodegenErrorType::InvalidBreak, statement.range()) - ); - } - } + ast::Stmt::Break(_) => { + // Unwind fblock stack until we find a loop, emitting cleanup for each fblock + self.compile_break_continue(statement.range(), true)?; + let dead = self.new_block(); + self.switch_to_block(dead); } - Stmt::Continue(_) => { - // Find the innermost loop in fblock stack - let found_loop = { - let code = self.current_code_info(); - let mut result = None; - for i in (0..code.fblock.len()).rev() { - match code.fblock[i].fb_type { - FBlockType::WhileLoop | FBlockType::ForLoop => { - result = Some(code.fblock[i].fb_block); - break; - } - _ => continue, - } - } - result - }; - - match found_loop { - Some(loop_block) => { - emit!(self, Instruction::Continue { target: loop_block }); - } - None => { - return Err( - self.error_ranged(CodegenErrorType::InvalidContinue, statement.range()) - ); - } - } + ast::Stmt::Continue(_) => { + // Unwind fblock stack until we find a loop, emitting cleanup for each fblock + self.compile_break_continue(statement.range(), false)?; + let dead = self.new_block(); + self.switch_to_block(dead); } - Stmt::Return(StmtReturn { value, .. }) => { + ast::Stmt::Return(ast::StmtReturn { value, .. }) => { if !self.ctx.in_func() { return Err( self.error_ranged(CodegenErrorType::InvalidReturn, statement.range()) ); } + match value { Some(v) => { if self.ctx.func == FunctionContext::AsyncFunction && self .current_code_info() .flags - .contains(bytecode::CodeFlags::IS_GENERATOR) + .contains(bytecode::CodeFlags::GENERATOR) { return Err(self.error_ranged( CodegenErrorType::AsyncReturnValue, @@ -1653,41 +2401,48 @@ impl Compiler { )); } self.compile_expression(v)?; + // Unwind fblock stack with preserve_tos=true (preserve return value) + self.unwind_fblock_stack(true, false)?; self.emit_return_value(); } None => { + // Unwind fblock stack with preserve_tos=false (no value to preserve) + self.unwind_fblock_stack(false, false)?; self.emit_return_const(ConstantData::None); } } + let dead = self.new_block(); + self.switch_to_block(dead); } - Stmt::Assign(StmtAssign { targets, value, .. }) => { + ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) => { self.compile_expression(value)?; for (i, target) in targets.iter().enumerate() { if i + 1 != targets.len() { - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); } self.compile_store(target)?; } } - Stmt::AugAssign(StmtAugAssign { + ast::Stmt::AugAssign(ast::StmtAugAssign { target, op, value, .. }) => self.compile_augassign(target, op, value)?, - Stmt::AnnAssign(StmtAnnAssign { + ast::Stmt::AnnAssign(ast::StmtAnnAssign { target, annotation, value, + simple, .. - }) => self.compile_annotated_assign(target, annotation, value.as_deref())?, - Stmt::Delete(StmtDelete { targets, .. }) => { + }) => self.compile_annotated_assign(target, annotation, value.as_deref(), *simple)?, + ast::Stmt::Delete(ast::StmtDelete { targets, .. }) => { for target in targets { self.compile_delete(target)?; } } - Stmt::Pass(_) => { + ast::Stmt::Pass(_) => { // No need to emit any code here :) } - Stmt::TypeAlias(StmtTypeAlias { + ast::Stmt::TypeAlias(ast::StmtTypeAlias { name, type_params, value, @@ -1710,27 +2465,95 @@ impl Compiler { }); if let Some(type_params) = type_params { - // For TypeAlias, we need to use push_symbol_table to properly handle the TypeAlias scope - self.push_symbol_table(); + // Outer scope for TypeParams + self.push_symbol_table()?; + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get().to_u32(); + let scope_name = format!("<generic parameters of {name_string}>"); + self.enter_scope(&scope_name, CompilerScope::TypeParams, key, lineno)?; + + // TypeParams scope is function-like + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; - // Compile type params and push to stack + // Compile type params inside the scope self.compile_type_params(type_params)?; - // Stack now has [name, type_params_tuple] - - // Compile value expression (can now see T1, T2) + // Stack: [type_params_tuple] + + // Inner closure for lazy value evaluation + self.push_symbol_table()?; + let inner_key = self.symbol_table_stack.len() - 1; + self.enter_scope("TypeAlias", CompilerScope::TypeParams, inner_key, lineno)?; + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + self.emit_format_validation()?; self.compile_expression(value)?; - // Stack: [name, type_params_tuple, value] - - // Pop the TypeAlias scope - self.pop_symbol_table(); + emit!(self, Instruction::ReturnValue); + let value_code = self.exit_scope(); + self.make_closure(value_code, bytecode::MakeFunctionFlags::empty())?; + // Stack: [type_params_tuple, value_closure] + + // Swap so unpack_sequence reverse gives correct order + emit!(self, Instruction::Swap { index: 2_u32 }); + // Stack: [value_closure, type_params_tuple] + + // Build tuple and return from TypeParams scope + emit!(self, Instruction::BuildTuple { size: 2 }); + emit!(self, Instruction::ReturnValue); + + let code = self.exit_scope(); + self.ctx = prev_ctx; + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { nargs: 0 }); + + // Unpack: (value_closure, type_params_tuple) + // UnpackSequence reverses → stack: [name, type_params_tuple, value_closure] + emit!(self, Instruction::UnpackSequence { size: 2 }); } else { // Push None for type_params self.emit_load_const(ConstantData::None); // Stack: [name, None] - // Compile value expression + // Create a closure for lazy evaluation of the value + self.push_symbol_table()?; + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get().to_u32(); + self.enter_scope("TypeAlias", CompilerScope::TypeParams, key, lineno)?; + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + self.emit_format_validation()?; + + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + self.compile_expression(value)?; - // Stack: [name, None, value] + emit!(self, Instruction::ReturnValue); + + let code = self.exit_scope(); + self.ctx = prev_ctx; + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; + // Stack: [name, None, closure] } // Build tuple of 3 elements and call intrinsic @@ -1743,32 +2566,33 @@ impl Compiler { ); self.store_name(&name_string)?; } - Stmt::IpyEscapeCommand(_) => todo!(), + ast::Stmt::IpyEscapeCommand(_) => todo!(), } Ok(()) } - fn compile_delete(&mut self, expression: &Expr) -> CompileResult<()> { - use ruff_python_ast::*; + fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> { match &expression { - Expr::Name(ExprName { id, .. }) => self.compile_name(id.as_str(), NameUsage::Delete)?, - Expr::Attribute(ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr.as_str(), NameUsage::Delete)?; + ast::Expr::Name(ast::ExprName { id, .. }) => { + self.compile_name(id.as_str(), NameUsage::Delete)? + } + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { self.compile_expression(value)?; let idx = self.name(attr.as_str()); emit!(self, Instruction::DeleteAttr { idx }); } - Expr::Subscript(ExprSubscript { + ast::Expr::Subscript(ast::ExprSubscript { value, slice, ctx, .. }) => { self.compile_subscript(value, slice, *ctx)?; } - Expr::Tuple(ExprTuple { elts, .. }) | Expr::List(ExprList { elts, .. }) => { + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) + | ast::Expr::List(ast::ExprList { elts, .. }) => { for element in elts { self.compile_delete(element)?; } } - Expr::BinOp(_) | Expr::UnaryOp(_) => { + ast::Expr::BinOp(_) | ast::Expr::UnaryOp(_) => { return Err(self.error(CodegenErrorType::Delete("expression"))); } _ => return Err(self.error(CodegenErrorType::Delete(expression.python_name()))), @@ -1776,7 +2600,7 @@ impl Compiler { Ok(()) } - fn enter_function(&mut self, name: &str, parameters: &Parameters) -> CompileResult<()> { + fn enter_function(&mut self, name: &str, parameters: &ast::Parameters) -> CompileResult<()> { // TODO: partition_in_place let mut kw_without_defaults = vec![]; let mut kw_with_defaults = vec![]; @@ -1789,14 +2613,14 @@ impl Compiler { } self.push_output( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, parameters.posonlyargs.len().to_u32(), (parameters.posonlyargs.len() + parameters.args.len()).to_u32(), parameters.kwonlyargs.len().to_u32(), name.to_owned(), - ); + )?; - let args_iter = std::iter::empty() + let args_iter = core::iter::empty() .chain(&parameters.posonlyargs) .chain(&parameters.args) .map(|arg| &arg.parameter) @@ -1807,51 +2631,76 @@ impl Compiler { } if let Some(name) = parameters.vararg.as_deref() { - self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS; + self.current_code_info().flags |= bytecode::CodeFlags::VARARGS; self.varname(name.name.as_str())?; } if let Some(name) = parameters.kwarg.as_deref() { - self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS; + self.current_code_info().flags |= bytecode::CodeFlags::VARKEYWORDS; self.varname(name.name.as_str())?; } Ok(()) } - fn prepare_decorators(&mut self, decorator_list: &[Decorator]) -> CompileResult<()> { + /// Push decorators onto the stack in source order. + /// For @dec1 @dec2 def foo(): stack becomes [dec1, NULL, dec2, NULL] + fn prepare_decorators(&mut self, decorator_list: &[ast::Decorator]) -> CompileResult<()> { for decorator in decorator_list { self.compile_expression(&decorator.expression)?; + emit!(self, Instruction::PushNull); } Ok(()) } - fn apply_decorators(&mut self, decorator_list: &[Decorator]) { - // Apply decorators: + /// Apply decorators in reverse order (LIFO from stack). + /// Stack [dec1, NULL, dec2, NULL, func] -> dec2(func) -> dec1(dec2(func)) + /// The forward loop works because each Call pops from TOS, naturally + /// applying decorators bottom-up (innermost first). + fn apply_decorators(&mut self, decorator_list: &[ast::Decorator]) { for _ in decorator_list { - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); + emit!(self, Instruction::Call { nargs: 1 }); } } /// Compile type parameter bound or default in a separate scope and return closure fn compile_type_param_bound_or_default( &mut self, - expr: &Expr, + expr: &ast::Expr, name: &str, allow_starred: bool, ) -> CompileResult<()> { // Push the next symbol table onto the stack - self.push_symbol_table(); + self.push_symbol_table()?; // Get the current symbol table let key = self.symbol_table_stack.len() - 1; - let lineno = expr.range().start().to_u32(); + let lineno = self.get_source_line_number().get().to_u32(); // Enter scope with the type parameter name self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?; + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + self.emit_format_validation()?; + + // TypeParams scope is function-like + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + // Compile the expression - if allow_starred && matches!(expr, Expr::Starred(_)) { - if let Expr::Starred(starred) = expr { + if allow_starred && matches!(expr, ast::Expr::Starred(_)) { + if let ast::Expr::Starred(starred) = expr { self.compile_expression(&starred.value)?; emit!(self, Instruction::UnpackSequence { size: 1 }); } @@ -1864,24 +2713,21 @@ impl Compiler { // Exit scope and create closure let code = self.exit_scope(); - // Note: exit_scope already calls pop_symbol_table, so we don't need to call it again + self.ctx = prev_ctx; - // Create type params function with closure + // Create closure for lazy evaluation self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; - // Call the function immediately - emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); - Ok(()) } /// Store each type parameter so it is accessible to the current scope, and leave a tuple of - /// all the type parameters on the stack. - fn compile_type_params(&mut self, type_params: &TypeParams) -> CompileResult<()> { + /// all the type parameters on the stack. Handles default values per PEP 695. + fn compile_type_params(&mut self, type_params: &ast::TypeParams) -> CompileResult<()> { // First, compile each type parameter and store it for type_param in &type_params.type_params { match type_param { - TypeParam::TypeVar(TypeParamTypeVar { + ast::TypeParam::TypeVar(ast::TypeParamTypeVar { name, bound, default, @@ -1914,7 +2760,6 @@ impl Compiler { ); } - // Handle default value if present (PEP 695) if let Some(default_expr) = default { let scope_name = format!("<TypeVar default of {name}>"); self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; @@ -1926,10 +2771,10 @@ impl Compiler { ); } - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.store_name(name.as_ref())?; } - TypeParam::ParamSpec(TypeParamParamSpec { name, default, .. }) => { + ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { name, default, .. }) => { self.emit_load_const(ConstantData::Str { value: name.as_str().into(), }); @@ -1940,7 +2785,6 @@ impl Compiler { } ); - // Handle default value if present (PEP 695) if let Some(default_expr) = default { let scope_name = format!("<ParamSpec default of {name}>"); self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; @@ -1952,10 +2796,12 @@ impl Compiler { ); } - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.store_name(name.as_ref())?; } - TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, default, .. }) => { + ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { + name, default, .. + }) => { self.emit_load_const(ConstantData::Str { value: name.as_str().into(), }); @@ -1966,7 +2812,6 @@ impl Compiler { } ); - // Handle default value if present (PEP 695) if let Some(default_expr) = default { // TypeVarTuple allows starred expressions let scope_name = format!("<TypeVarTuple default of {name}>"); @@ -1979,7 +2824,7 @@ impl Compiler { ); } - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.store_name(name.as_ref())?; } }; @@ -1995,104 +2840,351 @@ impl Compiler { fn compile_try_statement( &mut self, - body: &[Stmt], - handlers: &[ExceptHandler], - orelse: &[Stmt], - finalbody: &[Stmt], + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + finalbody: &[ast::Stmt], ) -> CompileResult<()> { let handler_block = self.new_block(); let finally_block = self.new_block(); + // finally needs TWO blocks: + // - finally_block: normal path (no exception active) + // - finally_except_block: exception path (PUSH_EXC_INFO -> body -> RERAISE) + let finally_except_block = if !finalbody.is_empty() { + Some(self.new_block()) + } else { + None + }; + let finally_cleanup_block = if finally_except_block.is_some() { + Some(self.new_block()) + } else { + None + }; + // End block - continuation point after try-finally + // Normal path jumps here to skip exception path blocks + let end_block = self.new_block(); + // Setup a finally block if we have a finally statement. + // Push fblock with handler info for exception table generation + // IMPORTANT: handler goes to finally_except_block (exception path), not finally_block if !finalbody.is_empty() { + // SETUP_FINALLY doesn't push lasti for try body handler + // Exception table: L1 to L2 -> L4 [1] (no lasti) + let setup_target = finally_except_block.unwrap_or(finally_block); emit!( self, - Instruction::SetupFinally { - handler: finally_block, + PseudoInstruction::SetupFinally { + target: setup_target } ); + // Store finally body in fb_datum for unwind_fblock to compile inline + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), // Clone finally body for unwind + )?; } let else_block = self.new_block(); + // if handlers is empty, compile body directly + // without wrapping in TryExcept (only FinallyTry is needed) + if handlers.is_empty() { + // Just compile body with FinallyTry fblock active (if finalbody exists) + self.compile_statements(body)?; + + // Pop FinallyTry fblock BEFORE compiling orelse/finally (normal path) + // This prevents exception table from covering the normal path + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + // Compile orelse (usually empty for try-finally without except) + self.compile_statements(orelse)?; + + // Snapshot sub_tables before first finally compilation + // This allows us to restore them for the second compilation (exception path) + let sub_table_cursor = if !finalbody.is_empty() && finally_except_block.is_some() { + self.symbol_table_stack.last().map(|t| t.next_sub_table) + } else { + None + }; + + // Compile finally body inline for normal path + if !finalbody.is_empty() { + self.compile_statements(finalbody)?; + } + + // Jump to end (skip exception path blocks) + emit!(self, PseudoInstruction::Jump { target: end_block }); + + if let Some(finally_except) = finally_except_block { + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + self.switch_to_block(finally_except); + // SETUP_CLEANUP before PUSH_EXC_INFO + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { target: cleanup }); + } + emit!(self, Instruction::PushExcInfo); + if let Some(cleanup) = finally_cleanup_block { + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + self.compile_statements(finalbody)?; + + // Pop FinallyEnd fblock BEFORE emitting RERAISE + // This ensures RERAISE routes to outer exception handler, not cleanup block + // Cleanup block is only for new exceptions raised during finally body execution + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + // Restore prev_exc as current exception before RERAISE + // Stack: [prev_exc, exc] -> COPY 2 -> [prev_exc, exc, prev_exc] + // POP_EXCEPT pops prev_exc and sets exc_info->exc_value = prev_exc + // Stack after POP_EXCEPT: [prev_exc, exc] + emit!(self, Instruction::Copy { index: 2_u32 }); + emit!(self, Instruction::PopExcept); + + // RERAISE 0: re-raise the original exception to outer handler + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack + } + ); + } + + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + emit!(self, Instruction::Copy { index: 3_u32 }); + emit!(self, Instruction::PopExcept); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack + } + ); + } + + self.switch_to_block(end_block); + return Ok(()); + } + // try: emit!( self, - Instruction::SetupExcept { - handler: handler_block, + PseudoInstruction::SetupFinally { + target: handler_block } ); + self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; self.compile_statements(body)?; - emit!(self, Instruction::PopBlock); - emit!(self, Instruction::Jump { target: else_block }); + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + emit!(self, PseudoInstruction::Jump { target: else_block }); // except handlers: self.switch_to_block(handler_block); - // Exception is on top of stack now + + // SETUP_CLEANUP(cleanup) for except block + // This handles exceptions during exception matching + // Exception table: L2 to L3 -> L5 [1] lasti + // After PUSH_EXC_INFO, stack is [prev_exc, exc] + // depth=1 means keep prev_exc on stack when routing to cleanup + let cleanup_block = self.new_block(); + emit!( + self, + PseudoInstruction::SetupCleanup { + target: cleanup_block + } + ); + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + + // Exception is on top of stack now, pushed by unwind_blocks + // PUSH_EXC_INFO transforms [exc] -> [prev_exc, exc] for PopExcept + emit!(self, Instruction::PushExcInfo); for handler in handlers { - let ExceptHandler::ExceptHandler(ExceptHandlerExceptHandler { - type_, name, body, .. + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + .. }) = &handler; let next_handler = self.new_block(); // If we gave a typ, // check if this handler can handle the exception: if let Some(exc_type) = type_ { - // Duplicate exception for test: - emit!(self, Instruction::CopyItem { index: 1_u32 }); - // Check exception type: + // Stack: [prev_exc, exc] self.compile_expression(exc_type)?; - emit!(self, Instruction::JumpIfNotExcMatch(next_handler)); + // Stack: [prev_exc, exc, type] + emit!(self, Instruction::CheckExcMatch); + // Stack: [prev_exc, exc, bool] + emit!( + self, + Instruction::PopJumpIfFalse { + target: next_handler + } + ); + // Stack: [prev_exc, exc] // We have a match, store in name (except x as y) if let Some(alias) = name { self.store_name(alias.as_str())? } else { // Drop exception from top of stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } } else { // Catch all! // Drop exception from top of stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } + // If name is bound, we need a cleanup handler for RERAISE + let handler_cleanup_block = if name.is_some() { + // SETUP_CLEANUP(cleanup_end) for named handler + let cleanup_end = self.new_block(); + emit!( + self, + PseudoInstruction::SetupCleanup { + target: cleanup_end + } + ); + self.push_fblock_full( + FBlockType::HandlerCleanup, + cleanup_end, + cleanup_end, + FBlockDatum::ExceptionName(name.as_ref().unwrap().as_str().to_owned()), + )?; + Some(cleanup_end) + } else { + // no SETUP_CLEANUP for unnamed handler + self.push_fblock(FBlockType::HandlerCleanup, finally_block, finally_block)?; + None + }; + // Handler code: self.compile_statements(body)?; - emit!(self, Instruction::PopException); - // Delete the exception variable if it was bound - if let Some(alias) = name { - // Set the variable to None before deleting - self.emit_load_const(ConstantData::None); - self.store_name(alias.as_str())?; - self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.pop_fblock(FBlockType::HandlerCleanup); + // PopBlock for inner SETUP_CLEANUP (named handler only) + if handler_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); } - if !finalbody.is_empty() { - emit!(self, Instruction::PopBlock); // pop excepthandler block - // We enter the finally block, without exception. - emit!(self, Instruction::EnterFinally); + // Create a block for normal path continuation (after handler body succeeds) + let handler_normal_exit = self.new_block(); + emit!( + self, + PseudoInstruction::Jump { + target: handler_normal_exit, + } + ); + + // cleanup_end block for named handler + // IMPORTANT: In CPython, cleanup_end is within outer SETUP_CLEANUP scope. + // so when RERAISE is executed, it goes to the cleanup block which does POP_EXCEPT. + // We MUST compile cleanup_end BEFORE popping ExceptionHandler so RERAISE routes to cleanup_block. + if let Some(cleanup_end) = handler_cleanup_block { + self.switch_to_block(cleanup_end); + if let Some(alias) = name { + // name = None; del name; before RERAISE + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + // RERAISE 1 (with lasti) - exception is on stack from exception table routing + // Stack at entry: [prev_exc (at handler_depth), lasti, exc] + // This RERAISE is within ExceptionHandler scope, so it routes to cleanup_block + // which does COPY 3; POP_EXCEPT; RERAISE + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack, + } + ); + } + + // Switch to normal exit block - this is where handler body success continues + self.switch_to_block(handler_normal_exit); + + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + // Now pop ExceptionHandler - the normal path continues from here + self.pop_fblock(FBlockType::ExceptionHandler); + emit!(self, Instruction::PopExcept); + + // Delete the exception variable if it was bound (normal path) + if let Some(alias) = name { + // Set the variable to None before deleting + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Pop FinallyTry block before jumping to finally body. + // The else_block path also pops this; both paths must agree + // on the except stack when entering finally_block. + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); } + // Jump to finally block emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: finally_block, } ); + // Re-push ExceptionHandler for next handler in the loop + // This will be popped at the end of handlers loop or when matched + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + // Emit a new label for the next handler self.switch_to_block(next_handler); } // If code flows here, we have an unhandled exception, // raise the exception again! + // RERAISE 0 + // Stack: [prev_exc, exc] - exception is on stack from PUSH_EXC_INFO + // NOTE: We emit RERAISE 0 BEFORE popping fblock so it is within cleanup handler scope + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack, + } + ); + + // Pop EXCEPTION_HANDLER fblock + // Pop after RERAISE so the instruction has the correct exception handler + self.pop_fblock(FBlockType::ExceptionHandler); + + // cleanup block (POP_EXCEPT_AND_RERAISE) + // Stack at entry: [prev_exc, lasti, exc] (depth=1 + lasti + exc pushed) + // COPY 3: copy prev_exc to top -> [prev_exc, lasti, exc, prev_exc] + // POP_EXCEPT: pop prev_exc from stack and restore -> [prev_exc, lasti, exc] + // RERAISE 1: reraise with lasti + self.switch_to_block(cleanup_block); + emit!(self, Instruction::Copy { index: 3_u32 }); + emit!(self, Instruction::PopExcept); emit!( self, - Instruction::Raise { - kind: bytecode::RaiseKind::Reraise, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack, } ); @@ -2101,47 +3193,499 @@ impl Compiler { self.switch_to_block(else_block); self.compile_statements(orelse)?; + // Pop the FinallyTry fblock before jumping to finally if !finalbody.is_empty() { - emit!(self, Instruction::PopBlock); // pop finally block - - // We enter the finallyhandler block, without return / exception. - emit!(self, Instruction::EnterFinally); + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); } - // finally: + // Snapshot sub_tables before first finally compilation (for double compilation issue) + let sub_table_cursor = if !finalbody.is_empty() && finally_except_block.is_some() { + self.symbol_table_stack.last().map(|t| t.next_sub_table) + } else { + None + }; + + // finally (normal path): self.switch_to_block(finally_block); if !finalbody.is_empty() { self.compile_statements(finalbody)?; - emit!(self, Instruction::EndFinally); + // Jump to end_block to skip exception path blocks + // This prevents fall-through to finally_except_block + emit!(self, PseudoInstruction::Jump { target: end_block }); + } + + // finally (exception path) + // This is where exceptions go to run finally before reraise + // Stack at entry: [lasti, exc] (from exception table with preserve_lasti=true) + if let Some(finally_except) = finally_except_block { + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + self.switch_to_block(finally_except); + + // SETUP_CLEANUP for finally body + // Exceptions during finally body need to go to cleanup block + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { target: cleanup }); + } + emit!(self, Instruction::PushExcInfo); + if let Some(cleanup) = finally_cleanup_block { + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + + // Run finally body + self.compile_statements(finalbody)?; + + // Pop FinallyEnd fblock BEFORE emitting RERAISE + // This ensures RERAISE routes to outer exception handler, not cleanup block + // Cleanup block is only for new exceptions raised during finally body execution + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + // Restore prev_exc as current exception before RERAISE + // Stack: [lasti, prev_exc, exc] -> COPY 2 -> [lasti, prev_exc, exc, prev_exc] + // POP_EXCEPT pops prev_exc and sets exc_info->exc_value = prev_exc + // Stack after POP_EXCEPT: [lasti, prev_exc, exc] + emit!(self, Instruction::Copy { index: 2_u32 }); + emit!(self, Instruction::PopExcept); + + // RERAISE 0: re-raise the original exception to outer handler + // Stack: [lasti, prev_exc, exc] - exception is on top + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack, + } + ); + } + + // finally cleanup block + // This handles exceptions that occur during the finally body itself + // Stack at entry: [lasti, prev_exc, lasti2, exc2] after exception table routing + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + // COPY 3: copy the exception from position 3 + emit!(self, Instruction::Copy { index: 3_u32 }); + // POP_EXCEPT: restore prev_exc as current exception + emit!(self, Instruction::PopExcept); + // RERAISE 1: reraise with lasti from stack + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack, + } + ); } + // End block - continuation point after try-finally + // Normal execution continues here after the finally block + self.switch_to_block(end_block); + Ok(()) } - fn compile_try_star_statement( + fn compile_try_star_except( &mut self, - _body: &[Stmt], - _handlers: &[ExceptHandler], - _orelse: &[Stmt], - _finalbody: &[Stmt], + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + finalbody: &[ast::Stmt], ) -> CompileResult<()> { - Err(self.error(CodegenErrorType::NotImplementedYet)) - } + // compiler_try_star_except + // Stack layout during handler processing: [prev_exc, orig, list, rest] + let handler_block = self.new_block(); + let finally_block = self.new_block(); + let else_block = self.new_block(); + let end_block = self.new_block(); + let reraise_star_block = self.new_block(); + let reraise_block = self.new_block(); + let finally_cleanup_block = if !finalbody.is_empty() { + Some(self.new_block()) + } else { + None + }; + let exit_block = self.new_block(); + + // Push fblock with handler info for exception table generation + if !finalbody.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + target: finally_block + } + ); + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), + )?; + } + + // SETUP_FINALLY for try body + emit!( + self, + PseudoInstruction::SetupFinally { + target: handler_block + } + ); + self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; + self.compile_statements(body)?; + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + emit!(self, PseudoInstruction::Jump { target: else_block }); + + // Exception handler entry + self.switch_to_block(handler_block); + // Stack: [exc] (from exception table) + + // PUSH_EXC_INFO + emit!(self, Instruction::PushExcInfo); + // Stack: [prev_exc, exc] + + // Push EXCEPTION_GROUP_HANDLER fblock + let eg_dummy1 = self.new_block(); + let eg_dummy2 = self.new_block(); + self.push_fblock(FBlockType::ExceptionGroupHandler, eg_dummy1, eg_dummy2)?; + + // Initialize handler stack before the loop + // BUILD_LIST 0 + COPY 2 to set up [prev_exc, orig, list, rest] + emit!(self, Instruction::BuildList { size: 0 }); + // Stack: [prev_exc, exc, []] + emit!(self, Instruction::Copy { index: 2 }); + // Stack: [prev_exc, orig, list, rest] + + let n = handlers.len(); + if n == 0 { + // Empty handlers (invalid AST) - append rest to list and proceed + // Stack: [prev_exc, orig, list, rest] + emit!(self, Instruction::ListAppend { i: 0 }); + // Stack: [prev_exc, orig, list] + emit!( + self, + PseudoInstruction::Jump { + target: reraise_star_block + } + ); + } + for (i, handler) in handlers.iter().enumerate() { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + .. + }) = handler; + + let no_match_block = self.new_block(); + let next_block = self.new_block(); + + // Compile exception type + if let Some(exc_type) = type_ { + // Check for unparenthesized tuple + if let ast::Expr::Tuple(ast::ExprTuple { elts, range, .. }) = exc_type.as_ref() + && let Some(first) = elts.first() + && range.start().to_u32() == first.range().start().to_u32() + { + return Err(self.error(CodegenErrorType::SyntaxError( + "multiple exception types must be parenthesized".to_owned(), + ))); + } + self.compile_expression(exc_type)?; + } else { + return Err(self.error(CodegenErrorType::SyntaxError( + "except* must specify an exception type".to_owned(), + ))); + } + // Stack: [prev_exc, orig, list, rest, type] + + // ADDOP(c, loc, CHECK_EG_MATCH); + emit!(self, Instruction::CheckEgMatch); + // Stack: [prev_exc, orig, list, new_rest, match] + + // ADDOP_I(c, loc, COPY, 1); + // ADDOP_JUMP(c, loc, POP_JUMP_IF_NONE, no_match); + emit!(self, Instruction::Copy { index: 1 }); + emit!( + self, + Instruction::PopJumpIfNone { + target: no_match_block + } + ); + + // Handler matched + // Stack: [prev_exc, orig, list, new_rest, match] + // Note: CheckEgMatch already sets the matched exception as current exception + let handler_except_block = self.new_block(); + + // Store match to name or pop + if let Some(alias) = name { + self.store_name(alias.as_str())?; + } else { + emit!(self, Instruction::PopTop); // pop match + } + // Stack: [prev_exc, orig, list, new_rest] + + // HANDLER_CLEANUP fblock for handler body + emit!( + self, + PseudoInstruction::SetupCleanup { + target: handler_except_block + } + ); + self.push_fblock_full( + FBlockType::HandlerCleanup, + next_block, + end_block, + if let Some(alias) = name { + FBlockDatum::ExceptionName(alias.as_str().to_owned()) + } else { + FBlockDatum::None + }, + )?; + + // Execute handler body + self.compile_statements(body)?; + + // Handler body completed normally + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::HandlerCleanup); + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Jump to next handler + emit!(self, PseudoInstruction::Jump { target: next_block }); + + // Handler raised an exception (cleanup_end label) + self.switch_to_block(handler_except_block); + // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] + // (lasti is pushed because push_lasti=true in HANDLER_CLEANUP fblock) + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // LIST_APPEND(3) - append raised_exc to list + // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] + // After pop: [prev_exc, orig, list, new_rest, lasti] (len=5) + // nth_value(i) = stack[len - i - 1], we need stack[2] = list + // stack[5 - i - 1] = 2 -> i = 2 + emit!(self, Instruction::ListAppend { i: 2 }); + // Stack: [prev_exc, orig, list, new_rest, lasti] + + // POP_TOP - pop lasti + emit!(self, Instruction::PopTop); + // Stack: [prev_exc, orig, list, new_rest] + + // JUMP except_with_error + // We directly JUMP to next_block since no_match_block falls through to it + emit!(self, PseudoInstruction::Jump { target: next_block }); + + // No match - pop match (None) + self.switch_to_block(no_match_block); + emit!(self, Instruction::PopTop); // pop match (None) + // Stack: [prev_exc, orig, list, new_rest] + // Falls through to next_block + + // except_with_error label + // All paths merge here at next_block + self.switch_to_block(next_block); + // Stack: [prev_exc, orig, list, rest] + + // After last handler, append rest to list + if i == n - 1 { + // Stack: [prev_exc, orig, list, rest] + // ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); + // PEEK(1) = stack[len-1] after pop + // RustPython nth_value(i) = stack[len-i-1] after pop + // For LIST_APPEND 1: stack[len-1] = stack[len-i-1] -> i = 0 + emit!(self, Instruction::ListAppend { i: 0 }); + // Stack: [prev_exc, orig, list] + emit!( + self, + PseudoInstruction::Jump { + target: reraise_star_block + } + ); + } + } + + // Pop EXCEPTION_GROUP_HANDLER fblock + self.pop_fblock(FBlockType::ExceptionGroupHandler); + + // Reraise star block + self.switch_to_block(reraise_star_block); + // Stack: [prev_exc, orig, list] + + // CALL_INTRINSIC_2 PREP_RERAISE_STAR + // Takes 2 args (orig, list) and produces result + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::PrepReraiseStar + } + ); + // Stack: [prev_exc, result] + + // COPY 1 + emit!(self, Instruction::Copy { index: 1 }); + // Stack: [prev_exc, result, result] + + // POP_JUMP_IF_NOT_NONE reraise + emit!( + self, + Instruction::PopJumpIfNotNone { + target: reraise_block + } + ); + // Stack: [prev_exc, result] + + // Nothing to reraise + // POP_TOP - pop result (None) + emit!(self, Instruction::PopTop); + // Stack: [prev_exc] + + // POP_BLOCK - no-op for us with exception tables (fblocks handle this) + // POP_EXCEPT - restore previous exception context + emit!(self, Instruction::PopExcept); + // Stack: [] + + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + emit!(self, PseudoInstruction::Jump { target: end_block }); + + // Reraise the result + self.switch_to_block(reraise_block); + // Stack: [prev_exc, result] + + // POP_BLOCK - no-op for us + // SWAP 2 + emit!(self, Instruction::Swap { index: 2 }); + // Stack: [result, prev_exc] + + // POP_EXCEPT + emit!(self, Instruction::PopExcept); + // Stack: [result] + + // RERAISE 0 + emit!(self, Instruction::Reraise { depth: 0 }); + + // try-else path + // NOTE: When we reach here in compilation, the nothing-to-reraise path above + // has already popped FinallyTry. But else_block is a different execution path + // that branches from try body success (where FinallyTry is still active). + // We need to re-push FinallyTry to reflect the correct fblock state for else path. + if !finalbody.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + target: finally_block + } + ); + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), + )?; + } + self.switch_to_block(else_block); + self.compile_statements(orelse)?; + + if !finalbody.is_empty() { + // Pop the FinallyTry fblock we just pushed for the else path + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + emit!(self, PseudoInstruction::Jump { target: end_block }); + + self.switch_to_block(end_block); + if !finalbody.is_empty() { + // Snapshot sub_tables before first finally compilation + let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); + + // Compile finally body inline for normal path + self.compile_statements(finalbody)?; + emit!(self, PseudoInstruction::Jump { target: exit_block }); + + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + // Exception handler path + self.switch_to_block(finally_block); + emit!(self, Instruction::PushExcInfo); - fn is_forbidden_arg_name(name: &str) -> bool { - is_forbidden_name(name) + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { target: cleanup }); + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + + self.compile_statements(finalbody)?; + + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + emit!(self, Instruction::Copy { index: 2_u32 }); + emit!(self, Instruction::PopExcept); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack + } + ); + + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + emit!(self, Instruction::Copy { index: 3_u32 }); + emit!(self, Instruction::PopExcept); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack + } + ); + } + } + + self.switch_to_block(exit_block); + + Ok(()) } /// Compile default arguments // = compiler_default_arguments fn compile_default_arguments( &mut self, - parameters: &Parameters, + parameters: &ast::Parameters, ) -> CompileResult<bytecode::MakeFunctionFlags> { let mut funcflags = bytecode::MakeFunctionFlags::empty(); // Handle positional defaults - let defaults: Vec<_> = std::iter::empty() + let defaults: Vec<_> = core::iter::empty() .chain(&parameters.posonlyargs) .chain(&parameters.args) .filter_map(|x| x.default.as_deref()) @@ -2173,7 +3717,7 @@ impl Compiler { // Compile kwdefaults and build dict for (arg, default) in &kw_with_defaults { self.emit_load_const(ConstantData::Str { - value: arg.name.as_str().into(), + value: self.mangle(arg.name.as_str()).into_owned().into(), }); self.compile_expression(default)?; } @@ -2194,8 +3738,8 @@ impl Compiler { fn compile_function_body( &mut self, name: &str, - parameters: &Parameters, - body: &[Stmt], + parameters: &ast::Parameters, + body: &[ast::Stmt], is_async: bool, funcflags: bytecode::MakeFunctionFlags, ) -> CompileResult<()> { @@ -2203,7 +3747,7 @@ impl Compiler { self.enter_function(name, parameters)?; self.current_code_info() .flags - .set(bytecode::CodeFlags::IS_COROUTINE, is_async); + .set(bytecode::CodeFlags::COROUTINE, is_async); // Set up context let prev_ctx = self.ctx; @@ -2215,24 +3759,34 @@ impl Compiler { } else { FunctionContext::Function }, + // A function starts a new async scope only if it's async + in_async_scope: is_async, }; // Set qualname self.set_qualname(); - // Handle docstring + // Handle docstring - store in co_consts[0] if present let (doc_str, body) = split_doc(body, &self.opts); - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + if let Some(doc) = &doc_str { + // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag + self.current_code_info() + .metadata + .consts + .insert_full(ConstantData::Str { + value: doc.to_string().into(), + }); + self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; + } + // If no docstring, don't add None to co_consts + // Note: RETURN_GENERATOR + POP_TOP for async functions is emitted in enter_scope() // Compile body statements self.compile_statements(body)?; // Emit None at end if needed match body.last() { - Some(Stmt::Return(_)) => {} + Some(ast::Stmt::Return(_)) => {} _ => { self.emit_return_const(ConstantData::None); } @@ -2245,31 +3799,42 @@ impl Compiler { // Create function object with closure self.make_closure(code, funcflags)?; - // Handle docstring if present - if let Some(doc) = doc_str { - emit!(self, Instruction::CopyItem { index: 1_u32 }); - self.emit_load_const(ConstantData::Str { - value: doc.to_string().into(), - }); - emit!(self, Instruction::Swap { index: 2 }); - let doc_attr = self.name("__doc__"); - emit!(self, Instruction::StoreAttr { idx: doc_attr }); - } + // Note: docstring is now retrieved from co_consts[0] by the VM + // when HAS_DOCSTRING flag is set, so no runtime __doc__ assignment needed Ok(()) } - /// Compile function annotations - // = compiler_visit_annotations - fn visit_annotations( + /// Compile function annotations as a closure (PEP 649) + /// Returns true if an __annotate__ closure was created + /// Uses symbol table's annotation_block for proper scoping. + fn compile_annotations_closure( &mut self, - parameters: &Parameters, - returns: Option<&Expr>, - ) -> CompileResult<u32> { - let mut num_annotations = 0; + func_name: &str, + parameters: &ast::Parameters, + returns: Option<&ast::Expr>, + ) -> CompileResult<bool> { + // Try to enter annotation scope - returns None if no annotation_block exists + let Some(saved_ctx) = self.enter_annotation_scope(func_name)? else { + return Ok(false); + }; - // Handle parameter annotations - let parameters_iter = std::iter::empty() + // Count annotations + let parameters_iter = core::iter::empty() + .chain(&parameters.posonlyargs) + .chain(&parameters.args) + .chain(&parameters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + + let num_annotations: u32 = + u32::try_from(parameters_iter.filter(|p| p.annotation.is_some()).count()) + .expect("too many annotations") + + if returns.is_some() { 1 } else { 0 }; + + // Compile annotations inside the annotation scope + let parameters_iter = core::iter::empty() .chain(&parameters.posonlyargs) .chain(&parameters.args) .chain(&parameters.kwonlyargs) @@ -2283,20 +3848,246 @@ impl Compiler { value: self.mangle(param.name.as_str()).into_owned().into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } } - // Handle return annotation last + // Handle return annotation if let Some(annotation) = returns { self.emit_load_const(ConstantData::Str { value: "return".into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } - Ok(num_annotations) + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + + // Exit the annotation scope and get the code object + let annotate_code = self.exit_annotation_scope(saved_ctx); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + Ok(true) + } + + /// Collect simple annotations from module body in AST order (including nested blocks) + /// Returns list of (name, annotation_expr) pairs + /// This must match the order that annotations are compiled to ensure + /// conditional_annotation_index stays in sync with __annotate__ enumeration. + fn collect_simple_annotations(body: &[ast::Stmt]) -> Vec<(&str, &ast::Expr)> { + fn walk<'a>(stmts: &'a [ast::Stmt], out: &mut Vec<(&'a str, &'a ast::Expr)>) { + for stmt in stmts { + match stmt { + ast::Stmt::AnnAssign(ast::StmtAnnAssign { + target, + annotation, + simple, + .. + }) if *simple && matches!(target.as_ref(), ast::Expr::Name(_)) => { + if let ast::Expr::Name(ast::ExprName { id, .. }) = target.as_ref() { + out.push((id.as_str(), annotation.as_ref())); + } + } + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + walk(body, out); + for clause in elif_else_clauses { + walk(&clause.body, out); + } + } + ast::Stmt::For(ast::StmtFor { body, orelse, .. }) + | ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { + walk(body, out); + walk(orelse, out); + } + ast::Stmt::With(ast::StmtWith { body, .. }) => walk(body, out), + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) => { + walk(body, out); + for handler in handlers { + let ast::ExceptHandler::ExceptHandler( + ast::ExceptHandlerExceptHandler { body, .. }, + ) = handler; + walk(body, out); + } + walk(orelse, out); + walk(finalbody, out); + } + ast::Stmt::Match(ast::StmtMatch { cases, .. }) => { + for case in cases { + walk(&case.body, out); + } + } + _ => {} + } + } + } + let mut annotations = Vec::new(); + walk(body, &mut annotations); + annotations + } + + /// Compile module-level __annotate__ function (PEP 649) + /// Returns true if __annotate__ was created and stored + fn compile_module_annotate(&mut self, body: &[ast::Stmt]) -> CompileResult<bool> { + // Collect simple annotations from module body first + let annotations = Self::collect_simple_annotations(body); + + if annotations.is_empty() { + return Ok(false); + } + + // Check if we have conditional annotations + let has_conditional = self.current_symbol_table().has_conditional_annotations; + + // Get parent scope type BEFORE pushing annotation symbol table + let parent_scope_type = self.current_symbol_table().typ; + // Try to push annotation symbol table from current scope + if !self.push_current_annotation_symbol_table() { + return Ok(false); + } + + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Enter annotation scope for code generation + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope( + "__annotate__", + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Add 'format' parameter to varnames + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + self.emit_format_validation()?; + + if has_conditional { + // PEP 649: Build dict incrementally, checking conditional annotations + // Start with empty dict + emit!(self, Instruction::BuildMap { size: 0 }); + + // Process each annotation + for (idx, (name, annotation)) in annotations.iter().enumerate() { + // Check if index is in __conditional_annotations__ + let not_set_block = self.new_block(); + + // LOAD_CONST index + self.emit_load_const(ConstantData::Integer { value: idx.into() }); + // Load __conditional_annotations__ from appropriate scope + // Class scope: LoadDeref (freevars), Module scope: LoadGlobal + if parent_scope_type == CompilerScope::Class { + let idx = self.get_free_var_index("__conditional_annotations__")?; + emit!(self, Instruction::LoadDeref(idx)); + } else { + let cond_annotations_name = self.name("__conditional_annotations__"); + emit!(self, Instruction::LoadGlobal(cond_annotations_name)); + } + // CONTAINS_OP (in) + emit!(self, Instruction::ContainsOp(bytecode::Invert::No)); + // POP_JUMP_IF_FALSE not_set + emit!( + self, + Instruction::PopJumpIfFalse { + target: not_set_block + } + ); + + // Annotation value + self.compile_annotation(annotation)?; + // COPY dict to TOS + emit!(self, Instruction::Copy { index: 2 }); + // LOAD_CONST name + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + // STORE_SUBSCR - dict[name] = value + emit!(self, Instruction::StoreSubscr); + + // not_set label + self.switch_to_block(not_set_block); + } + + // Return the dict + emit!(self, Instruction::ReturnValue); + } else { + // No conditional annotations - use simple BuildMap + let num_annotations = u32::try_from(annotations.len()).expect("too many annotations"); + + // Compile annotations inside the annotation scope + for (name, annotation) in annotations { + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + self.compile_annotation(annotation)?; + } + + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + } + + // Exit annotation scope - pop symbol table, restore to parent's annotation_block, and get code + let annotation_table = self.pop_symbol_table(); + // Restore annotation_block to module's symbol table + self.symbol_table_stack + .last_mut() + .expect("no module symbol table") + .annotation_block = Some(Box::new(annotation_table)); + // Restore context + self.ctx = saved_ctx; + // Exit code scope + let pop = self.code_stack.pop(); + let annotate_code = unwrap_internal( + self, + compiler_unwrap_option(self, pop).finalize_code(&self.opts), + ); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + // Store as __annotate_func__ for classes, __annotate__ for modules + let name = if parent_scope_type == CompilerScope::Class { + "__annotate_func__" + } else { + "__annotate__" + }; + self.store_name(name)?; + + Ok(true) } // = compiler_function @@ -2304,12 +4095,12 @@ impl Compiler { fn compile_function_def( &mut self, name: &str, - parameters: &Parameters, - body: &[Stmt], - decorator_list: &[Decorator], - returns: Option<&Expr>, // TODO: use type hint somehow.. + parameters: &ast::Parameters, + body: &[ast::Stmt], + decorator_list: &[ast::Decorator], + returns: Option<&ast::Expr>, // TODO: use type hint somehow.. is_async: bool, - type_params: Option<&TypeParams>, + type_params: Option<&ast::TypeParams>, ) -> CompileResult<()> { self.prepare_decorators(decorator_list)?; @@ -2319,6 +4110,9 @@ impl Compiler { let is_generic = type_params.is_some(); let mut num_typeparam_args = 0; + // Save context before entering TypeParams scope + let saved_ctx = self.ctx; + if is_generic { // Count args to pass to type params scope if funcflags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { @@ -2328,20 +4122,23 @@ impl Compiler { num_typeparam_args += 1; } - // SWAP if we have both - if num_typeparam_args == 2 { - emit!(self, Instruction::Swap { index: 2 }); - } - // Enter type params scope let type_params_name = format!("<generic parameters of {name}>"); self.push_output( - bytecode::CodeFlags::IS_OPTIMIZED | bytecode::CodeFlags::NEW_LOCALS, + bytecode::CodeFlags::OPTIMIZED | bytecode::CodeFlags::NEWLOCALS, 0, num_typeparam_args as u32, 0, type_params_name, - ); + )?; + + // TypeParams scope is function-like + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; // Add parameter names to varnames for the type params scope // These will be passed as arguments when the closure is called @@ -2368,18 +4165,12 @@ impl Compiler { } } - // Compile annotations - let mut annotations_flag = bytecode::MakeFunctionFlags::empty(); - let num_annotations = self.visit_annotations(parameters, returns)?; - if num_annotations > 0 { - annotations_flag = bytecode::MakeFunctionFlags::ANNOTATIONS; - emit!( - self, - Instruction::BuildMap { - size: num_annotations, - } - ); - } + // Compile annotations as closure (PEP 649) + let annotations_flag = if self.compile_annotations_closure(name, parameters, returns)? { + bytecode::MakeFunctionFlags::ANNOTATE + } else { + bytecode::MakeFunctionFlags::empty() + }; // Compile function body let final_funcflags = funcflags | annotations_flag; @@ -2407,27 +4198,45 @@ impl Compiler { // Exit type params scope and create closure let type_params_code = self.exit_scope(); + self.ctx = saved_ctx; // Make closure for type params code self.make_closure(type_params_code, bytecode::MakeFunctionFlags::empty())?; - // Call the closure + // Call the type params closure with defaults/kwdefaults as arguments. + // Call protocol: [callable, self_or_null, arg1, ..., argN] + // We need to reorder: [args..., closure] -> [closure, NULL, args...] + // Using Swap operations to move closure down and insert NULL. + // Note: num_typeparam_args is at most 2 (defaults tuple, kwdefaults dict). if num_typeparam_args > 0 { - emit!( - self, - Instruction::Swap { - index: (num_typeparam_args + 1) as u32 + match num_typeparam_args { + 1 => { + // Stack: [arg1, closure] + emit!(self, Instruction::Swap { index: 2 }); // [closure, arg1] + emit!(self, Instruction::PushNull); // [closure, arg1, NULL] + emit!(self, Instruction::Swap { index: 2 }); // [closure, NULL, arg1] } - ); + 2 => { + // Stack: [arg1, arg2, closure] + emit!(self, Instruction::Swap { index: 3 }); // [closure, arg2, arg1] + emit!(self, Instruction::Swap { index: 2 }); // [closure, arg1, arg2] + emit!(self, Instruction::PushNull); // [closure, arg1, arg2, NULL] + emit!(self, Instruction::Swap { index: 3 }); // [closure, NULL, arg2, arg1] + emit!(self, Instruction::Swap { index: 2 }); // [closure, NULL, arg1, arg2] + } + _ => unreachable!("only defaults and kwdefaults are supported"), + } emit!( self, - Instruction::CallFunctionPositional { + Instruction::Call { nargs: num_typeparam_args as u32 } ); } else { - // No arguments, just call the closure - emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); + // Stack: [closure] + emit!(self, Instruction::PushNull); + // Stack: [closure, NULL] + emit!(self, Instruction::Call { nargs: 0 }); } } @@ -2443,12 +4252,18 @@ impl Compiler { /// Determines if a variable should be CELL or FREE type // = get_ref_type fn get_ref_type(&self, name: &str) -> Result<SymbolScope, CodegenErrorType> { - // Special handling for __class__ and __classdict__ in class scope - if self.ctx.in_class && (name == "__class__" || name == "__classdict__") { + let table = self.symbol_table_stack.last().unwrap(); + + // Special handling for __class__, __classdict__, and __conditional_annotations__ in class scope + // This should only apply when we're actually IN a class body, + // not when we're in a method nested inside a class. + if table.typ == CompilerScope::Class + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { return Ok(SymbolScope::Cell); } - - let table = self.symbol_table_stack.last().unwrap(); match table.lookup(name) { Some(symbol) => match symbol.scope { SymbolScope::Cell => Ok(SymbolScope::Cell), @@ -2526,7 +4341,7 @@ impl Compiler { } }; - emit!(self, Instruction::LoadClosure(idx.to_u32())); + emit!(self, PseudoInstruction::LoadClosure(idx.to_u32())); } // Build tuple of closure variables @@ -2551,7 +4366,6 @@ impl Compiler { // Set closure if needed if has_freevars { - // Closure tuple is already on stack emit!( self, Instruction::SetFunctionAttribute { @@ -2562,7 +4376,6 @@ impl Compiler { // Set annotations if present if flags.contains(bytecode::MakeFunctionFlags::ANNOTATIONS) { - // Annotations dict is already on stack emit!( self, Instruction::SetFunctionAttribute { @@ -2571,9 +4384,18 @@ impl Compiler { ); } + // Set __annotate__ closure if present (PEP 649) + if flags.contains(bytecode::MakeFunctionFlags::ANNOTATE) { + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::ANNOTATE + } + ); + } + // Set kwdefaults if present if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { - // kwdefaults dict is already on stack emit!( self, Instruction::SetFunctionAttribute { @@ -2584,7 +4406,6 @@ impl Compiler { // Set defaults if present if flags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { - // defaults tuple is already on stack emit!( self, Instruction::SetFunctionAttribute { @@ -2595,7 +4416,6 @@ impl Compiler { // Set type_params if present if flags.contains(bytecode::MakeFunctionFlags::TYPE_PARAMS) { - // type_params tuple is already on stack emit!( self, Instruction::SetFunctionAttribute { @@ -2608,15 +4428,14 @@ impl Compiler { } // Python/compile.c find_ann - fn find_ann(body: &[Stmt]) -> bool { - use ruff_python_ast::*; + fn find_ann(body: &[ast::Stmt]) -> bool { for statement in body { let res = match &statement { - Stmt::AnnAssign(_) => true, - Stmt::For(StmtFor { body, orelse, .. }) => { + ast::Stmt::AnnAssign(_) => true, + ast::Stmt::For(ast::StmtFor { body, orelse, .. }) => { Self::find_ann(body) || Self::find_ann(orelse) } - Stmt::If(StmtIf { + ast::Stmt::If(ast::StmtIf { body, elif_else_clauses, .. @@ -2624,16 +4443,30 @@ impl Compiler { Self::find_ann(body) || elif_else_clauses.iter().any(|x| Self::find_ann(&x.body)) } - Stmt::While(StmtWhile { body, orelse, .. }) => { + ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { Self::find_ann(body) || Self::find_ann(orelse) } - Stmt::With(StmtWith { body, .. }) => Self::find_ann(body), - Stmt::Try(StmtTry { + ast::Stmt::With(ast::StmtWith { body, .. }) => Self::find_ann(body), + ast::Stmt::Match(ast::StmtMatch { cases, .. }) => { + cases.iter().any(|case| Self::find_ann(&case.body)) + } + ast::Stmt::Try(ast::StmtTry { body, + handlers, orelse, finalbody, .. - }) => Self::find_ann(body) || Self::find_ann(orelse) || Self::find_ann(finalbody), + }) => { + Self::find_ann(body) + || handlers.iter().any(|h| { + let ast::ExceptHandler::ExceptHandler( + ast::ExceptHandlerExceptHandler { body, .. }, + ) = h; + Self::find_ann(body) + }) + || Self::find_ann(orelse) + || Self::find_ann(finalbody) + } _ => false, }; if res { @@ -2648,13 +4481,13 @@ impl Compiler { fn compile_class_body( &mut self, name: &str, - body: &[Stmt], - type_params: Option<&TypeParams>, + body: &[ast::Stmt], + type_params: Option<&ast::TypeParams>, firstlineno: u32, ) -> CompileResult<CodeObject> { // 1. Enter class scope let key = self.symbol_table_stack.len(); - self.push_symbol_table(); + self.push_symbol_table()?; self.enter_scope(name, CompilerScope::Class, key, firstlineno)?; // Set qualname using the new method @@ -2670,20 +4503,20 @@ impl Compiler { let dunder_name = self.name("__name__"); emit!(self, Instruction::LoadGlobal(dunder_name)); let dunder_module = self.name("__module__"); - emit!(self, Instruction::StoreLocal(dunder_module)); + emit!(self, Instruction::StoreName(dunder_module)); // Store __qualname__ self.emit_load_const(ConstantData::Str { value: qualname.into(), }); let qualname_name = self.name("__qualname__"); - emit!(self, Instruction::StoreLocal(qualname_name)); + emit!(self, Instruction::StoreName(qualname_name)); // Store __doc__ only if there's an explicit docstring if let Some(doc) = doc_str { self.emit_load_const(ConstantData::Str { value: doc.into() }); let doc_name = self.name("__doc__"); - emit!(self, Instruction::StoreLocal(doc_name)); + emit!(self, Instruction::StoreName(doc_name)); } // Store __firstlineno__ (new in Python 3.12+) @@ -2691,22 +4524,41 @@ impl Compiler { value: BigInt::from(firstlineno), }); let firstlineno_name = self.name("__firstlineno__"); - emit!(self, Instruction::StoreLocal(firstlineno_name)); + emit!(self, Instruction::StoreName(firstlineno_name)); // Set __type_params__ if we have type parameters if type_params.is_some() { // Load .type_params from enclosing scope let dot_type_params = self.name(".type_params"); - emit!(self, Instruction::LoadNameAny(dot_type_params)); + emit!(self, Instruction::LoadName(dot_type_params)); // Store as __type_params__ let dunder_type_params = self.name("__type_params__"); - emit!(self, Instruction::StoreLocal(dunder_type_params)); + emit!(self, Instruction::StoreName(dunder_type_params)); } - // Setup annotations if needed + // PEP 649: Initialize __classdict__ cell for class annotation scope + if self.current_symbol_table().needs_classdict { + emit!(self, Instruction::LoadLocals); + let classdict_idx = self.get_cell_var_index("__classdict__")?; + emit!(self, Instruction::StoreDeref(classdict_idx)); + } + + // Handle class annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotation); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict for class + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Initialize __conditional_annotations__ set if needed for class + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + + // PEP 649: Generate __annotate__ function for class annotations + self.compile_module_annotate(body)?; + } } // 3. Compile the class body @@ -2723,10 +4575,10 @@ impl Compiler { .position(|var| *var == "__class__"); if let Some(classcell_idx) = classcell_idx { - emit!(self, Instruction::LoadClosure(classcell_idx.to_u32())); - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, PseudoInstruction::LoadClosure(classcell_idx.to_u32())); + emit!(self, Instruction::Copy { index: 1_u32 }); let classcell = self.name("__classcell__"); - emit!(self, Instruction::StoreLocal(classcell)); + emit!(self, Instruction::StoreName(classcell)); } else { self.emit_load_const(ConstantData::None); } @@ -2741,34 +4593,45 @@ impl Compiler { fn compile_class_def( &mut self, name: &str, - body: &[Stmt], - decorator_list: &[Decorator], - type_params: Option<&TypeParams>, - arguments: Option<&Arguments>, + body: &[ast::Stmt], + decorator_list: &[ast::Decorator], + type_params: Option<&ast::TypeParams>, + arguments: Option<&ast::Arguments>, ) -> CompileResult<()> { self.prepare_decorators(decorator_list)?; let is_generic = type_params.is_some(); let firstlineno = self.get_source_line_number().get().to_u32(); + // Save context before entering any scopes + let saved_ctx = self.ctx; + // Step 1: If generic, enter type params scope and compile type params if is_generic { let type_params_name = format!("<generic parameters of {name}>"); self.push_output( - bytecode::CodeFlags::IS_OPTIMIZED | bytecode::CodeFlags::NEW_LOCALS, + bytecode::CodeFlags::OPTIMIZED | bytecode::CodeFlags::NEWLOCALS, 0, 0, 0, type_params_name, - ); + )?; // Set private name for name mangling self.code_stack.last_mut().unwrap().private = Some(name.to_owned()); + // TypeParams scope is function-like + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + // Compile type parameters and store as .type_params self.compile_type_params(type_params.unwrap())?; let dot_type_params = self.name(".type_params"); - emit!(self, Instruction::StoreLocal(dot_type_params)); + emit!(self, Instruction::StoreName(dot_type_params)); } // Step 2: Compile class body (always done, whether generic or not) @@ -2777,6 +4640,7 @@ impl Compiler { func: FunctionContext::NoFunction, in_class: true, loop_data: None, + in_async_scope: false, }; let class_code = self.compile_class_body(name, body, type_params, firstlineno)?; self.ctx = prev_ctx; @@ -2788,64 +4652,125 @@ impl Compiler { let dot_generic_base = self.name(".generic_base"); // Create .generic_base - emit!(self, Instruction::LoadNameAny(dot_type_params)); + emit!(self, Instruction::LoadName(dot_type_params)); emit!( self, Instruction::CallIntrinsic1 { func: bytecode::IntrinsicFunction1::SubscriptGeneric } ); - emit!(self, Instruction::StoreLocal(dot_generic_base)); + emit!(self, Instruction::StoreName(dot_generic_base)); // Generate class creation code emit!(self, Instruction::LoadBuildClass); + emit!(self, Instruction::PushNull); // Set up the class function with type params let mut func_flags = bytecode::MakeFunctionFlags::empty(); - emit!(self, Instruction::LoadNameAny(dot_type_params)); + emit!(self, Instruction::LoadName(dot_type_params)); func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; // Create class function with closure self.make_closure(class_code, func_flags)?; self.emit_load_const(ConstantData::Str { value: name.into() }); - // Compile original bases - let base_count = if let Some(arguments) = arguments { - for arg in &arguments.args { - self.compile_expression(arg)?; + // Compile bases and call __build_class__ + // Check for starred bases or **kwargs + let has_starred = arguments.is_some_and(|args| { + args.args + .iter() + .any(|arg| matches!(arg, ast::Expr::Starred(_))) + }); + let has_double_star = + arguments.is_some_and(|args| args.keywords.iter().any(|kw| kw.arg.is_none())); + + if has_starred || has_double_star { + // Use CallFunctionEx for *bases or **kwargs + // Stack has: [__build_class__, NULL, class_func, name] + // Need to build: args tuple = (class_func, name, *bases, .generic_base) + + // Build a list starting with class_func and name (2 elements already on stack) + emit!(self, Instruction::BuildList { size: 2 }); + + // Add bases to the list + if let Some(arguments) = arguments { + for arg in &arguments.args { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = arg { + // Starred: compile and extend + self.compile_expression(value)?; + emit!(self, Instruction::ListExtend { i: 0 }); + } else { + // Non-starred: compile and append + self.compile_expression(arg)?; + emit!(self, Instruction::ListAppend { i: 0 }); + } + } } - arguments.args.len() + + // Add .generic_base as final element + emit!(self, Instruction::LoadName(dot_generic_base)); + emit!(self, Instruction::ListAppend { i: 0 }); + + // Convert list to tuple + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + + // Build kwargs if needed + if arguments.is_some_and(|args| !args.keywords.is_empty()) { + self.compile_keywords(&arguments.unwrap().keywords)?; + } else { + emit!(self, Instruction::PushNull); + } + emit!(self, Instruction::CallFunctionEx); } else { - 0 - }; + // Simple case: no starred bases, no **kwargs + // Compile bases normally + let base_count = if let Some(arguments) = arguments { + for arg in &arguments.args { + self.compile_expression(arg)?; + } + arguments.args.len() + } else { + 0 + }; - // Load .generic_base as the last base - emit!(self, Instruction::LoadNameAny(dot_generic_base)); + // Load .generic_base as the last base + emit!(self, Instruction::LoadName(dot_generic_base)); - let nargs = 2 + u32::try_from(base_count).expect("too many base classes") + 1; // function, name, bases..., generic_base + let nargs = 2 + u32::try_from(base_count).expect("too many base classes") + 1; - // Handle keyword arguments - if let Some(arguments) = arguments - && !arguments.keywords.is_empty() - { - for keyword in &arguments.keywords { - if let Some(name) = &keyword.arg { - self.emit_load_const(ConstantData::Str { + // Handle keyword arguments (no **kwargs here) + if let Some(arguments) = arguments + && !arguments.keywords.is_empty() + { + let mut kwarg_names = vec![]; + for keyword in &arguments.keywords { + let name = keyword.arg.as_ref().expect( + "keyword argument name must be set (no **kwargs in this branch)", + ); + kwarg_names.push(ConstantData::Str { value: name.as_str().into(), }); + self.compile_expression(&keyword.value)?; } - self.compile_expression(&keyword.value)?; + self.emit_load_const(ConstantData::Tuple { + elements: kwarg_names, + }); + emit!( + self, + Instruction::CallKw { + nargs: nargs + + u32::try_from(arguments.keywords.len()) + .expect("too many keyword arguments") + } + ); + } else { + emit!(self, Instruction::Call { nargs }); } - emit!( - self, - Instruction::CallFunctionKeyword { - nargs: nargs - + u32::try_from(arguments.keywords.len()) - .expect("too many keyword arguments") - } - ); - } else { - emit!(self, Instruction::CallFunctionPositional { nargs }); } // Return the created class @@ -2853,24 +4778,26 @@ impl Compiler { // Exit type params scope and wrap in function let type_params_code = self.exit_scope(); + self.ctx = saved_ctx; // Execute the type params function self.make_closure(type_params_code, bytecode::MakeFunctionFlags::empty())?; - emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { nargs: 0 }); } else { // Non-generic class: standard path emit!(self, Instruction::LoadBuildClass); + emit!(self, Instruction::PushNull); // Create class function with closure self.make_closure(class_code, bytecode::MakeFunctionFlags::empty())?; self.emit_load_const(ConstantData::Str { value: name.into() }); - let call = if let Some(arguments) = arguments { - self.compile_call_inner(2, arguments)? + if let Some(arguments) = arguments { + self.codegen_call_helper(2, arguments, self.current_source_range)?; } else { - CallType::Positional { nargs: 2 } - }; - self.compile_normal_call(call); + emit!(self, Instruction::Call { nargs: 2 }); + } } // Step 4: Apply decorators and store (common to both paths) @@ -2878,12 +4805,19 @@ impl Compiler { self.store_name(name) } - fn compile_while(&mut self, test: &Expr, body: &[Stmt], orelse: &[Stmt]) -> CompileResult<()> { + fn compile_while( + &mut self, + test: &ast::Expr, + body: &[ast::Stmt], + orelse: &[ast::Stmt], + ) -> CompileResult<()> { + self.enter_conditional_block(); + let while_block = self.new_block(); let else_block = self.new_block(); let after_block = self.new_block(); - emit!(self, Instruction::SetupLoop); + // Note: SetupLoop is no longer emitted (break/continue use direct jumps) self.switch_to_block(while_block); // Push fblock for while loop @@ -2896,7 +4830,7 @@ impl Compiler { self.ctx.loop_data = was_in_loop; emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: while_block, } ); @@ -2904,57 +4838,141 @@ impl Compiler { // Pop fblock self.pop_fblock(FBlockType::WhileLoop); - emit!(self, Instruction::PopBlock); + // Note: PopBlock is no longer emitted for loops self.compile_statements(orelse)?; self.switch_to_block(after_block); + + self.leave_conditional_block(); Ok(()) } fn compile_with( &mut self, - items: &[WithItem], - body: &[Stmt], + items: &[ast::WithItem], + body: &[ast::Stmt], is_async: bool, ) -> CompileResult<()> { + self.enter_conditional_block(); + + // Python 3.12+ style with statement: + // + // BEFORE_WITH # TOS: ctx_mgr -> [__exit__, __enter__ result] + // L1: STORE_NAME f # exception table: L1 to L2 -> L3 [1] lasti + // L2: ... body ... + // LOAD_CONST None # normal exit + // LOAD_CONST None + // LOAD_CONST None + // CALL 2 # __exit__(None, None, None) + // POP_TOP + // JUMP after + // L3: PUSH_EXC_INFO # exception handler + // WITH_EXCEPT_START # call __exit__(type, value, tb), push result + // TO_BOOL + // POP_JUMP_IF_TRUE suppress + // RERAISE 2 + // suppress: + // POP_TOP # pop exit result + // L5: POP_EXCEPT + // POP_TOP # pop __exit__ + // POP_TOP # pop prev_exc (or lasti depending on layout) + // JUMP after + // L6: COPY 3 # cleanup handler for reraise + // POP_EXCEPT + // RERAISE 1 + // after: ... + let with_range = self.current_source_range; let Some((item, items)) = items.split_first() else { return Err(self.error(CodegenErrorType::EmptyWithItems)); }; - let final_block = { - let final_block = self.new_block(); - self.compile_expression(&item.context_expr)?; + let exc_handler_block = self.new_block(); + let after_block = self.new_block(); - self.set_source_range(with_range); - if is_async { - emit!(self, Instruction::BeforeAsyncWith); - emit!(self, Instruction::GetAwaitable); - self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 - } - ); - emit!(self, Instruction::SetupAsyncWith { end: final_block }); - } else { - emit!(self, Instruction::SetupWith { end: final_block }); - } + // Compile context expression and load __enter__/__exit__ methods + self.compile_expression(&item.context_expr)?; + self.set_source_range(with_range); + + // Stack: [cm] + emit!(self, Instruction::Copy { index: 1_u32 }); // [cm, cm] - match &item.optional_vars { - Some(var) => { - self.set_source_range(var.range()); - self.compile_store(var)?; + if is_async { + if self.ctx.func != FunctionContext::AsyncFunction { + return Err(self.error(CodegenErrorType::InvalidAsyncWith)); + } + // Load __aexit__ and __aenter__, then call __aenter__ + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::AExit + } + ); // [cm, bound_aexit] + emit!(self, Instruction::Swap { index: 2_u32 }); // [bound_aexit, cm] + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::AEnter + } + ); // [bound_aexit, bound_aenter] + // bound_aenter is already bound, call with NULL self_or_null + emit!(self, Instruction::PushNull); // [bound_aexit, bound_aenter, NULL] + emit!(self, Instruction::Call { nargs: 0 }); // [bound_aexit, awaitable] + emit!(self, Instruction::GetAwaitable { arg: 1 }); + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + } else { + // Load __exit__ and __enter__, then call __enter__ + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::Exit } - None => { - emit!(self, Instruction::Pop); + ); // [cm, bound_exit] + emit!(self, Instruction::Swap { index: 2_u32 }); // [bound_exit, cm] + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::Enter } + ); // [bound_exit, bound_enter] + // bound_enter is already bound, call with NULL self_or_null + emit!(self, Instruction::PushNull); // [bound_exit, bound_enter, NULL] + emit!(self, Instruction::Call { nargs: 0 }); // [bound_exit, enter_result] + } + + // Stack: [..., __exit__, enter_result] + // Push fblock for exception table - handler goes to exc_handler_block + // preserve_lasti=true for with statements + emit!( + self, + PseudoInstruction::SetupWith { + target: exc_handler_block } - final_block - }; + ); + self.push_fblock( + if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }, + exc_handler_block, // block start (will become exit target after store) + after_block, + )?; + + // Store or pop the enter result + match &item.optional_vars { + Some(var) => { + self.set_source_range(var.range()); + self.compile_store(var)?; + } + None => { + emit!(self, Instruction::PopTop); + } + } + // Stack: [..., __exit__] + // Compile body or nested with if items.is_empty() { if body.is_empty() { return Err(self.error(CodegenErrorType::EmptyWithBody)); @@ -2965,76 +4983,163 @@ impl Compiler { self.compile_with(items, body, is_async)?; } - // sort of "stack up" the layers of with blocks: - // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a) + // Pop fblock before normal exit + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }); + + // ===== Normal exit path ===== + // Stack: [..., __exit__] + // Call __exit__(None, None, None) self.set_source_range(with_range); - emit!(self, Instruction::PopBlock); + emit!(self, Instruction::PushNull); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::Call { nargs: 3 }); + if is_async { + emit!(self, Instruction::GetAwaitable { arg: 2 }); + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + } + emit!(self, Instruction::PopTop); // Pop __exit__ result + emit!( + self, + PseudoInstruction::Jump { + target: after_block + } + ); + + // ===== Exception handler path ===== + // Stack at entry (after unwind): [..., __exit__, lasti, exc] + // PUSH_EXC_INFO -> [..., __exit__, lasti, prev_exc, exc] + self.switch_to_block(exc_handler_block); + + // Create blocks for exception handling + let cleanup_block = self.new_block(); + let suppress_block = self.new_block(); + + emit!( + self, + PseudoInstruction::SetupCleanup { + target: cleanup_block + } + ); + self.push_fblock(FBlockType::ExceptionHandler, exc_handler_block, after_block)?; - emit!(self, Instruction::EnterFinally); + // PUSH_EXC_INFO: [exc] -> [prev_exc, exc] + emit!(self, Instruction::PushExcInfo); - self.switch_to_block(final_block); - emit!(self, Instruction::WithCleanupStart); + // WITH_EXCEPT_START: call __exit__(type, value, tb) + // Stack: [..., __exit__, lasti, prev_exc, exc] + // __exit__ is at TOS-3, call with exception info + emit!(self, Instruction::WithExceptStart); if is_async { - emit!(self, Instruction::GetAwaitable); + emit!(self, Instruction::GetAwaitable { arg: 2 }); self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 - } - ); + self.compile_yield_from_sequence(true)?; } - emit!(self, Instruction::WithCleanupFinish); + // TO_BOOL + POP_JUMP_IF_TRUE: check if exception is suppressed + emit!(self, Instruction::ToBool); + emit!( + self, + Instruction::PopJumpIfTrue { + target: suppress_block + } + ); + + // Pop the nested fblock BEFORE RERAISE so that RERAISE's exception + // handler points to the outer handler (try-except), not cleanup_block. + // This is critical: when RERAISE propagates the exception, the exception + // table should route it to the outer try-except, not back to cleanup. + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::ExceptionHandler); + + // Not suppressed: RERAISE 2 + emit!(self, Instruction::Reraise { depth: 2 }); + + // ===== Suppress block ===== + // Exception was suppressed, clean up stack + // Stack: [..., __exit__, lasti, prev_exc, exc, True] + // Need to pop: True, exc, prev_exc, __exit__ + self.switch_to_block(suppress_block); + emit!(self, Instruction::PopTop); // pop True (TO_BOOL result) + emit!(self, Instruction::PopExcept); // pop exc and restore prev_exc + emit!(self, Instruction::PopTop); // pop __exit__ + emit!(self, Instruction::PopTop); // pop lasti + emit!( + self, + PseudoInstruction::Jump { + target: after_block + } + ); + + // ===== Cleanup block (for nested exception during __exit__) ===== + // Stack: [..., __exit__, lasti, prev_exc, lasti2, exc2] + // COPY 3: copy prev_exc to TOS + // POP_EXCEPT: restore exception state + // RERAISE 1: re-raise with lasti + // + // NOTE: We DON'T clear the fblock stack here because we want + // outer exception handlers (e.g., try-except wrapping this with statement) + // to be in the exception table for these instructions. + // If we cleared fblock, exceptions here would propagate uncaught. + self.switch_to_block(cleanup_block); + emit!(self, Instruction::Copy { index: 3 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 1 }); + + // ===== After block ===== + self.switch_to_block(after_block); + self.leave_conditional_block(); Ok(()) } fn compile_for( &mut self, - target: &Expr, - iter: &Expr, - body: &[Stmt], - orelse: &[Stmt], + target: &ast::Expr, + iter: &ast::Expr, + body: &[ast::Stmt], + orelse: &[ast::Stmt], is_async: bool, ) -> CompileResult<()> { + self.enter_conditional_block(); + // Start loop let for_block = self.new_block(); let else_block = self.new_block(); let after_block = self.new_block(); - emit!(self, Instruction::SetupLoop); - // The thing iterated: self.compile_expression(iter)?; if is_async { + if self.ctx.func != FunctionContext::AsyncFunction { + return Err(self.error(CodegenErrorType::InvalidAsyncFor)); + } emit!(self, Instruction::GetAIter); self.switch_to_block(for_block); - // Push fblock for async for loop + // codegen_async_for: push fblock BEFORE SETUP_FINALLY self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; - emit!( - self, - Instruction::SetupExcept { - handler: else_block, - } - ); + // SETUP_FINALLY to guard the __anext__ call + emit!(self, PseudoInstruction::SetupFinally { target: else_block }); emit!(self, Instruction::GetANext); self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 - } - ); + self.compile_yield_from_sequence(true)?; + // POP_BLOCK for SETUP_FINALLY - only GetANext/yield_from are protected + emit!(self, PseudoInstruction::PopBlock); + + // Success block for __anext__ self.compile_store(target)?; - emit!(self, Instruction::PopBlock); } else { // Retrieve Iterator emit!(self, Instruction::GetIter); @@ -3053,21 +5158,27 @@ impl Compiler { let was_in_loop = self.ctx.loop_data.replace((for_block, after_block)); self.compile_statements(body)?; self.ctx.loop_data = was_in_loop; - emit!(self, Instruction::Jump { target: for_block }); + emit!(self, PseudoInstruction::Jump { target: for_block }); self.switch_to_block(else_block); - // Pop fblock + // Except block for __anext__ / end of sync for + // No PopBlock here - for async, POP_BLOCK is already in for_block self.pop_fblock(FBlockType::ForLoop); if is_async { emit!(self, Instruction::EndAsyncFor); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + // FOR_ITER jumps to END_FOR, but VM skips it (+1) to reach POP_ITER + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); } - emit!(self, Instruction::PopBlock); self.compile_statements(orelse)?; self.switch_to_block(after_block); + self.leave_conditional_block(); Ok(()) } @@ -3084,8 +5195,9 @@ impl Compiler { } fn compile_error_forbidden_name(&mut self, name: &str) -> CodegenError { - // TODO: make into error (fine for now since it realistically errors out earlier) - panic!("Failing due to forbidden name {name:?}"); + self.error(CodegenErrorType::SyntaxError(format!( + "cannot use forbidden name '{name}' in pattern" + ))) } /// Ensures that `pc.fail_pop` has at least `n + 1` entries. @@ -3113,7 +5225,7 @@ impl Compiler { JumpOp::Jump => { emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: pc.fail_pop[pops] } ); @@ -3142,7 +5254,7 @@ impl Compiler { for &label in pc.fail_pop.iter().skip(1).rev() { self.switch_to_block(label); // Emit the POP instruction. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } // Finally, use the first label. self.switch_to_block(pc.fail_pop[0]); @@ -3180,13 +5292,13 @@ impl Compiler { /// to the list of captured names. fn pattern_helper_store_name( &mut self, - n: Option<&Identifier>, + n: Option<&ast::Identifier>, pc: &mut PatternContext, ) -> CompileResult<()> { match n { // If no name is provided, simply pop the top of the stack. None => { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } Some(name) => { @@ -3214,7 +5326,7 @@ impl Compiler { } } - fn pattern_unpack_helper(&mut self, elts: &[Pattern]) -> CompileResult<()> { + fn pattern_unpack_helper(&mut self, elts: &[ast::Pattern]) -> CompileResult<()> { let n = elts.len(); let mut seen_star = false; for (i, elt) in elts.iter().enumerate() { @@ -3250,7 +5362,7 @@ impl Compiler { fn pattern_helper_sequence_unpack( &mut self, - patterns: &[Pattern], + patterns: &[ast::Pattern], _star: Option<usize>, pc: &mut PatternContext, ) -> CompileResult<()> { @@ -3270,7 +5382,7 @@ impl Compiler { fn pattern_helper_sequence_subscr( &mut self, - patterns: &[Pattern], + patterns: &[ast::Pattern], star: usize, pc: &mut PatternContext, ) -> CompileResult<()> { @@ -3286,7 +5398,7 @@ impl Compiler { continue; } // Duplicate the subject. - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); if i < star { // For indices before the star, use a nonnegative index equal to i. self.emit_load_const(ConstantData::Integer { value: i.into() }); @@ -3306,19 +5418,24 @@ impl Compiler { ); } // Use BINARY_OP/NB_SUBSCR to extract the element. - emit!(self, Instruction::BinarySubscript); + emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr + } + ); // Compile the subpattern in irrefutable mode. self.compile_pattern_subpattern(pattern, pc)?; } // Pop the subject off the stack. pc.on_top -= 1; - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } fn compile_pattern_subpattern( &mut self, - p: &Pattern, + p: &ast::Pattern, pc: &mut PatternContext, ) -> CompileResult<()> { // Save the current allow_irrefutable state. @@ -3334,7 +5451,7 @@ impl Compiler { fn compile_pattern_as( &mut self, - p: &PatternMatchAs, + p: &ast::PatternMatchAs, pc: &mut PatternContext, ) -> CompileResult<()> { // If there is no sub-pattern, then it's an irrefutable match. @@ -3359,7 +5476,7 @@ impl Compiler { // Otherwise, there is a sub-pattern. Duplicate the object on top of the stack. pc.on_top += 1; - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); // Compile the sub-pattern. self.compile_pattern(p.pattern.as_ref().unwrap(), pc)?; // After success, decrement the on_top counter. @@ -3371,7 +5488,7 @@ impl Compiler { fn compile_pattern_star( &mut self, - p: &PatternMatchStar, + p: &ast::PatternMatchStar, pc: &mut PatternContext, ) -> CompileResult<()> { self.pattern_helper_store_name(p.name.as_ref(), pc)?; @@ -3382,8 +5499,8 @@ impl Compiler { /// and not duplicated. fn validate_kwd_attrs( &mut self, - attrs: &[Identifier], - _patterns: &[Pattern], + attrs: &[ast::Identifier], + _patterns: &[ast::Pattern], ) -> CompileResult<()> { let n_attrs = attrs.len(); for i in 0..n_attrs { @@ -3406,7 +5523,7 @@ impl Compiler { fn compile_pattern_class( &mut self, - p: &PatternMatchClass, + p: &ast::PatternMatchClass, pc: &mut PatternContext, ) -> CompileResult<()> { // Extract components from the MatchClass pattern. @@ -3427,12 +5544,9 @@ impl Compiler { // Check for too many sub-patterns. if nargs > u32::MAX as usize || (nargs + n_attrs).saturating_sub(1) > i32::MAX as usize { - let msg = format!( - "too many sub-patterns in class pattern {:?}", - match_class.cls - ); - panic!("{}", msg); - // return self.compiler_error(&msg); + return Err(self.error(CodegenErrorType::SyntaxError( + "too many sub-patterns in class pattern".to_owned(), + ))); } // Validate keyword attributes if any. @@ -3460,7 +5574,7 @@ impl Compiler { // 2. Emit MATCH_CLASS with nargs. emit!(self, Instruction::MatchClass(u32::try_from(nargs).unwrap())); // 3. Duplicate the top of the stack. - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); // 4. Load None. self.emit_load_const(ConstantData::None); // 5. Compare with IS_OP 1. @@ -3485,7 +5599,7 @@ impl Compiler { for subpattern in patterns.iter().chain(kwd_patterns.iter()) { // Check if this is a true wildcard (underscore pattern without name binding) let is_true_wildcard = match subpattern { - Pattern::MatchAs(match_as) => { + ast::Pattern::MatchAs(match_as) => { // Only consider it wildcard if both pattern and name are None (i.e., "_") match_as.pattern.is_none() && match_as.name.is_none() } @@ -3496,7 +5610,7 @@ impl Compiler { pc.on_top -= 1; if is_true_wildcard { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); continue; // Don't compile wildcard patterns } @@ -3508,7 +5622,7 @@ impl Compiler { fn compile_pattern_mapping( &mut self, - p: &PatternMatchMapping, + p: &ast::PatternMatchMapping, pc: &mut PatternContext, ) -> CompileResult<()> { let mapping = p; @@ -3547,7 +5661,7 @@ impl Compiler { if size == 0 && star_target.is_none() { // If the pattern is just "{}", we're done! Pop the subject pc.on_top -= 1; - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); return Ok(()); } @@ -3559,7 +5673,7 @@ impl Compiler { // Stack: [subject, len, size] emit!( self, - Instruction::CompareOperation { + Instruction::CompareOp { op: ComparisonOperator::GreaterOrEqual } ); @@ -3573,22 +5687,22 @@ impl Compiler { "too many sub-patterns in mapping pattern".to_string(), ))); } - #[allow(clippy::cast_possible_truncation)] - let size = size as u32; // checked right before + #[allow(clippy::cast_possible_truncation, reason = "checked right before")] + let size = size as u32; // Step 2: If we have keys to match if size > 0 { // Validate and compile keys - let mut seen = HashSet::new(); + let mut seen = IndexSet::default(); for key in keys { - let is_attribute = matches!(key, Expr::Attribute(_)); + let is_attribute = matches!(key, ast::Expr::Attribute(_)); let is_literal = matches!( key, - Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) + ast::Expr::NumberLiteral(_) + | ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) ); let key_repr = if is_literal { UnparseExpr::new(key, &self.source_file).to_string() @@ -3625,7 +5739,7 @@ impl Compiler { pc.on_top += 2; // subject and keys_tuple are underneath // Check if match succeeded - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); // Stack: [subject, keys_tuple, values_tuple, values_tuple_copy] // Check if copy is None (consumes the copy like POP_JUMP_IF_NONE) @@ -3649,7 +5763,7 @@ impl Compiler { } // After processing subpatterns, adjust on_top - // CPython: "Whatever happens next should consume the tuple of keys and the subject" + // "Whatever happens next should consume the tuple of keys and the subject" // Stack currently: [subject, keys_tuple, ...any captured values...] pc.on_top -= 2; @@ -3678,14 +5792,14 @@ impl Compiler { // Copy rest_dict which is at position (1 + remaining) from TOS emit!( self, - Instruction::CopyItem { + Instruction::Copy { index: 1 + remaining } ); // Stack: [rest_dict, k1, ..., kn, rest_dict] emit!(self, Instruction::Swap { index: 2 }); // Stack: [rest_dict, k1, ..., kn-1, rest_dict, kn] - emit!(self, Instruction::DeleteSubscript); + emit!(self, Instruction::DeleteSubscr); // Stack: [rest_dict, k1, ..., kn-1] (removed kn from rest_dict) remaining -= 1; } @@ -3702,8 +5816,8 @@ impl Compiler { // Non-rest pattern: just clean up the stack // Pop them as we're not using them - emit!(self, Instruction::Pop); // Pop keys_tuple - emit!(self, Instruction::Pop); // Pop subject + emit!(self, Instruction::PopTop); // Pop keys_tuple + emit!(self, Instruction::PopTop); // Pop subject } Ok(()) @@ -3711,13 +5825,17 @@ impl Compiler { fn compile_pattern_or( &mut self, - p: &PatternMatchOr, + p: &ast::PatternMatchOr, pc: &mut PatternContext, ) -> CompileResult<()> { // Ensure the pattern is a MatchOr. let end = self.new_block(); // Create a new jump target label. let size = p.patterns.len(); - assert!(size > 1, "MatchOr must have more than one alternative"); + if size <= 1 { + return Err(self.error(CodegenErrorType::SyntaxError( + "MatchOr requires at least 2 patterns".to_owned(), + ))); + } // Save the current pattern context. let old_pc = pc.clone(); @@ -3735,7 +5853,7 @@ impl Compiler { pc.fail_pop.clear(); pc.on_top = 0; // Emit a COPY(1) instruction before compiling the alternative. - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.compile_pattern(alt, pc)?; let n_stores = pc.stores.len(); @@ -3779,7 +5897,7 @@ impl Compiler { } } // Emit a jump to the common end label and reset any failure jump targets. - emit!(self, Instruction::Jump { target: end }); + emit!(self, PseudoInstruction::Jump { target: end }); self.emit_and_reset_fail_pop(pc)?; } @@ -3791,7 +5909,7 @@ impl Compiler { // In Rust, old_pc is a local clone, so we need not worry about that. // No alternative matched: pop the subject and fail. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); self.jump_to_fail_pop(pc, JumpOp::Jump)?; // Use the label "end". @@ -3813,17 +5931,17 @@ impl Compiler { // Old context and control will be dropped automatically. // Finally, pop the copy of the subject. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } fn compile_pattern_sequence( &mut self, - p: &PatternMatchSequence, + p: &ast::PatternMatchSequence, pc: &mut PatternContext, ) -> CompileResult<()> { // Ensure the pattern is a MatchSequence. - let patterns = &p.patterns; // a slice of Pattern + let patterns = &p.patterns; // a slice of ast::Pattern let size = patterns.len(); let mut star: Option<usize> = None; let mut only_wildcard = true; @@ -3863,7 +5981,7 @@ impl Compiler { self.emit_load_const(ConstantData::Integer { value: size.into() }); emit!( self, - Instruction::CompareOperation { + Instruction::CompareOp { op: ComparisonOperator::Equal } ); @@ -3876,7 +5994,7 @@ impl Compiler { }); emit!( self, - Instruction::CompareOperation { + Instruction::CompareOp { op: ComparisonOperator::GreaterOrEqual } ); @@ -3886,8 +6004,8 @@ impl Compiler { // Whatever comes next should consume the subject. pc.on_top -= 1; if only_wildcard { - // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. - emit!(self, Instruction::Pop); + // ast::Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. + emit!(self, Instruction::PopTop); } else if star_wildcard { self.pattern_helper_sequence_subscr(patterns, star.unwrap(), pc)?; } else { @@ -3898,14 +6016,14 @@ impl Compiler { fn compile_pattern_value( &mut self, - p: &PatternMatchValue, + p: &ast::PatternMatchValue, pc: &mut PatternContext, ) -> CompileResult<()> { // TODO: ensure literal or attribute lookup self.compile_expression(&p.value)?; emit!( self, - Instruction::CompareOperation { + Instruction::CompareOp { op: bytecode::ComparisonOperator::Equal } ); @@ -3916,14 +6034,14 @@ impl Compiler { fn compile_pattern_singleton( &mut self, - p: &PatternMatchSingleton, + p: &ast::PatternMatchSingleton, pc: &mut PatternContext, ) -> CompileResult<()> { // Load the singleton constant value. self.emit_load_const(match p.value { - Singleton::None => ConstantData::None, - Singleton::False => ConstantData::Boolean { value: false }, - Singleton::True => ConstantData::Boolean { value: true }, + ast::Singleton::None => ConstantData::None, + ast::Singleton::False => ConstantData::Boolean { value: false }, + ast::Singleton::True => ConstantData::Boolean { value: true }, }); // Compare using the "Is" operator. emit!(self, Instruction::IsOp(Invert::No)); @@ -3934,32 +6052,32 @@ impl Compiler { fn compile_pattern( &mut self, - pattern_type: &Pattern, + pattern_type: &ast::Pattern, pattern_context: &mut PatternContext, ) -> CompileResult<()> { match &pattern_type { - Pattern::MatchValue(pattern_type) => { + ast::Pattern::MatchValue(pattern_type) => { self.compile_pattern_value(pattern_type, pattern_context) } - Pattern::MatchSingleton(pattern_type) => { + ast::Pattern::MatchSingleton(pattern_type) => { self.compile_pattern_singleton(pattern_type, pattern_context) } - Pattern::MatchSequence(pattern_type) => { + ast::Pattern::MatchSequence(pattern_type) => { self.compile_pattern_sequence(pattern_type, pattern_context) } - Pattern::MatchMapping(pattern_type) => { + ast::Pattern::MatchMapping(pattern_type) => { self.compile_pattern_mapping(pattern_type, pattern_context) } - Pattern::MatchClass(pattern_type) => { + ast::Pattern::MatchClass(pattern_type) => { self.compile_pattern_class(pattern_type, pattern_context) } - Pattern::MatchStar(pattern_type) => { + ast::Pattern::MatchStar(pattern_type) => { self.compile_pattern_star(pattern_type, pattern_context) } - Pattern::MatchAs(pattern_type) => { + ast::Pattern::MatchAs(pattern_type) => { self.compile_pattern_as(pattern_type, pattern_context) } - Pattern::MatchOr(pattern_type) => { + ast::Pattern::MatchOr(pattern_type) => { self.compile_pattern_or(pattern_type, pattern_context) } } @@ -3967,8 +6085,8 @@ impl Compiler { fn compile_match_inner( &mut self, - subject: &Expr, - cases: &[MatchCase], + subject: &ast::Expr, + cases: &[ast::MatchCase], pattern_context: &mut PatternContext, ) -> CompileResult<()> { self.compile_expression(subject)?; @@ -3982,7 +6100,7 @@ impl Compiler { for (i, m) in cases.iter().enumerate().take(case_count) { // Only copy the subject if not on the last case if i != case_count - 1 { - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); } pattern_context.stores = Vec::with_capacity(1); @@ -4011,25 +6129,27 @@ impl Compiler { } if i != case_count - 1 { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } self.compile_statements(&m.body)?; - emit!(self, Instruction::Jump { target: end }); + emit!(self, PseudoInstruction::Jump { target: end }); self.emit_and_reset_fail_pop(pattern_context)?; } if has_default { let m = &cases[num_cases - 1]; if num_cases == 1 { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } else { emit!(self, Instruction::Nop); } if let Some(ref guard) = m.guard { // Compile guard and jump to end if false self.compile_expression(guard)?; - emit!(self, Instruction::JumpIfFalseOrPop { target: end }); + emit!(self, Instruction::Copy { index: 1_u32 }); + emit!(self, Instruction::PopJumpIfFalse { target: end }); + emit!(self, Instruction::PopTop); } self.compile_statements(&m.body)?; } @@ -4037,102 +6157,108 @@ impl Compiler { Ok(()) } - fn compile_match(&mut self, subject: &Expr, cases: &[MatchCase]) -> CompileResult<()> { + fn compile_match( + &mut self, + subject: &ast::Expr, + cases: &[ast::MatchCase], + ) -> CompileResult<()> { + self.enter_conditional_block(); let mut pattern_context = PatternContext::new(); self.compile_match_inner(subject, cases, &mut pattern_context)?; + self.leave_conditional_block(); Ok(()) } - fn compile_chained_comparison( + /// [CPython `compiler_addcompare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L2880-L2924) + fn compile_addcompare(&mut self, op: &ast::CmpOp) { + use bytecode::ComparisonOperator::*; + match op { + ast::CmpOp::Eq => emit!(self, Instruction::CompareOp { op: Equal }), + ast::CmpOp::NotEq => emit!(self, Instruction::CompareOp { op: NotEqual }), + ast::CmpOp::Lt => emit!(self, Instruction::CompareOp { op: Less }), + ast::CmpOp::LtE => emit!(self, Instruction::CompareOp { op: LessOrEqual }), + ast::CmpOp::Gt => emit!(self, Instruction::CompareOp { op: Greater }), + ast::CmpOp::GtE => { + emit!(self, Instruction::CompareOp { op: GreaterOrEqual }) + } + ast::CmpOp::In => emit!(self, Instruction::ContainsOp(Invert::No)), + ast::CmpOp::NotIn => emit!(self, Instruction::ContainsOp(Invert::Yes)), + ast::CmpOp::Is => emit!(self, Instruction::IsOp(Invert::No)), + ast::CmpOp::IsNot => emit!(self, Instruction::IsOp(Invert::Yes)), + } + } + + /// Compile a chained comparison. + /// + /// ```py + /// a == b == c == d + /// ``` + /// + /// Will compile into (pseudo code): + /// + /// ```py + /// result = a == b + /// if result: + /// result = b == c + /// if result: + /// result = c == d + /// ``` + /// + /// # See Also + /// - [CPython `compiler_compare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L4678-L4717) + fn compile_compare( &mut self, - left: &Expr, - ops: &[CmpOp], - exprs: &[Expr], + left: &ast::Expr, + ops: &[ast::CmpOp], + comparators: &[ast::Expr], ) -> CompileResult<()> { - assert!(!ops.is_empty()); - assert_eq!(exprs.len(), ops.len()); let (last_op, mid_ops) = ops.split_last().unwrap(); - let (last_val, mid_exprs) = exprs.split_last().unwrap(); - - use bytecode::ComparisonOperator::*; - let compile_cmpop = |c: &mut Self, op: &CmpOp| match op { - CmpOp::Eq => emit!(c, Instruction::CompareOperation { op: Equal }), - CmpOp::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }), - CmpOp::Lt => emit!(c, Instruction::CompareOperation { op: Less }), - CmpOp::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }), - CmpOp::Gt => emit!(c, Instruction::CompareOperation { op: Greater }), - CmpOp::GtE => { - emit!(c, Instruction::CompareOperation { op: GreaterOrEqual }) - } - CmpOp::In => emit!(c, Instruction::ContainsOp(Invert::No)), - CmpOp::NotIn => emit!(c, Instruction::ContainsOp(Invert::Yes)), - CmpOp::Is => emit!(c, Instruction::IsOp(Invert::No)), - CmpOp::IsNot => emit!(c, Instruction::IsOp(Invert::Yes)), - }; - - // a == b == c == d - // compile into (pseudo code): - // result = a == b - // if result: - // result = b == c - // if result: - // result = c == d + let (last_comparator, mid_comparators) = comparators.split_last().unwrap(); // initialize lhs outside of loop self.compile_expression(left)?; - let end_blocks = if mid_exprs.is_empty() { - None - } else { - let break_block = self.new_block(); - let after_block = self.new_block(); - Some((break_block, after_block)) - }; + if mid_comparators.is_empty() { + self.compile_expression(last_comparator)?; + self.compile_addcompare(last_op); + + return Ok(()); + } + + let cleanup = self.new_block(); // for all comparisons except the last (as the last one doesn't need a conditional jump) - for (op, val) in mid_ops.iter().zip(mid_exprs) { - self.compile_expression(val)?; + for (op, comparator) in mid_ops.iter().zip(mid_comparators) { + self.compile_expression(comparator)?; + // store rhs for the next comparison in chain emit!(self, Instruction::Swap { index: 2 }); - emit!(self, Instruction::CopyItem { index: 2_u32 }); + emit!(self, Instruction::Copy { index: 2 }); - compile_cmpop(self, op); + self.compile_addcompare(op); // if comparison result is false, we break with this value; if true, try the next one. - if let Some((break_block, _)) = end_blocks { - emit!( - self, - Instruction::JumpIfFalseOrPop { - target: break_block, - } - ); - } + emit!(self, Instruction::Copy { index: 1 }); + emit!(self, Instruction::PopJumpIfFalse { target: cleanup }); + emit!(self, Instruction::PopTop); } - // handle the last comparison - self.compile_expression(last_val)?; - compile_cmpop(self, last_op); - - if let Some((break_block, after_block)) = end_blocks { - emit!( - self, - Instruction::Jump { - target: after_block, - } - ); + self.compile_expression(last_comparator)?; + self.compile_addcompare(last_op); - // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. - self.switch_to_block(break_block); - emit!(self, Instruction::Swap { index: 2 }); - emit!(self, Instruction::Pop); + let end = self.new_block(); + emit!(self, PseudoInstruction::Jump { target: end }); - self.switch_to_block(after_block); - } + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.switch_to_block(cleanup); + emit!(self, Instruction::Swap { index: 2 }); + emit!(self, Instruction::PopTop); + self.switch_to_block(end); Ok(()) } - fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { + fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { value: UnparseExpr::new(annotation, &self.source_file) @@ -4145,8 +6271,7 @@ impl Compiler { // Special handling for starred annotations (*Ts -> Unpack[Ts]) let result = match annotation { - Expr::Starred(ExprStarred { value, .. }) => { - // Following CPython's approach: + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { // *args: *Ts (where Ts is a TypeVarTuple). // Do [annotation_value] = [*Ts]. self.compile_expression(value)?; @@ -4164,59 +6289,88 @@ impl Compiler { fn compile_annotated_assign( &mut self, - target: &Expr, - annotation: &Expr, - value: Option<&Expr>, + target: &ast::Expr, + annotation: &ast::Expr, + value: Option<&ast::Expr>, + simple: bool, ) -> CompileResult<()> { + // Perform the actual assignment first if let Some(value) = value { self.compile_expression(value)?; self.compile_store(target)?; } - // Annotations are only evaluated in a module or class. - if self.ctx.in_func() { - return Ok(()); - } - - // Compile annotation: - self.compile_annotation(annotation)?; - - if let Expr::Name(ExprName { id, .. }) = &target { - // Store as dict entry in __annotations__ dict: - let annotations = self.name("__annotations__"); - emit!(self, Instruction::LoadNameAny(annotations)); - self.emit_load_const(ConstantData::Str { - value: self.mangle(id.as_str()).into_owned().into(), - }); - emit!(self, Instruction::StoreSubscript); - } else { - // Drop annotation if not assigned to simple identifier. - emit!(self, Instruction::Pop); + // If we have a simple name in module or class scope, store annotation + if simple + && !self.ctx.in_func() + && let ast::Expr::Name(ast::ExprName { id, .. }) = target + { + if self.future_annotations { + // PEP 563: Store stringified annotation directly to __annotations__ + // Compile annotation as string + self.compile_annotation(annotation)?; + // Load __annotations__ + let annotations_name = self.name("__annotations__"); + emit!(self, Instruction::LoadName(annotations_name)); + // Load the variable name + self.emit_load_const(ConstantData::Str { + value: self.mangle(id.as_str()).into_owned().into(), + }); + // Store: __annotations__[name] = annotation + emit!(self, Instruction::StoreSubscr); + } else { + // PEP 649: Handle conditional annotations + if self.current_symbol_table().has_conditional_annotations { + // Allocate an index for every annotation when has_conditional_annotations + // This keeps indices aligned with compile_module_annotate's enumeration + let code_info = self.current_code_info(); + let annotation_index = code_info.next_conditional_annotation_index; + code_info.next_conditional_annotation_index += 1; + + // Determine if this annotation is conditional + // Module and Class scopes both need all annotations tracked + let scope_type = self.current_symbol_table().typ; + let in_conditional_block = self.current_code_info().in_conditional_block > 0; + let is_conditional = + matches!(scope_type, CompilerScope::Module | CompilerScope::Class) + || in_conditional_block; + + // Only add to __conditional_annotations__ set if actually conditional + if is_conditional { + self.load_name("__conditional_annotations__")?; + self.emit_load_const(ConstantData::Integer { + value: annotation_index.into(), + }); + emit!(self, Instruction::SetAdd { i: 0_u32 }); + emit!(self, Instruction::PopTop); + } + } + } } Ok(()) } - fn compile_store(&mut self, target: &Expr) -> CompileResult<()> { + fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> { match &target { - Expr::Name(ExprName { id, .. }) => self.store_name(id.as_str())?, - Expr::Subscript(ExprSubscript { + ast::Expr::Name(ast::ExprName { id, .. }) => self.store_name(id.as_str())?, + ast::Expr::Subscript(ast::ExprSubscript { value, slice, ctx, .. }) => { self.compile_subscript(value, slice, *ctx)?; } - Expr::Attribute(ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr.as_str(), NameUsage::Store)?; + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { self.compile_expression(value)?; let idx = self.name(attr.as_str()); emit!(self, Instruction::StoreAttr { idx }); } - Expr::List(ExprList { elts, .. }) | Expr::Tuple(ExprTuple { elts, .. }) => { + ast::Expr::List(ast::ExprList { elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { let mut seen_star = false; // Scan for star args: for (i, element) in elts.iter().enumerate() { - if let Expr::Starred(_) = &element { + if let ast::Expr::Starred(_) = &element { if seen_star { return Err(self.error(CodegenErrorType::MultipleStarArgs)); } else { @@ -4246,7 +6400,7 @@ impl Compiler { } for element in elts { - if let Expr::Starred(ExprStarred { value, .. }) = &element { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = &element { self.compile_store(value)?; } else { self.compile_store(element)?; @@ -4255,7 +6409,7 @@ impl Compiler { } _ => { return Err(self.error(match target { - Expr::Starred(_) => CodegenErrorType::SyntaxError( + ast::Expr::Starred(_) => CodegenErrorType::SyntaxError( "starred assignment target must be in a list or tuple".to_owned(), ), _ => CodegenErrorType::Assign(target.python_name()), @@ -4268,9 +6422,9 @@ impl Compiler { fn compile_augassign( &mut self, - target: &Expr, - op: &Operator, - value: &Expr, + target: &ast::Expr, + op: &ast::Operator, + value: &ast::Expr, ) -> CompileResult<()> { enum AugAssignKind<'a> { Name { id: &'a str }, @@ -4279,12 +6433,12 @@ impl Compiler { } let kind = match &target { - Expr::Name(ExprName { id, .. }) => { + ast::Expr::Name(ast::ExprName { id, .. }) => { let id = id.as_str(); self.compile_name(id, NameUsage::Load)?; AugAssignKind::Name { id } } - Expr::Subscript(ExprSubscript { + ast::Expr::Subscript(ast::ExprSubscript { value, slice, ctx: _, @@ -4294,18 +6448,22 @@ impl Compiler { // But we can't use compile_subscript directly because we need DUP_TOP2 self.compile_expression(value)?; self.compile_expression(slice)?; - emit!(self, Instruction::CopyItem { index: 2_u32 }); - emit!(self, Instruction::CopyItem { index: 2_u32 }); - emit!(self, Instruction::Subscript); + emit!(self, Instruction::Copy { index: 2_u32 }); + emit!(self, Instruction::Copy { index: 2_u32 }); + emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr + } + ); AugAssignKind::Subscript } - Expr::Attribute(ExprAttribute { value, attr, .. }) => { + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { let attr = attr.as_str(); - self.check_forbidden_name(attr, NameUsage::Store)?; self.compile_expression(value)?; - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); let idx = self.name(attr); - emit!(self, Instruction::LoadAttr { idx }); + self.emit_load_attr(idx); AugAssignKind::Attr { idx } } _ => { @@ -4325,7 +6483,7 @@ impl Compiler { // stack: CONTAINER SLICE RESULT emit!(self, Instruction::Swap { index: 3 }); emit!(self, Instruction::Swap { index: 2 }); - emit!(self, Instruction::StoreSubscript); + emit!(self, Instruction::StoreSubscr); } AugAssignKind::Attr { idx } => { // stack: CONTAINER RESULT @@ -4337,21 +6495,21 @@ impl Compiler { Ok(()) } - fn compile_op(&mut self, op: &Operator, inplace: bool) { + fn compile_op(&mut self, op: &ast::Operator, inplace: bool) { let bin_op = match op { - Operator::Add => BinaryOperator::Add, - Operator::Sub => BinaryOperator::Subtract, - Operator::Mult => BinaryOperator::Multiply, - Operator::MatMult => BinaryOperator::MatrixMultiply, - Operator::Div => BinaryOperator::TrueDivide, - Operator::FloorDiv => BinaryOperator::FloorDivide, - Operator::Mod => BinaryOperator::Remainder, - Operator::Pow => BinaryOperator::Power, - Operator::LShift => BinaryOperator::Lshift, - Operator::RShift => BinaryOperator::Rshift, - Operator::BitOr => BinaryOperator::Or, - Operator::BitXor => BinaryOperator::Xor, - Operator::BitAnd => BinaryOperator::And, + ast::Operator::Add => BinaryOperator::Add, + ast::Operator::Sub => BinaryOperator::Subtract, + ast::Operator::Mult => BinaryOperator::Multiply, + ast::Operator::MatMult => BinaryOperator::MatrixMultiply, + ast::Operator::Div => BinaryOperator::TrueDivide, + ast::Operator::FloorDiv => BinaryOperator::FloorDivide, + ast::Operator::Mod => BinaryOperator::Remainder, + ast::Operator::Pow => BinaryOperator::Power, + ast::Operator::LShift => BinaryOperator::Lshift, + ast::Operator::RShift => BinaryOperator::Rshift, + ast::Operator::BitOr => BinaryOperator::Or, + ast::Operator::BitXor => BinaryOperator::Xor, + ast::Operator::BitAnd => BinaryOperator::And, }; let op = if inplace { bin_op.as_inplace() } else { bin_op }; @@ -4368,15 +6526,15 @@ impl Compiler { /// (indicated by the condition parameter). fn compile_jump_if( &mut self, - expression: &Expr, + expression: &ast::Expr, condition: bool, target_block: BlockIdx, ) -> CompileResult<()> { // Compile expression for test, and jump to label if false match &expression { - Expr::BoolOp(ExprBoolOp { op, values, .. }) => { + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { match op { - BoolOp::And => { + ast::BoolOp::And => { if condition { // If all values are true. let end_block = self.new_block(); @@ -4397,7 +6555,7 @@ impl Compiler { } } } - BoolOp::Or => { + ast::BoolOp::Or => { if condition { // If any of the values is true. for value in values { @@ -4420,8 +6578,8 @@ impl Compiler { } } } - Expr::UnaryOp(ExprUnaryOp { - op: UnaryOp::Not, + ast::Expr::UnaryOp(ast::ExprUnaryOp { + op: ast::UnaryOp::Not, operand, .. }) => { @@ -4452,31 +6610,47 @@ impl Compiler { /// Compile a boolean operation as an expression. /// This means, that the last value remains on the stack. - fn compile_bool_op(&mut self, op: &BoolOp, values: &[Expr]) -> CompileResult<()> { - let after_block = self.new_block(); + fn compile_bool_op(&mut self, op: &ast::BoolOp, values: &[ast::Expr]) -> CompileResult<()> { + self.compile_bool_op_with_target(op, values, None) + } + /// Compile a boolean operation as an expression, with an optional + /// short-circuit target override. When `short_circuit_target` is `Some`, + /// the short-circuit jumps go to that block instead of the default + /// `after_block`, enabling jump threading to avoid redundant `__bool__` calls. + fn compile_bool_op_with_target( + &mut self, + op: &ast::BoolOp, + values: &[ast::Expr], + short_circuit_target: Option<BlockIdx>, + ) -> CompileResult<()> { + let after_block = self.new_block(); let (last_value, values) = values.split_last().unwrap(); - for value in values { - self.compile_expression(value)?; + let jump_target = short_circuit_target.unwrap_or(after_block); - match op { - BoolOp::And => { - emit!( - self, - Instruction::JumpIfFalseOrPop { - target: after_block, - } - ); - } - BoolOp::Or => { - emit!( - self, - Instruction::JumpIfTrueOrPop { - target: after_block, - } - ); - } + for value in values { + // Optimization: when a non-last value is a BoolOp with the opposite + // operator, redirect its short-circuit exits to skip the outer's + // redundant __bool__ test (jump threading). + if short_circuit_target.is_none() + && let ast::Expr::BoolOp(ast::ExprBoolOp { + op: inner_op, + values: inner_values, + .. + }) = value + && inner_op != op + { + let pop_block = self.new_block(); + self.compile_bool_op_with_target(inner_op, inner_values, Some(pop_block))?; + self.emit_short_circuit_test(op, after_block); + self.switch_to_block(pop_block); + emit!(self, Instruction::PopTop); + continue; } + + self.compile_expression(value)?; + self.emit_short_circuit_test(op, jump_target); + emit!(self, Instruction::PopTop); } // If all values did not qualify, take the value of the last value: @@ -4485,95 +6659,256 @@ impl Compiler { Ok(()) } - fn compile_dict(&mut self, items: &[DictItem]) -> CompileResult<()> { - // FIXME: correct order to build map, etc d = {**a, 'key': 2} should override - // 'key' in dict a - let mut size = 0; - let (packed, unpacked): (Vec<_>, Vec<_>) = items.iter().partition(|x| x.key.is_some()); - for item in packed { - self.compile_expression(item.key.as_ref().unwrap())?; - self.compile_expression(&item.value)?; - size += 1; + /// Emit `Copy 1` + conditional jump for short-circuit evaluation. + /// For `And`, emits `PopJumpIfFalse`; for `Or`, emits `PopJumpIfTrue`. + fn emit_short_circuit_test(&mut self, op: &ast::BoolOp, target: BlockIdx) { + emit!(self, Instruction::Copy { index: 1_u32 }); + match op { + ast::BoolOp::And => { + emit!(self, Instruction::PopJumpIfFalse { target }); + } + ast::BoolOp::Or => { + emit!(self, Instruction::PopJumpIfTrue { target }); + } + } + } + + fn compile_dict(&mut self, items: &[ast::DictItem]) -> CompileResult<()> { + let has_unpacking = items.iter().any(|item| item.key.is_none()); + + if !has_unpacking { + // Simple case: no ** unpacking, build all pairs directly + for item in items { + self.compile_expression(item.key.as_ref().unwrap())?; + self.compile_expression(&item.value)?; + } + emit!( + self, + Instruction::BuildMap { + size: u32::try_from(items.len()).expect("too many dict items"), + } + ); + return Ok(()); + } + + // Complex case with ** unpacking: preserve insertion order. + // Collect runs of regular k:v pairs and emit BUILD_MAP + DICT_UPDATE + // for each run, and DICT_UPDATE for each ** entry. + let mut have_dict = false; + let mut elements: u32 = 0; + + // Flush pending regular pairs as a BUILD_MAP, merging into the + // accumulator dict via DICT_UPDATE when one already exists. + macro_rules! flush_pending { + () => { + #[allow(unused_assignments)] + if elements > 0 { + emit!(self, Instruction::BuildMap { size: elements }); + if have_dict { + emit!(self, Instruction::DictUpdate { index: 1 }); + } else { + have_dict = true; + } + elements = 0; + } + }; } - emit!(self, Instruction::BuildMap { size }); - for item in unpacked { - self.compile_expression(&item.value)?; - emit!(self, Instruction::DictUpdate { index: 1 }); + for item in items { + if let Some(key) = &item.key { + // Regular key: value pair + self.compile_expression(key)?; + self.compile_expression(&item.value)?; + elements += 1; + } else { + // ** unpacking entry + flush_pending!(); + if !have_dict { + emit!(self, Instruction::BuildMap { size: 0 }); + have_dict = true; + } + self.compile_expression(&item.value)?; + emit!(self, Instruction::DictUpdate { index: 1 }); + } } + flush_pending!(); + if !have_dict { + emit!(self, Instruction::BuildMap { size: 0 }); + } + + Ok(()) + } + + /// Compile the yield-from/await sequence using SEND/END_SEND/CLEANUP_THROW. + /// compiler_add_yield_from + /// This generates: + /// send: + /// SEND exit + /// SETUP_FINALLY fail (via exception table) + /// YIELD_VALUE 1 + /// POP_BLOCK (implicit) + /// RESUME + /// JUMP send + /// fail: + /// CLEANUP_THROW + /// exit: + /// END_SEND + fn compile_yield_from_sequence(&mut self, is_await: bool) -> CompileResult<()> { + let send_block = self.new_block(); + let fail_block = self.new_block(); + let exit_block = self.new_block(); + + // send: + self.switch_to_block(send_block); + emit!(self, Instruction::Send { target: exit_block }); + + // SETUP_FINALLY fail - set up exception handler for YIELD_VALUE + emit!(self, PseudoInstruction::SetupFinally { target: fail_block }); + self.push_fblock( + FBlockType::TryExcept, // Use TryExcept for exception handler + send_block, + exit_block, + )?; + + // YIELD_VALUE with arg=1 (yield-from/await mode - not wrapped for async gen) + emit!(self, Instruction::YieldValue { arg: 1 }); + + // POP_BLOCK before RESUME + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + + // RESUME + emit!( + self, + Instruction::Resume { + arg: if is_await { + u32::from(bytecode::ResumeType::AfterAwait) + } else { + u32::from(bytecode::ResumeType::AfterYieldFrom) + } + } + ); + + // JUMP_BACKWARD_NO_INTERRUPT send + emit!( + self, + PseudoInstruction::JumpNoInterrupt { target: send_block } + ); + + // fail: CLEANUP_THROW + // Stack when exception: [receiver, yielded_value, exc] + // CLEANUP_THROW: [sub_iter, last_sent_val, exc] -> [None, value] + // After: stack is [None, value], fall through to exit + self.switch_to_block(fail_block); + emit!(self, Instruction::CleanupThrow); + // Fall through to exit block + + // exit: END_SEND + // Stack: [receiver, value] (from SEND) or [None, value] (from CLEANUP_THROW) + // END_SEND: [receiver/None, value] -> [value] + self.switch_to_block(exit_block); + emit!(self, Instruction::EndSend); + Ok(()) } - fn compile_expression(&mut self, expression: &Expr) -> CompileResult<()> { - use ruff_python_ast::*; + fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> { trace!("Compiling {expression:?}"); let range = expression.range(); self.set_source_range(range); match &expression { - Expr::Call(ExprCall { + ast::Expr::Call(ast::ExprCall { func, arguments, .. }) => self.compile_call(func, arguments)?, - Expr::BoolOp(ExprBoolOp { op, values, .. }) => self.compile_bool_op(op, values)?, - Expr::BinOp(ExprBinOp { + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { + self.compile_bool_op(op, values)? + } + ast::Expr::BinOp(ast::ExprBinOp { left, op, right, .. }) => { self.compile_expression(left)?; self.compile_expression(right)?; - // Perform operation: + // Restore full expression range before emitting the operation + self.set_source_range(range); self.compile_op(op, false); } - Expr::Subscript(ExprSubscript { + ast::Expr::Subscript(ast::ExprSubscript { value, slice, ctx, .. }) => { self.compile_subscript(value, slice, *ctx)?; } - Expr::UnaryOp(ExprUnaryOp { op, operand, .. }) => { + ast::Expr::UnaryOp(ast::ExprUnaryOp { op, operand, .. }) => { self.compile_expression(operand)?; - // Perform operation: - let op = match op { - UnaryOp::UAdd => bytecode::UnaryOperator::Plus, - UnaryOp::USub => bytecode::UnaryOperator::Minus, - UnaryOp::Not => bytecode::UnaryOperator::Not, - UnaryOp::Invert => bytecode::UnaryOperator::Invert, + // Restore full expression range before emitting the operation + self.set_source_range(range); + match op { + ast::UnaryOp::UAdd => emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::UnaryPositive + } + ), + ast::UnaryOp::USub => emit!(self, Instruction::UnaryNegative), + ast::UnaryOp::Not => { + emit!(self, Instruction::ToBool); + emit!(self, Instruction::UnaryNot); + } + ast::UnaryOp::Invert => emit!(self, Instruction::UnaryInvert), }; - emit!(self, Instruction::UnaryOperation { op }); } - Expr::Attribute(ExprAttribute { value, attr, .. }) => { - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::LoadAttr { idx }); + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + // Check for super() attribute access optimization + if let Some(super_type) = self.can_optimize_super_call(value, attr.as_str()) { + // super().attr or super(cls, self).attr optimization + // Stack: [global_super, class, self] → LOAD_SUPER_ATTR → [attr] + self.load_args_for_super(&super_type)?; + let idx = self.name(attr.as_str()); + match super_type { + SuperCallType::TwoArg { .. } => { + self.emit_load_super_attr(idx); + } + SuperCallType::ZeroArg => { + self.emit_load_zero_super_attr(idx); + } + } + } else { + // Normal attribute access + self.compile_expression(value)?; + let idx = self.name(attr.as_str()); + self.emit_load_attr(idx); + } } - Expr::Compare(ExprCompare { + ast::Expr::Compare(ast::ExprCompare { left, ops, comparators, .. }) => { - self.compile_chained_comparison(left, ops, comparators)?; + self.compile_compare(left, ops, comparators)?; } - // Expr::Constant(ExprConstant { value, .. }) => { + // ast::Expr::Constant(ExprConstant { value, .. }) => { // self.emit_load_const(compile_constant(value)); // } - Expr::List(ExprList { elts, .. }) => { + ast::Expr::List(ast::ExprList { elts, .. }) => { self.starunpack_helper(elts, 0, CollectionType::List)?; } - Expr::Tuple(ExprTuple { elts, .. }) => { + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { self.starunpack_helper(elts, 0, CollectionType::Tuple)?; } - Expr::Set(ExprSet { elts, .. }) => { + ast::Expr::Set(ast::ExprSet { elts, .. }) => { self.starunpack_helper(elts, 0, CollectionType::Set)?; } - Expr::Dict(ExprDict { items, .. }) => { + ast::Expr::Dict(ast::ExprDict { items, .. }) => { self.compile_dict(items)?; } - Expr::Slice(ExprSlice { + ast::Expr::Slice(ast::ExprSlice { lower, upper, step, .. }) => { - let mut compile_bound = |bound: Option<&Expr>| match bound { + let mut compile_bound = |bound: Option<&ast::Expr>| match bound { Some(exp) => self.compile_expression(exp), None => { self.emit_load_const(ConstantData::None); @@ -4591,7 +6926,7 @@ impl Compiler { }; emit!(self, Instruction::BuildSlice { argc }); } - Expr::Yield(ExprYield { value, .. }) => { + ast::Expr::Yield(ast::ExprYield { value, .. }) => { if !self.ctx.in_func() { return Err(self.error(CodegenErrorType::InvalidYield)); } @@ -4600,30 +6935,25 @@ impl Compiler { Some(expression) => self.compile_expression(expression)?, Option::None => self.emit_load_const(ConstantData::None), }; - emit!(self, Instruction::YieldValue); + // arg=0: direct yield (wrapped for async generators) + emit!(self, Instruction::YieldValue { arg: 0 }); emit!( self, Instruction::Resume { - arg: bytecode::ResumeType::AfterYield as u32 + arg: u32::from(bytecode::ResumeType::AfterYield) } ); } - Expr::Await(ExprAwait { value, .. }) => { + ast::Expr::Await(ast::ExprAwait { value, .. }) => { if self.ctx.func != FunctionContext::AsyncFunction { return Err(self.error(CodegenErrorType::InvalidAwait)); } self.compile_expression(value)?; - emit!(self, Instruction::GetAwaitable); + emit!(self, Instruction::GetAwaitable { arg: 0 }); self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 - } - ); + self.compile_yield_from_sequence(true)?; } - Expr::YieldFrom(ExprYieldFrom { value, .. }) => { + ast::Expr::YieldFrom(ast::ExprYieldFrom { value, .. }) => { match self.ctx.func { FunctionContext::NoFunction => { return Err(self.error(CodegenErrorType::InvalidYieldFrom)); @@ -4635,21 +6965,15 @@ impl Compiler { } self.mark_generator(); self.compile_expression(value)?; - emit!(self, Instruction::GetIter); + emit!(self, Instruction::GetYieldFromIter); self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterYieldFrom as u32 - } - ); + self.compile_yield_from_sequence(false)?; } - Expr::Name(ExprName { id, .. }) => self.load_name(id.as_str())?, - Expr::Lambda(ExprLambda { + ast::Expr::Name(ast::ExprName { id, .. }) => self.load_name(id.as_str())?, + ast::Expr::Lambda(ast::ExprLambda { parameters, body, .. }) => { - let default_params = Parameters::default(); + let default_params = ast::Parameters::default(); let params = parameters.as_deref().unwrap_or(&default_params); validate_duplicate_params(params).map_err(|e| self.error(e))?; @@ -4657,7 +6981,7 @@ impl Compiler { let name = "<lambda>".to_owned(); // Prepare defaults before entering function - let defaults: Vec<_> = std::iter::empty() + let defaults: Vec<_> = core::iter::empty() .chain(&params.posonlyargs) .chain(&params.args) .filter_map(|x| x.default.as_deref()) @@ -4685,7 +7009,7 @@ impl Compiler { let default_kw_count = kw_with_defaults.len(); for (arg, default) in &kw_with_defaults { self.emit_load_const(ConstantData::Str { - value: arg.name.as_str().into(), + value: self.mangle(arg.name.as_str()).into_owned().into(), }); self.compile_expression(default)?; } @@ -4713,12 +7037,11 @@ impl Compiler { loop_data: Option::None, in_class: prev_ctx.in_class, func: FunctionContext::Function, + // Lambda is never async, so new scope is not async + in_async_scope: false, }; - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + // Lambda cannot have docstrings, so no None is added to co_consts self.compile_expression(body)?; self.emit_return_value(); @@ -4729,14 +7052,17 @@ impl Compiler { self.ctx = prev_ctx; } - Expr::ListComp(ExprListComp { + ast::Expr::ListComp(ast::ExprListComp { elt, generators, .. }) => { self.compile_comprehension( "<listcomp>", - Some(Instruction::BuildList { - size: OpArgMarker::marker(), - }), + Some( + Instruction::BuildList { + size: OpArgMarker::marker(), + } + .into(), + ), generators, &|compiler| { compiler.compile_comprehension_element(elt)?; @@ -4749,17 +7075,20 @@ impl Compiler { Ok(()) }, ComprehensionType::List, - Self::contains_await(elt), + Self::contains_await(elt) || Self::generators_contain_await(generators), )?; } - Expr::SetComp(ExprSetComp { + ast::Expr::SetComp(ast::ExprSetComp { elt, generators, .. }) => { self.compile_comprehension( "<setcomp>", - Some(Instruction::BuildSet { - size: OpArgMarker::marker(), - }), + Some( + Instruction::BuildSet { + size: OpArgMarker::marker(), + } + .into(), + ), generators, &|compiler| { compiler.compile_comprehension_element(elt)?; @@ -4772,10 +7101,10 @@ impl Compiler { Ok(()) }, ComprehensionType::Set, - Self::contains_await(elt), + Self::contains_await(elt) || Self::generators_contain_await(generators), )?; } - Expr::DictComp(ExprDictComp { + ast::Expr::DictComp(ast::ExprDictComp { key, value, generators, @@ -4783,9 +7112,12 @@ impl Compiler { }) => { self.compile_comprehension( "<dictcomp>", - Some(Instruction::BuildMap { - size: OpArgMarker::marker(), - }), + Some( + Instruction::BuildMap { + size: OpArgMarker::marker(), + } + .into(), + ), generators, &|compiler| { // changed evaluation order for Py38 named expression PEP 572 @@ -4802,35 +7134,46 @@ impl Compiler { Ok(()) }, ComprehensionType::Dict, - Self::contains_await(key) || Self::contains_await(value), + Self::contains_await(key) + || Self::contains_await(value) + || Self::generators_contain_await(generators), )?; } - Expr::Generator(ExprGenerator { + ast::Expr::Generator(ast::ExprGenerator { elt, generators, .. }) => { + // Check if element or generators contain async content + // This makes the generator expression into an async generator + let element_contains_await = + Self::contains_await(elt) || Self::generators_contain_await(generators); self.compile_comprehension( "<genexpr>", None, generators, &|compiler| { + // Compile the element expression + // Note: if element is an async comprehension, compile_expression + // already handles awaiting it, so we don't need to await again here compiler.compile_comprehension_element(elt)?; + compiler.mark_generator(); - emit!(compiler, Instruction::YieldValue); + // arg=0: direct yield (wrapped for async generators) + emit!(compiler, Instruction::YieldValue { arg: 0 }); emit!( compiler, Instruction::Resume { - arg: bytecode::ResumeType::AfterYield as u32 + arg: u32::from(bytecode::ResumeType::AfterYield) } ); - emit!(compiler, Instruction::Pop); + emit!(compiler, Instruction::PopTop); Ok(()) }, ComprehensionType::Generator, - Self::contains_await(elt), + element_contains_await, )?; } - Expr::Starred(ExprStarred { value, .. }) => { + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { if self.in_annotation { // In annotation context, starred expressions are allowed (PEP 646) // For now, just compile the inner value without wrapping with Unpack @@ -4840,7 +7183,7 @@ impl Compiler { return Err(self.error(CodegenErrorType::InvalidStarExpr)); } } - Expr::If(ExprIf { + ast::Expr::If(ast::ExprIf { test, body, orelse, .. }) => { let else_block = self.new_block(); @@ -4851,7 +7194,7 @@ impl Compiler { self.compile_expression(body)?; emit!( self, - Instruction::Jump { + PseudoInstruction::Jump { target: after_block, } ); @@ -4864,23 +7207,23 @@ impl Compiler { self.switch_to_block(after_block); } - Expr::Named(ExprNamed { + ast::Expr::Named(ast::ExprNamed { target, value, node_index: _, range: _, }) => { self.compile_expression(value)?; - emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::Copy { index: 1_u32 }); self.compile_store(target)?; } - Expr::FString(fstring) => { + ast::Expr::FString(fstring) => { self.compile_expr_fstring(fstring)?; } - Expr::TString(_) => { - return Err(self.error(CodegenErrorType::NotImplementedYet)); + ast::Expr::TString(tstring) => { + self.compile_expr_tstring(tstring)?; } - Expr::StringLiteral(string) => { + ast::Expr::StringLiteral(string) => { let value = string.value.to_str(); if value.contains(char::REPLACEMENT_CHARACTER) { let value = string @@ -4899,42 +7242,42 @@ impl Compiler { }); } } - Expr::BytesLiteral(bytes) => { + ast::Expr::BytesLiteral(bytes) => { let iter = bytes.value.iter().flat_map(|x| x.iter().copied()); let v: Vec<u8> = iter.collect(); self.emit_load_const(ConstantData::Bytes { value: v }); } - Expr::NumberLiteral(number) => match &number.value { - Number::Int(int) => { + ast::Expr::NumberLiteral(number) => match &number.value { + ast::Number::Int(int) => { let value = ruff_int_to_bigint(int).map_err(|e| self.error(e))?; self.emit_load_const(ConstantData::Integer { value }); } - Number::Float(float) => { + ast::Number::Float(float) => { self.emit_load_const(ConstantData::Float { value: *float }); } - Number::Complex { real, imag } => { + ast::Number::Complex { real, imag } => { self.emit_load_const(ConstantData::Complex { value: Complex::new(*real, *imag), }); } }, - Expr::BooleanLiteral(b) => { + ast::Expr::BooleanLiteral(b) => { self.emit_load_const(ConstantData::Boolean { value: b.value }); } - Expr::NoneLiteral(_) => { + ast::Expr::NoneLiteral(_) => { self.emit_load_const(ConstantData::None); } - Expr::EllipsisLiteral(_) => { + ast::Expr::EllipsisLiteral(_) => { self.emit_load_const(ConstantData::Ellipsis); } - Expr::IpyEscapeCommand(_) => { + ast::Expr::IpyEscapeCommand(_) => { panic!("unexpected ipy escape command"); } } Ok(()) } - fn compile_keywords(&mut self, keywords: &[Keyword]) -> CompileResult<()> { + fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> { let mut size = 0; let groupby = keywords.iter().chunk_by(|e| e.arg.is_none()); for (is_unpacking, sub_keywords) in &groupby { @@ -4959,153 +7302,222 @@ impl Compiler { } } if size > 1 { - emit!(self, Instruction::BuildMapForCall { size }); + // Merge all dicts: first dict is accumulator, merge rest into it + for _ in 1..size { + emit!(self, Instruction::DictMerge { index: 1 }); + } } Ok(()) } - fn compile_call(&mut self, func: &Expr, args: &Arguments) -> CompileResult<()> { - let method = if let Expr::Attribute(ExprAttribute { value, attr, .. }) = &func { - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::LoadMethod { idx }); - true + fn compile_call(&mut self, func: &ast::Expr, args: &ast::Arguments) -> CompileResult<()> { + // Save the call expression's source range so CALL instructions use the + // call start line, not the last argument's line. + let call_range = self.current_source_range; + + // Method call: obj → LOAD_ATTR_METHOD → [method, self_or_null] → args → CALL + // Regular call: func → PUSH_NULL → args → CALL + if let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) = &func { + // Check for super() method call optimization + if let Some(super_type) = self.can_optimize_super_call(value, attr.as_str()) { + // super().method() or super(cls, self).method() optimization + // Stack: [global_super, class, self] → LOAD_SUPER_METHOD → [method, self] + self.load_args_for_super(&super_type)?; + let idx = self.name(attr.as_str()); + match super_type { + SuperCallType::TwoArg { .. } => { + self.emit_load_super_method(idx); + } + SuperCallType::ZeroArg => { + self.emit_load_zero_super_method(idx); + } + } + self.codegen_call_helper(0, args, call_range)?; + } else { + // Normal method call: compile object, then LOAD_ATTR with method flag + // LOAD_ATTR(method=1) pushes [method, self_or_null] on stack + self.compile_expression(value)?; + let idx = self.name(attr.as_str()); + self.emit_load_attr_method(idx); + self.codegen_call_helper(0, args, call_range)?; + } } else { + // Regular call: push func, then NULL for self_or_null slot + // Stack layout: [func, NULL, args...] - same as method call [func, self, args...] self.compile_expression(func)?; - false - }; - let call = self.compile_call_inner(0, args)?; - if method { - self.compile_method_call(call) - } else { - self.compile_normal_call(call) + emit!(self, Instruction::PushNull); + self.codegen_call_helper(0, args, call_range)?; } Ok(()) } - fn compile_normal_call(&mut self, ty: CallType) { - match ty { - CallType::Positional { nargs } => { - emit!(self, Instruction::CallFunctionPositional { nargs }) - } - CallType::Keyword { nargs } => emit!(self, Instruction::CallFunctionKeyword { nargs }), - CallType::Ex { has_kwargs } => emit!(self, Instruction::CallFunctionEx { has_kwargs }), + /// Compile subkwargs: emit key-value pairs for BUILD_MAP + fn codegen_subkwargs( + &mut self, + keywords: &[ast::Keyword], + begin: usize, + end: usize, + ) -> CompileResult<()> { + let n = end - begin; + assert!(n > 0); + + // For large kwargs, use BUILD_MAP(0) + MAP_ADD to avoid stack overflow + let big = n * 2 > 8; // STACK_USE_GUIDELINE approximation + + if big { + emit!(self, Instruction::BuildMap { size: 0 }); } - } - fn compile_method_call(&mut self, ty: CallType) { - match ty { - CallType::Positional { nargs } => { - emit!(self, Instruction::CallMethodPositional { nargs }) + + for kw in &keywords[begin..end] { + // Key first, then value - this is critical! + self.emit_load_const(ConstantData::Str { + value: kw.arg.as_ref().unwrap().as_str().into(), + }); + self.compile_expression(&kw.value)?; + + if big { + emit!(self, Instruction::MapAdd { i: 0 }); } - CallType::Keyword { nargs } => emit!(self, Instruction::CallMethodKeyword { nargs }), - CallType::Ex { has_kwargs } => emit!(self, Instruction::CallMethodEx { has_kwargs }), } + + if !big { + emit!(self, Instruction::BuildMap { size: n.to_u32() }); + } + + Ok(()) } - fn compile_call_inner( + /// Compile call arguments and emit the appropriate CALL instruction. + /// `call_range` is the source range of the call expression, used to set + /// the correct line number on the CALL instruction. + fn codegen_call_helper( &mut self, additional_positional: u32, - arguments: &Arguments, - ) -> CompileResult<CallType> { - let count = u32::try_from(arguments.len()).unwrap() + additional_positional; + arguments: &ast::Arguments, + call_range: TextRange, + ) -> CompileResult<()> { + let nelts = arguments.args.len(); + let nkwelts = arguments.keywords.len(); - // Normal arguments: - let (size, unpack) = self.gather_elements(additional_positional, &arguments.args)?; + // Check if we have starred args or **kwargs + let has_starred = arguments + .args + .iter() + .any(|arg| matches!(arg, ast::Expr::Starred(_))); let has_double_star = arguments.keywords.iter().any(|k| k.arg.is_none()); - for keyword in &arguments.keywords { - if let Some(name) = &keyword.arg { - self.check_forbidden_name(name.as_str(), NameUsage::Store)?; - } - } + // Check if exceeds stack guideline + let too_big = nelts + nkwelts * 2 > 8; - let call = if unpack || has_double_star { - // Create a tuple with positional args: - if unpack { - emit!(self, Instruction::BuildTupleFromTuples { size }); - } else { - emit!(self, Instruction::BuildTuple { size }); + if !has_starred && !has_double_star && !too_big { + // Simple call path: no * or ** args + for arg in &arguments.args { + self.compile_expression(arg)?; } - // Create an optional map with kw-args: - let has_kwargs = !arguments.keywords.is_empty(); - if has_kwargs { - self.compile_keywords(&arguments.keywords)?; - } - CallType::Ex { has_kwargs } - } else if !arguments.keywords.is_empty() { - let mut kwarg_names = vec![]; - for keyword in &arguments.keywords { - if let Some(name) = &keyword.arg { + if nkwelts > 0 { + // Compile keyword values and build kwnames tuple + let mut kwarg_names = Vec::with_capacity(nkwelts); + for keyword in &arguments.keywords { kwarg_names.push(ConstantData::Str { - value: name.as_str().into(), + value: keyword.arg.as_ref().unwrap().as_str().into(), }); - } else { - // This means **kwargs! - panic!("name must be set"); + self.compile_expression(&keyword.value)?; } - self.compile_expression(&keyword.value)?; - } - self.emit_load_const(ConstantData::Tuple { - elements: kwarg_names, - }); - CallType::Keyword { nargs: count } + // Restore call expression range for kwnames and CALL_KW + self.set_source_range(call_range); + self.emit_load_const(ConstantData::Tuple { + elements: kwarg_names, + }); + + let nargs = additional_positional + nelts.to_u32() + nkwelts.to_u32(); + emit!(self, Instruction::CallKw { nargs }); + } else { + self.set_source_range(call_range); + let nargs = additional_positional + nelts.to_u32(); + emit!(self, Instruction::Call { nargs }); + } } else { - CallType::Positional { nargs: count } - }; + // ex_call path: has * or ** args - Ok(call) - } + // Compile positional arguments + if additional_positional == 0 + && nelts == 1 + && matches!(arguments.args[0], ast::Expr::Starred(_)) + { + // Single starred arg: pass value directly to CallFunctionEx. + // Runtime will convert to tuple and validate with function name. + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = &arguments.args[0] { + self.compile_expression(value)?; + } + } else { + // Use starunpack_helper to build a list, then convert to tuple + self.starunpack_helper( + &arguments.args, + additional_positional, + CollectionType::List, + )?; + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + } - // Given a vector of expr / star expr generate code which gives either - // a list of expressions on the stack, or a list of tuples. - fn gather_elements(&mut self, before: u32, elements: &[Expr]) -> CompileResult<(u32, bool)> { - // First determine if we have starred elements: - let has_stars = elements.iter().any(|e| matches!(e, Expr::Starred(_))); + // Compile keyword arguments + if nkwelts > 0 { + let mut have_dict = false; + let mut nseen = 0usize; + + for (i, keyword) in arguments.keywords.iter().enumerate() { + if keyword.arg.is_none() { + // **kwargs unpacking + if nseen > 0 { + // Pack up preceding keywords using codegen_subkwargs + self.codegen_subkwargs(&arguments.keywords, i - nseen, i)?; + if have_dict { + emit!(self, Instruction::DictMerge { index: 1 }); + } + have_dict = true; + nseen = 0; + } - let size = if has_stars { - let mut size = 0; - let mut iter = elements.iter().peekable(); - let mut run_size = before; + if !have_dict { + emit!(self, Instruction::BuildMap { size: 0 }); + have_dict = true; + } - loop { - if iter.peek().is_none_or(|e| matches!(e, Expr::Starred(_))) { - emit!(self, Instruction::BuildTuple { size: run_size }); - run_size = 0; - size += 1; + self.compile_expression(&keyword.value)?; + emit!(self, Instruction::DictMerge { index: 1 }); + } else { + nseen += 1; + } } - match iter.next() { - Some(Expr::Starred(ExprStarred { value, .. })) => { - self.compile_expression(value)?; - // We need to collect each unpacked element into a - // tuple, since any side-effects during the conversion - // should be made visible before evaluating remaining - // expressions. - emit!(self, Instruction::BuildTupleFromIter); - size += 1; - } - Some(element) => { - self.compile_expression(element)?; - run_size += 1; + // Pack up any trailing keyword arguments + if nseen > 0 { + self.codegen_subkwargs(&arguments.keywords, nkwelts - nseen, nkwelts)?; + if have_dict { + emit!(self, Instruction::DictMerge { index: 1 }); } - None => break, + have_dict = true; } - } - size - } else { - for element in elements { - self.compile_expression(element)?; + assert!(have_dict); + } else { + emit!(self, Instruction::PushNull); } - before + elements.len().to_u32() - }; - Ok((size, has_stars)) + self.set_source_range(call_range); + emit!(self, Instruction::CallFunctionEx); + } + + Ok(()) } - fn compile_comprehension_element(&mut self, element: &Expr) -> CompileResult<()> { + fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> { self.compile_expression(element).map_err(|e| { if let CodegenErrorType::InvalidStarExpr = e.error { self.error(CodegenErrorType::SyntaxError( @@ -5120,8 +7532,8 @@ impl Compiler { fn compile_comprehension( &mut self, name: &str, - init_collection: Option<Instruction>, - generators: &[Comprehension], + init_collection: Option<AnyInstruction>, + generators: &[ast::Comprehension], compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, comprehension_type: ComprehensionType, element_contains_await: bool, @@ -5129,30 +7541,47 @@ impl Compiler { let prev_ctx = self.ctx; let has_an_async_gen = generators.iter().any(|g| g.is_async); - // async comprehensions are allowed in various contexts: - // - list/set/dict comprehensions in async functions - // - always for generator expressions - // Note: generators have to be treated specially since their async version is a fundamentally - // different type (aiter vs iter) instead of just an awaitable. - - // for if it actually is async, we check if any generator is async or if the element contains await + // Check for async comprehension outside async function (list/set/dict only, not generator expressions) + // Use in_async_scope to allow nested async comprehensions inside an async function + if comprehension_type != ComprehensionType::Generator + && (has_an_async_gen || element_contains_await) + && !prev_ctx.in_async_scope + { + return Err(self.error(CodegenErrorType::InvalidAsyncComprehension)); + } - // if the element expression contains await, but the context doesn't allow for async, - // then we continue on here with is_async=false and will produce a syntax once the await is hit + // Check if this comprehension should be inlined (PEP 709) + let is_inlined = self.is_inlined_comprehension_context(comprehension_type); + // async comprehensions are allowed in various contexts: + // - list/set/dict comprehensions in async functions (or nested within) + // - always for generator expressions let is_async_list_set_dict_comprehension = comprehension_type != ComprehensionType::Generator - && (has_an_async_gen || element_contains_await) // does it have to be async? (uses await or async for) - && prev_ctx.func == FunctionContext::AsyncFunction; // is it allowed to be async? (in an async function) + && (has_an_async_gen || element_contains_await) + && prev_ctx.in_async_scope; let is_async_generator_comprehension = comprehension_type == ComprehensionType::Generator && (has_an_async_gen || element_contains_await); - // since one is for generators, and one for not generators, they should never both be true debug_assert!(!(is_async_list_set_dict_comprehension && is_async_generator_comprehension)); let is_async = is_async_list_set_dict_comprehension || is_async_generator_comprehension; + // We must have at least one generator: + assert!(!generators.is_empty()); + + if is_inlined { + // PEP 709: Inlined comprehension - compile inline without new scope + return self.compile_inlined_comprehension( + init_collection, + generators, + compile_element, + has_an_async_gen, + ); + } + + // Non-inlined path: create a new code object (generator expressions, etc.) self.ctx = CompileContext { loop_data: None, in_class: prev_ctx.in_class, @@ -5161,50 +7590,256 @@ impl Compiler { } else { FunctionContext::Function }, + // Inherit in_async_scope from parent - nested async comprehensions are allowed + // if we're anywhere inside an async function + in_async_scope: prev_ctx.in_async_scope || is_async, }; - // We must have at least one generator: - assert!(!generators.is_empty()); - - let flags = bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED; + let flags = bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED; let flags = if is_async { - flags | bytecode::CodeFlags::IS_COROUTINE + flags | bytecode::CodeFlags::COROUTINE } else { flags }; - // Create magnificent function <listcomp>: - self.push_output(flags, 1, 1, 0, name.to_owned()); + // Create magnificent function <listcomp>: + self.push_output(flags, 1, 1, 0, name.to_owned())?; + + // Mark that we're in an inlined comprehension + self.current_code_info().in_inlined_comp = true; + + // Set qualname for comprehension + self.set_qualname(); + + let arg0 = self.varname(".0")?; + + let return_none = init_collection.is_none(); + // Create empty object of proper type: + if let Some(init_collection) = init_collection { + self._emit(init_collection, OpArg::new(0), BlockIdx::NULL) + } + + let mut loop_labels = vec![]; + for generator in generators { + let loop_block = self.new_block(); + let after_block = self.new_block(); + + if loop_labels.is_empty() { + // Load iterator onto stack (passed as first argument): + emit!(self, Instruction::LoadFast(arg0)); + } else { + // Evaluate iterated item: + self.compile_expression(&generator.iter)?; + + // Get iterator / turn item into an iterator + if generator.is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + } + } + + loop_labels.push((loop_block, after_block, generator.is_async)); + self.switch_to_block(loop_block); + if generator.is_async { + emit!( + self, + PseudoInstruction::SetupFinally { + target: after_block + } + ); + emit!(self, Instruction::GetANext); + self.push_fblock( + FBlockType::AsyncComprehensionGenerator, + loop_block, + after_block, + )?; + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + // POP_BLOCK before store: only __anext__/yield_from are + // protected by SetupFinally targeting END_ASYNC_FOR. + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::AsyncComprehensionGenerator); + self.compile_store(&generator.target)?; + } else { + emit!( + self, + Instruction::ForIter { + target: after_block, + } + ); + self.compile_store(&generator.target)?; + } + + // Now evaluate the ifs: + for if_condition in &generator.ifs { + self.compile_jump_if(if_condition, false, loop_block)? + } + } + + compile_element(self)?; + + for (loop_block, after_block, is_async) in loop_labels.iter().rev().copied() { + emit!(self, PseudoInstruction::Jump { target: loop_block }); + + self.switch_to_block(after_block); + if is_async { + // EndAsyncFor pops both the exception and the aiter + // (handler depth is before GetANext, so aiter is at handler depth) + emit!(self, Instruction::EndAsyncFor); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + } + + if return_none { + self.emit_load_const(ConstantData::None) + } + + self.emit_return_value(); + + let code = self.exit_scope(); + + self.ctx = prev_ctx; + + // Create comprehension function with closure + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; + emit!(self, Instruction::PushNull); + + // Evaluate iterated item: + self.compile_expression(&generators[0].iter)?; + + // Get iterator / turn item into an iterator + // Use is_async from the first generator, not has_an_async_gen which covers ALL generators + if generators[0].is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + }; + + // Call just created <listcomp> function: + emit!(self, Instruction::Call { nargs: 1 }); + if is_async_list_set_dict_comprehension { + emit!(self, Instruction::GetAwaitable { arg: 0 }); + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + } + + Ok(()) + } + + /// Collect variable names from an assignment target expression + fn collect_target_names(&self, target: &ast::Expr, names: &mut Vec<String>) { + match target { + ast::Expr::Name(name) => { + let name_str = name.id.to_string(); + if !names.contains(&name_str) { + names.push(name_str); + } + } + ast::Expr::Tuple(tuple) => { + for elt in &tuple.elts { + self.collect_target_names(elt, names); + } + } + ast::Expr::List(list) => { + for elt in &list.elts { + self.collect_target_names(elt, names); + } + } + ast::Expr::Starred(starred) => { + self.collect_target_names(&starred.value, names); + } + _ => { + // Other targets (attribute, subscript) don't bind local names + } + } + } + + /// Compile an inlined comprehension (PEP 709) + /// This generates bytecode inline without creating a new code object + fn compile_inlined_comprehension( + &mut self, + init_collection: Option<AnyInstruction>, + generators: &[ast::Comprehension], + compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, + _has_an_async_gen: bool, + ) -> CompileResult<()> { + // PEP 709: Consume the comprehension's sub_table (but we won't use it as a separate scope) + // We need to consume it to keep sub_tables in sync with AST traversal order. + // The symbols are already merged into parent scope by analyze_symbol_table. + let _comp_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table") + .sub_tables + .remove(0); + + // Collect local variables that need to be saved/restored + // These are variables bound in the comprehension (iteration vars from targets) + let mut pushed_locals: Vec<String> = Vec::new(); + for generator in generators { + self.collect_target_names(&generator.target, &mut pushed_locals); + } + + // Step 1: Compile the outermost iterator + self.compile_expression(&generators[0].iter)?; + // Use is_async from the first generator, not has_an_async_gen which covers ALL generators + if generators[0].is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + } + + // Step 2: Save local variables that will be shadowed by the comprehension + for name in &pushed_locals { + let idx = self.varname(name)?; + emit!(self, Instruction::LoadFastAndClear(idx)); + } - // Mark that we're in an inlined comprehension - self.current_code_info().in_inlined_comp = true; + // Step 3: SWAP iterator to TOS (above saved locals) + if !pushed_locals.is_empty() { + emit!( + self, + Instruction::Swap { + index: u32::try_from(pushed_locals.len() + 1).unwrap() + } + ); + } - // Set qualname for comprehension - self.set_qualname(); + // Step 4: Create the collection (list/set/dict) + // For generator expressions, init_collection is None + if let Some(init_collection) = init_collection { + self._emit(init_collection, OpArg::new(0), BlockIdx::NULL); + // SWAP to get iterator on top + emit!(self, Instruction::Swap { index: 2 }); + } - let arg0 = self.varname(".0")?; + // Set up exception handler for cleanup on exception + let cleanup_block = self.new_block(); + let end_block = self.new_block(); - let return_none = init_collection.is_none(); - // Create empty object of proper type: - if let Some(init_collection) = init_collection { - self._emit(init_collection, OpArg(0), BlockIdx::NULL) + if !pushed_locals.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + target: cleanup_block + } + ); + self.push_fblock(FBlockType::TryExcept, cleanup_block, end_block)?; } + // Step 5: Compile the comprehension loop(s) let mut loop_labels = vec![]; - for generator in generators { + for (i, generator) in generators.iter().enumerate() { let loop_block = self.new_block(); let after_block = self.new_block(); - // emit!(self, Instruction::SetupLoop); - - if loop_labels.is_empty() { - // Load iterator onto stack (passed as first argument): - emit!(self, Instruction::LoadFast(arg0)); - } else { - // Evaluate iterated item: + if i > 0 { + // For nested loops, compile the iterator expression self.compile_expression(&generator.iter)?; - - // Get iterator / turn item into an iterator if generator.is_async { emit!(self, Instruction::GetAIter); } else { @@ -5212,26 +7847,14 @@ impl Compiler { } } - loop_labels.push((loop_block, after_block)); + loop_labels.push((loop_block, after_block, generator.is_async)); self.switch_to_block(loop_block); + if generator.is_async { - emit!( - self, - Instruction::SetupExcept { - handler: after_block, - } - ); emit!(self, Instruction::GetANext); self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 - } - ); + self.compile_yield_from_sequence(true)?; self.compile_store(&generator.target)?; - emit!(self, Instruction::PopBlock); } else { emit!( self, @@ -5242,71 +7865,88 @@ impl Compiler { self.compile_store(&generator.target)?; } - // Now evaluate the ifs: + // Evaluate the if conditions for if_condition in &generator.ifs { - self.compile_jump_if(if_condition, false, loop_block)? + self.compile_jump_if(if_condition, false, loop_block)?; } } + // Step 6: Compile the element expression and append to collection compile_element(self)?; - for (loop_block, after_block) in loop_labels.iter().rev().copied() { - // Repeat: - emit!(self, Instruction::Jump { target: loop_block }); - - // End of for loop: + // Step 7: Close all loops + for (loop_block, after_block, is_async) in loop_labels.iter().rev().copied() { + emit!(self, PseudoInstruction::Jump { target: loop_block }); self.switch_to_block(after_block); - if has_an_async_gen { + if is_async { emit!(self, Instruction::EndAsyncFor); + // Pop the iterator + emit!(self, Instruction::PopTop); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); } } - if return_none { - self.emit_load_const(ConstantData::None) - } - - // Return freshly filled list: - self.emit_return_value(); - - // Fetch code for listcomp function: - let code = self.exit_scope(); + // Step 8: Clean up - restore saved locals + if !pushed_locals.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); - self.ctx = prev_ctx; + // Normal path: jump past cleanup + emit!(self, PseudoInstruction::Jump { target: end_block }); - // Create comprehension function with closure - self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; + // Exception cleanup path + self.switch_to_block(cleanup_block); + // Stack: [saved_locals..., collection, exception] + // Swap to get collection out from under exception + emit!(self, Instruction::Swap { index: 2 }); + emit!(self, Instruction::PopTop); // Pop incomplete collection - // Evaluate iterated item: - self.compile_expression(&generators[0].iter)?; + // Restore locals + emit!( + self, + Instruction::Swap { + index: u32::try_from(pushed_locals.len() + 1).unwrap() + } + ); + for name in pushed_locals.iter().rev() { + let idx = self.varname(name)?; + emit!(self, Instruction::StoreFast(idx)); + } + // Re-raise the exception + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::ReraiseFromStack + } + ); - // Get iterator / turn item into an iterator - if has_an_async_gen { - emit!(self, Instruction::GetAIter); - } else { - emit!(self, Instruction::GetIter); - }; + // Normal end path + self.switch_to_block(end_block); + } - // Call just created <listcomp> function: - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); - if is_async_list_set_dict_comprehension { - // async, but not a generator and not an async for - // in this case, we end up with an awaitable - // that evaluates to the list/set/dict, so here we add an await - emit!(self, Instruction::GetAwaitable); - self.emit_load_const(ConstantData::None); - emit!(self, Instruction::YieldFrom); + // SWAP result to TOS (above saved locals) + if !pushed_locals.is_empty() { emit!( self, - Instruction::Resume { - arg: bytecode::ResumeType::AfterAwait as u32 + Instruction::Swap { + index: u32::try_from(pushed_locals.len() + 1).unwrap() } ); } + // Restore saved locals + for name in pushed_locals.iter().rev() { + let idx = self.varname(name)?; + emit!(self, Instruction::StoreFast(idx)); + } + Ok(()) } - fn compile_future_features(&mut self, features: &[Alias]) -> Result<(), CodegenError> { + fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> { if let DoneWithFuture::Yes = self.done_with_future_stmts { return Err(self.error(CodegenErrorType::InvalidFuturePlacement)); } @@ -5328,30 +7968,31 @@ impl Compiler { } // Low level helper functions: - fn _emit(&mut self, instr: Instruction, arg: OpArg, target: BlockIdx) { + fn _emit<I: Into<AnyInstruction>>(&mut self, instr: I, arg: OpArg, target: BlockIdx) { let range = self.current_source_range; - let location = self - .source_file - .to_source_code() - .source_location(range.start(), PositionEncoding::Utf8); - // TODO: insert source filename + let source = self.source_file.to_source_code(); + let location = source.source_location(range.start(), PositionEncoding::Utf8); + let end_location = source.source_location(range.end(), PositionEncoding::Utf8); + let except_handler = None; self.current_block().instructions.push(ir::InstructionInfo { - instr, + instr: instr.into(), arg, target, location, - // range, + end_location, + except_handler, + lineno_override: None, }); } - fn emit_no_arg(&mut self, ins: Instruction) { - self._emit(ins, OpArg::null(), BlockIdx::NULL) + fn emit_no_arg<I: Into<AnyInstruction>>(&mut self, ins: I) { + self._emit(ins, OpArg::NULL, BlockIdx::NULL) } - fn emit_arg<A: OpArgType, T: EmitArg<A>>( + fn emit_arg<A: OpArgType, T: EmitArg<A>, I: Into<AnyInstruction>>( &mut self, arg: T, - f: impl FnOnce(OpArgMarker<A>) -> Instruction, + f: impl FnOnce(OpArgMarker<A>) -> I, ) { let (op, arg, target) = arg.emit(f); self._emit(op, arg, target) @@ -5370,17 +8011,75 @@ impl Compiler { } fn emit_return_const(&mut self, constant: ConstantData) { - let idx = self.arg_constant(constant); - self.emit_arg(idx, |idx| Instruction::ReturnConst { idx }) + self.emit_load_const(constant); + emit!(self, Instruction::ReturnValue) + } + + /// Emit LOAD_ATTR for attribute access (method=false). + /// Encodes: (name_idx << 1) | 0 + fn emit_load_attr(&mut self, name_idx: u32) { + let encoded = LoadAttr::builder() + .name_idx(name_idx) + .is_method(false) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadAttr { idx: arg }) + } + + /// Emit LOAD_ATTR with method flag set (for method calls). + /// Encodes: (name_idx << 1) | 1 + fn emit_load_attr_method(&mut self, name_idx: u32) { + let encoded = LoadAttr::builder() + .name_idx(name_idx) + .is_method(true) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadAttr { idx: arg }) + } + + /// Emit LOAD_SUPER_ATTR for 2-arg super().attr access. + /// Encodes: (name_idx << 2) | 0b10 (method=0, class=1) + fn emit_load_super_attr(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(false) + .has_class(true) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadSuperAttr { arg }) + } + + /// Emit LOAD_SUPER_ATTR for 2-arg super().method() call. + /// Encodes: (name_idx << 2) | 0b11 (method=1, class=1) + fn emit_load_super_method(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(true) + .has_class(true) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadSuperAttr { arg }) + } + + /// Emit LOAD_SUPER_ATTR for 0-arg super().attr access. + /// Encodes: (name_idx << 2) | 0b00 (method=0, class=0) + fn emit_load_zero_super_attr(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(false) + .has_class(false) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadSuperAttr { arg }) + } + + /// Emit LOAD_SUPER_ATTR for 0-arg super().method() call. + /// Encodes: (name_idx << 2) | 0b01 (method=1, class=0) + fn emit_load_zero_super_method(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(true) + .has_class(false) + .build(); + self.emit_arg(encoded, |arg| Instruction::LoadSuperAttr { arg }) } fn emit_return_value(&mut self) { - if let Some(inst) = self.current_block().instructions.last_mut() - && let Instruction::LoadConst { idx } = inst.instr - { - inst.instr = Instruction::ReturnConst { idx }; - return; - } emit!(self, Instruction::ReturnValue) } @@ -5388,6 +8087,212 @@ impl Compiler { self.code_stack.last_mut().expect("no code on stack") } + /// Enter a conditional block (if/for/while/match/try/with) + /// PEP 649: Track conditional annotation context + fn enter_conditional_block(&mut self) { + self.current_code_info().in_conditional_block += 1; + } + + /// Leave a conditional block + fn leave_conditional_block(&mut self) { + let code_info = self.current_code_info(); + debug_assert!(code_info.in_conditional_block > 0); + code_info.in_conditional_block -= 1; + } + + /// Compile break or continue statement with proper fblock cleanup. + /// compiler_break, compiler_continue + /// This handles unwinding through With blocks and exception handlers. + fn compile_break_continue( + &mut self, + range: ruff_text_size::TextRange, + is_break: bool, + ) -> CompileResult<()> { + // unwind_fblock_stack + // We need to unwind fblocks and compile cleanup code. For FinallyTry blocks, + // we need to compile the finally body inline, but we must temporarily pop + // the fblock so that nested break/continue in the finally body don't see it. + + // First, find the loop + let code = self.current_code_info(); + let mut loop_idx = None; + let mut is_for_loop = false; + + for i in (0..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::WhileLoop => { + loop_idx = Some(i); + is_for_loop = false; + break; + } + FBlockType::ForLoop => { + loop_idx = Some(i); + is_for_loop = true; + break; + } + FBlockType::ExceptionGroupHandler => { + return Err( + self.error_ranged(CodegenErrorType::BreakContinueReturnInExceptStar, range) + ); + } + _ => {} + } + } + + let Some(loop_idx) = loop_idx else { + if is_break { + return Err(self.error_ranged(CodegenErrorType::InvalidBreak, range)); + } else { + return Err(self.error_ranged(CodegenErrorType::InvalidContinue, range)); + } + }; + + let loop_block = code.fblock[loop_idx].fb_block; + let exit_block = code.fblock[loop_idx].fb_exit; + + // Collect the fblocks we need to unwind through, from top down to (but not including) the loop + #[derive(Clone)] + enum UnwindAction { + With { + is_async: bool, + }, + HandlerCleanup { + name: Option<String>, + }, + TryExcept, + FinallyTry { + body: Vec<ruff_python_ast::Stmt>, + fblock_idx: usize, + }, + FinallyEnd, + PopValue, // Pop return value when continue/break cancels a return + } + let mut unwind_actions = Vec::new(); + + { + let code = self.current_code_info(); + for i in (loop_idx + 1..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::With => { + unwind_actions.push(UnwindAction::With { is_async: false }); + } + FBlockType::AsyncWith => { + unwind_actions.push(UnwindAction::With { is_async: true }); + } + FBlockType::HandlerCleanup => { + let name = match &code.fblock[i].fb_datum { + FBlockDatum::ExceptionName(name) => Some(name.clone()), + _ => None, + }; + unwind_actions.push(UnwindAction::HandlerCleanup { name }); + } + FBlockType::TryExcept => { + unwind_actions.push(UnwindAction::TryExcept); + } + FBlockType::FinallyTry => { + // Need to execute finally body before break/continue + if let FBlockDatum::FinallyBody(ref body) = code.fblock[i].fb_datum { + unwind_actions.push(UnwindAction::FinallyTry { + body: body.clone(), + fblock_idx: i, + }); + } + } + FBlockType::FinallyEnd => { + // Inside finally block reached via exception - need to pop exception + unwind_actions.push(UnwindAction::FinallyEnd); + } + FBlockType::PopValue => { + // Pop the return value that was saved on stack + unwind_actions.push(UnwindAction::PopValue); + } + _ => {} + } + } + } + + // Emit cleanup for each fblock + for action in unwind_actions { + match action { + UnwindAction::With { is_async } => { + // codegen_unwind_fblock(WITH/ASYNC_WITH) + emit!(self, PseudoInstruction::PopBlock); + // compiler_call_exit_with_nones + emit!(self, Instruction::PushNull); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::Call { nargs: 3 }); + + if is_async { + emit!(self, Instruction::GetAwaitable { arg: 2 }); + self.emit_load_const(ConstantData::None); + self.compile_yield_from_sequence(true)?; + } + + emit!(self, Instruction::PopTop); + } + UnwindAction::HandlerCleanup { ref name } => { + // codegen_unwind_fblock(HANDLER_CLEANUP) + if name.is_some() { + // Named handler: PopBlock for inner SETUP_CLEANUP + emit!(self, PseudoInstruction::PopBlock); + } + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + if let Some(name) = name { + self.emit_load_const(ConstantData::None); + self.store_name(name)?; + self.compile_name(name, NameUsage::Delete)?; + } + } + UnwindAction::TryExcept => { + // codegen_unwind_fblock(TRY_EXCEPT) + emit!(self, PseudoInstruction::PopBlock); + } + UnwindAction::FinallyTry { body, fblock_idx } => { + // codegen_unwind_fblock(FINALLY_TRY) + emit!(self, PseudoInstruction::PopBlock); + + // compile finally body inline + // Temporarily pop the FinallyTry fblock so nested break/continue + // in the finally body won't see it again. + let code = self.current_code_info(); + let saved_fblock = code.fblock.remove(fblock_idx); + + self.compile_statements(&body)?; + + // Restore the fblock (though this break/continue will jump away, + // this keeps the fblock stack consistent for error checking) + let code = self.current_code_info(); + code.fblock.insert(fblock_idx, saved_fblock); + } + UnwindAction::FinallyEnd => { + // codegen_unwind_fblock(FINALLY_END) + emit!(self, Instruction::PopTop); // exc_value + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + } + UnwindAction::PopValue => { + // Pop the return value - continue/break cancels the pending return + emit!(self, Instruction::PopTop); + } + } + } + + // For break in a for loop, pop the iterator + if is_break && is_for_loop { + emit!(self, Instruction::PopIter); + } + + // Jump to target + let target = if is_break { exit_block } else { loop_block }; + emit!(self, PseudoInstruction::Jump { target }); + + Ok(()) + } + fn current_block(&mut self) -> &mut ir::Block { let info = self.current_code_info(); &mut info.blocks[info.current_block] @@ -5430,142 +8335,68 @@ impl Compiler { } fn mark_generator(&mut self) { - self.current_code_info().flags |= bytecode::CodeFlags::IS_GENERATOR + self.current_code_info().flags |= bytecode::CodeFlags::GENERATOR } /// Whether the expression contains an await expression and /// thus requires the function to be async. - /// Async with and async for are statements, so I won't check for them here - fn contains_await(expression: &Expr) -> bool { - use ruff_python_ast::*; + /// + /// Both: + /// ```py + /// async with: ... + /// async for: ... + /// ``` + /// are statements, so we won't check for them here + fn contains_await(expression: &ast::Expr) -> bool { + use ast::visitor::Visitor; + + #[derive(Default)] + struct AwaitVisitor { + found: bool, + } + + impl ast::visitor::Visitor<'_> for AwaitVisitor { + fn visit_expr(&mut self, expr: &ast::Expr) { + if self.found { + return; + } - match &expression { - Expr::Call(ExprCall { - func, arguments, .. - }) => { - Self::contains_await(func) - || arguments.args.iter().any(Self::contains_await) - || arguments - .keywords - .iter() - .any(|kw| Self::contains_await(&kw.value)) - } - Expr::BoolOp(ExprBoolOp { values, .. }) => values.iter().any(Self::contains_await), - Expr::BinOp(ExprBinOp { left, right, .. }) => { - Self::contains_await(left) || Self::contains_await(right) - } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - Self::contains_await(value) || Self::contains_await(slice) - } - Expr::UnaryOp(ExprUnaryOp { operand, .. }) => Self::contains_await(operand), - Expr::Attribute(ExprAttribute { value, .. }) => Self::contains_await(value), - Expr::Compare(ExprCompare { - left, comparators, .. - }) => Self::contains_await(left) || comparators.iter().any(Self::contains_await), - Expr::List(ExprList { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Tuple(ExprTuple { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Set(ExprSet { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Dict(ExprDict { items, .. }) => items - .iter() - .flat_map(|item| &item.key) - .any(Self::contains_await), - Expr::Slice(ExprSlice { - lower, upper, step, .. - }) => { - lower.as_deref().is_some_and(Self::contains_await) - || upper.as_deref().is_some_and(Self::contains_await) - || step.as_deref().is_some_and(Self::contains_await) - } - Expr::Yield(ExprYield { value, .. }) => { - value.as_deref().is_some_and(Self::contains_await) - } - Expr::Await(ExprAwait { .. }) => true, - Expr::YieldFrom(ExprYieldFrom { value, .. }) => Self::contains_await(value), - Expr::Name(ExprName { .. }) => false, - Expr::Lambda(ExprLambda { body, .. }) => Self::contains_await(body), - Expr::ListComp(ExprListComp { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::SetComp(ExprSetComp { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::DictComp(ExprDictComp { - key, - value, - generators, - .. - }) => { - Self::contains_await(key) - || Self::contains_await(value) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::Generator(ExprGenerator { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::Starred(expr) => Self::contains_await(&expr.value), - Expr::If(ExprIf { - test, body, orelse, .. - }) => { - Self::contains_await(test) - || Self::contains_await(body) - || Self::contains_await(orelse) + match expr { + ast::Expr::Await(_) => self.found = true, + // Note: We do NOT check for async comprehensions here. + // Async list/set/dict comprehensions are handled by compile_comprehension + // which already awaits the result. A generator expression containing + // an async comprehension as its element does NOT become an async generator, + // because the async comprehension is awaited when evaluating the element. + _ => ast::visitor::walk_expr(self, expr), + } } - - Expr::Named(ExprNamed { - target, - value, - node_index: _, - range: _, - }) => Self::contains_await(target) || Self::contains_await(value), - Expr::FString(fstring) => { - Self::interpolated_string_contains_await(fstring.value.elements()) - } - Expr::TString(tstring) => { - Self::interpolated_string_contains_await(tstring.value.elements()) - } - Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::NumberLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) - | Expr::EllipsisLiteral(_) - | Expr::IpyEscapeCommand(_) => false, - } - } - - fn interpolated_string_contains_await<'a>( - mut elements: impl Iterator<Item = &'a InterpolatedStringElement>, - ) -> bool { - fn interpolated_element_contains_await<F: Copy + Fn(&Expr) -> bool>( - expr_element: &InterpolatedElement, - contains_await: F, - ) -> bool { - contains_await(&expr_element.expression) - || expr_element - .format_spec - .iter() - .flat_map(|spec| spec.elements.interpolations()) - .any(|element| interpolated_element_contains_await(element, contains_await)) } - elements.any(|element| match element { - InterpolatedStringElement::Interpolation(expr_element) => { - interpolated_element_contains_await(expr_element, Self::contains_await) + let mut visitor = AwaitVisitor::default(); + visitor.visit_expr(expression); + visitor.found + } + + /// Check if any of the generators (except the first one's iter) contains an await expression. + /// The first generator's iter is evaluated outside the comprehension scope. + fn generators_contain_await(generators: &[ast::Comprehension]) -> bool { + for (i, generator) in generators.iter().enumerate() { + // First generator's iter is evaluated outside the comprehension + if i > 0 && Self::contains_await(&generator.iter) { + return true; } - InterpolatedStringElement::Literal(_) => false, - }) + // Check ifs in all generators + for if_expr in &generator.ifs { + if Self::contains_await(if_expr) { + return true; + } + } + } + false } - fn compile_expr_fstring(&mut self, fstring: &ExprFString) -> CompileResult<()> { + fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { let fstring = &fstring.value; for part in fstring { self.compile_fstring_part(part)?; @@ -5582,9 +8413,9 @@ impl Compiler { Ok(()) } - fn compile_fstring_part(&mut self, part: &FStringPart) -> CompileResult<()> { + fn compile_fstring_part(&mut self, part: &ast::FStringPart) -> CompileResult<()> { match part { - FStringPart::Literal(string) => { + ast::FStringPart::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure let source = self.source_file.slice(string.range); @@ -5600,24 +8431,24 @@ impl Compiler { } Ok(()) } - FStringPart::FString(fstring) => self.compile_fstring(fstring), + ast::FStringPart::FString(fstring) => self.compile_fstring(fstring), } } - fn compile_fstring(&mut self, fstring: &FString) -> CompileResult<()> { + fn compile_fstring(&mut self, fstring: &ast::FString) -> CompileResult<()> { self.compile_fstring_elements(fstring.flags, &fstring.elements) } fn compile_fstring_elements( &mut self, - flags: FStringFlags, - fstring_elements: &InterpolatedStringElements, + flags: ast::FStringFlags, + fstring_elements: &ast::InterpolatedStringElements, ) -> CompileResult<()> { let mut element_count = 0; for element in fstring_elements { element_count += 1; match element { - InterpolatedStringElement::Literal(string) => { + ast::InterpolatedStringElement::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure let source = self.source_file.slice(string.range); @@ -5634,25 +8465,29 @@ impl Compiler { }); } } - InterpolatedStringElement::Interpolation(fstring_expr) => { + ast::InterpolatedStringElement::Interpolation(fstring_expr) => { let mut conversion = match fstring_expr.conversion { - ConversionFlag::None => ConvertValueOparg::None, - ConversionFlag::Str => ConvertValueOparg::Str, - ConversionFlag::Repr => ConvertValueOparg::Repr, - ConversionFlag::Ascii => ConvertValueOparg::Ascii, + ast::ConversionFlag::None => ConvertValueOparg::None, + ast::ConversionFlag::Str => ConvertValueOparg::Str, + ast::ConversionFlag::Repr => ConvertValueOparg::Repr, + ast::ConversionFlag::Ascii => ConvertValueOparg::Ascii, }; - if let Some(DebugText { leading, trailing }) = &fstring_expr.debug_text { + if let Some(ast::DebugText { leading, trailing }) = &fstring_expr.debug_text { let range = fstring_expr.expression.range(); let source = self.source_file.slice(range); - let text = [leading, source, trailing].concat(); + let text = [ + strip_fstring_debug_comments(leading).as_str(), + source, + strip_fstring_debug_comments(trailing).as_str(), + ] + .concat(); self.emit_load_const(ConstantData::Str { value: text.into() }); element_count += 1; - // Match CPython behavior: If debug text is present, apply repr conversion. - // if no `format_spec` specified. - // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 + // If debug text is present, apply repr conversion when no `format_spec` specified. + // See action_helpers.c: fstring_find_expr_replacement if matches!( (conversion, &fstring_expr.format_spec), (ConvertValueOparg::None, None) @@ -5702,26 +8537,149 @@ impl Compiler { Ok(()) } + + fn compile_expr_tstring(&mut self, expr_tstring: &ast::ExprTString) -> CompileResult<()> { + // ast::TStringValue can contain multiple ast::TString parts (implicit concatenation) + // Each ast::TString part should be compiled and the results merged into a single Template + let tstring_value = &expr_tstring.value; + + // Collect all strings and compile all interpolations + let mut all_strings: Vec<Wtf8Buf> = Vec::new(); + let mut current_string = Wtf8Buf::new(); + let mut interp_count: u32 = 0; + + for tstring in tstring_value.iter() { + self.compile_tstring_into( + tstring, + &mut all_strings, + &mut current_string, + &mut interp_count, + )?; + } + + // Add trailing string + all_strings.push(core::mem::take(&mut current_string)); + + // Now build the Template: + // Stack currently has all interpolations from compile_tstring_into calls + + // 1. Build interpolations tuple from the interpolations on the stack + emit!(self, Instruction::BuildTuple { size: interp_count }); + + // 2. Load all string parts + let string_count: u32 = all_strings + .len() + .try_into() + .expect("t-string string count overflowed"); + for s in &all_strings { + self.emit_load_const(ConstantData::Str { value: s.clone() }); + } + + // 3. Build strings tuple + emit!(self, Instruction::BuildTuple { size: string_count }); + + // 4. Swap so strings is below interpolations: [interps, strings] -> [strings, interps] + emit!(self, Instruction::Swap { index: 2 }); + + // 5. Build the Template + emit!(self, Instruction::BuildTemplate); + + Ok(()) + } + + fn compile_tstring_into( + &mut self, + tstring: &ast::TString, + strings: &mut Vec<Wtf8Buf>, + current_string: &mut Wtf8Buf, + interp_count: &mut u32, + ) -> CompileResult<()> { + for element in &tstring.elements { + match element { + ast::InterpolatedStringElement::Literal(lit) => { + // Accumulate literal parts into current_string + current_string.push_str(&lit.value); + } + ast::InterpolatedStringElement::Interpolation(interp) => { + // Finish current string segment + strings.push(core::mem::take(current_string)); + + // Compile the interpolation value + self.compile_expression(&interp.expression)?; + + // Load the expression source string, including any + // whitespace between '{' and the expression start + let expr_range = interp.expression.range(); + let expr_source = if interp.range.start() < expr_range.start() + && interp.range.end() >= expr_range.end() + { + let after_brace = interp.range.start() + TextSize::new(1); + self.source_file + .slice(TextRange::new(after_brace, expr_range.end())) + } else { + // Fallback for programmatically constructed ASTs with dummy ranges + self.source_file.slice(expr_range) + }; + self.emit_load_const(ConstantData::Str { + value: expr_source.to_string().into(), + }); + + // Determine conversion code + let conversion: u32 = match interp.conversion { + ast::ConversionFlag::None => 0, + ast::ConversionFlag::Str => 1, + ast::ConversionFlag::Repr => 2, + ast::ConversionFlag::Ascii => 3, + }; + + // Handle format_spec + let has_format_spec = interp.format_spec.is_some(); + if let Some(format_spec) = &interp.format_spec { + // Compile format_spec as a string using fstring element compilation + // Use default ast::FStringFlags since format_spec syntax is independent of t-string flags + self.compile_fstring_elements( + ast::FStringFlags::empty(), + &format_spec.elements, + )?; + } + + // Emit BUILD_INTERPOLATION + // oparg encoding: (conversion << 2) | has_format_spec + let oparg = (conversion << 2) | u32::from(has_format_spec); + emit!(self, Instruction::BuildInterpolation { oparg }); + + *interp_count += 1; + } + } + } + + Ok(()) + } } trait EmitArg<Arg: OpArgType> { - fn emit( + fn emit<I: Into<AnyInstruction>>( self, - f: impl FnOnce(OpArgMarker<Arg>) -> Instruction, - ) -> (Instruction, OpArg, BlockIdx); + f: impl FnOnce(OpArgMarker<Arg>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx); } + impl<T: OpArgType> EmitArg<T> for T { - fn emit(self, f: impl FnOnce(OpArgMarker<T>) -> Instruction) -> (Instruction, OpArg, BlockIdx) { + fn emit<I: Into<AnyInstruction>>( + self, + f: impl FnOnce(OpArgMarker<T>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx) { let (marker, arg) = OpArgMarker::new(self); - (f(marker), arg, BlockIdx::NULL) + (f(marker).into(), arg, BlockIdx::NULL) } } + impl EmitArg<bytecode::Label> for BlockIdx { - fn emit( + fn emit<I: Into<AnyInstruction>>( self, - f: impl FnOnce(OpArgMarker<bytecode::Label>) -> Instruction, - ) -> (Instruction, OpArg, BlockIdx) { - (f(OpArgMarker::marker()), OpArg::null(), self) + f: impl FnOnce(OpArgMarker<bytecode::Label>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx) { + (f(OpArgMarker::marker()).into(), OpArg::NULL, self) } } @@ -5790,12 +8748,12 @@ fn expandtabs(input: &str, tab_size: usize) -> String { expanded_str } -fn split_doc<'a>(body: &'a [Stmt], opts: &CompileOpts) -> (Option<String>, &'a [Stmt]) { - if let Some((Stmt::Expr(expr), body_rest)) = body.split_first() { +fn split_doc<'a>(body: &'a [ast::Stmt], opts: &CompileOpts) -> (Option<String>, &'a [ast::Stmt]) { + if let Some((ast::Stmt::Expr(expr), body_rest)) = body.split_first() { let doc_comment = match &*expr.value { - Expr::StringLiteral(value) => Some(&value.value), + ast::Expr::StringLiteral(value) => Some(&value.value), // f-strings are not allowed in Python doc comments. - Expr::FString(_) => None, + ast::Expr::FString(_) => None, _ => None, }; if let Some(doc) = doc_comment { @@ -5809,7 +8767,7 @@ fn split_doc<'a>(body: &'a [Stmt], opts: &CompileOpts) -> (Option<String>, &'a [ (None, body) } -pub fn ruff_int_to_bigint(int: &Int) -> Result<BigInt, CodegenErrorType> { +pub fn ruff_int_to_bigint(int: &ast::Int) -> Result<BigInt, CodegenErrorType> { if let Some(small) = int.as_u64() { Ok(BigInt::from(small)) } else { @@ -5819,7 +8777,7 @@ pub fn ruff_int_to_bigint(int: &Int) -> Result<BigInt, CodegenErrorType> { /// Converts a `ruff` ast integer into a `BigInt`. /// Unlike small integers, big integers may be stored in one of four possible radix representations. -fn parse_big_integer(int: &Int) -> Result<BigInt, CodegenErrorType> { +fn parse_big_integer(int: &ast::Int) -> Result<BigInt, CodegenErrorType> { // TODO: Improve ruff API // Can we avoid this copy? let s = format!("{int}"); @@ -5859,38 +8817,58 @@ impl ToU32 for usize { } } +/// Strip Python comments from f-string debug text (leading/trailing around `=`). +/// A comment starts with `#` and extends to the end of the line. +/// The newline character itself is preserved. +fn strip_fstring_debug_comments(text: &str) -> String { + let mut result = String::with_capacity(text.len()); + let mut in_comment = false; + for ch in text.chars() { + if in_comment { + if ch == '\n' { + in_comment = false; + result.push(ch); + } + } else if ch == '#' { + in_comment = true; + } else { + result.push(ch); + } + } + result +} + #[cfg(test)] mod ruff_tests { use super::*; - use ruff_python_ast::name::Name; - use ruff_python_ast::*; + use ast::name::Name; /// Test if the compiler can correctly identify fstrings containing an `await` expression. #[test] fn test_fstring_contains_await() { let range = TextRange::default(); - let flags = FStringFlags::empty(); + let flags = ast::FStringFlags::empty(); // f'{x}' - let expr_x = Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, + let expr_x = ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, range, id: Name::new("x"), - ctx: ExprContext::Load, + ctx: ast::ExprContext::Load, }); - let not_present = &Expr::FString(ExprFString { - node_index: AtomicNodeIndex::NONE, + let not_present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, range, - value: FStringValue::single(FString { - node_index: AtomicNodeIndex::NONE, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, range, - elements: vec![InterpolatedStringElement::Interpolation( - InterpolatedElement { - node_index: AtomicNodeIndex::NONE, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, range, expression: Box::new(expr_x), debug_text: None, - conversion: ConversionFlag::None, + conversion: ast::ConversionFlag::None, format_spec: None, }, )] @@ -5901,29 +8879,29 @@ mod ruff_tests { assert!(!Compiler::contains_await(not_present)); // f'{await x}' - let expr_await_x = Expr::Await(ExprAwait { - node_index: AtomicNodeIndex::NONE, + let expr_await_x = ast::Expr::Await(ast::ExprAwait { + node_index: ast::AtomicNodeIndex::NONE, range, - value: Box::new(Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, + value: Box::new(ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, range, id: Name::new("x"), - ctx: ExprContext::Load, + ctx: ast::ExprContext::Load, })), }); - let present = &Expr::FString(ExprFString { - node_index: AtomicNodeIndex::NONE, + let present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, range, - value: FStringValue::single(FString { - node_index: AtomicNodeIndex::NONE, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, range, - elements: vec![InterpolatedStringElement::Interpolation( - InterpolatedElement { - node_index: AtomicNodeIndex::NONE, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, range, expression: Box::new(expr_await_x), debug_text: None, - conversion: ConversionFlag::None, + conversion: ast::ConversionFlag::None, format_spec: None, }, )] @@ -5934,45 +8912,45 @@ mod ruff_tests { assert!(Compiler::contains_await(present)); // f'{x:{await y}}' - let expr_x = Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, + let expr_x = ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, range, id: Name::new("x"), - ctx: ExprContext::Load, + ctx: ast::ExprContext::Load, }); - let expr_await_y = Expr::Await(ExprAwait { - node_index: AtomicNodeIndex::NONE, + let expr_await_y = ast::Expr::Await(ast::ExprAwait { + node_index: ast::AtomicNodeIndex::NONE, range, - value: Box::new(Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, + value: Box::new(ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, range, id: Name::new("y"), - ctx: ExprContext::Load, + ctx: ast::ExprContext::Load, })), }); - let present = &Expr::FString(ExprFString { - node_index: AtomicNodeIndex::NONE, + let present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, range, - value: FStringValue::single(FString { - node_index: AtomicNodeIndex::NONE, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, range, - elements: vec![InterpolatedStringElement::Interpolation( - InterpolatedElement { - node_index: AtomicNodeIndex::NONE, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, range, expression: Box::new(expr_x), debug_text: None, - conversion: ConversionFlag::None, - format_spec: Some(Box::new(InterpolatedStringFormatSpec { - node_index: AtomicNodeIndex::NONE, + conversion: ast::ConversionFlag::None, + format_spec: Some(Box::new(ast::InterpolatedStringFormatSpec { + node_index: ast::AtomicNodeIndex::NONE, range, - elements: vec![InterpolatedStringElement::Interpolation( - InterpolatedElement { - node_index: AtomicNodeIndex::NONE, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, range, expression: Box::new(expr_await_y), debug_text: None, - conversion: ConversionFlag::None, + conversion: ast::ConversionFlag::None, format_spec: None, }, )] @@ -6054,19 +9032,29 @@ if (True and False) or (False and True): )); } + #[test] + fn test_nested_bool_op() { + assert_dis_snapshot!(compile_exec( + "\ +x = Test() and False or False +" + )); + } + #[test] fn test_nested_double_async_with() { assert_dis_snapshot!(compile_exec( "\ -for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): - with self.subTest(type=type(stop_exc)): - try: - async with egg(): - raise stop_exc - except Exception as ex: - self.assertIs(ex, stop_exc) - else: - self.fail(f'{stop_exc} was suppressed') +async def test(): + for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): + with self.subTest(type=type(stop_exc)): + try: + async with egg(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail(f'{stop_exc} was suppressed') " )); } diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index a0e36bf29ae..086f9dfd739 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -1,8 +1,9 @@ +use alloc::fmt; +use core::fmt::Display; use rustpython_compiler_core::SourceLocation; -use std::fmt::{self, Display}; use thiserror::Error; -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub enum PatternUnreachableReason { NameCapture, Wildcard, @@ -75,6 +76,9 @@ pub enum CodegenErrorType { InvalidYield, InvalidYieldFrom, InvalidAwait, + InvalidAsyncFor, + InvalidAsyncWith, + InvalidAsyncComprehension, AsyncYieldFrom, AsyncReturnValue, InvalidFuturePlacement, @@ -88,10 +92,12 @@ pub enum CodegenErrorType { UnreachablePattern(PatternUnreachableReason), RepeatedAttributePattern, ConflictingNameBindPattern, + /// break/continue/return inside except* block + BreakContinueReturnInExceptStar, NotImplementedYet, // RustPython marker for unimplemented features } -impl std::error::Error for CodegenErrorType {} +impl core::error::Error for CodegenErrorType {} impl fmt::Display for CodegenErrorType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -101,15 +107,23 @@ impl fmt::Display for CodegenErrorType { Delete(target) => write!(f, "cannot delete {target}"), SyntaxError(err) => write!(f, "{}", err.as_str()), MultipleStarArgs => { - write!(f, "two starred expressions in assignment") + write!(f, "multiple starred expressions in assignment") } - InvalidStarExpr => write!(f, "cannot use starred expression here"), + InvalidStarExpr => write!(f, "can't use starred expression here"), InvalidBreak => write!(f, "'break' outside loop"), InvalidContinue => write!(f, "'continue' outside loop"), InvalidReturn => write!(f, "'return' outside function"), InvalidYield => write!(f, "'yield' outside function"), InvalidYieldFrom => write!(f, "'yield from' outside function"), InvalidAwait => write!(f, "'await' outside async function"), + InvalidAsyncFor => write!(f, "'async for' outside async function"), + InvalidAsyncWith => write!(f, "'async with' outside async function"), + InvalidAsyncComprehension => { + write!( + f, + "asynchronous comprehension outside of an asynchronous function" + ) + } AsyncYieldFrom => write!(f, "'yield from' inside async function"), AsyncReturnValue => { write!(f, "'return' with value inside async generator") @@ -148,6 +162,12 @@ impl fmt::Display for CodegenErrorType { ConflictingNameBindPattern => { write!(f, "alternative patterns bind different names") } + BreakContinueReturnInExceptStar => { + write!( + f, + "'break', 'continue' and 'return' cannot appear in an except* block" + ) + } NotImplementedYet => { write!(f, "RustPython does not implement this feature yet") } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index de0126f1122..e44c7223f2c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1,14 +1,28 @@ -use std::ops; +use core::ops; use crate::{IndexMap, IndexSet, error::InternalError}; +use malachite_bigint::BigInt; +use num_traits::ToPrimitive; + use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ - CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, InstrDisplayContext, Instruction, - Label, OpArg, PyCodeLocationInfoKind, + AnyInstruction, Arg, CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, + ExceptionTableEntry, InstrDisplayContext, Instruction, InstructionMetadata, Label, OpArg, + PseudoInstruction, PyCodeLocationInfoKind, encode_exception_table, }, + varint::{write_signed_varint, write_varint}, }; +/// Location info for linetable generation (allows line 0 for RESUME) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LineTableLocation { + line: i32, + end_line: i32, + col: i32, + end_col: i32, +} + /// Metadata for a code unit // = _PyCompile_CodeUnitMetadata #[derive(Clone, Debug)] @@ -81,14 +95,27 @@ impl ops::IndexMut<BlockIdx> for Vec<Block> { } } -#[derive(Debug, Clone)] +#[derive(Clone, Copy, Debug)] pub struct InstructionInfo { - pub instr: Instruction, + pub instr: AnyInstruction, pub arg: OpArg, pub target: BlockIdx, - // pub range: TextRange, pub location: SourceLocation, - // TODO: end_location for debug ranges + pub end_location: SourceLocation, + pub except_handler: Option<ExceptHandlerInfo>, + /// Override line number for linetable (e.g., line 0 for module RESUME) + pub lineno_override: Option<i32>, +} + +/// Exception handler information for an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExceptHandlerInfo { + /// Block to jump to when exception occurs + pub handler_block: BlockIdx, + /// Stack depth at handler entry + pub stack_depth: u32, + /// Whether to push lasti before exception + pub preserve_lasti: bool, } // spell-checker:ignore petgraph @@ -98,6 +125,13 @@ pub struct InstructionInfo { pub struct Block { pub instructions: Vec<InstructionInfo>, pub next: BlockIdx, + // Post-codegen analysis fields (set by label_exception_targets) + /// Whether this block is an exception handler target (b_except_handler) + pub except_handler: bool, + /// Whether to preserve lasti for this handler block (b_preserve_lasti) + pub preserve_lasti: bool, + /// Stack depth at block entry, set by stack depth analysis + pub start_depth: Option<u32>, } impl Default for Block { @@ -105,6 +139,9 @@ impl Default for Block { Self { instructions: Vec::new(), next: BlockIdx::NULL, + except_handler: false, + preserve_lasti: false, + start_depth: None, } } } @@ -130,14 +167,45 @@ pub struct CodeInfo { // Reference to the symbol table for this scope pub symbol_table_index: usize, + + // PEP 649: Track nesting depth inside conditional blocks (if/for/while/etc.) + // u_in_conditional_block + pub in_conditional_block: u32, + + // PEP 649: Next index for conditional annotation tracking + // u_next_conditional_annotation_index + pub next_conditional_annotation_index: u32, } impl CodeInfo { - pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult<CodeObject> { - if optimize > 0 { + pub fn finalize_code( + mut self, + opts: &crate::compile::CompileOpts, + ) -> crate::InternalResult<CodeObject> { + // Always fold tuple constants + self.fold_tuple_constants(); + // Python only applies LOAD_SMALL_INT conversion to module-level code + // (not inside functions). Module code lacks OPTIMIZED flag. + // Note: RustPython incorrectly sets NEWLOCALS on modules, so only check OPTIMIZED + let is_module_level = !self.flags.contains(CodeFlags::OPTIMIZED); + if is_module_level { + self.convert_to_load_small_int(); + } + self.remove_unused_consts(); + self.remove_nops(); + + if opts.optimize > 0 { self.dce(); + self.peephole_optimize(); } + // Always apply LOAD_FAST_BORROW optimization + self.optimize_load_fast_borrow(); + + // Post-codegen CFG analysis passes (flowgraph.c pipeline) + mark_except_handlers(&mut self.blocks); + label_exception_targets(&mut self.blocks); + let max_stackdepth = self.max_stackdepth()?; let cell2arg = self.cell2arg(); @@ -153,6 +221,8 @@ impl CodeInfo { in_inlined_comp: _, fblock: _, symbol_table_index: _, + in_conditional_block: _, + next_conditional_annotation_index: _, } = self; let CodeUnitMetadata { @@ -172,14 +242,33 @@ impl CodeInfo { let mut instructions = Vec::new(); let mut locations = Vec::new(); + let mut linetable_locations: Vec<LineTableLocation> = Vec::new(); + + // Convert pseudo ops and remove resulting NOPs + convert_pseudo_ops(&mut blocks, varname_cache.len() as u32); + for block in blocks + .iter_mut() + .filter(|b| b.next != BlockIdx::NULL || !b.instructions.is_empty()) + { + block + .instructions + .retain(|ins| !matches!(ins.instr.real(), Some(Instruction::Nop))); + } let mut block_to_offset = vec![Label(0); blocks.len()]; + // block_to_index: maps block idx to instruction index (for exception table) + // This is the index into the final instructions array, including EXTENDED_ARG + let mut block_to_index = vec![0u32; blocks.len()]; loop { let mut num_instructions = 0; for (idx, block) in iter_blocks(&blocks) { block_to_offset[idx.idx()] = Label(num_instructions as u32); + // block_to_index uses the same value as block_to_offset but as u32 + // because lasti in frame.rs is the index into instructions array + // and instructions array index == byte offset (each instruction is 1 CodeUnit) + block_to_index[idx.idx()] = num_instructions as u32; for instr in &block.instructions { - num_instructions += instr.arg.instr_size() + num_instructions += instr.arg.instr_size(); } } @@ -190,20 +279,64 @@ impl CodeInfo { let mut next_block = BlockIdx(0); while next_block != BlockIdx::NULL { let block = &mut blocks[next_block]; + // Track current instruction offset for jump direction resolution + let mut current_offset = block_to_offset[next_block.idx()].0; for info in &mut block.instructions { - let (op, arg, target) = (info.instr, &mut info.arg, info.target); + let target = info.target; if target != BlockIdx::NULL { - let new_arg = OpArg(block_to_offset[target.idx()].0); - recompile_extended_arg |= new_arg.instr_size() != arg.instr_size(); - *arg = new_arg; + let new_arg = OpArg::new(block_to_offset[target.idx()].0); + recompile_extended_arg |= new_arg.instr_size() != info.arg.instr_size(); + info.arg = new_arg; } - let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat_n(info.location, arg.instr_size())); + + // Convert JUMP pseudo to real instructions (direction depends on offset) + let op = match info.instr { + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) + if target != BlockIdx::NULL => + { + let target_offset = block_to_offset[target.idx()].0; + if target_offset > current_offset { + Instruction::JumpForward { + target: Arg::marker(), + } + } else { + Instruction::JumpBackward { + target: Arg::marker(), + } + } + } + AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) + if target != BlockIdx::NULL => + { + // JumpNoInterrupt is always backward (used in yield-from/await loops) + Instruction::JumpBackwardNoInterrupt { + target: Arg::marker(), + } + } + other => other.expect_real(), + }; + + let (extras, lo_arg) = info.arg.split(); + locations.extend(core::iter::repeat_n( + (info.location, info.end_location), + info.arg.instr_size(), + )); + // Collect linetable locations with lineno_override support + let lt_loc = LineTableLocation { + line: info + .lineno_override + .unwrap_or_else(|| info.location.line.get() as i32), + end_line: info.end_location.line.get() as i32, + col: info.location.character_offset.to_zero_indexed() as i32, + end_col: info.end_location.character_offset.to_zero_indexed() as i32, + }; + linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) .chain([CodeUnit { op, arg: lo_arg }]), ); + current_offset += info.arg.instr_size() as u32; } next_block = block.next; } @@ -213,11 +346,19 @@ impl CodeInfo { } instructions.clear(); - locations.clear() + locations.clear(); + linetable_locations.clear(); } - // Generate linetable from locations - let linetable = generate_linetable(&locations, first_line_number.get() as i32); + // Generate linetable from linetable_locations (supports line 0 for RESUME) + let linetable = generate_linetable( + &linetable_locations, + first_line_number.get() as i32, + opts.debug_ranges, + ); + + // Generate exception table before moving source_path + let exceptiontable = generate_exception_table(&blocks, &block_to_index); Ok(CodeObject { flags, @@ -239,7 +380,7 @@ impl CodeInfo { freevars: freevar_cache.into_iter().collect(), cell2arg, linetable, - exceptiontable: Box::new([]), // TODO: Generate actual exception table + exceptiontable, }) } @@ -250,8 +391,8 @@ impl CodeInfo { let total_args = self.metadata.argcount + self.metadata.kwonlyargcount - + self.flags.contains(CodeFlags::HAS_VARARGS) as u32 - + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32; + + self.flags.contains(CodeFlags::VARARGS) as u32 + + self.flags.contains(CodeFlags::VARKEYWORDS) as u32; let mut found_cellarg = false; let cell2arg = self @@ -278,7 +419,7 @@ impl CodeInfo { for block in &mut self.blocks { let mut last_instr = None; for (i, ins) in block.instructions.iter().enumerate() { - if ins.instr.unconditional_branch() { + if ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump() { last_instr = Some(i); break; } @@ -289,27 +430,378 @@ impl CodeInfo { } } - fn max_stackdepth(&self) -> crate::InternalResult<u32> { + /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + BUILD_TUPLE into LOAD_CONST tuple + /// fold_tuple_of_constants + fn fold_tuple_constants(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i < block.instructions.len() { + let instr = &block.instructions[i]; + // Look for BUILD_TUPLE + let Some(Instruction::BuildTuple { .. }) = instr.instr.real() else { + i += 1; + continue; + }; + + let tuple_size = u32::from(instr.arg) as usize; + if tuple_size == 0 || i < tuple_size { + i += 1; + continue; + } + + // Check if all preceding instructions are constant-loading + let start_idx = i - tuple_size; + let mut elements = Vec::with_capacity(tuple_size); + let mut all_const = true; + + for j in start_idx..i { + let load_instr = &block.instructions[j]; + match load_instr.instr.real() { + Some(Instruction::LoadConst { .. }) => { + let const_idx = u32::from(load_instr.arg) as usize; + if let Some(constant) = + self.metadata.consts.get_index(const_idx).cloned() + { + elements.push(constant); + } else { + all_const = false; + break; + } + } + Some(Instruction::LoadSmallInt { .. }) => { + // arg is the i32 value stored as u32 (two's complement) + let value = u32::from(load_instr.arg) as i32; + elements.push(ConstantData::Integer { + value: BigInt::from(value), + }); + } + _ => { + all_const = false; + break; + } + } + } + + if !all_const { + i += 1; + continue; + } + + // Note: The first small int is added to co_consts during compilation + // (in compile_default_arguments). + // We don't need to add it here again. + + // Create tuple constant and add to consts + let tuple_const = ConstantData::Tuple { elements }; + let (const_idx, _) = self.metadata.consts.insert_full(tuple_const); + + // Replace preceding LOAD instructions with NOP + for j in start_idx..i { + block.instructions[j].instr = Instruction::Nop.into(); + } + + // Replace BUILD_TUPLE with LOAD_CONST + block.instructions[i].instr = Instruction::LoadConst { idx: Arg::marker() }.into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + + i += 1; + } + } + } + + /// Peephole optimization: combine consecutive instructions into super-instructions + fn peephole_optimize(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i + 1 < block.instructions.len() { + let combined = { + let curr = &block.instructions[i]; + let next = &block.instructions[i + 1]; + + // Only combine if both are real instructions (not pseudo) + let (Some(curr_instr), Some(next_instr)) = + (curr.instr.real(), next.instr.real()) + else { + i += 1; + continue; + }; + + match (curr_instr, next_instr) { + // LoadFast + LoadFast -> LoadFastLoadFast (if both indices < 16) + (Instruction::LoadFast(_), Instruction::LoadFast(_)) => { + let idx1 = u32::from(curr.arg); + let idx2 = u32::from(next.arg); + if idx1 < 16 && idx2 < 16 { + let packed = (idx1 << 4) | idx2; + Some(( + Instruction::LoadFastLoadFast { arg: Arg::marker() }, + OpArg::new(packed), + )) + } else { + None + } + } + // StoreFast + StoreFast -> StoreFastStoreFast (if both indices < 16) + (Instruction::StoreFast(_), Instruction::StoreFast(_)) => { + let idx1 = u32::from(curr.arg); + let idx2 = u32::from(next.arg); + if idx1 < 16 && idx2 < 16 { + let packed = (idx1 << 4) | idx2; + Some(( + Instruction::StoreFastStoreFast { arg: Arg::marker() }, + OpArg::new(packed), + )) + } else { + None + } + } + _ => None, + } + }; + + if let Some((new_instr, new_arg)) = combined { + // Combine: keep first instruction's location, replace with combined instruction + block.instructions[i].instr = new_instr.into(); + block.instructions[i].arg = new_arg; + // Remove the second instruction + block.instructions.remove(i + 1); + // Don't increment i - check if we can combine again with the next instruction + } else { + i += 1; + } + } + } + } + + /// Convert LOAD_CONST for small integers to LOAD_SMALL_INT + /// maybe_instr_make_load_smallint + fn convert_to_load_small_int(&mut self) { + for block in &mut self.blocks { + for instr in &mut block.instructions { + // Check if it's a LOAD_CONST instruction + let Some(Instruction::LoadConst { .. }) = instr.instr.real() else { + continue; + }; + + // Get the constant value + let const_idx = u32::from(instr.arg) as usize; + let Some(constant) = self.metadata.consts.get_index(const_idx) else { + continue; + }; + + // Check if it's a small integer + let ConstantData::Integer { value } = constant else { + continue; + }; + + // Check if it's in small int range: -5 to 256 (_PY_IS_SMALL_INT) + if let Some(small) = value.to_i32().filter(|v| (-5..=256).contains(v)) { + // Convert LOAD_CONST to LOAD_SMALL_INT + instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into(); + // The arg is the i32 value stored as u32 (two's complement) + instr.arg = OpArg::new(small as u32); + } + } + } + } + + /// Remove constants that are no longer referenced by LOAD_CONST instructions. + /// remove_unused_consts + fn remove_unused_consts(&mut self) { + let nconsts = self.metadata.consts.len(); + if nconsts == 0 { + return; + } + + // Mark used constants + // The first constant (index 0) is always kept (may be docstring) + let mut used = vec![false; nconsts]; + used[0] = true; + + for block in &self.blocks { + for instr in &block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let idx = u32::from(instr.arg) as usize; + if idx < nconsts { + used[idx] = true; + } + } + } + } + + // Check if any constants can be removed + let n_used: usize = used.iter().filter(|&&u| u).count(); + if n_used == nconsts { + return; // Nothing to remove + } + + // Build old_to_new index mapping + let mut old_to_new = vec![0usize; nconsts]; + let mut new_idx = 0usize; + for (old_idx, &is_used) in used.iter().enumerate() { + if is_used { + old_to_new[old_idx] = new_idx; + new_idx += 1; + } + } + + // Build new consts list + let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect(); + self.metadata.consts.clear(); + for (old_idx, constant) in old_consts.into_iter().enumerate() { + if used[old_idx] { + self.metadata.consts.insert(constant); + } + } + + // Update LOAD_CONST instruction arguments + for block in &mut self.blocks { + for instr in &mut block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let old_idx = u32::from(instr.arg) as usize; + if old_idx < nconsts { + instr.arg = OpArg::new(old_to_new[old_idx] as u32); + } + } + } + } + } + + /// Remove NOP instructions from all blocks + fn remove_nops(&mut self) { + for block in &mut self.blocks { + block + .instructions + .retain(|ins| !matches!(ins.instr.real(), Some(Instruction::Nop))); + } + } + + /// Optimize LOAD_FAST to LOAD_FAST_BORROW where safe. + /// + /// A LOAD_FAST can be converted to LOAD_FAST_BORROW if its value is + /// consumed within the same basic block (not passed to another block). + /// This is a reference counting optimization in CPython; in RustPython + /// we implement it for bytecode compatibility. + fn optimize_load_fast_borrow(&mut self) { + // NOT_LOCAL marker: instruction didn't come from a LOAD_FAST + const NOT_LOCAL: usize = usize::MAX; + + for block in &mut self.blocks { + if block.instructions.is_empty() { + continue; + } + + // Track which instructions' outputs are still on stack at block end + // For each instruction, we track if its pushed value(s) are unconsumed + let mut unconsumed = vec![false; block.instructions.len()]; + + // Simulate stack: each entry is the instruction index that pushed it + // (or NOT_LOCAL if not from LOAD_FAST/LOAD_FAST_LOAD_FAST). + // + // CPython (flowgraph.c optimize_load_fast) pre-fills the stack with + // dummy refs for values inherited from predecessor blocks. We take + // the simpler approach of aborting the optimisation for the whole + // block on stack underflow. + let mut stack: Vec<usize> = Vec::new(); + let mut underflow = false; + + for (i, info) in block.instructions.iter().enumerate() { + let Some(instr) = info.instr.real() else { + continue; + }; + + let stack_effect_info = instr.stack_effect_info(info.arg.into()); + let (pushes, pops) = (stack_effect_info.pushed(), stack_effect_info.popped()); + + // Pop values from stack + for _ in 0..pops { + if stack.pop().is_none() { + // Stack underflow — block receives values from a predecessor. + // Abort optimisation for the entire block. + underflow = true; + break; + } + } + if underflow { + break; + } + + // Push values to stack with source instruction index + let source = match instr { + Instruction::LoadFast(_) | Instruction::LoadFastLoadFast { .. } => i, + _ => NOT_LOCAL, + }; + for _ in 0..pushes { + stack.push(source); + } + } + + if underflow { + continue; + } + + // Mark instructions whose values remain on stack at block end + for &src in &stack { + if src != NOT_LOCAL { + unconsumed[src] = true; + } + } + + // Convert LOAD_FAST to LOAD_FAST_BORROW where value is fully consumed + for (i, info) in block.instructions.iter_mut().enumerate() { + if unconsumed[i] { + continue; + } + let Some(instr) = info.instr.real() else { + continue; + }; + match instr { + Instruction::LoadFast(_) => { + info.instr = Instruction::LoadFastBorrow(Arg::marker()).into(); + } + Instruction::LoadFastLoadFast { .. } => { + info.instr = + Instruction::LoadFastBorrowLoadFastBorrow { arg: Arg::marker() }.into(); + } + _ => {} + } + } + } + } + + fn max_stackdepth(&mut self) -> crate::InternalResult<u32> { let mut maxdepth = 0u32; let mut stack = Vec::with_capacity(self.blocks.len()); let mut start_depths = vec![u32::MAX; self.blocks.len()]; start_depths[0] = 0; stack.push(BlockIdx(0)); const DEBUG: bool = false; - 'process_blocks: while let Some(block) = stack.pop() { - let mut depth = start_depths[block.idx()]; + // Global iteration limit as safety guard + // The algorithm is monotonic (depths only increase), so it should converge quickly. + // Max iterations = blocks * max_possible_depth_increases per block + let max_iterations = self.blocks.len() * 100; + let mut iterations = 0usize; + 'process_blocks: while let Some(block_idx) = stack.pop() { + iterations += 1; + if iterations > max_iterations { + // Safety guard: should never happen in valid code + // Return error instead of silently breaking to avoid underestimated stack depth + return Err(InternalError::StackOverflow); + } + let idx = block_idx.idx(); + let mut depth = start_depths[idx]; if DEBUG { - eprintln!("===BLOCK {}===", block.0); + eprintln!("===BLOCK {}===", block_idx.0); } - let block = &self.blocks[block]; + let block = &self.blocks[block_idx]; for ins in &block.instructions { let instr = &ins.instr; - let effect = instr.stack_effect(ins.arg, false); + let effect = instr.stack_effect(ins.arg.into()); if DEBUG { let display_arg = if ins.target == BlockIdx::NULL { ins.arg } else { - OpArg(ins.target.0) + OpArg::new(ins.target.0) }; let instr_display = instr.display(display_arg, self); eprint!("{instr_display}: {depth} {effect:+} => "); @@ -327,30 +819,45 @@ impl CodeInfo { if new_depth > maxdepth { maxdepth = new_depth } - // we don't want to worry about Break/Continue, they use unwinding to jump to - // their targets and as such the stack size is taken care of in frame.rs by setting - // it back to the level it was at when SetupLoop was run - if ins.target != BlockIdx::NULL - && !matches!( - instr, - Instruction::Continue { .. } | Instruction::Break { .. } - ) - { - let effect = instr.stack_effect(ins.arg, true); - let target_depth = depth.checked_add_signed(effect).ok_or({ - if effect < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow + // Process target blocks for branching instructions + if ins.target != BlockIdx::NULL { + if instr.is_block_push() { + // SETUP_* pseudo ops: target is a handler block. + // Handler entry depth uses the jump-path stack effect: + // SETUP_FINALLY: +1 (pushes exc) + // SETUP_CLEANUP: +2 (pushes lasti + exc) + // SETUP_WITH: +1 (pops __enter__ result, pushes lasti + exc) + let handler_effect: u32 = match instr.pseudo() { + Some(PseudoInstruction::SetupCleanup { .. }) => 2, + _ => 1, // SetupFinally and SetupWith + }; + let handler_depth = depth + handler_effect; + if handler_depth > maxdepth { + maxdepth = handler_depth; + } + stackdepth_push(&mut stack, &mut start_depths, ins.target, handler_depth); + } else { + // SEND jumps to END_SEND with receiver still on stack. + // END_SEND performs the receiver pop. + let jump_effect = match instr.real() { + Some(Instruction::Send { .. }) => 0i32, + _ => effect, + }; + let target_depth = depth.checked_add_signed(jump_effect).ok_or({ + if jump_effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; + if target_depth > maxdepth { + maxdepth = target_depth } - })?; - if target_depth > maxdepth { - maxdepth = target_depth + stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); } - stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); } depth = new_depth; - if instr.unconditional_branch() { + if instr.is_scope_exit() || instr.is_unconditional_jump() { continue 'process_blocks; } } @@ -362,6 +869,25 @@ impl CodeInfo { if DEBUG { eprintln!("DONE: {maxdepth}"); } + + // Fix up handler stack_depth in ExceptHandlerInfo using start_depths + // computed above: depth = start_depth - 1 - preserve_lasti + for block in self.blocks.iter_mut() { + for ins in &mut block.instructions { + if let Some(ref mut handler) = ins.except_handler { + let h_start = start_depths[handler.handler_block.idx()]; + if h_start != u32::MAX { + let adjustment = 1 + handler.preserve_lasti as u32; + debug_assert!( + h_start >= adjustment, + "handler start depth {h_start} too shallow for adjustment {adjustment}" + ); + handler.stack_depth = h_start.saturating_sub(adjustment); + } + } + } + } + Ok(maxdepth) } } @@ -392,8 +918,10 @@ fn stackdepth_push( target: BlockIdx, depth: u32, ) { - let block_depth = &mut start_depths[target.idx()]; - if *block_depth == u32::MAX || depth > *block_depth { + let idx = target.idx(); + let block_depth = &mut start_depths[idx]; + if depth > *block_depth || *block_depth == u32::MAX { + // Found a path with higher depth (or first visit): update max and queue *block_depth = depth; stack.push(target); } @@ -401,7 +929,7 @@ fn stackdepth_push( fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '_ { let mut next = BlockIdx(0); - std::iter::from_fn(move || { + core::iter::from_fn(move || { if next == BlockIdx::NULL { return None; } @@ -411,8 +939,12 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + ' }) } -/// Generate CPython 3.11+ format linetable from source locations -fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8]> { +/// Generate Python 3.11+ format linetable from source locations +fn generate_linetable( + locations: &[LineTableLocation], + first_line: i32, + debug_ranges: bool, +) -> Box<[u8]> { if locations.is_empty() { return Box::new([]); } @@ -436,18 +968,33 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] while length > 0 { let entry_length = length.min(8); - // Get line and column information - // SourceLocation always has row and column (both are OneIndexed) - let line = loc.line.get() as i32; - let col = loc.character_offset.to_zero_indexed() as i32; - + // Get line information + let line = loc.line; + let end_line = loc.end_line; let line_delta = line - prev_line; + let end_line_delta = end_line - line; - // Choose the appropriate encoding based on line delta and column info - // Note: SourceLocation always has valid column, so we never get NO_COLUMNS case - if line_delta == 0 { - let end_col = col; // Use same column for end (no range info available) + // When debug_ranges is disabled, only emit line info (NoColumns format) + if !debug_ranges { + // NoColumns format (code 13): line info only, no column data + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + + prev_line = line; + length -= entry_length; + i += entry_length; + continue; + } + // Get column information (only when debug_ranges is enabled) + let col = loc.col; + let end_col = loc.end_col; + + // Choose the appropriate encoding based on line delta and column info + if line_delta == 0 && end_line_delta == 0 { if col < 80 && end_col - col < 16 && end_col >= col { // Short form (codes 0-9) for common cases let code = (col / 8).min(9) as u8; // Short0 to Short9 @@ -470,42 +1017,37 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] ); write_signed_varint(&mut linetable, 0); // line_delta = 0 write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } - } else if line_delta > 0 && line_delta < 3 - /* && column.is_some() */ - { + } else if line_delta > 0 && line_delta < 3 && end_line_delta == 0 { // One-line form (codes 11-12) for line deltas 1-2 - let end_col = col; // Use same column for end - if col < 128 && end_col < 128 { - let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); // 11 for delta=1, 12 for delta=2 + let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); linetable.push(col as u8); linetable.push(end_col as u8); } else { - // Long form for columns >= 128 or negative line delta + // Long form for columns >= 128 linetable.push( 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), ); write_signed_varint(&mut linetable, line_delta); write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } } else { // Long form (code 14) for all other cases - // This handles: line_delta < 0, line_delta >= 3, or columns >= 128 - let end_col = col; // Use same column for end + // Handles: line_delta < 0, line_delta >= 3, multi-line spans, or columns >= 128 linetable.push( 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), ); write_signed_varint(&mut linetable, line_delta); - write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, end_line_delta as u32); + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } prev_line = line; @@ -517,27 +1059,271 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] linetable.into_boxed_slice() } -/// Write a variable-length unsigned integer (6-bit chunks) -/// Returns the number of bytes written -fn write_varint(buf: &mut Vec<u8>, mut val: u32) -> usize { - let start_len = buf.len(); - while val >= 64 { - buf.push(0x40 | (val & 0x3f) as u8); - val >>= 6; +/// Generate Python 3.11+ exception table from instruction handler info +fn generate_exception_table(blocks: &[Block], block_to_index: &[u32]) -> Box<[u8]> { + let mut entries: Vec<ExceptionTableEntry> = Vec::new(); + let mut current_entry: Option<(ExceptHandlerInfo, u32)> = None; // (handler_info, start_index) + let mut instr_index = 0u32; + + // Iterate through all instructions in block order + // instr_index is the index into the final instructions array (including EXTENDED_ARG) + // This matches how frame.rs uses lasti + for (_, block) in iter_blocks(blocks) { + for instr in &block.instructions { + // instr_size includes EXTENDED_ARG instructions + let instr_size = instr.arg.instr_size() as u32; + + match (&current_entry, instr.except_handler) { + // No current entry, no handler - nothing to do + (None, None) => {} + + // No current entry, handler starts - begin new entry + (None, Some(handler)) => { + current_entry = Some((handler, instr_index)); + } + + // Current entry exists, same handler - continue + (Some((curr_handler, _)), Some(handler)) + if curr_handler.handler_block == handler.handler_block + && curr_handler.stack_depth == handler.stack_depth + && curr_handler.preserve_lasti == handler.preserve_lasti => {} + + // Current entry exists, different handler - finish current, start new + (Some((curr_handler, start)), Some(handler)) => { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + *start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + current_entry = Some((handler, instr_index)); + } + + // Current entry exists, no handler - finish current entry + (Some((curr_handler, start)), None) => { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + *start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + current_entry = None; + } + } + + instr_index += instr_size; // Account for EXTENDED_ARG instructions + } + } + + // Finish any remaining entry + if let Some((curr_handler, start)) = current_entry { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + } + + encode_exception_table(&entries) +} + +/// Mark exception handler target blocks. +/// flowgraph.c mark_except_handlers +pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { + // Reset handler flags + for block in blocks.iter_mut() { + block.except_handler = false; + block.preserve_lasti = false; + } + // Mark target blocks of SETUP_* as except handlers + let targets: Vec<usize> = blocks + .iter() + .flat_map(|b| b.instructions.iter()) + .filter(|i| i.instr.is_block_push() && i.target != BlockIdx::NULL) + .map(|i| i.target.idx()) + .collect(); + for idx in targets { + blocks[idx].except_handler = true; + } +} + +/// Label exception targets: walk CFG with except stack, set per-instruction +/// handler info and block preserve_lasti flag. Converts POP_BLOCK to NOP. +/// flowgraph.c label_exception_targets + push_except_block +pub(crate) fn label_exception_targets(blocks: &mut [Block]) { + #[derive(Clone)] + struct ExceptEntry { + handler_block: BlockIdx, + preserve_lasti: bool, + } + + let num_blocks = blocks.len(); + if num_blocks == 0 { + return; + } + + let mut visited = vec![false; num_blocks]; + let mut block_stacks: Vec<Option<Vec<ExceptEntry>>> = vec![None; num_blocks]; + + // Entry block + visited[0] = true; + block_stacks[0] = Some(Vec::new()); + + let mut todo = vec![BlockIdx(0)]; + + while let Some(block_idx) = todo.pop() { + let bi = block_idx.idx(); + let mut stack = block_stacks[bi].take().unwrap_or_default(); + let mut last_yield_except_depth: i32 = -1; + + let instr_count = blocks[bi].instructions.len(); + for i in 0..instr_count { + // Read all needed fields (each temporary borrow ends immediately) + let target = blocks[bi].instructions[i].target; + let arg = blocks[bi].instructions[i].arg; + let is_push = blocks[bi].instructions[i].instr.is_block_push(); + let is_pop = blocks[bi].instructions[i].instr.is_pop_block(); + + if is_push { + // Determine preserve_lasti from instruction type (push_except_block) + let preserve_lasti = matches!( + blocks[bi].instructions[i].instr.pseudo(), + Some( + PseudoInstruction::SetupWith { .. } + | PseudoInstruction::SetupCleanup { .. } + ) + ); + + // Set preserve_lasti on handler block + if preserve_lasti && target != BlockIdx::NULL { + blocks[target.idx()].preserve_lasti = true; + } + + // Propagate except stack to handler block if not visited + if target != BlockIdx::NULL && !visited[target.idx()] { + visited[target.idx()] = true; + block_stacks[target.idx()] = Some(stack.clone()); + todo.push(target); + } + + // Push handler onto except stack + stack.push(ExceptEntry { + handler_block: target, + preserve_lasti, + }); + } else if is_pop { + debug_assert!( + !stack.is_empty(), + "POP_BLOCK with empty except stack at block {bi} instruction {i}" + ); + stack.pop(); + // POP_BLOCK → NOP + blocks[bi].instructions[i].instr = Instruction::Nop.into(); + } else { + // Set except_handler for this instruction from except stack top + // stack_depth placeholder: filled by fixup_handler_depths + let handler_info = stack.last().map(|e| ExceptHandlerInfo { + handler_block: e.handler_block, + stack_depth: 0, + preserve_lasti: e.preserve_lasti, + }); + blocks[bi].instructions[i].except_handler = handler_info; + + // Track YIELD_VALUE except stack depth + if matches!( + blocks[bi].instructions[i].instr.real(), + Some(Instruction::YieldValue { .. }) + ) { + last_yield_except_depth = stack.len() as i32; + } + + // Set RESUME DEPTH1 flag based on last yield's except depth + if matches!( + blocks[bi].instructions[i].instr.real(), + Some(Instruction::Resume { .. }) + ) { + const RESUME_AT_FUNC_START: u32 = 0; + const RESUME_OPARG_LOCATION_MASK: u32 = 0x3; + const RESUME_OPARG_DEPTH1_MASK: u32 = 0x4; + + if (u32::from(arg) & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START { + if last_yield_except_depth == 1 { + blocks[bi].instructions[i].arg = + OpArg::new(u32::from(arg) | RESUME_OPARG_DEPTH1_MASK); + } + last_yield_except_depth = -1; + } + } + + // For jump instructions, propagate except stack to target + if target != BlockIdx::NULL && !visited[target.idx()] { + visited[target.idx()] = true; + block_stacks[target.idx()] = Some(stack.clone()); + todo.push(target); + } + } + } + + // Propagate to fallthrough block (block.next) + let next = blocks[bi].next; + if next != BlockIdx::NULL && !visited[next.idx()] { + let has_fallthrough = blocks[bi] + .instructions + .last() + .map(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) + .unwrap_or(true); // Empty block falls through + if has_fallthrough { + visited[next.idx()] = true; + block_stacks[next.idx()] = Some(stack); + todo.push(next); + } + } } - buf.push(val as u8); - buf.len() - start_len } -/// Write a variable-length signed integer -/// Returns the number of bytes written -fn write_signed_varint(buf: &mut Vec<u8>, val: i32) -> usize { - let uval = if val < 0 { - // (unsigned int)(-val) has an undefined behavior for INT_MIN - // So we use (0 - val as u32) to handle it correctly - ((0u32.wrapping_sub(val as u32)) << 1) | 1 - } else { - (val as u32) << 1 - }; - write_varint(buf, uval) +/// Convert remaining pseudo ops to real instructions or NOP. +/// flowgraph.c convert_pseudo_ops +pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], varnames_len: u32) { + for block in blocks.iter_mut() { + for info in &mut block.instructions { + let Some(pseudo) = info.instr.pseudo() else { + continue; + }; + match pseudo { + // Block push pseudo ops → NOP + PseudoInstruction::SetupCleanup { .. } + | PseudoInstruction::SetupFinally { .. } + | PseudoInstruction::SetupWith { .. } => { + info.instr = Instruction::Nop.into(); + } + // PopBlock in reachable blocks is converted to NOP by + // label_exception_targets. Dead blocks may still have them. + PseudoInstruction::PopBlock => { + info.instr = Instruction::Nop.into(); + } + // LOAD_CLOSURE → LOAD_FAST (with varnames offset) + PseudoInstruction::LoadClosure(idx) => { + let new_idx = varnames_len + idx.get(info.arg); + info.arg = OpArg::new(new_idx); + info.instr = Instruction::LoadFast(Arg::marker()).into(); + } + // Jump pseudo ops are resolved during block linearization + PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. } => {} + // These should have been resolved earlier + PseudoInstruction::AnnotationsPlaceholder + | PseudoInstruction::JumpIfFalse { .. } + | PseudoInstruction::JumpIfTrue { .. } + | PseudoInstruction::StoreFastMaybeNull(_) => { + unreachable!("Unexpected pseudo instruction in convert_pseudo_ops: {pseudo:?}") + } + } + } + } } diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index 291b57d7f67..c1a318c19cd 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -1,10 +1,13 @@ //! Compile a Python AST or source code into bytecode consumable by RustPython. +#![cfg_attr(not(feature = "std"), no_std)] #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] #![doc(html_root_url = "https://docs.rs/rustpython-compiler/")] #[macro_use] extern crate log; +extern crate alloc; + type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>; type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>; @@ -16,7 +19,7 @@ pub mod symboltable; mod unparse; pub use compile::CompileOpts; -use ruff_python_ast::Expr; +use ruff_python_ast as ast; pub(crate) use compile::InternalResult; @@ -25,7 +28,7 @@ pub trait ToPythonName { fn python_name(&self) -> &'static str; } -impl ToPythonName for Expr { +impl ToPythonName for ast::Expr { fn python_name(&self) -> &'static str { match self { Self::BoolOp { .. } | Self::BinOp { .. } | Self::UnaryOp { .. } => "operator", diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap index 8b2907ef6ff..26583f5da0f 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap @@ -1,12 +1,14 @@ --- -source: compiler/codegen/src/compile.rs +source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" --- - 1 0 LoadConst (True) - 1 PopJumpIfFalse (6) - 2 LoadConst (False) - 3 PopJumpIfFalse (6) - 4 LoadConst (False) - 5 PopJumpIfFalse (6) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (7) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_FALSE (7) + 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (7) - 2 >> 6 ReturnConst (None) + 2 >> 7 LOAD_CONST (None) + 8 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap index fc91a74283b..21976b257ef 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap @@ -1,14 +1,16 @@ --- -source: compiler/codegen/src/compile.rs +source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" --- - 1 0 LoadConst (True) - 1 PopJumpIfFalse (4) - 2 LoadConst (False) - 3 PopJumpIfTrue (8) - >> 4 LoadConst (False) - 5 PopJumpIfFalse (8) - 6 LoadConst (True) - 7 PopJumpIfFalse (8) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (5) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_TRUE (9) + >> 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (9) + 7 LOAD_CONST (True) + 8 POP_JUMP_IF_FALSE (9) - 2 >> 8 ReturnConst (None) + 2 >> 9 LOAD_CONST (None) + 10 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap index 9be7c2af7bd..e1d41377db7 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap @@ -1,12 +1,14 @@ --- -source: compiler/codegen/src/compile.rs +source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" --- - 1 0 LoadConst (True) - 1 PopJumpIfTrue (6) - 2 LoadConst (False) - 3 PopJumpIfTrue (6) - 4 LoadConst (False) - 5 PopJumpIfFalse (6) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_TRUE (7) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_TRUE (7) + 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (7) - 2 >> 6 ReturnConst (None) + 2 >> 7 LOAD_CONST (None) + 8 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap new file mode 100644 index 00000000000..5b9a2182bdf --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap @@ -0,0 +1,20 @@ +--- +source: crates/codegen/src/compile.rs +assertion_line: 9071 +expression: "compile_exec(\"\\\nx = Test() and False or False\n\")" +--- + 1 0 RESUME (0) + 1 LOAD_NAME (0, Test) + 2 PUSH_NULL + 3 CALL (0) + 4 COPY (1) + 5 POP_JUMP_IF_FALSE (10) + 6 POP_TOP + 7 LOAD_CONST (False) + 8 COPY (1) + 9 POP_JUMP_IF_TRUE (12) + >> 10 POP_TOP + 11 LOAD_CONST (False) + >> 12 STORE_NAME (1, x) + 13 LOAD_CONST (None) + 14 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap index 435b73a14de..2a482f3ecb9 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -1,86 +1,165 @@ --- source: crates/codegen/src/compile.rs -expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" +expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" --- - 1 0 SetupLoop - 1 LoadNameAny (0, StopIteration) - 2 LoadConst ("spam") - 3 CallFunctionPositional(1) - 4 LoadNameAny (1, StopAsyncIteration) - 5 LoadConst ("ham") - 6 CallFunctionPositional(1) - 7 BuildTuple (2) - 8 GetIter - >> 9 ForIter (71) - 10 StoreLocal (2, stop_exc) + 1 0 RESUME (0) - 2 11 LoadNameAny (3, self) - 12 LoadMethod (4, subTest) - 13 LoadNameAny (5, type) - 14 LoadNameAny (2, stop_exc) - 15 CallFunctionPositional(1) - 16 LoadConst (("type")) - 17 CallMethodKeyword (1) - 18 SetupWith (68) - 19 Pop + 3 1 LOAD_CONST (<code object test at ??? file "source_path", line 1>): 1 0 RETURN_GENERATOR + 1 POP_TOP + 2 RESUME (0) - 3 20 SetupExcept (42) + 2 3 LOAD_GLOBAL (0, StopIteration) + 4 PUSH_NULL + 5 LOAD_CONST ("spam") + 6 CALL (1) + 7 LOAD_GLOBAL (1, StopAsyncIteration) + 8 PUSH_NULL + 9 LOAD_CONST ("ham") + 10 CALL (1) + 11 BUILD_TUPLE (2) + 12 GET_ITER + >> 13 FOR_ITER (141) + 14 STORE_FAST (0, stop_exc) - 4 21 LoadNameAny (6, egg) - 22 CallFunctionPositional(0) - 23 BeforeAsyncWith - 24 GetAwaitable - 25 LoadConst (None) - 26 YieldFrom - 27 Resume (3) - 28 SetupAsyncWith (34) - 29 Pop + 3 15 LOAD_GLOBAL (2, self) + 16 LOAD_ATTR (7, subTest, method=true) + 17 LOAD_GLOBAL (4, type) + 18 PUSH_NULL + 19 LOAD_FAST (0, stop_exc) + 20 CALL (1) + 21 LOAD_CONST (("type")) + 22 CALL_KW (1) + 23 COPY (1) + 24 LOAD_SPECIAL (__exit__) + 25 SWAP (2) + 26 LOAD_SPECIAL (__enter__) + 27 PUSH_NULL + 28 CALL (0) + 29 POP_TOP - 5 30 LoadNameAny (2, stop_exc) - 31 Raise (Raise) + 5 30 LOAD_GLOBAL (5, egg) + 31 PUSH_NULL + 32 CALL (0) + 33 COPY (1) + 34 LOAD_SPECIAL (__aexit__) + 35 SWAP (2) + 36 LOAD_SPECIAL (__aenter__) + 37 PUSH_NULL + 38 CALL (0) + 39 GET_AWAITABLE (1) + 40 LOAD_CONST (None) + >> 41 SEND (46) + 42 YIELD_VALUE (1) + 43 RESUME (3) + 44 JUMP_BACKWARD_NO_INTERRUPT(41) + 45 CLEANUP_THROW + >> 46 END_SEND + 47 POP_TOP - 4 32 PopBlock - 33 EnterFinally - >> 34 WithCleanupStart - 35 GetAwaitable - 36 LoadConst (None) - 37 YieldFrom - 38 Resume (3) - 39 WithCleanupFinish - 40 PopBlock - 41 Jump (58) - >> 42 CopyItem (1) + 6 48 LOAD_FAST (0, stop_exc) + 49 RAISE_VARARGS (Raise) - 6 43 LoadNameAny (7, Exception) - 44 JUMP_IF_NOT_EXC_MATCH(57) - 45 StoreLocal (8, ex) + 5 50 PUSH_NULL + 51 LOAD_CONST (None) + 52 LOAD_CONST (None) + 53 LOAD_CONST (None) + 54 CALL (3) + 55 GET_AWAITABLE (2) + 56 LOAD_CONST (None) + >> 57 SEND (62) + 58 YIELD_VALUE (1) + 59 RESUME (3) + 60 JUMP_BACKWARD_NO_INTERRUPT(57) + 61 CLEANUP_THROW + >> 62 END_SEND + 63 POP_TOP + 64 JUMP_FORWARD (86) + 65 PUSH_EXC_INFO + 66 WITH_EXCEPT_START + 67 GET_AWAITABLE (2) + 68 LOAD_CONST (None) + >> 69 SEND (74) + 70 YIELD_VALUE (1) + 71 RESUME (3) + 72 JUMP_BACKWARD_NO_INTERRUPT(69) + 73 CLEANUP_THROW + >> 74 END_SEND + 75 TO_BOOL + 76 POP_JUMP_IF_TRUE (78) + 77 RERAISE (2) + >> 78 POP_TOP + 79 POP_EXCEPT + 80 POP_TOP + 81 POP_TOP + 82 JUMP_FORWARD (86) + 83 COPY (3) + 84 POP_EXCEPT + 85 RERAISE (1) + >> 86 JUMP_FORWARD (112) + 87 PUSH_EXC_INFO - 7 46 LoadNameAny (3, self) - 47 LoadMethod (9, assertIs) - 48 LoadNameAny (8, ex) - 49 LoadNameAny (2, stop_exc) - 50 CallMethodPositional (2) - 51 Pop - 52 PopException - 53 LoadConst (None) - 54 StoreLocal (8, ex) - 55 DeleteLocal (8, ex) - 56 Jump (66) - >> 57 Raise (Reraise) + 7 88 LOAD_GLOBAL (6, Exception) + 89 CHECK_EXC_MATCH + 90 POP_JUMP_IF_FALSE (108) + 91 STORE_FAST (1, ex) - 9 >> 58 LoadNameAny (3, self) - 59 LoadMethod (10, fail) - 60 LoadNameAny (2, stop_exc) - 61 FORMAT_SIMPLE - 62 LoadConst (" was suppressed") - 63 BuildString (2) - 64 CallMethodPositional (1) - 65 Pop + 8 92 LOAD_GLOBAL (2, self) + 93 LOAD_ATTR (15, assertIs, method=true) + 94 LOAD_FAST (1, ex) + 95 LOAD_FAST (0, stop_exc) + 96 CALL (2) + 97 POP_TOP + 98 JUMP_FORWARD (103) + 99 LOAD_CONST (None) + 100 STORE_FAST (1, ex) + 101 DELETE_FAST (1, ex) + 102 RAISE_VARARGS (ReraiseFromStack) + >> 103 POP_EXCEPT + 104 LOAD_CONST (None) + 105 STORE_FAST (1, ex) + 106 DELETE_FAST (1, ex) + 107 JUMP_FORWARD (120) + >> 108 RAISE_VARARGS (ReraiseFromStack) + 109 COPY (3) + 110 POP_EXCEPT + 111 RAISE_VARARGS (ReraiseFromStack) - 2 >> 66 PopBlock - 67 EnterFinally - >> 68 WithCleanupStart - 69 WithCleanupFinish - 70 Jump (9) - >> 71 PopBlock - 72 ReturnConst (None) + 10 >> 112 LOAD_GLOBAL (2, self) + 113 LOAD_ATTR (17, fail, method=true) + 114 LOAD_FAST_BORROW (0, stop_exc) + 115 FORMAT_SIMPLE + 116 LOAD_CONST (" was suppressed") + 117 BUILD_STRING (2) + 118 CALL (1) + 119 POP_TOP + + 3 >> 120 PUSH_NULL + 121 LOAD_CONST (None) + 122 LOAD_CONST (None) + 123 LOAD_CONST (None) + 124 CALL (3) + 125 POP_TOP + 126 JUMP_FORWARD (140) + 127 PUSH_EXC_INFO + 128 WITH_EXCEPT_START + 129 TO_BOOL + 130 POP_JUMP_IF_TRUE (132) + 131 RERAISE (2) + >> 132 POP_TOP + 133 POP_EXCEPT + 134 POP_TOP + 135 POP_TOP + 136 JUMP_FORWARD (140) + 137 COPY (3) + 138 POP_EXCEPT + 139 RERAISE (1) + >> 140 JUMP_BACKWARD (13) + >> 141 END_FOR + 142 POP_ITER + 143 LOAD_CONST (None) + 144 RETURN_VALUE + + 2 MAKE_FUNCTION + 3 STORE_NAME (0, test) + 4 LOAD_CONST (None) + 5 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap deleted file mode 100644 index 589f3210cfa..00000000000 --- a/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap +++ /dev/null @@ -1,87 +0,0 @@ ---- -source: compiler/src/compile.rs -expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with woohoo():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" ---- - 1 0 SetupLoop (69) - 1 LoadNameAny (0, StopIteration) - 2 LoadConst ("spam") - 3 CallFunctionPositional (1) - 4 LoadNameAny (1, StopAsyncIteration) - 5 LoadConst ("ham") - 6 CallFunctionPositional (1) - 7 BuildTuple (2, false) - 8 GetIter - >> 9 ForIter (68) - 10 StoreLocal (2, stop_exc) - - 2 11 LoadNameAny (3, self) - 12 LoadMethod (subTest) - 13 LoadNameAny (5, type) - 14 LoadNameAny (2, stop_exc) - 15 CallFunctionPositional (1) - 16 LoadConst (("type")) - 17 CallMethodKeyword (1) - 18 SetupWith (65) - 19 Pop - - 3 20 SetupExcept (40) - - 4 21 LoadNameAny (6, woohoo) - 22 CallFunctionPositional (0) - 23 BeforeAsyncWith - 24 GetAwaitable - 25 LoadConst (None) - 26 YieldFrom - 27 SetupAsyncWith (33) - 28 Pop - - 5 29 LoadNameAny (2, stop_exc) - 30 Raise (Raise) - - 4 31 PopBlock - 32 EnterFinally - >> 33 WithCleanupStart - 34 GetAwaitable - 35 LoadConst (None) - 36 YieldFrom - 37 WithCleanupFinish - 38 PopBlock - 39 Jump (54) - >> 40 Duplicate - - 6 41 LoadNameAny (7, Exception) - 42 TestOperation (ExceptionMatch) - 43 PopJumpIfFalse (53) - 44 StoreLocal (8, ex) - - 7 45 LoadNameAny (3, self) - 46 LoadMethod (assertIs) - 47 LoadNameAny (8, ex) - 48 LoadNameAny (2, stop_exc) - 49 CallMethodPositional (2) - 50 Pop - 51 PopException - 52 Jump (63) - >> 53 Raise (Reraise) - - 9 >> 54 LoadNameAny (3, self) - 55 LoadMethod (fail) - 56 LoadConst ("") - - 1 57 LoadNameAny (2, stop_exc) - 58 FormatValue (None) - - 9 59 LoadConst (" was suppressed") - 60 BuildString (2) - 61 CallMethodPositional (1) - 62 Pop - - 2 >> 63 PopBlock - 64 EnterFinally - >> 65 WithCleanupStart - 66 WithCleanupFinish - 67 Jump (9) - >> 68 PopBlock - >> 69 LoadConst (None) - 70 ReturnValue - diff --git a/crates/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs index ede2f118c37..a7ad8c35a46 100644 --- a/crates/codegen/src/string_parser.rs +++ b/crates/codegen/src/string_parser.rs @@ -5,9 +5,9 @@ //! after ruff has already successfully parsed the string literal, meaning //! we don't need to do any validation or error handling. -use std::convert::Infallible; +use core::convert::Infallible; -use ruff_python_ast::{AnyStringFlags, StringFlags}; +use ruff_python_ast::{self as ast, StringFlags as _}; use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf}; // use ruff_python_parser::{LexicalError, LexicalErrorType}; @@ -24,11 +24,11 @@ struct StringParser { /// Current position of the parser in the source. cursor: usize, /// Flags that can be used to query information about the string. - flags: AnyStringFlags, + flags: ast::AnyStringFlags, } impl StringParser { - const fn new(source: Box<str>, flags: AnyStringFlags) -> Self { + const fn new(source: Box<str>, flags: ast::AnyStringFlags) -> Self { Self { source, cursor: 0, @@ -96,7 +96,7 @@ impl StringParser { } // OK because radix_bytes is always going to be in the ASCII range. - let radix_str = std::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); + let radix_str = core::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); let value = u32::from_str_radix(radix_str, 8).unwrap(); char::from_u32(value).unwrap() } @@ -272,15 +272,25 @@ impl StringParser { } } -pub(crate) fn parse_string_literal(source: &str, flags: AnyStringFlags) -> Box<Wtf8> { - let source = &source[flags.opener_len().to_usize()..]; - let source = &source[..source.len() - flags.quote_len().to_usize()]; +pub(crate) fn parse_string_literal(source: &str, flags: ast::AnyStringFlags) -> Box<Wtf8> { + let opener_len = flags.opener_len().to_usize(); + let quote_len = flags.quote_len().to_usize(); + if source.len() < opener_len + quote_len { + // Source unavailable (e.g., compiling from an AST object with no + // backing source text). Return the raw source as-is. + return Box::<Wtf8>::from(source); + } + let source = &source[opener_len..]; + let source = &source[..source.len() - quote_len]; StringParser::new(source.into(), flags) .parse_string() .unwrap_or_else(|x| match x {}) } -pub(crate) fn parse_fstring_literal_element(source: Box<str>, flags: AnyStringFlags) -> Box<Wtf8> { +pub(crate) fn parse_fstring_literal_element( + source: Box<str>, + flags: ast::AnyStringFlags, +) -> Box<Wtf8> { StringParser::new(source, flags) .parse_fstring_middle() .unwrap_or_else(|x| match x {}) diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 3c8454b9e22..22bea20fa6c 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -8,19 +8,14 @@ Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable. */ use crate::{ - IndexMap, + IndexMap, IndexSet, error::{CodegenError, CodegenErrorType}, }; +use alloc::{borrow::Cow, fmt}; use bitflags::bitflags; -use ruff_python_ast::{ - self as ast, Comprehension, Decorator, Expr, Identifier, ModExpression, ModModule, Parameter, - ParameterWithDefault, Parameters, Pattern, PatternMatchAs, PatternMatchClass, - PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchStar, PatternMatchValue, - Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, -}; +use ruff_python_ast as ast; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{PositionEncoding, SourceFile, SourceLocation}; -use std::{borrow::Cow, fmt}; /// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. #[derive(Clone)] @@ -44,6 +39,9 @@ pub struct SymbolTable { /// AST nodes. pub sub_tables: Vec<SymbolTable>, + /// Cursor pointing to the next sub-table to consume during compilation. + pub next_sub_table: usize, + /// Variable names in definition order (parameters first, then locals) pub varnames: Vec<String>, @@ -55,6 +53,26 @@ pub struct SymbolTable { /// Whether this type param scope can see the parent class scope pub can_see_class_scope: bool, + + /// Whether this comprehension scope should be inlined (PEP 709) + /// True for list/set/dict comprehensions in non-generator expressions + pub comp_inlined: bool, + + /// PEP 649: Reference to annotation scope for this block + /// Annotations are compiled as a separate `__annotate__` function + pub annotation_block: Option<Box<SymbolTable>>, + + /// PEP 649: Whether this scope has conditional annotations + /// (annotations inside if/for/while/etc. blocks or at module level) + pub has_conditional_annotations: bool, + + /// Whether `from __future__ import annotations` is active + pub future_annotations: bool, + + /// Names of type parameters that should still be mangled in type param scopes. + /// When Some, only names in this set are mangled; other names are left unmangled. + /// Set on type param blocks for generic classes; inherited by non-class child scopes. + pub mangled_names: Option<IndexSet<String>>, } impl SymbolTable { @@ -66,20 +84,32 @@ impl SymbolTable { is_nested, symbols: IndexMap::default(), sub_tables: vec![], + next_sub_table: 0, varnames: Vec::new(), needs_class_closure: false, needs_classdict: false, can_see_class_scope: false, + comp_inlined: false, + annotation_block: None, + has_conditional_annotations: false, + future_annotations: false, + mangled_names: None, } } - pub fn scan_program(program: &ModModule, source_file: SourceFile) -> SymbolTableResult<Self> { + pub fn scan_program( + program: &ast::ModModule, + source_file: SourceFile, + ) -> SymbolTableResult<Self> { let mut builder = SymbolTableBuilder::new(source_file); builder.scan_statements(program.body.as_ref())?; builder.finish() } - pub fn scan_expr(expr: &ModExpression, source_file: SourceFile) -> SymbolTableResult<Self> { + pub fn scan_expr( + expr: &ast::ModExpression, + source_file: SourceFile, + ) -> SymbolTableResult<Self> { let mut builder = SymbolTableBuilder::new(source_file); builder.scan_expression(expr.body.as_ref(), ExpressionContext::Load)?; builder.finish() @@ -99,6 +129,8 @@ pub enum CompilerScope { Lambda, Comprehension, TypeParams, + /// PEP 649: Annotation scope for deferred evaluation + Annotation, } impl fmt::Display for CompilerScope { @@ -111,9 +143,8 @@ impl fmt::Display for CompilerScope { Self::Lambda => write!(f, "lambda"), Self::Comprehension => write!(f, "comprehension"), Self::TypeParams => write!(f, "type parameter"), + Self::Annotation => write!(f, "annotation"), // TODO missing types from the C implementation - // if self._table.type == _symtable.TYPE_ANNOTATION: - // return "annotation" // if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: // return "TypeVar bound" // if self._table.type == _symtable.TYPE_TYPE_ALIAS: @@ -137,12 +168,12 @@ pub enum SymbolScope { bitflags! { #[derive(Copy, Clone, Debug, PartialEq)] pub struct SymbolFlags: u16 { - const REFERENCED = 0x001; - const ASSIGNED = 0x002; - const PARAMETER = 0x004; - const ANNOTATED = 0x008; - const IMPORTED = 0x010; - const NONLOCAL = 0x020; + const REFERENCED = 0x001; // USE + const ASSIGNED = 0x002; // DEF_LOCAL + const PARAMETER = 0x004; // DEF_PARAM + const ANNOTATED = 0x008; // DEF_ANNOT + const IMPORTED = 0x010; // DEF_IMPORT + const NONLOCAL = 0x020; // DEF_NONLOCAL // indicates if the symbol gets a value assigned by a named expression in a comprehension // this is required to correct the scope in the analysis. const ASSIGNED_IN_COMPREHENSION = 0x040; @@ -157,8 +188,12 @@ bitflags! { /// def method(self): /// return x // is_free_class /// ``` - const FREE_CLASS = 0x100; - const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits(); + const FREE_CLASS = 0x100; // DEF_FREE_CLASS + const GLOBAL = 0x200; // DEF_GLOBAL + const COMP_ITER = 0x400; // DEF_COMP_ITER + const COMP_CELL = 0x800; // DEF_COMP_CELL + const TYPE_PARAM = 0x1000; // DEF_TYPE_PARAM + const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits() | Self::TYPE_PARAM.bits(); } } @@ -215,8 +250,8 @@ impl SymbolTableError { type SymbolTableResult<T = ()> = Result<T, SymbolTableError>; -impl std::fmt::Debug for SymbolTable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for SymbolTable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, "SymbolTable({:?} symbols, {:?} sub scopes)", @@ -231,38 +266,44 @@ impl std::fmt::Debug for SymbolTable { */ fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { let mut analyzer = SymbolTableAnalyzer::default(); - analyzer.analyze_symbol_table(symbol_table) + // Discard the newfree set at the top level - it's only needed for propagation + // Pass None for class_entry at top level + let _newfree = analyzer.analyze_symbol_table(symbol_table, None)?; + Ok(()) } /* Drop __class__ and __classdict__ from free variables in class scope and set the appropriate flags. Equivalent to CPython's drop_class_free(). See: https://github.com/python/cpython/blob/main/Python/symtable.c#L884 + + This function removes __class__ and __classdict__ from the + `newfree` set (which contains free variables collected from all child scopes) + and sets the corresponding flags on the class's symbol table entry. */ -fn drop_class_free(symbol_table: &mut SymbolTable) { - // Check if __class__ is used as a free variable - if let Some(class_symbol) = symbol_table.symbols.get("__class__") - && class_symbol.scope == SymbolScope::Free - { +fn drop_class_free(symbol_table: &mut SymbolTable, newfree: &mut IndexSet<String>) { + // Check if __class__ is in the free variables collected from children + // If found, it means a child scope (method) references __class__ + if newfree.shift_remove("__class__") { symbol_table.needs_class_closure = true; - // Note: In CPython, the symbol is removed from the free set, - // but in RustPython we handle this differently during code generation } - // Check if __classdict__ is used as a free variable - if let Some(classdict_symbol) = symbol_table.symbols.get("__classdict__") - && classdict_symbol.scope == SymbolScope::Free - { + // Check if __classdict__ is in the free variables collected from children + if newfree.shift_remove("__classdict__") { symbol_table.needs_classdict = true; - // Note: In CPython, the symbol is removed from the free set, - // but in RustPython we handle this differently during code generation + } + + // Check if __conditional_annotations__ is in the free variables collected from children + // Remove it from free set - it's handled specially in class scope + if newfree.shift_remove("__conditional_annotations__") { + symbol_table.has_conditional_annotations = true; } } type SymbolMap = IndexMap<String, Symbol>; mod stack { - use std::panic; - use std::ptr::NonNull; + use alloc::vec::Vec; + use core::ptr::NonNull; pub struct StackStack<T> { v: Vec<NonNull<T>>, } @@ -274,14 +315,30 @@ mod stack { impl<T> StackStack<T> { /// Appends a reference to this stack for the duration of the function `f`. When `f` /// returns, the reference will be popped off the stack. + #[cfg(feature = "std")] + pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + self.v.push(x.into()); + let res = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| f(self))); + self.v.pop(); + res.unwrap_or_else(|x| std::panic::resume_unwind(x)) + } + + /// Appends a reference to this stack for the duration of the function `f`. When `f` + /// returns, the reference will be popped off the stack. + /// + /// Without std, panic cleanup is not guaranteed (no catch_unwind). + #[cfg(not(feature = "std"))] pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R where F: FnOnce(&mut Self) -> R, { self.v.push(x.into()); - let res = panic::catch_unwind(panic::AssertUnwindSafe(|| f(self))); + let result = f(self); self.v.pop(); - res.unwrap_or_else(|x| panic::resume_unwind(x)) + result } pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + '_ { @@ -324,33 +381,134 @@ struct SymbolTableAnalyzer { } impl SymbolTableAnalyzer { - fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult { - let symbols = std::mem::take(&mut symbol_table.symbols); + /// Analyze a symbol table and return the set of free variables. + /// See symtable.c analyze_block(). + /// class_entry: PEP 649 - enclosing class symbols for annotation scopes + fn analyze_symbol_table( + &mut self, + symbol_table: &mut SymbolTable, + class_entry: Option<&SymbolMap>, + ) -> SymbolTableResult<IndexSet<String>> { + let symbols = core::mem::take(&mut symbol_table.symbols); let sub_tables = &mut *symbol_table.sub_tables; + // Collect free variables from all child scopes + let mut newfree = IndexSet::default(); + + let annotation_block = &mut symbol_table.annotation_block; + + // PEP 649: Determine class_entry to pass to children + // If current scope is a class with annotation block that can_see_class_scope, + // we need to pass class symbols to the annotation scope + let is_class = symbol_table.typ == CompilerScope::Class; + + // Clone class symbols if needed for child scopes with can_see_class_scope + let needs_class_symbols = (is_class + && (sub_tables.iter().any(|st| st.can_see_class_scope) + || annotation_block + .as_ref() + .is_some_and(|b| b.can_see_class_scope))) + || (!is_class + && class_entry.is_some() + && sub_tables.iter().any(|st| st.can_see_class_scope)); + + let class_symbols_clone = if is_class && needs_class_symbols { + Some(symbols.clone()) + } else { + None + }; + let mut info = (symbols, symbol_table.typ); self.tables.with_append(&mut info, |list| { let inner_scope = unsafe { &mut *(list as *mut _ as *mut Self) }; - // Analyze sub scopes: + // Analyze sub scopes and collect their free variables for sub_table in sub_tables.iter_mut() { - inner_scope.analyze_symbol_table(sub_table)?; + // Pass class_entry to sub-scopes that can see the class scope + let child_class_entry = if sub_table.can_see_class_scope { + if is_class { + class_symbols_clone.as_ref() + } else { + class_entry + } + } else { + None + }; + let child_free = inner_scope.analyze_symbol_table(sub_table, child_class_entry)?; + // Propagate child's free variables to this scope + newfree.extend(child_free); + } + // PEP 649: Analyze annotation block if present + if let Some(annotation_table) = annotation_block { + // Pass class symbols to annotation scope if can_see_class_scope + let ann_class_entry = if annotation_table.can_see_class_scope { + if is_class { + class_symbols_clone.as_ref() + } else { + class_entry + } + } else { + None + }; + let child_free = + inner_scope.analyze_symbol_table(annotation_table, ann_class_entry)?; + // Propagate annotation's free variables to this scope + newfree.extend(child_free); } Ok(()) })?; symbol_table.symbols = info.0; - // Analyze symbols: + // PEP 709: Merge symbols from inlined comprehensions into parent scope + // Only merge symbols that are actually bound in the comprehension, + // not references to outer scope variables (Free symbols). + const BOUND_FLAGS: SymbolFlags = SymbolFlags::ASSIGNED + .union(SymbolFlags::PARAMETER) + .union(SymbolFlags::ITER) + .union(SymbolFlags::ASSIGNED_IN_COMPREHENSION); + + for sub_table in sub_tables.iter() { + if sub_table.comp_inlined { + for (name, sub_symbol) in &sub_table.symbols { + // Skip the .0 parameter - it's internal to the comprehension + if name == ".0" { + continue; + } + // Only merge symbols that are bound in the comprehension + // Skip Free references to outer scope variables + if !sub_symbol.flags.intersects(BOUND_FLAGS) { + continue; + } + // If the symbol doesn't exist in parent, add it + if !symbol_table.symbols.contains_key(name) { + let mut symbol = sub_symbol.clone(); + // Mark as local in parent scope + symbol.scope = SymbolScope::Local; + symbol_table.symbols.insert(name.clone(), symbol); + } + } + } + } + + // Analyze symbols in current scope for symbol in symbol_table.symbols.values_mut() { - self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?; + self.analyze_symbol(symbol, symbol_table.typ, sub_tables, class_entry)?; + + // Collect free variables from this scope + // These will be propagated to the parent scope + if symbol.scope == SymbolScope::Free || symbol.flags.contains(SymbolFlags::FREE_CLASS) { + newfree.insert(symbol.name.clone()); + } } - // Handle class-specific implicit cells (like CPython) + // Handle class-specific implicit cells + // This removes __class__ and __classdict__ from newfree if present + // and sets the corresponding flags on the symbol table if symbol_table.typ == CompilerScope::Class { - drop_class_free(symbol_table); + drop_class_free(symbol_table, &mut newfree); } - Ok(()) + Ok(newfree) } fn analyze_symbol( @@ -358,6 +516,7 @@ impl SymbolTableAnalyzer { symbol: &mut Symbol, st_typ: CompilerScope, sub_tables: &[SymbolTable], + class_entry: Option<&SymbolMap>, ) -> SymbolTableResult { if symbol .flags @@ -375,9 +534,9 @@ impl SymbolTableAnalyzer { if !self.tables.as_ref().is_empty() { let scope_depth = self.tables.as_ref().len(); // check if the name is already defined in any outer scope - // therefore if scope_depth < 2 - || self.found_in_outer_scope(&symbol.name) != Some(SymbolScope::Free) + || self.found_in_outer_scope(&symbol.name, st_typ) + != Some(SymbolScope::Free) { return Err(SymbolTableError { error: format!("no binding for nonlocal '{}' found", symbol.name), @@ -385,6 +544,25 @@ impl SymbolTableAnalyzer { location: None, }); } + // Check if the nonlocal binding refers to a type parameter + if symbol.flags.contains(SymbolFlags::NONLOCAL) { + for (symbols, _typ) in self.tables.iter().rev() { + if let Some(sym) = symbols.get(&symbol.name) { + if sym.flags.contains(SymbolFlags::TYPE_PARAM) { + return Err(SymbolTableError { + error: format!( + "nonlocal binding not allowed for type parameter '{}'", + symbol.name + ), + location: None, + }); + } + if sym.is_bound() { + break; + } + } + } + } } else { return Err(SymbolTableError { error: format!( @@ -407,8 +585,17 @@ impl SymbolTableAnalyzer { let scope = if symbol.is_bound() { self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) .unwrap_or(SymbolScope::Local) - } else if let Some(scope) = self.found_in_outer_scope(&symbol.name) { + } else if let Some(scope) = self.found_in_outer_scope(&symbol.name, st_typ) { + // If found in enclosing scope (function/TypeParams), use that scope + } else if let Some(class_symbols) = class_entry + && let Some(class_sym) = class_symbols.get(&symbol.name) + && class_sym.is_bound() + && class_sym.scope != SymbolScope::Free + { + // If name is bound in enclosing class, use GlobalImplicit + // so it can be accessed via __classdict__ + SymbolScope::GlobalImplicit } else if self.tables.is_empty() { // Don't make assumptions when we don't know. SymbolScope::Unknown @@ -423,14 +610,44 @@ impl SymbolTableAnalyzer { Ok(()) } - fn found_in_outer_scope(&mut self, name: &str) -> Option<SymbolScope> { + fn found_in_outer_scope(&mut self, name: &str, st_typ: CompilerScope) -> Option<SymbolScope> { let mut decl_depth = None; for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { if matches!(typ, CompilerScope::Module) - || matches!(typ, CompilerScope::Class if name != "__class__") + || matches!(typ, CompilerScope::Class if name != "__class__" && name != "__classdict__" && name != "__conditional_annotations__") { continue; } + + // PEP 649: Annotation scope is conceptually a sibling of the function, + // not a child. Skip the immediate parent function scope when looking + // for outer variables from annotation scope. + if st_typ == CompilerScope::Annotation + && i == 0 + && matches!( + typ, + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda + ) + { + continue; + } + + // __class__ and __classdict__ are implicitly declared in class scope + // This handles the case where nested scopes reference them + if (name == "__class__" || name == "__classdict__") + && matches!(typ, CompilerScope::Class) + { + decl_depth = Some(i); + break; + } + + // __conditional_annotations__ is implicitly declared in class scope + // for classes with conditional annotations + if name == "__conditional_annotations__" && matches!(typ, CompilerScope::Class) { + decl_depth = Some(i); + break; + } + if let Some(sym) = symbols.get(name) { match sym.scope { SymbolScope::GlobalExplicit => return Some(SymbolScope::GlobalExplicit), @@ -581,14 +798,25 @@ impl SymbolTableAnalyzer { self.analyze_symbol_comprehension(symbol, parent_offset + 1)?; } CompilerScope::TypeParams => { - todo!("analyze symbol comprehension for type params"); + // Named expression in comprehension cannot be used in type params + return Err(SymbolTableError { + error: "assignment expression within a comprehension cannot be used within the definition of a generic".to_string(), + location: None, + }); + } + CompilerScope::Annotation => { + // Named expression is not allowed in annotation scope + return Err(SymbolTableError { + error: "named expression cannot be used within an annotation".to_string(), + location: None, + }); } } Ok(()) } } -#[derive(Debug, Clone)] +#[derive(Clone, Copy, Debug)] enum SymbolUsage { Global, Nonlocal, @@ -611,6 +839,20 @@ struct SymbolTableBuilder { source_file: SourceFile, // Current scope's varnames being collected (temporary storage) current_varnames: Vec<String>, + // Stack to preserve parent varnames when entering nested scopes + varnames_stack: Vec<Vec<String>>, + // Track if we're inside an iterable definition expression (for nested comprehensions) + in_iter_def_exp: bool, + // Track if we're inside an annotation (yield/await/named expr not allowed) + in_annotation: bool, + // Track if we're inside a type alias (yield/await/named expr not allowed) + in_type_alias: bool, + // Track if we're scanning an inner loop iteration target (not the first generator) + in_comp_inner_loop_target: bool, + // Scope info for error messages (e.g., "a TypeVar bound") + scope_info: Option<&'static str>, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, } /// Enum to indicate in what mode an expression @@ -634,6 +876,13 @@ impl SymbolTableBuilder { future_annotations: false, source_file, current_varnames: Vec::new(), + varnames_stack: Vec::new(), + in_iter_def_exp: false, + in_annotation: false, + in_type_alias: false, + in_comp_inner_loop_target: false, + scope_info: None, + in_conditional_block: false, }; this.enter_scope("top", CompilerScope::Module, 0); this @@ -644,6 +893,8 @@ impl SymbolTableBuilder { let mut symbol_table = self.tables.pop().unwrap(); // Save varnames for the top-level module scope symbol_table.varnames = self.current_varnames; + // Propagate future_annotations to the symbol table + symbol_table.future_annotations = self.future_annotations; analyze_symbol_table(&mut symbol_table)?; Ok(symbol_table) } @@ -652,15 +903,34 @@ impl SymbolTableBuilder { let is_nested = self .tables .last() - .map(|table| table.is_nested || table.typ == CompilerScope::Function) + .map(|table| { + table.is_nested + || matches!( + table.typ, + CompilerScope::Function | CompilerScope::AsyncFunction + ) + }) .unwrap_or(false); - let table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + // Inherit mangled_names from parent for non-class scopes + let inherited_mangled_names = self + .tables + .last() + .and_then(|t| t.mangled_names.clone()) + .filter(|_| typ != CompilerScope::Class); + let mut table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + table.mangled_names = inherited_mangled_names; self.tables.push(table); - // Clear current_varnames for the new scope - self.current_varnames.clear(); + // Save parent's varnames and start fresh for the new scope + self.varnames_stack + .push(core::mem::take(&mut self.current_varnames)); } - fn enter_type_param_block(&mut self, name: &str, line_number: u32) -> SymbolTableResult { + fn enter_type_param_block( + &mut self, + name: &str, + line_number: u32, + for_class: bool, + ) -> SymbolTableResult { // Check if we're in a class scope let in_class = self .tables @@ -669,16 +939,21 @@ impl SymbolTableBuilder { self.enter_scope(name, CompilerScope::TypeParams, line_number); - // If we're in a class, mark that this type param scope can see the class scope + // Set properties on the newly created type param scope if let Some(table) = self.tables.last_mut() { table.can_see_class_scope = in_class; - - // Add __classdict__ as a USE symbol in type param scope if in class - if in_class { - self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + // For generic classes, create mangled_names set so that only + // type parameter names get mangled (not bases or other expressions) + if for_class { + table.mangled_names = Some(IndexSet::default()); } } + // Add __classdict__ as a USE symbol in type param scope if in class + if in_class { + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + // Register .type_params as a SET symbol (it will be converted to cell variable later) self.register_name(".type_params", SymbolUsage::Assigned, TextRange::default())?; @@ -689,8 +964,109 @@ impl SymbolTableBuilder { fn leave_scope(&mut self) { let mut table = self.tables.pop().unwrap(); // Save the collected varnames to the symbol table - table.varnames = std::mem::take(&mut self.current_varnames); + table.varnames = core::mem::take(&mut self.current_varnames); self.tables.last_mut().unwrap().sub_tables.push(table); + // Restore parent's varnames + self.current_varnames = self.varnames_stack.pop().unwrap_or_default(); + } + + /// Enter annotation scope (PEP 649) + /// Creates or reuses the annotation block for the current scope + fn enter_annotation_scope(&mut self, line_number: u32) { + let current = self.tables.last_mut().unwrap(); + let can_see_class_scope = + current.typ == CompilerScope::Class || current.can_see_class_scope; + let has_conditional = current.has_conditional_annotations; + + // Create annotation block if not exists + if current.annotation_block.is_none() { + let mut annotation_table = SymbolTable::new( + "__annotate__".to_owned(), + CompilerScope::Annotation, + line_number, + true, // is_nested + ); + // Annotation scope in class can see class scope + annotation_table.can_see_class_scope = can_see_class_scope; + // Add 'format' parameter + annotation_table.varnames.push("format".to_owned()); + current.annotation_block = Some(Box::new(annotation_table)); + } + + // Take the annotation block and push to stack for processing + let annotation_table = current.annotation_block.take().unwrap(); + self.tables.push(*annotation_table); + // Save parent's varnames and seed with existing annotation varnames (e.g., "format") + self.varnames_stack + .push(core::mem::take(&mut self.current_varnames)); + self.current_varnames = self.tables.last().unwrap().varnames.clone(); + + if can_see_class_scope && !self.future_annotations { + self.add_classdict_freevar(); + // Also add __conditional_annotations__ as free var if parent has conditional annotations + if has_conditional { + self.add_conditional_annotations_freevar(); + } + } + } + + /// Leave annotation scope (PEP 649) + /// Stores the annotation block back to parent instead of sub_tables + fn leave_annotation_scope(&mut self) { + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = core::mem::take(&mut self.current_varnames); + // Store back to parent's annotation_block (not sub_tables) + let parent = self.tables.last_mut().unwrap(); + parent.annotation_block = Some(Box::new(table)); + // Restore parent's varnames + self.current_varnames = self.varnames_stack.pop().unwrap_or_default(); + } + + fn add_classdict_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__classdict__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + fn add_conditional_annotations_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__conditional_annotations__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + /// Walk up the scope chain to determine if we're inside an async function. + /// Annotation and TypeParams scopes act as async barriers (always non-async). + /// Comprehension scopes are transparent (inherit parent's async context). + fn is_in_async_context(&self) -> bool { + for table in self.tables.iter().rev() { + match table.typ { + CompilerScope::AsyncFunction => return true, + CompilerScope::Function + | CompilerScope::Lambda + | CompilerScope::Class + | CompilerScope::Module + | CompilerScope::Annotation + | CompilerScope::TypeParams => return false, + // Comprehension inherits parent's async context + CompilerScope::Comprehension => continue, + } + } + false } fn line_index_start(&self, range: TextRange) -> u32 { @@ -700,39 +1076,106 @@ impl SymbolTableBuilder { .get() as _ } - fn scan_statements(&mut self, statements: &[Stmt]) -> SymbolTableResult { + fn scan_statements(&mut self, statements: &[ast::Stmt]) -> SymbolTableResult { for statement in statements { self.scan_statement(statement)?; } Ok(()) } - fn scan_parameters(&mut self, parameters: &[ParameterWithDefault]) -> SymbolTableResult { + fn scan_parameters(&mut self, parameters: &[ast::ParameterWithDefault]) -> SymbolTableResult { for parameter in parameters { self.scan_parameter(&parameter.parameter)?; } Ok(()) } - fn scan_parameter(&mut self, parameter: &Parameter) -> SymbolTableResult { + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.check_name( + parameter.name.as_str(), + ExpressionContext::Store, + parameter.name.range, + )?; + let usage = if parameter.annotation.is_some() { SymbolUsage::AnnotationParameter } else { SymbolUsage::Parameter }; + + // Check for duplicate parameter names + let table = self.tables.last().unwrap(); + if table.symbols.contains_key(parameter.name.as_str()) { + return Err(SymbolTableError { + error: format!( + "duplicate argument '{}' in function definition", + parameter.name + ), + location: Some( + self.source_file + .to_source_code() + .source_location(parameter.name.range.start(), PositionEncoding::Utf8), + ), + }); + } + self.register_ident(&parameter.name, usage) } - fn scan_annotation(&mut self, annotation: &Expr) -> SymbolTableResult { + fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult { + let current_scope = self.tables.last().map(|t| t.typ); + + // PEP 649: Check if this is a conditional annotation + // Module-level: always conditional (module may be partially executed) + // Class-level: conditional only when inside if/for/while/etc. + if !self.future_annotations { + let is_conditional = matches!(current_scope, Some(CompilerScope::Module)) + || (matches!(current_scope, Some(CompilerScope::Class)) + && self.in_conditional_block); + + if is_conditional && !self.tables.last().unwrap().has_conditional_annotations { + self.tables.last_mut().unwrap().has_conditional_annotations = true; + // Register __conditional_annotations__ as both Assigned and Used so that + // it becomes a Cell variable in class scope (children reference it as Free) + self.register_name( + "__conditional_annotations__", + SymbolUsage::Assigned, + annotation.range(), + )?; + self.register_name( + "__conditional_annotations__", + SymbolUsage::Used, + annotation.range(), + )?; + } + } + + // Create annotation scope for deferred evaluation + let line_number = self.line_index_start(annotation.range()); + self.enter_annotation_scope(line_number); + if self.future_annotations { - Ok(()) - } else { - self.scan_expression(annotation, ExpressionContext::Load) + // PEP 563: annotations are stringified at compile time + // Don't scan expression - symbols would fail to resolve + // Just create the annotation_block structure + self.leave_annotation_scope(); + return Ok(()); } + + // PEP 649: scan expression for symbol references + // Class annotations are evaluated in class locals (not module globals) + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + let result = self.scan_expression(annotation, ExpressionContext::Load); + self.in_annotation = was_in_annotation; + + self.leave_annotation_scope(); + + result } - fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { - use ruff_python_ast::*; + fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult { + use ast::*; if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement && module.as_ref().map(|id| id.as_str()) == Some("__future__") { @@ -759,30 +1202,70 @@ impl SymbolTableBuilder { type_params, returns, range, + is_async, .. }) => { self.scan_decorators(decorator_list, ExpressionContext::Load)?; self.register_ident(name, SymbolUsage::Assigned)?; - if let Some(expression) = returns { - self.scan_annotation(expression)?; + + // Save the parent's annotation_block before scanning function annotations, + // so function annotations don't interfere with parent scope annotations. + // This applies to both class scope (methods) and module scope (top-level functions). + let parent_scope_typ = self.tables.last().map(|t| t.typ); + let should_save_annotation_block = matches!( + parent_scope_typ, + Some(CompilerScope::Class) + | Some(CompilerScope::Module) + | Some(CompilerScope::Function) + | Some(CompilerScope::AsyncFunction) + ); + let saved_annotation_block = if should_save_annotation_block { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + // For generic functions, scan defaults before entering type_param_block + // (defaults are evaluated in the enclosing scope, not the type param scope) + let has_type_params = type_params.is_some(); + if has_type_params { + self.scan_parameter_defaults(parameters)?; } + + // For generic functions, enter type_param block FIRST so that + // annotation scopes are nested inside and can see type parameters. if let Some(type_params) = type_params { self.enter_type_param_block( &format!("<generic parameters of {}>", name.as_str()), self.line_index_start(type_params.range), + false, )?; self.scan_type_params(type_params)?; } + let has_return_annotation = if let Some(expression) = returns { + self.scan_annotation(expression)?; + true + } else { + false + }; self.enter_scope_with_parameters( name.as_str(), parameters, self.line_index_start(*range), + has_return_annotation, + *is_async, + has_type_params, // skip_defaults: already scanned above )?; self.scan_statements(body)?; self.leave_scope(); if type_params.is_some() { self.leave_scope(); } + + // Restore parent's annotation_block after processing the function + if let Some(block) = saved_annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } } Stmt::ClassDef(StmtClassDef { name, @@ -793,11 +1276,16 @@ impl SymbolTableBuilder { range, node_index: _, }) => { + // Save class_name for the entire ClassDef processing + let prev_class = self.class_name.take(); if let Some(type_params) = type_params { self.enter_type_param_block( &format!("<generic parameters of {}>", name.as_str()), self.line_index_start(type_params.range), + true, // for_class: enable selective mangling )?; + // Set class_name for mangling in type param scope + self.class_name = Some(name.to_string()); self.scan_type_params(type_params)?; } self.enter_scope( @@ -805,14 +1293,24 @@ impl SymbolTableBuilder { CompilerScope::Class, self.line_index_start(*range), ); - let prev_class = self.class_name.replace(name.to_string()); + // Reset in_conditional_block for new class scope + let saved_in_conditional = self.in_conditional_block; + self.in_conditional_block = false; + self.class_name = Some(name.to_string()); self.register_name("__module__", SymbolUsage::Assigned, *range)?; self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; self.register_name("__doc__", SymbolUsage::Assigned, *range)?; self.register_name("__class__", SymbolUsage::Assigned, *range)?; self.scan_statements(body)?; self.leave_scope(); - self.class_name = prev_class; + self.in_conditional_block = saved_in_conditional; + // For non-generic classes, restore class_name before base scanning. + // Bases are evaluated in the enclosing scope, not the class scope. + // For generic classes, bases are scanned within the type_param scope + // where class_name is already correctly set. + if type_params.is_none() { + self.class_name = prev_class.clone(); + } if let Some(arguments) = arguments { self.scan_expressions(&arguments.args, ExpressionContext::Load)?; for keyword in &arguments.keywords { @@ -822,6 +1320,8 @@ impl SymbolTableBuilder { if type_params.is_some() { self.leave_scope(); } + // Restore class_name after all ClassDef processing + self.class_name = prev_class; self.scan_decorators(decorator_list, ExpressionContext::Load)?; self.register_ident(name, SymbolUsage::Assigned)?; } @@ -835,6 +1335,9 @@ impl SymbolTableBuilder { .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for elif in elif_else_clauses { if let Some(test) = &elif.test { @@ -842,6 +1345,7 @@ impl SymbolTableBuilder { } self.scan_statements(&elif.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::For(StmtFor { target, @@ -852,15 +1356,23 @@ impl SymbolTableBuilder { }) => { self.scan_expression(target, ExpressionContext::Store)?; self.scan_expression(iter, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::While(StmtWhile { test, body, orelse, .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { // No symbols here. @@ -870,14 +1382,25 @@ impl SymbolTableBuilder { for name in names { if let Some(alias) = &name.asname { // `import my_module as my_alias` + self.check_name(alias.as_str(), ExpressionContext::Store, alias.range)?; self.register_ident(alias, SymbolUsage::Imported)?; + } else if name.name.as_str() == "*" { + // Star imports are only allowed at module level + if self.tables.last().unwrap().typ != CompilerScope::Module { + return Err(SymbolTableError { + error: "'import *' only allowed at module level".to_string(), + location: Some(self.source_file.to_source_code().source_location( + name.name.range.start(), + PositionEncoding::Utf8, + )), + }); + } + // Don't register star imports as symbols } else { - // `import module` - self.register_name( - name.name.split('.').next().unwrap(), - SymbolUsage::Imported, - name.name.range, - )?; + // `import module` or `from x import name` + let imported_name = name.name.split('.').next().unwrap(); + self.check_name(imported_name, ExpressionContext::Store, name.name.range)?; + self.register_name(imported_name, SymbolUsage::Imported, name.name.range)?; } } } @@ -914,7 +1437,26 @@ impl SymbolTableBuilder { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &**target { Expr::Name(ast::ExprName { id, .. }) if *simple => { - self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, *range)?; + let id_str = id.as_str(); + + self.check_name(id_str, ExpressionContext::Store, *range)?; + + self.register_name(id_str, SymbolUsage::AnnotationAssigned, *range)?; + // PEP 649: Register annotate function in module/class scope + let current_scope = self.tables.last().map(|t| t.typ); + match current_scope { + Some(CompilerScope::Module) => { + self.register_name("__annotate__", SymbolUsage::Assigned, *range)?; + } + Some(CompilerScope::Class) => { + self.register_name( + "__annotate_func__", + SymbolUsage::Assigned, + *range, + )?; + } + _ => {} + } } _ => { self.scan_expression(target, ExpressionContext::Store)?; @@ -932,7 +1474,11 @@ impl SymbolTableBuilder { self.scan_expression(expression, ExpressionContext::Store)?; } } + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Try(StmtTry { body, @@ -941,6 +1487,9 @@ impl SymbolTableBuilder { finalbody, .. }) => { + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for handler in handlers { let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -959,9 +1508,13 @@ impl SymbolTableBuilder { } self.scan_statements(orelse)?; self.scan_statements(finalbody)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Match(StmtMatch { subject, cases, .. }) => { self.scan_expression(subject, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; for case in cases { self.scan_pattern(&case.pattern)?; if let Some(guard) = &case.guard { @@ -969,6 +1522,7 @@ impl SymbolTableBuilder { } self.scan_statements(&case.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::Raise(StmtRaise { exc, cause, .. }) => { if let Some(expression) = exc { @@ -984,17 +1538,42 @@ impl SymbolTableBuilder { type_params, .. }) => { + let was_in_type_alias = self.in_type_alias; + self.in_type_alias = true; + // Check before entering any sub-scopes + let in_class = self + .tables + .last() + .is_some_and(|t| t.typ == CompilerScope::Class); + let is_generic = type_params.is_some(); if let Some(type_params) = type_params { self.enter_type_param_block( "TypeAlias", self.line_index_start(type_params.range), + false, )?; self.scan_type_params(type_params)?; - self.scan_expression(value, ExpressionContext::Load)?; + } + // Value scope for lazy evaluation + self.enter_scope( + "TypeAlias", + CompilerScope::TypeParams, + self.line_index_start(value.range()), + ); + // Evaluator takes a format parameter + self.register_name("format", SymbolUsage::Parameter, TextRange::default())?; + if in_class { + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = true; + } + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + self.scan_expression(value, ExpressionContext::Load)?; + self.leave_scope(); + if is_generic { self.leave_scope(); - } else { - self.scan_expression(value, ExpressionContext::Load)?; } + self.in_type_alias = was_in_type_alias; self.scan_expression(name, ExpressionContext::Store)?; } Stmt::IpyEscapeCommand(_) => todo!(), @@ -1004,7 +1583,7 @@ impl SymbolTableBuilder { fn scan_decorators( &mut self, - decorators: &[Decorator], + decorators: &[ast::Decorator], context: ExpressionContext, ) -> SymbolTableResult { for decorator in decorators { @@ -1015,7 +1594,7 @@ impl SymbolTableBuilder { fn scan_expressions( &mut self, - expressions: &[Expr], + expressions: &[ast::Expr], context: ExpressionContext, ) -> SymbolTableResult { for expression in expressions { @@ -1026,29 +1605,45 @@ impl SymbolTableBuilder { fn scan_expression( &mut self, - expression: &Expr, + expression: &ast::Expr, context: ExpressionContext, ) -> SymbolTableResult { - use ruff_python_ast::*; + use ast::*; + + // Check for expressions not allowed in certain contexts + // (type parameters, annotations, type aliases, TypeVar bounds/defaults) + if let Some(keyword) = match expression { + Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), + Expr::Await(_) => Some("await"), + Expr::Named(_) => Some("named"), + _ => None, + } { + // Determine the context name for the error message + // scope_info takes precedence (e.g., "a TypeVar bound") + let context_name = if let Some(scope_info) = self.scope_info { + Some(scope_info) + } else if let Some(table) = self.tables.last() + && table.typ == CompilerScope::TypeParams + { + Some("a type parameter") + } else if self.in_annotation { + Some("an annotation") + } else if self.in_type_alias { + Some("a type alias") + } else { + None + }; - // Check for expressions not allowed in type parameters scope - if let Some(table) = self.tables.last() - && table.typ == CompilerScope::TypeParams - && let Some(keyword) = match expression { - Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), - Expr::Await(_) => Some("await"), - Expr::Named(_) => Some("named"), - _ => None, + if let Some(context_name) = context_name { + return Err(SymbolTableError { + error: format!("{keyword} expression cannot be used within {context_name}"), + location: Some( + self.source_file + .to_source_code() + .source_location(expression.range().start(), PositionEncoding::Utf8), + ), + }); } - { - return Err(SymbolTableError { - error: format!("{keyword} expression cannot be used within a type parameter"), - location: Some( - self.source_file - .to_source_code() - .source_location(expression.range().start(), PositionEncoding::Utf8), - ), - }); } match expression { @@ -1085,8 +1680,9 @@ impl SymbolTableBuilder { self.scan_expression(slice, ExpressionContext::Load)?; } Expr::Attribute(ExprAttribute { - value, range: _, .. + value, attr, range, .. }) => { + self.check_name(attr.as_str(), context, *range)?; self.scan_expression(value, ExpressionContext::Load)?; } Expr::Dict(ExprDict { @@ -1162,7 +1758,13 @@ impl SymbolTableBuilder { range, .. }) => { - self.scan_comprehension("genexpr", elt, None, generators, *range)?; + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Generator expression - is_generator = true + self.scan_comprehension("<genexpr>", elt, None, generators, *range, true)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::ListComp(ExprListComp { elt, @@ -1170,7 +1772,13 @@ impl SymbolTableBuilder { range, node_index: _, }) => { - self.scan_comprehension("genexpr", elt, None, generators, *range)?; + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // List comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<listcomp>", elt, None, generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::SetComp(ExprSetComp { elt, @@ -1178,7 +1786,13 @@ impl SymbolTableBuilder { range, node_index: _, }) => { - self.scan_comprehension("genexpr", elt, None, generators, *range)?; + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Set comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<setcomp>", elt, None, generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::DictComp(ExprDictComp { key, @@ -1187,7 +1801,13 @@ impl SymbolTableBuilder { range, node_index: _, }) => { - self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Dict comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<dictcomp>", key, Some(value), generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::Call(ExprCall { func, @@ -1206,11 +1826,17 @@ impl SymbolTableBuilder { self.scan_expressions(&arguments.args, ExpressionContext::Load)?; for keyword in &arguments.keywords { + if let Some(arg) = &keyword.arg { + self.check_name(arg.as_str(), ExpressionContext::Store, keyword.range)?; + } self.scan_expression(&keyword.value, ExpressionContext::Load)?; } } Expr::Name(ExprName { id, range, .. }) => { let id = id.as_str(); + + self.check_name(id, context, *range)?; + // Determine the contextual usage of this symbol: match context { ExpressionContext::Delete => { @@ -1230,7 +1856,10 @@ impl SymbolTableBuilder { // Interesting stuff about the __class__ variable: // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object if context == ExpressionContext::Load - && self.tables.last().unwrap().typ == CompilerScope::Function + && matches!( + self.tables.last().unwrap().typ, + CompilerScope::Function | CompilerScope::AsyncFunction + ) && id == "super" { self.register_name("__class__", SymbolUsage::Used, *range)?; @@ -1247,6 +1876,9 @@ impl SymbolTableBuilder { "lambda", parameters, self.line_index_start(expression.range()), + false, // lambdas have no return annotation + false, // lambdas are never async + false, // don't skip defaults )?; } else { self.enter_scope( @@ -1276,14 +1908,19 @@ impl SymbolTableBuilder { } } Expr::TString(tstring) => { - return Err(SymbolTableError { - error: "not yet implemented".into(), - location: Some( - self.source_file - .to_source_code() - .source_location(tstring.range.start(), PositionEncoding::Utf8), - ), - }); + // Scan t-string interpolation expressions (similar to f-strings) + for expr in tstring + .value + .elements() + .filter_map(|x| x.as_interpolation()) + { + self.scan_expression(&expr.expression, ExpressionContext::Load)?; + if let Some(format_spec) = &expr.format_spec { + for element in format_spec.elements.interpolations() { + self.scan_expression(&element.expression, ExpressionContext::Load)? + } + } + } } // Constants Expr::StringLiteral(_) @@ -1312,8 +1949,8 @@ impl SymbolTableBuilder { node_index: _, }) => { // named expressions are not allowed in the definition of - // comprehension iterator definitions - if let ExpressionContext::IterDefinitionExp = context { + // comprehension iterator definitions (including nested comprehensions) + if context == ExpressionContext::IterDefinitionExp || self.in_iter_def_exp { return Err(SymbolTableError { error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), location: Some(self.source_file.to_source_code().source_location(target.range().start(), PositionEncoding::Utf8)), @@ -1328,6 +1965,7 @@ impl SymbolTableBuilder { // propagate inner names. if let Expr::Name(ExprName { id, .. }) = &**target { let id = id.as_str(); + self.check_name(id, ExpressionContext::Store, *range)?; let table = self.tables.last().unwrap(); if table.typ == CompilerScope::Comprehension { self.register_name( @@ -1352,11 +1990,26 @@ impl SymbolTableBuilder { fn scan_comprehension( &mut self, scope_name: &str, - elt1: &Expr, - elt2: Option<&Expr>, - generators: &[Comprehension], + elt1: &ast::Expr, + elt2: Option<&ast::Expr>, + generators: &[ast::Comprehension], range: TextRange, + is_generator: bool, ) -> SymbolTableResult { + // Check for async comprehension outside async function + // (list/set/dict comprehensions only, not generator expressions) + let has_async_gen = generators.iter().any(|g| g.is_async); + if has_async_gen && !is_generator && !self.is_in_async_context() { + return Err(SymbolTableError { + error: "asynchronous comprehension outside of an asynchronous function".to_owned(), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + // Comprehensions are compiled as functions, so create a scope for them: self.enter_scope( scope_name, @@ -1364,6 +2017,21 @@ impl SymbolTableBuilder { self.line_index_start(range), ); + // Mark non-generator comprehensions as inlined (PEP 709) + // inline_comp = entry->ste_comprehension && !entry->ste_generator && !ste->ste_can_see_class_scope + // We check is_generator and can_see_class_scope of parent + let parent_can_see_class = self + .tables + .get(self.tables.len().saturating_sub(2)) + .map(|t| t.can_see_class_scope) + .unwrap_or(false); + if !is_generator + && !parent_can_see_class + && let Some(table) = self.tables.last_mut() + { + table.comp_inlined = true; + } + // Register the passed argument to the generator function as the name ".0" self.register_name(".0", SymbolUsage::Parameter, range)?; @@ -1374,7 +2042,13 @@ impl SymbolTableBuilder { let mut is_first_generator = true; for generator in generators { + // Set flag for INNER_LOOP_CONFLICT check (only for inner loops, not the first) + if !is_first_generator { + self.in_comp_inner_loop_target = true; + } self.scan_expression(&generator.target, ExpressionContext::Iter)?; + self.in_comp_inner_loop_target = false; + if is_first_generator { is_first_generator = false; } else { @@ -1397,32 +2071,88 @@ impl SymbolTableBuilder { /// Scan type parameter bound or default in a separate scope // = symtable_visit_type_param_bound_or_default - fn scan_type_param_bound_or_default(&mut self, expr: &Expr, name: &str) -> SymbolTableResult { + fn scan_type_param_bound_or_default( + &mut self, + expr: &ast::Expr, + scope_name: &str, + scope_info: &'static str, + ) -> SymbolTableResult { // Enter a new TypeParams scope for the bound/default expression // This allows the expression to access outer scope symbols + let in_class = self.tables.last().is_some_and(|t| t.can_see_class_scope); let line_number = self.line_index_start(expr.range()); - self.enter_scope(name, CompilerScope::TypeParams, line_number); + self.enter_scope(scope_name, CompilerScope::TypeParams, line_number); + // Evaluator takes a format parameter + self.register_name("format", SymbolUsage::Parameter, TextRange::default())?; + + if in_class { + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = true; + } + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } - // Note: In CPython, can_see_class_scope is preserved in the new scope - // In RustPython, this is handled through the scope hierarchy + // Set scope_info for better error messages + let old_scope_info = self.scope_info; + self.scope_info = Some(scope_info); // Scan the expression in this new scope let result = self.scan_expression(expr, ExpressionContext::Load); - // Exit the scope + // Restore scope_info and exit the scope + self.scope_info = old_scope_info; self.leave_scope(); result } - fn scan_type_params(&mut self, type_params: &TypeParams) -> SymbolTableResult { + fn scan_type_params(&mut self, type_params: &ast::TypeParams) -> SymbolTableResult { + // Check for duplicate type parameter names + let mut seen_names: IndexSet<&str> = IndexSet::default(); + // Check for non-default type parameter after default type parameter + let mut default_seen = false; + for type_param in &type_params.type_params { + let (name, range, has_default) = match type_param { + ast::TypeParam::TypeVar(tv) => (tv.name.as_str(), tv.range, tv.default.is_some()), + ast::TypeParam::ParamSpec(ps) => (ps.name.as_str(), ps.range, ps.default.is_some()), + ast::TypeParam::TypeVarTuple(tvt) => { + (tvt.name.as_str(), tvt.range, tvt.default.is_some()) + } + }; + if !seen_names.insert(name) { + return Err(SymbolTableError { + error: format!("duplicate type parameter '{}'", name), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + if has_default { + default_seen = true; + } else if default_seen { + return Err(SymbolTableError { + error: format!( + "non-default type parameter '{}' follows default type parameter", + name + ), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + } + // Register .type_params as a type parameter (automatically becomes cell variable) self.register_name(".type_params", SymbolUsage::TypeParam, type_params.range)?; // First register all type parameters for type_param in &type_params.type_params { match type_param { - TypeParam::TypeVar(TypeParamTypeVar { + ast::TypeParam::TypeVar(ast::TypeParamTypeVar { name, bound, range: type_var_range, @@ -1433,21 +2163,28 @@ impl SymbolTableBuilder { // Process bound in a separate scope if let Some(binding) = bound { - let scope_name = if binding.is_tuple_expr() { - format!("<TypeVar constraint of {name}>") + let (scope_name, scope_info) = if binding.is_tuple_expr() { + ( + format!("<TypeVar constraint of {name}>"), + "a TypeVar constraint", + ) } else { - format!("<TypeVar bound of {name}>") + (format!("<TypeVar bound of {name}>"), "a TypeVar bound") }; - self.scan_type_param_bound_or_default(binding, &scope_name)?; + self.scan_type_param_bound_or_default(binding, &scope_name, scope_info)?; } // Process default in a separate scope if let Some(default_value) = default { let scope_name = format!("<TypeVar default of {name}>"); - self.scan_type_param_bound_or_default(default_value, &scope_name)?; + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a TypeVar default", + )?; } } - TypeParam::ParamSpec(TypeParamParamSpec { + ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { name, range: param_spec_range, default, @@ -1458,10 +2195,14 @@ impl SymbolTableBuilder { // Process default in a separate scope if let Some(default_value) = default { let scope_name = format!("<ParamSpec default of {name}>"); - self.scan_type_param_bound_or_default(default_value, &scope_name)?; + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a ParamSpec default", + )?; } } - TypeParam::TypeVarTuple(TypeParamTypeVarTuple { + ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { name, range: type_var_tuple_range, default, @@ -1472,7 +2213,11 @@ impl SymbolTableBuilder { // Process default in a separate scope if let Some(default_value) = default { let scope_name = format!("<TypeVarTuple default of {name}>"); - self.scan_type_param_bound_or_default(default_value, &scope_name)?; + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a TypeVarTuple default", + )?; } } } @@ -1480,22 +2225,24 @@ impl SymbolTableBuilder { Ok(()) } - fn scan_patterns(&mut self, patterns: &[Pattern]) -> SymbolTableResult { + fn scan_patterns(&mut self, patterns: &[ast::Pattern]) -> SymbolTableResult { for pattern in patterns { self.scan_pattern(pattern)?; } Ok(()) } - fn scan_pattern(&mut self, pattern: &Pattern) -> SymbolTableResult { - use Pattern::*; + fn scan_pattern(&mut self, pattern: &ast::Pattern) -> SymbolTableResult { + use ast::Pattern::*; match pattern { - MatchValue(PatternMatchValue { value, .. }) => { + MatchValue(ast::PatternMatchValue { value, .. }) => { self.scan_expression(value, ExpressionContext::Load)? } MatchSingleton(_) => {} - MatchSequence(PatternMatchSequence { patterns, .. }) => self.scan_patterns(patterns)?, - MatchMapping(PatternMatchMapping { + MatchSequence(ast::PatternMatchSequence { patterns, .. }) => { + self.scan_patterns(patterns)? + } + MatchMapping(ast::PatternMatchMapping { keys, patterns, rest, @@ -1507,19 +2254,19 @@ impl SymbolTableBuilder { self.register_ident(rest, SymbolUsage::Assigned)?; } } - MatchClass(PatternMatchClass { cls, arguments, .. }) => { + MatchClass(ast::PatternMatchClass { cls, arguments, .. }) => { self.scan_expression(cls, ExpressionContext::Load)?; self.scan_patterns(&arguments.patterns)?; for kw in &arguments.keywords { self.scan_pattern(&kw.pattern)?; } } - MatchStar(PatternMatchStar { name, .. }) => { + MatchStar(ast::PatternMatchStar { name, .. }) => { if let Some(name) = name { self.register_ident(name, SymbolUsage::Assigned)?; } } - MatchAs(PatternMatchAs { pattern, name, .. }) => { + MatchAs(ast::PatternMatchAs { pattern, name, .. }) => { if let Some(pattern) = pattern { self.scan_pattern(pattern)?; } @@ -1527,18 +2274,13 @@ impl SymbolTableBuilder { self.register_ident(name, SymbolUsage::Assigned)?; } } - MatchOr(PatternMatchOr { patterns, .. }) => self.scan_patterns(patterns)?, + MatchOr(ast::PatternMatchOr { patterns, .. }) => self.scan_patterns(patterns)?, } Ok(()) } - fn enter_scope_with_parameters( - &mut self, - name: &str, - parameters: &Parameters, - line_number: u32, - ) -> SymbolTableResult { - // Evaluate eventual default parameters: + /// Scan default parameter values (evaluated in the enclosing scope) + fn scan_parameter_defaults(&mut self, parameters: &ast::Parameters) -> SymbolTableResult { for default in parameters .posonlyargs .iter() @@ -1546,7 +2288,23 @@ impl SymbolTableBuilder { .chain(parameters.kwonlyargs.iter()) .filter_map(|arg| arg.default.as_ref()) { - self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext? + self.scan_expression(default, ExpressionContext::Load)?; + } + Ok(()) + } + + fn enter_scope_with_parameters( + &mut self, + name: &str, + parameters: &ast::Parameters, + line_number: u32, + has_return_annotation: bool, + is_async: bool, + skip_defaults: bool, + ) -> SymbolTableResult { + // Evaluate eventual default parameters (unless already scanned before type_param_block): + if !skip_defaults { + self.scan_parameter_defaults(parameters)?; } // Annotations are scanned in outer scope: @@ -1574,7 +2332,44 @@ impl SymbolTableBuilder { self.scan_annotation(annotation)?; } - self.enter_scope(name, CompilerScope::Function, line_number); + // Check if this function has any annotations (parameter or return) + let has_param_annotations = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .any(|p| p.parameter.annotation.is_some()) + || parameters + .vararg + .as_ref() + .is_some_and(|p| p.annotation.is_some()) + || parameters + .kwarg + .as_ref() + .is_some_and(|p| p.annotation.is_some()); + + let has_any_annotations = has_param_annotations || has_return_annotation; + + // Take annotation_block if this function has any annotations. + // When in class scope, the class's annotation_block was saved before scanning + // function annotations, so the current annotation_block belongs to this function. + let annotation_block = if has_any_annotations { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + let scope_type = if is_async { + CompilerScope::AsyncFunction + } else { + CompilerScope::Function + }; + self.enter_scope(name, scope_type, line_number); + + // Move annotation_block to function scope only if we have one + if let Some(block) = annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } // Fill scope with parameter names: self.scan_parameters(&parameters.posonlyargs)?; @@ -1589,10 +2384,41 @@ impl SymbolTableBuilder { Ok(()) } - fn register_ident(&mut self, ident: &Identifier, role: SymbolUsage) -> SymbolTableResult { + fn register_ident(&mut self, ident: &ast::Identifier, role: SymbolUsage) -> SymbolTableResult { self.register_name(ident.as_str(), role, ident.range) } + fn check_name( + &self, + name: &str, + context: ExpressionContext, + range: TextRange, + ) -> SymbolTableResult { + if name == "__debug__" { + let location = Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ); + match context { + ExpressionContext::Store | ExpressionContext::Iter => { + return Err(SymbolTableError { + error: "cannot assign to __debug__".to_owned(), + location, + }); + } + ExpressionContext::Delete => { + return Err(SymbolTableError { + error: "cannot delete __debug__".to_owned(), + location, + }); + } + _ => {} + } + } + Ok(()) + } + fn register_name( &mut self, name: &str, @@ -1604,13 +2430,44 @@ impl SymbolTableBuilder { .to_source_code() .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); + + // Note: __debug__ checks are handled by check_name function, so no check needed here. + let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); - let name = mangle_name(self.class_name.as_deref(), name); + // Add type param names to mangled_names set for selective mangling + if matches!(role, SymbolUsage::TypeParam) + && let Some(ref mut set) = table.mangled_names + { + set.insert(name.to_owned()); + } + + let name = maybe_mangle_name( + self.class_name.as_deref(), + table.mangled_names.as_ref(), + name, + ); // Some checks for the symbol that present on this scope level: let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) { let flags = &symbol.flags; + + // INNER_LOOP_CONFLICT: comprehension inner loop cannot rebind + // a variable that was used as a named expression target + // Example: [i for i in range(5) if (j := 0) for j in range(5)] + // Here 'j' is used in named expr first, then as inner loop iter target + if self.in_comp_inner_loop_target + && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) + { + return Err(SymbolTableError { + error: format!( + "comprehension inner loop cannot rebind assignment expression target '{}'", + name + ), + location, + }); + } + // Role already set.. match role { SymbolUsage::Global if !symbol.is_global() => { @@ -1745,6 +2602,7 @@ impl SymbolTableBuilder { } SymbolUsage::Global => { symbol.scope = SymbolScope::GlobalExplicit; + flags.insert(SymbolFlags::GLOBAL); } SymbolUsage::Used => { flags.insert(SymbolFlags::REFERENCED); @@ -1753,21 +2611,20 @@ impl SymbolTableBuilder { flags.insert(SymbolFlags::ITER); } SymbolUsage::TypeParam => { - // Type parameters are always cell variables in their scope - symbol.scope = SymbolScope::Cell; - flags.insert(SymbolFlags::ASSIGNED); + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::TYPE_PARAM); } } // and even more checking // it is not allowed to assign to iterator variables (by named expressions) - if flags.contains(SymbolFlags::ITER | SymbolFlags::ASSIGNED) - /*&& symbol.is_assign_named_expr_in_comprehension*/ + if flags.contains(SymbolFlags::ITER) + && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) { return Err(SymbolTableError { - error: - "assignment expression cannot be used in a comprehension iterable expression" - .to_string(), + error: format!( + "assignment expression cannot rebind comprehension iteration variable '{}'", + symbol.name + ), location, }); } @@ -1783,11 +2640,27 @@ pub(crate) fn mangle_name<'a>(class_name: Option<&str>, name: &'a str) -> Cow<'a if !name.starts_with("__") || name.ends_with("__") || name.contains('.') { return name.into(); } - // strip leading underscore - let class_name = class_name.strip_prefix(|c| c == '_').unwrap_or(class_name); + // Strip leading underscores from class name + let class_name = class_name.trim_start_matches('_'); let mut ret = String::with_capacity(1 + class_name.len() + name.len()); ret.push('_'); ret.push_str(class_name); ret.push_str(name); ret.into() } + +/// Selective mangling for type parameter scopes around generic classes. +/// If `mangled_names` is Some, only mangle names that are in the set; +/// other names are left unmangled. +pub(crate) fn maybe_mangle_name<'a>( + class_name: Option<&str>, + mangled_names: Option<&IndexSet<String>>, + name: &'a str, +) -> Cow<'a, str> { + if let Some(set) = mangled_names + && !set.contains(name) + { + return name.into(); + } + mangle_name(class_name, name) +} diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 74e35fd5e2a..a590323cb78 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -1,11 +1,9 @@ -use ruff_python_ast::{ - self as ruff, Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, - Parameter, ParameterWithDefault, Parameters, -}; +use alloc::fmt; +use core::fmt::Display as _; +use ruff_python_ast as ast; use ruff_text_size::Ranged; use rustpython_compiler_core::SourceFile; use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; -use std::fmt::{self, Display as _}; mod precedence { macro_rules! precedence { @@ -39,7 +37,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.f.write_str(s) } - fn p_id(&mut self, s: &Identifier) -> fmt::Result { + fn p_id(&mut self, s: &ast::Identifier) -> fmt::Result { self.f.write_str(s.as_str()) } @@ -51,14 +49,14 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { - self.p_if(!std::mem::take(first), s) + self.p_if(!core::mem::take(first), s) } fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { self.f.write_fmt(f) } - fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { + fn unparse_expr(&mut self, ast: &ast::Expr, level: u8) -> fmt::Result { macro_rules! op_prec { ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { match $x { @@ -82,13 +80,13 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { }}; } match &ast { - Expr::BoolOp(ruff::ExprBoolOp { + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, node_index: _, range: _range, }) => { - let (op, prec) = op_prec!(bin, op, BoolOp, And("and", AND), Or("or", OR)); + let (op, prec) = op_prec!(bin, op, ast::BoolOp, And("and", AND), Or("or", OR)); group_if!(prec, { let mut first = true; for val in values { @@ -97,7 +95,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } }) } - Expr::Named(ruff::ExprNamed { + ast::Expr::Named(ast::ExprNamed { target, value, node_index: _, @@ -109,18 +107,18 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(value, precedence::ATOM)?; }) } - Expr::BinOp(ruff::ExprBinOp { + ast::Expr::BinOp(ast::ExprBinOp { left, op, right, node_index: _, range: _range, }) => { - let right_associative = matches!(op, Operator::Pow); + let right_associative = matches!(op, ast::Operator::Pow); let (op, prec) = op_prec!( bin, op, - Operator, + ast::Operator, Add("+", ARITH), Sub("-", ARITH), Mult("*", TERM), @@ -141,7 +139,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(right, prec + !right_associative as u8)?; }) } - Expr::UnaryOp(ruff::ExprUnaryOp { + ast::Expr::UnaryOp(ast::ExprUnaryOp { op, operand, node_index: _, @@ -150,7 +148,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { let (op, prec) = op_prec!( un, op, - ruff::UnaryOp, + ast::UnaryOp, Invert("~", FACTOR), Not("not ", NOT), UAdd("+", FACTOR), @@ -161,7 +159,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(operand, prec)?; }) } - Expr::Lambda(ruff::ExprLambda { + ast::Expr::Lambda(ast::ExprLambda { parameters, body, node_index: _, @@ -177,7 +175,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { write!(self, ": {}", UnparseExpr::new(body, self.source))?; }) } - Expr::If(ruff::ExprIf { + ast::Expr::If(ast::ExprIf { test, body, orelse, @@ -192,7 +190,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(orelse, precedence::TEST)?; }) } - Expr::Dict(ruff::ExprDict { + ast::Expr::Dict(ast::ExprDict { items, node_index: _, range: _range, @@ -210,7 +208,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } self.p("}")?; } - Expr::Set(ruff::ExprSet { + ast::Expr::Set(ast::ExprSet { elts, node_index: _, range: _range, @@ -223,7 +221,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } self.p("}")?; } - Expr::ListComp(ruff::ExprListComp { + ast::Expr::ListComp(ast::ExprListComp { elt, generators, node_index: _, @@ -234,7 +232,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_comp(generators)?; self.p("]")?; } - Expr::SetComp(ruff::ExprSetComp { + ast::Expr::SetComp(ast::ExprSetComp { elt, generators, node_index: _, @@ -245,7 +243,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_comp(generators)?; self.p("}")?; } - Expr::DictComp(ruff::ExprDictComp { + ast::Expr::DictComp(ast::ExprDictComp { key, value, generators, @@ -259,7 +257,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_comp(generators)?; self.p("}")?; } - Expr::Generator(ruff::ExprGenerator { + ast::Expr::Generator(ast::ExprGenerator { parenthesized: _, elt, generators, @@ -271,7 +269,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_comp(generators)?; self.p(")")?; } - Expr::Await(ruff::ExprAwait { + ast::Expr::Await(ast::ExprAwait { value, node_index: _, range: _range, @@ -281,7 +279,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(value, precedence::ATOM)?; }) } - Expr::Yield(ruff::ExprYield { + ast::Expr::Yield(ast::ExprYield { value, node_index: _, range: _range, @@ -292,7 +290,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p("(yield)")?; } } - Expr::YieldFrom(ruff::ExprYieldFrom { + ast::Expr::YieldFrom(ast::ExprYieldFrom { value, node_index: _, range: _range, @@ -303,7 +301,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { UnparseExpr::new(value, self.source) )?; } - Expr::Compare(ruff::ExprCompare { + ast::Expr::Compare(ast::ExprCompare { left, ops, comparators, @@ -321,9 +319,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } }) } - Expr::Call(ruff::ExprCall { + ast::Expr::Call(ast::ExprCall { func, - arguments: Arguments { args, keywords, .. }, + arguments: ast::Arguments { args, keywords, .. }, node_index: _, range: _range, }) => { @@ -331,7 +329,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p("(")?; if let ( [ - Expr::Generator(ruff::ExprGenerator { + ast::Expr::Generator(ast::ExprGenerator { elt, generators, node_index: _, @@ -364,9 +362,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } self.p(")")?; } - Expr::FString(ruff::ExprFString { value, .. }) => self.unparse_fstring(value)?, - Expr::TString(_) => self.p("t\"\"")?, - Expr::StringLiteral(ruff::ExprStringLiteral { value, .. }) => { + ast::Expr::FString(ast::ExprFString { value, .. }) => self.unparse_fstring(value)?, + ast::Expr::TString(ast::ExprTString { value, .. }) => self.unparse_tstring(value)?, + ast::Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) => { if value.is_unicode() { self.p("u")? } @@ -374,12 +372,12 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { .str_repr() .fmt(self.f)? } - Expr::BytesLiteral(ruff::ExprBytesLiteral { value, .. }) => { + ast::Expr::BytesLiteral(ast::ExprBytesLiteral { value, .. }) => { AsciiEscape::new_repr(&value.bytes().collect::<Vec<_>>()) .bytes_repr() .fmt(self.f)? } - Expr::NumberLiteral(ruff::ExprNumberLiteral { value, .. }) => { + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value, .. }) => { #[allow(clippy::correctness, clippy::assertions_on_constants)] const { assert!(f64::MAX_10_EXP == 308) @@ -387,28 +385,28 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { let inf_str = "1e309"; match value { - ruff::Number::Int(int) => int.fmt(self.f)?, - &ruff::Number::Float(fp) => { + ast::Number::Int(int) => int.fmt(self.f)?, + &ast::Number::Float(fp) => { if fp.is_infinite() { self.p(inf_str)? } else { self.p(&rustpython_literal::float::to_string(fp))? } } - &ruff::Number::Complex { real, imag } => self + &ast::Number::Complex { real, imag } => self .p(&rustpython_literal::complex::to_string(real, imag) .replace("inf", inf_str))?, } } - Expr::BooleanLiteral(ruff::ExprBooleanLiteral { value, .. }) => { + ast::Expr::BooleanLiteral(ast::ExprBooleanLiteral { value, .. }) => { self.p(if *value { "True" } else { "False" })? } - Expr::NoneLiteral(ruff::ExprNoneLiteral { .. }) => self.p("None")?, - Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { .. }) => self.p("...")?, - Expr::Attribute(ruff::ExprAttribute { value, attr, .. }) => { + ast::Expr::NoneLiteral(ast::ExprNoneLiteral { .. }) => self.p("None")?, + ast::Expr::EllipsisLiteral(ast::ExprEllipsisLiteral { .. }) => self.p("...")?, + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { self.unparse_expr(value, precedence::ATOM)?; - let period = if let Expr::NumberLiteral(ruff::ExprNumberLiteral { - value: ruff::Number::Int(_), + let period = if let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + value: ast::Number::Int(_), .. }) = value.as_ref() { @@ -419,19 +417,19 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(period)?; self.p_id(attr)?; } - Expr::Subscript(ruff::ExprSubscript { value, slice, .. }) => { + ast::Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => { self.unparse_expr(value, precedence::ATOM)?; let lvl = precedence::TUPLE; self.p("[")?; self.unparse_expr(slice, lvl)?; self.p("]")?; } - Expr::Starred(ruff::ExprStarred { value, .. }) => { + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { self.p("*")?; self.unparse_expr(value, precedence::EXPR)?; } - Expr::Name(ruff::ExprName { id, .. }) => self.p(id.as_str())?, - Expr::List(ruff::ExprList { elts, .. }) => { + ast::Expr::Name(ast::ExprName { id, .. }) => self.p(id.as_str())?, + ast::Expr::List(ast::ExprList { elts, .. }) => { self.p("[")?; let mut first = true; for elt in elts { @@ -440,7 +438,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } self.p("]")?; } - Expr::Tuple(ruff::ExprTuple { elts, .. }) => { + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { if elts.is_empty() { self.p("()")?; } else { @@ -454,7 +452,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { }) } } - Expr::Slice(ruff::ExprSlice { + ast::Expr::Slice(ast::ExprSlice { lower, upper, step, @@ -473,12 +471,12 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.unparse_expr(step, precedence::TEST)?; } } - Expr::IpyEscapeCommand(_) => {} + ast::Expr::IpyEscapeCommand(_) => {} } Ok(()) } - fn unparse_arguments(&mut self, args: &Parameters) -> fmt::Result { + fn unparse_arguments(&mut self, args: &ast::Parameters) -> fmt::Result { let mut first = true; for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { self.p_delim(&mut first, ", ")?; @@ -503,7 +501,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } Ok(()) } - fn unparse_function_arg(&mut self, arg: &ParameterWithDefault) -> fmt::Result { + fn unparse_function_arg(&mut self, arg: &ast::ParameterWithDefault) -> fmt::Result { self.unparse_arg(&arg.parameter)?; if let Some(default) = &arg.default { write!(self, "={}", UnparseExpr::new(default, self.source))?; @@ -511,7 +509,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_arg(&mut self, arg: &Parameter) -> fmt::Result { + fn unparse_arg(&mut self, arg: &ast::Parameter) -> fmt::Result { self.p_id(&arg.name)?; if let Some(ann) = &arg.annotation { write!(self, ": {}", UnparseExpr::new(ann, self.source))?; @@ -519,7 +517,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { + fn unparse_comp(&mut self, generators: &[ast::Comprehension]) -> fmt::Result { for comp in generators { self.p(if comp.is_async { " async for " @@ -537,10 +535,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_fstring_body( - &mut self, - elements: &[ruff::InterpolatedStringElement], - ) -> fmt::Result { + fn unparse_fstring_body(&mut self, elements: &[ast::InterpolatedStringElement]) -> fmt::Result { for elem in elements { self.unparse_fstring_elem(elem)?; } @@ -549,15 +544,15 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { fn unparse_formatted( &mut self, - val: &Expr, - debug_text: Option<&ruff::DebugText>, - conversion: ConversionFlag, - spec: Option<&ruff::InterpolatedStringFormatSpec>, + val: &ast::Expr, + debug_text: Option<&ast::DebugText>, + conversion: ast::ConversionFlag, + spec: Option<&ast::InterpolatedStringFormatSpec>, ) -> fmt::Result { - let buffered = to_string_fmt(|f| { - Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1) - }); - if let Some(ruff::DebugText { leading, trailing }) = debug_text { + let buffered = + fmt::from_fn(|f| Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1)) + .to_string(); + if let Some(ast::DebugText { leading, trailing }) = debug_text { self.p(leading)?; self.p(self.source.slice(val.range()))?; self.p(trailing)?; @@ -566,16 +561,28 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { // put a space to avoid escaping the bracket "{ " } else { - "{" + // Preserve leading whitespace between '{' and the expression + let source_text = self.source.source_text(); + let start = val.range().start().to_usize(); + if start > 0 + && source_text + .as_bytes() + .get(start - 1) + .is_some_and(|b| b.is_ascii_whitespace()) + { + "{ " + } else { + "{" + } }; self.p(brace)?; self.p(&buffered)?; drop(buffered); - if conversion != ConversionFlag::None { + if conversion != ast::ConversionFlag::None { self.p("!")?; let buf = &[conversion as u8]; - let c = std::str::from_utf8(buf).unwrap(); + let c = core::str::from_utf8(buf).unwrap(); self.p(c)?; } @@ -589,9 +596,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_fstring_elem(&mut self, elem: &ruff::InterpolatedStringElement) -> fmt::Result { + fn unparse_fstring_elem(&mut self, elem: &ast::InterpolatedStringElement) -> fmt::Result { match elem { - ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { expression, debug_text, conversion, @@ -603,7 +610,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { *conversion, format_spec.as_deref(), ), - ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { value, .. }) => self.unparse_fstring_str(value), @@ -615,30 +622,44 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(&s) } - fn unparse_fstring(&mut self, value: &ruff::FStringValue) -> fmt::Result { + fn unparse_fstring(&mut self, value: &ast::FStringValue) -> fmt::Result { self.p("f")?; - let body = to_string_fmt(|f| { + let body = fmt::from_fn(|f| { value.iter().try_for_each(|part| match part { - ruff::FStringPart::Literal(lit) => f.write_str(lit), - ruff::FStringPart::FString(ruff::FString { elements, .. }) => { + ast::FStringPart::Literal(lit) => f.write_str(lit), + ast::FStringPart::FString(ast::FString { elements, .. }) => { Unparser::new(f, self.source).unparse_fstring_body(elements) } }) - }); + }) + .to_string(); // .unparse_fstring_body(elements)); UnicodeEscape::new_repr(body.as_str().as_ref()) .str_repr() .write(self.f) } + + fn unparse_tstring(&mut self, value: &ast::TStringValue) -> fmt::Result { + self.p("t")?; + let body = fmt::from_fn(|f| { + value.iter().try_for_each(|tstring| { + Unparser::new(f, self.source).unparse_fstring_body(&tstring.elements) + }) + }) + .to_string(); + UnicodeEscape::new_repr(body.as_str().as_ref()) + .str_repr() + .write(self.f) + } } pub struct UnparseExpr<'a> { - expr: &'a Expr, + expr: &'a ast::Expr, source: &'a SourceFile, } impl<'a> UnparseExpr<'a> { - pub const fn new(expr: &'a Expr, source: &'a SourceFile) -> Self { + pub const fn new(expr: &'a ast::Expr, source: &'a SourceFile) -> Self { Self { expr, source } } } @@ -648,14 +669,3 @@ impl fmt::Display for UnparseExpr<'_> { Unparser::new(f, self.source).unparse_expr(self.expr, precedence::TEST) } } - -fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result) -> String { - use std::cell::Cell; - struct Fmt<F>(Cell<Option<F>>); - impl<F: FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result> fmt::Display for Fmt<F> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.take().unwrap()(f) - } - } - Fmt(Cell::new(Some(f))).to_string() -} diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 9fd7ea3880a..054e52ae81a 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -9,6 +9,8 @@ repository.workspace = true license.workspace = true [features] +default = ["std"] +std = [] threading = ["parking_lot"] wasm_js = ["getrandom/wasm_js"] @@ -26,7 +28,6 @@ malachite-bigint = { workspace = true } malachite-q = { workspace = true } malachite-base = { workspace = true } num-traits = { workspace = true } -once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } unicode_names2 = { workspace = true } radium = { workspace = true } diff --git a/crates/common/src/borrow.rs b/crates/common/src/borrow.rs index 610084006e1..d8389479b33 100644 --- a/crates/common/src/borrow.rs +++ b/crates/common/src/borrow.rs @@ -2,10 +2,8 @@ use crate::lock::{ MapImmutable, PyImmutableMappedMutexGuard, PyMappedMutexGuard, PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutexGuard, PyRwLockReadGuard, PyRwLockWriteGuard, }; -use std::{ - fmt, - ops::{Deref, DerefMut}, -}; +use alloc::fmt; +use core::ops::{Deref, DerefMut}; macro_rules! impl_from { ($lt:lifetime, $gen:ident, $t:ty, $($var:ident($from:ty),)*) => { diff --git a/crates/common/src/boxvec.rs b/crates/common/src/boxvec.rs index 8687ba7f7f5..3260e76ca87 100644 --- a/crates/common/src/boxvec.rs +++ b/crates/common/src/boxvec.rs @@ -2,13 +2,13 @@ //! An unresizable vector backed by a `Box<[T]>` #![allow(clippy::needless_lifetimes)] - -use std::{ +use alloc::{fmt, slice}; +use core::{ borrow::{Borrow, BorrowMut}, - cmp, fmt, + cmp, mem::{self, MaybeUninit}, ops::{Bound, Deref, DerefMut, RangeBounds}, - ptr, slice, + ptr, }; pub struct BoxVec<T> { @@ -555,7 +555,7 @@ impl<T> Extend<T> for BoxVec<T> { }; let mut iter = iter.into_iter(); loop { - if std::ptr::eq(ptr, end_ptr) { + if core::ptr::eq(ptr, end_ptr) { break; } if let Some(elt) = iter.next() { @@ -693,7 +693,7 @@ impl<T> CapacityError<T> { const CAPERROR: &str = "insufficient capacity"; -impl<T> std::error::Error for CapacityError<T> {} +impl<T> core::error::Error for CapacityError<T> {} impl<T> fmt::Display for CapacityError<T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs index b553f0b6b10..7b9609e90ae 100644 --- a/crates/common/src/cformat.rs +++ b/crates/common/src/cformat.rs @@ -1,19 +1,20 @@ //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use alloc::fmt; use bitflags::bitflags; +use core::{ + cmp, + iter::{Enumerate, Peekable}, + str::FromStr, +}; use itertools::Itertools; use malachite_bigint::{BigInt, Sign}; use num_traits::Signed; use rustpython_literal::{float, format::Case}; -use std::{ - cmp, fmt, - iter::{Enumerate, Peekable}, - str::FromStr, -}; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum CFormatErrorType { UnmatchedKeyParentheses, MissingModuloSign, @@ -26,7 +27,7 @@ pub enum CFormatErrorType { // also contains how many chars the parsing function consumed pub type ParsingError = (CFormatErrorType, usize); -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct CFormatError { pub typ: CFormatErrorType, // FIXME pub index: usize, @@ -785,7 +786,7 @@ impl<S> CFormatStrOrBytes<S> { if !literal.is_empty() { parts.push(( part_index, - CFormatPart::Literal(std::mem::take(&mut literal)), + CFormatPart::Literal(core::mem::take(&mut literal)), )); } let spec = CFormatSpecKeyed::parse(iter).map_err(|err| CFormatError { @@ -816,7 +817,7 @@ impl<S> CFormatStrOrBytes<S> { impl<S> IntoIterator for CFormatStrOrBytes<S> { type Item = (usize, CFormatPart<S>); - type IntoIter = std::vec::IntoIter<Self::Item>; + type IntoIter = alloc::vec::IntoIter<Self::Item>; fn into_iter(self) -> Self::IntoIter { self.parts.into_iter() diff --git a/crates/common/src/crt_fd.rs b/crates/common/src/crt_fd.rs index 7b8279adbe3..ab7c94f8b3b 100644 --- a/crates/common/src/crt_fd.rs +++ b/crates/common/src/crt_fd.rs @@ -1,10 +1,14 @@ //! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's //! returned from libc::open, even on windows. -use std::{cmp, ffi, fmt, io}; +use alloc::fmt; +use core::cmp; +use std::{ffi, io}; +#[cfg(unix)] +use std::os::fd::AsFd; #[cfg(not(windows))] -use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; #[cfg(windows)] use std::os::windows::io::BorrowedHandle; @@ -60,8 +64,8 @@ type BorrowedInner<'fd> = BorrowedFd<'fd>; #[cfg(windows)] mod win { use super::*; - use std::marker::PhantomData; - use std::mem::ManuallyDrop; + use core::marker::PhantomData; + use core::mem::ManuallyDrop; #[repr(transparent)] pub(super) struct OwnedInner(i32); @@ -334,7 +338,21 @@ pub fn close(fd: Owned) -> io::Result<()> { } pub fn ftruncate(fd: Borrowed<'_>, len: Offset) -> io::Result<()> { - cvt(unsafe { suppress_iph!(c::ftruncate(fd.as_raw(), len)) })?; + let ret = unsafe { suppress_iph!(c::ftruncate(fd.as_raw(), len)) }; + // On Windows, _chsize_s returns 0 on success, or a positive error code (errno value) on failure. + // On other platforms, ftruncate returns 0 on success, or -1 on failure with errno set. + #[cfg(windows)] + { + if ret != 0 { + // _chsize_s returns errno directly, convert to Windows error code + let winerror = crate::os::errno_to_winerror(ret); + return Err(io::Error::from_raw_os_error(winerror)); + } + } + #[cfg(not(windows))] + { + cvt(ret)?; + } Ok(()) } diff --git a/crates/common/src/encodings.rs b/crates/common/src/encodings.rs index 39ca2661262..913f0521e16 100644 --- a/crates/common/src/encodings.rs +++ b/crates/common/src/encodings.rs @@ -1,4 +1,4 @@ -use std::ops::{self, Range}; +use core::ops::{self, Range}; use num_traits::ToPrimitive; @@ -260,8 +260,9 @@ pub mod errors { use crate::str::UnicodeEscapeCodepoint; use super::*; - use std::fmt::Write; + use core::fmt::Write; + #[derive(Clone, Copy)] pub struct Strict; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Strict { @@ -286,6 +287,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct Ignore; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Ignore { @@ -310,6 +312,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct Replace; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Replace { @@ -338,6 +341,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct XmlCharRefReplace; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for XmlCharRefReplace { @@ -358,6 +362,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct BackslashReplace; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for BackslashReplace { @@ -394,6 +399,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct NameReplace; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for NameReplace { @@ -422,6 +428,7 @@ pub mod errors { } } + #[derive(Clone, Copy)] pub struct SurrogateEscape; impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for SurrogateEscape { @@ -434,13 +441,22 @@ pub mod errors { let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; let num_chars = range.end.chars - range.start.chars; let mut out = Vec::with_capacity(num_chars); + let mut pos = range.start; for ch in err_str.code_points() { - let ch = ch.to_u32(); - if !(0xdc80..=0xdcff).contains(&ch) { - // Not a UTF-8b surrogate, fail with original exception - return Err(ctx.error_encoding(range, reason)); + let ch_u32 = ch.to_u32(); + if !(0xdc80..=0xdcff).contains(&ch_u32) { + if out.is_empty() { + // Can't handle even the first character + return Err(ctx.error_encoding(range, reason)); + } + // Return partial result, restart from this character + return Ok((EncodeReplace::Bytes(ctx.bytes(out)), pos)); } - out.push((ch - 0xdc00) as u8); + out.push((ch_u32 - 0xdc00) as u8); + pos += StrSize { + bytes: ch.len_wtf8(), + chars: 1, + }; } Ok((EncodeReplace::Bytes(ctx.bytes(out)), range.end)) } diff --git a/crates/common/src/fileutils.rs b/crates/common/src/fileutils.rs index a12c1cd82e5..a20140a6e04 100644 --- a/crates/common/src/fileutils.rs +++ b/crates/common/src/fileutils.rs @@ -9,7 +9,7 @@ pub use windows::{StatStruct, fstat}; #[cfg(not(windows))] pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result<StatStruct> { - let mut stat = std::mem::MaybeUninit::uninit(); + let mut stat = core::mem::MaybeUninit::uninit(); unsafe { let ret = libc::fstat(fd.as_raw(), stat.as_mut_ptr()); if ret == -1 { @@ -24,8 +24,9 @@ pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result<StatStruct> { pub mod windows { use crate::crt_fd; use crate::windows::ToWideString; + use alloc::ffi::CString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{CString, OsStr, OsString}; + use std::ffi::{OsStr, OsString}; use std::os::windows::io::AsRawHandle; use std::sync::OnceLock; use windows_sys::Win32::Foundation::{ @@ -46,7 +47,7 @@ pub mod windows { pub const SECS_BETWEEN_EPOCHS: i64 = 11644473600; // Seconds between 1.1.1601 and 1.1.1970 - #[derive(Default)] + #[derive(Clone, Copy, Default)] pub struct StatStruct { pub st_dev: libc::c_ulong, pub st_ino: u64, @@ -119,9 +120,9 @@ pub mod windows { }); } - let mut info = unsafe { std::mem::zeroed() }; - let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; - let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; + let mut info = unsafe { core::mem::zeroed() }; + let mut basic_info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let mut id_info: FILE_ID_INFO = unsafe { core::mem::zeroed() }; if unsafe { GetFileInformationByHandle(h as _, &mut info) } == 0 || unsafe { @@ -129,7 +130,7 @@ pub mod windows { h as _, FileBasicInfo, &mut basic_info as *mut _ as *mut _, - std::mem::size_of_val(&basic_info) as u32, + core::mem::size_of_val(&basic_info) as u32, ) } == 0 { @@ -141,7 +142,7 @@ pub mod windows { h as _, FileIdInfo, &mut id_info as *mut _ as *mut _, - std::mem::size_of_val(&id_info) as u32, + core::mem::size_of_val(&id_info) as u32, ) } == 0 { @@ -165,7 +166,7 @@ pub mod windows { } fn file_time_to_time_t_nsec(in_ptr: &FILETIME) -> (libc::time_t, libc::c_int) { - let in_val: i64 = unsafe { std::mem::transmute_copy(in_ptr) }; + let in_val: i64 = unsafe { core::mem::transmute_copy(in_ptr) }; let nsec_out = (in_val % 10_000_000) * 100; // FILETIME is in units of 100 nsec. let time_out = (in_val / 10_000_000) - SECS_BETWEEN_EPOCHS; (time_out, nsec_out as _) @@ -204,7 +205,7 @@ pub mod windows { let st_nlink = info.nNumberOfLinks as i32; let st_ino = if let Some(id_info) = id_info { - let file_id: [u64; 2] = unsafe { std::mem::transmute_copy(&id_info.FileId) }; + let file_id: [u64; 2] = unsafe { core::mem::transmute_copy(&id_info.FileId) }; file_id } else { let ino = ((info.nFileIndexHigh as u64) << 32) + info.nFileIndexLow as u64; @@ -256,6 +257,7 @@ pub mod windows { m as _ } + #[derive(Clone, Copy)] #[repr(C)] pub struct FILE_STAT_BASIC_INFORMATION { pub FileId: i64, @@ -275,8 +277,9 @@ pub mod windows { pub FileId128: [u64; 2], } - #[repr(C)] #[allow(dead_code)] + #[derive(Clone, Copy)] + #[repr(C)] pub enum FILE_INFO_BY_NAME_CLASS { FileStatByNameInfo, FileStatLxByNameInfo, @@ -303,7 +306,8 @@ pub mod windows { let GetFileInformationByName = GET_FILE_INFORMATION_BY_NAME .get_or_init(|| { - let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); + let library_name = + OsString::from("api-ms-win-core-file-l2-1-4.dll").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; if module.is_null() { return None; @@ -313,7 +317,7 @@ pub mod windows { unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) } { Some(unsafe { - std::mem::transmute::< + core::mem::transmute::< unsafe extern "system" fn() -> isize, unsafe extern "system" fn( *const u16, @@ -331,8 +335,8 @@ pub mod windows { .ok_or_else(|| std::io::Error::from_raw_os_error(ERROR_NOT_SUPPORTED as _))?; let file_name = file_name.to_wide_with_nul(); - let file_info_buffer_size = std::mem::size_of::<FILE_STAT_BASIC_INFORMATION>() as u32; - let mut file_info_buffer = std::mem::MaybeUninit::<FILE_STAT_BASIC_INFORMATION>::uninit(); + let file_info_buffer_size = core::mem::size_of::<FILE_STAT_BASIC_INFORMATION>() as u32; + let mut file_info_buffer = core::mem::MaybeUninit::<FILE_STAT_BASIC_INFORMATION>::uninit(); unsafe { if GetFileInformationByName( file_name.as_ptr(), @@ -441,7 +445,7 @@ pub mod windows { // Open a file using std::fs::File and convert to FILE* // Automatically handles path encoding and EINTR retries pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut libc::FILE> { - use std::ffi::CString; + use alloc::ffi::CString; use std::fs::File; // Currently only supports read mode diff --git a/crates/common/src/float_ops.rs b/crates/common/src/float_ops.rs index b431e793139..c6b7c71e494 100644 --- a/crates/common/src/float_ops.rs +++ b/crates/common/src/float_ops.rs @@ -1,6 +1,6 @@ +use core::f64; use malachite_bigint::{BigInt, ToBigInt}; use num_traits::{Float, Signed, ToPrimitive, Zero}; -use std::f64; pub const fn decompose_float(value: f64) -> (f64, i32) { if 0.0 == value { diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs index 447ae575f48..40bc9e53046 100644 --- a/crates/common/src/format.rs +++ b/crates/common/src/format.rs @@ -1,4 +1,6 @@ // spell-checker:ignore ddfe +use core::ops::Deref; +use core::{cmp, str::FromStr}; use itertools::{Itertools, PeekingNext}; use malachite_base::num::basic::floats::PrimitiveFloat; use malachite_bigint::{BigInt, Sign}; @@ -7,8 +9,6 @@ use num_traits::FromPrimitive; use num_traits::{Signed, cast::ToPrimitive}; use rustpython_literal::float; use rustpython_literal::format::Case; -use std::ops::Deref; -use std::{cmp, str::FromStr}; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; @@ -110,7 +110,7 @@ impl FormatParse for FormatSign { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum FormatGrouping { Comma, Underscore, @@ -136,7 +136,7 @@ impl From<&FormatGrouping> for char { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum FormatType { String, Binary, @@ -149,6 +149,7 @@ pub enum FormatType { GeneralFormat(Case), FixedPoint(Case), Percentage, + Unknown(char), } impl From<&FormatType> for char { @@ -170,6 +171,7 @@ impl From<&FormatType> for char { FormatType::FixedPoint(Case::Lower) => 'f', FormatType::FixedPoint(Case::Upper) => 'F', FormatType::Percentage => '%', + FormatType::Unknown(c) => *c, } } } @@ -194,12 +196,13 @@ impl FormatParse for FormatType { Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_wtf8()), Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_wtf8()), Some('%') => (Some(Self::Percentage), chars.as_wtf8()), + Some(c) => (Some(Self::Unknown(c)), chars.as_wtf8()), _ => (None, text), } } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct FormatSpec { conversion: Option<FormatConversion>, fill: Option<CodePoint>, @@ -429,7 +432,8 @@ impl FormatSpec { | FormatType::FixedPoint(_) | FormatType::GeneralFormat(_) | FormatType::Exponent(_) - | FormatType::Percentage, + | FormatType::Percentage + | FormatType::Number(_), ) => 3, None => 3, _ => panic!("Separators only valid for numbers!"), @@ -475,6 +479,7 @@ impl FormatSpec { let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); Ok(first_letter.collect::<String>() + &input.to_string()[1..]) } + Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(*c, "int")), _ => Err(FormatSpecError::InvalidFormatSpecifier), } } @@ -496,7 +501,8 @@ impl FormatSpec { | Some(FormatType::Hex(_)) | Some(FormatType::String) | Some(FormatType::Character) - | Some(FormatType::Number(Case::Upper)) => { + | Some(FormatType::Number(Case::Upper)) + | Some(FormatType::Unknown(_)) => { let ch = char::from(self.format_type.as_ref().unwrap()); Err(FormatSpecError::UnknownFormatCode(ch, "float")) } @@ -598,7 +604,7 @@ impl FormatSpec { (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), (_, _) => match num.to_u32() { - Some(n) if n <= 0x10ffff => Ok(std::char::from_u32(n).unwrap().to_string()), + Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()), Some(_) | None => Err(FormatSpecError::CodeNotInRange), }, }, @@ -609,6 +615,7 @@ impl FormatSpec { Some(float) => return self.format_float(float), _ => Err(FormatSpecError::UnableToConvert), }, + Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(c, "int")), None => self.format_int_radix(magnitude, 10), }?; let format_sign = self.sign.unwrap_or(FormatSign::Minus); @@ -707,7 +714,8 @@ impl FormatSpec { | Some(FormatType::String) | Some(FormatType::Character) | Some(FormatType::Number(Case::Upper)) - | Some(FormatType::Percentage) => { + | Some(FormatType::Percentage) + | Some(FormatType::Unknown(_)) => { let ch = char::from(self.format_type.as_ref().unwrap()); Err(FormatSpecError::UnknownFormatCode(ch, "complex")) } @@ -845,7 +853,7 @@ impl Deref for AsciiStr<'_> { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum FormatSpecError { DecimalDigitsTooMany, PrecisionTooBig, @@ -862,7 +870,7 @@ pub enum FormatSpecError { NotImplemented(char, &'static str), } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum FormatParseError { UnmatchedBracket, MissingStartBracket, diff --git a/crates/common/src/hash.rs b/crates/common/src/hash.rs index dcf424f7ba9..f8f3783d224 100644 --- a/crates/common/src/hash.rs +++ b/crates/common/src/hash.rs @@ -1,7 +1,7 @@ +use core::hash::{BuildHasher, Hash, Hasher}; use malachite_bigint::BigInt; use num_traits::ToPrimitive; use siphasher::sip::SipHasher24; -use std::hash::{BuildHasher, Hash, Hasher}; pub type PyHash = i64; pub type PyUHash = u64; @@ -19,12 +19,13 @@ pub const INF: PyHash = 314_159; pub const NAN: PyHash = 0; pub const IMAG: PyHash = MULTIPLIER; pub const ALGO: &str = "siphash24"; -pub const HASH_BITS: usize = std::mem::size_of::<PyHash>() * 8; +pub const HASH_BITS: usize = core::mem::size_of::<PyHash>() * 8; // SipHasher24 takes 2 u64s as a seed -pub const SEED_BITS: usize = std::mem::size_of::<u64>() * 2 * 8; +pub const SEED_BITS: usize = core::mem::size_of::<u64>() * 2 * 8; // pub const CUTOFF: usize = 7; +#[derive(Clone, Copy)] pub struct HashSecret { k0: u64, k1: u64, @@ -134,7 +135,7 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { Some(i) => mod_int(i), None => (value % MODULUS).to_i64().unwrap_or_else(|| unsafe { // SAFETY: MODULUS < i64::MAX, so value % MODULUS is guaranteed to be in the range of i64 - std::hint::unreachable_unchecked() + core::hint::unreachable_unchecked() }), }; fix_sentinel(ret) diff --git a/crates/common/src/int.rs b/crates/common/src/int.rs index ed09cc01a0a..9cfe2e0d738 100644 --- a/crates/common/src/int.rs +++ b/crates/common/src/int.rs @@ -7,18 +7,18 @@ pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); match rational.rounding_into(RoundingMode::Nearest) { // returned value is $t::MAX but still less than the original - (val, std::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, + (val, core::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, // returned value is $t::MIN but still greater than the original - (val, std::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, + (val, core::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, (val, _) => val, } } pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { - let sign = match std::cmp::PartialOrd::partial_cmp(&value, &0.0)? { - std::cmp::Ordering::Less => Sign::Minus, - std::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), - std::cmp::Ordering::Greater => Sign::Plus, + let sign = match core::cmp::PartialOrd::partial_cmp(&value, &0.0)? { + core::cmp::Ordering::Less => Sign::Minus, + core::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), + core::cmp::Ordering::Greater => Sign::Plus, }; Rational::try_from(value).ok().map(|x| { let (numer, denom) = x.into_numerator_and_denominator(); @@ -29,7 +29,7 @@ pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { }) } -#[derive(Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum BytesToIntError { InvalidLiteral { base: u32 }, InvalidBase, diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index c99ba0286a4..e514c17541f 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,6 +1,8 @@ //! A crate to hold types and functions common to all rustpython components. -#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate alloc; #[macro_use] mod macros; @@ -10,10 +12,10 @@ pub mod atomic; pub mod borrow; pub mod boxvec; pub mod cformat; -#[cfg(any(unix, windows, target_os = "wasi"))] +#[cfg(all(feature = "std", any(unix, windows, target_os = "wasi")))] pub mod crt_fd; pub mod encodings; -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] +#[cfg(all(feature = "std", any(not(target_arch = "wasm32"), target_os = "wasi")))] pub mod fileutils; pub mod float_ops; pub mod format; @@ -21,13 +23,14 @@ pub mod hash; pub mod int; pub mod linked_list; pub mod lock; +#[cfg(feature = "std")] pub mod os; pub mod rand; pub mod rc; pub mod refcount; pub mod static_cell; pub mod str; -#[cfg(windows)] +#[cfg(all(feature = "std", windows))] pub mod windows; pub use rustpython_wtf8 as wtf8; diff --git a/crates/common/src/linked_list.rs b/crates/common/src/linked_list.rs index 8afc1478e6b..48cdb4feb95 100644 --- a/crates/common/src/linked_list.rs +++ b/crates/common/src/linked_list.rs @@ -253,7 +253,7 @@ impl<L: Link> LinkedList<L, L::Target> { // === rustpython additions === pub fn iter(&self) -> impl Iterator<Item = &L::Target> { - std::iter::successors(self.head, |node| unsafe { + core::iter::successors(self.head, |node| unsafe { L::pointers(*node).as_ref().get_next() }) .map(|ptr| unsafe { ptr.as_ref() }) @@ -333,7 +333,7 @@ impl<T> Pointers<T> { } } - const fn get_prev(&self) -> Option<NonNull<T>> { + pub const fn get_prev(&self) -> Option<NonNull<T>> { // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -341,7 +341,7 @@ impl<T> Pointers<T> { ptr::read(prev) } } - const fn get_next(&self) -> Option<NonNull<T>> { + pub const fn get_next(&self) -> Option<NonNull<T>> { // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -351,7 +351,7 @@ impl<T> Pointers<T> { } } - const fn set_prev(&mut self, value: Option<NonNull<T>>) { + pub const fn set_prev(&mut self, value: Option<NonNull<T>>) { // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -359,7 +359,7 @@ impl<T> Pointers<T> { ptr::write(prev, value); } } - const fn set_next(&mut self, value: Option<NonNull<T>>) { + pub const fn set_next(&mut self, value: Option<NonNull<T>>) { // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); diff --git a/crates/common/src/lock.rs b/crates/common/src/lock.rs index ca5ffe8de37..f230011c028 100644 --- a/crates/common/src/lock.rs +++ b/crates/common/src/lock.rs @@ -10,12 +10,29 @@ cfg_if::cfg_if! { if #[cfg(feature = "threading")] { pub use parking_lot::{RawMutex, RawRwLock, RawThreadId}; - pub use once_cell::sync::{Lazy, OnceCell}; + pub use std::sync::{LazyLock, OnceLock as OnceCell}; + pub use core::cell::LazyCell; } else { mod cell_lock; pub use cell_lock::{RawCellMutex as RawMutex, RawCellRwLock as RawRwLock, SingleThreadId as RawThreadId}; - pub use once_cell::unsync::{Lazy, OnceCell}; + pub use core::cell::{LazyCell, OnceCell}; + + /// `core::cell::LazyCell` with `Sync` for use in `static` items. + /// SAFETY: Without threading, there can be no concurrent access. + pub struct LazyLock<T, F = fn() -> T>(core::cell::LazyCell<T, F>); + // SAFETY: Without threading, there can be no concurrent access. + unsafe impl<T, F> Sync for LazyLock<T, F> {} + + impl<T, F: FnOnce() -> T> LazyLock<T, F> { + pub const fn new(f: F) -> Self { Self(core::cell::LazyCell::new(f)) } + pub fn force(this: &Self) -> &T { core::cell::LazyCell::force(&this.0) } + } + + impl<T, F: FnOnce() -> T> core::ops::Deref for LazyLock<T, F> { + type Target = T; + fn deref(&self) -> &T { &self.0 } + } } } diff --git a/crates/common/src/lock/cell_lock.rs b/crates/common/src/lock/cell_lock.rs index 25a5cfedba1..ab9cfb08c84 100644 --- a/crates/common/src/lock/cell_lock.rs +++ b/crates/common/src/lock/cell_lock.rs @@ -1,16 +1,19 @@ // spell-checker:ignore upgradably sharedly +use core::{cell::Cell, num::NonZero}; use lock_api::{ GetThreadId, RawMutex, RawRwLock, RawRwLockDowngrade, RawRwLockRecursive, RawRwLockUpgrade, RawRwLockUpgradeDowngrade, }; -use std::{cell::Cell, num::NonZero}; pub struct RawCellMutex { locked: Cell<bool>, } unsafe impl RawMutex for RawCellMutex { - #[allow(clippy::declare_interior_mutable_const)] + #[allow( + clippy::declare_interior_mutable_const, + reason = "const lock initializer intentionally uses interior mutability" + )] const INIT: Self = Self { locked: Cell::new(false), }; @@ -60,7 +63,10 @@ impl RawCellRwLock { } unsafe impl RawRwLock for RawCellRwLock { - #[allow(clippy::declare_interior_mutable_const)] + #[allow( + clippy::declare_interior_mutable_const, + reason = "const rwlock initializer intentionally uses interior mutability" + )] const INIT: Self = Self { state: Cell::new(0), }; @@ -89,7 +95,7 @@ unsafe impl RawRwLock for RawCellRwLock { #[inline] unsafe fn unlock_shared(&self) { - self.state.set(self.state.get() - ONE_READER) + self.state.update(|x| x - ONE_READER) } #[inline] @@ -201,10 +207,14 @@ fn deadlock(lock_kind: &str, ty: &str) -> ! { panic!("deadlock: tried to {lock_kind}lock a Cell{ty} twice") } +#[derive(Clone, Copy)] pub struct SingleThreadId(()); + unsafe impl GetThreadId for SingleThreadId { const INIT: Self = Self(()); + fn nonzero_thread_id(&self) -> NonZero<usize> { - NonZero::new(1).unwrap() + // Safety: This is constant. + unsafe { NonZero::new_unchecked(1) } } } diff --git a/crates/common/src/lock/immutable_mutex.rs b/crates/common/src/lock/immutable_mutex.rs index 81c5c93be71..2013cf1c60d 100644 --- a/crates/common/src/lock/immutable_mutex.rs +++ b/crates/common/src/lock/immutable_mutex.rs @@ -1,7 +1,8 @@ #![allow(clippy::needless_lifetimes)] +use alloc::fmt; +use core::{marker::PhantomData, ops::Deref}; use lock_api::{MutexGuard, RawMutex}; -use std::{fmt, marker::PhantomData, ops::Deref}; /// A mutex guard that has an exclusive lock, but only an immutable reference; useful if you /// need to map a mutex guard with a function that returns an `&T`. Construct using the @@ -22,7 +23,7 @@ impl<'a, R: RawMutex, T: ?Sized> MapImmutable<'a, R, T> for MutexGuard<'a, R, T> { let raw = unsafe { MutexGuard::mutex(&s).raw() }; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, @@ -38,7 +39,7 @@ impl<'a, R: RawMutex, T: ?Sized> ImmutableMappedMutexGuard<'a, R, T> { { let raw = s.raw; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, diff --git a/crates/common/src/lock/thread_mutex.rs b/crates/common/src/lock/thread_mutex.rs index 2949a3c6c14..2cabf7ea4cd 100644 --- a/crates/common/src/lock/thread_mutex.rs +++ b/crates/common/src/lock/thread_mutex.rs @@ -1,14 +1,14 @@ #![allow(clippy::needless_lifetimes)] -use lock_api::{GetThreadId, GuardNoSend, RawMutex}; -use std::{ +use alloc::fmt; +use core::{ cell::UnsafeCell, - fmt, marker::PhantomData, ops::{Deref, DerefMut}, ptr::NonNull, sync::atomic::{AtomicUsize, Ordering}, }; +use lock_api::{GetThreadId, GuardNoSend, RawMutex}; // based off ReentrantMutex from lock_api @@ -20,7 +20,10 @@ pub struct RawThreadMutex<R: RawMutex, G: GetThreadId> { } impl<R: RawMutex, G: GetThreadId> RawThreadMutex<R, G> { - #[allow(clippy::declare_interior_mutable_const)] + #[allow( + clippy::declare_interior_mutable_const, + reason = "const initializer for lock primitive contains atomics by design" + )] pub const INIT: Self = Self { owner: AtomicUsize::new(0), mutex: R::INIT, @@ -121,19 +124,23 @@ impl<R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutex<R, G, T> { } } } -// Whether ThreadMutex::try_lock failed because the mutex was already locked on another thread or -// on the current thread + +#[derive(Clone, Copy)] pub enum TryLockThreadError { + /// Failed to lock because mutex was already locked on another thread. Other, + /// Failed to lock because mutex was already locked on current thread. Current, } struct LockedPlaceholder(&'static str); + impl fmt::Debug for LockedPlaceholder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.0) } } + impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug for ThreadMutex<R, G, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.try_lock() { @@ -174,7 +181,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { ) -> MappedThreadMutexGuard<'a, R, G, U> { let data = f(&mut s).into(); let mu = &s.mu.raw; - std::mem::forget(s); + core::mem::forget(s); MappedThreadMutexGuard { mu, data, @@ -188,7 +195,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { if let Some(data) = f(&mut s) { let data = data.into(); let mu = &s.mu.raw; - std::mem::forget(s); + core::mem::forget(s); Ok(MappedThreadMutexGuard { mu, data, @@ -241,7 +248,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G ) -> MappedThreadMutexGuard<'a, R, G, U> { let data = f(&mut s).into(); let mu = s.mu; - std::mem::forget(s); + core::mem::forget(s); MappedThreadMutexGuard { mu, data, @@ -255,7 +262,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G if let Some(data) = f(&mut s) { let data = data.into(); let mu = s.mu; - std::mem::forget(s); + core::mem::forget(s); Ok(MappedThreadMutexGuard { mu, data, diff --git a/crates/common/src/os.rs b/crates/common/src/os.rs index e77a81fd94f..ef8547289a2 100644 --- a/crates/common/src/os.rs +++ b/crates/common/src/os.rs @@ -1,7 +1,8 @@ // spell-checker:disable // TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here -use std::{io, process::ExitCode, str::Utf8Error}; +use core::str::Utf8Error; +use std::{io, process::ExitCode}; /// Convert exit code to std::process::ExitCode /// @@ -88,14 +89,66 @@ pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { #[cfg(not(unix))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - Ok(std::str::from_utf8(b)?.as_ref()) + Ok(core::str::from_utf8(b)?.as_ref()) } #[cfg(unix)] pub use std::os::unix::ffi; -#[cfg(target_os = "wasi")] + +// WASIp1 uses stable std::os::wasi::ffi +#[cfg(all(target_os = "wasi", not(target_env = "p2")))] pub use std::os::wasi::ffi; +// WASIp2: std::os::wasip2::ffi is unstable, so we provide a stable implementation +// leveraging WASI's UTF-8 string guarantee +#[cfg(all(target_os = "wasi", target_env = "p2"))] +pub mod ffi { + use std::ffi::{OsStr, OsString}; + + pub trait OsStrExt: sealed::Sealed { + fn as_bytes(&self) -> &[u8]; + fn from_bytes(slice: &[u8]) -> &Self; + } + + impl OsStrExt for OsStr { + fn as_bytes(&self) -> &[u8] { + // WASI strings are guaranteed to be UTF-8 + self.to_str().expect("wasip2 strings are UTF-8").as_bytes() + } + + fn from_bytes(slice: &[u8]) -> &OsStr { + // WASI strings are guaranteed to be UTF-8 + OsStr::new(std::str::from_utf8(slice).expect("wasip2 strings are UTF-8")) + } + } + + pub trait OsStringExt: sealed::Sealed { + fn from_vec(vec: Vec<u8>) -> Self; + fn into_vec(self) -> Vec<u8>; + } + + impl OsStringExt for OsString { + fn from_vec(vec: Vec<u8>) -> OsString { + // WASI strings are guaranteed to be UTF-8 + OsString::from(String::from_utf8(vec).expect("wasip2 strings are UTF-8")) + } + + fn into_vec(self) -> Vec<u8> { + // WASI strings are guaranteed to be UTF-8 + self.to_str() + .expect("wasip2 strings are UTF-8") + .as_bytes() + .to_vec() + } + } + + mod sealed { + pub trait Sealed {} + impl Sealed for std::ffi::OsStr {} + impl Sealed for std::ffi::OsString {} + } +} + #[cfg(windows)] pub fn errno_to_winerror(errno: i32) -> i32 { use libc::*; @@ -130,7 +183,10 @@ pub fn winerror_to_errno(winerror: i32) -> i32 { use libc::*; use windows_sys::Win32::{ Foundation::*, - Networking::WinSock::{WSAEACCES, WSAEBADF, WSAEFAULT, WSAEINTR, WSAEINVAL, WSAEMFILE}, + Networking::WinSock::{ + WSAEACCES, WSAEBADF, WSAECONNABORTED, WSAECONNREFUSED, WSAECONNRESET, WSAEFAULT, + WSAEINTR, WSAEINVAL, WSAEMFILE, + }, }; // Unwrap FACILITY_WIN32 HRESULT errors. // if ((winerror & 0xFFFF0000) == 0x80070000) { @@ -217,6 +273,11 @@ pub fn winerror_to_errno(winerror: i32) -> i32 { ERROR_BROKEN_PIPE | ERROR_NO_DATA => EPIPE, ERROR_DIR_NOT_EMPTY => ENOTEMPTY, ERROR_NO_UNICODE_TRANSLATION => EILSEQ, + // Connection-related Windows error codes - map to Winsock error codes + // which Python uses on Windows (errno.ECONNREFUSED = 10061, etc.) + ERROR_CONNECTION_REFUSED => WSAECONNREFUSED, + ERROR_CONNECTION_ABORTED => WSAECONNABORTED, + ERROR_NETNAME_DELETED => WSAECONNRESET, ERROR_INVALID_FUNCTION | ERROR_INVALID_ACCESS | ERROR_INVALID_DATA diff --git a/crates/common/src/rc.rs b/crates/common/src/rc.rs index 40c7cf97a8d..9e4cca228fd 100644 --- a/crates/common/src/rc.rs +++ b/crates/common/src/rc.rs @@ -1,7 +1,7 @@ #[cfg(not(feature = "threading"))] -use std::rc::Rc; +use alloc::rc::Rc; #[cfg(feature = "threading")] -use std::sync::Arc; +use alloc::sync::Arc; // type aliases instead of new-types because you can't do `fn method(self: PyRc<Self>)` with a // newtype; requires the arbitrary_self_types unstable feature diff --git a/crates/common/src/refcount.rs b/crates/common/src/refcount.rs index a5fbfa8fc36..4a8ed0bf51d 100644 --- a/crates/common/src/refcount.rs +++ b/crates/common/src/refcount.rs @@ -1,5 +1,14 @@ use crate::atomic::{Ordering::*, PyAtomic, Radium}; +#[inline(never)] +#[cold] +fn refcount_overflow() -> ! { + #[cfg(feature = "std")] + std::process::abort(); + #[cfg(not(feature = "std"))] + core::panic!("refcount overflow"); +} + /// from alloc::sync /// A soft limit on the amount of references that may be made to an `Arc`. /// @@ -36,7 +45,17 @@ impl RefCount { let old_size = self.strong.fetch_add(1, Relaxed); if old_size & Self::MASK == Self::MASK { - std::process::abort(); + refcount_overflow(); + } + } + + #[inline] + pub fn inc_by(&self, n: usize) { + debug_assert!(n <= Self::MASK); + let old_size = self.strong.fetch_add(n, Relaxed); + + if old_size & Self::MASK > Self::MASK - n { + refcount_overflow(); } } diff --git a/crates/common/src/static_cell.rs b/crates/common/src/static_cell.rs index a8beee08206..bf277e60ea7 100644 --- a/crates/common/src/static_cell.rs +++ b/crates/common/src/static_cell.rs @@ -1,4 +1,58 @@ -#[cfg(not(feature = "threading"))] +#[cfg(feature = "threading")] +mod threading { + use crate::lock::OnceCell; + + pub struct StaticCell<T: 'static> { + inner: OnceCell<T>, + } + + impl<T> StaticCell<T> { + #[doc(hidden)] + pub const fn _from_once_cell(inner: OnceCell<T>) -> Self { + Self { inner } + } + + pub fn get(&'static self) -> Option<&'static T> { + self.inner.get() + } + + pub fn set(&'static self, value: T) -> Result<(), T> { + self.inner.set(value) + } + + pub fn get_or_init<F>(&'static self, f: F) -> &'static T + where + F: FnOnce() -> T, + { + self.inner.get_or_init(f) + } + + pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> + where + F: FnOnce() -> Result<T, E>, + { + if let Some(val) = self.inner.get() { + return Ok(val); + } + let val = f()?; + let _ = self.inner.set(val); + Ok(self.inner.get().unwrap()) + } + } + + #[macro_export] + macro_rules! static_cell { + ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { + $($(#[$attr])* + $vis static $name: $crate::static_cell::StaticCell<$t> = + $crate::static_cell::StaticCell::_from_once_cell($crate::lock::OnceCell::new());)+ + }; + } +} +#[cfg(feature = "threading")] +pub use threading::*; + +#[cfg(all(not(feature = "threading"), feature = "std"))] mod non_threading { use crate::lock::OnceCell; use std::thread::LocalKey; @@ -22,12 +76,10 @@ mod non_threading { } pub fn set(&'static self, value: T) -> Result<(), T> { - // thread-safe because it's a unsync::OnceCell self.inner.with(|x| { if x.get().is_some() { Err(value) } else { - // will never fail let _ = x.set(leak(value)); Ok(()) } @@ -45,8 +97,15 @@ mod non_threading { where F: FnOnce() -> Result<T, E>, { - self.inner - .with(|x| x.get_or_try_init(|| f().map(leak)).copied()) + self.inner.with(|x| { + if let Some(val) = x.get() { + Ok(*val) + } else { + let val = leak(f()?); + let _ = x.set(val); + Ok(val) + } + }) } } @@ -65,43 +124,56 @@ mod non_threading { }; } } -#[cfg(not(feature = "threading"))] +#[cfg(all(not(feature = "threading"), feature = "std"))] pub use non_threading::*; -#[cfg(feature = "threading")] -mod threading { +// Same as `threading` variant, but wraps unsync::OnceCell with Sync. +#[cfg(all(not(feature = "threading"), not(feature = "std")))] +mod no_std { use crate::lock::OnceCell; + // unsync::OnceCell is !Sync, but without std there can be no threads. + struct SyncOnceCell<T>(OnceCell<T>); + // SAFETY: Without std, threading is impossible. + unsafe impl<T> Sync for SyncOnceCell<T> {} + pub struct StaticCell<T: 'static> { - inner: OnceCell<T>, + inner: SyncOnceCell<T>, } impl<T> StaticCell<T> { #[doc(hidden)] pub const fn _from_once_cell(inner: OnceCell<T>) -> Self { - Self { inner } + Self { + inner: SyncOnceCell(inner), + } } pub fn get(&'static self) -> Option<&'static T> { - self.inner.get() + self.inner.0.get() } pub fn set(&'static self, value: T) -> Result<(), T> { - self.inner.set(value) + self.inner.0.set(value) } pub fn get_or_init<F>(&'static self, f: F) -> &'static T where F: FnOnce() -> T, { - self.inner.get_or_init(f) + self.inner.0.get_or_init(f) } pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> where F: FnOnce() -> Result<T, E>, { - self.inner.get_or_try_init(f) + if let Some(val) = self.inner.0.get() { + return Ok(val); + } + let val = f()?; + let _ = self.inner.0.set(val); + Ok(self.inner.0.get().unwrap()) } } @@ -114,5 +186,5 @@ mod threading { }; } } -#[cfg(feature = "threading")] -pub use threading::*; +#[cfg(all(not(feature = "threading"), not(feature = "std")))] +pub use no_std::*; diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index 2d867130edd..79c407909ff 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -4,8 +4,8 @@ use crate::format::CharLen; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use ascii::{AsciiChar, AsciiStr, AsciiString}; use core::fmt; +use core::ops::{Bound, RangeBounds}; use core::sync::atomic::Ordering::Relaxed; -use std::ops::{Bound, RangeBounds}; #[cfg(not(target_arch = "wasm32"))] #[allow(non_camel_case_types)] @@ -22,7 +22,7 @@ pub enum StrKind { Wtf8, } -impl std::ops::BitOr for StrKind { +impl core::ops::BitOr for StrKind { type Output = Self; fn bitor(self, other: Self) -> Self { @@ -128,7 +128,7 @@ impl From<usize> for StrLen { } impl fmt::Debug for StrLen { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let len = self.0.load(Relaxed); if len == usize::MAX { f.write_str("<uncomputed>") @@ -262,7 +262,7 @@ impl StrData { pub fn as_str(&self) -> Option<&str> { self.kind .is_utf8() - .then(|| unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + .then(|| unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) } pub fn as_ascii(&self) -> Option<&AsciiStr> { @@ -282,7 +282,7 @@ impl StrData { PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) } StrKind::Utf8 => { - PyKindStr::Utf8(unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + PyKindStr::Utf8(unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) } StrKind::Wtf8 => PyKindStr::Wtf8(&self.data), } @@ -327,8 +327,8 @@ impl StrData { } } -impl std::fmt::Display for StrData { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for StrData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.data.fmt(f) } } @@ -421,7 +421,7 @@ pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> { }; let mut filled = Vec::new(); filled.extend_from_slice(sign); - filled.extend(std::iter::repeat_n(b'0', width - bytes.len())); + filled.extend(core::iter::repeat_n(b'0', width - bytes.len())); filled.extend_from_slice(s); filled } @@ -449,6 +449,7 @@ pub fn to_ascii(value: &str) -> AsciiString { unsafe { AsciiString::from_ascii_unchecked(ascii) } } +#[derive(Clone, Copy)] pub struct UnicodeEscapeCodepoint(pub CodePoint); impl fmt::Display for UnicodeEscapeCodepoint { @@ -465,8 +466,6 @@ impl fmt::Display for UnicodeEscapeCodepoint { } pub mod levenshtein { - use std::{cell::RefCell, thread_local}; - pub const MOVE_COST: usize = 2; const CASE_COST: usize = 1; const MAX_STRING_SIZE: usize = 40; @@ -488,13 +487,6 @@ pub mod levenshtein { } pub fn levenshtein_distance(a: &[u8], b: &[u8], max_cost: usize) -> usize { - thread_local! { - #[allow(clippy::declare_interior_mutable_const)] - static BUFFER: RefCell<[usize; MAX_STRING_SIZE]> = const { - RefCell::new([0usize; MAX_STRING_SIZE]) - }; - } - if a == b { return 0; } @@ -524,42 +516,42 @@ pub mod levenshtein { } if b_end < a_end { - std::mem::swap(&mut a_bytes, &mut b_bytes); - std::mem::swap(&mut a_begin, &mut b_begin); - std::mem::swap(&mut a_end, &mut b_end); + core::mem::swap(&mut a_bytes, &mut b_bytes); + core::mem::swap(&mut a_begin, &mut b_begin); + core::mem::swap(&mut a_end, &mut b_end); } if (b_end - a_end) * MOVE_COST > max_cost { return max_cost + 1; } - BUFFER.with_borrow_mut(|buffer| { - for (i, x) in buffer.iter_mut().take(a_end).enumerate() { - *x = (i + 1) * MOVE_COST; - } + let mut buffer = [0usize; MAX_STRING_SIZE]; - let mut result = 0usize; - for (b_index, b_code) in b_bytes[b_begin..(b_begin + b_end)].iter().enumerate() { - result = b_index * MOVE_COST; - let mut distance = result; - let mut minimum = usize::MAX; - for (a_index, a_code) in a_bytes[a_begin..(a_begin + a_end)].iter().enumerate() { - let substitute = distance + substitution_cost(*b_code, *a_code); - distance = buffer[a_index]; - let insert_delete = usize::min(result, distance) + MOVE_COST; - result = usize::min(insert_delete, substitute); - - buffer[a_index] = result; - if result < minimum { - minimum = result; - } - } - if minimum > max_cost { - return max_cost + 1; + for (i, x) in buffer.iter_mut().take(a_end).enumerate() { + *x = (i + 1) * MOVE_COST; + } + + let mut result = 0usize; + for (b_index, b_code) in b_bytes[b_begin..(b_begin + b_end)].iter().enumerate() { + result = b_index * MOVE_COST; + let mut distance = result; + let mut minimum = usize::MAX; + for (a_index, a_code) in a_bytes[a_begin..(a_begin + a_end)].iter().enumerate() { + let substitute = distance + substitution_cost(*b_code, *a_code); + distance = buffer[a_index]; + let insert_delete = usize::min(result, distance) + MOVE_COST; + result = usize::min(insert_delete, substitute); + + buffer[a_index] = result; + if result < minimum { + minimum = result; } } - result - }) + if minimum > max_cost { + return max_cost + 1; + } + } + result } } diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index 11d2a7b5f1b..a6d19795076 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -3,95 +3,100 @@ use crate::{ marshal::MarshalError, + varint::{read_varint, read_varint_with_start, write_varint, write_varint_with_start}, {OneIndexed, SourceLocation}, }; +use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec}; use bitflags::bitflags; +use core::{hash, mem, ops::Deref}; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; use rustpython_wtf8::{Wtf8, Wtf8Buf}; -use std::{collections::BTreeSet, fmt, hash, marker::PhantomData, mem, num::NonZeroU8, ops::Deref}; -/// Oparg values for [`Instruction::ConvertValue`]. -/// -/// ## See also -/// -/// - [CPython FVC_* flags](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Include/ceval.h#L129-L132) -#[repr(u8)] -#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] -pub enum ConvertValueOparg { - /// No conversion. - /// - /// ```python - /// f"{x}" - /// f"{x:4}" - /// ``` - None = 0, - /// Converts by calling `str(<value>)`. - /// - /// ```python - /// f"{x!s}" - /// f"{x!s:2}" - /// ``` - Str = 1, - /// Converts by calling `repr(<value>)`. - /// - /// ```python - /// f"{x!r}" - /// f"{x!r:2}" - /// ``` - Repr = 2, - /// Converts by calling `ascii(<value>)`. - /// - /// ```python - /// f"{x!a}" - /// f"{x!a:2}" - /// ``` - Ascii = 3, -} - -impl fmt::Display for ConvertValueOparg { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let out = match self { - Self::Str => "1 (str)", - Self::Repr => "2 (repr)", - Self::Ascii => "3 (ascii)", - // We should never reach this. `FVC_NONE` are being handled by `Instruction::FormatSimple` - Self::None => "", - }; +pub use crate::bytecode::{ + instruction::{ + AnyInstruction, Arg, Instruction, InstructionMetadata, PseudoInstruction, StackEffect, + }, + oparg::{ + BinaryOperator, BuildSliceArgCount, CommonConstant, ComparisonOperator, ConvertValueOparg, + IntrinsicFunction1, IntrinsicFunction2, Invert, Label, LoadAttr, LoadSuperAttr, + MakeFunctionFlags, NameIdx, OpArg, OpArgByte, OpArgState, OpArgType, RaiseKind, ResumeType, + SpecialMethod, UnpackExArgs, + }, +}; - write!(f, "{out}") +mod instruction; +mod oparg; + +/// Exception table entry for zero-cost exception handling +/// Format: (start, size, target, depth<<1|lasti) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExceptionTableEntry { + /// Start instruction offset (inclusive) + pub start: u32, + /// End instruction offset (exclusive) + pub end: u32, + /// Handler target offset + pub target: u32, + /// Stack depth at handler entry + pub depth: u16, + /// Whether to push lasti before exception + pub push_lasti: bool, +} + +impl ExceptionTableEntry { + pub const fn new(start: u32, end: u32, target: u32, depth: u16, push_lasti: bool) -> Self { + Self { + start, + end, + target, + depth, + push_lasti, + } } } -impl OpArgType for ConvertValueOparg { - #[inline] - fn from_op_arg(x: u32) -> Option<Self> { - Some(match x { - // Ruff `ConversionFlag::None` is `-1i8`, - // when its converted to `u8` its value is `u8::MAX` - 0 | 255 => Self::None, - 1 => Self::Str, - 2 => Self::Repr, - 3 => Self::Ascii, - _ => return None, - }) - } - - #[inline] - fn to_op_arg(self) -> u32 { - self as u32 +/// Encode exception table entries. +/// Uses 6-bit varint encoding with start marker (MSB) and continuation bit. +pub fn encode_exception_table(entries: &[ExceptionTableEntry]) -> alloc::boxed::Box<[u8]> { + let mut data = Vec::new(); + for entry in entries { + let size = entry.end.saturating_sub(entry.start); + let depth_lasti = ((entry.depth as u32) << 1) | (entry.push_lasti as u32); + + write_varint_with_start(&mut data, entry.start); + write_varint(&mut data, size); + write_varint(&mut data, entry.target); + write_varint(&mut data, depth_lasti); + } + data.into_boxed_slice() +} + +/// Find exception handler for given instruction offset. +pub fn find_exception_handler(table: &[u8], offset: u32) -> Option<ExceptionTableEntry> { + let mut pos = 0; + while pos < table.len() { + let start = read_varint_with_start(table, &mut pos)?; + let size = read_varint(table, &mut pos)?; + let target = read_varint(table, &mut pos)?; + let depth_lasti = read_varint(table, &mut pos)?; + + let end = start + size; + let depth = (depth_lasti >> 1) as u16; + let push_lasti = (depth_lasti & 1) != 0; + + if offset >= start && offset < end { + return Some(ExceptionTableEntry { + start, + end, + target, + depth, + push_lasti, + }); + } } -} - -/// Resume type for the RESUME instruction -#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] -#[repr(u32)] -pub enum ResumeType { - AtFuncStart = 0, - AfterYield = 1, - AfterYieldFrom = 2, - AfterAwait = 3, + None } /// CPython 3.11+ linetable location info codes @@ -257,7 +262,7 @@ impl ConstantBag for BasicBag { #[derive(Clone)] pub struct CodeObject<C: Constant = ConstantData> { pub instructions: CodeUnits, - pub locations: Box<[SourceLocation]>, + pub locations: Box<[(SourceLocation, SourceLocation)]>, pub flags: CodeFlags, /// Number of positional-only arguments pub posonlyarg_count: u32, @@ -284,611 +289,22 @@ pub struct CodeObject<C: Constant = ConstantData> { bitflags! { #[derive(Copy, Clone, Debug, PartialEq)] - pub struct CodeFlags: u16 { - const NEW_LOCALS = 0x01; - const IS_GENERATOR = 0x02; - const IS_COROUTINE = 0x04; - const HAS_VARARGS = 0x08; - const HAS_VARKEYWORDS = 0x10; - const IS_OPTIMIZED = 0x20; - } -} - -impl CodeFlags { - pub const NAME_MAPPING: &'static [(&'static str, Self)] = &[ - ("GENERATOR", Self::IS_GENERATOR), - ("COROUTINE", Self::IS_COROUTINE), - ( - "ASYNC_GENERATOR", - Self::from_bits_truncate(Self::IS_GENERATOR.bits() | Self::IS_COROUTINE.bits()), - ), - ("VARARGS", Self::HAS_VARARGS), - ("VARKEYWORDS", Self::HAS_VARKEYWORDS), - ]; -} - -/// an opcode argument that may be extended by a prior ExtendedArg -#[derive(Copy, Clone, PartialEq, Eq)] -#[repr(transparent)] -pub struct OpArgByte(pub u8); - -impl OpArgByte { - pub const fn null() -> Self { - Self(0) - } -} - -impl From<u8> for OpArgByte { - fn from(raw: u8) -> Self { - Self(raw) - } -} - -impl fmt::Debug for OpArgByte { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -/// a full 32-bit op_arg, including any possible ExtendedArg extension -#[derive(Copy, Clone, Debug)] -#[repr(transparent)] -pub struct OpArg(pub u32); - -impl OpArg { - pub const fn null() -> Self { - Self(0) - } - - /// Returns how many CodeUnits a instruction with this op_arg will be encoded as - #[inline] - pub const fn instr_size(self) -> usize { - (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1 - } - - /// returns the arg split into any necessary ExtendedArg components (in big-endian order) and - /// the arg for the real opcode itself - #[inline(always)] - pub fn split(self) -> (impl ExactSizeIterator<Item = OpArgByte>, OpArgByte) { - let mut it = self - .0 - .to_le_bytes() - .map(OpArgByte) - .into_iter() - .take(self.instr_size()); - let lo = it.next().unwrap(); - (it.rev(), lo) - } -} - -impl From<u32> for OpArg { - fn from(raw: u32) -> Self { - Self(raw) - } -} - -#[derive(Default, Copy, Clone)] -#[repr(transparent)] -pub struct OpArgState { - state: u32, -} - -impl OpArgState { - #[inline(always)] - pub fn get(&mut self, ins: CodeUnit) -> (Instruction, OpArg) { - let arg = self.extend(ins.arg); - if ins.op != Instruction::ExtendedArg { - self.reset(); - } - (ins.op, arg) - } - - #[inline(always)] - pub fn extend(&mut self, arg: OpArgByte) -> OpArg { - self.state = (self.state << 8) | u32::from(arg.0); - OpArg(self.state) - } - - #[inline(always)] - pub const fn reset(&mut self) { - self.state = 0 - } -} - -pub trait OpArgType: Copy { - fn from_op_arg(x: u32) -> Option<Self>; - - fn to_op_arg(self) -> u32; -} - -impl OpArgType for u32 { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(x) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - self - } -} - -impl OpArgType for bool { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(x != 0) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - self as u32 - } -} - -macro_rules! op_arg_enum_impl { - (enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $value:literal,)* }) => { - impl OpArgType for $name { - fn to_op_arg(self) -> u32 { - self as u32 - } - - fn from_op_arg(x: u32) -> Option<Self> { - Some(match u8::try_from(x).ok()? { - $($value => Self::$var,)* - _ => return None, - }) - } - } - }; -} - -macro_rules! op_arg_enum { - ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $value:literal,)* }) => { - $(#[$attr])* - $vis enum $name { - $($(#[$var_attr])* $var = $value,)* - } - - op_arg_enum_impl!(enum $name { - $($(#[$var_attr])* $var = $value,)* - }); - }; -} - -#[derive(Copy, Clone)] -pub struct Arg<T: OpArgType>(PhantomData<T>); - -impl<T: OpArgType> Arg<T> { - #[inline] - pub const fn marker() -> Self { - Self(PhantomData) - } - - #[inline] - pub fn new(arg: T) -> (Self, OpArg) { - (Self(PhantomData), OpArg(arg.to_op_arg())) - } - - #[inline] - pub fn new_single(arg: T) -> (Self, OpArgByte) - where - T: Into<u8>, - { - (Self(PhantomData), OpArgByte(arg.into())) - } - - #[inline(always)] - pub fn get(self, arg: OpArg) -> T { - self.try_get(arg).unwrap() - } - - #[inline(always)] - pub fn try_get(self, arg: OpArg) -> Option<T> { - T::from_op_arg(arg.0) - } - - /// # Safety - /// T::from_op_arg(self) must succeed - #[inline(always)] - pub unsafe fn get_unchecked(self, arg: OpArg) -> T { - // SAFETY: requirements forwarded from caller - unsafe { T::from_op_arg(arg.0).unwrap_unchecked() } - } -} - -impl<T: OpArgType> PartialEq for Arg<T> { - fn eq(&self, _: &Self) -> bool { - true - } -} - -impl<T: OpArgType> Eq for Arg<T> {} - -impl<T: OpArgType> fmt::Debug for Arg<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Arg<{}>", std::any::type_name::<T>()) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] -#[repr(transparent)] -// XXX: if you add a new instruction that stores a Label, make sure to add it in -// Instruction::label_arg -pub struct Label(pub u32); - -impl OpArgType for Label { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(Self(x)) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - self.0 - } -} - -impl fmt::Display for Label { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -op_arg_enum!( - /// The kind of Raise that occurred. - #[derive(Copy, Clone, Debug, PartialEq, Eq)] - #[repr(u8)] - pub enum RaiseKind { - Reraise = 0, - Raise = 1, - RaiseCause = 2, - } -); - -op_arg_enum!( - /// Intrinsic function for CALL_INTRINSIC_1 - #[derive(Copy, Clone, Debug, PartialEq, Eq)] - #[repr(u8)] - pub enum IntrinsicFunction1 { - // Invalid = 0, - Print = 1, - /// Import * operation - ImportStar = 2, - // StopIterationError = 3, - // AsyncGenWrap = 4, - // UnaryPositive = 5, - /// Convert list to tuple - ListToTuple = 6, - /// Type parameter related - TypeVar = 7, - ParamSpec = 8, - TypeVarTuple = 9, - /// Generic subscript for PEP 695 - SubscriptGeneric = 10, - TypeAlias = 11, - } -); - -op_arg_enum!( - /// Intrinsic function for CALL_INTRINSIC_2 - #[derive(Copy, Clone, Debug, PartialEq, Eq)] - #[repr(u8)] - pub enum IntrinsicFunction2 { - // PrepReraiseS tar = 1, - TypeVarWithBound = 2, - TypeVarWithConstraint = 3, - SetFunctionTypeParams = 4, - /// Set default value for type parameter (PEP 695) - SetTypeparamDefault = 5, - } -); - -pub type NameIdx = u32; - -/// A Single bytecode instruction. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum Instruction { - BeforeAsyncWith, - BinaryOp { - op: Arg<BinaryOperator>, - }, - BinarySubscript, - Break { - target: Arg<Label>, - }, - BuildListFromTuples { - size: Arg<u32>, - }, - BuildList { - size: Arg<u32>, - }, - BuildMapForCall { - size: Arg<u32>, - }, - BuildMap { - size: Arg<u32>, - }, - BuildSetFromTuples { - size: Arg<u32>, - }, - BuildSet { - size: Arg<u32>, - }, - BuildSlice { - argc: Arg<BuildSliceArgCount>, - }, - BuildString { - size: Arg<u32>, - }, - BuildTupleFromIter, - BuildTupleFromTuples { - size: Arg<u32>, - }, - BuildTuple { - size: Arg<u32>, - }, - CallFunctionEx { - has_kwargs: Arg<bool>, - }, - CallFunctionKeyword { - nargs: Arg<u32>, - }, - CallFunctionPositional { - nargs: Arg<u32>, - }, - CallIntrinsic1 { - func: Arg<IntrinsicFunction1>, - }, - CallIntrinsic2 { - func: Arg<IntrinsicFunction2>, - }, - CallMethodEx { - has_kwargs: Arg<bool>, - }, - CallMethodKeyword { - nargs: Arg<u32>, - }, - CallMethodPositional { - nargs: Arg<u32>, - }, - CompareOperation { - op: Arg<ComparisonOperator>, - }, - /// Performs `in` comparison, or `not in` if `invert` is 1. - ContainsOp(Arg<Invert>), - Continue { - target: Arg<Label>, - }, - /// Convert value to a string, depending on `oparg`: - /// - /// ```python - /// value = STACK.pop() - /// result = func(value) - /// STACK.append(result) - /// ``` - /// - /// Used for implementing formatted string literals (f-strings). - ConvertValue { - oparg: Arg<ConvertValueOparg>, - }, - CopyItem { - index: Arg<u32>, - }, - DeleteAttr { - idx: Arg<NameIdx>, - }, - DeleteDeref(Arg<NameIdx>), - DeleteFast(Arg<NameIdx>), - DeleteGlobal(Arg<NameIdx>), - DeleteLocal(Arg<NameIdx>), - DeleteSubscript, - DictUpdate { - index: Arg<u32>, - }, - EndAsyncFor, - /// Marker bytecode for the end of a finally sequence. - /// When this bytecode is executed, the eval loop does one of those things: - /// - Continue at a certain bytecode position - /// - Propagate the exception - /// - Return from a function - /// - Do nothing at all, just continue - EndFinally, - /// Enter a finally block, without returning, excepting, just because we are there. - EnterFinally, - ExtendedArg, - ForIter { - target: Arg<Label>, - }, - /// Formats the value on top of stack: - /// - /// ```python - /// value = STACK.pop() - /// result = value.__format__("") - /// STACK.append(result) - /// ``` - /// - /// Used for implementing formatted string literals (f-strings). - FormatSimple, - /// Formats the given value with the given format spec: - /// - /// ```python - /// spec = STACK.pop() - /// value = STACK.pop() - /// result = value.__format__(spec) - /// STACK.append(result) - /// ``` - /// - /// Used for implementing formatted string literals (f-strings). - FormatWithSpec, - GetAIter, - GetANext, - GetAwaitable, - GetIter, - GetLen, - /// from ... import ... - ImportFrom { - idx: Arg<NameIdx>, - }, - /// Importing by name - ImportName { - idx: Arg<NameIdx>, - }, - /// Performs `is` comparison, or `is not` if `invert` is 1. - IsOp(Arg<Invert>), - /// Peek at the top of the stack, and jump if this value is false. - /// Otherwise, pop top of stack. - JumpIfFalseOrPop { - target: Arg<Label>, - }, - /// Performs exception matching for except. - /// Tests whether the STACK[-2] is an exception matching STACK[-1]. - /// Pops STACK[-1] and pushes the boolean result of the test. - JumpIfNotExcMatch(Arg<Label>), - /// Peek at the top of the stack, and jump if this value is true. - /// Otherwise, pop top of stack. - JumpIfTrueOrPop { - target: Arg<Label>, - }, - Jump { - target: Arg<Label>, - }, - ListAppend { - i: Arg<u32>, - }, - LoadAttr { - idx: Arg<NameIdx>, - }, - LoadBuildClass, - LoadClassDeref(Arg<NameIdx>), - LoadClosure(Arg<NameIdx>), - LoadConst { - /// index into constants vec - idx: Arg<u32>, - }, - LoadDeref(Arg<NameIdx>), - LoadFast(Arg<NameIdx>), - LoadGlobal(Arg<NameIdx>), - LoadMethod { - idx: Arg<NameIdx>, - }, - LoadNameAny(Arg<NameIdx>), - MakeFunction, - MapAdd { - i: Arg<u32>, - }, - MatchClass(Arg<u32>), - MatchKeys, - MatchMapping, - MatchSequence, - Nop, - Pop, - PopBlock, - PopException, - /// Pop the top of the stack, and jump if this value is false. - PopJumpIfFalse { - target: Arg<Label>, - }, - /// Pop the top of the stack, and jump if this value is true. - PopJumpIfTrue { - target: Arg<Label>, - }, - Raise { - kind: Arg<RaiseKind>, - }, - /// Resume execution (e.g., at function start, after yield, etc.) - Resume { - arg: Arg<u32>, - }, - ReturnConst { - idx: Arg<u32>, - }, - ReturnValue, - Reverse { - amount: Arg<u32>, - }, - SetAdd { - i: Arg<u32>, - }, - SetFunctionAttribute { - attr: Arg<MakeFunctionFlags>, - }, - SetupAnnotation, - SetupAsyncWith { - end: Arg<Label>, - }, - - SetupExcept { - handler: Arg<Label>, - }, - /// Setup a finally handler, which will be called whenever one of this events occurs: - /// - the block is popped - /// - the function returns - /// - an exception is returned - SetupFinally { - handler: Arg<Label>, - }, - SetupLoop, - SetupWith { - end: Arg<Label>, - }, - StoreAttr { - idx: Arg<NameIdx>, - }, - StoreDeref(Arg<NameIdx>), - StoreFast(Arg<NameIdx>), - StoreGlobal(Arg<NameIdx>), - StoreLocal(Arg<NameIdx>), - StoreSubscript, - Subscript, - Swap { - index: Arg<u32>, - }, - ToBool, - UnaryOperation { - op: Arg<UnaryOperator>, - }, - UnpackEx { - args: Arg<UnpackExArgs>, - }, - UnpackSequence { - size: Arg<u32>, - }, - WithCleanupFinish, - WithCleanupStart, - YieldFrom, - YieldValue, - // If you add a new instruction here, be sure to keep LAST_INSTRUCTION updated -} - -// This must be kept up to date to avoid marshaling errors -const LAST_INSTRUCTION: Instruction = Instruction::YieldValue; - -const _: () = assert!(mem::size_of::<Instruction>() == 1); - -impl From<Instruction> for u8 { - #[inline] - fn from(ins: Instruction) -> Self { - // SAFETY: there's no padding bits - unsafe { std::mem::transmute::<Instruction, Self>(ins) } + pub struct CodeFlags: u32 { + const OPTIMIZED = 0x0001; + const NEWLOCALS = 0x0002; + const VARARGS = 0x0004; + const VARKEYWORDS = 0x0008; + const GENERATOR = 0x0020; + const COROUTINE = 0x0080; + const ITERABLE_COROUTINE = 0x0100; + /// If a code object represents a function and has a docstring, + /// this bit is set and the first item in co_consts is the docstring. + const HAS_DOCSTRING = 0x4000000; } } -impl TryFrom<u8> for Instruction { - type Error = MarshalError; - - #[inline] - fn try_from(value: u8) -> Result<Self, MarshalError> { - if value <= u8::from(LAST_INSTRUCTION) { - Ok(unsafe { std::mem::transmute::<u8, Self>(value) }) - } else { - Err(MarshalError::InvalidBytecode) - } - } -} - -#[derive(Copy, Clone)] #[repr(C)] +#[derive(Copy, Clone, Debug)] pub struct CodeUnit { pub op: Instruction, pub arg: OpArgByte, @@ -913,7 +329,7 @@ impl TryFrom<&[u8]> for CodeUnit { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct CodeUnits(Box<[CodeUnit]>); impl TryFrom<&[u8]> for CodeUnits { @@ -954,31 +370,6 @@ impl Deref for CodeUnits { } } -use self::Instruction::*; - -bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct MakeFunctionFlags: u8 { - const CLOSURE = 0x01; - const ANNOTATIONS = 0x02; - const KW_ONLY_DEFAULTS = 0x04; - const DEFAULTS = 0x08; - const TYPE_PARAMS = 0x10; - } -} - -impl OpArgType for MakeFunctionFlags { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Self::from_bits(x as u8) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - self.bits().into() - } -} - /// A Constant (which usually encapsulates data within it) /// /// # Examples @@ -1017,7 +408,7 @@ impl PartialEq for ConstantData { (Boolean { value: a }, Boolean { value: b }) => a == b, (Str { value: a }, Str { value: b }) => a == b, (Bytes { value: a }, Bytes { value: b }) => a == b, - (Code { code: a }, Code { code: b }) => std::ptr::eq(a.as_ref(), b.as_ref()), + (Code { code: a }, Code { code: b }) => core::ptr::eq(a.as_ref(), b.as_ref()), (Tuple { elements: a }, Tuple { elements: b }) => a == b, (None, None) => true, (Ellipsis, Ellipsis) => true, @@ -1043,7 +434,7 @@ impl hash::Hash for ConstantData { Boolean { value } => value.hash(state), Str { value } => value.hash(state), Bytes { value } => value.hash(state), - Code { code } => std::ptr::hash(code.as_ref(), state), + Code { code } => core::ptr::hash(code.as_ref(), state), Tuple { elements } => elements.hash(state), None => {} Ellipsis => {} @@ -1134,257 +525,6 @@ impl<C: Constant> BorrowedConstant<'_, C> { } } -op_arg_enum!( - /// The possible comparison operators - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum ComparisonOperator { - // be intentional with bits so that we can do eval_ord with just a bitwise and - // bits: | Equal | Greater | Less | - Less = 0b001, - Greater = 0b010, - NotEqual = 0b011, - Equal = 0b100, - LessOrEqual = 0b101, - GreaterOrEqual = 0b110, - } -); - -op_arg_enum!( - /// The possible Binary operators - /// - /// # Examples - /// - /// ```rust - /// use rustpython_compiler_core::bytecode::{Arg, BinaryOperator, Instruction}; - /// let (op, _) = Arg::new(BinaryOperator::Add); - /// let instruction = Instruction::BinaryOp { op }; - /// ``` - /// - /// See also: - /// - [_PyEval_BinaryOps](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Python/ceval.c#L316-L343) - #[repr(u8)] - #[derive(Clone, Copy, Debug, Eq, PartialEq)] - pub enum BinaryOperator { - /// `+` - Add = 0, - /// `&` - And = 1, - /// `//` - FloorDivide = 2, - /// `<<` - Lshift = 3, - /// `@` - MatrixMultiply = 4, - /// `*` - Multiply = 5, - /// `%` - Remainder = 6, - /// `|` - Or = 7, - /// `**` - Power = 8, - /// `>>` - Rshift = 9, - /// `-` - Subtract = 10, - /// `/` - TrueDivide = 11, - /// `^` - Xor = 12, - /// `+=` - InplaceAdd = 13, - /// `&=` - InplaceAnd = 14, - /// `//=` - InplaceFloorDivide = 15, - /// `<<=` - InplaceLshift = 16, - /// `@=` - InplaceMatrixMultiply = 17, - /// `*=` - InplaceMultiply = 18, - /// `%=` - InplaceRemainder = 19, - /// `|=` - InplaceOr = 20, - /// `**=` - InplacePower = 21, - /// `>>=` - InplaceRshift = 22, - /// `-=` - InplaceSubtract = 23, - /// `/=` - InplaceTrueDivide = 24, - /// `^=` - InplaceXor = 25, - } -); - -impl BinaryOperator { - /// Get the "inplace" version of the operator. - /// This has no effect if `self` is already an "inplace" operator. - /// - /// # Example - /// ```rust - /// use rustpython_compiler_core::bytecode::BinaryOperator; - /// - /// assert_eq!(BinaryOperator::Power.as_inplace(), BinaryOperator::InplacePower); - /// - /// assert_eq!(BinaryOperator::InplaceSubtract.as_inplace(), BinaryOperator::InplaceSubtract); - /// ``` - #[must_use] - pub const fn as_inplace(self) -> Self { - match self { - Self::Add => Self::InplaceAdd, - Self::And => Self::InplaceAnd, - Self::FloorDivide => Self::InplaceFloorDivide, - Self::Lshift => Self::InplaceLshift, - Self::MatrixMultiply => Self::InplaceMatrixMultiply, - Self::Multiply => Self::InplaceMultiply, - Self::Remainder => Self::InplaceRemainder, - Self::Or => Self::InplaceOr, - Self::Power => Self::InplacePower, - Self::Rshift => Self::InplaceRshift, - Self::Subtract => Self::InplaceSubtract, - Self::TrueDivide => Self::InplaceTrueDivide, - Self::Xor => Self::InplaceXor, - _ => self, - } - } -} - -impl fmt::Display for BinaryOperator { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let op = match self { - Self::Add => "+", - Self::And => "&", - Self::FloorDivide => "//", - Self::Lshift => "<<", - Self::MatrixMultiply => "@", - Self::Multiply => "*", - Self::Remainder => "%", - Self::Or => "|", - Self::Power => "**", - Self::Rshift => ">>", - Self::Subtract => "-", - Self::TrueDivide => "/", - Self::Xor => "^", - Self::InplaceAdd => "+=", - Self::InplaceAnd => "&=", - Self::InplaceFloorDivide => "//=", - Self::InplaceLshift => "<<=", - Self::InplaceMatrixMultiply => "@=", - Self::InplaceMultiply => "*=", - Self::InplaceRemainder => "%=", - Self::InplaceOr => "|=", - Self::InplacePower => "**=", - Self::InplaceRshift => ">>=", - Self::InplaceSubtract => "-=", - Self::InplaceTrueDivide => "/=", - Self::InplaceXor => "^=", - }; - write!(f, "{op}") - } -} - -op_arg_enum!( - /// The possible unary operators - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum UnaryOperator { - Not = 0, - Invert = 1, - Minus = 2, - Plus = 3, - } -); - -op_arg_enum!( - /// Whether or not to invert the operation. - #[repr(u8)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum Invert { - /// ```py - /// foo is bar - /// x in lst - /// ``` - No = 0, - /// ```py - /// foo is not bar - /// x not in lst - /// ``` - Yes = 1, - } -); - -/// Specifies if a slice is built with either 2 or 3 arguments. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BuildSliceArgCount { - /// ```py - /// x[5:10] - /// ``` - Two, - /// ```py - /// x[5:10:2] - /// ``` - Three, -} - -impl OpArgType for BuildSliceArgCount { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(match x { - 2 => Self::Two, - 3 => Self::Three, - _ => return None, - }) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - u32::from(self.argc().get()) - } -} - -impl BuildSliceArgCount { - /// Get the numeric value of `Self`. - #[must_use] - pub const fn argc(self) -> NonZeroU8 { - let inner = match self { - Self::Two => 2, - Self::Three => 3, - }; - // Safety: `inner` can be either 2 or 3. - unsafe { NonZeroU8::new_unchecked(inner) } - } -} - -#[derive(Copy, Clone)] -pub struct UnpackExArgs { - pub before: u8, - pub after: u8, -} - -impl OpArgType for UnpackExArgs { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - let [before, after, ..] = x.to_le_bytes(); - Some(Self { before, after }) - } - - #[inline(always)] - fn to_op_arg(self) -> u32 { - u32::from_le_bytes([self.before, self.after, 0, 0]) - } -} - -impl fmt::Display for UnpackExArgs { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "before: {}, after: {}", self.before, self.after) - } -} - /* Maintain a stack of blocks on the VM. pub enum BlockType { @@ -1430,14 +570,14 @@ impl<C: Constant> CodeObject<C> { let args = &self.varnames[..nargs]; let kwonlyargs = &self.varnames[nargs..varargs_pos]; - let vararg = if self.flags.contains(CodeFlags::HAS_VARARGS) { + let vararg = if self.flags.contains(CodeFlags::VARARGS) { let vararg = &self.varnames[varargs_pos]; varargs_pos += 1; Some(vararg) } else { None }; - let varkwarg = if self.flags.contains(CodeFlags::HAS_VARKEYWORDS) { + let varkwarg = if self.flags.contains(CodeFlags::VARKEYWORDS) { Some(&self.varnames[varargs_pos]) } else { None @@ -1472,14 +612,14 @@ impl<C: Constant> CodeObject<C> { level: usize, ) -> fmt::Result { let label_targets = self.label_targets(); - let line_digits = (3).max(self.locations.last().unwrap().line.digits().get()); + let line_digits = (3).max(self.locations.last().unwrap().0.line.digits().get()); let offset_digits = (4).max(1 + self.instructions.len().ilog10() as usize); let mut last_line = OneIndexed::MAX; let mut arg_state = OpArgState::default(); for (offset, &instruction) in self.instructions.iter().enumerate() { let (instruction, arg) = arg_state.get(instruction); // optional line number - let line = self.locations[offset].line; + let line = self.locations[offset].0.line; if line != last_line { if last_line != OneIndexed::MAX { writeln!(f)?; @@ -1608,368 +748,6 @@ impl<C: Constant> fmt::Display for CodeObject<C> { } } -impl Instruction { - /// Gets the label stored inside this instruction, if it exists - #[inline] - pub const fn label_arg(&self) -> Option<Arg<Label>> { - match self { - Jump { target: l } - | JumpIfNotExcMatch(l) - | PopJumpIfTrue { target: l } - | PopJumpIfFalse { target: l } - | JumpIfTrueOrPop { target: l } - | JumpIfFalseOrPop { target: l } - | ForIter { target: l } - | SetupFinally { handler: l } - | SetupExcept { handler: l } - | SetupWith { end: l } - | SetupAsyncWith { end: l } - | Break { target: l } - | Continue { target: l } => Some(*l), - _ => None, - } - } - - /// Whether this is an unconditional branching - /// - /// # Examples - /// - /// ``` - /// use rustpython_compiler_core::bytecode::{Arg, Instruction}; - /// let jump_inst = Instruction::Jump { target: Arg::marker() }; - /// assert!(jump_inst.unconditional_branch()) - /// ``` - pub const fn unconditional_branch(&self) -> bool { - matches!( - self, - Jump { .. } - | Continue { .. } - | Break { .. } - | ReturnValue - | ReturnConst { .. } - | Raise { .. } - ) - } - - /// What effect this instruction has on the stack - /// - /// # Examples - /// - /// ``` - /// use rustpython_compiler_core::bytecode::{Arg, Instruction, Label, UnaryOperator}; - /// let (target, jump_arg) = Arg::new(Label(0xF)); - /// let jump_instruction = Instruction::Jump { target }; - /// let (op, invert_arg) = Arg::new(UnaryOperator::Invert); - /// let invert_instruction = Instruction::UnaryOperation { op }; - /// assert_eq!(jump_instruction.stack_effect(jump_arg, true), 0); - /// assert_eq!(invert_instruction.stack_effect(invert_arg, false), 0); - /// ``` - /// - pub fn stack_effect(&self, arg: OpArg, jump: bool) -> i32 { - match self { - Nop => 0, - ImportName { .. } => -1, - ImportFrom { .. } => 1, - LoadFast(_) | LoadNameAny(_) | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) => 1, - StoreFast(_) | StoreLocal(_) | StoreGlobal(_) | StoreDeref(_) => -1, - DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_) | DeleteDeref(_) => 0, - LoadClosure(_) => 1, - Subscript => -1, - StoreSubscript => -3, - DeleteSubscript => -2, - LoadAttr { .. } => 0, - StoreAttr { .. } => -2, - DeleteAttr { .. } => -1, - LoadConst { .. } => 1, - UnaryOperation { .. } => 0, - BinaryOp { .. } | CompareOperation { .. } => -1, - BinarySubscript => -1, - CopyItem { .. } => 1, - Pop => -1, - Swap { .. } => 0, - ToBool => 0, - GetIter => 0, - GetLen => 1, - CallIntrinsic1 { .. } => 0, // Takes 1, pushes 1 - CallIntrinsic2 { .. } => -1, // Takes 2, pushes 1 - Continue { .. } => 0, - Break { .. } => 0, - Jump { .. } => 0, - PopJumpIfTrue { .. } | PopJumpIfFalse { .. } => -1, - JumpIfTrueOrPop { .. } | JumpIfFalseOrPop { .. } => { - if jump { - 0 - } else { - -1 - } - } - MakeFunction => { - // CPython 3.13 style: MakeFunction only pops code object - -1 + 1 // pop code, push function - } - SetFunctionAttribute { .. } => { - // pops attribute value and function, pushes function back - -2 + 1 - } - CallFunctionPositional { nargs } => -(nargs.get(arg) as i32) - 1 + 1, - CallMethodPositional { nargs } => -(nargs.get(arg) as i32) - 3 + 1, - CallFunctionKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 1 + 1, - CallMethodKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 3 + 1, - CallFunctionEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 1 + 1, - CallMethodEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 3 + 1, - ConvertValue { .. } => 0, - FormatSimple => 0, - FormatWithSpec => -1, - LoadMethod { .. } => -1 + 3, - ForIter { .. } => { - if jump { - -1 - } else { - 1 - } - } - IsOp(_) | ContainsOp(_) => -1, - JumpIfNotExcMatch(_) => -2, - ReturnValue => -1, - ReturnConst { .. } => 0, - Resume { .. } => 0, - YieldValue => 0, - YieldFrom => -1, - SetupAnnotation | SetupLoop | SetupFinally { .. } | EnterFinally | EndFinally => 0, - SetupExcept { .. } => jump as i32, - SetupWith { .. } => (!jump) as i32, - WithCleanupStart => 0, - WithCleanupFinish => -1, - PopBlock => 0, - Raise { kind } => -(kind.get(arg) as u8 as i32), - BuildString { size } - | BuildTuple { size, .. } - | BuildTupleFromTuples { size, .. } - | BuildList { size, .. } - | BuildListFromTuples { size, .. } - | BuildSet { size, .. } - | BuildSetFromTuples { size, .. } => -(size.get(arg) as i32) + 1, - BuildTupleFromIter => 0, - BuildMap { size } => { - let nargs = size.get(arg) * 2; - -(nargs as i32) + 1 - } - BuildMapForCall { size } => { - let nargs = size.get(arg); - -(nargs as i32) + 1 - } - DictUpdate { .. } => -1, - BuildSlice { argc } => { - // push 1 - // pops either 2/3 - 1 - (argc.get(arg).argc().get() as i32) - } - ListAppend { .. } | SetAdd { .. } => -1, - MapAdd { .. } => -2, - LoadBuildClass => 1, - UnpackSequence { size } => -1 + size.get(arg) as i32, - UnpackEx { args } => { - let UnpackExArgs { before, after } = args.get(arg); - -1 + before as i32 + 1 + after as i32 - } - PopException => 0, - Reverse { .. } => 0, - GetAwaitable => 0, - BeforeAsyncWith => 1, - SetupAsyncWith { .. } => { - if jump { - -1 - } else { - 0 - } - } - GetAIter => 0, - GetANext => 1, - EndAsyncFor => -2, - MatchMapping | MatchSequence => 1, // Push bool result - MatchKeys => 1, // Pop 2 (subject, keys), push 3 (subject, keys_or_none, values_or_none) - MatchClass(_) => -2, - ExtendedArg => 0, - } - } - - pub fn display<'a>( - &'a self, - arg: OpArg, - ctx: &'a impl InstrDisplayContext, - ) -> impl fmt::Display + 'a { - struct FmtFn<F>(F); - impl<F: Fn(&mut fmt::Formatter<'_>) -> fmt::Result> fmt::Display for FmtFn<F> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (self.0)(f) - } - } - FmtFn(move |f: &mut fmt::Formatter<'_>| self.fmt_dis(arg, f, ctx, false, 0, 0)) - } - - #[allow(clippy::too_many_arguments)] - fn fmt_dis( - &self, - arg: OpArg, - f: &mut fmt::Formatter<'_>, - ctx: &impl InstrDisplayContext, - expand_code_objects: bool, - pad: usize, - level: usize, - ) -> fmt::Result { - macro_rules! w { - ($variant:ident) => { - write!(f, stringify!($variant)) - }; - ($variant:ident, $map:ident = $arg_marker:expr) => {{ - let arg = $arg_marker.get(arg); - write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg)) - }}; - ($variant:ident, $arg_marker:expr) => { - write!(f, "{:pad$}({})", stringify!($variant), $arg_marker.get(arg)) - }; - ($variant:ident, ?$arg_marker:expr) => { - write!( - f, - "{:pad$}({:?})", - stringify!($variant), - $arg_marker.get(arg) - ) - }; - } - - let varname = |i: u32| ctx.get_varname(i as usize); - let name = |i: u32| ctx.get_name(i as usize); - let cell_name = |i: u32| ctx.get_cell_name(i as usize); - - let fmt_const = - |op: &str, arg: OpArg, f: &mut fmt::Formatter<'_>, idx: &Arg<u32>| -> fmt::Result { - let value = ctx.get_constant(idx.get(arg) as usize); - match value.borrow_constant() { - BorrowedConstant::Code { code } if expand_code_objects => { - write!(f, "{op:pad$}({code:?}):")?; - code.display_inner(f, true, level + 1)?; - Ok(()) - } - c => { - write!(f, "{op:pad$}(")?; - c.fmt_display(f)?; - write!(f, ")") - } - } - }; - - match self { - BeforeAsyncWith => w!(BeforeAsyncWith), - BinaryOp { op } => write!(f, "{:pad$}({})", "BINARY_OP", op.get(arg)), - BinarySubscript => w!(BinarySubscript), - Break { target } => w!(Break, target), - BuildListFromTuples { size } => w!(BuildListFromTuples, size), - BuildList { size } => w!(BuildList, size), - BuildMapForCall { size } => w!(BuildMapForCall, size), - BuildMap { size } => w!(BuildMap, size), - BuildSetFromTuples { size } => w!(BuildSetFromTuples, size), - BuildSet { size } => w!(BuildSet, size), - BuildSlice { argc } => w!(BuildSlice, ?argc), - BuildString { size } => w!(BuildString, size), - BuildTupleFromIter => w!(BuildTupleFromIter), - BuildTupleFromTuples { size } => w!(BuildTupleFromTuples, size), - BuildTuple { size } => w!(BuildTuple, size), - CallFunctionEx { has_kwargs } => w!(CallFunctionEx, has_kwargs), - CallFunctionKeyword { nargs } => w!(CallFunctionKeyword, nargs), - CallFunctionPositional { nargs } => w!(CallFunctionPositional, nargs), - CallIntrinsic1 { func } => w!(CallIntrinsic1, ?func), - CallIntrinsic2 { func } => w!(CallIntrinsic2, ?func), - CallMethodEx { has_kwargs } => w!(CallMethodEx, has_kwargs), - CallMethodKeyword { nargs } => w!(CallMethodKeyword, nargs), - CallMethodPositional { nargs } => w!(CallMethodPositional, nargs), - CompareOperation { op } => w!(CompareOperation, ?op), - ContainsOp(inv) => w!(CONTAINS_OP, ?inv), - Continue { target } => w!(Continue, target), - ConvertValue { oparg } => write!(f, "{:pad$}{}", "CONVERT_VALUE", oparg.get(arg)), - CopyItem { index } => w!(CopyItem, index), - DeleteAttr { idx } => w!(DeleteAttr, name = idx), - DeleteDeref(idx) => w!(DeleteDeref, cell_name = idx), - DeleteFast(idx) => w!(DeleteFast, varname = idx), - DeleteGlobal(idx) => w!(DeleteGlobal, name = idx), - DeleteLocal(idx) => w!(DeleteLocal, name = idx), - DeleteSubscript => w!(DeleteSubscript), - DictUpdate { index } => w!(DictUpdate, index), - EndAsyncFor => w!(EndAsyncFor), - EndFinally => w!(EndFinally), - EnterFinally => w!(EnterFinally), - ExtendedArg => w!(ExtendedArg, Arg::<u32>::marker()), - ForIter { target } => w!(ForIter, target), - FormatSimple => w!(FORMAT_SIMPLE), - FormatWithSpec => w!(FORMAT_WITH_SPEC), - GetAIter => w!(GetAIter), - GetANext => w!(GetANext), - GetAwaitable => w!(GetAwaitable), - GetIter => w!(GetIter), - GetLen => w!(GetLen), - ImportFrom { idx } => w!(ImportFrom, name = idx), - ImportName { idx } => w!(ImportName, name = idx), - IsOp(inv) => w!(IS_OP, ?inv), - JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, target), - JumpIfNotExcMatch(target) => w!(JUMP_IF_NOT_EXC_MATCH, target), - JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, target), - Jump { target } => w!(Jump, target), - ListAppend { i } => w!(ListAppend, i), - LoadAttr { idx } => w!(LoadAttr, name = idx), - LoadBuildClass => w!(LoadBuildClass), - LoadClassDeref(idx) => w!(LoadClassDeref, cell_name = idx), - LoadClosure(i) => w!(LoadClosure, cell_name = i), - LoadConst { idx } => fmt_const("LoadConst", arg, f, idx), - LoadDeref(idx) => w!(LoadDeref, cell_name = idx), - LoadFast(idx) => w!(LoadFast, varname = idx), - LoadGlobal(idx) => w!(LoadGlobal, name = idx), - LoadMethod { idx } => w!(LoadMethod, name = idx), - LoadNameAny(idx) => w!(LoadNameAny, name = idx), - MakeFunction => w!(MakeFunction), - MapAdd { i } => w!(MapAdd, i), - MatchClass(arg) => w!(MatchClass, arg), - MatchKeys => w!(MatchKeys), - MatchMapping => w!(MatchMapping), - MatchSequence => w!(MatchSequence), - Nop => w!(Nop), - Pop => w!(Pop), - PopBlock => w!(PopBlock), - PopException => w!(PopException), - PopJumpIfFalse { target } => w!(PopJumpIfFalse, target), - PopJumpIfTrue { target } => w!(PopJumpIfTrue, target), - Raise { kind } => w!(Raise, ?kind), - Resume { arg } => w!(Resume, arg), - ReturnConst { idx } => fmt_const("ReturnConst", arg, f, idx), - ReturnValue => w!(ReturnValue), - Reverse { amount } => w!(Reverse, amount), - SetAdd { i } => w!(SetAdd, i), - SetFunctionAttribute { attr } => w!(SetFunctionAttribute, ?attr), - SetupAnnotation => w!(SetupAnnotation), - SetupAsyncWith { end } => w!(SetupAsyncWith, end), - SetupExcept { handler } => w!(SetupExcept, handler), - SetupFinally { handler } => w!(SetupFinally, handler), - SetupLoop => w!(SetupLoop), - SetupWith { end } => w!(SetupWith, end), - StoreAttr { idx } => w!(StoreAttr, name = idx), - StoreDeref(idx) => w!(StoreDeref, cell_name = idx), - StoreFast(idx) => w!(StoreFast, varname = idx), - StoreGlobal(idx) => w!(StoreGlobal, name = idx), - StoreLocal(idx) => w!(StoreLocal, name = idx), - StoreSubscript => w!(StoreSubscript), - Subscript => w!(Subscript), - Swap { index } => w!(Swap, index), - ToBool => w!(ToBool), - UnaryOperation { op } => w!(UnaryOperation, ?op), - UnpackEx { args } => w!(UnpackEx, args), - UnpackSequence { size } => w!(UnpackSequence, size), - WithCleanupFinish => w!(WithCleanupFinish), - WithCleanupStart => w!(WithCleanupStart), - YieldFrom => w!(YieldFrom), - YieldValue => w!(YieldValue), - } - } -} - pub trait InstrDisplayContext { type Constant: Constant; @@ -2022,3 +800,75 @@ impl<C: Constant> fmt::Debug for CodeObject<C> { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloc::{vec, vec::Vec}; + + #[test] + fn test_exception_table_encode_decode() { + let entries = vec![ + ExceptionTableEntry::new(0, 10, 20, 2, false), + ExceptionTableEntry::new(15, 25, 30, 1, true), + ]; + + let encoded = encode_exception_table(&entries); + + // Find handler at offset 5 (in range [0, 10)) + let handler = find_exception_handler(&encoded, 5); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.start, 0); + assert_eq!(handler.end, 10); + assert_eq!(handler.target, 20); + assert_eq!(handler.depth, 2); + assert!(!handler.push_lasti); + + // Find handler at offset 20 (in range [15, 25)) + let handler = find_exception_handler(&encoded, 20); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.start, 15); + assert_eq!(handler.end, 25); + assert_eq!(handler.target, 30); + assert_eq!(handler.depth, 1); + assert!(handler.push_lasti); + + // No handler at offset 12 (not in any range) + let handler = find_exception_handler(&encoded, 12); + assert!(handler.is_none()); + + // No handler at offset 30 (past all ranges) + let handler = find_exception_handler(&encoded, 30); + assert!(handler.is_none()); + } + + #[test] + fn test_exception_table_empty() { + let entries: Vec<ExceptionTableEntry> = vec![]; + let encoded = encode_exception_table(&entries); + assert!(encoded.is_empty()); + assert!(find_exception_handler(&encoded, 0).is_none()); + } + + #[test] + fn test_exception_table_single_entry() { + let entries = vec![ExceptionTableEntry::new(5, 15, 100, 3, true)]; + let encoded = encode_exception_table(&entries); + + // Inside range + let handler = find_exception_handler(&encoded, 10); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.target, 100); + assert_eq!(handler.depth, 3); + assert!(handler.push_lasti); + + // At start boundary (inclusive) + assert!(find_exception_handler(&encoded, 5).is_some()); + + // At end boundary (exclusive) + assert!(find_exception_handler(&encoded, 15).is_none()); + } +} diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs new file mode 100644 index 00000000000..fd2d06761a3 --- /dev/null +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -0,0 +1,1294 @@ +use core::{fmt, marker::PhantomData, mem}; + +use crate::{ + bytecode::{ + BorrowedConstant, Constant, InstrDisplayContext, + oparg::{ + BinaryOperator, BuildSliceArgCount, CommonConstant, ComparisonOperator, + ConvertValueOparg, IntrinsicFunction1, IntrinsicFunction2, Invert, Label, LoadAttr, + LoadSuperAttr, MakeFunctionFlags, NameIdx, OpArg, OpArgByte, OpArgType, RaiseKind, + SpecialMethod, UnpackExArgs, + }, + }, + marshal::MarshalError, +}; + +/// A Single bytecode instruction that are executed by the VM. +/// +/// Currently aligned with CPython 3.14. +/// +/// ## See also +/// - [CPython opcode IDs](https://github.com/python/cpython/blob/v3.14.2/Include/opcode_ids.h) +#[derive(Clone, Copy, Debug)] +#[repr(u8)] +pub enum Instruction { + // No-argument instructions (opcode < HAVE_ARGUMENT=44) + Cache = 0, + BinarySlice = 1, + BuildTemplate = 2, + BinaryOpInplaceAddUnicode = 3, + CallFunctionEx = 4, + CheckEgMatch = 5, + CheckExcMatch = 6, + CleanupThrow = 7, + DeleteSubscr = 8, + EndFor = 9, + EndSend = 10, + ExitInitCheck = 11, // Placeholder + FormatSimple = 12, + FormatWithSpec = 13, + GetAIter = 14, + GetANext = 15, + GetIter = 16, + Reserved = 17, + GetLen = 18, + GetYieldFromIter = 19, + InterpreterExit = 20, // Placeholder + LoadBuildClass = 21, + LoadLocals = 22, + MakeFunction = 23, + MatchKeys = 24, + MatchMapping = 25, + MatchSequence = 26, + Nop = 27, + NotTaken = 28, + PopExcept = 29, + PopIter = 30, + PopTop = 31, + PushExcInfo = 32, + PushNull = 33, + ReturnGenerator = 34, + ReturnValue = 35, + SetupAnnotations = 36, + StoreSlice = 37, + StoreSubscr = 38, + ToBool = 39, + UnaryInvert = 40, + UnaryNegative = 41, + UnaryNot = 42, + WithExceptStart = 43, + // CPython 3.14 opcodes with arguments (44-120) + BinaryOp { + op: Arg<BinaryOperator>, + } = 44, + /// Build an Interpolation from value, expression string, and optional format_spec on stack. + /// + /// oparg encoding: (conversion << 2) | has_format_spec + /// - has_format_spec (bit 0): if 1, format_spec is on stack + /// - conversion (bits 2+): 0=None, 1=Str, 2=Repr, 3=Ascii + /// + /// Stack: [value, expression_str, format_spec?] -> [interpolation] + BuildInterpolation { + oparg: Arg<u32>, + } = 45, + BuildList { + size: Arg<u32>, + } = 46, + BuildMap { + size: Arg<u32>, + } = 47, + BuildSet { + size: Arg<u32>, + } = 48, + BuildSlice { + argc: Arg<BuildSliceArgCount>, + } = 49, + BuildString { + size: Arg<u32>, + } = 50, + BuildTuple { + size: Arg<u32>, + } = 51, + Call { + nargs: Arg<u32>, + } = 52, + CallIntrinsic1 { + func: Arg<IntrinsicFunction1>, + } = 53, + CallIntrinsic2 { + func: Arg<IntrinsicFunction2>, + } = 54, + CallKw { + nargs: Arg<u32>, + } = 55, + CompareOp { + op: Arg<ComparisonOperator>, + } = 56, + ContainsOp(Arg<Invert>) = 57, + ConvertValue { + oparg: Arg<ConvertValueOparg>, + } = 58, + Copy { + index: Arg<u32>, + } = 59, + CopyFreeVars { + count: Arg<u32>, + } = 60, + DeleteAttr { + idx: Arg<NameIdx>, + } = 61, + DeleteDeref(Arg<NameIdx>) = 62, + DeleteFast(Arg<NameIdx>) = 63, + DeleteGlobal(Arg<NameIdx>) = 64, + DeleteName(Arg<NameIdx>) = 65, + DictMerge { + index: Arg<u32>, + } = 66, + DictUpdate { + index: Arg<u32>, + } = 67, + EndAsyncFor = 68, + ExtendedArg = 69, + ForIter { + target: Arg<Label>, + } = 70, + GetAwaitable { + arg: Arg<u32>, + } = 71, + ImportFrom { + idx: Arg<NameIdx>, + } = 72, + ImportName { + idx: Arg<NameIdx>, + } = 73, + IsOp(Arg<Invert>) = 74, + JumpBackward { + target: Arg<Label>, + } = 75, + JumpBackwardNoInterrupt { + target: Arg<Label>, + } = 76, // Placeholder + JumpForward { + target: Arg<Label>, + } = 77, + ListAppend { + i: Arg<u32>, + } = 78, + ListExtend { + i: Arg<u32>, + } = 79, + LoadAttr { + idx: Arg<LoadAttr>, + } = 80, + LoadCommonConstant { + idx: Arg<CommonConstant>, + } = 81, + LoadConst { + idx: Arg<u32>, + } = 82, + LoadDeref(Arg<NameIdx>) = 83, + LoadFast(Arg<NameIdx>) = 84, + LoadFastAndClear(Arg<NameIdx>) = 85, + LoadFastBorrow(Arg<NameIdx>) = 86, + LoadFastBorrowLoadFastBorrow { + arg: Arg<u32>, + } = 87, + LoadFastCheck(Arg<NameIdx>) = 88, + LoadFastLoadFast { + arg: Arg<u32>, + } = 89, + LoadFromDictOrDeref(Arg<NameIdx>) = 90, + LoadFromDictOrGlobals(Arg<NameIdx>) = 91, + LoadGlobal(Arg<NameIdx>) = 92, + LoadName(Arg<NameIdx>) = 93, + LoadSmallInt { + idx: Arg<u32>, + } = 94, + LoadSpecial { + method: Arg<SpecialMethod>, + } = 95, + LoadSuperAttr { + arg: Arg<LoadSuperAttr>, + } = 96, + MakeCell(Arg<NameIdx>) = 97, + MapAdd { + i: Arg<u32>, + } = 98, + MatchClass(Arg<u32>) = 99, + PopJumpIfFalse { + target: Arg<Label>, + } = 100, + PopJumpIfNone { + target: Arg<Label>, + } = 101, + PopJumpIfNotNone { + target: Arg<Label>, + } = 102, + PopJumpIfTrue { + target: Arg<Label>, + } = 103, + RaiseVarargs { + kind: Arg<RaiseKind>, + } = 104, + Reraise { + depth: Arg<u32>, + } = 105, + Send { + target: Arg<Label>, + } = 106, + SetAdd { + i: Arg<u32>, + } = 107, + SetFunctionAttribute { + attr: Arg<MakeFunctionFlags>, + } = 108, + SetUpdate { + i: Arg<u32>, + } = 109, + StoreAttr { + idx: Arg<NameIdx>, + } = 110, + StoreDeref(Arg<NameIdx>) = 111, + StoreFast(Arg<NameIdx>) = 112, + StoreFastLoadFast { + store_idx: Arg<NameIdx>, + load_idx: Arg<NameIdx>, + } = 113, + StoreFastStoreFast { + arg: Arg<u32>, + } = 114, + StoreGlobal(Arg<NameIdx>) = 115, + StoreName(Arg<NameIdx>) = 116, + Swap { + index: Arg<u32>, + } = 117, + UnpackEx { + args: Arg<UnpackExArgs>, + } = 118, + UnpackSequence { + size: Arg<u32>, + } = 119, + YieldValue { + arg: Arg<u32>, + } = 120, + // CPython 3.14 RESUME (128) + Resume { + arg: Arg<u32>, + } = 128, + // CPython 3.14 specialized opcodes (129-211) + BinaryOpAddFloat = 129, // Placeholder + BinaryOpAddInt = 130, // Placeholder + BinaryOpAddUnicode = 131, // Placeholder + BinaryOpExtend = 132, // Placeholder + BinaryOpMultiplyFloat = 133, // Placeholder + BinaryOpMultiplyInt = 134, // Placeholder + BinaryOpSubscrDict = 135, // Placeholder + BinaryOpSubscrGetitem = 136, // Placeholder + BinaryOpSubscrListInt = 137, // Placeholder + BinaryOpSubscrListSlice = 138, // Placeholder + BinaryOpSubscrStrInt = 139, // Placeholder + BinaryOpSubscrTupleInt = 140, // Placeholder + BinaryOpSubtractFloat = 141, // Placeholder + BinaryOpSubtractInt = 142, // Placeholder + CallAllocAndEnterInit = 143, // Placeholder + CallBoundMethodExactArgs = 144, // Placeholder + CallBoundMethodGeneral = 145, // Placeholder + CallBuiltinClass = 146, // Placeholder + CallBuiltinFast = 147, // Placeholder + CallBuiltinFastWithKeywords = 148, // Placeholder + CallBuiltinO = 149, // Placeholder + CallIsinstance = 150, // Placeholder + CallKwBoundMethod = 151, // Placeholder + CallKwNonPy = 152, // Placeholder + CallKwPy = 153, // Placeholder + CallLen = 154, // Placeholder + CallListAppend = 155, // Placeholder + CallMethodDescriptorFast = 156, // Placeholder + CallMethodDescriptorFastWithKeywords = 157, // Placeholder + CallMethodDescriptorNoargs = 158, // Placeholder + CallMethodDescriptorO = 159, // Placeholder + CallNonPyGeneral = 160, // Placeholder + CallPyExactArgs = 161, // Placeholder + CallPyGeneral = 162, // Placeholder + CallStr1 = 163, // Placeholder + CallTuple1 = 164, // Placeholder + CallType1 = 165, // Placeholder + CompareOpFloat = 166, // Placeholder + CompareOpInt = 167, // Placeholder + CompareOpStr = 168, // Placeholder + ContainsOpDict = 169, // Placeholder + ContainsOpSet = 170, // Placeholder + ForIterGen = 171, // Placeholder + ForIterList = 172, // Placeholder + ForIterRange = 173, // Placeholder + ForIterTuple = 174, // Placeholder + JumpBackwardJit = 175, // Placeholder + JumpBackwardNoJit = 176, // Placeholder + LoadAttrClass = 177, // Placeholder + LoadAttrClassWithMetaclassCheck = 178, // Placeholder + LoadAttrGetattributeOverridden = 179, // Placeholder + LoadAttrInstanceValue = 180, // Placeholder + LoadAttrMethodLazyDict = 181, // Placeholder + LoadAttrMethodNoDict = 182, // Placeholder + LoadAttrMethodWithValues = 183, // Placeholder + LoadAttrModule = 184, // Placeholder + LoadAttrNondescriptorNoDict = 185, // Placeholder + LoadAttrNondescriptorWithValues = 186, // Placeholder + LoadAttrProperty = 187, // Placeholder + LoadAttrSlot = 188, // Placeholder + LoadAttrWithHint = 189, // Placeholder + LoadConstImmortal = 190, // Placeholder + LoadConstMortal = 191, // Placeholder + LoadGlobalBuiltin = 192, // Placeholder + LoadGlobalModule = 193, // Placeholder + LoadSuperAttrAttr = 194, // Placeholder + LoadSuperAttrMethod = 195, // Placeholder + ResumeCheck = 196, // Placeholder + SendGen = 197, // Placeholder + StoreAttrInstanceValue = 198, // Placeholder + StoreAttrSlot = 199, // Placeholder + StoreAttrWithHint = 200, // Placeholder + StoreSubscrDict = 201, // Placeholder + StoreSubscrListInt = 202, // Placeholder + ToBoolAlwaysTrue = 203, // Placeholder + ToBoolBool = 204, // Placeholder + ToBoolInt = 205, // Placeholder + ToBoolList = 206, // Placeholder + ToBoolNone = 207, // Placeholder + ToBoolStr = 208, // Placeholder + UnpackSequenceList = 209, // Placeholder + UnpackSequenceTuple = 210, // Placeholder + UnpackSequenceTwoTuple = 211, // Placeholder + // CPython 3.14 instrumented opcodes (234-254) + InstrumentedEndFor = 234, // Placeholder + InstrumentedPopIter = 235, // Placeholder + InstrumentedEndSend = 236, // Placeholder + InstrumentedForIter = 237, // Placeholder + InstrumentedInstruction = 238, // Placeholder + InstrumentedJumpForward = 239, // Placeholder + InstrumentedNotTaken = 240, + InstrumentedPopJumpIfTrue = 241, // Placeholder + InstrumentedPopJumpIfFalse = 242, // Placeholder + InstrumentedPopJumpIfNone = 243, // Placeholder + InstrumentedPopJumpIfNotNone = 244, // Placeholder + InstrumentedResume = 245, // Placeholder + InstrumentedReturnValue = 246, // Placeholder + InstrumentedYieldValue = 247, // Placeholder + InstrumentedEndAsyncFor = 248, // Placeholder + InstrumentedLoadSuperAttr = 249, // Placeholder + InstrumentedCall = 250, // Placeholder + InstrumentedCallKw = 251, // Placeholder + InstrumentedCallFunctionEx = 252, // Placeholder + InstrumentedJumpBackward = 253, // Placeholder + InstrumentedLine = 254, // Placeholder + EnterExecutor = 255, // Placeholder +} + +const _: () = assert!(mem::size_of::<Instruction>() == 1); + +impl From<Instruction> for u8 { + #[inline] + fn from(ins: Instruction) -> Self { + // SAFETY: there's no padding bits + unsafe { mem::transmute::<Instruction, Self>(ins) } + } +} + +impl TryFrom<u8> for Instruction { + type Error = MarshalError; + + #[inline] + fn try_from(value: u8) -> Result<Self, MarshalError> { + // CPython-compatible opcodes (0-120) + let cpython_start = u8::from(Self::Cache); + let cpython_end = u8::from(Self::YieldValue { arg: Arg::marker() }); + + // Resume has a non-contiguous opcode (128) + let resume_id = u8::from(Self::Resume { arg: Arg::marker() }); + let enter_executor_id = u8::from(Self::EnterExecutor); + + let specialized_start = u8::from(Self::BinaryOpAddFloat); + let specialized_end = u8::from(Self::UnpackSequenceTwoTuple); + + let instrumented_start = u8::from(Self::InstrumentedEndFor); + let instrumented_end = u8::from(Self::InstrumentedLine); + + // No RustPython-only opcodes anymore - all opcodes match CPython 3.14 + let custom_ops: &[u8] = &[]; + + if (cpython_start..=cpython_end).contains(&value) + || value == resume_id + || value == enter_executor_id + || custom_ops.contains(&value) + || (specialized_start..=specialized_end).contains(&value) + || (instrumented_start..=instrumented_end).contains(&value) + { + Ok(unsafe { mem::transmute::<u8, Self>(value) }) + } else { + Err(Self::Error::InvalidBytecode) + } + } +} + +impl InstructionMetadata for Instruction { + #[inline] + fn label_arg(&self) -> Option<Arg<Label>> { + match self { + Self::JumpBackward { target: l } + | Self::JumpBackwardNoInterrupt { target: l } + | Self::JumpForward { target: l } + | Self::PopJumpIfTrue { target: l } + | Self::PopJumpIfFalse { target: l } + | Self::PopJumpIfNone { target: l } + | Self::PopJumpIfNotNone { target: l } + | Self::ForIter { target: l } + | Self::Send { target: l } => Some(*l), + _ => None, + } + } + + fn is_unconditional_jump(&self) -> bool { + matches!( + self, + Self::JumpForward { .. } + | Self::JumpBackward { .. } + | Self::JumpBackwardNoInterrupt { .. } + ) + } + + fn is_scope_exit(&self) -> bool { + matches!( + self, + Self::ReturnValue | Self::RaiseVarargs { .. } | Self::Reraise { .. } + ) + } + + fn stack_effect_info(&self, oparg: u32) -> StackEffect { + // Reason for converting oparg to i32 is because of expressions like `1 + (oparg -1)` + // that causes underflow errors. + let oparg = i32::try_from(oparg).expect("oparg does not fit in an `i32`"); + + // NOTE: Please don't "simplify" expressions here (i.e. `1 + (oparg - 1)`) + // as it will be harder to see diff with what CPython auto-generates + let (pushed, popped) = match self { + Self::BinaryOp { .. } => (1, 2), + Self::BinaryOpAddFloat => (1, 2), + Self::BinaryOpAddInt => (1, 2), + Self::BinaryOpAddUnicode => (1, 2), + Self::BinaryOpExtend => (1, 2), + Self::BinaryOpInplaceAddUnicode => (0, 2), + Self::BinaryOpMultiplyFloat => (1, 2), + Self::BinaryOpMultiplyInt => (1, 2), + Self::BinaryOpSubscrDict => (1, 2), + Self::BinaryOpSubscrGetitem => (0, 2), + Self::BinaryOpSubscrListInt => (1, 2), + Self::BinaryOpSubscrListSlice => (1, 2), + Self::BinaryOpSubscrStrInt => (1, 2), + Self::BinaryOpSubscrTupleInt => (1, 2), + Self::BinaryOpSubtractFloat => (1, 2), + Self::BinaryOpSubtractInt => (1, 2), + Self::BinarySlice { .. } => (1, 3), + Self::BuildInterpolation { .. } => (1, 2 + (oparg & 1)), + Self::BuildList { .. } => (1, oparg), + Self::BuildMap { .. } => (1, oparg * 2), + Self::BuildSet { .. } => (1, oparg), + Self::BuildSlice { .. } => (1, oparg), + Self::BuildString { .. } => (1, oparg), + Self::BuildTemplate { .. } => (1, 2), + Self::BuildTuple { .. } => (1, oparg), + Self::Cache => (0, 0), + Self::Call { .. } => (1, 2 + oparg), + Self::CallAllocAndEnterInit => (0, 2 + oparg), + Self::CallBoundMethodExactArgs => (0, 2 + oparg), + Self::CallBoundMethodGeneral => (0, 2 + oparg), + Self::CallBuiltinClass => (1, 2 + oparg), + Self::CallBuiltinFast => (1, 2 + oparg), + Self::CallBuiltinFastWithKeywords => (1, 2 + oparg), + Self::CallBuiltinO => (1, 2 + oparg), + Self::CallFunctionEx => (1, 4), + Self::CallIntrinsic1 { .. } => (1, 1), + Self::CallIntrinsic2 { .. } => (1, 2), + Self::CallIsinstance => (1, 2 + oparg), + Self::CallKw { .. } => (1, 3 + oparg), + Self::CallKwBoundMethod => (0, 3 + oparg), + Self::CallKwNonPy => (1, 3 + oparg), + Self::CallKwPy => (0, 3 + oparg), + Self::CallLen => (1, 3), + Self::CallListAppend => (0, 3), + Self::CallMethodDescriptorFast => (1, 2 + oparg), + Self::CallMethodDescriptorFastWithKeywords => (1, 2 + oparg), + Self::CallMethodDescriptorNoargs => (1, 2 + oparg), + Self::CallMethodDescriptorO => (1, 2 + oparg), + Self::CallNonPyGeneral => (1, 2 + oparg), + Self::CallPyExactArgs => (0, 2 + oparg), + Self::CallPyGeneral => (0, 2 + oparg), + Self::CallStr1 => (1, 3), + Self::CallTuple1 => (1, 3), + Self::CallType1 => (1, 3), + Self::CheckEgMatch => (2, 2), + Self::CheckExcMatch => (2, 2), + Self::CleanupThrow => (2, 3), + Self::CompareOp { .. } => (1, 2), + Self::CompareOpFloat => (1, 2), + Self::CompareOpInt => (1, 2), + Self::CompareOpStr => (1, 2), + Self::ContainsOp(_) => (1, 2), + Self::ContainsOpDict => (1, 2), + Self::ContainsOpSet => (1, 2), + Self::ConvertValue { .. } => (1, 1), + Self::Copy { .. } => (2 + (oparg - 1), 1 + (oparg - 1)), + Self::CopyFreeVars { .. } => (0, 0), + Self::DeleteAttr { .. } => (0, 1), + Self::DeleteDeref(_) => (0, 0), + Self::DeleteFast(_) => (0, 0), + Self::DeleteGlobal(_) => (0, 0), + Self::DeleteName(_) => (0, 0), + Self::DeleteSubscr => (0, 2), + Self::DictMerge { .. } => (4 + (oparg - 1), 5 + (oparg - 1)), + Self::DictUpdate { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::EndAsyncFor => (0, 2), + Self::EndFor => (0, 1), + Self::EndSend => (1, 2), + Self::EnterExecutor => (0, 0), + Self::ExitInitCheck => (0, 1), + Self::ExtendedArg => (0, 0), + Self::ForIter { .. } => (2, 1), + Self::ForIterGen => (1, 1), + Self::ForIterList => (2, 1), + Self::ForIterRange => (2, 1), + Self::ForIterTuple => (2, 1), + Self::FormatSimple => (1, 1), + Self::FormatWithSpec => (1, 2), + Self::GetAIter => (1, 1), + Self::GetANext => (2, 1), + Self::GetAwaitable { .. } => (1, 1), + Self::GetIter => (1, 1), + Self::GetLen => (2, 1), + Self::GetYieldFromIter => (1, 1), + Self::ImportFrom { .. } => (2, 1), + Self::ImportName { .. } => (1, 2), + Self::InstrumentedCall => (1, 2 + oparg), + Self::InstrumentedCallFunctionEx => (1, 4), + Self::InstrumentedCallKw => (1, 3 + oparg), + Self::InstrumentedEndAsyncFor => (0, 2), + Self::InstrumentedEndFor => (1, 2), + Self::InstrumentedEndSend => (1, 2), + Self::InstrumentedForIter => (2, 1), + Self::InstrumentedInstruction => (0, 0), + Self::InstrumentedJumpBackward => (0, 0), + Self::InstrumentedJumpForward => (0, 0), + Self::InstrumentedLine => (0, 0), + Self::InstrumentedLoadSuperAttr => (1 + (oparg & 1), 3), + Self::InstrumentedNotTaken => (0, 0), + Self::InstrumentedPopIter => (0, 1), + Self::InstrumentedPopJumpIfFalse => (0, 1), + Self::InstrumentedPopJumpIfNone => (0, 1), + Self::InstrumentedPopJumpIfNotNone => (0, 1), + Self::InstrumentedPopJumpIfTrue => (0, 1), + Self::InstrumentedResume => (0, 0), + Self::InstrumentedReturnValue => (1, 1), + Self::InstrumentedYieldValue => (1, 1), + Self::InterpreterExit => (0, 1), + Self::IsOp(_) => (1, 2), + Self::JumpBackward { .. } => (0, 0), + Self::JumpBackwardJit => (0, 0), + Self::JumpBackwardNoInterrupt { .. } => (0, 0), + Self::JumpBackwardNoJit => (0, 0), + Self::JumpForward { .. } => (0, 0), + Self::ListAppend { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::ListExtend { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::LoadAttr { .. } => (1 + (oparg & 1), 1), + Self::LoadAttrClass => (1 + (oparg & 1), 1), + Self::LoadAttrClassWithMetaclassCheck => (1 + (oparg & 1), 1), + Self::LoadAttrGetattributeOverridden => (1, 1), + Self::LoadAttrInstanceValue => (1 + (oparg & 1), 1), + Self::LoadAttrMethodLazyDict => (2, 1), + Self::LoadAttrMethodNoDict => (2, 1), + Self::LoadAttrMethodWithValues => (2, 1), + Self::LoadAttrModule => (1 + (oparg & 1), 1), + Self::LoadAttrNondescriptorNoDict => (1, 1), + Self::LoadAttrNondescriptorWithValues => (1, 1), + Self::LoadAttrProperty => (0, 1), + Self::LoadAttrSlot => (1 + (oparg & 1), 1), + Self::LoadAttrWithHint => (1 + (oparg & 1), 1), + Self::LoadBuildClass => (1, 0), + Self::LoadCommonConstant { .. } => (1, 0), + Self::LoadConst { .. } => (1, 0), + Self::LoadConstImmortal => (1, 0), + Self::LoadConstMortal => (1, 0), + Self::LoadDeref(_) => (1, 0), + Self::LoadFast(_) => (1, 0), + Self::LoadFastAndClear(_) => (1, 0), + Self::LoadFastBorrow(_) => (1, 0), + Self::LoadFastBorrowLoadFastBorrow { .. } => (2, 0), + Self::LoadFastCheck(_) => (1, 0), + Self::LoadFastLoadFast { .. } => (2, 0), + Self::LoadFromDictOrDeref(_) => (1, 1), + Self::LoadFromDictOrGlobals(_) => (1, 1), + Self::LoadGlobal(_) => ( + 1, // TODO: Differs from CPython `1 + (oparg & 1)` + 0, + ), + Self::LoadGlobalBuiltin => (1 + (oparg & 1), 0), + Self::LoadGlobalModule => (1 + (oparg & 1), 0), + Self::LoadLocals => (1, 0), + Self::LoadName(_) => (1, 0), + Self::LoadSmallInt { .. } => (1, 0), + Self::LoadSpecial { .. } => (1, 1), + Self::LoadSuperAttr { .. } => (1 + (oparg & 1), 3), + Self::LoadSuperAttrAttr => (1, 3), + Self::LoadSuperAttrMethod => (2, 3), + Self::MakeCell(_) => (0, 0), + Self::MakeFunction { .. } => (1, 1), + Self::MapAdd { .. } => (1 + (oparg - 1), 3 + (oparg - 1)), + Self::MatchClass { .. } => (1, 3), + Self::MatchKeys { .. } => (3, 2), + Self::MatchMapping => (2, 1), + Self::MatchSequence => (2, 1), + Self::Nop => (0, 0), + Self::NotTaken => (0, 0), + Self::PopExcept => (0, 1), + Self::PopIter => (0, 1), + Self::PopJumpIfFalse { .. } => (0, 1), + Self::PopJumpIfNone { .. } => (0, 1), + Self::PopJumpIfNotNone { .. } => (0, 1), + Self::PopJumpIfTrue { .. } => (0, 1), + Self::PopTop => (0, 1), + Self::PushExcInfo => (2, 1), + Self::PushNull => (1, 0), + Self::RaiseVarargs { kind } => ( + 0, + // TODO: Differs from CPython: `oparg` + match kind.get((oparg as u32).into()) { + RaiseKind::BareRaise => 0, + RaiseKind::Raise => 1, + RaiseKind::RaiseCause => 2, + RaiseKind::ReraiseFromStack => 1, + }, + ), + Self::Reraise { .. } => ( + 1 + oparg, // TODO: Differs from CPython: `oparg` + 1 + oparg, + ), + Self::Reserved => (0, 0), + Self::Resume { .. } => (0, 0), + Self::ResumeCheck => (0, 0), + Self::ReturnGenerator => (1, 0), + Self::ReturnValue => ( + 0, // TODO: Differs from CPython: `1` + 1, + ), + Self::Send { .. } => (2, 2), + Self::SendGen => (1, 2), + Self::SetAdd { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::SetFunctionAttribute { .. } => (1, 2), + Self::SetUpdate { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::SetupAnnotations => (0, 0), + Self::StoreAttr { .. } => (0, 2), + Self::StoreAttrInstanceValue => (0, 2), + Self::StoreAttrSlot => (0, 2), + Self::StoreAttrWithHint => (0, 2), + Self::StoreDeref(_) => (0, 1), + Self::StoreFast(_) => (0, 1), + Self::StoreFastLoadFast { .. } => (1, 1), + Self::StoreFastStoreFast { .. } => (0, 2), + Self::StoreGlobal(_) => (0, 1), + Self::StoreName(_) => (0, 1), + Self::StoreSlice => (0, 4), + Self::StoreSubscr => (0, 3), + Self::StoreSubscrDict => (0, 3), + Self::StoreSubscrListInt => (0, 3), + Self::Swap { .. } => (2 + (oparg - 2), 2 + (oparg - 2)), + Self::ToBool => (1, 1), + Self::ToBoolAlwaysTrue => (1, 1), + Self::ToBoolBool => (1, 1), + Self::ToBoolInt => (1, 1), + Self::ToBoolList => (1, 1), + Self::ToBoolNone => (1, 1), + Self::ToBoolStr => (1, 1), + Self::UnaryInvert => (1, 1), + Self::UnaryNegative => (1, 1), + Self::UnaryNot => (1, 1), + Self::UnpackEx { .. } => (1 + (oparg & 0xFF) + (oparg >> 8), 1), + Self::UnpackSequence { .. } => (oparg, 1), + Self::UnpackSequenceList => (oparg, 1), + Self::UnpackSequenceTuple => (oparg, 1), + Self::UnpackSequenceTwoTuple => (2, 1), + Self::WithExceptStart => (6, 5), + Self::YieldValue { .. } => (1, 1), + }; + + debug_assert!((0..=i32::MAX).contains(&pushed)); + debug_assert!((0..=i32::MAX).contains(&popped)); + + StackEffect::new(pushed as u32, popped as u32) + } + + #[allow(clippy::too_many_arguments)] + fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize, + ) -> fmt::Result { + macro_rules! w { + ($variant:ident) => { + write!(f, stringify!($variant)) + }; + ($variant:ident, $map:ident = $arg_marker:expr) => {{ + let arg = $arg_marker.get(arg); + write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg)) + }}; + ($variant:ident, $arg_marker:expr) => { + write!(f, "{:pad$}({})", stringify!($variant), $arg_marker.get(arg)) + }; + ($variant:ident, ?$arg_marker:expr) => { + write!( + f, + "{:pad$}({:?})", + stringify!($variant), + $arg_marker.get(arg) + ) + }; + } + + let varname = |i: u32| ctx.get_varname(i as usize); + let name = |i: u32| ctx.get_name(i as usize); + let cell_name = |i: u32| ctx.get_cell_name(i as usize); + + let fmt_const = + |op: &str, arg: OpArg, f: &mut fmt::Formatter<'_>, idx: &Arg<u32>| -> fmt::Result { + let value = ctx.get_constant(idx.get(arg) as usize); + match value.borrow_constant() { + BorrowedConstant::Code { code } if expand_code_objects => { + write!(f, "{op:pad$}({code:?}):")?; + code.display_inner(f, true, level + 1)?; + Ok(()) + } + c => { + write!(f, "{op:pad$}(")?; + c.fmt_display(f)?; + write!(f, ")") + } + } + }; + + match self { + Self::BinaryOp { op } => write!(f, "{:pad$}({})", "BINARY_OP", op.get(arg)), + Self::BuildList { size } => w!(BUILD_LIST, size), + Self::BuildMap { size } => w!(BUILD_MAP, size), + Self::BuildSet { size } => w!(BUILD_SET, size), + Self::BuildSlice { argc } => w!(BUILD_SLICE, ?argc), + Self::BuildString { size } => w!(BUILD_STRING, size), + Self::BuildTuple { size } => w!(BUILD_TUPLE, size), + Self::Call { nargs } => w!(CALL, nargs), + Self::CallFunctionEx => w!(CALL_FUNCTION_EX), + Self::CallKw { nargs } => w!(CALL_KW, nargs), + Self::CallIntrinsic1 { func } => w!(CALL_INTRINSIC_1, ?func), + Self::CallIntrinsic2 { func } => w!(CALL_INTRINSIC_2, ?func), + Self::CheckEgMatch => w!(CHECK_EG_MATCH), + Self::CheckExcMatch => w!(CHECK_EXC_MATCH), + Self::CleanupThrow => w!(CLEANUP_THROW), + Self::CompareOp { op } => w!(COMPARE_OP, ?op), + Self::ContainsOp(inv) => w!(CONTAINS_OP, ?inv), + Self::ConvertValue { oparg } => write!(f, "{:pad$}{}", "CONVERT_VALUE", oparg.get(arg)), + Self::Copy { index } => w!(COPY, index), + Self::DeleteAttr { idx } => w!(DELETE_ATTR, name = idx), + Self::DeleteDeref(idx) => w!(DELETE_DEREF, cell_name = idx), + Self::DeleteFast(idx) => w!(DELETE_FAST, varname = idx), + Self::DeleteGlobal(idx) => w!(DELETE_GLOBAL, name = idx), + Self::DeleteName(idx) => w!(DELETE_NAME, name = idx), + Self::DeleteSubscr => w!(DELETE_SUBSCR), + Self::DictMerge { index } => w!(DICT_MERGE, index), + Self::DictUpdate { index } => w!(DICT_UPDATE, index), + Self::EndAsyncFor => w!(END_ASYNC_FOR), + Self::EndSend => w!(END_SEND), + Self::ExtendedArg => w!(EXTENDED_ARG, Arg::<u32>::marker()), + Self::ForIter { target } => w!(FOR_ITER, target), + Self::FormatSimple => w!(FORMAT_SIMPLE), + Self::FormatWithSpec => w!(FORMAT_WITH_SPEC), + Self::GetAIter => w!(GET_AITER), + Self::GetANext => w!(GET_ANEXT), + Self::GetAwaitable { arg } => w!(GET_AWAITABLE, arg), + Self::Reserved => w!(RESERVED), + Self::GetIter => w!(GET_ITER), + Self::GetLen => w!(GET_LEN), + Self::ImportFrom { idx } => w!(IMPORT_FROM, name = idx), + Self::ImportName { idx } => w!(IMPORT_NAME, name = idx), + Self::IsOp(inv) => w!(IS_OP, ?inv), + Self::JumpBackward { target } => w!(JUMP_BACKWARD, target), + Self::JumpBackwardNoInterrupt { target } => w!(JUMP_BACKWARD_NO_INTERRUPT, target), + Self::JumpForward { target } => w!(JUMP_FORWARD, target), + Self::ListAppend { i } => w!(LIST_APPEND, i), + Self::ListExtend { i } => w!(LIST_EXTEND, i), + Self::LoadAttr { idx } => { + let oparg = idx.get(arg); + let oparg_u32 = u32::from(oparg); + let attr_name = name(oparg.name_idx()); + if oparg.is_method() { + write!( + f, + "{:pad$}({}, {}, method=true)", + "LOAD_ATTR", oparg_u32, attr_name + ) + } else { + write!(f, "{:pad$}({}, {})", "LOAD_ATTR", oparg_u32, attr_name) + } + } + Self::LoadBuildClass => w!(LOAD_BUILD_CLASS), + Self::LoadFromDictOrDeref(i) => w!(LOAD_FROM_DICT_OR_DEREF, cell_name = i), + Self::LoadConst { idx } => fmt_const("LOAD_CONST", arg, f, idx), + Self::LoadSmallInt { idx } => w!(LOAD_SMALL_INT, idx), + Self::LoadDeref(idx) => w!(LOAD_DEREF, cell_name = idx), + Self::LoadFast(idx) => w!(LOAD_FAST, varname = idx), + Self::LoadFastAndClear(idx) => w!(LOAD_FAST_AND_CLEAR, varname = idx), + Self::LoadFastBorrow(idx) => w!(LOAD_FAST_BORROW, varname = idx), + Self::LoadFastCheck(idx) => w!(LOAD_FAST_CHECK, varname = idx), + Self::LoadFastLoadFast { arg: packed } => { + let oparg = packed.get(arg); + let idx1 = oparg >> 4; + let idx2 = oparg & 15; + let name1 = varname(idx1); + let name2 = varname(idx2); + write!(f, "{:pad$}({}, {})", "LOAD_FAST_LOAD_FAST", name1, name2) + } + Self::LoadFastBorrowLoadFastBorrow { arg: packed } => { + let oparg = packed.get(arg); + let idx1 = oparg >> 4; + let idx2 = oparg & 15; + let name1 = varname(idx1); + let name2 = varname(idx2); + write!( + f, + "{:pad$}({}, {})", + "LOAD_FAST_BORROW_LOAD_FAST_BORROW", name1, name2 + ) + } + Self::LoadFromDictOrGlobals(idx) => w!(LOAD_FROM_DICT_OR_GLOBALS, name = idx), + Self::LoadGlobal(idx) => w!(LOAD_GLOBAL, name = idx), + Self::LoadName(idx) => w!(LOAD_NAME, name = idx), + Self::LoadSpecial { method } => w!(LOAD_SPECIAL, method), + Self::LoadSuperAttr { arg: idx } => { + let oparg = idx.get(arg); + write!( + f, + "{:pad$}({}, {}, method={}, class={})", + "LOAD_SUPER_ATTR", + u32::from(oparg), + name(oparg.name_idx()), + oparg.is_load_method(), + oparg.has_class() + ) + } + Self::MakeFunction => w!(MAKE_FUNCTION), + Self::MapAdd { i } => w!(MAP_ADD, i), + Self::MatchClass(arg) => w!(MATCH_CLASS, arg), + Self::MatchKeys => w!(MATCH_KEYS), + Self::MatchMapping => w!(MATCH_MAPPING), + Self::MatchSequence => w!(MATCH_SEQUENCE), + Self::Nop => w!(NOP), + Self::PopExcept => w!(POP_EXCEPT), + Self::PopJumpIfFalse { target } => w!(POP_JUMP_IF_FALSE, target), + Self::PopJumpIfTrue { target } => w!(POP_JUMP_IF_TRUE, target), + Self::PopTop => w!(POP_TOP), + Self::EndFor => w!(END_FOR), + Self::PopIter => w!(POP_ITER), + Self::PushExcInfo => w!(PUSH_EXC_INFO), + Self::PushNull => w!(PUSH_NULL), + Self::RaiseVarargs { kind } => w!(RAISE_VARARGS, ?kind), + Self::Reraise { depth } => w!(RERAISE, depth), + Self::Resume { arg } => w!(RESUME, arg), + Self::ReturnValue => w!(RETURN_VALUE), + Self::ReturnGenerator => w!(RETURN_GENERATOR), + Self::Send { target } => w!(SEND, target), + Self::SetAdd { i } => w!(SET_ADD, i), + Self::SetFunctionAttribute { attr } => w!(SET_FUNCTION_ATTRIBUTE, ?attr), + Self::SetupAnnotations => w!(SETUP_ANNOTATIONS), + Self::SetUpdate { i } => w!(SET_UPDATE, i), + Self::StoreAttr { idx } => w!(STORE_ATTR, name = idx), + Self::StoreDeref(idx) => w!(STORE_DEREF, cell_name = idx), + Self::StoreFast(idx) => w!(STORE_FAST, varname = idx), + Self::StoreFastLoadFast { + store_idx, + load_idx, + } => { + write!(f, "STORE_FAST_LOAD_FAST")?; + write!(f, " ({}, {})", store_idx.get(arg), load_idx.get(arg)) + } + Self::StoreGlobal(idx) => w!(STORE_GLOBAL, name = idx), + Self::StoreName(idx) => w!(STORE_NAME, name = idx), + Self::StoreSubscr => w!(STORE_SUBSCR), + Self::Swap { index } => w!(SWAP, index), + Self::ToBool => w!(TO_BOOL), + Self::UnpackEx { args } => w!(UNPACK_EX, args), + Self::UnpackSequence { size } => w!(UNPACK_SEQUENCE, size), + Self::WithExceptStart => w!(WITH_EXCEPT_START), + Self::UnaryInvert => w!(UNARY_INVERT), + Self::UnaryNegative => w!(UNARY_NEGATIVE), + Self::UnaryNot => w!(UNARY_NOT), + Self::YieldValue { arg } => w!(YIELD_VALUE, arg), + Self::GetYieldFromIter => w!(GET_YIELD_FROM_ITER), + Self::BuildTemplate => w!(BUILD_TEMPLATE), + Self::BuildInterpolation { oparg } => w!(BUILD_INTERPOLATION, oparg), + _ => w!(RUSTPYTHON_PLACEHOLDER), + } + } +} + +/// Instructions used by the compiler. They are not executed by the VM. +/// +/// CPython 3.14.2 aligned (256-266). +#[derive(Clone, Copy, Debug)] +#[repr(u16)] +pub enum PseudoInstruction { + // CPython 3.14.2 pseudo instructions (256-266) + AnnotationsPlaceholder = 256, + Jump { target: Arg<Label> } = 257, + JumpIfFalse { target: Arg<Label> } = 258, + JumpIfTrue { target: Arg<Label> } = 259, + JumpNoInterrupt { target: Arg<Label> } = 260, + LoadClosure(Arg<NameIdx>) = 261, + PopBlock = 262, + SetupCleanup { target: Arg<Label> } = 263, + SetupFinally { target: Arg<Label> } = 264, + SetupWith { target: Arg<Label> } = 265, + StoreFastMaybeNull(Arg<NameIdx>) = 266, +} + +const _: () = assert!(mem::size_of::<PseudoInstruction>() == 2); + +impl From<PseudoInstruction> for u16 { + #[inline] + fn from(ins: PseudoInstruction) -> Self { + // SAFETY: there's no padding bits + unsafe { mem::transmute::<PseudoInstruction, Self>(ins) } + } +} + +impl TryFrom<u16> for PseudoInstruction { + type Error = MarshalError; + + #[inline] + fn try_from(value: u16) -> Result<Self, MarshalError> { + let start = u16::from(Self::AnnotationsPlaceholder); + let end = u16::from(Self::StoreFastMaybeNull(Arg::marker())); + + if (start..=end).contains(&value) { + Ok(unsafe { mem::transmute::<u16, Self>(value) }) + } else { + Err(Self::Error::InvalidBytecode) + } + } +} + +impl PseudoInstruction { + /// Returns true if this is a block push pseudo instruction + /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). + pub fn is_block_push(&self) -> bool { + matches!( + self, + Self::SetupCleanup { .. } | Self::SetupFinally { .. } | Self::SetupWith { .. } + ) + } +} + +impl InstructionMetadata for PseudoInstruction { + fn label_arg(&self) -> Option<Arg<Label>> { + match self { + Self::Jump { target: l } + | Self::JumpIfFalse { target: l } + | Self::JumpIfTrue { target: l } + | Self::JumpNoInterrupt { target: l } + | Self::SetupCleanup { target: l } + | Self::SetupFinally { target: l } + | Self::SetupWith { target: l } => Some(*l), + _ => None, + } + } + + fn is_scope_exit(&self) -> bool { + false + } + + fn stack_effect_info(&self, _oparg: u32) -> StackEffect { + // Reason for converting oparg to i32 is because of expressions like `1 + (oparg -1)` + // that causes underflow errors. + let _oparg = i32::try_from(_oparg).expect("oparg does not fit in an `i32`"); + + // NOTE: Please don't "simplify" expressions here (i.e. `1 + (oparg - 1)`) + // as it will be harder to see diff with what CPython auto-generates + let (pushed, popped) = match self { + Self::AnnotationsPlaceholder => (0, 0), + Self::Jump { .. } => (0, 0), + Self::JumpIfFalse { .. } => (1, 1), + Self::JumpIfTrue { .. } => (1, 1), + Self::JumpNoInterrupt { .. } => (0, 0), + Self::LoadClosure(_) => (1, 0), + Self::PopBlock => (0, 0), + // Normal path effect is 0 (these are NOPs on fall-through). + // Handler entry effects are computed directly in max_stackdepth(). + Self::SetupCleanup { .. } => (0, 0), + Self::SetupFinally { .. } => (0, 0), + Self::SetupWith { .. } => (0, 0), + Self::StoreFastMaybeNull(_) => (0, 1), + }; + + debug_assert!((0..=i32::MAX).contains(&pushed)); + debug_assert!((0..=i32::MAX).contains(&popped)); + + StackEffect::new(pushed as u32, popped as u32) + } + + fn is_unconditional_jump(&self) -> bool { + matches!(self, Self::Jump { .. } | Self::JumpNoInterrupt { .. }) + } + + fn fmt_dis( + &self, + _arg: OpArg, + _f: &mut fmt::Formatter<'_>, + _ctx: &impl InstrDisplayContext, + _expand_code_objects: bool, + _pad: usize, + _level: usize, + ) -> fmt::Result { + unimplemented!() + } +} + +#[derive(Clone, Copy, Debug)] +pub enum AnyInstruction { + Real(Instruction), + Pseudo(PseudoInstruction), +} + +impl From<Instruction> for AnyInstruction { + fn from(value: Instruction) -> Self { + Self::Real(value) + } +} + +impl From<PseudoInstruction> for AnyInstruction { + fn from(value: PseudoInstruction) -> Self { + Self::Pseudo(value) + } +} + +impl TryFrom<u8> for AnyInstruction { + type Error = MarshalError; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + Ok(Instruction::try_from(value)?.into()) + } +} + +impl TryFrom<u16> for AnyInstruction { + type Error = MarshalError; + + fn try_from(value: u16) -> Result<Self, Self::Error> { + match u8::try_from(value) { + Ok(v) => v.try_into(), + Err(_) => Ok(PseudoInstruction::try_from(value)?.into()), + } + } +} + +macro_rules! inst_either { + (fn $name:ident ( &self $(, $arg:ident : $arg_ty:ty )* ) -> $ret:ty ) => { + fn $name(&self $(, $arg : $arg_ty )* ) -> $ret { + match self { + Self::Real(op) => op.$name($($arg),*), + Self::Pseudo(op) => op.$name($($arg),*), + } + } + }; +} + +impl InstructionMetadata for AnyInstruction { + inst_either!(fn label_arg(&self) -> Option<Arg<Label>>); + + inst_either!(fn is_unconditional_jump(&self) -> bool); + + inst_either!(fn is_scope_exit(&self) -> bool); + + inst_either!(fn stack_effect(&self, oparg: u32) -> i32); + + inst_either!(fn stack_effect_info(&self, oparg: u32) -> StackEffect); + + inst_either!(fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize + ) -> fmt::Result); +} + +impl AnyInstruction { + /// Gets the inner value of [`Self::Real`]. + pub const fn real(self) -> Option<Instruction> { + match self { + Self::Real(ins) => Some(ins), + _ => None, + } + } + + /// Gets the inner value of [`Self::Pseudo`]. + pub const fn pseudo(self) -> Option<PseudoInstruction> { + match self { + Self::Pseudo(ins) => Some(ins), + _ => None, + } + } + + /// Same as [`Self::real`] but panics if wasn't called on [`Self::Real`]. + /// + /// # Panics + /// + /// If was called on something else other than [`Self::Real`]. + pub const fn expect_real(self) -> Instruction { + self.real() + .expect("Expected Instruction::Real, found Instruction::Pseudo") + } + + /// Same as [`Self::pseudo`] but panics if wasn't called on [`Self::Pseudo`]. + /// + /// # Panics + /// + /// If was called on something else other than [`Self::Pseudo`]. + pub const fn expect_pseudo(self) -> PseudoInstruction { + self.pseudo() + .expect("Expected Instruction::Pseudo, found Instruction::Real") + } + + /// Returns true if this is a block push pseudo instruction + /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). + pub fn is_block_push(&self) -> bool { + matches!(self, Self::Pseudo(p) if p.is_block_push()) + } + + /// Returns true if this is a POP_BLOCK pseudo instruction. + pub fn is_pop_block(&self) -> bool { + matches!(self, Self::Pseudo(PseudoInstruction::PopBlock)) + } +} + +/// What effect the instruction has on the stack. +#[derive(Clone, Copy)] +pub struct StackEffect { + /// How many items the instruction is pushing on the stack. + pushed: u32, + /// How many items the instruction is popping from the stack. + popped: u32, +} + +impl StackEffect { + /// Creates a new [`Self`]. + pub const fn new(pushed: u32, popped: u32) -> Self { + Self { pushed, popped } + } + + /// Get the calculated stack effect as [`i32`]. + pub fn effect(self) -> i32 { + self.into() + } + + /// Get the pushed count. + pub const fn pushed(self) -> u32 { + self.pushed + } + + /// Get the popped count. + pub const fn popped(self) -> u32 { + self.popped + } +} + +impl From<StackEffect> for i32 { + fn from(effect: StackEffect) -> Self { + (effect.pushed() as i32) - (effect.popped() as i32) + } +} + +pub trait InstructionMetadata { + /// Gets the label stored inside this instruction, if it exists. + fn label_arg(&self) -> Option<Arg<Label>>; + + fn is_scope_exit(&self) -> bool; + + fn is_unconditional_jump(&self) -> bool; + + /// Stack effect info for how many items are pushed/popped from the stack, + /// for this instruction. + fn stack_effect_info(&self, oparg: u32) -> StackEffect; + + /// Stack effect of [`Self::stack_effect_info`]. + fn stack_effect(&self, oparg: u32) -> i32 { + self.stack_effect_info(oparg).effect() + } + + #[allow(clippy::too_many_arguments)] + fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize, + ) -> fmt::Result; + + fn display(&self, arg: OpArg, ctx: &impl InstrDisplayContext) -> impl fmt::Display { + fmt::from_fn(move |f| self.fmt_dis(arg, f, ctx, false, 0, 0)) + } +} + +#[derive(Copy, Clone)] +pub struct Arg<T: OpArgType>(PhantomData<T>); + +impl<T: OpArgType> Arg<T> { + #[inline] + pub const fn marker() -> Self { + Self(PhantomData) + } + + #[inline] + pub fn new(arg: T) -> (Self, OpArg) { + (Self(PhantomData), OpArg::new(arg.into())) + } + + #[inline] + pub fn new_single(arg: T) -> (Self, OpArgByte) + where + T: Into<u8>, + { + (Self(PhantomData), OpArgByte::new(arg.into())) + } + + #[inline(always)] + pub fn get(self, arg: OpArg) -> T { + self.try_get(arg).unwrap() + } + + #[inline(always)] + pub fn try_get(self, arg: OpArg) -> Result<T, MarshalError> { + T::try_from(u32::from(arg)).map_err(|_| MarshalError::InvalidBytecode) + } + + /// # Safety + /// T::from_op_arg(self) must succeed + #[inline(always)] + pub unsafe fn get_unchecked(self, arg: OpArg) -> T { + // SAFETY: requirements forwarded from caller + unsafe { T::try_from(u32::from(arg)).unwrap_unchecked() } + } +} + +impl<T: OpArgType> PartialEq for Arg<T> { + fn eq(&self, _: &Self) -> bool { + true + } +} + +impl<T: OpArgType> Eq for Arg<T> {} + +impl<T: OpArgType> fmt::Debug for Arg<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Arg<{}>", core::any::type_name::<T>()) + } +} diff --git a/crates/compiler-core/src/bytecode/oparg.rs b/crates/compiler-core/src/bytecode/oparg.rs new file mode 100644 index 00000000000..849b3897f34 --- /dev/null +++ b/crates/compiler-core/src/bytecode/oparg.rs @@ -0,0 +1,833 @@ +use bitflags::bitflags; + +use core::fmt; + +use crate::{ + bytecode::{CodeUnit, instruction::Instruction}, + marshal::MarshalError, +}; + +pub trait OpArgType: Copy + Into<u32> + TryFrom<u32> {} + +/// Opcode argument that may be extended by a prior ExtendedArg. +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct OpArgByte(u8); + +impl OpArgByte { + pub const NULL: Self = Self::new(0); + + #[must_use] + pub const fn new(value: u8) -> Self { + Self(value) + } +} + +impl From<u8> for OpArgByte { + fn from(raw: u8) -> Self { + Self::new(raw) + } +} + +impl From<OpArgByte> for u8 { + fn from(value: OpArgByte) -> Self { + value.0 + } +} + +impl fmt::Debug for OpArgByte { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Full 32-bit op_arg, including any possible ExtendedArg extension. +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] +pub struct OpArg(u32); + +impl OpArg { + pub const NULL: Self = Self::new(0); + + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Returns how many CodeUnits a instruction with this op_arg will be encoded as + #[inline] + pub const fn instr_size(self) -> usize { + (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1 + } + + /// returns the arg split into any necessary ExtendedArg components (in big-endian order) and + /// the arg for the real opcode itself + #[inline(always)] + pub fn split(self) -> (impl ExactSizeIterator<Item = OpArgByte>, OpArgByte) { + let mut it = self + .0 + .to_le_bytes() + .map(OpArgByte) + .into_iter() + .take(self.instr_size()); + let lo = it.next().unwrap(); + (it.rev(), lo) + } +} + +impl From<u32> for OpArg { + fn from(raw: u32) -> Self { + Self::new(raw) + } +} + +impl From<OpArg> for u32 { + fn from(value: OpArg) -> Self { + value.0 + } +} + +#[derive(Default, Copy, Clone)] +#[repr(transparent)] +pub struct OpArgState { + state: u32, +} + +impl OpArgState { + #[inline(always)] + pub fn get(&mut self, ins: CodeUnit) -> (Instruction, OpArg) { + let arg = self.extend(ins.arg); + if !matches!(ins.op, Instruction::ExtendedArg) { + self.reset(); + } + (ins.op, arg) + } + + #[inline(always)] + pub fn extend(&mut self, arg: OpArgByte) -> OpArg { + self.state = (self.state << 8) | u32::from(arg.0); + self.state.into() + } + + #[inline(always)] + pub const fn reset(&mut self) { + self.state = 0 + } +} + +/// Helper macro for defining oparg enums in an optimal way. +/// +/// Will generate the following: +/// +/// - Enum which variant's aren't assigned any value (for optimizations). +/// - impl [`TryFrom<u8>`] +/// - impl [`TryFrom<u32>`] +/// - impl [`Into<u8>`] +/// - impl [`Into<u32>`] +/// - impl [`OpArgType`] +/// +/// # Note +/// If an enum variant has "alternative" values (i.e. `Foo = 0 | 1`), the first value will be the +/// result of converting to a number. +/// +/// # Examples +/// +/// ```ignore +/// oparg_enum!( +/// /// Oparg for the `X` opcode. +/// #[derive(Clone, Copy)] +/// pub enum MyOpArg { +/// /// Some doc. +/// Foo = 4, +/// Bar = 8, +/// Baz = 15 | 16, +/// Qux = 23 | 42 +/// } +/// ); +/// ``` +macro_rules! oparg_enum { + ( + $(#[$enum_meta:meta])* + $vis:vis enum $name:ident { + $( + $(#[$variant_meta:meta])* + $variant:ident = $value:literal $(| $alternatives:expr)* + ),* $(,)? + } + ) => { + $(#[$enum_meta])* + $vis enum $name { + $( + $(#[$variant_meta])* + $variant, // Do assign value to variant. + )* + } + + impl_oparg_enum!( + enum $name { + $( + $variant = $value $(| $alternatives)*, + )* + } + ); + }; +} + +macro_rules! impl_oparg_enum { + ( + enum $name:ident { + $( + $variant:ident = $value:literal $(| $alternatives:expr)* + ),* $(,)? + } + ) => { + impl TryFrom<u8> for $name { + type Error = $crate::marshal::MarshalError; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + Ok(match value { + $( + $value $(| $alternatives)* => Self::$variant, + )* + _ => return Err(Self::Error::InvalidBytecode), + }) + } + } + + impl TryFrom<u32> for $name { + type Error = $crate::marshal::MarshalError; + + fn try_from(value: u32) -> Result<Self, Self::Error> { + u8::try_from(value) + .map_err(|_| Self::Error::InvalidBytecode) + .map(TryInto::try_into)? + } + } + + impl From<$name> for u8 { + fn from(value: $name) -> Self { + match value { + $( + $name::$variant => $value, + )* + } + } + } + + impl From<$name> for u32 { + fn from(value: $name) -> Self { + Self::from(u8::from(value)) + } + } + + impl OpArgType for $name {} + }; +} + +oparg_enum!( + /// Oparg values for [`Instruction::ConvertValue`]. + /// + /// ## See also + /// + /// - [CPython FVC_* flags](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Include/ceval.h#L129-L132) + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] + pub enum ConvertValueOparg { + /// No conversion. + /// + /// ```python + /// f"{x}" + /// f"{x:4}" + /// ``` + // Ruff `ConversionFlag::None` is `-1i8`, when its converted to `u8` its value is `u8::MAX`. + None = 0 | 255, + /// Converts by calling `str(<value>)`. + /// + /// ```python + /// f"{x!s}" + /// f"{x!s:2}" + /// ``` + Str = 1, + /// Converts by calling `repr(<value>)`. + /// + /// ```python + /// f"{x!r}" + /// f"{x!r:2}" + /// ``` + Repr = 2, + /// Converts by calling `ascii(<value>)`. + /// + /// ```python + /// f"{x!a}" + /// f"{x!a:2}" + /// ``` + Ascii = 3, + } +); + +impl fmt::Display for ConvertValueOparg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let out = match self { + Self::Str => "1 (str)", + Self::Repr => "2 (repr)", + Self::Ascii => "3 (ascii)", + // We should never reach this. `FVC_NONE` are being handled by `Instruction::FormatSimple` + Self::None => "", + }; + + write!(f, "{out}") + } +} + +oparg_enum!( + /// Resume type for the RESUME instruction + #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] + pub enum ResumeType { + AtFuncStart = 0, + AfterYield = 1, + AfterYieldFrom = 2, + AfterAwait = 3, + } +); + +pub type NameIdx = u32; + +impl OpArgType for u32 {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +#[repr(transparent)] +pub struct Label(pub u32); + +impl Label { + pub const fn new(value: u32) -> Self { + Self(value) + } +} + +impl From<u32> for Label { + fn from(value: u32) -> Self { + Self::new(value) + } +} + +impl From<Label> for u32 { + fn from(value: Label) -> Self { + value.0 + } +} + +impl OpArgType for Label {} + +impl fmt::Display for Label { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +oparg_enum!( + /// The kind of Raise that occurred. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum RaiseKind { + /// Bare `raise` statement with no arguments. + /// Gets the current exception from VM state (topmost_exception). + /// Maps to RAISE_VARARGS with oparg=0. + BareRaise = 0, + /// `raise exc` - exception is on the stack. + /// Maps to RAISE_VARARGS with oparg=1. + Raise = 1, + /// `raise exc from cause` - exception and cause are on the stack. + /// Maps to RAISE_VARARGS with oparg=2. + RaiseCause = 2, + /// Reraise exception from the stack top. + /// Used in exception handler cleanup blocks (finally, except). + /// Gets exception from stack, not from VM state. + /// Maps to the RERAISE opcode. + ReraiseFromStack = 3, + } +); + +oparg_enum!( + /// Intrinsic function for CALL_INTRINSIC_1 + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum IntrinsicFunction1 { + // Invalid = 0, + Print = 1, + /// Import * operation + ImportStar = 2, + /// Convert StopIteration to RuntimeError in async context + StopIterationError = 3, + AsyncGenWrap = 4, + UnaryPositive = 5, + /// Convert list to tuple + ListToTuple = 6, + /// Type parameter related + TypeVar = 7, + ParamSpec = 8, + TypeVarTuple = 9, + /// Generic subscript for PEP 695 + SubscriptGeneric = 10, + TypeAlias = 11, + } +); + +oparg_enum!( + /// Intrinsic function for CALL_INTRINSIC_2 + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum IntrinsicFunction2 { + PrepReraiseStar = 1, + TypeVarWithBound = 2, + TypeVarWithConstraint = 3, + SetFunctionTypeParams = 4, + /// Set default value for type parameter (PEP 695) + SetTypeparamDefault = 5, + } +); + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct MakeFunctionFlags: u8 { + const CLOSURE = 0x01; + const ANNOTATIONS = 0x02; + const KW_ONLY_DEFAULTS = 0x04; + const DEFAULTS = 0x08; + const TYPE_PARAMS = 0x10; + /// PEP 649: __annotate__ function closure (instead of __annotations__ dict) + const ANNOTATE = 0x20; + } +} + +impl TryFrom<u32> for MakeFunctionFlags { + type Error = MarshalError; + + fn try_from(value: u32) -> Result<Self, Self::Error> { + Self::from_bits(value as u8).ok_or(Self::Error::InvalidBytecode) + } +} + +impl From<MakeFunctionFlags> for u32 { + fn from(value: MakeFunctionFlags) -> Self { + value.bits().into() + } +} + +impl OpArgType for MakeFunctionFlags {} + +oparg_enum!( + /// The possible comparison operators. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum ComparisonOperator { + // be intentional with bits so that we can do eval_ord with just a bitwise and + // bits: | Equal | Greater | Less | + Less = 0b001, + Greater = 0b010, + NotEqual = 0b011, + Equal = 0b100, + LessOrEqual = 0b101, + GreaterOrEqual = 0b110, + } +); + +oparg_enum!( + /// The possible Binary operators + /// + /// # Examples + /// + /// ```rust + /// use rustpython_compiler_core::bytecode::{Arg, BinaryOperator, Instruction}; + /// let (op, _) = Arg::new(BinaryOperator::Add); + /// let instruction = Instruction::BinaryOp { op }; + /// ``` + /// + /// See also: + /// - [_PyEval_BinaryOps](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Python/ceval.c#L316-L343) + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum BinaryOperator { + /// `+` + Add = 0, + /// `&` + And = 1, + /// `//` + FloorDivide = 2, + /// `<<` + Lshift = 3, + /// `@` + MatrixMultiply = 4, + /// `*` + Multiply = 5, + /// `%` + Remainder = 6, + /// `|` + Or = 7, + /// `**` + Power = 8, + /// `>>` + Rshift = 9, + /// `-` + Subtract = 10, + /// `/` + TrueDivide = 11, + /// `^` + Xor = 12, + /// `+=` + InplaceAdd = 13, + /// `&=` + InplaceAnd = 14, + /// `//=` + InplaceFloorDivide = 15, + /// `<<=` + InplaceLshift = 16, + /// `@=` + InplaceMatrixMultiply = 17, + /// `*=` + InplaceMultiply = 18, + /// `%=` + InplaceRemainder = 19, + /// `|=` + InplaceOr = 20, + /// `**=` + InplacePower = 21, + /// `>>=` + InplaceRshift = 22, + /// `-=` + InplaceSubtract = 23, + /// `/=` + InplaceTrueDivide = 24, + /// `^=` + InplaceXor = 25, + /// `[]` subscript + Subscr = 26, + } +); + +impl BinaryOperator { + /// Get the "inplace" version of the operator. + /// This has no effect if `self` is already an "inplace" operator. + /// + /// # Example + /// ```rust + /// use rustpython_compiler_core::bytecode::BinaryOperator; + /// + /// assert_eq!(BinaryOperator::Power.as_inplace(), BinaryOperator::InplacePower); + /// + /// assert_eq!(BinaryOperator::InplaceSubtract.as_inplace(), BinaryOperator::InplaceSubtract); + /// ``` + #[must_use] + pub const fn as_inplace(self) -> Self { + match self { + Self::Add => Self::InplaceAdd, + Self::And => Self::InplaceAnd, + Self::FloorDivide => Self::InplaceFloorDivide, + Self::Lshift => Self::InplaceLshift, + Self::MatrixMultiply => Self::InplaceMatrixMultiply, + Self::Multiply => Self::InplaceMultiply, + Self::Remainder => Self::InplaceRemainder, + Self::Or => Self::InplaceOr, + Self::Power => Self::InplacePower, + Self::Rshift => Self::InplaceRshift, + Self::Subtract => Self::InplaceSubtract, + Self::TrueDivide => Self::InplaceTrueDivide, + Self::Xor => Self::InplaceXor, + _ => self, + } + } +} + +impl fmt::Display for BinaryOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let op = match self { + Self::Add => "+", + Self::And => "&", + Self::FloorDivide => "//", + Self::Lshift => "<<", + Self::MatrixMultiply => "@", + Self::Multiply => "*", + Self::Remainder => "%", + Self::Or => "|", + Self::Power => "**", + Self::Rshift => ">>", + Self::Subtract => "-", + Self::TrueDivide => "/", + Self::Xor => "^", + Self::InplaceAdd => "+=", + Self::InplaceAnd => "&=", + Self::InplaceFloorDivide => "//=", + Self::InplaceLshift => "<<=", + Self::InplaceMatrixMultiply => "@=", + Self::InplaceMultiply => "*=", + Self::InplaceRemainder => "%=", + Self::InplaceOr => "|=", + Self::InplacePower => "**=", + Self::InplaceRshift => ">>=", + Self::InplaceSubtract => "-=", + Self::InplaceTrueDivide => "/=", + Self::InplaceXor => "^=", + Self::Subscr => "[]", + }; + write!(f, "{op}") + } +} + +oparg_enum!( + /// Whether or not to invert the operation. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum Invert { + /// ```py + /// foo is bar + /// x in lst + /// ``` + No = 0, + /// ```py + /// foo is not bar + /// x not in lst + /// ``` + Yes = 1, + } +); + +oparg_enum!( + /// Special method for LOAD_SPECIAL opcode (context managers). + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum SpecialMethod { + /// `__enter__` for sync context manager + Enter = 0, + /// `__exit__` for sync context manager + Exit = 1, + /// `__aenter__` for async context manager + AEnter = 2, + /// `__aexit__` for async context manager + AExit = 3, + } +); + +impl fmt::Display for SpecialMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let method_name = match self { + Self::Enter => "__enter__", + Self::Exit => "__exit__", + Self::AEnter => "__aenter__", + Self::AExit => "__aexit__", + }; + write!(f, "{method_name}") + } +} + +oparg_enum!( + /// Common constants for LOAD_COMMON_CONSTANT opcode. + /// pycore_opcode_utils.h CONSTANT_* + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum CommonConstant { + /// `AssertionError` exception type + AssertionError = 0, + /// `NotImplementedError` exception type + NotImplementedError = 1, + /// Built-in `tuple` type + BuiltinTuple = 2, + /// Built-in `all` function + BuiltinAll = 3, + /// Built-in `any` function + BuiltinAny = 4, + } +); + +impl fmt::Display for CommonConstant { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::AssertionError => "AssertionError", + Self::NotImplementedError => "NotImplementedError", + Self::BuiltinTuple => "tuple", + Self::BuiltinAll => "all", + Self::BuiltinAny => "any", + }; + write!(f, "{name}") + } +} + +oparg_enum!( + /// Specifies if a slice is built with either 2 or 3 arguments. + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum BuildSliceArgCount { + /// ```py + /// x[5:10] + /// ``` + Two = 2, + /// ```py + /// x[5:10:2] + /// ``` + Three = 3, + } +); + +#[derive(Copy, Clone)] +pub struct UnpackExArgs { + pub before: u8, + pub after: u8, +} + +impl From<u32> for UnpackExArgs { + fn from(value: u32) -> Self { + let [before, after, ..] = value.to_le_bytes(); + Self { before, after } + } +} + +impl From<UnpackExArgs> for u32 { + fn from(value: UnpackExArgs) -> Self { + Self::from_le_bytes([value.before, value.after, 0, 0]) + } +} + +impl OpArgType for UnpackExArgs {} + +impl fmt::Display for UnpackExArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "before: {}, after: {}", self.before, self.after) + } +} + +#[derive(Clone, Copy)] +pub struct LoadSuperAttr(u32); + +impl LoadSuperAttr { + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + #[must_use] + pub fn builder() -> LoadSuperAttrBuilder { + LoadSuperAttrBuilder::default() + } + + #[must_use] + pub const fn name_idx(self) -> u32 { + self.0 >> 2 + } + + #[must_use] + pub const fn is_load_method(self) -> bool { + (self.0 & 1) == 1 + } + + #[must_use] + pub const fn has_class(self) -> bool { + (self.0 & 2) == 2 + } +} + +impl OpArgType for LoadSuperAttr {} + +impl From<u32> for LoadSuperAttr { + fn from(value: u32) -> Self { + Self::new(value) + } +} + +impl From<LoadSuperAttr> for u32 { + fn from(value: LoadSuperAttr) -> Self { + value.0 + } +} + +#[derive(Clone, Copy, Default)] +pub struct LoadSuperAttrBuilder { + name_idx: u32, + is_load_method: bool, + has_class: bool, +} + +impl LoadSuperAttrBuilder { + #[must_use] + pub const fn build(self) -> LoadSuperAttr { + let value = + (self.name_idx << 2) | ((self.has_class as u32) << 1) | (self.is_load_method as u32); + LoadSuperAttr::new(value) + } + + #[must_use] + pub const fn name_idx(mut self, value: u32) -> Self { + self.name_idx = value; + self + } + + #[must_use] + pub const fn is_load_method(mut self, value: bool) -> Self { + self.is_load_method = value; + self + } + + #[must_use] + pub const fn has_class(mut self, value: bool) -> Self { + self.has_class = value; + self + } +} + +impl From<LoadSuperAttrBuilder> for LoadSuperAttr { + fn from(builder: LoadSuperAttrBuilder) -> Self { + builder.build() + } +} + +#[derive(Clone, Copy)] +pub struct LoadAttr(u32); + +impl LoadAttr { + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + #[must_use] + pub fn builder() -> LoadAttrBuilder { + LoadAttrBuilder::default() + } + + #[must_use] + pub const fn name_idx(self) -> u32 { + self.0 >> 1 + } + + #[must_use] + pub const fn is_method(self) -> bool { + (self.0 & 1) == 1 + } +} + +impl OpArgType for LoadAttr {} + +impl From<u32> for LoadAttr { + fn from(value: u32) -> Self { + Self::new(value) + } +} + +impl From<LoadAttr> for u32 { + fn from(value: LoadAttr) -> Self { + value.0 + } +} + +#[derive(Clone, Copy, Default)] +pub struct LoadAttrBuilder { + name_idx: u32, + is_method: bool, +} + +impl LoadAttrBuilder { + #[must_use] + pub const fn build(self) -> LoadAttr { + let value = (self.name_idx << 1) | (self.is_method as u32); + LoadAttr::new(value) + } + + #[must_use] + pub const fn name_idx(mut self, value: u32) -> Self { + self.name_idx = value; + self + } + + #[must_use] + pub const fn is_method(mut self, value: bool) -> Self { + self.is_method = value; + self + } +} diff --git a/crates/compiler-core/src/frozen.rs b/crates/compiler-core/src/frozen.rs index a79569ad7fb..81530610d0e 100644 --- a/crates/compiler-core/src/frozen.rs +++ b/crates/compiler-core/src/frozen.rs @@ -1,5 +1,6 @@ use crate::bytecode::*; use crate::marshal::{self, Read, ReadBorrowed, Write}; +use alloc::vec::Vec; /// A frozen module. Holds a frozen code object and whether it is part of a package #[derive(Copy, Clone)] diff --git a/crates/compiler-core/src/lib.rs b/crates/compiler-core/src/lib.rs index 08cdc0ec21f..245713d1a14 100644 --- a/crates/compiler-core/src/lib.rs +++ b/crates/compiler-core/src/lib.rs @@ -1,10 +1,14 @@ +#![no_std] #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] #![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")] +extern crate alloc; + pub mod bytecode; pub mod frozen; pub mod marshal; mod mode; +pub mod varint; pub use mode::Mode; diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 39e48071678..11df127920a 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -1,12 +1,13 @@ use crate::{OneIndexed, SourceLocation, bytecode::*}; +use alloc::{boxed::Box, vec::Vec}; +use core::convert::Infallible; use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -use std::convert::Infallible; -pub const FORMAT_VERSION: u32 = 4; +pub const FORMAT_VERSION: u32 = 5; -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub enum MarshalError { /// Unexpected End Of File Eof, @@ -20,8 +21,8 @@ pub enum MarshalError { BadType, } -impl std::fmt::Display for MarshalError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for MarshalError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::Eof => f.write_str("unexpected end of data"), Self::InvalidBytecode => f.write_str("invalid bytecode"), @@ -32,16 +33,17 @@ impl std::fmt::Display for MarshalError { } } -impl From<std::str::Utf8Error> for MarshalError { - fn from(_: std::str::Utf8Error) -> Self { +impl From<core::str::Utf8Error> for MarshalError { + fn from(_: core::str::Utf8Error) -> Self { Self::InvalidUtf8 } } -impl std::error::Error for MarshalError {} +impl core::error::Error for MarshalError {} -type Result<T, E = MarshalError> = std::result::Result<T, E>; +type Result<T, E = MarshalError> = core::result::Result<T, E>; +#[derive(Clone, Copy)] #[repr(u8)] enum Type { // Null = b'0', @@ -65,6 +67,7 @@ enum Type { // Unknown = b'?', Set = b'<', FrozenSet = b'>', + Slice = b':', // Added in version 5 Ascii = b'a', // AsciiInterned = b'A', // SmallTuple = b')', @@ -101,6 +104,7 @@ impl TryFrom<u8> for Type { // b'?' => Unknown, b'<' => Set, b'>' => FrozenSet, + b':' => Slice, b'a' => Ascii, // b'A' => AsciiInterned, // b')' => SmallTuple, @@ -119,7 +123,7 @@ pub trait Read { } fn read_str(&mut self, len: u32) -> Result<&str> { - Ok(std::str::from_utf8(self.read_slice(len)?)?) + Ok(core::str::from_utf8(self.read_slice(len)?)?) } fn read_wtf8(&mut self, len: u32) -> Result<&Wtf8> { @@ -147,7 +151,7 @@ pub(crate) trait ReadBorrowed<'a>: Read { fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>; fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> { - Ok(std::str::from_utf8(self.read_slice_borrow(len)?)?) + Ok(core::str::from_utf8(self.read_slice_borrow(len)?)?) } } @@ -155,13 +159,17 @@ impl Read for &[u8] { fn read_slice(&mut self, n: u32) -> Result<&[u8]> { self.read_slice_borrow(n) } + + fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> { + let (chunk, rest) = self.split_first_chunk::<N>().ok_or(MarshalError::Eof)?; + *self = rest; + Ok(chunk) + } } impl<'a> ReadBorrowed<'a> for &'a [u8] { fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> { - let data = self.get(..n as usize).ok_or(MarshalError::Eof)?; - *self = &self[n as usize..]; - Ok(data) + self.split_off(..n as usize).ok_or(MarshalError::Eof) } } @@ -190,14 +198,19 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>( let len = rdr.read_u32()?; let locations = (0..len) .map(|_| { - Ok(SourceLocation { + let start = SourceLocation { line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), - }) + }; + let end = SourceLocation { + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + }; + Ok((start, end)) }) - .collect::<Result<Box<[SourceLocation]>>>()?; + .collect::<Result<Box<[(SourceLocation, SourceLocation)]>>>()?; - let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?); + let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?); let posonlyarg_count = rdr.read_u32()?; let arg_count = rdr.read_u32()?; @@ -461,6 +474,12 @@ pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Res bag.make_bytes(value) } Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?), + Type::Slice => { + // Slice constants are not yet supported in RustPython + // This would require adding a Slice variant to ConstantData enum + // For now, return an error if we encounter a slice in marshal data + return Err(MarshalError::BadType); + } }; Ok(value) } @@ -648,12 +667,14 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) buf.write_slice(instructions_bytes); write_len(buf, code.locations.len()); - for loc in &*code.locations { - buf.write_u32(loc.line.get() as _); - buf.write_u32(loc.character_offset.to_zero_indexed() as _); + for (start, end) in &*code.locations { + buf.write_u32(start.line.get() as _); + buf.write_u32(start.character_offset.to_zero_indexed() as _); + buf.write_u32(end.line.get() as _); + buf.write_u32(end.character_offset.to_zero_indexed() as _); } - buf.write_u16(code.flags.bits()); + buf.write_u32(code.flags.bits()); buf.write_u32(code.posonlyarg_count); buf.write_u32(code.arg_count); diff --git a/crates/compiler-core/src/mode.rs b/crates/compiler-core/src/mode.rs index 35e9e77f590..181ea4fdfe7 100644 --- a/crates/compiler-core/src/mode.rs +++ b/crates/compiler-core/src/mode.rs @@ -7,7 +7,7 @@ pub enum Mode { BlockExpr, } -impl std::str::FromStr for Mode { +impl core::str::FromStr for Mode { type Err = ModeParseError; // To support `builtins.compile()` `mode` argument @@ -22,11 +22,11 @@ impl std::str::FromStr for Mode { } /// Returned when a given mode is not valid. -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub struct ModeParseError; -impl std::fmt::Display for ModeParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for ModeParseError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, r#"mode must be "exec", "eval", or "single""#) } } diff --git a/crates/compiler-core/src/varint.rs b/crates/compiler-core/src/varint.rs new file mode 100644 index 00000000000..f1ea6b17ec0 --- /dev/null +++ b/crates/compiler-core/src/varint.rs @@ -0,0 +1,140 @@ +//! Variable-length integer encoding utilities. +//! +//! Uses 6-bit chunks with a continuation bit (0x40) to encode integers. +//! Used for exception tables and line number tables. + +use alloc::vec::Vec; + +/// Write a variable-length unsigned integer using 6-bit chunks. +/// Returns the number of bytes written. +#[inline] +pub fn write_varint(buf: &mut Vec<u8>, mut val: u32) -> usize { + let start_len = buf.len(); + while val >= 64 { + buf.push(0x40 | (val & 0x3f) as u8); + val >>= 6; + } + buf.push(val as u8); + buf.len() - start_len +} + +/// Write a variable-length signed integer. +/// Returns the number of bytes written. +#[inline] +pub fn write_signed_varint(buf: &mut Vec<u8>, val: i32) -> usize { + let uval = if val < 0 { + // (0 - val as u32) handles INT_MIN correctly + ((0u32.wrapping_sub(val as u32)) << 1) | 1 + } else { + (val as u32) << 1 + }; + write_varint(buf, uval) +} + +/// Write a variable-length unsigned integer with a start marker (0x80 bit). +/// Used for exception table entries where each entry starts with the marker. +pub fn write_varint_with_start(data: &mut Vec<u8>, val: u32) { + let start_pos = data.len(); + write_varint(data, val); + // Set start bit on first byte + if let Some(first) = data.get_mut(start_pos) { + *first |= 0x80; + } +} + +/// Read a variable-length unsigned integer that starts with a start marker (0x80 bit). +/// Returns None if not at a valid start byte or end of data. +pub fn read_varint_with_start(data: &[u8], pos: &mut usize) -> Option<u32> { + if *pos >= data.len() { + return None; + } + let first = data[*pos]; + if first & 0x80 == 0 { + return None; // Not a start byte + } + *pos += 1; + + let mut val = (first & 0x3f) as u32; + let mut shift = 6; + let mut has_continuation = first & 0x40 != 0; + + while has_continuation && *pos < data.len() { + let byte = data[*pos]; + if byte & 0x80 != 0 { + break; // Next entry start + } + *pos += 1; + val |= ((byte & 0x3f) as u32) << shift; + shift += 6; + has_continuation = byte & 0x40 != 0; + } + Some(val) +} + +/// Read a variable-length unsigned integer. +/// Returns None if end of data or malformed. +pub fn read_varint(data: &[u8], pos: &mut usize) -> Option<u32> { + if *pos >= data.len() { + return None; + } + + let mut val = 0u32; + let mut shift = 0; + + loop { + if *pos >= data.len() { + return None; + } + let byte = data[*pos]; + if byte & 0x80 != 0 && shift > 0 { + break; // Next entry start + } + *pos += 1; + val |= ((byte & 0x3f) as u32) << shift; + shift += 6; + if byte & 0x40 == 0 { + break; + } + } + Some(val) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_write_read_varint() { + let mut buf = Vec::new(); + write_varint(&mut buf, 0); + write_varint(&mut buf, 63); + write_varint(&mut buf, 64); + write_varint(&mut buf, 4095); + + // Values: 0, 63, 64, 4095 + assert_eq!(buf.len(), 1 + 1 + 2 + 2); + } + + #[test] + fn test_write_read_signed_varint() { + let mut buf = Vec::new(); + write_signed_varint(&mut buf, 0); + write_signed_varint(&mut buf, 1); + write_signed_varint(&mut buf, -1); + write_signed_varint(&mut buf, i32::MIN); + + assert!(!buf.is_empty()); + } + + #[test] + fn test_varint_with_start() { + let mut buf = Vec::new(); + write_varint_with_start(&mut buf, 42); + write_varint_with_start(&mut buf, 100); + + let mut pos = 0; + assert_eq!(read_varint_with_start(&buf, &mut pos), Some(42)); + assert_eq!(read_varint_with_start(&buf, &mut pos), Some(100)); + assert_eq!(read_varint_with_start(&buf, &mut pos), None); + } +} diff --git a/crates/compiler-source/src/lib.rs b/crates/compiler-source/src/lib.rs index 2d967e218d2..e3fb55ef603 100644 --- a/crates/compiler-source/src/lib.rs +++ b/crates/compiler-source/src/lib.rs @@ -1,4 +1,4 @@ -pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, SourceLocation}; +pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, PositionEncoding, SourceLocation}; use ruff_text_size::TextRange; pub use ruff_text_size::TextSize; @@ -20,7 +20,8 @@ impl<'src> SourceCode<'src> { } pub fn source_location(&self, offset: TextSize) -> SourceLocation { - self.index.source_location(offset, self.text) + self.index + .source_location(offset, self.text, PositionEncoding::Utf8) } pub fn get_range(&'src self, range: TextRange) -> &'src str { diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 84e64f3c27f..815d45f2cd8 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -25,11 +25,14 @@ pub struct ParseError { pub error: parser::ParseErrorType, pub raw_location: ruff_text_size::TextRange, pub location: SourceLocation, + pub end_location: SourceLocation, pub source_path: String, + /// Set when the error is an unclosed bracket (converted from EOF). + pub is_unclosed_bracket: bool, } -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl ::core::fmt::Display for ParseError { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { self.error.fmt(f) } } @@ -44,14 +47,72 @@ pub enum CompileError { impl CompileError { pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { - let location = source_file - .to_source_code() - .source_location(error.location.start(), PositionEncoding::Utf8); + let source_code = source_file.to_source_code(); + let source_text = source_file.source_text(); + + // For EOF errors (unclosed brackets), find the unclosed bracket position + // and adjust both the error location and message + let mut is_unclosed_bracket = false; + let (error_type, location, end_location) = if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::Eof) + ) { + if let Some((bracket_char, bracket_offset)) = find_unclosed_bracket(source_text) { + let bracket_text_size = ruff_text_size::TextSize::new(bracket_offset as u32); + let loc = source_code.source_location(bracket_text_size, PositionEncoding::Utf8); + let end_loc = SourceLocation { + line: loc.line, + character_offset: loc.character_offset.saturating_add(1), + }; + let msg = format!("'{}' was never closed", bracket_char); + is_unclosed_bracket = true; + (parser::ParseErrorType::OtherError(msg), loc, end_loc) + } else { + let loc = + source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let end_loc = + source_code.source_location(error.location.end(), PositionEncoding::Utf8); + (error.error, loc, end_loc) + } + } else if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) + ) { + // For IndentationError, point the offset to the end of the line content + // instead of the beginning + let loc = source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let line_idx = loc.line.to_zero_indexed(); + let line = source_text.split('\n').nth(line_idx).unwrap_or(""); + let line_end_col = line.chars().count() + 1; // 1-indexed, past last char + let end_loc = SourceLocation { + line: loc.line, + character_offset: ruff_source_file::OneIndexed::new(line_end_col) + .unwrap_or(loc.character_offset), + }; + (error.error, end_loc, end_loc) + } else { + let loc = source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let mut end_loc = + source_code.source_location(error.location.end(), PositionEncoding::Utf8); + + // If the error range ends at the start of a new line (column 1), + // adjust it to the end of the previous line + if end_loc.character_offset.get() == 1 && end_loc.line > loc.line { + let prev_line_end = error.location.end() - ruff_text_size::TextSize::from(1); + end_loc = source_code.source_location(prev_line_end, PositionEncoding::Utf8); + end_loc.character_offset = end_loc.character_offset.saturating_add(1); + } + + (error.error, loc, end_loc) + }; + Self::Parse(ParseError { - error: error.error, + error: error_type, raw_location: error.location, location, + end_location, source_path: source_file.name().to_owned(), + is_unclosed_bracket, }) } @@ -70,6 +131,16 @@ impl CompileError { } } + pub fn python_end_location(&self) -> Option<(usize, usize)> { + match self { + CompileError::Codegen(_) => None, + CompileError::Parse(parse_error) => Some(( + parse_error.end_location.line.get(), + parse_error.end_location.character_offset.get(), + )), + } + } + pub fn source_path(&self) -> &str { match self { Self::Codegen(codegen_error) => &codegen_error.source_path, @@ -78,6 +149,106 @@ impl CompileError { } } +/// Find the last unclosed opening bracket in source code. +/// Returns the bracket character and its byte offset, or None if all brackets are balanced. +fn find_unclosed_bracket(source: &str) -> Option<(char, usize)> { + let mut stack: Vec<(char, usize)> = Vec::new(); + let mut in_string = false; + let mut string_quote = '\0'; + let mut triple_quote = false; + let mut escape_next = false; + let mut is_raw_string = false; + + let chars: Vec<(usize, char)> = source.char_indices().collect(); + let mut i = 0; + + while i < chars.len() { + let (byte_offset, ch) = chars[i]; + + if escape_next { + escape_next = false; + i += 1; + continue; + } + + if in_string { + if ch == '\\' && !is_raw_string { + escape_next = true; + } else if triple_quote { + if ch == string_quote + && i + 2 < chars.len() + && chars[i + 1].1 == string_quote + && chars[i + 2].1 == string_quote + { + in_string = false; + i += 3; + continue; + } + } else if ch == string_quote { + in_string = false; + } + i += 1; + continue; + } + + // Check for comments + if ch == '#' { + // Skip to end of line + while i < chars.len() && chars[i].1 != '\n' { + i += 1; + } + continue; + } + + // Check for string start (with optional prefix like r, b, f, u, rb, br, etc.) + if ch == '\'' || ch == '"' { + // Check up to 2 characters before the quote for string prefix + is_raw_string = false; + for look_back in 1..=2.min(i) { + let prev = chars[i - look_back].1; + if matches!(prev, 'r' | 'R') { + is_raw_string = true; + break; + } + if !matches!(prev, 'b' | 'B' | 'f' | 'F' | 'u' | 'U') { + break; + } + } + string_quote = ch; + if i + 2 < chars.len() && chars[i + 1].1 == ch && chars[i + 2].1 == ch { + triple_quote = true; + in_string = true; + i += 3; + continue; + } + triple_quote = false; + in_string = true; + i += 1; + continue; + } + + match ch { + '(' | '[' | '{' => stack.push((ch, byte_offset)), + ')' | ']' | '}' => { + let expected = match ch { + ')' => '(', + ']' => '[', + '}' => '{', + _ => unreachable!(), + }; + if stack.last().is_some_and(|&(open, _)| open == expected) { + stack.pop(); + } + } + _ => {} + } + + i += 1; + } + + stack.last().copied() +} + /// Compile a given source code into a bytecode object. pub fn compile( source: &str, diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index cdcc89b9984..16984139fcd 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -4,7 +4,7 @@ //! // either: //! source = "python_source_code", //! // or -//! file = "file/path/relative/to/$CARGO_MANIFEST_DIR", +//! file = "file/path/relative/to/this/file", //! //! // the mode to compile the code in //! mode = "exec", // or "eval" or "single" @@ -17,10 +17,9 @@ use crate::Diagnostic; use proc_macro2::{Span, TokenStream}; use quote::quote; use rustpython_compiler_core::{Mode, bytecode::CodeObject, frozen}; -use std::sync::LazyLock; use std::{ collections::HashMap, - env, fs, + fs, path::{Path, PathBuf}, }; use syn::{ @@ -29,17 +28,13 @@ use syn::{ spanned::Spanned, }; -static CARGO_MANIFEST_DIR: LazyLock<PathBuf> = LazyLock::new(|| { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) -}); - enum CompilationSourceKind { /// Source is a File (Path) - File(PathBuf), + File { base: PathBuf, rel_path: PathBuf }, /// Direct Raw source code SourceCode(String), /// Source is a directory - Dir(PathBuf), + Dir { base: PathBuf, rel_path: PathBuf }, } struct CompiledModule { @@ -58,11 +53,11 @@ pub trait Compiler { source: &str, mode: Mode, module_name: String, - ) -> Result<CodeObject, Box<dyn std::error::Error>>; + ) -> Result<CodeObject, Box<dyn core::error::Error>>; } impl CompilationSource { - fn compile_string<D: std::fmt::Display, F: FnOnce() -> D>( + fn compile_string<D: core::fmt::Display, F: FnOnce() -> D>( &self, source: &str, mode: Mode, @@ -85,12 +80,9 @@ impl CompilationSource { compiler: &dyn Compiler, ) -> Result<HashMap<String, CompiledModule>, Diagnostic> { match &self.kind { - CompilationSourceKind::Dir(rel_path) => self.compile_dir( - &CARGO_MANIFEST_DIR.join(rel_path), - String::new(), - mode, - compiler, - ), + CompilationSourceKind::Dir { base, rel_path } => { + self.compile_dir(base, &base.join(rel_path), String::new(), mode, compiler) + } _ => Ok(hashmap! { module_name.clone() => CompiledModule { code: self.compile_single(mode, module_name, compiler)?, @@ -107,8 +99,8 @@ impl CompilationSource { compiler: &dyn Compiler, ) -> Result<CodeObject, Diagnostic> { match &self.kind { - CompilationSourceKind::File(rel_path) => { - let path = CARGO_MANIFEST_DIR.join(rel_path); + CompilationSourceKind::File { base, rel_path } => { + let path = base.join(rel_path); let source = fs::read_to_string(&path).map_err(|err| { Diagnostic::spans_error( self.span, @@ -124,7 +116,7 @@ impl CompilationSource { compiler, || "string literal", ), - CompilationSourceKind::Dir(_) => { + CompilationSourceKind::Dir { .. } => { unreachable!("Can't use compile_single with directory source") } } @@ -132,6 +124,7 @@ impl CompilationSource { fn compile_dir( &self, + base: &Path, path: &Path, parent: String, mode: Mode, @@ -160,6 +153,7 @@ impl CompilationSource { })?; if path.is_dir() { code_map.extend(self.compile_dir( + base, &path, if parent.is_empty() { file_name.to_string() @@ -188,10 +182,7 @@ impl CompilationSource { ) })?; self.compile_string(&source, mode, module_name.clone(), compiler, || { - path.strip_prefix(&*CARGO_MANIFEST_DIR) - .ok() - .unwrap_or(&path) - .display() + path.strip_prefix(base).ok().unwrap_or(&path).display() }) }; let code = compile_path(&path).or_else(|e| { @@ -257,6 +248,16 @@ impl PyCompileArgs { .get_ident() .ok_or_else(|| meta.error("unknown arg"))?; let check_str = || meta.value()?.call(parse_str); + let str_path = || { + let s = check_str()?; + let mut base_path = s + .span() + .unwrap() + .local_file() + .ok_or_else(|| err_span!(s, "filepath literal has no span information"))?; + base_path.pop(); + Ok::<_, syn::Error>((base_path, PathBuf::from(s.value()))) + }; if ident == "mode" { let s = check_str()?; match s.value().parse() { @@ -274,9 +275,9 @@ impl PyCompileArgs { }); } else if ident == "file" { assert_source_empty(&source)?; - let path = check_str()?.value().into(); + let (base, rel_path) = str_path()?; source = Some(CompilationSource { - kind: CompilationSourceKind::File(path), + kind: CompilationSourceKind::File { base, rel_path }, span: (ident.span(), meta.input.cursor().span()), }); } else if ident == "dir" { @@ -285,9 +286,9 @@ impl PyCompileArgs { } assert_source_empty(&source)?; - let path = check_str()?.value().into(); + let (base, rel_path) = str_path()?; source = Some(CompilationSource { - kind: CompilationSourceKind::Dir(path), + kind: CompilationSourceKind::Dir { base, rel_path }, span: (ident.span(), meta.input.cursor().span()), }); } else if ident == "crate_name" { diff --git a/crates/derive-impl/src/from_args.rs b/crates/derive-impl/src/from_args.rs index 4633c9b3aac..9f2d0460fb0 100644 --- a/crates/derive-impl/src/from_args.rs +++ b/crates/derive-impl/src/from_args.rs @@ -18,7 +18,7 @@ enum ParameterKind { impl TryFrom<&Ident> for ParameterKind { type Error = (); - fn try_from(ident: &Ident) -> std::result::Result<Self, Self::Error> { + fn try_from(ident: &Ident) -> core::result::Result<Self, Self::Error> { Ok(match ident.to_string().as_str() { "positional" => Self::PositionalOnly, "any" => Self::PositionalOrKeyword, @@ -37,6 +37,7 @@ struct ArgAttribute { name: Option<String>, kind: ParameterKind, default: Option<DefaultValue>, + error_msg: Option<String>, } impl ArgAttribute { @@ -63,6 +64,7 @@ impl ArgAttribute { name: None, kind, default: None, + error_msg: None, }); return Ok(()); }; @@ -94,6 +96,12 @@ impl ArgAttribute { } let val = meta.value()?.parse::<syn::LitStr>()?; self.name = Some(val.value()) + } else if meta.path.is_ident("error_msg") { + if self.error_msg.is_some() { + return Err(meta.error("already have an error_msg")); + } + let val = meta.value()?.parse::<syn::LitStr>()?; + self.error_msg = Some(val.value()) } else { return Err(meta.error("Unrecognized pyarg attribute")); } @@ -105,12 +113,12 @@ impl ArgAttribute { impl TryFrom<&Field> for ArgAttribute { type Error = syn::Error; - fn try_from(field: &Field) -> std::result::Result<Self, Self::Error> { + fn try_from(field: &Field) -> core::result::Result<Self, Self::Error> { let mut pyarg_attrs = field .attrs .iter() .filter_map(Self::from_attribute) - .collect::<std::result::Result<Vec<_>, _>>()?; + .collect::<core::result::Result<Vec<_>, _>>()?; if pyarg_attrs.len() >= 2 { bail_span!(field, "Multiple pyarg attributes on field") @@ -146,8 +154,15 @@ fn generate_field((i, field): (usize, &Field)) -> Result<TokenStream> { .or(name_string) .ok_or_else(|| err_span!(field, "field in tuple struct must have name attribute"))?; - let middle = quote! { - .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x)).transpose()? + let middle = if let Some(error_msg) = &attr.error_msg { + quote! { + .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x) + .map_err(|_| vm.new_type_error(#error_msg))).transpose()? + } + } else { + quote! { + .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x)).transpose()? + } }; let ending = if let Some(default) = attr.default { @@ -234,7 +249,7 @@ pub fn impl_from_args(input: DeriveInput) -> Result<TokenStream> { fn from_args( vm: &::rustpython_vm::VirtualMachine, args: &mut ::rustpython_vm::function::FuncArgs - ) -> ::std::result::Result<Self, ::rustpython_vm::function::ArgumentError> { + ) -> ::core::result::Result<Self, ::rustpython_vm::function::ArgumentError> { Ok(Self { #fields }) } } diff --git a/crates/derive-impl/src/lib.rs b/crates/derive-impl/src/lib.rs index 51bb0af406f..c00299794de 100644 --- a/crates/derive-impl/src/lib.rs +++ b/crates/derive-impl/src/lib.rs @@ -26,6 +26,8 @@ use quote::ToTokens; use syn::{DeriveInput, Item}; use syn_ext::types::PunctuatedNestedMeta; +pub use pymodule::PyModuleArgs; + pub use compile_bytecode::Compiler; fn result_to_tokens(result: Result<TokenStream, impl Into<Diagnostic>>) -> TokenStream { @@ -54,7 +56,7 @@ pub fn pyexception(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { } } -pub fn pymodule(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { +pub fn pymodule(attr: PyModuleArgs, item: Item) -> TokenStream { result_to_tokens(pymodule::impl_pymodule(attr, item)) } diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index 5060dced2b0..f88fa059817 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -4,11 +4,11 @@ use crate::util::{ ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, format_doc, pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature, }; +use core::str::FromStr; use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; use quote::{ToTokens, quote, quote_spanned}; use rustpython_doc::DB; use std::collections::{HashMap, HashSet}; -use std::str::FromStr; use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; use syn_ext::ext::*; use syn_ext::types::*; @@ -25,8 +25,8 @@ enum AttrName { Member, } -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let s = match self { Self::Method => "pymethod", Self::ClassMethod => "pyclassmethod", @@ -44,7 +44,7 @@ impl std::fmt::Display for AttrName { impl FromStr for AttrName { type Err = String; - fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + fn from_str(s: &str) -> core::result::Result<Self, Self::Err> { Ok(match s { "pymethod" => Self::Method, "pyclassmethod" => Self::ClassMethod, @@ -63,6 +63,7 @@ impl FromStr for AttrName { #[derive(Default)] struct ImplContext { + is_trait: bool, attribute_items: ItemNursery, method_items: MethodNursery, getset_items: GetSetNursery, @@ -164,6 +165,7 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul with_impl, with_method_defs, with_slots, + itemsize, } = extract_impl_attrs(attr, &impl_ty)?; let payload_ty = attr_payload.unwrap_or(payload_guess); let method_def = &context.method_items; @@ -188,9 +190,17 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul #(#class_extensions)* } }, - parse_quote! { - fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #slots_impl + { + let itemsize_impl = itemsize.as_ref().map(|size| { + quote! { + slots.itemsize = #size; + } + }); + parse_quote! { + fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { + #itemsize_impl + #slots_impl + } } }, ]; @@ -222,8 +232,8 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul const METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_defs; fn extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #impl_ty::__extend_slots(slots); #with_slots + #impl_ty::__extend_slots(slots); } } } @@ -232,7 +242,10 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul } } Item::Trait(mut trai) => { - let mut context = ImplContext::default(); + let mut context = ImplContext { + is_trait: true, + ..Default::default() + }; let mut has_extend_slots = false; for item in &trai.items { let has = match item { @@ -442,6 +455,7 @@ fn generate_class_def( }); // If repr(transparent) with a base, the type has the same memory layout as base, // so basicsize should be 0 (no additional space beyond the base type) + // Otherwise, basicsize = sizeof(payload). The header size is added in __basicsize__ getter. let basicsize = if is_repr_transparent && base.is_some() { quote!(0) } else { @@ -560,51 +574,80 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result<Tok )?; const ALLOWED_TRAVERSE_OPTS: &[&str] = &["manual"]; - // try to know if it have a `#[pyclass(trace)]` exist on this struct - // TODO(discord9): rethink on auto detect `#[Derive(PyTrace)]` - - // 1. no `traverse` at all: generate a dummy try_traverse - // 2. `traverse = "manual"`: generate a try_traverse, but not #[derive(Traverse)] - // 3. `traverse`: generate a try_traverse, and #[derive(Traverse)] - let (maybe_trace_code, derive_trace) = { - if class_meta.inner()._has_key("traverse")? { - let maybe_trace_code = quote! { - impl ::rustpython_vm::object::MaybeTraverse for #ident { - const IS_TRACE: bool = true; - fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - ::rustpython_vm::object::Traverse::traverse(self, tracer_fn); - } + // Generate MaybeTraverse impl with both traverse and clear support + // + // For traverse: + // 1. no `traverse` at all: HAS_TRAVERSE = false, try_traverse does nothing + // 2. `traverse = "manual"`: HAS_TRAVERSE = true, but no #[derive(Traverse)] + // 3. `traverse`: HAS_TRAVERSE = true, and #[derive(Traverse)] + // + // For clear (tp_clear): + // 1. no `clear`: HAS_CLEAR = HAS_TRAVERSE (default: same as traverse) + // 2. `clear` or `clear = true`: HAS_CLEAR = true, try_clear calls Traverse::clear + // 3. `clear = false`: HAS_CLEAR = false (rare: traverse without clear) + let has_traverse = class_meta.inner()._has_key("traverse")?; + let has_clear = if class_meta.inner()._has_key("clear")? { + // If clear attribute is present, use its value + class_meta.inner()._bool("clear")? + } else { + // If clear attribute is absent, default to same as traverse + has_traverse + }; + + let derive_trace = if has_traverse { + // _optional_str returns Err when key exists without value (e.g., `traverse` vs `traverse = "manual"`) + // We want to derive Traverse in that case, so we handle Err as Ok(None) + let value = class_meta.inner()._optional_str("traverse").ok().flatten(); + if let Some(s) = value { + if !ALLOWED_TRAVERSE_OPTS.contains(&s.as_str()) { + bail_span!( + item, + "traverse attribute only accept {ALLOWED_TRAVERSE_OPTS:?} as value or no value at all", + ); + } + assert_eq!(s, "manual"); + quote! {} + } else { + quote! {#[derive(Traverse)]} + } + } else { + quote! {} + }; + + let maybe_traverse_code = { + let try_traverse_body = if has_traverse { + quote! { + ::rustpython_vm::object::Traverse::traverse(self, tracer_fn); + } + } else { + quote! { + // do nothing + } + }; + + let try_clear_body = if has_clear { + quote! { + ::rustpython_vm::object::Traverse::clear(self, out); + } + } else { + quote! { + // do nothing + } + }; + + quote! { + impl ::rustpython_vm::object::MaybeTraverse for #ident { + const HAS_TRAVERSE: bool = #has_traverse; + const HAS_CLEAR: bool = #has_clear; + + fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { + #try_traverse_body } - }; - // if the key `traverse` exist but not as key-value, _optional_str return Err(...) - // so we need to check if it is Ok(Some(...)) - let value = class_meta.inner()._optional_str("traverse"); - let derive_trace = if let Ok(Some(s)) = value { - if !ALLOWED_TRAVERSE_OPTS.contains(&s.as_str()) { - bail_span!( - item, - "traverse attribute only accept {ALLOWED_TRAVERSE_OPTS:?} as value or no value at all", - ); + + fn try_clear(&mut self, out: &mut ::std::vec::Vec<::rustpython_vm::PyObjectRef>) { + #try_clear_body } - assert_eq!(s, "manual"); - quote! {} - } else { - quote! {#[derive(Traverse)]} - }; - (maybe_trace_code, derive_trace) - } else { - ( - // a dummy impl, which do nothing - // #attrs - quote! { - impl ::rustpython_vm::object::MaybeTraverse for #ident { - fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - // do nothing - } - } - }, - quote! {}, - ) + } } }; @@ -621,13 +664,10 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result<Tok quote! { // static_assertions::const_assert!(std::mem::size_of::<#base_type>() <= std::mem::size_of::<#ident>()); impl ::rustpython_vm::PyPayload for #ident { - #[inline] - fn payload_type_id() -> ::std::any::TypeId { - <#base_type as ::rustpython_vm::PyPayload>::payload_type_id() - } + const PAYLOAD_TYPE_ID: ::core::any::TypeId = <#base_type as ::rustpython_vm::PyPayload>::PAYLOAD_TYPE_ID; #[inline] - fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { + unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { <Self as ::rustpython_vm::class::PyClassDef>::BASICSIZE <= obj.class().slots.basicsize && obj.class().fast_issubclass(<Self as ::rustpython_vm::class::StaticType>::static_type()) } @@ -664,7 +704,7 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result<Tok let ret = quote! { #derive_trace #item - #maybe_trace_code + #maybe_traverse_code #class_def #impl_payload #empty_impl @@ -710,21 +750,16 @@ pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> R }; // Check if with(Constructor) is specified. If Constructor trait is used, don't generate slot_new - let mut has_slot_new = false; - let mut extra_attrs = Vec::new(); + let mut with_items = vec![]; for nested in &attr { if let NestedMeta::Meta(Meta::List(MetaList { path, nested, .. })) = nested { // If we already found the constructor trait, no need to keep looking for it - if !has_slot_new && path.is_ident("with") { - // Check if Constructor is in the list + if path.is_ident("with") { for meta in nested { - if let NestedMeta::Meta(Meta::Path(p)) = meta - && p.is_ident("Constructor") - { - has_slot_new = true; - } + with_items.push(meta.get_ident().expect("with() has non-ident item").clone()); } + continue; } extra_attrs.push(NestedMeta::Meta(Meta::List(MetaList { path: path.clone(), @@ -734,67 +769,48 @@ pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> R } } - let mut has_slot_init = false; + let with_contains = |with_items: &[Ident], s: &str| { + // Check if Constructor is in the list + with_items.iter().any(|ident| ident == s) + }; + let syn::ItemImpl { generics, self_ty, items, .. } = &imp; - for item in items { - // FIXME: better detection or correct wrapper implementation - let Some(ident) = item.get_ident() else { - continue; - }; - let item_name = ident.to_string(); - match item_name.as_str() { - "slot_new" => { - has_slot_new = true; - } - "slot_init" => { - has_slot_init = true; - } - _ => continue, - } - } - // TODO: slot_new, slot_init must be Constructor or Initializer later - - let slot_new = if has_slot_new { + let slot_new = if with_contains(&with_items, "Constructor") { quote!() } else { + with_items.push(Ident::new("Constructor", Span::call_site())); quote! { - #[pyslot] - pub fn slot_new( - cls: ::rustpython_vm::builtins::PyTypeRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult { - <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_new(cls, args, vm) + impl ::rustpython_vm::types::Constructor for #self_ty { + type Args = ::rustpython_vm::function::FuncArgs; + + fn slot_new( + cls: ::rustpython_vm::builtins::PyTypeRef, + args: ::rustpython_vm::function::FuncArgs, + vm: &::rustpython_vm::VirtualMachine, + ) -> ::rustpython_vm::PyResult { + <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_new(cls, args, vm) + } + fn py_new( + _cls: &::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + _args: Self::Args, + _vm: &::rustpython_vm::VirtualMachine + ) -> ::rustpython_vm::PyResult<Self> { + unreachable!("slot_new is defined") + } } } }; - // We need this method, because of how `CPython` copies `__init__` - // from `BaseException` in `SimpleExtendsException` macro. - // See: `(initproc)BaseException_init` - // spell-checker:ignore initproc - let slot_init = if has_slot_init { - quote!() - } else { - // FIXME: this is a generic logic for types not only for exceptions - quote! { - #[pyslot] - #[pymethod(name="__init__")] - pub fn slot_init( - zelf: ::rustpython_vm::PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_init(zelf, args, vm) - } - } - }; + // SimpleExtendsException: inherits BaseException_init from the base class via MRO. + // Only exceptions that explicitly specify `with(Initializer)` will have + // their own __init__ in __dict__. + let slot_init = quote!(); let extra_attrs_tokens = if extra_attrs.is_empty() { quote!() @@ -803,13 +819,13 @@ pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> R }; Ok(quote! { - #[pyclass(flags(BASETYPE, HAS_DICT) #extra_attrs_tokens)] + #[pyclass(flags(BASETYPE, HAS_DICT), with(#(#with_items),*) #extra_attrs_tokens)] impl #generics #self_ty { #(#items)* - - #slot_new - #slot_init } + + #slot_new + #slot_init }) } @@ -892,6 +908,111 @@ where let item_meta = MethodItemMeta::from_attr(ident.clone(), &item_attr)?; let py_name = item_meta.method_name()?; + + // Disallow slot methods - they should be defined via trait implementations + // These are exposed as wrapper_descriptor via add_operators from SLOT_DEFS + if !args.context.is_trait { + const FORBIDDEN_SLOT_METHODS: &[(&str, &str)] = &[ + // Constructor/Initializer traits + ("__new__", "Constructor"), + ("__init__", "Initializer"), + // Representable trait + // ("__repr__", "Representable"), + // ("__str__", "???"), // allow __str__ + // Hashable trait + ("__hash__", "Hashable"), + // Callable trait + ("__call__", "Callable"), + // GetAttr/SetAttr traits + // NOTE: __getattribute__, __setattr__, __delattr__ are intentionally NOT forbidden + // because they need pymethod for subclass override mechanism to work properly. + // GetDescriptor/SetDescriptor traits + // ("__get__", "GetDescriptor"), + // ("__set__", "SetDescriptor"), + // ("__delete__", "SetDescriptor"), + // AsNumber trait + ("__add__", "AsNumber"), + ("__radd__", "AsNumber"), + ("__iadd__", "AsNumber"), + ("__sub__", "AsNumber"), + ("__rsub__", "AsNumber"), + ("__isub__", "AsNumber"), + ("__mul__", "AsNumber"), + ("__rmul__", "AsNumber"), + ("__imul__", "AsNumber"), + ("__truediv__", "AsNumber"), + ("__rtruediv__", "AsNumber"), + ("__itruediv__", "AsNumber"), + ("__floordiv__", "AsNumber"), + ("__rfloordiv__", "AsNumber"), + ("__ifloordiv__", "AsNumber"), + ("__mod__", "AsNumber"), + ("__rmod__", "AsNumber"), + ("__imod__", "AsNumber"), + ("__pow__", "AsNumber"), + ("__rpow__", "AsNumber"), + ("__ipow__", "AsNumber"), + ("__divmod__", "AsNumber"), + ("__rdivmod__", "AsNumber"), + ("__matmul__", "AsNumber"), + ("__rmatmul__", "AsNumber"), + ("__imatmul__", "AsNumber"), + ("__lshift__", "AsNumber"), + ("__rlshift__", "AsNumber"), + ("__ilshift__", "AsNumber"), + ("__rshift__", "AsNumber"), + ("__rrshift__", "AsNumber"), + ("__irshift__", "AsNumber"), + ("__and__", "AsNumber"), + ("__rand__", "AsNumber"), + ("__iand__", "AsNumber"), + ("__or__", "AsNumber"), + ("__ror__", "AsNumber"), + ("__ior__", "AsNumber"), + ("__xor__", "AsNumber"), + ("__rxor__", "AsNumber"), + ("__ixor__", "AsNumber"), + ("__neg__", "AsNumber"), + ("__pos__", "AsNumber"), + ("__abs__", "AsNumber"), + ("__invert__", "AsNumber"), + ("__int__", "AsNumber"), + ("__float__", "AsNumber"), + ("__index__", "AsNumber"), + ("__bool__", "AsNumber"), + // AsSequence trait + // ("__len__", "AsSequence (or AsMapping)"), + // ("__contains__", "AsSequence"), + // AsMapping trait + // ("__getitem__", "AsMapping (or AsSequence)"), + // ("__setitem__", "AsMapping (or AsSequence)"), + // ("__delitem__", "AsMapping (or AsSequence)"), + // IterNext trait + // ("__iter__", "IterNext"), + // ("__next__", "IterNext"), + // Comparable trait + ("__eq__", "Comparable"), + ("__ne__", "Comparable"), + ("__lt__", "Comparable"), + ("__le__", "Comparable"), + ("__gt__", "Comparable"), + ("__ge__", "Comparable"), + ]; + + if let Some((_, trait_name)) = FORBIDDEN_SLOT_METHODS + .iter() + .find(|(method, _)| *method == py_name.as_str()) + { + return Err(syn::Error::new( + ident.span(), + format!( + "#[pymethod] cannot define '{py_name}'. Use `impl {trait_name} for ...` instead. \ + Slot methods are exposed as wrapper_descriptor automatically.", + ), + )); + } + } + let raw = item_meta.raw()?; let sig_doc = text_signature(func.sig(), &py_name); @@ -954,7 +1075,10 @@ where } else if let Ok(f) = args.item.function_or_method() { (&f.sig().ident, f.span()) } else { - return Err(self.new_syn_error(args.item.span(), "can only be on a method")); + return Err(self.new_syn_error( + args.item.span(), + "can only be on a method or const function pointer", + )); }; let item_attr = args.attrs.remove(self.index()); @@ -1083,13 +1207,18 @@ where let item_meta = MemberItemMeta::from_attr(ident.clone(), &item_attr)?; let (py_name, member_item_kind) = item_meta.member_name()?; - let member_kind = match item_meta.member_kind()? { - Some(s) => match s.as_str() { - "bool" => MemberKind::Bool, - _ => unreachable!(), - }, - _ => MemberKind::ObjectEx, - }; + let member_kind = item_meta.member_kind()?; + if let Some(ref s) = member_kind { + match s.as_str() { + "bool" | "object" => {} + other => { + return Err(self.new_syn_error( + args.item.span(), + &format!("unknown member type '{other}'"), + )); + } + } + } // Add #[allow(non_snake_case)] for setter methods if matches!(member_item_kind, MemberItemKind::Set) { @@ -1269,37 +1398,47 @@ impl ToTokens for GetSetNursery { } } +/// Member kind as string, matching `rustpython_vm::builtins::descriptor::MemberKind` variants. +/// None means ObjectEx (default). Valid values: "bool", "object". +type MemberKindStr = Option<String>; + #[derive(Default)] -#[allow(clippy::type_complexity)] struct MemberNursery { - map: HashMap<(String, MemberKind), (Option<Ident>, Option<Ident>)>, + map: HashMap<String, MemberNurseryEntry>, validated: bool, } +struct MemberNurseryEntry { + kind: MemberKindStr, + getter: Option<Ident>, + setter: Option<Ident>, +} + enum MemberItemKind { Get, Set, } -#[derive(Eq, PartialEq, Hash)] -enum MemberKind { - Bool, - ObjectEx, -} - impl MemberNursery { fn add_item( &mut self, name: String, kind: MemberItemKind, - member_kind: MemberKind, + member_kind: MemberKindStr, item_ident: Ident, ) -> Result<()> { assert!(!self.validated, "new item is not allowed after validation"); - let entry = self.map.entry((name.clone(), member_kind)).or_default(); + let entry = self + .map + .entry(name.clone()) + .or_insert_with(|| MemberNurseryEntry { + kind: member_kind, + getter: None, + setter: None, + }); let func = match kind { - MemberItemKind::Get => &mut entry.0, - MemberItemKind::Set => &mut entry.1, + MemberItemKind::Get => &mut entry.getter, + MemberItemKind::Set => &mut entry.setter, }; if func.is_some() { bail_span!(item_ident, "Multiple member accessors with name '{}'", name); @@ -1310,10 +1449,10 @@ impl MemberNursery { fn validate(&mut self) -> Result<()> { let mut errors = Vec::new(); - for ((name, _), (getter, setter)) in &self.map { - if getter.is_none() { + for (name, entry) in &self.map { + if entry.getter.is_none() { errors.push(err_span!( - setter.as_ref().unwrap(), + entry.setter.as_ref().unwrap(), "Member '{}' is missing a getter", name )); @@ -1328,30 +1467,31 @@ impl MemberNursery { impl ToTokens for MemberNursery { fn to_tokens(&self, tokens: &mut TokenStream) { assert!(self.validated, "Call `validate()` before token generation"); - let properties = self - .map - .iter() - .map(|((name, member_kind), (getter, setter))| { - let setter = match setter { - Some(setter) => quote_spanned! { setter.span() => Some(Self::#setter)}, - None => quote! { None }, - }; - let member_kind = match member_kind { - MemberKind::Bool => { - quote!(::rustpython_vm::builtins::descriptor::MemberKind::Bool) - } - MemberKind::ObjectEx => { - quote!(::rustpython_vm::builtins::descriptor::MemberKind::ObjectEx) - } - }; - quote_spanned! { getter.span() => - class.set_str_attr( - #name, - ctx.new_member(#name, #member_kind, Self::#getter, #setter, class), - ctx, - ); + let properties = self.map.iter().map(|(name, entry)| { + let setter = match &entry.setter { + Some(setter) => quote_spanned! { setter.span() => Some(Self::#setter)}, + None => quote! { None }, + }; + let member_kind = match entry.kind.as_deref() { + Some("bool") => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::Bool) } - }); + Some("object") => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::Object) + } + _ => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::ObjectEx) + } + }; + let getter = entry.getter.as_ref().unwrap(); + quote_spanned! { getter.span() => + class.set_str_attr( + #name, + ctx.new_member(#name, #member_kind, Self::#getter, #setter, class), + ctx, + ); + } + }); tokens.extend(properties); } } @@ -1452,7 +1592,7 @@ impl ItemMeta for SlotItemMeta { fn from_nested<I>(item_ident: Ident, meta_ident: Ident, mut nested: I) -> Result<Self> where - I: std::iter::Iterator<Item = NestedMeta>, + I: core::iter::Iterator<Item = NestedMeta>, { let meta_map = if let Some(nested_meta) = nested.next() { match nested_meta { @@ -1496,7 +1636,9 @@ impl SlotItemMeta { } } else { let ident_str = self.inner().item_name(); - let name = if let Some(stripped) = ident_str.strip_prefix("slot_") { + // Convert to lowercase to handle both SLOT_NEW and slot_new + let ident_lower = ident_str.to_lowercase(); + let name = if let Some(stripped) = ident_lower.strip_prefix("slot_") { proc_macro2::Ident::new(stripped, inner.item_ident.span()) } else { inner.item_ident.clone() @@ -1590,6 +1732,7 @@ struct ExtractedImplAttrs { with_impl: TokenStream, with_method_defs: Vec<TokenStream>, with_slots: TokenStream, + itemsize: Option<syn::Expr>, } fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<ExtractedImplAttrs> { @@ -1608,8 +1751,8 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac } }]; let mut payload = None; + let mut itemsize = None; - let mut has_constructor = false; for attr in attr { match attr { NestedMeta::Meta(Meta::List(MetaList { path, nested, .. })) => { @@ -1634,9 +1777,6 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac "Try `#[pyclass(with(Constructor, ...))]` instead of `#[pyclass(with(DefaultConstructor, ...))]`. DefaultConstructor implicitly implements Constructor." ) } - if path.is_ident("Constructor") || path.is_ident("Unconstructible") { - has_constructor = true; - } ( quote!(<Self as #path>::__extend_py_class), quote!(<Self as #path>::__OWN_METHOD_DEFS), @@ -1648,9 +1788,24 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac #extend_class(ctx, class); }); with_method_defs.push(method_defs); - with_slots.push(quote_spanned! { item_span => - #extend_slots(slots); - }); + // For Initializer and Constructor traits, directly set the slot + // instead of calling __extend_slots. This ensures that the trait + // impl's override (e.g., slot_init in impl Initializer) is used, + // not the trait's default implementation. + let slot_code = if path.is_ident("Initializer") { + quote_spanned! { item_span => + slots.init.store(Some(<Self as ::rustpython_vm::types::Initializer>::slot_init as _)); + } + } else if path.is_ident("Constructor") { + quote_spanned! { item_span => + slots.new.store(Some(<Self as ::rustpython_vm::types::Constructor>::slot_new as _)); + } + } else { + quote_spanned! { item_span => + #extend_slots(slots); + } + }; + with_slots.push(slot_code); } } else if path.is_ident("flags") { for meta in nested { @@ -1682,6 +1837,8 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac } else { bail_span!(value, "payload must be a string literal") } + } else if path.is_ident("itemsize") { + itemsize = Some(value); } else { bail_span!(path, "Unknown pyimpl attribute") } @@ -1689,11 +1846,6 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac attr => bail_span!(attr, "Unknown pyimpl attribute"), } } - // TODO: DISALLOW_INSTANTIATION check is required - let _ = has_constructor; - // if !withs.is_empty() && !has_constructor { - // bail_span!(item, "#[pyclass(with(...))] does not have a Constructor. Either #[pyclass(with(Constructor, ...))] or #[pyclass(with(Unconstructible, ...))] is mandatory. Consider to add `impl DefaultConstructor for T {{}}` or `impl Unconstructible for T {{}}`.") - // } Ok(ExtractedImplAttrs { payload, @@ -1707,6 +1859,7 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<Extrac with_slots: quote! { #(#with_slots)* }, + itemsize, }) } diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index 2d5ff7cb0c2..ed86d142cef 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -5,13 +5,99 @@ use crate::util::{ ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, format_doc, iter_use_idents, pyclass_ident_and_attrs, text_signature, }; +use core::str::FromStr; use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; -use quote::{ToTokens, quote, quote_spanned}; +use quote::{ToTokens, format_ident, quote, quote_spanned}; use rustpython_doc::DB; -use std::{collections::HashSet, str::FromStr}; +use std::collections::HashSet; use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; use syn_ext::ext::*; -use syn_ext::types::PunctuatedNestedMeta; +use syn_ext::types::NestedMeta; + +/// A `with(...)` item that may be gated by `#[cfg(...)]` attributes. +pub struct WithItem { + pub cfg_attrs: Vec<Attribute>, + pub path: syn::Path, +} + +impl syn::parse::Parse for WithItem { + fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> { + let cfg_attrs = Attribute::parse_outer(input)?; + for attr in &cfg_attrs { + if !attr.path().is_ident("cfg") { + return Err(syn::Error::new_spanned( + attr, + "only #[cfg(...)] is supported in with()", + )); + } + } + let path = input.parse()?; + Ok(WithItem { cfg_attrs, path }) + } +} + +/// Parsed arguments for `#[pymodule(...)]`, supporting `#[cfg]` inside `with(...)`. +pub struct PyModuleArgs { + pub metas: Vec<NestedMeta>, + pub with_items: Vec<WithItem>, +} + +impl syn::parse::Parse for PyModuleArgs { + fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> { + let mut metas = Vec::new(); + let mut with_items = Vec::new(); + + while !input.is_empty() { + // Detect `with(...)` — an ident "with" followed by a paren group + if input.peek(Ident) && input.peek2(syn::token::Paren) { + let fork = input.fork(); + let ident: Ident = fork.parse()?; + if ident == "with" { + // Advance past "with" + let _: Ident = input.parse()?; + let content; + syn::parenthesized!(content in input); + let items = + syn::punctuated::Punctuated::<WithItem, syn::Token![,]>::parse_terminated( + &content, + )?; + with_items.extend(items); + if !input.is_empty() { + input.parse::<syn::Token![,]>()?; + } + continue; + } + } + metas.push(input.parse::<NestedMeta>()?); + if input.is_empty() { + break; + } + input.parse::<syn::Token![,]>()?; + } + + Ok(PyModuleArgs { metas, with_items }) + } +} + +/// Generate `#[cfg(not(...))]` attributes that negate the given `#[cfg(...)]` attributes. +fn negate_cfg_attrs(cfg_attrs: &[Attribute]) -> Vec<Attribute> { + if cfg_attrs.is_empty() { + return vec![]; + } + let predicates: Vec<_> = cfg_attrs + .iter() + .map(|attr| match &attr.meta { + syn::Meta::List(list) => list.tokens.clone(), + _ => unreachable!("only #[cfg(...)] should be here"), + }) + .collect(); + if predicates.len() == 1 { + let predicate = &predicates[0]; + vec![parse_quote!(#[cfg(not(#predicate))])] + } else { + vec![parse_quote!(#[cfg(not(all(#(#predicates),*)))])] + } +} #[derive(Clone, Copy, Eq, PartialEq)] enum AttrName { @@ -22,8 +108,8 @@ enum AttrName { StructSequence, } -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let s = match self { Self::Function => "pyfunction", Self::Attr => "pyattr", @@ -38,7 +124,7 @@ impl std::fmt::Display for AttrName { impl FromStr for AttrName { type Err = String; - fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + fn from_str(s: &str) -> core::result::Result<Self, Self::Err> { Ok(match s { "pyfunction" => Self::Function, "pyattr" => Self::Attr, @@ -57,18 +143,19 @@ struct ModuleContext { name: String, function_items: FunctionNursery, attribute_items: ItemNursery, - has_extend_module: bool, // TODO: check if `fn extend_module` exists + has_module_exec: bool, errors: Vec<syn::Error>, } -pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<TokenStream> { +pub fn impl_pymodule(args: PyModuleArgs, module_item: Item) -> Result<TokenStream> { + let PyModuleArgs { metas, with_items } = args; let (doc, mut module_item) = match module_item { Item::Mod(m) => (m.attrs.doc(), m), other => bail_span!(other, "#[pymodule] can only be on a full module"), }; let fake_ident = Ident::new("pymodule", module_item.span()); let module_meta = - ModuleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; + ModuleItemMeta::from_nested(module_item.ident.clone(), fake_ident, metas.into_iter())?; // generation resources let mut context = ModuleContext { @@ -81,6 +168,12 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To // collect to context for item in items.iter_mut() { + // Check if module_exec function is already defined + if let Item::Fn(func) = item + && func.sig.ident == "module_exec" + { + context.has_module_exec = true; + } if matches!(item, Item::Impl(_) | Item::Trait(_)) { // #[pyclass] implementations continue; @@ -112,7 +205,6 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To quote!(None) }; let is_submodule = module_meta.sub()?; - let withs = module_meta.with()?; if !is_submodule { items.extend([ parse_quote! { @@ -122,7 +214,7 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To pub(crate) const DOC: Option<&'static str> = #doc; }, parse_quote! { - pub(crate) fn __module_def( + pub(crate) fn module_def( ctx: &::rustpython_vm::Context, ) -> &'static ::rustpython_vm::builtins::PyModuleDef { DEF.get_or_init(|| { @@ -132,43 +224,81 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To methods: METHOD_DEFS, slots: Default::default(), }; - def.slots.exec = Some(extend_module); + def.slots.exec = Some(module_exec); def }) } }, - parse_quote! { - #[allow(dead_code)] - pub(crate) fn make_module( - vm: &::rustpython_vm::VirtualMachine - ) -> ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> { - use ::rustpython_vm::PyPayload; - let module = ::rustpython_vm::builtins::PyModule::from_def(__module_def(&vm.ctx)).into_ref(&vm.ctx); - __init_dict(vm, &module); - extend_module(vm, &module).unwrap(); - module - } - }, ]); } - if !is_submodule && !context.has_extend_module { + if !is_submodule && !context.has_module_exec { items.push(parse_quote! { - pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { - __extend_module(vm, module); + pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { + __module_exec(vm, module); Ok(()) } }); } - let method_defs = if withs.is_empty() { + // Split with_items into unconditional and cfg-gated groups + let (uncond_withs, cond_withs): (Vec<_>, Vec<_>) = + with_items.iter().partition(|w| w.cfg_attrs.is_empty()); + let uncond_paths: Vec<_> = uncond_withs.iter().map(|w| &w.path).collect(); + + let method_defs = if with_items.is_empty() { quote!(#function_items) } else { + // For cfg-gated with items, generate conditional const declarations + // so the total array size adapts to the cfg at compile time + let cond_const_names: Vec<_> = cond_withs + .iter() + .enumerate() + .map(|(i, _)| format_ident!("__WITH_METHODS_{}", i)) + .collect(); + let cond_const_decls: Vec<_> = cond_withs + .iter() + .zip(&cond_const_names) + .map(|(w, name)| { + let cfg_attrs = &w.cfg_attrs; + let neg_attrs = negate_cfg_attrs(&w.cfg_attrs); + let path = &w.path; + quote! { + #(#cfg_attrs)* + const #name: &'static [::rustpython_vm::function::PyMethodDef] = super::#path::METHOD_DEFS; + #(#neg_attrs)* + const #name: &'static [::rustpython_vm::function::PyMethodDef] = &[]; + } + }) + .collect(); + quote!({ const OWN_METHODS: &'static [::rustpython_vm::function::PyMethodDef] = &#function_items; + #(#cond_const_decls)* rustpython_vm::function::PyMethodDef::__const_concat_arrays::< - { OWN_METHODS.len() #(+ super::#withs::METHOD_DEFS.len())* }, - >(&[#(super::#withs::METHOD_DEFS,)* OWN_METHODS]) + { OWN_METHODS.len() + #(+ super::#uncond_paths::METHOD_DEFS.len())* + #(+ #cond_const_names.len())* + }, + >(&[ + #(super::#uncond_paths::METHOD_DEFS,)* + #(#cond_const_names,)* + OWN_METHODS + ]) }) }; + + // Generate __init_attributes calls, wrapping cfg-gated items + let init_with_calls: Vec<_> = with_items + .iter() + .map(|w| { + let cfg_attrs = &w.cfg_attrs; + let path = &w.path; + quote! { + #(#cfg_attrs)* + super::#path::__init_attributes(vm, module); + } + }) + .collect(); + items.extend([ parse_quote! { ::rustpython_vm::common::static_cell! { @@ -183,19 +313,16 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, ) { - #( - super::#withs::__init_attributes(vm, module); - )* + #(#init_with_calls)* let ctx = &vm.ctx; #attribute_items } }, parse_quote! { - pub(crate) fn __extend_module( + pub(crate) fn __module_exec( vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, ) { - module.__init_methods(vm).unwrap(); __init_attributes(vm, module); } }, @@ -573,7 +700,18 @@ impl ModuleItem for ClassItem { }; let class_new = quote_spanned!(ident.span() => let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx); - new_class.set_attr(rustpython_vm::identifier!(ctx, __module__), vm.new_pyobj(#module_name)); + // Only set __module__ string if the class doesn't already have a + // getset descriptor for __module__ (which provides instance-level + // module resolution, e.g. TypeAliasType) + { + let module_key = rustpython_vm::identifier!(ctx, __module__); + let has_module_getset = new_class.attributes.read() + .get(module_key) + .is_some_and(|v| v.downcastable::<rustpython_vm::builtins::PyGetSet>()); + if !has_module_getset { + new_class.set_attr(module_key, vm.new_pyobj(#module_name)); + } + } ); (class_name, class_new) }; @@ -651,7 +789,15 @@ impl ModuleItem for StructSequenceItem { // Generate the class creation code let class_new = quote_spanned!(pytype_ident.span() => let new_class = <#pytype_ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx); - new_class.set_attr(rustpython_vm::identifier!(ctx, __module__), vm.new_pyobj(#module_name)); + { + let module_key = rustpython_vm::identifier!(ctx, __module__); + let has_module_getset = new_class.attributes.read() + .get(module_key) + .is_some_and(|v| v.downcastable::<rustpython_vm::builtins::PyGetSet>()); + if !has_module_getset { + new_class.set_attr(module_key, vm.new_pyobj(#module_name)); + } + } ); // Handle py_attrs for custom names, or use class_name as default diff --git a/crates/derive-impl/src/pystructseq.rs b/crates/derive-impl/src/pystructseq.rs index 6c34844696d..c59a1df2e31 100644 --- a/crates/derive-impl/src/pystructseq.rs +++ b/crates/derive-impl/src/pystructseq.rs @@ -22,6 +22,8 @@ enum FieldKind { struct ParsedField { ident: Ident, kind: FieldKind, + /// Optional cfg attributes for conditional compilation + cfg_attrs: Vec<syn::Attribute>, } /// Parsed field info from struct @@ -31,27 +33,24 @@ struct FieldInfo { } impl FieldInfo { - fn named_fields(&self) -> Vec<Ident> { + fn named_fields(&self) -> Vec<&ParsedField> { self.fields .iter() .filter(|f| f.kind == FieldKind::Named) - .map(|f| f.ident.clone()) .collect() } - fn visible_fields(&self) -> Vec<Ident> { + fn visible_fields(&self) -> Vec<&ParsedField> { self.fields .iter() .filter(|f| f.kind != FieldKind::Skipped) - .map(|f| f.ident.clone()) .collect() } - fn skipped_fields(&self) -> Vec<Ident> { + fn skipped_fields(&self) -> Vec<&ParsedField> { self.fields .iter() .filter(|f| f.kind == FieldKind::Skipped) - .map(|f| f.ident.clone()) .collect() } @@ -82,8 +81,15 @@ fn parse_fields(input: &mut DeriveInput) -> Result<FieldInfo> { let mut skip = false; let mut unnamed = false; let mut attrs_to_remove = Vec::new(); + let mut cfg_attrs = Vec::new(); for (i, attr) in field.attrs.iter().enumerate() { + // Collect cfg attributes for conditional compilation + if attr.path().is_ident("cfg") { + cfg_attrs.push(attr.clone()); + continue; + } + if !attr.path().is_ident("pystruct_sequence") { continue; } @@ -135,7 +141,11 @@ fn parse_fields(input: &mut DeriveInput) -> Result<FieldInfo> { FieldKind::Named }; - parsed_fields.push(ParsedField { ident, kind }); + parsed_fields.push(ParsedField { + ident, + kind, + cfg_attrs, + }); } Ok(FieldInfo { @@ -194,18 +204,91 @@ pub(crate) fn impl_pystruct_sequence_data( let skipped_fields = field_info.skipped_fields(); let n_unnamed_fields = field_info.n_unnamed_fields(); - // Generate field index constants for visible fields + // Generate field index constants for visible fields (with cfg guards) let field_indices: Vec<_> = visible_fields .iter() .enumerate() .map(|(i, field)| { - let const_name = format_ident!("{}_INDEX", field.to_string().to_uppercase()); + let const_name = format_ident!("{}_INDEX", field.ident.to_string().to_uppercase()); + let cfg_attrs = &field.cfg_attrs; quote! { + #(#cfg_attrs)* pub const #const_name: usize = #i; } }) .collect(); + // Generate field name entries with cfg guards for named fields + let named_field_names: Vec<_> = named_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { stringify!(#ident), } + } else { + quote! { + #(#cfg_attrs)* + { stringify!(#ident) }, + } + } + }) + .collect(); + + // Generate field name entries with cfg guards for skipped fields + let skipped_field_names: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { stringify!(#ident), } + } else { + quote! { + #(#cfg_attrs)* + { stringify!(#ident) }, + } + } + }) + .collect(); + + // Generate into_tuple items with cfg guards + let visible_tuple_items: Vec<_> = visible_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm), + } + } else { + quote! { + #(#cfg_attrs)* + { ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm) }, + } + } + }) + .collect(); + + let skipped_tuple_items: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm), + } + } else { + quote! { + #(#cfg_attrs)* + { ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm) }, + } + } + }) + .collect(); + // Generate TryFromObject impl only when try_from_object=true let try_from_object_impl = if try_from_object { let n_required = visible_fields.len(); @@ -216,11 +299,12 @@ pub(crate) fn impl_pystruct_sequence_data( obj: ::rustpython_vm::PyObjectRef, ) -> ::rustpython_vm::PyResult<Self> { let seq: Vec<::rustpython_vm::PyObjectRef> = obj.try_into_value(vm)?; - if seq.len() < #n_required { + if seq.len() != #n_required { return Err(vm.new_type_error(format!( - "{} requires at least {} elements", + "{} requires a {}-sequence ({}-sequence given)", stringify!(#data_ident), - #n_required + #n_required, + seq.len() ))); } <Self as ::rustpython_vm::types::PyStructSequenceData>::try_from_elements(seq, vm) @@ -233,6 +317,44 @@ pub(crate) fn impl_pystruct_sequence_data( // Generate try_from_elements trait override only when try_from_object=true let try_from_elements_trait_override = if try_from_object { + let visible_field_inits: Vec<_> = visible_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { #ident: iter.next().unwrap().clone().try_into_value(vm)?, } + } else { + quote! { + #(#cfg_attrs)* + #ident: iter.next().unwrap().clone().try_into_value(vm)?, + } + } + }) + .collect(); + let skipped_field_inits: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } else { + quote! { + #(#cfg_attrs)* + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } + }) + .collect(); quote! { fn try_from_elements( elements: Vec<::rustpython_vm::PyObjectRef>, @@ -240,11 +362,8 @@ pub(crate) fn impl_pystruct_sequence_data( ) -> ::rustpython_vm::PyResult<Self> { let mut iter = elements.into_iter(); Ok(Self { - #(#visible_fields: iter.next().unwrap().clone().try_into_value(vm)?,)* - #(#skipped_fields: match iter.next() { - Some(v) => v.clone().try_into_value(vm)?, - None => vm.ctx.none(), - },)* + #(#visible_field_inits)* + #(#skipped_field_inits)* }) } } @@ -259,20 +378,14 @@ pub(crate) fn impl_pystruct_sequence_data( // PyStructSequenceData trait impl impl ::rustpython_vm::types::PyStructSequenceData for #data_ident { - const REQUIRED_FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#named_fields),)*]; - const OPTIONAL_FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#skipped_fields),)*]; + const REQUIRED_FIELD_NAMES: &'static [&'static str] = &[#(#named_field_names)*]; + const OPTIONAL_FIELD_NAMES: &'static [&'static str] = &[#(#skipped_field_names)*]; const UNNAMED_FIELDS_LEN: usize = #n_unnamed_fields; fn into_tuple(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::builtins::PyTuple { let items = vec![ - #(::rustpython_vm::convert::ToPyObject::to_pyobject( - self.#visible_fields, - vm, - ),)* - #(::rustpython_vm::convert::ToPyObject::to_pyobject( - self.#skipped_fields, - vm, - ),)* + #(#visible_tuple_items)* + #(#skipped_tuple_items)* ]; ::rustpython_vm::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) } @@ -480,13 +593,10 @@ pub(crate) fn impl_pystruct_sequence( // Subtype uses base type's payload_type_id impl ::rustpython_vm::PyPayload for #pytype_ident { - #[inline] - fn payload_type_id() -> ::std::any::TypeId { - <::rustpython_vm::builtins::PyTuple as ::rustpython_vm::PyPayload>::payload_type_id() - } + const PAYLOAD_TYPE_ID: ::core::any::TypeId = <::rustpython_vm::builtins::PyTuple as ::rustpython_vm::PyPayload>::PAYLOAD_TYPE_ID; #[inline] - fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { + unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { obj.class().fast_issubclass(<Self as ::rustpython_vm::class::StaticType>::static_type()) } @@ -497,9 +607,16 @@ pub(crate) fn impl_pystruct_sequence( // MaybeTraverse - delegate to inner PyTuple impl ::rustpython_vm::object::MaybeTraverse for #pytype_ident { + const HAS_TRAVERSE: bool = true; + const HAS_CLEAR: bool = true; + fn try_traverse(&self, traverse_fn: &mut ::rustpython_vm::object::TraverseFn<'_>) { self.0.try_traverse(traverse_fn) } + + fn try_clear(&mut self, out: &mut ::std::vec::Vec<::rustpython_vm::PyObjectRef>) { + self.0.try_clear(out) + } } // PySubclass for proper inheritance diff --git a/crates/derive-impl/src/pytraverse.rs b/crates/derive-impl/src/pytraverse.rs index c5c4bbd2704..c4ec3823298 100644 --- a/crates/derive-impl/src/pytraverse.rs +++ b/crates/derive-impl/src/pytraverse.rs @@ -37,7 +37,7 @@ fn field_to_traverse_code(field: &Field) -> Result<TokenStream> { .attrs .iter() .filter_map(pytraverse_arg) - .collect::<std::result::Result<Vec<_>, _>>()?; + .collect::<core::result::Result<Vec<_>, _>>()?; let do_trace = if pytraverse_attrs.len() > 1 { bail_span!( field, diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 379adc65b57..a4bf7e6a8fe 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -76,7 +76,7 @@ impl ItemNursery { impl ToTokens for ValidatedItemNursery { fn to_tokens(&self, tokens: &mut TokenStream) { let mut sorted = self.0.0.clone(); - sorted.sort_by(|a, b| a.sort_order.cmp(&b.sort_order)); + sorted.sort_by_key(|a| a.sort_order); tokens.extend(sorted.iter().map(|item| { let cfgs = &item.cfgs; let tokens = &item.tokens; @@ -97,7 +97,7 @@ pub(crate) struct ContentItemInner<T> { } pub(crate) trait ContentItem { - type AttrName: std::str::FromStr + std::fmt::Display; + type AttrName: core::str::FromStr + core::fmt::Display; fn inner(&self) -> &ContentItemInner<Self::AttrName>; fn index(&self) -> usize { @@ -125,7 +125,7 @@ impl ItemMetaInner { allowed_names: &[&'static str], ) -> Result<Self> where - I: std::iter::Iterator<Item = NestedMeta>, + I: core::iter::Iterator<Item = NestedMeta>, { let (meta_map, lits) = nested.into_unique_map_and_lits(|path| { if let Some(ident) = path.get_ident() { @@ -243,7 +243,7 @@ impl ItemMetaInner { pub fn _optional_list( &self, key: &str, - ) -> Result<Option<impl std::iter::Iterator<Item = &'_ NestedMeta>>> { + ) -> Result<Option<impl core::iter::Iterator<Item = &'_ NestedMeta>>> { let value = if let Some((_, meta)) = self.meta_map.get(key) { let Meta::List(MetaList { path: _, nested, .. @@ -269,7 +269,7 @@ pub(crate) trait ItemMeta: Sized { fn from_nested<I>(item_ident: Ident, meta_ident: Ident, nested: I) -> Result<Self> where - I: std::iter::Iterator<Item = NestedMeta>, + I: core::iter::Iterator<Item = NestedMeta>, { Ok(Self::from_inner(ItemMetaInner::from_nested( item_ident, @@ -315,7 +315,7 @@ impl ItemMeta for SimpleItemMeta { pub(crate) struct ModuleItemMeta(pub ItemMetaInner); impl ItemMeta for ModuleItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "with", "sub"]; + const ALLOWED_NAMES: &'static [&'static str] = &["name", "sub"]; fn from_inner(inner: ItemMetaInner) -> Self { Self(inner) @@ -330,20 +330,6 @@ impl ModuleItemMeta { pub fn sub(&self) -> Result<bool> { self.inner()._bool("sub") } - - pub fn with(&self) -> Result<Vec<&syn::Path>> { - let mut withs = Vec::new(); - let Some(nested) = self.inner()._optional_list("with")? else { - return Ok(withs); - }; - for meta in nested { - let NestedMeta::Meta(Meta::Path(path)) = meta else { - bail_span!(meta, "#[pymodule(with(...))] arguments should be paths") - }; - withs.push(path); - } - Ok(withs) - } } pub(crate) struct AttrItemMeta(pub ItemMetaInner); @@ -372,6 +358,7 @@ impl ItemMeta for ClassItemMeta { "ctx", "impl", "traverse", + "clear", // tp_clear ]; fn from_inner(inner: ItemMetaInner) -> Self { @@ -529,7 +516,7 @@ impl ExceptionItemMeta { } } -impl std::ops::Deref for ExceptionItemMeta { +impl core::ops::Deref for ExceptionItemMeta { type Target = ClassItemMeta; fn deref(&self) -> &Self::Target { &self.0 diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs index 655ad3b4c9e..224aad4ea3c 100644 --- a/crates/derive/src/lib.rs +++ b/crates/derive/src/lib.rs @@ -143,8 +143,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { } /// This attribute must be applied to an inline module. -/// It defines a Python module in the form a `make_module` function in the module; -/// this has to be used in a `get_module_inits` to properly register the module. +/// It defines a Python module in the form of a `module_def` function in the module; +/// this has to be used in a `add_native_module` to properly register the module. /// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. /// # Arguments /// - `name`: the name of the python module, @@ -209,7 +209,7 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { /// - `name`: the name of the function in Python, by default it is the same as the associated Rust function. #[proc_macro_attribute] pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr with Punctuated::parse_terminated); + let attr = parse_macro_input!(attr as derive_impl::PyModuleArgs); let item = parse_macro_input!(item); derive_impl::pymodule(attr, item).into() } @@ -274,7 +274,7 @@ impl derive_impl::Compiler for Compiler { source: &str, mode: rustpython_compiler::Mode, module_name: String, - ) -> Result<rustpython_compiler::CodeObject, Box<dyn std::error::Error>> { + ) -> Result<rustpython_compiler::CodeObject, Box<dyn core::error::Error>> { use rustpython_compiler::{CompileOpts, compile}; Ok(compile(source, mode, &module_name, CompileOpts::default())?) } diff --git a/crates/doc/generate.py b/crates/doc/generate.py index 73cb462bb9f..189e69705e1 100644 --- a/crates/doc/generate.py +++ b/crates/doc/generate.py @@ -49,7 +49,7 @@ def key(self) -> str: def doc(self) -> str: assert self.raw_doc is not None - return re.sub(UNICODE_ESCAPE, r"\\u{\1}", inspect.cleandoc(self.raw_doc)) + return re.sub(UNICODE_ESCAPE, r"\\u{\1}", self.raw_doc.strip()) def is_c_extension(module: types.ModuleType) -> bool: @@ -90,7 +90,15 @@ def is_child_of(obj: typing.Any, module: types.ModuleType) -> bool: ------- bool """ - return inspect.getmodule(obj) is module + if inspect.getmodule(obj) is module: + return True + # Some C modules (e.g. _ast) set __module__ to a different name (e.g. "ast"), + # causing inspect.getmodule() to return a different module object. + # Fall back to checking the module's namespace directly. + obj_name = getattr(obj, "__name__", None) + if obj_name is not None: + return module.__dict__.get(obj_name) is obj + return False def iter_modules() -> "Iterable[types.ModuleType]": diff --git a/crates/doc/src/data.inc.rs b/crates/doc/src/data.inc.rs index 4411587ca8b..d347962ae59 100644 --- a/crates/doc/src/data.inc.rs +++ b/crates/doc/src/data.inc.rs @@ -1,5 +1,5 @@ // This file was auto-generated by `.github/workflows/update-doc-db.yml`. -// CPython version: 3.13.9 +// CPython version: 3.14.3 // spell-checker: disable pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { @@ -12,8 +12,2905 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_abc._reset_caches" => "Internal ABC helper to reset both caches of a given class.\n\nShould be only used by refleak.py", "_abc._reset_registry" => "Internal ABC helper to reset registry of a given class.\n\nShould be only used by refleak.py", "_abc.get_cache_token" => "Returns the current ABC cache token.\n\nThe token is an opaque object (supporting equality testing) identifying the\ncurrent version of the ABC cache for virtual subclasses. The token changes\nwith every call to register() on any ABC.", + "_ast.AST.__delattr__" => "Implement delattr(self, name).", + "_ast.AST.__eq__" => "Return self==value.", + "_ast.AST.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AST.__ge__" => "Return self>=value.", + "_ast.AST.__getattribute__" => "Return getattr(self, name).", + "_ast.AST.__getstate__" => "Helper for pickle.", + "_ast.AST.__gt__" => "Return self>value.", + "_ast.AST.__hash__" => "Return hash(self).", + "_ast.AST.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AST.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AST.__le__" => "Return self<=value.", + "_ast.AST.__lt__" => "Return self<value.", + "_ast.AST.__ne__" => "Return self!=value.", + "_ast.AST.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AST.__reduce_ex__" => "Helper for pickle.", + "_ast.AST.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AST.__repr__" => "Return repr(self).", + "_ast.AST.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AST.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AST.__str__" => "Return str(self).", + "_ast.AST.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Add" => "Add", + "_ast.Add.__delattr__" => "Implement delattr(self, name).", + "_ast.Add.__eq__" => "Return self==value.", + "_ast.Add.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Add.__ge__" => "Return self>=value.", + "_ast.Add.__getattribute__" => "Return getattr(self, name).", + "_ast.Add.__getstate__" => "Helper for pickle.", + "_ast.Add.__gt__" => "Return self>value.", + "_ast.Add.__hash__" => "Return hash(self).", + "_ast.Add.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Add.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Add.__le__" => "Return self<=value.", + "_ast.Add.__lt__" => "Return self<value.", + "_ast.Add.__ne__" => "Return self!=value.", + "_ast.Add.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Add.__reduce_ex__" => "Helper for pickle.", + "_ast.Add.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Add.__repr__" => "Return repr(self).", + "_ast.Add.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Add.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Add.__str__" => "Return str(self).", + "_ast.Add.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Add.__weakref__" => "list of weak references to the object", + "_ast.And" => "And", + "_ast.And.__delattr__" => "Implement delattr(self, name).", + "_ast.And.__eq__" => "Return self==value.", + "_ast.And.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.And.__ge__" => "Return self>=value.", + "_ast.And.__getattribute__" => "Return getattr(self, name).", + "_ast.And.__getstate__" => "Helper for pickle.", + "_ast.And.__gt__" => "Return self>value.", + "_ast.And.__hash__" => "Return hash(self).", + "_ast.And.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.And.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.And.__le__" => "Return self<=value.", + "_ast.And.__lt__" => "Return self<value.", + "_ast.And.__ne__" => "Return self!=value.", + "_ast.And.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.And.__reduce_ex__" => "Helper for pickle.", + "_ast.And.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.And.__repr__" => "Return repr(self).", + "_ast.And.__setattr__" => "Implement setattr(self, name, value).", + "_ast.And.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.And.__str__" => "Return str(self).", + "_ast.And.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.And.__weakref__" => "list of weak references to the object", + "_ast.AnnAssign" => "AnnAssign(expr target, expr annotation, expr? value, int simple)", + "_ast.AnnAssign.__delattr__" => "Implement delattr(self, name).", + "_ast.AnnAssign.__eq__" => "Return self==value.", + "_ast.AnnAssign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AnnAssign.__ge__" => "Return self>=value.", + "_ast.AnnAssign.__getattribute__" => "Return getattr(self, name).", + "_ast.AnnAssign.__getstate__" => "Helper for pickle.", + "_ast.AnnAssign.__gt__" => "Return self>value.", + "_ast.AnnAssign.__hash__" => "Return hash(self).", + "_ast.AnnAssign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AnnAssign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AnnAssign.__le__" => "Return self<=value.", + "_ast.AnnAssign.__lt__" => "Return self<value.", + "_ast.AnnAssign.__ne__" => "Return self!=value.", + "_ast.AnnAssign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AnnAssign.__reduce_ex__" => "Helper for pickle.", + "_ast.AnnAssign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AnnAssign.__repr__" => "Return repr(self).", + "_ast.AnnAssign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AnnAssign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AnnAssign.__str__" => "Return str(self).", + "_ast.AnnAssign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AnnAssign.__weakref__" => "list of weak references to the object", + "_ast.Assert" => "Assert(expr test, expr? msg)", + "_ast.Assert.__delattr__" => "Implement delattr(self, name).", + "_ast.Assert.__eq__" => "Return self==value.", + "_ast.Assert.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Assert.__ge__" => "Return self>=value.", + "_ast.Assert.__getattribute__" => "Return getattr(self, name).", + "_ast.Assert.__getstate__" => "Helper for pickle.", + "_ast.Assert.__gt__" => "Return self>value.", + "_ast.Assert.__hash__" => "Return hash(self).", + "_ast.Assert.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Assert.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Assert.__le__" => "Return self<=value.", + "_ast.Assert.__lt__" => "Return self<value.", + "_ast.Assert.__ne__" => "Return self!=value.", + "_ast.Assert.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Assert.__reduce_ex__" => "Helper for pickle.", + "_ast.Assert.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Assert.__repr__" => "Return repr(self).", + "_ast.Assert.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Assert.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Assert.__str__" => "Return str(self).", + "_ast.Assert.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Assert.__weakref__" => "list of weak references to the object", + "_ast.Assign" => "Assign(expr* targets, expr value, string? type_comment)", + "_ast.Assign.__delattr__" => "Implement delattr(self, name).", + "_ast.Assign.__eq__" => "Return self==value.", + "_ast.Assign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Assign.__ge__" => "Return self>=value.", + "_ast.Assign.__getattribute__" => "Return getattr(self, name).", + "_ast.Assign.__getstate__" => "Helper for pickle.", + "_ast.Assign.__gt__" => "Return self>value.", + "_ast.Assign.__hash__" => "Return hash(self).", + "_ast.Assign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Assign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Assign.__le__" => "Return self<=value.", + "_ast.Assign.__lt__" => "Return self<value.", + "_ast.Assign.__ne__" => "Return self!=value.", + "_ast.Assign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Assign.__reduce_ex__" => "Helper for pickle.", + "_ast.Assign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Assign.__repr__" => "Return repr(self).", + "_ast.Assign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Assign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Assign.__str__" => "Return str(self).", + "_ast.Assign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Assign.__weakref__" => "list of weak references to the object", + "_ast.AsyncFor" => "AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "_ast.AsyncFor.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncFor.__eq__" => "Return self==value.", + "_ast.AsyncFor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncFor.__ge__" => "Return self>=value.", + "_ast.AsyncFor.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncFor.__getstate__" => "Helper for pickle.", + "_ast.AsyncFor.__gt__" => "Return self>value.", + "_ast.AsyncFor.__hash__" => "Return hash(self).", + "_ast.AsyncFor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncFor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncFor.__le__" => "Return self<=value.", + "_ast.AsyncFor.__lt__" => "Return self<value.", + "_ast.AsyncFor.__ne__" => "Return self!=value.", + "_ast.AsyncFor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncFor.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncFor.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncFor.__repr__" => "Return repr(self).", + "_ast.AsyncFor.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncFor.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncFor.__str__" => "Return str(self).", + "_ast.AsyncFor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncFor.__weakref__" => "list of weak references to the object", + "_ast.AsyncFunctionDef" => "AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)", + "_ast.AsyncFunctionDef.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncFunctionDef.__eq__" => "Return self==value.", + "_ast.AsyncFunctionDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncFunctionDef.__ge__" => "Return self>=value.", + "_ast.AsyncFunctionDef.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncFunctionDef.__getstate__" => "Helper for pickle.", + "_ast.AsyncFunctionDef.__gt__" => "Return self>value.", + "_ast.AsyncFunctionDef.__hash__" => "Return hash(self).", + "_ast.AsyncFunctionDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncFunctionDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncFunctionDef.__le__" => "Return self<=value.", + "_ast.AsyncFunctionDef.__lt__" => "Return self<value.", + "_ast.AsyncFunctionDef.__ne__" => "Return self!=value.", + "_ast.AsyncFunctionDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncFunctionDef.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncFunctionDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncFunctionDef.__repr__" => "Return repr(self).", + "_ast.AsyncFunctionDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncFunctionDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncFunctionDef.__str__" => "Return str(self).", + "_ast.AsyncFunctionDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncFunctionDef.__weakref__" => "list of weak references to the object", + "_ast.AsyncWith" => "AsyncWith(withitem* items, stmt* body, string? type_comment)", + "_ast.AsyncWith.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncWith.__eq__" => "Return self==value.", + "_ast.AsyncWith.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncWith.__ge__" => "Return self>=value.", + "_ast.AsyncWith.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncWith.__getstate__" => "Helper for pickle.", + "_ast.AsyncWith.__gt__" => "Return self>value.", + "_ast.AsyncWith.__hash__" => "Return hash(self).", + "_ast.AsyncWith.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncWith.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncWith.__le__" => "Return self<=value.", + "_ast.AsyncWith.__lt__" => "Return self<value.", + "_ast.AsyncWith.__ne__" => "Return self!=value.", + "_ast.AsyncWith.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncWith.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncWith.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncWith.__repr__" => "Return repr(self).", + "_ast.AsyncWith.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncWith.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncWith.__str__" => "Return str(self).", + "_ast.AsyncWith.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncWith.__weakref__" => "list of weak references to the object", + "_ast.Attribute" => "Attribute(expr value, identifier attr, expr_context ctx)", + "_ast.Attribute.__delattr__" => "Implement delattr(self, name).", + "_ast.Attribute.__eq__" => "Return self==value.", + "_ast.Attribute.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Attribute.__ge__" => "Return self>=value.", + "_ast.Attribute.__getattribute__" => "Return getattr(self, name).", + "_ast.Attribute.__getstate__" => "Helper for pickle.", + "_ast.Attribute.__gt__" => "Return self>value.", + "_ast.Attribute.__hash__" => "Return hash(self).", + "_ast.Attribute.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Attribute.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Attribute.__le__" => "Return self<=value.", + "_ast.Attribute.__lt__" => "Return self<value.", + "_ast.Attribute.__ne__" => "Return self!=value.", + "_ast.Attribute.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Attribute.__reduce_ex__" => "Helper for pickle.", + "_ast.Attribute.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Attribute.__repr__" => "Return repr(self).", + "_ast.Attribute.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Attribute.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Attribute.__str__" => "Return str(self).", + "_ast.Attribute.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Attribute.__weakref__" => "list of weak references to the object", + "_ast.AugAssign" => "AugAssign(expr target, operator op, expr value)", + "_ast.AugAssign.__delattr__" => "Implement delattr(self, name).", + "_ast.AugAssign.__eq__" => "Return self==value.", + "_ast.AugAssign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AugAssign.__ge__" => "Return self>=value.", + "_ast.AugAssign.__getattribute__" => "Return getattr(self, name).", + "_ast.AugAssign.__getstate__" => "Helper for pickle.", + "_ast.AugAssign.__gt__" => "Return self>value.", + "_ast.AugAssign.__hash__" => "Return hash(self).", + "_ast.AugAssign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AugAssign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AugAssign.__le__" => "Return self<=value.", + "_ast.AugAssign.__lt__" => "Return self<value.", + "_ast.AugAssign.__ne__" => "Return self!=value.", + "_ast.AugAssign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AugAssign.__reduce_ex__" => "Helper for pickle.", + "_ast.AugAssign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AugAssign.__repr__" => "Return repr(self).", + "_ast.AugAssign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AugAssign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AugAssign.__str__" => "Return str(self).", + "_ast.AugAssign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AugAssign.__weakref__" => "list of weak references to the object", + "_ast.Await" => "Await(expr value)", + "_ast.Await.__delattr__" => "Implement delattr(self, name).", + "_ast.Await.__eq__" => "Return self==value.", + "_ast.Await.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Await.__ge__" => "Return self>=value.", + "_ast.Await.__getattribute__" => "Return getattr(self, name).", + "_ast.Await.__getstate__" => "Helper for pickle.", + "_ast.Await.__gt__" => "Return self>value.", + "_ast.Await.__hash__" => "Return hash(self).", + "_ast.Await.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Await.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Await.__le__" => "Return self<=value.", + "_ast.Await.__lt__" => "Return self<value.", + "_ast.Await.__ne__" => "Return self!=value.", + "_ast.Await.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Await.__reduce_ex__" => "Helper for pickle.", + "_ast.Await.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Await.__repr__" => "Return repr(self).", + "_ast.Await.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Await.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Await.__str__" => "Return str(self).", + "_ast.Await.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Await.__weakref__" => "list of weak references to the object", + "_ast.BinOp" => "BinOp(expr left, operator op, expr right)", + "_ast.BinOp.__delattr__" => "Implement delattr(self, name).", + "_ast.BinOp.__eq__" => "Return self==value.", + "_ast.BinOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BinOp.__ge__" => "Return self>=value.", + "_ast.BinOp.__getattribute__" => "Return getattr(self, name).", + "_ast.BinOp.__getstate__" => "Helper for pickle.", + "_ast.BinOp.__gt__" => "Return self>value.", + "_ast.BinOp.__hash__" => "Return hash(self).", + "_ast.BinOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BinOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BinOp.__le__" => "Return self<=value.", + "_ast.BinOp.__lt__" => "Return self<value.", + "_ast.BinOp.__ne__" => "Return self!=value.", + "_ast.BinOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BinOp.__reduce_ex__" => "Helper for pickle.", + "_ast.BinOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BinOp.__repr__" => "Return repr(self).", + "_ast.BinOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BinOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BinOp.__str__" => "Return str(self).", + "_ast.BinOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BinOp.__weakref__" => "list of weak references to the object", + "_ast.BitAnd" => "BitAnd", + "_ast.BitAnd.__delattr__" => "Implement delattr(self, name).", + "_ast.BitAnd.__eq__" => "Return self==value.", + "_ast.BitAnd.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitAnd.__ge__" => "Return self>=value.", + "_ast.BitAnd.__getattribute__" => "Return getattr(self, name).", + "_ast.BitAnd.__getstate__" => "Helper for pickle.", + "_ast.BitAnd.__gt__" => "Return self>value.", + "_ast.BitAnd.__hash__" => "Return hash(self).", + "_ast.BitAnd.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitAnd.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitAnd.__le__" => "Return self<=value.", + "_ast.BitAnd.__lt__" => "Return self<value.", + "_ast.BitAnd.__ne__" => "Return self!=value.", + "_ast.BitAnd.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitAnd.__reduce_ex__" => "Helper for pickle.", + "_ast.BitAnd.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitAnd.__repr__" => "Return repr(self).", + "_ast.BitAnd.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitAnd.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitAnd.__str__" => "Return str(self).", + "_ast.BitAnd.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitAnd.__weakref__" => "list of weak references to the object", + "_ast.BitOr" => "BitOr", + "_ast.BitOr.__delattr__" => "Implement delattr(self, name).", + "_ast.BitOr.__eq__" => "Return self==value.", + "_ast.BitOr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitOr.__ge__" => "Return self>=value.", + "_ast.BitOr.__getattribute__" => "Return getattr(self, name).", + "_ast.BitOr.__getstate__" => "Helper for pickle.", + "_ast.BitOr.__gt__" => "Return self>value.", + "_ast.BitOr.__hash__" => "Return hash(self).", + "_ast.BitOr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitOr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitOr.__le__" => "Return self<=value.", + "_ast.BitOr.__lt__" => "Return self<value.", + "_ast.BitOr.__ne__" => "Return self!=value.", + "_ast.BitOr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitOr.__reduce_ex__" => "Helper for pickle.", + "_ast.BitOr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitOr.__repr__" => "Return repr(self).", + "_ast.BitOr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitOr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitOr.__str__" => "Return str(self).", + "_ast.BitOr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitOr.__weakref__" => "list of weak references to the object", + "_ast.BitXor" => "BitXor", + "_ast.BitXor.__delattr__" => "Implement delattr(self, name).", + "_ast.BitXor.__eq__" => "Return self==value.", + "_ast.BitXor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitXor.__ge__" => "Return self>=value.", + "_ast.BitXor.__getattribute__" => "Return getattr(self, name).", + "_ast.BitXor.__getstate__" => "Helper for pickle.", + "_ast.BitXor.__gt__" => "Return self>value.", + "_ast.BitXor.__hash__" => "Return hash(self).", + "_ast.BitXor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitXor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitXor.__le__" => "Return self<=value.", + "_ast.BitXor.__lt__" => "Return self<value.", + "_ast.BitXor.__ne__" => "Return self!=value.", + "_ast.BitXor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitXor.__reduce_ex__" => "Helper for pickle.", + "_ast.BitXor.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitXor.__repr__" => "Return repr(self).", + "_ast.BitXor.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitXor.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitXor.__str__" => "Return str(self).", + "_ast.BitXor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitXor.__weakref__" => "list of weak references to the object", + "_ast.BoolOp" => "BoolOp(boolop op, expr* values)", + "_ast.BoolOp.__delattr__" => "Implement delattr(self, name).", + "_ast.BoolOp.__eq__" => "Return self==value.", + "_ast.BoolOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BoolOp.__ge__" => "Return self>=value.", + "_ast.BoolOp.__getattribute__" => "Return getattr(self, name).", + "_ast.BoolOp.__getstate__" => "Helper for pickle.", + "_ast.BoolOp.__gt__" => "Return self>value.", + "_ast.BoolOp.__hash__" => "Return hash(self).", + "_ast.BoolOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BoolOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BoolOp.__le__" => "Return self<=value.", + "_ast.BoolOp.__lt__" => "Return self<value.", + "_ast.BoolOp.__ne__" => "Return self!=value.", + "_ast.BoolOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BoolOp.__reduce_ex__" => "Helper for pickle.", + "_ast.BoolOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BoolOp.__repr__" => "Return repr(self).", + "_ast.BoolOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BoolOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BoolOp.__str__" => "Return str(self).", + "_ast.BoolOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BoolOp.__weakref__" => "list of weak references to the object", + "_ast.Break" => "Break", + "_ast.Break.__delattr__" => "Implement delattr(self, name).", + "_ast.Break.__eq__" => "Return self==value.", + "_ast.Break.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Break.__ge__" => "Return self>=value.", + "_ast.Break.__getattribute__" => "Return getattr(self, name).", + "_ast.Break.__getstate__" => "Helper for pickle.", + "_ast.Break.__gt__" => "Return self>value.", + "_ast.Break.__hash__" => "Return hash(self).", + "_ast.Break.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Break.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Break.__le__" => "Return self<=value.", + "_ast.Break.__lt__" => "Return self<value.", + "_ast.Break.__ne__" => "Return self!=value.", + "_ast.Break.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Break.__reduce_ex__" => "Helper for pickle.", + "_ast.Break.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Break.__repr__" => "Return repr(self).", + "_ast.Break.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Break.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Break.__str__" => "Return str(self).", + "_ast.Break.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Break.__weakref__" => "list of weak references to the object", + "_ast.Call" => "Call(expr func, expr* args, keyword* keywords)", + "_ast.Call.__delattr__" => "Implement delattr(self, name).", + "_ast.Call.__eq__" => "Return self==value.", + "_ast.Call.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Call.__ge__" => "Return self>=value.", + "_ast.Call.__getattribute__" => "Return getattr(self, name).", + "_ast.Call.__getstate__" => "Helper for pickle.", + "_ast.Call.__gt__" => "Return self>value.", + "_ast.Call.__hash__" => "Return hash(self).", + "_ast.Call.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Call.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Call.__le__" => "Return self<=value.", + "_ast.Call.__lt__" => "Return self<value.", + "_ast.Call.__ne__" => "Return self!=value.", + "_ast.Call.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Call.__reduce_ex__" => "Helper for pickle.", + "_ast.Call.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Call.__repr__" => "Return repr(self).", + "_ast.Call.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Call.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Call.__str__" => "Return str(self).", + "_ast.Call.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Call.__weakref__" => "list of weak references to the object", + "_ast.ClassDef" => "ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)", + "_ast.ClassDef.__delattr__" => "Implement delattr(self, name).", + "_ast.ClassDef.__eq__" => "Return self==value.", + "_ast.ClassDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ClassDef.__ge__" => "Return self>=value.", + "_ast.ClassDef.__getattribute__" => "Return getattr(self, name).", + "_ast.ClassDef.__getstate__" => "Helper for pickle.", + "_ast.ClassDef.__gt__" => "Return self>value.", + "_ast.ClassDef.__hash__" => "Return hash(self).", + "_ast.ClassDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ClassDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ClassDef.__le__" => "Return self<=value.", + "_ast.ClassDef.__lt__" => "Return self<value.", + "_ast.ClassDef.__ne__" => "Return self!=value.", + "_ast.ClassDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ClassDef.__reduce_ex__" => "Helper for pickle.", + "_ast.ClassDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ClassDef.__repr__" => "Return repr(self).", + "_ast.ClassDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ClassDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ClassDef.__str__" => "Return str(self).", + "_ast.ClassDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ClassDef.__weakref__" => "list of weak references to the object", + "_ast.Compare" => "Compare(expr left, cmpop* ops, expr* comparators)", + "_ast.Compare.__delattr__" => "Implement delattr(self, name).", + "_ast.Compare.__eq__" => "Return self==value.", + "_ast.Compare.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Compare.__ge__" => "Return self>=value.", + "_ast.Compare.__getattribute__" => "Return getattr(self, name).", + "_ast.Compare.__getstate__" => "Helper for pickle.", + "_ast.Compare.__gt__" => "Return self>value.", + "_ast.Compare.__hash__" => "Return hash(self).", + "_ast.Compare.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Compare.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Compare.__le__" => "Return self<=value.", + "_ast.Compare.__lt__" => "Return self<value.", + "_ast.Compare.__ne__" => "Return self!=value.", + "_ast.Compare.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Compare.__reduce_ex__" => "Helper for pickle.", + "_ast.Compare.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Compare.__repr__" => "Return repr(self).", + "_ast.Compare.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Compare.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Compare.__str__" => "Return str(self).", + "_ast.Compare.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Compare.__weakref__" => "list of weak references to the object", + "_ast.Constant" => "Constant(constant value, string? kind)", + "_ast.Constant.__delattr__" => "Implement delattr(self, name).", + "_ast.Constant.__eq__" => "Return self==value.", + "_ast.Constant.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Constant.__ge__" => "Return self>=value.", + "_ast.Constant.__getattribute__" => "Return getattr(self, name).", + "_ast.Constant.__getstate__" => "Helper for pickle.", + "_ast.Constant.__gt__" => "Return self>value.", + "_ast.Constant.__hash__" => "Return hash(self).", + "_ast.Constant.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Constant.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Constant.__le__" => "Return self<=value.", + "_ast.Constant.__lt__" => "Return self<value.", + "_ast.Constant.__ne__" => "Return self!=value.", + "_ast.Constant.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Constant.__reduce_ex__" => "Helper for pickle.", + "_ast.Constant.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Constant.__repr__" => "Return repr(self).", + "_ast.Constant.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Constant.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Constant.__str__" => "Return str(self).", + "_ast.Constant.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Constant.__weakref__" => "list of weak references to the object", + "_ast.Continue" => "Continue", + "_ast.Continue.__delattr__" => "Implement delattr(self, name).", + "_ast.Continue.__eq__" => "Return self==value.", + "_ast.Continue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Continue.__ge__" => "Return self>=value.", + "_ast.Continue.__getattribute__" => "Return getattr(self, name).", + "_ast.Continue.__getstate__" => "Helper for pickle.", + "_ast.Continue.__gt__" => "Return self>value.", + "_ast.Continue.__hash__" => "Return hash(self).", + "_ast.Continue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Continue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Continue.__le__" => "Return self<=value.", + "_ast.Continue.__lt__" => "Return self<value.", + "_ast.Continue.__ne__" => "Return self!=value.", + "_ast.Continue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Continue.__reduce_ex__" => "Helper for pickle.", + "_ast.Continue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Continue.__repr__" => "Return repr(self).", + "_ast.Continue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Continue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Continue.__str__" => "Return str(self).", + "_ast.Continue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Continue.__weakref__" => "list of weak references to the object", + "_ast.Del" => "Del", + "_ast.Del.__delattr__" => "Implement delattr(self, name).", + "_ast.Del.__eq__" => "Return self==value.", + "_ast.Del.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Del.__ge__" => "Return self>=value.", + "_ast.Del.__getattribute__" => "Return getattr(self, name).", + "_ast.Del.__getstate__" => "Helper for pickle.", + "_ast.Del.__gt__" => "Return self>value.", + "_ast.Del.__hash__" => "Return hash(self).", + "_ast.Del.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Del.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Del.__le__" => "Return self<=value.", + "_ast.Del.__lt__" => "Return self<value.", + "_ast.Del.__ne__" => "Return self!=value.", + "_ast.Del.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Del.__reduce_ex__" => "Helper for pickle.", + "_ast.Del.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Del.__repr__" => "Return repr(self).", + "_ast.Del.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Del.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Del.__str__" => "Return str(self).", + "_ast.Del.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Del.__weakref__" => "list of weak references to the object", + "_ast.Delete" => "Delete(expr* targets)", + "_ast.Delete.__delattr__" => "Implement delattr(self, name).", + "_ast.Delete.__eq__" => "Return self==value.", + "_ast.Delete.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Delete.__ge__" => "Return self>=value.", + "_ast.Delete.__getattribute__" => "Return getattr(self, name).", + "_ast.Delete.__getstate__" => "Helper for pickle.", + "_ast.Delete.__gt__" => "Return self>value.", + "_ast.Delete.__hash__" => "Return hash(self).", + "_ast.Delete.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Delete.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Delete.__le__" => "Return self<=value.", + "_ast.Delete.__lt__" => "Return self<value.", + "_ast.Delete.__ne__" => "Return self!=value.", + "_ast.Delete.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Delete.__reduce_ex__" => "Helper for pickle.", + "_ast.Delete.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Delete.__repr__" => "Return repr(self).", + "_ast.Delete.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Delete.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Delete.__str__" => "Return str(self).", + "_ast.Delete.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Delete.__weakref__" => "list of weak references to the object", + "_ast.Dict" => "Dict(expr?* keys, expr* values)", + "_ast.Dict.__delattr__" => "Implement delattr(self, name).", + "_ast.Dict.__eq__" => "Return self==value.", + "_ast.Dict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Dict.__ge__" => "Return self>=value.", + "_ast.Dict.__getattribute__" => "Return getattr(self, name).", + "_ast.Dict.__getstate__" => "Helper for pickle.", + "_ast.Dict.__gt__" => "Return self>value.", + "_ast.Dict.__hash__" => "Return hash(self).", + "_ast.Dict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Dict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Dict.__le__" => "Return self<=value.", + "_ast.Dict.__lt__" => "Return self<value.", + "_ast.Dict.__ne__" => "Return self!=value.", + "_ast.Dict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Dict.__reduce_ex__" => "Helper for pickle.", + "_ast.Dict.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Dict.__repr__" => "Return repr(self).", + "_ast.Dict.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Dict.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Dict.__str__" => "Return str(self).", + "_ast.Dict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Dict.__weakref__" => "list of weak references to the object", + "_ast.DictComp" => "DictComp(expr key, expr value, comprehension* generators)", + "_ast.DictComp.__delattr__" => "Implement delattr(self, name).", + "_ast.DictComp.__eq__" => "Return self==value.", + "_ast.DictComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.DictComp.__ge__" => "Return self>=value.", + "_ast.DictComp.__getattribute__" => "Return getattr(self, name).", + "_ast.DictComp.__getstate__" => "Helper for pickle.", + "_ast.DictComp.__gt__" => "Return self>value.", + "_ast.DictComp.__hash__" => "Return hash(self).", + "_ast.DictComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.DictComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.DictComp.__le__" => "Return self<=value.", + "_ast.DictComp.__lt__" => "Return self<value.", + "_ast.DictComp.__ne__" => "Return self!=value.", + "_ast.DictComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.DictComp.__reduce_ex__" => "Helper for pickle.", + "_ast.DictComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.DictComp.__repr__" => "Return repr(self).", + "_ast.DictComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.DictComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.DictComp.__str__" => "Return str(self).", + "_ast.DictComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.DictComp.__weakref__" => "list of weak references to the object", + "_ast.Div" => "Div", + "_ast.Div.__delattr__" => "Implement delattr(self, name).", + "_ast.Div.__eq__" => "Return self==value.", + "_ast.Div.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Div.__ge__" => "Return self>=value.", + "_ast.Div.__getattribute__" => "Return getattr(self, name).", + "_ast.Div.__getstate__" => "Helper for pickle.", + "_ast.Div.__gt__" => "Return self>value.", + "_ast.Div.__hash__" => "Return hash(self).", + "_ast.Div.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Div.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Div.__le__" => "Return self<=value.", + "_ast.Div.__lt__" => "Return self<value.", + "_ast.Div.__ne__" => "Return self!=value.", + "_ast.Div.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Div.__reduce_ex__" => "Helper for pickle.", + "_ast.Div.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Div.__repr__" => "Return repr(self).", + "_ast.Div.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Div.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Div.__str__" => "Return str(self).", + "_ast.Div.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Div.__weakref__" => "list of weak references to the object", + "_ast.Eq" => "Eq", + "_ast.Eq.__delattr__" => "Implement delattr(self, name).", + "_ast.Eq.__eq__" => "Return self==value.", + "_ast.Eq.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Eq.__ge__" => "Return self>=value.", + "_ast.Eq.__getattribute__" => "Return getattr(self, name).", + "_ast.Eq.__getstate__" => "Helper for pickle.", + "_ast.Eq.__gt__" => "Return self>value.", + "_ast.Eq.__hash__" => "Return hash(self).", + "_ast.Eq.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Eq.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Eq.__le__" => "Return self<=value.", + "_ast.Eq.__lt__" => "Return self<value.", + "_ast.Eq.__ne__" => "Return self!=value.", + "_ast.Eq.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Eq.__reduce_ex__" => "Helper for pickle.", + "_ast.Eq.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Eq.__repr__" => "Return repr(self).", + "_ast.Eq.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Eq.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Eq.__str__" => "Return str(self).", + "_ast.Eq.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Eq.__weakref__" => "list of weak references to the object", + "_ast.ExceptHandler" => "ExceptHandler(expr? type, identifier? name, stmt* body)", + "_ast.ExceptHandler.__delattr__" => "Implement delattr(self, name).", + "_ast.ExceptHandler.__eq__" => "Return self==value.", + "_ast.ExceptHandler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ExceptHandler.__ge__" => "Return self>=value.", + "_ast.ExceptHandler.__getattribute__" => "Return getattr(self, name).", + "_ast.ExceptHandler.__getstate__" => "Helper for pickle.", + "_ast.ExceptHandler.__gt__" => "Return self>value.", + "_ast.ExceptHandler.__hash__" => "Return hash(self).", + "_ast.ExceptHandler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ExceptHandler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ExceptHandler.__le__" => "Return self<=value.", + "_ast.ExceptHandler.__lt__" => "Return self<value.", + "_ast.ExceptHandler.__ne__" => "Return self!=value.", + "_ast.ExceptHandler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ExceptHandler.__reduce_ex__" => "Helper for pickle.", + "_ast.ExceptHandler.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ExceptHandler.__repr__" => "Return repr(self).", + "_ast.ExceptHandler.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ExceptHandler.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ExceptHandler.__str__" => "Return str(self).", + "_ast.ExceptHandler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ExceptHandler.__weakref__" => "list of weak references to the object", + "_ast.Expr" => "Expr(expr value)", + "_ast.Expr.__delattr__" => "Implement delattr(self, name).", + "_ast.Expr.__eq__" => "Return self==value.", + "_ast.Expr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Expr.__ge__" => "Return self>=value.", + "_ast.Expr.__getattribute__" => "Return getattr(self, name).", + "_ast.Expr.__getstate__" => "Helper for pickle.", + "_ast.Expr.__gt__" => "Return self>value.", + "_ast.Expr.__hash__" => "Return hash(self).", + "_ast.Expr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Expr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Expr.__le__" => "Return self<=value.", + "_ast.Expr.__lt__" => "Return self<value.", + "_ast.Expr.__ne__" => "Return self!=value.", + "_ast.Expr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Expr.__reduce_ex__" => "Helper for pickle.", + "_ast.Expr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Expr.__repr__" => "Return repr(self).", + "_ast.Expr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Expr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Expr.__str__" => "Return str(self).", + "_ast.Expr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Expr.__weakref__" => "list of weak references to the object", + "_ast.Expression" => "Expression(expr body)", + "_ast.Expression.__delattr__" => "Implement delattr(self, name).", + "_ast.Expression.__eq__" => "Return self==value.", + "_ast.Expression.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Expression.__ge__" => "Return self>=value.", + "_ast.Expression.__getattribute__" => "Return getattr(self, name).", + "_ast.Expression.__getstate__" => "Helper for pickle.", + "_ast.Expression.__gt__" => "Return self>value.", + "_ast.Expression.__hash__" => "Return hash(self).", + "_ast.Expression.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Expression.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Expression.__le__" => "Return self<=value.", + "_ast.Expression.__lt__" => "Return self<value.", + "_ast.Expression.__ne__" => "Return self!=value.", + "_ast.Expression.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Expression.__reduce_ex__" => "Helper for pickle.", + "_ast.Expression.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Expression.__repr__" => "Return repr(self).", + "_ast.Expression.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Expression.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Expression.__str__" => "Return str(self).", + "_ast.Expression.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Expression.__weakref__" => "list of weak references to the object", + "_ast.FloorDiv" => "FloorDiv", + "_ast.FloorDiv.__delattr__" => "Implement delattr(self, name).", + "_ast.FloorDiv.__eq__" => "Return self==value.", + "_ast.FloorDiv.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FloorDiv.__ge__" => "Return self>=value.", + "_ast.FloorDiv.__getattribute__" => "Return getattr(self, name).", + "_ast.FloorDiv.__getstate__" => "Helper for pickle.", + "_ast.FloorDiv.__gt__" => "Return self>value.", + "_ast.FloorDiv.__hash__" => "Return hash(self).", + "_ast.FloorDiv.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FloorDiv.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FloorDiv.__le__" => "Return self<=value.", + "_ast.FloorDiv.__lt__" => "Return self<value.", + "_ast.FloorDiv.__ne__" => "Return self!=value.", + "_ast.FloorDiv.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FloorDiv.__reduce_ex__" => "Helper for pickle.", + "_ast.FloorDiv.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FloorDiv.__repr__" => "Return repr(self).", + "_ast.FloorDiv.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FloorDiv.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FloorDiv.__str__" => "Return str(self).", + "_ast.FloorDiv.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FloorDiv.__weakref__" => "list of weak references to the object", + "_ast.For" => "For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "_ast.For.__delattr__" => "Implement delattr(self, name).", + "_ast.For.__eq__" => "Return self==value.", + "_ast.For.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.For.__ge__" => "Return self>=value.", + "_ast.For.__getattribute__" => "Return getattr(self, name).", + "_ast.For.__getstate__" => "Helper for pickle.", + "_ast.For.__gt__" => "Return self>value.", + "_ast.For.__hash__" => "Return hash(self).", + "_ast.For.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.For.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.For.__le__" => "Return self<=value.", + "_ast.For.__lt__" => "Return self<value.", + "_ast.For.__ne__" => "Return self!=value.", + "_ast.For.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.For.__reduce_ex__" => "Helper for pickle.", + "_ast.For.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.For.__repr__" => "Return repr(self).", + "_ast.For.__setattr__" => "Implement setattr(self, name, value).", + "_ast.For.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.For.__str__" => "Return str(self).", + "_ast.For.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.For.__weakref__" => "list of weak references to the object", + "_ast.FormattedValue" => "FormattedValue(expr value, int conversion, expr? format_spec)", + "_ast.FormattedValue.__delattr__" => "Implement delattr(self, name).", + "_ast.FormattedValue.__eq__" => "Return self==value.", + "_ast.FormattedValue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FormattedValue.__ge__" => "Return self>=value.", + "_ast.FormattedValue.__getattribute__" => "Return getattr(self, name).", + "_ast.FormattedValue.__getstate__" => "Helper for pickle.", + "_ast.FormattedValue.__gt__" => "Return self>value.", + "_ast.FormattedValue.__hash__" => "Return hash(self).", + "_ast.FormattedValue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FormattedValue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FormattedValue.__le__" => "Return self<=value.", + "_ast.FormattedValue.__lt__" => "Return self<value.", + "_ast.FormattedValue.__ne__" => "Return self!=value.", + "_ast.FormattedValue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FormattedValue.__reduce_ex__" => "Helper for pickle.", + "_ast.FormattedValue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FormattedValue.__repr__" => "Return repr(self).", + "_ast.FormattedValue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FormattedValue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FormattedValue.__str__" => "Return str(self).", + "_ast.FormattedValue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FormattedValue.__weakref__" => "list of weak references to the object", + "_ast.FunctionDef" => "FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)", + "_ast.FunctionDef.__delattr__" => "Implement delattr(self, name).", + "_ast.FunctionDef.__eq__" => "Return self==value.", + "_ast.FunctionDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FunctionDef.__ge__" => "Return self>=value.", + "_ast.FunctionDef.__getattribute__" => "Return getattr(self, name).", + "_ast.FunctionDef.__getstate__" => "Helper for pickle.", + "_ast.FunctionDef.__gt__" => "Return self>value.", + "_ast.FunctionDef.__hash__" => "Return hash(self).", + "_ast.FunctionDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FunctionDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FunctionDef.__le__" => "Return self<=value.", + "_ast.FunctionDef.__lt__" => "Return self<value.", + "_ast.FunctionDef.__ne__" => "Return self!=value.", + "_ast.FunctionDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FunctionDef.__reduce_ex__" => "Helper for pickle.", + "_ast.FunctionDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FunctionDef.__repr__" => "Return repr(self).", + "_ast.FunctionDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FunctionDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FunctionDef.__str__" => "Return str(self).", + "_ast.FunctionDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FunctionDef.__weakref__" => "list of weak references to the object", + "_ast.FunctionType" => "FunctionType(expr* argtypes, expr returns)", + "_ast.FunctionType.__delattr__" => "Implement delattr(self, name).", + "_ast.FunctionType.__eq__" => "Return self==value.", + "_ast.FunctionType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FunctionType.__ge__" => "Return self>=value.", + "_ast.FunctionType.__getattribute__" => "Return getattr(self, name).", + "_ast.FunctionType.__getstate__" => "Helper for pickle.", + "_ast.FunctionType.__gt__" => "Return self>value.", + "_ast.FunctionType.__hash__" => "Return hash(self).", + "_ast.FunctionType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FunctionType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FunctionType.__le__" => "Return self<=value.", + "_ast.FunctionType.__lt__" => "Return self<value.", + "_ast.FunctionType.__ne__" => "Return self!=value.", + "_ast.FunctionType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FunctionType.__reduce_ex__" => "Helper for pickle.", + "_ast.FunctionType.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FunctionType.__repr__" => "Return repr(self).", + "_ast.FunctionType.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FunctionType.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FunctionType.__str__" => "Return str(self).", + "_ast.FunctionType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FunctionType.__weakref__" => "list of weak references to the object", + "_ast.GeneratorExp" => "GeneratorExp(expr elt, comprehension* generators)", + "_ast.GeneratorExp.__delattr__" => "Implement delattr(self, name).", + "_ast.GeneratorExp.__eq__" => "Return self==value.", + "_ast.GeneratorExp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.GeneratorExp.__ge__" => "Return self>=value.", + "_ast.GeneratorExp.__getattribute__" => "Return getattr(self, name).", + "_ast.GeneratorExp.__getstate__" => "Helper for pickle.", + "_ast.GeneratorExp.__gt__" => "Return self>value.", + "_ast.GeneratorExp.__hash__" => "Return hash(self).", + "_ast.GeneratorExp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.GeneratorExp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.GeneratorExp.__le__" => "Return self<=value.", + "_ast.GeneratorExp.__lt__" => "Return self<value.", + "_ast.GeneratorExp.__ne__" => "Return self!=value.", + "_ast.GeneratorExp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.GeneratorExp.__reduce_ex__" => "Helper for pickle.", + "_ast.GeneratorExp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.GeneratorExp.__repr__" => "Return repr(self).", + "_ast.GeneratorExp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.GeneratorExp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.GeneratorExp.__str__" => "Return str(self).", + "_ast.GeneratorExp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.GeneratorExp.__weakref__" => "list of weak references to the object", + "_ast.Global" => "Global(identifier* names)", + "_ast.Global.__delattr__" => "Implement delattr(self, name).", + "_ast.Global.__eq__" => "Return self==value.", + "_ast.Global.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Global.__ge__" => "Return self>=value.", + "_ast.Global.__getattribute__" => "Return getattr(self, name).", + "_ast.Global.__getstate__" => "Helper for pickle.", + "_ast.Global.__gt__" => "Return self>value.", + "_ast.Global.__hash__" => "Return hash(self).", + "_ast.Global.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Global.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Global.__le__" => "Return self<=value.", + "_ast.Global.__lt__" => "Return self<value.", + "_ast.Global.__ne__" => "Return self!=value.", + "_ast.Global.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Global.__reduce_ex__" => "Helper for pickle.", + "_ast.Global.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Global.__repr__" => "Return repr(self).", + "_ast.Global.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Global.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Global.__str__" => "Return str(self).", + "_ast.Global.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Global.__weakref__" => "list of weak references to the object", + "_ast.Gt" => "Gt", + "_ast.Gt.__delattr__" => "Implement delattr(self, name).", + "_ast.Gt.__eq__" => "Return self==value.", + "_ast.Gt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Gt.__ge__" => "Return self>=value.", + "_ast.Gt.__getattribute__" => "Return getattr(self, name).", + "_ast.Gt.__getstate__" => "Helper for pickle.", + "_ast.Gt.__gt__" => "Return self>value.", + "_ast.Gt.__hash__" => "Return hash(self).", + "_ast.Gt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Gt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Gt.__le__" => "Return self<=value.", + "_ast.Gt.__lt__" => "Return self<value.", + "_ast.Gt.__ne__" => "Return self!=value.", + "_ast.Gt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Gt.__reduce_ex__" => "Helper for pickle.", + "_ast.Gt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Gt.__repr__" => "Return repr(self).", + "_ast.Gt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Gt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Gt.__str__" => "Return str(self).", + "_ast.Gt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Gt.__weakref__" => "list of weak references to the object", + "_ast.GtE" => "GtE", + "_ast.GtE.__delattr__" => "Implement delattr(self, name).", + "_ast.GtE.__eq__" => "Return self==value.", + "_ast.GtE.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.GtE.__ge__" => "Return self>=value.", + "_ast.GtE.__getattribute__" => "Return getattr(self, name).", + "_ast.GtE.__getstate__" => "Helper for pickle.", + "_ast.GtE.__gt__" => "Return self>value.", + "_ast.GtE.__hash__" => "Return hash(self).", + "_ast.GtE.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.GtE.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.GtE.__le__" => "Return self<=value.", + "_ast.GtE.__lt__" => "Return self<value.", + "_ast.GtE.__ne__" => "Return self!=value.", + "_ast.GtE.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.GtE.__reduce_ex__" => "Helper for pickle.", + "_ast.GtE.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.GtE.__repr__" => "Return repr(self).", + "_ast.GtE.__setattr__" => "Implement setattr(self, name, value).", + "_ast.GtE.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.GtE.__str__" => "Return str(self).", + "_ast.GtE.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.GtE.__weakref__" => "list of weak references to the object", + "_ast.If" => "If(expr test, stmt* body, stmt* orelse)", + "_ast.If.__delattr__" => "Implement delattr(self, name).", + "_ast.If.__eq__" => "Return self==value.", + "_ast.If.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.If.__ge__" => "Return self>=value.", + "_ast.If.__getattribute__" => "Return getattr(self, name).", + "_ast.If.__getstate__" => "Helper for pickle.", + "_ast.If.__gt__" => "Return self>value.", + "_ast.If.__hash__" => "Return hash(self).", + "_ast.If.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.If.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.If.__le__" => "Return self<=value.", + "_ast.If.__lt__" => "Return self<value.", + "_ast.If.__ne__" => "Return self!=value.", + "_ast.If.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.If.__reduce_ex__" => "Helper for pickle.", + "_ast.If.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.If.__repr__" => "Return repr(self).", + "_ast.If.__setattr__" => "Implement setattr(self, name, value).", + "_ast.If.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.If.__str__" => "Return str(self).", + "_ast.If.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.If.__weakref__" => "list of weak references to the object", + "_ast.IfExp" => "IfExp(expr test, expr body, expr orelse)", + "_ast.IfExp.__delattr__" => "Implement delattr(self, name).", + "_ast.IfExp.__eq__" => "Return self==value.", + "_ast.IfExp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.IfExp.__ge__" => "Return self>=value.", + "_ast.IfExp.__getattribute__" => "Return getattr(self, name).", + "_ast.IfExp.__getstate__" => "Helper for pickle.", + "_ast.IfExp.__gt__" => "Return self>value.", + "_ast.IfExp.__hash__" => "Return hash(self).", + "_ast.IfExp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.IfExp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.IfExp.__le__" => "Return self<=value.", + "_ast.IfExp.__lt__" => "Return self<value.", + "_ast.IfExp.__ne__" => "Return self!=value.", + "_ast.IfExp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.IfExp.__reduce_ex__" => "Helper for pickle.", + "_ast.IfExp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.IfExp.__repr__" => "Return repr(self).", + "_ast.IfExp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.IfExp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.IfExp.__str__" => "Return str(self).", + "_ast.IfExp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.IfExp.__weakref__" => "list of weak references to the object", + "_ast.Import" => "Import(alias* names)", + "_ast.Import.__delattr__" => "Implement delattr(self, name).", + "_ast.Import.__eq__" => "Return self==value.", + "_ast.Import.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Import.__ge__" => "Return self>=value.", + "_ast.Import.__getattribute__" => "Return getattr(self, name).", + "_ast.Import.__getstate__" => "Helper for pickle.", + "_ast.Import.__gt__" => "Return self>value.", + "_ast.Import.__hash__" => "Return hash(self).", + "_ast.Import.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Import.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Import.__le__" => "Return self<=value.", + "_ast.Import.__lt__" => "Return self<value.", + "_ast.Import.__ne__" => "Return self!=value.", + "_ast.Import.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Import.__reduce_ex__" => "Helper for pickle.", + "_ast.Import.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Import.__repr__" => "Return repr(self).", + "_ast.Import.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Import.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Import.__str__" => "Return str(self).", + "_ast.Import.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Import.__weakref__" => "list of weak references to the object", + "_ast.ImportFrom" => "ImportFrom(identifier? module, alias* names, int? level)", + "_ast.ImportFrom.__delattr__" => "Implement delattr(self, name).", + "_ast.ImportFrom.__eq__" => "Return self==value.", + "_ast.ImportFrom.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ImportFrom.__ge__" => "Return self>=value.", + "_ast.ImportFrom.__getattribute__" => "Return getattr(self, name).", + "_ast.ImportFrom.__getstate__" => "Helper for pickle.", + "_ast.ImportFrom.__gt__" => "Return self>value.", + "_ast.ImportFrom.__hash__" => "Return hash(self).", + "_ast.ImportFrom.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ImportFrom.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ImportFrom.__le__" => "Return self<=value.", + "_ast.ImportFrom.__lt__" => "Return self<value.", + "_ast.ImportFrom.__ne__" => "Return self!=value.", + "_ast.ImportFrom.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ImportFrom.__reduce_ex__" => "Helper for pickle.", + "_ast.ImportFrom.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ImportFrom.__repr__" => "Return repr(self).", + "_ast.ImportFrom.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ImportFrom.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ImportFrom.__str__" => "Return str(self).", + "_ast.ImportFrom.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ImportFrom.__weakref__" => "list of weak references to the object", + "_ast.In" => "In", + "_ast.In.__delattr__" => "Implement delattr(self, name).", + "_ast.In.__eq__" => "Return self==value.", + "_ast.In.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.In.__ge__" => "Return self>=value.", + "_ast.In.__getattribute__" => "Return getattr(self, name).", + "_ast.In.__getstate__" => "Helper for pickle.", + "_ast.In.__gt__" => "Return self>value.", + "_ast.In.__hash__" => "Return hash(self).", + "_ast.In.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.In.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.In.__le__" => "Return self<=value.", + "_ast.In.__lt__" => "Return self<value.", + "_ast.In.__ne__" => "Return self!=value.", + "_ast.In.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.In.__reduce_ex__" => "Helper for pickle.", + "_ast.In.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.In.__repr__" => "Return repr(self).", + "_ast.In.__setattr__" => "Implement setattr(self, name, value).", + "_ast.In.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.In.__str__" => "Return str(self).", + "_ast.In.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.In.__weakref__" => "list of weak references to the object", + "_ast.Interactive" => "Interactive(stmt* body)", + "_ast.Interactive.__delattr__" => "Implement delattr(self, name).", + "_ast.Interactive.__eq__" => "Return self==value.", + "_ast.Interactive.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Interactive.__ge__" => "Return self>=value.", + "_ast.Interactive.__getattribute__" => "Return getattr(self, name).", + "_ast.Interactive.__getstate__" => "Helper for pickle.", + "_ast.Interactive.__gt__" => "Return self>value.", + "_ast.Interactive.__hash__" => "Return hash(self).", + "_ast.Interactive.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Interactive.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Interactive.__le__" => "Return self<=value.", + "_ast.Interactive.__lt__" => "Return self<value.", + "_ast.Interactive.__ne__" => "Return self!=value.", + "_ast.Interactive.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Interactive.__reduce_ex__" => "Helper for pickle.", + "_ast.Interactive.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Interactive.__repr__" => "Return repr(self).", + "_ast.Interactive.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Interactive.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Interactive.__str__" => "Return str(self).", + "_ast.Interactive.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Interactive.__weakref__" => "list of weak references to the object", + "_ast.Interpolation" => "Interpolation(expr value, constant str, int conversion, expr? format_spec)", + "_ast.Interpolation.__delattr__" => "Implement delattr(self, name).", + "_ast.Interpolation.__eq__" => "Return self==value.", + "_ast.Interpolation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Interpolation.__ge__" => "Return self>=value.", + "_ast.Interpolation.__getattribute__" => "Return getattr(self, name).", + "_ast.Interpolation.__getstate__" => "Helper for pickle.", + "_ast.Interpolation.__gt__" => "Return self>value.", + "_ast.Interpolation.__hash__" => "Return hash(self).", + "_ast.Interpolation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Interpolation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Interpolation.__le__" => "Return self<=value.", + "_ast.Interpolation.__lt__" => "Return self<value.", + "_ast.Interpolation.__ne__" => "Return self!=value.", + "_ast.Interpolation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Interpolation.__reduce_ex__" => "Helper for pickle.", + "_ast.Interpolation.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Interpolation.__repr__" => "Return repr(self).", + "_ast.Interpolation.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Interpolation.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Interpolation.__str__" => "Return str(self).", + "_ast.Interpolation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Interpolation.__weakref__" => "list of weak references to the object", + "_ast.Invert" => "Invert", + "_ast.Invert.__delattr__" => "Implement delattr(self, name).", + "_ast.Invert.__eq__" => "Return self==value.", + "_ast.Invert.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Invert.__ge__" => "Return self>=value.", + "_ast.Invert.__getattribute__" => "Return getattr(self, name).", + "_ast.Invert.__getstate__" => "Helper for pickle.", + "_ast.Invert.__gt__" => "Return self>value.", + "_ast.Invert.__hash__" => "Return hash(self).", + "_ast.Invert.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Invert.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Invert.__le__" => "Return self<=value.", + "_ast.Invert.__lt__" => "Return self<value.", + "_ast.Invert.__ne__" => "Return self!=value.", + "_ast.Invert.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Invert.__reduce_ex__" => "Helper for pickle.", + "_ast.Invert.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Invert.__repr__" => "Return repr(self).", + "_ast.Invert.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Invert.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Invert.__str__" => "Return str(self).", + "_ast.Invert.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Invert.__weakref__" => "list of weak references to the object", + "_ast.Is" => "Is", + "_ast.Is.__delattr__" => "Implement delattr(self, name).", + "_ast.Is.__eq__" => "Return self==value.", + "_ast.Is.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Is.__ge__" => "Return self>=value.", + "_ast.Is.__getattribute__" => "Return getattr(self, name).", + "_ast.Is.__getstate__" => "Helper for pickle.", + "_ast.Is.__gt__" => "Return self>value.", + "_ast.Is.__hash__" => "Return hash(self).", + "_ast.Is.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Is.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Is.__le__" => "Return self<=value.", + "_ast.Is.__lt__" => "Return self<value.", + "_ast.Is.__ne__" => "Return self!=value.", + "_ast.Is.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Is.__reduce_ex__" => "Helper for pickle.", + "_ast.Is.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Is.__repr__" => "Return repr(self).", + "_ast.Is.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Is.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Is.__str__" => "Return str(self).", + "_ast.Is.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Is.__weakref__" => "list of weak references to the object", + "_ast.IsNot" => "IsNot", + "_ast.IsNot.__delattr__" => "Implement delattr(self, name).", + "_ast.IsNot.__eq__" => "Return self==value.", + "_ast.IsNot.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.IsNot.__ge__" => "Return self>=value.", + "_ast.IsNot.__getattribute__" => "Return getattr(self, name).", + "_ast.IsNot.__getstate__" => "Helper for pickle.", + "_ast.IsNot.__gt__" => "Return self>value.", + "_ast.IsNot.__hash__" => "Return hash(self).", + "_ast.IsNot.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.IsNot.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.IsNot.__le__" => "Return self<=value.", + "_ast.IsNot.__lt__" => "Return self<value.", + "_ast.IsNot.__ne__" => "Return self!=value.", + "_ast.IsNot.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.IsNot.__reduce_ex__" => "Helper for pickle.", + "_ast.IsNot.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.IsNot.__repr__" => "Return repr(self).", + "_ast.IsNot.__setattr__" => "Implement setattr(self, name, value).", + "_ast.IsNot.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.IsNot.__str__" => "Return str(self).", + "_ast.IsNot.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.IsNot.__weakref__" => "list of weak references to the object", + "_ast.JoinedStr" => "JoinedStr(expr* values)", + "_ast.JoinedStr.__delattr__" => "Implement delattr(self, name).", + "_ast.JoinedStr.__eq__" => "Return self==value.", + "_ast.JoinedStr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.JoinedStr.__ge__" => "Return self>=value.", + "_ast.JoinedStr.__getattribute__" => "Return getattr(self, name).", + "_ast.JoinedStr.__getstate__" => "Helper for pickle.", + "_ast.JoinedStr.__gt__" => "Return self>value.", + "_ast.JoinedStr.__hash__" => "Return hash(self).", + "_ast.JoinedStr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.JoinedStr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.JoinedStr.__le__" => "Return self<=value.", + "_ast.JoinedStr.__lt__" => "Return self<value.", + "_ast.JoinedStr.__ne__" => "Return self!=value.", + "_ast.JoinedStr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.JoinedStr.__reduce_ex__" => "Helper for pickle.", + "_ast.JoinedStr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.JoinedStr.__repr__" => "Return repr(self).", + "_ast.JoinedStr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.JoinedStr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.JoinedStr.__str__" => "Return str(self).", + "_ast.JoinedStr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.JoinedStr.__weakref__" => "list of weak references to the object", + "_ast.LShift" => "LShift", + "_ast.LShift.__delattr__" => "Implement delattr(self, name).", + "_ast.LShift.__eq__" => "Return self==value.", + "_ast.LShift.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.LShift.__ge__" => "Return self>=value.", + "_ast.LShift.__getattribute__" => "Return getattr(self, name).", + "_ast.LShift.__getstate__" => "Helper for pickle.", + "_ast.LShift.__gt__" => "Return self>value.", + "_ast.LShift.__hash__" => "Return hash(self).", + "_ast.LShift.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.LShift.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.LShift.__le__" => "Return self<=value.", + "_ast.LShift.__lt__" => "Return self<value.", + "_ast.LShift.__ne__" => "Return self!=value.", + "_ast.LShift.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.LShift.__reduce_ex__" => "Helper for pickle.", + "_ast.LShift.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.LShift.__repr__" => "Return repr(self).", + "_ast.LShift.__setattr__" => "Implement setattr(self, name, value).", + "_ast.LShift.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.LShift.__str__" => "Return str(self).", + "_ast.LShift.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.LShift.__weakref__" => "list of weak references to the object", + "_ast.Lambda" => "Lambda(arguments args, expr body)", + "_ast.Lambda.__delattr__" => "Implement delattr(self, name).", + "_ast.Lambda.__eq__" => "Return self==value.", + "_ast.Lambda.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Lambda.__ge__" => "Return self>=value.", + "_ast.Lambda.__getattribute__" => "Return getattr(self, name).", + "_ast.Lambda.__getstate__" => "Helper for pickle.", + "_ast.Lambda.__gt__" => "Return self>value.", + "_ast.Lambda.__hash__" => "Return hash(self).", + "_ast.Lambda.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Lambda.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Lambda.__le__" => "Return self<=value.", + "_ast.Lambda.__lt__" => "Return self<value.", + "_ast.Lambda.__ne__" => "Return self!=value.", + "_ast.Lambda.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Lambda.__reduce_ex__" => "Helper for pickle.", + "_ast.Lambda.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Lambda.__repr__" => "Return repr(self).", + "_ast.Lambda.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Lambda.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Lambda.__str__" => "Return str(self).", + "_ast.Lambda.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Lambda.__weakref__" => "list of weak references to the object", + "_ast.List" => "List(expr* elts, expr_context ctx)", + "_ast.List.__delattr__" => "Implement delattr(self, name).", + "_ast.List.__eq__" => "Return self==value.", + "_ast.List.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.List.__ge__" => "Return self>=value.", + "_ast.List.__getattribute__" => "Return getattr(self, name).", + "_ast.List.__getstate__" => "Helper for pickle.", + "_ast.List.__gt__" => "Return self>value.", + "_ast.List.__hash__" => "Return hash(self).", + "_ast.List.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.List.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.List.__le__" => "Return self<=value.", + "_ast.List.__lt__" => "Return self<value.", + "_ast.List.__ne__" => "Return self!=value.", + "_ast.List.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.List.__reduce_ex__" => "Helper for pickle.", + "_ast.List.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.List.__repr__" => "Return repr(self).", + "_ast.List.__setattr__" => "Implement setattr(self, name, value).", + "_ast.List.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.List.__str__" => "Return str(self).", + "_ast.List.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.List.__weakref__" => "list of weak references to the object", + "_ast.ListComp" => "ListComp(expr elt, comprehension* generators)", + "_ast.ListComp.__delattr__" => "Implement delattr(self, name).", + "_ast.ListComp.__eq__" => "Return self==value.", + "_ast.ListComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ListComp.__ge__" => "Return self>=value.", + "_ast.ListComp.__getattribute__" => "Return getattr(self, name).", + "_ast.ListComp.__getstate__" => "Helper for pickle.", + "_ast.ListComp.__gt__" => "Return self>value.", + "_ast.ListComp.__hash__" => "Return hash(self).", + "_ast.ListComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ListComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ListComp.__le__" => "Return self<=value.", + "_ast.ListComp.__lt__" => "Return self<value.", + "_ast.ListComp.__ne__" => "Return self!=value.", + "_ast.ListComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ListComp.__reduce_ex__" => "Helper for pickle.", + "_ast.ListComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ListComp.__repr__" => "Return repr(self).", + "_ast.ListComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ListComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ListComp.__str__" => "Return str(self).", + "_ast.ListComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ListComp.__weakref__" => "list of weak references to the object", + "_ast.Load" => "Load", + "_ast.Load.__delattr__" => "Implement delattr(self, name).", + "_ast.Load.__eq__" => "Return self==value.", + "_ast.Load.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Load.__ge__" => "Return self>=value.", + "_ast.Load.__getattribute__" => "Return getattr(self, name).", + "_ast.Load.__getstate__" => "Helper for pickle.", + "_ast.Load.__gt__" => "Return self>value.", + "_ast.Load.__hash__" => "Return hash(self).", + "_ast.Load.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Load.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Load.__le__" => "Return self<=value.", + "_ast.Load.__lt__" => "Return self<value.", + "_ast.Load.__ne__" => "Return self!=value.", + "_ast.Load.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Load.__reduce_ex__" => "Helper for pickle.", + "_ast.Load.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Load.__repr__" => "Return repr(self).", + "_ast.Load.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Load.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Load.__str__" => "Return str(self).", + "_ast.Load.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Load.__weakref__" => "list of weak references to the object", + "_ast.Lt" => "Lt", + "_ast.Lt.__delattr__" => "Implement delattr(self, name).", + "_ast.Lt.__eq__" => "Return self==value.", + "_ast.Lt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Lt.__ge__" => "Return self>=value.", + "_ast.Lt.__getattribute__" => "Return getattr(self, name).", + "_ast.Lt.__getstate__" => "Helper for pickle.", + "_ast.Lt.__gt__" => "Return self>value.", + "_ast.Lt.__hash__" => "Return hash(self).", + "_ast.Lt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Lt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Lt.__le__" => "Return self<=value.", + "_ast.Lt.__lt__" => "Return self<value.", + "_ast.Lt.__ne__" => "Return self!=value.", + "_ast.Lt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Lt.__reduce_ex__" => "Helper for pickle.", + "_ast.Lt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Lt.__repr__" => "Return repr(self).", + "_ast.Lt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Lt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Lt.__str__" => "Return str(self).", + "_ast.Lt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Lt.__weakref__" => "list of weak references to the object", + "_ast.LtE" => "LtE", + "_ast.LtE.__delattr__" => "Implement delattr(self, name).", + "_ast.LtE.__eq__" => "Return self==value.", + "_ast.LtE.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.LtE.__ge__" => "Return self>=value.", + "_ast.LtE.__getattribute__" => "Return getattr(self, name).", + "_ast.LtE.__getstate__" => "Helper for pickle.", + "_ast.LtE.__gt__" => "Return self>value.", + "_ast.LtE.__hash__" => "Return hash(self).", + "_ast.LtE.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.LtE.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.LtE.__le__" => "Return self<=value.", + "_ast.LtE.__lt__" => "Return self<value.", + "_ast.LtE.__ne__" => "Return self!=value.", + "_ast.LtE.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.LtE.__reduce_ex__" => "Helper for pickle.", + "_ast.LtE.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.LtE.__repr__" => "Return repr(self).", + "_ast.LtE.__setattr__" => "Implement setattr(self, name, value).", + "_ast.LtE.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.LtE.__str__" => "Return str(self).", + "_ast.LtE.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.LtE.__weakref__" => "list of weak references to the object", + "_ast.MatMult" => "MatMult", + "_ast.MatMult.__delattr__" => "Implement delattr(self, name).", + "_ast.MatMult.__eq__" => "Return self==value.", + "_ast.MatMult.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatMult.__ge__" => "Return self>=value.", + "_ast.MatMult.__getattribute__" => "Return getattr(self, name).", + "_ast.MatMult.__getstate__" => "Helper for pickle.", + "_ast.MatMult.__gt__" => "Return self>value.", + "_ast.MatMult.__hash__" => "Return hash(self).", + "_ast.MatMult.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatMult.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatMult.__le__" => "Return self<=value.", + "_ast.MatMult.__lt__" => "Return self<value.", + "_ast.MatMult.__ne__" => "Return self!=value.", + "_ast.MatMult.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatMult.__reduce_ex__" => "Helper for pickle.", + "_ast.MatMult.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatMult.__repr__" => "Return repr(self).", + "_ast.MatMult.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatMult.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatMult.__str__" => "Return str(self).", + "_ast.MatMult.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatMult.__weakref__" => "list of weak references to the object", + "_ast.Match" => "Match(expr subject, match_case* cases)", + "_ast.Match.__delattr__" => "Implement delattr(self, name).", + "_ast.Match.__eq__" => "Return self==value.", + "_ast.Match.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Match.__ge__" => "Return self>=value.", + "_ast.Match.__getattribute__" => "Return getattr(self, name).", + "_ast.Match.__getstate__" => "Helper for pickle.", + "_ast.Match.__gt__" => "Return self>value.", + "_ast.Match.__hash__" => "Return hash(self).", + "_ast.Match.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Match.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Match.__le__" => "Return self<=value.", + "_ast.Match.__lt__" => "Return self<value.", + "_ast.Match.__ne__" => "Return self!=value.", + "_ast.Match.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Match.__reduce_ex__" => "Helper for pickle.", + "_ast.Match.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Match.__repr__" => "Return repr(self).", + "_ast.Match.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Match.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Match.__str__" => "Return str(self).", + "_ast.Match.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Match.__weakref__" => "list of weak references to the object", + "_ast.MatchAs" => "MatchAs(pattern? pattern, identifier? name)", + "_ast.MatchAs.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchAs.__eq__" => "Return self==value.", + "_ast.MatchAs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchAs.__ge__" => "Return self>=value.", + "_ast.MatchAs.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchAs.__getstate__" => "Helper for pickle.", + "_ast.MatchAs.__gt__" => "Return self>value.", + "_ast.MatchAs.__hash__" => "Return hash(self).", + "_ast.MatchAs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchAs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchAs.__le__" => "Return self<=value.", + "_ast.MatchAs.__lt__" => "Return self<value.", + "_ast.MatchAs.__ne__" => "Return self!=value.", + "_ast.MatchAs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchAs.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchAs.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchAs.__repr__" => "Return repr(self).", + "_ast.MatchAs.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchAs.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchAs.__str__" => "Return str(self).", + "_ast.MatchAs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchAs.__weakref__" => "list of weak references to the object", + "_ast.MatchClass" => "MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)", + "_ast.MatchClass.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchClass.__eq__" => "Return self==value.", + "_ast.MatchClass.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchClass.__ge__" => "Return self>=value.", + "_ast.MatchClass.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchClass.__getstate__" => "Helper for pickle.", + "_ast.MatchClass.__gt__" => "Return self>value.", + "_ast.MatchClass.__hash__" => "Return hash(self).", + "_ast.MatchClass.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchClass.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchClass.__le__" => "Return self<=value.", + "_ast.MatchClass.__lt__" => "Return self<value.", + "_ast.MatchClass.__ne__" => "Return self!=value.", + "_ast.MatchClass.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchClass.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchClass.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchClass.__repr__" => "Return repr(self).", + "_ast.MatchClass.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchClass.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchClass.__str__" => "Return str(self).", + "_ast.MatchClass.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchClass.__weakref__" => "list of weak references to the object", + "_ast.MatchMapping" => "MatchMapping(expr* keys, pattern* patterns, identifier? rest)", + "_ast.MatchMapping.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchMapping.__eq__" => "Return self==value.", + "_ast.MatchMapping.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchMapping.__ge__" => "Return self>=value.", + "_ast.MatchMapping.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchMapping.__getstate__" => "Helper for pickle.", + "_ast.MatchMapping.__gt__" => "Return self>value.", + "_ast.MatchMapping.__hash__" => "Return hash(self).", + "_ast.MatchMapping.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchMapping.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchMapping.__le__" => "Return self<=value.", + "_ast.MatchMapping.__lt__" => "Return self<value.", + "_ast.MatchMapping.__ne__" => "Return self!=value.", + "_ast.MatchMapping.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchMapping.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchMapping.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchMapping.__repr__" => "Return repr(self).", + "_ast.MatchMapping.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchMapping.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchMapping.__str__" => "Return str(self).", + "_ast.MatchMapping.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchMapping.__weakref__" => "list of weak references to the object", + "_ast.MatchOr" => "MatchOr(pattern* patterns)", + "_ast.MatchOr.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchOr.__eq__" => "Return self==value.", + "_ast.MatchOr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchOr.__ge__" => "Return self>=value.", + "_ast.MatchOr.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchOr.__getstate__" => "Helper for pickle.", + "_ast.MatchOr.__gt__" => "Return self>value.", + "_ast.MatchOr.__hash__" => "Return hash(self).", + "_ast.MatchOr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchOr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchOr.__le__" => "Return self<=value.", + "_ast.MatchOr.__lt__" => "Return self<value.", + "_ast.MatchOr.__ne__" => "Return self!=value.", + "_ast.MatchOr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchOr.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchOr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchOr.__repr__" => "Return repr(self).", + "_ast.MatchOr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchOr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchOr.__str__" => "Return str(self).", + "_ast.MatchOr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchOr.__weakref__" => "list of weak references to the object", + "_ast.MatchSequence" => "MatchSequence(pattern* patterns)", + "_ast.MatchSequence.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchSequence.__eq__" => "Return self==value.", + "_ast.MatchSequence.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchSequence.__ge__" => "Return self>=value.", + "_ast.MatchSequence.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchSequence.__getstate__" => "Helper for pickle.", + "_ast.MatchSequence.__gt__" => "Return self>value.", + "_ast.MatchSequence.__hash__" => "Return hash(self).", + "_ast.MatchSequence.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchSequence.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchSequence.__le__" => "Return self<=value.", + "_ast.MatchSequence.__lt__" => "Return self<value.", + "_ast.MatchSequence.__ne__" => "Return self!=value.", + "_ast.MatchSequence.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchSequence.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchSequence.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchSequence.__repr__" => "Return repr(self).", + "_ast.MatchSequence.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchSequence.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchSequence.__str__" => "Return str(self).", + "_ast.MatchSequence.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchSequence.__weakref__" => "list of weak references to the object", + "_ast.MatchSingleton" => "MatchSingleton(constant value)", + "_ast.MatchSingleton.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchSingleton.__eq__" => "Return self==value.", + "_ast.MatchSingleton.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchSingleton.__ge__" => "Return self>=value.", + "_ast.MatchSingleton.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchSingleton.__getstate__" => "Helper for pickle.", + "_ast.MatchSingleton.__gt__" => "Return self>value.", + "_ast.MatchSingleton.__hash__" => "Return hash(self).", + "_ast.MatchSingleton.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchSingleton.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchSingleton.__le__" => "Return self<=value.", + "_ast.MatchSingleton.__lt__" => "Return self<value.", + "_ast.MatchSingleton.__ne__" => "Return self!=value.", + "_ast.MatchSingleton.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchSingleton.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchSingleton.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchSingleton.__repr__" => "Return repr(self).", + "_ast.MatchSingleton.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchSingleton.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchSingleton.__str__" => "Return str(self).", + "_ast.MatchSingleton.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchSingleton.__weakref__" => "list of weak references to the object", + "_ast.MatchStar" => "MatchStar(identifier? name)", + "_ast.MatchStar.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchStar.__eq__" => "Return self==value.", + "_ast.MatchStar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchStar.__ge__" => "Return self>=value.", + "_ast.MatchStar.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchStar.__getstate__" => "Helper for pickle.", + "_ast.MatchStar.__gt__" => "Return self>value.", + "_ast.MatchStar.__hash__" => "Return hash(self).", + "_ast.MatchStar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchStar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchStar.__le__" => "Return self<=value.", + "_ast.MatchStar.__lt__" => "Return self<value.", + "_ast.MatchStar.__ne__" => "Return self!=value.", + "_ast.MatchStar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchStar.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchStar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchStar.__repr__" => "Return repr(self).", + "_ast.MatchStar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchStar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchStar.__str__" => "Return str(self).", + "_ast.MatchStar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchStar.__weakref__" => "list of weak references to the object", + "_ast.MatchValue" => "MatchValue(expr value)", + "_ast.MatchValue.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchValue.__eq__" => "Return self==value.", + "_ast.MatchValue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchValue.__ge__" => "Return self>=value.", + "_ast.MatchValue.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchValue.__getstate__" => "Helper for pickle.", + "_ast.MatchValue.__gt__" => "Return self>value.", + "_ast.MatchValue.__hash__" => "Return hash(self).", + "_ast.MatchValue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchValue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchValue.__le__" => "Return self<=value.", + "_ast.MatchValue.__lt__" => "Return self<value.", + "_ast.MatchValue.__ne__" => "Return self!=value.", + "_ast.MatchValue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchValue.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchValue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchValue.__repr__" => "Return repr(self).", + "_ast.MatchValue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchValue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchValue.__str__" => "Return str(self).", + "_ast.MatchValue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchValue.__weakref__" => "list of weak references to the object", + "_ast.Mod" => "Mod", + "_ast.Mod.__delattr__" => "Implement delattr(self, name).", + "_ast.Mod.__eq__" => "Return self==value.", + "_ast.Mod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Mod.__ge__" => "Return self>=value.", + "_ast.Mod.__getattribute__" => "Return getattr(self, name).", + "_ast.Mod.__getstate__" => "Helper for pickle.", + "_ast.Mod.__gt__" => "Return self>value.", + "_ast.Mod.__hash__" => "Return hash(self).", + "_ast.Mod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Mod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Mod.__le__" => "Return self<=value.", + "_ast.Mod.__lt__" => "Return self<value.", + "_ast.Mod.__ne__" => "Return self!=value.", + "_ast.Mod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Mod.__reduce_ex__" => "Helper for pickle.", + "_ast.Mod.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Mod.__repr__" => "Return repr(self).", + "_ast.Mod.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Mod.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Mod.__str__" => "Return str(self).", + "_ast.Mod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Mod.__weakref__" => "list of weak references to the object", + "_ast.Module" => "Module(stmt* body, type_ignore* type_ignores)", + "_ast.Module.__delattr__" => "Implement delattr(self, name).", + "_ast.Module.__eq__" => "Return self==value.", + "_ast.Module.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Module.__ge__" => "Return self>=value.", + "_ast.Module.__getattribute__" => "Return getattr(self, name).", + "_ast.Module.__getstate__" => "Helper for pickle.", + "_ast.Module.__gt__" => "Return self>value.", + "_ast.Module.__hash__" => "Return hash(self).", + "_ast.Module.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Module.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Module.__le__" => "Return self<=value.", + "_ast.Module.__lt__" => "Return self<value.", + "_ast.Module.__ne__" => "Return self!=value.", + "_ast.Module.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Module.__reduce_ex__" => "Helper for pickle.", + "_ast.Module.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Module.__repr__" => "Return repr(self).", + "_ast.Module.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Module.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Module.__str__" => "Return str(self).", + "_ast.Module.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Module.__weakref__" => "list of weak references to the object", + "_ast.Mult" => "Mult", + "_ast.Mult.__delattr__" => "Implement delattr(self, name).", + "_ast.Mult.__eq__" => "Return self==value.", + "_ast.Mult.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Mult.__ge__" => "Return self>=value.", + "_ast.Mult.__getattribute__" => "Return getattr(self, name).", + "_ast.Mult.__getstate__" => "Helper for pickle.", + "_ast.Mult.__gt__" => "Return self>value.", + "_ast.Mult.__hash__" => "Return hash(self).", + "_ast.Mult.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Mult.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Mult.__le__" => "Return self<=value.", + "_ast.Mult.__lt__" => "Return self<value.", + "_ast.Mult.__ne__" => "Return self!=value.", + "_ast.Mult.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Mult.__reduce_ex__" => "Helper for pickle.", + "_ast.Mult.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Mult.__repr__" => "Return repr(self).", + "_ast.Mult.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Mult.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Mult.__str__" => "Return str(self).", + "_ast.Mult.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Mult.__weakref__" => "list of weak references to the object", + "_ast.Name" => "Name(identifier id, expr_context ctx)", + "_ast.Name.__delattr__" => "Implement delattr(self, name).", + "_ast.Name.__eq__" => "Return self==value.", + "_ast.Name.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Name.__ge__" => "Return self>=value.", + "_ast.Name.__getattribute__" => "Return getattr(self, name).", + "_ast.Name.__getstate__" => "Helper for pickle.", + "_ast.Name.__gt__" => "Return self>value.", + "_ast.Name.__hash__" => "Return hash(self).", + "_ast.Name.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Name.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Name.__le__" => "Return self<=value.", + "_ast.Name.__lt__" => "Return self<value.", + "_ast.Name.__ne__" => "Return self!=value.", + "_ast.Name.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Name.__reduce_ex__" => "Helper for pickle.", + "_ast.Name.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Name.__repr__" => "Return repr(self).", + "_ast.Name.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Name.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Name.__str__" => "Return str(self).", + "_ast.Name.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Name.__weakref__" => "list of weak references to the object", + "_ast.NamedExpr" => "NamedExpr(expr target, expr value)", + "_ast.NamedExpr.__delattr__" => "Implement delattr(self, name).", + "_ast.NamedExpr.__eq__" => "Return self==value.", + "_ast.NamedExpr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NamedExpr.__ge__" => "Return self>=value.", + "_ast.NamedExpr.__getattribute__" => "Return getattr(self, name).", + "_ast.NamedExpr.__getstate__" => "Helper for pickle.", + "_ast.NamedExpr.__gt__" => "Return self>value.", + "_ast.NamedExpr.__hash__" => "Return hash(self).", + "_ast.NamedExpr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NamedExpr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NamedExpr.__le__" => "Return self<=value.", + "_ast.NamedExpr.__lt__" => "Return self<value.", + "_ast.NamedExpr.__ne__" => "Return self!=value.", + "_ast.NamedExpr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NamedExpr.__reduce_ex__" => "Helper for pickle.", + "_ast.NamedExpr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NamedExpr.__repr__" => "Return repr(self).", + "_ast.NamedExpr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NamedExpr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NamedExpr.__str__" => "Return str(self).", + "_ast.NamedExpr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NamedExpr.__weakref__" => "list of weak references to the object", + "_ast.Nonlocal" => "Nonlocal(identifier* names)", + "_ast.Nonlocal.__delattr__" => "Implement delattr(self, name).", + "_ast.Nonlocal.__eq__" => "Return self==value.", + "_ast.Nonlocal.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Nonlocal.__ge__" => "Return self>=value.", + "_ast.Nonlocal.__getattribute__" => "Return getattr(self, name).", + "_ast.Nonlocal.__getstate__" => "Helper for pickle.", + "_ast.Nonlocal.__gt__" => "Return self>value.", + "_ast.Nonlocal.__hash__" => "Return hash(self).", + "_ast.Nonlocal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Nonlocal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Nonlocal.__le__" => "Return self<=value.", + "_ast.Nonlocal.__lt__" => "Return self<value.", + "_ast.Nonlocal.__ne__" => "Return self!=value.", + "_ast.Nonlocal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Nonlocal.__reduce_ex__" => "Helper for pickle.", + "_ast.Nonlocal.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Nonlocal.__repr__" => "Return repr(self).", + "_ast.Nonlocal.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Nonlocal.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Nonlocal.__str__" => "Return str(self).", + "_ast.Nonlocal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Nonlocal.__weakref__" => "list of weak references to the object", + "_ast.Not" => "Not", + "_ast.Not.__delattr__" => "Implement delattr(self, name).", + "_ast.Not.__eq__" => "Return self==value.", + "_ast.Not.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Not.__ge__" => "Return self>=value.", + "_ast.Not.__getattribute__" => "Return getattr(self, name).", + "_ast.Not.__getstate__" => "Helper for pickle.", + "_ast.Not.__gt__" => "Return self>value.", + "_ast.Not.__hash__" => "Return hash(self).", + "_ast.Not.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Not.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Not.__le__" => "Return self<=value.", + "_ast.Not.__lt__" => "Return self<value.", + "_ast.Not.__ne__" => "Return self!=value.", + "_ast.Not.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Not.__reduce_ex__" => "Helper for pickle.", + "_ast.Not.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Not.__repr__" => "Return repr(self).", + "_ast.Not.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Not.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Not.__str__" => "Return str(self).", + "_ast.Not.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Not.__weakref__" => "list of weak references to the object", + "_ast.NotEq" => "NotEq", + "_ast.NotEq.__delattr__" => "Implement delattr(self, name).", + "_ast.NotEq.__eq__" => "Return self==value.", + "_ast.NotEq.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NotEq.__ge__" => "Return self>=value.", + "_ast.NotEq.__getattribute__" => "Return getattr(self, name).", + "_ast.NotEq.__getstate__" => "Helper for pickle.", + "_ast.NotEq.__gt__" => "Return self>value.", + "_ast.NotEq.__hash__" => "Return hash(self).", + "_ast.NotEq.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NotEq.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NotEq.__le__" => "Return self<=value.", + "_ast.NotEq.__lt__" => "Return self<value.", + "_ast.NotEq.__ne__" => "Return self!=value.", + "_ast.NotEq.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NotEq.__reduce_ex__" => "Helper for pickle.", + "_ast.NotEq.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NotEq.__repr__" => "Return repr(self).", + "_ast.NotEq.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NotEq.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NotEq.__str__" => "Return str(self).", + "_ast.NotEq.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NotEq.__weakref__" => "list of weak references to the object", + "_ast.NotIn" => "NotIn", + "_ast.NotIn.__delattr__" => "Implement delattr(self, name).", + "_ast.NotIn.__eq__" => "Return self==value.", + "_ast.NotIn.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NotIn.__ge__" => "Return self>=value.", + "_ast.NotIn.__getattribute__" => "Return getattr(self, name).", + "_ast.NotIn.__getstate__" => "Helper for pickle.", + "_ast.NotIn.__gt__" => "Return self>value.", + "_ast.NotIn.__hash__" => "Return hash(self).", + "_ast.NotIn.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NotIn.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NotIn.__le__" => "Return self<=value.", + "_ast.NotIn.__lt__" => "Return self<value.", + "_ast.NotIn.__ne__" => "Return self!=value.", + "_ast.NotIn.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NotIn.__reduce_ex__" => "Helper for pickle.", + "_ast.NotIn.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NotIn.__repr__" => "Return repr(self).", + "_ast.NotIn.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NotIn.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NotIn.__str__" => "Return str(self).", + "_ast.NotIn.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NotIn.__weakref__" => "list of weak references to the object", + "_ast.Or" => "Or", + "_ast.Or.__delattr__" => "Implement delattr(self, name).", + "_ast.Or.__eq__" => "Return self==value.", + "_ast.Or.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Or.__ge__" => "Return self>=value.", + "_ast.Or.__getattribute__" => "Return getattr(self, name).", + "_ast.Or.__getstate__" => "Helper for pickle.", + "_ast.Or.__gt__" => "Return self>value.", + "_ast.Or.__hash__" => "Return hash(self).", + "_ast.Or.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Or.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Or.__le__" => "Return self<=value.", + "_ast.Or.__lt__" => "Return self<value.", + "_ast.Or.__ne__" => "Return self!=value.", + "_ast.Or.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Or.__reduce_ex__" => "Helper for pickle.", + "_ast.Or.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Or.__repr__" => "Return repr(self).", + "_ast.Or.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Or.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Or.__str__" => "Return str(self).", + "_ast.Or.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Or.__weakref__" => "list of weak references to the object", + "_ast.ParamSpec" => "ParamSpec(identifier name, expr? default_value)", + "_ast.ParamSpec.__delattr__" => "Implement delattr(self, name).", + "_ast.ParamSpec.__eq__" => "Return self==value.", + "_ast.ParamSpec.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ParamSpec.__ge__" => "Return self>=value.", + "_ast.ParamSpec.__getattribute__" => "Return getattr(self, name).", + "_ast.ParamSpec.__getstate__" => "Helper for pickle.", + "_ast.ParamSpec.__gt__" => "Return self>value.", + "_ast.ParamSpec.__hash__" => "Return hash(self).", + "_ast.ParamSpec.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ParamSpec.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ParamSpec.__le__" => "Return self<=value.", + "_ast.ParamSpec.__lt__" => "Return self<value.", + "_ast.ParamSpec.__ne__" => "Return self!=value.", + "_ast.ParamSpec.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ParamSpec.__reduce_ex__" => "Helper for pickle.", + "_ast.ParamSpec.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ParamSpec.__repr__" => "Return repr(self).", + "_ast.ParamSpec.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ParamSpec.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ParamSpec.__str__" => "Return str(self).", + "_ast.ParamSpec.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ParamSpec.__weakref__" => "list of weak references to the object", + "_ast.Pass" => "Pass", + "_ast.Pass.__delattr__" => "Implement delattr(self, name).", + "_ast.Pass.__eq__" => "Return self==value.", + "_ast.Pass.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Pass.__ge__" => "Return self>=value.", + "_ast.Pass.__getattribute__" => "Return getattr(self, name).", + "_ast.Pass.__getstate__" => "Helper for pickle.", + "_ast.Pass.__gt__" => "Return self>value.", + "_ast.Pass.__hash__" => "Return hash(self).", + "_ast.Pass.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Pass.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Pass.__le__" => "Return self<=value.", + "_ast.Pass.__lt__" => "Return self<value.", + "_ast.Pass.__ne__" => "Return self!=value.", + "_ast.Pass.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Pass.__reduce_ex__" => "Helper for pickle.", + "_ast.Pass.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Pass.__repr__" => "Return repr(self).", + "_ast.Pass.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Pass.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Pass.__str__" => "Return str(self).", + "_ast.Pass.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Pass.__weakref__" => "list of weak references to the object", + "_ast.Pow" => "Pow", + "_ast.Pow.__delattr__" => "Implement delattr(self, name).", + "_ast.Pow.__eq__" => "Return self==value.", + "_ast.Pow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Pow.__ge__" => "Return self>=value.", + "_ast.Pow.__getattribute__" => "Return getattr(self, name).", + "_ast.Pow.__getstate__" => "Helper for pickle.", + "_ast.Pow.__gt__" => "Return self>value.", + "_ast.Pow.__hash__" => "Return hash(self).", + "_ast.Pow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Pow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Pow.__le__" => "Return self<=value.", + "_ast.Pow.__lt__" => "Return self<value.", + "_ast.Pow.__ne__" => "Return self!=value.", + "_ast.Pow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Pow.__reduce_ex__" => "Helper for pickle.", + "_ast.Pow.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Pow.__repr__" => "Return repr(self).", + "_ast.Pow.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Pow.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Pow.__str__" => "Return str(self).", + "_ast.Pow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Pow.__weakref__" => "list of weak references to the object", + "_ast.RShift" => "RShift", + "_ast.RShift.__delattr__" => "Implement delattr(self, name).", + "_ast.RShift.__eq__" => "Return self==value.", + "_ast.RShift.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.RShift.__ge__" => "Return self>=value.", + "_ast.RShift.__getattribute__" => "Return getattr(self, name).", + "_ast.RShift.__getstate__" => "Helper for pickle.", + "_ast.RShift.__gt__" => "Return self>value.", + "_ast.RShift.__hash__" => "Return hash(self).", + "_ast.RShift.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.RShift.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.RShift.__le__" => "Return self<=value.", + "_ast.RShift.__lt__" => "Return self<value.", + "_ast.RShift.__ne__" => "Return self!=value.", + "_ast.RShift.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.RShift.__reduce_ex__" => "Helper for pickle.", + "_ast.RShift.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.RShift.__repr__" => "Return repr(self).", + "_ast.RShift.__setattr__" => "Implement setattr(self, name, value).", + "_ast.RShift.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.RShift.__str__" => "Return str(self).", + "_ast.RShift.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.RShift.__weakref__" => "list of weak references to the object", + "_ast.Raise" => "Raise(expr? exc, expr? cause)", + "_ast.Raise.__delattr__" => "Implement delattr(self, name).", + "_ast.Raise.__eq__" => "Return self==value.", + "_ast.Raise.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Raise.__ge__" => "Return self>=value.", + "_ast.Raise.__getattribute__" => "Return getattr(self, name).", + "_ast.Raise.__getstate__" => "Helper for pickle.", + "_ast.Raise.__gt__" => "Return self>value.", + "_ast.Raise.__hash__" => "Return hash(self).", + "_ast.Raise.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Raise.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Raise.__le__" => "Return self<=value.", + "_ast.Raise.__lt__" => "Return self<value.", + "_ast.Raise.__ne__" => "Return self!=value.", + "_ast.Raise.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Raise.__reduce_ex__" => "Helper for pickle.", + "_ast.Raise.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Raise.__repr__" => "Return repr(self).", + "_ast.Raise.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Raise.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Raise.__str__" => "Return str(self).", + "_ast.Raise.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Raise.__weakref__" => "list of weak references to the object", + "_ast.Return" => "Return(expr? value)", + "_ast.Return.__delattr__" => "Implement delattr(self, name).", + "_ast.Return.__eq__" => "Return self==value.", + "_ast.Return.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Return.__ge__" => "Return self>=value.", + "_ast.Return.__getattribute__" => "Return getattr(self, name).", + "_ast.Return.__getstate__" => "Helper for pickle.", + "_ast.Return.__gt__" => "Return self>value.", + "_ast.Return.__hash__" => "Return hash(self).", + "_ast.Return.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Return.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Return.__le__" => "Return self<=value.", + "_ast.Return.__lt__" => "Return self<value.", + "_ast.Return.__ne__" => "Return self!=value.", + "_ast.Return.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Return.__reduce_ex__" => "Helper for pickle.", + "_ast.Return.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Return.__repr__" => "Return repr(self).", + "_ast.Return.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Return.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Return.__str__" => "Return str(self).", + "_ast.Return.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Return.__weakref__" => "list of weak references to the object", + "_ast.Set" => "Set(expr* elts)", + "_ast.Set.__delattr__" => "Implement delattr(self, name).", + "_ast.Set.__eq__" => "Return self==value.", + "_ast.Set.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Set.__ge__" => "Return self>=value.", + "_ast.Set.__getattribute__" => "Return getattr(self, name).", + "_ast.Set.__getstate__" => "Helper for pickle.", + "_ast.Set.__gt__" => "Return self>value.", + "_ast.Set.__hash__" => "Return hash(self).", + "_ast.Set.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Set.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Set.__le__" => "Return self<=value.", + "_ast.Set.__lt__" => "Return self<value.", + "_ast.Set.__ne__" => "Return self!=value.", + "_ast.Set.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Set.__reduce_ex__" => "Helper for pickle.", + "_ast.Set.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Set.__repr__" => "Return repr(self).", + "_ast.Set.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Set.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Set.__str__" => "Return str(self).", + "_ast.Set.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Set.__weakref__" => "list of weak references to the object", + "_ast.SetComp" => "SetComp(expr elt, comprehension* generators)", + "_ast.SetComp.__delattr__" => "Implement delattr(self, name).", + "_ast.SetComp.__eq__" => "Return self==value.", + "_ast.SetComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.SetComp.__ge__" => "Return self>=value.", + "_ast.SetComp.__getattribute__" => "Return getattr(self, name).", + "_ast.SetComp.__getstate__" => "Helper for pickle.", + "_ast.SetComp.__gt__" => "Return self>value.", + "_ast.SetComp.__hash__" => "Return hash(self).", + "_ast.SetComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.SetComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.SetComp.__le__" => "Return self<=value.", + "_ast.SetComp.__lt__" => "Return self<value.", + "_ast.SetComp.__ne__" => "Return self!=value.", + "_ast.SetComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.SetComp.__reduce_ex__" => "Helper for pickle.", + "_ast.SetComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.SetComp.__repr__" => "Return repr(self).", + "_ast.SetComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.SetComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.SetComp.__str__" => "Return str(self).", + "_ast.SetComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.SetComp.__weakref__" => "list of weak references to the object", + "_ast.Slice" => "Slice(expr? lower, expr? upper, expr? step)", + "_ast.Slice.__delattr__" => "Implement delattr(self, name).", + "_ast.Slice.__eq__" => "Return self==value.", + "_ast.Slice.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Slice.__ge__" => "Return self>=value.", + "_ast.Slice.__getattribute__" => "Return getattr(self, name).", + "_ast.Slice.__getstate__" => "Helper for pickle.", + "_ast.Slice.__gt__" => "Return self>value.", + "_ast.Slice.__hash__" => "Return hash(self).", + "_ast.Slice.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Slice.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Slice.__le__" => "Return self<=value.", + "_ast.Slice.__lt__" => "Return self<value.", + "_ast.Slice.__ne__" => "Return self!=value.", + "_ast.Slice.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Slice.__reduce_ex__" => "Helper for pickle.", + "_ast.Slice.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Slice.__repr__" => "Return repr(self).", + "_ast.Slice.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Slice.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Slice.__str__" => "Return str(self).", + "_ast.Slice.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Slice.__weakref__" => "list of weak references to the object", + "_ast.Starred" => "Starred(expr value, expr_context ctx)", + "_ast.Starred.__delattr__" => "Implement delattr(self, name).", + "_ast.Starred.__eq__" => "Return self==value.", + "_ast.Starred.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Starred.__ge__" => "Return self>=value.", + "_ast.Starred.__getattribute__" => "Return getattr(self, name).", + "_ast.Starred.__getstate__" => "Helper for pickle.", + "_ast.Starred.__gt__" => "Return self>value.", + "_ast.Starred.__hash__" => "Return hash(self).", + "_ast.Starred.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Starred.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Starred.__le__" => "Return self<=value.", + "_ast.Starred.__lt__" => "Return self<value.", + "_ast.Starred.__ne__" => "Return self!=value.", + "_ast.Starred.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Starred.__reduce_ex__" => "Helper for pickle.", + "_ast.Starred.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Starred.__repr__" => "Return repr(self).", + "_ast.Starred.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Starred.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Starred.__str__" => "Return str(self).", + "_ast.Starred.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Starred.__weakref__" => "list of weak references to the object", + "_ast.Store" => "Store", + "_ast.Store.__delattr__" => "Implement delattr(self, name).", + "_ast.Store.__eq__" => "Return self==value.", + "_ast.Store.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Store.__ge__" => "Return self>=value.", + "_ast.Store.__getattribute__" => "Return getattr(self, name).", + "_ast.Store.__getstate__" => "Helper for pickle.", + "_ast.Store.__gt__" => "Return self>value.", + "_ast.Store.__hash__" => "Return hash(self).", + "_ast.Store.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Store.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Store.__le__" => "Return self<=value.", + "_ast.Store.__lt__" => "Return self<value.", + "_ast.Store.__ne__" => "Return self!=value.", + "_ast.Store.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Store.__reduce_ex__" => "Helper for pickle.", + "_ast.Store.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Store.__repr__" => "Return repr(self).", + "_ast.Store.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Store.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Store.__str__" => "Return str(self).", + "_ast.Store.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Store.__weakref__" => "list of weak references to the object", + "_ast.Sub" => "Sub", + "_ast.Sub.__delattr__" => "Implement delattr(self, name).", + "_ast.Sub.__eq__" => "Return self==value.", + "_ast.Sub.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Sub.__ge__" => "Return self>=value.", + "_ast.Sub.__getattribute__" => "Return getattr(self, name).", + "_ast.Sub.__getstate__" => "Helper for pickle.", + "_ast.Sub.__gt__" => "Return self>value.", + "_ast.Sub.__hash__" => "Return hash(self).", + "_ast.Sub.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Sub.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Sub.__le__" => "Return self<=value.", + "_ast.Sub.__lt__" => "Return self<value.", + "_ast.Sub.__ne__" => "Return self!=value.", + "_ast.Sub.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Sub.__reduce_ex__" => "Helper for pickle.", + "_ast.Sub.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Sub.__repr__" => "Return repr(self).", + "_ast.Sub.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Sub.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Sub.__str__" => "Return str(self).", + "_ast.Sub.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Sub.__weakref__" => "list of weak references to the object", + "_ast.Subscript" => "Subscript(expr value, expr slice, expr_context ctx)", + "_ast.Subscript.__delattr__" => "Implement delattr(self, name).", + "_ast.Subscript.__eq__" => "Return self==value.", + "_ast.Subscript.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Subscript.__ge__" => "Return self>=value.", + "_ast.Subscript.__getattribute__" => "Return getattr(self, name).", + "_ast.Subscript.__getstate__" => "Helper for pickle.", + "_ast.Subscript.__gt__" => "Return self>value.", + "_ast.Subscript.__hash__" => "Return hash(self).", + "_ast.Subscript.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Subscript.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Subscript.__le__" => "Return self<=value.", + "_ast.Subscript.__lt__" => "Return self<value.", + "_ast.Subscript.__ne__" => "Return self!=value.", + "_ast.Subscript.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Subscript.__reduce_ex__" => "Helper for pickle.", + "_ast.Subscript.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Subscript.__repr__" => "Return repr(self).", + "_ast.Subscript.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Subscript.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Subscript.__str__" => "Return str(self).", + "_ast.Subscript.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Subscript.__weakref__" => "list of weak references to the object", + "_ast.TemplateStr" => "TemplateStr(expr* values)", + "_ast.TemplateStr.__delattr__" => "Implement delattr(self, name).", + "_ast.TemplateStr.__eq__" => "Return self==value.", + "_ast.TemplateStr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TemplateStr.__ge__" => "Return self>=value.", + "_ast.TemplateStr.__getattribute__" => "Return getattr(self, name).", + "_ast.TemplateStr.__getstate__" => "Helper for pickle.", + "_ast.TemplateStr.__gt__" => "Return self>value.", + "_ast.TemplateStr.__hash__" => "Return hash(self).", + "_ast.TemplateStr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TemplateStr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TemplateStr.__le__" => "Return self<=value.", + "_ast.TemplateStr.__lt__" => "Return self<value.", + "_ast.TemplateStr.__ne__" => "Return self!=value.", + "_ast.TemplateStr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TemplateStr.__reduce_ex__" => "Helper for pickle.", + "_ast.TemplateStr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TemplateStr.__repr__" => "Return repr(self).", + "_ast.TemplateStr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TemplateStr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TemplateStr.__str__" => "Return str(self).", + "_ast.TemplateStr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TemplateStr.__weakref__" => "list of weak references to the object", + "_ast.Try" => "Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "_ast.Try.__delattr__" => "Implement delattr(self, name).", + "_ast.Try.__eq__" => "Return self==value.", + "_ast.Try.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Try.__ge__" => "Return self>=value.", + "_ast.Try.__getattribute__" => "Return getattr(self, name).", + "_ast.Try.__getstate__" => "Helper for pickle.", + "_ast.Try.__gt__" => "Return self>value.", + "_ast.Try.__hash__" => "Return hash(self).", + "_ast.Try.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Try.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Try.__le__" => "Return self<=value.", + "_ast.Try.__lt__" => "Return self<value.", + "_ast.Try.__ne__" => "Return self!=value.", + "_ast.Try.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Try.__reduce_ex__" => "Helper for pickle.", + "_ast.Try.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Try.__repr__" => "Return repr(self).", + "_ast.Try.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Try.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Try.__str__" => "Return str(self).", + "_ast.Try.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Try.__weakref__" => "list of weak references to the object", + "_ast.TryStar" => "TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "_ast.TryStar.__delattr__" => "Implement delattr(self, name).", + "_ast.TryStar.__eq__" => "Return self==value.", + "_ast.TryStar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TryStar.__ge__" => "Return self>=value.", + "_ast.TryStar.__getattribute__" => "Return getattr(self, name).", + "_ast.TryStar.__getstate__" => "Helper for pickle.", + "_ast.TryStar.__gt__" => "Return self>value.", + "_ast.TryStar.__hash__" => "Return hash(self).", + "_ast.TryStar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TryStar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TryStar.__le__" => "Return self<=value.", + "_ast.TryStar.__lt__" => "Return self<value.", + "_ast.TryStar.__ne__" => "Return self!=value.", + "_ast.TryStar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TryStar.__reduce_ex__" => "Helper for pickle.", + "_ast.TryStar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TryStar.__repr__" => "Return repr(self).", + "_ast.TryStar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TryStar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TryStar.__str__" => "Return str(self).", + "_ast.TryStar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TryStar.__weakref__" => "list of weak references to the object", + "_ast.Tuple" => "Tuple(expr* elts, expr_context ctx)", + "_ast.Tuple.__delattr__" => "Implement delattr(self, name).", + "_ast.Tuple.__eq__" => "Return self==value.", + "_ast.Tuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Tuple.__ge__" => "Return self>=value.", + "_ast.Tuple.__getattribute__" => "Return getattr(self, name).", + "_ast.Tuple.__getstate__" => "Helper for pickle.", + "_ast.Tuple.__gt__" => "Return self>value.", + "_ast.Tuple.__hash__" => "Return hash(self).", + "_ast.Tuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Tuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Tuple.__le__" => "Return self<=value.", + "_ast.Tuple.__lt__" => "Return self<value.", + "_ast.Tuple.__ne__" => "Return self!=value.", + "_ast.Tuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Tuple.__reduce_ex__" => "Helper for pickle.", + "_ast.Tuple.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Tuple.__repr__" => "Return repr(self).", + "_ast.Tuple.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Tuple.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Tuple.__str__" => "Return str(self).", + "_ast.Tuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Tuple.__weakref__" => "list of weak references to the object", + "_ast.Tuple.dims" => "Deprecated. Use elts instead.", + "_ast.TypeAlias" => "TypeAlias(expr name, type_param* type_params, expr value)", + "_ast.TypeAlias.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeAlias.__eq__" => "Return self==value.", + "_ast.TypeAlias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeAlias.__ge__" => "Return self>=value.", + "_ast.TypeAlias.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeAlias.__getstate__" => "Helper for pickle.", + "_ast.TypeAlias.__gt__" => "Return self>value.", + "_ast.TypeAlias.__hash__" => "Return hash(self).", + "_ast.TypeAlias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeAlias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeAlias.__le__" => "Return self<=value.", + "_ast.TypeAlias.__lt__" => "Return self<value.", + "_ast.TypeAlias.__ne__" => "Return self!=value.", + "_ast.TypeAlias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeAlias.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeAlias.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeAlias.__repr__" => "Return repr(self).", + "_ast.TypeAlias.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeAlias.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeAlias.__str__" => "Return str(self).", + "_ast.TypeAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeAlias.__weakref__" => "list of weak references to the object", + "_ast.TypeIgnore" => "TypeIgnore(int lineno, string tag)", + "_ast.TypeIgnore.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeIgnore.__eq__" => "Return self==value.", + "_ast.TypeIgnore.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeIgnore.__ge__" => "Return self>=value.", + "_ast.TypeIgnore.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeIgnore.__getstate__" => "Helper for pickle.", + "_ast.TypeIgnore.__gt__" => "Return self>value.", + "_ast.TypeIgnore.__hash__" => "Return hash(self).", + "_ast.TypeIgnore.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeIgnore.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeIgnore.__le__" => "Return self<=value.", + "_ast.TypeIgnore.__lt__" => "Return self<value.", + "_ast.TypeIgnore.__ne__" => "Return self!=value.", + "_ast.TypeIgnore.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeIgnore.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeIgnore.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeIgnore.__repr__" => "Return repr(self).", + "_ast.TypeIgnore.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeIgnore.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeIgnore.__str__" => "Return str(self).", + "_ast.TypeIgnore.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeIgnore.__weakref__" => "list of weak references to the object", + "_ast.TypeVar" => "TypeVar(identifier name, expr? bound, expr? default_value)", + "_ast.TypeVar.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeVar.__eq__" => "Return self==value.", + "_ast.TypeVar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeVar.__ge__" => "Return self>=value.", + "_ast.TypeVar.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeVar.__getstate__" => "Helper for pickle.", + "_ast.TypeVar.__gt__" => "Return self>value.", + "_ast.TypeVar.__hash__" => "Return hash(self).", + "_ast.TypeVar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeVar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeVar.__le__" => "Return self<=value.", + "_ast.TypeVar.__lt__" => "Return self<value.", + "_ast.TypeVar.__ne__" => "Return self!=value.", + "_ast.TypeVar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeVar.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeVar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeVar.__repr__" => "Return repr(self).", + "_ast.TypeVar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeVar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeVar.__str__" => "Return str(self).", + "_ast.TypeVar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeVar.__weakref__" => "list of weak references to the object", + "_ast.TypeVarTuple" => "TypeVarTuple(identifier name, expr? default_value)", + "_ast.TypeVarTuple.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeVarTuple.__eq__" => "Return self==value.", + "_ast.TypeVarTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeVarTuple.__ge__" => "Return self>=value.", + "_ast.TypeVarTuple.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeVarTuple.__getstate__" => "Helper for pickle.", + "_ast.TypeVarTuple.__gt__" => "Return self>value.", + "_ast.TypeVarTuple.__hash__" => "Return hash(self).", + "_ast.TypeVarTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeVarTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeVarTuple.__le__" => "Return self<=value.", + "_ast.TypeVarTuple.__lt__" => "Return self<value.", + "_ast.TypeVarTuple.__ne__" => "Return self!=value.", + "_ast.TypeVarTuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeVarTuple.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeVarTuple.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeVarTuple.__repr__" => "Return repr(self).", + "_ast.TypeVarTuple.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeVarTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeVarTuple.__str__" => "Return str(self).", + "_ast.TypeVarTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeVarTuple.__weakref__" => "list of weak references to the object", + "_ast.UAdd" => "UAdd", + "_ast.UAdd.__delattr__" => "Implement delattr(self, name).", + "_ast.UAdd.__eq__" => "Return self==value.", + "_ast.UAdd.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.UAdd.__ge__" => "Return self>=value.", + "_ast.UAdd.__getattribute__" => "Return getattr(self, name).", + "_ast.UAdd.__getstate__" => "Helper for pickle.", + "_ast.UAdd.__gt__" => "Return self>value.", + "_ast.UAdd.__hash__" => "Return hash(self).", + "_ast.UAdd.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.UAdd.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.UAdd.__le__" => "Return self<=value.", + "_ast.UAdd.__lt__" => "Return self<value.", + "_ast.UAdd.__ne__" => "Return self!=value.", + "_ast.UAdd.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.UAdd.__reduce_ex__" => "Helper for pickle.", + "_ast.UAdd.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.UAdd.__repr__" => "Return repr(self).", + "_ast.UAdd.__setattr__" => "Implement setattr(self, name, value).", + "_ast.UAdd.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.UAdd.__str__" => "Return str(self).", + "_ast.UAdd.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.UAdd.__weakref__" => "list of weak references to the object", + "_ast.USub" => "USub", + "_ast.USub.__delattr__" => "Implement delattr(self, name).", + "_ast.USub.__eq__" => "Return self==value.", + "_ast.USub.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.USub.__ge__" => "Return self>=value.", + "_ast.USub.__getattribute__" => "Return getattr(self, name).", + "_ast.USub.__getstate__" => "Helper for pickle.", + "_ast.USub.__gt__" => "Return self>value.", + "_ast.USub.__hash__" => "Return hash(self).", + "_ast.USub.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.USub.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.USub.__le__" => "Return self<=value.", + "_ast.USub.__lt__" => "Return self<value.", + "_ast.USub.__ne__" => "Return self!=value.", + "_ast.USub.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.USub.__reduce_ex__" => "Helper for pickle.", + "_ast.USub.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.USub.__repr__" => "Return repr(self).", + "_ast.USub.__setattr__" => "Implement setattr(self, name, value).", + "_ast.USub.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.USub.__str__" => "Return str(self).", + "_ast.USub.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.USub.__weakref__" => "list of weak references to the object", + "_ast.UnaryOp" => "UnaryOp(unaryop op, expr operand)", + "_ast.UnaryOp.__delattr__" => "Implement delattr(self, name).", + "_ast.UnaryOp.__eq__" => "Return self==value.", + "_ast.UnaryOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.UnaryOp.__ge__" => "Return self>=value.", + "_ast.UnaryOp.__getattribute__" => "Return getattr(self, name).", + "_ast.UnaryOp.__getstate__" => "Helper for pickle.", + "_ast.UnaryOp.__gt__" => "Return self>value.", + "_ast.UnaryOp.__hash__" => "Return hash(self).", + "_ast.UnaryOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.UnaryOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.UnaryOp.__le__" => "Return self<=value.", + "_ast.UnaryOp.__lt__" => "Return self<value.", + "_ast.UnaryOp.__ne__" => "Return self!=value.", + "_ast.UnaryOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.UnaryOp.__reduce_ex__" => "Helper for pickle.", + "_ast.UnaryOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.UnaryOp.__repr__" => "Return repr(self).", + "_ast.UnaryOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.UnaryOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.UnaryOp.__str__" => "Return str(self).", + "_ast.UnaryOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.UnaryOp.__weakref__" => "list of weak references to the object", + "_ast.While" => "While(expr test, stmt* body, stmt* orelse)", + "_ast.While.__delattr__" => "Implement delattr(self, name).", + "_ast.While.__eq__" => "Return self==value.", + "_ast.While.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.While.__ge__" => "Return self>=value.", + "_ast.While.__getattribute__" => "Return getattr(self, name).", + "_ast.While.__getstate__" => "Helper for pickle.", + "_ast.While.__gt__" => "Return self>value.", + "_ast.While.__hash__" => "Return hash(self).", + "_ast.While.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.While.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.While.__le__" => "Return self<=value.", + "_ast.While.__lt__" => "Return self<value.", + "_ast.While.__ne__" => "Return self!=value.", + "_ast.While.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.While.__reduce_ex__" => "Helper for pickle.", + "_ast.While.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.While.__repr__" => "Return repr(self).", + "_ast.While.__setattr__" => "Implement setattr(self, name, value).", + "_ast.While.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.While.__str__" => "Return str(self).", + "_ast.While.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.While.__weakref__" => "list of weak references to the object", + "_ast.With" => "With(withitem* items, stmt* body, string? type_comment)", + "_ast.With.__delattr__" => "Implement delattr(self, name).", + "_ast.With.__eq__" => "Return self==value.", + "_ast.With.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.With.__ge__" => "Return self>=value.", + "_ast.With.__getattribute__" => "Return getattr(self, name).", + "_ast.With.__getstate__" => "Helper for pickle.", + "_ast.With.__gt__" => "Return self>value.", + "_ast.With.__hash__" => "Return hash(self).", + "_ast.With.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.With.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.With.__le__" => "Return self<=value.", + "_ast.With.__lt__" => "Return self<value.", + "_ast.With.__ne__" => "Return self!=value.", + "_ast.With.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.With.__reduce_ex__" => "Helper for pickle.", + "_ast.With.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.With.__repr__" => "Return repr(self).", + "_ast.With.__setattr__" => "Implement setattr(self, name, value).", + "_ast.With.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.With.__str__" => "Return str(self).", + "_ast.With.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.With.__weakref__" => "list of weak references to the object", + "_ast.Yield" => "Yield(expr? value)", + "_ast.Yield.__delattr__" => "Implement delattr(self, name).", + "_ast.Yield.__eq__" => "Return self==value.", + "_ast.Yield.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Yield.__ge__" => "Return self>=value.", + "_ast.Yield.__getattribute__" => "Return getattr(self, name).", + "_ast.Yield.__getstate__" => "Helper for pickle.", + "_ast.Yield.__gt__" => "Return self>value.", + "_ast.Yield.__hash__" => "Return hash(self).", + "_ast.Yield.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Yield.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Yield.__le__" => "Return self<=value.", + "_ast.Yield.__lt__" => "Return self<value.", + "_ast.Yield.__ne__" => "Return self!=value.", + "_ast.Yield.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Yield.__reduce_ex__" => "Helper for pickle.", + "_ast.Yield.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Yield.__repr__" => "Return repr(self).", + "_ast.Yield.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Yield.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Yield.__str__" => "Return str(self).", + "_ast.Yield.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Yield.__weakref__" => "list of weak references to the object", + "_ast.YieldFrom" => "YieldFrom(expr value)", + "_ast.YieldFrom.__delattr__" => "Implement delattr(self, name).", + "_ast.YieldFrom.__eq__" => "Return self==value.", + "_ast.YieldFrom.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.YieldFrom.__ge__" => "Return self>=value.", + "_ast.YieldFrom.__getattribute__" => "Return getattr(self, name).", + "_ast.YieldFrom.__getstate__" => "Helper for pickle.", + "_ast.YieldFrom.__gt__" => "Return self>value.", + "_ast.YieldFrom.__hash__" => "Return hash(self).", + "_ast.YieldFrom.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.YieldFrom.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.YieldFrom.__le__" => "Return self<=value.", + "_ast.YieldFrom.__lt__" => "Return self<value.", + "_ast.YieldFrom.__ne__" => "Return self!=value.", + "_ast.YieldFrom.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.YieldFrom.__reduce_ex__" => "Helper for pickle.", + "_ast.YieldFrom.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.YieldFrom.__repr__" => "Return repr(self).", + "_ast.YieldFrom.__setattr__" => "Implement setattr(self, name, value).", + "_ast.YieldFrom.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.YieldFrom.__str__" => "Return str(self).", + "_ast.YieldFrom.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.YieldFrom.__weakref__" => "list of weak references to the object", + "_ast.alias" => "alias(identifier name, identifier? asname)", + "_ast.alias.__delattr__" => "Implement delattr(self, name).", + "_ast.alias.__eq__" => "Return self==value.", + "_ast.alias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.alias.__ge__" => "Return self>=value.", + "_ast.alias.__getattribute__" => "Return getattr(self, name).", + "_ast.alias.__getstate__" => "Helper for pickle.", + "_ast.alias.__gt__" => "Return self>value.", + "_ast.alias.__hash__" => "Return hash(self).", + "_ast.alias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.alias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.alias.__le__" => "Return self<=value.", + "_ast.alias.__lt__" => "Return self<value.", + "_ast.alias.__ne__" => "Return self!=value.", + "_ast.alias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.alias.__reduce_ex__" => "Helper for pickle.", + "_ast.alias.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.alias.__repr__" => "Return repr(self).", + "_ast.alias.__setattr__" => "Implement setattr(self, name, value).", + "_ast.alias.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.alias.__str__" => "Return str(self).", + "_ast.alias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.alias.__weakref__" => "list of weak references to the object", + "_ast.arg" => "arg(identifier arg, expr? annotation, string? type_comment)", + "_ast.arg.__delattr__" => "Implement delattr(self, name).", + "_ast.arg.__eq__" => "Return self==value.", + "_ast.arg.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.arg.__ge__" => "Return self>=value.", + "_ast.arg.__getattribute__" => "Return getattr(self, name).", + "_ast.arg.__getstate__" => "Helper for pickle.", + "_ast.arg.__gt__" => "Return self>value.", + "_ast.arg.__hash__" => "Return hash(self).", + "_ast.arg.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.arg.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.arg.__le__" => "Return self<=value.", + "_ast.arg.__lt__" => "Return self<value.", + "_ast.arg.__ne__" => "Return self!=value.", + "_ast.arg.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.arg.__reduce_ex__" => "Helper for pickle.", + "_ast.arg.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.arg.__repr__" => "Return repr(self).", + "_ast.arg.__setattr__" => "Implement setattr(self, name, value).", + "_ast.arg.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.arg.__str__" => "Return str(self).", + "_ast.arg.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.arg.__weakref__" => "list of weak references to the object", + "_ast.arguments" => "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr?* kw_defaults, arg? kwarg, expr* defaults)", + "_ast.arguments.__delattr__" => "Implement delattr(self, name).", + "_ast.arguments.__eq__" => "Return self==value.", + "_ast.arguments.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.arguments.__ge__" => "Return self>=value.", + "_ast.arguments.__getattribute__" => "Return getattr(self, name).", + "_ast.arguments.__getstate__" => "Helper for pickle.", + "_ast.arguments.__gt__" => "Return self>value.", + "_ast.arguments.__hash__" => "Return hash(self).", + "_ast.arguments.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.arguments.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.arguments.__le__" => "Return self<=value.", + "_ast.arguments.__lt__" => "Return self<value.", + "_ast.arguments.__ne__" => "Return self!=value.", + "_ast.arguments.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.arguments.__reduce_ex__" => "Helper for pickle.", + "_ast.arguments.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.arguments.__repr__" => "Return repr(self).", + "_ast.arguments.__setattr__" => "Implement setattr(self, name, value).", + "_ast.arguments.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.arguments.__str__" => "Return str(self).", + "_ast.arguments.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.arguments.__weakref__" => "list of weak references to the object", + "_ast.boolop" => "boolop = And | Or", + "_ast.boolop.__delattr__" => "Implement delattr(self, name).", + "_ast.boolop.__eq__" => "Return self==value.", + "_ast.boolop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.boolop.__ge__" => "Return self>=value.", + "_ast.boolop.__getattribute__" => "Return getattr(self, name).", + "_ast.boolop.__getstate__" => "Helper for pickle.", + "_ast.boolop.__gt__" => "Return self>value.", + "_ast.boolop.__hash__" => "Return hash(self).", + "_ast.boolop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.boolop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.boolop.__le__" => "Return self<=value.", + "_ast.boolop.__lt__" => "Return self<value.", + "_ast.boolop.__ne__" => "Return self!=value.", + "_ast.boolop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.boolop.__reduce_ex__" => "Helper for pickle.", + "_ast.boolop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.boolop.__repr__" => "Return repr(self).", + "_ast.boolop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.boolop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.boolop.__str__" => "Return str(self).", + "_ast.boolop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.boolop.__weakref__" => "list of weak references to the object", + "_ast.cmpop" => "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", + "_ast.cmpop.__delattr__" => "Implement delattr(self, name).", + "_ast.cmpop.__eq__" => "Return self==value.", + "_ast.cmpop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.cmpop.__ge__" => "Return self>=value.", + "_ast.cmpop.__getattribute__" => "Return getattr(self, name).", + "_ast.cmpop.__getstate__" => "Helper for pickle.", + "_ast.cmpop.__gt__" => "Return self>value.", + "_ast.cmpop.__hash__" => "Return hash(self).", + "_ast.cmpop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.cmpop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.cmpop.__le__" => "Return self<=value.", + "_ast.cmpop.__lt__" => "Return self<value.", + "_ast.cmpop.__ne__" => "Return self!=value.", + "_ast.cmpop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.cmpop.__reduce_ex__" => "Helper for pickle.", + "_ast.cmpop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.cmpop.__repr__" => "Return repr(self).", + "_ast.cmpop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.cmpop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.cmpop.__str__" => "Return str(self).", + "_ast.cmpop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.cmpop.__weakref__" => "list of weak references to the object", + "_ast.comprehension" => "comprehension(expr target, expr iter, expr* ifs, int is_async)", + "_ast.comprehension.__delattr__" => "Implement delattr(self, name).", + "_ast.comprehension.__eq__" => "Return self==value.", + "_ast.comprehension.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.comprehension.__ge__" => "Return self>=value.", + "_ast.comprehension.__getattribute__" => "Return getattr(self, name).", + "_ast.comprehension.__getstate__" => "Helper for pickle.", + "_ast.comprehension.__gt__" => "Return self>value.", + "_ast.comprehension.__hash__" => "Return hash(self).", + "_ast.comprehension.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.comprehension.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.comprehension.__le__" => "Return self<=value.", + "_ast.comprehension.__lt__" => "Return self<value.", + "_ast.comprehension.__ne__" => "Return self!=value.", + "_ast.comprehension.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.comprehension.__reduce_ex__" => "Helper for pickle.", + "_ast.comprehension.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.comprehension.__repr__" => "Return repr(self).", + "_ast.comprehension.__setattr__" => "Implement setattr(self, name, value).", + "_ast.comprehension.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.comprehension.__str__" => "Return str(self).", + "_ast.comprehension.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.comprehension.__weakref__" => "list of weak references to the object", + "_ast.excepthandler" => "excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)", + "_ast.excepthandler.__delattr__" => "Implement delattr(self, name).", + "_ast.excepthandler.__eq__" => "Return self==value.", + "_ast.excepthandler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.excepthandler.__ge__" => "Return self>=value.", + "_ast.excepthandler.__getattribute__" => "Return getattr(self, name).", + "_ast.excepthandler.__getstate__" => "Helper for pickle.", + "_ast.excepthandler.__gt__" => "Return self>value.", + "_ast.excepthandler.__hash__" => "Return hash(self).", + "_ast.excepthandler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.excepthandler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.excepthandler.__le__" => "Return self<=value.", + "_ast.excepthandler.__lt__" => "Return self<value.", + "_ast.excepthandler.__ne__" => "Return self!=value.", + "_ast.excepthandler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.excepthandler.__reduce_ex__" => "Helper for pickle.", + "_ast.excepthandler.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.excepthandler.__repr__" => "Return repr(self).", + "_ast.excepthandler.__setattr__" => "Implement setattr(self, name, value).", + "_ast.excepthandler.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.excepthandler.__str__" => "Return str(self).", + "_ast.excepthandler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.excepthandler.__weakref__" => "list of weak references to the object", + "_ast.expr" => "expr = BoolOp(boolop op, expr* values)\n | NamedExpr(expr target, expr value)\n | BinOp(expr left, operator op, expr right)\n | UnaryOp(unaryop op, expr operand)\n | Lambda(arguments args, expr body)\n | IfExp(expr test, expr body, expr orelse)\n | Dict(expr?* keys, expr* values)\n | Set(expr* elts)\n | ListComp(expr elt, comprehension* generators)\n | SetComp(expr elt, comprehension* generators)\n | DictComp(expr key, expr value, comprehension* generators)\n | GeneratorExp(expr elt, comprehension* generators)\n | Await(expr value)\n | Yield(expr? value)\n | YieldFrom(expr value)\n | Compare(expr left, cmpop* ops, expr* comparators)\n | Call(expr func, expr* args, keyword* keywords)\n | FormattedValue(expr value, int conversion, expr? format_spec)\n | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n | JoinedStr(expr* values)\n | TemplateStr(expr* values)\n | Constant(constant value, string? kind)\n | Attribute(expr value, identifier attr, expr_context ctx)\n | Subscript(expr value, expr slice, expr_context ctx)\n | Starred(expr value, expr_context ctx)\n | Name(identifier id, expr_context ctx)\n | List(expr* elts, expr_context ctx)\n | Tuple(expr* elts, expr_context ctx)\n | Slice(expr? lower, expr? upper, expr? step)", + "_ast.expr.__delattr__" => "Implement delattr(self, name).", + "_ast.expr.__eq__" => "Return self==value.", + "_ast.expr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.expr.__ge__" => "Return self>=value.", + "_ast.expr.__getattribute__" => "Return getattr(self, name).", + "_ast.expr.__getstate__" => "Helper for pickle.", + "_ast.expr.__gt__" => "Return self>value.", + "_ast.expr.__hash__" => "Return hash(self).", + "_ast.expr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.expr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.expr.__le__" => "Return self<=value.", + "_ast.expr.__lt__" => "Return self<value.", + "_ast.expr.__ne__" => "Return self!=value.", + "_ast.expr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.expr.__reduce_ex__" => "Helper for pickle.", + "_ast.expr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.expr.__repr__" => "Return repr(self).", + "_ast.expr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.expr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.expr.__str__" => "Return str(self).", + "_ast.expr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.expr.__weakref__" => "list of weak references to the object", + "_ast.expr_context" => "expr_context = Load | Store | Del", + "_ast.expr_context.__delattr__" => "Implement delattr(self, name).", + "_ast.expr_context.__eq__" => "Return self==value.", + "_ast.expr_context.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.expr_context.__ge__" => "Return self>=value.", + "_ast.expr_context.__getattribute__" => "Return getattr(self, name).", + "_ast.expr_context.__getstate__" => "Helper for pickle.", + "_ast.expr_context.__gt__" => "Return self>value.", + "_ast.expr_context.__hash__" => "Return hash(self).", + "_ast.expr_context.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.expr_context.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.expr_context.__le__" => "Return self<=value.", + "_ast.expr_context.__lt__" => "Return self<value.", + "_ast.expr_context.__ne__" => "Return self!=value.", + "_ast.expr_context.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.expr_context.__reduce_ex__" => "Helper for pickle.", + "_ast.expr_context.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.expr_context.__repr__" => "Return repr(self).", + "_ast.expr_context.__setattr__" => "Implement setattr(self, name, value).", + "_ast.expr_context.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.expr_context.__str__" => "Return str(self).", + "_ast.expr_context.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.expr_context.__weakref__" => "list of weak references to the object", + "_ast.keyword" => "keyword(identifier? arg, expr value)", + "_ast.keyword.__delattr__" => "Implement delattr(self, name).", + "_ast.keyword.__eq__" => "Return self==value.", + "_ast.keyword.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.keyword.__ge__" => "Return self>=value.", + "_ast.keyword.__getattribute__" => "Return getattr(self, name).", + "_ast.keyword.__getstate__" => "Helper for pickle.", + "_ast.keyword.__gt__" => "Return self>value.", + "_ast.keyword.__hash__" => "Return hash(self).", + "_ast.keyword.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.keyword.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.keyword.__le__" => "Return self<=value.", + "_ast.keyword.__lt__" => "Return self<value.", + "_ast.keyword.__ne__" => "Return self!=value.", + "_ast.keyword.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.keyword.__reduce_ex__" => "Helper for pickle.", + "_ast.keyword.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.keyword.__repr__" => "Return repr(self).", + "_ast.keyword.__setattr__" => "Implement setattr(self, name, value).", + "_ast.keyword.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.keyword.__str__" => "Return str(self).", + "_ast.keyword.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.keyword.__weakref__" => "list of weak references to the object", + "_ast.match_case" => "match_case(pattern pattern, expr? guard, stmt* body)", + "_ast.match_case.__delattr__" => "Implement delattr(self, name).", + "_ast.match_case.__eq__" => "Return self==value.", + "_ast.match_case.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.match_case.__ge__" => "Return self>=value.", + "_ast.match_case.__getattribute__" => "Return getattr(self, name).", + "_ast.match_case.__getstate__" => "Helper for pickle.", + "_ast.match_case.__gt__" => "Return self>value.", + "_ast.match_case.__hash__" => "Return hash(self).", + "_ast.match_case.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.match_case.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.match_case.__le__" => "Return self<=value.", + "_ast.match_case.__lt__" => "Return self<value.", + "_ast.match_case.__ne__" => "Return self!=value.", + "_ast.match_case.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.match_case.__reduce_ex__" => "Helper for pickle.", + "_ast.match_case.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.match_case.__repr__" => "Return repr(self).", + "_ast.match_case.__setattr__" => "Implement setattr(self, name, value).", + "_ast.match_case.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.match_case.__str__" => "Return str(self).", + "_ast.match_case.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.match_case.__weakref__" => "list of weak references to the object", + "_ast.mod" => "mod = Module(stmt* body, type_ignore* type_ignores)\n | Interactive(stmt* body)\n | Expression(expr body)\n | FunctionType(expr* argtypes, expr returns)", + "_ast.mod.__delattr__" => "Implement delattr(self, name).", + "_ast.mod.__eq__" => "Return self==value.", + "_ast.mod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.mod.__ge__" => "Return self>=value.", + "_ast.mod.__getattribute__" => "Return getattr(self, name).", + "_ast.mod.__getstate__" => "Helper for pickle.", + "_ast.mod.__gt__" => "Return self>value.", + "_ast.mod.__hash__" => "Return hash(self).", + "_ast.mod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.mod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.mod.__le__" => "Return self<=value.", + "_ast.mod.__lt__" => "Return self<value.", + "_ast.mod.__ne__" => "Return self!=value.", + "_ast.mod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.mod.__reduce_ex__" => "Helper for pickle.", + "_ast.mod.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.mod.__repr__" => "Return repr(self).", + "_ast.mod.__setattr__" => "Implement setattr(self, name, value).", + "_ast.mod.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.mod.__str__" => "Return str(self).", + "_ast.mod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.mod.__weakref__" => "list of weak references to the object", + "_ast.operator" => "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv", + "_ast.operator.__delattr__" => "Implement delattr(self, name).", + "_ast.operator.__eq__" => "Return self==value.", + "_ast.operator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.operator.__ge__" => "Return self>=value.", + "_ast.operator.__getattribute__" => "Return getattr(self, name).", + "_ast.operator.__getstate__" => "Helper for pickle.", + "_ast.operator.__gt__" => "Return self>value.", + "_ast.operator.__hash__" => "Return hash(self).", + "_ast.operator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.operator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.operator.__le__" => "Return self<=value.", + "_ast.operator.__lt__" => "Return self<value.", + "_ast.operator.__ne__" => "Return self!=value.", + "_ast.operator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.operator.__reduce_ex__" => "Helper for pickle.", + "_ast.operator.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.operator.__repr__" => "Return repr(self).", + "_ast.operator.__setattr__" => "Implement setattr(self, name, value).", + "_ast.operator.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.operator.__str__" => "Return str(self).", + "_ast.operator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.operator.__weakref__" => "list of weak references to the object", + "_ast.pattern" => "pattern = MatchValue(expr value)\n | MatchSingleton(constant value)\n | MatchSequence(pattern* patterns)\n | MatchMapping(expr* keys, pattern* patterns, identifier? rest)\n | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n | MatchStar(identifier? name)\n | MatchAs(pattern? pattern, identifier? name)\n | MatchOr(pattern* patterns)", + "_ast.pattern.__delattr__" => "Implement delattr(self, name).", + "_ast.pattern.__eq__" => "Return self==value.", + "_ast.pattern.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.pattern.__ge__" => "Return self>=value.", + "_ast.pattern.__getattribute__" => "Return getattr(self, name).", + "_ast.pattern.__getstate__" => "Helper for pickle.", + "_ast.pattern.__gt__" => "Return self>value.", + "_ast.pattern.__hash__" => "Return hash(self).", + "_ast.pattern.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.pattern.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.pattern.__le__" => "Return self<=value.", + "_ast.pattern.__lt__" => "Return self<value.", + "_ast.pattern.__ne__" => "Return self!=value.", + "_ast.pattern.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.pattern.__reduce_ex__" => "Helper for pickle.", + "_ast.pattern.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.pattern.__repr__" => "Return repr(self).", + "_ast.pattern.__setattr__" => "Implement setattr(self, name, value).", + "_ast.pattern.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.pattern.__str__" => "Return str(self).", + "_ast.pattern.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.pattern.__weakref__" => "list of weak references to the object", + "_ast.stmt" => "stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)\n | Return(expr? value)\n | Delete(expr* targets)\n | Assign(expr* targets, expr value, string? type_comment)\n | TypeAlias(expr name, type_param* type_params, expr value)\n | AugAssign(expr target, operator op, expr value)\n | AnnAssign(expr target, expr annotation, expr? value, int simple)\n | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n | While(expr test, stmt* body, stmt* orelse)\n | If(expr test, stmt* body, stmt* orelse)\n | With(withitem* items, stmt* body, string? type_comment)\n | AsyncWith(withitem* items, stmt* body, string? type_comment)\n | Match(expr subject, match_case* cases)\n | Raise(expr? exc, expr? cause)\n | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n | Assert(expr test, expr? msg)\n | Import(alias* names)\n | ImportFrom(identifier? module, alias* names, int? level)\n | Global(identifier* names)\n | Nonlocal(identifier* names)\n | Expr(expr value)\n | Pass\n | Break\n | Continue", + "_ast.stmt.__delattr__" => "Implement delattr(self, name).", + "_ast.stmt.__eq__" => "Return self==value.", + "_ast.stmt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.stmt.__ge__" => "Return self>=value.", + "_ast.stmt.__getattribute__" => "Return getattr(self, name).", + "_ast.stmt.__getstate__" => "Helper for pickle.", + "_ast.stmt.__gt__" => "Return self>value.", + "_ast.stmt.__hash__" => "Return hash(self).", + "_ast.stmt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.stmt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.stmt.__le__" => "Return self<=value.", + "_ast.stmt.__lt__" => "Return self<value.", + "_ast.stmt.__ne__" => "Return self!=value.", + "_ast.stmt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.stmt.__reduce_ex__" => "Helper for pickle.", + "_ast.stmt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.stmt.__repr__" => "Return repr(self).", + "_ast.stmt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.stmt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.stmt.__str__" => "Return str(self).", + "_ast.stmt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.stmt.__weakref__" => "list of weak references to the object", + "_ast.type_ignore" => "type_ignore = TypeIgnore(int lineno, string tag)", + "_ast.type_ignore.__delattr__" => "Implement delattr(self, name).", + "_ast.type_ignore.__eq__" => "Return self==value.", + "_ast.type_ignore.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.type_ignore.__ge__" => "Return self>=value.", + "_ast.type_ignore.__getattribute__" => "Return getattr(self, name).", + "_ast.type_ignore.__getstate__" => "Helper for pickle.", + "_ast.type_ignore.__gt__" => "Return self>value.", + "_ast.type_ignore.__hash__" => "Return hash(self).", + "_ast.type_ignore.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.type_ignore.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.type_ignore.__le__" => "Return self<=value.", + "_ast.type_ignore.__lt__" => "Return self<value.", + "_ast.type_ignore.__ne__" => "Return self!=value.", + "_ast.type_ignore.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.type_ignore.__reduce_ex__" => "Helper for pickle.", + "_ast.type_ignore.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.type_ignore.__repr__" => "Return repr(self).", + "_ast.type_ignore.__setattr__" => "Implement setattr(self, name, value).", + "_ast.type_ignore.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.type_ignore.__str__" => "Return str(self).", + "_ast.type_ignore.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.type_ignore.__weakref__" => "list of weak references to the object", + "_ast.type_param" => "type_param = TypeVar(identifier name, expr? bound, expr? default_value)\n | ParamSpec(identifier name, expr? default_value)\n | TypeVarTuple(identifier name, expr? default_value)", + "_ast.type_param.__delattr__" => "Implement delattr(self, name).", + "_ast.type_param.__eq__" => "Return self==value.", + "_ast.type_param.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.type_param.__ge__" => "Return self>=value.", + "_ast.type_param.__getattribute__" => "Return getattr(self, name).", + "_ast.type_param.__getstate__" => "Helper for pickle.", + "_ast.type_param.__gt__" => "Return self>value.", + "_ast.type_param.__hash__" => "Return hash(self).", + "_ast.type_param.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.type_param.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.type_param.__le__" => "Return self<=value.", + "_ast.type_param.__lt__" => "Return self<value.", + "_ast.type_param.__ne__" => "Return self!=value.", + "_ast.type_param.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.type_param.__reduce_ex__" => "Helper for pickle.", + "_ast.type_param.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.type_param.__repr__" => "Return repr(self).", + "_ast.type_param.__setattr__" => "Implement setattr(self, name, value).", + "_ast.type_param.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.type_param.__str__" => "Return str(self).", + "_ast.type_param.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.type_param.__weakref__" => "list of weak references to the object", + "_ast.unaryop" => "unaryop = Invert | Not | UAdd | USub", + "_ast.unaryop.__delattr__" => "Implement delattr(self, name).", + "_ast.unaryop.__eq__" => "Return self==value.", + "_ast.unaryop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.unaryop.__ge__" => "Return self>=value.", + "_ast.unaryop.__getattribute__" => "Return getattr(self, name).", + "_ast.unaryop.__getstate__" => "Helper for pickle.", + "_ast.unaryop.__gt__" => "Return self>value.", + "_ast.unaryop.__hash__" => "Return hash(self).", + "_ast.unaryop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.unaryop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.unaryop.__le__" => "Return self<=value.", + "_ast.unaryop.__lt__" => "Return self<value.", + "_ast.unaryop.__ne__" => "Return self!=value.", + "_ast.unaryop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.unaryop.__reduce_ex__" => "Helper for pickle.", + "_ast.unaryop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.unaryop.__repr__" => "Return repr(self).", + "_ast.unaryop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.unaryop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.unaryop.__str__" => "Return str(self).", + "_ast.unaryop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.unaryop.__weakref__" => "list of weak references to the object", + "_ast.withitem" => "withitem(expr context_expr, expr? optional_vars)", + "_ast.withitem.__delattr__" => "Implement delattr(self, name).", + "_ast.withitem.__eq__" => "Return self==value.", + "_ast.withitem.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.withitem.__ge__" => "Return self>=value.", + "_ast.withitem.__getattribute__" => "Return getattr(self, name).", + "_ast.withitem.__getstate__" => "Helper for pickle.", + "_ast.withitem.__gt__" => "Return self>value.", + "_ast.withitem.__hash__" => "Return hash(self).", + "_ast.withitem.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.withitem.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.withitem.__le__" => "Return self<=value.", + "_ast.withitem.__lt__" => "Return self<value.", + "_ast.withitem.__ne__" => "Return self!=value.", + "_ast.withitem.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.withitem.__reduce_ex__" => "Helper for pickle.", + "_ast.withitem.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.withitem.__repr__" => "Return repr(self).", + "_ast.withitem.__setattr__" => "Implement setattr(self, name, value).", + "_ast.withitem.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.withitem.__str__" => "Return str(self).", + "_ast.withitem.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.withitem.__weakref__" => "list of weak references to the object", "_asyncio" => "Accelerator module for asyncio", - "_asyncio.Future" => "This class is *almost* compatible with concurrent.futures.Future.\n\nDifferences:\n\n- result() and exception() do not take a timeout argument and\n raise an exception when the future isn't done yet.\n\n- Callbacks registered with add_done_callback() are always called\n via the event loop's call_soon_threadsafe().\n\n- This class is not compatible with the wait() and as_completed()\n methods in the concurrent.futures package.", + "_asyncio.Future" => "This class is *almost* compatible with concurrent.futures.Future.\n\n Differences:\n\n - result() and exception() do not take a timeout argument and\n raise an exception when the future isn't done yet.\n\n - Callbacks registered with add_done_callback() are always called\n via the event loop's call_soon_threadsafe().\n\n - This class is not compatible with the wait() and as_completed()\n methods in the concurrent.futures package.", "_asyncio.Future.__await__" => "Return an iterator to be used in await expression.", "_asyncio.Future.__class_getitem__" => "See PEP 585", "_asyncio.Future.__del__" => "Called when the instance is about to be destroyed.", @@ -98,7 +2995,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_asyncio._swap_current_task" => "Temporarily swap in the supplied task and return the original one (or None).\n\nThis is intended for use during eager coroutine execution.", "_asyncio._unregister_eager_task" => "Unregister a task.\n\nReturns None.", "_asyncio._unregister_task" => "Unregister a task.\n\nReturns None.", + "_asyncio.all_tasks" => "Return a set of all tasks for the loop.", "_asyncio.current_task" => "Return a currently executed task.", + "_asyncio.future_add_to_awaited_by" => "Record that `fut` is awaited on by `waiter`.", "_asyncio.get_event_loop" => "Return an asyncio event loop.\n\nWhen called from a coroutine or a callback (e.g. scheduled with\ncall_soon or similar API), this function will always return the\nrunning event loop.\n\nIf there is no running event loop set, the function will return\nthe result of `get_event_loop_policy().get_event_loop()` call.", "_asyncio.get_running_loop" => "Return the running event loop. Raise a RuntimeError if there is none.\n\nThis function is thread-specific.", "_bisect" => "Bisection algorithms.\n\nThis module provides support for maintaining a list in sorted order without\nhaving to sort the list after each insertion. For long lists of items with\nexpensive comparison operations, this can be an improvement over the more\ncommon approach.", @@ -209,6 +3108,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_bz2.BZ2Decompressor.eof" => "True if the end-of-stream marker has been reached.", "_bz2.BZ2Decompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", "_bz2.BZ2Decompressor.unused_data" => "Data found after the end of the compressed stream.", + "_codecs._unregister_error" => "Un-register the specified error handler for the error handling `errors'.\n\nOnly custom error handlers can be un-registered. An exception is raised\nif the error handling is a built-in one (e.g., 'strict'), or if an error\noccurs.\n\nOtherwise, this returns True if a custom handler has been successfully\nun-registered, and False if no custom handler for the specified error\nhandling exists.", "_codecs.decode" => "Decodes obj using the codec registered for encoding.\n\nDefault encoding is 'utf-8'. errors may be given to set a\ndifferent error handling scheme. Default is 'strict' meaning that encoding\nerrors raise a ValueError. Other possible values are 'ignore', 'replace'\nand 'backslashreplace' as well as any other name registered with\ncodecs.register_error that can handle ValueErrors.", "_codecs.encode" => "Encodes obj using the codec registered for encoding.\n\nThe default encoding is 'utf-8'. errors may be given to set a\ndifferent error handling scheme. Default is 'strict' meaning that encoding\nerrors raise a ValueError. Other possible values are 'ignore', 'replace'\nand 'backslashreplace' as well as any other name registered with\ncodecs.register_error that can handle ValueErrors.", "_codecs.lookup" => "Looks up a codec tuple in the Python codec registry and returns a CodecInfo object.", @@ -217,7 +3117,213 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_codecs.register_error" => "Register the specified error handler under the name errors.\n\nhandler must be a callable object, that will be called with an exception\ninstance containing information about the location of the encoding/decoding\nerror and must return a (replacement, new position) tuple.", "_codecs.unregister" => "Unregister a codec search function and clear the registry's cache.\n\nIf the search function is not registered, do nothing.", "_collections" => "High performance data structures.\n- deque: ordered collection accessible from endpoints only\n- defaultdict: dict subclass with a default value factory", + "_collections.OrderedDict" => "Dictionary that remembers insertion order", + "_collections.OrderedDict.__class_getitem__" => "See PEP 585", + "_collections.OrderedDict.__contains__" => "True if the dictionary has the specified key, else False.", + "_collections.OrderedDict.__delattr__" => "Implement delattr(self, name).", + "_collections.OrderedDict.__delitem__" => "Delete self[key].", + "_collections.OrderedDict.__eq__" => "Return self==value.", + "_collections.OrderedDict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.OrderedDict.__ge__" => "Return self>=value.", + "_collections.OrderedDict.__getattribute__" => "Return getattr(self, name).", + "_collections.OrderedDict.__getitem__" => "Return self[key].", + "_collections.OrderedDict.__getstate__" => "Helper for pickle.", + "_collections.OrderedDict.__gt__" => "Return self>value.", + "_collections.OrderedDict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.OrderedDict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.OrderedDict.__ior__" => "Return self|=value.", + "_collections.OrderedDict.__iter__" => "Implement iter(self).", + "_collections.OrderedDict.__le__" => "Return self<=value.", + "_collections.OrderedDict.__len__" => "Return len(self).", + "_collections.OrderedDict.__lt__" => "Return self<value.", + "_collections.OrderedDict.__ne__" => "Return self!=value.", + "_collections.OrderedDict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.OrderedDict.__or__" => "Return self|value.", + "_collections.OrderedDict.__reduce__" => "Return state information for pickling", + "_collections.OrderedDict.__reduce_ex__" => "Helper for pickle.", + "_collections.OrderedDict.__repr__" => "Return repr(self).", + "_collections.OrderedDict.__reversed__" => "od.__reversed__() <==> reversed(od)", + "_collections.OrderedDict.__ror__" => "Return value|self.", + "_collections.OrderedDict.__setattr__" => "Implement setattr(self, name, value).", + "_collections.OrderedDict.__setitem__" => "Set self[key] to value.", + "_collections.OrderedDict.__str__" => "Return str(self).", + "_collections.OrderedDict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.OrderedDict.clear" => "Remove all items from ordered dict.", + "_collections.OrderedDict.copy" => "A shallow copy of ordered dict.", + "_collections.OrderedDict.fromkeys" => "Create a new ordered dictionary with keys from iterable and values set to value.", + "_collections.OrderedDict.get" => "Return the value for key if key is in the dictionary, else default.", + "_collections.OrderedDict.move_to_end" => "Move an existing element to the end (or beginning if last is false).\n\nRaise KeyError if the element does not exist.", + "_collections.OrderedDict.pop" => "od.pop(key[,default]) -> v, remove specified key and return the corresponding value.\n\nIf the key is not found, return the default if given; otherwise,\nraise a KeyError.", + "_collections.OrderedDict.popitem" => "Remove and return a (key, value) pair from the dictionary.\n\nPairs are returned in LIFO order if last is true or FIFO order if false.", + "_collections.OrderedDict.setdefault" => "Insert key with a value of default if key is not in the dictionary.\n\nReturn the value for key if key is in the dictionary, else default.", "_collections._count_elements" => "Count elements in the iterable, updating the mapping", + "_collections._deque_iterator.__delattr__" => "Implement delattr(self, name).", + "_collections._deque_iterator.__eq__" => "Return self==value.", + "_collections._deque_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._deque_iterator.__ge__" => "Return self>=value.", + "_collections._deque_iterator.__getattribute__" => "Return getattr(self, name).", + "_collections._deque_iterator.__getstate__" => "Helper for pickle.", + "_collections._deque_iterator.__gt__" => "Return self>value.", + "_collections._deque_iterator.__hash__" => "Return hash(self).", + "_collections._deque_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._deque_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._deque_iterator.__iter__" => "Implement iter(self).", + "_collections._deque_iterator.__le__" => "Return self<=value.", + "_collections._deque_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "_collections._deque_iterator.__lt__" => "Return self<value.", + "_collections._deque_iterator.__ne__" => "Return self!=value.", + "_collections._deque_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._deque_iterator.__next__" => "Implement next(self).", + "_collections._deque_iterator.__reduce__" => "Return state information for pickling.", + "_collections._deque_iterator.__reduce_ex__" => "Helper for pickle.", + "_collections._deque_iterator.__repr__" => "Return repr(self).", + "_collections._deque_iterator.__setattr__" => "Implement setattr(self, name, value).", + "_collections._deque_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._deque_iterator.__str__" => "Return str(self).", + "_collections._deque_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections._deque_reverse_iterator.__delattr__" => "Implement delattr(self, name).", + "_collections._deque_reverse_iterator.__eq__" => "Return self==value.", + "_collections._deque_reverse_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._deque_reverse_iterator.__ge__" => "Return self>=value.", + "_collections._deque_reverse_iterator.__getattribute__" => "Return getattr(self, name).", + "_collections._deque_reverse_iterator.__getstate__" => "Helper for pickle.", + "_collections._deque_reverse_iterator.__gt__" => "Return self>value.", + "_collections._deque_reverse_iterator.__hash__" => "Return hash(self).", + "_collections._deque_reverse_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._deque_reverse_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._deque_reverse_iterator.__iter__" => "Implement iter(self).", + "_collections._deque_reverse_iterator.__le__" => "Return self<=value.", + "_collections._deque_reverse_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "_collections._deque_reverse_iterator.__lt__" => "Return self<value.", + "_collections._deque_reverse_iterator.__ne__" => "Return self!=value.", + "_collections._deque_reverse_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._deque_reverse_iterator.__next__" => "Implement next(self).", + "_collections._deque_reverse_iterator.__reduce__" => "Return state information for pickling.", + "_collections._deque_reverse_iterator.__reduce_ex__" => "Helper for pickle.", + "_collections._deque_reverse_iterator.__repr__" => "Return repr(self).", + "_collections._deque_reverse_iterator.__setattr__" => "Implement setattr(self, name, value).", + "_collections._deque_reverse_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._deque_reverse_iterator.__str__" => "Return str(self).", + "_collections._deque_reverse_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections._tuplegetter.__delattr__" => "Implement delattr(self, name).", + "_collections._tuplegetter.__delete__" => "Delete an attribute of instance.", + "_collections._tuplegetter.__eq__" => "Return self==value.", + "_collections._tuplegetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._tuplegetter.__ge__" => "Return self>=value.", + "_collections._tuplegetter.__get__" => "Return an attribute of instance, which is of type owner.", + "_collections._tuplegetter.__getattribute__" => "Return getattr(self, name).", + "_collections._tuplegetter.__getstate__" => "Helper for pickle.", + "_collections._tuplegetter.__gt__" => "Return self>value.", + "_collections._tuplegetter.__hash__" => "Return hash(self).", + "_collections._tuplegetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._tuplegetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._tuplegetter.__le__" => "Return self<=value.", + "_collections._tuplegetter.__lt__" => "Return self<value.", + "_collections._tuplegetter.__ne__" => "Return self!=value.", + "_collections._tuplegetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._tuplegetter.__reduce_ex__" => "Helper for pickle.", + "_collections._tuplegetter.__repr__" => "Return repr(self).", + "_collections._tuplegetter.__set__" => "Set an attribute of instance to value.", + "_collections._tuplegetter.__setattr__" => "Implement setattr(self, name, value).", + "_collections._tuplegetter.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._tuplegetter.__str__" => "Return str(self).", + "_collections._tuplegetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.defaultdict" => "defaultdict(default_factory=None, /, [...]) --> dict with default factory\n\nThe default factory is called without arguments to produce\na new value when a key is not present, in __getitem__ only.\nA defaultdict compares equal to a dict with the same items.\nAll remaining arguments are treated the same as if they were\npassed to the dict constructor, including keyword arguments.", + "_collections.defaultdict.__class_getitem__" => "See PEP 585", + "_collections.defaultdict.__contains__" => "True if the dictionary has the specified key, else False.", + "_collections.defaultdict.__copy__" => "D.copy() -> a shallow copy of D.", + "_collections.defaultdict.__delattr__" => "Implement delattr(self, name).", + "_collections.defaultdict.__delitem__" => "Delete self[key].", + "_collections.defaultdict.__eq__" => "Return self==value.", + "_collections.defaultdict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.defaultdict.__ge__" => "Return self>=value.", + "_collections.defaultdict.__getattribute__" => "Return getattr(self, name).", + "_collections.defaultdict.__getitem__" => "Return self[key].", + "_collections.defaultdict.__getstate__" => "Helper for pickle.", + "_collections.defaultdict.__gt__" => "Return self>value.", + "_collections.defaultdict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.defaultdict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.defaultdict.__ior__" => "Return self|=value.", + "_collections.defaultdict.__iter__" => "Implement iter(self).", + "_collections.defaultdict.__le__" => "Return self<=value.", + "_collections.defaultdict.__len__" => "Return len(self).", + "_collections.defaultdict.__lt__" => "Return self<value.", + "_collections.defaultdict.__missing__" => "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n if self.default_factory is None: raise KeyError((key,))\n self[key] = value = self.default_factory()\n return value", + "_collections.defaultdict.__ne__" => "Return self!=value.", + "_collections.defaultdict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.defaultdict.__or__" => "Return self|value.", + "_collections.defaultdict.__reduce__" => "Return state information for pickling.", + "_collections.defaultdict.__reduce_ex__" => "Helper for pickle.", + "_collections.defaultdict.__repr__" => "Return repr(self).", + "_collections.defaultdict.__reversed__" => "Return a reverse iterator over the dict keys.", + "_collections.defaultdict.__ror__" => "Return value|self.", + "_collections.defaultdict.__setattr__" => "Implement setattr(self, name, value).", + "_collections.defaultdict.__setitem__" => "Set self[key] to value.", + "_collections.defaultdict.__sizeof__" => "Return the size of the dict in memory, in bytes.", + "_collections.defaultdict.__str__" => "Return str(self).", + "_collections.defaultdict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.defaultdict.clear" => "Remove all items from the dict.", + "_collections.defaultdict.copy" => "D.copy() -> a shallow copy of D.", + "_collections.defaultdict.default_factory" => "Factory for default value called by __missing__().", + "_collections.defaultdict.fromkeys" => "Create a new dictionary with keys from iterable and values set to value.", + "_collections.defaultdict.get" => "Return the value for key if key is in the dictionary, else default.", + "_collections.defaultdict.items" => "Return a set-like object providing a view on the dict's items.", + "_collections.defaultdict.keys" => "Return a set-like object providing a view on the dict's keys.", + "_collections.defaultdict.pop" => "D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n\nIf the key is not found, return the default if given; otherwise,\nraise a KeyError.", + "_collections.defaultdict.popitem" => "Remove and return a (key, value) pair as a 2-tuple.\n\nPairs are returned in LIFO (last-in, first-out) order.\nRaises KeyError if the dict is empty.", + "_collections.defaultdict.setdefault" => "Insert key with a value of default if key is not in the dictionary.\n\nReturn the value for key if key is in the dictionary, else default.", + "_collections.defaultdict.update" => "D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\nIf E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\nIf E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\nIn either case, this is followed by: for k in F: D[k] = F[k]", + "_collections.defaultdict.values" => "Return an object providing a view on the dict's values.", + "_collections.deque" => "A list-like sequence optimized for data accesses near its endpoints.", + "_collections.deque.__add__" => "Return self+value.", + "_collections.deque.__class_getitem__" => "See PEP 585", + "_collections.deque.__contains__" => "Return bool(key in self).", + "_collections.deque.__copy__" => "Return a shallow copy of a deque.", + "_collections.deque.__delattr__" => "Implement delattr(self, name).", + "_collections.deque.__delitem__" => "Delete self[key].", + "_collections.deque.__eq__" => "Return self==value.", + "_collections.deque.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.deque.__ge__" => "Return self>=value.", + "_collections.deque.__getattribute__" => "Return getattr(self, name).", + "_collections.deque.__getitem__" => "Return self[key].", + "_collections.deque.__getstate__" => "Helper for pickle.", + "_collections.deque.__gt__" => "Return self>value.", + "_collections.deque.__iadd__" => "Implement self+=value.", + "_collections.deque.__imul__" => "Implement self*=value.", + "_collections.deque.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.deque.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.deque.__iter__" => "Implement iter(self).", + "_collections.deque.__le__" => "Return self<=value.", + "_collections.deque.__len__" => "Return len(self).", + "_collections.deque.__lt__" => "Return self<value.", + "_collections.deque.__mul__" => "Return self*value.", + "_collections.deque.__ne__" => "Return self!=value.", + "_collections.deque.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.deque.__reduce__" => "Return state information for pickling.", + "_collections.deque.__reduce_ex__" => "Helper for pickle.", + "_collections.deque.__repr__" => "Return repr(self).", + "_collections.deque.__reversed__" => "Return a reverse iterator over the deque.", + "_collections.deque.__rmul__" => "Return value*self.", + "_collections.deque.__setattr__" => "Implement setattr(self, name, value).", + "_collections.deque.__setitem__" => "Set self[key] to value.", + "_collections.deque.__sizeof__" => "Return the size of the deque in memory, in bytes.", + "_collections.deque.__str__" => "Return str(self).", + "_collections.deque.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.deque.append" => "Add an element to the right side of the deque.", + "_collections.deque.appendleft" => "Add an element to the left side of the deque.", + "_collections.deque.clear" => "Remove all elements from the deque.", + "_collections.deque.copy" => "Return a shallow copy of a deque.", + "_collections.deque.count" => "Return number of occurrences of value.", + "_collections.deque.extend" => "Extend the right side of the deque with elements from the iterable.", + "_collections.deque.extendleft" => "Extend the left side of the deque with elements from the iterable.", + "_collections.deque.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_collections.deque.insert" => "Insert value before index.", + "_collections.deque.maxlen" => "maximum size of a deque or None if unbounded", + "_collections.deque.pop" => "Remove and return the rightmost element.", + "_collections.deque.popleft" => "Remove and return the leftmost element.", + "_collections.deque.remove" => "Remove first occurrence of value.", + "_collections.deque.reverse" => "Reverse *IN PLACE*.", + "_collections.deque.rotate" => "Rotate the deque n steps to the right. If n is negative, rotates left.", "_contextvars" => "Context Variables", "_contextvars.Context.__contains__" => "Return bool(key in self).", "_contextvars.Context.__delattr__" => "Implement delattr(self, name).", @@ -275,7 +3381,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_contextvars.ContextVar.set" => "Call to set a new value for the context variable in the current context.\n\nThe required value argument is the new value for the context variable.\n\nReturns a Token object that can be used to restore the variable to its previous\nvalue via the `ContextVar.reset()` method.", "_contextvars.Token.__class_getitem__" => "See PEP 585", "_contextvars.Token.__delattr__" => "Implement delattr(self, name).", + "_contextvars.Token.__enter__" => "Enter into Token context manager.", "_contextvars.Token.__eq__" => "Return self==value.", + "_contextvars.Token.__exit__" => "Exit from Token context manager, restore the linked ContextVar.", "_contextvars.Token.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", "_contextvars.Token.__ge__" => "Return self>=value.", "_contextvars.Token.__getattribute__" => "Return getattr(self, name).", @@ -317,8 +3425,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_csv.Dialect.__sizeof__" => "Size of object in memory, in bytes.", "_csv.Dialect.__str__" => "Return str(self).", "_csv.Dialect.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_csv.Error.__cause__" => "exception cause", - "_csv.Error.__context__" => "exception context", "_csv.Error.__delattr__" => "Implement delattr(self, name).", "_csv.Error.__eq__" => "Return self==value.", "_csv.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -339,8 +3445,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_csv.Error.__sizeof__" => "Size of object in memory, in bytes.", "_csv.Error.__str__" => "Return str(self).", "_csv.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_csv.Error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_csv.Error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_csv.Error.add_note" => "Add a note to the exception", + "_csv.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_csv.Reader" => "CSV reader\n\nReader objects are responsible for reading and parsing tabular data\nin CSV format.", "_csv.Reader.__delattr__" => "Implement delattr(self, name).", "_csv.Reader.__eq__" => "Return self==value.", @@ -397,11 +3503,67 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_csv.unregister_dialect" => "Delete the name/dialect mapping associated with a string name.", "_csv.writer" => "Return a writer object that will write user data on the given file object.\n\nThe \"fileobj\" argument can be any object that supports the file API.\nThe optional \"dialect\" argument defines a CSV dialect. The function\nalso accepts optional keyword arguments which override settings\nprovided by the dialect.", "_ctypes" => "Create and manipulate C compatible data types in Python.", + "_ctypes.ArgumentError.__delattr__" => "Implement delattr(self, name).", + "_ctypes.ArgumentError.__eq__" => "Return self==value.", + "_ctypes.ArgumentError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ctypes.ArgumentError.__ge__" => "Return self>=value.", + "_ctypes.ArgumentError.__getattribute__" => "Return getattr(self, name).", + "_ctypes.ArgumentError.__getstate__" => "Helper for pickle.", + "_ctypes.ArgumentError.__gt__" => "Return self>value.", + "_ctypes.ArgumentError.__hash__" => "Return hash(self).", + "_ctypes.ArgumentError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ctypes.ArgumentError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ctypes.ArgumentError.__le__" => "Return self<=value.", + "_ctypes.ArgumentError.__lt__" => "Return self<value.", + "_ctypes.ArgumentError.__ne__" => "Return self!=value.", + "_ctypes.ArgumentError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ctypes.ArgumentError.__reduce_ex__" => "Helper for pickle.", + "_ctypes.ArgumentError.__repr__" => "Return repr(self).", + "_ctypes.ArgumentError.__setattr__" => "Implement setattr(self, name, value).", + "_ctypes.ArgumentError.__sizeof__" => "Size of object in memory, in bytes.", + "_ctypes.ArgumentError.__str__" => "Return str(self).", + "_ctypes.ArgumentError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ctypes.ArgumentError.__weakref__" => "list of weak references to the object", + "_ctypes.ArgumentError.add_note" => "Add a note to the exception", + "_ctypes.ArgumentError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_ctypes.Array" => "Abstract base class for arrays.\n\nThe recommended way to create concrete array types is by multiplying any\nctypes data type with a non-negative integer. Alternatively, you can subclass\nthis type and define _length_ and _type_ class variables. Array elements can\nbe read and written using standard subscript and slice accesses for slice\nreads, the resulting object is not itself an Array.", + "_ctypes.CField" => "Structure/Union member", + "_ctypes.CField.__delattr__" => "Implement delattr(self, name).", + "_ctypes.CField.__delete__" => "Delete an attribute of instance.", + "_ctypes.CField.__eq__" => "Return self==value.", + "_ctypes.CField.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ctypes.CField.__ge__" => "Return self>=value.", + "_ctypes.CField.__get__" => "Return an attribute of instance, which is of type owner.", + "_ctypes.CField.__getattribute__" => "Return getattr(self, name).", + "_ctypes.CField.__getstate__" => "Helper for pickle.", + "_ctypes.CField.__gt__" => "Return self>value.", + "_ctypes.CField.__hash__" => "Return hash(self).", + "_ctypes.CField.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ctypes.CField.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ctypes.CField.__le__" => "Return self<=value.", + "_ctypes.CField.__lt__" => "Return self<value.", + "_ctypes.CField.__ne__" => "Return self!=value.", + "_ctypes.CField.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ctypes.CField.__reduce__" => "Helper for pickle.", + "_ctypes.CField.__reduce_ex__" => "Helper for pickle.", + "_ctypes.CField.__repr__" => "Return repr(self).", + "_ctypes.CField.__set__" => "Set an attribute of instance to value.", + "_ctypes.CField.__setattr__" => "Implement setattr(self, name, value).", + "_ctypes.CField.__sizeof__" => "Size of object in memory, in bytes.", + "_ctypes.CField.__str__" => "Return str(self).", + "_ctypes.CField.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ctypes.CField.bit_offset" => "additional offset in bits (relative to byte_offset); zero for non-bitfields", + "_ctypes.CField.bit_size" => "size of this field in bits", + "_ctypes.CField.byte_offset" => "offset in bytes of this field. For bitfields: excludes bit_offset.", + "_ctypes.CField.byte_size" => "size of this field in bytes", + "_ctypes.CField.is_anonymous" => "true if this field is anonymous", + "_ctypes.CField.is_bitfield" => "true if this is a bitfield", + "_ctypes.CField.name" => "name of this field", + "_ctypes.CField.offset" => "offset in bytes of this field (same as byte_offset)", + "_ctypes.CField.size" => "size in bytes of this field. For bitfields, this is a legacy packed value; use byte_size instead", + "_ctypes.CField.type" => "type of this field", "_ctypes.CFuncPtr" => "Function Pointer", "_ctypes.COMError" => "Raised when a COM method call failed.", - "_ctypes.COMError.__cause__" => "exception cause", - "_ctypes.COMError.__context__" => "exception context", "_ctypes.COMError.__delattr__" => "Implement delattr(self, name).", "_ctypes.COMError.__eq__" => "Return self==value.", "_ctypes.COMError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -422,28 +3584,26 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_ctypes.COMError.__sizeof__" => "Size of object in memory, in bytes.", "_ctypes.COMError.__str__" => "Return str(self).", "_ctypes.COMError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_ctypes.COMError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_ctypes.COMError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_ctypes.COMError.add_note" => "Add a note to the exception", + "_ctypes.COMError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_ctypes.CopyComPointer" => "CopyComPointer(src, dst) -> HRESULT value", "_ctypes.FormatError" => "FormatError([integer]) -> string\n\nConvert a win32 error code into a string. If the error code is not\ngiven, the return value of a call to GetLastError() is used.", "_ctypes.FreeLibrary" => "FreeLibrary(handle) -> void\n\nFree the handle of an executable previously loaded by LoadLibrary.", "_ctypes.LoadLibrary" => "LoadLibrary(name, load_flags) -> handle\n\nLoad an executable (usually a DLL), and return a handle to it.\nThe handle may be used to locate exported functions in this\nmodule. load_flags are as defined for LoadLibraryEx in the\nWindows API.", - "_ctypes.POINTER" => "Create and return a new ctypes pointer type.\n\n type\n A ctypes type.\n\nPointer types are cached and reused internally,\nso calling this function repeatedly is cheap.", "_ctypes.Structure" => "Structure base class", "_ctypes.Union" => "Union base class", "_ctypes._Pointer" => "XXX to be provided", "_ctypes._SimpleCData" => "XXX to be provided", "_ctypes._dyld_shared_cache_contains_path" => "check if path is in the shared cache", - "_ctypes.addressof" => "addressof(C instance) -> integer\nReturn the address of the C instance internal buffer", + "_ctypes.addressof" => "Return the address of the C instance internal buffer", "_ctypes.alignment" => "alignment(C type) -> integer\nalignment(C instance) -> integer\nReturn the alignment requirements of a C instance", "_ctypes.buffer_info" => "Return buffer interface information", - "_ctypes.byref" => "byref(C instance[, offset=0]) -> byref-object\nReturn a pointer lookalike to a C instance, only usable\nas function argument", + "_ctypes.byref" => "Return a pointer lookalike to a C instance, only usable as function argument.", "_ctypes.dlclose" => "dlclose a library", "_ctypes.dlopen" => "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library", "_ctypes.dlsym" => "find symbol in shared library", - "_ctypes.pointer" => "Create a new pointer instance, pointing to 'obj'.\n\nThe returned object is of the type POINTER(type(obj)). Note that if you\njust want to pass a pointer to an object to a foreign function call, you\nshould use byref(obj) which is much faster.", - "_ctypes.resize" => "Resize the memory buffer of a ctypes instance", - "_ctypes.sizeof" => "sizeof(C type) -> integer\nsizeof(C instance) -> integer\nReturn the size in bytes of a C instance", + "_ctypes.sizeof" => "Return the size in bytes of a C instance.", + "_curses.assume_default_colors" => "Allow use of default values for colors on terminals supporting this feature.\n\nAssign terminal default foreground/background colors to color number -1.\nChange the definition of the color-pair 0 to (fg, bg).\n\nUse this to support transparency in your application.", "_curses.baudrate" => "Return the output speed of the terminal in bits per second.", "_curses.beep" => "Emit a short attention sound.", "_curses.can_change_color" => "Return True if the programmer can change the colors displayed by the terminal.", @@ -453,13 +3613,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.curs_set" => "Set the cursor state.\n\n visibility\n 0 for invisible, 1 for normal visible, or 2 for very visible.\n\nIf the terminal supports the visibility requested, the previous cursor\nstate is returned; otherwise, an exception is raised. On many terminals,\nthe \"visible\" mode is an underline cursor and the \"very visible\" mode is\na block cursor.", "_curses.def_prog_mode" => "Save the current terminal mode as the \"program\" mode.\n\nThe \"program\" mode is the mode when the running program is using curses.\n\nSubsequent calls to reset_prog_mode() will restore this mode.", "_curses.def_shell_mode" => "Save the current terminal mode as the \"shell\" mode.\n\nThe \"shell\" mode is the mode when the running program is not using curses.\n\nSubsequent calls to reset_shell_mode() will restore this mode.", - "_curses.delay_output" => "Insert a pause in output.\n\nms\n Duration in milliseconds.", + "_curses.delay_output" => "Insert a pause in output.\n\n ms\n Duration in milliseconds.", "_curses.doupdate" => "Update the physical screen to match the virtual screen.", "_curses.echo" => "Enter echo mode.\n\n flag\n If false, the effect is the same as calling noecho().\n\nIn echo mode, each character input is echoed to the screen as it is entered.", "_curses.endwin" => "De-initialize the library, and return terminal to normal status.", "_curses.erasechar" => "Return the user's current erase character.", - "_curses.error.__cause__" => "exception cause", - "_curses.error.__context__" => "exception context", "_curses.error.__delattr__" => "Implement delattr(self, name).", "_curses.error.__eq__" => "Return self==value.", "_curses.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -481,8 +3639,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.error.__str__" => "Return str(self).", "_curses.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_curses.error.__weakref__" => "list of weak references to the object", - "_curses.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_curses.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_curses.error.add_note" => "Add a note to the exception", + "_curses.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_curses.flash" => "Flash the screen.\n\nThat is, change it to reverse-video and then change it back in a short interval.", "_curses.flushinp" => "Flush all input buffers.\n\nThis throws away any typeahead that has been typed by the user and has not\nyet been processed by the program.", "_curses.get_escdelay" => "Gets the curses ESCDELAY setting.\n\nGets the number of milliseconds to wait after reading an escape character,\nto distinguish between an individual escape character entered on the\nkeyboard from escape sequences sent by cursor and function keys.", @@ -495,20 +3653,20 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.has_extended_color_support" => "Return True if the module supports extended colors; otherwise, return False.\n\nExtended color support allows more than 256 color-pairs for terminals\nthat support more than 16 colors (e.g. xterm-256color).", "_curses.has_ic" => "Return True if the terminal has insert- and delete-character capabilities.", "_curses.has_il" => "Return True if the terminal has insert- and delete-line capabilities.", - "_curses.has_key" => "Return True if the current terminal type recognizes a key with that value.\n\nkey\n Key number.", + "_curses.has_key" => "Return True if the current terminal type recognizes a key with that value.\n\n key\n Key number.", "_curses.init_color" => "Change the definition of a color.\n\n color_number\n The number of the color to be changed (0 - (COLORS-1)).\n r\n Red component (0 - 1000).\n g\n Green component (0 - 1000).\n b\n Blue component (0 - 1000).\n\nWhen init_color() is used, all occurrences of that color on the screen\nimmediately change to the new definition. This function is a no-op on\nmost terminals; it is active only if can_change_color() returns true.", "_curses.init_pair" => "Change the definition of a color-pair.\n\n pair_number\n The number of the color-pair to be changed (1 - (COLOR_PAIRS-1)).\n fg\n Foreground color number (-1 - (COLORS-1)).\n bg\n Background color number (-1 - (COLORS-1)).\n\nIf the color-pair was previously initialized, the screen is refreshed and\nall occurrences of that color-pair are changed to the new definition.", "_curses.initscr" => "Initialize the library.\n\nReturn a WindowObject which represents the whole screen.", - "_curses.is_term_resized" => "Return True if resize_term() would modify the window structure, False otherwise.\n\nnlines\n Height.\nncols\n Width.", + "_curses.is_term_resized" => "Return True if resize_term() would modify the window structure, False otherwise.\n\n nlines\n Height.\n ncols\n Width.", "_curses.isendwin" => "Return True if endwin() has been called.", - "_curses.keyname" => "Return the name of specified key.\n\nkey\n Key number.", + "_curses.keyname" => "Return the name of specified key.\n\n key\n Key number.", "_curses.killchar" => "Return the user's current line kill character.", "_curses.longname" => "Return the terminfo long name field describing the current terminal.\n\nThe maximum length of a verbose description is 128 characters. It is defined\nonly after the call to initscr().", "_curses.meta" => "Enable/disable meta keys.\n\nIf yes is True, allow 8-bit characters to be input. If yes is False,\nallow only 7-bit characters.", "_curses.mouseinterval" => "Set and retrieve the maximum time between press and release in a click.\n\n interval\n Time in milliseconds.\n\nSet the maximum time that can elapse between press and release events in\norder for them to be recognized as a click, and return the previous interval\nvalue.", "_curses.mousemask" => "Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n\nReturn a tuple (availmask, oldmask). availmask indicates which of the\nspecified mouse events can be reported; on complete failure it returns 0.\noldmask is the previous value of the given window's mouse event mask.\nIf this function is never called, no mouse events are ever reported.", - "_curses.napms" => "Sleep for specified time.\n\nms\n Duration in milliseconds.", - "_curses.newpad" => "Create and return a pointer to a new pad data structure.\n\nnlines\n Height.\nncols\n Width.", + "_curses.napms" => "Sleep for specified time.\n\n ms\n Duration in milliseconds.", + "_curses.newpad" => "Create and return a pointer to a new pad data structure.\n\n nlines\n Height.\n ncols\n Width.", "_curses.newwin" => "newwin(nlines, ncols, [begin_y=0, begin_x=0])\nReturn a new window.\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nBy default, the window will extend from the specified position to the lower\nright corner of the screen.", "_curses.nl" => "Enter newline mode.\n\n flag\n If false, the effect is the same as calling nonl().\n\nThis mode translates the return key into newline on input, and translates\nnewline into return and line-feed on output. Newline mode is initially on.", "_curses.nocbreak" => "Leave cbreak mode.\n\nReturn to normal \"cooked\" mode with line buffering.", @@ -516,7 +3674,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.nonl" => "Leave newline mode.\n\nDisable translation of return into newline on input, and disable low-level\ntranslation of newline into newline/return on output.", "_curses.noqiflush" => "Disable queue flushing.\n\nWhen queue flushing is disabled, normal flush of input and output queues\nassociated with the INTR, QUIT and SUSP characters will not be done.", "_curses.noraw" => "Leave raw mode.\n\nReturn to normal \"cooked\" mode with line buffering.", - "_curses.pair_content" => "Return a tuple (fg, bg) containing the colors for the requested color pair.\n\npair_number\n The number of the color pair (0 - (COLOR_PAIRS-1)).", + "_curses.pair_content" => "Return a tuple (fg, bg) containing the colors for the requested color pair.\n\n pair_number\n The number of the color pair (0 - (COLOR_PAIRS-1)).", "_curses.pair_number" => "Return the number of the color-pair set by the specified attribute value.\n\ncolor_pair() is the counterpart to this function.", "_curses.putp" => "Emit the value of a specified terminfo capability for the current terminal.\n\nNote that the output of putp() always goes to standard output.", "_curses.qiflush" => "Enable queue flushing.\n\n flag\n If false, the effect is the same as calling noqiflush().\n\nIf queue flushing is enabled, all output in the display driver queue\nwill be flushed when the INTR, QUIT and SUSP characters are read.", @@ -530,20 +3688,20 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.set_escdelay" => "Sets the curses ESCDELAY setting.\n\n ms\n length of the delay in milliseconds.\n\nSets the number of milliseconds to wait after reading an escape character,\nto distinguish between an individual escape character entered on the\nkeyboard from escape sequences sent by cursor and function keys.", "_curses.set_tabsize" => "Sets the curses TABSIZE setting.\n\n size\n rendered cell width of a tab character.\n\nSets the number of columns used by the curses library when converting a tab\ncharacter to spaces as it adds the tab to a window.", "_curses.setsyx" => "Set the virtual screen cursor.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nIf y and x are both -1, then leaveok is set.", - "_curses.setupterm" => "Initialize the terminal.\n\nterm\n Terminal name.\n If omitted, the value of the TERM environment variable will be used.\nfd\n File descriptor to which any initialization sequences will be sent.\n If not supplied, the file descriptor for sys.stdout will be used.", + "_curses.setupterm" => "Initialize the terminal.\n\n term\n Terminal name.\n If omitted, the value of the TERM environment variable will be used.\n fd\n File descriptor to which any initialization sequences will be sent.\n If not supplied, the file descriptor for sys.stdout will be used.", "_curses.start_color" => "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n\nMust be called if the programmer wants to use colors, and before any other\ncolor manipulation routine is called. It is good practice to call this\nroutine right after initscr().\n\nIt also restores the colors on the terminal to the values they had when the\nterminal was just turned on.", "_curses.termattrs" => "Return a logical OR of all video attributes supported by the terminal.", "_curses.termname" => "Return the value of the environment variable TERM, truncated to 14 characters.", "_curses.tigetflag" => "Return the value of the Boolean capability.\n\n capname\n The terminfo capability name.\n\nThe value -1 is returned if capname is not a Boolean capability, or 0 if\nit is canceled or absent from the terminal description.", "_curses.tigetnum" => "Return the value of the numeric capability.\n\n capname\n The terminfo capability name.\n\nThe value -2 is returned if capname is not a numeric capability, or -1 if\nit is canceled or absent from the terminal description.", "_curses.tigetstr" => "Return the value of the string capability.\n\n capname\n The terminfo capability name.\n\nNone is returned if capname is not a string capability, or is canceled or\nabsent from the terminal description.", - "_curses.tparm" => "Instantiate the specified byte string with the supplied parameters.\n\nstr\n Parameterized byte string obtained from the terminfo database.", + "_curses.tparm" => "Instantiate the specified byte string with the supplied parameters.\n\n str\n Parameterized byte string obtained from the terminfo database.", "_curses.typeahead" => "Specify that the file descriptor fd be used for typeahead checking.\n\n fd\n File descriptor.\n\nIf fd is -1, then no typeahead checking is done.", "_curses.unctrl" => "Return a string which is a printable representation of the character ch.\n\nControl characters are displayed as a caret followed by the character,\nfor example as ^C. Printing characters are left as they are.", "_curses.unget_wch" => "Push ch so the next get_wch() will return it.", "_curses.ungetch" => "Push ch so the next getch() will return it.", "_curses.ungetmouse" => "Push a KEY_MOUSE event onto the input queue.\n\nThe following getmouse() will return the given state data.", - "_curses.use_default_colors" => "Allow use of default values for colors on terminals supporting this feature.\n\nUse this to support transparency in your application. The default color\nis assigned to the color number -1.", + "_curses.use_default_colors" => "Equivalent to assume_default_colors(-1, -1).", "_curses.use_env" => "Use environment variables LINES and COLUMNS.\n\nIf used, this function should be called before initscr() or newterm() are\ncalled.\n\nWhen flag is False, the values of lines and columns specified in the terminfo\ndatabase will be used, even if environment variables LINES and COLUMNS (used\nby default) are set, or if curses is running in a window (in which case\ndefault behavior would be to use the window size if LINES and COLUMNS are\nnot set).", "_curses.window.__delattr__" => "Implement delattr(self, name).", "_curses.window.__eq__" => "Return self==value.", @@ -570,26 +3728,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.window.addnstr" => "addnstr([y, x,] str, n, [attr])\nPaint at most n characters of the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to add.\n n\n Maximal number of characters.\n attr\n Attributes for characters.\n\nPaint at most n characters of the string str at (y, x) with\nattributes attr, overwriting anything previously on the display.\nBy default, the character position and attributes are the\ncurrent settings for the window object.", "_curses.window.addstr" => "addstr([y, x,] str, [attr])\nPaint the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to add.\n attr\n Attributes for characters.\n\nPaint the string str at (y, x) with attributes attr,\noverwriting anything previously on the display.\nBy default, the character position and attributes are the\ncurrent settings for the window object.", "_curses.window.attroff" => "Remove attribute attr from the \"background\" set.", - "_curses.window.attron" => "Add attribute attr from the \"background\" set.", + "_curses.window.attron" => "Add attribute attr to the \"background\" set.", "_curses.window.attrset" => "Set the \"background\" set of attributes.", - "_curses.window.bkgd" => "Set the background property of the window.\n\nch\n Background character.\nattr\n Background attributes.", - "_curses.window.bkgdset" => "Set the window's background.\n\nch\n Background character.\nattr\n Background attributes.", + "_curses.window.bkgd" => "Set the background property of the window.\n\n ch\n Background character.\n attr\n Background attributes.", + "_curses.window.bkgdset" => "Set the window's background.\n\n ch\n Background character.\n attr\n Background attributes.", "_curses.window.border" => "Draw a border around the edges of the window.\n\n ls\n Left side.\n rs\n Right side.\n ts\n Top side.\n bs\n Bottom side.\n tl\n Upper-left corner.\n tr\n Upper-right corner.\n bl\n Bottom-left corner.\n br\n Bottom-right corner.\n\nEach parameter specifies the character to use for a specific part of the\nborder. The characters can be specified as integers or as one-character\nstrings. A 0 value for any parameter will cause the default character to be\nused for that parameter.", "_curses.window.box" => "box([verch=0, horch=0])\nDraw a border around the edges of the window.\n\n verch\n Left and right side.\n horch\n Top and bottom side.\n\nSimilar to border(), but both ls and rs are verch and both ts and bs are\nhorch. The default corner characters are always used by this function.", + "_curses.window.chgat" => "chgat([y, x,] [n=-1,] attr)\nSet the attributes of characters.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Number of characters.\n attr\n Attributes for characters.\n\nSet the attributes of num characters at the current cursor position, or at\nposition (y, x) if supplied. If no value of num is given or num = -1, the\nattribute will be set on all the characters to the end of the line. This\nfunction does not move the cursor. The changed line will be touched using\nthe touchline() method so that the contents will be redisplayed by the next\nwindow refresh.", "_curses.window.delch" => "delch([y, x])\nDelete any character at (y, x).\n\n y\n Y-coordinate.\n x\n X-coordinate.", "_curses.window.derwin" => "derwin([nlines=0, ncols=0,] begin_y, begin_x)\nCreate a sub-window (window-relative coordinates).\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nderwin() is the same as calling subwin(), except that begin_y and begin_x\nare relative to the origin of the window, rather than relative to the entire\nscreen.", - "_curses.window.echochar" => "Add character ch with attribute attr, and refresh.\n\nch\n Character to add.\nattr\n Attributes for the character.", - "_curses.window.enclose" => "Return True if the screen-relative coordinates are enclosed by the window.\n\ny\n Y-coordinate.\nx\n X-coordinate.", + "_curses.window.echochar" => "Add character ch with attribute attr, and refresh.\n\n ch\n Character to add.\n attr\n Attributes for the character.", + "_curses.window.enclose" => "Return True if the screen-relative coordinates are enclosed by the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.", "_curses.window.encoding" => "the typecode character used to create the array", "_curses.window.get_wch" => "get_wch([y, x])\nGet a wide character from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nReturn a character for most keys, or an integer for function keys,\nkeypad keys, and other special keys.", "_curses.window.getbkgd" => "Return the window's current background character/attribute pair.", "_curses.window.getch" => "getch([y, x])\nGet a character code from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nThe integer returned does not have to be in ASCII range: function keys,\nkeypad keys and so on return numbers higher than 256. In no-delay mode, -1\nis returned if there is no input, else getch() waits until a key is pressed.", "_curses.window.getkey" => "getkey([y, x])\nGet a character (string) from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nReturning a string instead of an integer, as getch() does. Function keys,\nkeypad keys and other special keys return a multibyte string containing the\nkey name. In no-delay mode, an exception is raised if there is no input.", + "_curses.window.getstr" => "getstr([[y, x,] n=2047])\nRead a string from the user, with primitive line editing capacity.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Maximal number of characters.", "_curses.window.hline" => "hline([y, x,] ch, n, [attr=_curses.A_NORMAL])\nDisplay a horizontal line.\n\n y\n Starting Y-coordinate.\n x\n Starting X-coordinate.\n ch\n Character to draw.\n n\n Line length.\n attr\n Attributes for the characters.", "_curses.window.inch" => "inch([y, x])\nReturn the character at the given position in the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nThe bottom 8 bits are the character proper, and upper bits are the attributes.", "_curses.window.insch" => "insch([y, x,] ch, [attr=_curses.A_NORMAL])\nInsert a character before the current or specified position.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n ch\n Character to insert.\n attr\n Attributes for the character.\n\nAll characters to the right of the cursor are shifted one position right, with\nthe rightmost characters on the line being lost.", "_curses.window.insnstr" => "insnstr([y, x,] str, n, [attr])\nInsert at most n characters of the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to insert.\n n\n Maximal number of characters.\n attr\n Attributes for characters.\n\nInsert a character string (as many characters as will fit on the line)\nbefore the character under the cursor, up to n characters. If n is zero\nor negative, the entire string is inserted. All characters to the right\nof the cursor are shifted right, with the rightmost characters on the line\nbeing lost. The cursor position does not change (after moving to y, x, if\nspecified).", "_curses.window.insstr" => "insstr([y, x,] str, [attr])\nInsert the string before the current or specified position.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to insert.\n attr\n Attributes for characters.\n\nInsert a character string (as many characters as will fit on the line)\nbefore the character under the cursor. All characters to the right of\nthe cursor are shifted right, with the rightmost characters on the line\nbeing lost. The cursor position does not change (after moving to y, x,\nif specified).", + "_curses.window.instr" => "instr([y, x,] n=2047)\nReturn a string of characters, extracted from the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Maximal number of characters.\n\nReturn a string of characters, extracted from the window starting at the\ncurrent cursor position, or at y, x if specified. Attributes are stripped\nfrom the characters. If n is specified, instr() returns a string at most\nn characters long (exclusive of the trailing NUL).", "_curses.window.is_linetouched" => "Return True if the specified line was modified, otherwise return False.\n\n line\n Line number.\n\nRaise a curses.error exception if line is not valid for the given window.", "_curses.window.noutrefresh" => "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\nMark for refresh but wait.\n\nThis function updates the data structure representing the desired state of the\nwindow, but does not force an update of the physical screen. To accomplish\nthat, call doupdate().", "_curses.window.overlay" => "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\nOverlay the window on top of destwin.\n\nThe windows need not be the same size, only the overlapping region is copied.\nThis copy is non-destructive, which means that the current background\ncharacter does not overwrite the old contents of destwin.\n\nTo get fine-grained control over the copied region, the second form of\noverlay() can be used. sminrow and smincol are the upper-left coordinates\nof the source window, and the other variables mark a rectangle in the\ndestination window.", @@ -604,8 +3765,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses.window.touchline" => "touchline(start, count, [changed=True])\nPretend count lines have been changed, starting with line start.\n\nIf changed is supplied, it specifies whether the affected lines are marked\nas having been changed (changed=True) or unchanged (changed=False).", "_curses.window.vline" => "vline([y, x,] ch, n, [attr=_curses.A_NORMAL])\nDisplay a vertical line.\n\n y\n Starting Y-coordinate.\n x\n Starting X-coordinate.\n ch\n Character to draw.\n n\n Line length.\n attr\n Attributes for the character.", "_curses_panel.bottom_panel" => "Return the bottom panel in the panel stack.", - "_curses_panel.error.__cause__" => "exception cause", - "_curses_panel.error.__context__" => "exception context", "_curses_panel.error.__delattr__" => "Implement delattr(self, name).", "_curses_panel.error.__eq__" => "Return self==value.", "_curses_panel.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -627,8 +3786,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses_panel.error.__str__" => "Return str(self).", "_curses_panel.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_curses_panel.error.__weakref__" => "list of weak references to the object", - "_curses_panel.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_curses_panel.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_curses_panel.error.add_note" => "Add a note to the exception", + "_curses_panel.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_curses_panel.new_panel" => "Return a panel object, associating it with the given window win.", "_curses_panel.panel.__delattr__" => "Implement delattr(self, name).", "_curses_panel.panel.__eq__" => "Return self==value.", @@ -665,9 +3824,232 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_curses_panel.panel.window" => "Return the window object associated with the panel.", "_curses_panel.top_panel" => "Return the top panel in the panel stack.", "_curses_panel.update_panels" => "Updates the virtual screen after changes in the panel stack.\n\nThis does not call curses.doupdate(), so you'll have to do this yourself.", - "_datetime" => "Fast implementation of the datetime type.", - "_dbm.error.__cause__" => "exception cause", - "_dbm.error.__context__" => "exception context", + "_datetime" => "Fast implementation of the datetime module.", + "_datetime.date" => "date(year, month, day) --> date object", + "_datetime.date.__add__" => "Return self+value.", + "_datetime.date.__delattr__" => "Implement delattr(self, name).", + "_datetime.date.__eq__" => "Return self==value.", + "_datetime.date.__format__" => "Formats self with strftime.", + "_datetime.date.__ge__" => "Return self>=value.", + "_datetime.date.__getattribute__" => "Return getattr(self, name).", + "_datetime.date.__getstate__" => "Helper for pickle.", + "_datetime.date.__gt__" => "Return self>value.", + "_datetime.date.__hash__" => "Return hash(self).", + "_datetime.date.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.date.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.date.__le__" => "Return self<=value.", + "_datetime.date.__lt__" => "Return self<value.", + "_datetime.date.__ne__" => "Return self!=value.", + "_datetime.date.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.date.__radd__" => "Return value+self.", + "_datetime.date.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.date.__reduce_ex__" => "Helper for pickle.", + "_datetime.date.__replace__" => "The same as replace().", + "_datetime.date.__repr__" => "Return repr(self).", + "_datetime.date.__rsub__" => "Return value-self.", + "_datetime.date.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.date.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.date.__str__" => "Return str(self).", + "_datetime.date.__sub__" => "Return self-value.", + "_datetime.date.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.date.ctime" => "Return ctime() style string.", + "_datetime.date.fromisocalendar" => "int, int, int -> Construct a date from the ISO year, week number and weekday.\n\nThis is the inverse of the date.isocalendar() function", + "_datetime.date.fromisoformat" => "str -> Construct a date from a string in ISO 8601 format.", + "_datetime.date.fromordinal" => "int -> date corresponding to a proleptic Gregorian ordinal.", + "_datetime.date.fromtimestamp" => "Create a date from a POSIX timestamp.\n\nThe timestamp is a number, e.g. created via time.time(), that is interpreted\nas local time.", + "_datetime.date.isocalendar" => "Return a named tuple containing ISO year, week number, and weekday.", + "_datetime.date.isoformat" => "Return string in ISO 8601 format, YYYY-MM-DD.", + "_datetime.date.isoweekday" => "Return the day of the week represented by the date.\nMonday == 1 ... Sunday == 7", + "_datetime.date.replace" => "Return date with new specified fields.", + "_datetime.date.strftime" => "format -> strftime() style string.", + "_datetime.date.strptime" => "string, format -> new date parsed from a string (like time.strptime()).", + "_datetime.date.timetuple" => "Return time tuple, compatible with time.localtime().", + "_datetime.date.today" => "Current date or datetime: same as self.__class__.fromtimestamp(time.time()).", + "_datetime.date.toordinal" => "Return proleptic Gregorian ordinal. January 1 of year 1 is day 1.", + "_datetime.date.weekday" => "Return the day of the week represented by the date.\nMonday == 0 ... Sunday == 6", + "_datetime.datetime" => "datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\nThe year, month and day arguments are required. tzinfo may be None, or an\ninstance of a tzinfo subclass. The remaining arguments may be ints.", + "_datetime.datetime.__add__" => "Return self+value.", + "_datetime.datetime.__delattr__" => "Implement delattr(self, name).", + "_datetime.datetime.__eq__" => "Return self==value.", + "_datetime.datetime.__format__" => "Formats self with strftime.", + "_datetime.datetime.__ge__" => "Return self>=value.", + "_datetime.datetime.__getattribute__" => "Return getattr(self, name).", + "_datetime.datetime.__getstate__" => "Helper for pickle.", + "_datetime.datetime.__gt__" => "Return self>value.", + "_datetime.datetime.__hash__" => "Return hash(self).", + "_datetime.datetime.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.datetime.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.datetime.__le__" => "Return self<=value.", + "_datetime.datetime.__lt__" => "Return self<value.", + "_datetime.datetime.__ne__" => "Return self!=value.", + "_datetime.datetime.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.datetime.__radd__" => "Return value+self.", + "_datetime.datetime.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.datetime.__reduce_ex__" => "__reduce_ex__(proto) -> (cls, state)", + "_datetime.datetime.__replace__" => "The same as replace().", + "_datetime.datetime.__repr__" => "Return repr(self).", + "_datetime.datetime.__rsub__" => "Return value-self.", + "_datetime.datetime.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.datetime.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.datetime.__str__" => "Return str(self).", + "_datetime.datetime.__sub__" => "Return self-value.", + "_datetime.datetime.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.datetime.astimezone" => "tz -> convert to local time in new timezone tz", + "_datetime.datetime.combine" => "date, time -> datetime with same date and time fields", + "_datetime.datetime.ctime" => "Return ctime() style string.", + "_datetime.datetime.date" => "Return date object with same year, month and day.", + "_datetime.datetime.dst" => "Return self.tzinfo.dst(self).", + "_datetime.datetime.fromisocalendar" => "int, int, int -> Construct a date from the ISO year, week number and weekday.\n\nThis is the inverse of the date.isocalendar() function", + "_datetime.datetime.fromisoformat" => "string -> datetime from a string in most ISO 8601 formats", + "_datetime.datetime.fromordinal" => "int -> date corresponding to a proleptic Gregorian ordinal.", + "_datetime.datetime.fromtimestamp" => "timestamp[, tz] -> tz's local time from POSIX timestamp.", + "_datetime.datetime.isocalendar" => "Return a named tuple containing ISO year, week number, and weekday.", + "_datetime.datetime.isoformat" => "[sep] -> string in ISO 8601 format, YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\nsep is used to separate the year from the time, and defaults to 'T'.\nThe optional argument timespec specifies the number of additional terms\nof the time to include. Valid options are 'auto', 'hours', 'minutes',\n'seconds', 'milliseconds' and 'microseconds'.", + "_datetime.datetime.isoweekday" => "Return the day of the week represented by the date.\nMonday == 1 ... Sunday == 7", + "_datetime.datetime.now" => "Returns new datetime object representing current time local to tz.\n\n tz\n Timezone object.\n\nIf no tz is specified, uses local timezone.", + "_datetime.datetime.replace" => "Return datetime with new specified fields.", + "_datetime.datetime.strftime" => "format -> strftime() style string.", + "_datetime.datetime.strptime" => "string, format -> new datetime parsed from a string (like time.strptime()).", + "_datetime.datetime.time" => "Return time object with same time but with tzinfo=None.", + "_datetime.datetime.timestamp" => "Return POSIX timestamp as float.", + "_datetime.datetime.timetuple" => "Return time tuple, compatible with time.localtime().", + "_datetime.datetime.timetz" => "Return time object with same time and tzinfo.", + "_datetime.datetime.today" => "Current date or datetime: same as self.__class__.fromtimestamp(time.time()).", + "_datetime.datetime.toordinal" => "Return proleptic Gregorian ordinal. January 1 of year 1 is day 1.", + "_datetime.datetime.tzname" => "Return self.tzinfo.tzname(self).", + "_datetime.datetime.utcfromtimestamp" => "Construct a naive UTC datetime from a POSIX timestamp.", + "_datetime.datetime.utcnow" => "Return a new datetime representing UTC day and time.", + "_datetime.datetime.utcoffset" => "Return self.tzinfo.utcoffset(self).", + "_datetime.datetime.utctimetuple" => "Return UTC time tuple, compatible with time.localtime().", + "_datetime.datetime.weekday" => "Return the day of the week represented by the date.\nMonday == 0 ... Sunday == 6", + "_datetime.time" => "time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\nAll arguments are optional. tzinfo may be None, or an instance of\na tzinfo subclass. The remaining arguments may be ints.", + "_datetime.time.__delattr__" => "Implement delattr(self, name).", + "_datetime.time.__eq__" => "Return self==value.", + "_datetime.time.__format__" => "Formats self with strftime.", + "_datetime.time.__ge__" => "Return self>=value.", + "_datetime.time.__getattribute__" => "Return getattr(self, name).", + "_datetime.time.__getstate__" => "Helper for pickle.", + "_datetime.time.__gt__" => "Return self>value.", + "_datetime.time.__hash__" => "Return hash(self).", + "_datetime.time.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.time.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.time.__le__" => "Return self<=value.", + "_datetime.time.__lt__" => "Return self<value.", + "_datetime.time.__ne__" => "Return self!=value.", + "_datetime.time.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.time.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.time.__reduce_ex__" => "__reduce_ex__(proto) -> (cls, state)", + "_datetime.time.__replace__" => "The same as replace().", + "_datetime.time.__repr__" => "Return repr(self).", + "_datetime.time.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.time.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.time.__str__" => "Return str(self).", + "_datetime.time.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.time.dst" => "Return self.tzinfo.dst(self).", + "_datetime.time.fromisoformat" => "string -> time from a string in ISO 8601 format", + "_datetime.time.isoformat" => "Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n\nThe optional argument timespec specifies the number of additional terms\nof the time to include. Valid options are 'auto', 'hours', 'minutes',\n'seconds', 'milliseconds' and 'microseconds'.", + "_datetime.time.replace" => "Return time with new specified fields.", + "_datetime.time.strftime" => "format -> strftime() style string.", + "_datetime.time.strptime" => "string, format -> new time parsed from a string (like time.strptime()).", + "_datetime.time.tzname" => "Return self.tzinfo.tzname(self).", + "_datetime.time.utcoffset" => "Return self.tzinfo.utcoffset(self).", + "_datetime.timedelta" => "Difference between two datetime values.\n\ntimedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)\n\nAll arguments are optional and default to 0.\nArguments may be integers or floats, and may be positive or negative.", + "_datetime.timedelta.__abs__" => "abs(self)", + "_datetime.timedelta.__add__" => "Return self+value.", + "_datetime.timedelta.__bool__" => "True if self else False", + "_datetime.timedelta.__delattr__" => "Implement delattr(self, name).", + "_datetime.timedelta.__divmod__" => "Return divmod(self, value).", + "_datetime.timedelta.__eq__" => "Return self==value.", + "_datetime.timedelta.__floordiv__" => "Return self//value.", + "_datetime.timedelta.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.timedelta.__ge__" => "Return self>=value.", + "_datetime.timedelta.__getattribute__" => "Return getattr(self, name).", + "_datetime.timedelta.__getstate__" => "Helper for pickle.", + "_datetime.timedelta.__gt__" => "Return self>value.", + "_datetime.timedelta.__hash__" => "Return hash(self).", + "_datetime.timedelta.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.timedelta.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.timedelta.__le__" => "Return self<=value.", + "_datetime.timedelta.__lt__" => "Return self<value.", + "_datetime.timedelta.__mod__" => "Return self%value.", + "_datetime.timedelta.__mul__" => "Return self*value.", + "_datetime.timedelta.__ne__" => "Return self!=value.", + "_datetime.timedelta.__neg__" => "-self", + "_datetime.timedelta.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.timedelta.__pos__" => "+self", + "_datetime.timedelta.__radd__" => "Return value+self.", + "_datetime.timedelta.__rdivmod__" => "Return divmod(value, self).", + "_datetime.timedelta.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.timedelta.__reduce_ex__" => "Helper for pickle.", + "_datetime.timedelta.__repr__" => "Return repr(self).", + "_datetime.timedelta.__rfloordiv__" => "Return value//self.", + "_datetime.timedelta.__rmod__" => "Return value%self.", + "_datetime.timedelta.__rmul__" => "Return value*self.", + "_datetime.timedelta.__rsub__" => "Return value-self.", + "_datetime.timedelta.__rtruediv__" => "Return value/self.", + "_datetime.timedelta.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.timedelta.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.timedelta.__str__" => "Return str(self).", + "_datetime.timedelta.__sub__" => "Return self-value.", + "_datetime.timedelta.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.timedelta.__truediv__" => "Return self/value.", + "_datetime.timedelta.days" => "Number of days.", + "_datetime.timedelta.microseconds" => "Number of microseconds (>= 0 and less than 1 second).", + "_datetime.timedelta.seconds" => "Number of seconds (>= 0 and less than 1 day).", + "_datetime.timedelta.total_seconds" => "Total seconds in the duration.", + "_datetime.timezone" => "Fixed offset from UTC implementation of tzinfo.", + "_datetime.timezone.__delattr__" => "Implement delattr(self, name).", + "_datetime.timezone.__eq__" => "Return self==value.", + "_datetime.timezone.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.timezone.__ge__" => "Return self>=value.", + "_datetime.timezone.__getattribute__" => "Return getattr(self, name).", + "_datetime.timezone.__getinitargs__" => "pickle support", + "_datetime.timezone.__getstate__" => "Helper for pickle.", + "_datetime.timezone.__gt__" => "Return self>value.", + "_datetime.timezone.__hash__" => "Return hash(self).", + "_datetime.timezone.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.timezone.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.timezone.__le__" => "Return self<=value.", + "_datetime.timezone.__lt__" => "Return self<value.", + "_datetime.timezone.__ne__" => "Return self!=value.", + "_datetime.timezone.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.timezone.__reduce__" => "-> (cls, state)", + "_datetime.timezone.__reduce_ex__" => "Helper for pickle.", + "_datetime.timezone.__repr__" => "Return repr(self).", + "_datetime.timezone.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.timezone.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.timezone.__str__" => "Return str(self).", + "_datetime.timezone.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.timezone.dst" => "Return None.", + "_datetime.timezone.fromutc" => "datetime in UTC -> datetime in local time.", + "_datetime.timezone.tzname" => "If name is specified when timezone is created, returns the name. Otherwise returns offset as 'UTC(+|-)HH:MM'.", + "_datetime.timezone.utcoffset" => "Return fixed offset.", + "_datetime.tzinfo" => "Abstract base class for time zone info objects.", + "_datetime.tzinfo.__delattr__" => "Implement delattr(self, name).", + "_datetime.tzinfo.__eq__" => "Return self==value.", + "_datetime.tzinfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.tzinfo.__ge__" => "Return self>=value.", + "_datetime.tzinfo.__getattribute__" => "Return getattr(self, name).", + "_datetime.tzinfo.__getstate__" => "Helper for pickle.", + "_datetime.tzinfo.__gt__" => "Return self>value.", + "_datetime.tzinfo.__hash__" => "Return hash(self).", + "_datetime.tzinfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.tzinfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.tzinfo.__le__" => "Return self<=value.", + "_datetime.tzinfo.__lt__" => "Return self<value.", + "_datetime.tzinfo.__ne__" => "Return self!=value.", + "_datetime.tzinfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.tzinfo.__reduce__" => "-> (cls, state)", + "_datetime.tzinfo.__reduce_ex__" => "Helper for pickle.", + "_datetime.tzinfo.__repr__" => "Return repr(self).", + "_datetime.tzinfo.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.tzinfo.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.tzinfo.__str__" => "Return str(self).", + "_datetime.tzinfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.tzinfo.dst" => "datetime -> DST offset as timedelta positive east of UTC.", + "_datetime.tzinfo.fromutc" => "datetime in UTC -> datetime in local time.", + "_datetime.tzinfo.tzname" => "datetime -> string name of time zone.", + "_datetime.tzinfo.utcoffset" => "datetime -> timedelta showing offset from UTC, negative values indicating West of UTC", "_dbm.error.__delattr__" => "Implement delattr(self, name).", "_dbm.error.__eq__" => "Return self==value.", "_dbm.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -689,24 +4071,744 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_dbm.error.__str__" => "Return str(self).", "_dbm.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_dbm.error.__weakref__" => "list of weak references to the object", - "_dbm.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "_dbm.error.add_note" => "Add a note to the exception", "_dbm.error.errno" => "POSIX exception code", "_dbm.error.filename" => "exception filename", "_dbm.error.filename2" => "second exception filename", "_dbm.error.strerror" => "exception strerror", - "_dbm.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_dbm.open" => "Return a database object.\n\nfilename\n The filename to open.\nflags\n How to open the file. \"r\" for reading, \"w\" for writing, etc.\nmode\n If creating a new file, the mode bits for the new file\n (e.g. os.O_RDWR).", + "_dbm.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_dbm.open" => "Return a database object.\n\n filename\n The filename to open.\n flags\n How to open the file. \"r\" for reading, \"w\" for writing, etc.\n mode\n If creating a new file, the mode bits for the new file\n (e.g. os.O_RDWR).", "_decimal" => "C decimal arithmetic module", + "_decimal.Clamped.__delattr__" => "Implement delattr(self, name).", + "_decimal.Clamped.__eq__" => "Return self==value.", + "_decimal.Clamped.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Clamped.__ge__" => "Return self>=value.", + "_decimal.Clamped.__getattribute__" => "Return getattr(self, name).", + "_decimal.Clamped.__getstate__" => "Helper for pickle.", + "_decimal.Clamped.__gt__" => "Return self>value.", + "_decimal.Clamped.__hash__" => "Return hash(self).", + "_decimal.Clamped.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Clamped.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Clamped.__le__" => "Return self<=value.", + "_decimal.Clamped.__lt__" => "Return self<value.", + "_decimal.Clamped.__ne__" => "Return self!=value.", + "_decimal.Clamped.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Clamped.__reduce_ex__" => "Helper for pickle.", + "_decimal.Clamped.__repr__" => "Return repr(self).", + "_decimal.Clamped.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Clamped.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Clamped.__str__" => "Return str(self).", + "_decimal.Clamped.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Clamped.__weakref__" => "list of weak references to the object", + "_decimal.Clamped.add_note" => "Add a note to the exception", + "_decimal.Clamped.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Context" => "The context affects almost all operations and controls rounding,\nOver/Underflow, raising of exceptions and much more. A new context\ncan be constructed as follows:\n\n >>> c = Context(prec=28, Emin=-425000000, Emax=425000000,\n ... rounding=ROUND_HALF_EVEN, capitals=1, clamp=1,\n ... traps=[InvalidOperation, DivisionByZero, Overflow],\n ... flags=[])\n >>>", + "_decimal.Context.Etiny" => "Return a value equal to Emin - prec + 1, which is the minimum exponent value\nfor subnormal results. When underflow occurs, the exponent is set to Etiny.", + "_decimal.Context.Etop" => "Return a value equal to Emax - prec + 1. This is the maximum exponent\nif the _clamp field of the context is set to 1 (IEEE clamp mode). Etop()\nmust not be negative.", + "_decimal.Context.__delattr__" => "Implement delattr(self, name).", + "_decimal.Context.__eq__" => "Return self==value.", + "_decimal.Context.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Context.__ge__" => "Return self>=value.", + "_decimal.Context.__getattribute__" => "Return getattr(self, name).", + "_decimal.Context.__getstate__" => "Helper for pickle.", + "_decimal.Context.__gt__" => "Return self>value.", + "_decimal.Context.__hash__" => "Return hash(self).", + "_decimal.Context.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Context.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Context.__le__" => "Return self<=value.", + "_decimal.Context.__lt__" => "Return self<value.", + "_decimal.Context.__ne__" => "Return self!=value.", + "_decimal.Context.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Context.__reduce_ex__" => "Helper for pickle.", + "_decimal.Context.__repr__" => "Return repr(self).", + "_decimal.Context.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Context.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Context.__str__" => "Return str(self).", + "_decimal.Context.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Context.abs" => "Return the absolute value of x.", + "_decimal.Context.add" => "Return the sum of x and y.", + "_decimal.Context.canonical" => "Return a new instance of x.", + "_decimal.Context.clear_flags" => "Reset all flags to False.", + "_decimal.Context.clear_traps" => "Set all traps to False.", + "_decimal.Context.compare" => "Compare x and y numerically.", + "_decimal.Context.compare_signal" => "Compare x and y numerically. All NaNs signal.", + "_decimal.Context.compare_total" => "Compare x and y using their abstract representation.", + "_decimal.Context.compare_total_mag" => "Compare x and y using their abstract representation, ignoring sign.", + "_decimal.Context.copy" => "Return a duplicate of the context with all flags cleared.", + "_decimal.Context.copy_abs" => "Return a copy of x with the sign set to 0.", + "_decimal.Context.copy_decimal" => "Return a copy of Decimal x.", + "_decimal.Context.copy_negate" => "Return a copy of x with the sign inverted.", + "_decimal.Context.copy_sign" => "Copy the sign from y to x.", + "_decimal.Context.create_decimal" => "Create a new Decimal instance from num, using self as the context. Unlike the\nDecimal constructor, this function observes the context limits.", + "_decimal.Context.create_decimal_from_float" => "Create a new Decimal instance from float f. Unlike the Decimal.from_float()\nclass method, this function observes the context limits.", + "_decimal.Context.divide" => "Return x divided by y.", + "_decimal.Context.divide_int" => "Return x divided by y, truncated to an integer.", + "_decimal.Context.divmod" => "Return quotient and remainder of the division x / y.", + "_decimal.Context.exp" => "Return e ** x.", + "_decimal.Context.fma" => "Return x multiplied by y, plus z.", + "_decimal.Context.is_canonical" => "Return True if x is canonical, False otherwise.", + "_decimal.Context.is_finite" => "Return True if x is finite, False otherwise.", + "_decimal.Context.is_infinite" => "Return True if x is infinite, False otherwise.", + "_decimal.Context.is_nan" => "Return True if x is a qNaN or sNaN, False otherwise.", + "_decimal.Context.is_normal" => "Return True if x is a normal number, False otherwise.", + "_decimal.Context.is_qnan" => "Return True if x is a quiet NaN, False otherwise.", + "_decimal.Context.is_signed" => "Return True if x is negative, False otherwise.", + "_decimal.Context.is_snan" => "Return True if x is a signaling NaN, False otherwise.", + "_decimal.Context.is_subnormal" => "Return True if x is subnormal, False otherwise.", + "_decimal.Context.is_zero" => "Return True if x is a zero, False otherwise.", + "_decimal.Context.ln" => "Return the natural (base e) logarithm of x.", + "_decimal.Context.log10" => "Return the base 10 logarithm of x.", + "_decimal.Context.logb" => "Return the exponent of the magnitude of the operand's MSD.", + "_decimal.Context.logical_and" => "Applies the logical operation 'and' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))\n Decimal('1000')\n >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))\n Decimal('10')\n >>> ExtendedContext.logical_and(110, 1101)\n Decimal('100')\n >>> ExtendedContext.logical_and(Decimal(110), 1101)\n Decimal('100')\n >>> ExtendedContext.logical_and(110, Decimal(1101))\n Decimal('100')", + "_decimal.Context.logical_invert" => "Invert all the digits in the operand.\n\nThe operand must be a logical number.\n\n >>> ExtendedContext.logical_invert(Decimal('0'))\n Decimal('111111111')\n >>> ExtendedContext.logical_invert(Decimal('1'))\n Decimal('111111110')\n >>> ExtendedContext.logical_invert(Decimal('111111111'))\n Decimal('0')\n >>> ExtendedContext.logical_invert(Decimal('101010101'))\n Decimal('10101010')\n >>> ExtendedContext.logical_invert(1101)\n Decimal('111110010')", + "_decimal.Context.logical_or" => "Applies the logical operation 'or' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))\n Decimal('1110')\n >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))\n Decimal('1110')\n >>> ExtendedContext.logical_or(110, 1101)\n Decimal('1111')\n >>> ExtendedContext.logical_or(Decimal(110), 1101)\n Decimal('1111')\n >>> ExtendedContext.logical_or(110, Decimal(1101))\n Decimal('1111')", + "_decimal.Context.logical_xor" => "Applies the logical operation 'xor' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))\n Decimal('1')\n >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))\n Decimal('0')\n >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))\n Decimal('110')\n >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))\n Decimal('1101')\n >>> ExtendedContext.logical_xor(110, 1101)\n Decimal('1011')\n >>> ExtendedContext.logical_xor(Decimal(110), 1101)\n Decimal('1011')\n >>> ExtendedContext.logical_xor(110, Decimal(1101))\n Decimal('1011')", + "_decimal.Context.max" => "Compare the values numerically and return the maximum.", + "_decimal.Context.max_mag" => "Compare the values numerically with their sign ignored.", + "_decimal.Context.min" => "Compare the values numerically and return the minimum.", + "_decimal.Context.min_mag" => "Compare the values numerically with their sign ignored.", + "_decimal.Context.minus" => "Minus corresponds to the unary prefix minus operator in Python, but applies\nthe context to the result.", + "_decimal.Context.multiply" => "Return the product of x and y.", + "_decimal.Context.next_minus" => "Return the largest representable number smaller than x.", + "_decimal.Context.next_plus" => "Return the smallest representable number larger than x.", + "_decimal.Context.next_toward" => "Return the number closest to x, in the direction towards y.", + "_decimal.Context.normalize" => "Reduce x to its simplest form. Alias for reduce(x).", + "_decimal.Context.number_class" => "Return an indication of the class of x.", + "_decimal.Context.plus" => "Plus corresponds to the unary prefix plus operator in Python, but applies\nthe context to the result.", + "_decimal.Context.power" => "Compute a**b. If 'a' is negative, then 'b' must be integral. The result\nwill be inexact unless 'a' is integral and the result is finite and can\nbe expressed exactly in 'precision' digits. In the Python version the\nresult is always correctly rounded, in the C version the result is almost\nalways correctly rounded.\n\nIf modulo is given, compute (a**b) % modulo. The following restrictions\nhold:\n\n * all three arguments must be integral\n * 'b' must be nonnegative\n * at least one of 'a' or 'b' must be nonzero\n * modulo must be nonzero and less than 10**prec in absolute value", + "_decimal.Context.quantize" => "Return a value equal to x (rounded), having the exponent of y.", + "_decimal.Context.radix" => "Return 10.", + "_decimal.Context.remainder" => "Return the remainder from integer division. The sign of the result,\nif non-zero, is the same as that of the original dividend.", + "_decimal.Context.remainder_near" => "Return x - y * n, where n is the integer nearest the exact value of x / y\n(if the result is 0 then its sign will be the sign of x).", + "_decimal.Context.rotate" => "Return a copy of x, rotated by y places.", + "_decimal.Context.same_quantum" => "Return True if the two operands have the same exponent.", + "_decimal.Context.scaleb" => "Return the first operand after adding the second value to its exp.", + "_decimal.Context.shift" => "Return a copy of x, shifted by y places.", + "_decimal.Context.sqrt" => "Square root of a non-negative number to context precision.", + "_decimal.Context.subtract" => "Return the difference between x and y.", + "_decimal.Context.to_eng_string" => "Convert a number to a string, using engineering notation.", + "_decimal.Context.to_integral" => "Identical to to_integral_value(x).", + "_decimal.Context.to_integral_exact" => "Round to an integer. Signal if the result is rounded or inexact.", + "_decimal.Context.to_integral_value" => "Round to an integer.", + "_decimal.Context.to_sci_string" => "Convert a number to a string using scientific notation.", + "_decimal.ConversionSyntax.__delattr__" => "Implement delattr(self, name).", + "_decimal.ConversionSyntax.__eq__" => "Return self==value.", + "_decimal.ConversionSyntax.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.ConversionSyntax.__ge__" => "Return self>=value.", + "_decimal.ConversionSyntax.__getattribute__" => "Return getattr(self, name).", + "_decimal.ConversionSyntax.__getstate__" => "Helper for pickle.", + "_decimal.ConversionSyntax.__gt__" => "Return self>value.", + "_decimal.ConversionSyntax.__hash__" => "Return hash(self).", + "_decimal.ConversionSyntax.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.ConversionSyntax.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.ConversionSyntax.__le__" => "Return self<=value.", + "_decimal.ConversionSyntax.__lt__" => "Return self<value.", + "_decimal.ConversionSyntax.__ne__" => "Return self!=value.", + "_decimal.ConversionSyntax.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.ConversionSyntax.__reduce_ex__" => "Helper for pickle.", + "_decimal.ConversionSyntax.__repr__" => "Return repr(self).", + "_decimal.ConversionSyntax.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.ConversionSyntax.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.ConversionSyntax.__str__" => "Return str(self).", + "_decimal.ConversionSyntax.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.ConversionSyntax.__weakref__" => "list of weak references to the object", + "_decimal.ConversionSyntax.add_note" => "Add a note to the exception", + "_decimal.ConversionSyntax.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Decimal" => "Construct a new Decimal object. 'value' can be an integer, string, tuple,\nor another Decimal object. If no value is given, return Decimal('0'). The\ncontext does not affect the conversion and is only passed to determine if\nthe InvalidOperation trap is active.", + "_decimal.Decimal.__abs__" => "abs(self)", + "_decimal.Decimal.__add__" => "Return self+value.", + "_decimal.Decimal.__bool__" => "True if self else False", + "_decimal.Decimal.__delattr__" => "Implement delattr(self, name).", + "_decimal.Decimal.__divmod__" => "Return divmod(self, value).", + "_decimal.Decimal.__eq__" => "Return self==value.", + "_decimal.Decimal.__float__" => "float(self)", + "_decimal.Decimal.__floordiv__" => "Return self//value.", + "_decimal.Decimal.__ge__" => "Return self>=value.", + "_decimal.Decimal.__getattribute__" => "Return getattr(self, name).", + "_decimal.Decimal.__getstate__" => "Helper for pickle.", + "_decimal.Decimal.__gt__" => "Return self>value.", + "_decimal.Decimal.__hash__" => "Return hash(self).", + "_decimal.Decimal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Decimal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Decimal.__int__" => "int(self)", + "_decimal.Decimal.__le__" => "Return self<=value.", + "_decimal.Decimal.__lt__" => "Return self<value.", + "_decimal.Decimal.__mod__" => "Return self%value.", + "_decimal.Decimal.__mul__" => "Return self*value.", + "_decimal.Decimal.__ne__" => "Return self!=value.", + "_decimal.Decimal.__neg__" => "-self", + "_decimal.Decimal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Decimal.__pos__" => "+self", + "_decimal.Decimal.__pow__" => "Return pow(self, value, mod).", + "_decimal.Decimal.__radd__" => "Return value+self.", + "_decimal.Decimal.__rdivmod__" => "Return divmod(value, self).", + "_decimal.Decimal.__reduce_ex__" => "Helper for pickle.", + "_decimal.Decimal.__repr__" => "Return repr(self).", + "_decimal.Decimal.__rfloordiv__" => "Return value//self.", + "_decimal.Decimal.__rmod__" => "Return value%self.", + "_decimal.Decimal.__rmul__" => "Return value*self.", + "_decimal.Decimal.__rpow__" => "Return pow(value, self, mod).", + "_decimal.Decimal.__rsub__" => "Return value-self.", + "_decimal.Decimal.__rtruediv__" => "Return value/self.", + "_decimal.Decimal.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Decimal.__str__" => "Return str(self).", + "_decimal.Decimal.__sub__" => "Return self-value.", + "_decimal.Decimal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Decimal.__truediv__" => "Return self/value.", + "_decimal.Decimal.adjusted" => "Return the adjusted exponent of the number. Defined as exp + digits - 1.", + "_decimal.Decimal.as_integer_ratio" => "Decimal.as_integer_ratio() -> (int, int)\n\nReturn a pair of integers, whose ratio is exactly equal to the original\nDecimal and with a positive denominator. The ratio is in lowest terms.\nRaise OverflowError on infinities and a ValueError on NaNs.", + "_decimal.Decimal.as_tuple" => "Return a tuple representation of the number.", + "_decimal.Decimal.canonical" => "Return the canonical encoding of the argument. Currently, the encoding\nof a Decimal instance is always canonical, so this operation returns its\nargument unchanged.", + "_decimal.Decimal.compare" => "Compare self to other. Return a decimal value:\n\n a or b is a NaN ==> Decimal('NaN')\n a < b ==> Decimal('-1')\n a == b ==> Decimal('0')\n a > b ==> Decimal('1')", + "_decimal.Decimal.compare_signal" => "Identical to compare, except that all NaNs signal.", + "_decimal.Decimal.compare_total" => "Compare two operands using their abstract representation rather than\ntheir numerical value. Similar to the compare() method, but the result\ngives a total ordering on Decimal instances. Two Decimal instances with\nthe same numeric value but different representations compare unequal\nin this ordering:\n\n >>> Decimal('12.0').compare_total(Decimal('12'))\n Decimal('-1')\n\nQuiet and signaling NaNs are also included in the total ordering. The result\nof this function is Decimal('0') if both operands have the same representation,\nDecimal('-1') if the first operand is lower in the total order than the second,\nand Decimal('1') if the first operand is higher in the total order than the\nsecond operand. See the specification for details of the total order.\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.compare_total_mag" => "Compare two operands using their abstract representation rather than their\nvalue as in compare_total(), but ignoring the sign of each operand.\n\nx.compare_total_mag(y) is equivalent to x.copy_abs().compare_total(y.copy_abs()).\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.conjugate" => "Return self.", + "_decimal.Decimal.copy_abs" => "Return the absolute value of the argument. This operation is unaffected by\ncontext and is quiet: no flags are changed and no rounding is performed.", + "_decimal.Decimal.copy_negate" => "Return the negation of the argument. This operation is unaffected by context\nand is quiet: no flags are changed and no rounding is performed.", + "_decimal.Decimal.copy_sign" => "Return a copy of the first operand with the sign set to be the same as the\nsign of the second operand. For example:\n\n >>> Decimal('2.3').copy_sign(Decimal('-1.5'))\n Decimal('-2.3')\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.exp" => "Return the value of the (natural) exponential function e**x at the given\nnumber. The function always uses the ROUND_HALF_EVEN mode and the result\nis correctly rounded.", + "_decimal.Decimal.fma" => "Fused multiply-add. Return self*other+third with no rounding of the\nintermediate product self*other.\n\n >>> Decimal(2).fma(3, 5)\n Decimal('11')", + "_decimal.Decimal.from_float" => "Class method that converts a float to a decimal number, exactly.\nSince 0.1 is not exactly representable in binary floating point,\nDecimal.from_float(0.1) is not the same as Decimal('0.1').\n\n >>> Decimal.from_float(0.1)\n Decimal('0.1000000000000000055511151231257827021181583404541015625')\n >>> Decimal.from_float(float('nan'))\n Decimal('NaN')\n >>> Decimal.from_float(float('inf'))\n Decimal('Infinity')\n >>> Decimal.from_float(float('-inf'))\n Decimal('-Infinity')", + "_decimal.Decimal.from_number" => "Class method that converts a real number to a decimal number, exactly.\n\n >>> Decimal.from_number(314) # int\n Decimal('314')\n >>> Decimal.from_number(0.1) # float\n Decimal('0.1000000000000000055511151231257827021181583404541015625')\n >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n Decimal('3.14')", + "_decimal.Decimal.is_canonical" => "Return True if the argument is canonical and False otherwise. Currently,\na Decimal instance is always canonical, so this operation always returns\nTrue.", + "_decimal.Decimal.is_finite" => "Return True if the argument is a finite number, and False if the argument\nis infinite or a NaN.", + "_decimal.Decimal.is_infinite" => "Return True if the argument is either positive or negative infinity and\nFalse otherwise.", + "_decimal.Decimal.is_nan" => "Return True if the argument is a (quiet or signaling) NaN and False\notherwise.", + "_decimal.Decimal.is_normal" => "Return True if the argument is a normal finite non-zero number with an\nadjusted exponent greater than or equal to Emin. Return False if the\nargument is zero, subnormal, infinite or a NaN.", + "_decimal.Decimal.is_qnan" => "Return True if the argument is a quiet NaN, and False otherwise.", + "_decimal.Decimal.is_signed" => "Return True if the argument has a negative sign and False otherwise.\nNote that both zeros and NaNs can carry signs.", + "_decimal.Decimal.is_snan" => "Return True if the argument is a signaling NaN and False otherwise.", + "_decimal.Decimal.is_subnormal" => "Return True if the argument is subnormal, and False otherwise. A number is\nsubnormal if it is non-zero, finite, and has an adjusted exponent less\nthan Emin.", + "_decimal.Decimal.is_zero" => "Return True if the argument is a (positive or negative) zero and False\notherwise.", + "_decimal.Decimal.ln" => "Return the natural (base e) logarithm of the operand. The function always\nuses the ROUND_HALF_EVEN mode and the result is correctly rounded.", + "_decimal.Decimal.log10" => "Return the base ten logarithm of the operand. The function always uses the\nROUND_HALF_EVEN mode and the result is correctly rounded.", + "_decimal.Decimal.logb" => "For a non-zero number, return the adjusted exponent of the operand as a\nDecimal instance. If the operand is a zero, then Decimal('-Infinity') is\nreturned and the DivisionByZero condition is raised. If the operand is\nan infinity then Decimal('Infinity') is returned.", + "_decimal.Decimal.logical_and" => "Applies an 'and' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.logical_invert" => "Invert all its digits.\n\nThe self must be logical number.", + "_decimal.Decimal.logical_or" => "Applies an 'or' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.logical_xor" => "Applies an 'xor' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.max" => "Maximum of self and other. If one operand is a quiet NaN and the other is\nnumeric, the numeric operand is returned.", + "_decimal.Decimal.max_mag" => "Similar to the max() method, but the comparison is done using the absolute\nvalues of the operands.", + "_decimal.Decimal.min" => "Minimum of self and other. If one operand is a quiet NaN and the other is\nnumeric, the numeric operand is returned.", + "_decimal.Decimal.min_mag" => "Similar to the min() method, but the comparison is done using the absolute\nvalues of the operands.", + "_decimal.Decimal.next_minus" => "Return the largest number representable in the given context (or in the\ncurrent default context if no context is given) that is smaller than the\ngiven operand.", + "_decimal.Decimal.next_plus" => "Return the smallest number representable in the given context (or in the\ncurrent default context if no context is given) that is larger than the\ngiven operand.", + "_decimal.Decimal.next_toward" => "If the two operands are unequal, return the number closest to the first\noperand in the direction of the second operand. If both operands are\nnumerically equal, return a copy of the first operand with the sign set\nto be the same as the sign of the second operand.", + "_decimal.Decimal.normalize" => "Normalize the number by stripping the rightmost trailing zeros and\nconverting any result equal to Decimal('0') to Decimal('0e0'). Used\nfor producing canonical values for members of an equivalence class.\nFor example, Decimal('32.100') and Decimal('0.321000e+2') both normalize\nto the equivalent value Decimal('32.1').", + "_decimal.Decimal.number_class" => "Return a string describing the class of the operand. The returned value\nis one of the following ten strings:\n\n * '-Infinity', indicating that the operand is negative infinity.\n * '-Normal', indicating that the operand is a negative normal number.\n * '-Subnormal', indicating that the operand is negative and subnormal.\n * '-Zero', indicating that the operand is a negative zero.\n * '+Zero', indicating that the operand is a positive zero.\n * '+Subnormal', indicating that the operand is positive and subnormal.\n * '+Normal', indicating that the operand is a positive normal number.\n * '+Infinity', indicating that the operand is positive infinity.\n * 'NaN', indicating that the operand is a quiet NaN (Not a Number).\n * 'sNaN', indicating that the operand is a signaling NaN.", + "_decimal.Decimal.quantize" => "Return a value equal to the first operand after rounding and having the\nexponent of the second operand.\n\n >>> Decimal('1.41421356').quantize(Decimal('1.000'))\n Decimal('1.414')\n\nUnlike other operations, if the length of the coefficient after the quantize\noperation would be greater than precision, then an InvalidOperation is signaled.\nThis guarantees that, unless there is an error condition, the quantized exponent\nis always equal to that of the right-hand operand.\n\nAlso unlike other operations, quantize never signals Underflow, even if the\nresult is subnormal and inexact.\n\nIf the exponent of the second operand is larger than that of the first, then\nrounding may be necessary. In this case, the rounding mode is determined by the\nrounding argument if given, else by the given context argument; if neither\nargument is given, the rounding mode of the current thread's context is used.", + "_decimal.Decimal.radix" => "Return Decimal(10), the radix (base) in which the Decimal class does\nall its arithmetic. Included for compatibility with the specification.", + "_decimal.Decimal.remainder_near" => "Return the remainder from dividing self by other. This differs from\nself % other in that the sign of the remainder is chosen so as to minimize\nits absolute value. More precisely, the return value is self - n * other\nwhere n is the integer nearest to the exact value of self / other, and\nif two integers are equally near then the even one is chosen.\n\nIf the result is zero then its sign will be the sign of self.", + "_decimal.Decimal.rotate" => "Return the result of rotating the digits of the first operand by an amount\nspecified by the second operand. The second operand must be an integer in\nthe range -precision through precision. The absolute value of the second\noperand gives the number of places to rotate. If the second operand is\npositive then rotation is to the left; otherwise rotation is to the right.\nThe coefficient of the first operand is padded on the left with zeros to\nlength precision if necessary. The sign and exponent of the first operand are\nunchanged.", + "_decimal.Decimal.same_quantum" => "Test whether self and other have the same exponent or whether both are NaN.\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.scaleb" => "Return the first operand with the exponent adjusted the second. Equivalently,\nreturn the first operand multiplied by 10**other. The second operand must be\nan integer.", + "_decimal.Decimal.shift" => "Return the result of shifting the digits of the first operand by an amount\nspecified by the second operand. The second operand must be an integer in\nthe range -precision through precision. The absolute value of the second\noperand gives the number of places to shift. If the second operand is\npositive, then the shift is to the left; otherwise the shift is to the\nright. Digits shifted into the coefficient are zeros. The sign and exponent\nof the first operand are unchanged.", + "_decimal.Decimal.sqrt" => "Return the square root of the argument to full precision. The result is\ncorrectly rounded using the ROUND_HALF_EVEN rounding mode.", + "_decimal.Decimal.to_eng_string" => "Convert to an engineering-type string. Engineering notation has an exponent\nwhich is a multiple of 3, so there are up to 3 digits left of the decimal\nplace. For example, Decimal('123E+1') is converted to Decimal('1.23E+3').\n\nThe value of context.capitals determines whether the exponent sign is lower\nor upper case. Otherwise, the context does not affect the operation.", + "_decimal.Decimal.to_integral" => "Identical to the to_integral_value() method. The to_integral() name has been\nkept for compatibility with older versions.", + "_decimal.Decimal.to_integral_exact" => "Round to the nearest integer, signaling Inexact or Rounded as appropriate if\nrounding occurs. The rounding mode is determined by the rounding parameter\nif given, else by the given context. If neither parameter is given, then the\nrounding mode of the current default context is used.", + "_decimal.Decimal.to_integral_value" => "Round to the nearest integer without signaling Inexact or Rounded. The\nrounding mode is determined by the rounding parameter if given, else by\nthe given context. If neither parameter is given, then the rounding mode\nof the current default context is used.", + "_decimal.DecimalException.__delattr__" => "Implement delattr(self, name).", + "_decimal.DecimalException.__eq__" => "Return self==value.", + "_decimal.DecimalException.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DecimalException.__ge__" => "Return self>=value.", + "_decimal.DecimalException.__getattribute__" => "Return getattr(self, name).", + "_decimal.DecimalException.__getstate__" => "Helper for pickle.", + "_decimal.DecimalException.__gt__" => "Return self>value.", + "_decimal.DecimalException.__hash__" => "Return hash(self).", + "_decimal.DecimalException.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DecimalException.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DecimalException.__le__" => "Return self<=value.", + "_decimal.DecimalException.__lt__" => "Return self<value.", + "_decimal.DecimalException.__ne__" => "Return self!=value.", + "_decimal.DecimalException.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DecimalException.__reduce_ex__" => "Helper for pickle.", + "_decimal.DecimalException.__repr__" => "Return repr(self).", + "_decimal.DecimalException.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DecimalException.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DecimalException.__str__" => "Return str(self).", + "_decimal.DecimalException.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DecimalException.__weakref__" => "list of weak references to the object", + "_decimal.DecimalException.add_note" => "Add a note to the exception", + "_decimal.DecimalException.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DecimalTuple" => "DecimalTuple(sign, digits, exponent)", + "_decimal.DecimalTuple.__add__" => "Return self+value.", + "_decimal.DecimalTuple.__class_getitem__" => "See PEP 585", + "_decimal.DecimalTuple.__contains__" => "Return bool(key in self).", + "_decimal.DecimalTuple.__delattr__" => "Implement delattr(self, name).", + "_decimal.DecimalTuple.__eq__" => "Return self==value.", + "_decimal.DecimalTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DecimalTuple.__ge__" => "Return self>=value.", + "_decimal.DecimalTuple.__getattribute__" => "Return getattr(self, name).", + "_decimal.DecimalTuple.__getitem__" => "Return self[key].", + "_decimal.DecimalTuple.__getnewargs__" => "Return self as a plain tuple. Used by copy and pickle.", + "_decimal.DecimalTuple.__getstate__" => "Helper for pickle.", + "_decimal.DecimalTuple.__gt__" => "Return self>value.", + "_decimal.DecimalTuple.__hash__" => "Return hash(self).", + "_decimal.DecimalTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DecimalTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DecimalTuple.__iter__" => "Implement iter(self).", + "_decimal.DecimalTuple.__le__" => "Return self<=value.", + "_decimal.DecimalTuple.__len__" => "Return len(self).", + "_decimal.DecimalTuple.__lt__" => "Return self<value.", + "_decimal.DecimalTuple.__mul__" => "Return self*value.", + "_decimal.DecimalTuple.__ne__" => "Return self!=value.", + "_decimal.DecimalTuple.__new__" => "Create new instance of DecimalTuple(sign, digits, exponent)", + "_decimal.DecimalTuple.__reduce__" => "Helper for pickle.", + "_decimal.DecimalTuple.__reduce_ex__" => "Helper for pickle.", + "_decimal.DecimalTuple.__replace__" => "Return a new DecimalTuple object replacing specified fields with new values", + "_decimal.DecimalTuple.__repr__" => "Return a nicely formatted representation string", + "_decimal.DecimalTuple.__rmul__" => "Return value*self.", + "_decimal.DecimalTuple.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DecimalTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DecimalTuple.__str__" => "Return str(self).", + "_decimal.DecimalTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DecimalTuple._asdict" => "Return a new dict which maps field names to their values.", + "_decimal.DecimalTuple._make" => "Make a new DecimalTuple object from a sequence or iterable", + "_decimal.DecimalTuple._replace" => "Return a new DecimalTuple object replacing specified fields with new values", + "_decimal.DecimalTuple.count" => "Return number of occurrences of value.", + "_decimal.DecimalTuple.digits" => "Alias for field number 1", + "_decimal.DecimalTuple.exponent" => "Alias for field number 2", + "_decimal.DecimalTuple.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_decimal.DecimalTuple.sign" => "Alias for field number 0", + "_decimal.DivisionByZero.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionByZero.__eq__" => "Return self==value.", + "_decimal.DivisionByZero.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionByZero.__ge__" => "Return self>=value.", + "_decimal.DivisionByZero.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionByZero.__getstate__" => "Helper for pickle.", + "_decimal.DivisionByZero.__gt__" => "Return self>value.", + "_decimal.DivisionByZero.__hash__" => "Return hash(self).", + "_decimal.DivisionByZero.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionByZero.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionByZero.__le__" => "Return self<=value.", + "_decimal.DivisionByZero.__lt__" => "Return self<value.", + "_decimal.DivisionByZero.__ne__" => "Return self!=value.", + "_decimal.DivisionByZero.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionByZero.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionByZero.__repr__" => "Return repr(self).", + "_decimal.DivisionByZero.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionByZero.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionByZero.__str__" => "Return str(self).", + "_decimal.DivisionByZero.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionByZero.__weakref__" => "list of weak references to the object", + "_decimal.DivisionByZero.add_note" => "Add a note to the exception", + "_decimal.DivisionByZero.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DivisionImpossible.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionImpossible.__eq__" => "Return self==value.", + "_decimal.DivisionImpossible.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionImpossible.__ge__" => "Return self>=value.", + "_decimal.DivisionImpossible.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionImpossible.__getstate__" => "Helper for pickle.", + "_decimal.DivisionImpossible.__gt__" => "Return self>value.", + "_decimal.DivisionImpossible.__hash__" => "Return hash(self).", + "_decimal.DivisionImpossible.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionImpossible.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionImpossible.__le__" => "Return self<=value.", + "_decimal.DivisionImpossible.__lt__" => "Return self<value.", + "_decimal.DivisionImpossible.__ne__" => "Return self!=value.", + "_decimal.DivisionImpossible.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionImpossible.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionImpossible.__repr__" => "Return repr(self).", + "_decimal.DivisionImpossible.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionImpossible.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionImpossible.__str__" => "Return str(self).", + "_decimal.DivisionImpossible.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionImpossible.__weakref__" => "list of weak references to the object", + "_decimal.DivisionImpossible.add_note" => "Add a note to the exception", + "_decimal.DivisionImpossible.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DivisionUndefined.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionUndefined.__eq__" => "Return self==value.", + "_decimal.DivisionUndefined.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionUndefined.__ge__" => "Return self>=value.", + "_decimal.DivisionUndefined.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionUndefined.__getstate__" => "Helper for pickle.", + "_decimal.DivisionUndefined.__gt__" => "Return self>value.", + "_decimal.DivisionUndefined.__hash__" => "Return hash(self).", + "_decimal.DivisionUndefined.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionUndefined.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionUndefined.__le__" => "Return self<=value.", + "_decimal.DivisionUndefined.__lt__" => "Return self<value.", + "_decimal.DivisionUndefined.__ne__" => "Return self!=value.", + "_decimal.DivisionUndefined.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionUndefined.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionUndefined.__repr__" => "Return repr(self).", + "_decimal.DivisionUndefined.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionUndefined.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionUndefined.__str__" => "Return str(self).", + "_decimal.DivisionUndefined.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionUndefined.__weakref__" => "list of weak references to the object", + "_decimal.DivisionUndefined.add_note" => "Add a note to the exception", + "_decimal.DivisionUndefined.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.FloatOperation.__delattr__" => "Implement delattr(self, name).", + "_decimal.FloatOperation.__eq__" => "Return self==value.", + "_decimal.FloatOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.FloatOperation.__ge__" => "Return self>=value.", + "_decimal.FloatOperation.__getattribute__" => "Return getattr(self, name).", + "_decimal.FloatOperation.__getstate__" => "Helper for pickle.", + "_decimal.FloatOperation.__gt__" => "Return self>value.", + "_decimal.FloatOperation.__hash__" => "Return hash(self).", + "_decimal.FloatOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.FloatOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.FloatOperation.__le__" => "Return self<=value.", + "_decimal.FloatOperation.__lt__" => "Return self<value.", + "_decimal.FloatOperation.__ne__" => "Return self!=value.", + "_decimal.FloatOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.FloatOperation.__reduce_ex__" => "Helper for pickle.", + "_decimal.FloatOperation.__repr__" => "Return repr(self).", + "_decimal.FloatOperation.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.FloatOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.FloatOperation.__str__" => "Return str(self).", + "_decimal.FloatOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.FloatOperation.__weakref__" => "list of weak references to the object", + "_decimal.FloatOperation.add_note" => "Add a note to the exception", + "_decimal.FloatOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.IEEEContext" => "Return a context object initialized to the proper values for one of the\nIEEE interchange formats. The argument must be a multiple of 32 and less\nthan IEEE_CONTEXT_MAX_BITS.", + "_decimal.Inexact.__delattr__" => "Implement delattr(self, name).", + "_decimal.Inexact.__eq__" => "Return self==value.", + "_decimal.Inexact.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Inexact.__ge__" => "Return self>=value.", + "_decimal.Inexact.__getattribute__" => "Return getattr(self, name).", + "_decimal.Inexact.__getstate__" => "Helper for pickle.", + "_decimal.Inexact.__gt__" => "Return self>value.", + "_decimal.Inexact.__hash__" => "Return hash(self).", + "_decimal.Inexact.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Inexact.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Inexact.__le__" => "Return self<=value.", + "_decimal.Inexact.__lt__" => "Return self<value.", + "_decimal.Inexact.__ne__" => "Return self!=value.", + "_decimal.Inexact.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Inexact.__reduce_ex__" => "Helper for pickle.", + "_decimal.Inexact.__repr__" => "Return repr(self).", + "_decimal.Inexact.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Inexact.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Inexact.__str__" => "Return str(self).", + "_decimal.Inexact.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Inexact.__weakref__" => "list of weak references to the object", + "_decimal.Inexact.add_note" => "Add a note to the exception", + "_decimal.Inexact.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.InvalidContext.__delattr__" => "Implement delattr(self, name).", + "_decimal.InvalidContext.__eq__" => "Return self==value.", + "_decimal.InvalidContext.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.InvalidContext.__ge__" => "Return self>=value.", + "_decimal.InvalidContext.__getattribute__" => "Return getattr(self, name).", + "_decimal.InvalidContext.__getstate__" => "Helper for pickle.", + "_decimal.InvalidContext.__gt__" => "Return self>value.", + "_decimal.InvalidContext.__hash__" => "Return hash(self).", + "_decimal.InvalidContext.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.InvalidContext.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.InvalidContext.__le__" => "Return self<=value.", + "_decimal.InvalidContext.__lt__" => "Return self<value.", + "_decimal.InvalidContext.__ne__" => "Return self!=value.", + "_decimal.InvalidContext.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.InvalidContext.__reduce_ex__" => "Helper for pickle.", + "_decimal.InvalidContext.__repr__" => "Return repr(self).", + "_decimal.InvalidContext.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.InvalidContext.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.InvalidContext.__str__" => "Return str(self).", + "_decimal.InvalidContext.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.InvalidContext.__weakref__" => "list of weak references to the object", + "_decimal.InvalidContext.add_note" => "Add a note to the exception", + "_decimal.InvalidContext.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.InvalidOperation.__delattr__" => "Implement delattr(self, name).", + "_decimal.InvalidOperation.__eq__" => "Return self==value.", + "_decimal.InvalidOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.InvalidOperation.__ge__" => "Return self>=value.", + "_decimal.InvalidOperation.__getattribute__" => "Return getattr(self, name).", + "_decimal.InvalidOperation.__getstate__" => "Helper for pickle.", + "_decimal.InvalidOperation.__gt__" => "Return self>value.", + "_decimal.InvalidOperation.__hash__" => "Return hash(self).", + "_decimal.InvalidOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.InvalidOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.InvalidOperation.__le__" => "Return self<=value.", + "_decimal.InvalidOperation.__lt__" => "Return self<value.", + "_decimal.InvalidOperation.__ne__" => "Return self!=value.", + "_decimal.InvalidOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.InvalidOperation.__reduce_ex__" => "Helper for pickle.", + "_decimal.InvalidOperation.__repr__" => "Return repr(self).", + "_decimal.InvalidOperation.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.InvalidOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.InvalidOperation.__str__" => "Return str(self).", + "_decimal.InvalidOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.InvalidOperation.__weakref__" => "list of weak references to the object", + "_decimal.InvalidOperation.add_note" => "Add a note to the exception", + "_decimal.InvalidOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Overflow.__delattr__" => "Implement delattr(self, name).", + "_decimal.Overflow.__eq__" => "Return self==value.", + "_decimal.Overflow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Overflow.__ge__" => "Return self>=value.", + "_decimal.Overflow.__getattribute__" => "Return getattr(self, name).", + "_decimal.Overflow.__getstate__" => "Helper for pickle.", + "_decimal.Overflow.__gt__" => "Return self>value.", + "_decimal.Overflow.__hash__" => "Return hash(self).", + "_decimal.Overflow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Overflow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Overflow.__le__" => "Return self<=value.", + "_decimal.Overflow.__lt__" => "Return self<value.", + "_decimal.Overflow.__ne__" => "Return self!=value.", + "_decimal.Overflow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Overflow.__reduce_ex__" => "Helper for pickle.", + "_decimal.Overflow.__repr__" => "Return repr(self).", + "_decimal.Overflow.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Overflow.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Overflow.__str__" => "Return str(self).", + "_decimal.Overflow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Overflow.__weakref__" => "list of weak references to the object", + "_decimal.Overflow.add_note" => "Add a note to the exception", + "_decimal.Overflow.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Rounded.__delattr__" => "Implement delattr(self, name).", + "_decimal.Rounded.__eq__" => "Return self==value.", + "_decimal.Rounded.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Rounded.__ge__" => "Return self>=value.", + "_decimal.Rounded.__getattribute__" => "Return getattr(self, name).", + "_decimal.Rounded.__getstate__" => "Helper for pickle.", + "_decimal.Rounded.__gt__" => "Return self>value.", + "_decimal.Rounded.__hash__" => "Return hash(self).", + "_decimal.Rounded.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Rounded.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Rounded.__le__" => "Return self<=value.", + "_decimal.Rounded.__lt__" => "Return self<value.", + "_decimal.Rounded.__ne__" => "Return self!=value.", + "_decimal.Rounded.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Rounded.__reduce_ex__" => "Helper for pickle.", + "_decimal.Rounded.__repr__" => "Return repr(self).", + "_decimal.Rounded.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Rounded.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Rounded.__str__" => "Return str(self).", + "_decimal.Rounded.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Rounded.__weakref__" => "list of weak references to the object", + "_decimal.Rounded.add_note" => "Add a note to the exception", + "_decimal.Rounded.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Subnormal.__delattr__" => "Implement delattr(self, name).", + "_decimal.Subnormal.__eq__" => "Return self==value.", + "_decimal.Subnormal.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Subnormal.__ge__" => "Return self>=value.", + "_decimal.Subnormal.__getattribute__" => "Return getattr(self, name).", + "_decimal.Subnormal.__getstate__" => "Helper for pickle.", + "_decimal.Subnormal.__gt__" => "Return self>value.", + "_decimal.Subnormal.__hash__" => "Return hash(self).", + "_decimal.Subnormal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Subnormal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Subnormal.__le__" => "Return self<=value.", + "_decimal.Subnormal.__lt__" => "Return self<value.", + "_decimal.Subnormal.__ne__" => "Return self!=value.", + "_decimal.Subnormal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Subnormal.__reduce_ex__" => "Helper for pickle.", + "_decimal.Subnormal.__repr__" => "Return repr(self).", + "_decimal.Subnormal.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Subnormal.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Subnormal.__str__" => "Return str(self).", + "_decimal.Subnormal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Subnormal.__weakref__" => "list of weak references to the object", + "_decimal.Subnormal.add_note" => "Add a note to the exception", + "_decimal.Subnormal.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Underflow.__delattr__" => "Implement delattr(self, name).", + "_decimal.Underflow.__eq__" => "Return self==value.", + "_decimal.Underflow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Underflow.__ge__" => "Return self>=value.", + "_decimal.Underflow.__getattribute__" => "Return getattr(self, name).", + "_decimal.Underflow.__getstate__" => "Helper for pickle.", + "_decimal.Underflow.__gt__" => "Return self>value.", + "_decimal.Underflow.__hash__" => "Return hash(self).", + "_decimal.Underflow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Underflow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Underflow.__le__" => "Return self<=value.", + "_decimal.Underflow.__lt__" => "Return self<value.", + "_decimal.Underflow.__ne__" => "Return self!=value.", + "_decimal.Underflow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Underflow.__reduce_ex__" => "Helper for pickle.", + "_decimal.Underflow.__repr__" => "Return repr(self).", + "_decimal.Underflow.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Underflow.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Underflow.__str__" => "Return str(self).", + "_decimal.Underflow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Underflow.__weakref__" => "list of weak references to the object", + "_decimal.Underflow.add_note" => "Add a note to the exception", + "_decimal.Underflow.with_traceback" => "Set self.__traceback__ to tb and return self.", "_decimal.getcontext" => "Get the current default context.", "_decimal.localcontext" => "Return a context manager that will set the default context to a copy of ctx\non entry to the with-statement and restore the previous default context when\nexiting the with-statement. If no context is specified, a copy of the current\ndefault context is used.", "_decimal.setcontext" => "Set a new default context.", + "_elementtree.Element.__bool__" => "True if self else False", + "_elementtree.Element.__delattr__" => "Implement delattr(self, name).", + "_elementtree.Element.__delitem__" => "Delete self[key].", + "_elementtree.Element.__eq__" => "Return self==value.", + "_elementtree.Element.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.Element.__ge__" => "Return self>=value.", + "_elementtree.Element.__getattribute__" => "Return getattr(self, name).", + "_elementtree.Element.__getitem__" => "Return self[key].", + "_elementtree.Element.__gt__" => "Return self>value.", + "_elementtree.Element.__hash__" => "Return hash(self).", + "_elementtree.Element.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.Element.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.Element.__le__" => "Return self<=value.", + "_elementtree.Element.__len__" => "Return len(self).", + "_elementtree.Element.__lt__" => "Return self<value.", + "_elementtree.Element.__ne__" => "Return self!=value.", + "_elementtree.Element.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.Element.__reduce__" => "Helper for pickle.", + "_elementtree.Element.__reduce_ex__" => "Helper for pickle.", + "_elementtree.Element.__repr__" => "Return repr(self).", + "_elementtree.Element.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.Element.__setitem__" => "Set self[key] to value.", + "_elementtree.Element.__str__" => "Return str(self).", + "_elementtree.Element.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.Element.attrib" => "A dictionary containing the element's attributes", + "_elementtree.Element.tag" => "A string identifying what kind of data this element represents", + "_elementtree.Element.tail" => "A string of text directly after the end tag, or None", + "_elementtree.Element.text" => "A string of text directly after the start tag, or None", + "_elementtree.ParseError.__delattr__" => "Implement delattr(self, name).", + "_elementtree.ParseError.__eq__" => "Return self==value.", + "_elementtree.ParseError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.ParseError.__ge__" => "Return self>=value.", + "_elementtree.ParseError.__getattribute__" => "Return getattr(self, name).", + "_elementtree.ParseError.__getstate__" => "Helper for pickle.", + "_elementtree.ParseError.__gt__" => "Return self>value.", + "_elementtree.ParseError.__hash__" => "Return hash(self).", + "_elementtree.ParseError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.ParseError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.ParseError.__le__" => "Return self<=value.", + "_elementtree.ParseError.__lt__" => "Return self<value.", + "_elementtree.ParseError.__ne__" => "Return self!=value.", + "_elementtree.ParseError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.ParseError.__reduce_ex__" => "Helper for pickle.", + "_elementtree.ParseError.__repr__" => "Return repr(self).", + "_elementtree.ParseError.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.ParseError.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.ParseError.__str__" => "Return str(self).", + "_elementtree.ParseError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.ParseError.__weakref__" => "list of weak references to the object", + "_elementtree.ParseError._metadata" => "exception private metadata", + "_elementtree.ParseError.add_note" => "Add a note to the exception", + "_elementtree.ParseError.end_lineno" => "exception end lineno", + "_elementtree.ParseError.end_offset" => "exception end offset", + "_elementtree.ParseError.filename" => "exception filename", + "_elementtree.ParseError.lineno" => "exception lineno", + "_elementtree.ParseError.msg" => "exception msg", + "_elementtree.ParseError.offset" => "exception offset", + "_elementtree.ParseError.print_file_and_line" => "exception print_file_and_line", + "_elementtree.ParseError.text" => "exception text", + "_elementtree.ParseError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_elementtree.TreeBuilder.__delattr__" => "Implement delattr(self, name).", + "_elementtree.TreeBuilder.__eq__" => "Return self==value.", + "_elementtree.TreeBuilder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.TreeBuilder.__ge__" => "Return self>=value.", + "_elementtree.TreeBuilder.__getattribute__" => "Return getattr(self, name).", + "_elementtree.TreeBuilder.__getstate__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__gt__" => "Return self>value.", + "_elementtree.TreeBuilder.__hash__" => "Return hash(self).", + "_elementtree.TreeBuilder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.TreeBuilder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.TreeBuilder.__le__" => "Return self<=value.", + "_elementtree.TreeBuilder.__lt__" => "Return self<value.", + "_elementtree.TreeBuilder.__ne__" => "Return self!=value.", + "_elementtree.TreeBuilder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.TreeBuilder.__reduce__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__reduce_ex__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__repr__" => "Return repr(self).", + "_elementtree.TreeBuilder.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.TreeBuilder.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.TreeBuilder.__str__" => "Return str(self).", + "_elementtree.TreeBuilder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.XMLParser.__delattr__" => "Implement delattr(self, name).", + "_elementtree.XMLParser.__eq__" => "Return self==value.", + "_elementtree.XMLParser.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.XMLParser.__ge__" => "Return self>=value.", + "_elementtree.XMLParser.__getattribute__" => "Return getattr(self, name).", + "_elementtree.XMLParser.__getstate__" => "Helper for pickle.", + "_elementtree.XMLParser.__gt__" => "Return self>value.", + "_elementtree.XMLParser.__hash__" => "Return hash(self).", + "_elementtree.XMLParser.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.XMLParser.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.XMLParser.__le__" => "Return self<=value.", + "_elementtree.XMLParser.__lt__" => "Return self<value.", + "_elementtree.XMLParser.__ne__" => "Return self!=value.", + "_elementtree.XMLParser.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.XMLParser.__reduce__" => "Helper for pickle.", + "_elementtree.XMLParser.__reduce_ex__" => "Helper for pickle.", + "_elementtree.XMLParser.__repr__" => "Return repr(self).", + "_elementtree.XMLParser.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.XMLParser.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.XMLParser.__str__" => "Return str(self).", + "_elementtree.XMLParser.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_elementtree._set_factories" => "Change the factories used to create comments and processing instructions.\n\nFor internal use only.", "_functools" => "Tools that operate on functions.", - "_functools.cmp_to_key" => "Convert a cmp= function into a key= function.\n\nmycmp\n Function that compares two objects.", - "_functools.reduce" => "reduce(function, iterable[, initial], /) -> value\n\nApply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\nThis effectively reduces the iterable to a single value. If initial is present,\nit is placed before the items of the iterable in the calculation, and serves as\na default when the iterable is empty.\n\nFor example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\ncalculates ((((1 + 2) + 3) + 4) + 5).", + "_functools._PlaceholderType" => "The type of the Placeholder singleton.\n\nUsed as a placeholder for partial arguments.", + "_functools._PlaceholderType.__delattr__" => "Implement delattr(self, name).", + "_functools._PlaceholderType.__eq__" => "Return self==value.", + "_functools._PlaceholderType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools._PlaceholderType.__ge__" => "Return self>=value.", + "_functools._PlaceholderType.__getattribute__" => "Return getattr(self, name).", + "_functools._PlaceholderType.__getstate__" => "Helper for pickle.", + "_functools._PlaceholderType.__gt__" => "Return self>value.", + "_functools._PlaceholderType.__hash__" => "Return hash(self).", + "_functools._PlaceholderType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools._PlaceholderType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools._PlaceholderType.__le__" => "Return self<=value.", + "_functools._PlaceholderType.__lt__" => "Return self<value.", + "_functools._PlaceholderType.__ne__" => "Return self!=value.", + "_functools._PlaceholderType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools._PlaceholderType.__reduce_ex__" => "Helper for pickle.", + "_functools._PlaceholderType.__repr__" => "Return repr(self).", + "_functools._PlaceholderType.__setattr__" => "Implement setattr(self, name, value).", + "_functools._PlaceholderType.__sizeof__" => "Size of object in memory, in bytes.", + "_functools._PlaceholderType.__str__" => "Return str(self).", + "_functools._PlaceholderType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools._lru_cache_wrapper" => "Create a cached callable that wraps another function.\n\nuser_function: the function being cached\n\nmaxsize: 0 for no caching\n None for unlimited cache size\n n for a bounded cache\n\ntyped: False cache f(3) and f(3.0) as identical calls\n True cache f(3) and f(3.0) as distinct calls\n\ncache_info_type: namedtuple class with the fields:\n hits misses currsize maxsize", + "_functools._lru_cache_wrapper.__call__" => "Call self as a function.", + "_functools._lru_cache_wrapper.__delattr__" => "Implement delattr(self, name).", + "_functools._lru_cache_wrapper.__eq__" => "Return self==value.", + "_functools._lru_cache_wrapper.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools._lru_cache_wrapper.__ge__" => "Return self>=value.", + "_functools._lru_cache_wrapper.__get__" => "Return an attribute of instance, which is of type owner.", + "_functools._lru_cache_wrapper.__getattribute__" => "Return getattr(self, name).", + "_functools._lru_cache_wrapper.__getstate__" => "Helper for pickle.", + "_functools._lru_cache_wrapper.__gt__" => "Return self>value.", + "_functools._lru_cache_wrapper.__hash__" => "Return hash(self).", + "_functools._lru_cache_wrapper.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools._lru_cache_wrapper.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools._lru_cache_wrapper.__le__" => "Return self<=value.", + "_functools._lru_cache_wrapper.__lt__" => "Return self<value.", + "_functools._lru_cache_wrapper.__ne__" => "Return self!=value.", + "_functools._lru_cache_wrapper.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools._lru_cache_wrapper.__reduce_ex__" => "Helper for pickle.", + "_functools._lru_cache_wrapper.__repr__" => "Return repr(self).", + "_functools._lru_cache_wrapper.__setattr__" => "Implement setattr(self, name, value).", + "_functools._lru_cache_wrapper.__sizeof__" => "Size of object in memory, in bytes.", + "_functools._lru_cache_wrapper.__str__" => "Return str(self).", + "_functools._lru_cache_wrapper.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools._lru_cache_wrapper.cache_clear" => "Clear the cache and cache statistics", + "_functools._lru_cache_wrapper.cache_info" => "Report cache statistics", + "_functools.cmp_to_key" => "Convert a cmp= function into a key= function.\n\n mycmp\n Function that compares two objects.", + "_functools.partial" => "Create a new function with partial application of the given arguments\nand keywords.", + "_functools.partial.__call__" => "Call self as a function.", + "_functools.partial.__class_getitem__" => "See PEP 585", + "_functools.partial.__delattr__" => "Implement delattr(self, name).", + "_functools.partial.__eq__" => "Return self==value.", + "_functools.partial.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools.partial.__ge__" => "Return self>=value.", + "_functools.partial.__get__" => "Return an attribute of instance, which is of type owner.", + "_functools.partial.__getattribute__" => "Return getattr(self, name).", + "_functools.partial.__getstate__" => "Helper for pickle.", + "_functools.partial.__gt__" => "Return self>value.", + "_functools.partial.__hash__" => "Return hash(self).", + "_functools.partial.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools.partial.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools.partial.__le__" => "Return self<=value.", + "_functools.partial.__lt__" => "Return self<value.", + "_functools.partial.__ne__" => "Return self!=value.", + "_functools.partial.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools.partial.__reduce_ex__" => "Helper for pickle.", + "_functools.partial.__repr__" => "Return repr(self).", + "_functools.partial.__setattr__" => "Implement setattr(self, name, value).", + "_functools.partial.__sizeof__" => "Size of object in memory, in bytes.", + "_functools.partial.__str__" => "Return str(self).", + "_functools.partial.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools.partial.args" => "tuple of arguments to future partial calls", + "_functools.partial.func" => "function object to use in future partial calls", + "_functools.partial.keywords" => "dictionary of keyword arguments to future partial calls", + "_functools.reduce" => "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\nThis effectively reduces the iterable to a single value. If initial is present,\nit is placed before the items of the iterable in the calculation, and serves as\na default when the iterable is empty.\n\nFor example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\ncalculates ((((1 + 2) + 3) + 4) + 5).", "_gdbm" => "This module provides an interface to the GNU DBM (GDBM) library.\n\nThis module is quite similar to the dbm module, but uses GDBM instead to\nprovide some additional functionality. Please note that the file formats\ncreated by GDBM and dbm are incompatible.\n\nGDBM objects behave like mappings (dictionaries), except that keys and\nvalues are always immutable bytes-like objects or strings. Printing\na GDBM object doesn't print the keys and values, and the items() and\nvalues() methods are not supported.", - "_gdbm.error.__cause__" => "exception cause", - "_gdbm.error.__context__" => "exception context", "_gdbm.error.__delattr__" => "Implement delattr(self, name).", "_gdbm.error.__eq__" => "Return self==value.", "_gdbm.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -728,12 +4830,12 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_gdbm.error.__str__" => "Return str(self).", "_gdbm.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_gdbm.error.__weakref__" => "list of weak references to the object", - "_gdbm.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "_gdbm.error.add_note" => "Add a note to the exception", "_gdbm.error.errno" => "POSIX exception code", "_gdbm.error.filename" => "exception filename", "_gdbm.error.filename2" => "second exception filename", "_gdbm.error.strerror" => "exception strerror", - "_gdbm.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_gdbm.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_gdbm.open" => "Open a dbm database and return a dbm object.\n\nThe filename argument is the name of the database file.\n\nThe optional flags argument can be 'r' (to open an existing database\nfor reading only -- default), 'w' (to open an existing database for\nreading and writing), 'c' (which creates the database if it doesn't\nexist), or 'n' (which always creates a new empty database).\n\nSome versions of gdbm support additional flags which must be\nappended to one of the flags described above. The module constant\n'open_flags' is a string of valid additional flags. The 'f' flag\nopens the database in fast mode; altered data will not automatically\nbe written to the disk after every change. This results in faster\nwrites to the database, but may result in an inconsistent database\nif the program crashes while the database is still open. Use the\nsync() method to force any unwritten data to be written to the disk.\nThe 's' flag causes all database operations to be synchronized to\ndisk. The 'u' flag disables locking of the database file.\n\nThe optional mode argument is the Unix mode of the file, used only\nwhen the database has to be created. It defaults to octal 0o666.", "_hashlib" => "OpenSSL interface for hashlib module", "_hashlib.HASH" => "A hash is an object used to calculate a checksum of a string of information.\n\nMethods:\n\nupdate() -- updates the current digest with an additional string\ndigest() -- return the current digest value\nhexdigest() -- return the current digest as a string of hexadecimal digits\ncopy() -- return a copy of the current hash object\n\nAttributes:\n\nname -- the hash algorithm being used by this object\ndigest_size -- number of bytes in this hashes output", @@ -814,8 +4916,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_hashlib.HMAC.digest" => "Return the digest of the bytes passed to the update() method so far.", "_hashlib.HMAC.hexdigest" => "Return hexadecimal digest of the bytes passed to the update() method so far.\n\nThis may be used to exchange the value safely in email or other non-binary\nenvironments.", "_hashlib.HMAC.update" => "Update the HMAC object with msg.", - "_hashlib.UnsupportedDigestmodError.__cause__" => "exception cause", - "_hashlib.UnsupportedDigestmodError.__context__" => "exception context", "_hashlib.UnsupportedDigestmodError.__delattr__" => "Implement delattr(self, name).", "_hashlib.UnsupportedDigestmodError.__eq__" => "Return self==value.", "_hashlib.UnsupportedDigestmodError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -837,8 +4937,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_hashlib.UnsupportedDigestmodError.__str__" => "Return str(self).", "_hashlib.UnsupportedDigestmodError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_hashlib.UnsupportedDigestmodError.__weakref__" => "list of weak references to the object", - "_hashlib.UnsupportedDigestmodError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_hashlib.UnsupportedDigestmodError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_hashlib.UnsupportedDigestmodError.add_note" => "Add a note to the exception", + "_hashlib.UnsupportedDigestmodError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_hashlib.compare_digest" => "Return 'a == b'.\n\nThis function uses an approach designed to prevent\ntiming analysis, making it appropriate for cryptography.\n\na and b must both be of the same type: either str (ASCII only),\nor any bytes-like object.\n\nNote: If a and b are of different lengths, or if an error occurs,\na timing attack could theoretically reveal information about the\ntypes and lengths of a and b--but not their values.", "_hashlib.get_fips_mode" => "Determine the OpenSSL FIPS mode of operation.\n\nFor OpenSSL 3.0.0 and newer it returns the state of the default provider\nin the default OSSL context. It's not quite the same as FIPS_mode() but good\nenough for unittests.\n\nEffectively any non-zero return value indicates FIPS mode;\nvalues other than 1 may have additional significance.", "_hashlib.hmac_digest" => "Single-shot HMAC.", @@ -859,16 +4959,67 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_hashlib.pbkdf2_hmac" => "Password based key derivation function 2 (PKCS #5 v2.0) with HMAC as pseudorandom function.", "_hashlib.scrypt" => "scrypt password-based key derivation function.", "_heapq" => "Heap queue algorithm (a.k.a. priority queue).\n\nHeaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for\nall k, counting elements from 0. For the sake of comparison,\nnon-existing elements are considered to be infinite. The interesting\nproperty of a heap is that a[0] is always its smallest element.\n\nUsage:\n\nheap = [] # creates an empty heap\nheappush(heap, item) # pushes a new item on the heap\nitem = heappop(heap) # pops the smallest item from the heap\nitem = heap[0] # smallest item on the heap without popping it\nheapify(x) # transforms list into a heap, in-place, in linear time\nitem = heapreplace(heap, item) # pops and returns smallest item, and adds\n # new item; the heap size is unchanged\n\nOur API differs from textbook heap algorithms as follows:\n\n- We use 0-based indexing. This makes the relationship between the\n index for a node and the indexes for its children slightly less\n obvious, but is more suitable since Python uses 0-based indexing.\n\n- Our heappop() method returns the smallest item, not the largest.\n\nThese two make it possible to view the heap as a regular Python list\nwithout surprises: heap[0] is the smallest item, and heap.sort()\nmaintains the heap invariant!", - "_heapq._heapify_max" => "Maxheap variant of heapify.", - "_heapq._heappop_max" => "Maxheap variant of heappop.", - "_heapq._heapreplace_max" => "Maxheap variant of heapreplace.", "_heapq.heapify" => "Transform list into a heap, in-place, in O(len(heap)) time.", + "_heapq.heapify_max" => "Maxheap variant of heapify.", "_heapq.heappop" => "Pop the smallest item off the heap, maintaining the heap invariant.", + "_heapq.heappop_max" => "Maxheap variant of heappop.", "_heapq.heappush" => "Push item onto heap, maintaining the heap invariant.", + "_heapq.heappush_max" => "Push item onto max heap, maintaining the heap invariant.", "_heapq.heappushpop" => "Push item on the heap, then pop and return the smallest item from the heap.\n\nThe combined action runs more efficiently than heappush() followed by\na separate call to heappop().", + "_heapq.heappushpop_max" => "Maxheap variant of heappushpop.\n\nThe combined action runs more efficiently than heappush_max() followed by\na separate call to heappop_max().", "_heapq.heapreplace" => "Pop and return the current smallest value, and add the new item.\n\nThis is more efficient than heappop() followed by heappush(), and can be\nmore appropriate when using a fixed-size heap. Note that the value\nreturned may be larger than item! That constrains reasonable uses of\nthis routine unless written as part of a conditional replacement:\n\n if item > heap[0]:\n item = heapreplace(heap, item)", + "_heapq.heapreplace_max" => "Maxheap variant of heapreplace.", + "_hmac.HMAC.__delattr__" => "Implement delattr(self, name).", + "_hmac.HMAC.__eq__" => "Return self==value.", + "_hmac.HMAC.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hmac.HMAC.__ge__" => "Return self>=value.", + "_hmac.HMAC.__getattribute__" => "Return getattr(self, name).", + "_hmac.HMAC.__getstate__" => "Helper for pickle.", + "_hmac.HMAC.__gt__" => "Return self>value.", + "_hmac.HMAC.__hash__" => "Return hash(self).", + "_hmac.HMAC.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hmac.HMAC.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hmac.HMAC.__le__" => "Return self<=value.", + "_hmac.HMAC.__lt__" => "Return self<value.", + "_hmac.HMAC.__ne__" => "Return self!=value.", + "_hmac.HMAC.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hmac.HMAC.__reduce__" => "Helper for pickle.", + "_hmac.HMAC.__reduce_ex__" => "Helper for pickle.", + "_hmac.HMAC.__repr__" => "Return repr(self).", + "_hmac.HMAC.__setattr__" => "Implement setattr(self, name, value).", + "_hmac.HMAC.__sizeof__" => "Size of object in memory, in bytes.", + "_hmac.HMAC.__str__" => "Return str(self).", + "_hmac.HMAC.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hmac.HMAC.copy" => "Return a copy (\"clone\") of the HMAC object.", + "_hmac.HMAC.digest" => "Return the digest of the bytes passed to the update() method so far.\n\nThis method may raise a MemoryError.", + "_hmac.HMAC.hexdigest" => "Return hexadecimal digest of the bytes passed to the update() method so far.\n\nThis may be used to exchange the value safely in email or other non-binary\nenvironments.\n\nThis method may raise a MemoryError.", + "_hmac.HMAC.update" => "Update the HMAC object with the given message.", + "_hmac.UnknownHashError.__delattr__" => "Implement delattr(self, name).", + "_hmac.UnknownHashError.__eq__" => "Return self==value.", + "_hmac.UnknownHashError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hmac.UnknownHashError.__ge__" => "Return self>=value.", + "_hmac.UnknownHashError.__getattribute__" => "Return getattr(self, name).", + "_hmac.UnknownHashError.__getstate__" => "Helper for pickle.", + "_hmac.UnknownHashError.__gt__" => "Return self>value.", + "_hmac.UnknownHashError.__hash__" => "Return hash(self).", + "_hmac.UnknownHashError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hmac.UnknownHashError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hmac.UnknownHashError.__le__" => "Return self<=value.", + "_hmac.UnknownHashError.__lt__" => "Return self<value.", + "_hmac.UnknownHashError.__ne__" => "Return self!=value.", + "_hmac.UnknownHashError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hmac.UnknownHashError.__reduce_ex__" => "Helper for pickle.", + "_hmac.UnknownHashError.__repr__" => "Return repr(self).", + "_hmac.UnknownHashError.__setattr__" => "Implement setattr(self, name, value).", + "_hmac.UnknownHashError.__sizeof__" => "Size of object in memory, in bytes.", + "_hmac.UnknownHashError.__str__" => "Return str(self).", + "_hmac.UnknownHashError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hmac.UnknownHashError.__weakref__" => "list of weak references to the object", + "_hmac.UnknownHashError.add_note" => "Add a note to the exception", + "_hmac.UnknownHashError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_hmac.new" => "Return a new HMAC object.", "_imp" => "(Extremely) low-level import machinery bits as used by importlib.", - "_imp._fix_co_filename" => "Changes code.co_filename to specify the passed-in file path.\n\ncode\n Code object to change.\npath\n File path to use.", + "_imp._fix_co_filename" => "Changes code.co_filename to specify the passed-in file path.\n\n code\n Code object to change.\n path\n File path to use.", "_imp._frozen_module_names" => "Returns the list of available frozen modules.", "_imp._override_frozen_modules_for_tests" => "(internal-only) Override PyConfig.use_frozen_modules.\n\n(-1: \"off\", 1: \"on\", 0: no override)\nSee frozen_modules() in Lib/test/support/import_helper.py.", "_imp._override_multi_interp_extensions_check" => "(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.\n\n(-1: \"never\", 1: \"always\", 0: no override)", @@ -887,8 +5038,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_imp.lock_held" => "Return True if the import lock is currently held, else False.\n\nOn platforms without threads, return False.", "_imp.release_lock" => "Release the interpreter's import lock.\n\nOn platforms without threads, this function does nothing.", "_interpchannels" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", - "_interpchannels.ChannelClosedError.__cause__" => "exception cause", - "_interpchannels.ChannelClosedError.__context__" => "exception context", "_interpchannels.ChannelClosedError.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelClosedError.__eq__" => "Return self==value.", "_interpchannels.ChannelClosedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -910,10 +5059,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelClosedError.__str__" => "Return str(self).", "_interpchannels.ChannelClosedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_interpchannels.ChannelClosedError.__weakref__" => "list of weak references to the object", - "_interpchannels.ChannelClosedError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_interpchannels.ChannelClosedError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_interpchannels.ChannelEmptyError.__cause__" => "exception cause", - "_interpchannels.ChannelEmptyError.__context__" => "exception context", + "_interpchannels.ChannelClosedError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelClosedError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpchannels.ChannelEmptyError.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelEmptyError.__eq__" => "Return self==value.", "_interpchannels.ChannelEmptyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -935,10 +5082,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelEmptyError.__str__" => "Return str(self).", "_interpchannels.ChannelEmptyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_interpchannels.ChannelEmptyError.__weakref__" => "list of weak references to the object", - "_interpchannels.ChannelEmptyError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_interpchannels.ChannelEmptyError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_interpchannels.ChannelError.__cause__" => "exception cause", - "_interpchannels.ChannelError.__context__" => "exception context", + "_interpchannels.ChannelEmptyError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelEmptyError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpchannels.ChannelError.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelError.__eq__" => "Return self==value.", "_interpchannels.ChannelError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -960,8 +5105,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelError.__str__" => "Return str(self).", "_interpchannels.ChannelError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_interpchannels.ChannelError.__weakref__" => "list of weak references to the object", - "_interpchannels.ChannelError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_interpchannels.ChannelError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_interpchannels.ChannelError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpchannels.ChannelID" => "A channel ID identifies a channel and may be used as an int.", "_interpchannels.ChannelID.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelID.__eq__" => "Return self==value.", @@ -1036,8 +5181,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelInfo.recv_released" => "current interpreter *was* bound to the recv end", "_interpchannels.ChannelInfo.send_associated" => "current interpreter is bound to the send end", "_interpchannels.ChannelInfo.send_released" => "current interpreter *was* bound to the send end", - "_interpchannels.ChannelNotEmptyError.__cause__" => "exception cause", - "_interpchannels.ChannelNotEmptyError.__context__" => "exception context", "_interpchannels.ChannelNotEmptyError.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelNotEmptyError.__eq__" => "Return self==value.", "_interpchannels.ChannelNotEmptyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -1059,10 +5202,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelNotEmptyError.__str__" => "Return str(self).", "_interpchannels.ChannelNotEmptyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_interpchannels.ChannelNotEmptyError.__weakref__" => "list of weak references to the object", - "_interpchannels.ChannelNotEmptyError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_interpchannels.ChannelNotEmptyError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_interpchannels.ChannelNotFoundError.__cause__" => "exception cause", - "_interpchannels.ChannelNotFoundError.__context__" => "exception context", + "_interpchannels.ChannelNotEmptyError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelNotEmptyError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpchannels.ChannelNotFoundError.__delattr__" => "Implement delattr(self, name).", "_interpchannels.ChannelNotFoundError.__eq__" => "Return self==value.", "_interpchannels.ChannelNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -1084,8 +5225,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.ChannelNotFoundError.__str__" => "Return str(self).", "_interpchannels.ChannelNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_interpchannels.ChannelNotFoundError.__weakref__" => "list of weak references to the object", - "_interpchannels.ChannelNotFoundError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_interpchannels.ChannelNotFoundError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_interpchannels.ChannelNotFoundError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpchannels.close" => "channel_close(cid, *, send=None, recv=None, force=False)\n\nClose the channel for all interpreters.\n\nIf the channel is empty then the keyword args are ignored and both\nends are immediately closed. Otherwise, if 'force' is True then\nall queued items are released and both ends are immediately\nclosed.\n\nIf the channel is not empty *and* 'force' is False then following\nhappens:\n\n * recv is True (regardless of send):\n - raise ChannelNotEmptyError\n * recv is None and send is None:\n - raise ChannelNotEmptyError\n * send is True and recv is not True:\n - fully close the 'send' end\n - close the 'recv' end to interpreters not already receiving\n - fully close it once empty\n\nClosing an already closed channel results in a ChannelClosedError.\n\nOnce the channel's ID has no more ref counts in any interpreter\nthe channel will be destroyed.", "_interpchannels.create" => "channel_create(unboundop) -> cid\n\nCreate a new cross-interpreter channel and return a unique generated ID.", "_interpchannels.destroy" => "channel_destroy(cid)\n\nClose and finalize the channel. Afterward attempts to use the channel\nwill behave as though it never existed.", @@ -1099,16 +5240,63 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpchannels.send" => "channel_send(cid, obj, *, blocking=True, timeout=None)\n\nAdd the object's data to the channel's queue.\nBy default this waits for the object to be received.", "_interpchannels.send_buffer" => "channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\nAdd the object's buffer to the channel's queue.\nBy default this waits for the object to be received.", "_interpqueues" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", + "_interpqueues.QueueError" => "Indicates that a queue-related error happened.", + "_interpqueues.QueueError.__delattr__" => "Implement delattr(self, name).", + "_interpqueues.QueueError.__eq__" => "Return self==value.", + "_interpqueues.QueueError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpqueues.QueueError.__ge__" => "Return self>=value.", + "_interpqueues.QueueError.__getattribute__" => "Return getattr(self, name).", + "_interpqueues.QueueError.__getstate__" => "Helper for pickle.", + "_interpqueues.QueueError.__gt__" => "Return self>value.", + "_interpqueues.QueueError.__hash__" => "Return hash(self).", + "_interpqueues.QueueError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpqueues.QueueError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpqueues.QueueError.__le__" => "Return self<=value.", + "_interpqueues.QueueError.__lt__" => "Return self<value.", + "_interpqueues.QueueError.__ne__" => "Return self!=value.", + "_interpqueues.QueueError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpqueues.QueueError.__reduce_ex__" => "Helper for pickle.", + "_interpqueues.QueueError.__repr__" => "Return repr(self).", + "_interpqueues.QueueError.__setattr__" => "Implement setattr(self, name, value).", + "_interpqueues.QueueError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpqueues.QueueError.__str__" => "Return str(self).", + "_interpqueues.QueueError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpqueues.QueueError.__weakref__" => "list of weak references to the object", + "_interpqueues.QueueError.add_note" => "Add a note to the exception", + "_interpqueues.QueueError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpqueues.QueueNotFoundError.__delattr__" => "Implement delattr(self, name).", + "_interpqueues.QueueNotFoundError.__eq__" => "Return self==value.", + "_interpqueues.QueueNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpqueues.QueueNotFoundError.__ge__" => "Return self>=value.", + "_interpqueues.QueueNotFoundError.__getattribute__" => "Return getattr(self, name).", + "_interpqueues.QueueNotFoundError.__getstate__" => "Helper for pickle.", + "_interpqueues.QueueNotFoundError.__gt__" => "Return self>value.", + "_interpqueues.QueueNotFoundError.__hash__" => "Return hash(self).", + "_interpqueues.QueueNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpqueues.QueueNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpqueues.QueueNotFoundError.__le__" => "Return self<=value.", + "_interpqueues.QueueNotFoundError.__lt__" => "Return self<value.", + "_interpqueues.QueueNotFoundError.__ne__" => "Return self!=value.", + "_interpqueues.QueueNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpqueues.QueueNotFoundError.__reduce_ex__" => "Helper for pickle.", + "_interpqueues.QueueNotFoundError.__repr__" => "Return repr(self).", + "_interpqueues.QueueNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "_interpqueues.QueueNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpqueues.QueueNotFoundError.__str__" => "Return str(self).", + "_interpqueues.QueueNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpqueues.QueueNotFoundError.__weakref__" => "list of weak references to the object", + "_interpqueues.QueueNotFoundError.add_note" => "Add a note to the exception", + "_interpqueues.QueueNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_interpqueues.bind" => "bind(qid)\n\nTake a reference to the identified queue.\nThe queue is not destroyed until there are no references left.", - "_interpqueues.create" => "create(maxsize, fmt, unboundop) -> qid\n\nCreate a new cross-interpreter queue and return its unique generated ID.\nIt is a new reference as though bind() had been called on the queue.\n\nThe caller is responsible for calling destroy() for the new queue\nbefore the runtime is finalized.", + "_interpqueues.create" => "create(maxsize, unboundop, fallback) -> qid\n\nCreate a new cross-interpreter queue and return its unique generated ID.\nIt is a new reference as though bind() had been called on the queue.\n\nThe caller is responsible for calling destroy() for the new queue\nbefore the runtime is finalized.", "_interpqueues.destroy" => "destroy(qid)\n\nClear and destroy the queue. Afterward attempts to use the queue\nwill behave as though it never existed.", - "_interpqueues.get" => "get(qid) -> (obj, fmt)\n\nReturn a new object from the data at the front of the queue.\nThe object's format is also returned.\n\nIf there is nothing to receive then raise QueueEmpty.", + "_interpqueues.get" => "get(qid) -> (obj, unboundop)\n\nReturn a new object from the data at the front of the queue.\nThe unbound op is also returned.\n\nIf there is nothing to receive then raise QueueEmpty.", "_interpqueues.get_count" => "get_count(qid)\n\nReturn the number of items in the queue.", "_interpqueues.get_maxsize" => "get_maxsize(qid)\n\nReturn the maximum number of items in the queue.", "_interpqueues.get_queue_defaults" => "get_queue_defaults(qid)\n\nReturn the queue's default values, set when it was created.", "_interpqueues.is_full" => "is_full(qid)\n\nReturn true if the queue has a maxsize and has reached it.", - "_interpqueues.list_all" => "list_all() -> [(qid, fmt)]\n\nReturn the list of IDs for all queues.\nEach corresponding default format is also included.", - "_interpqueues.put" => "put(qid, obj, fmt)\n\nAdd the object's data to the queue.", + "_interpqueues.list_all" => "list_all() -> [(qid, unboundop, fallback)]\n\nReturn the list of IDs for all queues.\nEach corresponding default unbound op and fallback is also included.", + "_interpqueues.put" => "put(qid, obj)\n\nAdd the object's data to the queue.", "_interpqueues.release" => "release(qid)\n\nRelease a reference to the queue.\nThe queue is destroyed once there are no references left.", "_interpreters" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", "_interpreters.CrossInterpreterBufferView.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", @@ -1133,9 +5321,78 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpreters.CrossInterpreterBufferView.__sizeof__" => "Size of object in memory, in bytes.", "_interpreters.CrossInterpreterBufferView.__str__" => "Return str(self).", "_interpreters.CrossInterpreterBufferView.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_interpreters.call" => "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\nCall the provided object in the identified interpreter.\nPass the given args and kwargs, if possible.\n\n\"callable\" may be a plain function with no free vars that takes\nno arguments.\n\nThe function's code object is used and all its state\nis ignored, including its __globals__ dict.", + "_interpreters.InterpreterError" => "A cross-interpreter operation failed", + "_interpreters.InterpreterError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.InterpreterError.__eq__" => "Return self==value.", + "_interpreters.InterpreterError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.InterpreterError.__ge__" => "Return self>=value.", + "_interpreters.InterpreterError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.InterpreterError.__getstate__" => "Helper for pickle.", + "_interpreters.InterpreterError.__gt__" => "Return self>value.", + "_interpreters.InterpreterError.__hash__" => "Return hash(self).", + "_interpreters.InterpreterError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.InterpreterError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.InterpreterError.__le__" => "Return self<=value.", + "_interpreters.InterpreterError.__lt__" => "Return self<value.", + "_interpreters.InterpreterError.__ne__" => "Return self!=value.", + "_interpreters.InterpreterError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.InterpreterError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.InterpreterError.__repr__" => "Return repr(self).", + "_interpreters.InterpreterError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.InterpreterError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.InterpreterError.__str__" => "Return str(self).", + "_interpreters.InterpreterError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.InterpreterError.add_note" => "Add a note to the exception", + "_interpreters.InterpreterError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.InterpreterNotFoundError" => "An interpreter was not found", + "_interpreters.InterpreterNotFoundError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.InterpreterNotFoundError.__eq__" => "Return self==value.", + "_interpreters.InterpreterNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.InterpreterNotFoundError.__ge__" => "Return self>=value.", + "_interpreters.InterpreterNotFoundError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.InterpreterNotFoundError.__getstate__" => "Helper for pickle.", + "_interpreters.InterpreterNotFoundError.__gt__" => "Return self>value.", + "_interpreters.InterpreterNotFoundError.__hash__" => "Return hash(self).", + "_interpreters.InterpreterNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.InterpreterNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.InterpreterNotFoundError.__le__" => "Return self<=value.", + "_interpreters.InterpreterNotFoundError.__lt__" => "Return self<value.", + "_interpreters.InterpreterNotFoundError.__ne__" => "Return self!=value.", + "_interpreters.InterpreterNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.InterpreterNotFoundError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.InterpreterNotFoundError.__repr__" => "Return repr(self).", + "_interpreters.InterpreterNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.InterpreterNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.InterpreterNotFoundError.__str__" => "Return str(self).", + "_interpreters.InterpreterNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.InterpreterNotFoundError.add_note" => "Add a note to the exception", + "_interpreters.InterpreterNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.NotShareableError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.NotShareableError.__eq__" => "Return self==value.", + "_interpreters.NotShareableError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.NotShareableError.__ge__" => "Return self>=value.", + "_interpreters.NotShareableError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.NotShareableError.__getstate__" => "Helper for pickle.", + "_interpreters.NotShareableError.__gt__" => "Return self>value.", + "_interpreters.NotShareableError.__hash__" => "Return hash(self).", + "_interpreters.NotShareableError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.NotShareableError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.NotShareableError.__le__" => "Return self<=value.", + "_interpreters.NotShareableError.__lt__" => "Return self<value.", + "_interpreters.NotShareableError.__ne__" => "Return self!=value.", + "_interpreters.NotShareableError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.NotShareableError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.NotShareableError.__repr__" => "Return repr(self).", + "_interpreters.NotShareableError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.NotShareableError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.NotShareableError.__str__" => "Return str(self).", + "_interpreters.NotShareableError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.NotShareableError.__weakref__" => "list of weak references to the object", + "_interpreters.NotShareableError.add_note" => "Add a note to the exception", + "_interpreters.NotShareableError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.call" => "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\nCall the provided object in the identified interpreter.\nPass the given args and kwargs, if possible.", "_interpreters.capture_exception" => "capture_exception(exc=None) -> types.SimpleNamespace\n\nReturn a snapshot of an exception. If \"exc\" is None\nthen the current exception, if any, is used (but not cleared).\n\nThe returned snapshot is the same as what _interpreters.exec() returns.", - "_interpreters.create" => "create([config], *, reqrefs=False) -> ID\n\nCreate a new interpreter and return a unique generated ID.\n\nThe caller is responsible for destroying the interpreter before exiting,\ntypically by using _interpreters.destroy(). This can be managed \nautomatically by passing \"reqrefs=True\" and then using _incref() and\n_decref()` appropriately.\n\n\"config\" must be a valid interpreter config or the name of a\npredefined config (\"isolated\" or \"legacy\"). The default\nis \"isolated\".", + "_interpreters.create" => "create([config], *, reqrefs=False) -> ID\n\nCreate a new interpreter and return a unique generated ID.\n\nThe caller is responsible for destroying the interpreter before exiting,\ntypically by using _interpreters.destroy(). This can be managed \nautomatically by passing \"reqrefs=True\" and then using _incref() and\n_decref() appropriately.\n\n\"config\" must be a valid interpreter config or the name of a\npredefined config (\"isolated\" or \"legacy\"). The default\nis \"isolated\".", "_interpreters.destroy" => "destroy(id, *, restrict=False)\n\nDestroy the identified interpreter.\n\nAttempting to destroy the current interpreter raises InterpreterError.\nSo does an unrecognized ID.", "_interpreters.exec" => "exec(id, code, shared=None, *, restrict=False)\n\nExecute the provided code in the identified interpreter.\nThis is equivalent to running the builtin exec() under the target\ninterpreter, using the __dict__ of its __main__ module as both\nglobals and locals.\n\n\"code\" may be a string containing the text of a Python script.\n\nFunctions (and code objects) are also supported, with some restrictions.\nThe code/function must not take any arguments or be a closure\n(i.e. have cell vars). Methods and other callables are not supported.\n\nIf a function is provided, its code object is used and all its state\nis ignored, including its __globals__ dict.", "_interpreters.get_config" => "get_config(id, *, restrict=False) -> types.SimpleNamespace\n\nReturn a representation of the config used to initialize the interpreter.", @@ -1149,7 +5406,35 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_interpreters.run_string" => "run_string(id, script, shared=None, *, restrict=False)\n\nExecute the provided string in the identified interpreter.\n\n(See _interpreters.exec().", "_interpreters.set___main___attrs" => "set___main___attrs(id, ns, *, restrict=False)\n\nBind the given attributes in the interpreter's __main__ module.", "_interpreters.whence" => "whence(id) -> int\n\nReturn an identifier for where the interpreter was created.", - "_io" => "The io module provides the Python interfaces to stream handling. The\nbuiltin open function is defined in this module.\n\nAt the top of the I/O hierarchy is the abstract base class IOBase. It\ndefines the basic interface to a stream. Note, however, that there is no\nseparation between reading and writing to streams; implementations are\nallowed to raise an OSError if they do not support a given operation.\n\nExtending IOBase is RawIOBase which deals simply with the reading and\nwriting of raw bytes to a stream. FileIO subclasses RawIOBase to provide\nan interface to OS files.\n\nBufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\nsubclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\nstreams that are readable, writable, and both respectively.\nBufferedRandom provides a buffered interface to random access\nstreams. BytesIO is a simple stream of in-memory bytes.\n\nAnother IOBase subclass, TextIOBase, deals with the encoding and decoding\nof streams into text. TextIOWrapper, which extends it, is a buffered text\ninterface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\nis an in-memory stream for text.\n\nArgument names are not part of the specification, and only the arguments\nof open() are intended to be used as keyword arguments.\n\ndata:\n\nDEFAULT_BUFFER_SIZE\n\n An int containing the default buffer size used by the module's buffered\n I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n possible.", + "_io" => "The io module provides the Python interfaces to stream handling. The\nbuiltin open function is defined in this module.\n\nAt the top of the I/O hierarchy is the abstract base class IOBase. It\ndefines the basic interface to a stream. Note, however, that there is no\nseparation between reading and writing to streams; implementations are\nallowed to raise an OSError if they do not support a given operation.\n\nExtending IOBase is RawIOBase which deals simply with the reading and\nwriting of raw bytes to a stream. FileIO subclasses RawIOBase to provide\nan interface to OS files.\n\nBufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\nsubclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\nstreams that are readable, writable, and both respectively.\nBufferedRandom provides a buffered interface to random access\nstreams. BytesIO is a simple stream of in-memory bytes.\n\nAnother IOBase subclass, TextIOBase, deals with the encoding and decoding\nof streams into text. TextIOWrapper, which extends it, is a buffered text\ninterface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\nis an in-memory stream for text.\n\nArgument names are not part of the specification, and only the arguments\nof open() are intended to be used as keyword arguments.\n\ndata:\n\nDEFAULT_BUFFER_SIZE\n\n An int containing the default buffer size used by the module's buffered\n I/O classes.", + "_io.BlockingIOError" => "I/O operation would block.", + "_io.BlockingIOError.__delattr__" => "Implement delattr(self, name).", + "_io.BlockingIOError.__eq__" => "Return self==value.", + "_io.BlockingIOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BlockingIOError.__ge__" => "Return self>=value.", + "_io.BlockingIOError.__getattribute__" => "Return getattr(self, name).", + "_io.BlockingIOError.__getstate__" => "Helper for pickle.", + "_io.BlockingIOError.__gt__" => "Return self>value.", + "_io.BlockingIOError.__hash__" => "Return hash(self).", + "_io.BlockingIOError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BlockingIOError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BlockingIOError.__le__" => "Return self<=value.", + "_io.BlockingIOError.__lt__" => "Return self<value.", + "_io.BlockingIOError.__ne__" => "Return self!=value.", + "_io.BlockingIOError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BlockingIOError.__reduce_ex__" => "Helper for pickle.", + "_io.BlockingIOError.__repr__" => "Return repr(self).", + "_io.BlockingIOError.__setattr__" => "Implement setattr(self, name, value).", + "_io.BlockingIOError.__sizeof__" => "Size of object in memory, in bytes.", + "_io.BlockingIOError.__str__" => "Return str(self).", + "_io.BlockingIOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BlockingIOError.add_note" => "Add a note to the exception", + "_io.BlockingIOError.errno" => "POSIX exception code", + "_io.BlockingIOError.filename" => "exception filename", + "_io.BlockingIOError.filename2" => "second exception filename", + "_io.BlockingIOError.strerror" => "exception strerror", + "_io.BlockingIOError.winerror" => "Win32 exception code", + "_io.BlockingIOError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_io.BufferedRWPair" => "A buffered reader and writer object together.\n\nA buffered reader object and buffered writer object put together to\nform a sequential IO object that can read and write. This is typically\nused with a socket or two-way pipe.\n\nreader and writer are RawIOBase objects that are readable and\nwriteable respectively. If the buffer_size is omitted it defaults to\nDEFAULT_BUFFER_SIZE.", "_io.BufferedRWPair.__del__" => "Called when the instance is about to be destroyed.", "_io.BufferedRWPair.__delattr__" => "Implement delattr(self, name).", @@ -1333,6 +5618,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_io.FileIO.__sizeof__" => "Size of object in memory, in bytes.", "_io.FileIO.__str__" => "Return str(self).", "_io.FileIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.FileIO._blksize" => "Stat st_blksize if available", "_io.FileIO.close" => "Close the file.\n\nA closed file cannot be used for further I/O operations. close() may be\ncalled more than once without error.", "_io.FileIO.closed" => "True if the file is closed", "_io.FileIO.closefd" => "True if the file descriptor will be closed by close().", @@ -1340,9 +5626,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_io.FileIO.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", "_io.FileIO.isatty" => "True if the file is connected to a TTY device.", "_io.FileIO.mode" => "String giving the file mode", - "_io.FileIO.read" => "Read at most size bytes, returned as bytes.\n\nOnly makes one system call, so less data may be returned than requested.\nIn non-blocking mode, returns None if no data is available.\nReturn an empty bytes object at EOF.", + "_io.FileIO.read" => "Read at most size bytes, returned as bytes.\n\nIf size is less than 0, read all bytes in the file making multiple read calls.\nSee ``FileIO.readall``.\n\nAttempts to make only one system call, retrying only per PEP 475 (EINTR). This\nmeans less data may be returned than requested.\n\nIn non-blocking mode, returns None if no data is available. Return an empty\nbytes object at EOF.", "_io.FileIO.readable" => "True if file was opened in a read mode.", - "_io.FileIO.readall" => "Read all data from the file, returned as bytes.\n\nIn non-blocking mode, returns as much as is immediately available,\nor None if no data is available. Return an empty bytes object at EOF.", + "_io.FileIO.readall" => "Read all data from the file, returned as bytes.\n\nReads until either there is an error or read() returns size 0 (indicates EOF).\nIf the file is already at EOF, returns an empty bytes object.\n\nIn non-blocking mode, returns as much data as could be read before EAGAIN. If no\ndata is available (EAGAIN is returned before bytes are read) returns None.", "_io.FileIO.readinto" => "Same as RawIOBase.readinto().", "_io.FileIO.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", "_io.FileIO.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", @@ -1447,6 +5733,34 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_io.TextIOWrapper.seek" => "Set the stream position, and return the new stream position.\n\n cookie\n Zero or an opaque number returned by tell().\n whence\n The relative position to seek from.\n\nFour operations are supported, given by the following argument\ncombinations:\n\n- seek(0, SEEK_SET): Rewind to the start of the stream.\n- seek(cookie, SEEK_SET): Restore a previous position;\n 'cookie' must be a number returned by tell().\n- seek(0, SEEK_END): Fast-forward to the end of the stream.\n- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n\nAny other argument combinations are invalid,\nand may raise exceptions.", "_io.TextIOWrapper.tell" => "Return the stream position as an opaque number.\n\nThe return value of tell() can be given as input to seek(), to restore a\nprevious stream position.", "_io.TextIOWrapper.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.UnsupportedOperation.__delattr__" => "Implement delattr(self, name).", + "_io.UnsupportedOperation.__eq__" => "Return self==value.", + "_io.UnsupportedOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.UnsupportedOperation.__ge__" => "Return self>=value.", + "_io.UnsupportedOperation.__getattribute__" => "Return getattr(self, name).", + "_io.UnsupportedOperation.__getstate__" => "Helper for pickle.", + "_io.UnsupportedOperation.__gt__" => "Return self>value.", + "_io.UnsupportedOperation.__hash__" => "Return hash(self).", + "_io.UnsupportedOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.UnsupportedOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.UnsupportedOperation.__le__" => "Return self<=value.", + "_io.UnsupportedOperation.__lt__" => "Return self<value.", + "_io.UnsupportedOperation.__ne__" => "Return self!=value.", + "_io.UnsupportedOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.UnsupportedOperation.__reduce_ex__" => "Helper for pickle.", + "_io.UnsupportedOperation.__repr__" => "Return repr(self).", + "_io.UnsupportedOperation.__setattr__" => "Implement setattr(self, name, value).", + "_io.UnsupportedOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_io.UnsupportedOperation.__str__" => "Return str(self).", + "_io.UnsupportedOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.UnsupportedOperation.__weakref__" => "list of weak references to the object", + "_io.UnsupportedOperation.add_note" => "Add a note to the exception", + "_io.UnsupportedOperation.errno" => "POSIX exception code", + "_io.UnsupportedOperation.filename" => "exception filename", + "_io.UnsupportedOperation.filename2" => "second exception filename", + "_io.UnsupportedOperation.strerror" => "exception strerror", + "_io.UnsupportedOperation.winerror" => "Win32 exception code", + "_io.UnsupportedOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", "_io._BufferedIOBase" => "Base class for buffered IO objects.\n\nThe main difference with RawIOBase is that the read() method\nsupports omitting the size argument, and does not have a default\nimplementation that defers to readinto().\n\nIn addition, read(), readinto() and write() may raise\nBlockingIOError if the underlying raw stream is in non-blocking\nmode and not ready; unlike their raw counterparts, they will never\nreturn None.\n\nA typical implementation should not inherit from a RawIOBase\nimplementation, but wrap one.", "_io._BufferedIOBase.__del__" => "Called when the instance is about to be destroyed.", "_io._BufferedIOBase.__delattr__" => "Implement delattr(self, name).", @@ -1678,7 +5992,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_io._WindowsConsoleIO.writable" => "True if console is an output buffer.", "_io._WindowsConsoleIO.write" => "Write buffer b to file, return number of bytes written.\n\nOnly makes one system call, so not all of the data may be written.\nThe number of bytes actually written is returned.", "_io._WindowsConsoleIO.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", - "_io.open" => "Open file and return a stream. Raise OSError upon failure.\n\nfile is either a text or byte string giving the name (and the path\nif the file isn't in the current working directory) of the file to\nbe opened or an integer file descriptor of the file to be\nwrapped. (If a file descriptor is given, it is closed when the\nreturned I/O object is closed, unless closefd is set to False.)\n\nmode is an optional string that specifies the mode in which the file\nis opened. It defaults to 'r' which means open for reading in text\nmode. Other common values are 'w' for writing (truncating the file if\nit already exists), 'x' for creating and writing to a new file, and\n'a' for appending (which on some Unix systems, means that all writes\nappend to the end of the file regardless of the current seek position).\nIn text mode, if encoding is not specified the encoding used is platform\ndependent: locale.getencoding() is called to get the current locale encoding.\n(For reading and writing raw bytes use binary mode and leave encoding\nunspecified.) The available modes are:\n\n========= ===============================================================\nCharacter Meaning\n--------- ---------------------------------------------------------------\n'r' open for reading (default)\n'w' open for writing, truncating the file first\n'x' create a new file and open it for writing\n'a' open for writing, appending to the end of the file if it exists\n'b' binary mode\n't' text mode (default)\n'+' open a disk file for updating (reading and writing)\n========= ===============================================================\n\nThe default mode is 'rt' (open for reading text). For binary random\naccess, the mode 'w+b' opens and truncates the file to 0 bytes, while\n'r+b' opens the file without truncation. The 'x' mode implies 'w' and\nraises an `FileExistsError` if the file already exists.\n\nPython distinguishes between files opened in binary and text modes,\neven when the underlying operating system doesn't. Files opened in\nbinary mode (appending 'b' to the mode argument) return contents as\nbytes objects without any decoding. In text mode (the default, or when\n't' is appended to the mode argument), the contents of the file are\nreturned as strings, the bytes having been first decoded using a\nplatform-dependent encoding or using the specified encoding if given.\n\nbuffering is an optional integer used to set the buffering policy.\nPass 0 to switch buffering off (only allowed in binary mode), 1 to select\nline buffering (only usable in text mode), and an integer > 1 to indicate\nthe size of a fixed-size chunk buffer. When no buffering argument is\ngiven, the default buffering policy works as follows:\n\n* Binary files are buffered in fixed-size chunks; the size of the buffer\n is chosen using a heuristic trying to determine the underlying device's\n \"block size\" and falling back on `io.DEFAULT_BUFFER_SIZE`.\n On many systems, the buffer will typically be 4096 or 8192 bytes long.\n\n* \"Interactive\" text files (files for which isatty() returns True)\n use line buffering. Other text files use the policy described above\n for binary files.\n\nencoding is the name of the encoding used to decode or encode the\nfile. This should only be used in text mode. The default encoding is\nplatform dependent, but any encoding supported by Python can be\npassed. See the codecs module for the list of supported encodings.\n\nerrors is an optional string that specifies how encoding errors are to\nbe handled---this argument should not be used in binary mode. Pass\n'strict' to raise a ValueError exception if there is an encoding error\n(the default of None has the same effect), or pass 'ignore' to ignore\nerrors. (Note that ignoring encoding errors can lead to data loss.)\nSee the documentation for codecs.register or run 'help(codecs.Codec)'\nfor a list of the permitted encoding error strings.\n\nnewline controls how universal newlines works (it only applies to text\nmode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\nfollows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf closefd is False, the underlying file descriptor will be kept open\nwhen the file is closed. This does not work when a file name is given\nand must be True in that case.\n\nA custom opener can be used by passing a callable as *opener*. The\nunderlying file descriptor for the file object is then obtained by\ncalling *opener* with (*file*, *flags*). *opener* must return an open\nfile descriptor (passing os.open as *opener* results in functionality\nsimilar to passing None).\n\nopen() returns a file object whose type depends on the mode, and\nthrough which the standard file operations such as reading and writing\nare performed. When open() is used to open a file in a text mode ('w',\n'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\na file in a binary mode, the returned class varies: in read binary\nmode, it returns a BufferedReader; in write binary and append binary\nmodes, it returns a BufferedWriter, and in read/write mode, it returns\na BufferedRandom.\n\nIt is also possible to use a string or bytearray as a file for both\nreading and writing. For strings StringIO can be used like a file\nopened in a text mode, and for bytes a BytesIO can be used like a file\nopened in a binary mode.", + "_io.open" => "Open file and return a stream. Raise OSError upon failure.\n\nfile is either a text or byte string giving the name (and the path\nif the file isn't in the current working directory) of the file to\nbe opened or an integer file descriptor of the file to be\nwrapped. (If a file descriptor is given, it is closed when the\nreturned I/O object is closed, unless closefd is set to False.)\n\nmode is an optional string that specifies the mode in which the file\nis opened. It defaults to 'r' which means open for reading in text\nmode. Other common values are 'w' for writing (truncating the file if\nit already exists), 'x' for creating and writing to a new file, and\n'a' for appending (which on some Unix systems, means that all writes\nappend to the end of the file regardless of the current seek position).\nIn text mode, if encoding is not specified the encoding used is platform\ndependent: locale.getencoding() is called to get the current locale encoding.\n(For reading and writing raw bytes use binary mode and leave encoding\nunspecified.) The available modes are:\n\n========= ===============================================================\nCharacter Meaning\n--------- ---------------------------------------------------------------\n'r' open for reading (default)\n'w' open for writing, truncating the file first\n'x' create a new file and open it for writing\n'a' open for writing, appending to the end of the file if it exists\n'b' binary mode\n't' text mode (default)\n'+' open a disk file for updating (reading and writing)\n========= ===============================================================\n\nThe default mode is 'rt' (open for reading text). For binary random\naccess, the mode 'w+b' opens and truncates the file to 0 bytes, while\n'r+b' opens the file without truncation. The 'x' mode implies 'w' and\nraises an `FileExistsError` if the file already exists.\n\nPython distinguishes between files opened in binary and text modes,\neven when the underlying operating system doesn't. Files opened in\nbinary mode (appending 'b' to the mode argument) return contents as\nbytes objects without any decoding. In text mode (the default, or when\n't' is appended to the mode argument), the contents of the file are\nreturned as strings, the bytes having been first decoded using a\nplatform-dependent encoding or using the specified encoding if given.\n\nbuffering is an optional integer used to set the buffering policy.\nPass 0 to switch buffering off (only allowed in binary mode), 1 to select\nline buffering (only usable in text mode), and an integer > 1 to indicate\nthe size of a fixed-size chunk buffer. When no buffering argument is\ngiven, the default buffering policy works as follows:\n\n* Binary files are buffered in fixed-size chunks; the size of the buffer\n is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n when the device block size is available.\n On most systems, the buffer will typically be 128 kilobytes long.\n\n* \"Interactive\" text files (files for which isatty() returns True)\n use line buffering. Other text files use the policy described above\n for binary files.\n\nencoding is the name of the encoding used to decode or encode the\nfile. This should only be used in text mode. The default encoding is\nplatform dependent, but any encoding supported by Python can be\npassed. See the codecs module for the list of supported encodings.\n\nerrors is an optional string that specifies how encoding errors are to\nbe handled---this argument should not be used in binary mode. Pass\n'strict' to raise a ValueError exception if there is an encoding error\n(the default of None has the same effect), or pass 'ignore' to ignore\nerrors. (Note that ignoring encoding errors can lead to data loss.)\nSee the documentation for codecs.register or run 'help(codecs.Codec)'\nfor a list of the permitted encoding error strings.\n\nnewline controls how universal newlines works (it only applies to text\nmode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\nfollows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf closefd is False, the underlying file descriptor will be kept open\nwhen the file is closed. This does not work when a file name is given\nand must be True in that case.\n\nA custom opener can be used by passing a callable as *opener*. The\nunderlying file descriptor for the file object is then obtained by\ncalling *opener* with (*file*, *flags*). *opener* must return an open\nfile descriptor (passing os.open as *opener* results in functionality\nsimilar to passing None).\n\nopen() returns a file object whose type depends on the mode, and\nthrough which the standard file operations such as reading and writing\nare performed. When open() is used to open a file in a text mode ('w',\n'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\na file in a binary mode, the returned class varies: in read binary\nmode, it returns a BufferedReader; in write binary and append binary\nmodes, it returns a BufferedWriter, and in read/write mode, it returns\na BufferedRandom.\n\nIt is also possible to use a string or bytearray as a file for both\nreading and writing. For strings StringIO can be used like a file\nopened in a text mode, and for bytes a BytesIO can be used like a file\nopened in a binary mode.", "_io.open_code" => "Opens the provided file with the intent to import the contents.\n\nThis may perform extra validation beyond open(), but is otherwise interchangeable\nwith calling open(path, 'rb').", "_io.text_encoding" => "A helper function to choose the text encoding.\n\nWhen encoding is not None, this function returns it.\nOtherwise, this function returns the default text encoding\n(i.e. \"locale\" or \"utf-8\" depends on UTF-8 mode).\n\nThis function emits an EncodingWarning if encoding is None and\nsys.flags.warn_default_encoding is true.\n\nThis can be used in APIs with an encoding=None parameter.\nHowever, please consider using encoding=\"utf-8\" for new APIs.", "_json" => "json speedups", @@ -1745,6 +6059,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_json.make_scanner.strict" => "strict", "_json.scanstring" => "scanstring(string, end, strict=True) -> (string, end)\n\nScan the string s for a JSON string. End is the index of the\ncharacter in s after the quote that started the JSON string.\nUnescapes all valid JSON string escape sequences and raises ValueError\non attempt to decode an invalid string. If strict is False then literal\ncontrol characters are allowed in the string.\n\nReturns a tuple of the decoded string and the index of the character in s\nafter the end quote.", "_locale" => "Support for POSIX locales.", + "_locale.Error.__delattr__" => "Implement delattr(self, name).", + "_locale.Error.__eq__" => "Return self==value.", + "_locale.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_locale.Error.__ge__" => "Return self>=value.", + "_locale.Error.__getattribute__" => "Return getattr(self, name).", + "_locale.Error.__getstate__" => "Helper for pickle.", + "_locale.Error.__gt__" => "Return self>value.", + "_locale.Error.__hash__" => "Return hash(self).", + "_locale.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_locale.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_locale.Error.__le__" => "Return self<=value.", + "_locale.Error.__lt__" => "Return self<value.", + "_locale.Error.__ne__" => "Return self!=value.", + "_locale.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_locale.Error.__reduce_ex__" => "Helper for pickle.", + "_locale.Error.__repr__" => "Return repr(self).", + "_locale.Error.__setattr__" => "Implement setattr(self, name, value).", + "_locale.Error.__sizeof__" => "Size of object in memory, in bytes.", + "_locale.Error.__str__" => "Return str(self).", + "_locale.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_locale.Error.__weakref__" => "list of weak references to the object", + "_locale.Error.add_note" => "Add a note to the exception", + "_locale.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_locale.bind_textdomain_codeset" => "Bind the C library's domain to codeset.", "_locale.bindtextdomain" => "Bind the C library's domain to dir.", "_locale.dcgettext" => "Return translation of msg in domain and category.", @@ -1758,7 +6095,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_locale.strxfrm" => "Return a string that can be used as a key for locale-aware comparisons.", "_locale.textdomain" => "Set the C library's textdmain to domain, returning the new domain.", "_lsprof" => "Fast profiler", - "_lsprof.Profiler" => "Profiler(timer=None, timeunit=None, subcalls=True, builtins=True)\n\nBuilds a profiler object using the specified timer function.\nThe default timer is a fast built-in one based on real time.\nFor custom timer functions returning integers, timeunit can\nbe a float specifying a scale (i.e. how long each integer unit\nis, in seconds).", + "_lsprof.Profiler" => "Build a profiler object using the specified timer function.\n\nThe default timer is a fast built-in one based on real time.\nFor custom timer functions returning integers, 'timeunit' can\nbe a float specifying a scale (that is, how long each integer unit\nis, in seconds).", "_lsprof.Profiler.__delattr__" => "Implement delattr(self, name).", "_lsprof.Profiler.__eq__" => "Return self==value.", "_lsprof.Profiler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -1780,9 +6117,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_lsprof.Profiler.__sizeof__" => "Size of object in memory, in bytes.", "_lsprof.Profiler.__str__" => "Return str(self).", "_lsprof.Profiler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_lsprof.Profiler.clear" => "clear()\n\nClear all profiling information collected so far.", - "_lsprof.Profiler.disable" => "disable()\n\nStop collecting profiling information.", - "_lsprof.Profiler.enable" => "enable(subcalls=True, builtins=True)\n\nStart collecting profiling information.\nIf 'subcalls' is True, also records for each function\nstatistics separated according to its current caller.\nIf 'builtins' is True, records the time spent in\nbuilt-in functions separately from their caller.", + "_lsprof.Profiler.clear" => "Clear all profiling information collected so far.", + "_lsprof.Profiler.disable" => "Stop collecting profiling information.", + "_lsprof.Profiler.enable" => "Start collecting profiling information.\n\n subcalls\n If True, also records for each function\n statistics separated according to its current caller.\n builtins\n If True, records the time spent in\n built-in functions separately from their caller.", "_lsprof.Profiler.getstats" => "list of profiler_entry objects.\n\ngetstats() -> list of profiler_entry objects\n\nReturn all information collected by the profiler.\nEach profiler_entry is a tuple-like object with the\nfollowing attributes:\n\n code code object\n callcount how many times this was called\n reccallcount how many times called recursively\n totaltime total time in this entry\n inlinetime inline time in this entry (not in subcalls)\n calls details of the calls\n\nThe calls attribute is either None or a list of\nprofiler_subentry objects:\n\n code called code object\n callcount how many times this is called\n reccallcount how many times this is called recursively\n totaltime total time spent in this call\n inlinetime inline time (not in further subcalls)", "_lsprof.profiler_entry.__add__" => "Return self+value.", "_lsprof.profiler_entry.__class_getitem__" => "See PEP 585", @@ -1909,8 +6246,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_lzma.LZMADecompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", "_lzma.LZMADecompressor.unused_data" => "Data found after the end of the compressed stream.", "_lzma.LZMAError" => "Call to liblzma failed.", - "_lzma.LZMAError.__cause__" => "exception cause", - "_lzma.LZMAError.__context__" => "exception context", "_lzma.LZMAError.__delattr__" => "Implement delattr(self, name).", "_lzma.LZMAError.__eq__" => "Return self==value.", "_lzma.LZMAError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -1932,8 +6267,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_lzma.LZMAError.__str__" => "Return str(self).", "_lzma.LZMAError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_lzma.LZMAError.__weakref__" => "list of weak references to the object", - "_lzma.LZMAError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_lzma.LZMAError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_lzma.LZMAError.add_note" => "Add a note to the exception", + "_lzma.LZMAError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_lzma._decode_filter_properties" => "Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n\nThe result does not include the filter ID itself, only the options.", "_lzma._encode_filter_properties" => "Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n\nThe result does not include the filter ID itself, only the options.", "_lzma.is_check_supported" => "Test whether the given integrity check is supported.\n\nAlways returns True for CHECK_NONE and CHECK_CRC32.", @@ -2091,6 +6426,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_opcode.get_intrinsic1_descs" => "Return a list of names of the unary intrinsics.", "_opcode.get_intrinsic2_descs" => "Return a list of names of the binary intrinsics.", "_opcode.get_nb_ops" => "Return array of symbols of binary ops.\n\nIndexed by the BINARY_OP oparg value.", + "_opcode.get_special_method_names" => "Return a list of special method names.", "_opcode.get_specialization_stats" => "Return the specialization stats", "_opcode.has_arg" => "Return True if the opcode uses its oparg, False otherwise.", "_opcode.has_const" => "Return True if the opcode accesses a constant, False otherwise.", @@ -2106,6 +6442,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_operator.abs" => "Same as abs(a).", "_operator.add" => "Same as a + b.", "_operator.and_" => "Same as a & b.", + "_operator.attrgetter" => "Return a callable object that fetches the given attribute(s) from its operand.\nAfter f = attrgetter('name'), the call f(r) returns r.name.\nAfter g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\nAfter h = attrgetter('name.first', 'name.last'), the call h(r) returns\n(r.name.first, r.name.last).", + "_operator.attrgetter.__call__" => "Call self as a function.", + "_operator.attrgetter.__delattr__" => "Implement delattr(self, name).", + "_operator.attrgetter.__eq__" => "Return self==value.", + "_operator.attrgetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.attrgetter.__ge__" => "Return self>=value.", + "_operator.attrgetter.__getattribute__" => "Return getattr(self, name).", + "_operator.attrgetter.__getstate__" => "Helper for pickle.", + "_operator.attrgetter.__gt__" => "Return self>value.", + "_operator.attrgetter.__hash__" => "Return hash(self).", + "_operator.attrgetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.attrgetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.attrgetter.__le__" => "Return self<=value.", + "_operator.attrgetter.__lt__" => "Return self<value.", + "_operator.attrgetter.__ne__" => "Return self!=value.", + "_operator.attrgetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.attrgetter.__reduce__" => "Return state information for pickling", + "_operator.attrgetter.__reduce_ex__" => "Helper for pickle.", + "_operator.attrgetter.__repr__" => "Return repr(self).", + "_operator.attrgetter.__setattr__" => "Implement setattr(self, name, value).", + "_operator.attrgetter.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.attrgetter.__str__" => "Return str(self).", + "_operator.attrgetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_operator.call" => "Same as obj(*args, **kwargs).", "_operator.concat" => "Same as a + b, for a and b sequences.", "_operator.contains" => "Same as b in a (note reversed operands).", @@ -2132,8 +6491,33 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_operator.ipow" => "Same as a **= b.", "_operator.irshift" => "Same as a >>= b.", "_operator.is_" => "Same as a is b.", + "_operator.is_none" => "Same as a is None.", "_operator.is_not" => "Same as a is not b.", + "_operator.is_not_none" => "Same as a is not None.", "_operator.isub" => "Same as a -= b.", + "_operator.itemgetter" => "Return a callable object that fetches the given item(s) from its operand.\nAfter f = itemgetter(2), the call f(r) returns r[2].\nAfter g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])", + "_operator.itemgetter.__call__" => "Call self as a function.", + "_operator.itemgetter.__delattr__" => "Implement delattr(self, name).", + "_operator.itemgetter.__eq__" => "Return self==value.", + "_operator.itemgetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.itemgetter.__ge__" => "Return self>=value.", + "_operator.itemgetter.__getattribute__" => "Return getattr(self, name).", + "_operator.itemgetter.__getstate__" => "Helper for pickle.", + "_operator.itemgetter.__gt__" => "Return self>value.", + "_operator.itemgetter.__hash__" => "Return hash(self).", + "_operator.itemgetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.itemgetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.itemgetter.__le__" => "Return self<=value.", + "_operator.itemgetter.__lt__" => "Return self<value.", + "_operator.itemgetter.__ne__" => "Return self!=value.", + "_operator.itemgetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.itemgetter.__reduce__" => "Return state information for pickling", + "_operator.itemgetter.__reduce_ex__" => "Helper for pickle.", + "_operator.itemgetter.__repr__" => "Return repr(self).", + "_operator.itemgetter.__setattr__" => "Implement setattr(self, name, value).", + "_operator.itemgetter.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.itemgetter.__str__" => "Return str(self).", + "_operator.itemgetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_operator.itruediv" => "Same as a /= b.", "_operator.ixor" => "Same as a ^= b.", "_operator.le" => "Same as a <= b.", @@ -2141,6 +6525,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_operator.lshift" => "Same as a << b.", "_operator.lt" => "Same as a < b.", "_operator.matmul" => "Same as a @ b.", + "_operator.methodcaller" => "Return a callable object that calls the given method on its operand.\nAfter f = methodcaller('name'), the call f(r) returns r.name().\nAfter g = methodcaller('name', 'date', foo=1), the call g(r) returns\nr.name('date', foo=1).", + "_operator.methodcaller.__call__" => "Call self as a function.", + "_operator.methodcaller.__delattr__" => "Implement delattr(self, name).", + "_operator.methodcaller.__eq__" => "Return self==value.", + "_operator.methodcaller.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.methodcaller.__ge__" => "Return self>=value.", + "_operator.methodcaller.__getattribute__" => "Return getattr(self, name).", + "_operator.methodcaller.__getstate__" => "Helper for pickle.", + "_operator.methodcaller.__gt__" => "Return self>value.", + "_operator.methodcaller.__hash__" => "Return hash(self).", + "_operator.methodcaller.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.methodcaller.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.methodcaller.__le__" => "Return self<=value.", + "_operator.methodcaller.__lt__" => "Return self<value.", + "_operator.methodcaller.__ne__" => "Return self!=value.", + "_operator.methodcaller.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.methodcaller.__reduce__" => "Return state information for pickling", + "_operator.methodcaller.__reduce_ex__" => "Helper for pickle.", + "_operator.methodcaller.__repr__" => "Return repr(self).", + "_operator.methodcaller.__setattr__" => "Implement setattr(self, name, value).", + "_operator.methodcaller.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.methodcaller.__str__" => "Return str(self).", + "_operator.methodcaller.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_operator.mod" => "Same as a % b.", "_operator.mul" => "Same as a * b.", "_operator.ne" => "Same as a != b.", @@ -2210,8 +6617,32 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_overlapped.UnregisterWaitEx" => "Unregister wait handle.", "_overlapped.WSAConnect" => "Bind a remote address to a connectionless (UDP) socket.", "_pickle" => "Optimized C implementation for the Python pickle module.", - "_pickle.PickleError.__cause__" => "exception cause", - "_pickle.PickleError.__context__" => "exception context", + "_pickle.PickleBuffer" => "Wrapper for potentially out-of-band buffers", + "_pickle.PickleBuffer.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "_pickle.PickleBuffer.__delattr__" => "Implement delattr(self, name).", + "_pickle.PickleBuffer.__eq__" => "Return self==value.", + "_pickle.PickleBuffer.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.PickleBuffer.__ge__" => "Return self>=value.", + "_pickle.PickleBuffer.__getattribute__" => "Return getattr(self, name).", + "_pickle.PickleBuffer.__getstate__" => "Helper for pickle.", + "_pickle.PickleBuffer.__gt__" => "Return self>value.", + "_pickle.PickleBuffer.__hash__" => "Return hash(self).", + "_pickle.PickleBuffer.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.PickleBuffer.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.PickleBuffer.__le__" => "Return self<=value.", + "_pickle.PickleBuffer.__lt__" => "Return self<value.", + "_pickle.PickleBuffer.__ne__" => "Return self!=value.", + "_pickle.PickleBuffer.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.PickleBuffer.__reduce__" => "Helper for pickle.", + "_pickle.PickleBuffer.__reduce_ex__" => "Helper for pickle.", + "_pickle.PickleBuffer.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "_pickle.PickleBuffer.__repr__" => "Return repr(self).", + "_pickle.PickleBuffer.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.PickleBuffer.__sizeof__" => "Size of object in memory, in bytes.", + "_pickle.PickleBuffer.__str__" => "Return str(self).", + "_pickle.PickleBuffer.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.PickleBuffer.raw" => "Return a memoryview of the raw memory underlying this buffer.\nWill raise BufferError is the buffer isn't contiguous.", + "_pickle.PickleBuffer.release" => "Release the underlying buffer exposed by the PickleBuffer object.", "_pickle.PickleError.__delattr__" => "Implement delattr(self, name).", "_pickle.PickleError.__eq__" => "Return self==value.", "_pickle.PickleError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2233,9 +6664,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_pickle.PickleError.__str__" => "Return str(self).", "_pickle.PickleError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_pickle.PickleError.__weakref__" => "list of weak references to the object", - "_pickle.PickleError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_pickle.PickleError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_pickle.Pickler" => "This takes a binary file for writing a pickle data stream.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 4. It was introduced in Python 3.4, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are\nserialized into *file* as part of the pickle stream.\n\nIf *buffer_callback* is not None, then it can be called any number\nof times with a buffer view. If the callback returns a false value\n(such as None), the given buffer is out-of-band; otherwise the\nbuffer is serialized in-band, i.e. inside the pickle stream.\n\nIt is an error if *buffer_callback* is not None and *protocol*\nis None or smaller than 5.", + "_pickle.PickleError.add_note" => "Add a note to the exception", + "_pickle.PickleError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_pickle.Pickler" => "This takes a binary file for writing a pickle data stream.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are\nserialized into *file* as part of the pickle stream.\n\nIf *buffer_callback* is not None, then it can be called any number\nof times with a buffer view. If the callback returns a false value\n(such as None), the given buffer is out-of-band; otherwise the\nbuffer is serialized in-band, i.e. inside the pickle stream.\n\nIt is an error if *buffer_callback* is not None and *protocol*\nis None or smaller than 5.", "_pickle.Pickler.__delattr__" => "Implement delattr(self, name).", "_pickle.Pickler.__eq__" => "Return self==value.", "_pickle.Pickler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2259,8 +6690,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_pickle.Pickler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_pickle.Pickler.clear_memo" => "Clears the pickler's \"memo\".\n\nThe memo is the data structure that remembers which objects the\npickler has already seen, so that shared or recursive objects are\npickled by reference and not by value. This method is useful when\nre-using picklers.", "_pickle.Pickler.dump" => "Write a pickled representation of the given object to the open file.", - "_pickle.PicklingError.__cause__" => "exception cause", - "_pickle.PicklingError.__context__" => "exception context", "_pickle.PicklingError.__delattr__" => "Implement delattr(self, name).", "_pickle.PicklingError.__eq__" => "Return self==value.", "_pickle.PicklingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2282,8 +6711,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_pickle.PicklingError.__str__" => "Return str(self).", "_pickle.PicklingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_pickle.PicklingError.__weakref__" => "list of weak references to the object", - "_pickle.PicklingError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_pickle.PicklingError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_pickle.PicklingError.add_note" => "Add a note to the exception", + "_pickle.PicklingError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_pickle.Unpickler" => "This takes a binary file for reading a pickle data stream.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nThe argument *file* must have two methods, a read() method that takes\nan integer argument, and a readline() method that requires no\narguments. Both methods should return bytes. Thus *file* can be a\nbinary file object opened for reading, an io.BytesIO object, or any\nother custom object that meets this interface.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", "_pickle.Unpickler.__delattr__" => "Implement delattr(self, name).", "_pickle.Unpickler.__eq__" => "Return self==value.", @@ -2308,8 +6737,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_pickle.Unpickler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_pickle.Unpickler.find_class" => "Return an object from a specified module.\n\nIf necessary, the module will be imported. Subclasses may override\nthis method (e.g. to restrict unpickling of arbitrary classes and\nfunctions).\n\nThis method is called whenever a class or a function object is\nneeded. Both arguments passed are str objects.", "_pickle.Unpickler.load" => "Load a pickle.\n\nRead a pickled object representation from the open file object given\nin the constructor, and return the reconstituted object hierarchy\nspecified therein.", - "_pickle.UnpicklingError.__cause__" => "exception cause", - "_pickle.UnpicklingError.__context__" => "exception context", "_pickle.UnpicklingError.__delattr__" => "Implement delattr(self, name).", "_pickle.UnpicklingError.__eq__" => "Return self==value.", "_pickle.UnpicklingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2331,10 +6758,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_pickle.UnpicklingError.__str__" => "Return str(self).", "_pickle.UnpicklingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_pickle.UnpicklingError.__weakref__" => "list of weak references to the object", - "_pickle.UnpicklingError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_pickle.UnpicklingError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "_pickle.dump" => "Write a pickled representation of obj to the open file object file.\n\nThis is equivalent to ``Pickler(file, protocol).dump(obj)``, but may\nbe more efficient.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 4. It was introduced in Python 3.4, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", - "_pickle.dumps" => "Return the pickled representation of the object as a bytes object.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 4. It was introduced in Python 3.4, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nIf *fix_imports* is True and *protocol* is less than 3, pickle will\ntry to map the new Python 3 names to the old module names used in\nPython 2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", + "_pickle.UnpicklingError.add_note" => "Add a note to the exception", + "_pickle.UnpicklingError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_pickle.dump" => "Write a pickled representation of obj to the open file object file.\n\nThis is equivalent to ``Pickler(file, protocol).dump(obj)``, but may\nbe more efficient.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", + "_pickle.dumps" => "Return the pickled representation of the object as a bytes object.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nIf *fix_imports* is True and *protocol* is less than 3, pickle will\ntry to map the new Python 3 names to the old module names used in\nPython 2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", "_pickle.load" => "Read and return an object from the pickle data stored in a file.\n\nThis is equivalent to ``Unpickler(file).load()``, but may be more\nefficient.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nThe argument *file* must have two methods, a read() method that takes\nan integer argument, and a readline() method that requires no\narguments. Both methods should return bytes. Thus *file* can be a\nbinary file object opened for reading, an io.BytesIO object, or any\nother custom object that meets this interface.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", "_pickle.loads" => "Read and return an object from the given pickle data.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", "_posixshmem" => "POSIX shared memory module", @@ -2344,8 +6771,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_posixsubprocess.fork_exec" => "Spawn a fresh new child process.\n\nFork a child process, close parent file descriptors as appropriate in the\nchild and duplicate the few that are needed before calling exec() in the\nchild process.\n\nIf close_fds is True, close file descriptors 3 and higher, except those listed\nin the sorted tuple pass_fds.\n\nThe preexec_fn, if supplied, will be called immediately before closing file\ndescriptors and exec.\n\nWARNING: preexec_fn is NOT SAFE if your application uses threads.\n It may trigger infrequent, difficult to debug deadlocks.\n\nIf an error occurs in the child process before the exec, it is\nserialized and written to the errpipe_write fd per subprocess.py.\n\nReturns: the child process's PID.\n\nRaises: Only on an error in the parent process.", "_queue" => "C implementation of the Python queue module.\nThis module is an implementation detail, please do not use it directly.", "_queue.Empty" => "Exception raised by Queue.get(block=0)/get_nowait().", - "_queue.Empty.__cause__" => "exception cause", - "_queue.Empty.__context__" => "exception context", "_queue.Empty.__delattr__" => "Implement delattr(self, name).", "_queue.Empty.__eq__" => "Return self==value.", "_queue.Empty.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2367,8 +6792,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_queue.Empty.__str__" => "Return str(self).", "_queue.Empty.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_queue.Empty.__weakref__" => "list of weak references to the object", - "_queue.Empty.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_queue.Empty.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_queue.Empty.add_note" => "Add a note to the exception", + "_queue.Empty.with_traceback" => "Set self.__traceback__ to tb and return self.", "_queue.SimpleQueue" => "Simple, unbounded, reentrant FIFO queue.", "_queue.SimpleQueue.__class_getitem__" => "See PEP 585", "_queue.SimpleQueue.__delattr__" => "Implement delattr(self, name).", @@ -2426,6 +6851,204 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_random.Random.random" => "random() -> x in the interval [0, 1).", "_random.Random.seed" => "seed([n]) -> None.\n\nDefaults to use urandom and falls back to a combination\nof the current time and the process identifier.", "_random.Random.setstate" => "setstate(state) -> None. Restores generator state.", + "_remote_debugging.AwaitedInfo" => "Information about what a thread is awaiting", + "_remote_debugging.AwaitedInfo.__add__" => "Return self+value.", + "_remote_debugging.AwaitedInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.AwaitedInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.AwaitedInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.AwaitedInfo.__eq__" => "Return self==value.", + "_remote_debugging.AwaitedInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.AwaitedInfo.__ge__" => "Return self>=value.", + "_remote_debugging.AwaitedInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.AwaitedInfo.__getitem__" => "Return self[key].", + "_remote_debugging.AwaitedInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.AwaitedInfo.__gt__" => "Return self>value.", + "_remote_debugging.AwaitedInfo.__hash__" => "Return hash(self).", + "_remote_debugging.AwaitedInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.AwaitedInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.AwaitedInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.AwaitedInfo.__le__" => "Return self<=value.", + "_remote_debugging.AwaitedInfo.__len__" => "Return len(self).", + "_remote_debugging.AwaitedInfo.__lt__" => "Return self<value.", + "_remote_debugging.AwaitedInfo.__mul__" => "Return self*value.", + "_remote_debugging.AwaitedInfo.__ne__" => "Return self!=value.", + "_remote_debugging.AwaitedInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.AwaitedInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.AwaitedInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.AwaitedInfo.__repr__" => "Return repr(self).", + "_remote_debugging.AwaitedInfo.__rmul__" => "Return value*self.", + "_remote_debugging.AwaitedInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.AwaitedInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.AwaitedInfo.__str__" => "Return str(self).", + "_remote_debugging.AwaitedInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.AwaitedInfo.awaited_by" => "List of tasks awaited by this thread", + "_remote_debugging.AwaitedInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.AwaitedInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.AwaitedInfo.thread_id" => "Thread ID", + "_remote_debugging.CoroInfo" => "Information about a coroutine", + "_remote_debugging.CoroInfo.__add__" => "Return self+value.", + "_remote_debugging.CoroInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.CoroInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.CoroInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.CoroInfo.__eq__" => "Return self==value.", + "_remote_debugging.CoroInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.CoroInfo.__ge__" => "Return self>=value.", + "_remote_debugging.CoroInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.CoroInfo.__getitem__" => "Return self[key].", + "_remote_debugging.CoroInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.CoroInfo.__gt__" => "Return self>value.", + "_remote_debugging.CoroInfo.__hash__" => "Return hash(self).", + "_remote_debugging.CoroInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.CoroInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.CoroInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.CoroInfo.__le__" => "Return self<=value.", + "_remote_debugging.CoroInfo.__len__" => "Return len(self).", + "_remote_debugging.CoroInfo.__lt__" => "Return self<value.", + "_remote_debugging.CoroInfo.__mul__" => "Return self*value.", + "_remote_debugging.CoroInfo.__ne__" => "Return self!=value.", + "_remote_debugging.CoroInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.CoroInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.CoroInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.CoroInfo.__repr__" => "Return repr(self).", + "_remote_debugging.CoroInfo.__rmul__" => "Return value*self.", + "_remote_debugging.CoroInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.CoroInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.CoroInfo.__str__" => "Return str(self).", + "_remote_debugging.CoroInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.CoroInfo.call_stack" => "Coroutine call stack", + "_remote_debugging.CoroInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.CoroInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.CoroInfo.task_name" => "Task name", + "_remote_debugging.FrameInfo" => "Information about a frame", + "_remote_debugging.FrameInfo.__add__" => "Return self+value.", + "_remote_debugging.FrameInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.FrameInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.FrameInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.FrameInfo.__eq__" => "Return self==value.", + "_remote_debugging.FrameInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.FrameInfo.__ge__" => "Return self>=value.", + "_remote_debugging.FrameInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.FrameInfo.__getitem__" => "Return self[key].", + "_remote_debugging.FrameInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.FrameInfo.__gt__" => "Return self>value.", + "_remote_debugging.FrameInfo.__hash__" => "Return hash(self).", + "_remote_debugging.FrameInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.FrameInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.FrameInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.FrameInfo.__le__" => "Return self<=value.", + "_remote_debugging.FrameInfo.__len__" => "Return len(self).", + "_remote_debugging.FrameInfo.__lt__" => "Return self<value.", + "_remote_debugging.FrameInfo.__mul__" => "Return self*value.", + "_remote_debugging.FrameInfo.__ne__" => "Return self!=value.", + "_remote_debugging.FrameInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.FrameInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.FrameInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.FrameInfo.__repr__" => "Return repr(self).", + "_remote_debugging.FrameInfo.__rmul__" => "Return value*self.", + "_remote_debugging.FrameInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.FrameInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.FrameInfo.__str__" => "Return str(self).", + "_remote_debugging.FrameInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.FrameInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.FrameInfo.filename" => "Source code filename", + "_remote_debugging.FrameInfo.funcname" => "Function name", + "_remote_debugging.FrameInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.FrameInfo.lineno" => "Line number", + "_remote_debugging.RemoteUnwinder" => "RemoteUnwinder(pid): Inspect stack of a remote Python process.", + "_remote_debugging.RemoteUnwinder.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.RemoteUnwinder.__eq__" => "Return self==value.", + "_remote_debugging.RemoteUnwinder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.RemoteUnwinder.__ge__" => "Return self>=value.", + "_remote_debugging.RemoteUnwinder.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.RemoteUnwinder.__getstate__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__gt__" => "Return self>value.", + "_remote_debugging.RemoteUnwinder.__hash__" => "Return hash(self).", + "_remote_debugging.RemoteUnwinder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.RemoteUnwinder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.RemoteUnwinder.__le__" => "Return self<=value.", + "_remote_debugging.RemoteUnwinder.__lt__" => "Return self<value.", + "_remote_debugging.RemoteUnwinder.__ne__" => "Return self!=value.", + "_remote_debugging.RemoteUnwinder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.RemoteUnwinder.__reduce__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__repr__" => "Return repr(self).", + "_remote_debugging.RemoteUnwinder.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.RemoteUnwinder.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.RemoteUnwinder.__str__" => "Return str(self).", + "_remote_debugging.RemoteUnwinder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.RemoteUnwinder.get_all_awaited_by" => "Get all tasks and their awaited_by relationships from the remote process.\n\nThis provides a tree structure showing which tasks are waiting for other tasks.\n\nFor each task, returns:\n1. The call stack frames leading to where the task is currently executing\n2. The name of the task\n3. A list of tasks that this task is waiting for, with their own frames/names/etc\n\nReturns a list of [frames, task_name, subtasks] where:\n- frames: List of (func_name, filename, lineno) showing the call stack\n- task_name: String identifier for the task\n- subtasks: List of tasks being awaited by this task, in same format\n\nRaises:\n RuntimeError: If AsyncioDebug section is not available in the remote process\n MemoryError: If memory allocation fails\n OSError: If reading from the remote process fails\n\nExample output:\n[\n [\n [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n \"c2_root\",\n [\n [\n [(\"c1\", \"script.py\", 23)],\n \"sub_main_2\",\n [...]\n ],\n [...]\n ]\n ]\n]", + "_remote_debugging.RemoteUnwinder.get_async_stack_trace" => "Get the currently running async tasks and their dependency graphs from the remote process.\n\nThis returns information about running tasks and all tasks that are waiting for them,\nforming a complete dependency graph for each thread's active task.\n\nFor each thread with a running task, returns the running task plus all tasks that\ntransitively depend on it (tasks waiting for the running task, tasks waiting for\nthose tasks, etc.).\n\nReturns a list of per-thread results, where each thread result contains:\n- Thread ID\n- List of task information for the running task and all its waiters\n\nEach task info contains:\n- Task ID (memory address)\n- Task name\n- Call stack frames: List of (func_name, filename, lineno)\n- List of tasks waiting for this task (recursive structure)\n\nRaises:\n RuntimeError: If AsyncioDebug section is not available in the target process\n MemoryError: If memory allocation fails\n OSError: If reading from the remote process fails\n\nExample output (similar structure to get_all_awaited_by but only for running tasks):\n[\n (140234, [\n (4345585712, 'main_task',\n [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n [\n (4345585800, 'worker_1', [...], [...]),\n (4345585900, 'worker_2', [...], [...])\n ])\n ])\n]", + "_remote_debugging.RemoteUnwinder.get_stack_trace" => "Returns a list of stack traces for threads in the target process.\n\nEach element in the returned list is a tuple of (thread_id, frame_list), where:\n- thread_id is the OS thread identifier\n- frame_list is a list of tuples (function_name, filename, line_number) representing\n the Python stack frames for that thread, ordered from most recent to oldest\n\nThe threads returned depend on the initialization parameters:\n- If only_active_thread was True: returns only the thread holding the GIL\n- If all_threads was True: returns all threads\n- Otherwise: returns only the main thread\n\nExample:\n [\n (1234, [\n ('process_data', 'worker.py', 127),\n ('run_worker', 'worker.py', 45),\n ('main', 'app.py', 23)\n ]),\n (1235, [\n ('handle_request', 'server.py', 89),\n ('serve_forever', 'server.py', 52)\n ])\n ]\n\nRaises:\n RuntimeError: If there is an error copying memory from the target process\n OSError: If there is an error accessing the target process\n PermissionError: If access to the target process is denied\n UnicodeDecodeError: If there is an error decoding strings from the target process", + "_remote_debugging.TaskInfo" => "Information about an asyncio task", + "_remote_debugging.TaskInfo.__add__" => "Return self+value.", + "_remote_debugging.TaskInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.TaskInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.TaskInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.TaskInfo.__eq__" => "Return self==value.", + "_remote_debugging.TaskInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.TaskInfo.__ge__" => "Return self>=value.", + "_remote_debugging.TaskInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.TaskInfo.__getitem__" => "Return self[key].", + "_remote_debugging.TaskInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.TaskInfo.__gt__" => "Return self>value.", + "_remote_debugging.TaskInfo.__hash__" => "Return hash(self).", + "_remote_debugging.TaskInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.TaskInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.TaskInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.TaskInfo.__le__" => "Return self<=value.", + "_remote_debugging.TaskInfo.__len__" => "Return len(self).", + "_remote_debugging.TaskInfo.__lt__" => "Return self<value.", + "_remote_debugging.TaskInfo.__mul__" => "Return self*value.", + "_remote_debugging.TaskInfo.__ne__" => "Return self!=value.", + "_remote_debugging.TaskInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.TaskInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.TaskInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.TaskInfo.__repr__" => "Return repr(self).", + "_remote_debugging.TaskInfo.__rmul__" => "Return value*self.", + "_remote_debugging.TaskInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.TaskInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.TaskInfo.__str__" => "Return str(self).", + "_remote_debugging.TaskInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.TaskInfo.awaited_by" => "Tasks awaiting this task", + "_remote_debugging.TaskInfo.coroutine_stack" => "Coroutine call stack", + "_remote_debugging.TaskInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.TaskInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.TaskInfo.task_id" => "Task ID (memory address)", + "_remote_debugging.TaskInfo.task_name" => "Task name", + "_remote_debugging.ThreadInfo" => "Information about a thread", + "_remote_debugging.ThreadInfo.__add__" => "Return self+value.", + "_remote_debugging.ThreadInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.ThreadInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.ThreadInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.ThreadInfo.__eq__" => "Return self==value.", + "_remote_debugging.ThreadInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.ThreadInfo.__ge__" => "Return self>=value.", + "_remote_debugging.ThreadInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.ThreadInfo.__getitem__" => "Return self[key].", + "_remote_debugging.ThreadInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.ThreadInfo.__gt__" => "Return self>value.", + "_remote_debugging.ThreadInfo.__hash__" => "Return hash(self).", + "_remote_debugging.ThreadInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.ThreadInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.ThreadInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.ThreadInfo.__le__" => "Return self<=value.", + "_remote_debugging.ThreadInfo.__len__" => "Return len(self).", + "_remote_debugging.ThreadInfo.__lt__" => "Return self<value.", + "_remote_debugging.ThreadInfo.__mul__" => "Return self*value.", + "_remote_debugging.ThreadInfo.__ne__" => "Return self!=value.", + "_remote_debugging.ThreadInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.ThreadInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.ThreadInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.ThreadInfo.__repr__" => "Return repr(self).", + "_remote_debugging.ThreadInfo.__rmul__" => "Return value*self.", + "_remote_debugging.ThreadInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.ThreadInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.ThreadInfo.__str__" => "Return str(self).", + "_remote_debugging.ThreadInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.ThreadInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.ThreadInfo.frame_info" => "Frame information", + "_remote_debugging.ThreadInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.ThreadInfo.thread_id" => "Thread ID", "_sha1.SHA1Type.__delattr__" => "Implement delattr(self, name).", "_sha1.SHA1Type.__eq__" => "Return self==value.", "_sha1.SHA1Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2713,6 +7336,33 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_sha3.shake_256.hexdigest" => "Return the digest value as a string of hexadecimal digits.", "_sha3.shake_256.update" => "Update this hash object's state with the provided bytes-like object.", "_signal" => "This module provides mechanisms to use signal handlers in Python.\n\nFunctions:\n\nalarm() -- cause SIGALRM after a specified time [Unix only]\nsetitimer() -- cause a signal (described below) after a specified\n float time and the timer may restart then [Unix only]\ngetitimer() -- get current value of timer [Unix only]\nsignal() -- set the action for a given signal\ngetsignal() -- get the signal action for a given signal\npause() -- wait until a signal arrives [Unix only]\ndefault_int_handler() -- default SIGINT handler\n\nsignal constants:\nSIG_DFL -- used to refer to the system default handler\nSIG_IGN -- used to ignore the signal\nNSIG -- number of defined signals\nSIGINT, SIGTERM, etc. -- signal numbers\n\nitimer constants:\nITIMER_REAL -- decrements in real time, and delivers SIGALRM upon\n expiration\nITIMER_VIRTUAL -- decrements only when the process is executing,\n and delivers SIGVTALRM upon expiration\nITIMER_PROF -- decrements both when the process is executing and\n when the system is executing on behalf of the process.\n Coupled with ITIMER_VIRTUAL, this timer is usually\n used to profile the time spent by the application\n in user and kernel space. SIGPROF is delivered upon\n expiration.\n\n\n*** IMPORTANT NOTICE ***\nA signal handler function is called with two arguments:\nthe first is the signal number, the second is the interrupted stack frame.", + "_signal.ItimerError.__delattr__" => "Implement delattr(self, name).", + "_signal.ItimerError.__eq__" => "Return self==value.", + "_signal.ItimerError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_signal.ItimerError.__ge__" => "Return self>=value.", + "_signal.ItimerError.__getattribute__" => "Return getattr(self, name).", + "_signal.ItimerError.__getstate__" => "Helper for pickle.", + "_signal.ItimerError.__gt__" => "Return self>value.", + "_signal.ItimerError.__hash__" => "Return hash(self).", + "_signal.ItimerError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_signal.ItimerError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_signal.ItimerError.__le__" => "Return self<=value.", + "_signal.ItimerError.__lt__" => "Return self<value.", + "_signal.ItimerError.__ne__" => "Return self!=value.", + "_signal.ItimerError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_signal.ItimerError.__reduce_ex__" => "Helper for pickle.", + "_signal.ItimerError.__repr__" => "Return repr(self).", + "_signal.ItimerError.__setattr__" => "Implement setattr(self, name, value).", + "_signal.ItimerError.__sizeof__" => "Size of object in memory, in bytes.", + "_signal.ItimerError.__str__" => "Return str(self).", + "_signal.ItimerError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_signal.ItimerError.__weakref__" => "list of weak references to the object", + "_signal.ItimerError.add_note" => "Add a note to the exception", + "_signal.ItimerError.errno" => "POSIX exception code", + "_signal.ItimerError.filename" => "exception filename", + "_signal.ItimerError.filename2" => "second exception filename", + "_signal.ItimerError.strerror" => "exception strerror", + "_signal.ItimerError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_signal.alarm" => "Arrange for SIGALRM to arrive after the given number of seconds.", "_signal.default_int_handler" => "The default handler for SIGINT installed by Python.\n\nIt raises KeyboardInterrupt.", "_signal.getitimer" => "Returns current value of given itimer.", @@ -2731,6 +7381,45 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_signal.sigwait" => "Wait for a signal.\n\nSuspend execution of the calling thread until the delivery of one of the\nsignals specified in the signal set sigset. The function accepts the signal\nand returns the signal number.", "_signal.sigwaitinfo" => "Wait synchronously until one of the signals in *sigset* is delivered.\n\nReturns a struct_siginfo containing information about the signal.", "_signal.strsignal" => "Return the system description of the given signal.\n\nReturns the description of signal *signalnum*, such as \"Interrupt\"\nfor :const:`SIGINT`. Returns :const:`None` if *signalnum* has no\ndescription. Raises :exc:`ValueError` if *signalnum* is invalid.", + "_signal.struct_siginfo" => "struct_siginfo: Result from sigwaitinfo or sigtimedwait.\n\nThis object may be accessed either as a tuple of\n(si_signo, si_code, si_errno, si_pid, si_uid, si_status, si_band),\nor via the attributes si_signo, si_code, and so on.", + "_signal.struct_siginfo.__add__" => "Return self+value.", + "_signal.struct_siginfo.__class_getitem__" => "See PEP 585", + "_signal.struct_siginfo.__contains__" => "Return bool(key in self).", + "_signal.struct_siginfo.__delattr__" => "Implement delattr(self, name).", + "_signal.struct_siginfo.__eq__" => "Return self==value.", + "_signal.struct_siginfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_signal.struct_siginfo.__ge__" => "Return self>=value.", + "_signal.struct_siginfo.__getattribute__" => "Return getattr(self, name).", + "_signal.struct_siginfo.__getitem__" => "Return self[key].", + "_signal.struct_siginfo.__getstate__" => "Helper for pickle.", + "_signal.struct_siginfo.__gt__" => "Return self>value.", + "_signal.struct_siginfo.__hash__" => "Return hash(self).", + "_signal.struct_siginfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_signal.struct_siginfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_signal.struct_siginfo.__iter__" => "Implement iter(self).", + "_signal.struct_siginfo.__le__" => "Return self<=value.", + "_signal.struct_siginfo.__len__" => "Return len(self).", + "_signal.struct_siginfo.__lt__" => "Return self<value.", + "_signal.struct_siginfo.__mul__" => "Return self*value.", + "_signal.struct_siginfo.__ne__" => "Return self!=value.", + "_signal.struct_siginfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_signal.struct_siginfo.__reduce_ex__" => "Helper for pickle.", + "_signal.struct_siginfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_signal.struct_siginfo.__repr__" => "Return repr(self).", + "_signal.struct_siginfo.__rmul__" => "Return value*self.", + "_signal.struct_siginfo.__setattr__" => "Implement setattr(self, name, value).", + "_signal.struct_siginfo.__sizeof__" => "Size of object in memory, in bytes.", + "_signal.struct_siginfo.__str__" => "Return str(self).", + "_signal.struct_siginfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_signal.struct_siginfo.count" => "Return number of occurrences of value.", + "_signal.struct_siginfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_signal.struct_siginfo.si_band" => "band event for SIGPOLL", + "_signal.struct_siginfo.si_code" => "signal code", + "_signal.struct_siginfo.si_errno" => "errno associated with this signal", + "_signal.struct_siginfo.si_pid" => "sending process ID", + "_signal.struct_siginfo.si_signo" => "signal number", + "_signal.struct_siginfo.si_status" => "exit value or signal", + "_signal.struct_siginfo.si_uid" => "real user ID of sending process", "_signal.valid_signals" => "Return a set of valid signal numbers on this platform.\n\nThe signal numbers returned by this function can be safely passed to\nfunctions like `pthread_sigmask`.", "_socket" => "Implementation module for socket operations.\n\nSee the socket module for documentation.", "_socket.CMSG_LEN" => "CMSG_LEN(length) -> control message length\n\nReturn the total length, without trailing padding, of an ancillary\ndata item with associated data of the given length. This value can\noften be used as the buffer size for recvmsg() to receive a single\nitem of ancillary data, but RFC 3542 requires portable applications to\nuse CMSG_SPACE() and thus include space for padding, even when the\nitem will be the last in the buffer. Raises OverflowError if length\nis outside the permissible range of values.", @@ -2794,6 +7483,34 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_socket.SocketType.type" => "the socket type", "_socket.close" => "close(integer) -> None\n\nClose an integer socket file descriptor. This is like os.close(), but for\nsockets; on some platforms os.close() won't work for socket file descriptors.", "_socket.dup" => "dup(integer) -> integer\n\nDuplicate an integer socket file descriptor. This is like os.dup(), but for\nsockets; on some platforms os.dup() won't work for socket file descriptors.", + "_socket.gaierror.__delattr__" => "Implement delattr(self, name).", + "_socket.gaierror.__eq__" => "Return self==value.", + "_socket.gaierror.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.gaierror.__ge__" => "Return self>=value.", + "_socket.gaierror.__getattribute__" => "Return getattr(self, name).", + "_socket.gaierror.__getstate__" => "Helper for pickle.", + "_socket.gaierror.__gt__" => "Return self>value.", + "_socket.gaierror.__hash__" => "Return hash(self).", + "_socket.gaierror.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.gaierror.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.gaierror.__le__" => "Return self<=value.", + "_socket.gaierror.__lt__" => "Return self<value.", + "_socket.gaierror.__ne__" => "Return self!=value.", + "_socket.gaierror.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.gaierror.__reduce_ex__" => "Helper for pickle.", + "_socket.gaierror.__repr__" => "Return repr(self).", + "_socket.gaierror.__setattr__" => "Implement setattr(self, name, value).", + "_socket.gaierror.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.gaierror.__str__" => "Return str(self).", + "_socket.gaierror.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.gaierror.__weakref__" => "list of weak references to the object", + "_socket.gaierror.add_note" => "Add a note to the exception", + "_socket.gaierror.errno" => "POSIX exception code", + "_socket.gaierror.filename" => "exception filename", + "_socket.gaierror.filename2" => "second exception filename", + "_socket.gaierror.strerror" => "exception strerror", + "_socket.gaierror.winerror" => "Win32 exception code", + "_socket.gaierror.with_traceback" => "Set self.__traceback__ to tb and return self.", "_socket.getaddrinfo" => "getaddrinfo(host, port [, family, type, proto, flags])\n -> list of (family, type, proto, canonname, sockaddr)\n\nResolve host and port into addrinfo struct.", "_socket.getdefaulttimeout" => "getdefaulttimeout() -> timeout\n\nReturns the default timeout in seconds (float) for new socket objects.\nA value of None indicates that new socket objects have no timeout.\nWhen the socket module is first imported, the default is None.", "_socket.gethostbyaddr" => "gethostbyaddr(host) -> (name, aliaslist, addresslist)\n\nReturn the true host name, a list of aliases, and a list of IP addresses,\nfor a host. The host argument is a string giving a host name or IP number.", @@ -2804,16 +7521,44 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_socket.getprotobyname" => "getprotobyname(name) -> integer\n\nReturn the protocol number for the named protocol. (Rarely used.)", "_socket.getservbyname" => "getservbyname(servicename[, protocolname]) -> integer\n\nReturn a port number from a service name and protocol name.\nThe optional protocol name, if given, should be 'tcp' or 'udp',\notherwise any protocol will match.", "_socket.getservbyport" => "getservbyport(port[, protocolname]) -> string\n\nReturn the service name from a port number and protocol name.\nThe optional protocol name, if given, should be 'tcp' or 'udp',\notherwise any protocol will match.", - "_socket.htonl" => "htonl(integer) -> integer\n\nConvert a 32-bit integer from host to network byte order.", + "_socket.herror.__delattr__" => "Implement delattr(self, name).", + "_socket.herror.__eq__" => "Return self==value.", + "_socket.herror.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.herror.__ge__" => "Return self>=value.", + "_socket.herror.__getattribute__" => "Return getattr(self, name).", + "_socket.herror.__getstate__" => "Helper for pickle.", + "_socket.herror.__gt__" => "Return self>value.", + "_socket.herror.__hash__" => "Return hash(self).", + "_socket.herror.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.herror.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.herror.__le__" => "Return self<=value.", + "_socket.herror.__lt__" => "Return self<value.", + "_socket.herror.__ne__" => "Return self!=value.", + "_socket.herror.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.herror.__reduce_ex__" => "Helper for pickle.", + "_socket.herror.__repr__" => "Return repr(self).", + "_socket.herror.__setattr__" => "Implement setattr(self, name, value).", + "_socket.herror.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.herror.__str__" => "Return str(self).", + "_socket.herror.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.herror.__weakref__" => "list of weak references to the object", + "_socket.herror.add_note" => "Add a note to the exception", + "_socket.herror.errno" => "POSIX exception code", + "_socket.herror.filename" => "exception filename", + "_socket.herror.filename2" => "second exception filename", + "_socket.herror.strerror" => "exception strerror", + "_socket.herror.winerror" => "Win32 exception code", + "_socket.herror.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_socket.htonl" => "Convert a 32-bit unsigned integer from host to network byte order.", "_socket.htons" => "Convert a 16-bit unsigned integer from host to network byte order.", - "_socket.if_indextoname" => "if_indextoname(if_index)\n\nReturns the interface name corresponding to the interface index if_index.", + "_socket.if_indextoname" => "Returns the interface name corresponding to the interface index if_index.", "_socket.if_nameindex" => "if_nameindex()\n\nReturns a list of network interface information (index, name) tuples.", "_socket.if_nametoindex" => "Returns the interface index corresponding to the interface name if_name.", "_socket.inet_aton" => "Convert an IP address in string format (123.45.67.89) to the 32-bit packed binary format used in low-level network functions.", "_socket.inet_ntoa" => "Convert an IP address from 32-bit packed binary format to string format.", "_socket.inet_ntop" => "inet_ntop(af, packed_ip) -> string formatted IP address\n\nConvert a packed IP address of the given family to string format.", "_socket.inet_pton" => "inet_pton(af, ip) -> packed IP address string\n\nConvert an IP address from string format to a packed string suitable\nfor use with low-level network functions.", - "_socket.ntohl" => "ntohl(integer) -> integer\n\nConvert a 32-bit integer from network to host byte order.", + "_socket.ntohl" => "Convert a 32-bit unsigned integer from network to host byte order.", "_socket.ntohs" => "Convert a 16-bit unsigned integer from network to host byte order.", "_socket.setdefaulttimeout" => "setdefaulttimeout(timeout)\n\nSet the default timeout in seconds (float) for new socket objects.\nA value of None indicates that new socket objects have no timeout.\nWhen the socket module is first imported, the default is None.", "_socket.sethostname" => "sethostname(name)\n\nSets the hostname to name.", @@ -2875,13 +7620,407 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_socket.socket.timeout" => "the socket timeout", "_socket.socket.type" => "the socket type", "_socket.socketpair" => "socketpair([family[, type [, proto]]]) -> (socket object, socket object)\n\nCreate a pair of socket objects from the sockets returned by the platform\nsocketpair() function.\nThe arguments are the same as for socket() except the default family is\nAF_UNIX if defined on the platform; otherwise, the default is AF_INET.", + "_sqlite3.Blob.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Blob.__delitem__" => "Delete self[key].", + "_sqlite3.Blob.__enter__" => "Blob context manager enter.", + "_sqlite3.Blob.__eq__" => "Return self==value.", + "_sqlite3.Blob.__exit__" => "Blob context manager exit.", + "_sqlite3.Blob.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Blob.__ge__" => "Return self>=value.", + "_sqlite3.Blob.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Blob.__getitem__" => "Return self[key].", + "_sqlite3.Blob.__getstate__" => "Helper for pickle.", + "_sqlite3.Blob.__gt__" => "Return self>value.", + "_sqlite3.Blob.__hash__" => "Return hash(self).", + "_sqlite3.Blob.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Blob.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Blob.__le__" => "Return self<=value.", + "_sqlite3.Blob.__len__" => "Return len(self).", + "_sqlite3.Blob.__lt__" => "Return self<value.", + "_sqlite3.Blob.__ne__" => "Return self!=value.", + "_sqlite3.Blob.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Blob.__reduce__" => "Helper for pickle.", + "_sqlite3.Blob.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Blob.__repr__" => "Return repr(self).", + "_sqlite3.Blob.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Blob.__setitem__" => "Set self[key] to value.", + "_sqlite3.Blob.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Blob.__str__" => "Return str(self).", + "_sqlite3.Blob.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Blob.close" => "Close the blob.", + "_sqlite3.Blob.read" => "Read data at the current offset position.\n\n length\n Read length in bytes.\n\nIf the end of the blob is reached, the data up to end of file will be returned.\nWhen length is not specified, or is negative, Blob.read() will read until the\nend of the blob.", + "_sqlite3.Blob.seek" => "Set the current access position to offset.\n\nThe origin argument defaults to os.SEEK_SET (absolute blob positioning).\nOther values for origin are os.SEEK_CUR (seek relative to the current position)\nand os.SEEK_END (seek relative to the blob's end).", + "_sqlite3.Blob.tell" => "Return the current access position for the blob.", + "_sqlite3.Blob.write" => "Write data at the current offset.\n\nThis function cannot change the blob length. Writing beyond the end of the\nblob will result in an exception being raised.", + "_sqlite3.Connection" => "SQLite database connection object.", + "_sqlite3.Connection.__call__" => "Call self as a function.", + "_sqlite3.Connection.__del__" => "Called when the instance is about to be destroyed.", + "_sqlite3.Connection.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Connection.__enter__" => "Called when the connection is used as a context manager.\n\nReturns itself as a convenience to the caller.", + "_sqlite3.Connection.__eq__" => "Return self==value.", + "_sqlite3.Connection.__exit__" => "Called when the connection is used as a context manager.\n\nIf there was any exception, a rollback takes place; otherwise we commit.", + "_sqlite3.Connection.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Connection.__ge__" => "Return self>=value.", + "_sqlite3.Connection.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Connection.__getstate__" => "Helper for pickle.", + "_sqlite3.Connection.__gt__" => "Return self>value.", + "_sqlite3.Connection.__hash__" => "Return hash(self).", + "_sqlite3.Connection.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Connection.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Connection.__le__" => "Return self<=value.", + "_sqlite3.Connection.__lt__" => "Return self<value.", + "_sqlite3.Connection.__ne__" => "Return self!=value.", + "_sqlite3.Connection.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Connection.__reduce__" => "Helper for pickle.", + "_sqlite3.Connection.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Connection.__repr__" => "Return repr(self).", + "_sqlite3.Connection.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Connection.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Connection.__str__" => "Return str(self).", + "_sqlite3.Connection.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Connection.backup" => "Makes a backup of the database.", + "_sqlite3.Connection.blobopen" => "Open and return a BLOB object.\n\n table\n Table name.\n column\n Column name.\n rowid\n Row id.\n readonly\n Open the BLOB without write permissions.\n name\n Database name.", + "_sqlite3.Connection.close" => "Close the database connection.\n\nAny pending transaction is not committed implicitly.", + "_sqlite3.Connection.commit" => "Commit any pending transaction to the database.\n\nIf there is no open transaction, this method is a no-op.", + "_sqlite3.Connection.create_aggregate" => "Creates a new aggregate.\n\nNote: Passing keyword arguments 'name', 'n_arg' and 'aggregate_class'\nto _sqlite3.Connection.create_aggregate() is deprecated. Parameters\n'name', 'n_arg' and 'aggregate_class' will become positional-only in\nPython 3.15.", + "_sqlite3.Connection.create_collation" => "Creates a collation function.", + "_sqlite3.Connection.create_function" => "Creates a new function.\n\nNote: Passing keyword arguments 'name', 'narg' and 'func' to\n_sqlite3.Connection.create_function() is deprecated. Parameters\n'name', 'narg' and 'func' will become positional-only in Python 3.15.", + "_sqlite3.Connection.create_window_function" => "Creates or redefines an aggregate window function. Non-standard.\n\n name\n The name of the SQL aggregate window function to be created or\n redefined.\n num_params\n The number of arguments the step and inverse methods takes.\n aggregate_class\n A class with step(), finalize(), value(), and inverse() methods.\n Set to None to clear the window function.", + "_sqlite3.Connection.cursor" => "Return a cursor for the connection.", + "_sqlite3.Connection.deserialize" => "Load a serialized database.\n\n data\n The serialized database content.\n name\n Which database to reopen with the deserialization.\n\nThe deserialize interface causes the database connection to disconnect from the\ntarget database, and then reopen it as an in-memory database based on the given\nserialized data.\n\nThe deserialize interface will fail with SQLITE_BUSY if the database is\ncurrently in a read transaction or is involved in a backup operation.", + "_sqlite3.Connection.enable_load_extension" => "Enable dynamic loading of SQLite extension modules.", + "_sqlite3.Connection.execute" => "Executes an SQL statement.", + "_sqlite3.Connection.executemany" => "Repeatedly executes an SQL statement.", + "_sqlite3.Connection.executescript" => "Executes multiple SQL statements at once.", + "_sqlite3.Connection.getconfig" => "Query a boolean connection configuration option.\n\n op\n The configuration verb; one of the sqlite3.SQLITE_DBCONFIG codes.", + "_sqlite3.Connection.getlimit" => "Get connection run-time limits.\n\n category\n The limit category to be queried.", + "_sqlite3.Connection.interrupt" => "Abort any pending database operation.", + "_sqlite3.Connection.iterdump" => "Returns iterator to the dump of the database in an SQL text format.\n\n filter\n An optional LIKE pattern for database objects to dump", + "_sqlite3.Connection.load_extension" => "Load SQLite extension module.", + "_sqlite3.Connection.rollback" => "Roll back to the start of any pending transaction.\n\nIf there is no open transaction, this method is a no-op.", + "_sqlite3.Connection.serialize" => "Serialize a database into a byte string.\n\n name\n Which database to serialize.\n\nFor an ordinary on-disk database file, the serialization is just a copy of the\ndisk file. For an in-memory database or a \"temp\" database, the serialization is\nthe same sequence of bytes which would be written to disk if that database\nwere backed up to disk.", + "_sqlite3.Connection.set_authorizer" => "Set authorizer callback.\n\nNote: Passing keyword argument 'authorizer_callback' to\n_sqlite3.Connection.set_authorizer() is deprecated. Parameter\n'authorizer_callback' will become positional-only in Python 3.15.", + "_sqlite3.Connection.set_progress_handler" => "Set progress handler callback.\n\n progress_handler\n A callable that takes no arguments.\n If the callable returns non-zero, the current query is terminated,\n and an exception is raised.\n n\n The number of SQLite virtual machine instructions that are\n executed between invocations of 'progress_handler'.\n\nIf 'progress_handler' is None or 'n' is 0, the progress handler is disabled.\n\nNote: Passing keyword argument 'progress_handler' to\n_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n'progress_handler' will become positional-only in Python 3.15.", + "_sqlite3.Connection.set_trace_callback" => "Set a trace callback called for each SQL statement (passed as unicode).\n\nNote: Passing keyword argument 'trace_callback' to\n_sqlite3.Connection.set_trace_callback() is deprecated. Parameter\n'trace_callback' will become positional-only in Python 3.15.", + "_sqlite3.Connection.setconfig" => "Set a boolean connection configuration option.\n\n op\n The configuration verb; one of the sqlite3.SQLITE_DBCONFIG codes.", + "_sqlite3.Connection.setlimit" => "Set connection run-time limits.\n\n category\n The limit category to be set.\n limit\n The new limit. If the new limit is a negative number, the limit is\n unchanged.\n\nAttempts to increase a limit above its hard upper bound are silently truncated\nto the hard upper bound. Regardless of whether or not the limit was changed,\nthe prior value of the limit is returned.", + "_sqlite3.Cursor" => "SQLite database cursor class.", + "_sqlite3.Cursor.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Cursor.__eq__" => "Return self==value.", + "_sqlite3.Cursor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Cursor.__ge__" => "Return self>=value.", + "_sqlite3.Cursor.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Cursor.__getstate__" => "Helper for pickle.", + "_sqlite3.Cursor.__gt__" => "Return self>value.", + "_sqlite3.Cursor.__hash__" => "Return hash(self).", + "_sqlite3.Cursor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Cursor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Cursor.__iter__" => "Implement iter(self).", + "_sqlite3.Cursor.__le__" => "Return self<=value.", + "_sqlite3.Cursor.__lt__" => "Return self<value.", + "_sqlite3.Cursor.__ne__" => "Return self!=value.", + "_sqlite3.Cursor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Cursor.__next__" => "Implement next(self).", + "_sqlite3.Cursor.__reduce__" => "Helper for pickle.", + "_sqlite3.Cursor.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Cursor.__repr__" => "Return repr(self).", + "_sqlite3.Cursor.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Cursor.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Cursor.__str__" => "Return str(self).", + "_sqlite3.Cursor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Cursor.close" => "Closes the cursor.", + "_sqlite3.Cursor.execute" => "Executes an SQL statement.", + "_sqlite3.Cursor.executemany" => "Repeatedly executes an SQL statement.", + "_sqlite3.Cursor.executescript" => "Executes multiple SQL statements at once.", + "_sqlite3.Cursor.fetchall" => "Fetches all rows from the resultset.", + "_sqlite3.Cursor.fetchmany" => "Fetches several rows from the resultset.\n\n size\n The default value is set by the Cursor.arraysize attribute.", + "_sqlite3.Cursor.fetchone" => "Fetches one row from the resultset.", + "_sqlite3.Cursor.setinputsizes" => "Required by DB-API. Does nothing in sqlite3.", + "_sqlite3.Cursor.setoutputsize" => "Required by DB-API. Does nothing in sqlite3.", + "_sqlite3.DataError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.DataError.__eq__" => "Return self==value.", + "_sqlite3.DataError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.DataError.__ge__" => "Return self>=value.", + "_sqlite3.DataError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.DataError.__getstate__" => "Helper for pickle.", + "_sqlite3.DataError.__gt__" => "Return self>value.", + "_sqlite3.DataError.__hash__" => "Return hash(self).", + "_sqlite3.DataError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.DataError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.DataError.__le__" => "Return self<=value.", + "_sqlite3.DataError.__lt__" => "Return self<value.", + "_sqlite3.DataError.__ne__" => "Return self!=value.", + "_sqlite3.DataError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.DataError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.DataError.__repr__" => "Return repr(self).", + "_sqlite3.DataError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.DataError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.DataError.__str__" => "Return str(self).", + "_sqlite3.DataError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.DataError.__weakref__" => "list of weak references to the object", + "_sqlite3.DataError.add_note" => "Add a note to the exception", + "_sqlite3.DataError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.DatabaseError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.DatabaseError.__eq__" => "Return self==value.", + "_sqlite3.DatabaseError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.DatabaseError.__ge__" => "Return self>=value.", + "_sqlite3.DatabaseError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.DatabaseError.__getstate__" => "Helper for pickle.", + "_sqlite3.DatabaseError.__gt__" => "Return self>value.", + "_sqlite3.DatabaseError.__hash__" => "Return hash(self).", + "_sqlite3.DatabaseError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.DatabaseError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.DatabaseError.__le__" => "Return self<=value.", + "_sqlite3.DatabaseError.__lt__" => "Return self<value.", + "_sqlite3.DatabaseError.__ne__" => "Return self!=value.", + "_sqlite3.DatabaseError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.DatabaseError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.DatabaseError.__repr__" => "Return repr(self).", + "_sqlite3.DatabaseError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.DatabaseError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.DatabaseError.__str__" => "Return str(self).", + "_sqlite3.DatabaseError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.DatabaseError.__weakref__" => "list of weak references to the object", + "_sqlite3.DatabaseError.add_note" => "Add a note to the exception", + "_sqlite3.DatabaseError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.Error.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Error.__eq__" => "Return self==value.", + "_sqlite3.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Error.__ge__" => "Return self>=value.", + "_sqlite3.Error.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Error.__getstate__" => "Helper for pickle.", + "_sqlite3.Error.__gt__" => "Return self>value.", + "_sqlite3.Error.__hash__" => "Return hash(self).", + "_sqlite3.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Error.__le__" => "Return self<=value.", + "_sqlite3.Error.__lt__" => "Return self<value.", + "_sqlite3.Error.__ne__" => "Return self!=value.", + "_sqlite3.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Error.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Error.__repr__" => "Return repr(self).", + "_sqlite3.Error.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Error.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Error.__str__" => "Return str(self).", + "_sqlite3.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Error.__weakref__" => "list of weak references to the object", + "_sqlite3.Error.add_note" => "Add a note to the exception", + "_sqlite3.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.IntegrityError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.IntegrityError.__eq__" => "Return self==value.", + "_sqlite3.IntegrityError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.IntegrityError.__ge__" => "Return self>=value.", + "_sqlite3.IntegrityError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.IntegrityError.__getstate__" => "Helper for pickle.", + "_sqlite3.IntegrityError.__gt__" => "Return self>value.", + "_sqlite3.IntegrityError.__hash__" => "Return hash(self).", + "_sqlite3.IntegrityError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.IntegrityError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.IntegrityError.__le__" => "Return self<=value.", + "_sqlite3.IntegrityError.__lt__" => "Return self<value.", + "_sqlite3.IntegrityError.__ne__" => "Return self!=value.", + "_sqlite3.IntegrityError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.IntegrityError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.IntegrityError.__repr__" => "Return repr(self).", + "_sqlite3.IntegrityError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.IntegrityError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.IntegrityError.__str__" => "Return str(self).", + "_sqlite3.IntegrityError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.IntegrityError.__weakref__" => "list of weak references to the object", + "_sqlite3.IntegrityError.add_note" => "Add a note to the exception", + "_sqlite3.IntegrityError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.InterfaceError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.InterfaceError.__eq__" => "Return self==value.", + "_sqlite3.InterfaceError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.InterfaceError.__ge__" => "Return self>=value.", + "_sqlite3.InterfaceError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.InterfaceError.__getstate__" => "Helper for pickle.", + "_sqlite3.InterfaceError.__gt__" => "Return self>value.", + "_sqlite3.InterfaceError.__hash__" => "Return hash(self).", + "_sqlite3.InterfaceError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.InterfaceError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.InterfaceError.__le__" => "Return self<=value.", + "_sqlite3.InterfaceError.__lt__" => "Return self<value.", + "_sqlite3.InterfaceError.__ne__" => "Return self!=value.", + "_sqlite3.InterfaceError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.InterfaceError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.InterfaceError.__repr__" => "Return repr(self).", + "_sqlite3.InterfaceError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.InterfaceError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.InterfaceError.__str__" => "Return str(self).", + "_sqlite3.InterfaceError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.InterfaceError.__weakref__" => "list of weak references to the object", + "_sqlite3.InterfaceError.add_note" => "Add a note to the exception", + "_sqlite3.InterfaceError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.InternalError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.InternalError.__eq__" => "Return self==value.", + "_sqlite3.InternalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.InternalError.__ge__" => "Return self>=value.", + "_sqlite3.InternalError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.InternalError.__getstate__" => "Helper for pickle.", + "_sqlite3.InternalError.__gt__" => "Return self>value.", + "_sqlite3.InternalError.__hash__" => "Return hash(self).", + "_sqlite3.InternalError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.InternalError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.InternalError.__le__" => "Return self<=value.", + "_sqlite3.InternalError.__lt__" => "Return self<value.", + "_sqlite3.InternalError.__ne__" => "Return self!=value.", + "_sqlite3.InternalError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.InternalError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.InternalError.__repr__" => "Return repr(self).", + "_sqlite3.InternalError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.InternalError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.InternalError.__str__" => "Return str(self).", + "_sqlite3.InternalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.InternalError.__weakref__" => "list of weak references to the object", + "_sqlite3.InternalError.add_note" => "Add a note to the exception", + "_sqlite3.InternalError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.NotSupportedError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.NotSupportedError.__eq__" => "Return self==value.", + "_sqlite3.NotSupportedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.NotSupportedError.__ge__" => "Return self>=value.", + "_sqlite3.NotSupportedError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.NotSupportedError.__getstate__" => "Helper for pickle.", + "_sqlite3.NotSupportedError.__gt__" => "Return self>value.", + "_sqlite3.NotSupportedError.__hash__" => "Return hash(self).", + "_sqlite3.NotSupportedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.NotSupportedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.NotSupportedError.__le__" => "Return self<=value.", + "_sqlite3.NotSupportedError.__lt__" => "Return self<value.", + "_sqlite3.NotSupportedError.__ne__" => "Return self!=value.", + "_sqlite3.NotSupportedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.NotSupportedError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.NotSupportedError.__repr__" => "Return repr(self).", + "_sqlite3.NotSupportedError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.NotSupportedError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.NotSupportedError.__str__" => "Return str(self).", + "_sqlite3.NotSupportedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.NotSupportedError.__weakref__" => "list of weak references to the object", + "_sqlite3.NotSupportedError.add_note" => "Add a note to the exception", + "_sqlite3.NotSupportedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.OperationalError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.OperationalError.__eq__" => "Return self==value.", + "_sqlite3.OperationalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.OperationalError.__ge__" => "Return self>=value.", + "_sqlite3.OperationalError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.OperationalError.__getstate__" => "Helper for pickle.", + "_sqlite3.OperationalError.__gt__" => "Return self>value.", + "_sqlite3.OperationalError.__hash__" => "Return hash(self).", + "_sqlite3.OperationalError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.OperationalError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.OperationalError.__le__" => "Return self<=value.", + "_sqlite3.OperationalError.__lt__" => "Return self<value.", + "_sqlite3.OperationalError.__ne__" => "Return self!=value.", + "_sqlite3.OperationalError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.OperationalError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.OperationalError.__repr__" => "Return repr(self).", + "_sqlite3.OperationalError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.OperationalError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.OperationalError.__str__" => "Return str(self).", + "_sqlite3.OperationalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.OperationalError.__weakref__" => "list of weak references to the object", + "_sqlite3.OperationalError.add_note" => "Add a note to the exception", + "_sqlite3.OperationalError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.PrepareProtocol" => "PEP 246 style object adaption protocol type.", + "_sqlite3.PrepareProtocol.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.PrepareProtocol.__eq__" => "Return self==value.", + "_sqlite3.PrepareProtocol.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.PrepareProtocol.__ge__" => "Return self>=value.", + "_sqlite3.PrepareProtocol.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.PrepareProtocol.__getstate__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__gt__" => "Return self>value.", + "_sqlite3.PrepareProtocol.__hash__" => "Return hash(self).", + "_sqlite3.PrepareProtocol.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.PrepareProtocol.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.PrepareProtocol.__le__" => "Return self<=value.", + "_sqlite3.PrepareProtocol.__lt__" => "Return self<value.", + "_sqlite3.PrepareProtocol.__ne__" => "Return self!=value.", + "_sqlite3.PrepareProtocol.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.PrepareProtocol.__reduce__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__repr__" => "Return repr(self).", + "_sqlite3.PrepareProtocol.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.PrepareProtocol.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.PrepareProtocol.__str__" => "Return str(self).", + "_sqlite3.PrepareProtocol.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.ProgrammingError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.ProgrammingError.__eq__" => "Return self==value.", + "_sqlite3.ProgrammingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.ProgrammingError.__ge__" => "Return self>=value.", + "_sqlite3.ProgrammingError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.ProgrammingError.__getstate__" => "Helper for pickle.", + "_sqlite3.ProgrammingError.__gt__" => "Return self>value.", + "_sqlite3.ProgrammingError.__hash__" => "Return hash(self).", + "_sqlite3.ProgrammingError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.ProgrammingError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.ProgrammingError.__le__" => "Return self<=value.", + "_sqlite3.ProgrammingError.__lt__" => "Return self<value.", + "_sqlite3.ProgrammingError.__ne__" => "Return self!=value.", + "_sqlite3.ProgrammingError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.ProgrammingError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.ProgrammingError.__repr__" => "Return repr(self).", + "_sqlite3.ProgrammingError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.ProgrammingError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.ProgrammingError.__str__" => "Return str(self).", + "_sqlite3.ProgrammingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.ProgrammingError.__weakref__" => "list of weak references to the object", + "_sqlite3.ProgrammingError.add_note" => "Add a note to the exception", + "_sqlite3.ProgrammingError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.Row.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Row.__eq__" => "Return self==value.", + "_sqlite3.Row.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Row.__ge__" => "Return self>=value.", + "_sqlite3.Row.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Row.__getitem__" => "Return self[key].", + "_sqlite3.Row.__getstate__" => "Helper for pickle.", + "_sqlite3.Row.__gt__" => "Return self>value.", + "_sqlite3.Row.__hash__" => "Return hash(self).", + "_sqlite3.Row.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Row.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Row.__iter__" => "Implement iter(self).", + "_sqlite3.Row.__le__" => "Return self<=value.", + "_sqlite3.Row.__len__" => "Return len(self).", + "_sqlite3.Row.__lt__" => "Return self<value.", + "_sqlite3.Row.__ne__" => "Return self!=value.", + "_sqlite3.Row.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Row.__reduce__" => "Helper for pickle.", + "_sqlite3.Row.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Row.__repr__" => "Return repr(self).", + "_sqlite3.Row.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Row.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Row.__str__" => "Return str(self).", + "_sqlite3.Row.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Row.keys" => "Returns the keys of the row.", + "_sqlite3.Warning.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Warning.__eq__" => "Return self==value.", + "_sqlite3.Warning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Warning.__ge__" => "Return self>=value.", + "_sqlite3.Warning.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Warning.__getstate__" => "Helper for pickle.", + "_sqlite3.Warning.__gt__" => "Return self>value.", + "_sqlite3.Warning.__hash__" => "Return hash(self).", + "_sqlite3.Warning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Warning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Warning.__le__" => "Return self<=value.", + "_sqlite3.Warning.__lt__" => "Return self<value.", + "_sqlite3.Warning.__ne__" => "Return self!=value.", + "_sqlite3.Warning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Warning.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Warning.__repr__" => "Return repr(self).", + "_sqlite3.Warning.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Warning.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Warning.__str__" => "Return str(self).", + "_sqlite3.Warning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Warning.__weakref__" => "list of weak references to the object", + "_sqlite3.Warning.add_note" => "Add a note to the exception", + "_sqlite3.Warning.with_traceback" => "Set self.__traceback__ to tb and return self.", "_sqlite3.adapt" => "Adapt given object to given protocol.", "_sqlite3.complete_statement" => "Checks if a string contains a complete SQL statement.", "_sqlite3.connect" => "Open a connection to the SQLite database file 'database'.\n\nYou can use \":memory:\" to open a database connection to a database that\nresides in RAM instead of on disk.\n\nNote: Passing more than 1 positional argument to _sqlite3.connect() is\ndeprecated. Parameters 'timeout', 'detect_types', 'isolation_level',\n'check_same_thread', 'factory', 'cached_statements' and 'uri' will\nbecome keyword-only parameters in Python 3.15.", "_sqlite3.enable_callback_tracebacks" => "Enable or disable callback functions throwing errors to stderr.", "_sqlite3.register_adapter" => "Register a function to adapt Python objects to SQLite values.", "_sqlite3.register_converter" => "Register a function to convert SQLite values to Python objects.", - "_sre.template" => "template\n A list containing interleaved literal strings (str or bytes) and group\n indices (int), as returned by re._parser.parse_template():\n [literal1, group1, ..., literalN, groupN]", + "_sre.template" => "template\n A list containing interleaved literal strings (str or bytes) and group\n indices (int), as returned by re._parser.parse_template():\n [literal1, group1, ..., literalN, groupN]", "_ssl" => "Implementation module for SSL socket operations. See the socket module\nfor documentation.", "_ssl.Certificate.__delattr__" => "Implement delattr(self, name).", "_ssl.Certificate.__eq__" => "Return self==value.", @@ -2933,6 +8072,92 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_ssl.RAND_add" => "Mix string into the OpenSSL PRNG state.\n\nentropy (a float) is a lower bound on the entropy contained in\nstring. See RFC 4086.", "_ssl.RAND_bytes" => "Generate n cryptographically strong pseudo-random bytes.", "_ssl.RAND_status" => "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n\nIt is necessary to seed the PRNG with RAND_add() on some platforms before\nusing the ssl() function.", + "_ssl.SSLCertVerificationError" => "A certificate could not be verified.", + "_ssl.SSLCertVerificationError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLCertVerificationError.__eq__" => "Return self==value.", + "_ssl.SSLCertVerificationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLCertVerificationError.__ge__" => "Return self>=value.", + "_ssl.SSLCertVerificationError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLCertVerificationError.__getstate__" => "Helper for pickle.", + "_ssl.SSLCertVerificationError.__gt__" => "Return self>value.", + "_ssl.SSLCertVerificationError.__hash__" => "Return hash(self).", + "_ssl.SSLCertVerificationError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLCertVerificationError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLCertVerificationError.__le__" => "Return self<=value.", + "_ssl.SSLCertVerificationError.__lt__" => "Return self<value.", + "_ssl.SSLCertVerificationError.__ne__" => "Return self!=value.", + "_ssl.SSLCertVerificationError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLCertVerificationError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLCertVerificationError.__repr__" => "Return repr(self).", + "_ssl.SSLCertVerificationError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLCertVerificationError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLCertVerificationError.__str__" => "Return str(self).", + "_ssl.SSLCertVerificationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLCertVerificationError.__weakref__" => "list of weak references to the object", + "_ssl.SSLCertVerificationError.add_note" => "Add a note to the exception", + "_ssl.SSLCertVerificationError.errno" => "POSIX exception code", + "_ssl.SSLCertVerificationError.filename" => "exception filename", + "_ssl.SSLCertVerificationError.filename2" => "second exception filename", + "_ssl.SSLCertVerificationError.strerror" => "exception strerror", + "_ssl.SSLCertVerificationError.winerror" => "Win32 exception code", + "_ssl.SSLCertVerificationError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLEOFError" => "SSL/TLS connection terminated abruptly.", + "_ssl.SSLEOFError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLEOFError.__eq__" => "Return self==value.", + "_ssl.SSLEOFError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLEOFError.__ge__" => "Return self>=value.", + "_ssl.SSLEOFError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLEOFError.__getstate__" => "Helper for pickle.", + "_ssl.SSLEOFError.__gt__" => "Return self>value.", + "_ssl.SSLEOFError.__hash__" => "Return hash(self).", + "_ssl.SSLEOFError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLEOFError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLEOFError.__le__" => "Return self<=value.", + "_ssl.SSLEOFError.__lt__" => "Return self<value.", + "_ssl.SSLEOFError.__ne__" => "Return self!=value.", + "_ssl.SSLEOFError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLEOFError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLEOFError.__repr__" => "Return repr(self).", + "_ssl.SSLEOFError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLEOFError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLEOFError.__str__" => "Return str(self).", + "_ssl.SSLEOFError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLEOFError.__weakref__" => "list of weak references to the object", + "_ssl.SSLEOFError.add_note" => "Add a note to the exception", + "_ssl.SSLEOFError.errno" => "POSIX exception code", + "_ssl.SSLEOFError.filename" => "exception filename", + "_ssl.SSLEOFError.filename2" => "second exception filename", + "_ssl.SSLEOFError.strerror" => "exception strerror", + "_ssl.SSLEOFError.winerror" => "Win32 exception code", + "_ssl.SSLEOFError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLError" => "An error occurred in the SSL implementation.", + "_ssl.SSLError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLError.__eq__" => "Return self==value.", + "_ssl.SSLError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLError.__ge__" => "Return self>=value.", + "_ssl.SSLError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLError.__getstate__" => "Helper for pickle.", + "_ssl.SSLError.__gt__" => "Return self>value.", + "_ssl.SSLError.__hash__" => "Return hash(self).", + "_ssl.SSLError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLError.__le__" => "Return self<=value.", + "_ssl.SSLError.__lt__" => "Return self<value.", + "_ssl.SSLError.__ne__" => "Return self!=value.", + "_ssl.SSLError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLError.__repr__" => "Return repr(self).", + "_ssl.SSLError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLError.__str__" => "Return str(self).", + "_ssl.SSLError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLError.add_note" => "Add a note to the exception", + "_ssl.SSLError.errno" => "POSIX exception code", + "_ssl.SSLError.filename" => "exception filename", + "_ssl.SSLError.filename2" => "second exception filename", + "_ssl.SSLError.strerror" => "exception strerror", + "_ssl.SSLError.winerror" => "Win32 exception code", + "_ssl.SSLError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_ssl.SSLSession.__delattr__" => "Implement delattr(self, name).", "_ssl.SSLSession.__eq__" => "Return self==value.", "_ssl.SSLSession.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -2958,6 +8183,122 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_ssl.SSLSession.ticket_lifetime_hint" => "Ticket life time hint.", "_ssl.SSLSession.time" => "Session creation time (seconds since epoch).", "_ssl.SSLSession.timeout" => "Session timeout (delta in seconds).", + "_ssl.SSLSyscallError" => "System error when attempting SSL operation.", + "_ssl.SSLSyscallError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLSyscallError.__eq__" => "Return self==value.", + "_ssl.SSLSyscallError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLSyscallError.__ge__" => "Return self>=value.", + "_ssl.SSLSyscallError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLSyscallError.__getstate__" => "Helper for pickle.", + "_ssl.SSLSyscallError.__gt__" => "Return self>value.", + "_ssl.SSLSyscallError.__hash__" => "Return hash(self).", + "_ssl.SSLSyscallError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLSyscallError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLSyscallError.__le__" => "Return self<=value.", + "_ssl.SSLSyscallError.__lt__" => "Return self<value.", + "_ssl.SSLSyscallError.__ne__" => "Return self!=value.", + "_ssl.SSLSyscallError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLSyscallError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLSyscallError.__repr__" => "Return repr(self).", + "_ssl.SSLSyscallError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLSyscallError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLSyscallError.__str__" => "Return str(self).", + "_ssl.SSLSyscallError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLSyscallError.__weakref__" => "list of weak references to the object", + "_ssl.SSLSyscallError.add_note" => "Add a note to the exception", + "_ssl.SSLSyscallError.errno" => "POSIX exception code", + "_ssl.SSLSyscallError.filename" => "exception filename", + "_ssl.SSLSyscallError.filename2" => "second exception filename", + "_ssl.SSLSyscallError.strerror" => "exception strerror", + "_ssl.SSLSyscallError.winerror" => "Win32 exception code", + "_ssl.SSLSyscallError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLWantReadError" => "Non-blocking SSL socket needs to read more data\nbefore the requested operation can be completed.", + "_ssl.SSLWantReadError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLWantReadError.__eq__" => "Return self==value.", + "_ssl.SSLWantReadError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLWantReadError.__ge__" => "Return self>=value.", + "_ssl.SSLWantReadError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLWantReadError.__getstate__" => "Helper for pickle.", + "_ssl.SSLWantReadError.__gt__" => "Return self>value.", + "_ssl.SSLWantReadError.__hash__" => "Return hash(self).", + "_ssl.SSLWantReadError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLWantReadError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLWantReadError.__le__" => "Return self<=value.", + "_ssl.SSLWantReadError.__lt__" => "Return self<value.", + "_ssl.SSLWantReadError.__ne__" => "Return self!=value.", + "_ssl.SSLWantReadError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLWantReadError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLWantReadError.__repr__" => "Return repr(self).", + "_ssl.SSLWantReadError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLWantReadError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLWantReadError.__str__" => "Return str(self).", + "_ssl.SSLWantReadError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLWantReadError.__weakref__" => "list of weak references to the object", + "_ssl.SSLWantReadError.add_note" => "Add a note to the exception", + "_ssl.SSLWantReadError.errno" => "POSIX exception code", + "_ssl.SSLWantReadError.filename" => "exception filename", + "_ssl.SSLWantReadError.filename2" => "second exception filename", + "_ssl.SSLWantReadError.strerror" => "exception strerror", + "_ssl.SSLWantReadError.winerror" => "Win32 exception code", + "_ssl.SSLWantReadError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLWantWriteError" => "Non-blocking SSL socket needs to write more data\nbefore the requested operation can be completed.", + "_ssl.SSLWantWriteError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLWantWriteError.__eq__" => "Return self==value.", + "_ssl.SSLWantWriteError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLWantWriteError.__ge__" => "Return self>=value.", + "_ssl.SSLWantWriteError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLWantWriteError.__getstate__" => "Helper for pickle.", + "_ssl.SSLWantWriteError.__gt__" => "Return self>value.", + "_ssl.SSLWantWriteError.__hash__" => "Return hash(self).", + "_ssl.SSLWantWriteError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLWantWriteError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLWantWriteError.__le__" => "Return self<=value.", + "_ssl.SSLWantWriteError.__lt__" => "Return self<value.", + "_ssl.SSLWantWriteError.__ne__" => "Return self!=value.", + "_ssl.SSLWantWriteError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLWantWriteError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLWantWriteError.__repr__" => "Return repr(self).", + "_ssl.SSLWantWriteError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLWantWriteError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLWantWriteError.__str__" => "Return str(self).", + "_ssl.SSLWantWriteError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLWantWriteError.__weakref__" => "list of weak references to the object", + "_ssl.SSLWantWriteError.add_note" => "Add a note to the exception", + "_ssl.SSLWantWriteError.errno" => "POSIX exception code", + "_ssl.SSLWantWriteError.filename" => "exception filename", + "_ssl.SSLWantWriteError.filename2" => "second exception filename", + "_ssl.SSLWantWriteError.strerror" => "exception strerror", + "_ssl.SSLWantWriteError.winerror" => "Win32 exception code", + "_ssl.SSLWantWriteError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLZeroReturnError" => "SSL/TLS session closed cleanly.", + "_ssl.SSLZeroReturnError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLZeroReturnError.__eq__" => "Return self==value.", + "_ssl.SSLZeroReturnError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLZeroReturnError.__ge__" => "Return self>=value.", + "_ssl.SSLZeroReturnError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLZeroReturnError.__getstate__" => "Helper for pickle.", + "_ssl.SSLZeroReturnError.__gt__" => "Return self>value.", + "_ssl.SSLZeroReturnError.__hash__" => "Return hash(self).", + "_ssl.SSLZeroReturnError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLZeroReturnError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLZeroReturnError.__le__" => "Return self<=value.", + "_ssl.SSLZeroReturnError.__lt__" => "Return self<value.", + "_ssl.SSLZeroReturnError.__ne__" => "Return self!=value.", + "_ssl.SSLZeroReturnError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLZeroReturnError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLZeroReturnError.__repr__" => "Return repr(self).", + "_ssl.SSLZeroReturnError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLZeroReturnError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLZeroReturnError.__str__" => "Return str(self).", + "_ssl.SSLZeroReturnError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLZeroReturnError.__weakref__" => "list of weak references to the object", + "_ssl.SSLZeroReturnError.add_note" => "Add a note to the exception", + "_ssl.SSLZeroReturnError.errno" => "POSIX exception code", + "_ssl.SSLZeroReturnError.filename" => "exception filename", + "_ssl.SSLZeroReturnError.filename2" => "second exception filename", + "_ssl.SSLZeroReturnError.strerror" => "exception strerror", + "_ssl.SSLZeroReturnError.winerror" => "Win32 exception code", + "_ssl.SSLZeroReturnError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_ssl._SSLContext.__delattr__" => "Implement delattr(self, name).", "_ssl._SSLContext.__eq__" => "Return self==value.", "_ssl._SSLContext.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3023,7 +8364,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_ssl.get_default_verify_paths" => "Return search paths and environment vars that are used by SSLContext's set_default_verify_paths() to load default CAs.\n\nThe values are 'cert_file_env', 'cert_file', 'cert_dir_env', 'cert_dir'.", "_ssl.nid2obj" => "Lookup NID, short name, long name and OID of an ASN1_OBJECT by NID.", "_ssl.txt2obj" => "Lookup NID, short name, long name and OID of an ASN1_OBJECT.\n\nBy default objects are looked up by OID. With name=True short and\nlong name are also matched.", - "_stat" => "S_IFMT_: file type bits\nS_IFDIR: directory\nS_IFCHR: character device\nS_IFBLK: block device\nS_IFREG: regular file\nS_IFIFO: fifo (named pipe)\nS_IFLNK: symbolic link\nS_IFSOCK: socket file\nS_IFDOOR: door\nS_IFPORT: event port\nS_IFWHT: whiteout\n\nS_ISUID: set UID bit\nS_ISGID: set GID bit\nS_ENFMT: file locking enforcement\nS_ISVTX: sticky bit\nS_IREAD: Unix V7 synonym for S_IRUSR\nS_IWRITE: Unix V7 synonym for S_IWUSR\nS_IEXEC: Unix V7 synonym for S_IXUSR\nS_IRWXU: mask for owner permissions\nS_IRUSR: read by owner\nS_IWUSR: write by owner\nS_IXUSR: execute by owner\nS_IRWXG: mask for group permissions\nS_IRGRP: read by group\nS_IWGRP: write by group\nS_IXGRP: execute by group\nS_IRWXO: mask for others (not in group) permissions\nS_IROTH: read by others\nS_IWOTH: write by others\nS_IXOTH: execute by others\n\nUF_SETTABLE: mask of owner changable flags\nUF_NODUMP: do not dump file\nUF_IMMUTABLE: file may not be changed\nUF_APPEND: file may only be appended to\nUF_OPAQUE: directory is opaque when viewed through a union stack\nUF_NOUNLINK: file may not be renamed or deleted\nUF_COMPRESSED: macOS: file is hfs-compressed\nUF_TRACKED: used for dealing with document IDs\nUF_DATAVAULT: entitlement required for reading and writing\nUF_HIDDEN: macOS: file should not be displayed\nSF_SETTABLE: mask of super user changeable flags\nSF_ARCHIVED: file may be archived\nSF_IMMUTABLE: file may not be changed\nSF_APPEND: file may only be appended to\nSF_RESTRICTED: entitlement required for writing\nSF_NOUNLINK: file may not be renamed or deleted\nSF_SNAPSHOT: file is a snapshot file\nSF_FIRMLINK: file is a firmlink\nSF_DATALESS: file is a dataless object\n\nOn macOS:\nSF_SUPPORTED: mask of super user supported flags\nSF_SYNTHETIC: mask of read-only synthetic flags\n\nST_MODE\nST_INO\nST_DEV\nST_NLINK\nST_UID\nST_GID\nST_SIZE\nST_ATIME\nST_MTIME\nST_CTIME\n\nFILE_ATTRIBUTE_*: Windows file attribute constants\n (only present on Windows)", + "_stat" => "S_IFMT_: file type bits\nS_IFDIR: directory\nS_IFCHR: character device\nS_IFBLK: block device\nS_IFREG: regular file\nS_IFIFO: fifo (named pipe)\nS_IFLNK: symbolic link\nS_IFSOCK: socket file\nS_IFDOOR: door\nS_IFPORT: event port\nS_IFWHT: whiteout\n\nS_ISUID: set UID bit\nS_ISGID: set GID bit\nS_ENFMT: file locking enforcement\nS_ISVTX: sticky bit\nS_IREAD: Unix V7 synonym for S_IRUSR\nS_IWRITE: Unix V7 synonym for S_IWUSR\nS_IEXEC: Unix V7 synonym for S_IXUSR\nS_IRWXU: mask for owner permissions\nS_IRUSR: read by owner\nS_IWUSR: write by owner\nS_IXUSR: execute by owner\nS_IRWXG: mask for group permissions\nS_IRGRP: read by group\nS_IWGRP: write by group\nS_IXGRP: execute by group\nS_IRWXO: mask for others (not in group) permissions\nS_IROTH: read by others\nS_IWOTH: write by others\nS_IXOTH: execute by others\n\nUF_SETTABLE: mask of owner changeable flags\nUF_NODUMP: do not dump file\nUF_IMMUTABLE: file may not be changed\nUF_APPEND: file may only be appended to\nUF_OPAQUE: directory is opaque when viewed through a union stack\nUF_NOUNLINK: file may not be renamed or deleted\nUF_COMPRESSED: macOS: file is hfs-compressed\nUF_TRACKED: used for dealing with document IDs\nUF_DATAVAULT: entitlement required for reading and writing\nUF_HIDDEN: macOS: file should not be displayed\nSF_SETTABLE: mask of super user changeable flags\nSF_ARCHIVED: file may be archived\nSF_IMMUTABLE: file may not be changed\nSF_APPEND: file may only be appended to\nSF_RESTRICTED: entitlement required for writing\nSF_NOUNLINK: file may not be renamed or deleted\nSF_SNAPSHOT: file is a snapshot file\nSF_FIRMLINK: file is a firmlink\nSF_DATALESS: file is a dataless object\n\nOn macOS:\nSF_SUPPORTED: mask of super user supported flags\nSF_SYNTHETIC: mask of read-only synthetic flags\n\nST_MODE\nST_INO\nST_DEV\nST_NLINK\nST_UID\nST_GID\nST_SIZE\nST_ATIME\nST_MTIME\nST_CTIME\n\nFILE_ATTRIBUTE_*: Windows file attribute constants\n (only present on Windows)", "_stat.S_IFMT" => "Return the portion of the file's mode that describes the file type.", "_stat.S_IMODE" => "Return the portion of the file's mode that can be set by os.chmod().", "_stat.S_ISBLK" => "S_ISBLK(mode) -> bool\n\nReturn True if mode is from a block special device file.", @@ -3041,7 +8382,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_string" => "string helper module", "_string.formatter_field_name_split" => "split the argument as a field name", "_string.formatter_parser" => "parse the argument as a format string", - "_struct" => "Functions to convert between Python values and C structs.\nPython bytes objects are used to hold the data representing the C struct\nand also as format strings (explained below) to describe the layout of data\nin the C struct.\n\nThe optional first format char indicates byte order, size and alignment:\n @: native order, size & alignment (default)\n =: native order, std. size & alignment\n <: little-endian, std. size & alignment\n >: big-endian, std. size & alignment\n !: same as >\n\nThe remaining chars indicate types of args and must match exactly;\nthese can be preceded by a decimal repeat count:\n x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n ?: _Bool (requires C99; if not available, char is used instead)\n h:short; H:unsigned short; i:int; I:unsigned int;\n l:long; L:unsigned long; f:float; d:double; e:half-float.\nSpecial cases (preceding decimal count indicates length):\n s:string (array of char); p: pascal string (with count byte).\nSpecial cases (only available in native format):\n n:ssize_t; N:size_t;\n P:an integer type that is wide enough to hold a pointer.\nSpecial case (not in native mode unless 'long long' in platform C):\n q:long long; Q:unsigned long long\nWhitespace between formats is ignored.\n\nThe variable struct.error is an exception raised on errors.", + "_struct" => "Functions to convert between Python values and C structs.\nPython bytes objects are used to hold the data representing the C struct\nand also as format strings (explained below) to describe the layout of data\nin the C struct.\n\nThe optional first format char indicates byte order, size and alignment:\n @: native order, size & alignment (default)\n =: native order, std. size & alignment\n <: little-endian, std. size & alignment\n >: big-endian, std. size & alignment\n !: same as >\n\nThe remaining chars indicate types of args and must match exactly;\nthese can be preceded by a decimal repeat count:\n x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n ?:_Bool; h:short; H:unsigned short; i:int; I:unsigned int;\n l:long; L:unsigned long; f:float; d:double; e:half-float.\n F:float complex; D:double complex.\nSpecial cases (preceding decimal count indicates length):\n s:string (array of char); p: pascal string (with count byte).\nSpecial cases (only available in native format):\n n:ssize_t; N:size_t;\n P:an integer type that is wide enough to hold a pointer.\nSpecial case (not in native mode unless 'long long' in platform C):\n q:long long; Q:unsigned long long\nWhitespace between formats is ignored.\n\nThe variable struct.error is an exception raised on errors.", "_struct.Struct" => "Struct(fmt) --> compiled struct object", "_struct.Struct.__delattr__" => "Implement delattr(self, name).", "_struct.Struct.__eq__" => "Return self==value.", @@ -3073,6 +8414,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_struct.Struct.unpack_from" => "Return a tuple containing unpacked values.\n\nValues are unpacked according to the format string Struct.format.\n\nThe buffer's size in bytes, starting at position offset, must be\nat least Struct.size.\n\nSee help(struct) for more on format strings.", "_struct._clearcache" => "Clear the internal cache.", "_struct.calcsize" => "Return size in bytes of the struct described by the format string.", + "_struct.error.__delattr__" => "Implement delattr(self, name).", + "_struct.error.__eq__" => "Return self==value.", + "_struct.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_struct.error.__ge__" => "Return self>=value.", + "_struct.error.__getattribute__" => "Return getattr(self, name).", + "_struct.error.__getstate__" => "Helper for pickle.", + "_struct.error.__gt__" => "Return self>value.", + "_struct.error.__hash__" => "Return hash(self).", + "_struct.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_struct.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_struct.error.__le__" => "Return self<=value.", + "_struct.error.__lt__" => "Return self<value.", + "_struct.error.__ne__" => "Return self!=value.", + "_struct.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_struct.error.__reduce_ex__" => "Helper for pickle.", + "_struct.error.__repr__" => "Return repr(self).", + "_struct.error.__setattr__" => "Implement setattr(self, name, value).", + "_struct.error.__sizeof__" => "Size of object in memory, in bytes.", + "_struct.error.__str__" => "Return str(self).", + "_struct.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_struct.error.__weakref__" => "list of weak references to the object", + "_struct.error.add_note" => "Add a note to the exception", + "_struct.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "_struct.iter_unpack" => "Return an iterator yielding tuples unpacked from the given bytes.\n\nThe bytes are unpacked according to the format string, like\na repeated invocation of unpack_from().\n\nRequires that the bytes length be a multiple of the format struct size.", "_struct.pack" => "pack(format, v1, v2, ...) -> bytes\n\nReturn a bytes object containing the values v1, v2, ... packed according\nto the format string. See help(struct) for more on format strings.", "_struct.pack_into" => "pack_into(format, buffer, offset, v1, v2, ...)\n\nPack the values v1, v2, ... according to the format string and write\nthe packed bytes into the writable buffer buf starting at offset. Note\nthat the offset is a required argument. See help(struct) for more\non format strings.", @@ -3141,6 +8505,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_thread.RLock._recursion_count" => "For internal use by reentrancy checks.", "_thread.RLock._release_save" => "For internal use by `threading.Condition`.", "_thread.RLock.acquire" => "Lock the lock. `blocking` indicates whether we should wait\nfor the lock to be available or not. If `blocking` is False\nand another thread holds the lock, the method will return False\nimmediately. If `blocking` is True and another thread holds\nthe lock, the method will wait for the lock to be released,\ntake it and then return True.\n(note: the blocking operation is interruptible.)\n\nIn all other cases, the method will return True immediately.\nPrecisely, if the current thread already holds the lock, its\ninternal counter is simply incremented. If nobody holds the lock,\nthe lock is taken and its internal counter initialized to 1.", + "_thread.RLock.locked" => "locked()\n\nReturn a boolean indicating whether this object is locked right now.", "_thread.RLock.release" => "Release the lock, allowing another thread that is blocked waiting for\nthe lock to acquire the lock. The lock must be in the locked state,\nand must be locked by the same thread that unlocks it; otherwise a\n`RuntimeError` is raised.\n\nDo note that if the lock was acquire()d several times in a row by the\ncurrent thread, release() needs to be called as many times for the lock\nto be available for other threads.", "_thread._ExceptHookArgs" => "ExceptHookArgs\n\nType used to pass arguments to threading.excepthook.", "_thread._ExceptHookArgs.__add__" => "Return self+value.", @@ -3202,6 +8567,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_thread._count" => "Return the number of currently running Python threads, excluding\nthe main thread. The returned number comprises all threads created\nthrough `start_new_thread()` as well as `threading.Thread`, and not\nyet finished.\n\nThis function is meant for internal and specialized purposes only.\nIn most applications `threading.enumerate()` should be used instead.", "_thread._excepthook" => "Handle uncaught Thread.run() exception.", "_thread._get_main_thread_ident" => "Internal only. Return a non-zero integer that uniquely identifies the main thread\nof the main interpreter.", + "_thread._get_name" => "Get the name of the current thread.", "_thread._is_main_interpreter" => "Return True if the current interpreter is the main Python interpreter.", "_thread._local" => "Thread-local data", "_thread._local.__delattr__" => "Implement delattr(self, name).", @@ -3265,12 +8631,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_thread.lock.locked_lock" => "An obsolete synonym of locked().", "_thread.lock.release" => "Release the lock, allowing another thread that is blocked waiting for\nthe lock to acquire the lock. The lock must be in the locked state,\nbut it needn't be locked by the same thread that unlocks it.", "_thread.lock.release_lock" => "An obsolete synonym of release().", + "_thread.set_name" => "Set the name of the current thread.", "_thread.stack_size" => "Return the thread stack size used when creating new threads. The\noptional size argument specifies the stack size (in bytes) to be used\nfor subsequently created threads, and must be 0 (use platform or\nconfigured default) or a positive integer value of at least 32,768 (32k).\nIf changing the thread stack size is unsupported, a ThreadError\nexception is raised. If the specified size is invalid, a ValueError\nexception is raised, and the stack size is unmodified. 32k bytes\n currently the minimum supported stack size value to guarantee\nsufficient stack space for the interpreter itself.\n\nNote that some platforms may have particular restrictions on values for\nthe stack size, such as requiring a minimum stack size larger than 32 KiB or\nrequiring allocation in multiples of the system memory page size\n- platform documentation should be referred to for more information\n(4 KiB pages are common; using multiples of 4096 for the stack size is\nthe suggested approach in the absence of more specific information).", "_thread.start_joinable_thread" => "*For internal use only*: start a new thread.\n\nLike start_new_thread(), this starts a new thread calling the given function.\nUnlike start_new_thread(), this returns a handle object with methods to join\nor detach the given thread.\nThis function is not for third-party code, please use the\n`threading` module instead. During finalization the runtime will not wait for\nthe thread to exit if daemon is True. If handle is provided it must be a\nnewly created thread._ThreadHandle instance.", "_thread.start_new" => "An obsolete synonym of start_new_thread().", "_thread.start_new_thread" => "Start a new thread and return its identifier.\n\nThe thread will call the function with positional arguments from the\ntuple args and keyword arguments taken from the optional dictionary\nkwargs. The thread exits when the function returns; the return value\nis ignored. The thread will also exit when the function raises an\nunhandled exception; a stack trace will be printed unless the exception\nis SystemExit.", - "_tkinter.TclError.__cause__" => "exception cause", - "_tkinter.TclError.__context__" => "exception context", "_tkinter.TclError.__delattr__" => "Implement delattr(self, name).", "_tkinter.TclError.__eq__" => "Return self==value.", "_tkinter.TclError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3292,8 +8657,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_tkinter.TclError.__str__" => "Return str(self).", "_tkinter.TclError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_tkinter.TclError.__weakref__" => "list of weak references to the object", - "_tkinter.TclError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "_tkinter.TclError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "_tkinter.TclError.add_note" => "Add a note to the exception", + "_tkinter.TclError.with_traceback" => "Set self.__traceback__ to tb and return self.", "_tkinter.Tcl_Obj.__delattr__" => "Implement delattr(self, name).", "_tkinter.Tcl_Obj.__eq__" => "Return self==value.", "_tkinter.Tcl_Obj.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3360,7 +8725,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_tkinter.TkttType.__sizeof__" => "Size of object in memory, in bytes.", "_tkinter.TkttType.__str__" => "Return str(self).", "_tkinter.TkttType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "_tkinter.create" => "wantTk\n if false, then Tk_Init() doesn't get called\nsync\n if true, then pass -sync to wish\nuse\n if not None, then pass -use to wish", + "_tkinter.create" => "wantTk\n if false, then Tk_Init() doesn't get called\n sync\n if true, then pass -sync to wish\n use\n if not None, then pass -use to wish", "_tkinter.getbusywaitinterval" => "Return the current busy-wait interval between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.", "_tkinter.setbusywaitinterval" => "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n\nIt should be set to a divisor of the maximum time between frames in an animation.", "_tokenize.TokenizerIter.__delattr__" => "Implement delattr(self, name).", @@ -3397,15 +8762,499 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_tracemalloc.reset_peak" => "Set the peak size of memory blocks traced by tracemalloc to the current size.\n\nDo nothing if the tracemalloc module is not tracing memory allocations.", "_tracemalloc.start" => "Start tracing Python memory allocations.\n\nAlso set the maximum number of frames stored in the traceback of a\ntrace to nframe.", "_tracemalloc.stop" => "Stop tracing Python memory allocations.\n\nAlso clear traces of memory blocks allocated by Python.", + "_types" => "Define names for built-in types.", + "_types.GenericAlias" => "Represent a PEP 585 generic type\n\nE.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).", + "_types.GenericAlias.__call__" => "Call self as a function.", + "_types.GenericAlias.__delattr__" => "Implement delattr(self, name).", + "_types.GenericAlias.__eq__" => "Return self==value.", + "_types.GenericAlias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.GenericAlias.__ge__" => "Return self>=value.", + "_types.GenericAlias.__getattribute__" => "Return getattr(self, name).", + "_types.GenericAlias.__getitem__" => "Return self[key].", + "_types.GenericAlias.__getstate__" => "Helper for pickle.", + "_types.GenericAlias.__gt__" => "Return self>value.", + "_types.GenericAlias.__hash__" => "Return hash(self).", + "_types.GenericAlias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.GenericAlias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.GenericAlias.__iter__" => "Implement iter(self).", + "_types.GenericAlias.__le__" => "Return self<=value.", + "_types.GenericAlias.__lt__" => "Return self<value.", + "_types.GenericAlias.__ne__" => "Return self!=value.", + "_types.GenericAlias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.GenericAlias.__or__" => "Return self|value.", + "_types.GenericAlias.__parameters__" => "Type variables in the GenericAlias.", + "_types.GenericAlias.__reduce_ex__" => "Helper for pickle.", + "_types.GenericAlias.__repr__" => "Return repr(self).", + "_types.GenericAlias.__ror__" => "Return value|self.", + "_types.GenericAlias.__setattr__" => "Implement setattr(self, name, value).", + "_types.GenericAlias.__sizeof__" => "Size of object in memory, in bytes.", + "_types.GenericAlias.__str__" => "Return str(self).", + "_types.GenericAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.NoneType" => "The type of the None singleton.", + "_types.NoneType.__bool__" => "True if self else False", + "_types.NoneType.__delattr__" => "Implement delattr(self, name).", + "_types.NoneType.__eq__" => "Return self==value.", + "_types.NoneType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.NoneType.__ge__" => "Return self>=value.", + "_types.NoneType.__getattribute__" => "Return getattr(self, name).", + "_types.NoneType.__getstate__" => "Helper for pickle.", + "_types.NoneType.__gt__" => "Return self>value.", + "_types.NoneType.__hash__" => "Return hash(self).", + "_types.NoneType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.NoneType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.NoneType.__le__" => "Return self<=value.", + "_types.NoneType.__lt__" => "Return self<value.", + "_types.NoneType.__ne__" => "Return self!=value.", + "_types.NoneType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.NoneType.__reduce__" => "Helper for pickle.", + "_types.NoneType.__reduce_ex__" => "Helper for pickle.", + "_types.NoneType.__repr__" => "Return repr(self).", + "_types.NoneType.__setattr__" => "Implement setattr(self, name, value).", + "_types.NoneType.__sizeof__" => "Size of object in memory, in bytes.", + "_types.NoneType.__str__" => "Return str(self).", + "_types.NoneType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.NotImplementedType" => "The type of the NotImplemented singleton.", + "_types.NotImplementedType.__bool__" => "True if self else False", + "_types.NotImplementedType.__delattr__" => "Implement delattr(self, name).", + "_types.NotImplementedType.__eq__" => "Return self==value.", + "_types.NotImplementedType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.NotImplementedType.__ge__" => "Return self>=value.", + "_types.NotImplementedType.__getattribute__" => "Return getattr(self, name).", + "_types.NotImplementedType.__getstate__" => "Helper for pickle.", + "_types.NotImplementedType.__gt__" => "Return self>value.", + "_types.NotImplementedType.__hash__" => "Return hash(self).", + "_types.NotImplementedType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.NotImplementedType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.NotImplementedType.__le__" => "Return self<=value.", + "_types.NotImplementedType.__lt__" => "Return self<value.", + "_types.NotImplementedType.__ne__" => "Return self!=value.", + "_types.NotImplementedType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.NotImplementedType.__reduce_ex__" => "Helper for pickle.", + "_types.NotImplementedType.__repr__" => "Return repr(self).", + "_types.NotImplementedType.__setattr__" => "Implement setattr(self, name, value).", + "_types.NotImplementedType.__sizeof__" => "Size of object in memory, in bytes.", + "_types.NotImplementedType.__str__" => "Return str(self).", + "_types.NotImplementedType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.SimpleNamespace" => "A simple attribute-based namespace.", + "_types.SimpleNamespace.__delattr__" => "Implement delattr(self, name).", + "_types.SimpleNamespace.__eq__" => "Return self==value.", + "_types.SimpleNamespace.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.SimpleNamespace.__ge__" => "Return self>=value.", + "_types.SimpleNamespace.__getattribute__" => "Return getattr(self, name).", + "_types.SimpleNamespace.__getstate__" => "Helper for pickle.", + "_types.SimpleNamespace.__gt__" => "Return self>value.", + "_types.SimpleNamespace.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.SimpleNamespace.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.SimpleNamespace.__le__" => "Return self<=value.", + "_types.SimpleNamespace.__lt__" => "Return self<value.", + "_types.SimpleNamespace.__ne__" => "Return self!=value.", + "_types.SimpleNamespace.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.SimpleNamespace.__reduce__" => "Return state information for pickling", + "_types.SimpleNamespace.__reduce_ex__" => "Helper for pickle.", + "_types.SimpleNamespace.__replace__" => "Return a copy of the namespace object with new values for the specified attributes.", + "_types.SimpleNamespace.__repr__" => "Return repr(self).", + "_types.SimpleNamespace.__setattr__" => "Implement setattr(self, name, value).", + "_types.SimpleNamespace.__sizeof__" => "Size of object in memory, in bytes.", + "_types.SimpleNamespace.__str__" => "Return str(self).", + "_types.SimpleNamespace.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_typing" => "Primitives and accelerators for the typing module.", + "_typing.Generic" => "Abstract base class for generic types.\n\nOn Python 3.12 and newer, generic classes implicitly inherit from\nGeneric when they declare a parameter list after the class's name::\n\n class Mapping[KT, VT]:\n def __getitem__(self, key: KT) -> VT:\n ...\n # Etc.\n\nOn older versions of Python, however, generic classes have to\nexplicitly inherit from Generic.\n\nAfter a class has been declared to be generic, it can then be used as\nfollows::\n\n def lookup_name[KT, VT](mapping: Mapping[KT, VT], key: KT, default: VT) -> VT:\n try:\n return mapping[key]\n except KeyError:\n return default", + "_typing.Generic.__class_getitem__" => "Parameterizes a generic class.\n\nAt least, parameterizing a generic class is the *main* thing this\nmethod does. For example, for some generic class `Foo`, this is called\nwhen we do `Foo[int]` - there, with `cls=Foo` and `params=int`.\n\nHowever, note that this method is also called when defining generic\nclasses in the first place with `class Foo[T]: ...`.", + "_typing.Generic.__delattr__" => "Implement delattr(self, name).", + "_typing.Generic.__eq__" => "Return self==value.", + "_typing.Generic.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.Generic.__ge__" => "Return self>=value.", + "_typing.Generic.__getattribute__" => "Return getattr(self, name).", + "_typing.Generic.__getstate__" => "Helper for pickle.", + "_typing.Generic.__gt__" => "Return self>value.", + "_typing.Generic.__hash__" => "Return hash(self).", + "_typing.Generic.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.Generic.__init_subclass__" => "Function to initialize subclasses.", + "_typing.Generic.__le__" => "Return self<=value.", + "_typing.Generic.__lt__" => "Return self<value.", + "_typing.Generic.__ne__" => "Return self!=value.", + "_typing.Generic.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.Generic.__reduce__" => "Helper for pickle.", + "_typing.Generic.__reduce_ex__" => "Helper for pickle.", + "_typing.Generic.__repr__" => "Return repr(self).", + "_typing.Generic.__setattr__" => "Implement setattr(self, name, value).", + "_typing.Generic.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.Generic.__str__" => "Return str(self).", + "_typing.Generic.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpec" => "Parameter specification variable.\n\nThe preferred way to construct a parameter specification is via the\ndedicated syntax for generic functions, classes, and type aliases,\nwhere the use of '**' creates a parameter specification::\n\n type IntFunc[**P] = Callable[P, int]\n\nThe following syntax creates a parameter specification that defaults\nto a callable accepting two positional-only arguments of types int\nand str:\n\n type IntFuncDefault[**P = [int, str]] = Callable[P, int]\n\nFor compatibility with Python 3.11 and earlier, ParamSpec objects\ncan also be created as follows::\n\n P = ParamSpec('P')\n DefaultP = ParamSpec('DefaultP', default=[int, str])\n\nParameter specification variables exist primarily for the benefit of\nstatic type checkers. They are used to forward the parameter types of\none callable to another callable, a pattern commonly found in\nhigher-order functions and decorators. They are only valid when used\nin ``Concatenate``, or as the first argument to ``Callable``, or as\nparameters for user-defined Generics. See class Generic for more\ninformation on generic types.\n\nAn example for annotating a decorator::\n\n def add_logging[**P, T](f: Callable[P, T]) -> Callable[P, T]:\n '''A type-safe decorator to add logging to a function.'''\n def inner(*args: P.args, **kwargs: P.kwargs) -> T:\n logging.info(f'{f.__name__} was called')\n return f(*args, **kwargs)\n return inner\n\n @add_logging\n def add_two(x: float, y: float) -> float:\n '''Add two numbers together.'''\n return x + y\n\nParameter specification variables can be introspected. e.g.::\n\n >>> P = ParamSpec(\"P\")\n >>> P.__name__\n 'P'\n\nNote that only parameter specification variables defined in the global\nscope can be pickled.", + "_typing.ParamSpec.__default__" => "The default value for this ParamSpec.", + "_typing.ParamSpec.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpec.__eq__" => "Return self==value.", + "_typing.ParamSpec.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpec.__ge__" => "Return self>=value.", + "_typing.ParamSpec.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpec.__getstate__" => "Helper for pickle.", + "_typing.ParamSpec.__gt__" => "Return self>value.", + "_typing.ParamSpec.__hash__" => "Return hash(self).", + "_typing.ParamSpec.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpec.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpec.__le__" => "Return self<=value.", + "_typing.ParamSpec.__lt__" => "Return self<value.", + "_typing.ParamSpec.__ne__" => "Return self!=value.", + "_typing.ParamSpec.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpec.__or__" => "Return self|value.", + "_typing.ParamSpec.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpec.__repr__" => "Return repr(self).", + "_typing.ParamSpec.__ror__" => "Return value|self.", + "_typing.ParamSpec.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpec.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpec.__str__" => "Return str(self).", + "_typing.ParamSpec.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpec.args" => "Represents positional arguments.", + "_typing.ParamSpec.kwargs" => "Represents keyword arguments.", + "_typing.ParamSpecArgs" => "The args for a ParamSpec object.\n\nGiven a ParamSpec object P, P.args is an instance of ParamSpecArgs.\n\nParamSpecArgs objects have a reference back to their ParamSpec::\n\n >>> P = ParamSpec(\"P\")\n >>> P.args.__origin__ is P\n True\n\nThis type is meant for runtime introspection and has no special meaning\nto static type checkers.", + "_typing.ParamSpecArgs.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpecArgs.__eq__" => "Return self==value.", + "_typing.ParamSpecArgs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpecArgs.__ge__" => "Return self>=value.", + "_typing.ParamSpecArgs.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpecArgs.__getstate__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__gt__" => "Return self>value.", + "_typing.ParamSpecArgs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpecArgs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpecArgs.__le__" => "Return self<=value.", + "_typing.ParamSpecArgs.__lt__" => "Return self<value.", + "_typing.ParamSpecArgs.__ne__" => "Return self!=value.", + "_typing.ParamSpecArgs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpecArgs.__reduce__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__repr__" => "Return repr(self).", + "_typing.ParamSpecArgs.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpecArgs.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpecArgs.__str__" => "Return str(self).", + "_typing.ParamSpecArgs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpecKwargs" => "The kwargs for a ParamSpec object.\n\nGiven a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs.\n\nParamSpecKwargs objects have a reference back to their ParamSpec::\n\n >>> P = ParamSpec(\"P\")\n >>> P.kwargs.__origin__ is P\n True\n\nThis type is meant for runtime introspection and has no special meaning\nto static type checkers.", + "_typing.ParamSpecKwargs.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpecKwargs.__eq__" => "Return self==value.", + "_typing.ParamSpecKwargs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpecKwargs.__ge__" => "Return self>=value.", + "_typing.ParamSpecKwargs.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpecKwargs.__getstate__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__gt__" => "Return self>value.", + "_typing.ParamSpecKwargs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpecKwargs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpecKwargs.__le__" => "Return self<=value.", + "_typing.ParamSpecKwargs.__lt__" => "Return self<value.", + "_typing.ParamSpecKwargs.__ne__" => "Return self!=value.", + "_typing.ParamSpecKwargs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpecKwargs.__reduce__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__repr__" => "Return repr(self).", + "_typing.ParamSpecKwargs.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpecKwargs.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpecKwargs.__str__" => "Return str(self).", + "_typing.ParamSpecKwargs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeAliasType" => "Type alias.\n\nType aliases are created through the type statement::\n\n type Alias = int\n\nIn this example, Alias and int will be treated equivalently by static\ntype checkers.\n\nAt runtime, Alias is an instance of TypeAliasType. The __name__\nattribute holds the name of the type alias. The value of the type alias\nis stored in the __value__ attribute. It is evaluated lazily, so the\nvalue is computed only if the attribute is accessed.\n\nType aliases can also be generic::\n\n type ListOrSet[T] = list[T] | set[T]\n\nIn this case, the type parameters of the alias are stored in the\n__type_params__ attribute.\n\nSee PEP 695 for more information.", + "_typing.TypeAliasType.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeAliasType.__eq__" => "Return self==value.", + "_typing.TypeAliasType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeAliasType.__ge__" => "Return self>=value.", + "_typing.TypeAliasType.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeAliasType.__getitem__" => "Return self[key].", + "_typing.TypeAliasType.__getstate__" => "Helper for pickle.", + "_typing.TypeAliasType.__gt__" => "Return self>value.", + "_typing.TypeAliasType.__hash__" => "Return hash(self).", + "_typing.TypeAliasType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeAliasType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeAliasType.__iter__" => "Implement iter(self).", + "_typing.TypeAliasType.__le__" => "Return self<=value.", + "_typing.TypeAliasType.__lt__" => "Return self<value.", + "_typing.TypeAliasType.__ne__" => "Return self!=value.", + "_typing.TypeAliasType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeAliasType.__or__" => "Return self|value.", + "_typing.TypeAliasType.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeAliasType.__repr__" => "Return repr(self).", + "_typing.TypeAliasType.__ror__" => "Return value|self.", + "_typing.TypeAliasType.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeAliasType.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeAliasType.__str__" => "Return str(self).", + "_typing.TypeAliasType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeVar" => "Type variable.\n\nThe preferred way to construct a type variable is via the dedicated\nsyntax for generic functions, classes, and type aliases::\n\n class Sequence[T]: # T is a TypeVar\n ...\n\nThis syntax can also be used to create bound and constrained type\nvariables::\n\n # S is a TypeVar bound to str\n class StrSequence[S: str]:\n ...\n\n # A is a TypeVar constrained to str or bytes\n class StrOrBytesSequence[A: (str, bytes)]:\n ...\n\nType variables can also have defaults:\n\n class IntDefault[T = int]:\n ...\n\nHowever, if desired, reusable type variables can also be constructed\nmanually, like so::\n\n T = TypeVar('T') # Can be anything\n S = TypeVar('S', bound=str) # Can be any subtype of str\n A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n D = TypeVar('D', default=int) # Defaults to int\n\nType variables exist primarily for the benefit of static type\ncheckers. They serve as the parameters for generic types as well\nas for generic function and type alias definitions.\n\nThe variance of type variables is inferred by type checkers when they\nare created through the type parameter syntax and when\n``infer_variance=True`` is passed. Manually created type variables may\nbe explicitly marked covariant or contravariant by passing\n``covariant=True`` or ``contravariant=True``. By default, manually\ncreated type variables are invariant. See PEP 484 and PEP 695 for more\ndetails.", + "_typing.TypeVar.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeVar.__eq__" => "Return self==value.", + "_typing.TypeVar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeVar.__ge__" => "Return self>=value.", + "_typing.TypeVar.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeVar.__getstate__" => "Helper for pickle.", + "_typing.TypeVar.__gt__" => "Return self>value.", + "_typing.TypeVar.__hash__" => "Return hash(self).", + "_typing.TypeVar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeVar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeVar.__le__" => "Return self<=value.", + "_typing.TypeVar.__lt__" => "Return self<value.", + "_typing.TypeVar.__ne__" => "Return self!=value.", + "_typing.TypeVar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeVar.__or__" => "Return self|value.", + "_typing.TypeVar.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeVar.__repr__" => "Return repr(self).", + "_typing.TypeVar.__ror__" => "Return value|self.", + "_typing.TypeVar.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeVar.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeVar.__str__" => "Return str(self).", + "_typing.TypeVar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeVarTuple" => "Type variable tuple. A specialized form of type variable that enables\nvariadic generics.\n\nThe preferred way to construct a type variable tuple is via the\ndedicated syntax for generic functions, classes, and type aliases,\nwhere a single '*' indicates a type variable tuple::\n\n def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n return (*tup[1:], tup[0])\n\nType variables tuples can have default values:\n\n type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\nFor compatibility with Python 3.11 and earlier, TypeVarTuple objects\ncan also be created as follows::\n\n Ts = TypeVarTuple('Ts') # Can be given any name\n DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\nJust as a TypeVar (type variable) is a placeholder for a single type,\na TypeVarTuple is a placeholder for an *arbitrary* number of types. For\nexample, if we define a generic class using a TypeVarTuple::\n\n class C[*Ts]: ...\n\nThen we can parameterize that class with an arbitrary number of type\narguments::\n\n C[int] # Fine\n C[int, str] # Also fine\n C[()] # Even this is fine\n\nFor more details, see PEP 646.\n\nNote that only TypeVarTuples defined in the global scope can be\npickled.", + "_typing.TypeVarTuple.__default__" => "The default value for this TypeVarTuple.", + "_typing.TypeVarTuple.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeVarTuple.__eq__" => "Return self==value.", + "_typing.TypeVarTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeVarTuple.__ge__" => "Return self>=value.", + "_typing.TypeVarTuple.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeVarTuple.__getstate__" => "Helper for pickle.", + "_typing.TypeVarTuple.__gt__" => "Return self>value.", + "_typing.TypeVarTuple.__hash__" => "Return hash(self).", + "_typing.TypeVarTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeVarTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeVarTuple.__iter__" => "Implement iter(self).", + "_typing.TypeVarTuple.__le__" => "Return self<=value.", + "_typing.TypeVarTuple.__lt__" => "Return self<value.", + "_typing.TypeVarTuple.__ne__" => "Return self!=value.", + "_typing.TypeVarTuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeVarTuple.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeVarTuple.__repr__" => "Return repr(self).", + "_typing.TypeVarTuple.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeVarTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeVarTuple.__str__" => "Return str(self).", + "_typing.TypeVarTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.Union" => "Represent a union type\n\nE.g. for int | str", + "_typing.Union.__class_getitem__" => "See PEP 585", + "_typing.Union.__delattr__" => "Implement delattr(self, name).", + "_typing.Union.__eq__" => "Return self==value.", + "_typing.Union.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.Union.__ge__" => "Return self>=value.", + "_typing.Union.__getattribute__" => "Return getattr(self, name).", + "_typing.Union.__getitem__" => "Return self[key].", + "_typing.Union.__getstate__" => "Helper for pickle.", + "_typing.Union.__gt__" => "Return self>value.", + "_typing.Union.__hash__" => "Return hash(self).", + "_typing.Union.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.Union.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.Union.__le__" => "Return self<=value.", + "_typing.Union.__lt__" => "Return self<value.", + "_typing.Union.__ne__" => "Return self!=value.", + "_typing.Union.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.Union.__or__" => "Return self|value.", + "_typing.Union.__origin__" => "Always returns the type", + "_typing.Union.__parameters__" => "Type variables in the types.UnionType.", + "_typing.Union.__reduce__" => "Helper for pickle.", + "_typing.Union.__reduce_ex__" => "Helper for pickle.", + "_typing.Union.__repr__" => "Return repr(self).", + "_typing.Union.__ror__" => "Return value|self.", + "_typing.Union.__setattr__" => "Implement setattr(self, name, value).", + "_typing.Union.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.Union.__str__" => "Return str(self).", + "_typing.Union.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_warnings" => "_warnings provides basic warning filtering support.\nIt is a helper module to speed up interpreter start-up.", - "_warnings.warn" => "Issue a warning, or maybe ignore it or raise an exception.\n\nmessage\n Text of the warning message.\ncategory\n The Warning category subclass. Defaults to UserWarning.\nstacklevel\n How far up the call stack to make this warning appear. A value of 2 for\n example attributes the warning to the caller of the code calling warn().\nsource\n If supplied, the destroyed object which emitted a ResourceWarning\nskip_file_prefixes\n An optional tuple of module filename prefixes indicating frames to skip\n during stacklevel computations for stack frame attribution.", + "_warnings.warn" => "Issue a warning, or maybe ignore it or raise an exception.\n\n message\n Text of the warning message.\n category\n The Warning category subclass. Defaults to UserWarning.\n stacklevel\n How far up the call stack to make this warning appear. A value of 2 for\n example attributes the warning to the caller of the code calling warn().\n source\n If supplied, the destroyed object which emitted a ResourceWarning\n skip_file_prefixes\n An optional tuple of module filename prefixes indicating frames to skip\n during stacklevel computations for stack frame attribution.", "_warnings.warn_explicit" => "Issue a warning, or maybe ignore it or raise an exception.", "_weakref" => "Weak-reference support module.", + "_weakref.CallableProxyType.__abs__" => "abs(self)", + "_weakref.CallableProxyType.__add__" => "Return self+value.", + "_weakref.CallableProxyType.__and__" => "Return self&value.", + "_weakref.CallableProxyType.__bool__" => "True if self else False", + "_weakref.CallableProxyType.__call__" => "Call self as a function.", + "_weakref.CallableProxyType.__contains__" => "Return bool(key in self).", + "_weakref.CallableProxyType.__delattr__" => "Implement delattr(self, name).", + "_weakref.CallableProxyType.__delitem__" => "Delete self[key].", + "_weakref.CallableProxyType.__divmod__" => "Return divmod(self, value).", + "_weakref.CallableProxyType.__eq__" => "Return self==value.", + "_weakref.CallableProxyType.__float__" => "float(self)", + "_weakref.CallableProxyType.__floordiv__" => "Return self//value.", + "_weakref.CallableProxyType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.CallableProxyType.__ge__" => "Return self>=value.", + "_weakref.CallableProxyType.__getattribute__" => "Return getattr(self, name).", + "_weakref.CallableProxyType.__getitem__" => "Return self[key].", + "_weakref.CallableProxyType.__getstate__" => "Helper for pickle.", + "_weakref.CallableProxyType.__gt__" => "Return self>value.", + "_weakref.CallableProxyType.__iadd__" => "Return self+=value.", + "_weakref.CallableProxyType.__iand__" => "Return self&=value.", + "_weakref.CallableProxyType.__ifloordiv__" => "Return self//=value.", + "_weakref.CallableProxyType.__ilshift__" => "Return self<<=value.", + "_weakref.CallableProxyType.__imatmul__" => "Return self@=value.", + "_weakref.CallableProxyType.__imod__" => "Return self%=value.", + "_weakref.CallableProxyType.__imul__" => "Return self*=value.", + "_weakref.CallableProxyType.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "_weakref.CallableProxyType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.CallableProxyType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.CallableProxyType.__int__" => "int(self)", + "_weakref.CallableProxyType.__invert__" => "~self", + "_weakref.CallableProxyType.__ior__" => "Return self|=value.", + "_weakref.CallableProxyType.__ipow__" => "Return self**=value.", + "_weakref.CallableProxyType.__irshift__" => "Return self>>=value.", + "_weakref.CallableProxyType.__isub__" => "Return self-=value.", + "_weakref.CallableProxyType.__iter__" => "Implement iter(self).", + "_weakref.CallableProxyType.__itruediv__" => "Return self/=value.", + "_weakref.CallableProxyType.__ixor__" => "Return self^=value.", + "_weakref.CallableProxyType.__le__" => "Return self<=value.", + "_weakref.CallableProxyType.__len__" => "Return len(self).", + "_weakref.CallableProxyType.__lshift__" => "Return self<<value.", + "_weakref.CallableProxyType.__lt__" => "Return self<value.", + "_weakref.CallableProxyType.__matmul__" => "Return self@value.", + "_weakref.CallableProxyType.__mod__" => "Return self%value.", + "_weakref.CallableProxyType.__mul__" => "Return self*value.", + "_weakref.CallableProxyType.__ne__" => "Return self!=value.", + "_weakref.CallableProxyType.__neg__" => "-self", + "_weakref.CallableProxyType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.CallableProxyType.__next__" => "Implement next(self).", + "_weakref.CallableProxyType.__or__" => "Return self|value.", + "_weakref.CallableProxyType.__pos__" => "+self", + "_weakref.CallableProxyType.__pow__" => "Return pow(self, value, mod).", + "_weakref.CallableProxyType.__radd__" => "Return value+self.", + "_weakref.CallableProxyType.__rand__" => "Return value&self.", + "_weakref.CallableProxyType.__rdivmod__" => "Return divmod(value, self).", + "_weakref.CallableProxyType.__reduce__" => "Helper for pickle.", + "_weakref.CallableProxyType.__reduce_ex__" => "Helper for pickle.", + "_weakref.CallableProxyType.__repr__" => "Return repr(self).", + "_weakref.CallableProxyType.__rfloordiv__" => "Return value//self.", + "_weakref.CallableProxyType.__rlshift__" => "Return value<<self.", + "_weakref.CallableProxyType.__rmatmul__" => "Return value@self.", + "_weakref.CallableProxyType.__rmod__" => "Return value%self.", + "_weakref.CallableProxyType.__rmul__" => "Return value*self.", + "_weakref.CallableProxyType.__ror__" => "Return value|self.", + "_weakref.CallableProxyType.__rpow__" => "Return pow(value, self, mod).", + "_weakref.CallableProxyType.__rrshift__" => "Return value>>self.", + "_weakref.CallableProxyType.__rshift__" => "Return self>>value.", + "_weakref.CallableProxyType.__rsub__" => "Return value-self.", + "_weakref.CallableProxyType.__rtruediv__" => "Return value/self.", + "_weakref.CallableProxyType.__rxor__" => "Return value^self.", + "_weakref.CallableProxyType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.CallableProxyType.__setitem__" => "Set self[key] to value.", + "_weakref.CallableProxyType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.CallableProxyType.__str__" => "Return str(self).", + "_weakref.CallableProxyType.__sub__" => "Return self-value.", + "_weakref.CallableProxyType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_weakref.CallableProxyType.__truediv__" => "Return self/value.", + "_weakref.CallableProxyType.__xor__" => "Return self^value.", + "_weakref.ProxyType.__abs__" => "abs(self)", + "_weakref.ProxyType.__add__" => "Return self+value.", + "_weakref.ProxyType.__and__" => "Return self&value.", + "_weakref.ProxyType.__bool__" => "True if self else False", + "_weakref.ProxyType.__contains__" => "Return bool(key in self).", + "_weakref.ProxyType.__delattr__" => "Implement delattr(self, name).", + "_weakref.ProxyType.__delitem__" => "Delete self[key].", + "_weakref.ProxyType.__divmod__" => "Return divmod(self, value).", + "_weakref.ProxyType.__eq__" => "Return self==value.", + "_weakref.ProxyType.__float__" => "float(self)", + "_weakref.ProxyType.__floordiv__" => "Return self//value.", + "_weakref.ProxyType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ProxyType.__ge__" => "Return self>=value.", + "_weakref.ProxyType.__getattribute__" => "Return getattr(self, name).", + "_weakref.ProxyType.__getitem__" => "Return self[key].", + "_weakref.ProxyType.__getstate__" => "Helper for pickle.", + "_weakref.ProxyType.__gt__" => "Return self>value.", + "_weakref.ProxyType.__iadd__" => "Return self+=value.", + "_weakref.ProxyType.__iand__" => "Return self&=value.", + "_weakref.ProxyType.__ifloordiv__" => "Return self//=value.", + "_weakref.ProxyType.__ilshift__" => "Return self<<=value.", + "_weakref.ProxyType.__imatmul__" => "Return self@=value.", + "_weakref.ProxyType.__imod__" => "Return self%=value.", + "_weakref.ProxyType.__imul__" => "Return self*=value.", + "_weakref.ProxyType.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "_weakref.ProxyType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ProxyType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ProxyType.__int__" => "int(self)", + "_weakref.ProxyType.__invert__" => "~self", + "_weakref.ProxyType.__ior__" => "Return self|=value.", + "_weakref.ProxyType.__ipow__" => "Return self**=value.", + "_weakref.ProxyType.__irshift__" => "Return self>>=value.", + "_weakref.ProxyType.__isub__" => "Return self-=value.", + "_weakref.ProxyType.__iter__" => "Implement iter(self).", + "_weakref.ProxyType.__itruediv__" => "Return self/=value.", + "_weakref.ProxyType.__ixor__" => "Return self^=value.", + "_weakref.ProxyType.__le__" => "Return self<=value.", + "_weakref.ProxyType.__len__" => "Return len(self).", + "_weakref.ProxyType.__lshift__" => "Return self<<value.", + "_weakref.ProxyType.__lt__" => "Return self<value.", + "_weakref.ProxyType.__matmul__" => "Return self@value.", + "_weakref.ProxyType.__mod__" => "Return self%value.", + "_weakref.ProxyType.__mul__" => "Return self*value.", + "_weakref.ProxyType.__ne__" => "Return self!=value.", + "_weakref.ProxyType.__neg__" => "-self", + "_weakref.ProxyType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ProxyType.__next__" => "Implement next(self).", + "_weakref.ProxyType.__or__" => "Return self|value.", + "_weakref.ProxyType.__pos__" => "+self", + "_weakref.ProxyType.__pow__" => "Return pow(self, value, mod).", + "_weakref.ProxyType.__radd__" => "Return value+self.", + "_weakref.ProxyType.__rand__" => "Return value&self.", + "_weakref.ProxyType.__rdivmod__" => "Return divmod(value, self).", + "_weakref.ProxyType.__reduce__" => "Helper for pickle.", + "_weakref.ProxyType.__reduce_ex__" => "Helper for pickle.", + "_weakref.ProxyType.__repr__" => "Return repr(self).", + "_weakref.ProxyType.__rfloordiv__" => "Return value//self.", + "_weakref.ProxyType.__rlshift__" => "Return value<<self.", + "_weakref.ProxyType.__rmatmul__" => "Return value@self.", + "_weakref.ProxyType.__rmod__" => "Return value%self.", + "_weakref.ProxyType.__rmul__" => "Return value*self.", + "_weakref.ProxyType.__ror__" => "Return value|self.", + "_weakref.ProxyType.__rpow__" => "Return pow(value, self, mod).", + "_weakref.ProxyType.__rrshift__" => "Return value>>self.", + "_weakref.ProxyType.__rshift__" => "Return self>>value.", + "_weakref.ProxyType.__rsub__" => "Return value-self.", + "_weakref.ProxyType.__rtruediv__" => "Return value/self.", + "_weakref.ProxyType.__rxor__" => "Return value^self.", + "_weakref.ProxyType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ProxyType.__setitem__" => "Set self[key] to value.", + "_weakref.ProxyType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ProxyType.__str__" => "Return str(self).", + "_weakref.ProxyType.__sub__" => "Return self-value.", + "_weakref.ProxyType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_weakref.ProxyType.__truediv__" => "Return self/value.", + "_weakref.ProxyType.__xor__" => "Return self^value.", + "_weakref.ReferenceType.__call__" => "Call self as a function.", + "_weakref.ReferenceType.__class_getitem__" => "See PEP 585", + "_weakref.ReferenceType.__delattr__" => "Implement delattr(self, name).", + "_weakref.ReferenceType.__eq__" => "Return self==value.", + "_weakref.ReferenceType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ReferenceType.__ge__" => "Return self>=value.", + "_weakref.ReferenceType.__getattribute__" => "Return getattr(self, name).", + "_weakref.ReferenceType.__getstate__" => "Helper for pickle.", + "_weakref.ReferenceType.__gt__" => "Return self>value.", + "_weakref.ReferenceType.__hash__" => "Return hash(self).", + "_weakref.ReferenceType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ReferenceType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ReferenceType.__le__" => "Return self<=value.", + "_weakref.ReferenceType.__lt__" => "Return self<value.", + "_weakref.ReferenceType.__ne__" => "Return self!=value.", + "_weakref.ReferenceType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ReferenceType.__reduce__" => "Helper for pickle.", + "_weakref.ReferenceType.__reduce_ex__" => "Helper for pickle.", + "_weakref.ReferenceType.__repr__" => "Return repr(self).", + "_weakref.ReferenceType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ReferenceType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ReferenceType.__str__" => "Return str(self).", + "_weakref.ReferenceType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_weakref._remove_dead_weakref" => "Atomically remove key from dict if it points to a dead weakref.", "_weakref.getweakrefcount" => "Return the number of weak references to 'object'.", "_weakref.getweakrefs" => "Return a list of all weak reference objects pointing to 'object'.", "_weakref.proxy" => "Create a proxy object that weakly references 'object'.\n\n'callback', if given, is called with a reference to the\nproxy when 'object' is about to be finalized.", + "_weakref.ref.__call__" => "Call self as a function.", + "_weakref.ref.__class_getitem__" => "See PEP 585", + "_weakref.ref.__delattr__" => "Implement delattr(self, name).", + "_weakref.ref.__eq__" => "Return self==value.", + "_weakref.ref.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ref.__ge__" => "Return self>=value.", + "_weakref.ref.__getattribute__" => "Return getattr(self, name).", + "_weakref.ref.__getstate__" => "Helper for pickle.", + "_weakref.ref.__gt__" => "Return self>value.", + "_weakref.ref.__hash__" => "Return hash(self).", + "_weakref.ref.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ref.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ref.__le__" => "Return self<=value.", + "_weakref.ref.__lt__" => "Return self<value.", + "_weakref.ref.__ne__" => "Return self!=value.", + "_weakref.ref.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ref.__reduce__" => "Helper for pickle.", + "_weakref.ref.__reduce_ex__" => "Helper for pickle.", + "_weakref.ref.__repr__" => "Return repr(self).", + "_weakref.ref.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ref.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ref.__str__" => "Return str(self).", + "_weakref.ref.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "_winapi.BatchedWaitForMultipleObjects" => "Supports a larger number of handles than WaitForMultipleObjects\n\nNote that the handles may be waited on other threads, which could cause\nissues for objects like mutexes that become associated with the thread\nthat was waiting for them. Objects may also be left signalled, even if\nthe wait fails.\n\nIt is recommended to use WaitForMultipleObjects whenever possible, and\nonly switch to BatchedWaitForMultipleObjects for scenarios where you\ncontrol all the handles involved, such as your own thread pool or\nfiles, and all wait objects are left unmodified by a wait (for example,\nmanual reset events, threads, and files/pipes).\n\nOverlapped handles returned from this module use manual reset events.", "_winapi.CloseHandle" => "Close handle.", "_winapi.CopyFile2" => "Copies a file from one name to a new name.\n\nThis is implemented using the CopyFile2 API, which preserves all stat\nand metadata information apart from security attributes.\n\nprogress_routine is reserved for future use, but is currently not\nimplemented. Its value is ignored.", @@ -3448,6 +9297,142 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "_winapi._mimetypes_read_windows_registry" => "Optimized function for reading all known MIME types from the registry.\n\n*on_type_read* is a callable taking *type* and *ext* arguments, as for\nMimeTypes.add_type.", "_wmi.exec_query" => "Runs a WMI query against the local machine.\n\nThis returns a single string with 'name=value' pairs in a flat array separated\nby null characters.", "_zoneinfo" => "C implementation of the zoneinfo module", + "_zoneinfo.ZoneInfo.__delattr__" => "Implement delattr(self, name).", + "_zoneinfo.ZoneInfo.__eq__" => "Return self==value.", + "_zoneinfo.ZoneInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zoneinfo.ZoneInfo.__ge__" => "Return self>=value.", + "_zoneinfo.ZoneInfo.__getattribute__" => "Return getattr(self, name).", + "_zoneinfo.ZoneInfo.__getstate__" => "Helper for pickle.", + "_zoneinfo.ZoneInfo.__gt__" => "Return self>value.", + "_zoneinfo.ZoneInfo.__hash__" => "Return hash(self).", + "_zoneinfo.ZoneInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zoneinfo.ZoneInfo.__init_subclass__" => "Function to initialize subclasses.", + "_zoneinfo.ZoneInfo.__le__" => "Return self<=value.", + "_zoneinfo.ZoneInfo.__lt__" => "Return self<value.", + "_zoneinfo.ZoneInfo.__ne__" => "Return self!=value.", + "_zoneinfo.ZoneInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zoneinfo.ZoneInfo.__reduce__" => "Function for serialization with the pickle protocol.", + "_zoneinfo.ZoneInfo.__reduce_ex__" => "Helper for pickle.", + "_zoneinfo.ZoneInfo.__repr__" => "Return repr(self).", + "_zoneinfo.ZoneInfo.__setattr__" => "Implement setattr(self, name, value).", + "_zoneinfo.ZoneInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_zoneinfo.ZoneInfo.__str__" => "Return str(self).", + "_zoneinfo.ZoneInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zoneinfo.ZoneInfo.dst" => "Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime.", + "_zoneinfo.ZoneInfo.fromutc" => "Given a datetime with local time in UTC, retrieve an adjusted datetime in local time.", + "_zoneinfo.ZoneInfo.tzname" => "Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime.", + "_zoneinfo.ZoneInfo.utcoffset" => "Retrieve a timedelta representing the UTC offset in a zone at the given datetime.", + "_zstd" => "Implementation module for Zstandard compression.", + "_zstd.ZstdCompressor" => "Create a compressor object for compressing data incrementally.\n\n level\n The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n options\n A dict object that contains advanced compression parameters.\n zstd_dict\n A ZstdDict object, a pre-trained Zstandard dictionary.\n\nThread-safe at method level. For one-shot compression, use the compress()\nfunction instead.", + "_zstd.ZstdCompressor.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdCompressor.__eq__" => "Return self==value.", + "_zstd.ZstdCompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdCompressor.__ge__" => "Return self>=value.", + "_zstd.ZstdCompressor.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdCompressor.__getstate__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__gt__" => "Return self>value.", + "_zstd.ZstdCompressor.__hash__" => "Return hash(self).", + "_zstd.ZstdCompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdCompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdCompressor.__le__" => "Return self<=value.", + "_zstd.ZstdCompressor.__lt__" => "Return self<value.", + "_zstd.ZstdCompressor.__ne__" => "Return self!=value.", + "_zstd.ZstdCompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdCompressor.__reduce__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__repr__" => "Return repr(self).", + "_zstd.ZstdCompressor.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdCompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdCompressor.__str__" => "Return str(self).", + "_zstd.ZstdCompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdCompressor.compress" => "Provide data to the compressor object.\n\n mode\n Can be these 3 values ZstdCompressor.CONTINUE,\n ZstdCompressor.FLUSH_BLOCK, ZstdCompressor.FLUSH_FRAME\n\nReturn a chunk of compressed data if possible, or b'' otherwise. When you have\nfinished providing data to the compressor, call the flush() method to finish\nthe compression process.", + "_zstd.ZstdCompressor.flush" => "Finish the compression process.\n\n mode\n Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n ZstdCompressor.FLUSH_BLOCK\n\nFlush any remaining data left in internal buffers. Since Zstandard data\nconsists of one or more independent frames, the compressor object can still\nbe used after this method is called.", + "_zstd.ZstdCompressor.last_mode" => "The last mode used to this compressor object, its value can be .CONTINUE,\n.FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\nIt can be used to get the current state of a compressor, such as, data\nflushed, or a frame ended.", + "_zstd.ZstdCompressor.set_pledged_input_size" => "Set the uncompressed content size to be written into the frame header.\n\n size\n The size of the uncompressed data to be provided to the compressor.\n\nThis method can be used to ensure the header of the frame about to be written\nincludes the size of the data, unless the CompressionParameter.content_size_flag\nis set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n\nIt is important to ensure that the pledged data size matches the actual data\nsize. If they do not match the compressed output data may be corrupted and the\nfinal chunk written may be lost.", + "_zstd.ZstdDecompressor" => "Create a decompressor object for decompressing data incrementally.\n\n zstd_dict\n A ZstdDict object, a pre-trained Zstandard dictionary.\n options\n A dict object that contains advanced decompression parameters.\n\nThread-safe at method level. For one-shot decompression, use the decompress()\nfunction instead.", + "_zstd.ZstdDecompressor.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdDecompressor.__eq__" => "Return self==value.", + "_zstd.ZstdDecompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdDecompressor.__ge__" => "Return self>=value.", + "_zstd.ZstdDecompressor.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdDecompressor.__getstate__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__gt__" => "Return self>value.", + "_zstd.ZstdDecompressor.__hash__" => "Return hash(self).", + "_zstd.ZstdDecompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdDecompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdDecompressor.__le__" => "Return self<=value.", + "_zstd.ZstdDecompressor.__lt__" => "Return self<value.", + "_zstd.ZstdDecompressor.__ne__" => "Return self!=value.", + "_zstd.ZstdDecompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdDecompressor.__reduce__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__repr__" => "Return repr(self).", + "_zstd.ZstdDecompressor.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdDecompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdDecompressor.__str__" => "Return str(self).", + "_zstd.ZstdDecompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdDecompressor.decompress" => "Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise.\n\n data\n A bytes-like object, Zstandard data to be decompressed.\n max_length\n Maximum size of returned data. When it is negative, the size of\n output buffer is unlimited. When it is nonnegative, returns at\n most max_length bytes of decompressed data.\n\nIf *max_length* is nonnegative, returns at most *max_length* bytes of\ndecompressed data. If this limit is reached and further output can be\nproduced, *self.needs_input* will be set to ``False``. In this case, the next\ncall to *decompress()* may provide *data* as b'' to obtain more of the output.\n\nIf all of the input data was decompressed and returned (either because this\nwas less than *max_length* bytes, or because *max_length* was negative),\n*self.needs_input* will be set to True.\n\nAttempting to decompress data after the end of a frame is reached raises an\nEOFError. Any data found after the end of the frame is ignored and saved in\nthe self.unused_data attribute.", + "_zstd.ZstdDecompressor.eof" => "True means the end of the first frame has been reached. If decompress data\nafter that, an EOFError exception will be raised.", + "_zstd.ZstdDecompressor.needs_input" => "If the max_length output limit in .decompress() method has been reached,\nand the decompressor has (or may has) unconsumed input data, it will be set\nto False. In this case, passing b'' to the .decompress() method may output\nfurther data.", + "_zstd.ZstdDecompressor.unused_data" => "A bytes object of un-consumed input data.\n\nWhen ZstdDecompressor object stops after a frame is\ndecompressed, unused input data after the frame. Otherwise this will be b''.", + "_zstd.ZstdDict" => "Represents a Zstandard dictionary.\n\n dict_content\n The content of a Zstandard dictionary as a bytes-like object.\n is_raw\n If true, perform no checks on *dict_content*, useful for some\n advanced cases. Otherwise, check that the content represents\n a Zstandard dictionary created by the zstd library or CLI.\n\nThe dictionary can be used for compression or decompression, and can be shared\nby multiple ZstdCompressor or ZstdDecompressor objects.", + "_zstd.ZstdDict.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdDict.__eq__" => "Return self==value.", + "_zstd.ZstdDict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdDict.__ge__" => "Return self>=value.", + "_zstd.ZstdDict.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdDict.__getstate__" => "Helper for pickle.", + "_zstd.ZstdDict.__gt__" => "Return self>value.", + "_zstd.ZstdDict.__hash__" => "Return hash(self).", + "_zstd.ZstdDict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdDict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdDict.__le__" => "Return self<=value.", + "_zstd.ZstdDict.__len__" => "Return len(self).", + "_zstd.ZstdDict.__lt__" => "Return self<value.", + "_zstd.ZstdDict.__ne__" => "Return self!=value.", + "_zstd.ZstdDict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdDict.__reduce__" => "Helper for pickle.", + "_zstd.ZstdDict.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdDict.__repr__" => "Return repr(self).", + "_zstd.ZstdDict.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdDict.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdDict.__str__" => "Return str(self).", + "_zstd.ZstdDict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdDict.as_digested_dict" => "Load as a digested dictionary to compressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_digested_dict)\n\n1. Some advanced compression parameters of compressor may be overridden\n by parameters of digested dictionary.\n2. ZstdDict has a digested dictionaries cache for each compression level.\n It's faster when loading again a digested dictionary with the same\n compression level.\n3. No need to use this for decompression.", + "_zstd.ZstdDict.as_prefix" => "Load as a prefix to compressor/decompressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_prefix)\n\n1. Prefix is compatible with long distance matching, while dictionary is not.\n2. It only works for the first frame, then the compressor/decompressor will\n return to no prefix state.\n3. When decompressing, must use the same prefix as when compressing.", + "_zstd.ZstdDict.as_undigested_dict" => "Load as an undigested dictionary to compressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_undigested_dict)\n\n1. The advanced compression parameters of compressor will not be overridden.\n2. Loading an undigested dictionary is costly. If load an undigested dictionary\n multiple times, consider reusing a compressor object.\n3. No need to use this for decompression.", + "_zstd.ZstdDict.dict_content" => "The content of a Zstandard dictionary, as a bytes object.", + "_zstd.ZstdDict.dict_id" => "The Zstandard dictionary, an int between 0 and 2**32.\n\nA non-zero value represents an ordinary Zstandard dictionary,\nconforming to the standardised format.\n\nA value of zero indicates a 'raw content' dictionary,\nwithout any restrictions on format or content.", + "_zstd.ZstdError" => "An error occurred in the zstd library.", + "_zstd.ZstdError.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdError.__eq__" => "Return self==value.", + "_zstd.ZstdError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdError.__ge__" => "Return self>=value.", + "_zstd.ZstdError.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdError.__getstate__" => "Helper for pickle.", + "_zstd.ZstdError.__gt__" => "Return self>value.", + "_zstd.ZstdError.__hash__" => "Return hash(self).", + "_zstd.ZstdError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdError.__le__" => "Return self<=value.", + "_zstd.ZstdError.__lt__" => "Return self<value.", + "_zstd.ZstdError.__ne__" => "Return self!=value.", + "_zstd.ZstdError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdError.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdError.__repr__" => "Return repr(self).", + "_zstd.ZstdError.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdError.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdError.__str__" => "Return str(self).", + "_zstd.ZstdError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdError.__weakref__" => "list of weak references to the object", + "_zstd.ZstdError.add_note" => "Add a note to the exception", + "_zstd.ZstdError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_zstd.finalize_dict" => "Finalize a Zstandard dictionary.\n\n custom_dict_bytes\n Custom dictionary content.\n samples_bytes\n Concatenation of samples.\n samples_sizes\n Tuple of samples' sizes.\n dict_size\n The size of the dictionary.\n compression_level\n Optimize for a specific Zstandard compression level, 0 means default.", + "_zstd.get_frame_info" => "Get Zstandard frame infomation from a frame header.\n\n frame_buffer\n A bytes-like object, containing the header of a Zstandard frame.", + "_zstd.get_frame_size" => "Get the size of a Zstandard frame, including the header and optional checksum.\n\n frame_buffer\n A bytes-like object, it should start from the beginning of a frame,\n and contains at least one complete frame.", + "_zstd.get_param_bounds" => "Get CompressionParameter/DecompressionParameter bounds.\n\n parameter\n The parameter to get bounds.\n is_compress\n True for CompressionParameter, False for DecompressionParameter.", + "_zstd.set_parameter_types" => "Set CompressionParameter and DecompressionParameter types for validity check.\n\n c_parameter_type\n CompressionParameter IntEnum type object\n d_parameter_type\n DecompressionParameter IntEnum type object", + "_zstd.train_dict" => "Train a Zstandard dictionary on sample data.\n\n samples_bytes\n Concatenation of samples.\n samples_sizes\n Tuple of samples' sizes.\n dict_size\n The size of the dictionary.", "array" => "This module defines an object type which can efficiently represent\nan array of basic values: characters, integers, floating-point\nnumbers. Arrays are sequence types and behave very much like lists,\nexcept that the type of objects stored in them is constrained.", "array.ArrayType" => "array(typecode [, initializer]) -> array\n\nReturn a new array whose items are restricted by typecode, and\ninitialized from the optional initializer value, which must be a list,\nstring or iterable over elements of the appropriate type.\n\nArrays represent basic values and behave very much like lists, except\nthe type of objects stored in them is constrained. The type is specified\nat object creation time by using a type code, which is a single character.\nThe following type codes are defined:\n\n Type code C Type Minimum size in bytes\n 'b' signed integer 1\n 'B' unsigned integer 1\n 'u' Unicode character 2 (see note)\n 'h' signed integer 2\n 'H' unsigned integer 2\n 'i' signed integer 2\n 'I' unsigned integer 2\n 'l' signed integer 4\n 'L' unsigned integer 4\n 'q' signed integer 8 (see note)\n 'Q' unsigned integer 8 (see note)\n 'f' floating-point 4\n 'd' floating-point 8\n\nNOTE: The 'u' typecode corresponds to Python's unicode character. On\nnarrow builds this is 2-bytes on wide builds this is 4-bytes.\n\nNOTE: The 'q' and 'Q' type codes are only available if the platform\nC compiler used to build Python supports 'long long', or, on Windows,\n'__int64'.\n\nMethods:\n\nappend() -- append a new item to the end of the array\nbuffer_info() -- return information giving the current memory info\nbyteswap() -- byteswap all the items of the array\ncount() -- return number of occurrences of an object\nextend() -- extend array by appending multiple elements from an iterable\nfromfile() -- read items from a file object\nfromlist() -- append items from the list\nfrombytes() -- append items from the string\nindex() -- return index of first occurrence of an object\ninsert() -- insert a new item into the array at a provided position\npop() -- remove and return item (default last)\nremove() -- remove first occurrence of an object\nreverse() -- reverse the order of the items in the array\ntofile() -- write all items to a file object\ntolist() -- return the array converted to an ordinary list\ntobytes() -- return the array converted to a string\n\nAttributes:\n\ntypecode -- the typecode character used to create the array\nitemsize -- the length in bytes of one array item", "array.ArrayType.__add__" => "Return self+value.", @@ -3570,11 +9555,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "atexit._clear" => "Clear the list of previously registered exit functions.", "atexit._ncallbacks" => "Return the number of registered exit functions.", "atexit._run_exitfuncs" => "Run all registered exit functions.\n\nIf a callback raises an exception, it is logged with sys.unraisablehook.", - "atexit.register" => "Register a function to be executed upon normal program termination\n\nfunc - function to be called at exit\nargs - optional arguments to pass to func\nkwargs - optional keyword arguments to pass to func\n\nfunc is returned to facilitate usage as a decorator.", + "atexit.register" => "Register a function to be executed upon normal program termination\n\n func - function to be called at exit\n args - optional arguments to pass to func\n kwargs - optional keyword arguments to pass to func\n\n func is returned to facilitate usage as a decorator.", "atexit.unregister" => "Unregister an exit function which was previously registered using\natexit.register\n\n func - function to be unregistered", "binascii" => "Conversion between binary data and ASCII", - "binascii.Error.__cause__" => "exception cause", - "binascii.Error.__context__" => "exception context", "binascii.Error.__delattr__" => "Implement delattr(self, name).", "binascii.Error.__eq__" => "Return self==value.", "binascii.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3596,10 +9579,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "binascii.Error.__str__" => "Return str(self).", "binascii.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "binascii.Error.__weakref__" => "list of weak references to the object", - "binascii.Error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "binascii.Error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "binascii.Incomplete.__cause__" => "exception cause", - "binascii.Incomplete.__context__" => "exception context", + "binascii.Error.add_note" => "Add a note to the exception", + "binascii.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", "binascii.Incomplete.__delattr__" => "Implement delattr(self, name).", "binascii.Incomplete.__eq__" => "Return self==value.", "binascii.Incomplete.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3621,9 +9602,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "binascii.Incomplete.__str__" => "Return str(self).", "binascii.Incomplete.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "binascii.Incomplete.__weakref__" => "list of weak references to the object", - "binascii.Incomplete.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "binascii.Incomplete.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "binascii.a2b_base64" => "Decode a line of base64 data.\n\nstrict_mode\n When set to True, bytes that are not part of the base64 standard are not allowed.\n The same applies to excess data after padding (= / ==).", + "binascii.Incomplete.add_note" => "Add a note to the exception", + "binascii.Incomplete.with_traceback" => "Set self.__traceback__ to tb and return self.", + "binascii.a2b_base64" => "Decode a line of base64 data.\n\n strict_mode\n When set to True, bytes that are not part of the base64 standard are not allowed.\n The same applies to excess data after padding (= / ==).", "binascii.a2b_hex" => "Binary data of hexadecimal representation.\n\nhexstr must contain an even number of hex digits (upper or lower case).\nThis function is also available as \"unhexlify()\".", "binascii.a2b_qp" => "Decode a string of qp-encoded data.", "binascii.a2b_uu" => "Decode a line of uuencoded data.", @@ -3637,8 +9618,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "binascii.unhexlify" => "Binary data of hexadecimal representation.\n\nhexstr must contain an even number of hex digits (upper or lower case).", "builtins" => "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", "builtins.ArithmeticError" => "Base class for arithmetic errors.", - "builtins.ArithmeticError.__cause__" => "exception cause", - "builtins.ArithmeticError.__context__" => "exception context", "builtins.ArithmeticError.__delattr__" => "Implement delattr(self, name).", "builtins.ArithmeticError.__eq__" => "Return self==value.", "builtins.ArithmeticError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3659,11 +9638,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ArithmeticError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ArithmeticError.__str__" => "Return str(self).", "builtins.ArithmeticError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ArithmeticError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ArithmeticError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ArithmeticError.add_note" => "Add a note to the exception", + "builtins.ArithmeticError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.AssertionError" => "Assertion failed.", - "builtins.AssertionError.__cause__" => "exception cause", - "builtins.AssertionError.__context__" => "exception context", "builtins.AssertionError.__delattr__" => "Implement delattr(self, name).", "builtins.AssertionError.__eq__" => "Return self==value.", "builtins.AssertionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3684,11 +9661,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.AssertionError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.AssertionError.__str__" => "Return str(self).", "builtins.AssertionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.AssertionError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.AssertionError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.AssertionError.add_note" => "Add a note to the exception", + "builtins.AssertionError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.AttributeError" => "Attribute not found.", - "builtins.AttributeError.__cause__" => "exception cause", - "builtins.AttributeError.__context__" => "exception context", "builtins.AttributeError.__delattr__" => "Implement delattr(self, name).", "builtins.AttributeError.__eq__" => "Return self==value.", "builtins.AttributeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3708,13 +9683,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.AttributeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.AttributeError.__str__" => "Return str(self).", "builtins.AttributeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.AttributeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.AttributeError.add_note" => "Add a note to the exception", "builtins.AttributeError.name" => "attribute name", "builtins.AttributeError.obj" => "object", - "builtins.AttributeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.AttributeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BaseException" => "Common base class for all exceptions", - "builtins.BaseException.__cause__" => "exception cause", - "builtins.BaseException.__context__" => "exception context", "builtins.BaseException.__delattr__" => "Implement delattr(self, name).", "builtins.BaseException.__eq__" => "Return self==value.", "builtins.BaseException.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3735,12 +9708,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BaseException.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BaseException.__str__" => "Return str(self).", "builtins.BaseException.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BaseException.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.BaseException.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BaseException.add_note" => "Add a note to the exception", + "builtins.BaseException.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BaseExceptionGroup" => "A combination of multiple unrelated exceptions.", - "builtins.BaseExceptionGroup.__cause__" => "exception cause", "builtins.BaseExceptionGroup.__class_getitem__" => "See PEP 585", - "builtins.BaseExceptionGroup.__context__" => "exception context", "builtins.BaseExceptionGroup.__delattr__" => "Implement delattr(self, name).", "builtins.BaseExceptionGroup.__eq__" => "Return self==value.", "builtins.BaseExceptionGroup.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3761,13 +9732,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BaseExceptionGroup.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BaseExceptionGroup.__str__" => "Return str(self).", "builtins.BaseExceptionGroup.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BaseExceptionGroup.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.BaseExceptionGroup.add_note" => "Add a note to the exception", "builtins.BaseExceptionGroup.exceptions" => "nested exceptions", "builtins.BaseExceptionGroup.message" => "exception message", - "builtins.BaseExceptionGroup.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BaseExceptionGroup.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BlockingIOError" => "I/O operation would block.", - "builtins.BlockingIOError.__cause__" => "exception cause", - "builtins.BlockingIOError.__context__" => "exception context", "builtins.BlockingIOError.__delattr__" => "Implement delattr(self, name).", "builtins.BlockingIOError.__eq__" => "Return self==value.", "builtins.BlockingIOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3788,16 +9757,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BlockingIOError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BlockingIOError.__str__" => "Return str(self).", "builtins.BlockingIOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BlockingIOError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.BlockingIOError.add_note" => "Add a note to the exception", "builtins.BlockingIOError.errno" => "POSIX exception code", "builtins.BlockingIOError.filename" => "exception filename", "builtins.BlockingIOError.filename2" => "second exception filename", "builtins.BlockingIOError.strerror" => "exception strerror", "builtins.BlockingIOError.winerror" => "Win32 exception code", - "builtins.BlockingIOError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BlockingIOError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BrokenPipeError" => "Broken pipe.", - "builtins.BrokenPipeError.__cause__" => "exception cause", - "builtins.BrokenPipeError.__context__" => "exception context", "builtins.BrokenPipeError.__delattr__" => "Implement delattr(self, name).", "builtins.BrokenPipeError.__eq__" => "Return self==value.", "builtins.BrokenPipeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3818,16 +9785,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BrokenPipeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BrokenPipeError.__str__" => "Return str(self).", "builtins.BrokenPipeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BrokenPipeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.BrokenPipeError.add_note" => "Add a note to the exception", "builtins.BrokenPipeError.errno" => "POSIX exception code", "builtins.BrokenPipeError.filename" => "exception filename", "builtins.BrokenPipeError.filename2" => "second exception filename", "builtins.BrokenPipeError.strerror" => "exception strerror", "builtins.BrokenPipeError.winerror" => "Win32 exception code", - "builtins.BrokenPipeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BrokenPipeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BufferError" => "Buffer error.", - "builtins.BufferError.__cause__" => "exception cause", - "builtins.BufferError.__context__" => "exception context", "builtins.BufferError.__delattr__" => "Implement delattr(self, name).", "builtins.BufferError.__eq__" => "Return self==value.", "builtins.BufferError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3848,11 +9813,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BufferError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BufferError.__str__" => "Return str(self).", "builtins.BufferError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BufferError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.BufferError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BufferError.add_note" => "Add a note to the exception", + "builtins.BufferError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.BytesWarning" => "Base class for warnings about bytes and buffer related problems, mostly\nrelated to conversion from str or comparing to str.", - "builtins.BytesWarning.__cause__" => "exception cause", - "builtins.BytesWarning.__context__" => "exception context", "builtins.BytesWarning.__delattr__" => "Implement delattr(self, name).", "builtins.BytesWarning.__eq__" => "Return self==value.", "builtins.BytesWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3873,11 +9836,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.BytesWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.BytesWarning.__str__" => "Return str(self).", "builtins.BytesWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.BytesWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.BytesWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.BytesWarning.add_note" => "Add a note to the exception", + "builtins.BytesWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ChildProcessError" => "Child process error.", - "builtins.ChildProcessError.__cause__" => "exception cause", - "builtins.ChildProcessError.__context__" => "exception context", "builtins.ChildProcessError.__delattr__" => "Implement delattr(self, name).", "builtins.ChildProcessError.__eq__" => "Return self==value.", "builtins.ChildProcessError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3898,16 +9859,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ChildProcessError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ChildProcessError.__str__" => "Return str(self).", "builtins.ChildProcessError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ChildProcessError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ChildProcessError.add_note" => "Add a note to the exception", "builtins.ChildProcessError.errno" => "POSIX exception code", "builtins.ChildProcessError.filename" => "exception filename", "builtins.ChildProcessError.filename2" => "second exception filename", "builtins.ChildProcessError.strerror" => "exception strerror", "builtins.ChildProcessError.winerror" => "Win32 exception code", - "builtins.ChildProcessError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ChildProcessError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ConnectionAbortedError" => "Connection aborted.", - "builtins.ConnectionAbortedError.__cause__" => "exception cause", - "builtins.ConnectionAbortedError.__context__" => "exception context", "builtins.ConnectionAbortedError.__delattr__" => "Implement delattr(self, name).", "builtins.ConnectionAbortedError.__eq__" => "Return self==value.", "builtins.ConnectionAbortedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3928,16 +9887,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ConnectionAbortedError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ConnectionAbortedError.__str__" => "Return str(self).", "builtins.ConnectionAbortedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ConnectionAbortedError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ConnectionAbortedError.add_note" => "Add a note to the exception", "builtins.ConnectionAbortedError.errno" => "POSIX exception code", "builtins.ConnectionAbortedError.filename" => "exception filename", "builtins.ConnectionAbortedError.filename2" => "second exception filename", "builtins.ConnectionAbortedError.strerror" => "exception strerror", "builtins.ConnectionAbortedError.winerror" => "Win32 exception code", - "builtins.ConnectionAbortedError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ConnectionAbortedError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ConnectionError" => "Connection error.", - "builtins.ConnectionError.__cause__" => "exception cause", - "builtins.ConnectionError.__context__" => "exception context", "builtins.ConnectionError.__delattr__" => "Implement delattr(self, name).", "builtins.ConnectionError.__eq__" => "Return self==value.", "builtins.ConnectionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3958,16 +9915,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ConnectionError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ConnectionError.__str__" => "Return str(self).", "builtins.ConnectionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ConnectionError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ConnectionError.add_note" => "Add a note to the exception", "builtins.ConnectionError.errno" => "POSIX exception code", "builtins.ConnectionError.filename" => "exception filename", "builtins.ConnectionError.filename2" => "second exception filename", "builtins.ConnectionError.strerror" => "exception strerror", "builtins.ConnectionError.winerror" => "Win32 exception code", - "builtins.ConnectionError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ConnectionError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ConnectionRefusedError" => "Connection refused.", - "builtins.ConnectionRefusedError.__cause__" => "exception cause", - "builtins.ConnectionRefusedError.__context__" => "exception context", "builtins.ConnectionRefusedError.__delattr__" => "Implement delattr(self, name).", "builtins.ConnectionRefusedError.__eq__" => "Return self==value.", "builtins.ConnectionRefusedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -3988,16 +9943,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ConnectionRefusedError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ConnectionRefusedError.__str__" => "Return str(self).", "builtins.ConnectionRefusedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ConnectionRefusedError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ConnectionRefusedError.add_note" => "Add a note to the exception", "builtins.ConnectionRefusedError.errno" => "POSIX exception code", "builtins.ConnectionRefusedError.filename" => "exception filename", "builtins.ConnectionRefusedError.filename2" => "second exception filename", "builtins.ConnectionRefusedError.strerror" => "exception strerror", "builtins.ConnectionRefusedError.winerror" => "Win32 exception code", - "builtins.ConnectionRefusedError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ConnectionRefusedError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ConnectionResetError" => "Connection reset.", - "builtins.ConnectionResetError.__cause__" => "exception cause", - "builtins.ConnectionResetError.__context__" => "exception context", "builtins.ConnectionResetError.__delattr__" => "Implement delattr(self, name).", "builtins.ConnectionResetError.__eq__" => "Return self==value.", "builtins.ConnectionResetError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4018,16 +9971,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ConnectionResetError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ConnectionResetError.__str__" => "Return str(self).", "builtins.ConnectionResetError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ConnectionResetError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ConnectionResetError.add_note" => "Add a note to the exception", "builtins.ConnectionResetError.errno" => "POSIX exception code", "builtins.ConnectionResetError.filename" => "exception filename", "builtins.ConnectionResetError.filename2" => "second exception filename", "builtins.ConnectionResetError.strerror" => "exception strerror", "builtins.ConnectionResetError.winerror" => "Win32 exception code", - "builtins.ConnectionResetError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ConnectionResetError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.DeprecationWarning" => "Base class for warnings about deprecated features.", - "builtins.DeprecationWarning.__cause__" => "exception cause", - "builtins.DeprecationWarning.__context__" => "exception context", "builtins.DeprecationWarning.__delattr__" => "Implement delattr(self, name).", "builtins.DeprecationWarning.__eq__" => "Return self==value.", "builtins.DeprecationWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4048,8 +9999,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.DeprecationWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.DeprecationWarning.__str__" => "Return str(self).", "builtins.DeprecationWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.DeprecationWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.DeprecationWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.DeprecationWarning.add_note" => "Add a note to the exception", + "builtins.DeprecationWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.DynamicClassAttribute" => "Route attribute access on a class to __getattr__.\n\nThis is a descriptor, used to define attributes that act differently when\naccessed through an instance and through a class. Instance access remains\nnormal, but access to an attribute through a class will be routed to the\nclass's __getattr__ method; this is done by raising AttributeError.\n\nThis allows one to have properties active on an instance, and have virtual\nattributes on the class with the same name. (Enum used this between Python\nversions 3.4 - 3.9 .)\n\nSubclass from this to use a different method of accessing virtual attributes\nand still be treated properly by the inspect module. (Enum uses this since\nPython 3.10 .)", "builtins.DynamicClassAttribute.__delattr__" => "Implement delattr(self, name).", "builtins.DynamicClassAttribute.__eq__" => "Return self==value.", @@ -4073,8 +10024,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.DynamicClassAttribute.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.DynamicClassAttribute.__weakref__" => "list of weak references to the object", "builtins.EOFError" => "Read beyond end of file.", - "builtins.EOFError.__cause__" => "exception cause", - "builtins.EOFError.__context__" => "exception context", "builtins.EOFError.__delattr__" => "Implement delattr(self, name).", "builtins.EOFError.__eq__" => "Return self==value.", "builtins.EOFError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4095,11 +10044,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.EOFError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.EOFError.__str__" => "Return str(self).", "builtins.EOFError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.EOFError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.EOFError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.EOFError.add_note" => "Add a note to the exception", + "builtins.EOFError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.EncodingWarning" => "Base class for warnings about encodings.", - "builtins.EncodingWarning.__cause__" => "exception cause", - "builtins.EncodingWarning.__context__" => "exception context", "builtins.EncodingWarning.__delattr__" => "Implement delattr(self, name).", "builtins.EncodingWarning.__eq__" => "Return self==value.", "builtins.EncodingWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4120,11 +10067,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.EncodingWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.EncodingWarning.__str__" => "Return str(self).", "builtins.EncodingWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.EncodingWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.EncodingWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.EncodingWarning.add_note" => "Add a note to the exception", + "builtins.EncodingWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.EnvironmentError" => "Base class for I/O related errors.", - "builtins.EnvironmentError.__cause__" => "exception cause", - "builtins.EnvironmentError.__context__" => "exception context", "builtins.EnvironmentError.__delattr__" => "Implement delattr(self, name).", "builtins.EnvironmentError.__eq__" => "Return self==value.", "builtins.EnvironmentError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4145,16 +10090,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.EnvironmentError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.EnvironmentError.__str__" => "Return str(self).", "builtins.EnvironmentError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.EnvironmentError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.EnvironmentError.add_note" => "Add a note to the exception", "builtins.EnvironmentError.errno" => "POSIX exception code", "builtins.EnvironmentError.filename" => "exception filename", "builtins.EnvironmentError.filename2" => "second exception filename", "builtins.EnvironmentError.strerror" => "exception strerror", "builtins.EnvironmentError.winerror" => "Win32 exception code", - "builtins.EnvironmentError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.EnvironmentError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.Exception" => "Common base class for all non-exit exceptions.", - "builtins.Exception.__cause__" => "exception cause", - "builtins.Exception.__context__" => "exception context", "builtins.Exception.__delattr__" => "Implement delattr(self, name).", "builtins.Exception.__eq__" => "Return self==value.", "builtins.Exception.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4175,11 +10118,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.Exception.__sizeof__" => "Size of object in memory, in bytes.", "builtins.Exception.__str__" => "Return str(self).", "builtins.Exception.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.Exception.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.Exception.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "builtins.ExceptionGroup.__cause__" => "exception cause", + "builtins.Exception.add_note" => "Add a note to the exception", + "builtins.Exception.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ExceptionGroup.__class_getitem__" => "See PEP 585", - "builtins.ExceptionGroup.__context__" => "exception context", "builtins.ExceptionGroup.__delattr__" => "Implement delattr(self, name).", "builtins.ExceptionGroup.__eq__" => "Return self==value.", "builtins.ExceptionGroup.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4201,13 +10142,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ExceptionGroup.__str__" => "Return str(self).", "builtins.ExceptionGroup.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.ExceptionGroup.__weakref__" => "list of weak references to the object", - "builtins.ExceptionGroup.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ExceptionGroup.add_note" => "Add a note to the exception", "builtins.ExceptionGroup.exceptions" => "nested exceptions", "builtins.ExceptionGroup.message" => "exception message", - "builtins.ExceptionGroup.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ExceptionGroup.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.FileExistsError" => "File already exists.", - "builtins.FileExistsError.__cause__" => "exception cause", - "builtins.FileExistsError.__context__" => "exception context", "builtins.FileExistsError.__delattr__" => "Implement delattr(self, name).", "builtins.FileExistsError.__eq__" => "Return self==value.", "builtins.FileExistsError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4228,16 +10167,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.FileExistsError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.FileExistsError.__str__" => "Return str(self).", "builtins.FileExistsError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.FileExistsError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.FileExistsError.add_note" => "Add a note to the exception", "builtins.FileExistsError.errno" => "POSIX exception code", "builtins.FileExistsError.filename" => "exception filename", "builtins.FileExistsError.filename2" => "second exception filename", "builtins.FileExistsError.strerror" => "exception strerror", "builtins.FileExistsError.winerror" => "Win32 exception code", - "builtins.FileExistsError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.FileExistsError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.FileNotFoundError" => "File not found.", - "builtins.FileNotFoundError.__cause__" => "exception cause", - "builtins.FileNotFoundError.__context__" => "exception context", "builtins.FileNotFoundError.__delattr__" => "Implement delattr(self, name).", "builtins.FileNotFoundError.__eq__" => "Return self==value.", "builtins.FileNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4258,16 +10195,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.FileNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.FileNotFoundError.__str__" => "Return str(self).", "builtins.FileNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.FileNotFoundError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.FileNotFoundError.add_note" => "Add a note to the exception", "builtins.FileNotFoundError.errno" => "POSIX exception code", "builtins.FileNotFoundError.filename" => "exception filename", "builtins.FileNotFoundError.filename2" => "second exception filename", "builtins.FileNotFoundError.strerror" => "exception strerror", "builtins.FileNotFoundError.winerror" => "Win32 exception code", - "builtins.FileNotFoundError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.FileNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.FloatingPointError" => "Floating-point operation failed.", - "builtins.FloatingPointError.__cause__" => "exception cause", - "builtins.FloatingPointError.__context__" => "exception context", "builtins.FloatingPointError.__delattr__" => "Implement delattr(self, name).", "builtins.FloatingPointError.__eq__" => "Return self==value.", "builtins.FloatingPointError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4288,11 +10223,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.FloatingPointError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.FloatingPointError.__str__" => "Return str(self).", "builtins.FloatingPointError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.FloatingPointError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.FloatingPointError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.FloatingPointError.add_note" => "Add a note to the exception", + "builtins.FloatingPointError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.FutureWarning" => "Base class for warnings about constructs that will change semantically\nin the future.", - "builtins.FutureWarning.__cause__" => "exception cause", - "builtins.FutureWarning.__context__" => "exception context", "builtins.FutureWarning.__delattr__" => "Implement delattr(self, name).", "builtins.FutureWarning.__eq__" => "Return self==value.", "builtins.FutureWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4313,11 +10246,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.FutureWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.FutureWarning.__str__" => "Return str(self).", "builtins.FutureWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.FutureWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.FutureWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.FutureWarning.add_note" => "Add a note to the exception", + "builtins.FutureWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.GeneratorExit" => "Request that a generator exit.", - "builtins.GeneratorExit.__cause__" => "exception cause", - "builtins.GeneratorExit.__context__" => "exception context", "builtins.GeneratorExit.__delattr__" => "Implement delattr(self, name).", "builtins.GeneratorExit.__eq__" => "Return self==value.", "builtins.GeneratorExit.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4338,8 +10269,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.GeneratorExit.__sizeof__" => "Size of object in memory, in bytes.", "builtins.GeneratorExit.__str__" => "Return str(self).", "builtins.GeneratorExit.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.GeneratorExit.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.GeneratorExit.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.GeneratorExit.add_note" => "Add a note to the exception", + "builtins.GeneratorExit.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.GenericAlias" => "Represent a PEP 585 generic type\n\nE.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).", "builtins.GenericAlias.__call__" => "Call self as a function.", "builtins.GenericAlias.__delattr__" => "Implement delattr(self, name).", @@ -4368,8 +10299,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.GenericAlias.__str__" => "Return str(self).", "builtins.GenericAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.IOError" => "Base class for I/O related errors.", - "builtins.IOError.__cause__" => "exception cause", - "builtins.IOError.__context__" => "exception context", "builtins.IOError.__delattr__" => "Implement delattr(self, name).", "builtins.IOError.__eq__" => "Return self==value.", "builtins.IOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4390,16 +10319,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.IOError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.IOError.__str__" => "Return str(self).", "builtins.IOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.IOError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.IOError.add_note" => "Add a note to the exception", "builtins.IOError.errno" => "POSIX exception code", "builtins.IOError.filename" => "exception filename", "builtins.IOError.filename2" => "second exception filename", "builtins.IOError.strerror" => "exception strerror", "builtins.IOError.winerror" => "Win32 exception code", - "builtins.IOError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.IOError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ImportError" => "Import can't find module, or can't find name in module.", - "builtins.ImportError.__cause__" => "exception cause", - "builtins.ImportError.__context__" => "exception context", "builtins.ImportError.__delattr__" => "Implement delattr(self, name).", "builtins.ImportError.__eq__" => "Return self==value.", "builtins.ImportError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4420,15 +10347,13 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ImportError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ImportError.__str__" => "Return str(self).", "builtins.ImportError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ImportError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ImportError.add_note" => "Add a note to the exception", "builtins.ImportError.msg" => "exception message", "builtins.ImportError.name" => "module name", "builtins.ImportError.name_from" => "name imported from module", "builtins.ImportError.path" => "module path", - "builtins.ImportError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ImportError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ImportWarning" => "Base class for warnings about probable mistakes in module imports", - "builtins.ImportWarning.__cause__" => "exception cause", - "builtins.ImportWarning.__context__" => "exception context", "builtins.ImportWarning.__delattr__" => "Implement delattr(self, name).", "builtins.ImportWarning.__eq__" => "Return self==value.", "builtins.ImportWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4449,11 +10374,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ImportWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ImportWarning.__str__" => "Return str(self).", "builtins.ImportWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ImportWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ImportWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ImportWarning.add_note" => "Add a note to the exception", + "builtins.ImportWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.IndentationError" => "Improper indentation.", - "builtins.IndentationError.__cause__" => "exception cause", - "builtins.IndentationError.__context__" => "exception context", "builtins.IndentationError.__delattr__" => "Implement delattr(self, name).", "builtins.IndentationError.__eq__" => "Return self==value.", "builtins.IndentationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4474,7 +10397,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.IndentationError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.IndentationError.__str__" => "Return str(self).", "builtins.IndentationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.IndentationError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.IndentationError._metadata" => "exception private metadata", + "builtins.IndentationError.add_note" => "Add a note to the exception", "builtins.IndentationError.end_lineno" => "exception end lineno", "builtins.IndentationError.end_offset" => "exception end offset", "builtins.IndentationError.filename" => "exception filename", @@ -4483,10 +10407,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.IndentationError.offset" => "exception offset", "builtins.IndentationError.print_file_and_line" => "exception print_file_and_line", "builtins.IndentationError.text" => "exception text", - "builtins.IndentationError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.IndentationError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.IndexError" => "Sequence index out of range.", - "builtins.IndexError.__cause__" => "exception cause", - "builtins.IndexError.__context__" => "exception context", "builtins.IndexError.__delattr__" => "Implement delattr(self, name).", "builtins.IndexError.__eq__" => "Return self==value.", "builtins.IndexError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4507,11 +10429,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.IndexError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.IndexError.__str__" => "Return str(self).", "builtins.IndexError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.IndexError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.IndexError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.IndexError.add_note" => "Add a note to the exception", + "builtins.IndexError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.InterruptedError" => "Interrupted by signal.", - "builtins.InterruptedError.__cause__" => "exception cause", - "builtins.InterruptedError.__context__" => "exception context", "builtins.InterruptedError.__delattr__" => "Implement delattr(self, name).", "builtins.InterruptedError.__eq__" => "Return self==value.", "builtins.InterruptedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4532,16 +10452,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.InterruptedError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.InterruptedError.__str__" => "Return str(self).", "builtins.InterruptedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.InterruptedError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.InterruptedError.add_note" => "Add a note to the exception", "builtins.InterruptedError.errno" => "POSIX exception code", "builtins.InterruptedError.filename" => "exception filename", "builtins.InterruptedError.filename2" => "second exception filename", "builtins.InterruptedError.strerror" => "exception strerror", "builtins.InterruptedError.winerror" => "Win32 exception code", - "builtins.InterruptedError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.InterruptedError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.IsADirectoryError" => "Operation doesn't work on directories.", - "builtins.IsADirectoryError.__cause__" => "exception cause", - "builtins.IsADirectoryError.__context__" => "exception context", "builtins.IsADirectoryError.__delattr__" => "Implement delattr(self, name).", "builtins.IsADirectoryError.__eq__" => "Return self==value.", "builtins.IsADirectoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4562,16 +10480,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.IsADirectoryError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.IsADirectoryError.__str__" => "Return str(self).", "builtins.IsADirectoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.IsADirectoryError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.IsADirectoryError.add_note" => "Add a note to the exception", "builtins.IsADirectoryError.errno" => "POSIX exception code", "builtins.IsADirectoryError.filename" => "exception filename", "builtins.IsADirectoryError.filename2" => "second exception filename", "builtins.IsADirectoryError.strerror" => "exception strerror", "builtins.IsADirectoryError.winerror" => "Win32 exception code", - "builtins.IsADirectoryError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.IsADirectoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.KeyError" => "Mapping key not found.", - "builtins.KeyError.__cause__" => "exception cause", - "builtins.KeyError.__context__" => "exception context", "builtins.KeyError.__delattr__" => "Implement delattr(self, name).", "builtins.KeyError.__eq__" => "Return self==value.", "builtins.KeyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4592,11 +10508,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.KeyError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.KeyError.__str__" => "Return str(self).", "builtins.KeyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.KeyError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.KeyError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.KeyError.add_note" => "Add a note to the exception", + "builtins.KeyError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.KeyboardInterrupt" => "Program interrupted by user.", - "builtins.KeyboardInterrupt.__cause__" => "exception cause", - "builtins.KeyboardInterrupt.__context__" => "exception context", "builtins.KeyboardInterrupt.__delattr__" => "Implement delattr(self, name).", "builtins.KeyboardInterrupt.__eq__" => "Return self==value.", "builtins.KeyboardInterrupt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4617,11 +10531,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.KeyboardInterrupt.__sizeof__" => "Size of object in memory, in bytes.", "builtins.KeyboardInterrupt.__str__" => "Return str(self).", "builtins.KeyboardInterrupt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.KeyboardInterrupt.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.KeyboardInterrupt.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.KeyboardInterrupt.add_note" => "Add a note to the exception", + "builtins.KeyboardInterrupt.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.LookupError" => "Base class for lookup errors.", - "builtins.LookupError.__cause__" => "exception cause", - "builtins.LookupError.__context__" => "exception context", "builtins.LookupError.__delattr__" => "Implement delattr(self, name).", "builtins.LookupError.__eq__" => "Return self==value.", "builtins.LookupError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4642,11 +10554,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.LookupError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.LookupError.__str__" => "Return str(self).", "builtins.LookupError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.LookupError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.LookupError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.LookupError.add_note" => "Add a note to the exception", + "builtins.LookupError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.MemoryError" => "Out of memory.", - "builtins.MemoryError.__cause__" => "exception cause", - "builtins.MemoryError.__context__" => "exception context", "builtins.MemoryError.__delattr__" => "Implement delattr(self, name).", "builtins.MemoryError.__eq__" => "Return self==value.", "builtins.MemoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4667,11 +10577,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.MemoryError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.MemoryError.__str__" => "Return str(self).", "builtins.MemoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.MemoryError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.MemoryError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.MemoryError.add_note" => "Add a note to the exception", + "builtins.MemoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ModuleNotFoundError" => "Module not found.", - "builtins.ModuleNotFoundError.__cause__" => "exception cause", - "builtins.ModuleNotFoundError.__context__" => "exception context", "builtins.ModuleNotFoundError.__delattr__" => "Implement delattr(self, name).", "builtins.ModuleNotFoundError.__eq__" => "Return self==value.", "builtins.ModuleNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4692,15 +10600,13 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ModuleNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ModuleNotFoundError.__str__" => "Return str(self).", "builtins.ModuleNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ModuleNotFoundError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ModuleNotFoundError.add_note" => "Add a note to the exception", "builtins.ModuleNotFoundError.msg" => "exception message", "builtins.ModuleNotFoundError.name" => "module name", "builtins.ModuleNotFoundError.name_from" => "name imported from module", "builtins.ModuleNotFoundError.path" => "module path", - "builtins.ModuleNotFoundError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ModuleNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.NameError" => "Name not found globally.", - "builtins.NameError.__cause__" => "exception cause", - "builtins.NameError.__context__" => "exception context", "builtins.NameError.__delattr__" => "Implement delattr(self, name).", "builtins.NameError.__eq__" => "Return self==value.", "builtins.NameError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4721,9 +10627,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.NameError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.NameError.__str__" => "Return str(self).", "builtins.NameError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.NameError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.NameError.add_note" => "Add a note to the exception", "builtins.NameError.name" => "name", - "builtins.NameError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.NameError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.NoneType" => "The type of the None singleton.", "builtins.NoneType.__bool__" => "True if self else False", "builtins.NoneType.__delattr__" => "Implement delattr(self, name).", @@ -4748,8 +10654,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.NoneType.__str__" => "Return str(self).", "builtins.NoneType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.NotADirectoryError" => "Operation only works on directories.", - "builtins.NotADirectoryError.__cause__" => "exception cause", - "builtins.NotADirectoryError.__context__" => "exception context", "builtins.NotADirectoryError.__delattr__" => "Implement delattr(self, name).", "builtins.NotADirectoryError.__eq__" => "Return self==value.", "builtins.NotADirectoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4770,16 +10674,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.NotADirectoryError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.NotADirectoryError.__str__" => "Return str(self).", "builtins.NotADirectoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.NotADirectoryError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.NotADirectoryError.add_note" => "Add a note to the exception", "builtins.NotADirectoryError.errno" => "POSIX exception code", "builtins.NotADirectoryError.filename" => "exception filename", "builtins.NotADirectoryError.filename2" => "second exception filename", "builtins.NotADirectoryError.strerror" => "exception strerror", "builtins.NotADirectoryError.winerror" => "Win32 exception code", - "builtins.NotADirectoryError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.NotADirectoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.NotImplementedError" => "Method or function hasn't been implemented yet.", - "builtins.NotImplementedError.__cause__" => "exception cause", - "builtins.NotImplementedError.__context__" => "exception context", "builtins.NotImplementedError.__delattr__" => "Implement delattr(self, name).", "builtins.NotImplementedError.__eq__" => "Return self==value.", "builtins.NotImplementedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4800,8 +10702,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.NotImplementedError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.NotImplementedError.__str__" => "Return str(self).", "builtins.NotImplementedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.NotImplementedError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.NotImplementedError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.NotImplementedError.add_note" => "Add a note to the exception", + "builtins.NotImplementedError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.NotImplementedType" => "The type of the NotImplemented singleton.", "builtins.NotImplementedType.__bool__" => "True if self else False", "builtins.NotImplementedType.__delattr__" => "Implement delattr(self, name).", @@ -4825,8 +10727,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.NotImplementedType.__str__" => "Return str(self).", "builtins.NotImplementedType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.OSError" => "Base class for I/O related errors.", - "builtins.OSError.__cause__" => "exception cause", - "builtins.OSError.__context__" => "exception context", "builtins.OSError.__delattr__" => "Implement delattr(self, name).", "builtins.OSError.__eq__" => "Return self==value.", "builtins.OSError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4847,16 +10747,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.OSError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.OSError.__str__" => "Return str(self).", "builtins.OSError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.OSError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.OSError.add_note" => "Add a note to the exception", "builtins.OSError.errno" => "POSIX exception code", "builtins.OSError.filename" => "exception filename", "builtins.OSError.filename2" => "second exception filename", "builtins.OSError.strerror" => "exception strerror", "builtins.OSError.winerror" => "Win32 exception code", - "builtins.OSError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.OSError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.OverflowError" => "Result too large to be represented.", - "builtins.OverflowError.__cause__" => "exception cause", - "builtins.OverflowError.__context__" => "exception context", "builtins.OverflowError.__delattr__" => "Implement delattr(self, name).", "builtins.OverflowError.__eq__" => "Return self==value.", "builtins.OverflowError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4877,11 +10775,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.OverflowError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.OverflowError.__str__" => "Return str(self).", "builtins.OverflowError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.OverflowError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.OverflowError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.OverflowError.add_note" => "Add a note to the exception", + "builtins.OverflowError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.PendingDeprecationWarning" => "Base class for warnings about features which will be deprecated\nin the future.", - "builtins.PendingDeprecationWarning.__cause__" => "exception cause", - "builtins.PendingDeprecationWarning.__context__" => "exception context", "builtins.PendingDeprecationWarning.__delattr__" => "Implement delattr(self, name).", "builtins.PendingDeprecationWarning.__eq__" => "Return self==value.", "builtins.PendingDeprecationWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4902,11 +10798,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.PendingDeprecationWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.PendingDeprecationWarning.__str__" => "Return str(self).", "builtins.PendingDeprecationWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.PendingDeprecationWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.PendingDeprecationWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.PendingDeprecationWarning.add_note" => "Add a note to the exception", + "builtins.PendingDeprecationWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.PermissionError" => "Not enough permissions.", - "builtins.PermissionError.__cause__" => "exception cause", - "builtins.PermissionError.__context__" => "exception context", "builtins.PermissionError.__delattr__" => "Implement delattr(self, name).", "builtins.PermissionError.__eq__" => "Return self==value.", "builtins.PermissionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4927,16 +10821,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.PermissionError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.PermissionError.__str__" => "Return str(self).", "builtins.PermissionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.PermissionError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.PermissionError.add_note" => "Add a note to the exception", "builtins.PermissionError.errno" => "POSIX exception code", "builtins.PermissionError.filename" => "exception filename", "builtins.PermissionError.filename2" => "second exception filename", "builtins.PermissionError.strerror" => "exception strerror", "builtins.PermissionError.winerror" => "Win32 exception code", - "builtins.PermissionError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.PermissionError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ProcessLookupError" => "Process not found.", - "builtins.ProcessLookupError.__cause__" => "exception cause", - "builtins.ProcessLookupError.__context__" => "exception context", "builtins.ProcessLookupError.__delattr__" => "Implement delattr(self, name).", "builtins.ProcessLookupError.__eq__" => "Return self==value.", "builtins.ProcessLookupError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4957,16 +10849,36 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ProcessLookupError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ProcessLookupError.__str__" => "Return str(self).", "builtins.ProcessLookupError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ProcessLookupError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.ProcessLookupError.add_note" => "Add a note to the exception", "builtins.ProcessLookupError.errno" => "POSIX exception code", "builtins.ProcessLookupError.filename" => "exception filename", "builtins.ProcessLookupError.filename2" => "second exception filename", "builtins.ProcessLookupError.strerror" => "exception strerror", "builtins.ProcessLookupError.winerror" => "Win32 exception code", - "builtins.ProcessLookupError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ProcessLookupError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.PyCapsule" => "Capsule objects let you wrap a C \"void *\" pointer in a Python\nobject. They're a way of passing data through the Python interpreter\nwithout creating your own custom type.\n\nCapsules are used for communication between extension modules.\nThey provide a way for an extension module to export a C interface\nto other extension modules, so that extension modules can use the\nPython import mechanism to link to one another.", + "builtins.PyCapsule.__delattr__" => "Implement delattr(self, name).", + "builtins.PyCapsule.__eq__" => "Return self==value.", + "builtins.PyCapsule.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.PyCapsule.__ge__" => "Return self>=value.", + "builtins.PyCapsule.__getattribute__" => "Return getattr(self, name).", + "builtins.PyCapsule.__getstate__" => "Helper for pickle.", + "builtins.PyCapsule.__gt__" => "Return self>value.", + "builtins.PyCapsule.__hash__" => "Return hash(self).", + "builtins.PyCapsule.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.PyCapsule.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.PyCapsule.__le__" => "Return self<=value.", + "builtins.PyCapsule.__lt__" => "Return self<value.", + "builtins.PyCapsule.__ne__" => "Return self!=value.", + "builtins.PyCapsule.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.PyCapsule.__reduce__" => "Helper for pickle.", + "builtins.PyCapsule.__reduce_ex__" => "Helper for pickle.", + "builtins.PyCapsule.__repr__" => "Return repr(self).", + "builtins.PyCapsule.__setattr__" => "Implement setattr(self, name, value).", + "builtins.PyCapsule.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.PyCapsule.__str__" => "Return str(self).", + "builtins.PyCapsule.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.PythonFinalizationError" => "Operation blocked during Python finalization.", - "builtins.PythonFinalizationError.__cause__" => "exception cause", - "builtins.PythonFinalizationError.__context__" => "exception context", "builtins.PythonFinalizationError.__delattr__" => "Implement delattr(self, name).", "builtins.PythonFinalizationError.__eq__" => "Return self==value.", "builtins.PythonFinalizationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -4987,11 +10899,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.PythonFinalizationError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.PythonFinalizationError.__str__" => "Return str(self).", "builtins.PythonFinalizationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.PythonFinalizationError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.PythonFinalizationError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.PythonFinalizationError.add_note" => "Add a note to the exception", + "builtins.PythonFinalizationError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.RecursionError" => "Recursion limit exceeded.", - "builtins.RecursionError.__cause__" => "exception cause", - "builtins.RecursionError.__context__" => "exception context", "builtins.RecursionError.__delattr__" => "Implement delattr(self, name).", "builtins.RecursionError.__eq__" => "Return self==value.", "builtins.RecursionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5012,11 +10922,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.RecursionError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.RecursionError.__str__" => "Return str(self).", "builtins.RecursionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.RecursionError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.RecursionError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.RecursionError.add_note" => "Add a note to the exception", + "builtins.RecursionError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ReferenceError" => "Weak ref proxy used after referent went away.", - "builtins.ReferenceError.__cause__" => "exception cause", - "builtins.ReferenceError.__context__" => "exception context", "builtins.ReferenceError.__delattr__" => "Implement delattr(self, name).", "builtins.ReferenceError.__eq__" => "Return self==value.", "builtins.ReferenceError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5037,11 +10945,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ReferenceError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ReferenceError.__str__" => "Return str(self).", "builtins.ReferenceError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ReferenceError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ReferenceError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ReferenceError.add_note" => "Add a note to the exception", + "builtins.ReferenceError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ResourceWarning" => "Base class for warnings about resource usage.", - "builtins.ResourceWarning.__cause__" => "exception cause", - "builtins.ResourceWarning.__context__" => "exception context", "builtins.ResourceWarning.__delattr__" => "Implement delattr(self, name).", "builtins.ResourceWarning.__eq__" => "Return self==value.", "builtins.ResourceWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5062,11 +10968,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ResourceWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ResourceWarning.__str__" => "Return str(self).", "builtins.ResourceWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ResourceWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ResourceWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ResourceWarning.add_note" => "Add a note to the exception", + "builtins.ResourceWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.RuntimeError" => "Unspecified run-time error.", - "builtins.RuntimeError.__cause__" => "exception cause", - "builtins.RuntimeError.__context__" => "exception context", "builtins.RuntimeError.__delattr__" => "Implement delattr(self, name).", "builtins.RuntimeError.__eq__" => "Return self==value.", "builtins.RuntimeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5087,11 +10991,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.RuntimeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.RuntimeError.__str__" => "Return str(self).", "builtins.RuntimeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.RuntimeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.RuntimeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.RuntimeError.add_note" => "Add a note to the exception", + "builtins.RuntimeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.RuntimeWarning" => "Base class for warnings about dubious runtime behavior.", - "builtins.RuntimeWarning.__cause__" => "exception cause", - "builtins.RuntimeWarning.__context__" => "exception context", "builtins.RuntimeWarning.__delattr__" => "Implement delattr(self, name).", "builtins.RuntimeWarning.__eq__" => "Return self==value.", "builtins.RuntimeWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5112,8 +11014,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.RuntimeWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.RuntimeWarning.__str__" => "Return str(self).", "builtins.RuntimeWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.RuntimeWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.RuntimeWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.RuntimeWarning.add_note" => "Add a note to the exception", + "builtins.RuntimeWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.SimpleNamespace" => "A simple attribute-based namespace.", "builtins.SimpleNamespace.__delattr__" => "Implement delattr(self, name).", "builtins.SimpleNamespace.__eq__" => "Return self==value.", @@ -5137,8 +11039,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SimpleNamespace.__str__" => "Return str(self).", "builtins.SimpleNamespace.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.StopAsyncIteration" => "Signal the end from iterator.__anext__().", - "builtins.StopAsyncIteration.__cause__" => "exception cause", - "builtins.StopAsyncIteration.__context__" => "exception context", "builtins.StopAsyncIteration.__delattr__" => "Implement delattr(self, name).", "builtins.StopAsyncIteration.__eq__" => "Return self==value.", "builtins.StopAsyncIteration.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5159,11 +11059,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.StopAsyncIteration.__sizeof__" => "Size of object in memory, in bytes.", "builtins.StopAsyncIteration.__str__" => "Return str(self).", "builtins.StopAsyncIteration.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.StopAsyncIteration.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.StopAsyncIteration.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.StopAsyncIteration.add_note" => "Add a note to the exception", + "builtins.StopAsyncIteration.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.StopIteration" => "Signal the end from iterator.__next__().", - "builtins.StopIteration.__cause__" => "exception cause", - "builtins.StopIteration.__context__" => "exception context", "builtins.StopIteration.__delattr__" => "Implement delattr(self, name).", "builtins.StopIteration.__eq__" => "Return self==value.", "builtins.StopIteration.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5184,12 +11082,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.StopIteration.__sizeof__" => "Size of object in memory, in bytes.", "builtins.StopIteration.__str__" => "Return str(self).", "builtins.StopIteration.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.StopIteration.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.StopIteration.add_note" => "Add a note to the exception", "builtins.StopIteration.value" => "generator return value", - "builtins.StopIteration.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.StopIteration.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.SyntaxError" => "Invalid syntax.", - "builtins.SyntaxError.__cause__" => "exception cause", - "builtins.SyntaxError.__context__" => "exception context", "builtins.SyntaxError.__delattr__" => "Implement delattr(self, name).", "builtins.SyntaxError.__eq__" => "Return self==value.", "builtins.SyntaxError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5210,7 +11106,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SyntaxError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.SyntaxError.__str__" => "Return str(self).", "builtins.SyntaxError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.SyntaxError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.SyntaxError._metadata" => "exception private metadata", + "builtins.SyntaxError.add_note" => "Add a note to the exception", "builtins.SyntaxError.end_lineno" => "exception end lineno", "builtins.SyntaxError.end_offset" => "exception end offset", "builtins.SyntaxError.filename" => "exception filename", @@ -5219,10 +11116,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SyntaxError.offset" => "exception offset", "builtins.SyntaxError.print_file_and_line" => "exception print_file_and_line", "builtins.SyntaxError.text" => "exception text", - "builtins.SyntaxError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.SyntaxError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.SyntaxWarning" => "Base class for warnings about dubious syntax.", - "builtins.SyntaxWarning.__cause__" => "exception cause", - "builtins.SyntaxWarning.__context__" => "exception context", "builtins.SyntaxWarning.__delattr__" => "Implement delattr(self, name).", "builtins.SyntaxWarning.__eq__" => "Return self==value.", "builtins.SyntaxWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5243,11 +11138,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SyntaxWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.SyntaxWarning.__str__" => "Return str(self).", "builtins.SyntaxWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.SyntaxWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.SyntaxWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.SyntaxWarning.add_note" => "Add a note to the exception", + "builtins.SyntaxWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.SystemError" => "Internal error in the Python interpreter.\n\nPlease report this to the Python maintainer, along with the traceback,\nthe Python version, and the hardware/OS platform and version.", - "builtins.SystemError.__cause__" => "exception cause", - "builtins.SystemError.__context__" => "exception context", "builtins.SystemError.__delattr__" => "Implement delattr(self, name).", "builtins.SystemError.__eq__" => "Return self==value.", "builtins.SystemError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5268,11 +11161,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SystemError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.SystemError.__str__" => "Return str(self).", "builtins.SystemError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.SystemError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.SystemError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.SystemError.add_note" => "Add a note to the exception", + "builtins.SystemError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.SystemExit" => "Request to exit from the interpreter.", - "builtins.SystemExit.__cause__" => "exception cause", - "builtins.SystemExit.__context__" => "exception context", "builtins.SystemExit.__delattr__" => "Implement delattr(self, name).", "builtins.SystemExit.__eq__" => "Return self==value.", "builtins.SystemExit.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5293,12 +11184,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.SystemExit.__sizeof__" => "Size of object in memory, in bytes.", "builtins.SystemExit.__str__" => "Return str(self).", "builtins.SystemExit.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.SystemExit.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.SystemExit.add_note" => "Add a note to the exception", "builtins.SystemExit.code" => "exception code", - "builtins.SystemExit.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.SystemExit.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.TabError" => "Improper mixture of spaces and tabs.", - "builtins.TabError.__cause__" => "exception cause", - "builtins.TabError.__context__" => "exception context", "builtins.TabError.__delattr__" => "Implement delattr(self, name).", "builtins.TabError.__eq__" => "Return self==value.", "builtins.TabError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5319,7 +11208,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.TabError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.TabError.__str__" => "Return str(self).", "builtins.TabError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.TabError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.TabError._metadata" => "exception private metadata", + "builtins.TabError.add_note" => "Add a note to the exception", "builtins.TabError.end_lineno" => "exception end lineno", "builtins.TabError.end_offset" => "exception end offset", "builtins.TabError.filename" => "exception filename", @@ -5328,10 +11218,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.TabError.offset" => "exception offset", "builtins.TabError.print_file_and_line" => "exception print_file_and_line", "builtins.TabError.text" => "exception text", - "builtins.TabError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.TabError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.TimeoutError" => "Timeout expired.", - "builtins.TimeoutError.__cause__" => "exception cause", - "builtins.TimeoutError.__context__" => "exception context", "builtins.TimeoutError.__delattr__" => "Implement delattr(self, name).", "builtins.TimeoutError.__eq__" => "Return self==value.", "builtins.TimeoutError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5352,16 +11240,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.TimeoutError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.TimeoutError.__str__" => "Return str(self).", "builtins.TimeoutError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.TimeoutError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.TimeoutError.add_note" => "Add a note to the exception", "builtins.TimeoutError.errno" => "POSIX exception code", "builtins.TimeoutError.filename" => "exception filename", "builtins.TimeoutError.filename2" => "second exception filename", "builtins.TimeoutError.strerror" => "exception strerror", "builtins.TimeoutError.winerror" => "Win32 exception code", - "builtins.TimeoutError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.TimeoutError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.TypeError" => "Inappropriate argument type.", - "builtins.TypeError.__cause__" => "exception cause", - "builtins.TypeError.__context__" => "exception context", "builtins.TypeError.__delattr__" => "Implement delattr(self, name).", "builtins.TypeError.__eq__" => "Return self==value.", "builtins.TypeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5382,11 +11268,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.TypeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.TypeError.__str__" => "Return str(self).", "builtins.TypeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.TypeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.TypeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.TypeError.add_note" => "Add a note to the exception", + "builtins.TypeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnboundLocalError" => "Local name referenced but not bound to a value.", - "builtins.UnboundLocalError.__cause__" => "exception cause", - "builtins.UnboundLocalError.__context__" => "exception context", "builtins.UnboundLocalError.__delattr__" => "Implement delattr(self, name).", "builtins.UnboundLocalError.__eq__" => "Return self==value.", "builtins.UnboundLocalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5407,12 +11291,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnboundLocalError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnboundLocalError.__str__" => "Return str(self).", "builtins.UnboundLocalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnboundLocalError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.UnboundLocalError.add_note" => "Add a note to the exception", "builtins.UnboundLocalError.name" => "name", - "builtins.UnboundLocalError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UnboundLocalError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnicodeDecodeError" => "Unicode decoding error.", - "builtins.UnicodeDecodeError.__cause__" => "exception cause", - "builtins.UnicodeDecodeError.__context__" => "exception context", "builtins.UnicodeDecodeError.__delattr__" => "Implement delattr(self, name).", "builtins.UnicodeDecodeError.__eq__" => "Return self==value.", "builtins.UnicodeDecodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5433,16 +11315,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnicodeDecodeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnicodeDecodeError.__str__" => "Return str(self).", "builtins.UnicodeDecodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnicodeDecodeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.UnicodeDecodeError.add_note" => "Add a note to the exception", "builtins.UnicodeDecodeError.encoding" => "exception encoding", "builtins.UnicodeDecodeError.end" => "exception end", "builtins.UnicodeDecodeError.object" => "exception object", "builtins.UnicodeDecodeError.reason" => "exception reason", "builtins.UnicodeDecodeError.start" => "exception start", - "builtins.UnicodeDecodeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UnicodeDecodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnicodeEncodeError" => "Unicode encoding error.", - "builtins.UnicodeEncodeError.__cause__" => "exception cause", - "builtins.UnicodeEncodeError.__context__" => "exception context", "builtins.UnicodeEncodeError.__delattr__" => "Implement delattr(self, name).", "builtins.UnicodeEncodeError.__eq__" => "Return self==value.", "builtins.UnicodeEncodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5463,16 +11343,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnicodeEncodeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnicodeEncodeError.__str__" => "Return str(self).", "builtins.UnicodeEncodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnicodeEncodeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.UnicodeEncodeError.add_note" => "Add a note to the exception", "builtins.UnicodeEncodeError.encoding" => "exception encoding", "builtins.UnicodeEncodeError.end" => "exception end", "builtins.UnicodeEncodeError.object" => "exception object", "builtins.UnicodeEncodeError.reason" => "exception reason", "builtins.UnicodeEncodeError.start" => "exception start", - "builtins.UnicodeEncodeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UnicodeEncodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnicodeError" => "Unicode related error.", - "builtins.UnicodeError.__cause__" => "exception cause", - "builtins.UnicodeError.__context__" => "exception context", "builtins.UnicodeError.__delattr__" => "Implement delattr(self, name).", "builtins.UnicodeError.__eq__" => "Return self==value.", "builtins.UnicodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5493,11 +11371,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnicodeError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnicodeError.__str__" => "Return str(self).", "builtins.UnicodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnicodeError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.UnicodeError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UnicodeError.add_note" => "Add a note to the exception", + "builtins.UnicodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnicodeTranslateError" => "Unicode translation error.", - "builtins.UnicodeTranslateError.__cause__" => "exception cause", - "builtins.UnicodeTranslateError.__context__" => "exception context", "builtins.UnicodeTranslateError.__delattr__" => "Implement delattr(self, name).", "builtins.UnicodeTranslateError.__eq__" => "Return self==value.", "builtins.UnicodeTranslateError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5518,16 +11394,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnicodeTranslateError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnicodeTranslateError.__str__" => "Return str(self).", "builtins.UnicodeTranslateError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnicodeTranslateError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.UnicodeTranslateError.add_note" => "Add a note to the exception", "builtins.UnicodeTranslateError.encoding" => "exception encoding", "builtins.UnicodeTranslateError.end" => "exception end", "builtins.UnicodeTranslateError.object" => "exception object", "builtins.UnicodeTranslateError.reason" => "exception reason", "builtins.UnicodeTranslateError.start" => "exception start", - "builtins.UnicodeTranslateError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UnicodeTranslateError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.UnicodeWarning" => "Base class for warnings about Unicode related problems, mostly\nrelated to conversion problems.", - "builtins.UnicodeWarning.__cause__" => "exception cause", - "builtins.UnicodeWarning.__context__" => "exception context", "builtins.UnicodeWarning.__delattr__" => "Implement delattr(self, name).", "builtins.UnicodeWarning.__eq__" => "Return self==value.", "builtins.UnicodeWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5548,37 +11422,37 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UnicodeWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UnicodeWarning.__str__" => "Return str(self).", "builtins.UnicodeWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UnicodeWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.UnicodeWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", - "builtins.UnionType" => "Represent a PEP 604 union type\n\nE.g. for int | str", - "builtins.UnionType.__delattr__" => "Implement delattr(self, name).", - "builtins.UnionType.__eq__" => "Return self==value.", - "builtins.UnionType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", - "builtins.UnionType.__ge__" => "Return self>=value.", - "builtins.UnionType.__getattribute__" => "Return getattr(self, name).", - "builtins.UnionType.__getitem__" => "Return self[key].", - "builtins.UnionType.__getstate__" => "Helper for pickle.", - "builtins.UnionType.__gt__" => "Return self>value.", - "builtins.UnionType.__hash__" => "Return hash(self).", - "builtins.UnionType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", - "builtins.UnionType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", - "builtins.UnionType.__le__" => "Return self<=value.", - "builtins.UnionType.__lt__" => "Return self<value.", - "builtins.UnionType.__ne__" => "Return self!=value.", - "builtins.UnionType.__new__" => "Create and return a new object. See help(type) for accurate signature.", - "builtins.UnionType.__or__" => "Return self|value.", - "builtins.UnionType.__parameters__" => "Type variables in the types.UnionType.", - "builtins.UnionType.__reduce__" => "Helper for pickle.", - "builtins.UnionType.__reduce_ex__" => "Helper for pickle.", - "builtins.UnionType.__repr__" => "Return repr(self).", - "builtins.UnionType.__ror__" => "Return value|self.", - "builtins.UnionType.__setattr__" => "Implement setattr(self, name, value).", - "builtins.UnionType.__sizeof__" => "Size of object in memory, in bytes.", - "builtins.UnionType.__str__" => "Return str(self).", - "builtins.UnionType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeWarning.add_note" => "Add a note to the exception", + "builtins.UnicodeWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.Union" => "Represent a union type\n\nE.g. for int | str", + "builtins.Union.__class_getitem__" => "See PEP 585", + "builtins.Union.__delattr__" => "Implement delattr(self, name).", + "builtins.Union.__eq__" => "Return self==value.", + "builtins.Union.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.Union.__ge__" => "Return self>=value.", + "builtins.Union.__getattribute__" => "Return getattr(self, name).", + "builtins.Union.__getitem__" => "Return self[key].", + "builtins.Union.__getstate__" => "Helper for pickle.", + "builtins.Union.__gt__" => "Return self>value.", + "builtins.Union.__hash__" => "Return hash(self).", + "builtins.Union.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.Union.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.Union.__le__" => "Return self<=value.", + "builtins.Union.__lt__" => "Return self<value.", + "builtins.Union.__ne__" => "Return self!=value.", + "builtins.Union.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.Union.__or__" => "Return self|value.", + "builtins.Union.__origin__" => "Always returns the type", + "builtins.Union.__parameters__" => "Type variables in the types.UnionType.", + "builtins.Union.__reduce__" => "Helper for pickle.", + "builtins.Union.__reduce_ex__" => "Helper for pickle.", + "builtins.Union.__repr__" => "Return repr(self).", + "builtins.Union.__ror__" => "Return value|self.", + "builtins.Union.__setattr__" => "Implement setattr(self, name, value).", + "builtins.Union.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.Union.__str__" => "Return str(self).", + "builtins.Union.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.UserWarning" => "Base class for warnings generated by user code.", - "builtins.UserWarning.__cause__" => "exception cause", - "builtins.UserWarning.__context__" => "exception context", "builtins.UserWarning.__delattr__" => "Implement delattr(self, name).", "builtins.UserWarning.__eq__" => "Return self==value.", "builtins.UserWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5599,11 +11473,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.UserWarning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.UserWarning.__str__" => "Return str(self).", "builtins.UserWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.UserWarning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.UserWarning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.UserWarning.add_note" => "Add a note to the exception", + "builtins.UserWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ValueError" => "Inappropriate argument value (of correct type).", - "builtins.ValueError.__cause__" => "exception cause", - "builtins.ValueError.__context__" => "exception context", "builtins.ValueError.__delattr__" => "Implement delattr(self, name).", "builtins.ValueError.__eq__" => "Return self==value.", "builtins.ValueError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5624,11 +11496,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ValueError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ValueError.__str__" => "Return str(self).", "builtins.ValueError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ValueError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ValueError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ValueError.add_note" => "Add a note to the exception", + "builtins.ValueError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.Warning" => "Base class for warning categories.", - "builtins.Warning.__cause__" => "exception cause", - "builtins.Warning.__context__" => "exception context", "builtins.Warning.__delattr__" => "Implement delattr(self, name).", "builtins.Warning.__eq__" => "Return self==value.", "builtins.Warning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5649,11 +11519,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.Warning.__sizeof__" => "Size of object in memory, in bytes.", "builtins.Warning.__str__" => "Return str(self).", "builtins.Warning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.Warning.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.Warning.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.Warning.add_note" => "Add a note to the exception", + "builtins.Warning.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.WindowsError" => "Base class for I/O related errors.", - "builtins.WindowsError.__cause__" => "exception cause", - "builtins.WindowsError.__context__" => "exception context", "builtins.WindowsError.__delattr__" => "Implement delattr(self, name).", "builtins.WindowsError.__eq__" => "Return self==value.", "builtins.WindowsError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5674,16 +11542,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.WindowsError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.WindowsError.__str__" => "Return str(self).", "builtins.WindowsError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.WindowsError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins.WindowsError.add_note" => "Add a note to the exception", "builtins.WindowsError.errno" => "POSIX exception code", "builtins.WindowsError.filename" => "exception filename", "builtins.WindowsError.filename2" => "second exception filename", "builtins.WindowsError.strerror" => "exception strerror", "builtins.WindowsError.winerror" => "Win32 exception code", - "builtins.WindowsError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.WindowsError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.ZeroDivisionError" => "Second argument to a division or modulo operation was zero.", - "builtins.ZeroDivisionError.__cause__" => "exception cause", - "builtins.ZeroDivisionError.__context__" => "exception context", "builtins.ZeroDivisionError.__delattr__" => "Implement delattr(self, name).", "builtins.ZeroDivisionError.__eq__" => "Return self==value.", "builtins.ZeroDivisionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5704,11 +11570,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.ZeroDivisionError.__sizeof__" => "Size of object in memory, in bytes.", "builtins.ZeroDivisionError.__str__" => "Return str(self).", "builtins.ZeroDivisionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.ZeroDivisionError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "builtins.ZeroDivisionError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins.ZeroDivisionError.add_note" => "Add a note to the exception", + "builtins.ZeroDivisionError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins._IncompleteInputError" => "incomplete input.", - "builtins._IncompleteInputError.__cause__" => "exception cause", - "builtins._IncompleteInputError.__context__" => "exception context", "builtins._IncompleteInputError.__delattr__" => "Implement delattr(self, name).", "builtins._IncompleteInputError.__eq__" => "Return self==value.", "builtins._IncompleteInputError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -5729,7 +11593,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins._IncompleteInputError.__sizeof__" => "Size of object in memory, in bytes.", "builtins._IncompleteInputError.__str__" => "Return str(self).", "builtins._IncompleteInputError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins._IncompleteInputError.add_note" => "Exception.add_note(note) --\nadd a note to the exception", + "builtins._IncompleteInputError._metadata" => "exception private metadata", + "builtins._IncompleteInputError.add_note" => "Add a note to the exception", "builtins._IncompleteInputError.end_lineno" => "exception end lineno", "builtins._IncompleteInputError.end_offset" => "exception end offset", "builtins._IncompleteInputError.filename" => "exception filename", @@ -5738,7 +11603,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins._IncompleteInputError.offset" => "exception offset", "builtins._IncompleteInputError.print_file_and_line" => "exception print_file_and_line", "builtins._IncompleteInputError.text" => "exception text", - "builtins._IncompleteInputError.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "builtins._IncompleteInputError.with_traceback" => "Set self.__traceback__ to tb and return self.", "builtins.__build_class__" => "__build_class__(func, name, /, *bases, [metaclass], **kwds) -> class\n\nInternal helper function used by the class statement.", "builtins.__import__" => "Import a module.\n\nBecause this function is meant for use by the Python\ninterpreter and not for general use, it is better to use\nimportlib.import_module() to programmatically import a module.\n\nThe globals argument is only used to determine the context;\nthey are not modified. The locals argument is unused. The fromlist\nshould be a list of names to emulate ``from name import ...``, or an\nempty list to emulate ``import name``.\nWhen importing a module from a package, note that __import__('A.B', ...)\nreturns package A when fromlist is empty, but its submodule B when\nfromlist is not empty. The level argument is used to determine whether to\nperform absolute or relative imports: 0 is absolute, while a positive number\nis the number of parent directories to search relative to the current module.", "builtins.abs" => "Return the absolute value of the argument.", @@ -5776,7 +11641,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.async_generator.ag_await" => "object being awaited on, or None", "builtins.async_generator.asend" => "asend(v) -> send 'v' in generator.", "builtins.async_generator.athrow" => "athrow(value)\nathrow(type[,value[,tb]])\n\nraise exception in generator.\nthe (type, val, tb) signature is deprecated, \nand may be removed in a future version of Python.", - "builtins.bin" => "Return the binary representation of an integer.\n\n>>> bin(2796202)\n'0b1010101010101010101010'", + "builtins.bin" => "Return the binary representation of an integer.\n\n >>> bin(2796202)\n '0b1010101010101010101010'", "builtins.bool" => "Returns True when the argument is true, False otherwise.\nThe builtins True and False are the only two instances of the class bool.\nThe class bool is a subclass of the class int, and cannot be subclassed.", "builtins.bool.__abs__" => "abs(self)", "builtins.bool.__add__" => "Return self+value.", @@ -5842,12 +11707,12 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bool.bit_length" => "Number of bits necessary to represent self in binary.\n\n>>> bin(37)\n'0b100101'\n>>> (37).bit_length()\n6", "builtins.bool.conjugate" => "Returns self, the complex conjugate of any int.", "builtins.bool.denominator" => "the denominator of a rational number in lowest terms", - "builtins.bool.from_bytes" => "Return the integer represented by the given array of bytes.\n\nbytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\nbyteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\nsigned\n Indicates whether two's complement is used to represent the integer.", + "builtins.bool.from_bytes" => "Return the integer represented by the given array of bytes.\n\n bytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Indicates whether two's complement is used to represent the integer.", "builtins.bool.imag" => "the imaginary part of a complex number", "builtins.bool.is_integer" => "Returns True. Exists for duck type compatibility with float.is_integer.", "builtins.bool.numerator" => "the numerator of a rational number in lowest terms", "builtins.bool.real" => "the real part of a complex number", - "builtins.bool.to_bytes" => "Return an array of bytes representing an integer.\n\nlength\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\nbyteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\nsigned\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", + "builtins.bool.to_bytes" => "Return an array of bytes representing an integer.\n\n length\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", "builtins.breakpoint" => "Call sys.breakpointhook(*args, **kws). sys.breakpointhook() must accept\nwhatever arguments are passed.\n\nBy default, this drops you into the pdb debugger.", "builtins.builtin_function_or_method.__call__" => "Call self as a function.", "builtins.builtin_function_or_method.__delattr__" => "Implement delattr(self, name).", @@ -5907,21 +11772,21 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytearray.__sizeof__" => "Returns the size of the bytearray object in memory, in bytes.", "builtins.bytearray.__str__" => "Return str(self).", "builtins.bytearray.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.bytearray.append" => "Append a single item to the end of the bytearray.\n\nitem\n The item to be appended.", + "builtins.bytearray.append" => "Append a single item to the end of the bytearray.\n\n item\n The item to be appended.", "builtins.bytearray.capitalize" => "B.capitalize() -> copy of B\n\nReturn a copy of B with only its first character capitalized (ASCII)\nand the rest lower-cased.", "builtins.bytearray.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character.", "builtins.bytearray.clear" => "Remove all items from the bytearray.", "builtins.bytearray.copy" => "Return a copy of B.", - "builtins.bytearray.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\nstart\n Optional start position. Default: start of the bytes.\nend\n Optional stop position. Default: end of the bytes.", - "builtins.bytearray.decode" => "Decode the bytearray using the codec registered for encoding.\n\nencoding\n The encoding with which to decode the bytearray.\nerrors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", - "builtins.bytearray.endswith" => "Return True if the bytearray ends with the specified suffix, False otherwise.\n\nsuffix\n A bytes or a tuple of bytes to try.\nstart\n Optional start position. Default: start of the bytearray.\nend\n Optional stop position. Default: end of the bytearray.", + "builtins.bytearray.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytearray.decode" => "Decode the bytearray using the codec registered for encoding.\n\n encoding\n The encoding with which to decode the bytearray.\n errors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", + "builtins.bytearray.endswith" => "Return True if the bytearray ends with the specified suffix, False otherwise.\n\n suffix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytearray.\n end\n Optional stop position. Default: end of the bytearray.", "builtins.bytearray.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", - "builtins.bytearray.extend" => "Append all the items from the iterator or sequence to the end of the bytearray.\n\niterable_of_ints\n The iterable of items to append.", + "builtins.bytearray.extend" => "Append all the items from the iterator or sequence to the end of the bytearray.\n\n iterable_of_ints\n The iterable of items to append.", "builtins.bytearray.find" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", "builtins.bytearray.fromhex" => "Create a bytearray object from a string of hexadecimal numbers.\n\nSpaces between two numbers are accepted.\nExample: bytearray.fromhex('B9 01EF') -> bytearray(b'\\\\xb9\\\\x01\\\\xef')", "builtins.bytearray.hex" => "Create a string of hexadecimal numbers from a bytearray object.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nExample:\n>>> value = bytearray([0xb9, 0x01, 0xef])\n>>> value.hex()\n'b901ef'\n>>> value.hex(':')\n'b9:01:ef'\n>>> value.hex(':', 2)\n'b9:01ef'\n>>> value.hex(':', -2)\n'b901:ef'", "builtins.bytearray.index" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", - "builtins.bytearray.insert" => "Insert a single item into the bytearray before the given index.\n\nindex\n The index where the value is to be inserted.\nitem\n The item to be inserted.", + "builtins.bytearray.insert" => "Insert a single item into the bytearray before the given index.\n\n index\n The index where the value is to be inserted.\n item\n The item to be inserted.", "builtins.bytearray.isalnum" => "B.isalnum() -> bool\n\nReturn True if all characters in B are alphanumeric\nand there is at least one character in B, False otherwise.", "builtins.bytearray.isalpha" => "B.isalpha() -> bool\n\nReturn True if all characters in B are alphabetic\nand there is at least one character in B, False otherwise.", "builtins.bytearray.isascii" => "B.isascii() -> bool\n\nReturn True if B is empty or all characters in B are ASCII,\nFalse otherwise.", @@ -5937,10 +11802,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytearray.maketrans" => "Return a translation table usable for the bytes or bytearray translate method.\n\nThe returned table will be one where each byte in frm is mapped to the byte at\nthe same position in to.\n\nThe bytes objects frm and to must be of the same length.", "builtins.bytearray.partition" => "Partition the bytearray into three parts using the given separator.\n\nThis will search for the separator sep in the bytearray. If the separator is\nfound, returns a 3-tuple containing the part before the separator, the\nseparator itself, and the part after it as new bytearray objects.\n\nIf the separator is not found, returns a 3-tuple containing the copy of the\noriginal bytearray object and two empty bytearray objects.", "builtins.bytearray.pop" => "Remove and return a single item from B.\n\n index\n The index from where to remove the item.\n -1 (the default value) means remove the last item.\n\nIf no index argument is given, will pop the last item.", - "builtins.bytearray.remove" => "Remove the first occurrence of a value in the bytearray.\n\nvalue\n The value to remove.", + "builtins.bytearray.remove" => "Remove the first occurrence of a value in the bytearray.\n\n value\n The value to remove.", "builtins.bytearray.removeprefix" => "Return a bytearray with the given prefix string removed if present.\n\nIf the bytearray starts with the prefix string, return\nbytearray[len(prefix):]. Otherwise, return a copy of the original\nbytearray.", "builtins.bytearray.removesuffix" => "Return a bytearray with the given suffix string removed if present.\n\nIf the bytearray ends with the suffix string and that suffix is not\nempty, return bytearray[:-len(suffix)]. Otherwise, return a copy of\nthe original bytearray.", "builtins.bytearray.replace" => "Return a copy with all occurrences of substring old replaced by new.\n\n count\n Maximum number of occurrences to replace.\n -1 (the default value) means replace all occurrences.\n\nIf the optional argument count is given, only the first count occurrences are\nreplaced.", + "builtins.bytearray.resize" => "Resize the internal buffer of bytearray to len.\n\n size\n New size to resize to.", "builtins.bytearray.reverse" => "Reverse the order of the values in B in place.", "builtins.bytearray.rfind" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", "builtins.bytearray.rindex" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", @@ -5948,9 +11814,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytearray.rpartition" => "Partition the bytearray into three parts using the given separator.\n\nThis will search for the separator sep in the bytearray, starting at the end.\nIf the separator is found, returns a 3-tuple containing the part before the\nseparator, the separator itself, and the part after it as new bytearray\nobjects.\n\nIf the separator is not found, returns a 3-tuple containing two empty bytearray\nobjects and the copy of the original bytearray object.", "builtins.bytearray.rsplit" => "Return a list of the sections in the bytearray, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytearray.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.\n\nSplitting is done starting at the end of the bytearray and working to the front.", "builtins.bytearray.rstrip" => "Strip trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip trailing ASCII whitespace.", - "builtins.bytearray.split" => "Return a list of the sections in the bytearray, using sep as the delimiter.\n\nsep\n The delimiter according which to split the bytearray.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\nmaxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", + "builtins.bytearray.split" => "Return a list of the sections in the bytearray, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytearray.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", "builtins.bytearray.splitlines" => "Return a list of the lines in the bytearray, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", - "builtins.bytearray.startswith" => "Return True if the bytearray starts with the specified prefix, False otherwise.\n\nprefix\n A bytes or a tuple of bytes to try.\nstart\n Optional start position. Default: start of the bytearray.\nend\n Optional stop position. Default: end of the bytearray.", + "builtins.bytearray.startswith" => "Return True if the bytearray starts with the specified prefix, False otherwise.\n\n prefix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytearray.\n end\n Optional stop position. Default: end of the bytearray.", "builtins.bytearray.strip" => "Strip leading and trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading and trailing ASCII whitespace.", "builtins.bytearray.swapcase" => "B.swapcase() -> copy of B\n\nReturn a copy of B with uppercase ASCII characters converted\nto lowercase ASCII and vice versa.", "builtins.bytearray.title" => "B.title() -> copy of B\n\nReturn a titlecased version of B, i.e. ASCII words start with uppercase\ncharacters, all remaining cased characters have lowercase.", @@ -6017,9 +11883,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytes.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.bytes.capitalize" => "B.capitalize() -> copy of B\n\nReturn a copy of B with only its first character capitalized (ASCII)\nand the rest lower-cased.", "builtins.bytes.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character.", - "builtins.bytes.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\nstart\n Optional start position. Default: start of the bytes.\nend\n Optional stop position. Default: end of the bytes.", - "builtins.bytes.decode" => "Decode the bytes using the codec registered for encoding.\n\nencoding\n The encoding with which to decode the bytes.\nerrors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", - "builtins.bytes.endswith" => "Return True if the bytes ends with the specified suffix, False otherwise.\n\nsuffix\n A bytes or a tuple of bytes to try.\nstart\n Optional start position. Default: start of the bytes.\nend\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.decode" => "Decode the bytes using the codec registered for encoding.\n\n encoding\n The encoding with which to decode the bytes.\n errors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", + "builtins.bytes.endswith" => "Return True if the bytes ends with the specified suffix, False otherwise.\n\n suffix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", "builtins.bytes.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", "builtins.bytes.find" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", "builtins.bytes.fromhex" => "Create a bytes object from a string of hexadecimal numbers.\n\nSpaces between two numbers are accepted.\nExample: bytes.fromhex('B9 01EF') -> b'\\\\xb9\\\\x01\\\\xef'.", @@ -6048,9 +11914,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytes.rpartition" => "Partition the bytes into three parts using the given separator.\n\nThis will search for the separator sep in the bytes, starting at the end. If\nthe separator is found, returns a 3-tuple containing the part before the\nseparator, the separator itself, and the part after it.\n\nIf the separator is not found, returns a 3-tuple containing two empty bytes\nobjects and the original bytes object.", "builtins.bytes.rsplit" => "Return a list of the sections in the bytes, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytes.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.\n\nSplitting is done starting at the end of the bytes and working to the front.", "builtins.bytes.rstrip" => "Strip trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip trailing ASCII whitespace.", - "builtins.bytes.split" => "Return a list of the sections in the bytes, using sep as the delimiter.\n\nsep\n The delimiter according which to split the bytes.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\nmaxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", + "builtins.bytes.split" => "Return a list of the sections in the bytes, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytes.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", "builtins.bytes.splitlines" => "Return a list of the lines in the bytes, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", - "builtins.bytes.startswith" => "Return True if the bytes starts with the specified prefix, False otherwise.\n\nprefix\n A bytes or a tuple of bytes to try.\nstart\n Optional start position. Default: start of the bytes.\nend\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.startswith" => "Return True if the bytes starts with the specified prefix, False otherwise.\n\n prefix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", "builtins.bytes.strip" => "Strip leading and trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading and trailing ASCII whitespace.", "builtins.bytes.swapcase" => "B.swapcase() -> copy of B\n\nReturn a copy of B with uppercase ASCII characters converted\nto lowercase ASCII and vice versa.", "builtins.bytes.title" => "B.title() -> copy of B\n\nReturn a titlecased version of B, i.e. ASCII words start with uppercase\ncharacters, all remaining cased characters have lowercase.", @@ -6083,7 +11949,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.bytes_iterator.__str__" => "Return str(self).", "builtins.bytes_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.callable" => "Return whether the object is callable (i.e., some kind of function).\n\nNote that classes are callable, as are instances of classes with a\n__call__() method.", - "builtins.cell" => "Create a new cell object.\n\n contents\n the contents of the cell. If not specified, the cell will be empty,\n and \nfurther attempts to access its cell_contents attribute will\n raise a ValueError.", + "builtins.cell" => "Create a new cell object.\n\n contents\n the contents of the cell. If not specified, the cell will be empty,\n and \n further attempts to access its cell_contents attribute will\n raise a ValueError.", "builtins.cell.__delattr__" => "Implement delattr(self, name).", "builtins.cell.__eq__" => "Return self==value.", "builtins.cell.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -6214,6 +12080,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.complex.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.complex.__truediv__" => "Return self/value.", "builtins.complex.conjugate" => "Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j.", + "builtins.complex.from_number" => "Convert number to a complex floating-point number.", "builtins.complex.imag" => "the imaginary part of a complex number", "builtins.complex.real" => "the real part of a complex number", "builtins.coroutine.__await__" => "Return an iterator to be used in await expression.", @@ -6544,6 +12411,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.float.__trunc__" => "Return the Integral closest to x between 0 and x.", "builtins.float.as_integer_ratio" => "Return a pair of integers, whose ratio is exactly equal to the original float.\n\nThe ratio is in lowest terms and has a positive denominator. Raise\nOverflowError on infinities and a ValueError on NaNs.\n\n>>> (10.0).as_integer_ratio()\n(10, 1)\n>>> (0.0).as_integer_ratio()\n(0, 1)\n>>> (-.25).as_integer_ratio()\n(-1, 4)", "builtins.float.conjugate" => "Return self, the complex conjugate of any float.", + "builtins.float.from_number" => "Convert real number to a floating-point number.", "builtins.float.fromhex" => "Create a floating-point number from a hexadecimal string.\n\n>>> float.fromhex('0x1.ffffp10')\n2047.984375\n>>> float.fromhex('-0x1p-1074')\n-5e-324", "builtins.float.hex" => "Return a hexadecimal representation of a floating-point number.\n\n>>> (-0.1).hex()\n'-0x1.999999999999ap-4'\n>>> 3.14159.hex()\n'0x1.921f9f01b866ep+1'", "builtins.float.imag" => "the imaginary part of a complex number", @@ -6568,10 +12436,19 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.frame.__reduce_ex__" => "Helper for pickle.", "builtins.frame.__repr__" => "Return repr(self).", "builtins.frame.__setattr__" => "Implement setattr(self, name, value).", - "builtins.frame.__sizeof__" => "F.__sizeof__() -> size of F in memory, in bytes", + "builtins.frame.__sizeof__" => "Return the size of the frame in memory, in bytes.", "builtins.frame.__str__" => "Return str(self).", "builtins.frame.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.frame.clear" => "F.clear(): clear most references held by the frame", + "builtins.frame.clear" => "Clear all references held by the frame.", + "builtins.frame.f_builtins" => "Return the built-in variables in the frame.", + "builtins.frame.f_code" => "Return the code object being executed in this frame.", + "builtins.frame.f_generator" => "Return the generator or coroutine associated with this frame, or None.", + "builtins.frame.f_globals" => "Return the global variables in the frame.", + "builtins.frame.f_lasti" => "Return the index of the last attempted instruction in the frame.", + "builtins.frame.f_lineno" => "Return the current line number in the frame.", + "builtins.frame.f_locals" => "Return the mapping used by the frame to look up local variables.", + "builtins.frame.f_trace" => "Return the trace function for this frame, or None if no trace function is set.", + "builtins.frame.f_trace_opcodes" => "Return True if opcode tracing is enabled, False otherwise.", "builtins.frozenset" => "Build an immutable unordered collection of unique elements.", "builtins.frozenset.__and__" => "Return self&value.", "builtins.frozenset.__class_getitem__" => "See PEP 585", @@ -6614,7 +12491,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.frozenset.issuperset" => "Report whether this set contains another set.", "builtins.frozenset.symmetric_difference" => "Return a new set with elements in either the set or other but not both.", "builtins.frozenset.union" => "Return a new set with elements from the set and all others.", - "builtins.function" => "Create a function object.\n\ncode\n a code object\nglobals\n the globals dictionary\nname\n a string that overrides the name from the code object\nargdefs\n a tuple that specifies the default argument values\nclosure\n a tuple that supplies the bindings for free variables\nkwdefaults\n a dictionary that specifies the default keyword argument values", + "builtins.function" => "Create a function object.\n\n code\n a code object\n globals\n the globals dictionary\n name\n a string that overrides the name from the code object\n argdefs\n a tuple that specifies the default argument values\n closure\n a tuple that supplies the bindings for free variables\n kwdefaults\n a dictionary that specifies the default keyword argument values", + "builtins.function.__annotate__" => "Get the code object for a function.", "builtins.function.__call__" => "Call self as a function.", "builtins.function.__delattr__" => "Implement delattr(self, name).", "builtins.function.__eq__" => "Return self==value.", @@ -6696,7 +12574,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.globals" => "Return the dictionary containing the current scope's global variables.\n\nNOTE: Updates to this dictionary *will* affect name lookups in the current\nglobal scope and vice-versa.", "builtins.hasattr" => "Return whether the object has an attribute with the given name.\n\nThis is done by calling getattr(obj, name) and catching AttributeError.", "builtins.hash" => "Return the hash value for the given object.\n\nTwo objects that compare equal must also have the same hash value, but the\nreverse is not necessarily true.", - "builtins.hex" => "Return the hexadecimal representation of an integer.\n\n>>> hex(12648430)\n'0xc0ffee'", + "builtins.hex" => "Return the hexadecimal representation of an integer.\n\n >>> hex(12648430)\n '0xc0ffee'", "builtins.id" => "Return the identity of an object.\n\nThis is guaranteed to be unique among simultaneously existing objects.\n(CPython uses the object's memory address.)", "builtins.input" => "Read a string from standard input. The trailing newline is stripped.\n\nThe prompt string, if given, is printed to standard output without a\ntrailing newline before reading input.\n\nIf the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\nOn *nix systems, readline is used if available.", "builtins.int" => "int([x]) -> integer\nint(x, base=10) -> integer\n\nConvert a number or string to an integer, or return 0 if no arguments\nare given. If x is a number, return x.__int__(). For floating-point\nnumbers, this truncates towards zero.\n\nIf x is not a number or if base is given, then x must be a string,\nbytes, or bytearray instance representing an integer literal in the\ngiven base. The literal can be preceded by '+' or '-' and be surrounded\nby whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\nBase 0 means to interpret the base from the string as an integer literal.\n>>> int('0b100', base=0)\n4", @@ -6764,12 +12642,12 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.int.bit_length" => "Number of bits necessary to represent self in binary.\n\n>>> bin(37)\n'0b100101'\n>>> (37).bit_length()\n6", "builtins.int.conjugate" => "Returns self, the complex conjugate of any int.", "builtins.int.denominator" => "the denominator of a rational number in lowest terms", - "builtins.int.from_bytes" => "Return the integer represented by the given array of bytes.\n\nbytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\nbyteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\nsigned\n Indicates whether two's complement is used to represent the integer.", + "builtins.int.from_bytes" => "Return the integer represented by the given array of bytes.\n\n bytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Indicates whether two's complement is used to represent the integer.", "builtins.int.imag" => "the imaginary part of a complex number", "builtins.int.is_integer" => "Returns True. Exists for duck type compatibility with float.is_integer.", "builtins.int.numerator" => "the numerator of a rational number in lowest terms", "builtins.int.real" => "the real part of a complex number", - "builtins.int.to_bytes" => "Return an array of bytes representing an integer.\n\nlength\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\nbyteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\nsigned\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", + "builtins.int.to_bytes" => "Return an array of bytes representing an integer.\n\n length\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", "builtins.isinstance" => "Return whether an object is an instance of a class or of a subclass thereof.\n\nA tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\ncheck against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\nor ...`` etc.", "builtins.issubclass" => "Return whether 'cls' is derived from another class or is the same class.\n\nA tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\ncheck against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\nor ...``.", "builtins.iter" => "iter(iterable) -> iterator\niter(callable, sentinel) -> iterator\n\nGet an iterator from an object. In the first form, the argument must\nsupply its own iterator, or be a sequence.\nIn the second form, the callable is called until it returns the sentinel.", @@ -6845,7 +12723,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.list_iterator.__str__" => "Return str(self).", "builtins.list_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.locals" => "Return a dictionary containing the current scope's local variables.\n\nNOTE: Whether or not updates to this dictionary will affect name lookups in\nthe local scope and vice-versa is *implementation dependent* and not\ncovered by any backwards compatibility guarantees.", - "builtins.map" => "Make an iterator that computes the function using arguments from\neach of the iterables. Stops when the shortest iterable is exhausted.", + "builtins.map" => "Make an iterator that computes the function using arguments from\neach of the iterables. Stops when the shortest iterable is exhausted.\n\nIf strict is true and one of the arguments is exhausted before the others,\nraise a ValueError.", "builtins.map.__delattr__" => "Implement delattr(self, name).", "builtins.map.__eq__" => "Return self==value.", "builtins.map.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -6866,6 +12744,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.map.__reduce_ex__" => "Helper for pickle.", "builtins.map.__repr__" => "Return repr(self).", "builtins.map.__setattr__" => "Implement setattr(self, name, value).", + "builtins.map.__setstate__" => "Set state information for unpickling.", "builtins.map.__sizeof__" => "Size of object in memory, in bytes.", "builtins.map.__str__" => "Return str(self).", "builtins.map.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -6954,6 +12833,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.memory_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "builtins.memoryview" => "Create a new memoryview object which references the given object.", "builtins.memoryview.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "builtins.memoryview.__class_getitem__" => "See PEP 585", "builtins.memoryview.__delattr__" => "Implement delattr(self, name).", "builtins.memoryview.__delitem__" => "Delete self[key].", "builtins.memoryview.__eq__" => "Return self==value.", @@ -6986,17 +12866,19 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.memoryview.c_contiguous" => "A bool indicating whether the memory is C contiguous.", "builtins.memoryview.cast" => "Cast a memoryview to a new format or shape.", "builtins.memoryview.contiguous" => "A bool indicating whether the memory is contiguous.", + "builtins.memoryview.count" => "Count the number of occurrences of a value.", "builtins.memoryview.f_contiguous" => "A bool indicating whether the memory is Fortran contiguous.", - "builtins.memoryview.format" => "A string containing the format (in struct module style)\nfor each element in the view.", + "builtins.memoryview.format" => "A string containing the format (in struct module style)\n for each element in the view.", "builtins.memoryview.hex" => "Return the data in the buffer as a str of hexadecimal numbers.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nExample:\n>>> value = memoryview(b'\\xb9\\x01\\xef')\n>>> value.hex()\n'b901ef'\n>>> value.hex(':')\n'b9:01:ef'\n>>> value.hex(':', 2)\n'b9:01ef'\n>>> value.hex(':', -2)\n'b901:ef'", + "builtins.memoryview.index" => "Return the index of the first occurrence of a value.\n\nRaises ValueError if the value is not present.", "builtins.memoryview.itemsize" => "The size in bytes of each element of the memoryview.", - "builtins.memoryview.nbytes" => "The amount of space in bytes that the array would use in\na contiguous representation.", - "builtins.memoryview.ndim" => "An integer indicating how many dimensions of a multi-dimensional\narray the memory represents.", + "builtins.memoryview.nbytes" => "The amount of space in bytes that the array would use in\n a contiguous representation.", + "builtins.memoryview.ndim" => "An integer indicating how many dimensions of a multi-dimensional\n array the memory represents.", "builtins.memoryview.obj" => "The underlying object of the memoryview.", "builtins.memoryview.readonly" => "A bool indicating whether the memory is read only.", "builtins.memoryview.release" => "Release the underlying buffer exposed by the memoryview object.", - "builtins.memoryview.shape" => "A tuple of ndim integers giving the shape of the memory\nas an N-dimensional array.", - "builtins.memoryview.strides" => "A tuple of ndim integers giving the size in bytes to access\neach element for each dimension of the array.", + "builtins.memoryview.shape" => "A tuple of ndim integers giving the shape of the memory\n as an N-dimensional array.", + "builtins.memoryview.strides" => "A tuple of ndim integers giving the size in bytes to access\n each element for each dimension of the array.", "builtins.memoryview.suboffsets" => "A tuple of integers used internally for PIL-style arrays.", "builtins.memoryview.tobytes" => "Return the data in the buffer as a byte string.\n\nOrder can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the\noriginal array is converted to C or Fortran order. For contiguous views,\n'A' returns an exact copy of the physical memory. In particular, in-memory\nFortran order is preserved. For non-contiguous views, the data is converted\nto C first. order=None is the same as order='C'.", "builtins.memoryview.tolist" => "Return the data in the buffer as a list of elements.", @@ -7115,10 +12997,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.object.__sizeof__" => "Size of object in memory, in bytes.", "builtins.object.__str__" => "Return str(self).", "builtins.object.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "builtins.oct" => "Return the octal representation of an integer.\n\n>>> oct(342391)\n'0o1234567'", + "builtins.oct" => "Return the octal representation of an integer.\n\n >>> oct(342391)\n '0o1234567'", + "builtins.open" => "Open file and return a stream. Raise OSError upon failure.\n\nfile is either a text or byte string giving the name (and the path\nif the file isn't in the current working directory) of the file to\nbe opened or an integer file descriptor of the file to be\nwrapped. (If a file descriptor is given, it is closed when the\nreturned I/O object is closed, unless closefd is set to False.)\n\nmode is an optional string that specifies the mode in which the file\nis opened. It defaults to 'r' which means open for reading in text\nmode. Other common values are 'w' for writing (truncating the file if\nit already exists), 'x' for creating and writing to a new file, and\n'a' for appending (which on some Unix systems, means that all writes\nappend to the end of the file regardless of the current seek position).\nIn text mode, if encoding is not specified the encoding used is platform\ndependent: locale.getencoding() is called to get the current locale encoding.\n(For reading and writing raw bytes use binary mode and leave encoding\nunspecified.) The available modes are:\n\n========= ===============================================================\nCharacter Meaning\n--------- ---------------------------------------------------------------\n'r' open for reading (default)\n'w' open for writing, truncating the file first\n'x' create a new file and open it for writing\n'a' open for writing, appending to the end of the file if it exists\n'b' binary mode\n't' text mode (default)\n'+' open a disk file for updating (reading and writing)\n========= ===============================================================\n\nThe default mode is 'rt' (open for reading text). For binary random\naccess, the mode 'w+b' opens and truncates the file to 0 bytes, while\n'r+b' opens the file without truncation. The 'x' mode implies 'w' and\nraises an `FileExistsError` if the file already exists.\n\nPython distinguishes between files opened in binary and text modes,\neven when the underlying operating system doesn't. Files opened in\nbinary mode (appending 'b' to the mode argument) return contents as\nbytes objects without any decoding. In text mode (the default, or when\n't' is appended to the mode argument), the contents of the file are\nreturned as strings, the bytes having been first decoded using a\nplatform-dependent encoding or using the specified encoding if given.\n\nbuffering is an optional integer used to set the buffering policy.\nPass 0 to switch buffering off (only allowed in binary mode), 1 to select\nline buffering (only usable in text mode), and an integer > 1 to indicate\nthe size of a fixed-size chunk buffer. When no buffering argument is\ngiven, the default buffering policy works as follows:\n\n* Binary files are buffered in fixed-size chunks; the size of the buffer\n is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n when the device block size is available.\n On most systems, the buffer will typically be 128 kilobytes long.\n\n* \"Interactive\" text files (files for which isatty() returns True)\n use line buffering. Other text files use the policy described above\n for binary files.\n\nencoding is the name of the encoding used to decode or encode the\nfile. This should only be used in text mode. The default encoding is\nplatform dependent, but any encoding supported by Python can be\npassed. See the codecs module for the list of supported encodings.\n\nerrors is an optional string that specifies how encoding errors are to\nbe handled---this argument should not be used in binary mode. Pass\n'strict' to raise a ValueError exception if there is an encoding error\n(the default of None has the same effect), or pass 'ignore' to ignore\nerrors. (Note that ignoring encoding errors can lead to data loss.)\nSee the documentation for codecs.register or run 'help(codecs.Codec)'\nfor a list of the permitted encoding error strings.\n\nnewline controls how universal newlines works (it only applies to text\nmode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\nfollows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf closefd is False, the underlying file descriptor will be kept open\nwhen the file is closed. This does not work when a file name is given\nand must be True in that case.\n\nA custom opener can be used by passing a callable as *opener*. The\nunderlying file descriptor for the file object is then obtained by\ncalling *opener* with (*file*, *flags*). *opener* must return an open\nfile descriptor (passing os.open as *opener* results in functionality\nsimilar to passing None).\n\nopen() returns a file object whose type depends on the mode, and\nthrough which the standard file operations such as reading and writing\nare performed. When open() is used to open a file in a text mode ('w',\n'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\na file in a binary mode, the returned class varies: in read binary\nmode, it returns a BufferedReader; in write binary and append binary\nmodes, it returns a BufferedWriter, and in read/write mode, it returns\na BufferedRandom.\n\nIt is also possible to use a string or bytearray as a file for both\nreading and writing. For strings StringIO can be used like a file\nopened in a text mode, and for bytes a BytesIO can be used like a file\nopened in a binary mode.", "builtins.ord" => "Return the ordinal value of a character.\n\nIf the argument is a one-character string, return the Unicode code\npoint of that character.\n\nIf the argument is a bytes or bytearray object of length 1, return its\nsingle byte value.", "builtins.pow" => "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n\nSome types, such as ints, are able to use a more efficient algorithm when\ninvoked using the three argument form.", - "builtins.print" => "Prints the values to a stream, or to sys.stdout by default.\n\nsep\n string inserted between values, default a space.\nend\n string appended after the last value, default a newline.\nfile\n a file-like object (stream); defaults to the current sys.stdout.\nflush\n whether to forcibly flush the stream.", + "builtins.print" => "Prints the values to a stream, or to sys.stdout by default.\n\n sep\n string inserted between values, default a space.\n end\n string appended after the last value, default a newline.\n file\n a file-like object (stream); defaults to the current sys.stdout.\n flush\n whether to forcibly flush the stream.", "builtins.property" => "Property attribute.\n\n fget\n function to be used for getting an attribute value\n fset\n function to be used for setting an attribute value\n fdel\n function to be used for del'ing an attribute\n doc\n docstring\n\nTypical use is to define a managed attribute x:\n\nclass C(object):\n def getx(self): return self._x\n def setx(self, value): self._x = value\n def delx(self): del self._x\n x = property(getx, setx, delx, \"I'm the 'x' property.\")\n\nDecorators make defining new properties or modifying existing ones easy:\n\nclass C(object):\n @property\n def x(self):\n \"I am the 'x' property.\"\n return self._x\n @x.setter\n def x(self, value):\n self._x = value\n @x.deleter\n def x(self):\n del self._x", "builtins.property.__delattr__" => "Implement delattr(self, name).", "builtins.property.__delete__" => "Delete an attribute of instance.", @@ -7392,8 +13275,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.str.casefold" => "Return a version of the string suitable for caseless comparisons.", "builtins.str.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character (default is a space).", "builtins.str.count" => "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.", - "builtins.str.encode" => "Encode the string using the codec registered for encoding.\n\nencoding\n The encoding in which to encode the string.\nerrors\n The error handling scheme to use for encoding errors.\n The default is 'strict' meaning that encoding errors raise a\n UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n 'xmlcharrefreplace' as well as any other name registered with\n codecs.register_error that can handle UnicodeEncodeErrors.", - "builtins.str.endswith" => "Return True if the string ends with the specified suffix, False otherwise.\n\nsuffix\n A string or a tuple of strings to try.\nstart\n Optional start position. Default: start of the string.\nend\n Optional stop position. Default: end of the string.", + "builtins.str.encode" => "Encode the string using the codec registered for encoding.\n\n encoding\n The encoding in which to encode the string.\n errors\n The error handling scheme to use for encoding errors.\n The default is 'strict' meaning that encoding errors raise a\n UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n 'xmlcharrefreplace' as well as any other name registered with\n codecs.register_error that can handle UnicodeEncodeErrors.", + "builtins.str.endswith" => "Return True if the string ends with the specified suffix, False otherwise.\n\n suffix\n A string or a tuple of strings to try.\n start\n Optional start position. Default: start of the string.\n end\n Optional stop position. Default: end of the string.", "builtins.str.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", "builtins.str.find" => "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.\nReturn -1 on failure.", "builtins.str.format" => "Return a formatted version of the string, using substitutions from args and kwargs.\nThe substitutions are identified by braces ('{' and '}').", @@ -7428,7 +13311,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "builtins.str.rstrip" => "Return a copy of the string with trailing whitespace removed.\n\nIf chars is given and not None, remove characters in chars instead.", "builtins.str.split" => "Return a list of the substrings in the string, using sep as the separator string.\n\n sep\n The separator used to split the string.\n\n When set to None (the default value), will split on any whitespace\n character (including \\n \\r \\t \\f and spaces) and will discard\n empty strings from the result.\n maxsplit\n Maximum number of splits.\n -1 (the default value) means no limit.\n\nSplitting starts at the front of the string and works to the end.\n\nNote, str.split() is mainly useful for data that has been intentionally\ndelimited. With natural text that includes punctuation, consider using\nthe regular expression module.", "builtins.str.splitlines" => "Return a list of the lines in the string, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", - "builtins.str.startswith" => "Return True if the string starts with the specified prefix, False otherwise.\n\nprefix\n A string or a tuple of strings to try.\nstart\n Optional start position. Default: start of the string.\nend\n Optional stop position. Default: end of the string.", + "builtins.str.startswith" => "Return True if the string starts with the specified prefix, False otherwise.\n\n prefix\n A string or a tuple of strings to try.\n start\n Optional start position. Default: start of the string.\n end\n Optional stop position. Default: end of the string.", "builtins.str.strip" => "Return a copy of the string with leading and trailing whitespace removed.\n\nIf chars is given and not None, remove characters in chars instead.", "builtins.str.swapcase" => "Convert uppercase characters to lowercase and lowercase characters to uppercase.", "builtins.str.title" => "Return a version of the string where each word is titlecased.\n\nMore specifically, words start with uppercased characters and all remaining\ncased characters have lower case.", @@ -7701,6 +13584,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "faulthandler._stack_overflow" => "Recursive call to raise a stack overflow.", "faulthandler.cancel_dump_traceback_later" => "Cancel the previous call to dump_traceback_later().", "faulthandler.disable" => "Disable the fault handler.", + "faulthandler.dump_c_stack" => "Dump the C stack of the current thread.", "faulthandler.dump_traceback" => "Dump the traceback of the current thread, or of all threads if all_threads is True, into file.", "faulthandler.dump_traceback_later" => "Dump the traceback of all threads in timeout seconds,\nor each timeout seconds if repeat is True. If exit is True, call _exit(1) which is not safe.", "faulthandler.enable" => "Enable the fault handler.", @@ -7708,10 +13592,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "faulthandler.register" => "Register a handler for the signal 'signum': dump the traceback of the current thread, or of all threads if all_threads is True, into file.", "faulthandler.unregister" => "Unregister the handler of the signal 'signum' registered by register().", "fcntl" => "This module performs file control and I/O control on file\ndescriptors. It is an interface to the fcntl() and ioctl() Unix\nroutines. File descriptors can be obtained with the fileno() method of\na file or socket object.", - "fcntl.fcntl" => "Perform the operation `cmd` on file descriptor fd.\n\nThe values used for `cmd` are operating system dependent, and are available\nas constants in the fcntl module, using the same names as used in\nthe relevant C header files. The argument arg is optional, and\ndefaults to 0; it may be an int or a string. If arg is given as a string,\nthe return value of fcntl is a string of that length, containing the\nresulting value put in the arg buffer by the operating system. The length\nof the arg string is not allowed to exceed 1024 bytes. If the arg given\nis an integer or if none is specified, the result value is an integer\ncorresponding to the return value of the fcntl call in the C code.", - "fcntl.flock" => "Perform the lock operation `operation` on file descriptor `fd`.\n\nSee the Unix manual page for flock(2) for details (On some systems, this\nfunction is emulated using fcntl()).", - "fcntl.ioctl" => "Perform the operation `request` on file descriptor `fd`.\n\nThe values used for `request` are operating system dependent, and are available\nas constants in the fcntl or termios library modules, using the same names as\nused in the relevant C header files.\n\nThe argument `arg` is optional, and defaults to 0; it may be an int or a\nbuffer containing character data (most likely a string or an array).\n\nIf the argument is a mutable buffer (such as an array) and if the\nmutate_flag argument (which is only allowed in this case) is true then the\nbuffer is (in effect) passed to the operating system and changes made by\nthe OS will be reflected in the contents of the buffer after the call has\nreturned. The return value is the integer returned by the ioctl system\ncall.\n\nIf the argument is a mutable buffer and the mutable_flag argument is false,\nthe behavior is as if a string had been passed.\n\nIf the argument is an immutable buffer (most likely a string) then a copy\nof the buffer is passed to the operating system and the return value is a\nstring of the same length containing whatever the operating system put in\nthe buffer. The length of the arg buffer in this case is not allowed to\nexceed 1024 bytes.\n\nIf the arg given is an integer or if none is specified, the result value is\nan integer corresponding to the return value of the ioctl call in the C\ncode.", - "fcntl.lockf" => "A wrapper around the fcntl() locking calls.\n\n`fd` is the file descriptor of the file to lock or unlock, and operation is one\nof the following values:\n\n LOCK_UN - unlock\n LOCK_SH - acquire a shared lock\n LOCK_EX - acquire an exclusive lock\n\nWhen operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\nLOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the\nlock cannot be acquired, an OSError will be raised and the exception will\nhave an errno attribute set to EACCES or EAGAIN (depending on the operating\nsystem -- for portability, check for either value).\n\n`len` is the number of bytes to lock, with the default meaning to lock to\nEOF. `start` is the byte offset, relative to `whence`, to that the lock\nstarts. `whence` is as with fileobj.seek(), specifically:\n\n 0 - relative to the start of the file (SEEK_SET)\n 1 - relative to the current buffer position (SEEK_CUR)\n 2 - relative to the end of the file (SEEK_END)", + "fcntl.fcntl" => "Perform the operation cmd on file descriptor fd.\n\nThe values used for cmd are operating system dependent, and are\navailable as constants in the fcntl module, using the same names as used\nin the relevant C header files. The argument arg is optional, and\ndefaults to 0; it may be an integer, a bytes-like object or a string.\nIf arg is given as a string, it will be encoded to binary using the\nUTF-8 encoding.\n\nIf the arg given is an integer or if none is specified, the result value\nis an integer corresponding to the return value of the fcntl() call in\nthe C code.\n\nIf arg is given as a bytes-like object, the return value of fcntl() is a\nbytes object of that length, containing the resulting value put in the\narg buffer by the operating system. The length of the arg buffer is not\nallowed to exceed 1024 bytes.", + "fcntl.flock" => "Perform the lock operation on file descriptor fd.\n\nSee the Unix manual page for flock(2) for details (On some systems, this\nfunction is emulated using fcntl()).", + "fcntl.ioctl" => "Perform the operation request on file descriptor fd.\n\nThe values used for request are operating system dependent, and are\navailable as constants in the fcntl or termios library modules, using\nthe same names as used in the relevant C header files.\n\nThe argument arg is optional, and defaults to 0; it may be an integer, a\nbytes-like object or a string. If arg is given as a string, it will be\nencoded to binary using the UTF-8 encoding.\n\nIf the arg given is an integer or if none is specified, the result value\nis an integer corresponding to the return value of the ioctl() call in\nthe C code.\n\nIf the argument is a mutable buffer (such as a bytearray) and the\nmutate_flag argument is true (default) then the buffer is (in effect)\npassed to the operating system and changes made by the OS will be\nreflected in the contents of the buffer after the call has returned.\nThe return value is the integer returned by the ioctl() system call.\n\nIf the argument is a mutable buffer and the mutable_flag argument is\nfalse, the behavior is as if an immutable buffer had been passed.\n\nIf the argument is an immutable buffer then a copy of the buffer is\npassed to the operating system and the return value is a bytes object of\nthe same length containing whatever the operating system put in the\nbuffer. The length of the arg buffer in this case is not allowed to\nexceed 1024 bytes.", + "fcntl.lockf" => "A wrapper around the fcntl() locking calls.\n\nfd is the file descriptor of the file to lock or unlock, and operation\nis one of the following values:\n\n LOCK_UN - unlock\n LOCK_SH - acquire a shared lock\n LOCK_EX - acquire an exclusive lock\n\nWhen operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\nLOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\nthe lock cannot be acquired, an OSError will be raised and the exception\nwill have an errno attribute set to EACCES or EAGAIN (depending on the\noperating system -- for portability, check for either value).\n\nlen is the number of bytes to lock, with the default meaning to lock to\nEOF. start is the byte offset, relative to whence, to that the lock\nstarts. whence is as with fileobj.seek(), specifically:\n\n 0 - relative to the start of the file (SEEK_SET)\n 1 - relative to the current buffer position (SEEK_CUR)\n 2 - relative to the end of the file (SEEK_END)", "gc" => "This module provides access to the garbage collector for reference cycles.\n\nenable() -- Enable automatic garbage collection.\ndisable() -- Disable automatic garbage collection.\nisenabled() -- Returns true if automatic collection is enabled.\ncollect() -- Do a full collection right now.\nget_count() -- Return the current collection counts.\nget_stats() -- Return list of dictionaries containing per-generation stats.\nset_debug() -- Set debugging flags.\nget_debug() -- Get debugging flags.\nset_threshold() -- Set the collection thresholds.\nget_threshold() -- Return the current collection thresholds.\nget_objects() -- Return a list of all objects tracked by the collector.\nis_tracked() -- Returns true if a given object is tracked.\nis_finalized() -- Returns true if a given object has been already finalized.\nget_referrers() -- Return the list of objects that refer to an object.\nget_referents() -- Return the list of objects that an object refers to.\nfreeze() -- Freeze all tracked objects and ignore them for future collections.\nunfreeze() -- Unfreeze all objects in the permanent generation.\nget_freeze_count() -- Return the number of objects in the permanent generation.", "gc.collect" => "Run the garbage collector.\n\nWith no arguments, run a full collection. The optional argument\nmay be an integer specifying which generation to collect. A ValueError\nis raised if the generation number is invalid.\n\nThe number of unreachable objects is returned.", "gc.disable" => "Disable automatic garbage collection.", @@ -7788,7 +13672,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools._grouper.__ne__" => "Return self!=value.", "itertools._grouper.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools._grouper.__next__" => "Implement next(self).", - "itertools._grouper.__reduce__" => "Return state information for pickling.", + "itertools._grouper.__reduce__" => "Helper for pickle.", "itertools._grouper.__reduce_ex__" => "Helper for pickle.", "itertools._grouper.__repr__" => "Return repr(self).", "itertools._grouper.__setattr__" => "Implement setattr(self, name, value).", @@ -7813,11 +13697,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools._tee.__ne__" => "Return self!=value.", "itertools._tee.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools._tee.__next__" => "Implement next(self).", - "itertools._tee.__reduce__" => "Return state information for pickling.", + "itertools._tee.__reduce__" => "Helper for pickle.", "itertools._tee.__reduce_ex__" => "Helper for pickle.", "itertools._tee.__repr__" => "Return repr(self).", "itertools._tee.__setattr__" => "Implement setattr(self, name, value).", - "itertools._tee.__setstate__" => "Set state information for unpickling.", "itertools._tee.__sizeof__" => "Size of object in memory, in bytes.", "itertools._tee.__str__" => "Return str(self).", "itertools._tee.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -7836,7 +13719,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools._tee_dataobject.__lt__" => "Return self<value.", "itertools._tee_dataobject.__ne__" => "Return self!=value.", "itertools._tee_dataobject.__new__" => "Create and return a new object. See help(type) for accurate signature.", - "itertools._tee_dataobject.__reduce__" => "Return state information for pickling.", + "itertools._tee_dataobject.__reduce__" => "Helper for pickle.", "itertools._tee_dataobject.__reduce_ex__" => "Helper for pickle.", "itertools._tee_dataobject.__repr__" => "Return repr(self).", "itertools._tee_dataobject.__setattr__" => "Implement setattr(self, name, value).", @@ -7860,11 +13743,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.accumulate.__ne__" => "Return self!=value.", "itertools.accumulate.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.accumulate.__next__" => "Implement next(self).", - "itertools.accumulate.__reduce__" => "Return state information for pickling.", + "itertools.accumulate.__reduce__" => "Helper for pickle.", "itertools.accumulate.__reduce_ex__" => "Helper for pickle.", "itertools.accumulate.__repr__" => "Return repr(self).", "itertools.accumulate.__setattr__" => "Implement setattr(self, name, value).", - "itertools.accumulate.__setstate__" => "Set state information for unpickling.", "itertools.accumulate.__sizeof__" => "Size of object in memory, in bytes.", "itertools.accumulate.__str__" => "Return str(self).", "itertools.accumulate.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -7910,11 +13792,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.chain.__ne__" => "Return self!=value.", "itertools.chain.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.chain.__next__" => "Implement next(self).", - "itertools.chain.__reduce__" => "Return state information for pickling.", + "itertools.chain.__reduce__" => "Helper for pickle.", "itertools.chain.__reduce_ex__" => "Helper for pickle.", "itertools.chain.__repr__" => "Return repr(self).", "itertools.chain.__setattr__" => "Implement setattr(self, name, value).", - "itertools.chain.__setstate__" => "Set state information for unpickling.", "itertools.chain.__sizeof__" => "Size of object in memory, in bytes.", "itertools.chain.__str__" => "Return str(self).", "itertools.chain.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -7936,11 +13817,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.combinations.__ne__" => "Return self!=value.", "itertools.combinations.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.combinations.__next__" => "Implement next(self).", - "itertools.combinations.__reduce__" => "Return state information for pickling.", + "itertools.combinations.__reduce__" => "Helper for pickle.", "itertools.combinations.__reduce_ex__" => "Helper for pickle.", "itertools.combinations.__repr__" => "Return repr(self).", "itertools.combinations.__setattr__" => "Implement setattr(self, name, value).", - "itertools.combinations.__setstate__" => "Set state information for unpickling.", "itertools.combinations.__sizeof__" => "Returns size in memory, in bytes.", "itertools.combinations.__str__" => "Return str(self).", "itertools.combinations.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -7961,11 +13841,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.combinations_with_replacement.__ne__" => "Return self!=value.", "itertools.combinations_with_replacement.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.combinations_with_replacement.__next__" => "Implement next(self).", - "itertools.combinations_with_replacement.__reduce__" => "Return state information for pickling.", + "itertools.combinations_with_replacement.__reduce__" => "Helper for pickle.", "itertools.combinations_with_replacement.__reduce_ex__" => "Helper for pickle.", "itertools.combinations_with_replacement.__repr__" => "Return repr(self).", "itertools.combinations_with_replacement.__setattr__" => "Implement setattr(self, name, value).", - "itertools.combinations_with_replacement.__setstate__" => "Set state information for unpickling.", "itertools.combinations_with_replacement.__sizeof__" => "Returns size in memory, in bytes.", "itertools.combinations_with_replacement.__str__" => "Return str(self).", "itertools.combinations_with_replacement.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -7986,7 +13865,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.compress.__ne__" => "Return self!=value.", "itertools.compress.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.compress.__next__" => "Implement next(self).", - "itertools.compress.__reduce__" => "Return state information for pickling.", + "itertools.compress.__reduce__" => "Helper for pickle.", "itertools.compress.__reduce_ex__" => "Helper for pickle.", "itertools.compress.__repr__" => "Return repr(self).", "itertools.compress.__setattr__" => "Implement setattr(self, name, value).", @@ -8010,7 +13889,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.count.__ne__" => "Return self!=value.", "itertools.count.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.count.__next__" => "Implement next(self).", - "itertools.count.__reduce__" => "Return state information for pickling.", + "itertools.count.__reduce__" => "Helper for pickle.", "itertools.count.__reduce_ex__" => "Helper for pickle.", "itertools.count.__repr__" => "Return repr(self).", "itertools.count.__setattr__" => "Implement setattr(self, name, value).", @@ -8034,11 +13913,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.cycle.__ne__" => "Return self!=value.", "itertools.cycle.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.cycle.__next__" => "Implement next(self).", - "itertools.cycle.__reduce__" => "Return state information for pickling.", + "itertools.cycle.__reduce__" => "Helper for pickle.", "itertools.cycle.__reduce_ex__" => "Helper for pickle.", "itertools.cycle.__repr__" => "Return repr(self).", "itertools.cycle.__setattr__" => "Implement setattr(self, name, value).", - "itertools.cycle.__setstate__" => "Set state information for unpickling.", "itertools.cycle.__sizeof__" => "Size of object in memory, in bytes.", "itertools.cycle.__str__" => "Return str(self).", "itertools.cycle.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8059,11 +13937,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.dropwhile.__ne__" => "Return self!=value.", "itertools.dropwhile.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.dropwhile.__next__" => "Implement next(self).", - "itertools.dropwhile.__reduce__" => "Return state information for pickling.", + "itertools.dropwhile.__reduce__" => "Helper for pickle.", "itertools.dropwhile.__reduce_ex__" => "Helper for pickle.", "itertools.dropwhile.__repr__" => "Return repr(self).", "itertools.dropwhile.__setattr__" => "Implement setattr(self, name, value).", - "itertools.dropwhile.__setstate__" => "Set state information for unpickling.", "itertools.dropwhile.__sizeof__" => "Size of object in memory, in bytes.", "itertools.dropwhile.__str__" => "Return str(self).", "itertools.dropwhile.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8084,14 +13961,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.filterfalse.__ne__" => "Return self!=value.", "itertools.filterfalse.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.filterfalse.__next__" => "Implement next(self).", - "itertools.filterfalse.__reduce__" => "Return state information for pickling.", + "itertools.filterfalse.__reduce__" => "Helper for pickle.", "itertools.filterfalse.__reduce_ex__" => "Helper for pickle.", "itertools.filterfalse.__repr__" => "Return repr(self).", "itertools.filterfalse.__setattr__" => "Implement setattr(self, name, value).", "itertools.filterfalse.__sizeof__" => "Size of object in memory, in bytes.", "itertools.filterfalse.__str__" => "Return str(self).", "itertools.filterfalse.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "itertools.groupby" => "make an iterator that returns consecutive keys and groups from the iterable\n\niterable\n Elements to divide into groups according to the key function.\nkey\n A function for computing the group category for each element.\n If the key function is not specified or is None, the element itself\n is used for grouping.", + "itertools.groupby" => "make an iterator that returns consecutive keys and groups from the iterable\n\n iterable\n Elements to divide into groups according to the key function.\n key\n A function for computing the group category for each element.\n If the key function is not specified or is None, the element itself\n is used for grouping.", "itertools.groupby.__delattr__" => "Implement delattr(self, name).", "itertools.groupby.__eq__" => "Return self==value.", "itertools.groupby.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -8108,11 +13985,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.groupby.__ne__" => "Return self!=value.", "itertools.groupby.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.groupby.__next__" => "Implement next(self).", - "itertools.groupby.__reduce__" => "Return state information for pickling.", + "itertools.groupby.__reduce__" => "Helper for pickle.", "itertools.groupby.__reduce_ex__" => "Helper for pickle.", "itertools.groupby.__repr__" => "Return repr(self).", "itertools.groupby.__setattr__" => "Implement setattr(self, name, value).", - "itertools.groupby.__setstate__" => "Set state information for unpickling.", "itertools.groupby.__sizeof__" => "Size of object in memory, in bytes.", "itertools.groupby.__str__" => "Return str(self).", "itertools.groupby.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8133,15 +14009,14 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.islice.__ne__" => "Return self!=value.", "itertools.islice.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.islice.__next__" => "Implement next(self).", - "itertools.islice.__reduce__" => "Return state information for pickling.", + "itertools.islice.__reduce__" => "Helper for pickle.", "itertools.islice.__reduce_ex__" => "Helper for pickle.", "itertools.islice.__repr__" => "Return repr(self).", "itertools.islice.__setattr__" => "Implement setattr(self, name, value).", - "itertools.islice.__setstate__" => "Set state information for unpickling.", "itertools.islice.__sizeof__" => "Size of object in memory, in bytes.", "itertools.islice.__str__" => "Return str(self).", "itertools.islice.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", - "itertools.pairwise" => "Return an iterator of overlapping pairs taken from the input iterator.\n\ns -> (s0,s1), (s1,s2), (s2, s3), ...", + "itertools.pairwise" => "Return an iterator of overlapping pairs taken from the input iterator.\n\n s -> (s0,s1), (s1,s2), (s2, s3), ...", "itertools.pairwise.__delattr__" => "Implement delattr(self, name).", "itertools.pairwise.__eq__" => "Return self==value.", "itertools.pairwise.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -8182,11 +14057,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.permutations.__ne__" => "Return self!=value.", "itertools.permutations.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.permutations.__next__" => "Implement next(self).", - "itertools.permutations.__reduce__" => "Return state information for pickling.", + "itertools.permutations.__reduce__" => "Helper for pickle.", "itertools.permutations.__reduce_ex__" => "Helper for pickle.", "itertools.permutations.__repr__" => "Return repr(self).", "itertools.permutations.__setattr__" => "Implement setattr(self, name, value).", - "itertools.permutations.__setstate__" => "Set state information for unpickling.", "itertools.permutations.__sizeof__" => "Returns size in memory, in bytes.", "itertools.permutations.__str__" => "Return str(self).", "itertools.permutations.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8207,11 +14081,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.product.__ne__" => "Return self!=value.", "itertools.product.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.product.__next__" => "Implement next(self).", - "itertools.product.__reduce__" => "Return state information for pickling.", + "itertools.product.__reduce__" => "Helper for pickle.", "itertools.product.__reduce_ex__" => "Helper for pickle.", "itertools.product.__repr__" => "Return repr(self).", "itertools.product.__setattr__" => "Implement setattr(self, name, value).", - "itertools.product.__setstate__" => "Set state information for unpickling.", "itertools.product.__sizeof__" => "Returns size in memory, in bytes.", "itertools.product.__str__" => "Return str(self).", "itertools.product.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8233,7 +14106,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.repeat.__ne__" => "Return self!=value.", "itertools.repeat.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.repeat.__next__" => "Implement next(self).", - "itertools.repeat.__reduce__" => "Return state information for pickling.", + "itertools.repeat.__reduce__" => "Helper for pickle.", "itertools.repeat.__reduce_ex__" => "Helper for pickle.", "itertools.repeat.__repr__" => "Return repr(self).", "itertools.repeat.__setattr__" => "Implement setattr(self, name, value).", @@ -8257,7 +14130,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.starmap.__ne__" => "Return self!=value.", "itertools.starmap.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.starmap.__next__" => "Implement next(self).", - "itertools.starmap.__reduce__" => "Return state information for pickling.", + "itertools.starmap.__reduce__" => "Helper for pickle.", "itertools.starmap.__reduce_ex__" => "Helper for pickle.", "itertools.starmap.__repr__" => "Return repr(self).", "itertools.starmap.__setattr__" => "Implement setattr(self, name, value).", @@ -8281,11 +14154,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.takewhile.__ne__" => "Return self!=value.", "itertools.takewhile.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.takewhile.__next__" => "Implement next(self).", - "itertools.takewhile.__reduce__" => "Return state information for pickling.", + "itertools.takewhile.__reduce__" => "Helper for pickle.", "itertools.takewhile.__reduce_ex__" => "Helper for pickle.", "itertools.takewhile.__repr__" => "Return repr(self).", "itertools.takewhile.__setattr__" => "Implement setattr(self, name, value).", - "itertools.takewhile.__setstate__" => "Set state information for unpickling.", "itertools.takewhile.__sizeof__" => "Size of object in memory, in bytes.", "itertools.takewhile.__str__" => "Return str(self).", "itertools.takewhile.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8307,11 +14179,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "itertools.zip_longest.__ne__" => "Return self!=value.", "itertools.zip_longest.__new__" => "Create and return a new object. See help(type) for accurate signature.", "itertools.zip_longest.__next__" => "Implement next(self).", - "itertools.zip_longest.__reduce__" => "Return state information for pickling.", + "itertools.zip_longest.__reduce__" => "Helper for pickle.", "itertools.zip_longest.__reduce_ex__" => "Helper for pickle.", "itertools.zip_longest.__repr__" => "Return repr(self).", "itertools.zip_longest.__setattr__" => "Implement setattr(self, name, value).", - "itertools.zip_longest.__setstate__" => "Set state information for unpickling.", "itertools.zip_longest.__sizeof__" => "Size of object in memory, in bytes.", "itertools.zip_longest.__str__" => "Return str(self).", "itertools.zip_longest.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", @@ -8350,7 +14221,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "math.fsum" => "Return an accurate floating-point sum of values in the iterable seq.\n\nAssumes IEEE-754 floating-point arithmetic.", "math.gamma" => "Gamma function at x.", "math.gcd" => "Greatest Common Divisor.", - "math.hypot" => "hypot(*coordinates) -> value\n\nMultidimensional Euclidean distance from the origin to a point.\n\nRoughly equivalent to:\n sqrt(sum(x**2 for x in coordinates))\n\nFor a two dimensional point (x, y), gives the hypotenuse\nusing the Pythagorean theorem: sqrt(x*x + y*y).\n\nFor example, the hypotenuse of a 3/4/5 right triangle is:\n\n >>> hypot(3.0, 4.0)\n 5.0", + "math.hypot" => "Multidimensional Euclidean distance from the origin to a point.\n\nRoughly equivalent to:\n sqrt(sum(x**2 for x in coordinates))\n\nFor a two dimensional point (x, y), gives the hypotenuse\nusing the Pythagorean theorem: sqrt(x*x + y*y).\n\nFor example, the hypotenuse of a 3/4/5 right triangle is:\n\n >>> hypot(3.0, 4.0)\n 5.0", "math.isclose" => "Determine whether two floating-point numbers are close in value.\n\n rel_tol\n maximum difference for being considered \"close\", relative to the\n magnitude of the input values\n abs_tol\n maximum difference for being considered \"close\", regardless of the\n magnitude of the input values\n\nReturn True if a is close in value to b, and False otherwise.\n\nFor the values to be considered close, the difference between them\nmust be smaller than at least one of the tolerances.\n\n-inf, inf and NaN behave similarly to the IEEE 754 Standard. That\nis, NaN is not close to anything, even itself. inf and -inf are\nonly close to themselves.", "math.isfinite" => "Return True if x is neither an infinity nor a NaN, and False otherwise.", "math.isinf" => "Return True if x is a positive or negative infinity, and False otherwise.", @@ -8373,7 +14244,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "math.sin" => "Return the sine of x (measured in radians).", "math.sinh" => "Return the hyperbolic sine of x.", "math.sqrt" => "Return the square root of x.", - "math.sumprod" => "Return the sum of products of values from two iterables p and q.\n\nRoughly equivalent to:\n\n sum(itertools.starmap(operator.mul, zip(p, q, strict=True)))\n\nFor float and mixed int/float inputs, the intermediate products\nand sums are computed with extended precision.", + "math.sumprod" => "Return the sum of products of values from two iterables p and q.\n\nRoughly equivalent to:\n\n sum(map(operator.mul, p, q, strict=True))\n\nFor float and mixed int/float inputs, the intermediate products\nand sums are computed with extended precision.", "math.tan" => "Return the tangent of x (measured in radians).", "math.tanh" => "Return the hyperbolic tangent of x.", "math.trunc" => "Truncates the Real x to the nearest Integral toward 0.\n\nUses the __trunc__ magic method.", @@ -8455,6 +14326,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "nt.DirEntry.path" => "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)", "nt.DirEntry.stat" => "Return stat_result object for the entry; cached per entry.", "nt._add_dll_directory" => "Add a path to the DLL search path.\n\nThis search path is used when resolving dependencies for imported\nextension modules (the module itself is resolved through sys.path),\nand also by ctypes.\n\nReturns an opaque value that may be passed to os.remove_dll_directory\nto remove this directory from the search path.", + "nt._create_environ" => "Create the environment dictionary.", "nt._exit" => "Exit to the system with specified status, without normal exit processing.", "nt._findfirstfile" => "A function to get the real file name without accessing the file in Windows.", "nt._getdiskusage" => "Return disk usage statistics about the given path as a (total, free) tuple.", @@ -8484,8 +14356,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "nt.device_encoding" => "Return a string describing the encoding of a terminal's file descriptor.\n\nThe file descriptor must be attached to a terminal.\nIf the device is not a terminal, return None.", "nt.dup" => "Return a duplicate of a file descriptor.", "nt.dup2" => "Duplicate file descriptor.", - "nt.execv" => "Execute an executable path with arguments, replacing current process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.", - "nt.execve" => "Execute an executable path with arguments, replacing current process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.\nenv\n Dictionary of strings mapping to strings.", + "nt.execv" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "nt.execve" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", "nt.fchmod" => "Change the access permissions of the file given by file descriptor fd.\n\n fd\n The file descriptor of the file to be modified.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n\nEquivalent to os.chmod(fd, mode).", "nt.fspath" => "Return the file system path representation of the object.\n\nIf the object is str or bytes, then allow it to pass through as-is. If the\nobject defines __fspath__(), then return the result of that method. All other\ntypes raise a TypeError.", "nt.fstat" => "Perform a stat system call on the given file descriptor.\n\nLike stat(), but for an open file descriptor.\nEquivalent to os.stat(fd).", @@ -8515,6 +14387,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "nt.pipe" => "Create a pipe.\n\nReturns a tuple of two file descriptors:\n (read_fd, write_fd)", "nt.putenv" => "Change or add an environment variable.", "nt.read" => "Read from a file descriptor. Returns a bytes object.", + "nt.readinto" => "Read into a buffer object from a file descriptor.\n\nThe buffer should be mutable and bytes-like. On success, returns the number of\nbytes read. Less bytes may be read than the size of the buffer. The underlying\nsystem call will be retried when interrupted by a signal, unless the signal\nhandler raises an exception. Other errors will not be retried and an error will\nbe raised.\n\nReturns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n(which can be used to check for errors without reading data). Never returns\nnegative.", "nt.readlink" => "Return a string representing the path to which the symbolic link points.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\nand path should be relative; path will then be relative to that directory.\n\ndir_fd may not be implemented on your platform. If it is unavailable,\nusing it will raise a NotImplementedError.", "nt.remove" => "Remove a file (same as unlink()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", "nt.rename" => "Rename a file or directory.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nsrc_dir_fd and dst_dir_fd, may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", @@ -8524,13 +14397,128 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "nt.set_blocking" => "Set the blocking mode of the specified file descriptor.\n\nSet the O_NONBLOCK flag if blocking is False,\nclear the O_NONBLOCK flag otherwise.", "nt.set_handle_inheritable" => "Set the inheritable flag of the specified handle.", "nt.set_inheritable" => "Set the inheritable flag of the specified file descriptor.", - "nt.spawnv" => "Execute the program specified by path in a new process.\n\nmode\n Mode of process creation.\npath\n Path of executable file.\nargv\n Tuple or list of strings.", - "nt.spawnve" => "Execute the program specified by path in a new process.\n\nmode\n Mode of process creation.\npath\n Path of executable file.\nargv\n Tuple or list of strings.\nenv\n Dictionary of strings mapping to strings.", + "nt.spawnv" => "Execute the program specified by path in a new process.\n\n mode\n Mode of process creation.\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "nt.spawnve" => "Execute the program specified by path in a new process.\n\n mode\n Mode of process creation.\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", "nt.startfile" => "Start a file with its associated application.\n\nWhen \"operation\" is not specified or \"open\", this acts like\ndouble-clicking the file in Explorer, or giving the file name as an\nargument to the DOS \"start\" command: the file is opened with whatever\napplication (if any) its extension is associated.\nWhen another \"operation\" is given, it specifies what should be done with\nthe file. A typical operation is \"print\".\n\n\"arguments\" is passed to the application, but should be omitted if the\nfile is a document.\n\n\"cwd\" is the working directory for the operation. If \"filepath\" is\nrelative, it will be resolved against this directory. This argument\nshould usually be an absolute path.\n\n\"show_cmd\" can be used to override the recommended visibility option.\nSee the Windows ShellExecute documentation for values.\n\nstartfile returns as soon as the associated application is launched.\nThere is no option to wait for the application to close, and no way\nto retrieve the application's exit status.\n\nThe filepath is relative to the current directory. If you want to use\nan absolute path, make sure the first character is not a slash (\"/\");\nthe underlying Win32 ShellExecute function doesn't work if it is.", "nt.stat" => "Perform a stat system call on the given path.\n\n path\n Path to be examined; can be string, bytes, a path-like object or\n open-file-descriptor int.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be a relative string; path will then be relative to\n that directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n stat will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nIt's an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.", + "nt.stat_result" => "stat_result: Result from stat, fstat, or lstat.\n\nThis object may be accessed either as a tuple of\n (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\nor via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\nPosix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\nor st_flags, they are available as attributes only.\n\nSee os.stat for more information.", + "nt.stat_result.__add__" => "Return self+value.", + "nt.stat_result.__class_getitem__" => "See PEP 585", + "nt.stat_result.__contains__" => "Return bool(key in self).", + "nt.stat_result.__delattr__" => "Implement delattr(self, name).", + "nt.stat_result.__eq__" => "Return self==value.", + "nt.stat_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.stat_result.__ge__" => "Return self>=value.", + "nt.stat_result.__getattribute__" => "Return getattr(self, name).", + "nt.stat_result.__getitem__" => "Return self[key].", + "nt.stat_result.__getstate__" => "Helper for pickle.", + "nt.stat_result.__gt__" => "Return self>value.", + "nt.stat_result.__hash__" => "Return hash(self).", + "nt.stat_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.stat_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.stat_result.__iter__" => "Implement iter(self).", + "nt.stat_result.__le__" => "Return self<=value.", + "nt.stat_result.__len__" => "Return len(self).", + "nt.stat_result.__lt__" => "Return self<value.", + "nt.stat_result.__mul__" => "Return self*value.", + "nt.stat_result.__ne__" => "Return self!=value.", + "nt.stat_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.stat_result.__reduce_ex__" => "Helper for pickle.", + "nt.stat_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.stat_result.__repr__" => "Return repr(self).", + "nt.stat_result.__rmul__" => "Return value*self.", + "nt.stat_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.stat_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.stat_result.__str__" => "Return str(self).", + "nt.stat_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.stat_result.count" => "Return number of occurrences of value.", + "nt.stat_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.stat_result.st_atime" => "time of last access", + "nt.stat_result.st_atime_ns" => "time of last access in nanoseconds", + "nt.stat_result.st_birthtime" => "time of creation", + "nt.stat_result.st_birthtime_ns" => "time of creation in nanoseconds", + "nt.stat_result.st_ctime" => "time of last change", + "nt.stat_result.st_ctime_ns" => "time of last change in nanoseconds", + "nt.stat_result.st_dev" => "device", + "nt.stat_result.st_file_attributes" => "Windows file attribute bits", + "nt.stat_result.st_gid" => "group ID of owner", + "nt.stat_result.st_ino" => "inode", + "nt.stat_result.st_mode" => "protection bits", + "nt.stat_result.st_mtime" => "time of last modification", + "nt.stat_result.st_mtime_ns" => "time of last modification in nanoseconds", + "nt.stat_result.st_nlink" => "number of hard links", + "nt.stat_result.st_reparse_tag" => "Windows reparse tag", + "nt.stat_result.st_size" => "total size, in bytes", + "nt.stat_result.st_uid" => "user ID of owner", + "nt.statvfs_result" => "statvfs_result: Result from statvfs or fstatvfs.\n\nThis object may be accessed either as a tuple of\n (bsize, frsize, blocks, bfree, bavail, files, ffree, favail, flag, namemax),\nor via the attributes f_bsize, f_frsize, f_blocks, f_bfree, and so on.\n\nSee os.statvfs for more information.", + "nt.statvfs_result.__add__" => "Return self+value.", + "nt.statvfs_result.__class_getitem__" => "See PEP 585", + "nt.statvfs_result.__contains__" => "Return bool(key in self).", + "nt.statvfs_result.__delattr__" => "Implement delattr(self, name).", + "nt.statvfs_result.__eq__" => "Return self==value.", + "nt.statvfs_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.statvfs_result.__ge__" => "Return self>=value.", + "nt.statvfs_result.__getattribute__" => "Return getattr(self, name).", + "nt.statvfs_result.__getitem__" => "Return self[key].", + "nt.statvfs_result.__getstate__" => "Helper for pickle.", + "nt.statvfs_result.__gt__" => "Return self>value.", + "nt.statvfs_result.__hash__" => "Return hash(self).", + "nt.statvfs_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.statvfs_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.statvfs_result.__iter__" => "Implement iter(self).", + "nt.statvfs_result.__le__" => "Return self<=value.", + "nt.statvfs_result.__len__" => "Return len(self).", + "nt.statvfs_result.__lt__" => "Return self<value.", + "nt.statvfs_result.__mul__" => "Return self*value.", + "nt.statvfs_result.__ne__" => "Return self!=value.", + "nt.statvfs_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.statvfs_result.__reduce_ex__" => "Helper for pickle.", + "nt.statvfs_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.statvfs_result.__repr__" => "Return repr(self).", + "nt.statvfs_result.__rmul__" => "Return value*self.", + "nt.statvfs_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.statvfs_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.statvfs_result.__str__" => "Return str(self).", + "nt.statvfs_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.statvfs_result.count" => "Return number of occurrences of value.", + "nt.statvfs_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", "nt.strerror" => "Translate an error code to a message string.", "nt.symlink" => "Create a symbolic link pointing to src named dst.\n\ntarget_is_directory is required on Windows if the target is to be\n interpreted as a directory. (On Windows, symlink requires\n Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n target_is_directory is ignored on non-Windows platforms.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", "nt.system" => "Execute the command in a subshell.", + "nt.terminal_size" => "A tuple of (columns, lines) for holding terminal window size", + "nt.terminal_size.__add__" => "Return self+value.", + "nt.terminal_size.__class_getitem__" => "See PEP 585", + "nt.terminal_size.__contains__" => "Return bool(key in self).", + "nt.terminal_size.__delattr__" => "Implement delattr(self, name).", + "nt.terminal_size.__eq__" => "Return self==value.", + "nt.terminal_size.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.terminal_size.__ge__" => "Return self>=value.", + "nt.terminal_size.__getattribute__" => "Return getattr(self, name).", + "nt.terminal_size.__getitem__" => "Return self[key].", + "nt.terminal_size.__getstate__" => "Helper for pickle.", + "nt.terminal_size.__gt__" => "Return self>value.", + "nt.terminal_size.__hash__" => "Return hash(self).", + "nt.terminal_size.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.terminal_size.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.terminal_size.__iter__" => "Implement iter(self).", + "nt.terminal_size.__le__" => "Return self<=value.", + "nt.terminal_size.__len__" => "Return len(self).", + "nt.terminal_size.__lt__" => "Return self<value.", + "nt.terminal_size.__mul__" => "Return self*value.", + "nt.terminal_size.__ne__" => "Return self!=value.", + "nt.terminal_size.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.terminal_size.__reduce_ex__" => "Helper for pickle.", + "nt.terminal_size.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.terminal_size.__repr__" => "Return repr(self).", + "nt.terminal_size.__rmul__" => "Return value*self.", + "nt.terminal_size.__setattr__" => "Implement setattr(self, name, value).", + "nt.terminal_size.__sizeof__" => "Size of object in memory, in bytes.", + "nt.terminal_size.__str__" => "Return str(self).", + "nt.terminal_size.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.terminal_size.columns" => "width of the terminal window in characters", + "nt.terminal_size.count" => "Return number of occurrences of value.", + "nt.terminal_size.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.terminal_size.lines" => "height of the terminal window in characters", "nt.times" => "Return a collection containing process timing information.\n\nThe object returned behaves like a named tuple with these fields:\n (utime, stime, cutime, cstime, elapsed_time)\nAll fields are floating-point numbers.", "nt.times_result" => "times_result: Result from os.times().\n\nThis object may be accessed either as a tuple of\n (user, system, children_user, children_system, elapsed),\nor via the attributes user, system, children_user, children_system,\nand elapsed.\n\nSee os.times for more information.", "nt.times_result.__add__" => "Return self+value.", @@ -8655,6 +14643,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.WIFSTOPPED" => "Return True if the process returning status was stopped.", "posix.WSTOPSIG" => "Return the signal that stopped the process that provided the status value.", "posix.WTERMSIG" => "Return the signal that terminated the process that provided the status value.", + "posix._create_environ" => "Create the environment dictionary.", "posix._exit" => "Exit to the system with specified status, without normal exit processing.", "posix._fcopyfile" => "Efficiently copy content or metadata of 2 regular file descriptors (macOS).", "posix._inputhook" => "Calls PyOS_CallInputHook droppong the GIL first", @@ -8680,8 +14669,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.eventfd" => "Creates and returns an event notification file descriptor.", "posix.eventfd_read" => "Read eventfd value", "posix.eventfd_write" => "Write eventfd value.", - "posix.execv" => "Execute an executable path with arguments, replacing current process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.", - "posix.execve" => "Execute an executable path with arguments, replacing current process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.\nenv\n Dictionary of strings mapping to strings.", + "posix.execv" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "posix.execve" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", "posix.fchdir" => "Change to the directory of the given file descriptor.\n\nfd must be opened on a directory, not a file.\nEquivalent to os.chdir(fd).", "posix.fchmod" => "Change the access permissions of the file given by file descriptor fd.\n\n fd\n The file descriptor of the file to be modified.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n\nEquivalent to os.chmod(fd, mode).", "posix.fchown" => "Change the owner and group id of the file specified by file descriptor.\n\nEquivalent to os.chown(fd, uid, gid).", @@ -8702,7 +14691,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.getegid" => "Return the current process's effective group id.", "posix.geteuid" => "Return the current process's effective user id.", "posix.getgid" => "Return the current process's group id.", - "posix.getgrouplist" => "Returns a list of groups to which a user belongs.\n\nuser\n username to lookup\ngroup\n base group id of the user", + "posix.getgrouplist" => "Returns a list of groups to which a user belongs.\n\n user\n username to lookup\n group\n base group id of the user", "posix.getgroups" => "Return list of supplemental group IDs for the process.", "posix.getloadavg" => "Return average recent system load information.\n\nReturn the number of processes in the system run queue averaged over\nthe last 1, 5, and 15 minutes as a tuple of three floats.\nRaises OSError if the load average was unobtainable.", "posix.getlogin" => "Return the actual login name.", @@ -8728,7 +14717,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.link" => "Create a hard link to a file.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of src is a symbolic\n link, link will create a link to the symbolic link itself instead of the\n file the link points to.\nsrc_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n platform. If they are unavailable, using them will raise a\n NotImplementedError.", "posix.listdir" => "Return a list containing the names of the files in the directory.\n\npath can be specified as either str, bytes, or a path-like object. If path is bytes,\n the filenames returned will also be bytes; in all other circumstances\n the filenames returned will be str.\nIf path is None, uses the path='.'.\nOn some platforms, path may also be specified as an open file descriptor;\\\n the file descriptor must refer to a directory.\n If this functionality is unavailable, using it raises NotImplementedError.\n\nThe list is in arbitrary order. It does not include the special\nentries '.' and '..' even if they are present in the directory.", "posix.listxattr" => "Return a list of extended attributes on path.\n\npath may be either None, a string, a path-like object, or an open file descriptor.\nif path is None, listxattr will examine the current directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, listxattr will examine the symbolic link itself instead of the file\n the link points to.", - "posix.lockf" => "Apply, test or remove a POSIX lock on an open file descriptor.\n\nfd\n An open file descriptor.\ncommand\n One of F_LOCK, F_TLOCK, F_ULOCK or F_TEST.\nlength\n The number of bytes to lock, starting at the current position.", + "posix.lockf" => "Apply, test or remove a POSIX lock on an open file descriptor.\n\n fd\n An open file descriptor.\n command\n One of F_LOCK, F_TLOCK, F_ULOCK or F_TEST.\n length\n The number of bytes to lock, starting at the current position.", "posix.login_tty" => "Prepare the tty of which fd is a file descriptor for a new login session.\n\nMake the calling process a session leader; make the tty the\ncontrolling tty, the stdin, the stdout, and the stderr of the\ncalling process; close fd.", "posix.lseek" => "Set the position of a file descriptor. Return the new position.\n\n fd\n An open file descriptor, as returned by os.open().\n position\n Position, interpreted relative to 'whence'.\n whence\n The relative position to seek from. Valid values are:\n - SEEK_SET: seek from the start of the file.\n - SEEK_CUR: seek from the current file position.\n - SEEK_END: seek from the end of the file.\n\nThe return value is the number of bytes relative to the beginning of the file.", "posix.lstat" => "Perform a stat system call on the given path, without following symbolic links.\n\nLike stat(), but do not follow symbolic links.\nEquivalent to stat(path, follow_symlinks=False).", @@ -8748,8 +14737,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.posix_fadvise" => "Announce an intention to access data in a specific pattern.\n\nAnnounce an intention to access data in a specific pattern, thus allowing\nthe kernel to make optimizations.\nThe advice applies to the region of the file specified by fd starting at\noffset and continuing for length bytes.\nadvice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\nPOSIX_FADV_RANDOM, POSIX_FADV_NOREUSE, POSIX_FADV_WILLNEED, or\nPOSIX_FADV_DONTNEED.", "posix.posix_fallocate" => "Ensure a file has allocated at least a particular number of bytes on disk.\n\nEnsure that the file specified by fd encompasses a range of bytes\nstarting at offset bytes from the beginning and continuing for length bytes.", "posix.posix_openpt" => "Open and return a file descriptor for a master pseudo-terminal device.\n\nPerforms a posix_openpt() C function call. The oflag argument is used to\nset file status flags and file access modes as specified in the manual page\nof posix_openpt() of your system.", - "posix.posix_spawn" => "Execute the program specified by path in a new process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.\nenv\n Dictionary of strings mapping to strings.\nfile_actions\n A sequence of file action tuples.\nsetpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\nresetids\n If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\nsetsid\n If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\nsetsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\nsetsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\nscheduler\n A tuple with the scheduler policy (optional) and parameters.", - "posix.posix_spawnp" => "Execute the program specified by path in a new process.\n\npath\n Path of executable file.\nargv\n Tuple or list of strings.\nenv\n Dictionary of strings mapping to strings.\nfile_actions\n A sequence of file action tuples.\nsetpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\nresetids\n If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\nsetsid\n If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\nsetsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\nsetsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\nscheduler\n A tuple with the scheduler policy (optional) and parameters.", + "posix.posix_spawn" => "Execute the program specified by path in a new process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.\n file_actions\n A sequence of file action tuples.\n setpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n resetids\n If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n setsid\n If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n setsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n setsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n scheduler\n A tuple with the scheduler policy (optional) and parameters.", + "posix.posix_spawnp" => "Execute the program specified by path in a new process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.\n file_actions\n A sequence of file action tuples.\n setpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n resetids\n If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n setsid\n If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n setsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n setsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n scheduler\n A tuple with the scheduler policy (optional) and parameters.", "posix.pread" => "Read a number of bytes from a file descriptor starting at a particular offset.\n\nRead length bytes from file descriptor fd, starting at offset bytes from\nthe beginning of the file. The file offset remains unchanged.", "posix.preadv" => "Reads from a file descriptor into a number of mutable bytes-like objects.\n\nCombines the functionality of readv() and pread(). As readv(), it will\ntransfer data into each buffer until it is full and then move on to the next\nbuffer in the sequence to hold the rest of the data. Its fourth argument,\nspecifies the file offset at which the input operation is to be performed. It\nwill return the total number of bytes read (which can be less than the total\ncapacity of all the objects).\n\nThe flags argument contains a bitwise OR of zero or more of the following flags:\n\n- RWF_HIPRI\n- RWF_NOWAIT\n\nUsing non-zero flags requires Linux 4.6 or newer.", "posix.ptsname" => "Return the name of the slave pseudo-terminal device.\n\n fd\n File descriptor of a master pseudo-terminal device.\n\nIf the ptsname_r() C function is available, it is called;\notherwise, performs a ptsname() C function call.", @@ -8757,6 +14746,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.pwrite" => "Write bytes to a file descriptor starting at a particular offset.\n\nWrite buffer to fd, starting at offset bytes from the beginning of\nthe file. Returns the number of bytes written. Does not change the\ncurrent file offset.", "posix.pwritev" => "Writes the contents of bytes-like objects to a file descriptor at a given offset.\n\nCombines the functionality of writev() and pwrite(). All buffers must be a sequence\nof bytes-like objects. Buffers are processed in array order. Entire contents of first\nbuffer is written before proceeding to second, and so on. The operating system may\nset a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\nThis function writes the contents of each object to the file descriptor and returns\nthe total number of bytes written.\n\nThe flags argument contains a bitwise OR of zero or more of the following flags:\n\n- RWF_DSYNC\n- RWF_SYNC\n- RWF_APPEND\n\nUsing non-zero flags requires Linux 4.7 or newer.", "posix.read" => "Read from a file descriptor. Returns a bytes object.", + "posix.readinto" => "Read into a buffer object from a file descriptor.\n\nThe buffer should be mutable and bytes-like. On success, returns the number of\nbytes read. Less bytes may be read than the size of the buffer. The underlying\nsystem call will be retried when interrupted by a signal, unless the signal\nhandler raises an exception. Other errors will not be retried and an error will\nbe raised.\n\nReturns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n(which can be used to check for errors without reading data). Never returns\nnegative.", "posix.readlink" => "Return a string representing the path to which the symbolic link points.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\nand path should be relative; path will then be relative to that directory.\n\ndir_fd may not be implemented on your platform. If it is unavailable,\nusing it will raise a NotImplementedError.", "posix.readv" => "Read from a file descriptor fd into an iterable of buffers.\n\nThe buffers should be mutable buffers accepting bytes.\nreadv will transfer data into each buffer until it is full\nand then move on to the next buffer in the sequence to hold\nthe rest of the data.\n\nreadv returns the total number of bytes read,\nwhich may be less than the total capacity of all the buffers.", "posix.register_at_fork" => "Register callables to be called when forking a new process.\n\n before\n A callable to be called in the parent before the fork() syscall.\n after_in_child\n A callable to be called in the child after fork().\n after_in_parent\n A callable to be called in the parent after fork().\n\n'before' callbacks are called in reverse order.\n'after_in_child' and 'after_in_parent' callbacks are called in order.", @@ -8771,7 +14761,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.sched_getaffinity" => "Return the affinity of the process identified by pid (or the current process if zero).\n\nThe affinity is returned as a set of CPU identifiers.", "posix.sched_getparam" => "Returns scheduling parameters for the process identified by pid.\n\nIf pid is 0, returns parameters for the calling process.\nReturn value is an instance of sched_param.", "posix.sched_getscheduler" => "Get the scheduling policy for the process identified by pid.\n\nPassing 0 for pid returns the scheduling policy for the calling process.", - "posix.sched_param" => "Currently has only one field: sched_priority\n\nsched_priority\n A scheduling parameter.", + "posix.sched_param" => "Currently has only one field: sched_priority\n\n sched_priority\n A scheduling parameter.", "posix.sched_param.__add__" => "Return self+value.", "posix.sched_param.__class_getitem__" => "See PEP 585", "posix.sched_param.__contains__" => "Return bool(key in self).", @@ -8816,7 +14806,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.seteuid" => "Set the current process's effective user id.", "posix.setgid" => "Set the current process's group id.", "posix.setgroups" => "Set the groups of the current process to list.", - "posix.setns" => "Move the calling thread into different namespaces.\n\nfd\n A file descriptor to a namespace.\nnstype\n Type of namespace.", + "posix.setns" => "Move the calling thread into different namespaces.\n\n fd\n A file descriptor to a namespace.\n nstype\n Type of namespace.", "posix.setpgid" => "Call the system call setpgid(pid, pgrp).", "posix.setpgrp" => "Make the current process the leader of its process group.", "posix.setpriority" => "Set program scheduling priority.", @@ -8829,7 +14819,90 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.setxattr" => "Set extended attribute attribute on path to value.\n\npath may be either a string, a path-like object, or an open file descriptor.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, setxattr will modify the symbolic link itself instead of the file\n the link points to.", "posix.splice" => "Transfer count bytes from one pipe to a descriptor or vice versa.\n\n src\n Source file descriptor.\n dst\n Destination file descriptor.\n count\n Number of bytes to copy.\n offset_src\n Starting offset in src.\n offset_dst\n Starting offset in dst.\n flags\n Flags to modify the semantics of the call.\n\nIf offset_src is None, then src is read from the current position;\nrespectively for offset_dst. The offset associated to the file\ndescriptor that refers to a pipe must be None.", "posix.stat" => "Perform a stat system call on the given path.\n\n path\n Path to be examined; can be string, bytes, a path-like object or\n open-file-descriptor int.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be a relative string; path will then be relative to\n that directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n stat will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nIt's an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.", + "posix.stat_result" => "stat_result: Result from stat, fstat, or lstat.\n\nThis object may be accessed either as a tuple of\n (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\nor via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\nPosix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\nor st_flags, they are available as attributes only.\n\nSee os.stat for more information.", + "posix.stat_result.__add__" => "Return self+value.", + "posix.stat_result.__class_getitem__" => "See PEP 585", + "posix.stat_result.__contains__" => "Return bool(key in self).", + "posix.stat_result.__delattr__" => "Implement delattr(self, name).", + "posix.stat_result.__eq__" => "Return self==value.", + "posix.stat_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.stat_result.__ge__" => "Return self>=value.", + "posix.stat_result.__getattribute__" => "Return getattr(self, name).", + "posix.stat_result.__getitem__" => "Return self[key].", + "posix.stat_result.__getstate__" => "Helper for pickle.", + "posix.stat_result.__gt__" => "Return self>value.", + "posix.stat_result.__hash__" => "Return hash(self).", + "posix.stat_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.stat_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.stat_result.__iter__" => "Implement iter(self).", + "posix.stat_result.__le__" => "Return self<=value.", + "posix.stat_result.__len__" => "Return len(self).", + "posix.stat_result.__lt__" => "Return self<value.", + "posix.stat_result.__mul__" => "Return self*value.", + "posix.stat_result.__ne__" => "Return self!=value.", + "posix.stat_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.stat_result.__reduce_ex__" => "Helper for pickle.", + "posix.stat_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.stat_result.__repr__" => "Return repr(self).", + "posix.stat_result.__rmul__" => "Return value*self.", + "posix.stat_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.stat_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.stat_result.__str__" => "Return str(self).", + "posix.stat_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.stat_result.count" => "Return number of occurrences of value.", + "posix.stat_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.stat_result.st_atime" => "time of last access", + "posix.stat_result.st_atime_ns" => "time of last access in nanoseconds", + "posix.stat_result.st_birthtime" => "time of creation", + "posix.stat_result.st_blksize" => "blocksize for filesystem I/O", + "posix.stat_result.st_blocks" => "number of blocks allocated", + "posix.stat_result.st_ctime" => "time of last change", + "posix.stat_result.st_ctime_ns" => "time of last change in nanoseconds", + "posix.stat_result.st_dev" => "device", + "posix.stat_result.st_flags" => "user defined flags for file", + "posix.stat_result.st_gen" => "generation number", + "posix.stat_result.st_gid" => "group ID of owner", + "posix.stat_result.st_ino" => "inode", + "posix.stat_result.st_mode" => "protection bits", + "posix.stat_result.st_mtime" => "time of last modification", + "posix.stat_result.st_mtime_ns" => "time of last modification in nanoseconds", + "posix.stat_result.st_nlink" => "number of hard links", + "posix.stat_result.st_rdev" => "device type (if inode device)", + "posix.stat_result.st_size" => "total size, in bytes", + "posix.stat_result.st_uid" => "user ID of owner", "posix.statvfs" => "Perform a statvfs system call on the given path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", + "posix.statvfs_result" => "statvfs_result: Result from statvfs or fstatvfs.\n\nThis object may be accessed either as a tuple of\n (bsize, frsize, blocks, bfree, bavail, files, ffree, favail, flag, namemax),\nor via the attributes f_bsize, f_frsize, f_blocks, f_bfree, and so on.\n\nSee os.statvfs for more information.", + "posix.statvfs_result.__add__" => "Return self+value.", + "posix.statvfs_result.__class_getitem__" => "See PEP 585", + "posix.statvfs_result.__contains__" => "Return bool(key in self).", + "posix.statvfs_result.__delattr__" => "Implement delattr(self, name).", + "posix.statvfs_result.__eq__" => "Return self==value.", + "posix.statvfs_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.statvfs_result.__ge__" => "Return self>=value.", + "posix.statvfs_result.__getattribute__" => "Return getattr(self, name).", + "posix.statvfs_result.__getitem__" => "Return self[key].", + "posix.statvfs_result.__getstate__" => "Helper for pickle.", + "posix.statvfs_result.__gt__" => "Return self>value.", + "posix.statvfs_result.__hash__" => "Return hash(self).", + "posix.statvfs_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.statvfs_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.statvfs_result.__iter__" => "Implement iter(self).", + "posix.statvfs_result.__le__" => "Return self<=value.", + "posix.statvfs_result.__len__" => "Return len(self).", + "posix.statvfs_result.__lt__" => "Return self<value.", + "posix.statvfs_result.__mul__" => "Return self*value.", + "posix.statvfs_result.__ne__" => "Return self!=value.", + "posix.statvfs_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.statvfs_result.__reduce_ex__" => "Helper for pickle.", + "posix.statvfs_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.statvfs_result.__repr__" => "Return repr(self).", + "posix.statvfs_result.__rmul__" => "Return value*self.", + "posix.statvfs_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.statvfs_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.statvfs_result.__str__" => "Return str(self).", + "posix.statvfs_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.statvfs_result.count" => "Return number of occurrences of value.", + "posix.statvfs_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", "posix.strerror" => "Translate an error code to a message string.", "posix.symlink" => "Create a symbolic link pointing to src named dst.\n\ntarget_is_directory is required on Windows if the target is to be\n interpreted as a directory. (On Windows, symlink requires\n Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n target_is_directory is ignored on non-Windows platforms.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", "posix.sync" => "Force write of everything to disk.", @@ -8837,11 +14910,45 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.system" => "Execute the command in a subshell.", "posix.tcgetpgrp" => "Return the process group associated with the terminal specified by fd.", "posix.tcsetpgrp" => "Set the process group associated with the terminal specified by fd.", - "posix.timerfd_create" => "Create and return a timer file descriptor.\n\nclockid\n A valid clock ID constant as timer file descriptor.\n\n time.CLOCK_REALTIME\n time.CLOCK_MONOTONIC\n time.CLOCK_BOOTTIME\nflags\n 0 or a bit mask of os.TFD_NONBLOCK or os.TFD_CLOEXEC.\n\n os.TFD_NONBLOCK\n If *TFD_NONBLOCK* is set as a flag, read doesn't blocks.\n If *TFD_NONBLOCK* is not set as a flag, read block until the timer fires.\n\n os.TFD_CLOEXEC\n If *TFD_CLOEXEC* is set as a flag, enable the close-on-exec flag", - "posix.timerfd_gettime" => "Return a tuple of a timer file descriptor's (interval, next expiration) in float seconds.\n\nfd\n A timer file descriptor.", - "posix.timerfd_gettime_ns" => "Return a tuple of a timer file descriptor's (interval, next expiration) in nanoseconds.\n\nfd\n A timer file descriptor.", - "posix.timerfd_settime" => "Alter a timer file descriptor's internal timer in seconds.\n\nfd\n A timer file descriptor.\nflags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\ninitial\n The initial expiration time, in seconds.\ninterval\n The timer's interval, in seconds.", - "posix.timerfd_settime_ns" => "Alter a timer file descriptor's internal timer in nanoseconds.\n\nfd\n A timer file descriptor.\nflags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\ninitial\n initial expiration timing in seconds.\ninterval\n interval for the timer in seconds.", + "posix.terminal_size" => "A tuple of (columns, lines) for holding terminal window size", + "posix.terminal_size.__add__" => "Return self+value.", + "posix.terminal_size.__class_getitem__" => "See PEP 585", + "posix.terminal_size.__contains__" => "Return bool(key in self).", + "posix.terminal_size.__delattr__" => "Implement delattr(self, name).", + "posix.terminal_size.__eq__" => "Return self==value.", + "posix.terminal_size.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.terminal_size.__ge__" => "Return self>=value.", + "posix.terminal_size.__getattribute__" => "Return getattr(self, name).", + "posix.terminal_size.__getitem__" => "Return self[key].", + "posix.terminal_size.__getstate__" => "Helper for pickle.", + "posix.terminal_size.__gt__" => "Return self>value.", + "posix.terminal_size.__hash__" => "Return hash(self).", + "posix.terminal_size.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.terminal_size.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.terminal_size.__iter__" => "Implement iter(self).", + "posix.terminal_size.__le__" => "Return self<=value.", + "posix.terminal_size.__len__" => "Return len(self).", + "posix.terminal_size.__lt__" => "Return self<value.", + "posix.terminal_size.__mul__" => "Return self*value.", + "posix.terminal_size.__ne__" => "Return self!=value.", + "posix.terminal_size.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.terminal_size.__reduce_ex__" => "Helper for pickle.", + "posix.terminal_size.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.terminal_size.__repr__" => "Return repr(self).", + "posix.terminal_size.__rmul__" => "Return value*self.", + "posix.terminal_size.__setattr__" => "Implement setattr(self, name, value).", + "posix.terminal_size.__sizeof__" => "Size of object in memory, in bytes.", + "posix.terminal_size.__str__" => "Return str(self).", + "posix.terminal_size.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.terminal_size.columns" => "width of the terminal window in characters", + "posix.terminal_size.count" => "Return number of occurrences of value.", + "posix.terminal_size.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.terminal_size.lines" => "height of the terminal window in characters", + "posix.timerfd_create" => "Create and return a timer file descriptor.\n\n clockid\n A valid clock ID constant as timer file descriptor.\n\n time.CLOCK_REALTIME\n time.CLOCK_MONOTONIC\n time.CLOCK_BOOTTIME\n flags\n 0 or a bit mask of os.TFD_NONBLOCK or os.TFD_CLOEXEC.\n\n os.TFD_NONBLOCK\n If *TFD_NONBLOCK* is set as a flag, read doesn't blocks.\n If *TFD_NONBLOCK* is not set as a flag, read block until the timer fires.\n\n os.TFD_CLOEXEC\n If *TFD_CLOEXEC* is set as a flag, enable the close-on-exec flag", + "posix.timerfd_gettime" => "Return a tuple of a timer file descriptor's (interval, next expiration) in float seconds.\n\n fd\n A timer file descriptor.", + "posix.timerfd_gettime_ns" => "Return a tuple of a timer file descriptor's (interval, next expiration) in nanoseconds.\n\n fd\n A timer file descriptor.", + "posix.timerfd_settime" => "Alter a timer file descriptor's internal timer in seconds.\n\n fd\n A timer file descriptor.\n flags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n initial\n The initial expiration time, in seconds.\n interval\n The timer's interval, in seconds.", + "posix.timerfd_settime_ns" => "Alter a timer file descriptor's internal timer in nanoseconds.\n\n fd\n A timer file descriptor.\n flags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n initial\n initial expiration timing in seconds.\n interval\n interval for the timer in seconds.", "posix.times" => "Return a collection containing process timing information.\n\nThe object returned behaves like a named tuple with these fields:\n (utime, stime, cutime, cstime, elapsed_time)\nAll fields are floating-point numbers.", "posix.times_result" => "times_result: Result from os.times().\n\nThis object may be accessed either as a tuple of\n (user, system, children_user, children_system, elapsed),\nor via the attributes user, system, children_user, children_system,\nand elapsed.\n\nSee os.times for more information.", "posix.times_result.__add__" => "Return self+value.", @@ -8881,7 +14988,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.times_result.system" => "system time", "posix.times_result.user" => "user time", "posix.truncate" => "Truncate a file, specified by path, to a specific length.\n\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", - "posix.ttyname" => "Return the name of the terminal device connected to 'fd'.\n\nfd\n Integer file descriptor handle.", + "posix.ttyname" => "Return the name of the terminal device connected to 'fd'.\n\n fd\n Integer file descriptor handle.", "posix.umask" => "Set the current numeric umask and return the previous umask.", "posix.uname" => "Return an object identifying the current operating system.\n\nThe object behaves like a named tuple with the following fields:\n (sysname, nodename, release, version, machine)", "posix.uname_result" => "uname_result: Result from os.uname().\n\nThis object may be accessed either as a tuple of\n (sysname, nodename, release, version, machine),\nor via the attributes sysname, nodename, release, version, and machine.\n\nSee os.uname for more information.", @@ -8924,7 +15031,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "posix.unlink" => "Remove a file (same as remove()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", "posix.unlockpt" => "Unlock a pseudo-terminal master/slave pair.\n\n fd\n File descriptor of a master pseudo-terminal device.\n\nPerforms an unlockpt() C function call.", "posix.unsetenv" => "Delete an environment variable.", - "posix.unshare" => "Disassociate parts of a process (or thread) execution context.\n\nflags\n Namespaces to be unshared.", + "posix.unshare" => "Disassociate parts of a process (or thread) execution context.\n\n flags\n Namespaces to be unshared.", "posix.urandom" => "Return a bytes object containing random bytes suitable for cryptographic use.", "posix.utime" => "Set the access and modified time of path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\n\nIf times is not None, it must be a tuple (atime, mtime);\n atime and mtime should be expressed as float seconds since the epoch.\nIf ns is specified, it must be a tuple (atime_ns, mtime_ns);\n atime_ns and mtime_ns should be expressed as integer nanoseconds\n since the epoch.\nIf times is None and ns is unspecified, utime uses the current time.\nSpecifying tuples for both times and ns is an error.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, utime will modify the symbolic link itself instead of the file the\n link points to.\nIt is an error to use dir_fd or follow_symlinks when specifying path\n as an open file descriptor.\ndir_fd and follow_symlinks may not be available on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", "posix.wait" => "Wait for completion of a child process.\n\nReturns a tuple of information about the child process:\n (pid, status)", @@ -9012,14 +15119,39 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "pwd.struct_passwd.pw_uid" => "user id", "pyexpat" => "Python wrapper for Expat parser.", "pyexpat.ErrorString" => "Returns string error for given number.", + "pyexpat.ExpatError.__delattr__" => "Implement delattr(self, name).", + "pyexpat.ExpatError.__eq__" => "Return self==value.", + "pyexpat.ExpatError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pyexpat.ExpatError.__ge__" => "Return self>=value.", + "pyexpat.ExpatError.__getattribute__" => "Return getattr(self, name).", + "pyexpat.ExpatError.__getstate__" => "Helper for pickle.", + "pyexpat.ExpatError.__gt__" => "Return self>value.", + "pyexpat.ExpatError.__hash__" => "Return hash(self).", + "pyexpat.ExpatError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pyexpat.ExpatError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pyexpat.ExpatError.__le__" => "Return self<=value.", + "pyexpat.ExpatError.__lt__" => "Return self<value.", + "pyexpat.ExpatError.__ne__" => "Return self!=value.", + "pyexpat.ExpatError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pyexpat.ExpatError.__reduce_ex__" => "Helper for pickle.", + "pyexpat.ExpatError.__repr__" => "Return repr(self).", + "pyexpat.ExpatError.__setattr__" => "Implement setattr(self, name, value).", + "pyexpat.ExpatError.__sizeof__" => "Size of object in memory, in bytes.", + "pyexpat.ExpatError.__str__" => "Return str(self).", + "pyexpat.ExpatError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.ExpatError.__weakref__" => "list of weak references to the object", + "pyexpat.ExpatError.add_note" => "Add a note to the exception", + "pyexpat.ExpatError.with_traceback" => "Set self.__traceback__ to tb and return self.", "pyexpat.ParserCreate" => "Return a new XML parser object.", "pyexpat.XMLParserType" => "XML parser", "pyexpat.XMLParserType.ExternalEntityParserCreate" => "Create a parser for parsing an external entity based on the information passed to the ExternalEntityRefHandler.", "pyexpat.XMLParserType.GetBase" => "Return base URL string for the parser.", "pyexpat.XMLParserType.GetInputContext" => "Return the untranslated text of the input that caused the current event.\n\nIf the event was generated by a large amount of text (such as a start tag\nfor an element with many attributes), not all of the text may be available.", "pyexpat.XMLParserType.GetReparseDeferralEnabled" => "Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0.", - "pyexpat.XMLParserType.Parse" => "Parse XML data.\n\n`isfinal' should be true at end of input.", + "pyexpat.XMLParserType.Parse" => "Parse XML data.\n\n'isfinal' should be true at end of input.", "pyexpat.XMLParserType.ParseFile" => "Parse XML data from file-like object.", + "pyexpat.XMLParserType.SetAllocTrackerActivationThreshold" => "Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n\nBy default, parser objects have an allocation activation threshold of 64 MiB.", + "pyexpat.XMLParserType.SetAllocTrackerMaximumAmplification" => "Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n\nThe amplification factor is calculated as \"allocated / direct\" while parsing,\nwhere \"direct\" is the number of bytes read from the primary document in parsing\nand \"allocated\" is the number of bytes of dynamic memory allocated in the parser\nhierarchy.\n\nThe 'max_factor' value must be a non-NaN floating point value greater than\nor equal to 1.0. Amplification factors greater than 100.0 can be observed\nnear the start of parsing even with benign files in practice. In particular,\nthe activation threshold should be carefully chosen to avoid false positives.\n\nBy default, parser objects have a maximum amplification factor of 100.0.", "pyexpat.XMLParserType.SetBase" => "Set the base URL for the parser.", "pyexpat.XMLParserType.SetParamEntityParsing" => "Controls parsing of parameter entities (including the external DTD subset).\n\nPossible flag values are XML_PARAM_ENTITY_PARSING_NEVER,\nXML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE and\nXML_PARAM_ENTITY_PARSING_ALWAYS. Returns true if setting the flag\nwas successful.", "pyexpat.XMLParserType.SetReparseDeferralEnabled" => "Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0.", @@ -9045,6 +15177,29 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "pyexpat.XMLParserType.__sizeof__" => "Size of object in memory, in bytes.", "pyexpat.XMLParserType.__str__" => "Return str(self).", "pyexpat.XMLParserType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.error.__delattr__" => "Implement delattr(self, name).", + "pyexpat.error.__eq__" => "Return self==value.", + "pyexpat.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pyexpat.error.__ge__" => "Return self>=value.", + "pyexpat.error.__getattribute__" => "Return getattr(self, name).", + "pyexpat.error.__getstate__" => "Helper for pickle.", + "pyexpat.error.__gt__" => "Return self>value.", + "pyexpat.error.__hash__" => "Return hash(self).", + "pyexpat.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pyexpat.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pyexpat.error.__le__" => "Return self<=value.", + "pyexpat.error.__lt__" => "Return self<value.", + "pyexpat.error.__ne__" => "Return self!=value.", + "pyexpat.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pyexpat.error.__reduce_ex__" => "Helper for pickle.", + "pyexpat.error.__repr__" => "Return repr(self).", + "pyexpat.error.__setattr__" => "Implement setattr(self, name, value).", + "pyexpat.error.__sizeof__" => "Size of object in memory, in bytes.", + "pyexpat.error.__str__" => "Return str(self).", + "pyexpat.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.error.__weakref__" => "list of weak references to the object", + "pyexpat.error.add_note" => "Add a note to the exception", + "pyexpat.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "readline" => "Importing this module enables command line editing using GNU readline.", "readline.add_history" => "Add an item to the history buffer.", "readline.append_history_file" => "Append the last nelements items of the history list to file.\n\nThe default filename is ~/.history.", @@ -9148,10 +15303,10 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "select.epoll.closed" => "True if the epoll handler is closed", "select.epoll.fileno" => "Return the epoll control file descriptor.", "select.epoll.fromfd" => "Create an epoll object from a given control fd.", - "select.epoll.modify" => "Modify event mask for a registered file descriptor.\n\nfd\n the target file descriptor of the operation\neventmask\n a bit set composed of the various EPOLL constants", + "select.epoll.modify" => "Modify event mask for a registered file descriptor.\n\n fd\n the target file descriptor of the operation\n eventmask\n a bit set composed of the various EPOLL constants", "select.epoll.poll" => "Wait for events on the epoll file descriptor.\n\n timeout\n the maximum time to wait in seconds (as float);\n a timeout of None or -1 makes poll wait indefinitely\n maxevents\n the maximum number of events returned; -1 means no limit\n\nReturns a list containing any descriptors that have events to report,\nas a list of (fd, events) 2-tuples.", "select.epoll.register" => "Registers a new fd or raises an OSError if the fd is already registered.\n\n fd\n the target file descriptor of the operation\n eventmask\n a bit set composed of the various EPOLL constants\n\nThe epoll interface supports all file descriptors that support poll.", - "select.epoll.unregister" => "Remove a registered file descriptor from the epoll object.\n\nfd\n the target file descriptor of the operation", + "select.epoll.unregister" => "Remove a registered file descriptor from the epoll object.\n\n fd\n the target file descriptor of the operation", "select.kevent" => "kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0)\n\nThis object is the equivalent of the struct kevent for the C API.\n\nSee the kqueue manpage for more detailed information about the meaning\nof the arguments.\n\nOne minor note: while you might hope that udata could store a\nreference to a python object, it cannot, because it is impossible to\nkeep a proper reference count of the object once it's passed into the\nkernel. Therefore, I have restricted it to only storing an integer. I\nrecommend ignoring it and simply using the 'ident' field to key off\nof. You could also set up a dictionary on the python side to store a\nudata->object mapping.", "select.kevent.__delattr__" => "Implement delattr(self, name).", "select.kevent.__eq__" => "Return self==value.", @@ -9198,7 +15353,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "select.kqueue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "select.kqueue.close" => "Close the kqueue control file descriptor.\n\nFurther operations on the kqueue object will raise an exception.", "select.kqueue.closed" => "True if the kqueue handler is closed", - "select.kqueue.control" => "Calls the kernel kevent function.\n\nchangelist\n Must be an iterable of kevent objects describing the changes to be made\n to the kernel's watch list or None.\nmaxevents\n The maximum number of events that the kernel will return.\ntimeout\n The maximum time to wait in seconds, or else None to wait forever.\n This accepts floats for smaller timeouts, too.", + "select.kqueue.control" => "Calls the kernel kevent function.\n\n changelist\n Must be an iterable of kevent objects describing the changes to be made\n to the kernel's watch list or None.\n maxevents\n The maximum number of events that the kernel will return.\n timeout\n The maximum time to wait in seconds, or else None to wait forever.\n This accepts floats for smaller timeouts, too.", "select.kqueue.fileno" => "Return the kqueue control file descriptor.", "select.kqueue.fromfd" => "Create a kqueue object from a given control fd.", "select.poll" => "Returns a polling object.\n\nThis object supports registering and unregistering file descriptors, and then\npolling them for I/O events.", @@ -9211,14 +15366,17 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "sys._baserepl" => "Private function for getting the base REPL", "sys._clear_internal_caches" => "Clear all internal performance-related caches.", "sys._clear_type_cache" => "Clear the internal type lookup cache.", + "sys._clear_type_descriptors" => "Private function for clearing certain descriptors from a type's dictionary.\n\nSee gh-135228 for context.", "sys._current_exceptions" => "Return a dict mapping each thread's identifier to its current raised exception.\n\nThis function should be used for specialized purposes only.", "sys._current_frames" => "Return a dict mapping each thread's thread id to its current stack frame.\n\nThis function should be used for specialized purposes only.", "sys._debugmallocstats" => "Print summary info to stderr about the state of pymalloc's structures.\n\nIn Py_DEBUG mode, also perform some expensive internal consistency\nchecks.", + "sys._dump_tracelets" => "Dump the graph of tracelets in graphviz format", "sys._enablelegacywindowsfsencoding" => "Changes the default filesystem encoding to mbcs:replace.\n\nThis is done for consistency with earlier versions of Python. See PEP\n529 for more information.\n\nThis is equivalent to defining the PYTHONLEGACYWINDOWSFSENCODING\nenvironment variable before launching Python.", "sys._get_cpu_count_config" => "Private function for getting PyConfig.cpu_count", "sys._getframe" => "Return a frame object from the call stack.\n\nIf optional integer depth is given, return the frame object that many\ncalls below the top of the stack. If that is deeper than the call\nstack, ValueError is raised. The default for depth is zero, returning\nthe frame at the top of the call stack.\n\nThis function should be used for internal and specialized purposes\nonly.", "sys._getframemodulename" => "Return the name of the module for a calling frame.\n\nThe default depth returns the module containing the call to this API.\nA more typical use in a library will pass a depth of 1 to get the user's\nmodule rather than the library module.\n\nIf no frame, module, or name can be found, returns None.", "sys._is_gil_enabled" => "Return True if the GIL is currently enabled and False otherwise.", + "sys._is_immortal" => "Return True if the given object is \"immortal\" per PEP 683.\n\nThis function should be used for specialized purposes only.", "sys._is_interned" => "Return True if the given string is \"interned\".", "sys._setprofileallthreads" => "Set the profiling function in all running threads belonging to the current interpreter.\n\nIt will be called on each function call and return. See the profiler\nchapter in the library manual.", "sys._settraceallthreads" => "Set the global debug tracing function in all running threads belonging to the current interpreter.\n\nIt will be called on each function call. See the debugger chapter\nin the library manual.", @@ -9251,7 +15409,9 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "sys.getwindowsversion" => "Return info about the running version of Windows as a named tuple.\n\nThe members are named: major, minor, build, platform, service_pack,\nservice_pack_major, service_pack_minor, suite_mask, product_type and\nplatform_version. For backward compatibility, only the first 5 items\nare available by indexing. All elements are numbers, except\nservice_pack and platform_type which are strings, and platform_version\nwhich is a 3-tuple. Platform is always 2. Product_type may be 1 for a\nworkstation, 2 for a domain controller, 3 for a server.\nPlatform_version is a 3-tuple containing a version number that is\nintended for identifying the OS rather than feature detection.", "sys.intern" => "``Intern'' the given string.\n\nThis enters the string in the (global) table of interned strings whose\npurpose is to speed up dictionary lookups. Return the string itself or\nthe previously interned string object with the same value.", "sys.is_finalizing" => "Return True if Python is exiting.", + "sys.is_remote_debug_enabled" => "Return True if remote debugging is enabled, False otherwise.", "sys.is_stack_trampoline_active" => "Return *True* if a stack profiler trampoline is active.", + "sys.remote_exec" => "Executes a file containing Python code in a given remote Python process.\n\nThis function returns immediately, and the code will be executed by the\ntarget process's main thread at the next available opportunity, similarly\nto how signals are handled. There is no interface to determine when the\ncode has been executed. The caller is responsible for making sure that\nthe file still exists whenever the remote process tries to read it and that\nit hasn't been overwritten.\n\nThe remote process must be running a CPython interpreter of the same major\nand minor version as the local process. If either the local or remote\ninterpreter is pre-release (alpha, beta, or release candidate) then the\nlocal and remote interpreters must be the same exact version.\n\nArgs:\n pid (int): The process ID of the target Python process.\n script (str|bytes): The path to a file containing\n the Python code to be executed.", "sys.set_asyncgen_hooks" => "set_asyncgen_hooks([firstiter] [, finalizer])\n\nSet a finalizer for async generators objects.", "sys.set_coroutine_origin_tracking_depth" => "Enable or disable origin tracking for coroutine objects in this thread.\n\nCoroutine objects will track 'depth' frames of traceback information\nabout where they came from, available in their cr_origin attribute.\n\nSet a depth of 0 to disable.", "sys.set_int_max_str_digits" => "Set the maximum string digits limit for non-binary int<->str conversions.", @@ -9268,8 +15428,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "syslog.setlogmask" => "Set the priority mask to maskpri and return the previous mask value.", "syslog.syslog" => "syslog([priority=LOG_INFO,] message)\nSend the string message to the system logger.", "termios" => "This module provides an interface to the Posix calls for tty I/O control.\nFor a complete description of these calls, see the Posix or Unix manual\npages. It is only available for those Unix versions that support Posix\ntermios style tty I/O control.\n\nAll functions in this module take a file descriptor fd as their first\nargument. This can be an integer file descriptor, such as returned by\nsys.stdin.fileno(), or a file object, such as sys.stdin itself.", - "termios.error.__cause__" => "exception cause", - "termios.error.__context__" => "exception context", "termios.error.__delattr__" => "Implement delattr(self, name).", "termios.error.__eq__" => "Return self==value.", "termios.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -9291,8 +15449,8 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "termios.error.__str__" => "Return str(self).", "termios.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "termios.error.__weakref__" => "list of weak references to the object", - "termios.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "termios.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "termios.error.add_note" => "Add a note to the exception", + "termios.error.with_traceback" => "Set self.__traceback__ to tb and return self.", "termios.tcdrain" => "Wait until all output written to file descriptor fd has been transmitted.", "termios.tcflow" => "Suspend or resume input or output on file descriptor fd.\n\nThe action argument can be termios.TCOOFF to suspend output,\ntermios.TCOON to restart output, termios.TCIOFF to suspend input,\nor termios.TCION to restart input.", "termios.tcflush" => "Discard queued data on file descriptor fd.\n\nThe queue selector specifies which queue: termios.TCIFLUSH for the input\nqueue, termios.TCOFLUSH for the output queue, or termios.TCIOFLUSH for\nboth queues.", @@ -9323,7 +15481,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "time.sleep" => "sleep(seconds)\n\nDelay execution for a given number of seconds. The argument may be\na floating-point number for subsecond precision.", "time.strftime" => "strftime(format[, tuple]) -> string\n\nConvert a time tuple to a string according to a format specification.\nSee the library reference manual for formatting codes. When the time tuple\nis not present, current time as returned by localtime() is used.\n\nCommonly used format codes:\n\n%Y Year with century as a decimal number.\n%m Month as a decimal number [01,12].\n%d Day of the month as a decimal number [01,31].\n%H Hour (24-hour clock) as a decimal number [00,23].\n%M Minute as a decimal number [00,59].\n%S Second as a decimal number [00,61].\n%z Time zone offset from UTC.\n%a Locale's abbreviated weekday name.\n%A Locale's full weekday name.\n%b Locale's abbreviated month name.\n%B Locale's full month name.\n%c Locale's appropriate date and time representation.\n%I Hour (12-hour clock) as a decimal number [01,12].\n%p Locale's equivalent of either AM or PM.\n\nOther codes may be available on your platform. See documentation for\nthe C library strftime function.", "time.strptime" => "strptime(string, format) -> struct_time\n\nParse a string to a time tuple according to a format specification.\nSee the library reference manual for formatting codes (same as\nstrftime()).\n\nCommonly used format codes:\n\n%Y Year with century as a decimal number.\n%m Month as a decimal number [01,12].\n%d Day of the month as a decimal number [01,31].\n%H Hour (24-hour clock) as a decimal number [00,23].\n%M Minute as a decimal number [00,59].\n%S Second as a decimal number [00,61].\n%z Time zone offset from UTC.\n%a Locale's abbreviated weekday name.\n%A Locale's full weekday name.\n%b Locale's abbreviated month name.\n%B Locale's full month name.\n%c Locale's appropriate date and time representation.\n%I Hour (12-hour clock) as a decimal number [01,12].\n%p Locale's equivalent of either AM or PM.\n\nOther codes may be available on your platform. See documentation for\nthe C library strftime function.", - "time.struct_time" => "The time value as returned by gmtime(), localtime(), and strptime(), and\naccepted by asctime(), mktime() and strftime(). May be considered as a\nsequence of 9 integers.\n\nNote that several fields' values are not the same as those defined by\nthe C language standard for struct tm. For example, the value of the\nfield tm_year is the actual year, not year - 1900. See individual\nfields' descriptions for details.", + "time.struct_time" => "The time value as returned by gmtime(), localtime(), and strptime(), and\n accepted by asctime(), mktime() and strftime(). May be considered as a\n sequence of 9 integers.\n\n Note that several fields' values are not the same as those defined by\n the C language standard for struct tm. For example, the value of the\n field tm_year is the actual year, not year - 1900. See individual\n fields' descriptions for details.", "time.struct_time.__add__" => "Return self+value.", "time.struct_time.__class_getitem__" => "See PEP 585", "time.struct_time.__contains__" => "Return bool(key in self).", @@ -9371,7 +15529,7 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "time.time" => "time() -> floating-point number\n\nReturn the current time in seconds since the Epoch.\nFractions of a second may be present if the system clock provides them.", "time.time_ns" => "time_ns() -> int\n\nReturn the current time in nanoseconds since the Epoch.", "time.tzset" => "tzset()\n\nInitialize, or reinitialize, the local timezone to the value stored in\nos.environ['TZ']. The TZ environment variable should be specified in\nstandard Unix timezone format as documented in the tzset man page\n(eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently\nfall back to UTC. If the TZ environment variable is not set, the local\ntimezone is set to the systems best guess of wallclock time.\nChanging the TZ environment variable without calling tzset *may* change\nthe local timezone used by methods such as localtime, but this behaviour\nshould not be relied on.", - "unicodedata" => "This module provides access to the Unicode Character Database which\ndefines character properties for all Unicode characters. The data in\nthis database is based on the UnicodeData.txt file version\n15.1.0 which is publicly available from ftp://ftp.unicode.org/.\n\nThe module uses the same names and symbols as defined by the\nUnicodeData File Format 15.1.0.", + "unicodedata" => "This module provides access to the Unicode Character Database which\ndefines character properties for all Unicode characters. The data in\nthis database is based on the UnicodeData.txt file version\n16.0.0 which is publicly available from ftp://ftp.unicode.org/.\n\nThe module uses the same names and symbols as defined by the\nUnicodeData File Format 16.0.0.", "unicodedata.UCD.__delattr__" => "Implement delattr(self, name).", "unicodedata.UCD.__eq__" => "Return self==value.", "unicodedata.UCD.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -9419,21 +15577,21 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "unicodedata.name" => "Returns the name assigned to the character chr as a string.\n\nIf no name is defined, default is returned, or, if not given,\nValueError is raised.", "unicodedata.normalize" => "Return the normal form 'form' for the Unicode string unistr.\n\nValid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.", "unicodedata.numeric" => "Converts a Unicode character into its equivalent numeric value.\n\nReturns the numeric value assigned to the character chr as float.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", - "winreg" => "This module provides access to the Windows registry API.\n\nFunctions:\n\nCloseKey() - Closes a registry key.\nConnectRegistry() - Establishes a connection to a predefined registry handle\n on another computer.\nCreateKey() - Creates the specified key, or opens it if it already exists.\nDeleteKey() - Deletes the specified key.\nDeleteValue() - Removes a named value from the specified registry key.\nEnumKey() - Enumerates subkeys of the specified open registry key.\nEnumValue() - Enumerates values of the specified open registry key.\nExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n string.\nFlushKey() - Writes all the attributes of the specified key to the registry.\nLoadKey() - Creates a subkey under HKEY_USER or HKEY_LOCAL_MACHINE and\n stores registration information from a specified file into that\n subkey.\nOpenKey() - Opens the specified key.\nOpenKeyEx() - Alias of OpenKey().\nQueryValue() - Retrieves the value associated with the unnamed value for a\n specified key in the registry.\nQueryValueEx() - Retrieves the type and data for a specified value name\n associated with an open registry key.\nQueryInfoKey() - Returns information about the specified key.\nSaveKey() - Saves the specified key, and all its subkeys a file.\nSetValue() - Associates a value with a specified key.\nSetValueEx() - Stores data in the value field of an open registry key.\n\nSpecial objects:\n\nHKEYType -- type object for HKEY objects\nerror -- exception raised for Win32 errors\n\nInteger constants:\nMany constants are defined - see the documentation for each function\nto see what constants are used, and where.", + "winreg" => "This module provides access to the Windows registry API.\n\nFunctions:\n\nCloseKey() - Closes a registry key.\nConnectRegistry() - Establishes a connection to a predefined registry handle\n on another computer.\nCreateKey() - Creates the specified key, or opens it if it already exists.\nDeleteKey() - Deletes the specified key.\nDeleteValue() - Removes a named value from the specified registry key.\nDeleteTree() - Deletes the specified key and all its subkeys and values recursively.\nEnumKey() - Enumerates subkeys of the specified open registry key.\nEnumValue() - Enumerates values of the specified open registry key.\nExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n string.\nFlushKey() - Writes all the attributes of the specified key to the registry.\nLoadKey() - Creates a subkey under HKEY_USER or HKEY_LOCAL_MACHINE and\n stores registration information from a specified file into that\n subkey.\nOpenKey() - Opens the specified key.\nOpenKeyEx() - Alias of OpenKey().\nQueryValue() - Retrieves the value associated with the unnamed value for a\n specified key in the registry.\nQueryValueEx() - Retrieves the type and data for a specified value name\n associated with an open registry key.\nQueryInfoKey() - Returns information about the specified key.\nSaveKey() - Saves the specified key, and all its subkeys a file.\nSetValue() - Associates a value with a specified key.\nSetValueEx() - Stores data in the value field of an open registry key.\n\nSpecial objects:\n\nHKEYType -- type object for HKEY objects\nerror -- exception raised for Win32 errors\n\nInteger constants:\nMany constants are defined - see the documentation for each function\nto see what constants are used, and where.", "winreg.CloseKey" => "Closes a previously opened registry key.\n\n hkey\n A previously opened key.\n\nNote that if the key is not closed using this method, it will be\nclosed when the hkey object is destroyed by Python.", "winreg.ConnectRegistry" => "Establishes a connection to the registry on another computer.\n\n computer_name\n The name of the remote computer, of the form r\"\\\\computername\". If\n None, the local computer is used.\n key\n The predefined key to connect to.\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", "winreg.CreateKey" => "Creates or opens the specified key.\n\n key\n An already open key, or one of the predefined HKEY_* constants.\n sub_key\n The name of the key this method opens or creates.\n\nIf key is one of the predefined keys, sub_key may be None. In that case,\nthe handle returned is the same key handle passed in to the function.\n\nIf the key already exists, this function opens the existing key.\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", "winreg.CreateKeyEx" => "Creates or opens the specified key.\n\n key\n An already open key, or one of the predefined HKEY_* constants.\n sub_key\n The name of the key this method opens or creates.\n reserved\n A reserved integer, and must be zero. Default is zero.\n access\n An integer that specifies an access mask that describes the\n desired security access for the key. Default is KEY_WRITE.\n\nIf key is one of the predefined keys, sub_key may be None. In that case,\nthe handle returned is the same key handle passed in to the function.\n\nIf the key already exists, this function opens the existing key\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", "winreg.DeleteKey" => "Deletes the specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that must be the name of a subkey of the key identified by\n the key parameter. This value must not be None, and the key may not\n have subkeys.\n\nThis method can not delete keys with subkeys.\n\nIf the function succeeds, the entire key, including all of its values,\nis removed. If the function fails, an OSError exception is raised.", "winreg.DeleteKeyEx" => "Deletes the specified key (intended for 64-bit OS).\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that must be the name of a subkey of the key identified by\n the key parameter. This value must not be None, and the key may not\n have subkeys.\n access\n An integer that specifies an access mask that describes the\n desired security access for the key. Default is KEY_WOW64_64KEY.\n reserved\n A reserved integer, and must be zero. Default is zero.\n\nWhile this function is intended to be used for 64-bit OS, it is also\n available on 32-bit systems.\n\nThis method can not delete keys with subkeys.\n\nIf the function succeeds, the entire key, including all of its values,\nis removed. If the function fails, an OSError exception is raised.\nOn unsupported Windows versions, NotImplementedError is raised.", - "winreg.DeleteValue" => "Removes a named value from a registry key.\n\nkey\n An already open key, or any one of the predefined HKEY_* constants.\nvalue\n A string that identifies the value to remove.", + "winreg.DeleteValue" => "Removes a named value from a registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n value\n A string that identifies the value to remove.", "winreg.DisableReflectionKey" => "Disables registry reflection for 32bit processes running on a 64bit OS.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nWill generally raise NotImplementedError if executed on a 32bit OS.\n\nIf the key is not on the reflection list, the function succeeds but has\nno effect. Disabling reflection for a key does not affect reflection\nof any subkeys.", "winreg.EnableReflectionKey" => "Restores registry reflection for the specified disabled key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nWill generally raise NotImplementedError if executed on a 32bit OS.\nRestoring reflection for a key does not affect reflection of any\nsubkeys.", "winreg.EnumKey" => "Enumerates subkeys of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n index\n An integer that identifies the index of the key to retrieve.\n\nThe function retrieves the name of one subkey each time it is called.\nIt is typically called repeatedly until an OSError exception is\nraised, indicating no more values are available.", "winreg.EnumValue" => "Enumerates values of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n index\n An integer that identifies the index of the value to retrieve.\n\nThe function retrieves the name of one subkey each time it is called.\nIt is typically called repeatedly, until an OSError exception\nis raised, indicating no more values.\n\nThe result is a tuple of 3 items:\n value_name\n A string that identifies the value.\n value_data\n An object that holds the value data, and whose type depends\n on the underlying registry type.\n data_type\n An integer that identifies the type of the value data.", "winreg.ExpandEnvironmentStrings" => "Expand environment vars.", "winreg.FlushKey" => "Writes all the attributes of a key to the registry.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nIt is not necessary to call FlushKey to change a key. Registry changes\nare flushed to disk by the registry using its lazy flusher. Registry\nchanges are also flushed to disk at system shutdown. Unlike\nCloseKey(), the FlushKey() method returns only when all the data has\nbeen written to the registry.\n\nAn application should only call FlushKey() if it requires absolute\ncertainty that registry changes are on disk. If you don't know whether\na FlushKey() call is required, it probably isn't.", - "winreg.HKEYType" => "PyHKEY Object - A Python object, representing a win32 registry key.\n\nThis object wraps a Windows HKEY object, automatically closing it when\nthe object is destroyed. To guarantee cleanup, you can call either\nthe Close() method on the PyHKEY, or the CloseKey() method.\n\nAll functions which accept a handle object also accept an integer --\nhowever, use of the handle object is encouraged.\n\nFunctions:\nClose() - Closes the underlying handle.\nDetach() - Returns the integer Win32 handle, detaching it from the object\n\nProperties:\nhandle - The integer Win32 handle.\n\nOperations:\n__bool__ - Handles with an open object return true, otherwise false.\n__int__ - Converting a handle to an integer returns the Win32 handle.\nrich comparison - Handle objects are compared using the handle value.", + "winreg.HKEYType" => "PyHKEY Object - A Python object, representing a win32 registry key.\n\nThis object wraps a Windows HKEY object, automatically closing it when\nthe object is destroyed. To guarantee cleanup, you can call either\nthe Close() method on the PyHKEY, or the CloseKey() method.\n\nAll functions which accept a handle object also accept an integer --\nhowever, use of the handle object is encouraged.\n\nFunctions:\nClose() - Closes the underlying handle.\nDetach() - Returns the integer Win32 handle, detaching it from the object\n\nProperties:\nhandle - The integer Win32 handle.\n\nOperations:\n__bool__ - Handles with an open object return true, otherwise false.\n__int__ - Converting a handle to an integer returns the Win32 handle.\n__enter__, __exit__ - Context manager support for 'with' statement,\nautomatically closes handle.", "winreg.HKEYType.Close" => "Closes the underlying Windows handle.\n\nIf the handle is already closed, no error is raised.", "winreg.HKEYType.Detach" => "Detaches the Windows handle from the handle object.\n\nThe result is the value of the handle before it is detached. If the\nhandle is already detached, this will return zero.\n\nAfter calling this function, the handle is effectively invalidated,\nbut the handle is not closed. You would call this function when you\nneed the underlying win32 handle to exist beyond the lifetime of the\nhandle object.", "winreg.HKEYType.__abs__" => "abs(self)", @@ -9496,12 +15654,12 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "winreg.SaveKey" => "Saves the specified key, and all its subkeys to the specified file.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n file_name\n The name of the file to save registry data to. This file cannot\n already exist. If this filename includes an extension, it cannot be\n used on file allocation table (FAT) file systems by the LoadKey(),\n ReplaceKey() or RestoreKey() methods.\n\nIf key represents a key on a remote computer, the path described by\nfile_name is relative to the remote computer.\n\nThe caller of this method must possess the SeBackupPrivilege\nsecurity privilege. This function passes NULL for security_attributes\nto the API.", "winreg.SetValue" => "Associates a value with a specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that names the subkey with which the value is associated.\n type\n An integer that specifies the type of the data. Currently this must\n be REG_SZ, meaning only strings are supported.\n value\n A string that specifies the new value.\n\nIf the key specified by the sub_key parameter does not exist, the\nSetValue function creates it.\n\nValue lengths are limited by available memory. Long values (more than\n2048 bytes) should be stored as files with the filenames stored in\nthe configuration registry to help the registry perform efficiently.\n\nThe key identified by the key parameter must have been opened with\nKEY_SET_VALUE access.", "winreg.SetValueEx" => "Stores data in the value field of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n value_name\n A string containing the name of the value to set, or None.\n reserved\n Can be anything - zero is always passed to the API.\n type\n An integer that specifies the type of the data, one of:\n REG_BINARY -- Binary data in any form.\n REG_DWORD -- A 32-bit number.\n REG_DWORD_LITTLE_ENDIAN -- A 32-bit number in little-endian format. Equivalent to REG_DWORD\n REG_DWORD_BIG_ENDIAN -- A 32-bit number in big-endian format.\n REG_EXPAND_SZ -- A null-terminated string that contains unexpanded\n references to environment variables (for example,\n %PATH%).\n REG_LINK -- A Unicode symbolic link.\n REG_MULTI_SZ -- A sequence of null-terminated strings, terminated\n by two null characters. Note that Python handles\n this termination automatically.\n REG_NONE -- No defined value type.\n REG_QWORD -- A 64-bit number.\n REG_QWORD_LITTLE_ENDIAN -- A 64-bit number in little-endian format. Equivalent to REG_QWORD.\n REG_RESOURCE_LIST -- A device-driver resource list.\n REG_SZ -- A null-terminated string.\n value\n A string that specifies the new value.\n\nThis method can also set additional value and type information for the\nspecified key. The key identified by the key parameter must have been\nopened with KEY_SET_VALUE access.\n\nTo open the key, use the CreateKeyEx() or OpenKeyEx() methods.\n\nValue lengths are limited by available memory. Long values (more than\n2048 bytes) should be stored as files with the filenames stored in\nthe configuration registry to help the registry perform efficiently.", - "winsound" => "PlaySound(sound, flags) - play a sound\nSND_FILENAME - sound is a wav file name\nSND_ALIAS - sound is a registry sound association name\nSND_LOOP - Play the sound repeatedly; must also specify SND_ASYNC\nSND_MEMORY - sound is a memory image of a wav file\nSND_PURGE - stop all instances of the specified sound\nSND_ASYNC - PlaySound returns immediately\nSND_NODEFAULT - Do not play a default beep if the sound can not be found\nSND_NOSTOP - Do not interrupt any sounds currently playing\nSND_NOWAIT - Return immediately if the sound driver is busy\nSND_APPLICATION - sound is an application-specific alias in the registry.\nBeep(frequency, duration) - Make a beep through the PC speaker.\nMessageBeep(type) - Call Windows MessageBeep.", - "winsound.Beep" => "A wrapper around the Windows Beep API.\n\nfrequency\n Frequency of the sound in hertz.\n Must be in the range 37 through 32,767.\nduration\n How long the sound should play, in milliseconds.", + "winsound" => "PlaySound(sound, flags) - play a sound\nSND_FILENAME - sound is a wav file name\nSND_ALIAS - sound is a registry sound association name\nSND_LOOP - Play the sound repeatedly; must also specify SND_ASYNC\nSND_MEMORY - sound is a memory image of a wav file\nSND_PURGE - stop all instances of the specified sound\nSND_ASYNC - PlaySound returns immediately\nSND_NODEFAULT - Do not play a default beep if the sound can not be found\nSND_NOSTOP - Do not interrupt any sounds currently playing\nSND_NOWAIT - Return immediately if the sound driver is busy\nSND_APPLICATION - sound is an application-specific alias in the registry.\nSND_SENTRY - Triggers a SoundSentry event when the sound is played.\nSND_SYNC - Play the sound synchronously, default behavior.\nSND_SYSTEM - Assign sound to the audio session for system notification sounds.\n\nBeep(frequency, duration) - Make a beep through the PC speaker.\nMessageBeep(type) - Call Windows MessageBeep.", + "winsound.Beep" => "A wrapper around the Windows Beep API.\n\n frequency\n Frequency of the sound in hertz.\n Must be in the range 37 through 32,767.\n duration\n How long the sound should play, in milliseconds.", "winsound.MessageBeep" => "Call Windows MessageBeep(x).\n\nx defaults to MB_OK.", - "winsound.PlaySound" => "A wrapper around the Windows PlaySound API.\n\nsound\n The sound to play; a filename, data, or None.\nflags\n Flag values, ored together. See module documentation.", + "winsound.PlaySound" => "A wrapper around the Windows PlaySound API.\n\n sound\n The sound to play; a filename, data, or None.\n flags\n Flag values, ored together. See module documentation.", "zlib" => "The functions in this module allow compression and decompression using the\nzlib library, which is based on GNU zip.\n\nadler32(string[, start]) -- Compute an Adler-32 checksum.\ncompress(data[, level]) -- Compress data, with compression level 0-9 or -1.\ncompressobj([level[, ...]]) -- Return a compressor object.\ncrc32(string[, start]) -- Compute a CRC-32 checksum.\ndecompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\ndecompressobj([wbits[, zdict]]) -- Return a decompressor object.\n\n'wbits' is window buffer size and container format.\nCompressor objects support compress() and flush() methods; decompressor\nobjects support decompress() and flush().", - "zlib._ZlibDecompressor" => "Create a decompressor object for decompressing data incrementally.\n\nwbits = 15\nzdict\n The predefined compression dictionary. This is a sequence of bytes\n (such as a bytes object) containing subsequences that are expected\n to occur frequently in the data that is to be compressed. Those\n subsequences that are expected to be most common should come at the\n end of the dictionary. This must be the same dictionary as used by the\n compressor that produced the input data.", + "zlib._ZlibDecompressor" => "Create a decompressor object for decompressing data incrementally.\n\n wbits = 15\n zdict\n The predefined compression dictionary. This is a sequence of bytes\n (such as a bytes object) containing subsequences that are expected\n to occur frequently in the data that is to be compressed. Those\n subsequences that are expected to be most common should come at the\n end of the dictionary. This must be the same dictionary as used by the\n compressor that produced the input data.", "zlib._ZlibDecompressor.__delattr__" => "Implement delattr(self, name).", "zlib._ZlibDecompressor.__eq__" => "Return self==value.", "zlib._ZlibDecompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -9528,13 +15686,11 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "zlib._ZlibDecompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", "zlib._ZlibDecompressor.unused_data" => "Data found after the end of the compressed stream.", "zlib.adler32" => "Compute an Adler-32 checksum of data.\n\n value\n Starting value of the checksum.\n\nThe returned checksum is an integer.", - "zlib.compress" => "Returns a bytes object containing compressed data.\n\ndata\n Binary data to be compressed.\nlevel\n Compression level, in 0-9 or -1.\nwbits\n The window buffer size and container format.", - "zlib.compressobj" => "Return a compressor object.\n\nlevel\n The compression level (an integer in the range 0-9 or -1; default is\n currently equivalent to 6). Higher compression levels are slower,\n but produce smaller results.\nmethod\n The compression algorithm. If given, this must be DEFLATED.\nwbits\n +9 to +15: The base-two logarithm of the window size. Include a zlib\n container.\n -9 to -15: Generate a raw stream.\n +25 to +31: Include a gzip container.\nmemLevel\n Controls the amount of memory used for internal compression state.\n Valid values range from 1 to 9. Higher values result in higher memory\n usage, faster compression, and smaller output.\nstrategy\n Used to tune the compression algorithm. Possible values are\n Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY.\nzdict\n The predefined compression dictionary - a sequence of bytes\n containing subsequences that are likely to occur in the input data.", + "zlib.compress" => "Returns a bytes object containing compressed data.\n\n data\n Binary data to be compressed.\n level\n Compression level, in 0-9 or -1.\n wbits\n The window buffer size and container format.", + "zlib.compressobj" => "Return a compressor object.\n\n level\n The compression level (an integer in the range 0-9 or -1; default is\n currently equivalent to 6). Higher compression levels are slower,\n but produce smaller results.\n method\n The compression algorithm. If given, this must be DEFLATED.\n wbits\n +9 to +15: The base-two logarithm of the window size. Include a zlib\n container.\n -9 to -15: Generate a raw stream.\n +25 to +31: Include a gzip container.\n memLevel\n Controls the amount of memory used for internal compression state.\n Valid values range from 1 to 9. Higher values result in higher memory\n usage, faster compression, and smaller output.\n strategy\n Used to tune the compression algorithm. Possible values are\n Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY.\n zdict\n The predefined compression dictionary - a sequence of bytes\n containing subsequences that are likely to occur in the input data.", "zlib.crc32" => "Compute a CRC-32 checksum of data.\n\n value\n Starting value of the checksum.\n\nThe returned checksum is an integer.", - "zlib.decompress" => "Returns a bytes object containing the uncompressed data.\n\ndata\n Compressed data.\nwbits\n The window buffer size and container format.\nbufsize\n The initial output buffer size.", - "zlib.decompressobj" => "Return a decompressor object.\n\nwbits\n The window buffer size and container format.\nzdict\n The predefined compression dictionary. This must be the same\n dictionary as used by the compressor that produced the input data.", - "zlib.error.__cause__" => "exception cause", - "zlib.error.__context__" => "exception context", + "zlib.decompress" => "Returns a bytes object containing the uncompressed data.\n\n data\n Compressed data.\n wbits\n The window buffer size and container format.\n bufsize\n The initial output buffer size.", + "zlib.decompressobj" => "Return a decompressor object.\n\n wbits\n The window buffer size and container format.\n zdict\n The predefined compression dictionary. This must be the same\n dictionary as used by the compressor that produced the input data.", "zlib.error.__delattr__" => "Implement delattr(self, name).", "zlib.error.__eq__" => "Return self==value.", "zlib.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", @@ -9556,6 +15712,6 @@ pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { "zlib.error.__str__" => "Return str(self).", "zlib.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", "zlib.error.__weakref__" => "list of weak references to the object", - "zlib.error.add_note" => "Exception.add_note(note) --\nadd a note to the exception", - "zlib.error.with_traceback" => "Exception.with_traceback(tb) --\nset self.__traceback__ to tb and return self.", + "zlib.error.add_note" => "Add a note to the exception", + "zlib.error.with_traceback" => "Set self.__traceback__ to tb and return self.", }; diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index cc81f17fc55..8b7f5d8f75b 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -1,3 +1,5 @@ +#![no_std] + include!("./data.inc.rs"); #[cfg(test)] diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index a79e48bac2d..8c1c871d0bc 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -17,12 +17,13 @@ num-traits = { workspace = true } thiserror = { workspace = true } libffi = { workspace = true } -cranelift = "0.126" -cranelift-jit = "0.126" -cranelift-module = "0.126" +cranelift = "0.128.3" +cranelift-jit = "0.128.3" +cranelift-module = "0.128.3" [dev-dependencies] rustpython-derive = { workspace = true } +rustpython-wtf8 = { workspace = true } approx = "0.5.1" diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs index 8a70b9a73cb..19931038fe0 100644 --- a/crates/jit/src/instructions.rs +++ b/crates/jit/src/instructions.rs @@ -4,8 +4,8 @@ use cranelift::codegen::ir::FuncRef; use cranelift::prelude::*; use num_traits::cast::ToPrimitive; use rustpython_compiler_core::bytecode::{ - self, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator, Instruction, Label, - OpArg, OpArgState, UnaryOperator, + self, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator, Instruction, + IntrinsicFunction1, Label, OpArg, OpArgState, }; use std::collections::HashMap; @@ -27,6 +27,7 @@ enum JitValue { Float(Value), Bool(Value), None, + Null, Tuple(Vec<JitValue>), FuncRef(FuncRef), } @@ -45,14 +46,14 @@ impl JitValue { JitValue::Int(_) => Some(JitType::Int), JitValue::Float(_) => Some(JitType::Float), JitValue::Bool(_) => Some(JitType::Bool), - JitValue::None | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, + JitValue::None | JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, } } fn into_value(self) -> Option<Value> { match self { JitValue::Int(val) | JitValue::Float(val) | JitValue::Bool(val) => Some(val), - JitValue::None | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, + JitValue::None | JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, } } } @@ -139,7 +140,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } JitValue::Bool(val) => Ok(val), JitValue::None => Ok(self.builder.ins().iconst(types::I8, 0)), - JitValue::Tuple(_) | JitValue::FuncRef(_) => Err(JitCompileError::NotSupported), + JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => { + Err(JitCompileError::NotSupported) + } } } @@ -172,12 +175,21 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { // Create or get the block for this label: let target_block = self.get_or_create_block(label); - // If the current block isn't terminated, jump: + // If the current block isn't terminated, add a fallthrough jump if let Some(cur) = self.builder.current_block() && cur != target_block - && self.builder.func.layout.last_inst(cur).is_none() { - self.builder.ins().jump(target_block, &[]); + // Check if the block needs a terminator by examining the last instruction + let needs_terminator = match self.builder.func.layout.last_inst(cur) { + None => true, // Empty block needs terminator + Some(inst) => { + // Check if the last instruction is a terminator + !self.builder.func.dfg.insts[inst].opcode().is_terminator() + } + }; + if needs_terminator { + self.builder.ins().jump(target_block, &[]); + } } // Switch to the target block if self.builder.current_block() != Some(target_block) { @@ -196,20 +208,27 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { // Actually compile this instruction: self.add_instruction(func_ref, bytecode, instruction, arg)?; - // If that was a return instruction, mark future instructions unreachable + // If that was an unconditional branch or return, mark future instructions unreachable match instruction { - Instruction::ReturnValue | Instruction::ReturnConst { .. } => { + Instruction::ReturnValue + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpForward { .. } => { in_unreachable_code = true; } _ => {} } } - // After processing, if the current block is unterminated, insert a trap or fallthrough - if let Some(cur) = self.builder.current_block() - && self.builder.func.layout.last_inst(cur).is_none() - { - self.builder.ins().trap(TrapCode::user(0).unwrap()); + // After processing, if the current block is unterminated, insert a trap + if let Some(cur) = self.builder.current_block() { + let needs_terminator = match self.builder.func.layout.last_inst(cur) { + None => true, + Some(inst) => !self.builder.func.dfg.insts[inst].opcode().is_terminator(), + }; + if needs_terminator { + self.builder.ins().trap(TrapCode::user(0).unwrap()); + } } Ok(()) } @@ -439,7 +458,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { self.stack.push(JitValue::Tuple(elements)); Ok(()) } - Instruction::CallFunctionPositional { nargs } => { + Instruction::Call { nargs } => { let nargs = nargs.get(arg); let mut args = Vec::new(); @@ -448,6 +467,12 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { args.push(arg.into_value().unwrap()); } + // Pop self_or_null (should be Null for JIT-compiled recursive calls) + let self_or_null = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + if !matches!(self_or_null, JitValue::Null) { + return Err(JitCompileError::NotSupported); + } + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { JitValue::FuncRef(reference) => { let call = self.builder.ins().call(reference, &args); @@ -459,7 +484,26 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { _ => Err(JitCompileError::BadBytecode), } } - Instruction::CompareOperation { op, .. } => { + Instruction::PushNull => { + self.stack.push(JitValue::Null); + Ok(()) + } + Instruction::CallIntrinsic1 { func } => { + match func.get(arg) { + IntrinsicFunction1::UnaryPositive => { + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Int(val) => { + // Nothing to do + self.stack.push(JitValue::Int(val)); + Ok(()) + } + _ => Err(JitCompileError::NotSupported), + } + } + _ => Err(JitCompileError::NotSupported), + } + } + Instruction::CompareOp { op, .. } => { let op = op.get(arg); // the rhs is popped off first let b = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; @@ -515,7 +559,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } Instruction::ExtendedArg => Ok(()), - Instruction::Jump { target } => { + Instruction::JumpBackward { target } + | Instruction::JumpBackwardNoInterrupt { target } + | Instruction::JumpForward { target } => { let target_block = self.get_or_create_block(target.get(arg)); self.builder.ins().jump(target_block, &[]); Ok(()) @@ -526,7 +572,13 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { self.stack.push(val); Ok(()) } - Instruction::LoadFast(idx) => { + Instruction::LoadSmallInt { idx } => { + let small_int = idx.get(arg) as i64; + let val = self.builder.ins().iconst(types::I64, small_int); + self.stack.push(JitValue::Int(val)); + Ok(()) + } + Instruction::LoadFast(idx) | Instruction::LoadFastBorrow(idx) => { let local = self.variables[idx.get(arg) as usize] .as_ref() .ok_or(JitCompileError::BadBytecode)?; @@ -536,6 +588,22 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { )); Ok(()) } + Instruction::LoadFastLoadFast { arg: packed } + | Instruction::LoadFastBorrowLoadFastBorrow { arg: packed } => { + let oparg = packed.get(arg); + let idx1 = oparg >> 4; + let idx2 = oparg & 0xF; + for idx in [idx1, idx2] { + let local = self.variables[idx as usize] + .as_ref() + .ok_or(JitCompileError::BadBytecode)?; + self.stack.push(JitValue::from_type_and_value( + local.ty.clone(), + self.builder.use_var(local.var), + )); + } + Ok(()) + } Instruction::LoadGlobal(idx) => { let name = &bytecode.names[idx.get(arg) as usize]; @@ -547,14 +615,6 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } } Instruction::Nop => Ok(()), - Instruction::Pop => { - self.stack.pop(); - Ok(()) - } - Instruction::PopBlock => { - // TODO: block support - Ok(()) - } Instruction::PopJumpIfFalse { target } => { let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; let val = self.boolean_val(cond)?; @@ -581,25 +641,18 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { Ok(()) } + Instruction::PopTop => { + self.stack.pop(); + Ok(()) + } Instruction::Resume { arg: _resume_arg } => { // TODO: Implement the resume instruction Ok(()) } - Instruction::ReturnConst { idx } => { - let val = self - .prepare_const(bytecode.constants[idx.get(arg) as usize].borrow_constant())?; - self.return_value(val) - } Instruction::ReturnValue => { let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; self.return_value(val) } - Instruction::SetupLoop => { - let loop_head = self.builder.create_block(); - self.builder.ins().jump(loop_head, &[]); - self.builder.switch_to_block(loop_head); - Ok(()) - } Instruction::StoreFast(idx) => { let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; self.store_variable(idx.get(arg), val) @@ -611,28 +664,30 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { self.stack.swap(i, j); Ok(()) } - Instruction::UnaryOperation { op, .. } => { - let op = op.get(arg); + Instruction::ToBool => { let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - match (op, a) { - (UnaryOperator::Minus, JitValue::Int(val)) => { - // Compile minus as 0 - a. + let value = self.boolean_val(a)?; + self.stack.push(JitValue::Bool(value)); + Ok(()) + } + Instruction::UnaryNot => { + let boolean = match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Bool(val) => val, + _ => return Err(JitCompileError::BadBytecode), + }; + let not_boolean = self.builder.ins().bxor_imm(boolean, 1); + self.stack.push(JitValue::Bool(not_boolean)); + Ok(()) + } + Instruction::UnaryNegative => { + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Int(val) => { + // Compile minus as 0 - val. let zero = self.builder.ins().iconst(types::I64, 0); let out = self.compile_sub(zero, val); self.stack.push(JitValue::Int(out)); Ok(()) } - (UnaryOperator::Plus, JitValue::Int(val)) => { - // Nothing to do - self.stack.push(JitValue::Int(val)); - Ok(()) - } - (UnaryOperator::Not, a) => { - let boolean = self.boolean_val(a)?; - let not_boolean = self.builder.ins().bxor_imm(boolean, 1); - self.stack.push(JitValue::Bool(not_boolean)); - Ok(()) - } _ => Err(JitCompileError::NotSupported), } } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 91911fd8d14..1e278617661 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -1,11 +1,14 @@ mod instructions; +extern crate alloc; + +use alloc::fmt; +use core::mem::ManuallyDrop; use cranelift::prelude::*; use cranelift_jit::{JITBuilder, JITModule}; use cranelift_module::{FuncId, Linkage, Module, ModuleError}; use instructions::FunctionCompiler; use rustpython_compiler_core::bytecode; -use std::{fmt, mem::ManuallyDrop}; #[derive(Debug, thiserror::Error)] #[non_exhaustive] @@ -157,7 +160,7 @@ impl CompiledCode { Ok(unsafe { self.invoke_raw(&cif_args) }) } - unsafe fn invoke_raw(&self, cif_args: &[libffi::middle::Arg]) -> Option<AbiValue> { + unsafe fn invoke_raw(&self, cif_args: &[libffi::middle::Arg<'_>]) -> Option<AbiValue> { unsafe { let cif = self.sig.to_cif(); let value = cif.call::<UnTypedAbiValue>( @@ -219,7 +222,7 @@ pub enum AbiValue { } impl AbiValue { - fn to_libffi_arg(&self) -> libffi::middle::Arg { + fn to_libffi_arg(&self) -> libffi::middle::Arg<'_> { match self { AbiValue::Int(i) => libffi::middle::Arg::new(i), AbiValue::Float(f) => libffi::middle::Arg::new(f), @@ -350,26 +353,25 @@ impl<'a> ArgsBuilder<'a> { } pub fn into_args(self) -> Option<Args<'a>> { - self.values - .iter() - .map(|v| v.as_ref().map(AbiValue::to_libffi_arg)) - .collect::<Option<_>>() - .map(|cif_args| Args { - _values: self.values, - cif_args, - code: self.code, - }) + // Ensure all values are set + if self.values.iter().any(|v| v.is_none()) { + return None; + } + Some(Args { + values: self.values.into_iter().map(|v| v.unwrap()).collect(), + code: self.code, + }) } } pub struct Args<'a> { - _values: Vec<Option<AbiValue>>, - cif_args: Vec<libffi::middle::Arg>, + values: Vec<AbiValue>, code: &'a CompiledCode, } impl Args<'_> { pub fn invoke(&self) -> Option<AbiValue> { - unsafe { self.code.invoke_raw(&self.cif_args) } + let cif_args: Vec<_> = self.values.iter().map(AbiValue::to_libffi_arg).collect(); + unsafe { self.code.invoke_raw(&cif_args) } } } diff --git a/crates/jit/tests/common.rs b/crates/jit/tests/common.rs index 5dafeaeb807..a862d9eef69 100644 --- a/crates/jit/tests/common.rs +++ b/crates/jit/tests/common.rs @@ -1,21 +1,22 @@ +use core::ops::ControlFlow; use rustpython_compiler_core::bytecode::{ CodeObject, ConstantData, Instruction, OpArg, OpArgState, }; use rustpython_jit::{CompiledCode, JitType}; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; use std::collections::HashMap; -use std::ops::ControlFlow; #[derive(Debug, Clone)] pub struct Function { code: Box<CodeObject>, - annotations: HashMap<String, StackValue>, + annotations: HashMap<Wtf8Buf, StackValue>, } impl Function { pub fn compile(self) -> CompiledCode { let mut arg_types = Vec::new(); for arg in self.code.arg_names().args { - let arg_type = match self.annotations.get(arg) { + let arg_type = match self.annotations.get(AsRef::<Wtf8>::as_ref(arg.as_str())) { Some(StackValue::String(annotation)) => match annotation.as_str() { "int" => JitType::Int, "float" => JitType::Float, @@ -27,7 +28,7 @@ impl Function { arg_types.push(arg_type); } - let ret_type = match self.annotations.get("return") { + let ret_type = match self.annotations.get(AsRef::<Wtf8>::as_ref("return")) { Some(StackValue::String(annotation)) => match annotation.as_str() { "int" => Some(JitType::Int), "float" => Some(JitType::Float), @@ -45,7 +46,7 @@ impl Function { enum StackValue { String(String), None, - Map(HashMap<String, StackValue>), + Map(HashMap<Wtf8Buf, StackValue>), Code(Box<CodeObject>), Function(Function), } @@ -63,6 +64,97 @@ impl From<ConstantData> for StackValue { } } +/// Extract annotations from an annotate function's bytecode. +/// The annotate function uses BUILD_MAP with key-value pairs loaded before it. +/// Keys are parameter names (from LOAD_CONST), values are type names (from LOAD_NAME/LOAD_GLOBAL). +fn extract_annotations_from_annotate_code(code: &CodeObject) -> HashMap<Wtf8Buf, StackValue> { + let mut annotations = HashMap::new(); + let mut stack: Vec<(bool, usize)> = Vec::new(); // (is_const, index) + let mut op_arg_state = OpArgState::default(); + + for &word in code.instructions.iter() { + let (instruction, arg) = op_arg_state.get(word); + + match instruction { + Instruction::LoadConst { idx } => { + stack.push((true, idx.get(arg) as usize)); + } + Instruction::LoadName(idx) | Instruction::LoadGlobal(idx) => { + stack.push((false, idx.get(arg) as usize)); + } + Instruction::BuildMap { size, .. } => { + let count = size.get(arg) as usize; + // Stack has key-value pairs in order: k1, v1, k2, v2, ... + // So we need count * 2 items from the stack + let start = stack.len().saturating_sub(count * 2); + let pairs: Vec<_> = stack.drain(start..).collect(); + + for chunk in pairs.chunks(2) { + if chunk.len() == 2 { + let (key_is_const, key_idx) = chunk[0]; + let (val_is_const, val_idx) = chunk[1]; + + // Key should be a const string (parameter name) + if key_is_const + && let ConstantData::Str { value } = &code.constants[key_idx] + { + let param_name = value; + // Value can be a name (type ref) or a const string (forward ref) + let type_name = if val_is_const { + match code.constants.get(val_idx) { + Some(ConstantData::Str { value }) => value + .as_str() + .map(|s| s.to_owned()) + .unwrap_or_else(|_| value.to_string_lossy().into_owned()), + Some(other) => panic!( + "Unsupported annotation const for '{:?}' at idx {}: {:?}", + param_name, val_idx, other + ), + None => panic!( + "Annotation const idx out of bounds for '{:?}': {} (len={})", + param_name, + val_idx, + code.constants.len() + ), + } + } else { + match code.names.get(val_idx) { + Some(name) => name.clone(), + None => panic!( + "Annotation name idx out of bounds for '{:?}': {} (len={})", + param_name, + val_idx, + code.names.len() + ), + } + }; + annotations.insert(param_name.clone(), StackValue::String(type_name)); + } + } + } + // Return after processing BUILD_MAP - we got our annotations + return annotations; + } + Instruction::Resume { .. } + | Instruction::LoadFast(_) + | Instruction::CompareOp { .. } + | Instruction::ExtendedArg => { + // Ignore these instructions for annotation extraction + } + Instruction::ReturnValue => { + // End of function - return what we have + return annotations; + } + _ => { + // For other instructions, clear the stack tracking as we don't understand the effect + stack.clear(); + } + } + } + + annotations +} + pub struct StackMachine { stack: Vec<StackValue>, locals: HashMap<String, StackValue>, @@ -92,14 +184,17 @@ impl StackMachine { names: &[String], ) -> ControlFlow<()> { match instruction { + Instruction::Resume { .. } => { + // No-op for JIT tests - just marks function entry point + } Instruction::LoadConst { idx } => { let idx = idx.get(arg); self.stack.push(constants[idx as usize].clone().into()) } - Instruction::LoadNameAny(idx) => self + Instruction::LoadName(idx) => self .stack .push(StackValue::String(names[idx.get(arg) as usize].clone())), - Instruction::StoreLocal(idx) => { + Instruction::StoreName(idx) => { let idx = idx.get(arg); self.locals .insert(names[idx as usize].clone(), self.stack.pop().unwrap()); @@ -114,7 +209,7 @@ impl StackMachine { for _ in 0..size.get(arg) { let value = self.stack.pop().unwrap(); let name = if let Some(StackValue::String(name)) = self.stack.pop() { - name + Wtf8Buf::from(name) } else { unimplemented!("no string keys isn't yet supported in py_function!") }; @@ -143,13 +238,31 @@ impl StackMachine { }; let attr_value = self.stack.pop().expect("Expected attribute value on stack"); - // For now, we only handle ANNOTATIONS flag in JIT tests - if attr - .get(arg) + let flags = attr.get(arg); + + // Handle ANNOTATE flag (PEP 649 style - Python 3.14+) + // The attr_value is a function that returns annotations when called + if flags.contains(rustpython_compiler_core::bytecode::MakeFunctionFlags::ANNOTATE) { + if let StackValue::Function(annotate_func) = attr_value { + // Parse the annotate function's bytecode to extract annotations + // The pattern is: LOAD_CONST (key), LOAD_NAME (value), ... BUILD_MAP + let annotate_code = &annotate_func.code; + let annotations = extract_annotations_from_annotate_code(annotate_code); + + let updated_func = Function { + code: func.code, + annotations, + }; + self.stack.push(StackValue::Function(updated_func)); + } else { + panic!("Expected annotate function for ANNOTATE flag"); + } + } + // Handle old ANNOTATIONS flag (Python 3.12 style) + else if flags .contains(rustpython_compiler_core::bytecode::MakeFunctionFlags::ANNOTATIONS) { if let StackValue::Map(annotations) = attr_value { - // Update function's annotations let updated_func = Function { code: func.code, annotations, @@ -160,15 +273,9 @@ impl StackMachine { } } else { // For other attributes, just push the function back unchanged - // (since JIT tests mainly care about type annotations) self.stack.push(StackValue::Function(func)); } } - Instruction::ReturnConst { idx } => { - let idx = idx.get(arg); - self.stack.push(constants[idx as usize].clone().into()); - return ControlFlow::Break(()); - } Instruction::ReturnValue => return ControlFlow::Break(()), Instruction::ExtendedArg => {} _ => unimplemented!( diff --git a/crates/literal/src/complex.rs b/crates/literal/src/complex.rs index 076f2807c99..c91d1c1439e 100644 --- a/crates/literal/src/complex.rs +++ b/crates/literal/src/complex.rs @@ -1,4 +1,6 @@ use crate::float; +use alloc::borrow::ToOwned; +use alloc::string::{String, ToString}; /// Convert a complex number to a string. pub fn to_string(re: f64, im: f64) -> String { diff --git a/crates/literal/src/escape.rs b/crates/literal/src/escape.rs index 6bdd94e9860..1099c0a02bc 100644 --- a/crates/literal/src/escape.rs +++ b/crates/literal/src/escape.rs @@ -1,3 +1,4 @@ +use alloc::string::String; use rustpython_wtf8::{CodePoint, Wtf8}; #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)] @@ -55,9 +56,9 @@ pub unsafe trait Escape { /// # Safety /// /// This string must only contain printable characters. - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; - fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result; + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result; + fn write_body(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { if self.changed() { self.write_body_slow(formatter) } else { @@ -117,7 +118,7 @@ impl<'a> UnicodeEscape<'a> { pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>); impl StrRepr<'_, '_> { - pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { let quote = self.0.layout().quote.to_char(); formatter.write_char(quote)?; self.0.write_body(formatter)?; @@ -131,8 +132,8 @@ impl StrRepr<'_, '_> { } } -impl std::fmt::Display for StrRepr<'_, '_> { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for StrRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.write(formatter) } } @@ -216,8 +217,8 @@ impl UnicodeEscape<'_> { fn write_char( ch: CodePoint, quote: Quote, - formatter: &mut impl std::fmt::Write, - ) -> std::fmt::Result { + formatter: &mut impl core::fmt::Write, + ) -> core::fmt::Result { let Some(ch) = ch.to_char() else { return write!(formatter, "\\u{:04x}", ch.to_u32()); }; @@ -260,15 +261,15 @@ unsafe impl Escape for UnicodeEscape<'_> { &self.layout } - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { formatter.write_str(unsafe { // SAFETY: this function must be called only when source is printable characters (i.e. no surrogates) - std::str::from_utf8_unchecked(self.source.as_bytes()) + core::str::from_utf8_unchecked(self.source.as_bytes()) }) } #[cold] - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { for ch in self.source.code_points() { Self::write_char(ch, self.layout().quote, formatter)?; } @@ -378,7 +379,11 @@ impl AsciiEscape<'_> { } } - fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_char( + ch: u8, + quote: Quote, + formatter: &mut impl core::fmt::Write, + ) -> core::fmt::Result { match ch { b'\t' => formatter.write_str("\\t"), b'\n' => formatter.write_str("\\n"), @@ -404,15 +409,15 @@ unsafe impl Escape for AsciiEscape<'_> { &self.layout } - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { formatter.write_str(unsafe { // SAFETY: this function must be called only when source is printable ascii characters - std::str::from_utf8_unchecked(self.source) + core::str::from_utf8_unchecked(self.source) }) } #[cold] - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { for ch in self.source { Self::write_char(*ch, self.layout().quote, formatter)?; } @@ -423,7 +428,7 @@ unsafe impl Escape for AsciiEscape<'_> { pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>); impl BytesRepr<'_, '_> { - pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { let quote = self.0.layout().quote.to_char(); formatter.write_char('b')?; formatter.write_char(quote)?; @@ -438,8 +443,8 @@ impl BytesRepr<'_, '_> { } } -impl std::fmt::Display for BytesRepr<'_, '_> { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for BytesRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.write(formatter) } } diff --git a/crates/literal/src/float.rs b/crates/literal/src/float.rs index e2bc54a8f1b..0fc51782438 100644 --- a/crates/literal/src/float.rs +++ b/crates/literal/src/float.rs @@ -1,6 +1,9 @@ use crate::format::Case; +use alloc::borrow::ToOwned; +use alloc::format; +use alloc::string::{String, ToString}; +use core::f64; use num_traits::{Float, Zero}; -use std::f64; pub fn parse_str(literal: &str) -> Option<f64> { parse_inner(literal.trim().as_bytes()) @@ -23,7 +26,7 @@ fn parse_inner(literal: &[u8]) -> Option<f64> { } pub fn is_integer(v: f64) -> bool { - (v - v.round()).abs() < f64::EPSILON + v.is_finite() && v.fract() == 0.0 } fn format_nan(case: Case) -> String { @@ -55,7 +58,7 @@ pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form match magnitude { magnitude if magnitude.is_finite() => { let point = decimal_point_or_empty(precision, alternate_form); - let precision = std::cmp::min(precision, u16::MAX as usize); + let precision = core::cmp::min(precision, u16::MAX as usize); format!("{magnitude:.precision$}{point}") } magnitude if magnitude.is_nan() => format_nan(case), diff --git a/crates/literal/src/lib.rs b/crates/literal/src/lib.rs index 29971070129..a863dd87738 100644 --- a/crates/literal/src/lib.rs +++ b/crates/literal/src/lib.rs @@ -1,3 +1,7 @@ +#![no_std] + +extern crate alloc; + pub mod char; pub mod complex; pub mod escape; diff --git a/crates/pylib/build.rs b/crates/pylib/build.rs index 9b135690f82..f96ef9b477c 100644 --- a/crates/pylib/build.rs +++ b/crates/pylib/build.rs @@ -11,15 +11,25 @@ fn main() { process_python_libs("./Lib/**/*"); } - if cfg!(windows) - && let Ok(real_path) = std::fs::read_to_string("Lib") - { - let canonicalized_path = std::fs::canonicalize(real_path) - .expect("failed to resolve RUSTPYTHONPATH during build time"); - // Strip the extended path prefix (\\?\) that canonicalize adds on Windows - let path_str = canonicalized_path.to_str().unwrap(); - let path_str = path_str.strip_prefix(r"\\?\").unwrap_or(path_str); - println!("cargo:rustc-env=win_lib_path={path_str}"); + if cfg!(windows) { + // On Windows, the Lib entry can be either: + // 1. A text file containing the relative path (git without symlink support) + // 2. A proper symlink (git with symlink support) + // We handle both cases to resolve to the actual Lib directory. + let lib_path = if let Ok(real_path) = std::fs::read_to_string("Lib") { + // Case 1: Text file containing relative path + std::path::PathBuf::from(real_path.trim()) + } else { + // Case 2: Symlink or directory - canonicalize directly + std::path::PathBuf::from("Lib") + }; + + if let Ok(canonicalized_path) = std::fs::canonicalize(&lib_path) { + // Strip the extended path prefix (\\?\) that canonicalize adds on Windows + let path_str = canonicalized_path.to_str().unwrap(); + let path_str = path_str.strip_prefix(r"\\?\").unwrap_or(path_str); + println!("cargo:rustc-env=win_lib_path={path_str}"); + } } } diff --git a/crates/pylib/src/lib.rs b/crates/pylib/src/lib.rs index f8a47ba67da..db957f633fc 100644 --- a/crates/pylib/src/lib.rs +++ b/crates/pylib/src/lib.rs @@ -2,6 +2,8 @@ //! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`, //! in order to automatically include the python part of the standard library into the binary. +#![no_std] + // windows needs to read the symlink out of `Lib` as git turns it into a text file, // so build.rs sets this env var pub const LIB_PATH: &str = match option_env!("win_lib_path") { @@ -11,4 +13,4 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") { #[cfg(feature = "freeze-stdlib")] pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = - rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core"); + rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core"); diff --git a/crates/sre_engine/benches/benches.rs b/crates/sre_engine/benches/benches.rs index 127f72e2747..9905a8db70f 100644 --- a/crates/sre_engine/benches/benches.rs +++ b/crates/sre_engine/benches/benches.rs @@ -15,7 +15,7 @@ impl Pattern { fn state_range<'a, S: StrDrive>( &self, string: S, - range: std::ops::Range<usize>, + range: core::ops::Range<usize>, ) -> (Request<'a, S>, State) { let req = Request::new(string, range.start, range.end, self.code, false); let state = State::default(); diff --git a/crates/sre_engine/src/constants.rs b/crates/sre_engine/src/constants.rs index 9fe792ce17d..b38ecb109b8 100644 --- a/crates/sre_engine/src/constants.rs +++ b/crates/sre_engine/src/constants.rs @@ -3,20 +3,21 @@ * * regular expression matching engine * - * NOTE: This file is generated by sre_constants.py. If you need - * to change anything in here, edit sre_constants.py and run it. + * Auto-generated by scripts/generate_sre_constants.py from + * Lib/re/_constants.py. * * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. * - * See the _sre.c file for information on usage and redistribution. + * See the sre.c file for information on usage and redistribution. */ use bitflags::bitflags; -pub const SRE_MAGIC: usize = 20221023; -#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)] -#[repr(u32)] +pub const SRE_MAGIC: usize = 20230612; + #[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] pub enum SreOpcode { FAILURE = 0, SUCCESS = 1, @@ -62,9 +63,10 @@ pub enum SreOpcode { NOT_LITERAL_UNI_IGNORE = 41, RANGE_UNI_IGNORE = 42, } -#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)] -#[repr(u32)] + #[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] pub enum SreAtCode { BEGINNING = 0, BEGINNING_LINE = 1, @@ -79,9 +81,10 @@ pub enum SreAtCode { UNI_BOUNDARY = 10, UNI_NON_BOUNDARY = 11, } -#[derive(num_enum::TryFromPrimitive, Debug)] -#[repr(u32)] + #[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Clone, Copy, Debug)] +#[repr(u32)] pub enum SreCatCode { DIGIT = 0, NOT_DIGIT = 1, @@ -102,10 +105,10 @@ pub enum SreCatCode { UNI_LINEBREAK = 16, UNI_NOT_LINEBREAK = 17, } + bitflags! { #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct SreFlag: u16 { - const TEMPLATE = 1; const IGNORECASE = 2; const LOCALE = 4; const MULTILINE = 8; @@ -116,7 +119,9 @@ bitflags! { const ASCII = 256; } } + bitflags! { + #[derive(Clone, Copy)] pub struct SreInfo: u32 { const PREFIX = 1; const LITERAL = 2; diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index 9cc2e4788a5..36a7a9d485e 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -6,8 +6,9 @@ use crate::string::{ }; use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; +use alloc::{vec, vec::Vec}; +use core::{convert::TryFrom, ptr::null}; use optional::Optioned; -use std::{convert::TryFrom, ptr::null}; #[derive(Debug, Clone, Copy)] pub struct Request<'a, S> { @@ -27,8 +28,8 @@ impl<'a, S: StrDrive> Request<'a, S> { pattern_codes: &'a [u32], match_all: bool, ) -> Self { - let end = std::cmp::min(end, string.count()); - let start = std::cmp::min(start, end); + let end = core::cmp::min(end, string.count()); + let start = core::cmp::min(start, end); Self { string, @@ -283,8 +284,10 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo let mut context_stack = vec![]; let mut popped_result = false; - // NOTE: 'result loop is not an actual loop but break label - #[allow(clippy::never_loop)] + #[allow( + clippy::never_loop, + reason = "'result loop is not an actual loop but break label" + )] 'coro: loop { popped_result = 'result: loop { let yielded = 'context: loop { @@ -1332,7 +1335,7 @@ fn _count<S: StrDrive>( ctx: &mut MatchContext, max_count: usize, ) -> usize { - let max_count = std::cmp::min(max_count, ctx.remaining_chars(req)); + let max_count = core::cmp::min(max_count, ctx.remaining_chars(req)); let end = ctx.cursor.position + max_count; let opcode = SreOpcode::try_from(ctx.peek_code(req, 0)).unwrap(); diff --git a/crates/sre_engine/src/lib.rs b/crates/sre_engine/src/lib.rs index 08c21de9df8..df598974ba9 100644 --- a/crates/sre_engine/src/lib.rs +++ b/crates/sre_engine/src/lib.rs @@ -1,3 +1,7 @@ +#![no_std] + +extern crate alloc; + pub mod constants; pub mod engine; pub mod string; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 0d3325b6a1d..489819bfb3e 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -9,7 +9,7 @@ pub struct StringCursor { impl Default for StringCursor { fn default() -> Self { Self { - ptr: std::ptr::null(), + ptr: core::ptr::null(), position: 0, } } diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index a5328697ca8..987bca8e88f 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -11,7 +11,8 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["compiler"] +default = ["compiler", "host_env"] +host_env = ["rustpython-vm/host_env"] compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] sqlite = ["dep:libsqlite3-sys"] @@ -21,7 +22,8 @@ ssl-rustls = ["ssl", "rustls", "rustls-native-certs", "rustls-pemfile", "rustls- ssl-rustls-fips = ["ssl-rustls", "aws-lc-rs/fips"] ssl-openssl = ["ssl", "openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] ssl-vendor = ["ssl-openssl", "openssl/vendored"] -tkinter = ["dep:tk-sys", "dep:tcl-sys"] +tkinter = ["dep:tk-sys", "dep:tcl-sys", "dep:widestring"] +flame-it = ["flame"] [dependencies] # rustpython crates @@ -33,6 +35,7 @@ ahash = { workspace = true } ascii = { workspace = true } cfg-if = { workspace = true } crossbeam-utils = { workspace = true } +flame = { workspace = true, optional = true } hex = { workspace = true } itertools = { workspace = true } indexmap = { workspace = true } @@ -55,15 +58,17 @@ xml = "1.2" # random rand_core = { workspace = true } -mt19937 = "3.1" +mt19937 = "<=3.2" # upgrade it once rand is upgraded # Crypto: -digest = "0.10.3" +digest = "0.10.7" md-5 = "0.10.1" sha-1 = "0.10.0" sha2 = "0.10.2" sha3 = "0.10.1" blake2 = "0.10.4" +hmac = "0.12" +pbkdf2 = { version = "0.12", features = ["hmac"] } ## unicode stuff unicode_names2 = { workspace = true } @@ -83,13 +88,14 @@ unicode-bidi-mirroring = { workspace = true } # compression adler32 = "1.2.0" crc32fast = "1.3.2" -flate2 = { version = "<=1.1.5", default-features = false, features = ["zlib-rs"] } +flate2 = { version = "1.1.9", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.5" } bzip2 = "0.6" # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } +widestring = { workspace = true, optional = true } chrono.workspace = true # uuid @@ -114,11 +120,11 @@ dns-lookup = "3.0" # OpenSSL dependencies (optional, for ssl-openssl feature) openssl = { version = "0.10.72", optional = true } openssl-sys = { version = "0.9.110", optional = true } -openssl-probe = { version = "0.1.5", optional = true } +openssl-probe = { version = "0.2.1", optional = true } foreign-types-shared = { version = "0.1.1", optional = true } # Rustls dependencies (optional, for ssl-rustls feature) -rustls = { version = "0.23.35", default-features = false, features = ["std", "tls12", "aws_lc_rs"], optional = true } +rustls = { version = "0.23.36", default-features = false, features = ["std", "tls12", "aws_lc_rs"], optional = true } rustls-native-certs = { version = "0.8", optional = true } rustls-pemfile = { version = "2.2", optional = true } rustls-platform-verifier = { version = "0.6", optional = true } @@ -127,12 +133,12 @@ x509-parser = { version = "0.18", optional = true } der = { version = "0.7", features = ["alloc", "oid"], optional = true } pem-rfc7468 = { version = "1.0", features = ["alloc"], optional = true } webpki-roots = { version = "1.0", optional = true } -aws-lc-rs = { version = "1.14.1", optional = true } +aws-lc-rs = { version = "1.15.2", optional = true } oid-registry = { version = "0.8", features = ["x509", "pkcs1", "nist_algs"], optional = true } pkcs8 = { version = "0.10", features = ["encryption", "pkcs5", "pem"], optional = true } [target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] -libsqlite3-sys = { version = "0.28", features = ["bundled"], optional = true } +libsqlite3-sys = { version = "0.36", features = ["bundled"], optional = true } lzma-sys = "0.1" xz2 = "0.1" @@ -152,6 +158,7 @@ features = [ "Win32_Storage_FileSystem", "Win32_System_Diagnostics_Debug", "Win32_System_Environment", + "Win32_System_Console", "Win32_System_IO", "Win32_System_Threading" ] diff --git a/crates/stdlib/build.rs b/crates/stdlib/build.rs index 83ebd81ead6..95c34c4fb3c 100644 --- a/crates/stdlib/build.rs +++ b/crates/stdlib/build.rs @@ -4,7 +4,10 @@ fn main() { println!(r#"cargo::rustc-check-cfg=cfg(osslconf, values("OPENSSL_NO_COMP"))"#); println!(r#"cargo::rustc-check-cfg=cfg(openssl_vendored)"#); - #[allow(clippy::unusual_byte_groupings)] + #[allow( + clippy::unusual_byte_groupings, + reason = "hex groups follow OpenSSL version field boundaries" + )] let ossl_vers = [ (0x1_00_01_00_0, "ossl101"), (0x1_00_02_00_0, "ossl102"), @@ -25,7 +28,10 @@ fn main() { #[cfg(feature = "ssl-openssl")] { - #[allow(clippy::unusual_byte_groupings)] + #[allow( + clippy::unusual_byte_groupings, + reason = "OpenSSL version number is parsed with grouped hex fields" + )] if let Ok(v) = std::env::var("DEP_OPENSSL_VERSION_NUMBER") { println!("cargo:rustc-env=OPENSSL_API_VERSION={v}"); // cfg setup from openssl crate's build script diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs new file mode 100644 index 00000000000..d7b5dad3c9d --- /dev/null +++ b/crates/stdlib/src/_asyncio.rs @@ -0,0 +1,2800 @@ +//! _asyncio module - provides native asyncio support +//! +//! This module provides native implementations of Future and Task classes, + +pub(crate) use _asyncio::module_def; + +#[pymodule] +pub(crate) mod _asyncio { + use crate::{ + common::lock::PyRwLock, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyDict, PyDictRef, PyGenericAlias, PyList, + PyListRef, PyModule, PySet, PyTuple, PyType, PyTypeRef, + }, + extend_module, + function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue}, + protocol::PyIterReturn, + recursion::ReprGuard, + types::{ + Callable, Constructor, Destructor, Initializer, IterNext, Iterable, Representable, + SelfIter, + }, + warn, + }, + }; + use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering}; + use crossbeam_utils::atomic::AtomicCell; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Initialize module-level state + let weakref_module = vm.import("weakref", 0)?; + let weak_set_class = vm + .get_attribute_opt(weakref_module, vm.ctx.intern_str("WeakSet"))? + .ok_or_else(|| vm.new_attribute_error("WeakSet not found"))?; + let scheduled_tasks = weak_set_class.call((), vm)?; + let eager_tasks = PySet::default().into_ref(&vm.ctx); + let current_tasks = PyDict::default().into_ref(&vm.ctx); + + extend_module!(vm, module, { + "_scheduled_tasks" => scheduled_tasks, + "_eager_tasks" => eager_tasks, + "_current_tasks" => current_tasks, + }); + + // Register fork handler to clear task state in child process + #[cfg(unix)] + { + let on_fork = vm + .get_attribute_opt(module.to_owned().into(), vm.ctx.intern_str("_on_fork"))? + .expect("_on_fork not found in _asyncio module"); + vm.state.after_forkers_child.lock().push(on_fork); + } + + Ok(()) + } + + #[derive(FromArgs)] + struct AddDoneCallbackArgs { + #[pyarg(positional)] + func: PyObjectRef, + #[pyarg(named, optional)] + context: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct CancelArgs { + #[pyarg(any, optional)] + msg: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct LoopArg { + #[pyarg(any, name = "loop", optional)] + loop_: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct GetStackArgs { + #[pyarg(named, optional)] + limit: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct PrintStackArgs { + #[pyarg(named, optional)] + limit: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + file: OptionalOption<PyObjectRef>, + } + + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + enum FutureState { + Pending, + Cancelled, + Finished, + } + + impl FutureState { + fn as_str(&self) -> &'static str { + match self { + FutureState::Pending => "PENDING", + FutureState::Cancelled => "CANCELLED", + FutureState::Finished => "FINISHED", + } + } + } + + /// asyncio.Future implementation + #[pyattr] + #[pyclass(name = "Future", module = "_asyncio", traverse)] + #[derive(Debug, PyPayload)] + #[repr(C)] // Required for inheritance - ensures base field is at offset 0 in subclasses + struct PyFuture { + fut_loop: PyRwLock<Option<PyObjectRef>>, + fut_callback0: PyRwLock<Option<PyObjectRef>>, + fut_context0: PyRwLock<Option<PyObjectRef>>, + fut_callbacks: PyRwLock<Option<PyObjectRef>>, + fut_exception: PyRwLock<Option<PyObjectRef>>, + fut_exception_tb: PyRwLock<Option<PyObjectRef>>, + fut_result: PyRwLock<Option<PyObjectRef>>, + fut_source_tb: PyRwLock<Option<PyObjectRef>>, + fut_cancel_msg: PyRwLock<Option<PyObjectRef>>, + fut_cancelled_exc: PyRwLock<Option<PyObjectRef>>, + fut_awaited_by: PyRwLock<Option<PyObjectRef>>, + #[pytraverse(skip)] + fut_state: AtomicCell<FutureState>, + #[pytraverse(skip)] + fut_awaited_by_is_set: AtomicBool, + #[pytraverse(skip)] + fut_log_tb: AtomicBool, + #[pytraverse(skip)] + fut_blocking: AtomicBool, + } + + impl Constructor for PyFuture { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(PyFuture::new_empty()) + } + } + + impl Initializer for PyFuture { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Future does not accept positional arguments + if !args.args.is_empty() { + return Err(vm.new_type_error("Future() takes no positional arguments".to_string())); + } + // Extract only 'loop' keyword argument + let loop_ = args.kwargs.get("loop").cloned(); + PyFuture::py_init(&zelf, loop_, vm) + } + } + + #[pyclass( + flags(BASETYPE, HAS_DICT), + with(Constructor, Initializer, Destructor, Representable, Iterable) + )] + impl PyFuture { + fn new_empty() -> Self { + Self { + fut_loop: PyRwLock::new(None), + fut_callback0: PyRwLock::new(None), + fut_context0: PyRwLock::new(None), + fut_callbacks: PyRwLock::new(None), + fut_exception: PyRwLock::new(None), + fut_exception_tb: PyRwLock::new(None), + fut_result: PyRwLock::new(None), + fut_source_tb: PyRwLock::new(None), + fut_cancel_msg: PyRwLock::new(None), + fut_cancelled_exc: PyRwLock::new(None), + fut_awaited_by: PyRwLock::new(None), + fut_state: AtomicCell::new(FutureState::Pending), + fut_awaited_by_is_set: AtomicBool::new(false), + fut_log_tb: AtomicBool::new(false), + fut_blocking: AtomicBool::new(false), + } + } + + fn py_init( + zelf: &PyRef<Self>, + loop_: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get the event loop + let loop_obj = match loop_ { + Some(l) if !vm.is_none(&l) => l, + _ => get_event_loop(vm)?, + }; + *zelf.fut_loop.write() = Some(loop_obj.clone()); + + // Check if loop has get_debug method and call it + if let Ok(Some(get_debug)) = + vm.get_attribute_opt(loop_obj.clone(), vm.ctx.intern_str("get_debug")) + && let Ok(debug) = get_debug.call((), vm) + && debug.try_to_bool(vm).unwrap_or(false) + { + // Get source traceback + if let Ok(tb_module) = vm.import("traceback", 0) + && let Ok(Some(extract_stack)) = + vm.get_attribute_opt(tb_module, vm.ctx.intern_str("extract_stack")) + && let Ok(tb) = extract_stack.call((), vm) + { + *zelf.fut_source_tb.write() = Some(tb); + } + } + + Ok(()) + } + + #[pymethod] + fn result(&self, vm: &VirtualMachine) -> PyResult { + match self.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Result is not ready.")), + FutureState::Cancelled => { + let exc = self.make_cancelled_error_impl(vm); + Err(exc) + } + FutureState::Finished => { + self.fut_log_tb.store(false, Ordering::Relaxed); + if let Some(exc) = self.fut_exception.read().clone() { + let exc: PyBaseExceptionRef = exc.downcast().unwrap(); + // Restore the original traceback to prevent traceback accumulation + if let Some(tb) = self.fut_exception_tb.read().clone() { + let _ = exc.set___traceback__(tb, vm); + } + Err(exc) + } else { + Ok(self + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + } + + #[pymethod] + fn exception(&self, vm: &VirtualMachine) -> PyResult { + match self.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Exception is not set.")), + FutureState::Cancelled => { + let exc = self.make_cancelled_error_impl(vm); + Err(exc) + } + FutureState::Finished => { + self.fut_log_tb.store(false, Ordering::Relaxed); + Ok(self + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + + #[pymethod] + fn set_result(zelf: PyRef<Self>, result: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + if zelf.fut_state.load() != FutureState::Pending { + return Err(new_invalid_state_error(vm, "invalid state")); + } + *zelf.fut_result.write() = Some(result); + zelf.fut_state.store(FutureState::Finished); + Self::schedule_callbacks(&zelf, vm)?; + Ok(()) + } + + #[pymethod] + fn set_exception( + zelf: PyRef<Self>, + exception: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + if zelf.fut_state.load() != FutureState::Pending { + return Err(new_invalid_state_error(vm, "invalid state")); + } + + // Normalize the exception + let exc = if exception.fast_isinstance(vm.ctx.types.type_type) { + exception.call((), vm)? + } else { + exception + }; + + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error(format!( + "exception must be a BaseException, not {}", + exc.class().name() + ))); + } + + // Wrap StopIteration in RuntimeError + let exc = if exc.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + let msg = "StopIteration interacts badly with generators and cannot be raised into a Future"; + let runtime_err = vm.new_runtime_error(msg.to_string()); + // Set cause and context to the original StopIteration + let stop_iter: PyRef<PyBaseException> = exc.downcast().unwrap(); + runtime_err.set___cause__(Some(stop_iter.clone())); + runtime_err.set___context__(Some(stop_iter)); + runtime_err.into() + } else { + exc + }; + + // Save the original traceback for later restoration + if let Ok(exc_ref) = exc.clone().downcast::<PyBaseException>() { + let tb = exc_ref.__traceback__().map(|tb| tb.into()); + *zelf.fut_exception_tb.write() = tb; + } + + *zelf.fut_exception.write() = Some(exc); + zelf.fut_state.store(FutureState::Finished); + zelf.fut_log_tb.store(true, Ordering::Relaxed); + Self::schedule_callbacks(&zelf, vm)?; + Ok(()) + } + + #[pymethod] + fn add_done_callback( + zelf: PyRef<Self>, + args: AddDoneCallbackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + let ctx = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + + if zelf.fut_state.load() != FutureState::Pending { + Self::call_soon_with_context(&zelf, args.func, Some(ctx), vm)?; + } else if zelf.fut_callback0.read().is_none() { + *zelf.fut_callback0.write() = Some(args.func); + *zelf.fut_context0.write() = Some(ctx); + } else { + let tuple = vm.ctx.new_tuple(vec![args.func, ctx]); + let mut callbacks = zelf.fut_callbacks.write(); + if callbacks.is_none() { + *callbacks = Some(vm.ctx.new_list(vec![tuple.into()]).into()); + } else { + let list = callbacks.as_ref().unwrap(); + vm.call_method(list, "append", (tuple,))?; + } + } + Ok(()) + } + + #[pymethod] + fn remove_done_callback(&self, func: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + let mut cleared_callback0 = 0usize; + + // Check fut_callback0 first + // Clone to release lock before comparison (which may run Python code) + let cb0 = self.fut_callback0.read().clone(); + if let Some(cb0) = cb0 { + let cmp = vm.identical_or_equal(&cb0, &func)?; + if cmp { + *self.fut_callback0.write() = None; + *self.fut_context0.write() = None; + cleared_callback0 = 1; + } + } + + // Check if fut_callbacks exists + let callbacks = self.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => return Ok(cleared_callback0), + }; + + let list: PyListRef = callbacks.downcast().unwrap(); + let len = list.borrow_vec().len(); + + if len == 0 { + *self.fut_callbacks.write() = None; + return Ok(cleared_callback0); + } + + // Special case for single callback + if len == 1 { + let item = list.borrow_vec().first().cloned(); + if let Some(item) = item { + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + if cmp { + *self.fut_callbacks.write() = None; + return Ok(1 + cleared_callback0); + } + } + return Ok(cleared_callback0); + } + + // Multiple callbacks - iterate with index, checking validity each time + // to handle evil comparisons + let mut new_callbacks = Vec::with_capacity(len); + let mut i = 0usize; + let mut removed = 0usize; + + loop { + // Re-check fut_callbacks on each iteration (evil code may have cleared it) + let callbacks = self.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => break, + }; + let list: PyListRef = callbacks.downcast().unwrap(); + let current_len = list.borrow_vec().len(); + if i >= current_len { + break; + } + + // Get item and release lock before comparison + let item = list.borrow_vec().get(i).cloned(); + let item = match item { + Some(item) => item, + None => break, + }; + + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + + if !cmp { + new_callbacks.push(item); + } else { + removed += 1; + } + i += 1; + } + + // Update fut_callbacks with filtered list + if new_callbacks.is_empty() { + *self.fut_callbacks.write() = None; + } else { + *self.fut_callbacks.write() = Some(vm.ctx.new_list(new_callbacks).into()); + } + + Ok(removed + cleared_callback0) + } + + #[pymethod] + fn cancel(zelf: PyRef<Self>, args: CancelArgs, vm: &VirtualMachine) -> PyResult<bool> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + if zelf.fut_state.load() != FutureState::Pending { + // Clear log_tb even when cancel fails + zelf.fut_log_tb.store(false, Ordering::Relaxed); + return Ok(false); + } + + *zelf.fut_cancel_msg.write() = args.msg.flatten(); + zelf.fut_state.store(FutureState::Cancelled); + Self::schedule_callbacks(&zelf, vm)?; + Ok(true) + } + + #[pymethod] + fn cancelled(&self) -> bool { + self.fut_state.load() == FutureState::Cancelled + } + + #[pymethod] + fn done(&self) -> bool { + self.fut_state.load() != FutureState::Pending + } + + #[pymethod] + fn get_loop(&self, vm: &VirtualMachine) -> PyResult { + self.fut_loop + .read() + .clone() + .ok_or_else(|| vm.new_runtime_error("Future object is not initialized.")) + } + + #[pymethod] + fn _make_cancelled_error(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.make_cancelled_error_impl(vm) + } + + fn make_cancelled_error_impl(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + if let Some(exc) = self.fut_cancelled_exc.read().clone() + && let Ok(exc) = exc.downcast::<PyBaseException>() + { + return exc; + } + + let msg = self.fut_cancel_msg.read().clone(); + let args = if let Some(m) = msg { vec![m] } else { vec![] }; + + let exc = match get_cancelled_error_type(vm) { + Ok(cancelled_error) => vm.new_exception(cancelled_error, args), + Err(_) => vm.new_runtime_error("cancelled"), + }; + *self.fut_cancelled_exc.write() = Some(exc.clone().into()); + exc + } + + fn schedule_callbacks(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Collect all callbacks first to avoid holding locks during callback execution + // This prevents deadlock when callbacks access the future's properties + let mut callbacks_to_call: Vec<(PyObjectRef, Option<PyObjectRef>)> = Vec::new(); + + // Take callback0 - release lock before collecting from list + let cb0 = zelf.fut_callback0.write().take(); + let ctx0 = zelf.fut_context0.write().take(); + if let Some(cb) = cb0 { + callbacks_to_call.push((cb, ctx0)); + } + + // Take callbacks list and collect items + let callbacks_list = zelf.fut_callbacks.write().take(); + if let Some(callbacks) = callbacks_list + && let Ok(list) = callbacks.downcast::<PyList>() + { + // Clone the items while holding the list lock, then release + let items: Vec<_> = list.borrow_vec().iter().cloned().collect(); + for item in items { + if let Some(tuple) = item.downcast_ref::<PyTuple>() + && let (Some(cb), Some(ctx)) = (tuple.first(), tuple.get(1)) + { + callbacks_to_call.push((cb.clone(), Some(ctx.clone()))); + } + } + } + + // Now call all callbacks without holding any locks + for (cb, ctx) in callbacks_to_call { + Self::call_soon_with_context(zelf, cb, ctx, vm)?; + } + + Ok(()) + } + + fn call_soon_with_context( + zelf: &PyRef<Self>, + callback: PyObjectRef, + context: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let loop_obj = zelf.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + // call_soon(callback, *args, context=context) + // callback receives the future as its argument + let future_arg: PyObjectRef = zelf.clone().into(); + let args = if let Some(ctx) = context { + FuncArgs::new( + vec![callback, future_arg], + KwArgs::new([("context".to_owned(), ctx)].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![callback, future_arg], KwArgs::default()) + }; + vm.call_method(&loop_obj, "call_soon", args)?; + } + Ok(()) + } + + // Properties + #[pygetset] + fn _state(&self) -> &'static str { + self.fut_state.load().as_str() + } + + #[pygetset] + fn _asyncio_future_blocking(&self) -> bool { + self.fut_blocking.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__asyncio_future_blocking( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.fut_blocking.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("cannot delete attribute".to_string())) + } + } + } + + #[pygetset] + fn _loop(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _callbacks(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let mut result = Vec::new(); + + if let Some(cb0) = self.fut_callback0.read().clone() { + let ctx0 = self + .fut_context0 + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + result.push(vm.ctx.new_tuple(vec![cb0, ctx0]).into()); + } + + if let Some(callbacks) = self.fut_callbacks.read().clone() { + let list: PyListRef = callbacks.downcast().unwrap(); + for item in list.borrow_vec().iter() { + result.push(item.clone()); + } + } + + // Return None if no callbacks + if result.is_empty() { + Ok(vm.ctx.none()) + } else { + Ok(vm.ctx.new_list(result).into()) + } + } + + #[pygetset] + fn _result(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _exception(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _log_traceback(&self) -> bool { + self.fut_log_tb.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_traceback( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + if v { + return Err(vm.new_value_error( + "_log_traceback can only be set to False".to_string(), + )); + } + self.fut_log_tb.store(false, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("cannot delete attribute".to_string())) + } + } + } + + #[pygetset] + fn _source_traceback(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_source_tb + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _cancel_message(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_cancel_msg + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set__cancel_message(&self, value: PySetterValue) { + match value { + PySetterValue::Assign(v) => *self.fut_cancel_msg.write() = Some(v), + PySetterValue::Delete => *self.fut_cancel_msg.write() = None, + } + } + + #[pygetset] + fn _asyncio_awaited_by(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let awaited_by = self.fut_awaited_by.read().clone(); + match awaited_by { + None => Ok(vm.ctx.none()), + Some(obj) => { + if self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Already a Set + Ok(obj) + } else { + // Single object - create a Set for the return value + let new_set = PySet::default().into_ref(&vm.ctx); + new_set.add(obj, vm)?; + Ok(new_set.into()) + } + } + } + } + + /// Add waiter to fut_awaited_by with single-object optimization + fn awaited_by_add(&self, waiter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut awaited_by = self.fut_awaited_by.write(); + if awaited_by.is_none() { + // First waiter - store directly + *awaited_by = Some(waiter); + return Ok(()); + } + + if self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Already a Set - add to it + let set = awaited_by.as_ref().unwrap(); + vm.call_method(set, "add", (waiter,))?; + } else { + // Single object - convert to Set + let existing = awaited_by.take().unwrap(); + let new_set = PySet::default().into_ref(&vm.ctx); + new_set.add(existing, vm)?; + new_set.add(waiter, vm)?; + *awaited_by = Some(new_set.into()); + self.fut_awaited_by_is_set.store(true, Ordering::Relaxed); + } + Ok(()) + } + + /// Discard waiter from fut_awaited_by with single-object optimization + fn awaited_by_discard(&self, waiter: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let mut awaited_by = self.fut_awaited_by.write(); + if awaited_by.is_none() { + return Ok(()); + } + + let obj = awaited_by.as_ref().unwrap(); + if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Single object - check if it matches + if obj.is(waiter) { + *awaited_by = None; + } + } else { + // It's a Set - use discard + vm.call_method(obj, "discard", (waiter.to_owned(),))?; + } + Ok(()) + } + + #[pymethod] + fn __iter__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Self::__await__(zelf, vm) + } + + #[pymethod] + fn __await__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + }) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Destructor for PyFuture { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Check if we should log the traceback + // Don't log if log_tb is false or if the future was cancelled + if !zelf.fut_log_tb.load(Ordering::Relaxed) { + return Ok(()); + } + + if zelf.fut_state.load() == FutureState::Cancelled { + return Ok(()); + } + + let exc = zelf.fut_exception.read().clone(); + let exc = match exc { + Some(e) => e, + None => return Ok(()), + }; + + let loop_obj = zelf.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(()), + }; + + // Create context dict for call_exception_handler + let context = PyDict::default().into_ref(&vm.ctx); + let class_name = zelf.class().name().to_string(); + let message = format!("{} exception was never retrieved", class_name); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("exception"), exc, vm)?; + context.set_item(vm.ctx.intern_str("future"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + // Call loop.call_exception_handler(context) + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + Ok(()) + } + } + + impl Representable for PyFuture { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class_name = zelf.class().name().to_string(); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let info = get_future_repr_info(zelf.as_object(), vm)?; + Ok(format!("<{} {}>", class_name, info)) + } else { + Ok(format!("<{} ...>", class_name)) + } + } + } + + impl Iterable for PyFuture { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + } + .into_pyobject(_vm)) + } + } + + fn get_future_repr_info(future: &PyObject, vm: &VirtualMachine) -> PyResult<String> { + // Try to use asyncio.base_futures._future_repr_info + // Import from sys.modules if available, otherwise try regular import + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let module = + if let Ok(m) = sys_modules.get_item(&*vm.ctx.new_str("asyncio.base_futures"), vm) { + m + } else { + // vm.import returns the top-level module, get base_futures submodule + match vm + .import("asyncio.base_futures", 0) + .and_then(|asyncio| asyncio.get_attr(vm.ctx.intern_str("base_futures"), vm)) + { + Ok(m) => m, + Err(_) => return get_future_repr_info_fallback(future, vm), + } + }; + + let func = match vm.get_attribute_opt(module, vm.ctx.intern_str("_future_repr_info")) { + Ok(Some(f)) => f, + _ => return get_future_repr_info_fallback(future, vm), + }; + + let info = match func.call((future.to_owned(),), vm) { + Ok(i) => i, + Err(_) => return get_future_repr_info_fallback(future, vm), + }; + + let list: PyListRef = match info.downcast() { + Ok(l) => l, + Err(_) => return get_future_repr_info_fallback(future, vm), + }; + + let parts: Vec<String> = list + .borrow_vec() + .iter() + .filter_map(|x: &PyObjectRef| x.str(vm).ok().map(|s| s.as_str().to_string())) + .collect(); + Ok(parts.join(" ")) + } + + fn get_future_repr_info_fallback(future: &PyObject, vm: &VirtualMachine) -> PyResult<String> { + // Fallback: build repr from properties directly + if let Ok(Some(state)) = + vm.get_attribute_opt(future.to_owned(), vm.ctx.intern_str("_state")) + { + let state_str = state + .str(vm) + .map(|s| s.as_str().to_lowercase()) + .unwrap_or_else(|_| "unknown".to_string()); + return Ok(state_str); + } + Ok("state=unknown".to_string()) + } + + fn get_task_repr_info(task: &PyObject, vm: &VirtualMachine) -> PyResult<String> { + // vm.import returns the top-level module, get base_tasks submodule + match vm + .import("asyncio.base_tasks", 0) + .and_then(|asyncio| asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)) + { + Ok(base_tasks) => { + match vm.get_attribute_opt(base_tasks, vm.ctx.intern_str("_task_repr_info")) { + Ok(Some(func)) => { + let info: PyObjectRef = func.call((task.to_owned(),), vm)?; + let list: PyListRef = info.downcast().map_err(|_| { + vm.new_type_error("_task_repr_info should return a list") + })?; + let parts: Vec<String> = list + .borrow_vec() + .iter() + .map(|x: &PyObjectRef| x.str(vm).map(|s| s.as_str().to_string())) + .collect::<PyResult<Vec<_>>>()?; + Ok(parts.join(" ")) + } + _ => get_future_repr_info(task, vm), + } + } + Err(_) => get_future_repr_info(task, vm), + } + } + + #[pyattr] + #[pyclass(name = "FutureIter", module = "_asyncio", traverse)] + #[derive(Debug, PyPayload)] + struct PyFutureIter { + future: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(IterNext, Iterable))] + impl PyFutureIter { + #[pymethod] + fn send(&self, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let future = self.future.read().clone(); + let future = match future { + Some(f) => f, + None => return Err(vm.new_stop_iteration(None)), + }; + + // Try to get blocking flag (check Task first since it inherits from Future) + let blocking = if let Some(task) = future.downcast_ref::<PyTask>() { + task.base.fut_blocking.load(Ordering::Relaxed) + } else if let Some(fut) = future.downcast_ref::<PyFuture>() { + fut.fut_blocking.load(Ordering::Relaxed) + } else { + // For non-native futures, check the attribute + vm.get_attribute_opt( + future.clone(), + vm.ctx.intern_str("_asyncio_future_blocking"), + )? + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false) + }; + + // Check if future is done + let done = vm.call_method(&future, "done", ())?; + if done.try_to_bool(vm)? { + *self.future.write() = None; + let result = vm.call_method(&future, "result", ())?; + return Err(vm.new_stop_iteration(Some(result))); + } + + // If still pending and blocking is already set, raise RuntimeError + // This means await wasn't used with future + if blocking { + return Err(vm.new_runtime_error("await wasn't used with future")); + } + + // First call: set blocking flag and yield the future (check Task first) + if let Some(task) = future.downcast_ref::<PyTask>() { + task.base.fut_blocking.store(true, Ordering::Relaxed); + } else if let Some(fut) = future.downcast_ref::<PyFuture>() { + fut.fut_blocking.store(true, Ordering::Relaxed); + } else { + future.set_attr( + vm.ctx.intern_str("_asyncio_future_blocking"), + vm.ctx.true_value.clone(), + vm, + )?; + } + Ok(future) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // Warn about deprecated (type, val, tb) signature + if exc_val.is_present() || exc_tb.is_present() { + warn::warn( + vm.ctx + .new_str( + "the (type, val, tb) signature of throw() is deprecated, \ + use throw(val) instead", + ) + .into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + + *self.future.write() = None; + + // Validate tb if present + if let OptionalArg::Present(ref tb) = exc_tb + && !vm.is_none(tb) + && !tb.fast_isinstance(vm.ctx.types.traceback_type) + { + return Err(vm.new_type_error(format!( + "throw() third argument must be a traceback object, not '{}'", + tb.class().name() + ))); + } + + let exc = if exc_type.fast_isinstance(vm.ctx.types.type_type) { + // exc_type is a class + let exc_class: PyTypeRef = exc_type.clone().downcast().unwrap(); + // Must be a subclass of BaseException + if !exc_class.fast_issubclass(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error( + "exceptions must be classes or instances deriving from BaseException, not type".to_string() + )); + } + + let val = exc_val.unwrap_or_none(vm); + if vm.is_none(&val) { + exc_type.call((), vm)? + } else if val.fast_isinstance(&exc_class) { + val + } else { + exc_type.call((val,), vm)? + } + } else if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + // exc_type is an exception instance + if let OptionalArg::Present(ref val) = exc_val + && !vm.is_none(val) + { + return Err(vm.new_type_error( + "instance exception may not have a separate value".to_string(), + )); + } + exc_type + } else { + // exc_type is neither a class nor an exception instance + return Err(vm.new_type_error(format!( + "exceptions must be classes or instances deriving from BaseException, not {}", + exc_type.class().name() + ))); + }; + + if let OptionalArg::Present(tb) = exc_tb + && !vm.is_none(&tb) + { + exc.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; + } + + Err(exc.downcast().unwrap()) + } + + #[pymethod] + fn close(&self) { + *self.future.write() = None; + } + } + + impl SelfIter for PyFutureIter {} + impl IterNext for PyFutureIter { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) + } + } + + #[pyattr] + #[pyclass(name = "Task", module = "_asyncio", base = PyFuture, traverse)] + #[derive(Debug)] + #[repr(C)] + struct PyTask { + // Base class (must be first field for inheritance) + base: PyFuture, + // Task-specific fields + task_coro: PyRwLock<Option<PyObjectRef>>, + task_fut_waiter: PyRwLock<Option<PyObjectRef>>, + task_name: PyRwLock<Option<PyObjectRef>>, + task_context: PyRwLock<Option<PyObjectRef>>, + #[pytraverse(skip)] + task_must_cancel: AtomicBool, + #[pytraverse(skip)] + task_num_cancels_requested: AtomicI32, + #[pytraverse(skip)] + task_log_destroy_pending: AtomicBool, + } + + #[derive(FromArgs)] + struct TaskInitArgs { + #[pyarg(positional)] + coro: PyObjectRef, + #[pyarg(named, name = "loop", optional)] + loop_: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + name: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + context: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + eager_start: OptionalOption<bool>, + } + + static TASK_NAME_COUNTER: AtomicU64 = AtomicU64::new(0); + + impl Constructor for PyTask { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + base: PyFuture::new_empty(), + task_coro: PyRwLock::new(None), + task_fut_waiter: PyRwLock::new(None), + task_name: PyRwLock::new(None), + task_context: PyRwLock::new(None), + task_must_cancel: AtomicBool::new(false), + task_num_cancels_requested: AtomicI32::new(0), + task_log_destroy_pending: AtomicBool::new(true), + }) + } + } + + impl Initializer for PyTask { + type Args = TaskInitArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + PyTask::py_init(&zelf, args, vm) + } + } + + #[pyclass( + flags(BASETYPE, HAS_DICT), + with(Constructor, Initializer, Destructor, Representable, Iterable) + )] + impl PyTask { + fn py_init(zelf: &PyRef<Self>, args: TaskInitArgs, vm: &VirtualMachine) -> PyResult<()> { + // Validate coroutine + if !is_coroutine(args.coro.clone(), vm)? { + return Err(vm.new_type_error(format!( + "a coroutine was expected, got {}", + args.coro.repr(vm)? + ))); + } + + // Get the event loop + let loop_obj = match args.loop_.flatten() { + Some(l) => l, + None => get_running_loop(vm) + .map_err(|_| vm.new_runtime_error("no current event loop"))?, + }; + *zelf.base.fut_loop.write() = Some(loop_obj.clone()); + + // Check if loop has get_debug method and capture source traceback if enabled + if let Ok(Some(get_debug)) = + vm.get_attribute_opt(loop_obj.clone(), vm.ctx.intern_str("get_debug")) + && let Ok(debug) = get_debug.call((), vm) + && debug.try_to_bool(vm).unwrap_or(false) + { + // Get source traceback + if let Ok(tb_module) = vm.import("traceback", 0) + && let Ok(Some(extract_stack)) = + vm.get_attribute_opt(tb_module, vm.ctx.intern_str("extract_stack")) + && let Ok(tb) = extract_stack.call((), vm) + { + *zelf.base.fut_source_tb.write() = Some(tb); + } + } + + // Get or create context + let context = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + *zelf.task_context.write() = Some(context); + + // Set coroutine + *zelf.task_coro.write() = Some(args.coro); + + // Set task name + let name = match args.name.flatten() { + Some(n) => { + if !n.fast_isinstance(vm.ctx.types.str_type) { + n.str(vm)?.into() + } else { + n + } + } + None => { + let counter = TASK_NAME_COUNTER.fetch_add(1, Ordering::SeqCst); + vm.ctx.new_str(format!("Task-{}", counter + 1)).into() + } + }; + *zelf.task_name.write() = Some(name); + + let eager_start = args.eager_start.flatten().unwrap_or(false); + + // Check if we should do eager start: only if the loop is running + let do_eager_start = if eager_start { + let is_running = vm.call_method(&loop_obj, "is_running", ())?; + is_running.is_true(vm)? + } else { + false + }; + + if do_eager_start { + // Eager start: run first step synchronously (loop is already running) + task_eager_start(zelf, vm)?; + } else { + // Non-eager or loop not running: schedule the first step + _register_task(zelf.clone().into(), vm)?; + let task_obj: PyObjectRef = zelf.clone().into(); + let step_wrapper = TaskStepMethWrapper::new(task_obj).into_ref(&vm.ctx); + vm.call_method(&loop_obj, "call_soon", (step_wrapper,))?; + } + + Ok(()) + } + + // Future methods delegation + #[pymethod] + fn result(&self, vm: &VirtualMachine) -> PyResult { + match self.base.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Result is not ready.")), + FutureState::Cancelled => Err(self.make_cancelled_error_impl(vm)), + FutureState::Finished => { + self.base.fut_log_tb.store(false, Ordering::Relaxed); + if let Some(exc) = self.base.fut_exception.read().clone() { + let exc: PyBaseExceptionRef = exc.downcast().unwrap(); + // Restore the original traceback to prevent traceback accumulation + if let Some(tb) = self.base.fut_exception_tb.read().clone() { + let _ = exc.set___traceback__(tb, vm); + } + Err(exc) + } else { + Ok(self + .base + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + } + + #[pymethod] + fn exception(&self, vm: &VirtualMachine) -> PyResult { + match self.base.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Exception is not set.")), + FutureState::Cancelled => Err(self.make_cancelled_error_impl(vm)), + FutureState::Finished => { + self.base.fut_log_tb.store(false, Ordering::Relaxed); + Ok(self + .base + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + + #[pymethod] + fn set_result( + _zelf: PyObjectRef, + _result: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Err(vm.new_runtime_error("Task does not support set_result operation")) + } + + #[pymethod] + fn set_exception(&self, _exception: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_runtime_error("Task does not support set_exception operation")) + } + + fn make_cancelled_error_impl(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + if let Some(exc) = self.base.fut_cancelled_exc.read().clone() + && let Ok(exc) = exc.downcast::<PyBaseException>() + { + return exc; + } + + let msg = self.base.fut_cancel_msg.read().clone(); + let args = if let Some(m) = msg { vec![m] } else { vec![] }; + + let exc = match get_cancelled_error_type(vm) { + Ok(cancelled_error) => vm.new_exception(cancelled_error, args), + Err(_) => vm.new_runtime_error("cancelled"), + }; + *self.base.fut_cancelled_exc.write() = Some(exc.clone().into()); + exc + } + + #[pymethod] + fn add_done_callback( + zelf: PyRef<Self>, + args: AddDoneCallbackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.base.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + let ctx = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + + if zelf.base.fut_state.load() != FutureState::Pending { + Self::call_soon_with_context(&zelf, args.func, Some(ctx), vm)?; + } else if zelf.base.fut_callback0.read().is_none() { + *zelf.base.fut_callback0.write() = Some(args.func); + *zelf.base.fut_context0.write() = Some(ctx); + } else { + let tuple = vm.ctx.new_tuple(vec![args.func, ctx]); + let mut callbacks = zelf.base.fut_callbacks.write(); + if callbacks.is_none() { + *callbacks = Some(vm.ctx.new_list(vec![tuple.into()]).into()); + } else { + let list = callbacks.as_ref().unwrap(); + vm.call_method(list, "append", (tuple,))?; + } + } + Ok(()) + } + + #[pymethod] + fn remove_done_callback(&self, func: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.base.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.".to_string())); + } + let mut cleared_callback0 = 0usize; + + // Check fut_callback0 first + // Clone to release lock before comparison (which may run Python code) + let cb0 = self.base.fut_callback0.read().clone(); + if let Some(cb0) = cb0 { + let cmp = vm.identical_or_equal(&cb0, &func)?; + if cmp { + *self.base.fut_callback0.write() = None; + *self.base.fut_context0.write() = None; + cleared_callback0 = 1; + } + } + + // Check if fut_callbacks exists + let callbacks = self.base.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => return Ok(cleared_callback0), + }; + + let list: PyListRef = callbacks.downcast().unwrap(); + let len = list.borrow_vec().len(); + + if len == 0 { + *self.base.fut_callbacks.write() = None; + return Ok(cleared_callback0); + } + + // Special case for single callback + if len == 1 { + let item = list.borrow_vec().first().cloned(); + if let Some(item) = item { + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + if cmp { + *self.base.fut_callbacks.write() = None; + return Ok(1 + cleared_callback0); + } + } + return Ok(cleared_callback0); + } + + // Multiple callbacks - iterate with index, checking validity each time + // to handle evil comparisons + let mut new_callbacks = Vec::with_capacity(len); + let mut i = 0usize; + let mut removed = 0usize; + + loop { + // Re-check fut_callbacks on each iteration (evil code may have cleared it) + let callbacks = self.base.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => break, + }; + let list: PyListRef = callbacks.downcast().unwrap(); + let current_len = list.borrow_vec().len(); + if i >= current_len { + break; + } + + // Get item and release lock before comparison + let item = list.borrow_vec().get(i).cloned(); + let item = match item { + Some(item) => item, + None => break, + }; + + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + + if !cmp { + new_callbacks.push(item); + } else { + removed += 1; + } + i += 1; + } + + // Update fut_callbacks with filtered list + if new_callbacks.is_empty() { + *self.base.fut_callbacks.write() = None; + } else { + *self.base.fut_callbacks.write() = Some(vm.ctx.new_list(new_callbacks).into()); + } + + Ok(removed + cleared_callback0) + } + + fn schedule_callbacks(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Collect all callbacks first to avoid holding locks during callback execution + // This prevents deadlock when callbacks access the future's properties + let mut callbacks_to_call: Vec<(PyObjectRef, Option<PyObjectRef>)> = Vec::new(); + + // Take callback0 - release lock before collecting from list + let cb0 = zelf.base.fut_callback0.write().take(); + let ctx0 = zelf.base.fut_context0.write().take(); + if let Some(cb) = cb0 { + callbacks_to_call.push((cb, ctx0)); + } + + // Take callbacks list and collect items + let callbacks_list = zelf.base.fut_callbacks.write().take(); + if let Some(callbacks) = callbacks_list + && let Ok(list) = callbacks.downcast::<PyList>() + { + // Clone the items while holding the list lock, then release + let items: Vec<_> = list.borrow_vec().iter().cloned().collect(); + for item in items { + if let Some(tuple) = item.downcast_ref::<PyTuple>() + && let (Some(cb), Some(ctx)) = (tuple.first(), tuple.get(1)) + { + callbacks_to_call.push((cb.clone(), Some(ctx.clone()))); + } + } + } + + // Now call all callbacks without holding any locks + for (cb, ctx) in callbacks_to_call { + Self::call_soon_with_context(zelf, cb, ctx, vm)?; + } + + Ok(()) + } + + fn call_soon_with_context( + zelf: &PyRef<Self>, + callback: PyObjectRef, + context: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + // call_soon(callback, *args, context=context) + // callback receives the task as its argument + let task_arg: PyObjectRef = zelf.clone().into(); + let args = if let Some(ctx) = context { + FuncArgs::new( + vec![callback, task_arg], + KwArgs::new([("context".to_owned(), ctx)].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![callback, task_arg], KwArgs::default()) + }; + vm.call_method(&loop_obj, "call_soon", args)?; + } + Ok(()) + } + + #[pymethod] + fn cancel(&self, args: CancelArgs, vm: &VirtualMachine) -> PyResult<bool> { + if self.base.fut_state.load() != FutureState::Pending { + // Clear log_tb even when cancel fails (task is already done) + self.base.fut_log_tb.store(false, Ordering::Relaxed); + return Ok(false); + } + + self.task_num_cancels_requested + .fetch_add(1, Ordering::SeqCst); + + let msg_value = args.msg.flatten(); + + if let Some(fut_waiter) = self.task_fut_waiter.read().clone() { + // Call cancel with msg=msg keyword argument + let cancel_args = if let Some(ref m) = msg_value { + FuncArgs::new( + vec![], + KwArgs::new([("msg".to_owned(), m.clone())].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![], KwArgs::default()) + }; + let cancel_result = vm.call_method(&fut_waiter, "cancel", cancel_args)?; + if cancel_result.try_to_bool(vm)? { + return Ok(true); + } + } + + self.task_must_cancel.store(true, Ordering::Relaxed); + *self.base.fut_cancel_msg.write() = msg_value; + Ok(true) + } + + #[pymethod] + fn cancelled(&self) -> bool { + self.base.fut_state.load() == FutureState::Cancelled + } + + #[pymethod] + fn done(&self) -> bool { + self.base.fut_state.load() != FutureState::Pending + } + + #[pymethod] + fn cancelling(&self) -> i32 { + self.task_num_cancels_requested.load(Ordering::SeqCst) + } + + #[pymethod] + fn uncancel(&self) -> i32 { + let prev = self + .task_num_cancels_requested + .fetch_sub(1, Ordering::SeqCst); + if prev <= 0 { + self.task_num_cancels_requested.store(0, Ordering::SeqCst); + 0 + } else { + let new_val = prev - 1; + // When cancelling count reaches 0, reset _must_cancel + if new_val == 0 { + self.task_must_cancel.store(false, Ordering::SeqCst); + } + new_val + } + } + + #[pymethod] + fn get_coro(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_coro + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn get_context(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_context + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn get_name(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_name + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn set_name(&self, name: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let name = if !name.fast_isinstance(vm.ctx.types.str_type) { + name.str(vm)?.into() + } else { + name + }; + *self.task_name.write() = Some(name); + Ok(()) + } + + #[pymethod] + fn get_loop(&self, vm: &VirtualMachine) -> PyResult { + self.base + .fut_loop + .read() + .clone() + .ok_or_else(|| vm.new_runtime_error("Task object is not initialized.")) + } + + #[pymethod] + fn get_stack(zelf: PyRef<Self>, args: GetStackArgs, vm: &VirtualMachine) -> PyResult { + let limit = args.limit.flatten().unwrap_or_else(|| vm.ctx.none()); + // vm.import returns the top-level module, get base_tasks submodule + let asyncio = vm.import("asyncio.base_tasks", 0)?; + let base_tasks = asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)?; + let get_stack_func = base_tasks.get_attr(vm.ctx.intern_str("_task_get_stack"), vm)?; + get_stack_func.call((zelf, limit), vm) + } + + #[pymethod] + fn print_stack( + zelf: PyRef<Self>, + args: PrintStackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let limit = args.limit.flatten().unwrap_or_else(|| vm.ctx.none()); + let file = args.file.flatten().unwrap_or_else(|| vm.ctx.none()); + // vm.import returns the top-level module, get base_tasks submodule + let asyncio = vm.import("asyncio.base_tasks", 0)?; + let base_tasks = asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)?; + let print_stack_func = + base_tasks.get_attr(vm.ctx.intern_str("_task_print_stack"), vm)?; + print_stack_func.call((zelf, limit, file), vm)?; + Ok(()) + } + + #[pymethod] + fn _make_cancelled_error(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.make_cancelled_error_impl(vm) + } + + // Properties + #[pygetset] + fn _state(&self) -> &'static str { + self.base.fut_state.load().as_str() + } + + #[pygetset] + fn _asyncio_future_blocking(&self) -> bool { + self.base.fut_blocking.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__asyncio_future_blocking( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.base.fut_blocking.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("cannot delete attribute".to_string())) + } + } + } + + #[pygetset] + fn _loop(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _log_destroy_pending(&self) -> bool { + self.task_log_destroy_pending.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_destroy_pending( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.task_log_destroy_pending.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("can't delete _log_destroy_pending".to_owned())) + } + } + } + + #[pygetset] + fn _log_traceback(&self) -> bool { + self.base.fut_log_tb.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_traceback( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + if v { + return Err(vm.new_value_error( + "_log_traceback can only be set to False".to_string(), + )); + } + self.base.fut_log_tb.store(false, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("cannot delete attribute".to_string())) + } + } + } + + #[pygetset] + fn _must_cancel(&self) -> bool { + self.task_must_cancel.load(Ordering::Relaxed) + } + + #[pygetset] + fn _coro(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_coro + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _fut_waiter(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_fut_waiter + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _source_traceback(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_source_tb + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _result(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _exception(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _cancel_message(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_cancel_msg + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set__cancel_message(&self, value: PySetterValue) { + match value { + PySetterValue::Assign(v) => *self.base.fut_cancel_msg.write() = Some(v), + PySetterValue::Delete => *self.base.fut_cancel_msg.write() = None, + } + } + + #[pygetset] + fn _callbacks(&self, vm: &VirtualMachine) -> PyObjectRef { + let mut result: Vec<PyObjectRef> = Vec::new(); + if let Some(cb) = self.base.fut_callback0.read().clone() { + let ctx = self + .base + .fut_context0 + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + result.push(vm.ctx.new_tuple(vec![cb, ctx]).into()); + } + if let Some(callbacks) = self.base.fut_callbacks.read().clone() + && let Ok(list) = callbacks.downcast::<PyList>() + { + for item in list.borrow_vec().iter() { + result.push(item.clone()); + } + } + // Return None if no callbacks + if result.is_empty() { + vm.ctx.none() + } else { + vm.ctx.new_list(result).into() + } + } + + #[pymethod] + fn __iter__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + }) + } + + #[pymethod] + fn __await__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + }) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Destructor for PyTask { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + + // Check if task is pending and log_destroy_pending is True + if zelf.base.fut_state.load() == FutureState::Pending + && zelf.task_log_destroy_pending.load(Ordering::Relaxed) + { + if let Some(loop_obj) = loop_obj.clone() { + let context = PyDict::default().into_ref(&vm.ctx); + let task_repr = zelf + .as_object() + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Task>")); + let message = + format!("Task was destroyed but it is pending!\ntask: {}", task_repr); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("task"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.base.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + } + return Ok(()); + } + + // Check if we should log the traceback for exception + if !zelf.base.fut_log_tb.load(Ordering::Relaxed) { + return Ok(()); + } + + let exc = zelf.base.fut_exception.read().clone(); + let exc = match exc { + Some(e) => e, + None => return Ok(()), + }; + + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(()), + }; + + // Create context dict for call_exception_handler + let context = PyDict::default().into_ref(&vm.ctx); + let class_name = zelf.class().name().to_string(); + let message = format!("{} exception was never retrieved", class_name); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("exception"), exc, vm)?; + context.set_item(vm.ctx.intern_str("future"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.base.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + // Call loop.call_exception_handler(context) + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + Ok(()) + } + } + + impl Representable for PyTask { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class_name = zelf.class().name().to_string(); + + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + // Try to use _task_repr_info if available + if let Ok(info) = get_task_repr_info(zelf.as_object(), vm) + && info != "state=unknown" + { + return Ok(format!("<{} {}>", class_name, info)); + } + + // Fallback: build repr from task properties directly + let state = zelf.base.fut_state.load().as_str().to_lowercase(); + let name = zelf + .task_name + .read() + .as_ref() + .and_then(|n| n.str(vm).ok()) + .map(|s| s.as_str().to_string()) + .unwrap_or_else(|| "?".to_string()); + let coro_repr = zelf + .task_coro + .read() + .as_ref() + .and_then(|c| c.repr(vm).ok()) + .map(|s| s.as_str().to_string()) + .unwrap_or_else(|| "?".to_string()); + + Ok(format!( + "<{} {} name='{}' coro={}>", + class_name, state, name, coro_repr + )) + } else { + Ok(format!("<{} ...>", class_name)) + } + } + } + + impl Iterable for PyTask { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + } + .into_pyobject(_vm)) + } + } + + /// Eager start: run first step synchronously + fn task_eager_start(zelf: &PyRef<PyTask>, vm: &VirtualMachine) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Err(vm.new_runtime_error("Task has no loop")), + }; + + // Register task before running step + let task_obj: PyObjectRef = zelf.clone().into(); + _register_task(task_obj.clone(), vm)?; + + // Register as eager task + _register_eager_task(task_obj.clone(), vm)?; + + // Swap current task - save previous task + let prev_task = _swap_current_task(loop_obj.clone(), task_obj.clone(), vm)?; + + // Get coro and context + let coro = zelf.task_coro.read().clone(); + let context = zelf.task_context.read().clone(); + + // Run the first step with context (using context.run(callable, *args)) + let step_result = if let Some(ctx) = context { + // Call context.run(coro.send, None) + let coro_ref = match coro { + Some(c) => c, + None => { + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + _unregister_eager_task(task_obj.clone(), vm)?; + return Ok(()); + } + }; + let send_method = coro_ref.get_attr(vm.ctx.intern_str("send"), vm)?; + vm.call_method(&ctx, "run", (send_method, vm.ctx.none())) + } else { + // Run without context + match coro { + Some(c) => vm.call_method(&c, "send", (vm.ctx.none(),)), + None => { + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + _unregister_eager_task(task_obj.clone(), vm)?; + return Ok(()); + } + } + }; + + // Restore previous task + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + + // Unregister from eager tasks + _unregister_eager_task(task_obj.clone(), vm)?; + + // Handle the result + match step_result { + Ok(result) => { + task_step_handle_result(zelf, result, vm)?; + } + Err(e) => { + task_step_handle_exception(zelf, e, vm)?; + } + } + + // If task is no longer pending, clear the coroutine + if zelf.base.fut_state.load() != FutureState::Pending { + *zelf.task_coro.write() = None; + } + + Ok(()) + } + + /// Task step implementation + fn task_step_impl( + task: &PyObjectRef, + exc: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let task_ref: PyRef<PyTask> = task + .clone() + .downcast() + .map_err(|_| vm.new_type_error("task_step called with non-Task object"))?; + + if task_ref.base.fut_state.load() != FutureState::Pending { + // Task is already done - report InvalidStateError via exception handler + let loop_obj = task_ref.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + let exc = new_invalid_state_error(vm, "step(): already done"); + let context = vm.ctx.new_dict(); + context.set_item("message", vm.new_pyobj("step(): already done"), vm)?; + context.set_item("exception", exc.clone().into(), vm)?; + context.set_item("task", task.clone(), vm)?; + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + } + return Ok(vm.ctx.none()); + } + + *task_ref.task_fut_waiter.write() = None; + + let coro = task_ref.task_coro.read().clone(); + let coro = match coro { + Some(c) => c, + None => return Ok(vm.ctx.none()), + }; + + // Get event loop for enter/leave task + let loop_obj = task_ref.base.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(vm.ctx.none()), + }; + + // Get task context + let context = task_ref.task_context.read().clone(); + + // Enter task - register as current task + _enter_task(loop_obj.clone(), task.clone(), vm)?; + + // Determine the exception to throw (if any) + // If task_must_cancel is set and exc is None or not CancelledError, create CancelledError + let exc_to_throw = if task_ref.task_must_cancel.load(Ordering::Relaxed) { + task_ref.task_must_cancel.store(false, Ordering::Relaxed); + if let Some(ref e) = exc { + if is_cancelled_error_obj(e, vm) { + exc.clone() + } else { + Some(task_ref.make_cancelled_error_impl(vm).into()) + } + } else { + Some(task_ref.make_cancelled_error_impl(vm).into()) + } + } else { + exc + }; + + // Run coroutine step within task's context + let result = if let Some(ctx) = context { + // Use context.run(callable, *args) to run within the task's context + if let Some(ref exc_obj) = exc_to_throw { + let throw_method = coro.get_attr(vm.ctx.intern_str("throw"), vm)?; + vm.call_method(&ctx, "run", (throw_method, exc_obj.clone())) + } else { + let send_method = coro.get_attr(vm.ctx.intern_str("send"), vm)?; + vm.call_method(&ctx, "run", (send_method, vm.ctx.none())) + } + } else { + // Fallback: run without context + if let Some(ref exc_obj) = exc_to_throw { + vm.call_method(&coro, "throw", (exc_obj.clone(),)) + } else { + vm.call_method(&coro, "send", (vm.ctx.none(),)) + } + }; + + // Leave task - unregister as current task (must happen even on error) + let _ = _leave_task(loop_obj, task.clone(), vm); + + match result { + Ok(result) => { + task_step_handle_result(&task_ref, result, vm)?; + } + Err(e) => { + task_step_handle_exception(&task_ref, e, vm)?; + } + } + + Ok(vm.ctx.none()) + } + + fn task_step_handle_result( + task: &PyRef<PyTask>, + result: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if task awaits on itself + let task_obj: PyObjectRef = task.clone().into(); + if result.is(&task_obj) { + let msg = format!( + "Task cannot await on itself: {}", + task_obj.repr(vm)?.as_str() + ); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task_obj, vm)?; + return Ok(()); + } + + let blocking = vm + .get_attribute_opt( + result.clone(), + vm.ctx.intern_str("_asyncio_future_blocking"), + )? + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false); + + if blocking { + result.set_attr( + vm.ctx.intern_str("_asyncio_future_blocking"), + vm.ctx.new_bool(false), + vm, + )?; + + // Get the future's loop, similar to get_future_loop: + // 1. If it's our native Future/Task, access fut_loop directly (check Task first) + // 2. Otherwise try get_loop(), falling back to _loop on AttributeError + let fut_loop = if let Ok(task) = result.clone().downcast::<PyTask>() { + task.base + .fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } else if let Ok(fut) = result.clone().downcast::<PyFuture>() { + fut.fut_loop.read().clone().unwrap_or_else(|| vm.ctx.none()) + } else { + // Try get_loop(), fall back to _loop on AttributeError + match vm.call_method(&result, "get_loop", ()) { + Ok(loop_obj) => loop_obj, + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => { + result.get_attr(vm.ctx.intern_str("_loop"), vm)? + } + Err(e) => return Err(e), + } + }; + let task_loop = task.base.fut_loop.read().clone(); + if let Some(task_loop) = task_loop + && !fut_loop.is(&task_loop) + { + let task_repr = task + .as_object() + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Task>")); + let result_repr = result + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Future>")); + let msg = format!( + "Task {} got Future {} attached to a different loop", + task_repr, result_repr + ); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + return Ok(()); + } + + *task.task_fut_waiter.write() = Some(result.clone()); + + let task_obj: PyObjectRef = task.clone().into(); + let wakeup_wrapper = TaskWakeupMethWrapper::new(task_obj.clone()).into_ref(&vm.ctx); + vm.call_method(&result, "add_done_callback", (wakeup_wrapper,))?; + + // Track awaited_by relationship for introspection + future_add_to_awaited_by(result.clone(), task_obj, vm)?; + + // If task_must_cancel is set, cancel the awaited future immediately + // This propagates the cancellation through the future chain + if task.task_must_cancel.load(Ordering::Relaxed) { + let cancel_msg = task.base.fut_cancel_msg.read().clone(); + let cancel_args = if let Some(ref m) = cancel_msg { + FuncArgs::new( + vec![], + KwArgs::new([("msg".to_owned(), m.clone())].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![], KwArgs::default()) + }; + let cancel_result = vm.call_method(&result, "cancel", cancel_args)?; + if cancel_result.try_to_bool(vm).unwrap_or(false) { + task.task_must_cancel.store(false, Ordering::Relaxed); + } + } + } else if vm.is_none(&result) { + let loop_obj = task.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + let task_obj: PyObjectRef = task.clone().into(); + let step_wrapper = TaskStepMethWrapper::new(task_obj).into_ref(&vm.ctx); + vm.call_method(&loop_obj, "call_soon", (step_wrapper,))?; + } + } else { + let msg = format!("Task got bad yield: {}", result.repr(vm)?.as_str()); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } + + Ok(()) + } + + fn task_step_handle_exception( + task: &PyRef<PyTask>, + exc: PyBaseExceptionRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check for KeyboardInterrupt or SystemExit - these should be re-raised + let should_reraise = exc.fast_isinstance(vm.ctx.exceptions.keyboard_interrupt) + || exc.fast_isinstance(vm.ctx.exceptions.system_exit); + + if exc.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + // Check if task was cancelled while running + if task.task_must_cancel.load(Ordering::Relaxed) { + // Task was cancelled - treat as cancelled instead of result + task.task_must_cancel.store(false, Ordering::Relaxed); + let cancelled_exc = task.base.make_cancelled_error_impl(vm); + task.base.fut_state.store(FutureState::Cancelled); + *task.base.fut_cancelled_exc.write() = Some(cancelled_exc.into()); + } else { + let result = exc.get_arg(0).unwrap_or_else(|| vm.ctx.none()); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_result.write() = Some(result); + } + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } else if is_cancelled_error(&exc, vm) { + task.base.fut_state.store(FutureState::Cancelled); + *task.base.fut_cancelled_exc.write() = Some(exc.clone().into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } else { + task.base.fut_state.store(FutureState::Finished); + // Save the original traceback for later restoration + let tb = exc.__traceback__().map(|tb| tb.into()); + *task.base.fut_exception_tb.write() = tb; + *task.base.fut_exception.write() = Some(exc.clone().into()); + task.base.fut_log_tb.store(true, Ordering::Relaxed); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } + + // Re-raise KeyboardInterrupt and SystemExit after storing in task + if should_reraise { + return Err(exc); + } + + Ok(()) + } + + fn task_wakeup_impl(task: &PyObjectRef, fut: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let task_ref: PyRef<PyTask> = task + .clone() + .downcast() + .map_err(|_| vm.new_type_error("task_wakeup called with non-Task object"))?; + + // Remove awaited_by relationship before resuming + future_discard_from_awaited_by(fut.clone(), task.clone(), vm)?; + + *task_ref.task_fut_waiter.write() = None; + + // Call result() on the awaited future to get either result or exception + // If result() raises an exception (like CancelledError), pass it to task_step + let exc = match vm.call_method(fut, "result", ()) { + Ok(_) => None, + Err(e) => Some(e.into()), + }; + + // Call task_step directly instead of using call_soon + // This allows the awaiting task to continue in the same event loop iteration + task_step_impl(task, exc, vm) + } + + // Module Functions + + fn get_all_tasks_set(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _scheduled_tasks WeakSet + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_scheduled_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_scheduled_tasks not found")) + } + + fn get_eager_tasks_set(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _eager_tasks Set + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_eager_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_eager_tasks not found")) + } + + fn get_current_tasks_dict(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _current_tasks Dict + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_current_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_current_tasks not found")) + } + + #[pyfunction] + fn _get_running_loop(vm: &VirtualMachine) -> PyObjectRef { + vm.asyncio_running_loop + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pyfunction] + fn _set_running_loop(loop_: OptionalOption<PyObjectRef>, vm: &VirtualMachine) { + *vm.asyncio_running_loop.borrow_mut() = loop_.flatten(); + } + + #[pyfunction] + fn get_running_loop(vm: &VirtualMachine) -> PyResult { + vm.asyncio_running_loop + .borrow() + .clone() + .ok_or_else(|| vm.new_runtime_error("no running event loop")) + } + + #[pyfunction] + fn get_event_loop(vm: &VirtualMachine) -> PyResult { + if let Some(loop_) = vm.asyncio_running_loop.borrow().clone() { + return Ok(loop_); + } + + let asyncio_events = vm.import("asyncio.events", 0)?; + let get_event_loop_policy = vm + .get_attribute_opt(asyncio_events, vm.ctx.intern_str("get_event_loop_policy"))? + .ok_or_else(|| vm.new_attribute_error("get_event_loop_policy"))?; + let policy = get_event_loop_policy.call((), vm)?; + let get_event_loop = vm + .get_attribute_opt(policy, vm.ctx.intern_str("get_event_loop"))? + .ok_or_else(|| vm.new_attribute_error("get_event_loop"))?; + get_event_loop.call((), vm) + } + + #[pyfunction] + fn current_task(args: LoopArg, vm: &VirtualMachine) -> PyResult { + let loop_obj = match args.loop_.flatten() { + Some(l) if !vm.is_none(&l) => l, + _ => { + // When loop is None or not provided, use the running loop + match vm.asyncio_running_loop.borrow().clone() { + Some(l) => l, + None => return Err(vm.new_runtime_error("no running event loop")), + } + } + }; + + // Fast path: if the loop is the current thread's running loop, + // return the per-thread running task directly + let is_current_loop = vm + .asyncio_running_loop + .borrow() + .as_ref() + .is_some_and(|rl| rl.is(&loop_obj)); + + if is_current_loop { + return Ok(vm + .asyncio_running_task + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none())); + } + + // Slow path: look up in the module-level dict for cross-thread queries + let current_tasks = get_current_tasks_dict(vm)?; + let dict: PyDictRef = current_tasks.downcast().unwrap(); + + match dict.get_item(&*loop_obj, vm) { + Ok(task) => Ok(task), + Err(_) => Ok(vm.ctx.none()), + } + } + + #[pyfunction] + fn all_tasks(args: LoopArg, vm: &VirtualMachine) -> PyResult { + let loop_obj = match args.loop_.flatten() { + Some(l) if !vm.is_none(&l) => l, + _ => get_running_loop(vm)?, + }; + + let all_tasks_set = get_all_tasks_set(vm)?; + let result_set = PySet::default().into_ref(&vm.ctx); + + let iter = vm.call_method(&all_tasks_set, "__iter__", ())?; + loop { + match vm.call_method(&iter, "__next__", ()) { + Ok(task) => { + // Try get_loop() method first, fallback to _loop property + let task_loop = if let Ok(l) = vm.call_method(&task, "get_loop", ()) { + Some(l) + } else if let Ok(Some(l)) = + vm.get_attribute_opt(task.clone(), vm.ctx.intern_str("_loop")) + { + Some(l) + } else { + None + }; + + if let Some(task_loop) = task_loop + && task_loop.is(&loop_obj) + && let Ok(done) = vm.call_method(&task, "done", ()) + && !done.try_to_bool(vm).unwrap_or(true) + { + result_set.add(task, vm)?; + } + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) => break, + Err(e) => return Err(e), + } + } + + Ok(result_set.into()) + } + + #[pyfunction] + fn _register_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let all_tasks_set = get_all_tasks_set(vm)?; + vm.call_method(&all_tasks_set, "add", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _unregister_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let all_tasks_set = get_all_tasks_set(vm)?; + vm.call_method(&all_tasks_set, "discard", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _register_eager_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let eager_tasks_set = get_eager_tasks_set(vm)?; + vm.call_method(&eager_tasks_set, "add", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _unregister_eager_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let eager_tasks_set = get_eager_tasks_set(vm)?; + vm.call_method(&eager_tasks_set, "discard", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _enter_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Per-thread check, matching CPython's ts->asyncio_running_task + { + let running_task = vm.asyncio_running_task.borrow(); + if running_task.is_some() { + return Err(vm.new_runtime_error(format!( + "Cannot enter into task {:?} while another task {:?} is being executed.", + task, + running_task.as_ref().unwrap() + ))); + } + } + + *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); + + // Also update the module-level dict for cross-thread queries + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + let _ = dict.set_item(&*loop_, task, vm); + } + Ok(()) + } + + #[pyfunction] + fn _leave_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Per-thread check, matching CPython's ts->asyncio_running_task + { + let running_task = vm.asyncio_running_task.borrow(); + match running_task.as_ref() { + None => { + return Err(vm.new_runtime_error( + "_leave_task: task is not the current task".to_owned(), + )); + } + Some(current) if !current.is(&task) => { + return Err(vm.new_runtime_error( + "_leave_task: task is not the current task".to_owned(), + )); + } + _ => {} + } + } + + *vm.asyncio_running_task.borrow_mut() = None; + + // Also update the module-level dict + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + let _ = dict.del_item(&*loop_, vm); + } + Ok(()) + } + + #[pyfunction] + fn _swap_current_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Per-thread swap, matching CPython's swap_current_task + let prev = vm + .asyncio_running_task + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + + if vm.is_none(&task) { + *vm.asyncio_running_task.borrow_mut() = None; + } else { + *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); + } + + // Also update the module-level dict for cross-thread queries + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + if vm.is_none(&task) { + let _ = dict.del_item(&*loop_, vm); + } else { + let _ = dict.set_item(&*loop_, task, vm); + } + } + + Ok(prev) + } + + /// Reset task state after fork in child process. + #[pyfunction] + fn _on_fork(vm: &VirtualMachine) -> PyResult<()> { + // Clear current_tasks dict so child process doesn't inherit parent's tasks + if let Ok(current_tasks) = get_current_tasks_dict(vm) { + vm.call_method(&current_tasks, "clear", ())?; + } + // Clear the running loop and task + *vm.asyncio_running_loop.borrow_mut() = None; + *vm.asyncio_running_task.borrow_mut() = None; + Ok(()) + } + + #[pyfunction] + fn future_add_to_awaited_by( + fut: PyObjectRef, + waiter: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Only operate on native Future/Task objects (including subclasses). + // Non-native objects are silently ignored. + if let Some(task) = fut.downcast_ref::<PyTask>() { + return task.base.awaited_by_add(waiter, vm); + } + if let Some(future) = fut.downcast_ref::<PyFuture>() { + return future.awaited_by_add(waiter, vm); + } + Ok(()) + } + + #[pyfunction] + fn future_discard_from_awaited_by( + fut: PyObjectRef, + waiter: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Only operate on native Future/Task objects (including subclasses). + // Non-native objects are silently ignored. + if let Some(task) = fut.downcast_ref::<PyTask>() { + return task.base.awaited_by_discard(&waiter, vm); + } + if let Some(future) = fut.downcast_ref::<PyFuture>() { + return future.awaited_by_discard(&waiter, vm); + } + Ok(()) + } + + // TaskStepMethWrapper - wrapper for task step callback with proper repr + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct TaskStepMethWrapper { + task: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(Callable, Representable))] + impl TaskStepMethWrapper { + fn new(task: PyObjectRef) -> Self { + Self { + task: PyRwLock::new(Some(task)), + } + } + + // __self__ property returns the task, used by _format_handle in base_events.py + #[pygetset] + fn __self__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task.read().clone().unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match self.task.read().as_ref() { + Some(t) => vm.get_attribute_opt(t.clone(), vm.ctx.intern_str("__qualname__")), + None => Ok(None), + } + } + } + + impl Callable for TaskStepMethWrapper { + type Args = (); + fn call(zelf: &Py<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + let task = zelf.task.read().clone(); + match task { + Some(t) => task_step_impl(&t, None, vm), + None => Ok(vm.ctx.none()), + } + } + } + + impl Representable for TaskStepMethWrapper { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} object at {:#x}>", + zelf.class().name(), + zelf.get_id() + )) + } + } + + /// TaskWakeupMethWrapper - wrapper for task wakeup callback with proper repr + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct TaskWakeupMethWrapper { + task: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(Callable, Representable))] + impl TaskWakeupMethWrapper { + fn new(task: PyObjectRef) -> Self { + Self { + task: PyRwLock::new(Some(task)), + } + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match self.task.read().as_ref() { + Some(t) => vm.get_attribute_opt(t.clone(), vm.ctx.intern_str("__qualname__")), + None => Ok(None), + } + } + } + + impl Callable for TaskWakeupMethWrapper { + type Args = (PyObjectRef,); + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let task = zelf.task.read().clone(); + match task { + Some(t) => task_wakeup_impl(&t, &args.0, vm), + None => Ok(vm.ctx.none()), + } + } + } + + impl Representable for TaskWakeupMethWrapper { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} object at {:#x}>", + zelf.class().name(), + zelf.get_id() + )) + } + } + + fn is_coroutine(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + if obj.class().is(vm.ctx.types.coroutine_type) { + return Ok(true); + } + + let asyncio_coroutines = vm.import("asyncio.coroutines", 0)?; + if let Some(iscoroutine) = + vm.get_attribute_opt(asyncio_coroutines, vm.ctx.intern_str("iscoroutine"))? + { + let result = iscoroutine.call((obj,), vm)?; + result.try_to_bool(vm) + } else { + Ok(false) + } + } + + fn new_invalid_state_error(vm: &VirtualMachine, msg: &str) -> PyBaseExceptionRef { + match vm.import("asyncio.exceptions", 0) { + Ok(module) => { + match vm.get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError")) { + Ok(Some(exc_type)) => match exc_type.call((msg,), vm) { + Ok(exc) => exc.downcast().unwrap(), + Err(_) => vm.new_runtime_error(msg.to_string()), + }, + _ => vm.new_runtime_error(msg.to_string()), + } + } + Err(_) => vm.new_runtime_error(msg.to_string()), + } + } + + fn get_copy_context(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let contextvars = vm.import("contextvars", 0)?; + let copy_context = vm + .get_attribute_opt(contextvars, vm.ctx.intern_str("copy_context"))? + .ok_or_else(|| vm.new_attribute_error("copy_context not found"))?; + copy_context.call((), vm) + } + + fn get_cancelled_error_type(vm: &VirtualMachine) -> PyResult<PyTypeRef> { + let module = vm.import("asyncio.exceptions", 0)?; + let exc_type = vm + .get_attribute_opt(module, vm.ctx.intern_str("CancelledError"))? + .ok_or_else(|| vm.new_attribute_error("CancelledError not found"))?; + exc_type + .downcast() + .map_err(|_| vm.new_type_error("CancelledError is not a type".to_string())) + } + + fn is_cancelled_error(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => exc.fast_isinstance(&cancelled_error), + Err(_) => false, + } + } + + fn is_cancelled_error_obj(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => obj.fast_isinstance(&cancelled_error), + Err(_) => false, + } + } +} diff --git a/crates/stdlib/src/opcode.rs b/crates/stdlib/src/_opcode.rs similarity index 51% rename from crates/stdlib/src/opcode.rs rename to crates/stdlib/src/_opcode.rs index c355b59df91..f2b447e78b6 100644 --- a/crates/stdlib/src/opcode.rs +++ b/crates/stdlib/src/_opcode.rs @@ -1,34 +1,47 @@ -pub(crate) use opcode::make_module; +pub(crate) use _opcode::module_def; #[pymodule] -mod opcode { +mod _opcode { use crate::vm::{ AsObject, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyBool, PyInt, PyIntRef, PyNone}, - bytecode::Instruction, - match_class, + builtins::{PyInt, PyIntRef}, + bytecode::{AnyInstruction, Instruction, InstructionMetadata, PseudoInstruction}, }; - use std::ops::Deref; + use core::ops::Deref; - struct Opcode(Instruction); + #[derive(Clone, Copy)] + struct Opcode(AnyInstruction); impl Deref for Opcode { - type Target = Instruction; + type Target = AnyInstruction; fn deref(&self) -> &Self::Target { &self.0 } } + impl TryFrom<i32> for Opcode { + type Error = (); + + fn try_from(value: i32) -> Result<Self, Self::Error> { + Ok(Self( + u16::try_from(value) + .map_err(|_| ())? + .try_into() + .map_err(|_| ())?, + )) + } + } + impl Opcode { - // https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/opcode_ids.h#L238 - const HAVE_ARGUMENT: i32 = 44; + // https://github.com/python/cpython/blob/v3.14.2/Include/opcode_ids.h#L252 + const HAVE_ARGUMENT: i32 = 43; pub fn try_from_pyint(raw: PyIntRef, vm: &VirtualMachine) -> PyResult<Self> { let instruction = raw - .try_to_primitive::<u8>(vm) + .try_to_primitive::<u16>(vm) .and_then(|v| { - Instruction::try_from(v).map_err(|_| { + AnyInstruction::try_from(v).map_err(|_| { vm.new_exception_empty(vm.ctx.exceptions.value_error.to_owned()) }) }) @@ -37,77 +50,109 @@ mod opcode { Ok(Self(instruction)) } - /// https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/internal/pycore_opcode_metadata.h#L914-L916 - #[must_use] - pub const fn is_valid(opcode: i32) -> bool { - opcode >= 0 && opcode < 268 && opcode != 255 + const fn inner(self) -> AnyInstruction { + self.0 } - // All `has_*` methods below mimics - // https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Include/internal/pycore_opcode_metadata.h#L966-L1190 + /// Check if opcode is valid (can be converted to an AnyInstruction) + #[must_use] + pub fn is_valid(opcode: i32) -> bool { + Self::try_from(opcode).is_ok() + } + /// Check if instruction has an argument #[must_use] - pub const fn has_arg(opcode: i32) -> bool { + pub fn has_arg(opcode: i32) -> bool { Self::is_valid(opcode) && opcode > Self::HAVE_ARGUMENT } + /// Check if instruction uses co_consts #[must_use] - pub const fn has_const(opcode: i32) -> bool { - Self::is_valid(opcode) && matches!(opcode, 83 | 103 | 240) + pub fn has_const(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real(Instruction::LoadConst { .. })) + ) } + /// Check if instruction uses co_names #[must_use] - pub const fn has_name(opcode: i32) -> bool { - Self::is_valid(opcode) - && matches!( - opcode, - 63 | 66 - | 67 - | 74 - | 75 - | 82 - | 90 - | 91 - | 92 - | 93 - | 108 - | 113 - | 114 - | 259 - | 260 - | 261 - | 262 - ) + pub fn has_name(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteAttr { .. } + | Instruction::DeleteGlobal(_) + | Instruction::DeleteName(_) + | Instruction::ImportFrom { .. } + | Instruction::ImportName { .. } + | Instruction::LoadAttr { .. } + | Instruction::LoadGlobal(_) + | Instruction::LoadName(_) + | Instruction::StoreAttr { .. } + | Instruction::StoreGlobal(_) + | Instruction::StoreName(_) + )) + ) } + /// Check if instruction is a jump #[must_use] - pub const fn has_jump(opcode: i32) -> bool { - Self::is_valid(opcode) - && matches!( - opcode, - 72 | 77 | 78 | 79 | 97 | 98 | 99 | 100 | 104 | 256 | 257 - ) + pub fn has_jump(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::ForIter { .. } + | Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::Send { .. } + ) | AnyInstruction::Pseudo(PseudoInstruction::Jump { .. })) + ) } + /// Check if instruction uses co_freevars/co_cellvars #[must_use] - pub const fn has_free(opcode: i32) -> bool { - Self::is_valid(opcode) && matches!(opcode, 64 | 84 | 89 | 94 | 109) + pub fn has_free(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteDeref(_) + | Instruction::LoadFromDictOrDeref(_) + | Instruction::LoadDeref(_) + | Instruction::StoreDeref(_) + )) + ) } + /// Check if instruction uses co_varnames (local variables) #[must_use] - pub const fn has_local(opcode: i32) -> bool { - Self::is_valid(opcode) - && matches!(opcode, 65 | 85 | 86 | 87 | 88 | 110 | 111 | 112 | 258 | 267) + pub fn has_local(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteFast(_) + | Instruction::LoadFast(_) + | Instruction::LoadFastAndClear(_) + | Instruction::StoreFast(_) + | Instruction::StoreFastLoadFast { .. } + )) + ) } + /// Check if instruction has exception info #[must_use] - pub const fn has_exc(opcode: i32) -> bool { - Self::is_valid(opcode) && matches!(opcode, 264..=266) + pub fn has_exc(_opcode: i32) -> bool { + // No instructions have exception info in RustPython + // (exception handling is done via exception table) + false } } + // prepare specialization #[pyattr] const ENABLE_SPECIALIZATION: i8 = 1; + #[pyattr] + const ENABLE_SPECIALIZATION_FT: i8 = 1; #[derive(FromArgs)] struct StackEffectArgs { @@ -131,7 +176,12 @@ mod opcode { ))); } v.downcast_ref::<PyInt>() - .ok_or_else(|| vm.new_type_error(""))? + .ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + v.class().name() + )) + })? .try_to_primitive::<u32>(vm) }) .unwrap_or(Ok(0))?; @@ -139,19 +189,16 @@ mod opcode { let jump = args .jump .map(|v| { - match_class!(match v { - b @ PyBool => Ok(b.is(&vm.ctx.true_value)), - _n @ PyNone => Ok(false), - _ => { - Err(vm.new_value_error("stack_effect: jump must be False, True or None")) - } + v.try_to_bool(vm).map_err(|_| { + vm.new_value_error("stack_effect: jump must be False, True or None") }) }) .unwrap_or(Ok(false))?; let opcode = Opcode::try_from_pyint(args.opcode, vm)?; - Ok(opcode.stack_effect(oparg.into(), jump)) + let _ = jump; // Python API accepts jump but it's not used + Ok(opcode.stack_effect(oparg)) } #[pyfunction] @@ -259,6 +306,7 @@ mod opcode { ("NB_INPLACE_SUBTRACT", "-="), ("NB_INPLACE_TRUE_DIVIDE", "/="), ("NB_INPLACE_XOR", "^="), + ("NB_SUBSCR", "[]"), ] .into_iter() .map(|(a, b)| { @@ -270,8 +318,19 @@ mod opcode { } #[pyfunction] - fn get_executor(_code: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - // TODO + fn get_special_method_names(vm: &VirtualMachine) -> Vec<PyObjectRef> { + ["__enter__", "__exit__", "__aenter__", "__aexit__"] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_executor( + _code: PyObjectRef, + _offset: i32, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { Ok(vm.ctx.none()) } diff --git a/crates/stdlib/src/_remote_debugging.rs b/crates/stdlib/src/_remote_debugging.rs new file mode 100644 index 00000000000..57aa9876a01 --- /dev/null +++ b/crates/stdlib/src/_remote_debugging.rs @@ -0,0 +1,107 @@ +pub(crate) use _remote_debugging::module_def; + +#[pymodule] +mod _remote_debugging { + use crate::vm::{ + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::PyType, + function::FuncArgs, + types::{Constructor, PyStructSequence}, + }; + + #[pystruct_sequence_data] + struct FrameInfoData { + filename: String, + lineno: i64, + funcname: String, + } + + #[pyattr] + #[pystruct_sequence( + name = "FrameInfo", + module = "_remote_debugging", + data = "FrameInfoData" + )] + struct FrameInfo; + + #[pyclass(with(PyStructSequence))] + impl FrameInfo {} + + #[pystruct_sequence_data] + struct TaskInfoData { + task_id: PyObjectRef, + task_name: PyObjectRef, + coroutine_stack: PyObjectRef, + awaited_by: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "TaskInfo", module = "_remote_debugging", data = "TaskInfoData")] + struct TaskInfo; + + #[pyclass(with(PyStructSequence))] + impl TaskInfo {} + + #[pystruct_sequence_data] + struct CoroInfoData { + call_stack: PyObjectRef, + task_name: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "CoroInfo", module = "_remote_debugging", data = "CoroInfoData")] + struct CoroInfo; + + #[pyclass(with(PyStructSequence))] + impl CoroInfo {} + + #[pystruct_sequence_data] + struct ThreadInfoData { + thread_id: PyObjectRef, + frame_info: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence( + name = "ThreadInfo", + module = "_remote_debugging", + data = "ThreadInfoData" + )] + struct ThreadInfo; + + #[pyclass(with(PyStructSequence))] + impl ThreadInfo {} + + #[pystruct_sequence_data] + struct AwaitedInfoData { + thread_id: PyObjectRef, + awaited_by: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence( + name = "AwaitedInfo", + module = "_remote_debugging", + data = "AwaitedInfoData" + )] + struct AwaitedInfo; + + #[pyclass(with(PyStructSequence))] + impl AwaitedInfo {} + + #[pyattr] + #[pyclass(name = "RemoteUnwinder", module = "_remote_debugging")] + #[derive(Debug, PyPayload)] + struct RemoteUnwinder {} + + impl Constructor for RemoteUnwinder { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Err(vm.new_not_implemented_error("_remote_debugging is not available".to_owned())) + } + } + + #[pyclass(with(Constructor))] + impl RemoteUnwinder {} +} diff --git a/crates/stdlib/src/sqlite.rs b/crates/stdlib/src/_sqlite3.rs similarity index 92% rename from crates/stdlib/src/sqlite.rs rename to crates/stdlib/src/_sqlite3.rs index deff3c3a66a..f7ae445fe81 100644 --- a/crates/stdlib/src/sqlite.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -8,19 +8,16 @@ // spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath // spell-checker:ignore dbmoved vnode nbytes -use rustpython_vm::{AsObject, PyRef, VirtualMachine, builtins::PyModule}; - -// pub(crate) use _sqlite::make_module; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - // TODO: sqlite version check - let module = _sqlite::make_module(vm); - _sqlite::setup_module(module.as_object(), vm); - module -} +pub(crate) use _sqlite3::module_def; #[pymodule] -mod _sqlite { - use crossbeam_utils::atomic::AtomicCell; +mod _sqlite3 { + use core::{ + ffi::{CStr, c_int, c_longlong, c_uint, c_void}, + fmt::Debug, + ops::Deref, + ptr::{NonNull, null, null_mut}, + }; use libsqlite3_sys::{ SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, SQLITE_TRACE_STMT, @@ -30,21 +27,21 @@ mod _sqlite { sqlite3_bind_null, sqlite3_bind_parameter_count, sqlite3_bind_parameter_name, sqlite3_bind_text, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, sqlite3_blob_read, sqlite3_blob_write, sqlite3_busy_timeout, sqlite3_changes, - sqlite3_close_v2, sqlite3_column_blob, sqlite3_column_bytes, sqlite3_column_count, - sqlite3_column_decltype, sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, - sqlite3_column_text, sqlite3_column_type, sqlite3_complete, sqlite3_context, - sqlite3_context_db_handle, sqlite3_create_collation_v2, sqlite3_create_function_v2, - sqlite3_create_window_function, sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, - sqlite3_errmsg, sqlite3_exec, sqlite3_expanded_sql, sqlite3_extended_errcode, - sqlite3_finalize, sqlite3_get_autocommit, sqlite3_interrupt, sqlite3_last_insert_rowid, - sqlite3_libversion, sqlite3_limit, sqlite3_open_v2, sqlite3_prepare_v2, - sqlite3_progress_handler, sqlite3_reset, sqlite3_result_blob, sqlite3_result_double, - sqlite3_result_error, sqlite3_result_error_nomem, sqlite3_result_error_toobig, - sqlite3_result_int64, sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, - sqlite3_sleep, sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, - sqlite3_threadsafe, sqlite3_total_changes, sqlite3_trace_v2, sqlite3_user_data, - sqlite3_value, sqlite3_value_blob, sqlite3_value_bytes, sqlite3_value_double, - sqlite3_value_int64, sqlite3_value_text, sqlite3_value_type, + sqlite3_column_blob, sqlite3_column_bytes, sqlite3_column_count, sqlite3_column_decltype, + sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text, + sqlite3_column_type, sqlite3_complete, sqlite3_context, sqlite3_context_db_handle, + sqlite3_create_collation_v2, sqlite3_create_function_v2, sqlite3_create_window_function, + sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec, + sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit, + sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit, + sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset, + sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error, + sqlite3_result_error_nomem, sqlite3_result_error_toobig, sqlite3_result_int64, + sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep, + sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe, + sqlite3_total_changes, sqlite3_trace_v2, sqlite3_user_data, sqlite3_value, + sqlite3_value_blob, sqlite3_value_bytes, sqlite3_value_double, sqlite3_value_int64, + sqlite3_value_text, sqlite3_value_type, }; use malachite_bigint::Sign; use rustpython_common::{ @@ -59,8 +56,8 @@ mod _sqlite { TryFromBorrowedObject, VirtualMachine, atomic_func, builtins::{ PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, - PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - PyUtf8Str, PyUtf8StrRef, + PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyTypeRef, PyUtf8Str, PyUtf8StrRef, }, convert::IntoObject, function::{ @@ -75,17 +72,11 @@ mod _sqlite { sliceable::{SaturatedSliceIter, SliceableSequenceOp}, types::{ AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, - Initializer, IterNext, Iterable, PyComparisonOp, SelfIter, Unconstructible, + Initializer, IterNext, Iterable, PyComparisonOp, SelfIter, }, utils::ToCString, }; - use std::{ - ffi::{CStr, c_int, c_longlong, c_uint, c_void}, - fmt::Debug, - ops::Deref, - ptr::{NonNull, null, null_mut}, - thread::ThreadId, - }; + use std::thread::ThreadId; macro_rules! exceptions { ($(($x:ident, $base:expr)),*) => { @@ -161,6 +152,8 @@ mod _sqlite { const PARSE_DECLTYPES: c_int = 1; #[pyattr] const PARSE_COLNAMES: c_int = 2; + #[pyattr] + const LEGACY_TRANSACTION_CONTROL: c_int = -1; #[pyattr] use libsqlite3_sys::{ @@ -301,6 +294,41 @@ mod _sqlite { SQLITE_IOERR_CORRUPTFS ); + /// Autocommit mode setting for sqlite3 connections. + /// - Legacy (default): use isolation_level to control transactions + /// - Enabled: autocommit mode (no automatic transactions) + /// - Disabled: manual commit mode + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + enum AutocommitMode { + #[default] + Legacy, + Enabled, + Disabled, + } + + impl TryFromBorrowedObject<'_> for AutocommitMode { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Self> { + if obj.is(&vm.ctx.true_value) { + Ok(Self::Enabled) + } else if obj.is(&vm.ctx.false_value) { + Ok(Self::Disabled) + } else if let Ok(val) = obj.try_to_value::<c_int>(vm) { + if val == LEGACY_TRANSACTION_CONTROL { + Ok(Self::Legacy) + } else { + Err(vm.new_value_error(format!( + "autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {val}" + ))) + } + } else { + Err(vm.new_type_error(format!( + "autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {}", + obj.class().name() + ))) + } + } + } + #[derive(FromArgs)] struct ConnectArgs { #[pyarg(any)] @@ -321,6 +349,8 @@ mod _sqlite { cached_statements: c_int, #[pyarg(any, default = false)] uri: bool, + #[pyarg(any, default)] + autocommit: AutocommitMode, } unsafe impl Traverse for ConnectArgs { @@ -387,6 +417,12 @@ mod _sqlite { name: PyStrRef, } + #[derive(FromArgs)] + struct CursorArgs { + #[pyarg(any, default)] + factory: OptionalArg<PyObjectRef>, + } + struct CallbackData { obj: NonNull<PyObject>, vm: *const VirtualMachine, @@ -415,7 +451,7 @@ mod _sqlite { ) { let context = SqliteContext::from(context); let (func, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; - let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; let f = || -> PyResult<()> { let db = context.db_handle(); @@ -442,7 +478,7 @@ mod _sqlite { ) { let context = SqliteContext::from(context); let (cls, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; - let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; let instance = context.aggregate_context::<*const PyObject>(); if unsafe { (*instance).is_null() } { match cls.call((), vm) { @@ -488,7 +524,7 @@ mod _sqlite { let text2 = vm.ctx.new_str(text2); let val = callable.call((text1, text2), vm)?; - let Some(val) = val.to_number().index(vm) else { + let Some(val) = val.number().index(vm) else { return Ok(0); }; @@ -520,7 +556,7 @@ mod _sqlite { ) { let context = SqliteContext::from(context); let (_, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; - let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; let instance = context.aggregate_context::<*const PyObject>(); let instance = unsafe { &**instance }; @@ -808,26 +844,26 @@ mod _sqlite { .expect("enable traceback not initialize") } - pub(super) fn setup_module(module: &PyObject, vm: &VirtualMachine) { + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + for (name, code) in ERROR_CODES { let name = vm.ctx.intern_str(*name); let code = vm.new_pyobj(*code); - module.set_attr(name, code, vm).unwrap(); + module.set_attr(name, code, vm)?; } - setup_module_exceptions(module, vm); + setup_module_exceptions(module.as_object(), vm); let _ = CONVERTERS.set(vm.ctx.new_dict()); let _ = ADAPTERS.set(vm.ctx.new_dict()); let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); let _ = ENABLE_TRACEBACK.set(Radium::new(false)); - module - .set_attr("converters", converters().to_owned(), vm) - .unwrap(); - module - .set_attr("adapters", adapters().to_owned(), vm) - .unwrap(); + module.set_attr("converters", converters().to_owned(), vm)?; + module.set_attr("adapters", adapters().to_owned(), vm)?; + + Ok(()) } #[pyattr] @@ -842,10 +878,11 @@ mod _sqlite { thread_ident: PyMutex<ThreadId>, // TODO: Use atomic row_factory: PyAtomicRef<Option<PyObject>>, text_factory: PyAtomicRef<PyObject>, + autocommit: PyMutex<AutocommitMode>, } impl Debug for Connection { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { write!(f, "Sqlite3 Connection") } } @@ -879,6 +916,7 @@ mod _sqlite { thread_ident: PyMutex::new(std::thread::current().id()), row_factory: PyAtomicRef::from(None), text_factory: PyAtomicRef::from(text_factory), + autocommit: PyMutex::new(args.autocommit), }) } } @@ -920,12 +958,14 @@ mod _sqlite { detect_types, isolation_level, check_same_thread, + autocommit, .. } = args; zelf.detect_types.store(detect_types, Ordering::Relaxed); zelf.check_same_thread .store(check_same_thread, Ordering::Relaxed); + *zelf.autocommit.lock() = autocommit; *zelf.thread_ident.lock() = std::thread::current().id(); let _ = unsafe { zelf.isolation_level.swap(isolation_level) }; @@ -981,22 +1021,29 @@ mod _sqlite { #[pymethod] fn cursor( zelf: PyRef<Self>, - factory: OptionalArg<ArgCallable>, + args: CursorArgs, vm: &VirtualMachine, - ) -> PyResult<PyRef<Cursor>> { + ) -> PyResult<PyObjectRef> { zelf.db_lock(vm).map(drop)?; - let cursor = if let OptionalArg::Present(factory) = factory { - let cursor = factory.invoke((zelf.clone(),), vm)?; - let cursor = cursor.downcast::<Cursor>().map_err(|x| { - vm.new_type_error(format!("factory must return a cursor, not {}", x.class())) - })?; - let _ = unsafe { cursor.row_factory.swap(zelf.row_factory.to_owned()) }; - cursor - } else { - let row_factory = zelf.row_factory.to_owned(); - Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx) + let factory = match args.factory { + OptionalArg::Present(f) => f, + OptionalArg::Missing => Cursor::class(&vm.ctx).to_owned().into(), }; + + let cursor = factory.call((zelf.clone(),), vm)?; + + if !cursor.class().fast_issubclass(Cursor::class(&vm.ctx)) { + return Err(vm.new_type_error(format!( + "factory must return a cursor, not {}", + cursor.class() + ))); + } + + if let Some(cursor_ref) = cursor.downcast_ref::<Cursor>() { + let _ = unsafe { cursor_ref.row_factory.swap(zelf.row_factory.to_owned()) }; + } + Ok(cursor) } @@ -1349,14 +1396,14 @@ mod _sqlite { fn set_trace_callback(&self, callable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let db = self.db_lock(vm)?; let Some(data) = CallbackData::new(callable, vm) else { - unsafe { sqlite3_trace_v2(db.db, SQLITE_TRACE_STMT as u32, None, null_mut()) }; + unsafe { sqlite3_trace_v2(db.db, SQLITE_TRACE_STMT, None, null_mut()) }; return Ok(()); }; let ret = unsafe { sqlite3_trace_v2( db.db, - SQLITE_TRACE_STMT as u32, + SQLITE_TRACE_STMT, Some(CallbackData::trace_callback), Box::into_raw(Box::new(data)).cast(), ) @@ -1466,6 +1513,43 @@ mod _sqlite { } } + #[pygetset] + fn autocommit(&self, vm: &VirtualMachine) -> PyObjectRef { + match *self.autocommit.lock() { + AutocommitMode::Enabled => vm.ctx.true_value.clone().into(), + AutocommitMode::Disabled => vm.ctx.false_value.clone().into(), + AutocommitMode::Legacy => vm.ctx.new_int(LEGACY_TRANSACTION_CONTROL).into(), + } + } + #[pygetset(setter)] + fn set_autocommit(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mode = AutocommitMode::try_from_borrowed_object(vm, &val)?; + let db = self.db_lock(vm)?; + + // Handle transaction state based on mode change + match mode { + AutocommitMode::Enabled => { + // If there's a pending transaction, commit it + if !db.is_autocommit() { + db._exec(b"COMMIT�", vm)?; + } + } + AutocommitMode::Disabled => { + // If not in a transaction, begin one + if db.is_autocommit() { + db._exec(b"BEGIN�", vm)?; + } + } + AutocommitMode::Legacy => { + // Legacy mode doesn't change transaction state + } + } + + drop(db); + *self.autocommit.lock() = mode; + Ok(()) + } + #[pygetset] fn text_factory(&self) -> PyObjectRef { self.text_factory.to_owned() @@ -1540,7 +1624,7 @@ mod _sqlite { size: Option<c_int>, } - #[pyclass(with(Constructor, IterNext, Iterable), flags(BASETYPE))] + #[pyclass(with(Constructor, Initializer, IterNext, Iterable), flags(BASETYPE))] impl Cursor { fn new( connection: PyRef<Connection>, @@ -1571,24 +1655,6 @@ mod _sqlite { } } - #[pymethod] - fn __init__(&self, _connection: PyRef<Connection>, _vm: &VirtualMachine) -> PyResult<()> { - let mut guard = self.inner.lock(); - if guard.is_some() { - // Already initialized (e.g., from a call to super().__init__) - return Ok(()); - } - *guard = Some(CursorInner { - description: None, - row_cast_map: vec![], - lastrowid: -1, - rowcount: -1, - statement: None, - closed: false, - }); - Ok(()) - } - fn check_cursor_state(inner: Option<&CursorInner>, vm: &VirtualMachine) -> PyResult<()> { match inner { Some(inner) if inner.closed => Err(new_programming_error( @@ -1641,9 +1707,11 @@ mod _sqlite { let db = zelf.connection.db_lock(vm)?; + // Start implicit transaction for DML statements unless in autocommit mode if stmt.is_dml && db.is_autocommit() && zelf.connection.isolation_level.deref().is_some() + && *zelf.connection.autocommit.lock() != AutocommitMode::Enabled { db.begin_transaction( zelf.connection @@ -1734,9 +1802,11 @@ mod _sqlite { let db = zelf.connection.db_lock(vm)?; + // Start implicit transaction for DML statements unless in autocommit mode if stmt.is_dml && db.is_autocommit() && zelf.connection.isolation_level.deref().is_some() + && *zelf.connection.autocommit.lock() != AutocommitMode::Enabled { db.begin_transaction( zelf.connection @@ -1949,9 +2019,42 @@ mod _sqlite { } } + impl Initializer for Cursor { + type Args = PyRef<Connection>; + + fn init(zelf: PyRef<Self>, _connection: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + let mut guard = zelf.inner.lock(); + if guard.is_some() { + // Already initialized (e.g., from a call to super().__init__) + return Ok(()); + } + *guard = Some(CursorInner { + description: None, + row_cast_map: vec![], + lastrowid: -1, + rowcount: -1, + statement: None, + closed: false, + }); + Ok(()) + } + } + impl SelfIter for Cursor {} impl IterNext for Cursor { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // Check if connection is closed first, and if so, clear statement to release file lock + if zelf.connection.is_closed() { + let mut guard = zelf.inner.lock(); + if let Some(stmt) = guard.as_mut().and_then(|inner| inner.statement.take()) { + stmt.lock().reset(); + } + return Err(new_programming_error( + vm, + "Cannot operate on a closed database.".to_owned(), + )); + } + let mut inner = zelf.inner(vm)?; let Some(stmt) = &inner.statement else { return Ok(PyIterReturn::StopIteration(None)); @@ -1974,7 +2077,7 @@ mod _sqlite { } else { let nbytes = st.column_bytes(i); let blob = unsafe { - std::slice::from_raw_parts(blob.cast::<u8>(), nbytes as usize) + core::slice::from_raw_parts(blob.cast::<u8>(), nbytes as usize) }; let blob = vm.ctx.new_bytes(blob.to_vec()); converter.call((blob,), vm)? @@ -2197,8 +2300,6 @@ mod _sqlite { inner: PyMutex<Option<BlobInner>>, } - impl Unconstructible for Blob {} - #[derive(Debug)] struct BlobInner { blob: SqliteBlob, @@ -2211,7 +2312,7 @@ mod _sqlite { } } - #[pyclass(with(AsMapping, Unconstructible, AsNumber, AsSequence))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(AsMapping, AsNumber, AsSequence))] impl Blob { #[pymethod] fn close(&self) { @@ -2548,19 +2649,19 @@ mod _sqlite { impl AsSequence for Blob { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: AtomicCell::new(None), - concat: AtomicCell::new(None), - repeat: AtomicCell::new(None), - item: AtomicCell::new(None), - ass_item: AtomicCell::new(None), + length: None, + concat: None, + repeat: None, + item: None, + ass_item: None, contains: atomic_func!(|seq, _needle, vm| { Err(vm.new_type_error(format!( "argument of type '{}' is not iterable", seq.obj.class().name(), ))) }), - inplace_concat: AtomicCell::new(None), - inplace_repeat: AtomicCell::new(None), + inplace_concat: None, + inplace_repeat: None, }; &AS_SEQUENCE } @@ -2583,7 +2684,7 @@ mod _sqlite { } impl Debug for Statement { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { write!( f, "{} Statement", @@ -2592,9 +2693,7 @@ mod _sqlite { } } - impl Unconstructible for Statement {} - - #[pyclass(with(Unconstructible))] + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl Statement { fn new( connection: &Connection, @@ -2660,8 +2759,16 @@ mod _sqlite { } } + // sqlite3_close_v2 is not exported by libsqlite3-sys, so we declare it manually. + // It handles "zombie close" - if there are still unfinalized statements, + // the database will be closed when the last statement is finalized. + unsafe extern "C" { + fn sqlite3_close_v2(db: *mut sqlite3) -> c_int; + } + impl Drop for Sqlite { fn drop(&mut self) { + // Use sqlite3_close_v2 for safe closing even with active statements unsafe { sqlite3_close_v2(self.raw.db) }; } } @@ -2981,7 +3088,7 @@ mod _sqlite { fn bind_parameters(self, parameters: &PyObject, vm: &VirtualMachine) -> PyResult<()> { if let Some(dict) = parameters.downcast_ref::<PyDict>() { self.bind_parameters_name(dict, vm) - } else if let Ok(seq) = PySequence::try_protocol(parameters, vm) { + } else if let Ok(seq) = parameters.try_sequence(vm) { self.bind_parameters_sequence(seq, vm) } else { Err(new_programming_error( @@ -3193,7 +3300,9 @@ mod _sqlite { } fn aggregate_context<T>(self) -> *mut T { - unsafe { sqlite3_aggregate_context(self.ctx, std::mem::size_of::<T>() as c_int).cast() } + unsafe { + sqlite3_aggregate_context(self.ctx, core::mem::size_of::<T>() as c_int).cast() + } } fn result_exception(self, vm: &VirtualMachine, exc: PyBaseExceptionRef, msg: &str) { @@ -3299,7 +3408,7 @@ mod _sqlite { } else if nbytes < 0 { Err(vm.new_system_error("negative size with ptr")) } else { - Ok(unsafe { std::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) + Ok(unsafe { core::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) } } diff --git a/crates/stdlib/src/_testconsole.rs b/crates/stdlib/src/_testconsole.rs new file mode 100644 index 00000000000..0db508e3da5 --- /dev/null +++ b/crates/stdlib/src/_testconsole.rs @@ -0,0 +1,76 @@ +pub(crate) use _testconsole::module_def; + +#[pymodule] +mod _testconsole { + use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, convert::IntoPyException, function::ArgBytesLike, + }; + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + + type Handle = windows_sys::Win32::Foundation::HANDLE; + + #[pyfunction] + fn write_input(file: PyObjectRef, s: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Console::{INPUT_RECORD, KEY_EVENT, WriteConsoleInputW}; + + // Get the fd from the file object via fileno() + let fd_obj = vm.call_method(&file, "fileno", ())?; + let fd: i32 = fd_obj.try_into_value(vm)?; + + let handle = unsafe { libc::get_osfhandle(fd) } as Handle; + if handle == INVALID_HANDLE_VALUE { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + let data = s.borrow_buf(); + let data = &*data; + + // Interpret as UTF-16-LE pairs + if !data.len().is_multiple_of(2) { + return Err(vm.new_value_error("buffer must contain UTF-16-LE data (even length)")); + } + let wchars: Vec<u16> = data + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + + let size = wchars.len() as u32; + + // Create INPUT_RECORD array + let mut records: Vec<INPUT_RECORD> = Vec::with_capacity(wchars.len()); + for &wc in &wchars { + // SAFETY: zeroing and accessing the union field for KEY_EVENT + let mut rec: INPUT_RECORD = unsafe { core::mem::zeroed() }; + rec.EventType = KEY_EVENT as u16; + rec.Event.KeyEvent.bKeyDown = 1; // TRUE + rec.Event.KeyEvent.wRepeatCount = 1; + rec.Event.KeyEvent.uChar.UnicodeChar = wc; + records.push(rec); + } + + let mut total: u32 = 0; + while total < size { + let mut wrote: u32 = 0; + let res = unsafe { + WriteConsoleInputW( + handle, + records[total as usize..].as_ptr(), + size - total, + &mut wrote, + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + total += wrote; + } + + Ok(()) + } + + #[pyfunction] + fn read_output(_file: PyObjectRef) -> Option<()> { + // Stub, same as CPython + None + } +} diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 4fcba1f8725..9877d0dcc4f 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -1,33 +1,6 @@ // spell-checker:ignore typecode tofile tolist fromfile -use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = array::make_module(vm); - - let array = module - .get_attr("array", vm) - .expect("Expect array has array type."); - - let collections_abc = vm - .import("collections.abc", 0) - .expect("Expect collections exist."); - let abc = collections_abc - .get_attr("abc", vm) - .expect("Expect collections has abc submodule."); - let mutable_sequence = abc - .get_attr("MutableSequence", vm) - .expect("Expect collections.abc has MutableSequence type."); - - let register = &mutable_sequence - .get_attr("register", vm) - .expect("Expect collections.abc.MutableSequence has register method."); - register - .call((array,), vm) - .expect("Expect collections.abc.MutableSequence.register(array.array) not fail."); - - module -} +pub(crate) use array::module_def; #[pymodule(name = "array")] mod array { @@ -46,7 +19,7 @@ mod array { builtins::{ PositionIterInternal, PyByteArray, PyBytes, PyBytesRef, PyDictRef, PyFloat, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTupleRef, PyType, - PyTypeRef, + PyTypeRef, builtins_iter, }, class_or_notimplemented, convert::{ToPyObject, ToPyResult, TryFromBorrowedObject, TryFromObject}, @@ -62,17 +35,19 @@ mod array { SaturatedSlice, SequenceIndex, SequenceIndexOp, SliceableSequenceMutOp, SliceableSequenceOp, }, + stdlib::warnings, types::{ AsBuffer, AsMapping, AsSequence, Comparable, Constructor, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, }, }; + use alloc::fmt; + use core::cmp::Ordering; use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; - use std::{cmp::Ordering, fmt, os::raw}; - + use std::os::raw; macro_rules! def_array_enum { ($(($n:ident, $t:ty, $c:literal, $scode:literal)),*$(,)?) => { #[derive(Debug, Clone)] @@ -104,14 +79,14 @@ mod array { const fn itemsize_of_typecode(c: char) -> Option<usize> { match c { - $($c => Some(std::mem::size_of::<$t>()),)* + $($c => Some(core::mem::size_of::<$t>()),)* _ => None, } } const fn itemsize(&self) -> usize { match self { - $(ArrayContentType::$n(_) => std::mem::size_of::<$t>(),)* + $(ArrayContentType::$n(_) => core::mem::size_of::<$t>(),)* } } @@ -181,6 +156,13 @@ mod array { } } + fn clear(&mut self) -> PyResult<()>{ + match self { + $(ArrayContentType::$n(v) => v.clear(),)* + }; + Ok(()) + } + fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ match self { $(ArrayContentType::$n(v) => { @@ -201,10 +183,10 @@ mod array { if v.is_empty() { // safe because every configuration of bytes for the types we // support are valid - let b = std::mem::ManuallyDrop::new(b); + let b = core::mem::ManuallyDrop::new(b); let ptr = b.as_ptr() as *mut $t; - let len = b.len() / std::mem::size_of::<$t>(); - let capacity = b.capacity() / std::mem::size_of::<$t>(); + let len = b.len() / core::mem::size_of::<$t>(); + let capacity = b.capacity() / core::mem::size_of::<$t>(); *v = unsafe { Vec::from_raw_parts(ptr, len, capacity) }; } else { self.frombytes(&b); @@ -220,8 +202,8 @@ mod array { // support are valid if b.len() > 0 { let ptr = b.as_ptr() as *const $t; - let ptr_len = b.len() / std::mem::size_of::<$t>(); - let slice = unsafe { std::slice::from_raw_parts(ptr, ptr_len) }; + let ptr_len = b.len() / core::mem::size_of::<$t>(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ptr_len) }; v.extend_from_slice(slice); } })* @@ -249,8 +231,8 @@ mod array { $(ArrayContentType::$n(v) => { // safe because we're just reading memory as bytes let ptr = v.as_ptr() as *const u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts(ptr, ptr_len) } + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts(ptr, ptr_len) } })* } } @@ -260,8 +242,8 @@ mod array { $(ArrayContentType::$n(v) => { // safe because we're just reading memory as bytes let ptr = v.as_ptr() as *mut u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts_mut(ptr, ptr_len) } + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts_mut(ptr, ptr_len) } })* } } @@ -564,11 +546,11 @@ mod array { } fn f32_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f32> { - ArgIntoFloat::try_from_object(vm, obj).map(|x| *x as f32) + ArgIntoFloat::try_from_object(vm, obj).map(|x| x.into_float() as f32) } fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f64> { - ArgIntoFloat::try_from_object(vm, obj).map(Into::into) + ArgIntoFloat::try_from_object(vm, obj).map(|x| x.into_float()) } fn pyfloat_from_f32(value: f32) -> PyFloat { @@ -664,6 +646,15 @@ mod array { return Err(vm.new_type_error("array.array() takes no keyword arguments")); } + if spec == 'u' { + warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "The 'u' type code is deprecated and will be removed in Python 3.16".to_owned(), + 1, + vm, + )?; + } + let mut array = ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err))?; @@ -744,6 +735,11 @@ mod array { zelf.try_resizable(vm)?.push(x, vm) } + #[pymethod] + fn clear(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?.clear() + } + #[pymethod] fn buffer_info(&self) -> (usize, usize) { let array = self.read(); @@ -785,18 +781,18 @@ mod array { if item_size == 2 { // safe because every configuration of bytes for the types we support are valid let utf16 = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( bytes.as_ptr() as *const u16, - bytes.len() / std::mem::size_of::<u16>(), + bytes.len() / core::mem::size_of::<u16>(), ) }; Ok(Wtf8Buf::from_wide(utf16)) } else { // safe because every configuration of bytes for the types we support are valid let chars = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( bytes.as_ptr() as *const u32, - bytes.len() / std::mem::size_of::<u32>(), + bytes.len() / core::mem::size_of::<u32>(), ) }; chars @@ -938,8 +934,10 @@ mod array { /* XXX Make the block size settable */ const BLOCKSIZE: usize = 64 * 1024; - let bytes = self.read(); - let bytes = bytes.get_bytes(); + let bytes = { + let bytes = self.read(); + bytes.get_bytes().to_vec() + }; for b in bytes.chunks(BLOCKSIZE) { let b = PyBytes::from(b.to_vec()).into_ref(&vm.ctx); @@ -993,7 +991,6 @@ mod array { } } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.getitem_inner(&needle, vm) } @@ -1035,7 +1032,6 @@ mod array { } } - #[pymethod] fn __setitem__( zelf: &Py<Self>, needle: PyObjectRef, @@ -1052,12 +1048,10 @@ mod array { } } - #[pymethod] fn __delitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.delitem_inner(&needle, vm) } - #[pymethod] fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { if let Some(other) = other.downcast_ref::<Self>() { self.read() @@ -1071,7 +1065,6 @@ mod array { } } - #[pymethod] fn __iadd__( zelf: PyRef<Self>, other: PyObjectRef, @@ -1090,21 +1083,17 @@ mod array { Ok(zelf) } - #[pymethod(name = "__rmul__")] - #[pymethod] fn __mul__(&self, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { self.read() .mul(value, vm) .map(|x| Self::from(x).into_ref(&vm.ctx)) } - #[pymethod] fn __imul__(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { zelf.try_resizable(vm)?.imul(value, vm)?; Ok(zelf) } - #[pymethod] pub(crate) fn __len__(&self) -> usize { self.read().len() } @@ -1179,7 +1168,6 @@ mod array { )) } - #[pymethod] fn __contains__(&self, value: PyObjectRef, vm: &VirtualMachine) -> bool { let array = self.array.read(); for element in array @@ -1399,7 +1387,7 @@ mod array { internal: PyMutex<PositionIterInternal<PyArrayRef>>, } - #[pyclass(with(IterNext, Iterable), flags(HAS_DICT))] + #[pyclass(with(IterNext, Iterable), flags(HAS_DICT, DISALLOW_INSTANTIATION))] impl PyArrayIter { #[pymethod] fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { @@ -1410,9 +1398,13 @@ mod array { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } } @@ -1514,7 +1506,7 @@ mod array { impl MachineFormatCode { fn from_typecode(code: char) -> Option<Self> { - use std::mem::size_of; + use core::mem::size_of; let signed = code.is_ascii_uppercase(); let big_endian = cfg!(target_endian = "big"); let int_size = match code { @@ -1588,7 +1580,7 @@ mod array { macro_rules! chunk_to_obj { ($BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); + let b = <[u8; ::core::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); if $BIG_ENDIAN { <$TY>::from_be_bytes(b) } else { @@ -1599,7 +1591,7 @@ mod array { chunk_to_obj!($BYTE, $TY, $BIG_ENDIAN).to_pyobject($VM) }; ($VM:ident, $BYTE:ident, $SIGNED_TY:ty, $UNSIGNED_TY:ty, $SIGNED:ident, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); + let b = <[u8; ::core::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); match ($SIGNED, $BIG_ENDIAN) { (false, false) => <$UNSIGNED_TY>::from_le_bytes(b).to_pyobject($VM), (false, true) => <$UNSIGNED_TY>::from_be_bytes(b).to_pyobject($VM), @@ -1665,4 +1657,25 @@ mod array { }; PyArray::from(array).into_ref_with_type(vm, cls) } + + // Register array.array as collections.abc.MutableSequence + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::vm::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + let array_type = module + .get_attr("array", vm) + .expect("array module has array type"); + + // vm.import returns the top-level module, so we need to get abc submodule + let collections_abc = vm.import("collections.abc", 0)?; + let abc = collections_abc.get_attr("abc", vm)?; + let mutable_sequence = abc.get_attr("MutableSequence", vm)?; + let register = mutable_sequence.get_attr("register", vm)?; + register.call((array_type,), vm)?; + + Ok(()) + } } diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs index a2316d3c204..ee55d482e4c 100644 --- a/crates/stdlib/src/binascii.rs +++ b/crates/stdlib/src/binascii.rs @@ -1,7 +1,7 @@ // spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode pub(super) use decl::crc32; -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException}; const PAD: u8 = 61u8; @@ -359,7 +359,7 @@ mod decl { } _ => unsafe { // quad_pos is only assigned in this match statement to constants - std::hint::unreachable_unchecked() + core::hint::unreachable_unchecked() }, } } diff --git a/crates/stdlib/src/bisect.rs b/crates/stdlib/src/bisect.rs index 46e689ac068..69b6e8aee46 100644 --- a/crates/stdlib/src/bisect.rs +++ b/crates/stdlib/src/bisect.rs @@ -1,4 +1,4 @@ -pub(crate) use _bisect::make_module; +pub(crate) use _bisect::module_def; #[pymodule] mod _bisect { @@ -24,7 +24,7 @@ mod _bisect { #[inline] fn handle_default(arg: OptionalArg<ArgIndex>, vm: &VirtualMachine) -> PyResult<Option<isize>> { arg.into_option() - .map(|v| v.try_to_primitive(vm)) + .map(|v| v.into_int_ref().try_to_primitive(vm)) .transpose() } diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs index 4209c966e86..53687c7027a 100644 --- a/crates/stdlib/src/blake2.rs +++ b/crates/stdlib/src/blake2.rs @@ -1,6 +1,6 @@ // spell-checker:ignore usedforsecurity HASHXOF -pub(crate) use _blake2::make_module; +pub(crate) use _blake2::module_def; #[pymodule] mod _blake2 { @@ -9,11 +9,11 @@ mod _blake2 { #[pyfunction] fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_blake2b(args).into_pyobject(vm)) + Ok(local_blake2b(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_blake2s(args).into_pyobject(vm)) + Ok(local_blake2s(args, vm)?.into_pyobject(vm)) } } diff --git a/crates/stdlib/src/bz2.rs b/crates/stdlib/src/bz2.rs index a2a40953cff..575f33c4b8f 100644 --- a/crates/stdlib/src/bz2.rs +++ b/crates/stdlib/src/bz2.rs @@ -1,6 +1,6 @@ // spell-checker:ignore compresslevel -pub(crate) use _bz2::make_module; +pub(crate) use _bz2::module_def; #[pymodule] mod _bz2 { @@ -15,9 +15,10 @@ mod _bz2 { object::PyResult, types::Constructor, }; + use alloc::fmt; use bzip2::{Decompress, Status, write::BzEncoder}; use rustpython_vm::convert::ToPyException; - use std::{fmt, io::Write}; + use std::io::Write; const BUFSIZ: usize = 8192; diff --git a/crates/stdlib/src/cmath.rs b/crates/stdlib/src/cmath.rs index e5d1d55a578..e7ea317d212 100644 --- a/crates/stdlib/src/cmath.rs +++ b/crates/stdlib/src/cmath.rs @@ -1,6 +1,5 @@ -// TODO: Keep track of rust-num/num-complex/issues/2. A common trait could help with duplication -// that exists between cmath and math. -pub(crate) use cmath::make_module; +pub(crate) use cmath::module_def; + #[pymodule] mod cmath { use crate::vm::{ @@ -9,137 +8,141 @@ mod cmath { }; use num_complex::Complex64; + use crate::math::pymath_exception; + // Constants - #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; + #[pyattr(name = "e")] + const E: f64 = pymath::cmath::E; + #[pyattr(name = "pi")] + const PI: f64 = pymath::cmath::PI; + #[pyattr(name = "tau")] + const TAU: f64 = pymath::cmath::TAU; #[pyattr(name = "inf")] - const INF: f64 = f64::INFINITY; + const INF: f64 = pymath::cmath::INF; #[pyattr(name = "nan")] - const NAN: f64 = f64::NAN; + const NAN: f64 = pymath::cmath::NAN; #[pyattr(name = "infj")] - const INFJ: Complex64 = Complex64::new(0., f64::INFINITY); + const INFJ: Complex64 = pymath::cmath::INFJ; #[pyattr(name = "nanj")] - const NANJ: Complex64 = Complex64::new(0., f64::NAN); + const NANJ: Complex64 = pymath::cmath::NANJ; #[pyfunction] - fn phase(z: ArgIntoComplex) -> f64 { - z.arg() + fn phase(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<f64> { + pymath::cmath::phase(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn polar(x: ArgIntoComplex) -> (f64, f64) { - x.to_polar() + fn polar(x: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + pymath::cmath::polar(x.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn rect(r: ArgIntoFloat, phi: ArgIntoFloat) -> Complex64 { - Complex64::from_polar(*r, *phi) + fn rect(r: ArgIntoFloat, phi: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::rect(r.into_float(), phi.into_float()) + .map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn isinf(z: ArgIntoComplex) -> bool { - let Complex64 { re, im } = *z; - re.is_infinite() || im.is_infinite() + pymath::cmath::isinf(z.into_complex()) } #[pyfunction] fn isfinite(z: ArgIntoComplex) -> bool { - z.is_finite() + pymath::cmath::isfinite(z.into_complex()) } #[pyfunction] fn isnan(z: ArgIntoComplex) -> bool { - z.is_nan() + pymath::cmath::isnan(z.into_complex()) } #[pyfunction] fn exp(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { - let z = *z; - result_or_overflow(z, z.exp(), vm) + pymath::cmath::exp(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn sqrt(z: ArgIntoComplex) -> Complex64 { - z.sqrt() + fn sqrt(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sqrt(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn sin(z: ArgIntoComplex) -> Complex64 { - z.sin() + fn sin(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sin(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn asin(z: ArgIntoComplex) -> Complex64 { - z.asin() + fn asin(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::asin(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn cos(z: ArgIntoComplex) -> Complex64 { - z.cos() + fn cos(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::cos(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn acos(z: ArgIntoComplex) -> Complex64 { - z.acos() + fn acos(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::acos(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn log(z: ArgIntoComplex, base: OptionalArg<ArgIntoComplex>) -> Complex64 { - // TODO: Complex64.log with a negative base yields wrong results. - // Issue is with num_complex::Complex64 implementation of log - // which returns NaN when base is negative. - // log10(z) / log10(base) yields correct results but division - // doesn't handle pos/neg zero nicely. (i.e log(1, 0.5)) - z.log( - base.into_option() - .map(|base| base.re) - .unwrap_or(std::f64::consts::E), + fn log( + z: ArgIntoComplex, + base: OptionalArg<ArgIntoComplex>, + vm: &VirtualMachine, + ) -> PyResult<Complex64> { + pymath::cmath::log( + z.into_complex(), + base.into_option().map(|b| b.into_complex()), ) + .map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn log10(z: ArgIntoComplex) -> Complex64 { - z.log(10.0) + fn log10(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::log10(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn acosh(z: ArgIntoComplex) -> Complex64 { - z.acosh() + fn acosh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::acosh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn atan(z: ArgIntoComplex) -> Complex64 { - z.atan() + fn atan(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::atan(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn atanh(z: ArgIntoComplex) -> Complex64 { - z.atanh() + fn atanh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::atanh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn tan(z: ArgIntoComplex) -> Complex64 { - z.tan() + fn tan(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::tan(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn tanh(z: ArgIntoComplex) -> Complex64 { - z.tanh() + fn tanh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::tanh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn sinh(z: ArgIntoComplex) -> Complex64 { - z.sinh() + fn sinh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sinh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn cosh(z: ArgIntoComplex) -> Complex64 { - z.cosh() + fn cosh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::cosh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn asinh(z: ArgIntoComplex) -> Complex64 { - z.asinh() + fn asinh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::asinh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) } #[derive(FromArgs)] @@ -156,54 +159,12 @@ mod cmath { #[pyfunction] fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { - let a = *args.a; - let b = *args.b; - let rel_tol = args.rel_tol.map_or(1e-09, Into::into); - let abs_tol = args.abs_tol.map_or(0.0, Into::into); - - if rel_tol < 0.0 || abs_tol < 0.0 { - return Err(vm.new_value_error("tolerances must be non-negative")); - } - - if a == b { - /* short circuit exact equality -- needed to catch two infinities of - the same sign. And perhaps speeds things up a bit sometimes. - */ - return Ok(true); - } - - /* This catches the case of two infinities of opposite sign, or - one infinity and one finite number. Two infinities of opposite - sign would otherwise have an infinite relative tolerance. - Two infinities of the same sign are caught by the equality check - above. - */ - if a.is_infinite() || b.is_infinite() { - return Ok(false); - } - - let diff = c_abs(b - a); - - Ok(diff <= (rel_tol * c_abs(b)) || (diff <= (rel_tol * c_abs(a))) || diff <= abs_tol) - } - - #[inline] - fn c_abs(Complex64 { re, im }: Complex64) -> f64 { - re.hypot(im) - } - - #[inline] - fn result_or_overflow( - value: Complex64, - result: Complex64, - vm: &VirtualMachine, - ) -> PyResult<Complex64> { - if !result.is_finite() && value.is_finite() { - // CPython doesn't return `inf` when called with finite - // values, it raises OverflowError instead. - Err(vm.new_overflow_error("math range error")) - } else { - Ok(result) - } + let a = args.a.into_complex(); + let b = args.b.into_complex(); + let rel_tol = args.rel_tol.into_option().map(|v| v.into_float()); + let abs_tol = args.abs_tol.into_option().map(|v| v.into_float()); + + pymath::cmath::isclose(a, b, rel_tol, abs_tol) + .map_err(|_| vm.new_value_error("tolerances must be non-negative")) } } diff --git a/crates/stdlib/src/compression.rs b/crates/stdlib/src/compression.rs index 7f4e3432eab..a857b4e53de 100644 --- a/crates/stdlib/src/compression.rs +++ b/crates/stdlib/src/compression.rs @@ -107,7 +107,7 @@ impl<'a> Chunker<'a> { pub fn advance(&mut self, consumed: usize) { self.data1 = &self.data1[consumed..]; if self.data1.is_empty() { - self.data1 = std::mem::take(&mut self.data2); + self.data1 = core::mem::take(&mut self.data2); } } } @@ -140,7 +140,7 @@ pub fn _decompress_chunks<D: Decompressor>( let chunk = data.chunk(); let flush = calc_flush(chunk.len() == data.len()); loop { - let additional = std::cmp::min(bufsize, max_length - buf.capacity()); + let additional = core::cmp::min(bufsize, max_length - buf.capacity()); if additional == 0 { return Ok((buf, false)); } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 56c2657f585..883ef6f6820 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -1,19 +1,8 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType}; +pub(crate) use _contextvars::module_def; + +use crate::vm::PyRef; use _contextvars::PyContext; -use std::cell::RefCell; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _contextvars::make_module(vm); - let token_type = module.get_attr("Token", vm).unwrap(); - token_type - .set_attr( - "MISSING", - _contextvars::ContextTokenMissing::static_type().to_owned(), - vm, - ) - .unwrap(); - module -} +use core::cell::RefCell; thread_local! { // TODO: Vec doesn't seem to match copy behavior @@ -24,20 +13,20 @@ thread_local! { mod _contextvars { use crate::vm::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, - builtins::{PyGenericAlias, PyStrRef, PyType, PyTypeRef}, + builtins::{PyGenericAlias, PyList, PyStrRef, PyType, PyTypeRef}, class::StaticType, common::hash::PyHash, function::{ArgCallable, FuncArgs, OptionalArg}, protocol::{PyMappingMethods, PySequenceMethods}, - types::{AsMapping, AsSequence, Constructor, Hashable, Representable}, + types::{AsMapping, AsSequence, Constructor, Hashable, Iterable, Representable}, }; - use crossbeam_utils::atomic::AtomicCell; - use indexmap::IndexMap; - use std::sync::LazyLock; - use std::{ + use core::{ cell::{Cell, RefCell, UnsafeCell}, sync::atomic::Ordering, }; + use crossbeam_utils::atomic::AtomicCell; + use indexmap::IndexMap; + use rustpython_common::lock::LazyLock; // TODO: Real hamt implementation type Hamt = IndexMap<PyRef<ContextVar>, PyObjectRef, ahash::RandomState>; @@ -90,11 +79,11 @@ mod _contextvars { } } - fn borrow_vars(&self) -> impl std::ops::Deref<Target = Hamt> + '_ { + fn borrow_vars(&self) -> impl core::ops::Deref<Target = Hamt> + '_ { self.inner.vars.hamt.borrow() } - fn borrow_vars_mut(&self) -> impl std::ops::DerefMut<Target = Hamt> + '_ { + fn borrow_vars_mut(&self) -> impl core::ops::DerefMut<Target = Hamt> + '_ { self.inner.vars.hamt.borrow_mut() } @@ -163,7 +152,7 @@ mod _contextvars { } } - #[pyclass(with(Constructor, AsMapping, AsSequence))] + #[pyclass(with(Constructor, AsMapping, AsSequence, Iterable))] impl PyContext { #[pymethod] fn run( @@ -179,17 +168,20 @@ mod _contextvars { } #[pymethod] - fn copy(&self) -> Self { + fn copy(&self, vm: &VirtualMachine) -> Self { + // Deep copy the vars - clone the underlying Hamt data, not just the PyRef + let vars_copy = HamtObject { + hamt: RefCell::new(self.inner.vars.hamt.borrow().clone()), + }; Self { inner: ContextInner { idx: Cell::new(usize::MAX), - vars: self.inner.vars.clone(), + vars: vars_copy.into_ref(&vm.ctx), entered: Cell::new(false), }, } } - #[pymethod] fn __getitem__( &self, var: PyRef<ContextVar>, @@ -202,16 +194,10 @@ mod _contextvars { Ok(item.to_owned()) } - #[pymethod] fn __len__(&self) -> usize { self.borrow_vars().len() } - #[pymethod] - fn __iter__(&self) -> PyResult { - unimplemented!("Context.__iter__ is currently under construction") - } - #[pymethod] fn get( &self, @@ -240,6 +226,15 @@ mod _contextvars { let vars = zelf.borrow_vars(); vars.values().map(|value| value.to_owned()).collect() } + + // TODO: wrong return type + #[pymethod] + fn items(zelf: PyRef<Self>, vm: &VirtualMachine) -> Vec<PyObjectRef> { + let vars = zelf.borrow_vars(); + vars.iter() + .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone().into(), v.clone()]).into()) + .collect() + } } impl Constructor for PyContext { @@ -264,7 +259,7 @@ mod _contextvars { Err(vm.new_key_error(needle.to_owned().into())) } }), - ass_subscript: AtomicCell::new(None), + ass_subscript: None, }; &AS_MAPPING } @@ -283,6 +278,15 @@ mod _contextvars { } } + impl Iterable for PyContext { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let vars = zelf.borrow_vars(); + let keys: Vec<PyObjectRef> = vars.keys().map(|k| k.clone().into()).collect(); + let list = vm.ctx.new_list(keys); + <PyList as Iterable>::iter(list, vm) + } + } + #[pyattr] #[pyclass(name, traverse)] #[derive(PyPayload)] @@ -293,13 +297,13 @@ mod _contextvars { #[pytraverse(skip)] cached: AtomicCell<Option<ContextVarCache>>, #[pytraverse(skip)] - cached_id: std::sync::atomic::AtomicUsize, // cached_tsid in CPython + cached_id: core::sync::atomic::AtomicUsize, // cached_tsid in CPython #[pytraverse(skip)] hash: UnsafeCell<PyHash>, } - impl std::fmt::Debug for ContextVar { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for ContextVar { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("ContextVar").finish() } } @@ -308,7 +312,7 @@ mod _contextvars { impl PartialEq for ContextVar { fn eq(&self, other: &Self) -> bool { - std::ptr::eq(self, other) + core::ptr::eq(self, other) } } impl Eq for ContextVar {} @@ -512,9 +516,9 @@ mod _contextvars { } } - impl std::hash::Hash for ContextVar { + impl core::hash::Hash for ContextVar { #[inline] - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { unsafe { *self.hash.get() }.hash(state) } } @@ -576,6 +580,22 @@ mod _contextvars { ) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn __exit__( + zelf: &Py<Self>, + _ty: PyObjectRef, + _val: PyObjectRef, + _tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + ContextVar::reset(&zelf.var, zelf.to_owned(), vm) + } } impl Constructor for ContextToken { @@ -614,6 +634,19 @@ mod _contextvars { #[pyfunction] fn copy_context(vm: &VirtualMachine) -> PyContext { - PyContext::current(vm).copy() + PyContext::current(vm).copy(vm) + } + + // Set Token.MISSING attribute + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::vm::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + let token_type = module.get_attr("Token", vm)?; + token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?; + + Ok(()) } } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 3c7cc2ff807..b898dc8c106 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -1,22 +1,24 @@ -pub(crate) use _csv::make_module; +pub(crate) use _csv::module_def; #[pymodule] mod _csv { use crate::common::lock::PyMutex; use crate::vm::{ - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, builtins::{PyBaseExceptionRef, PyInt, PyNone, PyStr, PyType, PyTypeRef}, function::{ArgIterable, ArgumentError, FromArgs, FuncArgs, OptionalArg}, protocol::{PyIter, PyIterReturn}, raise_if_stop, types::{Constructor, IterNext, Iterable, SelfIter}, }; + use alloc::fmt; use csv_core::Terminator; use itertools::{self, Itertools}; use parking_lot::Mutex; - use rustpython_vm::match_class; - use std::sync::LazyLock; - use std::{collections::HashMap, fmt}; + use rustpython_common::lock::LazyLock; + use rustpython_vm::{match_class, sliceable::SliceableSequenceOp}; + use std::collections::HashMap; #[pyattr] const QUOTE_MINIMAL: i32 = QuoteStyle::Minimal as i32; @@ -130,52 +132,61 @@ mod _csv { /// /// * If the 'delimiter' attribute is not a single-character string, a type error is returned. /// * If the 'obj' is not of string type and does not have a 'delimiter' attribute, a type error is returned. - fn parse_delimiter_from_obj(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<u8> { + fn parse_delimiter_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<u8> { if let Ok(attr) = obj.get_attr("delimiter", vm) { parse_delimiter_from_obj(vm, &attr) } else { - match_class!(match obj.clone() { + match_class!(match obj.to_owned() { s @ PyStr => { Ok(s.as_str().bytes().exactly_one().map_err(|_| { - let msg = r#""delimiter" must be a 1-character string"#; - vm.new_type_error(msg.to_owned()) + vm.new_type_error(format!( + r#""delimiter" must be a unicode character, not a string of length {}"#, + s.len() + )) })?) } attr => { - let msg = format!("\"delimiter\" must be string, not {}", attr.class()); + let msg = format!( + r#""delimiter" must be a unicode character, not {}"#, + attr.class() + ); Err(vm.new_type_error(msg)) } }) } } - fn parse_quotechar_from_obj(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Option<u8>> { + + fn parse_quotechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<u8>> { match_class!(match obj.get_attr("quotechar", vm)? { s @ PyStr => { Ok(Some(s.as_str().bytes().exactly_one().map_err(|_| { vm.new_exception_msg( super::_csv::error(vm), - r#""quotechar" must be a 1-character string"#.to_owned(), + format!(r#""quotechar" must be a unicode character or None, not a string of length {}"#, s.len()), ) })?)) } _n @ PyNone => { Ok(None) } - _ => { + attr => { Err(vm.new_exception_msg( super::_csv::error(vm), - r#""quotechar" must be string or None, not int"#.to_owned(), + format!( + r#""quotechar" must be a unicode character or None, not {}"#, + attr.class() + ), )) } }) } - fn parse_escapechar_from_obj(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Option<u8>> { + fn parse_escapechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<u8>> { match_class!(match obj.get_attr("escapechar", vm)? { s @ PyStr => { Ok(Some(s.as_str().bytes().exactly_one().map_err(|_| { vm.new_exception_msg( super::_csv::error(vm), - r#""escapechar" must be a 1-character string"#.to_owned(), + format!(r#""escapechar" must be a unicode character or None, not a string of length {}"#, s.len()), ) })?)) } @@ -184,17 +195,14 @@ mod _csv { } attr => { let msg = format!( - "\"escapechar\" must be string or None, not {}", + r#""escapechar" must be a unicode character or None, not {}"#, attr.class() ); Err(vm.new_type_error(msg.to_owned())) } }) } - fn prase_lineterminator_from_obj( - vm: &VirtualMachine, - obj: &PyObjectRef, - ) -> PyResult<Terminator> { + fn prase_lineterminator_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Terminator> { match_class!(match obj.get_attr("lineterminator", vm)? { s @ PyStr => { Ok(if s.as_bytes().eq(b"\r\n") { @@ -211,13 +219,15 @@ mod _csv { )); }) } - _ => { - let msg = "\"lineterminator\" must be a string".to_string(); - Err(vm.new_type_error(msg.to_owned())) + attr => { + Err(vm.new_type_error(format!( + r#""lineterminator" must be a string, not {}"#, + attr.class() + ))) } }) } - fn prase_quoting_from_obj(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<QuoteStyle> { + fn prase_quoting_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<QuoteStyle> { match_class!(match obj.get_attr("quoting", vm)? { i @ PyInt => { Ok(i.try_to_primitive::<isize>(vm)?.try_into().map_err(|_| { @@ -226,7 +236,7 @@ mod _csv { })?) } attr => { - let msg = format!("\"quoting\" must be string or None, not {}", attr.class()); + let msg = format!(r#""quoting" must be string or None, not {}"#, attr.class()); Err(vm.new_type_error(msg.to_owned())) } }) @@ -908,7 +918,7 @@ mod _csv { } } - #[pyclass(with(IterNext, Iterable))] + #[pyclass(with(IterNext, Iterable), flags(DISALLOW_INSTANTIATION))] impl Reader { #[pygetset] fn line_num(&self) -> u64 { @@ -1008,7 +1018,7 @@ mod _csv { return Err(new_csv_error(vm, "filed too long to read".to_string())); } prev_end = end; - let s = std::str::from_utf8(&buffer[range.clone()]) + let s = core::str::from_utf8(&buffer[range.clone()]) // not sure if this is possible - the input was all strings .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?; // Rustpython TODO! @@ -1059,7 +1069,7 @@ mod _csv { } } - #[pyclass] + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl Writer { #[pygetset(name = "dialect")] const fn get_dialect(&self, _vm: &VirtualMachine) -> PyDialect { @@ -1118,7 +1128,7 @@ mod _csv { loop { handle_res!(writer.terminator(&mut buffer[buffer_offset..])); } - let s = std::str::from_utf8(&buffer[..buffer_offset]) + let s = core::str::from_utf8(&buffer[..buffer_offset]) .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; self.write.call((s,), vm) diff --git a/crates/stdlib/src/dis.rs b/crates/stdlib/src/dis.rs deleted file mode 100644 index 341137f91f4..00000000000 --- a/crates/stdlib/src/dis.rs +++ /dev/null @@ -1,58 +0,0 @@ -pub(crate) use decl::make_module; - -#[pymodule(name = "dis")] -mod decl { - use crate::vm::{ - PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyCode, PyDictRef, PyStrRef}, - bytecode::CodeFlags, - }; - - #[pyfunction] - fn dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let co = if let Ok(co) = obj.get_attr("__code__", vm) { - // Method or function: - PyRef::try_from_object(vm, co)? - } else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) { - #[cfg(not(feature = "compiler"))] - { - let _ = co_str; - return Err( - vm.new_runtime_error("dis.dis() with str argument requires `compiler` feature") - ); - } - #[cfg(feature = "compiler")] - { - vm.compile( - co_str.as_str(), - crate::vm::compiler::Mode::Exec, - "<dis>".to_owned(), - ) - .map_err(|err| vm.new_syntax_error(&err, Some(co_str.as_str())))? - } - } else { - PyRef::try_from_object(vm, obj)? - }; - disassemble(co) - } - - #[pyfunction] - fn disassemble(co: PyRef<PyCode>) -> PyResult<()> { - print!("{}", &co.code); - Ok(()) - } - - #[pyattr(name = "COMPILER_FLAG_NAMES")] - fn compiler_flag_names(vm: &VirtualMachine) -> PyDictRef { - let dict = vm.ctx.new_dict(); - for (name, flag) in CodeFlags::NAME_MAPPING { - dict.set_item( - &*vm.new_pyobj(flag.bits()), - vm.ctx.new_str(*name).into(), - vm, - ) - .unwrap(); - } - dict - } -} diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index f45c9909c6f..f618f8f6731 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -1,19 +1,20 @@ -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[allow(static_mut_refs)] // TODO: group code only with static mut refs #[pymodule(name = "faulthandler")] mod decl { use crate::vm::{ - PyObjectRef, PyResult, VirtualMachine, builtins::PyFloat, frame::Frame, - function::OptionalArg, py_io::Write, + PyObjectRef, PyResult, VirtualMachine, + frame::Frame, + function::{ArgIntoFloat, OptionalArg}, }; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicI32, Ordering}; + use core::time::Duration; use parking_lot::{Condvar, Mutex}; #[cfg(any(unix, windows))] use rustpython_common::os::{get_errno, set_errno}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; use std::thread; - use std::time::Duration; /// fault_handler_t #[cfg(unix)] @@ -40,7 +41,7 @@ mod decl { enabled: false, name, // SAFETY: sigaction is a C struct that can be zero-initialized - previous: unsafe { std::mem::zeroed() }, + previous: unsafe { core::mem::zeroed() }, } } } @@ -64,11 +65,7 @@ mod decl { #[cfg(windows)] const FAULTHANDLER_NSIGNALS: usize = 4; - // CPython uses static arrays for signal handlers which requires mutable static access. - // This is safe because: - // 1. Signal handlers run in a single-threaded context (from the OS perspective) - // 2. FAULTHANDLER_HANDLERS is only modified during enable/disable operations - // 3. This matches CPython's faulthandler.c implementation + // Signal handlers use mutable statics matching faulthandler.c implementation. #[cfg(unix)] static mut FAULTHANDLER_HANDLERS: [FaultHandler; FAULTHANDLER_NSIGNALS] = [ FaultHandler::new(libc::SIGBUS, "Bus error"), @@ -99,6 +96,9 @@ mod decl { all_threads: AtomicBool::new(true), }; + #[cfg(feature = "threading")] + type ThreadFrameSlot = Arc<rustpython_vm::vm::thread::ThreadSlot>; + // Watchdog thread state for dump_traceback_later struct WatchdogState { cancel: bool, @@ -107,50 +107,32 @@ mod decl { repeat: bool, exit: bool, header: String, + #[cfg(feature = "threading")] + thread_frame_slots: Vec<(u64, ThreadFrameSlot)>, } type WatchdogHandle = Arc<(Mutex<WatchdogState>, Condvar)>; static WATCHDOG: Mutex<Option<WatchdogHandle>> = Mutex::new(None); - // Frame snapshot for signal-safe traceback (RustPython-specific) - - /// Frame information snapshot for signal-safe access - #[cfg(any(unix, windows))] - #[derive(Clone, Copy)] - struct FrameSnapshot { - filename: [u8; 256], - filename_len: usize, - lineno: u32, - funcname: [u8; 128], - funcname_len: usize, - } + // Signal-safe output functions + // PUTS macro #[cfg(any(unix, windows))] - impl FrameSnapshot { - const EMPTY: Self = Self { - filename: [0; 256], - filename_len: 0, - lineno: 0, - funcname: [0; 128], - funcname_len: 0, + fn puts(fd: i32, s: &str) { + let _ = unsafe { + #[cfg(windows)] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len() as u32) + } + #[cfg(not(windows))] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len()) + } }; } #[cfg(any(unix, windows))] - const MAX_SNAPSHOT_FRAMES: usize = 100; - - /// Signal-safe global storage for frame snapshots - #[cfg(any(unix, windows))] - static mut FRAME_SNAPSHOTS: [FrameSnapshot; MAX_SNAPSHOT_FRAMES] = - [FrameSnapshot::EMPTY; MAX_SNAPSHOT_FRAMES]; - #[cfg(any(unix, windows))] - static SNAPSHOT_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - - // Signal-safe output functions - - // PUTS macro - #[cfg(any(unix, windows))] - fn puts(fd: i32, s: &str) { + fn puts_bytes(fd: i32, s: &[u8]) { let _ = unsafe { #[cfg(windows)] { @@ -232,59 +214,69 @@ mod decl { // write_thread_id (traceback.c:1240-1256) #[cfg(any(unix, windows))] - fn write_thread_id(fd: i32, is_current: bool) { + fn write_thread_id(fd: i32, thread_id: u64, is_current: bool) { if is_current { - puts(fd, "Current thread 0x"); + puts(fd, "Current thread "); } else { - puts(fd, "Thread 0x"); + puts(fd, "Thread "); } - let thread_id = current_thread_id(); - // Use appropriate width based on platform pointer size - dump_hexadecimal(fd, thread_id, std::mem::size_of::<usize>() * 2); + dump_hexadecimal(fd, thread_id, core::mem::size_of::<usize>() * 2); puts(fd, " (most recent call first):\n"); } - // dump_frame (traceback.c:1037-1087) + /// Dump the current thread's live frame chain to fd (signal-safe). + /// Walks the `Frame.previous` pointer chain starting from the + /// thread-local current frame pointer. #[cfg(any(unix, windows))] - fn dump_frame(fd: i32, filename: &[u8], lineno: u32, funcname: &[u8]) { - puts(fd, " File \""); - let _ = unsafe { - #[cfg(windows)] - { - libc::write( - fd, - filename.as_ptr() as *const libc::c_void, - filename.len() as u32, - ) - } - #[cfg(not(windows))] - { - libc::write(fd, filename.as_ptr() as *const libc::c_void, filename.len()) + fn dump_live_frames(fd: i32) { + const MAX_FRAME_DEPTH: usize = 100; + + let mut frame_ptr = crate::vm::vm::thread::get_current_frame(); + if frame_ptr.is_null() { + puts(fd, " <no Python frame>\n"); + return; + } + let mut depth = 0; + while !frame_ptr.is_null() && depth < MAX_FRAME_DEPTH { + let frame = unsafe { &*frame_ptr }; + dump_frame_from_raw(fd, frame); + frame_ptr = frame.previous_frame(); + depth += 1; + } + if depth >= MAX_FRAME_DEPTH && !frame_ptr.is_null() { + puts(fd, " ...\n"); + } + } + + /// Dump a single frame's info to fd (signal-safe), reading live data. + #[cfg(any(unix, windows))] + fn dump_frame_from_raw(fd: i32, frame: &Frame) { + let filename = frame.code.source_path().as_str(); + let funcname = frame.code.obj_name.as_str(); + let lasti = frame.lasti(); + let lineno = if lasti == 0 { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(1) as u32 + } else { + let idx = (lasti as usize).saturating_sub(1); + if idx < frame.code.locations.len() { + frame.code.locations[idx].0.line.get() as u32 + } else { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(0) as u32 } }; + + puts(fd, " File \""); + dump_ascii(fd, filename); puts(fd, "\", line "); dump_decimal(fd, lineno as usize); puts(fd, " in "); - let _ = unsafe { - #[cfg(windows)] - { - libc::write( - fd, - funcname.as_ptr() as *const libc::c_void, - funcname.len() as u32, - ) - } - #[cfg(not(windows))] - { - libc::write(fd, funcname.as_ptr() as *const libc::c_void, funcname.len()) - } - }; + dump_ascii(fd, funcname); puts(fd, "\n"); } - // faulthandler_dump_traceback + // faulthandler_dump_traceback (signal-safe, for fatal errors) #[cfg(any(unix, windows))] - fn faulthandler_dump_traceback(fd: i32, _all_threads: bool) { + fn faulthandler_dump_traceback(fd: i32, all_threads: bool) { static REENTRANT: AtomicBool = AtomicBool::new(false); if REENTRANT.swap(true, Ordering::SeqCst) { @@ -292,76 +284,85 @@ mod decl { } // Write thread header - write_thread_id(fd, true); - - // Try to dump traceback from snapshot - let count = SNAPSHOT_COUNT.load(Ordering::Acquire); - if count > 0 { - // Using index access instead of iterator because FRAME_SNAPSHOTS is static mut - #[allow(clippy::needless_range_loop)] - for i in 0..count { - unsafe { - let snap = &FRAME_SNAPSHOTS[i]; - if snap.filename_len > 0 { - dump_frame( - fd, - &snap.filename[..snap.filename_len], - snap.lineno, - &snap.funcname[..snap.funcname_len], - ); - } - } - } + if all_threads { + write_thread_id(fd, current_thread_id(), true); } else { - puts(fd, " <no Python frame>\n"); + puts(fd, "Stack (most recent call first):\n"); } + dump_live_frames(fd); + REENTRANT.store(false, Ordering::SeqCst); } - const MAX_FUNCTION_NAME_LEN: usize = 500; + /// MAX_STRING_LENGTH in traceback.c + const MAX_STRING_LENGTH: usize = 500; - fn truncate_name(name: &str) -> String { - if name.len() > MAX_FUNCTION_NAME_LEN { - format!("{}...", &name[..MAX_FUNCTION_NAME_LEN]) - } else { - name.to_string() + /// Truncate a UTF-8 string to at most `max_bytes` without splitting a + /// multi-byte codepoint. Signal-safe (no allocation, no panic). + #[cfg(any(unix, windows))] + fn safe_truncate(s: &str, max_bytes: usize) -> (&str, bool) { + if s.len() <= max_bytes { + return (s, false); + } + let mut end = max_bytes; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; } + (&s[..end], true) } - fn get_file_for_output( - file: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - match file { - OptionalArg::Present(f) => { - // If it's an integer, we can't use it directly as a file object - // For now, just return it and let the caller handle it - Ok(f) - } - OptionalArg::Missing => { - // Get sys.stderr - let stderr = vm.sys_module.get_attr("stderr", vm)?; - if vm.is_none(&stderr) { - return Err(vm.new_runtime_error("sys.stderr is None".to_owned())); - } - Ok(stderr) - } + /// Write a string to fd, truncating with "..." if it exceeds MAX_STRING_LENGTH. + /// Mirrors `_Py_DumpASCII` truncation behavior. + #[cfg(any(unix, windows))] + fn dump_ascii(fd: i32, s: &str) { + let (truncated_s, was_truncated) = safe_truncate(s, MAX_STRING_LENGTH); + puts(fd, truncated_s); + if was_truncated { + puts(fd, "..."); } } - fn collect_frame_info(frame: &crate::vm::PyRef<Frame>) -> String { - let func_name = truncate_name(frame.code.obj_name.as_str()); - // If lasti is 0, execution hasn't started yet - use first line number or 1 - let line = if frame.lasti() == 0 { - frame.code.first_line_number.map(|n| n.get()).unwrap_or(1) + /// Write a frame's info to an fd using signal-safe I/O. + #[cfg(any(unix, windows))] + fn dump_frame_from_ref(fd: i32, frame: &crate::vm::Py<Frame>) { + let funcname = frame.code.obj_name.as_str(); + let filename = frame.code.source_path().as_str(); + let lineno = if frame.lasti() == 0 { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(1) as u32 } else { - frame.current_location().line.get() + frame.current_location().line.get() as u32 }; - format!( - " File \"{}\", line {} in {}", - frame.code.source_path, line, func_name - ) + + puts(fd, " File \""); + dump_ascii(fd, filename); + puts(fd, "\", line "); + dump_decimal(fd, lineno as usize); + puts(fd, " in "); + dump_ascii(fd, funcname); + puts(fd, "\n"); + } + + /// Dump traceback for a thread given its frame stack (for cross-thread dumping). + /// # Safety + /// Each `FramePtr` must point to a live frame (caller holds the Mutex). + #[cfg(all(any(unix, windows), feature = "threading"))] + fn dump_traceback_thread_frames( + fd: i32, + thread_id: u64, + is_current: bool, + frames: &[rustpython_vm::vm::FramePtr], + ) { + write_thread_id(fd, thread_id, is_current); + + if frames.is_empty() { + puts(fd, " <no Python frame>\n"); + } else { + for fp in frames.iter().rev() { + // SAFETY: caller holds the Mutex, so the owning thread can't pop. + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } } #[derive(FromArgs)] @@ -374,22 +375,71 @@ mod decl { #[pyfunction] fn dump_traceback(args: DumpTracebackArgs, vm: &VirtualMachine) -> PyResult<()> { - let _ = args.all_threads; // TODO: implement all_threads support - - let file = get_file_for_output(args.file, vm)?; + let fd = get_fd_from_file_opt(args.file, vm)?; - // Collect frame info first to avoid RefCell borrow conflict - let frame_lines: Vec<String> = vm.frames.borrow().iter().map(collect_frame_info).collect(); + #[cfg(any(unix, windows))] + { + if args.all_threads { + dump_all_threads(fd, vm); + } else { + puts(fd, "Stack (most recent call first):\n"); + let frames = vm.frames.borrow(); + for fp in frames.iter().rev() { + // SAFETY: the frame is alive while it's in the Vec + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } - // Now write to file (in reverse order - most recent call first) - let mut writer = crate::vm::py_io::PyWriter(file, vm); - writeln!(writer, "Stack (most recent call first):")?; - for line in frame_lines.iter().rev() { - writeln!(writer, "{}", line)?; + #[cfg(not(any(unix, windows)))] + { + let _ = (fd, args.all_threads); } + Ok(()) } + /// Dump tracebacks of all threads. + #[cfg(any(unix, windows))] + fn dump_all_threads(fd: i32, vm: &VirtualMachine) { + // Get all threads' frame stacks from the shared registry + #[cfg(feature = "threading")] + { + let current_tid = rustpython_vm::stdlib::thread::get_ident(); + let registry = vm.state.thread_frames.lock(); + + // First dump non-current threads, then current thread last + for (&tid, slot) in registry.iter() { + if tid == current_tid { + continue; + } + let frames_guard = slot.frames.lock(); + dump_traceback_thread_frames(fd, tid, false, &frames_guard); + puts(fd, "\n"); + } + + // Now dump current thread (use vm.frames for most up-to-date data) + write_thread_id(fd, current_tid, true); + let frames = vm.frames.borrow(); + if frames.is_empty() { + puts(fd, " <no Python frame>\n"); + } else { + for fp in frames.iter().rev() { + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + + #[cfg(not(feature = "threading"))] + { + write_thread_id(fd, current_thread_id(), true); + let frames = vm.frames.borrow(); + for fp in frames.iter().rev() { + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + #[derive(FromArgs)] #[allow(unused)] struct EnableArgs { @@ -429,7 +479,7 @@ mod decl { } handler.enabled = false; unsafe { - libc::sigaction(handler.signum, &handler.previous, std::ptr::null_mut()); + libc::sigaction(handler.signum, &handler.previous, core::ptr::null_mut()); } } @@ -461,9 +511,8 @@ mod decl { .find(|h| h.signum == signum) }; - // faulthandler_fatal_error if let Some(h) = handler { - // Disable handler first (restores previous) + // Disable handler (restores previous) unsafe { faulthandler_disable_fatal_handler(h); } @@ -477,18 +526,24 @@ mod decl { puts(fd, "\n\n"); } - // faulthandler_dump_traceback let all_threads = FATAL_ERROR.all_threads.load(Ordering::Relaxed); faulthandler_dump_traceback(fd, all_threads); - // restore errno set_errno(save_errno); - // raise - // Called immediately thanks to SA_NODEFER flag + // Reset to default handler and re-raise to ensure process terminates. + // We cannot just restore the previous handler because Rust's runtime + // may have installed its own SIGSEGV handler (for stack overflow detection) + // that doesn't terminate the process on software-raised signals. unsafe { + libc::signal(signum, libc::SIG_DFL); libc::raise(signum); } + + // Fallback if raise() somehow didn't terminate the process + unsafe { + libc::_exit(1); + } } // faulthandler_fatal_error for Windows @@ -526,14 +581,84 @@ mod decl { set_errno(save_errno); - // On Windows, don't explicitly call the previous handler for SIGSEGV - if signum == libc::SIGSEGV { - return; - } - unsafe { + libc::signal(signum, libc::SIG_DFL); libc::raise(signum); } + + // Fallback + std::process::exit(1); + } + + // Windows vectored exception handler (faulthandler.c:417-480) + #[cfg(windows)] + static EXC_HANDLER: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + + #[cfg(windows)] + fn faulthandler_ignore_exception(code: u32) -> bool { + // bpo-30557: ignore exceptions which are not errors + if (code & 0x80000000) == 0 { + return true; + } + // bpo-31701: ignore MSC and COM exceptions + if code == 0xE06D7363 || code == 0xE0434352 { + return true; + } + false + } + + #[cfg(windows)] + unsafe extern "system" fn faulthandler_exc_handler( + exc_info: *mut windows_sys::Win32::System::Diagnostics::Debug::EXCEPTION_POINTERS, + ) -> i32 { + const EXCEPTION_CONTINUE_SEARCH: i32 = 0; + + if !FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return EXCEPTION_CONTINUE_SEARCH; + } + + let record = unsafe { &*(*exc_info).ExceptionRecord }; + let code = record.ExceptionCode as u32; + + if faulthandler_ignore_exception(code) { + return EXCEPTION_CONTINUE_SEARCH; + } + + let fd = FATAL_ERROR.fd.load(Ordering::Relaxed); + + puts(fd, "Windows fatal exception: "); + match code { + 0xC0000005 => puts(fd, "access violation"), + 0xC000008C => puts(fd, "float divide by zero"), + 0xC0000091 => puts(fd, "float overflow"), + 0xC0000094 => puts(fd, "int divide by zero"), + 0xC0000095 => puts(fd, "integer overflow"), + 0xC0000006 => puts(fd, "page error"), + 0xC00000FD => puts(fd, "stack overflow"), + 0xC000001D => puts(fd, "illegal instruction"), + _ => { + puts(fd, "code "); + dump_hexadecimal(fd, code as u64, 8); + } + } + puts(fd, "\n\n"); + + // Disable SIGSEGV handler for access violations to avoid double output + if code == 0xC0000005 { + unsafe { + for handler in FAULTHANDLER_HANDLERS.iter_mut() { + if handler.signum == libc::SIGSEGV { + faulthandler_disable_fatal_handler(handler); + break; + } + } + } + } + + let all_threads = FATAL_ERROR.all_threads.load(Ordering::Relaxed); + faulthandler_dump_traceback(fd, all_threads); + + EXCEPTION_CONTINUE_SEARCH } // faulthandler_enable @@ -549,8 +674,8 @@ mod decl { continue; } - let mut action: libc::sigaction = std::mem::zeroed(); - action.sa_sigaction = faulthandler_fatal_error as libc::sighandler_t; + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_fatal_error as *const () as libc::sighandler_t; // SA_NODEFER flag action.sa_flags = libc::SA_NODEFER; @@ -580,7 +705,7 @@ mod decl { handler.previous = libc::signal( handler.signum, - faulthandler_fatal_error as libc::sighandler_t, + faulthandler_fatal_error as *const () as libc::sighandler_t, ); // SIG_ERR is -1 as sighandler_t (which is usize on Windows) @@ -592,6 +717,14 @@ mod decl { } } + // Register Windows vectored exception handler + #[cfg(windows)] + { + use windows_sys::Win32::System::Diagnostics::Debug::AddVectoredExceptionHandler; + let h = unsafe { AddVectoredExceptionHandler(1, Some(faulthandler_exc_handler)) }; + EXC_HANDLER.store(h as usize, Ordering::Relaxed); + } + FATAL_ERROR.enabled.store(true, Ordering::Relaxed); true } @@ -608,6 +741,18 @@ mod decl { faulthandler_disable_fatal_handler(handler); } } + + // Remove Windows vectored exception handler + #[cfg(windows)] + { + use windows_sys::Win32::System::Diagnostics::Debug::RemoveVectoredExceptionHandler; + let h = EXC_HANDLER.swap(0, Ordering::Relaxed); + if h != 0 { + unsafe { + RemoveVectoredExceptionHandler(h as *mut core::ffi::c_void); + } + } + } } #[cfg(not(any(unix, windows)))] @@ -643,16 +788,17 @@ mod decl { let hour = min / 60; let min = min % 60; + // Match Python's timedelta str format: H:MM:SS.ffffff (no leading zero for hours) if us != 0 { - format!("Timeout ({:02}:{:02}:{:02}.{:06})!\n", hour, min, sec, us) + format!("Timeout ({}:{:02}:{:02}.{:06})!\n", hour, min, sec, us) } else { - format!("Timeout ({:02}:{:02}:{:02})!\n", hour, min, sec) + format!("Timeout ({}:{:02}:{:02})!\n", hour, min, sec) } } fn get_fd_from_file_opt(file: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<i32> { match file { - OptionalArg::Present(f) => { + OptionalArg::Present(f) if !vm.is_none(&f) => { // Check if it's an integer (file descriptor) if let Ok(fd) = f.try_to_value::<i32>(vm) { if fd < 0 { @@ -674,8 +820,8 @@ mod decl { let _ = vm.call_method(&f, "flush", ()); Ok(fd) } - OptionalArg::Missing => { - // Get sys.stderr + _ => { + // file=None or file not passed: fall back to sys.stderr let stderr = vm.sys_module.get_attr("stderr", vm)?; if vm.is_none(&stderr) { return Err(vm.new_runtime_error("sys.stderr is None".to_owned())); @@ -706,8 +852,12 @@ mod decl { } // Extract values before releasing lock for I/O - let (repeat, exit, fd, header) = - (guard.repeat, guard.exit, guard.fd, guard.header.clone()); + let repeat = guard.repeat; + let exit = guard.exit; + let fd = guard.fd; + let header = guard.header.clone(); + #[cfg(feature = "threading")] + let thread_frame_slots = guard.thread_frame_slots.clone(); drop(guard); // Release lock before I/O // Timeout occurred, dump traceback @@ -716,35 +866,21 @@ mod decl { #[cfg(not(target_arch = "wasm32"))] { - let header_bytes = header.as_bytes(); - #[cfg(windows)] - unsafe { - libc::write( - fd, - header_bytes.as_ptr() as *const libc::c_void, - header_bytes.len() as u32, - ); - } - #[cfg(not(windows))] - unsafe { - libc::write( - fd, - header_bytes.as_ptr() as *const libc::c_void, - header_bytes.len(), - ); - } - - // Note: We cannot dump actual Python traceback from a separate thread - // because we don't have access to the VM's frame stack. - // Just output a message indicating timeout occurred. - let msg = b"<timeout: cannot dump traceback from watchdog thread>\n"; - #[cfg(windows)] - unsafe { - libc::write(fd, msg.as_ptr() as *const libc::c_void, msg.len() as u32); + puts_bytes(fd, header.as_bytes()); + + // Use thread frame slots when threading is enabled (includes all threads). + // Fall back to live frame walking for non-threaded builds. + #[cfg(feature = "threading")] + { + for (tid, slot) in &thread_frame_slots { + let frames = slot.frames.lock(); + dump_traceback_thread_frames(fd, *tid, false, &frames); + } } - #[cfg(not(windows))] - unsafe { - libc::write(fd, msg.as_ptr() as *const libc::c_void, msg.len()); + #[cfg(not(feature = "threading"))] + { + write_thread_id(fd, current_thread_id(), false); + dump_live_frames(fd); } if exit { @@ -761,8 +897,8 @@ mod decl { #[derive(FromArgs)] #[allow(unused)] struct DumpTracebackLaterArgs { - #[pyarg(positional)] - timeout: PyObjectRef, + #[pyarg(positional, error_msg = "timeout must be a number (int or float)")] + timeout: ArgIntoFloat, #[pyarg(any, default = false)] repeat: bool, #[pyarg(any, default)] @@ -773,18 +909,7 @@ mod decl { #[pyfunction] fn dump_traceback_later(args: DumpTracebackLaterArgs, vm: &VirtualMachine) -> PyResult<()> { - use num_traits::ToPrimitive; - // Convert timeout to f64 (accepting int or float) - let timeout: f64 = if let Some(float) = args.timeout.downcast_ref::<PyFloat>() { - float.to_f64() - } else if let Some(int) = args.timeout.try_index_opt(vm).transpose()? { - int.as_bigint() - .to_i64() - .ok_or_else(|| vm.new_overflow_error("timeout value is too large".to_owned()))? - as f64 - } else { - return Err(vm.new_type_error("timeout must be a number (int or float)".to_owned())); - }; + let timeout: f64 = args.timeout.into_float(); if timeout <= 0.0 { return Err(vm.new_value_error("timeout must be greater than 0".to_owned())); @@ -800,6 +925,16 @@ mod decl { let header = format_timeout(timeout_us); + // Snapshot thread frame slots so watchdog can dump tracebacks + #[cfg(feature = "threading")] + let thread_frame_slots: Vec<(u64, ThreadFrameSlot)> = { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .map(|(&id, slot)| (id, Arc::clone(slot))) + .collect() + }; + // Cancel any previous watchdog cancel_dump_traceback_later(); @@ -812,6 +947,8 @@ mod decl { repeat: args.repeat, exit: args.exit, header, + #[cfg(feature = "threading")] + thread_frame_slots, }), Condvar::new(), )); @@ -853,14 +990,13 @@ mod decl { const NSIG: usize = 64; - #[derive(Clone)] + #[derive(Clone, Copy)] pub struct UserSignal { pub enabled: bool, pub fd: i32, - #[allow(dead_code)] pub all_threads: bool, pub chain: bool, - pub previous: libc::sighandler_t, + pub previous: libc::sigaction, } impl Default for UserSignal { @@ -870,7 +1006,8 @@ mod decl { fd: 2, // stderr all_threads: true, chain: false, - previous: libc::SIG_DFL, + // SAFETY: sigaction is a C struct that can be zero-initialized + previous: unsafe { core::mem::zeroed() }, } } } @@ -900,7 +1037,7 @@ mod decl { && signum < v.len() && v[signum].enabled { - let old = v[signum].clone(); + let old = v[signum]; v[signum] = UserSignal::default(); return Some(old); } @@ -918,35 +1055,33 @@ mod decl { #[cfg(unix)] extern "C" fn faulthandler_user_signal(signum: libc::c_int) { + let save_errno = get_errno(); + let user = match user_signals::get_user_signal(signum as usize) { Some(u) if u.enabled => u, _ => return, }; - // Write traceback header - let header = b"Current thread 0x0000 (most recent call first):\n"; - let _ = unsafe { - libc::write( - user.fd, - header.as_ptr() as *const libc::c_void, - header.len(), - ) - }; - - // Note: We cannot easily access RustPython's frame stack from a signal handler - // because signal handlers run asynchronously. We just output a placeholder. - let msg = b" <signal handler invoked, traceback unavailable in signal context>\n"; - let _ = unsafe { libc::write(user.fd, msg.as_ptr() as *const libc::c_void, msg.len()) }; + faulthandler_dump_traceback(user.fd, user.all_threads); - // If chain is enabled, call the previous handler - if user.chain && user.previous != libc::SIG_DFL && user.previous != libc::SIG_IGN { - // Re-register the old handler and raise the signal + if user.chain { + // Restore the previous handler and re-raise + unsafe { + libc::sigaction(signum, &user.previous, core::ptr::null_mut()); + } + set_errno(save_errno); unsafe { - libc::signal(signum, user.previous); libc::raise(signum); - // Re-register our handler - libc::signal(signum, faulthandler_user_signal as libc::sighandler_t); } + // Re-install our handler with the same flags as register() + let save_errno2 = get_errno(); + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_user_signal as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_NODEFER; + libc::sigaction(signum, &action, core::ptr::null_mut()); + } + set_errno(save_errno2); } } @@ -994,22 +1129,31 @@ mod decl { // Get current handler to save as previous let previous = if !user_signals::is_enabled(signum) { - // Install signal handler - let prev = unsafe { - libc::signal(args.signum, faulthandler_user_signal as libc::sighandler_t) - }; - if prev == libc::SIG_ERR { - return Err(vm.new_os_error(format!( - "Failed to register signal handler for signal {}", - args.signum - ))); + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_user_signal as *const () as libc::sighandler_t; + // SA_RESTART by default; SA_NODEFER only when chaining + // (faulthandler.c:860-864) + action.sa_flags = if args.chain { + libc::SA_NODEFER + } else { + libc::SA_RESTART + }; + + let mut prev: libc::sigaction = core::mem::zeroed(); + if libc::sigaction(args.signum, &action, &mut prev) != 0 { + return Err(vm.new_os_error(format!( + "Failed to register signal handler for signal {}", + args.signum + ))); + } + prev } - prev } else { // Already registered, keep previous handler user_signals::get_user_signal(signum) .map(|u| u.previous) - .unwrap_or(libc::SIG_DFL) + .unwrap_or(unsafe { core::mem::zeroed() }) }; user_signals::set_user_signal( @@ -1034,7 +1178,7 @@ mod decl { if let Some(old) = user_signals::clear_user_signal(signum as usize) { // Restore previous handler unsafe { - libc::signal(signum, old.previous); + libc::sigaction(signum, &old.previous, core::ptr::null_mut()); } Ok(true) } else { @@ -1045,14 +1189,15 @@ mod decl { // Test functions for faulthandler testing #[pyfunction] - fn _read_null() { - // This function intentionally causes a segmentation fault by reading from NULL - // Used for testing faulthandler + fn _read_null(_vm: &VirtualMachine) { #[cfg(not(target_arch = "wasm32"))] - unsafe { + { suppress_crash_report(); - let ptr: *const i32 = std::ptr::null(); - std::ptr::read_volatile(ptr); + + unsafe { + let ptr: *const i32 = core::ptr::null(); + core::ptr::read_volatile(ptr); + } } } @@ -1064,39 +1209,28 @@ mod decl { } #[pyfunction] - fn _sigsegv(_args: SigsegvArgs) { - // Raise SIGSEGV signal + fn _sigsegv(_args: SigsegvArgs, _vm: &VirtualMachine) { #[cfg(not(target_arch = "wasm32"))] { suppress_crash_report(); - // Reset SIGSEGV to default behavior before raising - // This ensures the process will actually crash + // Write to NULL pointer to trigger a real hardware SIGSEGV, + // matching CPython's *((volatile int *)NULL) = 0; + // Using raise(SIGSEGV) doesn't work reliably because Rust's runtime + // installs its own signal handler that may swallow software signals. unsafe { - libc::signal(libc::SIGSEGV, libc::SIG_DFL); - } - - #[cfg(windows)] - { - // On Windows, we need to raise SIGSEGV multiple times - loop { - unsafe { - libc::raise(libc::SIGSEGV); - } - } - } - #[cfg(not(windows))] - unsafe { - libc::raise(libc::SIGSEGV); + let ptr: *mut i32 = core::ptr::null_mut(); + core::ptr::write_volatile(ptr, 0); } } } #[pyfunction] - fn _sigabrt() { + fn _sigabrt(_vm: &VirtualMachine) { #[cfg(not(target_arch = "wasm32"))] { suppress_crash_report(); + unsafe { libc::abort(); } @@ -1104,17 +1238,11 @@ mod decl { } #[pyfunction] - fn _sigfpe() { + fn _sigfpe(_vm: &VirtualMachine) { #[cfg(not(target_arch = "wasm32"))] { suppress_crash_report(); - // Reset SIGFPE to default behavior before raising - unsafe { - libc::signal(libc::SIGFPE, libc::SIG_DFL); - } - - // Raise SIGFPE unsafe { libc::raise(libc::SIGFPE); } @@ -1132,7 +1260,7 @@ mod decl { panic!("Fatal Python error: in new thread"); }); // Wait a bit for the thread to panic - std::thread::sleep(std::time::Duration::from_secs(1)); + std::thread::sleep(core::time::Duration::from_secs(1)); } } @@ -1198,12 +1326,12 @@ mod decl { #[cfg(windows)] #[pyfunction] - fn _raise_exception(args: RaiseExceptionArgs) { + fn _raise_exception(args: RaiseExceptionArgs, _vm: &VirtualMachine) { use windows_sys::Win32::System::Diagnostics::Debug::RaiseException; suppress_crash_report(); unsafe { - RaiseException(args.code, args.flags, 0, std::ptr::null()); + RaiseException(args.code, args.flags, 0, core::ptr::null()); } } } diff --git a/crates/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs index dc6a0b8171e..407a2dfd6b3 100644 --- a/crates/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use fcntl::make_module; +pub(crate) use fcntl::module_def; #[pymodule] mod fcntl { @@ -92,11 +92,15 @@ mod fcntl { #[pyfunction] fn ioctl( io::Fildes(fd): io::Fildes, - request: u32, + request: i64, arg: OptionalArg<Either<Either<ArgMemoryBuffer, ArgStrOrBytesLike>, i32>>, mutate_flag: OptionalArg<bool>, vm: &VirtualMachine, ) -> PyResult { + // Convert to unsigned - handles both positive u32 values and negative i32 values + // that represent the same bit pattern (e.g., TIOCSWINSZ on some platforms). + // First truncate to u32 (takes lower 32 bits), then zero-extend to c_ulong. + let request = (request as u32) as libc::c_ulong; let arg = arg.unwrap_or_else(|| Either::B(0)); match arg { Either::A(buf_kind) => { @@ -173,7 +177,7 @@ mod fcntl { }; } - let mut l: libc::flock = unsafe { std::mem::zeroed() }; + let mut l: libc::flock = unsafe { core::mem::zeroed() }; l.l_type = if cmd == libc::LOCK_UN { try_into_l_type!(libc::F_UNLCK) } else if (cmd & libc::LOCK_SH) != 0 { diff --git a/crates/stdlib/src/gc.rs b/crates/stdlib/src/gc.rs deleted file mode 100644 index 5fc96a302f7..00000000000 --- a/crates/stdlib/src/gc.rs +++ /dev/null @@ -1,76 +0,0 @@ -pub(crate) use gc::make_module; - -#[pymodule] -mod gc { - use crate::vm::{PyResult, VirtualMachine, function::FuncArgs}; - - #[pyfunction] - fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 { - 0 - } - - #[pyfunction] - fn isenabled(_args: FuncArgs, _vm: &VirtualMachine) -> bool { - false - } - - #[pyfunction] - fn enable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn disable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_count(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_objects(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_stats(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn get_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn is_tracked(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn set_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } - - #[pyfunction] - fn set_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) - } -} diff --git a/crates/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs index 4664d5fc575..c1a52eee62e 100644 --- a/crates/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -1,5 +1,5 @@ // spell-checker:disable -pub(crate) use grp::make_module; +pub(crate) use grp::module_def; #[pymodule] mod grp { @@ -10,8 +10,8 @@ mod grp { exceptions, types::PyStructSequence, }; + use core::ptr::NonNull; use nix::unistd; - use std::ptr::NonNull; #[pystruct_sequence_data] struct GroupData { @@ -30,7 +30,7 @@ mod grp { impl GroupData { fn from_unistd_group(group: unistd::Group, vm: &VirtualMachine) -> Self { - let cstr_lossy = |s: std::ffi::CString| { + let cstr_lossy = |s: alloc::ffi::CString| { s.into_string() .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) }; diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index e7b03a2ff12..5097d804a79 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -1,27 +1,80 @@ -// spell-checker:ignore usedforsecurity HASHXOF +// spell-checker:ignore usedforsecurity HASHXOF hashopenssl dklen +// NOTE: Function names like `openssl_md5` match CPython's `_hashopenssl.c` interface +// for compatibility, but the implementation uses pure Rust crates (md5, sha2, etc.), +// not OpenSSL. -pub(crate) use _hashlib::make_module; +pub(crate) use _hashlib::module_def; #[pymodule] pub mod _hashlib { use crate::common::lock::PyRwLock; use crate::vm::{ Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyBytes, PyStrRef, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyBytes, PyFrozenSet, PyStr, PyStrRef, PyTypeRef, PyValueError, + }, + class::StaticType, convert::ToPyObject, function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, - protocol::PyBuffer, - types::Representable, + types::{Constructor, Representable}, }; use blake2::{Blake2b512, Blake2s256}; - use digest::{DynDigest, core_api::BlockSizeUser}; + use digest::{DynDigest, OutputSizeUser, core_api::BlockSizeUser}; use digest::{ExtendableOutput, Update}; use dyn_clone::{DynClone, clone_trait_object}; + use hmac::Mac; use md5::Md5; use sha1::Sha1; use sha2::{Sha224, Sha256, Sha384, Sha512}; use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; + const HASH_ALGORITHMS: &[&str] = &[ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha3_224", + "sha3_256", + "sha3_384", + "sha3_512", + "shake_128", + "shake_256", + "blake2b", + "blake2s", + ]; + + #[pyattr] + const _GIL_MINSIZE: usize = 2048; + + #[pyattr] + #[pyexception(name = "UnsupportedDigestmodError", base = PyValueError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct UnsupportedDigestmodError(PyValueError); + + #[pyattr] + fn openssl_md_meth_names(vm: &VirtualMachine) -> PyObjectRef { + PyFrozenSet::from_iter( + vm, + HASH_ALGORITHMS.iter().map(|n| vm.ctx.new_str(*n).into()), + ) + .expect("failed to create openssl_md_meth_names frozenset") + .into_ref(&vm.ctx) + .into() + } + + #[pyattr] + fn _constructors(vm: &VirtualMachine) -> PyObjectRef { + let dict = vm.ctx.new_dict(); + for name in HASH_ALGORITHMS { + let s = vm.ctx.new_str(*name); + dict.set_item(&*s, s.clone().into(), vm).unwrap(); + } + dict.into() + } + #[derive(FromArgs, Debug)] #[allow(unused)] struct NewHashArgs { @@ -31,15 +84,19 @@ pub mod _hashlib { data: OptionalArg<ArgBytesLike>, #[pyarg(named, default = true)] usedforsecurity: bool, + #[pyarg(named, optional)] + string: OptionalArg<ArgBytesLike>, } #[derive(FromArgs)] #[allow(unused)] pub struct BlakeHashArgs { - #[pyarg(positional, optional)] + #[pyarg(any, optional)] pub data: OptionalArg<ArgBytesLike>, #[pyarg(named, default = true)] usedforsecurity: bool, + #[pyarg(named, optional)] + pub string: OptionalArg<ArgBytesLike>, } impl From<NewHashArgs> for BlakeHashArgs { @@ -47,6 +104,7 @@ pub mod _hashlib { Self { data: args.data, usedforsecurity: args.usedforsecurity, + string: args.string, } } } @@ -55,20 +113,45 @@ pub mod _hashlib { #[allow(unused)] pub struct HashArgs { #[pyarg(any, optional)] - pub string: OptionalArg<ArgBytesLike>, + pub data: OptionalArg<ArgBytesLike>, #[pyarg(named, default = true)] usedforsecurity: bool, + #[pyarg(named, optional)] + pub string: OptionalArg<ArgBytesLike>, } impl From<NewHashArgs> for HashArgs { fn from(args: NewHashArgs) -> Self { Self { - string: args.data, + data: args.data, usedforsecurity: args.usedforsecurity, + string: args.string, } } } + const KECCAK_WIDTH_BITS: usize = 1600; + + fn keccak_suffix(name: &str) -> Option<u8> { + match name { + "sha3_224" | "sha3_256" | "sha3_384" | "sha3_512" => Some(0x06), + "shake_128" | "shake_256" => Some(0x1f), + _ => None, + } + } + + fn keccak_rate_bits(name: &str, block_size: usize) -> Option<usize> { + keccak_suffix(name).map(|_| block_size * 8) + } + + fn keccak_capacity_bits(name: &str, block_size: usize) -> Option<usize> { + keccak_rate_bits(name, block_size).map(|rate| KECCAK_WIDTH_BITS - rate) + } + + fn missing_hash_attribute<T>(vm: &VirtualMachine, class_name: &str, attr: &str) -> PyResult<T> { + Err(vm.new_attribute_error(format!("'{class_name}' object has no attribute '{attr}'"))) + } + #[derive(FromArgs)] #[allow(unused)] struct XofDigestArgs { @@ -77,9 +160,200 @@ pub mod _hashlib { } impl XofDigestArgs { + // Match CPython's SHAKE output guard in Modules/sha3module.c. + const MAX_SHAKE_OUTPUT_LENGTH: usize = 1 << 29; + fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { - usize::try_from(self.length) - .map_err(|_| vm.new_value_error("length must be non-negative")) + let length = usize::try_from(self.length) + .map_err(|_| vm.new_value_error("length must be non-negative"))?; + if length >= Self::MAX_SHAKE_OUTPUT_LENGTH { + return Err(vm.new_value_error("length is too large")); + } + Ok(length) + } + } + + #[derive(FromArgs)] + #[allow(unused)] + struct HmacDigestArgs { + #[pyarg(positional)] + key: ArgBytesLike, + #[pyarg(positional)] + msg: ArgBytesLike, + #[pyarg(positional)] + digest: PyObjectRef, + } + + #[derive(FromArgs)] + #[allow(unused)] + struct Pbkdf2HmacArgs { + #[pyarg(any)] + hash_name: PyStrRef, + #[pyarg(any)] + password: ArgBytesLike, + #[pyarg(any)] + salt: ArgBytesLike, + #[pyarg(any)] + iterations: i64, + #[pyarg(any, optional)] + dklen: OptionalArg<PyObjectRef>, + } + + fn resolve_data( + data: OptionalArg<ArgBytesLike>, + string: OptionalArg<ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<OptionalArg<ArgBytesLike>> { + match (data.into_option(), string.into_option()) { + (Some(d), None) => Ok(OptionalArg::Present(d)), + (None, Some(s)) => Ok(OptionalArg::Present(s)), + (None, None) => Ok(OptionalArg::Missing), + (Some(_), Some(_)) => Err(vm.new_type_error( + "'data' and 'string' are mutually exclusive \ + and support for 'string' keyword parameter \ + is slated for removal in a future version." + .to_owned(), + )), + } + } + + fn resolve_digestmod(digestmod: &PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + if let Some(name) = digestmod.downcast_ref::<PyStr>() { + return Ok(name.as_str().to_lowercase()); + } + if let Ok(name_obj) = digestmod.get_attr("__name__", vm) + && let Some(name) = name_obj.downcast_ref::<PyStr>() + && let Some(algo) = name.as_str().strip_prefix("openssl_") + { + return Ok(algo.to_owned()); + } + Err(vm.new_exception_msg( + UnsupportedDigestmodError::static_type().to_owned(), + "unsupported digestmod".to_owned(), + )) + } + + fn hash_digest_size(name: &str) -> Option<usize> { + match name { + "md5" => Some(16), + "sha1" => Some(20), + "sha224" => Some(28), + "sha256" => Some(32), + "sha384" => Some(48), + "sha512" => Some(64), + "sha3_224" => Some(28), + "sha3_256" => Some(32), + "sha3_384" => Some(48), + "sha3_512" => Some(64), + "blake2b" => Some(64), + "blake2s" => Some(32), + _ => None, + } + } + + fn unsupported_hash(name: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception_msg( + UnsupportedDigestmodError::static_type().to_owned(), + format!("unsupported hash type {name}"), + ) + } + + // Object-safe HMAC trait for type-erased dispatch + trait DynHmac: Send + Sync { + fn dyn_update(&mut self, data: &[u8]); + fn dyn_finalize(&self) -> Vec<u8>; + fn dyn_clone(&self) -> Box<dyn DynHmac>; + } + + struct TypedHmac<D>(D); + + impl<D> DynHmac for TypedHmac<D> + where + D: Mac + Clone + Send + Sync + 'static, + { + fn dyn_update(&mut self, data: &[u8]) { + Mac::update(&mut self.0, data); + } + + fn dyn_finalize(&self) -> Vec<u8> { + self.0.clone().finalize().into_bytes().to_vec() + } + + fn dyn_clone(&self) -> Box<dyn DynHmac> { + Box::new(TypedHmac(self.0.clone())) + } + } + + #[pyattr] + #[pyclass(module = "_hashlib", name = "HMAC")] + #[derive(PyPayload)] + pub struct PyHmac { + algo_name: String, + digest_size: usize, + block_size: usize, + ctx: PyRwLock<Box<dyn DynHmac>>, + } + + impl core::fmt::Debug for PyHmac { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "HMAC {}", self.algo_name) + } + } + + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] + impl PyHmac { + #[pyslot] + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_hashlib.HMAC' instances".to_owned())) + } + + #[pygetset] + fn name(&self) -> String { + format!("hmac-{}", self.algo_name) + } + + #[pygetset] + fn digest_size(&self) -> usize { + self.digest_size + } + + #[pygetset] + fn block_size(&self) -> usize { + self.block_size + } + + #[pymethod] + fn update(&self, msg: ArgBytesLike) { + msg.with_ref(|bytes| self.ctx.write().dyn_update(bytes)); + } + + #[pymethod] + fn digest(&self) -> PyBytes { + self.ctx.read().dyn_finalize().into() + } + + #[pymethod] + fn hexdigest(&self) -> String { + hex::encode(self.ctx.read().dyn_finalize()) + } + + #[pymethod] + fn copy(&self) -> Self { + Self { + algo_name: self.algo_name.clone(), + digest_size: self.digest_size, + block_size: self.block_size, + ctx: PyRwLock::new(self.ctx.read().dyn_clone()), + } + } + } + + impl Representable for PyHmac { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} HMAC object @ {:#x}>", + zelf.algo_name, zelf as *const _ as usize + )) } } @@ -91,13 +365,13 @@ pub mod _hashlib { pub ctx: PyRwLock<HashWrapper>, } - impl std::fmt::Debug for PyHasher { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyHasher { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "HASH {}", self.name) } } - #[pyclass(with(Representable))] + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] impl PyHasher { fn new(name: &str, d: HashWrapper) -> Self { Self { @@ -126,6 +400,32 @@ pub mod _hashlib { self.ctx.read().block_size() } + #[pygetset] + fn _capacity_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_capacity_bits(&self.name, block_size) { + Some(capacity) => Ok(capacity), + None => missing_hash_attribute(vm, "HASH", "_capacity_bits"), + } + } + + #[pygetset] + fn _rate_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_rate_bits(&self.name, block_size) { + Some(rate) => Ok(rate), + None => missing_hash_attribute(vm, "HASH", "_rate_bits"), + } + } + + #[pygetset] + fn _suffix(&self, vm: &VirtualMachine) -> PyResult<PyBytes> { + match keccak_suffix(&self.name) { + Some(suffix) => Ok(vec![suffix].into()), + None => missing_hash_attribute(vm, "HASH", "_suffix"), + } + } + #[pymethod] fn update(&self, data: ArgBytesLike) { data.with_ref(|bytes| self.ctx.write().update(bytes)); @@ -164,13 +464,13 @@ pub mod _hashlib { ctx: PyRwLock<HashXofWrapper>, } - impl std::fmt::Debug for PyHasherXof { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyHasherXof { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "HASHXOF {}", self.name) } } - #[pyclass] + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] impl PyHasherXof { fn new(name: &str, d: HashXofWrapper) -> Self { Self { @@ -199,6 +499,32 @@ pub mod _hashlib { self.ctx.read().block_size() } + #[pygetset] + fn _capacity_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_capacity_bits(&self.name, block_size) { + Some(capacity) => Ok(capacity), + None => missing_hash_attribute(vm, "HASHXOF", "_capacity_bits"), + } + } + + #[pygetset] + fn _rate_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_rate_bits(&self.name, block_size) { + Some(rate) => Ok(rate), + None => missing_hash_attribute(vm, "HASHXOF", "_rate_bits"), + } + } + + #[pygetset] + fn _suffix(&self, vm: &VirtualMachine) -> PyResult<PyBytes> { + match keccak_suffix(&self.name) { + Some(suffix) => Ok(vec![suffix].into()), + None => missing_hash_attribute(vm, "HASHXOF", "_suffix"), + } + } + #[pymethod] fn update(&self, data: ArgBytesLike) { data.with_ref(|bytes| self.ctx.write().update(bytes)); @@ -220,95 +546,174 @@ pub mod _hashlib { } } + impl Representable for PyHasherXof { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} _hashlib.HASHXOF object @ {:#x}>", + zelf.name, zelf as *const _ as usize + )) + } + } + #[pyfunction(name = "new")] fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let data = resolve_data(args.data, args.string, vm)?; match args.name.as_str().to_lowercase().as_str() { - "md5" => Ok(local_md5(args.into()).into_pyobject(vm)), - "sha1" => Ok(local_sha1(args.into()).into_pyobject(vm)), - "sha224" => Ok(local_sha224(args.into()).into_pyobject(vm)), - "sha256" => Ok(local_sha256(args.into()).into_pyobject(vm)), - "sha384" => Ok(local_sha384(args.into()).into_pyobject(vm)), - "sha512" => Ok(local_sha512(args.into()).into_pyobject(vm)), - "sha3_224" => Ok(local_sha3_224(args.into()).into_pyobject(vm)), - "sha3_256" => Ok(local_sha3_256(args.into()).into_pyobject(vm)), - "sha3_384" => Ok(local_sha3_384(args.into()).into_pyobject(vm)), - "sha3_512" => Ok(local_sha3_512(args.into()).into_pyobject(vm)), - "shake_128" => Ok(local_shake_128(args.into()).into_pyobject(vm)), - "shake_256" => Ok(local_shake_256(args.into()).into_pyobject(vm)), - "blake2b" => Ok(local_blake2b(args.into()).into_pyobject(vm)), - "blake2s" => Ok(local_blake2s(args.into()).into_pyobject(vm)), + "md5" => Ok(PyHasher::new("md5", HashWrapper::new::<Md5>(data)).into_pyobject(vm)), + "sha1" => Ok(PyHasher::new("sha1", HashWrapper::new::<Sha1>(data)).into_pyobject(vm)), + "sha224" => { + Ok(PyHasher::new("sha224", HashWrapper::new::<Sha224>(data)).into_pyobject(vm)) + } + "sha256" => { + Ok(PyHasher::new("sha256", HashWrapper::new::<Sha256>(data)).into_pyobject(vm)) + } + "sha384" => { + Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(data)).into_pyobject(vm)) + } + "sha512" => { + Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(data)).into_pyobject(vm)) + } + "sha3_224" => { + Ok(PyHasher::new("sha3_224", HashWrapper::new::<Sha3_224>(data)).into_pyobject(vm)) + } + "sha3_256" => { + Ok(PyHasher::new("sha3_256", HashWrapper::new::<Sha3_256>(data)).into_pyobject(vm)) + } + "sha3_384" => { + Ok(PyHasher::new("sha3_384", HashWrapper::new::<Sha3_384>(data)).into_pyobject(vm)) + } + "sha3_512" => { + Ok(PyHasher::new("sha3_512", HashWrapper::new::<Sha3_512>(data)).into_pyobject(vm)) + } + "shake_128" => Ok( + PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(data)) + .into_pyobject(vm), + ), + "shake_256" => Ok( + PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(data)) + .into_pyobject(vm), + ), + "blake2b" => Ok( + PyHasher::new("blake2b", HashWrapper::new::<Blake2b512>(data)).into_pyobject(vm), + ), + "blake2s" => Ok( + PyHasher::new("blake2s", HashWrapper::new::<Blake2s256>(data)).into_pyobject(vm), + ), other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {other}"))), } } #[pyfunction(name = "openssl_md5")] - pub fn local_md5(args: HashArgs) -> PyHasher { - PyHasher::new("md5", HashWrapper::new::<Md5>(args.string)) + pub fn local_md5(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("md5", HashWrapper::new::<Md5>(data))) } #[pyfunction(name = "openssl_sha1")] - pub fn local_sha1(args: HashArgs) -> PyHasher { - PyHasher::new("sha1", HashWrapper::new::<Sha1>(args.string)) + pub fn local_sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha1", HashWrapper::new::<Sha1>(data))) } #[pyfunction(name = "openssl_sha224")] - pub fn local_sha224(args: HashArgs) -> PyHasher { - PyHasher::new("sha224", HashWrapper::new::<Sha224>(args.string)) + pub fn local_sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha224", HashWrapper::new::<Sha224>(data))) } #[pyfunction(name = "openssl_sha256")] - pub fn local_sha256(args: HashArgs) -> PyHasher { - PyHasher::new("sha256", HashWrapper::new::<Sha256>(args.string)) + pub fn local_sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha256", HashWrapper::new::<Sha256>(data))) } #[pyfunction(name = "openssl_sha384")] - pub fn local_sha384(args: HashArgs) -> PyHasher { - PyHasher::new("sha384", HashWrapper::new::<Sha384>(args.string)) + pub fn local_sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(data))) } #[pyfunction(name = "openssl_sha512")] - pub fn local_sha512(args: HashArgs) -> PyHasher { - PyHasher::new("sha512", HashWrapper::new::<Sha512>(args.string)) + pub fn local_sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(data))) } #[pyfunction(name = "openssl_sha3_224")] - pub fn local_sha3_224(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_224", HashWrapper::new::<Sha3_224>(args.string)) + pub fn local_sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_224", + HashWrapper::new::<Sha3_224>(data), + )) } #[pyfunction(name = "openssl_sha3_256")] - pub fn local_sha3_256(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_256", HashWrapper::new::<Sha3_256>(args.string)) + pub fn local_sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_256", + HashWrapper::new::<Sha3_256>(data), + )) } #[pyfunction(name = "openssl_sha3_384")] - pub fn local_sha3_384(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_384", HashWrapper::new::<Sha3_384>(args.string)) + pub fn local_sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_384", + HashWrapper::new::<Sha3_384>(data), + )) } #[pyfunction(name = "openssl_sha3_512")] - pub fn local_sha3_512(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_512", HashWrapper::new::<Sha3_512>(args.string)) + pub fn local_sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_512", + HashWrapper::new::<Sha3_512>(data), + )) } #[pyfunction(name = "openssl_shake_128")] - pub fn local_shake_128(args: HashArgs) -> PyHasherXof { - PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) + pub fn local_shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasherXof> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasherXof::new( + "shake_128", + HashXofWrapper::new_shake_128(data), + )) } #[pyfunction(name = "openssl_shake_256")] - pub fn local_shake_256(args: HashArgs) -> PyHasherXof { - PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) + pub fn local_shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasherXof> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasherXof::new( + "shake_256", + HashXofWrapper::new_shake_256(data), + )) } #[pyfunction(name = "openssl_blake2b")] - pub fn local_blake2b(args: BlakeHashArgs) -> PyHasher { - PyHasher::new("blake2b", HashWrapper::new::<Blake2b512>(args.data)) + pub fn local_blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "blake2b", + HashWrapper::new::<Blake2b512>(data), + )) } #[pyfunction(name = "openssl_blake2s")] - pub fn local_blake2s(args: BlakeHashArgs) -> PyHasher { - PyHasher::new("blake2s", HashWrapper::new::<Blake2s256>(args.data)) + pub fn local_blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "blake2s", + HashWrapper::new::<Blake2s256>(data), + )) + } + + #[pyfunction] + fn get_fips_mode() -> i32 { + 0 } #[pyfunction] @@ -338,16 +743,134 @@ pub mod _hashlib { #[allow(unused)] pub struct NewHMACHashArgs { #[pyarg(positional)] - name: PyBuffer, + key: ArgBytesLike, #[pyarg(any, optional)] - data: OptionalArg<ArgBytesLike>, - #[pyarg(named, default = true)] - digestmod: bool, // TODO: RUSTPYTHON support functions & name functions + msg: OptionalArg<Option<ArgBytesLike>>, + #[pyarg(named, optional)] + digestmod: OptionalArg<PyObjectRef>, } #[pyfunction] - fn hmac_new(_args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - Err(vm.new_type_error("cannot create 'hmac' instances")) // TODO: RUSTPYTHON support hmac + fn hmac_new(args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult<PyHmac> { + let digestmod = args.digestmod.into_option().ok_or_else(|| { + vm.new_type_error("Missing required parameter 'digestmod'.".to_owned()) + })?; + let name = resolve_digestmod(&digestmod, vm)?; + + let key_buf = args.key.borrow_buf(); + let msg_data = args.msg.flatten(); + + macro_rules! make_hmac { + ($hash_ty:ty) => {{ + let mut mac = <hmac::Hmac<$hash_ty> as Mac>::new_from_slice(&key_buf) + .map_err(|_| vm.new_value_error("invalid key length".to_owned()))?; + if let Some(ref m) = msg_data { + m.with_ref(|bytes| Mac::update(&mut mac, bytes)); + } + Ok(PyHmac { + algo_name: name, + digest_size: <$hash_ty as OutputSizeUser>::output_size(), + block_size: <$hash_ty as BlockSizeUser>::block_size(), + ctx: PyRwLock::new(Box::new(TypedHmac(mac))), + }) + }}; + } + + match name.as_str() { + "md5" => make_hmac!(Md5), + "sha1" => make_hmac!(Sha1), + "sha224" => make_hmac!(Sha224), + "sha256" => make_hmac!(Sha256), + "sha384" => make_hmac!(Sha384), + "sha512" => make_hmac!(Sha512), + "sha3_224" => make_hmac!(Sha3_224), + "sha3_256" => make_hmac!(Sha3_256), + "sha3_384" => make_hmac!(Sha3_384), + "sha3_512" => make_hmac!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } + } + + #[pyfunction] + fn hmac_digest(args: HmacDigestArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + let name = resolve_digestmod(&args.digest, vm)?; + + let key_buf = args.key.borrow_buf(); + let msg_buf = args.msg.borrow_buf(); + + macro_rules! do_hmac { + ($hash_ty:ty) => {{ + let mut mac = <hmac::Hmac<$hash_ty> as Mac>::new_from_slice(&key_buf) + .map_err(|_| vm.new_value_error("invalid key length".to_owned()))?; + Mac::update(&mut mac, &msg_buf); + Ok(mac.finalize().into_bytes().to_vec().into()) + }}; + } + + match name.as_str() { + "md5" => do_hmac!(Md5), + "sha1" => do_hmac!(Sha1), + "sha224" => do_hmac!(Sha224), + "sha256" => do_hmac!(Sha256), + "sha384" => do_hmac!(Sha384), + "sha512" => do_hmac!(Sha512), + "sha3_224" => do_hmac!(Sha3_224), + "sha3_256" => do_hmac!(Sha3_256), + "sha3_384" => do_hmac!(Sha3_384), + "sha3_512" => do_hmac!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } + } + + #[pyfunction] + fn pbkdf2_hmac(args: Pbkdf2HmacArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + let name = args.hash_name.as_str().to_lowercase(); + + if args.iterations < 1 { + return Err(vm.new_value_error("iteration value must be greater than 0.".to_owned())); + } + let rounds = u32::try_from(args.iterations) + .map_err(|_| vm.new_overflow_error("iteration value is too great.".to_owned()))?; + + let dklen: usize = match args.dklen.into_option() { + Some(obj) if vm.is_none(&obj) => { + hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))? + } + Some(obj) => { + let len: i64 = obj.try_into_value(vm)?; + if len < 1 { + return Err(vm.new_value_error("key length must be greater than 0.".to_owned())); + } + usize::try_from(len) + .map_err(|_| vm.new_overflow_error("key length is too great.".to_owned()))? + } + None => hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))?, + }; + + let password_buf = args.password.borrow_buf(); + let salt_buf = args.salt.borrow_buf(); + let mut dk = vec![0u8; dklen]; + + macro_rules! do_pbkdf2 { + ($hash_ty:ty) => {{ + pbkdf2::pbkdf2_hmac::<$hash_ty>(&password_buf, &salt_buf, rounds, &mut dk); + Ok(dk.into()) + }}; + } + + match name.as_str() { + "md5" => do_pbkdf2!(Md5), + "sha1" => do_pbkdf2!(Sha1), + "sha224" => do_pbkdf2!(Sha224), + "sha256" => do_pbkdf2!(Sha256), + "sha384" => do_pbkdf2!(Sha384), + "sha512" => do_pbkdf2!(Sha512), + "sha3_224" => do_pbkdf2!(Sha3_224), + "sha3_256" => do_pbkdf2!(Sha3_256), + "sha3_384" => do_pbkdf2!(Sha3_384), + "sha3_512" => do_pbkdf2!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } } pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} @@ -355,7 +878,6 @@ pub mod _hashlib { clone_trait_object!(ThreadSafeDynDigest); - /// Generic wrapper patching around the hashing libraries. #[derive(Clone)] pub struct HashWrapper { block_size: usize, diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index eb6ed3a5f64..3baeba629c8 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -1,4 +1,4 @@ -pub(crate) use _json::make_module; +pub(crate) use _json::module_def; mod machinery; #[pymodule] @@ -7,14 +7,37 @@ mod _json { use crate::vm::{ AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyStrRef, PyType}, - convert::{ToPyObject, ToPyResult}, + convert::ToPyResult, function::{IntoFuncArgs, OptionalArg}, protocol::PyIterReturn, types::{Callable, Constructor}, }; + use core::str::FromStr; use malachite_bigint::BigInt; use rustpython_common::wtf8::Wtf8Buf; - use std::str::FromStr; + use std::collections::HashMap; + + /// Skip JSON whitespace characters (space, tab, newline, carriage return). + /// Works with a byte slice and returns the number of bytes skipped. + /// Since all JSON whitespace chars are ASCII, bytes == chars. + #[inline] + fn skip_whitespace(bytes: &[u8]) -> usize { + flame_guard!("_json::skip_whitespace"); + let mut count = 0; + for &b in bytes { + match b { + b' ' | b'\t' | b'\n' | b'\r' => count += 1, + _ => break, + } + } + count + } + + /// Check if a byte slice starts with a given ASCII pattern. + #[inline] + fn starts_with_bytes(bytes: &[u8], pattern: &[u8]) -> bool { + bytes.len() >= pattern.len() && &bytes[..pattern.len()] == pattern + } #[pyattr(name = "make_scanner")] #[pyclass(name = "Scanner", traverse)] @@ -68,57 +91,64 @@ mod _json { impl JsonScanner { fn parse( &self, - s: &str, pystr: PyStrRef, - idx: usize, + char_idx: usize, + byte_idx: usize, scan_once: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<PyIterReturn> { - let c = match s.chars().next() { - Some(c) => c, + flame_guard!("JsonScanner::parse"); + let bytes = pystr.as_str().as_bytes(); + let wtf8 = pystr.as_wtf8(); + + let first_byte = match bytes.get(byte_idx) { + Some(&b) => b, None => { return Ok(PyIterReturn::StopIteration(Some( - vm.ctx.new_int(idx).into(), + vm.ctx.new_int(char_idx).into(), ))); } }; - let next_idx = idx + c.len_utf8(); - match c { - '"' => { - return scanstring(pystr, next_idx, OptionalArg::Present(self.strict), vm) - .map(|x| PyIterReturn::Return(x.to_pyobject(vm))); + + match first_byte { + b'"' => { + // Parse string - pass slice starting after the quote + let (wtf8_result, chars_consumed, _bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone(), vm))?; + let end_char_idx = char_idx + 1 + chars_consumed; + return Ok(PyIterReturn::Return( + vm.new_tuple((wtf8_result, end_char_idx)).into(), + )); } - '{' => { - // TODO: parse the object in rust - let parse_obj = self.ctx.get_attr("parse_object", vm)?; - let result = parse_obj.call( - ( - (pystr, next_idx), - self.strict, - scan_once, - self.object_hook.clone(), - self.object_pairs_hook.clone(), - ), - vm, - ); - return PyIterReturn::from_pyresult(result, vm); + b'{' => { + // Parse object in Rust + let mut memo = HashMap::new(); + return self + .parse_object(pystr, char_idx + 1, byte_idx + 1, &scan_once, &mut memo, vm) + .map(|(obj, end_char, _end_byte)| { + PyIterReturn::Return(vm.new_tuple((obj, end_char)).into()) + }); } - '[' => { - // TODO: parse the array in rust - let parse_array = self.ctx.get_attr("parse_array", vm)?; - return PyIterReturn::from_pyresult( - parse_array.call(((pystr, next_idx), scan_once), vm), - vm, - ); + b'[' => { + // Parse array in Rust + let mut memo = HashMap::new(); + return self + .parse_array(pystr, char_idx + 1, byte_idx + 1, &scan_once, &mut memo, vm) + .map(|(obj, end_char, _end_byte)| { + PyIterReturn::Return(vm.new_tuple((obj, end_char)).into()) + }); } _ => {} } + let s = &pystr.as_str()[byte_idx..]; + macro_rules! parse_const { ($s:literal, $val:expr) => { if s.starts_with($s) { return Ok(PyIterReturn::Return( - vm.new_tuple(($val, idx + $s.len())).into(), + vm.new_tuple(($val, char_idx + $s.len())).into(), )); } }; @@ -129,15 +159,20 @@ mod _json { parse_const!("false", false); if let Some((res, len)) = self.parse_number(s, vm) { - return Ok(PyIterReturn::Return(vm.new_tuple((res?, idx + len)).into())); + return Ok(PyIterReturn::Return( + vm.new_tuple((res?, char_idx + len)).into(), + )); } macro_rules! parse_constant { ($s:literal) => { if s.starts_with($s) { return Ok(PyIterReturn::Return( - vm.new_tuple((self.parse_constant.call(($s,), vm)?, idx + $s.len())) - .into(), + vm.new_tuple(( + self.parse_constant.call(($s,), vm)?, + char_idx + $s.len(), + )) + .into(), )); } }; @@ -148,11 +183,12 @@ mod _json { parse_constant!("-Infinity"); Ok(PyIterReturn::StopIteration(Some( - vm.ctx.new_int(idx).into(), + vm.ctx.new_int(char_idx).into(), ))) } fn parse_number(&self, s: &str, vm: &VirtualMachine) -> Option<(PyResult, usize)> { + flame_guard!("JsonScanner::parse_number"); let mut has_neg = false; let mut has_decimal = false; let mut has_exponent = false; @@ -187,36 +223,452 @@ mod _json { }; Some((ret, buf.len())) } + + /// Parse a JSON object starting after the opening '{'. + /// Returns (parsed_object, end_char_index, end_byte_index). + fn parse_object( + &self, + pystr: PyStrRef, + start_char_idx: usize, + start_byte_idx: usize, + scan_once: &PyObjectRef, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + flame_guard!("JsonScanner::parse_object"); + + let bytes = pystr.as_str().as_bytes(); + let wtf8 = pystr.as_wtf8(); + let mut char_idx = start_char_idx; + let mut byte_idx = start_byte_idx; + + // Skip initial whitespace + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for empty object + match bytes.get(byte_idx) { + Some(b'}') => { + return self.finalize_object(vec![], char_idx + 1, byte_idx + 1, vm); + } + Some(b'"') => { + // Continue to parse first key + } + _ => { + return Err(self.make_decode_error( + "Expecting property name enclosed in double quotes", + pystr, + char_idx, + vm, + )); + } + } + + let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); + + loop { + // We're now at '"', skip it + char_idx += 1; + byte_idx += 1; + + // Parse key string using scanstring with byte slice + let (key_wtf8, chars_consumed, bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx..], char_idx, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone(), vm))?; + + char_idx += chars_consumed; + byte_idx += bytes_consumed; + + // Key memoization - reuse existing key strings + let key_str = key_wtf8.to_string(); + let key: PyObjectRef = match memo.get(&key_str) { + Some(cached) => cached.clone().into(), + None => { + let py_key = vm.ctx.new_str(key_str.clone()); + memo.insert(key_str, py_key.clone()); + py_key.into() + } + }; + + // Skip whitespace after key + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Expect ':' delimiter + match bytes.get(byte_idx) { + Some(b':') => { + char_idx += 1; + byte_idx += 1; + } + _ => { + return Err(self.make_decode_error( + "Expecting ':' delimiter", + pystr, + char_idx, + vm, + )); + } + } + + // Skip whitespace after ':' + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Parse value recursively + let (value, value_char_end, value_byte_end) = + self.call_scan_once(scan_once, pystr.clone(), char_idx, byte_idx, memo, vm)?; + + pairs.push((key, value)); + char_idx = value_char_end; + byte_idx = value_byte_end; + + // Skip whitespace after value + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for ',' or '}' + match bytes.get(byte_idx) { + Some(b'}') => { + char_idx += 1; + byte_idx += 1; + break; + } + Some(b',') => { + let comma_char_idx = char_idx; + char_idx += 1; + byte_idx += 1; + + // Skip whitespace after comma + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Next must be '"' + match bytes.get(byte_idx) { + Some(b'"') => { + // Continue to next key-value pair + } + Some(b'}') => { + // Trailing comma before end of object + return Err(self.make_decode_error( + "Illegal trailing comma before end of object", + pystr, + comma_char_idx, + vm, + )); + } + _ => { + return Err(self.make_decode_error( + "Expecting property name enclosed in double quotes", + pystr, + char_idx, + vm, + )); + } + } + } + _ => { + return Err(self.make_decode_error( + "Expecting ',' delimiter", + pystr, + char_idx, + vm, + )); + } + } + } + + self.finalize_object(pairs, char_idx, byte_idx, vm) + } + + /// Parse a JSON array starting after the opening '['. + /// Returns (parsed_array, end_char_index, end_byte_index). + fn parse_array( + &self, + pystr: PyStrRef, + start_char_idx: usize, + start_byte_idx: usize, + scan_once: &PyObjectRef, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + flame_guard!("JsonScanner::parse_array"); + + let bytes = pystr.as_str().as_bytes(); + let mut char_idx = start_char_idx; + let mut byte_idx = start_byte_idx; + + // Skip initial whitespace + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for empty array + if bytes.get(byte_idx) == Some(&b']') { + return Ok((vm.ctx.new_list(vec![]).into(), char_idx + 1, byte_idx + 1)); + } + + let mut values: Vec<PyObjectRef> = Vec::new(); + + loop { + // Parse value + let (value, value_char_end, value_byte_end) = + self.call_scan_once(scan_once, pystr.clone(), char_idx, byte_idx, memo, vm)?; + + values.push(value); + char_idx = value_char_end; + byte_idx = value_byte_end; + + // Skip whitespace after value + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + match bytes.get(byte_idx) { + Some(b']') => { + char_idx += 1; + byte_idx += 1; + break; + } + Some(b',') => { + let comma_char_idx = char_idx; + char_idx += 1; + byte_idx += 1; + + // Skip whitespace after comma + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for trailing comma + if bytes.get(byte_idx) == Some(&b']') { + return Err(self.make_decode_error( + "Illegal trailing comma before end of array", + pystr, + comma_char_idx, + vm, + )); + } + } + _ => { + return Err(self.make_decode_error( + "Expecting ',' delimiter", + pystr, + char_idx, + vm, + )); + } + } + } + + Ok((vm.ctx.new_list(values).into(), char_idx, byte_idx)) + } + + /// Finalize object construction with hooks. + fn finalize_object( + &self, + pairs: Vec<(PyObjectRef, PyObjectRef)>, + end_char_idx: usize, + end_byte_idx: usize, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + let result = if let Some(ref pairs_hook) = self.object_pairs_hook { + // object_pairs_hook takes priority - pass list of tuples + let pairs_list: Vec<PyObjectRef> = pairs + .into_iter() + .map(|(k, v)| vm.new_tuple((k, v)).into()) + .collect(); + pairs_hook.call((vm.ctx.new_list(pairs_list),), vm)? + } else { + // Build a dict from pairs + let dict = vm.ctx.new_dict(); + for (key, value) in pairs { + dict.set_item(&*key, value, vm)?; + } + + // Apply object_hook if present + let dict_obj: PyObjectRef = dict.into(); + if let Some(ref hook) = self.object_hook { + hook.call((dict_obj,), vm)? + } else { + dict_obj + } + }; + + Ok((result, end_char_idx, end_byte_idx)) + } + + /// Call scan_once and handle the result. + /// Returns (value, end_char_idx, end_byte_idx). + fn call_scan_once( + &self, + scan_once: &PyObjectRef, + pystr: PyStrRef, + char_idx: usize, + byte_idx: usize, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + let s = pystr.as_str(); + let bytes = s.as_bytes(); + let wtf8 = pystr.as_wtf8(); + + let first_byte = match bytes.get(byte_idx) { + Some(&b) => b, + None => return Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)), + }; + + match first_byte { + b'"' => { + // String - pass slice starting after the quote + let (wtf8_result, chars_consumed, bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone(), vm))?; + let py_str = vm.ctx.new_str(wtf8_result.to_string()); + Ok(( + py_str.into(), + char_idx + 1 + chars_consumed, + byte_idx + 1 + bytes_consumed, + )) + } + b'{' => { + // Object + self.parse_object(pystr, char_idx + 1, byte_idx + 1, scan_once, memo, vm) + } + b'[' => { + // Array + self.parse_array(pystr, char_idx + 1, byte_idx + 1, scan_once, memo, vm) + } + b'n' if starts_with_bytes(&bytes[byte_idx..], b"null") => { + // null + Ok((vm.ctx.none(), char_idx + 4, byte_idx + 4)) + } + b't' if starts_with_bytes(&bytes[byte_idx..], b"true") => { + // true + Ok((vm.ctx.new_bool(true).into(), char_idx + 4, byte_idx + 4)) + } + b'f' if starts_with_bytes(&bytes[byte_idx..], b"false") => { + // false + Ok((vm.ctx.new_bool(false).into(), char_idx + 5, byte_idx + 5)) + } + b'N' if starts_with_bytes(&bytes[byte_idx..], b"NaN") => { + // NaN + let result = self.parse_constant.call(("NaN",), vm)?; + Ok((result, char_idx + 3, byte_idx + 3)) + } + b'I' if starts_with_bytes(&bytes[byte_idx..], b"Infinity") => { + // Infinity + let result = self.parse_constant.call(("Infinity",), vm)?; + Ok((result, char_idx + 8, byte_idx + 8)) + } + b'-' => { + // -Infinity or negative number + if starts_with_bytes(&bytes[byte_idx..], b"-Infinity") { + let result = self.parse_constant.call(("-Infinity",), vm)?; + return Ok((result, char_idx + 9, byte_idx + 9)); + } + // Negative number - numbers are ASCII so len == bytes + if let Some((result, len)) = self.parse_number(&s[byte_idx..], vm) { + return Ok((result?, char_idx + len, byte_idx + len)); + } + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + b'0'..=b'9' => { + // Positive number - numbers are ASCII so len == bytes + if let Some((result, len)) = self.parse_number(&s[byte_idx..], vm) { + return Ok((result?, char_idx + len, byte_idx + len)); + } + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + _ => { + // Fall back to scan_once for unrecognized input + // Note: This path requires char_idx for Python compatibility + let result = scan_once.call((pystr.clone(), char_idx as isize), vm); + + match result { + Ok(tuple) => { + use crate::vm::builtins::PyTupleRef; + let tuple: PyTupleRef = tuple.try_into_value(vm)?; + if tuple.len() != 2 { + return Err(vm.new_value_error("scan_once must return 2-tuple")); + } + let value = tuple.as_slice()[0].clone(); + let end_char_idx: isize = tuple.as_slice()[1].try_to_value(vm)?; + // For fallback, we need to calculate byte_idx from char_idx + // This is expensive but fallback should be rare + let end_byte_idx = s + .char_indices() + .nth(end_char_idx as usize) + .map(|(i, _)| i) + .unwrap_or(s.len()); + Ok((value, end_char_idx as usize, end_byte_idx)) + } + Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + Err(err) => Err(err), + } + } + } + } + + /// Create a decode error. + fn make_decode_error( + &self, + msg: &str, + s: PyStrRef, + pos: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let err = machinery::DecodeError::new(msg, pos); + py_decode_error(err, s, vm) + } } impl Callable for JsonScanner { type Args = (PyStrRef, isize); - fn call(zelf: &Py<Self>, (pystr, idx): Self::Args, vm: &VirtualMachine) -> PyResult { - if idx < 0 { + fn call(zelf: &Py<Self>, (pystr, char_idx): Self::Args, vm: &VirtualMachine) -> PyResult { + if char_idx < 0 { return Err(vm.new_value_error("idx cannot be negative")); } - let idx = idx as usize; - let mut chars = pystr.as_str().chars(); - if idx > 0 && chars.nth(idx - 1).is_none() { - PyIterReturn::StopIteration(Some(vm.ctx.new_int(idx).into())).to_pyresult(vm) + let char_idx = char_idx as usize; + let s = pystr.as_str(); + + // Calculate byte index from char index (O(char_idx) but only at entry point) + let byte_idx = if char_idx == 0 { + 0 } else { - zelf.parse( - chars.as_str(), - pystr.clone(), - idx, - zelf.to_owned().into(), - vm, - ) - .and_then(|x| x.to_pyresult(vm)) - } + match s.char_indices().nth(char_idx) { + Some((byte_i, _)) => byte_i, + None => { + // char_idx is beyond the string length + return PyIterReturn::StopIteration(Some(vm.ctx.new_int(char_idx).into())) + .to_pyresult(vm); + } + } + }; + + zelf.parse( + pystr.clone(), + char_idx, + byte_idx, + zelf.to_owned().into(), + vm, + ) + .and_then(|x| x.to_pyresult(vm)) } } fn encode_string(s: &str, ascii_only: bool) -> String { + flame_guard!("_json::encode_string"); let mut buf = Vec::<u8>::with_capacity(s.len() + 2); machinery::write_json_string(s, ascii_only, &mut buf) // SAFETY: writing to a vec can't fail - .unwrap_or_else(|_| unsafe { std::hint::unreachable_unchecked() }); + .unwrap_or_else(|_| unsafe { core::hint::unreachable_unchecked() }); // SAFETY: we only output valid utf8 from write_json_string unsafe { String::from_utf8_unchecked(buf) } } @@ -253,7 +705,29 @@ mod _json { strict: OptionalArg<bool>, vm: &VirtualMachine, ) -> PyResult<(Wtf8Buf, usize)> { - machinery::scanstring(s.as_wtf8(), end, strict.unwrap_or(true)) - .map_err(|e| py_decode_error(e, s, vm)) + flame_guard!("_json::scanstring"); + let wtf8 = s.as_wtf8(); + + // Convert char index `end` to byte index + let byte_idx = if end == 0 { + 0 + } else { + wtf8.code_point_indices() + .nth(end) + .map(|(i, _)| i) + .ok_or_else(|| { + py_decode_error( + machinery::DecodeError::new("Unterminated string starting at", end - 1), + s.clone(), + vm, + ) + })? + }; + + let (result, chars_consumed, _bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx..], end, strict.unwrap_or(true)) + .map_err(|e| py_decode_error(e, s, vm))?; + + Ok((result, end + chars_consumed)) } } diff --git a/crates/stdlib/src/json/machinery.rs b/crates/stdlib/src/json/machinery.rs index 57b8ae441f7..2102d437396 100644 --- a/crates/stdlib/src/json/machinery.rs +++ b/crates/stdlib/src/json/machinery.rs @@ -30,6 +30,7 @@ use std::io; use itertools::Itertools; +use memchr::memchr2; use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; static ESCAPE_CHARS: [&str; 0x20] = [ @@ -43,7 +44,10 @@ static ESCAPE_CHARS: [&str; 0x20] = [ // And which one need to be escaped (1) // The characters that need escaping are 0x00 to 0x1F, 0x22 ("), 0x5C (\), 0x7F (DEL) // Non-ASCII unicode characters can be safely included in a JSON string -#[allow(clippy::unusual_byte_groupings)] // it's groups of 16, come on clippy +#[allow( + clippy::unusual_byte_groupings, + reason = "groups of 16 are intentional here" +)] static NEEDS_ESCAPING_BITSET: [u64; 4] = [ //fedcba9876543210_fedcba9876543210_fedcba9876543210_fedcba9876543210 0b0000000000000000_0000000000000100_1111111111111111_1111111111111111, // 3_2_1_0 @@ -108,7 +112,7 @@ pub struct DecodeError { pub pos: usize, } impl DecodeError { - fn new(msg: impl Into<String>, pos: usize) -> Self { + pub fn new(msg: impl Into<String>, pos: usize) -> Self { let msg = msg.into(); Self { msg, pos } } @@ -126,24 +130,63 @@ impl StrOrChar<'_> { } } } +/// Scan a JSON string starting right after the opening quote. +/// +/// # Arguments +/// * `s` - The string slice starting at the first character after the opening `"` +/// * `char_offset` - The character index where this slice starts (for error messages) +/// * `strict` - Whether to reject control characters +/// +/// # Returns +/// * `Ok((result, chars_consumed, bytes_consumed))` - The decoded string and how much was consumed +/// * `Err(DecodeError)` - If the string is malformed pub fn scanstring<'a>( s: &'a Wtf8, - end: usize, + char_offset: usize, strict: bool, -) -> Result<(Wtf8Buf, usize), DecodeError> { +) -> Result<(Wtf8Buf, usize, usize), DecodeError> { + flame_guard!("machinery::scanstring"); + let unterminated_err = || DecodeError::new("Unterminated string starting at", char_offset - 1); + + let bytes = s.as_bytes(); + + // Fast path: use memchr to find " or \ quickly + if let Some(pos) = memchr2(b'"', b'\\', bytes) + && bytes[pos] == b'"' + { + let content_bytes = &bytes[..pos]; + + // In strict mode, check for control characters (0x00-0x1F) + let has_control_char = strict && content_bytes.iter().any(|&b| b < 0x20); + + if !has_control_char { + flame_guard!("machinery::scanstring::fast_path"); + let result_slice = &s[..pos]; + let char_count = result_slice.code_points().count(); + let mut out = Wtf8Buf::with_capacity(pos); + out.push_wtf8(result_slice); + // +1 for the closing quote + return Ok((out, char_count + 1, pos + 1)); + } + } + + // Slow path: chunk-based parsing for strings with escapes or control chars + flame_guard!("machinery::scanstring::slow_path"); let mut chunks: Vec<StrOrChar<'a>> = Vec::new(); let mut output_len = 0usize; let mut push_chunk = |chunk: StrOrChar<'a>| { output_len += chunk.len(); chunks.push(chunk); }; - let unterminated_err = || DecodeError::new("Unterminated string starting at", end - 1); - let mut chars = s.code_point_indices().enumerate().skip(end).peekable(); - let &(_, (mut chunk_start, _)) = chars.peek().ok_or_else(unterminated_err)?; + + let mut chars = s.code_point_indices().enumerate().peekable(); + let mut chunk_start: usize = 0; + while let Some((char_i, (byte_i, c))) = chars.next() { match c.to_char_lossy() { '"' => { push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); + flame_guard!("machinery::scanstring::assemble_chunks"); let mut out = Wtf8Buf::with_capacity(output_len); for x in chunks { match x { @@ -151,11 +194,12 @@ pub fn scanstring<'a>( StrOrChar::Char(c) => out.push(c), } } - return Ok((out, char_i + 1)); + // +1 for the closing quote + return Ok((out, char_i + 1, byte_i + 1)); } '\\' => { push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); - let (_, (_, c)) = chars.next().ok_or_else(unterminated_err)?; + let (next_char_i, (_, c)) = chars.next().ok_or_else(unterminated_err)?; let esc = match c.to_char_lossy() { '"' => "\"", '\\' => "\\", @@ -166,20 +210,21 @@ pub fn scanstring<'a>( 'r' => "\r", 't' => "\t", 'u' => { - let mut uni = decode_unicode(&mut chars, char_i)?; + let mut uni = decode_unicode(&mut chars, char_offset + char_i)?; chunk_start = byte_i + 6; if let Some(lead) = uni.to_lead_surrogate() { // uni is a surrogate -- try to find its pair let mut chars2 = chars.clone(); - if let Some(((pos2, _), (_, _))) = chars2 + if let Some(((_, (byte_pos2, _)), (_, _))) = chars2 .next_tuple() .filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u') { - let uni2 = decode_unicode(&mut chars2, pos2)?; + let uni2 = + decode_unicode(&mut chars2, char_offset + next_char_i + 1)?; if let Some(trail) = uni2.to_trail_surrogate() { // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates uni = lead.merge(trail).into(); - chunk_start = pos2 + 6; + chunk_start = byte_pos2 + 6; chars = chars2; } } @@ -188,7 +233,10 @@ pub fn scanstring<'a>( continue; } _ => { - return Err(DecodeError::new(format!("Invalid \\escape: {c:?}"), char_i)); + return Err(DecodeError::new( + format!("Invalid \\escape: {c:?}"), + char_offset + char_i, + )); } }; chunk_start = byte_i + 2; @@ -197,7 +245,7 @@ pub fn scanstring<'a>( '\x00'..='\x1f' if strict => { return Err(DecodeError::new( format!("Invalid control character {c:?} at"), - char_i, + char_offset + char_i, )); } _ => {} @@ -211,12 +259,13 @@ fn decode_unicode<I>(it: &mut I, pos: usize) -> Result<CodePoint, DecodeError> where I: Iterator<Item = (usize, (usize, CodePoint))>, { + flame_guard!("machinery::decode_unicode"); let err = || DecodeError::new("Invalid \\uXXXX escape", pos); - let mut uni = 0; - for x in (0..4).rev() { + let mut uni = 0u16; + for _ in 0..4 { let (_, (_, c)) = it.next().ok_or_else(err)?; let d = c.to_char().and_then(|c| c.to_digit(16)).ok_or_else(err)? as u16; - uni += d * 16u16.pow(x); + uni = (uni << 4) | d; } Ok(uni.into()) } diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index c9b5ca32b57..8c234c22f89 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -2,22 +2,24 @@ // how `mod` works, but we want this sometimes for pymodule declarations #![allow(clippy::module_inception)] -#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] #[macro_use] extern crate rustpython_derive; +extern crate alloc; +#[macro_use] +pub(crate) mod macros; + +mod _asyncio; +mod _remote_debugging; pub mod array; mod binascii; mod bisect; +mod bz2; mod cmath; +mod compression; // internal module mod contextvars; mod csv; -mod dis; -mod gc; - -mod bz2; -mod compression; // internal module #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] mod lzma; mod zlib; @@ -32,13 +34,16 @@ mod sha512; mod json; -#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))] +#[cfg(all( + feature = "host_env", + not(any(target_os = "ios", target_arch = "wasm32")) +))] mod locale; +mod _opcode; mod math; -#[cfg(any(unix, windows))] +#[cfg(all(feature = "host_env", any(unix, windows)))] mod mmap; -mod opcode; mod pyexpat; mod pystruct; mod random; @@ -46,182 +51,202 @@ mod statistics; mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] pub mod socket; -#[cfg(all(unix, not(target_os = "redox")))] +#[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] mod syslog; mod unicodedata; +#[cfg(feature = "host_env")] mod faulthandler; -#[cfg(any(unix, target_os = "wasi"))] +#[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] mod fcntl; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] mod multiprocessing; -#[cfg(unix)] +#[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "android") +))] +mod posixshmem; +#[cfg(all(feature = "host_env", unix))] mod posixsubprocess; // libc is missing constants on redox -#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] +#[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) +))] +mod _sqlite3; +#[cfg(all(feature = "host_env", windows))] +mod _testconsole; +#[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "android", target_os = "redox")) +))] mod grp; -#[cfg(windows)] +#[cfg(all(feature = "host_env", windows))] mod overlapped; -#[cfg(all(unix, not(target_os = "redox")))] +#[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] mod resource; -#[cfg(target_os = "macos")] +#[cfg(all(feature = "host_env", target_os = "macos"))] mod scproxy; -#[cfg(any(unix, windows, target_os = "wasi"))] +#[cfg(all(feature = "host_env", any(unix, windows, target_os = "wasi")))] mod select; + #[cfg(all( - feature = "sqlite", - not(any(target_os = "android", target_arch = "wasm32")) + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-openssl" ))] -mod sqlite; - -#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] mod openssl; -#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))] +#[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-rustls" +))] mod ssl; #[cfg(all(feature = "ssl-openssl", feature = "ssl-rustls"))] compile_error!("features \"ssl-openssl\" and \"ssl-rustls\" are mutually exclusive"); -#[cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))] +#[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "ios") +))] mod termios; -#[cfg(not(any( - target_os = "android", - target_os = "ios", - target_os = "windows", - target_arch = "wasm32", - target_os = "redox", -)))] +#[cfg(all( + feature = "host_env", + not(any( + target_os = "android", + target_os = "ios", + target_os = "windows", + target_arch = "wasm32", + target_os = "redox", + )) +))] mod uuid; -#[cfg(feature = "tkinter")] +#[cfg(all(feature = "host_env", feature = "tkinter"))] mod tkinter; use rustpython_common as common; use rustpython_vm as vm; -use crate::vm::{builtins, stdlib::StdlibInitFunc}; -use std::borrow::Cow; +use crate::vm::{Context, builtins}; -pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInitFunc)> { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ] - .into_iter() - }}; - } - modules! { - #[cfg(all())] - { - "array" => array::make_module, - "binascii" => binascii::make_module, - "_bisect" => bisect::make_module, - "_bz2" => bz2::make_module, - "cmath" => cmath::make_module, - "_contextvars" => contextvars::make_module, - "_csv" => csv::make_module, - "_dis" => dis::make_module, - "faulthandler" => faulthandler::make_module, - "gc" => gc::make_module, - "_hashlib" => hashlib::make_module, - "_sha1" => sha1::make_module, - "_sha3" => sha3::make_module, - "_sha256" => sha256::make_module, - "_sha512" => sha512::make_module, - "_md5" => md5::make_module, - "_blake2" => blake2::make_module, - "_json" => json::make_module, - "math" => math::make_module, - "pyexpat" => pyexpat::make_module, - "_opcode" => opcode::make_module, - "_random" => random::make_module, - "_statistics" => statistics::make_module, - "_struct" => pystruct::make_module, - "unicodedata" => unicodedata::make_module, - "zlib" => zlib::make_module, - "_statistics" => statistics::make_module, - "_suggestions" => suggestions::make_module, - // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, - } - #[cfg(any(unix, target_os = "wasi"))] - { - "fcntl" => fcntl::make_module, - } - #[cfg(any(unix, windows, target_os = "wasi"))] - { - "select" => select::make_module, - } - #[cfg(not(target_arch = "wasm32"))] - { - "_multiprocessing" => multiprocessing::make_module, - "_socket" => socket::make_module, - } +/// Returns module definitions for multi-phase init modules. +/// These modules are added to sys.modules BEFORE their exec function runs, +/// allowing safe circular imports. +pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> { + vec![ + _asyncio::module_def(ctx), + _opcode::module_def(ctx), + _remote_debugging::module_def(ctx), + array::module_def(ctx), + binascii::module_def(ctx), + bisect::module_def(ctx), + blake2::module_def(ctx), + bz2::module_def(ctx), + cmath::module_def(ctx), + contextvars::module_def(ctx), + csv::module_def(ctx), + #[cfg(feature = "host_env")] + faulthandler::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] + fcntl::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "android", target_os = "redox")) + ))] + grp::module_def(ctx), + hashlib::module_def(ctx), + json::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(any(target_os = "ios", target_arch = "wasm32")) + ))] + locale::module_def(ctx), #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] - { - "_lzma" => lzma::make_module, - } - #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] - { - "_sqlite3" => sqlite::make_module, - } - #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))] - { - "_ssl" => ssl::make_module, - } - #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] - { - "_ssl" => openssl::make_module, - } - #[cfg(windows)] - { - "_overlapped" => overlapped::make_module, - } - // Unix-only - #[cfg(unix)] - { - "_posixsubprocess" => posixsubprocess::make_module, - } - #[cfg(any(unix, windows))] - { - "mmap" => mmap::make_module, - } - #[cfg(all(unix, not(target_os = "redox")))] - { - "syslog" => syslog::make_module, - "resource" => resource::make_module, - } - #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] - { - "termios" => termios::make_module, - } - #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] - { - "grp" => grp::make_module, - } - #[cfg(target_os = "macos")] - { - "_scproxy" => scproxy::make_module, - } - #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "windows", target_arch = "wasm32", target_os = "redox")))] - { - "_uuid" => uuid::make_module, - } - #[cfg(not(any(target_os = "ios", target_arch = "wasm32")))] - { - "_locale" => locale::make_module, - } - #[cfg(feature = "tkinter")] - { - "_tkinter" => tkinter::make_module, - } - } + lzma::module_def(ctx), + math::module_def(ctx), + md5::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, windows)))] + mmap::module_def(ctx), + #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] + multiprocessing::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-openssl" + ))] + openssl::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + _testconsole::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + overlapped::module_def(ctx), + #[cfg(all(feature = "host_env", unix))] + posixsubprocess::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "android") + ))] + posixshmem::module_def(ctx), + pyexpat::module_def(ctx), + pystruct::module_def(ctx), + random::module_def(ctx), + #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] + resource::module_def(ctx), + #[cfg(all(feature = "host_env", target_os = "macos"))] + scproxy::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, windows, target_os = "wasi")))] + select::module_def(ctx), + sha1::module_def(ctx), + sha256::module_def(ctx), + sha3::module_def(ctx), + sha512::module_def(ctx), + #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] + socket::module_def(ctx), + #[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) + ))] + _sqlite3::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-rustls" + ))] + ssl::module_def(ctx), + statistics::module_def(ctx), + suggestions::module_def(ctx), + #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] + syslog::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "ios", target_os = "redox")) + ))] + termios::module_def(ctx), + #[cfg(all(feature = "host_env", feature = "tkinter"))] + tkinter::module_def(ctx), + unicodedata::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(any( + target_os = "android", + target_os = "ios", + target_os = "windows", + target_arch = "wasm32", + target_os = "redox" + )) + ))] + uuid::module_def(ctx), + zlib::module_def(ctx), + ] } diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs index 6cca8b9123b..496325b5038 100644 --- a/crates/stdlib/src/locale.rs +++ b/crates/stdlib/src/locale.rs @@ -1,6 +1,6 @@ // spell-checker:ignore abday abmon yesexpr noexpr CRNCYSTR RADIXCHAR AMPM THOUSEP -pub(crate) use _locale::make_module; +pub(crate) use _locale::module_def; #[cfg(windows)] #[repr(C)] @@ -41,16 +41,16 @@ use libc::localeconv; #[pymodule] mod _locale { + use alloc::ffi::CString; + use core::{ffi::CStr, ptr}; use rustpython_vm::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, convert::ToPyException, function::OptionalArg, }; - use std::{ - ffi::{CStr, CString}, - ptr, - }; + #[cfg(windows)] + use windows_sys::Win32::Globalization::GetACP; #[cfg(all( unix, @@ -101,10 +101,42 @@ mod _locale { unsafe fn pystr_from_raw_cstr(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { let slice = unsafe { CStr::from_ptr(raw_ptr) }; - let string = slice - .to_str() - .expect("localeconv always return decodable string"); - Ok(vm.new_pyobj(string)) + + // Fast path: ASCII/UTF-8 + if let Ok(s) = slice.to_str() { + return Ok(vm.new_pyobj(s)); + } + + // On Windows, locale strings use the ANSI code page encoding + #[cfg(windows)] + { + use windows_sys::Win32::Globalization::{CP_ACP, MultiByteToWideChar}; + let bytes = slice.to_bytes(); + unsafe { + let len = MultiByteToWideChar( + CP_ACP, + 0, + bytes.as_ptr(), + bytes.len() as i32, + ptr::null_mut(), + 0, + ); + if len > 0 { + let mut wide = vec![0u16; len as usize]; + MultiByteToWideChar( + CP_ACP, + 0, + bytes.as_ptr(), + bytes.len() as i32, + wide.as_mut_ptr(), + len, + ); + return Ok(vm.new_pyobj(String::from_utf16_lossy(&wide))); + } + } + } + + Ok(vm.new_pyobj(String::from_utf8_lossy(slice.to_bytes()).into_owned())) } #[pyattr(name = "Error", once)] @@ -262,4 +294,39 @@ mod _locale { pystr_from_raw_cstr(vm, result) } } + + /// Get the current locale encoding. + #[pyfunction] + fn getencoding() -> String { + #[cfg(windows)] + { + // On Windows, use GetACP() to get the ANSI code page + let acp = unsafe { GetACP() }; + format!("cp{}", acp) + } + #[cfg(not(windows))] + { + // On Unix, use nl_langinfo(CODESET) or fallback to UTF-8 + #[cfg(all( + unix, + not(any(target_os = "ios", target_os = "android", target_os = "redox")) + ))] + { + unsafe { + let codeset = libc::nl_langinfo(libc::CODESET); + if !codeset.is_null() + && let Ok(s) = CStr::from_ptr(codeset).to_str() + && !s.is_empty() + { + return s.to_string(); + } + } + "UTF-8".to_string() + } + #[cfg(any(target_os = "ios", target_os = "android", target_os = "redox"))] + { + "UTF-8".to_string() + } + } + } } diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 855a5eae562..80e4ce80755 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -1,6 +1,6 @@ // spell-checker:ignore ARMTHUMB -pub(crate) use _lzma::make_module; +pub(crate) use _lzma::module_def; #[pymodule] mod _lzma { @@ -8,6 +8,7 @@ mod _lzma { CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, }; + use alloc::fmt; #[pyattr] use lzma_sys::{ LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64, @@ -38,7 +39,6 @@ mod _lzma { use rustpython_vm::function::ArgBytesLike; use rustpython_vm::types::Constructor; use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use std::fmt; use xz2::stream::{Action, Check, Error, Filters, LzmaOptions, Status, Stream}; #[cfg(windows)] @@ -51,7 +51,9 @@ mod _lzma { #[pyattr] const FILTER_DELTA: i32 = 3; #[pyattr] - const CHECK_UNKNOWN: i32 = 16; + const CHECK_ID_MAX: i32 = 15; + #[pyattr] + const CHECK_UNKNOWN: i32 = CHECK_ID_MAX + 1; // the variant ids are hardcoded to be equivalent to the C enum values enum Format { diff --git a/crates/stdlib/src/macros.rs b/crates/stdlib/src/macros.rs new file mode 100644 index 00000000000..385f4b1c4ab --- /dev/null +++ b/crates/stdlib/src/macros.rs @@ -0,0 +1,7 @@ +#[macro_export] +macro_rules! flame_guard { + ($name:expr) => { + #[cfg(feature = "flame-it")] + let _guard = ::flame::start_guard($name); + }; +} diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 62b0ef73ad3..80463dcaa22 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -1,70 +1,48 @@ -pub(crate) use math::make_module; +pub(crate) use math::module_def; -use crate::{builtins::PyBaseExceptionRef, vm::VirtualMachine}; +use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; #[pymodule] mod math { use crate::vm::{ - PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::{PyFloat, PyInt, PyIntRef, PyStrInterned, try_bigint_to_f64, try_f64_to_bigint}, function::{ArgIndex, ArgIntoFloat, ArgIterable, Either, OptionalArg, PosArgs}, identifier, }; - use itertools::Itertools; use malachite_bigint::BigInt; - use num_traits::{One, Signed, ToPrimitive, Zero}; - use rustpython_common::{float_ops, int::true_div}; - use std::cmp::Ordering; + use num_traits::{Signed, ToPrimitive}; + + use super::{float_repr, pymath_exception}; // Constants #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; + use core::f64::consts::{E as e, PI as pi, TAU as tau}; - use super::pymath_error_to_exception; #[pyattr(name = "inf")] const INF: f64 = f64::INFINITY; #[pyattr(name = "nan")] const NAN: f64 = f64::NAN; - // Helper macro: - macro_rules! call_math_func { - ( $fun:ident, $name:ident, $vm:ident ) => {{ - let value = *$name; - let result = value.$fun(); - result_or_overflow(value, result, $vm) - }}; - } - - #[inline] - fn result_or_overflow(value: f64, result: f64, vm: &VirtualMachine) -> PyResult<f64> { - if !result.is_finite() && value.is_finite() { - // CPython doesn't return `inf` when called with finite - // values, it raises OverflowError instead. - Err(vm.new_overflow_error("math range error")) - } else { - Ok(result) - } - } - // Number theory functions: #[pyfunction] fn fabs(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(abs, x, vm) + pymath::math::fabs(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn isfinite(x: ArgIntoFloat) -> bool { - x.is_finite() + pymath::math::isfinite(x.into_float()) } #[pyfunction] fn isinf(x: ArgIntoFloat) -> bool { - x.is_infinite() + pymath::math::isinf(x.into_float()) } #[pyfunction] fn isnan(x: ArgIntoFloat) -> bool { - x.is_nan() + pymath::math::isnan(x.into_float()) } #[derive(FromArgs)] @@ -81,418 +59,288 @@ mod math { #[pyfunction] fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { - let a = *args.a; - let b = *args.b; - let rel_tol = args.rel_tol.map_or(1e-09, |value| value.into()); - let abs_tol = args.abs_tol.map_or(0.0, |value| value.into()); - - if rel_tol < 0.0 || abs_tol < 0.0 { - return Err(vm.new_value_error("tolerances must be non-negative")); - } - - if a == b { - /* short circuit exact equality -- needed to catch two infinities of - the same sign. And perhaps speeds things up a bit sometimes. - */ - return Ok(true); - } + let a = args.a.into_float(); + let b = args.b.into_float(); + let rel_tol = args.rel_tol.into_option().map(|v| v.into_float()); + let abs_tol = args.abs_tol.into_option().map(|v| v.into_float()); - /* This catches the case of two infinities of opposite sign, or - one infinity and one finite number. Two infinities of opposite - sign would otherwise have an infinite relative tolerance. - Two infinities of the same sign are caught by the equality check - above. - */ - - if a.is_infinite() || b.is_infinite() { - return Ok(false); - } - - let diff = (b - a).abs(); - - Ok((diff <= (rel_tol * b).abs()) || (diff <= (rel_tol * a).abs()) || (diff <= abs_tol)) + pymath::math::isclose(a, b, rel_tol, abs_tol) + .map_err(|_| vm.new_value_error("tolerances must be non-negative")) } #[pyfunction] - fn copysign(x: ArgIntoFloat, y: ArgIntoFloat) -> f64 { - x.copysign(*y) + fn copysign(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::copysign(x.into_float(), y.into_float()) + .map_err(|err| pymath_exception(err, vm)) } // Power and logarithmic functions: #[pyfunction] fn exp(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp, x, vm) + pymath::math::exp(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn exp2(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp2, x, vm) + pymath::math::exp2(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn expm1(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp_m1, x, vm) + pymath::math::expm1(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn log(x: PyObjectRef, base: OptionalArg<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let base = base.map(|b| *b).unwrap_or(std::f64::consts::E); - if base.is_sign_negative() { - return Err(vm.new_value_error("math domain error")); + let base = base.into_option().map(|v| v.into_float()); + // Check base first for proper error messages + if let Some(b) = base { + if b <= 0.0 { + return Err(vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(b) + ))); + } + if b == 1.0 { + return Err(vm.new_value_error("math domain error".to_owned())); + } + } + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log_bigint(i.as_bigint(), base).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input".to_owned()), + _ => pymath_exception(err, vm), + }); } - log2(x, vm).map(|log_x| log_x / base.log2()) + let val = x.try_float(vm)?.to_f64(); + pymath::math::log(val, base).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn log1p(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || x > -1.0_f64 { - Ok(x.ln_1p()) - } else { - Err(vm.new_value_error("math domain error")) - } - } - - /// Generates the base-2 logarithm of a BigInt `x` - fn int_log2(x: &BigInt) -> f64 { - // log2(x) = log2(2^n * 2^-n * x) = n + log2(x/2^n) - // If we set 2^n to be the greatest power of 2 below x, then x/2^n is in [1, 2), and can - // thus be converted into a float. - let n = x.bits() as u32 - 1; - let frac = true_div(x, &BigInt::from(2).pow(n)); - f64::from(n) + frac.log2() + pymath::math::log1p(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn log2(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { - match x.try_float(vm) { - Ok(x) => { - let x = x.to_f64(); - if x.is_nan() || x > 0.0_f64 { - Ok(x.log2()) - } else { - Err(vm.new_value_error("math domain error")) - } - } - Err(float_err) => { - if let Ok(x) = x.try_int(vm) { - let x = x.as_bigint(); - if x.is_positive() { - Ok(int_log2(x)) - } else { - Err(vm.new_value_error("math domain error")) - } - } else { - // Return the float error, as it will be more intuitive to users - Err(float_err) - } - } + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log2_bigint(i.as_bigint()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input".to_owned()), + _ => pymath_exception(err, vm), + }); } + let val = x.try_float(vm)?.to_f64(); + pymath::math::log2(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn log10(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { - log2(x, vm).map(|log_x| log_x / 10f64.log2()) - } - - #[pyfunction] - fn pow(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - if x < 0.0 && x.is_finite() && y.fract() != 0.0 && y.is_finite() - || x == 0.0 && y < 0.0 && y != f64::NEG_INFINITY - { - return Err(vm.new_value_error("math domain error")); - } - - let value = x.powf(y); - - if x.is_finite() && y.is_finite() && value.is_infinite() { - return Err(vm.new_overflow_error("math range error")); + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log10_bigint(i.as_bigint()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input".to_owned()), + _ => pymath_exception(err, vm), + }); } - - Ok(value) + let val = x.try_float(vm)?.to_f64(); + pymath::math::log10(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] - fn sqrt(value: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let value = *value; - if value.is_nan() { - return Ok(value); - } - if value.is_sign_negative() { - if value.is_zero() { - return Ok(-0.0f64); - } - return Err(vm.new_value_error("math domain error")); - } - Ok(value.sqrt()) + fn pow(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::pow(x.into_float(), y.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn isqrt(x: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { - let value = x.as_bigint(); - - if value.is_negative() { - return Err(vm.new_value_error("isqrt() argument must be nonnegative")); - } - Ok(value.sqrt()) + fn sqrt(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::sqrt(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a nonnegative input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } // Trigonometric functions: #[pyfunction] fn acos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || (-1.0_f64..=1.0_f64).contains(&x) { - Ok(x.acos()) - } else { - Err(vm.new_value_error("math domain error")) - } + let val = x.into_float(); + pymath::math::acos(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number in range from -1 up to 1, got {}", + float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn asin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || (-1.0_f64..=1.0_f64).contains(&x) { - Ok(x.asin()) - } else { - Err(vm.new_value_error("math domain error")) - } + let val = x.into_float(); + pymath::math::asin(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number in range from -1 up to 1, got {}", + float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn atan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(atan, x, vm) + pymath::math::atan(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn atan2(y: ArgIntoFloat, x: ArgIntoFloat) -> f64 { - y.atan2(*x) + fn atan2(y: ArgIntoFloat, x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::atan2(y.into_float(), x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn cos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - if x.is_infinite() { - return Err(vm.new_value_error("math domain error")); - } - call_math_func!(cos, x, vm) + let val = x.into_float(); + pymath::math::cos(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn hypot(coordinates: PosArgs<ArgIntoFloat>) -> f64 { - let mut coordinates = ArgIntoFloat::vec_into_f64(coordinates.into_vec()); - let mut max = 0.0; - let mut has_nan = false; - for f in &mut coordinates { - *f = f.abs(); - if f.is_nan() { - has_nan = true; - } else if *f > max { - max = *f - } - } - // inf takes precedence over nan - if max.is_infinite() { - return max; - } - if has_nan { - return f64::NAN; - } - coordinates.sort_unstable_by(|x, y| x.total_cmp(y).reverse()); - vector_norm(&coordinates) - } - - /// Implementation of accurate hypotenuse algorithm from Borges 2019. - /// See https://arxiv.org/abs/1904.09481. - /// This assumes that its arguments are positive finite and have been scaled to avoid overflow - /// and underflow. - fn accurate_hypot(max: f64, min: f64) -> f64 { - if min <= max * (f64::EPSILON / 2.0).sqrt() { - return max; - } - let hypot = max.mul_add(max, min * min).sqrt(); - let hypot_sq = hypot * hypot; - let max_sq = max * max; - let correction = (-min).mul_add(min, hypot_sq - max_sq) + hypot.mul_add(hypot, -hypot_sq) - - max.mul_add(max, -max_sq); - hypot - correction / (2.0 * hypot) - } - - /// Calculates the norm of the vector given by `v`. - /// `v` is assumed to be a list of non-negative finite floats, sorted in descending order. - fn vector_norm(v: &[f64]) -> f64 { - // Drop zeros from the vector. - let zero_count = v.iter().rev().cloned().take_while(|x| *x == 0.0).count(); - let v = &v[..v.len() - zero_count]; - if v.is_empty() { - return 0.0; - } - if v.len() == 1 { - return v[0]; - } - // Calculate scaling to avoid overflow / underflow. - let max = *v.first().unwrap(); - let min = *v.last().unwrap(); - let scale = if max > (f64::MAX / v.len() as f64).sqrt() { - max - } else if min < f64::MIN_POSITIVE.sqrt() { - // ^ This can be an `else if`, because if the max is near f64::MAX and the min is near - // f64::MIN_POSITIVE, then the min is relatively unimportant and will be effectively - // ignored. - min - } else { - 1.0 - }; - let mut norm = v - .iter() - .copied() - .map(|x| x / scale) - .reduce(accurate_hypot) - .unwrap_or_default(); - if v.len() > 2 { - // For larger lists of numbers, we can accumulate a rounding error, so a correction is - // needed, similar to that in `accurate_hypot()`. - // First, we estimate [sum of squares - norm^2], then we add the first-order - // approximation of the square root of that to `norm`. - let correction = v - .iter() - .copied() - .map(|x| (x / scale).powi(2)) - .chain(std::iter::once(-norm * norm)) - // Pairwise summation of floats gives less rounding error than a naive sum. - .tree_reduce(std::ops::Add::add) - .expect("expected at least 1 element"); - norm = norm + correction / (2.0 * norm); - } - norm * scale + let coords = ArgIntoFloat::vec_into_f64(coordinates.into_vec()); + pymath::math::hypot(&coords) } #[pyfunction] fn dist(p: Vec<ArgIntoFloat>, q: Vec<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let mut max = 0.0; - let mut has_nan = false; - let p = ArgIntoFloat::vec_into_f64(p); let q = ArgIntoFloat::vec_into_f64(q); - let mut diffs = vec![]; - if p.len() != q.len() { return Err(vm.new_value_error("both points must have the same number of dimensions")); } - - for i in 0..p.len() { - let px = p[i]; - let qx = q[i]; - - let x = (px - qx).abs(); - if x.is_nan() { - has_nan = true; - } - - diffs.push(x); - if x > max { - max = x; - } - } - - if max.is_infinite() { - return Ok(max); - } - if has_nan { - return Ok(f64::NAN); - } - diffs.sort_unstable_by(|x, y| x.total_cmp(y).reverse()); - Ok(vector_norm(&diffs)) + Ok(pymath::math::dist(&p, &q)) } #[pyfunction] fn sin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - if x.is_infinite() { - return Err(vm.new_value_error("math domain error")); - } - call_math_func!(sin, x, vm) + let val = x.into_float(); + pymath::math::sin(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn tan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - if x.is_infinite() { - return Err(vm.new_value_error("math domain error")); - } - call_math_func!(tan, x, vm) + let val = x.into_float(); + pymath::math::tan(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn degrees(x: ArgIntoFloat) -> f64 { - *x * (180.0 / std::f64::consts::PI) + pymath::math::degrees(x.into_float()) } #[pyfunction] fn radians(x: ArgIntoFloat) -> f64 { - *x * (std::f64::consts::PI / 180.0) + pymath::math::radians(x.into_float()) } // Hyperbolic functions: #[pyfunction] fn acosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_sign_negative() || x.is_zero() { - Err(vm.new_value_error("math domain error")) - } else { - Ok(x.acosh()) - } + pymath::math::acosh(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn asinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(asinh, x, vm) + pymath::math::asinh(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn atanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x >= 1.0_f64 || x <= -1.0_f64 { - Err(vm.new_value_error("math domain error")) - } else { - Ok(x.atanh()) - } + let val = x.into_float(); + pymath::math::atanh(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number between -1 and 1, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn cosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(cosh, x, vm) + pymath::math::cosh(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn sinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(sinh, x, vm) + pymath::math::sinh(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn tanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(tanh, x, vm) + pymath::math::tanh(x.into_float()).map_err(|err| pymath_exception(err, vm)) } // Special functions: #[pyfunction] - fn erf(x: ArgIntoFloat) -> f64 { - pymath::erf(*x) + fn erf(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::erf(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn erfc(x: ArgIntoFloat) -> f64 { - pymath::erfc(*x) + fn erfc(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::erfc(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn gamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - pymath::gamma(*x).map_err(|err| pymath_error_to_exception(err, vm)) + let val = x.into_float(); + pymath::math::gamma(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a noninteger or positive integer, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) } #[pyfunction] fn lgamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - pymath::lgamma(*x).map_err(|err| pymath_error_to_exception(err, vm)) + pymath::math::lgamma(x.into_float()).map_err(|err| pymath_exception(err, vm)) } fn try_magic_method( @@ -517,37 +365,43 @@ mod math { #[pyfunction] fn ceil(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let result_or_err = try_magic_method(identifier!(vm, __ceil__), vm, &x); - if result_or_err.is_err() - && let Some(v) = x.try_float_opt(vm) - { + // Only call __ceil__ if the class defines it - if it exists but is not callable, + // the error should be propagated (not fall back to float conversion) + if x.class().has_attr(identifier!(vm, __ceil__)) { + return try_magic_method(identifier!(vm, __ceil__), vm, &x); + } + // __ceil__ not defined - fall back to float conversion + if let Some(v) = x.try_float_opt(vm) { let v = try_f64_to_bigint(v?.to_f64().ceil(), vm)?; return Ok(vm.ctx.new_int(v).into()); } - result_or_err + Err(vm.new_type_error(format!( + "type '{}' doesn't define '__ceil__' method", + x.class().name(), + ))) } #[pyfunction] fn floor(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let result_or_err = try_magic_method(identifier!(vm, __floor__), vm, &x); - if result_or_err.is_err() - && let Some(v) = x.try_float_opt(vm) - { + // Only call __floor__ if the class defines it - if it exists but is not callable, + // the error should be propagated (not fall back to float conversion) + if x.class().has_attr(identifier!(vm, __floor__)) { + return try_magic_method(identifier!(vm, __floor__), vm, &x); + } + // __floor__ not defined - fall back to float conversion + if let Some(v) = x.try_float_opt(vm) { let v = try_f64_to_bigint(v?.to_f64().floor(), vm)?; return Ok(vm.ctx.new_int(v).into()); } - result_or_err + Err(vm.new_type_error(format!( + "type '{}' doesn't define '__floor__' method", + x.class().name(), + ))) } #[pyfunction] fn frexp(x: ArgIntoFloat) -> (f64, i32) { - let value = *x; - if value.is_finite() { - let (m, exp) = float_ops::decompose_float(value); - (m * value.signum(), exp) - } else { - (value, 0) - } + pymath::math::frexp(x.into_float()) } #[pyfunction] @@ -560,312 +414,24 @@ mod math { Either::A(f) => f.to_f64(), Either::B(z) => try_bigint_to_f64(z.as_bigint(), vm)?, }; - - if value == 0_f64 || !value.is_finite() { - // NaNs, zeros and infinities are returned unchanged - return Ok(value); - } - - // Using IEEE 754 bit manipulation to handle large exponents correctly. - // Direct multiplication would overflow for large i values, especially when computing - // the largest finite float (i=1024, x<1.0). By directly modifying the exponent bits, - // we avoid intermediate overflow to infinity. - - // Scale subnormals to normal range first, then adjust exponent. - let (mant, exp0) = if value.abs() < f64::MIN_POSITIVE { - let scaled = value * (1u64 << 54) as f64; // multiply by 2^54 - let (mant_scaled, exp_scaled) = float_ops::decompose_float(scaled); - (mant_scaled, exp_scaled - 54) // adjust exponent back - } else { - float_ops::decompose_float(value) - }; - - let i_big = i.as_bigint(); - let overflow_bound = BigInt::from(1024_i32 - exp0); // i > 1024 - exp0 => overflow - if i_big > &overflow_bound { - return Err(vm.new_overflow_error("math range error")); - } - if i_big == &overflow_bound && mant == 1.0 { - return Err(vm.new_overflow_error("math range error")); - } - let underflow_bound = BigInt::from(-1074_i32 - exp0); // i < -1074 - exp0 => 0.0 with sign - if i_big < &underflow_bound { - return Ok(0.0f64.copysign(value)); - } - - let i_small: i32 = i_big - .to_i32() - .expect("exponent within [-1074-exp0, 1024-exp0] must fit in i32"); - let exp = exp0 + i_small; - - const SIGN_MASK: u64 = 0x8000_0000_0000_0000; - const FRAC_MASK: u64 = 0x000F_FFFF_FFFF_FFFF; - let sign_bit: u64 = if value.is_sign_negative() { - SIGN_MASK - } else { - 0 - }; - let mant_bits = mant.to_bits() & FRAC_MASK; - if exp >= -1021 { - let e_bits = (1022_i32 + exp) as u64; - let result_bits = sign_bit | (e_bits << 52) | mant_bits; - return Ok(f64::from_bits(result_bits)); - } - - let full_mant: u64 = (1u64 << 52) | mant_bits; - let shift: u32 = (-exp - 1021) as u32; - let frac_shifted = full_mant >> shift; - let lost_bits = full_mant & ((1u64 << shift) - 1); - - let half = 1u64 << (shift - 1); - let frac = if (lost_bits > half) || (lost_bits == half && (frac_shifted & 1) == 1) { - frac_shifted + 1 - } else { - frac_shifted - }; - - let result_bits = if frac >= (1u64 << 52) { - sign_bit | (1u64 << 52) - } else { - sign_bit | frac - }; - Ok(f64::from_bits(result_bits)) - } - - fn math_perf_arb_len_int_op<F>(args: PosArgs<ArgIndex>, op: F, default: BigInt) -> BigInt - where - F: Fn(&BigInt, &PyInt) -> BigInt, - { - let arg_vec = args.into_vec(); - - if arg_vec.is_empty() { - return default; - } else if arg_vec.len() == 1 { - return op(arg_vec[0].as_bigint(), &arg_vec[0]); - } - - let mut res = arg_vec[0].as_bigint().clone(); - for num in &arg_vec[1..] { - res = op(&res, num) - } - res - } - - #[pyfunction] - fn gcd(args: PosArgs<ArgIndex>) -> BigInt { - use num_integer::Integer; - math_perf_arb_len_int_op(args, |x, y| x.gcd(y.as_bigint()), BigInt::zero()) - } - - #[pyfunction] - fn lcm(args: PosArgs<ArgIndex>) -> BigInt { - use num_integer::Integer; - math_perf_arb_len_int_op(args, |x, y| x.lcm(y.as_bigint()), BigInt::one()) + pymath::math::ldexp_bigint(value, i.as_bigint()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] - fn cbrt(x: ArgIntoFloat) -> f64 { - x.cbrt() + fn cbrt(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::cbrt(x.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn fsum(seq: ArgIterable<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let mut partials = Vec::with_capacity(32); - let mut special_sum = 0.0; - let mut inf_sum = 0.0; - - for obj in seq.iter(vm)? { - let mut x = *obj?; - - let xsave = x; - let mut i = 0; - // This inner loop applies `hi`/`lo` summation to each - // partial so that the list of partial sums remains exact. - for j in 0..partials.len() { - let mut y: f64 = partials[j]; - if x.abs() < y.abs() { - std::mem::swap(&mut x, &mut y); - } - // Rounded `x+y` is stored in `hi` with round-off stored in - // `lo`. Together `hi+lo` are exactly equal to `x+y`. - let hi = x + y; - let lo = y - (hi - x); - if lo != 0.0 { - partials[i] = lo; - i += 1; - } - x = hi; - } - - partials.truncate(i); - if x != 0.0 { - if !x.is_finite() { - // a non-finite x could arise either as - // a result of intermediate overflow, or - // as a result of a nan or inf in the - // summands - if xsave.is_finite() { - return Err(vm.new_overflow_error("intermediate overflow in fsum")); - } - if xsave.is_infinite() { - inf_sum += xsave; - } - special_sum += xsave; - // reset partials - partials.clear(); - } else { - partials.push(x); - } - } - } - if special_sum != 0.0 { - return if inf_sum.is_nan() { - Err(vm.new_value_error("-inf + inf in fsum")) - } else { - Ok(special_sum) - }; - } - - let mut n = partials.len(); - if n > 0 { - n -= 1; - let mut hi = partials[n]; - - let mut lo = 0.0; - while n > 0 { - let x = hi; - - n -= 1; - let y = partials[n]; - - hi = x + y; - lo = y - (hi - x); - if lo != 0.0 { - break; - } - } - if n > 0 && ((lo < 0.0 && partials[n - 1] < 0.0) || (lo > 0.0 && partials[n - 1] > 0.0)) - { - let y = lo + lo; - let x = hi + y; - - // Make half-even rounding work across multiple partials. - // Needed so that sum([1e-16, 1, 1e16]) will round-up the last - // digit to two instead of down to zero (the 1e-16 makes the 1 - // slightly closer to two). With a potential 1 ULP rounding - // error fixed-up, math.fsum() can guarantee commutativity. - if y == x - hi { - hi = x; - } - } - - Ok(hi) - } else { - Ok(0.0) - } - } - - #[pyfunction] - fn factorial(x: PyIntRef, vm: &VirtualMachine) -> PyResult<BigInt> { - let value = x.as_bigint(); - let one = BigInt::one(); - if value.is_negative() { - return Err(vm.new_value_error("factorial() not defined for negative values")); - } else if *value <= one { - return Ok(one); - } - // start from 2, since we know that value > 1 and 1*2=2 - let mut current = one + 1; - let mut product = BigInt::from(2u8); - while current < *value { - current += 1; - product *= &current; - } - Ok(product) - } - - #[pyfunction] - fn perm( - n: ArgIndex, - k: OptionalArg<Option<ArgIndex>>, - vm: &VirtualMachine, - ) -> PyResult<BigInt> { - let n = n.as_bigint(); - let k_ref; - let v = match k.flatten() { - Some(k) => { - k_ref = k; - k_ref.as_bigint() - } - None => n, - }; - - if n.is_negative() || v.is_negative() { - return Err(vm.new_value_error("perm() not defined for negative values")); - } - if v > n { - return Ok(BigInt::zero()); - } - let mut result = BigInt::one(); - let mut current = n.clone(); - let tmp = n - v; - while current > tmp { - result *= &current; - current -= 1; - } - Ok(result) - } - - #[pyfunction] - fn comb(n: ArgIndex, k: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { - let mut k = k.as_bigint(); - let n = n.as_bigint(); - let one = BigInt::one(); - let zero = BigInt::zero(); - - if n.is_negative() || k.is_negative() { - return Err(vm.new_value_error("comb() not defined for negative values")); - } - - let temp = n - k; - if temp.is_negative() { - return Ok(zero); - } - - if temp < *k { - k = &temp - } - - if k.is_zero() { - return Ok(one); - } - - let mut result = n.clone(); - let mut factor = n.clone(); - let mut current = one; - while current < *k { - factor -= 1; - current += 1; - - result *= &factor; - result /= &current; - } - - Ok(result) + let values: Result<Vec<f64>, _> = + seq.iter(vm)?.map(|r| r.map(|v| v.into_float())).collect(); + pymath::math::fsum(values?).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn modf(x: ArgIntoFloat) -> (f64, f64) { - let x = *x; - if !x.is_finite() { - if x.is_infinite() { - return (0.0_f64.copysign(x), x); - } else if x.is_nan() { - return (x, x); - } - } - - (x.fract(), x.trunc()) + pymath::math::modf(x.into_float()) } #[derive(FromArgs)] @@ -880,87 +446,36 @@ mod math { #[pyfunction] fn nextafter(arg: NextAfterArgs, vm: &VirtualMachine) -> PyResult<f64> { - let steps: Option<i64> = arg - .steps - .map(|v| v.try_to_primitive(vm)) - .transpose()? - .into_option(); - match steps { + let x = arg.x.into_float(); + let y = arg.y.into_float(); + + let steps = match arg.steps.into_option() { Some(steps) => { + let steps: i64 = steps.into_int_ref().try_to_primitive(vm)?; if steps < 0 { return Err(vm.new_value_error("steps must be a non-negative integer")); } - Ok(float_ops::nextafter_with_steps( - *arg.x, - *arg.y, - steps as u64, - )) + Some(steps as u64) } - None => Ok(float_ops::nextafter(*arg.x, *arg.y)), - } + None => None, + }; + Ok(pymath::math::nextafter(x, y, steps)) } #[pyfunction] fn ulp(x: ArgIntoFloat) -> f64 { - float_ops::ulp(*x) - } - - fn fmod(x: f64, y: f64) -> f64 { - if y.is_infinite() && x.is_finite() { - return x; - } - - x % y + pymath::math::ulp(x.into_float()) } #[pyfunction(name = "fmod")] fn py_fmod(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - let r = fmod(x, y); - - if r.is_nan() && !x.is_nan() && !y.is_nan() { - return Err(vm.new_value_error("math domain error")); - } - - Ok(r) + pymath::math::fmod(x.into_float(), y.into_float()).map_err(|err| pymath_exception(err, vm)) } #[pyfunction] fn remainder(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - if x.is_finite() && y.is_finite() { - if y == 0.0 { - return Err(vm.new_value_error("math domain error")); - } - - let abs_x = x.abs(); - let abs_y = y.abs(); - let modulus = abs_x % abs_y; - - let c = abs_y - modulus; - let r = match modulus.partial_cmp(&c) { - Some(Ordering::Less) => modulus, - Some(Ordering::Greater) => -c, - _ => modulus - 2.0 * fmod(0.5 * (abs_x - modulus), abs_y), - }; - - return Ok(1.0_f64.copysign(x) * r); - } - if x.is_infinite() && !y.is_nan() { - return Err(vm.new_value_error("math domain error")); - } - if x.is_nan() || y.is_nan() { - return Ok(f64::NAN); - } - if y.is_infinite() { - Ok(x) - } else { - Err(vm.new_value_error("math domain error")) - } + pymath::math::remainder(x.into_float(), y.into_float()) + .map_err(|err| pymath_exception(err, vm)) } #[derive(FromArgs)] @@ -973,15 +488,118 @@ mod math { #[pyfunction] fn prod(args: ProdArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use crate::vm::builtins::PyInt; + let iter = args.iterable; + let start = args.start; + + // Check if start is provided and what type it is (exact types only, not subclasses) + let (mut obj_result, start_is_int, start_is_float) = match &start { + OptionalArg::Present(s) => { + let is_int = s.class().is(vm.ctx.types.int_type); + let is_float = s.class().is(vm.ctx.types.float_type); + (Some(s.clone()), is_int, is_float) + } + OptionalArg::Missing => (None, true, false), // Default is int 1 + }; + + let mut item_iter = iter.iter(vm)?; + + // Integer fast path + if start_is_int && !start_is_float { + let mut int_result: i64 = match &start { + OptionalArg::Present(s) => { + if let Some(i) = s.downcast_ref::<PyInt>() { + match i.as_bigint().try_into() { + Ok(v) => v, + Err(_) => { + // Start overflows i64, fall through to generic path + obj_result = Some(s.clone()); + i64::MAX // Will be ignored + } + } + } else { + 1 + } + } + OptionalArg::Missing => 1, + }; - let mut result = args.start.unwrap_or_else(|| vm.new_pyobj(1)); + if obj_result.is_none() { + loop { + let item = match item_iter.next() { + Some(r) => r?, + None => return Ok(vm.ctx.new_int(int_result).into()), + }; + + // Only use fast path for exact int type (not subclasses) + if item.class().is(vm.ctx.types.int_type) + && let Some(int_item) = item.downcast_ref::<PyInt>() + && let Ok(b) = int_item.as_bigint().try_into() as Result<i64, _> + && let Some(product) = int_result.checked_mul(b) + { + int_result = product; + continue; + } - // TODO: CPython has optimized implementation for this - // refer: https://github.com/python/cpython/blob/main/Modules/mathmodule.c#L3093-L3193 - for obj in iter.iter(vm)? { - let obj = obj?; - result = vm._mul(&result, &obj)?; + // Overflow or non-int: restore to PyObject and continue + obj_result = Some(vm.ctx.new_int(int_result).into()); + let temp = vm._mul(obj_result.as_ref().unwrap(), &item)?; + obj_result = Some(temp); + break; + } + } + } + + // Float fast path + let obj_float = obj_result + .as_ref() + .and_then(|obj| obj.clone().downcast::<PyFloat>().ok()); + if obj_float.is_some() || start_is_float { + let mut flt_result: f64 = if let Some(ref f) = obj_float { + f.to_f64() + } else if start_is_float && let OptionalArg::Present(s) = &start { + s.downcast_ref::<PyFloat>() + .map(|f| f.to_f64()) + .unwrap_or(1.0) + } else { + 1.0 + }; + + loop { + let item = match item_iter.next() { + Some(r) => r?, + None => return Ok(vm.ctx.new_float(flt_result).into()), + }; + + // Only use fast path for exact float/int types (not subclasses) + if item.class().is(vm.ctx.types.float_type) + && let Some(f) = item.downcast_ref::<PyFloat>() + { + flt_result *= f.to_f64(); + continue; + } + if item.class().is(vm.ctx.types.int_type) + && let Some(i) = item.downcast_ref::<PyInt>() + && let Ok(v) = i.as_bigint().try_into() as Result<i64, _> + { + flt_result *= v as f64; + continue; + } + + // Non-exact-float/int: restore and continue with generic path + obj_result = Some(vm.ctx.new_float(flt_result).into()); + let temp = vm._mul(obj_result.as_ref().unwrap(), &item)?; + obj_result = Some(temp); + break; + } + } + + // Generic path for remaining items + let mut result = obj_result.unwrap_or_else(|| vm.ctx.new_int(1).into()); + for item in item_iter { + let item = item?; + result = vm._mul(&result, &item)?; } Ok(result) @@ -993,29 +611,145 @@ mod math { q: ArgIterable<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<PyObjectRef> { + use crate::vm::builtins::PyInt; + let mut p_iter = p.iter(vm)?; let mut q_iter = q.iter(vm)?; - // We cannot just create a float because the iterator may contain - // anything as long as it supports __add__ and __mul__. - let mut result = vm.new_pyobj(0); + + // Fast path state + let mut int_path_enabled = true; + let mut int_total: i64 = 0; + let mut int_total_in_use = false; + let mut flt_p_values: Vec<f64> = Vec::new(); + let mut flt_q_values: Vec<f64> = Vec::new(); + + // Fallback accumulator for generic Python path + let mut obj_total: Option<PyObjectRef> = None; + loop { let m_p = p_iter.next(); let m_q = q_iter.next(); - match (m_p, m_q) { - (Some(r_p), Some(r_q)) => { - let p = r_p?; - let q = r_q?; - let tmp = vm._mul(&p, &q)?; - result = vm._add(&result, &tmp)?; + + let (p_i, q_i, finished) = match (m_p, m_q) { + (Some(r_p), Some(r_q)) => (Some(r_p?), Some(r_q?), false), + (None, None) => (None, None, true), + _ => return Err(vm.new_value_error("Inputs are not the same length")), + }; + + // Integer fast path (only for exact int types, not subclasses) + if int_path_enabled { + if !finished { + let (p_i, q_i) = (p_i.as_ref().unwrap(), q_i.as_ref().unwrap()); + if p_i.class().is(vm.ctx.types.int_type) + && q_i.class().is(vm.ctx.types.int_type) + && let (Some(p_int), Some(q_int)) = + (p_i.downcast_ref::<PyInt>(), q_i.downcast_ref::<PyInt>()) + && let (Ok(p_val), Ok(q_val)) = ( + p_int.as_bigint().try_into() as Result<i64, _>, + q_int.as_bigint().try_into() as Result<i64, _>, + ) + && let Some(prod) = p_val.checked_mul(q_val) + && let Some(new_total) = int_total.checked_add(prod) + { + int_total = new_total; + int_total_in_use = true; + continue; + } } - (None, None) => break, - _ => { - return Err(vm.new_value_error("Inputs are not the same length")); + // Finalize int path + int_path_enabled = false; + if int_total_in_use { + let int_obj: PyObjectRef = vm.ctx.new_int(int_total).into(); + obj_total = Some(match obj_total { + Some(total) => vm._add(&total, &int_obj)?, + None => int_obj, + }); + int_total = 0; + int_total_in_use = false; } } + + // Float fast path - only when at least one value is exact float type + // (not subclasses, to preserve custom __mul__/__add__ behavior) + { + if !finished { + let (p_i, q_i) = (p_i.as_ref().unwrap(), q_i.as_ref().unwrap()); + + let p_is_exact_float = p_i.class().is(vm.ctx.types.float_type); + let q_is_exact_float = q_i.class().is(vm.ctx.types.float_type); + let p_is_exact_int = p_i.class().is(vm.ctx.types.int_type); + let q_is_exact_int = q_i.class().is(vm.ctx.types.int_type); + let p_is_exact_numeric = p_is_exact_float || p_is_exact_int; + let q_is_exact_numeric = q_is_exact_float || q_is_exact_int; + let has_exact_float = p_is_exact_float || q_is_exact_float; + + // Only use float path if at least one is exact float and both are exact int/float + if has_exact_float && p_is_exact_numeric && q_is_exact_numeric { + let p_flt = if let Some(f) = p_i.downcast_ref::<PyFloat>() { + Some(f.to_f64()) + } else if let Some(i) = p_i.downcast_ref::<PyInt>() { + // PyLong_AsDouble fails for integers too large for f64 + try_bigint_to_f64(i.as_bigint(), vm).ok() + } else { + None + }; + + let q_flt = if let Some(f) = q_i.downcast_ref::<PyFloat>() { + Some(f.to_f64()) + } else if let Some(i) = q_i.downcast_ref::<PyInt>() { + // PyLong_AsDouble fails for integers too large for f64 + try_bigint_to_f64(i.as_bigint(), vm).ok() + } else { + None + }; + + if let (Some(p_val), Some(q_val)) = (p_flt, q_flt) { + flt_p_values.push(p_val); + flt_q_values.push(q_val); + continue; + } + } + } + // Finalize float path + if !flt_p_values.is_empty() { + let flt_result = pymath::math::sumprod(&flt_p_values, &flt_q_values); + let flt_obj: PyObjectRef = vm.ctx.new_float(flt_result).into(); + obj_total = Some(match obj_total { + Some(total) => vm._add(&total, &flt_obj)?, + None => flt_obj, + }); + flt_p_values.clear(); + flt_q_values.clear(); + } + } + + if finished { + break; + } + + // Generic Python path + let (p_i, q_i) = (p_i.unwrap(), q_i.unwrap()); + + // Collect current + remaining elements + let p_remaining: Result<Vec<PyObjectRef>, _> = + core::iter::once(Ok(p_i)).chain(p_iter).collect(); + let q_remaining: Result<Vec<PyObjectRef>, _> = + core::iter::once(Ok(q_i)).chain(q_iter).collect(); + let (p_vec, q_vec) = (p_remaining?, q_remaining?); + + if p_vec.len() != q_vec.len() { + return Err(vm.new_value_error("Inputs are not the same length")); + } + + let mut total = obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into()); + for (p_item, q_item) in p_vec.into_iter().zip(q_vec) { + let prod = vm._mul(&p_item, &q_item)?; + total = vm._add(&total, &prod)?; + } + return Ok(total); } - Ok(result) + Ok(obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into())) } #[pyfunction] @@ -1025,27 +759,202 @@ mod math { z: ArgIntoFloat, vm: &VirtualMachine, ) -> PyResult<f64> { - let result = (*x).mul_add(*y, *z); + pymath::math::fma(x.into_float(), y.into_float(), z.into_float()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("invalid operation in fma"), + pymath::Error::ERANGE => vm.new_overflow_error("overflow in fma"), + }) + } + + // Integer functions: + + #[pyfunction] + fn isqrt(x: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { + let value = x.into_int_ref(); + pymath::math::integer::isqrt(value.as_bigint()) + .map_err(|_| vm.new_value_error("isqrt() argument must be nonnegative")) + } - if result.is_finite() { - return Ok(result); + #[pyfunction] + fn gcd(args: PosArgs<ArgIndex>) -> BigInt { + let ints: Vec<_> = args + .into_vec() + .into_iter() + .map(|x| x.into_int_ref()) + .collect(); + let refs: Vec<_> = ints.iter().map(|x| x.as_bigint()).collect(); + pymath::math::integer::gcd(&refs) + } + + #[pyfunction] + fn lcm(args: PosArgs<ArgIndex>) -> BigInt { + let ints: Vec<_> = args + .into_vec() + .into_iter() + .map(|x| x.into_int_ref()) + .collect(); + let refs: Vec<_> = ints.iter().map(|x| x.as_bigint()).collect(); + pymath::math::integer::lcm(&refs) + } + + #[pyfunction] + fn factorial(x: PyIntRef, vm: &VirtualMachine) -> PyResult<BigInt> { + // Check for negative before overflow - negative values are always invalid + if x.as_bigint().is_negative() { + return Err(vm.new_value_error("factorial() not defined for negative values")); } + let n: i64 = x.try_to_primitive(vm).map_err(|_| { + vm.new_overflow_error("factorial() argument should not exceed 9223372036854775807") + })?; + pymath::math::integer::factorial(n) + .map(|r| r.into()) + .map_err(|_| vm.new_value_error("factorial() not defined for negative values")) + } + + #[pyfunction] + fn perm( + n: ArgIndex, + k: OptionalArg<Option<ArgIndex>>, + vm: &VirtualMachine, + ) -> PyResult<BigInt> { + let n_int = n.into_int_ref(); + let n_big = n_int.as_bigint(); - if result.is_nan() { - if !x.is_nan() && !y.is_nan() && !z.is_nan() { - return Err(vm.new_value_error("invalid operation in fma")); + if n_big.is_negative() { + return Err(vm.new_value_error("n must be a non-negative integer")); + } + + // k = None means k = n (factorial) + let k_int = k.flatten().map(|k| k.into_int_ref()); + let k_big: Option<&BigInt> = k_int.as_ref().map(|k| k.as_bigint()); + + if let Some(k_val) = k_big { + if k_val.is_negative() { + return Err(vm.new_value_error("k must be a non-negative integer")); + } + if k_val > n_big { + return Ok(BigInt::from(0u8)); } - } else if x.is_finite() && y.is_finite() && z.is_finite() { - return Err(vm.new_overflow_error("overflow in fma")); } - Ok(result) + // Convert k to u64 (required by pymath) + let ki: u64 = match k_big { + None => match n_big.to_u64() { + Some(n) => n, + None => { + return Err(vm.new_overflow_error(format!("n must not exceed {}", u64::MAX))); + } + }, + Some(k_val) => match k_val.to_u64() { + Some(k) => k, + None => { + return Err(vm.new_overflow_error(format!("k must not exceed {}", u64::MAX))); + } + }, + }; + + // Fast path: n fits in i64 + if let Some(ni) = n_big.to_i64() + && ni >= 0 + && ki > 1 + { + let result = pymath::math::integer::perm(ni, Some(ki as i64)) + .map_err(|_| vm.new_value_error("perm() error"))?; + return Ok(result.into()); + } + + // BigInt path: use perm_bigint + let result = pymath::math::perm_bigint(n_big, ki); + Ok(result.into()) + } + + #[pyfunction] + fn comb(n: ArgIndex, k: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { + let n_int = n.into_int_ref(); + let n_big = n_int.as_bigint(); + let k_int = k.into_int_ref(); + let k_big = k_int.as_bigint(); + + if n_big.is_negative() { + return Err(vm.new_value_error("n must be a non-negative integer")); + } + if k_big.is_negative() { + return Err(vm.new_value_error("k must be a non-negative integer")); + } + + // Fast path: n fits in i64 + if let Some(ni) = n_big.to_i64() + && ni >= 0 + { + // k overflow or k > n means result is 0 + let ki = match k_big.to_i64() { + Some(k) if k >= 0 && k <= ni => k, + _ => return Ok(BigInt::from(0u8)), + }; + // Apply symmetry: use min(k, n-k) + let ki = ki.min(ni - ki); + if ki > 1 { + let result = pymath::math::integer::comb(ni, ki) + .map_err(|_| vm.new_value_error("comb() error"))?; + return Ok(result.into()); + } + // ki <= 1 cases + if ki == 0 { + return Ok(BigInt::from(1u8)); + } + return Ok(n_big.clone()); // ki == 1 + } + + // BigInt path: n doesn't fit in i64 + // Apply symmetry: k = min(k, n - k) + let n_minus_k = n_big - k_big; + if n_minus_k.is_negative() { + return Ok(BigInt::from(0u8)); + } + let effective_k = if &n_minus_k < k_big { + &n_minus_k + } else { + k_big + }; + + // k must fit in u64 + let ki: u64 = match effective_k.to_u64() { + Some(k) => k, + None => { + return Err( + vm.new_overflow_error(format!("min(n - k, k) must not exceed {}", u64::MAX)) + ); + } + }; + + let result = pymath::math::comb_bigint(n_big, ki); + Ok(result.into()) } } -fn pymath_error_to_exception(err: pymath::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { +pub(crate) fn pymath_exception(err: pymath::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { match err { pymath::Error::EDOM => vm.new_value_error("math domain error"), pymath::Error::ERANGE => vm.new_overflow_error("math range error"), } } + +/// Format a float in Python style (ensures trailing .0 for integers). +fn float_repr(value: f64) -> String { + if value.is_nan() { + "nan".to_owned() + } else if value.is_infinite() { + if value.is_sign_positive() { + "inf".to_owned() + } else { + "-inf".to_owned() + } + } else { + let s = format!("{}", value); + // If no decimal point and not in scientific notation, add .0 + if !s.contains('.') && !s.contains('e') && !s.contains('E') { + format!("{}.0", s) + } else { + s + } + } +} diff --git a/crates/stdlib/src/md5.rs b/crates/stdlib/src/md5.rs index dca48242bbd..2ff6cd24ff7 100644 --- a/crates/stdlib/src/md5.rs +++ b/crates/stdlib/src/md5.rs @@ -1,4 +1,4 @@ -pub(crate) use _md5::make_module; +pub(crate) use _md5::module_def; #[pymodule] mod _md5 { @@ -7,6 +7,6 @@ mod _md5 { #[pyfunction] fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_md5(args).into_pyobject(vm)) + Ok(local_md5(args, vm)?.into_pyobject(vm)) } } diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 5309917a999..6ab898db6b8 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -1,6 +1,6 @@ // spell-checker:disable //! mmap module -pub(crate) use mmap::make_module; +pub(crate) use mmap::module_def; #[pymodule] mod mmap { @@ -21,11 +21,11 @@ mod mmap { sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, }; + use core::ops::{Deref, DerefMut}; use crossbeam_utils::atomic::AtomicCell; use memmap2::{Mmap, MmapMut, MmapOptions}; use num_traits::Signed; use std::io::{self, Write}; - use std::ops::{Deref, DerefMut}; #[cfg(unix)] use nix::{sys::stat::fstat, unistd}; @@ -576,7 +576,7 @@ mod mmap { SetFilePointerEx( duplicated_handle, required_size, - std::ptr::null_mut(), + core::ptr::null_mut(), FILE_BEGIN, ) }; @@ -617,7 +617,7 @@ mod mmap { // Keep the handle alive let raw = owned_handle.as_raw_handle() as isize; - std::mem::forget(owned_handle); + core::mem::forget(owned_handle); (raw, mmap) } else { // Anonymous mapping @@ -651,9 +651,10 @@ mod mmap { impl AsBuffer for PyMmap { fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let readonly = matches!(zelf.access, AccessMode::Read); let buf = PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(zelf.__len__(), true), + BufferDescriptor::simple(zelf.__len__(), readonly), &BUFFER_METHODS, ); @@ -686,7 +687,7 @@ mod mmap { impl AsSequence for PyMmap { fn as_sequence() -> &'static PySequenceMethods { - use std::sync::LazyLock; + use rustpython_common::lock::LazyLock; static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyMmap::sequence_downcast(seq).__len__())), item: atomic_func!(|seq, i, vm| { @@ -730,7 +731,6 @@ mod mmap { .into() } - #[pymethod] fn __len__(&self) -> usize { self.size.load() } @@ -1056,7 +1056,7 @@ mod mmap { // 3. Replace the old mmap let old_size = self.size.load(); - let copy_size = std::cmp::min(old_size, newsize); + let copy_size = core::cmp::min(old_size, newsize); // Create new anonymous mmap let mut new_mmap_opts = MmapOptions::new(); @@ -1088,7 +1088,7 @@ mod mmap { SetFilePointerEx( handle as HANDLE, required_size, - std::ptr::null_mut(), + core::ptr::null_mut(), FILE_BEGIN, ) }; @@ -1240,12 +1240,10 @@ mod mmap { Ok(()) } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { self.getitem_inner(&needle, vm) } - #[pymethod] fn __setitem__( zelf: &Py<Self>, needle: PyObjectRef, @@ -1269,7 +1267,7 @@ mod mmap { #[cfg(windows)] #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + core::mem::size_of::<Self>() } } @@ -1299,7 +1297,7 @@ mod mmap { }; // Don't close the file handle - we're borrowing it - std::mem::forget(file); + core::mem::forget(file); result } diff --git a/crates/stdlib/src/multiprocessing.rs b/crates/stdlib/src/multiprocessing.rs index 9ff2d3dc318..b18cbbb24d9 100644 --- a/crates/stdlib/src/multiprocessing.rs +++ b/crates/stdlib/src/multiprocessing.rs @@ -1,4 +1,4 @@ -pub(crate) use _multiprocessing::make_module; +pub(crate) use _multiprocessing::module_def; #[cfg(windows)] #[pymodule] @@ -41,6 +41,701 @@ mod _multiprocessing { } } -#[cfg(not(windows))] +// Unix platforms (Linux, macOS, etc.) +// macOS has broken sem_timedwait/sem_getvalue - we use polled fallback +#[cfg(unix)] +#[pymodule] +mod _multiprocessing { + use crate::vm::{ + Context, FromArgs, Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyDict, PyType, PyTypeRef}, + function::{FuncArgs, KwArgs}, + types::Constructor, + }; + use alloc::ffi::CString; + use core::sync::atomic::{AtomicI32, AtomicU64, Ordering}; + use libc::sem_t; + use nix::errno::Errno; + + /// Error type for sem_timedwait operations + #[cfg(target_vendor = "apple")] + enum SemWaitError { + Timeout, + SignalException(PyBaseExceptionRef), + OsError(Errno), + } + + /// macOS fallback for sem_timedwait using select + sem_trywait polling + /// Matches sem_timedwait_save in semaphore.c + #[cfg(target_vendor = "apple")] + fn sem_timedwait_polled( + sem: *mut sem_t, + deadline: &libc::timespec, + vm: &VirtualMachine, + ) -> Result<(), SemWaitError> { + let mut delay: u64 = 0; + + loop { + // poll: try to acquire + if unsafe { libc::sem_trywait(sem) } == 0 { + return Ok(()); + } + let err = Errno::last(); + if err != Errno::EAGAIN { + return Err(SemWaitError::OsError(err)); + } + + // get current time + let mut now = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + if unsafe { libc::gettimeofday(&mut now, core::ptr::null_mut()) } < 0 { + return Err(SemWaitError::OsError(Errno::last())); + } + + // check for timeout + let deadline_usec = deadline.tv_sec * 1_000_000 + deadline.tv_nsec / 1000; + #[allow(clippy::unnecessary_cast)] + let now_usec = now.tv_sec as i64 * 1_000_000 + now.tv_usec as i64; + + if now_usec >= deadline_usec { + return Err(SemWaitError::Timeout); + } + + // calculate how much time is left + let difference = (deadline_usec - now_usec) as u64; + + // check delay not too long -- maximum is 20 msecs + delay += 1000; + if delay > 20000 { + delay = 20000; + } + if delay > difference { + delay = difference; + } + + // sleep using select + let mut tv_delay = libc::timeval { + tv_sec: (delay / 1_000_000) as _, + tv_usec: (delay % 1_000_000) as _, + }; + unsafe { + libc::select( + 0, + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + &mut tv_delay, + ) + }; + + // check for signals - preserve the exception (e.g., KeyboardInterrupt) + if let Err(exc) = vm.check_signals() { + return Err(SemWaitError::SignalException(exc)); + } + } + } + + // These match the values in Lib/multiprocessing/synchronize.py + const RECURSIVE_MUTEX: i32 = 0; + const SEMAPHORE: i32 = 1; + + // #define ISMINE(o) (o->count > 0 && PyThread_get_thread_ident() == o->last_tid) + macro_rules! ismine { + ($self:expr) => { + $self.count.load(Ordering::Acquire) > 0 + && $self.last_tid.load(Ordering::Acquire) == current_thread_id() + }; + } + + #[derive(FromArgs)] + struct SemLockNewArgs { + #[pyarg(positional)] + kind: i32, + #[pyarg(positional)] + value: i32, + #[pyarg(positional)] + maxvalue: i32, + #[pyarg(positional)] + name: String, + #[pyarg(positional)] + unlink: bool, + } + + #[pyattr] + #[pyclass(name = "SemLock", module = "_multiprocessing")] + #[derive(Debug, PyPayload)] + struct SemLock { + handle: SemHandle, + kind: i32, + maxvalue: i32, + name: Option<String>, + last_tid: AtomicU64, // unsigned long + count: AtomicI32, // int + } + + #[derive(Debug)] + struct SemHandle { + raw: *mut sem_t, + } + + unsafe impl Send for SemHandle {} + unsafe impl Sync for SemHandle {} + + impl SemHandle { + fn create( + name: &str, + value: u32, + unlink: bool, + vm: &VirtualMachine, + ) -> PyResult<(Self, Option<String>)> { + let cname = semaphore_name(vm, name)?; + // SEM_CREATE(name, val, max) sem_open(name, O_CREAT | O_EXCL, 0600, val) + let raw = unsafe { + libc::sem_open(cname.as_ptr(), libc::O_CREAT | libc::O_EXCL, 0o600, value) + }; + if raw == libc::SEM_FAILED { + let err = Errno::last(); + return Err(os_error(vm, err)); + } + if unlink { + // SEM_UNLINK(name) sem_unlink(name) + unsafe { + libc::sem_unlink(cname.as_ptr()); + } + Ok((SemHandle { raw }, None)) + } else { + Ok((SemHandle { raw }, Some(name.to_owned()))) + } + } + + fn open_existing(name: &str, vm: &VirtualMachine) -> PyResult<Self> { + let cname = semaphore_name(vm, name)?; + let raw = unsafe { libc::sem_open(cname.as_ptr(), 0) }; + if raw == libc::SEM_FAILED { + let err = Errno::last(); + return Err(os_error(vm, err)); + } + Ok(SemHandle { raw }) + } + + #[inline] + fn as_ptr(&self) -> *mut sem_t { + self.raw + } + } + + impl Drop for SemHandle { + fn drop(&mut self) { + // Guard against default/uninitialized state. + // Note: SEM_FAILED is (sem_t*)-1, not null, but valid handles are never null + // and SEM_FAILED is never stored (error is returned immediately on sem_open failure). + if !self.raw.is_null() { + // SEM_CLOSE(sem) sem_close(sem) + unsafe { + libc::sem_close(self.raw); + } + } + } + } + + #[pyclass(with(Constructor), flags(BASETYPE))] + impl SemLock { + #[pygetset] + fn handle(&self) -> isize { + self.handle.as_ptr() as isize + } + + #[pygetset] + fn kind(&self) -> i32 { + self.kind + } + + #[pygetset] + fn maxvalue(&self) -> i32 { + self.maxvalue + } + + #[pygetset] + fn name(&self) -> Option<String> { + self.name.clone() + } + + /// Acquire the semaphore/lock. + // _multiprocessing_SemLock_acquire_impl + #[pymethod] + fn acquire(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<bool> { + // block=True, timeout=None + + let blocking: bool = args + .kwargs + .get("block") + .or_else(|| args.args.first()) + .map(|o| o.clone().try_to_bool(vm)) + .transpose()? + .unwrap_or(true); + + let timeout_obj = args + .kwargs + .get("timeout") + .or_else(|| args.args.get(1)) + .cloned(); + + if self.kind == RECURSIVE_MUTEX && ismine!(self) { + self.count.fetch_add(1, Ordering::Release); + return Ok(true); + } + + // timeout_obj != Py_None + let use_deadline = timeout_obj.as_ref().is_some_and(|o| !vm.is_none(o)); + + let deadline = if use_deadline { + let timeout_obj = timeout_obj.unwrap(); + // This accepts both int and float, converting to f64 + let timeout: f64 = timeout_obj.try_float(vm)?.to_f64(); + let timeout = if timeout < 0.0 { 0.0 } else { timeout }; + + let mut tv = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + let res = unsafe { libc::gettimeofday(&mut tv, core::ptr::null_mut()) }; + if res < 0 { + return Err(vm.new_os_error("gettimeofday failed".to_string())); + } + + // deadline calculation: + // long sec = (long) timeout; + // long nsec = (long) (1e9 * (timeout - sec) + 0.5); + // deadline.tv_sec = now.tv_sec + sec; + // deadline.tv_nsec = now.tv_usec * 1000 + nsec; + // deadline.tv_sec += (deadline.tv_nsec / 1000000000); + // deadline.tv_nsec %= 1000000000; + let sec = timeout as libc::c_long; + let nsec = (1e9 * (timeout - sec as f64) + 0.5) as libc::c_long; + let mut deadline = libc::timespec { + tv_sec: tv.tv_sec + sec as libc::time_t, + tv_nsec: (tv.tv_usec as libc::c_long * 1000 + nsec) as _, + }; + deadline.tv_sec += (deadline.tv_nsec / 1_000_000_000) as libc::time_t; + deadline.tv_nsec %= 1_000_000_000; + Some(deadline) + } else { + None + }; + + // Check whether we can acquire without releasing the GIL and blocking + let mut res; + loop { + res = unsafe { libc::sem_trywait(self.handle.as_ptr()) }; + if res >= 0 { + break; + } + let err = Errno::last(); + if err == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + + // if (res < 0 && errno == EAGAIN && blocking) + if res < 0 && Errno::last() == Errno::EAGAIN && blocking { + // Couldn't acquire immediately, need to block + #[cfg(not(target_vendor = "apple"))] + { + loop { + // Py_BEGIN_ALLOW_THREADS / Py_END_ALLOW_THREADS + // RustPython doesn't have GIL, so we just do the wait + if let Some(ref dl) = deadline { + res = unsafe { libc::sem_timedwait(self.handle.as_ptr(), dl) }; + } else { + res = unsafe { libc::sem_wait(self.handle.as_ptr()) }; + } + + if res >= 0 { + break; + } + let err = Errno::last(); + if err == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + } + #[cfg(target_vendor = "apple")] + { + // macOS: use polled fallback since sem_timedwait is not available + if let Some(ref dl) = deadline { + match sem_timedwait_polled(self.handle.as_ptr(), dl, vm) { + Ok(()) => res = 0, + Err(SemWaitError::Timeout) => { + // Timeout occurred - return false directly + return Ok(false); + } + Err(SemWaitError::SignalException(exc)) => { + // Propagate the original exception (e.g., KeyboardInterrupt) + return Err(exc); + } + Err(SemWaitError::OsError(e)) => { + return Err(os_error(vm, e)); + } + } + } else { + // No timeout: use sem_wait (available on macOS) + loop { + res = unsafe { libc::sem_wait(self.handle.as_ptr()) }; + if res >= 0 { + break; + } + let err = Errno::last(); + if err == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + } + } + } + + // result handling: + if res < 0 { + let err = Errno::last(); + match err { + Errno::EAGAIN | Errno::ETIMEDOUT => return Ok(false), + Errno::EINTR => { + // EINTR should be handled by the check_signals() loop above + // If we reach here, check signals again and propagate any exception + return vm.check_signals().map(|_| false); + } + _ => return Err(os_error(vm, err)), + } + } + + self.count.fetch_add(1, Ordering::Release); + self.last_tid.store(current_thread_id(), Ordering::Release); + + Ok(true) + } + + /// Release the semaphore/lock. + // _multiprocessing_SemLock_release_impl + #[pymethod] + fn release(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.kind == RECURSIVE_MUTEX { + // if (!ISMINE(self)) + if !ismine!(self) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.assertion_error.to_owned(), + "attempt to release recursive lock not owned by thread".to_owned(), + )); + } + // if (self->count > 1) { --self->count; Py_RETURN_NONE; } + if self.count.load(Ordering::Acquire) > 1 { + self.count.fetch_sub(1, Ordering::Release); + return Ok(()); + } + // assert(self->count == 1); + } else { + // SEMAPHORE case: check value before releasing + #[cfg(not(target_vendor = "apple"))] + { + // Linux: use sem_getvalue + let mut sval: libc::c_int = 0; + let res = unsafe { libc::sem_getvalue(self.handle.as_ptr(), &mut sval) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + if sval >= self.maxvalue { + return Err(vm.new_value_error( + "semaphore or lock released too many times".to_owned(), + )); + } + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE + // We will only check properly the maxvalue == 1 case + if self.maxvalue == 1 { + // make sure that already locked + if unsafe { libc::sem_trywait(self.handle.as_ptr()) } < 0 { + if Errno::last() != Errno::EAGAIN { + return Err(os_error(vm, Errno::last())); + } + // it is already locked as expected + } else { + // it was not locked so undo wait and raise + if unsafe { libc::sem_post(self.handle.as_ptr()) } < 0 { + return Err(os_error(vm, Errno::last())); + } + return Err(vm.new_value_error( + "semaphore or lock released too many times".to_owned(), + )); + } + } + } + } + + let res = unsafe { libc::sem_post(self.handle.as_ptr()) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + + self.count.fetch_sub(1, Ordering::Release); + Ok(()) + } + + /// Enter the semaphore/lock (context manager). + // _multiprocessing_SemLock___enter___impl + #[pymethod(name = "__enter__")] + fn enter(&self, vm: &VirtualMachine) -> PyResult<bool> { + // return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); + self.acquire( + FuncArgs::new::<Vec<_>, KwArgs>( + vec![vm.ctx.new_bool(true).into()], + KwArgs::default(), + ), + vm, + ) + } + + /// Exit the semaphore/lock (context manager). + // _multiprocessing_SemLock___exit___impl + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.release(vm) + } + + /// Rebuild a SemLock from pickled state. + // _multiprocessing_SemLock__rebuild_impl + #[pyclassmethod(name = "_rebuild")] + fn rebuild( + cls: PyTypeRef, + _handle: isize, + kind: i32, + maxvalue: i32, + name: Option<String>, + vm: &VirtualMachine, + ) -> PyResult { + let Some(ref name_str) = name else { + return Err(vm.new_value_error("cannot rebuild SemLock without name".to_owned())); + }; + let handle = SemHandle::open_existing(name_str, vm)?; + // return newsemlockobject(type, handle, kind, maxvalue, name_copy); + let zelf = SemLock { + handle, + kind, + maxvalue, + name, + last_tid: AtomicU64::new(0), + count: AtomicI32::new(0), + }; + zelf.into_ref_with_type(vm, cls).map(Into::into) + } + + /// Rezero the net acquisition count after fork(). + // _multiprocessing_SemLock__after_fork_impl + #[pymethod] + fn _after_fork(&self) { + self.count.store(0, Ordering::Release); + // Also reset last_tid for safety + self.last_tid.store(0, Ordering::Release); + } + + /// SemLock objects cannot be pickled directly. + /// Use multiprocessing.synchronize.SemLock wrapper which handles pickling. + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'SemLock' object".to_owned())) + } + + /// Num of `acquire()`s minus num of `release()`s for this process. + // _multiprocessing_SemLock__count_impl + #[pymethod] + fn _count(&self) -> i32 { + self.count.load(Ordering::Acquire) + } + + /// Whether the lock is owned by this thread. + // _multiprocessing_SemLock__is_mine_impl + #[pymethod] + fn _is_mine(&self) -> bool { + ismine!(self) + } + + /// Get the value of the semaphore. + // _multiprocessing_SemLock__get_value_impl + #[pymethod] + fn _get_value(&self, vm: &VirtualMachine) -> PyResult<i32> { + #[cfg(not(target_vendor = "apple"))] + { + // Linux: use sem_getvalue + let mut sval: libc::c_int = 0; + let res = unsafe { libc::sem_getvalue(self.handle.as_ptr(), &mut sval) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + // some posix implementations use negative numbers to indicate + // the number of waiting threads + Ok(if sval < 0 { 0 } else { sval }) + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE - raise NotImplementedError + Err(vm.new_not_implemented_error(String::new())) + } + } + + /// Return whether semaphore has value zero. + // _multiprocessing_SemLock__is_zero_impl + #[pymethod] + fn _is_zero(&self, vm: &VirtualMachine) -> PyResult<bool> { + #[cfg(not(target_vendor = "apple"))] + { + Ok(self._get_value(vm)? == 0) + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE + // Try to acquire - if EAGAIN, value is 0 + if unsafe { libc::sem_trywait(self.handle.as_ptr()) } < 0 { + if Errno::last() == Errno::EAGAIN { + return Ok(true); + } + return Err(os_error(vm, Errno::last())); + } + // Successfully acquired - undo and return false + if unsafe { libc::sem_post(self.handle.as_ptr()) } < 0 { + return Err(os_error(vm, Errno::last())); + } + Ok(false) + } + } + + #[extend_class] + fn extend_class(ctx: &Context, class: &Py<PyType>) { + class.set_attr( + ctx.intern_str("RECURSIVE_MUTEX"), + ctx.new_int(RECURSIVE_MUTEX).into(), + ); + class.set_attr(ctx.intern_str("SEMAPHORE"), ctx.new_int(SEMAPHORE).into()); + // SEM_VALUE_MAX from system, or INT_MAX if negative + // We use a reasonable default + let sem_value_max: i32 = unsafe { + let val = libc::sysconf(libc::_SC_SEM_VALUE_MAX); + if val < 0 || val > i32::MAX as libc::c_long { + i32::MAX + } else { + val as i32 + } + }; + class.set_attr( + ctx.intern_str("SEM_VALUE_MAX"), + ctx.new_int(sem_value_max).into(), + ); + } + } + + impl Constructor for SemLock { + type Args = SemLockNewArgs; + + // Create a new SemLock. + // _multiprocessing_SemLock_impl + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if args.kind != RECURSIVE_MUTEX && args.kind != SEMAPHORE { + return Err(vm.new_value_error("unrecognized kind".to_owned())); + } + // Value validation + if args.value < 0 || args.value > args.maxvalue { + return Err(vm.new_value_error("invalid value".to_owned())); + } + + let value = args.value as u32; + let (handle, name) = SemHandle::create(&args.name, value, args.unlink, vm)?; + + // return newsemlockobject(type, handle, kind, maxvalue, name_copy); + Ok(SemLock { + handle, + kind: args.kind, + maxvalue: args.maxvalue, + name, + last_tid: AtomicU64::new(0), + count: AtomicI32::new(0), + }) + } + } + + /// Function to unlink semaphore names. + // _PyMp_sem_unlink. + #[pyfunction] + fn sem_unlink(name: String, vm: &VirtualMachine) -> PyResult<()> { + let cname = semaphore_name(vm, &name)?; + let res = unsafe { libc::sem_unlink(cname.as_ptr()) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + Ok(()) + } + + /// Module-level flags dict. + #[pyattr] + fn flags(vm: &VirtualMachine) -> PyRef<PyDict> { + let flags = vm.ctx.new_dict(); + // HAVE_SEM_OPEN is always 1 on Unix (we wouldn't be here otherwise) + flags + .set_item("HAVE_SEM_OPEN", vm.ctx.new_int(1).into(), vm) + .unwrap(); + + #[cfg(not(target_vendor = "apple"))] + { + // Linux: HAVE_SEM_TIMEDWAIT is available + flags + .set_item("HAVE_SEM_TIMEDWAIT", vm.ctx.new_int(1).into(), vm) + .unwrap(); + } + + #[cfg(target_vendor = "apple")] + { + // macOS: sem_getvalue is broken + flags + .set_item("HAVE_BROKEN_SEM_GETVALUE", vm.ctx.new_int(1).into(), vm) + .unwrap(); + } + + flags + } + + fn semaphore_name(vm: &VirtualMachine, name: &str) -> PyResult<CString> { + // POSIX semaphore names must start with / + let mut full = String::with_capacity(name.len() + 1); + if !name.starts_with('/') { + full.push('/'); + } + full.push_str(name); + CString::new(full).map_err(|_| vm.new_value_error("embedded null character".to_owned())) + } + + fn os_error(vm: &VirtualMachine, err: Errno) -> PyBaseExceptionRef { + // _PyMp_SetError maps to PyErr_SetFromErrno + let exc_type = match err { + Errno::EEXIST => vm.ctx.exceptions.file_exists_error.to_owned(), + Errno::ENOENT => vm.ctx.exceptions.file_not_found_error.to_owned(), + _ => vm.ctx.exceptions.os_error.to_owned(), + }; + vm.new_os_subtype_error(exc_type, Some(err as i32), err.desc().to_owned()) + .upcast() + } + + /// Get current thread identifier. + /// PyThread_get_thread_ident on Unix (pthread_self). + fn current_thread_id() -> u64 { + unsafe { libc::pthread_self() as u64 } + } +} + +#[cfg(all(not(unix), not(windows)))] #[pymodule] mod _multiprocessing {} diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index ea67d605f76..44d690f9190 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -2,6 +2,10 @@ mod cert; +// SSL exception types (shared with rustls backend) +#[path = "ssl/error.rs"] +mod ssl_error; + // Conditional compilation for OpenSSL version-specific error codes cfg_if::cfg_if! { if #[cfg(ossl310)] { @@ -19,51 +23,57 @@ cfg_if::cfg_if! { } } -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; -use openssl_probe::ProbeResult; +pub(crate) use _ssl::module_def; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - // if openssl is vendored, it doesn't know the locations - // of system certificates - cache the probe result now. - #[cfg(openssl_vendored)] - LazyLock::force(&PROBE); - _ssl::make_module(vm) -} +use openssl_probe::ProbeResult; +use rustpython_common::lock::LazyLock; // define our own copy of ProbeResult so we can handle the vendor case // easily, without having to have a bunch of cfgs cfg_if::cfg_if! { if #[cfg(openssl_vendored)] { - use std::sync::LazyLock; static PROBE: LazyLock<ProbeResult> = LazyLock::new(openssl_probe::probe); - fn probe() -> &'static ProbeResult { &PROBE } } else { - fn probe() -> &'static ProbeResult { - &ProbeResult { cert_file: None, cert_dir: None } - } + static PROBE: LazyLock<ProbeResult> = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] }); } } +fn probe() -> &'static ProbeResult { + &PROBE +} + #[allow(non_upper_case_globals)] -#[pymodule(with(cert::ssl_cert, ossl101, ossl111, windows))] +#[pymodule(with( + cert::ssl_cert, + ssl_error::ssl_error, + #[cfg(ossl101)] ossl101, + #[cfg(ossl111)] ossl111, + #[cfg(windows)] windows))] mod _ssl { use super::{bio, probe}; + + // Import error types and helpers used in this module (others are exposed via pymodule(with(...))) + use super::ssl_error::{ + PySSLCertVerificationError, PySSLError, create_ssl_eof_error, create_ssl_want_read_error, + create_ssl_want_write_error, + }; use crate::{ common::lock::{ PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, }, socket::{self, PySocket}, vm::{ - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseExceptionRef, PyBytesRef, PyListRef, PyOSError, PyStrRef, PyTypeRef, PyWeak, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyModule, PyStrRef, + PyType, PyWeak, }, class_or_notimplemented, convert::ToPyException, exceptions, function::{ - ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, - OptionalArg, PyComparisonValue, + ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, OptionalArg, + PyComparisonValue, }, types::{Comparable, Constructor, PyComparisonOp}, utils::ToCString, @@ -85,13 +95,22 @@ mod _ssl { fmt, io::{Read, Write}, path::{Path, PathBuf}, - sync::LazyLock, time::Instant, }; // Import certificate types from parent module use super::cert::{self, cert_to_certificate, cert_to_py}; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // if openssl is vendored, it doesn't know the locations + // of system certificates - cache the probe result now. + #[cfg(openssl_vendored)] + rustpython_common::lock::LazyLock::force(&super::PROBE); + + __module_exec(vm, module); + Ok(()) + } + // Re-export PySSLCertificate to make it available in the _ssl module // It will be automatically exposed to Python via #[pyclass] #[allow(unused_imports)] @@ -237,8 +256,6 @@ mod _ssl { #[pyattr] const VERIFY_DEFAULT: u32 = 0; #[pyattr] - const SSL_ERROR_EOF: u32 = 8; // custom for python - #[pyattr] const HAS_SNI: bool = true; #[pyattr] const HAS_ECDH: bool = true; @@ -299,85 +316,6 @@ mod _ssl { parse_version_info(openssl_api_version) } - // SSL Exception Types - - /// An error occurred in the SSL implementation. - #[pyattr] - #[pyexception(name = "SSLError", base = PyOSError)] - #[derive(Debug)] - pub struct PySslError {} - - #[pyexception] - impl PySslError { - // Returns strerror attribute if available, otherwise str(args) - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - // Try to get strerror attribute first (OSError compatibility) - if let Ok(strerror) = exc.as_object().get_attr("strerror", vm) - && !vm.is_none(&strerror) - { - return strerror.str(vm); - } - - // Otherwise return str(args) - exc.args().as_object().str(vm) - } - } - - /// A certificate could not be verified. - #[pyattr] - #[pyexception(name = "SSLCertVerificationError", base = PySslError)] - #[derive(Debug)] - pub struct PySslCertVerificationError {} - - #[pyexception] - impl PySslCertVerificationError {} - - /// SSL/TLS session closed cleanly. - #[pyattr] - #[pyexception(name = "SSLZeroReturnError", base = PySslError)] - #[derive(Debug)] - pub struct PySslZeroReturnError {} - - #[pyexception] - impl PySslZeroReturnError {} - - /// Non-blocking SSL socket needs to read more data. - #[pyattr] - #[pyexception(name = "SSLWantReadError", base = PySslError)] - #[derive(Debug)] - pub struct PySslWantReadError {} - - #[pyexception] - impl PySslWantReadError {} - - /// Non-blocking SSL socket needs to write more data. - #[pyattr] - #[pyexception(name = "SSLWantWriteError", base = PySslError)] - #[derive(Debug)] - pub struct PySslWantWriteError {} - - #[pyexception] - impl PySslWantWriteError {} - - /// System error when attempting SSL operation. - #[pyattr] - #[pyexception(name = "SSLSyscallError", base = PySslError)] - #[derive(Debug)] - pub struct PySslSyscallError {} - - #[pyexception] - impl PySslSyscallError {} - - /// SSL/TLS connection terminated abruptly. - #[pyattr] - #[pyexception(name = "SSLEOFError", base = PySslError)] - #[derive(Debug)] - pub struct PySslEOFError {} - - #[pyexception] - impl PySslEOFError {} - type OpensslVersionInfo = (u8, u8, u8, u8, u8); const fn parse_version_info(mut n: i64) -> OpensslVersionInfo { let status = (n & 0xF) as u8; @@ -516,7 +454,7 @@ mod _ssl { }); let cert_dir = probe .cert_dir - .as_ref() + .first() .map(PathBuf::from) .unwrap_or_else(|| { path_from_cstr(unsafe { CStr::from_ptr(sys::X509_get_default_cert_dir()) }) @@ -581,21 +519,56 @@ mod _ssl { Ok(buf) } - // Callback data stored in SSL context for SNI + // Callback data stored in SSL ex_data for SNI/msg callbacks struct SniCallbackData { ssl_context: PyRef<PySslContext>, - vm_ptr: *const VirtualMachine, + // Use weak reference to avoid reference cycle: + // PySslSocket -> SslStream -> SSL -> ex_data -> SniCallbackData -> PySslSocket + ssl_socket_weak: PyRef<PyWeak>, + } + + // Thread-local storage for VirtualMachine pointer during handshake + // SNI callback is only called during handshake which is synchronous + thread_local! { + static HANDSHAKE_VM: core::cell::Cell<Option<*const VirtualMachine>> = const { core::cell::Cell::new(None) }; + // SSL pointer during handshake - needed because connection lock is held during handshake + // and callbacks may need to access SSL without acquiring the lock + static HANDSHAKE_SSL_PTR: core::cell::Cell<Option<*mut sys::SSL>> = const { core::cell::Cell::new(None) }; } - impl Drop for SniCallbackData { + // RAII guard to set/clear thread-local handshake context + struct HandshakeVmGuard { + _ssl_ptr: *mut sys::SSL, + } + + impl HandshakeVmGuard { + fn new(vm: &VirtualMachine, ssl_ptr: *mut sys::SSL) -> Self { + HANDSHAKE_VM.with(|cell| cell.set(Some(vm as *const _))); + HANDSHAKE_SSL_PTR.with(|cell| cell.set(Some(ssl_ptr))); + HandshakeVmGuard { _ssl_ptr: ssl_ptr } + } + } + + impl Drop for HandshakeVmGuard { fn drop(&mut self) { - // PyRef will handle reference counting + HANDSHAKE_VM.with(|cell| cell.set(None)); + HANDSHAKE_SSL_PTR.with(|cell| cell.set(None)); + } + } + + // Get SSL pointer - either from thread-local (during handshake) or from connection + fn get_ssl_ptr_for_context_change(connection: &PyRwLock<SslConnection>) -> *mut sys::SSL { + // First check if we're in a handshake callback (lock already held) + if let Some(ptr) = HANDSHAKE_SSL_PTR.with(|cell| cell.get()) { + return ptr; } + // Otherwise, acquire the lock normally + connection.read().ssl().as_ptr() } // Get or create an ex_data index for SNI callback data fn get_sni_ex_data_index() -> libc::c_int { - use std::sync::LazyLock; + use rustpython_common::lock::LazyLock; static SNI_EX_DATA_IDX: LazyLock<libc::c_int> = LazyLock::new(|| unsafe { sys::SSL_get_ex_new_index( 0, @@ -609,7 +582,51 @@ mod _ssl { } // Free function for callback data + // NOTE: We don't free the data here because it's managed manually in do_handshake + // to avoid use-after-free when the SSL object is dropped after timeout unsafe extern "C" fn sni_callback_data_free( + _parent: *mut libc::c_void, + _ptr: *mut libc::c_void, + _ad: *mut sys::CRYPTO_EX_DATA, + _idx: libc::c_int, + _argl: libc::c_long, + _argp: *mut libc::c_void, + ) { + // Intentionally empty - data is freed in cleanup_sni_ex_data() + } + + // Clean up SNI callback data from SSL ex_data + // Called after handshake to free the data and release references + unsafe fn cleanup_sni_ex_data(ssl_ptr: *mut sys::SSL) { + unsafe { + let idx = get_sni_ex_data_index(); + let data_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); + if !data_ptr.is_null() { + // Free the Box<SniCallbackData> - this releases references to context and socket + let _ = Box::from_raw(data_ptr as *mut SniCallbackData); + // Clear the ex_data to prevent double-free + sys::SSL_set_ex_data(ssl_ptr, idx, std::ptr::null_mut()); + } + } + } + + // Get or create an ex_data index for msg_callback data + fn get_msg_callback_ex_data_index() -> libc::c_int { + use rustpython_common::lock::LazyLock; + static MSG_CB_EX_DATA_IDX: LazyLock<libc::c_int> = LazyLock::new(|| unsafe { + sys::SSL_get_ex_new_index( + 0, + std::ptr::null_mut(), + None, + None, + Some(msg_callback_data_free), + ) + }); + *MSG_CB_EX_DATA_IDX + } + + // Free function for msg_callback data - called by OpenSSL when SSL is freed + unsafe extern "C" fn msg_callback_data_free( _parent: *mut libc::c_void, ptr: *mut libc::c_void, _ad: *mut sys::CRYPTO_EX_DATA, @@ -619,7 +636,9 @@ mod _ssl { ) { if !ptr.is_null() { unsafe { - let _ = Box::from_raw(ptr as *mut SniCallbackData); + // Reconstruct PyObjectRef and drop to decrement reference count + let raw = std::ptr::NonNull::new_unchecked(ptr as *mut PyObject); + let _ = PyObjectRef::from_raw(raw); } } } @@ -657,9 +676,13 @@ mod _ssl { let callback_data = &*(data_ptr as *const SniCallbackData); - // SAFETY: vm_ptr is stored during wrap_socket and is valid for the lifetime - // of the SSL connection. The handshake happens synchronously in the same thread. - let vm = &*callback_data.vm_ptr; + // Get VM from thread-local storage (set by HandshakeVmGuard in do_handshake) + let Some(vm_ptr) = HANDSHAKE_VM.with(|cell| cell.get()) else { + // VM not available - this shouldn't happen during handshake + *al = SSL_AD_INTERNAL_ERROR; + return SSL_TLSEXT_ERR_ALERT_FATAL; + }; + let vm = &*vm_ptr; // Get server name let servername = sys::SSL_get_servername(ssl_ptr, TLSEXT_NAMETYPE_host_name); @@ -673,20 +696,11 @@ mod _ssl { } }; - // Get SSL socket from SSL ex_data (stored as PySslSocket pointer) - let ssl_socket_ptr = sys::SSL_get_ex_data(ssl_ptr, 0); // Index 0 for SSL socket - let ssl_socket_obj = if !ssl_socket_ptr.is_null() { - let ssl_socket = &*(ssl_socket_ptr as *const PySslSocket); - // Try to get owner first - ssl_socket - .owner - .read() - .as_ref() - .and_then(|weak| weak.upgrade()) - .unwrap_or_else(|| vm.ctx.none()) - } else { - vm.ctx.none() - }; + // Get SSL socket from callback data via weak reference + let ssl_socket_obj = callback_data + .ssl_socket_weak + .upgrade() + .unwrap_or_else(|| vm.ctx.none()); // Call the Python callback match callback.call( @@ -733,11 +747,21 @@ mod _ssl { } } + // OpenSSL record type constants for msg_callback + const SSL3_RT_CHANGE_CIPHER_SPEC: i32 = 20; + const SSL3_RT_ALERT: i32 = 21; + const SSL3_RT_HANDSHAKE: i32 = 22; + const SSL3_RT_HEADER: i32 = 256; + const SSL3_RT_INNER_CONTENT_TYPE: i32 = 257; + // Special value for change cipher spec (CPython compatibility) + const SSL3_MT_CHANGE_CIPHER_SPEC: i32 = 0x0101; + // Message callback function called by OpenSSL - // Based on CPython's _PySSL_msg_callback in Modules/_ssl/debughelpers.c + // Called during SSL operations to report protocol messages. + // debughelpers.c:_PySSL_msg_callback unsafe extern "C" fn _msg_callback( write_p: libc::c_int, - version: libc::c_int, + mut version: libc::c_int, content_type: libc::c_int, buf: *const libc::c_void, len: usize, @@ -749,13 +773,15 @@ mod _ssl { } unsafe { - // Get SSL socket from SSL_get_app_data (index 0) - let ssl_socket_ptr = sys::SSL_get_ex_data(ssl_ptr, 0); + // Get SSL socket from ex_data using the dedicated index + let idx = get_msg_callback_ex_data_index(); + let ssl_socket_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); if ssl_socket_ptr.is_null() { return; } - let ssl_socket = &*(ssl_socket_ptr as *const PySslSocket); + // ssl_socket_ptr is a pointer to Box<Py<PySslSocket>>, set in _wrap_socket/_wrap_bio + let ssl_socket: &Py<PySslSocket> = &*(ssl_socket_ptr as *const Py<PySslSocket>); // Get the callback from the context let callback_opt = ssl_socket.ctx.read().msg_callback.lock().clone(); @@ -763,15 +789,12 @@ mod _ssl { return; }; - // Get callback data from SSL ex_data (for VM) - let idx = get_sni_ex_data_index(); - let data_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); - if data_ptr.is_null() { + // Get VM from thread-local storage (set by HandshakeVmGuard in do_handshake) + let Some(vm_ptr) = HANDSHAKE_VM.with(|cell| cell.get()) else { + // VM not available - this shouldn't happen during handshake return; - } - - let callback_data = &*(data_ptr as *const SniCallbackData); - let vm = &*callback_data.vm_ptr; + }; + let vm = &*vm_ptr; // Get SSL socket owner object let ssl_socket_obj = ssl_socket @@ -788,16 +811,50 @@ mod _ssl { // Determine direction string let direction_str = if write_p != 0 { "write" } else { "read" }; + // Calculate msg_type based on content_type (debughelpers.c behavior) + let msg_type = match content_type { + SSL3_RT_CHANGE_CIPHER_SPEC => SSL3_MT_CHANGE_CIPHER_SPEC, + SSL3_RT_ALERT => { + // byte 1 is alert type + if len >= 2 { buf_slice[1] as i32 } else { -1 } + } + SSL3_RT_HANDSHAKE => { + // byte 0 is handshake type + if !buf_slice.is_empty() { + buf_slice[0] as i32 + } else { + -1 + } + } + SSL3_RT_HEADER => { + // Frame header: version in bytes 1..2, type in byte 0 + if len >= 3 { + version = ((buf_slice[1] as i32) << 8) | (buf_slice[2] as i32); + buf_slice[0] as i32 + } else { + -1 + } + } + SSL3_RT_INNER_CONTENT_TYPE => { + // Inner content type in byte 0 + if !buf_slice.is_empty() { + buf_slice[0] as i32 + } else { + -1 + } + } + _ => -1, + }; + // Call the Python callback // Signature: callback(conn, direction, version, content_type, msg_type, data) - // For simplicity, we'll pass msg_type as 0 (would need more parsing to get the actual type) match callback.call( ( ssl_socket_obj, vm.ctx.new_str(direction_str), vm.ctx.new_int(version), vm.ctx.new_int(content_type), - vm.ctx.new_int(0), // msg_type - would need parsing + vm.ctx.new_int(msg_type), msg_bytes, ), vm, @@ -834,6 +891,9 @@ mod _ssl { post_handshake_auth: PyMutex<bool>, sni_callback: PyMutex<Option<PyObjectRef>>, msg_callback: PyMutex<Option<PyObjectRef>>, + psk_client_callback: PyMutex<Option<PyObjectRef>>, + psk_server_callback: PyMutex<Option<PyObjectRef>>, + psk_identity_hint: PyMutex<Option<String>>, } impl fmt::Debug for PySslContext { @@ -849,7 +909,11 @@ mod _ssl { impl Constructor for PySslContext { type Args = i32; - fn py_new(cls: PyTypeRef, proto_version: Self::Args, vm: &VirtualMachine) -> PyResult { + fn py_new( + _cls: &Py<PyType>, + proto_version: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { let proto = SslVersion::try_from(proto_version) .map_err(|_| vm.new_value_error("invalid protocol version"))?; let method = match proto { @@ -875,6 +939,7 @@ mod _ssl { SslVerifyMode::NONE }); + // Start with OP_ALL but remove options that CPython doesn't include by default let mut options = SslOptions::ALL & !SslOptions::DONT_INSERT_EMPTY_FRAGMENTS; if proto != SslVersion::Ssl2 { options |= SslOptions::NO_SSLV2; @@ -888,6 +953,8 @@ mod _ssl { options |= SslOptions::SINGLE_ECDH_USE; options |= SslOptions::ENABLE_MIDDLEBOX_COMPAT; builder.set_options(options); + // Remove NO_TLSv1 and NO_TLSv1_1 which newer OpenSSL adds to OP_ALL + builder.clear_options(SslOptions::NO_TLSV1 | SslOptions::NO_TLSV1_1); let mode = ssl::SslMode::ACCEPT_MOVING_WRITE_BUFFER | ssl::SslMode::AUTO_RETRY; builder.set_mode(mode); @@ -931,16 +998,17 @@ mod _ssl { sys::X509_VERIFY_PARAM_set_flags(param, sys::X509_V_FLAG_TRUSTED_FIRST); } - PySslContext { + Ok(PySslContext { ctx: PyRwLock::new(builder), check_hostname: AtomicCell::new(check_hostname), protocol: proto, post_handshake_auth: PyMutex::new(false), sni_callback: PyMutex::new(None), msg_callback: PyMutex::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into) + psk_client_callback: PyMutex::new(None), + psk_server_callback: PyMutex::new(None), + psk_identity_hint: PyMutex::new(None), + }) } } @@ -980,12 +1048,9 @@ mod _ssl { if ciphers.contains('\0') { return Err(exceptions::cstring_error(vm)); } - self.builder().set_cipher_list(ciphers).map_err(|_| { - vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "No cipher can be selected.".to_owned(), - ) - }) + self.builder() + .set_cipher_list(ciphers) + .map_err(|_| new_ssl_error(vm, "No cipher can be selected.")) } #[pymethod] @@ -1066,9 +1131,26 @@ mod _ssl { self.ctx.read().options().bits() as _ } #[pygetset(setter)] - fn set_options(&self, opts: libc::c_ulong) { - self.builder() - .set_options(SslOptions::from_bits_truncate(opts as _)); + fn set_options(&self, new_opts: libc::c_ulong) { + let mut ctx = self.builder(); + // Get current options + let current = ctx.options().bits() as libc::c_ulong; + + // Calculate options to clear and set + let clear = current & !new_opts; + let set = !current & new_opts; + + // Clear options first (using raw FFI since openssl crate doesn't expose clear_options) + if clear != 0 { + unsafe { + sys::SSL_CTX_clear_options(ctx.as_ptr(), clear); + } + } + + // Then set new options + if set != 0 { + ctx.set_options(SslOptions::from_bits_truncate(set as _)); + } } #[pygetset] fn protocol(&self) -> i32 { @@ -1125,16 +1207,10 @@ mod _ssl { let set = !flags & new_flags; if clear != 0 && sys::X509_VERIFY_PARAM_clear_flags(param, clear) == 0 { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "Failed to clear verify flags".to_owned(), - )); + return Err(new_ssl_error(vm, "Failed to clear verify flags")); } if set != 0 && sys::X509_VERIFY_PARAM_set_flags(param, set) == 0 { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "Failed to set verify flags".to_owned(), - )); + return Err(new_ssl_error(vm, "Failed to set verify flags")); } Ok(()) } @@ -1301,7 +1377,7 @@ mod _ssl { ssl::select_next_proto(&server, client).ok_or(ssl::AlpnError::NOACK)?; let pos = memchr::memmem::find(client, proto) .expect("selected alpn proto should be present in client protos"); - Ok(&client[pos..proto.len()]) + Ok(&client[pos..pos + proto.len()]) }); Ok(()) } @@ -1313,6 +1389,78 @@ mod _ssl { } } + #[pymethod] + fn set_psk_client_callback( + &self, + callback: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Cannot add PSK client callback to a server context + if self.protocol == SslVersion::TlsServer { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot add PSK client callback to a PROTOCOL_TLS_SERVER context" + .to_owned(), + ) + .upcast()); + } + + if vm.is_none(&callback) { + *self.psk_client_callback.lock() = None; + unsafe { + sys::SSL_CTX_set_psk_client_callback(self.builder().as_ptr(), None); + } + } else { + if !callback.is_callable() { + return Err(vm.new_type_error("callback must be callable".to_owned())); + } + *self.psk_client_callback.lock() = Some(callback); + // Note: The actual callback will be invoked via SSL app_data mechanism + // when do_handshake is called + } + Ok(()) + } + + #[pymethod] + fn set_psk_server_callback( + &self, + callback: PyObjectRef, + identity_hint: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Cannot add PSK server callback to a client context + if self.protocol == SslVersion::TlsClient { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot add PSK server callback to a PROTOCOL_TLS_CLIENT context" + .to_owned(), + ) + .upcast()); + } + + if vm.is_none(&callback) { + *self.psk_server_callback.lock() = None; + *self.psk_identity_hint.lock() = None; + unsafe { + sys::SSL_CTX_set_psk_server_callback(self.builder().as_ptr(), None); + } + } else { + if !callback.is_callable() { + return Err(vm.new_type_error("callback must be callable".to_owned())); + } + *self.psk_server_callback.lock() = Some(callback); + if let OptionalArg::Present(hint) = identity_hint { + *self.psk_identity_hint.lock() = Some(hint.as_str().to_owned()); + } + // Note: The actual callback will be invoked via SSL app_data mechanism + } + Ok(()) + } + #[pymethod] fn load_verify_locations( &self, @@ -1332,16 +1480,33 @@ mod _ssl { // validate cadata type and load cadata if let Some(cadata) = args.cadata { - let certs = match cadata { + let (certs, is_pem) = match cadata { Either::A(s) => { - if !s.is_ascii() { + if !s.as_str().is_ascii() { return Err(invalid_cadata(vm)); } - X509::stack_from_pem(s.as_bytes()) + (X509::stack_from_pem(s.as_bytes()), true) } - Either::B(b) => b.with_ref(x509_stack_from_der), + Either::B(b) => (b.with_ref(x509_stack_from_der), false), }; let certs = certs.map_err(|e| convert_openssl_error(vm, e))?; + + // If no certificates were loaded, raise an error + if certs.is_empty() { + let msg = if is_pem { + "no start line: cadata does not contain a certificate" + } else { + "not enough data: cadata does not contain a certificate" + }; + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + msg.to_owned(), + ) + .upcast()); + } + let store = ctx.cert_store_mut(); for cert in certs { store @@ -1353,6 +1518,29 @@ mod _ssl { if args.cafile.is_some() || args.capath.is_some() { let cafile_path = args.cafile.map(|p| p.to_path_buf(vm)).transpose()?; let capath_path = args.capath.map(|p| p.to_path_buf(vm)).transpose()?; + // Check file/directory existence before calling OpenSSL to get proper errno + if let Some(ref path) = cafile_path + && !path.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", path.display()), + ) + .upcast()); + } + if let Some(ref path) = capath_path + && !path.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", path.display()), + ) + .upcast()); + } ctx.load_verify_locations(cafile_path.as_deref(), capath_path.as_deref()) .map_err(|e| convert_openssl_error(vm, e))?; } @@ -1363,10 +1551,10 @@ mod _ssl { #[pymethod] fn get_ca_certs( &self, - binary_form: OptionalArg<bool>, + args: GetCertArgs, vm: &VirtualMachine, ) -> PyResult<Vec<PyObjectRef>> { - let binary_form = binary_form.unwrap_or(false); + let binary_form = args.binary_form.unwrap_or(false); let ctx = self.ctx(); #[cfg(ossl300)] let certs = ctx.cert_store().all_certificates(); @@ -1416,7 +1604,8 @@ mod _ssl { X509_LU_X509 => { x509_count += 1; let x509_ptr = sys::X509_OBJECT_get0_X509(obj_ptr); - if !x509_ptr.is_null() && X509_check_ca(x509_ptr) == 1 { + // X509_check_ca returns non-zero for any CA type + if !x509_ptr.is_null() && X509_check_ca(x509_ptr) != 0 { ca_count += 1; } } @@ -1476,10 +1665,13 @@ mod _ssl { let fp = rustpython_common::fileutils::fopen(path.as_path(), "rb").map_err(|e| { match e.kind() { - std::io::ErrorKind::NotFound => vm.new_exception_msg( - vm.ctx.exceptions.file_not_found_error.to_owned(), - e.to_string(), - ), + std::io::ErrorKind::NotFound => vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + e.to_string(), + ) + .upcast(), _ => vm.new_os_error(e.to_string()), } })?; @@ -1636,29 +1828,162 @@ mod _ssl { #[pymethod] fn load_cert_chain(&self, args: LoadCertChainArgs, vm: &VirtualMachine) -> PyResult<()> { + use openssl::pkey::PKey; + use std::cell::RefCell; + let LoadCertChainArgs { certfile, keyfile, password, } = args; - // TODO: requires passing a callback to C - if password.is_some() { - return Err(vm.new_not_implemented_error("password arg not yet supported")); - } + let mut ctx = self.builder(); let key_path = keyfile.map(|path| path.to_path_buf(vm)).transpose()?; let cert_path = certfile.to_path_buf(vm)?; - ctx.set_certificate_chain_file(&cert_path) - .and_then(|()| { - ctx.set_private_key_file( - key_path.as_ref().unwrap_or(&cert_path), - ssl::SslFiletype::PEM, + + // Check file existence before calling OpenSSL to get proper errno + if !cert_path.exists() { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", cert_path.display()), ) - }) - .and_then(|()| ctx.check_private_key()) + .upcast()); + } + if let Some(ref kp) = key_path + && !kp.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", kp.display()), + ) + .upcast()); + } + + // Load certificate chain + ctx.set_certificate_chain_file(&cert_path) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Load private key - handle password if provided + let key_file_path = key_path.as_ref().unwrap_or(&cert_path); + + // PEM_BUFSIZE = 1024 (maximum password length in OpenSSL) + const PEM_BUFSIZE: usize = 1024; + + // Read key file data + let key_data = std::fs::read(key_file_path) + .map_err(|e| crate::vm::convert::ToPyException::to_pyexception(&e, vm))?; + + let pkey = if let Some(ref pw_obj) = password { + if pw_obj.is_callable() { + // Callable password - use callback that calls Python function + // Store any Python error that occurs in the callback + let py_error: RefCell<Option<PyBaseExceptionRef>> = RefCell::new(None); + + let result = PKey::private_key_from_pem_callback(&key_data, |buf| { + // Call the Python password callback + let pw_result = pw_obj.call((), vm); + match pw_result { + Ok(result) => { + // Extract password bytes + match Self::extract_password_bytes( + &result, + "password callback must return a string", + vm, + ) { + Ok(pw) => { + // Check password length + if pw.len() > PEM_BUFSIZE { + *py_error.borrow_mut() = + Some(vm.new_value_error(format!( + "password cannot be longer than {} bytes", + PEM_BUFSIZE + ))); + return Err(openssl::error::ErrorStack::get()); + } + let len = core::cmp::min(pw.len(), buf.len()); + buf[..len].copy_from_slice(&pw[..len]); + Ok(len) + } + Err(e) => { + *py_error.borrow_mut() = Some(e); + Err(openssl::error::ErrorStack::get()) + } + } + } + Err(e) => { + *py_error.borrow_mut() = Some(e); + Err(openssl::error::ErrorStack::get()) + } + } + }); + + // Check for Python error first + if let Some(py_err) = py_error.into_inner() { + return Err(py_err); + } + + result.map_err(|e| convert_openssl_error(vm, e))? + } else { + // Direct password (string/bytes) + let pw = Self::extract_password_bytes( + pw_obj, + "password should be a string or bytes", + vm, + )?; + + // Check password length + if pw.len() > PEM_BUFSIZE { + return Err(vm.new_value_error(format!( + "password cannot be longer than {} bytes", + PEM_BUFSIZE + ))); + } + + PKey::private_key_from_pem_passphrase(&key_data, &pw) + .map_err(|e| convert_openssl_error(vm, e))? + } + } else { + // No password - use SSL_CTX_use_PrivateKey_file directly for correct error messages + ctx.set_private_key_file(key_file_path, ssl::SslFiletype::PEM) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Verify key matches certificate and return early + return ctx + .check_private_key() + .map_err(|e| convert_openssl_error(vm, e)); + }; + + ctx.set_private_key(&pkey) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Verify key matches certificate + ctx.check_private_key() .map_err(|e| convert_openssl_error(vm, e)) } + // Helper to extract password bytes from string/bytes/bytearray + fn extract_password_bytes( + obj: &PyObject, + bad_type_error: &str, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + use crate::vm::builtins::{PyByteArray, PyBytes, PyStr}; + + if let Some(s) = obj.downcast_ref::<PyStr>() { + Ok(s.as_str().as_bytes().to_vec()) + } else if let Some(b) = obj.downcast_ref::<PyBytes>() { + Ok(b.as_bytes().to_vec()) + } else if let Some(ba) = obj.downcast_ref::<PyByteArray>() { + Ok(ba.borrow_buf().to_vec()) + } else { + Err(vm.new_type_error(bad_type_error.to_owned())) + } + } + // Helper function to create SSL socket // = CPython's newPySSLSocket() fn new_py_ssl_socket( @@ -1669,15 +1994,15 @@ mod _ssl { ) -> PyResult<(ssl::Ssl, SslServerOrClient, Option<PyStrRef>)> { // Validate socket type and context protocol if server_side && ctx_ref.protocol == SslVersion::TlsClient { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context".to_owned(), + return Err(new_ssl_error( + vm, + "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", )); } if !server_side && ctx_ref.protocol == SslVersion::TlsServer { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "Cannot create a client socket with a PROTOCOL_TLS_SERVER context".to_owned(), + return Err(new_ssl_error( + vm, + "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", )); } @@ -1712,7 +2037,7 @@ mod _ssl { )); } if hostname_str.contains('\0') { - return Err(vm.new_value_error("embedded null byte in server_hostname")); + return Err(vm.new_type_error("embedded null character")); } let ip = hostname_str.parse::<std::net::IpAddr>(); if ip.is_err() { @@ -1790,21 +2115,30 @@ mod _ssl { let py_ref = py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; - // Set SNI callback data if callback is configured - if zelf.sni_callback.lock().is_some() { - unsafe { - let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + // Check if SNI callback is configured (minimize lock time) + let has_sni_callback = zelf.sni_callback.lock().is_some(); - // Store callback data in SSL ex_data + // Set up ex_data for callbacks + unsafe { + let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + + // Clone and store via into_raw() - increments refcount and returns stable pointer + // The refcount will be decremented by msg_callback_data_free when SSL is freed + let cloned: PyObjectRef = py_ref.clone().into(); + let raw_ptr = cloned.into_raw(); + let msg_cb_idx = get_msg_callback_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, msg_cb_idx, raw_ptr.as_ptr() as *mut _); + + // Set SNI callback data if needed + if has_sni_callback { + let ssl_socket_weak = py_ref.as_object().downgrade(None, vm)?; + // Store callback data in SSL ex_data - use weak reference to avoid cycle let callback_data = Box::new(SniCallbackData { ssl_context: zelf.clone(), - vm_ptr: vm as *const _, + ssl_socket_weak, }); - let idx = get_sni_ex_data_index(); - sys::SSL_set_ex_data(ssl_ptr, idx, Box::into_raw(callback_data) as *mut _); - - // Store PyRef pointer (heap-allocated) in ex_data index 0 - sys::SSL_set_ex_data(ssl_ptr, 0, &*py_ref as *const _ as *mut _); + let sni_idx = get_sni_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, sni_idx, Box::into_raw(callback_data) as *mut _); } } @@ -1850,21 +2184,30 @@ mod _ssl { let py_ref = py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; - // Set SNI callback data if callback is configured - if zelf.sni_callback.lock().is_some() { - unsafe { - let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + // Check if SNI callback is configured (minimize lock time) + let has_sni_callback = zelf.sni_callback.lock().is_some(); - // Store callback data in SSL ex_data + // Set up ex_data for callbacks + unsafe { + let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + + // Clone and store via into_raw() - increments refcount and returns stable pointer + // The refcount will be decremented by msg_callback_data_free when SSL is freed + let cloned: PyObjectRef = py_ref.clone().into(); + let raw_ptr = cloned.into_raw(); + let msg_cb_idx = get_msg_callback_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, msg_cb_idx, raw_ptr.as_ptr() as *mut _); + + // Set SNI callback data if needed + if has_sni_callback { + let ssl_socket_weak = py_ref.as_object().downgrade(None, vm)?; + // Store callback data in SSL ex_data - use weak reference to avoid cycle let callback_data = Box::new(SniCallbackData { ssl_context: zelf.clone(), - vm_ptr: vm as *const _, + ssl_socket_weak, }); - let idx = get_sni_ex_data_index(); - sys::SSL_set_ex_data(ssl_ptr, idx, Box::into_raw(callback_data) as *mut _); - - // Store PyRef pointer (heap-allocated) in ex_data index 0 - sys::SSL_set_ex_data(ssl_ptr, 0, &*py_ref as *const _ as *mut _); + let sni_idx = get_sni_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, sni_idx, Box::into_raw(callback_data) as *mut _); } } @@ -1921,7 +2264,13 @@ mod _ssl { #[pyarg(any, optional)] keyfile: Option<FsPath>, #[pyarg(any, optional)] - password: Option<Either<PyStrRef, ArgCallable>>, + password: Option<PyObjectRef>, + } + + #[derive(FromArgs)] + struct GetCertArgs { + #[pyarg(any, optional)] + binary_form: OptionalArg<bool>, } // Err is true if the socket is blocking @@ -1930,7 +2279,6 @@ mod _ssl { enum SelectRet { Nonblocking, TimedOut, - IsBlocking, Closed, Ok, } @@ -1953,12 +2301,14 @@ mod _ssl { Some(s) => s, None => return SelectRet::Closed, }; - let deadline = match &deadline { + // For blocking sockets without timeout, call sock_select with None timeout + // to actually block waiting for data instead of busy-looping + let timeout = match &deadline { Ok(deadline) => match deadline.checked_duration_since(Instant::now()) { - Some(deadline) => deadline, + Some(d) => Some(d), None => return SelectRet::TimedOut, }, - Err(true) => return SelectRet::IsBlocking, + Err(true) => None, // Blocking: no timeout, wait indefinitely Err(false) => return SelectRet::Nonblocking, }; let res = socket::sock_select( @@ -1967,7 +2317,7 @@ mod _ssl { SslNeeds::Read => socket::SelectKind::Read, SslNeeds::Write => socket::SelectKind::Write, }, - Some(deadline), + timeout, ); match res { Ok(true) => SelectRet::TimedOut, @@ -1991,10 +2341,7 @@ mod _ssl { } fn socket_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "Underlying socket has been closed.".to_owned(), - ) + new_ssl_error(vm, "Underlying socket has been closed.") } // BIO stream wrapper to implement Read/Write traits for MemoryBIO @@ -2107,6 +2454,14 @@ mod _ssl { SslConnection::Bio(stream) => stream.get_shutdown(), } } + + // Check if incoming BIO has EOF (for BIO mode only) + fn is_bio_eof(&self) -> bool { + match self { + SslConnection::Socket(_) => false, + SslConnection::Bio(stream) => stream.get_ref().inbio.eof_written.load(), + } + } } #[pyattr] @@ -2151,12 +2506,13 @@ mod _ssl { } #[pygetset(setter)] fn set_context(&self, value: PyRef<PySslContext>, vm: &VirtualMachine) -> PyResult<()> { - // Update the SSL context in the underlying SSL object - let stream = self.connection.read(); + // Get SSL pointer - use thread-local during handshake to avoid deadlock + // (connection lock is already held during handshake) + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); // Set the new SSL_CTX on the SSL object unsafe { - let result = SSL_set_SSL_CTX(stream.ssl().as_ptr(), value.ctx().as_ptr()); + let result = SSL_set_SSL_CTX(ssl_ptr, value.ctx().as_ptr()); if result.is_null() { return Err(vm.new_runtime_error("Failed to set SSL context".to_owned())); } @@ -2174,10 +2530,10 @@ mod _ssl { #[pymethod] fn getpeercert( &self, - binary: OptionalArg<bool>, + args: GetCertArgs, vm: &VirtualMachine, ) -> PyResult<Option<PyObjectRef>> { - let binary = binary.unwrap_or(false); + let binary = args.binary_form.unwrap_or(false); let stream = self.connection.read(); if !stream.ssl().is_init_finished() { return Err(vm.new_value_error("handshake not done yet")); @@ -2210,12 +2566,13 @@ mod _ssl { #[pymethod] fn get_unverified_chain(&self, vm: &VirtualMachine) -> PyResult<Option<PyListRef>> { let stream = self.connection.read(); - let Some(chain) = stream.ssl().peer_cert_chain() else { + let ssl = stream.ssl(); + let Some(chain) = ssl.peer_cert_chain() else { return Ok(None); }; // Return Certificate objects - let certs: Vec<PyObjectRef> = chain + let mut certs: Vec<PyObjectRef> = chain .iter() .map(|cert| unsafe { sys::X509_up_ref(cert.as_ptr()); @@ -2223,6 +2580,16 @@ mod _ssl { cert_to_certificate(vm, owned) }) .collect::<PyResult<_>>()?; + + // SSL_get_peer_cert_chain does not include peer cert for server-side sockets + // Add it manually at the beginning + if matches!(self.socket_type, SslServerOrClient::Server) + && let Some(peer_cert) = ssl.peer_certificate() + { + let peer_obj = cert_to_certificate(vm, peer_cert)?; + certs.insert(0, peer_obj); + } + Ok(Some(vm.ctx.new_list(certs))) } @@ -2261,31 +2628,42 @@ mod _ssl { #[pymethod] fn version(&self) -> Option<&'static str> { - let v = self.connection.read().ssl().version_str(); + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + // Return None if handshake is not complete (CPython behavior) + if unsafe { sys::SSL_is_init_finished(ssl_ptr) } == 0 { + return None; + } + let v = unsafe { ssl::SslRef::from_ptr(ssl_ptr).version_str() }; if v == "unknown" { None } else { Some(v) } } #[pymethod] fn cipher(&self) -> Option<CipherTuple> { - self.connection - .read() - .ssl() - .current_cipher() - .map(cipher_to_tuple) + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { ssl::SslRef::from_ptr(ssl_ptr).current_cipher() }.map(cipher_to_tuple) + } + + #[pymethod] + fn pending(&self) -> i32 { + let stream = self.connection.read(); + unsafe { sys::SSL_pending(stream.ssl().as_ptr()) } } #[pymethod] fn shared_ciphers(&self, vm: &VirtualMachine) -> Option<PyListRef> { #[cfg(ossl110)] { - let stream = self.connection.read(); + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); unsafe { - let server_ciphers = SSL_get_ciphers(stream.ssl().as_ptr()); + let server_ciphers = SSL_get_ciphers(ssl_ptr); if server_ciphers.is_null() { return None; } - let client_ciphers = SSL_get_client_ciphers(stream.ssl().as_ptr()); + let client_ciphers = SSL_get_client_ciphers(ssl_ptr); if client_ciphers.is_null() { return None; } @@ -2341,12 +2719,13 @@ mod _ssl { fn selected_alpn_protocol(&self) -> Option<String> { #[cfg(ossl102)] { - let stream = self.connection.read(); + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); unsafe { - let mut out: *const libc::c_uchar = std::ptr::null(); + let mut out: *const libc::c_uchar = core::ptr::null(); let mut outlen: libc::c_uint = 0; - sys::SSL_get0_alpn_selected(stream.ssl().as_ptr(), &mut out, &mut outlen); + sys::SSL_get0_alpn_selected(ssl_ptr, &mut out, &mut outlen); if out.is_null() { None @@ -2426,42 +2805,124 @@ mod _ssl { } #[pymethod] - fn shutdown(&self, vm: &VirtualMachine) -> PyResult<PyRef<PySocket>> { + fn shutdown(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<PySocket>>> { let stream = self.connection.read(); + let ssl_ptr = stream.ssl().as_ptr(); - // BIO mode doesn't have an underlying socket + // BIO mode: just try shutdown once and raise SSLWantReadError if needed if stream.is_bio() { - return Err(vm.new_not_implemented_error( - "shutdown() is not supported for BIO-based SSL objects".to_owned(), - )); + let ret = unsafe { sys::SSL_shutdown(ssl_ptr) }; + if ret < 0 { + let err = unsafe { sys::SSL_get_error(ssl_ptr, ret) }; + if err == sys::SSL_ERROR_WANT_READ { + return Err(create_ssl_want_read_error(vm).upcast()); + } else if err == sys::SSL_ERROR_WANT_WRITE { + return Err(create_ssl_want_write_error(vm).upcast()); + } else { + return Err(new_ssl_error( + vm, + format!("SSL shutdown failed: error code {}", err), + )); + } + } else if ret == 0 { + // Sent close-notify, waiting for peer's - raise SSLWantReadError + return Err(create_ssl_want_read_error(vm).upcast()); + } + return Ok(None); } - let ssl_ptr = stream.ssl().as_ptr(); + // Socket mode: loop with select to wait for peer's close-notify + let socket_stream = stream.get_ref().expect("get_ref() failed for socket mode"); + let deadline = socket_stream.timeout_deadline(); + + // Track how many times we've seen ret == 0 (max 2 tries) + let mut zeros = 0; - // Perform SSL shutdown - let ret = unsafe { sys::SSL_shutdown(ssl_ptr) }; + loop { + let ret = unsafe { sys::SSL_shutdown(ssl_ptr) }; + + // ret > 0: complete shutdown + if ret > 0 { + break; + } - if ret < 0 { - // Error occurred + // ret == 0: sent our close-notify, need to receive peer's + if ret == 0 { + zeros += 1; + if zeros > 1 { + // Already tried twice, break out (legacy behavior) + break; + } + // Wait briefly for peer's close_notify before retrying + match socket_stream.select(SslNeeds::Read, &deadline) { + SelectRet::TimedOut => { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.timeout_error.to_owned(), + "The read operation timed out".to_owned(), + )); + } + SelectRet::Closed => { + return Err(socket_closed_error(vm)); + } + SelectRet::Nonblocking => { + // Non-blocking socket: return SSLWantReadError + return Err(create_ssl_want_read_error(vm).upcast()); + } + SelectRet::Ok => { + // Data available, continue to retry + } + } + continue; + } + + // ret < 0: error or would-block let err = unsafe { sys::SSL_get_error(ssl_ptr, ret) }; - if err == sys::SSL_ERROR_WANT_READ || err == sys::SSL_ERROR_WANT_WRITE { - // Non-blocking would block - this is okay for shutdown - // Return the underlying socket + let needs = if err == sys::SSL_ERROR_WANT_READ { + SslNeeds::Read + } else if err == sys::SSL_ERROR_WANT_WRITE { + SslNeeds::Write } else { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), + // Real error + return Err(new_ssl_error( + vm, format!("SSL shutdown failed: error code {}", err), )); + }; + + // Wait on the socket + match socket_stream.select(needs, &deadline) { + SelectRet::TimedOut => { + let msg = if err == sys::SSL_ERROR_WANT_READ { + "The read operation timed out" + } else { + "The write operation timed out" + }; + return Err(vm.new_exception_msg( + vm.ctx.exceptions.timeout_error.to_owned(), + msg.to_owned(), + )); + } + SelectRet::Closed => { + return Err(socket_closed_error(vm)); + } + SelectRet::Nonblocking => { + // Non-blocking socket, raise SSLWantReadError/SSLWantWriteError + if err == sys::SSL_ERROR_WANT_READ { + return Err(create_ssl_want_read_error(vm).upcast()); + } else { + return Err(create_ssl_want_write_error(vm).upcast()); + } + } + SelectRet::Ok => { + // Socket is ready, retry shutdown + continue; + } } } // Return the underlying socket - // Get the socket from the stream (SocketStream wraps PyRef<PySocket>) - let socket = stream - .get_ref() - .expect("unwrap() called on bio mode; should only be called in socket mode"); - Ok(socket.0.clone()) + Ok(Some(socket_stream.0.clone())) } #[cfg(osslconf = "OPENSSL_NO_COMP")] @@ -2472,8 +2933,9 @@ mod _ssl { #[cfg(not(osslconf = "OPENSSL_NO_COMP"))] #[pymethod] fn compression(&self) -> Option<&'static str> { - let stream = self.connection.read(); - let comp_method = unsafe { sys::SSL_get_current_compression(stream.ssl().as_ptr()) }; + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + let comp_method = unsafe { sys::SSL_get_current_compression(ssl_ptr) }; if comp_method.is_null() { return None; } @@ -2490,16 +2952,24 @@ mod _ssl { let mut stream = self.connection.write(); let ssl_ptr = stream.ssl().as_ptr(); + // Set up thread-local VM and SSL pointer for callbacks + // This allows callbacks to access SSL without acquiring the connection lock + let _vm_guard = HandshakeVmGuard::new(vm, ssl_ptr); + // BIO mode: no timeout/select logic, just do handshake if stream.is_bio() { - return stream.do_handshake().map_err(|e| { + let result = stream.do_handshake().map_err(|e| { let exc = convert_ssl_error(vm, e); // If it's a cert verification error, set verify info - if exc.class().is(PySslCertVerificationError::class(&vm.ctx)) { + if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { set_verify_error_info(&exc, ssl_ptr, vm); } exc }); + // Clean up SNI ex_data after handshake (success or failure) + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return result; } // Socket mode: handle timeout and blocking @@ -2509,7 +2979,12 @@ mod _ssl { .timeout_deadline(); loop { let err = match stream.do_handshake() { - Ok(()) => return Ok(()), + Ok(()) => { + // Clean up SNI ex_data after successful handshake + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Ok(()); + } Err(e) => e, }; let (needs, state) = stream @@ -2518,14 +2993,22 @@ mod _ssl { .socket_needs(&err, &timeout); match state { SelectRet::TimedOut => { + // Clean up SNI ex_data before returning error + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; return Err(socket::timeout_error_msg( vm, "The handshake operation timed out".to_owned(), - )); + ) + .upcast()); + } + SelectRet::Closed => { + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Err(socket_closed_error(vm)); } - SelectRet::Closed => return Err(socket_closed_error(vm)), SelectRet::Nonblocking => {} - SelectRet::IsBlocking | SelectRet::Ok => { + SelectRet::Ok => { // For blocking sockets, select() has completed successfully // Continue the handshake loop (matches CPython's SOCKET_IS_BLOCKING behavior) if needs.is_some() { @@ -2535,9 +3018,12 @@ mod _ssl { } let exc = convert_ssl_error(vm, err); // If it's a cert verification error, set verify info - if exc.class().is(PySslCertVerificationError::class(&vm.ctx)) { + if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { set_verify_error_info(&exc, ssl_ptr, vm); } + // Clean up SNI ex_data before returning error + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; return Err(exc); } } @@ -2564,7 +3050,8 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The write operation timed out".to_owned(), - )); + ) + .upcast()); } SelectRet::Closed => return Err(socket_closed_error(vm)), _ => {} @@ -2583,11 +3070,12 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The write operation timed out".to_owned(), - )); + ) + .upcast()); } SelectRet::Closed => return Err(socket_closed_error(vm)), SelectRet::Nonblocking => {} - SelectRet::IsBlocking | SelectRet::Ok => { + SelectRet::Ok => { // For blocking sockets, select() has completed successfully // Continue the write loop (matches CPython's SOCKET_IS_BLOCKING behavior) if needs.is_some() { @@ -2660,19 +3148,41 @@ mod _ssl { #[pygetset] fn session_reused(&self) -> bool { - let stream = self.connection.read(); - unsafe { sys::SSL_session_reused(stream.ssl().as_ptr()) != 0 } + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { sys::SSL_session_reused(ssl_ptr) != 0 } } #[pymethod] fn read( &self, - n: usize, + n: isize, buffer: OptionalArg<ArgMemoryBuffer>, vm: &VirtualMachine, ) -> PyResult { + // Handle negative n: + // - If buffer is None and n < 0: raise ValueError + // - If buffer is present and n <= 0: use buffer length + // This matches _ssl__SSLSocket_read_impl in CPython + let read_len: usize = match &buffer { + OptionalArg::Present(buf) => { + let buf_len = buf.borrow_buf_mut().len(); + if n <= 0 || (n as usize) > buf_len { + buf_len + } else { + n as usize + } + } + OptionalArg::Missing => { + if n < 0 { + return Err(vm.new_value_error("size should not be negative".to_owned())); + } + n as usize + } + }; + // Special case: reading 0 bytes should return empty bytes immediately - if n == 0 { + if read_len == 0 { return if buffer.is_present() { Ok(vm.ctx.new_int(0).into()) } else { @@ -2684,13 +3194,13 @@ mod _ssl { let mut inner_buffer = if let OptionalArg::Present(buffer) = &buffer { Either::A(buffer.borrow_buf_mut()) } else { - Either::B(vec![0u8; n]) + Either::B(vec![0u8; read_len]) }; let buf = match &mut inner_buffer { Either::A(b) => &mut **b, Either::B(b) => b.as_mut_slice(), }; - let buf = match buf.get_mut(..n) { + let buf = match buf.get_mut(..read_len) { Some(b) => b, None => buf, }; @@ -2699,7 +3209,18 @@ mod _ssl { let count = if stream.is_bio() { match stream.ssl_read(buf) { Ok(count) => count, - Err(e) => return Err(convert_ssl_error(vm, e)), + Err(e) => { + // Handle ZERO_RETURN (EOF) - raise SSLEOFError + if e.code() == ssl::ErrorCode::ZERO_RETURN { + return Err(create_ssl_eof_error(vm).upcast()); + } + // If WANT_READ and the incoming BIO has EOF written, + // this is an unexpected EOF (transport closed without TLS close_notify) + if e.code() == ssl::ErrorCode::WANT_READ && stream.is_bio_eof() { + return Err(create_ssl_eof_error(vm).upcast()); + } + return Err(convert_ssl_error(vm, e)); + } } } else { // Socket mode: handle timeout and blocking @@ -2726,11 +3247,12 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The read operation timed out".to_owned(), - )); + ) + .upcast()); } SelectRet::Closed => return Err(socket_closed_error(vm)), SelectRet::Nonblocking => {} - SelectRet::IsBlocking | SelectRet::Ok => { + SelectRet::Ok => { // For blocking sockets, select() has completed successfully // Continue the read loop (matches CPython's SOCKET_IS_BLOCKING behavior) if needs.is_some() { @@ -3069,7 +3591,7 @@ mod _ssl { impl Constructor for PySslMemoryBio { type Args = (); - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { unsafe { let bio = sys::BIO_new(sys::BIO_s_mem()); if bio.is_null() { @@ -3079,12 +3601,10 @@ mod _ssl { sys::BIO_set_retry_read(bio); BIO_set_mem_eof_return(bio, -1); - PySslMemoryBio { + Ok(PySslMemoryBio { bio, eof_written: AtomicCell::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) + }) } } } @@ -3109,22 +3629,10 @@ mod _ssl { let len = size.unwrap_or(-1); let len = if len < 0 || len > avail { avail } else { len }; - // Check if EOF has been written and no data available - // This matches CPython's behavior where read() returns b'' when EOF is set - if len == 0 && self.eof_written.load() { - return Ok(Vec::new()); - } - + // When no data available, return empty bytes (CPython behavior) + // CPython returns empty bytes directly without calling BIO_read() if len == 0 { - // No data available and no EOF - would block - // Call BIO_read() to get the proper error (SSL_ERROR_WANT_READ) - let mut test_buf = [0u8; 1]; - let nbytes = sys::BIO_read(self.bio, test_buf.as_mut_ptr() as *mut _, 1); - if nbytes < 0 { - return Err(convert_openssl_error(vm, ErrorStack::get())); - } - // Shouldn't reach here, but if we do, return what we got - return Ok(test_buf[..nbytes as usize].to_vec()); + return Ok(Vec::new()); } let mut buf = vec![0u8; len as usize]; @@ -3142,10 +3650,7 @@ mod _ssl { #[pymethod] fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<i32> { if self.eof_written.load() { - return Err(vm.new_exception_msg( - PySslError::class(&vm.ctx).to_owned(), - "cannot write() after write_eof()".to_owned(), - )); + return Err(new_ssl_error(vm, "cannot write() after write_eof()")); } data.with_ref(|buf| unsafe { @@ -3234,6 +3739,12 @@ mod _ssl { } } + /// Helper function to create SSL error with proper OSError subtype handling + fn new_ssl_error(vm: &VirtualMachine, msg: impl ToString) -> PyBaseExceptionRef { + vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), None, msg.to_string()) + .upcast() + } + #[track_caller] pub(crate) fn convert_openssl_error( vm: &VirtualMachine, @@ -3254,12 +3765,7 @@ mod _ssl { } else { vm.ctx.exceptions.os_error.to_owned() }; - let exc = vm.new_exception(exc_type, vec![vm.ctx.new_int(reason).into()]); - // Set errno attribute explicitly - let _ = exc - .as_object() - .set_attr("errno", vm.ctx.new_int(reason), vm); - return exc; + return vm.new_os_subtype_error(exc_type, Some(reason), "").upcast(); } let caller = std::panic::Location::caller(); @@ -3295,9 +3801,9 @@ mod _ssl { // Use SSLCertVerificationError for certificate verification failures let cls = if is_cert_verify_error { - PySslCertVerificationError::class(&vm.ctx).to_owned() + PySSLCertVerificationError::class(&vm.ctx).to_owned() } else { - PySslError::class(&vm.ctx).to_owned() + PySSLError::class(&vm.ctx).to_owned() }; // Build message @@ -3309,13 +3815,8 @@ mod _ssl { // Create exception instance let reason = sys::ERR_GET_REASON(e.code()); - let exc = vm.new_exception( - cls, - vec![vm.ctx.new_int(reason).into(), vm.ctx.new_str(msg).into()], - ); - - // Set attributes on instance, not class - let exc_obj: PyObjectRef = exc.into(); + let exc = vm.new_os_subtype_error(cls, Some(reason), msg); + let exc_obj: PyObjectRef = exc.upcast::<PyBaseException>().into(); // Set reason attribute (always set, even if just the error string) let reason_value = vm.ctx.new_str(errstr); @@ -3343,15 +3844,16 @@ mod _ssl { ) } None => { - let cls = PySslError::class(&vm.ctx).to_owned(); - vm.new_exception_empty(cls) + let cls = PySSLError::class(&vm.ctx).to_owned(); + vm.new_os_subtype_error(cls, None, "unknown SSL error") + .upcast() } } } // Helper function to set verify_code and verify_message on SSLCertVerificationError fn set_verify_error_info( - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, ssl_ptr: *const sys::SSL, vm: &VirtualMachine, ) { @@ -3381,29 +3883,18 @@ mod _ssl { ) -> PyBaseExceptionRef { let e = e.borrow(); let (cls, msg) = match e.code() { - ssl::ErrorCode::WANT_READ => ( - PySslWantReadError::class(&vm.ctx).to_owned(), - "The operation did not complete (read)", - ), - ssl::ErrorCode::WANT_WRITE => ( - PySslWantWriteError::class(&vm.ctx).to_owned(), - "The operation did not complete (write)", - ), + ssl::ErrorCode::WANT_READ => { + return create_ssl_want_read_error(vm).upcast(); + } + ssl::ErrorCode::WANT_WRITE => { + return create_ssl_want_write_error(vm).upcast(); + } ssl::ErrorCode::SYSCALL => match e.io_error() { Some(io_err) => return io_err.to_pyexception(vm), // When no I/O error and OpenSSL error queue is empty, // this is an EOF in violation of protocol -> SSLEOFError - // Need to set args[0] = SSL_ERROR_EOF for suppress_ragged_eofs check None => { - return vm.new_exception( - PySslEOFError::class(&vm.ctx).to_owned(), - vec![ - vm.ctx.new_int(SSL_ERROR_EOF).into(), - vm.ctx - .new_str("EOF occurred in violation of protocol") - .into(), - ], - ); + return create_ssl_eof_error(vm).upcast(); } }, ssl::ErrorCode::SSL => { @@ -3416,30 +3907,22 @@ mod _ssl { let reason = sys::ERR_GET_REASON(err_code); let lib = sys::ERR_GET_LIB(err_code); if lib == ERR_LIB_SSL && reason == SSL_R_UNEXPECTED_EOF_WHILE_READING { - return vm.new_exception( - PySslEOFError::class(&vm.ctx).to_owned(), - vec![ - vm.ctx.new_int(SSL_ERROR_EOF).into(), - vm.ctx - .new_str("EOF occurred in violation of protocol") - .into(), - ], - ); + return create_ssl_eof_error(vm).upcast(); } } return convert_openssl_error(vm, ssl_err.clone()); } ( - PySslError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx).to_owned(), "A failure in the SSL library occurred", ) } _ => ( - PySslError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx).to_owned(), "A failure in the SSL library occurred", ), }; - vm.new_exception_msg(cls, msg.to_owned()) + vm.new_os_subtype_error(cls, None, msg).upcast() } // SSL_FILETYPE_ASN1 part of _add_ca_certs in CPython @@ -3449,30 +3932,33 @@ mod _ssl { let bio = bio::MemBioSlice::new(der)?; let mut certs = vec![]; + let mut was_bio_eof = false; loop { + // Check for EOF before attempting to parse (like CPython's _add_ca_certs) + // BIO_ctrl with BIO_CTRL_EOF returns 1 if EOF, 0 otherwise + if sys::BIO_ctrl(bio.as_ptr(), sys::BIO_CTRL_EOF, 0, std::ptr::null_mut()) != 0 { + was_bio_eof = true; + break; + } + let cert = sys::d2i_X509_bio(bio.as_ptr(), std::ptr::null_mut()); if cert.is_null() { + // Parse error (not just EOF) break; } certs.push(X509::from_ptr(cert)); } - if certs.is_empty() { - // No certificates loaded at all + // If we loaded some certs but didn't reach EOF, there's garbage data + // (like cacert_der + b"A") - this is an error + if !certs.is_empty() && !was_bio_eof { + // Return the error from the last failed parse attempt return Err(ErrorStack::get()); } - // Successfully loaded at least one certificate from DER data. - // Clear any trailing errors from EOF. - // CPython clears errors when: - // - DER: was_bio_eof is set (EOF reached) - // - PEM: PEM_R_NO_START_LINE error (normal EOF) - // Both cases mean successful completion with loaded certs. - eprintln!( - "[x509_stack_from_der] SUCCESS: Clearing errors and returning {} certs", - certs.len() - ); + // Clear any errors (including parse errors when no certs loaded) + // Let the caller decide how to handle empty results sys::ERR_clear_error(); Ok(certs) } @@ -3542,10 +4028,13 @@ mod _ssl { ) -> Result<(), PyBaseExceptionRef> { let root = Path::new(CERT_DIR); if !root.is_dir() { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.file_not_found_error.to_owned(), - CERT_DIR.to_string(), - )); + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + None, + CERT_DIR.to_string(), + ) + .upcast()); } let mut combined_pem = String::new(); @@ -3585,18 +4074,6 @@ mod _ssl { } } -#[cfg(not(ossl101))] -#[pymodule(sub)] -mod ossl101 {} - -#[cfg(not(ossl111))] -#[pymodule(sub)] -mod ossl111 {} - -#[cfg(not(windows))] -#[pymodule(sub)] -mod windows {} - #[allow(non_upper_case_globals)] #[cfg(ossl101)] #[pymodule(sub)] diff --git a/crates/stdlib/src/openssl/cert.rs b/crates/stdlib/src/openssl/cert.rs index 1139f0e26f0..b63d824a837 100644 --- a/crates/stdlib/src/openssl/cert.rs +++ b/crates/stdlib/src/openssl/cert.rs @@ -5,16 +5,19 @@ pub(super) use ssl_cert::{PySSLCertificate, cert_to_certificate, cert_to_py, obj #[pymodule(sub)] pub(crate) mod ssl_cert { use crate::{ - common::ascii, + common::{ascii, hash::PyHash}, vm::{ - PyObjectRef, PyPayload, PyResult, VirtualMachine, + Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class_or_notimplemented, convert::{ToPyException, ToPyObject}, - function::{FsPath, OptionalArg}, + function::{FsPath, OptionalArg, PyComparisonValue}, + types::{Comparable, Hashable, PyComparisonOp, Representable}, }, }; use foreign_types_shared::ForeignTypeRef; use openssl::{ asn1::Asn1ObjectRef, + nid::Nid, x509::{self, X509, X509Ref}, }; use openssl_sys as sys; @@ -54,7 +57,7 @@ pub(crate) mod ssl_cert { #[pyclass(module = "ssl", name = "Certificate")] #[derive(PyPayload)] pub(crate) struct PySSLCertificate { - cert: X509, + pub(crate) cert: X509, } impl fmt::Debug for PySSLCertificate { @@ -63,7 +66,7 @@ pub(crate) mod ssl_cert { } } - #[pyclass] + #[pyclass(with(Comparable, Hashable, Representable))] impl PySSLCertificate { #[pymethod] fn public_bytes( @@ -83,12 +86,14 @@ pub(crate) mod ssl_cert { Ok(vm.ctx.new_bytes(der).into()) } ENCODING_PEM => { - // PEM encoding + // PEM encoding - returns string let pem = self .cert .to_pem() .map_err(|e| convert_openssl_error(vm, e))?; - Ok(vm.ctx.new_bytes(pem).into()) + let pem_str = String::from_utf8(pem) + .map_err(|_| vm.new_value_error("Invalid UTF-8 in PEM"))?; + Ok(vm.ctx.new_str(pem_str).into()) } _ => Err(vm.new_value_error("Unsupported format")), } @@ -100,6 +105,66 @@ pub(crate) mod ssl_cert { } } + impl Comparable for PySSLCertificate { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let other = class_or_notimplemented!(Self, other); + + // Only support equality comparison + if !matches!(op, PyComparisonOp::Eq | PyComparisonOp::Ne) { + return Ok(PyComparisonValue::NotImplemented); + } + + // Compare DER encodings + let self_der = zelf + .cert + .to_der() + .map_err(|e| convert_openssl_error(vm, e))?; + let other_der = other + .cert + .to_der() + .map_err(|e| convert_openssl_error(vm, e))?; + + let eq = self_der == other_der; + Ok(op.eval_ord(eq.cmp(&true)).into()) + } + } + + impl Hashable for PySSLCertificate { + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + // Use subject name hash as certificate hash + let hash = unsafe { sys::X509_subject_name_hash(zelf.cert.as_ptr()) }; + Ok(hash as PyHash) + } + } + + impl Representable for PySSLCertificate { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // Build subject string like "CN=localhost, O=Python" + let subject = zelf.cert.subject_name(); + let mut parts: Vec<String> = Vec::new(); + for entry in subject.entries() { + // Use short name (SN) if available, otherwise use OID + let name = match entry.object().nid().short_name() { + Ok(sn) => sn.to_string(), + Err(_) => obj2txt(entry.object(), true).unwrap_or_default(), + }; + if let Ok(value) = entry.data().as_utf8() { + parts.push(format!("{}={}", name, value)); + } + } + if parts.is_empty() { + Ok("<Certificate>".to_string()) + } else { + Ok(format!("<Certificate '{}'>", parts.join(", "))) + } + } + } + fn name_to_py(vm: &VirtualMachine, name: &x509::X509NameRef) -> PyResult { let list = name .entries() @@ -133,11 +198,15 @@ pub(crate) mod ssl_cert { .to_bn() .and_then(|bn| bn.to_hex_str()) .map_err(|e| convert_openssl_error(vm, e))?; - dict.set_item( - "serialNumber", - vm.ctx.new_str(serial_num.to_owned()).into(), - vm, - )?; + // Serial number must have even length (each byte = 2 hex chars) + // BigNum::to_hex_str() strips leading zeros, so we need to pad + let serial_str = serial_num.to_string(); + let serial_str = if serial_str.len() % 2 == 1 { + format!("0{}", serial_str) + } else { + serial_str + }; + dict.set_item("serialNumber", vm.ctx.new_str(serial_str).into(), vm)?; dict.set_item( "notBefore", @@ -165,7 +234,8 @@ pub(crate) mod ssl_cert { format!("{}.{}.{}.{}", ip[0], ip[1], ip[2], ip[3]) } else if ip.len() == 16 { // IPv6 - format with all zeros visible (not compressed) - let ip_addr = std::net::Ipv6Addr::from(ip[0..16]); + let ip_addr = + std::net::Ipv6Addr::from(<[u8; 16]>::try_from(&ip[0..16]).unwrap()); let s = ip_addr.segments(); format!( "{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}", @@ -187,10 +257,23 @@ pub(crate) mod ssl_cert { return vm.new_tuple((ascii!("DirName"), py_name)).into(); } - // TODO: Handle Registered ID (GEN_RID) - // CPython implementation uses i2t_ASN1_OBJECT to convert OID - // This requires accessing GENERAL_NAME union which is complex in Rust - // For now, we return <unsupported> for unhandled types + // Check for Registered ID (GEN_RID) + // Access raw GENERAL_NAME to check type + let ptr = gen_name.as_ptr(); + unsafe { + if (*ptr).type_ == sys::GEN_RID { + // d is ASN1_OBJECT* for GEN_RID + let oid_ptr = (*ptr).d as *const sys::ASN1_OBJECT; + if !oid_ptr.is_null() { + let oid_ref = Asn1ObjectRef::from_ptr(oid_ptr as *mut _); + if let Some(oid_str) = obj2txt(oid_ref, true) { + return vm + .new_tuple((ascii!("Registered ID"), oid_str)) + .into(); + } + } + } + } // For othername and other unsupported types vm.new_tuple((ascii!("othername"), ascii!("<unsupported>"))) @@ -201,6 +284,60 @@ pub(crate) mod ssl_cert { dict.set_item("subjectAltName", vm.ctx.new_tuple(san).into(), vm)?; }; + // Authority Information Access: OCSP URIs + if let Ok(ocsp_list) = cert.ocsp_responders() + && !ocsp_list.is_empty() + { + let uris: Vec<PyObjectRef> = ocsp_list + .iter() + .map(|s| vm.ctx.new_str(s.to_string()).into()) + .collect(); + dict.set_item("OCSP", vm.ctx.new_tuple(uris).into(), vm)?; + } + + // Authority Information Access: CA Issuers URIs + if let Some(aia) = cert.authority_info() { + let ca_issuers: Vec<PyObjectRef> = aia + .iter() + .filter_map(|ad| { + // Check if method is CA Issuers (NID_ad_ca_issuers) + if ad.method().nid() != Nid::AD_CA_ISSUERS { + return None; + } + // Get URI from location + ad.location() + .uri() + .map(|uri| vm.ctx.new_str(uri.to_owned()).into()) + }) + .collect(); + if !ca_issuers.is_empty() { + dict.set_item("caIssuers", vm.ctx.new_tuple(ca_issuers).into(), vm)?; + } + } + + // CRL Distribution Points + if let Some(crl_dps) = cert.crl_distribution_points() { + let mut crl_uris: Vec<PyObjectRef> = Vec::new(); + for dp in crl_dps.iter() { + if let Some(dp_name) = dp.distpoint() + && let Some(fullname) = dp_name.fullname() + { + for gn in fullname.iter() { + if let Some(uri) = gn.uri() { + crl_uris.push(vm.ctx.new_str(uri.to_owned()).into()); + } + } + } + } + if !crl_uris.is_empty() { + dict.set_item( + "crlDistributionPoints", + vm.ctx.new_tuple(crl_uris).into(), + vm, + )?; + } + } + Ok(dict.into()) } diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index d8f14baf35e..779fe31efe5 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use _overlapped::make_module; +pub(crate) use _overlapped::module_def; #[allow(non_snake_case)] #[pymodule] @@ -8,19 +8,28 @@ mod _overlapped { // straight-forward port of Modules/overlapped.c use crate::vm::{ - Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef, PyType}, + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef, PyModule, PyStrRef, PyTupleRef, PyType}, common::lock::PyMutex, - convert::{ToPyException, ToPyObject}, + convert::ToPyObject, + function::OptionalArg, + object::{Traverse, TraverseFn}, protocol::PyBuffer, - types::Constructor, + types::{Constructor, Destructor}, }; use windows_sys::Win32::{ Foundation::{self, GetLastError, HANDLE}, - Networking::WinSock::SOCKADDR_IN6, + Networking::WinSock::{AF_INET, AF_INET6, SOCKADDR, SOCKADDR_IN, SOCKADDR_IN6}, System::IO::OVERLAPPED, }; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_socket", 0)?; + initialize_winsock_extensions(vm)?; + __module_exec(vm, module); + Ok(()) + } + #[pyattr] use windows_sys::Win32::{ Foundation::{ @@ -35,13 +44,102 @@ mod _overlapped { #[pyattr] const INVALID_HANDLE_VALUE: isize = - unsafe { std::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; + unsafe { core::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; #[pyattr] const NULL: isize = 0; + // Function pointers for Winsock extension functions + static ACCEPT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static CONNECT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static DISCONNECT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static TRANSMIT_FILE: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + + fn initialize_winsock_extensions(vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{ + INVALID_SOCKET, IPPROTO_TCP, SIO_GET_EXTENSION_FUNCTION_POINTER, SOCK_STREAM, + SOCKET_ERROR, WSAGetLastError, WSAIoctl, closesocket, socket, + }; + + // GUIDs for extension functions + const WSAID_ACCEPTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0xb5367df1, + data2: 0xcbac, + data3: 0x11cf, + data4: [0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92], + }; + const WSAID_CONNECTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0x25a207b9, + data2: 0xddf3, + data3: 0x4660, + data4: [0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e], + }; + const WSAID_DISCONNECTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0x7fda2e11, + data2: 0x8630, + data3: 0x436f, + data4: [0xa0, 0x31, 0xf5, 0x36, 0xa6, 0xee, 0xc1, 0x57], + }; + const WSAID_TRANSMITFILE: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0xb5367df0, + data2: 0xcbac, + data3: 0x11cf, + data4: [0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92], + }; + + // Check all four locks to prevent partial initialization + if ACCEPT_EX.get().is_some() + && CONNECT_EX.get().is_some() + && DISCONNECT_EX.get().is_some() + && TRANSMIT_FILE.get().is_some() + { + return Ok(()); + } + + let s = unsafe { socket(AF_INET as i32, SOCK_STREAM, IPPROTO_TCP) }; + if s == INVALID_SOCKET { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + let mut dw_bytes: u32 = 0; + + macro_rules! get_extension { + ($guid:expr, $lock:expr) => {{ + let mut func_ptr: usize = 0; + let ret = unsafe { + WSAIoctl( + s, + SIO_GET_EXTENSION_FUNCTION_POINTER, + &$guid as *const _ as *const _, + core::mem::size_of_val(&$guid) as u32, + &mut func_ptr as *mut _ as *mut _, + core::mem::size_of::<usize>() as u32, + &mut dw_bytes, + core::ptr::null_mut(), + None, + ) + }; + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + unsafe { closesocket(s) }; + return Err(set_from_windows_err(err, vm)); + } + let _ = $lock.set(func_ptr); + }}; + } + + get_extension!(WSAID_ACCEPTEX, ACCEPT_EX); + get_extension!(WSAID_CONNECTEX, CONNECT_EX); + get_extension!(WSAID_DISCONNECTEX, DISCONNECT_EX); + get_extension!(WSAID_TRANSMITFILE, TRANSMIT_FILE); + + unsafe { closesocket(s) }; + Ok(()) + } + #[pyattr] - #[pyclass(name)] + #[pyclass(name, traverse)] #[derive(PyPayload)] struct Overlapped { inner: PyMutex<OverlappedInner>, @@ -57,11 +155,39 @@ mod _overlapped { unsafe impl Sync for OverlappedInner {} unsafe impl Send for OverlappedInner {} - impl std::fmt::Debug for Overlapped { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + unsafe impl Traverse for OverlappedInner { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + match &self.data { + OverlappedData::Read(buf) | OverlappedData::Accept(buf) => { + buf.traverse(traverse_fn); + } + OverlappedData::ReadInto(buf) | OverlappedData::Write(buf) => { + buf.traverse(traverse_fn); + } + OverlappedData::WriteTo(wt) => { + wt.buf.traverse(traverse_fn); + } + OverlappedData::ReadFrom(rf) => { + if let Some(result) = &rf.result { + result.traverse(traverse_fn); + } + rf.allocated_buffer.traverse(traverse_fn); + } + OverlappedData::ReadFromInto(rfi) => { + if let Some(result) = &rfi.result { + result.traverse(traverse_fn); + } + rfi.user_buffer.traverse(traverse_fn); + } + _ => {} + } + } + } + + impl core::fmt::Debug for Overlapped { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let zelf = self.inner.lock(); f.debug_struct("Overlapped") - // .field("overlapped", &(self.overlapped as *const _ as usize)) .field("handle", &zelf.handle) .field("error", &zelf.error) .field("data", &zelf.data) @@ -69,41 +195,43 @@ mod _overlapped { } } - #[allow(dead_code)] // TODO: remove when done #[derive(Debug)] enum OverlappedData { None, NotStarted, Read(PyBytesRef), + // Fields below store buffers that must be kept alive during async operations + #[allow(dead_code)] ReadInto(PyBuffer), + #[allow(dead_code)] Write(PyBuffer), - Accept(PyObjectRef), - Connect, + #[allow(dead_code)] + Accept(PyBytesRef), + Connect(Vec<u8>), // Store address bytes to keep them alive during async operation Disconnect, ConnectNamedPipe, + #[allow(dead_code)] // Reserved for named pipe support WaitNamedPipeAndConnect, TransmitFile, ReadFrom(OverlappedReadFrom), - WriteTo(PyBuffer), + WriteTo(OverlappedWriteTo), // Store address bytes for WSASendTo ReadFromInto(OverlappedReadFromInto), } struct OverlappedReadFrom { // A (buffer, (host, port)) tuple - result: PyObjectRef, + result: Option<PyObjectRef>, // The actual read buffer - allocated_buffer: PyObjectRef, - #[allow(dead_code)] - address: SOCKADDR_IN6, // TODO: remove when done - address_length: libc::c_int, + allocated_buffer: PyBytesRef, + address: SOCKADDR_IN6, + address_length: i32, } - impl std::fmt::Debug for OverlappedReadFrom { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for OverlappedReadFrom { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("OverlappedReadFrom") .field("result", &self.result) .field("allocated_buffer", &self.allocated_buffer) - // .field("address", &self.address) .field("address_length", &self.address_length) .finish() } @@ -111,25 +239,37 @@ mod _overlapped { struct OverlappedReadFromInto { // A (number of bytes read, (host, port)) tuple - result: PyObjectRef, + result: Option<PyObjectRef>, /* Buffer passed by the user */ user_buffer: PyBuffer, - #[allow(dead_code)] - address: SOCKADDR_IN6, // TODO: remove when done - address_length: libc::c_int, + address: SOCKADDR_IN6, + address_length: i32, } - impl std::fmt::Debug for OverlappedReadFromInto { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for OverlappedReadFromInto { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("OverlappedReadFromInto") .field("result", &self.result) .field("user_buffer", &self.user_buffer) - // .field("address", &self.address) .field("address_length", &self.address_length) .finish() } } + struct OverlappedWriteTo { + buf: PyBuffer, + address: Vec<u8>, // Keep address alive during async operation + } + + impl core::fmt::Debug for OverlappedWriteTo { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedWriteTo") + .field("buf", &self.buf) + .field("address", &self.address.len()) + .finish() + } + } + fn mark_as_completed(ov: &mut OVERLAPPED) { ov.Internal = 0; if !ov.hEvent.is_null() { @@ -137,23 +277,142 @@ mod _overlapped { } } - fn from_windows_err(err: u32, vm: &VirtualMachine) -> PyBaseExceptionRef { - use Foundation::{ERROR_CONNECTION_ABORTED, ERROR_CONNECTION_REFUSED}; - debug_assert_ne!(err, 0, "call errno_err instead"); - let exc = match err { - ERROR_CONNECTION_REFUSED => vm.ctx.exceptions.connection_refused_error, - ERROR_CONNECTION_ABORTED => vm.ctx.exceptions.connection_aborted_error, - err => return std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm), + fn set_from_windows_err(err: u32, vm: &VirtualMachine) -> PyBaseExceptionRef { + let err = if err == 0 { + unsafe { GetLastError() } + } else { + err }; - // TODO: set errno and winerror - vm.new_exception_empty(exc.to_owned()) + let errno = crate::vm::common::os::winerror_to_errno(err as i32); + let message = std::io::Error::from_raw_os_error(err as i32).to_string(); + let exc = vm.new_errno_error(errno, message); + let _ = exc + .as_object() + .set_attr("winerror", err.to_pyobject(vm), vm); + exc.upcast() } fn HasOverlappedIoCompleted(overlapped: &OVERLAPPED) -> bool { overlapped.Internal != (Foundation::STATUS_PENDING as usize) } - #[pyclass(with(Constructor))] + /// Parse a Python address tuple to SOCKADDR + fn parse_address(addr_obj: &PyTupleRef, vm: &VirtualMachine) -> PyResult<(Vec<u8>, i32)> { + use windows_sys::Win32::Networking::WinSock::{WSAGetLastError, WSAStringToAddressW}; + + match addr_obj.len() { + 2 => { + // IPv4: (host, port) + let host: PyStrRef = addr_obj[0].clone().try_into_value(vm)?; + let port: u16 = addr_obj[1].clone().try_to_value(vm)?; + + let mut addr: SOCKADDR_IN = unsafe { core::mem::zeroed() }; + addr.sin_family = AF_INET; + + let host_wide: Vec<u16> = host.as_str().encode_utf16().chain([0]).collect(); + let mut addr_len = core::mem::size_of::<SOCKADDR_IN>() as i32; + + let ret = unsafe { + WSAStringToAddressW( + host_wide.as_ptr(), + AF_INET as i32, + core::ptr::null(), + &mut addr as *mut _ as *mut SOCKADDR, + &mut addr_len, + ) + }; + + if ret < 0 { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + // Restore port (WSAStringToAddressW overwrites it) + addr.sin_port = port.to_be(); + + let bytes = unsafe { + core::slice::from_raw_parts( + &addr as *const _ as *const u8, + core::mem::size_of::<SOCKADDR_IN>(), + ) + }; + Ok((bytes.to_vec(), addr_len)) + } + 4 => { + // IPv6: (host, port, flowinfo, scope_id) + let host: PyStrRef = addr_obj[0].clone().try_into_value(vm)?; + let port: u16 = addr_obj[1].clone().try_to_value(vm)?; + let flowinfo: u32 = addr_obj[2].clone().try_to_value(vm)?; + let scope_id: u32 = addr_obj[3].clone().try_to_value(vm)?; + + let mut addr: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + addr.sin6_family = AF_INET6; + + let host_wide: Vec<u16> = host.as_str().encode_utf16().chain([0]).collect(); + let mut addr_len = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + let ret = unsafe { + WSAStringToAddressW( + host_wide.as_ptr(), + AF_INET6 as i32, + core::ptr::null(), + &mut addr as *mut _ as *mut SOCKADDR, + &mut addr_len, + ) + }; + + if ret < 0 { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + // Restore fields that WSAStringToAddressW might overwrite + addr.sin6_port = port.to_be(); + addr.sin6_flowinfo = flowinfo; + addr.Anonymous.sin6_scope_id = scope_id; + + let bytes = unsafe { + core::slice::from_raw_parts( + &addr as *const _ as *const u8, + core::mem::size_of::<SOCKADDR_IN6>(), + ) + }; + Ok((bytes.to_vec(), addr_len)) + } + _ => Err(vm.new_value_error("illegal address_as_bytes argument".to_owned())), + } + } + + /// Parse a SOCKADDR_IN6 (which can also hold IPv4 addresses) to a Python address tuple + fn unparse_address(addr: &SOCKADDR_IN6, _addr_len: i32, vm: &VirtualMachine) -> PyResult { + use core::net::{Ipv4Addr, Ipv6Addr}; + + unsafe { + let family = addr.sin6_family; + if family == AF_INET { + // IPv4 address stored in SOCKADDR_IN6 structure + let addr_in = &*(addr as *const SOCKADDR_IN6 as *const SOCKADDR_IN); + let ip_bytes = addr_in.sin_addr.S_un.S_un_b; + let ip_str = + Ipv4Addr::new(ip_bytes.s_b1, ip_bytes.s_b2, ip_bytes.s_b3, ip_bytes.s_b4) + .to_string(); + let port = u16::from_be(addr_in.sin_port); + Ok((ip_str, port).to_pyobject(vm)) + } else if family == AF_INET6 { + // IPv6 address + let ip_bytes = addr.sin6_addr.u.Byte; + let ip_str = Ipv6Addr::from(ip_bytes).to_string(); + let port = u16::from_be(addr.sin6_port); + let flowinfo = u32::from_be(addr.sin6_flowinfo); + let scope_id = addr.Anonymous.sin6_scope_id; + Ok((ip_str, port, flowinfo, scope_id).to_pyobject(vm)) + } else { + Err(vm.new_value_error("recvfrom returned unsupported address family".to_owned())) + } + } + } + + #[pyclass(with(Constructor, Destructor))] impl Overlapped { #[pygetset] fn address(&self, _vm: &VirtualMachine) -> usize { @@ -168,139 +427,1189 @@ mod _overlapped { && !matches!(inner.data, OverlappedData::NotStarted) } - fn WSARecv_inner( - inner: &mut OverlappedInner, + #[pygetset] + fn error(&self, _vm: &VirtualMachine) -> u32 { + let inner = self.inner.lock(); + inner.error + } + + #[pygetset] + fn event(&self, _vm: &VirtualMachine) -> isize { + let inner = self.inner.lock(); + inner.overlapped.hEvent as isize + } + + #[pymethod] + fn cancel(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let inner = zelf.inner.lock(); + if matches!( + inner.data, + OverlappedData::NotStarted | OverlappedData::WaitNamedPipeAndConnect + ) { + return Ok(()); + } + let ret = if !HasOverlappedIoCompleted(&inner.overlapped) { + unsafe { + windows_sys::Win32::System::IO::CancelIoEx(inner.handle, &inner.overlapped) + } + } else { + 1 + }; + // CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between + if ret == 0 && unsafe { GetLastError() } != Foundation::ERROR_NOT_FOUND { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pymethod] + fn getresult(zelf: &Py<Self>, wait: OptionalArg<bool>, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + + let mut inner = zelf.inner.lock(); + let wait = wait.unwrap_or(false); + + // Check operation state + if matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation not yet attempted".to_owned())); + } + if matches!(inner.data, OverlappedData::NotStarted) { + return Err(vm.new_value_error("operation failed to start".to_owned())); + } + + // Get the result + let mut transferred: u32 = 0; + let ret = unsafe { + windows_sys::Win32::System::IO::GetOverlappedResult( + inner.handle, + &inner.overlapped, + &mut transferred, + if wait { 1 } else { 0 }, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + // Handle errors + match err { + ERROR_SUCCESS | ERROR_MORE_DATA => {} + ERROR_BROKEN_PIPE => { + let allow_broken_pipe = match &inner.data { + OverlappedData::Read(_) | OverlappedData::ReadInto(_) => true, + OverlappedData::ReadFrom(_) => true, + OverlappedData::ReadFromInto(rfi) => rfi.result.is_some(), + _ => false, + }; + if !allow_broken_pipe { + return Err(set_from_windows_err(err, vm)); + } + } + _ => return Err(set_from_windows_err(err, vm)), + } + + // Return result based on operation type + match &mut inner.data { + OverlappedData::Read(buf) => { + let len = buf.as_bytes().len(); + let result = if transferred as usize != len { + let resized = vm + .ctx + .new_bytes(buf.as_bytes()[..transferred as usize].to_vec()); + *buf = resized.clone(); + resized + } else { + buf.clone() + }; + Ok(result.into()) + } + OverlappedData::ReadFrom(rf) => { + let len = rf.allocated_buffer.as_bytes().len(); + let resized_buf = if transferred as usize != len { + let resized = vm.ctx.new_bytes( + rf.allocated_buffer.as_bytes()[..transferred as usize].to_vec(), + ); + rf.allocated_buffer = resized.clone(); + resized + } else { + rf.allocated_buffer.clone() + }; + let addr_tuple = unparse_address(&rf.address, rf.address_length, vm)?; + if let Some(result) = &rf.result { + return Ok(result.clone()); + } + let result = vm.ctx.new_tuple(vec![resized_buf.into(), addr_tuple]); + rf.result = Some(result.clone().into()); + Ok(result.into()) + } + OverlappedData::ReadFromInto(rfi) => { + let addr_tuple = unparse_address(&rfi.address, rfi.address_length, vm)?; + if let Some(result) = &rfi.result { + return Ok(result.clone()); + } + let result = vm + .ctx + .new_tuple(vec![vm.ctx.new_int(transferred).into(), addr_tuple]); + rfi.result = Some(result.clone().into()); + Ok(result.into()) + } + _ => Ok(vm.ctx.new_int(transferred).into()), + } + } + + // ReadFile + #[pymethod] + fn ReadFile(zelf: &Py<Self>, handle: isize, size: u32, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Storage::FileSystem::ReadFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + #[cfg(target_pointer_width = "32")] + let size = core::cmp::min(size, isize::MAX as u32); + + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; + let buf = vm.ctx.new_bytes(buf); + inner.handle = handle as HANDLE; + inner.data = OverlappedData::Read(buf.clone()); + + let mut nread: u32 = 0; + let ret = unsafe { + ReadFile( + handle as HANDLE, + buf.as_bytes().as_ptr() as *mut _, + size, + &mut nread, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ReadFileInto + #[pymethod] + fn ReadFileInto( + zelf: &Py<Self>, handle: isize, - buf: &[u8], - mut flags: u32, + buf: PyBuffer, vm: &VirtualMachine, ) -> PyResult { use windows_sys::Win32::Foundation::{ ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, }; + use windows_sys::Win32::Storage::FileSystem::ReadFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); + } - let wsabuf = windows_sys::Win32::Networking::WinSock::WSABUF { - buf: buf.as_ptr() as *mut _, - len: buf.len() as _, + // For async read, buffer must be contiguous - we can't use a temporary copy + // because Windows writes data directly to the buffer after this call returns + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); }; - let mut n_read: u32 = 0; - // TODO: optimization with MaybeUninit + + inner.data = OverlappedData::ReadInto(buf.clone()); + + let mut nread: u32 = 0; let ret = unsafe { - windows_sys::Win32::Networking::WinSock::WSARecv( - handle as _, - &wsabuf, - 1, - &mut n_read, - &mut flags, + ReadFile( + handle as HANDLE, + contiguous.as_ptr() as *mut _, + buf_len as u32, + &mut nread, &mut inner.overlapped, - None, ) }; - let err = if ret < 0 { - unsafe { windows_sys::Win32::Networking::WinSock::WSAGetLastError() as u32 } + + let err = if ret != 0 { + ERROR_SUCCESS } else { - Foundation::ERROR_SUCCESS + unsafe { GetLastError() } }; inner.error = err; + match err { ERROR_BROKEN_PIPE => { mark_as_completed(&mut inner.overlapped); - Err(from_windows_err(err, vm)) + Err(set_from_windows_err(err, vm)) } ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), - _ => Err(from_windows_err(err, vm)), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } } } + // WSARecv #[pymethod] fn WSARecv( zelf: &Py<Self>, handle: isize, size: u32, - flags: u32, + flags: OptionalArg<u32>, vm: &VirtualMachine, ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecv}; + let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { - return Err(vm.new_value_error("operation already attempted")); + return Err(vm.new_value_error("operation already attempted".to_owned())); } + let mut flags = flags.unwrap_or(0); + #[cfg(target_pointer_width = "32")] - let size = std::cmp::min(size, std::isize::MAX as _); + let size = core::cmp::min(size, isize::MAX as u32); - let buf = vec![0u8; std::cmp::max(size, 1) as usize]; + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; let buf = vm.ctx.new_bytes(buf); - inner.handle = handle as _; + inner.handle = handle as HANDLE; + inner.data = OverlappedData::Read(buf.clone()); - let r = Self::WSARecv_inner(&mut inner, handle as _, buf.as_bytes(), flags, vm); - inner.data = OverlappedData::Read(buf); - r - } + let wsabuf = WSABUF { + buf: buf.as_bytes().as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; - #[pymethod] - fn cancel(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { - let inner = zelf.inner.lock(); - if matches!( - inner.data, - OverlappedData::NotStarted | OverlappedData::WaitNamedPipeAndConnect - ) { - return Ok(()); - } - let ret = if !HasOverlappedIoCompleted(&inner.overlapped) { - unsafe { - windows_sys::Win32::System::IO::CancelIoEx(inner.handle, &inner.overlapped) - } + let ret = unsafe { + WSARecv( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } } else { - 1 + ERROR_SUCCESS }; - // CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between - if ret == 0 && unsafe { GetLastError() } != Foundation::ERROR_NOT_FOUND { - return Err(vm.new_last_os_error()); + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } } - Ok(()) } - } - impl Constructor for Overlapped { - type Args = (isize,); - - fn py_new( - _cls: &Py<PyType>, - (mut event,): Self::Args, + // WSARecvInto + #[pymethod] + fn WSARecvInto( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, vm: &VirtualMachine, - ) -> PyResult<Self> { - if event == INVALID_HANDLE_VALUE { - event = unsafe { - windows_sys::Win32::System::Threading::CreateEventA( - std::ptr::null(), - Foundation::TRUE, - Foundation::FALSE, - std::ptr::null(), - ) as isize - }; - if event == NULL { - return Err(vm.new_last_os_error()); - } + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecv}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); } - let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; - if event != NULL { - overlapped.hEvent = event as _; + let mut flags = flags; + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); } - let inner = OverlappedInner { - overlapped, - handle: NULL as _, - error: 0, - data: OverlappedData::None, + + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); }; - Ok(Overlapped { + + inner.data = OverlappedData::ReadInto(buf.clone()); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut nread: u32 = 0; + + let ret = unsafe { + WSARecv( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WriteFile + #[pymethod] + fn WriteFile( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Storage::FileSystem::WriteFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); + } + + // For async write, buffer must be contiguous - we can't use a temporary copy + // because Windows reads from the buffer after this call returns + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); + }; + + inner.data = OverlappedData::Write(buf.clone()); + + let mut written: u32 = 0; + let ret = unsafe { + WriteFile( + handle as HANDLE, + contiguous.as_ptr() as *const _, + buf_len as u32, + &mut written, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSASend + #[pymethod] + fn WSASend( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSASend}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); + } + + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); + }; + + inner.data = OverlappedData::Write(buf.clone()); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut written: u32 = 0; + + let ret = unsafe { + WSASend( + handle as _, + &wsabuf, + 1, + &mut written, + flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // AcceptEx + #[pymethod] + fn AcceptEx( + zelf: &Py<Self>, + listen_socket: isize, + accept_socket: isize, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + // Buffer size: local address + remote address + let size = core::mem::size_of::<SOCKADDR_IN6>() + 16; + let buf = vec![0u8; size * 2]; + let buf = vm.ctx.new_bytes(buf); + + inner.handle = listen_socket as HANDLE; + inner.data = OverlappedData::Accept(buf.clone()); + + let mut bytes_received: u32 = 0; + + type AcceptExFn = unsafe extern "system" fn( + sListenSocket: usize, + sAcceptSocket: usize, + lpOutputBuffer: *mut core::ffi::c_void, + dwReceiveDataLength: u32, + dwLocalAddressLength: u32, + dwRemoteAddressLength: u32, + lpdwBytesReceived: *mut u32, + lpOverlapped: *mut OVERLAPPED, + ) -> i32; + + let accept_ex: AcceptExFn = unsafe { core::mem::transmute(*ACCEPT_EX.get().unwrap()) }; + + let ret = unsafe { + accept_ex( + listen_socket as _, + accept_socket as _, + buf.as_bytes().as_ptr() as *mut _, + 0, + size as u32, + size as u32, + &mut bytes_received, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ConnectEx + #[pymethod] + fn ConnectEx( + zelf: &Py<Self>, + socket: isize, + address: PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + inner.handle = socket as HANDLE; + // Store addr_bytes in OverlappedData to keep it alive during async operation + inner.data = OverlappedData::Connect(addr_bytes); + + type ConnectExFn = unsafe extern "system" fn( + s: usize, + name: *const SOCKADDR, + namelen: i32, + lpSendBuffer: *const core::ffi::c_void, + dwSendDataLength: u32, + lpdwBytesSent: *mut u32, + lpOverlapped: *mut OVERLAPPED, + ) -> i32; + + let connect_ex: ConnectExFn = + unsafe { core::mem::transmute(*CONNECT_EX.get().unwrap()) }; + + // Get pointer to the stored address data + let addr_ptr = match &inner.data { + OverlappedData::Connect(bytes) => bytes.as_ptr(), + _ => unreachable!(), + }; + + let ret = unsafe { + connect_ex( + socket as _, + addr_ptr as *const SOCKADDR, + addr_len, + core::ptr::null(), + 0, + core::ptr::null_mut(), + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // DisconnectEx + #[pymethod] + fn DisconnectEx( + zelf: &Py<Self>, + socket: isize, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = socket as HANDLE; + inner.data = OverlappedData::Disconnect; + + type DisconnectExFn = unsafe extern "system" fn( + s: usize, + lpOverlapped: *mut OVERLAPPED, + dwFlags: u32, + dwReserved: u32, + ) -> i32; + + let disconnect_ex: DisconnectExFn = + unsafe { core::mem::transmute(*DISCONNECT_EX.get().unwrap()) }; + + let ret = unsafe { disconnect_ex(socket as _, &mut inner.overlapped, flags, 0) }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // TransmitFile + #[allow( + clippy::too_many_arguments, + reason = "mirrors Windows TransmitFile argument structure" + )] + #[pymethod] + fn TransmitFile( + zelf: &Py<Self>, + socket: isize, + file: isize, + offset: u32, + offset_high: u32, + count_to_write: u32, + count_per_send: u32, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = socket as HANDLE; + inner.data = OverlappedData::TransmitFile; + inner.overlapped.Anonymous.Anonymous.Offset = offset; + inner.overlapped.Anonymous.Anonymous.OffsetHigh = offset_high; + + type TransmitFileFn = unsafe extern "system" fn( + hSocket: usize, + hFile: HANDLE, + nNumberOfBytesToWrite: u32, + nNumberOfBytesPerSend: u32, + lpOverlapped: *mut OVERLAPPED, + lpTransmitBuffers: *const core::ffi::c_void, + dwReserved: u32, + ) -> i32; + + let transmit_file: TransmitFileFn = + unsafe { core::mem::transmute(*TRANSMIT_FILE.get().unwrap()) }; + + let ret = unsafe { + transmit_file( + socket as _, + file as HANDLE, + count_to_write, + count_per_send, + &mut inner.overlapped, + core::ptr::null(), + flags, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ConnectNamedPipe + #[pymethod] + fn ConnectNamedPipe(zelf: &Py<Self>, pipe: isize, vm: &VirtualMachine) -> PyResult<bool> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_PENDING, ERROR_PIPE_CONNECTED, ERROR_SUCCESS, + }; + use windows_sys::Win32::System::Pipes::ConnectNamedPipe; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + inner.handle = pipe as HANDLE; + inner.data = OverlappedData::ConnectNamedPipe; + + let ret = unsafe { ConnectNamedPipe(pipe as HANDLE, &mut inner.overlapped) }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_PIPE_CONNECTED => { + mark_as_completed(&mut inner.overlapped); + Ok(true) + } + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(false), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSASendTo + #[pymethod] + fn WSASendTo( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, + address: PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSASendTo}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); + } + + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); + }; + + // Store both buffer and address in OverlappedData to keep them alive + inner.data = OverlappedData::WriteTo(OverlappedWriteTo { + buf: buf.clone(), + address: addr_bytes, + }); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut written: u32 = 0; + + // Get pointer to the stored address data + let addr_ptr = match &inner.data { + OverlappedData::WriteTo(wt) => wt.address.as_ptr(), + _ => unreachable!(), + }; + + let ret = unsafe { + WSASendTo( + handle as _, + &wsabuf, + 1, + &mut written, + flags, + addr_ptr as *const SOCKADDR, + addr_len, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecvFrom + #[pymethod] + fn WSARecvFrom( + zelf: &Py<Self>, + handle: isize, + size: u32, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecvFrom}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + let mut flags = flags.unwrap_or(0); + + #[cfg(target_pointer_width = "32")] + let size = core::cmp::min(size, isize::MAX as u32); + + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; + let buf = vm.ctx.new_bytes(buf); + inner.handle = handle as HANDLE; + + let address: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + let address_length = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + inner.data = OverlappedData::ReadFrom(OverlappedReadFrom { + result: None, + allocated_buffer: buf.clone(), + address, + address_length, + }); + + let wsabuf = WSABUF { + buf: buf.as_bytes().as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; + + // Get mutable reference to address in inner.data + let (addr_ptr, addr_len_ptr) = match &mut inner.data { + OverlappedData::ReadFrom(rf) => ( + &mut rf.address as *mut SOCKADDR_IN6, + &mut rf.address_length as *mut i32, + ), + _ => unreachable!(), + }; + + let ret = unsafe { + WSARecvFrom( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + addr_ptr as *mut SOCKADDR, + addr_len_ptr, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecvFromInto + #[pymethod] + fn WSARecvFromInto( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + size: u32, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecvFrom}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted".to_owned())); + } + + let mut flags = flags.unwrap_or(0); + inner.handle = handle as HANDLE; + + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous".to_owned())); + }; + + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large".to_owned())); + } + + let address: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + let address_length = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + inner.data = OverlappedData::ReadFromInto(OverlappedReadFromInto { + result: None, + user_buffer: buf.clone(), + address, + address_length, + }); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; + + // Get mutable reference to address in inner.data + let (addr_ptr, addr_len_ptr) = match &mut inner.data { + OverlappedData::ReadFromInto(rfi) => ( + &mut rfi.address as *mut SOCKADDR_IN6, + &mut rfi.address_length as *mut i32, + ), + _ => unreachable!(), + }; + + let ret = unsafe { + WSARecvFrom( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + addr_ptr as *mut SOCKADDR, + addr_len_ptr, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + } + + impl Constructor for Overlapped { + type Args = (OptionalArg<isize>,); + + fn py_new(_cls: &Py<PyType>, (event,): Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut event = event.unwrap_or(INVALID_HANDLE_VALUE); + + if event == INVALID_HANDLE_VALUE { + event = unsafe { + windows_sys::Win32::System::Threading::CreateEventW( + core::ptr::null(), + Foundation::TRUE, + Foundation::FALSE, + core::ptr::null(), + ) as isize + }; + if event == NULL { + return Err(set_from_windows_err(0, vm)); + } + } + + let mut overlapped: OVERLAPPED = unsafe { core::mem::zeroed() }; + if event != NULL { + overlapped.hEvent = event as HANDLE; + } + let inner = OverlappedInner { + overlapped, + handle: NULL as HANDLE, + error: 0, + data: OverlappedData::None, + }; + Ok(Overlapped { inner: PyMutex::new(inner), }) } } - unsafe fn u64_to_handle(raw_ptr_value: u64) -> HANDLE { - raw_ptr_value as HANDLE + impl Destructor for Overlapped { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Foundation::{ + ERROR_NOT_FOUND, ERROR_OPERATION_ABORTED, ERROR_SUCCESS, + }; + use windows_sys::Win32::System::IO::{CancelIoEx, GetOverlappedResult}; + + let mut inner = zelf.inner.lock(); + let olderr = unsafe { GetLastError() }; + + // Cancel pending I/O and wait for completion + if !HasOverlappedIoCompleted(&inner.overlapped) + && !matches!(inner.data, OverlappedData::NotStarted) + { + let cancelled = unsafe { CancelIoEx(inner.handle, &inner.overlapped) } != 0; + let mut transferred: u32 = 0; + let ret = unsafe { + GetOverlappedResult( + inner.handle, + &inner.overlapped, + &mut transferred, + if cancelled { 1 } else { 0 }, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + match err { + ERROR_SUCCESS | ERROR_NOT_FOUND | ERROR_OPERATION_ABORTED => {} + _ => { + let msg = format!( + "{:?} still has pending operation at deallocation, the process may crash", + zelf + ); + let exc = vm.new_runtime_error(msg); + let err_msg = Some(format!( + "Exception ignored while deallocating overlapped operation {:?}", + zelf + )); + let obj: PyObjectRef = zelf.to_owned().into(); + vm.run_unraisable(exc, err_msg, obj); + } + } + } + + // Close the event handle + if !inner.overlapped.hEvent.is_null() { + unsafe { + Foundation::CloseHandle(inner.overlapped.hEvent); + } + inner.overlapped.hEvent = core::ptr::null_mut(); + } + + // Restore last error + unsafe { Foundation::SetLastError(olderr) }; + + Ok(()) + } + } + + #[pyfunction] + fn ConnectPipe(address: String, vm: &VirtualMachine) -> PyResult<isize> { + use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_OVERLAPPED, OPEN_EXISTING, + }; + + let address_wide: Vec<u16> = address.encode_utf16().chain(core::iter::once(0)).collect(); + + let handle = unsafe { + CreateFileW( + address_wide.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_OVERLAPPED, + core::ptr::null_mut(), + ) + }; + + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(set_from_windows_err(0, vm)); + } + + Ok(handle as isize) } #[pyfunction] @@ -313,14 +1622,14 @@ mod _overlapped { ) -> PyResult<isize> { let r = unsafe { windows_sys::Win32::System::IO::CreateIoCompletionPort( - handle as _, - port as _, + handle as HANDLE, + port as HANDLE, key, concurrency, ) as isize }; - if r as usize == 0 { - return Err(vm.new_last_os_error()); + if r == 0 { + return Err(set_from_windows_err(0, vm)); } Ok(r) } @@ -329,10 +1638,10 @@ mod _overlapped { fn GetQueuedCompletionStatus(port: isize, msecs: u32, vm: &VirtualMachine) -> PyResult { let mut bytes_transferred = 0; let mut completion_key = 0; - let mut overlapped: *mut OVERLAPPED = std::ptr::null_mut(); + let mut overlapped: *mut OVERLAPPED = core::ptr::null_mut(); let ret = unsafe { windows_sys::Win32::System::IO::GetQueuedCompletionStatus( - port as _, + port as HANDLE, &mut bytes_transferred, &mut completion_key, &mut overlapped, @@ -342,25 +1651,290 @@ mod _overlapped { let err = if ret != 0 { Foundation::ERROR_SUCCESS } else { - unsafe { Foundation::GetLastError() } + unsafe { GetLastError() } }; if overlapped.is_null() { if err == Foundation::WAIT_TIMEOUT { return Ok(vm.ctx.none()); } else { - return Err(vm.new_last_os_error()); + return Err(set_from_windows_err(err, vm)); } } let value = vm.ctx.new_tuple(vec![ err.to_pyobject(vm), - completion_key.to_pyobject(vm), bytes_transferred.to_pyobject(vm), + completion_key.to_pyobject(vm), (overlapped as usize).to_pyobject(vm), ]); Ok(value.into()) } + #[pyfunction] + fn PostQueuedCompletionStatus( + port: isize, + bytes: u32, + key: usize, + address: usize, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ret = unsafe { + windows_sys::Win32::System::IO::PostQueuedCompletionStatus( + port as HANDLE, + bytes, + key, + address as *mut OVERLAPPED, + ) + }; + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + // Registry to track callback data for proper cleanup + // Uses Arc for reference counting to prevent use-after-free when callback + // and UnregisterWait race - the data stays alive until both are done + static WAIT_CALLBACK_REGISTRY: std::sync::OnceLock< + std::sync::Mutex<std::collections::HashMap<isize, alloc::sync::Arc<PostCallbackData>>>, + > = std::sync::OnceLock::new(); + + fn wait_callback_registry() -> &'static std::sync::Mutex< + std::collections::HashMap<isize, alloc::sync::Arc<PostCallbackData>>, + > { + WAIT_CALLBACK_REGISTRY + .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) + } + + // Callback data for RegisterWaitWithQueue + // Uses Arc to ensure the data stays alive while callback is executing + struct PostCallbackData { + completion_port: HANDLE, + overlapped: *mut OVERLAPPED, + } + + // SAFETY: The pointers are handles/addresses passed from Python and are + // only used to call Windows APIs. They are not dereferenced as Rust pointers. + unsafe impl Send for PostCallbackData {} + unsafe impl Sync for PostCallbackData {} + + unsafe extern "system" fn post_to_queue_callback( + parameter: *mut core::ffi::c_void, + timer_or_wait_fired: bool, + ) { + // Reconstruct Arc from raw pointer - this gives us ownership of one reference + // The Arc prevents use-after-free since we own a reference count + let data = unsafe { alloc::sync::Arc::from_raw(parameter as *const PostCallbackData) }; + + unsafe { + let _ = windows_sys::Win32::System::IO::PostQueuedCompletionStatus( + data.completion_port, + if timer_or_wait_fired { 1 } else { 0 }, + 0, + data.overlapped, + ); + } + // Arc is dropped here, decrementing refcount + // Memory is freed only when all references (callback + registry) are gone + } + + #[pyfunction] + fn RegisterWaitWithQueue( + object: isize, + completion_port: isize, + overlapped: usize, + timeout: u32, + vm: &VirtualMachine, + ) -> PyResult<isize> { + use windows_sys::Win32::System::Threading::{ + RegisterWaitForSingleObject, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE, + }; + + let data = alloc::sync::Arc::new(PostCallbackData { + completion_port: completion_port as HANDLE, + overlapped: overlapped as *mut OVERLAPPED, + }); + + // Create raw pointer for the callback - this increments refcount + let data_ptr = alloc::sync::Arc::into_raw(data.clone()); + + let mut new_wait_object: HANDLE = core::ptr::null_mut(); + let ret = unsafe { + RegisterWaitForSingleObject( + &mut new_wait_object, + object as HANDLE, + Some(post_to_queue_callback), + data_ptr as *mut _, + timeout, + WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE, + ) + }; + + if ret == 0 { + // Registration failed - reconstruct Arc to drop the extra reference + unsafe { + let _ = alloc::sync::Arc::from_raw(data_ptr); + } + return Err(set_from_windows_err(0, vm)); + } + + // Store in registry for cleanup tracking + let wait_handle = new_wait_object as isize; + if let Ok(mut registry) = wait_callback_registry().lock() { + registry.insert(wait_handle, data); + } + + Ok(wait_handle) + } + + // Helper to cleanup callback data when unregistering + // Just removes from registry - Arc ensures memory stays alive if callback is running + fn cleanup_wait_callback_data(wait_handle: isize) { + if let Ok(mut registry) = wait_callback_registry().lock() { + // Removing from registry drops one Arc reference + // If callback already ran, this frees the memory + // If callback is still pending/running, it holds the other reference + registry.remove(&wait_handle); + } + } + + #[pyfunction] + fn UnregisterWait(wait_handle: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::UnregisterWait; + + let ret = unsafe { UnregisterWait(wait_handle as HANDLE) }; + // Cleanup callback data regardless of UnregisterWait result + // (callback may have already fired, or may never fire) + cleanup_wait_callback_data(wait_handle); + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pyfunction] + fn UnregisterWaitEx(wait_handle: isize, event: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::UnregisterWaitEx; + + let ret = unsafe { UnregisterWaitEx(wait_handle as HANDLE, event as HANDLE) }; + // Cleanup callback data regardless of UnregisterWaitEx result + cleanup_wait_callback_data(wait_handle); + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pyfunction] + fn BindLocal(socket: isize, family: i32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{ + INADDR_ANY, SOCKET_ERROR, WSAGetLastError, bind, + }; + + let ret = if family == AF_INET as i32 { + let mut addr: SOCKADDR_IN = unsafe { core::mem::zeroed() }; + addr.sin_family = AF_INET; + addr.sin_port = 0; + addr.sin_addr.S_un.S_addr = INADDR_ANY; + unsafe { + bind( + socket as _, + &addr as *const _ as *const SOCKADDR, + core::mem::size_of::<SOCKADDR_IN>() as i32, + ) + } + } else if family == AF_INET6 as i32 { + // in6addr_any is all zeros, which we have from zeroed() + let mut addr: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + addr.sin6_family = AF_INET6; + addr.sin6_port = 0; + unsafe { + bind( + socket as _, + &addr as *const _ as *const SOCKADDR, + core::mem::size_of::<SOCKADDR_IN6>() as i32, + ) + } + } else { + return Err(vm.new_value_error("expected tuple of length 2 or 4".to_owned())); + }; + + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + Ok(()) + } + + #[pyfunction] + fn FormatMessage(error_code: u32, _vm: &VirtualMachine) -> PyResult<String> { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::System::Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }; + + // LANG_NEUTRAL = 0, SUBLANG_DEFAULT = 1 + const LANG_NEUTRAL: u32 = 0; + const SUBLANG_DEFAULT: u32 = 1; + + let mut buffer: *mut u16 = core::ptr::null_mut(); + + let len = unsafe { + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + core::ptr::null(), + error_code, + (SUBLANG_DEFAULT << 10) | LANG_NEUTRAL, + &mut buffer as *mut _ as *mut u16, + 0, + core::ptr::null(), + ) + }; + + if len == 0 || buffer.is_null() { + if !buffer.is_null() { + unsafe { LocalFree(buffer as *mut _) }; + } + return Ok(format!("unknown error code {}", error_code)); + } + + // Convert to Rust string, trimming trailing whitespace + let slice = unsafe { core::slice::from_raw_parts(buffer, len as usize) }; + let msg = String::from_utf16_lossy(slice).trim_end().to_string(); + + unsafe { LocalFree(buffer as *mut _) }; + + Ok(msg) + } + + #[pyfunction] + fn WSAConnect(socket: isize, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{SOCKET_ERROR, WSAConnect, WSAGetLastError}; + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + let ret = unsafe { + WSAConnect( + socket as _, + addr_bytes.as_ptr() as *const SOCKADDR, + addr_len, + core::ptr::null(), + core::ptr::null_mut(), + core::ptr::null(), + core::ptr::null(), + ) + }; + + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + Ok(()) + } + #[pyfunction] fn CreateEvent( event_attributes: PyObjectRef, @@ -370,45 +1944,44 @@ mod _overlapped { vm: &VirtualMachine, ) -> PyResult<isize> { if !vm.is_none(&event_attributes) { - return Err(vm.new_value_error("EventAttributes must be None")); + return Err(vm.new_value_error("EventAttributes must be None".to_owned())); } - let name = match name { - Some(name) => { - let name = widestring::WideCString::from_str(&name).unwrap(); - name.as_ptr() - } - None => std::ptr::null(), - }; + let name_wide: Option<Vec<u16>> = + name.map(|n| n.encode_utf16().chain(core::iter::once(0)).collect()); + let name_ptr = name_wide + .as_ref() + .map(|n| n.as_ptr()) + .unwrap_or(core::ptr::null()); + let event = unsafe { windows_sys::Win32::System::Threading::CreateEventW( - std::ptr::null(), - manual_reset as _, - initial_state as _, - name, + core::ptr::null(), + if manual_reset { 1 } else { 0 }, + if initial_state { 1 } else { 0 }, + name_ptr, ) as isize }; if event == NULL { - return Err(vm.new_last_os_error()); + return Err(set_from_windows_err(0, vm)); } Ok(event) } #[pyfunction] - fn SetEvent(handle: u64, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { windows_sys::Win32::System::Threading::SetEvent(u64_to_handle(handle)) }; + fn SetEvent(handle: isize, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { windows_sys::Win32::System::Threading::SetEvent(handle as HANDLE) }; if ret == 0 { - return Err(vm.new_last_os_error()); + return Err(set_from_windows_err(0, vm)); } Ok(()) } #[pyfunction] - fn ResetEvent(handle: u64, vm: &VirtualMachine) -> PyResult<()> { - let ret = - unsafe { windows_sys::Win32::System::Threading::ResetEvent(u64_to_handle(handle)) }; + fn ResetEvent(handle: isize, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { windows_sys::Win32::System::Threading::ResetEvent(handle as HANDLE) }; if ret == 0 { - return Err(vm.new_last_os_error()); + return Err(set_from_windows_err(0, vm)); } Ok(()) } diff --git a/crates/stdlib/src/posixshmem.rs b/crates/stdlib/src/posixshmem.rs new file mode 100644 index 00000000000..2a142d8b6f3 --- /dev/null +++ b/crates/stdlib/src/posixshmem.rs @@ -0,0 +1,50 @@ +#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] +pub(crate) use _posixshmem::module_def; + +#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] +#[pymodule] +mod _posixshmem { + use alloc::ffi::CString; + + use crate::{ + common::os::errno_io_error, + vm::{FromArgs, PyResult, VirtualMachine, builtins::PyStrRef, convert::IntoPyException}, + }; + + #[derive(FromArgs)] + struct ShmOpenArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(any)] + flags: libc::c_int, + #[pyarg(any, default = 0o600)] + mode: libc::mode_t, + } + + #[pyfunction] + fn shm_open(args: ShmOpenArgs, vm: &VirtualMachine) -> PyResult<libc::c_int> { + let name = CString::new(args.name.as_str()).map_err(|e| e.into_pyexception(vm))?; + let mode: libc::c_uint = args.mode as _; + #[cfg(target_os = "freebsd")] + let mode = mode.try_into().unwrap(); + // SAFETY: `name` is a NUL-terminated string and `shm_open` does not write through it. + let fd = unsafe { libc::shm_open(name.as_ptr(), args.flags, mode) }; + if fd == -1 { + Err(errno_io_error().into_pyexception(vm)) + } else { + Ok(fd) + } + } + + #[pyfunction] + fn shm_unlink(name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let name = CString::new(name.as_str()).map_err(|e| e.into_pyexception(vm))?; + // SAFETY: `name` is a valid NUL-terminated string and `shm_unlink` only reads it. + let ret = unsafe { libc::shm_unlink(name.as_ptr()) }; + if ret == -1 { + Err(errno_io_error().into_pyexception(vm)) + } else { + Ok(()) + } + } +} diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index 4da6a6858dd..fec2ceb16d5 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -13,16 +13,16 @@ use nix::{ unistd::{self, Pid}, }; use std::{ - convert::Infallible as Never, - ffi::{CStr, CString}, io::prelude::*, - marker::PhantomData, - ops::Deref, os::fd::{AsFd, AsRawFd, BorrowedFd, IntoRawFd, OwnedFd, RawFd}, }; use unistd::{Gid, Uid}; -pub(crate) use _posixsubprocess::make_module; +use alloc::ffi::CString; + +use core::{convert::Infallible as Never, ffi::CStr, marker::PhantomData, ops::Deref}; + +pub(crate) use _posixsubprocess::module_def; #[pymodule] mod _posixsubprocess { @@ -33,9 +33,18 @@ mod _posixsubprocess { #[pyfunction] fn fork_exec(args: ForkExecArgs<'_>, vm: &VirtualMachine) -> PyResult<libc::pid_t> { - if args.preexec_fn.is_some() { - return Err(vm.new_not_implemented_error("preexec_fn not supported yet")); + // Check for interpreter shutdown when preexec_fn is used + if args.preexec_fn.is_some() + && vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_python_finalization_error( + "preexec_fn not supported at interpreter shutdown".to_owned(), + )); } + let extra_groups = args .groups_list .as_ref() @@ -49,7 +58,7 @@ mod _posixsubprocess { extra_groups: extra_groups.as_deref(), }; match unsafe { nix::unistd::fork() }.map_err(|err| err.into_pyexception(vm))? { - nix::unistd::ForkResult::Child => exec(&args, procargs), + nix::unistd::ForkResult::Child => exec(&args, procargs, vm), nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()), } } @@ -90,7 +99,7 @@ impl<'a, T: AsRef<CStr>> FromIterator<&'a T> for CharPtrVec<'a> { let vec = iter .into_iter() .map(|x| x.as_ref().as_ptr()) - .chain(std::iter::once(std::ptr::null())) + .chain(core::iter::once(core::ptr::null())) .collect(); Self { vec, @@ -217,7 +226,6 @@ gen_args! { uid: Option<Uid>, child_umask: i32, preexec_fn: Option<PyObjectRef>, - _use_vfork: bool, } // can't reallocate inside of exec(), so we reallocate prior to fork() and pass this along @@ -227,13 +235,19 @@ struct ProcArgs<'a> { extra_groups: Option<&'a [Gid]>, } -fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>) -> ! { +fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, vm: &VirtualMachine) -> ! { let mut ctx = ExecErrorContext::NoExec; - match exec_inner(args, procargs, &mut ctx) { + match exec_inner(args, procargs, &mut ctx, vm) { Ok(x) => match x {}, Err(e) => { let mut pipe = args.errpipe_write; - let _ = write!(pipe, "OSError:{}:{}", e as i32, ctx.as_msg()); + if matches!(ctx, ExecErrorContext::PreExec) { + // For preexec_fn errors, use SubprocessError format (errno=0) + let _ = write!(pipe, "SubprocessError:0:{}", ctx.as_msg()); + } else { + // errno is written in hex format + let _ = write!(pipe, "OSError:{:x}:{}", e as i32, ctx.as_msg()); + } std::process::exit(255) } } @@ -242,6 +256,7 @@ fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>) -> ! { enum ExecErrorContext { NoExec, ChDir, + PreExec, Exec, } @@ -250,6 +265,7 @@ impl ExecErrorContext { match self { Self::NoExec => "noexec", Self::ChDir => "noexec:chdir", + Self::PreExec => "Exception occurred in preexec_fn.", Self::Exec => "", } } @@ -259,6 +275,7 @@ fn exec_inner( args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, ctx: &mut ExecErrorContext, + vm: &VirtualMachine, ) -> nix::Result<Never> { for &fd in args.fds_to_keep.as_slice() { if fd.as_raw_fd() != args.errpipe_write.as_raw_fd() { @@ -315,11 +332,14 @@ fn exec_inner( } if args.child_umask >= 0 { - // TODO: umask(child_umask); + unsafe { libc::umask(args.child_umask as libc::mode_t) }; } if args.restore_signals { - // TODO: restore signals SIGPIPE, SIGXFZ, SIGXFSZ to SIG_DFL + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + libc::signal(libc::SIGXFSZ, libc::SIG_DFL); + } } if args.call_setsid { @@ -345,6 +365,18 @@ fn exec_inner( nix::Error::result(ret)?; } + // Call preexec_fn after all process setup but before closing FDs + if let Some(ref preexec_fn) = args.preexec_fn { + match preexec_fn.call((), vm) { + Ok(_) => {} + Err(_e) => { + // Cannot safely stringify exception after fork + *ctx = ExecErrorContext::PreExec; + return Err(Errno::UnknownErrno); + } + } + } + *ctx = ExecErrorContext::Exec; if args.close_fds { diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs index 871ba7d5987..7d603c72ed5 100644 --- a/crates/stdlib/src/pyexpat.rs +++ b/crates/stdlib/src/pyexpat.rs @@ -1,16 +1,8 @@ -/// Pyexpat builtin module -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, extend_module}; +//! Pyexpat builtin module -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _pyexpat::make_module(vm); +// spell-checker: ignore libexpat - extend_module!(vm, &module, { - "errors" => _errors::make_module(vm), - "model" => _model::make_module(vm), - }); - - module -} +pub(crate) use _pyexpat::module_def; macro_rules! create_property { ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { @@ -25,18 +17,54 @@ macro_rules! create_property { }; } +macro_rules! create_bool_property { + ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { + let attr = $ctx.new_static_getset( + $name, + $class, + move |this: &PyExpatLikeXmlParser| this.$element.read().clone(), + move |this: &PyExpatLikeXmlParser, + value: PyObjectRef, + vm: &VirtualMachine| + -> PyResult<()> { + let bool_value = value.is_true(vm)?; + *this.$element.write() = vm.ctx.new_bool(bool_value).into(); + Ok(()) + }, + ); + + $attributes.insert($ctx.intern_str($name), attr.into()); + }; +} + #[pymodule(name = "pyexpat")] mod _pyexpat { use crate::vm::{ Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyStr, PyStrRef, PyType}, - function::ArgBytesLike, - function::{IntoFuncArgs, OptionalArg}, + builtins::{PyBytesRef, PyException, PyModule, PyStr, PyStrRef, PyType}, + extend_module, + function::{ArgBytesLike, Either, IntoFuncArgs, OptionalArg}, + types::Constructor, }; use rustpython_common::lock::PyRwLock; use std::io::Cursor; use xml::reader::XmlEvent; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Add submodules + let model = super::_model::module_def(&vm.ctx).create_module(vm)?; + let errors = super::_errors::module_def(&vm.ctx).create_module(vm)?; + + extend_module!(vm, module, { + "model" => model, + "errors" => errors, + }); + + Ok(()) + } + type MutableObject = PyRwLock<PyObjectRef>; #[pyattr(name = "version_info")] @@ -46,11 +74,36 @@ mod _pyexpat { #[pyclass(name = "xmlparser", module = false, traverse)] #[derive(Debug, PyPayload)] pub struct PyExpatLikeXmlParser { + #[pytraverse(skip)] + namespace_separator: Option<String>, start_element: MutableObject, end_element: MutableObject, character_data: MutableObject, entity_decl: MutableObject, buffer_text: MutableObject, + namespace_prefixes: MutableObject, + ordered_attributes: MutableObject, + specified_attributes: MutableObject, + intern: MutableObject, + // Additional handlers (stubs for compatibility) + processing_instruction: MutableObject, + unparsed_entity_decl: MutableObject, + notation_decl: MutableObject, + start_namespace_decl: MutableObject, + end_namespace_decl: MutableObject, + comment: MutableObject, + start_cdata_section: MutableObject, + end_cdata_section: MutableObject, + default: MutableObject, + default_expand: MutableObject, + not_standalone: MutableObject, + external_entity_ref: MutableObject, + start_doctype_decl: MutableObject, + end_doctype_decl: MutableObject, + xml_decl: MutableObject, + element_decl: MutableObject, + attlist_decl: MutableObject, + skipped_entity: MutableObject, } type PyExpatLikeXmlParserRef = PyRef<PyExpatLikeXmlParser>; @@ -59,18 +112,49 @@ mod _pyexpat { where T: IntoFuncArgs, { - handler.read().call(args, vm).ok(); + // Clone the handler while holding the read lock, then release the lock + let handler = handler.read().clone(); + handler.call(args, vm).ok(); } #[pyclass] impl PyExpatLikeXmlParser { - fn new(vm: &VirtualMachine) -> PyResult<PyExpatLikeXmlParserRef> { + fn new( + namespace_separator: Option<String>, + intern: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyExpatLikeXmlParserRef> { + let intern_dict = intern.unwrap_or_else(|| vm.ctx.new_dict().into()); Ok(Self { + namespace_separator, start_element: MutableObject::new(vm.ctx.none()), end_element: MutableObject::new(vm.ctx.none()), character_data: MutableObject::new(vm.ctx.none()), entity_decl: MutableObject::new(vm.ctx.none()), buffer_text: MutableObject::new(vm.ctx.new_bool(false).into()), + namespace_prefixes: MutableObject::new(vm.ctx.new_bool(false).into()), + ordered_attributes: MutableObject::new(vm.ctx.new_bool(false).into()), + specified_attributes: MutableObject::new(vm.ctx.new_bool(false).into()), + intern: MutableObject::new(intern_dict), + // Additional handlers (stubs for compatibility) + processing_instruction: MutableObject::new(vm.ctx.none()), + unparsed_entity_decl: MutableObject::new(vm.ctx.none()), + notation_decl: MutableObject::new(vm.ctx.none()), + start_namespace_decl: MutableObject::new(vm.ctx.none()), + end_namespace_decl: MutableObject::new(vm.ctx.none()), + comment: MutableObject::new(vm.ctx.none()), + start_cdata_section: MutableObject::new(vm.ctx.none()), + end_cdata_section: MutableObject::new(vm.ctx.none()), + default: MutableObject::new(vm.ctx.none()), + default_expand: MutableObject::new(vm.ctx.none()), + not_standalone: MutableObject::new(vm.ctx.none()), + external_entity_ref: MutableObject::new(vm.ctx.none()), + start_doctype_decl: MutableObject::new(vm.ctx.none()), + end_doctype_decl: MutableObject::new(vm.ctx.none()), + xml_decl: MutableObject::new(vm.ctx.none()), + element_decl: MutableObject::new(vm.ctx.none()), + attlist_decl: MutableObject::new(vm.ctx.none()), + skipped_entity: MutableObject::new(vm.ctx.none()), } .into_ref(&vm.ctx)) } @@ -89,7 +173,120 @@ mod _pyexpat { character_data ); create_property!(ctx, attributes, "EntityDeclHandler", class, entity_decl); - create_property!(ctx, attributes, "buffer_text", class, buffer_text); + create_bool_property!(ctx, attributes, "buffer_text", class, buffer_text); + create_bool_property!( + ctx, + attributes, + "namespace_prefixes", + class, + namespace_prefixes + ); + create_bool_property!( + ctx, + attributes, + "ordered_attributes", + class, + ordered_attributes + ); + create_bool_property!( + ctx, + attributes, + "specified_attributes", + class, + specified_attributes + ); + create_property!(ctx, attributes, "intern", class, intern); + // Additional handlers (stubs for compatibility) + create_property!( + ctx, + attributes, + "ProcessingInstructionHandler", + class, + processing_instruction + ); + create_property!( + ctx, + attributes, + "UnparsedEntityDeclHandler", + class, + unparsed_entity_decl + ); + create_property!(ctx, attributes, "NotationDeclHandler", class, notation_decl); + create_property!( + ctx, + attributes, + "StartNamespaceDeclHandler", + class, + start_namespace_decl + ); + create_property!( + ctx, + attributes, + "EndNamespaceDeclHandler", + class, + end_namespace_decl + ); + create_property!(ctx, attributes, "CommentHandler", class, comment); + create_property!( + ctx, + attributes, + "StartCdataSectionHandler", + class, + start_cdata_section + ); + create_property!( + ctx, + attributes, + "EndCdataSectionHandler", + class, + end_cdata_section + ); + create_property!(ctx, attributes, "DefaultHandler", class, default); + create_property!( + ctx, + attributes, + "DefaultHandlerExpand", + class, + default_expand + ); + create_property!( + ctx, + attributes, + "NotStandaloneHandler", + class, + not_standalone + ); + create_property!( + ctx, + attributes, + "ExternalEntityRefHandler", + class, + external_entity_ref + ); + create_property!( + ctx, + attributes, + "StartDoctypeDeclHandler", + class, + start_doctype_decl + ); + create_property!( + ctx, + attributes, + "EndDoctypeDeclHandler", + class, + end_doctype_decl + ); + create_property!(ctx, attributes, "XmlDeclHandler", class, xml_decl); + create_property!(ctx, attributes, "ElementDeclHandler", class, element_decl); + create_property!(ctx, attributes, "AttlistDeclHandler", class, attlist_decl); + create_property!( + ctx, + attributes, + "SkippedEntityHandler", + class, + skipped_entity + ); } fn create_config(&self) -> xml::ParserConfig { @@ -99,7 +296,19 @@ mod _pyexpat { .whitespace_to_characters(true) } - fn do_parse<T>(&self, vm: &VirtualMachine, parser: xml::EventReader<T>) + /// Construct element name with namespace if separator is set + fn make_name(&self, name: &xml::name::OwnedName) -> String { + match (&self.namespace_separator, &name.namespace) { + (Some(sep), Some(ns)) => format!("{}{}{}", ns, sep, name.local_name), + _ => name.local_name.clone(), + } + } + + fn do_parse<T>( + &self, + vm: &VirtualMachine, + parser: xml::EventReader<T>, + ) -> Result<(), xml::reader::Error> where T: std::io::Read, { @@ -110,70 +319,118 @@ mod _pyexpat { }) => { let dict = vm.ctx.new_dict(); for attribute in attributes { + let attr_name = self.make_name(&attribute.name); dict.set_item( - attribute.name.local_name.as_str(), + attr_name.as_str(), vm.ctx.new_str(attribute.value).into(), vm, ) .unwrap(); } - let name_str = PyStr::from(name.local_name).into_ref(&vm.ctx); + let name_str = PyStr::from(self.make_name(&name)).into_ref(&vm.ctx); invoke_handler(vm, &self.start_element, (name_str, dict)); } Ok(XmlEvent::EndElement { name, .. }) => { - let name_str = PyStr::from(name.local_name).into_ref(&vm.ctx); + let name_str = PyStr::from(self.make_name(&name)).into_ref(&vm.ctx); invoke_handler(vm, &self.end_element, (name_str,)); } Ok(XmlEvent::Characters(chars)) => { let str = PyStr::from(chars).into_ref(&vm.ctx); invoke_handler(vm, &self.character_data, (str,)); } + Err(e) => return Err(e), _ => {} } } + Ok(()) } #[pymethod(name = "Parse")] - fn parse(&self, data: PyStrRef, _isfinal: OptionalArg<bool>, vm: &VirtualMachine) { - let reader = Cursor::<Vec<u8>>::new(data.as_bytes().to_vec()); + fn parse( + &self, + data: Either<PyStrRef, PyBytesRef>, + _isfinal: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult<i32> { + let bytes = match data { + Either::A(s) => s.as_bytes().to_vec(), + Either::B(b) => b.as_bytes().to_vec(), + }; + // Empty data is valid - used to finalize parsing + if bytes.is_empty() { + return Ok(1); + } + let reader = Cursor::<Vec<u8>>::new(bytes); let parser = self.create_config().create_reader(reader); - self.do_parse(vm, parser); + // Note: xml-rs is stricter than libexpat; some errors are silently ignored + // to maintain compatibility with existing Python code + let _ = self.do_parse(vm, parser); + Ok(1) } #[pymethod(name = "ParseFile")] - fn parse_file(&self, file: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // todo: read chunks at a time + fn parse_file(&self, file: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { let read_res = vm.call_method(&file, "read", ())?; let bytes_like = ArgBytesLike::try_from_object(vm, read_res)?; let buf = bytes_like.borrow_buf().to_vec(); + if buf.is_empty() { + return Ok(1); + } let reader = Cursor::new(buf); let parser = self.create_config().create_reader(reader); - self.do_parse(vm, parser); - - // todo: return value - Ok(()) + // Note: xml-rs is stricter than libexpat; some errors are silently ignored + let _ = self.do_parse(vm, parser); + Ok(1) } } #[derive(FromArgs)] - #[allow(dead_code)] struct ParserCreateArgs { #[pyarg(any, optional)] - encoding: OptionalArg<PyStrRef>, + encoding: Option<PyStrRef>, #[pyarg(any, optional)] - namespace_separator: OptionalArg<PyStrRef>, + namespace_separator: Option<PyStrRef>, #[pyarg(any, optional)] - intern: OptionalArg<PyStrRef>, + intern: Option<PyObjectRef>, } #[pyfunction(name = "ParserCreate")] fn parser_create( - _args: ParserCreateArgs, + args: ParserCreateArgs, vm: &VirtualMachine, ) -> PyResult<PyExpatLikeXmlParserRef> { - PyExpatLikeXmlParser::new(vm) + // Validate namespace_separator: must be at most one character + let ns_sep = match args.namespace_separator { + Some(ref s) => { + let chars: Vec<char> = s.as_str().chars().collect(); + if chars.len() > 1 { + return Err(vm.new_value_error( + "namespace_separator must be at most one character, omitted, or None" + .to_owned(), + )); + } + Some(s.as_str().to_owned()) + } + None => None, + }; + + // encoding parameter is currently not used (xml-rs handles encoding from XML declaration) + let _ = args.encoding; + + PyExpatLikeXmlParser::new(ns_sep, args.intern, vm) } + + // TODO: Tie this exception to the module's state. + #[pyattr] + #[pyattr(name = "error")] + #[pyexception(name = "ExpatError", base = PyException)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyExpatError(PyException); + + #[pyexception] + impl PyExpatError {} } #[pymodule(name = "model")] diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index 798e5f5de80..d3be417edb3 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -5,7 +5,7 @@ //! Use this rust module to do byte packing: //! <https://docs.rs/byteorder/1.2.6/byteorder/> -pub(crate) use _struct::make_module; +pub(crate) use _struct::module_def; #[pymodule] pub(crate) mod _struct { @@ -16,7 +16,7 @@ pub(crate) mod _struct { function::{ArgBytesLike, ArgMemoryBuffer, PosArgs}, match_class, protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, Representable, SelfIter, Unconstructible}, + types::{Constructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; @@ -28,7 +28,7 @@ pub(crate) mod _struct { // CPython turns str to bytes but we do reversed way here // The only performance difference is this transition cost let fmt = match_class!(match obj { - s @ PyStr => s.is_ascii().then_some(s), + s @ PyStr => s.isascii().then_some(s), b @ PyBytes => ascii::AsciiStr::from_ascii(&b) .ok() .map(|s| vm.ctx.new_str(s)), @@ -71,7 +71,7 @@ pub(crate) mod _struct { } else { ("unpack_from", "unpacking") }; - if offset >= buffer_len { + if offset + needed > buffer_len { let msg = format!( "{op} requires a buffer of at least {required} bytes for {op_action} {needed} \ bytes at offset {offset} (actual buffer size is {buffer_len})", @@ -189,7 +189,7 @@ pub(crate) mod _struct { } } - #[pyclass(with(Unconstructible, IterNext, Iterable))] + #[pyclass(with(IterNext, Iterable), flags(DISALLOW_INSTANTIATION))] impl UnpackIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -197,7 +197,7 @@ pub(crate) mod _struct { } } impl SelfIter for UnpackIterator {} - impl Unconstructible for UnpackIterator {} + impl IterNext for UnpackIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { let size = zelf.format_spec.size; diff --git a/crates/stdlib/src/random.rs b/crates/stdlib/src/random.rs index be31d3011d7..35e6473d8f6 100644 --- a/crates/stdlib/src/random.rs +++ b/crates/stdlib/src/random.rs @@ -1,6 +1,6 @@ //! Random module. -pub(crate) use _random::make_module; +pub(crate) use _random::module_def; #[pymodule] mod _random { diff --git a/crates/stdlib/src/re.rs b/crates/stdlib/src/re.rs index 5af45671522..fdb14d427fc 100644 --- a/crates/stdlib/src/re.rs +++ b/crates/stdlib/src/re.rs @@ -1,4 +1,4 @@ -pub(crate) use re::make_module; +pub(crate) use re::module_def; #[pymodule] mod re { diff --git a/crates/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs index 052f45e0cad..34c8161e0cd 100644 --- a/crates/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use resource::make_module; +pub(crate) use resource::module_def; #[pymodule] mod resource { @@ -9,7 +9,8 @@ mod resource { convert::{ToPyException, ToPyObject}, types::PyStructSequence, }; - use std::{io, mem}; + use core::mem; + use std::io; cfg_if::cfg_if! { if #[cfg(target_os = "android")] { diff --git a/crates/stdlib/src/scproxy.rs b/crates/stdlib/src/scproxy.rs index f49b6890a69..09e7cdc6046 100644 --- a/crates/stdlib/src/scproxy.rs +++ b/crates/stdlib/src/scproxy.rs @@ -1,12 +1,12 @@ -pub(crate) use _scproxy::make_module; +pub(crate) use _scproxy::module_def; #[pymodule] mod _scproxy { // straight-forward port of Modules/_scproxy.c use crate::vm::{ - PyResult, VirtualMachine, - builtins::{PyDictRef, PyStr}, + Py, PyResult, VirtualMachine, + builtins::{PyDict, PyDictRef, PyStr}, convert::ToPyObject, }; use system_configuration::core_foundation::{ @@ -22,7 +22,7 @@ mod _scproxy { fn proxy_dict() -> Option<CFDictionary<CFString, CFType>> { // Py_BEGIN_ALLOW_THREADS - let proxy_dict = unsafe { SCDynamicStoreCopyProxies(std::ptr::null()) }; + let proxy_dict = unsafe { SCDynamicStoreCopyProxies(core::ptr::null()) }; // Py_END_ALLOW_THREADS if proxy_dict.is_null() { None @@ -74,7 +74,7 @@ mod _scproxy { let result = vm.ctx.new_dict(); - let set_proxy = |result: &PyDictRef, + let set_proxy = |result: &Py<PyDict>, proto: &str, enabled_key: CFStringRef, host_key: CFStringRef, @@ -91,7 +91,7 @@ mod _scproxy { .find(host_key) .and_then(|v| v.downcast::<CFString>()) { - let h = std::borrow::Cow::<str>::from(&host); + let h = alloc::borrow::Cow::<str>::from(&host); let v = if let Some(port) = proxy_dict .find(port_key) .and_then(|v| v.downcast::<CFNumber>()) diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index 5639a66d2cc..bc8aded5478 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -1,23 +1,12 @@ // spell-checker:disable +pub(crate) use decl::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, - builtins::PyModule, + PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, }; -use std::{io, mem}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - - #[cfg(unix)] - { - use crate::vm::class::PyClassImpl; - decl::poll::PyPoll::make_class(&vm.ctx); - } - - decl::make_module(vm) -} +use core::mem; +use std::io; #[cfg(unix)] mod platform { @@ -72,7 +61,7 @@ mod platform { #[cfg(target_os = "wasi")] mod platform { pub use libc::{FD_SETSIZE, timeval}; - pub use std::os::wasi::io::RawFd; + pub use std::os::fd::RawFd; pub fn check_err(x: i32) -> bool { x < 0 @@ -158,7 +147,7 @@ impl FdSet { pub fn new() -> Self { // it's just ints, and all the code that's actually // interacting with it is in C, so it's safe to zero - let mut fdset = std::mem::MaybeUninit::zeroed(); + let mut fdset = core::mem::MaybeUninit::zeroed(); unsafe { platform::FD_ZERO(fdset.as_mut_ptr()) }; Self(fdset) } @@ -191,7 +180,7 @@ pub fn select( ) -> io::Result<i32> { let timeout = match timeout { Some(tv) => tv as *mut timeval, - None => std::ptr::null_mut(), + None => core::ptr::null_mut(), }; let ret = unsafe { platform::select( @@ -220,13 +209,27 @@ fn sec_to_timeval(sec: f64) -> timeval { mod decl { use super::*; use crate::vm::{ - PyObjectRef, PyResult, VirtualMachine, - builtins::PyTypeRef, + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyModule, PyTypeRef}, convert::ToPyException, function::{Either, OptionalOption}, stdlib::time, }; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + #[cfg(unix)] + { + use crate::vm::class::PyClassImpl; + poll::PyPoll::make_class(&vm.ctx); + } + + __module_exec(vm, module); + Ok(()) + } + #[pyattr] fn error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.exceptions.os_error.to_owned() @@ -336,12 +339,10 @@ mod decl { function::OptionalArg, stdlib::io::Fildes, }; + use core::{convert::TryFrom, time::Duration}; use libc::pollfd; use num_traits::{Signed, ToPrimitive}; - use std::{ - convert::TryFrom, - time::{Duration, Instant}, - }; + use std::time::Instant; #[derive(Default)] pub(super) struct TimeoutArg<const MILLIS: bool>(pub Option<Duration>); @@ -546,7 +547,7 @@ mod decl { pub(super) mod epoll { use super::*; use crate::vm::{ - Py, PyPayload, + Py, PyPayload, PyRef, builtins::PyType, common::lock::{PyRwLock, PyRwLockReadGuard}, convert::{IntoPyException, ToPyObject}, @@ -554,8 +555,8 @@ mod decl { stdlib::io::Fildes, types::Constructor, }; + use core::ops::Deref; use rustix::event::epoll::{self, EventData, EventFlags}; - use std::ops::Deref; use std::os::fd::{AsRawFd, IntoRawFd, OwnedFd}; use std::time::Instant; diff --git a/crates/stdlib/src/sha1.rs b/crates/stdlib/src/sha1.rs index 04845bb76b5..3e3d4928c79 100644 --- a/crates/stdlib/src/sha1.rs +++ b/crates/stdlib/src/sha1.rs @@ -1,4 +1,4 @@ -pub(crate) use _sha1::make_module; +pub(crate) use _sha1::module_def; #[pymodule] mod _sha1 { @@ -7,6 +7,6 @@ mod _sha1 { #[pyfunction] fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha1(args).into_pyobject(vm)) + Ok(local_sha1(args, vm)?.into_pyobject(vm)) } } diff --git a/crates/stdlib/src/sha256.rs b/crates/stdlib/src/sha256.rs index 5d031968aeb..b4c26dc0dd6 100644 --- a/crates/stdlib/src/sha256.rs +++ b/crates/stdlib/src/sha256.rs @@ -1,22 +1,23 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let _ = vm.import("_hashlib", 0); - _sha256::make_module(vm) -} - #[pymodule] mod _sha256 { use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha224(args).into_pyobject(vm)) + Ok(local_sha224(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha256(args).into_pyobject(vm)) + Ok(local_sha256(args, vm)?.into_pyobject(vm)) + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) } } + +pub(crate) use _sha256::module_def; diff --git a/crates/stdlib/src/sha3.rs b/crates/stdlib/src/sha3.rs index 07b61d9aed2..0eb2dfa84d5 100644 --- a/crates/stdlib/src/sha3.rs +++ b/crates/stdlib/src/sha3.rs @@ -1,4 +1,4 @@ -pub(crate) use _sha3::make_module; +pub(crate) use _sha3::module_def; #[pymodule] mod _sha3 { @@ -10,31 +10,31 @@ mod _sha3 { #[pyfunction] fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_224(args).into_pyobject(vm)) + Ok(local_sha3_224(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_256(args).into_pyobject(vm)) + Ok(local_sha3_256(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_384(args).into_pyobject(vm)) + Ok(local_sha3_384(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_512(args).into_pyobject(vm)) + Ok(local_sha3_512(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_shake_128(args).into_pyobject(vm)) + Ok(local_shake_128(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_shake_256(args).into_pyobject(vm)) + Ok(local_shake_256(args, vm)?.into_pyobject(vm)) } } diff --git a/crates/stdlib/src/sha512.rs b/crates/stdlib/src/sha512.rs index baf63fdacf0..b7c6f02ed66 100644 --- a/crates/stdlib/src/sha512.rs +++ b/crates/stdlib/src/sha512.rs @@ -1,22 +1,23 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let _ = vm.import("_hashlib", 0); - _sha512::make_module(vm) -} - #[pymodule] mod _sha512 { use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha384(args).into_pyobject(vm)) + Ok(local_sha384(args, vm)?.into_pyobject(vm)) } #[pyfunction] fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha512(args).into_pyobject(vm)) + Ok(local_sha512(args, vm)?.into_pyobject(vm)) + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) } } + +pub(crate) use _sha512::module_def; diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index 08b05b56aa8..8c307eb54ec 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -1,36 +1,48 @@ // spell-checker:disable -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _socket::module_def; + #[cfg(feature = "ssl")] pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - _socket::make_module(vm) -} - #[pymodule] mod _socket { use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; use crate::vm::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyListRef, PyOSError, PyStrRef, PyTupleRef, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyListRef, PyModule, PyOSError, PyStrRef, PyTupleRef, PyTypeRef, + }, common::os::ErrorExt, convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, - function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption}, + function::{ + ArgBytesLike, ArgIntoFloat, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, + OptionalArg, OptionalOption, + }, types::{Constructor, DefaultConstructor, Initializer, Representable}, utils::ToCString, }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + __module_exec(vm, module); + Ok(()) + } + use core::{ + mem::MaybeUninit, + net::{Ipv4Addr, Ipv6Addr, SocketAddr}, + time::Duration, + }; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; use socket2::Socket; use std::{ ffi, io::{self, Read, Write}, - mem::MaybeUninit, - net::{self, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, ToSocketAddrs}, - time::{Duration, Instant}, + net::{self, Shutdown, ToSocketAddrs}, + time::Instant, }; #[cfg(unix)] @@ -58,10 +70,14 @@ mod _socket { NI_MAXHOST, NI_MAXSERV, NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, RCVALL_IPLEVEL, RCVALL_OFF, RCVALL_ON, RCVALL_SOCKETLEVELONLY, SD_BOTH as SHUT_RDWR, SD_RECEIVE as SHUT_RD, SD_SEND as SHUT_WR, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, - SIO_RCVALL, SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, - SO_USELOOPBACK, SOCK_DGRAM, SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET, SOCK_STREAM, - SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, WSAECONNRESET, WSAENOTSOCK, - WSAEWOULDBLOCK, + SIO_RCVALL, SO_BROADCAST, SO_ERROR, SO_KEEPALIVE, SO_LINGER, SO_OOBINLINE, SO_RCVBUF, + SO_REUSEADDR, SO_SNDBUF, SO_TYPE, SO_USELOOPBACK, SOCK_DGRAM, SOCK_RAW, SOCK_RDM, + SOCK_SEQPACKET, SOCK_STREAM, SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, + WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + INVALID_SOCKET, SOCKET_ERROR, WSA_FLAG_OVERLAPPED, WSADuplicateSocketW, + WSAGetLastError, WSAIoctl, WSAPROTOCOL_INFOW, WSASocketW, }; pub use windows_sys::Win32::Networking::WinSock::{ SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, @@ -82,6 +98,7 @@ mod _socket { pub const AI_PASSIVE: i32 = windows_sys::Win32::Networking::WinSock::AI_PASSIVE as _; pub const AI_NUMERICHOST: i32 = windows_sys::Win32::Networking::WinSock::AI_NUMERICHOST as _; + pub const FROM_PROTOCOL_INFO: i32 = -1; } // constants #[pyattr(name = "has_ipv6")] @@ -93,8 +110,8 @@ mod _socket { IPPROTO_ICMPV6, IPPROTO_IP, IPPROTO_IPV6, IPPROTO_TCP, IPPROTO_TCP as SOL_TCP, IPPROTO_UDP, MSG_CTRUNC, MSG_DONTROUTE, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, SHUT_RD, SHUT_RDWR, SHUT_WR, - SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, SOCK_DGRAM, - SOCK_STREAM, SOL_SOCKET, TCP_NODELAY, + SO_BROADCAST, SO_ERROR, SO_KEEPALIVE, SO_LINGER, SO_OOBINLINE, SO_RCVBUF, SO_REUSEADDR, + SO_SNDBUF, SO_TYPE, SOCK_DGRAM, SOCK_STREAM, SOL_SOCKET, TCP_NODELAY, }; #[cfg(not(target_os = "redox"))] @@ -138,6 +155,82 @@ mod _socket { SOL_CAN_RAW, }; + // CAN BCM opcodes + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_SETUP: i32 = 1; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_DELETE: i32 = 2; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_READ: i32 = 3; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_SEND: i32 = 4; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_SETUP: i32 = 5; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_DELETE: i32 = 6; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_READ: i32 = 7; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_STATUS: i32 = 8; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_EXPIRED: i32 = 9; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_STATUS: i32 = 10; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_TIMEOUT: i32 = 11; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_CHANGED: i32 = 12; + + // CAN BCM flags (linux/can/bcm.h) + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_SETTIMER: i32 = 0x0001; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_STARTTIMER: i32 = 0x0002; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_COUNTEVT: i32 = 0x0004; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_ANNOUNCE: i32 = 0x0008; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_CP_CAN_ID: i32 = 0x0010; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_FILTER_ID: i32 = 0x0020; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_CHECK_DLC: i32 = 0x0040; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_NO_AUTOTIMER: i32 = 0x0080; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_ANNOUNCE_RESUME: i32 = 0x0100; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_RESET_MULTI_IDX: i32 = 0x0200; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_RTR_FRAME: i32 = 0x0400; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_CAN_FD_FRAME: i32 = 0x0800; + #[cfg(all(target_os = "linux", target_env = "gnu"))] #[pyattr] use c::SOL_RDS; @@ -150,6 +243,48 @@ mod _socket { #[pyattr] use c::{AF_SYSTEM, PF_SYSTEM, SYSPROTO_CONTROL, TCP_KEEPALIVE}; + // RFC3542 IPv6 socket options for macOS (netinet6/in6.h) + // Not available in libc, define manually + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVHOPLIMIT: i32 = 37; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVRTHDR: i32 = 38; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVHOPOPTS: i32 = 39; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVDSTOPTS: i32 = 40; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_USE_MIN_MTU: i32 = 42; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVPATHMTU: i32 = 43; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_PATHMTU: i32 = 44; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_NEXTHOP: i32 = 48; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_HOPOPTS: i32 = 49; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_DSTOPTS: i32 = 50; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDR: i32 = 51; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDRDSTOPTS: i32 = 57; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDR_TYPE_0: i32 = 0; + #[cfg(windows)] #[pyattr] use c::{ @@ -168,8 +303,8 @@ mod _socket { #[cfg(any(unix, target_os = "android"))] #[pyattr] use c::{ - EAI_SYSTEM, MSG_EOR, SO_ACCEPTCONN, SO_DEBUG, SO_DONTROUTE, SO_KEEPALIVE, SO_RCVBUF, - SO_RCVLOWAT, SO_RCVTIMEO, SO_SNDBUF, SO_SNDLOWAT, SO_SNDTIMEO, + EAI_SYSTEM, MSG_EOR, SO_ACCEPTCONN, SO_DEBUG, SO_DONTROUTE, SO_RCVLOWAT, SO_RCVTIMEO, + SO_SNDLOWAT, SO_SNDTIMEO, }; #[cfg(any(target_os = "android", target_os = "linux"))] @@ -415,6 +550,7 @@ mod _socket { target_os = "dragonfly", target_os = "freebsd", target_os = "linux", + target_vendor = "apple", windows ))] #[pyattr] @@ -783,6 +919,21 @@ mod _socket { .map(|sock| sock as RawSocket) } + #[cfg(target_os = "linux")] + #[derive(FromArgs)] + struct SendmsgAfalgArgs { + #[pyarg(any, default)] + msg: Vec<ArgBytesLike>, + #[pyarg(named)] + op: u32, + #[pyarg(named, default)] + iv: Option<ArgBytesLike>, + #[pyarg(named, default)] + assoclen: OptionalArg<isize>, + #[pyarg(named, default)] + flags: i32, + } + #[pyattr(name = "socket")] #[pyattr(name = "SocketType")] #[pyclass(name = "socket")] @@ -795,7 +946,7 @@ mod _socket { sock: PyRwLock<Option<Socket>>, } - const _: () = assert!(std::mem::size_of::<Option<Socket>>() == std::mem::size_of::<Socket>()); + const _: () = assert!(core::mem::size_of::<Option<Socket>>() == core::mem::size_of::<Socket>()); impl Default for PySocket { fn default() -> Self { @@ -849,10 +1000,64 @@ mod _socket { sock: Socket, ) -> io::Result<()> { self.family.store(family); - self.kind.store(socket_kind); + // Mask out SOCK_NONBLOCK and SOCK_CLOEXEC flags from stored type + // to ensure consistent cross-platform behavior + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + let masked_kind = socket_kind & !(c::SOCK_NONBLOCK | c::SOCK_CLOEXEC); + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] + let masked_kind = socket_kind; + self.kind.store(masked_kind); self.proto.store(proto); let mut s = self.sock.write(); let sock = s.insert(sock); + // If SOCK_NONBLOCK is set, use timeout 0 (non-blocking) + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + let timeout = if socket_kind & c::SOCK_NONBLOCK != 0 { + 0.0 + } else { + DEFAULT_TIMEOUT.load() + }; + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] let timeout = DEFAULT_TIMEOUT.load(); self.timeout.store(timeout); if timeout >= 0.0 { @@ -996,6 +1201,131 @@ mod _socket { } Ok(addr6.into()) } + #[cfg(target_os = "linux")] + c::AF_CAN => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_CAN address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + if tuple.is_empty() || tuple.len() > 2 { + return Err(vm + .new_type_error( + "AF_CAN address must be a tuple (interface,) or (interface, addr)", + ) + .into()); + } + let interface: PyStrRef = tuple[0].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_CAN interface must be str, not {}", + caller, + obj.class().name() + )) + })?; + let ifname = interface.as_str(); + + // Get interface index + let ifindex = if ifname.is_empty() { + 0 // Bind to all CAN interfaces + } else { + // Check interface name length (IFNAMSIZ is typically 16) + if ifname.len() >= 16 { + return Err(vm + .new_os_error("interface name too long".to_owned()) + .into()); + } + let cstr = alloc::ffi::CString::new(ifname) + .map_err(|_| vm.new_os_error("invalid interface name".to_owned()))?; + let idx = unsafe { libc::if_nametoindex(cstr.as_ptr()) }; + if idx == 0 { + return Err(io::Error::last_os_error().into()); + } + idx as i32 + }; + + // Create sockaddr_can + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + let can_addr = + &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_can; + unsafe { + (*can_addr).can_family = libc::AF_CAN as libc::sa_family_t; + (*can_addr).can_ifindex = ifindex; + } + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(storage) }; + Ok(unsafe { + socket2::SockAddr::new( + storage, + core::mem::size_of::<libc::sockaddr_can>() as libc::socklen_t, + ) + }) + } + #[cfg(target_os = "linux")] + c::AF_ALG => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + if tuple.len() != 2 { + return Err(vm + .new_type_error("AF_ALG address must be a tuple (type, name)") + .into()); + } + let alg_type: PyStrRef = tuple[0].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG type must be str, not {}", + caller, + obj.class().name() + )) + })?; + let alg_name: PyStrRef = tuple[1].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG name must be str, not {}", + caller, + obj.class().name() + )) + })?; + + let type_str = alg_type.as_str(); + let name_str = alg_name.as_str(); + + // salg_type is 14 bytes, salg_name is 64 bytes + if type_str.len() >= 14 { + return Err(vm.new_value_error("type too long".to_owned()).into()); + } + if name_str.len() >= 64 { + return Err(vm.new_value_error("name too long".to_owned()).into()); + } + + // Create sockaddr_alg + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + let alg_addr = + &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_alg; + unsafe { + (*alg_addr).salg_family = libc::AF_ALG as libc::sa_family_t; + // Copy type string + for (i, b) in type_str.bytes().enumerate() { + (*alg_addr).salg_type[i] = b; + } + // Copy name string + for (i, b) in name_str.bytes().enumerate() { + (*alg_addr).salg_name[i] = b; + } + } + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(storage) }; + Ok(unsafe { + socket2::SockAddr::new( + storage, + core::mem::size_of::<libc::sockaddr_alg>() as libc::socklen_t, + ) + }) + } _ => Err(vm.new_os_error(format!("{caller}(): bad family")).into()), } } @@ -1083,11 +1413,90 @@ mod _socket { let mut family = family.unwrap_or(-1); let mut socket_kind = socket_kind.unwrap_or(-1); let mut proto = proto.unwrap_or(-1); + + let sock; + + // On Windows, fileno can be bytes from socket.share() for fromshare() + #[cfg(windows)] + if let Some(fileno_obj) = fileno.flatten() { + use crate::vm::builtins::PyBytes; + if let Ok(bytes) = fileno_obj.clone().downcast::<PyBytes>() { + let bytes_data = bytes.as_bytes(); + let expected_size = core::mem::size_of::<c::WSAPROTOCOL_INFOW>(); + + if bytes_data.len() != expected_size { + return Err(vm + .new_value_error(format!( + "socket descriptor string has wrong size, should be {} bytes", + expected_size + )) + .into()); + } + + let mut info: c::WSAPROTOCOL_INFOW = unsafe { core::mem::zeroed() }; + unsafe { + core::ptr::copy_nonoverlapping( + bytes_data.as_ptr(), + &mut info as *mut c::WSAPROTOCOL_INFOW as *mut u8, + expected_size, + ); + } + + let fd = unsafe { + c::WSASocketW( + c::FROM_PROTOCOL_INFO, + c::FROM_PROTOCOL_INFO, + c::FROM_PROTOCOL_INFO, + &info, + 0, + c::WSA_FLAG_OVERLAPPED, + ) + }; + + if fd == c::INVALID_SOCKET { + return Err(Self::wsa_error().into()); + } + + crate::vm::stdlib::nt::raw_set_handle_inheritable(fd as _, false)?; + + family = info.iAddressFamily; + socket_kind = info.iSocketType; + proto = info.iProtocol; + + sock = unsafe { sock_from_raw_unchecked(fd as RawSocket) }; + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + // Not bytes, treat as regular fileno + let fileno = get_raw_sock(fileno_obj, vm)?; + sock = sock_from_raw(fileno, vm)?; + match sock.local_addr() { + Ok(addr) if family == -1 => family = addr.family() as i32, + Err(e) + if family == -1 + || matches!( + e.raw_os_error(), + Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) + ) => + { + core::mem::forget(sock); + return Err(e.into()); + } + _ => {} + } + if socket_kind == -1 { + socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); + } + proto = 0; + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + #[cfg(not(windows))] let fileno = fileno .flatten() .map(|obj| get_raw_sock(obj, vm)) .transpose()?; - let sock; + #[cfg(not(windows))] if let Some(fileno) = fileno { sock = sock_from_raw(fileno, vm)?; match sock.local_addr() { @@ -1099,7 +1508,7 @@ mod _socket { Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) ) => { - std::mem::forget(sock); + core::mem::forget(sock); return Err(e.into()); } _ => {} @@ -1121,7 +1530,11 @@ mod _socket { proto = 0; } } - } else { + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + // No fileno provided, create new socket + { if family == -1 { family = c::AF_INET as _ } @@ -1171,7 +1584,9 @@ mod _socket { &self, vm: &VirtualMachine, ) -> Result<(RawSocket, PyObjectRef), IoOrPyException> { - let (sock, addr) = self.sock_op(vm, SelectKind::Read, || self.sock()?.accept())?; + // Use accept_raw() instead of accept() to avoid socket2's set_common_flags() + // which tries to set SO_NOSIGPIPE and fails with EINVAL on Unix domain sockets on macOS + let (sock, addr) = self.sock_op(vm, SelectKind::Read, || self.sock()?.accept_raw())?; let fd = into_sock_fileno(sock); Ok((fd, get_addr_tuple(&addr, vm))) } @@ -1403,13 +1818,257 @@ mod _socket { .map_err(|e| e.into_pyexception(vm)) } + /// sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) -> int + /// + /// Set operation mode and target IV for an AF_ALG socket. + #[cfg(target_os = "linux")] + #[pymethod] + fn sendmsg_afalg(&self, args: SendmsgAfalgArgs, vm: &VirtualMachine) -> PyResult<usize> { + let msg = args.msg; + let op = args.op; + let iv = args.iv; + let flags = args.flags; + + // Validate assoclen - must be non-negative if provided + let assoclen: Option<u32> = match args.assoclen { + OptionalArg::Present(val) if val < 0 => { + return Err(vm.new_type_error("assoclen must be non-negative".to_owned())); + } + OptionalArg::Present(val) => Some(val as u32), + OptionalArg::Missing => None, + }; + + // Build control messages for AF_ALG + let mut control_buf = Vec::new(); + + // Add ALG_SET_OP control message + { + let op_bytes = op.to_ne_bytes(); + let space = + unsafe { libc::CMSG_SPACE(core::mem::size_of::<u32>() as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(core::mem::size_of::<u32>() as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_OP; + let data = libc::CMSG_DATA(cmsg); + core::ptr::copy_nonoverlapping(op_bytes.as_ptr(), data, op_bytes.len()); + } + } + + // Add ALG_SET_IV control message if iv is provided + if let Some(iv_data) = iv { + let iv_bytes = iv_data.borrow_buf(); + // struct af_alg_iv { __u32 ivlen; __u8 iv[]; } + let iv_struct_size = 4 + iv_bytes.len(); + let space = unsafe { libc::CMSG_SPACE(iv_struct_size as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(iv_struct_size as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_IV; + let data = libc::CMSG_DATA(cmsg); + // Write ivlen + let ivlen = (iv_bytes.len() as u32).to_ne_bytes(); + core::ptr::copy_nonoverlapping(ivlen.as_ptr(), data, 4); + // Write iv + core::ptr::copy_nonoverlapping(iv_bytes.as_ptr(), data.add(4), iv_bytes.len()); + } + } + + // Add ALG_SET_AEAD_ASSOCLEN control message if assoclen is provided + if let Some(assoclen_val) = assoclen { + let assoclen_bytes = assoclen_val.to_ne_bytes(); + let space = + unsafe { libc::CMSG_SPACE(core::mem::size_of::<u32>() as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(core::mem::size_of::<u32>() as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_AEAD_ASSOCLEN; + let data = libc::CMSG_DATA(cmsg); + core::ptr::copy_nonoverlapping( + assoclen_bytes.as_ptr(), + data, + assoclen_bytes.len(), + ); + } + } + + // Build buffers + let buffers = msg.iter().map(|buf| buf.borrow_buf()).collect::<Vec<_>>(); + let iovecs: Vec<libc::iovec> = buffers + .iter() + .map(|buf| libc::iovec { + iov_base: buf.as_ptr() as *mut _, + iov_len: buf.len(), + }) + .collect(); + + // Set up msghdr + let mut msghdr: libc::msghdr = unsafe { core::mem::zeroed() }; + msghdr.msg_iov = iovecs.as_ptr() as *mut _; + msghdr.msg_iovlen = iovecs.len() as _; + if !control_buf.is_empty() { + msghdr.msg_control = control_buf.as_mut_ptr() as *mut _; + msghdr.msg_controllen = control_buf.len() as _; + } + + self.sock_op(vm, SelectKind::Write, || { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let ret = unsafe { libc::sendmsg(fd as libc::c_int, &msghdr, flags) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ret as usize) + } + }) + .map_err(|e| e.into_pyexception(vm)) + } + + /// recvmsg(bufsize[, ancbufsize[, flags]]) -> (data, ancdata, msg_flags, address) + /// + /// Receive normal data and ancillary data from the socket. + #[cfg(all(unix, not(target_os = "redox")))] + #[pymethod] + fn recvmsg( + &self, + bufsize: isize, + ancbufsize: OptionalArg<isize>, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + use core::mem::MaybeUninit; + + if bufsize < 0 { + return Err(vm.new_value_error("negative buffer size in recvmsg".to_owned())); + } + let bufsize = bufsize as usize; + + let ancbufsize = ancbufsize.unwrap_or(0); + if ancbufsize < 0 { + return Err( + vm.new_value_error("negative ancillary buffer size in recvmsg".to_owned()) + ); + } + let ancbufsize = ancbufsize as usize; + let flags = flags.unwrap_or(0); + + // Allocate buffers + let mut data_buf: Vec<MaybeUninit<u8>> = vec![MaybeUninit::uninit(); bufsize]; + let mut anc_buf: Vec<MaybeUninit<u8>> = vec![MaybeUninit::uninit(); ancbufsize]; + let mut addr_storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + + // Set up iovec + let mut iov = [libc::iovec { + iov_base: data_buf.as_mut_ptr().cast(), + iov_len: bufsize, + }]; + + // Set up msghdr + let mut msg: libc::msghdr = unsafe { core::mem::zeroed() }; + msg.msg_name = (&mut addr_storage as *mut libc::sockaddr_storage).cast(); + msg.msg_namelen = core::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t; + msg.msg_iov = iov.as_mut_ptr(); + msg.msg_iovlen = 1; + if ancbufsize > 0 { + msg.msg_control = anc_buf.as_mut_ptr().cast(); + msg.msg_controllen = ancbufsize as _; + } + + let n = self + .sock_op(vm, SelectKind::Read, || { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let ret = unsafe { libc::recvmsg(fd as libc::c_int, &mut msg, flags) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ret as usize) + } + }) + .map_err(|e| e.into_pyexception(vm))?; + + // Build data bytes + let data = unsafe { + data_buf.set_len(n); + core::mem::transmute::<Vec<MaybeUninit<u8>>, Vec<u8>>(data_buf) + }; + + // Build ancdata list + let ancdata = Self::parse_ancillary_data(&msg, vm)?; + + // Build address tuple + let address = if msg.msg_namelen > 0 { + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(addr_storage) }; + let addr = unsafe { socket2::SockAddr::new(storage, msg.msg_namelen) }; + get_addr_tuple(&addr, vm) + } else { + vm.ctx.none() + }; + + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytes(data).into(), + ancdata, + vm.ctx.new_int(msg.msg_flags).into(), + address, + ])) + } + + /// Parse ancillary data from a received message header + #[cfg(all(unix, not(target_os = "redox")))] + fn parse_ancillary_data(msg: &libc::msghdr, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let mut result = Vec::new(); + + // Calculate buffer end for truncation handling + let ctrl_buf = msg.msg_control as *const u8; + let ctrl_end = unsafe { ctrl_buf.add(msg.msg_controllen as _) }; + + let mut cmsg: *mut libc::cmsghdr = unsafe { libc::CMSG_FIRSTHDR(msg) }; + while !cmsg.is_null() { + let cmsg_ref = unsafe { &*cmsg }; + let data_ptr = unsafe { libc::CMSG_DATA(cmsg) }; + + // Calculate data length, respecting buffer truncation + let data_len_from_cmsg = + cmsg_ref.cmsg_len as usize - (data_ptr as usize - cmsg as usize); + let available = ctrl_end as usize - data_ptr as usize; + let data_len = data_len_from_cmsg.min(available); + + let data = unsafe { core::slice::from_raw_parts(data_ptr, data_len) }; + + let tuple = vm.ctx.new_tuple(vec![ + vm.ctx.new_int(cmsg_ref.cmsg_level).into(), + vm.ctx.new_int(cmsg_ref.cmsg_type).into(), + vm.ctx.new_bytes(data.to_vec()).into(), + ]); + + result.push(tuple.into()); + + cmsg = unsafe { libc::CMSG_NXTHDR(msg, cmsg) }; + } + + Ok(vm.ctx.new_list(result).into()) + } + // based on nix's implementation #[cfg(all(unix, not(target_os = "redox")))] fn pack_cmsgs_to_send( cmsgs: &[(i32, i32, ArgBytesLike)], vm: &VirtualMachine, ) -> PyResult<Vec<u8>> { - use std::{mem, ptr}; + use core::{mem, ptr}; if cmsgs.is_empty() { return Ok(vec![]); @@ -1447,7 +2106,7 @@ mod _socket { unsafe { (*pmhdr).cmsg_level = *lvl; (*pmhdr).cmsg_type = *typ; - (*pmhdr).cmsg_len = data.len() as _; + (*pmhdr).cmsg_len = libc::CMSG_LEN(data.len() as _) as _; ptr::copy_nonoverlapping(data.as_ptr(), libc::CMSG_DATA(pmhdr), data.len()); } @@ -1460,13 +2119,45 @@ mod _socket { #[pymethod] fn close(&self) -> io::Result<()> { - let sock = self.detach(); - if sock != INVALID_SOCKET as i64 { - close_inner(sock as RawSocket)?; + let sock = self.sock.write().take(); + if let Some(sock) = sock { + close_inner(into_sock_fileno(sock))?; } Ok(()) } + #[pymethod] + fn __del__(&self, vm: &VirtualMachine) { + // Emit ResourceWarning if socket is still open + if self.sock.read().is_some() { + let laddr = if let Ok(sock) = self.sock() + && let Ok(addr) = sock.local_addr() + && let Ok(repr) = get_addr_tuple(&addr, vm).repr(vm) + { + format!(", laddr={}", repr.as_str()) + } else { + String::new() + }; + + let msg = format!( + "unclosed <socket.socket fd={}, family={}, type={}, proto={}{}>", + self.fileno(), + self.family.load(), + self.kind.load(), + self.proto.load(), + laddr + ); + let _ = crate::vm::warn::warn( + vm.ctx.new_str(msg).into(), + Some(vm.ctx.exceptions.resource_warning.to_owned()), + 1, + None, + vm, + ); + } + let _ = self.close(); + } + #[pymethod] #[inline] fn detach(&self) -> i64 { @@ -1514,12 +2205,29 @@ mod _socket { } #[pymethod] - fn settimeout(&self, timeout: Option<Duration>) -> io::Result<()> { - self.timeout - .store(timeout.map_or(-1.0, |d| d.as_secs_f64())); + fn settimeout(&self, timeout: Option<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<()> { + let timeout = match timeout { + Some(t) => { + let f = t.into_float(); + if f.is_nan() { + return Err( + vm.new_value_error("Invalid value NaN (not a number)".to_owned()) + ); + } + if f < 0.0 || !f.is_finite() { + return Err(vm.new_value_error("Timeout value out of range".to_owned())); + } + Some(f) + } + None => None, + }; + self.timeout.store(timeout.unwrap_or(-1.0)); // even if timeout is > 0 the socket needs to be nonblocking in order for us to select() on // it - self.sock()?.set_nonblocking(timeout.is_some()) + self.sock() + .map_err(|e| e.into_pyexception(vm))? + .set_nonblocking(timeout.is_some()) + .map_err(|e| e.into_pyexception(vm)) } #[pymethod] @@ -1535,7 +2243,7 @@ mod _socket { let buflen = buflen.unwrap_or(0); if buflen == 0 { let mut flag: libc::c_int = 0; - let mut flagsize = std::mem::size_of::<libc::c_int>() as _; + let mut flagsize = core::mem::size_of::<libc::c_int>() as _; let ret = unsafe { c::getsockopt( fd as _, @@ -1595,11 +2303,11 @@ mod _socket { level, name, val as *const i32 as *const _, - std::mem::size_of::<i32>() as _, + core::mem::size_of::<i32>() as _, ) }, (None, OptionalArg::Present(optlen)) => unsafe { - c::setsockopt(fd as _, level, name, std::ptr::null(), optlen as _) + c::setsockopt(fd as _, level, name, core::ptr::null(), optlen as _) }, _ => { return Err(vm @@ -1629,6 +2337,136 @@ mod _socket { Ok(self.sock()?.shutdown(how)?) } + #[cfg(windows)] + fn wsa_error() -> io::Error { + io::Error::from_raw_os_error(unsafe { c::WSAGetLastError() }) + } + + #[cfg(windows)] + #[pymethod] + fn ioctl( + &self, + cmd: PyObjectRef, + option: PyObjectRef, + vm: &VirtualMachine, + ) -> Result<u32, IoOrPyException> { + use crate::vm::builtins::PyInt; + use crate::vm::convert::TryFromObject; + + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let mut recv: u32 = 0; + + // Convert cmd to u32, returning ValueError for invalid/negative values + let cmd_int = cmd + .downcast::<PyInt>() + .map_err(|_| vm.new_type_error("an integer is required"))?; + let cmd_val = cmd_int.as_bigint(); + let cmd: u32 = cmd_val + .to_u32() + .ok_or_else(|| vm.new_value_error(format!("invalid ioctl command {}", cmd_val)))?; + + match cmd { + c::SIO_RCVALL | c::SIO_LOOPBACK_FAST_PATH => { + // Option must be an integer, not None + if vm.is_none(&option) { + return Err(vm + .new_type_error("an integer is required (got type NoneType)") + .into()); + } + let option_val: u32 = TryFromObject::try_from_object(vm, option)?; + let ret = unsafe { + c::WSAIoctl( + fd as _, + cmd, + &option_val as *const u32 as *const _, + core::mem::size_of::<u32>() as u32, + core::ptr::null_mut(), + 0, + &mut recv, + core::ptr::null_mut(), + None, + ) + }; + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + Ok(recv) + } + c::SIO_KEEPALIVE_VALS => { + let tuple: PyTupleRef = option + .downcast() + .map_err(|_| vm.new_type_error("SIO_KEEPALIVE_VALS requires a tuple"))?; + if tuple.len() != 3 { + return Err(vm + .new_type_error( + "SIO_KEEPALIVE_VALS requires (onoff, keepalivetime, keepaliveinterval)", + ) + .into()); + } + + #[repr(C)] + struct TcpKeepalive { + onoff: u32, + keepalivetime: u32, + keepaliveinterval: u32, + } + + let ka = TcpKeepalive { + onoff: TryFromObject::try_from_object(vm, tuple[0].clone())?, + keepalivetime: TryFromObject::try_from_object(vm, tuple[1].clone())?, + keepaliveinterval: TryFromObject::try_from_object(vm, tuple[2].clone())?, + }; + + let ret = unsafe { + c::WSAIoctl( + fd as _, + cmd, + &ka as *const TcpKeepalive as *const _, + core::mem::size_of::<TcpKeepalive>() as u32, + core::ptr::null_mut(), + 0, + &mut recv, + core::ptr::null_mut(), + None, + ) + }; + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + Ok(recv) + } + _ => Err(vm + .new_value_error(format!("invalid ioctl command {}", cmd)) + .into()), + } + } + + #[cfg(windows)] + #[pymethod] + fn share(&self, process_id: u32, _vm: &VirtualMachine) -> Result<Vec<u8>, IoOrPyException> { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + + let mut info: MaybeUninit<c::WSAPROTOCOL_INFOW> = MaybeUninit::uninit(); + + let ret = unsafe { c::WSADuplicateSocketW(fd as _, process_id, info.as_mut_ptr()) }; + + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + + let info = unsafe { info.assume_init() }; + let bytes = unsafe { + core::slice::from_raw_parts( + &info as *const c::WSAPROTOCOL_INFOW as *const u8, + core::mem::size_of::<c::WSAPROTOCOL_INFOW>(), + ) + }; + + Ok(bytes.to_vec()) + } + #[pygetset(name = "type")] fn kind(&self) -> i32 { self.kind.load() @@ -1651,7 +2489,7 @@ mod _socket { } impl ToSocketAddrs for Address { - type Iter = std::vec::IntoIter<SocketAddr>; + type Iter = alloc::vec::IntoIter<SocketAddr>; fn to_socket_addrs(&self) -> io::Result<Self::Iter> { (self.host.as_str(), self.port).to_socket_addrs() } @@ -1729,6 +2567,50 @@ mod _socket { let path = ffi::OsStr::from_bytes(&path[..nul_pos]); return vm.fsdecode(path).into(); } + #[cfg(target_os = "linux")] + { + let family = addr.family(); + if family == libc::AF_CAN as libc::sa_family_t { + // AF_CAN address: (interface_name,) or (interface_name, can_id) + let can_addr = unsafe { &*(addr.as_ptr() as *const libc::sockaddr_can) }; + let ifindex = can_addr.can_ifindex; + let ifname = if ifindex == 0 { + String::new() + } else { + let mut buf = [0u8; libc::IF_NAMESIZE]; + let ret = unsafe { + libc::if_indextoname( + ifindex as libc::c_uint, + buf.as_mut_ptr() as *mut libc::c_char, + ) + }; + if ret.is_null() { + String::new() + } else { + let nul_pos = memchr::memchr(b'\0', &buf).unwrap_or(buf.len()); + String::from_utf8_lossy(&buf[..nul_pos]).into_owned() + } + }; + return vm.ctx.new_tuple(vec![vm.ctx.new_str(ifname).into()]).into(); + } + if family == libc::AF_ALG as libc::sa_family_t { + // AF_ALG address: (type, name) + let alg_addr = unsafe { &*(addr.as_ptr() as *const libc::sockaddr_alg) }; + let type_bytes = &alg_addr.salg_type; + let name_bytes = &alg_addr.salg_name; + let type_nul = memchr::memchr(b'\0', type_bytes).unwrap_or(type_bytes.len()); + let name_nul = memchr::memchr(b'\0', name_bytes).unwrap_or(name_bytes.len()); + let type_str = String::from_utf8_lossy(&type_bytes[..type_nul]).into_owned(); + let name_str = String::from_utf8_lossy(&name_bytes[..name_nul]).into_owned(); + return vm + .ctx + .new_tuple(vec![ + vm.ctx.new_str(type_str).into(), + vm.ctx.new_str(name_str).into(), + ]) + .into(); + } + } // TODO: support more address families (String::new(), 0).to_pyobject(vm) } @@ -1767,7 +2649,7 @@ mod _socket { } fn cstr_opt_as_ptr(x: &OptionalArg<ffi::CString>) -> *const libc::c_char { - x.as_ref().map_or_else(std::ptr::null, |s| s.as_ptr()) + x.as_ref().map_or_else(core::ptr::null, |s| s.as_ptr()) } #[pyfunction] @@ -1925,9 +2807,9 @@ mod _socket { #[derive(FromArgs)] struct GAIOptions { #[pyarg(positional)] - host: Option<PyStrRef>, + host: Option<ArgStrOrBytesLike>, #[pyarg(positional)] - port: Option<Either<PyStrRef, i32>>, + port: Option<Either<ArgStrOrBytesLike, i32>>, #[pyarg(positional, default = c::AF_UNSPEC)] family: i32, @@ -1951,14 +2833,54 @@ mod _socket { flags: opts.flags, }; - let host = opts.host.as_ref().map(|s| s.as_str()); - let port = opts.port.as_ref().map(|p| -> std::borrow::Cow<'_, str> { - match p { - Either::A(s) => s.as_str().into(), - Either::B(i) => i.to_string().into(), + // Encode host: str uses IDNA encoding, bytes must be valid UTF-8 + let host_encoded: Option<String> = match opts.host.as_ref() { + Some(ArgStrOrBytesLike::Str(s)) => { + let encoded = + vm.state + .codec_registry + .encode_text(s.to_owned(), "idna", None, vm)?; + let host_str = core::str::from_utf8(encoded.as_bytes()) + .map_err(|_| vm.new_runtime_error("idna output is not utf8".to_owned()))?; + Some(host_str.to_owned()) } - }); - let port = port.as_ref().map(|p| p.as_ref()); + Some(ArgStrOrBytesLike::Buf(b)) => { + let bytes = b.borrow_buf(); + let host_str = core::str::from_utf8(&bytes).map_err(|_| { + vm.new_unicode_decode_error("host bytes is not utf8".to_owned()) + })?; + Some(host_str.to_owned()) + } + None => None, + }; + let host = host_encoded.as_deref(); + + // Encode port: str/bytes as service name, int as port number + let port_encoded: Option<String> = match opts.port.as_ref() { + Some(Either::A(sb)) => { + let port_str = match sb { + ArgStrOrBytesLike::Str(s) => { + // For str, check for surrogates and raise UnicodeEncodeError if found + s.to_str() + .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))? + .to_owned() + } + ArgStrOrBytesLike::Buf(b) => { + // For bytes, check if it's valid UTF-8 + let bytes = b.borrow_buf(); + core::str::from_utf8(&bytes) + .map_err(|_| { + vm.new_unicode_decode_error("port is not utf8".to_owned()) + })? + .to_owned() + } + }; + Some(port_str) + } + Some(Either::B(i)) => Some(i.to_string()), + None => None, + }; + let port = port_encoded.as_deref(); let addrs = dns_lookup::getaddrinfo(host, port, Some(hints)) .map_err(|err| convert_socket_error(vm, err, SocketError::GaiError))?; @@ -2202,7 +3124,6 @@ mod _socket { } #[cfg(windows)] { - use std::ptr; use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; let table = MibTable::get_raw().map_err(|err| err.into_pyexception(vm))?; @@ -2229,14 +3150,14 @@ mod _socket { } } struct MibTable { - ptr: ptr::NonNull<IpHelper::MIB_IF_TABLE2>, + ptr: core::ptr::NonNull<IpHelper::MIB_IF_TABLE2>, } impl MibTable { fn get_raw() -> io::Result<Self> { - let mut ptr = ptr::null_mut(); + let mut ptr = core::ptr::null_mut(); let ret = unsafe { IpHelper::GetIfTable2Ex(IpHelper::MibIfTableRaw, &mut ptr) }; if ret == 0 { - let ptr = unsafe { ptr::NonNull::new_unchecked(ptr) }; + let ptr = unsafe { core::ptr::NonNull::new_unchecked(ptr) }; Ok(Self { ptr }) } else { Err(io::Error::from_raw_os_error(ret as i32)) @@ -2248,7 +3169,7 @@ mod _socket { unsafe { let p = self.ptr.as_ptr(); let ptr = &raw const (*p).Table as *const IpHelper::MIB_IF_ROW2; - std::slice::from_raw_parts(ptr, (*p).NumEntries as usize) + core::slice::from_raw_parts(ptr, (*p).NumEntries as usize) } } } @@ -2316,7 +3237,7 @@ mod _socket { .state .codec_registry .encode_text(pyname, "idna", None, vm)?; - let name = std::str::from_utf8(name.as_bytes()) + let name = core::str::from_utf8(name.as_bytes()) .map_err(|_| vm.new_runtime_error("idna output is not utf8"))?; let mut res = dns_lookup::getaddrinfo(Some(name), None, Some(hints)) .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; @@ -2465,14 +3386,28 @@ mod _socket { } #[pyfunction] - fn setdefaulttimeout(timeout: Option<Duration>) { - DEFAULT_TIMEOUT.store(timeout.map_or(-1.0, |d| d.as_secs_f64())); + fn setdefaulttimeout(timeout: Option<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<()> { + let val = match timeout { + Some(t) => { + let f = t.into_float(); + if f.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)".to_owned())); + } + if f < 0.0 || !f.is_finite() { + return Err(vm.new_value_error("Timeout value out of range".to_owned())); + } + f + } + None => -1.0, + }; + DEFAULT_TIMEOUT.store(val); + Ok(()) } #[pyfunction] fn dup(x: PyObjectRef, vm: &VirtualMachine) -> Result<RawSocket, IoOrPyException> { let sock = get_raw_sock(x, vm)?; - let sock = std::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); + let sock = core::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); let newsock = sock.try_clone()?; let fd = into_sock_fileno(newsock); #[cfg(windows)] diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index c23062d639d..a1b089078a8 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -1,4 +1,4 @@ -// spell-checker: ignore ssleof aesccm aesgcm getblocking setblocking ENDTLS +// spell-checker: ignore ssleof aesccm aesgcm capath getblocking setblocking ENDTLS TLSEXT //! Pure Rust SSL/TLS implementation using rustls //! @@ -22,11 +22,14 @@ mod cert; // OpenSSL compatibility layer (abstracts rustls operations) mod compat; -pub(crate) use _ssl::make_module; +// SSL exception types (shared with openssl backend) +mod error; + +pub(crate) use _ssl::module_def; #[allow(non_snake_case)] #[allow(non_upper_case_globals)] -#[pymodule] +#[pymodule(with(error::ssl_error))] mod _ssl { use crate::{ common::{ @@ -37,23 +40,27 @@ mod _ssl { vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{ - PyBaseExceptionRef, PyBytesRef, PyListRef, PyOSError, PyStrRef, PyType, PyTypeRef, - }, + builtins::{PyBaseExceptionRef, PyBytesRef, PyListRef, PyStrRef, PyType, PyTypeRef}, convert::IntoPyException, - function::{ArgBytesLike, ArgMemoryBuffer, FuncArgs, OptionalArg, PyComparisonValue}, + function::{ + ArgBytesLike, ArgMemoryBuffer, Either, FuncArgs, OptionalArg, PyComparisonValue, + }, stdlib::warnings, types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }, }; - use std::{ - collections::HashMap, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, - time::{Duration, SystemTime}, + + // Import error types used in this module (others are exposed via pymodule(with(...))) + use super::error::{ + PySSLError, create_ssl_eof_error, create_ssl_want_read_error, create_ssl_want_write_error, + create_ssl_zero_return_error, + }; + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, }; + use std::{collections::HashMap, time::SystemTime}; // Rustls imports use parking_lot::{Mutex as ParkingMutex, RwLock as ParkingRwLock}; @@ -201,28 +208,6 @@ mod _ssl { #[pyattr] const OP_ALL: i32 = 0x00000BFB; // Combined "safe" options (reduced for i32, excluding OP_LEGACY_SERVER_CONNECT for OpenSSL 3.0.0+ compatibility) - // Error types - #[pyattr] - const SSL_ERROR_NONE: i32 = 0; - #[pyattr] - const SSL_ERROR_SSL: i32 = 1; - #[pyattr] - const SSL_ERROR_WANT_READ: i32 = 2; - #[pyattr] - const SSL_ERROR_WANT_WRITE: i32 = 3; - #[pyattr] - const SSL_ERROR_WANT_X509_LOOKUP: i32 = 4; - #[pyattr] - const SSL_ERROR_SYSCALL: i32 = 5; - #[pyattr] - const SSL_ERROR_ZERO_RETURN: i32 = 6; - #[pyattr] - const SSL_ERROR_WANT_CONNECT: i32 = 7; - #[pyattr] - const SSL_ERROR_EOF: i32 = 8; - #[pyattr] - const SSL_ERROR_INVALID_ERROR_CODE: i32 = 10; - // Alert types (matching _TLSAlertType enum) #[pyattr] const ALERT_DESCRIPTION_CLOSE_NOTIFY: i32 = 0; @@ -342,106 +327,6 @@ mod _ssl { #[pyattr] const ENCODING_PEM_AUX: i32 = 0x101; // PEM + 0x100 - #[pyattr] - #[pyexception(name = "SSLError", base = PyOSError)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLError(PyOSError); - - #[pyexception] - impl PySSLError { - // Returns strerror attribute if available, otherwise str(args) - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - // Try to get strerror attribute first (OSError compatibility) - if let Ok(strerror) = exc.as_object().get_attr("strerror", vm) - && !vm.is_none(&strerror) - { - return strerror.str(vm); - } - - // Otherwise return str(args) - let args = exc.args(); - if args.len() == 1 { - args.as_slice()[0].str(vm) - } else { - args.as_object().str(vm) - } - } - } - - #[pyattr] - #[pyexception(name = "SSLZeroReturnError", base = PySSLError)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLZeroReturnError(PySSLError); - - #[pyexception] - impl PySSLZeroReturnError {} - - #[pyattr] - #[pyexception(name = "SSLWantReadError", base = PySSLError, impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLWantReadError(PySSLError); - - #[pyattr] - #[pyexception(name = "SSLWantWriteError", base = PySSLError, impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLWantWriteError(PySSLError); - - #[pyattr] - #[pyexception(name = "SSLSyscallError", base = PySSLError, impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLSyscallError(PySSLError); - - #[pyattr] - #[pyexception(name = "SSLEOFError", base = PySSLError, impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLEOFError(PySSLError); - - #[pyattr] - #[pyexception(name = "SSLCertVerificationError", base = PySSLError, impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PySSLCertVerificationError(PySSLError); - - // Helper functions to create SSL exceptions with proper errno attribute - pub(super) fn create_ssl_want_read_error(vm: &VirtualMachine) -> PyRef<PyOSError> { - vm.new_os_subtype_error( - PySSLWantReadError::class(&vm.ctx).to_owned(), - Some(SSL_ERROR_WANT_READ), - "The operation did not complete (read)", - ) - } - - pub(super) fn create_ssl_want_write_error(vm: &VirtualMachine) -> PyRef<PyOSError> { - vm.new_os_subtype_error( - PySSLWantWriteError::class(&vm.ctx).to_owned(), - Some(SSL_ERROR_WANT_WRITE), - "The operation did not complete (write)", - ) - } - - pub(crate) fn create_ssl_eof_error(vm: &VirtualMachine) -> PyRef<PyOSError> { - vm.new_os_subtype_error( - PySSLEOFError::class(&vm.ctx).to_owned(), - None, - "EOF occurred in violation of protocol", - ) - } - - pub(crate) fn create_ssl_zero_return_error(vm: &VirtualMachine) -> PyRef<PyOSError> { - vm.new_os_subtype_error( - PySSLZeroReturnError::class(&vm.ctx).to_owned(), - None, - "TLS/SSL connection has been closed (EOF)", - ) - } - /// Validate server hostname for TLS SNI /// /// Checks that the hostname: @@ -939,24 +824,30 @@ mod _ssl { #[derive(FromArgs)] struct LoadVerifyLocationsArgs { - #[pyarg(any, optional)] - cafile: OptionalArg<Option<PyObjectRef>>, - #[pyarg(any, optional)] - capath: OptionalArg<Option<PyObjectRef>>, - #[pyarg(any, optional)] - cadata: OptionalArg<PyObjectRef>, + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + cafile: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + capath: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + #[pyarg(any, optional, error_msg = "cadata should be a str or bytes")] + cadata: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, } #[derive(FromArgs)] struct LoadCertChainArgs { - #[pyarg(any)] - certfile: PyObjectRef, - #[pyarg(any, optional)] - keyfile: OptionalArg<Option<PyObjectRef>>, + #[pyarg(any, error_msg = "path should be a str or bytes")] + certfile: Either<PyStrRef, ArgBytesLike>, + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + keyfile: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, #[pyarg(any, optional)] password: OptionalArg<PyObjectRef>, } + #[derive(FromArgs)] + struct GetCertArgs { + #[pyarg(any, optional)] + binary_form: OptionalArg<bool>, + } + #[pyclass(with(Constructor), flags(BASETYPE))] impl PySSLContext { // Helper method to convert DER certificate bytes to Python dict @@ -1341,7 +1232,7 @@ mod _ssl { // Check that at least one argument is provided let has_cafile = matches!(&args.cafile, OptionalArg::Present(Some(_))); let has_capath = matches!(&args.capath, OptionalArg::Present(Some(_))); - let has_cadata = matches!(&args.cadata, OptionalArg::Present(obj) if !vm.is_none(obj)); + let has_cadata = matches!(&args.cadata, OptionalArg::Present(Some(_))); if !has_cafile && !has_capath && !has_cadata { return Err( @@ -1362,10 +1253,8 @@ mod _ssl { None }; - let cadata_parsed = if let OptionalArg::Present(ref cadata_obj) = args.cadata - && !vm.is_none(cadata_obj) - { - let is_string = PyStrRef::try_from_object(vm, cadata_obj.clone()).is_ok(); + let cadata_parsed = if let OptionalArg::Present(Some(ref cadata_obj)) = args.cadata { + let is_string = matches!(cadata_obj, Either::A(_)); let data_vec = self.parse_cadata_arg(cadata_obj, vm)?; Some((data_vec, is_string)) } else { @@ -1423,7 +1312,7 @@ mod _ssl { /// Helper: Get path from Python's os.environ fn get_env_path( - environ: &PyObjectRef, + environ: &PyObject, var_name: &str, vm: &VirtualMachine, ) -> PyResult<String> { @@ -1804,12 +1693,8 @@ mod _ssl { } #[pymethod] - fn get_ca_certs( - &self, - binary_form: OptionalArg<bool>, - vm: &VirtualMachine, - ) -> PyResult<PyListRef> { - let binary_form = binary_form.unwrap_or(false); + fn get_ca_certs(&self, args: GetCertArgs, vm: &VirtualMachine) -> PyResult<PyListRef> { + let binary_form = args.binary_form.unwrap_or(false); let ca_certs_der = self.ca_certs_der.read(); let mut certs = Vec::new(); @@ -2018,6 +1903,8 @@ mod _ssl { pending_context: PyRwLock::new(None), client_hello_buffer: PyMutex::new(None), shutdown_state: PyMutex::new(ShutdownState::NotStarted), + pending_tls_output: PyMutex::new(Vec::new()), + write_buffered_len: PyMutex::new(0), deferred_cert_error: Arc::new(ParkingRwLock::new(None)), }; @@ -2088,6 +1975,8 @@ mod _ssl { pending_context: PyRwLock::new(None), client_hello_buffer: PyMutex::new(None), shutdown_state: PyMutex::new(ShutdownState::NotStarted), + pending_tls_output: PyMutex::new(Vec::new()), + write_buffered_len: PyMutex::new(0), deferred_cert_error: Arc::new(ParkingRwLock::new(None)), }; @@ -2101,14 +1990,14 @@ mod _ssl { // Helper functions (private): /// Parse path argument (str or bytes) to string - fn parse_path_arg(arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - if let Ok(s) = PyStrRef::try_from_object(vm, arg.clone()) { - Ok(s.as_str().to_owned()) - } else if let Ok(b) = ArgBytesLike::try_from_object(vm, arg.clone()) { - String::from_utf8(b.borrow_buf().to_vec()) - .map_err(|_| vm.new_value_error("path contains invalid UTF-8".to_owned())) - } else { - Err(vm.new_type_error("path should be a str or bytes".to_owned())) + fn parse_path_arg( + arg: &Either<PyStrRef, ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<String> { + match arg { + Either::A(s) => Ok(s.as_str().to_owned()), + Either::B(b) => String::from_utf8(b.borrow_buf().to_vec()) + .map_err(|_| vm.new_value_error("path contains invalid UTF-8".to_owned())), } } @@ -2279,13 +2168,14 @@ mod _ssl { } /// Helper: Parse cadata argument (str or bytes) - fn parse_cadata_arg(&self, arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - if let Ok(s) = PyStrRef::try_from_object(vm, arg.clone()) { - Ok(s.as_str().as_bytes().to_vec()) - } else if let Ok(b) = ArgBytesLike::try_from_object(vm, arg.clone()) { - Ok(b.borrow_buf().to_vec()) - } else { - Err(vm.new_type_error("cadata should be a str or bytes".to_owned())) + fn parse_cadata_arg( + &self, + arg: &Either<PyStrRef, ArgBytesLike>, + _vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + match arg { + Either::A(s) => Ok(s.as_str().as_bytes().to_vec()), + Either::B(b) => Ok(b.borrow_buf().to_vec()), } } @@ -2409,7 +2299,7 @@ mod _ssl { // SSLSocket - represents a TLS-wrapped socket #[pyattr] - #[pyclass(name = "_SSLSocket", module = "ssl")] + #[pyclass(name = "_SSLSocket", module = "ssl", traverse)] #[derive(Debug, PyPayload)] pub(crate) struct PySSLSocket { // Underlying socket @@ -2417,14 +2307,19 @@ mod _ssl { // SSL context context: PyRwLock<PyRef<PySSLContext>>, // Server-side or client-side + #[pytraverse(skip)] server_side: bool, // Server hostname for SNI + #[pytraverse(skip)] server_hostname: PyRwLock<Option<String>>, // TLS connection state + #[pytraverse(skip)] connection: PyMutex<Option<TlsConnection>>, // Handshake completed flag + #[pytraverse(skip)] handshake_done: PyMutex<bool>, // Session was reused (for session resumption tracking) + #[pytraverse(skip)] session_was_reused: PyMutex<bool>, // Owner (SSLSocket instance that owns this _SSLSocket) owner: PyRwLock<Option<PyObjectRef>>, @@ -2432,22 +2327,37 @@ mod _ssl { session: PyRwLock<Option<PyObjectRef>>, // Verified certificate chain (built during verification) #[allow(dead_code)] + #[pytraverse(skip)] verified_chain: PyRwLock<Option<Vec<CertificateDer<'static>>>>, // MemoryBIO mode (optional) incoming_bio: Option<PyRef<PyMemoryBIO>>, outgoing_bio: Option<PyRef<PyMemoryBIO>>, // SNI certificate resolver state (for server-side only) + #[pytraverse(skip)] sni_state: PyRwLock<Option<Arc<ParkingMutex<SniCertName>>>>, // Pending context change (for SNI callback deferred handling) pending_context: PyRwLock<Option<PyRef<PySSLContext>>>, // Buffer to store ClientHello for connection recreation + #[pytraverse(skip)] client_hello_buffer: PyMutex<Option<Vec<u8>>>, // Shutdown state for tracking close-notify exchange + #[pytraverse(skip)] shutdown_state: PyMutex<ShutdownState>, + // Pending TLS output buffer for non-blocking sockets + // Stores unsent TLS bytes when sock_send() would block + // This prevents data loss when write_tls() drains rustls' internal buffer + // but the socket cannot accept all the data immediately + #[pytraverse(skip)] + pub(crate) pending_tls_output: PyMutex<Vec<u8>>, + // Tracks bytes already buffered in rustls for the current write operation + // Prevents duplicate writes when retrying after WantWrite/WantRead + #[pytraverse(skip)] + pub(crate) write_buffered_len: PyMutex<usize>, // Deferred client certificate verification error (for TLS 1.3) // Stores error message if client cert verification failed during handshake // Error is raised on first I/O operation after handshake // Using Arc to share with the certificate verifier + #[pytraverse(skip)] deferred_cert_error: Arc<ParkingRwLock<Option<String>>>, } @@ -2702,6 +2612,36 @@ mod _ssl { Ok(timed_out) } + // Internal implementation with explicit timeout override + pub(crate) fn sock_wait_for_io_with_timeout( + &self, + kind: SelectKind, + timeout: Option<core::time::Duration>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + if self.is_bio_mode() { + // BIO mode doesn't use select + return Ok(false); + } + + if let Some(t) = timeout + && t.is_zero() + { + // Non-blocking mode - don't use select + return Ok(false); + } + + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + + let timed_out = sock_select(&socket, kind, timeout) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + Ok(timed_out) + } + // SNI (Server Name Indication) Helper Methods: // These methods support the server-side handshake SNI callback mechanism @@ -2750,7 +2690,31 @@ mod _ssl { }; let initial_context: PyObjectRef = self.context.read().clone().into(); - let result = callback.call((ssl_sock, server_name_py, initial_context), vm)?; + // catches exceptions from the callback and reports them as unraisable + let result = match callback.call((ssl_sock, server_name_py, initial_context), vm) { + Ok(result) => result, + Err(exc) => { + vm.run_unraisable( + exc, + Some("in ssl servername callback".to_owned()), + callback.clone(), + ); + // Return SSL error like SSL_TLSEXT_ERR_ALERT_FATAL + let ssl_exc: PyBaseExceptionRef = vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "SNI callback raised exception", + ) + .upcast(); + let _ = ssl_exc.as_object().set_attr( + "reason", + vm.ctx.new_str("TLSV1_ALERT_INTERNAL_ERROR"), + vm, + ); + return Err(ssl_exc); + } + }; // Check return value type (must be None or integer) if !vm.is_none(&result) { @@ -2806,7 +2770,7 @@ mod _ssl { // Helper to call socket methods, bypassing any SSL wrapper pub(crate) fn sock_recv(&self, size: usize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - // In BIO mode, read from incoming BIO + // In BIO mode, read from incoming BIO (flags not supported) if let Some(ref bio) = self.incoming_bio { let bio_obj: PyObjectRef = bio.clone().into(); let read_method = bio_obj.get_attr("read", vm)?; @@ -2817,21 +2781,29 @@ mod _ssl { let socket_mod = vm.import("socket", 0)?; let socket_class = socket_mod.get_attr("socket", vm)?; - // Call socket.socket.recv(self.sock, size) + // Call socket.socket.recv(self.sock, size, flags) let recv_method = socket_class.get_attr("recv", vm)?; recv_method.call((self.sock.clone(), vm.ctx.new_int(size)), vm) } - pub(crate) fn sock_send( - &self, - data: Vec<u8>, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { + /// Peek at socket data without consuming it (MSG_PEEK). + /// Used during TLS shutdown to avoid consuming post-TLS cleartext data. + pub(crate) fn sock_peek(&self, size: usize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let socket_mod = vm.import("socket", 0)?; + let socket_class = socket_mod.get_attr("socket", vm)?; + let recv_method = socket_class.get_attr("recv", vm)?; + let msg_peek = socket_mod.get_attr("MSG_PEEK", vm)?; + recv_method.call((self.sock.clone(), vm.ctx.new_int(size), msg_peek), vm) + } + + /// Socket send - just sends data, caller must handle pending flush + /// Use flush_pending_tls_output before this if ordering is important + pub(crate) fn sock_send(&self, data: &[u8], vm: &VirtualMachine) -> PyResult<PyObjectRef> { // In BIO mode, write to outgoing BIO if let Some(ref bio) = self.outgoing_bio { let bio_obj: PyObjectRef = bio.clone().into(); let write_method = bio_obj.get_attr("write", vm)?; - return write_method.call((vm.ctx.new_bytes(data),), vm); + return write_method.call((vm.ctx.new_bytes(data.to_vec()),), vm); } // Normal socket mode @@ -2840,7 +2812,208 @@ mod _ssl { // Call socket.socket.send(self.sock, data) let send_method = socket_class.get_attr("send", vm)?; - send_method.call((self.sock.clone(), vm.ctx.new_bytes(data)), vm) + send_method.call((self.sock.clone(), vm.ctx.new_bytes(data.to_vec())), vm) + } + + /// Flush any pending TLS output data to the socket + /// Optional deadline parameter allows respecting a read deadline during flush + pub(crate) fn flush_pending_tls_output( + &self, + vm: &VirtualMachine, + deadline: Option<std::time::Instant>, + ) -> PyResult<()> { + let mut pending = self.pending_tls_output.lock(); + if pending.is_empty() { + return Ok(()); + } + + let socket_timeout = self.get_socket_timeout(vm)?; + let is_non_blocking = socket_timeout.map(|t| t.is_zero()).unwrap_or(false); + + let mut sent_total = 0; + + while sent_total < pending.len() { + // Calculate timeout: use deadline if provided, otherwise use socket timeout + let timeout_to_use = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + // Deadline already passed + *pending = pending[sent_total..].to_vec(); + return Err( + timeout_error_msg(vm, "The operation timed out".to_string()).upcast() + ); + } + Some(dl - now) + } else { + socket_timeout + }; + + // Use sock_select directly with calculated timeout + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + let timed_out = sock_select(&socket, SelectKind::Write, timeout_to_use) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + if timed_out { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + if is_non_blocking { + return Err(create_ssl_want_write_error(vm).upcast()); + } + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + match self.sock_send(&pending[sent_total..], vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent == 0 { + if is_non_blocking { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + return Err(create_ssl_want_write_error(vm).upcast()); + } + // Socket said ready but sent 0 bytes - retry + continue; + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + if is_non_blocking { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + // Keep unsent data in pending buffer for other errors too + *pending = pending[sent_total..].to_vec(); + return Err(e); + } + } + } + + // All data sent successfully + pending.clear(); + Ok(()) + } + + /// Send TLS output data to socket, saving unsent bytes to pending buffer + /// This prevents data loss when rustls' write_tls() drains its internal buffer + /// but the socket cannot accept all the data immediately + fn send_tls_output(&self, buf: Vec<u8>, vm: &VirtualMachine) -> PyResult<()> { + if buf.is_empty() { + return Ok(()); + } + + let timeout = self.get_socket_timeout(vm)?; + let is_non_blocking = timeout.map(|t| t.is_zero()).unwrap_or(false); + + let mut sent_total = 0; + while sent_total < buf.len() { + let timed_out = self.sock_wait_for_io_impl(SelectKind::Write, vm)?; + if timed_out { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + match self.sock_send(&buf[sent_total..], vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent == 0 { + if is_non_blocking { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + if is_non_blocking { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + // Save unsent data for other errors too + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(e); + } + } + } + + Ok(()) + } + + /// Flush all pending TLS output data, respecting socket timeout + /// Used during handshake completion and shutdown() to ensure all data is sent + pub(crate) fn blocking_flush_all_pending(&self, vm: &VirtualMachine) -> PyResult<()> { + // Get socket timeout to respect during flush + let timeout = self.get_socket_timeout(vm)?; + if timeout.map(|t| t.is_zero()).unwrap_or(false) { + return self.flush_pending_tls_output(vm, None); + } + + loop { + let pending_data = { + let pending = self.pending_tls_output.lock(); + if pending.is_empty() { + return Ok(()); + } + pending.clone() + }; + + // Wait for socket to be writable, respecting socket timeout + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + let timed_out = sock_select(&socket, SelectKind::Write, timeout) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + if timed_out { + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + // Try to send pending data (use raw to avoid recursion) + match self.sock_send(&pending_data, vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent > 0 { + let mut pending = self.pending_tls_output.lock(); + pending.drain(..sent); + } + // If sent == 0, loop will retry with sock_select + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + continue; + } + return Err(e); + } + } + } } #[pymethod] @@ -3238,7 +3411,7 @@ mod _ssl { // When server_hostname=None, use an IP address to suppress SNI // no hostname = no SNI extension ServerName::IpAddress( - std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)).into(), + core::net::IpAddr::V4(core::net::Ipv4Addr::new(127, 0, 0, 1)).into(), ) }; @@ -3335,15 +3508,16 @@ mod _ssl { }; } - // Ensure handshake is done + // Ensure handshake is done - if not, complete it first + // This matches OpenSSL behavior where SSL_read() auto-completes handshake if !*self.handshake_done.lock() { - return Err(vm.new_value_error("Handshake not completed")); + self.do_handshake(vm)?; } // Check if connection has been shut down - // After unwrap()/shutdown(), read operations should fail with SSLError + // Only block after shutdown is COMPLETED, not during shutdown process let shutdown_state = *self.shutdown_state.lock(); - if shutdown_state != ShutdownState::NotStarted { + if shutdown_state == ShutdownState::Completed { return Err(vm .new_os_subtype_error( PySSLError::class(&vm.ctx).to_owned(), @@ -3378,7 +3552,7 @@ mod _ssl { }; // Use compat layer for unified read logic with proper EOF handling - // This matches CPython's SSL_read_ex() approach + // This matches SSL_read_ex() approach let mut buf = vec![0u8; len]; let read_result = { let mut conn_guard = self.connection.lock(); @@ -3396,18 +3570,74 @@ mod _ssl { return_data(buf, &buffer, vm) } Err(crate::ssl::compat::SslError::Eof) => { + // If plaintext is still buffered, return it before EOF. + let pending = { + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(conn) => conn, + None => return Err(create_ssl_eof_error(vm).upcast()), + }; + use std::io::BufRead; + let mut reader = conn.reader(); + reader.fill_buf().map(|buf| buf.len()).unwrap_or(0) + }; + if pending > 0 { + let mut buf = vec![0u8; pending.min(len)]; + let read_retry = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + crate::ssl::compat::ssl_read(conn, &mut buf, self, vm) + }; + if let Ok(n) = read_retry { + buf.truncate(n); + return return_data(buf, &buffer, vm); + } + } // EOF occurred in violation of protocol (unexpected closure) - Err(vm - .new_os_subtype_error( - PySSLEOFError::class(&vm.ctx).to_owned(), - None, - "EOF occurred in violation of protocol", - ) - .upcast()) + Err(create_ssl_eof_error(vm).upcast()) } Err(crate::ssl::compat::SslError::ZeroReturn) => { - // Clean closure with close_notify - return empty data - return_data(vec![], &buffer, vm) + // If plaintext is still buffered, return it before clean EOF. + let pending = { + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(conn) => conn, + None => return Err(create_ssl_zero_return_error(vm).upcast()), + }; + use std::io::BufRead; + let mut reader = conn.reader(); + reader.fill_buf().map(|buf| buf.len()).unwrap_or(0) + }; + if pending > 0 { + let mut buf = vec![0u8; pending.min(len)]; + let read_retry = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + crate::ssl::compat::ssl_read(conn, &mut buf, self, vm) + }; + if let Ok(n) = read_retry { + buf.truncate(n); + return return_data(buf, &buffer, vm); + } + } + // Clean closure with close_notify + // CPython behavior depends on whether we've sent our close_notify: + // - If we've already sent close_notify (unwrap was called): raise SSLZeroReturnError + // - If we haven't sent close_notify yet: return empty bytes + let our_shutdown_state = *self.shutdown_state.lock(); + if our_shutdown_state == ShutdownState::SentCloseNotify + || our_shutdown_state == ShutdownState::Completed + { + // We already sent close_notify, now receiving peer's → SSLZeroReturnError + Err(create_ssl_zero_return_error(vm).upcast()) + } else { + // We haven't sent close_notify yet → return empty bytes + return_data(vec![], &buffer, vm) + } } Err(crate::ssl::compat::SslError::WantRead) => { // Non-blocking mode: would block @@ -3461,20 +3691,18 @@ mod _ssl { let data_bytes = data.borrow_buf(); let data_len = data_bytes.len(); - // return 0 immediately for empty write if data_len == 0 { return Ok(0); } - // Ensure handshake is done + // Ensure handshake is done (SSL_write auto-completes handshake) if !*self.handshake_done.lock() { - return Err(vm.new_value_error("Handshake not completed")); + self.do_handshake(vm)?; } - // Check if connection has been shut down - // After unwrap()/shutdown(), write operations should fail with SSLError - let shutdown_state = *self.shutdown_state.lock(); - if shutdown_state != ShutdownState::NotStarted { + // Check shutdown state + // Only block after shutdown is COMPLETED, not during shutdown process + if *self.shutdown_state.lock() == ShutdownState::Completed { return Err(vm .new_os_subtype_error( PySSLError::class(&vm.ctx).to_owned(), @@ -3484,86 +3712,41 @@ mod _ssl { .upcast()); } - { + // Call ssl_write (matches CPython's SSL_write_ex loop) + let result = { let mut conn_guard = self.connection.lock(); let conn = conn_guard .as_mut() .ok_or_else(|| vm.new_value_error("Connection not established"))?; - let is_bio = self.is_bio_mode(); - let data: &[u8] = data_bytes.as_ref(); - - // Write data in chunks to avoid filling the internal TLS buffer - // rustls has a limited internal buffer, so we need to flush periodically - const CHUNK_SIZE: usize = 16384; // 16KB chunks (typical TLS record size) - let mut written = 0; - - while written < data.len() { - let chunk_end = std::cmp::min(written + CHUNK_SIZE, data.len()); - let chunk = &data[written..chunk_end]; - - // Write chunk to TLS layer - { - let mut writer = conn.writer(); - use std::io::Write; - writer - .write_all(chunk) - .map_err(|e| vm.new_os_error(format!("Write failed: {e}")))?; - } - - written = chunk_end; - - // Flush TLS data to socket after each chunk - if conn.wants_write() { - if is_bio { - self.write_pending_tls(conn, vm)?; - } else { - // Socket mode: flush all pending TLS data - while conn.wants_write() { - let mut buf = Vec::new(); - conn.write_tls(&mut buf).map_err(|e| { - vm.new_os_error(format!("TLS write failed: {e}")) - })?; - - if !buf.is_empty() { - let timed_out = - self.sock_wait_for_io_impl(SelectKind::Write, vm)?; - if timed_out { - return Err(vm.new_os_error("Write operation timed out")); - } + crate::ssl::compat::ssl_write(conn, data_bytes.as_ref(), self, vm) + }; - match self.sock_send(buf, vm) { - Ok(_) => {} - Err(e) => { - if is_blocking_io_error(&e, vm) { - return Err( - create_ssl_want_write_error(vm).upcast() - ); - } - return Err(e); - } - } - } - } - } - } + match result { + Ok(n) => { + self.check_deferred_cert_error(vm)?; + Ok(n) } + Err(crate::ssl::compat::SslError::WantRead) => { + Err(create_ssl_want_read_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::WantWrite) => { + Err(create_ssl_want_write_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::Timeout(msg)) => { + Err(timeout_error_msg(vm, msg).upcast()) + } + Err(e) => Err(e.into_py_err(vm)), } - - // Check for deferred certificate verification errors (TLS 1.3) - // Must be checked AFTER write completes, as the error may be set during I/O - self.check_deferred_cert_error(vm)?; - - Ok(data_len) } #[pymethod] fn getpeercert( &self, - binary_form: OptionalArg<bool>, + args: GetCertArgs, vm: &VirtualMachine, ) -> PyResult<Option<PyObjectRef>> { - let binary = binary_form.unwrap_or(false); + let binary = args.binary_form.unwrap_or(false); // Check if handshake is complete if !*self.handshake_done.lock() { @@ -3861,19 +4044,49 @@ mod _ssl { .as_mut() .ok_or_else(|| vm.new_value_error("Connection not established"))?; + let is_bio = self.is_bio_mode(); + // Step 1: Send our close_notify if not already sent if current_state == ShutdownState::NotStarted { + // First, flush ALL pending TLS data BEFORE sending close_notify + // This is CRITICAL - close_notify must come AFTER all application data + // Otherwise data loss occurs when peer receives close_notify first + + // Step 1a: Flush any pending TLS records from rustls internal buffer + // This ensures all application data is converted to TLS records + while conn.wants_write() { + let mut buf = Vec::new(); + conn.write_tls(&mut buf) + .map_err(|e| vm.new_os_error(format!("TLS write failed: {e}")))?; + if !buf.is_empty() { + self.send_tls_output(buf, vm)?; + } + } + + // Step 1b: Flush pending_tls_output buffer to socket + if !is_bio { + // Socket mode: blocking flush to ensure data order + // Must complete before sending close_notify + self.blocking_flush_all_pending(vm)?; + } else { + // BIO mode: non-blocking flush (caller handles pending data) + let _ = self.flush_pending_tls_output(vm, None); + } + conn.send_close_notify(); // Write close_notify to outgoing buffer/BIO self.write_pending_tls(conn, vm)?; + // Ensure close_notify and any pending TLS data are flushed + if !is_bio { + self.flush_pending_tls_output(vm, None)?; + } // Update state *self.shutdown_state.lock() = ShutdownState::SentCloseNotify; } // Step 2: Try to read and process peer's close_notify - let is_bio = self.is_bio_mode(); // First check if we already have peer's close_notify // This can happen if it was received during a previous read() call @@ -3881,46 +4094,156 @@ mod _ssl { // If peer hasn't closed yet, try to read from socket if !peer_closed { - // Check if socket is in blocking mode (timeout is None) - let is_blocking = if !is_bio { + // Check socket timeout mode + let timeout_mode = if !is_bio { // Get socket timeout match self.sock.get_attr("gettimeout", vm) { Ok(method) => match method.call((), vm) { - Ok(timeout) => vm.is_none(&timeout), - Err(_) => false, + Ok(timeout) => { + if vm.is_none(&timeout) { + // timeout=None means blocking + Some(None) + } else if let Ok(t) = timeout.try_float(vm).map(|f| f.to_f64()) { + if t == 0.0 { + // timeout=0 means non-blocking + Some(Some(0.0)) + } else { + // timeout>0 means timeout mode + Some(Some(t)) + } + } else { + None + } + } + Err(_) => None, }, - Err(_) => false, + Err(_) => None, } } else { - false + None // BIO mode }; if is_bio { // In BIO mode: non-blocking read attempt - let _ = self.try_read_close_notify(conn, vm); - } else if is_blocking { - // Blocking socket mode: Return immediately without waiting for peer - // - // Reasons we don't read from socket here: - // 1. STARTTLS scenario: application data may arrive before/instead of close_notify - // - Example: client sends ENDTLS, immediately sends plain "msg 5" - // - Server's unwrap() would read "msg 5" and try to parse as TLS → FAIL - // 2. CPython's SSL_shutdown() typically returns immediately without waiting - // 3. Bidirectional shutdown is the application's responsibility - // 4. Reading from socket would consume application data incorrectly - // - // Therefore: Just send our close_notify and return success immediately. - // The peer's close_notify (if any) will remain in the socket buffer. - // - // Mark shutdown as complete and return the underlying socket - drop(conn_guard); - *self.shutdown_state.lock() = ShutdownState::Completed; - *self.connection.lock() = None; - return Ok(self.sock.clone()); + if self.try_read_close_notify(conn, vm)? { + peer_closed = true; + } + } else if let Some(timeout) = timeout_mode { + match timeout { + Some(0.0) => { + // Non-blocking: return immediately after sending close_notify. + // Don't wait for peer's close_notify to avoid blocking. + drop(conn_guard); + // Best-effort flush; WouldBlock is expected in non-blocking mode. + // Other errors indicate close_notify may not have been sent, + // but we still complete shutdown to avoid inconsistent state. + let _ = self.flush_pending_tls_output(vm, None); + *self.shutdown_state.lock() = ShutdownState::Completed; + *self.connection.lock() = None; + return Ok(self.sock.clone()); + } + _ => { + // Blocking or timeout mode: wait for peer's close_notify. + // This is proper TLS shutdown - we should receive peer's + // close_notify before closing the connection. + drop(conn_guard); + + // Flush our close_notify first + if timeout.is_none() { + self.blocking_flush_all_pending(vm)?; + } else { + self.flush_pending_tls_output(vm, None)?; + } + + // Calculate deadline for timeout mode + let deadline = timeout.map(|t| { + std::time::Instant::now() + core::time::Duration::from_secs_f64(t) + }); + + // Wait for peer's close_notify + loop { + // Re-acquire connection lock for each iteration + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(c) => c, + None => break, // Connection already closed + }; + + // Check if peer already sent close_notify + if self.check_peer_closed(conn, vm)? { + break; + } + + drop(conn_guard); + + // Check timeout + let remaining_timeout = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + // Timeout reached - raise TimeoutError + return Err(vm.new_exception_msg( + vm.ctx.exceptions.timeout_error.to_owned(), + "The read operation timed out".to_owned(), + )); + } + Some(dl - now) + } else { + None // Blocking mode: no timeout + }; + + // Wait for socket to be readable + let timed_out = self.sock_wait_for_io_with_timeout( + SelectKind::Read, + remaining_timeout, + vm, + )?; + + if timed_out { + // Timeout waiting for peer's close_notify + // Raise TimeoutError + return Err(vm.new_exception_msg( + vm.ctx.exceptions.timeout_error.to_owned(), + "The read operation timed out".to_owned(), + )); + } + + // Try to read data from socket + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(c) => c, + None => break, + }; + + // Read and process any incoming TLS data + match self.try_read_close_notify(conn, vm) { + Ok(closed) => { + if closed { + break; + } + // Check again after processing + if self.check_peer_closed(conn, vm)? { + break; + } + } + Err(_) => { + // Socket error - peer likely closed connection + break; + } + } + } + + // Shutdown complete + *self.shutdown_state.lock() = ShutdownState::Completed; + *self.connection.lock() = None; + return Ok(self.sock.clone()); + } + } } // Step 3: Check again if peer has sent close_notify (non-blocking/BIO mode only) - peer_closed = self.check_peer_closed(conn, vm)?; + if !peer_closed { + peer_closed = self.check_peer_closed(conn, vm)?; + } } drop(conn_guard); // Release lock before returning @@ -3942,6 +4265,10 @@ mod _ssl { // Helper: Write all pending TLS data (including close_notify) to outgoing buffer/BIO fn write_pending_tls(&self, conn: &mut TlsConnection, vm: &VirtualMachine) -> PyResult<()> { + // First, flush any previously pending TLS output + // Must succeed before sending new data to maintain order + self.flush_pending_tls_output(vm, None)?; + loop { if !conn.wants_write() { break; @@ -3956,42 +4283,135 @@ mod _ssl { break; } - // Send to outgoing BIO or socket - self.sock_send(buf[..written].to_vec(), vm)?; + // Send TLS data, saving unsent bytes to pending buffer if needed + self.send_tls_output(buf[..written].to_vec(), vm)?; } Ok(()) } - // Helper: Try to read incoming data from BIO (non-blocking) + // Helper: Try to read incoming data from socket/BIO + // Returns true if peer closed connection (with or without close_notify) fn try_read_close_notify( &self, conn: &mut TlsConnection, vm: &VirtualMachine, - ) -> PyResult<()> { - // Try to read incoming data from BIO - // This is non-blocking in BIO mode - if no data, recv returns empty + ) -> PyResult<bool> { + // In socket mode, peek first to avoid consuming post-TLS cleartext + // data. During STARTTLS, after close_notify exchange, the socket + // transitions to cleartext. Without peeking, sock_recv may consume + // cleartext data meant for the application after unwrap(). + if self.incoming_bio.is_none() { + return self.try_read_close_notify_socket(conn, vm); + } + + // BIO mode: read from incoming BIO match self.sock_recv(SSL3_RT_MAX_PLAIN_LENGTH, vm) { Ok(bytes_obj) => { let bytes = ArgBytesLike::try_from_object(vm, bytes_obj)?; let data = bytes.borrow_buf(); - if !data.is_empty() { - // Feed data to TLS connection - let data_slice: &[u8] = data.as_ref(); - let mut cursor = std::io::Cursor::new(data_slice); - let _ = conn.read_tls(&mut cursor); + if data.is_empty() { + if let Some(ref bio) = self.incoming_bio { + // BIO mode: check if EOF was signaled via write_eof() + let bio_obj: PyObjectRef = bio.clone().into(); + let eof_attr = bio_obj.get_attr("eof", vm)?; + let is_eof = eof_attr.try_to_bool(vm)?; + if !is_eof { + return Ok(false); + } + } + return Ok(true); + } - // Process packets - let _ = conn.process_new_packets(); + let data_slice: &[u8] = data.as_ref(); + let mut cursor = std::io::Cursor::new(data_slice); + let _ = conn.read_tls(&mut cursor); + let _ = conn.process_new_packets(); + Ok(false) + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); } + Ok(true) } - Err(_) => { - // No data available or error - that's OK in BIO mode + } + } + + /// Socket-mode close_notify reader that respects TLS record boundaries. + /// Uses MSG_PEEK to inspect data before consuming, preventing accidental + /// consumption of post-TLS cleartext data during STARTTLS transitions. + /// + /// Equivalent to OpenSSL's `SSL_set_read_ahead(ssl, 0)` — rustls has no + /// such knob, so we enforce record-level reads manually via peek. + fn try_read_close_notify_socket( + &self, + conn: &mut TlsConnection, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // Peek at the first 5 bytes (TLS record header size) + let peeked_obj = match self.sock_peek(5, vm) { + Ok(obj) => obj, + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); + } + return Ok(true); } + }; + + let peeked = ArgBytesLike::try_from_object(vm, peeked_obj)?; + let peek_data = peeked.borrow_buf(); + + if peek_data.is_empty() { + return Ok(true); // EOF } - Ok(()) + // TLS record content types: ChangeCipherSpec(20), Alert(21), + // Handshake(22), ApplicationData(23) + let content_type = peek_data[0]; + if !(20..=23).contains(&content_type) { + // Not a TLS record - post-TLS cleartext data. + // Peer has completed TLS shutdown; don't consume this data. + return Ok(true); + } + + // Determine how many bytes to read for exactly one TLS record + let recv_size = if peek_data.len() >= 5 { + let record_length = u16::from_be_bytes([peek_data[3], peek_data[4]]) as usize; + 5 + record_length + } else { + // Partial header available - read just these bytes for now + peek_data.len() + }; + + drop(peek_data); + drop(peeked); + + // Now consume exactly one TLS record from the socket + match self.sock_recv(recv_size, vm) { + Ok(bytes_obj) => { + let bytes = ArgBytesLike::try_from_object(vm, bytes_obj)?; + let data = bytes.borrow_buf(); + + if data.is_empty() { + return Ok(true); + } + + let data_slice: &[u8] = data.as_ref(); + let mut cursor = std::io::Cursor::new(data_slice); + let _ = conn.read_tls(&mut cursor); + let _ = conn.process_new_packets(); + Ok(false) + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); + } + Ok(true) + } + } } // Helper: Check if peer has sent close_notify @@ -4149,6 +4569,20 @@ mod _ssl { } } + // Clean up SSL socket resources on drop + impl Drop for PySSLSocket { + fn drop(&mut self) { + // Only clear connection state. + // Do NOT clear pending_tls_output - it may contain data that hasn't + // been flushed to the socket yet. SSLSocket._real_close() in Python + // doesn't call shutdown(), so when the socket is closed, pending TLS + // data would be lost if we clear it here. + // All fields (Vec, primitives) are automatically freed when the + // struct is dropped, so explicit clearing is unnecessary. + let _ = self.connection.lock().take(); + } + } + // MemoryBIO - provides in-memory buffer for SSL/TLS I/O #[pyattr] #[pyclass(name = "MemoryBIO", module = "ssl")] @@ -4290,8 +4724,8 @@ mod _ssl { #[pygetset] fn id(&self, vm: &VirtualMachine) -> PyBytesRef { // Return session ID (hash of session data for uniqueness) + use core::hash::{Hash, Hasher}; use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); self.session_data.hash(&mut hasher); @@ -4586,7 +5020,7 @@ mod _ssl { let store_name_wide: Vec<u16> = store_name .as_str() .encode_utf16() - .chain(std::iter::once(0)) + .chain(core::iter::once(0)) .collect(); // Open system store @@ -4601,7 +5035,7 @@ mod _ssl { let mut result = Vec::new(); - let mut crl_context: *const CRL_CONTEXT = std::ptr::null(); + let mut crl_context: *const CRL_CONTEXT = core::ptr::null(); loop { crl_context = unsafe { CertEnumCRLsInStore(store, crl_context) }; if crl_context.is_null() { @@ -4610,7 +5044,7 @@ mod _ssl { let crl = unsafe { &*crl_context }; let crl_bytes = - unsafe { std::slice::from_raw_parts(crl.pbCrlEncoded, crl.cbCrlEncoded as usize) }; + unsafe { core::slice::from_raw_parts(crl.pbCrlEncoded, crl.cbCrlEncoded as usize) }; let enc_type = if crl.dwCertEncodingType == X509_ASN_ENCODING { vm.new_pyobj("x509_asn") @@ -4701,8 +5135,8 @@ mod _ssl { // Implement Hashable trait for PySSLCertificate impl Hashable for PySSLCertificate { fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + use core::hash::{Hash, Hasher}; use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); zelf.der_bytes.hash(&mut hasher); diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs index b3cb7d6c14e..cd39972cf41 100644 --- a/crates/stdlib/src/ssl/cert.rs +++ b/crates/stdlib/src/ssl/cert.rs @@ -9,6 +9,7 @@ //! - Building and verifying certificate chains //! - Loading certificates from files, directories, and bytes +use alloc::sync::Arc; use chrono::{DateTime, Utc}; use parking_lot::RwLock as ParkingRwLock; use rustls::{ @@ -19,7 +20,6 @@ use rustls::{ }; use rustpython_vm::{PyObjectRef, PyResult, VirtualMachine}; use std::collections::HashSet; -use std::sync::Arc; use x509_parser::prelude::*; use super::compat::{VERIFY_X509_PARTIAL_CHAIN, VERIFY_X509_STRICT}; @@ -51,8 +51,9 @@ const ALL_SIGNATURE_SCHEMES: &[SignatureScheme] = &[ /// operations, reducing code duplication and ensuring uniform error messages /// across the codebase. mod cert_error { + use alloc::sync::Arc; + use core::fmt::{Debug, Display}; use std::io; - use std::sync::Arc; /// Create InvalidData error with formatted message pub fn invalid_data(msg: impl Into<String>) -> io::Error { @@ -67,11 +68,11 @@ mod cert_error { invalid_data(format!("no start line: {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse PEM certificate: {e}")) } - pub fn parse_failed_debug(e: impl std::fmt::Debug) -> io::Error { + pub fn parse_failed_debug(e: impl Debug) -> io::Error { invalid_data(format!("Failed to parse PEM certificate: {e:?}")) } @@ -88,7 +89,7 @@ mod cert_error { invalid_data(format!("not enough data: {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse DER certificate: {e}")) } } @@ -101,15 +102,15 @@ mod cert_error { invalid_data(format!("No private key found in {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse private key: {e}")) } - pub fn parse_encrypted_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_encrypted_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse encrypted private key: {e}")) } - pub fn decrypt_failed(e: impl std::fmt::Display) -> io::Error { + pub fn decrypt_failed(e: impl Display) -> io::Error { io::Error::other(format!( "Failed to decrypt private key (wrong password?): {e}", )) @@ -383,7 +384,7 @@ pub fn cert_der_to_dict_helper(vm: &VirtualMachine, cert_der: &[u8]) -> PyResult s.to_string() } else { let value_bytes = attr.attr_value().data; - match std::str::from_utf8(value_bytes) { + match core::str::from_utf8(value_bytes) { Ok(s) => s.to_string(), Err(_) => String::from_utf8_lossy(value_bytes).into_owned(), } @@ -1126,7 +1127,7 @@ pub(super) fn load_cert_chain_from_file( cert_path: &str, key_path: &str, password: Option<&str>, -) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), Box<dyn std::error::Error>> { +) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), Box<dyn core::error::Error>> { // Load certificate file - preserve io::Error for errno let cert_contents = std::fs::read(cert_path)?; @@ -1727,13 +1728,13 @@ fn verify_ip_address( cert: &X509Certificate<'_>, expected_ip: &rustls::pki_types::IpAddr, ) -> Result<(), rustls::Error> { - use std::net::IpAddr; + use core::net::IpAddr; use x509_parser::extensions::GeneralName; // Convert rustls IpAddr to std::net::IpAddr for comparison let expected_std_ip: IpAddr = match expected_ip { - rustls::pki_types::IpAddr::V4(octets) => IpAddr::V4(std::net::Ipv4Addr::from(*octets)), - rustls::pki_types::IpAddr::V6(octets) => IpAddr::V6(std::net::Ipv6Addr::from(*octets)), + rustls::pki_types::IpAddr::V4(octets) => IpAddr::V4(core::net::Ipv4Addr::from(*octets)), + rustls::pki_types::IpAddr::V6(octets) => IpAddr::V6(core::net::Ipv6Addr::from(*octets)), }; // Check Subject Alternative Names for IP addresses @@ -1745,7 +1746,7 @@ fn verify_ip_address( 4 => { // IPv4 if let Ok(octets) = <[u8; 4]>::try_from(*cert_ip_bytes) { - IpAddr::V4(std::net::Ipv4Addr::from(octets)) + IpAddr::V4(core::net::Ipv4Addr::from(octets)) } else { continue; } @@ -1753,7 +1754,7 @@ fn verify_ip_address( 16 => { // IPv6 if let Ok(octets) = <[u8; 16]>::try_from(*cert_ip_bytes) { - IpAddr::V6(std::net::Ipv6Addr::from(octets)) + IpAddr::V6(core::net::Ipv6Addr::from(octets)) } else { continue; } diff --git a/crates/stdlib/src/ssl/compat.rs b/crates/stdlib/src/ssl/compat.rs index 798542f210a..5bf2cd8b60f 100644 --- a/crates/stdlib/src/ssl/compat.rs +++ b/crates/stdlib/src/ssl/compat.rs @@ -13,6 +13,7 @@ mod ssl_data; use crate::socket::{SelectKind, timeout_error_msg}; use crate::vm::VirtualMachine; +use alloc::sync::Arc; use parking_lot::RwLock as ParkingRwLock; use rustls::RootCertStore; use rustls::client::ClientConfig; @@ -23,16 +24,19 @@ use rustls::server::ResolvesServerCert; use rustls::server::ServerConfig; use rustls::server::ServerConnection; use rustls::sign::CertifiedKey; -use rustpython_vm::builtins::PyBaseExceptionRef; +use rustpython_vm::builtins::{PyBaseException, PyBaseExceptionRef}; use rustpython_vm::convert::IntoPyException; use rustpython_vm::function::ArgBytesLike; -use rustpython_vm::{AsObject, PyObjectRef, PyPayload, PyResult, TryFromObject}; +use rustpython_vm::{AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject}; use std::io::Read; -use std::sync::{Arc, Once}; +use std::sync::Once; -// Import PySSLSocket and helper functions from parent module -use super::_ssl::{ - PySSLCertVerificationError, PySSLError, PySSLSocket, create_ssl_eof_error, +// Import PySSLSocket from parent module +use super::_ssl::PySSLSocket; + +// Import error types and helper functions from error module +use super::error::{ + PySSLCertVerificationError, PySSLError, create_ssl_eof_error, create_ssl_syscall_error, create_ssl_want_read_error, create_ssl_want_write_error, create_ssl_zero_return_error, }; @@ -388,6 +392,8 @@ pub(super) enum SslError { ZeroReturn, /// Unexpected EOF without close_notify (protocol violation) Eof, + /// Non-TLS data received before handshake completed + PreauthData, /// Certificate verification error CertVerification(rustls::CertificateError), /// I/O error @@ -547,8 +553,8 @@ impl SslError { SslError::WantWrite => create_ssl_want_write_error(vm).upcast(), SslError::Timeout(msg) => timeout_error_msg(vm, msg).upcast(), SslError::Syscall(msg) => { - // Create SSLError with library=None for syscall errors during SSL operations - Self::create_ssl_error_with_reason(vm, None, &msg, msg.clone()) + // SSLSyscallError with errno=SSL_ERROR_SYSCALL (5) + create_ssl_syscall_error(vm, msg).upcast() } SslError::Ssl(msg) => vm .new_os_subtype_error( @@ -559,6 +565,15 @@ impl SslError { .upcast(), SslError::ZeroReturn => create_ssl_zero_return_error(vm).upcast(), SslError::Eof => create_ssl_eof_error(vm).upcast(), + SslError::PreauthData => { + // Non-TLS data received before handshake + Self::create_ssl_error_with_reason( + vm, + None, + "before TLS handshake with data", + "before TLS handshake with data", + ) + } SslError::CertVerification(cert_err) => { // Use the proper cert verification error creator create_ssl_cert_verification_error(vm, &cert_err).expect("unlikely to happen") @@ -984,10 +999,114 @@ pub(super) fn create_client_config(options: ClientConfigOptions) -> Result<Clien } /// Helper function - check if error is BlockingIOError -pub(super) fn is_blocking_io_error(err: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { +pub(super) fn is_blocking_io_error(err: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { err.fast_isinstance(vm.ctx.exceptions.blocking_io_error) } +// Socket I/O Helper Functions + +/// Send all bytes to socket, handling partial sends with blocking wait +/// +/// Loops until all bytes are sent. For blocking sockets, this will wait +/// until all data is sent. For non-blocking sockets, returns WantWrite +/// if no progress can be made. +/// Optional deadline parameter allows respecting a read deadline during flush. +fn send_all_bytes( + socket: &PySSLSocket, + buf: Vec<u8>, + vm: &VirtualMachine, + deadline: Option<std::time::Instant>, +) -> SslResult<()> { + // First, flush any previously pending TLS data with deadline + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + + if buf.is_empty() { + return Ok(()); + } + + let mut sent_total = 0; + while sent_total < buf.len() { + // Check deadline before each send attempt + if let Some(dl) = deadline + && std::time::Instant::now() >= dl + { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout("The operation timed out".to_string())); + } + + // Wait for socket to be writable before sending + let timed_out = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + socket + .sock_wait_for_io_with_timeout(SelectKind::Write, Some(dl - now), vm) + .map_err(SslError::Py)? + } else { + socket + .sock_wait_for_io_impl(SelectKind::Write, vm) + .map_err(SslError::Py)? + }; + if timed_out { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + + match socket.sock_send(&buf[sent_total..], vm) { + Ok(result) => { + let sent: usize = result + .try_to_value::<isize>(vm) + .map_err(SslError::Py)? + .try_into() + .map_err(|_| SslError::Syscall("Invalid send return value".to_string()))?; + if sent == 0 { + // No progress - save unsent bytes to pending buffer + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::WantWrite); + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + // Save unsent bytes to pending buffer + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::WantWrite); + } + // For other errors, also save unsent bytes + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Py(e)); + } + } + } + Ok(()) +} + // Handshake Helper Functions /// Write TLS handshake data to socket/BIO @@ -1002,6 +1121,12 @@ fn handshake_write_loop( ) -> SslResult<bool> { let mut made_progress = false; + // Flush any previously pending TLS data before generating new output + // Must succeed before sending new data to maintain order + socket + .flush_pending_tls_output(vm, None) + .map_err(SslError::Py)?; + while conn.wants_write() || force_initial_write { if force_initial_write && !conn.wants_write() { // No data to write on first iteration - break to avoid infinite loop @@ -1014,20 +1139,9 @@ fn handshake_write_loop( .map_err(SslError::Io)?; if written > 0 && !buf.is_empty() { - // Send directly without select - blocking sockets will wait automatically - // Handle BlockingIOError from non-blocking sockets - match socket.sock_send(buf, vm) { - Ok(_) => { - made_progress = true; - } - Err(e) => { - if is_blocking_io_error(&e, vm) { - // Non-blocking socket would block - return SSLWantWriteError - return Err(SslError::WantWrite); - } - return Err(SslError::Py(e)); - } - } + // Send all bytes to socket, handling partial sends + send_all_bytes(socket, buf, vm, None)?; + made_progress = true; } else if written == 0 { // No data written but wants_write is true - should not happen normally // Break to avoid infinite loop @@ -1145,7 +1259,7 @@ fn handle_handshake_complete( // Do NOT loop on wants_write() - avoid infinite loop/deadlock let tls_data = ssl_write_tls_records(conn)?; if !tls_data.is_empty() { - socket.sock_send(tls_data, vm).map_err(SslError::Py)?; + send_all_bytes(socket, tls_data, vm, None)?; } // IMPORTANT: Don't check wants_write() again! @@ -1153,15 +1267,36 @@ fn handle_handshake_complete( } } else if conn.wants_write() { // Send all pending data (e.g., TLS 1.3 NewSessionTicket) to socket + // Must drain ALL rustls buffer - don't break on WantWrite while conn.wants_write() { let tls_data = ssl_write_tls_records(conn)?; if tls_data.is_empty() { break; } - socket.sock_send(tls_data, vm).map_err(SslError::Py)?; + match send_all_bytes(socket, tls_data, vm, None) { + Ok(()) => {} + Err(SslError::WantWrite) => { + // Socket buffer full, data saved to pending_tls_output + // Flush pending and continue draining rustls buffer + socket + .blocking_flush_all_pending(vm) + .map_err(SslError::Py)?; + } + Err(e) => return Err(e), + } } } + // CRITICAL: Ensure all pending TLS data is sent before returning + // TLS 1.3 Finished must reach server before handshake is considered complete + // Without this, server may not process application data + if !socket.is_bio_mode() { + // Flush pending_tls_output to ensure all TLS data reaches the server + socket + .blocking_flush_all_pending(vm) + .map_err(SslError::Py)?; + } + Ok(true) } @@ -1234,14 +1369,23 @@ pub(super) fn ssl_do_handshake( // Send close_notify on error if !is_bio { conn.send_close_notify(); - // Actually send the close_notify alert + // Flush any pending TLS data before sending close_notify + let _ = socket.flush_pending_tls_output(vm, None); + // Actually send the close_notify alert using send_all_bytes + // for proper partial send handling and retry logic if let Ok(alert_data) = ssl_write_tls_records(conn) && !alert_data.is_empty() { - let _ = socket.sock_send(alert_data, vm); + let _ = send_all_bytes(socket, alert_data, vm, None); } } + // InvalidMessage during handshake means non-TLS data was received + // before the handshake completed (e.g., HTTP request to TLS server) + if matches!(e, rustls::Error::InvalidMessage(_)) { + return Err(SslError::PreauthData); + } + // Certificate verification errors are already handled by from_rustls return Err(SslError::from_rustls(e)); @@ -1283,9 +1427,7 @@ pub(super) fn ssl_do_handshake( break; } // Send to outgoing BIO - socket - .sock_send(buf[..n].to_vec(), vm) - .map_err(SslError::Py)?; + send_all_bytes(socket, buf[..n].to_vec(), vm, None)?; // Check if there's more to write if !conn.wants_write() { break; @@ -1331,9 +1473,17 @@ pub(super) fn ssl_do_handshake( } } - // If we exit the loop without completing handshake, return error - // Check rustls state to provide better error message + // If we exit the loop without completing handshake, return appropriate error if conn.is_handshaking() { + // For non-blocking sockets, return WantRead/WantWrite to signal caller + // should retry when socket is ready. This matches OpenSSL behavior. + if conn.wants_write() { + return Err(SslError::WantWrite); + } + if conn.wants_read() { + return Err(SslError::WantRead); + } + // Neither wants_read nor wants_write - this is a real error Err(SslError::Syscall(format!( "SSL handshake failed: incomplete after {iteration_count} iterations", ))) @@ -1367,6 +1517,16 @@ pub(super) fn ssl_read( None // BIO mode has no deadline }; + // CRITICAL: Flush any pending TLS output before reading + // This ensures data from previous write() calls is sent before we wait for response. + // Without this, write() may leave data in pending_tls_output (if socket buffer was full), + // and read() would timeout waiting for a response that the server never received. + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + // Loop to handle TLS records and post-handshake messages // Matches SSL_read behavior which loops until data is available // - CPython uses OpenSSL's SSL_read which loops on SSL_ERROR_WANT_READ/WANT_WRITE @@ -1375,6 +1535,7 @@ pub(super) fn ssl_read( // - Blocking sockets: sock_select() and recv() wait at kernel level (no CPU busy-wait) // - Non-blocking sockets: immediate return on first WantRead // - Deadline prevents timeout issues + loop { // Check deadline if let Some(deadline) = deadline @@ -1391,11 +1552,61 @@ pub(super) fn ssl_read( // Try to read plaintext from rustls buffer if let Some(n) = try_read_plaintext(conn, buf)? { + if n == 0 { + // EOF from TLS - close_notify received + // Return ZeroReturn so Python raises SSLZeroReturnError + return Err(SslError::ZeroReturn); + } return Ok(n); } - // No plaintext available and cannot read more TLS records + // No plaintext available and rustls doesn't want to read more TLS records if !needs_more_tls { + // Check if connection needs to write data first (e.g., TLS key update, renegotiation) + // This mirrors the handshake logic which checks both wants_read() and wants_write() + if conn.wants_write() && !is_bio { + // Check deadline BEFORE attempting flush + if let Some(deadline) = deadline + && std::time::Instant::now() >= deadline + { + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + + // Flush pending TLS data before continuing + // CRITICAL: Pass deadline so flush respects read timeout + let tls_data = ssl_write_tls_records(conn)?; + if !tls_data.is_empty() { + // Use best-effort send - don't fail READ just because WRITE couldn't complete + match send_all_bytes(socket, tls_data, vm, deadline) { + Ok(()) => {} + Err(SslError::WantWrite) => { + // Socket buffer full - acceptable during READ operation + // Pending data will be sent on next write/read call + } + Err(SslError::Timeout(_)) => { + // Timeout during flush is acceptable during READ + // Pending data stays buffered for next operation + } + Err(e) => return Err(e), + } + } + + // Check deadline AFTER flush attempt + if let Some(deadline) = deadline + && std::time::Instant::now() >= deadline + { + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + + // After flushing, rustls may want to read again - continue loop + continue; + } + + // BIO mode: check for EOF if is_bio && let Some(bio_obj) = socket.incoming_bio() { let is_eof = bio_obj .get_attr("eof", vm) @@ -1405,31 +1616,77 @@ pub(super) fn ssl_read( return Err(SslError::Eof); } } + + // For non-blocking sockets, return WantRead so caller can poll and retry. + // For blocking sockets (or sockets with timeout), wait for more data. + if !is_bio { + let timeout = socket.get_socket_timeout(vm).map_err(SslError::Py)?; + if let Some(t) = timeout + && t.is_zero() + { + // Non-blocking socket: check if peer has closed before returning WantRead + // If close_notify was received, we should return ZeroReturn (EOF), not WantRead + // This is critical for asyncore-based applications that rely on recv() returning + // 0 or raising SSL_ERROR_ZERO_RETURN to detect connection close. + let io_state = conn.process_new_packets().map_err(SslError::from_rustls)?; + if io_state.peer_has_closed() { + return Err(SslError::ZeroReturn); + } + // Non-blocking socket: return immediately + return Err(SslError::WantRead); + } + // Blocking socket or socket with timeout: try to read more data from socket. + // Even though rustls says it doesn't want to read, more TLS records may arrive. + // This handles the case where rustls processed all buffered TLS records but + // more data is coming over the network. + let data = match socket.sock_recv(2048, vm) { + Ok(data) => data, + Err(e) => { + if is_connection_closed_error(&e, vm) { + return Err(SslError::Eof); + } + return Err(SslError::Py(e)); + } + }; + + let bytes_read = data + .clone() + .try_into_value::<rustpython_vm::builtins::PyBytes>(vm) + .map(|b| b.as_bytes().len()) + .unwrap_or(0); + + if bytes_read == 0 { + // No more data available - check if this is clean shutdown or unexpected EOF + // If close_notify was already received, return ZeroReturn (clean closure) + // Otherwise, return Eof (unexpected EOF) + let io_state = conn.process_new_packets().map_err(SslError::from_rustls)?; + if io_state.peer_has_closed() { + return Err(SslError::ZeroReturn); + } + return Err(SslError::Eof); + } + + // Feed data to rustls and process + ssl_read_tls_records(conn, data, false, vm)?; + conn.process_new_packets().map_err(SslError::from_rustls)?; + + // Continue loop to try reading plaintext + continue; + } + return Err(SslError::WantRead); } // Read and process TLS records - // This will block for blocking sockets match ssl_ensure_data_available(conn, socket, vm) { Ok(_bytes_read) => { // Successfully read and processed TLS data // Continue loop to try reading plaintext } Err(SslError::Io(ref io_err)) if io_err.to_string().contains("message buffer full") => { - // Buffer is full - we need to consume plaintext before reading more - // Try to read plaintext now - match try_read_plaintext(conn, buf)? { - Some(n) if n > 0 => { - // Have plaintext - return it - // Python will call read() again if it needs more data - return Ok(n); - } - _ => { - // No plaintext available yet - this is unusual - // Return WantRead to let Python retry - return Err(SslError::WantRead); - } - } + // This case should be rare now that ssl_read_tls_records handles buffer full + // Just continue loop to try again + continue; } Err(e) => { // Other errors - check for buffered plaintext before propagating @@ -1448,6 +1705,187 @@ pub(super) fn ssl_read( } } +/// Equivalent to OpenSSL's SSL_write() +/// +/// Writes application data to TLS connection. +/// Automatically handles TLS record I/O as needed. +/// +/// = SSL_write_ex() +pub(super) fn ssl_write( + conn: &mut TlsConnection, + data: &[u8], + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<usize> { + if data.is_empty() { + return Ok(0); + } + + let is_bio = socket.is_bio_mode(); + + // Get socket timeout and calculate deadline (= _PyDeadline_Init) + let deadline = if !is_bio { + match socket.get_socket_timeout(vm).map_err(SslError::Py)? { + Some(timeout) if !timeout.is_zero() => Some(std::time::Instant::now() + timeout), + _ => None, + } + } else { + None + }; + + // Flush any pending TLS output before writing new data + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + + // Check if we already have data buffered from a previous retry + // (prevents duplicate writes when retrying after WantWrite/WantRead) + let already_buffered = *socket.write_buffered_len.lock(); + + // Only write plaintext if not already buffered + // Track how much we wrote for partial write handling + let mut bytes_written_to_rustls = 0usize; + + if already_buffered == 0 { + // Write plaintext to rustls (= SSL_write_ex internal buffer write) + bytes_written_to_rustls = { + let mut writer = conn.writer(); + use std::io::Write; + // Use write() instead of write_all() to support partial writes. + // In BIO mode (asyncio), when the internal buffer is full, + // we want to write as much as possible and return that count, + // rather than failing completely. + match writer.write(data) { + Ok(0) if !data.is_empty() => { + // Buffer is full and nothing could be written. + // In BIO mode, return WantWrite so the caller can + // drain the outgoing BIO and retry. + if is_bio { + return Err(SslError::WantWrite); + } + return Err(SslError::Syscall("Write failed: buffer full".to_string())); + } + Ok(n) => n, + Err(e) => { + if is_bio { + // In BIO mode, treat write errors as WantWrite + return Err(SslError::WantWrite); + } + return Err(SslError::Syscall(format!("Write failed: {e}"))); + } + } + }; + // Mark data as buffered (only the portion we actually wrote) + *socket.write_buffered_len.lock() = bytes_written_to_rustls; + } else if already_buffered != data.len() { + // Caller is retrying with different data - this is a protocol error + // Clear the buffer state and return an SSL error (bad write retry) + *socket.write_buffered_len.lock() = 0; + return Err(SslError::Ssl("bad write retry".to_string())); + } + // else: already_buffered == data.len(), this is a valid retry + + // Loop to send TLS records, handling WANT_READ/WANT_WRITE + // Matches CPython's do-while loop on SSL_ERROR_WANT_READ/WANT_WRITE + loop { + // Check deadline + if let Some(dl) = deadline + && std::time::Instant::now() >= dl + { + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + + // Check if rustls has TLS data to send + if !conn.wants_write() { + // All TLS data sent successfully + break; + } + + // Get TLS records from rustls + let tls_data = ssl_write_tls_records(conn)?; + if tls_data.is_empty() { + break; + } + + // Send TLS data to socket + match send_all_bytes(socket, tls_data, vm, deadline) { + Ok(()) => { + // Successfully sent, continue loop to check for more data + } + Err(SslError::WantWrite) => { + // Non-blocking socket would block - return WANT_WRITE + // If we had a partial write to rustls, return partial success + // instead of error to match OpenSSL partial-write semantics + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Keep write_buffered_len set so we don't re-buffer on retry + return Err(SslError::WantWrite); + } + Err(SslError::WantRead) => { + // Need to read before write can complete (e.g., renegotiation) + if is_bio { + // If we had a partial write to rustls, return partial success + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Keep write_buffered_len set so we don't re-buffer on retry + return Err(SslError::WantRead); + } + // For socket mode, try to read TLS data + let recv_result = socket.sock_recv(4096, vm).map_err(SslError::Py)?; + ssl_read_tls_records(conn, recv_result, false, vm)?; + conn.process_new_packets().map_err(SslError::from_rustls)?; + // Continue loop + } + Err(e @ SslError::Timeout(_)) => { + // If we had a partial write to rustls, return partial success + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Preserve buffered state so retry doesn't duplicate data + // (send_all_bytes saved unsent TLS bytes to pending_tls_output) + return Err(e); + } + Err(e) => { + // Clear buffer state on error + *socket.write_buffered_len.lock() = 0; + return Err(e); + } + } + } + + // Final flush to ensure all data is sent + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + + // Determine how many bytes we actually wrote + let actual_written = if bytes_written_to_rustls > 0 { + // Fresh write: return what we wrote to rustls + bytes_written_to_rustls + } else if already_buffered > 0 { + // Retry of previous write: return the full buffered amount + already_buffered + } else { + data.len() + }; + + // Write completed successfully - clear buffer state + *socket.write_buffered_len.lock() = 0; + + Ok(actual_written) +} + // Helper functions (private-ish, used by public SSL functions) /// Write TLS records from rustls to socket @@ -1484,31 +1922,29 @@ fn ssl_read_tls_records( // 1. Clean shutdown: received TLS close_notify → return ZeroReturn (0 bytes) // 2. Unexpected EOF: no close_notify → return Eof (SSLEOFError) // - // SSL_ERROR_ZERO_RETURN vs SSL_ERROR_SYSCALL(errno=0) logic + // SSL_ERROR_ZERO_RETURN vs SSL_ERROR_EOF logic // CPython checks SSL_get_shutdown() & SSL_RECEIVED_SHUTDOWN // // Process any buffered TLS records (may contain close_notify) - let _ = conn.process_new_packets(); - - // IMPORTANT: CPython's default behavior (suppress_ragged_eofs=True) - // treats empty recv() as clean shutdown, returning 0 bytes instead of raising SSLEOFError. - // - // This is necessary for HTTP/1.0 servers that: - // 1. Send response without Content-Length header - // 2. Signal end-of-response by closing connection (TCP FIN) - // 3. Don't send TLS close_notify before TCP close - // - // While this could theoretically allow truncation attacks, - // it's the standard behavior for compatibility with real-world servers. - // Python only raises SSLEOFError when suppress_ragged_eofs=False is explicitly set. - // - // TODO: Implement suppress_ragged_eofs parameter if needed for strict security mode. - return Err(SslError::ZeroReturn); + match conn.process_new_packets() { + Ok(io_state) => { + if io_state.peer_has_closed() { + // Received close_notify - normal SSL closure (SSL_ERROR_ZERO_RETURN) + return Err(SslError::ZeroReturn); + } else { + // No close_notify - ragged EOF (SSL_ERROR_EOF → SSLEOFError) + // CPython raises SSLEOFError here, which SSLSocket.read() handles + // based on suppress_ragged_eofs setting + return Err(SslError::Eof); + } + } + Err(e) => return Err(SslError::from_rustls(e)), + } } } // Feed all received data to read_tls - loop to consume all data - // read_tls may not consume all data in one call + // read_tls may not consume all data in one call, and buffer may become full let mut offset = 0; while offset < bytes_data.len() { let remaining = &bytes_data[offset..]; @@ -1517,12 +1953,33 @@ fn ssl_read_tls_records( match conn.read_tls(&mut cursor) { Ok(read_bytes) => { if read_bytes == 0 { - // No more data can be consumed - break; + // Buffer is full - process existing packets to make room + conn.process_new_packets().map_err(SslError::from_rustls)?; + + // Try again - if we still can't consume, break + let mut retry_cursor = std::io::Cursor::new(remaining); + match conn.read_tls(&mut retry_cursor) { + Ok(0) => { + // Still can't consume - break to avoid infinite loop + break; + } + Ok(n) => { + offset += n; + } + Err(e) => { + return Err(SslError::Io(e)); + } + } + } else { + offset += read_bytes; } - offset += read_bytes; } Err(e) => { + // Check if it's a buffer full error (unlikely but handle it) + if e.to_string().contains("buffer full") { + conn.process_new_packets().map_err(SslError::from_rustls)?; + continue; + } // Real error - propagate it return Err(SslError::Io(e)); } @@ -1534,7 +1991,7 @@ fn ssl_read_tls_records( /// Check if an exception is a connection closed error /// In SSL context, these errors indicate unexpected connection termination without proper TLS shutdown -fn is_connection_closed_error(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { +fn is_connection_closed_error(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { use rustpython_vm::stdlib::errno::errors; // Check for ConnectionAbortedError, ConnectionResetError (Python exception types) @@ -1583,8 +2040,10 @@ fn ssl_ensure_data_available( .sock_wait_for_io_impl(SelectKind::Read, vm) .map_err(SslError::Py)?; if timed_out { - // Socket not ready within timeout - return Err(SslError::WantRead); + // Socket not ready within timeout - raise socket.timeout + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); } } // else: non-blocking socket (timeout=0) or blocking socket (timeout=None) - skip select @@ -1593,6 +2052,9 @@ fn ssl_ensure_data_available( let data = match socket.sock_recv(2048, vm) { Ok(data) => data, Err(e) => { + if is_blocking_io_error(&e, vm) { + return Err(SslError::WantRead); + } // Before returning socket error, check if rustls already has a queued TLS alert // This mirrors CPython/OpenSSL behavior: SSL errors take precedence over socket errors // On Windows, TCP RST may arrive before we read the alert, but rustls may have diff --git a/crates/stdlib/src/ssl/error.rs b/crates/stdlib/src/ssl/error.rs new file mode 100644 index 00000000000..cbc59e0e8f6 --- /dev/null +++ b/crates/stdlib/src/ssl/error.rs @@ -0,0 +1,146 @@ +// SSL exception types shared between ssl (rustls) and openssl backends + +pub(crate) use ssl_error::*; + +#[pymodule(sub)] +pub(crate) mod ssl_error { + use crate::vm::{ + Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseException, PyOSError, PyStrRef}, + types::Constructor, + }; + + // Error type constants - exposed as pyattr and available for internal use + #[pyattr] + pub(crate) const SSL_ERROR_NONE: i32 = 0; + #[pyattr] + pub(crate) const SSL_ERROR_SSL: i32 = 1; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_READ: i32 = 2; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_WRITE: i32 = 3; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_X509_LOOKUP: i32 = 4; + #[pyattr] + pub(crate) const SSL_ERROR_SYSCALL: i32 = 5; + #[pyattr] + pub(crate) const SSL_ERROR_ZERO_RETURN: i32 = 6; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_CONNECT: i32 = 7; + #[pyattr] + pub(crate) const SSL_ERROR_EOF: i32 = 8; + #[pyattr] + pub(crate) const SSL_ERROR_INVALID_ERROR_CODE: i32 = 10; + + #[pyattr] + #[pyexception(name = "SSLError", base = PyOSError)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLError(PyOSError); + + #[pyexception] + impl PySSLError { + // Returns strerror attribute if available, otherwise str(args) + #[pymethod] + fn __str__(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use crate::vm::AsObject; + // Try to get strerror attribute first (OSError compatibility) + if let Ok(strerror) = exc.as_object().get_attr("strerror", vm) + && !vm.is_none(&strerror) + { + return strerror.str(vm); + } + + // Otherwise return str(args) + let args = exc.args(); + if args.len() == 1 { + args.as_slice()[0].str(vm) + } else { + args.as_object().str(vm) + } + } + } + + #[pyattr] + #[pyexception(name = "SSLZeroReturnError", base = PySSLError)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLZeroReturnError(PySSLError); + + #[pyexception] + impl PySSLZeroReturnError {} + + #[pyattr] + #[pyexception(name = "SSLWantReadError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLWantReadError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLWantWriteError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLWantWriteError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLSyscallError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLSyscallError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLEOFError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLEOFError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLCertVerificationError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLCertVerificationError(PySSLError); + + // Helper functions to create SSL exceptions with proper errno attribute + pub fn create_ssl_want_read_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLWantReadError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_WANT_READ), + "The operation did not complete (read)", + ) + } + + pub fn create_ssl_want_write_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLWantWriteError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_WANT_WRITE), + "The operation did not complete (write)", + ) + } + + pub fn create_ssl_eof_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLEOFError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_EOF), + "EOF occurred in violation of protocol", + ) + } + + pub fn create_ssl_zero_return_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLZeroReturnError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_ZERO_RETURN), + "TLS/SSL connection has been closed (EOF)", + ) + } + + pub fn create_ssl_syscall_error( + vm: &VirtualMachine, + msg: impl Into<String>, + ) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLSyscallError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_SYSCALL), + msg.into(), + ) + } +} diff --git a/crates/stdlib/src/ssl/oid.rs b/crates/stdlib/src/ssl/oid.rs index 2e13733a2a2..d85898c0f79 100644 --- a/crates/stdlib/src/ssl/oid.rs +++ b/crates/stdlib/src/ssl/oid.rs @@ -132,7 +132,8 @@ impl OidTable { } /// Global OID table -static OID_TABLE: std::sync::LazyLock<OidTable> = std::sync::LazyLock::new(OidTable::build); +static OID_TABLE: rustpython_common::lock::LazyLock<OidTable> = + rustpython_common::lock::LazyLock::new(OidTable::build); /// Macro to define OID entry using oid-registry constant macro_rules! oid_static { diff --git a/crates/stdlib/src/statistics.rs b/crates/stdlib/src/statistics.rs index 9f5b294c009..2f7eb85284a 100644 --- a/crates/stdlib/src/statistics.rs +++ b/crates/stdlib/src/statistics.rs @@ -1,11 +1,14 @@ -pub(crate) use _statistics::make_module; +pub(crate) use _statistics::module_def; #[pymodule] mod _statistics { use crate::vm::{PyResult, VirtualMachine, function::ArgIntoFloat}; // See https://github.com/python/cpython/blob/6846d6712a0894f8e1a91716c11dd79f42864216/Modules/_statisticsmodule.c#L28-L120 - #[allow(clippy::excessive_precision)] + #[allow( + clippy::excessive_precision, + reason = "constants are kept at CPython precision" + )] fn normal_dist_inv_cdf(p: f64, mu: f64, sigma: f64) -> Option<f64> { if p <= 0.0 || p >= 1.0 { return None; @@ -53,7 +56,10 @@ mod _statistics { let r = (-(r.ln())).sqrt(); let num; let den; - #[allow(clippy::excessive_precision)] + #[allow( + clippy::excessive_precision, + reason = "piecewise polynomial coefficients match CPython" + )] if r <= 5.0 { let r = r - 1.6; // Hash sum-49.33206503301610289036 @@ -126,7 +132,7 @@ mod _statistics { sigma: ArgIntoFloat, vm: &VirtualMachine, ) -> PyResult<f64> { - normal_dist_inv_cdf(*p, *mu, *sigma) + normal_dist_inv_cdf(p.into_float(), mu.into_float(), sigma.into_float()) .ok_or_else(|| vm.new_value_error("inv_cdf undefined for these parameters")) } } diff --git a/crates/stdlib/src/suggestions.rs b/crates/stdlib/src/suggestions.rs index e49e9dd4a4b..e0667dfb553 100644 --- a/crates/stdlib/src/suggestions.rs +++ b/crates/stdlib/src/suggestions.rs @@ -1,4 +1,4 @@ -pub(crate) use _suggestions::make_module; +pub(crate) use _suggestions::module_def; #[pymodule] mod _suggestions { diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index adba6f297ce..8f446a8e161 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -1,6 +1,6 @@ // spell-checker:ignore logoption openlog setlogmask upto NDELAY ODELAY -pub(crate) use syslog::make_module; +pub(crate) use syslog::module_def; #[pymodule(name = "syslog")] mod syslog { @@ -11,7 +11,8 @@ mod syslog { function::{OptionalArg, OptionalOption}, utils::ToCString, }; - use std::{ffi::CStr, os::raw::c_char}; + use core::ffi::CStr; + use std::os::raw::c_char; #[pyattr] use libc::{ @@ -26,7 +27,7 @@ mod syslog { use libc::{LOG_AUTHPRIV, LOG_CRON, LOG_PERROR}; fn get_argv(vm: &VirtualMachine) -> Option<PyStrRef> { - if let Some(argv) = vm.state.settings.argv.first() + if let Some(argv) = vm.state.config.settings.argv.first() && !argv.is_empty() { return Some( @@ -50,7 +51,7 @@ mod syslog { fn as_ptr(&self) -> *const c_char { match self { Self::Explicit(cstr) => cstr.as_ptr(), - Self::Implicit => std::ptr::null(), + Self::Implicit => core::ptr::null(), } } } diff --git a/crates/stdlib/src/termios.rs b/crates/stdlib/src/termios.rs index a9ae1375c6f..de402724434 100644 --- a/crates/stdlib/src/termios.rs +++ b/crates/stdlib/src/termios.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use self::termios::make_module; +pub(crate) use self::termios::module_def; #[pymodule] mod termios { @@ -261,13 +261,12 @@ mod termios { } fn termios_error(err: std::io::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception( + vm.new_os_subtype_error( error_type(vm), - vec![ - err.posix_errno().to_pyobject(vm), - vm.ctx.new_str(err.to_string()).into(), - ], + Some(err.posix_errno()), + vm.ctx.new_str(err.to_string()), ) + .upcast() } #[pyattr(name = "error", once)] diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 687458b193b..b258002c129 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -1,11 +1,12 @@ // spell-checker:ignore createcommand -pub(crate) use self::_tkinter::make_module; +pub(crate) use self::_tkinter::module_def; #[pymodule] mod _tkinter { + use rustpython_vm::convert::IntoPyException; use rustpython_vm::types::Constructor; - use rustpython_vm::{PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; + use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use rustpython_vm::builtins::{PyInt, PyStr, PyType}; use std::{ffi, ptr}; @@ -59,8 +60,8 @@ mod _tkinter { value: *mut tk_sys::Tcl_Obj, } - impl std::fmt::Debug for TclObject { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for TclObject { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "TclObject") } } @@ -72,6 +73,7 @@ mod _tkinter { impl TclObject {} static QUIT_MAIN_LOOP: AtomicBool = AtomicBool::new(false); + static ERROR_IN_CMD: AtomicBool = AtomicBool::new(false); #[pyattr] #[pyclass(name = "tkapp")] @@ -107,8 +109,8 @@ mod _tkinter { unsafe impl Send for TkApp {} unsafe impl Sync for TkApp {} - impl std::fmt::Debug for TkApp { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for TkApp { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "TkApp") } } @@ -125,7 +127,7 @@ mod _tkinter { interactive: i32, #[pyarg(any)] wantobjects: i32, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] want_tk: bool, #[pyarg(any)] sync: i32, @@ -136,11 +138,7 @@ mod _tkinter { impl Constructor for TkApp { type Args = TkAppConstructorArgs; - fn py_new( - _zelf: PyRef<PyType>, - args: Self::Args, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { create(args, vm) } } @@ -168,10 +166,18 @@ mod _tkinter { ))) } + #[derive(Debug, FromArgs)] + struct TkAppGetVarArgs { + #[pyarg(any)] + name: PyObjectRef, + #[pyarg(any, default)] + name2: Option<String>, + } + // TODO: DISALLOW_INSTANTIATION #[pyclass(with(Constructor))] impl TkApp { - fn from_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { + fn tcl_obj_to_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { let mut res = -1; unsafe { if tk_sys::Tcl_GetBooleanFromObj(self.interpreter, obj, &mut res) @@ -184,16 +190,16 @@ mod _tkinter { res != 0 } - fn from_object( + fn tcl_obj_to_pyobject( &self, obj: *mut tk_sys::Tcl_Obj, vm: &VirtualMachine, ) -> PyResult<PyObjectRef> { let type_ptr = unsafe { (*obj).typePtr }; - if type_ptr == ptr::null() { + if type_ptr.is_null() { return self.unicode_from_object(obj, vm); } else if type_ptr == self.old_boolean_type || type_ptr == self.boolean_type { - return Ok(vm.ctx.new_bool(self.from_bool(obj)).into()); + return Ok(vm.ctx.new_bool(self.tcl_obj_to_bool(obj)).into()); } else if type_ptr == self.string_type || type_ptr == self.utf32_string_type || type_ptr == self.pixel_type @@ -202,7 +208,7 @@ mod _tkinter { } // TODO: handle other types - return Ok(TclObject { value: obj }.into_pyobject(vm)); + Ok(TclObject { value: obj }.into_pyobject(vm)) } fn unicode_from_string( @@ -226,8 +232,8 @@ mod _tkinter { vm: &VirtualMachine, ) -> PyResult<PyObjectRef> { let type_ptr = unsafe { (*obj).typePtr }; - if type_ptr != ptr::null() - && self.interpreter != ptr::null_mut() + if !type_ptr.is_null() + && !self.interpreter.is_null() && (type_ptr == self.string_type || type_ptr == self.utf32_string_type) { let len = ptr::null_mut(); @@ -247,30 +253,75 @@ mod _tkinter { Self::unicode_from_string(s, len as _, vm) } - #[pymethod] - fn getvar(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + fn var_invoke(&self) { + if self.threaded && self.thread_id != Some(unsafe { tk_sys::Tcl_GetCurrentThread() }) { + // TODO: do stuff + } + } + + fn inner_getvar( + &self, + args: TkAppGetVarArgs, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let TkAppGetVarArgs { name, name2 } = args; // TODO: technically not thread safe - let name = varname_converter(arg, vm)?; + let name = varname_converter(name, vm)?; + let name = ffi::CString::new(name).map_err(|e| e.into_pyexception(vm))?; + let name2 = + ffi::CString::new(name2.unwrap_or_default()).map_err(|e| e.into_pyexception(vm))?; + let name2_ptr = if name2.is_empty() { + ptr::null() + } else { + name2.as_ptr() + }; let res = unsafe { tk_sys::Tcl_GetVar2Ex( self.interpreter, - ptr::null(), name.as_ptr() as _, - tk_sys::TCL_LEAVE_ERR_MSG as _, + name2_ptr as _, + flags as _, ) }; - if res == ptr::null_mut() { - todo!(); + if res.is_null() { + // TODO: Should be tk error + unsafe { + let err_obj = tk_sys::Tcl_GetObjResult(self.interpreter); + let err_str_obj = tk_sys::Tcl_GetString(err_obj); + let err_cstr = ffi::CStr::from_ptr(err_str_obj as _); + return Err(vm.new_type_error(format!("{err_cstr:?}"))); + } } let res = if self.want_objects { - self.from_object(res, vm) + self.tcl_obj_to_pyobject(res, vm) } else { self.unicode_from_object(res, vm) }?; Ok(res) } + #[pymethod] + fn getvar(&self, args: TkAppGetVarArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.var_invoke(); + self.inner_getvar(args, tk_sys::TCL_LEAVE_ERR_MSG, vm) + } + + #[pymethod] + fn globalgetvar( + &self, + args: TkAppGetVarArgs, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + self.var_invoke(); + self.inner_getvar( + args, + tk_sys::TCL_LEAVE_ERR_MSG | tk_sys::TCL_GLOBAL_ONLY, + vm, + ) + } + #[pymethod] fn getint(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { if let Some(int) = arg.downcast_ref::<PyInt>() { @@ -289,7 +340,21 @@ mod _tkinter { #[pymethod] fn mainloop(&self, threshold: Option<i32>) -> PyResult<()> { let threshold = threshold.unwrap_or(0); - todo!(); + // self.dispatching = true; + QUIT_MAIN_LOOP.store(false, Ordering::Relaxed); + while unsafe { tk_sys::Tk_GetNumMainWindows() } > threshold + && !QUIT_MAIN_LOOP.load(Ordering::Relaxed) + && !ERROR_IN_CMD.load(Ordering::Relaxed) + { + if self.threaded { + unsafe { tk_sys::Tcl_DoOneEvent(0 as _) }; + } else { + unsafe { tk_sys::Tcl_DoOneEvent(tk_sys::TCL_DONT_WAIT as _) }; + // TODO: sleep for the proper time + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + Ok(()) } #[pymethod] @@ -299,13 +364,15 @@ mod _tkinter { } #[pyfunction] - fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult<TkApp> { unsafe { let interp = tk_sys::Tcl_CreateInterp(); let want_objects = args.wantobjects != 0; - let threaded = { + let threaded = !{ let part1 = String::from("tcl_platform"); let part2 = String::from("threaded"); + let part1 = ffi::CString::new(part1).map_err(|e| e.into_pyexception(vm))?; + let part2 = ffi::CString::new(part2).map_err(|e| e.into_pyexception(vm))?; let part1_ptr = part1.as_ptr(); let part2_ptr = part2.as_ptr(); tk_sys::Tcl_GetVar2Ex( @@ -314,7 +381,8 @@ mod _tkinter { part2_ptr as _, tk_sys::TCL_GLOBAL_ONLY as ffi::c_int, ) - } != ptr::null_mut(); + } + .is_null(); let thread_id = tk_sys::Tcl_GetCurrentThread(); let dispatching = false; let trace = None; @@ -339,8 +407,8 @@ mod _tkinter { let double_type = tk_sys::Tcl_GetObjType(double_str.as_ptr() as _); let int_str = String::from("int"); let int_type = tk_sys::Tcl_GetObjType(int_str.as_ptr() as _); - let int_type = if int_type == ptr::null() { - let mut value = *tk_sys::Tcl_NewIntObj(0); + let int_type = if int_type.is_null() { + let mut value = *tk_sys::Tcl_NewWideIntObj(0); let res = value.typePtr; tk_sys::Tcl_DecrRefCount(&mut value); res @@ -374,33 +442,37 @@ mod _tkinter { } if args.interactive != 0 { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "tcl_interactive".as_ptr() as _, + ptr::null(), "1".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); } else { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "tcl_interactive".as_ptr() as _, + ptr::null(), "0".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); } let argv0 = args.class_name.clone().to_lowercase(); - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "argv0".as_ptr() as _, + ptr::null(), argv0.as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); if !args.want_tk { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "_tkinter_skip_tk_init".as_ptr() as _, + ptr::null(), "1".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); @@ -418,11 +490,12 @@ mod _tkinter { argv.push_str("-use "); argv.push_str(&args.use_.unwrap()); } - argv.push_str("\0"); + argv.push('\0'); let argv_ptr = argv.as_ptr() as *mut *mut i8; - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "argv".as_ptr() as _, + ptr::null(), argv_ptr as *const i8, tk_sys::TCL_GLOBAL_ONLY as i32, ); @@ -460,8 +533,7 @@ mod _tkinter { string_type, utf32_string_type, pixel_type, - } - .into_pyobject(vm)) + }) } } diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index 46e18357260..a575c1ae7e5 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -4,35 +4,12 @@ // spell-checker:ignore nfkc unistr unidata +pub(crate) use unicodedata::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyModule, - builtins::PyStr, convert::TryFromBorrowedObject, + PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject, }; -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = unicodedata::make_module(vm); - - let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION) - .into_ref(&vm.ctx) - .into(); - - for attr in [ - "category", - "lookup", - "name", - "bidirectional", - "east_asian_width", - "normalize", - "mirrored", - ] { - crate::vm::extend_module!(vm, &module, { - attr => ucd.get_attr(attr, vm).unwrap(), - }); - } - - module -} - enum NormalizeForm { Nfc, Nfkc, @@ -60,12 +37,13 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { #[pymodule] mod unicodedata { use crate::vm::{ - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef, + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyModule, PyStrRef}, function::OptionalArg, }; use itertools::Itertools; use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; - use ucd::{Codepoint, EastAsianWidth}; + use ucd::{Codepoint, DecompositionType, EastAsianWidth, Number, NumericType}; use unic_char_property::EnumeratedCharProperty; use unic_normal::StrNormalForm; use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion}; @@ -73,6 +51,33 @@ mod unicodedata { use unic_ucd_category::GeneralCategory; use unicode_bidi_mirroring::is_mirroring; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Add UCD methods as module-level functions + let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into(); + + for attr in [ + "category", + "lookup", + "name", + "bidirectional", + "combining", + "decimal", + "decomposition", + "digit", + "east_asian_width", + "is_normalized", + "mirrored", + "normalize", + "numeric", + ] { + module.set_attr(attr, ucd.get_attr(attr, vm)?, vm)?; + } + + Ok(()) + } + #[pyattr] #[pyclass(name = "UCD")] #[derive(Debug, PyPayload)] @@ -105,7 +110,7 @@ mod unicodedata { } } - #[pyclass] + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl Ucd { #[pymethod] fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { @@ -126,7 +131,11 @@ mod unicodedata { { return Ok(character.to_string()); } - Err(vm.new_lookup_error(format!("undefined character name '{name}'"))) + Err(vm.new_key_error( + vm.ctx + .new_str(format!("undefined character name '{name}'")) + .into(), + )) } #[pymethod] @@ -190,6 +199,19 @@ mod unicodedata { Ok(normalized_text) } + #[pymethod] + fn is_normalized(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<bool> { + use super::NormalizeForm::*; + let text = unistr.as_wtf8(); + let normalized: Wtf8Buf = match form { + Nfc => text.map_utf8(|s| s.nfc()).collect(), + Nfkc => text.map_utf8(|s| s.nfkc()).collect(), + Nfd => text.map_utf8(|s| s.nfd()).collect(), + Nfkd => text.map_utf8(|s| s.nfkd()).collect(), + }; + Ok(text == &*normalized) + } + #[pymethod] fn mirrored(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { match self.extract_char(character, vm)? { @@ -205,12 +227,120 @@ mod unicodedata { } } + #[pymethod] + fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { + Ok(self + .extract_char(character, vm)? + .and_then(|c| c.to_char()) + .map_or(0, |ch| ch.canonical_combining_class() as i32)) + } + + #[pymethod] + fn decomposition(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + let ch = match self.extract_char(character, vm)?.and_then(|c| c.to_char()) { + Some(ch) => ch, + None => return Ok(String::new()), + }; + let chars: Vec<char> = ch.decomposition_map().collect(); + // If decomposition maps to just the character itself, there's no decomposition + if chars.len() == 1 && chars[0] == ch { + return Ok(String::new()); + } + let hex_parts = chars.iter().map(|c| format!("{:04X}", *c as u32)).join(" "); + let tag = match ch.decomposition_type() { + Some(DecompositionType::Canonical) | None => return Ok(hex_parts), + Some(dt) => decomposition_type_tag(dt), + }; + Ok(format!("<{tag}> {hex_parts}")) + } + + #[pymethod] + fn digit( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch + && matches!( + ch.numeric_type(), + Some(NumericType::Decimal) | Some(NumericType::Digit) + ) + && let Some(Number::Integer(n)) = ch.numeric_value() + { + return Ok(vm.ctx.new_int(n).into()); + } + default.ok_or_else(|| vm.new_value_error("not a digit")) + } + + #[pymethod] + fn decimal( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch + && ch.numeric_type() == Some(NumericType::Decimal) + && let Some(Number::Integer(n)) = ch.numeric_value() + { + return Ok(vm.ctx.new_int(n).into()); + } + default.ok_or_else(|| vm.new_value_error("not a decimal")) + } + + #[pymethod] + fn numeric( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch { + match ch.numeric_value() { + Some(Number::Integer(n)) => { + return Ok(vm.ctx.new_float(n as f64).into()); + } + Some(Number::Rational(num, den)) => { + return Ok(vm.ctx.new_float(num as f64 / den as f64).into()); + } + None => {} + } + } + default.ok_or_else(|| vm.new_value_error("not a numeric character")) + } + #[pygetset] fn unidata_version(&self) -> String { self.unic_version.to_string() } } + fn decomposition_type_tag(dt: DecompositionType) -> &'static str { + match dt { + DecompositionType::Canonical => "canonical", + DecompositionType::Compat => "compat", + DecompositionType::Circle => "circle", + DecompositionType::Final => "final", + DecompositionType::Font => "font", + DecompositionType::Fraction => "fraction", + DecompositionType::Initial => "initial", + DecompositionType::Isolated => "isolated", + DecompositionType::Medial => "medial", + DecompositionType::Narrow => "narrow", + DecompositionType::Nobreak => "noBreak", + DecompositionType::Small => "small", + DecompositionType::Square => "square", + DecompositionType::Sub => "sub", + DecompositionType::Super => "super", + DecompositionType::Vertical => "vertical", + DecompositionType::Wide => "wide", + } + } + trait EastAsianWidthAbbrName { fn abbr_name(&self) -> &'static str; } diff --git a/crates/stdlib/src/uuid.rs b/crates/stdlib/src/uuid.rs index 3f75db402c8..3bc01610d43 100644 --- a/crates/stdlib/src/uuid.rs +++ b/crates/stdlib/src/uuid.rs @@ -1,4 +1,4 @@ -pub(crate) use _uuid::make_module; +pub(crate) use _uuid::module_def; #[pymodule] mod _uuid { diff --git a/crates/stdlib/src/zlib.rs b/crates/stdlib/src/zlib.rs index 328452ae9d5..40269f12bbf 100644 --- a/crates/stdlib/src/zlib.rs +++ b/crates/stdlib/src/zlib.rs @@ -1,6 +1,6 @@ // spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker -pub(crate) use zlib::make_module; +pub(crate) use zlib::module_def; #[pymodule] mod zlib { @@ -39,7 +39,7 @@ mod zlib { #[pyattr(name = "ZLIB_RUNTIME_VERSION")] #[pyattr] const ZLIB_VERSION: &str = unsafe { - match std::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { + match core::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { Ok(s) => s, Err(_) => unreachable!(), } @@ -225,7 +225,7 @@ mod zlib { inner: PyMutex<PyDecompressInner>, } - #[pyclass] + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl PyDecompress { #[pygetset] fn eof(&self) -> bool { @@ -322,7 +322,7 @@ mod zlib { }; let inner = &mut *self.inner.lock(); - let data = std::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); + let data = core::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); let (ret, _) = Self::decompress_inner(inner, &data, length, None, true, vm)?; @@ -383,7 +383,7 @@ mod zlib { inner: PyMutex<CompressState<CompressInner>>, } - #[pyclass] + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl PyCompress { #[pymethod] fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> { diff --git a/crates/venvlauncher/Cargo.toml b/crates/venvlauncher/Cargo.toml new file mode 100644 index 00000000000..ac3ea106b7a --- /dev/null +++ b/crates/venvlauncher/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "rustpython-venvlauncher" +description = "Lightweight venv launcher for RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +# Free-threaded variants (RustPython uses Py_GIL_DISABLED=true) +[[bin]] +name = "venvlaunchert" +path = "src/main.rs" + +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_System_Threading", + "Win32_System_Environment", + "Win32_Storage_FileSystem", + "Win32_System_Console", + "Win32_Security", +] } + +[lints] +workspace = true diff --git a/crates/venvlauncher/build.rs b/crates/venvlauncher/build.rs new file mode 100644 index 00000000000..404f46a484f --- /dev/null +++ b/crates/venvlauncher/build.rs @@ -0,0 +1,21 @@ +//! Build script for venvlauncher +//! +//! Sets the Windows subsystem to GUI for venvwlauncher variants. +//! Only MSVC toolchain is supported on Windows (same as CPython). + +fn main() { + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap(); + let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + + // Only apply on Windows with MSVC toolchain + if target_os == "windows" && target_env == "msvc" { + let exe_name = std::env::var("CARGO_BIN_NAME").unwrap_or_default(); + + // venvwlauncher and venvwlaunchert should be Windows GUI applications + // (no console window) + if exe_name.contains("venvw") { + println!("cargo:rustc-link-arg=/SUBSYSTEM:WINDOWS"); + println!("cargo:rustc-link-arg=/ENTRY:mainCRTStartup"); + } + } +} diff --git a/crates/venvlauncher/src/main.rs b/crates/venvlauncher/src/main.rs new file mode 100644 index 00000000000..7087e791e37 --- /dev/null +++ b/crates/venvlauncher/src/main.rs @@ -0,0 +1,155 @@ +//! RustPython venv launcher +//! +//! A lightweight launcher that reads pyvenv.cfg and delegates execution +//! to the actual Python interpreter. This mimics CPython's venvlauncher.c. +//! Windows only. + +#[cfg(not(windows))] +compile_error!("venvlauncher is only supported on Windows"); + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +fn main() -> ExitCode { + match run() { + Ok(code) => ExitCode::from(code as u8), + Err(e) => { + eprintln!("venvlauncher error: {}", e); + ExitCode::from(1) + } + } +} + +fn run() -> Result<u32, Box<dyn core::error::Error>> { + // 1. Get own executable path + let exe_path = env::current_exe()?; + let exe_name = exe_path + .file_name() + .ok_or("Failed to get executable name")? + .to_string_lossy(); + + // 2. Determine target executable name based on launcher name + // pythonw.exe / venvwlauncher -> pythonw.exe (GUI, no console) + // python.exe / venvlauncher -> python.exe (console) + let exe_name_lower = exe_name.to_lowercase(); + let target_exe = if exe_name_lower.contains("pythonw") || exe_name_lower.contains("venvw") { + "pythonw.exe" + } else { + "python.exe" + }; + + // 3. Find pyvenv.cfg + // The launcher is in Scripts/ directory, pyvenv.cfg is in parent (venv root) + let scripts_dir = exe_path.parent().ok_or("Failed to get Scripts directory")?; + let venv_dir = scripts_dir.parent().ok_or("Failed to get venv directory")?; + let cfg_path = venv_dir.join("pyvenv.cfg"); + + if !cfg_path.exists() { + return Err(format!("pyvenv.cfg not found: {}", cfg_path.display()).into()); + } + + // 4. Parse home= from pyvenv.cfg + let home = read_home(&cfg_path)?; + + // 5. Locate python executable in home directory + let python_path = PathBuf::from(&home).join(target_exe); + if !python_path.exists() { + return Err(format!("Python not found: {}", python_path.display()).into()); + } + + // 6. Set __PYVENV_LAUNCHER__ environment variable + // This tells Python it was launched from a venv + // SAFETY: We are in a single-threaded context (program entry point) + unsafe { + env::set_var("__PYVENV_LAUNCHER__", &exe_path); + } + + // 7. Launch Python with same arguments + let args: Vec<String> = env::args().skip(1).collect(); + launch_process(&python_path, &args) +} + +/// Parse the `home=` value from pyvenv.cfg +fn read_home(cfg_path: &Path) -> Result<String, Box<dyn core::error::Error>> { + let content = fs::read_to_string(cfg_path)?; + + for line in content.lines() { + let line = line.trim(); + // Skip comments and empty lines + if line.is_empty() || line.starts_with('#') { + continue; + } + + // Look for "home = <path>" or "home=<path>" + if let Some(rest) = line.strip_prefix("home") { + let rest = rest.trim_start(); + if let Some(value) = rest.strip_prefix('=') { + return Ok(value.trim().to_string()); + } + } + } + + Err("'home' key not found in pyvenv.cfg".into()) +} + +/// Launch the Python process and wait for it to complete +fn launch_process(exe: &Path, args: &[String]) -> Result<u32, Box<dyn core::error::Error>> { + use std::process::Command; + + let status = Command::new(exe).args(args).status()?; + + Ok(status.code().unwrap_or(1) as u32) +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_read_home() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "home = C:\\Python314").unwrap(); + writeln!(file, "include-system-site-packages = false").unwrap(); + writeln!(file, "version = 3.14.0").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "C:\\Python314"); + + fs::remove_file(&cfg_path).unwrap(); + } + + #[test] + fn test_read_home_no_spaces() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv2.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "home=C:\\Python313").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "C:\\Python313"); + + fs::remove_file(&cfg_path).unwrap(); + } + + #[test] + fn test_read_home_with_comments() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv3.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "# This is a comment").unwrap(); + writeln!(file, "home = D:\\RustPython").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "D:\\RustPython"); + + fs::remove_file(&cfg_path).unwrap(); + } +} diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index b74aba41145..78b1608673e 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -10,7 +10,8 @@ repository.workspace = true license.workspace = true [features] -default = ["compiler", "wasmbind", "stdio"] +default = ["compiler", "wasmbind", "gc", "host_env", "stdio"] +host_env = [] stdio = [] importlib = [] encodings = ["importlib"] @@ -19,6 +20,7 @@ flame-it = ["flame", "flamer"] freeze-stdlib = ["encodings"] jit = ["rustpython-jit"] threading = ["rustpython-common/threading"] +gc = [] compiler = ["parser", "codegen", "rustpython-compiler"] ast = ["ruff_python_ast", "ruff_text_size"] codegen = ["rustpython-codegen", "ast"] @@ -61,7 +63,6 @@ num-complex = { workspace = true } num-integer = { workspace = true } num-traits = { workspace = true } num_enum = { workspace = true } -once_cell = { workspace = true } parking_lot = { workspace = true } paste = { workspace = true } scoped-tls = { workspace = true } @@ -77,6 +78,7 @@ memchr = { workspace = true } caseless = "0.2.2" flamer = { version = "0.5", optional = true } half = "2" +psm = "0.1" optional = { workspace = true } result-like = "0.5.0" timsort = "0.1.2" @@ -117,6 +119,7 @@ workspace = true features = [ "Win32_Foundation", "Win32_Globalization", + "Win32_Media_Audio", "Win32_Networking_WinSock", "Win32_Security", "Win32_Security_Authorization", diff --git a/crates/vm/Lib/core_modules/encodings_ascii.py b/crates/vm/Lib/core_modules/encodings_ascii.py new file mode 120000 index 00000000000..c0507e75b03 --- /dev/null +++ b/crates/vm/Lib/core_modules/encodings_ascii.py @@ -0,0 +1 @@ +../../../../Lib/encodings/ascii.py \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/__reducelib.py b/crates/vm/Lib/python_builtins/__reducelib.py deleted file mode 100644 index 0067cd0a818..00000000000 --- a/crates/vm/Lib/python_builtins/__reducelib.py +++ /dev/null @@ -1,86 +0,0 @@ -# Modified from code from the PyPy project: -# https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objectobject.py - -# The MIT License - -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import copyreg - - -def _abstract_method_error(typ): - methods = ", ".join(sorted(typ.__abstractmethods__)) - err = "Can't instantiate abstract class %s with abstract methods %s" - raise TypeError(err % (typ.__name__, methods)) - - -def reduce_2(obj): - cls = obj.__class__ - - try: - getnewargs = obj.__getnewargs__ - except AttributeError: - args = () - else: - args = getnewargs() - if not isinstance(args, tuple): - raise TypeError("__getnewargs__ should return a tuple") - - try: - getstate = obj.__getstate__ - except AttributeError: - state = getattr(obj, "__dict__", None) - names = slotnames(cls) # not checking for list - if names is not None: - slots = {} - for name in names: - try: - value = getattr(obj, name) - except AttributeError: - pass - else: - slots[name] = value - if slots: - state = state, slots - else: - state = getstate() - - listitems = iter(obj) if isinstance(obj, list) else None - dictitems = iter(obj.items()) if isinstance(obj, dict) else None - - newobj = copyreg.__newobj__ - - args2 = (cls,) + args - return newobj, args2, state, listitems, dictitems - - -def slotnames(cls): - if not isinstance(cls, type): - return None - - try: - return cls.__dict__["__slotnames__"] - except KeyError: - pass - - slotnames = copyreg._slotnames(cls) - if not isinstance(slotnames, list) and slotnames is not None: - raise TypeError("copyreg._slotnames didn't return a list or None") - return slotnames diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index ef6d24c100e..79b62a58abf 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -6,6 +6,8 @@ use crate::{ }; use num_traits::{cast::ToPrimitive, sign::Signed}; +use core::ops::Range; + #[derive(FromArgs)] pub struct SplitArgs<T: TryFromObject> { #[pyarg(any, default)] @@ -43,7 +45,7 @@ pub struct StartsEndsWithArgs { } impl StartsEndsWithArgs { - pub fn get_value(self, len: usize) -> (PyObjectRef, Option<std::ops::Range<usize>>) { + pub fn get_value(self, len: usize) -> (PyObjectRef, Option<Range<usize>>) { let range = if self.start.is_some() || self.end.is_some() { Some(adjust_indices(self.start, self.end, len)) } else { @@ -56,7 +58,7 @@ impl StartsEndsWithArgs { pub fn prepare<S, F>(self, s: &S, len: usize, substr: F) -> Option<(PyObjectRef, &S)> where S: ?Sized + AnyStr, - F: Fn(&S, std::ops::Range<usize>) -> &S, + F: Fn(&S, Range<usize>) -> &S, { let (affix, range) = self.get_value(len); let substr = if let Some(range) = range { @@ -83,11 +85,7 @@ fn saturate_to_isize(py_int: PyIntRef) -> isize { } // help get optional string indices -pub fn adjust_indices( - start: Option<PyIntRef>, - end: Option<PyIntRef>, - len: usize, -) -> std::ops::Range<usize> { +pub fn adjust_indices(start: Option<PyIntRef>, end: Option<PyIntRef>, len: usize) -> Range<usize> { let mut start = start.map_or(0, saturate_to_isize); let mut end = end.map_or(len as isize, saturate_to_isize); if end > len as isize { @@ -111,7 +109,7 @@ pub trait StringRange { fn is_normal(&self) -> bool; } -impl StringRange for std::ops::Range<usize> { +impl StringRange for Range<usize> { fn is_normal(&self) -> bool { self.start <= self.end } @@ -144,9 +142,9 @@ pub trait AnyStr { fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; fn elements(&self) -> impl Iterator<Item = Self::Char>; - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self; + fn get_bytes(&self, range: Range<usize>) -> &Self; // FIXME: get_chars is expensive for str - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self; + fn get_chars(&self, range: Range<usize>) -> &Self; fn bytes_len(&self) -> usize; // NOTE: str::chars().count() consumes the O(n) time. But pystr::char_len does cache. // So using chars_len directly is too expensive and the below method shouldn't be implemented. @@ -254,7 +252,7 @@ pub trait AnyStr { } #[inline] - fn py_find<F>(&self, needle: &Self, range: std::ops::Range<usize>, find: F) -> Option<usize> + fn py_find<F>(&self, needle: &Self, range: Range<usize>, find: F) -> Option<usize> where F: Fn(&Self, &Self) -> Option<usize>, { @@ -268,7 +266,7 @@ pub trait AnyStr { } #[inline] - fn py_count<F>(&self, needle: &Self, range: std::ops::Range<usize>, count: F) -> usize + fn py_count<F>(&self, needle: &Self, range: Range<usize>, count: F) -> usize where F: Fn(&Self, &Self) -> usize, { @@ -283,9 +281,9 @@ pub trait AnyStr { let mut u = Self::Container::with_capacity( (left + right) * fillchar.bytes_len() + self.bytes_len(), ); - u.extend(std::iter::repeat_n(fillchar, left)); + u.extend(core::iter::repeat_n(fillchar, left)); u.push_str(self); - u.extend(std::iter::repeat_n(fillchar, right)); + u.extend(core::iter::repeat_n(fillchar, right)); u } @@ -305,7 +303,7 @@ pub trait AnyStr { fn py_join( &self, - mut iter: impl std::iter::Iterator<Item = PyResult<impl AnyStrWrapper<Self> + TryFromObject>>, + mut iter: impl core::iter::Iterator<Item = PyResult<impl AnyStrWrapper<Self> + TryFromObject>>, ) -> PyResult<Self::Container> { let mut joined = if let Some(elem) = iter.next() { elem?.as_ref().unwrap().to_container() @@ -328,7 +326,7 @@ pub trait AnyStr { ) -> PyResult<(Self::Container, bool, Self::Container)> where F: Fn() -> S, - S: std::iter::Iterator<Item = &'a Self>, + S: core::iter::Iterator<Item = &'a Self>, { if sub.is_empty() { return Err(vm.new_value_error("empty separator")); diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index 13cebfc6a28..33670f1c30a 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -5,11 +5,13 @@ use crate::{ convert::ToPyObject, function::{ArgBytesLike, ArgIntoBool, ArgIntoFloat}, }; +use alloc::fmt; +use core::{iter::Peekable, mem}; use half::f16; use itertools::Itertools; use malachite_bigint::BigInt; use num_traits::{PrimInt, ToPrimitive}; -use std::{fmt, iter::Peekable, mem, os::raw}; +use std::os::raw; type PackFunc = fn(&VirtualMachine, PyObjectRef, &mut [u8]) -> PyResult<()>; type UnpackFunc = fn(&VirtualMachine, &[u8]) -> PyObjectRef; @@ -88,7 +90,9 @@ pub(crate) enum FormatType { Half = b'e', Float = b'f', Double = b'd', + LongDouble = b'g', VoidP = b'P', + PyObject = b'O', } impl fmt::Debug for FormatType { @@ -148,7 +152,9 @@ impl FormatType { Half => nonnative_info!(f16, $end), Float => nonnative_info!(f32, $end), Double => nonnative_info!(f64, $end), - _ => unreachable!(), // size_t or void* + LongDouble => nonnative_info!(f64, $end), // long double same as double + PyObject => nonnative_info!(usize, $end), // pointer size + _ => unreachable!(), // size_t or void* } }}; } @@ -183,7 +189,9 @@ impl FormatType { Half => native_info!(f16), Float => native_info!(raw::c_float), Double => native_info!(raw::c_double), + LongDouble => native_info!(raw::c_double), // long double same as double for now VoidP => native_info!(*mut raw::c_void), + PyObject => native_info!(*mut raw::c_void), // pointer to PyObject }, Endianness::Big => match_nonnative!(self, BigEndian), Endianness::Little => match_nonnative!(self, LittleEndian), @@ -220,6 +228,11 @@ impl FormatCode { let mut arg_count = 0usize; let mut codes = vec![]; while chars.peek().is_some() { + // Skip whitespace before repeat count or format char + while let Some(b' ' | b'\t' | b'\n' | b'\r') = chars.peek() { + chars.next(); + } + // determine repeat operator: let repeat = match chars.peek() { Some(b'0'..=b'9') => { @@ -239,15 +252,81 @@ impl FormatCode { }; // determine format char: - let c = chars - .next() - .ok_or_else(|| "repeat count given without format specifier".to_owned())?; + let c = match chars.next() { + Some(c) => c, + None => { + // If we have a repeat count but only whitespace follows, error + if repeat != 1 { + return Err("repeat count given without format specifier".to_owned()); + } + // Otherwise, we're done parsing + break; + } + }; // Check for embedded null character if c == 0 { return Err("embedded null character".to_owned()); } + // PEP3118: Handle extended format specifiers + // T{...} - struct, X{} - function pointer, (...) - array shape, :name: - field name + if c == b'T' || c == b'X' { + // Skip struct/function pointer: consume until matching '}' + if chars.peek() == Some(&b'{') { + chars.next(); // consume '{' + let mut depth = 1; + while depth > 0 { + match chars.next() { + Some(b'{') => depth += 1, + Some(b'}') => depth -= 1, + None => return Err("unmatched '{' in format".to_owned()), + _ => {} + } + } + continue; + } + } + + if c == b'(' { + // Skip array shape: consume until matching ')' + let mut depth = 1; + while depth > 0 { + match chars.next() { + Some(b'(') => depth += 1, + Some(b')') => depth -= 1, + None => return Err("unmatched '(' in format".to_owned()), + _ => {} + } + } + continue; + } + + if c == b':' { + // Skip field name: consume until next ':' + loop { + match chars.next() { + Some(b':') => break, + None => return Err("unmatched ':' in format".to_owned()), + _ => {} + } + } + continue; + } + + if c == b'{' + || c == b'}' + || c == b'&' + || c == b'<' + || c == b'>' + || c == b'@' + || c == b'=' + || c == b'!' + { + // Skip standalone braces (pointer targets, etc.), pointer prefix, and nested endianness markers + continue; + } + let code = FormatType::try_from(c) .ok() .filter(|c| match c { @@ -468,7 +547,7 @@ macro_rules! make_pack_prim_int { } #[inline] fn unpack_int<E: ByteOrder>(data: &[u8]) -> Self { - let mut x = [0; std::mem::size_of::<$T>()]; + let mut x = [0; core::mem::size_of::<$T>()]; x.copy_from_slice(data); E::convert(<$T>::from_ne_bytes(x)) } @@ -524,7 +603,7 @@ macro_rules! make_pack_float { arg: PyObjectRef, data: &mut [u8], ) -> PyResult<()> { - let f = *ArgIntoFloat::try_from_object(vm, arg)? as $T; + let f = ArgIntoFloat::try_from_object(vm, arg)?.into_float() as $T; f.to_bits().pack_int::<E>(data); Ok(()) } @@ -542,7 +621,7 @@ make_pack_float!(f64); impl Packable for f16 { fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - let f_64 = *ArgIntoFloat::try_from_object(vm, arg)?; + let f_64 = ArgIntoFloat::try_from_object(vm, arg)?.into_float(); // "from_f64 should be preferred in any non-`const` context" except it gives the wrong result :/ let f_16 = Self::from_f64_const(f_64); if f_16.is_infinite() != f_64.is_infinite() { @@ -570,7 +649,7 @@ impl Packable for *mut raw::c_void { impl Packable for bool { fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - let v = *ArgIntoBool::try_from_object(vm, arg)? as u8; + let v = ArgIntoBool::try_from_object(vm, arg)?.into_bool() as u8; v.pack_int::<E>(data); Ok(()) } @@ -604,7 +683,7 @@ fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResul } let b = ArgBytesLike::try_from_object(vm, arg)?; b.with_ref(|data| { - let string_length = std::cmp::min(std::cmp::min(data.len(), 255), buf.len() - 1); + let string_length = core::cmp::min(core::cmp::min(data.len(), 255), buf.len() - 1); buf[0] = string_length as u8; write_string(&mut buf[1..], data); }); @@ -612,7 +691,7 @@ fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResul } fn write_string(buf: &mut [u8], data: &[u8]) { - let len_from_data = std::cmp::min(data.len(), buf.len()); + let len_from_data = core::cmp::min(data.len(), buf.len()); buf[..len_from_data].copy_from_slice(&data[..len_from_data]); for byte in &mut buf[len_from_data..] { *byte = 0 @@ -631,7 +710,7 @@ fn unpack_pascal(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { return vm.ctx.new_bytes(vec![]).into(); } }; - let len = std::cmp::min(len as usize, data.len()); + let len = core::cmp::min(len as usize, data.len()); vm.ctx.new_bytes(data[..len].to_vec()).into() } diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs index 073513184ff..f5b85410eef 100644 --- a/crates/vm/src/builtins/asyncgenerator.rs +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -1,19 +1,20 @@ -use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use super::{PyCode, PyGenerator, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, class::PyClassImpl, common::lock::PyMutex, - coroutine::Coro, + coroutine::{Coro, warn_deprecated_throw_signature}, frame::FrameRef, function::OptionalArg, + object::{Traverse, TraverseFn}, protocol::PyIterReturn, - types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; -#[pyclass(name = "async_generator", module = false)] +#[pyclass(name = "async_generator", module = false, traverse = "manual")] #[derive(Debug)] pub struct PyAsyncGen { inner: Coro, @@ -23,6 +24,13 @@ pub struct PyAsyncGen { // ag_origin_or_finalizer - stores the finalizer callback ag_finalizer: PyMutex<Option<PyObjectRef>>, } + +unsafe impl Traverse for PyAsyncGen { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + self.ag_finalizer.traverse(tracer_fn); + } +} type PyAsyncGenRef = PyRef<PyAsyncGen>; impl PyPayload for PyAsyncGen { @@ -32,7 +40,7 @@ impl PyPayload for PyAsyncGen { } } -#[pyclass(with(PyRef, Unconstructible, Representable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(PyRef, Representable, Destructor))] impl PyAsyncGen { pub const fn as_coro(&self) -> &Coro { &self.inner @@ -57,14 +65,14 @@ impl PyAsyncGen { zelf.ag_hooks_inited.store(true); - // Get and store finalizer from thread-local storage - let finalizer = crate::vm::thread::ASYNC_GEN_FINALIZER.with_borrow(|f| f.as_ref().cloned()); + // Get and store finalizer from VM + let finalizer = vm.async_gen_finalizer.borrow().clone(); if let Some(finalizer) = finalizer { *zelf.ag_finalizer.lock() = Some(finalizer); } // Call firstiter hook - let firstiter = crate::vm::thread::ASYNC_GEN_FIRSTITER.with_borrow(|f| f.as_ref().cloned()); + let firstiter = vm.async_gen_firstiter.borrow().clone(); if let Some(firstiter) = firstiter { let obj: PyObjectRef = zelf.to_owned().into(); firstiter.call((obj,), vm)?; @@ -73,17 +81,20 @@ impl PyAsyncGen { Ok(()) } - /// Call finalizer hook if set - #[allow(dead_code)] + /// Call finalizer hook if set. fn call_finalizer(zelf: &Py<Self>, vm: &VirtualMachine) { - // = gen_dealloc let finalizer = zelf.ag_finalizer.lock().clone(); if let Some(finalizer) = finalizer && !zelf.inner.closed.load() { - // Call finalizer, ignore any errors (PyErr_WriteUnraisable) + // Create a strong reference for the finalizer call. + // This keeps the object alive during the finalizer execution. let obj: PyObjectRef = zelf.to_owned().into(); - let _ = finalizer.call((obj,), vm); + + // Call the finalizer. Any exceptions are handled as unraisable. + if let Err(e) = finalizer.call((obj,), vm) { + vm.run_unraisable(e, Some("async generator finalizer".to_owned()), finalizer); + } } } @@ -112,8 +123,12 @@ impl PyAsyncGen { self.inner.frame().yield_from_target() } #[pygetset] - fn ag_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() + fn ag_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } } #[pygetset] fn ag_running(&self, _vm: &VirtualMachine) -> bool { @@ -165,6 +180,7 @@ impl PyRef<PyAsyncGen> { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult<PyAsyncGenAThrow> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; PyAsyncGen::init_hooks(&self, vm)?; Ok(PyAsyncGenAThrow { ag: self, @@ -201,11 +217,20 @@ impl Representable for PyAsyncGen { } } -impl Unconstructible for PyAsyncGen {} - -#[pyclass(module = false, name = "async_generator_wrapped_value")] +#[pyclass( + module = false, + name = "async_generator_wrapped_value", + traverse = "manual" +)] #[derive(Debug)] pub(crate) struct PyAsyncGenWrappedValue(pub PyObjectRef); + +unsafe impl Traverse for PyAsyncGenWrappedValue { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn); + } +} + impl PyPayload for PyAsyncGenWrappedValue { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -248,7 +273,7 @@ enum AwaitableState { Closed, } -#[pyclass(module = false, name = "async_generator_asend")] +#[pyclass(module = false, name = "async_generator_asend", traverse = "manual")] #[derive(Debug)] pub(crate) struct PyAsyncGenASend { ag: PyAsyncGenRef, @@ -256,6 +281,13 @@ pub(crate) struct PyAsyncGenASend { value: PyObjectRef, } +unsafe impl Traverse for PyAsyncGenASend { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.ag.traverse(tracer_fn); + self.value.traverse(tracer_fn); + } +} + impl PyPayload for PyAsyncGenASend { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -297,7 +329,7 @@ impl PyAsyncGenASend { let res = self.ag.inner.send(self.ag.as_object(), val, vm); let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); if res.is_err() { - self.close(); + self.set_closed(); } res } @@ -310,10 +342,26 @@ impl PyAsyncGenASend { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - if let AwaitableState::Closed = self.state.load() { - return Err(vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()")); + match self.state.load() { + AwaitableState::Closed => { + return Err( + vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()") + ); + } + AwaitableState::Init => { + if self.ag.running_async.load() { + self.state.store(AwaitableState::Closed); + return Err( + vm.new_runtime_error("anext(): asynchronous generator is already running") + ); + } + self.ag.running_async.store(true); + self.state.store(AwaitableState::Iter); + } + AwaitableState::Iter => {} } + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; let res = self.ag.inner.throw( self.ag.as_object(), exc_type, @@ -323,13 +371,36 @@ impl PyAsyncGenASend { ); let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); if res.is_err() { - self.close(); + self.set_closed(); } res } #[pymethod] - fn close(&self) { + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if matches!(self.state.load(), AwaitableState::Closed) { + return Ok(()); + } + let result = self.throw( + vm.ctx.exceptions.generator_exit.to_owned().into(), + OptionalArg::Missing, + OptionalArg::Missing, + vm, + ); + match result { + Ok(_) => Err(vm.new_runtime_error("coroutine ignored GeneratorExit")), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) + || e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + || e.fast_isinstance(vm.ctx.exceptions.generator_exit) => + { + Ok(()) + } + Err(e) => Err(e), + } + } + + fn set_closed(&self) { self.state.store(AwaitableState::Closed); } } @@ -341,7 +412,7 @@ impl IterNext for PyAsyncGenASend { } } -#[pyclass(module = false, name = "async_generator_athrow")] +#[pyclass(module = false, name = "async_generator_athrow", traverse = "manual")] #[derive(Debug)] pub(crate) struct PyAsyncGenAThrow { ag: PyAsyncGenRef, @@ -350,6 +421,13 @@ pub(crate) struct PyAsyncGenAThrow { value: (PyObjectRef, PyObjectRef, PyObjectRef), } +unsafe impl Traverse for PyAsyncGenAThrow { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.ag.traverse(tracer_fn); + self.value.traverse(tracer_fn); + } +} + impl PyPayload for PyAsyncGenAThrow { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -433,6 +511,31 @@ impl PyAsyncGenAThrow { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult { + match self.state.load() { + AwaitableState::Closed => { + return Err(vm.new_runtime_error("cannot reuse already awaited aclose()/athrow()")); + } + AwaitableState::Init => { + if self.ag.running_async.load() { + self.state.store(AwaitableState::Closed); + let msg = if self.aclose { + "aclose(): asynchronous generator is already running" + } else { + "athrow(): asynchronous generator is already running" + }; + return Err(vm.new_runtime_error(msg.to_owned())); + } + if self.ag.inner.closed() { + self.state.store(AwaitableState::Closed); + return Err(vm.new_stop_iteration(None)); + } + self.ag.running_async.store(true); + self.state.store(AwaitableState::Iter); + } + AwaitableState::Iter => {} + } + + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; let ret = self.ag.inner.throw( self.ag.as_object(), exc_type, @@ -453,8 +556,27 @@ impl PyAsyncGenAThrow { } #[pymethod] - fn close(&self) { - self.state.store(AwaitableState::Closed); + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if matches!(self.state.load(), AwaitableState::Closed) { + return Ok(()); + } + let result = self.throw( + vm.ctx.exceptions.generator_exit.to_owned().into(), + OptionalArg::Missing, + OptionalArg::Missing, + vm, + ); + match result { + Ok(_) => Err(vm.new_runtime_error("coroutine ignored GeneratorExit")), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) + || e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + || e.fast_isinstance(vm.ctx.exceptions.generator_exit) => + { + Ok(()) + } + Err(e) => Err(e), + } } fn ignored_close(&self, res: &PyResult<PyIterReturn>) -> bool { @@ -465,11 +587,13 @@ impl PyAsyncGenAThrow { } fn yield_close(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { self.ag.running_async.store(false); + self.ag.inner.closed.store(true); self.state.store(AwaitableState::Closed); vm.new_runtime_error("async generator ignored GeneratorExit") } fn check_error(&self, exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyBaseExceptionRef { self.ag.running_async.store(false); + self.ag.inner.closed.store(true); self.state.store(AwaitableState::Closed); if self.aclose && (exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) @@ -491,7 +615,7 @@ impl IterNext for PyAsyncGenAThrow { /// Awaitable wrapper for anext() builtin with default value. /// When StopAsyncIteration is raised, it converts it to StopIteration(default). -#[pyclass(module = false, name = "anext_awaitable")] +#[pyclass(module = false, name = "anext_awaitable", traverse = "manual")] #[derive(Debug)] pub struct PyAnextAwaitable { wrapped: PyObjectRef, @@ -499,6 +623,13 @@ pub struct PyAnextAwaitable { state: AtomicCell<AwaitableState>, } +unsafe impl Traverse for PyAnextAwaitable { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.wrapped.traverse(tracer_fn); + self.default_value.traverse(tracer_fn); + } +} + impl PyPayload for PyAnextAwaitable { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -548,12 +679,24 @@ impl PyAnextAwaitable { // Coroutine - get __await__ later wrapped.clone() } else { + // Check for generator with CO_ITERABLE_COROUTINE flag + if let Some(generator) = wrapped.downcast_ref::<PyGenerator>() + && generator + .as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + { + // Return the generator itself as the iterator + return Ok(wrapped.clone()); + } // Try to get __await__ method if let Some(await_method) = vm.get_method(wrapped.clone(), identifier!(vm, __await__)) { await_method?.call((), vm)? } else { return Err(vm.new_type_error(format!( - "object {} can't be used in 'await' expression", + "'{}' object can't be awaited", wrapped.class().name() ))); } @@ -603,6 +746,7 @@ impl PyAnextAwaitable { vm: &VirtualMachine, ) -> PyResult { self.check_closed(vm)?; + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; self.state.store(AwaitableState::Iter); let awaitable = self.get_awaitable_iter(vm)?; let result = vm.call_method( @@ -645,6 +789,27 @@ impl IterNext for PyAnextAwaitable { } } +/// _PyGen_Finalize for async generators +impl Destructor for PyAsyncGen { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Generator is already closed, nothing to do + if zelf.inner.closed.load() { + return Ok(()); + } + + // Call the async generator finalizer hook if set. + Self::call_finalizer(zelf, vm); + + Ok(()) + } +} + +impl Drop for PyAsyncGen { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + pub fn init(ctx: &Context) { PyAsyncGen::extend_class(ctx, ctx.types.async_generator); PyAsyncGenASend::extend_class(ctx, ctx.types.async_generator_asend); diff --git a/crates/vm/src/builtins/bool.rs b/crates/vm/src/builtins/bool.rs index 6b3ddd8241a..24ded08ab10 100644 --- a/crates/vm/src/builtins/bool.rs +++ b/crates/vm/src/builtins/bool.rs @@ -8,9 +8,8 @@ use crate::{ protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, }; -use malachite_bigint::Sign; +use core::fmt::{Debug, Formatter}; use num_traits::Zero; -use std::fmt::{Debug, Formatter}; impl ToPyObject for bool { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { @@ -42,46 +41,28 @@ impl PyObjectRef { if self.is(&vm.ctx.false_value) { return Ok(false); } - let rs_bool = if let Some(nb_bool) = self.class().slots.as_number.boolean.load() { - nb_bool(self.as_object().to_number(), vm)? - } else { - // TODO: Fully implement AsNumber and remove this block - match vm.get_method(self.clone(), identifier!(vm, __bool__)) { - Some(method_or_err) => { - // If descriptor returns Error, propagate it further - let method = method_or_err?; - let bool_obj = method.call((), vm)?; - if !bool_obj.fast_isinstance(vm.ctx.types.bool_type) { - return Err(vm.new_type_error(format!( - "__bool__ should return bool, returned type {}", - bool_obj.class().name() - ))); - } - - get_value(&bool_obj) - } - None => match vm.get_method(self, identifier!(vm, __len__)) { - Some(method_or_err) => { - let method = method_or_err?; - let bool_obj = method.call((), vm)?; - let int_obj = bool_obj.downcast_ref::<PyInt>().ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - bool_obj.class().name() - )) - })?; - - let len_val = int_obj.as_bigint(); - if len_val.sign() == Sign::Minus { - return Err(vm.new_value_error("__len__() should return >= 0")); - } - !len_val.is_zero() - } - None => true, - }, - } - }; - Ok(rs_bool) + + let slots = &self.class().slots; + + // 1. Try nb_bool slot first + if let Some(nb_bool) = slots.as_number.boolean.load() { + return nb_bool(self.as_object().number(), vm); + } + + // 2. Try mp_length slot (mapping protocol) + if let Some(mp_length) = slots.as_mapping.length.load() { + let len = mp_length(self.as_object().mapping_unchecked(), vm)?; + return Ok(len != 0); + } + + // 3. Try sq_length slot (sequence protocol) + if let Some(sq_length) = slots.as_sequence.length.load() { + let len = sq_length(self.as_object().sequence_unchecked(), vm)?; + return Ok(len != 0); + } + + // 4. Default: objects without __bool__ or __len__ are truthy + Ok(true) } } @@ -90,7 +71,7 @@ impl PyObjectRef { pub struct PyBool(pub PyInt); impl Debug for PyBool { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { let value = !self.0.as_bigint().is_zero(); write!(f, "PyBool({})", value) } @@ -126,10 +107,10 @@ impl PyBool { .and_then(|format_spec| format_spec.format_bool(new_bool)) .map_err(|err| err.into_pyexception(vm)) } +} - #[pymethod(name = "__ror__")] - #[pymethod] - fn __or__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +impl PyBool { + pub(crate) fn __or__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if lhs.fast_isinstance(vm.ctx.types.bool_type) && rhs.fast_isinstance(vm.ctx.types.bool_type) { @@ -143,9 +124,7 @@ impl PyBool { } } - #[pymethod(name = "__rand__")] - #[pymethod] - fn __and__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub(crate) fn __and__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if lhs.fast_isinstance(vm.ctx.types.bool_type) && rhs.fast_isinstance(vm.ctx.types.bool_type) { @@ -159,9 +138,7 @@ impl PyBool { } } - #[pymethod(name = "__rxor__")] - #[pymethod] - fn __xor__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub(crate) fn __xor__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if lhs.fast_isinstance(vm.ctx.types.bool_type) && rhs.fast_isinstance(vm.ctx.types.bool_type) { diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index d1ce107e374..0dff221f166 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -5,11 +5,12 @@ use crate::{ common::wtf8::Wtf8, convert::TryFromObject, function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, - types::{Callable, Comparable, PyComparisonOp, Representable, Unconstructible}, + types::{Callable, Comparable, PyComparisonOp, Representable}, }; -use std::fmt; +use alloc::fmt; // PyCFunctionObject in CPython +#[repr(C)] #[pyclass(name = "builtin_function_or_method", module = false)] pub struct PyNativeFunction { pub(crate) value: &'static PyMethodDef, @@ -51,11 +52,11 @@ impl PyNativeFunction { } // PyCFunction_GET_SELF - pub const fn get_self(&self) -> Option<&PyObjectRef> { + pub fn get_self(&self) -> Option<&PyObject> { if self.value.flags.contains(PyMethodFlags::STATIC) { return None; } - self.zelf.as_ref() + self.zelf.as_deref() } pub const fn as_func(&self) -> &'static dyn PyNativeFn { @@ -68,13 +69,65 @@ impl Callable for PyNativeFunction { #[inline] fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { if let Some(z) = &zelf.zelf { - args.prepend_arg(z.clone()); + // STATIC methods store the class in zelf for qualname/repr purposes, + // but should not prepend it to args (the Rust function doesn't expect it). + if !zelf.value.flags.contains(PyMethodFlags::STATIC) { + args.prepend_arg(z.clone()); + } } (zelf.value.func)(vm, args) } } -#[pyclass(with(Callable, Unconstructible), flags(HAS_DICT))] +// meth_richcompare in CPython +impl Comparable for PyNativeFunction { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if let Some(other) = other.downcast_ref::<Self>() { + let eq = match (zelf.zelf.as_ref(), other.zelf.as_ref()) { + (Some(z), Some(o)) => z.is(o), + (None, None) => true, + _ => false, + }; + let eq = eq && core::ptr::eq(zelf.value, other.value); + Ok(eq.into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } +} + +// meth_repr in CPython +impl Representable for PyNativeFunction { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + if let Some(bound) = zelf + .zelf + .as_ref() + .filter(|b| !b.class().is(vm.ctx.types.module_type)) + { + Ok(format!( + "<built-in method {} of {} object at {:#x}>", + zelf.value.name, + bound.class().name(), + bound.get_id() + )) + } else { + Ok(format!("<built-in function {}>", zelf.value.name)) + } + } +} + +#[pyclass( + with(Callable, Comparable, Representable), + flags(HAS_DICT, DISALLOW_INSTANTIATION) +)] impl PyNativeFunction { #[pygetset] fn __module__(zelf: NativeFunctionOrMethod) -> Option<&'static PyStrInterned> { @@ -86,20 +139,19 @@ impl PyNativeFunction { zelf.0.value.name } + // meth_get__qualname__ in CPython #[pygetset] fn __qualname__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult<PyStrRef> { let zelf = zelf.0; - let flags = zelf.value.flags; - // if flags.contains(PyMethodFlags::CLASS) || flags.contains(PyMethodFlags::STATIC) { let qualname = if let Some(bound) = &zelf.zelf { - let prefix = if flags.contains(PyMethodFlags::CLASS) { - bound - .get_attr("__qualname__", vm) - .unwrap() - .str(vm) - .unwrap() - .to_string() + if bound.class().is(vm.ctx.types.module_type) { + return Ok(vm.ctx.intern_str(zelf.value.name).to_owned()); + } + let prefix = if bound.class().is(vm.ctx.types.type_type) { + // m_self is a type: use PyType_GetQualName(m_self) + bound.get_attr("__qualname__", vm)?.str(vm)?.to_string() } else { + // m_self is an instance: use Py_TYPE(m_self).__qualname__ bound.class().name().to_string() }; vm.ctx.new_str(format!("{}.{}", prefix, &zelf.value.name)) @@ -114,15 +166,23 @@ impl PyNativeFunction { zelf.0.value.doc } - #[pygetset(name = "__self__")] - fn __self__(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none() + // meth_get__self__ in CPython + #[pygetset] + fn __self__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyObjectRef { + zelf.0.zelf.clone().unwrap_or_else(|| vm.ctx.none()) } + // meth_reduce in CPython #[pymethod] - const fn __reduce__(&self) -> &'static str { - // TODO: return (getattr, (self.object, self.name)) if this is a method - self.value.name + fn __reduce__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult { + let zelf = zelf.0; + if zelf.zelf.is_none() || zelf.module.is_some() { + Ok(vm.ctx.new_str(zelf.value.name).into()) + } else { + let getattr = vm.builtins.get_attr("getattr", vm)?; + let target = zelf.zelf.clone().unwrap(); + Ok(vm.new_tuple((getattr, (target, zelf.value.name))).into()) + } } #[pymethod] @@ -138,56 +198,20 @@ impl PyNativeFunction { } } -impl Representable for PyNativeFunction { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!("<built-in function {}>", zelf.value.name)) - } -} - -impl Unconstructible for PyNativeFunction {} - -// `PyCMethodObject` in CPython -#[pyclass(name = "builtin_method", module = false, base = PyNativeFunction, ctx = "builtin_method_type")] +// PyCMethodObject in CPython +// repr(C) ensures `func` is at offset 0, allowing safe cast from PyNativeMethod to PyNativeFunction +#[repr(C)] +#[pyclass(name = "builtin_function_or_method", module = false, base = PyNativeFunction, ctx = "builtin_function_or_method_type")] pub struct PyNativeMethod { pub(crate) func: PyNativeFunction, pub(crate) class: &'static Py<PyType>, // TODO: the actual life is &'self } -#[pyclass( - with(Unconstructible, Callable, Comparable, Representable), - flags(HAS_DICT) -)] -impl PyNativeMethod { - #[pygetset] - fn __qualname__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let prefix = zelf.class.name().to_string(); - Ok(vm - .ctx - .new_str(format!("{}.{}", prefix, &zelf.func.value.name))) - } - - #[pymethod] - fn __reduce__( - &self, - vm: &VirtualMachine, - ) -> PyResult<(PyObjectRef, (PyObjectRef, &'static str))> { - // TODO: return (getattr, (self.object, self.name)) if this is a method - let getattr = vm.builtins.get_attr("getattr", vm)?; - let target = self - .func - .zelf - .clone() - .unwrap_or_else(|| self.class.to_owned().into()); - let name = self.func.value.name; - Ok((getattr, (target, name))) - } - - #[pygetset(name = "__self__")] - fn __self__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> Option<PyObjectRef> { - zelf.func.zelf.clone() - } -} +// All Python-visible behavior (getters, slots) is registered by PyNativeFunction::extend_class. +// PyNativeMethod only extends the Rust-side struct with the defining class reference. +// The func field at offset 0 (#[repr(C)]) allows NativeFunctionOrMethod to read it safely. +#[pyclass(flags(HAS_DICT, DISALLOW_INSTANTIATION))] +impl PyNativeMethod {} impl fmt::Debug for PyNativeMethod { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -200,65 +224,21 @@ impl fmt::Debug for PyNativeMethod { } } -impl Comparable for PyNativeMethod { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - _vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - if let Some(other) = other.downcast_ref::<Self>() { - let eq = match (zelf.func.zelf.as_ref(), other.func.zelf.as_ref()) { - (Some(z), Some(o)) => z.is(o), - (None, None) => true, - _ => false, - }; - let eq = eq && std::ptr::eq(zelf.func.value, other.func.value); - Ok(eq.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - }) - } -} - -impl Callable for PyNativeMethod { - type Args = FuncArgs; - - #[inline] - fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { - if let Some(zelf) = &zelf.func.zelf { - args.prepend_arg(zelf.clone()); - } - (zelf.func.value.func)(vm, args) - } -} - -impl Representable for PyNativeMethod { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<built-in method {} of {} object at ...>", - &zelf.func.value.name, - zelf.class.name() - )) - } -} - -impl Unconstructible for PyNativeMethod {} - pub fn init(context: &Context) { PyNativeFunction::extend_class(context, context.types.builtin_function_or_method_type); - PyNativeMethod::extend_class(context, context.types.builtin_method_type); } +/// Wrapper that provides access to the common PyNativeFunction data +/// for both PyNativeFunction and PyNativeMethod (which has func as its first field). struct NativeFunctionOrMethod(PyRef<PyNativeFunction>); impl TryFromObject for NativeFunctionOrMethod { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { let class = vm.ctx.types.builtin_function_or_method_type; if obj.fast_isinstance(class) { + // Both PyNativeFunction and PyNativeMethod share the same type now. + // PyNativeMethod has `func: PyNativeFunction` as its first field, + // so we can safely treat the data pointer as PyNativeFunction for reading. Ok(Self(unsafe { obj.downcast_unchecked() })) } else { Err(vm.new_downcast_type_error(class, &obj)) diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 32eaa2b3e27..83143070e07 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -1,7 +1,7 @@ //! Implementation of the python bytearray object. use super::{ PositionIterInternal, PyBytes, PyBytesRef, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, - PyTuple, PyTupleRef, PyType, PyTypeRef, + PyTuple, PyTupleRef, PyType, PyTypeRef, iter::builtins_iter, }; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, @@ -33,11 +33,11 @@ use crate::{ types::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, DefaultConstructor, Initializer, IterNext, Iterable, PyComparisonOp, Representable, - SelfIter, Unconstructible, + SelfIter, }, }; use bstr::ByteSlice; -use std::mem::size_of; +use core::mem::size_of; #[pyclass(module = false, name = "bytearray", unhashable = true)] #[derive(Debug, Default)] @@ -206,7 +206,6 @@ impl PyByteArray { self.inner().capacity() } - #[pymethod] fn __len__(&self) -> usize { self.borrow_buf().len() } @@ -216,12 +215,10 @@ impl PyByteArray { size_of::<Self>() + self.borrow_buf().len() * size_of::<u8>() } - #[pymethod] fn __add__(&self, other: ArgBytesLike) -> Self { self.inner().add(&other.borrow_buf()).into() } - #[pymethod] fn __contains__( &self, needle: Either<PyBytesInner, PyIntRef>, @@ -230,7 +227,6 @@ impl PyByteArray { self.inner().contains(needle, vm) } - #[pymethod] fn __iadd__( zelf: PyRef<Self>, other: ArgBytesLike, @@ -242,12 +238,10 @@ impl PyByteArray { Ok(zelf) } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } - #[pymethod] pub fn __delitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self._delitem(&needle, vm) } @@ -328,8 +322,8 @@ impl PyByteArray { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_bytes(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes); let args = vec![bytes.into()].into(); PyType::call(&cls, args, vm) @@ -525,32 +519,32 @@ impl PyByteArray { self.inner().title().into() } - #[pymethod(name = "__rmul__")] - #[pymethod] fn __mul__(&self, value: ArgSize, vm: &VirtualMachine) -> PyResult<Self> { self.repeat(value.into(), vm) } - #[pymethod] fn __imul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Self::irepeat(&zelf, value.into(), vm)?; Ok(zelf) } - #[pymethod(name = "__mod__")] - fn mod_(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { let formatted = self.inner().cformat(values, vm)?; Ok(formatted.into()) } #[pymethod] - fn __rmod__(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() + fn reverse(&self) { + self.borrow_buf_mut().reverse(); } #[pymethod] - fn reverse(&self) { - self.borrow_buf_mut().reverse(); + fn resize(&self, size: isize, vm: &VirtualMachine) -> PyResult<()> { + if size < 0 { + return Err(vm.new_value_error("bytearray.resize(): new size must be >= 0".to_owned())); + } + self.try_resizable(vm)?.elements.resize(size as usize, 0); + Ok(()) } // TODO: Uncomment when Python adds __class_getitem__ to bytearray @@ -562,7 +556,6 @@ impl PyByteArray { #[pyclass] impl Py<PyByteArray> { - #[pymethod] fn __setitem__( &self, needle: PyObjectRef, @@ -687,7 +680,7 @@ impl Initializer for PyByteArray { fn init(zelf: PyRef<Self>, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { // First unpack bytearray and *then* get a lock to set it. let mut inner = options.get_bytearray_inner(vm)?; - std::mem::swap(&mut *zelf.inner_mut(), &mut inner); + core::mem::swap(&mut *zelf.inner_mut(), &mut inner); Ok(()) } } @@ -823,7 +816,7 @@ impl AsNumber for PyByteArray { static AS_NUMBER: PyNumberMethods = PyNumberMethods { remainder: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::<PyByteArray>() { - a.mod_(b.to_owned(), vm).to_pyresult(vm) + a.__mod__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -865,7 +858,7 @@ impl PyPayload for PyByteArrayIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyByteArrayIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -873,9 +866,13 @@ impl PyByteArrayIterator { } #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } #[pymethod] @@ -886,8 +883,6 @@ impl PyByteArrayIterator { } } -impl Unconstructible for PyByteArrayIterator {} - impl SelfIter for PyByteArrayIterator {} impl IterNext for PyByteArrayIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 70a33401271..623885ec0fe 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -1,7 +1,8 @@ use super::{ PositionIterInternal, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, - PyType, PyTypeRef, + PyType, PyTypeRef, iter::builtins_iter, }; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, @@ -25,12 +26,11 @@ use crate::{ sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, - IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, + IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, }; use bstr::ByteSlice; -use std::sync::LazyLock; -use std::{mem::size_of, ops::Deref}; +use core::{mem::size_of, ops::Deref}; #[pyclass(module = false, name = "bytes")] #[derive(Clone, Debug)] @@ -187,6 +187,7 @@ impl PyRef<PyBytes> { } #[pyclass( + itemsize = 1, flags(BASETYPE, _MATCH_SELF), with( Py, @@ -204,7 +205,6 @@ impl PyRef<PyBytes> { )] impl PyBytes { #[inline] - #[pymethod] pub const fn __len__(&self) -> usize { self.inner.len() } @@ -224,12 +224,10 @@ impl PyBytes { size_of::<Self>() + self.len() * size_of::<u8>() } - #[pymethod] fn __add__(&self, other: ArgBytesLike) -> Vec<u8> { self.inner.add(&other.borrow_buf()) } - #[pymethod] fn __contains__( &self, needle: Either<PyBytesInner, PyIntRef>, @@ -243,7 +241,6 @@ impl PyBytes { PyBytesInner::maketrans(from, to, vm) } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } @@ -319,8 +316,8 @@ impl PyBytes { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_bytes(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes).into(); PyType::call(&cls, vec![bytes].into(), vm) } @@ -512,23 +509,15 @@ impl PyBytes { self.inner.title().into() } - #[pymethod(name = "__rmul__")] - #[pymethod] fn __mul__(zelf: PyRef<Self>, value: ArgIndex, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.repeat(value.try_to_primitive(vm)?, vm) + zelf.repeat(value.into_int_ref().try_to_primitive(vm)?, vm) } - #[pymethod(name = "__mod__")] - fn mod_(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { let formatted = self.inner.cformat(values, vm)?; Ok(formatted.into()) } - #[pymethod] - fn __rmod__(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - #[pymethod] fn __getnewargs__(&self, vm: &VirtualMachine) -> PyTupleRef { let param: Vec<PyObjectRef> = self.elements().map(|x| x.to_pyobject(vm)).collect(); @@ -677,7 +666,7 @@ impl AsNumber for PyBytes { static AS_NUMBER: PyNumberMethods = PyNumberMethods { remainder: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::<PyBytes>() { - a.mod_(b.to_owned(), vm).to_pyresult(vm) + a.__mod__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -749,7 +738,7 @@ impl PyPayload for PyBytesIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyBytesIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -758,9 +747,13 @@ impl PyBytesIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } #[pymethod] @@ -770,7 +763,6 @@ impl PyBytesIterator { .set_state(state, |obj, pos| pos.min(obj.len()), vm) } } -impl Unconstructible for PyBytesIterator {} impl SelfIter for PyBytesIterator {} impl IterNext for PyBytesIterator { diff --git a/crates/vm/src/builtins/capsule.rs b/crates/vm/src/builtins/capsule.rs new file mode 100644 index 00000000000..7e263c4cde2 --- /dev/null +++ b/crates/vm/src/builtins/capsule.rs @@ -0,0 +1,33 @@ +use super::PyType; +use crate::{Context, Py, PyPayload, PyResult, class::PyClassImpl, types::Representable}; + +/// PyCapsule - a container for C pointers. +/// In RustPython, this is a minimal implementation for compatibility. +#[pyclass(module = false, name = "PyCapsule")] +#[derive(Debug, Clone, Copy)] +pub struct PyCapsule { + // Capsules store opaque pointers; we don't expose the actual pointer functionality + // since RustPython doesn't have the same C extension model as CPython. + _private: (), +} + +impl PyPayload for PyCapsule { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.capsule_type + } +} + +#[pyclass(with(Representable), flags(DISALLOW_INSTANTIATION))] +impl PyCapsule {} + +impl Representable for PyCapsule { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &crate::VirtualMachine) -> PyResult<String> { + Ok("<capsule object>".to_string()) + } +} + +pub fn init(context: &Context) { + PyCapsule::extend_class(context, context.types.capsule_type); +} diff --git a/crates/vm/src/builtins/classmethod.rs b/crates/vm/src/builtins/classmethod.rs index 911960bf691..d2f1377be04 100644 --- a/crates/vm/src/builtins/classmethod.rs +++ b/crates/vm/src/builtins/classmethod.rs @@ -3,7 +3,7 @@ use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyMutex, - function::FuncArgs, + function::{FuncArgs, PySetterValue}, types::{Constructor, GetDescriptor, Initializer, Representable}, }; @@ -57,11 +57,11 @@ impl GetDescriptor for PyClassMethod { ) -> PyResult { let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; let cls = cls.unwrap_or_else(|| _obj.class().to_owned().into()); - let call_descr_get: PyResult<PyObjectRef> = zelf.callable.lock().get_attr("__get__", vm); + // Clone and release lock before calling Python code to prevent deadlock + let callable = zelf.callable.lock().clone(); + let call_descr_get: PyResult<PyObjectRef> = callable.get_attr("__get__", vm); match call_descr_get { - Err(_) => Ok(PyBoundMethod::new(cls, zelf.callable.lock().clone()) - .into_ref(&vm.ctx) - .into()), + Err(_) => Ok(PyBoundMethod::new(cls, callable).into_ref(&vm.ctx).into()), Ok(call_descr_get) => call_descr_get.call((cls.clone(), cls), vm), } } @@ -124,7 +124,7 @@ impl PyClassMethod { } #[pyclass( - with(GetDescriptor, Constructor, Representable), + with(GetDescriptor, Constructor, Initializer, Representable), flags(BASETYPE, HAS_DICT) )] impl PyClassMethod { @@ -158,6 +158,27 @@ impl PyClassMethod { self.callable.lock().get_attr("__annotations__", vm) } + #[pygetset(setter)] + fn set___annotations__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotations__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotate__", vm) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotate__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + #[pygetset] fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyObjectRef { match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index e46cc711bb3..790f3fd8695 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -2,19 +2,24 @@ use super::{PyBytesRef, PyStrRef, PyTupleRef, PyType}; use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrInterned, bytecode::{self, AsBag, BorrowedConstant, CodeFlags, Constant, ConstantBag}, class::{PyClassImpl, StaticType}, - convert::ToPyObject, + convert::{ToPyException, ToPyObject}, frozen, function::OptionalArg, types::{Constructor, Representable}, }; +use alloc::fmt; +use core::{ + borrow::Borrow, + ops::Deref, + sync::atomic::{AtomicPtr, Ordering}, +}; use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::{OneIndexed, bytecode::CodeUnits, bytecode::PyCodeLocationInfoKind}; -use std::{borrow::Borrow, fmt, ops::Deref}; /// State for iterating through code address ranges struct PyCodeAddressRange<'a> { @@ -151,7 +156,7 @@ pub struct ReplaceArgs { #[pyarg(named, optional)] co_names: OptionalArg<Vec<PyObjectRef>>, #[pyarg(named, optional)] - co_flags: OptionalArg<u16>, + co_flags: OptionalArg<u32>, #[pyarg(named, optional)] co_varnames: OptionalArg<Vec<PyObjectRef>>, #[pyarg(named, optional)] @@ -322,6 +327,7 @@ impl<B: AsRef<[u8]>> IntoCodeObject for frozen::FrozenCodeObject<B> { #[pyclass(module = false, name = "code")] pub struct PyCode { pub code: CodeObject, + source_path: AtomicPtr<PyStrInterned>, } impl Deref for PyCode { @@ -332,8 +338,62 @@ impl Deref for PyCode { } impl PyCode { - pub const fn new(code: CodeObject) -> Self { - Self { code } + pub fn new(code: CodeObject) -> Self { + let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned; + Self { + code, + source_path: AtomicPtr::new(sp), + } + } + + pub fn source_path(&self) -> &'static PyStrInterned { + // SAFETY: always points to a valid &'static PyStrInterned (interned strings are never deallocated) + unsafe { &*self.source_path.load(Ordering::Relaxed) } + } + + pub fn set_source_path(&self, new: &'static PyStrInterned) { + self.source_path.store( + new as *const PyStrInterned as *mut PyStrInterned, + Ordering::Relaxed, + ); + } + pub fn from_pyc_path(path: &std::path::Path, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let name = match path.file_stem() { + Some(stem) => stem.display().to_string(), + None => "".to_owned(), + }; + let content = std::fs::read(path).map_err(|e| e.to_pyexception(vm))?; + Self::from_pyc( + &content, + Some(&name), + Some(&path.display().to_string()), + Some("<source>"), + vm, + ) + } + pub fn from_pyc( + pyc_bytes: &[u8], + name: Option<&str>, + bytecode_path: Option<&str>, + source_path: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + if !crate::import::check_pyc_magic_number_bytes(pyc_bytes) { + return Err(vm.new_value_error("pyc bytes has wrong MAGIC")); + } + let bootstrap_external = vm.import("_frozen_importlib_external", 0)?; + let compile_bytecode = bootstrap_external.get_attr("_compile_bytecode", vm)?; + // 16 is the pyc header length + let Some((_, code_bytes)) = pyc_bytes.split_at_checked(16) else { + return Err(vm.new_value_error(format!( + "pyc_bytes header is broken. 16 bytes expected but {} bytes given.", + pyc_bytes.len() + ))); + }; + let code_bytes_obj = vm.ctx.new_bytes(code_bytes.to_vec()); + let compiled = + compile_bytecode.call((code_bytes_obj, name, bytecode_path, source_path), vm)?; + compiled.try_downcast(vm) } } @@ -358,7 +418,7 @@ impl Representable for PyCode { "<code object {} at {:#x} file {:?}, line {}>", code.obj_name, zelf.get_id(), - code.source_path.as_str(), + zelf.source_path().as_str(), code.first_line_number.map_or(-1, |n| n.get() as i32) )) } @@ -372,7 +432,7 @@ pub struct PyCodeNewArgs { kwonlyargcount: u32, nlocals: u32, stacksize: u32, - flags: u16, + flags: u32, co_code: PyBytesRef, consts: PyTupleRef, names: PyTupleRef, @@ -466,20 +526,22 @@ impl Constructor for PyCode { .collect::<Vec<_>>() .into_boxed_slice(); - // Create locations + // Create locations (start and end pairs) let row = if args.firstlineno > 0 { OneIndexed::new(args.firstlineno as usize).unwrap_or(OneIndexed::MIN) } else { OneIndexed::MIN }; - let locations: Box<[rustpython_compiler_core::SourceLocation]> = vec![ - rustpython_compiler_core::SourceLocation { - line: row, - character_offset: OneIndexed::from_zero_indexed(0), - }; - instructions.len() - ] - .into_boxed_slice(); + let loc = rustpython_compiler_core::SourceLocation { + line: row, + character_offset: OneIndexed::from_zero_indexed(0), + }; + let locations: Box< + [( + rustpython_compiler_core::SourceLocation, + rustpython_compiler_core::SourceLocation, + )], + > = vec![(loc, loc); instructions.len()].into_boxed_slice(); // Build the CodeObject let code = CodeObject { @@ -531,7 +593,7 @@ impl PyCode { #[pygetset] pub fn co_filename(&self) -> PyStrRef { - self.code.source_path.to_owned() + self.source_path().to_owned() } #[pygetset] @@ -587,7 +649,7 @@ impl PyCode { } #[pygetset] - const fn co_flags(&self) -> u16 { + const fn co_flags(&self) -> u32 { self.code.flags.bits() } @@ -601,7 +663,7 @@ impl PyCode { pub fn co_code(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { // SAFETY: CodeUnit is #[repr(C)] with size 2, so we can safely transmute to bytes let bytes = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( self.code.instructions.as_ptr() as *const u8, self.code.instructions.len() * 2, ) @@ -609,6 +671,12 @@ impl PyCode { vm.ctx.new_bytes(bytes.to_vec()) } + #[pygetset] + pub fn _co_code_adaptive(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // RustPython doesn't have adaptive/specialized bytecode, so return regular co_code + self.co_code(vm) + } + #[pygetset] pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef { let names = self @@ -808,7 +876,6 @@ impl PyCode { Some(line + end_line_delta) }; - // Convert Option to PyObject (None or int) let line_obj = final_line.to_pyobject(vm); let end_line_obj = final_endline.to_pyobject(vm); let column_obj = column.to_pyobject(vm); @@ -860,7 +927,7 @@ impl PyCode { let source_path = match co_filename { OptionalArg::Present(source_path) => source_path, - OptionalArg::Missing => self.code.source_path.to_owned(), + OptionalArg::Missing => self.source_path().to_owned(), }; let first_line_number = match co_firstlineno { @@ -1002,7 +1069,26 @@ impl PyCode { let idx_err = |vm: &VirtualMachine| vm.new_index_error("tuple index out of range"); let idx = usize::try_from(opcode).map_err(|_| idx_err(vm))?; - let name = self.code.varnames.get(idx).ok_or_else(|| idx_err(vm))?; + + let varnames_len = self.code.varnames.len(); + let cellvars_len = self.code.cellvars.len(); + + let name = if idx < varnames_len { + // Index in varnames + self.code.varnames.get(idx).ok_or_else(|| idx_err(vm))? + } else if idx < varnames_len + cellvars_len { + // Index in cellvars + self.code + .cellvars + .get(idx - varnames_len) + .ok_or_else(|| idx_err(vm))? + } else { + // Index in freevars + self.code + .freevars + .get(idx - varnames_len - cellvars_len) + .ok_or_else(|| idx_err(vm))? + }; Ok(name.to_object()) } } diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index 8bf46cfdd5c..dd68061557d 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -5,19 +5,15 @@ use crate::{ class::PyClassImpl, common::format::FormatSpec, convert::{IntoPyException, ToPyObject, ToPyResult}, - function::{ - FuncArgs, OptionalArg, OptionalOption, - PyArithmeticValue::{self, *}, - PyComparisonValue, - }, + function::{FuncArgs, OptionalArg, PyComparisonValue}, protocol::PyNumberMethods, stdlib::warnings, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use core::num::Wrapping; use num_complex::Complex64; use num_traits::Zero; use rustpython_common::hash; -use std::num::Wrapping; /// Create a complex number from a real part and an optional imaginary part. /// @@ -268,133 +264,11 @@ impl PyComplex { self.value.im } - #[pymethod] - fn __abs__(&self, vm: &VirtualMachine) -> PyResult<f64> { - let Complex64 { im, re } = self.value; - let is_finite = im.is_finite() && re.is_finite(); - let abs_result = re.hypot(im); - if is_finite && abs_result.is_infinite() { - Err(vm.new_overflow_error("absolute value too large")) - } else { - Ok(abs_result) - } - } - - #[inline] - fn op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> - where - F: Fn(Complex64, Complex64) -> PyResult<Complex64>, - { - to_op_complex(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[pymethod(name = "__radd__")] - #[pymethod] - fn __add__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a + b), vm) - } - - #[pymethod] - fn __sub__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a - b), vm) - } - - #[pymethod] - fn __rsub__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(b - a), vm) - } - #[pymethod] fn conjugate(&self) -> Complex64 { self.value.conj() } - #[pymethod(name = "__rmul__")] - #[pymethod] - fn __mul__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a * b), vm) - } - - #[pymethod] - fn __truediv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_div(a, b, vm), vm) - } - - #[pymethod] - fn __rtruediv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_div(b, a, vm), vm) - } - - #[pymethod] - const fn __pos__(&self) -> Complex64 { - self.value - } - - #[pymethod] - fn __neg__(&self) -> Complex64 { - -self.value - } - - #[pymethod] - fn __pow__( - &self, - other: PyObjectRef, - mod_val: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - if mod_val.flatten().is_some() { - Err(vm.new_value_error("complex modulo not allowed")) - } else { - self.op(other, |a, b| inner_pow(a, b, vm), vm) - } - } - - #[pymethod] - fn __rpow__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_pow(b, a, vm), vm) - } - - #[pymethod] - fn __bool__(&self) -> bool { - !Complex64::is_zero(&self.value) - } - #[pymethod] const fn __getnewargs__(&self) -> (f64, f64) { let Complex64 { re, im } = self.value; @@ -402,9 +276,13 @@ impl PyComplex { } #[pymethod] - fn __format__(&self, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + fn __format__(zelf: &Py<Self>, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + // Empty format spec: equivalent to str(self) + if spec.is_empty() { + return Ok(zelf.as_object().str(vm)?.as_str().to_owned()); + } FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_complex(&self.value)) + .and_then(|format_spec| format_spec.format_complex(&zelf.value)) .map_err(|err| err.into_pyexception(vm)) } } @@ -489,7 +367,12 @@ impl AsNumber for PyComplex { }), absolute: Some(|number, vm| { let value = PyComplex::number_downcast(number).value; - value.norm().to_pyresult(vm) + let result = value.norm(); + // Check for overflow: hypot returns inf for finite inputs that overflow + if result.is_infinite() && value.re.is_finite() && value.im.is_finite() { + return Err(vm.new_overflow_error("absolute value too large".to_owned())); + } + result.to_pyresult(vm) }), boolean: Some(|number, _vm| Ok(!PyComplex::number_downcast(number).value.is_zero())), true_divide: Some(|a, b, vm| PyComplex::number_op(a, b, inner_div, vm)), diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index 0909cdfb444..c8547b8a41f 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -2,21 +2,28 @@ use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, - coroutine::Coro, + coroutine::{Coro, warn_deprecated_throw_signature}, frame::FrameRef, function::OptionalArg, + object::{Traverse, TraverseFn}, protocol::PyIterReturn, - types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; -#[pyclass(module = false, name = "coroutine")] +#[pyclass(module = false, name = "coroutine", traverse = "manual")] #[derive(Debug)] // PyCoro_Type in CPython pub struct PyCoroutine { inner: Coro, } +unsafe impl Traverse for PyCoroutine { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + } +} + impl PyPayload for PyCoroutine { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -24,7 +31,10 @@ impl PyPayload for PyCoroutine { } } -#[pyclass(with(Py, Unconstructible, IterNext, Representable))] +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with(Py, IterNext, Representable, Destructor) +)] impl PyCoroutine { pub const fn as_coro(&self) -> &Coro { &self.inner @@ -69,8 +79,12 @@ impl PyCoroutine { self.inner.frame().yield_from_target() } #[pygetset] - fn cr_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() + fn cr_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } } #[pygetset] fn cr_running(&self, _vm: &VirtualMachine) -> bool { @@ -108,6 +122,7 @@ impl Py<PyCoroutine> { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult<PyIterReturn> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; self.inner.throw( self.as_object(), exc_type, @@ -118,13 +133,11 @@ impl Py<PyCoroutine> { } #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { self.inner.close(self.as_object(), vm) } } -impl Unconstructible for PyCoroutine {} - impl Representable for PyCoroutine { #[inline] fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { @@ -139,7 +152,23 @@ impl IterNext for PyCoroutine { } } -#[pyclass(module = false, name = "coroutine_wrapper")] +impl Destructor for PyCoroutine { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + if zelf.inner.closed() || zelf.inner.running() { + return Ok(()); + } + if zelf.inner.frame().lasti() == 0 { + zelf.inner.closed.store(true); + return Ok(()); + } + if let Err(e) = zelf.inner.close(zelf.as_object(), vm) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + Ok(()) + } +} + +#[pyclass(module = false, name = "coroutine_wrapper", traverse = "manual")] #[derive(Debug)] // PyCoroWrapper_Type in CPython pub struct PyCoroutineWrapper { @@ -147,6 +176,12 @@ pub struct PyCoroutineWrapper { closed: AtomicCell<bool>, } +unsafe impl Traverse for PyCoroutineWrapper { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.coro.traverse(tracer_fn); + } +} + impl PyPayload for PyCoroutineWrapper { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -183,6 +218,7 @@ impl PyCoroutineWrapper { vm: &VirtualMachine, ) -> PyResult<PyIterReturn> { self.check_closed(vm)?; + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; let result = self.coro.throw(exc_type, exc_val, exc_tb, vm); // Mark as closed if exhausted if let Ok(PyIterReturn::StopIteration(_)) = &result { @@ -192,7 +228,7 @@ impl PyCoroutineWrapper { } #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { self.closed.store(true); self.coro.close(vm) } @@ -205,6 +241,12 @@ impl IterNext for PyCoroutineWrapper { } } +impl Drop for PyCoroutine { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + pub fn init(ctx: &Context) { PyCoroutine::extend_class(ctx, ctx.types.coroutine_type); PyCoroutineWrapper::extend_class(ctx, ctx.types.coroutine_wrapper_type); diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index bc3aded3253..e1b92746a0f 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -3,8 +3,17 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyTypeRef, builtin_func::PyNativeMethod, type_}, class::PyClassImpl, - function::{FuncArgs, PyMethodDef, PyMethodFlags, PySetterValue}, - types::{Callable, GetDescriptor, Representable, Unconstructible}, + common::hash::PyHash, + convert::{ToPyObject, ToPyResult}, + function::{ArgSize, FuncArgs, PyMethodDef, PyMethodFlags, PySetterValue}, + protocol::{PyNumberBinaryFunc, PyNumberTernaryFunc, PyNumberUnaryFunc}, + types::{ + Callable, Comparable, DelFunc, DescrGetFunc, DescrSetFunc, GenericMethod, GetDescriptor, + GetattroFunc, HashFunc, Hashable, InitFunc, IterFunc, IterNextFunc, MapAssSubscriptFunc, + MapLenFunc, MapSubscriptFunc, PyComparisonOp, Representable, RichCompareFunc, + SeqAssItemFunc, SeqConcatFunc, SeqContainsFunc, SeqItemFunc, SeqLenFunc, SeqRepeatFunc, + SetattroFunc, StringifyFunc, + }, }; use rustpython_common::lock::PyRwLock; @@ -50,8 +59,8 @@ impl PyPayload for PyMethodDescriptor { } } -impl std::fmt::Debug for PyMethodDescriptor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMethodDescriptor { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "method descriptor for '{}'", self.common.name) } } @@ -105,8 +114,8 @@ impl PyMethodDescriptor { } #[pyclass( - with(GetDescriptor, Callable, Unconstructible, Representable), - flags(METHOD_DESCRIPTOR) + with(GetDescriptor, Callable, Representable), + flags(METHOD_DESCRIPTOR, DISALLOW_INSTANTIATION) )] impl PyMethodDescriptor { #[pygetset] @@ -159,10 +168,9 @@ impl Representable for PyMethodDescriptor { } } -impl Unconstructible for PyMethodDescriptor {} - #[derive(Debug)] pub enum MemberKind { + Object = 6, Bool = 14, ObjectEx = 16, } @@ -211,8 +219,8 @@ impl PyMemberDef { } } -impl std::fmt::Debug for PyMemberDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMemberDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyMemberDef") .field("name", &self.name) .field("kind", &self.kind) @@ -221,7 +229,7 @@ impl std::fmt::Debug for PyMemberDef { } } -// PyMemberDescrObject in CPython +// = PyMemberDescrObject #[pyclass(name = "member_descriptor", module = false)] #[derive(Debug)] pub struct PyMemberDescriptor { @@ -246,8 +254,20 @@ fn calculate_qualname(descr: &PyDescriptorOwned, vm: &VirtualMachine) -> PyResul } } -#[pyclass(with(GetDescriptor, Unconstructible, Representable), flags(BASETYPE))] +#[pyclass(with(GetDescriptor, Representable), flags(DISALLOW_INSTANTIATION))] impl PyMemberDescriptor { + #[pymember] + fn __objclass__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: &Py<Self> = zelf.try_to_value(vm)?; + Ok(zelf.common.typ.clone().into()) + } + + #[pymember] + fn __name__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: &Py<Self> = zelf.try_to_value(vm)?; + Ok(zelf.common.name.to_owned().into()) + } + #[pygetset] fn __doc__(&self) -> Option<String> { self.member.doc.to_owned() @@ -266,6 +286,23 @@ impl PyMemberDescriptor { }) } + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + Ok(vm + .ctx + .new_tuple(vec![ + builtins_getattr, + vm.ctx + .new_tuple(vec![ + self.common.typ.clone().into(), + vm.ctx.new_str(self.common.name.as_str()).into(), + ]) + .into(), + ]) + .into()) + } + #[pyslot] fn descr_set( zelf: &PyObject, @@ -296,6 +333,7 @@ fn get_slot_from_object( vm: &VirtualMachine, ) -> PyResult { let slot = match member.kind { + MemberKind::Object => obj.get_slot(offset).unwrap_or_else(|| vm.ctx.none()), MemberKind::Bool => obj .get_slot(offset) .unwrap_or_else(|| vm.ctx.new_bool(false).into()), @@ -315,32 +353,43 @@ fn set_slot_at_object( vm: &VirtualMachine, ) -> PyResult<()> { match member.kind { + MemberKind::Object => match value { + PySetterValue::Assign(v) => { + obj.set_slot(offset, Some(v)); + } + PySetterValue::Delete => { + obj.set_slot(offset, None); + } + }, MemberKind::Bool => { match value { PySetterValue::Assign(v) => { if !v.class().is(vm.ctx.types.bool_type) { return Err(vm.new_type_error("attribute value type must be bool")); } - obj.set_slot(offset, Some(v)) } - PySetterValue::Delete => obj.set_slot(offset, None), - }; - } - MemberKind::ObjectEx => { - let value = match value { - PySetterValue::Assign(v) => Some(v), - PySetterValue::Delete => None, + PySetterValue::Delete => { + return Err(vm.new_type_error("can't delete numeric/char attribute".to_owned())); + } }; - obj.set_slot(offset, value); } + MemberKind::ObjectEx => match value { + PySetterValue::Assign(v) => { + obj.set_slot(offset, Some(v)); + } + PySetterValue::Delete => { + if obj.get_slot(offset).is_none() { + return Err(vm.new_attribute_error(member.name.clone())); + } + obj.set_slot(offset, None); + } + }, } Ok(()) } -impl Unconstructible for PyMemberDescriptor {} - impl Representable for PyMemberDescriptor { #[inline] fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { @@ -356,26 +405,23 @@ impl GetDescriptor for PyMemberDescriptor { fn descr_get( zelf: PyObjectRef, obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult { let descr = Self::_as_pyref(&zelf, vm)?; match obj { - Some(x) => descr.member.get(x, vm), - None => { - // When accessed from class (not instance), for __doc__ member descriptor, - // return the class's docstring if available - // When accessed from class (not instance), check if the class has - // an attribute with the same name as this member descriptor - if let Some(cls) = cls - && let Ok(cls_type) = cls.downcast::<PyType>() - && let Some(interned) = vm.ctx.interned_str(descr.member.name.as_str()) - && let Some(attr) = cls_type.attributes.read().get(&interned) - { - return Ok(attr.clone()); + Some(x) => { + if !x.class().fast_issubclass(&descr.common.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' for '{}' objects doesn't apply to a '{}' object", + descr.common.name, + descr.common.typ.name(), + x.class().name() + ))); } - Ok(zelf) + descr.member.get(x, vm) } + None => Ok(zelf), } } } @@ -383,4 +429,483 @@ impl GetDescriptor for PyMemberDescriptor { pub fn init(ctx: &Context) { PyMemberDescriptor::extend_class(ctx, ctx.types.member_descriptor_type); PyMethodDescriptor::extend_class(ctx, ctx.types.method_descriptor_type); + PyWrapper::extend_class(ctx, ctx.types.wrapper_descriptor_type); + PyMethodWrapper::extend_class(ctx, ctx.types.method_wrapper_type); +} + +// PyWrapper - wrapper_descriptor + +/// Each variant knows how to call the wrapped function with proper types +#[derive(Clone, Copy)] +pub enum SlotFunc { + // Basic slots + Init(InitFunc), + Hash(HashFunc), + Str(StringifyFunc), + Repr(StringifyFunc), + Iter(IterFunc), + IterNext(IterNextFunc), + Call(GenericMethod), + Del(DelFunc), + + // Attribute access slots + GetAttro(GetattroFunc), + SetAttro(SetattroFunc), // __setattr__ + DelAttro(SetattroFunc), // __delattr__ (same func type, different PySetterValue) + + // Rich comparison slots (with comparison op) + RichCompare(RichCompareFunc, PyComparisonOp), + + // Descriptor slots + DescrGet(DescrGetFunc), + DescrSet(DescrSetFunc), // __set__ + DescrDel(DescrSetFunc), // __delete__ (same func type, different PySetterValue) + + // Sequence sub-slots (sq_*) + SeqLength(SeqLenFunc), + SeqConcat(SeqConcatFunc), + SeqRepeat(SeqRepeatFunc), + SeqItem(SeqItemFunc), + SeqSetItem(SeqAssItemFunc), // __setitem__ (same func type, value = Some) + SeqDelItem(SeqAssItemFunc), // __delitem__ (same func type, value = None) + SeqContains(SeqContainsFunc), + + // Mapping sub-slots (mp_*) + MapLength(MapLenFunc), + MapSubscript(MapSubscriptFunc), + MapSetSubscript(MapAssSubscriptFunc), // __setitem__ (same func type, value = Some) + MapDelSubscript(MapAssSubscriptFunc), // __delitem__ (same func type, value = None) + + // Number sub-slots (nb_*) - grouped by signature + NumBoolean(PyNumberUnaryFunc<bool>), // __bool__ + NumUnary(PyNumberUnaryFunc), // __int__, __float__, __index__ + NumBinary(PyNumberBinaryFunc), // __add__, __sub__, __mul__, etc. + NumBinaryRight(PyNumberBinaryFunc), // __radd__, __rsub__, etc. (swapped args) + NumTernary(PyNumberTernaryFunc), // __pow__ + NumTernaryRight(PyNumberTernaryFunc), // __rpow__ (swapped first two args) +} + +impl core::fmt::Debug for SlotFunc { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + SlotFunc::Init(_) => write!(f, "SlotFunc::Init(...)"), + SlotFunc::Hash(_) => write!(f, "SlotFunc::Hash(...)"), + SlotFunc::Str(_) => write!(f, "SlotFunc::Str(...)"), + SlotFunc::Repr(_) => write!(f, "SlotFunc::Repr(...)"), + SlotFunc::Iter(_) => write!(f, "SlotFunc::Iter(...)"), + SlotFunc::IterNext(_) => write!(f, "SlotFunc::IterNext(...)"), + SlotFunc::Call(_) => write!(f, "SlotFunc::Call(...)"), + SlotFunc::Del(_) => write!(f, "SlotFunc::Del(...)"), + SlotFunc::GetAttro(_) => write!(f, "SlotFunc::GetAttro(...)"), + SlotFunc::SetAttro(_) => write!(f, "SlotFunc::SetAttro(...)"), + SlotFunc::DelAttro(_) => write!(f, "SlotFunc::DelAttro(...)"), + SlotFunc::RichCompare(_, op) => write!(f, "SlotFunc::RichCompare(..., {:?})", op), + SlotFunc::DescrGet(_) => write!(f, "SlotFunc::DescrGet(...)"), + SlotFunc::DescrSet(_) => write!(f, "SlotFunc::DescrSet(...)"), + SlotFunc::DescrDel(_) => write!(f, "SlotFunc::DescrDel(...)"), + // Sequence sub-slots + SlotFunc::SeqLength(_) => write!(f, "SlotFunc::SeqLength(...)"), + SlotFunc::SeqConcat(_) => write!(f, "SlotFunc::SeqConcat(...)"), + SlotFunc::SeqRepeat(_) => write!(f, "SlotFunc::SeqRepeat(...)"), + SlotFunc::SeqItem(_) => write!(f, "SlotFunc::SeqItem(...)"), + SlotFunc::SeqSetItem(_) => write!(f, "SlotFunc::SeqSetItem(...)"), + SlotFunc::SeqDelItem(_) => write!(f, "SlotFunc::SeqDelItem(...)"), + SlotFunc::SeqContains(_) => write!(f, "SlotFunc::SeqContains(...)"), + // Mapping sub-slots + SlotFunc::MapLength(_) => write!(f, "SlotFunc::MapLength(...)"), + SlotFunc::MapSubscript(_) => write!(f, "SlotFunc::MapSubscript(...)"), + SlotFunc::MapSetSubscript(_) => write!(f, "SlotFunc::MapSetSubscript(...)"), + SlotFunc::MapDelSubscript(_) => write!(f, "SlotFunc::MapDelSubscript(...)"), + // Number sub-slots + SlotFunc::NumBoolean(_) => write!(f, "SlotFunc::NumBoolean(...)"), + SlotFunc::NumUnary(_) => write!(f, "SlotFunc::NumUnary(...)"), + SlotFunc::NumBinary(_) => write!(f, "SlotFunc::NumBinary(...)"), + SlotFunc::NumBinaryRight(_) => write!(f, "SlotFunc::NumBinaryRight(...)"), + SlotFunc::NumTernary(_) => write!(f, "SlotFunc::NumTernary(...)"), + SlotFunc::NumTernaryRight(_) => write!(f, "SlotFunc::NumTernaryRight(...)"), + } + } +} + +impl SlotFunc { + /// Call the wrapped slot function with proper type handling + pub fn call(&self, obj: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + match self { + SlotFunc::Init(func) => { + func(obj, args, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::Hash(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err( + vm.new_type_error("__hash__() takes no arguments (1 given)".to_owned()) + ); + } + let hash = func(&obj, vm)?; + Ok(vm.ctx.new_int(hash).into()) + } + SlotFunc::Repr(func) | SlotFunc::Str(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + let name = match self { + SlotFunc::Repr(_) => "__repr__", + SlotFunc::Str(_) => "__str__", + _ => unreachable!(), + }; + return Err(vm.new_type_error(format!("{name}() takes no arguments (1 given)"))); + } + let s = func(&obj, vm)?; + Ok(s.into()) + } + SlotFunc::Iter(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err( + vm.new_type_error("__iter__() takes no arguments (1 given)".to_owned()) + ); + } + func(obj, vm) + } + SlotFunc::IterNext(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err( + vm.new_type_error("__next__() takes no arguments (1 given)".to_owned()) + ); + } + func(&obj, vm).to_pyresult(vm) + } + SlotFunc::Call(func) => func(&obj, args, vm), + SlotFunc::Del(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err( + vm.new_type_error("__del__() takes no arguments (1 given)".to_owned()) + ); + } + func(&obj, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::GetAttro(func) => { + let (name,): (PyRef<PyStr>,) = args.bind(vm)?; + func(&obj, &name, vm) + } + SlotFunc::SetAttro(func) => { + let (name, value): (PyRef<PyStr>, PyObjectRef) = args.bind(vm)?; + func(&obj, &name, PySetterValue::Assign(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::DelAttro(func) => { + let (name,): (PyRef<PyStr>,) = args.bind(vm)?; + func(&obj, &name, PySetterValue::Delete, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::RichCompare(func, op) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, &other, *op, vm).map(|r| match r { + crate::function::Either::A(obj) => obj, + crate::function::Either::B(cmp_val) => cmp_val.to_pyobject(vm), + }) + } + SlotFunc::DescrGet(func) => { + let (instance, owner): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let owner = owner.into_option(); + let instance_opt = if vm.is_none(&instance) { + None + } else { + Some(instance) + }; + func(obj, instance_opt, owner, vm) + } + SlotFunc::DescrSet(func) => { + let (instance, value): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + func(&obj, instance, PySetterValue::Assign(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::DescrDel(func) => { + let (instance,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, instance, PySetterValue::Delete, vm)?; + Ok(vm.ctx.none()) + } + // Sequence sub-slots + SlotFunc::SeqLength(func) => { + args.bind::<()>(vm)?; + let len = func(obj.sequence_unchecked(), vm)?; + Ok(vm.ctx.new_int(len).into()) + } + SlotFunc::SeqConcat(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(obj.sequence_unchecked(), &other, vm) + } + SlotFunc::SeqRepeat(func) => { + let (n,): (ArgSize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), n.into(), vm) + } + SlotFunc::SeqItem(func) => { + let (index,): (isize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, vm) + } + SlotFunc::SeqSetItem(func) => { + let (index, value): (isize, PyObjectRef) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, Some(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::SeqDelItem(func) => { + let (index,): (isize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, None, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::SeqContains(func) => { + let (item,): (PyObjectRef,) = args.bind(vm)?; + let result = func(obj.sequence_unchecked(), &item, vm)?; + Ok(vm.ctx.new_bool(result).into()) + } + // Mapping sub-slots + SlotFunc::MapLength(func) => { + args.bind::<()>(vm)?; + let len = func(obj.mapping_unchecked(), vm)?; + Ok(vm.ctx.new_int(len).into()) + } + SlotFunc::MapSubscript(func) => { + let (key,): (PyObjectRef,) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, vm) + } + SlotFunc::MapSetSubscript(func) => { + let (key, value): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, Some(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::MapDelSubscript(func) => { + let (key,): (PyObjectRef,) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, None, vm)?; + Ok(vm.ctx.none()) + } + // Number sub-slots + SlotFunc::NumBoolean(func) => { + args.bind::<()>(vm)?; + let result = func(obj.number(), vm)?; + Ok(vm.ctx.new_bool(result).into()) + } + SlotFunc::NumUnary(func) => { + args.bind::<()>(vm)?; + func(obj.number(), vm) + } + SlotFunc::NumBinary(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, &other, vm) + } + SlotFunc::NumBinaryRight(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&other, &obj, vm) // Swapped: other op obj + } + SlotFunc::NumTernary(func) => { + let (y, z): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let z = z.unwrap_or_else(|| vm.ctx.none()); + func(&obj, &y, &z, vm) + } + SlotFunc::NumTernaryRight(func) => { + let (y, z): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let z = z.unwrap_or_else(|| vm.ctx.none()); + func(&y, &obj, &z, vm) // Swapped: y ** obj % z + } + } + } +} + +/// wrapper_descriptor: wraps a slot function as a Python method +// = PyWrapperDescrObject +#[pyclass(name = "wrapper_descriptor", module = false)] +#[derive(Debug)] +pub struct PyWrapper { + pub typ: &'static Py<PyType>, + pub name: &'static PyStrInterned, + pub wrapped: SlotFunc, + pub doc: Option<&'static str>, +} + +impl PyPayload for PyWrapper { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.wrapper_descriptor_type + } +} + +impl GetDescriptor for PyWrapper { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + match obj { + None => Ok(zelf), + Some(obj) => { + let zelf = zelf.downcast::<Self>().unwrap(); + Ok(PyMethodWrapper { wrapper: zelf, obj }.into_pyobject(vm)) + } + } + } +} + +impl Callable for PyWrapper { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // list.__init__(l, [1,2,3]) form - first arg is self + let (obj, rest): (PyObjectRef, FuncArgs) = args.bind(vm)?; + + if !obj.fast_isinstance(zelf.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' requires a '{}' object but received a '{}'", + zelf.name.as_str(), + zelf.typ.name(), + obj.class().name() + ))); + } + + zelf.wrapped.call(obj, rest, vm) + } +} + +#[pyclass( + with(GetDescriptor, Callable, Representable), + flags(DISALLOW_INSTANTIATION) +)] +impl PyWrapper { + #[pygetset] + fn __name__(&self) -> &'static PyStrInterned { + self.name + } + + #[pygetset] + fn __qualname__(&self) -> String { + format!("{}.{}", self.typ.name(), self.name) + } + + #[pygetset] + fn __objclass__(&self) -> PyTypeRef { + self.typ.to_owned() + } + + #[pygetset] + fn __doc__(&self) -> Option<&'static str> { + self.doc + } +} + +impl Representable for PyWrapper { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<slot wrapper '{}' of '{}' objects>", + zelf.name.as_str(), + zelf.typ.name() + )) + } +} + +// PyMethodWrapper - method-wrapper + +/// method-wrapper: a slot wrapper bound to an instance +/// Returned when accessing l.__init__ on an instance +#[pyclass(name = "method-wrapper", module = false, traverse)] +#[derive(Debug)] +pub struct PyMethodWrapper { + pub wrapper: PyRef<PyWrapper>, + #[pytraverse(skip)] + pub obj: PyObjectRef, +} + +impl PyPayload for PyMethodWrapper { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.method_wrapper_type + } +} + +impl Callable for PyMethodWrapper { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // bpo-37619: Check type compatibility before calling wrapped slot + if !zelf.obj.fast_isinstance(zelf.wrapper.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' requires a '{}' object but received a '{}'", + zelf.wrapper.name.as_str(), + zelf.wrapper.typ.name(), + zelf.obj.class().name() + ))); + } + zelf.wrapper.wrapped.call(zelf.obj.clone(), args, vm) + } +} + +#[pyclass( + with(Callable, Representable, Hashable, Comparable), + flags(DISALLOW_INSTANTIATION) +)] +impl PyMethodWrapper { + #[pygetset] + fn __self__(&self) -> PyObjectRef { + self.obj.clone() + } + + #[pygetset] + fn __name__(&self) -> &'static PyStrInterned { + self.wrapper.name + } + + #[pygetset] + fn __objclass__(&self) -> PyTypeRef { + self.wrapper.typ.to_owned() + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + Ok(vm + .ctx + .new_tuple(vec![ + builtins_getattr, + vm.ctx + .new_tuple(vec![ + zelf.obj.clone(), + vm.ctx.new_str(zelf.wrapper.name.as_str()).into(), + ]) + .into(), + ]) + .into()) + } +} + +impl Representable for PyMethodWrapper { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<method-wrapper '{}' of {} object at {:#x}>", + zelf.wrapper.name.as_str(), + zelf.obj.class().name(), + zelf.obj.get_id() + )) + } +} + +impl Hashable for PyMethodWrapper { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + let obj_hash = zelf.obj.hash(vm)?; + let wrapper_hash = zelf.wrapper.as_object().get_id() as PyHash; + Ok(obj_hash ^ wrapper_hash) + } +} + +impl Comparable for PyMethodWrapper { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<crate::function::PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + let eq = zelf.wrapper.is(&other.wrapper) && vm.bool_eq(&zelf.obj, &other.obj)?; + Ok(eq.into()) + }) + } } diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 77126d4ee62..1d79a5a5906 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -2,6 +2,8 @@ use super::{ IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, }; +use crate::common::lock::LazyLock; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromObject, atomic_func, @@ -19,23 +21,37 @@ use crate::{ recursion::ReprGuard, types::{ AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, DefaultConstructor, - Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, + Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, vm::VirtualMachine, }; +use alloc::fmt; use rustpython_common::lock::PyMutex; -use std::fmt; -use std::sync::LazyLock; pub type DictContentType = dict_inner::Dict; -#[pyclass(module = false, name = "dict", unhashable = true, traverse)] +#[pyclass(module = false, name = "dict", unhashable = true, traverse = "manual")] #[derive(Default)] pub struct PyDict { entries: DictContentType, } pub type PyDictRef = PyRef<PyDict>; +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyDict { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.entries.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + // Pop all entries and collect both keys and values + for (key, value) in self.entries.drain_entries() { + out.push(key); + out.push(value); + } + } +} + impl fmt::Debug for PyDict { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter @@ -62,6 +78,24 @@ impl PyDict { &self.entries } + /// Returns all keys as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn keys_vec(&self) -> Vec<PyObjectRef> { + self.entries.keys() + } + + /// Returns all values as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn values_vec(&self) -> Vec<PyObjectRef> { + self.entries.values() + } + + /// Returns all items as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn items_vec(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + self.entries.items() + } + // Used in update and ior. pub(crate) fn merge_object(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let casted: Result<PyRefExact<Self>, _> = other.downcast_exact(vm); @@ -70,13 +104,21 @@ impl PyDict { Err(other) => other, }; let dict = &self.entries; - if let Some(keys) = vm.get_method(other.clone(), vm.ctx.intern_str("keys")) { - let keys = keys?.call((), vm)?.get_iter(vm)?; - while let PyIterReturn::Return(key) = keys.next(vm)? { - let val = other.get_item(&*key, vm)?; - dict.insert(vm, &*key, val)?; + // Use get_attr to properly invoke __getattribute__ for proxy objects + let keys_result = other.get_attr(vm.ctx.intern_str("keys"), vm); + let has_keys = match keys_result { + Ok(keys_method) => { + let keys = keys_method.call((), vm)?.get_iter(vm)?; + while let PyIterReturn::Return(key) = keys.next(vm)? { + let val = other.get_item(&*key, vm)?; + dict.insert(vm, &*key, val)?; + } + true } - } else { + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => false, + Err(e) => return Err(e), + }; + if !has_keys { let iter = other.get_iter(vm)?; loop { fn err(vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -204,22 +246,19 @@ impl PyDict { } } - #[pymethod] pub fn __len__(&self) -> usize { self.entries.len() } #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + self.entries.sizeof() + core::mem::size_of::<Self>() + self.entries.sizeof() } - #[pymethod] fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self.entries.contains(vm, &*key) } - #[pymethod] fn __delitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner_delitem(&*key, vm) } @@ -229,7 +268,6 @@ impl PyDict { self.entries.clear() } - #[pymethod] fn __setitem__( &self, key: PyObjectRef, @@ -286,7 +324,6 @@ impl PyDict { Ok(()) } - #[pymethod] fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let other_dict: Result<PyDictRef, _> = other.downcast(); if let Ok(other) = other_dict { @@ -367,7 +404,6 @@ impl Py<PyDict> { Ok(Implemented(true)) } - #[pymethod] #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner_getitem(&*key, vm) @@ -396,13 +432,11 @@ impl PyRef<PyDict> { PyDictReverseKeyIterator::new(self) } - #[pymethod] fn __ior__(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { self.merge_object(other, vm)?; Ok(self) } - #[pymethod] fn __ror__(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let other_dict: Result<Self, _> = other.downcast(); if let Ok(other) = other_dict { @@ -751,12 +785,11 @@ impl ExactSizeIterator for DictIter<'_> { #[pyclass] trait DictView: PyPayload + PyClassDef + Iterable + Representable { - type ReverseIter: PyPayload + std::fmt::Debug; + type ReverseIter: PyPayload + core::fmt::Debug; - fn dict(&self) -> &PyDictRef; + fn dict(&self) -> &Py<PyDict>; fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef; - #[pymethod] fn __len__(&self) -> usize { self.dict().__len__() } @@ -785,7 +818,7 @@ macro_rules! dict_view { impl DictView for $name { type ReverseIter = $reverse_iter_name; - fn dict(&self) -> &PyDictRef { + fn dict(&self) -> &Py<PyDict> { &self.dict } @@ -848,7 +881,7 @@ macro_rules! dict_view { } } - #[pyclass(with(Unconstructible, IterNext, Iterable))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl $iter_name { fn new(dict: PyDictRef) -> Self { $iter_name { @@ -865,7 +898,7 @@ macro_rules! dict_view { #[allow(clippy::redundant_closure_call)] #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_iter(vm).to_owned(); + let iter = builtins_iter(vm); let internal = self.internal.lock(); let entries = match &internal.status { IterStatus::Active(dict) => dict @@ -878,8 +911,6 @@ macro_rules! dict_view { } } - impl Unconstructible for $iter_name {} - impl SelfIter for $iter_name {} impl IterNext for $iter_name { #[allow(clippy::redundant_closure_call)] @@ -923,7 +954,7 @@ macro_rules! dict_view { } } - #[pyclass(with(Unconstructible, IterNext, Iterable))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl $reverse_iter_name { fn new(dict: PyDictRef) -> Self { let size = dict.size(); @@ -937,7 +968,7 @@ macro_rules! dict_view { #[allow(clippy::redundant_closure_call)] #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_reversed(vm).to_owned(); + let iter = builtins_reversed(vm); let internal = self.internal.lock(); // TODO: entries must be reversed too let entries = match &internal.status { @@ -957,8 +988,6 @@ macro_rules! dict_view { .rev_length_hint(|_| self.size.entries_size) } } - impl Unconstructible for $reverse_iter_name {} - impl SelfIter for $reverse_iter_name {} impl IterNext for $reverse_iter_name { #[allow(clippy::redundant_closure_call)] @@ -1044,38 +1073,30 @@ trait ViewSetOps: DictView { PySetInner::from_iter(iter, vm) } - #[pymethod(name = "__rxor__")] - #[pymethod] fn __xor__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { let zelf = Self::to_set(zelf, vm)?; let inner = zelf.symmetric_difference(other, vm)?; Ok(PySet { inner }) } - #[pymethod(name = "__rand__")] - #[pymethod] fn __and__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { let zelf = Self::to_set(zelf, vm)?; let inner = zelf.intersection(other, vm)?; Ok(PySet { inner }) } - #[pymethod(name = "__ror__")] - #[pymethod] fn __or__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { let zelf = Self::to_set(zelf, vm)?; let inner = zelf.union(other, vm)?; Ok(PySet { inner }) } - #[pymethod] fn __sub__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { let zelf = Self::to_set(zelf, vm)?; let inner = zelf.difference(other, vm)?; Ok(PySet { inner }) } - #[pymethod] fn __rsub__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { let left = PySetInner::from_iter(other.iter(vm)?, vm)?; let right = ArgIterable::try_from_object(vm, Self::iter(zelf, vm)?)?; @@ -1126,28 +1147,28 @@ trait ViewSetOps: DictView { } impl ViewSetOps for PyDictKeys {} -#[pyclass(with( - DictView, - Unconstructible, - Comparable, - Iterable, - ViewSetOps, - AsSequence, - AsNumber, - Representable -))] +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with( + DictView, + Comparable, + Iterable, + ViewSetOps, + AsSequence, + AsNumber, + Representable + ) +)] impl PyDictKeys { - #[pymethod] fn __contains__(zelf: PyObjectRef, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - zelf.to_sequence().contains(&key, vm) + zelf.sequence_unchecked().contains(&key, vm) } #[pygetset] fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) + PyMappingProxy::from(zelf.dict().to_owned()) } } -impl Unconstructible for PyDictKeys {} impl Comparable for PyDictKeys { fn cmp( @@ -1190,27 +1211,27 @@ impl AsNumber for PyDictKeys { } impl ViewSetOps for PyDictItems {} -#[pyclass(with( - DictView, - Unconstructible, - Comparable, - Iterable, - ViewSetOps, - AsSequence, - AsNumber, - Representable -))] +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with( + DictView, + Comparable, + Iterable, + ViewSetOps, + AsSequence, + AsNumber, + Representable + ) +)] impl PyDictItems { - #[pymethod] fn __contains__(zelf: PyObjectRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - zelf.to_sequence().contains(&needle, vm) + zelf.sequence_unchecked().contains(&needle, vm) } #[pygetset] fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) + PyMappingProxy::from(zelf.dict().to_owned()) } } -impl Unconstructible for PyDictItems {} impl Comparable for PyDictItems { fn cmp( @@ -1264,14 +1285,16 @@ impl AsNumber for PyDictItems { } } -#[pyclass(with(DictView, Unconstructible, Iterable, AsSequence, Representable))] +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with(DictView, Iterable, AsSequence, Representable) +)] impl PyDictValues { #[pygetset] fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) + PyMappingProxy::from(zelf.dict().to_owned()) } } -impl Unconstructible for PyDictValues {} impl AsSequence for PyDictValues { fn as_sequence() -> &'static PySequenceMethods { diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index 29d51f420e1..3cd8f4e734a 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -1,5 +1,6 @@ use super::{ IterStatus, PositionIterInternal, PyGenericAlias, PyIntRef, PyTupleRef, PyType, PyTypeRef, + iter::builtins_reversed, }; use crate::common::lock::{PyMutex, PyRwLock}; use crate::{ @@ -124,9 +125,13 @@ impl PyReverseSequenceIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_reversed_reduce(|x| x.clone(), vm) + let func = builtins_reversed(vm); + self.internal.lock().reduce( + func, + |x| x.clone(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } } diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 26182d748a9..89e42ec0f39 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -8,8 +8,7 @@ use crate::{ common::{float_ops, format::FormatSpec, hash}, convert::{IntoPyException, ToPyObject, ToPyResult}, function::{ - ArgBytesLike, FuncArgs, OptionalArg, OptionalOption, - PyArithmeticValue::{self, *}, + ArgBytesLike, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue::*, PyComparisonValue, }, protocol::PyNumberMethods, @@ -117,7 +116,7 @@ fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v1.is_zero() && v2.is_sign_negative() { - let msg = "0.0 cannot be raised to a negative power"; + let msg = "zero to a negative power"; Err(vm.new_zero_division_error(msg.to_owned())) } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { let v1 = Complex64::new(v1, 0.); @@ -215,9 +214,13 @@ fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { )] impl PyFloat { #[pymethod] - fn __format__(&self, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + fn __format__(zelf: &Py<Self>, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + // Empty format spec: equivalent to str(self) + if spec.is_empty() { + return Ok(zelf.as_object().str(vm)?.as_str().to_owned()); + } FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_float(self.value)) + .and_then(|format_spec| format_spec.format_float(zelf.value)) .map_err(|err| err.into_pyexception(vm)) } @@ -239,182 +242,6 @@ impl PyFloat { .to_owned()) } - #[pymethod] - const fn __abs__(&self) -> f64 { - self.value.abs() - } - - #[inline] - fn simple_op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> - where - F: Fn(f64, f64) -> PyResult<f64>, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[inline] - fn complex_op<F>(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult - where - F: Fn(f64, f64) -> PyResult, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(vm.ctx.not_implemented()), - |other| op(self.value, other), - ) - } - - #[inline] - fn tuple_op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> - where - F: Fn(f64, f64) -> PyResult<(f64, f64)>, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[pymethod(name = "__radd__")] - #[pymethod] - fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a + b), vm) - } - - #[pymethod] - const fn __bool__(&self) -> bool { - self.value != 0.0 - } - - #[pymethod] - fn __divmod__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> { - self.tuple_op(other, |a, b| inner_divmod(a, b, vm), vm) - } - - #[pymethod] - fn __rdivmod__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> { - self.tuple_op(other, |a, b| inner_divmod(b, a, vm), vm) - } - - #[pymethod] - fn __floordiv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_floordiv(a, b, vm), vm) - } - - #[pymethod] - fn __rfloordiv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_floordiv(b, a, vm), vm) - } - - #[pymethod(name = "__mod__")] - fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_mod(a, b, vm), vm) - } - - #[pymethod] - fn __rmod__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_mod(b, a, vm), vm) - } - - #[pymethod] - const fn __pos__(&self) -> f64 { - self.value - } - - #[pymethod] - const fn __neg__(&self) -> f64 { - -self.value - } - - #[pymethod] - fn __pow__( - &self, - other: PyObjectRef, - mod_val: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - if mod_val.flatten().is_some() { - Err(vm.new_type_error("floating point pow() does not accept a 3rd argument")) - } else { - self.complex_op(other, |a, b| float_pow(a, b, vm), vm) - } - } - - #[pymethod] - fn __rpow__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.complex_op(other, |a, b| float_pow(b, a, vm), vm) - } - - #[pymethod] - fn __sub__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a - b), vm) - } - - #[pymethod] - fn __rsub__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(b - a), vm) - } - - #[pymethod] - fn __truediv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_div(a, b, vm), vm) - } - - #[pymethod] - fn __rtruediv__( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_div(b, a, vm), vm) - } - - #[pymethod(name = "__rmul__")] - #[pymethod] - fn __mul__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a * b), vm) - } - #[pymethod] fn __trunc__(&self, vm: &VirtualMachine) -> PyResult<BigInt> { try_to_bigint(self.value, vm) @@ -460,16 +287,6 @@ impl PyFloat { Ok(value) } - #[pymethod] - fn __int__(&self, vm: &VirtualMachine) -> PyResult<BigInt> { - self.__trunc__(vm) - } - - #[pymethod] - const fn __float__(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - #[pygetset] const fn real(zelf: PyRef<Self>) -> PyRef<Self> { zelf @@ -507,6 +324,21 @@ impl PyFloat { }) } + #[pyclassmethod] + fn from_number(cls: PyTypeRef, number: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if number.class().is(vm.ctx.types.float_type) && cls.is(vm.ctx.types.float_type) { + return Ok(number); + } + + let value = number.try_float(vm)?.to_f64(); + let result = vm.ctx.new_float(value); + if cls.is(vm.ctx.types.float_type) { + Ok(result.into()) + } else { + PyType::call(&cls, vec![result.into()].into(), vm) + } + } + #[pyclassmethod] fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { let result = crate::literal::float::from_hex(string.as_str().trim()) diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 17dc88ac042..ed2e1e672fd 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -4,11 +4,11 @@ use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; use crate::{ - AsObject, Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, class::PyClassImpl, - frame::{Frame, FrameRef}, + frame::{Frame, FrameOwner, FrameRef}, function::PySetterValue, - types::{Representable, Unconstructible}, + types::Representable, }; use num_traits::Zero; @@ -16,8 +16,6 @@ pub fn init(context: &Context) { Frame::extend_class(context, context.types.frame_type); } -impl Unconstructible for Frame {} - impl Representable for Frame { #[inline] fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { @@ -31,21 +29,24 @@ impl Representable for Frame { } } -#[pyclass(with(Unconstructible, Py))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(Py))] impl Frame { - #[pymethod] - const fn clear(&self) { - // TODO - } - #[pygetset] fn f_globals(&self) -> PyDictRef { self.globals.clone() } + #[pygetset] + fn f_builtins(&self) -> PyObjectRef { + self.builtins.clone() + } + #[pygetset] fn f_locals(&self, vm: &VirtualMachine) -> PyResult { - self.locals(vm).map(Into::into) + let result = self.locals(vm).map(Into::into); + self.locals_dirty + .store(true, core::sync::atomic::Ordering::Release); + result } #[pygetset] @@ -55,12 +56,19 @@ impl Frame { #[pygetset] fn f_lasti(&self) -> u32 { - self.lasti() + // Return byte offset (each instruction is 2 bytes) for compatibility + self.lasti() * 2 } #[pygetset] pub fn f_lineno(&self) -> usize { - self.current_location().line.get() + // If lasti is 0, execution hasn't started yet - use first line number + // Similar to PyCode_Addr2Line which returns co_firstlineno for addr_q < 0 + if self.lasti() == 0 { + self.code.first_line_number.map(|n| n.get()).unwrap_or(1) + } else { + self.current_location().line.get() + } } #[pygetset] @@ -105,22 +113,125 @@ impl Frame { PySetterValue::Delete => Err(vm.new_type_error("can't delete numeric/char attribute")), } } + + #[pymember(type = "bool")] + fn f_trace_opcodes(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + let trace_opcodes = zelf.trace_opcodes.lock(); + Ok(vm.ctx.new_bool(*trace_opcodes).into()) + } + + #[pymember(type = "bool", setter)] + fn set_f_trace_opcodes( + vm: &VirtualMachine, + zelf: PyObjectRef, + value: PySetterValue, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + + let value: PyIntRef = value + .downcast() + .map_err(|_| vm.new_type_error("attribute value type must be bool"))?; + + let mut trace_opcodes = zelf.trace_opcodes.lock(); + *trace_opcodes = !value.as_bigint().is_zero(); + + // TODO: Implement the equivalent of _PyEval_SetOpcodeTrace() + + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("can't delete numeric/char attribute")), + } + } } #[pyclass] impl Py<Frame> { + #[pymethod] + // = frame_clear_impl + fn clear(&self, vm: &VirtualMachine) -> PyResult<()> { + let owner = FrameOwner::from_i8(self.owner.load(core::sync::atomic::Ordering::Acquire)); + match owner { + FrameOwner::Generator => { + // Generator frame: check if suspended (lasti > 0 means + // FRAME_SUSPENDED). lasti == 0 means FRAME_CREATED and + // can be cleared. + if self.lasti() != 0 { + return Err(vm.new_runtime_error("cannot clear a suspended frame".to_owned())); + } + } + FrameOwner::Thread => { + // Thread-owned frame: always executing, cannot clear. + return Err(vm.new_runtime_error("cannot clear an executing frame".to_owned())); + } + FrameOwner::FrameObject => { + // Detached frame: safe to clear. + } + } + + // Clear fastlocals + { + let mut fastlocals = self.fastlocals.lock(); + for slot in fastlocals.iter_mut() { + *slot = None; + } + } + + // Clear the evaluation stack + self.clear_value_stack(); + + // Clear temporary refs + self.temporary_refs.lock().clear(); + + Ok(()) + } + + #[pygetset] + fn f_generator(&self) -> Option<PyObjectRef> { + self.generator.to_owned() + } + #[pygetset] pub fn f_back(&self, vm: &VirtualMachine) -> Option<PyRef<Frame>> { - // TODO: actually store f_back inside Frame struct + let previous = self.previous_frame(); + if previous.is_null() { + return None; + } - // get the frame in the frame stack that appears before this one. - // won't work if this frame isn't in the frame stack, hence the todo above - vm.frames + if let Some(frame) = vm + .frames .borrow() .iter() - .rev() - .skip_while(|p| !p.is(self.as_object())) - .nth(1) - .cloned() + .find(|fp| { + // SAFETY: the caller keeps the FrameRef alive while it's in the Vec + let py: &crate::Py<Frame> = unsafe { fp.as_ref() }; + let ptr: *const Frame = &**py; + core::ptr::eq(ptr, previous) + }) + .map(|fp| unsafe { fp.as_ref() }.to_owned()) + { + return Some(frame); + } + + #[cfg(feature = "threading")] + { + let registry = vm.state.thread_frames.lock(); + for slot in registry.values() { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so FramePtr is valid for the duration of the lock. + if let Some(frame) = frames.iter().find_map(|fp| { + let f = unsafe { fp.as_ref() }; + let ptr: *const Frame = &**f; + core::ptr::eq(ptr, previous).then(|| f.to_owned()) + }) { + return Some(frame); + } + } + } + + None } } diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 0459cecbdd2..9a6a6d49e3c 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -2,8 +2,8 @@ mod jit; use super::{ - PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStr, PyStrRef, PyTuple, PyTupleRef, - PyType, + PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyModule, PyStr, PyStrRef, PyTuple, + PyTupleRef, PyType, }; #[cfg(feature = "jit")] use crate::common::lock::OnceCell; @@ -25,6 +25,37 @@ use itertools::Itertools; #[cfg(feature = "jit")] use rustpython_jit::CompiledCode; +fn format_missing_args( + qualname: impl core::fmt::Display, + kind: &str, + missing: &mut Vec<impl core::fmt::Display>, +) -> String { + let count = missing.len(); + let last = if missing.len() > 1 { + missing.pop() + } else { + None + }; + let (and, right): (&str, String) = if let Some(last) = last { + ( + if missing.len() == 1 { + "' and '" + } else { + "', and '" + }, + format!("{last}"), + ) + } else { + ("", String::new()) + }; + format!( + "{qualname}() missing {count} required {kind} argument{}: '{}{}{right}'", + if count == 1 { "" } else { "s" }, + missing.iter().join("', '"), + and, + ) +} + #[pyclass(module = false, name = "function", traverse = "manual")] #[derive(Debug)] pub struct PyFunction { @@ -36,7 +67,8 @@ pub struct PyFunction { name: PyMutex<PyStrRef>, qualname: PyMutex<PyStrRef>, type_params: PyMutex<PyTupleRef>, - annotations: PyMutex<PyDictRef>, + annotations: PyMutex<Option<PyDictRef>>, + annotate: PyMutex<Option<PyObjectRef>>, module: PyMutex<PyObjectRef>, doc: PyMutex<PyObjectRef>, #[cfg(feature = "jit")] @@ -50,6 +82,71 @@ unsafe impl Traverse for PyFunction { closure.as_untyped().traverse(tracer_fn); } self.defaults_and_kwdefaults.traverse(tracer_fn); + // Traverse additional fields that may contain references + self.type_params.lock().traverse(tracer_fn); + self.annotations.lock().traverse(tracer_fn); + self.module.lock().traverse(tracer_fn); + self.doc.lock().traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec<crate::PyObjectRef>) { + // Pop closure if present (equivalent to Py_CLEAR(func_closure)) + if let Some(closure) = self.closure.take() { + out.push(closure.into()); + } + + // Pop defaults and kwdefaults + if let Some(mut guard) = self.defaults_and_kwdefaults.try_lock() { + if let Some(defaults) = guard.0.take() { + out.push(defaults.into()); + } + if let Some(kwdefaults) = guard.1.take() { + out.push(kwdefaults.into()); + } + } + + // Clear annotations and annotate (Py_CLEAR) + if let Some(mut guard) = self.annotations.try_lock() + && let Some(annotations) = guard.take() + { + out.push(annotations.into()); + } + if let Some(mut guard) = self.annotate.try_lock() + && let Some(annotate) = guard.take() + { + out.push(annotate); + } + + // Clear module, doc, and type_params (Py_CLEAR) + if let Some(mut guard) = self.module.try_lock() { + let old_module = + core::mem::replace(&mut *guard, Context::genesis().none.to_owned().into()); + out.push(old_module); + } + if let Some(mut guard) = self.doc.try_lock() { + let old_doc = + core::mem::replace(&mut *guard, Context::genesis().none.to_owned().into()); + out.push(old_doc); + } + if let Some(mut guard) = self.type_params.try_lock() { + let old_type_params = + core::mem::replace(&mut *guard, Context::genesis().empty_tuple.to_owned()); + out.push(old_type_params.into()); + } + + // Replace name and qualname with empty string to break potential str subclass cycles + // name and qualname could be str subclasses, so they could have reference cycles + if let Some(mut guard) = self.name.try_lock() { + let old_name = core::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_name.into()); + } + if let Some(mut guard) = self.qualname.try_lock() { + let old_qualname = + core::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_qualname.into()); + } + + // Note: globals, builtins, code are NOT cleared (required to be non-NULL) } } @@ -65,11 +162,28 @@ impl PyFunction { let builtins = globals.get_item("__builtins__", vm).unwrap_or_else(|_| { // If not in globals, inherit from current execution context if let Some(frame) = vm.current_frame() { - frame.builtins.clone().into() + frame.builtins.clone() } else { - vm.builtins.clone().into() + vm.builtins.dict().into() } }); + // If builtins is a module, use its __dict__ instead + let builtins = if let Some(module) = builtins.downcast_ref::<PyModule>() { + module.dict().into() + } else { + builtins + }; + + // Get docstring from co_consts[0] if HAS_DOCSTRING flag is set + let doc = if code.code.flags.contains(bytecode::CodeFlags::HAS_DOCSTRING) { + code.code + .constants + .first() + .map(|c| c.as_object().to_owned()) + .unwrap_or_else(|| vm.ctx.none()) + } else { + vm.ctx.none() + }; let qualname = vm.ctx.new_str(code.qualname.as_str()); let func = Self { @@ -81,9 +195,10 @@ impl PyFunction { name, qualname: PyMutex::new(qualname), type_params: PyMutex::new(vm.ctx.empty_tuple.clone()), - annotations: PyMutex::new(vm.ctx.new_dict()), + annotations: PyMutex::new(None), + annotate: PyMutex::new(None), module: PyMutex::new(module), - doc: PyMutex::new(vm.ctx.none()), + doc: PyMutex::new(doc), #[cfg(feature = "jit")] jitted_code: OnceCell::new(), }; @@ -124,24 +239,62 @@ impl PyFunction { let mut vararg_offset = total_args; // Pack other positional arguments in to *args: - if code.flags.contains(bytecode::CodeFlags::HAS_VARARGS) { + if code.flags.contains(bytecode::CodeFlags::VARARGS) { let vararg_value = vm.ctx.new_tuple(args_iter.collect()); fastlocals[vararg_offset] = Some(vararg_value.into()); vararg_offset += 1; } else { // Check the number of positional arguments if nargs > n_expected_args { + let n_defaults = self + .defaults_and_kwdefaults + .lock() + .0 + .as_ref() + .map_or(0, |d| d.len()); + let n_required = n_expected_args - n_defaults; + let takes_msg = if n_defaults > 0 { + format!("from {} to {}", n_required, n_expected_args) + } else { + n_expected_args.to_string() + }; + + // Count keyword-only arguments that were actually provided + let kw_only_given = if code.kwonlyarg_count > 0 { + let start = code.arg_count as usize; + let end = start + code.kwonlyarg_count as usize; + code.varnames[start..end] + .iter() + .filter(|name| func_args.kwargs.contains_key(name.as_str())) + .count() + } else { + 0 + }; + + let given_msg = if kw_only_given > 0 { + format!( + "{} positional argument{} (and {} keyword-only argument{}) were", + nargs, + if nargs == 1 { "" } else { "s" }, + kw_only_given, + if kw_only_given == 1 { "" } else { "s" }, + ) + } else { + format!("{} {}", nargs, if nargs == 1 { "was" } else { "were" }) + }; + return Err(vm.new_type_error(format!( - "{}() takes {} positional arguments but {} were given", + "{}() takes {} positional argument{} but {} given", self.__qualname__(), - n_expected_args, - nargs + takes_msg, + if n_expected_args == 1 { "" } else { "s" }, + given_msg, ))); } } // Do we support `**kwargs` ? - let kwargs = if code.flags.contains(bytecode::CodeFlags::HAS_VARKEYWORDS) { + let kwargs = if code.flags.contains(bytecode::CodeFlags::VARKEYWORDS) { let d = vm.ctx.new_dict(); fastlocals[vararg_offset] = Some(d.clone().into()); Some(d) @@ -149,7 +302,7 @@ impl PyFunction { None }; - let arg_pos = |range: std::ops::Range<_>, name: &str| { + let arg_pos = |range: core::ops::Range<_>, name: &str| { code.varnames .iter() .enumerate() @@ -221,41 +374,17 @@ impl PyFunction { } }) .collect(); - let missing_args_len = missing.len(); if !missing.is_empty() { - let last = if missing.len() > 1 { - missing.pop() - } else { - None - }; - - let (and, right) = if let Some(last) = last { - ( - if missing.len() == 1 { - "' and '" - } else { - "', and '" - }, - last.as_str(), - ) - } else { - ("", "") - }; - - return Err(vm.new_type_error(format!( - "{}() missing {} required positional argument{}: '{}{}{}'", + return Err(vm.new_type_error(format_missing_args( self.__qualname__(), - missing_args_len, - if missing_args_len == 1 { "" } else { "s" }, - missing.iter().join("', '"), - and, - right, + "positional", + &mut missing, ))); } if let Some(defaults) = defaults { - let n = std::cmp::min(nargs, n_expected_args); + let n = core::cmp::min(nargs, n_expected_args); let i = n.saturating_sub(n_required); // We have sufficient defaults, so iterate over the corresponding names and use @@ -270,8 +399,7 @@ impl PyFunction { }; if code.kwonlyarg_count > 0 { - // TODO: compile a list of missing arguments - // let mut missing = vec![]; + let mut missing = Vec::new(); // Check if kw only arguments are all present: for (slot, kwarg) in fastlocals .iter_mut() @@ -288,9 +416,15 @@ impl PyFunction { } // No default value and not specified. - return Err( - vm.new_type_error(format!("Missing required kw only argument: '{kwarg}'")) - ); + missing.push(kwarg); + } + + if !missing.is_empty() { + return Err(vm.new_type_error(format_missing_args( + self.__qualname__(), + "keyword-only", + &mut missing, + ))); } } @@ -344,7 +478,7 @@ impl PyFunction { ))); } }; - *self.annotations.lock() = annotations; + *self.annotations.lock() = Some(annotations); } else if attr == bytecode::MakeFunctionFlags::CLOSURE { // For closure, we need special handling // The closure tuple contains cell objects @@ -368,6 +502,12 @@ impl PyFunction { )) })?; *self.type_params.lock() = type_params; + } else if attr == bytecode::MakeFunctionFlags::ANNOTATE { + // PEP 649: Store the __annotate__ function closure + if !attr_value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable".to_owned())); + } + *self.annotate.lock() = Some(attr_value); } else { unreachable!("This is a compiler bug"); } @@ -400,7 +540,7 @@ impl Py<PyFunction> { let code = self.code.lock().clone(); - let locals = if code.flags.contains(bytecode::CodeFlags::NEW_LOCALS) { + let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { ArgMapping::from_dict_exact(vm.ctx.new_dict()) } else if let Some(locals) = locals { locals @@ -412,7 +552,7 @@ impl Py<PyFunction> { let frame = Frame::new( code.clone(), Scope::new(Some(locals), self.globals.clone()), - vm.builtins.dict(), + self.builtins.clone(), self.closure.as_ref().map_or(&[], |c| c.as_slice()), Some(self.to_owned().into()), vm, @@ -422,17 +562,26 @@ impl Py<PyFunction> { self.fill_locals_from_args(&frame, func_args, vm)?; // If we have a generator, create a new generator - let is_gen = code.flags.contains(bytecode::CodeFlags::IS_GENERATOR); - let is_coro = code.flags.contains(bytecode::CodeFlags::IS_COROUTINE); + let is_gen = code.flags.contains(bytecode::CodeFlags::GENERATOR); + let is_coro = code.flags.contains(bytecode::CodeFlags::COROUTINE); match (is_gen, is_coro) { (true, false) => { - Ok(PyGenerator::new(frame, self.__name__(), self.__qualname__()).into_pyobject(vm)) + let obj = PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) } (false, true) => { - Ok(PyCoroutine::new(frame, self.__name__(), self.__qualname__()).into_pyobject(vm)) + let obj = PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) } (true, true) => { - Ok(PyAsyncGen::new(frame, self.__name__(), self.__qualname__()).into_pyobject(vm)) + let obj = PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) } (false, false) => vm.run_frame(frame), } @@ -554,13 +703,107 @@ impl PyFunction { } #[pygetset] - fn __annotations__(&self) -> PyDictRef { - self.annotations.lock().clone() + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult<PyDictRef> { + // First check if we have cached annotations + { + let annotations = self.annotations.lock(); + if let Some(ref ann) = *annotations { + return Ok(ann.clone()); + } + } + + // Check for callable __annotate__ and clone it before calling + let annotate_fn = { + let annotate = self.annotate.lock(); + if let Some(ref func) = *annotate + && func.is_callable() + { + Some(func.clone()) + } else { + None + } + }; + + // Release locks before calling __annotate__ to avoid deadlock + if let Some(annotate_fn) = annotate_fn { + let one = vm.ctx.new_int(1); + let ann_dict = annotate_fn.call((one,), vm)?; + let ann_dict = ann_dict + .downcast::<crate::builtins::PyDict>() + .map_err(|obj| { + vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + obj.class().name() + )) + })?; + + // Cache the result + *self.annotations.lock() = Some(ann_dict.clone()); + return Ok(ann_dict); + } + + // No __annotate__ or not callable, create empty dict + let new_dict = vm.ctx.new_dict(); + *self.annotations.lock() = Some(new_dict.clone()); + Ok(new_dict) } #[pygetset(setter)] - fn set___annotations__(&self, annotations: PyDictRef) { - *self.annotations.lock() = annotations + fn set___annotations__( + &self, + value: PySetterValue<Option<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(Some(value)) => { + let annotations = value.downcast::<crate::builtins::PyDict>().map_err(|_| { + vm.new_type_error("__annotations__ must be set to a dict object") + })?; + *self.annotations.lock() = Some(annotations); + *self.annotate.lock() = None; + } + PySetterValue::Assign(None) => { + *self.annotations.lock() = None; + *self.annotate.lock() = None; + } + PySetterValue::Delete => { + // del only clears cached annotations; __annotate__ is preserved + *self.annotations.lock() = None; + } + } + Ok(()) + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.annotate + .lock() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set___annotate__( + &self, + value: PySetterValue<Option<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let annotate = match value { + PySetterValue::Assign(Some(value)) => { + if !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + // Clear cached __annotations__ when __annotate__ is set + *self.annotations.lock() = None; + Some(value) + } + PySetterValue::Assign(None) => None, + PySetterValue::Delete => { + return Err(vm.new_type_error("__annotate__ cannot be deleted")); + } + }; + *self.annotate.lock() = annotate; + Ok(()) } #[pygetset] @@ -609,15 +852,16 @@ impl PyFunction { #[cfg(feature = "jit")] #[pymethod] fn __jit__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { - zelf.jitted_code - .get_or_try_init(|| { - let arg_types = jit::get_jit_arg_types(&zelf, vm)?; - let ret_type = jit::jit_ret_type(&zelf, vm)?; - let code = zelf.code.lock(); - rustpython_jit::compile(&code.code, &arg_types, ret_type) - .map_err(|err| jit::new_jit_error(err.to_string(), vm)) - }) - .map(drop) + if zelf.jitted_code.get().is_some() { + return Ok(()); + } + let arg_types = jit::get_jit_arg_types(&zelf, vm)?; + let ret_type = jit::jit_ret_type(&zelf, vm)?; + let code = zelf.code.lock(); + let compiled = rustpython_jit::compile(&code.code, &arg_types, ret_type) + .map_err(|err| jit::new_jit_error(err.to_string(), vm))?; + let _ = zelf.jitted_code.set(compiled); + Ok(()) } } @@ -629,12 +873,11 @@ impl GetDescriptor for PyFunction { vm: &VirtualMachine, ) -> PyResult { let (_zelf, obj) = Self::_unwrap(&zelf, obj, vm)?; - let obj = if vm.is_none(&obj) && !Self::_cls_is(&cls, obj.class()) { + Ok(if vm.is_none(&obj) && !Self::_cls_is(&cls, obj.class()) { zelf } else { PyBoundMethod::new(obj, zelf).into_ref(&vm.ctx).into() - }; - Ok(obj) + }) } } @@ -663,14 +906,14 @@ pub struct PyFunctionNewArgs { code: PyRef<PyCode>, #[pyarg(positional)] globals: PyDictRef, - #[pyarg(any, optional)] + #[pyarg(any, optional, error_msg = "arg 3 (name) must be None or string")] name: OptionalArg<PyStrRef>, - #[pyarg(any, optional)] - defaults: OptionalArg<PyTupleRef>, - #[pyarg(any, optional)] - closure: OptionalArg<PyTupleRef>, - #[pyarg(any, optional)] - kwdefaults: OptionalArg<PyDictRef>, + #[pyarg(any, optional, error_msg = "arg 4 (defaults) must be None or tuple")] + argdefs: Option<PyTupleRef>, + #[pyarg(any, optional, error_msg = "arg 5 (closure) must be None or tuple")] + closure: Option<PyTupleRef>, + #[pyarg(any, optional, error_msg = "arg 6 (kwdefaults) must be None or dict")] + kwdefaults: Option<PyDictRef>, } impl Constructor for PyFunction { @@ -678,7 +921,7 @@ impl Constructor for PyFunction { fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { // Handle closure - must be a tuple of cells - let closure = if let Some(closure_tuple) = args.closure.into_option() { + let closure = if let Some(closure_tuple) = args.closure { // Check that closure length matches code's free variables if closure_tuple.len() != args.code.freevars.len() { return Err(vm.new_value_error(format!( @@ -709,10 +952,10 @@ impl Constructor for PyFunction { if let Some(closure_tuple) = closure { func.closure = Some(closure_tuple); } - if let Some(defaults) = args.defaults.into_option() { - func.defaults_and_kwdefaults.lock().0 = Some(defaults); + if let Some(argdefs) = args.argdefs { + func.defaults_and_kwdefaults.lock().0 = Some(argdefs); } - if let Some(kwdefaults) = args.kwdefaults.into_option() { + if let Some(kwdefaults) = args.kwdefaults { func.defaults_and_kwdefaults.lock().1 = Some(kwdefaults); } @@ -865,7 +1108,6 @@ impl PyPayload for PyBoundMethod { impl Representable for PyBoundMethod { #[inline] fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - #[allow(clippy::needless_match)] // False positive on nightly let func_name = if let Some(qname) = vm.get_attribute_opt(zelf.function.clone(), "__qualname__")? { Some(qname) diff --git a/crates/vm/src/builtins/function/jit.rs b/crates/vm/src/builtins/function/jit.rs index 21d8c9c0abf..a28335900da 100644 --- a/crates/vm/src/builtins/function/jit.rs +++ b/crates/vm/src/builtins/function/jit.rs @@ -1,6 +1,8 @@ use crate::{ AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyDictRef, PyFunction, PyStrInterned, bool_, float, int}, + builtins::{ + PyBaseExceptionRef, PyDict, PyDictRef, PyFunction, PyStrInterned, bool_, float, int, + }, bytecode::CodeFlags, convert::ToPyObject, function::FuncArgs, @@ -42,7 +44,7 @@ pub fn new_jit_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_exception_msg(jit_error, msg) } -fn get_jit_arg_type(dict: &PyDictRef, name: &str, vm: &VirtualMachine) -> PyResult<JitType> { +fn get_jit_arg_type(dict: &Py<PyDict>, name: &str, vm: &VirtualMachine) -> PyResult<JitType> { if let Some(value) = dict.get_item_opt(name, vm)? { if value.is(vm.ctx.types.int_type) { Ok(JitType::Int) @@ -70,7 +72,7 @@ pub fn get_jit_arg_types(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult if code .flags - .intersects(CodeFlags::HAS_VARARGS | CodeFlags::HAS_VARKEYWORDS) + .intersects(CodeFlags::VARARGS | CodeFlags::VARKEYWORDS) { return Err(new_jit_error( "Can't jit functions with variable number of arguments".to_owned(), diff --git a/crates/vm/src/builtins/generator.rs b/crates/vm/src/builtins/generator.rs index da981b5a6c2..cdbb2af440a 100644 --- a/crates/vm/src/builtins/generator.rs +++ b/crates/vm/src/builtins/generator.rs @@ -6,19 +6,26 @@ use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, - coroutine::Coro, + coroutine::{Coro, warn_deprecated_throw_signature}, frame::FrameRef, function::OptionalArg, + object::{Traverse, TraverseFn}, protocol::PyIterReturn, - types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, }; -#[pyclass(module = false, name = "generator")] +#[pyclass(module = false, name = "generator", traverse = "manual")] #[derive(Debug)] pub struct PyGenerator { inner: Coro, } +unsafe impl Traverse for PyGenerator { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + } +} + impl PyPayload for PyGenerator { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -26,7 +33,10 @@ impl PyPayload for PyGenerator { } } -#[pyclass(with(Py, Unconstructible, IterNext, Iterable))] +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with(Py, IterNext, Iterable, Representable, Destructor) +)] impl PyGenerator { pub const fn as_coro(&self) -> &Coro { &self.inner @@ -59,8 +69,12 @@ impl PyGenerator { } #[pygetset] - fn gi_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() + fn gi_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } } #[pygetset] @@ -78,6 +92,11 @@ impl PyGenerator { self.inner.frame().yield_from_target() } + #[pygetset] + fn gi_suspended(&self, _vm: &VirtualMachine) -> bool { + self.inner.suspended() + } + #[pyclassmethod] fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) @@ -99,6 +118,7 @@ impl Py<PyGenerator> { exc_tb: OptionalArg, vm: &VirtualMachine, ) -> PyResult<PyIterReturn> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; self.inner.throw( self.as_object(), exc_type, @@ -109,13 +129,11 @@ impl Py<PyGenerator> { } #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { self.inner.close(self.as_object(), vm) } } -impl Unconstructible for PyGenerator {} - impl Representable for PyGenerator { #[inline] fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { @@ -130,6 +148,31 @@ impl IterNext for PyGenerator { } } +impl Destructor for PyGenerator { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // _PyGen_Finalize: close the generator if it's still suspended + if zelf.inner.closed() || zelf.inner.running() { + return Ok(()); + } + // Generator was never started, just mark as closed + if zelf.inner.frame().lasti() == 0 { + zelf.inner.closed.store(true); + return Ok(()); + } + // Throw GeneratorExit to run finally blocks + if let Err(e) = zelf.inner.close(zelf.as_object(), vm) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + Ok(()) + } +} + +impl Drop for PyGenerator { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + pub fn init(ctx: &Context) { PyGenerator::extend_class(ctx, ctx.types.generator_type); } diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index 494b580e563..8aabca1ae10 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -1,27 +1,26 @@ -// spell-checker:ignore iparam -use std::sync::LazyLock; +// spell-checker:ignore iparam gaiterobject +use crate::common::lock::LazyLock; use super::type_; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, atomic_func, - builtins::{PyList, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef}, + builtins::{PyList, PyStr, PyTuple, PyTupleRef, PyType}, class::PyClassImpl, common::hash, convert::ToPyObject, function::{FuncArgs, PyComparisonValue}, protocol::{PyMappingMethods, PyNumberMethods}, types::{ - AsMapping, AsNumber, Callable, Comparable, Constructor, GetAttr, Hashable, Iterable, - PyComparisonOp, Representable, + AsMapping, AsNumber, Callable, Comparable, Constructor, GetAttr, Hashable, IterNext, + Iterable, PyComparisonOp, Representable, }, }; -use std::fmt; +use alloc::fmt; -// attr_exceptions -static ATTR_EXCEPTIONS: [&str; 12] = [ +// Attributes that are looked up on the GenericAlias itself, not on __origin__ +static ATTR_EXCEPTIONS: [&str; 9] = [ "__class__", - "__bases__", "__origin__", "__args__", "__unpacked__", @@ -30,13 +29,14 @@ static ATTR_EXCEPTIONS: [&str; 12] = [ "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", - "__copy__", - "__deepcopy__", ]; +// Attributes that are blocked from being looked up on __origin__ +static ATTR_BLOCKED: [&str; 3] = ["__bases__", "__copy__", "__deepcopy__"]; + #[pyclass(module = "types", name = "GenericAlias")] pub struct PyGenericAlias { - origin: PyTypeRef, + origin: PyObjectRef, args: PyTupleRef, parameters: PyTupleRef, starred: bool, // for __unpacked__ attribute @@ -62,7 +62,7 @@ impl Constructor for PyGenericAlias { if !args.kwargs.is_empty() { return Err(vm.new_type_error("GenericAlias() takes no keyword arguments")); } - let (origin, arguments): (_, PyObjectRef) = args.bind(vm)?; + let (origin, arguments): (PyObjectRef, PyObjectRef) = args.bind(vm)?; let args = if let Ok(tuple) = arguments.try_to_ref::<PyTuple>(vm) { tuple.to_owned() } else { @@ -87,10 +87,15 @@ impl Constructor for PyGenericAlias { flags(BASETYPE) )] impl PyGenericAlias { - pub fn new(origin: PyTypeRef, args: PyTupleRef, starred: bool, vm: &VirtualMachine) -> Self { + pub fn new( + origin: impl Into<PyObjectRef>, + args: PyTupleRef, + starred: bool, + vm: &VirtualMachine, + ) -> Self { let parameters = make_parameters(&args, vm); Self { - origin, + origin: origin.into(), args, parameters, starred, @@ -98,7 +103,11 @@ impl PyGenericAlias { } /// Create a GenericAlias from an origin and PyObjectRef arguments (helper for compatibility) - pub fn from_args(origin: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> Self { + pub fn from_args( + origin: impl Into<PyObjectRef>, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> Self { let args = if let Ok(tuple) = args.try_to_ref::<PyTuple>(vm) { tuple.to_owned() } else { @@ -138,15 +147,35 @@ impl PyGenericAlias { } } + fn repr_arg(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + // ParamSpec args can be lists - format their items with repr_item + if obj.class().is(vm.ctx.types.list_type) { + let list = obj.downcast_ref::<crate::builtins::PyList>().unwrap(); + let len = list.borrow_vec().len(); + let mut parts = Vec::with_capacity(len); + // Use indexed access so list mutation during repr causes IndexError + for i in 0..len { + let item = + list.borrow_vec().get(i).cloned().ok_or_else(|| { + vm.new_index_error("list index out of range".to_owned()) + })?; + parts.push(repr_item(item, vm)?); + } + Ok(format!("[{}]", parts.join(", "))) + } else { + repr_item(obj, vm) + } + } + let repr_str = format!( "{}[{}]", - repr_item(self.origin.clone().into(), vm)?, + repr_item(self.origin.clone(), vm)?, if self.args.is_empty() { "()".to_owned() } else { self.args .iter() - .map(|o| repr_item(o.clone(), vm)) + .map(|o| repr_arg(o.clone(), vm)) .collect::<PyResult<Vec<_>>>()? .join(", ") } @@ -172,7 +201,7 @@ impl PyGenericAlias { #[pygetset] fn __origin__(&self) -> PyObjectRef { - self.origin.clone().into() + self.origin.clone() } #[pygetset] @@ -182,14 +211,13 @@ impl PyGenericAlias { #[pygetset] fn __typing_unpacked_tuple_args__(&self, vm: &VirtualMachine) -> PyObjectRef { - if self.starred && self.origin.is(vm.ctx.types.tuple_type) { + if self.starred && self.origin.is(vm.ctx.types.tuple_type.as_object()) { self.args.clone().into() } else { vm.ctx.none() } } - #[pymethod] fn __getitem__(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let new_args = subs_parameters( zelf.to_owned().into(), @@ -214,11 +242,29 @@ impl PyGenericAlias { } #[pymethod] - fn __reduce__(zelf: &Py<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyTypeRef, PyTupleRef)) { - ( - vm.ctx.types.generic_alias_type.to_owned(), - (zelf.origin.clone(), zelf.args.clone()), - ) + fn __reduce__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + if zelf.starred { + // (next, (iter(GenericAlias(origin, args)),)) + let next_fn = vm.builtins.get_attr("next", vm)?; + let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm); + let iter_obj = PyGenericAliasIterator { + obj: crate::common::lock::PyMutex::new(Some(non_starred.into_pyobject(vm))), + } + .into_pyobject(vm); + Ok(PyTuple::new_ref( + vec![next_fn, PyTuple::new_ref(vec![iter_obj], &vm.ctx).into()], + &vm.ctx, + )) + } else { + Ok(PyTuple::new_ref( + vec![ + vm.ctx.types.generic_alias_type.to_owned().into(), + PyTuple::new_ref(vec![zelf.origin.clone(), zelf.args.clone().into()], &vm.ctx) + .into(), + ], + &vm.ctx, + )) + } } #[pymethod] @@ -236,20 +282,21 @@ impl PyGenericAlias { Err(vm.new_type_error("issubclass() argument 2 cannot be a parameterized generic")) } - #[pymethod] - fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { type_::or_(other, zelf, vm) } - #[pymethod] - fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { type_::or_(zelf, other, vm) } } pub(crate) fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { + make_parameters_from_slice(args.as_slice(), vm) +} + +fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTupleRef { let mut parameters: Vec<PyObjectRef> = Vec::with_capacity(args.len()); - let mut iparam = 0; for arg in args { // We don't want __parameters__ descriptor of a bare Python class. @@ -259,46 +306,43 @@ pub(crate) fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupl // Check for __typing_subst__ attribute if arg.get_attr(identifier!(vm, __typing_subst__), vm).is_ok() { - // Use tuple_add equivalent logic if tuple_index(&parameters, arg).is_none() { - if iparam >= parameters.len() { - parameters.resize(iparam + 1, vm.ctx.none()); - } - parameters[iparam] = arg.clone(); - iparam += 1; + parameters.push(arg.clone()); } } else if let Ok(subparams) = arg.get_attr(identifier!(vm, __parameters__), vm) && let Ok(sub_params) = subparams.try_to_ref::<PyTuple>(vm) { - let len2 = sub_params.len(); - // Resize if needed - if iparam + len2 > parameters.len() { - parameters.resize(iparam + len2, vm.ctx.none()); - } for sub_param in sub_params { - // Use tuple_add equivalent logic - if tuple_index(&parameters[..iparam], sub_param).is_none() { - if iparam >= parameters.len() { - parameters.resize(iparam + 1, vm.ctx.none()); - } - parameters[iparam] = sub_param.clone(); - iparam += 1; + if tuple_index(&parameters, sub_param).is_none() { + parameters.push(sub_param.clone()); + } + } + } else if arg.try_to_ref::<PyTuple>(vm).is_ok() || arg.try_to_ref::<PyList>(vm).is_ok() { + // Recursively extract parameters from lists/tuples (ParamSpec args) + let items: Vec<PyObjectRef> = if let Ok(t) = arg.try_to_ref::<PyTuple>(vm) { + t.as_slice().to_vec() + } else { + let list = arg.downcast_ref::<PyList>().unwrap(); + list.borrow_vec().to_vec() + }; + let sub = make_parameters_from_slice(&items, vm); + for sub_param in sub.iter() { + if tuple_index(&parameters, sub_param).is_none() { + parameters.push(sub_param.clone()); } } } } - // Resize to actual size - parameters.truncate(iparam); PyTuple::new_ref(parameters, &vm.ctx) } #[inline] -fn tuple_index(vec: &[PyObjectRef], item: &PyObjectRef) -> Option<usize> { +fn tuple_index(vec: &[PyObjectRef], item: &PyObject) -> Option<usize> { vec.iter().position(|element| element.is(item)) } -fn is_unpacked_typevartuple(arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { +fn is_unpacked_typevartuple(arg: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { if arg.class().is(vm.ctx.types.type_type) { return Ok(false); } @@ -312,7 +356,7 @@ fn is_unpacked_typevartuple(arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult< fn subs_tvars( obj: PyObjectRef, - params: &PyTupleRef, + params: &Py<PyTuple>, arg_items: &[PyObjectRef], vm: &VirtualMachine, ) -> PyResult { @@ -436,7 +480,7 @@ pub fn subs_parameters( let arg_items = if let Ok(tuple) = item.try_to_ref::<PyTuple>(vm) { tuple.as_slice().to_vec() } else { - vec![item] + vec![item.clone()] }; let n_items = arg_items.len(); @@ -460,32 +504,55 @@ pub fn subs_parameters( continue; } - // Check if this is an unpacked TypeVarTuple's _is_unpacked_typevartuple + // Recursively substitute params in lists/tuples + let is_list = arg.try_to_ref::<PyList>(vm).is_ok(); + if arg.try_to_ref::<PyTuple>(vm).is_ok() || is_list { + let sub_items: Vec<PyObjectRef> = if let Ok(t) = arg.try_to_ref::<PyTuple>(vm) { + t.as_slice().to_vec() + } else { + arg.downcast_ref::<PyList>().unwrap().borrow_vec().to_vec() + }; + let sub_tuple = PyTuple::new_ref(sub_items, &vm.ctx); + let sub_result = subs_parameters( + alias.clone(), + sub_tuple, + parameters.clone(), + item.clone(), + vm, + )?; + let substituted: PyObjectRef = if is_list { + // Convert tuple back to list + PyList::from(sub_result.as_slice().to_vec()) + .into_ref(&vm.ctx) + .into() + } else { + sub_result.into() + }; + new_args.push(substituted); + continue; + } + + // Check if this is an unpacked TypeVarTuple let unpack = is_unpacked_typevartuple(arg, vm)?; - // Try __typing_subst__ method first, + // Try __typing_subst__ method first let substituted_arg = if let Ok(subst) = arg.get_attr(identifier!(vm, __typing_subst__), vm) { - // Find parameter index's tuple_index if let Some(iparam) = tuple_index(parameters.as_slice(), arg) { subst.call((arg_items[iparam].clone(),), vm)? } else { - // This shouldn't happen in well-formed generics but handle gracefully subs_tvars(arg.clone(), &parameters, &arg_items, vm)? } } else { - // Use subs_tvars for objects with __parameters__ subs_tvars(arg.clone(), &parameters, &arg_items, vm)? }; if unpack { - // Handle unpacked TypeVarTuple's tuple_extend if let Ok(tuple) = substituted_arg.try_to_ref::<PyTuple>(vm) { for elem in tuple { new_args.push(elem.clone()); } } else { - // This shouldn't happen but handle gracefully new_args.push(substituted_arg); } } else { @@ -512,7 +579,7 @@ impl AsMapping for PyGenericAlias { impl AsNumber for PyGenericAlias { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| Ok(PyGenericAlias::__or__(a.to_owned(), b.to_owned(), vm))), + or: Some(|a, b, vm| PyGenericAlias::__or__(a.to_owned(), b.to_owned(), vm)), ..PyNumberMethods::NOT_IMPLEMENTED }; &AS_NUMBER @@ -522,7 +589,7 @@ impl AsNumber for PyGenericAlias { impl Callable for PyGenericAlias { type Args = FuncArgs; fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyType::call(&zelf.origin, args, vm).map(|obj| { + zelf.origin.call(args, vm).map(|obj| { if let Err(exc) = obj.set_attr(identifier!(vm, __orig_class__), zelf.to_owned(), vm) && !exc.fast_isinstance(vm.ctx.exceptions.attribute_error) && !exc.fast_isinstance(vm.ctx.exceptions.type_error) @@ -543,17 +610,17 @@ impl Comparable for PyGenericAlias { ) -> PyResult<PyComparisonValue> { op.eq_only(|| { let other = class_or_notimplemented!(Self, other); + if zelf.starred != other.starred { + return Ok(PyComparisonValue::Implemented(false)); + } Ok(PyComparisonValue::Implemented( - if !zelf.__origin__().rich_compare_bool( - &other.__origin__(), - PyComparisonOp::Eq, - vm, - )? { - false - } else { - zelf.__args__() - .rich_compare_bool(&other.__args__(), PyComparisonOp::Eq, vm)? - }, + zelf.__origin__() + .rich_compare_bool(&other.__origin__(), PyComparisonOp::Eq, vm)? + && zelf.__args__().rich_compare_bool( + &other.__args__(), + PyComparisonOp::Eq, + vm, + )?, )) }) } @@ -562,14 +629,20 @@ impl Comparable for PyGenericAlias { impl Hashable for PyGenericAlias { #[inline] fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { - Ok(zelf.origin.as_object().hash(vm)? ^ zelf.args.as_object().hash(vm)?) + Ok(zelf.origin.hash(vm)? ^ zelf.args.as_object().hash(vm)?) } } impl GetAttr for PyGenericAlias { fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let attr_str = attr.as_str(); for exc in &ATTR_EXCEPTIONS { - if *(*exc) == attr.to_string() { + if *exc == attr_str { + return zelf.as_object().generic_getattr(attr, vm); + } + } + for blocked in &ATTR_BLOCKED { + if *blocked == attr_str { return zelf.as_object().generic_getattr(attr, vm); } } @@ -585,52 +658,88 @@ impl Representable for PyGenericAlias { } impl Iterable for PyGenericAlias { - // ga_iter - // spell-checker:ignore gaiterobject - // TODO: gaiterobject fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - // CPython's ga_iter creates an iterator that yields one starred GenericAlias - // we don't have gaiterobject yet + Ok(PyGenericAliasIterator { + obj: crate::common::lock::PyMutex::new(Some(zelf.into())), + } + .into_pyobject(vm)) + } +} - let starred_alias = Self::new( - zelf.origin.clone(), - zelf.args.clone(), - true, // starred - vm, - ); - let starred_ref = PyRef::new_ref( - starred_alias, - vm.ctx.types.generic_alias_type.to_owned(), - None, - ); - let items = vec![starred_ref.into()]; - let iter_tuple = PyTuple::new_ref(items, &vm.ctx); - Ok(iter_tuple.to_pyobject(vm).get_iter(vm)?.into()) +// gaiterobject - yields one starred GenericAlias then exhausts +#[pyclass(module = "types", name = "generic_alias_iterator")] +#[derive(Debug, PyPayload)] +pub struct PyGenericAliasIterator { + obj: crate::common::lock::PyMutex<Option<PyObjectRef>>, +} + +#[pyclass(with(Representable, Iterable, IterNext))] +impl PyGenericAliasIterator { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let iter_fn = vm.builtins.get_attr("iter", vm)?; + let guard = self.obj.lock(); + let arg: PyObjectRef = if let Some(ref obj) = *guard { + // Not yet exhausted: (iter, (obj,)) + PyTuple::new_ref(vec![obj.clone()], &vm.ctx).into() + } else { + // Exhausted: (iter, ((),)) + let empty = PyTuple::new_ref(vec![], &vm.ctx); + PyTuple::new_ref(vec![empty.into()], &vm.ctx).into() + }; + Ok(PyTuple::new_ref(vec![iter_fn, arg], &vm.ctx)) + } +} + +impl Representable for PyGenericAliasIterator { + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<generic_alias_iterator>".to_owned()) } } -/// Creates a GenericAlias from type parameters, equivalent to CPython's _Py_subscript_generic -/// This is used for PEP 695 classes to create Generic[T] from type parameters +impl Iterable for PyGenericAliasIterator { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(zelf.into()) + } +} + +impl crate::types::IterNext for PyGenericAliasIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<crate::protocol::PyIterReturn> { + use crate::protocol::PyIterReturn; + let mut guard = zelf.obj.lock(); + let obj = match guard.take() { + Some(obj) => obj, + None => return Ok(PyIterReturn::StopIteration(None)), + }; + // Create a starred GenericAlias from the original + let alias = obj.downcast_ref::<PyGenericAlias>().ok_or_else(|| { + vm.new_type_error("generic_alias_iterator expected GenericAlias".to_owned()) + })?; + let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm); + Ok(PyIterReturn::Return(starred.into_pyobject(vm))) + } +} + +/// Creates a GenericAlias from type parameters, equivalent to _Py_subscript_generic. +/// This is used for PEP 695 classes to create Generic[T] from type parameters. // _Py_subscript_generic pub fn subscript_generic(type_params: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Get typing module and _GenericAlias let typing_module = vm.import("typing", 0)?; let generic_type = typing_module.get_attr("Generic", vm)?; - - // Call typing._GenericAlias(Generic, type_params) let generic_alias_class = typing_module.get_attr("_GenericAlias", vm)?; - let args = if let Ok(tuple) = type_params.try_to_ref::<PyTuple>(vm) { + let params = if let Ok(tuple) = type_params.try_to_ref::<PyTuple>(vm) { tuple.to_owned() } else { PyTuple::new_ref(vec![type_params], &vm.ctx) }; - // Create _GenericAlias instance + let args = crate::stdlib::typing::unpack_typevartuples(&params, vm)?; + generic_alias_class.call((generic_type, args.to_pyobject(vm)), vm) } pub fn init(context: &Context) { - let generic_alias_type = &context.types.generic_alias_type; - PyGenericAlias::extend_class(context, generic_alias_type); + PyGenericAlias::extend_class(context, context.types.generic_alias_type); + PyGenericAliasIterator::extend_class(context, context.types.generic_alias_iterator_type); } diff --git a/crates/vm/src/builtins/getset.rs b/crates/vm/src/builtins/getset.rs index 4b966bbc31b..86f0524b12c 100644 --- a/crates/vm/src/builtins/getset.rs +++ b/crates/vm/src/builtins/getset.rs @@ -7,7 +7,7 @@ use crate::{ builtins::type_::PointerSlot, class::PyClassImpl, function::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc, PySetterValue}, - types::{GetDescriptor, Unconstructible}, + types::{GetDescriptor, Representable}, }; #[pyclass(module = false, name = "getset_descriptor")] @@ -19,8 +19,8 @@ pub struct PyGetSet { // doc: Option<String>, } -impl std::fmt::Debug for PyGetSet { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyGetSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, "PyGetSet {{ name: {}, getter: {}, setter: {} }}", @@ -96,7 +96,7 @@ impl PyGetSet { } } -#[pyclass(with(GetDescriptor, Unconstructible))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(GetDescriptor, Representable))] impl PyGetSet { // Descriptor methods @@ -118,19 +118,6 @@ impl PyGetSet { ))) } } - #[pymethod] - fn __set__( - zelf: PyObjectRef, - obj: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Assign(value), vm) - } - #[pymethod] - fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Delete, vm) - } #[pygetset] fn __name__(&self) -> String { @@ -152,7 +139,23 @@ impl PyGetSet { Ok(unsafe { zelf.class.borrow_static() }.to_owned().into()) } } -impl Unconstructible for PyGetSet {} + +impl Representable for PyGetSet { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class = unsafe { zelf.class.borrow_static() }; + // Special case for object type + if core::ptr::eq(class, vm.ctx.types.object_type) { + Ok(format!("<attribute '{}'>", zelf.name)) + } else { + Ok(format!( + "<attribute '{}' of '{}' objects>", + zelf.name, + class.name() + )) + } + } +} pub(crate) fn init(context: &Context) { PyGetSet::extend_class(context, context.types.getset_type); diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 8fe85267cd0..bbbc7d17673 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -18,11 +18,11 @@ use crate::{ protocol::{PyNumberMethods, handle_bytes_to_int_err}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use alloc::fmt; +use core::ops::{Neg, Not}; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; -use std::fmt; -use std::ops::{Neg, Not}; #[pyclass(module = false, name = "int")] #[derive(Debug)] @@ -122,7 +122,7 @@ fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { fn inner_mod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { if int2.is_zero() { - Err(vm.new_zero_division_error("integer modulo by zero")) + Err(vm.new_zero_division_error("division by zero")) } else { Ok(vm.ctx.new_int(int1.mod_floor(int2)).into()) } @@ -130,7 +130,7 @@ fn inner_mod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { fn inner_floordiv(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { if int2.is_zero() { - Err(vm.new_zero_division_error("integer division by zero")) + Err(vm.new_zero_division_error("division by zero")) } else { Ok(vm.ctx.new_int(int1.div_floor(int2)).into()) } @@ -138,7 +138,7 @@ fn inner_floordiv(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult fn inner_divmod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { if int2.is_zero() { - return Err(vm.new_zero_division_error("integer division or modulo by zero")); + return Err(vm.new_zero_division_error("division by zero")); } let (div, modulo) = int1.div_mod_floor(int2); Ok(vm.new_tuple((div, modulo)).into()) @@ -287,10 +287,17 @@ impl PyInt { where I: PrimInt + TryFrom<&'a BigInt>, { + // TODO: Python 3.14+: ValueError for negative int to unsigned type + // See stdlib_socket.py socket.htonl(-1) + // + // if I::min_value() == I::zero() && self.as_bigint().sign() == Sign::Minus { + // return Err(vm.new_value_error("Cannot convert negative int".to_owned())); + // } + I::try_from(self.as_bigint()).map_err(|_| { vm.new_overflow_error(format!( "Python int too large to convert to Rust {}", - std::any::type_name::<I>() + core::any::type_name::<I>() )) }) } @@ -320,87 +327,20 @@ impl PyInt { } #[pyclass( + itemsize = 4, flags(BASETYPE, _MATCH_SELF), with(PyRef, Comparable, Hashable, Constructor, AsNumber, Representable) )] impl PyInt { - #[pymethod(name = "__radd__")] - #[pymethod] - fn __add__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a + b) - } - - #[pymethod] - fn __sub__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a - b) - } - - #[pymethod] - fn __rsub__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| b - a) - } - - #[pymethod(name = "__rmul__")] - #[pymethod] - fn __mul__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a * b) - } - - #[pymethod] - fn __truediv__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_truediv(a, b, vm), vm) - } - - #[pymethod] - fn __rtruediv__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_truediv(b, a, vm), vm) - } - - #[pymethod] - fn __floordiv__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_floordiv(a, b, vm), vm) - } - - #[pymethod] - fn __rfloordiv__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_floordiv(b, a, vm), vm) - } - - #[pymethod] - fn __lshift__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_lshift(a, b, vm), vm) - } - - #[pymethod] - fn __rlshift__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_lshift(b, a, vm), vm) - } - - #[pymethod] - fn __rshift__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_rshift(a, b, vm), vm) - } - - #[pymethod] - fn __rrshift__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_rshift(b, a, vm), vm) - } - - #[pymethod(name = "__rxor__")] - #[pymethod] - pub fn __xor__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + pub(crate) fn __xor__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { self.int_op(other, |a, b| a ^ b) } - #[pymethod(name = "__ror__")] - #[pymethod] - pub fn __or__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + pub(crate) fn __or__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { self.int_op(other, |a, b| a | b) } - #[pymethod(name = "__rand__")] - #[pymethod] - pub fn __and__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + pub(crate) fn __and__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { self.int_op(other, |a, b| a & b) } @@ -446,61 +386,13 @@ impl PyInt { ) } - #[pymethod] - fn __pow__( - &self, - other: PyObjectRef, - r#mod: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - match r#mod.flatten() { - Some(modulus) => self.modpow(other, modulus, vm), - None => self.general_op(other, |a, b| inner_pow(a, b, vm), vm), - } - } - - #[pymethod] - fn __rpow__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_pow(b, a, vm), vm) - } - - #[pymethod(name = "__mod__")] - fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_mod(a, b, vm), vm) - } - - #[pymethod] - fn __rmod__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_mod(b, a, vm), vm) - } - - #[pymethod] - fn __divmod__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_divmod(a, b, vm), vm) - } - - #[pymethod] - fn __rdivmod__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_divmod(b, a, vm), vm) - } - - #[pymethod] - fn __neg__(&self) -> BigInt { - -(&self.value) - } - - #[pymethod] - fn __abs__(&self) -> BigInt { - self.value.abs() - } - #[pymethod] fn __round__( zelf: PyRef<Self>, - ndigits: OptionalArg<PyIntRef>, + ndigits: OptionalOption<PyIntRef>, vm: &VirtualMachine, ) -> PyResult<PyRef<Self>> { - if let OptionalArg::Present(ndigits) = ndigits { + if let Some(ndigits) = ndigits.flatten() { let ndigits = ndigits.as_bigint(); // round(12345, -2) == 12300 // If precision >= 0, then any integer is already rounded correctly @@ -536,16 +428,6 @@ impl PyInt { Ok(zelf) } - #[pymethod] - fn __pos__(&self) -> BigInt { - self.value.clone() - } - - #[pymethod] - fn __float__(&self, vm: &VirtualMachine) -> PyResult<f64> { - try_to_float(&self.value, vm) - } - #[pymethod] fn __trunc__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { zelf.__int__(vm) @@ -562,30 +444,19 @@ impl PyInt { } #[pymethod] - fn __index__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.__int__(vm) - } - - #[pymethod] - fn __invert__(&self) -> BigInt { - !(&self.value) - } - - #[pymethod] - fn __format__(&self, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + fn __format__(zelf: &Py<Self>, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + // Empty format spec on a subclass: equivalent to str(self) + if spec.is_empty() && !zelf.class().is(vm.ctx.types.int_type) { + return Ok(zelf.as_object().str(vm)?.as_str().to_owned()); + } FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_int(&self.value)) + .and_then(|format_spec| format_spec.format_int(&zelf.value)) .map_err(|err| err.into_pyexception(vm)) } - #[pymethod] - fn __bool__(&self) -> bool { - !self.value.is_zero() - } - #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + (((self.value.bits() + 7) & !7) / 8) as usize + core::mem::size_of::<Self>() + (((self.value.bits() + 7) & !7) / 8) as usize } #[pymethod] @@ -705,8 +576,7 @@ impl PyInt { #[pyclass] impl PyRef<PyInt> { - #[pymethod] - fn __int__(self, vm: &VirtualMachine) -> PyRefExact<PyInt> { + pub(crate) fn __int__(self, vm: &VirtualMachine) -> PyRefExact<PyInt> { self.into_exact_or(&vm.ctx, |zelf| unsafe { // TODO: this is actually safe. we need better interface PyRefExact::new_unchecked(vm.ctx.new_bigint(&zelf.value)) diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs new file mode 100644 index 00000000000..afdce51be9b --- /dev/null +++ b/crates/vm/src/builtins/interpolation.rs @@ -0,0 +1,221 @@ +use super::{ + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, genericalias::PyGenericAlias, + tuple::IntoPyTuple, +}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::hash::PyHash, + convert::ToPyObject, + function::{OptionalArg, PyComparisonValue}, + types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; + +/// Interpolation object for t-strings (PEP 750). +/// +/// Represents an interpolated expression within a template string. +#[pyclass(module = "string.templatelib", name = "Interpolation")] +#[derive(Debug, Clone)] +pub struct PyInterpolation { + pub value: PyObjectRef, + pub expression: PyStrRef, + pub conversion: PyObjectRef, // None or 's', 'r', 'a' + pub format_spec: PyStrRef, +} + +impl PyPayload for PyInterpolation { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.interpolation_type + } +} + +impl PyInterpolation { + pub fn new( + value: PyObjectRef, + expression: PyStrRef, + conversion: PyObjectRef, + format_spec: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<Self> { + // Validate conversion like _PyInterpolation_Build does + let is_valid = vm.is_none(&conversion) + || conversion + .downcast_ref::<PyStr>() + .is_some_and(|s| matches!(s.as_str(), "s" | "r" | "a")); + if !is_valid { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + "Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'".to_owned(), + )); + } + Ok(Self { + value, + expression, + conversion, + format_spec, + }) + } +} + +impl Constructor for PyInterpolation { + type Args = InterpolationArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let conversion: PyObjectRef = if let Some(s) = args.conversion { + let s_str = s.as_str(); + if s_str.len() != 1 || !matches!(s_str.chars().next(), Some('s' | 'r' | 'a')) { + return Err(vm.new_value_error( + "Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'", + )); + } + s.into() + } else { + vm.ctx.none() + }; + + let expression = args + .expression + .unwrap_or_else(|| vm.ctx.empty_str.to_owned()); + let format_spec = args + .format_spec + .unwrap_or_else(|| vm.ctx.empty_str.to_owned()); + + Ok(PyInterpolation { + value: args.value, + expression, + conversion, + format_spec, + }) + } +} + +#[derive(FromArgs)] +pub struct InterpolationArgs { + #[pyarg(positional)] + value: PyObjectRef, + #[pyarg(any, optional)] + expression: OptionalArg<PyStrRef>, + #[pyarg( + any, + optional, + error_msg = "Interpolation() argument 'conversion' must be str or None" + )] + conversion: Option<PyStrRef>, + #[pyarg(any, optional)] + format_spec: OptionalArg<PyStrRef>, +} + +#[pyclass(with(Constructor, Comparable, Hashable, Representable))] +impl PyInterpolation { + #[pyattr] + fn __match_args__(ctx: &Context) -> PyTupleRef { + ctx.new_tuple(vec![ + ctx.intern_str("value").to_owned().into(), + ctx.intern_str("expression").to_owned().into(), + ctx.intern_str("conversion").to_owned().into(), + ctx.intern_str("format_spec").to_owned().into(), + ]) + } + + #[pygetset] + fn value(&self) -> PyObjectRef { + self.value.clone() + } + + #[pygetset] + fn expression(&self) -> PyStrRef { + self.expression.clone() + } + + #[pygetset] + fn conversion(&self) -> PyObjectRef { + self.conversion.clone() + } + + #[pygetset] + fn format_spec(&self) -> PyStrRef { + self.format_spec.clone() + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { + let cls = zelf.class().to_owned(); + let args = ( + zelf.value.clone(), + zelf.expression.clone(), + zelf.conversion.clone(), + zelf.format_spec.clone(), + ); + (cls, args.to_pyobject(vm)).into_pytuple(vm) + } +} + +impl Comparable for PyInterpolation { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + + let eq = vm.bool_eq(&zelf.value, &other.value)? + && vm.bool_eq(zelf.expression.as_object(), other.expression.as_object())? + && vm.bool_eq(&zelf.conversion, &other.conversion)? + && vm.bool_eq(zelf.format_spec.as_object(), other.format_spec.as_object())?; + + Ok(eq.into()) + }) + } +} + +impl Hashable for PyInterpolation { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + // Hash based on (value, expression, conversion, format_spec) + let value_hash = zelf.value.hash(vm)?; + let expr_hash = zelf.expression.as_object().hash(vm)?; + let conv_hash = zelf.conversion.hash(vm)?; + let spec_hash = zelf.format_spec.as_object().hash(vm)?; + + // Combine hashes + Ok(value_hash + .wrapping_add(expr_hash.wrapping_mul(3)) + .wrapping_add(conv_hash.wrapping_mul(5)) + .wrapping_add(spec_hash.wrapping_mul(7))) + } +} + +impl Representable for PyInterpolation { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let value_repr = zelf.value.repr(vm)?; + let expr_repr = zelf.expression.repr(vm)?; + + let conv_str = if vm.is_none(&zelf.conversion) { + "None".to_owned() + } else { + zelf.conversion.repr(vm)?.as_str().to_owned() + }; + + let spec_repr = zelf.format_spec.repr(vm)?; + + Ok(format!( + "Interpolation({}, {}, {}, {})", + value_repr.as_str(), + expr_repr.as_str(), + conv_str, + spec_repr.as_str() + )) + } +} + +pub fn init(context: &Context) { + PyInterpolation::extend_class(context, context.types.interpolation_type); +} diff --git a/crates/vm/src/builtins/iter.rs b/crates/vm/src/builtins/iter.rs index 56dfc14d164..b96fb68559a 100644 --- a/crates/vm/src/builtins/iter.rs +++ b/crates/vm/src/builtins/iter.rs @@ -4,17 +4,14 @@ use super::{PyInt, PyTupleRef, PyType}; use crate::{ - Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, function::ArgCallable, object::{Traverse, TraverseFn}, - protocol::{PyIterReturn, PySequence, PySequenceMethods}, + protocol::PyIterReturn, types::{IterNext, Iterable, SelfIter}, }; -use rustpython_common::{ - lock::{PyMutex, PyRwLock, PyRwLockUpgradableReadGuard}, - static_cell, -}; +use rustpython_common::lock::{PyMutex, PyRwLock, PyRwLockUpgradableReadGuard}; /// Marks status of iterator. #[derive(Debug, Clone)] @@ -71,33 +68,29 @@ impl<T> PositionIterInternal<T> { } } - fn _reduce<F>(&self, func: PyObjectRef, f: F, vm: &VirtualMachine) -> PyTupleRef + /// Build a pickle-compatible reduce tuple. + /// + /// `func` must be resolved **before** acquiring any lock that guards this + /// `PositionIterInternal`, so that the builtins lookup cannot trigger + /// reentrant iterator access and deadlock. + pub fn reduce<F, E>( + &self, + func: PyObjectRef, + active: F, + empty: E, + vm: &VirtualMachine, + ) -> PyTupleRef where F: FnOnce(&T) -> PyObjectRef, + E: FnOnce(&VirtualMachine) -> PyObjectRef, { if let IterStatus::Active(obj) = &self.status { - vm.new_tuple((func, (f(obj),), self.position)) + vm.new_tuple((func, (active(obj),), self.position)) } else { - vm.new_tuple((func, (vm.ctx.new_list(Vec::new()),))) + vm.new_tuple((func, (empty(vm),))) } } - pub fn builtins_iter_reduce<F>(&self, f: F, vm: &VirtualMachine) -> PyTupleRef - where - F: FnOnce(&T) -> PyObjectRef, - { - let iter = builtins_iter(vm).to_owned(); - self._reduce(iter, f, vm) - } - - pub fn builtins_reversed_reduce<F>(&self, f: F, vm: &VirtualMachine) -> PyTupleRef - where - F: FnOnce(&T) -> PyObjectRef, - { - let reversed = builtins_reversed(vm).to_owned(); - self._reduce(reversed, f, vm) - } - fn _next<F, OP>(&mut self, f: F, op: OP) -> PyResult<PyIterReturn> where F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, @@ -160,26 +153,17 @@ impl<T> PositionIterInternal<T> { } } -pub fn builtins_iter(vm: &VirtualMachine) -> &PyObject { - static_cell! { - static INSTANCE: PyObjectRef; - } - INSTANCE.get_or_init(|| vm.builtins.get_attr("iter", vm).unwrap()) +pub fn builtins_iter(vm: &VirtualMachine) -> PyObjectRef { + vm.builtins.get_attr("iter", vm).unwrap() } -pub fn builtins_reversed(vm: &VirtualMachine) -> &PyObject { - static_cell! { - static INSTANCE: PyObjectRef; - } - INSTANCE.get_or_init(|| vm.builtins.get_attr("reversed", vm).unwrap()) +pub fn builtins_reversed(vm: &VirtualMachine) -> PyObjectRef { + vm.builtins.get_attr("reversed", vm).unwrap() } #[pyclass(module = false, name = "iterator", traverse)] #[derive(Debug)] pub struct PySequenceIterator { - // cached sequence methods - #[pytraverse(skip)] - seq_methods: &'static PySequenceMethods, internal: PyMutex<PositionIterInternal<PyObjectRef>>, } @@ -193,9 +177,8 @@ impl PyPayload for PySequenceIterator { #[pyclass(with(IterNext, Iterable))] impl PySequenceIterator { pub fn new(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { - let seq = PySequence::try_protocol(&obj, vm)?; + let _seq = obj.try_sequence(vm)?; Ok(Self { - seq_methods: seq.methods, internal: PyMutex::new(PositionIterInternal::new(obj, 0)), }) } @@ -204,10 +187,7 @@ impl PySequenceIterator { fn __length_hint__(&self, vm: &VirtualMachine) -> PyObjectRef { let internal = self.internal.lock(); if let IterStatus::Active(obj) = &internal.status { - let seq = PySequence { - obj, - methods: self.seq_methods, - }; + let seq = obj.sequence_unchecked(); seq.length(vm) .map(|x| PyInt::from(x).into_pyobject(vm)) .unwrap_or_else(|_| vm.ctx.not_implemented()) @@ -218,7 +198,13 @@ impl PySequenceIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal.lock().builtins_iter_reduce(|x| x.clone(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } #[pymethod] @@ -231,10 +217,7 @@ impl SelfIter for PySequenceIterator {} impl IterNext for PySequenceIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { zelf.internal.lock().next(|obj, pos| { - let seq = PySequence { - obj, - methods: zelf.seq_methods, - }; + let seq = obj.sequence_unchecked(); PyIterReturn::from_getitem_result(seq.get_item(pos as isize, vm), vm) }) } @@ -262,24 +245,47 @@ impl PyCallableIterator { status: PyRwLock::new(IterStatus::Active(callable)), } } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + let status = self.status.read(); + if let IterStatus::Active(callable) = &*status { + let callable_obj: PyObjectRef = callable.clone().into(); + vm.new_tuple((func, (callable_obj, self.sentinel.clone()))) + } else { + vm.new_tuple((func, (vm.ctx.empty_tuple.clone(),))) + } + } } impl SelfIter for PyCallableIterator {} impl IterNext for PyCallableIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let status = zelf.status.upgradable_read(); - let next = if let IterStatus::Active(callable) = &*status { - let ret = callable.invoke((), vm)?; - if vm.bool_eq(&ret, &zelf.sentinel)? { - *PyRwLockUpgradableReadGuard::upgrade(status) = IterStatus::Exhausted; - PyIterReturn::StopIteration(None) - } else { - PyIterReturn::Return(ret) + // Clone the callable and release the lock before invoking, + // so that reentrant next() calls don't deadlock. + let callable = { + let status = zelf.status.read(); + match &*status { + IterStatus::Active(callable) => callable.clone(), + IterStatus::Exhausted => return Ok(PyIterReturn::StopIteration(None)), } - } else { - PyIterReturn::StopIteration(None) }; - Ok(next) + + let ret = callable.invoke((), vm)?; + + // Re-check: a reentrant call may have exhausted the iterator. + let status = zelf.status.upgradable_read(); + if !matches!(&*status, IterStatus::Active(_)) { + return Ok(PyIterReturn::StopIteration(None)); + } + + if vm.bool_eq(&ret, &zelf.sentinel)? { + *PyRwLockUpgradableReadGuard::upgrade(status) = IterStatus::Exhausted; + Ok(PyIterReturn::StopIteration(None)) + } else { + Ok(PyIterReturn::Return(ret)) + } } } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index b2927462cac..7e22f73f8ec 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -1,8 +1,12 @@ -use super::{PositionIterInternal, PyGenericAlias, PyTupleRef, PyType, PyTypeRef}; +use super::{ + PositionIterInternal, PyGenericAlias, PyTupleRef, PyType, PyTypeRef, + iter::{builtins_iter, builtins_reversed}, +}; use crate::atomic_func; use crate::common::lock::{ PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, }; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, class::PyClassImpl, @@ -15,14 +19,15 @@ use crate::{ sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, types::{ AsMapping, AsSequence, Comparable, Constructor, Initializer, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, vm::VirtualMachine, }; -use std::{fmt, ops::DerefMut}; +use alloc::fmt; +use core::ops::DerefMut; -#[pyclass(module = false, name = "list", unhashable = true, traverse)] +#[pyclass(module = false, name = "list", unhashable = true, traverse = "manual")] #[derive(Default)] pub struct PyList { elements: PyRwLock<Vec<PyObjectRef>>, @@ -49,6 +54,22 @@ impl FromIterator<PyObjectRef> for PyList { } } +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyList { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + // During GC, we use interior mutability to access elements. + // This is safe because during GC collection, the object is unreachable + // and no other code should be accessing it. + if let Some(mut guard) = self.elements.try_write() { + out.extend(guard.drain(..)); + } + } +} + impl PyPayload for PyList { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -144,7 +165,6 @@ impl PyList { Ok(Self::from(elements).into_ref(&vm.ctx)) } - #[pymethod] fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { self.concat(&other, vm) } @@ -159,7 +179,6 @@ impl PyList { Ok(zelf.to_owned().into()) } - #[pymethod] fn __iadd__( zelf: PyRef<Self>, other: PyObjectRef, @@ -172,7 +191,7 @@ impl PyList { #[pymethod] fn clear(&self) { - let _removed = std::mem::take(self.borrow_vec_mut().deref_mut()); + let _removed = core::mem::take(self.borrow_vec_mut().deref_mut()); } #[pymethod] @@ -181,15 +200,14 @@ impl PyList { } #[allow(clippy::len_without_is_empty)] - #[pymethod] pub fn __len__(&self) -> usize { self.borrow_vec().len() } #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() - + self.elements.read().capacity() * std::mem::size_of::<PyObjectRef>() + core::mem::size_of::<Self>() + + self.elements.read().capacity() * core::mem::size_of::<PyObjectRef>() } #[pymethod] @@ -207,7 +225,13 @@ impl PyList { fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { - SequenceIndex::Int(i) => self.borrow_vec().getitem_by_index(vm, i), + SequenceIndex::Int(i) => { + let vec = self.borrow_vec(); + let pos = vec + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("list index out of range"))?; + Ok(vec.do_get(pos)) + } SequenceIndex::Slice(slice) => self .borrow_vec() .getitem_by_slice(vm, slice) @@ -215,7 +239,6 @@ impl PyList { } } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } @@ -230,7 +253,6 @@ impl PyList { } } - #[pymethod] fn __setitem__( &self, needle: PyObjectRef, @@ -240,13 +262,10 @@ impl PyList { self._setitem(&needle, value, vm) } - #[pymethod] - #[pymethod(name = "__rmul__")] fn __mul__(&self, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { self.repeat(n.into(), vm) } - #[pymethod] fn __imul__(zelf: PyRef<Self>, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Self::irepeat(zelf, n.into(), vm) } @@ -256,7 +275,6 @@ impl PyList { self.mut_count(vm, &needle) } - #[pymethod] pub(crate) fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self.mut_contains(vm, &needle) } @@ -314,7 +332,6 @@ impl PyList { } } - #[pymethod] fn __delitem__(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self._delitem(&subscript, vm) } @@ -324,9 +341,9 @@ impl PyList { // replace list contents with [] for duration of sort. // this prevents keyfunc from messing with the list and makes it easy to // check if it tries to append elements to it. - let mut elements = std::mem::take(self.borrow_vec_mut().deref_mut()); + let mut elements = core::mem::take(self.borrow_vec_mut().deref_mut()); let res = do_sort(vm, &mut elements, options.key, options.reverse); - std::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); + core::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); res?; if !elements.is_empty() { @@ -354,7 +371,11 @@ where } else { let iter = obj.to_owned().get_iter(vm)?; let iter = iter.iter::<PyObjectRef>(vm)?; - let len = obj.to_sequence().length_opt(vm).transpose()?.unwrap_or(0); + let len = obj + .sequence_unchecked() + .length_opt(vm) + .transpose()? + .unwrap_or(0); let mut v = Vec::with_capacity(len); for x in iter { v.push(f(x?)?); @@ -367,11 +388,11 @@ where impl MutObjectSequenceOp for PyList { type Inner = [PyObjectRef]; - fn do_get(index: usize, inner: &[PyObjectRef]) -> Option<&PyObjectRef> { - inner.get(index) + fn do_get(index: usize, inner: &[PyObjectRef]) -> Option<&PyObject> { + inner.get(index).map(|r| r.as_ref()) } - fn do_lock(&self) -> impl std::ops::Deref<Target = [PyObjectRef]> { + fn do_lock(&self) -> impl core::ops::Deref<Target = [PyObjectRef]> { self.borrow_vec() } } @@ -393,7 +414,7 @@ impl Initializer for PyList { } else { vec![] }; - std::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); + core::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); Ok(()) } } @@ -433,9 +454,12 @@ impl AsSequence for PyList { .map(|x| x.into()) }), item: atomic_func!(|seq, i, vm| { - PyList::sequence_downcast(seq) - .borrow_vec() - .getitem_by_index(vm, i) + let list = PyList::sequence_downcast(seq); + let vec = list.borrow_vec(); + let pos = vec + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("list index out of range"))?; + Ok(vec.do_get(pos)) }), ass_item: atomic_func!(|seq, i, value, vm| { let zelf = PyList::sequence_downcast(seq); @@ -496,7 +520,11 @@ impl Representable for PyList { let s = if zelf.__len__() == 0 { "[]".to_owned() } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - collection_repr(None, "[", "]", zelf.borrow_vec().iter(), vm)? + // Clone elements before calling repr to release the read lock. + // Element repr may mutate the list (e.g., list.clear()), which + // needs a write lock and would deadlock if read lock is held. + let elements: Vec<PyObjectRef> = zelf.borrow_vec().to_vec(); + collection_repr(None, "[", "]", elements.iter(), vm)? } else { "[...]".to_owned() }; @@ -510,12 +538,17 @@ fn do_sort( key_func: Option<PyObjectRef>, reverse: bool, ) -> PyResult<()> { - let op = if reverse { - PyComparisonOp::Lt - } else { - PyComparisonOp::Gt + // CPython uses __lt__ for all comparisons in sort. + // try_sort_by_gt expects is_gt(a, b) = true when a should come AFTER b. + let cmp = |a: &PyObjectRef, b: &PyObjectRef| { + if reverse { + // Descending: a comes after b when a < b + a.rich_compare_bool(b, PyComparisonOp::Lt, vm) + } else { + // Ascending: a comes after b when b < a + b.rich_compare_bool(a, PyComparisonOp::Lt, vm) + } }; - let cmp = |a: &PyObjectRef, b: &PyObjectRef| a.rich_compare_bool(b, op, vm); if let Some(ref key_func) = key_func { let mut items = values @@ -544,7 +577,7 @@ impl PyPayload for PyListIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyListIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -560,12 +593,15 @@ impl PyListIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.new_list(Vec::new()).into(), + vm, + ) } } -impl Unconstructible for PyListIterator {} impl SelfIter for PyListIterator {} impl IterNext for PyListIterator { @@ -590,7 +626,7 @@ impl PyPayload for PyListReverseIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyListReverseIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -606,12 +642,15 @@ impl PyListReverseIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_reversed_reduce(|x| x.clone().into(), vm) + let func = builtins_reversed(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.new_list(Vec::new()).into(), + vm, + ) } } -impl Unconstructible for PyListReverseIterator {} impl SelfIter for PyListReverseIterator {} impl IterNext for PyListReverseIterator { diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs index f5cee945ece..f83030824f1 100644 --- a/crates/vm/src/builtins/map.rs +++ b/crates/vm/src/builtins/map.rs @@ -42,7 +42,7 @@ impl PyMap { fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult<usize> { self.iterators.iter().try_fold(0, |prev, cur| { let cur = cur.as_ref().to_owned().length_hint(0, vm)?; - let max = std::cmp::max(prev, cur); + let max = core::cmp::max(prev, cur); Ok(max) }) } diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index fb8ff5de9cc..11525c3f80a 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -1,18 +1,19 @@ use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyTuple, PyType, PyTypeRef}; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, class::PyClassImpl, + common::hash, convert::ToPyObject, function::{ArgMapping, OptionalArg, PyComparisonValue}, object::{Traverse, TraverseFn}, - protocol::{PyMapping, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, types::{ - AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp, - Representable, + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, Iterable, + PyComparisonOp, Representable, }, }; -use std::sync::LazyLock; #[pyclass(module = false, name = "mappingproxy", traverse)] #[derive(Debug)] @@ -62,14 +63,12 @@ impl Constructor for PyMappingProxy { type Args = PyObjectRef; fn py_new(_cls: &Py<PyType>, mapping: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(methods) = PyMapping::find_methods(&mapping) + if mapping.mapping_unchecked().check() && !mapping.downcastable::<PyList>() && !mapping.downcastable::<PyTuple>() { return Ok(Self { - mapping: MappingProxyInner::Mapping(ArgMapping::with_methods(mapping, unsafe { - methods.borrow_static() - })), + mapping: MappingProxyInner::Mapping(ArgMapping::new(mapping)), }); } Err(vm.new_type_error(format!( @@ -85,6 +84,7 @@ impl Constructor for PyMappingProxy { Constructor, AsSequence, Comparable, + Hashable, AsNumber, Representable ))] @@ -113,7 +113,6 @@ impl PyMappingProxy { )?)) } - #[pymethod] pub fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.get_inner(key.clone(), vm)? .ok_or_else(|| vm.new_key_error(key)) @@ -124,11 +123,12 @@ impl PyMappingProxy { MappingProxyInner::Class(class) => Ok(key .as_interned_str(vm) .is_some_and(|key| class.attributes.read().contains_key(key))), - MappingProxyInner::Mapping(mapping) => mapping.to_sequence().contains(key, vm), + MappingProxyInner::Mapping(mapping) => { + mapping.obj().sequence_unchecked().contains(key, vm) + } } } - #[pymethod] pub fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self._contains(&key, vm) } @@ -163,7 +163,9 @@ impl PyMappingProxy { #[pymethod] pub fn copy(&self, vm: &VirtualMachine) -> PyResult { match &self.mapping { - MappingProxyInner::Mapping(d) => vm.call_method(d, identifier!(vm, copy).as_str(), ()), + MappingProxyInner::Mapping(d) => { + vm.call_method(d.obj(), identifier!(vm, copy).as_str(), ()) + } MappingProxyInner::Class(c) => { Ok(PyDict::from_attributes(c.attributes.read().clone(), vm)?.to_pyobject(vm)) } @@ -175,7 +177,6 @@ impl PyMappingProxy { PyGenericAlias::from_args(cls, args, vm) } - #[pymethod] fn __len__(&self, vm: &VirtualMachine) -> PyResult<usize> { let obj = self.to_object(vm)?; obj.length(vm) @@ -190,7 +191,6 @@ impl PyMappingProxy { ) } - #[pymethod] fn __ior__(&self, _args: PyObjectRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error(format!( r#""'|=' is not supported by {}; use '|' instead""#, @@ -198,8 +198,6 @@ impl PyMappingProxy { ))) } - #[pymethod(name = "__ror__")] - #[pymethod] fn __or__(&self, args: PyObjectRef, vm: &VirtualMachine) -> PyResult { vm._or(self.copy(vm)?.as_ref(), args.as_ref()) } @@ -219,6 +217,15 @@ impl Comparable for PyMappingProxy { } } +impl Hashable for PyMappingProxy { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { + // Delegate hash to the underlying mapping + let obj = zelf.to_object(vm)?; + obj.hash(vm) + } +} + impl AsMapping for PyMappingProxy { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { @@ -237,6 +244,7 @@ impl AsMapping for PyMappingProxy { impl AsSequence for PyMappingProxy { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, vm| PyMappingProxy::sequence_downcast(seq).__len__(vm)), contains: atomic_func!( |seq, target, vm| PyMappingProxy::sequence_downcast(seq)._contains(target, vm) ), diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index c1b1496e8c6..329a6be6737 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -1,7 +1,8 @@ use super::{ PositionIterInternal, PyBytes, PyBytesRef, PyGenericAlias, PyInt, PyListRef, PySlice, PyStr, - PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, iter::builtins_iter, }; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, atomic_func, @@ -23,14 +24,13 @@ use crate::{ sliceable::SequenceIndexOp, types::{ AsBuffer, AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + PyComparisonOp, Representable, SelfIter, }, }; +use core::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use rustpython_common::lock::PyMutex; -use std::sync::LazyLock; -use std::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; #[derive(FromArgs)] pub struct PyMemoryViewNewArgs { @@ -552,8 +552,7 @@ impl Py<PyMemoryView> { flags(SEQUENCE) )] impl PyMemoryView { - // TODO: Uncomment when Python adds __class_getitem__ to memoryview - // #[pyclassmethod] + #[pyclassmethod] fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) } @@ -617,13 +616,22 @@ impl PyMemoryView { #[pygetset] fn suboffsets(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { self.try_not_released(vm)?; - Ok(vm.ctx.new_tuple( - self.desc - .dim_desc - .iter() - .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) - .collect(), - )) + let has_suboffsets = self + .desc + .dim_desc + .iter() + .any(|(_, _, suboffset)| *suboffset != 0); + if has_suboffsets { + Ok(vm.ctx.new_tuple( + self.desc + .dim_desc + .iter() + .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) + .collect(), + )) + } else { + Ok(vm.ctx.empty_tuple.clone()) + } } #[pygetset] @@ -659,7 +667,6 @@ impl PyMemoryView { self.release(); } - #[pymethod] fn __getitem__(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { zelf.try_not_released(vm)?; if zelf.desc.ndim() == 0 { @@ -682,7 +689,6 @@ impl PyMemoryView { } } - #[pymethod] fn __delitem__(&self, _needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.desc.readonly { return Err(vm.new_type_error("cannot modify read-only memory")); @@ -690,7 +696,6 @@ impl PyMemoryView { Err(vm.new_type_error("cannot delete memory")) } - #[pymethod] fn __len__(&self, vm: &VirtualMachine) -> PyResult<usize> { self.try_not_released(vm)?; if self.desc.ndim() == 0 { @@ -743,6 +748,63 @@ impl PyMemoryView { self.contiguous_or_collect(|x| bytes_to_hex(x, sep, bytes_per_sep, vm)) } + #[pymethod] + fn count(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + self.try_not_released(vm)?; + if self.desc.ndim() != 1 { + return Err( + vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") + ); + } + let len = self.desc.dim_desc[0].0; + let mut count = 0; + for i in 0..len { + let item = self.getitem_by_idx(i as isize, vm)?; + if vm.bool_eq(&item, &value)? { + count += 1; + } + } + Ok(count) + } + + #[pymethod] + fn index( + &self, + value: PyObjectRef, + start: OptionalArg<isize>, + stop: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult<usize> { + self.try_not_released(vm)?; + if self.desc.ndim() != 1 { + return Err( + vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") + ); + } + let len = self.desc.dim_desc[0].0; + let start = start.unwrap_or(0); + let stop = stop.unwrap_or(len as isize); + + let start = if start < 0 { + (start + len as isize).max(0) as usize + } else { + (start as usize).min(len) + }; + let stop = if stop < 0 { + (stop + len as isize).max(0) as usize + } else { + (stop as usize).min(len) + }; + + for i in start..stop { + let item = self.getitem_by_idx(i as isize, vm)?; + if vm.bool_eq(&item, &value)? { + return Ok(i); + } + } + Err(vm.new_value_error("memoryview.index(x): x not in memoryview")) + } + fn cast_to_1d(&self, format: PyStrRef, vm: &VirtualMachine) -> PyResult<Self> { let format_spec = Self::parse_format(format.as_str(), vm)?; let itemsize = format_spec.size(); @@ -848,7 +910,6 @@ impl PyMemoryView { #[pyclass] impl Py<PyMemoryView> { - #[pymethod] fn __setitem__( &self, needle: PyObjectRef, @@ -1033,15 +1094,16 @@ impl Comparable for PyMemoryView { impl Hashable for PyMemoryView { fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - zelf.hash - .get_or_try_init(|| { - zelf.try_not_released(vm)?; - if !zelf.desc.readonly { - return Err(vm.new_value_error("cannot hash writable memoryview object")); - } - Ok(zelf.contiguous_or_collect(|bytes| vm.state.hash_secret.hash_bytes(bytes))) - }) - .copied() + if let Some(val) = zelf.hash.get() { + return Ok(*val); + } + zelf.try_not_released(vm)?; + if !zelf.desc.readonly { + return Err(vm.new_value_error("cannot hash writable memoryview object")); + } + let val = zelf.contiguous_or_collect(|bytes| vm.state.hash_secret.hash_bytes(bytes)); + let _ = zelf.hash.set(val); + Ok(*zelf.hash.get().unwrap()) } } @@ -1132,16 +1194,19 @@ impl PyPayload for PyMemoryViewIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyMemoryViewIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } } -impl Unconstructible for PyMemoryViewIterator {} impl SelfIter for PyMemoryViewIterator {} impl IterNext for PyMemoryViewIterator { diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index 6f379a4babf..fa7ab1b854e 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -9,6 +9,8 @@ pub(crate) mod bytearray; pub use bytearray::PyByteArray; pub(crate) mod bytes; pub use bytes::{PyBytes, PyBytesRef}; +pub(crate) mod capsule; +pub use capsule::PyCapsule; pub(crate) mod classmethod; pub use classmethod::PyClassMethod; pub(crate) mod code; @@ -36,6 +38,8 @@ pub(crate) mod getset; pub use getset::PyGetSet; pub(crate) mod int; pub use int::{PyInt, PyIntRef}; +pub(crate) mod interpolation; +pub use interpolation::PyInterpolation; pub(crate) mod iter; pub use iter::*; pub(crate) mod list; @@ -47,7 +51,7 @@ pub use mappingproxy::PyMappingProxy; pub(crate) mod memory; pub use memory::PyMemoryView; pub(crate) mod module; -pub use module::{PyModule, PyModuleDef}; +pub use module::{PyModule, PyModuleDef, PyModuleSlots}; pub(crate) mod namespace; pub use namespace::PyNamespace; pub(crate) mod object; @@ -76,6 +80,8 @@ pub(crate) mod slice; pub use slice::{PyEllipsis, PySlice}; pub(crate) mod staticmethod; pub use staticmethod::PyStaticMethod; +pub(crate) mod template; +pub use template::{PyTemplate, PyTemplateIter}; pub(crate) mod traceback; pub use traceback::PyTraceback; pub(crate) mod tuple; @@ -88,7 +94,7 @@ pub(crate) mod zip; pub use zip::PyZip; #[path = "union.rs"] pub(crate) mod union_; -pub use union_::PyUnion; +pub use union_::{PyUnion, make_union}; pub(crate) mod descriptor; pub use float::try_to_bigint as try_f64_to_bigint; diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index f8e42b28e0b..a2221fb6b9a 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -4,7 +4,8 @@ use crate::{ builtins::{PyStrInterned, pystr::AsPyStr}, class::PyClassImpl, convert::ToPyObject, - function::{FuncArgs, PyMethodDef}, + function::{FuncArgs, PyMethodDef, PySetterValue}, + import::{get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name}, types::{GetAttr, Initializer, Representable}, }; @@ -32,8 +33,8 @@ pub struct PyModuleSlots { pub exec: Option<ModuleExec>, } -impl std::fmt::Debug for PyModuleSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyModuleSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyModuleSlots") .field("create", &self.create.is_some()) .field("exec", &self.exec.is_some()) @@ -41,7 +42,50 @@ impl std::fmt::Debug for PyModuleSlots { } } -#[allow(clippy::new_without_default)] // avoid Default implementation +impl PyModuleDef { + /// Create a module from this definition (Phase 1 of multi-phase init). + /// + /// This performs: + /// 1. Create module object (using create slot if provided) + /// 2. Initialize module dict from def + /// 3. Add methods to module + /// + /// Does NOT add to sys.modules or call exec slot. + pub fn create_module(&'static self, vm: &VirtualMachine) -> PyResult<PyRef<PyModule>> { + use crate::PyPayload; + + // Create module (use create slot if provided, else default creation) + let module = if let Some(create) = self.slots.create { + // Custom module creation + let spec = vm.ctx.new_str(self.name.as_str()); + create(vm, spec.as_object(), self)? + } else { + // Default module creation + PyModule::from_def(self).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + Ok(module) + } + + /// Execute the module's exec slot (Phase 2 of multi-phase init). + /// + /// Calls the exec slot if present. Returns Ok(()) if no exec slot. + pub fn exec_module(&'static self, vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + if let Some(exec) = self.slots.exec { + exec(vm, module)?; + } + Ok(()) + } +} + +#[allow( + clippy::new_without_default, + reason = "avoid a misleading Default implementation" +)] #[pyclass(module = false, name = "module")] #[derive(Debug)] pub struct PyModule { @@ -109,20 +153,96 @@ impl Py<PyModule> { if let Ok(getattr) = self.dict().get_item(identifier!(vm, __getattr__), vm) { return getattr.call((name.to_owned(),), vm); } - let module_name = if let Some(name) = self.name(vm) { - format!(" '{name}'") + let dict = self.dict(); + + // Get the raw __name__ object (may be a str subclass) + let mod_name_obj = dict + .get_item_opt(identifier!(vm, __name__), vm) + .ok() + .flatten(); + let mod_name_str = mod_name_obj + .as_ref() + .and_then(|n| n.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned())); + + // If __name__ is not set or not a string, use a simpler error message + let mod_display = match mod_name_str.as_deref() { + Some(s) => s, + None => { + return Err(vm.new_attribute_error(format!("module has no attribute '{name}'"))); + } + }; + + let spec = dict + .get_item_opt(vm.ctx.intern_str("__spec__"), vm) + .ok() + .flatten() + .filter(|s| !vm.is_none(s)); + + let origin = get_spec_file_origin(&spec, vm); + + let is_possibly_shadowing = origin + .as_ref() + .map(|o| is_possibly_shadowing_path(o, vm)) + .unwrap_or(false); + // Use the ORIGINAL __name__ object for stdlib check (may raise TypeError + // if __name__ is an unhashable str subclass) + let is_possibly_shadowing_stdlib = if is_possibly_shadowing { + if let Some(ref mod_name) = mod_name_obj { + is_stdlib_module_name(mod_name, vm)? + } else { + false + } } else { - "".to_owned() + false }; - Err(vm.new_attribute_error(format!("module{module_name} has no attribute '{name}'"))) - } - fn name(&self, vm: &VirtualMachine) -> Option<PyStrRef> { - let name = self - .as_object() - .generic_getattr_opt(identifier!(vm, __name__), None, vm) - .unwrap_or_default()?; - name.downcast::<PyStr>().ok() + if is_possibly_shadowing_stdlib { + let origin = origin.as_ref().unwrap(); + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}' \ + (consider renaming '{origin}' since it has the same \ + name as the standard library module named '{mod_display}' \ + and prevents importing that standard library module)" + ))) + } else { + let is_initializing = PyModule::is_initializing(&dict, vm); + if is_initializing { + if is_possibly_shadowing { + let origin = origin.as_ref().unwrap(); + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}' \ + (consider renaming '{origin}' if it has the same name \ + as a library you intended to import)" + ))) + } else if let Some(ref origin) = origin { + Err(vm.new_attribute_error(format!( + "partially initialized module '{mod_display}' from '{origin}' \ + has no attribute '{name}' \ + (most likely due to a circular import)" + ))) + } else { + Err(vm.new_attribute_error(format!( + "partially initialized module '{mod_display}' \ + has no attribute '{name}' \ + (most likely due to a circular import)" + ))) + } + } else { + // Check for uninitialized submodule + let submodule_initializing = + is_uninitialized_submodule(mod_name_str.as_ref(), name, vm); + if submodule_initializing { + Err(vm.new_attribute_error(format!( + "cannot access submodule '{name}' of module '{mod_display}' \ + (most likely due to a circular import)" + ))) + } else { + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}'" + ))) + } + } + } } // TODO: to be replaced by the commented-out dict method above once dictoffset land @@ -182,6 +302,116 @@ impl PyModule { let attrs = dict.into_iter().map(|(k, _v)| k).collect(); Ok(attrs) } + + #[pygetset] + fn __annotate__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let dict = zelf.dict(); + // Get __annotate__ from dict; if not present, insert None and return it + // See: module_get_annotate() + if let Some(annotate) = dict.get_item_opt(identifier!(vm, __annotate__), vm)? { + Ok(annotate) + } else { + let none = vm.ctx.none(); + dict.set_item(identifier!(vm, __annotate__), none.clone(), vm)?; + Ok(none) + } + } + + #[pygetset(setter)] + fn set___annotate__( + zelf: &Py<Self>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + let dict = zelf.dict(); + dict.set_item(identifier!(vm, __annotate__), value.clone(), vm)?; + // Clear __annotations__ if value is not None + if !vm.is_none(&value) { + dict.del_item(identifier!(vm, __annotations__), vm).ok(); + } + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("cannot delete __annotate__ attribute")), + } + } + + #[pygetset] + fn __annotations__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let dict = zelf.dict(); + + // Check if __annotations__ is already in dict (explicitly set) + if let Some(annotations) = dict.get_item_opt(identifier!(vm, __annotations__), vm)? { + return Ok(annotations); + } + + // Check if module is initializing + let is_initializing = Self::is_initializing(&dict, vm); + + // PEP 649: Get __annotate__ and call it if callable + let annotations = if let Some(annotate) = + dict.get_item_opt(identifier!(vm, __annotate__), vm)? + && annotate.is_callable() + { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result + } else { + vm.ctx.new_dict().into() + }; + + // Cache result unless module is initializing + if !is_initializing { + dict.set_item(identifier!(vm, __annotations__), annotations.clone(), vm)?; + } + + Ok(annotations) + } + + /// Check if module is initializing via __spec__._initializing + fn is_initializing(dict: &PyDictRef, vm: &VirtualMachine) -> bool { + if let Ok(Some(spec)) = dict.get_item_opt(vm.ctx.intern_str("__spec__"), vm) + && let Ok(initializing) = spec.get_attr(vm.ctx.intern_str("_initializing"), vm) + { + return initializing.try_to_bool(vm).unwrap_or(false); + } + false + } + + #[pygetset(setter)] + fn set___annotations__( + zelf: &Py<Self>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let dict = zelf.dict(); + match value { + PySetterValue::Assign(value) => { + dict.set_item(identifier!(vm, __annotations__), value, vm)?; + // Clear __annotate__ from dict + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + PySetterValue::Delete => { + if dict.del_item(identifier!(vm, __annotations__), vm).is_err() { + return Err(vm.new_attribute_error("__annotations__".to_owned())); + } + // Also clear __annotate__ + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + } + } } impl Initializer for PyModule { @@ -208,8 +438,8 @@ impl GetAttr for PyModule { impl Representable for PyModule { #[inline] fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let importlib = vm.import("_frozen_importlib", 0)?; - let module_repr = importlib.get_attr("_module_repr", vm)?; + // Use cached importlib reference (like interp->importlib) + let module_repr = vm.importlib.get_attr("_module_repr", vm)?; let repr = module_repr.call((zelf.to_owned(),), vm)?; repr.downcast() .map_err(|_| vm.new_type_error("_module_repr did not return a string")) @@ -224,3 +454,32 @@ impl Representable for PyModule { pub(crate) fn init(context: &Context) { PyModule::extend_class(context, context.types.module_type); } + +/// Check if {module_name}.{name} is an uninitialized submodule in sys.modules. +fn is_uninitialized_submodule( + module_name: Option<&String>, + name: &Py<PyStr>, + vm: &VirtualMachine, +) -> bool { + let mod_name = match module_name { + Some(n) => n.as_str(), + None => return false, + }; + let full_name = format!("{mod_name}.{name}"); + let sys_modules = match vm.sys_module.get_attr("modules", vm).ok() { + Some(m) => m, + None => return false, + }; + let sub_mod = match sys_modules.get_item(&full_name, vm).ok() { + Some(m) => m, + None => return false, + }; + let spec = match sub_mod.get_attr("__spec__", vm).ok() { + Some(s) if !vm.is_none(&s) => s, + _ => return false, + }; + spec.get_attr("_initializing", vm) + .ok() + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false) +} diff --git a/crates/vm/src/builtins/namespace.rs b/crates/vm/src/builtins/namespace.rs index 03969c35e7b..2cc1693302a 100644 --- a/crates/vm/src/builtins/namespace.rs +++ b/crates/vm/src/builtins/namespace.rs @@ -42,15 +42,76 @@ impl PyNamespace { ); result.into_pytuple(vm) } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments")); + } + + // Create a new instance of the same type + let cls: PyObjectRef = zelf.class().to_owned().into(); + let result = cls.call((), vm)?; + + // Copy the current namespace dict to the new instance + let src_dict = zelf.dict().unwrap(); + let dst_dict = result.dict().unwrap(); + for (key, value) in src_dict { + dst_dict.set_item(&*key, value, vm)?; + } + + // Update with the provided kwargs + for (name, value) in args.kwargs { + let name = vm.ctx.new_str(name); + result.set_attr(&name, value, vm)?; + } + + Ok(result) + } } impl Initializer for PyNamespace { type Args = FuncArgs; fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - if !args.args.is_empty() { - return Err(vm.new_type_error("no positional arguments expected")); + // SimpleNamespace accepts 0 or 1 positional argument (a mapping) + if args.args.len() > 1 { + return Err(vm.new_type_error(format!( + "{} expected at most 1 positional argument, got {}", + zelf.class().name(), + args.args.len() + ))); } + + // If there's a positional argument, treat it as a mapping + if let Some(mapping) = args.args.first() { + // Convert to dict if not already + let dict: PyRef<PyDict> = if let Some(d) = mapping.downcast_ref::<PyDict>() { + d.to_owned() + } else { + // Call dict() on the mapping + let dict_type: PyObjectRef = vm.ctx.types.dict_type.to_owned().into(); + dict_type + .call((mapping.clone(),), vm)? + .downcast() + .map_err(|_| vm.new_type_error("dict() did not return a dict"))? + }; + + // Validate keys are strings and set attributes + for (key, value) in dict.into_iter() { + let key_str = key + .downcast_ref::<crate::builtins::PyStr>() + .ok_or_else(|| { + vm.new_type_error(format!( + "keywords must be strings, not '{}'", + key.class().name() + )) + })?; + zelf.as_object().set_attr(key_str, value, vm)?; + } + } + + // Apply keyword arguments (these override positional mapping values) for (name, value) in args.kwargs { let name = vm.ctx.new_str(name); zelf.as_object().set_attr(&name, value, vm)?; diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index 6f917cd853c..e4c51061685 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -2,11 +2,11 @@ use super::{PyDictRef, PyList, PyStr, PyStrRef, PyType, PyTypeRef}; use crate::common::hash::PyHash; use crate::types::PyTypeFlags; use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, convert::ToPyResult, function::{Either, FuncArgs, PyArithmeticValue, PyComparisonValue, PySetterValue}, - types::{Constructor, PyComparisonOp}, + types::{Constructor, Initializer, PyComparisonOp}, }; use itertools::Itertools; @@ -115,6 +115,50 @@ impl Constructor for PyBaseObject { } } +impl Initializer for PyBaseObject { + type Args = FuncArgs; + + // object_init: excess_args validation + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if args.is_empty() { + return Ok(()); + } + + let typ = zelf.class(); + let object_type = &vm.ctx.types.object_type; + + let typ_init = typ.slots.init.load().map(|f| f as usize); + let object_init = object_type.slots.init.load().map(|f| f as usize); + + // if (type->tp_init != object_init) → first error + if typ_init != object_init { + return Err(vm.new_type_error( + "object.__init__() takes exactly one argument (the instance to initialize)" + .to_owned(), + )); + } + + // if (type->tp_new == object_new) → second error + if let (Some(typ_new), Some(object_new)) = ( + typ.get_attr(identifier!(vm, __new__)), + object_type.get_attr(identifier!(vm, __new__)), + ) && typ_new.is(&object_new) + { + return Err(vm.new_type_error(format!( + "{}.__init__() takes exactly one argument (the instance to initialize)", + typ.name() + ))); + } + + // Both conditions false → OK (e.g., tuple, dict with custom __new__) + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + // TODO: implement _PyType_GetSlotNames properly fn type_slot_names(typ: &Py<PyType>, vm: &VirtualMachine) -> PyResult<Option<super::PyListRef>> { // let attributes = typ.attributes.read(); @@ -141,15 +185,12 @@ fn type_slot_names(typ: &Py<PyType>, vm: &VirtualMachine) -> PyResult<Option<sup Ok(result) } -// object_getstate_default in CPython +// object_getstate_default fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) -> PyResult { - // TODO: itemsize - // if required && obj.class().slots.itemsize > 0 { - // return vm.new_type_error(format!( - // "cannot pickle {:.200} objects", - // obj.class().name() - // )); - // } + // Check itemsize + if required && obj.class().slots.itemsize > 0 { + return Err(vm.new_type_error(format!("cannot pickle {:.200} objects", obj.class().name()))); + } let state = if obj.dict().is_none_or(|d| d.is_empty()) { vm.ctx.none() @@ -165,22 +206,36 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) type_slot_names(obj.class(), vm).map_err(|_| vm.new_type_error("cannot pickle object"))?; if required { - let mut basicsize = obj.class().slots.basicsize; - // if obj.class().slots.dict_offset > 0 - // && !obj.class().slots.flags.has_feature(PyTypeFlags::MANAGED_DICT) - // { - // basicsize += std::mem::size_of::<PyObjectRef>(); - // } - // if obj.class().slots.weaklist_offset > 0 { - // basicsize += std::mem::size_of::<PyObjectRef>(); - // } + // Start with PyBaseObject_Type's basicsize + let mut basicsize = vm.ctx.types.object_type.slots.basicsize; + + // Add __dict__ size if type has dict + if obj.class().slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + basicsize += core::mem::size_of::<PyObjectRef>(); + } + + // Add __weakref__ size if type has weakref support + let has_weakref = if let Some(ref ext) = obj.class().heaptype_ext { + match &ext.slots { + None => true, // Heap type without __slots__ has automatic weakref + Some(slots) => slots.iter().any(|s| s.as_str() == "__weakref__"), + } + } else { + let weakref_name = vm.ctx.intern_str("__weakref__"); + obj.class().attributes.read().contains_key(weakref_name) + }; + if has_weakref { + basicsize += core::mem::size_of::<PyObjectRef>(); + } + + // Add slots size if let Some(ref slot_names) = slot_names { - basicsize += std::mem::size_of::<PyObjectRef>() * slot_names.__len__(); + basicsize += core::mem::size_of::<PyObjectRef>() * slot_names.__len__(); } + + // Fail if actual type's basicsize > expected basicsize if obj.class().slots.basicsize > basicsize { - return Err( - vm.new_type_error(format!("cannot pickle {:.200} object", obj.class().name())) - ); + return Err(vm.new_type_error(format!("cannot pickle '{}' object", obj.class().name()))); } } @@ -190,6 +245,12 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) let slots = vm.ctx.new_dict(); for i in 0..slot_names_len { let borrowed_names = slot_names.borrow_vec(); + // Check if slotnames changed during iteration + if borrowed_names.len() != slot_names_len { + return Err(vm.new_runtime_error( + "__slotnames__ changed size during iteration".to_owned(), + )); + } let name = borrowed_names[i].downcast_ref::<PyStr>().unwrap(); let Ok(value) = obj.get_attr(name, vm) else { continue; @@ -206,7 +267,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) Ok(state) } -// object_getstate in CPython +// object_getstate // fn object_getstate( // obj: &PyObject, // required: bool, @@ -235,7 +296,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) // getstate.call((), vm) // } -#[pyclass(with(Constructor), flags(BASETYPE))] +#[pyclass(with(Constructor, Initializer), flags(BASETYPE))] impl PyBaseObject { #[pymethod(raw)] fn __getstate__(vm: &VirtualMachine, args: FuncArgs) -> PyResult { @@ -269,10 +330,7 @@ impl PyBaseObject { } } PyComparisonOp::Ne => { - let cmp = zelf - .class() - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); + let cmp = zelf.class().slots.richcompare.load().unwrap(); let value = match cmp(zelf, other, PyComparisonOp::Eq, vm)? { Either::A(obj) => PyArithmeticValue::from_object(vm, obj) .map(|obj| obj.try_to_bool(vm)) @@ -286,66 +344,6 @@ impl PyBaseObject { Ok(res) } - /// Return self==value. - #[pymethod] - fn __eq__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Eq, vm) - } - - /// Return self!=value. - #[pymethod] - fn __ne__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Ne, vm) - } - - /// Return self<value. - #[pymethod] - fn __lt__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Lt, vm) - } - - /// Return self<=value. - #[pymethod] - fn __le__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Le, vm) - } - - /// Return self>=value. - #[pymethod] - fn __ge__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Ge, vm) - } - - /// Return self>value. - #[pymethod] - fn __gt__( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Gt, vm) - } - /// Implement setattr(self, name, value). #[pymethod] fn __setattr__( @@ -374,8 +372,8 @@ impl PyBaseObject { } /// Return str(self). - #[pymethod] - fn __str__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + #[pyslot] + fn slot_str(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { // FIXME: try tp_repr first and fallback to object.__repr__ zelf.repr(vm) } @@ -410,12 +408,6 @@ impl PyBaseObject { } } - /// Return repr(self). - #[pymethod] - fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Self::slot_repr(&zelf, vm) - } - #[pyclassmethod] fn __subclasshook__(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { vm.ctx.not_implemented() @@ -444,31 +436,58 @@ impl PyBaseObject { obj.str(vm) } - #[pyslot] - #[pymethod] - fn __init__(_zelf: PyObjectRef, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<()> { - Ok(()) - } - - #[pygetset(name = "__class__")] - fn get_class(obj: PyObjectRef) -> PyTypeRef { + #[pygetset] + fn __class__(obj: PyObjectRef) -> PyTypeRef { obj.class().to_owned() } - #[pygetset(name = "__class__", setter)] - fn set_class(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pygetset(setter)] + fn set___class__( + instance: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { match value.downcast::<PyType>() { Ok(cls) => { - let both_module = instance.class().fast_issubclass(vm.ctx.types.module_type) + let current_cls = instance.class(); + let both_module = current_cls.fast_issubclass(vm.ctx.types.module_type) && cls.fast_issubclass(vm.ctx.types.module_type); - let both_mutable = !instance - .class() + let both_mutable = !current_cls .slots .flags .has_feature(PyTypeFlags::IMMUTABLETYPE) && !cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE); // FIXME(#1979) cls instances might have a payload if both_mutable || both_module { + let has_dict = + |typ: &Py<PyType>| typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT); + // Compare slots tuples + let slots_equal = match ( + current_cls + .heaptype_ext + .as_ref() + .and_then(|e| e.slots.as_ref()), + cls.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + ) { + (Some(a), Some(b)) => { + a.len() == b.len() + && a.iter() + .zip(b.iter()) + .all(|(x, y)| x.as_str() == y.as_str()) + } + (None, None) => true, + _ => false, + }; + if current_cls.slots.basicsize != cls.slots.basicsize + || !slots_equal + || has_dict(current_cls) != has_dict(&cls) + { + return Err(vm.new_type_error(format!( + "__class__ assignment: '{}' object layout differs from '{}'", + cls.name(), + current_cls.name() + ))); + } instance.set_class(cls, vm); Ok(()) } else { @@ -523,12 +542,6 @@ impl PyBaseObject { Ok(zelf.get_id() as _) } - /// Return hash(self). - #[pymethod] - fn __hash__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { - Self::slot_hash(&zelf, vm) - } - #[pymethod] fn __sizeof__(zelf: PyObjectRef) -> usize { zelf.class().slots.basicsize @@ -545,14 +558,193 @@ pub fn object_set_dict(obj: PyObjectRef, dict: PyDictRef, vm: &VirtualMachine) - } pub fn init(ctx: &Context) { + // Manually set init slot - derive macro doesn't generate extend_slots + // for trait impl that overrides #[pyslot] method + ctx.types + .object_type + .slots + .init + .store(Some(<PyBaseObject as Initializer>::slot_init)); PyBaseObject::extend_class(ctx, ctx.types.object_type); } +/// Get arguments for __new__ from __getnewargs_ex__ or __getnewargs__ +/// Returns (args, kwargs) tuple where either can be None +fn get_new_arguments( + obj: &PyObject, + vm: &VirtualMachine, +) -> PyResult<(Option<super::PyTupleRef>, Option<super::PyDictRef>)> { + // First try __getnewargs_ex__ + if let Some(getnewargs_ex) = vm.get_special_method(obj, identifier!(vm, __getnewargs_ex__))? { + let newargs = getnewargs_ex.invoke((), vm)?; + + let newargs_tuple: PyRef<super::PyTuple> = newargs.downcast().map_err(|obj| { + vm.new_type_error(format!( + "__getnewargs_ex__ should return a tuple, not '{}'", + obj.class().name() + )) + })?; + + if newargs_tuple.len() != 2 { + return Err(vm.new_value_error(format!( + "__getnewargs_ex__ should return a tuple of length 2, not {}", + newargs_tuple.len() + ))); + } + + let args = newargs_tuple.as_slice()[0].clone(); + let kwargs = newargs_tuple.as_slice()[1].clone(); + + let args_tuple: PyRef<super::PyTuple> = args.downcast().map_err(|obj| { + vm.new_type_error(format!( + "first item of the tuple returned by __getnewargs_ex__ must be a tuple, not '{}'", + obj.class().name() + )) + })?; + + let kwargs_dict: PyRef<super::PyDict> = kwargs.downcast().map_err(|obj| { + vm.new_type_error(format!( + "second item of the tuple returned by __getnewargs_ex__ must be a dict, not '{}'", + obj.class().name() + )) + })?; + + return Ok((Some(args_tuple), Some(kwargs_dict))); + } + + // Fall back to __getnewargs__ + if let Some(getnewargs) = vm.get_special_method(obj, identifier!(vm, __getnewargs__))? { + let args = getnewargs.invoke((), vm)?; + + let args_tuple: PyRef<super::PyTuple> = args.downcast().map_err(|obj| { + vm.new_type_error(format!( + "__getnewargs__ should return a tuple, not '{}'", + obj.class().name() + )) + })?; + + return Ok((Some(args_tuple), None)); + } + + // No __getnewargs_ex__ or __getnewargs__ + Ok((None, None)) +} + +/// Check if __getstate__ is overridden by comparing with object.__getstate__ +fn is_getstate_overridden(obj: &PyObject, vm: &VirtualMachine) -> bool { + let obj_cls = obj.class(); + let object_type = vm.ctx.types.object_type; + + // If the class is object itself, not overridden + if obj_cls.is(object_type) { + return false; + } + + // Check if __getstate__ in the MRO comes from object or elsewhere + // If the type has its own __getstate__, it's overridden + if let Some(getstate) = obj_cls.get_attr(identifier!(vm, __getstate__)) + && let Some(obj_getstate) = object_type.get_attr(identifier!(vm, __getstate__)) + { + return !getstate.is(&obj_getstate); + } + false +} + +/// object_getstate - calls __getstate__ method or default implementation +fn object_getstate(obj: &PyObject, required: bool, vm: &VirtualMachine) -> PyResult { + // If __getstate__ is not overridden, use the default implementation with required flag + if !is_getstate_overridden(obj, vm) { + return object_getstate_default(obj, required, vm); + } + + // __getstate__ is overridden, call it without required + let getstate = obj.get_attr(identifier!(vm, __getstate__), vm)?; + getstate.call((), vm) +} + +/// Get list items iterator if obj is a list (or subclass), None iterator otherwise +fn get_items_iter(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + let listitems: PyObjectRef = if obj.fast_isinstance(vm.ctx.types.list_type) { + obj.get_iter(vm)?.into() + } else { + vm.ctx.none() + }; + + let dictitems: PyObjectRef = if obj.fast_isinstance(vm.ctx.types.dict_type) { + let items = vm.call_method(obj, "items", ())?; + items.get_iter(vm)?.into() + } else { + vm.ctx.none() + }; + + Ok((listitems, dictitems)) +} + +/// reduce_newobj - creates reduce tuple for protocol >= 2 +fn reduce_newobj(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Check if type has tp_new + let cls = obj.class(); + if cls.slots.new.load().is_none() { + return Err(vm.new_type_error(format!("cannot pickle '{}' object", cls.name()))); + } + + let (args, kwargs) = get_new_arguments(&obj, vm)?; + + let copyreg = vm.import("copyreg", 0)?; + + let has_args = args.is_some(); + + let (newobj, newargs): (PyObjectRef, PyObjectRef) = if kwargs.is_none() + || kwargs.as_ref().is_some_and(|k| k.is_empty()) + { + // Use copyreg.__newobj__ + let newobj = copyreg.get_attr("__newobj__", vm)?; + + let args_vec: Vec<PyObjectRef> = args.map(|a| a.as_slice().to_vec()).unwrap_or_default(); + + // Create (cls, *args) tuple + let mut newargs_vec: Vec<PyObjectRef> = vec![cls.to_owned().into()]; + newargs_vec.extend(args_vec); + let newargs = vm.ctx.new_tuple(newargs_vec); + + (newobj, newargs.into()) + } else { + // args == NULL with non-empty kwargs is BadInternalCall + let Some(args) = args else { + return Err(vm.new_system_error("bad internal call".to_owned())); + }; + // Use copyreg.__newobj_ex__ + let newobj = copyreg.get_attr("__newobj_ex__", vm)?; + let args_tuple: PyObjectRef = args.into(); + let kwargs_dict: PyObjectRef = kwargs + .map(|k| k.into()) + .unwrap_or_else(|| vm.ctx.new_dict().into()); + + let newargs = vm + .ctx + .new_tuple(vec![cls.to_owned().into(), args_tuple, kwargs_dict]); + (newobj, newargs.into()) + }; + + // Determine if state is required + // required = !(has_args || is_list || is_dict) + let is_list = obj.fast_isinstance(vm.ctx.types.list_type); + let is_dict = obj.fast_isinstance(vm.ctx.types.dict_type); + let required = !(has_args || is_list || is_dict); + + let state = object_getstate(&obj, required, vm)?; + + let (listitems, dictitems) = get_items_iter(&obj, vm)?; + + let result = vm + .ctx + .new_tuple(vec![newobj, newargs, state, listitems, dictitems]); + Ok(result.into()) +} + fn common_reduce(obj: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { if proto >= 2 { - let reducelib = vm.import("__reducelib", 0)?; - let reduce_2 = reducelib.get_attr("reduce_2", vm)?; - reduce_2.call((obj,), vm) + reduce_newobj(obj, vm) } else { let copyreg = vm.import("copyreg", 0)?; let reduce_ex = copyreg.get_attr("_reduce_ex", vm)?; diff --git a/crates/vm/src/builtins/property.rs b/crates/vm/src/builtins/property.rs index 1a2e04ee8b0..8cc9ba92e92 100644 --- a/crates/vm/src/builtins/property.rs +++ b/crates/vm/src/builtins/property.rs @@ -10,7 +10,7 @@ use crate::{ function::{FuncArgs, PySetterValue}, types::{Constructor, GetDescriptor, Initializer}, }; -use std::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; #[pyclass(module = false, name = "property", traverse)] #[derive(Debug)] @@ -21,7 +21,7 @@ pub struct PyProperty { doc: PyRwLock<Option<PyObjectRef>>, name: PyRwLock<Option<PyObjectRef>>, #[pytraverse(skip)] - getter_doc: std::sync::atomic::AtomicBool, + getter_doc: core::sync::atomic::AtomicBool, } impl PyPayload for PyProperty { @@ -55,7 +55,8 @@ impl GetDescriptor for PyProperty { let (zelf, obj) = Self::_unwrap(&zelf_obj, obj, vm)?; if vm.is_none(&obj) { Ok(zelf_obj) - } else if let Some(getter) = zelf.getter.read().as_ref() { + } else if let Some(getter) = zelf.getter.read().clone() { + // Clone and release lock before calling Python code to prevent deadlock getter.call((obj,), vm) } else { let error_msg = zelf.format_property_error(&obj, "getter", vm)?; @@ -70,12 +71,12 @@ impl PyProperty { // Returns the name if available, None if not found, or propagates errors fn get_property_name(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { // First check if name was set via __set_name__ - if let Some(name) = self.name.read().as_ref() { - return Ok(Some(name.clone())); + if let Some(name) = self.name.read().clone() { + return Ok(Some(name)); } - let getter = self.getter.read(); - let Some(getter) = getter.as_ref() else { + // Clone and release lock before calling Python code to prevent deadlock + let Some(getter) = self.getter.read().clone() else { return Ok(None); }; @@ -105,7 +106,8 @@ impl PyProperty { let zelf = zelf.try_to_ref::<Self>(vm)?; match value { PySetterValue::Assign(value) => { - if let Some(setter) = zelf.setter.read().as_ref() { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(setter) = zelf.setter.read().clone() { setter.call((obj, value), vm).map(drop) } else { let error_msg = zelf.format_property_error(&obj, "setter", vm)?; @@ -113,7 +115,8 @@ impl PyProperty { } } PySetterValue::Delete => { - if let Some(deleter) = zelf.deleter.read().as_ref() { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(deleter) = zelf.deleter.read().clone() { deleter.call((obj,), vm).map(drop) } else { let error_msg = zelf.format_property_error(&obj, "deleter", vm)?; @@ -122,19 +125,6 @@ impl PyProperty { } } } - #[pymethod] - fn __set__( - zelf: PyObjectRef, - obj: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Assign(value), vm) - } - #[pymethod] - fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Delete, vm) - } // Access functions @@ -266,30 +256,31 @@ impl PyProperty { #[pygetset] fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyResult { // Helper to check if a method is abstract - let is_abstract = |method: &PyObjectRef| -> PyResult<bool> { + let is_abstract = |method: &PyObject| -> PyResult<bool> { match method.get_attr("__isabstractmethod__", vm) { Ok(isabstract) => isabstract.try_to_bool(vm), Err(_) => Ok(false), } }; + // Clone and release lock before calling Python code to prevent deadlock // Check getter - if let Some(getter) = self.getter.read().as_ref() - && is_abstract(getter)? + if let Some(getter) = self.getter.read().clone() + && is_abstract(&getter)? { return Ok(vm.ctx.new_bool(true).into()); } // Check setter - if let Some(setter) = self.setter.read().as_ref() - && is_abstract(setter)? + if let Some(setter) = self.setter.read().clone() + && is_abstract(&setter)? { return Ok(vm.ctx.new_bool(true).into()); } // Check deleter - if let Some(deleter) = self.deleter.read().as_ref() - && is_abstract(deleter)? + if let Some(deleter) = self.deleter.read().clone() + && is_abstract(&deleter)? { return Ok(vm.ctx.new_bool(true).into()); } @@ -299,7 +290,8 @@ impl PyProperty { #[pygetset(setter)] fn set___isabstractmethod__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if let Some(getter) = self.getter.read().to_owned() { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(getter) = self.getter.read().clone() { getter.set_attr("__isabstractmethod__", value, vm)?; } Ok(()) @@ -309,7 +301,7 @@ impl PyProperty { #[cold] fn format_property_error( &self, - obj: &PyObjectRef, + obj: &PyObject, error_type: &str, vm: &VirtualMachine, ) -> PyResult<String> { @@ -356,7 +348,7 @@ impl Initializer for PyProperty { let mut getter_doc = false; // Helper to get doc from getter - let get_getter_doc = |fget: &PyObjectRef| -> Option<PyObjectRef> { + let get_getter_doc = |fget: &PyObject| -> Option<PyObjectRef> { fget.get_attr("__doc__", vm) .ok() .filter(|doc| !vm.is_none(doc)) diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index 3edd130ee28..92de2463e2c 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -2,24 +2,24 @@ use super::{ PyGenericAlias, PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, builtins_iter, tuple::tuple_hash, }; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, atomic_func, class::PyClassImpl, common::hash::PyHash, function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, - protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, + protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, types::{ - AsMapping, AsSequence, Comparable, Hashable, IterNext, Iterable, PyComparisonOp, - Representable, SelfIter, Unconstructible, + AsMapping, AsNumber, AsSequence, Comparable, Hashable, IterNext, Iterable, PyComparisonOp, + Representable, SelfIter, }, }; +use core::cmp::max; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Signed, ToPrimitive, Zero}; -use std::cmp::max; -use std::sync::LazyLock; // Search flag passed to iter_search enum SearchType { @@ -176,6 +176,7 @@ pub fn init(context: &Context) { with( Py, AsMapping, + AsNumber, AsSequence, Hashable, Comparable, @@ -264,16 +265,10 @@ impl PyRange { ) } - #[pymethod] fn __len__(&self) -> BigInt { self.compute_length() } - #[pymethod] - fn __bool__(&self) -> bool { - !self.is_empty() - } - #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef) { let range_parameters: Vec<PyObjectRef> = [&self.start, &self.stop, &self.step] @@ -284,7 +279,6 @@ impl PyRange { (vm.ctx.types.range_type.to_owned(), range_parameters_tuple) } - #[pymethod] fn __getitem__(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult { match RangeIndex::try_from_object(vm, subscript)? { RangeIndex::Slice(slice) => { @@ -346,7 +340,6 @@ impl Py<PyRange> { } } - #[pymethod] fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> bool { self.contains_inner(&needle, vm) } @@ -426,6 +419,19 @@ impl AsSequence for PyRange { } } +impl AsNumber for PyRange { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyRange>().unwrap(); + Ok(!zelf.is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + impl Hashable for PyRange { fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { let length = zelf.compute_length(); @@ -548,7 +554,7 @@ impl PyPayload for PyLongRangeIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyLongRangeIterator { #[pymethod] fn __length_hint__(&self) -> BigInt { @@ -577,7 +583,6 @@ impl PyLongRangeIterator { ) } } -impl Unconstructible for PyLongRangeIterator {} impl SelfIter for PyLongRangeIterator {} impl IterNext for PyLongRangeIterator { @@ -614,7 +619,7 @@ impl PyPayload for PyRangeIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyRangeIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -640,7 +645,6 @@ impl PyRangeIterator { ) } } -impl Unconstructible for PyRangeIterator {} impl SelfIter for PyRangeIterator {} impl IterNext for PyRangeIterator { @@ -666,7 +670,7 @@ fn range_iter_reduce( index: usize, vm: &VirtualMachine, ) -> PyResult<PyTupleRef> { - let iter = builtins_iter(vm).to_owned(); + let iter = builtins_iter(vm); let stop = start.clone() + length * step.clone(); let range = PyRange { start: PyInt::from(start).into_ref(&vm.ctx), diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 7fde8d32781..17f406bbcbe 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -5,6 +5,7 @@ use super::{ IterStatus, PositionIterInternal, PyDict, PyDictRef, PyGenericAlias, PyTupleRef, PyType, PyTypeRef, builtins_iter, }; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, @@ -18,17 +19,18 @@ use crate::{ types::AsNumber, types::{ AsSequence, Comparable, Constructor, DefaultConstructor, Hashable, Initializer, IterNext, - Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, + Iterable, PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, vm::VirtualMachine, }; +use alloc::fmt; +use core::borrow::Borrow; +use core::ops::Deref; use rustpython_common::{ atomic::{Ordering, PyAtomic, Radium}, hash, }; -use std::sync::LazyLock; -use std::{fmt, ops::Deref}; pub type SetContentType = dict_inner::Dict<()>; @@ -50,7 +52,7 @@ impl PySet { fn fold_op( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, vm: &VirtualMachine, ) -> PyResult<Self> { @@ -68,7 +70,7 @@ impl PySet { Ok(Self { inner: self .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, }) } } @@ -111,7 +113,7 @@ impl PyFrozenSet { fn fold_op( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, vm: &VirtualMachine, ) -> PyResult<Self> { @@ -130,7 +132,7 @@ impl PyFrozenSet { Ok(Self { inner: self .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, ..Default::default() }) } @@ -191,7 +193,7 @@ impl PySetInner { fn fold_op<O>( &self, - others: impl std::iter::Iterator<Item = O>, + others: impl core::iter::Iterator<Item = O>, op: fn(&Self, O, &VirtualMachine) -> PyResult<Self>, vm: &VirtualMachine, ) -> PyResult<Self> { @@ -352,7 +354,7 @@ impl PySetInner { fn update( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -395,7 +397,7 @@ impl PySetInner { fn intersection_update( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, vm: &VirtualMachine, ) -> PyResult<()> { let temp_inner = self.fold_op(others, Self::intersection, vm)?; @@ -408,7 +410,7 @@ impl PySetInner { fn difference_update( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -422,7 +424,7 @@ impl PySetInner { fn symmetric_difference_update( &self, - others: impl std::iter::Iterator<Item = ArgIterable>, + others: impl core::iter::Iterator<Item = ArgIterable>, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -532,14 +534,17 @@ fn reduce_set( flags(BASETYPE, _MATCH_SELF) )] impl PySet { - #[pymethod] fn __len__(&self) -> usize { self.inner.len() } + fn __contains__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.contains(needle, vm) + } + #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + self.inner.sizeof() + core::mem::size_of::<Self>() + self.inner.sizeof() } #[pymethod] @@ -549,11 +554,6 @@ impl PySet { } } - #[pymethod] - fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.contains(&needle, vm) - } - #[pymethod] fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { self.fold_op(others.into_iter(), PySetInner::union, vm) @@ -593,8 +593,6 @@ impl PySet { self.inner.isdisjoint(other, vm) } - #[pymethod(name = "__ror__")] - #[pymethod] fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { if let Ok(other) = AnySet::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( @@ -607,8 +605,6 @@ impl PySet { } } - #[pymethod(name = "__rand__")] - #[pymethod] fn __and__( &self, other: PyObjectRef, @@ -625,7 +621,6 @@ impl PySet { } } - #[pymethod] fn __sub__( &self, other: PyObjectRef, @@ -642,7 +637,6 @@ impl PySet { } } - #[pymethod] fn __rsub__( zelf: PyRef<Self>, other: PyObjectRef, @@ -659,8 +653,6 @@ impl PySet { } } - #[pymethod(name = "__rxor__")] - #[pymethod] fn __xor__( &self, other: PyObjectRef, @@ -704,7 +696,6 @@ impl PySet { self.inner.pop(vm) } - #[pymethod] fn __ior__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { zelf.inner.update(set.into_iterable_iter(vm)?, vm)?; Ok(zelf) @@ -728,10 +719,11 @@ impl PySet { Ok(()) } - #[pymethod] fn __iand__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .intersection_update(std::iter::once(set.into_iterable(vm)?), vm)?; + if !set.is(zelf.as_object()) { + zelf.inner + .intersection_update(core::iter::once(set.into_iterable(vm)?), vm)?; + } Ok(zelf) } @@ -741,10 +733,13 @@ impl PySet { Ok(()) } - #[pymethod] fn __isub__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .difference_update(set.into_iterable_iter(vm)?, vm)?; + if set.is(zelf.as_object()) { + zelf.inner.clear(); + } else { + zelf.inner + .difference_update(set.into_iterable_iter(vm)?, vm)?; + } Ok(zelf) } @@ -759,10 +754,13 @@ impl PySet { Ok(()) } - #[pymethod] fn __ixor__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .symmetric_difference_update(set.into_iterable_iter(vm)?, vm)?; + if set.is(zelf.as_object()) { + zelf.inner.clear(); + } else { + zelf.inner + .symmetric_difference_update(set.into_iterable_iter(vm)?, vm)?; + } Ok(zelf) } @@ -798,9 +796,9 @@ impl AsSequence for PySet { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PySet::sequence_downcast(seq).__len__())), - contains: atomic_func!(|seq, needle, vm| PySet::sequence_downcast(seq) - .inner - .contains(needle, vm)), + contains: atomic_func!( + |seq, needle, vm| PySet::sequence_downcast(seq).__contains__(needle, vm) + ), ..PySequenceMethods::NOT_IMPLEMENTED }); &AS_SEQUENCE @@ -829,30 +827,77 @@ impl Iterable for PySet { impl AsNumber for PySet { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { + // Binary ops check both operands are sets (like CPython's set_sub, etc.) + // This is needed because __rsub__ swaps operands: a.__rsub__(b) calls subtract(b, a) subtract: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PySet>() { a.__sub__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + // When called via __rsub__, a might be PyFrozenSet + a.__sub__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), and: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PySet>() { a.__and__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__and__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), xor: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PySet>() { a.__xor__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__xor__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), or: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PySet>() { a.__or__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__or__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -971,14 +1016,17 @@ impl Constructor for PyFrozenSet { ) )] impl PyFrozenSet { - #[pymethod] fn __len__(&self) -> usize { self.inner.len() } + fn __contains__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.contains(needle, vm) + } + #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + self.inner.sizeof() + core::mem::size_of::<Self>() + self.inner.sizeof() } #[pymethod] @@ -994,11 +1042,6 @@ impl PyFrozenSet { } } - #[pymethod] - fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.contains(&needle, vm) - } - #[pymethod] fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { self.fold_op(others.into_iter(), PySetInner::union, vm) @@ -1038,8 +1081,6 @@ impl PyFrozenSet { self.inner.isdisjoint(other, vm) } - #[pymethod(name = "__ror__")] - #[pymethod] fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { if let Ok(set) = AnySet::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( @@ -1052,8 +1093,6 @@ impl PyFrozenSet { } } - #[pymethod(name = "__rand__")] - #[pymethod] fn __and__( &self, other: PyObjectRef, @@ -1070,7 +1109,6 @@ impl PyFrozenSet { } } - #[pymethod] fn __sub__( &self, other: PyObjectRef, @@ -1087,7 +1125,6 @@ impl PyFrozenSet { } } - #[pymethod] fn __rsub__( zelf: PyRef<Self>, other: PyObjectRef, @@ -1105,8 +1142,6 @@ impl PyFrozenSet { } } - #[pymethod(name = "__rxor__")] - #[pymethod] fn __xor__( &self, other: PyObjectRef, @@ -1141,9 +1176,9 @@ impl AsSequence for PyFrozenSet { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyFrozenSet::sequence_downcast(seq).__len__())), - contains: atomic_func!(|seq, needle, vm| PyFrozenSet::sequence_downcast(seq) - .inner - .contains(needle, vm)), + contains: atomic_func!( + |seq, needle, vm| PyFrozenSet::sequence_downcast(seq).__contains__(needle, vm) + ), ..PySequenceMethods::NOT_IMPLEMENTED }); &AS_SEQUENCE @@ -1195,30 +1230,53 @@ impl Iterable for PyFrozenSet { impl AsNumber for PyFrozenSet { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { + // Binary ops check both operands are sets (like CPython's set_sub, etc.) + // __rsub__ swaps operands. Result type follows first operand's type. subtract: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PyFrozenSet>() { a.__sub__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + // When called via __rsub__, a might be PySet - return set (not frozenset) + a.__sub__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), and: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PyFrozenSet>() { a.__and__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__and__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), xor: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PyFrozenSet>() { a.__xor__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__xor__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), or: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } if let Some(a) = a.downcast_ref::<PyFrozenSet>() { a.__or__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__or__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -1250,7 +1308,21 @@ struct AnySet { object: PyObjectRef, } +impl Borrow<PyObject> for AnySet { + #[inline(always)] + fn borrow(&self) -> &PyObject { + &self.object + } +} + impl AnySet { + /// Check if object is a set or frozenset (including subclasses) + /// Equivalent to CPython's PyAnySet_Check + fn check(obj: &PyObject, vm: &VirtualMachine) -> bool { + let ctx = &vm.ctx; + obj.fast_isinstance(ctx.types.set_type) || obj.fast_isinstance(ctx.types.frozenset_type) + } + fn into_iterable(self, vm: &VirtualMachine) -> PyResult<ArgIterable> { self.object.try_into_value(vm) } @@ -1258,8 +1330,8 @@ impl AnySet { fn into_iterable_iter( self, vm: &VirtualMachine, - ) -> PyResult<impl std::iter::Iterator<Item = ArgIterable>> { - Ok(std::iter::once(self.into_iterable(vm)?)) + ) -> PyResult<impl core::iter::Iterator<Item = ArgIterable>> { + Ok(core::iter::once(self.into_iterable(vm)?)) } fn as_inner(&self) -> &PySetInner { @@ -1304,7 +1376,7 @@ impl PyPayload for PySetIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PySetIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -1318,7 +1390,7 @@ impl PySetIterator { ) -> PyResult<(PyObjectRef, (PyObjectRef,))> { let internal = zelf.internal.lock(); Ok(( - builtins_iter(vm).to_owned(), + builtins_iter(vm), (vm.ctx .new_list(match &internal.status { IterStatus::Exhausted => vec![], @@ -1330,7 +1402,6 @@ impl PySetIterator { )) } } -impl Unconstructible for PySetIterator {} impl SelfIter for PySetIterator {} impl IterNext for PySetIterator { diff --git a/crates/vm/src/builtins/singletons.rs b/crates/vm/src/builtins/singletons.rs index bdc032cc865..169104efeb3 100644 --- a/crates/vm/src/builtins/singletons.rs +++ b/crates/vm/src/builtins/singletons.rs @@ -50,12 +50,7 @@ impl Constructor for PyNone { } #[pyclass(with(Constructor, AsNumber, Representable))] -impl PyNone { - #[pymethod] - const fn __bool__(&self) -> bool { - false - } -} +impl PyNone {} impl Representable for PyNone { #[inline] @@ -103,22 +98,28 @@ impl Constructor for PyNotImplemented { } } -#[pyclass(with(Constructor))] +#[pyclass(with(Constructor, AsNumber, Representable))] impl PyNotImplemented { - // TODO: As per https://bugs.python.org/issue35712, using NotImplemented - // in boolean contexts will need to raise a DeprecationWarning in 3.9 - // and, eventually, a TypeError. - #[pymethod] - const fn __bool__(&self) -> bool { - true - } - #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyStrRef { vm.ctx.names.NotImplemented.to_owned() } } +impl AsNumber for PyNotImplemented { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|_number, vm| { + Err(vm.new_type_error( + "NotImplemented should not be used in a boolean context".to_owned(), + )) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + impl Representable for PyNotImplemented { #[inline] fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 09a6c462ed5..3aa23b0746c 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -174,6 +174,7 @@ impl PySlice { #[pymethod] fn indices(&self, length: ArgIndex, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let length = length.into_int_ref(); let length = length.as_bigint(); if length.is_negative() { return Err(vm.new_value_error("length should not be negative.")); diff --git a/crates/vm/src/builtins/staticmethod.rs b/crates/vm/src/builtins/staticmethod.rs index 5d2474a567c..ac363415a9f 100644 --- a/crates/vm/src/builtins/staticmethod.rs +++ b/crates/vm/src/builtins/staticmethod.rs @@ -3,7 +3,7 @@ use crate::{ Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyMutex, - function::FuncArgs, + function::{FuncArgs, PySetterValue}, types::{Callable, Constructor, GetDescriptor, Initializer, Representable}, }; @@ -121,6 +121,27 @@ impl PyStaticMethod { self.callable.lock().get_attr("__annotations__", vm) } + #[pygetset(setter)] + fn set___annotations__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotations__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotate__", vm) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotate__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + #[pygetset] fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyObjectRef { match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 9b05e195722..a1f0de1810f 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -1,8 +1,12 @@ use super::{ PositionIterInternal, PyBytesRef, PyDict, PyTupleRef, PyType, PyTypeRef, int::{PyInt, PyIntRef}, - iter::IterStatus::{self, Exhausted}, + iter::{ + IterStatus::{self, Exhausted}, + builtins_iter, + }, }; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromBorrowedObject, VirtualMachine, @@ -21,11 +25,13 @@ use crate::{ sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + PyComparisonOp, Representable, SelfIter, }, }; +use alloc::{borrow::Cow, fmt}; use ascii::{AsciiChar, AsciiStr, AsciiString}; use bstr::ByteSlice; +use core::{char, mem, ops::Range}; use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_common::{ @@ -37,8 +43,6 @@ use rustpython_common::{ str::DeduceStrKind, wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}, }; -use std::{borrow::Cow, char, fmt, ops::Range}; -use std::{mem, sync::LazyLock}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; use unic_ucd_ident::{is_xid_continue, is_xid_start}; @@ -191,8 +195,8 @@ impl From<StrData> for PyStr { } } -impl<'a> From<std::borrow::Cow<'a, str>> for PyStr { - fn from(s: std::borrow::Cow<'a, str>) -> Self { +impl<'a> From<alloc::borrow::Cow<'a, str>> for PyStr { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { s.into_owned().into() } } @@ -231,7 +235,10 @@ pub trait AsPyStr<'a> where Self: 'a, { - #[allow(clippy::wrong_self_convention)] // to implement on refs + #[allow( + clippy::wrong_self_convention, + reason = "this trait is intentionally implemented for references" + )] fn as_pystr(self, ctx: &Context) -> &'a Py<PyStr>; } @@ -282,7 +289,7 @@ impl PyPayload for PyStrIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyStrIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -300,15 +307,16 @@ impl PyStrIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .0 - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().0.reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_str.to_owned().into(), + vm, + ) } } -impl Unconstructible for PyStrIterator {} - impl SelfIter for PyStrIterator {} impl IterNext for PyStrIterator { @@ -443,7 +451,7 @@ impl PyStr { self.data.as_str() } - fn ensure_valid_utf8(&self, vm: &VirtualMachine) -> PyResult<()> { + pub(crate) fn ensure_valid_utf8(&self, vm: &VirtualMachine) -> PyResult<()> { if self.is_utf8() { Ok(()) } else { @@ -531,7 +539,6 @@ impl Py<PyStr> { #[pyclass( flags(BASETYPE, _MATCH_SELF), with( - PyRef, AsMapping, AsNumber, AsSequence, @@ -543,7 +550,6 @@ impl Py<PyStr> { ) )] impl PyStr { - #[pymethod] fn __add__(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(other) = other.downcast_ref::<Self>() { let bytes = zelf.as_wtf8().py_add(other.as_wtf8()); @@ -575,7 +581,6 @@ impl PyStr { } } - #[pymethod] fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self._contains(&needle, vm) } @@ -588,7 +593,6 @@ impl PyStr { Ok(item) } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } @@ -627,19 +631,17 @@ impl PyStr { self.data.char_len() } - #[pymethod(name = "isascii")] + #[pymethod] #[inline(always)] - pub const fn is_ascii(&self) -> bool { + pub const fn isascii(&self) -> bool { matches!(self.kind(), StrKind::Ascii) } #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::<Self>() + self.byte_len() * std::mem::size_of::<u8>() + core::mem::size_of::<Self>() + self.byte_len() * core::mem::size_of::<u8>() } - #[pymethod(name = "__rmul__")] - #[pymethod] fn __mul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Self::repeat(zelf, value.into(), vm) } @@ -671,8 +673,19 @@ impl PyStr { // casefold is much more aggressive than lower #[pymethod] - fn casefold(&self) -> String { - caseless::default_case_fold_str(self.as_str()) + fn casefold(&self) -> Self { + match self.as_str_kind() { + PyKindStr::Ascii(s) => caseless::default_case_fold_str(s.as_str()).into(), + PyKindStr::Utf8(s) => caseless::default_case_fold_str(s).into(), + PyKindStr::Wtf8(w) => w + .chunks() + .map(|c| match c { + Wtf8Chunk::Utf8(s) => Wtf8Buf::from_string(caseless::default_case_fold_str(s)), + Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c), + }) + .collect::<Wtf8Buf>() + .into(), + } } #[pymethod] @@ -938,16 +951,10 @@ impl PyStr { && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) } - #[pymethod(name = "__mod__")] - fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { cformat_string(vm, self.as_wtf8(), values) } - #[pymethod] - fn __rmod__(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - #[pymethod] fn format(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { let format_str = @@ -962,7 +969,7 @@ impl PyStr { format_map(&format_string, &mapping, vm) } - #[pymethod(name = "__format__")] + #[pymethod] fn __format__( zelf: PyRef<PyStr>, spec: PyStrRef, @@ -1037,7 +1044,7 @@ impl PyStr { #[pymethod] fn replace(&self, args: ReplaceArgs) -> Wtf8Buf { - use std::cmp::Ordering; + use core::cmp::Ordering; let s = self.as_wtf8(); let ReplaceArgs { old, new, count } = args; @@ -1328,7 +1335,7 @@ impl PyStr { } #[pymethod] - fn isidentifier(&self) -> bool { + pub fn isidentifier(&self) -> bool { let Some(s) = self.to_str() else { return false }; let mut chars = s.chars(); let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c)); @@ -1353,7 +1360,7 @@ impl PyStr { let ch = bigint .as_bigint() .to_u32() - .and_then(std::char::from_u32) + .and_then(core::char::from_u32) .ok_or_else(|| { vm.new_value_error("character mapping must be in range(0x110000)") })?; @@ -1450,15 +1457,16 @@ impl PyStr { fn __getnewargs__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { (zelf.as_str(),).to_pyobject(vm) } -} -#[pyclass] -impl PyRef<PyStr> { #[pymethod] - fn __str__(self, vm: &VirtualMachine) -> PyRefExact<PyStr> { - self.into_exact_or(&vm.ctx, |zelf| { - PyStr::from(zelf.data.clone()).into_exact_ref(&vm.ctx) - }) + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + if zelf.class().is(vm.ctx.types.str_type) { + // Already exact str, just return a reference + Ok(zelf.to_owned()) + } else { + // Subclass, create a new exact str + Ok(PyStr::from(zelf.data.clone()).into_ref(&vm.ctx)) + } } } @@ -1485,7 +1493,7 @@ impl PyRef<PyStr> { } struct CharLenStr<'a>(&'a str, usize); -impl std::ops::Deref for CharLenStr<'_> { +impl core::ops::Deref for CharLenStr<'_> { type Target = str; fn deref(&self) -> &Self::Target { @@ -1554,7 +1562,7 @@ impl AsNumber for PyStr { static AS_NUMBER: PyNumberMethods = PyNumberMethods { remainder: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::<PyStr>() { - a.modulo(b.to_owned(), vm).to_pyresult(vm) + a.__mod__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -1696,7 +1704,7 @@ pub struct FindArgs { } impl FindArgs { - fn get_value(self, len: usize) -> (PyStrRef, std::ops::Range<usize>) { + fn get_value(self, len: usize) -> (PyStrRef, core::ops::Range<usize>) { let range = adjust_indices(self.start, self.end, len); (self.sub, range) } @@ -1925,9 +1933,16 @@ impl fmt::Display for PyUtf8Str { } impl MaybeTraverse for PyUtf8Str { + const HAS_TRAVERSE: bool = true; + const HAS_CLEAR: bool = false; + fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>) { self.0.try_traverse(traverse_fn); } + + fn try_clear(&mut self, _out: &mut Vec<PyObjectRef>) { + // No clear needed for PyUtf8Str + } } impl PyPayload for PyUtf8Str { @@ -1936,11 +1951,9 @@ impl PyPayload for PyUtf8Str { ctx.types.str_type } - fn payload_type_id() -> std::any::TypeId { - std::any::TypeId::of::<PyStr>() - } + const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::<PyStr>(); - fn validate_downcastable_from(obj: &PyObject) -> bool { + unsafe fn validate_downcastable_from(obj: &PyObject) -> bool { // SAFETY: we know the object is a PyStr in this context let wtf8 = unsafe { obj.downcast_unchecked_ref::<PyStr>() }; wtf8.is_utf8() @@ -1996,8 +2009,8 @@ impl From<char> for PyUtf8Str { } } -impl<'a> From<std::borrow::Cow<'a, str>> for PyUtf8Str { - fn from(s: std::borrow::Cow<'a, str>) -> Self { +impl<'a> From<alloc::borrow::Cow<'a, str>> for PyUtf8Str { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { s.into_owned().into() } } @@ -2119,11 +2132,11 @@ impl AnyStr for str { Self::chars(self) } - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { rustpython_common::str::get_chars(self, range) } @@ -2230,11 +2243,11 @@ impl AnyStr for Wtf8 { self.code_points() } - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { rustpython_common::str::get_codepoints(self, range) } @@ -2352,11 +2365,11 @@ impl AnyStr for AsciiStr { self.chars() } - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } @@ -2427,8 +2440,8 @@ impl PyStrInterned { } } -impl std::fmt::Display for PyStrInterned { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for PyStrInterned { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.data.fmt(f) } } diff --git a/crates/vm/src/builtins/super.rs b/crates/vm/src/builtins/super.rs index f0d873abfbb..b7bc3004332 100644 --- a/crates/vm/src/builtins/super.rs +++ b/crates/vm/src/builtins/super.rs @@ -60,7 +60,7 @@ impl Constructor for PySuper { #[derive(FromArgs)] pub struct InitArgs { - #[pyarg(positional, optional)] + #[pyarg(positional, optional, error_msg = "super() argument 1 must be a type")] py_type: OptionalArg<PyTypeRef>, #[pyarg(positional, optional)] py_obj: OptionalArg<PyObjectRef>, diff --git a/crates/vm/src/builtins/template.rs b/crates/vm/src/builtins/template.rs new file mode 100644 index 00000000000..0496504cbef --- /dev/null +++ b/crates/vm/src/builtins/template.rs @@ -0,0 +1,343 @@ +use super::{PyStr, PyTupleRef, PyType, PyTypeRef, genericalias::PyGenericAlias}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + class::PyClassImpl, + common::lock::LazyLock, + function::{FuncArgs, PyComparisonValue}, + protocol::{PyIterReturn, PySequenceMethods}, + types::{ + AsSequence, Comparable, Constructor, IterNext, Iterable, PyComparisonOp, Representable, + SelfIter, + }, +}; + +use super::interpolation::PyInterpolation; + +/// Template object for t-strings (PEP 750). +/// +/// Represents a template string with interpolated expressions. +#[pyclass(module = "string.templatelib", name = "Template")] +#[derive(Debug, Clone)] +pub struct PyTemplate { + pub strings: PyTupleRef, + pub interpolations: PyTupleRef, +} + +impl PyPayload for PyTemplate { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.template_type + } +} + +impl PyTemplate { + pub fn new(strings: PyTupleRef, interpolations: PyTupleRef) -> Self { + Self { + strings, + interpolations, + } + } +} + +impl Constructor for PyTemplate { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("Template.__new__ only accepts *args arguments")); + } + + let mut strings: Vec<PyObjectRef> = Vec::new(); + let mut interpolations: Vec<PyObjectRef> = Vec::new(); + let mut last_was_str = false; + + for item in args.args.iter() { + if let Ok(s) = item.clone().downcast::<PyStr>() { + if last_was_str { + // Concatenate adjacent strings + if let Some(last) = strings.last_mut() { + let last_str = last.downcast_ref::<PyStr>().unwrap(); + let concatenated = format!("{}{}", last_str.as_str(), s.as_str()); + *last = vm.ctx.new_str(concatenated).into(); + } + } else { + strings.push(s.into()); + } + last_was_str = true; + } else if item.class().is(vm.ctx.types.interpolation_type) { + if !last_was_str { + // Add empty string before interpolation + strings.push(vm.ctx.empty_str.to_owned().into()); + } + interpolations.push(item.clone()); + last_was_str = false; + } else { + return Err(vm.new_type_error(format!( + "Template.__new__ *args need to be of type 'str' or 'Interpolation', got {}", + item.class().name() + ))); + } + } + + if !last_was_str { + // Add trailing empty string + strings.push(vm.ctx.empty_str.to_owned().into()); + } + + Ok(PyTemplate { + strings: vm.ctx.new_tuple(strings), + interpolations: vm.ctx.new_tuple(interpolations), + }) + } +} + +#[pyclass(with(Constructor, Comparable, Iterable, Representable, AsSequence))] +impl PyTemplate { + #[pygetset] + fn strings(&self) -> PyTupleRef { + self.strings.clone() + } + + #[pygetset] + fn interpolations(&self) -> PyTupleRef { + self.interpolations.clone() + } + + #[pygetset] + fn values(&self, vm: &VirtualMachine) -> PyTupleRef { + let values: Vec<PyObjectRef> = self + .interpolations + .iter() + .map(|interp| { + interp + .downcast_ref::<PyInterpolation>() + .map(|i| i.value.clone()) + .unwrap_or_else(|| interp.clone()) + }) + .collect(); + vm.ctx.new_tuple(values) + } + + fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let other = other.downcast_ref::<PyTemplate>().ok_or_else(|| { + vm.new_type_error(format!( + "can only concatenate Template (not '{}') to Template", + other.class().name() + )) + })?; + + // Concatenate the two templates + let mut new_strings: Vec<PyObjectRef> = Vec::new(); + let mut new_interps: Vec<PyObjectRef> = Vec::new(); + + // Add all strings from self except the last one + let self_strings_len = self.strings.len(); + for i in 0..self_strings_len.saturating_sub(1) { + new_strings.push(self.strings.get(i).unwrap().clone()); + } + + // Add all interpolations from self + for interp in self.interpolations.iter() { + new_interps.push(interp.clone()); + } + + // Concatenate last string of self with first string of other + let last_self = self + .strings + .get(self_strings_len.saturating_sub(1)) + .and_then(|s| s.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned())) + .unwrap_or_default(); + let first_other = other + .strings + .first() + .and_then(|s| s.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned())) + .unwrap_or_default(); + let concatenated = format!("{}{}", last_self, first_other); + new_strings.push(vm.ctx.new_str(concatenated).into()); + + // Add remaining strings from other (skip first) + for i in 1..other.strings.len() { + new_strings.push(other.strings.get(i).unwrap().clone()); + } + + // Add all interpolations from other + for interp in other.interpolations.iter() { + new_interps.push(interp.clone()); + } + + let template = PyTemplate { + strings: vm.ctx.new_tuple(new_strings), + interpolations: vm.ctx.new_tuple(new_interps), + }; + + Ok(template.into_ref(&vm.ctx)) + } + + fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + self.concat(&other, vm) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + // Import string.templatelib._template_unpickle + // We need to import string first, then get templatelib from it, + // because import("string.templatelib", 0) with empty from_list returns the top-level module + let string_mod = vm.import("string.templatelib", 0)?; + let templatelib = string_mod.get_attr("templatelib", vm)?; + let unpickle_func = templatelib.get_attr("_template_unpickle", vm)?; + + // Return (func, (strings, interpolations)) + let args = vm.ctx.new_tuple(vec![ + self.strings.clone().into(), + self.interpolations.clone().into(), + ]); + Ok(vm.ctx.new_tuple(vec![unpickle_func, args.into()])) + } +} + +impl AsSequence for PyTemplate { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + concat: atomic_func!(|seq, other, vm| { + let zelf = PyTemplate::sequence_downcast(seq); + zelf.concat(other, vm).map(|t| t.into()) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl Comparable for PyTemplate { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + + let eq = vm.bool_eq(zelf.strings.as_object(), other.strings.as_object())? + && vm.bool_eq( + zelf.interpolations.as_object(), + other.interpolations.as_object(), + )?; + + Ok(eq.into()) + }) + } +} + +impl Iterable for PyTemplate { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyTemplateIter::new(zelf).into_pyobject(vm)) + } +} + +impl Representable for PyTemplate { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let mut parts = Vec::new(); + + let strings_len = zelf.strings.len(); + let interps_len = zelf.interpolations.len(); + + for i in 0..strings_len.max(interps_len * 2 + 1) { + if i % 2 == 0 { + // String position + let idx = i / 2; + if idx < strings_len { + let s = zelf.strings.get(idx).unwrap(); + parts.push(s.repr(vm)?.as_str().to_owned()); + } + } else { + // Interpolation position + let idx = i / 2; + if idx < interps_len { + let interp = zelf.interpolations.get(idx).unwrap(); + parts.push(interp.repr(vm)?.as_str().to_owned()); + } + } + } + + Ok(format!("Template({})", parts.join(", "))) + } +} + +/// Iterator for Template objects +#[pyclass(module = "string.templatelib", name = "TemplateIter")] +#[derive(Debug)] +pub struct PyTemplateIter { + template: PyRef<PyTemplate>, + index: core::sync::atomic::AtomicUsize, + from_strings: core::sync::atomic::AtomicBool, +} + +impl PyPayload for PyTemplateIter { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.template_iter_type + } +} + +impl PyTemplateIter { + fn new(template: PyRef<PyTemplate>) -> Self { + Self { + template, + index: core::sync::atomic::AtomicUsize::new(0), + from_strings: core::sync::atomic::AtomicBool::new(true), + } + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyTemplateIter {} + +impl SelfIter for PyTemplateIter {} + +impl IterNext for PyTemplateIter { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + use core::sync::atomic::Ordering; + + loop { + let from_strings = zelf.from_strings.load(Ordering::SeqCst); + let index = zelf.index.load(Ordering::SeqCst); + + if from_strings { + if index < zelf.template.strings.len() { + let item = zelf.template.strings.get(index).unwrap(); + zelf.from_strings.store(false, Ordering::SeqCst); + + // Skip empty strings + if let Some(s) = item.downcast_ref::<PyStr>() + && s.as_str().is_empty() + { + continue; + } + return Ok(PyIterReturn::Return(item.clone())); + } else { + return Ok(PyIterReturn::StopIteration(None)); + } + } else if index < zelf.template.interpolations.len() { + let item = zelf.template.interpolations.get(index).unwrap(); + zelf.index.fetch_add(1, Ordering::SeqCst); + zelf.from_strings.store(true, Ordering::SeqCst); + return Ok(PyIterReturn::Return(item.clone())); + } else { + return Ok(PyIterReturn::StopIteration(None)); + } + } + } +} + +pub fn init(context: &Context) { + PyTemplate::extend_class(context, context.types.template_type); + PyTemplateIter::extend_class(context, context.types.template_iter_type); +} diff --git a/crates/vm/src/builtins/traceback.rs b/crates/vm/src/builtins/traceback.rs index 6bf4070c9b5..66d999b26b4 100644 --- a/crates/vm/src/builtins/traceback.rs +++ b/crates/vm/src/builtins/traceback.rs @@ -1,7 +1,7 @@ -use super::PyType; +use super::{PyList, PyType}; use crate::{ - Context, Py, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, frame::FrameRef, - types::Constructor, + AsObject, Context, Py, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, + frame::FrameRef, function::PySetterValue, types::Constructor, }; use rustpython_common::lock::PyMutex; use rustpython_compiler_core::OneIndexed; @@ -62,9 +62,43 @@ impl PyTraceback { self.next.lock().as_ref().cloned() } + #[pymethod] + fn __dir__(&self, vm: &VirtualMachine) -> PyList { + PyList::from( + ["tb_frame", "tb_next", "tb_lasti", "tb_lineno"] + .iter() + .map(|&s| vm.ctx.new_str(s).into()) + .collect::<Vec<_>>(), + ) + } + #[pygetset(setter)] - fn set_tb_next(&self, value: Option<PyRef<Self>>) { - *self.next.lock() = value; + fn set_tb_next( + zelf: &Py<Self>, + value: PySetterValue<Option<PyRef<Self>>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let value = match value { + PySetterValue::Assign(v) => v, + PySetterValue::Delete => { + return Err(vm.new_type_error("can't delete tb_next attribute".to_owned())); + } + }; + if let Some(ref new_next) = value { + let mut cursor = new_next.clone(); + loop { + if cursor.is(zelf) { + return Err(vm.new_value_error("traceback loop detected".to_owned())); + } + let next = cursor.next.lock().clone(); + match next { + Some(n) => cursor = n, + None => break, + } + } + } + *zelf.next.lock() = value; + Ok(()) } } @@ -81,7 +115,7 @@ impl Constructor for PyTraceback { impl PyTracebackRef { pub fn iter(&self) -> impl Iterator<Item = Self> { - std::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) + core::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) } } @@ -97,7 +131,7 @@ impl serde::Serialize for PyTraceback { let mut struc = s.serialize_struct("PyTraceback", 3)?; struc.serialize_field("name", self.frame.code.obj_name.as_str())?; struc.serialize_field("lineno", &self.lineno.get())?; - struc.serialize_field("filename", self.frame.code.source_path.as_str())?; + struc.serialize_field("filename", self.frame.code.source_path().as_str())?; struc.end() } } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index fada8840bb1..67c51127cb4 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -1,8 +1,12 @@ -use super::{PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use super::{ + PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef, iter::builtins_iter, +}; +use crate::common::lock::LazyLock; use crate::common::{ hash::{PyHash, PyUHash}, lock::PyMutex, }; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, @@ -10,20 +14,20 @@ use crate::{ convert::{ToPyObject, TransmuteFromObject}, function::{ArgSize, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, iter::PyExactSizeIterator, - protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, + protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{OptionalRangeArgs, SequenceExt}, sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ - AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, + PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, vm::VirtualMachine, }; -use std::{fmt, sync::LazyLock}; +use alloc::fmt; -#[pyclass(module = false, name = "tuple", traverse)] +#[pyclass(module = false, name = "tuple", traverse = "manual")] pub struct PyTuple<R = PyObjectRef> { elements: Box<[R]>, } @@ -35,6 +39,19 @@ impl<R> fmt::Debug for PyTuple<R> { } } +// SAFETY: Traverse properly visits all owned PyObjectRefs +// Note: Only impl for PyTuple<PyObjectRef> (the default) +unsafe impl Traverse for PyTuple { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + let elements = core::mem::take(&mut self.elements); + out.extend(elements.into_vec()); + } +} + impl PyPayload for PyTuple { #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { @@ -158,7 +175,7 @@ impl<R> AsRef<[R]> for PyTuple<R> { } } -impl<R> std::ops::Deref for PyTuple<R> { +impl<R> core::ops::Deref for PyTuple<R> { type Target = [R]; fn deref(&self) -> &[R] { @@ -166,18 +183,18 @@ impl<R> std::ops::Deref for PyTuple<R> { } } -impl<'a, R> std::iter::IntoIterator for &'a PyTuple<R> { +impl<'a, R> core::iter::IntoIterator for &'a PyTuple<R> { type Item = &'a R; - type IntoIter = std::slice::Iter<'a, R>; + type IntoIter = core::slice::Iter<'a, R>; fn into_iter(self) -> Self::IntoIter { self.iter() } } -impl<'a, R> std::iter::IntoIterator for &'a Py<PyTuple<R>> { +impl<'a, R> core::iter::IntoIterator for &'a Py<PyTuple<R>> { type Item = &'a R; - type IntoIter = std::slice::Iter<'a, R>; + type IntoIter = core::slice::Iter<'a, R>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -200,7 +217,7 @@ impl<R> PyTuple<R> { } #[inline] - pub fn iter(&self) -> std::slice::Iter<'_, R> { + pub fn iter(&self) -> core::slice::Iter<'_, R> { self.elements.iter() } } @@ -249,27 +266,19 @@ impl<T> PyTuple<PyRef<T>> { // SAFETY: PyRef<T> has the same layout as PyObjectRef unsafe { let elements: Vec<PyObjectRef> = - std::mem::transmute::<Vec<PyRef<T>>, Vec<PyObjectRef>>(elements); + core::mem::transmute::<Vec<PyRef<T>>, Vec<PyObjectRef>>(elements); let tuple = PyTuple::<PyObjectRef>::new_ref(elements, ctx); - std::mem::transmute::<PyRef<PyTuple>, PyRef<Self>>(tuple) + core::mem::transmute::<PyRef<PyTuple>, PyRef<Self>>(tuple) } } } #[pyclass( + itemsize = core::mem::size_of::<crate::PyObjectRef>(), flags(BASETYPE, SEQUENCE, _MATCH_SELF), - with( - AsMapping, - AsSequence, - Hashable, - Comparable, - Iterable, - Constructor, - Representable - ) + with(AsMapping, AsNumber, AsSequence, Hashable, Comparable, Iterable, Constructor, Representable) )] impl PyTuple { - #[pymethod] fn __add__( zelf: PyRef<Self>, other: PyObjectRef, @@ -292,11 +301,6 @@ impl PyTuple { PyArithmeticValue::from_option(added.ok()) } - #[pymethod] - const fn __bool__(&self) -> bool { - !self.elements.is_empty() - } - #[pymethod] fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { let mut count: usize = 0; @@ -309,13 +313,10 @@ impl PyTuple { } #[inline] - #[pymethod] pub const fn __len__(&self) -> usize { self.elements.len() } - #[pymethod(name = "__rmul__")] - #[pymethod] fn __mul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Self::repeat(zelf, value.into(), vm) } @@ -330,7 +331,6 @@ impl PyTuple { } } - #[pymethod] fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } @@ -360,7 +360,6 @@ impl PyTuple { Ok(false) } - #[pymethod] fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self._contains(&needle, vm) } @@ -429,6 +428,19 @@ impl AsSequence for PyTuple { } } +impl AsNumber for PyTuple { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyTuple>().unwrap(); + Ok(!zelf.elements.is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + impl Hashable for PyTuple { #[inline] fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { @@ -496,21 +508,21 @@ impl PyRef<PyTuple<PyObjectRef>> { <PyRef<T> as TransmuteFromObject>::check(vm, elem)?; } // SAFETY: We just verified all elements are of type T - Ok(unsafe { std::mem::transmute::<Self, PyRef<PyTuple<PyRef<T>>>>(self) }) + Ok(unsafe { core::mem::transmute::<Self, PyRef<PyTuple<PyRef<T>>>>(self) }) } } impl<T: PyPayload> PyRef<PyTuple<PyRef<T>>> { pub fn into_untyped(self) -> PyRef<PyTuple> { // SAFETY: PyTuple<PyRef<T>> has the same layout as PyTuple - unsafe { std::mem::transmute::<Self, PyRef<PyTuple>>(self) } + unsafe { core::mem::transmute::<Self, PyRef<PyTuple>>(self) } } } impl<T: PyPayload> Py<PyTuple<PyRef<T>>> { pub fn as_untyped(&self) -> &Py<PyTuple> { // SAFETY: PyTuple<PyRef<T>> has the same layout as PyTuple - unsafe { std::mem::transmute::<&Self, &Py<PyTuple>>(self) } + unsafe { core::mem::transmute::<&Self, &Py<PyTuple>>(self) } } } @@ -533,7 +545,7 @@ impl PyPayload for PyTupleIterator { } } -#[pyclass(with(Unconstructible, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyTupleIterator { #[pymethod] fn __length_hint__(&self) -> usize { @@ -549,12 +561,15 @@ impl PyTupleIterator { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } } -impl Unconstructible for PyTupleIterator {} impl SelfIter for PyTupleIterator {} impl IterNext for PyTupleIterator { diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 0f619b1399a..49257117484 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1,6 +1,6 @@ use super::{ - PyClassMethod, PyDictRef, PyList, PyStr, PyStrInterned, PyStrRef, PyTupleRef, PyWeak, - mappingproxy::PyMappingProxy, object, union_, + PyClassMethod, PyDictRef, PyList, PyStaticMethod, PyStr, PyStrInterned, PyStrRef, PyTupleRef, + PyWeak, mappingproxy::PyMappingProxy, object, union_, }; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, @@ -20,19 +20,19 @@ use crate::{ borrow::BorrowedValue, lock::{PyRwLock, PyRwLockReadGuard}, }, - convert::ToPyResult, function::{FuncArgs, KwArgs, OptionalArg, PyMethodDef, PySetterValue}, object::{Traverse, TraverseFn}, - protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + protocol::{PyIterReturn, PyNumberMethods}, types::{ - AsNumber, Callable, Constructor, GetAttr, PyTypeFlags, PyTypeSlots, Representable, SetAttr, - TypeDataRef, TypeDataRefMut, TypeDataSlot, + AsNumber, Callable, Constructor, GetAttr, Initializer, PyTypeFlags, PyTypeSlots, + Representable, SLOT_DEFS, SetAttr, TypeDataRef, TypeDataRefMut, TypeDataSlot, }, }; +use core::{any::Any, borrow::Borrow, ops::Deref, pin::Pin, ptr::NonNull}; use indexmap::{IndexMap, map::Entry}; use itertools::Itertools; use num_traits::ToPrimitive; -use std::{any::Any, borrow::Borrow, collections::HashSet, ops::Deref, pin::Pin, ptr::NonNull}; +use std::collections::HashSet; #[pyclass(module = false, name = "type", traverse = "manual")] pub struct PyType { @@ -57,6 +57,33 @@ unsafe impl crate::object::Traverse for PyType { .map(|(_, v)| v.traverse(tracer_fn)) .count(); } + + /// type_clear: break reference cycles in type objects + fn clear(&mut self, out: &mut Vec<crate::PyObjectRef>) { + if let Some(base) = self.base.take() { + out.push(base.into()); + } + if let Some(mut guard) = self.bases.try_write() { + for base in guard.drain(..) { + out.push(base.into()); + } + } + if let Some(mut guard) = self.mro.try_write() { + for typ in guard.drain(..) { + out.push(typ.into()); + } + } + if let Some(mut guard) = self.subclasses.try_write() { + for weak in guard.drain(..) { + out.push(weak.into()); + } + } + if let Some(mut guard) = self.attributes.try_write() { + for (_, val) in guard.drain(..) { + out.push(val); + } + } + } } // PyHeapTypeObject in CPython @@ -64,8 +91,6 @@ pub struct HeapTypeExt { pub name: PyRwLock<PyStrRef>, pub qualname: PyRwLock<PyStrRef>, pub slots: Option<PyRef<PyTuple<PyStrRef>>>, - pub sequence_methods: PySequenceMethods, - pub mapping_methods: PyMappingMethods, pub type_data: PyRwLock<Option<TypeDataSlot>>, } @@ -100,17 +125,6 @@ impl<T> AsRef<T> for PointerSlot<T> { } } -impl<T> PointerSlot<T> { - pub unsafe fn from_heaptype<F>(typ: &PyType, f: F) -> Option<Self> - where - F: FnOnce(&HeapTypeExt) -> &T, - { - typ.heaptype_ext - .as_ref() - .map(|ext| Self(NonNull::from(f(ext)))) - } -} - pub type PyTypeRef = PyRef<PyType>; cfg_if::cfg_if! { @@ -131,14 +145,14 @@ unsafe impl Traverse for PyAttributes { } } -impl std::fmt::Display for PyType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.name(), f) +impl core::fmt::Display for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Display::fmt(&self.name(), f) } } -impl std::fmt::Debug for PyType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "[PyType {}]", &self.name()) } } @@ -175,12 +189,12 @@ fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py<PyType>, b: &Py<PyType>) -> b impl PyType { pub fn new_simple_heap( name: &str, - base: &PyTypeRef, + base: &Py<PyType>, ctx: &Context, ) -> Result<PyRef<Self>, String> { Self::new_heap( name, - vec![base.clone()], + vec![base.to_owned()], Default::default(), Default::default(), Self::static_type().to_owned(), @@ -191,20 +205,21 @@ impl PyType { name: &str, bases: Vec<PyRef<Self>>, attrs: PyAttributes, - slots: PyTypeSlots, + mut slots: PyTypeSlots, metaclass: PyRef<Self>, ctx: &Context, ) -> Result<PyRef<Self>, String> { // TODO: ensure clean slot name // assert_eq!(slots.name.borrow(), ""); + // Set HEAPTYPE flag for heap-allocated types + slots.flags |= PyTypeFlags::HEAPTYPE; + let name = ctx.new_str(name); let heaptype_ext = HeapTypeExt { name: PyRwLock::new(name.clone()), qualname: PyRwLock::new(name), slots: None, - sequence_methods: PySequenceMethods::default(), - mapping_methods: PyMappingMethods::default(), type_data: PyRwLock::new(None), }; let base = bases[0].clone(); @@ -328,6 +343,8 @@ impl PyType { slots.basicsize = base.slots.basicsize; } + Self::inherit_readonly_slots(&mut slots, &base); + if let Some(qualname) = attrs.get(identifier!(ctx, __qualname__)) && !qualname.fast_isinstance(ctx.types.str_type) { @@ -350,6 +367,7 @@ impl PyType { metaclass, None, ); + new_type.mro.write().insert(0, new_type.clone()); new_type.init_slots(ctx); @@ -378,12 +396,14 @@ impl PyType { // Inherit SEQUENCE and MAPPING flags from base class // For static types, we only have a single base - Self::inherit_patma_flags(&mut slots, std::slice::from_ref(&base)); + Self::inherit_patma_flags(&mut slots, core::slice::from_ref(&base)); if slots.basicsize == 0 { slots.basicsize = base.slots.basicsize; } + Self::inherit_readonly_slots(&mut slots, &base); + let bases = PyRwLock::new(vec![base.clone()]); let mro = base.mro_map_collect(|x| x.to_owned()); @@ -401,6 +421,17 @@ impl PyType { None, ); + // Static types are not tracked by GC. + // They are immortal and never participate in collectable cycles. + new_type.as_object().clear_gc_tracked(); + + new_type.mro.write().insert(0, new_type.clone()); + + // Note: inherit_slots is called in PyClassImpl::init_class after + // slots are fully initialized by make_slots() + + Self::set_new(&new_type.slots, &new_type.base); + let weakref_type = super::PyWeak::static_type(); for base in new_type.bases.read().iter() { base.subclasses.write().push( @@ -415,14 +446,19 @@ impl PyType { } pub(crate) fn init_slots(&self, ctx: &Context) { + // Inherit slots from MRO (mro[0] is self, so skip it) + let mro: Vec<_> = self.mro.read()[1..].to_vec(); + for base in mro.iter() { + self.inherit_slots(base); + } + + // Wire dunder methods to slots #[allow(clippy::mutable_key_type)] let mut slot_name_set = std::collections::HashSet::new(); - for cls in self.mro.read().iter() { + // mro[0] is self, so skip it; self.attributes is checked separately below + for cls in self.mro.read()[1..].iter() { for &name in cls.attributes.read().keys() { - if name == identifier!(ctx, __new__) { - continue; - } if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") { slot_name_set.insert(name); } @@ -433,9 +469,43 @@ impl PyType { slot_name_set.insert(name); } } - for attr_name in slot_name_set { + // Sort for deterministic iteration order (important for slot processing) + let mut slot_names: Vec<_> = slot_name_set.into_iter().collect(); + slot_names.sort_by_key(|name| name.as_str()); + for attr_name in slot_names { self.update_slot::<true>(attr_name, ctx); } + + Self::set_new(&self.slots, &self.base); + } + + fn set_new(slots: &PyTypeSlots, base: &Option<PyTypeRef>) { + if slots.flags.contains(PyTypeFlags::DISALLOW_INSTANTIATION) { + slots.new.store(None) + } else if slots.new.load().is_none() { + slots.new.store( + base.as_ref() + .map(|base| base.slots.new.load()) + .unwrap_or(None), + ) + } + } + + /// Inherit readonly slots from base type at creation time. + /// These slots are not AtomicCell and must be set before the type is used. + fn inherit_readonly_slots(slots: &mut PyTypeSlots, base: &Self) { + if slots.as_buffer.is_none() { + slots.as_buffer = base.slots.as_buffer; + } + } + + /// Inherit slots from base type. inherit_slots + pub(crate) fn inherit_slots(&self, base: &Self) { + // Use SLOT_DEFS to iterate all slots + // Note: as_buffer is handled in inherit_readonly_slots (not AtomicCell) + for def in SLOT_DEFS { + def.accessor.copyslot_if_none(self, base); + } } // This is used for class initialization where the vm is not yet available. @@ -469,18 +539,12 @@ impl PyType { /// Equivalent to CPython's find_name_in_mro /// Look in tp_dict of types in MRO - bypasses descriptors and other attribute access machinery fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option<PyObjectRef> { - // First check in our own dict - if let Some(value) = self.attributes.read().get(name) { - return Some(value.clone()); - } - - // Then check in MRO - for base in self.mro.read().iter() { - if let Some(value) = base.attributes.read().get(name) { + // mro[0] is self, so we just iterate through the entire MRO + for cls in self.mro.read().iter() { + if let Some(value) = cls.attributes.read().get(name) { return Some(value.clone()); } } - None } @@ -496,8 +560,7 @@ impl PyType { } pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { - self.mro - .read() + self.mro.read()[1..] .iter() .find_map(|class| class.attributes.read().get(attr_name).cloned()) } @@ -505,9 +568,7 @@ impl PyType { // This is the internal has_attr implementation for fast lookup on a class. pub fn has_attr(&self, attr_name: &'static PyStrInterned) -> bool { self.attributes.read().contains_key(attr_name) - || self - .mro - .read() + || self.mro.read()[1..] .iter() .any(|c| c.attributes.read().contains_key(attr_name)) } @@ -516,10 +577,8 @@ impl PyType { // Gather all members here: let mut attributes = PyAttributes::default(); - for bc in std::iter::once(self) - .chain(self.mro.read().iter().map(|cls| -> &Self { cls })) - .rev() - { + // mro[0] is self, so we iterate through the entire MRO in reverse + for bc in self.mro.read().iter().map(|cls| -> &Self { cls }).rev() { for (name, value) in bc.attributes.read().iter() { attributes.insert(name.to_owned(), value.clone()); } @@ -546,10 +605,10 @@ impl PyType { static_f: impl FnOnce(&'static str) -> R, heap_f: impl FnOnce(&'a HeapTypeExt) -> R, ) -> R { - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - static_f(self.slots.name) + if let Some(ref ext) = self.heaptype_ext { + heap_f(ext) } else { - heap_f(self.heaptype_ext.as_ref().unwrap()) + static_f(self.slots.name) } } @@ -627,41 +686,27 @@ impl Py<PyType> { /// so only use this if `cls` is known to have not overridden the base __subclasscheck__ magic /// method. pub fn fast_issubclass(&self, cls: &impl Borrow<PyObject>) -> bool { - self.as_object().is(cls.borrow()) || self.mro.read().iter().any(|c| c.is(cls.borrow())) + self.as_object().is(cls.borrow()) || self.mro.read()[1..].iter().any(|c| c.is(cls.borrow())) } pub fn mro_map_collect<F, R>(&self, f: F) -> Vec<R> where F: Fn(&Self) -> R, { - std::iter::once(self) - .chain(self.mro.read().iter().map(|x| x.deref())) - .map(f) - .collect() + self.mro.read().iter().map(|x| x.deref()).map(f).collect() } pub fn mro_collect(&self) -> Vec<PyRef<PyType>> { - std::iter::once(self) - .chain(self.mro.read().iter().map(|x| x.deref())) + self.mro + .read() + .iter() + .map(|x| x.deref()) .map(|x| x.to_owned()) .collect() } - pub(crate) fn mro_find_map<F, R>(&self, f: F) -> Option<R> - where - F: Fn(&Self) -> Option<R>, - { - // the hot path will be primitive types which usually hit the result from itself. - // try std::intrinsics::likely once it is stabilized - if let Some(r) = f(self) { - Some(r) - } else { - self.mro.read().iter().find_map(|cls| f(cls)) - } - } - pub fn iter_base_chain(&self) -> impl Iterator<Item = &Self> { - std::iter::successors(Some(self), |cls| cls.base.as_deref()) + core::iter::successors(Some(self), |cls| cls.base.as_deref()) } pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { @@ -673,7 +718,16 @@ impl Py<PyType> { } #[pyclass( - with(Py, Constructor, GetAttr, SetAttr, Callable, AsNumber, Representable), + with( + Py, + Constructor, + Initializer, + GetAttr, + SetAttr, + Callable, + AsNumber, + Representable + ), flags(BASETYPE) )] impl PyType { @@ -715,8 +769,11 @@ impl PyType { *zelf.bases.write() = bases; // Recursively update the mros of this class and all subclasses fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { - *cls.mro.write() = + let mut mro = PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; + // Preserve self (mro[0]) when updating MRO + mro.insert(0, cls.mro.read()[0].to_owned()); + *cls.mro.write() = mro; for subclass in cls.subclasses.write().iter() { let subclass = subclass.upgrade().unwrap(); let subclass: &Py<PyType> = subclass.downcast_ref().unwrap(); @@ -753,8 +810,13 @@ impl PyType { } #[pygetset] - const fn __basicsize__(&self) -> usize { - self.slots.basicsize + fn __basicsize__(&self) -> usize { + crate::object::SIZEOF_PYOBJECT_HEAD + self.slots.basicsize + } + + #[pygetset] + fn __itemsize__(&self) -> usize { + self.slots.itemsize } #[pygetset] @@ -787,13 +849,7 @@ impl PyType { #[pygetset(setter)] fn set___qualname__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - // TODO: we should replace heaptype flag check to immutable flag check - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - return Err(vm.new_type_error(format!( - "cannot set '__qualname__' attribute of immutable type '{}'", - self.name() - ))); - }; + self.check_set_special_type_attr(identifier!(vm, __qualname__), vm)?; let value = value.ok_or_else(|| { vm.new_type_error(format!( "cannot delete '__qualname__' attribute of immutable type '{}'", @@ -803,16 +859,18 @@ impl PyType { let str_value = downcast_qualname(value, vm)?; - let heap_type = self - .heaptype_ext - .as_ref() - .expect("HEAPTYPE should have heaptype_ext"); + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__qualname__' attribute of immutable type '{}'", + self.name() + )) + })?; // Use std::mem::replace to swap the new value in and get the old value out, // then drop the old value after releasing the lock let _old_qualname = { let mut qualname_guard = heap_type.qualname.write(); - std::mem::replace(&mut *qualname_guard, str_value) + core::mem::replace(&mut *qualname_guard, str_value) }; // old_qualname is dropped here, outside the lock scope @@ -820,33 +878,129 @@ impl PyType { } #[pygetset] - fn __annotations__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotate__'", + self.name() + ))); + } + + let mut attrs = self.attributes.write(); + // First try __annotate__, in case that's been set explicitly + if let Some(annotate) = attrs.get(identifier!(vm, __annotate__)).cloned() { + return Ok(annotate); + } + // Then try __annotate_func__ + if let Some(annotate) = attrs.get(identifier!(vm, __annotate_func__)).cloned() { + // TODO: Apply descriptor tp_descr_get if needed + return Ok(annotate); + } + // Set __annotate_func__ = None and return None + let none = vm.ctx.none(); + attrs.insert(identifier!(vm, __annotate_func__), none.clone()); + Ok(none) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let value = match value { + PySetterValue::Delete => { + return Err(vm.new_type_error("cannot delete __annotate__ attribute".to_owned())); + } + PySetterValue::Assign(v) => v, + }; + + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '__annotate__' attribute of immutable type '{}'", + self.name() + ))); + } + + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None".to_owned())); + } + + let mut attrs = self.attributes.write(); + // Clear cached annotations only when setting to a new callable + if !vm.is_none(&value) { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + attrs.insert(identifier!(vm, __annotate_func__), value.clone()); + + Ok(()) + } + + #[pygetset] + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let attrs = self.attributes.read(); + if let Some(annotations) = attrs.get(identifier!(vm, __annotations__)).cloned() { + // Ignore the __annotations__ descriptor stored on type itself. + if !annotations.class().is(vm.ctx.types.getset_type) { + if vm.is_none(&annotations) + || annotations.class().is(vm.ctx.types.dict_type) + || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + { + return Ok(annotations); + } + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + self.name() + ))); + } + } + // Then try __annotations_cache__ + if let Some(annotations) = attrs.get(identifier!(vm, __annotations_cache__)).cloned() { + if vm.is_none(&annotations) + || annotations.class().is(vm.ctx.types.dict_type) + || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + { + return Ok(annotations); + } return Err(vm.new_attribute_error(format!( "type object '{}' has no attribute '__annotations__'", self.name() ))); } + drop(attrs); - let __annotations__ = identifier!(vm, __annotations__); - let annotations = self.attributes.read().get(__annotations__).cloned(); + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + self.name() + ))); + } - let annotations = if let Some(annotations) = annotations { - annotations + // Get __annotate__ and call it if callable + let annotate = self.__annotate__(vm)?; + let annotations = if annotate.is_callable() { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result } else { - let annotations: PyObjectRef = vm.ctx.new_dict().into(); - let removed = self - .attributes - .write() - .insert(__annotations__, annotations.clone()); - debug_assert!(removed.is_none()); - annotations + vm.ctx.new_dict().into() }; + + // Cache the result in __annotations_cache__ + self.attributes + .write() + .insert(identifier!(vm, __annotations_cache__), annotations.clone()); Ok(annotations) } #[pygetset(setter)] - fn set___annotations__(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { + fn set___annotations__( + &self, + value: crate::function::PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { return Err(vm.new_type_error(format!( "cannot set '__annotations__' attribute of immutable type '{}'", @@ -854,21 +1008,43 @@ impl PyType { ))); } - let __annotations__ = identifier!(vm, __annotations__); - if let Some(value) = value { - self.attributes.write().insert(__annotations__, value); - } else { - self.attributes - .read() - .get(__annotations__) - .cloned() - .ok_or_else(|| { - vm.new_attribute_error(format!( - "'{}' object has no attribute '__annotations__'", - self.name() - )) - })?; + let mut attrs = self.attributes.write(); + let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + + match value { + crate::function::PySetterValue::Assign(value) => { + // SET path: store the value (including None) + let key = if has_annotations { + identifier!(vm, __annotations__) + } else { + identifier!(vm, __annotations_cache__) + }; + attrs.insert(key, value); + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + } + crate::function::PySetterValue::Delete => { + // DELETE path: remove the key + let removed = if has_annotations { + attrs + .swap_remove(identifier!(vm, __annotations__)) + .is_some() + } else { + attrs + .swap_remove(identifier!(vm, __annotations_cache__)) + .is_some() + }; + if !removed { + return Err(vm.new_attribute_error("__annotations__".to_owned())); + } + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + } } + attrs.swap_remove(identifier!(vm, __annotate_func__)); + attrs.swap_remove(identifier!(vm, __annotate__)); Ok(()) } @@ -887,14 +1063,24 @@ impl PyType { Some(found) } }) - .unwrap_or_else(|| vm.ctx.new_str(ascii!("builtins")).into()) + .unwrap_or_else(|| { + // For non-heap types, extract module from tp_name (e.g. "typing.TypeAliasType" -> "typing") + let slot_name = self.slot_name(); + if let Some((module, _)) = slot_name.rsplit_once('.') { + vm.ctx.intern_str(module).to_object() + } else { + vm.ctx.new_str(ascii!("builtins")).into() + } + }) } #[pygetset(setter)] - fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) { + fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.check_set_special_type_attr(identifier!(vm, __module__), vm)?; self.attributes .write() .insert(identifier!(vm, __module__), value); + Ok(()) } #[pyclassmethod] @@ -920,13 +1106,11 @@ impl PyType { ) } - #[pymethod] - pub fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { or_(other, zelf, vm) } - #[pymethod] - pub fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { or_(zelf, other, vm) } @@ -944,7 +1128,6 @@ impl PyType { fn check_set_special_type_attr( &self, - _value: &PyObject, name: &PyStrInterned, vm: &VirtualMachine, ) -> PyResult<()> { @@ -960,7 +1143,7 @@ impl PyType { #[pygetset(setter)] fn set___name__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.check_set_special_type_attr(&value, identifier!(vm, __name__), vm)?; + self.check_set_special_type_attr(identifier!(vm, __name__), vm)?; let name = value.downcast::<PyStr>().map_err(|value| { vm.new_type_error(format!( "can only assign string to {}.__name__, not '{}'", @@ -971,12 +1154,20 @@ impl PyType { if name.as_bytes().contains(&0) { return Err(vm.new_value_error("type name must not contain null characters")); } + name.ensure_valid_utf8(vm)?; + + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__name__' attribute of immutable type '{}'", + self.slot_name() + )) + })?; // Use std::mem::replace to swap the new value in and get the old value out, - // then drop the old value after releasing the lock (similar to CPython's Py_SETREF) + // then drop the old value after releasing the lock let _old_name = { - let mut name_guard = self.heaptype_ext.as_ref().unwrap().name.write(); - std::mem::replace(&mut *name_guard, name) + let mut name_guard = heap_type.name.write(); + core::mem::replace(&mut *name_guard, name) }; // old_name is dropped here, outside the lock scope @@ -1013,7 +1204,7 @@ impl PyType { match value { PySetterValue::Assign(ref val) => { let key = identifier!(vm, __type_params__); - self.check_set_special_type_attr(val.as_ref(), key, vm)?; + self.check_set_special_type_attr(key, vm)?; let mut attrs = self.attributes.write(); attrs.insert(key, val.clone().into()); } @@ -1062,6 +1253,7 @@ impl Constructor for PyType { if name.as_bytes().contains(&0) { return Err(vm.new_value_error("type name must not contain null characters")); } + name.ensure_valid_utf8(vm)?; let (metatype, base, bases, base_is_type) = if bases.is_empty() { let base = vm.ctx.types.object_type.to_owned(); @@ -1114,6 +1306,13 @@ impl Constructor for PyType { }); let mut attributes = dict.to_attributes(vm); + // Check __doc__ for surrogates - raises UnicodeEncodeError during type creation + if let Some(doc) = attributes.get(identifier!(vm, __doc__)) + && let Some(doc_str) = doc.downcast_ref::<PyStr>() + { + doc_str.ensure_valid_utf8(vm)?; + } + if let Some(f) = attributes.get_mut(identifier!(vm, __init_subclass__)) && f.class().is(vm.ctx.types.function_type) { @@ -1126,6 +1325,12 @@ impl Constructor for PyType { *f = PyClassMethod::from(f.clone()).into_pyobject(vm); } + if let Some(f) = attributes.get_mut(identifier!(vm, __new__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyStaticMethod::from(f.clone()).into_pyobject(vm); + } + if let Some(current_frame) = vm.current_frame() { let entry = attributes.entry(identifier!(vm, __module__)); if matches!(entry, Entry::Vacant(_)) { @@ -1146,45 +1351,144 @@ impl Constructor for PyType { attributes.insert(identifier!(vm, __hash__), vm.ctx.none.clone().into()); } - let (heaptype_slots, add_dict): (Option<PyRef<PyTuple<PyStrRef>>>, bool) = - if let Some(x) = attributes.get(identifier!(vm, __slots__)) { - let slots = if x.class().is(vm.ctx.types.str_type) { - let x = unsafe { x.downcast_unchecked_ref::<PyStr>() }; - PyTuple::new_ref_typed(vec![x.to_owned()], &vm.ctx) - } else { - let iter = x.get_iter(vm)?; - let elements = { - let mut elements = Vec::new(); - while let PyIterReturn::Return(element) = iter.next(vm)? { - elements.push(element); + let (heaptype_slots, add_dict): (Option<PyRef<PyTuple<PyStrRef>>>, bool) = if let Some(x) = + attributes.get(identifier!(vm, __slots__)) + { + // Check if __slots__ is bytes - not allowed + if x.class().is(vm.ctx.types.bytes_type) { + return Err( + vm.new_type_error("__slots__ items must be strings, not 'bytes'".to_owned()) + ); + } + + let slots = if x.class().is(vm.ctx.types.str_type) { + let x = unsafe { x.downcast_unchecked_ref::<PyStr>() }; + PyTuple::new_ref_typed(vec![x.to_owned()], &vm.ctx) + } else { + let iter = x.get_iter(vm)?; + let elements = { + let mut elements = Vec::new(); + while let PyIterReturn::Return(element) = iter.next(vm)? { + // Check if any slot item is bytes + if element.class().is(vm.ctx.types.bytes_type) { + return Err(vm.new_type_error( + "__slots__ items must be strings, not 'bytes'".to_owned(), + )); } - elements - }; - let tuple = elements.into_pytuple(vm); - tuple.try_into_typed(vm)? + elements.push(element); + } + elements }; + let tuple = elements.into_pytuple(vm); + tuple.try_into_typed(vm)? + }; + + // Check if base has itemsize > 0 - can't add arbitrary slots to variable-size types + // Types like int, bytes, tuple have itemsize > 0 and don't allow custom slots + // But types like weakref.ref have itemsize = 0 and DO allow slots + let has_custom_slots = slots + .iter() + .any(|s| s.as_str() != "__dict__" && s.as_str() != "__weakref__"); + if has_custom_slots && base.slots.itemsize > 0 { + return Err(vm.new_type_error(format!( + "nonempty __slots__ not supported for subtype of '{}'", + base.name() + ))); + } - // Check if __dict__ is in slots - let dict_name = "__dict__"; - let has_dict = slots.iter().any(|s| s.as_str() == dict_name); - - // Filter out __dict__ from slots - let filtered_slots = if has_dict { - let filtered: Vec<PyStrRef> = slots - .iter() - .filter(|s| s.as_str() != dict_name) - .cloned() - .collect(); - PyTuple::new_ref_typed(filtered, &vm.ctx) + // Validate slot names and track duplicates + let mut seen_dict = false; + let mut seen_weakref = false; + for slot in slots.iter() { + // Use isidentifier for validation (handles Unicode properly) + if !slot.isidentifier() { + return Err(vm.new_type_error("__slots__ must be identifiers".to_owned())); + } + + let slot_name = slot.as_str(); + + // Check for duplicate __dict__ + if slot_name == "__dict__" { + if seen_dict { + return Err(vm.new_type_error( + "__dict__ slot disallowed: we already got one".to_owned(), + )); + } + seen_dict = true; + } + + // Check for duplicate __weakref__ + if slot_name == "__weakref__" { + if seen_weakref { + return Err(vm.new_type_error( + "__weakref__ slot disallowed: we already got one".to_owned(), + )); + } + seen_weakref = true; + } + + // Check if slot name conflicts with class attributes + if attributes.contains_key(vm.ctx.intern_str(slot_name)) { + return Err(vm.new_value_error(format!( + "'{}' in __slots__ conflicts with a class variable", + slot_name + ))); + } + } + + // Check if base class already has __dict__ - can't redefine it + if seen_dict && base.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + return Err( + vm.new_type_error("__dict__ slot disallowed: we already got one".to_owned()) + ); + } + + // Check if base class already has __weakref__ - can't redefine it + // A base has weakref support if: + // 1. It's a heap type without explicit __slots__ (automatic weakref), OR + // 2. It's a heap type with __weakref__ in its __slots__ + if seen_weakref { + let base_has_weakref = if let Some(ref ext) = base.heaptype_ext { + match &ext.slots { + // Heap type without __slots__ - has automatic weakref + None => true, + // Heap type with __slots__ - check if __weakref__ is in slots + Some(base_slots) => base_slots.iter().any(|s| s.as_str() == "__weakref__"), + } } else { - slots + // Builtin type - check if it has __weakref__ descriptor + let weakref_name = vm.ctx.intern_str("__weakref__"); + base.attributes.read().contains_key(weakref_name) }; - (Some(filtered_slots), has_dict) + if base_has_weakref { + return Err(vm.new_type_error( + "__weakref__ slot disallowed: we already got one".to_owned(), + )); + } + } + + // Check if __dict__ is in slots + let dict_name = "__dict__"; + let has_dict = slots.iter().any(|s| s.as_str() == dict_name); + + // Filter out __dict__ from slots + let filtered_slots = if has_dict { + let filtered: Vec<PyStrRef> = slots + .iter() + .filter(|s| s.as_str() != dict_name) + .cloned() + .collect(); + PyTuple::new_ref_typed(filtered, &vm.ctx) } else { - (None, false) + slots }; + (Some(filtered_slots), has_dict) + } else { + (None, false) + }; + // FIXME: this is a temporary fix. multi bases with multiple slots will break object let base_member_count = bases .iter() @@ -1195,10 +1499,16 @@ impl Constructor for PyType { let member_count: usize = base_member_count + heaptype_member_count; let mut flags = PyTypeFlags::heap_type_flags(); + + // Check if we may add dict + // We can only add a dict if the primary base class doesn't already have one + // In CPython, this checks tp_dictoffset == 0 + let may_add_dict = !base.slots.flags.has_feature(PyTypeFlags::HAS_DICT); + // Add HAS_DICT and MANAGED_DICT if: - // 1. __slots__ is not defined, OR + // 1. __slots__ is not defined AND base doesn't have dict, OR // 2. __dict__ is in __slots__ - if heaptype_slots.is_none() || add_dict { + if (heaptype_slots.is_none() && may_add_dict) || add_dict { flags |= PyTypeFlags::HAS_DICT | PyTypeFlags::MANAGED_DICT; } @@ -1206,14 +1516,13 @@ impl Constructor for PyType { let slots = PyTypeSlots { flags, member_count, + itemsize: base.slots.itemsize, ..PyTypeSlots::heap_default() }; let heaptype_ext = HeapTypeExt { name: PyRwLock::new(name), qualname: PyRwLock::new(qualname), slots: heaptype_slots.clone(), - sequence_methods: PySequenceMethods::default(), - mapping_methods: PyMappingMethods::default(), type_data: PyRwLock::new(None), }; (slots, heaptype_ext) @@ -1280,9 +1589,15 @@ impl Constructor for PyType { // Only add if: // 1. base is not type (type subclasses inherit __dict__ from type) // 2. the class has HAS_DICT flag (i.e., __slots__ was not defined or __dict__ is in __slots__) + // 3. no base class in MRO already provides __dict__ descriptor if !base_is_type && typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { let __dict__ = identifier!(vm, __dict__); - if !typ.attributes.read().contains_key(&__dict__) { + let has_inherited_dict = typ + .mro + .read() + .iter() + .any(|base| base.attributes.read().contains_key(&__dict__)); + if !typ.attributes.read().contains_key(&__dict__) && !has_inherited_dict { unsafe { let descriptor = vm.ctx @@ -1313,15 +1628,17 @@ impl Constructor for PyType { }) .collect::<PyResult<Vec<_>>>()?; for (obj, name, set_name) in attributes { - set_name.call((typ.clone(), name), vm).map_err(|e| { - let err = vm.new_runtime_error(format!( - "Error calling __set_name__ on '{}' instance {} in '{}'", + set_name.call((typ.clone(), name), vm).inspect_err(|e| { + // PEP 678: Add a note to the original exception instead of wrapping it + // (Python 3.12+, gh-77757) + let note = format!( + "Error calling __set_name__ on '{}' instance '{}' in '{}'", obj.class().name(), name, typ.name() - )); - err.set___cause__(Some(e)); - err + ); + // Ignore result - adding a note is best-effort, the original exception is what matters + drop(vm.call_method(e.as_object(), "add_note", (vm.ctx.new_str(note.as_str()),))); })?; } @@ -1374,6 +1691,26 @@ fn get_doc_from_internal_doc<'a>(name: &str, internal_doc: &'a str) -> &'a str { internal_doc } +impl Initializer for PyType { + type Args = FuncArgs; + + // type_init + fn slot_init(_zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // type.__init__() takes 1 or 3 arguments + if args.args.len() == 1 && !args.kwargs.is_empty() { + return Err(vm.new_type_error("type.__init__() takes no keyword arguments".to_owned())); + } + if args.args.len() != 1 && args.args.len() != 3 { + return Err(vm.new_type_error("type.__init__() takes 1 or 3 arguments".to_owned())); + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + impl GetAttr for PyType { fn getattro(zelf: &Py<Self>, name_str: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { #[cold] @@ -1398,11 +1735,9 @@ impl GetAttr for PyType { if let Some(ref attr) = mcl_attr { let attr_class = attr.class(); - let has_descr_set = attr_class - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some(); + let has_descr_set = attr_class.slots.descr_set.load().is_some(); if has_descr_set { - let descr_get = attr_class.mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = attr_class.slots.descr_get.load(); if let Some(descr_get) = descr_get { let mcl = mcl.to_owned().into(); return descr_get(attr.clone(), Some(zelf.to_owned().into()), Some(mcl), vm); @@ -1413,7 +1748,7 @@ impl GetAttr for PyType { let zelf_attr = zelf.get_attr(name); if let Some(attr) = zelf_attr { - let descr_get = attr.class().mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = attr.class().slots.descr_get.load(); if let Some(descr_get) = descr_get { descr_get(attr, None, Some(zelf.to_owned().into()), vm) } else { @@ -1429,8 +1764,8 @@ impl GetAttr for PyType { #[pyclass] impl Py<PyType> { - #[pygetset(name = "__mro__")] - fn get_mro(&self) -> PyTuple { + #[pygetset] + fn __mro__(&self) -> PyTuple { let elements: Vec<PyObjectRef> = self.mro_map_collect(|x| x.as_object().to_owned()); PyTuple::new_unchecked(elements.into_boxed_slice()) } @@ -1451,9 +1786,7 @@ impl Py<PyType> { // CPython returns None if __doc__ is not in the type's own dict if let Some(doc_attr) = self.get_direct_attr(vm.ctx.intern_str("__doc__")) { // If it's a descriptor, call its __get__ method - let descr_get = doc_attr - .class() - .mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = doc_attr.class().slots.descr_get.load(); if let Some(descr_get) = descr_get { descr_get(doc_attr, None, Some(self.to_owned().into()), vm) } else { @@ -1475,7 +1808,7 @@ impl Py<PyType> { })?; // Check if we can set this special type attribute - self.check_set_special_type_attr(&value, identifier!(vm, __doc__), vm)?; + self.check_set_special_type_attr(identifier!(vm, __doc__), vm)?; // Set the __doc__ in the type's dict self.attributes @@ -1528,8 +1861,15 @@ impl SetAttr for PyType { ) -> PyResult<()> { // TODO: pass PyRefExact instead of &str let attr_name = vm.ctx.intern_str(attr_name.as_str()); + if zelf.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '{}' attribute of immutable type '{}'", + attr_name, + zelf.slot_name() + ))); + } if let Some(attr) = zelf.get_class_attr(attr_name) { - let descr_set = attr.class().mro_find_map(|cls| cls.slots.descr_set.load()); + let descr_set = attr.class().slots.descr_set.load(); if let Some(descriptor) = descr_set { return descriptor(&attr, zelf.to_owned().into(), value, vm); } @@ -1563,15 +1903,28 @@ impl Callable for PyType { type Args = FuncArgs; fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { vm_trace!("type_call: {:?}", zelf); - let obj = call_slot_new(zelf.to_owned(), zelf.to_owned(), args.clone(), vm)?; - if (zelf.is(vm.ctx.types.type_type) && args.kwargs.is_empty()) || !obj.fast_isinstance(zelf) - { + if zelf.is(vm.ctx.types.type_type) { + let num_args = args.args.len(); + if num_args == 1 && args.kwargs.is_empty() { + return Ok(args.args[0].obj_type()); + } + if num_args != 3 { + return Err(vm.new_type_error("type() takes 1 or 3 arguments".to_owned())); + } + } + + let obj = if let Some(slot_new) = zelf.slots.new.load() { + slot_new(zelf.to_owned(), args.clone(), vm)? + } else { + return Err(vm.new_type_error(format!("cannot create '{}' instances", zelf.slots.name))); + }; + + if !obj.class().fast_issubclass(zelf) { return Ok(obj); } - let init = obj.class().mro_find_map(|cls| cls.slots.init.load()); - if let Some(init_method) = init { + if let Some(init_method) = obj.class().slots.init.load() { init_method(obj.clone(), args, vm)?; } Ok(obj) @@ -1581,7 +1934,7 @@ impl Callable for PyType { impl AsNumber for PyType { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| or_(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + or: Some(|a, b, vm| or_(a.to_owned(), b.to_owned(), vm)), ..PyNumberMethods::NOT_IMPLEMENTED }; &AS_NUMBER @@ -1673,7 +2026,9 @@ fn subtype_set_dict(obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) - // Call the descriptor's tp_descr_set let descr_set = descr .class() - .mro_find_map(|cls| cls.slots.descr_set.load()) + .slots + .descr_set + .load() .ok_or_else(|| raise_dict_descriptor_error(&obj, vm))?; descr_set(&descr, obj, PySetterValue::Assign(value), vm) } else { @@ -1700,20 +2055,50 @@ pub(crate) fn call_slot_new( args: FuncArgs, vm: &VirtualMachine, ) -> PyResult { + // Check DISALLOW_INSTANTIATION flag on subtype (the type being instantiated) + if subtype + .slots + .flags + .has_feature(PyTypeFlags::DISALLOW_INSTANTIATION) + { + return Err(vm.new_type_error(format!("cannot create '{}' instances", subtype.slot_name()))); + } + + // "is not safe" check (tp_new_wrapper logic) + // Check that the user doesn't do something silly and unsafe like + // object.__new__(dict). To do this, we check that the most derived base + // that's not a heap type is this type. + let mut staticbase = subtype.clone(); + while staticbase.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + if let Some(base) = staticbase.base.as_ref() { + staticbase = base.clone(); + } else { + break; + } + } + + // Check if staticbase's tp_new differs from typ's tp_new + let typ_new = typ.slots.new.load(); + let staticbase_new = staticbase.slots.new.load(); + if typ_new.map(|f| f as usize) != staticbase_new.map(|f| f as usize) { + return Err(vm.new_type_error(format!( + "{}.__new__({}) is not safe, use {}.__new__()", + typ.slot_name(), + subtype.slot_name(), + staticbase.slot_name() + ))); + } + let slot_new = typ - .deref() - .mro_find_map(|cls| cls.slots.new.load()) + .slots + .new + .load() .expect("Should be able to find a new slot somewhere in the mro"); slot_new(subtype, args, vm) } -pub(super) fn or_(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if !union_::is_unionable(zelf.clone(), vm) || !union_::is_unionable(other.clone(), vm) { - return vm.ctx.not_implemented(); - } - - let tuple = PyTuple::new_ref(vec![zelf, other], &vm.ctx); - union_::make_union(&tuple, vm) +pub(crate) fn or_(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + union_::or_op(zelf, other, vm) } fn take_next_base(bases: &mut [Vec<PyTypeRef>]) -> Option<PyTypeRef> { @@ -1747,9 +2132,10 @@ fn linearise_mro(mut bases: Vec<Vec<PyTypeRef>>) -> Result<Vec<PyTypeRef>, Strin // We start at index 1 to skip direct bases. // This will not catch duplicate bases, but such a thing is already tested for. if later_mro[1..].iter().any(|cls| cls.is(base)) { - return Err( - "Unable to find mro order which keeps local precedence ordering".to_owned(), - ); + return Err(format!( + "Cannot create a consistent method resolution order (MRO) for bases {}", + bases.iter().map(|x| x.first().unwrap()).format(", ") + )); } } } @@ -1781,6 +2167,8 @@ fn calculate_meta_class( let mut winner = metatype; for base in bases { let base_type = base.class(); + + // First try fast_issubclass for PyType instances if winner.fast_issubclass(base_type) { continue; } else if base_type.fast_issubclass(&winner) { @@ -1788,6 +2176,19 @@ fn calculate_meta_class( continue; } + // If fast_issubclass didn't work, fall back to general is_subclass + // This handles cases where metaclasses are not PyType subclasses + let winner_is_subclass = winner.as_object().is_subclass(base_type.as_object(), vm)?; + if winner_is_subclass { + continue; + } + + let base_type_is_subclass = base_type.as_object().is_subclass(winner.as_object(), vm)?; + if base_type_is_subclass { + winner = base_type.to_owned(); + continue; + } + return Err(vm.new_type_error( "metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass \ of the metaclasses of all its bases", @@ -1796,6 +2197,11 @@ fn calculate_meta_class( Ok(winner) } +/// Returns true if the two types have different instance layouts. +fn shape_differs(t1: &Py<PyType>, t2: &Py<PyType>) -> bool { + t1.__basicsize__() != t2.__basicsize__() || t1.slots.itemsize != t2.slots.itemsize +} + fn solid_base<'a>(typ: &'a Py<PyType>, vm: &VirtualMachine) -> &'a Py<PyType> { let base = if let Some(base) = &typ.base { solid_base(base, vm) @@ -1803,12 +2209,7 @@ fn solid_base<'a>(typ: &'a Py<PyType>, vm: &VirtualMachine) -> &'a Py<PyType> { vm.ctx.types.object_type }; - // TODO: requires itemsize comparison too - if typ.__basicsize__() != base.__basicsize__() { - typ - } else { - base - } + if shape_differs(typ, base) { typ } else { base } } fn best_base<'a>(bases: &'a [PyTypeRef], vm: &VirtualMachine) -> PyResult<&'a Py<PyType>> { diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index 16d2b7831cd..03939422c0c 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -1,23 +1,28 @@ use super::{genericalias, type_}; +use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, - builtins::{PyFrozenSet, PyGenericAlias, PyStr, PyTuple, PyTupleRef, PyType}, + builtins::{PyFrozenSet, PySet, PyStr, PyTuple, PyTupleRef, PyType}, class::PyClassImpl, common::hash, - convert::{ToPyObject, ToPyResult}, + convert::ToPyObject, function::PyComparisonValue, protocol::{PyMappingMethods, PyNumberMethods}, + stdlib::typing::{TypeAliasType, call_typing_func_object}, types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, }; -use std::fmt; -use std::sync::LazyLock; +use alloc::fmt; const CLS_ATTRS: &[&str] = &["__module__"]; -#[pyclass(module = "types", name = "UnionType", traverse)] +#[pyclass(module = "typing", name = "Union", traverse)] pub struct PyUnion { args: PyTupleRef, + /// Frozenset of hashable args, or None if all args were hashable + hashable_args: Option<PyRef<PyFrozenSet>>, + /// Tuple of initially unhashable args, or None if all args were hashable + unhashable_args: Option<PyTupleRef>, parameters: PyTupleRef, } @@ -35,14 +40,20 @@ impl PyPayload for PyUnion { } impl PyUnion { - pub fn new(args: PyTupleRef, vm: &VirtualMachine) -> Self { - let parameters = make_parameters(&args, vm); - Self { args, parameters } + /// Create a new union from dedup result (internal use) + fn from_components(result: UnionComponents, vm: &VirtualMachine) -> PyResult<Self> { + let parameters = make_parameters(&result.args, vm)?; + Ok(Self { + args: result.args, + hashable_args: result.hashable_args, + unhashable_args: result.unhashable_args, + parameters, + }) } /// Direct access to args field, matching CPython's _Py_union_args #[inline] - pub const fn args(&self) -> &PyTupleRef { + pub fn args(&self) -> &Py<PyTuple> { &self.args } @@ -87,10 +98,25 @@ impl PyUnion { } #[pyclass( - flags(BASETYPE), + flags(DISALLOW_INSTANTIATION), with(Hashable, Comparable, AsMapping, AsNumber, Representable) )] impl PyUnion { + #[pygetset] + fn __name__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str("Union").into() + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str("Union").into() + } + + #[pygetset] + fn __origin__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.union_type.to_owned().into() + } + #[pygetset] fn __parameters__(&self) -> PyObjectRef { self.parameters.clone().into() @@ -135,32 +161,81 @@ impl PyUnion { } } - #[pymethod(name = "__ror__")] - #[pymethod] - fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { type_::or_(zelf, other, vm) } + #[pymethod] + fn __mro_entries__(zelf: PyRef<Self>, _args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("Cannot subclass {}", zelf.repr(vm)?))) + } + #[pyclassmethod] fn __class_getitem__( - cls: crate::builtins::PyTypeRef, + _cls: crate::builtins::PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { - PyGenericAlias::from_args(cls, args, vm) + ) -> PyResult { + // Convert args to tuple if not already + let args_tuple = if let Some(tuple) = args.downcast_ref::<PyTuple>() { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![args], &vm.ctx) + }; + + // Check for empty union + if args_tuple.is_empty() { + return Err(vm.new_type_error("Cannot create empty Union")); + } + + // Create union using make_union to properly handle None -> NoneType conversion + make_union(&args_tuple, vm) } } -pub fn is_unionable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { - obj.class().is(vm.ctx.types.none_type) +fn is_unionable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { + let cls = obj.class(); + cls.is(vm.ctx.types.none_type) || obj.downcastable::<PyType>() - || obj.class().is(vm.ctx.types.generic_alias_type) - || obj.class().is(vm.ctx.types.union_type) + || cls.fast_issubclass(vm.ctx.types.generic_alias_type) + || cls.is(vm.ctx.types.union_type) + || obj.downcast_ref::<TypeAliasType>().is_some() +} + +fn type_check(arg: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Fast path to avoid calling into typing.py + if is_unionable(arg.clone(), vm) { + return Ok(arg); + } + let message_str: PyObjectRef = vm + .ctx + .new_str("Union[arg, ...]: each arg must be a type.") + .into(); + call_typing_func_object(vm, "_type_check", (arg, message_str)) } -fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { +fn has_union_operands(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> bool { + let union_type = vm.ctx.types.union_type; + a.class().is(union_type) || b.class().is(union_type) +} + +pub fn or_op(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !has_union_operands(zelf.clone(), other.clone(), vm) + && (!is_unionable(zelf.clone(), vm) || !is_unionable(other.clone(), vm)) + { + return Ok(vm.ctx.not_implemented()); + } + + let left = type_check(zelf, vm)?; + let right = type_check(other, vm)?; + let tuple = PyTuple::new_ref(vec![left, right], &vm.ctx); + make_union(&tuple, vm) +} + +fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { let parameters = genericalias::make_parameters(args, vm); - dedup_and_flatten_args(&parameters, vm) + let result = dedup_and_flatten_args(&parameters, vm)?; + Ok(result.args) } fn flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { @@ -179,6 +254,12 @@ fn flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { flattened_args.extend(pyref.args.iter().cloned()); } else if vm.is_none(arg) { flattened_args.push(vm.ctx.types.none_type.to_owned().into()); + } else if arg.downcast_ref::<PyStr>().is_some() { + // Convert string to ForwardRef + match string_to_forwardref(arg.clone(), vm) { + Ok(fr) => flattened_args.push(fr), + Err(_) => flattened_args.push(arg.clone()), + } } else { flattened_args.push(arg.clone()); }; @@ -187,31 +268,105 @@ fn flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { PyTuple::new_ref(flattened_args, &vm.ctx) } -fn dedup_and_flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { +fn string_to_forwardref(arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Import annotationlib.ForwardRef and create a ForwardRef + let annotationlib = vm.import("annotationlib", 0)?; + let forwardref_cls = annotationlib.get_attr("ForwardRef", vm)?; + forwardref_cls.call((arg,), vm) +} + +/// Components for creating a PyUnion after deduplication +struct UnionComponents { + /// All unique args in order + args: PyTupleRef, + /// Frozenset of hashable args (for fast equality comparison) + hashable_args: Option<PyRef<PyFrozenSet>>, + /// Tuple of unhashable args at creation time (for hash error message) + unhashable_args: Option<PyTupleRef>, +} + +fn dedup_and_flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult<UnionComponents> { let args = flatten_args(args, vm); + // Use set-based deduplication like CPython: + // - For hashable elements: use Python's set semantics (hash + equality) + // - For unhashable elements: use equality comparison + // + // This avoids calling __eq__ when hashes differ, matching CPython behavior + // where `int | BadType` doesn't raise even if BadType.__eq__ raises. + let mut new_args: Vec<PyObjectRef> = Vec::with_capacity(args.len()); + + // Track hashable elements using a Python set (uses hash + equality) + let hashable_set = PySet::default().into_ref(&vm.ctx); + let mut hashable_list: Vec<PyObjectRef> = Vec::new(); + let mut unhashable_list: Vec<PyObjectRef> = Vec::new(); + for arg in &*args { - if !new_args.iter().any(|param| { - param - .rich_compare_bool(arg, PyComparisonOp::Eq, vm) - .expect("types are always comparable") - }) { - new_args.push(arg.clone()); + // Try to hash the element first + match arg.hash(vm) { + Ok(_) => { + // Element is hashable - use set for deduplication + // Set membership uses hash first, then equality only if hashes match + let contains = vm + .call_method(hashable_set.as_ref(), "__contains__", (arg.clone(),)) + .and_then(|r| r.try_to_bool(vm))?; + if !contains { + hashable_set.add(arg.clone(), vm)?; + hashable_list.push(arg.clone()); + new_args.push(arg.clone()); + } + } + Err(_) => { + // Element is unhashable - use equality comparison + let mut is_duplicate = false; + for existing in &unhashable_list { + match existing.rich_compare_bool(arg, PyComparisonOp::Eq, vm) { + Ok(true) => { + is_duplicate = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), + } + } + if !is_duplicate { + unhashable_list.push(arg.clone()); + new_args.push(arg.clone()); + } + } } } new_args.shrink_to_fit(); - PyTuple::new_ref(new_args, &vm.ctx) + // Create hashable_args frozenset if there are hashable elements + let hashable_args = if !hashable_list.is_empty() { + Some(PyFrozenSet::from_iter(vm, hashable_list.into_iter())?.into_ref(&vm.ctx)) + } else { + None + }; + + // Create unhashable_args tuple if there are unhashable elements + let unhashable_args = if !unhashable_list.is_empty() { + Some(PyTuple::new_ref(unhashable_list, &vm.ctx)) + } else { + None + }; + + Ok(UnionComponents { + args: PyTuple::new_ref(new_args, &vm.ctx), + hashable_args, + unhashable_args, + }) } -pub fn make_union(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyObjectRef { - let args = dedup_and_flatten_args(args, vm); - match args.len() { - 1 => args[0].to_owned(), - _ => PyUnion::new(args, vm).to_pyobject(vm), - } +pub fn make_union(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult { + let result = dedup_and_flatten_args(args, vm)?; + Ok(match result.args.len() { + 1 => result.args[0].to_owned(), + _ => PyUnion::from_components(result, vm)?.to_pyobject(vm), + }) } impl PyUnion { @@ -223,14 +378,15 @@ impl PyUnion { needle, vm, )?; - let mut res; + let res; if new_args.is_empty() { - res = make_union(&new_args, vm); + res = make_union(&new_args, vm)?; } else { - res = new_args[0].to_owned(); + let mut tmp = new_args[0].to_owned(); for arg in new_args.iter().skip(1) { - res = vm._or(&res, arg)?; + tmp = vm._or(&tmp, arg)?; } + res = tmp; } Ok(res) @@ -253,7 +409,7 @@ impl AsMapping for PyUnion { impl AsNumber for PyUnion { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| PyUnion::__or__(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + or: Some(|a, b, vm| PyUnion::__or__(a.to_owned(), b.to_owned(), vm)), ..PyNumberMethods::NOT_IMPLEMENTED }; &AS_NUMBER @@ -269,15 +425,62 @@ impl Comparable for PyUnion { ) -> PyResult<PyComparisonValue> { op.eq_only(|| { let other = class_or_notimplemented!(Self, other); - let a = PyFrozenSet::from_iter(vm, zelf.args.into_iter().cloned())?; - let b = PyFrozenSet::from_iter(vm, other.args.into_iter().cloned())?; - Ok(PyComparisonValue::Implemented( - a.into_pyobject(vm).as_object().rich_compare_bool( - b.into_pyobject(vm).as_object(), - PyComparisonOp::Eq, - vm, - )?, - )) + + // Check if lengths are equal + if zelf.args.len() != other.args.len() { + return Ok(PyComparisonValue::Implemented(false)); + } + + // Fast path: if both unions have all hashable args, compare frozensets directly + // Always use Eq here since eq_only handles Ne by negating the result + if zelf.unhashable_args.is_none() + && other.unhashable_args.is_none() + && let (Some(a), Some(b)) = (&zelf.hashable_args, &other.hashable_args) + { + let eq = a + .as_object() + .rich_compare_bool(b.as_object(), PyComparisonOp::Eq, vm)?; + return Ok(PyComparisonValue::Implemented(eq)); + } + + // Slow path: O(n^2) nested loop comparison for unhashable elements + // Check if all elements in zelf.args are in other.args + for arg_a in &*zelf.args { + let mut found = false; + for arg_b in &*other.args { + match arg_a.rich_compare_bool(arg_b, PyComparisonOp::Eq, vm) { + Ok(true) => { + found = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), // Propagate comparison errors + } + } + if !found { + return Ok(PyComparisonValue::Implemented(false)); + } + } + + // Check if all elements in other.args are in zelf.args (for symmetry) + for arg_b in &*other.args { + let mut found = false; + for arg_a in &*zelf.args { + match arg_b.rich_compare_bool(arg_a, PyComparisonOp::Eq, vm) { + Ok(true) => { + found = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), // Propagate comparison errors + } + } + if !found { + return Ok(PyComparisonValue::Implemented(false)); + } + } + + Ok(PyComparisonValue::Implemented(true)) }) } } @@ -285,7 +488,36 @@ impl Comparable for PyUnion { impl Hashable for PyUnion { #[inline] fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { - let set = PyFrozenSet::from_iter(vm, zelf.args.into_iter().cloned())?; + // If there are any unhashable args from creation time, the union is unhashable + if let Some(ref unhashable_args) = zelf.unhashable_args { + let n = unhashable_args.len(); + // Try to hash each previously unhashable arg to get an error + for arg in unhashable_args.iter() { + arg.hash(vm)?; + } + // All previously unhashable args somehow became hashable + // But still raise an error to maintain consistent hashing + return Err(vm.new_type_error(format!( + "union contains {} unhashable element{}", + n, + if n > 1 { "s" } else { "" } + ))); + } + + // If we have a stored frozenset of hashable args, use that + if let Some(ref hashable_args) = zelf.hashable_args { + return PyFrozenSet::hash(hashable_args, vm); + } + + // Fallback: compute hash from args + let mut args_to_hash = Vec::new(); + for arg in &*zelf.args { + match arg.hash(vm) { + Ok(_) => args_to_hash.push(arg.clone()), + Err(e) => return Err(e), + } + } + let set = PyFrozenSet::from_iter(vm, args_to_hash.into_iter())?; PyFrozenSet::hash(&set.into_ref(&vm.ctx), vm) } } diff --git a/crates/vm/src/builtins/weakproxy.rs b/crates/vm/src/builtins/weakproxy.rs index a9221ec876f..437e0dc886e 100644 --- a/crates/vm/src/builtins/weakproxy.rs +++ b/crates/vm/src/builtins/weakproxy.rs @@ -1,17 +1,17 @@ use super::{PyStr, PyStrRef, PyType, PyTypeRef, PyWeak}; +use crate::common::lock::LazyLock; use crate::{ Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, class::PyClassImpl, common::hash::PyHash, function::{OptionalArg, PyComparisonValue, PySetterValue}, - protocol::{PyIter, PyIterReturn, PyMappingMethods, PySequenceMethods}, + protocol::{PyIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, stdlib::builtins::reversed, types::{ - AsMapping, AsSequence, Comparable, Constructor, GetAttr, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SetAttr, + AsMapping, AsNumber, AsSequence, Comparable, Constructor, GetAttr, Hashable, IterNext, + Iterable, PyComparisonOp, Representable, SetAttr, }, }; -use std::sync::LazyLock; #[pyclass(module = false, name = "weakproxy", unhashable = true, traverse)] #[derive(Debug)] @@ -68,6 +68,7 @@ crate::common::static_cell! { SetAttr, Constructor, Comparable, + AsNumber, AsSequence, AsMapping, Representable, @@ -79,19 +80,14 @@ impl PyWeakProxy { } #[pymethod] - fn __str__(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - self.try_upgrade(vm)?.str(vm) + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + zelf.try_upgrade(vm)?.str(vm) } fn len(&self, vm: &VirtualMachine) -> PyResult<usize> { self.try_upgrade(vm)?.length(vm) } - #[pymethod] - fn __bool__(&self, vm: &VirtualMachine) -> PyResult<bool> { - self.try_upgrade(vm)?.is_true(vm) - } - #[pymethod] fn __bytes__(&self, vm: &VirtualMachine) -> PyResult { self.try_upgrade(vm)?.bytes(vm) @@ -102,9 +98,10 @@ impl PyWeakProxy { let obj = self.try_upgrade(vm)?; reversed(obj, vm) } - #[pymethod] fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.try_upgrade(vm)?.to_sequence().contains(&needle, vm) + self.try_upgrade(vm)? + .sequence_unchecked() + .contains(&needle, vm) } fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -169,6 +166,19 @@ impl SetAttr for PyWeakProxy { } } +impl AsNumber for PyWeakProxy { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: LazyLock<PyNumberMethods> = LazyLock::new(|| PyNumberMethods { + boolean: Some(|number, vm| { + let zelf = number.obj.downcast_ref::<PyWeakProxy>().unwrap(); + zelf.try_upgrade(vm)?.is_true(vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }); + &AS_NUMBER + } +} + impl Comparable for PyWeakProxy { fn cmp( zelf: &Py<Self>, diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs index 88d6dbac3ed..19f3f44f071 100644 --- a/crates/vm/src/builtins/weakref.rs +++ b/crates/vm/src/builtins/weakref.rs @@ -4,15 +4,18 @@ use crate::common::{ hash::{self, PyHash}, }; use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, function::{FuncArgs, OptionalArg}, - types::{Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, + types::{ + Callable, Comparable, Constructor, Hashable, Initializer, PyComparisonOp, Representable, + }, }; pub use crate::object::PyWeak; #[derive(FromArgs)] +#[allow(dead_code)] pub struct WeakNewArgs { #[pyarg(positional)] referent: PyObjectRef, @@ -39,8 +42,20 @@ impl Constructor for PyWeak { type Args = WeakNewArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let Self::Args { referent, callback } = args.bind(vm)?; - let weak = referent.downgrade_with_typ(callback.into_option(), cls, vm)?; + // PyArg_UnpackTuple: only process positional args, ignore kwargs. + // Subclass __init__ will handle extra kwargs. + let mut positional = args.args.into_iter(); + let referent = positional.next().ok_or_else(|| { + vm.new_type_error("__new__ expected at least 1 argument, got 0".to_owned()) + })?; + let callback = positional.next(); + if let Some(_extra) = positional.next() { + return Err(vm.new_type_error(format!( + "__new__ expected at most 2 arguments, got {}", + 3 + positional.count() + ))); + } + let weak = referent.downgrade_with_typ(callback, cls, vm)?; Ok(weak.into()) } @@ -49,8 +64,24 @@ impl Constructor for PyWeak { } } +impl Initializer for PyWeak { + type Args = WeakNewArgs; + + // weakref_tp_init: accepts args but does nothing (all init done in slot_new) + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } +} + #[pyclass( - with(Callable, Hashable, Comparable, Constructor, Representable), + with( + Callable, + Hashable, + Comparable, + Constructor, + Initializer, + Representable + ), flags(BASETYPE) )] impl PyWeak { diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index 8593f16fcd9..7e1c1c2220c 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -151,7 +151,7 @@ impl ByteInnerFindOptions { self, len: usize, vm: &VirtualMachine, - ) -> PyResult<(Vec<u8>, std::ops::Range<usize>)> { + ) -> PyResult<(Vec<u8>, core::ops::Range<usize>)> { let sub = match self.sub { Either::A(v) => v.elements.to_vec(), Either::B(int) => vec![int.as_bigint().byte_or(vm)?], @@ -423,12 +423,13 @@ impl PyBytesInner { pub fn fromhex(bytes: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> { let mut iter = bytes.iter().enumerate(); - let mut bytes: Vec<u8> = Vec::with_capacity(bytes.len() / 2); - let i = loop { + let mut result: Vec<u8> = Vec::with_capacity(bytes.len() / 2); + // None means odd number of hex digits, Some(i) means invalid char at position i + let invalid_char: Option<usize> = loop { let (i, &b) = match iter.next() { Some(val) => val, None => { - return Ok(bytes); + return Ok(result); } }; @@ -440,27 +441,49 @@ impl PyBytesInner { b'0'..=b'9' => b - b'0', b'a'..=b'f' => 10 + b - b'a', b'A'..=b'F' => 10 + b - b'A', - _ => break i, + _ => break Some(i), }; let (i, b) = match iter.next() { Some(val) => val, - None => break i + 1, + None => break None, // odd number of hex digits }; let bot = match b { b'0'..=b'9' => b - b'0', b'a'..=b'f' => 10 + b - b'a', b'A'..=b'F' => 10 + b - b'A', - _ => break i, + _ => break Some(i), }; - bytes.push((top << 4) + bot); + result.push((top << 4) + bot); }; - Err(vm.new_value_error(format!( - "non-hexadecimal number found in fromhex() arg at position {i}" - ))) + match invalid_char { + None => Err(vm.new_value_error( + "fromhex() arg must contain an even number of hexadecimal digits".to_owned(), + )), + Some(i) => Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {i}" + ))), + } + } + + /// Parse hex string from str or bytes-like object + pub fn fromhex_object(string: PyObjectRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + if let Some(s) = string.downcast_ref::<PyStr>() { + Self::fromhex(s.as_bytes(), vm) + } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &string) { + let borrowed = buffer.as_contiguous().ok_or_else(|| { + vm.new_buffer_error("fromhex() requires a contiguous buffer".to_owned()) + })?; + Self::fromhex(&borrowed, vm) + } else { + Err(vm.new_type_error(format!( + "fromhex() argument must be str or bytes-like, not {}", + string.class().name() + ))) + } } #[inline] @@ -719,7 +742,7 @@ impl PyBytesInner { // len(self)>=1, from="", len(to)>=1, max_count>=1 fn replace_interleave(&self, to: Self, max_count: Option<usize>) -> Vec<u8> { let place_count = self.elements.len() + 1; - let count = max_count.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; + let count = max_count.map_or(place_count, |v| core::cmp::min(v, place_count)) - 1; let capacity = self.elements.len() + count * to.len(); let mut result = Vec::with_capacity(capacity); let to_slice = to.elements.as_slice(); @@ -952,7 +975,7 @@ where fn count_substring(haystack: &[u8], needle: &[u8], max_count: Option<usize>) -> usize { let substrings = haystack.find_iter(needle); if let Some(max_count) = max_count { - std::cmp::min(substrings.take(max_count).count(), max_count) + core::cmp::min(substrings.take(max_count).count(), max_count) } else { substrings.count() } @@ -1025,11 +1048,11 @@ impl AnyStr for [u8] { self.iter().copied() } - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { &self[range] } @@ -1120,7 +1143,7 @@ fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { let len = bytes.len(); let buf = if bytes_per_sep < 0 { - let bytes_per_sep = std::cmp::min(len, (-bytes_per_sep) as usize); + let bytes_per_sep = core::cmp::min(len, (-bytes_per_sep) as usize); let chunks = (len - 1) / bytes_per_sep; let chunked = chunks * bytes_per_sep; let unchunked = len - chunked; @@ -1139,7 +1162,7 @@ fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { hex::encode_to_slice(&bytes[chunked..], &mut buf[j..j + unchunked * 2]).unwrap(); buf } else { - let bytes_per_sep = std::cmp::min(len, bytes_per_sep as usize); + let bytes_per_sep = core::cmp::min(len, bytes_per_sep as usize); let chunks = (len - 1) / bytes_per_sep; let chunked = chunks * bytes_per_sep; let unchunked = len - chunked; diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index 507079e7deb..939b1c7760f 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -6,7 +6,8 @@ use crate::common::cformat::*; use crate::common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use crate::{ - AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyFloat, PyInt, PyStr, try_f64_to_bigint, tuple, }, @@ -207,7 +208,7 @@ fn spec_format_string( fn try_update_quantity_from_element( vm: &VirtualMachine, - element: Option<&PyObjectRef>, + element: Option<&PyObject>, ) -> PyResult<CFormatQuantity> { match element { Some(width_obj) => { @@ -224,7 +225,7 @@ fn try_update_quantity_from_element( fn try_conversion_flag_from_tuple( vm: &VirtualMachine, - element: Option<&PyObjectRef>, + element: Option<&PyObject>, ) -> PyResult<CConversionFlags> { match element { Some(width_obj) => { @@ -254,8 +255,11 @@ fn try_update_quantity_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( return Ok(()); }; let element = elements.next(); - f.insert(try_conversion_flag_from_tuple(vm, element)?); - let quantity = try_update_quantity_from_element(vm, element)?; + f.insert(try_conversion_flag_from_tuple( + vm, + element.map(|v| v.as_ref()), + )?); + let quantity = try_update_quantity_from_element(vm, element.map(|v| v.as_ref()))?; *q = Some(quantity); Ok(()) } @@ -268,7 +272,7 @@ fn try_update_precision_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( let Some(CFormatPrecision::Quantity(CFormatQuantity::FromValuesTuple)) = p else { return Ok(()); }; - let quantity = try_update_quantity_from_element(vm, elements.next())?; + let quantity = try_update_quantity_from_element(vm, elements.next().map(|v| v.as_ref()))?; *p = Some(CFormatPrecision::Quantity(quantity)); Ok(()) } @@ -338,7 +342,7 @@ pub(crate) fn cformat_bytes( let values = if let Some(tup) = values_obj.downcast_ref::<tuple::PyTuple>() { tup.as_slice() } else { - std::slice::from_ref(&values_obj) + core::slice::from_ref(&values_obj) }; let mut value_iter = values.iter(); @@ -431,7 +435,7 @@ pub(crate) fn cformat_string( let values = if let Some(tup) = values_obj.downcast_ref::<tuple::PyTuple>() { tup.as_slice() } else { - std::slice::from_ref(&values_obj) + core::slice::from_ref(&values_obj) }; let mut value_iter = values.iter(); diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs index 6b5a02dea73..3075b59f0bb 100644 --- a/crates/vm/src/class.rs +++ b/crates/vm/src/class.rs @@ -1,14 +1,68 @@ //! Utilities to define a new Python class use crate::{ - builtins::{PyBaseObject, PyType, PyTypeRef}, + PyPayload, + builtins::{PyBaseObject, PyType, PyTypeRef, descriptor::PyWrapper}, function::PyMethodDef, object::Py, - types::{PyTypeFlags, PyTypeSlots, hash_not_implemented}, + types::{PyTypeFlags, PyTypeSlots, SLOT_DEFS, hash_not_implemented}, vm::Context, }; use rustpython_common::static_cell; +/// Add slot wrapper descriptors to a type's dict +/// +/// Iterates SLOT_DEFS and creates a PyWrapper for each slot that: +/// 1. Has a function set in the type's slots +/// 2. Doesn't already have an attribute in the type's dict +pub fn add_operators(class: &'static Py<PyType>, ctx: &Context) { + for def in SLOT_DEFS.iter() { + // Skip __new__ - it has special handling + if def.name == "__new__" { + continue; + } + + // Special handling for __hash__ = None + if def.name == "__hash__" + && class + .slots + .hash + .load() + .is_some_and(|h| h as usize == hash_not_implemented as *const () as usize) + { + class.set_attr(ctx.names.__hash__, ctx.none.clone().into()); + continue; + } + + // __getattr__ should only have a wrapper if the type explicitly defines it. + // Unlike __getattribute__, __getattr__ is not present on object by default. + // Both map to TpGetattro, but only __getattribute__ gets a wrapper from the slot. + if def.name == "__getattr__" { + continue; + } + + // Get the slot function wrapped in SlotFunc + let Some(slot_func) = def.accessor.get_slot_func_with_op(&class.slots, def.op) else { + continue; + }; + + // Check if attribute already exists in dict + let attr_name = ctx.intern_str(def.name); + if class.attributes.read().contains_key(attr_name) { + continue; + } + + // Create and add the wrapper + let wrapper = PyWrapper { + typ: class, + name: attr_name, + wrapped: slot_func, + doc: Some(def.doc), + }; + class.set_attr(attr_name, wrapper.into_ref(ctx).into()); + } +} + pub trait StaticType { // Ideally, saving PyType is better than PyTypeRef fn static_cell() -> &'static static_cell::StaticCell<PyTypeRef>; @@ -66,6 +120,7 @@ pub trait PyClassDef { const TP_NAME: &'static str; const DOC: Option<&'static str> = None; const BASICSIZE: usize; + const ITEMSIZE: usize = 0; const UNHASHABLE: bool = false; // due to restriction of rust trait system, object.__base__ is None @@ -110,23 +165,44 @@ pub trait PyClassImpl: PyClassDef { } } if let Some(module_name) = Self::MODULE_NAME { - class.set_attr( - identifier!(ctx, __module__), - ctx.new_str(module_name).into(), - ); + let module_key = identifier!(ctx, __module__); + // Don't overwrite a getset descriptor for __module__ (e.g. TypeAliasType + // has an instance-level __module__ getset that should not be replaced) + let has_getset = class + .attributes + .read() + .get(module_key) + .is_some_and(|v| v.downcastable::<crate::builtins::PyGetSet>()); + if !has_getset { + class.set_attr(module_key, ctx.new_str(module_name).into()); + } } - if class.slots.new.load().is_some() { - let bound_new = Context::genesis().slot_new_wrapper.build_bound_method( - ctx, - class.to_owned().into(), - class, - ); - class.set_attr(identifier!(ctx, __new__), bound_new.into()); + // Don't add __new__ attribute if slot_new is inherited from object + // (Python doesn't add __new__ to __dict__ for inherited slots) + // Exception: object itself should have __new__ in its dict + if let Some(slot_new) = class.slots.new.load() { + let object_new = ctx.types.object_type.slots.new.load(); + let is_object_itself = core::ptr::eq(class, ctx.types.object_type); + let is_inherited_from_object = !is_object_itself + && object_new.is_some_and(|obj_new| slot_new as usize == obj_new as usize); + + if !is_inherited_from_object { + let bound_new = Context::genesis().slot_new_wrapper.build_bound_method( + ctx, + class.to_owned().into(), + class, + ); + class.set_attr(identifier!(ctx, __new__), bound_new.into()); + } } - if class.slots.hash.load().map_or(0, |h| h as usize) == hash_not_implemented as usize { - class.set_attr(ctx.names.__hash__, ctx.none.clone().into()); + // Add slot wrappers using SLOT_DEFS array + add_operators(class, ctx); + + // Inherit slots from base types after slots are fully initialized + for base in class.bases.read().iter() { + class.inherit_slots(base); } class.extend_methods(class.slots.methods, ctx); @@ -141,7 +217,7 @@ pub trait PyClassImpl: PyClassDef { Self::extend_class(ctx, unsafe { // typ will be saved in static_cell let r: &Py<PyType> = &typ; - let r: &'static Py<PyType> = std::mem::transmute(r); + let r: &'static Py<PyType> = core::mem::transmute(r); r }); typ @@ -158,6 +234,7 @@ pub trait PyClassImpl: PyClassDef { flags: Self::TP_FLAGS, name: Self::TP_NAME, basicsize: Self::BASICSIZE, + itemsize: Self::ITEMSIZE, doc: Self::DOC, methods: Self::METHOD_DEFS, ..Default::default() diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index dac637c396d..9cd75eee55c 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -8,6 +8,7 @@ use rustpython_common::{ wtf8::{CodePoint, Wtf8, Wtf8Buf}, }; +use crate::common::lock::OnceCell; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, @@ -16,12 +17,9 @@ use crate::{ convert::ToPyObject, function::{ArgBytesLike, PyMethodDef}, }; -use once_cell::unsync::OnceCell; -use std::{ - borrow::Cow, - collections::HashMap, - ops::{self, Range}, -}; +use alloc::borrow::Cow; +use core::ops::{self, Range}; +use std::collections::HashMap; pub struct CodecsRegistry { inner: PyRwLock<RegistryInner>, @@ -52,7 +50,7 @@ impl PyCodec { self.0 } #[inline] - pub const fn as_tuple(&self) -> &PyTupleRef { + pub fn as_tuple(&self) -> &Py<PyTuple> { &self.0 } @@ -222,10 +220,11 @@ impl CodecsRegistry { } pub(crate) fn register_manual(&self, name: &str, codec: PyCodec) -> PyResult<()> { + let name = normalize_encoding_name(name); self.inner .write() .search_cache - .insert(name.to_owned(), codec); + .insert(name.into_owned(), codec); Ok(()) } @@ -285,7 +284,9 @@ impl CodecsRegistry { vm: &VirtualMachine, ) -> PyResult { let codec = self.lookup(encoding, vm)?; - codec.encode(obj, errors, vm) + codec.encode(obj, errors, vm).inspect_err(|exc| { + Self::add_codec_note(exc, "encoding", encoding, vm); + }) } pub fn decode( @@ -296,7 +297,9 @@ impl CodecsRegistry { vm: &VirtualMachine, ) -> PyResult { let codec = self.lookup(encoding, vm)?; - codec.decode(obj, errors, vm) + codec.decode(obj, errors, vm).inspect_err(|exc| { + Self::add_codec_note(exc, "decoding", encoding, vm); + }) } pub fn encode_text( @@ -308,12 +311,15 @@ impl CodecsRegistry { ) -> PyResult<PyBytesRef> { let codec = self._lookup_text_encoding(encoding, "codecs.encode()", vm)?; codec - .encode(obj.into(), errors, vm)? + .encode(obj.into(), errors, vm) + .inspect_err(|exc| { + Self::add_codec_note(exc, "encoding", encoding, vm); + })? .downcast() .map_err(|obj| { vm.new_type_error(format!( "'{}' encoder returned '{}' instead of 'bytes'; use codecs.encode() to \ - encode arbitrary types", + encode to arbitrary types", encoding, obj.class().name(), )) @@ -328,20 +334,55 @@ impl CodecsRegistry { vm: &VirtualMachine, ) -> PyResult<PyStrRef> { let codec = self._lookup_text_encoding(encoding, "codecs.decode()", vm)?; - codec.decode(obj, errors, vm)?.downcast().map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.decode() \ - to encode arbitrary types", - encoding, - obj.class().name(), - )) - }) + codec + .decode(obj, errors, vm) + .inspect_err(|exc| { + Self::add_codec_note(exc, "decoding", encoding, vm); + })? + .downcast() + .map_err(|obj| { + vm.new_type_error(format!( + "'{}' decoder returned '{}' instead of 'str'; use codecs.decode() to \ + decode to arbitrary types", + encoding, + obj.class().name(), + )) + }) + } + + fn add_codec_note( + exc: &crate::builtins::PyBaseExceptionRef, + operation: &str, + encoding: &str, + vm: &VirtualMachine, + ) { + let note = format!("{operation} with '{encoding}' codec failed"); + let _ = vm.call_method(exc.as_object(), "add_note", (vm.ctx.new_str(note),)); } pub fn register_error(&self, name: String, handler: PyObjectRef) -> Option<PyObjectRef> { self.inner.write().errors.insert(name, handler) } + pub fn unregister_error(&self, name: &str, vm: &VirtualMachine) -> PyResult<bool> { + const BUILTIN_ERROR_HANDLERS: &[&str] = &[ + "strict", + "ignore", + "replace", + "xmlcharrefreplace", + "backslashreplace", + "namereplace", + "surrogatepass", + "surrogateescape", + ]; + if BUILTIN_ERROR_HANDLERS.contains(&name) { + return Err(vm.new_value_error(format!( + "cannot un-register built-in error handler '{name}'" + ))); + } + Ok(self.inner.write().errors.remove(name).is_some()) + } + pub fn lookup_error_opt(&self, name: &str) -> Option<PyObjectRef> { self.inner.read().errors.get(name).cloned() } @@ -353,19 +394,28 @@ impl CodecsRegistry { } fn normalize_encoding_name(encoding: &str) -> Cow<'_, str> { - if let Some(i) = encoding.find(|c: char| c == ' ' || c.is_ascii_uppercase()) { - let mut out = encoding.as_bytes().to_owned(); - for byte in &mut out[i..] { - if *byte == b' ' { - *byte = b'-'; - } else { - byte.make_ascii_lowercase(); + // _Py_normalize_encoding: collapse non-alphanumeric/non-dot chars into + // single underscore, strip non-ASCII, lowercase ASCII letters. + let needs_transform = encoding + .bytes() + .any(|b| b.is_ascii_uppercase() || !b.is_ascii_alphanumeric() && b != b'.'); + if !needs_transform { + return encoding.into(); + } + let mut out = String::with_capacity(encoding.len()); + let mut punct = false; + for c in encoding.chars() { + if c.is_ascii_alphanumeric() || c == '.' { + if punct && !out.is_empty() { + out.push('_'); } + out.push(c.to_ascii_lowercase()); + punct = false; + } else { + punct = true; } - String::from_utf8(out).unwrap().into() - } else { - encoding.into() } + out.into() } #[derive(Eq, PartialEq)] @@ -418,7 +468,7 @@ impl StandardEncoding { } else { None } - } else if encoding == "CP_UTF8" { + } else if encoding == "cp65001" { Some(Self::Utf8) } else { None @@ -478,7 +528,7 @@ impl<'a> DecodeErrorHandler<PyDecodeContext<'a>> for SurrogatePass { let p = &s[byte_range.start..]; fn slice<const N: usize>(p: &[u8]) -> Option<[u8; N]> { - p.get(..N).map(|x| x.try_into().unwrap()) + p.first_chunk().copied() } let c = match standard_encoding { @@ -811,22 +861,25 @@ impl<'a> ErrorsHandler<'a> { }, None => Self { errors: identifier!(vm, strict).as_ref(), - resolved: OnceCell::with_value(ResolvedError::Standard(StandardError::Strict)), + resolved: OnceCell::from(ResolvedError::Standard(StandardError::Strict)), }, } } #[inline] fn resolve(&self, vm: &VirtualMachine) -> PyResult<&ResolvedError> { - self.resolved.get_or_try_init(|| { - if let Ok(standard) = self.errors.as_str().parse() { - Ok(ResolvedError::Standard(standard)) - } else { - vm.state - .codec_registry - .lookup_error(self.errors.as_str(), vm) - .map(ResolvedError::Handler) - } - }) + if let Some(val) = self.resolved.get() { + return Ok(val); + } + let val = if let Ok(standard) = self.errors.as_str().parse() { + ResolvedError::Standard(standard) + } else { + vm.state + .codec_registry + .lookup_error(self.errors.as_str(), vm) + .map(ResolvedError::Handler)? + }; + let _ = self.resolved.set(val); + Ok(self.resolved.get().unwrap()) } } impl StrBuffer for PyStrRef { @@ -948,7 +1001,7 @@ where encoding: s_encoding.as_str(), data: &s, pos: StrSize::default(), - exception: OnceCell::with_value(err.downcast().unwrap()), + exception: OnceCell::from(err.downcast().unwrap()), }; let mut iter = s.as_wtf8().code_point_indices(); let start = StrSize { @@ -988,7 +1041,7 @@ where data: PyDecodeData::Original(s.borrow_buf()), orig_bytes: s.as_object().downcast_ref(), pos: 0, - exception: OnceCell::with_value(err.downcast().unwrap()), + exception: OnceCell::from(err.downcast().unwrap()), }; let (replace, restart) = handler.handle_decode_error(&mut ctx, range, None)?; Ok((replace.into(), restart)) @@ -1011,7 +1064,7 @@ where encoding: "", data: &s, pos: StrSize::default(), - exception: OnceCell::with_value(err.downcast().unwrap()), + exception: OnceCell::from(err.downcast().unwrap()), }; let mut iter = s.as_wtf8().code_point_indices(); let start = StrSize { diff --git a/crates/vm/src/convert/try_from.rs b/crates/vm/src/convert/try_from.rs index 4f921e9c5de..b8d1b53e2e7 100644 --- a/crates/vm/src/convert/try_from.rs +++ b/crates/vm/src/convert/try_from.rs @@ -122,14 +122,27 @@ impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py<T> { } } -impl TryFromObject for std::time::Duration { +impl TryFromObject for core::time::Duration { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { if let Some(float) = obj.downcast_ref::<PyFloat>() { let f = float.to_f64(); + if f.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)".to_owned())); + } if f < 0.0 { - return Err(vm.new_value_error("negative duration")); + return Err(vm.new_value_error("negative duration".to_owned())); + } + if !f.is_finite() || f > u64::MAX as f64 { + return Err(vm.new_overflow_error( + "timestamp too large to convert to C PyTime_t".to_owned(), + )); } - Ok(Self::from_secs_f64(f)) + // Convert float to Duration using floor rounding (_PyTime_ROUND_FLOOR) + let secs = f.trunc() as u64; + let frac = f.fract(); + // Use floor to round down the nanoseconds + let nanos = (frac * 1_000_000_000.0).floor() as u32; + Ok(Self::new(secs, nanos)) } else if let Some(int) = obj.try_index_opt(vm) { let int = int?; let bigint = int.as_bigint(); diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index 4e76490ed65..ac44e33f799 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -1,8 +1,11 @@ use crate::{ - AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyStrRef}, + AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::PyStrRef, common::lock::PyMutex, - frame::{ExecutionResult, FrameRef}, + exceptions::types::PyBaseException, + frame::{ExecutionResult, Frame, FrameOwner, FrameRef}, + function::OptionalArg, + object::{PyAtomicRef, Traverse, TraverseFn}, protocol::PyIterReturn, }; use crossbeam_utils::atomic::AtomicCell; @@ -33,7 +36,18 @@ pub struct Coro { // _weakreflist name: PyMutex<PyStrRef>, qualname: PyMutex<PyStrRef>, - exception: PyMutex<Option<PyBaseExceptionRef>>, // exc_state + exception: PyAtomicRef<Option<PyBaseException>>, // exc_state +} + +unsafe impl Traverse for Coro { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.frame.traverse(tracer_fn); + self.name.traverse(tracer_fn); + self.qualname.traverse(tracer_fn); + if let Some(exc) = self.exception.deref() { + exc.traverse(tracer_fn); + } + } } fn gen_name(jen: &PyObject, vm: &VirtualMachine) -> &'static str { @@ -53,7 +67,7 @@ impl Coro { frame, closed: AtomicCell::new(false), running: AtomicCell::new(false), - exception: PyMutex::default(), + exception: PyAtomicRef::from(None), name: PyMutex::new(name), qualname: PyMutex::new(qualname), } @@ -61,7 +75,14 @@ impl Coro { fn maybe_close(&self, res: &PyResult<ExecutionResult>) { match res { - Ok(ExecutionResult::Return(_)) | Err(_) => self.closed.store(true), + Ok(ExecutionResult::Return(_)) | Err(_) => { + self.closed.store(true); + // Frame is no longer suspended; allow frame.clear() to succeed. + self.frame.owner.store( + FrameOwner::FrameObject as i8, + core::sync::atomic::Ordering::Release, + ); + } Ok(ExecutionResult::Yield(_)) => {} } } @@ -73,17 +94,22 @@ impl Coro { func: F, ) -> PyResult<ExecutionResult> where - F: FnOnce(FrameRef) -> PyResult<ExecutionResult>, + F: FnOnce(&Py<Frame>) -> PyResult<ExecutionResult>, { if self.running.compare_exchange(false, true).is_err() { return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); } - vm.push_exception(self.exception.lock().take()); - - let result = vm.with_frame(self.frame.clone(), func); + // SAFETY: running.compare_exchange guarantees exclusive access + let gen_exc = unsafe { self.exception.swap(None) }; + let exception_ptr = &self.exception as *const PyAtomicRef<Option<PyBaseException>>; - *self.exception.lock() = vm.pop_exception(); + let result = vm.resume_gen_frame(&self.frame, gen_exc, |f| { + let result = func(f); + // SAFETY: exclusive access guaranteed by running flag + let _old = unsafe { (*exception_ptr).swap(vm.current_exception()) }; + result + }); self.running.store(false); result @@ -98,6 +124,7 @@ impl Coro { if self.closed.load() { return Ok(PyIterReturn::StopIteration(None)); } + self.frame.locals_to_fast(vm)?; let value = if self.frame.lasti() > 0 { Some(value) } else if !vm.is_none(&value) { @@ -139,17 +166,37 @@ impl Coro { exc_tb: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<PyIterReturn> { + // Validate throw arguments (matching CPython _gen_throw) + if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) && !vm.is_none(&exc_val) + { + return Err( + vm.new_type_error("instance exception may not have a separate value".to_owned()) + ); + } + if !vm.is_none(&exc_tb) && !exc_tb.fast_isinstance(vm.ctx.types.traceback_type) { + return Err( + vm.new_type_error("throw() third argument must be a traceback object".to_owned()) + ); + } if self.closed.load() { return Err(vm.normalize_exception(exc_type, exc_val, exc_tb)?); } + // Validate exception type before entering generator context. + // Invalid types propagate to caller without closing the generator. + crate::exceptions::ExceptionCtor::try_from_object(vm, exc_type.clone())?; let result = self.run_with_context(jen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); self.maybe_close(&result); Ok(result?.into_iter_return(vm)) } - pub fn close(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + pub fn close(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult<PyObjectRef> { if self.closed.load() { - return Ok(()); + return Ok(vm.ctx.none()); + } + // If generator hasn't started (FRAME_CREATED), just mark as closed + if self.frame.lasti() == 0 { + self.closed.store(true); + return Ok(vm.ctx.none()); } let result = self.run_with_context(jen, vm, |f| { f.gen_throw( @@ -165,10 +212,15 @@ impl Coro { Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) } Err(e) if !is_gen_exit(&e, vm) => Err(e), - _ => Ok(()), + Ok(ExecutionResult::Return(value)) => Ok(value), + _ => Ok(vm.ctx.none()), } } + pub fn suspended(&self) -> bool { + !self.closed.load() && !self.running.load() && self.frame.lasti() > 0 + } + pub fn running(&self) -> bool { self.running.load() } @@ -198,15 +250,85 @@ impl Coro { } pub fn repr(&self, jen: &PyObject, id: usize, vm: &VirtualMachine) -> String { + let qualname = self.qualname(); format!( "<{} object {} at {:#x}>", gen_name(jen, vm), - self.name.lock(), + qualname.as_str(), id ) } } -pub fn is_gen_exit(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { +pub fn is_gen_exit(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { exc.fast_isinstance(vm.ctx.exceptions.generator_exit) } + +/// Get an awaitable iterator from an object. +/// +/// Returns the object itself if it's a coroutine or iterable coroutine (generator with +/// CO_ITERABLE_COROUTINE flag). Otherwise calls `__await__()` and validates the result. +pub fn get_awaitable_iter(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::{PyCoroutine, PyGenerator}; + use crate::protocol::PyIter; + + if obj.downcastable::<PyCoroutine>() + || obj.downcast_ref::<PyGenerator>().is_some_and(|g| { + g.as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + }) + { + return Ok(obj); + } + + if let Some(await_method) = vm.get_method(obj.clone(), identifier!(vm, __await__)) { + let result = await_method?.call((), vm)?; + // __await__() must NOT return a coroutine (PEP 492) + if result.downcastable::<PyCoroutine>() + || result.downcast_ref::<PyGenerator>().is_some_and(|g| { + g.as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + }) + { + return Err(vm.new_type_error("__await__() returned a coroutine".to_owned())); + } + if !PyIter::check(&result) { + return Err(vm.new_type_error(format!( + "__await__() returned non-iterator of type '{}'", + result.class().name() + ))); + } + return Ok(result); + } + + Err(vm.new_type_error(format!("'{}' object can't be awaited", obj.class().name()))) +} + +/// Emit DeprecationWarning for the deprecated 3-argument throw() signature. +pub fn warn_deprecated_throw_signature( + exc_val: &OptionalArg, + exc_tb: &OptionalArg, + vm: &VirtualMachine, +) -> PyResult<()> { + if exc_val.is_present() || exc_tb.is_present() { + crate::warn::warn( + vm.ctx + .new_str( + "the (type, val, tb) signature of throw() is deprecated, \ + use throw(val) instead", + ) + .into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + Ok(()) +} diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 02e237afb00..f2a379d99a5 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -16,8 +16,9 @@ use crate::{ }, object::{Traverse, TraverseFn}, }; +use alloc::fmt; +use core::{mem::size_of, ops::ControlFlow}; use num_traits::ToPrimitive; -use std::{fmt, mem::size_of, ops::ControlFlow}; // HashIndex is intended to be same size with hash::PyHash // but it doesn't mean the values are compatible with actual PyHash value @@ -281,7 +282,7 @@ impl<T: Clone> Dict<T> { continue; }; if entry.index == index_index { - let removed = std::mem::replace(&mut entry.value, value); + let removed = core::mem::replace(&mut entry.value, value); // defer dec RC break Some(removed); } else { @@ -357,7 +358,7 @@ impl<T: Clone> Dict<T> { inner.used = 0; inner.filled = 0; // defer dec rc - std::mem::take(&mut inner.entries) + core::mem::take(&mut inner.entries) }; } @@ -552,6 +553,22 @@ impl<T: Clone> Dict<T> { .collect() } + pub fn values(&self) -> Vec<T> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| v.value.clone())) + .collect() + } + + pub fn items(&self) -> Vec<(PyObjectRef, T)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.value.clone()))) + .collect() + } + pub fn try_fold_keys<Acc, Fold>(&self, init: Acc, f: Fold) -> PyResult<Acc> where Fold: FnMut(Acc, &PyObject) -> PyResult<Acc>, @@ -633,7 +650,7 @@ impl<T: Clone> Dict<T> { // returns Err(()) if changed since lookup fn pop_inner(&self, lookup: LookupResult) -> PopInnerResult<T> { - self.pop_inner_if(lookup, |_| Ok::<_, std::convert::Infallible>(true)) + self.pop_inner_if(lookup, |_| Ok::<_, core::convert::Infallible>(true)) .unwrap_or_else(|x| match x {}) } @@ -707,6 +724,17 @@ impl<T: Clone> Dict<T> { + inner.indices.len() * size_of::<i64>() + inner.entries.len() * size_of::<DictEntry<T>>() } + + /// Pop all entries from the dict, returning (key, value) pairs. + /// This is used for circular reference resolution in GC. + /// Requires &mut self to avoid lock contention. + pub fn drain_entries(&mut self) -> impl Iterator<Item = (PyObjectRef, T)> + '_ { + let inner = self.inner.get_mut(); + inner.used = 0; + inner.filled = 0; + inner.indices.iter_mut().for_each(|i| *i = IndexEntry::FREE); + inner.entries.drain(..).flatten().map(|e| (e.key, e.value)) + } } type LookupResult = (IndexEntry, IndexIndex); diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 8d033b26110..7ad27c078af 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -43,13 +43,14 @@ pub(super) mod types { use super::*; use crate::PyPayload; use crate::builtins::PyGenericAlias; + use crate::types::{Constructor, Initializer}; #[pyexception(name, base = PyBaseException, ctx = "base_exception_group")] #[derive(Debug)] #[repr(transparent)] pub struct PyBaseExceptionGroup(PyBaseException); - #[pyexception] + #[pyexception(with(Constructor, Initializer))] impl PyBaseExceptionGroup { #[pyclassmethod] fn __class_getitem__( @@ -60,117 +61,6 @@ pub(super) mod types { PyGenericAlias::from_args(cls, args, vm) } - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // Validate exactly 2 positional arguments - if args.args.len() != 2 { - return Err(vm.new_type_error(format!( - "BaseExceptionGroup.__new__() takes exactly 2 positional arguments ({} given)", - args.args.len() - ))); - } - - // Validate message is str - let message = args.args[0].clone(); - if !message.fast_isinstance(vm.ctx.types.str_type) { - return Err(vm.new_type_error(format!( - "argument 1 must be str, not {}", - message.class().name() - ))); - } - - // Validate exceptions is a sequence (not set or None) - let exceptions_arg = &args.args[1]; - - // Check for set/frozenset (not a sequence - unordered) - if exceptions_arg.fast_isinstance(vm.ctx.types.set_type) - || exceptions_arg.fast_isinstance(vm.ctx.types.frozenset_type) - { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } - - // Check for None - if exceptions_arg.is(&vm.ctx.none) { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } - - let exceptions: Vec<PyObjectRef> = exceptions_arg.try_to_value(vm).map_err(|_| { - vm.new_type_error("second argument (exceptions) must be a sequence") - })?; - - // Validate non-empty - if exceptions.is_empty() { - return Err(vm.new_value_error( - "second argument (exceptions) must be a non-empty sequence".to_owned(), - )); - } - - // Validate all items are BaseException instances - let mut has_non_exception = false; - for (i, exc) in exceptions.iter().enumerate() { - if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { - return Err(vm.new_value_error(format!( - "Item {} of second argument (exceptions) is not an exception", - i - ))); - } - // Check if any exception is not an Exception subclass - // With dynamic ExceptionGroup (inherits from both BaseExceptionGroup and Exception), - // ExceptionGroup instances are automatically instances of Exception - if !exc.fast_isinstance(vm.ctx.exceptions.exception_type) { - has_non_exception = true; - } - } - - // Get the dynamic ExceptionGroup type - let exception_group_type = crate::exception_group::exception_group(); - - // Determine the actual class to use - let actual_cls = if cls.is(exception_group_type) { - // ExceptionGroup cannot contain BaseExceptions that are not Exception - if has_non_exception { - return Err( - vm.new_type_error("Cannot nest BaseExceptions in an ExceptionGroup") - ); - } - cls - } else if cls.is(vm.ctx.exceptions.base_exception_group) { - // Auto-convert to ExceptionGroup if all are Exception subclasses - if !has_non_exception { - exception_group_type.to_owned() - } else { - cls - } - } else { - // User-defined subclass - if has_non_exception && cls.fast_issubclass(vm.ctx.exceptions.exception_type) { - return Err(vm.new_type_error(format!( - "Cannot nest BaseExceptions in '{}'", - cls.name() - ))); - } - cls - }; - - // Create the exception with (message, exceptions_tuple) as args - let exceptions_tuple = vm.ctx.new_tuple(exceptions); - let init_args = vec![message, exceptions_tuple.into()]; - PyBaseException::new(init_args, vm) - .into_ref_with_type(vm, actual_cls) - .map(Into::into) - } - - #[pyslot] - #[pymethod(name = "__init__")] - fn slot_init(_zelf: PyObjectRef, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<()> { - // CPython's BaseExceptionGroup.__init__ just calls BaseException.__init__ - // which stores args as-is. Since __new__ already set up the correct args - // (message, exceptions_tuple), we don't need to do anything here. - // This also allows subclasses to pass extra arguments to __new__ without - // __init__ complaining about argument count. - Ok(()) - } - #[pymethod] fn derive( zelf: PyRef<PyBaseException>, @@ -292,7 +182,7 @@ pub(super) mod types { } #[pymethod] - fn __str__(zelf: PyRef<PyBaseException>, vm: &VirtualMachine) -> PyResult<String> { + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { let message = zelf .get_arg(0) .map(|m| m.str(vm)) @@ -306,15 +196,10 @@ pub(super) mod types { .unwrap_or(0); let suffix = if num_excs == 1 { "" } else { "s" }; - Ok(format!( + Ok(vm.ctx.new_str(format!( "{} ({} sub-exception{})", message, num_excs, suffix - )) - } - - #[pymethod] - fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Self::slot_repr(&zelf, vm) + ))) } #[pyslot] @@ -351,13 +236,145 @@ pub(super) mod types { } } + impl Constructor for PyBaseExceptionGroup { + type Args = crate::function::PosArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let args = args.into_vec(); + // Validate exactly 2 positional arguments + if args.len() != 2 { + return Err(vm.new_type_error(format!( + "BaseExceptionGroup.__new__() takes exactly 2 positional arguments ({} given)", + args.len() + ))); + } + + // Validate message is str + let message = args[0].clone(); + if !message.fast_isinstance(vm.ctx.types.str_type) { + return Err(vm.new_type_error(format!( + "argument 1 must be str, not {}", + message.class().name() + ))); + } + + // Validate exceptions is a sequence (not set or None) + let exceptions_arg = &args[1]; + + // Check for set/frozenset (not a sequence - unordered) + if exceptions_arg.fast_isinstance(vm.ctx.types.set_type) + || exceptions_arg.fast_isinstance(vm.ctx.types.frozenset_type) + { + return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); + } + + // Check for None + if exceptions_arg.is(&vm.ctx.none) { + return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); + } + + let exceptions: Vec<PyObjectRef> = exceptions_arg.try_to_value(vm).map_err(|_| { + vm.new_type_error("second argument (exceptions) must be a sequence") + })?; + + // Validate non-empty + if exceptions.is_empty() { + return Err(vm.new_value_error( + "second argument (exceptions) must be a non-empty sequence".to_owned(), + )); + } + + // Validate all items are BaseException instances + let mut has_non_exception = false; + for (i, exc) in exceptions.iter().enumerate() { + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_value_error(format!( + "Item {} of second argument (exceptions) is not an exception", + i + ))); + } + // Check if any exception is not an Exception subclass + // With dynamic ExceptionGroup (inherits from both BaseExceptionGroup and Exception), + // ExceptionGroup instances are automatically instances of Exception + if !exc.fast_isinstance(vm.ctx.exceptions.exception_type) { + has_non_exception = true; + } + } + + // Get the dynamic ExceptionGroup type + let exception_group_type = crate::exception_group::exception_group(); + + // Determine the actual class to use + let actual_cls = if cls.is(exception_group_type) { + // ExceptionGroup cannot contain BaseExceptions that are not Exception + if has_non_exception { + return Err( + vm.new_type_error("Cannot nest BaseExceptions in an ExceptionGroup") + ); + } + cls + } else if cls.is(vm.ctx.exceptions.base_exception_group) { + // Auto-convert to ExceptionGroup if all are Exception subclasses + if !has_non_exception { + exception_group_type.to_owned() + } else { + cls + } + } else { + // User-defined subclass + if has_non_exception && cls.fast_issubclass(vm.ctx.exceptions.exception_type) { + return Err(vm.new_type_error(format!( + "Cannot nest BaseExceptions in '{}'", + cls.name() + ))); + } + cls + }; + + // Create the exception with (message, exceptions_tuple) as args + let exceptions_tuple = vm.ctx.new_tuple(exceptions); + let init_args = vec![message, exceptions_tuple.into()]; + PyBaseException::new(init_args, vm) + .into_ref_with_type(vm, actual_cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Initializer for PyBaseExceptionGroup { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // BaseExceptionGroup_init: no kwargs allowed + if !args.kwargs.is_empty() { + return Err(vm.new_type_error(format!( + "{} does not take keyword arguments", + zelf.class().name() + ))); + } + // Do NOT call PyBaseException::slot_init here. + // slot_new already set args to (message, exceptions_tuple). + // Calling base init would overwrite with original args (message, exceptions_list). + let _ = (zelf, args, vm); + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is overridden") + } + } + // Helper functions for ExceptionGroup - fn is_base_exception_group(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { + fn is_base_exception_group(obj: &PyObject, vm: &VirtualMachine) -> bool { obj.fast_isinstance(vm.ctx.exceptions.base_exception_group) } fn get_exceptions_tuple( - exc: &PyRef<PyBaseException>, + exc: &Py<PyBaseException>, vm: &VirtualMachine, ) -> PyResult<Vec<PyObjectRef>> { let obj = exc @@ -376,7 +393,7 @@ pub(super) mod types { } fn get_condition_matcher( - condition: &PyObjectRef, + condition: &PyObject, vm: &VirtualMachine, ) -> PyResult<ConditionMatcher> { // If it's a type and subclass of BaseException @@ -409,19 +426,19 @@ pub(super) mod types { // If it's callable (but not a type) if condition.is_callable() && condition.downcast_ref::<PyType>().is_none() { - return Ok(ConditionMatcher::Callable(condition.clone())); + return Ok(ConditionMatcher::Callable(condition.to_owned())); } Err(vm.new_type_error("expected a function, exception type or tuple of exception types")) } impl ConditionMatcher { - fn check(&self, exc: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + fn check(&self, exc: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { match self { ConditionMatcher::Type(typ) => Ok(exc.fast_isinstance(typ)), ConditionMatcher::Types(types) => Ok(types.iter().any(|t| exc.fast_isinstance(t))), ConditionMatcher::Callable(func) => { - let result = func.call((exc.clone(),), vm)?; + let result = func.call((exc.to_owned(),), vm)?; result.try_to_bool(vm) } } @@ -429,7 +446,7 @@ pub(super) mod types { } fn derive_and_copy_attributes( - orig: &PyRef<PyBaseException>, + orig: &Py<PyBaseException>, excs: Vec<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<PyObjectRef> { diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 04dd78fb448..192549d096b 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -8,8 +8,8 @@ use crate::{ traceback::{PyTraceback, PyTracebackRef}, }, class::{PyClassImpl, StaticType}, - convert::{ToPyException, ToPyObject}, - function::{ArgIterable, FuncArgs, IntoFuncArgs}, + convert::{IntoPyException, ToPyException, ToPyObject}, + function::{ArgIterable, FuncArgs, IntoFuncArgs, PySetterValue}, py_io::{self, Write}, stdlib::sys, suggestion::offer_suggestions, @@ -33,8 +33,8 @@ unsafe impl Traverse for PyBaseException { } } -impl std::fmt::Debug for PyBaseException { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyBaseException { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("PyBaseException") } @@ -79,7 +79,7 @@ impl VirtualMachine { pub fn write_exception<W: Write>( &self, output: &mut W, - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, ) -> Result<(), W::Error> { let seen = &mut HashSet::<usize>::new(); self.write_exception_recursive(output, exc, seen) @@ -88,7 +88,7 @@ impl VirtualMachine { fn write_exception_recursive<W: Write>( &self, output: &mut W, - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, seen: &mut HashSet<usize>, ) -> Result<(), W::Error> { // This function should not be called directly, @@ -132,7 +132,7 @@ impl VirtualMachine { pub fn write_exception_inner<W: Write>( &self, output: &mut W, - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, ) -> Result<(), W::Error> { let vm = self; if let Some(tb) = exc.traceback.read().clone() { @@ -177,7 +177,7 @@ impl VirtualMachine { fn write_syntaxerror<W: Write>( &self, output: &mut W, - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, exc_type: &Py<PyType>, args_repr: &[PyRef<PyStr>], ) -> Result<(), W::Error> { @@ -210,8 +210,8 @@ impl VirtualMachine { } if let Some(text) = maybe_text { - // if text ends with \n, remove it - let r_text = text.as_str().trim_end_matches('\n'); + // if text ends with \n or \r\n, remove it + let r_text = text.as_str().trim_end_matches(['\n', '\r']); let l_text = r_text.trim_start_matches([' ', '\n', '\x0c']); // \x0c is \f let spaces = (r_text.len() - l_text.len()) as isize; @@ -223,37 +223,49 @@ impl VirtualMachine { if let Some(offset) = maybe_offset { let maybe_end_offset: Option<isize> = getattr("end_offset").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); - - let mut end_offset = match maybe_end_offset { - Some(0) | None => offset, - Some(end_offset) => end_offset, + let maybe_end_lineno: Option<isize> = + getattr("end_lineno").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + let maybe_lineno_int: Option<isize> = + getattr("lineno").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + + // Only show caret if end_lineno is same as lineno (or not set) + let same_line = match (maybe_lineno_int, maybe_end_lineno) { + (Some(lineno), Some(end_lineno)) => lineno == end_lineno, + _ => true, }; - if offset == end_offset || end_offset == -1 { - end_offset = offset + 1; - } + if same_line { + let mut end_offset = match maybe_end_offset { + Some(0) | None => offset, + Some(end_offset) => end_offset, + }; - // Convert 1-based column offset to 0-based index into stripped text - let colno = offset - 1 - spaces; - let end_colno = end_offset - 1 - spaces; - if colno >= 0 { - let caret_space = l_text - .chars() - .take(colno as usize) - .map(|c| if c.is_whitespace() { c } else { ' ' }) - .collect::<String>(); - - let mut error_width = end_colno - colno; - if error_width < 1 { - error_width = 1; + if offset == end_offset || end_offset == -1 { + end_offset = offset + 1; } - writeln!( - output, - " {}{}", - caret_space, - "^".repeat(error_width as usize) - )?; + // Convert 1-based column offset to 0-based index into stripped text + let colno = offset - 1 - spaces; + let end_colno = end_offset - 1 - spaces; + if colno >= 0 { + let caret_space = l_text + .chars() + .take(colno as usize) + .map(|c| if c.is_whitespace() { c } else { ' ' }) + .collect::<String>(); + + let mut error_width = end_colno - colno; + if error_width < 1 { + error_width = 1; + } + + writeln!( + output, + " {}{}", + caret_space, + "^".repeat(error_width as usize) + )?; + } } } } @@ -336,7 +348,13 @@ impl VirtualMachine { ) -> PyResult<PyBaseExceptionRef> { // TODO: fast-path built-in exceptions by directly instantiating them? Is that really worth it? let res = PyType::call(&cls, args.into_args(self), self)?; - PyBaseExceptionRef::try_from_object(self, res) + res.downcast::<PyBaseException>().map_err(|obj| { + self.new_type_error(format!( + "calling {} should have returned an instance of BaseException, not {}", + cls, + obj.class() + )) + }) } } @@ -369,9 +387,9 @@ fn print_source_line<W: Write>( /// Print exception occurrence location from traceback element fn write_traceback_entry<W: Write>( output: &mut W, - tb_entry: &PyTracebackRef, + tb_entry: &Py<PyTraceback>, ) -> Result<(), W::Error> { - let filename = tb_entry.frame.code.source_path.as_str(); + let filename = tb_entry.frame.code.source_path().as_str(); writeln!( output, r##" File "{}", line {}, in {}"##, @@ -493,6 +511,7 @@ pub struct ExceptionZoo { pub runtime_error: &'static Py<PyType>, pub not_implemented_error: &'static Py<PyType>, pub recursion_error: &'static Py<PyType>, + pub python_finalization_error: &'static Py<PyType>, pub syntax_error: &'static Py<PyType>, pub incomplete_input_error: &'static Py<PyType>, pub indentation_error: &'static Py<PyType>, @@ -560,7 +579,7 @@ impl PyBaseException { } #[pyclass( - with(PyRef, Constructor, Initializer, Representable), + with(Py, PyRef, Constructor, Initializer, Representable), flags(BASETYPE, HAS_DICT) )] impl PyBaseException { @@ -624,8 +643,8 @@ impl PyBaseException { *self.context.write() = context; } - #[pygetset(name = "__suppress_context__")] - pub(super) fn get_suppress_context(&self) -> bool { + #[pygetset] + pub(super) fn __suppress_context__(&self) -> bool { self.suppress_context.load() } @@ -633,15 +652,18 @@ impl PyBaseException { fn set_suppress_context(&self, suppress_context: bool) { self.suppress_context.store(suppress_context); } +} +#[pyclass] +impl Py<PyBaseException> { #[pymethod] - pub(super) fn __str__(&self, vm: &VirtualMachine) -> PyStrRef { + pub(super) fn __str__(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { let str_args = vm.exception_args_as_string(self.args(), true); - match str_args.into_iter().exactly_one() { + Ok(match str_args.into_iter().exactly_one() { Err(i) if i.len() == 0 => vm.ctx.empty_str.to_owned(), Ok(s) => s, Err(i) => PyStr::from(format!("({})", i.format(", "))).into_ref(&vm.ctx), - } + }) } } @@ -801,6 +823,7 @@ impl ExceptionZoo { let runtime_error = PyRuntimeError::init_builtin_type(); let not_implemented_error = PyNotImplementedError::init_builtin_type(); let recursion_error = PyRecursionError::init_builtin_type(); + let python_finalization_error = PyPythonFinalizationError::init_builtin_type(); let syntax_error = PySyntaxError::init_builtin_type(); let incomplete_input_error = PyIncompleteInputError::init_builtin_type(); @@ -876,6 +899,7 @@ impl ExceptionZoo { runtime_error, not_implemented_error, recursion_error, + python_finalization_error, syntax_error, incomplete_input_error, indentation_error, @@ -907,7 +931,10 @@ impl ExceptionZoo { } // TODO: remove it after fixing `errno` / `winerror` problem - #[allow(clippy::redundant_clone)] + #[allow( + clippy::redundant_clone, + reason = "temporary workaround until errno/winerror handling is fixed" + )] pub fn extend(ctx: &Context) { use self::types::*; @@ -989,9 +1016,19 @@ impl ExceptionZoo { extend_exception!(PyRuntimeError, ctx, excs.runtime_error); extend_exception!(PyNotImplementedError, ctx, excs.not_implemented_error); extend_exception!(PyRecursionError, ctx, excs.recursion_error); + extend_exception!( + PyPythonFinalizationError, + ctx, + excs.python_finalization_error + ); extend_exception!(PySyntaxError, ctx, excs.syntax_error, { - "msg" => ctx.new_readonly_getset("msg", excs.syntax_error, make_arg_getter(0)), + "msg" => ctx.new_static_getset( + "msg", + excs.syntax_error, + make_arg_getter(0), + syntax_error_set_msg, + ), // TODO: members "filename" => ctx.none(), "lineno" => ctx.none(), @@ -1038,27 +1075,47 @@ fn make_arg_getter(idx: usize) -> impl Fn(PyBaseExceptionRef) -> Option<PyObject move |exc| exc.get_arg(idx) } +fn syntax_error_set_msg( + exc: PyBaseExceptionRef, + value: PySetterValue, + vm: &VirtualMachine, +) -> PyResult<()> { + let mut args = exc.args.write(); + let mut new_args = args.as_slice().to_vec(); + // Ensure the message slot at index 0 always exists for SyntaxError.args. + if new_args.is_empty() { + new_args.push(vm.ctx.none()); + } + match value { + PySetterValue::Assign(value) => new_args[0] = value, + PySetterValue::Delete => new_args[0] = vm.ctx.none(), + } + *args = PyTuple::new_ref(new_args, &vm.ctx); + Ok(()) +} + fn system_exit_code(exc: PyBaseExceptionRef) -> Option<PyObjectRef> { - exc.args.read().first().map(|code| { - match_class!(match code { - ref tup @ PyTuple => match tup.as_slice() { - [x] => x.clone(), - _ => code.clone(), - }, - other => other.clone(), - }) - }) + // SystemExit.code based on args length: + // - size == 0: code is None + // - size == 1: code is args[0] + // - size > 1: code is args (the whole tuple) + let args = exc.args.read(); + match args.len() { + 0 => None, + 1 => Some(args.first().unwrap().clone()), + _ => Some(args.as_object().to_owned()), + } } #[cfg(feature = "serde")] pub struct SerializeException<'vm, 's> { vm: &'vm VirtualMachine, - exc: &'s PyBaseExceptionRef, + exc: &'s Py<PyBaseException>, } #[cfg(feature = "serde")] impl<'vm, 's> SerializeException<'vm, 's> { - pub fn new(vm: &'vm VirtualMachine, exc: &'s PyBaseExceptionRef) -> Self { + pub fn new(vm: &'vm VirtualMachine, exc: &'s Py<PyBaseException>) -> Self { SerializeException { vm, exc } } } @@ -1112,7 +1169,7 @@ impl serde::Serialize for SerializeException<'_, '_> { .__context__() .map(|exc| SerializeExceptionOwned { vm: self.vm, exc }), )?; - struc.serialize_field("suppress_context", &self.exc.get_suppress_context())?; + struc.serialize_field("suppress_context", &self.exc.__suppress_context__())?; let args = { struct Args<'vm>(&'vm VirtualMachine, PyTupleRef); @@ -1146,7 +1203,7 @@ pub fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_value_error("embedded null character") } -impl ToPyException for std::ffi::NulError { +impl ToPyException for alloc::ffi::NulError { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { cstring_error(vm) } @@ -1193,6 +1250,198 @@ pub(crate) fn errno_to_exc_type(_errno: i32, _vm: &VirtualMachine) -> Option<&'s None } +pub(crate) trait ToOSErrorBuilder { + fn to_os_error_builder(&self, vm: &VirtualMachine) -> OSErrorBuilder; +} + +pub(crate) struct OSErrorBuilder { + exc_type: PyTypeRef, + errno: Option<i32>, + strerror: Option<PyObjectRef>, + filename: Option<PyObjectRef>, + #[cfg(windows)] + winerror: Option<PyObjectRef>, + filename2: Option<PyObjectRef>, +} + +impl OSErrorBuilder { + #[must_use] + pub fn with_subtype( + exc_type: PyTypeRef, + errno: Option<i32>, + strerror: impl ToPyObject, + vm: &VirtualMachine, + ) -> Self { + let strerror = strerror.to_pyobject(vm); + Self { + exc_type, + errno, + strerror: Some(strerror), + filename: None, + #[cfg(windows)] + winerror: None, + filename2: None, + } + } + + #[must_use] + pub fn with_errno(errno: i32, strerror: impl ToPyObject, vm: &VirtualMachine) -> Self { + let exc_type = errno_to_exc_type(errno, vm) + .unwrap_or(vm.ctx.exceptions.os_error) + .to_owned(); + Self::with_subtype(exc_type, Some(errno), strerror, vm) + } + + #[must_use] + #[allow(dead_code)] + pub(crate) fn filename(mut self, filename: PyObjectRef) -> Self { + self.filename.replace(filename); + self + } + + #[must_use] + #[allow(dead_code)] + pub(crate) fn filename2(mut self, filename: PyObjectRef) -> Self { + self.filename2.replace(filename); + self + } + + #[must_use] + #[cfg(windows)] + pub(crate) fn winerror(mut self, winerror: PyObjectRef) -> Self { + self.winerror.replace(winerror); + self + } + + /// Strip winerror from the builder. Used for C runtime errors + /// (e.g. `_wopen`, `open`) that should produce `[Errno X]` format + /// instead of `[WinError X]`. + #[must_use] + #[cfg(windows)] + pub(crate) fn without_winerror(mut self) -> Self { + self.winerror = None; + self + } + + pub fn build(self, vm: &VirtualMachine) -> PyRef<types::PyOSError> { + use types::PyOSError; + + let OSErrorBuilder { + exc_type, + errno, + strerror, + filename, + #[cfg(windows)] + winerror, + filename2, + } = self; + + let args = if let Some(errno) = errno { + #[cfg(windows)] + let winerror = winerror.to_pyobject(vm); + #[cfg(not(windows))] + let winerror = vm.ctx.none(); + + vec![ + errno.to_pyobject(vm), + strerror.to_pyobject(vm), + filename.to_pyobject(vm), + winerror, + filename2.to_pyobject(vm), + ] + } else { + vec![strerror.to_pyobject(vm)] + }; + + let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) + .expect("new_os_error usage error"); + let os_error = payload + .into_ref_with_type(vm, exc_type) + .expect("new_os_error usage error"); + PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) + .expect("new_os_error usage error"); + os_error + } +} + +impl IntoPyException for OSErrorBuilder { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.build(vm).upcast() + } +} + +impl ToOSErrorBuilder for std::io::Error { + fn to_os_error_builder(&self, vm: &VirtualMachine) -> OSErrorBuilder { + use crate::common::os::ErrorExt; + + let errno = self.posix_errno(); + #[cfg(windows)] + let msg = 'msg: { + // Use C runtime's strerror for POSIX errno values. + // For Windows-specific error codes, fall back to FormatMessage. + const MAX_POSIX_ERRNO: i32 = 127; + if errno > 0 && errno <= MAX_POSIX_ERRNO { + let ptr = unsafe { libc::strerror(errno) }; + if !ptr.is_null() { + let s = unsafe { core::ffi::CStr::from_ptr(ptr) }.to_string_lossy(); + if !s.starts_with("Unknown error") { + break 'msg s.into_owned(); + } + } + } + self.to_string() + }; + #[cfg(unix)] + let msg = { + let ptr = unsafe { libc::strerror(errno) }; + if !ptr.is_null() { + unsafe { core::ffi::CStr::from_ptr(ptr) } + .to_string_lossy() + .into_owned() + } else { + self.to_string() + } + }; + #[cfg(not(any(windows, unix)))] + let msg = self.to_string(); + + #[allow(unused_mut)] + let mut builder = OSErrorBuilder::with_errno(errno, msg, vm); + #[cfg(windows)] + if let Some(winerror) = self.raw_os_error() { + builder = builder.winerror(winerror.to_pyobject(vm)); + } + builder + } +} + +impl ToPyException for std::io::Error { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + let builder = self.to_os_error_builder(vm); + builder.into_pyexception(vm) + } +} + +impl IntoPyException for std::io::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.to_pyexception(vm) + } +} + +#[cfg(unix)] +impl IntoPyException for nix::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + std::io::Error::from(self).into_pyexception(vm) + } +} + +#[cfg(unix)] +impl IntoPyException for rustix::io::Errno { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + std::io::Error::from(self).into_pyexception(vm) + } +} + pub(super) mod types { use crate::common::lock::PyRwLock; use crate::object::{MaybeTraverse, Traverse, TraverseFn}; @@ -1205,7 +1454,7 @@ pub(super) mod types { tuple::IntoPyTuple, }, convert::ToPyResult, - function::{ArgBytesLike, FuncArgs}, + function::{ArgBytesLike, FuncArgs, KwArgs}, types::{Constructor, Initializer}, }; use crossbeam_utils::atomic::AtomicCell; @@ -1232,11 +1481,29 @@ pub(super) mod types { pub(super) args: PyRwLock<PyTupleRef>, } - #[pyexception(name, base = PyBaseException, ctx = "system_exit", impl)] + #[pyexception(name, base = PyBaseException, ctx = "system_exit")] #[derive(Debug)] #[repr(transparent)] pub struct PySystemExit(PyBaseException); + // SystemExit_init: has its own __init__ that sets the code attribute + #[pyexception(with(Initializer))] + impl PySystemExit {} + + impl Initializer for PySystemExit { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Call BaseException_init first (handles args) + PyBaseException::slot_init(zelf, args, vm) + // Note: code is computed dynamically via system_exit_code getter + // so we don't need to set it here explicitly + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + #[pyexception(name, base = PyBaseException, ctx = "generator_exit", impl)] #[derive(Debug)] #[repr(transparent)] @@ -1257,18 +1524,19 @@ pub(super) mod types { #[repr(transparent)] pub struct PyStopIteration(PyException); - #[pyexception] - impl PyStopIteration { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { + #[pyexception(with(Initializer))] + impl PyStopIteration {} + + impl Initializer for PyStopIteration { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { zelf.set_attr("value", vm.unwrap_or_none(args.args.first().cloned()), vm)?; Ok(()) } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } } #[pyexception(name, base = PyException, ctx = "stop_async_iteration", impl)] @@ -1305,27 +1573,38 @@ pub(super) mod types { #[repr(transparent)] pub struct PyAttributeError(PyException); - #[pyexception] - impl PyAttributeError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - zelf.set_attr( - "name", - vm.unwrap_or_none(args.kwargs.get("name").cloned()), - vm, - )?; - zelf.set_attr( - "obj", - vm.unwrap_or_none(args.kwargs.get("obj").cloned()), - vm, - )?; + #[pyexception(with(Initializer))] + impl PyAttributeError {} + + impl Initializer for PyAttributeError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name' and 'obj' kwargs are allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + let obj = kwargs.swap_remove("obj"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "AttributeError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set attributes + zelf.set_attr("name", vm.unwrap_or_none(name), vm)?; + zelf.set_attr("obj", vm.unwrap_or_none(obj), vm)?; Ok(()) } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } } #[pyexception(name, base = PyException, ctx = "buffer_error", impl)] @@ -1343,18 +1622,33 @@ pub(super) mod types { #[repr(transparent)] pub struct PyImportError(PyException); - #[pyexception] + #[pyexception(with(Initializer))] impl PyImportError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { + #[pymethod] + fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { + let obj = exc.as_object().to_owned(); + let mut result: Vec<PyObjectRef> = vec![ + obj.class().to_owned().into(), + vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), + ]; + + if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { + result.push(dict.into()); + } + + result.into_pytuple(vm) + } + } + + impl Initializer for PyImportError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name', 'path', 'name_from' kwargs are allowed let mut kwargs = args.kwargs.clone(); let name = kwargs.swap_remove("name"); let path = kwargs.swap_remove("path"); + let name_from = kwargs.swap_remove("name_from"); // Check for any remaining invalid keyword arguments if let Some(invalid_key) = kwargs.keys().next() { @@ -1366,21 +1660,12 @@ pub(super) mod types { let dict = zelf.dict().unwrap(); dict.set_item("name", vm.unwrap_or_none(name), vm)?; dict.set_item("path", vm.unwrap_or_none(path), vm)?; + dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; PyBaseException::slot_init(zelf, args, vm) } - #[pymethod] - fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { - let obj = exc.as_object().to_owned(); - let mut result: Vec<PyObjectRef> = vec![ - obj.class().to_owned().into(), - vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), - ]; - if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { - result.push(dict.into()); - } - - result.into_pytuple(vm) + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") } } @@ -1407,16 +1692,16 @@ pub(super) mod types { #[pyexception] impl PyKeyError { #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyStrRef { - let args = exc.args(); - if args.len() == 1 { + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let args = zelf.args(); + Ok(if args.len() == 1 { vm.exception_args_as_string(args, false) .into_iter() .exactly_one() .unwrap() } else { - exc.__str__(vm) - } + zelf.__str__(vm)? + }) } } @@ -1425,17 +1710,52 @@ pub(super) mod types { #[repr(transparent)] pub struct PyMemoryError(PyException); - #[pyexception(name, base = PyException, ctx = "name_error", impl)] + #[pyexception(name, base = PyException, ctx = "name_error")] #[derive(Debug)] #[repr(transparent)] pub struct PyNameError(PyException); + // NameError_init: handles the .name. kwarg + #[pyexception(with(Initializer))] + impl PyNameError {} + + impl Initializer for PyNameError { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name' kwarg is allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "NameError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set name attribute if provided + if let Some(name) = name { + zelf.set_attr("name", name, vm)?; + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + #[pyexception(name, base = PyNameError, ctx = "unbound_local_error", impl)] #[derive(Debug)] #[repr(transparent)] pub struct PyUnboundLocalError(PyNameError); #[pyexception(name, base = PyException, ctx = "os_error")] + #[repr(C)] pub struct PyOSError { base: PyException, errno: PyAtomicRef<Option<PyObject>>, @@ -1444,6 +1764,9 @@ pub(super) mod types { filename2: PyAtomicRef<Option<PyObject>>, #[cfg(windows)] winerror: PyAtomicRef<Option<PyObject>>, + // For BlockingIOError: characters written before blocking occurred + // -1 means not set (AttributeError when accessed) + written: AtomicCell<isize>, } impl crate::class::PySubclass for PyOSError { @@ -1453,8 +1776,8 @@ pub(super) mod types { } } - impl std::fmt::Debug for PyOSError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyOSError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyOSError").finish_non_exhaustive() } } @@ -1518,6 +1841,7 @@ pub(super) mod types { filename2: filename2.into(), #[cfg(windows)] winerror: None.into(), + written: AtomicCell::new(-1), }) } @@ -1545,11 +1869,10 @@ pub(super) mod types { } } - #[pyexception(with(Constructor))] - impl PyOSError { - #[pyslot] - #[pymethod(name = "__init__")] - pub fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + impl Initializer for PyOSError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let len = args.args.len(); let mut new_args = args; @@ -1560,16 +1883,37 @@ pub(super) mod types { #[allow(deprecated)] let exc: &Py<PyOSError> = zelf.downcast_ref::<PyOSError>().unwrap(); + // Check if this is BlockingIOError - need to handle characters_written + let is_blocking_io_error = + zelf.class() + .is(vm.ctx.exceptions.blocking_io_error.as_ref()); + // SAFETY: slot_init is called during object initialization, // so fields are None and swap result can be safely ignored + let mut set_filename = true; if len <= 5 { - // Only set errno/strerror when args len is 2-5 (CPython behavior) + // Only set errno/strerror when args len is 2-5 if 2 <= len { let _ = unsafe { exc.errno.swap(Some(new_args.args[0].clone())) }; let _ = unsafe { exc.strerror.swap(Some(new_args.args[1].clone())) }; } if 3 <= len { - let _ = unsafe { exc.filename.swap(Some(new_args.args[2].clone())) }; + let third_arg = &new_args.args[2]; + // BlockingIOError's 3rd argument can be the number of characters written + if is_blocking_io_error + && !vm.is_none(third_arg) + && crate::protocol::PyNumber::check(third_arg) + && let Ok(written) = third_arg.try_index(vm) + && let Ok(n) = written.try_to_primitive::<isize>(vm) + { + exc.written.store(n); + set_filename = false; + // Clear filename that was set in py_new + let _ = unsafe { exc.filename.swap(None) }; + } + if set_filename { + let _ = unsafe { exc.filename.swap(Some(third_arg.clone())) }; + } } #[cfg(windows)] if 4 <= len { @@ -1594,53 +1938,112 @@ pub(super) mod types { } } - // args are truncated to 2 for compatibility (only when 2-5 args) - if (3..=5).contains(&len) { + // args are truncated to 2 for compatibility (only when 2-5 args and filename is not None) + // truncation happens inside "if (filename && filename != Py_None)" block + let has_filename = exc.filename.to_owned().filter(|f| !vm.is_none(f)).is_some(); + if (3..=5).contains(&len) && has_filename { new_args.args.truncate(2); } PyBaseException::slot_init(zelf, new_args, vm) } - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let args = exc.args(); - let obj = exc.as_object().to_owned(); + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } - let str = if args.len() == 2 { - // SAFETY: len() == 2 is checked so get_arg 1 or 2 won't panic - let errno = exc.get_arg(0).unwrap().str(vm)?; - let msg = exc.get_arg(1).unwrap().str(vm)?; + #[pyexception(with(Constructor, Initializer))] + impl PyOSError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let obj = zelf.as_object(); + + // Get OSError fields directly + let errno_field = obj.get_attr("errno", vm).ok().filter(|v| !vm.is_none(v)); + let strerror = obj.get_attr("strerror", vm).ok().filter(|v| !vm.is_none(v)); + let filename = obj.get_attr("filename", vm).ok().filter(|v| !vm.is_none(v)); + let filename2 = obj + .get_attr("filename2", vm) + .ok() + .filter(|v| !vm.is_none(v)); + #[cfg(windows)] + let winerror = obj.get_attr("winerror", vm).ok().filter(|v| !vm.is_none(v)); - // On Windows, use [WinError X] format when winerror is set - #[cfg(windows)] - let (label, code) = match obj.get_attr("winerror", vm) { - Ok(winerror) if !vm.is_none(&winerror) => ("WinError", winerror.str(vm)?), - _ => ("Errno", errno.clone()), - }; - #[cfg(not(windows))] - let (label, code) = ("Errno", errno.clone()); - - let s = match obj.get_attr("filename", vm) { - Ok(filename) if !vm.is_none(&filename) => match obj.get_attr("filename2", vm) { - Ok(filename2) if !vm.is_none(&filename2) => format!( - "[{} {}] {}: '{}' -> '{}'", - label, + // Windows: winerror takes priority over errno + #[cfg(windows)] + if let Some(ref win_err) = winerror { + let code = win_err.str(vm)?; + if let Some(ref f) = filename { + let msg = strerror + .as_ref() + .map(|s| s.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + if let Some(ref f2) = filename2 { + return Ok(vm.ctx.new_str(format!( + "[WinError {}] {}: {} -> {}", code, msg, - filename.str(vm)?, - filename2.str(vm)? - ), - _ => format!("[{} {}] {}: '{}'", label, code, msg, filename.str(vm)?), - }, - _ => { - format!("[{label} {code}] {msg}") + f.repr(vm)?, + f2.repr(vm)? + ))); } - }; - vm.ctx.new_str(s) - } else { - exc.__str__(vm) - }; - Ok(str) + return Ok(vm.ctx.new_str(format!( + "[WinError {}] {}: {}", + code, + msg, + f.repr(vm)? + ))); + } + // winerror && strerror (no filename) + if let Some(ref s) = strerror { + return Ok(vm + .ctx + .new_str(format!("[WinError {}] {}", code, s.str(vm)?))); + } + } + + // Non-Windows or fallback: use errno + if let Some(ref f) = filename { + let errno_str = errno_field + .as_ref() + .map(|e| e.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + let msg = strerror + .as_ref() + .map(|s| s.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + if let Some(ref f2) = filename2 { + return Ok(vm.ctx.new_str(format!( + "[Errno {}] {}: {} -> {}", + errno_str, + msg, + f.repr(vm)?, + f2.repr(vm)? + ))); + } + return Ok(vm.ctx.new_str(format!( + "[Errno {}] {}: {}", + errno_str, + msg, + f.repr(vm)? + ))); + } + + // errno && strerror (no filename) + if let (Some(e), Some(s)) = (&errno_field, &strerror) { + return Ok(vm + .ctx + .new_str(format!("[Errno {}] {}", e.str(vm)?, s.str(vm)?))); + } + + // fallback to BaseException.__str__ + zelf.__str__(vm) } #[pymethod] @@ -1691,8 +2094,8 @@ pub(super) mod types { self.errno.swap_to_temporary_refs(value, vm); } - #[pygetset(name = "strerror")] - fn get_strerror(&self) -> Option<PyObjectRef> { + #[pygetset] + fn strerror(&self) -> Option<PyObjectRef> { self.strerror.to_owned() } @@ -1732,6 +2135,47 @@ pub(super) mod types { fn set_winerror(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { self.winerror.swap_to_temporary_refs(value, vm); } + + #[pygetset] + fn characters_written(&self, vm: &VirtualMachine) -> PyResult<isize> { + let written = self.written.load(); + if written == -1 { + Err(vm.new_attribute_error("characters_written".to_owned())) + } else { + Ok(written) + } + } + + #[pygetset(setter)] + fn set_characters_written( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + None => { + // Deleting the attribute + if self.written.load() == -1 { + Err(vm.new_attribute_error("characters_written".to_owned())) + } else { + self.written.store(-1); + Ok(()) + } + } + Some(v) => { + let n = v + .try_index(vm)? + .try_to_primitive::<isize>(vm) + .map_err(|_| { + vm.new_value_error( + "cannot convert characters_written value to isize".to_owned(), + ) + })?; + self.written.store(n); + Ok(()) + } + } + } } #[pyexception(name, base = PyOSError, ctx = "blocking_io_error", impl)] @@ -1839,15 +2283,68 @@ pub(super) mod types { #[repr(transparent)] pub struct PyRecursionError(PyRuntimeError); + #[pyexception(name, base = PyRuntimeError, ctx = "python_finalization_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyPythonFinalizationError(PyRuntimeError); + #[pyexception(name, base = PyException, ctx = "syntax_error")] #[derive(Debug)] #[repr(transparent)] pub struct PySyntaxError(PyException); - #[pyexception] + #[pyexception(with(Initializer))] impl PySyntaxError { - #[pyslot] - #[pymethod(name = "__init__")] + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + fn basename(filename: &str) -> &str { + let splitted = if cfg!(windows) { + filename.rsplit(&['/', '\\']).next() + } else { + filename.rsplit('/').next() + }; + splitted.unwrap_or(filename) + } + + let maybe_lineno = zelf.as_object().get_attr("lineno", vm).ok().map(|obj| { + obj.str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<lineno str() failed>")) + }); + let maybe_filename = zelf.as_object().get_attr("filename", vm).ok().map(|obj| { + obj.str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<filename str() failed>")) + }); + + let msg = match zelf.as_object().get_attr("msg", vm) { + Ok(obj) => obj + .str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<msg str() failed>")), + Err(_) => { + // Fallback to the base formatting if the msg attribute was deleted or attribute lookup fails for any reason. + return Py::<PyBaseException>::__str__(zelf, vm); + } + }; + + let msg_with_location_info: String = match (maybe_lineno, maybe_filename) { + (Some(lineno), Some(filename)) => { + format!("{} ({}, line {})", msg, basename(filename.as_str()), lineno) + } + (Some(lineno), None) => { + format!("{msg} (line {lineno})") + } + (None, Some(filename)) => { + format!("{} ({})", msg, basename(filename.as_str())) + } + (None, None) => msg.to_string(), + }; + + Ok(vm.ctx.new_str(msg_with_location_info)) + } + } + + impl Initializer for PySyntaxError { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let len = args.args.len(); let new_args = args; @@ -1882,77 +2379,22 @@ pub(super) mod types { PyBaseException::slot_init(zelf, new_args, vm) } - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyStrRef { - fn basename(filename: &str) -> &str { - let splitted = if cfg!(windows) { - filename.rsplit(&['/', '\\']).next() - } else { - filename.rsplit('/').next() - }; - splitted.unwrap_or(filename) - } - - let maybe_lineno = exc.as_object().get_attr("lineno", vm).ok().map(|obj| { - obj.str(vm) - .unwrap_or_else(|_| vm.ctx.new_str("<lineno str() failed>")) - }); - let maybe_filename = exc.as_object().get_attr("filename", vm).ok().map(|obj| { - obj.str(vm) - .unwrap_or_else(|_| vm.ctx.new_str("<filename str() failed>")) - }); - - let args = exc.args(); - - let msg = if args.len() == 1 { - vm.exception_args_as_string(args, false) - .into_iter() - .exactly_one() - .unwrap() - } else { - return exc.__str__(vm); - }; - - let msg_with_location_info: String = match (maybe_lineno, maybe_filename) { - (Some(lineno), Some(filename)) => { - format!("{} ({}, line {})", msg, basename(filename.as_str()), lineno) - } - (Some(lineno), None) => { - format!("{msg} (line {lineno})") - } - (None, Some(filename)) => { - format!("{} ({})", msg, basename(filename.as_str())) - } - (None, None) => msg.to_string(), - }; - - vm.ctx.new_str(msg_with_location_info) + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") } } + // MiddlingExtendsException: inherits __init__ from SyntaxError via MRO #[pyexception( name = "_IncompleteInputError", base = PySyntaxError, - ctx = "incomplete_input_error" + ctx = "incomplete_input_error", + impl )] #[derive(Debug)] #[repr(transparent)] pub struct PyIncompleteInputError(PySyntaxError); - #[pyexception] - impl PyIncompleteInputError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - _args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?; - Ok(()) - } - } - #[pyexception(name, base = PySyntaxError, ctx = "indentation_error", impl)] #[derive(Debug)] #[repr(transparent)] @@ -1988,102 +2430,112 @@ pub(super) mod types { #[repr(transparent)] pub struct PyUnicodeDecodeError(PyUnicodeError); - #[pyexception] + #[pyexception(with(Initializer))] impl PyUnicodeDecodeError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - type Args = (PyStrRef, ArgBytesLike, isize, isize, PyStrRef); - let (encoding, object, start, end, reason): Args = args.bind(vm)?; - zelf.set_attr("encoding", encoding, vm)?; - let object_as_bytes = vm.ctx.new_bytes(object.borrow_buf().to_vec()); - zelf.set_attr("object", object_as_bytes, vm)?; - zelf.set_attr("start", vm.ctx.new_int(start), vm)?; - zelf.set_attr("end", vm.ctx.new_int(end), vm)?; - zelf.set_attr("reason", reason, vm)?; - Ok(()) - } - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<String> { - let Ok(object) = exc.as_object().get_attr("object", vm) else { - return Ok("".to_owned()); + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); }; let object: ArgBytesLike = object.try_into_value(vm)?; - let encoding: PyStrRef = exc + let encoding: PyStrRef = zelf .as_object() .get_attr("encoding", vm)? .try_into_value(vm)?; - let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; - if start < object.len() && end <= object.len() && end == start + 1 { + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str(if start < object.len() && end <= object.len() && end == start + 1 { let b = object.borrow_buf()[start]; - Ok(format!( + format!( "'{encoding}' codec can't decode byte {b:#02x} in position {start}: {reason}" - )) + ) } else { - Ok(format!( + format!( "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", end - 1, - )) - } + ) + })) } } - #[pyexception(name, base = PyUnicodeError, ctx = "unicode_encode_error")] - #[derive(Debug)] - #[repr(transparent)] - pub struct PyUnicodeEncodeError(PyUnicodeError); + impl Initializer for PyUnicodeDecodeError { + type Args = FuncArgs; - #[pyexception] - impl PyUnicodeEncodeError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - type Args = (PyStrRef, PyStrRef, isize, isize, PyStrRef); + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + type Args = (PyStrRef, ArgBytesLike, isize, isize, PyStrRef); let (encoding, object, start, end, reason): Args = args.bind(vm)?; zelf.set_attr("encoding", encoding, vm)?; - zelf.set_attr("object", object, vm)?; + let object_as_bytes = vm.ctx.new_bytes(object.borrow_buf().to_vec()); + zelf.set_attr("object", object_as_bytes, vm)?; zelf.set_attr("start", vm.ctx.new_int(start), vm)?; zelf.set_attr("end", vm.ctx.new_int(end), vm)?; zelf.set_attr("reason", reason, vm)?; Ok(()) } + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyUnicodeError, ctx = "unicode_encode_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeEncodeError(PyUnicodeError); + + #[pyexception(with(Initializer))] + impl PyUnicodeEncodeError { #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<String> { - let Ok(object) = exc.as_object().get_attr("object", vm) else { - return Ok("".to_owned()); + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); }; let object: PyStrRef = object.try_into_value(vm)?; - let encoding: PyStrRef = exc + let encoding: PyStrRef = zelf .as_object() .get_attr("encoding", vm)? .try_into_value(vm)?; - let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; - if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str(if start < object.char_len() && end <= object.char_len() && end == start + 1 { let ch = object.as_wtf8().code_points().nth(start).unwrap(); - Ok(format!( + format!( "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", UnicodeEscapeCodepoint(ch) - )) + ) } else { - Ok(format!( + format!( "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", end - 1, - )) - } + ) + })) + } + } + + impl Initializer for PyUnicodeEncodeError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + type Args = (PyStrRef, PyStrRef, isize, isize, PyStrRef); + let (encoding, object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("encoding", encoding, vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") } } @@ -2092,15 +2544,41 @@ pub(super) mod types { #[repr(transparent)] pub struct PyUnicodeTranslateError(PyUnicodeError); - #[pyexception] + #[pyexception(with(Initializer))] impl PyUnicodeTranslateError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); + }; + let object: PyStrRef = object.try_into_value(vm)?; + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str( + if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let ch = object.as_wtf8().code_points().nth(start).unwrap(); + format!( + "can't translate character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + ) + } else { + format!( + "can't translate characters in position {start}-{}: {reason}", + end - 1, + ) + }, + )) + } + } + + impl Initializer for PyUnicodeTranslateError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { type Args = (PyStrRef, isize, isize, PyStrRef); let (object, start, end, reason): Args = args.bind(vm)?; zelf.set_attr("object", object, vm)?; @@ -2110,27 +2588,8 @@ pub(super) mod types { Ok(()) } - #[pymethod] - fn __str__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<String> { - let Ok(object) = exc.as_object().get_attr("object", vm) else { - return Ok("".to_owned()); - }; - let object: PyStrRef = object.try_into_value(vm)?; - let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; - if start < object.char_len() && end <= object.char_len() && end == start + 1 { - let ch = object.as_wtf8().code_points().nth(start).unwrap(); - Ok(format!( - "can't translate character '{}' in position {start}: {reason}", - UnicodeEscapeCodepoint(ch) - )) - } else { - Ok(format!( - "can't translate characters in position {start}-{}: {reason}", - end - 1, - )) - } + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") } } @@ -2202,3 +2661,281 @@ pub(super) mod types { #[repr(transparent)] pub struct PyEncodingWarning(PyWarning); } + +/// Check if match_type is valid for except* (must be exception type, not ExceptionGroup). +fn check_except_star_type_valid(match_type: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let base_exc: PyObjectRef = vm.ctx.exceptions.base_exception_type.to_owned().into(); + let base_eg: PyObjectRef = vm.ctx.exceptions.base_exception_group.to_owned().into(); + + // Helper to check a single type + let check_one = |exc_type: &PyObjectRef| -> PyResult<()> { + // Must be a subclass of BaseException + if !exc_type.is_subclass(&base_exc, vm)? { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed".to_owned(), + )); + } + // Must not be a subclass of BaseExceptionGroup + if exc_type.is_subclass(&base_eg, vm)? { + return Err(vm.new_type_error( + "catching ExceptionGroup with except* is not allowed. Use except instead." + .to_owned(), + )); + } + Ok(()) + }; + + // If it's a tuple, check each element + if let Ok(tuple) = match_type.clone().downcast::<PyTuple>() { + for item in tuple.iter() { + check_one(item)?; + } + } else { + check_one(match_type)?; + } + Ok(()) +} + +/// Match exception against except* handler type. +/// Returns (rest, match) tuple. +pub fn exception_group_match( + exc_value: &PyObjectRef, + match_type: &PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, PyObjectRef)> { + // Implements _PyEval_ExceptionGroupMatch + + // If exc_value is None, return (None, None) + if vm.is_none(exc_value) { + return Ok((vm.ctx.none(), vm.ctx.none())); + } + + // Validate match_type and reject ExceptionGroup/BaseExceptionGroup + check_except_star_type_valid(match_type, vm)?; + + // Check if exc_value matches match_type + if exc_value.is_instance(match_type, vm)? { + // Full match of exc itself + let is_eg = exc_value.fast_isinstance(vm.ctx.exceptions.base_exception_group); + let matched = if is_eg { + exc_value.clone() + } else { + // Naked exception - wrap it in ExceptionGroup + let excs = vm.ctx.new_tuple(vec![exc_value.clone()]); + let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + let wrapped = eg_type.call((vm.ctx.new_str(""), excs), vm)?; + // Copy traceback from original exception + if let Ok(exc) = exc_value.clone().downcast::<types::PyBaseException>() + && let Some(tb) = exc.__traceback__() + && let Ok(wrapped_exc) = wrapped.clone().downcast::<types::PyBaseException>() + { + let _ = wrapped_exc.set___traceback__(tb.into(), vm); + } + wrapped + }; + return Ok((vm.ctx.none(), matched)); + } + + // Check for partial match if it's an exception group + if exc_value.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + let pair = vm.call_method(exc_value, "split", (match_type.clone(),))?; + if !pair.class().is(vm.ctx.types.tuple_type) { + return Err(vm.new_type_error(format!( + "{}.split must return a tuple, not {}", + exc_value.class().name(), + pair.class().name() + ))); + } + let pair_tuple: PyTupleRef = pair.try_into_value(vm)?; + if pair_tuple.len() < 2 { + return Err(vm.new_type_error(format!( + "{}.split must return a 2-tuple, got tuple of size {}", + exc_value.class().name(), + pair_tuple.len() + ))); + } + let matched = pair_tuple[0].clone(); + let rest = pair_tuple[1].clone(); + return Ok((rest, matched)); + } + + // No match + Ok((exc_value.clone(), vm.ctx.none())) +} + +/// Prepare exception for reraise in except* block. +/// Implements _PyExc_PrepReraiseStar +pub fn prep_reraise_star(orig: PyObjectRef, excs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyList; + + let excs_list = excs + .downcast::<PyList>() + .map_err(|_| vm.new_type_error("expected list for prep_reraise_star"))?; + + let excs_vec: Vec<PyObjectRef> = excs_list.borrow_vec().to_vec(); + + // If no exceptions to process, return None + if excs_vec.is_empty() { + return Ok(vm.ctx.none()); + } + + // Special case: naked exception (not an ExceptionGroup) + // Only one except* clause could have executed, so there's at most one exception to raise + if !orig.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + // Find first non-None exception + let first = excs_vec.into_iter().find(|e| !vm.is_none(e)); + return Ok(first.unwrap_or_else(|| vm.ctx.none())); + } + + // Split excs into raised (new) and reraised (from original) by comparing metadata + let mut raised: Vec<PyObjectRef> = Vec::new(); + let mut reraised: Vec<PyObjectRef> = Vec::new(); + + for exc in excs_vec { + if vm.is_none(&exc) { + continue; + } + // Check if this exception came from the original group + if is_exception_from_orig(&exc, &orig, vm) { + reraised.push(exc); + } else { + raised.push(exc); + } + } + + // If no exceptions to reraise, return None + if raised.is_empty() && reraised.is_empty() { + return Ok(vm.ctx.none()); + } + + // Project reraised exceptions onto original structure to preserve nesting + let reraised_eg = exception_group_projection(&orig, &reraised, vm)?; + + // If no new raised exceptions, just return the reraised projection + if raised.is_empty() { + return Ok(reraised_eg); + } + + // Combine raised with reraised_eg + if !vm.is_none(&reraised_eg) { + raised.push(reraised_eg); + } + + // If only one exception, return it directly + if raised.len() == 1 { + return Ok(raised.into_iter().next().unwrap()); + } + + // Create new ExceptionGroup for multiple exceptions + let excs_tuple = vm.ctx.new_tuple(raised); + let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + eg_type.call((vm.ctx.new_str(""), excs_tuple), vm) +} + +/// Check if an exception came from the original group (for reraise detection). +/// Instead of comparing metadata (which can be modified when caught), we compare +/// leaf exception object IDs. split() preserves leaf exception identity. +fn is_exception_from_orig(exc: &PyObjectRef, orig: &PyObjectRef, vm: &VirtualMachine) -> bool { + // Collect leaf exception IDs from exc + let mut exc_leaf_ids = HashSet::new(); + collect_exception_group_leaf_ids(exc, &mut exc_leaf_ids, vm); + + if exc_leaf_ids.is_empty() { + return false; + } + + // Collect leaf exception IDs from orig + let mut orig_leaf_ids = HashSet::new(); + collect_exception_group_leaf_ids(orig, &mut orig_leaf_ids, vm); + + // If ALL of exc's leaves are in orig's leaves, it's a reraise + exc_leaf_ids.iter().all(|id| orig_leaf_ids.contains(id)) +} + +/// Collect all leaf exception IDs from an exception (group). +fn collect_exception_group_leaf_ids( + exc: &PyObjectRef, + leaf_ids: &mut HashSet<usize>, + vm: &VirtualMachine, +) { + if vm.is_none(exc) { + return; + } + + // If not an exception group, it's a leaf - add its ID + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + leaf_ids.insert(exc.get_id()); + return; + } + + // Recurse into exception group's exceptions + if let Ok(excs_attr) = exc.get_attr("exceptions", vm) + && let Ok(tuple) = excs_attr.downcast::<PyTuple>() + { + for e in tuple.iter() { + collect_exception_group_leaf_ids(e, leaf_ids, vm); + } + } +} + +/// Project orig onto keep list, preserving nested structure. +/// Returns an exception group containing only the exceptions from orig +/// that are also in the keep list. +fn exception_group_projection( + orig: &PyObjectRef, + keep: &[PyObjectRef], + vm: &VirtualMachine, +) -> PyResult { + if keep.is_empty() { + return Ok(vm.ctx.none()); + } + + // Collect all leaf IDs from keep list + let mut leaf_ids = HashSet::new(); + for e in keep { + collect_exception_group_leaf_ids(e, &mut leaf_ids, vm); + } + + // Split orig by matching leaf IDs, preserving structure + split_by_leaf_ids(orig, &leaf_ids, vm) +} + +/// Recursively split an exception (group) by leaf IDs. +/// Returns the projection containing only matching leaves with preserved structure. +fn split_by_leaf_ids( + exc: &PyObjectRef, + leaf_ids: &HashSet<usize>, + vm: &VirtualMachine, +) -> PyResult { + if vm.is_none(exc) { + return Ok(vm.ctx.none()); + } + + // If not an exception group, check if it's in our set + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + if leaf_ids.contains(&exc.get_id()) { + return Ok(exc.clone()); + } + return Ok(vm.ctx.none()); + } + + // Exception group - recurse and reconstruct + let excs_attr = exc.get_attr("exceptions", vm)?; + let tuple: PyTupleRef = excs_attr.try_into_value(vm)?; + + let mut matched = Vec::new(); + for e in tuple.iter() { + let m = split_by_leaf_ids(e, leaf_ids, vm)?; + if !vm.is_none(&m) { + matched.push(m); + } + } + + if matched.is_empty() { + return Ok(vm.ctx.none()); + } + + // Reconstruct using derive() to preserve the structure (not necessarily the subclass type) + let matched_tuple = vm.ctx.new_tuple(matched); + vm.call_method(exc, "derive", (matched_tuple,)) +} diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 4a460a95884..62df1b298e6 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1,58 +1,37 @@ +#[cfg(feature = "flame")] +use crate::bytecode::InstructionMetadata; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator, PyList, PySet, - PySlice, PyStr, PyStrInterned, PyStrRef, PyTraceback, PyType, + PyBaseException, PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator, + PyInterpolation, PyList, PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, + PyType, asyncgenerator::PyAsyncGenWrappedValue, function::{PyCell, PyCellRef, PyFunction}, tuple::{PyTuple, PyTupleRef}, }, - bytecode, + bytecode::{self, Instruction, LoadAttr, LoadSuperAttr, SpecialMethod}, convert::{IntoObject, ToPyResult}, coroutine::Coro, exceptions::ExceptionCtor, function::{ArgMapping, Either, FuncArgs}, + object::PyAtomicBorrow, + object::{Traverse, TraverseFn}, protocol::{PyIter, PyIterReturn}, scope::Scope, stdlib::{builtins, typing}, types::PyTypeFlags, vm::{Context, PyMethod}, }; +use alloc::fmt; +use core::iter::zip; +use core::sync::atomic; +use core::sync::atomic::AtomicPtr; use indexmap::IndexMap; use itertools::Itertools; + use rustpython_common::{boxvec::BoxVec, lock::PyMutex, wtf8::Wtf8Buf}; use rustpython_compiler_core::SourceLocation; -#[cfg(feature = "threading")] -use std::sync::atomic; -use std::{fmt, iter::zip}; - -#[derive(Clone, Debug)] -struct Block { - /// The type of block. - typ: BlockType, - /// The level of the value stack when the block was entered. - level: usize, -} - -#[derive(Clone, Debug)] -enum BlockType { - Loop, - TryExcept { - handler: bytecode::Label, - }, - Finally { - handler: bytecode::Label, - }, - - /// Active finally sequence - FinallyHandler { - reason: Option<UnwindReason>, - prev_exc: Option<PyBaseExceptionRef>, - }, - ExceptHandler { - prev_exc: Option<PyBaseExceptionRef>, - }, -} pub type FrameRef = PyRef<Frame>; @@ -67,33 +46,48 @@ enum UnwindReason { /// We hit an exception, so unwind any try-except and finally blocks. The exception should be /// on top of the vm exception stack. Raising { exception: PyBaseExceptionRef }, - - // NoWorries, - /// We are unwinding blocks, since we hit break - Break { target: bytecode::Label }, - - /// We are unwinding blocks since we hit a continue statements. - Continue { target: bytecode::Label }, } #[derive(Debug)] struct FrameState { // We need 1 stack per frame /// The main data frame of the stack machine - stack: BoxVec<PyObjectRef>, - /// Block frames, for controlling loops and exceptions - blocks: Vec<Block>, + stack: BoxVec<Option<PyObjectRef>>, /// index of last instruction ran #[cfg(feature = "threading")] lasti: u32, } +/// Tracks who owns a frame. +// = `_PyFrameOwner` +#[repr(i8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FrameOwner { + /// Being executed by a thread (FRAME_OWNED_BY_THREAD). + Thread = 0, + /// Owned by a generator/coroutine (FRAME_OWNED_BY_GENERATOR). + Generator = 1, + /// Not executing; held only by a frame object or traceback + /// (FRAME_OWNED_BY_FRAME_OBJECT). + FrameObject = 2, +} + +impl FrameOwner { + pub(crate) fn from_i8(v: i8) -> Self { + match v { + 0 => Self::Thread, + 1 => Self::Generator, + _ => Self::FrameObject, + } + } +} + #[cfg(feature = "threading")] type Lasti = atomic::AtomicU32; #[cfg(not(feature = "threading"))] -type Lasti = std::cell::Cell<u32>; +type Lasti = core::cell::Cell<u32>; -#[pyclass(module = false, name = "frame")] +#[pyclass(module = false, name = "frame", traverse = "manual")] pub struct Frame { pub code: PyRef<PyCode>, pub func_obj: Option<PyObjectRef>, @@ -102,7 +96,7 @@ pub struct Frame { pub(crate) cells_frees: Box<[PyCellRef]>, pub locals: ArgMapping, pub globals: PyDictRef, - pub builtins: PyDictRef, + pub builtins: PyObjectRef, // on feature=threading, this is a duplicate of FrameState.lasti, but it's faster to do an // atomic store than it is to do a fetch_add, for every instruction executed @@ -114,7 +108,21 @@ pub struct Frame { // member pub trace_lines: PyMutex<bool>, + pub trace_opcodes: PyMutex<bool>, pub temporary_refs: PyMutex<Vec<PyObjectRef>>, + /// Back-reference to owning generator/coroutine/async generator. + /// Borrowed reference (not ref-counted) to avoid Generator↔Frame cycle. + /// Cleared by the generator's Drop impl. + pub generator: PyAtomicBorrow, + /// Previous frame in the call chain for signal-safe traceback walking. + /// Mirrors `_PyInterpreterFrame.previous`. + pub(crate) previous: AtomicPtr<Frame>, + /// Who owns this frame. Mirrors `_PyInterpreterFrame.owner`. + /// Used by `frame.clear()` to reject clearing an executing frame, + /// even when called from a different thread. + pub(crate) owner: atomic::AtomicI8, + /// Set when f_locals is accessed. Cleared after locals_to_fast() sync. + pub(crate) locals_dirty: atomic::AtomicBool, } impl PyPayload for Frame { @@ -124,6 +132,28 @@ impl PyPayload for Frame { } } +unsafe impl Traverse for FrameState { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.stack.traverse(tracer_fn); + } +} + +unsafe impl Traverse for Frame { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.code.traverse(tracer_fn); + self.func_obj.traverse(tracer_fn); + self.fastlocals.traverse(tracer_fn); + self.cells_frees.traverse(tracer_fn); + self.locals.traverse(tracer_fn); + self.globals.traverse(tracer_fn); + self.builtins.traverse(tracer_fn); + self.trace.traverse(tracer_fn); + self.state.traverse(tracer_fn); + self.temporary_refs.traverse(tracer_fn); + // generator is a borrowed reference, not traversed + } +} + // Running a frame can result in one of the below: pub enum ExecutionResult { Return(PyObjectRef), @@ -137,25 +167,38 @@ impl Frame { pub(crate) fn new( code: PyRef<PyCode>, scope: Scope, - builtins: PyDictRef, + builtins: PyObjectRef, closure: &[PyCellRef], func_obj: Option<PyObjectRef>, vm: &VirtualMachine, ) -> Self { - let cells_frees = std::iter::repeat_with(|| PyCell::default().into_ref(&vm.ctx)) - .take(code.cellvars.len()) - .chain(closure.iter().cloned()) - .collect(); + let nlocals = code.varnames.len(); + let num_cells = code.cellvars.len(); + let nfrees = closure.len(); + + let cells_frees: Box<[PyCellRef]> = + core::iter::repeat_with(|| PyCell::default().into_ref(&vm.ctx)) + .take(num_cells) + .chain(closure.iter().cloned()) + .collect(); + + // Extend fastlocals to include varnames + cellvars + freevars (localsplus) + let total_locals = nlocals + num_cells + nfrees; + let mut fastlocals_vec: Vec<Option<PyObjectRef>> = vec![None; total_locals]; + + // Store cell objects at cellvars and freevars positions + for (i, cell) in cells_frees.iter().enumerate() { + fastlocals_vec[nlocals + i] = Some(cell.clone().into()); + } let state = FrameState { stack: BoxVec::new(code.max_stackdepth as usize), - blocks: Vec::new(), #[cfg(feature = "threading")] lasti: 0, }; Self { - fastlocals: PyMutex::new(vec![None; code.varnames.len()].into_boxed_slice()), + fastlocals: PyMutex::new(fastlocals_vec.into_boxed_slice()), cells_frees, locals: scope.locals, globals: scope.globals, @@ -166,12 +209,42 @@ impl Frame { state: PyMutex::new(state), trace: PyMutex::new(vm.ctx.none()), trace_lines: PyMutex::new(true), + trace_opcodes: PyMutex::new(false), temporary_refs: PyMutex::new(vec![]), + generator: PyAtomicBorrow::new(), + previous: AtomicPtr::new(core::ptr::null_mut()), + owner: atomic::AtomicI8::new(FrameOwner::FrameObject as i8), + locals_dirty: atomic::AtomicBool::new(false), } } + /// Clear the evaluation stack. Used by frame.clear() to break reference cycles. + pub(crate) fn clear_value_stack(&self) { + self.state.lock().stack.clear(); + } + + /// Store a borrowed back-reference to the owning generator/coroutine. + /// The caller must ensure the generator outlives the frame. + pub fn set_generator(&self, generator: &PyObject) { + self.generator.store(generator); + self.owner + .store(FrameOwner::Generator as i8, atomic::Ordering::Release); + } + + /// Clear the generator back-reference. Called when the generator is finalized. + pub fn clear_generator(&self) { + self.generator.clear(); + self.owner + .store(FrameOwner::FrameObject as i8, atomic::Ordering::Release); + } + pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1] + self.code.locations[self.lasti() as usize - 1].0 + } + + /// Get the previous frame pointer for signal-safe traceback walking. + pub fn previous_frame(&self) -> *const Frame { + self.previous.load(atomic::Ordering::Relaxed) } pub fn lasti(&self) -> u32 { @@ -185,11 +258,33 @@ impl Frame { } } + /// Sync locals dict back to fastlocals. Called before generator/coroutine resume + /// to apply any modifications made via f_locals. + pub fn locals_to_fast(&self, vm: &VirtualMachine) -> PyResult<()> { + if !self.locals_dirty.load(atomic::Ordering::Acquire) { + return Ok(()); + } + let code = &**self.code; + let mut fastlocals = self.fastlocals.lock(); + for (i, &varname) in code.varnames.iter().enumerate() { + if i >= fastlocals.len() { + break; + } + match self.locals.mapping().subscript(varname, vm) { + Ok(value) => fastlocals[i] = Some(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} + Err(e) => return Err(e), + } + } + self.locals_dirty.store(false, atomic::Ordering::Release); + Ok(()) + } + pub fn locals(&self, vm: &VirtualMachine) -> PyResult<ArgMapping> { let locals = &self.locals; let code = &**self.code; let map = &code.varnames; - let j = std::cmp::min(map.len(), code.varnames.len()); + let j = core::cmp::min(map.len(), code.varnames.len()); if !code.varnames.is_empty() { let fastlocals = self.fastlocals.lock(); for (&k, v) in zip(&map[..j], &**fastlocals) { @@ -216,7 +311,7 @@ impl Frame { Ok(()) }; map_to_dict(&code.cellvars, &self.cells_frees)?; - if code.flags.contains(bytecode::CodeFlags::IS_OPTIMIZED) { + if code.flags.contains(bytecode::CodeFlags::OPTIMIZED) { map_to_dict(&code.freevars, &self.cells_frees[code.cellvars.len()..])?; } } @@ -271,7 +366,21 @@ impl Py<Frame> { } pub fn yield_from_target(&self) -> Option<PyObjectRef> { - self.with_exec(|exec| exec.yield_from_target().map(PyObject::to_owned)) + // Use try_lock to avoid deadlock when the frame is currently executing. + // A running coroutine has no yield-from target. + let mut state = self.state.try_lock()?; + let exec = ExecutingFrame { + code: &self.code, + fastlocals: &self.fastlocals, + cells_frees: &self.cells_frees, + locals: &self.locals, + globals: &self.globals, + builtins: &self.builtins, + lasti: &self.lasti, + object: self, + state: &mut state, + }; + exec.yield_from_target().map(PyObject::to_owned) } pub fn is_internal_frame(&self) -> bool { @@ -282,19 +391,14 @@ impl Py<Frame> { } pub fn next_external_frame(&self, vm: &VirtualMachine) -> Option<FrameRef> { - self.f_back(vm).map(|mut back| { - loop { - back = if let Some(back) = back.to_owned().f_back(vm) { - back - } else { - break back; - }; - - if !back.is_internal_frame() { - break back; - } + let mut frame = self.f_back(vm); + while let Some(ref f) = frame { + if !f.is_internal_frame() { + break; } - }) + frame = f.f_back(vm); + } + frame } } @@ -306,7 +410,7 @@ struct ExecutingFrame<'a> { cells_frees: &'a [PyCellRef], locals: &'a ArgMapping, globals: &'a PyDictRef, - builtins: &'a PyDictRef, + builtins: &'a PyObjectRef, object: &'a Py<Frame>, lasti: &'a Lasti, state: &'a mut FrameState, @@ -359,12 +463,20 @@ impl ExecutingFrame<'_> { // Execute until return or exception: let instructions = &self.code.instructions; let mut arg_state = bytecode::OpArgState::default(); + let mut prev_line: u32 = 0; loop { let idx = self.lasti() as usize; - // eprintln!( - // "location: {:?} {}", - // self.code.locations[idx], self.code.source_path - // ); + // Fire 'line' trace event when line number changes. + // Only fire if this frame has a per-frame trace function set + // (frames entered before sys.settrace() have trace=None). + if vm.use_tracing.get() + && !vm.is_none(&self.object.trace.lock()) + && let Some((loc, _)) = self.code.locations.get(idx) + && loc.line.get() as u32 != prev_line + { + prev_line = loc.line.get() as u32; + vm.trace_event(crate::protocol::TraceEvent::Line, None)?; + } self.update_lasti(|i| *i += 1); let bytecode::CodeUnit { op, arg } = instructions[idx]; let arg = arg_state.extend(arg); @@ -382,34 +494,95 @@ impl ExecutingFrame<'_> { frame: &mut ExecutingFrame<'_>, exception: PyBaseExceptionRef, idx: usize, + is_reraise: bool, + is_new_raise: bool, vm: &VirtualMachine, ) -> FrameResult { // 1. Extract traceback from exception's '__traceback__' attr. // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. - // 3. Unwind block stack till appropriate handler is found. - - let loc = frame.code.locations[idx]; - let next = exception.__traceback__(); - let new_traceback = PyTraceback::new( - next, - frame.object.to_owned(), - frame.lasti(), - loc.line, - ); - vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); - - vm.contextualize_exception(&exception); + // 3. First, try to find handler in exception table + + // RERAISE instructions should not add traceback entries - they're just + // re-raising an already-processed exception + if !is_reraise { + // Check if the exception already has traceback entries before + // we add ours. If it does, it was propagated from a callee + // function and we should not re-contextualize it. + let had_prior_traceback = exception.__traceback__().is_some(); + + // PyTraceBack_Here always adds a new entry without + // checking for duplicates. Each time an exception passes through + // a frame (e.g., in a loop with repeated raise statements), + // a new traceback entry is added. + let (loc, _end_loc) = frame.code.locations[idx]; + let next = exception.__traceback__(); + + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + + // _PyErr_SetObject sets __context__ only when the exception + // is first raised. When an exception propagates through frames, + // __context__ must not be overwritten. We contextualize when: + // - It's an explicit raise (raise/raise from) + // - The exception had no prior traceback (originated here) + if is_new_raise || !had_prior_traceback { + vm.contextualize_exception(&exception); + } + } + // Use exception table for zero-cost exception handling frame.unwind_blocks(vm, UnwindReason::Raising { exception }) } - match handle_exception(self, exception, idx, vm) { + // Check if this is a RERAISE instruction + // Both AnyInstruction::Raise { kind: Reraise/ReraiseFromStack } and + // AnyInstruction::Reraise are reraise operations that should not add + // new traceback entries. + // EndAsyncFor and CleanupThrow also re-raise non-matching exceptions. + let is_reraise = match op { + Instruction::RaiseVarargs { kind } => matches!( + kind.get(arg), + bytecode::RaiseKind::BareRaise | bytecode::RaiseKind::ReraiseFromStack + ), + Instruction::Reraise { .. } + | Instruction::EndAsyncFor + | Instruction::CleanupThrow => true, + _ => false, + }; + + // Explicit raise instructions (raise/raise from) - these always + // need contextualization even if the exception has prior traceback + let is_new_raise = matches!( + op, + Instruction::RaiseVarargs { kind } + if matches!( + kind.get(arg), + bytecode::RaiseKind::Raise | bytecode::RaiseKind::RaiseCause + ) + ); + + match handle_exception(self, exception, idx, is_reraise, is_new_raise, vm) { Ok(None) => {} Ok(Some(result)) => break Ok(result), - // TODO: append line number to traceback? - // traceback.append(); - Err(exception) => break Err(exception), + Err(exception) => { + // Restore lasti from traceback so frame.f_lineno matches tb_lineno + // The traceback was created with the correct lasti when exception + // was first raised, but frame.lasti may have changed during cleanup + if let Some(tb) = exception.__traceback__() + && core::ptr::eq::<Py<Frame>>(&*tb.frame, self.object) + { + // This traceback entry is for this frame - restore its lasti + // tb.lasti is in bytes (idx * 2), convert back to instruction index + self.update_lasti(|i| *i = tb.lasti / 2); + } + break Err(exception); + } } } } @@ -420,19 +593,43 @@ impl ExecutingFrame<'_> { } fn yield_from_target(&self) -> Option<&PyObject> { - if let Some(bytecode::CodeUnit { - op: bytecode::Instruction::YieldFrom, - .. - }) = self.code.instructions.get(self.lasti() as usize) - { - Some(self.top_value()) - } else { - None + // checks gi_frame_state == FRAME_SUSPENDED_YIELD_FROM + // which is set when YIELD_VALUE with oparg >= 1 is executed. + // In RustPython, we check: + // 1. lasti points to RESUME (after YIELD_VALUE) + // 2. The previous instruction was YIELD_VALUE with arg >= 1 + // 3. Stack top is the delegate (receiver) + // + // First check if stack is empty - if so, we can't be in yield-from + if self.state.stack.is_empty() { + return None; + } + let lasti = self.lasti() as usize; + if let Some(unit) = self.code.instructions.get(lasti) { + match &unit.op { + Instruction::Send { .. } => return Some(self.top_value()), + Instruction::Resume { .. } => { + // Check if previous instruction was YIELD_VALUE with arg >= 1 + // This indicates yield-from/await context + if lasti > 0 + && let Some(prev_unit) = self.code.instructions.get(lasti - 1) + && let Instruction::YieldValue { .. } = &prev_unit.op + { + // YIELD_VALUE arg: 0 = direct yield, >= 1 = yield-from/await + // OpArgByte.0 is the raw byte value + if u8::from(prev_unit.arg) >= 1 { + // In yield-from/await context, delegate is on top of stack + return Some(self.top_value()); + } + } + } + _ => {} + } } + None } - /// Ok(Err(e)) means that an error occurred while calling throw() and the generator should try - /// sending it + /// Handle throw() on a generator/coroutine. fn gen_throw( &mut self, vm: &VirtualMachine, @@ -441,36 +638,123 @@ impl ExecutingFrame<'_> { exc_tb: PyObjectRef, ) -> PyResult<ExecutionResult> { if let Some(jen) = self.yield_from_target() { - // borrow checker shenanigans - we only need to use exc_type/val/tb if the following - // variable is Some - let thrower = if let Some(coro) = self.builtin_coro(jen) { - Some(Either::A(coro)) + // Check if the exception is GeneratorExit (type or instance). + // For GeneratorExit, close the sub-iterator instead of throwing. + let is_gen_exit = if let Some(typ) = exc_type.downcast_ref::<PyType>() { + typ.fast_issubclass(vm.ctx.exceptions.generator_exit) } else { - vm.get_attribute_opt(jen.to_owned(), "throw")? - .map(Either::B) + exc_type.fast_isinstance(vm.ctx.exceptions.generator_exit) }; - if let Some(thrower) = thrower { - let ret = match thrower { - Either::A(coro) => coro - .throw(jen, exc_type, exc_val, exc_tb, vm) - .to_pyresult(vm), // FIXME: - Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), + + if is_gen_exit { + // gen_close_iter: close the sub-iterator + let close_result = if let Some(coro) = self.builtin_coro(jen) { + coro.close(jen, vm).map(|_| ()) + } else if let Some(close_meth) = vm.get_attribute_opt(jen.to_owned(), "close")? { + close_meth.call((), vm).map(|_| ()) + } else { + Ok(()) }; - return ret.map(ExecutionResult::Yield).or_else(|err| { - self.pop_value(); - self.update_lasti(|i| *i += 1); - if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) { - let val = vm.unwrap_or_none(err.get_arg(0)); - self.push_value(val); - self.run(vm) - } else { - let (ty, val, tb) = vm.split_exception(err); - self.gen_throw(vm, ty, val, tb) + if let Err(err) = close_result { + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = err.__traceback__(); + let new_traceback = PyTraceback::new( + next, + self.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); } - }); + + self.push_value(vm.ctx.none()); + vm.contextualize_exception(&err); + return match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + Ok(None) => self.run(vm), + Ok(Some(result)) => Ok(result), + Err(exception) => Err(exception), + }; + } + // Fall through to throw_here to raise GeneratorExit in the generator + } else { + // For non-GeneratorExit, delegate throw to sub-iterator + let thrower = if let Some(coro) = self.builtin_coro(jen) { + Some(Either::A(coro)) + } else { + vm.get_attribute_opt(jen.to_owned(), "throw")? + .map(Either::B) + }; + if let Some(thrower) = thrower { + let ret = match thrower { + Either::A(coro) => coro + .throw(jen, exc_type, exc_val, exc_tb, vm) + .to_pyresult(vm), + Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), + }; + return ret.map(ExecutionResult::Yield).or_else(|err| { + // Add traceback entry for the yield-from/await point. + // gen_send_ex2 resumes the frame with a pending exception, + // which goes through error: → PyTraceBack_Here. We add the + // entry here before calling unwind_blocks. + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = err.__traceback__(); + let new_traceback = PyTraceback::new( + next, + self.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + + self.push_value(vm.ctx.none()); + vm.contextualize_exception(&err); + match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + Ok(None) => self.run(vm), + Ok(Some(result)) => Ok(result), + Err(exception) => Err(exception), + } + }); + } + } + } + // throw_here: no delegate has throw method, or not in yield-from + // Validate the exception type first. Invalid types propagate directly to + // the caller. Valid types with failed instantiation (e.g. __new__ returns + // wrong type) get thrown into the generator via PyErr_SetObject path. + let ctor = ExceptionCtor::try_from_object(vm, exc_type)?; + let exception = match ctor.instantiate_value(exc_val, vm) { + Ok(exc) => { + if let Some(tb) = Option::<PyRef<PyTraceback>>::try_from_object(vm, exc_tb)? { + exc.set_traceback_typed(Some(tb)); + } + exc } + Err(err) => err, + }; + + // Add traceback entry for the generator frame at the yield site + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = exception.__traceback__(); + let new_traceback = + PyTraceback::new(next, self.object.to_owned(), idx as u32 * 2, loc.line); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); } - let exception = vm.normalize_exception(exc_type, exc_val, exc_tb)?; + + // when raising an exception, set __context__ to the current exception + // This is done in _PyErr_SetObject + vm.contextualize_exception(&exception); + + // always pushes Py_None before calling gen_send_ex with exc=1 + // This is needed for exception handler to have correct stack state + self.push_value(vm.ctx.none()); + match self.unwind_blocks(vm, UnwindReason::Raising { exception }) { Ok(None) => self.run(vm), Ok(Some(result)) => Ok(result), @@ -487,7 +771,7 @@ impl ExecutingFrame<'_> { } else { let name = self.code.freevars[i - self.code.cellvars.len()]; vm.new_name_error( - format!("free variable '{name}' referenced before assignment in enclosing scope"), + format!("cannot access free variable '{name}' where it is not associated with a value in enclosing scope"), name.to_owned(), ) } @@ -497,7 +781,7 @@ impl ExecutingFrame<'_> { #[inline(always)] fn execute_instruction( &mut self, - instruction: bytecode::Instruction, + instruction: Instruction, arg: bytecode::OpArg, extend_arg: &mut bool, vm: &VirtualMachine, @@ -531,77 +815,37 @@ impl ExecutingFrame<'_> { } match instruction { - bytecode::Instruction::BeforeAsyncWith => { - let mgr = self.pop_value(); - let error_string = || -> String { - format!( - "'{:.200}' object does not support the asynchronous context manager protocol", - mgr.class().name(), - ) - }; - - let aenter_res = vm - .get_special_method(&mgr, identifier!(vm, __aenter__))? - .ok_or_else(|| vm.new_type_error(error_string()))? - .invoke((), vm)?; - let aexit = mgr - .get_attr(identifier!(vm, __aexit__), vm) - .map_err(|_exc| { - vm.new_type_error({ - format!("{} (missed __aexit__ method)", error_string()) - }) - })?; - self.push_value(aexit); - self.push_value(aenter_res); - - Ok(None) - } - bytecode::Instruction::BinaryOp { op } => self.execute_bin_op(vm, op.get(arg)), - bytecode::Instruction::BinarySubscript => { - let key = self.pop_value(); + Instruction::BinaryOp { op } => self.execute_bin_op(vm, op.get(arg)), + // TODO: In CPython, this does in-place unicode concatenation when + // refcount is 1. Falls back to regular iadd for now. + Instruction::BinaryOpInplaceAddUnicode => { + self.execute_bin_op(vm, bytecode::BinaryOperator::InplaceAdd) + } + Instruction::BinarySlice => { + // Stack: [container, start, stop] -> [result] + let stop = self.pop_value(); + let start = self.pop_value(); let container = self.pop_value(); - self.state - .stack - .push(container.get_item(key.as_object(), vm)?); - Ok(None) - } - - bytecode::Instruction::Break { target } => self.unwind_blocks( - vm, - UnwindReason::Break { - target: target.get(arg), - }, - ), - bytecode::Instruction::BuildListFromTuples { size } => { - // SAFETY: compiler guarantees `size` tuples are on the stack - let elements = unsafe { self.flatten_tuples(size.get(arg) as usize) }; - let list_obj = vm.ctx.new_list(elements); - self.push_value(list_obj.into()); + let slice: PyObjectRef = PySlice { + start: Some(start), + stop, + step: None, + } + .into_ref(&vm.ctx) + .into(); + let result = container.get_item(&*slice, vm)?; + self.push_value(result); Ok(None) } - bytecode::Instruction::BuildList { size } => { - let elements = self.pop_multiple(size.get(arg) as usize).collect(); + Instruction::BuildList { size } => { + let sz = size.get(arg) as usize; + let elements = self.pop_multiple(sz).collect(); let list_obj = vm.ctx.new_list(elements); self.push_value(list_obj.into()); Ok(None) } - bytecode::Instruction::BuildMapForCall { size } => { - self.execute_build_map_for_call(vm, size.get(arg)) - } - bytecode::Instruction::BuildMap { size } => self.execute_build_map(vm, size.get(arg)), - bytecode::Instruction::BuildSetFromTuples { size } => { - let set = PySet::default().into_ref(&vm.ctx); - for element in self.pop_multiple(size.get(arg) as usize) { - // SAFETY: trust compiler - let tup = unsafe { element.downcast_unchecked::<PyTuple>() }; - for item in tup.iter() { - set.add(item.clone(), vm)?; - } - } - self.push_value(set.into()); - Ok(None) - } - bytecode::Instruction::BuildSet { size } => { + Instruction::BuildMap { size } => self.execute_build_map(vm, size.get(arg)), + Instruction::BuildSet { size } => { let set = PySet::default().into_ref(&vm.ctx); for element in self.pop_multiple(size.get(arg) as usize) { set.add(element, vm)?; @@ -609,11 +853,9 @@ impl ExecutingFrame<'_> { self.push_value(set.into()); Ok(None) } - bytecode::Instruction::BuildSlice { argc } => { - self.execute_build_slice(vm, argc.get(arg)) - } + Instruction::BuildSlice { argc } => self.execute_build_slice(vm, argc.get(arg)), /* - bytecode::Instruction::ToBool => { + Instruction::ToBool => { dbg!("Shouldn't be called outside of match statements for now") let value = self.pop_value(); // call __bool__ @@ -622,77 +864,118 @@ impl ExecutingFrame<'_> { Ok(None) } */ - bytecode::Instruction::BuildString { size } => { - let s = self + Instruction::BuildString { size } => { + let s: Wtf8Buf = self .pop_multiple(size.get(arg) as usize) - .as_slice() - .iter() - .map(|pyobj| pyobj.downcast_ref::<PyStr>().unwrap()) - .collect::<Wtf8Buf>(); - let str_obj = vm.ctx.new_str(s); - self.push_value(str_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildTupleFromIter => { - if !self.top_value().class().is(vm.ctx.types.tuple_type) { - let elements: Vec<_> = self.pop_value().try_to_value(vm)?; - let list_obj = vm.ctx.new_tuple(elements); - self.push_value(list_obj.into()); - } + .map(|pyobj| pyobj.downcast::<PyStr>().unwrap()) + .collect(); + self.push_value(vm.ctx.new_str(s).into()); Ok(None) } - bytecode::Instruction::BuildTupleFromTuples { size } => { - // SAFETY: compiler guarantees `size` tuples are on the stack - let elements = unsafe { self.flatten_tuples(size.get(arg) as usize) }; + Instruction::BuildTuple { size } => { + let elements = self.pop_multiple(size.get(arg) as usize).collect(); let list_obj = vm.ctx.new_tuple(elements); self.push_value(list_obj.into()); Ok(None) } - bytecode::Instruction::BuildTuple { size } => { - let elements = self.pop_multiple(size.get(arg) as usize).collect(); - let list_obj = vm.ctx.new_tuple(elements); - self.push_value(list_obj.into()); + Instruction::BuildTemplate => { + // Stack: [strings_tuple, interpolations_tuple] -> [template] + let interpolations = self.pop_value(); + let strings = self.pop_value(); + + let strings = strings + .downcast::<PyTuple>() + .map_err(|_| vm.new_type_error("BUILD_TEMPLATE expected tuple for strings"))?; + let interpolations = interpolations.downcast::<PyTuple>().map_err(|_| { + vm.new_type_error("BUILD_TEMPLATE expected tuple for interpolations") + })?; + + let template = PyTemplate::new(strings, interpolations); + self.push_value(template.into_pyobject(vm)); + Ok(None) + } + Instruction::BuildInterpolation { oparg } => { + // oparg encoding: (conversion << 2) | has_format_spec + // Stack: [value, expression_str, (format_spec)?] -> [interpolation] + let oparg_val = oparg.get(arg); + let has_format_spec = (oparg_val & 1) != 0; + let conversion_code = oparg_val >> 2; + + let format_spec = if has_format_spec { + self.pop_value().downcast::<PyStr>().map_err(|_| { + vm.new_type_error("BUILD_INTERPOLATION expected str for format_spec") + })? + } else { + vm.ctx.empty_str.to_owned() + }; + + let expression = self.pop_value().downcast::<PyStr>().map_err(|_| { + vm.new_type_error("BUILD_INTERPOLATION expected str for expression") + })?; + let value = self.pop_value(); + + // conversion: 0=None, 1=Str, 2=Repr, 3=Ascii + let conversion: PyObjectRef = match conversion_code { + 0 => vm.ctx.none(), + 1 => vm.ctx.new_str("s").into(), + 2 => vm.ctx.new_str("r").into(), + 3 => vm.ctx.new_str("a").into(), + _ => vm.ctx.none(), // should not happen + }; + + let interpolation = + PyInterpolation::new(value, expression, conversion, format_spec, vm)?; + self.push_value(interpolation.into_pyobject(vm)); Ok(None) } - bytecode::Instruction::CallFunctionEx { has_kwargs } => { - let args = self.collect_ex_args(vm, has_kwargs.get(arg))?; + Instruction::Call { nargs } => { + // Stack: [callable, self_or_null, arg1, ..., argN] + let args = self.collect_positional_args(nargs.get(arg)); self.execute_call(args, vm) } - bytecode::Instruction::CallFunctionKeyword { nargs } => { + Instruction::CallKw { nargs } => { + // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] let args = self.collect_keyword_args(nargs.get(arg)); self.execute_call(args, vm) } - bytecode::Instruction::CallFunctionPositional { nargs } => { - let args = self.collect_positional_args(nargs.get(arg)); + Instruction::CallFunctionEx => { + // Stack: [callable, self_or_null, args_tuple, kwargs_or_null] + let args = self.collect_ex_args(vm)?; self.execute_call(args, vm) } - bytecode::Instruction::CallIntrinsic1 { func } => { + Instruction::CallIntrinsic1 { func } => { let value = self.pop_value(); let result = self.call_intrinsic_1(func.get(arg), value, vm)?; self.push_value(result); Ok(None) } - bytecode::Instruction::CallIntrinsic2 { func } => { + Instruction::CallIntrinsic2 { func } => { let value2 = self.pop_value(); let value1 = self.pop_value(); let result = self.call_intrinsic_2(func.get(arg), value1, value2, vm)?; self.push_value(result); Ok(None) } - bytecode::Instruction::CallMethodEx { has_kwargs } => { - let args = self.collect_ex_args(vm, has_kwargs.get(arg))?; - self.execute_method_call(args, vm) - } - bytecode::Instruction::CallMethodKeyword { nargs } => { - let args = self.collect_keyword_args(nargs.get(arg)); - self.execute_method_call(args, vm) - } - bytecode::Instruction::CallMethodPositional { nargs } => { - let args = self.collect_positional_args(nargs.get(arg)); - self.execute_method_call(args, vm) + Instruction::CheckEgMatch => { + let match_type = self.pop_value(); + let exc_value = self.pop_value(); + let (rest, matched) = + crate::exceptions::exception_group_match(&exc_value, &match_type, vm)?; + + // Set matched exception as current exception (if not None) + // This mirrors CPython's PyErr_SetHandledException(match_o) in CHECK_EG_MATCH + if !vm.is_none(&matched) + && let Some(exc) = matched.downcast_ref::<PyBaseException>() + { + vm.set_exception(Some(exc.to_owned())); + } + + self.push_value(rest); + self.push_value(matched); + Ok(None) } - bytecode::Instruction::CompareOperation { op } => self.execute_compare(vm, op.get(arg)), - bytecode::Instruction::ContainsOp(invert) => { + Instruction::CompareOp { op } => self.execute_compare(vm, op.get(arg)), + Instruction::ContainsOp(invert) => { let b = self.pop_value(); let a = self.pop_value(); @@ -703,37 +986,35 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } - bytecode::Instruction::Continue { target } => self.unwind_blocks( - vm, - UnwindReason::Continue { - target: target.get(arg), - }, - ), - - bytecode::Instruction::ConvertValue { oparg: conversion } => { + Instruction::ConvertValue { oparg: conversion } => { self.convert_value(conversion.get(arg), vm) } - bytecode::Instruction::CopyItem { index } => { + Instruction::Copy { index } => { // CopyItem { index: 1 } copies TOS // CopyItem { index: 2 } copies second from top // This is 1-indexed to match CPython let idx = index.get(arg) as usize; - let value = self - .state - .stack - .len() - .checked_sub(idx) - .map(|i| &self.state.stack[i]) - .unwrap(); - self.push_value(value.clone()); + let stack_len = self.state.stack.len(); + if stack_len < idx { + eprintln!("CopyItem ERROR: stack_len={}, idx={}", stack_len, idx); + eprintln!(" code: {}", self.code.obj_name); + eprintln!(" lasti: {}", self.lasti()); + panic!("CopyItem: stack underflow"); + } + let value = self.state.stack[stack_len - idx].clone(); + self.push_value_opt(value); Ok(None) } - bytecode::Instruction::DeleteAttr { idx } => self.delete_attr(vm, idx.get(arg)), - bytecode::Instruction::DeleteDeref(i) => { + Instruction::CopyFreeVars { .. } => { + // Free vars are already set up at frame creation time in RustPython + Ok(None) + } + Instruction::DeleteAttr { idx } => self.delete_attr(vm, idx.get(arg)), + Instruction::DeleteDeref(i) => { self.cells_frees[i.get(arg) as usize].set(None); Ok(None) } - bytecode::Instruction::DeleteFast(idx) => { + Instruction::DeleteFast(idx) => { let mut fastlocals = self.fastlocals.lock(); let idx = idx.get(arg) as usize; if fastlocals[idx].is_none() { @@ -748,7 +1029,7 @@ impl ExecutingFrame<'_> { fastlocals[idx] = None; Ok(None) } - bytecode::Instruction::DeleteGlobal(idx) => { + Instruction::DeleteGlobal(idx) => { let name = self.code.names[idx.get(arg) as usize]; match self.globals.del_item(name, vm) { Ok(()) => {} @@ -759,7 +1040,7 @@ impl ExecutingFrame<'_> { } Ok(None) } - bytecode::Instruction::DeleteLocal(idx) => { + Instruction::DeleteName(idx) => { let name = self.code.names[idx.get(arg) as usize]; let res = self.locals.mapping().ass_subscript(name, None, vm); @@ -772,8 +1053,8 @@ impl ExecutingFrame<'_> { } Ok(None) } - bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), - bytecode::Instruction::DictUpdate { index } => { + Instruction::DeleteSubscr => self.execute_delete_subscript(vm), + Instruction::DictUpdate { index } => { // Stack before: [..., dict, ..., source] (source at TOS) // Stack after: [..., dict, ...] (source consumed) // The dict to update is at position TOS-i (before popping source) @@ -809,56 +1090,85 @@ impl ExecutingFrame<'_> { dict.merge_object(source, vm)?; Ok(None) } - bytecode::Instruction::EndAsyncFor => { - let exc = self.pop_value(); - let except_block = self.pop_block(); // pushed by TryExcept unwind - debug_assert_eq!(except_block.level, self.state.stack.len()); - let _async_iterator = self.pop_value(); // __anext__ provider in the loop - if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { - vm.take_exception().expect("Should have exception in stack"); - Ok(None) + Instruction::DictMerge { index } => { + let source = self.pop_value(); + let idx = index.get(arg); + + // Get the dict to merge into (same logic as DICT_UPDATE) + let dict_ref = if idx <= 1 { + self.top_value() } else { - Err(exc.downcast().unwrap()) + self.nth_value(idx - 1) + }; + + let dict: &Py<PyDict> = unsafe { dict_ref.downcast_unchecked_ref() }; + + // Get callable for error messages + // Stack: [callable, self_or_null, args_tuple, kwargs_dict] + let callable = self.nth_value(idx + 2); + let func_str = Self::object_function_str(callable, vm); + + // Check if source is a mapping + if vm + .get_method(source.clone(), vm.ctx.intern_str("keys")) + .is_none() + { + return Err(vm.new_type_error(format!( + "{} argument after ** must be a mapping, not {}", + func_str, + source.class().name() + ))); } - } - bytecode::Instruction::EndFinally => { - // Pop the finally handler from the stack, and recall - // what was the reason we were in this finally clause. - let block = self.pop_block(); - if let BlockType::FinallyHandler { reason, prev_exc } = block.typ { - vm.set_exception(prev_exc); - if let Some(reason) = reason { - self.unwind_blocks(vm, reason) - } else { - Ok(None) + // Merge keys, checking for duplicates + let keys_iter = vm.call_method(&source, "keys", ())?; + for key in keys_iter.try_to_value::<Vec<PyObjectRef>>(vm)? { + if dict.contains_key(&*key, vm) { + let key_str = key.str(vm)?; + return Err(vm.new_type_error(format!( + "{} got multiple values for keyword argument '{}'", + func_str, + key_str.as_str() + ))); } - } else { - self.fatal( - "Block type must be finally handler when reaching EndFinally instruction!", - ); + let value = vm.call_method(&source, "__getitem__", (key.clone(),))?; + dict.set_item(&*key, value, vm)?; } - } - bytecode::Instruction::EnterFinally => { - self.push_block(BlockType::FinallyHandler { - reason: None, - prev_exc: vm.current_exception(), - }); Ok(None) } - bytecode::Instruction::ExtendedArg => { + Instruction::EndAsyncFor => { + // Pops (awaitable, exc) from stack. + // If exc is StopAsyncIteration, clears it (normal loop end). + // Otherwise re-raises. + let exc = self.pop_value(); + let _awaitable = self.pop_value(); + + let exc = exc + .downcast::<PyBaseException>() + .expect("EndAsyncFor expects exception on stack"); + + if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { + // StopAsyncIteration - normal end of async for loop + vm.set_exception(None); + Ok(None) + } else { + // Other exception - re-raise + Err(exc) + } + } + Instruction::ExtendedArg => { *extend_arg = true; Ok(None) } - bytecode::Instruction::ForIter { target } => self.execute_for_iter(vm, target.get(arg)), - bytecode::Instruction::FormatSimple => { + Instruction::ForIter { target } => self.execute_for_iter(vm, target.get(arg)), + Instruction::FormatSimple => { let value = self.pop_value(); let formatted = vm.format(&value, vm.ctx.new_str(""))?; self.push_value(formatted.into()); Ok(None) } - bytecode::Instruction::FormatWithSpec => { + Instruction::FormatWithSpec => { let spec = self.pop_value(); let value = self.pop_value(); let formatted = vm.format(&value, spec.downcast::<PyStr>().unwrap())?; @@ -866,13 +1176,13 @@ impl ExecutingFrame<'_> { Ok(None) } - bytecode::Instruction::GetAIter => { + Instruction::GetAIter => { let aiterable = self.pop_value(); let aiter = vm.call_special_method(&aiterable, identifier!(vm, __aiter__), ())?; self.push_value(aiter); Ok(None) } - bytecode::Instruction::GetANext => { + Instruction::GetANext => { #[cfg(debug_assertions)] // remove when GetANext is fully implemented let orig_stack_len = self.state.stack.len(); @@ -917,59 +1227,99 @@ impl ExecutingFrame<'_> { debug_assert_eq!(orig_stack_len + 1, self.state.stack.len()); Ok(None) } - bytecode::Instruction::GetAwaitable => { - use crate::protocol::PyIter; + Instruction::GetAwaitable { arg: oparg } => { + let iterable = self.pop_value(); - let awaited_obj = self.pop_value(); - let awaitable = if awaited_obj.downcastable::<PyCoroutine>() { - awaited_obj - } else { - let await_method = vm.get_method_or_type_error( - awaited_obj.clone(), - identifier!(vm, __await__), - || { - format!( - "object {} can't be used in 'await' expression", - awaited_obj.class().name(), - ) - }, - )?; - let result = await_method.call((), vm)?; - // Check that __await__ returned an iterator - if !PyIter::check(&result) { - return Err(vm.new_type_error(format!( - "__await__() returned non-iterator of type '{}'", - result.class().name() - ))); + let iter = match crate::coroutine::get_awaitable_iter(iterable.clone(), vm) { + Ok(iter) => iter, + Err(e) => { + // _PyEval_FormatAwaitableError: override error for async with + // when the type doesn't have __await__ + let oparg_val = oparg.get(arg); + if vm + .get_method(iterable.clone(), identifier!(vm, __await__)) + .is_none() + { + if oparg_val == 1 { + return Err(vm.new_type_error(format!( + "'async with' received an object from __aenter__ \ + that does not implement __await__: {}", + iterable.class().name() + ))); + } else if oparg_val == 2 { + return Err(vm.new_type_error(format!( + "'async with' received an object from __aexit__ \ + that does not implement __await__: {}", + iterable.class().name() + ))); + } + } + return Err(e); } - result }; - self.push_value(awaitable); + + // Check if coroutine is already being awaited + if let Some(coro) = iter.downcast_ref::<PyCoroutine>() + && coro.as_coro().frame().yield_from_target().is_some() + { + return Err( + vm.new_runtime_error("coroutine is being awaited already".to_owned()) + ); + } + + self.push_value(iter); Ok(None) } - bytecode::Instruction::GetIter => { + Instruction::GetIter => { let iterated_obj = self.pop_value(); let iter_obj = iterated_obj.get_iter(vm)?; self.push_value(iter_obj.into()); Ok(None) } - bytecode::Instruction::GetLen => { + Instruction::GetYieldFromIter => { + // GET_YIELD_FROM_ITER: prepare iterator for yield from + // If iterable is a coroutine, ensure we're in a coroutine context + // If iterable is a generator, use it directly + // Otherwise, call iter() on it + let iterable = self.pop_value(); + let iter = if iterable.class().is(vm.ctx.types.coroutine_type) { + // Coroutine requires CO_COROUTINE or CO_ITERABLE_COROUTINE flag + if !self.code.flags.intersects( + bytecode::CodeFlags::COROUTINE | bytecode::CodeFlags::ITERABLE_COROUTINE, + ) { + return Err(vm.new_type_error( + "cannot 'yield from' a coroutine object in a non-coroutine generator" + .to_owned(), + )); + } + iterable + } else if iterable.class().is(vm.ctx.types.generator_type) { + // Generator can be used directly + iterable + } else { + // Otherwise, get iterator + iterable.get_iter(vm)?.into() + }; + self.push_value(iter); + Ok(None) + } + Instruction::GetLen => { // STACK.append(len(STACK[-1])) let obj = self.top_value(); let len = obj.length(vm)?; self.push_value(vm.ctx.new_int(len).into()); Ok(None) } - bytecode::Instruction::ImportFrom { idx } => { + Instruction::ImportFrom { idx } => { let obj = self.import_from(vm, idx.get(arg))?; self.push_value(obj); Ok(None) } - bytecode::Instruction::ImportName { idx } => { + Instruction::ImportName { idx } => { self.import(vm, Some(self.code.names[idx.get(arg) as usize]))?; Ok(None) } - bytecode::Instruction::IsOp(invert) => { + Instruction::IsOp(invert) => { let b = self.pop_value(); let a = self.pop_value(); let res = a.is(&b); @@ -981,40 +1331,19 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } - bytecode::Instruction::JumpIfFalseOrPop { target } => { - self.jump_if_or_pop(vm, target.get(arg), false) + Instruction::JumpForward { target } => { + self.jump(target.get(arg)); + Ok(None) } - bytecode::Instruction::JumpIfNotExcMatch(target) => { - let b = self.pop_value(); - let a = self.pop_value(); - if let Some(tuple_of_exceptions) = b.downcast_ref::<PyTuple>() { - for exception in tuple_of_exceptions { - if !exception - .is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? - { - return Err(vm.new_type_error( - "catching classes that do not inherit from BaseException is not allowed", - )); - } - } - } else if !b.is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? { - return Err(vm.new_type_error( - "catching classes that do not inherit from BaseException is not allowed", - )); - } - - let value = a.is_instance(&b, vm)?; - self.push_value(vm.ctx.new_bool(value).into()); - self.pop_jump_if(vm, target.get(arg), false) - } - bytecode::Instruction::JumpIfTrueOrPop { target } => { - self.jump_if_or_pop(vm, target.get(arg), true) + Instruction::JumpBackward { target } => { + self.jump(target.get(arg)); + Ok(None) } - bytecode::Instruction::Jump { target } => { + Instruction::JumpBackwardNoInterrupt { target } => { self.jump(target.get(arg)); Ok(None) } - bytecode::Instruction::ListAppend { i } => { + Instruction::ListAppend { i } => { let item = self.pop_value(); let obj = self.nth_value(i.get(arg)); let list: &Py<PyList> = unsafe { @@ -1024,19 +1353,87 @@ impl ExecutingFrame<'_> { list.append(item); Ok(None) } - bytecode::Instruction::LoadAttr { idx } => self.load_attr(vm, idx.get(arg)), - bytecode::Instruction::LoadBuildClass => { - self.push_value(vm.builtins.get_attr(identifier!(vm, __build_class__), vm)?); + Instruction::ListExtend { i } => { + let iterable = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let list: &Py<PyList> = unsafe { + // SAFETY: compiler guarantees correct type + obj.downcast_unchecked_ref() + }; + let type_name = iterable.class().name().to_owned(); + // Only rewrite the error if the type is truly not iterable + // (no __iter__ and no __getitem__). Preserve original TypeError + // from custom iterables that raise during iteration. + let not_iterable = iterable.class().slots.iter.load().is_none() + && iterable + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + list.extend(iterable, vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "Value after * must be an iterable, not {type_name}" + )) + } else { + e + } + })?; + Ok(None) + } + Instruction::LoadAttr { idx } => self.load_attr(vm, idx.get(arg)), + Instruction::LoadSuperAttr { arg: idx } => self.load_super_attr(vm, idx.get(arg)), + Instruction::LoadBuildClass => { + let build_class = + if let Some(builtins_dict) = self.builtins.downcast_ref::<PyDict>() { + builtins_dict + .get_item_opt(identifier!(vm, __build_class__), vm)? + .ok_or_else(|| { + vm.new_name_error( + "__build_class__ not found".to_owned(), + identifier!(vm, __build_class__).to_owned(), + ) + })? + } else { + self.builtins + .get_item(identifier!(vm, __build_class__), vm) + .map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_name_error( + "__build_class__ not found".to_owned(), + identifier!(vm, __build_class__).to_owned(), + ) + } else { + e + } + })? + }; + self.push_value(build_class); Ok(None) } - bytecode::Instruction::LoadClassDeref(i) => { + Instruction::LoadLocals => { + // Push the locals dict onto the stack + let locals = self.locals.clone().into_object(); + self.push_value(locals); + Ok(None) + } + Instruction::LoadFromDictOrDeref(i) => { + // Pop dict from stack (locals or classdict depending on context) + let class_dict = self.pop_value(); let i = i.get(arg) as usize; let name = if i < self.code.cellvars.len() { self.code.cellvars[i] } else { self.code.freevars[i - self.code.cellvars.len()] }; - let value = self.locals.mapping().subscript(name, vm).ok(); + // Only treat KeyError as "not found", propagate other exceptions + let value = if let Some(dict_obj) = class_dict.downcast_ref::<PyDict>() { + dict_obj.get_item_opt(name, vm)? + } else { + match class_dict.get_item(name, vm) { + Ok(v) => Some(v), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => None, + Err(e) => return Err(e), + } + }; self.push_value(match value { Some(v) => v, None => self.cells_frees[i] @@ -1045,24 +1442,64 @@ impl ExecutingFrame<'_> { }); Ok(None) } - bytecode::Instruction::LoadClosure(i) => { - let value = self.cells_frees[i.get(arg) as usize].clone(); - self.push_value(value.into()); + Instruction::LoadFromDictOrGlobals(idx) => { + // PEP 649: Pop dict from stack (classdict), check there first, then globals + let dict = self.pop_value(); + let name = self.code.names[idx.get(arg) as usize]; + + // Only treat KeyError as "not found", propagate other exceptions + let value = if let Some(dict_obj) = dict.downcast_ref::<PyDict>() { + dict_obj.get_item_opt(name, vm)? + } else { + // Not an exact dict, use mapping protocol + match dict.get_item(name, vm) { + Ok(v) => Some(v), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => None, + Err(e) => return Err(e), + } + }; + + self.push_value(match value { + Some(v) => v, + None => self.load_global_or_builtin(name, vm)?, + }); Ok(None) } - bytecode::Instruction::LoadConst { idx } => { + Instruction::LoadConst { idx } => { self.push_value(self.code.constants[idx.get(arg) as usize].clone().into()); Ok(None) } - bytecode::Instruction::LoadDeref(i) => { - let i = i.get(arg) as usize; - let x = self.cells_frees[i] + Instruction::LoadCommonConstant { idx } => { + use bytecode::CommonConstant; + let value = match idx.get(arg) { + CommonConstant::AssertionError => { + vm.ctx.exceptions.assertion_error.to_owned().into() + } + CommonConstant::NotImplementedError => { + vm.ctx.exceptions.not_implemented_error.to_owned().into() + } + CommonConstant::BuiltinTuple => vm.ctx.types.tuple_type.to_owned().into(), + CommonConstant::BuiltinAll => vm.builtins.get_attr("all", vm)?, + CommonConstant::BuiltinAny => vm.builtins.get_attr("any", vm)?, + }; + self.push_value(value); + Ok(None) + } + Instruction::LoadSmallInt { idx } => { + // Push small integer (-5..=256) directly without constant table lookup + let value = vm.ctx.new_int(idx.get(arg) as i32); + self.push_value(value.into()); + Ok(None) + } + Instruction::LoadDeref(i) => { + let idx = i.get(arg) as usize; + let x = self.cells_frees[idx] .get() - .ok_or_else(|| self.unbound_cell_exception(i, vm))?; + .ok_or_else(|| self.unbound_cell_exception(idx, vm))?; self.push_value(x); Ok(None) } - bytecode::Instruction::LoadFast(idx) => { + Instruction::LoadFast(idx) => { #[cold] fn reference_error( varname: &'static PyStrInterned, @@ -1080,28 +1517,120 @@ impl ExecutingFrame<'_> { self.push_value(x); Ok(None) } - bytecode::Instruction::LoadGlobal(idx) => { - let name = &self.code.names[idx.get(arg) as usize]; - let x = self.load_global_or_builtin(name, vm)?; + Instruction::LoadFastAndClear(idx) => { + // Load value and clear the slot (for inlined comprehensions) + // If slot is empty, push None (not an error - variable may not exist yet) + let idx = idx.get(arg) as usize; + let x = self.fastlocals.lock()[idx] + .take() + .unwrap_or_else(|| vm.ctx.none()); self.push_value(x); Ok(None) } - bytecode::Instruction::LoadMethod { idx } => { - let obj = self.pop_value(); - let method_name = self.code.names[idx.get(arg) as usize]; - let method = PyMethod::get(obj, method_name, vm)?; - let (target, is_method, func) = match method { - PyMethod::Function { target, func } => (target, true, func), - PyMethod::Attribute(val) => (vm.ctx.none(), false, val), - }; - // TODO: figure out a better way to communicate PyMethod::Attribute - CPython uses - // target==NULL, maybe we could use a sentinel value or something? - self.push_value(target); - self.push_value(vm.ctx.new_bool(is_method).into()); - self.push_value(func); + Instruction::LoadFastCheck(idx) => { + // Same as LoadFast but explicitly checks for unbound locals + // (LoadFast in RustPython already does this check) + let idx = idx.get(arg) as usize; + let x = self.fastlocals.lock()[idx].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx] + ), + ) + })?; + self.push_value(x); + Ok(None) + } + Instruction::LoadFastLoadFast { arg: packed } => { + // Load two local variables at once + // oparg encoding: (idx1 << 4) | idx2 + let oparg = packed.get(arg); + let idx1 = (oparg >> 4) as usize; + let idx2 = (oparg & 15) as usize; + let fastlocals = self.fastlocals.lock(); + let x1 = fastlocals[idx1].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx1] + ), + ) + })?; + let x2 = fastlocals[idx2].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx2] + ), + ) + })?; + drop(fastlocals); + self.push_value(x1); + self.push_value(x2); + Ok(None) + } + // TODO: Implement true borrow optimization (skip Arc::clone). + // CPython's LOAD_FAST_BORROW uses PyStackRef_Borrow to avoid refcount + // increment for values that are consumed within the same basic block. + // Possible approaches: + // - Store raw pointers with careful lifetime management + // - Add a "borrowed" variant to stack slots + // - Use arena allocation for short-lived stack values + // Currently this just clones like LoadFast. + Instruction::LoadFastBorrow(idx) => { + let idx = idx.get(arg) as usize; + let x = self.fastlocals.lock()[idx].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx] + ), + ) + })?; + self.push_value(x); Ok(None) } - bytecode::Instruction::LoadNameAny(idx) => { + // TODO: Same as LoadFastBorrow - implement true borrow optimization. + Instruction::LoadFastBorrowLoadFastBorrow { arg: packed } => { + let oparg = packed.get(arg); + let idx1 = (oparg >> 4) as usize; + let idx2 = (oparg & 15) as usize; + let fastlocals = self.fastlocals.lock(); + let x1 = fastlocals[idx1].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx1] + ), + ) + })?; + let x2 = fastlocals[idx2].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx2] + ), + ) + })?; + drop(fastlocals); + self.push_value(x1); + self.push_value(x2); + Ok(None) + } + Instruction::LoadGlobal(idx) => { + let name = &self.code.names[idx.get(arg) as usize]; + let x = self.load_global_or_builtin(name, vm)?; + self.push_value(x); + Ok(None) + } + Instruction::LoadName(idx) => { let name = self.code.names[idx.get(arg) as usize]; let result = self.locals.mapping().subscript(name, vm); match result { @@ -1113,8 +1642,41 @@ impl ExecutingFrame<'_> { } Ok(None) } - bytecode::Instruction::MakeFunction => self.execute_make_function(vm), - bytecode::Instruction::MapAdd { i } => { + Instruction::LoadSpecial { method } => { + // Stack effect: 0 (replaces TOS with bound method) + // Input: [..., obj] + // Output: [..., bound_method] + use crate::vm::PyMethod; + + let obj = self.pop_value(); + let oparg = method.get(arg); + let method_name = get_special_method_name(oparg, vm); + + let bound = match vm.get_special_method(&obj, method_name)? { + Some(PyMethod::Function { target, func }) => { + // Create bound method: PyBoundMethod(object=target, function=func) + crate::builtins::PyBoundMethod::new(target, func) + .into_ref(&vm.ctx) + .into() + } + Some(PyMethod::Attribute(bound)) => bound, + None => { + return Err(vm.new_type_error(get_special_method_error_msg( + oparg, + &obj.class().name(), + special_method_can_suggest(&obj, oparg, vm)?, + ))); + } + }; + self.push_value(bound); + Ok(None) + } + Instruction::MakeFunction => self.execute_make_function(vm), + Instruction::MakeCell(_) => { + // Cell creation is handled at frame creation time in RustPython + Ok(None) + } + Instruction::MapAdd { i } => { let value = self.pop_value(); let key = self.pop_value(); let obj = self.nth_value(i.get(arg)); @@ -1125,7 +1687,7 @@ impl ExecutingFrame<'_> { dict.set_item(&*key, value, vm)?; Ok(None) } - bytecode::Instruction::MatchClass(nargs) => { + Instruction::MatchClass(nargs) => { // STACK[-1] is a tuple of keyword attribute names, STACK[-2] is the class being matched against, and STACK[-3] is the match subject. // nargs is the number of positional sub-patterns. let kwd_attrs = self.pop_value(); @@ -1246,7 +1808,7 @@ impl ExecutingFrame<'_> { } Ok(None) } - bytecode::Instruction::MatchKeys => { + Instruction::MatchKeys => { // MATCH_KEYS doesn't pop subject and keys, only reads them let keys_tuple = self.top_value(); // stack[-1] let subject = self.nth_value(1); // stack[-2] @@ -1267,8 +1829,6 @@ impl ExecutingFrame<'_> { .get_method(subject.to_owned(), vm.ctx.intern_str("get")) .transpose()? { - // dummy = object() - // CPython: dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); let dummy = vm .ctx .new_base_object(vm.ctx.types.object_type.to_owned(), None); @@ -1314,7 +1874,7 @@ impl ExecutingFrame<'_> { } Ok(None) } - bytecode::Instruction::MatchMapping => { + Instruction::MatchMapping => { // Pop and push back the subject to keep it on stack let subject = self.pop_value(); @@ -1325,7 +1885,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(is_mapping).into()); Ok(None) } - bytecode::Instruction::MatchSequence => { + Instruction::MatchSequence => { // Pop and push back the subject to keep it on stack let subject = self.pop_value(); @@ -1336,33 +1896,78 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(is_sequence).into()); Ok(None) } - bytecode::Instruction::Nop => Ok(None), - bytecode::Instruction::Pop => { - // Pop value from stack and ignore. - self.pop_value(); + Instruction::Nop => Ok(None), + // NOT_TAKEN is a branch prediction hint - functionally a NOP + Instruction::NotTaken => Ok(None), + // Instrumented version of NOT_TAKEN - NOP without monitoring + Instruction::InstrumentedNotTaken => Ok(None), + // CACHE is used by adaptive interpreter for inline caching - NOP for us + Instruction::Cache => Ok(None), + Instruction::ReturnGenerator => { + // In RustPython, generators/coroutines are created in function.rs + // before the frame starts executing. The RETURN_GENERATOR instruction + // pushes None so that the following POP_TOP has something to consume. + // This matches CPython's semantics where the sent value (None for first call) + // is on the stack when the generator resumes. + self.push_value(vm.ctx.none()); + Ok(None) + } + Instruction::PopExcept => { + // Pop prev_exc from value stack and restore it + let prev_exc = self.pop_value(); + if vm.is_none(&prev_exc) { + vm.set_exception(None); + } else if let Ok(exc) = prev_exc.downcast::<PyBaseException>() { + vm.set_exception(Some(exc)); + } + + // NOTE: We do NOT clear the traceback of the exception that was just handled. + // Python preserves exception tracebacks even after the exception is no longer + // the "current exception". This is important for code that catches an exception, + // stores it, and later inspects its traceback. + // Reference cycles (Exception → Traceback → Frame → locals) are handled by + // Python's garbage collector which can detect and break cycles. + Ok(None) } - bytecode::Instruction::PopBlock => { - self.pop_block(); + Instruction::PopJumpIfFalse { target } => self.pop_jump_if(vm, target.get(arg), false), + Instruction::PopJumpIfTrue { target } => self.pop_jump_if(vm, target.get(arg), true), + Instruction::PopJumpIfNone { target } => { + let value = self.pop_value(); + if vm.is_none(&value) { + self.jump(target.get(arg)); + } Ok(None) } - bytecode::Instruction::PopException => { - let block = self.pop_block(); - if let BlockType::ExceptHandler { prev_exc } = block.typ { - vm.set_exception(prev_exc); - Ok(None) - } else { - self.fatal("block type must be ExceptHandler here.") + Instruction::PopJumpIfNotNone { target } => { + let value = self.pop_value(); + if !vm.is_none(&value) { + self.jump(target.get(arg)); } + Ok(None) } - bytecode::Instruction::PopJumpIfFalse { target } => { - self.pop_jump_if(vm, target.get(arg), false) + Instruction::PopTop => { + // Pop value from stack and ignore. + self.pop_value(); + Ok(None) } - bytecode::Instruction::PopJumpIfTrue { target } => { - self.pop_jump_if(vm, target.get(arg), true) + Instruction::EndFor => { + // Pop the next value from stack (cleanup after loop body) + self.pop_value(); + Ok(None) } - bytecode::Instruction::Raise { kind } => self.execute_raise(vm, kind.get(arg)), - bytecode::Instruction::Resume { arg: resume_arg } => { + Instruction::PopIter => { + // Pop the iterator from stack (end of for loop) + self.pop_value(); + Ok(None) + } + Instruction::PushNull => { + // Push NULL for self_or_null slot in call protocol + self.push_null(); + Ok(None) + } + Instruction::RaiseVarargs { kind } => self.execute_raise(vm, kind.get(arg)), + Instruction::Resume { arg: resume_arg } => { // Resume execution after yield, await, or at function start // In CPython, this checks instrumentation and eval breaker // For now, we just check for signals/interrupts @@ -1374,20 +1979,11 @@ impl ExecutingFrame<'_> { // } Ok(None) } - bytecode::Instruction::ReturnConst { idx } => { - let value = self.code.constants[idx.get(arg) as usize].clone().into(); - self.unwind_blocks(vm, UnwindReason::Returning { value }) - } - bytecode::Instruction::ReturnValue => { + Instruction::ReturnValue => { let value = self.pop_value(); self.unwind_blocks(vm, UnwindReason::Returning { value }) } - bytecode::Instruction::Reverse { amount } => { - let stack_len = self.state.stack.len(); - self.state.stack[stack_len - amount.get(arg) as usize..stack_len].reverse(); - Ok(None) - } - bytecode::Instruction::SetAdd { i } => { + Instruction::SetAdd { i } => { let item = self.pop_value(); let obj = self.nth_value(i.get(arg)); let set: &Py<PySet> = unsafe { @@ -1397,87 +1993,164 @@ impl ExecutingFrame<'_> { set.add(item, vm)?; Ok(None) } - bytecode::Instruction::SetFunctionAttribute { attr } => { - self.execute_set_function_attribute(vm, attr.get(arg)) - } - bytecode::Instruction::SetupAnnotation => self.setup_annotations(vm), - bytecode::Instruction::SetupAsyncWith { end } => { - let enter_res = self.pop_value(); - self.push_block(BlockType::Finally { - handler: end.get(arg), - }); - self.push_value(enter_res); + Instruction::SetUpdate { i } => { + let iterable = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let set: &Py<PySet> = unsafe { + // SAFETY: compiler guarantees correct type + obj.downcast_unchecked_ref() + }; + let iter = PyIter::try_from_object(vm, iterable)?; + while let PyIterReturn::Return(item) = iter.next(vm)? { + set.add(item, vm)?; + } Ok(None) } - bytecode::Instruction::SetupExcept { handler } => { - self.push_block(BlockType::TryExcept { - handler: handler.get(arg), - }); + Instruction::PushExcInfo => { + // Stack: [exc] -> [prev_exc, exc] + let exc = self.pop_value(); + let prev_exc = vm + .current_exception() + .map(|e| e.into()) + .unwrap_or_else(|| vm.ctx.none()); + + // Set exc as the current exception + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() { + vm.set_exception(Some(exc_ref.to_owned())); + } + + self.push_value(prev_exc); + self.push_value(exc); Ok(None) } - bytecode::Instruction::SetupFinally { handler } => { - self.push_block(BlockType::Finally { - handler: handler.get(arg), - }); + Instruction::CheckExcMatch => { + // Stack: [exc, type] -> [exc, bool] + let exc_type = self.pop_value(); + let exc = self.top_value(); + + // Validate that exc_type inherits from BaseException + if let Some(tuple_of_exceptions) = exc_type.downcast_ref::<PyTuple>() { + for exception in tuple_of_exceptions { + if !exception + .is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? + { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed", + )); + } + } + } else if !exc_type.is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed", + )); + } + + let result = exc.is_instance(&exc_type, vm)?; + self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } - bytecode::Instruction::SetupLoop => { - self.push_block(BlockType::Loop); - Ok(None) + Instruction::Reraise { depth } => { + // inst(RERAISE, (values[oparg], exc -- values[oparg])) + // + // Stack layout: [values..., exc] where len(values) == oparg + // RERAISE pops exc and oparg additional values from the stack. + // values[0] is lasti used to set frame->instr_ptr for traceback. + // We skip the lasti update since RustPython's traceback is already correct. + let depth_val = depth.get(arg) as usize; + + // Pop exception from TOS + let exc = self.pop_value(); + + // Pop the depth values (lasti and possibly other items like prev_exc) + for _ in 0..depth_val { + self.pop_value(); + } + + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() { + Err(exc_ref.to_owned()) + } else { + // Fallback: use current exception if TOS is not an exception + let exc = vm + .topmost_exception() + .ok_or_else(|| vm.new_runtime_error("No active exception to re-raise"))?; + Err(exc) + } } - bytecode::Instruction::SetupWith { end } => { - let context_manager = self.pop_value(); - let error_string = || -> String { - format!( - "'{:.200}' object does not support the context manager protocol", - context_manager.class().name(), - ) - }; - let enter_res = vm - .get_special_method(&context_manager, identifier!(vm, __enter__))? - .ok_or_else(|| vm.new_type_error(error_string()))? - .invoke((), vm)?; - - let exit = context_manager - .get_attr(identifier!(vm, __exit__), vm) - .map_err(|_exc| { - vm.new_type_error({ - format!("{} (missed __exit__ method)", error_string()) - }) - })?; - self.push_value(exit); - self.push_block(BlockType::Finally { - handler: end.get(arg), - }); - self.push_value(enter_res); - Ok(None) + Instruction::SetFunctionAttribute { attr } => { + self.execute_set_function_attribute(vm, attr.get(arg)) } - bytecode::Instruction::StoreAttr { idx } => self.store_attr(vm, idx.get(arg)), - bytecode::Instruction::StoreDeref(i) => { + Instruction::SetupAnnotations => self.setup_annotations(vm), + Instruction::StoreAttr { idx } => self.store_attr(vm, idx.get(arg)), + Instruction::StoreDeref(i) => { let value = self.pop_value(); self.cells_frees[i.get(arg) as usize].set(Some(value)); Ok(None) } - bytecode::Instruction::StoreFast(idx) => { + Instruction::StoreFast(idx) => { let value = self.pop_value(); self.fastlocals.lock()[idx.get(arg) as usize] = Some(value); Ok(None) } - bytecode::Instruction::StoreGlobal(idx) => { + Instruction::StoreFastLoadFast { + store_idx, + load_idx, + } => { + // Store to one slot and load from another (often the same) - for inlined comprehensions + let value = self.pop_value(); + let mut locals = self.fastlocals.lock(); + locals[store_idx.get(arg) as usize] = Some(value); + let load_value = locals[load_idx.get(arg) as usize] + .clone() + .expect("StoreFastLoadFast: load slot should have value after store"); + drop(locals); + self.push_value(load_value); + Ok(None) + } + Instruction::StoreFastStoreFast { arg: packed } => { + // Store two values to two local variables at once + // STORE_FAST idx1 executes first: pops TOS -> locals[idx1] + // STORE_FAST idx2 executes second: pops new TOS -> locals[idx2] + // oparg encoding: (idx1 << 4) | idx2 + let oparg = packed.get(arg); + let idx1 = (oparg >> 4) as usize; + let idx2 = (oparg & 15) as usize; + let value1 = self.pop_value(); // TOS -> idx1 + let value2 = self.pop_value(); // second -> idx2 + let mut fastlocals = self.fastlocals.lock(); + fastlocals[idx1] = Some(value1); + fastlocals[idx2] = Some(value2); + Ok(None) + } + Instruction::StoreGlobal(idx) => { let value = self.pop_value(); self.globals .set_item(self.code.names[idx.get(arg) as usize], value, vm)?; Ok(None) } - bytecode::Instruction::StoreLocal(idx) => { + Instruction::StoreName(idx) => { let name = self.code.names[idx.get(arg) as usize]; let value = self.pop_value(); self.locals.mapping().ass_subscript(name, Some(value), vm)?; Ok(None) } - bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), - bytecode::Instruction::Subscript => self.execute_subscript(vm), - bytecode::Instruction::Swap { index } => { + Instruction::StoreSlice => { + // Stack: [value, container, start, stop] -> [] + let stop = self.pop_value(); + let start = self.pop_value(); + let container = self.pop_value(); + let value = self.pop_value(); + let slice: PyObjectRef = PySlice { + start: Some(start), + stop, + step: None, + } + .into_ref(&vm.ctx) + .into(); + container.set_item(&*slice, value, vm)?; + Ok(None) + } + Instruction::StoreSubscr => self.execute_store_subscript(vm), + Instruction::Swap { index } => { let len = self.state.stack.len(); debug_assert!(len > 0, "stack underflow in SWAP"); let i = len - 1; // TOS index @@ -1494,92 +2167,177 @@ impl ExecutingFrame<'_> { self.state.stack.swap(i, j); Ok(None) } - bytecode::Instruction::ToBool => { + Instruction::ToBool => { let obj = self.pop_value(); let bool_val = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(bool_val).into()); Ok(None) } - bytecode::Instruction::UnaryOperation { op } => self.execute_unary_op(vm, op.get(arg)), - bytecode::Instruction::UnpackEx { args } => { + Instruction::UnpackEx { args } => { let args = args.get(arg); self.execute_unpack_ex(vm, args.before, args.after) } - bytecode::Instruction::UnpackSequence { size } => { - self.unpack_sequence(size.get(arg), vm) - } - bytecode::Instruction::WithCleanupFinish => { - let block = self.pop_block(); - let (reason, prev_exc) = match block.typ { - BlockType::FinallyHandler { reason, prev_exc } => (reason, prev_exc), - _ => self.fatal("WithCleanupFinish expects a FinallyHandler block on stack"), - }; - - vm.set_exception(prev_exc); - - let suppress_exception = self.pop_value().try_to_bool(vm)?; + Instruction::UnpackSequence { size } => self.unpack_sequence(size.get(arg), vm), + Instruction::WithExceptStart => { + // Stack: [..., __exit__, lasti, prev_exc, exc] + // Call __exit__(type, value, tb) and push result + // __exit__ is at TOS-3 (below lasti, prev_exc, and exc) + let exc = vm.current_exception(); - if suppress_exception { - Ok(None) - } else if let Some(reason) = reason { - self.unwind_blocks(vm, reason) - } else { - Ok(None) - } - } - bytecode::Instruction::WithCleanupStart => { - let block = self.current_block().unwrap(); - let reason = match block.typ { - BlockType::FinallyHandler { reason, .. } => reason, - _ => self.fatal("WithCleanupStart expects a FinallyHandler block on stack"), - }; - let exc = match reason { - Some(UnwindReason::Raising { exception }) => Some(exception), - _ => None, - }; - - let exit = self.top_value(); + let stack_len = self.state.stack.len(); + let exit = expect_unchecked( + self.state.stack[stack_len - 4].clone(), + "WithExceptStart: __exit__ is NULL", + ); - let args = if let Some(exc) = exc { - vm.split_exception(exc) + let args = if let Some(ref exc) = exc { + vm.split_exception(exc.clone()) } else { (vm.ctx.none(), vm.ctx.none(), vm.ctx.none()) }; let exit_res = exit.call(args, vm)?; - self.replace_top(exit_res); + // Push result on top of stack + self.push_value(exit_res); Ok(None) } - bytecode::Instruction::YieldFrom => self.execute_yield_from(vm), - bytecode::Instruction::YieldValue => { + Instruction::YieldValue { arg: oparg } => { let value = self.pop_value(); - let value = if self.code.flags.contains(bytecode::CodeFlags::IS_COROUTINE) { + // arg=0: direct yield (wrapped for async generators) + // arg=1: yield from await/yield-from (NOT wrapped) + let wrap = oparg.get(arg) == 0; + let value = if wrap && self.code.flags.contains(bytecode::CodeFlags::COROUTINE) { PyAsyncGenWrappedValue(value).into_pyobject(vm) } else { value }; Ok(Some(ExecutionResult::Yield(value))) } + Instruction::Send { target } => { + // (receiver, v -- receiver, retval) + // Pops v, sends it to receiver. On yield, pushes retval + // (so stack = [..., receiver, retval]). On return/StopIteration, + // also pushes retval and jumps to END_SEND which will pop receiver. + let exit_label = target.get(arg); + let val = self.pop_value(); + let receiver = self.top_value(); + + match self._send(receiver, val, vm)? { + PyIterReturn::Return(value) => { + self.push_value(value); + Ok(None) + } + PyIterReturn::StopIteration(value) => { + let value = vm.unwrap_or_none(value); + self.push_value(value); + self.jump(exit_label); + Ok(None) + } + } + } + Instruction::EndSend => { + // Stack: (receiver, value) -> (value) + // Pops receiver, leaves value + let value = self.pop_value(); + self.pop_value(); // discard receiver + self.push_value(value); + Ok(None) + } + Instruction::ExitInitCheck => { + // Check that __init__ returned None + let should_be_none = self.pop_value(); + if !vm.is_none(&should_be_none) { + return Err(vm.new_type_error(format!( + "__init__() should return None, not '{}'", + should_be_none.class().name() + ))); + } + Ok(None) + } + Instruction::CleanupThrow => { + // CLEANUP_THROW: (sub_iter, last_sent_val, exc) -> (None, value) OR re-raise + // If StopIteration: pop all 3, extract value, push (None, value) + // Otherwise: pop all 3, return Err(exc) for unwind_blocks to handle + // + // Unlike CPython where exception_unwind pops the triple as part of + // stack cleanup to handler depth, RustPython pops here explicitly + // and lets unwind_blocks find outer handlers. + // Compiler sets handler_depth = base + 2 (before exc is pushed). + + // First peek at exc_value (top of stack) without popping + let exc = self.top_value(); + + // Check if it's a StopIteration + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() + && exc_ref.fast_isinstance(vm.ctx.exceptions.stop_iteration) + { + // Extract value from StopIteration + let value = exc_ref.get_arg(0).unwrap_or_else(|| vm.ctx.none()); + // Now pop all three + self.pop_value(); // exc + self.pop_value(); // last_sent_val + self.pop_value(); // sub_iter + self.push_value(vm.ctx.none()); + self.push_value(value); + return Ok(None); + } + + // Re-raise other exceptions: pop all three and return Err(exc) + let exc = self.pop_value(); // exc + self.pop_value(); // last_sent_val + self.pop_value(); // sub_iter + + let exc = exc + .downcast::<PyBaseException>() + .map_err(|_| vm.new_type_error("exception expected".to_owned()))?; + Err(exc) + } + Instruction::UnaryInvert => { + let a = self.pop_value(); + let value = vm._invert(&a)?; + self.push_value(value); + Ok(None) + } + Instruction::UnaryNegative => { + let a = self.pop_value(); + let value = vm._neg(&a)?; + self.push_value(value); + Ok(None) + } + Instruction::UnaryNot => { + let obj = self.pop_value(); + let value = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(!value).into()); + Ok(None) + } + _ => { + unreachable!("{instruction:?} instruction should not be executed") + } } } #[inline] fn load_global_or_builtin(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - self.globals - .get_chain(self.builtins, name, vm)? - .ok_or_else(|| { - vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + if let Some(builtins_dict) = self.builtins.downcast_ref::<PyDict>() { + // Fast path: builtins is a dict + self.globals + .get_chain(builtins_dict, name, vm)? + .ok_or_else(|| { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + }) + } else { + // Slow path: builtins is not a dict, use generic __getitem__ + if let Some(value) = self.globals.get_item_opt(name, vm)? { + return Ok(value); + } + self.builtins.get_item(name, vm).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + } else { + e + } }) - } - - unsafe fn flatten_tuples(&mut self, size: usize) -> Vec<PyObjectRef> { - let mut elements = Vec::new(); - for tup in self.pop_multiple(size) { - // SAFETY: caller ensures that the elements are tuples - let tup = unsafe { tup.downcast_unchecked::<PyTuple>() }; - elements.extend(tup.iter().cloned()); } - elements } #[cfg_attr(feature = "flame-it", flame("Frame"))] @@ -1616,46 +2374,134 @@ impl ExecutingFrame<'_> { sys_modules.get_item(&full_mod_name, vm).ok() })(); - if let Some(sub_module) = fallback_module { - return Ok(sub_module); + if let Some(sub_module) = fallback_module { + return Ok(sub_module); + } + + use crate::import::{ + get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name, + }; + + // Get module name for the error message + let mod_name_obj = module.get_attr(identifier!(vm, __name__), vm).ok(); + let mod_name_str = mod_name_obj + .as_ref() + .and_then(|n| n.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned())); + let module_name = mod_name_str.as_deref().unwrap_or("<unknown module name>"); + + let spec = module + .get_attr("__spec__", vm) + .ok() + .filter(|s| !vm.is_none(s)); + + let origin = get_spec_file_origin(&spec, vm); + + let is_possibly_shadowing = origin + .as_ref() + .map(|o| is_possibly_shadowing_path(o, vm)) + .unwrap_or(false); + let is_possibly_shadowing_stdlib = if is_possibly_shadowing { + if let Some(ref mod_name) = mod_name_obj { + is_stdlib_module_name(mod_name, vm)? + } else { + false + } + } else { + false + }; + + let msg = if is_possibly_shadowing_stdlib { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' since it has the same \ + name as the standard library module named '{module_name}' \ + and prevents importing that standard library module)" + ) + } else { + let is_init = is_module_initializing(module, vm); + if is_init { + if is_possibly_shadowing { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' if it has the same name \ + as a library you intended to import)" + ) + } else if let Some(ref path) = origin { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import) ({path})" + ) + } else { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import)" + ) + } + } else if let Some(ref path) = origin { + format!("cannot import name '{name}' from '{module_name}' ({path})") + } else { + format!("cannot import name '{name}' from '{module_name}' (unknown location)") + } + }; + let err = vm.new_import_error(msg, vm.ctx.new_str(module_name)); + + if let Some(ref path) = origin { + let _ignore = err + .as_object() + .set_attr("path", vm.ctx.new_str(path.as_str()), vm); } - if is_module_initializing(module, vm) { - let module_name = module - .get_attr(identifier!(vm, __name__), vm) - .ok() - .and_then(|n| n.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned())) - .unwrap_or_else(|| "<unknown>".to_owned()); + // name_from = the attribute name that failed to import (best-effort metadata) + let _ignore = err.as_object().set_attr("name_from", name.to_owned(), vm); - let msg = format!( - "cannot import name '{name}' from partially initialized module '{module_name}' (most likely due to a circular import)", - ); - Err(vm.new_import_error(msg, name.to_owned())) - } else { - Err(vm.new_import_error(format!("cannot import name '{name}'"), name.to_owned())) - } + Err(err) } #[cfg_attr(feature = "flame-it", flame("Frame"))] fn import_star(&mut self, vm: &VirtualMachine) -> PyResult<()> { let module = self.pop_value(); - // Grab all the names from the module and put them in the context - if let Some(dict) = module.dict() { - let filter_pred: Box<dyn Fn(&str) -> bool> = - if let Ok(all) = dict.get_item(identifier!(vm, __all__), vm) { - let all: Vec<PyStrRef> = all.try_to_value(vm)?; - let all: Vec<String> = all - .into_iter() - .map(|name| name.as_str().to_owned()) - .collect(); - Box::new(move |name| all.contains(&name.to_owned())) + let Some(dict) = module.dict() else { + return Ok(()); + }; + + let mod_name = module + .get_attr(identifier!(vm, __name__), vm) + .ok() + .and_then(|n| n.downcast::<PyStr>().ok()); + + let require_str = |obj: PyObjectRef, attr: &str| -> PyResult<PyRef<PyStr>> { + obj.downcast().map_err(|obj: PyObjectRef| { + let source = if let Some(ref mod_name) = mod_name { + format!("{}.{attr}", mod_name.as_str()) } else { - Box::new(|name| !name.starts_with('_')) + attr.to_owned() }; + let repr = obj.repr(vm).unwrap_or_else(|_| vm.ctx.new_str("?")); + vm.new_type_error(format!( + "{} in {} must be str, not {}", + repr.as_str(), + source, + obj.class().name() + )) + }) + }; + + if let Ok(all) = dict.get_item(identifier!(vm, __all__), vm) { + let items: Vec<PyObjectRef> = all.try_to_value(vm)?; + for item in items { + let name = require_str(item, "__all__")?; + let value = module.get_attr(&*name, vm)?; + self.locals + .mapping() + .ass_subscript(&name, Some(value), vm)?; + } + } else { for (k, v) in dict { - let k = PyStrRef::try_from_object(vm, k)?; - if filter_pred(k.as_str()) { + let k = require_str(k, "__dict__")?; + if !k.as_str().starts_with('_') { self.locals.mapping().ass_subscript(&k, Some(v), vm)?; } } @@ -1669,76 +2515,52 @@ impl ExecutingFrame<'_> { /// Optionally returns an exception. #[cfg_attr(feature = "flame-it", flame("Frame"))] fn unwind_blocks(&mut self, vm: &VirtualMachine, reason: UnwindReason) -> FrameResult { - // First unwind all existing blocks on the block stack: - while let Some(block) = self.current_block() { - // eprintln!("unwinding block: {:.60?} {:.60?}", block.typ, reason); - match block.typ { - BlockType::Loop => match reason { - UnwindReason::Break { target } => { - self.pop_block(); - self.jump(target); - return Ok(None); - } - UnwindReason::Continue { target } => { - self.jump(target); - return Ok(None); - } - _ => { - self.pop_block(); - } - }, - BlockType::Finally { handler } => { - self.pop_block(); - let prev_exc = vm.current_exception(); - if let UnwindReason::Raising { exception } = &reason { - vm.set_exception(Some(exception.clone())); - } - self.push_block(BlockType::FinallyHandler { - reason: Some(reason), - prev_exc, - }); - self.jump(handler); - return Ok(None); + // use exception table for exception handling + match reason { + UnwindReason::Raising { exception } => { + // Look up handler in exception table + // lasti points to NEXT instruction (already incremented in run loop) + // The exception occurred at the previous instruction + // Python uses signed int where INSTR_OFFSET() - 1 = -1 before first instruction + // We use u32, so check for 0 explicitly (equivalent to CPython's -1) + if self.lasti() == 0 { + // No instruction executed yet, no handler can match + return Err(exception); } - BlockType::TryExcept { handler } => { - self.pop_block(); - if let UnwindReason::Raising { exception } = reason { - self.push_block(BlockType::ExceptHandler { - prev_exc: vm.current_exception(), - }); - vm.contextualize_exception(&exception); - vm.set_exception(Some(exception.clone())); - self.push_value(exception.into()); - self.jump(handler); - return Ok(None); + let offset = self.lasti() - 1; + if let Some(entry) = + bytecode::find_exception_handler(&self.code.exceptiontable, offset) + { + // 1. Pop stack to entry.depth + while self.state.stack.len() > entry.depth as usize { + self.state.stack.pop(); } - } - BlockType::FinallyHandler { prev_exc, .. } - | BlockType::ExceptHandler { prev_exc } => { - self.pop_block(); - vm.set_exception(prev_exc); + + // 2. If push_lasti=true (SETUP_CLEANUP), push lasti before exception + // pushes lasti as PyLong + if entry.push_lasti { + self.push_value(vm.ctx.new_int(offset as i32).into()); + } + + // 3. Push exception onto stack + // always push exception, PUSH_EXC_INFO transforms [exc] -> [prev_exc, exc] + // Note: Do NOT call vm.set_exception here! PUSH_EXC_INFO will do it. + // PUSH_EXC_INFO needs to get prev_exc from vm.current_exception() BEFORE setting the new one. + self.push_value(exception.into()); + + // 4. Jump to handler + self.jump(bytecode::Label(entry.target)); + + Ok(None) + } else { + // No handler found, propagate exception + Err(exception) } } - } - - // We do not have any more blocks to unwind. Inspect the reason we are here: - match reason { - UnwindReason::Raising { exception } => Err(exception), UnwindReason::Returning { value } => Ok(Some(ExecutionResult::Return(value))), - UnwindReason::Break { .. } | UnwindReason::Continue { .. } => { - self.fatal("break or continue must occur within a loop block.") - } // UnwindReason::NoWorries => Ok(None), } } - fn execute_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { - let b_ref = self.pop_value(); - let a_ref = self.pop_value(); - let value = a_ref.get_item(&*b_ref, vm)?; - self.push_value(value); - Ok(None) - } - fn execute_store_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); @@ -1765,35 +2587,6 @@ impl ExecutingFrame<'_> { Ok(None) } - fn execute_build_map_for_call(&mut self, vm: &VirtualMachine, size: u32) -> FrameResult { - let size = size as usize; - let map_obj = vm.ctx.new_dict(); - for obj in self.pop_multiple(size) { - // Use keys() method for all mapping objects to preserve order - Self::iterate_mapping_keys(vm, &obj, "keyword argument", |key| { - // Check for keyword argument restrictions - if key.downcast_ref::<PyStr>().is_none() { - return Err(vm.new_type_error("keywords must be strings")); - } - if map_obj.contains_key(&*key, vm) { - let key_repr = &key.repr(vm)?; - let msg = format!( - "got multiple values for keyword argument {}", - key_repr.as_str() - ); - return Err(vm.new_type_error(msg)); - } - - let value = obj.get_item(&*key, vm)?; - map_obj.set_item(&*key, value, vm)?; - Ok(()) - })?; - } - - self.push_value(map_obj.into()); - Ok(None) - } - fn execute_build_slice( &mut self, vm: &VirtualMachine, @@ -1837,13 +2630,16 @@ impl ExecutingFrame<'_> { FuncArgs::with_kwargs_names(args, kwarg_names) } - fn collect_ex_args(&mut self, vm: &VirtualMachine, has_kwargs: bool) -> PyResult<FuncArgs> { - let kwargs = if has_kwargs { - let kw_obj = self.pop_value(); + fn collect_ex_args(&mut self, vm: &VirtualMachine) -> PyResult<FuncArgs> { + let kwargs_or_null = self.pop_value_opt(); + let kwargs = if let Some(kw_obj) = kwargs_or_null { let mut kwargs = IndexMap::new(); - // Use keys() method for all mapping objects to preserve order - Self::iterate_mapping_keys(vm, &kw_obj, "argument after **", |key| { + // Stack: [callable, self_or_null, args_tuple] + let callable = self.nth_value(2); + let func_str = Self::object_function_str(callable, vm); + + Self::iterate_mapping_keys(vm, &kw_obj, &func_str, |key| { let key_str = key .downcast_ref::<PyStr>() .ok_or_else(|| vm.new_type_error("keywords must be strings"))?; @@ -1855,26 +2651,75 @@ impl ExecutingFrame<'_> { } else { IndexMap::new() }; - // SAFETY: trust compiler - let args = unsafe { self.pop_value().downcast_unchecked::<PyTuple>() } - .as_slice() - .to_vec(); + let args_obj = self.pop_value(); + let args = if let Some(tuple) = args_obj.downcast_ref::<PyTuple>() { + tuple.as_slice().to_vec() + } else { + // Single *arg passed directly; convert to sequence at runtime. + // Stack: [callable, self_or_null] + let callable = self.nth_value(1); + let func_str = Self::object_function_str(callable, vm); + let not_iterable = args_obj.class().slots.iter.load().is_none() + && args_obj + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + args_obj.try_to_value::<Vec<PyObjectRef>>(vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "{} argument after * must be an iterable, not {}", + func_str, + args_obj.class().name() + )) + } else { + e + } + })? + }; Ok(FuncArgs { args, kwargs }) } + /// Returns a display string for a callable object for use in error messages. + /// For objects with `__qualname__`, returns "module.qualname()" or "qualname()". + /// For other objects, returns repr(obj). + fn object_function_str(obj: &PyObject, vm: &VirtualMachine) -> String { + let Ok(qualname) = obj.get_attr(vm.ctx.intern_str("__qualname__"), vm) else { + return obj + .repr(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| "?".to_owned()); + }; + let Some(qualname_str) = qualname.downcast_ref::<PyStr>() else { + return obj + .repr(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| "?".to_owned()); + }; + if let Ok(module) = obj.get_attr(vm.ctx.intern_str("__module__"), vm) + && let Some(module_str) = module.downcast_ref::<PyStr>() + && module_str.as_str() != "builtins" + { + return format!("{}.{}()", module_str.as_str(), qualname_str.as_str()); + } + format!("{}()", qualname_str.as_str()) + } + /// Helper function to iterate over mapping keys using the keys() method. /// This ensures proper order preservation for OrderedDict and other custom mappings. fn iterate_mapping_keys<F>( vm: &VirtualMachine, - mapping: &PyObjectRef, - error_prefix: &str, + mapping: &PyObject, + func_str: &str, mut key_handler: F, ) -> PyResult<()> where F: FnMut(PyObjectRef) -> PyResult<()>, { - let Some(keys_method) = vm.get_method(mapping.clone(), vm.ctx.intern_str("keys")) else { - return Err(vm.new_type_error(format!("{error_prefix} must be a mapping"))); + let Some(keys_method) = vm.get_method(mapping.to_owned(), vm.ctx.intern_str("keys")) else { + return Err(vm.new_type_error(format!( + "{} argument after ** must be a mapping, not {}", + func_str, + mapping.class().name() + ))); }; let keys = keys_method?.call((), vm)?.get_iter(vm)?; @@ -1886,31 +2731,25 @@ impl ExecutingFrame<'_> { #[inline] fn execute_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { - let func_ref = self.pop_value(); - let value = func_ref.call(args, vm)?; - self.push_value(value); - Ok(None) - } - - #[inline] - fn execute_method_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { - let func = self.pop_value(); - let is_method = self.pop_value().is(&vm.ctx.true_value); - let target = self.pop_value(); - - // TODO: It was PyMethod before #4873. Check if it's correct. - let func = if is_method { - if let Some(descr_get) = func.class().mro_find_map(|cls| cls.slots.descr_get.load()) { - let cls = target.class().to_owned().into(); - descr_get(func, Some(target), Some(cls), vm)? - } else { - func + // Stack: [callable, self_or_null, ...] + let self_or_null = self.pop_value_opt(); // Option<PyObjectRef> + let callable = self.pop_value(); + + // If self_or_null is Some (not NULL), prepend it to args + let final_args = if let Some(self_val) = self_or_null { + // Method call: prepend self to args + let mut all_args = vec![self_val]; + all_args.extend(args.args); + FuncArgs { + args: all_args, + kwargs: args.kwargs, } } else { - drop(target); // should be None - func + // Regular attribute call: self_or_null is NULL + args }; - let value = func.call(args, vm)?; + + let value = callable.call(final_args, vm)?; self.push_value(value); Ok(None) } @@ -1931,15 +2770,30 @@ impl ExecutingFrame<'_> { }) } // if there's no cause arg, we keep the cause as is - bytecode::RaiseKind::Raise | bytecode::RaiseKind::Reraise => None, + _ => None, }; let exception = match kind { bytecode::RaiseKind::RaiseCause | bytecode::RaiseKind::Raise => { ExceptionCtor::try_from_object(vm, self.pop_value())?.instantiate(vm)? } - bytecode::RaiseKind::Reraise => vm - .topmost_exception() - .ok_or_else(|| vm.new_runtime_error("No active exception to reraise"))?, + bytecode::RaiseKind::BareRaise => { + // RAISE_VARARGS 0: bare `raise` gets exception from VM state + // This is the current exception set by PUSH_EXC_INFO + vm.topmost_exception().ok_or_else(|| { + vm.new_runtime_error("No active exception to reraise".to_owned()) + })? + } + bytecode::RaiseKind::ReraiseFromStack => { + // RERAISE: gets exception from stack top + // Used in cleanup blocks where exception is on stack after COPY 3 + let exc = self.pop_value(); + exc.downcast::<PyBaseException>().map_err(|obj| { + vm.new_type_error(format!( + "exceptions must derive from BaseException, not {}", + obj.class().name() + )) + })? + } }; #[cfg(debug_assertions)] debug!("Exception raised: {exception:?} with cause: {cause:?}"); @@ -1974,31 +2828,23 @@ impl ExecutingFrame<'_> { } } - fn execute_yield_from(&mut self, vm: &VirtualMachine) -> FrameResult { - // Value send into iterator: - let val = self.pop_value(); - let coro = self.top_value(); - let result = self._send(coro, val, vm)?; - - // PyIterReturn returned from e.g. gen.__next__() or gen.send() - match result { - PyIterReturn::Return(value) => { - // Set back program counter: - self.update_lasti(|i| *i -= 1); - Ok(Some(ExecutionResult::Yield(value))) - } - PyIterReturn::StopIteration(value) => { - let value = vm.unwrap_or_none(value); - self.replace_top(value); - Ok(None) - } - } - } - fn execute_unpack_ex(&mut self, vm: &VirtualMachine, before: u8, after: u8) -> FrameResult { let (before, after) = (before as usize, after as usize); let value = self.pop_value(); - let elements: Vec<_> = value.try_to_value(vm)?; + let not_iterable = value.class().slots.iter.load().is_none() + && value + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + let elements: Vec<_> = value.try_to_value(vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "cannot unpack non-iterable {} object", + value.class().name() + )) + } else { + e + } + })?; let min_expected = before + after; let middle = elements.len().checked_sub(min_expected).ok_or_else(|| { @@ -2013,14 +2859,16 @@ impl ExecutingFrame<'_> { // Elements on stack from right-to-left: self.state .stack - .extend(elements.drain(before + middle..).rev()); + .extend(elements.drain(before + middle..).rev().map(Some)); let middle_elements = elements.drain(before..).collect(); let t = vm.ctx.new_list(middle_elements); self.push_value(t.into()); // Lastly the first reversed values: - self.state.stack.extend(elements.into_iter().rev()); + self.state + .stack + .extend(elements.into_iter().rev().map(Some)); Ok(None) } @@ -2047,23 +2895,6 @@ impl ExecutingFrame<'_> { Ok(None) } - #[inline] - fn jump_if_or_pop( - &mut self, - vm: &VirtualMachine, - target: bytecode::Label, - flag: bool, - ) -> FrameResult { - let obj = self.top_value(); - let value = obj.to_owned().try_to_bool(vm)?; - if value == flag { - self.jump(target); - } else { - self.pop_value(); - } - Ok(None) - } - /// The top of stack contains the iterator, lets push it forward fn execute_for_iter(&mut self, vm: &VirtualMachine, target: bytecode::Label) -> FrameResult { let top_of_stack = PyIter::new(self.top_value()); @@ -2076,15 +2907,30 @@ impl ExecutingFrame<'_> { Ok(None) } Ok(PyIterReturn::StopIteration(_)) => { - // Pop iterator from stack: - self.pop_value(); - - // End of for loop - self.jump(target); + // Check if target instruction is END_FOR (CPython 3.14 pattern) + // If so, skip it and jump to target + 1 instruction (POP_ITER) + let target_idx = target.0 as usize; + let jump_target = if let Some(unit) = self.code.instructions.get(target_idx) { + if matches!(unit.op, bytecode::Instruction::EndFor) + && matches!( + self.code.instructions.get(target_idx + 1).map(|u| &u.op), + Some(bytecode::Instruction::PopIter) + ) + { + // Skip END_FOR, jump to POP_ITER + bytecode::Label(target.0 + 1) + } else { + // Legacy pattern: jump directly to target (POP_TOP/POP_ITER) + target + } + } else { + target + }; + self.jump(jump_target); Ok(None) } Err(next_error) => { - // Pop iterator from stack: + // On error, pop iterator and propagate self.pop_value(); Err(next_error) } @@ -2109,12 +2955,12 @@ impl ExecutingFrame<'_> { vm: &VirtualMachine, attr: bytecode::MakeFunctionFlags, ) -> FrameResult { - // CPython 3.13 style: SET_FUNCTION_ATTRIBUTE sets attributes on a function + // SET_FUNCTION_ATTRIBUTE sets attributes on a function // Stack: [..., attr_value, func] -> [..., func] // Stack order: func is at -1, attr_value is at -2 - let func = self.pop_value(); - let attr_value = self.replace_top(func); + let func = self.pop_value_opt(); + let attr_value = expect_unchecked(self.replace_top(func), "attr_value must not be null"); let func = self.top_value(); // Get the function reference and call the new method @@ -2166,32 +3012,13 @@ impl ExecutingFrame<'_> { bytecode::BinaryOperator::InplaceXor => vm._ixor(a_ref, b_ref), bytecode::BinaryOperator::InplaceOr => vm._ior(a_ref, b_ref), bytecode::BinaryOperator::InplaceAnd => vm._iand(a_ref, b_ref), + bytecode::BinaryOperator::Subscr => a_ref.get_item(b_ref.as_object(), vm), }?; self.push_value(value); Ok(None) } - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_unary_op( - &mut self, - vm: &VirtualMachine, - op: bytecode::UnaryOperator, - ) -> FrameResult { - let a = self.pop_value(); - let value = match op { - bytecode::UnaryOperator::Minus => vm._neg(&a)?, - bytecode::UnaryOperator::Plus => vm._pos(&a)?, - bytecode::UnaryOperator::Invert => vm._invert(&a)?, - bytecode::UnaryOperator::Not => { - let value = a.try_to_bool(vm)?; - vm.ctx.new_bool(!value).into() - } - }; - self.push_value(value); - Ok(None) - } - #[cold] fn setup_annotations(&mut self, vm: &VirtualMachine) -> FrameResult { let __annotations__ = identifier!(vm, __annotations__); @@ -2216,10 +3043,55 @@ impl ExecutingFrame<'_> { Ok(None) } + /// _PyEval_UnpackIterableStackRef fn unpack_sequence(&mut self, size: u32, vm: &VirtualMachine) -> FrameResult { let value = self.pop_value(); - let elements: Vec<_> = value.try_to_value(vm).map_err(|e| { - if e.class().is(vm.ctx.exceptions.type_error) { + let size = size as usize; + + // Fast path for exact tuple/list types (not subclasses) — check + // length directly without creating an iterator, matching + // UNPACK_SEQUENCE_TUPLE / UNPACK_SEQUENCE_LIST specializations. + let cls = value.class(); + let fast_elements: Option<Vec<PyObjectRef>> = if cls.is(vm.ctx.types.tuple_type) { + Some(value.downcast_ref::<PyTuple>().unwrap().as_slice().to_vec()) + } else if cls.is(vm.ctx.types.list_type) { + Some( + value + .downcast_ref::<PyList>() + .unwrap() + .borrow_vec() + .to_vec(), + ) + } else { + None + }; + if let Some(elements) = fast_elements { + return match elements.len().cmp(&size) { + core::cmp::Ordering::Equal => { + self.state + .stack + .extend(elements.into_iter().rev().map(Some)); + Ok(None) + } + core::cmp::Ordering::Greater => Err(vm.new_value_error(format!( + "too many values to unpack (expected {size}, got {})", + elements.len() + ))), + core::cmp::Ordering::Less => Err(vm.new_value_error(format!( + "not enough values to unpack (expected {size}, got {})", + elements.len() + ))), + }; + } + + // General path — iterate up to `size + 1` elements to avoid + // consuming the entire iterator (fixes hang on infinite sequences). + let not_iterable = value.class().slots.iter.load().is_none() + && value + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + let iter = PyIter::try_from_object(vm, value.clone()).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { vm.new_type_error(format!( "cannot unpack non-iterable {} object", value.class().name() @@ -2228,21 +3100,48 @@ impl ExecutingFrame<'_> { e } })?; - let msg = match elements.len().cmp(&(size as usize)) { - std::cmp::Ordering::Equal => { - self.state.stack.extend(elements.into_iter().rev()); - return Ok(None); + + let mut elements = Vec::with_capacity(size); + for _ in 0..size { + match iter.next(vm)? { + PyIterReturn::Return(item) => elements.push(item), + PyIterReturn::StopIteration(_) => { + return Err(vm.new_value_error(format!( + "not enough values to unpack (expected {size}, got {})", + elements.len() + ))); + } + } + } + + // Check that the iterator is exhausted. + match iter.next(vm)? { + PyIterReturn::Return(_) => { + // For exact dict types, show "got N" using the container's + // size (PyDict_Size). Exact tuple/list are handled by the + // fast path above and never reach here. + let msg = if value.class().is(vm.ctx.types.dict_type) { + if let Ok(got) = value.length(vm) { + if got > size { + format!("too many values to unpack (expected {size}, got {got})") + } else { + format!("too many values to unpack (expected {size})") + } + } else { + format!("too many values to unpack (expected {size})") + } + } else { + format!("too many values to unpack (expected {size})") + }; + Err(vm.new_value_error(msg)) } - std::cmp::Ordering::Greater => { - format!("too many values to unpack (expected {size})") + PyIterReturn::StopIteration(_) => { + self.state + .stack + .extend(elements.into_iter().rev().map(Some)); + Ok(None) } - std::cmp::Ordering::Less => format!( - "not enough values to unpack (expected {}, got {})", - size, - elements.len() - ), - }; - Err(vm.new_value_error(msg)) + } } fn convert_value( @@ -2291,11 +3190,66 @@ impl ExecutingFrame<'_> { Ok(None) } - fn load_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { - let attr_name = self.code.names[attr as usize]; + fn load_attr(&mut self, vm: &VirtualMachine, oparg: LoadAttr) -> FrameResult { + let attr_name = self.code.names[oparg.name_idx() as usize]; let parent = self.pop_value(); - let obj = parent.get_attr(attr_name, vm)?; - self.push_value(obj); + + if oparg.is_method() { + // Method call: push [method, self_or_null] + let method = PyMethod::get(parent.clone(), attr_name, vm)?; + match method { + PyMethod::Function { target: _, func } => { + self.push_value(func); + self.push_value(parent); + } + PyMethod::Attribute(val) => { + self.push_value(val); + self.push_null(); + } + } + } else { + // Regular attribute access + let obj = parent.get_attr(attr_name, vm)?; + self.push_value(obj); + } + Ok(None) + } + + fn load_super_attr(&mut self, vm: &VirtualMachine, oparg: LoadSuperAttr) -> FrameResult { + let attr_name = self.code.names[oparg.name_idx() as usize]; + + // Stack layout (bottom to top): [super, class, self] + // Pop in LIFO order: self, class, super + let self_obj = self.pop_value(); + let class = self.pop_value(); + let global_super = self.pop_value(); + + // Create super object - pass args based on has_class flag + // When super is shadowed, has_class=false means call with 0 args + let super_obj = if oparg.has_class() { + global_super.call((class.clone(), self_obj.clone()), vm)? + } else { + global_super.call((), vm)? + }; + + if oparg.is_load_method() { + // Method load: push [method, self_or_null] + let method = PyMethod::get(super_obj, attr_name, vm)?; + match method { + PyMethod::Function { target: _, func } => { + self.push_value(func); + self.push_value(self_obj); + } + PyMethod::Attribute(val) => { + self.push_value(val); + self.push_null(); + } + } + } else { + // Regular attribute access + let obj = super_obj.get_attr(attr_name, vm)?; + self.push_value(obj); + } Ok(None) } @@ -2314,69 +3268,44 @@ impl ExecutingFrame<'_> { Ok(None) } - fn push_block(&mut self, typ: BlockType) { - // eprintln!("block pushed: {:.60?} {}", typ, self.state.stack.len()); - self.state.blocks.push(Block { - typ, - level: self.state.stack.len(), - }); + // Block stack functions removed - exception table handles all exception/cleanup + + #[inline] + #[track_caller] // not a real track_caller but push_value is less useful for debugging + fn push_value_opt(&mut self, obj: Option<PyObjectRef>) { + match self.state.stack.try_push(obj) { + Ok(()) => {} + Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stackdepth"), + } } + #[inline] #[track_caller] - fn pop_block(&mut self) -> Block { - let block = self.state.blocks.pop().expect("No more blocks to pop!"); - // eprintln!( - // "block popped: {:.60?} {} -> {} ", - // block.typ, - // self.state.stack.len(), - // block.level - // ); - #[cfg(debug_assertions)] - if self.state.stack.len() < block.level { - dbg!(&self); - panic!( - "stack size reversion: current size({}) < truncates target({}).", - self.state.stack.len(), - block.level - ); - } - self.state.stack.truncate(block.level); - block + fn push_value(&mut self, obj: PyObjectRef) { + self.push_value_opt(Some(obj)); } #[inline] - fn current_block(&self) -> Option<Block> { - self.state.blocks.last().cloned() + fn push_null(&mut self) { + self.push_value_opt(None); } + /// Pop a value from the stack, returning None if the stack slot is NULL #[inline] - #[track_caller] // not a real track_caller but push_value is not very useful - fn push_value(&mut self, obj: PyObjectRef) { - // eprintln!( - // "push_value {} / len: {} +1", - // obj.class().name(), - // self.state.stack.len() - // ); - match self.state.stack.try_push(obj) { - Ok(()) => {} - Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stackdepth"), + fn pop_value_opt(&mut self) -> Option<PyObjectRef> { + match self.state.stack.pop() { + Some(slot) => slot, // slot is Option<PyObjectRef> + None => self.fatal("tried to pop from empty stack"), } } #[inline] - #[track_caller] // not a real track_caller but pop_value is not very useful + #[track_caller] fn pop_value(&mut self) -> PyObjectRef { - match self.state.stack.pop() { - Some(x) => { - // eprintln!( - // "pop_value {} / len: {}", - // x.class().name(), - // self.state.stack.len() - // ); - x - } - None => self.fatal("tried to pop value but there was nothing on the stack"), - } + expect_unchecked( + self.pop_value_opt(), + "pop value but null found. This is a compiler bug.", + ) } fn call_intrinsic_1( @@ -2399,6 +3328,7 @@ impl ExecutingFrame<'_> { self.import_star(vm)?; Ok(vm.ctx.none()) } + bytecode::IntrinsicFunction1::UnaryPositive => vm._pos(&arg), bytecode::IntrinsicFunction1::SubscriptGeneric => { // Used for PEP 695: Generic[*type_params] crate::builtins::genericalias::subscript_generic(arg, vm) @@ -2437,7 +3367,7 @@ impl ExecutingFrame<'_> { let name = tuple.as_slice()[0].clone(); let type_params_obj = tuple.as_slice()[1].clone(); - let value = tuple.as_slice()[2].clone(); + let compute_value = tuple.as_slice()[2].clone(); let type_params: PyTupleRef = if vm.is_none(&type_params_obj) { vm.ctx.empty_tuple.clone() @@ -2450,7 +3380,7 @@ impl ExecutingFrame<'_> { let name = name.downcast::<crate::builtins::PyStr>().map_err(|_| { vm.new_type_error("TypeAliasType name must be a string".to_owned()) })?; - let type_alias = typing::TypeAliasType::new(name, type_params, value); + let type_alias = typing::TypeAliasType::new(name, type_params, compute_value); Ok(type_alias.into_ref(&vm.ctx).into()) } bytecode::IntrinsicFunction1::ListToTuple => { @@ -2460,6 +3390,30 @@ impl ExecutingFrame<'_> { .map_err(|_| vm.new_type_error("LIST_TO_TUPLE expects a list"))?; Ok(vm.ctx.new_tuple(list.borrow_vec().to_vec()).into()) } + bytecode::IntrinsicFunction1::StopIterationError => { + // Convert StopIteration to RuntimeError + // Used to ensure async generators don't raise StopIteration directly + // _PyGen_FetchStopIterationValue + // Use fast_isinstance to handle subclasses of StopIteration + if arg.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + Err(vm.new_runtime_error("coroutine raised StopIteration")) + } else { + // If not StopIteration, just re-raise the original exception + Err(arg.downcast().unwrap_or_else(|obj| { + vm.new_runtime_error(format!( + "unexpected exception type: {:?}", + obj.class() + )) + })) + } + } + bytecode::IntrinsicFunction1::AsyncGenWrap => { + // Wrap value for async generator + // Creates an AsyncGenWrappedValue + Ok(crate::builtins::asyncgenerator::PyAsyncGenWrappedValue(arg) + .into_ref(&vm.ctx) + .into()) + } } } @@ -2494,26 +3448,51 @@ impl ExecutingFrame<'_> { .into(); Ok(type_var) } + bytecode::IntrinsicFunction2::PrepReraiseStar => { + // arg1 = orig (original exception) + // arg2 = excs (list of exceptions raised/reraised in except* blocks) + // Returns: exception to reraise, or None if nothing to reraise + crate::exceptions::prep_reraise_star(arg1, arg2, vm) + } } } - fn pop_multiple(&mut self, count: usize) -> crate::common::boxvec::Drain<'_, PyObjectRef> { + /// Pop multiple values from the stack. Panics if any slot is NULL. + fn pop_multiple(&mut self, count: usize) -> impl ExactSizeIterator<Item = PyObjectRef> + '_ { let stack_len = self.state.stack.len(); - self.state.stack.drain(stack_len - count..) + if count > stack_len { + let instr = self.code.instructions.get(self.lasti() as usize); + let op_name = instr + .map(|i| format!("{:?}", i.op)) + .unwrap_or_else(|| "None".to_string()); + panic!( + "Stack underflow in pop_multiple: trying to pop {} elements from stack with {} elements. lasti={}, code={}, op={}, source_path={}", + count, + stack_len, + self.lasti(), + self.code.obj_name, + op_name, + self.code.source_path() + ); + } + self.state.stack.drain(stack_len - count..).map(|obj| { + expect_unchecked(obj, "pop_multiple but null found. This is a compiler bug.") + }) } #[inline] - fn replace_top(&mut self, mut top: PyObjectRef) -> PyObjectRef { + fn replace_top(&mut self, mut top: Option<PyObjectRef>) -> Option<PyObjectRef> { let last = self.state.stack.last_mut().unwrap(); - std::mem::swap(&mut top, last); + core::mem::swap(last, &mut top); top } #[inline] - #[track_caller] // not a real track_caller but top_value is not very useful + #[track_caller] fn top_value(&self) -> &PyObject { match &*self.state.stack { - [.., last] => last, + [.., Some(last)] => last, + [.., None] => self.fatal("tried to get top of stack but got NULL"), [] => self.fatal("tried to get top of stack but stack is empty"), } } @@ -2522,7 +3501,10 @@ impl ExecutingFrame<'_> { #[track_caller] fn nth_value(&self, depth: u32) -> &PyObject { let stack = &self.state.stack; - &stack[stack.len() - depth as usize - 1] + match &stack[stack.len() - depth as usize - 1] { + Some(obj) => obj, + None => unsafe { core::hint::unreachable_unchecked() }, + } } #[cold] @@ -2537,30 +3519,103 @@ impl ExecutingFrame<'_> { impl fmt::Debug for Frame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = self.state.lock(); - let stack_str = state.stack.iter().fold(String::new(), |mut s, elem| { - if elem.downcastable::<Self>() { - s.push_str("\n > {frame}"); - } else { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); + let stack_str = state.stack.iter().fold(String::new(), |mut s, slot| { + match slot { + Some(elem) if elem.downcastable::<Self>() => { + s.push_str("\n > {frame}"); + } + Some(elem) => { + core::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); + } + None => { + s.push_str("\n > NULL"); + } } s }); - let block_str = state.blocks.iter().fold(String::new(), |mut s, elem| { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); - s - }); // TODO: fix this up let locals = self.locals.clone(); write!( f, - "Frame Object {{ \n Stack:{}\n Blocks:{}\n Locals:{:?}\n}}", + "Frame Object {{ \n Stack:{}\n Locals:{:?}\n}}", stack_str, - block_str, locals.into_object() ) } } +/// _PyEval_SpecialMethodCanSuggest +fn special_method_can_suggest( + obj: &PyObjectRef, + oparg: SpecialMethod, + vm: &VirtualMachine, +) -> PyResult<bool> { + Ok(match oparg { + SpecialMethod::Enter | SpecialMethod::Exit => { + vm.get_special_method(obj, get_special_method_name(SpecialMethod::AEnter, vm))? + .is_some() + && vm + .get_special_method(obj, get_special_method_name(SpecialMethod::AExit, vm))? + .is_some() + } + SpecialMethod::AEnter | SpecialMethod::AExit => { + vm.get_special_method(obj, get_special_method_name(SpecialMethod::Enter, vm))? + .is_some() + && vm + .get_special_method(obj, get_special_method_name(SpecialMethod::Exit, vm))? + .is_some() + } + }) +} + +fn get_special_method_name(oparg: SpecialMethod, vm: &VirtualMachine) -> &'static PyStrInterned { + match oparg { + SpecialMethod::Enter => identifier!(vm, __enter__), + SpecialMethod::Exit => identifier!(vm, __exit__), + SpecialMethod::AEnter => identifier!(vm, __aenter__), + SpecialMethod::AExit => identifier!(vm, __aexit__), + } +} + +/// _Py_SpecialMethod _Py_SpecialMethods +fn get_special_method_error_msg( + oparg: SpecialMethod, + class_name: &str, + can_suggest: bool, +) -> String { + if can_suggest { + match oparg { + SpecialMethod::Enter => format!( + "'{class_name}' object does not support the context manager protocol (missed __enter__ method) but it supports the asynchronous context manager protocol. Did you mean to use 'async with'?" + ), + SpecialMethod::Exit => format!( + "'{class_name}' object does not support the context manager protocol (missed __exit__ method) but it supports the asynchronous context manager protocol. Did you mean to use 'async with'?" + ), + SpecialMethod::AEnter => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aenter__ method) but it supports the context manager protocol. Did you mean to use 'with'?" + ), + SpecialMethod::AExit => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aexit__ method) but it supports the context manager protocol. Did you mean to use 'with'?" + ), + } + } else { + match oparg { + SpecialMethod::Enter => format!( + "'{class_name}' object does not support the context manager protocol (missed __enter__ method)" + ), + SpecialMethod::Exit => format!( + "'{class_name}' object does not support the context manager protocol (missed __exit__ method)" + ), + SpecialMethod::AEnter => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aenter__ method)" + ), + SpecialMethod::AExit => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aexit__ method)" + ), + } + } +} + fn is_module_initializing(module: &PyObject, vm: &VirtualMachine) -> bool { let Ok(spec) = module.get_attr(&vm.ctx.new_str("__spec__"), vm) else { return false; @@ -2573,3 +3628,11 @@ fn is_module_initializing(module: &PyObject, vm: &VirtualMachine) -> bool { }; initializing_attr.try_to_bool(vm).unwrap_or(false) } + +fn expect_unchecked(optional: Option<PyObjectRef>, err_msg: &'static str) -> PyObjectRef { + if cfg!(debug_assertions) { + optional.expect(err_msg) + } else { + unsafe { optional.unwrap_unchecked() } + } +} diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index d657ff6be8f..a4877cf4042 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -4,9 +4,9 @@ use crate::{ convert::ToPyObject, object::{Traverse, TraverseFn}, }; +use core::ops::RangeInclusive; use indexmap::IndexMap; use itertools::Itertools; -use std::ops::RangeInclusive; pub trait IntoFuncArgs: Sized { fn into_args(self, vm: &VirtualMachine) -> FuncArgs; @@ -100,7 +100,7 @@ impl From<KwArgs> for FuncArgs { impl FromArgs for FuncArgs { fn from_args(_vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - Ok(std::mem::take(args)) + Ok(core::mem::take(args)) } } @@ -424,7 +424,7 @@ impl<T> PosArgs<T> { self.0 } - pub fn iter(&self) -> std::slice::Iter<'_, T> { + pub fn iter(&self) -> core::slice::Iter<'_, T> { self.0.iter() } } @@ -469,7 +469,7 @@ where impl<T> IntoIterator for PosArgs<T> { type Item = T; - type IntoIter = std::vec::IntoIter<T>; + type IntoIter = alloc::vec::IntoIter<T>; fn into_iter(self) -> Self::IntoIter { self.0.into_iter() diff --git a/crates/vm/src/function/builtin.rs b/crates/vm/src/function/builtin.rs index 1a91e4344ba..444df64a8ef 100644 --- a/crates/vm/src/function/builtin.rs +++ b/crates/vm/src/function/builtin.rs @@ -3,7 +3,7 @@ use crate::{ Py, PyPayload, PyRef, PyResult, VirtualMachine, convert::ToPyResult, object::PyThreadingConstraint, }; -use std::marker::PhantomData; +use core::marker::PhantomData; /// A built-in Python function. // PyCFunction in CPython @@ -54,14 +54,14 @@ const fn zst_ref_out_of_thin_air<T: 'static>(x: T) -> &'static T { // if T is zero-sized, there's no issue forgetting it - even if it does have a Drop impl, it // would never get called anyway if we consider this semantically a Box::leak(Box::new(x))-type // operation. if T isn't zero-sized, we don't have to worry about it because we'll fail to compile. - std::mem::forget(x); + core::mem::forget(x); const { - if std::mem::size_of::<T>() != 0 { + if core::mem::size_of::<T>() != 0 { panic!("can't use a non-zero-sized type here") } // SAFETY: we just confirmed that T is zero-sized, so we can // pull a value of it out of thin air. - unsafe { std::ptr::NonNull::<T>::dangling().as_ref() } + unsafe { core::ptr::NonNull::<T>::dangling().as_ref() } } } @@ -218,7 +218,7 @@ into_py_native_fn_tuple!( #[cfg(test)] mod tests { use super::*; - use std::mem::size_of_val; + use core::mem::size_of_val; #[test] fn test_into_native_fn_noalloc() { diff --git a/crates/vm/src/function/either.rs b/crates/vm/src/function/either.rs index 8700c6150db..9ee7f028bd2 100644 --- a/crates/vm/src/function/either.rs +++ b/crates/vm/src/function/either.rs @@ -1,7 +1,7 @@ use crate::{ AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, convert::ToPyObject, }; -use std::borrow::Borrow; +use core::borrow::Borrow; pub enum Either<A, B> { A(A), diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 2bc331844c6..7d3a0dcbbd5 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -5,8 +5,10 @@ use crate::{ function::PyStr, protocol::PyBuffer, }; -use std::{borrow::Cow, ffi::OsStr, path::PathBuf}; +use alloc::borrow::Cow; +use std::{ffi::OsStr, path::PathBuf}; +/// Helper to implement os.fspath() #[derive(Clone)] pub enum FsPath { Str(PyStrRef), @@ -27,7 +29,7 @@ impl FsPath { ) } - // PyOS_FSPath in CPython + // PyOS_FSPath pub fn try_from( obj: PyObjectRef, check_for_nul: bool, @@ -110,8 +112,8 @@ impl FsPath { Ok(path) } - pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) + pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) } #[cfg(windows)] diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs index 5e109176c5e..52624cbbf86 100644 --- a/crates/vm/src/function/method.rs +++ b/crates/vm/src/function/method.rs @@ -188,7 +188,7 @@ impl PyMethodDef { ) -> PyRef<PyNativeMethod> { PyRef::new_ref( self.to_bound_method(obj, class), - ctx.types.builtin_method_type.to_owned(), + ctx.types.builtin_function_or_method_type.to_owned(), None, ) } @@ -211,7 +211,13 @@ impl PyMethodDef { class: &'static Py<PyType>, ) -> PyRef<PyNativeMethod> { debug_assert!(self.flags.contains(PyMethodFlags::STATIC)); - let func = self.to_function(); + // Set zelf to the class, matching CPython's m_self = type for static methods. + // Callable::call skips prepending when STATIC flag is set. + let func = PyNativeFunction { + zelf: Some(class.to_owned().into()), + value: self, + module: None, + }; PyNativeMethod { func, class }.into_ref(ctx) } @@ -251,14 +257,14 @@ impl PyMethodDef { } } -impl std::fmt::Debug for PyMethodDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMethodDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyMethodDef") .field("name", &self.name) .field( "func", &(unsafe { - std::mem::transmute::<&dyn PyNativeFn, [usize; 2]>(self.func)[1] as *const u8 + core::mem::transmute::<&dyn PyNativeFn, [usize; 2]>(self.func)[1] as *const u8 }), ) .field("flags", &self.flags) diff --git a/crates/vm/src/function/number.rs b/crates/vm/src/function/number.rs index 7bb37b8f549..b53208bcd93 100644 --- a/crates/vm/src/function/number.rs +++ b/crates/vm/src/function/number.rs @@ -1,9 +1,9 @@ use super::argument::OptionalArg; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyIntRef}; +use core::ops::Deref; use malachite_bigint::BigInt; use num_complex::Complex64; use num_traits::PrimInt; -use std::ops::Deref; /// A Python complex-like object. /// @@ -20,17 +20,16 @@ pub struct ArgIntoComplex { value: Complex64, } -impl From<ArgIntoComplex> for Complex64 { - fn from(arg: ArgIntoComplex) -> Self { - arg.value +impl ArgIntoComplex { + #[inline] + pub fn into_complex(self) -> Complex64 { + self.value } } -impl Deref for ArgIntoComplex { - type Target = Complex64; - - fn deref(&self) -> &Self::Target { - &self.value +impl From<ArgIntoComplex> for Complex64 { + fn from(arg: ArgIntoComplex) -> Self { + arg.value } } @@ -60,9 +59,14 @@ pub struct ArgIntoFloat { } impl ArgIntoFloat { + #[inline] + pub fn into_float(self) -> f64 { + self.value + } + pub fn vec_into_f64(v: Vec<Self>) -> Vec<f64> { // TODO: Vec::into_raw_parts once stabilized - let mut v = std::mem::ManuallyDrop::new(v); + let mut v = core::mem::ManuallyDrop::new(v); let (p, l, c) = (v.as_mut_ptr(), v.len(), v.capacity()); // SAFETY: IntoPyFloat is repr(transparent) over f64 unsafe { Vec::from_raw_parts(p.cast(), l, c) } @@ -75,13 +79,6 @@ impl From<ArgIntoFloat> for f64 { } } -impl Deref for ArgIntoFloat { - type Target = f64; - fn deref(&self) -> &Self::Target { - &self.value - } -} - impl TryFromObject for ArgIntoFloat { // Equivalent to PyFloat_AsDouble. fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { @@ -106,6 +103,11 @@ pub struct ArgIntoBool { impl ArgIntoBool { pub const TRUE: Self = Self { value: true }; pub const FALSE: Self = Self { value: false }; + + #[inline] + pub fn into_bool(self) -> bool { + self.value + } } impl From<ArgIntoBool> for bool { @@ -114,13 +116,6 @@ impl From<ArgIntoBool> for bool { } } -impl Deref for ArgIntoBool { - type Target = bool; - fn deref(&self) -> &Self::Target { - &self.value - } -} - impl TryFromObject for ArgIntoBool { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { Ok(Self { @@ -136,20 +131,25 @@ pub struct ArgIndex { value: PyIntRef, } -impl From<ArgIndex> for PyIntRef { - fn from(arg: ArgIndex) -> Self { - arg.value +impl ArgIndex { + #[inline] + pub fn into_int_ref(self) -> PyIntRef { + self.value } } -impl Deref for ArgIndex { - type Target = PyIntRef; - - fn deref(&self) -> &Self::Target { +impl AsRef<PyIntRef> for ArgIndex { + fn as_ref(&self) -> &PyIntRef { &self.value } } +impl From<ArgIndex> for PyIntRef { + fn from(arg: ArgIndex) -> Self { + arg.value + } +} + impl TryFromObject for ArgIndex { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { Ok(Self { diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index 1e670b96389..402f6d0365b 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -1,13 +1,13 @@ use super::IntoFuncArgs; use crate::{ AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, - builtins::{PyDict, PyDictRef, iter::PySequenceIterator}, + builtins::{PyDictRef, iter::PySequenceIterator}, convert::ToPyObject, object::{Traverse, TraverseFn}, - protocol::{PyIter, PyIterIter, PyMapping, PyMappingMethods}, - types::{AsMapping, GenericMethod}, + protocol::{PyIter, PyIterIter, PyMapping}, + types::GenericMethod, }; -use std::{borrow::Borrow, marker::PhantomData, ops::Deref}; +use core::{borrow::Borrow, marker::PhantomData}; #[derive(Clone, Traverse)] pub struct ArgCallable { @@ -24,8 +24,8 @@ impl ArgCallable { } } -impl std::fmt::Debug for ArgCallable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for ArgCallable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("ArgCallable") .field("obj", &self.obj) .field("call", &format!("{:08x}", self.call as usize)) @@ -104,14 +104,11 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let iter_fn = { - let cls = obj.class(); - let iter_fn = cls.mro_find_map(|x| x.slots.iter.load()); - if iter_fn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { - return Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))); - } - iter_fn - }; + let cls = obj.class(); + let iter_fn = cls.slots.iter.load(); + if iter_fn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { + return Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))); + } Ok(Self { iterable: obj, iter_fn, @@ -123,30 +120,27 @@ where #[derive(Debug, Clone, Traverse)] pub struct ArgMapping { obj: PyObjectRef, - #[pytraverse(skip)] - methods: &'static PyMappingMethods, } impl ArgMapping { #[inline] - pub const fn with_methods(obj: PyObjectRef, methods: &'static PyMappingMethods) -> Self { - Self { obj, methods } + pub const fn new(obj: PyObjectRef) -> Self { + Self { obj } } #[inline(always)] pub fn from_dict_exact(dict: PyDictRef) -> Self { - Self { - obj: dict.into(), - methods: PyDict::as_mapping(), - } + Self { obj: dict.into() } + } + + #[inline(always)] + pub fn obj(&self) -> &PyObject { + &self.obj } #[inline(always)] pub fn mapping(&self) -> PyMapping<'_> { - PyMapping { - obj: &self.obj, - methods: self.methods, - } + self.obj.mapping_unchecked() } } @@ -164,14 +158,6 @@ impl AsRef<PyObject> for ArgMapping { } } -impl Deref for ArgMapping { - type Target = PyObject; - #[inline(always)] - fn deref(&self) -> &PyObject { - &self.obj - } -} - impl From<ArgMapping> for PyObjectRef { #[inline(always)] fn from(value: ArgMapping) -> Self { @@ -188,9 +174,8 @@ impl ToPyObject for ArgMapping { impl TryFromObject for ArgMapping { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let mapping = PyMapping::try_protocol(&obj, vm)?; - let methods = mapping.methods; - Ok(Self { obj, methods }) + let _mapping = obj.try_mapping(vm)?; + Ok(Self { obj }) } } @@ -215,7 +200,7 @@ impl<T> ArgSequence<T> { } } -impl<T> std::ops::Deref for ArgSequence<T> { +impl<T> core::ops::Deref for ArgSequence<T> { type Target = [T]; #[inline(always)] fn deref(&self) -> &[T] { @@ -225,14 +210,14 @@ impl<T> std::ops::Deref for ArgSequence<T> { impl<'a, T> IntoIterator for &'a ArgSequence<T> { type Item = &'a T; - type IntoIter = std::slice::Iter<'a, T>; + type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { self.iter() } } impl<T> IntoIterator for ArgSequence<T> { type Item = T; - type IntoIter = std::vec::IntoIter<T>; + type IntoIter = alloc::vec::IntoIter<T>; fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs new file mode 100644 index 00000000000..87dd1152d9c --- /dev/null +++ b/crates/vm/src/gc_state.rs @@ -0,0 +1,506 @@ +//! Garbage Collection State and Algorithm +//! +//! This module implements CPython-compatible generational garbage collection +//! for RustPython, using an intrusive doubly-linked list approach. + +use crate::common::lock::PyMutex; +use crate::{PyObject, PyObjectRef}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::collections::HashSet; +use std::sync::{Mutex, RwLock}; + +bitflags::bitflags! { + /// GC debug flags (see Include/internal/pycore_gc.h) + #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] + pub struct GcDebugFlags: u32 { + /// Print collection statistics + const STATS = 1 << 0; + /// Print collectable objects + const COLLECTABLE = 1 << 1; + /// Print uncollectable objects + const UNCOLLECTABLE = 1 << 2; + /// Save all garbage in gc.garbage + const SAVEALL = 1 << 5; + /// DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | DEBUG_SAVEALL + const LEAK = Self::COLLECTABLE.bits() | Self::UNCOLLECTABLE.bits() | Self::SAVEALL.bits(); + } +} + +/// Statistics for a single generation (gc_generation_stats) +#[derive(Debug, Default, Clone, Copy)] +pub struct GcStats { + pub collections: usize, + pub collected: usize, + pub uncollectable: usize, +} + +/// A single GC generation with intrusive linked list +pub struct GcGeneration { + /// Number of objects in this generation + count: AtomicUsize, + /// Threshold for triggering collection + threshold: AtomicU32, + /// Collection statistics + stats: PyMutex<GcStats>, +} + +impl GcGeneration { + pub const fn new(threshold: u32) -> Self { + Self { + count: AtomicUsize::new(0), + threshold: AtomicU32::new(threshold), + stats: PyMutex::new(GcStats { + collections: 0, + collected: 0, + uncollectable: 0, + }), + } + } + + pub fn count(&self) -> usize { + self.count.load(Ordering::SeqCst) + } + + pub fn threshold(&self) -> u32 { + self.threshold.load(Ordering::SeqCst) + } + + pub fn set_threshold(&self, value: u32) { + self.threshold.store(value, Ordering::SeqCst); + } + + pub fn stats(&self) -> GcStats { + let guard = self.stats.lock(); + GcStats { + collections: guard.collections, + collected: guard.collected, + uncollectable: guard.uncollectable, + } + } + + pub fn update_stats(&self, collected: usize, uncollectable: usize) { + let mut guard = self.stats.lock(); + guard.collections += 1; + guard.collected += collected; + guard.uncollectable += uncollectable; + } +} + +/// Wrapper for raw pointer to make it Send + Sync +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct GcObjectPtr(NonNull<PyObject>); + +// SAFETY: We only use this for tracking objects, and proper synchronization is used +unsafe impl Send for GcObjectPtr {} +unsafe impl Sync for GcObjectPtr {} + +/// Global GC state +pub struct GcState { + /// 3 generations (0 = youngest, 2 = oldest) + pub generations: [GcGeneration; 3], + /// Permanent generation (frozen objects) + pub permanent: GcGeneration, + /// GC enabled flag + pub enabled: AtomicBool, + /// Per-generation object tracking (for correct gc_refs algorithm) + /// Objects start in gen0, survivors move to gen1, then gen2 + generation_objects: [RwLock<HashSet<GcObjectPtr>>; 3], + /// Frozen/permanent objects (excluded from normal GC) + permanent_objects: RwLock<HashSet<GcObjectPtr>>, + /// Debug flags + pub debug: AtomicU32, + /// gc.garbage list (uncollectable objects with __del__) + pub garbage: PyMutex<Vec<PyObjectRef>>, + /// gc.callbacks list + pub callbacks: PyMutex<Vec<PyObjectRef>>, + /// Mutex for collection (prevents concurrent collections). + /// Used by collect_inner when the actual collection algorithm is enabled. + #[allow(dead_code)] + collecting: Mutex<()>, + /// Allocation counter for gen0 + alloc_count: AtomicUsize, + /// Registry of all tracked objects (for cycle detection) + tracked_objects: RwLock<HashSet<GcObjectPtr>>, + /// Objects that have been finalized (__del__ already called) + /// Prevents calling __del__ multiple times on resurrected objects + finalized_objects: RwLock<HashSet<GcObjectPtr>>, +} + +// SAFETY: All fields are either inherently Send/Sync (atomics, RwLock, Mutex) or protected by PyMutex. +// PyMutex<Vec<PyObjectRef>> is safe to share/send across threads because access is synchronized. +// PyObjectRef itself is Send, and interior mutability is guarded by the mutex. +unsafe impl Send for GcState {} +unsafe impl Sync for GcState {} + +impl Default for GcState { + fn default() -> Self { + Self::new() + } +} + +impl GcState { + pub fn new() -> Self { + Self { + generations: [ + GcGeneration::new(2000), // young + GcGeneration::new(10), // old[0] + GcGeneration::new(0), // old[1] + ], + permanent: GcGeneration::new(0), + enabled: AtomicBool::new(true), + generation_objects: [ + RwLock::new(HashSet::new()), + RwLock::new(HashSet::new()), + RwLock::new(HashSet::new()), + ], + permanent_objects: RwLock::new(HashSet::new()), + debug: AtomicU32::new(0), + garbage: PyMutex::new(Vec::new()), + callbacks: PyMutex::new(Vec::new()), + collecting: Mutex::new(()), + alloc_count: AtomicUsize::new(0), + tracked_objects: RwLock::new(HashSet::new()), + finalized_objects: RwLock::new(HashSet::new()), + } + } + + /// Check if GC is enabled + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Enable GC + pub fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + /// Disable GC + pub fn disable(&self) { + self.enabled.store(false, Ordering::SeqCst); + } + + /// Get debug flags + pub fn get_debug(&self) -> GcDebugFlags { + GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::SeqCst)) + } + + /// Set debug flags + pub fn set_debug(&self, flags: GcDebugFlags) { + self.debug.store(flags.bits(), Ordering::SeqCst); + } + + /// Get thresholds for all generations + pub fn get_threshold(&self) -> (u32, u32, u32) { + ( + self.generations[0].threshold(), + self.generations[1].threshold(), + self.generations[2].threshold(), + ) + } + + /// Set thresholds + pub fn set_threshold(&self, t0: u32, t1: Option<u32>, t2: Option<u32>) { + self.generations[0].set_threshold(t0); + if let Some(t1) = t1 { + self.generations[1].set_threshold(t1); + } + if let Some(t2) = t2 { + self.generations[2].set_threshold(t2); + } + } + + /// Get counts for all generations + pub fn get_count(&self) -> (usize, usize, usize) { + ( + self.generations[0].count(), + self.generations[1].count(), + self.generations[2].count(), + ) + } + + /// Get statistics for all generations + pub fn get_stats(&self) -> [GcStats; 3] { + [ + self.generations[0].stats(), + self.generations[1].stats(), + self.generations[2].stats(), + ] + } + + /// Track a new object (add to gen0) + /// Called when IS_TRACE objects are created + /// + /// # Safety + /// obj must be a valid pointer to a PyObject + pub unsafe fn track_object(&self, obj: NonNull<PyObject>) { + let gc_ptr = GcObjectPtr(obj); + + // _PyObject_GC_TRACK + let obj_ref = unsafe { obj.as_ref() }; + obj_ref.set_gc_tracked(); + + // Add to generation 0 tracking first (for correct gc_refs algorithm) + // Only increment count if we successfully add to the set + if let Ok(mut gen0) = self.generation_objects[0].write() + && gen0.insert(gc_ptr) + { + self.generations[0].count.fetch_add(1, Ordering::SeqCst); + self.alloc_count.fetch_add(1, Ordering::SeqCst); + } + + // Also add to global tracking (for get_objects, etc.) + if let Ok(mut tracked) = self.tracked_objects.write() { + tracked.insert(gc_ptr); + } + } + + /// Untrack an object (remove from GC lists) + /// Called when objects are deallocated + /// + /// # Safety + /// obj must be a valid pointer to a PyObject + pub unsafe fn untrack_object(&self, obj: NonNull<PyObject>) { + let gc_ptr = GcObjectPtr(obj); + + // Remove from generation tracking lists and decrement the correct generation's count + for (gen_idx, generation) in self.generation_objects.iter().enumerate() { + if let Ok(mut gen_set) = generation.write() + && gen_set.remove(&gc_ptr) + { + // Decrement count for the generation we removed from + let count = self.generations[gen_idx].count.load(Ordering::SeqCst); + if count > 0 { + self.generations[gen_idx] + .count + .fetch_sub(1, Ordering::SeqCst); + } + break; // Object can only be in one generation + } + } + + // Remove from global tracking + if let Ok(mut tracked) = self.tracked_objects.write() { + tracked.remove(&gc_ptr); + } + + // Remove from permanent tracking + if let Ok(mut permanent) = self.permanent_objects.write() + && permanent.remove(&gc_ptr) + { + let count = self.permanent.count.load(Ordering::SeqCst); + if count > 0 { + self.permanent.count.fetch_sub(1, Ordering::SeqCst); + } + } + + // Remove from finalized set + if let Ok(mut finalized) = self.finalized_objects.write() { + finalized.remove(&gc_ptr); + } + } + + /// Check if an object has been finalized + pub fn is_finalized(&self, obj: NonNull<PyObject>) -> bool { + let gc_ptr = GcObjectPtr(obj); + if let Ok(finalized) = self.finalized_objects.read() { + finalized.contains(&gc_ptr) + } else { + false + } + } + + /// Mark an object as finalized + pub fn mark_finalized(&self, obj: NonNull<PyObject>) { + let gc_ptr = GcObjectPtr(obj); + if let Ok(mut finalized) = self.finalized_objects.write() { + finalized.insert(gc_ptr); + } + } + + /// Get tracked objects (for gc.get_objects) + /// If generation is None, returns all tracked objects. + /// If generation is Some(n), returns objects in generation n only. + pub fn get_objects(&self, generation: Option<i32>) -> Vec<PyObjectRef> { + match generation { + None => { + // Return all tracked objects + if let Ok(tracked) = self.tracked_objects.read() { + tracked + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + Some(obj.to_owned()) + } else { + None + } + }) + .collect() + } else { + Vec::new() + } + } + Some(g) if (0..=2).contains(&g) => { + // Return objects in specific generation + let gen_idx = g as usize; + if let Ok(gen_set) = self.generation_objects[gen_idx].read() { + gen_set + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + Some(obj.to_owned()) + } else { + None + } + }) + .collect() + } else { + Vec::new() + } + } + _ => Vec::new(), + } + } + + /// Check if automatic GC should run and run it if needed. + /// Called after object allocation. + /// Returns true if GC was run, false otherwise. + pub fn maybe_collect(&self) -> bool { + if !self.is_enabled() { + return false; + } + + // _PyObject_GC_Alloc checks thresholds + + // Check gen0 threshold + let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; + let threshold0 = self.generations[0].threshold(); + if threshold0 > 0 && count0 >= threshold0 { + self.collect(0); + return true; + } + + false + } + + /// Perform garbage collection on the given generation. + /// Returns (collected_count, uncollectable_count). + /// + /// Currently a stub — the actual collection algorithm requires EBR + /// and will be added in a follow-up. + pub fn collect(&self, _generation: usize) -> (usize, usize) { + // gc_collect_main + // Reset gen0 count even though we're not actually collecting + self.generations[0].count.store(0, Ordering::SeqCst); + (0, 0) + } + + /// Force collection even if GC is disabled (for manual gc.collect() calls). + /// gc.collect() always runs regardless of gc.isenabled() + /// Currently a stub. + pub fn collect_force(&self, _generation: usize) -> (usize, usize) { + // Reset gen0 count even though we're not actually collecting + self.generations[0].count.store(0, Ordering::SeqCst); + (0, 0) + } + + /// Get count of frozen objects + pub fn get_freeze_count(&self) -> usize { + self.permanent.count() + } + + /// Freeze all tracked objects (move to permanent generation) + pub fn freeze(&self) { + // Move all objects from gen0-2 to permanent + let mut objects_to_freeze: Vec<GcObjectPtr> = Vec::new(); + + for (gen_idx, generation) in self.generation_objects.iter().enumerate() { + if let Ok(mut gen_set) = generation.write() { + objects_to_freeze.extend(gen_set.drain()); + self.generations[gen_idx].count.store(0, Ordering::SeqCst); + } + } + + // Add to permanent set + if let Ok(mut permanent) = self.permanent_objects.write() { + let count = objects_to_freeze.len(); + for ptr in objects_to_freeze { + permanent.insert(ptr); + } + self.permanent.count.fetch_add(count, Ordering::SeqCst); + } + } + + /// Unfreeze all objects (move from permanent to gen2) + pub fn unfreeze(&self) { + let mut objects_to_unfreeze: Vec<GcObjectPtr> = Vec::new(); + + if let Ok(mut permanent) = self.permanent_objects.write() { + objects_to_unfreeze.extend(permanent.drain()); + self.permanent.count.store(0, Ordering::SeqCst); + } + + // Add to generation 2 + if let Ok(mut gen2) = self.generation_objects[2].write() { + let count = objects_to_unfreeze.len(); + for ptr in objects_to_unfreeze { + gen2.insert(ptr); + } + self.generations[2].count.fetch_add(count, Ordering::SeqCst); + } + } +} + +use std::sync::OnceLock; + +/// Global GC state instance +/// Using a static because GC needs to be accessible from object allocation/deallocation +static GC_STATE: OnceLock<GcState> = OnceLock::new(); + +/// Get a reference to the global GC state +pub fn gc_state() -> &'static GcState { + GC_STATE.get_or_init(GcState::new) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gc_state_default() { + let state = GcState::new(); + assert!(state.is_enabled()); + assert_eq!(state.get_debug(), GcDebugFlags::empty()); + assert_eq!(state.get_threshold(), (2000, 10, 0)); + assert_eq!(state.get_count(), (0, 0, 0)); + } + + #[test] + fn test_gc_enable_disable() { + let state = GcState::new(); + assert!(state.is_enabled()); + state.disable(); + assert!(!state.is_enabled()); + state.enable(); + assert!(state.is_enabled()); + } + + #[test] + fn test_gc_threshold() { + let state = GcState::new(); + state.set_threshold(100, Some(20), Some(30)); + assert_eq!(state.get_threshold(), (100, 20, 30)); + } + + #[test] + fn test_gc_debug_flags() { + let state = GcState::new(); + state.set_debug(GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE); + assert_eq!( + state.get_debug(), + GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE + ); + } +} diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs new file mode 100644 index 00000000000..31fa0617b45 --- /dev/null +++ b/crates/vm/src/getpath.rs @@ -0,0 +1,411 @@ +//! Path configuration for RustPython (ref: Modules/getpath.py) +//! +//! This module implements Python path calculation logic following getpath.py. +//! It uses landmark-based search to locate prefix, exec_prefix, and stdlib directories. +//! +//! The main entry point is `init_path_config()` which computes Paths from Settings. + +use crate::vm::{Paths, Settings}; +use std::env; +use std::path::{Path, PathBuf}; + +// Platform-specific landmarks (ref: getpath.py PLATFORM CONSTANTS) + +#[cfg(not(windows))] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules/Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib/os.py"; + + pub fn stdlib_subdir() -> String { + format!("lib/python{}.{}", version::MAJOR, version::MINOR) + } + + pub fn stdlib_landmarks() -> [String; 2] { + let subdir = stdlib_subdir(); + [format!("{}/os.py", subdir), format!("{}/os.pyc", subdir)] + } + + pub fn platstdlib_landmark() -> String { + format!( + "lib/python{}.{}/lib-dynload", + version::MAJOR, + version::MINOR + ) + } + + pub fn zip_landmark() -> String { + format!("lib/python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +#[cfg(windows)] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules\\Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib\\os.py"; + pub const STDLIB_SUBDIR: &str = "Lib"; + + pub fn stdlib_landmarks() -> [String; 2] { + ["Lib\\os.py".into(), "Lib\\os.pyc".into()] + } + + pub fn platstdlib_landmark() -> String { + "DLLs".into() + } + + pub fn zip_landmark() -> String { + format!("python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +// Helper functions (ref: getpath.py HELPER FUNCTIONS) + +/// Search upward from a directory for landmark files/directories +/// Returns the directory where a landmark was found +fn search_up<P, F>(start: P, landmarks: &[&str], test: F) -> Option<PathBuf> +where + P: AsRef<Path>, + F: Fn(&Path) -> bool, +{ + let mut current = start.as_ref().to_path_buf(); + loop { + for landmark in landmarks { + let path = current.join(landmark); + if test(&path) { + return Some(current); + } + } + if !current.pop() { + return None; + } + } +} + +/// Search upward for a file landmark +fn search_up_file<P: AsRef<Path>>(start: P, landmarks: &[&str]) -> Option<PathBuf> { + search_up(start, landmarks, |p| p.is_file()) +} + +/// Search upward for a directory landmark +#[cfg(not(windows))] +fn search_up_dir<P: AsRef<Path>>(start: P, landmarks: &[&str]) -> Option<PathBuf> { + search_up(start, landmarks, |p| p.is_dir()) +} + +// Path computation functions + +/// Compute path configuration from Settings +/// +/// This function should be called before interpreter initialization. +/// It returns a Paths struct with all computed path values. +pub fn init_path_config(settings: &Settings) -> Paths { + let mut paths = Paths::default(); + + // Step 0: Get executable path + let executable = get_executable_path(); + let real_executable = executable + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + + // Step 1: Check for __PYVENV_LAUNCHER__ environment variable + // When launched from a venv launcher, __PYVENV_LAUNCHER__ contains the venv's python.exe path + // In this case: + // - sys.executable should be the launcher path (where user invoked Python) + // - sys._base_executable should be the real Python executable + let exe_dir = if let Ok(launcher) = env::var("__PYVENV_LAUNCHER__") { + paths.executable = launcher.clone(); + paths.base_executable = real_executable; + PathBuf::from(&launcher).parent().map(PathBuf::from) + } else { + paths.executable = real_executable; + executable + .as_ref() + .and_then(|p| p.parent().map(PathBuf::from)) + }; + + // Step 2: Check for venv (pyvenv.cfg) and get 'home' + let (venv_prefix, home_dir) = detect_venv(&exe_dir); + let search_dir = home_dir.clone().or(exe_dir.clone()); + + // Step 3: Check for build directory + let build_prefix = detect_build_directory(&search_dir); + + // Step 4: Calculate prefix via landmark search + // When in venv, search_dir is home_dir, so this gives us the base Python's prefix + let calculated_prefix = calculate_prefix(&search_dir, &build_prefix); + + // Step 5: Set prefix and base_prefix + if venv_prefix.is_some() { + // In venv: prefix = venv directory, base_prefix = original Python's prefix + paths.prefix = venv_prefix + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|| calculated_prefix.clone()); + paths.base_prefix = calculated_prefix; + } else { + // Not in venv: prefix == base_prefix + paths.prefix = calculated_prefix.clone(); + paths.base_prefix = calculated_prefix; + } + + // Step 6: Calculate exec_prefix + paths.exec_prefix = if venv_prefix.is_some() { + // In venv: exec_prefix = prefix (venv directory) + paths.prefix.clone() + } else { + calculate_exec_prefix(&search_dir, &paths.prefix) + }; + paths.base_exec_prefix = paths.base_prefix.clone(); + + // Step 7: Calculate base_executable (if not already set by __PYVENV_LAUNCHER__) + if paths.base_executable.is_empty() { + paths.base_executable = calculate_base_executable(executable.as_ref(), &home_dir); + } + + // Step 8: Build module_search_paths + paths.module_search_paths = + build_module_search_paths(settings, &paths.prefix, &paths.exec_prefix); + + // Step 9: Calculate stdlib_dir + paths.stdlib_dir = calculate_stdlib_dir(&paths.prefix); + + paths +} + +/// Get default prefix value +fn default_prefix() -> String { + std::option_env!("RUSTPYTHON_PREFIX") + .map(String::from) + .unwrap_or_else(|| { + if cfg!(windows) { + "C:".to_owned() + } else { + "/usr/local".to_owned() + } + }) +} + +/// Detect virtual environment by looking for pyvenv.cfg +/// Returns (venv_prefix, home_dir from pyvenv.cfg) +fn detect_venv(exe_dir: &Option<PathBuf>) -> (Option<PathBuf>, Option<PathBuf>) { + // Try exe_dir/../pyvenv.cfg first (standard venv layout: venv/bin/python) + if let Some(dir) = exe_dir + && let Some(venv_dir) = dir.parent() + { + let cfg = venv_dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(venv_dir.to_path_buf()), Some(PathBuf::from(home))); + } + } + + // Try exe_dir/pyvenv.cfg (alternative layout) + if let Some(dir) = exe_dir { + let cfg = dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(dir.clone()), Some(PathBuf::from(home))); + } + } + + (None, None) +} + +/// Detect if running from a build directory +fn detect_build_directory(exe_dir: &Option<PathBuf>) -> Option<PathBuf> { + let dir = exe_dir.as_ref()?; + + // Check for pybuilddir.txt (indicates build directory) + if dir.join(platform::BUILDDIR_TXT).exists() { + return Some(dir.clone()); + } + + // Check for Modules/Setup.local (build landmark) + if dir.join(platform::BUILD_LANDMARK).exists() { + return Some(dir.clone()); + } + + // Search up for Lib/os.py (build stdlib landmark) + search_up_file(dir, &[platform::BUILDSTDLIB_LANDMARK]) +} + +/// Calculate prefix by searching for landmarks +fn calculate_prefix(exe_dir: &Option<PathBuf>, build_prefix: &Option<PathBuf>) -> String { + // 1. If build directory detected, use it + if let Some(bp) = build_prefix { + return bp.to_string_lossy().into_owned(); + } + + if let Some(dir) = exe_dir { + // 2. Search for ZIP landmark + let zip = platform::zip_landmark(); + if let Some(prefix) = search_up_file(dir, &[&zip]) { + return prefix.to_string_lossy().into_owned(); + } + + // 3. Search for stdlib landmarks (os.py) + let landmarks = platform::stdlib_landmarks(); + let refs: Vec<&str> = landmarks.iter().map(|s| s.as_str()).collect(); + if let Some(prefix) = search_up_file(dir, &refs) { + return prefix.to_string_lossy().into_owned(); + } + } + + // 4. Fallback to default + default_prefix() +} + +/// Calculate exec_prefix +fn calculate_exec_prefix(exe_dir: &Option<PathBuf>, prefix: &str) -> String { + #[cfg(windows)] + { + // Windows: exec_prefix == prefix + let _ = exe_dir; // silence unused warning + prefix.to_owned() + } + + #[cfg(not(windows))] + { + // POSIX: search for lib-dynload directory + if let Some(dir) = exe_dir { + let landmark = platform::platstdlib_landmark(); + if let Some(exec_prefix) = search_up_dir(dir, &[&landmark]) { + return exec_prefix.to_string_lossy().into_owned(); + } + } + // Fallback: same as prefix + prefix.to_owned() + } +} + +/// Calculate base_executable +fn calculate_base_executable(executable: Option<&PathBuf>, home_dir: &Option<PathBuf>) -> String { + // If in venv and we have home, construct base_executable from home + if let (Some(exe), Some(home)) = (executable, home_dir) + && let Some(exe_name) = exe.file_name() + { + let base = home.join(exe_name); + return base.to_string_lossy().into_owned(); + } + + // Otherwise, base_executable == executable + executable + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Calculate stdlib_dir (sys._stdlib_dir) +/// Returns None if the stdlib directory doesn't exist +fn calculate_stdlib_dir(prefix: &str) -> Option<String> { + #[cfg(not(windows))] + let stdlib_dir = PathBuf::from(prefix).join(platform::stdlib_subdir()); + + #[cfg(windows)] + let stdlib_dir = PathBuf::from(prefix).join(platform::STDLIB_SUBDIR); + + if stdlib_dir.is_dir() { + Some(stdlib_dir.to_string_lossy().into_owned()) + } else { + None + } +} + +/// Build the complete module_search_paths (sys.path) +fn build_module_search_paths(settings: &Settings, prefix: &str, exec_prefix: &str) -> Vec<String> { + let mut paths = Vec::new(); + + // 1. PYTHONPATH/RUSTPYTHONPATH from settings + paths.extend(settings.path_list.iter().cloned()); + + // 2. ZIP file path + let zip_path = PathBuf::from(prefix).join(platform::zip_landmark()); + paths.push(zip_path.to_string_lossy().into_owned()); + + // 3. stdlib and platstdlib directories + #[cfg(not(windows))] + { + // POSIX: stdlib first, then lib-dynload + let stdlib_dir = PathBuf::from(prefix).join(platform::stdlib_subdir()); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + } + + #[cfg(windows)] + { + // Windows: DLLs first, then Lib + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + + let stdlib_dir = PathBuf::from(prefix).join(platform::STDLIB_SUBDIR); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + } + + paths +} + +/// Get the current executable path +fn get_executable_path() -> Option<PathBuf> { + #[cfg(not(target_arch = "wasm32"))] + { + let exec_arg = env::args_os().next()?; + which::which(exec_arg).ok() + } + #[cfg(target_arch = "wasm32")] + { + let exec_arg = env::args().next()?; + Some(PathBuf::from(exec_arg)) + } +} + +/// Parse pyvenv.cfg and extract the 'home' key value +fn parse_pyvenv_home(pyvenv_cfg: &Path) -> Option<String> { + let content = std::fs::read_to_string(pyvenv_cfg).ok()?; + + for line in content.lines() { + if let Some((key, value)) = line.split_once('=') + && key.trim().to_lowercase() == "home" + { + return Some(value.trim().to_string()); + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_init_path_config() { + let settings = Settings::default(); + let paths = init_path_config(&settings); + // Just verify it doesn't panic and returns valid paths + assert!(!paths.prefix.is_empty()); + } + + #[test] + fn test_search_up() { + // Test with a path that doesn't have any landmarks + let result = search_up_file(std::env::temp_dir(), &["nonexistent_landmark_xyz"]); + assert!(result.is_none()); + } + + #[test] + fn test_default_prefix() { + let prefix = default_prefix(); + assert!(!prefix.is_empty()); + } +} diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index c119405fe1d..1957ccca663 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -1,13 +1,17 @@ //! Import mechanics use crate::{ - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - builtins::{PyBaseExceptionRef, PyCode, list, traceback::PyTraceback}, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, + builtins::{PyCode, PyStr, PyStrRef, traceback::PyTraceback}, + exceptions::types::PyBaseException, scope::Scope, - version::get_git_revision, - vm::{VirtualMachine, thread}, + vm::{VirtualMachine, resolve_frozen_alias, thread}, }; +pub(crate) fn check_pyc_magic_number_bytes(buf: &[u8]) -> bool { + buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES[..2]) +} + pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult<PyObjectRef> { flame_guard!("init importlib"); @@ -26,15 +30,18 @@ pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult<PyObjectR Ok(bootstrap) })?; vm.import_func = importlib.get_attr(identifier!(vm, __import__), vm)?; + vm.importlib = importlib.clone(); Ok(importlib) } +#[cfg(feature = "host_env")] pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef) -> PyResult<()> { + use crate::{TryFromObject, builtins::PyListRef}; + thread::enter_vm(vm, || { flame_guard!("install_external"); // same deal as imports above - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] import_builtin(vm, crate::stdlib::os::MODULE_NAME)?; #[cfg(windows)] import_builtin(vm, "winreg")?; @@ -43,21 +50,11 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef let install_external = importlib.get_attr("_install_external_importers", vm)?; install_external.call((), vm)?; - // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable. - let importlib_external = vm.import("_frozen_importlib_external", 0)?; - let mut magic = get_git_revision().into_bytes(); - magic.truncate(4); - if magic.len() != 4 { - // os_random is expensive, but this is only ever called once - magic = rustpython_common::rand::os_random::<4>().to_vec(); - } - let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); - importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; let zipimport_res = (|| -> PyResult<()> { let zipimport = vm.import("zipimport", 0)?; let zipimporter = zipimport.get_attr("zipimporter", vm)?; let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; - let path_hooks = list::PyListRef::try_from_object(vm, path_hooks)?; + let path_hooks = PyListRef::try_from_object(vm, path_hooks)?; path_hooks.insert(0, zipimporter); Ok(()) })(); @@ -79,25 +76,50 @@ pub fn make_frozen(vm: &VirtualMachine, name: &str) -> PyResult<PyRef<PyCode>> { } pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { - let frozen = make_frozen(vm, module_name)?; - let module = import_code_obj(vm, module_name, frozen, false)?; + let frozen = vm.state.frozen.get(module_name).ok_or_else(|| { + vm.new_import_error( + format!("No such frozen object named {module_name}"), + vm.ctx.new_str(module_name), + ) + })?; + let module = import_code_obj(vm, module_name, vm.ctx.new_code(frozen.code), false)?; debug_assert!(module.get_attr(identifier!(vm, __name__), vm).is_ok()); - // TODO: give a correct origname here - module.set_attr("__origname__", vm.ctx.new_str(module_name.to_owned()), vm)?; + let origname = resolve_frozen_alias(module_name); + module.set_attr("__origname__", vm.ctx.new_str(origname), vm)?; Ok(module) } pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { - let make_module_func = vm.state.module_inits.get(module_name).ok_or_else(|| { - vm.new_import_error( - format!("Cannot import builtin module {module_name}"), - vm.ctx.new_str(module_name), - ) - })?; - let module = make_module_func(vm); let sys_modules = vm.sys_module.get_attr("modules", vm)?; - sys_modules.set_item(module_name, module.as_object().to_owned(), vm)?; - Ok(module.into()) + + // Check if already in sys.modules (handles recursive imports) + if let Ok(module) = sys_modules.get_item(module_name, vm) { + return Ok(module); + } + + // Try multi-phase init first (preferred for modules that import other modules) + if let Some(&def) = vm.state.module_defs.get(module_name) { + // Phase 1: Create and initialize module + let module = def.create_module(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(module_name, module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + // If exec fails, remove the partially-initialized module from sys.modules + if let Err(e) = def.exec_module(vm, &module) { + let _ = sys_modules.del_item(module_name, vm); + return Err(e); + } + + return Ok(module.into()); + } + + // Module not found in module_defs + Err(vm.new_import_error( + format!("Cannot import builtin module {module_name}"), + vm.ctx.new_str(module_name), + )) } #[cfg(feature = "rustpython-compiler")] @@ -131,6 +153,27 @@ pub fn import_source(vm: &VirtualMachine, module_name: &str, content: &str) -> P import_code_obj(vm, module_name, code, false) } +/// If `__spec__._initializing` is true, wait for the module to finish +/// initializing by calling `_lock_unlock_module`. +fn import_ensure_initialized( + module: &PyObjectRef, + name: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + let initializing = match vm.get_attribute_opt(module.clone(), vm.ctx.intern_str("__spec__"))? { + Some(spec) => match vm.get_attribute_opt(spec, vm.ctx.intern_str("_initializing"))? { + Some(v) => v.try_to_bool(vm)?, + None => false, + }, + None => false, + }; + if initializing { + let lock_unlock = vm.importlib.get_attr("_lock_unlock_module", vm)?; + lock_unlock.call((vm.ctx.new_str(name),), vm)?; + } + Ok(()) +} + pub fn import_code_obj( vm: &VirtualMachine, module_name: &str, @@ -146,7 +189,7 @@ pub fn import_code_obj( if set_file_attr { attrs.set_item( identifier!(vm, __file__), - code_obj.source_path.to_object(), + code_obj.source_path().to_object(), vm, )?; } @@ -173,7 +216,7 @@ fn remove_importlib_frames_inner( return (None, false); }; - let file_name = traceback.frame.code.source_path.as_str(); + let file_name = traceback.frame.code.source_path().as_str(); let (inner_tb, mut now_in_importlib) = remove_importlib_frames_inner(vm, traceback.next.lock().clone(), always_trim); @@ -204,8 +247,8 @@ fn remove_importlib_frames_inner( // TODO: This function should do nothing on verbose mode. // TODO: Fix this function after making PyTraceback.next mutable -pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyBaseExceptionRef) { - if vm.state.settings.verbose != 0 { +pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &Py<PyBaseException>) { + if vm.state.config.settings.verbose != 0 { return; } @@ -216,3 +259,354 @@ pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyBaseExceptionRef) { exc.set_traceback_typed(trimmed_tb); } } + +/// Get origin path from a module spec, checking has_location first. +pub(crate) fn get_spec_file_origin( + spec: &Option<PyObjectRef>, + vm: &VirtualMachine, +) -> Option<String> { + let spec = spec.as_ref()?; + let has_location = spec + .get_attr("has_location", vm) + .ok() + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false); + if !has_location { + return None; + } + spec.get_attr("origin", vm).ok().and_then(|origin| { + if vm.is_none(&origin) { + None + } else { + origin + .downcast_ref::<PyStr>() + .and_then(|s| s.to_str().map(|s| s.to_owned())) + } + }) +} + +/// Check if a module file possibly shadows another module of the same name. +/// Compares the module's directory with the original sys.path[0] (derived from sys.argv[0]). +pub(crate) fn is_possibly_shadowing_path(origin: &str, vm: &VirtualMachine) -> bool { + use std::path::Path; + + if vm.state.config.settings.safe_path { + return false; + } + + let origin_path = Path::new(origin); + let parent = match origin_path.parent() { + Some(p) => p, + None => return false, + }; + // For packages (__init__.py), look one directory further up + let root = if origin_path.file_name() == Some("__init__.py".as_ref()) { + parent.parent().unwrap_or(Path::new("")) + } else { + parent + }; + + // Compute original sys.path[0] from sys.argv[0] (the script path). + // See: config->sys_path_0, which is set once + // at initialization and never changes even if sys.path is modified. + let sys_path_0 = (|| -> Option<String> { + let argv = vm.sys_module.get_attr("argv", vm).ok()?; + let argv0 = argv.get_item(&0usize, vm).ok()?; + let argv0_str = argv0.downcast_ref::<PyStr>()?; + let s = argv0_str.as_str(); + + // For -c and REPL, original sys.path[0] is "" + if s == "-c" || s.is_empty() { + return Some(String::new()); + } + // For scripts, original sys.path[0] is dirname(argv[0]) + Some( + Path::new(s) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or("") + .to_owned(), + ) + })(); + + let sys_path_0 = match sys_path_0 { + Some(p) => p, + None => return false, + }; + + let cmp_path = if sys_path_0.is_empty() { + match std::env::current_dir() { + Ok(d) => d.to_string_lossy().to_string(), + Err(_) => return false, + } + } else { + sys_path_0 + }; + + root.to_str() == Some(cmp_path.as_str()) +} + +/// Check if a module name is in sys.stdlib_module_names. +/// Takes the original __name__ object to preserve str subclass behavior. +/// Propagates errors (e.g. TypeError for unhashable str subclass). +pub(crate) fn is_stdlib_module_name(name: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + let stdlib_names = match vm.sys_module.get_attr("stdlib_module_names", vm) { + Ok(names) => names, + Err(_) => return Ok(false), + }; + if !stdlib_names.class().fast_issubclass(vm.ctx.types.set_type) + && !stdlib_names + .class() + .fast_issubclass(vm.ctx.types.frozenset_type) + { + return Ok(false); + } + let result = vm.call_method(&stdlib_names, "__contains__", (name.clone(),))?; + result.try_to_bool(vm) +} + +/// PyImport_ImportModuleLevelObject +pub(crate) fn import_module_level( + name: &Py<PyStr>, + globals: Option<PyObjectRef>, + fromlist: Option<PyObjectRef>, + level: i32, + vm: &VirtualMachine, +) -> PyResult { + if level < 0 { + return Err(vm.new_value_error("level must be >= 0".to_owned())); + } + + let name_str = match name.to_str() { + Some(s) => s, + None => { + // Name contains surrogates. Like CPython, try sys.modules + // lookup with the Python string key directly. + if level == 0 { + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + return sys_modules.get_item(name, vm).map_err(|_| { + vm.new_import_error(format!("No module named '{}'", name), name.to_owned()) + }); + } + return Err(vm.new_import_error(format!("No module named '{}'", name), name.to_owned())); + } + }; + + // Resolve absolute name + let abs_name = if level > 0 { + // When globals is not provided (Rust None), raise KeyError + // matching resolve_name() where globals==NULL + if globals.is_none() { + return Err(vm.new_key_error(vm.ctx.new_str("'__name__' not in globals").into())); + } + let globals_ref = globals.as_ref().unwrap(); + // When globals is Python None, treat like empty mapping + let empty_dict_obj; + let globals_ref = if vm.is_none(globals_ref) { + empty_dict_obj = vm.ctx.new_dict().into(); + &empty_dict_obj + } else { + globals_ref + }; + let package = calc_package(Some(globals_ref), vm)?; + if package.is_empty() { + return Err(vm.new_import_error( + "attempted relative import with no known parent package".to_owned(), + vm.ctx.new_str(""), + )); + } + resolve_name(name_str, &package, level as usize, vm)? + } else { + if name_str.is_empty() { + return Err(vm.new_value_error("Empty module name".to_owned())); + } + name_str.to_owned() + }; + + // import_get_module + import_find_and_load + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let module = match sys_modules.get_item(&*abs_name, vm) { + Ok(m) if !vm.is_none(&m) => { + import_ensure_initialized(&m, &abs_name, vm)?; + m + } + _ => { + let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; + let abs_name_obj = vm.ctx.new_str(&*abs_name); + find_and_load.call((abs_name_obj, vm.import_func.clone()), vm)? + } + }; + + // Handle fromlist + let has_from = match fromlist.as_ref().filter(|fl| !vm.is_none(fl)) { + Some(fl) => fl.clone().try_to_bool(vm)?, + None => false, + }; + + if has_from { + let fromlist = fromlist.unwrap(); + // Only call _handle_fromlist if the module looks like a package + // (has __path__). Non-module objects without __name__/__path__ would + // crash inside _handle_fromlist; IMPORT_FROM handles per-attribute + // errors with proper ImportError conversion. + let has_path = vm + .get_attribute_opt(module.clone(), vm.ctx.intern_str("__path__"))? + .is_some(); + if has_path { + let handle_fromlist = vm.importlib.get_attr("_handle_fromlist", vm)?; + handle_fromlist.call((module, fromlist, vm.import_func.clone()), vm) + } else { + Ok(module) + } + } else if level == 0 || !name_str.is_empty() { + match name_str.find('.') { + None => Ok(module), + Some(dot) => { + let to_return = if level == 0 { + name_str[..dot].to_owned() + } else { + let cut_off = name_str.len() - dot; + abs_name[..abs_name.len() - cut_off].to_owned() + }; + match sys_modules.get_item(&*to_return, vm) { + Ok(m) => Ok(m), + Err(_) if level == 0 => { + // For absolute imports (level 0), try importing the + // parent. Matches _bootstrap.__import__ behavior. + let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; + let to_return_obj = vm.ctx.new_str(&*to_return); + find_and_load.call((to_return_obj, vm.import_func.clone()), vm) + } + Err(_) => { + // For relative imports (level > 0), raise KeyError + let to_return_obj: PyObjectRef = vm + .ctx + .new_str(format!("'{to_return}' not in sys.modules as expected")) + .into(); + Err(vm.new_key_error(to_return_obj)) + } + } + } + } + } else { + Ok(module) + } +} + +/// resolve_name in import.c - resolve relative import name +fn resolve_name(name: &str, package: &str, level: usize, vm: &VirtualMachine) -> PyResult<String> { + // Python: bits = package.rsplit('.', level - 1) + // Rust: rsplitn(level, '.') gives maxsplit=level-1 + let parts: Vec<&str> = package.rsplitn(level, '.').collect(); + if parts.len() < level { + return Err(vm.new_import_error( + "attempted relative import beyond top-level package".to_owned(), + vm.ctx.new_str(name), + )); + } + // rsplitn returns parts right-to-left, so last() is the leftmost (base) + let base = parts.last().unwrap(); + if name.is_empty() { + Ok(base.to_string()) + } else { + Ok(format!("{base}.{name}")) + } +} + +/// _calc___package__ - calculate package from globals for relative imports +fn calc_package(globals: Option<&PyObjectRef>, vm: &VirtualMachine) -> PyResult<String> { + let globals = globals.ok_or_else(|| { + vm.new_import_error( + "attempted relative import with no known parent package".to_owned(), + vm.ctx.new_str(""), + ) + })?; + + let package = globals.get_item("__package__", vm).ok(); + let spec = globals.get_item("__spec__", vm).ok(); + + if let Some(ref pkg) = package + && !vm.is_none(pkg) + { + let pkg_str: PyStrRef = pkg + .clone() + .downcast() + .map_err(|_| vm.new_type_error("package must be a string".to_owned()))?; + // Warn if __package__ != __spec__.parent + if let Some(ref spec) = spec + && !vm.is_none(spec) + && let Ok(parent) = spec.get_attr("parent", vm) + && !pkg_str.is(&parent) + && pkg_str + .as_object() + .rich_compare_bool(&parent, crate::types::PyComparisonOp::Ne, vm) + .unwrap_or(false) + { + let parent_repr = parent + .repr(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_default(); + let msg = format!( + "__package__ != __spec__.parent ('{}' != {})", + pkg_str.as_str(), + parent_repr + ); + let warn = vm + .import("_warnings", 0) + .and_then(|w| w.get_attr("warn", vm)); + if let Ok(warn_fn) = warn { + let _ = warn_fn.call( + ( + vm.ctx.new_str(msg), + vm.ctx.exceptions.deprecation_warning.to_owned(), + ), + vm, + ); + } + } + return Ok(pkg_str.as_str().to_owned()); + } else if let Some(ref spec) = spec + && !vm.is_none(spec) + && let Ok(parent) = spec.get_attr("parent", vm) + && !vm.is_none(&parent) + { + let parent_str: PyStrRef = parent + .downcast() + .map_err(|_| vm.new_type_error("package set to non-string".to_owned()))?; + return Ok(parent_str.as_str().to_owned()); + } + + // Fall back to __name__ and __path__ + let warn = vm + .import("_warnings", 0) + .and_then(|w| w.get_attr("warn", vm)); + if let Ok(warn_fn) = warn { + let _ = warn_fn.call( + ( + vm.ctx.new_str("can't resolve package from __spec__ or __package__, falling back on __name__ and __path__"), + vm.ctx.exceptions.import_warning.to_owned(), + ), + vm, + ); + } + + let mod_name = globals.get_item("__name__", vm).map_err(|_| { + vm.new_import_error( + "attempted relative import with no known parent package".to_owned(), + vm.ctx.new_str(""), + ) + })?; + let mod_name_str: PyStrRef = mod_name + .downcast() + .map_err(|_| vm.new_type_error("__name__ must be a string".to_owned()))?; + let mut package = mod_name_str.as_str().to_owned(); + // If not a package (no __path__), strip last component. + // Uses rpartition('.')[0] semantics: returns empty string when no dot. + if globals.get_item("__path__", vm).is_err() { + package = match package.rfind('.') { + Some(dot) => package[..dot].to_owned(), + None => String::new(), + }; + } + Ok(package) +} diff --git a/crates/vm/src/intern.rs b/crates/vm/src/intern.rs index a5b2a798d53..a50b8871cb9 100644 --- a/crates/vm/src/intern.rs +++ b/crates/vm/src/intern.rs @@ -6,10 +6,8 @@ use crate::{ common::lock::PyRwLock, convert::ToPyObject, }; -use std::{ - borrow::{Borrow, ToOwned}, - ops::Deref, -}; +use alloc::borrow::ToOwned; +use core::{borrow::Borrow, ops::Deref}; #[derive(Debug)] pub struct StringPool { @@ -86,8 +84,8 @@ pub struct CachedPyStrRef { inner: PyRefExact<PyStr>, } -impl std::hash::Hash for CachedPyStrRef { - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { +impl core::hash::Hash for CachedPyStrRef { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { self.inner.as_wtf8().hash(state) } } @@ -100,7 +98,7 @@ impl PartialEq for CachedPyStrRef { impl Eq for CachedPyStrRef {} -impl std::borrow::Borrow<Wtf8> for CachedPyStrRef { +impl core::borrow::Borrow<Wtf8> for CachedPyStrRef { #[inline] fn borrow(&self) -> &Wtf8 { self.as_wtf8() @@ -119,7 +117,7 @@ impl CachedPyStrRef { /// the given cache must be alive while returned reference is alive #[inline] const unsafe fn as_interned_str(&self) -> &'static PyStrInterned { - unsafe { std::mem::transmute_copy(self) } + unsafe { core::mem::transmute_copy(self) } } #[inline] @@ -135,7 +133,7 @@ pub struct PyInterned<T> { impl<T: PyPayload> PyInterned<T> { #[inline] pub fn leak(cache: PyRef<T>) -> &'static Self { - unsafe { std::mem::transmute(cache) } + unsafe { core::mem::transmute(cache) } } #[inline] @@ -163,9 +161,9 @@ impl<T: PyPayload> Borrow<PyObject> for PyInterned<T> { // NOTE: std::hash::Hash of Self and Self::Borrowed *must* be the same // This is ok only because PyObject doesn't implement Hash -impl<T: PyPayload> std::hash::Hash for PyInterned<T> { +impl<T: PyPayload> core::hash::Hash for PyInterned<T> { #[inline(always)] - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { self.get_id().hash(state) } } @@ -188,15 +186,15 @@ impl<T> Deref for PyInterned<T> { impl<T: PyPayload> PartialEq for PyInterned<T> { #[inline(always)] fn eq(&self, other: &Self) -> bool { - std::ptr::eq(self, other) + core::ptr::eq(self, other) } } impl<T: PyPayload> Eq for PyInterned<T> {} -impl<T: std::fmt::Debug + PyPayload> std::fmt::Debug for PyInterned<T> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Debug::fmt(&**self, f)?; +impl<T: core::fmt::Debug + PyPayload> core::fmt::Debug for PyInterned<T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(&**self, f)?; write!(f, "@{:p}", self.as_ptr()) } } @@ -308,7 +306,7 @@ impl MaybeInternedString for Py<PyStr> { #[inline(always)] fn as_interned(&self) -> Option<&'static PyStrInterned> { if self.as_object().is_interned() { - Some(unsafe { std::mem::transmute::<&Self, &PyInterned<PyStr>>(self) }) + Some(unsafe { core::mem::transmute::<&Self, &PyInterned<PyStr>>(self) }) } else { None } diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index 923b33d2acc..8a15688f591 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -24,6 +24,7 @@ extern crate bitflags; #[macro_use] extern crate log; // extern crate env_logger; +extern crate alloc; #[macro_use] extern crate rustpython_derive; @@ -60,12 +61,13 @@ pub mod exceptions; pub mod format; pub mod frame; pub mod function; +pub mod getpath; pub mod import; mod intern; pub mod iter; pub mod object; -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] +#[cfg(feature = "host_env")] pub mod ospath; pub mod prelude; @@ -75,6 +77,7 @@ pub mod py_io; #[cfg(feature = "serde")] pub mod py_serde; +pub mod gc_state; pub mod readline; pub mod recursion; pub mod scope; @@ -97,7 +100,7 @@ pub use self::object::{ AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, PyWeakRef, }; -pub use self::vm::{Context, Interpreter, Settings, VirtualMachine}; +pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine}; pub use rustpython_common as common; pub use rustpython_compiler_core::{bytecode, frozen}; diff --git a/crates/vm/src/macros.rs b/crates/vm/src/macros.rs index 1284c202782..32f3e4566ea 100644 --- a/crates/vm/src/macros.rs +++ b/crates/vm/src/macros.rs @@ -116,12 +116,12 @@ macro_rules! match_class { // The default arm, binding the original object to the specified identifier. (match ($obj:expr) { $binding:ident => $default:expr $(,)? }) => {{ - #[allow(clippy::redundant_locals)] + #[allow(clippy::redundant_locals, reason = "macro arm intentionally binds expression once to a local")] let $binding = $obj; $default }}; (match ($obj:expr) { ref $binding:ident => $default:expr $(,)? }) => {{ - #[allow(clippy::redundant_locals)] + #[allow(clippy::redundant_locals, reason = "macro arm intentionally binds expression once to a local reference")] let $binding = &$obj; $default }}; @@ -146,8 +146,8 @@ macro_rules! match_class { }; (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { match $obj.downcast_ref::<$class>() { - ::std::option::Option::Some($binding) => $expr, - ::std::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), + core::option::Option::Some($binding) => $expr, + core::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), } }; @@ -190,7 +190,7 @@ macro_rules! identifier( macro_rules! identifier_utf8( ($as_ctx:expr, $name:ident) => { // Safety: All known identifiers are ascii strings. - #[allow(clippy::macro_metavars_in_unsafe)] + #[allow(clippy::macro_metavars_in_unsafe, reason = "known identifiers are ASCII and downcast target is fixed")] unsafe { $as_ctx.as_ref().names.$name.as_object().downcast_unchecked_ref::<$crate::builtins::PyUtf8Str>() } }; ); diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index e04b87de594..4d61a208241 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -19,9 +19,9 @@ use crate::object::traverse_object::PyObjVTable; use crate::{ builtins::{PyDictRef, PyType, PyTypeRef}, common::{ - atomic::{OncePtr, PyAtomic, Radium}, - linked_list::{Link, LinkedList, Pointers}, - lock::{PyMutex, PyMutexGuard, PyRwLock}, + atomic::{Ordering, PyAtomic, Radium}, + linked_list::{Link, Pointers}, + lock::PyRwLock, refcount::RefCount, }, vm::VirtualMachine, @@ -31,11 +31,13 @@ use crate::{ object::traverse::{MaybeTraverse, Traverse, TraverseFn}, }; use itertools::Itertools; -use std::{ + +use alloc::fmt; + +use core::{ any::TypeId, borrow::Borrow, cell::UnsafeCell, - fmt, marker::PhantomData, mem::ManuallyDrop, ops::Deref, @@ -79,10 +81,30 @@ use std::{ #[derive(Debug)] pub(super) struct Erased; -pub(super) unsafe fn drop_dealloc_obj<T: PyPayload>(x: *mut PyObject) { - drop(unsafe { Box::from_raw(x as *mut PyInner<T>) }); +/// Default dealloc: handles __del__, weakref clearing, tp_clear, and memory free. +/// Equivalent to subtype_dealloc. +pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) { + let obj_ref = unsafe { &*(obj as *const PyObject) }; + if let Err(()) = obj_ref.drop_slow_inner() { + return; // resurrected by __del__ + } + + // Extract child references before deallocation to break circular refs (tp_clear). + // This ensures that when edges are dropped after the object is freed, + // any pointers back to this object are already gone. + let mut edges = Vec::new(); + if let Some(clear_fn) = obj_ref.0.vtable.clear { + unsafe { clear_fn(obj, &mut edges) }; + } + + // Deallocate the object memory + drop(unsafe { Box::from_raw(obj as *mut PyInner<T>) }); + + // Drop child references - may trigger recursive destruction. + // The object is already deallocated, so circular refs are broken. + drop(edges); } -pub(super) unsafe fn debug_obj<T: PyPayload + std::fmt::Debug>( +pub(super) unsafe fn debug_obj<T: PyPayload + core::fmt::Debug>( x: &PyObject, f: &mut fmt::Formatter<'_>, ) -> fmt::Result { @@ -91,21 +113,52 @@ pub(super) unsafe fn debug_obj<T: PyPayload + std::fmt::Debug>( } /// Call `try_trace` on payload -pub(super) unsafe fn try_trace_obj<T: PyPayload>(x: &PyObject, tracer_fn: &mut TraverseFn<'_>) { +pub(super) unsafe fn try_traverse_obj<T: PyPayload>(x: &PyObject, tracer_fn: &mut TraverseFn<'_>) { let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) }; let payload = &x.payload; payload.try_traverse(tracer_fn) } +/// Call `try_clear` on payload to extract child references (tp_clear) +pub(super) unsafe fn try_clear_obj<T: PyPayload>(x: *mut PyObject, out: &mut Vec<PyObjectRef>) { + let x = unsafe { &mut *(x as *mut PyInner<T>) }; + x.payload.try_clear(out); +} + +bitflags::bitflags! { + /// GC bits for free-threading support (like ob_gc_bits in Py_GIL_DISABLED) + /// These bits are stored in a separate atomic field for lock-free access. + /// See Include/internal/pycore_gc.h + #[derive(Copy, Clone, Debug, Default)] + pub(crate) struct GcBits: u8 { + /// Tracked by the GC + const TRACKED = 1 << 0; + /// tp_finalize was called (prevents __del__ from being called twice) + const FINALIZED = 1 << 1; + /// Object is unreachable (during GC collection) + const UNREACHABLE = 1 << 2; + /// Object is frozen (immutable) + const FROZEN = 1 << 3; + /// Memory the object references is shared between multiple threads + /// and needs special handling when freeing due to possible in-flight lock-free reads + const SHARED = 1 << 4; + /// Memory of the object itself is shared between multiple threads + /// Objects with this bit that are GC objects will automatically be delay-freed + const SHARED_INLINE = 1 << 5; + /// Use deferred reference counting + const DEFERRED = 1 << 6; + } +} + /// This is an actual python object. It consists of a `typ` which is the /// python class, and carries some rust payload optionally. This rust /// payload can be a rust float or rust int in case of float and int objects. #[repr(C)] pub(super) struct PyInner<T> { pub(super) ref_count: RefCount, - // TODO: move typeid into vtable once TypeId::of is const - pub(super) typeid: TypeId, pub(super) vtable: &'static PyObjVTable, + /// GC bits for free-threading (like ob_gc_bits) + pub(super) gc_bits: PyAtomic<u8>, pub(super) typ: PyAtomicRef<PyType>, // __class__ member pub(super) dict: Option<InstanceDict>, @@ -114,6 +167,7 @@ pub(super) struct PyInner<T> { pub(super) payload: T, } +pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>(); impl<T: fmt::Debug> fmt::Debug for PyInner<T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -137,8 +191,77 @@ unsafe impl Traverse for PyObject { } } +// === Stripe lock for weakref list protection (WEAKREF_LIST_LOCK) === + +#[cfg(feature = "threading")] +mod weakref_lock { + use core::sync::atomic::{AtomicU8, Ordering}; + + const NUM_WEAKREF_LOCKS: usize = 64; + + static LOCKS: [AtomicU8; NUM_WEAKREF_LOCKS] = [const { AtomicU8::new(0) }; NUM_WEAKREF_LOCKS]; + + pub(super) struct WeakrefLockGuard { + idx: usize, + } + + impl Drop for WeakrefLockGuard { + fn drop(&mut self) { + LOCKS[self.idx].store(0, Ordering::Release); + } + } + + pub(super) fn lock(addr: usize) -> WeakrefLockGuard { + let idx = (addr >> 4) % NUM_WEAKREF_LOCKS; + loop { + if LOCKS[idx] + .compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return WeakrefLockGuard { idx }; + } + core::hint::spin_loop(); + } + } + + /// Reset all weakref stripe locks after fork in child process. + /// Locks held by parent threads would cause infinite spin in the child. + #[cfg(unix)] + pub(crate) fn reset_all_after_fork() { + for lock in &LOCKS { + lock.store(0, Ordering::Release); + } + } +} + +#[cfg(not(feature = "threading"))] +mod weakref_lock { + pub(super) struct WeakrefLockGuard; + + impl Drop for WeakrefLockGuard { + fn drop(&mut self) {} + } + + pub(super) fn lock(_addr: usize) -> WeakrefLockGuard { + WeakrefLockGuard + } +} + +/// Reset weakref stripe locks after fork. Must be called before any +/// Python code runs in the child process. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn reset_weakref_locks_after_fork() { + weakref_lock::reset_all_after_fork(); +} + +// === WeakRefList: inline on every object (tp_weaklist) === + pub(super) struct WeakRefList { - inner: OncePtr<PyMutex<WeakListInner>>, + /// Head of the intrusive doubly-linked list of weakrefs. + head: PyAtomic<*mut Py<PyWeak>>, + /// Cached generic weakref (no callback, exact weakref type). + /// Matches try_reuse_basic_ref in weakrefobject.c. + generic: PyAtomic<*mut Py<PyWeak>>, } impl fmt::Debug for WeakRefList { @@ -147,33 +270,43 @@ impl fmt::Debug for WeakRefList { } } -struct WeakListInner { - list: LinkedList<WeakLink, Py<PyWeak>>, - generic_weakref: Option<NonNull<Py<PyWeak>>>, - obj: Option<NonNull<PyObject>>, - // one for each live PyWeak with a reference to this, + 1 for the referent object if it's not dead - ref_count: usize, -} +/// Unlink a node from the weakref list. Must be called under stripe lock. +/// +/// # Safety +/// `node` must be a valid pointer to a node currently in the list owned by `wrl`. +unsafe fn unlink_weakref(wrl: &WeakRefList, node: NonNull<Py<PyWeak>>) { + unsafe { + let mut ptrs = WeakLink::pointers(node); + let prev = ptrs.as_ref().get_prev(); + let next = ptrs.as_ref().get_next(); + + if let Some(prev) = prev { + WeakLink::pointers(prev).as_mut().set_next(next); + } else { + // node is the head + wrl.head.store( + next.map_or(ptr::null_mut(), |p| p.as_ptr()), + Ordering::Relaxed, + ); + } + if let Some(next) = next { + WeakLink::pointers(next).as_mut().set_prev(prev); + } -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl Send for WeakListInner {} - unsafe impl Sync for WeakListInner {} + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); } } impl WeakRefList { pub fn new() -> Self { Self { - inner: OncePtr::new(), + head: Radium::new(ptr::null_mut()), + generic: Radium::new(ptr::null_mut()), } } - /// returns None if there have never been any weakrefs in this list - fn try_lock(&self) -> Option<PyMutexGuard<'_, WeakListInner>> { - self.inner.get().map(|mu| unsafe { mu.as_ref().lock() }) - } - + /// get_or_create_weakref fn add( &self, obj: &PyObject, @@ -183,119 +316,193 @@ impl WeakRefList { dict: Option<PyDictRef>, ) -> PyRef<PyWeak> { let is_generic = cls_is_weakref && callback.is_none(); - let inner_ptr = self.inner.get_or_init(|| { - Box::new(PyMutex::new(WeakListInner { - list: LinkedList::default(), - generic_weakref: None, - obj: Some(NonNull::from(obj)), - ref_count: 1, - })) - }); - let mut inner = unsafe { inner_ptr.as_ref().lock() }; - if is_generic && let Some(generic_weakref) = inner.generic_weakref { - let generic_weakref = unsafe { generic_weakref.as_ref() }; - if generic_weakref.0.ref_count.get() != 0 { - return generic_weakref.to_owned(); + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + + // try_reuse_basic_ref: reuse cached generic weakref + if is_generic { + let generic_ptr = self.generic.load(Ordering::Relaxed); + if !generic_ptr.is_null() { + let generic = unsafe { &*generic_ptr }; + if generic.0.ref_count.safe_inc() { + return unsafe { PyRef::from_raw(generic_ptr) }; + } } } - let obj = PyWeak { + + // Allocate new PyWeak with wr_object pointing to referent + let weak_payload = PyWeak { pointers: Pointers::new(), - parent: inner_ptr, + wr_object: Radium::new(obj as *const PyObject as *mut PyObject), callback: UnsafeCell::new(callback), hash: Radium::new(crate::common::hash::SENTINEL), }; - let weak = PyRef::new_ref(obj, cls, dict); - // SAFETY: we don't actually own the PyObjectWeak's inside `list`, and every time we take - // one out of the list we immediately wrap it in ManuallyDrop or forget it - inner.list.push_front(unsafe { ptr::read(&weak) }); - inner.ref_count += 1; - if is_generic { - inner.generic_weakref = Some(NonNull::from(&*weak)); + let weak = PyRef::new_ref(weak_payload, cls, dict); + + // Insert into linked list under stripe lock + let node_ptr = NonNull::from(&*weak); + unsafe { + let mut ptrs = WeakLink::pointers(node_ptr); + if is_generic { + // Generic ref goes to head (insert_head for basic ref) + let old_head = self.head.load(Ordering::Relaxed); + ptrs.as_mut().set_next(NonNull::new(old_head)); + ptrs.as_mut().set_prev(None); + if let Some(old_head) = NonNull::new(old_head) { + WeakLink::pointers(old_head) + .as_mut() + .set_prev(Some(node_ptr)); + } + self.head.store(node_ptr.as_ptr(), Ordering::Relaxed); + self.generic.store(node_ptr.as_ptr(), Ordering::Relaxed); + } else { + // Non-generic refs go after generic ref (insert_after) + let generic_ptr = self.generic.load(Ordering::Relaxed); + if let Some(after) = NonNull::new(generic_ptr) { + let after_next = WeakLink::pointers(after).as_ref().get_next(); + ptrs.as_mut().set_prev(Some(after)); + ptrs.as_mut().set_next(after_next); + WeakLink::pointers(after).as_mut().set_next(Some(node_ptr)); + if let Some(next) = after_next { + WeakLink::pointers(next).as_mut().set_prev(Some(node_ptr)); + } + } else { + // No generic ref; insert at head + let old_head = self.head.load(Ordering::Relaxed); + ptrs.as_mut().set_next(NonNull::new(old_head)); + ptrs.as_mut().set_prev(None); + if let Some(old_head) = NonNull::new(old_head) { + WeakLink::pointers(old_head) + .as_mut() + .set_prev(Some(node_ptr)); + } + self.head.store(node_ptr.as_ptr(), Ordering::Relaxed); + } + } } + weak } - fn clear(&self) { - let to_dealloc = { - let ptr = match self.inner.get() { - Some(ptr) => ptr, - None => return, - }; - let mut inner = unsafe { ptr.as_ref().lock() }; - inner.obj = None; - // TODO: can be an arrayvec - let mut v = Vec::with_capacity(16); - loop { - let inner2 = &mut *inner; - let iter = inner2 - .list - .drain_filter(|_| true) - .filter_map(|wr| { - // we don't have actual ownership of the reference counts in the list. - // but, now we do want ownership (and so incref these *while the lock - // is held*) to avoid weird things if PyWeakObj::drop happens after - // this but before we reach the loop body below - let wr = ManuallyDrop::new(wr); - - if Some(NonNull::from(&**wr)) == inner2.generic_weakref { - inner2.generic_weakref = None - } - - // if strong_count == 0 there's some reentrancy going on. we don't - // want to call the callback - (wr.as_object().strong_count() > 0).then(|| (*wr).clone()) - }) - .take(16); - v.extend(iter); - if v.is_empty() { - break; + /// Clear all weakrefs and call their callbacks. + /// Called when the owner object is being dropped. + // PyObject_ClearWeakRefs + fn clear(&self, obj: &PyObject) { + let obj_addr = obj as *const PyObject as usize; + let _lock = weakref_lock::lock(obj_addr); + + // Clear generic cache + self.generic.store(ptr::null_mut(), Ordering::Relaxed); + + // Walk the list, collecting weakrefs with callbacks + let mut callbacks: Vec<(PyRef<PyWeak>, PyObjectRef)> = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + + let wr = unsafe { node.as_ref() }; + + // Mark weakref as dead + wr.0.payload + .wr_object + .store(ptr::null_mut(), Ordering::Relaxed); + + // Unlink from list + unsafe { + let mut ptrs = WeakLink::pointers(node); + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); + } + + // Collect callback if present and weakref is still alive + if wr.0.ref_count.get() > 0 { + let cb = unsafe { wr.0.payload.callback.get().replace(None) }; + if let Some(cb) = cb { + callbacks.push((wr.to_owned(), cb)); } - PyMutexGuard::unlocked(&mut inner, || { - for wr in v.drain(..) { - let cb = unsafe { wr.callback.get().replace(None) }; - if let Some(cb) = cb { - crate::vm::thread::with_vm(&cb, |vm| { - // TODO: handle unraisable exception - let _ = cb.call((wr.clone(),), vm); - }); - } - } - }) } - inner.ref_count -= 1; - (inner.ref_count == 0).then_some(ptr) - }; - if let Some(ptr) = to_dealloc { - unsafe { Self::dealloc(ptr) } + + current = next; } - } + self.head.store(ptr::null_mut(), Ordering::Relaxed); - fn count(&self) -> usize { - self.try_lock() - // we assume the object is still alive (and this is only - // called from PyObject::weak_count so it should be) - .map(|inner| inner.ref_count - 1) - .unwrap_or(0) + // Invoke callbacks outside the lock + drop(_lock); + for (wr, cb) in callbacks { + crate::vm::thread::with_vm(&cb, |vm| { + let _ = cb.call((wr.clone(),), vm); + }); + } } - unsafe fn dealloc(ptr: NonNull<PyMutex<WeakListInner>>) { - drop(unsafe { Box::from_raw(ptr.as_ptr()) }); + /// Clear all weakrefs but DON'T call callbacks. Instead, return them for later invocation. + /// Used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks are invoked. + /// handle_weakrefs() clears all weakrefs first, then invokes callbacks. + fn clear_for_gc_collect_callbacks(&self, obj: &PyObject) -> Vec<(PyRef<PyWeak>, PyObjectRef)> { + let obj_addr = obj as *const PyObject as usize; + let _lock = weakref_lock::lock(obj_addr); + + // Clear generic cache + self.generic.store(ptr::null_mut(), Ordering::Relaxed); + + let mut callbacks = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + + let wr = unsafe { node.as_ref() }; + + // Mark weakref as dead + wr.0.payload + .wr_object + .store(ptr::null_mut(), Ordering::Relaxed); + + // Unlink from list + unsafe { + let mut ptrs = WeakLink::pointers(node); + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); + } + + // Collect callback without invoking + if wr.0.ref_count.get() > 0 { + let cb = unsafe { wr.0.payload.callback.get().replace(None) }; + if let Some(cb) = cb { + callbacks.push((wr.to_owned(), cb)); + } + } + + current = next; + } + self.head.store(ptr::null_mut(), Ordering::Relaxed); + + callbacks } - fn get_weak_references(&self) -> Vec<PyRef<PyWeak>> { - let inner = match self.try_lock() { - Some(inner) => inner, - None => return vec![], - }; - let mut v = Vec::with_capacity(inner.ref_count - 1); - v.extend(inner.iter().map(|wr| wr.to_owned())); - v + fn count(&self, obj: &PyObject) -> usize { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + let mut count = 0usize; + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + if unsafe { node.as_ref() }.0.ref_count.get() > 0 { + count += 1; + } + current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + } + count } -} -impl WeakListInner { - fn iter(&self) -> impl Iterator<Item = &Py<PyWeak>> { - self.list.iter().filter(|wr| wr.0.ref_count.get() > 0) + fn get_weak_references(&self, obj: &PyObject) -> Vec<PyRef<PyWeak>> { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + let mut v = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let wr = unsafe { node.as_ref() }; + if wr.0.ref_count.get() > 0 { + v.push(wr.to_owned()); + } + current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + } + v } } @@ -318,7 +525,6 @@ unsafe impl Link for WeakLink { #[inline(always)] unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle { - // SAFETY: requirements forwarded from caller unsafe { PyRef::from_raw(ptr.as_ptr()) } } @@ -329,62 +535,90 @@ unsafe impl Link for WeakLink { } } +/// PyWeakReference: each weakref holds a direct pointer to its referent. #[pyclass(name = "weakref", module = false)] #[derive(Debug)] pub struct PyWeak { pointers: Pointers<Py<PyWeak>>, - parent: NonNull<PyMutex<WeakListInner>>, - // this is treated as part of parent's mutex - you must hold that lock to access it + /// Direct pointer to the referent object, null when dead. + /// Equivalent to wr_object in PyWeakReference. + wr_object: PyAtomic<*mut PyObject>, + /// Protected by stripe lock (keyed on wr_object address). callback: UnsafeCell<Option<PyObjectRef>>, pub(crate) hash: PyAtomic<crate::common::hash::PyHash>, } cfg_if::cfg_if! { if #[cfg(feature = "threading")] { - #[allow(clippy::non_send_fields_in_send_ty)] // false positive? unsafe impl Send for PyWeak {} unsafe impl Sync for PyWeak {} } } impl PyWeak { + /// _PyWeakref_GET_REF: attempt to upgrade the weakref to a strong reference. pub(crate) fn upgrade(&self) -> Option<PyObjectRef> { - let guard = unsafe { self.parent.as_ref().lock() }; - let obj_ptr = guard.obj?; + let obj_ptr = self.wr_object.load(Ordering::Acquire); + if obj_ptr.is_null() { + return None; + } + + let _lock = weakref_lock::lock(obj_ptr as usize); + + // Double-check under lock (clear may have run between our check and lock) + let obj_ptr = self.wr_object.load(Ordering::Relaxed); + if obj_ptr.is_null() { + return None; + } + unsafe { - if !obj_ptr.as_ref().0.ref_count.safe_inc() { + if !(*obj_ptr).0.ref_count.safe_inc() { return None; } - Some(PyObjectRef::from_raw(obj_ptr)) + Some(PyObjectRef::from_raw(NonNull::new_unchecked(obj_ptr))) } } pub(crate) fn is_dead(&self) -> bool { - let guard = unsafe { self.parent.as_ref().lock() }; - guard.obj.is_none() + self.wr_object.load(Ordering::Acquire).is_null() } + /// weakref_dealloc: remove from list if still linked. fn drop_inner(&self) { - let dealloc = { - let mut guard = unsafe { self.parent.as_ref().lock() }; - let offset = std::mem::offset_of!(PyInner<Self>, payload); - let py_inner = (self as *const Self) - .cast::<u8>() - .wrapping_sub(offset) - .cast::<PyInner<Self>>(); - let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py<Self>) }; - // the list doesn't have ownership over its PyRef<PyWeak>! we're being dropped - // right now so that should be obvious!! - std::mem::forget(unsafe { guard.list.remove(node_ptr) }); - guard.ref_count -= 1; - if Some(node_ptr) == guard.generic_weakref { - guard.generic_weakref = None; - } - guard.ref_count == 0 - }; - if dealloc { - unsafe { WeakRefList::dealloc(self.parent) } + let obj_ptr = self.wr_object.load(Ordering::Acquire); + if obj_ptr.is_null() { + return; // Already cleared by WeakRefList::clear() + } + + let _lock = weakref_lock::lock(obj_ptr as usize); + + // Double-check under lock + let obj_ptr = self.wr_object.load(Ordering::Relaxed); + if obj_ptr.is_null() { + return; // Cleared between our check and lock acquisition } + + let obj = unsafe { &*obj_ptr }; + let wrl = &obj.0.weak_list; + + // Compute our Py<PyWeak> node pointer from payload address + let offset = std::mem::offset_of!(PyInner<Self>, payload); + let py_inner = (self as *const Self) + .cast::<u8>() + .wrapping_sub(offset) + .cast::<PyInner<Self>>(); + let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py<Self>) }; + + // Unlink from list + unsafe { unlink_weakref(wrl, node_ptr) }; + + // Update generic cache if this was it + if wrl.generic.load(Ordering::Relaxed) == node_ptr.as_ptr() { + wrl.generic.store(ptr::null_mut(), Ordering::Relaxed); + } + + // Mark as dead + self.wr_object.store(ptr::null_mut(), Ordering::Relaxed); } } @@ -392,7 +626,6 @@ impl Drop for PyWeak { #[inline(always)] fn drop(&mut self) { // we do NOT have actual exclusive access! - // no clue if doing this actually reduces chance of UB let me: &Self = self; me.drop_inner(); } @@ -437,22 +670,22 @@ impl InstanceDict { #[inline] pub fn replace(&self, d: PyDictRef) -> PyDictRef { - std::mem::replace(&mut self.d.write(), d) + core::mem::replace(&mut self.d.write(), d) } } -impl<T: PyPayload + std::fmt::Debug> PyInner<T> { +impl<T: PyPayload + core::fmt::Debug> PyInner<T> { fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> Box<Self> { let member_count = typ.slots.member_count; Box::new(Self { ref_count: RefCount::new(), - typeid: T::payload_type_id(), vtable: PyObjVTable::of::<T>(), + gc_bits: Radium::new(0), typ: PyAtomicRef::from(typ), dict: dict.map(InstanceDict::new), weak_list: WeakRefList::new(), payload, - slots: std::iter::repeat_with(|| PyRwLock::new(None)) + slots: core::iter::repeat_with(|| PyRwLock::new(None)) .take(member_count) .collect_vec() .into_boxed_slice(), @@ -512,7 +745,7 @@ impl PyObjectRef { #[inline(always)] pub const fn into_raw(self) -> NonNull<PyObject> { let ptr = self.ptr; - std::mem::forget(self); + core::mem::forget(self); ptr } @@ -630,13 +863,14 @@ impl PyObject { } pub fn get_weak_references(&self) -> Option<Vec<PyRef<PyWeak>>> { - self.weak_ref_list().map(|wrl| wrl.get_weak_references()) + self.weak_ref_list() + .map(|wrl| wrl.get_weak_references(self)) } #[deprecated(note = "use downcastable instead")] #[inline(always)] pub fn payload_is<T: PyPayload>(&self) -> bool { - self.0.typeid == T::payload_type_id() + self.0.vtable.typeid == T::PAYLOAD_TYPE_ID } /// Force to return payload as T. @@ -719,13 +953,13 @@ impl PyObject { #[inline] pub(crate) fn typeid(&self) -> TypeId { - self.0.typeid + self.0.vtable.typeid } /// Check if this object can be downcast to T. #[inline(always)] pub fn downcastable<T: PyPayload>(&self) -> bool { - T::downcastable_from(self) + self.typeid() == T::PAYLOAD_TYPE_ID && unsafe { T::validate_downcastable_from(self) } } /// Attempt to downcast this reference to a subclass. @@ -772,7 +1006,7 @@ impl PyObject { #[inline] pub fn weak_count(&self) -> Option<usize> { - self.weak_ref_list().map(|wrl| wrl.count()) + self.weak_ref_list().map(|wrl| wrl.count(self)) } #[inline(always)] @@ -780,6 +1014,40 @@ impl PyObject { self } + /// Check if the object has been finalized (__del__ already called). + /// _PyGC_FINALIZED in Py_GIL_DISABLED mode. + #[inline] + pub(crate) fn gc_finalized(&self) -> bool { + GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)).contains(GcBits::FINALIZED) + } + + /// Mark the object as finalized. Should be called before __del__. + /// _PyGC_SET_FINALIZED in Py_GIL_DISABLED mode. + #[inline] + fn set_gc_finalized(&self) { + self.set_gc_bit(GcBits::FINALIZED); + } + + /// Set a GC bit atomically. + #[inline] + pub(crate) fn set_gc_bit(&self, bit: GcBits) { + self.0.gc_bits.fetch_or(bit.bits(), Ordering::Relaxed); + } + + /// _PyObject_GC_TRACK + #[inline] + pub(crate) fn set_gc_tracked(&self) { + self.set_gc_bit(GcBits::TRACKED); + } + + /// _PyObject_GC_UNTRACK + #[inline] + pub(crate) fn clear_gc_tracked(&self) { + self.0 + .gc_bits + .fetch_and(!GcBits::TRACKED.bits(), Ordering::Relaxed); + } + #[inline(always)] // the outer function is never inlined fn drop_slow_inner(&self) -> Result<(), ()> { // __del__ is mostly not implemented @@ -790,15 +1058,23 @@ impl PyObject { slot_del: fn(&PyObject, &VirtualMachine) -> PyResult<()>, ) -> Result<(), ()> { let ret = crate::vm::thread::with_vm(zelf, |vm| { - zelf.0.ref_count.inc(); + // Temporarily resurrect (0→2) so ref_count stays positive + // during __del__, preventing safe_inc from seeing 0. + zelf.0.ref_count.inc_by(2); + if let Err(e) = slot_del(zelf, vm) { let del_method = zelf.get_class_attr(identifier!(vm, __del__)).unwrap(); vm.run_unraisable(e, None, del_method); } + + // Undo the temporary resurrection. Always remove both + // temporary refs; the second dec returns true only when + // ref_count drops to 0 (no resurrection). + zelf.0.ref_count.dec(); zelf.0.ref_count.dec() }); match ret { - // the decref right above set ref_count back to 0 + // the decref set ref_count back to 0 Some(true) => Ok(()), // we've been resurrected by __del__ Some(false) => Err(()), @@ -809,28 +1085,33 @@ impl PyObject { } } - // CPython-compatible drop implementation - let del = self.class().mro_find_map(|cls| cls.slots.del.load()); - if let Some(slot_del) = del { + // __del__ should only be called once (like _PyGC_FINALIZED check in GIL_DISABLED) + // We call __del__ BEFORE clearing weakrefs to allow the finalizer to access + // the object's weak references if needed. + let del = self.class().slots.del.load(); + if let Some(slot_del) = del + && !self.gc_finalized() + { + self.set_gc_finalized(); call_slot_del(self, slot_del)?; } + + // Clear weak refs AFTER __del__. + // Note: This differs from GC behavior which clears weakrefs before finalizers, + // but for direct deallocation (drop_slow_inner), we need to allow the finalizer + // to run without triggering use-after-free from WeakRefList operations. if let Some(wrl) = self.weak_ref_list() { - wrl.clear(); + wrl.clear(self); } Ok(()) } - /// Can only be called when ref_count has dropped to zero. `ptr` must be valid + /// _Py_Dealloc: dispatch to type's dealloc #[inline(never)] unsafe fn drop_slow(ptr: NonNull<Self>) { - if let Err(()) = unsafe { ptr.as_ref().drop_slow_inner() } { - // abort drop for whatever reason - return; - } - let drop_dealloc = unsafe { ptr.as_ref().0.vtable.drop_dealloc }; - // call drop only when there are no references in scope - stacked borrows stuff - unsafe { drop_dealloc(ptr.as_ptr()) } + let dealloc = unsafe { ptr.as_ref().0.vtable.dealloc }; + unsafe { dealloc(ptr.as_ptr()) } } /// # Safety @@ -850,6 +1131,119 @@ impl PyObject { pub(crate) fn set_slot(&self, offset: usize, value: Option<PyObjectRef>) { *self.0.slots[offset].write() = value; } + + /// _PyObject_GC_IS_TRACKED + pub fn is_gc_tracked(&self) -> bool { + GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)).contains(GcBits::TRACKED) + } + + /// Get the referents (objects directly referenced) of this object. + /// Uses the full traverse including dict and slots. + pub fn gc_get_referents(&self) -> Vec<PyObjectRef> { + let mut result = Vec::new(); + self.0.traverse(&mut |child: &PyObject| { + result.push(child.to_owned()); + }); + result + } + + /// Call __del__ if present, without triggering object deallocation. + /// Used by GC to call finalizers before breaking cycles. + /// This allows proper resurrection detection. + /// CPython: PyObject_CallFinalizerFromDealloc in Objects/object.c + pub fn try_call_finalizer(&self) { + let del = self.class().slots.del.load(); + if let Some(slot_del) = del + && !self.gc_finalized() + { + // Mark as finalized BEFORE calling __del__ to prevent double-call + // This ensures drop_slow_inner() won't call __del__ again + self.set_gc_finalized(); + let result = crate::vm::thread::with_vm(self, |vm| { + if let Err(e) = slot_del(self, vm) + && let Some(del_method) = self.get_class_attr(identifier!(vm, __del__)) + { + vm.run_unraisable(e, None, del_method); + } + }); + let _ = result; + } + } + + /// Clear weakrefs but collect callbacks instead of calling them. + /// This is used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks run. + /// Returns collected callbacks as (PyRef<PyWeak>, callback) pairs. + // = handle_weakrefs + pub fn gc_clear_weakrefs_collect_callbacks(&self) -> Vec<(PyRef<PyWeak>, PyObjectRef)> { + if let Some(wrl) = self.weak_ref_list() { + wrl.clear_for_gc_collect_callbacks(self) + } else { + vec![] + } + } + + /// Get raw pointers to referents without incrementing reference counts. + /// This is used during GC to avoid reference count manipulation. + /// tp_traverse visits objects without incref + /// + /// # Safety + /// The returned pointers are only valid as long as the object is alive + /// and its contents haven't been modified. + pub unsafe fn gc_get_referent_ptrs(&self) -> Vec<NonNull<PyObject>> { + let mut result = Vec::new(); + // Traverse the entire object including dict and slots + self.0.traverse(&mut |child: &PyObject| { + result.push(NonNull::from(child)); + }); + result + } + + /// Pop edges from this object for cycle breaking. + /// Returns extracted child references that were removed from this object (tp_clear). + /// This is used during garbage collection to break circular references. + /// + /// # Safety + /// - ptr must be a valid pointer to a PyObject + /// - The caller must have exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_clear_raw(ptr: *mut PyObject) -> Vec<PyObjectRef> { + let mut result = Vec::new(); + let obj = unsafe { &*ptr }; + + // 1. Clear payload-specific references (vtable.clear / tp_clear) + if let Some(clear_fn) = obj.0.vtable.clear { + unsafe { clear_fn(ptr, &mut result) }; + } + + // 2. Clear member slots (subtype_clear) + for slot in obj.0.slots.iter() { + if let Some(val) = slot.write().take() { + result.push(val); + } + } + + result + } + + /// Clear this object for cycle breaking (tp_clear). + /// This version takes &self but should only be called during GC + /// when exclusive access is guaranteed. + /// + /// # Safety + /// - The caller must guarantee exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_clear(&self) -> Vec<PyObjectRef> { + // SAFETY: During GC collection, this object is unreachable (gc_refs == 0), + // meaning no other code has a reference to it. The only references are + // internal cycle references which we're about to break. + unsafe { Self::gc_clear_raw(self as *const _ as *mut PyObject) } + } + + /// Check if this object has clear capability (tp_clear) + // Py_TPFLAGS_HAVE_GC types have tp_clear + pub fn gc_has_clear(&self) -> bool { + self.0.vtable.clear.is_some() || self.0.dict.is_some() || !self.0.slots.is_empty() + } } impl Borrow<PyObject> for PyObjectRef { @@ -945,12 +1339,12 @@ impl<T: PyPayload> Borrow<PyObject> for Py<T> { } } -impl<T> std::hash::Hash for Py<T> +impl<T> core::hash::Hash for Py<T> where - T: std::hash::Hash + PyPayload, + T: core::hash::Hash + PyPayload, { #[inline] - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { self.deref().hash(state) } } @@ -977,7 +1371,7 @@ where } } -impl<T: PyPayload + std::fmt::Debug> fmt::Debug for Py<T> { +impl<T: PyPayload + core::fmt::Debug> fmt::Debug for Py<T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } @@ -1058,24 +1452,39 @@ impl<T: PyPayload> PyRef<T> { pub const fn leak(pyref: Self) -> &'static Py<T> { let ptr = pyref.ptr; - std::mem::forget(pyref); + core::mem::forget(pyref); unsafe { ptr.as_ref() } } } -impl<T: PyPayload + std::fmt::Debug> PyRef<T> { +impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> { #[inline(always)] pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option<PyDictRef>) -> Self { + let has_dict = dict.is_some(); + let is_heaptype = typ.heaptype_ext.is_some(); let inner = Box::into_raw(PyInner::new(payload, typ, dict)); - Self { - ptr: unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }, + let ptr = unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }; + + // Track object if: + // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR + // - has instance dict (user-defined class instances), OR + // - heap type (all heap type instances are GC-tracked, like Py_TPFLAGS_HAVE_GC) + if <T as crate::object::MaybeTraverse>::HAS_TRAVERSE || has_dict || is_heaptype { + let gc = crate::gc_state::gc_state(); + unsafe { + gc.track_object(ptr.cast()); + } + // Check if automatic GC should run + gc.maybe_collect(); } + + Self { ptr } } } -impl<T: crate::class::PySubclass + std::fmt::Debug> PyRef<T> +impl<T: crate::class::PySubclass + core::fmt::Debug> PyRef<T> where - T::Base: std::fmt::Debug, + T::Base: core::fmt::Debug, { /// Converts this reference to the base type (ownership transfer). /// # Safety @@ -1085,7 +1494,7 @@ where let obj: PyObjectRef = self.into(); match obj.downcast() { Ok(base_ref) => base_ref, - Err(_) => unsafe { std::hint::unreachable_unchecked() }, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, } } #[inline] @@ -1097,11 +1506,33 @@ where let obj: PyObjectRef = self.into(); match obj.downcast::<U>() { Ok(upcast_ref) => upcast_ref, - Err(_) => unsafe { std::hint::unreachable_unchecked() }, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, } } } +impl<T: crate::class::PySubclass> Py<T> { + /// Converts `&Py<T>` to `&Py<T::Base>`. + #[inline] + pub fn to_base(&self) -> &Py<T::Base> { + debug_assert!(self.as_object().downcast_ref::<T::Base>().is_some()); + // SAFETY: T is #[repr(transparent)] over T::Base, + // so Py<T> and Py<T::Base> have the same layout. + unsafe { &*(self as *const Py<T> as *const Py<T::Base>) } + } + + /// Converts `&Py<T>` to `&Py<U>` where U is an ancestor type. + #[inline] + pub fn upcast_ref<U: PyPayload + StaticType>(&self) -> &Py<U> + where + T: StaticType, + { + debug_assert!(T::static_type().is_subtype(U::static_type())); + // SAFETY: T is a subtype of U, so Py<T> can be viewed as Py<U>. + unsafe { &*(self as *const Py<T> as *const Py<U>) } + } +} + impl<T> Borrow<PyObject> for PyRef<T> where T: PyPayload, @@ -1153,12 +1584,12 @@ impl<T> Deref for PyRef<T> { } } -impl<T> std::hash::Hash for PyRef<T> +impl<T> core::hash::Hash for PyRef<T> where - T: std::hash::Hash + PyPayload, + T: core::hash::Hash + PyPayload, { #[inline] - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { self.deref().hash(state) } } @@ -1199,7 +1630,7 @@ macro_rules! partially_init { ) => {{ // check all the fields are there but *don't* actually run it - #[allow(clippy::diverging_sub_expression)] // FIXME: better way than using `if false`? + #[allow(clippy::diverging_sub_expression, reason = "intentional compile-time field check in an unreachable branch")] if false { #[allow(invalid_value, dead_code, unreachable_code)] let _ = {$ty { @@ -1207,10 +1638,10 @@ macro_rules! partially_init { $($uninit_field: unreachable!(),)* }}; } - let mut m = ::std::mem::MaybeUninit::<$ty>::uninit(); + let mut m = ::core::mem::MaybeUninit::<$ty>::uninit(); #[allow(unused_unsafe)] unsafe { - $(::std::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* + $(::core::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* } m }}; @@ -1218,7 +1649,7 @@ macro_rules! partially_init { pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { use crate::{builtins::object, class::PyClassImpl}; - use std::mem::MaybeUninit; + use core::mem::MaybeUninit; // `type` inherits from `object` // and both `type` and `object are instances of `type`. @@ -1251,8 +1682,8 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { let type_type_ptr = Box::into_raw(Box::new(partially_init!( PyInner::<PyType> { ref_count: RefCount::new(), - typeid: TypeId::of::<PyType>(), vtable: PyObjVTable::of::<PyType>(), + gc_bits: Radium::new(0), dict: None, weak_list: WeakRefList::new(), payload: type_payload, @@ -1263,8 +1694,8 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { let object_type_ptr = Box::into_raw(Box::new(partially_init!( PyInner::<PyType> { ref_count: RefCount::new(), - typeid: TypeId::of::<PyType>(), vtable: PyObjVTable::of::<PyType>(), + gc_bits: Radium::new(0), dict: None, weak_list: WeakRefList::new(), payload: object_payload, @@ -1285,12 +1716,16 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { ptr::write(&mut (*type_type_ptr).typ, PyAtomicRef::from(type_type)); let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); + // object's mro is [object] + (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); - (*type_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.base = Some(object_type.clone()); let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + // type's mro is [type, object] + (*type_type_ptr).payload.mro = + PyRwLock::new(vec![type_type.clone(), object_type.clone()]); (type_type, object_type) } @@ -1306,6 +1741,14 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { heaptype_ext: None, }; let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); + // Static type: untrack from GC (was tracked by new_ref because PyType has HAS_TRAVERSE) + unsafe { + crate::gc_state::gc_state() + .untrack_object(core::ptr::NonNull::from(weakref_type.as_object())); + } + weakref_type.as_object().clear_gc_tracked(); + // weakref's mro is [weakref, object] + weakref_type.mro.write().insert(0, weakref_type.clone()); object_type.subclasses.write().push( type_type diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 88f5fdc66d7..0fd251499f1 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -12,9 +12,10 @@ use crate::{ convert::{IntoPyException, ToPyObject, ToPyResult, TryFromObject}, vm::Context, }; -use std::{ +use alloc::fmt; + +use core::{ borrow::Borrow, - fmt, marker::PhantomData, ops::Deref, ptr::{NonNull, null_mut}, @@ -108,7 +109,7 @@ impl<T: PyPayload> AsRef<Py<T>> for PyExact<T> { } } -impl<T: PyPayload> std::borrow::ToOwned for PyExact<T> { +impl<T: PyPayload> alloc::borrow::ToOwned for PyExact<T> { type Owned = PyRefExact<T>; fn to_owned(&self) -> Self::Owned { @@ -462,6 +463,64 @@ impl PyAtomicRef<Option<PyObject>> { } } +/// Atomic borrowed (non-ref-counted) optional reference to a Python object. +/// Unlike `PyAtomicRef`, this does NOT own the reference. +/// The pointed-to object must outlive this reference. +pub struct PyAtomicBorrow { + inner: PyAtomic<*mut u8>, +} + +// Safety: Access patterns ensure the pointed-to object outlives this reference. +// The owner (generator/coroutine) clears this in its Drop impl before deallocation. +unsafe impl Send for PyAtomicBorrow {} +unsafe impl Sync for PyAtomicBorrow {} + +impl PyAtomicBorrow { + pub fn new() -> Self { + Self { + inner: Radium::new(null_mut()), + } + } + + pub fn store(&self, obj: &PyObject) { + let ptr = obj as *const PyObject as *mut u8; + Radium::store(&self.inner, ptr, Ordering::Relaxed); + } + + pub fn load(&self) -> Option<&PyObject> { + let ptr = Radium::load(&self.inner, Ordering::Relaxed); + if ptr.is_null() { + None + } else { + Some(unsafe { &*(ptr as *const PyObject) }) + } + } + + pub fn clear(&self) { + Radium::store(&self.inner, null_mut(), Ordering::Relaxed); + } + + pub fn to_owned(&self) -> Option<PyObjectRef> { + self.load().map(|obj| obj.to_owned()) + } +} + +impl Default for PyAtomicBorrow { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for PyAtomicBorrow { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "PyAtomicBorrow({:?})", + Radium::load(&self.inner, Ordering::Relaxed) + ) + } +} + pub trait AsObject where Self: Borrow<PyObject>, @@ -581,7 +640,7 @@ impl ToPyObject for &PyObject { // explicitly implementing `ToPyObject`. impl<T> ToPyObject for T where - T: PyPayload + std::fmt::Debug + Sized, + T: PyPayload + core::fmt::Debug + Sized, { #[inline(always)] fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 034523afe5a..a279201a0b0 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -7,4 +7,5 @@ mod traverse_object; pub use self::core::*; pub use self::ext::*; pub use self::payload::*; +pub(crate) use core::SIZEOF_PYOBJECT_HEAD; pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index cf903871179..98c61817568 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -26,24 +26,17 @@ pub(crate) fn cold_downcast_type_error( } pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { - #[inline] - fn payload_type_id() -> std::any::TypeId { - std::any::TypeId::of::<Self>() - } - - /// # Safety: this function should only be called if `payload_type_id` matches the type of `obj`. - #[inline] - fn downcastable_from(obj: &PyObject) -> bool { - obj.typeid() == Self::payload_type_id() && Self::validate_downcastable_from(obj) - } + const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::<Self>(); + /// # Safety + /// This function should only be called if `payload_type_id` matches the type of `obj`. #[inline] - fn validate_downcastable_from(_obj: &PyObject) -> bool { + unsafe fn validate_downcastable_from(_obj: &PyObject) -> bool { true } fn try_downcast_from(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if Self::downcastable_from(obj) { + if obj.downcastable::<Self>() { return Ok(()); } @@ -56,7 +49,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { self.into_ref(&vm.ctx).into() } @@ -64,7 +57,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn _into_ref(self, cls: PyTypeRef, ctx: &Context) -> PyRef<Self> where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { Some(ctx.new_dict()) @@ -77,7 +70,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_exact_ref(self, ctx: &Context) -> PyRefExact<Self> where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { unsafe { // Self::into_ref() always returns exact typed PyRef @@ -88,7 +81,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref(self, ctx: &Context) -> PyRef<Self> where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let cls = Self::class(ctx); self._into_ref(cls.to_owned(), ctx) @@ -97,7 +90,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult<PyRef<Self>> where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let exact_class = Self::class(&vm.ctx); if cls.fast_issubclass(exact_class) { @@ -106,7 +99,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline(never)] fn _into_ref_size_error( vm: &VirtualMachine, - cls: &PyTypeRef, + cls: &Py<PyType>, exact_class: &Py<PyType>, ) -> PyBaseExceptionRef { vm.new_type_error(format!( @@ -123,7 +116,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline(never)] fn _into_ref_with_type_error( vm: &VirtualMachine, - cls: &PyTypeRef, + cls: &Py<PyType>, exact_class: &Py<PyType>, ) -> PyBaseExceptionRef { vm.new_type_error(format!( @@ -138,11 +131,11 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { } pub trait PyObjectPayload: - PyPayload + std::any::Any + std::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static + PyPayload + core::any::Any + core::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static { } -impl<T: PyPayload + std::fmt::Debug + 'static> PyObjectPayload for T {} +impl<T: PyPayload + core::fmt::Debug + 'static> PyObjectPayload for T {} pub trait SlotOffset { fn offset() -> usize; diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 31bee8becea..367076b78e3 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -1,4 +1,4 @@ -use std::ptr::NonNull; +use core::ptr::NonNull; use rustpython_common::lock::{PyMutex, PyRwLock}; @@ -12,9 +12,13 @@ pub type TraverseFn<'a> = dyn FnMut(&PyObject) + 'a; /// Every PyObjectPayload impl `MaybeTrace`, which may or may not be traceable pub trait MaybeTraverse { /// if is traceable, will be used by vtable to determine - const IS_TRACE: bool = false; + const HAS_TRAVERSE: bool = false; + /// if has clear implementation for circular reference resolution (tp_clear) + const HAS_CLEAR: bool = false; // if this type is traceable, then call with tracer_fn, default to do nothing fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>); + // if this type has clear, extract child refs for circular reference resolution (tp_clear) + fn try_clear(&mut self, _out: &mut Vec<PyObjectRef>) {} } /// Type that need traverse it's children should impl [`Traverse`] (not [`MaybeTraverse`]) @@ -28,6 +32,11 @@ pub unsafe trait Traverse { /// /// - _**DO NOT**_ clone a [`PyObjectRef`] or [`PyRef<T>`] in [`Traverse::traverse()`] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); + + /// Extract all owned child PyObjectRefs for circular reference resolution (tp_clear). + /// Called just before object deallocation to break circular references. + /// Default implementation does nothing. + fn clear(&mut self, _out: &mut Vec<PyObjectRef>) {} } unsafe impl Traverse for PyObjectRef { diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index 281b0e56eb5..3f88c6b7481 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -1,29 +1,43 @@ -use std::fmt; +use alloc::fmt; +use core::any::TypeId; use crate::{ - PyObject, + PyObject, PyObjectRef, object::{ - Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, drop_dealloc_obj, - try_trace_obj, + Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc, + try_clear_obj, try_traverse_obj, }, }; use super::{Traverse, TraverseFn}; pub(in crate::object) struct PyObjVTable { - pub(in crate::object) drop_dealloc: unsafe fn(*mut PyObject), + pub(in crate::object) typeid: TypeId, + /// dealloc: handles __del__, weakref clearing, and memory free. + pub(in crate::object) dealloc: unsafe fn(*mut PyObject), pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter<'_>) -> fmt::Result, pub(in crate::object) trace: Option<unsafe fn(&PyObject, &mut TraverseFn<'_>)>, + /// Clear for circular reference resolution (tp_clear). + /// Called just before deallocation to extract child references. + pub(in crate::object) clear: Option<unsafe fn(*mut PyObject, &mut Vec<PyObjectRef>)>, } impl PyObjVTable { pub const fn of<T: PyObjectPayload>() -> &'static Self { &Self { - drop_dealloc: drop_dealloc_obj::<T>, + typeid: T::PAYLOAD_TYPE_ID, + dealloc: default_dealloc::<T>, debug: debug_obj::<T>, trace: const { - if T::IS_TRACE { - Some(try_trace_obj::<T>) + if T::HAS_TRAVERSE { + Some(try_traverse_obj::<T>) + } else { + None + } + }, + clear: const { + if T::HAS_CLEAR { + Some(try_clear_obj::<T>) } else { None } @@ -41,11 +55,18 @@ unsafe impl Traverse for InstanceDict { unsafe impl Traverse for PyInner<Erased> { /// Because PyObject hold a `PyInner<Erased>`, so we need to trace it fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) - // 2. call vtable's trace function to trace payload - // self.typ.trace(tracer_fn); + // For heap type instances, traverse the type reference. + // PyAtomicRef holds a strong reference (via PyRef::leak), so GC must + // account for it to correctly detect instance ↔ type cycles. + // Static types are always alive and don't need this. + let typ = &*self.typ; + if typ.heaptype_ext.is_some() { + // Safety: Py<PyType> and PyObject share the same memory layout + let typ_obj: &PyObject = unsafe { &*(typ as *const _ as *const PyObject) }; + tracer_fn(typ_obj); + } self.dict.traverse(tracer_fn); - // weak_list keeps a *pointer* to a struct for maintenance of weak ref, so no ownership, no trace + // weak_list is inline atomic pointers, no heap allocation, no trace self.slots.traverse(tracer_fn); if let Some(f) = self.vtable.trace { @@ -60,12 +81,14 @@ unsafe impl Traverse for PyInner<Erased> { unsafe impl<T: MaybeTraverse> Traverse for PyInner<T> { /// Type is known, so we can call `try_trace` directly instead of using erased type vtable fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) - // 2. call corresponding `try_trace` function to trace payload - // (No need to call vtable's trace function because we already know the type) - // self.typ.trace(tracer_fn); + // For heap type instances, traverse the type reference (same as erased version) + let typ = &*self.typ; + if typ.heaptype_ext.is_some() { + let typ_obj: &PyObject = unsafe { &*(typ as *const _ as *const PyObject) }; + tracer_fn(typ_obj); + } self.dict.traverse(tracer_fn); - // weak_list keeps a *pointer* to a struct for maintenance of weak ref, so no ownership, no trace + // weak_list is inline atomic pointers, no heap allocation, no trace self.slots.traverse(tracer_fn); T::try_traverse(&self.payload, tracer_fn); } diff --git a/crates/vm/src/ospath.rs b/crates/vm/src/ospath.rs index add40f9b20c..00195460ea3 100644 --- a/crates/vm/src/ospath.rs +++ b/crates/vm/src/ospath.rs @@ -1,23 +1,195 @@ use rustpython_common::crt_fd; use crate::{ - PyObjectRef, PyResult, VirtualMachine, - builtins::PyBaseExceptionRef, + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyStr}, + class::StaticType, convert::{IntoPyException, ToPyException, ToPyObject, TryFromObject}, function::FsPath, - object::AsObject, }; use std::path::{Path, PathBuf}; -// path_ without allow_fd in CPython +/// path_converter +#[derive(Clone, Copy, Default)] +pub struct PathConverter { + /// Function name for error messages (e.g., "rename") + pub function_name: Option<&'static str>, + /// Argument name for error messages (e.g., "src", "dst") + pub argument_name: Option<&'static str>, + /// If true, embedded null characters are allowed + pub non_strict: bool, +} + +impl PathConverter { + pub const fn new() -> Self { + Self { + function_name: None, + argument_name: None, + non_strict: false, + } + } + + pub const fn function(mut self, name: &'static str) -> Self { + self.function_name = Some(name); + self + } + + pub const fn argument(mut self, name: &'static str) -> Self { + self.argument_name = Some(name); + self + } + + pub const fn non_strict(mut self) -> Self { + self.non_strict = true; + self + } + + /// Generate error message prefix like "rename: " + fn error_prefix(&self) -> String { + match self.function_name { + Some(func) => format!("{}: ", func), + None => String::new(), + } + } + + /// Get argument name for error messages, defaults to "path" + fn arg_name(&self) -> &'static str { + self.argument_name.unwrap_or("path") + } + + /// Format a type error message + fn type_error_msg(&self, type_name: &str, allow_fd: bool) -> String { + let expected = if allow_fd { + "string, bytes, os.PathLike or integer" + } else { + "string, bytes or os.PathLike" + }; + format!( + "{}{} should be {}, not {}", + self.error_prefix(), + self.arg_name(), + expected, + type_name + ) + } + + /// Convert to OsPathOrFd (path or file descriptor) + pub(crate) fn try_path_or_fd<'fd>( + &self, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<OsPathOrFd<'fd>> { + // Handle fd (before __fspath__ check, like CPython) + if let Some(int) = obj.try_index_opt(vm) { + // Warn if bool is used as a file descriptor + if obj + .class() + .is(crate::builtins::bool_::PyBool::static_type()) + { + crate::stdlib::warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + let fd = int?.try_to_primitive(vm)?; + return unsafe { crt_fd::Borrowed::try_borrow_raw(fd) } + .map(OsPathOrFd::Fd) + .map_err(|e| e.into_pyexception(vm)); + } + + self.try_path_inner(obj, true, vm).map(OsPathOrFd::Path) + } + + /// Convert to OsPath only (no fd support) + fn try_path_inner( + &self, + obj: PyObjectRef, + allow_fd: bool, + vm: &VirtualMachine, + ) -> PyResult<OsPath> { + // Try direct str/bytes match + let obj = match self.try_match_str_bytes(obj.clone(), vm)? { + Ok(path) => return Ok(path), + Err(obj) => obj, + }; + + // Call __fspath__ + let type_error_msg = || self.type_error_msg(&obj.class().name(), allow_fd); + let method = + vm.get_method_or_type_error(obj.clone(), identifier!(vm, __fspath__), type_error_msg)?; + if vm.is_none(&method) { + return Err(vm.new_type_error(type_error_msg())); + } + let result = method.call((), vm)?; + + // Match __fspath__ result + self.try_match_str_bytes(result.clone(), vm)?.map_err(|_| { + vm.new_type_error(format!( + "{}expected {}.__fspath__() to return str or bytes, not {}", + self.error_prefix(), + obj.class().name(), + result.class().name(), + )) + }) + } + + /// Try to match str or bytes, returns Err(obj) if neither + fn try_match_str_bytes( + &self, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Result<OsPath, PyObjectRef>> { + let check_nul = |b: &[u8]| { + if self.non_strict || memchr::memchr(b'\0', b).is_none() { + Ok(()) + } else { + Err(vm.new_value_error(format!( + "{}embedded null character in {}", + self.error_prefix(), + self.arg_name() + ))) + } + }; + + match_class!(match obj { + s @ PyStr => { + check_nul(s.as_bytes())?; + let path = vm.fsencode(&s)?.into_owned(); + Ok(Ok(OsPath { + path, + origin: Some(s.into()), + })) + } + b @ PyBytes => { + check_nul(&b)?; + let path = FsPath::bytes_as_os_str(&b, vm)?.to_owned(); + Ok(Ok(OsPath { + path, + origin: Some(b.into()), + })) + } + obj => Ok(Err(obj)), + }) + } + + /// Convert to OsPath directly + pub fn try_path(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<OsPath> { + self.try_path_inner(obj, false, vm) + } +} + +/// path_t output - the converted path #[derive(Clone)] pub struct OsPath { pub path: std::ffi::OsString, - pub(super) mode: OutputMode, + /// Original Python object for identity preservation in OSError + pub(super) origin: Option<PyObjectRef>, } #[derive(Debug, Copy, Clone)] -pub(super) enum OutputMode { +pub enum OutputMode { String, Bytes, } @@ -40,19 +212,19 @@ impl OutputMode { impl OsPath { pub fn new_str(path: impl Into<std::ffi::OsString>) -> Self { let path = path.into(); - Self { - path, - mode: OutputMode::String, - } + Self { path, origin: None } } pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult<Self> { let path = fspath.as_os_str(vm)?.into_owned(); - let mode = match fspath { - FsPath::Str(_) => OutputMode::String, - FsPath::Bytes(_) => OutputMode::Bytes, + let origin = match fspath { + FsPath::Str(s) => s.into(), + FsPath::Bytes(b) => b.into(), }; - Ok(Self { path, mode }) + Ok(Self { + path, + origin: Some(origin), + }) } /// Convert an object to OsPath using the os.fspath-style error message. @@ -71,12 +243,12 @@ impl OsPath { self.path.into_encoded_bytes() } - pub fn to_string_lossy(&self) -> std::borrow::Cow<'_, str> { + pub fn to_string_lossy(&self) -> alloc::borrow::Cow<'_, str> { self.path.to_string_lossy() } - pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(self.into_bytes()).map_err(|err| err.to_pyexception(vm)) + pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.into_bytes()).map_err(|err| err.to_pyexception(vm)) } #[cfg(windows)] @@ -85,7 +257,20 @@ impl OsPath { } pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { - self.mode.process_path(self.path.clone(), vm) + if let Some(ref origin) = self.origin { + origin.clone() + } else { + // Default to string when no origin (e.g., from new_str) + OutputMode::String.process_path(self.path.clone(), vm) + } + } + + /// Get the output mode based on origin type (bytes -> Bytes, otherwise -> String) + pub fn mode(&self) -> OutputMode { + match &self.origin { + Some(obj) if obj.downcast_ref::<PyBytes>().is_some() => OutputMode::Bytes, + _ => OutputMode::String, + } } } @@ -96,15 +281,8 @@ impl AsRef<Path> for OsPath { } impl TryFromObject for OsPath { - // TODO: path_converter with allow_fd=0 in CPython fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let fspath = FsPath::try_from( - obj, - true, - "should be string, bytes, os.PathLike or integer", - vm, - )?; - Self::from_fspath(fspath, vm) + PathConverter::new().try_path(obj, vm) } } @@ -117,15 +295,7 @@ pub(crate) enum OsPathOrFd<'fd> { impl TryFromObject for OsPathOrFd<'_> { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - match obj.try_index_opt(vm) { - Some(int) => { - let fd = int?.try_to_primitive(vm)?; - unsafe { crt_fd::Borrowed::try_borrow_raw(fd) } - .map(Self::Fd) - .map_err(|e| e.into_pyexception(vm)) - } - None => obj.try_into_value(vm).map(Self::Path), - } + PathConverter::new().try_path_or_fd(obj, vm) } } @@ -144,62 +314,34 @@ impl OsPathOrFd<'_> { } } -// TODO: preserve the input `PyObjectRef` of filename and filename2 (Failing check `self.assertIs(err.filename, name, str(func)`) -pub struct IOErrorBuilder<'a> { - error: &'a std::io::Error, - filename: Option<OsPathOrFd<'a>>, - filename2: Option<OsPathOrFd<'a>>, -} - -impl<'a> IOErrorBuilder<'a> { - pub const fn new(error: &'a std::io::Error) -> Self { - Self { - error, - filename: None, - filename2: None, - } - } - - pub(crate) fn filename(mut self, filename: impl Into<OsPathOrFd<'a>>) -> Self { - let filename = filename.into(); - self.filename.replace(filename); - self - } - - pub(crate) fn filename2(mut self, filename: impl Into<OsPathOrFd<'a>>) -> Self { - let filename = filename.into(); - self.filename2.replace(filename); - self - } - - pub(crate) fn with_filename( - error: &'a std::io::Error, +impl crate::exceptions::OSErrorBuilder { + #[must_use] + pub(crate) fn with_filename<'a>( + error: &std::io::Error, filename: impl Into<OsPathOrFd<'a>>, vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - let zelf = IOErrorBuilder { - error, - filename: Some(filename.into()), - filename2: None, - }; - zelf.to_pyexception(vm) + ) -> crate::builtins::PyBaseExceptionRef { + // TODO: return type to PyRef<PyOSError> + use crate::exceptions::ToOSErrorBuilder; + let builder = error.to_os_error_builder(vm); + let builder = builder.filename(filename.into().filename(vm)); + builder.build(vm).upcast() } -} - -impl ToPyException for IOErrorBuilder<'_> { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - let exc = self.error.to_pyexception(vm); - if let Some(filename) = &self.filename { - exc.as_object() - .set_attr("filename", filename.filename(vm), vm) - .unwrap(); - } - if let Some(filename2) = &self.filename2 { - exc.as_object() - .set_attr("filename2", filename2.filename(vm), vm) - .unwrap(); - } - exc + /// Like `with_filename`, but strips winerror on Windows. + /// Use for C runtime errors (open, fstat, etc.) that should produce + /// `[Errno X]` format instead of `[WinError X]`. + #[must_use] + pub(crate) fn with_filename_from_errno<'a>( + error: &std::io::Error, + filename: impl Into<OsPathOrFd<'a>>, + vm: &VirtualMachine, + ) -> crate::builtins::PyBaseExceptionRef { + use crate::exceptions::ToOSErrorBuilder; + let builder = error.to_os_error_builder(vm); + #[cfg(windows)] + let builder = builder.without_winerror(); + let builder = builder.filename(filename.into().filename(vm)); + builder.build(vm).upcast() } } diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs index 1b1a4a14df5..0fe4d15458b 100644 --- a/crates/vm/src/protocol/buffer.rs +++ b/crates/vm/src/protocol/buffer.rs @@ -9,10 +9,10 @@ use crate::{ }, object::PyObjectPayload, sliceable::SequenceIndexOp, - types::Unconstructible, }; +use alloc::borrow::Cow; +use core::{fmt::Debug, ops::Range}; use itertools::Itertools; -use std::{borrow::Cow, fmt::Debug, ops::Range}; pub struct BufferMethods { pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, @@ -22,7 +22,7 @@ pub struct BufferMethods { } impl Debug for BufferMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("BufferMethods") .field("obj_bytes", &(self.obj_bytes as usize)) .field("obj_bytes_mut", &(self.obj_bytes_mut as usize)) @@ -135,8 +135,8 @@ impl PyBuffer { pub(crate) unsafe fn drop_without_release(&mut self) { // SAFETY: requirements forwarded from caller unsafe { - std::ptr::drop_in_place(&mut self.obj); - std::ptr::drop_in_place(&mut self.desc); + core::ptr::drop_in_place(&mut self.obj); + core::ptr::drop_in_place(&mut self.desc); } } } @@ -144,8 +144,7 @@ impl PyBuffer { impl<'a> TryFromBorrowedObject<'a> for PyBuffer { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { let cls = obj.class(); - let as_buffer = cls.mro_find_map(|cls| cls.slots.as_buffer); - if let Some(f) = as_buffer { + if let Some(f) = cls.slots.as_buffer { return f(obj, vm); } Err(vm.new_type_error(format!( @@ -202,15 +201,26 @@ impl BufferDescriptor { #[cfg(debug_assertions)] pub fn validate(self) -> Self { - assert!(self.itemsize != 0); - assert!(self.ndim() != 0); - let mut shape_product = 1; - for (shape, stride, suboffset) in self.dim_desc.iter().cloned() { - shape_product *= shape; - assert!(suboffset >= 0); - assert!(stride != 0); + // ndim=0 is valid for scalar types (e.g., ctypes Structure) + if self.ndim() == 0 { + // Empty structures (len=0) can have itemsize=0 + if self.len > 0 { + assert!(self.itemsize != 0); + } + assert!(self.itemsize == self.len); + } else { + let mut shape_product = 1; + let has_zero_dim = self.dim_desc.iter().any(|(s, _, _)| *s == 0); + for (shape, stride, suboffset) in self.dim_desc.iter().cloned() { + shape_product *= shape; + assert!(suboffset >= 0); + // For empty arrays (any dimension is 0), strides can be 0 + if !has_zero_dim { + assert!(stride != 0); + } + } + assert!(shape_product * self.itemsize == self.len); } - assert!(shape_product * self.itemsize == self.len); self } @@ -402,10 +412,10 @@ pub struct VecBuffer { data: PyMutex<Vec<u8>>, } -#[pyclass(flags(BASETYPE), with(Unconstructible))] +#[pyclass(flags(BASETYPE, DISALLOW_INSTANTIATION))] impl VecBuffer { pub fn take(&self) -> Vec<u8> { - std::mem::take(&mut self.data.lock()) + core::mem::take(&mut self.data.lock()) } } @@ -417,8 +427,6 @@ impl From<Vec<u8>> for VecBuffer { } } -impl Unconstructible for VecBuffer {} - impl PyRef<VecBuffer> { pub fn into_pybuffer(self, readonly: bool) -> PyBuffer { let len = self.data.lock().len(); diff --git a/crates/vm/src/protocol/callable.rs b/crates/vm/src/protocol/callable.rs index 1444b6bf73a..9a621dee4f8 100644 --- a/crates/vm/src/protocol/callable.rs +++ b/crates/vm/src/protocol/callable.rs @@ -1,7 +1,8 @@ use crate::{ + builtins::{PyBoundMethod, PyFunction}, function::{FuncArgs, IntoFuncArgs}, types::GenericMethod, - {AsObject, PyObject, PyResult, VirtualMachine}, + {PyObject, PyObjectRef, PyResult, VirtualMachine}, }; impl PyObject { @@ -42,31 +43,53 @@ pub struct PyCallable<'a> { impl<'a> PyCallable<'a> { pub fn new(obj: &'a PyObject) -> Option<Self> { - let call = obj.class().mro_find_map(|cls| cls.slots.call.load())?; + let call = obj.class().slots.call.load()?; Some(PyCallable { obj, call }) } pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { let args = args.into_args(vm); - vm.trace_event(TraceEvent::Call)?; - let result = (self.call)(self.obj, args, vm); - vm.trace_event(TraceEvent::Return)?; - result + // Python functions get 'call'/'return' events from with_frame(). + // Bound methods delegate to the inner callable, which fires its own events. + // All other callables (built-in functions, etc.) get 'c_call'/'c_return'/'c_exception'. + let is_python_callable = self.obj.downcast_ref::<PyFunction>().is_some() + || self.obj.downcast_ref::<PyBoundMethod>().is_some(); + if is_python_callable { + (self.call)(self.obj, args, vm) + } else { + let callable = self.obj.to_owned(); + vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?; + let result = (self.call)(self.obj, args, vm); + if result.is_ok() { + vm.trace_event(TraceEvent::CReturn, Some(callable))?; + } else { + let _ = vm.trace_event(TraceEvent::CException, Some(callable)); + } + result + } } } /// Trace events for sys.settrace and sys.setprofile. -enum TraceEvent { +pub(crate) enum TraceEvent { Call, Return, + Line, + CCall, + CReturn, + CException, } -impl std::fmt::Display for TraceEvent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for TraceEvent { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { use TraceEvent::*; match self { Call => write!(f, "call"), Return => write!(f, "return"), + Line => write!(f, "line"), + CCall => write!(f, "c_call"), + CReturn => write!(f, "c_return"), + CException => write!(f, "c_exception"), } } } @@ -74,28 +97,27 @@ impl std::fmt::Display for TraceEvent { impl VirtualMachine { /// Call registered trace function. #[inline] - fn trace_event(&self, event: TraceEvent) -> PyResult<()> { + pub(crate) fn trace_event(&self, event: TraceEvent, arg: Option<PyObjectRef>) -> PyResult<()> { if self.use_tracing.get() { - self._trace_event_inner(event) + self._trace_event_inner(event, arg) } else { Ok(()) } } - fn _trace_event_inner(&self, event: TraceEvent) -> PyResult<()> { + fn _trace_event_inner(&self, event: TraceEvent, arg: Option<PyObjectRef>) -> PyResult<()> { let trace_func = self.trace_func.borrow().to_owned(); let profile_func = self.profile_func.borrow().to_owned(); if self.is_none(&trace_func) && self.is_none(&profile_func) { return Ok(()); } - let frame_ref = self.current_frame(); - if frame_ref.is_none() { + let Some(frame_ref) = self.current_frame() else { return Ok(()); - } + }; - let frame = frame_ref.unwrap().as_object().to_owned(); + let frame: PyObjectRef = frame_ref.into(); let event = self.ctx.new_str(event.to_string()).into(); - let args = vec![frame, event, self.ctx.none()]; + let args = vec![frame, event, arg.unwrap_or_else(|| self.ctx.none())]; // temporarily disable tracing, during the call to the // tracing function itself. diff --git a/crates/vm/src/protocol/iter.rs b/crates/vm/src/protocol/iter.rs index 18f2b5243e2..aa6ab6769cd 100644 --- a/crates/vm/src/protocol/iter.rs +++ b/crates/vm/src/protocol/iter.rs @@ -4,8 +4,8 @@ use crate::{ convert::{ToPyObject, ToPyResult}, object::{Traverse, TraverseFn}, }; -use std::borrow::Borrow; -use std::ops::Deref; +use core::borrow::Borrow; +use core::ops::Deref; /// Iterator Protocol // https://docs.python.org/3/c-api/iter.html @@ -23,9 +23,7 @@ unsafe impl<O: Borrow<PyObject>> Traverse for PyIter<O> { impl PyIter<PyObjectRef> { pub fn check(obj: &PyObject) -> bool { - obj.class() - .mro_find_map(|x| x.slots.iternext.load()) - .is_some() + obj.class().slots.iternext.load().is_some() } } @@ -37,18 +35,19 @@ where Self(obj) } pub fn next(&self, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let iternext = { - self.0 - .borrow() - .class() - .mro_find_map(|x| x.slots.iternext.load()) - .ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object is not an iterator", - self.0.borrow().class().name() - )) - })? - }; + let iternext = self + .0 + .borrow() + .class() + .slots + .iternext + .load() + .ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object is not an iterator", + self.0.borrow().class().name() + )) + })?; iternext(self.0.borrow(), vm) } @@ -126,10 +125,7 @@ impl TryFromObject for PyIter<PyObjectRef> { // in the vm when a for loop is entered. Next, it is used when the builtin // function 'iter' is called. fn try_from_object(vm: &VirtualMachine, iter_target: PyObjectRef) -> PyResult<Self> { - let get_iter = { - let cls = iter_target.class(); - cls.mro_find_map(|x| x.slots.iter.load()) - }; + let get_iter = iter_target.class().slots.iter.load(); if let Some(get_iter) = get_iter { let iter = get_iter(iter_target, vm)?; if Self::check(&iter) { @@ -227,7 +223,7 @@ where vm: &'a VirtualMachine, obj: O, // creating PyIter<O> is zero-cost length_hint: Option<usize>, - _phantom: std::marker::PhantomData<T>, + _phantom: core::marker::PhantomData<T>, } unsafe impl<T, O> Traverse for PyIterIter<'_, T, O> @@ -248,7 +244,7 @@ where vm, obj, length_hint, - _phantom: std::marker::PhantomData, + _phantom: core::marker::PhantomData, } } } diff --git a/crates/vm/src/protocol/mapping.rs b/crates/vm/src/protocol/mapping.rs index a942303dbb4..6c200043e35 100644 --- a/crates/vm/src/protocol/mapping.rs +++ b/crates/vm/src/protocol/mapping.rs @@ -3,7 +3,6 @@ use crate::{ builtins::{ PyDict, PyStrInterned, dict::{PyDictItems, PyDictKeys, PyDictValues}, - type_::PointerSlot, }, convert::ToPyResult, object::{Traverse, TraverseFn}, @@ -13,15 +12,9 @@ use crossbeam_utils::atomic::AtomicCell; // Mapping protocol // https://docs.python.org/3/c-api/mapping.html -impl PyObject { - pub fn to_mapping(&self) -> PyMapping<'_> { - PyMapping::from(self) - } -} - #[allow(clippy::type_complexity)] #[derive(Default)] -pub struct PyMappingMethods { +pub struct PyMappingSlots { pub length: AtomicCell<Option<fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>>>, pub subscript: AtomicCell<Option<fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult>>, pub ass_subscript: AtomicCell< @@ -29,38 +22,72 @@ pub struct PyMappingMethods { >, } -impl std::fmt::Debug for PyMappingMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "mapping methods") +impl core::fmt::Debug for PyMappingSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PyMappingSlots") } } -impl PyMappingMethods { - fn check(&self) -> bool { +impl PyMappingSlots { + pub fn has_subscript(&self) -> bool { self.subscript.load().is_some() } - #[allow(clippy::declare_interior_mutable_const)] + /// Copy from static PyMappingMethods + pub fn copy_from(&self, methods: &PyMappingMethods) { + if let Some(f) = methods.length { + self.length.store(Some(f)); + } + if let Some(f) = methods.subscript { + self.subscript.store(Some(f)); + } + if let Some(f) = methods.ass_subscript { + self.ass_subscript.store(Some(f)); + } + } +} + +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PyMappingMethods { + pub length: Option<fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>>, + pub subscript: Option<fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub ass_subscript: + Option<fn(PyMapping<'_>, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, +} + +impl core::fmt::Debug for PyMappingMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PyMappingMethods") + } +} + +impl PyMappingMethods { pub const NOT_IMPLEMENTED: Self = Self { - length: AtomicCell::new(None), - subscript: AtomicCell::new(None), - ass_subscript: AtomicCell::new(None), + length: None, + subscript: None, + ass_subscript: None, }; } -impl<'a> From<&'a PyObject> for PyMapping<'a> { - fn from(obj: &'a PyObject) -> Self { - static GLOBAL_NOT_IMPLEMENTED: PyMappingMethods = PyMappingMethods::NOT_IMPLEMENTED; - let methods = Self::find_methods(obj) - .map_or(&GLOBAL_NOT_IMPLEMENTED, |x| unsafe { x.borrow_static() }); - Self { obj, methods } +impl PyObject { + pub fn mapping_unchecked(&self) -> PyMapping<'_> { + PyMapping { obj: self } + } + + pub fn try_mapping(&self, vm: &VirtualMachine) -> PyResult<PyMapping<'_>> { + let mapping = self.mapping_unchecked(); + if mapping.check() { + Ok(mapping) + } else { + Err(vm.new_type_error(format!("{} is not a mapping object", self.class()))) + } } } #[derive(Copy, Clone)] pub struct PyMapping<'a> { pub obj: &'a PyObject, - pub methods: &'static PyMappingMethods, } unsafe impl Traverse for PyMapping<'_> { @@ -76,34 +103,19 @@ impl AsRef<PyObject> for PyMapping<'_> { } } -impl<'a> PyMapping<'a> { - pub fn try_protocol(obj: &'a PyObject, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(methods) = Self::find_methods(obj) - && methods.as_ref().check() - { - return Ok(Self { - obj, - methods: unsafe { methods.borrow_static() }, - }); - } - - Err(vm.new_type_error(format!("{} is not a mapping object", obj.class()))) - } -} - impl PyMapping<'_> { - // PyMapping::Check #[inline] - pub fn check(obj: &PyObject) -> bool { - Self::find_methods(obj).is_some_and(|x| x.as_ref().check()) + pub fn slots(&self) -> &PyMappingSlots { + &self.obj.class().slots.as_mapping } - pub fn find_methods(obj: &PyObject) -> Option<PointerSlot<PyMappingMethods>> { - obj.class().mro_find_map(|cls| cls.slots.as_mapping.load()) + #[inline] + pub fn check(&self) -> bool { + self.slots().has_subscript() } pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.methods.length.load().map(|f| f(self, vm)) + self.slots().length.load().map(|f| f(self, vm)) } pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { @@ -130,7 +142,7 @@ impl PyMapping<'_> { fn _subscript(self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { let f = - self.methods.subscript.load().ok_or_else(|| { + self.slots().subscript.load().ok_or_else(|| { vm.new_type_error(format!("{} is not a mapping", self.obj.class())) })?; f(self, needle, vm) @@ -142,7 +154,7 @@ impl PyMapping<'_> { value: Option<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<()> { - let f = self.methods.ass_subscript.load().ok_or_else(|| { + let f = self.slots().ass_subscript.load().ok_or_else(|| { vm.new_type_error(format!( "'{}' object does not support item assignment", self.obj.class() diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs index d5c7e239a24..411aa4dfad3 100644 --- a/crates/vm/src/protocol/mod.rs +++ b/crates/vm/src/protocol/mod.rs @@ -8,10 +8,11 @@ mod sequence; pub use buffer::{BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer}; pub use callable::PyCallable; +pub(crate) use callable::TraceEvent; pub use iter::{PyIter, PyIterIter, PyIterReturn}; -pub use mapping::{PyMapping, PyMappingMethods}; +pub use mapping::{PyMapping, PyMappingMethods, PyMappingSlots}; pub use number::{ PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots, - PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err, + PyNumberTernaryFunc, PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err, }; -pub use sequence::{PySequence, PySequenceMethods}; +pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots}; diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs index 1242ee52795..542afce2c6c 100644 --- a/crates/vm/src/protocol/number.rs +++ b/crates/vm/src/protocol/number.rs @@ -1,4 +1,4 @@ -use std::ops::Deref; +use core::ops::Deref; use crossbeam_utils::atomic::AtomicCell; @@ -20,8 +20,8 @@ pub type PyNumberTernaryFunc = fn(&PyObject, &PyObject, &PyObject, &VirtualMachi impl PyObject { #[inline] - pub const fn to_number(&self) -> PyNumber<'_> { - PyNumber(self) + pub const fn number(&self) -> PyNumber<'_> { + PyNumber { obj: self } } pub fn try_index_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { @@ -30,7 +30,7 @@ impl PyObject { } else if let Some(i) = self.downcast_ref::<PyInt>() { Some(Ok(vm.ctx.new_bigint(i.as_bigint()))) } else { - self.to_number().index(vm) + self.number().index(vm) } } @@ -56,7 +56,7 @@ impl PyObject { if let Some(i) = self.downcast_ref_if_exact::<PyInt>(vm) { Ok(i.to_owned()) - } else if let Some(i) = self.to_number().int(vm).or_else(|| self.try_index_opt(vm)) { + } else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) { i } else if let Ok(Some(f)) = vm.get_special_method(self, identifier!(vm, __trunc__)) { warnings::warn( @@ -92,7 +92,7 @@ impl PyObject { pub fn try_float_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { if let Some(float) = self.downcast_ref_if_exact::<PyFloat>(vm) { Some(Ok(float.to_owned())) - } else if let Some(f) = self.to_number().float(vm) { + } else if let Some(f) = self.number().float(vm) { Some(f) } else { self.try_index_opt(vm) @@ -256,6 +256,7 @@ pub struct PyNumberSlots { pub int: AtomicCell<Option<PyNumberUnaryFunc>>, pub float: AtomicCell<Option<PyNumberUnaryFunc>>, + // Right variants (internal - not exposed in SlotAccessor) pub right_add: AtomicCell<Option<PyNumberBinaryFunc>>, pub right_subtract: AtomicCell<Option<PyNumberBinaryFunc>>, pub right_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, @@ -295,8 +296,7 @@ pub struct PyNumberSlots { impl From<&PyNumberMethods> for PyNumberSlots { fn from(value: &PyNumberMethods) -> Self { - // right_* functions will use the same left function as PyNumberMethods - // allows both f(self, other) and f(other, self) + // right_* slots use the same function as left ops for native types Self { add: AtomicCell::new(value.add), subtract: AtomicCell::new(value.subtract), @@ -352,6 +352,129 @@ impl From<&PyNumberMethods> for PyNumberSlots { } impl PyNumberSlots { + /// Copy from static PyNumberMethods + pub fn copy_from(&self, methods: &PyNumberMethods) { + if let Some(f) = methods.add { + self.add.store(Some(f)); + self.right_add.store(Some(f)); + } + if let Some(f) = methods.subtract { + self.subtract.store(Some(f)); + self.right_subtract.store(Some(f)); + } + if let Some(f) = methods.multiply { + self.multiply.store(Some(f)); + self.right_multiply.store(Some(f)); + } + if let Some(f) = methods.remainder { + self.remainder.store(Some(f)); + self.right_remainder.store(Some(f)); + } + if let Some(f) = methods.divmod { + self.divmod.store(Some(f)); + self.right_divmod.store(Some(f)); + } + if let Some(f) = methods.power { + self.power.store(Some(f)); + self.right_power.store(Some(f)); + } + if let Some(f) = methods.negative { + self.negative.store(Some(f)); + } + if let Some(f) = methods.positive { + self.positive.store(Some(f)); + } + if let Some(f) = methods.absolute { + self.absolute.store(Some(f)); + } + if let Some(f) = methods.boolean { + self.boolean.store(Some(f)); + } + if let Some(f) = methods.invert { + self.invert.store(Some(f)); + } + if let Some(f) = methods.lshift { + self.lshift.store(Some(f)); + self.right_lshift.store(Some(f)); + } + if let Some(f) = methods.rshift { + self.rshift.store(Some(f)); + self.right_rshift.store(Some(f)); + } + if let Some(f) = methods.and { + self.and.store(Some(f)); + self.right_and.store(Some(f)); + } + if let Some(f) = methods.xor { + self.xor.store(Some(f)); + self.right_xor.store(Some(f)); + } + if let Some(f) = methods.or { + self.or.store(Some(f)); + self.right_or.store(Some(f)); + } + if let Some(f) = methods.int { + self.int.store(Some(f)); + } + if let Some(f) = methods.float { + self.float.store(Some(f)); + } + if let Some(f) = methods.inplace_add { + self.inplace_add.store(Some(f)); + } + if let Some(f) = methods.inplace_subtract { + self.inplace_subtract.store(Some(f)); + } + if let Some(f) = methods.inplace_multiply { + self.inplace_multiply.store(Some(f)); + } + if let Some(f) = methods.inplace_remainder { + self.inplace_remainder.store(Some(f)); + } + if let Some(f) = methods.inplace_power { + self.inplace_power.store(Some(f)); + } + if let Some(f) = methods.inplace_lshift { + self.inplace_lshift.store(Some(f)); + } + if let Some(f) = methods.inplace_rshift { + self.inplace_rshift.store(Some(f)); + } + if let Some(f) = methods.inplace_and { + self.inplace_and.store(Some(f)); + } + if let Some(f) = methods.inplace_xor { + self.inplace_xor.store(Some(f)); + } + if let Some(f) = methods.inplace_or { + self.inplace_or.store(Some(f)); + } + if let Some(f) = methods.floor_divide { + self.floor_divide.store(Some(f)); + self.right_floor_divide.store(Some(f)); + } + if let Some(f) = methods.true_divide { + self.true_divide.store(Some(f)); + self.right_true_divide.store(Some(f)); + } + if let Some(f) = methods.inplace_floor_divide { + self.inplace_floor_divide.store(Some(f)); + } + if let Some(f) = methods.inplace_true_divide { + self.inplace_true_divide.store(Some(f)); + } + if let Some(f) = methods.index { + self.index.store(Some(f)); + } + if let Some(f) = methods.matrix_multiply { + self.matrix_multiply.store(Some(f)); + self.right_matrix_multiply.store(Some(f)); + } + if let Some(f) = methods.inplace_matrix_multiply { + self.inplace_matrix_multiply.store(Some(f)); + } + } + pub fn left_binary_op(&self, op_slot: PyNumberBinaryOp) -> Option<PyNumberBinaryFunc> { use PyNumberBinaryOp::*; match op_slot { @@ -420,11 +543,13 @@ impl PyNumberSlots { } } #[derive(Copy, Clone)] -pub struct PyNumber<'a>(&'a PyObject); +pub struct PyNumber<'a> { + pub obj: &'a PyObject, +} unsafe impl Traverse for PyNumber<'_> { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.0.traverse(tracer_fn) + self.obj.traverse(tracer_fn) } } @@ -432,36 +557,17 @@ impl Deref for PyNumber<'_> { type Target = PyObject; fn deref(&self) -> &Self::Target { - self.0 + self.obj } } impl<'a> PyNumber<'a> { - pub(crate) const fn obj(self) -> &'a PyObject { - self.0 - } - - // PyNumber_Check + // PyNumber_Check - slots are now inherited pub fn check(obj: &PyObject) -> bool { - let cls = &obj.class(); - // TODO: when we finally have a proper slot inheritance, mro_find_map can be removed - // methods.int.load().is_some() - // || methods.index.load().is_some() - // || methods.float.load().is_some() - // || obj.downcastable::<PyComplex>() - let has_number = cls - .mro_find_map(|x| { - let methods = &x.slots.as_number; - if methods.int.load().is_some() - || methods.index.load().is_some() - || methods.float.load().is_some() - { - Some(()) - } else { - None - } - }) - .is_some(); + let methods = &obj.class().slots.as_number; + let has_number = methods.int.load().is_some() + || methods.index.load().is_some() + || methods.float.load().is_some(); has_number || obj.downcastable::<PyComplex>() } } @@ -469,114 +575,106 @@ impl<'a> PyNumber<'a> { impl PyNumber<'_> { // PyIndex_Check pub fn is_index(self) -> bool { - self.class() - .mro_find_map(|x| x.slots.as_number.index.load()) - .is_some() + self.class().slots.as_number.index.load().is_some() } #[inline] pub fn int(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { - self.class() - .mro_find_map(|x| x.slots.as_number.int.load()) - .map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyInt>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__int__ returned non-int (type {ret_class}). \ + self.class().slots.as_number.int.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyInt>() { + warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__int__ returned non-int (type {ret_class}). \ The ability to return an instance of a strict subclass of int \ is deprecated, and may be removed in a future version of Python." - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__int__ returned non-int(type {})", - self.class(), - ret_class - ))) - } - }) + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__int__ returned non-int(type {})", + self.class(), + ret_class + ))) + } + }) } #[inline] pub fn index(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { - self.class() - .mro_find_map(|x| x.slots.as_number.index.load()) - .map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyInt>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__index__ returned non-int (type {ret_class}). \ + self.class().slots.as_number.index.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyInt>() { + warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__index__ returned non-int (type {ret_class}). \ The ability to return an instance of a strict subclass of int \ is deprecated, and may be removed in a future version of Python." - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__index__ returned non-int(type {})", - self.class(), - ret_class - ))) - } - }) + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__index__ returned non-int(type {})", + self.class(), + ret_class + ))) + } + }) } #[inline] pub fn float(self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { - self.class() - .mro_find_map(|x| x.slots.as_number.float.load()) - .map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyFloat>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyFloat>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__float__ returned non-float (type {ret_class}). \ + self.class().slots.as_number.float.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyFloat>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyFloat>() { + warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__float__ returned non-float (type {ret_class}). \ The ability to return an instance of a strict subclass of float \ is deprecated, and may be removed in a future version of Python." - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__float__ returned non-float(type {})", - self.class(), - ret_class - ))) - } - }) + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__float__ returned non-float(type {})", + self.class(), + ret_class + ))) + } + }) } } diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index f2e52a94004..d2e4b31aaea 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -12,7 +12,7 @@ use crate::{ dict_inner::DictKey, function::{Either, FuncArgs, PyArithmeticValue, PySetterValue}, object::PyPayload, - protocol::{PyIter, PyMapping, PySequence}, + protocol::PyIter, types::{Constructor, PyComparisonOp}, }; @@ -136,10 +136,7 @@ impl PyObject { #[inline] pub(crate) fn get_attr_inner(&self, attr_name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { vm_trace!("object.__getattribute__: {:?} {:?}", self, attr_name); - let getattro = self - .class() - .mro_find_map(|cls| cls.slots.getattro.load()) - .unwrap(); + let getattro = self.class().slots.getattro.load().unwrap(); getattro(self, attr_name, vm).inspect_err(|exc| { vm.set_attribute_error_context(exc, self.to_owned(), attr_name.to_owned()); }) @@ -153,21 +150,20 @@ impl PyObject { ) -> PyResult<()> { let setattro = { let cls = self.class(); - cls.mro_find_map(|cls| cls.slots.setattro.load()) - .ok_or_else(|| { - let has_getattr = cls.mro_find_map(|cls| cls.slots.getattro.load()).is_some(); - vm.new_type_error(format!( - "'{}' object has {} attributes ({} {})", - cls.name(), - if has_getattr { "only read-only" } else { "no" }, - if attr_value.is_assign() { - "assign to" - } else { - "del" - }, - attr_name - )) - })? + cls.slots.setattro.load().ok_or_else(|| { + let has_getattr = cls.slots.getattro.load().is_some(); + vm.new_type_error(format!( + "'{}' object has {} attributes ({} {})", + cls.name(), + if has_getattr { "only read-only" } else { "no" }, + if attr_value.is_assign() { + "assign to" + } else { + "del" + }, + attr_name + )) + })? }; setattro(self, attr_name, attr_value, vm) } @@ -197,7 +193,7 @@ impl PyObject { .interned_str(attr_name) .and_then(|attr_name| self.get_class_attr(attr_name)) { - let descr_set = attr.class().mro_find_map(|cls| cls.slots.descr_set.load()); + let descr_set = attr.class().slots.descr_set.load(); if let Some(descriptor) = descr_set { return descriptor(&attr, self.to_owned(), value, vm); } @@ -239,11 +235,9 @@ impl PyObject { let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { Some(descr) => { let descr_cls = descr.class(); - let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = descr_cls.slots.descr_get.load(); if let Some(descr_get) = descr_get - && descr_cls - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some() + && descr_cls.slots.descr_set.load().is_some() { let cls = obj_cls.to_owned().into(); return descr_get(descr, Some(self.to_owned()), Some(cls), vm).map(Some); @@ -293,10 +287,7 @@ impl PyObject { ) -> PyResult<Either<PyObjectRef, bool>> { let swapped = op.swapped(); let call_cmp = |obj: &Self, other: &Self, op| { - let cmp = obj - .class() - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); + let cmp = obj.class().slots.richcompare.load().unwrap(); let r = match cmp(obj, other, op, vm)? { Either::A(obj) => PyArithmeticValue::from_object(vm, obj).map(Either::A), Either::B(arithmetic) => arithmetic.map(Either::B), @@ -331,7 +322,12 @@ impl PyObject { match op { PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), - _ => Err(vm.new_unsupported_bin_op_error(self, other, op.operator_token())), + _ => Err(vm.new_type_error(format!( + "'{}' not supported between instances of '{}' and '{}'", + op.operator_token(), + self.class().name(), + other.class().name() + ))), } } #[inline(always)] @@ -353,18 +349,15 @@ impl PyObject { pub fn repr(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { vm.with_recursion("while getting the repr of an object", || { - // TODO: RustPython does not implement type slots inheritance yet - self.class() - .mro_find_map(|cls| cls.slots.repr.load()) - .map_or_else( - || { - Err(vm.new_runtime_error(format!( + self.class().slots.repr.load().map_or_else( + || { + Err(vm.new_runtime_error(format!( "BUG: object of type '{}' has no __repr__ method. This is a bug in RustPython.", self.class().name() ))) - }, - |repr| repr(self, vm), - ) + }, + |repr| repr(self, vm), + ) }) } @@ -659,7 +652,7 @@ impl PyObject { } pub fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { - if let Some(hash) = self.class().mro_find_map(|cls| cls.slots.hash.load()) { + if let Some(hash) = self.class().slots.hash.load() { return hash(self, vm); } @@ -681,9 +674,9 @@ impl PyObject { } pub fn length_opt(&self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.to_sequence() + self.sequence_unchecked() .length_opt(vm) - .or_else(|| self.to_mapping().length_opt(vm)) + .or_else(|| self.mapping_unchecked().length_opt(vm)) } pub fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { @@ -702,9 +695,9 @@ impl PyObject { let needle = needle.to_pyobject(vm); - if let Ok(mapping) = PyMapping::try_protocol(self, vm) { + if let Ok(mapping) = self.try_mapping(vm) { mapping.subscript(&needle, vm) - } else if let Ok(seq) = PySequence::try_protocol(self, vm) { + } else if let Ok(seq) = self.try_sequence(vm) { let i = needle.key_as_isize(vm)?; seq.get_item(i, vm) } else { @@ -716,9 +709,14 @@ impl PyObject { if let Some(class_getitem) = vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class_getitem__))? + && !vm.is_none(&class_getitem) { return class_getitem.call((needle,), vm); } + return Err(vm.new_type_error(format!( + "type '{}' is not subscriptable", + self.downcast_ref::<PyType>().unwrap().name() + ))); } Err(vm.new_type_error(format!("'{}' object is not subscriptable", self.class()))) } @@ -734,14 +732,14 @@ impl PyObject { return dict.set_item(needle, value, vm); } - let mapping = self.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { + let mapping = self.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { let needle = needle.to_pyobject(vm); return f(mapping, &needle, Some(value), vm); } - let seq = self.to_sequence(); - if let Some(f) = seq.methods.ass_item.load() { + let seq = self.sequence_unchecked(); + if let Some(f) = seq.slots().ass_item.load() { let i = needle.key_as_isize(vm)?; return f(seq, i, Some(value), vm); } @@ -757,13 +755,13 @@ impl PyObject { return dict.del_item(needle, vm); } - let mapping = self.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { + let mapping = self.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { let needle = needle.to_pyobject(vm); return f(mapping, &needle, None, vm); } - let seq = self.to_sequence(); - if let Some(f) = seq.methods.ass_item.load() { + let seq = self.sequence_unchecked(); + if let Some(f) = seq.slots().ass_item.load() { let i = needle.key_as_isize(vm)?; return f(seq, i, None, vm); } @@ -781,7 +779,7 @@ impl PyObject { let res = obj_cls.lookup_ref(attr, vm)?; // If it's a descriptor, call its __get__ method - let descr_get = res.class().mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = res.class().slots.descr_get.load(); if let Some(descr_get) = descr_get { let obj_cls = obj_cls.to_owned().into(); // CPython ignores exceptions in _PyObject_LookupSpecial and returns NULL diff --git a/crates/vm/src/protocol/sequence.rs b/crates/vm/src/protocol/sequence.rs index fb71446a5a4..cee46a29089 100644 --- a/crates/vm/src/protocol/sequence.rs +++ b/crates/vm/src/protocol/sequence.rs @@ -1,33 +1,20 @@ use crate::{ PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyList, PyListRef, PySlice, PyTuple, PyTupleRef, type_::PointerSlot}, + builtins::{PyList, PyListRef, PySlice, PyTuple, PyTupleRef}, convert::ToPyObject, function::PyArithmeticValue, object::{Traverse, TraverseFn}, - protocol::{PyMapping, PyNumberBinaryOp}, + protocol::PyNumberBinaryOp, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; -use std::fmt::Debug; // Sequence Protocol // https://docs.python.org/3/c-api/sequence.html -impl PyObject { - #[inline] - pub fn to_sequence(&self) -> PySequence<'_> { - static GLOBAL_NOT_IMPLEMENTED: PySequenceMethods = PySequenceMethods::NOT_IMPLEMENTED; - PySequence { - obj: self, - methods: PySequence::find_methods(self) - .map_or(&GLOBAL_NOT_IMPLEMENTED, |x| unsafe { x.borrow_static() }), - } - } -} - #[allow(clippy::type_complexity)] #[derive(Default)] -pub struct PySequenceMethods { +pub struct PySequenceSlots { pub length: AtomicCell<Option<fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>>>, pub concat: AtomicCell<Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>>, pub repeat: AtomicCell<Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>>, @@ -42,66 +29,118 @@ pub struct PySequenceMethods { pub inplace_repeat: AtomicCell<Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>>, } -impl Debug for PySequenceMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Sequence Methods") +impl core::fmt::Debug for PySequenceSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PySequenceSlots") } } -impl PySequenceMethods { - #[allow(clippy::declare_interior_mutable_const)] - pub const NOT_IMPLEMENTED: Self = Self { - length: AtomicCell::new(None), - concat: AtomicCell::new(None), - repeat: AtomicCell::new(None), - item: AtomicCell::new(None), - ass_item: AtomicCell::new(None), - contains: AtomicCell::new(None), - inplace_concat: AtomicCell::new(None), - inplace_repeat: AtomicCell::new(None), - }; +impl PySequenceSlots { + pub fn has_item(&self) -> bool { + self.item.load().is_some() + } + + /// Copy from static PySequenceMethods + pub fn copy_from(&self, methods: &PySequenceMethods) { + if let Some(f) = methods.length { + self.length.store(Some(f)); + } + if let Some(f) = methods.concat { + self.concat.store(Some(f)); + } + if let Some(f) = methods.repeat { + self.repeat.store(Some(f)); + } + if let Some(f) = methods.item { + self.item.store(Some(f)); + } + if let Some(f) = methods.ass_item { + self.ass_item.store(Some(f)); + } + if let Some(f) = methods.contains { + self.contains.store(Some(f)); + } + if let Some(f) = methods.inplace_concat { + self.inplace_concat.store(Some(f)); + } + if let Some(f) = methods.inplace_repeat { + self.inplace_repeat.store(Some(f)); + } + } } -#[derive(Copy, Clone)] -pub struct PySequence<'a> { - pub obj: &'a PyObject, - pub methods: &'static PySequenceMethods, +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PySequenceMethods { + pub length: Option<fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>>, + pub concat: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub repeat: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, + pub item: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, + pub ass_item: + Option<fn(PySequence<'_>, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, + pub contains: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult<bool>>, + pub inplace_concat: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub inplace_repeat: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, } -unsafe impl Traverse for PySequence<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.obj.traverse(tracer_fn) +impl core::fmt::Debug for PySequenceMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PySequenceMethods") } } -impl<'a> PySequence<'a> { +impl PySequenceMethods { + pub const NOT_IMPLEMENTED: Self = Self { + length: None, + concat: None, + repeat: None, + item: None, + ass_item: None, + contains: None, + inplace_concat: None, + inplace_repeat: None, + }; +} + +impl PyObject { #[inline] - pub const fn with_methods(obj: &'a PyObject, methods: &'static PySequenceMethods) -> Self { - Self { obj, methods } + pub fn sequence_unchecked(&self) -> PySequence<'_> { + PySequence { obj: self } } - pub fn try_protocol(obj: &'a PyObject, vm: &VirtualMachine) -> PyResult<Self> { - let seq = obj.to_sequence(); + pub fn try_sequence(&self, vm: &VirtualMachine) -> PyResult<PySequence<'_>> { + let seq = self.sequence_unchecked(); if seq.check() { Ok(seq) } else { - Err(vm.new_type_error(format!("'{}' is not a sequence", obj.class()))) + Err(vm.new_type_error(format!("'{}' is not a sequence", self.class()))) } } } +#[derive(Copy, Clone)] +pub struct PySequence<'a> { + pub obj: &'a PyObject, +} + +unsafe impl Traverse for PySequence<'_> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.obj.traverse(tracer_fn) + } +} + impl PySequence<'_> { - pub fn check(&self) -> bool { - self.methods.item.load().is_some() + #[inline] + pub fn slots(&self) -> &PySequenceSlots { + &self.obj.class().slots.as_sequence } - pub fn find_methods(obj: &PyObject) -> Option<PointerSlot<PySequenceMethods>> { - let cls = obj.class(); - cls.mro_find_map(|x| x.slots.as_sequence.load()) + pub fn check(&self) -> bool { + self.slots().has_item() } pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.methods.length.load().map(|f| f(self, vm)) + self.slots().length.load().map(|f| f(self, vm)) } pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { @@ -114,12 +153,12 @@ impl PySequence<'_> { } pub fn concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.concat.load() { + if let Some(f) = self.slots().concat.load() { return f(self, other, vm); } // if both arguments appear to be sequences, try fallback to __add__ - if self.check() && other.to_sequence().check() { + if self.check() && other.sequence_unchecked().check() { let ret = vm.binary_op1(self.obj, other, PyNumberBinaryOp::Add)?; if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { return Ok(ret); @@ -133,7 +172,7 @@ impl PySequence<'_> { } pub fn repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.repeat.load() { + if let Some(f) = self.slots().repeat.load() { return f(self, n, vm); } @@ -149,15 +188,15 @@ impl PySequence<'_> { } pub fn inplace_concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.inplace_concat.load() { + if let Some(f) = self.slots().inplace_concat.load() { return f(self, other, vm); } - if let Some(f) = self.methods.concat.load() { + if let Some(f) = self.slots().concat.load() { return f(self, other, vm); } // if both arguments appear to be sequences, try fallback to __iadd__ - if self.check() && other.to_sequence().check() { + if self.check() && other.sequence_unchecked().check() { let ret = vm._iadd(self.obj, other)?; if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { return Ok(ret); @@ -171,10 +210,10 @@ impl PySequence<'_> { } pub fn inplace_repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.inplace_repeat.load() { + if let Some(f) = self.slots().inplace_repeat.load() { return f(self, n, vm); } - if let Some(f) = self.methods.repeat.load() { + if let Some(f) = self.slots().repeat.load() { return f(self, n, vm); } @@ -189,7 +228,7 @@ impl PySequence<'_> { } pub fn get_item(self, i: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.item.load() { + if let Some(f) = self.slots().item.load() { return f(self, i, vm); } Err(vm.new_type_error(format!( @@ -199,7 +238,7 @@ impl PySequence<'_> { } fn _ass_item(self, i: isize, value: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { - if let Some(f) = self.methods.ass_item.load() { + if let Some(f) = self.slots().ass_item.load() { return f(self, i, value, vm); } Err(vm.new_type_error(format!( @@ -222,7 +261,7 @@ impl PySequence<'_> { } pub fn get_slice(&self, start: isize, stop: isize, vm: &VirtualMachine) -> PyResult { - if let Ok(mapping) = PyMapping::try_protocol(self.obj, vm) { + if let Ok(mapping) = self.obj.try_mapping(vm) { let slice = PySlice { start: Some(start.to_pyobject(vm)), stop: stop.to_pyobject(vm), @@ -241,8 +280,8 @@ impl PySequence<'_> { value: Option<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<()> { - let mapping = self.obj.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { + let mapping = self.obj.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { let slice = PySlice { start: Some(start.to_pyobject(vm)), stop: stop.to_pyobject(vm), @@ -355,7 +394,7 @@ impl PySequence<'_> { } pub fn contains(self, target: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - if let Some(f) = self.methods.contains.load() { + if let Some(f) = self.slots().contains.load() { return f(self, target, vm); } diff --git a/crates/vm/src/py_io.rs b/crates/vm/src/py_io.rs index 6b82bbd4788..5649463b30e 100644 --- a/crates/vm/src/py_io.rs +++ b/crates/vm/src/py_io.rs @@ -3,7 +3,9 @@ use crate::{ builtins::{PyBaseExceptionRef, PyBytes, PyStr}, common::ascii, }; -use std::{fmt, io, ops}; +use alloc::fmt; +use core::ops; +use std::io; pub trait Write { type Error; @@ -70,11 +72,14 @@ pub fn file_readline(obj: &PyObject, size: Option<usize>, vm: &VirtualMachine) - }; let ret = match_class!(match ret { s @ PyStr => { - let s_val = s.as_str(); - if s_val.is_empty() { + // Use as_wtf8() to handle strings with surrogates (e.g., surrogateescape) + let s_wtf8 = s.as_wtf8(); + if s_wtf8.is_empty() { return Err(eof_err()); } - if let Some(no_nl) = s_val.strip_suffix('\n') { + // '\n' is ASCII, so we can check bytes directly + if s_wtf8.as_bytes().last() == Some(&b'\n') { + let no_nl = &s_wtf8[..s_wtf8.len() - 1]; vm.ctx.new_str(no_nl).into() } else { s.into() diff --git a/crates/vm/src/py_serde.rs b/crates/vm/src/py_serde.rs index f9a5f4bc060..945068113f1 100644 --- a/crates/vm/src/py_serde.rs +++ b/crates/vm/src/py_serde.rs @@ -130,7 +130,7 @@ impl<'de> DeserializeSeed<'de> for PyObjectDeserializer<'de> { impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { type Value = PyObjectRef; - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.write_str("a type that can deserialize in Python") } diff --git a/crates/vm/src/readline.rs b/crates/vm/src/readline.rs index 77402dc6839..d62d520ecbd 100644 --- a/crates/vm/src/readline.rs +++ b/crates/vm/src/readline.rs @@ -5,7 +5,7 @@ use std::{io, path::Path}; -type OtherError = Box<dyn std::error::Error>; +type OtherError = Box<dyn core::error::Error>; type OtherResult<T> = Result<T, OtherError>; pub enum ReadlineResult { diff --git a/crates/vm/src/scope.rs b/crates/vm/src/scope.rs index 9311fa5c2db..74392dc9b73 100644 --- a/crates/vm/src/scope.rs +++ b/crates/vm/src/scope.rs @@ -1,5 +1,5 @@ use crate::{VirtualMachine, builtins::PyDictRef, function::ArgMapping}; -use std::fmt; +use alloc::fmt; #[derive(Clone)] pub struct Scope { @@ -34,7 +34,7 @@ impl Scope { Self::new(locals, globals) } - // pub fn get_locals(&self) -> &PyDictRef { + // pub fn get_locals(&self) -> &Py<PyDict> { // match self.locals.first() { // Some(dict) => dict, // None => &self.globals, diff --git a/crates/vm/src/sequence.rs b/crates/vm/src/sequence.rs index e75c0a6da55..0bc12fd2631 100644 --- a/crates/vm/src/sequence.rs +++ b/crates/vm/src/sequence.rs @@ -6,13 +6,13 @@ use crate::{ types::PyComparisonOp, vm::{MAX_MEMORY_SIZE, VirtualMachine}, }; +use core::ops::{Deref, Range}; use optional::Optioned; -use std::ops::{Deref, Range}; pub trait MutObjectSequenceOp { type Inner: ?Sized; - fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObjectRef>; + fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObject>; fn do_lock(&self) -> impl Deref<Target = Self::Inner>; fn mut_count(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<usize> { @@ -76,7 +76,7 @@ pub trait MutObjectSequenceOp { } borrower = Some(guard); } else { - let elem = elem.clone(); + let elem = elem.to_owned(); drop(guard); if elem.rich_compare_bool(needle, PyComparisonOp::Eq, vm)? { @@ -100,7 +100,7 @@ where fn mul(&self, vm: &VirtualMachine, n: isize) -> PyResult<Vec<T>> { let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; - if n > 1 && std::mem::size_of_val(self.as_ref()) >= MAX_MEMORY_SIZE / n { + if n > 1 && core::mem::size_of_val(self.as_ref()) >= MAX_MEMORY_SIZE / n { return Err(vm.new_memory_error("")); } diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 1074c8e8f11..f146a7321a0 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -1,24 +1,47 @@ #![cfg_attr(target_os = "wasi", allow(dead_code))] -use crate::{PyResult, VirtualMachine}; -use std::{ - fmt, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc, - }, -}; +use crate::{PyObjectRef, PyResult, VirtualMachine}; +use alloc::fmt; +use core::cell::{Cell, RefCell}; +#[cfg(windows)] +use core::sync::atomic::AtomicIsize; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; pub(crate) const NSIG: usize = 64; + +pub(crate) fn new_signal_handlers() -> Box<RefCell<[Option<PyObjectRef>; NSIG]>> { + Box::new(const { RefCell::new([const { None }; NSIG]) }) +} static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); // hack to get around const array repeat expressions, rust issue #79270 -#[allow(clippy::declare_interior_mutable_const)] +#[allow( + clippy::declare_interior_mutable_const, + reason = "workaround for const array repeat limitation (rust issue #79270)" +)] const ATOMIC_FALSE: AtomicBool = AtomicBool::new(false); pub(crate) static TRIGGERS: [AtomicBool; NSIG] = [ATOMIC_FALSE; NSIG]; +#[cfg(windows)] +static SIGINT_EVENT: AtomicIsize = AtomicIsize::new(0); + +thread_local! { + /// Prevent recursive signal handler invocation. When a Python signal + /// handler is running, new signals are deferred until it completes. + static IN_SIGNAL_HANDLER: Cell<bool> = const { Cell::new(false) }; +} + +struct SignalHandlerGuard; + +impl Drop for SignalHandlerGuard { + fn drop(&mut self) { + IN_SIGNAL_HANDLER.with(|h| h.set(false)); + } +} + #[cfg_attr(feature = "flame-it", flame)] #[inline(always)] pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { - if vm.signal_handlers.is_none() { + if vm.signal_handlers.get().is_none() { return Ok(()); } @@ -31,8 +54,15 @@ pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { #[inline(never)] #[cold] fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { + if IN_SIGNAL_HANDLER.with(|h| h.replace(true)) { + // Already inside a signal handler — defer pending signals + set_triggered(); + return Ok(()); + } + let _guard = SignalHandlerGuard; + // unwrap should never fail since we check above - let signal_handlers = vm.signal_handlers.as_ref().unwrap().borrow(); + let signal_handlers = vm.signal_handlers.get().unwrap().borrow(); for (signum, trigger) in TRIGGERS.iter().enumerate().skip(1) { let triggered = trigger.swap(false, Ordering::Relaxed); if triggered @@ -54,6 +84,17 @@ pub(crate) fn set_triggered() { ANY_TRIGGERED.store(true, Ordering::Release); } +/// Reset all signal trigger state after fork in child process. +/// Stale triggers from the parent must not fire in the child. +#[cfg(unix)] +#[cfg(feature = "host_env")] +pub(crate) fn clear_after_fork() { + ANY_TRIGGERED.store(false, Ordering::Release); + for trigger in &TRIGGERS { + trigger.store(false, Ordering::Relaxed); + } +} + pub fn assert_in_range(signum: i32, vm: &VirtualMachine) -> PyResult<()> { if (1..NSIG as i32).contains(&signum) { Ok(()) @@ -66,7 +107,7 @@ pub fn assert_in_range(signum: i32, vm: &VirtualMachine) -> PyResult<()> { /// /// Missing signal handler for the given signal number is silently ignored. #[allow(dead_code)] -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] pub fn set_interrupt_ex(signum: i32, vm: &VirtualMachine) -> PyResult<()> { use crate::stdlib::signal::_signal::{SIG_DFL, SIG_IGN, run_signal}; assert_in_range(signum, vm)?; @@ -122,3 +163,14 @@ pub fn user_signal_channel() -> (UserSignalSender, UserSignalReceiver) { let (tx, rx) = mpsc::channel(); (UserSignalSender { tx }, UserSignalReceiver { rx }) } + +#[cfg(windows)] +pub fn set_sigint_event(handle: isize) { + SIGINT_EVENT.store(handle, Ordering::Release); +} + +#[cfg(windows)] +pub fn get_sigint_event() -> Option<isize> { + let handle = SIGINT_EVENT.load(Ordering::Acquire); + if handle == 0 { None } else { Some(handle) } +} diff --git a/crates/vm/src/sliceable.rs b/crates/vm/src/sliceable.rs index 786b66fb36a..e416f5a1b49 100644 --- a/crates/vm/src/sliceable.rs +++ b/crates/vm/src/sliceable.rs @@ -3,9 +3,9 @@ use crate::{ PyObject, PyResult, VirtualMachine, builtins::{int::PyInt, slice::PySlice}, }; +use core::ops::Range; use malachite_bigint::BigInt; use num_traits::{Signed, ToPrimitive}; -use std::ops::Range; pub trait SliceableSequenceMutOp where diff --git a/crates/vm/src/stdlib/_abc.rs b/crates/vm/src/stdlib/_abc.rs new file mode 100644 index 00000000000..72642c249f5 --- /dev/null +++ b/crates/vm/src/stdlib/_abc.rs @@ -0,0 +1,481 @@ +//! Implementation of the `_abc` module. +//! +//! This module provides the C implementation of Abstract Base Classes (ABCs) +//! as defined in PEP 3119. + +pub(crate) use _abc::module_def; + +#[pymodule] +mod _abc { + use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyFrozenSet, PyList, PySet, PyStr, PyTupleRef, PyTypeRef, PyWeak}, + common::lock::PyRwLock, + convert::ToPyObject, + protocol::PyIterReturn, + types::Constructor, + }; + use core::sync::atomic::{AtomicU64, Ordering}; + + // Global invalidation counter + static ABC_INVALIDATION_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn get_invalidation_counter() -> u64 { + ABC_INVALIDATION_COUNTER.load(Ordering::SeqCst) + } + + fn increment_invalidation_counter() { + ABC_INVALIDATION_COUNTER.fetch_add(1, Ordering::SeqCst); + } + + /// Internal state held by ABC machinery. + #[pyattr] + #[pyclass(name = "_abc_data", module = "_abc")] + #[derive(Debug, PyPayload)] + struct AbcData { + // WeakRef sets for registry and caches + registry: PyRwLock<Option<PyRef<PySet>>>, + cache: PyRwLock<Option<PyRef<PySet>>>, + negative_cache: PyRwLock<Option<PyRef<PySet>>>, + negative_cache_version: AtomicU64, + } + + #[pyclass(with(Constructor))] + impl AbcData { + fn new() -> Self { + AbcData { + registry: PyRwLock::new(None), + cache: PyRwLock::new(None), + negative_cache: PyRwLock::new(None), + negative_cache_version: AtomicU64::new(get_invalidation_counter()), + } + } + + fn get_cache_version(&self) -> u64 { + self.negative_cache_version.load(Ordering::SeqCst) + } + + fn set_cache_version(&self, version: u64) { + self.negative_cache_version.store(version, Ordering::SeqCst); + } + } + + impl Constructor for AbcData { + type Args = (); + + fn py_new( + _cls: &crate::Py<crate::builtins::PyType>, + _args: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(AbcData::new()) + } + } + + /// Get the _abc_impl attribute from an ABC class + fn get_impl(cls: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<AbcData>> { + let impl_obj = cls.get_attr("_abc_impl", vm)?; + impl_obj + .downcast::<AbcData>() + .map_err(|_| vm.new_type_error("_abc_impl is set to a wrong type".to_owned())) + } + + /// Check if obj is in the weak set + fn in_weak_set( + set_lock: &PyRwLock<Option<PyRef<PySet>>>, + obj: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<bool> { + let set_opt = set_lock.read(); + let set = match &*set_opt { + Some(s) if !s.elements().is_empty() => s.clone(), + _ => return Ok(false), + }; + drop(set_opt); + + // Create a weak reference to the object + let weak_ref = match obj.downgrade(None, vm) { + Ok(w) => w, + Err(e) => { + // If we can't create a weakref (e.g., TypeError), the object can't be in the set + if e.class().is(vm.ctx.exceptions.type_error) { + return Ok(false); + } + return Err(e); + } + }; + + // Use vm.call_method to call __contains__ + let weak_ref_obj: PyObjectRef = weak_ref.into(); + vm.call_method(set.as_ref(), "__contains__", (weak_ref_obj,))? + .try_to_bool(vm) + } + + /// Add obj to the weak set + fn add_to_weak_set( + set_lock: &PyRwLock<Option<PyRef<PySet>>>, + obj: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut set_opt = set_lock.write(); + let set = match &*set_opt { + Some(s) => s.clone(), + None => { + let new_set = PySet::default().into_ref(&vm.ctx); + *set_opt = Some(new_set.clone()); + new_set + } + }; + drop(set_opt); + + // Create a weak reference to the object + let weak_ref = obj.downgrade(None, vm)?; + set.add(weak_ref.into(), vm)?; + Ok(()) + } + + /// Returns the current ABC cache token. + #[pyfunction] + fn get_cache_token() -> u64 { + get_invalidation_counter() + } + + /// Compute set of abstract method names. + fn compute_abstract_methods(cls: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let mut abstracts = Vec::new(); + + // Stage 1: direct abstract methods + let ns = cls.get_attr("__dict__", vm)?; + let items = vm.call_method(&ns, "items", ())?; + let iter = items.get_iter(vm)?; + + while let PyIterReturn::Return(item) = iter.next(vm)? { + let tuple: PyTupleRef = item + .downcast() + .map_err(|_| vm.new_type_error("items() returned non-tuple".to_owned()))?; + let elements = tuple.as_slice(); + if elements.len() != 2 { + return Err( + vm.new_type_error("items() returned item which size is not 2".to_owned()) + ); + } + let key = &elements[0]; + let value = &elements[1]; + + // Check if value has __isabstractmethod__ = True + if let Ok(is_abstract) = value.get_attr("__isabstractmethod__", vm) + && is_abstract.try_to_bool(vm)? + { + abstracts.push(key.clone()); + } + } + + // Stage 2: inherited abstract methods + let bases: PyTupleRef = cls + .get_attr("__bases__", vm)? + .downcast() + .map_err(|_| vm.new_type_error("__bases__ is not a tuple".to_owned()))?; + + for base in bases.iter() { + if let Ok(base_abstracts) = base.get_attr("__abstractmethods__", vm) { + let iter = base_abstracts.get_iter(vm)?; + while let PyIterReturn::Return(key) = iter.next(vm)? { + // Try to get the attribute from cls - key should be a string + if let Some(key_str) = key.downcast_ref::<PyStr>() + && let Some(value) = vm.get_attribute_opt(cls.to_owned(), key_str)? + && let Ok(is_abstract) = value.get_attr("__isabstractmethod__", vm) + && is_abstract.try_to_bool(vm)? + { + abstracts.push(key); + } + } + } + } + + // Set __abstractmethods__ + let abstracts_set = PyFrozenSet::from_iter(vm, abstracts.into_iter())?; + cls.set_attr("__abstractmethods__", abstracts_set.into_pyobject(vm), vm)?; + + Ok(()) + } + + /// Internal ABC helper for class set-up. Should be never used outside abc module. + #[pyfunction] + fn _abc_init(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + compute_abstract_methods(&cls, vm)?; + + // Set up inheritance registry + let data = AbcData::new(); + cls.set_attr("_abc_impl", data.to_pyobject(vm), vm)?; + + Ok(()) + } + + /// Internal ABC helper for subclass registration. Should be never used outside abc module. + #[pyfunction] + fn _abc_register( + cls: PyObjectRef, + subclass: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // Type check + if !subclass.class().fast_issubclass(vm.ctx.types.type_type) { + return Err(vm.new_type_error("Can only register classes".to_owned())); + } + + // Check if already a subclass + if subclass.is_subclass(&cls, vm)? { + return Ok(subclass); + } + + // Check for cycles + if cls.is_subclass(&subclass, vm)? { + return Err(vm.new_runtime_error("Refusing to create an inheritance cycle".to_owned())); + } + + // Add to registry + let impl_data = get_impl(&cls, vm)?; + add_to_weak_set(&impl_data.registry, &subclass, vm)?; + + // Invalidate negative cache + increment_invalidation_counter(); + + Ok(subclass) + } + + /// Internal ABC helper for instance checks. Should be never used outside abc module. + #[pyfunction] + fn _abc_instancecheck( + cls: PyObjectRef, + instance: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let impl_data = get_impl(&cls, vm)?; + + // Get instance.__class__ + let subclass = instance.get_attr("__class__", vm)?; + + // Check cache + if in_weak_set(&impl_data.cache, &subclass, vm)? { + return Ok(vm.ctx.true_value.clone().into()); + } + + let subtype: PyObjectRef = instance.class().to_owned().into(); + if subtype.is(&subclass) { + let invalidation_counter = get_invalidation_counter(); + if impl_data.get_cache_version() == invalidation_counter + && in_weak_set(&impl_data.negative_cache, &subclass, vm)? + { + return Ok(vm.ctx.false_value.clone().into()); + } + // Fall back to __subclasscheck__ + return vm.call_method(&cls, "__subclasscheck__", (subclass,)); + } + + // Call __subclasscheck__ on subclass + let result = vm.call_method(&cls, "__subclasscheck__", (subclass.clone(),))?; + + match result.clone().try_to_bool(vm) { + Ok(true) => Ok(result), + Ok(false) => { + // Also try with subtype + vm.call_method(&cls, "__subclasscheck__", (subtype,)) + } + Err(e) => Err(e), + } + } + + /// Check if subclass is in registry (recursive) + fn subclasscheck_check_registry( + impl_data: &AbcData, + subclass: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<Option<bool>> { + // Fast path: check if subclass is in weakref directly + if in_weak_set(&impl_data.registry, subclass, vm)? { + return Ok(Some(true)); + } + + let registry_opt = impl_data.registry.read(); + let registry = match &*registry_opt { + Some(s) => s.clone(), + None => return Ok(None), + }; + drop(registry_opt); + + // Make a local copy to protect against concurrent modifications + let registry_copy = PyFrozenSet::from_iter(vm, registry.elements().into_iter())?; + + for weak_ref_obj in registry_copy.elements() { + if let Ok(weak_ref) = weak_ref_obj.downcast::<PyWeak>() + && let Some(rkey) = weak_ref.upgrade() + && subclass.to_owned().is_subclass(&rkey, vm)? + { + add_to_weak_set(&impl_data.cache, subclass, vm)?; + return Ok(Some(true)); + } + } + + Ok(None) + } + + /// Internal ABC helper for subclass checks. Should be never used outside abc module. + #[pyfunction] + fn _abc_subclasscheck( + cls: PyObjectRef, + subclass: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // Type check + if !subclass.class().fast_issubclass(vm.ctx.types.type_type) { + return Err(vm.new_type_error("issubclass() arg 1 must be a class".to_owned())); + } + + let impl_data = get_impl(&cls, vm)?; + + // 1. Check cache + if in_weak_set(&impl_data.cache, &subclass, vm)? { + return Ok(true); + } + + // 2. Check negative cache; may have to invalidate + let invalidation_counter = get_invalidation_counter(); + if impl_data.get_cache_version() < invalidation_counter { + // Invalidate the negative cache + // Clone set ref and drop lock before calling into VM to avoid reentrancy + let set = impl_data.negative_cache.read().clone(); + if let Some(ref set) = set { + vm.call_method(set.as_ref(), "clear", ())?; + } + impl_data.set_cache_version(invalidation_counter); + } else if in_weak_set(&impl_data.negative_cache, &subclass, vm)? { + return Ok(false); + } + + // 3. Check the subclass hook + let ok = vm.call_method(&cls, "__subclasshook__", (subclass.clone(),))?; + if ok.is(&vm.ctx.true_value) { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + if ok.is(&vm.ctx.false_value) { + add_to_weak_set(&impl_data.negative_cache, &subclass, vm)?; + return Ok(false); + } + if !ok.is(&vm.ctx.not_implemented) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.assertion_error.to_owned(), + "__subclasshook__ must return either False, True, or NotImplemented".to_owned(), + )); + } + + // 4. Check if it's a direct subclass + let subclass_type: PyTypeRef = subclass + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected a type object".to_owned()))?; + let cls_type: PyTypeRef = cls + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected a type object".to_owned()))?; + if subclass_type.fast_issubclass(&cls_type) { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + + // 5. Check if it's a subclass of a registered class (recursive) + if let Some(result) = subclasscheck_check_registry(&impl_data, &subclass, vm)? { + return Ok(result); + } + + // 6. Check if it's a subclass of a subclass (recursive) + let subclasses: PyRef<PyList> = vm + .call_method(&cls, "__subclasses__", ())? + .downcast() + .map_err(|_| vm.new_type_error("__subclasses__() must return a list".to_owned()))?; + + for scls in subclasses.borrow_vec().iter() { + if subclass.is_subclass(scls, vm)? { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + } + + // No dice; update negative cache + add_to_weak_set(&impl_data.negative_cache, &subclass, vm)?; + Ok(false) + } + + /// Internal ABC helper for cache and registry debugging. + #[pyfunction] + fn _get_dump(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let impl_data = get_impl(&cls, vm)?; + + let registry = { + let r = impl_data.registry.read(); + match &*r { + Some(s) => { + // Use copy method to get a shallow copy + vm.call_method(s.as_ref(), "copy", ())? + } + None => PySet::default().to_pyobject(vm), + } + }; + + let cache = { + let c = impl_data.cache.read(); + match &*c { + Some(s) => vm.call_method(s.as_ref(), "copy", ())?, + None => PySet::default().to_pyobject(vm), + } + }; + + let negative_cache = { + let nc = impl_data.negative_cache.read(); + match &*nc { + Some(s) => vm.call_method(s.as_ref(), "copy", ())?, + None => PySet::default().to_pyobject(vm), + } + }; + + let version = impl_data.get_cache_version(); + + Ok(vm.ctx.new_tuple(vec![ + registry, + cache, + negative_cache, + vm.ctx.new_int(version).into(), + ])) + } + + /// Internal ABC helper to reset registry of a given class. + #[pyfunction] + fn _reset_registry(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let impl_data = get_impl(&cls, vm)?; + // Clone set ref and drop lock before calling into VM to avoid reentrancy + let set = impl_data.registry.read().clone(); + if let Some(ref set) = set { + vm.call_method(set.as_ref(), "clear", ())?; + } + Ok(()) + } + + /// Internal ABC helper to reset both caches of a given class. + #[pyfunction] + fn _reset_caches(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let impl_data = get_impl(&cls, vm)?; + + // Clone set refs and drop locks before calling into VM to avoid reentrancy + let cache = impl_data.cache.read().clone(); + if let Some(ref set) = cache { + vm.call_method(set.as_ref(), "clear", ())?; + } + + let negative_cache = impl_data.negative_cache.read().clone(); + if let Some(ref set) = negative_cache { + vm.call_method(set.as_ref(), "clear", ())?; + } + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_types.rs b/crates/vm/src/stdlib/_types.rs new file mode 100644 index 00000000000..385ecf8cf5a --- /dev/null +++ b/crates/vm/src/stdlib/_types.rs @@ -0,0 +1,157 @@ +//! Implementation of the `_types` module. +//! +//! This module exposes built-in types that are used by the `types` module. + +pub(crate) use _types::module_def; + +#[pymodule] +#[allow(non_snake_case)] +mod _types { + use crate::{PyObjectRef, VirtualMachine}; + + #[pyattr] + fn AsyncGeneratorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.async_generator.to_owned().into() + } + + #[pyattr] + fn BuiltinFunctionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .types + .builtin_function_or_method_type + .to_owned() + .into() + } + + #[pyattr] + fn BuiltinMethodType(vm: &VirtualMachine) -> PyObjectRef { + // Same as BuiltinFunctionType in CPython + vm.ctx + .types + .builtin_function_or_method_type + .to_owned() + .into() + } + + #[pyattr] + fn CapsuleType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.capsule_type.to_owned().into() + } + + #[pyattr] + fn CellType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.cell_type.to_owned().into() + } + + #[pyattr] + fn CodeType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.code_type.to_owned().into() + } + + #[pyattr] + fn CoroutineType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.coroutine_type.to_owned().into() + } + + #[pyattr] + fn EllipsisType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.ellipsis_type.to_owned().into() + } + + #[pyattr] + fn FrameType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.frame_type.to_owned().into() + } + + #[pyattr] + fn FunctionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.function_type.to_owned().into() + } + + #[pyattr] + fn GeneratorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.generator_type.to_owned().into() + } + + #[pyattr] + fn GenericAlias(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.generic_alias_type.to_owned().into() + } + + #[pyattr] + fn GetSetDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.getset_type.to_owned().into() + } + + #[pyattr] + fn LambdaType(vm: &VirtualMachine) -> PyObjectRef { + // Same as FunctionType in CPython + vm.ctx.types.function_type.to_owned().into() + } + + #[pyattr] + fn MappingProxyType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.mappingproxy_type.to_owned().into() + } + + #[pyattr] + fn MemberDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.member_descriptor_type.to_owned().into() + } + + #[pyattr] + fn MethodDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.method_descriptor_type.to_owned().into() + } + + #[pyattr] + fn ClassMethodDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + // TODO: implement as separate type + vm.ctx.types.method_descriptor_type.to_owned().into() + } + + #[pyattr] + fn MethodType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.bound_method_type.to_owned().into() + } + + #[pyattr] + fn MethodWrapperType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.method_wrapper_type.to_owned().into() + } + + #[pyattr] + fn ModuleType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.module_type.to_owned().into() + } + + #[pyattr] + fn NoneType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.none_type.to_owned().into() + } + + #[pyattr] + fn NotImplementedType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.not_implemented_type.to_owned().into() + } + + #[pyattr] + fn SimpleNamespace(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.namespace_type.to_owned().into() + } + + #[pyattr] + fn TracebackType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.traceback_type.to_owned().into() + } + + #[pyattr] + fn UnionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.union_type.to_owned().into() + } + + #[pyattr] + fn WrapperDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.wrapper_descriptor_type.to_owned().into() + } +} diff --git a/crates/vm/src/stdlib/_wmi.rs b/crates/vm/src/stdlib/_wmi.rs new file mode 100644 index 00000000000..f25f4b23bfe --- /dev/null +++ b/crates/vm/src/stdlib/_wmi.rs @@ -0,0 +1,698 @@ +// spell-checker:disable +#![allow(non_snake_case)] + +pub(crate) use _wmi::module_def; + +// COM/WMI FFI declarations (not inside pymodule to avoid macro issues) +mod wmi_ffi { + #![allow(unsafe_op_in_unsafe_fn)] + use core::ffi::c_void; + + pub type HRESULT = i32; + + #[repr(C)] + pub struct GUID { + pub data1: u32, + pub data2: u16, + pub data3: u16, + pub data4: [u8; 8], + } + + // Opaque VARIANT type (24 bytes covers both 32-bit and 64-bit) + #[repr(C, align(8))] + pub struct VARIANT([u64; 3]); + + impl VARIANT { + pub fn zeroed() -> Self { + VARIANT([0u64; 3]) + } + } + + // CLSID_WbemLocator = {4590F811-1D3A-11D0-891F-00AA004B2E24} + pub const CLSID_WBEM_LOCATOR: GUID = GUID { + data1: 0x4590F811, + data2: 0x1D3A, + data3: 0x11D0, + data4: [0x89, 0x1F, 0x00, 0xAA, 0x00, 0x4B, 0x2E, 0x24], + }; + + // IID_IWbemLocator = {DC12A687-737F-11CF-884D-00AA004B2E24} + pub const IID_IWBEM_LOCATOR: GUID = GUID { + data1: 0xDC12A687, + data2: 0x737F, + data3: 0x11CF, + data4: [0x88, 0x4D, 0x00, 0xAA, 0x00, 0x4B, 0x2E, 0x24], + }; + + // COM constants + pub const COINIT_APARTMENTTHREADED: u32 = 0x2; + pub const CLSCTX_INPROC_SERVER: u32 = 0x1; + pub const RPC_C_AUTHN_LEVEL_DEFAULT: u32 = 0; + pub const RPC_C_IMP_LEVEL_IMPERSONATE: u32 = 3; + pub const RPC_C_AUTHN_LEVEL_CALL: u32 = 3; + pub const RPC_C_AUTHN_WINNT: u32 = 10; + pub const RPC_C_AUTHZ_NONE: u32 = 0; + pub const EOAC_NONE: u32 = 0; + pub const RPC_E_TOO_LATE: HRESULT = 0x80010119_u32 as i32; + + // WMI constants + pub const WBEM_FLAG_FORWARD_ONLY: i32 = 0x20; + pub const WBEM_FLAG_RETURN_IMMEDIATELY: i32 = 0x10; + pub const WBEM_S_FALSE: HRESULT = 1; + pub const WBEM_S_NO_MORE_DATA: HRESULT = 0x40005; + pub const WBEM_INFINITE: i32 = -1; + pub const WBEM_FLAVOR_MASK_ORIGIN: i32 = 0x60; + pub const WBEM_FLAVOR_ORIGIN_SYSTEM: i32 = 0x40; + + #[link(name = "ole32")] + unsafe extern "system" { + pub fn CoInitializeEx(pvReserved: *mut c_void, dwCoInit: u32) -> HRESULT; + pub fn CoUninitialize(); + pub fn CoInitializeSecurity( + pSecDesc: *const c_void, + cAuthSvc: i32, + asAuthSvc: *const c_void, + pReserved1: *const c_void, + dwAuthnLevel: u32, + dwImpLevel: u32, + pAuthList: *const c_void, + dwCapabilities: u32, + pReserved3: *const c_void, + ) -> HRESULT; + pub fn CoCreateInstance( + rclsid: *const GUID, + pUnkOuter: *mut c_void, + dwClsContext: u32, + riid: *const GUID, + ppv: *mut *mut c_void, + ) -> HRESULT; + pub fn CoSetProxyBlanket( + pProxy: *mut c_void, + dwAuthnSvc: u32, + dwAuthzSvc: u32, + pServerPrincName: *const u16, + dwAuthnLevel: u32, + dwImpLevel: u32, + pAuthInfo: *const c_void, + dwCapabilities: u32, + ) -> HRESULT; + } + + #[link(name = "oleaut32")] + unsafe extern "system" { + pub fn SysAllocString(psz: *const u16) -> *mut u16; + pub fn SysFreeString(bstrString: *mut u16); + pub fn VariantClear(pvarg: *mut VARIANT) -> HRESULT; + } + + #[link(name = "propsys")] + unsafe extern "system" { + pub fn VariantToString(varIn: *const VARIANT, pszBuf: *mut u16, cchBuf: u32) -> HRESULT; + } + + /// Release a COM object (IUnknown::Release, vtable index 2) + pub unsafe fn com_release(this: *mut c_void) { + if !this.is_null() { + let vtable = *(this as *const *const usize); + let release: unsafe extern "system" fn(*mut c_void) -> u32 = + core::mem::transmute(*vtable.add(2)); + release(this); + } + } + + /// IWbemLocator::ConnectServer (vtable index 3) + #[allow(clippy::too_many_arguments)] + pub unsafe fn locator_connect_server( + this: *mut c_void, + network_resource: *const u16, + user: *const u16, + password: *const u16, + locale: *const u16, + security_flags: i32, + authority: *const u16, + ctx: *mut c_void, + services: *mut *mut c_void, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + *const u16, + *const u16, + *const u16, + *const u16, + i32, + *const u16, + *mut c_void, + *mut *mut c_void, + ) -> HRESULT = core::mem::transmute(*vtable.add(3)); + method( + this, + network_resource, + user, + password, + locale, + security_flags, + authority, + ctx, + services, + ) + } + + /// IWbemServices::ExecQuery (vtable index 20) + pub unsafe fn services_exec_query( + this: *mut c_void, + query_language: *const u16, + query: *const u16, + flags: i32, + ctx: *mut c_void, + enumerator: *mut *mut c_void, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + *const u16, + *const u16, + i32, + *mut c_void, + *mut *mut c_void, + ) -> HRESULT = core::mem::transmute(*vtable.add(20)); + method(this, query_language, query, flags, ctx, enumerator) + } + + /// IEnumWbemClassObject::Next (vtable index 4) + pub unsafe fn enum_next( + this: *mut c_void, + timeout: i32, + count: u32, + objects: *mut *mut c_void, + returned: *mut u32, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + i32, + u32, + *mut *mut c_void, + *mut u32, + ) -> HRESULT = core::mem::transmute(*vtable.add(4)); + method(this, timeout, count, objects, returned) + } + + /// IWbemClassObject::BeginEnumeration (vtable index 8) + pub unsafe fn object_begin_enumeration(this: *mut c_void, enum_flags: i32) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn(*mut c_void, i32) -> HRESULT = + core::mem::transmute(*vtable.add(8)); + method(this, enum_flags) + } + + /// IWbemClassObject::Next (vtable index 9) + pub unsafe fn object_next( + this: *mut c_void, + flags: i32, + name: *mut *mut u16, + val: *mut VARIANT, + cim_type: *mut i32, + flavor: *mut i32, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + i32, + *mut *mut u16, + *mut VARIANT, + *mut i32, + *mut i32, + ) -> HRESULT = core::mem::transmute(*vtable.add(9)); + method(this, flags, name, val, cim_type, flavor) + } + + /// IWbemClassObject::EndEnumeration (vtable index 10) + pub unsafe fn object_end_enumeration(this: *mut c_void) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn(*mut c_void) -> HRESULT = + core::mem::transmute(*vtable.add(10)); + method(this) + } +} + +#[pymodule] +mod _wmi { + use super::wmi_ffi::*; + use crate::builtins::PyStrRef; + use crate::convert::ToPyException; + use crate::{PyResult, VirtualMachine}; + use core::ffi::c_void; + use core::ptr::{null, null_mut}; + use windows_sys::Win32::Foundation::{ + CloseHandle, ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, + HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; + use windows_sys::Win32::System::Pipes::CreatePipe; + use windows_sys::Win32::System::Threading::{ + CreateEventW, CreateThread, GetExitCodeThread, SetEvent, WaitForSingleObject, + }; + + const BUFFER_SIZE: usize = 8192; + + fn hresult_from_win32(err: u32) -> HRESULT { + if err == 0 { + 0 + } else { + ((err & 0xFFFF) | 0x80070000) as HRESULT + } + } + + fn succeeded(hr: HRESULT) -> bool { + hr >= 0 + } + + fn failed(hr: HRESULT) -> bool { + hr < 0 + } + + fn wide_str(s: &str) -> Vec<u16> { + s.encode_utf16().chain(core::iter::once(0)).collect() + } + + unsafe fn wcslen(s: *const u16) -> usize { + unsafe { + let mut len = 0; + while *s.add(len) != 0 { + len += 1; + } + len + } + } + + unsafe fn wait_event(event: HANDLE, timeout: u32) -> u32 { + unsafe { + match WaitForSingleObject(event, timeout) { + WAIT_OBJECT_0 => 0, + WAIT_TIMEOUT => WAIT_TIMEOUT, + _ => GetLastError(), + } + } + } + + struct QueryThreadData { + query: Vec<u16>, + write_pipe: HANDLE, + init_event: HANDLE, + connect_event: HANDLE, + } + + // SAFETY: QueryThreadData contains HANDLEs (isize) which are safe to send across threads + unsafe impl Send for QueryThreadData {} + + unsafe extern "system" fn query_thread(param: *mut c_void) -> u32 { + unsafe { query_thread_impl(param) } + } + + unsafe fn query_thread_impl(param: *mut c_void) -> u32 { + unsafe { + let data = Box::from_raw(param as *mut QueryThreadData); + let write_pipe = data.write_pipe; + let init_event = data.init_event; + let connect_event = data.connect_event; + + let mut locator: *mut c_void = null_mut(); + let mut services: *mut c_void = null_mut(); + let mut enumerator: *mut c_void = null_mut(); + let mut hr: HRESULT = 0; + + // gh-125315: Copy the query string first + let bstr_query = SysAllocString(data.query.as_ptr()); + if bstr_query.is_null() { + hr = hresult_from_win32(ERROR_NOT_ENOUGH_MEMORY); + } + + drop(data); + + if succeeded(hr) { + hr = CoInitializeEx(null_mut(), COINIT_APARTMENTTHREADED); + } + + if failed(hr) { + CloseHandle(write_pipe); + if !bstr_query.is_null() { + SysFreeString(bstr_query); + } + return hr as u32; + } + + hr = CoInitializeSecurity( + null(), + -1, + null(), + null(), + RPC_C_AUTHN_LEVEL_DEFAULT, + RPC_C_IMP_LEVEL_IMPERSONATE, + null(), + EOAC_NONE, + null(), + ); + // gh-96684: CoInitializeSecurity will fail if another part of the app has + // already called it. + if hr == RPC_E_TOO_LATE { + hr = 0; + } + + if succeeded(hr) { + hr = CoCreateInstance( + &CLSID_WBEM_LOCATOR, + null_mut(), + CLSCTX_INPROC_SERVER, + &IID_IWBEM_LOCATOR, + &mut locator, + ); + } + if succeeded(hr) && SetEvent(init_event) == 0 { + hr = hresult_from_win32(GetLastError()); + } + + if succeeded(hr) { + let root_cimv2 = wide_str("ROOT\\CIMV2"); + let bstr_root = SysAllocString(root_cimv2.as_ptr()); + hr = locator_connect_server( + locator, + bstr_root, + null(), + null(), + null(), + 0, + null(), + null_mut(), + &mut services, + ); + if !bstr_root.is_null() { + SysFreeString(bstr_root); + } + } + if succeeded(hr) && SetEvent(connect_event) == 0 { + hr = hresult_from_win32(GetLastError()); + } + + if succeeded(hr) { + hr = CoSetProxyBlanket( + services, + RPC_C_AUTHN_WINNT, + RPC_C_AUTHZ_NONE, + null(), + RPC_C_AUTHN_LEVEL_CALL, + RPC_C_IMP_LEVEL_IMPERSONATE, + null(), + EOAC_NONE, + ); + } + if succeeded(hr) { + let wql = wide_str("WQL"); + let bstr_wql = SysAllocString(wql.as_ptr()); + hr = services_exec_query( + services, + bstr_wql, + bstr_query, + WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, + null_mut(), + &mut enumerator, + ); + if !bstr_wql.is_null() { + SysFreeString(bstr_wql); + } + } + + // Enumerate results and write to pipe + let mut value: *mut c_void; + let mut start_of_enum = true; + let null_sep: u16 = 0; + let eq_sign: u16 = b'=' as u16; + + while succeeded(hr) { + let mut got: u32 = 0; + let mut written: u32 = 0; + value = null_mut(); + hr = enum_next(enumerator, WBEM_INFINITE, 1, &mut value, &mut got); + + if hr == WBEM_S_FALSE { + hr = 0; + break; + } + if failed(hr) || got != 1 || value.is_null() { + continue; + } + + if !start_of_enum + && WriteFile( + write_pipe, + &null_sep as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + { + hr = hresult_from_win32(GetLastError()); + com_release(value); + break; + } + start_of_enum = false; + + hr = object_begin_enumeration(value, 0); + if failed(hr) { + com_release(value); + break; + } + + while succeeded(hr) { + let mut prop_name: *mut u16 = null_mut(); + let mut prop_value = VARIANT::zeroed(); + let mut flavor: i32 = 0; + + hr = object_next( + value, + 0, + &mut prop_name, + &mut prop_value, + null_mut(), + &mut flavor, + ); + + if hr == WBEM_S_NO_MORE_DATA { + hr = 0; + break; + } + + if succeeded(hr) + && (flavor & WBEM_FLAVOR_MASK_ORIGIN) != WBEM_FLAVOR_ORIGIN_SYSTEM + { + let mut prop_str = [0u16; BUFFER_SIZE]; + hr = + VariantToString(&prop_value, prop_str.as_mut_ptr(), BUFFER_SIZE as u32); + + if succeeded(hr) { + let cb_str1 = (wcslen(prop_name) * 2) as u32; + let cb_str2 = (wcslen(prop_str.as_ptr()) * 2) as u32; + + if WriteFile( + write_pipe, + prop_name as *const _, + cb_str1, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + &eq_sign as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + prop_str.as_ptr() as *const _, + cb_str2, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + &null_sep as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + { + hr = hresult_from_win32(GetLastError()); + } + } + + VariantClear(&mut prop_value); + SysFreeString(prop_name); + } + } + + object_end_enumeration(value); + com_release(value); + } + + // Cleanup + if !bstr_query.is_null() { + SysFreeString(bstr_query); + } + if !enumerator.is_null() { + com_release(enumerator); + } + if !services.is_null() { + com_release(services); + } + if !locator.is_null() { + com_release(locator); + } + CoUninitialize(); + CloseHandle(write_pipe); + + hr as u32 + } + } + + /// Runs a WMI query against the local machine. + /// + /// This returns a single string with 'name=value' pairs in a flat array separated + /// by null characters. + #[pyfunction] + fn exec_query(query: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + let query_str = query.as_str(); + + if !query_str + .get(..7) + .is_some_and(|s| s.eq_ignore_ascii_case("select ")) + { + return Err(vm.new_value_error("only SELECT queries are supported".to_owned())); + } + + let query_wide = wide_str(query_str); + + let mut h_thread: HANDLE = null_mut(); + let mut err: u32 = 0; + let mut buffer = [0u16; BUFFER_SIZE]; + let mut offset: u32 = 0; + let mut bytes_read: u32 = 0; + + let mut read_pipe: HANDLE = null_mut(); + let mut write_pipe: HANDLE = null_mut(); + + unsafe { + let init_event = CreateEventW(null(), 1, 0, null()); + let connect_event = CreateEventW(null(), 1, 0, null()); + + if init_event.is_null() + || connect_event.is_null() + || CreatePipe(&mut read_pipe, &mut write_pipe, null(), 0) == 0 + { + err = GetLastError(); + } else { + let thread_data = Box::new(QueryThreadData { + query: query_wide, + write_pipe, + init_event, + connect_event, + }); + let thread_data_ptr = Box::into_raw(thread_data); + + h_thread = CreateThread( + null(), + 0, + Some(query_thread), + thread_data_ptr as *const _ as *mut _, + 0, + null_mut(), + ); + + if h_thread.is_null() { + err = GetLastError(); + // Thread didn't start, so recover data and close write pipe + let data = Box::from_raw(thread_data_ptr); + CloseHandle(data.write_pipe); + } + } + + // gh-112278: Timeout for COM init and WMI connection + if err == 0 { + err = wait_event(init_event, 1000); + if err == 0 { + err = wait_event(connect_event, 100); + } + } + + // Read results from pipe + while err == 0 { + let buf_ptr = (buffer.as_mut_ptr() as *mut u8).add(offset as usize); + let buf_remaining = (BUFFER_SIZE * 2) as u32 - offset; + + if ReadFile( + read_pipe, + buf_ptr as *mut _, + buf_remaining, + &mut bytes_read, + null_mut(), + ) != 0 + { + offset += bytes_read; + if offset >= (BUFFER_SIZE * 2) as u32 { + err = ERROR_MORE_DATA; + } + } else { + err = GetLastError(); + } + } + + if !read_pipe.is_null() { + CloseHandle(read_pipe); + } + + if !h_thread.is_null() { + let thread_err: u32; + match WaitForSingleObject(h_thread, 100) { + WAIT_OBJECT_0 => { + let mut exit_code: u32 = 0; + if GetExitCodeThread(h_thread, &mut exit_code) == 0 { + thread_err = GetLastError(); + } else { + thread_err = exit_code; + } + } + WAIT_TIMEOUT => { + thread_err = WAIT_TIMEOUT; + } + _ => { + thread_err = GetLastError(); + } + } + if err == 0 || err == ERROR_BROKEN_PIPE { + err = thread_err; + } + + CloseHandle(h_thread); + } + + CloseHandle(init_event); + CloseHandle(connect_event); + } + + if err == ERROR_MORE_DATA { + return Err(vm.new_os_error(format!( + "Query returns more than {} characters", + BUFFER_SIZE, + ))); + } else if err != 0 { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + + if offset == 0 { + return Ok(String::new()); + } + + let char_count = (offset as usize) / 2 - 1; + Ok(String::from_utf16_lossy(&buffer[..char_count])) + } +} diff --git a/crates/vm/src/stdlib/ast.rs b/crates/vm/src/stdlib/ast.rs index 00aad0213f3..92366b6e8e3 100644 --- a/crates/vm/src/stdlib/ast.rs +++ b/crates/vm/src/stdlib/ast.rs @@ -1,26 +1,27 @@ //! `ast` standard module for abstract syntax trees. + //! //! This module makes use of the parser logic, and translates all ast nodes //! into python ast.AST objects. +pub(crate) use python::_ast::module_def; + mod pyast; use crate::builtins::{PyInt, PyStr}; -use crate::stdlib::ast::module::{Mod, ModInteractive}; +use crate::stdlib::ast::module::{Mod, ModFunctionType, ModInteractive}; use crate::stdlib::ast::node::BoxedSlice; -use crate::stdlib::ast::python::_ast; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromObject, VirtualMachine, builtins::PyIntRef, builtins::{PyDict, PyModule, PyStrRef, PyType}, class::{PyClassImpl, StaticType}, - compiler::core::bytecode::OpArgType, compiler::{CompileError, ParseError}, convert::ToPyObject, }; use node::Node; -use ruff_python_ast as ruff; +use ruff_python_ast as ast; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustpython_compiler_core::{ LineIndex, OneIndexed, PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation, @@ -35,6 +36,8 @@ use rustpython_codegen as codegen; pub(crate) use python::_ast::NodeAst; mod python; +mod repr; +mod validate; mod argument; mod basic; @@ -53,6 +56,18 @@ mod string; mod type_ignore; mod type_parameters; +/// Return the cached singleton instance for an operator/context node type, +/// or create a new instance if none exists. +fn singleton_node_to_object(vm: &VirtualMachine, node_type: &'static Py<PyType>) -> PyObjectRef { + if let Some(instance) = node_type.get_attr(vm.ctx.intern_str("_instance")) { + return instance; + } + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() +} + fn get_node_field(vm: &VirtualMachine, obj: &PyObject, field: &'static str, typ: &str) -> PyResult { vm.get_attribute_opt(obj.to_owned(), field)? .ok_or_else(|| vm.new_type_error(format!(r#"required field "{field}" missing from {typ}"#))) @@ -160,6 +175,20 @@ fn text_range_to_source_range(source_file: &SourceFile, text_range: TextRange) - } } +fn get_opt_int_field( + vm: &VirtualMachine, + obj: &PyObject, + field: &'static str, +) -> PyResult<Option<PyRefExact<PyInt>>> { + match get_node_field_opt(vm, obj, field)? { + Some(val) => val + .downcast_exact(vm) + .map(Some) + .map_err(|_| vm.new_type_error(format!(r#"field "{field}" must have integer type"#))), + None => Ok(None), + } +} + fn range_from_object( vm: &VirtualMachine, source_file: &SourceFile, @@ -168,17 +197,56 @@ fn range_from_object( ) -> PyResult<TextRange> { let start_row = get_int_field(vm, &object, "lineno", name)?; let start_column = get_int_field(vm, &object, "col_offset", name)?; - let end_row = get_int_field(vm, &object, "end_lineno", name)?; - let end_column = get_int_field(vm, &object, "end_col_offset", name)?; + // end_lineno and end_col_offset are optional, default to start values + let end_row = + get_opt_int_field(vm, &object, "end_lineno")?.unwrap_or_else(|| start_row.clone()); + let end_column = + get_opt_int_field(vm, &object, "end_col_offset")?.unwrap_or_else(|| start_column.clone()); + + // lineno=0 or negative values as a special case (no location info). + // Use default values (line 1, col 0) when lineno <= 0. + let start_row_val: i32 = start_row.try_to_primitive(vm)?; + let end_row_val: i32 = end_row.try_to_primitive(vm)?; + let start_col_val: i32 = start_column.try_to_primitive(vm)?; + let end_col_val: i32 = end_column.try_to_primitive(vm)?; + + if start_row_val > end_row_val { + return Err(vm.new_value_error(format!( + "AST node line range ({}, {}) is not valid", + start_row_val, end_row_val + ))); + } + if (start_row_val < 0 && end_row_val != start_row_val) + || (start_col_val < 0 && end_col_val != start_col_val) + { + return Err(vm.new_value_error(format!( + "AST node column range ({}, {}) for line range ({}, {}) is not valid", + start_col_val, end_col_val, start_row_val, end_row_val + ))); + } + if start_row_val == end_row_val && start_col_val > end_col_val { + return Err(vm.new_value_error(format!( + "line {}, column {}-{} is not a valid range", + start_row_val, start_col_val, end_col_val + ))); + } let location = PySourceRange { start: PySourceLocation { - row: Row(OneIndexed::new(start_row.try_to_primitive(vm)?).unwrap()), - column: Column(TextSize::new(start_column.try_to_primitive(vm)?)), + row: Row(if start_row_val > 0 { + OneIndexed::new(start_row_val as usize).unwrap_or(OneIndexed::MIN) + } else { + OneIndexed::MIN + }), + column: Column(TextSize::new(start_col_val.max(0) as u32)), }, end: PySourceLocation { - row: Row(OneIndexed::new(end_row.try_to_primitive(vm)?).unwrap()), - column: Column(TextSize::new(end_column.try_to_primitive(vm)?)), + row: Row(if end_row_val > 0 { + OneIndexed::new(end_row_val as usize).unwrap_or(OneIndexed::MIN) + } else { + OneIndexed::MIN + }), + column: Column(TextSize::new(end_col_val.max(0) as u32)), }, }; @@ -232,28 +300,449 @@ fn node_add_location( .unwrap(); } +/// Return the expected AST mod type class for a compile() mode string. +pub(crate) fn mode_type_and_name( + ctx: &Context, + mode: &str, +) -> Option<(PyRef<PyType>, &'static str)> { + match mode { + "exec" => Some((pyast::NodeModModule::make_class(ctx), "Module")), + "eval" => Some((pyast::NodeModExpression::make_class(ctx), "Expression")), + "single" => Some((pyast::NodeModInteractive::make_class(ctx), "Interactive")), + "func_type" => Some((pyast::NodeModFunctionType::make_class(ctx), "FunctionType")), + _ => None, + } +} + +/// Create an empty `arguments` AST node (no parameters). +fn empty_arguments_object(vm: &VirtualMachine) -> PyObjectRef { + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + for list_field in [ + "posonlyargs", + "args", + "kwonlyargs", + "kw_defaults", + "defaults", + ] { + dict.set_item(list_field, vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); + } + for none_field in ["vararg", "kwarg"] { + dict.set_item(none_field, vm.ctx.none(), vm).unwrap(); + } + node.into() +} + #[cfg(feature = "parser")] pub(crate) fn parse( vm: &VirtualMachine, source: &str, mode: parser::Mode, + optimize: u8, + target_version: Option<ast::PythonVersion>, + type_comments: bool, ) -> Result<PyObjectRef, CompileError> { let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); - let top = parser::parse(source, mode.into()) - .map_err(|parse_error| ParseError { + let mut options = parser::ParseOptions::from(mode); + let target_version = target_version.unwrap_or(ast::PythonVersion::PY314); + options = options.with_target_version(target_version); + let parsed = parser::parse(source, options).map_err(|parse_error| { + let range = text_range_to_source_range(&source_file, parse_error.location); + ParseError { error: parse_error.error, raw_location: parse_error.location, - location: text_range_to_source_range(&source_file, parse_error.location) - .start - .to_source_location(), + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), source_path: "<unknown>".to_string(), - })? - .into_syntax(); + is_unclosed_bracket: false, + } + })?; + + if let Some(error) = parsed.unsupported_syntax_errors().first() { + let range = text_range_to_source_range(&source_file, error.range()); + return Err(ParseError { + error: parser::ParseErrorType::OtherError(error.to_string()), + raw_location: error.range(), + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), + source_path: "<unknown>".to_string(), + is_unclosed_bracket: false, + } + .into()); + } + + let mut top = parsed.into_syntax(); + if optimize > 0 { + fold_match_value_constants(&mut top); + } + if optimize >= 2 { + strip_docstrings(&mut top); + } let top = match top { - ruff::Mod::Module(m) => Mod::Module(m), - ruff::Mod::Expression(e) => Mod::Expression(e), + ast::Mod::Module(m) => Mod::Module(m), + ast::Mod::Expression(e) => Mod::Expression(e), }; - Ok(top.ast_to_object(vm, &source_file)) + let obj = top.ast_to_object(vm, &source_file); + if type_comments && obj.class().is(pyast::NodeModModule::static_type()) { + let type_ignores = type_ignores_from_source(vm, source)?; + let dict = obj.as_object().dict().unwrap(); + dict.set_item("type_ignores", vm.ctx.new_list(type_ignores).into(), vm) + .unwrap(); + } + Ok(obj) +} + +#[cfg(feature = "parser")] +pub(crate) fn wrap_interactive(vm: &VirtualMachine, module_obj: PyObjectRef) -> PyResult { + if !module_obj.class().is(pyast::NodeModModule::static_type()) { + return Err(vm.new_type_error("expected Module node".to_owned())); + } + let body = get_node_field(vm, &module_obj, "body", "Module")?; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body, vm).unwrap(); + Ok(node.into()) +} + +#[cfg(feature = "parser")] +pub(crate) fn parse_func_type( + vm: &VirtualMachine, + source: &str, + optimize: u8, + target_version: Option<ast::PythonVersion>, +) -> Result<PyObjectRef, CompileError> { + let _ = optimize; + let _ = target_version; + let source = source.trim(); + let mut depth = 0i32; + let mut split_at = None; + let mut chars = source.chars().peekable(); + let mut idx = 0usize; + while let Some(ch) = chars.next() { + match ch { + '(' | '[' | '{' => depth += 1, + ')' | ']' | '}' => depth -= 1, + '-' if depth == 0 && chars.peek() == Some(&'>') => { + split_at = Some(idx); + break; + } + _ => {} + } + idx += ch.len_utf8(); + } + + let Some(split_at) = split_at else { + return Err(ParseError { + error: parser::ParseErrorType::OtherError("invalid func_type".to_owned()), + raw_location: TextRange::default(), + location: SourceLocation::default(), + end_location: SourceLocation::default(), + source_path: "<unknown>".to_owned(), + is_unclosed_bracket: false, + } + .into()); + }; + + let left = source[..split_at].trim(); + let right = source[split_at + 2..].trim(); + + let parse_expr = |expr_src: &str| -> Result<ast::Expr, CompileError> { + let source_file = SourceFileBuilder::new("".to_owned(), expr_src.to_owned()).finish(); + let parsed = parser::parse_expression(expr_src).map_err(|parse_error| { + let range = text_range_to_source_range(&source_file, parse_error.location); + ParseError { + error: parse_error.error, + raw_location: parse_error.location, + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), + source_path: "<unknown>".to_string(), + is_unclosed_bracket: false, + } + })?; + Ok(*parsed.into_syntax().body) + }; + + let arg_expr = parse_expr(left)?; + let returns = parse_expr(right)?; + + let argtypes: Vec<ast::Expr> = match arg_expr { + ast::Expr::Tuple(tup) => tup.elts, + ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => vec![arg_expr], + other => vec![other], + }; + + let func_type = ModFunctionType { + argtypes: argtypes.into_boxed_slice(), + returns, + range: TextRange::default(), + }; + let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); + Ok(func_type.ast_to_object(vm, &source_file)) +} + +fn type_ignores_from_source( + vm: &VirtualMachine, + source: &str, +) -> Result<Vec<PyObjectRef>, CompileError> { + let mut ignores = Vec::new(); + for (idx, line) in source.lines().enumerate() { + let Some(pos) = line.find("#") else { + continue; + }; + let comment = &line[pos + 1..]; + let comment = comment.trim_start(); + let Some(rest) = comment.strip_prefix("type: ignore") else { + continue; + }; + let tag = rest.trim_start(); + let tag = if tag.is_empty() { "" } else { tag }; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + let lineno = idx + 1; + dict.set_item("lineno", vm.ctx.new_int(lineno).into(), vm) + .unwrap(); + dict.set_item("tag", vm.ctx.new_str(tag).into(), vm) + .unwrap(); + ignores.push(node.into()); + } + Ok(ignores) +} + +#[cfg(feature = "parser")] +fn fold_match_value_constants(top: &mut ast::Mod) { + match top { + ast::Mod::Module(module) => fold_stmts(&mut module.body), + ast::Mod::Expression(_expr) => {} + } +} + +#[cfg(feature = "parser")] +fn strip_docstrings(top: &mut ast::Mod) { + match top { + ast::Mod::Module(module) => strip_docstring_in_body(&mut module.body), + ast::Mod::Expression(_expr) => {} + } +} + +#[cfg(feature = "parser")] +fn strip_docstring_in_body(body: &mut Vec<ast::Stmt>) { + if let Some(range) = take_docstring(body) + && body.is_empty() + { + let start_offset = range.start(); + let end_offset = start_offset + TextSize::from(4); + let pass_range = TextRange::new(start_offset, end_offset); + body.push(ast::Stmt::Pass(ast::StmtPass { + node_index: Default::default(), + range: pass_range, + })); + } + for stmt in body { + match stmt { + ast::Stmt::FunctionDef(def) => strip_docstring_in_body(&mut def.body), + ast::Stmt::ClassDef(def) => strip_docstring_in_body(&mut def.body), + _ => {} + } + } +} + +#[cfg(feature = "parser")] +fn take_docstring(body: &mut Vec<ast::Stmt>) -> Option<TextRange> { + let ast::Stmt::Expr(expr_stmt) = body.first()? else { + return None; + }; + if matches!(expr_stmt.value.as_ref(), ast::Expr::StringLiteral(_)) { + let range = expr_stmt.range; + body.remove(0); + return Some(range); + } + None +} + +#[cfg(feature = "parser")] +fn fold_stmts(stmts: &mut [ast::Stmt]) { + for stmt in stmts { + fold_stmt(stmt); + } +} + +#[cfg(feature = "parser")] +fn fold_stmt(stmt: &mut ast::Stmt) { + use ast::Stmt; + match stmt { + Stmt::FunctionDef(def) => fold_stmts(&mut def.body), + Stmt::ClassDef(def) => fold_stmts(&mut def.body), + Stmt::For(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + } + Stmt::While(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + } + Stmt::If(stmt) => { + fold_stmts(&mut stmt.body); + for clause in &mut stmt.elif_else_clauses { + fold_stmts(&mut clause.body); + } + } + Stmt::With(stmt) => { + fold_stmts(&mut stmt.body); + } + Stmt::Try(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + fold_stmts(&mut stmt.finalbody); + } + Stmt::Match(stmt) => { + for case in &mut stmt.cases { + fold_pattern(&mut case.pattern); + if let Some(expr) = case.guard.as_deref_mut() { + fold_expr(expr); + } + fold_stmts(&mut case.body); + } + } + _ => {} + } +} + +#[cfg(feature = "parser")] +fn fold_pattern(pattern: &mut ast::Pattern) { + use ast::Pattern; + match pattern { + Pattern::MatchValue(value) => fold_expr(&mut value.value), + Pattern::MatchSequence(seq) => { + for pattern in &mut seq.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchMapping(mapping) => { + for key in &mut mapping.keys { + fold_expr(key); + } + for pattern in &mut mapping.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchClass(class) => { + for pattern in &mut class.arguments.patterns { + fold_pattern(pattern); + } + for keyword in &mut class.arguments.keywords { + fold_pattern(&mut keyword.pattern); + } + } + Pattern::MatchAs(match_as) => { + if let Some(pattern) = match_as.pattern.as_deref_mut() { + fold_pattern(pattern); + } + } + Pattern::MatchOr(match_or) => { + for pattern in &mut match_or.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => {} + } +} + +#[cfg(feature = "parser")] +fn fold_expr(expr: &mut ast::Expr) { + use ast::Expr; + if let Expr::UnaryOp(unary) = expr { + fold_expr(&mut unary.operand); + if matches!(unary.op, ast::UnaryOp::USub) + && let Expr::NumberLiteral(number_literal) = unary.operand.as_ref() + { + let number = match &number_literal.value { + ast::Number::Int(value) => { + if *value == ast::Int::ZERO { + Some(ast::Number::Int(ast::Int::ZERO)) + } else { + None + } + } + ast::Number::Float(value) => Some(ast::Number::Float(-value)), + ast::Number::Complex { real, imag } => Some(ast::Number::Complex { + real: -real, + imag: -imag, + }), + }; + if let Some(number) = number { + *expr = Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: unary.node_index.clone(), + range: unary.range, + value: number, + }); + return; + } + } + } + if let Expr::BinOp(binop) = expr { + fold_expr(&mut binop.left); + fold_expr(&mut binop.right); + + let Expr::NumberLiteral(left) = binop.left.as_ref() else { + return; + }; + let Expr::NumberLiteral(right) = binop.right.as_ref() else { + return; + }; + + if let Some(number) = fold_number_binop(&left.value, &binop.op, &right.value) { + *expr = Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: binop.node_index.clone(), + range: binop.range, + value: number, + }); + } + } +} + +#[cfg(feature = "parser")] +fn fold_number_binop( + left: &ast::Number, + op: &ast::Operator, + right: &ast::Number, +) -> Option<ast::Number> { + let (left_real, left_imag, left_is_complex) = number_to_complex(left)?; + let (right_real, right_imag, right_is_complex) = number_to_complex(right)?; + + if !(left_is_complex || right_is_complex) { + return None; + } + + match op { + ast::Operator::Add => Some(ast::Number::Complex { + real: left_real + right_real, + imag: left_imag + right_imag, + }), + ast::Operator::Sub => Some(ast::Number::Complex { + real: left_real - right_real, + imag: left_imag - right_imag, + }), + _ => None, + } +} + +#[cfg(feature = "parser")] +fn number_to_complex(number: &ast::Number) -> Option<(f64, f64, bool)> { + match number { + ast::Number::Complex { real, imag } => Some((*real, *imag, true)), + ast::Number::Float(value) => Some((*value, 0.0, false)), + ast::Number::Int(value) => value.as_i64().map(|value| (value as f64, 0.0, false)), + } } #[cfg(feature = "codegen")] @@ -271,14 +760,15 @@ pub(crate) fn compile( let source_file = SourceFileBuilder::new(filename.to_owned(), "".to_owned()).finish(); let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; + validate::validate_mod(vm, &ast)?; let ast = match ast { - Mod::Module(m) => ruff::Mod::Module(m), - Mod::Interactive(ModInteractive { range, body }) => ruff::Mod::Module(ruff::ModModule { + Mod::Module(m) => ast::Mod::Module(m), + Mod::Interactive(ModInteractive { range, body }) => ast::Mod::Module(ast::ModModule { node_index: Default::default(), range, body, }), - Mod::Expression(e) => ruff::Mod::Expression(e), + Mod::Expression(e) => ast::Mod::Expression(e), Mod::FunctionType(_) => todo!(), }; // TODO: create a textual representation of the ast @@ -289,16 +779,27 @@ pub(crate) fn compile( Ok(vm.ctx.new_code(code).into()) } +#[cfg(feature = "codegen")] +pub(crate) fn validate_ast_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<()> { + let source_file = SourceFileBuilder::new("<ast>".to_owned(), "".to_owned()).finish(); + let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; + validate::validate_mod(vm, &ast)?; + Ok(()) +} + // Used by builtins::compile() -pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; +pub const PY_CF_ONLY_AST: i32 = 0x0400; // The following flags match the values from Include/cpython/compile.h // Caveat emptor: These flags are undocumented on purpose and depending // on their effect outside the standard library is **unsupported**. +pub const PY_CF_SOURCE_IS_UTF8: i32 = 0x0100; pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; +pub const PY_CF_IGNORE_COOKIE: i32 = 0x0800; pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; -pub const PY_CF_OPTIMIZED_AST: i32 = 0x8000 | PY_COMPILE_FLAG_AST_ONLY; +pub const PY_CF_OPTIMIZED_AST: i32 = 0x8000 | PY_CF_ONLY_AST; pub const PY_CF_TYPE_COMMENTS: i32 = 0x1000; +pub const PY_CF_ALLOW_TOP_LEVEL_AWAIT: i32 = 0x2000; // __future__ flags - sync with Lib/__future__.py // TODO: These flags aren't being used in rust code @@ -317,8 +818,11 @@ const CO_FUTURE_GENERATOR_STOP: i32 = 0x800000; const CO_FUTURE_ANNOTATIONS: i32 = 0x1000000; // Used by builtins::compile() - the summary of all flags -pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY +pub const PY_COMPILE_FLAGS_MASK: i32 = PY_CF_ONLY_AST + | PY_CF_SOURCE_IS_UTF8 | PY_CF_DONT_IMPLY_DEDENT + | PY_CF_IGNORE_COOKIE + | PY_CF_ALLOW_TOP_LEVEL_AWAIT | PY_CF_ALLOW_INCOMPLETE_INPUT | PY_CF_OPTIMIZED_AST | PY_CF_TYPE_COMMENTS @@ -332,9 +836,3 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY | CO_FUTURE_BARRY_AS_BDFL | CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS; - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _ast::make_module(vm); - pyast::extend_module_nodes(vm, &module); - module -} diff --git a/crates/vm/src/stdlib/ast/argument.rs b/crates/vm/src/stdlib/ast/argument.rs index a13200e6502..626024f5bd6 100644 --- a/crates/vm/src/stdlib/ast/argument.rs +++ b/crates/vm/src/stdlib/ast/argument.rs @@ -3,7 +3,7 @@ use rustpython_compiler_core::SourceFile; pub(super) struct PositionalArguments { pub range: TextRange, - pub args: Box<[ruff::Expr]>, + pub args: Box<[ast::Expr]>, } impl Node for PositionalArguments { @@ -27,7 +27,7 @@ impl Node for PositionalArguments { pub(super) struct KeywordArguments { pub range: TextRange, - pub keywords: Box<[ruff::Keyword]>, + pub keywords: Box<[ast::Keyword]>, } impl Node for KeywordArguments { @@ -53,10 +53,10 @@ impl Node for KeywordArguments { pub(super) fn merge_function_call_arguments( pos_args: PositionalArguments, key_args: KeywordArguments, -) -> ruff::Arguments { +) -> ast::Arguments { let range = pos_args.range.cover(key_args.range); - ruff::Arguments { + ast::Arguments { node_index: Default::default(), range, args: pos_args.args, @@ -65,9 +65,9 @@ pub(super) fn merge_function_call_arguments( } pub(super) fn split_function_call_arguments( - args: ruff::Arguments, + args: ast::Arguments, ) -> (PositionalArguments, KeywordArguments) { - let ruff::Arguments { + let ast::Arguments { node_index: _, range: _, args, @@ -100,13 +100,13 @@ pub(super) fn split_function_call_arguments( } pub(super) fn split_class_def_args( - args: Option<Box<ruff::Arguments>>, + args: Option<Box<ast::Arguments>>, ) -> (Option<PositionalArguments>, Option<KeywordArguments>) { let args = match args { None => return (None, None), Some(args) => *args, }; - let ruff::Arguments { + let ast::Arguments { node_index: _, range: _, args, @@ -141,7 +141,7 @@ pub(super) fn split_class_def_args( pub(super) fn merge_class_def_args( positional_arguments: Option<PositionalArguments>, keyword_arguments: Option<KeywordArguments>, -) -> Option<Box<ruff::Arguments>> { +) -> Option<Box<ast::Arguments>> { if positional_arguments.is_none() && keyword_arguments.is_none() { return None; } @@ -157,7 +157,7 @@ pub(super) fn merge_class_def_args( vec![].into_boxed_slice() }; - Some(Box::new(ruff::Arguments { + Some(Box::new(ast::Arguments { node_index: Default::default(), range: Default::default(), // TODO args, diff --git a/crates/vm/src/stdlib/ast/basic.rs b/crates/vm/src/stdlib/ast/basic.rs index d8565029d6c..ca518eaa520 100644 --- a/crates/vm/src/stdlib/ast/basic.rs +++ b/crates/vm/src/stdlib/ast/basic.rs @@ -2,10 +2,10 @@ use super::*; use rustpython_codegen::compile::ruff_int_to_bigint; use rustpython_compiler_core::SourceFile; -impl Node for ruff::Identifier { +impl Node for ast::Identifier { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let id = self.as_str(); - vm.ctx.new_str(id).into() + vm.ctx.intern_str(id).to_object() } fn ast_from_object( @@ -18,7 +18,7 @@ impl Node for ruff::Identifier { } } -impl Node for ruff::Int { +impl Node for ast::Int { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { vm.ctx.new_int(ruff_int_to_bigint(&self).unwrap()).into() } diff --git a/crates/vm/src/stdlib/ast/constant.rs b/crates/vm/src/stdlib/ast/constant.rs index 83b2a7f7015..1030a037f17 100644 --- a/crates/vm/src/stdlib/ast/constant.rs +++ b/crates/vm/src/stdlib/ast/constant.rs @@ -1,6 +1,6 @@ use super::*; use crate::builtins::{PyComplex, PyFrozenSet, PyTuple}; -use ruff::str_prefix::StringLiteralPrefix; +use ast::str_prefix::StringLiteralPrefix; use rustpython_compiler_core::SourceFile; #[derive(Debug)] @@ -22,7 +22,7 @@ impl Constant { } } - pub(super) const fn new_int(value: ruff::Int, range: TextRange) -> Self { + pub(super) const fn new_int(value: ast::Int, range: TextRange) -> Self { Self { range, value: ConstantLiteral::Int(value), @@ -71,7 +71,7 @@ impl Constant { } } - pub(crate) fn into_expr(self) -> ruff::Expr { + pub(crate) fn into_expr(self) -> ast::Expr { constant_to_ruff_expr(self) } } @@ -85,7 +85,7 @@ pub(crate) enum ConstantLiteral { prefix: StringLiteralPrefix, }, Bytes(Box<[u8]>), - Int(ruff::Int), + Int(ast::Int), Tuple(Vec<ConstantLiteral>), FrozenSet(Vec<ConstantLiteral>), Float(f64), @@ -244,48 +244,48 @@ impl Node for ConstantLiteral { } } -fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { +fn constant_to_ruff_expr(value: Constant) -> ast::Expr { let Constant { value, range } = value; match value { - ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { + ConstantLiteral::None => ast::Expr::NoneLiteral(ast::ExprNoneLiteral { node_index: Default::default(), range, }), - ConstantLiteral::Bool(value) => ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { + ConstantLiteral::Bool(value) => ast::Expr::BooleanLiteral(ast::ExprBooleanLiteral { node_index: Default::default(), range, value, }), ConstantLiteral::Str { value, prefix } => { - ruff::Expr::StringLiteral(ruff::ExprStringLiteral { + ast::Expr::StringLiteral(ast::ExprStringLiteral { node_index: Default::default(), range, - value: ruff::StringLiteralValue::single(ruff::StringLiteral { + value: ast::StringLiteralValue::single(ast::StringLiteral { node_index: Default::default(), range, value, - flags: ruff::StringLiteralFlags::empty().with_prefix(prefix), + flags: ast::StringLiteralFlags::empty().with_prefix(prefix), }), }) } ConstantLiteral::Bytes(value) => { - ruff::Expr::BytesLiteral(ruff::ExprBytesLiteral { + ast::Expr::BytesLiteral(ast::ExprBytesLiteral { node_index: Default::default(), range, - value: ruff::BytesLiteralValue::single(ruff::BytesLiteral { + value: ast::BytesLiteralValue::single(ast::BytesLiteral { node_index: Default::default(), range, value, - flags: ruff::BytesLiteralFlags::empty(), // TODO + flags: ast::BytesLiteralFlags::empty(), // TODO }), }) } - ConstantLiteral::Int(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + ConstantLiteral::Int(value) => ast::Expr::NumberLiteral(ast::ExprNumberLiteral { node_index: Default::default(), range, - value: ruff::Number::Int(value), + value: ast::Number::Int(value), }), - ConstantLiteral::Tuple(value) => ruff::Expr::Tuple(ruff::ExprTuple { + ConstantLiteral::Tuple(value) => ast::Expr::Tuple(ast::ExprTuple { node_index: Default::default(), range, elts: value @@ -297,48 +297,58 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { }) }) .collect(), - ctx: ruff::ExprContext::Load, + ctx: ast::ExprContext::Load, // TODO: Does this matter? parenthesized: true, }), - ConstantLiteral::FrozenSet(value) => ruff::Expr::Call(ruff::ExprCall { - node_index: Default::default(), - range, - // idk lol - func: Box::new(ruff::Expr::Name(ruff::ExprName { - node_index: Default::default(), - range: TextRange::default(), - id: ruff::name::Name::new_static("frozenset"), - ctx: ruff::ExprContext::Load, - })), - arguments: ruff::Arguments { + ConstantLiteral::FrozenSet(value) => { + let args = if value.is_empty() { + Vec::new() + } else { + vec![ast::Expr::Set(ast::ExprSet { + node_index: Default::default(), + range: TextRange::default(), + elts: value + .into_iter() + .map(|value| { + constant_to_ruff_expr(Constant { + range: TextRange::default(), + value, + }) + }) + .collect(), + })] + }; + ast::Expr::Call(ast::ExprCall { node_index: Default::default(), range, - args: value - .into_iter() - .map(|value| { - constant_to_ruff_expr(Constant { - range: TextRange::default(), - value, - }) - }) - .collect(), - keywords: Box::default(), - }, - }), - ConstantLiteral::Float(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + func: Box::new(ast::Expr::Name(ast::ExprName { + node_index: Default::default(), + range: TextRange::default(), + id: ast::name::Name::new_static("frozenset"), + ctx: ast::ExprContext::Load, + })), + arguments: ast::Arguments { + node_index: Default::default(), + range, + args: args.into(), + keywords: Box::default(), + }, + }) + } + ConstantLiteral::Float(value) => ast::Expr::NumberLiteral(ast::ExprNumberLiteral { node_index: Default::default(), range, - value: ruff::Number::Float(value), + value: ast::Number::Float(value), }), ConstantLiteral::Complex { real, imag } => { - ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { node_index: Default::default(), range, - value: ruff::Number::Complex { real, imag }, + value: ast::Number::Complex { real, imag }, }) } - ConstantLiteral::Ellipsis => ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { + ConstantLiteral::Ellipsis => ast::Expr::EllipsisLiteral(ast::ExprEllipsisLiteral { node_index: Default::default(), range, }), @@ -348,17 +358,17 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { pub(super) fn number_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprNumberLiteral, + constant: ast::ExprNumberLiteral, ) -> PyObjectRef { - let ruff::ExprNumberLiteral { + let ast::ExprNumberLiteral { node_index: _, range, value, } = constant; let c = match value { - ruff::Number::Int(n) => Constant::new_int(n, range), - ruff::Number::Float(n) => Constant::new_float(n, range), - ruff::Number::Complex { real, imag } => Constant::new_complex(real, imag, range), + ast::Number::Int(n) => Constant::new_int(n, range), + ast::Number::Float(n) => Constant::new_float(n, range), + ast::Number::Complex { real, imag } => Constant::new_complex(real, imag, range), }; c.ast_to_object(vm, source_file) } @@ -366,9 +376,9 @@ pub(super) fn number_literal_to_object( pub(super) fn string_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprStringLiteral, + constant: ast::ExprStringLiteral, ) -> PyObjectRef { - let ruff::ExprStringLiteral { + let ast::ExprStringLiteral { node_index: _, range, value, @@ -384,9 +394,9 @@ pub(super) fn string_literal_to_object( pub(super) fn bytes_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprBytesLiteral, + constant: ast::ExprBytesLiteral, ) -> PyObjectRef { - let ruff::ExprBytesLiteral { + let ast::ExprBytesLiteral { node_index: _, range, value, @@ -399,9 +409,9 @@ pub(super) fn bytes_literal_to_object( pub(super) fn boolean_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprBooleanLiteral, + constant: ast::ExprBooleanLiteral, ) -> PyObjectRef { - let ruff::ExprBooleanLiteral { + let ast::ExprBooleanLiteral { node_index: _, range, value, @@ -413,9 +423,9 @@ pub(super) fn boolean_literal_to_object( pub(super) fn none_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprNoneLiteral, + constant: ast::ExprNoneLiteral, ) -> PyObjectRef { - let ruff::ExprNoneLiteral { + let ast::ExprNoneLiteral { node_index: _, range, } = constant; @@ -426,9 +436,9 @@ pub(super) fn none_literal_to_object( pub(super) fn ellipsis_literal_to_object( vm: &VirtualMachine, source_file: &SourceFile, - constant: ruff::ExprEllipsisLiteral, + constant: ast::ExprEllipsisLiteral, ) -> PyObjectRef { - let ruff::ExprEllipsisLiteral { + let ast::ExprEllipsisLiteral { node_index: _, range, } = constant; diff --git a/crates/vm/src/stdlib/ast/elif_else_clause.rs b/crates/vm/src/stdlib/ast/elif_else_clause.rs index 581fc499b8a..0afdbc02ac1 100644 --- a/crates/vm/src/stdlib/ast/elif_else_clause.rs +++ b/crates/vm/src/stdlib/ast/elif_else_clause.rs @@ -2,12 +2,12 @@ use super::*; use rustpython_compiler_core::SourceFile; pub(super) fn ast_to_object( - clause: ruff::ElifElseClause, - mut rest: std::vec::IntoIter<ruff::ElifElseClause>, + clause: ast::ElifElseClause, + mut rest: alloc::vec::IntoIter<ast::ElifElseClause>, vm: &VirtualMachine, source_file: &SourceFile, ) -> PyObjectRef { - let ruff::ElifElseClause { + let ast::ElifElseClause { node_index: _, range, test, @@ -29,6 +29,10 @@ pub(super) fn ast_to_object( let orelse = if let Some(next) = rest.next() { if next.test.is_some() { + let next = ast::ElifElseClause { + range: TextRange::new(next.range.start(), range.end()), + ..next + }; vm.ctx .new_list(vec![ast_to_object(next, rest, vm, source_file)]) .into() @@ -48,18 +52,20 @@ pub(super) fn ast_from_object( vm: &VirtualMachine, source_file: &SourceFile, object: PyObjectRef, -) -> PyResult<ruff::StmtIf> { +) -> PyResult<ast::StmtIf> { let test = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "test", "If")?)?; let body = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "body", "If")?)?; - let orelse: Vec<ruff::Stmt> = Node::ast_from_object( + let orelse: Vec<ast::Stmt> = Node::ast_from_object( vm, source_file, get_node_field(vm, &object, "orelse", "If")?, )?; let range = range_from_object(vm, source_file, object, "If")?; - let elif_else_clauses = if let [ruff::Stmt::If(_)] = &*orelse { - let Some(ruff::Stmt::If(ruff::StmtIf { + let elif_else_clauses = if orelse.is_empty() { + vec![] + } else if let [ast::Stmt::If(_)] = &*orelse { + let Some(ast::Stmt::If(ast::StmtIf { node_index: _, range, test, @@ -71,7 +77,7 @@ pub(super) fn ast_from_object( }; elif_else_clauses.insert( 0, - ruff::ElifElseClause { + ast::ElifElseClause { node_index: Default::default(), range, test: Some(*test), @@ -80,7 +86,7 @@ pub(super) fn ast_from_object( ); elif_else_clauses } else { - vec![ruff::ElifElseClause { + vec![ast::ElifElseClause { node_index: Default::default(), range, test: None, @@ -88,7 +94,7 @@ pub(super) fn ast_from_object( }] }; - Ok(ruff::StmtIf { + Ok(ast::StmtIf { node_index: Default::default(), test, body, diff --git a/crates/vm/src/stdlib/ast/exception.rs b/crates/vm/src/stdlib/ast/exception.rs index b5b3ca2709a..2daabecc84c 100644 --- a/crates/vm/src/stdlib/ast/exception.rs +++ b/crates/vm/src/stdlib/ast/exception.rs @@ -2,29 +2,29 @@ use super::*; use rustpython_compiler_core::SourceFile; // sum -impl Node for ruff::ExceptHandler { +impl Node for ast::ExceptHandler { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self { Self::ExceptHandler(cons) => cons.ast_to_object(vm, source_file), } } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); + let cls = object.class(); Ok( - if _cls.is(pyast::NodeExceptHandlerExceptHandler::static_type()) { - Self::ExceptHandler(ruff::ExceptHandlerExceptHandler::ast_from_object( - _vm, + if cls.is(pyast::NodeExceptHandlerExceptHandler::static_type()) { + Self::ExceptHandler(ast::ExceptHandlerExceptHandler::ast_from_object( + vm, source_file, - _object, + object, )?) } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of excepthandler, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }, ) @@ -32,51 +32,51 @@ impl Node for ruff::ExceptHandler { } // constructor -impl Node for ruff::ExceptHandlerExceptHandler { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::ExceptHandlerExceptHandler { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, type_, name, body, - range: _range, + range, } = self; let node = NodeAst .into_ref_with_type( - _vm, + vm, pyast::NodeExceptHandlerExceptHandler::static_type().to_owned(), ) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("type", type_.ast_to_object(_vm, source_file), _vm) + dict.set_item("type", type_.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), - type_: get_node_field_opt(_vm, &_object, "type")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + type_: get_node_field_opt(vm, &object, "type")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, body: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "body", "ExceptHandler")?, + get_node_field(vm, &object, "body", "ExceptHandler")?, )?, - range: range_from_object(_vm, source_file, _object, "ExceptHandler")?, + range: range_from_object(vm, source_file, object, "ExceptHandler")?, }) } } diff --git a/crates/vm/src/stdlib/ast/expression.rs b/crates/vm/src/stdlib/ast/expression.rs index 83d77374380..5e55b7b676b 100644 --- a/crates/vm/src/stdlib/ast/expression.rs +++ b/crates/vm/src/stdlib/ast/expression.rs @@ -7,7 +7,7 @@ use crate::stdlib::ast::{ use rustpython_compiler_core::SourceFile; // sum -impl Node for ruff::Expr { +impl Node for ast::Expr { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self { Self::BoolOp(cons) => cons.ast_to_object(vm, source_file), @@ -36,7 +36,7 @@ impl Node for ruff::Expr { Self::NumberLiteral(cons) => constant::number_literal_to_object(vm, source_file, cons), Self::StringLiteral(cons) => constant::string_literal_to_object(vm, source_file, cons), Self::FString(cons) => string::fstring_to_object(vm, source_file, cons), - Self::TString(_) => unimplemented!(), + Self::TString(cons) => string::tstring_to_object(vm, source_file, cons), Self::BytesLiteral(cons) => constant::bytes_literal_to_object(vm, source_file, cons), Self::BooleanLiteral(cons) => { constant::boolean_literal_to_object(vm, source_file, cons) @@ -59,81 +59,80 @@ impl Node for ruff::Expr { ) -> PyResult<Self> { let cls = object.class(); Ok(if cls.is(pyast::NodeExprBoolOp::static_type()) { - Self::BoolOp(ruff::ExprBoolOp::ast_from_object(vm, source_file, object)?) + Self::BoolOp(ast::ExprBoolOp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprNamedExpr::static_type()) { - Self::Named(ruff::ExprNamed::ast_from_object(vm, source_file, object)?) + Self::Named(ast::ExprNamed::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprBinOp::static_type()) { - Self::BinOp(ruff::ExprBinOp::ast_from_object(vm, source_file, object)?) + Self::BinOp(ast::ExprBinOp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprUnaryOp::static_type()) { - Self::UnaryOp(ruff::ExprUnaryOp::ast_from_object(vm, source_file, object)?) + Self::UnaryOp(ast::ExprUnaryOp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprLambda::static_type()) { - Self::Lambda(ruff::ExprLambda::ast_from_object(vm, source_file, object)?) + Self::Lambda(ast::ExprLambda::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprIfExp::static_type()) { - Self::If(ruff::ExprIf::ast_from_object(vm, source_file, object)?) + Self::If(ast::ExprIf::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprDict::static_type()) { - Self::Dict(ruff::ExprDict::ast_from_object(vm, source_file, object)?) + Self::Dict(ast::ExprDict::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprSet::static_type()) { - Self::Set(ruff::ExprSet::ast_from_object(vm, source_file, object)?) + Self::Set(ast::ExprSet::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprListComp::static_type()) { - Self::ListComp(ruff::ExprListComp::ast_from_object( - vm, - source_file, - object, - )?) + Self::ListComp(ast::ExprListComp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprSetComp::static_type()) { - Self::SetComp(ruff::ExprSetComp::ast_from_object(vm, source_file, object)?) + Self::SetComp(ast::ExprSetComp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprDictComp::static_type()) { - Self::DictComp(ruff::ExprDictComp::ast_from_object( - vm, - source_file, - object, - )?) + Self::DictComp(ast::ExprDictComp::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprGeneratorExp::static_type()) { - Self::Generator(ruff::ExprGenerator::ast_from_object( + Self::Generator(ast::ExprGenerator::ast_from_object( vm, source_file, object, )?) } else if cls.is(pyast::NodeExprAwait::static_type()) { - Self::Await(ruff::ExprAwait::ast_from_object(vm, source_file, object)?) + Self::Await(ast::ExprAwait::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprYield::static_type()) { - Self::Yield(ruff::ExprYield::ast_from_object(vm, source_file, object)?) + Self::Yield(ast::ExprYield::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprYieldFrom::static_type()) { - Self::YieldFrom(ruff::ExprYieldFrom::ast_from_object( + Self::YieldFrom(ast::ExprYieldFrom::ast_from_object( vm, source_file, object, )?) } else if cls.is(pyast::NodeExprCompare::static_type()) { - Self::Compare(ruff::ExprCompare::ast_from_object(vm, source_file, object)?) + Self::Compare(ast::ExprCompare::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprCall::static_type()) { - Self::Call(ruff::ExprCall::ast_from_object(vm, source_file, object)?) + Self::Call(ast::ExprCall::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprAttribute::static_type()) { - Self::Attribute(ruff::ExprAttribute::ast_from_object( + Self::Attribute(ast::ExprAttribute::ast_from_object( vm, source_file, object, )?) } else if cls.is(pyast::NodeExprSubscript::static_type()) { - Self::Subscript(ruff::ExprSubscript::ast_from_object( + Self::Subscript(ast::ExprSubscript::ast_from_object( vm, source_file, object, )?) } else if cls.is(pyast::NodeExprStarred::static_type()) { - Self::Starred(ruff::ExprStarred::ast_from_object(vm, source_file, object)?) + Self::Starred(ast::ExprStarred::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprName::static_type()) { - Self::Name(ruff::ExprName::ast_from_object(vm, source_file, object)?) + Self::Name(ast::ExprName::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprList::static_type()) { - Self::List(ruff::ExprList::ast_from_object(vm, source_file, object)?) + Self::List(ast::ExprList::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprTuple::static_type()) { - Self::Tuple(ruff::ExprTuple::ast_from_object(vm, source_file, object)?) + Self::Tuple(ast::ExprTuple::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprSlice::static_type()) { - Self::Slice(ruff::ExprSlice::ast_from_object(vm, source_file, object)?) + Self::Slice(ast::ExprSlice::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeExprConstant::static_type()) { Constant::ast_from_object(vm, source_file, object)?.into_expr() } else if cls.is(pyast::NodeExprJoinedStr::static_type()) { JoinedStr::ast_from_object(vm, source_file, object)?.into_expr() + } else if cls.is(pyast::NodeExprTemplateStr::static_type()) { + let template = string::TemplateStr::ast_from_object(vm, source_file, object)?; + return string::template_str_to_expr(vm, template); + } else if cls.is(pyast::NodeExprInterpolation::static_type()) { + let interpolation = + string::TStringInterpolation::ast_from_object(vm, source_file, object)?; + return string::interpolation_to_expr(vm, interpolation); } else { return Err(vm.new_type_error(format!( "expected some sort of expr, but got {}", @@ -144,7 +143,7 @@ impl Node for ruff::Expr { } // constructor -impl Node for ruff::ExprBoolOp { +impl Node for ast::ExprBoolOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -187,7 +186,7 @@ impl Node for ruff::ExprBoolOp { } // constructor -impl Node for ruff::ExprNamed { +impl Node for ast::ExprNamed { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -230,7 +229,7 @@ impl Node for ruff::ExprNamed { } // constructor -impl Node for ruff::ExprBinOp { +impl Node for ast::ExprBinOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -281,7 +280,7 @@ impl Node for ruff::ExprBinOp { } // constructor -impl Node for ruff::ExprUnaryOp { +impl Node for ast::ExprUnaryOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -323,7 +322,7 @@ impl Node for ruff::ExprUnaryOp { } // constructor -impl Node for ruff::ExprLambda { +impl Node for ast::ExprLambda { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -335,8 +334,12 @@ impl Node for ruff::ExprLambda { .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("args", parameters.ast_to_object(vm, source_file), vm) - .unwrap(); + // Lambda with no parameters should have an empty arguments object, not None + let args = match parameters { + Some(params) => params.ast_to_object(vm, source_file), + None => empty_arguments_object(vm), + }; + dict.set_item("args", args, vm).unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); node_add_location(&dict, _range, vm, source_file); @@ -366,7 +369,7 @@ impl Node for ruff::ExprLambda { } // constructor -impl Node for ruff::ExprIf { +impl Node for ast::ExprIf { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -417,7 +420,7 @@ impl Node for ruff::ExprIf { } // constructor -impl Node for ruff::ExprDict { +impl Node for ast::ExprDict { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -449,7 +452,7 @@ impl Node for ruff::ExprDict { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult<Self> { - let keys: Vec<Option<ruff::Expr>> = Node::ast_from_object( + let keys: Vec<Option<ast::Expr>> = Node::ast_from_object( vm, source_file, get_node_field(vm, &object, "keys", "Dict")?, @@ -459,10 +462,15 @@ impl Node for ruff::ExprDict { source_file, get_node_field(vm, &object, "values", "Dict")?, )?; + if keys.len() != values.len() { + return Err(vm.new_value_error( + "Dict doesn't have the same number of keys as values".to_owned(), + )); + } let items = keys .into_iter() .zip(values) - .map(|(key, value)| ruff::DictItem { key, value }) + .map(|(key, value)| ast::DictItem { key, value }) .collect(); Ok(Self { node_index: Default::default(), @@ -473,7 +481,7 @@ impl Node for ruff::ExprDict { } // constructor -impl Node for ruff::ExprSet { +impl Node for ast::ExprSet { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -507,7 +515,7 @@ impl Node for ruff::ExprSet { } // constructor -impl Node for ruff::ExprListComp { +impl Node for ast::ExprListComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -550,7 +558,7 @@ impl Node for ruff::ExprListComp { } // constructor -impl Node for ruff::ExprSetComp { +impl Node for ast::ExprSetComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -593,7 +601,7 @@ impl Node for ruff::ExprSetComp { } // constructor -impl Node for ruff::ExprDictComp { +impl Node for ast::ExprDictComp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -644,15 +652,25 @@ impl Node for ruff::ExprDictComp { } // constructor -impl Node for ruff::ExprGenerator { +impl Node for ast::ExprGenerator { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, elt, generators, range, - parenthesized: _, + parenthesized, } = self; + let range = if parenthesized { + range + } else { + TextRange::new( + range + .start() + .saturating_sub(ruff_text_size::TextSize::from(1)), + range.end() + ruff_text_size::TextSize::from(1), + ) + }; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::static_type().to_owned()) .unwrap(); @@ -690,7 +708,7 @@ impl Node for ruff::ExprGenerator { } // constructor -impl Node for ruff::ExprAwait { +impl Node for ast::ExprAwait { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -724,7 +742,7 @@ impl Node for ruff::ExprAwait { } // constructor -impl Node for ruff::ExprYield { +impl Node for ast::ExprYield { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -757,7 +775,7 @@ impl Node for ruff::ExprYield { } // constructor -impl Node for ruff::ExprYieldFrom { +impl Node for ast::ExprYieldFrom { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -792,7 +810,7 @@ impl Node for ruff::ExprYieldFrom { } // constructor -impl Node for ruff::ExprCompare { +impl Node for ast::ExprCompare { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -853,7 +871,7 @@ impl Node for ruff::ExprCompare { } // constructor -impl Node for ruff::ExprCall { +impl Node for ast::ExprCall { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -914,7 +932,7 @@ impl Node for ruff::ExprCall { } // constructor -impl Node for ruff::ExprAttribute { +impl Node for ast::ExprAttribute { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -965,7 +983,7 @@ impl Node for ruff::ExprAttribute { } // constructor -impl Node for ruff::ExprSubscript { +impl Node for ast::ExprSubscript { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1015,7 +1033,7 @@ impl Node for ruff::ExprSubscript { } // constructor -impl Node for ruff::ExprStarred { +impl Node for ast::ExprStarred { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1057,7 +1075,7 @@ impl Node for ruff::ExprStarred { } // constructor -impl Node for ruff::ExprName { +impl Node for ast::ExprName { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1095,7 +1113,7 @@ impl Node for ruff::ExprName { } // constructor -impl Node for ruff::ExprList { +impl Node for ast::ExprList { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1138,7 +1156,7 @@ impl Node for ruff::ExprList { } // constructor -impl Node for ruff::ExprTuple { +impl Node for ast::ExprTuple { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1183,7 +1201,7 @@ impl Node for ruff::ExprTuple { } // constructor -impl Node for ruff::ExprSlice { +impl Node for ast::ExprSlice { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1228,7 +1246,7 @@ impl Node for ruff::ExprSlice { } // sum -impl Node for ruff::ExprContext { +impl Node for ast::ExprContext { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { Self::Load => pyast::NodeExprContextLoad::static_type(), @@ -1238,10 +1256,7 @@ impl Node for ruff::ExprContext { unimplemented!("Invalid expression context is not allowed in Python AST") } }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + singleton_node_to_object(vm, node_type) } fn ast_from_object( @@ -1249,12 +1264,12 @@ impl Node for ruff::ExprContext { _source_file: &SourceFile, object: PyObjectRef, ) -> PyResult<Self> { - let _cls = object.class(); - Ok(if _cls.is(pyast::NodeExprContextLoad::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeExprContextLoad::static_type()) { Self::Load - } else if _cls.is(pyast::NodeExprContextStore::static_type()) { + } else if cls.is(pyast::NodeExprContextStore::static_type()) { Self::Store - } else if _cls.is(pyast::NodeExprContextDel::static_type()) { + } else if cls.is(pyast::NodeExprContextDel::static_type()) { Self::Del } else { return Err(vm.new_type_error(format!( @@ -1266,7 +1281,7 @@ impl Node for ruff::ExprContext { } // product -impl Node for ruff::Comprehension { +impl Node for ast::Comprehension { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, diff --git a/crates/vm/src/stdlib/ast/module.rs b/crates/vm/src/stdlib/ast/module.rs index 6fae8f10a33..cfedba606b0 100644 --- a/crates/vm/src/stdlib/ast/module.rs +++ b/crates/vm/src/stdlib/ast/module.rs @@ -18,9 +18,9 @@ use rustpython_compiler_core::SourceFile; /// - `FunctionType`: A function signature with argument and return type /// annotations, representing the type hints of a function (e.g., `def add(x: int, y: int) -> int`). pub(super) enum Mod { - Module(ruff::ModModule), + Module(ast::ModModule), Interactive(ModInteractive), - Expression(ruff::ModExpression), + Expression(ast::ModExpression), FunctionType(ModFunctionType), } @@ -42,11 +42,11 @@ impl Node for Mod { ) -> PyResult<Self> { let cls = object.class(); Ok(if cls.is(pyast::NodeModModule::static_type()) { - Self::Module(ruff::ModModule::ast_from_object(vm, source_file, object)?) + Self::Module(ast::ModModule::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeModInteractive::static_type()) { Self::Interactive(ModInteractive::ast_from_object(vm, source_file, object)?) } else if cls.is(pyast::NodeModExpression::static_type()) { - Self::Expression(ruff::ModExpression::ast_from_object( + Self::Expression(ast::ModExpression::ast_from_object( vm, source_file, object, @@ -63,7 +63,7 @@ impl Node for Mod { } // constructor -impl Node for ruff::ModModule { +impl Node for ast::ModModule { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -86,7 +86,7 @@ impl Node for ruff::ModModule { vm, ) .unwrap(); - node_add_location(&dict, range, vm, source_file); + let _ = range; node.into() } @@ -113,7 +113,7 @@ impl Node for ruff::ModModule { pub(super) struct ModInteractive { pub(crate) range: TextRange, - pub(crate) body: Vec<ruff::Stmt>, + pub(crate) body: Vec<ast::Stmt>, } // constructor @@ -126,7 +126,7 @@ impl Node for ModInteractive { let dict = node.as_object().dict().unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, range, vm, source_file); + let _ = range; node.into() } @@ -147,7 +147,7 @@ impl Node for ModInteractive { } // constructor -impl Node for ruff::ModExpression { +impl Node for ast::ModExpression { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -160,7 +160,7 @@ impl Node for ruff::ModExpression { let dict = node.as_object().dict().unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, range, vm, source_file); + let _ = range; node.into() } @@ -182,8 +182,8 @@ impl Node for ruff::ModExpression { } pub(super) struct ModFunctionType { - pub(crate) argtypes: Box<[ruff::Expr]>, - pub(crate) returns: ruff::Expr, + pub(crate) argtypes: Box<[ast::Expr]>, + pub(crate) returns: ast::Expr, pub(crate) range: TextRange, } @@ -207,7 +207,7 @@ impl Node for ModFunctionType { .unwrap(); dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, range, vm, source_file); + let _ = range; node.into() } diff --git a/crates/vm/src/stdlib/ast/operator.rs b/crates/vm/src/stdlib/ast/operator.rs index c394152da2c..09e63b5d6ce 100644 --- a/crates/vm/src/stdlib/ast/operator.rs +++ b/crates/vm/src/stdlib/ast/operator.rs @@ -2,39 +2,36 @@ use super::*; use rustpython_compiler_core::SourceFile; // sum -impl Node for ruff::BoolOp { +impl Node for ast::BoolOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { Self::And => pyast::NodeBoolOpAnd::static_type(), Self::Or => pyast::NodeBoolOpOr::static_type(), }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + singleton_node_to_object(vm, node_type) } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, _source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeBoolOpAnd::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeBoolOpAnd::static_type()) { Self::And - } else if _cls.is(pyast::NodeBoolOpOr::static_type()) { + } else if cls.is(pyast::NodeBoolOpOr::static_type()) { Self::Or } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of boolop, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } } // sum -impl Node for ruff::Operator { +impl Node for ast::Operator { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { Self::Add => pyast::NodeOperatorAdd::static_type(), @@ -51,55 +48,52 @@ impl Node for ruff::Operator { Self::BitAnd => pyast::NodeOperatorBitAnd::static_type(), Self::FloorDiv => pyast::NodeOperatorFloorDiv::static_type(), }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + singleton_node_to_object(vm, node_type) } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, _source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeOperatorAdd::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeOperatorAdd::static_type()) { Self::Add - } else if _cls.is(pyast::NodeOperatorSub::static_type()) { + } else if cls.is(pyast::NodeOperatorSub::static_type()) { Self::Sub - } else if _cls.is(pyast::NodeOperatorMult::static_type()) { + } else if cls.is(pyast::NodeOperatorMult::static_type()) { Self::Mult - } else if _cls.is(pyast::NodeOperatorMatMult::static_type()) { + } else if cls.is(pyast::NodeOperatorMatMult::static_type()) { Self::MatMult - } else if _cls.is(pyast::NodeOperatorDiv::static_type()) { + } else if cls.is(pyast::NodeOperatorDiv::static_type()) { Self::Div - } else if _cls.is(pyast::NodeOperatorMod::static_type()) { + } else if cls.is(pyast::NodeOperatorMod::static_type()) { Self::Mod - } else if _cls.is(pyast::NodeOperatorPow::static_type()) { + } else if cls.is(pyast::NodeOperatorPow::static_type()) { Self::Pow - } else if _cls.is(pyast::NodeOperatorLShift::static_type()) { + } else if cls.is(pyast::NodeOperatorLShift::static_type()) { Self::LShift - } else if _cls.is(pyast::NodeOperatorRShift::static_type()) { + } else if cls.is(pyast::NodeOperatorRShift::static_type()) { Self::RShift - } else if _cls.is(pyast::NodeOperatorBitOr::static_type()) { + } else if cls.is(pyast::NodeOperatorBitOr::static_type()) { Self::BitOr - } else if _cls.is(pyast::NodeOperatorBitXor::static_type()) { + } else if cls.is(pyast::NodeOperatorBitXor::static_type()) { Self::BitXor - } else if _cls.is(pyast::NodeOperatorBitAnd::static_type()) { + } else if cls.is(pyast::NodeOperatorBitAnd::static_type()) { Self::BitAnd - } else if _cls.is(pyast::NodeOperatorFloorDiv::static_type()) { + } else if cls.is(pyast::NodeOperatorFloorDiv::static_type()) { Self::FloorDiv } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of operator, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } } // sum -impl Node for ruff::UnaryOp { +impl Node for ast::UnaryOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { Self::Invert => pyast::NodeUnaryOpInvert::static_type(), @@ -107,37 +101,34 @@ impl Node for ruff::UnaryOp { Self::UAdd => pyast::NodeUnaryOpUAdd::static_type(), Self::USub => pyast::NodeUnaryOpUSub::static_type(), }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + singleton_node_to_object(vm, node_type) } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, _source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeUnaryOpInvert::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeUnaryOpInvert::static_type()) { Self::Invert - } else if _cls.is(pyast::NodeUnaryOpNot::static_type()) { + } else if cls.is(pyast::NodeUnaryOpNot::static_type()) { Self::Not - } else if _cls.is(pyast::NodeUnaryOpUAdd::static_type()) { + } else if cls.is(pyast::NodeUnaryOpUAdd::static_type()) { Self::UAdd - } else if _cls.is(pyast::NodeUnaryOpUSub::static_type()) { + } else if cls.is(pyast::NodeUnaryOpUSub::static_type()) { Self::USub } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of unaryop, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } } // sum -impl Node for ruff::CmpOp { +impl Node for ast::CmpOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { Self::Eq => pyast::NodeCmpOpEq::static_type(), @@ -151,42 +142,39 @@ impl Node for ruff::CmpOp { Self::In => pyast::NodeCmpOpIn::static_type(), Self::NotIn => pyast::NodeCmpOpNotIn::static_type(), }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + singleton_node_to_object(vm, node_type) } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, _source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeCmpOpEq::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeCmpOpEq::static_type()) { Self::Eq - } else if _cls.is(pyast::NodeCmpOpNotEq::static_type()) { + } else if cls.is(pyast::NodeCmpOpNotEq::static_type()) { Self::NotEq - } else if _cls.is(pyast::NodeCmpOpLt::static_type()) { + } else if cls.is(pyast::NodeCmpOpLt::static_type()) { Self::Lt - } else if _cls.is(pyast::NodeCmpOpLtE::static_type()) { + } else if cls.is(pyast::NodeCmpOpLtE::static_type()) { Self::LtE - } else if _cls.is(pyast::NodeCmpOpGt::static_type()) { + } else if cls.is(pyast::NodeCmpOpGt::static_type()) { Self::Gt - } else if _cls.is(pyast::NodeCmpOpGtE::static_type()) { + } else if cls.is(pyast::NodeCmpOpGtE::static_type()) { Self::GtE - } else if _cls.is(pyast::NodeCmpOpIs::static_type()) { + } else if cls.is(pyast::NodeCmpOpIs::static_type()) { Self::Is - } else if _cls.is(pyast::NodeCmpOpIsNot::static_type()) { + } else if cls.is(pyast::NodeCmpOpIsNot::static_type()) { Self::IsNot - } else if _cls.is(pyast::NodeCmpOpIn::static_type()) { + } else if cls.is(pyast::NodeCmpOpIn::static_type()) { Self::In - } else if _cls.is(pyast::NodeCmpOpNotIn::static_type()) { + } else if cls.is(pyast::NodeCmpOpNotIn::static_type()) { Self::NotIn } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of cmpop, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } diff --git a/crates/vm/src/stdlib/ast/other.rs b/crates/vm/src/stdlib/ast/other.rs index cf8a8319749..5c0803ac594 100644 --- a/crates/vm/src/stdlib/ast/other.rs +++ b/crates/vm/src/stdlib/ast/other.rs @@ -1,10 +1,9 @@ use super::*; -use num_traits::ToPrimitive; -use rustpython_compiler_core::{SourceFile, bytecode}; +use rustpython_compiler_core::SourceFile; -impl Node for ruff::ConversionFlag { +impl Node for ast::ConversionFlag { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() + vm.ctx.new_int(self as i8).into() } fn ast_from_object( @@ -12,23 +11,22 @@ impl Node for ruff::ConversionFlag { _source_file: &SourceFile, object: PyObjectRef, ) -> PyResult<Self> { - i32::try_from_object(vm, object)? - .to_u32() - .and_then(bytecode::ConvertValueOparg::from_op_arg) - .map(|flag| match flag { - bytecode::ConvertValueOparg::None => Self::None, - bytecode::ConvertValueOparg::Str => Self::Str, - bytecode::ConvertValueOparg::Repr => Self::Repr, - bytecode::ConvertValueOparg::Ascii => Self::Ascii, - }) - .ok_or_else(|| vm.new_value_error("invalid conversion flag")) + // Python's AST uses ASCII codes: 's', 'r', 'a', -1=None + // Note: 255 is -1i8 as u8 (ruff's ConversionFlag::None) + match i32::try_from_object(vm, object)? { + -1 | 255 => Ok(Self::None), + x if x == b's' as i32 => Ok(Self::Str), + x if x == b'r' as i32 => Ok(Self::Repr), + x if x == b'a' as i32 => Ok(Self::Ascii), + _ => Err(vm.new_value_error("invalid conversion flag")), + } } } // /// This is just a string, not strictly an AST node. But it makes AST conversions easier. -impl Node for ruff::name::Name { +impl Node for ast::name::Name { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { - vm.ctx.new_str(self.as_str()).to_pyobject(vm) + vm.ctx.intern_str(self.as_str()).to_object() } fn ast_from_object( @@ -43,9 +41,9 @@ impl Node for ruff::name::Name { } } -impl Node for ruff::Decorator { +impl Node for ast::Decorator { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - ruff::Expr::ast_to_object(self.expression, vm, source_file) + ast::Expr::ast_to_object(self.expression, vm, source_file) } fn ast_from_object( @@ -53,7 +51,7 @@ impl Node for ruff::Decorator { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult<Self> { - let expression = ruff::Expr::ast_from_object(vm, source_file, object)?; + let expression = ast::Expr::ast_from_object(vm, source_file, object)?; let range = expression.range(); Ok(Self { node_index: Default::default(), @@ -64,7 +62,7 @@ impl Node for ruff::Decorator { } // product -impl Node for ruff::Alias { +impl Node for ast::Alias { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -105,7 +103,7 @@ impl Node for ruff::Alias { } // product -impl Node for ruff::WithItem { +impl Node for ast::WithItem { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, diff --git a/crates/vm/src/stdlib/ast/parameter.rs b/crates/vm/src/stdlib/ast/parameter.rs index 87fa736687b..15ff237e50d 100644 --- a/crates/vm/src/stdlib/ast/parameter.rs +++ b/crates/vm/src/stdlib/ast/parameter.rs @@ -2,7 +2,7 @@ use super::*; use rustpython_compiler_core::SourceFile; // product -impl Node for ruff::Parameters { +impl Node for ast::Parameters { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -42,7 +42,7 @@ impl Node for ruff::Parameters { .unwrap(); dict.set_item("defaults", defaults.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, range, vm, source_file); + let _ = range; node.into() } @@ -61,7 +61,7 @@ impl Node for ruff::Parameters { source_file, get_node_field(vm, &object, "kw_defaults", "arguments")?, )?; - let kwonlyargs = merge_keyword_parameter_defaults(kwonlyargs, kw_defaults); + let kwonlyargs = merge_keyword_parameter_defaults(vm, kwonlyargs, kw_defaults)?; let posonlyargs = Node::ast_from_object( vm, @@ -78,7 +78,8 @@ impl Node for ruff::Parameters { source_file, get_node_field(vm, &object, "defaults", "arguments")?, )?; - let (posonlyargs, args) = merge_positional_parameter_defaults(posonlyargs, args, defaults); + let (posonlyargs, args) = + merge_positional_parameter_defaults(vm, posonlyargs, args, defaults)?; Ok(Self { node_index: Default::default(), @@ -101,7 +102,7 @@ impl Node for ruff::Parameters { } // product -impl Node for ruff::Parameter { +impl Node for ast::Parameter { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -126,8 +127,8 @@ impl Node for ruff::Parameter { _vm, ) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, range, _vm, source_file); node.into() } @@ -156,7 +157,7 @@ impl Node for ruff::Parameter { } // product -impl Node for ruff::Keyword { +impl Node for ast::Keyword { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -197,7 +198,7 @@ impl Node for ruff::Keyword { struct PositionalParameters { pub _range: TextRange, // TODO: Use this - pub args: Box<[ruff::Parameter]>, + pub args: Box<[ast::Parameter]>, } impl Node for PositionalParameters { @@ -220,7 +221,7 @@ impl Node for PositionalParameters { struct KeywordParameters { pub _range: TextRange, // TODO: Use this - pub keywords: Box<[ruff::Parameter]>, + pub keywords: Box<[ast::Parameter]>, } impl Node for KeywordParameters { @@ -243,7 +244,7 @@ impl Node for KeywordParameters { struct ParameterDefaults { pub _range: TextRange, // TODO: Use this - defaults: Box<[Option<Box<ruff::Expr>>]>, + defaults: Box<[Option<Box<ast::Expr>>]>, } impl Node for ParameterDefaults { @@ -265,8 +266,8 @@ impl Node for ParameterDefaults { } fn extract_positional_parameter_defaults( - pos_only_args: Vec<ruff::ParameterWithDefault>, - args: Vec<ruff::ParameterWithDefault>, + pos_only_args: Vec<ast::ParameterWithDefault>, + args: Vec<ast::ParameterWithDefault>, ) -> ( PositionalParameters, PositionalParameters, @@ -321,19 +322,20 @@ fn extract_positional_parameter_defaults( /// Merges the keyword parameters with their default values, opposite of [`extract_positional_parameter_defaults`]. fn merge_positional_parameter_defaults( + vm: &VirtualMachine, posonlyargs: PositionalParameters, args: PositionalParameters, defaults: ParameterDefaults, -) -> ( - Vec<ruff::ParameterWithDefault>, - Vec<ruff::ParameterWithDefault>, -) { +) -> PyResult<( + Vec<ast::ParameterWithDefault>, + Vec<ast::ParameterWithDefault>, +)> { let posonlyargs = posonlyargs.args; let args = args.args; let defaults = defaults.defaults; let mut posonlyargs: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(posonlyargs) - .map(|parameter| ruff::ParameterWithDefault { + .map(|parameter| ast::ParameterWithDefault { node_index: Default::default(), range: Default::default(), parameter, @@ -341,7 +343,7 @@ fn merge_positional_parameter_defaults( }) .collect(); let mut args: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(args) - .map(|parameter| ruff::ParameterWithDefault { + .map(|parameter| ast::ParameterWithDefault { node_index: Default::default(), range: Default::default(), parameter, @@ -352,7 +354,11 @@ fn merge_positional_parameter_defaults( // If an argument has a default value, insert it // Note that "defaults" will only contain default values for the last "n" parameters // so we need to skip the first "total_argument_count - n" arguments. - let default_argument_count = posonlyargs.len() + args.len() - defaults.len(); + let total_args = posonlyargs.len() + args.len(); + if defaults.len() > total_args { + return Err(vm.new_value_error("more positional defaults than args on arguments")); + } + let default_argument_count = total_args - defaults.len(); for (arg, default) in posonlyargs .iter_mut() .chain(args.iter_mut()) @@ -362,11 +368,11 @@ fn merge_positional_parameter_defaults( arg.default = default; } - (posonlyargs, args) + Ok((posonlyargs, args)) } fn extract_keyword_parameter_defaults( - kw_only_args: Vec<ruff::ParameterWithDefault>, + kw_only_args: Vec<ast::ParameterWithDefault>, ) -> (KeywordParameters, ParameterDefaults) { let mut defaults = vec![]; defaults.extend(kw_only_args.iter().map(|item| item.default.clone())); @@ -400,15 +406,21 @@ fn extract_keyword_parameter_defaults( /// Merges the keyword parameters with their default values, opposite of [`extract_keyword_parameter_defaults`]. fn merge_keyword_parameter_defaults( + vm: &VirtualMachine, kw_only_args: KeywordParameters, defaults: ParameterDefaults, -) -> Vec<ruff::ParameterWithDefault> { - std::iter::zip(kw_only_args.keywords, defaults.defaults) - .map(|(parameter, default)| ruff::ParameterWithDefault { +) -> PyResult<Vec<ast::ParameterWithDefault>> { + if kw_only_args.keywords.len() != defaults.defaults.len() { + return Err( + vm.new_value_error("length of kwonlyargs is not the same as kw_defaults on arguments") + ); + } + Ok(core::iter::zip(kw_only_args.keywords, defaults.defaults) + .map(|(parameter, default)| ast::ParameterWithDefault { node_index: Default::default(), parameter, default, range: Default::default(), }) - .collect() + .collect()) } diff --git a/crates/vm/src/stdlib/ast/pattern.rs b/crates/vm/src/stdlib/ast/pattern.rs index d8128cb0622..3b665a95b55 100644 --- a/crates/vm/src/stdlib/ast/pattern.rs +++ b/crates/vm/src/stdlib/ast/pattern.rs @@ -2,47 +2,47 @@ use super::*; use rustpython_compiler_core::SourceFile; // product -impl Node for ruff::MatchCase { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::MatchCase { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, pattern, guard, body, - range: _range, + range: _, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodeMatchCase::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeMatchCase::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm, source_file), _vm) + dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("guard", guard.ast_to_object(_vm, source_file), _vm) + dict.set_item("guard", guard.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), pattern: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "pattern", "match_case")?, + get_node_field(vm, &object, "pattern", "match_case")?, )?, - guard: get_node_field_opt(_vm, &_object, "guard")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + guard: get_node_field_opt(vm, &object, "guard")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, body: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "body", "match_case")?, + get_node_field(vm, &object, "body", "match_case")?, )?, range: Default::default(), }) @@ -50,7 +50,7 @@ impl Node for ruff::MatchCase { } // sum -impl Node for ruff::Pattern { +impl Node for ast::Pattern { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self { Self::MatchValue(cons) => cons.ast_to_object(vm, source_file), @@ -64,146 +64,146 @@ impl Node for ruff::Pattern { } } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodePatternMatchValue::static_type()) { - Self::MatchValue(ruff::PatternMatchValue::ast_from_object( - _vm, + let cls = object.class(); + Ok(if cls.is(pyast::NodePatternMatchValue::static_type()) { + Self::MatchValue(ast::PatternMatchValue::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchSingleton::static_type()) { - Self::MatchSingleton(ruff::PatternMatchSingleton::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchSingleton::static_type()) { + Self::MatchSingleton(ast::PatternMatchSingleton::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchSequence::static_type()) { - Self::MatchSequence(ruff::PatternMatchSequence::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchSequence::static_type()) { + Self::MatchSequence(ast::PatternMatchSequence::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchMapping::static_type()) { - Self::MatchMapping(ruff::PatternMatchMapping::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchMapping::static_type()) { + Self::MatchMapping(ast::PatternMatchMapping::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchClass::static_type()) { - Self::MatchClass(ruff::PatternMatchClass::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchClass::static_type()) { + Self::MatchClass(ast::PatternMatchClass::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchStar::static_type()) { - Self::MatchStar(ruff::PatternMatchStar::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchStar::static_type()) { + Self::MatchStar(ast::PatternMatchStar::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchAs::static_type()) { - Self::MatchAs(ruff::PatternMatchAs::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchAs::static_type()) { + Self::MatchAs(ast::PatternMatchAs::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodePatternMatchOr::static_type()) { - Self::MatchOr(ruff::PatternMatchOr::ast_from_object( - _vm, + } else if cls.is(pyast::NodePatternMatchOr::static_type()) { + Self::MatchOr(ast::PatternMatchOr::ast_from_object( + vm, source_file, - _object, + object, )?) } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of pattern, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } } // constructor -impl Node for ruff::PatternMatchValue { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchValue { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, value, - range: _range, + range, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodePatternMatchValue::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchValue::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + dict.set_item("value", value.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), value: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "value", "MatchValue")?, + get_node_field(vm, &object, "value", "MatchValue")?, )?, - range: range_from_object(_vm, source_file, _object, "MatchValue")?, + range: range_from_object(vm, source_file, object, "MatchValue")?, }) } } // constructor -impl Node for ruff::PatternMatchSingleton { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchSingleton { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, value, - range: _range, + range, } = self; let node = NodeAst .into_ref_with_type( - _vm, + vm, pyast::NodePatternMatchSingleton::static_type().to_owned(), ) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + dict.set_item("value", value.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), value: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "value", "MatchSingleton")?, + get_node_field(vm, &object, "value", "MatchSingleton")?, )?, - range: range_from_object(_vm, source_file, _object, "MatchSingleton")?, + range: range_from_object(vm, source_file, object, "MatchSingleton")?, }) } } -impl Node for ruff::Singleton { +impl Node for ast::Singleton { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { match self { - ruff::Singleton::None => vm.ctx.none(), - ruff::Singleton::True => vm.ctx.new_bool(true).into(), - ruff::Singleton::False => vm.ctx.new_bool(false).into(), + ast::Singleton::None => vm.ctx.none(), + ast::Singleton::True => vm.ctx.new_bool(true).into(), + ast::Singleton::False => vm.ctx.new_bool(false).into(), } } @@ -213,11 +213,11 @@ impl Node for ruff::Singleton { object: PyObjectRef, ) -> PyResult<Self> { if vm.is_none(&object) { - Ok(ruff::Singleton::None) + Ok(ast::Singleton::None) } else if object.is(&vm.ctx.true_value) { - Ok(ruff::Singleton::True) + Ok(ast::Singleton::True) } else if object.is(&vm.ctx.false_value) { - Ok(ruff::Singleton::False) + Ok(ast::Singleton::False) } else { Err(vm.new_value_error(format!( "Expected None, True, or False, got {:?}", @@ -228,103 +228,100 @@ impl Node for ruff::Singleton { } // constructor -impl Node for ruff::PatternMatchSequence { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchSequence { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, patterns, - range: _range, + range, } = self; let node = NodeAst .into_ref_with_type( - _vm, + vm, pyast::NodePatternMatchSequence::static_type().to_owned(), ) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm, source_file), _vm) + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), patterns: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "patterns", "MatchSequence")?, + get_node_field(vm, &object, "patterns", "MatchSequence")?, )?, - range: range_from_object(_vm, source_file, _object, "MatchSequence")?, + range: range_from_object(vm, source_file, object, "MatchSequence")?, }) } } // constructor -impl Node for ruff::PatternMatchMapping { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchMapping { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, keys, patterns, rest, - range: _range, + range, } = self; let node = NodeAst - .into_ref_with_type( - _vm, - pyast::NodePatternMatchMapping::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodePatternMatchMapping::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("keys", keys.ast_to_object(_vm, source_file), _vm) + dict.set_item("keys", keys.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm, source_file), _vm) + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("rest", rest.ast_to_object(_vm, source_file), _vm) + dict.set_item("rest", rest.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), keys: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "keys", "MatchMapping")?, + get_node_field(vm, &object, "keys", "MatchMapping")?, )?, patterns: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "patterns", "MatchMapping")?, + get_node_field(vm, &object, "patterns", "MatchMapping")?, )?, - rest: get_node_field_opt(_vm, &_object, "rest")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + rest: get_node_field_opt(vm, &object, "rest")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, - range: range_from_object(_vm, source_file, _object, "MatchMapping")?, + range: range_from_object(vm, source_file, object, "MatchMapping")?, }) } } // constructor -impl Node for ruff::PatternMatchClass { +impl Node for ast::PatternMatchClass { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, cls, arguments, - range: _range, + range, } = self; let (patterns, kwd_attrs, kwd_patterns) = split_pattern_match_class(arguments); let node = NodeAst @@ -343,7 +340,7 @@ impl Node for ruff::PatternMatchClass { vm, ) .unwrap(); - node_add_location(&dict, _range, vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } @@ -357,16 +354,21 @@ impl Node for ruff::PatternMatchClass { source_file, get_node_field(vm, &object, "patterns", "MatchClass")?, )?; - let kwd_attrs = Node::ast_from_object( + let kwd_attrs: PatternMatchClassKeywordAttributes = Node::ast_from_object( vm, source_file, get_node_field(vm, &object, "kwd_attrs", "MatchClass")?, )?; - let kwd_patterns = Node::ast_from_object( + let kwd_patterns: PatternMatchClassKeywordPatterns = Node::ast_from_object( vm, source_file, get_node_field(vm, &object, "kwd_patterns", "MatchClass")?, )?; + if kwd_attrs.0.len() != kwd_patterns.0.len() { + return Err(vm.new_value_error( + "MatchClass has mismatched kwd_attrs and kwd_patterns".to_owned(), + )); + } let (patterns, keywords) = merge_pattern_match_class(patterns, kwd_attrs, kwd_patterns); Ok(Self { @@ -377,7 +379,7 @@ impl Node for ruff::PatternMatchClass { get_node_field(vm, &object, "cls", "MatchClass")?, )?, range: range_from_object(vm, source_file, object, "MatchClass")?, - arguments: ruff::PatternArguments { + arguments: ast::PatternArguments { node_index: Default::default(), range: Default::default(), patterns, @@ -387,7 +389,7 @@ impl Node for ruff::PatternMatchClass { } } -struct PatternMatchClassPatterns(Vec<ruff::Pattern>); +struct PatternMatchClassPatterns(Vec<ast::Pattern>); impl Node for PatternMatchClassPatterns { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -403,7 +405,7 @@ impl Node for PatternMatchClassPatterns { } } -struct PatternMatchClassKeywordAttributes(Vec<ruff::Identifier>); +struct PatternMatchClassKeywordAttributes(Vec<ast::Identifier>); impl Node for PatternMatchClassKeywordAttributes { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -419,7 +421,7 @@ impl Node for PatternMatchClassKeywordAttributes { } } -struct PatternMatchClassKeywordPatterns(Vec<ruff::Pattern>); +struct PatternMatchClassKeywordPatterns(Vec<ast::Pattern>); impl Node for PatternMatchClassKeywordPatterns { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -435,113 +437,113 @@ impl Node for PatternMatchClassKeywordPatterns { } } // constructor -impl Node for ruff::PatternMatchStar { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchStar { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, name, - range: _range, + range, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodePatternMatchStar::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchStar::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, - range: range_from_object(_vm, source_file, _object, "MatchStar")?, + range: range_from_object(vm, source_file, object, "MatchStar")?, }) } } // constructor -impl Node for ruff::PatternMatchAs { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchAs { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, pattern, name, - range: _range, + range, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodePatternMatchAs::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchAs::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm, source_file), _vm) + dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), - pattern: get_node_field_opt(_vm, &_object, "pattern")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + pattern: get_node_field_opt(vm, &object, "pattern")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, - range: range_from_object(_vm, source_file, _object, "MatchAs")?, + range: range_from_object(vm, source_file, object, "MatchAs")?, }) } } // constructor -impl Node for ruff::PatternMatchOr { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::PatternMatchOr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, patterns, - range: _range, + range, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodePatternMatchOr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchOr::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm, source_file), _vm) + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), patterns: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "patterns", "MatchOr")?, + get_node_field(vm, &object, "patterns", "MatchOr")?, )?, - range: range_from_object(_vm, source_file, _object, "MatchOr")?, + range: range_from_object(vm, source_file, object, "MatchOr")?, }) } } fn split_pattern_match_class( - arguments: ruff::PatternArguments, + arguments: ast::PatternArguments, ) -> ( PatternMatchClassPatterns, PatternMatchClassKeywordAttributes, @@ -562,12 +564,12 @@ fn merge_pattern_match_class( patterns: PatternMatchClassPatterns, kwd_attrs: PatternMatchClassKeywordAttributes, kwd_patterns: PatternMatchClassKeywordPatterns, -) -> (Vec<ruff::Pattern>, Vec<ruff::PatternKeyword>) { +) -> (Vec<ast::Pattern>, Vec<ast::PatternKeyword>) { let keywords = kwd_attrs .0 .into_iter() .zip(kwd_patterns.0) - .map(|(attr, pattern)| ruff::PatternKeyword { + .map(|(attr, pattern)| ast::PatternKeyword { range: Default::default(), node_index: Default::default(), attr, diff --git a/crates/vm/src/stdlib/ast/pyast.rs b/crates/vm/src/stdlib/ast/pyast.rs index e36635fe4b9..14245a56c09 100644 --- a/crates/vm/src/stdlib/ast/pyast.rs +++ b/crates/vm/src/stdlib/ast/pyast.rs @@ -1,7 +1,11 @@ #![allow(clippy::all)] use super::*; +use crate::builtins::{PyGenericAlias, PyTuple, PyTupleRef, PyTypeRef, make_union}; use crate::common::ascii; +use crate::convert::ToPyObject; +use crate::function::FuncArgs; +use crate::types::Initializer; macro_rules! impl_node { ( @@ -14,29 +18,7 @@ macro_rules! impl_node { #[repr(transparent)] $vis struct $name($base); - #[pyclass(flags(HAS_DICT, BASETYPE))] - impl $name { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - $( - ctx.new_str(ascii!($field)).into() - ),* - ]).into(), - ); - - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - $( - ctx.new_str(ascii!($attr)).into() - ),* - ]).into(), - ); - } - } + impl_base_node!($name, fields: [$($field),*], attributes: [$($attr),*]); }; // Without attributes ( @@ -78,11 +60,104 @@ macro_rules! impl_node { }; } +macro_rules! impl_base_node { + // Base node without fields/attributes (e.g. NodeMod, NodeExpr) + ($name:ident) => { + #[pyclass(flags(HAS_DICT, BASETYPE))] + impl $name { + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + super::python::_ast::ast_reduce(zelf, vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + super::python::_ast::ast_replace(zelf, args, vm) + } + + #[extend_class] + fn extend_class(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types in CPython, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _attributes), + ctx.empty_tuple.clone().into(), + ); + } + } + }; + // Leaf node with fields and attributes + ($name:ident, fields: [$($field:expr),*], attributes: [$($attr:expr),*]) => { + #[pyclass(flags(HAS_DICT, BASETYPE))] + impl $name { + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + super::python::_ast::ast_reduce(zelf, vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + super::python::_ast::ast_replace(zelf, args, vm) + } + + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types in CPython, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($field)).into() + ),* + ]) + .into(), + ); + + class.set_str_attr( + "__match_args__", + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($field)).into() + ),* + ]), + ctx, + ); + + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($attr)).into() + ),* + ]) + .into(), + ); + + // Signal that this is a built-in AST node with field defaults + class.set_attr( + ctx.intern_str("_field_types"), + ctx.new_dict().into(), + ); + } + } + }; +} + #[pyclass(module = "_ast", name = "mod", base = NodeAst)] pub(crate) struct NodeMod(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeMod {} +impl_base_node!(NodeMod); impl_node!( #[pyclass(module = "_ast", name = "Module", base = NodeMod)] @@ -106,8 +181,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeStmt(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmt {} +impl_base_node!(NodeStmt); impl_node!( #[pyclass(module = "_ast", name = "FunctionType", base = NodeMod)] @@ -306,8 +380,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeExpr(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExpr {} +impl_base_node!(NodeExpr); impl_node!( #[pyclass(module = "_ast", name = "Continue", base = NodeStmt)] @@ -449,12 +522,84 @@ impl_node!( ); impl_node!( - #[pyclass(module = "_ast", name = "Constant", base = NodeExpr)] - pub(crate) struct NodeExprConstant, - fields: ["value", "kind"], + #[pyclass(module = "_ast", name = "TemplateStr", base = NodeExpr)] + pub(crate) struct NodeExprTemplateStr, + fields: ["values"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); +impl_node!( + #[pyclass(module = "_ast", name = "Interpolation", base = NodeExpr)] + pub(crate) struct NodeExprInterpolation, + fields: ["value", "str", "conversion", "format_spec"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +// NodeExprConstant needs custom Initializer to default kind to None +#[pyclass(module = "_ast", name = "Constant", base = NodeExpr)] +#[repr(transparent)] +pub(crate) struct NodeExprConstant(NodeExpr); + +#[pyclass(flags(HAS_DICT, BASETYPE), with(Initializer))] +impl NodeExprConstant { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("kind")).into(), + ]) + .into(), + ); + + class.set_str_attr( + "__match_args__", + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("kind")).into(), + ]), + ctx, + ); + + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} + +impl Initializer for NodeExprConstant { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + <NodeAst as Initializer>::slot_init(zelf.clone(), args, vm)?; + // kind defaults to None if not provided + let dict = zelf.as_object().dict().unwrap(); + if !dict.contains_key("kind", vm) { + dict.set_item("kind", vm.ctx.none(), vm)?; + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + impl_node!( #[pyclass(module = "_ast", name = "Attribute", base = NodeExpr)] pub(crate) struct NodeExprAttribute, @@ -501,8 +646,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeExprContext(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContext {} +impl_base_node!(NodeExprContext); impl_node!( #[pyclass(module = "_ast", name = "Slice", base = NodeExpr)] @@ -525,8 +669,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeBoolOp(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOp {} +impl_base_node!(NodeBoolOp); impl_node!( #[pyclass(module = "_ast", name = "Del", base = NodeExprContext)] @@ -542,8 +685,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeOperator(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperator {} +impl_base_node!(NodeOperator); impl_node!( #[pyclass(module = "_ast", name = "Or", base = NodeBoolOp)] @@ -614,8 +756,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeUnaryOp(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOp {} +impl_base_node!(NodeUnaryOp); impl_node!( #[pyclass(module = "_ast", name = "FloorDiv", base = NodeOperator)] @@ -641,8 +782,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeCmpOp(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOp {} +impl_base_node!(NodeCmpOp); impl_node!( #[pyclass(module = "_ast", name = "USub", base = NodeUnaryOp)] @@ -703,8 +843,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeExceptHandler(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExceptHandler {} +impl_base_node!(NodeExceptHandler); impl_node!( #[pyclass(module = "_ast", name = "comprehension", base = NodeAst)] @@ -756,8 +895,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodePattern(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePattern {} +impl_base_node!(NodePattern); impl_node!( #[pyclass(module = "_ast", name = "match_case", base = NodeAst)] @@ -818,8 +956,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeTypeIgnore(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeIgnore {} +impl_base_node!(NodeTypeIgnore); impl_node!( #[pyclass(module = "_ast", name = "MatchOr", base = NodePattern)] @@ -832,8 +969,7 @@ impl_node!( #[repr(transparent)] pub(crate) struct NodeTypeParam(NodeAst); -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParam {} +impl_base_node!(NodeTypeParam); impl_node!( #[pyclass(module = "_ast", name = "TypeIgnore", base = NodeTypeIgnore)] @@ -844,26 +980,550 @@ impl_node!( impl_node!( #[pyclass(module = "_ast", name = "TypeVar", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVar, - fields: ["name", "bound"], + fields: ["name", "bound", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( #[pyclass(module = "_ast", name = "ParamSpec", base = NodeTypeParam)] pub(crate) struct NodeTypeParamParamSpec, - fields: ["name"], + fields: ["name", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( #[pyclass(module = "_ast", name = "TypeVarTuple", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVarTuple, - fields: ["name"], - attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], -); + fields: ["name", "default_value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +/// Marker for how to resolve an ASDL field type into a Python type object. +#[derive(Clone, Copy)] +enum FieldType { + /// AST node type reference (e.g. "expr", "stmt") + Node(&'static str), + /// Built-in type reference (e.g. "str", "int", "object") + Builtin(&'static str), + /// list[NodeType] — Py_GenericAlias(list, node_type) + ListOf(&'static str), + /// list[BuiltinType] — Py_GenericAlias(list, builtin_type) + ListOfBuiltin(&'static str), + /// NodeType | None — Union[node_type, None] + Optional(&'static str), + /// BuiltinType | None — Union[builtin_type, None] + OptionalBuiltin(&'static str), +} + +/// Field type annotations for all concrete AST node classes. +/// Derived from add_ast_annotations() in Python-ast.c. +const FIELD_TYPES: &[(&str, &[(&str, FieldType)])] = &[ + // -- mod -- + ( + "Module", + &[ + ("body", FieldType::ListOf("stmt")), + ("type_ignores", FieldType::ListOf("type_ignore")), + ], + ), + ("Interactive", &[("body", FieldType::ListOf("stmt"))]), + ("Expression", &[("body", FieldType::Node("expr"))]), + ( + "FunctionType", + &[ + ("argtypes", FieldType::ListOf("expr")), + ("returns", FieldType::Node("expr")), + ], + ), + // -- stmt -- + ( + "FunctionDef", + &[ + ("name", FieldType::Builtin("str")), + ("args", FieldType::Node("arguments")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("returns", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ( + "AsyncFunctionDef", + &[ + ("name", FieldType::Builtin("str")), + ("args", FieldType::Node("arguments")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("returns", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ( + "ClassDef", + &[ + ("name", FieldType::Builtin("str")), + ("bases", FieldType::ListOf("expr")), + ("keywords", FieldType::ListOf("keyword")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ("Return", &[("value", FieldType::Optional("expr"))]), + ("Delete", &[("targets", FieldType::ListOf("expr"))]), + ( + "Assign", + &[ + ("targets", FieldType::ListOf("expr")), + ("value", FieldType::Node("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "TypeAlias", + &[ + ("name", FieldType::Node("expr")), + ("type_params", FieldType::ListOf("type_param")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "AugAssign", + &[ + ("target", FieldType::Node("expr")), + ("op", FieldType::Node("operator")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "AnnAssign", + &[ + ("target", FieldType::Node("expr")), + ("annotation", FieldType::Node("expr")), + ("value", FieldType::Optional("expr")), + ("simple", FieldType::Builtin("int")), + ], + ), + ( + "For", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "AsyncFor", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "While", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ], + ), + ( + "If", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ], + ), + ( + "With", + &[ + ("items", FieldType::ListOf("withitem")), + ("body", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "AsyncWith", + &[ + ("items", FieldType::ListOf("withitem")), + ("body", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "Match", + &[ + ("subject", FieldType::Node("expr")), + ("cases", FieldType::ListOf("match_case")), + ], + ), + ( + "Raise", + &[ + ("exc", FieldType::Optional("expr")), + ("cause", FieldType::Optional("expr")), + ], + ), + ( + "Try", + &[ + ("body", FieldType::ListOf("stmt")), + ("handlers", FieldType::ListOf("excepthandler")), + ("orelse", FieldType::ListOf("stmt")), + ("finalbody", FieldType::ListOf("stmt")), + ], + ), + ( + "TryStar", + &[ + ("body", FieldType::ListOf("stmt")), + ("handlers", FieldType::ListOf("excepthandler")), + ("orelse", FieldType::ListOf("stmt")), + ("finalbody", FieldType::ListOf("stmt")), + ], + ), + ( + "Assert", + &[ + ("test", FieldType::Node("expr")), + ("msg", FieldType::Optional("expr")), + ], + ), + ("Import", &[("names", FieldType::ListOf("alias"))]), + ( + "ImportFrom", + &[ + ("module", FieldType::OptionalBuiltin("str")), + ("names", FieldType::ListOf("alias")), + ("level", FieldType::OptionalBuiltin("int")), + ], + ), + ("Global", &[("names", FieldType::ListOfBuiltin("str"))]), + ("Nonlocal", &[("names", FieldType::ListOfBuiltin("str"))]), + ("Expr", &[("value", FieldType::Node("expr"))]), + // -- expr -- + ( + "BoolOp", + &[ + ("op", FieldType::Node("boolop")), + ("values", FieldType::ListOf("expr")), + ], + ), + ( + "NamedExpr", + &[ + ("target", FieldType::Node("expr")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "BinOp", + &[ + ("left", FieldType::Node("expr")), + ("op", FieldType::Node("operator")), + ("right", FieldType::Node("expr")), + ], + ), + ( + "UnaryOp", + &[ + ("op", FieldType::Node("unaryop")), + ("operand", FieldType::Node("expr")), + ], + ), + ( + "Lambda", + &[ + ("args", FieldType::Node("arguments")), + ("body", FieldType::Node("expr")), + ], + ), + ( + "IfExp", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::Node("expr")), + ("orelse", FieldType::Node("expr")), + ], + ), + ( + "Dict", + &[ + ("keys", FieldType::ListOf("expr")), + ("values", FieldType::ListOf("expr")), + ], + ), + ("Set", &[("elts", FieldType::ListOf("expr"))]), + ( + "ListComp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "SetComp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "DictComp", + &[ + ("key", FieldType::Node("expr")), + ("value", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "GeneratorExp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ("Await", &[("value", FieldType::Node("expr"))]), + ("Yield", &[("value", FieldType::Optional("expr"))]), + ("YieldFrom", &[("value", FieldType::Node("expr"))]), + ( + "Compare", + &[ + ("left", FieldType::Node("expr")), + ("ops", FieldType::ListOf("cmpop")), + ("comparators", FieldType::ListOf("expr")), + ], + ), + ( + "Call", + &[ + ("func", FieldType::Node("expr")), + ("args", FieldType::ListOf("expr")), + ("keywords", FieldType::ListOf("keyword")), + ], + ), + ( + "FormattedValue", + &[ + ("value", FieldType::Node("expr")), + ("conversion", FieldType::Builtin("int")), + ("format_spec", FieldType::Optional("expr")), + ], + ), + ("JoinedStr", &[("values", FieldType::ListOf("expr"))]), + ("TemplateStr", &[("values", FieldType::ListOf("expr"))]), + ( + "Interpolation", + &[ + ("value", FieldType::Node("expr")), + ("str", FieldType::Builtin("object")), + ("conversion", FieldType::Builtin("int")), + ("format_spec", FieldType::Optional("expr")), + ], + ), + ( + "Constant", + &[ + ("value", FieldType::Builtin("object")), + ("kind", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "Attribute", + &[ + ("value", FieldType::Node("expr")), + ("attr", FieldType::Builtin("str")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Subscript", + &[ + ("value", FieldType::Node("expr")), + ("slice", FieldType::Node("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Starred", + &[ + ("value", FieldType::Node("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Name", + &[ + ("id", FieldType::Builtin("str")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "List", + &[ + ("elts", FieldType::ListOf("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Tuple", + &[ + ("elts", FieldType::ListOf("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Slice", + &[ + ("lower", FieldType::Optional("expr")), + ("upper", FieldType::Optional("expr")), + ("step", FieldType::Optional("expr")), + ], + ), + // -- misc -- + ( + "comprehension", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("ifs", FieldType::ListOf("expr")), + ("is_async", FieldType::Builtin("int")), + ], + ), + ( + "ExceptHandler", + &[ + ("type", FieldType::Optional("expr")), + ("name", FieldType::OptionalBuiltin("str")), + ("body", FieldType::ListOf("stmt")), + ], + ), + ( + "arguments", + &[ + ("posonlyargs", FieldType::ListOf("arg")), + ("args", FieldType::ListOf("arg")), + ("vararg", FieldType::Optional("arg")), + ("kwonlyargs", FieldType::ListOf("arg")), + ("kw_defaults", FieldType::ListOf("expr")), + ("kwarg", FieldType::Optional("arg")), + ("defaults", FieldType::ListOf("expr")), + ], + ), + ( + "arg", + &[ + ("arg", FieldType::Builtin("str")), + ("annotation", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "keyword", + &[ + ("arg", FieldType::OptionalBuiltin("str")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "alias", + &[ + ("name", FieldType::Builtin("str")), + ("asname", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "withitem", + &[ + ("context_expr", FieldType::Node("expr")), + ("optional_vars", FieldType::Optional("expr")), + ], + ), + ( + "match_case", + &[ + ("pattern", FieldType::Node("pattern")), + ("guard", FieldType::Optional("expr")), + ("body", FieldType::ListOf("stmt")), + ], + ), + // -- pattern -- + ("MatchValue", &[("value", FieldType::Node("expr"))]), + ("MatchSingleton", &[("value", FieldType::Builtin("object"))]), + ( + "MatchSequence", + &[("patterns", FieldType::ListOf("pattern"))], + ), + ( + "MatchMapping", + &[ + ("keys", FieldType::ListOf("expr")), + ("patterns", FieldType::ListOf("pattern")), + ("rest", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "MatchClass", + &[ + ("cls", FieldType::Node("expr")), + ("patterns", FieldType::ListOf("pattern")), + ("kwd_attrs", FieldType::ListOfBuiltin("str")), + ("kwd_patterns", FieldType::ListOf("pattern")), + ], + ), + ("MatchStar", &[("name", FieldType::OptionalBuiltin("str"))]), + ( + "MatchAs", + &[ + ("pattern", FieldType::Optional("pattern")), + ("name", FieldType::OptionalBuiltin("str")), + ], + ), + ("MatchOr", &[("patterns", FieldType::ListOf("pattern"))]), + // -- type_ignore -- + ( + "TypeIgnore", + &[ + ("lineno", FieldType::Builtin("int")), + ("tag", FieldType::Builtin("str")), + ], + ), + // -- type_param -- + ( + "TypeVar", + &[ + ("name", FieldType::Builtin("str")), + ("bound", FieldType::Optional("expr")), + ("default_value", FieldType::Optional("expr")), + ], + ), + ( + "ParamSpec", + &[ + ("name", FieldType::Builtin("str")), + ("default_value", FieldType::Optional("expr")), + ], + ), + ( + "TypeVarTuple", + &[ + ("name", FieldType::Builtin("str")), + ("default_value", FieldType::Optional("expr")), + ], + ), +]; pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { extend_module!(vm, module, { + "AST" => NodeAst::make_class(&vm.ctx), "mod" => NodeMod::make_class(&vm.ctx), "Module" => NodeModModule::make_class(&vm.ctx), "Interactive" => NodeModInteractive::make_class(&vm.ctx), @@ -918,6 +1578,8 @@ pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { "Call" => NodeExprCall::make_class(&vm.ctx), "FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx), "JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx), + "TemplateStr" => NodeExprTemplateStr::make_class(&vm.ctx), + "Interpolation" => NodeExprInterpolation::make_class(&vm.ctx), "Constant" => NodeExprConstant::make_class(&vm.ctx), "Attribute" => NodeExprAttribute::make_class(&vm.ctx), "Subscript" => NodeExprSubscript::make_class(&vm.ctx), @@ -987,5 +1649,187 @@ pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { "TypeVar" => NodeTypeParamTypeVar::make_class(&vm.ctx), "ParamSpec" => NodeTypeParamParamSpec::make_class(&vm.ctx), "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_class(&vm.ctx), - }) + }); + + // Populate _field_types with real Python type objects + populate_field_types(vm, module); + populate_singletons(vm, module); + force_ast_module_name(vm, module); + populate_repr(vm, module); +} + +fn populate_field_types(vm: &VirtualMachine, module: &Py<PyModule>) { + let list_type: PyTypeRef = vm.ctx.types.list_type.to_owned(); + let none_type: PyObjectRef = vm.ctx.types.none_type.to_owned().into(); + + // Resolve a builtin type name to a Python type object + let resolve_builtin = |name: &str| -> PyObjectRef { + let ty: &Py<PyType> = match name { + "str" => vm.ctx.types.str_type, + "int" => vm.ctx.types.int_type, + "object" => vm.ctx.types.object_type, + "bool" => vm.ctx.types.bool_type, + _ => unreachable!("unknown builtin type: {name}"), + }; + ty.to_owned().into() + }; + + // Resolve an AST node type name by looking it up from the module + let resolve_node = |name: &str| -> PyObjectRef { + module + .get_attr(vm.ctx.intern_str(name), vm) + .unwrap_or_else(|_| panic!("AST node type '{name}' not found in module")) + }; + + let field_types_attr = vm.ctx.intern_str("_field_types"); + let annotations_attr = vm.ctx.intern_str("__annotations__"); + let empty_dict: PyObjectRef = vm.ctx.new_dict().into(); + + for &(class_name, fields) in FIELD_TYPES { + if fields.is_empty() { + continue; + } + + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let dict = vm.ctx.new_dict(); + + for &(field_name, ref field_type) in fields { + let type_obj = match field_type { + FieldType::Node(name) => resolve_node(name), + FieldType::Builtin(name) => resolve_builtin(name), + FieldType::ListOf(name) => { + let elem = resolve_node(name); + let args = PyTuple::new_ref(vec![elem], &vm.ctx); + PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + } + FieldType::ListOfBuiltin(name) => { + let elem = resolve_builtin(name); + let args = PyTuple::new_ref(vec![elem], &vm.ctx); + PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + } + FieldType::Optional(name) => { + let base = resolve_node(name); + let union_args = PyTuple::new_ref(vec![base, none_type.clone()], &vm.ctx); + make_union(&union_args, vm).expect("failed to create union type") + } + FieldType::OptionalBuiltin(name) => { + let base = resolve_builtin(name); + let union_args = PyTuple::new_ref(vec![base, none_type.clone()], &vm.ctx); + make_union(&union_args, vm).expect("failed to create union type") + } + }; + dict.set_item(vm.ctx.intern_str(field_name), type_obj, vm) + .expect("failed to set field type"); + } + + let dict_obj: PyObjectRef = dict.into(); + if let Some(type_obj) = class.downcast_ref::<PyType>() { + type_obj.set_attr(field_types_attr, dict_obj.clone()); + type_obj.set_attr(annotations_attr, dict_obj); + + // Set None as class-level default for optional fields. + // When ast_type_init skips optional fields, the instance + // inherits None from the class (init_types in Python-ast.c). + let none = vm.ctx.none(); + for &(field_name, ref field_type) in fields { + if matches!( + field_type, + FieldType::Optional(_) | FieldType::OptionalBuiltin(_) + ) { + type_obj.set_attr(vm.ctx.intern_str(field_name), none.clone()); + } + } + } + } + + // CPython sets __annotations__ for all built-in AST node classes, even + // when _field_types is an empty dict (e.g., operators, Load/Store/Del). + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + if let Some(field_types) = type_obj.get_attr(field_types_attr) { + type_obj.set_attr(annotations_attr, field_types); + } + } + + // Base AST classes (e.g., expr, stmt) should still expose __annotations__. + const BASE_AST_TYPES: &[&str] = &[ + "mod", + "stmt", + "expr", + "expr_context", + "boolop", + "operator", + "unaryop", + "cmpop", + "excepthandler", + "pattern", + "type_ignore", + "type_param", + ]; + for &class_name in BASE_AST_TYPES { + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let Some(type_obj) = class.downcast_ref::<PyType>() else { + continue; + }; + if type_obj.get_attr(field_types_attr).is_none() { + type_obj.set_attr(field_types_attr, empty_dict.clone()); + } + if type_obj.get_attr(annotations_attr).is_none() { + type_obj.set_attr(annotations_attr, empty_dict.clone()); + } + } +} + +fn populate_singletons(vm: &VirtualMachine, module: &Py<PyModule>) { + let instance_attr = vm.ctx.intern_str("_instance"); + const SINGLETON_TYPES: &[&str] = &[ + // expr_context + "Load", "Store", "Del", // boolop + "And", "Or", // operator + "Add", "Sub", "Mult", "MatMult", "Div", "Mod", "Pow", "LShift", "RShift", "BitOr", + "BitXor", "BitAnd", "FloorDiv", // unaryop + "Invert", "Not", "UAdd", "USub", // cmpop + "Eq", "NotEq", "Lt", "LtE", "Gt", "GtE", "Is", "IsNot", "In", "NotIn", + ]; + + for &class_name in SINGLETON_TYPES { + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let Some(type_obj) = class.downcast_ref::<PyType>() else { + continue; + }; + let instance = vm + .ctx + .new_base_object(type_obj.to_owned(), Some(vm.ctx.new_dict())); + type_obj.set_attr(instance_attr, instance); + } +} + +fn force_ast_module_name(vm: &VirtualMachine, module: &Py<PyModule>) { + let ast_name = vm.ctx.new_str("ast"); + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + type_obj.set_attr(identifier!(vm, __module__), ast_name.clone().into()); + } +} + +fn populate_repr(_vm: &VirtualMachine, module: &Py<PyModule>) { + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + type_obj + .slots + .repr + .store(Some(super::python::_ast::ast_repr)); + } } diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 042db4aa74e..0de6f45b912 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -1,62 +1,253 @@ -use super::{PY_CF_OPTIMIZED_AST, PY_CF_TYPE_COMMENTS, PY_COMPILE_FLAG_AST_ONLY}; +use super::{ + PY_CF_ALLOW_INCOMPLETE_INPUT, PY_CF_ALLOW_TOP_LEVEL_AWAIT, PY_CF_DONT_IMPLY_DEDENT, + PY_CF_IGNORE_COOKIE, PY_CF_ONLY_AST, PY_CF_OPTIMIZED_AST, PY_CF_SOURCE_IS_UTF8, + PY_CF_TYPE_COMMENTS, +}; #[pymodule] pub(crate) mod _ast { use crate::{ - AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyStrRef, PyTupleRef, PyType, PyTypeRef}, - function::FuncArgs, - types::Constructor, + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef}, + class::{PyClassImpl, StaticType}, + function::{FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, + stdlib::ast::repr, + types::{Constructor, Initializer}, + warn, }; + use indexmap::IndexMap; #[pyattr] #[pyclass(module = "_ast", name = "AST")] #[derive(Debug, PyPayload)] pub(crate) struct NodeAst; - #[pyclass(with(Constructor), flags(BASETYPE, HAS_DICT))] + #[pyclass(with(Constructor, Initializer), flags(BASETYPE, HAS_DICT))] impl NodeAst { - #[pyslot] + #[extend_class] + fn extend_class(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + let empty_tuple = ctx.empty_tuple.clone(); + class.set_str_attr("_fields", empty_tuple.clone(), ctx); + class.set_str_attr("_attributes", empty_tuple.clone(), ctx); + class.set_str_attr("__match_args__", empty_tuple.clone(), ctx); + + const AST_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyObjectRef, vm: &VirtualMachine| -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + }, + PyMethodFlags::METHOD, + None, + ); + const AST_REPLACE: PyMethodDef = PyMethodDef::new_const( + "__replace__", + |zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine| -> PyResult { + ast_replace(zelf, args, vm) + }, + PyMethodFlags::METHOD, + None, + ); + + class.set_str_attr("__reduce__", AST_REDUCE.to_proper_method(class, ctx), ctx); + class.set_str_attr("__replace__", AST_REPLACE.to_proper_method(class, ctx), ctx); + class.slots.repr.store(Some(ast_repr)); + } + + #[pyattr] + fn _fields(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pyattr] + fn _attributes(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pyattr] + fn __match_args__(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + } + #[pymethod] - fn __init__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let fields = zelf.get_attr("_fields", vm)?; + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + ast_replace(zelf, args, vm) + } + } + + pub(crate) fn ast_reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let dict = zelf.as_object().dict(); + let cls = zelf.class(); + let type_obj: PyObjectRef = cls.to_owned().into(); + + let Some(dict) = dict else { + return Ok(vm.ctx.new_tuple(vec![type_obj])); + }; + + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + if let Some(fields) = fields { let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; - let n_args = args.args.len(); - if n_args > fields.len() { + let mut positional: Vec<PyObjectRef> = Vec::new(); + for field in fields { + if dict.get_item_opt::<str>(field.as_str(), vm)?.is_some() { + positional.push(vm.ctx.none()); + } else { + break; + } + } + let args: PyObjectRef = vm.ctx.new_tuple(positional).into(); + let dict_obj: PyObjectRef = dict.into(); + return Ok(vm.ctx.new_tuple(vec![type_obj, args, dict_obj])); + } + + Ok(vm + .ctx + .new_tuple(vec![type_obj, vm.ctx.new_tuple(vec![]).into(), dict.into()])) + } + + pub(crate) fn ast_replace(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments".to_owned())); + } + + let cls = zelf.class(); + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + let attributes = cls.get_attr(vm.ctx.intern_str("_attributes")); + let dict = zelf.as_object().dict(); + + let mut expecting: std::collections::HashSet<String> = std::collections::HashSet::new(); + if let Some(fields) = fields.clone() { + let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; + for field in fields { + expecting.insert(field.as_str().to_owned()); + } + } + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyStrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + expecting.insert(attr.as_str().to_owned()); + } + } + + for (key, _value) in &args.kwargs { + if !expecting.remove(key) { return Err(vm.new_type_error(format!( - "{} constructor takes at most {} positional argument{}", - zelf.class().name(), - fields.len(), - if fields.len() == 1 { "" } else { "s" }, + "{}.__replace__ got an unexpected keyword argument '{}'.", + cls.name(), + key ))); } - for (name, arg) in fields.iter().zip(args.args) { - zelf.set_attr(name, arg, vm)?; + } + + if let Some(dict) = dict.as_ref() { + for (key, _value) in dict.items_vec() { + if let Ok(key) = key.downcast::<PyStr>() { + expecting.remove(key.as_str()); + } } - for (key, value) in args.kwargs { - if let Some(pos) = fields.iter().position(|f| f.as_str() == key) - && pos < n_args - { - return Err(vm.new_type_error(format!( - "{} got multiple values for argument '{}'", - zelf.class().name(), - key - ))); + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyStrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + expecting.remove(attr.as_str()); } - zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; } - Ok(()) } - #[pyattr(name = "_fields")] - fn fields(ctx: &Context) -> PyTupleRef { - ctx.empty_tuple.clone() + // Discard optional fields (T | None). + if let Some(field_types) = cls.get_attr(vm.ctx.intern_str("_field_types")) + && let Ok(field_types) = field_types.downcast::<crate::builtins::PyDict>() + { + for (key, value) in field_types.items_vec() { + let Ok(key) = key.downcast::<PyStr>() else { + continue; + }; + if value.fast_isinstance(vm.ctx.types.union_type) { + expecting.remove(key.as_str()); + } + } + } + + if !expecting.is_empty() { + let mut names: Vec<String> = expecting + .into_iter() + .map(|name| format!("{name:?}")) + .collect(); + names.sort(); + let missing = names.join(", "); + let count = names.len(); + return Err(vm.new_type_error(format!( + "{}.__replace__ missing {} keyword argument{}: {}.", + cls.name(), + count, + if count == 1 { "" } else { "s" }, + missing + ))); } + + let payload = vm.ctx.new_dict(); + if let Some(dict) = dict { + if let Some(fields) = fields.clone() { + let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; + for field in fields { + if let Some(value) = dict.get_item_opt::<str>(field.as_str(), vm)? { + payload.set_item(field.as_object(), value, vm)?; + } + } + } + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyStrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + if let Some(value) = dict.get_item_opt::<str>(attr.as_str(), vm)? { + payload.set_item(attr.as_object(), value, vm)?; + } + } + } + } + for (key, value) in args.kwargs { + payload.set_item(vm.ctx.intern_str(key), value, vm)?; + } + + let type_obj: PyObjectRef = cls.to_owned().into(); + let kwargs = payload + .items_vec() + .into_iter() + .map(|(key, value)| { + let key = key + .downcast::<PyStr>() + .map_err(|_| vm.new_type_error("keywords must be strings".to_owned()))?; + Ok((key.as_str().to_owned(), value)) + }) + .collect::<PyResult<IndexMap<String, PyObjectRef>>>()?; + let result = type_obj.call(FuncArgs::new(vec![], KwArgs::new(kwargs)), vm)?; + Ok(result) + } + + pub(crate) fn ast_repr(zelf: &crate::PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let repr = repr::repr_ast_node(vm, &zelf.to_owned(), 3)?; + Ok(vm.ctx.new_str(repr)) } impl Constructor for NodeAst { type Args = FuncArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if args.args.is_empty() + && args.kwargs.is_empty() + && let Some(instance) = cls.get_attr(vm.ctx.intern_str("_instance")) + { + return Ok(instance); + } + // AST nodes accept extra arguments (unlike object.__new__) // This matches CPython's behavior where AST has its own tp_new let dict = if cls @@ -70,8 +261,9 @@ pub(crate) mod _ast { }; let zelf = vm.ctx.new_base_object(cls, dict); - // Initialize the instance with the provided arguments - Self::__init__(zelf.clone(), args, vm)?; + // type.__call__ does not invoke slot_init after slot_new + // for types with a custom slot_new, so we must call it here. + Self::slot_init(zelf.clone(), args, vm)?; Ok(zelf) } @@ -81,12 +273,249 @@ pub(crate) mod _ast { } } + impl Initializer for NodeAst { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let fields = zelf + .class() + .get_attr(vm.ctx.intern_str("_fields")) + .ok_or_else(|| { + let module = zelf + .class() + .get_attr(vm.ctx.intern_str("__module__")) + .and_then(|obj| obj.try_to_value::<String>(vm).ok()) + .unwrap_or_else(|| "ast".to_owned()); + vm.new_attribute_error(format!( + "type object '{}.{}' has no attribute '_fields'", + module, + zelf.class().name() + )) + })?; + let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; + let n_args = args.args.len(); + if n_args > fields.len() { + return Err(vm.new_type_error(format!( + "{} constructor takes at most {} positional argument{}", + zelf.class().name(), + fields.len(), + if fields.len() == 1 { "" } else { "s" }, + ))); + } + + // Track which fields were set + let mut set_fields = std::collections::HashSet::new(); + let mut attributes: Option<Vec<PyStrRef>> = None; + + for (name, arg) in fields.iter().zip(args.args) { + zelf.set_attr(name, arg, vm)?; + set_fields.insert(name.as_str().to_string()); + } + for (key, value) in args.kwargs { + if let Some(pos) = fields.iter().position(|f| f.as_str() == key) + && pos < n_args + { + return Err(vm.new_type_error(format!( + "{} got multiple values for argument '{}'", + zelf.class().name(), + key + ))); + } + + if fields.iter().all(|field| field.as_str() != key) { + let attrs = if let Some(attrs) = &attributes { + attrs + } else { + let attrs = zelf + .class() + .get_attr(vm.ctx.intern_str("_attributes")) + .and_then(|attr| attr.try_to_value::<Vec<PyStrRef>>(vm).ok()) + .unwrap_or_default(); + attributes = Some(attrs); + attributes.as_ref().unwrap() + }; + if attrs.iter().all(|attr| attr.as_str() != key) { + let message = vm.ctx.new_str(format!( + "{}.__init__ got an unexpected keyword argument '{}'. \ +Support for arbitrary keyword arguments is deprecated and will be removed in Python 3.15.", + zelf.class().name(), + key + )); + warn::warn( + message.into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + } + + set_fields.insert(key.clone()); + zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; + } + + // Use _field_types to determine defaults for unset fields. + // Only built-in AST node classes have _field_types populated. + let field_types = zelf.class().get_attr(vm.ctx.intern_str("_field_types")); + if let Some(Ok(ft_dict)) = + field_types.map(|ft| ft.downcast::<crate::builtins::PyDict>()) + { + let expr_ctx_type: PyObjectRef = + super::super::pyast::NodeExprContext::make_class(&vm.ctx).into(); + + for field in &fields { + if set_fields.contains(field.as_str()) { + continue; + } + if let Some(ftype) = ft_dict.get_item_opt::<str>(field.as_str(), vm)? { + if ftype.fast_isinstance(vm.ctx.types.union_type) { + // Optional field (T | None) — no default + } else if ftype.fast_isinstance(vm.ctx.types.generic_alias_type) { + // List field (list[T]) — default to [] + let empty_list: PyObjectRef = vm.ctx.new_list(vec![]).into(); + zelf.set_attr(vm.ctx.intern_str(field.as_str()), empty_list, vm)?; + } else if ftype.is(&expr_ctx_type) { + // expr_context — default to Load() + let load_type = + super::super::pyast::NodeExprContextLoad::make_class(&vm.ctx); + let load_instance = load_type + .get_attr(vm.ctx.intern_str("_instance")) + .unwrap_or_else(|| { + vm.ctx.new_base_object(load_type, Some(vm.ctx.new_dict())) + }); + zelf.set_attr(vm.ctx.intern_str(field.as_str()), load_instance, vm)?; + } else { + // Required field missing: emit DeprecationWarning (CPython behavior). + let message = vm.ctx.new_str(format!( + "{}.__init__ missing 1 required positional argument: '{}'", + zelf.class().name(), + field.as_str() + )); + warn::warn( + message.into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + } + } + } + + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyattr(name = "PyCF_SOURCE_IS_UTF8")] + use super::PY_CF_SOURCE_IS_UTF8; + + #[pyattr(name = "PyCF_DONT_IMPLY_DEDENT")] + use super::PY_CF_DONT_IMPLY_DEDENT; + #[pyattr(name = "PyCF_ONLY_AST")] - use super::PY_COMPILE_FLAG_AST_ONLY; + use super::PY_CF_ONLY_AST; - #[pyattr(name = "PyCF_OPTIMIZED_AST")] - use super::PY_CF_OPTIMIZED_AST; + #[pyattr(name = "PyCF_IGNORE_COOKIE")] + use super::PY_CF_IGNORE_COOKIE; #[pyattr(name = "PyCF_TYPE_COMMENTS")] use super::PY_CF_TYPE_COMMENTS; + + #[pyattr(name = "PyCF_ALLOW_TOP_LEVEL_AWAIT")] + use super::PY_CF_ALLOW_TOP_LEVEL_AWAIT; + + #[pyattr(name = "PyCF_ALLOW_INCOMPLETE_INPUT")] + use super::PY_CF_ALLOW_INCOMPLETE_INPUT; + + #[pyattr(name = "PyCF_OPTIMIZED_AST")] + use super::PY_CF_OPTIMIZED_AST; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::pyast::extend_module_nodes(vm, module); + + let ast_type = module + .get_attr("AST", vm)? + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("AST is not a type".to_owned()))?; + let ctx = &vm.ctx; + let empty_tuple = ctx.empty_tuple.clone(); + ast_type.set_str_attr("_fields", empty_tuple.clone(), ctx); + ast_type.set_str_attr("_attributes", empty_tuple.clone(), ctx); + ast_type.set_str_attr("__match_args__", empty_tuple.clone(), ctx); + + const AST_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyObjectRef, vm: &VirtualMachine| -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + }, + PyMethodFlags::METHOD, + None, + ); + const AST_REPLACE: PyMethodDef = PyMethodDef::new_const( + "__replace__", + |zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine| -> PyResult { + ast_replace(zelf, args, vm) + }, + PyMethodFlags::METHOD, + None, + ); + let base_type = NodeAst::static_type(); + ast_type.set_str_attr( + "__reduce__", + AST_REDUCE.to_proper_method(base_type, ctx), + ctx, + ); + ast_type.set_str_attr( + "__replace__", + AST_REPLACE.to_proper_method(base_type, ctx), + ctx, + ); + ast_type.slots.repr.store(Some(ast_repr)); + + const EXPR_DOC: &str = "expr = BoolOp(boolop op, expr* values)\n\ + | NamedExpr(expr target, expr value)\n\ + | BinOp(expr left, operator op, expr right)\n\ + | UnaryOp(unaryop op, expr operand)\n\ + | Lambda(arguments args, expr body)\n\ + | IfExp(expr test, expr body, expr orelse)\n\ + | Dict(expr?* keys, expr* values)\n\ + | Set(expr* elts)\n\ + | ListComp(expr elt, comprehension* generators)\n\ + | SetComp(expr elt, comprehension* generators)\n\ + | DictComp(expr key, expr value, comprehension* generators)\n\ + | GeneratorExp(expr elt, comprehension* generators)\n\ + | Await(expr value)\n\ + | Yield(expr? value)\n\ + | YieldFrom(expr value)\n\ + | Compare(expr left, cmpop* ops, expr* comparators)\n\ + | Call(expr func, expr* args, keyword* keywords)\n\ + | FormattedValue(expr value, int conversion, expr? format_spec)\n\ + | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n\ + | JoinedStr(expr* values)\n\ + | TemplateStr(expr* values)\n\ + | Constant(constant value, string? kind)\n\ + | Attribute(expr value, identifier attr, expr_context ctx)\n\ + | Subscript(expr value, expr slice, expr_context ctx)\n\ + | Starred(expr value, expr_context ctx)\n\ + | Name(identifier id, expr_context ctx)\n\ + | List(expr* elts, expr_context ctx)\n\ + | Tuple(expr* elts, expr_context ctx)\n\ + | Slice(expr? lower, expr? upper, expr? step)"; + let expr_type = super::super::pyast::NodeExpr::static_type(); + expr_type.set_attr( + identifier!(vm.ctx, __doc__), + vm.ctx.new_str(EXPR_DOC).into(), + ); + Ok(()) + } } diff --git a/crates/vm/src/stdlib/ast/repr.rs b/crates/vm/src/stdlib/ast/repr.rs new file mode 100644 index 00000000000..0810814cd06 --- /dev/null +++ b/crates/vm/src/stdlib/ast/repr.rs @@ -0,0 +1,147 @@ +use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyList, PyTuple}, + class::PyClassImpl, + stdlib::ast::NodeAst, +}; + +fn repr_ast_list(vm: &VirtualMachine, items: Vec<PyObjectRef>, depth: usize) -> PyResult<String> { + if items.is_empty() { + let empty_list: PyObjectRef = vm.ctx.new_list(vec![]).into(); + return Ok(empty_list.repr(vm)?.to_string()); + } + + let mut parts: Vec<String> = Vec::new(); + let first = &items[0]; + let last = items.last().unwrap(); + + for (idx, item) in [first, last].iter().enumerate() { + if idx == 1 && items.len() == 1 { + break; + } + let repr = if item.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { + repr_ast_node(vm, item, depth.saturating_sub(1))? + } else { + item.repr(vm)?.to_string() + }; + parts.push(repr); + } + + let mut rendered = String::from("["); + if !parts.is_empty() { + rendered.push_str(&parts[0]); + } + if items.len() > 2 { + if !parts[0].is_empty() { + rendered.push_str(", ..."); + } + if parts.len() > 1 { + rendered.push_str(", "); + rendered.push_str(&parts[1]); + } + } else if parts.len() > 1 { + rendered.push_str(", "); + rendered.push_str(&parts[1]); + } + rendered.push(']'); + Ok(rendered) +} + +fn repr_ast_tuple(vm: &VirtualMachine, items: Vec<PyObjectRef>, depth: usize) -> PyResult<String> { + if items.is_empty() { + let empty_tuple: PyObjectRef = vm.ctx.empty_tuple.clone().into(); + return Ok(empty_tuple.repr(vm)?.to_string()); + } + + let mut parts: Vec<String> = Vec::new(); + let first = &items[0]; + let last = items.last().unwrap(); + + for (idx, item) in [first, last].iter().enumerate() { + if idx == 1 && items.len() == 1 { + break; + } + let repr = if item.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { + repr_ast_node(vm, item, depth.saturating_sub(1))? + } else { + item.repr(vm)?.to_string() + }; + parts.push(repr); + } + + let mut rendered = String::from("("); + if !parts.is_empty() { + rendered.push_str(&parts[0]); + } + if items.len() > 2 { + if !parts[0].is_empty() { + rendered.push_str(", ..."); + } + if parts.len() > 1 { + rendered.push_str(", "); + rendered.push_str(&parts[1]); + } + } else if parts.len() > 1 { + rendered.push_str(", "); + rendered.push_str(&parts[1]); + } + if items.len() == 1 { + rendered.push(','); + } + rendered.push(')'); + Ok(rendered) +} + +pub(crate) fn repr_ast_node( + vm: &VirtualMachine, + obj: &PyObjectRef, + depth: usize, +) -> PyResult<String> { + let cls = obj.class(); + if depth == 0 { + return Ok(format!("{}(...)", cls.name())); + } + + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + let fields = match fields { + Some(fields) => fields.try_to_value::<Vec<crate::builtins::PyStrRef>>(vm)?, + None => return Ok(format!("{}(...)", cls.name())), + }; + + if fields.is_empty() { + return Ok(format!("{}()", cls.name())); + } + + let mut rendered = String::new(); + rendered.push_str(&cls.name()); + rendered.push('('); + + for (idx, field) in fields.iter().enumerate() { + let value = obj.get_attr(field, vm)?; + let value_repr = if value.fast_isinstance(vm.ctx.types.list_type) { + let list = value + .downcast::<PyList>() + .expect("list type should downcast"); + repr_ast_list(vm, list.borrow_vec().to_vec(), depth)? + } else if value.fast_isinstance(vm.ctx.types.tuple_type) { + let tuple = value + .downcast::<PyTuple>() + .expect("tuple type should downcast"); + repr_ast_tuple(vm, tuple.as_slice().to_vec(), depth)? + } else if value.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { + repr_ast_node(vm, &value, depth.saturating_sub(1))? + } else { + value.repr(vm)?.to_string() + }; + + if idx > 0 { + rendered.push_str(", "); + } + rendered.push_str(field.as_str()); + rendered.push('='); + rendered.push_str(&value_repr); + } + + rendered.push(')'); + Ok(rendered) +} diff --git a/crates/vm/src/stdlib/ast/statement.rs b/crates/vm/src/stdlib/ast/statement.rs index 5925ca1fc2a..8b6ceb490a1 100644 --- a/crates/vm/src/stdlib/ast/statement.rs +++ b/crates/vm/src/stdlib/ast/statement.rs @@ -3,7 +3,7 @@ use crate::stdlib::ast::argument::{merge_class_def_args, split_class_def_args}; use rustpython_compiler_core::SourceFile; // sum -impl Node for ruff::Stmt { +impl Node for ast::Stmt { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self { Self::FunctionDef(cons) => cons.ast_to_object(vm, source_file), @@ -44,117 +44,93 @@ impl Node for ruff::Stmt { ) -> PyResult<Self> { let _cls = _object.class(); Ok(if _cls.is(pyast::NodeStmtFunctionDef::static_type()) { - Self::FunctionDef(ruff::StmtFunctionDef::ast_from_object( + Self::FunctionDef(ast::StmtFunctionDef::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()) { - Self::FunctionDef(ruff::StmtFunctionDef::ast_from_object( + Self::FunctionDef(ast::StmtFunctionDef::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtClassDef::static_type()) { - Self::ClassDef(ruff::StmtClassDef::ast_from_object( + Self::ClassDef(ast::StmtClassDef::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtReturn::static_type()) { - Self::Return(ruff::StmtReturn::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Return(ast::StmtReturn::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtDelete::static_type()) { - Self::Delete(ruff::StmtDelete::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Delete(ast::StmtDelete::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtAssign::static_type()) { - Self::Assign(ruff::StmtAssign::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Assign(ast::StmtAssign::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtTypeAlias::static_type()) { - Self::TypeAlias(ruff::StmtTypeAlias::ast_from_object( + Self::TypeAlias(ast::StmtTypeAlias::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtAugAssign::static_type()) { - Self::AugAssign(ruff::StmtAugAssign::ast_from_object( + Self::AugAssign(ast::StmtAugAssign::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtAnnAssign::static_type()) { - Self::AnnAssign(ruff::StmtAnnAssign::ast_from_object( + Self::AnnAssign(ast::StmtAnnAssign::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtFor::static_type()) { - Self::For(ruff::StmtFor::ast_from_object(_vm, source_file, _object)?) + Self::For(ast::StmtFor::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtAsyncFor::static_type()) { - Self::For(ruff::StmtFor::ast_from_object(_vm, source_file, _object)?) + Self::For(ast::StmtFor::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtWhile::static_type()) { - Self::While(ruff::StmtWhile::ast_from_object(_vm, source_file, _object)?) + Self::While(ast::StmtWhile::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtIf::static_type()) { - Self::If(ruff::StmtIf::ast_from_object(_vm, source_file, _object)?) + Self::If(ast::StmtIf::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtWith::static_type()) { - Self::With(ruff::StmtWith::ast_from_object(_vm, source_file, _object)?) + Self::With(ast::StmtWith::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtAsyncWith::static_type()) { - Self::With(ruff::StmtWith::ast_from_object(_vm, source_file, _object)?) + Self::With(ast::StmtWith::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtMatch::static_type()) { - Self::Match(ruff::StmtMatch::ast_from_object(_vm, source_file, _object)?) + Self::Match(ast::StmtMatch::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtRaise::static_type()) { - Self::Raise(ruff::StmtRaise::ast_from_object(_vm, source_file, _object)?) + Self::Raise(ast::StmtRaise::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtTry::static_type()) { - Self::Try(ruff::StmtTry::ast_from_object(_vm, source_file, _object)?) + Self::Try(ast::StmtTry::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtTryStar::static_type()) { - Self::Try(ruff::StmtTry::ast_from_object(_vm, source_file, _object)?) + Self::Try(ast::StmtTry::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtAssert::static_type()) { - Self::Assert(ruff::StmtAssert::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Assert(ast::StmtAssert::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtImport::static_type()) { - Self::Import(ruff::StmtImport::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Import(ast::StmtImport::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtImportFrom::static_type()) { - Self::ImportFrom(ruff::StmtImportFrom::ast_from_object( + Self::ImportFrom(ast::StmtImportFrom::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtGlobal::static_type()) { - Self::Global(ruff::StmtGlobal::ast_from_object( - _vm, - source_file, - _object, - )?) + Self::Global(ast::StmtGlobal::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtNonlocal::static_type()) { - Self::Nonlocal(ruff::StmtNonlocal::ast_from_object( + Self::Nonlocal(ast::StmtNonlocal::ast_from_object( _vm, source_file, _object, )?) } else if _cls.is(pyast::NodeStmtExpr::static_type()) { - Self::Expr(ruff::StmtExpr::ast_from_object(_vm, source_file, _object)?) + Self::Expr(ast::StmtExpr::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtPass::static_type()) { - Self::Pass(ruff::StmtPass::ast_from_object(_vm, source_file, _object)?) + Self::Pass(ast::StmtPass::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtBreak::static_type()) { - Self::Break(ruff::StmtBreak::ast_from_object(_vm, source_file, _object)?) + Self::Break(ast::StmtBreak::ast_from_object(_vm, source_file, _object)?) } else if _cls.is(pyast::NodeStmtContinue::static_type()) { - Self::Continue(ruff::StmtContinue::ast_from_object( + Self::Continue(ast::StmtContinue::ast_from_object( _vm, source_file, _object, @@ -169,7 +145,7 @@ impl Node for ruff::Stmt { } // constructor -impl Node for ruff::StmtFunctionDef { +impl Node for ast::StmtFunctionDef { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -183,6 +159,9 @@ impl Node for ruff::StmtFunctionDef { is_async, range: _range, } = self; + let source_code = source_file.to_source_code(); + let def_line = source_code.line_index(name.range.start()); + let range = TextRange::new(source_code.line_start(def_line), _range.end()); let cls = if !is_async { pyast::NodeStmtFunctionDef::static_type().to_owned() @@ -206,16 +185,17 @@ impl Node for ruff::StmtFunctionDef { .unwrap(); dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) .unwrap(); - // TODO: Ruff ignores type_comment during parsing - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", vm.ctx.none(), vm).unwrap(); dict.set_item( "type_params", - type_params.ast_to_object(vm, source_file), + type_params + .map(|tp| tp.ast_to_object(vm, source_file)) + .unwrap_or_else(|| vm.ctx.new_list(vec![]).into()), vm, ) .unwrap(); - node_add_location(&dict, _range, vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( @@ -225,6 +205,7 @@ impl Node for ruff::StmtFunctionDef { ) -> PyResult<Self> { let _cls = _object.class(); let is_async = _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()); + let range = range_from_object(_vm, source_file, _object.clone(), "FunctionDef")?; Ok(Self { node_index: Default::default(), name: Node::ast_from_object( @@ -257,16 +238,17 @@ impl Node for ruff::StmtFunctionDef { type_params: Node::ast_from_object( _vm, source_file, - get_node_field(_vm, &_object, "type_params", "FunctionDef")?, + get_node_field_opt(_vm, &_object, "type_params")? + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), )?, - range: range_from_object(_vm, source_file, _object, "FunctionDef")?, + range, is_async, }) } } // constructor -impl Node for ruff::StmtClassDef { +impl Node for ast::StmtClassDef { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -278,16 +260,31 @@ impl Node for ruff::StmtClassDef { range: _range, } = self; let (bases, keywords) = split_class_def_args(arguments); + let source_code = source_file.to_source_code(); + let class_line = source_code.line_index(name.range.start()); + let range = TextRange::new(source_code.line_start(class_line), _range.end()); let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtClassDef::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) .unwrap(); - dict.set_item("bases", bases.ast_to_object(_vm, source_file), _vm) - .unwrap(); - dict.set_item("keywords", keywords.ast_to_object(_vm, source_file), _vm) - .unwrap(); + dict.set_item( + "bases", + bases + .map(|b| b.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), + _vm, + ) + .unwrap(); + dict.set_item( + "keywords", + keywords + .map(|k| k.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), + _vm, + ) + .unwrap(); dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) .unwrap(); dict.set_item( @@ -298,11 +295,13 @@ impl Node for ruff::StmtClassDef { .unwrap(); dict.set_item( "type_params", - type_params.ast_to_object(_vm, source_file), + type_params + .map(|tp| tp.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), _vm, ) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, _vm, source_file); node.into() } fn ast_from_object( @@ -341,14 +340,15 @@ impl Node for ruff::StmtClassDef { type_params: Node::ast_from_object( _vm, source_file, - get_node_field(_vm, &_object, "type_params", "ClassDef")?, + get_node_field_opt(_vm, &_object, "type_params")? + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), )?, range: range_from_object(_vm, source_file, _object, "ClassDef")?, }) } } // constructor -impl Node for ruff::StmtReturn { +impl Node for ast::StmtReturn { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -379,7 +379,7 @@ impl Node for ruff::StmtReturn { } } // constructor -impl Node for ruff::StmtDelete { +impl Node for ast::StmtDelete { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -413,7 +413,7 @@ impl Node for ruff::StmtDelete { } // constructor -impl Node for ruff::StmtAssign { +impl Node for ast::StmtAssign { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -461,7 +461,7 @@ impl Node for ruff::StmtAssign { } // constructor -impl Node for ruff::StmtTypeAlias { +impl Node for ast::StmtTypeAlias { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -478,7 +478,9 @@ impl Node for ruff::StmtTypeAlias { .unwrap(); dict.set_item( "type_params", - type_params.ast_to_object(_vm, source_file), + type_params + .map(|tp| tp.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), _vm, ) .unwrap(); @@ -503,7 +505,7 @@ impl Node for ruff::StmtTypeAlias { type_params: Node::ast_from_object( _vm, source_file, - get_node_field(_vm, &_object, "type_params", "TypeAlias")?, + get_node_field_opt(_vm, &_object, "type_params")?.unwrap_or_else(|| _vm.ctx.none()), )?, value: Node::ast_from_object( _vm, @@ -516,7 +518,7 @@ impl Node for ruff::StmtTypeAlias { } // constructor -impl Node for ruff::StmtAugAssign { +impl Node for ast::StmtAugAssign { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -566,7 +568,7 @@ impl Node for ruff::StmtAugAssign { } // constructor -impl Node for ruff::StmtAnnAssign { +impl Node for ast::StmtAnnAssign { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -626,7 +628,7 @@ impl Node for ruff::StmtAnnAssign { } // constructor -impl Node for ruff::StmtFor { +impl Node for ast::StmtFor { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -655,8 +657,8 @@ impl Node for ruff::StmtFor { .unwrap(); dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } @@ -704,7 +706,7 @@ impl Node for ruff::StmtFor { } // constructor -impl Node for ruff::StmtWhile { +impl Node for ast::StmtWhile { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -754,7 +756,7 @@ impl Node for ruff::StmtWhile { } } // constructor -impl Node for ruff::StmtIf { +impl Node for ast::StmtIf { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -764,7 +766,7 @@ impl Node for ruff::StmtIf { elif_else_clauses, } = self; elif_else_clause::ast_to_object( - ruff::ElifElseClause { + ast::ElifElseClause { node_index: Default::default(), range, test: Some(*test), @@ -784,7 +786,7 @@ impl Node for ruff::StmtIf { } } // constructor -impl Node for ruff::StmtWith { +impl Node for ast::StmtWith { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -807,8 +809,8 @@ impl Node for ruff::StmtWith { .unwrap(); dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } @@ -844,7 +846,7 @@ impl Node for ruff::StmtWith { } } // constructor -impl Node for ruff::StmtMatch { +impl Node for ast::StmtMatch { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -885,7 +887,7 @@ impl Node for ruff::StmtMatch { } } // constructor -impl Node for ruff::StmtRaise { +impl Node for ast::StmtRaise { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -922,7 +924,7 @@ impl Node for ruff::StmtRaise { } } // constructor -impl Node for ruff::StmtTry { +impl Node for ast::StmtTry { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -996,7 +998,7 @@ impl Node for ruff::StmtTry { } } // constructor -impl Node for ruff::StmtAssert { +impl Node for ast::StmtAssert { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1035,7 +1037,7 @@ impl Node for ruff::StmtAssert { } } // constructor -impl Node for ruff::StmtImport { +impl Node for ast::StmtImport { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1068,7 +1070,7 @@ impl Node for ruff::StmtImport { } } // constructor -impl Node for ruff::StmtImportFrom { +impl Node for ast::StmtImportFrom { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1105,16 +1107,25 @@ impl Node for ruff::StmtImportFrom { source_file, get_node_field(vm, &_object, "names", "ImportFrom")?, )?, - level: get_node_field(vm, &_object, "level", "ImportFrom")? - .downcast_exact::<PyInt>(vm) - .unwrap() - .try_to_primitive::<u32>(vm)?, + level: get_node_field_opt(vm, &_object, "level")? + .map(|obj| -> PyResult<u32> { + let int: PyRef<PyInt> = obj.try_into_value(vm)?; + let value: i64 = int.try_to_primitive(vm)?; + if value < 0 { + return Err(vm.new_value_error("Negative ImportFrom level".to_owned())); + } + u32::try_from(value).map_err(|_| { + vm.new_overflow_error("ImportFrom level out of range".to_owned()) + }) + }) + .transpose()? + .unwrap_or(0), range: range_from_object(vm, source_file, _object, "ImportFrom")?, }) } } // constructor -impl Node for ruff::StmtGlobal { +impl Node for ast::StmtGlobal { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1147,7 +1158,7 @@ impl Node for ruff::StmtGlobal { } } // constructor -impl Node for ruff::StmtNonlocal { +impl Node for ast::StmtNonlocal { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1180,7 +1191,7 @@ impl Node for ruff::StmtNonlocal { } } // constructor -impl Node for ruff::StmtExpr { +impl Node for ast::StmtExpr { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1213,7 +1224,7 @@ impl Node for ruff::StmtExpr { } } // constructor -impl Node for ruff::StmtPass { +impl Node for ast::StmtPass { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1223,7 +1234,28 @@ impl Node for ruff::StmtPass { .into_ref_with_type(_vm, pyast::NodeStmtPass::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm, source_file); + let location = super::text_range_to_source_range(source_file, _range); + let start_row = location.start.row.get(); + let start_col = location.start.column.get(); + let mut end_row = location.end.row.get(); + let mut end_col = location.end.column.get(); + + // Align with CPython: when docstring optimization replaces a lone + // docstring with `pass`, the end position is on the same line even if + // it extends past the physical line length. + if end_row != start_row && _range.len() == TextSize::from(4) { + end_row = start_row; + end_col = start_col + 4; + } + + dict.set_item("lineno", _vm.ctx.new_int(start_row).into(), _vm) + .unwrap(); + dict.set_item("col_offset", _vm.ctx.new_int(start_col).into(), _vm) + .unwrap(); + dict.set_item("end_lineno", _vm.ctx.new_int(end_row).into(), _vm) + .unwrap(); + dict.set_item("end_col_offset", _vm.ctx.new_int(end_col).into(), _vm) + .unwrap(); node.into() } fn ast_from_object( @@ -1238,7 +1270,7 @@ impl Node for ruff::StmtPass { } } // constructor -impl Node for ruff::StmtBreak { +impl Node for ast::StmtBreak { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, @@ -1265,7 +1297,7 @@ impl Node for ruff::StmtBreak { } // constructor -impl Node for ruff::StmtContinue { +impl Node for ast::StmtContinue { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, diff --git a/crates/vm/src/stdlib/ast/string.rs b/crates/vm/src/stdlib/ast/string.rs index f3df8d99262..4b6a6e8489f 100644 --- a/crates/vm/src/stdlib/ast/string.rs +++ b/crates/vm/src/stdlib/ast/string.rs @@ -1,76 +1,37 @@ use super::constant::{Constant, ConstantLiteral}; use super::*; +use crate::warn; +use ast::str_prefix::StringLiteralPrefix; -fn ruff_fstring_value_into_iter( - mut fstring_value: ruff::FStringValue, -) -> impl Iterator<Item = ruff::FStringPart> + 'static { - let default = ruff::FStringPart::FString(ruff::FString { +fn ruff_fstring_element_into_iter( + mut fstring_element: ast::InterpolatedStringElements, +) -> impl Iterator<Item = ast::InterpolatedStringElement> { + let default = ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { node_index: Default::default(), range: Default::default(), - elements: Default::default(), - flags: ruff::FStringFlags::empty(), + value: Default::default(), }); - (0..fstring_value.as_slice().len()).map(move |i| { - let tmp = fstring_value.iter_mut().nth(i).unwrap(); - std::mem::replace(tmp, default.clone()) - }) -} - -fn ruff_fstring_element_into_iter( - mut fstring_element: ruff::InterpolatedStringElements, -) -> impl Iterator<Item = ruff::InterpolatedStringElement> + 'static { - let default = - ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { - node_index: Default::default(), - range: Default::default(), - value: Default::default(), - }); - (0..fstring_element.into_iter().len()).map(move |i| { - let fstring_element = &mut fstring_element; - let tmp = fstring_element.into_iter().nth(i).unwrap(); - std::mem::replace(tmp, default.clone()) - }) -} - -fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec<JoinedStrPart> { - match fstring_part { - ruff::FStringPart::Literal(ruff::StringLiteral { - range, - value, - flags, - node_index: _, - }) => { - vec![JoinedStrPart::Constant(Constant::new_str( - value, - flags.prefix(), - range, - ))] - } - ruff::FStringPart::FString(ruff::FString { - range: _, - elements, - flags: _, // TODO - node_index: _, - }) => ruff_fstring_element_into_iter(elements) - .map(ruff_fstring_element_to_joined_str_part) - .collect(), - } + fstring_element + .iter_mut() + .map(move |elem| core::mem::replace(elem, default.clone())) + .collect::<Vec<_>>() + .into_iter() } fn ruff_fstring_element_to_joined_str_part( - element: ruff::InterpolatedStringElement, + element: ast::InterpolatedStringElement, ) -> JoinedStrPart { match element { - ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { range, value, node_index: _, }) => JoinedStrPart::Constant(Constant::new_str( value, - ruff::str_prefix::StringLiteralPrefix::Empty, + ast::str_prefix::StringLiteralPrefix::Empty, range, )), - ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { range, expression, debug_text: _, // TODO: What is this? @@ -86,57 +47,252 @@ fn ruff_fstring_element_to_joined_str_part( } } +fn push_joined_str_literal( + output: &mut Vec<JoinedStrPart>, + pending: &mut Option<(String, StringLiteralPrefix, TextRange)>, +) { + if let Some((value, prefix, range)) = pending.take() + && !value.is_empty() + { + output.push(JoinedStrPart::Constant(Constant::new_str( + value, prefix, range, + ))); + } +} + +fn normalize_joined_str_parts(values: Vec<JoinedStrPart>) -> Vec<JoinedStrPart> { + let mut output = Vec::with_capacity(values.len()); + let mut pending: Option<(String, StringLiteralPrefix, TextRange)> = None; + + for part in values { + match part { + JoinedStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, prefix } = constant.value else { + push_joined_str_literal(&mut output, &mut pending); + output.push(JoinedStrPart::Constant(constant)); + continue; + }; + let value: String = value.into(); + if let Some((pending_value, _, _)) = pending.as_mut() { + pending_value.push_str(&value); + } else { + pending = Some((value, prefix, constant.range)); + } + } + JoinedStrPart::FormattedValue(value) => { + push_joined_str_literal(&mut output, &mut pending); + output.push(JoinedStrPart::FormattedValue(value)); + } + } + } + + push_joined_str_literal(&mut output, &mut pending); + output +} + +fn push_template_str_literal( + output: &mut Vec<TemplateStrPart>, + pending: &mut Option<(String, StringLiteralPrefix, TextRange)>, +) { + if let Some((value, prefix, range)) = pending.take() + && !value.is_empty() + { + output.push(TemplateStrPart::Constant(Constant::new_str( + value, prefix, range, + ))); + } +} + +fn normalize_template_str_parts(values: Vec<TemplateStrPart>) -> Vec<TemplateStrPart> { + let mut output = Vec::with_capacity(values.len()); + let mut pending: Option<(String, StringLiteralPrefix, TextRange)> = None; + + for part in values { + match part { + TemplateStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, prefix } = constant.value else { + push_template_str_literal(&mut output, &mut pending); + output.push(TemplateStrPart::Constant(constant)); + continue; + }; + let value: String = value.into(); + if let Some((pending_value, _, _)) = pending.as_mut() { + pending_value.push_str(&value); + } else { + pending = Some((value, prefix, constant.range)); + } + } + TemplateStrPart::Interpolation(value) => { + push_template_str_literal(&mut output, &mut pending); + output.push(TemplateStrPart::Interpolation(value)); + } + } + } + + push_template_str_literal(&mut output, &mut pending); + output +} + +fn warn_invalid_escape_sequences_in_format_spec( + vm: &VirtualMachine, + source_file: &SourceFile, + range: TextRange, +) { + let source = source_file.source_text(); + let start = range.start().to_usize(); + let end = range.end().to_usize(); + if start >= end || end > source.len() { + return; + } + let mut raw = &source[start..end]; + if raw.starts_with(':') { + raw = &raw[1..]; + } + + let mut chars = raw.chars().peekable(); + while let Some(ch) = chars.next() { + if ch != '\\' { + continue; + } + let Some(next) = chars.next() else { + break; + }; + let valid = match next { + '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => true, + '\n' => true, + '\r' => { + if let Some('\n') = chars.peek().copied() { + chars.next(); + } + true + } + '0'..='7' => { + for _ in 0..2 { + if let Some('0'..='7') = chars.peek().copied() { + chars.next(); + } else { + break; + } + } + true + } + 'x' => { + for _ in 0..2 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'u' => { + for _ in 0..4 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'U' => { + for _ in 0..8 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'N' => { + if let Some('{') = chars.peek().copied() { + chars.next(); + for c in chars.by_ref() { + if c == '}' { + break; + } + } + } + true + } + _ => false, + }; + if !valid { + let message = vm.ctx.new_str(format!( + "\"\\{next}\" is an invalid escape sequence. Such sequences will not work in the future. Did you mean \"\\\\{next}\"? A raw string is also an option." + )); + let _ = warn::warn( + message.into(), + Some(vm.ctx.exceptions.syntax_warning.to_owned()), + 1, + None, + vm, + ); + } + } +} + fn ruff_format_spec_to_joined_str( - format_spec: Option<Box<ruff::InterpolatedStringFormatSpec>>, + format_spec: Option<Box<ast::InterpolatedStringFormatSpec>>, ) -> Option<Box<JoinedStr>> { match format_spec { None => None, Some(format_spec) => { - let ruff::InterpolatedStringFormatSpec { + let ast::InterpolatedStringFormatSpec { range, elements, node_index: _, } = *format_spec; + let range = if range.start() > ruff_text_size::TextSize::from(0) { + TextRange::new( + range.start() - ruff_text_size::TextSize::from(1), + range.end(), + ) + } else { + range + }; let values: Vec<_> = ruff_fstring_element_into_iter(elements) .map(ruff_fstring_element_to_joined_str_part) .collect(); - let values = values.into_boxed_slice(); + let values = normalize_joined_str_parts(values).into_boxed_slice(); Some(Box::new(JoinedStr { range, values })) } } } fn ruff_fstring_element_to_ruff_fstring_part( - element: ruff::InterpolatedStringElement, -) -> ruff::FStringPart { + element: ast::InterpolatedStringElement, +) -> ast::FStringPart { match element { - ruff::InterpolatedStringElement::Literal(value) => { - let ruff::InterpolatedStringLiteralElement { + ast::InterpolatedStringElement::Literal(value) => { + let ast::InterpolatedStringLiteralElement { node_index, range, value, } = value; - ruff::FStringPart::Literal(ruff::StringLiteral { + ast::FStringPart::Literal(ast::StringLiteral { node_index, range, value, - flags: ruff::StringLiteralFlags::empty(), + flags: ast::StringLiteralFlags::empty(), }) } - ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { range, .. - }) => ruff::FStringPart::FString(ruff::FString { + }) => ast::FStringPart::FString(ast::FString { node_index: Default::default(), range, elements: vec![element].into(), - flags: ruff::FStringFlags::empty(), + flags: ast::FStringFlags::empty(), }), } } fn joined_str_to_ruff_format_spec( joined_str: Option<Box<JoinedStr>>, -) -> Option<Box<ruff::InterpolatedStringFormatSpec>> { +) -> Option<Box<ast::InterpolatedStringFormatSpec>> { match joined_str { None => None, Some(joined_str) => { @@ -144,7 +300,7 @@ fn joined_str_to_ruff_format_spec( let elements: Vec<_> = Box::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) .collect(); - let format_spec = ruff::InterpolatedStringFormatSpec { + let format_spec = ast::InterpolatedStringFormatSpec { node_index: Default::default(), range, elements: elements.into(), @@ -161,32 +317,32 @@ pub(super) struct JoinedStr { } impl JoinedStr { - pub(super) fn into_expr(self) -> ruff::Expr { + pub(super) fn into_expr(self) -> ast::Expr { let Self { range, values } = self; - ruff::Expr::FString(ruff::ExprFString { + ast::Expr::FString(ast::ExprFString { node_index: Default::default(), range: Default::default(), value: match values.len() { // ruff represents an empty fstring like this: - 0 => ruff::FStringValue::single(ruff::FString { + 0 => ast::FStringValue::single(ast::FString { node_index: Default::default(), range, elements: vec![].into(), - flags: ruff::FStringFlags::empty(), + flags: ast::FStringFlags::empty(), }), - 1 => ruff::FStringValue::single( + 1 => ast::FStringValue::single( Box::<[_]>::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) - .map(|element| ruff::FString { + .map(|element| ast::FString { node_index: Default::default(), range, elements: vec![element].into(), - flags: ruff::FStringFlags::empty(), + flags: ast::FStringFlags::empty(), }) .next() .expect("FString has exactly one part"), ), - _ => ruff::FStringValue::concatenated( + _ => ast::FStringValue::concatenated( Box::<[_]>::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) .map(ruff_fstring_element_to_ruff_fstring_part) @@ -197,10 +353,10 @@ impl JoinedStr { } } -fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::InterpolatedStringElement { +fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ast::InterpolatedStringElement { match part { JoinedStrPart::FormattedValue(value) => { - ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { node_index: Default::default(), range: value.range, expression: value.value.clone(), @@ -210,7 +366,7 @@ fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::Interpo }) } JoinedStrPart::Constant(value) => { - ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { node_index: Default::default(), range: value.range, value: match value.value { @@ -294,8 +450,8 @@ impl Node for JoinedStrPart { #[derive(Debug)] pub(super) struct FormattedValue { - value: Box<ruff::Expr>, - conversion: ruff::ConversionFlag, + value: Box<ast::Expr>, + conversion: ast::ConversionFlag, format_spec: Option<Box<JoinedStr>>, range: TextRange, } @@ -353,17 +509,451 @@ impl Node for FormattedValue { pub(super) fn fstring_to_object( vm: &VirtualMachine, source_file: &SourceFile, - expression: ruff::ExprFString, + expression: ast::ExprFString, +) -> PyObjectRef { + let ast::ExprFString { + range, + mut value, + node_index: _, + } = expression; + let default_part = ast::FStringPart::FString(ast::FString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::FStringFlags::empty(), + }); + let mut values = Vec::new(); + for i in 0..value.as_slice().len() { + let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); + match part { + ast::FStringPart::Literal(ast::StringLiteral { + range, + value, + flags, + node_index: _, + }) => { + values.push(JoinedStrPart::Constant(Constant::new_str( + value, + flags.prefix(), + range, + ))); + } + ast::FStringPart::FString(ast::FString { + range: _, + elements, + flags: _, + node_index: _, + }) => { + for element in ruff_fstring_element_into_iter(elements) { + values.push(ruff_fstring_element_to_joined_str_part(element)); + } + } + } + } + let values = normalize_joined_str_parts(values); + for part in &values { + if let JoinedStrPart::FormattedValue(value) = part + && let Some(format_spec) = &value.format_spec + { + warn_invalid_escape_sequences_in_format_spec(vm, source_file, format_spec.range); + } + } + let c = JoinedStr { + range, + values: values.into_boxed_slice(), + }; + c.ast_to_object(vm, source_file) +} + +// ===== TString (Template String) Support ===== + +fn ruff_tstring_element_to_template_str_part( + element: ast::InterpolatedStringElement, + source_file: &SourceFile, +) -> TemplateStrPart { + match element { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + }) => TemplateStrPart::Constant(Constant::new_str( + value, + ast::str_prefix::StringLiteralPrefix::Empty, + range, + )), + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + range, + expression, + debug_text, + conversion, + format_spec, + node_index: _, + }) => { + let expr_range = + extend_expr_range_with_wrapping_parens(source_file, range, expression.range()) + .unwrap_or_else(|| expression.range()); + let expr_str = if let Some(debug_text) = debug_text { + let expr_source = source_file.slice(expr_range); + let mut expr_with_debug = String::with_capacity( + debug_text.leading.len() + expr_source.len() + debug_text.trailing.len(), + ); + expr_with_debug.push_str(&debug_text.leading); + expr_with_debug.push_str(expr_source); + expr_with_debug.push_str(&debug_text.trailing); + strip_interpolation_expr(&expr_with_debug) + } else { + tstring_interpolation_expr_str(source_file, range, expr_range) + }; + TemplateStrPart::Interpolation(TStringInterpolation { + value: expression, + str: expr_str, + conversion, + format_spec: ruff_format_spec_to_joined_str(format_spec), + range, + }) + } + } +} + +fn tstring_interpolation_expr_str( + source_file: &SourceFile, + interpolation_range: TextRange, + expr_range: TextRange, +) -> String { + let expr_range = + extend_expr_range_with_wrapping_parens(source_file, interpolation_range, expr_range) + .unwrap_or(expr_range); + let start = interpolation_range.start() + TextSize::from(1); + let start = if start > expr_range.end() { + expr_range.start() + } else { + start + }; + let expr_source = source_file.slice(TextRange::new(start, expr_range.end())); + strip_interpolation_expr(expr_source) +} + +fn extend_expr_range_with_wrapping_parens( + source_file: &SourceFile, + interpolation_range: TextRange, + expr_range: TextRange, +) -> Option<TextRange> { + let left_slice = source_file.slice(TextRange::new( + interpolation_range.start(), + expr_range.start(), + )); + let mut left_char: Option<(usize, char)> = None; + for (idx, ch) in left_slice + .char_indices() + .collect::<Vec<_>>() + .into_iter() + .rev() + { + if !ch.is_whitespace() { + left_char = Some((idx, ch)); + break; + } + } + let (left_idx, left_ch) = left_char?; + if left_ch != '(' { + return None; + } + + let right_slice = + source_file.slice(TextRange::new(expr_range.end(), interpolation_range.end())); + let mut right_char: Option<(usize, char)> = None; + for (idx, ch) in right_slice.char_indices() { + if !ch.is_whitespace() { + right_char = Some((idx, ch)); + break; + } + } + let (right_idx, right_ch) = right_char?; + if right_ch != ')' { + return None; + } + + let left_pos = interpolation_range.start() + TextSize::from(left_idx as u32); + let right_pos = expr_range.end() + TextSize::from(right_idx as u32); + Some(TextRange::new(left_pos, right_pos + TextSize::from(1))) +} + +fn strip_interpolation_expr(expr_source: &str) -> String { + let mut end = expr_source.len(); + for (idx, ch) in expr_source.char_indices().rev() { + if ch.is_whitespace() || ch == '=' { + end = idx; + continue; + } + end = idx + ch.len_utf8(); + break; + } + expr_source[..end].to_owned() +} + +#[derive(Debug)] +pub(super) struct TemplateStr { + pub(super) range: TextRange, + pub(super) values: Box<[TemplateStrPart]>, +} + +pub(super) fn template_str_to_expr( + vm: &VirtualMachine, + template: TemplateStr, +) -> PyResult<ast::Expr> { + let TemplateStr { range, values } = template; + let elements = template_parts_to_elements(vm, values)?; + let tstring = ast::TString { + range, + node_index: Default::default(), + elements, + flags: ast::TStringFlags::empty(), + }; + Ok(ast::Expr::TString(ast::ExprTString { + node_index: Default::default(), + range, + value: ast::TStringValue::single(tstring), + })) +} + +pub(super) fn interpolation_to_expr( + vm: &VirtualMachine, + interpolation: TStringInterpolation, +) -> PyResult<ast::Expr> { + let part = TemplateStrPart::Interpolation(interpolation); + let elements = template_parts_to_elements(vm, vec![part].into_boxed_slice())?; + let range = TextRange::default(); + let tstring = ast::TString { + range, + node_index: Default::default(), + elements, + flags: ast::TStringFlags::empty(), + }; + Ok(ast::Expr::TString(ast::ExprTString { + node_index: Default::default(), + range, + value: ast::TStringValue::single(tstring), + })) +} + +fn template_parts_to_elements( + vm: &VirtualMachine, + values: Box<[TemplateStrPart]>, +) -> PyResult<ast::InterpolatedStringElements> { + let mut elements = Vec::with_capacity(values.len()); + for value in values.into_vec() { + elements.push(template_part_to_element(vm, value)?); + } + Ok(ast::InterpolatedStringElements::from(elements)) +} + +fn template_part_to_element( + vm: &VirtualMachine, + part: TemplateStrPart, +) -> PyResult<ast::InterpolatedStringElement> { + match part { + TemplateStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, .. } = constant.value else { + return Err( + vm.new_type_error("TemplateStr constant values must be strings".to_owned()) + ); + }; + Ok(ast::InterpolatedStringElement::Literal( + ast::InterpolatedStringLiteralElement { + range: constant.range, + node_index: Default::default(), + value, + }, + )) + } + TemplateStrPart::Interpolation(interpolation) => { + let TStringInterpolation { + value, + conversion, + format_spec, + range, + .. + } = interpolation; + let format_spec = joined_str_to_ruff_format_spec(format_spec); + Ok(ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + range, + node_index: Default::default(), + expression: value, + debug_text: None, + conversion, + format_spec, + }, + )) + } + } +} + +// constructor +impl Node for TemplateStr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { values, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprTemplateStr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "values", + BoxedSlice(values).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let values: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "values", "TemplateStr")?, + )?; + Ok(Self { + values: values.0, + range: range_from_object(vm, source_file, object, "TemplateStr")?, + }) + } +} + +#[derive(Debug)] +pub(super) enum TemplateStrPart { + Interpolation(TStringInterpolation), + Constant(Constant), +} + +// constructor +impl Node for TemplateStrPart { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::Interpolation(value) => value.ast_to_object(vm, source_file), + Self::Constant(value) => value.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + if cls.is(pyast::NodeExprInterpolation::static_type()) { + Ok(Self::Interpolation(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } else { + Ok(Self::Constant(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } + } +} + +#[derive(Debug)] +pub(super) struct TStringInterpolation { + value: Box<ast::Expr>, + str: String, + conversion: ast::ConversionFlag, + format_spec: Option<Box<JoinedStr>>, + range: TextRange, +} + +// constructor +impl Node for TStringInterpolation { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + value, + str, + conversion, + format_spec, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprInterpolation::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("str", vm.ctx.new_str(str).into(), vm) + .unwrap(); + dict.set_item("conversion", conversion.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "format_spec", + format_spec.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let str_obj = get_node_field(vm, &object, "str", "Interpolation")?; + let str_val: String = str_obj.try_into_value(vm)?; + Ok(Self { + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Interpolation")?, + )?, + str: str_val, + conversion: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "conversion", "Interpolation")?, + )?, + format_spec: get_node_field_opt(vm, &object, "format_spec")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "Interpolation")?, + }) + } +} + +pub(super) fn tstring_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + expression: ast::ExprTString, ) -> PyObjectRef { - let ruff::ExprFString { + let ast::ExprTString { range, - value, + mut value, node_index: _, } = expression; - let values: Vec<_> = ruff_fstring_value_into_iter(value) - .flat_map(fstring_part_to_joined_str_part) - .collect(); - let values = values.into_boxed_slice(); - let c = JoinedStr { range, values }; + let default_tstring = ast::TString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::TStringFlags::empty(), + }; + let mut values = Vec::new(); + for i in 0..value.as_slice().len() { + let tstring = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_tstring.clone()); + for element in ruff_fstring_element_into_iter(tstring.elements) { + values.push(ruff_tstring_element_to_template_str_part( + element, + source_file, + )); + } + } + let values = normalize_template_str_parts(values); + let c = TemplateStr { + range, + values: values.into_boxed_slice(), + }; c.ast_to_object(vm, source_file) } diff --git a/crates/vm/src/stdlib/ast/type_ignore.rs b/crates/vm/src/stdlib/ast/type_ignore.rs index de929fcf623..6e90ba9b80e 100644 --- a/crates/vm/src/stdlib/ast/type_ignore.rs +++ b/crates/vm/src/stdlib/ast/type_ignore.rs @@ -13,21 +13,21 @@ impl Node for TypeIgnore { } } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeTypeIgnoreTypeIgnore::static_type()) { + let cls = object.class(); + Ok(if cls.is(pyast::NodeTypeIgnoreTypeIgnore::static_type()) { Self::TypeIgnore(TypeIgnoreTypeIgnore::ast_from_object( - _vm, + vm, source_file, - _object, + object, )?) } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of type_ignore, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } diff --git a/crates/vm/src/stdlib/ast/type_parameters.rs b/crates/vm/src/stdlib/ast/type_parameters.rs index 017470f7e64..0424ffbd768 100644 --- a/crates/vm/src/stdlib/ast/type_parameters.rs +++ b/crates/vm/src/stdlib/ast/type_parameters.rs @@ -1,17 +1,17 @@ use super::*; use rustpython_compiler_core::SourceFile; -impl Node for ruff::TypeParams { +impl Node for ast::TypeParams { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { self.type_params.ast_to_object(vm, source_file) } fn ast_from_object( - _vm: &VirtualMachine, - _source_file: &SourceFile, - _object: PyObjectRef, + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, ) -> PyResult<Self> { - let type_params: Vec<ruff::TypeParam> = Node::ast_from_object(_vm, _source_file, _object)?; + let type_params: Vec<ast::TypeParam> = Node::ast_from_object(vm, source_file, object)?; let range = Option::zip(type_params.first(), type_params.last()) .map(|(first, last)| first.range().cover(last.range())) .unwrap_or_default(); @@ -28,7 +28,7 @@ impl Node for ruff::TypeParams { } // sum -impl Node for ruff::TypeParam { +impl Node for ast::TypeParam { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self { Self::TypeVar(cons) => cons.ast_to_object(vm, source_file), @@ -38,178 +38,172 @@ impl Node for ruff::TypeParam { } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(pyast::NodeTypeParamTypeVar::static_type()) { - Self::TypeVar(ruff::TypeParamTypeVar::ast_from_object( - _vm, + let cls = object.class(); + Ok(if cls.is(pyast::NodeTypeParamTypeVar::static_type()) { + Self::TypeVar(ast::TypeParamTypeVar::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodeTypeParamParamSpec::static_type()) { - Self::ParamSpec(ruff::TypeParamParamSpec::ast_from_object( - _vm, + } else if cls.is(pyast::NodeTypeParamParamSpec::static_type()) { + Self::ParamSpec(ast::TypeParamParamSpec::ast_from_object( + vm, source_file, - _object, + object, )?) - } else if _cls.is(pyast::NodeTypeParamTypeVarTuple::static_type()) { - Self::TypeVarTuple(ruff::TypeParamTypeVarTuple::ast_from_object( - _vm, + } else if cls.is(pyast::NodeTypeParamTypeVarTuple::static_type()) { + Self::TypeVarTuple(ast::TypeParamTypeVarTuple::ast_from_object( + vm, source_file, - _object, + object, )?) } else { - return Err(_vm.new_type_error(format!( + return Err(vm.new_type_error(format!( "expected some sort of type_param, but got {}", - _object.repr(_vm)? + object.repr(vm)? ))); }) } } // constructor -impl Node for ruff::TypeParamTypeVar { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::TypeParamTypeVar { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, name, bound, - range: _range, - default: _, + range, + default, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("bound", bound.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("bound", bound.ast_to_object(_vm, source_file), _vm) + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), name: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "name", "TypeVar")?, + get_node_field(vm, &object, "name", "TypeVar")?, )?, - bound: get_node_field_opt(_vm, &_object, "bound")? - .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + bound: get_node_field_opt(vm, &object, "bound")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, default: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "default_value", "TypeVar")?, + get_node_field(vm, &object, "default_value", "TypeVar")?, )?, - range: range_from_object(_vm, source_file, _object, "TypeVar")?, + range: range_from_object(vm, source_file, object, "TypeVar")?, }) } } // constructor -impl Node for ruff::TypeParamParamSpec { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::TypeParamParamSpec { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, name, - range: _range, + range, default, } = self; let node = NodeAst - .into_ref_with_type(_vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item( - "default_value", - default.ast_to_object(_vm, source_file), - _vm, - ) - .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), name: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "name", "ParamSpec")?, + get_node_field(vm, &object, "name", "ParamSpec")?, )?, default: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "default_value", "ParamSpec")?, + get_node_field(vm, &object, "default_value", "ParamSpec")?, )?, - range: range_from_object(_vm, source_file, _object, "ParamSpec")?, + range: range_from_object(vm, source_file, object, "ParamSpec")?, }) } } // constructor -impl Node for ruff::TypeParamTypeVarTuple { - fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { +impl Node for ast::TypeParamTypeVarTuple { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, name, - range: _range, + range, default, } = self; let node = NodeAst .into_ref_with_type( - _vm, + vm, pyast::NodeTypeParamTypeVarTuple::static_type().to_owned(), ) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item( - "default_value", - default.ast_to_object(_vm, source_file), - _vm, - ) - .unwrap(); - node_add_location(&dict, _range, _vm, source_file); + node_add_location(&dict, range, vm, source_file); node.into() } fn ast_from_object( - _vm: &VirtualMachine, + vm: &VirtualMachine, source_file: &SourceFile, - _object: PyObjectRef, + object: PyObjectRef, ) -> PyResult<Self> { Ok(Self { node_index: Default::default(), name: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "name", "TypeVarTuple")?, + get_node_field(vm, &object, "name", "TypeVarTuple")?, )?, default: Node::ast_from_object( - _vm, + vm, source_file, - get_node_field(_vm, &_object, "default_value", "TypeVarTuple")?, + get_node_field(vm, &object, "default_value", "TypeVarTuple")?, )?, - range: range_from_object(_vm, source_file, _object, "TypeVarTuple")?, + range: range_from_object(vm, source_file, object, "TypeVarTuple")?, }) } } diff --git a/crates/vm/src/stdlib/ast/validate.rs b/crates/vm/src/stdlib/ast/validate.rs new file mode 100644 index 00000000000..ea5c2be840c --- /dev/null +++ b/crates/vm/src/stdlib/ast/validate.rs @@ -0,0 +1,670 @@ +// spell-checker: ignore assignlist ifexp + +use super::module::Mod; +use crate::{PyResult, VirtualMachine}; +use ruff_python_ast as ast; + +fn expr_context_name(ctx: ast::ExprContext) -> &'static str { + match ctx { + ast::ExprContext::Load => "Load", + ast::ExprContext::Store => "Store", + ast::ExprContext::Del => "Del", + ast::ExprContext::Invalid => "Invalid", + } +} + +fn validate_name(vm: &VirtualMachine, name: &ast::name::Name) -> PyResult<()> { + match name.as_str() { + "None" | "True" | "False" => Err(vm.new_value_error(format!( + "identifier field can't represent '{}' constant", + name.as_str() + ))), + _ => Ok(()), + } +} + +fn validate_comprehension(vm: &VirtualMachine, gens: &[ast::Comprehension]) -> PyResult<()> { + if gens.is_empty() { + return Err(vm.new_value_error("comprehension with no generators".to_owned())); + } + for comp in gens { + validate_expr(vm, &comp.target, ast::ExprContext::Store)?; + validate_expr(vm, &comp.iter, ast::ExprContext::Load)?; + validate_exprs(vm, &comp.ifs, ast::ExprContext::Load, false)?; + } + Ok(()) +} + +fn validate_keywords(vm: &VirtualMachine, keywords: &[ast::Keyword]) -> PyResult<()> { + for keyword in keywords { + validate_expr(vm, &keyword.value, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_parameters(vm: &VirtualMachine, params: &ast::Parameters) -> PyResult<()> { + for param in params + .posonlyargs + .iter() + .chain(&params.args) + .chain(&params.kwonlyargs) + { + if let Some(annotation) = &param.parameter.annotation { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + if let Some(default) = &param.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + if let Some(vararg) = &params.vararg + && let Some(annotation) = &vararg.annotation + { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + if let Some(kwarg) = &params.kwarg + && let Some(annotation) = &kwarg.annotation + { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_nonempty_seq( + vm: &VirtualMachine, + len: usize, + what: &'static str, + owner: &'static str, +) -> PyResult<()> { + if len == 0 { + return Err(vm.new_value_error(format!("empty {what} on {owner}"))); + } + Ok(()) +} + +fn validate_assignlist( + vm: &VirtualMachine, + targets: &[ast::Expr], + ctx: ast::ExprContext, +) -> PyResult<()> { + validate_nonempty_seq( + vm, + targets.len(), + "targets", + if ctx == ast::ExprContext::Del { + "Delete" + } else { + "Assign" + }, + )?; + validate_exprs(vm, targets, ctx, false) +} + +fn validate_body(vm: &VirtualMachine, body: &[ast::Stmt], owner: &'static str) -> PyResult<()> { + validate_nonempty_seq(vm, body.len(), "body", owner)?; + validate_stmts(vm, body) +} + +fn validate_interpolated_elements<'a>( + vm: &VirtualMachine, + elements: impl IntoIterator<Item = ast::InterpolatedStringElementRef<'a>>, +) -> PyResult<()> { + for element in elements { + if let ast::InterpolatedStringElementRef::Interpolation(interpolation) = element { + validate_expr(vm, &interpolation.expression, ast::ExprContext::Load)?; + if let Some(format_spec) = &interpolation.format_spec { + for spec_element in &format_spec.elements { + if let ast::InterpolatedStringElement::Interpolation(spec_interp) = spec_element + { + validate_expr(vm, &spec_interp.expression, ast::ExprContext::Load)?; + } + } + } + } + } + Ok(()) +} + +fn validate_pattern_match_value(vm: &VirtualMachine, expr: &ast::Expr) -> PyResult<()> { + validate_expr(vm, expr, ast::ExprContext::Load)?; + match expr { + ast::Expr::NumberLiteral(_) | ast::Expr::StringLiteral(_) | ast::Expr::BytesLiteral(_) => { + Ok(()) + } + ast::Expr::Attribute(_) => Ok(()), + ast::Expr::UnaryOp(op) => match &*op.operand { + ast::Expr::NumberLiteral(_) => Ok(()), + _ => Err(vm.new_value_error( + "patterns may only match literals and attribute lookups".to_owned(), + )), + }, + ast::Expr::BinOp(bin) => match (&*bin.left, &*bin.right) { + (ast::Expr::NumberLiteral(_), ast::Expr::NumberLiteral(_)) => Ok(()), + _ => Err(vm.new_value_error( + "patterns may only match literals and attribute lookups".to_owned(), + )), + }, + ast::Expr::FString(_) | ast::Expr::TString(_) => Ok(()), + ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::EllipsisLiteral(_) => { + Err(vm.new_value_error("unexpected constant inside of a literal pattern".to_owned())) + } + _ => Err( + vm.new_value_error("patterns may only match literals and attribute lookups".to_owned()) + ), + } +} + +fn validate_capture(vm: &VirtualMachine, name: &ast::Identifier) -> PyResult<()> { + if name.as_str() == "_" { + return Err(vm.new_value_error("can't capture name '_' in patterns".to_owned())); + } + validate_name(vm, name.id()) +} + +fn validate_pattern(vm: &VirtualMachine, pattern: &ast::Pattern, star_ok: bool) -> PyResult<()> { + match pattern { + ast::Pattern::MatchValue(value) => validate_pattern_match_value(vm, &value.value), + ast::Pattern::MatchSingleton(singleton) => match singleton.value { + ast::Singleton::None | ast::Singleton::True | ast::Singleton::False => Ok(()), + }, + ast::Pattern::MatchSequence(seq) => validate_patterns(vm, &seq.patterns, true), + ast::Pattern::MatchMapping(mapping) => { + if mapping.keys.len() != mapping.patterns.len() { + return Err(vm.new_value_error( + "MatchMapping doesn't have the same number of keys as patterns".to_owned(), + )); + } + if let Some(rest) = &mapping.rest { + validate_capture(vm, rest)?; + } + for key in &mapping.keys { + if let ast::Expr::BooleanLiteral(_) | ast::Expr::NoneLiteral(_) = key { + continue; + } + validate_pattern_match_value(vm, key)?; + } + validate_patterns(vm, &mapping.patterns, false) + } + ast::Pattern::MatchClass(match_class) => { + validate_expr(vm, &match_class.cls, ast::ExprContext::Load)?; + let mut cls = match_class.cls.as_ref(); + loop { + match cls { + ast::Expr::Name(_) => break, + ast::Expr::Attribute(attr) => { + cls = &attr.value; + } + _ => { + return Err(vm.new_value_error( + "MatchClass cls field can only contain Name or Attribute nodes." + .to_owned(), + )); + } + } + } + for keyword in &match_class.arguments.keywords { + validate_name(vm, keyword.attr.id())?; + } + validate_patterns(vm, &match_class.arguments.patterns, false)?; + for keyword in &match_class.arguments.keywords { + validate_pattern(vm, &keyword.pattern, false)?; + } + Ok(()) + } + ast::Pattern::MatchStar(star) => { + if !star_ok { + return Err(vm.new_value_error("can't use MatchStar here".to_owned())); + } + if let Some(name) = &star.name { + validate_capture(vm, name)?; + } + Ok(()) + } + ast::Pattern::MatchAs(match_as) => { + if let Some(name) = &match_as.name { + validate_capture(vm, name)?; + } + match &match_as.pattern { + None => Ok(()), + Some(pattern) => { + if match_as.name.is_none() { + return Err(vm.new_value_error( + "MatchAs must specify a target name if a pattern is given".to_owned(), + )); + } + validate_pattern(vm, pattern, false) + } + } + } + ast::Pattern::MatchOr(match_or) => { + if match_or.patterns.len() < 2 { + return Err(vm.new_value_error("MatchOr requires at least 2 patterns".to_owned())); + } + validate_patterns(vm, &match_or.patterns, false) + } + } +} + +fn validate_patterns( + vm: &VirtualMachine, + patterns: &[ast::Pattern], + star_ok: bool, +) -> PyResult<()> { + for pattern in patterns { + validate_pattern(vm, pattern, star_ok)?; + } + Ok(()) +} + +fn validate_typeparam(vm: &VirtualMachine, tp: &ast::TypeParam) -> PyResult<()> { + match tp { + ast::TypeParam::TypeVar(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(bound) = &tp.bound { + validate_expr(vm, bound, ast::ExprContext::Load)?; + } + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + ast::TypeParam::ParamSpec(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + ast::TypeParam::TypeVarTuple(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + } + Ok(()) +} + +fn validate_type_params( + vm: &VirtualMachine, + type_params: &Option<Box<ast::TypeParams>>, +) -> PyResult<()> { + if let Some(type_params) = type_params { + for tp in &type_params.type_params { + validate_typeparam(vm, tp)?; + } + } + Ok(()) +} + +fn validate_exprs( + vm: &VirtualMachine, + exprs: &[ast::Expr], + ctx: ast::ExprContext, + _null_ok: bool, +) -> PyResult<()> { + for expr in exprs { + validate_expr(vm, expr, ctx)?; + } + Ok(()) +} + +fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) -> PyResult<()> { + let mut check_ctx = true; + let actual_ctx = match expr { + ast::Expr::Attribute(attr) => attr.ctx, + ast::Expr::Subscript(sub) => sub.ctx, + ast::Expr::Starred(star) => star.ctx, + ast::Expr::Name(name) => { + validate_name(vm, name.id())?; + name.ctx + } + ast::Expr::List(list) => list.ctx, + ast::Expr::Tuple(tuple) => tuple.ctx, + _ => { + if ctx != ast::ExprContext::Load { + return Err(vm.new_value_error(format!( + "expression which can't be assigned to in {} context", + expr_context_name(ctx) + ))); + } + check_ctx = false; + ast::ExprContext::Invalid + } + }; + if check_ctx && actual_ctx != ctx { + return Err(vm.new_value_error(format!( + "expression must have {} context but has {} instead", + expr_context_name(ctx), + expr_context_name(actual_ctx) + ))); + } + + match expr { + ast::Expr::BoolOp(op) => { + if op.values.len() < 2 { + return Err(vm.new_value_error("BoolOp with less than 2 values".to_owned())); + } + validate_exprs(vm, &op.values, ast::ExprContext::Load, false) + } + ast::Expr::Named(named) => { + if !matches!(&*named.target, ast::Expr::Name(_)) { + return Err(vm.new_type_error("NamedExpr target must be a Name".to_owned())); + } + validate_expr(vm, &named.value, ast::ExprContext::Load) + } + ast::Expr::BinOp(bin) => { + validate_expr(vm, &bin.left, ast::ExprContext::Load)?; + validate_expr(vm, &bin.right, ast::ExprContext::Load) + } + ast::Expr::UnaryOp(unary) => validate_expr(vm, &unary.operand, ast::ExprContext::Load), + ast::Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + validate_parameters(vm, parameters)?; + } + validate_expr(vm, &lambda.body, ast::ExprContext::Load) + } + ast::Expr::If(ifexp) => { + validate_expr(vm, &ifexp.test, ast::ExprContext::Load)?; + validate_expr(vm, &ifexp.body, ast::ExprContext::Load)?; + validate_expr(vm, &ifexp.orelse, ast::ExprContext::Load) + } + ast::Expr::Dict(dict) => { + for item in &dict.items { + if let Some(key) = &item.key { + validate_expr(vm, key, ast::ExprContext::Load)?; + } + validate_expr(vm, &item.value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::Set(set) => validate_exprs(vm, &set.elts, ast::ExprContext::Load, false), + ast::Expr::ListComp(list) => { + validate_comprehension(vm, &list.generators)?; + validate_expr(vm, &list.elt, ast::ExprContext::Load) + } + ast::Expr::SetComp(set) => { + validate_comprehension(vm, &set.generators)?; + validate_expr(vm, &set.elt, ast::ExprContext::Load) + } + ast::Expr::DictComp(dict) => { + validate_comprehension(vm, &dict.generators)?; + validate_expr(vm, &dict.key, ast::ExprContext::Load)?; + validate_expr(vm, &dict.value, ast::ExprContext::Load) + } + ast::Expr::Generator(generator) => { + validate_comprehension(vm, &generator.generators)?; + validate_expr(vm, &generator.elt, ast::ExprContext::Load) + } + ast::Expr::Yield(yield_expr) => { + if let Some(value) = &yield_expr.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::YieldFrom(yield_expr) => { + validate_expr(vm, &yield_expr.value, ast::ExprContext::Load) + } + ast::Expr::Await(await_expr) => { + validate_expr(vm, &await_expr.value, ast::ExprContext::Load) + } + ast::Expr::Compare(compare) => { + if compare.comparators.is_empty() { + return Err(vm.new_value_error("Compare with no comparators".to_owned())); + } + if compare.comparators.len() != compare.ops.len() { + return Err(vm.new_value_error( + "Compare has a different number of comparators and operands".to_owned(), + )); + } + validate_exprs(vm, &compare.comparators, ast::ExprContext::Load, false)?; + validate_expr(vm, &compare.left, ast::ExprContext::Load) + } + ast::Expr::Call(call) => { + validate_expr(vm, &call.func, ast::ExprContext::Load)?; + validate_exprs(vm, &call.arguments.args, ast::ExprContext::Load, false)?; + validate_keywords(vm, &call.arguments.keywords) + } + ast::Expr::FString(fstring) => validate_interpolated_elements( + vm, + fstring + .value + .elements() + .map(ast::InterpolatedStringElementRef::from), + ), + ast::Expr::TString(tstring) => validate_interpolated_elements( + vm, + tstring + .value + .elements() + .map(ast::InterpolatedStringElementRef::from), + ), + ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::NumberLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::EllipsisLiteral(_) => Ok(()), + ast::Expr::Attribute(attr) => validate_expr(vm, &attr.value, ast::ExprContext::Load), + ast::Expr::Subscript(sub) => { + validate_expr(vm, &sub.slice, ast::ExprContext::Load)?; + validate_expr(vm, &sub.value, ast::ExprContext::Load) + } + ast::Expr::Starred(star) => validate_expr(vm, &star.value, ctx), + ast::Expr::Name(_) => Ok(()), + ast::Expr::List(list) => validate_exprs(vm, &list.elts, ctx, false), + ast::Expr::Tuple(tuple) => validate_exprs(vm, &tuple.elts, ctx, false), + ast::Expr::Slice(slice) => { + if let Some(lower) = &slice.lower { + validate_expr(vm, lower, ast::ExprContext::Load)?; + } + if let Some(upper) = &slice.upper { + validate_expr(vm, upper, ast::ExprContext::Load)?; + } + if let Some(step) = &slice.step { + validate_expr(vm, step, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::IpyEscapeCommand(_) => Ok(()), + } +} + +fn validate_decorators(vm: &VirtualMachine, decorators: &[ast::Decorator]) -> PyResult<()> { + for decorator in decorators { + validate_expr(vm, &decorator.expression, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { + match stmt { + ast::Stmt::FunctionDef(func) => { + let owner = if func.is_async { + "AsyncFunctionDef" + } else { + "FunctionDef" + }; + validate_body(vm, &func.body, owner)?; + validate_type_params(vm, &func.type_params)?; + validate_parameters(vm, &func.parameters)?; + validate_decorators(vm, &func.decorator_list)?; + if let Some(returns) = &func.returns { + validate_expr(vm, returns, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::ClassDef(class_def) => { + validate_body(vm, &class_def.body, "ClassDef")?; + validate_type_params(vm, &class_def.type_params)?; + if let Some(arguments) = &class_def.arguments { + validate_exprs(vm, &arguments.args, ast::ExprContext::Load, false)?; + validate_keywords(vm, &arguments.keywords)?; + } + validate_decorators(vm, &class_def.decorator_list) + } + ast::Stmt::Return(ret) => { + if let Some(value) = &ret.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::Delete(del) => validate_assignlist(vm, &del.targets, ast::ExprContext::Del), + ast::Stmt::Assign(assign) => { + validate_assignlist(vm, &assign.targets, ast::ExprContext::Store)?; + validate_expr(vm, &assign.value, ast::ExprContext::Load) + } + ast::Stmt::AugAssign(assign) => { + validate_expr(vm, &assign.target, ast::ExprContext::Store)?; + validate_expr(vm, &assign.value, ast::ExprContext::Load) + } + ast::Stmt::AnnAssign(assign) => { + if assign.simple && !matches!(&*assign.target, ast::Expr::Name(_)) { + return Err(vm.new_type_error("AnnAssign with simple non-Name target".to_owned())); + } + validate_expr(vm, &assign.target, ast::ExprContext::Store)?; + if let Some(value) = &assign.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + validate_expr(vm, &assign.annotation, ast::ExprContext::Load) + } + ast::Stmt::TypeAlias(alias) => { + if !matches!(&*alias.name, ast::Expr::Name(_)) { + return Err(vm.new_type_error("TypeAlias with non-Name name".to_owned())); + } + validate_expr(vm, &alias.name, ast::ExprContext::Store)?; + validate_type_params(vm, &alias.type_params)?; + validate_expr(vm, &alias.value, ast::ExprContext::Load) + } + ast::Stmt::For(for_stmt) => { + let owner = if for_stmt.is_async { "AsyncFor" } else { "For" }; + validate_expr(vm, &for_stmt.target, ast::ExprContext::Store)?; + validate_expr(vm, &for_stmt.iter, ast::ExprContext::Load)?; + validate_body(vm, &for_stmt.body, owner)?; + validate_stmts(vm, &for_stmt.orelse) + } + ast::Stmt::While(while_stmt) => { + validate_expr(vm, &while_stmt.test, ast::ExprContext::Load)?; + validate_body(vm, &while_stmt.body, "While")?; + validate_stmts(vm, &while_stmt.orelse) + } + ast::Stmt::If(if_stmt) => { + validate_expr(vm, &if_stmt.test, ast::ExprContext::Load)?; + validate_body(vm, &if_stmt.body, "If")?; + for clause in &if_stmt.elif_else_clauses { + if let Some(test) = &clause.test { + validate_expr(vm, test, ast::ExprContext::Load)?; + } + validate_body(vm, &clause.body, "If")?; + } + Ok(()) + } + ast::Stmt::With(with_stmt) => { + let owner = if with_stmt.is_async { + "AsyncWith" + } else { + "With" + }; + validate_nonempty_seq(vm, with_stmt.items.len(), "items", owner)?; + for item in &with_stmt.items { + validate_expr(vm, &item.context_expr, ast::ExprContext::Load)?; + if let Some(optional_vars) = &item.optional_vars { + validate_expr(vm, optional_vars, ast::ExprContext::Store)?; + } + } + validate_body(vm, &with_stmt.body, owner) + } + ast::Stmt::Match(match_stmt) => { + validate_expr(vm, &match_stmt.subject, ast::ExprContext::Load)?; + validate_nonempty_seq(vm, match_stmt.cases.len(), "cases", "Match")?; + for case in &match_stmt.cases { + validate_pattern(vm, &case.pattern, false)?; + if let Some(guard) = &case.guard { + validate_expr(vm, guard, ast::ExprContext::Load)?; + } + validate_body(vm, &case.body, "match_case")?; + } + Ok(()) + } + ast::Stmt::Raise(raise) => { + if let Some(exc) = &raise.exc { + validate_expr(vm, exc, ast::ExprContext::Load)?; + if let Some(cause) = &raise.cause { + validate_expr(vm, cause, ast::ExprContext::Load)?; + } + } else if raise.cause.is_some() { + return Err(vm.new_value_error("Raise with cause but no exception".to_owned())); + } + Ok(()) + } + ast::Stmt::Try(try_stmt) => { + let owner = if try_stmt.is_star { "TryStar" } else { "Try" }; + validate_body(vm, &try_stmt.body, owner)?; + if try_stmt.handlers.is_empty() && try_stmt.finalbody.is_empty() { + return Err(vm.new_value_error(format!( + "{owner} has neither except handlers nor finalbody" + ))); + } + if try_stmt.handlers.is_empty() && !try_stmt.orelse.is_empty() { + return Err( + vm.new_value_error(format!("{owner} has orelse but no except handlers")) + ); + } + for handler in &try_stmt.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + if let Some(type_expr) = &handler.type_ { + validate_expr(vm, type_expr, ast::ExprContext::Load)?; + } + validate_body(vm, &handler.body, "ExceptHandler")?; + } + validate_stmts(vm, &try_stmt.finalbody)?; + validate_stmts(vm, &try_stmt.orelse) + } + ast::Stmt::Assert(assert_stmt) => { + validate_expr(vm, &assert_stmt.test, ast::ExprContext::Load)?; + if let Some(msg) = &assert_stmt.msg { + validate_expr(vm, msg, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::Import(import) => { + validate_nonempty_seq(vm, import.names.len(), "names", "Import")?; + Ok(()) + } + ast::Stmt::ImportFrom(import) => { + validate_nonempty_seq(vm, import.names.len(), "names", "ImportFrom")?; + Ok(()) + } + ast::Stmt::Global(global) => { + validate_nonempty_seq(vm, global.names.len(), "names", "Global")?; + Ok(()) + } + ast::Stmt::Nonlocal(nonlocal) => { + validate_nonempty_seq(vm, nonlocal.names.len(), "names", "Nonlocal")?; + Ok(()) + } + ast::Stmt::Expr(expr) => validate_expr(vm, &expr.value, ast::ExprContext::Load), + ast::Stmt::Pass(_) + | ast::Stmt::Break(_) + | ast::Stmt::Continue(_) + | ast::Stmt::IpyEscapeCommand(_) => Ok(()), + } +} + +fn validate_stmts(vm: &VirtualMachine, stmts: &[ast::Stmt]) -> PyResult<()> { + for stmt in stmts { + validate_stmt(vm, stmt)?; + } + Ok(()) +} + +pub(super) fn validate_mod(vm: &VirtualMachine, module: &Mod) -> PyResult<()> { + match module { + Mod::Module(module) => validate_stmts(vm, &module.body), + Mod::Interactive(module) => validate_stmts(vm, &module.body), + Mod::Expression(expr) => validate_expr(vm, &expr.body, ast::ExprContext::Load), + Mod::FunctionType(func_type) => { + validate_exprs(vm, &func_type.argtypes, ast::ExprContext::Load, false)?; + validate_expr(vm, &func_type.returns, ast::ExprContext::Load) + } + } +} diff --git a/crates/vm/src/stdlib/atexit.rs b/crates/vm/src/stdlib/atexit.rs index b1832b5481d..338fae3b2b7 100644 --- a/crates/vm/src/stdlib/atexit.rs +++ b/crates/vm/src/stdlib/atexit.rs @@ -1,5 +1,5 @@ pub use atexit::_run_exitfuncs; -pub(crate) use atexit::make_module; +pub(crate) use atexit::module_def; #[pymodule] mod atexit { @@ -34,7 +34,7 @@ mod atexit { #[pyfunction] pub fn _run_exitfuncs(vm: &VirtualMachine) { - let funcs: Vec<_> = std::mem::take(&mut *vm.state.atexit_funcs.lock()); + let funcs: Vec<_> = core::mem::take(&mut *vm.state.atexit_funcs.lock()); for (func, args) in funcs.into_iter().rev() { if let Err(e) = func.call(args, vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 542476d68c7..1b54a26e732 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -2,15 +2,13 @@ //! //! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html). use crate::{Py, VirtualMachine, builtins::PyModule, class::PyClassImpl}; -pub(crate) use builtins::{__module_def, DOC}; +pub(crate) use builtins::{DOC, module_def}; pub use builtins::{ascii, print, reversed}; #[pymodule] mod builtins { - use std::io::IsTerminal; - use crate::{ - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, enumerate::PyReverseSequenceIterator, @@ -47,7 +45,7 @@ mod builtins { #[pyfunction] fn all(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { for item in iterable.iter(vm)? { - if !*item? { + if !item?.into_bool() { return Ok(false); } } @@ -57,7 +55,7 @@ mod builtins { #[pyfunction] fn any(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { for item in iterable.iter(vm)? { - if *item? { + if item?.into_bool() { return Ok(true); } } @@ -124,15 +122,13 @@ mod builtins { { use crate::{class::PyClassImpl, stdlib::ast}; - if args._feature_version.is_present() { - // TODO: add support for _feature_version - } + let feature_version = feature_version_from_arg(args._feature_version, vm)?; let mode_str = args.mode.as_str(); let optimize: i32 = args.optimize.map_or(Ok(-1), |v| v.try_to_primitive(vm))?; let optimize: u8 = if optimize == -1 { - vm.state.settings.optimize + vm.state.config.settings.optimize } else { optimize .try_into() @@ -143,6 +139,37 @@ mod builtins { .source .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) { + use num_traits::Zero; + let flags: i32 = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + let is_ast_only = !(flags & ast::PY_CF_ONLY_AST).is_zero(); + + // func_type mode requires PyCF_ONLY_AST + if mode_str == "func_type" && !is_ast_only { + return Err(vm.new_value_error( + "compile() mode 'func_type' requires flag PyCF_ONLY_AST".to_owned(), + )); + } + + // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation + if is_ast_only { + let (expected_type, expected_name) = ast::mode_type_and_name(&vm.ctx, mode_str) + .ok_or_else(|| { + vm.new_value_error( + "compile() mode must be 'exec', 'eval', 'single' or 'func_type'" + .to_owned(), + ) + })?; + if !args.source.fast_isinstance(&expected_type) { + return Err(vm.new_type_error(format!( + "expected {} node, got {}", + expected_name, + args.source.class().name() + ))); + } + ast::validate_ast_object(vm, args.source.clone())?; + return Ok(args.source); + } + #[cfg(not(feature = "rustpython-codegen"))] { return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())); @@ -177,7 +204,7 @@ mod builtins { let source = source.borrow_bytes(); // TODO: compiler::compile should probably get bytes - let source = std::str::from_utf8(&source) + let source = core::str::from_utf8(&source) .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; @@ -187,14 +214,32 @@ mod builtins { } let allow_incomplete = !(flags & ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero(); + let type_comments = !(flags & ast::PY_CF_TYPE_COMMENTS).is_zero(); + + let optimize_level = optimize; - if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { + if (flags & ast::PY_CF_ONLY_AST).is_zero() { #[cfg(not(feature = "compiler"))] { Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned())) } #[cfg(feature = "compiler")] { + if let Some(feature_version) = feature_version { + let mode = mode_str + .parse::<parser::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + let _ = ast::parse( + vm, + source, + mode, + optimize_level, + Some(feature_version), + type_comments, + ) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))?; + } + let mode = mode_str .parse::<crate::compiler::Mode>() .map_err(|err| vm.new_value_error(err.to_string()))?; @@ -215,16 +260,53 @@ mod builtins { Ok(code.into()) } } else { + if mode_str == "func_type" { + return ast::parse_func_type(vm, source, optimize_level, feature_version) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm)); + } + let mode = mode_str .parse::<parser::Mode>() .map_err(|err| vm.new_value_error(err.to_string()))?; - ast::parse(vm, source, mode) - .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm)) + let parsed = ast::parse( + vm, + source, + mode, + optimize_level, + feature_version, + type_comments, + ) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))?; + + if mode_str == "single" { + return ast::wrap_interactive(vm, parsed); + } + + Ok(parsed) } } } } + #[cfg(feature = "ast")] + fn feature_version_from_arg( + feature_version: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<Option<ruff_python_ast::PythonVersion>> { + let minor = match feature_version.into_option() { + Some(minor) => minor, + None => return Ok(None), + }; + + if minor < 0 { + return Ok(None); + } + + let minor = u8::try_from(minor) + .map_err(|_| vm.new_value_error("compile() _feature_version out of range"))?; + Ok(Some(ruff_python_ast::PythonVersion { major: 3, minor })) + } + #[pyfunction] fn delattr(obj: PyObjectRef, attr: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { @@ -261,14 +343,14 @@ mod builtins { func_name: &'static str, ) -> PyResult<crate::scope::Scope> { fn validate_globals_dict( - globals: &PyObjectRef, + globals: &PyObject, vm: &VirtualMachine, func_name: &'static str, ) -> PyResult<()> { if !globals.fast_isinstance(vm.ctx.types.dict_type) { return Err(match func_name { "eval" => { - let is_mapping = crate::protocol::PyMapping::check(globals); + let is_mapping = globals.mapping_unchecked().check(); vm.new_type_error(if is_mapping { "globals must be a real dict; try eval(expr, {}, mapping)" .to_owned() @@ -302,7 +384,7 @@ mod builtins { ) } None => ( - vm.current_globals().clone(), + vm.current_globals(), if let Some(locals) = self.locals { locals } else { @@ -335,7 +417,7 @@ mod builtins { )); } - let source = std::str::from_utf8(source).map_err(|err| { + let source = core::str::from_utf8(source).map_err(|err| { let msg = format!( "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", source[err.valid_up_to()], @@ -421,7 +503,7 @@ mod builtins { #[pyfunction] fn globals(vm: &VirtualMachine) -> PyDictRef { - vm.current_globals().clone() + vm.current_globals() } #[pyfunction] @@ -453,6 +535,7 @@ mod builtins { #[pyfunction] fn hex(number: ArgIndex) -> String { + let number = number.into_int_ref(); let n = number.as_bigint(); format!("{n:#x}") } @@ -464,6 +547,8 @@ mod builtins { #[pyfunction] fn input(prompt: OptionalArg<PyStrRef>, vm: &VirtualMachine) -> PyResult { + use std::io::IsTerminal; + let stdin = sys::get_stdin(vm)?; let stdout = sys::get_stdout(vm)?; let stderr = sys::get_stderr(vm)?; @@ -476,8 +561,13 @@ mod builtins { .is_ok_and(|fd| fd == expected) }; - // everything is normal, we can just rely on rustyline to use stdin/stdout - if fd_matches(&stdin, 0) && fd_matches(&stdout, 1) && std::io::stdin().is_terminal() { + // Check if we should use rustyline (interactive terminal, not PTY child) + let use_rustyline = fd_matches(&stdin, 0) + && fd_matches(&stdout, 1) + && std::io::stdin().is_terminal() + && !is_pty_child(); + + if use_rustyline { let prompt = prompt.as_ref().map_or("", |s| s.as_str()); let mut readline = Readline::new(()); match readline.readline(prompt) { @@ -502,6 +592,21 @@ mod builtins { } } + /// Check if we're running in a PTY child process (e.g., after pty.fork()). + /// pty.fork() calls setsid(), making the child a session leader. + /// In this case, rustyline may hang because it uses raw mode. + #[cfg(unix)] + fn is_pty_child() -> bool { + use nix::unistd::{getpid, getsid}; + // If this process is a session leader, we're likely in a PTY child + getsid(None) == Ok(getpid()) + } + + #[cfg(not(unix))] + fn is_pty_child() -> bool { + false + } + #[pyfunction] fn isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { obj.is_instance(&typ, vm) @@ -585,7 +690,7 @@ mod builtins { } let candidates = match args.args.len().cmp(&1) { - std::cmp::Ordering::Greater => { + core::cmp::Ordering::Greater => { if default.is_some() { return Err(vm.new_type_error(format!( "Cannot specify a default for {func_name}() with multiple positional arguments" @@ -593,8 +698,8 @@ mod builtins { } args.args } - std::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, - std::cmp::Ordering::Less => { + core::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, + core::cmp::Ordering::Less => { // zero arguments means type error: return Err( vm.new_type_error(format!("{func_name} expected at least 1 argument, got 0")) @@ -607,7 +712,7 @@ mod builtins { Some(x) => x, None => { return default.ok_or_else(|| { - vm.new_value_error(format!("{func_name}() arg is an empty sequence")) + vm.new_value_error(format!("{func_name}() iterable argument is empty")) }); } }; @@ -667,6 +772,7 @@ mod builtins { #[pyfunction] fn oct(number: ArgIndex, vm: &VirtualMachine) -> PyResult { + let number = number.into_int_ref(); let n = number.as_bigint(); let s = if n.is_negative() { format!("-0o{:o}", n.abs()) @@ -766,7 +872,7 @@ mod builtins { .unwrap_or_else(|| PyStr::from("\n").into_ref(&vm.ctx)); write(end)?; - if *options.flush { + if options.flush.into() { vm.call_method(&file, "flush", ())?; } @@ -885,9 +991,24 @@ mod builtins { Ok(sum) } + #[derive(FromArgs)] + struct ImportArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(any, default)] + globals: Option<PyObjectRef>, + #[allow(dead_code)] + #[pyarg(any, default)] + locals: Option<PyObjectRef>, + #[pyarg(any, default)] + fromlist: Option<PyObjectRef>, + #[pyarg(any, default)] + level: i32, + } + #[pyfunction] - fn __import__(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm.import_func.call(args, vm) + fn __import__(args: ImportArgs, vm: &VirtualMachine) -> PyResult { + crate::import::import_module_level(&args.name, args.globals, args.fromlist, args.level, vm) } #[pyfunction] @@ -1078,9 +1199,10 @@ pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) { crate::protocol::VecBuffer::make_class(&vm.ctx); - builtins::extend_module(vm, module).unwrap(); + module.__init_methods(vm).unwrap(); + builtins::module_exec(vm, module).unwrap(); - let debug_mode: bool = vm.state.settings.optimize == 0; + let debug_mode: bool = vm.state.config.settings.optimize == 0; // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) let exception_group = crate::exception_group::exception_group(); diff --git a/crates/vm/src/stdlib/codecs.rs b/crates/vm/src/stdlib/codecs.rs index 5f1b721dfb4..85869e066cb 100644 --- a/crates/vm/src/stdlib/codecs.rs +++ b/crates/vm/src/stdlib/codecs.rs @@ -1,6 +1,10 @@ -pub(crate) use _codecs::make_module; +// spell-checker: ignore unencodable pused -#[pymodule] +pub(crate) use _codecs::module_def; + +use crate::common::static_cell::StaticCell; + +#[pymodule(with(#[cfg(windows)] _codecs_windows))] mod _codecs { use crate::codecs::{ErrorsHandler, PyDecodeContext, PyEncodeContext}; use crate::common::encodings; @@ -9,6 +13,7 @@ mod _codecs { AsObject, PyObjectRef, PyResult, VirtualMachine, builtins::{PyStrRef, PyUtf8StrRef}, codecs, + exceptions::cstring_error, function::{ArgBytesLike, FuncArgs}, }; @@ -24,6 +29,9 @@ mod _codecs { #[pyfunction] fn lookup(encoding: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + if encoding.as_str().contains('\0') { + return Err(cstring_error(vm)); + } vm.state .codec_registry .lookup(encoding.as_str(), vm) @@ -79,9 +87,32 @@ mod _codecs { #[pyfunction] fn lookup_error(name: PyStrRef, vm: &VirtualMachine) -> PyResult { + if name.as_wtf8().as_bytes().contains(&0) { + return Err(cstring_error(vm)); + } + if !name.as_wtf8().is_utf8() { + return Err(vm.new_unicode_encode_error( + "'utf-8' codec can't encode character: surrogates not allowed".to_owned(), + )); + } vm.state.codec_registry.lookup_error(name.as_str(), vm) } + #[pyfunction] + fn _unregister_error(errors: PyStrRef, vm: &VirtualMachine) -> PyResult<bool> { + if errors.as_wtf8().as_bytes().contains(&0) { + return Err(cstring_error(vm)); + } + if !errors.as_wtf8().is_utf8() { + return Err(vm.new_unicode_encode_error( + "'utf-8' codec can't encode character: surrogates not allowed".to_owned(), + )); + } + vm.state + .codec_registry + .unregister_error(errors.as_str(), vm) + } + type EncodeResult = PyResult<(Vec<u8>, usize)>; #[derive(FromArgs)] @@ -176,7 +207,7 @@ mod _codecs { #[pyfunction] fn latin_1_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - if args.s.is_ascii() { + if args.s.isascii() { return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); } do_codec!(latin_1::encode, args, vm) @@ -189,7 +220,7 @@ mod _codecs { #[pyfunction] fn ascii_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - if args.s.is_ascii() { + if args.s.isascii() { return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); } do_codec!(ascii::encode, args, vm) @@ -202,30 +233,141 @@ mod _codecs { // TODO: implement these codecs in Rust! - use crate::common::static_cell::StaticCell; - #[inline] - fn delegate_pycodecs( - cell: &'static StaticCell<PyObjectRef>, - name: &'static str, - args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult { - let f = cell.get_or_try_init(|| { - let module = vm.import("_pycodecs", 0)?; - module.get_attr(name, vm) - })?; - f.call(args, vm) - } macro_rules! delegate_pycodecs { ($name:ident, $args:ident, $vm:ident) => {{ rustpython_common::static_cell!( static FUNC: PyObjectRef; ); - delegate_pycodecs(&FUNC, stringify!($name), $args, $vm) + super::delegate_pycodecs(&FUNC, stringify!($name), $args, $vm) }}; } - #[cfg(windows)] + #[pyfunction] + fn readbuffer_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(readbuffer_encode, args, vm) + } + #[pyfunction] + fn escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(escape_encode, args, vm) + } + #[pyfunction] + fn escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(escape_decode, args, vm) + } + #[pyfunction] + fn unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(unicode_escape_encode, args, vm) + } + #[pyfunction] + fn unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(unicode_escape_decode, args, vm) + } + #[pyfunction] + fn raw_unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(raw_unicode_escape_encode, args, vm) + } + #[pyfunction] + fn raw_unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(raw_unicode_escape_decode, args, vm) + } + #[pyfunction] + fn utf_7_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_7_encode, args, vm) + } + #[pyfunction] + fn utf_7_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_7_decode, args, vm) + } + #[pyfunction] + fn utf_16_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_encode, args, vm) + } + #[pyfunction] + fn utf_16_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_decode, args, vm) + } + #[pyfunction] + fn charmap_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_encode, args, vm) + } + #[pyfunction] + fn charmap_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_decode, args, vm) + } + #[pyfunction] + fn charmap_build(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_build, args, vm) + } + #[pyfunction] + fn utf_16_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_le_encode, args, vm) + } + #[pyfunction] + fn utf_16_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_le_decode, args, vm) + } + #[pyfunction] + fn utf_16_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_be_encode, args, vm) + } + #[pyfunction] + fn utf_16_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_be_decode, args, vm) + } + #[pyfunction] + fn utf_16_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_ex_decode, args, vm) + } + #[pyfunction] + fn utf_32_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_ex_decode, args, vm) + } + #[pyfunction] + fn utf_32_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_encode, args, vm) + } + #[pyfunction] + fn utf_32_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_decode, args, vm) + } + #[pyfunction] + fn utf_32_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_le_encode, args, vm) + } + #[pyfunction] + fn utf_32_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_le_decode, args, vm) + } + #[pyfunction] + fn utf_32_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_be_encode, args, vm) + } + #[pyfunction] + fn utf_32_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_be_decode, args, vm) + } +} + +#[inline] +fn delegate_pycodecs( + cell: &'static StaticCell<crate::PyObjectRef>, + name: &'static str, + args: crate::function::FuncArgs, + vm: &crate::VirtualMachine, +) -> crate::PyResult { + let f = cell.get_or_try_init(|| { + let module = vm.import("_pycodecs", 0)?; + module.get_attr(name, vm) + })?; + f.call(args, vm) +} + +#[cfg(windows)] +#[pymodule(sub)] +mod _codecs_windows { + use crate::{PyResult, VirtualMachine}; + use crate::{builtins::PyStrRef, function::ArgBytesLike}; + #[derive(FromArgs)] struct MbcsEncodeArgs { #[pyarg(positional)] @@ -234,7 +376,6 @@ mod _codecs { errors: Option<PyStrRef>, } - #[cfg(windows)] #[pyfunction] fn mbcs_encode(args: MbcsEncodeArgs, vm: &VirtualMachine) -> PyResult<(Vec<u8>, usize)> { use crate::common::windows::ToWideString; @@ -268,10 +409,10 @@ mod _codecs { WC_NO_BEST_FIT_CHARS, wide.as_ptr(), wide.len() as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, - std::ptr::null(), - std::ptr::null_mut(), + core::ptr::null(), + core::ptr::null_mut(), ) }; @@ -291,11 +432,11 @@ mod _codecs { wide.len() as i32, buffer.as_mut_ptr().cast(), size, - std::ptr::null(), + core::ptr::null(), if errors == "strict" { &mut used_default_char } else { - std::ptr::null_mut() + core::ptr::null_mut() }, ) }; @@ -315,13 +456,6 @@ mod _codecs { Ok((buffer, char_len)) } - #[cfg(not(windows))] - #[pyfunction] - fn mbcs_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(mbcs_encode, args, vm) - } - - #[cfg(windows)] #[derive(FromArgs)] struct MbcsDecodeArgs { #[pyarg(positional)] @@ -333,7 +467,6 @@ mod _codecs { r#final: bool, } - #[cfg(windows)] #[pyfunction] fn mbcs_decode(args: MbcsDecodeArgs, vm: &VirtualMachine) -> PyResult<(String, usize)> { use windows_sys::Win32::Globalization::{ @@ -355,7 +488,7 @@ mod _codecs { MB_ERR_INVALID_CHARS, data.as_ptr().cast(), len as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, ) }; @@ -368,7 +501,7 @@ mod _codecs { 0, data.as_ptr().cast(), len as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, ) }; @@ -421,13 +554,6 @@ mod _codecs { Ok((s, len)) } - #[cfg(not(windows))] - #[pyfunction] - fn mbcs_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(mbcs_decode, args, vm) - } - - #[cfg(windows)] #[derive(FromArgs)] struct OemEncodeArgs { #[pyarg(positional)] @@ -436,7 +562,6 @@ mod _codecs { errors: Option<PyStrRef>, } - #[cfg(windows)] #[pyfunction] fn oem_encode(args: OemEncodeArgs, vm: &VirtualMachine) -> PyResult<(Vec<u8>, usize)> { use crate::common::windows::ToWideString; @@ -470,10 +595,10 @@ mod _codecs { WC_NO_BEST_FIT_CHARS, wide.as_ptr(), wide.len() as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, - std::ptr::null(), - std::ptr::null_mut(), + core::ptr::null(), + core::ptr::null_mut(), ) }; @@ -493,11 +618,11 @@ mod _codecs { wide.len() as i32, buffer.as_mut_ptr().cast(), size, - std::ptr::null(), + core::ptr::null(), if errors == "strict" { &mut used_default_char } else { - std::ptr::null_mut() + core::ptr::null_mut() }, ) }; @@ -517,13 +642,6 @@ mod _codecs { Ok((buffer, char_len)) } - #[cfg(not(windows))] - #[pyfunction] - fn oem_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(oem_encode, args, vm) - } - - #[cfg(windows)] #[derive(FromArgs)] struct OemDecodeArgs { #[pyarg(positional)] @@ -535,7 +653,6 @@ mod _codecs { r#final: bool, } - #[cfg(windows)] #[pyfunction] fn oem_decode(args: OemDecodeArgs, vm: &VirtualMachine) -> PyResult<(String, usize)> { use windows_sys::Win32::Globalization::{ @@ -557,7 +674,7 @@ mod _codecs { MB_ERR_INVALID_CHARS, data.as_ptr().cast(), len as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, ) }; @@ -570,7 +687,7 @@ mod _codecs { 0, data.as_ptr().cast(), len as i32, - std::ptr::null_mut(), + core::ptr::null_mut(), 0, ) }; @@ -623,87 +740,647 @@ mod _codecs { Ok((s, len)) } - #[cfg(not(windows))] - #[pyfunction] - fn oem_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(oem_decode, args, vm) + #[derive(FromArgs)] + struct CodePageEncodeArgs { + #[pyarg(positional)] + code_page: i32, + #[pyarg(positional)] + s: PyStrRef, + #[pyarg(positional, optional)] + errors: Option<PyStrRef>, } - #[pyfunction] - fn readbuffer_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(readbuffer_encode, args, vm) - } - #[pyfunction] - fn escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(escape_encode, args, vm) - } - #[pyfunction] - fn escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(escape_decode, args, vm) - } - #[pyfunction] - fn unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(unicode_escape_encode, args, vm) - } - #[pyfunction] - fn unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(unicode_escape_decode, args, vm) - } - #[pyfunction] - fn raw_unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(raw_unicode_escape_encode, args, vm) - } - #[pyfunction] - fn raw_unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(raw_unicode_escape_decode, args, vm) - } - #[pyfunction] - fn utf_7_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_7_encode, args, vm) - } - #[pyfunction] - fn utf_7_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_7_decode, args, vm) - } - #[pyfunction] - fn utf_16_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_encode, args, vm) - } - #[pyfunction] - fn utf_16_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_decode, args, vm) + fn code_page_encoding_name(code_page: u32) -> String { + match code_page { + 0 => "mbcs".to_string(), + cp => format!("cp{cp}"), + } } - #[pyfunction] - fn charmap_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_encode, args, vm) + + /// Get WideCharToMultiByte flags for encoding. + /// Matches encode_code_page_flags() in CPython. + fn encode_code_page_flags(code_page: u32, errors: &str) -> u32 { + use windows_sys::Win32::Globalization::{WC_ERR_INVALID_CHARS, WC_NO_BEST_FIT_CHARS}; + if code_page == 65001 { + // CP_UTF8 + WC_ERR_INVALID_CHARS + } else if code_page == 65000 { + // CP_UTF7 only supports flags=0 + 0 + } else if errors == "replace" { + 0 + } else { + WC_NO_BEST_FIT_CHARS + } } - #[pyfunction] - fn charmap_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_decode, args, vm) + + /// Try to encode the entire wide string at once (fast/strict path). + /// Returns Ok(Some(bytes)) on success, Ok(None) if there are unencodable chars, + /// or Err on OS error. + fn try_encode_code_page_strict( + code_page: u32, + wide: &[u16], + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<u8>>> { + use windows_sys::Win32::Globalization::WideCharToMultiByte; + + let flags = encode_code_page_flags(code_page, "strict"); + + let use_default_char = code_page != 65001 && code_page != 65000; + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let size = unsafe { + WideCharToMultiByte( + code_page, + flags, + wide.as_ptr(), + wide.len() as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + pused, + ) + }; + + if size <= 0 { + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1113 { + // ERROR_NO_UNICODE_TRANSLATION + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_encode: {err}"))); + } + + if use_default_char && used_default_char != 0 { + return Ok(None); + } + + let mut buffer = vec![0u8; size as usize]; + used_default_char = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let result = unsafe { + WideCharToMultiByte( + code_page, + flags, + wide.as_ptr(), + wide.len() as i32, + buffer.as_mut_ptr().cast(), + size, + core::ptr::null(), + pused, + ) + }; + + if result <= 0 { + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1113 { + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_encode: {err}"))); + } + + if use_default_char && used_default_char != 0 { + return Ok(None); + } + + buffer.truncate(result as usize); + Ok(Some(buffer)) } - #[pyfunction] - fn charmap_build(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_build, args, vm) + + /// Encode character by character with error handling. + fn encode_code_page_errors( + code_page: u32, + s: &PyStrRef, + errors: &str, + encoding_name: &str, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, usize)> { + use crate::builtins::{PyBytes, PyStr, PyTuple}; + use windows_sys::Win32::Globalization::WideCharToMultiByte; + + let char_len = s.char_len(); + let flags = encode_code_page_flags(code_page, errors); + let use_default_char = code_page != 65001 && code_page != 65000; + let encoding_str = vm.ctx.new_str(encoding_name); + let reason_str = vm.ctx.new_str("invalid character"); + + // For strict mode, find the first unencodable character and raise + if errors == "strict" { + // Find the failing position by trying each character + let mut fail_pos = 0; + for cp in s.as_wtf8().code_points() { + let ch = cp.to_u32(); + if (0xD800..=0xDFFF).contains(&ch) { + break; + } + let mut wchars = [0u16; 2]; + let wchar_len = if ch < 0x10000 { + wchars[0] = ch as u16; + 1 + } else { + wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; + wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; + 2 + }; + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + let outsize = unsafe { + WideCharToMultiByte( + code_page, + flags, + wchars.as_ptr(), + wchar_len, + core::ptr::null_mut(), + 0, + core::ptr::null(), + pused, + ) + }; + if outsize <= 0 || (use_default_char && used_default_char != 0) { + break; + } + fail_pos += 1; + } + return Err(vm.new_unicode_encode_error_real( + encoding_str, + s.clone(), + fail_pos, + fail_pos + 1, + reason_str, + )); + } + + let error_handler = vm.state.codec_registry.lookup_error(errors, vm)?; + let mut output = Vec::new(); + + // Collect code points for random access + let code_points: Vec<u32> = s.as_wtf8().code_points().map(|cp| cp.to_u32()).collect(); + + let mut pos = 0usize; + while pos < code_points.len() { + let ch = code_points[pos]; + + // Convert code point to UTF-16 + let mut wchars = [0u16; 2]; + let wchar_len; + let is_surrogate = (0xD800..=0xDFFF).contains(&ch); + + if is_surrogate { + wchar_len = 0; // Can't encode surrogates normally + } else if ch < 0x10000 { + wchars[0] = ch as u16; + wchar_len = 1; + } else { + wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; + wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; + wchar_len = 2; + } + + if !is_surrogate { + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let mut buf = [0u8; 8]; + let outsize = unsafe { + WideCharToMultiByte( + code_page, + flags, + wchars.as_ptr(), + wchar_len, + buf.as_mut_ptr().cast(), + buf.len() as i32, + core::ptr::null(), + pused, + ) + }; + + if outsize > 0 && (!use_default_char || used_default_char == 0) { + output.extend_from_slice(&buf[..outsize as usize]); + pos += 1; + continue; + } + } + + // Character can't be encoded - call error handler + let exc = vm.new_unicode_encode_error_real( + encoding_str.clone(), + s.clone(), + pos, + pos + 1, + reason_str.clone(), + ); + + let res = error_handler.call((exc,), vm)?; + let tuple_err = + || vm.new_type_error("encoding error handler must return (str/bytes, int) tuple"); + let tuple: &PyTuple = res.downcast_ref().ok_or_else(&tuple_err)?; + let tuple_slice = tuple.as_slice(); + if tuple_slice.len() != 2 { + return Err(tuple_err()); + } + + let replacement = &tuple_slice[0]; + let new_pos_obj = tuple_slice[1].clone(); + + if let Some(bytes) = replacement.downcast_ref::<PyBytes>() { + output.extend_from_slice(bytes); + } else if let Some(rep_str) = replacement.downcast_ref::<PyStr>() { + // Replacement string - try to encode each character + for rcp in rep_str.as_wtf8().code_points() { + let rch = rcp.to_u32(); + if rch > 127 { + return Err(vm.new_unicode_encode_error_real( + encoding_str.clone(), + s.clone(), + pos, + pos + 1, + vm.ctx + .new_str("unable to encode error handler result to ASCII"), + )); + } + output.push(rch as u8); + } + } else { + return Err(tuple_err()); + } + + let new_pos: isize = new_pos_obj.try_into_value(vm).map_err(|_| tuple_err())?; + pos = if new_pos < 0 { + (code_points.len() as isize + new_pos).max(0) as usize + } else { + new_pos as usize + }; + } + + Ok((output, char_len)) } + #[pyfunction] - fn utf_16_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_le_encode, args, vm) + fn code_page_encode( + args: CodePageEncodeArgs, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, usize)> { + use crate::common::windows::ToWideString; + + if args.code_page < 0 { + return Err(vm.new_value_error("invalid code page number".to_owned())); + } + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let code_page = args.code_page as u32; + let char_len = args.s.char_len(); + + if char_len == 0 { + return Ok((Vec::new(), 0)); + } + + let encoding_name = code_page_encoding_name(code_page); + + // Fast path: try encoding the whole string at once (only if no surrogates) + if let Some(str_data) = args.s.to_str() { + let wide: Vec<u16> = std::ffi::OsStr::new(str_data).to_wide(); + if let Some(result) = try_encode_code_page_strict(code_page, &wide, vm)? { + return Ok((result, char_len)); + } + } + + // Slow path: character by character with error handling + encode_code_page_errors(code_page, &args.s, errors, &encoding_name, vm) } - #[pyfunction] - fn utf_16_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_le_decode, args, vm) + + #[derive(FromArgs)] + struct CodePageDecodeArgs { + #[pyarg(positional)] + code_page: i32, + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyStrRef>, + #[pyarg(positional, default = false)] + r#final: bool, } - #[pyfunction] - fn utf_16_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_be_encode, args, vm) + + /// Try to decode the entire buffer with strict flags (fast path). + /// Returns Ok(Some(wide_chars)) on success, Ok(None) on decode error, + /// or Err on OS error. + fn try_decode_code_page_strict( + code_page: u32, + data: &[u8], + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<u16>>> { + use windows_sys::Win32::Globalization::{MB_ERR_INVALID_CHARS, MultiByteToWideChar}; + + let mut flags = MB_ERR_INVALID_CHARS; + + loop { + let size = unsafe { + MultiByteToWideChar( + code_page, + flags, + data.as_ptr().cast(), + data.len() as i32, + core::ptr::null_mut(), + 0, + ) + }; + if size > 0 { + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + code_page, + flags, + data.as_ptr().cast(), + data.len() as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result > 0 { + buffer.truncate(result as usize); + return Ok(Some(buffer)); + } + } + + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + // ERROR_INVALID_FLAGS = 1004 + if flags != 0 && err_code == 1004 { + flags = 0; + continue; + } + // ERROR_NO_UNICODE_TRANSLATION = 1113 + if err_code == 1113 { + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_decode: {err}"))); + } } - #[pyfunction] - fn utf_16_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_be_decode, args, vm) + + /// Decode byte by byte with error handling (slow path). + fn decode_code_page_errors( + code_page: u32, + data: &[u8], + errors: &str, + is_final: bool, + encoding_name: &str, + vm: &VirtualMachine, + ) -> PyResult<(PyStrRef, usize)> { + use crate::builtins::PyTuple; + use crate::common::wtf8::Wtf8Buf; + use windows_sys::Win32::Globalization::{MB_ERR_INVALID_CHARS, MultiByteToWideChar}; + + let len = data.len(); + let encoding_str = vm.ctx.new_str(encoding_name); + let reason_str = vm + .ctx + .new_str("No mapping for the Unicode character exists in the target code page."); + + // For strict+final, find the failing position and raise + if errors == "strict" && is_final { + // Find the exact failing byte position by trying byte by byte + let mut fail_pos = 0; + let mut flags_s: u32 = MB_ERR_INVALID_CHARS; + let mut buf = [0u16; 2]; + while fail_pos < len { + let mut in_size = 1; + let mut found = false; + while in_size <= 4 && fail_pos + in_size <= len { + let outsize = unsafe { + MultiByteToWideChar( + code_page, + flags_s, + data[fail_pos..].as_ptr().cast(), + in_size as i32, + buf.as_mut_ptr(), + 2, + ) + }; + if outsize > 0 { + fail_pos += in_size; + found = true; + break; + } + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1004 && flags_s != 0 { + flags_s = 0; + continue; + } + in_size += 1; + } + if !found { + break; + } + } + let object = vm.ctx.new_bytes(data.to_vec()); + return Err(vm.new_unicode_decode_error_real( + encoding_str, + object, + fail_pos, + fail_pos + 1, + reason_str, + )); + } + + let error_handler = if errors != "strict" + && errors != "ignore" + && errors != "replace" + && errors != "backslashreplace" + && errors != "surrogateescape" + { + Some(vm.state.codec_registry.lookup_error(errors, vm)?) + } else { + None + }; + + let mut wide_buf: Vec<u16> = Vec::new(); + let mut pos = 0usize; + let mut flags: u32 = MB_ERR_INVALID_CHARS; + + while pos < len { + // Try to decode with increasing byte counts (1, 2, 3, 4) + let mut in_size = 1; + let mut outsize; + let mut buffer = [0u16; 2]; + + loop { + outsize = unsafe { + MultiByteToWideChar( + code_page, + flags, + data[pos..].as_ptr().cast(), + in_size as i32, + buffer.as_mut_ptr(), + 2, + ) + }; + if outsize > 0 { + break; + } + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1004 && flags != 0 { + // ERROR_INVALID_FLAGS - retry with flags=0 + flags = 0; + continue; + } + if err_code != 1113 && err_code != 122 { + // Not ERROR_NO_UNICODE_TRANSLATION and not ERROR_INSUFFICIENT_BUFFER + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_decode: {err}"))); + } + in_size += 1; + if in_size > 4 || pos + in_size > len { + break; + } + } + + if outsize <= 0 { + // Can't decode this byte sequence + if pos + in_size >= len && !is_final { + // Incomplete sequence at end, not final - stop here + break; + } + + // Handle the error based on error mode + match errors { + "ignore" => { + pos += 1; + } + "replace" => { + wide_buf.push(0xFFFD); + pos += 1; + } + "backslashreplace" => { + let byte = data[pos]; + for ch in format!("\\x{byte:02x}").encode_utf16() { + wide_buf.push(ch); + } + pos += 1; + } + "surrogateescape" => { + let byte = data[pos]; + wide_buf.push(0xDC00 + byte as u16); + pos += 1; + } + "strict" => { + let object = vm.ctx.new_bytes(data.to_vec()); + return Err(vm.new_unicode_decode_error_real( + encoding_str, + object, + pos, + pos + 1, + reason_str, + )); + } + _ => { + // Custom error handler + let object = vm.ctx.new_bytes(data.to_vec()); + let exc = vm.new_unicode_decode_error_real( + encoding_str.clone(), + object, + pos, + pos + 1, + reason_str.clone(), + ); + let handler = error_handler.as_ref().unwrap(); + let res = handler.call((exc,), vm)?; + let tuple_err = || { + vm.new_type_error("decoding error handler must return (str, int) tuple") + }; + let tuple: &PyTuple = res.downcast_ref().ok_or_else(&tuple_err)?; + let tuple_slice = tuple.as_slice(); + if tuple_slice.len() != 2 { + return Err(tuple_err()); + } + + let replacement: PyStrRef = tuple_slice[0] + .clone() + .try_into_value(vm) + .map_err(|_| tuple_err())?; + let new_pos: isize = tuple_slice[1] + .clone() + .try_into_value(vm) + .map_err(|_| tuple_err())?; + + for cp in replacement.as_wtf8().code_points() { + let u = cp.to_u32(); + if u < 0x10000 { + wide_buf.push(u as u16); + } else { + wide_buf.push(((u - 0x10000) >> 10) as u16 + 0xD800); + wide_buf.push(((u - 0x10000) & 0x3FF) as u16 + 0xDC00); + } + } + + pos = if new_pos < 0 { + (len as isize + new_pos).max(0) as usize + } else { + new_pos as usize + }; + } + } + } else { + // Successfully decoded + wide_buf.extend_from_slice(&buffer[..outsize as usize]); + pos += in_size; + } + } + + let s = Wtf8Buf::from_wide(&wide_buf); + Ok((vm.ctx.new_str(s), pos)) } + #[pyfunction] - fn utf_16_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_ex_decode, args, vm) + fn code_page_decode( + args: CodePageDecodeArgs, + vm: &VirtualMachine, + ) -> PyResult<(PyStrRef, usize)> { + use crate::common::wtf8::Wtf8Buf; + + if args.code_page < 0 { + return Err(vm.new_value_error("invalid code page number".to_owned())); + } + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let code_page = args.code_page as u32; + let data = args.data.borrow_buf(); + let is_final = args.r#final; + + if data.is_empty() { + return Ok((vm.ctx.empty_str.to_owned(), 0)); + } + + let encoding_name = code_page_encoding_name(code_page); + + // Fast path: try to decode the whole buffer with strict flags + match try_decode_code_page_strict(code_page, &data, vm)? { + Some(wide) => { + let s = Wtf8Buf::from_wide(&wide); + return Ok((vm.ctx.new_str(s), data.len())); + } + None => { + // Decode error - fall through to slow path + } + } + + // Slow path: byte by byte with error handling + decode_code_page_errors(code_page, &data, errors, is_final, &encoding_name, vm) } - // TODO: utf-32 functions } diff --git a/crates/vm/src/stdlib/collections.rs b/crates/vm/src/stdlib/collections.rs index 32596b65386..0f84db80e74 100644 --- a/crates/vm/src/stdlib/collections.rs +++ b/crates/vm/src/stdlib/collections.rs @@ -1,4 +1,4 @@ -pub(crate) use _collections::make_module; +pub(crate) use _collections::module_def; #[pymodule] mod _collections { @@ -12,19 +12,19 @@ mod _collections { common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, function::{KwArgs, OptionalArg, PyComparisonValue}, iter::PyExactSizeIterator, - protocol::{PyIterReturn, PySequenceMethods}, + protocol::{PyIterReturn, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{MutObjectSequenceOp, OptionalRangeArgs}, sliceable::SequenceIndexOp, types::{ - AsSequence, Comparable, Constructor, DefaultConstructor, Initializer, IterNext, - Iterable, PyComparisonOp, Representable, SelfIter, + AsNumber, AsSequence, Comparable, Constructor, DefaultConstructor, Initializer, + IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, }; + use alloc::collections::VecDeque; + use core::cmp::max; use crossbeam_utils::atomic::AtomicCell; - use std::cmp::max; - use std::collections::VecDeque; #[pyattr] #[pyclass(module = "collections", name = "deque", unhashable = true)] @@ -60,6 +60,7 @@ mod _collections { with( Constructor, Initializer, + AsNumber, AsSequence, Comparable, Iterable, @@ -157,7 +158,7 @@ mod _collections { let mut created = VecDeque::from(elements); let mut borrowed = self.borrow_deque_mut(); created.append(&mut borrowed); - std::mem::swap(&mut created, &mut borrowed); + core::mem::swap(&mut created, &mut borrowed); Ok(()) } @@ -277,7 +278,6 @@ mod _collections { self.maxlen } - #[pymethod] fn __getitem__(&self, idx: isize, vm: &VirtualMachine) -> PyResult { let deque = self.borrow_deque(); idx.wrapped_at(deque.len()) @@ -285,7 +285,6 @@ mod _collections { .ok_or_else(|| vm.new_index_error("deque index out of range")) } - #[pymethod] fn __setitem__(&self, idx: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let mut deque = self.borrow_deque_mut(); idx.wrapped_at(deque.len()) @@ -294,7 +293,6 @@ mod _collections { .ok_or_else(|| vm.new_index_error("deque index out of range")) } - #[pymethod] fn __delitem__(&self, idx: isize, vm: &VirtualMachine) -> PyResult<()> { let mut deque = self.borrow_deque_mut(); idx.wrapped_at(deque.len()) @@ -302,7 +300,6 @@ mod _collections { .ok_or_else(|| vm.new_index_error("deque index out of range")) } - #[pymethod] fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { self._contains(&needle, vm) } @@ -331,8 +328,6 @@ mod _collections { Ok(deque) } - #[pymethod] - #[pymethod(name = "__rmul__")] fn __mul__(&self, n: isize, vm: &VirtualMachine) -> PyResult<Self> { let deque = self._mul(n, vm)?; Ok(Self { @@ -342,28 +337,16 @@ mod _collections { }) } - #[pymethod] fn __imul__(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { let mul_deque = zelf._mul(n, vm)?; *zelf.borrow_deque_mut() = mul_deque; Ok(zelf) } - #[pymethod] fn __len__(&self) -> usize { self.borrow_deque().len() } - #[pymethod] - fn __bool__(&self) -> bool { - !self.borrow_deque().is_empty() - } - - #[pymethod] - fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { - self.concat(&other, vm) - } - fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<Self> { if let Some(o) = other.downcast_ref::<Self>() { let mut deque = self.borrow_deque().clone(); @@ -389,7 +372,6 @@ mod _collections { } } - #[pymethod] fn __iadd__( zelf: PyRef<Self>, other: PyObjectRef, @@ -422,11 +404,11 @@ mod _collections { impl MutObjectSequenceOp for PyDeque { type Inner = VecDeque<PyObjectRef>; - fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObjectRef> { - inner.get(index) + fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObject> { + inner.get(index).map(|r| r.as_ref()) } - fn do_lock(&self) -> impl std::ops::Deref<Target = Self::Inner> { + fn do_lock(&self) -> impl core::ops::Deref<Target = Self::Inner> { self.borrow_deque() } } @@ -484,7 +466,7 @@ mod _collections { // `maxlen` is better to be defined as UnsafeCell in common practice, // but then more type works without any safety benefits let unsafe_maxlen = - &zelf.maxlen as *const _ as *const std::cell::UnsafeCell<Option<usize>>; + &zelf.maxlen as *const _ as *const core::cell::UnsafeCell<Option<usize>>; *(*unsafe_maxlen).get() = maxlen; } if let Some(elements) = elements { @@ -496,6 +478,19 @@ mod _collections { } } + impl AsNumber for PyDeque { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyDeque>().unwrap(); + Ok(!zelf.borrow_deque().is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + impl AsSequence for PyDeque { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index 70aee7378d3..441a5ce37e4 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -1,77 +1,358 @@ // spell-checker:disable -pub(crate) mod array; -pub(crate) mod base; -pub(crate) mod field; -pub(crate) mod function; -pub(crate) mod library; -pub(crate) mod pointer; -pub(crate) mod structure; -pub(crate) mod thunk; -pub(crate) mod union; -pub(crate) mod util; - -use crate::builtins::PyModule; -use crate::class::PyClassImpl; -use crate::{Py, PyRef, VirtualMachine}; - -pub use crate::stdlib::ctypes::base::{CDataObject, PyCData, PyCSimple, PyCSimpleType}; - -pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { - let ctx = &vm.ctx; - PyCSimpleType::make_class(ctx); - array::PyCArrayType::make_class(ctx); - field::PyCFieldType::make_class(ctx); - pointer::PyCPointerType::make_class(ctx); - structure::PyCStructType::make_class(ctx); - union::PyCUnionType::make_class(ctx); - extend_module!(vm, module, { - "_CData" => PyCData::make_class(ctx), - "_SimpleCData" => PyCSimple::make_class(ctx), - "Array" => array::PyCArray::make_class(ctx), - "CField" => field::PyCField::make_class(ctx), - "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), - "_Pointer" => pointer::PyCPointer::make_class(ctx), - "_pointer_type_cache" => ctx.new_dict(), - "Structure" => structure::PyCStructure::make_class(ctx), - "CThunkObject" => thunk::PyCThunk::make_class(ctx), - "Union" => union::PyCUnion::make_class(ctx), - }) +mod array; +mod base; +mod function; +mod library; +mod pointer; +mod simple; +mod structure; +mod union; + +use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStr, PyType}, + class::PyClassImpl, + types::TypeDataRef, +}; +use core::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, + c_ulonglong, c_ushort, +}; +use core::mem; +use widestring::WideChar; + +pub use array::PyCArray; +pub use base::{FfiArgValue, PyCData, PyCField, StgInfo, StgInfoFlags}; +pub use pointer::PyCPointer; +pub use simple::{PyCSimple, PyCSimpleType}; +pub use structure::PyCStructure; +pub use union::PyCUnion; + +/// Extension for PyType to get StgInfo +/// PyStgInfo_FromType +impl Py<PyType> { + /// Get StgInfo from a ctypes type object + /// + /// Returns a TypeDataRef to StgInfo if the type has one and is initialized, error otherwise. + /// Abstract classes (whose metaclass __init__ was not called) will have uninitialized StgInfo. + fn stg_info<'a>(&'a self, vm: &VirtualMachine) -> PyResult<TypeDataRef<'a, StgInfo>> { + self.stg_info_opt() + .ok_or_else(|| vm.new_type_error("abstract class")) + } + + /// Get StgInfo if initialized, None otherwise. + fn stg_info_opt(&self) -> Option<TypeDataRef<'_, StgInfo>> { + self.get_type_data::<StgInfo>() + .filter(|info| info.initialized) + } + + /// Get _type_ attribute as String (type code like "i", "d", etc.) + fn type_code(&self, vm: &VirtualMachine) -> Option<String> { + self.as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t: PyObjectRef| t.downcast_ref::<PyStr>().map(|s| s.to_string())) + } + + /// Mark all base classes as finalized + fn mark_bases_final(&self) { + for base in self.bases.read().iter() { + if let Some(mut stg) = base.get_type_data_mut::<StgInfo>() { + stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + let mut stg = StgInfo::default(); + stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = base.init_type_data(stg); + } + } + } +} + +impl PyType { + /// Check if StgInfo is already initialized. + /// Raises SystemError if already initialized. + pub(crate) fn check_not_initialized(&self, vm: &VirtualMachine) -> PyResult<()> { + if let Some(stg_info) = self.get_type_data::<StgInfo>() + && stg_info.initialized + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + format!("class \"{}\" already initialized", self.name()), + )); + } + Ok(()) + } + + /// Check if StgInfo is already initialized, returning true if so. + /// Unlike check_not_initialized, does not raise an error. + pub(crate) fn is_initialized(&self) -> bool { + self.get_type_data::<StgInfo>() + .is_some_and(|stg_info| stg_info.initialized) + } } -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _ctypes::make_module(vm); - extend_module_nodes(vm, &module); - module +// Dynamic type check helpers for PyCData +pub(crate) use _ctypes::module_def; + +// These check if an object's type's metaclass is a subclass of a specific metaclass + +/// Size of long double - platform dependent +/// x86_64 macOS/Linux: 16 bytes (80-bit extended + padding) +/// ARM64: 16 bytes (128-bit) +/// Windows: 8 bytes (same as double) +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + not(target_os = "windows") +))] +const LONG_DOUBLE_SIZE: usize = 16; + +#[cfg(target_os = "windows")] +const LONG_DOUBLE_SIZE: usize = mem::size_of::<c_double>(); + +#[cfg(not(any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + not(target_os = "windows") + ), + target_os = "windows" +)))] +const LONG_DOUBLE_SIZE: usize = mem::size_of::<c_double>(); + +/// Type information for ctypes simple types +struct TypeInfo { + pub size: usize, + pub ffi_type_fn: fn() -> libffi::middle::Type, +} + +/// Get type information (size and ffi_type) for a ctypes type code +fn type_info(ty: &str) -> Option<TypeInfo> { + use libffi::middle::Type; + match ty { + "c" => Some(TypeInfo { + size: mem::size_of::<c_schar>(), + ffi_type_fn: Type::u8, + }), + "u" => Some(TypeInfo { + size: mem::size_of::<WideChar>(), + ffi_type_fn: if mem::size_of::<WideChar>() == 2 { + Type::u16 + } else { + Type::u32 + }, + }), + "b" => Some(TypeInfo { + size: mem::size_of::<c_schar>(), + ffi_type_fn: Type::i8, + }), + "B" => Some(TypeInfo { + size: mem::size_of::<c_uchar>(), + ffi_type_fn: Type::u8, + }), + "h" | "v" => Some(TypeInfo { + size: mem::size_of::<c_short>(), + ffi_type_fn: Type::i16, + }), + "H" => Some(TypeInfo { + size: mem::size_of::<c_ushort>(), + ffi_type_fn: Type::u16, + }), + "i" => Some(TypeInfo { + size: mem::size_of::<c_int>(), + ffi_type_fn: Type::i32, + }), + "I" => Some(TypeInfo { + size: mem::size_of::<c_uint>(), + ffi_type_fn: Type::u32, + }), + "l" => Some(TypeInfo { + size: mem::size_of::<c_long>(), + ffi_type_fn: if mem::size_of::<c_long>() == 8 { + Type::i64 + } else { + Type::i32 + }, + }), + "L" => Some(TypeInfo { + size: mem::size_of::<c_ulong>(), + ffi_type_fn: if mem::size_of::<c_ulong>() == 8 { + Type::u64 + } else { + Type::u32 + }, + }), + "q" => Some(TypeInfo { + size: mem::size_of::<c_longlong>(), + ffi_type_fn: Type::i64, + }), + "Q" => Some(TypeInfo { + size: mem::size_of::<c_ulonglong>(), + ffi_type_fn: Type::u64, + }), + "f" => Some(TypeInfo { + size: mem::size_of::<c_float>(), + ffi_type_fn: Type::f32, + }), + "d" => Some(TypeInfo { + size: mem::size_of::<c_double>(), + ffi_type_fn: Type::f64, + }), + "g" => Some(TypeInfo { + // long double - platform dependent size + // x86_64 macOS/Linux: 16 bytes (80-bit extended + padding) + // ARM64: 16 bytes (128-bit) + // Windows: 8 bytes (same as double) + // Note: Use f64 as FFI type since Rust doesn't support long double natively + size: LONG_DOUBLE_SIZE, + ffi_type_fn: Type::f64, + }), + "?" => Some(TypeInfo { + size: mem::size_of::<c_uchar>(), + ffi_type_fn: Type::u8, + }), + "z" | "Z" | "P" | "X" | "O" => Some(TypeInfo { + size: mem::size_of::<usize>(), + ffi_type_fn: Type::pointer, + }), + "void" => Some(TypeInfo { + size: 0, + ffi_type_fn: Type::void, + }), + _ => None, + } +} + +/// Get size for a ctypes type code +fn get_size(ty: &str) -> usize { + type_info(ty).map(|t| t.size).expect("invalid type code") +} + +/// Get alignment for simple type codes from type_info(). +/// For primitive C types (c_int, c_long, etc.), alignment equals size. +fn get_align(ty: &str) -> usize { + get_size(ty) } #[pymodule] pub(crate) mod _ctypes { - use super::base::{CDataObject, PyCData, PyCSimple}; - use crate::builtins::PyTypeRef; + use super::library; + use super::{PyCArray, PyCData, PyCPointer, PyCSimple, PyCStructure, PyCUnion}; + use crate::builtins::{PyType, PyTypeRef}; use crate::class::StaticType; use crate::convert::ToPyObject; - use crate::function::{Either, FuncArgs, OptionalArg}; - use crate::stdlib::ctypes::library; - use crate::{AsObject, PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use crossbeam_utils::atomic::AtomicCell; - use std::ffi::{ - c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, - c_ulonglong, c_ushort, - }; - use std::mem; - use widestring::WideChar; - - /// CArgObject - returned by byref() + use crate::function::{Either, OptionalArg}; + use crate::types::Representable; + use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use num_traits::ToPrimitive; + + /// CArgObject - returned by byref() and paramfunc + /// tagPyCArgObject #[pyclass(name = "CArgObject", module = "_ctypes", no_attr)] #[derive(Debug, PyPayload)] pub struct CArgObject { + /// Type tag ('P', 'V', 'i', 'd', etc.) + pub tag: u8, + /// The actual FFI value (mirrors union value) + pub value: super::FfiArgValue, + /// Reference to original object (for memory safety) pub obj: PyObjectRef, + /// Size for struct/union ('V' tag) #[allow(dead_code)] + pub size: usize, + /// Offset for byref() pub offset: isize, } - #[pyclass] + /// is_literal_char - check if character is printable literal (not \\ or ') + fn is_literal_char(c: u8) -> bool { + c < 128 && c.is_ascii_graphic() && c != b'\\' && c != b'\'' + } + + impl Representable for CArgObject { + // PyCArg_repr - use tag and value fields directly + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + use super::base::FfiArgValue; + + let tag_char = zelf.tag as char; + + // Format value based on tag + match zelf.tag { + b'b' | b'h' | b'i' | b'l' | b'q' => { + // Signed integers + let n = match zelf.value { + FfiArgValue::I8(v) => v as i64, + FfiArgValue::I16(v) => v as i64, + FfiArgValue::I32(v) => v as i64, + FfiArgValue::I64(v) => v, + _ => 0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, n)) + } + b'B' | b'H' | b'I' | b'L' | b'Q' => { + // Unsigned integers + let n = match zelf.value { + FfiArgValue::U8(v) => v as u64, + FfiArgValue::U16(v) => v as u64, + FfiArgValue::U32(v) => v as u64, + FfiArgValue::U64(v) => v, + _ => 0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, n)) + } + b'f' => { + let v = match zelf.value { + FfiArgValue::F32(v) => v as f64, + _ => 0.0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, v)) + } + b'd' | b'g' => { + let v = match zelf.value { + FfiArgValue::F64(v) => v, + FfiArgValue::F32(v) => v as f64, + _ => 0.0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, v)) + } + b'c' => { + // c_char - single byte + let byte = match zelf.value { + FfiArgValue::I8(v) => v as u8, + FfiArgValue::U8(v) => v, + _ => 0, + }; + if is_literal_char(byte) { + Ok(format!("<cparam '{}' ('{}')>", tag_char, byte as char)) + } else { + Ok(format!("<cparam '{}' ('\\x{:02x}')>", tag_char, byte)) + } + } + b'z' | b'Z' | b'P' | b'V' => { + // Pointer types + let ptr = match zelf.value { + FfiArgValue::Pointer(v) => v, + _ => 0, + }; + if ptr == 0 { + Ok(format!("<cparam '{}' (nil)>", tag_char)) + } else { + Ok(format!("<cparam '{}' ({:#x})>", tag_char, ptr)) + } + } + _ => { + // Default fallback + let addr = zelf.get_id(); + if is_literal_char(zelf.tag) { + Ok(format!("<cparam '{}' at {:#x}>", tag_char, addr)) + } else { + Ok(format!("<cparam {:#04x} at {:#x}>", zelf.tag, addr)) + } + } + } + } + } + + #[pyclass(with(Representable))] impl CArgObject { #[pygetset] fn _obj(&self) -> PyObjectRef { @@ -83,43 +364,39 @@ pub(crate) mod _ctypes { const __VERSION__: &str = "1.1.0"; // TODO: get properly - #[pyattr(name = "RTLD_LOCAL")] + #[pyattr] const RTLD_LOCAL: i32 = 0; // TODO: get properly - #[pyattr(name = "RTLD_GLOBAL")] + #[pyattr] const RTLD_GLOBAL: i32 = 0; - #[cfg(target_os = "windows")] - #[pyattr(name = "SIZEOF_TIME_T")] - pub const SIZEOF_TIME_T: usize = 8; - #[cfg(not(target_os = "windows"))] - #[pyattr(name = "SIZEOF_TIME_T")] - pub const SIZEOF_TIME_T: usize = 4; + #[pyattr] + const SIZEOF_TIME_T: usize = core::mem::size_of::<libc::time_t>(); - #[pyattr(name = "CTYPES_MAX_ARGCOUNT")] - pub const CTYPES_MAX_ARGCOUNT: usize = 1024; + #[pyattr] + const CTYPES_MAX_ARGCOUNT: usize = 1024; #[pyattr] - pub const FUNCFLAG_STDCALL: u32 = 0x0; + const FUNCFLAG_STDCALL: u32 = 0x0; #[pyattr] - pub const FUNCFLAG_CDECL: u32 = 0x1; + const FUNCFLAG_CDECL: u32 = 0x1; #[pyattr] - pub const FUNCFLAG_HRESULT: u32 = 0x2; + const FUNCFLAG_HRESULT: u32 = 0x2; #[pyattr] - pub const FUNCFLAG_PYTHONAPI: u32 = 0x4; + const FUNCFLAG_PYTHONAPI: u32 = 0x4; #[pyattr] - pub const FUNCFLAG_USE_ERRNO: u32 = 0x8; + const FUNCFLAG_USE_ERRNO: u32 = 0x8; #[pyattr] - pub const FUNCFLAG_USE_LASTERROR: u32 = 0x10; + const FUNCFLAG_USE_LASTERROR: u32 = 0x10; #[pyattr] - pub const TYPEFLAG_ISPOINTER: u32 = 0x100; + const TYPEFLAG_ISPOINTER: u32 = 0x100; #[pyattr] - pub const TYPEFLAG_HASPOINTER: u32 = 0x200; + const TYPEFLAG_HASPOINTER: u32 = 0x200; #[pyattr] - pub const DICTFLAG_FINAL: u32 = 0x1000; + const DICTFLAG_FINAL: u32 = 0x1000; #[pyattr(name = "ArgumentError", once)] fn argument_error(vm: &VirtualMachine) -> PyTypeRef { @@ -130,369 +407,138 @@ pub(crate) mod _ctypes { ) } - #[pyattr(name = "FormatError", once)] - fn format_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "_ctypes", - "FormatError", - Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), - ) - } - - pub fn get_size(ty: &str) -> usize { - match ty { - "u" => mem::size_of::<WideChar>(), - "c" | "b" => mem::size_of::<c_schar>(), - "h" => mem::size_of::<c_short>(), - "H" => mem::size_of::<c_short>(), - "i" => mem::size_of::<c_int>(), - "I" => mem::size_of::<c_uint>(), - "l" => mem::size_of::<c_long>(), - "q" => mem::size_of::<c_longlong>(), - "L" => mem::size_of::<c_ulong>(), - "Q" => mem::size_of::<c_ulonglong>(), - "f" => mem::size_of::<c_float>(), - "d" | "g" => mem::size_of::<c_double>(), - "?" | "B" => mem::size_of::<c_uchar>(), - "P" | "z" | "Z" => mem::size_of::<usize>(), - "O" => mem::size_of::<PyObjectRef>(), - _ => unreachable!(), - } - } - - /// Get alignment for a simple type - for C types, alignment equals size - pub fn get_align(ty: &str) -> usize { - get_size(ty) - } - - /// Get the size of a ctypes type from its type object - #[allow(dead_code)] - pub fn get_size_from_type(cls: &PyTypeRef, vm: &VirtualMachine) -> PyResult<usize> { - // Try to get _type_ attribute for simple types - if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) - && let Ok(s) = type_attr.str(vm) - { - let s = s.to_string(); - if s.len() == 1 && SIMPLE_TYPE_CHARS.contains(s.as_str()) { - return Ok(get_size(&s)); - } + #[cfg(target_os = "windows")] + #[pyattr(name = "COMError", once)] + fn com_error(vm: &VirtualMachine) -> PyTypeRef { + use crate::builtins::type_::PyAttributes; + use crate::function::FuncArgs; + use crate::types::{PyTypeFlags, PyTypeSlots}; + + // Sets hresult, text, details as instance attributes in __init__ + // This function has InitFunc signature for direct slots.init use + fn comerror_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let (hresult, text, details): ( + Option<PyObjectRef>, + Option<PyObjectRef>, + Option<PyObjectRef>, + ) = args.bind(vm)?; + let hresult = hresult.unwrap_or_else(|| vm.ctx.none()); + let text = text.unwrap_or_else(|| vm.ctx.none()); + let details = details.unwrap_or_else(|| vm.ctx.none()); + + // Set instance attributes + zelf.set_attr("hresult", hresult.clone(), vm)?; + zelf.set_attr("text", text.clone(), vm)?; + zelf.set_attr("details", details.clone(), vm)?; + + // self.args = args[1:] = (text, details) + // via: PyObject_SetAttrString(self, "args", PySequence_GetSlice(args, 1, size)) + let args_tuple: PyObjectRef = vm.ctx.new_tuple(vec![text, details]).into(); + zelf.set_attr("args", args_tuple, vm)?; + + Ok(()) } - // Fall back to sizeof - size_of(cls.clone().into(), vm) - } - /// Convert bytes to appropriate Python object based on ctypes type - pub fn bytes_to_pyobject( - cls: &PyTypeRef, - bytes: &[u8], - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - // Try to get _type_ attribute - if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) - && let Ok(s) = type_attr.str(vm) - { - let ty = s.to_string(); - return match ty.as_str() { - "c" => { - // c_char - single byte - Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) - } - "b" => { - // c_byte - signed char - let val = if !bytes.is_empty() { bytes[0] as i8 } else { 0 }; - Ok(vm.ctx.new_int(val).into()) - } - "B" => { - // c_ubyte - unsigned char - let val = if !bytes.is_empty() { bytes[0] } else { 0 }; - Ok(vm.ctx.new_int(val).into()) - } - "h" => { - // c_short - const SIZE: usize = mem::size_of::<c_short>(); - let val = if bytes.len() >= SIZE { - c_short::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "H" => { - // c_ushort - const SIZE: usize = mem::size_of::<c_ushort>(); - let val = if bytes.len() >= SIZE { - c_ushort::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "i" => { - // c_int - const SIZE: usize = mem::size_of::<c_int>(); - let val = if bytes.len() >= SIZE { - c_int::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "I" => { - // c_uint - const SIZE: usize = mem::size_of::<c_uint>(); - let val = if bytes.len() >= SIZE { - c_uint::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "l" => { - // c_long - const SIZE: usize = mem::size_of::<c_long>(); - let val = if bytes.len() >= SIZE { - c_long::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "L" => { - // c_ulong - const SIZE: usize = mem::size_of::<c_ulong>(); - let val = if bytes.len() >= SIZE { - c_ulong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "q" => { - // c_longlong - const SIZE: usize = mem::size_of::<c_longlong>(); - let val = if bytes.len() >= SIZE { - c_longlong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "Q" => { - // c_ulonglong - const SIZE: usize = mem::size_of::<c_ulonglong>(); - let val = if bytes.len() >= SIZE { - c_ulonglong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "f" => { - // c_float - const SIZE: usize = mem::size_of::<c_float>(); - let val = if bytes.len() >= SIZE { - c_float::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0.0 - }; - Ok(vm.ctx.new_float(val as f64).into()) - } - "d" | "g" => { - // c_double - const SIZE: usize = mem::size_of::<c_double>(); - let val = if bytes.len() >= SIZE { - c_double::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) - } else { - 0.0 - }; - Ok(vm.ctx.new_float(val).into()) - } - "?" => { - // c_bool - let val = !bytes.is_empty() && bytes[0] != 0; - Ok(vm.ctx.new_bool(val).into()) - } - "P" | "z" | "Z" => { - // Pointer types - return as integer address - let val = if bytes.len() >= mem::size_of::<libc::uintptr_t>() { - const UINTPTR_LEN: usize = mem::size_of::<libc::uintptr_t>(); - let mut arr = [0u8; UINTPTR_LEN]; - arr[..bytes.len().min(UINTPTR_LEN)] - .copy_from_slice(&bytes[..bytes.len().min(UINTPTR_LEN)]); - usize::from_ne_bytes(arr) - } else { - 0 - }; - Ok(vm.ctx.new_int(val).into()) - } - "u" => { - // c_wchar - wide character - let val = if bytes.len() >= mem::size_of::<WideChar>() { - let wc = if mem::size_of::<WideChar>() == 2 { - u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 - } else { - u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) - }; - char::from_u32(wc).unwrap_or('\0') - } else { - '\0' - }; - Ok(vm.ctx.new_str(val.to_string()).into()) - } - _ => Ok(vm.ctx.none()), - }; - } - // Default: return bytes as-is - Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) - } + // Create exception type with IMMUTABLETYPE flag + let mut attrs = PyAttributes::default(); + attrs.insert( + vm.ctx.intern_str("__module__"), + vm.ctx.new_str("_ctypes").into(), + ); + attrs.insert( + vm.ctx.intern_str("__doc__"), + vm.ctx + .new_str("Raised when a COM method call failed.") + .into(), + ); + + // Create slots with IMMUTABLETYPE flag + let slots = PyTypeSlots { + name: "COMError", + flags: PyTypeFlags::heap_type_flags() + | PyTypeFlags::HAS_DICT + | PyTypeFlags::IMMUTABLETYPE, + ..PyTypeSlots::default() + }; - const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfguzZPqQ?O"; + let exc_type = PyType::new_heap( + "COMError", + vec![vm.ctx.exceptions.exception_type.to_owned()], + attrs, + slots, + vm.ctx.types.type_type.to_owned(), + &vm.ctx, + ) + .unwrap(); - pub fn new_simple_type( - cls: Either<&PyObjectRef, &PyTypeRef>, - vm: &VirtualMachine, - ) -> PyResult<PyCSimple> { - let cls = match cls { - Either::A(obj) => obj, - Either::B(typ) => typ.as_object(), - }; + // Set our custom init after new_heap, which runs init_slots that would + // otherwise overwrite slots.init with init_wrapper (due to __init__ in MRO). + exc_type.slots.init.store(Some(comerror_init)); - if let Ok(_type_) = cls.get_attr("_type_", vm) { - if _type_.is_instance((&vm.ctx.types.str_type).as_ref(), vm)? { - let tp_str = _type_.str(vm)?.to_string(); - - if tp_str.len() != 1 { - Err(vm.new_value_error( - format!("class must define a '_type_' attribute which must be a string of length 1, str: {tp_str}"), - )) - } else if !SIMPLE_TYPE_CHARS.contains(tp_str.as_str()) { - Err(vm.new_attribute_error(format!("class must define a '_type_' attribute which must be\n a single character string containing one of {SIMPLE_TYPE_CHARS}, currently it is {tp_str}."))) - } else { - let size = get_size(&tp_str); - let cdata = CDataObject::from_bytes(vec![0u8; size], None); - Ok(PyCSimple { - _base: PyCData::new(cdata.clone()), - _type_: tp_str, - value: AtomicCell::new(vm.ctx.none()), - cdata: rustpython_common::lock::PyRwLock::new(cdata), - }) - } - } else { - Err(vm.new_type_error("class must define a '_type_' string attribute")) - } - } else { - Err(vm.new_attribute_error("class must define a '_type_' attribute")) - } + exc_type } /// Get the size of a ctypes type or instance - #[pyfunction(name = "sizeof")] - pub fn size_of(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - use super::pointer::PyCPointer; - use super::structure::{PyCStructType, PyCStructure}; + #[pyfunction] + pub fn sizeof(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + use super::structure::PyCStructType; use super::union::PyCUnionType; - use super::util::StgInfo; - use crate::builtins::PyType; - - // 1. Check TypeDataSlot on class (for instances) - if let Some(stg_info) = obj.class().get_type_data::<StgInfo>() { - return Ok(stg_info.size); - } - // 2. Check TypeDataSlot on type itself (for type objects) - if let Some(type_obj) = obj.downcast_ref::<PyType>() - && let Some(stg_info) = type_obj.get_type_data::<StgInfo>() - { - return Ok(stg_info.size); - } - - // 3. Instances with cdata buffer - if let Some(structure) = obj.downcast_ref::<PyCStructure>() { - return Ok(structure.cdata.read().size()); - } - if let Some(simple) = obj.downcast_ref::<PyCSimple>() { - return Ok(simple.cdata.read().size()); - } - if obj.fast_isinstance(PyCPointer::static_type()) { - return Ok(std::mem::size_of::<usize>()); - } - - // 3. Type objects - if let Ok(type_ref) = obj.clone().downcast::<crate::builtins::PyType>() { - // Structure types - check if metaclass is or inherits from PyCStructType - if type_ref + // 1. Check if obj is a TYPE object (not instance) - PyStgInfo_FromType + if let Some(type_obj) = obj.downcast_ref::<PyType>() { + // Type object - return StgInfo.size + if let Some(stg_info) = type_obj.stg_info_opt() { + return Ok(stg_info.size); + } + // Fallback for type objects without StgInfo + // Array types + if type_obj + .class() + .fast_issubclass(super::array::PyCArrayType::static_type()) + && let Ok(stg) = type_obj.stg_info(vm) + { + return Ok(stg.size); + } + // Structure types + if type_obj .class() .fast_issubclass(PyCStructType::static_type()) { - return calculate_struct_size(&type_ref, vm); + return super::structure::calculate_struct_size(type_obj, vm); } - // Union types - check if metaclass is or inherits from PyCUnionType - if type_ref + // Union types + if type_obj .class() .fast_issubclass(PyCUnionType::static_type()) { - return calculate_union_size(&type_ref, vm); + return super::union::calculate_union_size(type_obj, vm); } - // Simple types (c_int, c_char, etc.) - if type_ref.fast_issubclass(PyCSimple::static_type()) { - let instance = new_simple_type(Either::B(&type_ref), vm)?; - return Ok(get_size(&instance._type_)); + // Simple types + if type_obj.fast_issubclass(PyCSimple::static_type()) { + if let Ok(type_attr) = type_obj.as_object().get_attr("_type_", vm) + && let Ok(type_str) = type_attr.str(vm) + { + return Ok(super::get_size(type_str.as_ref())); + } + return Ok(core::mem::size_of::<usize>()); } // Pointer types - if type_ref.fast_issubclass(PyCPointer::static_type()) { - return Ok(std::mem::size_of::<usize>()); + if type_obj.fast_issubclass(PyCPointer::static_type()) { + return Ok(core::mem::size_of::<usize>()); } + return Err(vm.new_type_error("this type has no size")); } - Err(vm.new_type_error("this type has no size")) - } - - /// Calculate Structure type size from _fields_ (sum of field sizes) - fn calculate_struct_size( - cls: &crate::builtins::PyTypeRef, - vm: &VirtualMachine, - ) -> PyResult<usize> { - use crate::AsObject; - - if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { - let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm).unwrap_or_default(); - let mut total_size = 0usize; - - for field in fields.iter() { - if let Some(tuple) = field.downcast_ref::<crate::builtins::PyTuple>() - && let Some(field_type) = tuple.get(1) - { - // Recursively calculate field type size - total_size += size_of(field_type.clone(), vm)?; - } - } - return Ok(total_size); + // 2. Instance object - return actual buffer size (b_size) + // CDataObject_Check + return obj->b_size + if let Some(cdata) = obj.downcast_ref::<PyCData>() { + return Ok(cdata.size()); } - Ok(0) - } - - /// Calculate Union type size from _fields_ (max field size) - fn calculate_union_size( - cls: &crate::builtins::PyTypeRef, - vm: &VirtualMachine, - ) -> PyResult<usize> { - use crate::AsObject; - - if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { - let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm).unwrap_or_default(); - let mut max_size = 0usize; - - for field in fields.iter() { - if let Some(tuple) = field.downcast_ref::<crate::builtins::PyTuple>() - && let Some(field_type) = tuple.get(1) - { - let field_size = size_of(field_type.clone(), vm)?; - max_size = max_size.max(field_size); - } - } - return Ok(max_size); + if obj.fast_isinstance(PyCPointer::static_type()) { + return Ok(core::mem::size_of::<usize>()); } - Ok(0) + + Err(vm.new_type_error("this type has no size")) } #[cfg(windows)] @@ -513,28 +559,47 @@ pub(crate) mod _ctypes { #[cfg(not(windows))] #[pyfunction(name = "dlopen")] fn load_library_unix( - name: Option<String>, - _load_flags: OptionalArg<i32>, + name: Option<crate::function::FsPath>, + load_flags: OptionalArg<i32>, vm: &VirtualMachine, ) -> PyResult<usize> { - // TODO: audit functions first - // TODO: load_flags + // Default mode: RTLD_NOW | RTLD_LOCAL, always force RTLD_NOW + let mode = load_flags.unwrap_or(libc::RTLD_NOW | libc::RTLD_LOCAL) | libc::RTLD_NOW; + match name { Some(name) => { let cache = library::libcache(); let mut cache_write = cache.write(); + let os_str = name.as_os_str(vm)?; let (id, _) = cache_write - .get_or_insert_lib(&name, vm) - .map_err(|e| vm.new_os_error(e.to_string()))?; + .get_or_insert_lib_with_mode(&*os_str, mode, vm) + .map_err(|e| { + let name_str = os_str.to_string_lossy(); + vm.new_os_error(format!("{}: {}", name_str, e)) + })?; Ok(id) } None => { - // If None, call libc::dlopen(null, mode) to get the current process handle - let handle = unsafe { libc::dlopen(std::ptr::null(), libc::RTLD_NOW) }; + // dlopen(NULL, mode) to get the current process handle (for pythonapi) + let handle = unsafe { libc::dlopen(core::ptr::null(), mode) }; if handle.is_null() { - return Err(vm.new_os_error("dlopen() error")); + let err = unsafe { libc::dlerror() }; + let msg = if err.is_null() { + "dlopen() error".to_string() + } else { + unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + } + }; + return Err(vm.new_os_error(msg)); } - Ok(handle as usize) + // Add to library cache so symbol lookup works + let cache = library::libcache(); + let mut cache_write = cache.write(); + let id = cache_write.insert_raw_handle(handle); + Ok(id) } } } @@ -547,8 +612,56 @@ pub(crate) mod _ctypes { Ok(()) } + #[cfg(not(windows))] + #[pyfunction] + fn dlclose(handle: usize, _vm: &VirtualMachine) -> PyResult<()> { + // Remove from cache, which triggers SharedLibrary drop. + // libloading::Library calls dlclose automatically on Drop. + let cache = library::libcache(); + let mut cache_write = cache.write(); + cache_write.drop_lib(handle); + Ok(()) + } + + #[cfg(not(windows))] + #[pyfunction] + fn dlsym( + handle: usize, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let symbol_name = alloc::ffi::CString::new(name.as_str()) + .map_err(|_| vm.new_value_error("symbol name contains null byte"))?; + + // Clear previous error + unsafe { libc::dlerror() }; + + let ptr = unsafe { libc::dlsym(handle as *mut libc::c_void, symbol_name.as_ptr()) }; + + // Check for error via dlerror first + let err = unsafe { libc::dlerror() }; + if !err.is_null() { + let msg = unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + }; + return Err(vm.new_os_error(msg)); + } + + // Treat NULL symbol address as error + // This handles cases like GNU IFUNCs that resolve to NULL + if ptr.is_null() { + return Err(vm.new_os_error(format!("symbol '{}' not found", name.as_str()))); + } + + Ok(ptr as usize) + } + #[pyfunction(name = "POINTER")] - pub fn create_pointer_type(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn create_pointer_type(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyStr; + // Get the _pointer_type_cache let ctypes_module = vm.import("_ctypes", 0)?; let cache = ctypes_module.get_attr("_pointer_type_cache", vm)?; @@ -563,33 +676,60 @@ pub(crate) mod _ctypes { // Get the _Pointer base class let pointer_base = ctypes_module.get_attr("_Pointer", vm)?; + // Create a new type that inherits from _Pointer + let pointer_base_type = pointer_base + .clone() + .downcast::<crate::builtins::PyType>() + .map_err(|_| vm.new_type_error("_Pointer must be a type"))?; + let metaclass = pointer_base_type.class().to_owned(); + + let bases = vm.ctx.new_tuple(vec![pointer_base]); + let dict = vm.ctx.new_dict(); + + // PyUnicode_CheckExact(cls) - string creates incomplete pointer type + if let Some(s) = cls.downcast_ref::<PyStr>() { + // Incomplete pointer type: _type_ not set, cache key is id(result) + let name = format!("LP_{}", s.as_str()); + + let new_type = metaclass + .as_object() + .call((vm.ctx.new_str(name), bases, dict), vm)?; + + // Store with id(result) as key for incomplete pointer types + let id_key: PyObjectRef = vm.ctx.new_int(new_type.get_id() as i64).into(); + vm.call_method(&cache, "__setitem__", (id_key, new_type.clone()))?; + + return Ok(new_type); + } + + // PyType_Check(cls) - type creates complete pointer type + if !cls.class().fast_issubclass(vm.ctx.types.type_type.as_ref()) { + return Err(vm.new_type_error("must be a ctypes type")); + } + // Create the name for the pointer type let name = if let Ok(type_obj) = cls.get_attr("__name__", vm) { format!("LP_{}", type_obj.str(vm)?) - } else if let Ok(s) = cls.str(vm) { - format!("LP_{}", s) } else { "LP_unknown".to_string() }; - // Create a new type that inherits from _Pointer - let type_type = &vm.ctx.types.type_type; - let bases = vm.ctx.new_tuple(vec![pointer_base]); - let dict = vm.ctx.new_dict(); + // Complete pointer type: set _type_ attribute dict.set_item("_type_", cls.clone(), vm)?; - let new_type = type_type + // Call the metaclass (PyCPointerType) to create the new type + let new_type = metaclass .as_object() .call((vm.ctx.new_str(name), bases, dict), vm)?; - // Store in cache using __setitem__ + // Store in cache with cls as key vm.call_method(&cache, "__setitem__", (cls, new_type.clone()))?; Ok(new_type) } - #[pyfunction(name = "pointer")] - pub fn create_pointer_inst(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pyfunction] + fn pointer(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { // Get the type of the object let obj_type = obj.class().to_owned(); @@ -607,7 +747,7 @@ pub(crate) mod _ctypes { #[cfg(target_os = "windows")] #[pyfunction(name = "_check_HRESULT")] - pub fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult<i32> { + fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult<i32> { // TODO: fixme if hr < 0 { // vm.ctx.new_windows_error(hr) @@ -619,18 +759,17 @@ pub(crate) mod _ctypes { #[pyfunction] fn addressof(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - if obj.is_instance(PyCSimple::static_type().as_ref(), vm)? { - let simple = obj.downcast_ref::<PyCSimple>().unwrap(); - Ok(simple.value.as_ptr() as usize) + // All ctypes objects should return cdata buffer pointer + if let Some(cdata) = obj.downcast_ref::<PyCData>() { + Ok(cdata.buffer.read().as_ptr() as usize) } else { Err(vm.new_type_error("expected a ctypes instance")) } } #[pyfunction] - fn byref(obj: PyObjectRef, offset: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { - use super::base::PyCData; - use crate::class::StaticType; + pub fn byref(obj: PyObjectRef, offset: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { + use super::FfiArgValue; // Check if obj is a ctypes instance if !obj.fast_isinstance(PyCData::static_type()) @@ -644,9 +783,23 @@ pub(crate) mod _ctypes { let offset_val = offset.unwrap_or(0); + // Get buffer address: (char *)((CDataObject *)obj)->b_ptr + offset + let ptr_val = if let Some(simple) = obj.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + (buffer.as_ptr() as isize + offset_val) as usize + } else if let Some(cdata) = obj.downcast_ref::<PyCData>() { + let buffer = cdata.buffer.read(); + (buffer.as_ptr() as isize + offset_val) as usize + } else { + 0 + }; + // Create CArgObject to hold the reference Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), obj, + size: 0, offset: offset_val, } .to_pyobject(vm)) @@ -654,11 +807,6 @@ pub(crate) mod _ctypes { #[pyfunction] fn alignment(tp: Either<PyTypeRef, PyObjectRef>, vm: &VirtualMachine) -> PyResult<usize> { - use super::base::PyCSimpleType; - use super::pointer::PyCPointer; - use super::structure::PyCStructure; - use super::union::PyCUnion; - use super::util::StgInfo; use crate::builtins::PyType; let obj = match &tp { @@ -667,23 +815,27 @@ pub(crate) mod _ctypes { }; // 1. Check TypeDataSlot on class (for instances) - if let Some(stg_info) = obj.class().get_type_data::<StgInfo>() { + if let Some(stg_info) = obj.class().stg_info_opt() { return Ok(stg_info.align); } // 2. Check TypeDataSlot on type itself (for type objects) if let Some(type_obj) = obj.downcast_ref::<PyType>() - && let Some(stg_info) = type_obj.get_type_data::<StgInfo>() + && let Some(stg_info) = type_obj.stg_info_opt() { return Ok(stg_info.align); } - // 3. Fallback for simple types without TypeDataSlot - if obj.fast_isinstance(PyCSimple::static_type()) { - // Get stg_info from the type by reading _type_ attribute - let cls = obj.class().to_owned(); - let stg_info = PyCSimpleType::get_stg_info(&cls, vm); - return Ok(stg_info.align); + // 3. Fallback for simple types + if obj.fast_isinstance(PyCSimple::static_type()) + && let Ok(stg) = obj.class().stg_info(vm) + { + return Ok(stg.align); + } + if obj.fast_isinstance(PyCArray::static_type()) + && let Ok(stg) = obj.class().stg_info(vm) + { + return Ok(stg.align); } if obj.fast_isinstance(PyCStructure::static_type()) { // Calculate alignment from _fields_ @@ -692,7 +844,7 @@ pub(crate) mod _ctypes { } if obj.fast_isinstance(PyCPointer::static_type()) { // Pointer alignment is always pointer size - return Ok(std::mem::align_of::<usize>()); + return Ok(core::mem::align_of::<usize>()); } if obj.fast_isinstance(PyCUnion::static_type()) { // Calculate alignment from _fields_ @@ -715,8 +867,8 @@ pub(crate) mod _ctypes { // Simple type: _type_ is a single character string if let Ok(s) = type_attr.str(vm) { let ty = s.to_string(); - if ty.len() == 1 && SIMPLE_TYPE_CHARS.contains(ty.as_str()) { - return Ok(get_align(&ty)); + if ty.len() == 1 && super::simple::SIMPLE_TYPE_CHARS.contains(ty.as_str()) { + return Ok(super::get_align(&ty)); } } } @@ -754,119 +906,422 @@ pub(crate) mod _ctypes { } #[pyfunction] - fn resize(_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - // TODO: RUSTPYTHON - Err(vm.new_value_error("not implemented")) + fn resize(obj: PyObjectRef, size: isize, vm: &VirtualMachine) -> PyResult<()> { + use alloc::borrow::Cow; + + // 1. Get StgInfo from object's class (validates ctypes instance) + let stg_info = obj + .class() + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))?; + + // 2. Validate size + if size < 0 || (size as usize) < stg_info.size { + return Err(vm.new_value_error(format!("minimum size is {}", stg_info.size))); + } + + // 3. Get PyCData via upcast (works for all ctypes types due to repr(transparent)) + let cdata = obj + .downcast_ref::<PyCData>() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))?; + + // 4. Check if buffer is owned (not borrowed from external memory) + { + let buffer = cdata.buffer.read(); + if matches!(&*buffer, Cow::Borrowed(_)) { + return Err(vm.new_value_error( + "Memory cannot be resized because this object doesn't own it".to_owned(), + )); + } + } + + // 5. Resize the buffer + let new_size = size as usize; + let mut buffer = cdata.buffer.write(); + let old_data = buffer.to_vec(); + let mut new_data = vec![0u8; new_size]; + let copy_len = old_data.len().min(new_size); + new_data[..copy_len].copy_from_slice(&old_data[..copy_len]); + *buffer = Cow::Owned(new_data); + + Ok(()) } #[pyfunction] fn get_errno() -> i32 { - errno::errno().0 + super::function::get_errno_value() } #[pyfunction] - fn set_errno(value: i32) { - errno::set_errno(errno::Errno(value)); + fn set_errno(value: i32) -> i32 { + super::function::set_errno_value(value) } #[cfg(windows)] #[pyfunction] fn get_last_error() -> PyResult<u32> { - Ok(unsafe { windows_sys::Win32::Foundation::GetLastError() }) + Ok(super::function::get_last_error_value()) } #[cfg(windows)] #[pyfunction] - fn set_last_error(value: u32) -> PyResult<()> { - unsafe { windows_sys::Win32::Foundation::SetLastError(value) }; - Ok(()) + fn set_last_error(value: u32) -> u32 { + super::function::set_last_error_value(value) } #[pyattr] fn _memmove_addr(_vm: &VirtualMachine) -> usize { let f = libc::memmove; - f as usize + f as *const () as usize } #[pyattr] fn _memset_addr(_vm: &VirtualMachine) -> usize { let f = libc::memset; - f as usize + f as *const () as usize } #[pyattr] fn _string_at_addr(_vm: &VirtualMachine) -> usize { - let f = libc::strnlen; - f as usize + super::function::INTERNAL_STRING_AT_ADDR } #[pyattr] fn _wstring_at_addr(_vm: &VirtualMachine) -> usize { - // Return address of wcsnlen or similar wide string function - #[cfg(not(target_os = "windows"))] - { - let f = libc::wcslen; - f as usize - } - #[cfg(target_os = "windows")] - { - // FIXME: On Windows, use wcslen from ucrt - 0 - } + super::function::INTERNAL_WSTRING_AT_ADDR } #[pyattr] fn _cast_addr(_vm: &VirtualMachine) -> usize { - // todo!("Implement _cast_addr") - 0 + super::function::INTERNAL_CAST_ADDR } - #[pyfunction(name = "_cast")] - pub fn pycfunction_cast( + #[pyattr] + fn _memoryview_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_MEMORYVIEW_AT_ADDR + } + + #[pyfunction] + fn _cast( obj: PyObjectRef, - _obj2: PyObjectRef, + src: PyObjectRef, ctype: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - use super::array::PyCArray; - use super::base::PyCData; - use super::pointer::PyCPointer; - use crate::class::StaticType; + super::function::cast_impl(obj, src, ctype, vm) + } + + /// Python-level cast function (PYFUNCTYPE wrapper) + #[pyfunction] + fn cast(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult { + super::function::cast_impl(obj.clone(), obj, typ, vm) + } + + /// Return buffer interface information for a ctypes type or object. + /// Returns a tuple (format, ndim, shape) where: + /// - format: PEP 3118 format string + /// - ndim: number of dimensions + /// - shape: tuple of dimension sizes + #[pyfunction] + fn buffer_info(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Determine if obj is a type or an instance + let is_type = obj.class().fast_issubclass(vm.ctx.types.type_type.as_ref()); + let cls = if is_type { + obj.clone() + } else { + obj.class().to_owned().into() + }; + + // Get format from type - try _type_ first (for simple types), then _stg_info_format_ + let format = if let Ok(type_attr) = cls.get_attr("_type_", vm) { + type_attr.str(vm)?.to_string() + } else if let Ok(format_attr) = cls.get_attr("_stg_info_format_", vm) { + format_attr.str(vm)?.to_string() + } else { + return Err(vm.new_type_error("not a ctypes type or object")); + }; + + // Non-array types have ndim=0 and empty shape + // TODO: Implement ndim/shape for arrays when StgInfo supports it + let ndim = 0; + let shape: Vec<PyObjectRef> = vec![]; + + let shape_tuple = vm.ctx.new_tuple(shape); + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_str(format).into(), + vm.ctx.new_int(ndim).into(), + shape_tuple.into(), + ]) + .into()) + } + + /// Unpickle a ctypes object. + #[pyfunction] + fn _unpickle(typ: PyObjectRef, state: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !state.class().is(vm.ctx.types.tuple_type.as_ref()) { + return Err(vm.new_type_error("state must be a tuple")); + } + let obj = vm.call_method(&typ, "__new__", (typ.clone(),))?; + vm.call_method(&obj, "__setstate__", (state,))?; + Ok(obj) + } + + /// Call a function at the given address with the given arguments. + #[pyfunction] + fn call_function( + func_addr: usize, + args: crate::builtins::PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + call_function_internal(func_addr, args, 0, vm) + } - // Python signature: _cast(obj, obj, ctype) - // Python passes the same object twice (obj and _obj2 are the same) - // We ignore _obj2 as it's redundant + /// Call a cdecl function at the given address with the given arguments. + #[pyfunction] + fn call_cdeclfunction( + func_addr: usize, + args: crate::builtins::PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + call_function_internal(func_addr, args, FUNCFLAG_CDECL, vm) + } + + fn call_function_internal( + func_addr: usize, + args: crate::builtins::PyTupleRef, + _flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use libffi::middle::{Arg, Cif, CodePtr, Type}; - // Check if this is a pointer type (has _type_ attribute) - if ctype.get_attr("_type_", vm).is_err() { - return Err(vm.new_type_error("cast() argument 2 must be a pointer type".to_string())); + if func_addr == 0 { + return Err(vm.new_value_error("NULL function pointer")); } - // Create an instance of the target pointer type with no arguments - let result = ctype.call((), vm)?; + let mut ffi_args: Vec<Arg<'_>> = Vec::with_capacity(args.len()); + let mut arg_values: Vec<isize> = Vec::with_capacity(args.len()); + let mut arg_types: Vec<Type> = Vec::with_capacity(args.len()); + + for arg in args.iter() { + if vm.is_none(arg) { + arg_values.push(0); + arg_types.push(Type::pointer()); + } else if let Ok(int_val) = arg.try_int(vm) { + let val = int_val.as_bigint().to_i64().unwrap_or(0) as isize; + arg_values.push(val); + arg_types.push(Type::isize()); + } else if let Some(bytes) = arg.downcast_ref::<crate::builtins::PyBytes>() { + let ptr = bytes.as_bytes().as_ptr() as isize; + arg_values.push(ptr); + arg_types.push(Type::pointer()); + } else if let Some(s) = arg.downcast_ref::<crate::builtins::PyStr>() { + let ptr = s.as_str().as_ptr() as isize; + arg_values.push(ptr); + arg_types.push(Type::pointer()); + } else { + return Err(vm.new_type_error(format!( + "Don't know how to convert parameter of type '{}'", + arg.class().name() + ))); + } + } - // Get the pointer value from the source object - // If obj is a CData instance (including arrays), use the object itself - // If obj is an integer, use it directly as the pointer value - let ptr_value: PyObjectRef = if obj.fast_isinstance(PyCData::static_type()) - || obj.fast_isinstance(PyCArray::static_type()) - || obj.fast_isinstance(PyCPointer::static_type()) - { - // For CData objects (including arrays and pointers), store the object itself - obj.clone() - } else if let Ok(int_val) = obj.try_int(vm) { - // For integers, treat as pointer address - vm.ctx.new_int(int_val.as_bigint().clone()).into() - } else { - return Err(vm.new_type_error(format!( - "cast() argument 1 must be a ctypes instance or an integer, not {}", - obj.class().name() - ))); + for val in &arg_values { + ffi_args.push(Arg::new(val)); + } + + let cif = Cif::new(arg_types, Type::c_int()); + let code_ptr = CodePtr::from_ptr(func_addr as *const _); + let result: libc::c_int = unsafe { cif.call(code_ptr, &ffi_args) }; + Ok(vm.ctx.new_int(result).into()) + } + + /// Convert a pointer (as integer) to a Python object. + #[pyfunction(name = "PyObj_FromPtr")] + fn py_obj_from_ptr(ptr: usize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let raw_ptr = ptr as *mut crate::object::PyObject; + unsafe { + let obj = crate::PyObjectRef::from_raw(core::ptr::NonNull::new_unchecked(raw_ptr)); + let obj = core::mem::ManuallyDrop::new(obj); + Ok((*obj).clone()) + } + } + + #[pyfunction(name = "Py_INCREF")] + fn py_incref(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { + // TODO: + obj + } + + #[pyfunction(name = "Py_DECREF")] + fn py_decref(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { + // TODO: + obj + } + + #[cfg(target_os = "macos")] + #[pyfunction] + fn _dyld_shared_cache_contains_path( + path: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + use alloc::ffi::CString; + + let path = match path { + Some(p) if !vm.is_none(&p) => p, + _ => return Ok(false), }; - // Set the contents of the pointer by setting the attribute - result.set_attr("contents", ptr_value, vm)?; + let path_str = path.str(vm)?.to_string(); + let c_path = + CString::new(path_str).map_err(|_| vm.new_value_error("path contains null byte"))?; + unsafe extern "C" { + fn _dyld_shared_cache_contains_path(path: *const libc::c_char) -> bool; + } + + let result = unsafe { _dyld_shared_cache_contains_path(c_path.as_ptr()) }; Ok(result) } + + #[cfg(windows)] + #[pyfunction(name = "FormatError")] + fn format_error_func(code: OptionalArg<u32>, _vm: &VirtualMachine) -> PyResult<String> { + use windows_sys::Win32::Foundation::{GetLastError, LocalFree}; + use windows_sys::Win32::System::Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }; + + let error_code = code.unwrap_or_else(|| unsafe { GetLastError() }); + + let mut buffer: *mut u16 = core::ptr::null_mut(); + let len = unsafe { + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + core::ptr::null(), + error_code, + 0, + &mut buffer as *mut *mut u16 as *mut u16, + 0, + core::ptr::null(), + ) + }; + + if len == 0 || buffer.is_null() { + return Ok("<no description>".to_string()); + } + + let message = unsafe { + let slice = core::slice::from_raw_parts(buffer, len as usize); + let msg = String::from_utf16_lossy(slice).trim_end().to_string(); + LocalFree(buffer as *mut _); + msg + }; + + Ok(message) + } + + #[cfg(windows)] + #[pyfunction(name = "CopyComPointer")] + fn copy_com_pointer(src: PyObjectRef, dst: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { + use windows_sys::Win32::Foundation::{E_POINTER, S_OK}; + + // 1. Extract pointer-to-pointer address from dst (byref() result) + let pdst: usize = if let Some(carg) = dst.downcast_ref::<CArgObject>() { + // byref() result: object buffer address + offset + let base = if let Some(cdata) = carg.obj.downcast_ref::<PyCData>() { + cdata.buffer.read().as_ptr() as usize + } else { + return Ok(E_POINTER); + }; + (base as isize + carg.offset) as usize + } else { + return Ok(E_POINTER); + }; + + if pdst == 0 { + return Ok(E_POINTER); + } + + // 2. Extract COM pointer value from src + let src_ptr: usize = if vm.is_none(&src) { + 0 + } else if let Some(cdata) = src.downcast_ref::<PyCData>() { + // c_void_p etc: read pointer value from buffer + let buffer = cdata.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + usize::from_ne_bytes( + buffer[..core::mem::size_of::<usize>()] + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ) + } else { + 0 + } + } else { + return Ok(E_POINTER); + }; + + // 3. Call IUnknown::AddRef if src is non-NULL + if src_ptr != 0 { + unsafe { + // IUnknown vtable: [QueryInterface, AddRef, Release, ...] + let iunknown = src_ptr as *mut *const usize; + let vtable = *iunknown; + debug_assert!(!vtable.is_null(), "IUnknown vtable is null"); + let addref_fn: extern "system" fn(*mut core::ffi::c_void) -> u32 = + core::mem::transmute(*vtable.add(1)); // AddRef is index 1 + addref_fn(src_ptr as *mut core::ffi::c_void); + } + } + + // 4. Copy pointer: *pdst = src + unsafe { + *(pdst as *mut usize) = src_ptr; + } + + Ok(S_OK) + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + use super::*; + + __module_exec(vm, module); + + let ctx = &vm.ctx; + PyCSimpleType::make_class(ctx); + array::PyCArrayType::make_class(ctx); + pointer::PyCPointerType::make_class(ctx); + structure::PyCStructType::make_class(ctx); + union::PyCUnionType::make_class(ctx); + function::PyCFuncPtrType::make_class(ctx); + function::RawMemoryBuffer::make_class(ctx); + + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + "Array" => PyCArray::make_class(ctx), + "CField" => PyCField::make_class(ctx), + "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), + "_Pointer" => PyCPointer::make_class(ctx), + "_pointer_type_cache" => ctx.new_dict(), + "_array_type_cache" => ctx.new_dict(), + "Structure" => PyCStructure::make_class(ctx), + "CThunkObject" => function::PyCThunk::make_class(ctx), + "Union" => PyCUnion::make_class(ctx), + }); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/ctypes/array.rs b/crates/vm/src/stdlib/ctypes/array.rs index fe12a781d9f..843dd7d5b8c 100644 --- a/crates/vm/src/stdlib/ctypes/array.rs +++ b/crates/vm/src/stdlib/ctypes/array.rs @@ -1,41 +1,126 @@ -use crate::atomic_func; -use crate::builtins::{PyBytes, PyInt}; -use crate::class::StaticType; -use crate::function::FuncArgs; -use crate::protocol::{ - BufferDescriptor, BufferMethods, PyBuffer, PyNumberMethods, PySequenceMethods, -}; -use crate::stdlib::ctypes::base::CDataObject; -use crate::stdlib::ctypes::util::StgInfo; -use crate::types::{AsBuffer, AsNumber, AsSequence}; -use crate::{AsObject, Py, PyObjectRef, PyPayload}; +use super::StgInfo; +use super::base::{CDATA_BUFFER_METHODS, PyCData}; +use super::type_info; use crate::{ - PyResult, VirtualMachine, - builtins::{PyType, PyTypeRef}, - types::Constructor, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + atomic_func, + builtins::{ + PyBytes, PyInt, PyList, PySlice, PyStr, PyType, PyTypeRef, genericalias::PyGenericAlias, + }, + class::StaticType, + function::{ArgBytesLike, FuncArgs, PySetterValue}, + protocol::{BufferDescriptor, PyBuffer, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + types::{AsBuffer, AsMapping, AsNumber, AsSequence, Constructor, Initializer}, }; -use crossbeam_utils::atomic::AtomicCell; -use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; -use rustpython_vm::stdlib::ctypes::_ctypes::get_size; -use rustpython_vm::stdlib::ctypes::base::PyCData; +use alloc::borrow::Cow; +use num_traits::{Signed, ToPrimitive}; + +/// Get itemsize from a PEP 3118 format string +/// Extracts the type code (last char after endianness prefix) and returns its size +fn get_size_from_format(fmt: &str) -> usize { + // Format is like "<f", ">q", etc. - strip endianness prefix and get type code + let code = fmt + .trim_start_matches(['<', '>', '@', '=', '!', '&']) + .chars() + .next() + .map(|c| c.to_string()); + code.map(|c| type_info(&c).map(|t| t.size).unwrap_or(1)) + .unwrap_or(1) +} -/// PyCArrayType - metatype for Array types -/// CPython stores array info (type, length) in StgInfo via type_data -#[pyclass(name = "PyCArrayType", base = PyType, module = "_ctypes")] -#[derive(Debug)] -#[repr(transparent)] -pub struct PyCArrayType(PyType); +/// Creates array type for (element_type, length) +/// Uses _array_type_cache to ensure identical calls return the same type object +pub(super) fn array_type_from_ctype( + itemtype: PyObjectRef, + length: usize, + vm: &VirtualMachine, +) -> PyResult { + // PyCArrayType_from_ctype + + // Get the _array_type_cache from _ctypes module + let ctypes_module = vm.import("_ctypes", 0)?; + let cache = ctypes_module.get_attr("_array_type_cache", vm)?; + + // Create cache key: (itemtype, length) tuple + let length_obj: PyObjectRef = vm.ctx.new_int(length).into(); + let cache_key = vm.ctx.new_tuple(vec![itemtype.clone(), length_obj]); + + // Check if already in cache + if let Ok(cached) = vm.call_method(&cache, "__getitem__", (cache_key.clone(),)) + && !vm.is_none(&cached) + { + return Ok(cached); + } -/// Create a new Array type with StgInfo stored in type_data (CPython style) -pub fn create_array_type_with_stg_info(stg_info: StgInfo, vm: &VirtualMachine) -> PyResult { - // Get PyCArrayType as metaclass - let metaclass = PyCArrayType::static_type().to_owned(); + // Cache miss - create new array type + let itemtype_ref = itemtype + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("Expected a type object"))?; + + let item_stg = itemtype_ref + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + + let element_size = item_stg.size; + let element_align = item_stg.align; + let item_format = item_stg.format.clone(); + let item_shape = item_stg.shape.clone(); + let item_flags = item_stg.flags; + + // Check overflow before multiplication + let total_size = element_size + .checked_mul(length) + .ok_or_else(|| vm.new_overflow_error("array too large"))?; + + // format name: "c_int_Array_5" + let type_name = format!("{}_Array_{}", itemtype_ref.name(), length); + + // Get item type code before moving itemtype + let item_type_code = itemtype_ref + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + let stg_info = StgInfo::new_array( + total_size, + element_align, + length, + itemtype_ref.clone(), + element_size, + item_format.as_deref(), + &item_shape, + item_flags, + ); - // Create a unique name for the array type - let type_name = format!("Array_{}", stg_info.length); + let new_type = create_array_type_with_name(stg_info, &type_name, vm)?; - // Create args for type(): (name, bases, dict) + // Special case for character arrays - add value/raw attributes + let new_type_ref: PyTypeRef = new_type + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + match item_type_code.as_deref() { + Some("c") => add_char_array_getsets(&new_type_ref, vm), + Some("u") => add_wchar_array_getsets(&new_type_ref, vm), + _ => {} + } + + // Store in cache + vm.call_method(&cache, "__setitem__", (cache_key, new_type.clone()))?; + + Ok(new_type) +} + +/// create_array_type_with_name - create array type with specified name +fn create_array_type_with_name( + stg_info: StgInfo, + type_name: &str, + vm: &VirtualMachine, +) -> PyResult { + let metaclass = PyCArrayType::static_type().to_owned(); let name = vm.ctx.new_str(type_name); let bases = vm .ctx @@ -47,170 +132,215 @@ pub fn create_array_type_with_stg_info(stg_info: StgInfo, vm: &VirtualMachine) - crate::function::KwArgs::default(), ); - // Create the new type using PyType::slot_new with PyCArrayType as metaclass let new_type = crate::builtins::type_::PyType::slot_new(metaclass, args, vm)?; - // Set StgInfo in type_data let type_ref: PyTypeRef = new_type .clone() .downcast() - .map_err(|_| vm.new_type_error("Failed to create array type".to_owned()))?; + .map_err(|_| vm.new_type_error("Failed to create array type"))?; - if type_ref.init_type_data(stg_info.clone()).is_err() { - // Type data already initialized - update it - if let Some(mut existing) = type_ref.get_type_data_mut::<StgInfo>() { - *existing = stg_info; - } + // Set class attributes for _type_ and _length_ + if let Some(element_type) = stg_info.element_type.clone() { + new_type.set_attr("_type_", element_type, vm)?; } + new_type.set_attr("_length_", vm.ctx.new_int(stg_info.length), vm)?; + + super::base::set_or_init_stginfo(&type_ref, stg_info); Ok(new_type) } -impl Constructor for PyCArrayType { - type Args = PyObjectRef; +/// PyCArrayType - metatype for Array types +#[pyclass(name = "PyCArrayType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCArrayType(PyType); - fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("use slot_new") - } -} +// PyCArrayType implements Initializer for slots.init (PyCArrayType_init) +impl Initializer for PyCArrayType { + type Args = FuncArgs; -#[pyclass(flags(IMMUTABLETYPE), with(Constructor, AsNumber))] -impl PyCArrayType { - #[pygetset(name = "_type_")] - fn typ(zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - zelf.downcast_ref::<PyType>() - .and_then(|t| t.get_type_data::<StgInfo>()) - .and_then(|stg| stg.element_type.clone()) - .unwrap_or_else(|| vm.ctx.none()) - } + fn init(zelf: PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // zelf is the newly created array type (e.g., T in "class T(Array)") + let new_type: &PyType = &zelf.0; + + new_type.check_not_initialized(vm)?; + + // 1. Get _length_ from class dict first + let direct_length = new_type + .attributes + .read() + .get(vm.ctx.intern_str("_length_")) + .cloned(); + + // 2. Get _type_ from class dict first + let direct_type = new_type + .attributes + .read() + .get(vm.ctx.intern_str("_type_")) + .cloned(); + + // 3. Find parent StgInfo from MRO (for inheritance) + // Note: PyType.mro does NOT include self, so no skip needed + let parent_stg_info = new_type + .mro + .read() + .iter() + .find_map(|base| base.stg_info_opt().map(|s| s.clone())); + + // 4. Resolve _length_ (direct or inherited) + let length = if let Some(length_attr) = direct_length { + // Direct _length_ defined - validate it (PyLong_Check) + let length_int = length_attr + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("The '_length_' attribute must be an integer"))?; + let bigint = length_int.as_bigint(); + // Check sign first - negative values are ValueError + if bigint.is_negative() { + return Err(vm.new_value_error("The '_length_' attribute must not be negative")); + } + // Positive values that don't fit in usize are OverflowError + bigint + .to_usize() + .ok_or_else(|| vm.new_overflow_error("The '_length_' attribute is too large"))? + } else if let Some(ref parent_info) = parent_stg_info { + // Inherit from parent + parent_info.length + } else { + return Err(vm.new_attribute_error("class must define a '_length_' attribute")); + }; - #[pygetset(name = "_length_")] - fn length(zelf: PyObjectRef) -> usize { - zelf.downcast_ref::<PyType>() - .and_then(|t| t.get_type_data::<StgInfo>()) - .map(|stg| stg.length) - .unwrap_or(0) - } + // 5. Resolve _type_ and get item_info (direct or inherited) + let (element_type, item_size, item_align, item_format, item_shape, item_flags) = + if let Some(type_attr) = direct_type { + // Direct _type_ defined - validate it (PyStgInfo_FromType) + let type_ref = type_attr + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_type_ must be a type"))?; + let (size, align, format, shape, flags) = { + let item_info = type_ref + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + ( + item_info.size, + item_info.align, + item_info.format.clone(), + item_info.shape.clone(), + item_info.flags, + ) + }; + (type_ref, size, align, format, shape, flags) + } else if let Some(ref parent_info) = parent_stg_info { + // Inherit from parent + let parent_type = parent_info + .element_type + .clone() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + ( + parent_type, + parent_info.element_size, + parent_info.align, + parent_info.format.clone(), + parent_info.shape.clone(), + parent_info.flags, + ) + } else { + return Err(vm.new_attribute_error("class must define a '_type_' attribute")); + }; - #[pymethod] - fn __mul__(zelf: PyObjectRef, n: isize, vm: &VirtualMachine) -> PyResult { - if n < 0 { - return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + // 6. Check overflow (item_size != 0 && length > MAX / item_size) + if item_size != 0 && length > usize::MAX / item_size { + return Err(vm.new_overflow_error("array too large")); } - // Get inner array info from TypeDataSlot - let type_ref = zelf.downcast_ref::<PyType>().unwrap(); - let (_inner_length, inner_size) = type_ref - .get_type_data::<StgInfo>() - .map(|stg| (stg.length, stg.size)) - .unwrap_or((0, 0)); - - // The element type of the new array is the current array type itself - let current_array_type: PyObjectRef = zelf.clone(); - - // Element size is the total size of the inner array - let new_element_size = inner_size; - let total_size = new_element_size * (n as usize); - + // 7. Initialize StgInfo (PyStgInfo_Init + field assignment) let stg_info = StgInfo::new_array( - total_size, - new_element_size, - n as usize, - current_array_type, - new_element_size, + item_size * length, // size = item_size * length + item_align, // align = item_info->align + length, // length + element_type.clone(), + item_size, // element_size + item_format.as_deref(), + &item_shape, + item_flags, ); - create_array_type_with_stg_info(stg_info, vm) - } + // 8. Store StgInfo in type_data + super::base::set_or_init_stginfo(new_type, stg_info); - #[pyclassmethod] - fn in_dll( - zelf: PyObjectRef, - dll: PyObjectRef, - name: crate::builtins::PyStrRef, - vm: &VirtualMachine, - ) -> PyResult { - use libloading::Symbol; - - // Get the library handle from dll object - let handle = if let Ok(int_handle) = dll.try_int(vm) { - // dll is an integer handle - int_handle - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - } else { - // dll is a CDLL/PyDLL/WinDLL object with _handle attribute - dll.get_attr("_handle", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - }; + // 9. Get type code before moving element_type + let item_type_code = element_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + // 10. Set class attributes for _type_ and _length_ + zelf.as_object().set_attr("_type_", element_type, vm)?; + zelf.as_object() + .set_attr("_length_", vm.ctx.new_int(length), vm)?; + + // 11. Special case for character arrays - add value/raw attributes + // if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) + // add_getset((PyTypeObject*)self, CharArray_getsets); + // else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) + // add_getset((PyTypeObject*)self, WCharArray_getsets); + + // Get type ref for add_getset + let type_ref: PyTypeRef = zelf.as_object().to_owned().downcast().unwrap(); + match item_type_code.as_deref() { + Some("c") => add_char_array_getsets(&type_ref, vm), + Some("u") => add_wchar_array_getsets(&type_ref, vm), + _ => {} + } - // Get the library from cache - let library_cache = crate::stdlib::ctypes::library::libcache().read(); - let library = library_cache - .get_lib(handle) - .ok_or_else(|| vm.new_attribute_error("Library not found".to_owned()))?; + Ok(()) + } +} - // Get symbol address from library - let symbol_name = format!("{}\0", name.as_str()); - let inner_lib = library.lib.lock(); +#[pyclass(flags(IMMUTABLETYPE), with(Initializer, AsNumber))] +impl PyCArrayType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } - let symbol_address = if let Some(lib) = &*inner_lib { - unsafe { - // Try to get the symbol from the library - let symbol: Symbol<'_, *mut u8> = lib.get(symbol_name.as_bytes()).map_err(|e| { - vm.new_attribute_error(format!("{}: symbol '{}' not found", e, name.as_str())) - })?; - *symbol as usize - } - } else { - return Err(vm.new_attribute_error("Library is closed".to_owned())); - }; + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } - // Get size from the array type via TypeDataSlot - let type_ref = zelf.downcast_ref::<PyType>().unwrap(); - let (element_type, length, element_size) = type_ref - .get_type_data::<StgInfo>() - .map(|stg| { - ( - stg.element_type.clone().unwrap_or_else(|| vm.ctx.none()), - stg.length, - stg.element_size, - ) - }) - .unwrap_or_else(|| (vm.ctx.none(), 0, 0)); - let total_size = element_size * length; + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the array type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } - // Read data from symbol address - let data = if symbol_address != 0 && total_size > 0 { - unsafe { - let ptr = symbol_address as *const u8; - std::slice::from_raw_parts(ptr, total_size).to_vec() + // 2. Check for CArgObject (PyCArg_CheckExact) + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() { + // Check if the wrapped object is an instance of the requested type + if carg.obj.is_instance(cls.as_object(), vm)? { + return Ok(value); // Return the CArgObject as-is } - } else { - vec![0; total_size] - }; - - // Create instance - let cdata = CDataObject::from_bytes(data, None); - let instance = PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), } - .into_pyobject(vm); - // Store base reference to keep dll alive - if let Ok(array_ref) = instance.clone().downcast::<PyCArray>() { - array_ref.cdata.write().base = Some(dll); + // 3. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCArrayType::from_param(cls.as_object().to_owned(), as_parameter, vm); } - Ok(instance) + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) } } @@ -223,8 +353,28 @@ impl AsNumber for PyCArrayType { .try_index(vm)? .as_bigint() .to_isize() - .ok_or_else(|| vm.new_overflow_error("array size too large".to_owned()))?; - PyCArrayType::__mul__(a.to_owned(), n, vm) + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + + // Check for overflow before creating the new array type + let zelf_type = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("Expected type"))?; + + if let Some(stg_info) = zelf_type.stg_info_opt() { + let current_size = stg_info.size; + // Check if current_size * n would overflow + if current_size != 0 && (n as usize) > isize::MAX as usize / current_size { + return Err(vm.new_overflow_error("array too large")); + } + } + + // Use cached array type creation + // The element type of the new array is the current array type itself + array_type_from_ctype(a.to_owned(), n as usize, vm) }), ..PyNumberMethods::NOT_IMPLEMENTED }; @@ -232,27 +382,28 @@ impl AsNumber for PyCArrayType { } } +/// PyCArray - Array instance +/// All array metadata (element_type, length, element_size) is stored in the type's StgInfo #[pyclass( name = "Array", base = PyCData, metaclass = "PyCArrayType", module = "_ctypes" )] -pub struct PyCArray { - _base: PyCData, - /// Element type - can be a simple type (c_int) or an array type (c_int * 5) - pub(super) typ: PyRwLock<PyObjectRef>, - pub(super) length: AtomicCell<usize>, - pub(super) element_size: AtomicCell<usize>, - pub(super) cdata: PyRwLock<CDataObject>, -} +#[derive(Debug)] +#[repr(transparent)] +pub struct PyCArray(pub PyCData); -impl std::fmt::Debug for PyCArray { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyCArray") - .field("typ", &self.typ) - .field("length", &self.length) - .finish() +impl PyCArray { + /// Get the type code of array element type (e.g., "c" for c_char, "u" for c_wchar) + fn get_element_type_code(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<String> { + zelf.class() + .stg_info_opt() + .and_then(|info| info.element_type.clone())? + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())) } } @@ -260,60 +411,29 @@ impl Constructor for PyCArray { type Args = FuncArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // Get _type_ and _length_ from the class - let type_attr = cls.as_object().get_attr("_type_", vm).ok(); - let length_attr = cls.as_object().get_attr("_length_", vm).ok(); - - let element_type = type_attr.unwrap_or_else(|| vm.ctx.types.object_type.to_owned().into()); - let length = if let Some(len_obj) = length_attr { - len_obj.try_int(vm)?.as_bigint().to_usize().unwrap_or(0) - } else { - 0 + // Check for abstract class - StgInfo must exist and be initialized + // Extract values in a block to drop the borrow before using cls + let (length, total_size) = { + let stg = cls.stg_info(vm)?; + (stg.length, stg.size) }; - // Get element size from _type_ - let element_size = if let Ok(type_code) = element_type.get_attr("_type_", vm) { - if let Ok(s) = type_code.str(vm) { - let s = s.to_string(); - if s.len() == 1 { - get_size(&s) - } else { - std::mem::size_of::<usize>() - } - } else { - std::mem::size_of::<usize>() - } - } else { - std::mem::size_of::<usize>() - }; + // Check for too many initializers + if args.args.len() > length { + return Err(vm.new_index_error("too many initializers")); + } - let total_size = element_size * length; - let mut buffer = vec![0u8; total_size]; + // Create array with zero-initialized buffer + let buffer = vec![0u8; total_size]; + let instance = PyCArray(PyCData::from_bytes_with_length(buffer, None, length)) + .into_ref_with_type(vm, cls)?; - // Initialize from positional arguments + // Initialize elements using setitem_by_index (Array_init pattern) for (i, value) in args.args.iter().enumerate() { - if i >= length { - break; - } - let offset = i * element_size; - if let Ok(int_val) = value.try_int(vm) { - let bytes = PyCArray::int_to_bytes(int_val.as_bigint(), element_size); - if offset + element_size <= buffer.len() { - buffer[offset..offset + element_size].copy_from_slice(&bytes); - } - } + PyCArray::setitem_by_index(&instance, i as isize, value.clone(), vm)?; } - let cdata = CDataObject::from_bytes(buffer, None); - PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), - } - .into_ref_with_type(vm, cls) - .map(Into::into) + Ok(instance.into()) } fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { @@ -321,19 +441,35 @@ impl Constructor for PyCArray { } } +impl Initializer for PyCArray { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Re-initialize array elements when __init__ is called + for (i, value) in args.args.iter().enumerate() { + PyCArray::setitem_by_index(&zelf, i as isize, value.clone(), vm)?; + } + Ok(()) + } +} + impl AsSequence for PyCArray { fn as_sequence() -> &'static PySequenceMethods { - use std::sync::LazyLock; + use crate::common::lock::LazyLock; static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyCArray::sequence_downcast(seq).length.load())), + length: atomic_func!(|seq, _vm| { + let zelf = PyCArray::sequence_downcast(seq); + Ok(zelf.class().stg_info_opt().map_or(0, |i| i.length)) + }), item: atomic_func!(|seq, i, vm| { - PyCArray::getitem_by_index(PyCArray::sequence_downcast(seq), i, vm) + let zelf = PyCArray::sequence_downcast(seq); + PyCArray::getitem_by_index(zelf, i, vm) }), ass_item: atomic_func!(|seq, i, value, vm| { let zelf = PyCArray::sequence_downcast(seq); match value { Some(v) => PyCArray::setitem_by_index(zelf, i, v, vm), - None => Err(vm.new_type_error("cannot delete array elements".to_owned())), + None => Err(vm.new_type_error("cannot delete array elements")), } }), ..PySequenceMethods::NOT_IMPLEMENTED @@ -342,473 +478,890 @@ impl AsSequence for PyCArray { } } +impl AsMapping for PyCArray { + fn as_mapping() -> &'static PyMappingMethods { + use crate::common::lock::LazyLock; + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, _vm| { + let zelf = PyCArray::mapping_downcast(mapping); + Ok(zelf.class().stg_info_opt().map_or(0, |i| i.length)) + }), + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyCArray::mapping_downcast(mapping); + PyCArray::__getitem__(zelf, needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyCArray::mapping_downcast(mapping); + match value { + Some(value) => PyCArray::__setitem__(zelf, needle.to_owned(), value, vm), + None => PyCArray::__delitem__(zelf, needle.to_owned(), vm), + } + }), + }); + &AS_MAPPING + } +} + #[pyclass( flags(BASETYPE, IMMUTABLETYPE), - with(Constructor, AsSequence, AsBuffer) + with(Constructor, Initializer, AsSequence, AsMapping, AsBuffer) )] impl PyCArray { - #[pygetset] - fn _objects(&self) -> Option<PyObjectRef> { - self.cdata.read().objects.clone() + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) } fn int_to_bytes(i: &malachite_bigint::BigInt, size: usize) -> Vec<u8> { + // Try unsigned first (handles values like 0xFFFFFFFF that overflow signed) + // then fall back to signed (handles negative values) match size { - 1 => vec![i.to_i8().unwrap_or(0) as u8], - 2 => i.to_i16().unwrap_or(0).to_ne_bytes().to_vec(), - 4 => i.to_i32().unwrap_or(0).to_ne_bytes().to_vec(), - 8 => i.to_i64().unwrap_or(0).to_ne_bytes().to_vec(), + 1 => { + if let Some(v) = i.to_u8() { + vec![v] + } else { + vec![i.to_i8().unwrap_or(0) as u8] + } + } + 2 => { + if let Some(v) = i.to_u16() { + v.to_ne_bytes().to_vec() + } else { + i.to_i16().unwrap_or(0).to_ne_bytes().to_vec() + } + } + 4 => { + if let Some(v) = i.to_u32() { + v.to_ne_bytes().to_vec() + } else { + i.to_i32().unwrap_or(0).to_ne_bytes().to_vec() + } + } + 8 => { + if let Some(v) = i.to_u64() { + v.to_ne_bytes().to_vec() + } else { + i.to_i64().unwrap_or(0).to_ne_bytes().to_vec() + } + } _ => vec![0u8; size], } } - fn bytes_to_int(bytes: &[u8], size: usize, vm: &VirtualMachine) -> PyObjectRef { - match size { - 1 => vm.ctx.new_int(bytes[0] as i8).into(), - 2 => { + fn bytes_to_int( + bytes: &[u8], + size: usize, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyObjectRef { + // Unsigned type codes: B (uchar), H (ushort), I (uint), L (ulong), Q (ulonglong) + let is_unsigned = matches!( + type_code, + Some("B") | Some("H") | Some("I") | Some("L") | Some("Q") + ); + + match (size, is_unsigned) { + (1, false) => vm.ctx.new_int(bytes[0] as i8).into(), + (1, true) => vm.ctx.new_int(bytes[0]).into(), + (2, false) => { let val = i16::from_ne_bytes([bytes[0], bytes[1]]); vm.ctx.new_int(val).into() } - 4 => { + (2, true) => { + let val = u16::from_ne_bytes([bytes[0], bytes[1]]); + vm.ctx.new_int(val).into() + } + (4, false) => { let val = i32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); vm.ctx.new_int(val).into() } - 8 => { + (4, true) => { + let val = u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + vm.ctx.new_int(val).into() + } + (8, false) => { let val = i64::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]); vm.ctx.new_int(val).into() } + (8, true) => { + let val = u64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]); + vm.ctx.new_int(val).into() + } _ => vm.ctx.new_int(0).into(), } } - fn getitem_by_index(zelf: &PyCArray, i: isize, vm: &VirtualMachine) -> PyResult { - let length = zelf.length.load() as isize; + fn getitem_by_index(zelf: &Py<PyCArray>, i: isize, vm: &VirtualMachine) -> PyResult { + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length) as isize; let index = if i < 0 { length + i } else { i }; if index < 0 || index >= length { - return Err(vm.new_index_error("array index out of range".to_owned())); + return Err(vm.new_index_error("invalid index")); } let index = index as usize; - let element_size = zelf.element_size.load(); + let element_size = stg.as_ref().map_or(0, |i| i.element_size); let offset = index * element_size; - let buffer = zelf.cdata.read().buffer.clone(); - if offset + element_size <= buffer.len() { - let bytes = &buffer[offset..offset + element_size]; - Ok(Self::bytes_to_int(bytes, element_size, vm)) + let type_code = Self::get_element_type_code(zelf, vm); + + // Get target buffer and offset (base's buffer if available, otherwise own) + let base_obj = zelf.0.base.read().clone(); + let (buffer_lock, final_offset) = if let Some(cdata) = base_obj + .as_ref() + .and_then(|b| b.downcast_ref::<super::PyCData>()) + { + (&cdata.buffer, zelf.0.base_offset.load() + offset) } else { - Ok(vm.ctx.new_int(0).into()) + (&zelf.0.buffer, offset) + }; + + let buffer = buffer_lock.read(); + Self::read_element_from_buffer( + &buffer, + final_offset, + element_size, + type_code.as_deref(), + vm, + ) + } + + /// Helper to read an element value from a buffer at given offset + fn read_element_from_buffer( + buffer: &[u8], + offset: usize, + element_size: usize, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult { + match type_code { + Some("c") => { + // Return single byte as bytes + if offset < buffer.len() { + Ok(vm.ctx.new_bytes(vec![buffer[offset]]).into()) + } else { + Ok(vm.ctx.new_bytes(vec![0]).into()) + } + } + Some("u") => { + // Return single wchar as str + if let Some(code) = wchar_from_bytes(&buffer[offset..]) { + let s = char::from_u32(code) + .map(|c| c.to_string()) + .unwrap_or_default(); + Ok(vm.ctx.new_str(s).into()) + } else { + Ok(vm.ctx.new_str("").into()) + } + } + Some("z") => { + // c_char_p: pointer to bytes - dereference to get string + if offset + element_size > buffer.len() { + return Ok(vm.ctx.none()); + } + let ptr_bytes = &buffer[offset..offset + element_size]; + let ptr_val = usize::from_ne_bytes( + ptr_bytes + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ); + if ptr_val == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated string from pointer address + unsafe { + let ptr = ptr_val as *const u8; + let mut len = 0; + while *ptr.add(len) != 0 { + len += 1; + } + let bytes = core::slice::from_raw_parts(ptr, len); + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) + } + } + Some("Z") => { + // c_wchar_p: pointer to wchar_t - dereference to get string + if offset + element_size > buffer.len() { + return Ok(vm.ctx.none()); + } + let ptr_bytes = &buffer[offset..offset + element_size]; + let ptr_val = usize::from_ne_bytes( + ptr_bytes + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ); + if ptr_val == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated wide string using WCHAR_SIZE + unsafe { + let ptr = ptr_val as *const u8; + let mut chars = Vec::new(); + let mut pos = 0usize; + loop { + let code = if WCHAR_SIZE == 2 { + let bytes = core::slice::from_raw_parts(ptr.add(pos), 2); + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + let bytes = core::slice::from_raw_parts(ptr.add(pos), 4); + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }; + if code == 0 { + break; + } + if let Some(ch) = char::from_u32(code) { + chars.push(ch); + } + pos += WCHAR_SIZE; + } + let s: String = chars.into_iter().collect(); + Ok(vm.ctx.new_str(s).into()) + } + } + Some("f") => { + // c_float + let val = buffer[offset..] + .first_chunk::<4>() + .copied() + .map_or(0.0, f32::from_ne_bytes); + Ok(vm.ctx.new_float(val as f64).into()) + } + Some("d") | Some("g") => { + // c_double / c_longdouble - read f64 from first 8 bytes + let val = buffer[offset..] + .first_chunk::<8>() + .copied() + .map_or(0.0, f64::from_ne_bytes); + Ok(vm.ctx.new_float(val).into()) + } + _ => { + if let Some(bytes) = buffer[offset..].get(..element_size) { + Ok(Self::bytes_to_int(bytes, element_size, type_code, vm)) + } else { + Ok(vm.ctx.new_int(0).into()) + } + } + } + } + + /// Helper to write an element value to a buffer at given offset + /// This is extracted to share code between direct write and base-buffer write + #[allow(clippy::too_many_arguments)] + fn write_element_to_buffer( + buffer: &mut [u8], + offset: usize, + element_size: usize, + type_code: Option<&str>, + value: &PyObject, + zelf: &Py<PyCArray>, + index: usize, + vm: &VirtualMachine, + ) -> PyResult<()> { + match type_code { + Some("c") => { + if let Some(b) = value.downcast_ref::<PyBytes>() { + if offset < buffer.len() { + buffer[offset] = b.as_bytes().first().copied().unwrap_or(0); + } + } else if let Ok(int_val) = value.try_int(vm) { + if offset < buffer.len() { + buffer[offset] = int_val.as_bigint().to_u8().unwrap_or(0); + } + } else { + return Err(vm.new_type_error("an integer or bytes of length 1 is required")); + } + } + Some("u") => { + if let Some(s) = value.downcast_ref::<PyStr>() { + let code = s.as_str().chars().next().map(|c| c as u32).unwrap_or(0); + if offset + WCHAR_SIZE <= buffer.len() { + wchar_to_bytes(code, &mut buffer[offset..]); + } + } else { + return Err(vm.new_type_error("unicode string expected")); + } + } + Some("z") => { + let (ptr_val, converted) = if value.is(&vm.ctx.none) { + (0usize, None) + } else if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + zelf.0.keep_alive(index, kept_alive); + (ptr, Some(value.to_owned())) + } else if let Ok(int_val) = value.try_index(vm) { + (int_val.as_bigint().to_usize().unwrap_or(0), None) + } else { + return Err(vm.new_type_error( + "bytes or integer address expected instead of {}".to_owned(), + )); + }; + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&ptr_val.to_ne_bytes()); + } + if let Some(c) = converted { + return zelf.0.keep_ref(index, c, vm); + } + } + Some("Z") => { + let (ptr_val, converted) = if value.is(&vm.ctx.none) { + (0usize, None) + } else if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); + (ptr, Some(holder)) + } else if let Ok(int_val) = value.try_index(vm) { + (int_val.as_bigint().to_usize().unwrap_or(0), None) + } else { + return Err(vm.new_type_error("unicode string or integer address expected")); + }; + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&ptr_val.to_ne_bytes()); + } + if let Some(c) = converted { + return zelf.0.keep_ref(index, c, vm); + } + } + Some("f") => { + // c_float: convert int/float to f32 bytes + let f32_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() as f32 + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) as f32 + } else { + return Err(vm.new_type_error("a float is required")); + }; + if offset + 4 <= buffer.len() { + buffer[offset..offset + 4].copy_from_slice(&f32_val.to_ne_bytes()); + } + } + Some("d") | Some("g") => { + // c_double / c_longdouble: convert int/float to f64 bytes + let f64_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) + } else { + return Err(vm.new_type_error("a float is required")); + }; + if offset + 8 <= buffer.len() { + buffer[offset..offset + 8].copy_from_slice(&f64_val.to_ne_bytes()); + } + // For "g" type, remaining bytes stay zero + } + _ => { + // Handle ctypes instances (copy their buffer) + if let Some(cdata) = value.downcast_ref::<PyCData>() { + let src_buffer = cdata.buffer.read(); + let copy_len = src_buffer.len().min(element_size); + if offset + copy_len <= buffer.len() { + buffer[offset..offset + copy_len].copy_from_slice(&src_buffer[..copy_len]); + } + // Other types: use int_to_bytes + } else if let Ok(int_val) = value.try_int(vm) { + let bytes = Self::int_to_bytes(int_val.as_bigint(), element_size); + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&bytes); + } + } else { + return Err(vm.new_type_error(format!( + "expected {} instance, not {}", + type_code.unwrap_or("value"), + value.class().name() + ))); + } + } } + + // KeepRef + if super::base::PyCData::should_keep_ref(value) { + let to_keep = super::base::PyCData::get_kept_objects(value, vm); + zelf.0.keep_ref(index, to_keep, vm)?; + } + + Ok(()) } fn setitem_by_index( - zelf: &PyCArray, + zelf: &Py<PyCArray>, i: isize, value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - let length = zelf.length.load() as isize; + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length) as isize; let index = if i < 0 { length + i } else { i }; if index < 0 || index >= length { - return Err(vm.new_index_error("array index out of range".to_owned())); + return Err(vm.new_index_error("invalid index")); } let index = index as usize; - let element_size = zelf.element_size.load(); + let element_size = stg.as_ref().map_or(0, |i| i.element_size); let offset = index * element_size; + let type_code = Self::get_element_type_code(zelf, vm); + + // Get target buffer and offset (base's buffer if available, otherwise own) + let base_obj = zelf.0.base.read().clone(); + let (buffer_lock, final_offset) = if let Some(cdata) = base_obj + .as_ref() + .and_then(|b| b.downcast_ref::<super::PyCData>()) + { + (&cdata.buffer, zelf.0.base_offset.load() + offset) + } else { + (&zelf.0.buffer, offset) + }; - let int_val = value.try_int(vm)?; - let bytes = Self::int_to_bytes(int_val.as_bigint(), element_size); - - let mut cdata = zelf.cdata.write(); - if offset + element_size <= cdata.buffer.len() { - cdata.buffer[offset..offset + element_size].copy_from_slice(&bytes); + let mut buffer = buffer_lock.write(); + + // For shared memory (Cow::Borrowed), we need to write directly to the memory + // For owned memory (Cow::Owned), we can write to the owned buffer + match &mut *buffer { + Cow::Borrowed(slice) => { + // SAFETY: For from_buffer, the slice points to writable shared memory. + // Python's from_buffer requires writable buffer, so this is safe. + let ptr = slice.as_ptr() as *mut u8; + let len = slice.len(); + let owned_slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; + Self::write_element_to_buffer( + owned_slice, + final_offset, + element_size, + type_code.as_deref(), + &value, + zelf, + index, + vm, + ) + } + Cow::Owned(vec) => Self::write_element_to_buffer( + vec, + final_offset, + element_size, + type_code.as_deref(), + &value, + zelf, + index, + vm, + ), } - Ok(()) } - #[pymethod] - fn __getitem__(&self, index: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(i) = index.downcast_ref::<PyInt>() { + // Array_subscript + fn __getitem__(zelf: &Py<Self>, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { let i = i.as_bigint().to_isize().ok_or_else(|| { - vm.new_index_error("cannot fit index into an index-sized integer".to_owned()) + vm.new_index_error("cannot fit index into an index-sized integer") })?; - Self::getitem_by_index(self, i, vm) + // getitem_by_index handles negative index normalization + Self::getitem_by_index(zelf, i, vm) + } + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::getitem_by_slice(zelf, slice, vm) } else { - Err(vm.new_type_error("array indices must be integers".to_owned())) + Err(vm.new_type_error("indices must be integers")) } } - #[pymethod] + // Array_subscript slice handling + fn getitem_by_slice(zelf: &Py<Self>, slice: &PySlice, vm: &VirtualMachine) -> PyResult { + use crate::sliceable::SaturatedSliceIter; + + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length); + + // PySlice_Unpack + PySlice_AdjustIndices + let sat_slice = slice.to_saturated(vm)?; + let (range, step, slice_len) = sat_slice.adjust_indices(length); + + let type_code = Self::get_element_type_code(zelf, vm); + let element_size = stg.as_ref().map_or(0, |i| i.element_size); + let start = range.start; + + match type_code.as_deref() { + // c_char → bytes (item_info->getfunc == "c") + Some("c") => { + if slice_len == 0 { + return Ok(vm.ctx.new_bytes(vec![]).into()); + } + let buffer = zelf.0.buffer.read(); + // step == 1 optimization: direct memcpy + if step == 1 { + let start_offset = start * element_size; + let end_offset = start_offset + slice_len; + if end_offset <= buffer.len() { + return Ok(vm + .ctx + .new_bytes(buffer[start_offset..end_offset].to_vec()) + .into()); + } + } + // Non-contiguous: iterate + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = Vec::with_capacity(slice_len); + for idx in iter { + let offset = idx * element_size; + if offset < buffer.len() { + result.push(buffer[offset]); + } + } + Ok(vm.ctx.new_bytes(result).into()) + } + // c_wchar → str (item_info->getfunc == "u") + Some("u") => { + if slice_len == 0 { + return Ok(vm.ctx.new_str("").into()); + } + let buffer = zelf.0.buffer.read(); + // step == 1 optimization: direct conversion + if step == 1 { + let start_offset = start * WCHAR_SIZE; + let end_offset = start_offset + slice_len * WCHAR_SIZE; + if end_offset <= buffer.len() { + let wchar_bytes = &buffer[start_offset..end_offset]; + let result: String = wchar_bytes + .chunks(WCHAR_SIZE) + .filter_map(|chunk| wchar_from_bytes(chunk).and_then(char::from_u32)) + .collect(); + return Ok(vm.ctx.new_str(result).into()); + } + } + // Non-contiguous: iterate + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = String::with_capacity(slice_len); + for idx in iter { + let offset = idx * WCHAR_SIZE; + if let Some(code_point) = wchar_from_bytes(&buffer[offset..]) + && let Some(c) = char::from_u32(code_point) + { + result.push(c); + } + } + Ok(vm.ctx.new_str(result).into()) + } + // Other types → list (PyList_New + Array_item for each) + _ => { + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = Vec::with_capacity(slice_len); + for idx in iter { + result.push(Self::getitem_by_index(zelf, idx as isize, vm)?); + } + Ok(PyList::from(result).into_ref(&vm.ctx).into()) + } + } + } + + // Array_ass_subscript fn __setitem__( - &self, - index: PyObjectRef, + zelf: &Py<Self>, + item: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - if let Some(i) = index.downcast_ref::<PyInt>() { + // Array does not support item deletion + // (handled implicitly - value is always provided in __setitem__) + + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { let i = i.as_bigint().to_isize().ok_or_else(|| { - vm.new_index_error("cannot fit index into an index-sized integer".to_owned()) + vm.new_index_error("cannot fit index into an index-sized integer") })?; - Self::setitem_by_index(self, i, value, vm) + // setitem_by_index handles negative index normalization + Self::setitem_by_index(zelf, i, value, vm) + } + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::setitem_by_slice(zelf, slice, value, vm) } else { - Err(vm.new_type_error("array indices must be integers".to_owned())) + Err(vm.new_type_error("indices must be integer")) } } - #[pymethod] - fn __len__(&self) -> usize { - self.length.load() - } - - #[pygetset(name = "_type_")] - fn typ(&self) -> PyObjectRef { - self.typ.read().clone() + // Array does not support item deletion + fn __delitem__(&self, _item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("Array does not support item deletion")) } - #[pygetset(name = "_length_")] - fn length_getter(&self) -> usize { - self.length.load() - } + // Array_ass_subscript slice handling + fn setitem_by_slice( + zelf: &Py<Self>, + slice: &PySlice, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use crate::sliceable::SaturatedSliceIter; - #[pygetset] - fn value(&self, vm: &VirtualMachine) -> PyObjectRef { - // Return bytes representation of the buffer - let buffer = self.cdata.read().buffer.clone(); - vm.ctx.new_bytes(buffer.clone()).into() - } + let length = zelf.class().stg_info_opt().map_or(0, |i| i.length); - #[pygetset(setter)] - fn set_value(&self, value: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { - if let Some(bytes) = value.downcast_ref::<PyBytes>() { - let mut cdata = self.cdata.write(); - let src = bytes.as_bytes(); - let len = std::cmp::min(src.len(), cdata.buffer.len()); - cdata.buffer[..len].copy_from_slice(&src[..len]); - } - Ok(()) - } + // PySlice_Unpack + PySlice_AdjustIndices + let sat_slice = slice.to_saturated(vm)?; + let (range, step, slice_len) = sat_slice.adjust_indices(length); - #[pygetset] - fn raw(&self, vm: &VirtualMachine) -> PyObjectRef { - let cdata = self.cdata.read(); - vm.ctx.new_bytes(cdata.buffer.clone()).into() - } + // other_len = PySequence_Length(value); + let items: Vec<PyObjectRef> = vm.extract_elements_with(&value, Ok)?; + let other_len = items.len(); - #[pygetset(setter)] - fn set_raw(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if let Some(bytes) = value.downcast_ref::<PyBytes>() { - let mut cdata = self.cdata.write(); - let src = bytes.as_bytes(); - let len = std::cmp::min(src.len(), cdata.buffer.len()); - cdata.buffer[..len].copy_from_slice(&src[..len]); - Ok(()) - } else { - Err(vm.new_type_error("expected bytes".to_owned())) + if other_len != slice_len { + return Err(vm.new_value_error("Can only assign sequence of same size")); } - } - - #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; + // Use SaturatedSliceIter for correct index iteration (handles negative step) + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); - // Create instance with data from address - if address == 0 || size == 0 { - return Err(vm.new_value_error("NULL pointer access".to_owned())); - } - unsafe { - let ptr = address as *const u8; - let bytes = std::slice::from_raw_parts(ptr, size); - // Get element type and length from cls - let element_type = cls.as_object().get_attr("_type_", vm)?; - let element_type: PyTypeRef = element_type - .downcast() - .map_err(|_| vm.new_type_error("_type_ must be a type".to_owned()))?; - let length = cls - .as_object() - .get_attr("_length_", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .unwrap_or(0); - let element_size = if length > 0 { size / length } else { 0 }; - - let cdata = CDataObject::from_bytes(bytes.to_vec(), None); - Ok(PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type.into()), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), - } - .into_pyobject(vm)) + for (idx, item) in iter.zip(items) { + Self::setitem_by_index(zelf, idx as isize, item, vm)?; } + Ok(()) } - #[pyclassmethod] - fn from_buffer( - cls: PyTypeRef, - source: PyObjectRef, - offset: crate::function::OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult { - use crate::TryFromObject; - use crate::protocol::PyBuffer; - use crate::stdlib::ctypes::_ctypes::size_of; + fn __len__(zelf: &Py<Self>, _vm: &VirtualMachine) -> usize { + zelf.class().stg_info_opt().map_or(0, |i| i.length) + } +} - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); - } - let offset = offset as usize; +impl AsBuffer for PyCArray { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let buffer_len = zelf.0.buffer.read().len(); + + // Get format and shape from type's StgInfo + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCArray type must have StgInfo"); + let format = stg_info.format.clone(); + let shape = stg_info.shape.clone(); + + let desc = if let Some(fmt) = format + && !shape.is_empty() + { + // itemsize is the size of the base element type (item_info->size) + // For empty arrays, we still need the element size, not 0 + let total_elements: usize = shape.iter().product(); + let has_zero_dim = shape.contains(&0); + let itemsize = if total_elements > 0 && buffer_len > 0 { + buffer_len / total_elements + } else { + // For empty arrays, get itemsize from format type code + get_size_from_format(&fmt) + }; + + // Build dim_desc from shape (C-contiguous: row-major order) + // stride[i] = product(shape[i+1:]) * itemsize + // For empty arrays (any dimension is 0), all strides are 0 + let mut dim_desc = Vec::with_capacity(shape.len()); + let mut stride = itemsize as isize; + + for &dim_size in shape.iter().rev() { + let current_stride = if has_zero_dim { 0 } else { stride }; + dim_desc.push((dim_size, current_stride, 0)); + stride *= dim_size as isize; + } + dim_desc.reverse(); + + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize, + format: alloc::borrow::Cow::Owned(fmt), + dim_desc, + } + } else { + // Fallback to simple buffer if no format/shape info + BufferDescriptor::simple(buffer_len, false) + }; - // Get buffer from source - let buffer = PyBuffer::try_from_object(vm, source.clone())?; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} - // Check if buffer is writable - if buffer.desc.readonly { - return Err(vm.new_type_error("underlying buffer is not writable".to_owned())); - } +// CharArray and WCharArray getsets - added dynamically via add_getset - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; - - // Check if buffer is large enough - let buffer_len = buffer.desc.len; - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); - } +// CharArray_get_value +fn char_array_get_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + let len = buffer.iter().position(|&b| b == 0).unwrap_or(buffer.len()); + Ok(vm.ctx.new_bytes(buffer[..len].to_vec()).into()) +} - // Read bytes from buffer at offset - let bytes = buffer.obj_bytes(); - let data = &bytes[offset..offset + size]; +// CharArray_set_value +fn char_array_set_value(obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let bytes = value + .downcast_ref::<PyBytes>() + .ok_or_else(|| vm.new_type_error("bytes expected"))?; + let mut buffer = zelf.0.buffer.write(); + let src = bytes.as_bytes(); + + if src.len() > buffer.len() { + return Err(vm.new_value_error("byte string too long")); + } - // Get element type and length from cls - let element_type = cls.as_object().get_attr("_type_", vm)?; - let element_type: PyTypeRef = element_type - .downcast() - .map_err(|_| vm.new_type_error("_type_ must be a type".to_owned()))?; - let length = cls - .as_object() - .get_attr("_length_", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .unwrap_or(0); - let element_size = if length > 0 { size / length } else { 0 }; - - let cdata = CDataObject::from_bytes(data.to_vec(), Some(buffer.obj.clone())); - Ok(PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type.into()), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), - } - .into_pyobject(vm)) + buffer.to_mut()[..src.len()].copy_from_slice(src); + if src.len() < buffer.len() { + buffer.to_mut()[src.len()] = 0; } + Ok(()) +} - #[pyclassmethod] - fn from_buffer_copy( - cls: PyTypeRef, - source: crate::function::ArgBytesLike, - offset: crate::function::OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; +// CharArray_get_raw +fn char_array_get_raw(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + Ok(vm.ctx.new_bytes(buffer.to_vec()).into()) +} - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); - } - let offset = offset as usize; - - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; - - // Borrow bytes from source - let source_bytes = source.borrow_buf(); - let buffer_len = source_bytes.len(); - - // Check if buffer is large enough - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); - } +// CharArray_set_raw +fn char_array_set_raw( + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + let value = value.ok_or_else(|| vm.new_attribute_error("cannot delete attribute"))?; + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let bytes_like = ArgBytesLike::try_from_object(vm, value)?; + let mut buffer = zelf.0.buffer.write(); + let src = bytes_like.borrow_buf(); + if src.len() > buffer.len() { + return Err(vm.new_value_error("byte string too long")); + } + buffer.to_mut()[..src.len()].copy_from_slice(&src); + Ok(()) +} - // Copy bytes from buffer at offset - let data = &source_bytes[offset..offset + size]; +// WCharArray_get_value +fn wchar_array_get_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + Ok(vm.ctx.new_str(wstring_from_bytes(&buffer)).into()) +} - // Get element type and length from cls - let element_type = cls.as_object().get_attr("_type_", vm)?; - let element_type: PyTypeRef = element_type - .downcast() - .map_err(|_| vm.new_type_error("_type_ must be a type".to_owned()))?; - let length = cls - .as_object() - .get_attr("_length_", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .unwrap_or(0); - let element_size = if length > 0 { size / length } else { 0 }; - - let cdata = CDataObject::from_bytes(data.to_vec(), None); - Ok(PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type.into()), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), - } - .into_pyobject(vm)) +// WCharArray_set_value +fn wchar_array_set_value( + obj: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let s = value + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("unicode string expected"))?; + let mut buffer = zelf.0.buffer.write(); + let wchar_count = buffer.len() / WCHAR_SIZE; + let char_count = s.as_str().chars().count(); + + if char_count > wchar_count { + return Err(vm.new_value_error("string too long")); } - #[pyclassmethod] - fn in_dll( - cls: PyTypeRef, - dll: PyObjectRef, - name: crate::builtins::PyStrRef, - vm: &VirtualMachine, - ) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; - use libloading::Symbol; - - // Get the library handle from dll object - let handle = if let Ok(int_handle) = dll.try_int(vm) { - // dll is an integer handle - int_handle - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - } else { - // dll is a CDLL/PyDLL/WinDLL object with _handle attribute - dll.get_attr("_handle", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - }; + for (i, ch) in s.as_str().chars().enumerate() { + let offset = i * WCHAR_SIZE; + wchar_to_bytes(ch as u32, &mut buffer.to_mut()[offset..]); + } - // Get the library from cache - let library_cache = crate::stdlib::ctypes::library::libcache().read(); - let library = library_cache - .get_lib(handle) - .ok_or_else(|| vm.new_attribute_error("Library not found".to_owned()))?; + let terminator_offset = char_count * WCHAR_SIZE; + if terminator_offset + WCHAR_SIZE <= buffer.len() { + wchar_to_bytes(0, &mut buffer.to_mut()[terminator_offset..]); + } + Ok(()) +} - // Get symbol address from library - let symbol_name = format!("{}\0", name.as_str()); - let inner_lib = library.lib.lock(); +/// add_getset for c_char arrays - adds 'value' and 'raw' attributes +/// add_getset((PyTypeObject*)self, CharArray_getsets) +fn add_char_array_getsets(array_type: &Py<PyType>, vm: &VirtualMachine) { + // SAFETY: getset is owned by array_type which outlives the getset + let value_getset = unsafe { + vm.ctx.new_getset( + "value", + array_type, + char_array_get_value, + char_array_set_value, + ) + }; + let raw_getset = unsafe { + vm.ctx + .new_getset("raw", array_type, char_array_get_raw, char_array_set_raw) + }; + + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("value"), value_getset.into()); + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("raw"), raw_getset.into()); +} - let symbol_address = if let Some(lib) = &*inner_lib { - unsafe { - // Try to get the symbol from the library - let symbol: Symbol<'_, *mut u8> = lib.get(symbol_name.as_bytes()).map_err(|e| { - vm.new_attribute_error(format!("{}: symbol '{}' not found", e, name.as_str())) - })?; - *symbol as usize - } - } else { - return Err(vm.new_attribute_error("Library is closed".to_owned())); - }; +/// add_getset for c_wchar arrays - adds only 'value' attribute (no 'raw') +fn add_wchar_array_getsets(array_type: &Py<PyType>, vm: &VirtualMachine) { + // SAFETY: getset is owned by array_type which outlives the getset + let value_getset = unsafe { + vm.ctx.new_getset( + "value", + array_type, + wchar_array_get_value, + wchar_array_set_value, + ) + }; - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("value"), value_getset.into()); +} - // Read data from symbol address - let data = if symbol_address != 0 && size > 0 { - unsafe { - let ptr = symbol_address as *const u8; - std::slice::from_raw_parts(ptr, size).to_vec() - } - } else { - vec![0; size] - }; +// wchar_t helpers - Platform-independent wide character handling +// Windows: sizeof(wchar_t) == 2 (UTF-16) +// Linux/macOS: sizeof(wchar_t) == 4 (UTF-32) - // Get element type and length from cls - let element_type = cls.as_object().get_attr("_type_", vm)?; - let element_type: PyTypeRef = element_type - .downcast() - .map_err(|_| vm.new_type_error("_type_ must be a type".to_owned()))?; - let length = cls - .as_object() - .get_attr("_length_", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .unwrap_or(0); - let element_size = if length > 0 { size / length } else { 0 }; - - // Create instance - let cdata = CDataObject::from_bytes(data, None); - let instance = PyCArray { - _base: PyCData::new(cdata.clone()), - typ: PyRwLock::new(element_type.into()), - length: AtomicCell::new(length), - element_size: AtomicCell::new(element_size), - cdata: PyRwLock::new(cdata), - } - .into_pyobject(vm); +/// Size of wchar_t on this platform +pub(super) const WCHAR_SIZE: usize = core::mem::size_of::<libc::wchar_t>(); - // Store base reference to keep dll alive - if let Ok(array_ref) = instance.clone().downcast::<PyCArray>() { - array_ref.cdata.write().base = Some(dll); - } - - Ok(instance) +/// Read a single wchar_t from bytes (platform-endian) +#[inline] +pub(super) fn wchar_from_bytes(bytes: &[u8]) -> Option<u32> { + if bytes.len() < WCHAR_SIZE { + return None; } + Some(if WCHAR_SIZE == 2 { + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }) } -impl PyCArray { - #[allow(unused)] - pub fn to_arg(&self, _vm: &VirtualMachine) -> PyResult<libffi::middle::Arg> { - let cdata = self.cdata.read(); - Ok(libffi::middle::Arg::new(&cdata.buffer)) +/// Write a single wchar_t to bytes (platform-endian) +#[inline] +pub(super) fn wchar_to_bytes(ch: u32, buffer: &mut [u8]) { + if WCHAR_SIZE == 2 { + if buffer.len() >= 2 { + buffer[..2].copy_from_slice(&(ch as u16).to_ne_bytes()); + } + } else if buffer.len() >= 4 { + buffer[..4].copy_from_slice(&ch.to_ne_bytes()); } } -static ARRAY_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - rustpython_common::lock::PyMappedRwLockReadGuard::map( - rustpython_common::lock::PyRwLockReadGuard::map( - buffer.obj_as::<PyCArray>().cdata.read(), - |x: &CDataObject| x, - ), - |x: &CDataObject| x.buffer.as_slice(), - ) - .into() - }, - obj_bytes_mut: |buffer| { - rustpython_common::lock::PyMappedRwLockWriteGuard::map( - rustpython_common::lock::PyRwLockWriteGuard::map( - buffer.obj_as::<PyCArray>().cdata.write(), - |x: &mut CDataObject| x, - ), - |x: &mut CDataObject| x.buffer.as_mut_slice(), - ) - .into() - }, - release: |_| {}, - retain: |_| {}, -}; - -impl AsBuffer for PyCArray { - fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { - let buffer_len = zelf.cdata.read().buffer.len(); - let buf = PyBuffer::new( - zelf.to_owned().into(), - BufferDescriptor::simple(buffer_len, false), // readonly=false for ctypes - &ARRAY_BUFFER_METHODS, - ); - Ok(buf) +/// Read a null-terminated wchar_t string from bytes, returns String +fn wstring_from_bytes(buffer: &[u8]) -> String { + let mut chars = Vec::new(); + for chunk in buffer.chunks(WCHAR_SIZE) { + if chunk.len() < WCHAR_SIZE { + break; + } + let code = if WCHAR_SIZE == 2 { + u16::from_ne_bytes([chunk[0], chunk[1]]) as u32 + } else { + u32::from_ne_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) + }; + if code == 0 { + break; // null terminator + } + if let Some(ch) = char::from_u32(code) { + chars.push(ch); + } } + chars.into_iter().collect() } diff --git a/crates/vm/src/stdlib/ctypes/base.rs b/crates/vm/src/stdlib/ctypes/base.rs index a4664ad3671..ba2b987330a 100644 --- a/crates/vm/src/stdlib/ctypes/base.rs +++ b/crates/vm/src/stdlib/ctypes/base.rs @@ -1,1063 +1,2606 @@ -use super::_ctypes::bytes_to_pyobject; -use super::util::StgInfo; -use crate::builtins::{PyBytes, PyFloat, PyInt, PyNone, PyStr, PyStrRef, PyType, PyTypeRef}; -use crate::function::{ArgBytesLike, Either, FuncArgs, KwArgs, OptionalArg}; -use crate::protocol::{BufferDescriptor, BufferMethods, PyBuffer, PyNumberMethods}; -use crate::stdlib::ctypes::_ctypes::new_simple_type; -use crate::types::{AsBuffer, AsNumber, Constructor}; -use crate::{AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine}; +use super::array::{WCHAR_SIZE, wchar_from_bytes, wchar_to_bytes}; +use crate::builtins::{PyBytes, PyDict, PyMemoryView, PyStr, PyTuple, PyType, PyTypeRef}; +use crate::class::StaticType; +use crate::function::{ArgBytesLike, OptionalArg, PySetterValue}; +use crate::protocol::{BufferMethods, PyBuffer}; +use crate::types::{Constructor, GetDescriptor, Representable}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, +}; +use alloc::borrow::Cow; +use core::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_short, c_uint, c_ulong, c_ulonglong, c_ushort, +}; +use core::fmt::Debug; +use core::mem; use crossbeam_utils::atomic::AtomicCell; -use num_traits::ToPrimitive; +use num_traits::{Signed, ToPrimitive}; use rustpython_common::lock::PyRwLock; -use std::ffi::{c_uint, c_ulong, c_ulonglong, c_ushort}; -use std::fmt::Debug; +use widestring::WideChar; + +// StgInfo - Storage information for ctypes types +// Stored in TypeDataSlot of heap types (PyType::init_type_data/get_type_data) + +// Flag constants +bitflags::bitflags! { + #[derive(Default, Copy, Clone, Debug, PartialEq, Eq)] + pub struct StgInfoFlags: u32 { + // Function calling convention flags + /// Standard call convention (Windows) + const FUNCFLAG_STDCALL = 0x0; + /// C calling convention + const FUNCFLAG_CDECL = 0x1; + /// Function returns HRESULT + const FUNCFLAG_HRESULT = 0x2; + /// Use Python API calling convention + const FUNCFLAG_PYTHONAPI = 0x4; + /// Capture errno after call + const FUNCFLAG_USE_ERRNO = 0x8; + /// Capture last error after call (Windows) + const FUNCFLAG_USE_LASTERROR = 0x10; + + // Type flags + /// Type is a pointer type + const TYPEFLAG_ISPOINTER = 0x100; + /// Type contains pointer fields + const TYPEFLAG_HASPOINTER = 0x200; + /// Type is or contains a union + const TYPEFLAG_HASUNION = 0x400; + /// Type contains bitfield members + const TYPEFLAG_HASBITFIELD = 0x800; + + // Dict flags + /// Type is finalized (_fields_ has been set) + const DICTFLAG_FINAL = 0x1000; + } +} -/// Get the type code string from a ctypes type (e.g., "i" for c_int) -pub fn get_type_code(cls: &PyTypeRef, vm: &VirtualMachine) -> Option<String> { - cls.as_object() - .get_attr("_type_", vm) - .ok() - .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())) +/// ParamFunc - determines how a type is passed to foreign functions +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum ParamFunc { + #[default] + None, + /// Array types are passed as pointers (tag = 'P') + Array, + /// Simple types use their specific conversion (tag = type code) + Simple, + /// Pointer types (tag = 'P') + Pointer, + /// Structure types (tag = 'V' for value) + Structure, + /// Union types (tag = 'V' for value) + Union, +} + +#[derive(Clone)] +pub struct StgInfo { + pub initialized: bool, + pub size: usize, // number of bytes + pub align: usize, // alignment requirements + pub length: usize, // number of fields (for arrays/structures) + pub proto: Option<PyTypeRef>, // Only for Pointer/ArrayObject + pub flags: StgInfoFlags, // type flags (TYPEFLAG_*, DICTFLAG_*) + + // Array-specific fields + pub element_type: Option<PyTypeRef>, // _type_ for arrays + pub element_size: usize, // size of each element + + // PEP 3118 buffer protocol fields + pub format: Option<String>, // struct format string (e.g., "i", "(5)i") + pub shape: Vec<usize>, // shape for multi-dimensional arrays + + // Function parameter conversion + pub(super) paramfunc: ParamFunc, // how to pass to foreign functions + + // Byte order (for _swappedbytes_) + pub big_endian: bool, // true if big endian, false if little endian + + // FFI field types for structure/union passing (inherited from base class) + pub ffi_field_types: Vec<libffi::middle::Type>, + + // Cached pointer type (non-inheritable via descriptor) + pub pointer_type: Option<PyObjectRef>, } -pub fn ffi_type_from_str(_type_: &str) -> Option<libffi::middle::Type> { - match _type_ { - "c" => Some(libffi::middle::Type::u8()), - "u" => Some(libffi::middle::Type::u32()), - "b" => Some(libffi::middle::Type::i8()), - "B" => Some(libffi::middle::Type::u8()), - "h" => Some(libffi::middle::Type::i16()), - "H" => Some(libffi::middle::Type::u16()), - "i" => Some(libffi::middle::Type::i32()), - "I" => Some(libffi::middle::Type::u32()), - "l" => Some(libffi::middle::Type::i32()), - "L" => Some(libffi::middle::Type::u32()), - "q" => Some(libffi::middle::Type::i64()), - "Q" => Some(libffi::middle::Type::u64()), - "f" => Some(libffi::middle::Type::f32()), - "d" => Some(libffi::middle::Type::f64()), - "g" => Some(libffi::middle::Type::f64()), - "?" => Some(libffi::middle::Type::u8()), - "z" => Some(libffi::middle::Type::u64()), - "Z" => Some(libffi::middle::Type::u64()), - "P" => Some(libffi::middle::Type::u64()), - _ => None, +// StgInfo is stored in type_data which requires Send + Sync. +// The PyTypeRef in proto/element_type fields is protected by the type system's locking mechanism. +// ctypes objects are not thread-safe by design; users must synchronize access. +unsafe impl Send for StgInfo {} +unsafe impl Sync for StgInfo {} + +impl core::fmt::Debug for StgInfo { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("StgInfo") + .field("initialized", &self.initialized) + .field("size", &self.size) + .field("align", &self.align) + .field("length", &self.length) + .field("proto", &self.proto) + .field("flags", &self.flags) + .field("element_type", &self.element_type) + .field("element_size", &self.element_size) + .field("format", &self.format) + .field("shape", &self.shape) + .field("paramfunc", &self.paramfunc) + .field("big_endian", &self.big_endian) + .field("ffi_field_types", &self.ffi_field_types.len()) + .finish() } } -#[allow(dead_code)] -fn set_primitive(_type_: &str, value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - match _type_ { - "c" => { - if value - .clone() - .downcast_exact::<PyBytes>(vm) - .is_ok_and(|v| v.len() == 1) - || value - .clone() - .downcast_exact::<PyBytes>(vm) - .is_ok_and(|v| v.len() == 1) - || value - .clone() - .downcast_exact::<PyInt>(vm) - .map_or(Ok(false), |v| { - let n = v.as_bigint().to_i64(); - if let Some(n) = n { - Ok((0..=255).contains(&n)) - } else { - Ok(false) - } - })? - { - Ok(value.clone()) - } else { - Err(vm.new_type_error("one character bytes, bytearray or integer expected")) - } +impl Default for StgInfo { + fn default() -> Self { + StgInfo { + initialized: false, + size: 0, + align: 1, + length: 0, + proto: None, + flags: StgInfoFlags::empty(), + element_type: None, + element_size: 0, + format: None, + shape: Vec::new(), + paramfunc: ParamFunc::None, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, } - "u" => { - if let Ok(b) = value.str(vm).map(|v| v.to_string().chars().count() == 1) { - if b { - Ok(value.clone()) + } +} + +impl StgInfo { + pub fn new(size: usize, align: usize) -> Self { + StgInfo { + initialized: true, + size, + align, + length: 0, + proto: None, + flags: StgInfoFlags::empty(), + element_type: None, + element_size: 0, + format: None, + shape: Vec::new(), + paramfunc: ParamFunc::None, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, + } + } + + /// Create StgInfo for an array type + /// item_format: the innermost element's format string (kept as-is, e.g., "<i") + /// item_shape: the element's shape (will be prepended with length) + /// item_flags: the element type's flags (for HASPOINTER inheritance) + #[allow(clippy::too_many_arguments)] + pub fn new_array( + size: usize, + align: usize, + length: usize, + element_type: PyTypeRef, + element_size: usize, + item_format: Option<&str>, + item_shape: &[usize], + item_flags: StgInfoFlags, + ) -> Self { + // Format is kept from innermost element (e.g., "<i" for c_int arrays) + // The array dimensions go into shape only, not format + let format = item_format.map(|f| f.to_owned()); + + // Build shape: [length, ...item_shape] + let mut shape = vec![length]; + shape.extend_from_slice(item_shape); + + // Inherit HASPOINTER flag from element type + // if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + // stginfo->flags |= TYPEFLAG_HASPOINTER; + let flags = if item_flags + .intersects(StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER) + { + StgInfoFlags::TYPEFLAG_HASPOINTER + } else { + StgInfoFlags::empty() + }; + + StgInfo { + initialized: true, + size, + align, + length, + proto: None, + flags, + element_type: Some(element_type), + element_size, + format, + shape, + paramfunc: ParamFunc::Array, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, + } + } + + /// Get libffi type for this StgInfo + /// Note: For very large types, returns pointer type to avoid overflow + pub fn to_ffi_type(&self) -> libffi::middle::Type { + // Limit to avoid overflow in libffi (MAX_STRUCT_SIZE is platform-dependent) + const MAX_FFI_STRUCT_SIZE: usize = 1024 * 1024; // 1MB limit for safety + + match self.paramfunc { + ParamFunc::Structure | ParamFunc::Union => { + if !self.ffi_field_types.is_empty() { + libffi::middle::Type::structure(self.ffi_field_types.iter().cloned()) + } else if self.size <= MAX_FFI_STRUCT_SIZE { + // Small struct without field types: use bytes array + libffi::middle::Type::structure(core::iter::repeat_n( + libffi::middle::Type::u8(), + self.size, + )) } else { - Err(vm.new_type_error("one character unicode string expected")) + // Large struct: treat as pointer (passed by reference) + libffi::middle::Type::pointer() } - } else { - Err(vm.new_type_error(format!( - "unicode string expected instead of {} instance", - value.class().name() - ))) } - } - "b" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { - if value.clone().downcast_exact::<PyInt>(vm).is_ok() { - Ok(value.clone()) - } else { - Err(vm.new_type_error(format!( - "an integer is required (got type {})", - value.class().name() - ))) + ParamFunc::Array => { + if self.size > MAX_FFI_STRUCT_SIZE || self.length > MAX_FFI_STRUCT_SIZE { + // Large array: treat as pointer + libffi::middle::Type::pointer() + } else if let Some(ref fmt) = self.format { + let elem_type = Self::format_to_ffi_type(fmt); + libffi::middle::Type::structure(core::iter::repeat_n(elem_type, self.length)) + } else { + libffi::middle::Type::structure(core::iter::repeat_n( + libffi::middle::Type::u8(), + self.size, + )) + } } - } - "f" | "d" | "g" => { - // float allows int - if value.clone().downcast_exact::<PyFloat>(vm).is_ok() - || value.clone().downcast_exact::<PyInt>(vm).is_ok() - { - Ok(value.clone()) - } else { - Err(vm.new_type_error(format!("must be real number, not {}", value.class().name()))) + ParamFunc::Pointer => libffi::middle::Type::pointer(), + _ => { + // Simple type: derive from format + if let Some(ref fmt) = self.format { + Self::format_to_ffi_type(fmt) + } else { + libffi::middle::Type::u8() + } } } - "?" => Ok(PyObjectRef::from( - vm.ctx.new_bool(value.clone().try_to_bool(vm)?), - )), - "B" => { - if value.clone().downcast_exact::<PyInt>(vm).is_ok() { - // Store as-is, conversion to unsigned happens in the getter - Ok(value.clone()) - } else { - Err(vm.new_type_error(format!("int expected instead of {}", value.class().name()))) - } + } + + /// Convert format string to libffi type + fn format_to_ffi_type(fmt: &str) -> libffi::middle::Type { + // Strip endian prefix if present + let code = fmt.trim_start_matches(['<', '>', '!', '@', '=']); + match code { + "b" => libffi::middle::Type::i8(), + "B" => libffi::middle::Type::u8(), + "h" => libffi::middle::Type::i16(), + "H" => libffi::middle::Type::u16(), + "i" | "l" => libffi::middle::Type::i32(), + "I" | "L" => libffi::middle::Type::u32(), + "q" => libffi::middle::Type::i64(), + "Q" => libffi::middle::Type::u64(), + "f" => libffi::middle::Type::f32(), + "d" => libffi::middle::Type::f64(), + "P" | "z" | "Z" | "O" => libffi::middle::Type::pointer(), + _ => libffi::middle::Type::u8(), // default } - "z" => { - if value.clone().downcast_exact::<PyInt>(vm).is_ok() - || value.clone().downcast_exact::<PyBytes>(vm).is_ok() - { - Ok(value.clone()) - } else { - Err(vm.new_type_error(format!( - "bytes or integer address expected instead of {} instance", - value.class().name() - ))) - } + } + + /// Check if this type is finalized (cannot set _fields_ again) + pub fn is_final(&self) -> bool { + self.flags.contains(StgInfoFlags::DICTFLAG_FINAL) + } + + /// Get proto type reference (for Pointer/Array types) + pub fn proto(&self) -> &Py<PyType> { + self.proto.as_deref().expect("type has proto") + } +} + +/// __pointer_type__ getter for ctypes metaclasses. +/// Reads from StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_get(zelf: &Py<PyType>, vm: &VirtualMachine) -> PyResult { + zelf.stg_info_opt() + .and_then(|info| info.pointer_type.clone()) + .ok_or_else(|| { + vm.new_attribute_error(format!( + "type {} has no attribute '__pointer_type__'", + zelf.name() + )) + }) +} + +/// __pointer_type__ setter for ctypes metaclasses. +/// Writes to StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_set( + zelf: &Py<PyType>, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(mut info) = zelf.get_type_data_mut::<StgInfo>() { + info.pointer_type = Some(value); + Ok(()) + } else { + Err(vm.new_attribute_error(format!("cannot set __pointer_type__ on {}", zelf.name()))) + } +} + +/// Get PEP3118 format string for a field type +/// Returns the format string considering byte order +pub(super) fn get_field_format( + field_type: &PyObject, + big_endian: bool, + vm: &VirtualMachine, +) -> String { + let endian_prefix = if big_endian { ">" } else { "<" }; + + // 1. Check StgInfo for format + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + && let Some(fmt) = &stg_info.format + { + // For structures (T{...}), arrays ((n)...), and pointers (&...), return as-is + // These complex types have their own endianness markers inside + if fmt.starts_with('T') + || fmt.starts_with('(') + || fmt.starts_with('&') + || fmt.starts_with("X{") + { + return fmt.clone(); } - "Z" => { - if value.clone().downcast_exact::<PyStr>(vm).is_ok() { - Ok(value.clone()) - } else { - Err(vm.new_type_error(format!( - "unicode string or integer address expected instead of {} instance", - value.class().name() - ))) - } + + // For simple types, replace existing endian prefix with the correct one + let base_fmt = fmt.trim_start_matches(['<', '>', '@', '=', '!']); + if !base_fmt.is_empty() { + return format!("{}{}", endian_prefix, base_fmt); } - _ => { - // "P" - if value.clone().downcast_exact::<PyInt>(vm).is_ok() - || value.clone().downcast_exact::<PyNone>(vm).is_ok() - { - Ok(value.clone()) - } else { - Err(vm.new_type_error("cannot be converted to pointer")) - } + return fmt.clone(); + } + + // 2. Try to get _type_ attribute for simple types + if let Ok(type_attr) = field_type.get_attr("_type_", vm) + && let Some(type_str) = type_attr.downcast_ref::<PyStr>() + { + let s = type_str.as_str(); + if !s.is_empty() { + return format!("{}{}", endian_prefix, s); } + return s.to_string(); } + + // Default: single byte + "B".to_string() } -/// Common data object for all ctypes types -#[derive(Debug, Clone)] -pub struct CDataObject { - /// pointer to memory block (b_ptr + b_size) - pub buffer: Vec<u8>, +/// Compute byte order based on swapped flag +#[inline] +pub(super) fn is_big_endian(is_swapped: bool) -> bool { + if is_swapped { + !cfg!(target_endian = "big") + } else { + cfg!(target_endian = "big") + } +} + +/// Shared BufferMethods for all ctypes types (PyCArray, PyCSimple, PyCStructure, PyCUnion) +/// All these types are #[repr(transparent)] wrappers around PyCData +pub(super) static CDATA_BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| { + rustpython_common::lock::PyRwLockReadGuard::map( + buffer.obj_as::<PyCData>().buffer.read(), + |x| &**x, + ) + .into() + }, + obj_bytes_mut: |buffer| { + rustpython_common::lock::PyRwLockWriteGuard::map( + buffer.obj_as::<PyCData>().buffer.write(), + |x| x.to_mut().as_mut_slice(), + ) + .into() + }, + release: |_| {}, + retain: |_| {}, +}; + +/// Convert Vec<T> to Vec<u8> by reinterpreting the memory (same allocation). +fn vec_to_bytes<T>(vec: Vec<T>) -> Vec<u8> { + let len = vec.len() * core::mem::size_of::<T>(); + let cap = vec.capacity() * core::mem::size_of::<T>(); + let ptr = vec.as_ptr() as *mut u8; + core::mem::forget(vec); + unsafe { Vec::from_raw_parts(ptr, len, cap) } +} + +/// Ensure PyBytes data is null-terminated. Returns (kept_alive_obj, pointer). +/// The caller must keep the returned object alive to keep the pointer valid. +pub(super) fn ensure_z_null_terminated( + bytes: &PyBytes, + vm: &VirtualMachine, +) -> (PyObjectRef, usize) { + let data = bytes.as_bytes(); + let mut buffer = data.to_vec(); + if !buffer.ends_with(&[0]) { + buffer.push(0); + } + let ptr = buffer.as_ptr() as usize; + let kept_alive: PyObjectRef = vm.ctx.new_bytes(buffer).into(); + (kept_alive, ptr) +} + +/// Convert str to null-terminated wchar_t buffer. Returns (PyBytes holder, pointer). +pub(super) fn str_to_wchar_bytes(s: &str, vm: &VirtualMachine) -> (PyObjectRef, usize) { + let wchars: Vec<libc::wchar_t> = s + .chars() + .map(|c| c as libc::wchar_t) + .chain(core::iter::once(0)) + .collect(); + let ptr = wchars.as_ptr() as usize; + let bytes = vec_to_bytes(wchars); + let holder: PyObjectRef = vm.ctx.new_bytes(bytes).into(); + (holder, ptr) +} + +/// PyCData - base type for all ctypes data types +#[pyclass(name = "_CData", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub struct PyCData { + /// Memory buffer - Owned (self-owned) or Borrowed (external reference) + /// + /// SAFETY: Borrowed variant's 'static lifetime is not actually static. + /// When created via from_address or from_base_obj, only valid for the lifetime of the source memory. + /// Same behavior as CPython's b_ptr (user responsibility, kept alive via b_base). + pub buffer: PyRwLock<Cow<'static, [u8]>>, /// pointer to base object or None (b_base) - #[allow(dead_code)] - pub base: Option<PyObjectRef>, + pub base: PyRwLock<Option<PyObjectRef>>, + /// byte offset within base's buffer (for field access) + pub base_offset: AtomicCell<usize>, /// index into base's b_objects list (b_index) - #[allow(dead_code)] - pub index: usize, + pub index: AtomicCell<usize>, /// dictionary of references we need to keep (b_objects) - pub objects: Option<PyObjectRef>, + pub objects: PyRwLock<Option<PyObjectRef>>, + /// number of references we need (b_length) + pub length: AtomicCell<usize>, + /// References kept alive but not visible in _objects. + /// Used for null-terminated c_char_p buffer copies, since + /// RustPython's PyBytes lacks CPython's internal trailing null. + /// Keyed by unique_key (hierarchical) so nested fields don't collide. + pub(super) kept_refs: PyRwLock<std::collections::HashMap<String, PyObjectRef>>, } -impl CDataObject { +impl PyCData { /// Create from StgInfo (PyCData_MallocBuffer pattern) pub fn from_stg_info(stg_info: &StgInfo) -> Self { - CDataObject { - buffer: vec![0u8; stg_info.size], - base: None, - index: 0, - objects: None, + PyCData { + buffer: PyRwLock::new(Cow::Owned(vec![0u8; stg_info.size])), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(None), + length: AtomicCell::new(stg_info.length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } /// Create from existing bytes (copies data) pub fn from_bytes(data: Vec<u8>, objects: Option<PyObjectRef>) -> Self { - CDataObject { - buffer: data, - base: None, - index: 0, - objects, + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(objects), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } - /// Create from base object (copies data from base's buffer at offset) - #[allow(dead_code)] - pub fn from_base( - base: PyObjectRef, - _offset: usize, - size: usize, - index: usize, + /// Create from bytes with specified length (for arrays) + pub fn from_bytes_with_length( + data: Vec<u8>, objects: Option<PyObjectRef>, + length: usize, ) -> Self { - CDataObject { - buffer: vec![0u8; size], - base: Some(base), - index, - objects, + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(objects), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } - #[inline] - pub fn size(&self) -> usize { - self.buffer.len() + /// Create from external memory address + /// + /// # Safety + /// The returned slice's 'static lifetime is a lie. + /// Actually only valid for the lifetime of the memory pointed to by ptr. + /// PyCData_AtAddress + pub unsafe fn at_address(ptr: *const u8, size: usize) -> Self { + // = PyCData_AtAddress + // SAFETY: Caller must ensure ptr is valid for the lifetime of returned PyCData + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(None), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } } -} - -#[pyclass(name = "_CData", module = "_ctypes")] -#[derive(Debug, PyPayload)] -pub struct PyCData { - pub cdata: PyRwLock<CDataObject>, -} -impl PyCData { - pub fn new(cdata: CDataObject) -> Self { - Self { - cdata: PyRwLock::new(cdata), + /// Create from base object with offset and data copy + /// + /// Similar to from_base_with_offset, but also stores a copy of the data. + /// This is used for arrays where we need our own buffer for the buffer protocol, + /// but still maintain the base reference for KeepRef and tracking. + pub fn from_base_with_data( + base_obj: PyObjectRef, + offset: usize, + idx: usize, + length: usize, + data: Vec<u8>, + ) -> Self { + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), // Has its own buffer copy + base: PyRwLock::new(Some(base_obj)), // But still tracks base + base_offset: AtomicCell::new(offset), // And offset for writes + index: AtomicCell::new(idx), + objects: PyRwLock::new(None), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } -} -#[pyclass(flags(BASETYPE))] -impl PyCData { - #[pygetset] - fn _objects(&self) -> Option<PyObjectRef> { - self.cdata.read().objects.clone() + /// Create from base object's buffer + /// + /// This creates a borrowed view into the base's buffer at the given address. + /// The base object is stored in b_base to keep the memory alive. + /// + /// # Safety + /// ptr must point into base_obj's buffer and remain valid as long as base_obj is alive. + pub unsafe fn from_base_obj( + ptr: *mut u8, + size: usize, + base_obj: PyObjectRef, + idx: usize, + ) -> Self { + // = PyCData_FromBaseObj + // SAFETY: ptr points into base_obj's buffer, kept alive via base reference + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(Some(base_obj)), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(idx), + objects: PyRwLock::new(None), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } } -} -#[pyclass(module = "_ctypes", name = "PyCSimpleType", base = PyType)] -#[derive(Debug)] -#[repr(transparent)] -pub struct PyCSimpleType(PyType); - -#[pyclass(flags(BASETYPE), with(AsNumber))] -impl PyCSimpleType { - /// Get stg_info for a simple type by reading _type_ attribute - pub fn get_stg_info(cls: &PyTypeRef, vm: &VirtualMachine) -> StgInfo { - if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) - && let Ok(type_str) = type_attr.str(vm) - { - let tp_str = type_str.to_string(); - if tp_str.len() == 1 { - let size = super::_ctypes::get_size(&tp_str); - let align = super::_ctypes::get_align(&tp_str); - return StgInfo::new(size, align); - } + /// Create from buffer protocol object (for from_buffer method) + /// + /// Unlike from_bytes, this shares memory with the source buffer. + /// The source object is stored in objects dict to keep the buffer alive. + /// Python stores with key -1 via KeepRef(result, -1, mv). + /// + /// # Safety + /// ptr must point to valid memory that remains valid as long as source is alive. + pub unsafe fn from_buffer_shared( + ptr: *const u8, + size: usize, + length: usize, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> Self { + // SAFETY: Caller must ensure ptr is valid for the lifetime of source + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + + // Python stores the reference in a dict with key "-1" (unique_key pattern) + let objects_dict = vm.ctx.new_dict(); + objects_dict + .set_item("-1", source, vm) + .expect("Failed to store buffer reference"); + + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(Some(objects_dict.into())), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } - StgInfo::default() - } - #[allow(clippy::new_ret_no_self)] - #[pymethod] - fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { - Ok(PyObjectRef::from( - new_simple_type(Either::B(&cls), vm)? - .into_ref_with_type(vm, cls)? - .clone(), - )) } - #[pyclassmethod] - fn from_param(cls: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // 1. If the value is already an instance of the requested type, return it - if value.fast_isinstance(&cls) { - return Ok(value); + /// Common implementation for from_buffer class method. + /// Validates buffer, creates memoryview, and returns PyCData sharing memory with source. + /// + /// CDataType_from_buffer_impl + pub fn from_buffer_impl( + cls: &Py<PyType>, + source: PyObjectRef, + offset: isize, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let (size, length) = { + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + (stg_info.size, stg_info.length) + }; + + if offset < 0 { + return Err(vm.new_value_error("offset cannot be negative")); } + let offset = offset as usize; - // 2. Get the type code to determine conversion rules - let type_code = get_type_code(&cls, vm); + // Get buffer from source (this exports the buffer) + let buffer = PyBuffer::try_from_object(vm, source)?; - // 3. Handle None for pointer types (c_char_p, c_wchar_p, c_void_p) - if vm.is_none(&value) && matches!(type_code.as_deref(), Some("z") | Some("Z") | Some("P")) { - return Ok(value); + // Check if buffer is writable + if buffer.desc.readonly { + return Err(vm.new_type_error("underlying buffer is not writable")); } - // 4. Try to convert value based on type code - match type_code.as_deref() { - // Integer types: accept integers - Some("b" | "B" | "h" | "H" | "i" | "I" | "l" | "L" | "q" | "Q") => { - if value.try_int(vm).is_ok() { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - } - // Float types: accept numbers - Some("f" | "d" | "g") => { - if value.try_float(vm).is_ok() || value.try_int(vm).is_ok() { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - } - // c_char: 1 byte character - Some("c") => { - if let Some(bytes) = value.downcast_ref::<PyBytes>() - && bytes.len() == 1 - { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - if let Ok(int_val) = value.try_int(vm) - && int_val.as_bigint().to_u8().is_some() - { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - return Err(vm.new_type_error( - "one character bytes, bytearray or integer expected".to_string(), - )); - } - // c_wchar: 1 unicode character - Some("u") => { - if let Some(s) = value.downcast_ref::<PyStr>() - && s.as_str().chars().count() == 1 - { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - return Err(vm.new_type_error("one character unicode string expected".to_string())); - } - // c_char_p: bytes pointer - Some("z") => { - if value.downcast_ref::<PyBytes>().is_some() { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - } - // c_wchar_p: unicode pointer - Some("Z") => { - if value.downcast_ref::<PyStr>().is_some() { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - } - // c_void_p: most flexible - accepts int, bytes, str - Some("P") => { - if value.try_int(vm).is_ok() - || value.downcast_ref::<PyBytes>().is_some() - || value.downcast_ref::<PyStr>().is_some() - { - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(value.clone()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - } - // c_bool - Some("?") => { - let bool_val = value.is_true(vm)?; - let simple = new_simple_type(Either::B(&cls), vm)?; - simple.value.store(vm.ctx.new_bool(bool_val).into()); - return simple.into_ref_with_type(vm, cls.clone()).map(Into::into); - } - _ => {} + // Check if buffer is C contiguous + if !buffer.desc.is_contiguous() { + return Err(vm.new_type_error("underlying buffer is not C contiguous")); } - // 5. Check for _as_parameter_ attribute - if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { - return PyCSimpleType::from_param(cls, as_parameter, vm); + // Check if buffer is large enough + let buffer_len = buffer.desc.len; + if offset + size > buffer_len { + return Err(vm.new_value_error(format!( + "Buffer size too small ({} instead of at least {} bytes)", + buffer_len, + offset + size + ))); } - // 6. Type-specific error messages - match type_code.as_deref() { - Some("z") => Err(vm.new_type_error(format!( - "'{}' object cannot be interpreted as ctypes.c_char_p", - value.class().name() - ))), - Some("Z") => Err(vm.new_type_error(format!( - "'{}' object cannot be interpreted as ctypes.c_wchar_p", - value.class().name() - ))), - _ => Err(vm.new_type_error("wrong type".to_string())), - } - } + // Get buffer pointer - the memory is owned by source + let ptr = { + let bytes = buffer.obj_bytes(); + bytes.as_ptr().wrapping_add(offset) + }; - #[pymethod] - fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { - PyCSimple::repeat(cls, n, vm) + // Create memoryview to keep buffer exported (prevents source from being modified) + // mv = PyMemoryView_FromObject(obj); KeepRef(result, -1, mv); + let memoryview = PyMemoryView::from_buffer(buffer, vm)?; + let mv_obj = memoryview.into_pyobject(vm); + + // Create CData that shares memory with the buffer + Ok(unsafe { Self::from_buffer_shared(ptr, size, length, mv_obj, vm) }) } -} -impl AsNumber for PyCSimpleType { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - multiply: Some(|a, b, vm| { - // a is a PyCSimpleType instance (type object like c_char) - // b is int (array size) - let cls = a - .downcast_ref::<PyType>() - .ok_or_else(|| vm.new_type_error("expected type".to_owned()))?; - let n = b - .try_index(vm)? - .as_bigint() - .to_isize() - .ok_or_else(|| vm.new_overflow_error("array size too large".to_owned()))?; - PyCSimple::repeat(cls.to_owned(), n, vm) - }), - ..PyNumberMethods::NOT_IMPLEMENTED + /// Common implementation for from_buffer_copy class method. + /// Copies data from buffer and creates new independent instance. + /// + /// CDataType_from_buffer_copy_impl + pub fn from_buffer_copy_impl( + cls: &Py<PyType>, + source: &[u8], + offset: isize, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let (size, length) = { + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + (stg_info.size, stg_info.length) }; - &AS_NUMBER - } -} -#[pyclass( - module = "_ctypes", - name = "_SimpleCData", - base = PyCData, - metaclass = "PyCSimpleType" -)] -pub struct PyCSimple { - pub _base: PyCData, - pub _type_: String, - pub value: AtomicCell<PyObjectRef>, - pub cdata: PyRwLock<CDataObject>, -} + if offset < 0 { + return Err(vm.new_value_error("offset cannot be negative")); + } + let offset = offset as usize; -impl Debug for PyCSimple { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyCSimple") - .field("_type_", &self._type_) - .finish() + // Check if buffer is large enough + if offset + size > source.len() { + return Err(vm.new_value_error(format!( + "Buffer size too small ({} instead of at least {} bytes)", + source.len(), + offset + size + ))); + } + + // Copy bytes from buffer at offset + let data = source[offset..offset + size].to_vec(); + + Ok(Self::from_bytes_with_length(data, None, length)) } -} -fn value_to_bytes_endian( - _type_: &str, - value: &PyObjectRef, - swapped: bool, - vm: &VirtualMachine, -) -> Vec<u8> { - // Helper macro for endian conversion - macro_rules! to_bytes { - ($val:expr) => { - if swapped { - // Use opposite endianness - #[cfg(target_endian = "little")] - { - $val.to_be_bytes().to_vec() - } - #[cfg(target_endian = "big")] - { - $val.to_le_bytes().to_vec() - } - } else { - $val.to_ne_bytes().to_vec() - } - }; + #[inline] + pub fn size(&self) -> usize { + self.buffer.read().len() } - match _type_ { - "c" => { - // c_char - single byte - if let Some(bytes) = value.downcast_ref::<PyBytes>() - && !bytes.is_empty() - { - return vec![bytes.as_bytes()[0]]; - } - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_u8() - { - return vec![v]; - } - vec![0] + /// Check if this buffer is borrowed (external memory reference) + #[inline] + pub fn is_borrowed(&self) -> bool { + matches!(&*self.buffer.read(), Cow::Borrowed(_)) + } + + /// Write bytes at offset - handles both borrowed and owned buffers + /// + /// For borrowed buffers (from from_address), writes directly to external memory. + /// For owned buffers, writes through to_mut() as normal. + /// + /// # Safety + /// For borrowed buffers, caller must ensure the memory is writable. + pub fn write_bytes_at_offset(&self, offset: usize, bytes: &[u8]) { + let buffer = self.buffer.read(); + if offset + bytes.len() > buffer.len() { + return; // Out of bounds } - "u" => { - // c_wchar - 4 bytes (wchar_t on most platforms) - if let Ok(s) = value.str(vm) - && let Some(c) = s.as_str().chars().next() - { - return to_bytes!(c as u32); + + match &*buffer { + Cow::Borrowed(slice) => { + // For borrowed memory, write directly + // SAFETY: We assume the caller knows this memory is writable + // (e.g., from from_address pointing to a ctypes buffer) + unsafe { + let ptr = slice.as_ptr() as *mut u8; + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), bytes.len()); + } } - vec![0; 4] - } - "b" => { - // c_byte - signed char (1 byte) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_i8() - { - return vec![v as u8]; + Cow::Owned(_) => { + // For owned memory, use to_mut() through write lock + drop(buffer); + let mut buffer = self.buffer.write(); + buffer.to_mut()[offset..offset + bytes.len()].copy_from_slice(bytes); } - vec![0] } - "B" => { - // c_ubyte - unsigned char (1 byte) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_u8() - { - return vec![v]; - } - vec![0] + } + + /// Generate unique key for nested references (unique_key) + /// Creates a hierarchical key by walking up the b_base chain. + /// Format: "index:parent_index:grandparent_index:..." + pub fn unique_key(&self, index: usize) -> String { + let mut key = format!("{index:x}"); + // Walk up the base chain to build hierarchical key + if self.base.read().is_some() { + let parent_index = self.index.load(); + key.push_str(&format!(":{parent_index:x}")); } - "h" => { - // c_short (2 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_i16() - { - return to_bytes!(v); - } - vec![0; 2] + key + } + + /// Keep a reference in the objects dictionary (KeepRef) + /// + /// Stores 'keep' in this object's b_objects dict at key 'index'. + /// If keep is None, does nothing (optimization). + /// This function stores the value directly - caller should use get_kept_objects() + /// first if they want to store the _objects of a CData instead of the object itself. + /// + /// If this object has a base (is embedded in another structure/union/array), + /// the reference is stored in the root object's b_objects with a hierarchical key. + pub fn keep_ref(&self, index: usize, keep: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Optimization: no need to store None + if vm.is_none(&keep) { + return Ok(()); } - "H" => { - // c_ushort (2 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_u16() - { - return to_bytes!(v); - } - vec![0; 2] + + // Build hierarchical key + let key = self.unique_key(index); + + // If we have a base object, find root and store there + if let Some(base_obj) = self.base.read().clone() { + // Find root by walking up the base chain + let root_obj = Self::find_root_object(&base_obj); + Self::store_in_object(&root_obj, &key, keep, vm)?; + return Ok(()); } - "i" => { - // c_int (4 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_i32() - { - return to_bytes!(v); + + // No base - store in own objects dict + let mut objects = self.objects.write(); + + // Initialize b_objects if needed + if objects.is_none() { + if self.length.load() > 0 { + // Need to store multiple references - create a dict + *objects = Some(vm.ctx.new_dict().into()); + } else { + // Only one reference needed - store directly + *objects = Some(keep); + return Ok(()); } - vec![0; 4] } - "I" => { - // c_uint (4 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_u32() - { - return to_bytes!(v); - } - vec![0; 4] + + // If b_objects is not a dict, convert it to a dict first + // This preserves the existing reference (e.g., from cast) when adding new references + if let Some(obj) = objects.as_ref() + && obj.downcast_ref::<PyDict>().is_none() + { + // Convert existing single reference to a dict + let dict = vm.ctx.new_dict(); + // Store the original object with a special key (id-based) + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + dict.set_item(&*id_key, obj.clone(), vm)?; + *objects = Some(dict.into()); } - "l" => { - // c_long (platform dependent) - if let Ok(int_val) = value.try_to_value::<libc::c_long>(vm) { - return to_bytes!(int_val); - } - const SIZE: usize = std::mem::size_of::<libc::c_long>(); - vec![0; SIZE] + + // Store in dict with unique key + if let Some(dict_obj) = objects.as_ref() + && let Some(dict) = dict_obj.downcast_ref::<PyDict>() + { + let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); + dict.set_item(&*key_obj, keep, vm)?; } - "L" => { - // c_ulong (platform dependent) - if let Ok(int_val) = value.try_to_value::<libc::c_ulong>(vm) { - return to_bytes!(int_val); + + Ok(()) + } + + /// Keep a reference alive without exposing it in _objects. + /// Walks up to root object (same as keep_ref) so the reference + /// lives as long as the owning ctypes object. + /// Uses unique_key (hierarchical) so nested fields don't collide. + pub fn keep_alive(&self, index: usize, obj: PyObjectRef) { + let key = self.unique_key(index); + if let Some(base_obj) = self.base.read().clone() { + let root = Self::find_root_object(&base_obj); + if let Some(cdata) = root.downcast_ref::<PyCData>() { + cdata.kept_refs.write().insert(key, obj); + return; } - const SIZE: usize = std::mem::size_of::<libc::c_ulong>(); - vec![0; SIZE] } - "q" => { - // c_longlong (8 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_i64() - { - return to_bytes!(v); - } - vec![0; 8] + self.kept_refs.write().insert(key, obj); + } + + /// Find the root object (one without a base) by walking up the base chain + fn find_root_object(obj: &PyObject) -> PyObjectRef { + // Try to get base from different ctypes types + let base = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + cdata.base.read().clone() + } else { + None + }; + + // Recurse if there's a base, otherwise this is the root + if let Some(base_obj) = base { + Self::find_root_object(&base_obj) + } else { + obj.to_owned() } - "Q" => { - // c_ulonglong (8 bytes) - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_u64() - { - return to_bytes!(v); - } - vec![0; 8] - } - "f" => { - // c_float (4 bytes) - int도 허용 - if let Ok(float_val) = value.try_float(vm) { - return to_bytes!(float_val.to_f64() as f32); - } - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_f64() - { - return to_bytes!(v as f32); - } - vec![0; 4] - } - "d" | "g" => { - // c_double (8 bytes) - int도 허용 - if let Ok(float_val) = value.try_float(vm) { - return to_bytes!(float_val.to_f64()); - } - if let Ok(int_val) = value.try_int(vm) - && let Some(v) = int_val.as_bigint().to_f64() - { - return to_bytes!(v); - } - vec![0; 8] - } - "?" => { - // c_bool (1 byte) - if let Ok(b) = value.clone().try_to_bool(vm) { - return vec![if b { 1 } else { 0 }]; - } - vec![0] - } - "P" | "z" | "Z" => { - // Pointer types (platform pointer size) - vec![0; std::mem::size_of::<usize>()] - } - _ => vec![0], } -} -impl Constructor for PyCSimple { - type Args = (OptionalArg,); - - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: Self::Args = args.bind(vm)?; - let attributes = cls.get_attributes(); - let _type_ = attributes - .iter() - .find(|(k, _)| { - k.to_object() - .str(vm) - .map(|s| s.to_string() == "_type_") - .unwrap_or(false) - }) - .ok_or_else(|| { - vm.new_type_error(format!( - "cannot create '{}' instances: no _type_ attribute", - cls.name() - )) - })? - .1 - .str(vm)? - .to_string(); - let value = if let Some(ref v) = args.0.into_option() { - set_primitive(_type_.as_str(), v, vm)? + /// Store a value in an object's _objects dict with the given key + fn store_in_object( + obj: &PyObject, + key: &str, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get the objects dict from the object + let objects_lock = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + &cdata.objects } else { - match _type_.as_str() { - "c" | "u" => PyObjectRef::from(vm.ctx.new_bytes(vec![0])), - "b" | "B" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { - PyObjectRef::from(vm.ctx.new_int(0)) - } - "f" | "d" | "g" => PyObjectRef::from(vm.ctx.new_float(0.0)), - "?" => PyObjectRef::from(vm.ctx.new_bool(false)), - _ => vm.ctx.none(), // "z" | "Z" | "P" - } + return Ok(()); // Unknown type, skip }; - // Check if this is a swapped endian type - let swapped = cls - .as_object() - .get_attr("_swappedbytes_", vm) - .map(|v| v.is_true(vm).unwrap_or(false)) - .unwrap_or(false); + let mut objects = objects_lock.write(); - let buffer = value_to_bytes_endian(&_type_, &value, swapped, vm); - let cdata = CDataObject::from_bytes(buffer, None); - PyCSimple { - _base: PyCData::new(cdata.clone()), - _type_, - value: AtomicCell::new(value), - cdata: PyRwLock::new(cdata), + // Initialize if needed + if objects.is_none() { + *objects = Some(vm.ctx.new_dict().into()); } - .into_ref_with_type(vm, cls) - .map(Into::into) + + // If not a dict, convert to dict + if let Some(obj) = objects.as_ref() + && obj.downcast_ref::<PyDict>().is_none() + { + let dict = vm.ctx.new_dict(); + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + dict.set_item(&*id_key, obj.clone(), vm)?; + *objects = Some(dict.into()); + } + + // Store in dict + if let Some(dict_obj) = objects.as_ref() + && let Some(dict) = dict_obj.downcast_ref::<PyDict>() + { + let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); + dict.set_item(&*key_obj, value, vm)?; + } + + Ok(()) } - fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("use slot_new") + /// Get kept objects from a CData instance + /// Returns the _objects of the CData, or an empty dict if None. + pub fn get_kept_objects(value: &PyObject, vm: &VirtualMachine) -> PyObjectRef { + value + .downcast_ref::<PyCData>() + .and_then(|cdata| cdata.objects.read().clone()) + .unwrap_or_else(|| vm.ctx.new_dict().into()) } -} -#[pyclass(flags(BASETYPE), with(Constructor, AsBuffer))] -impl PyCSimple { - #[pygetset] - fn _objects(&self) -> Option<PyObjectRef> { - self.cdata.read().objects.clone() - } - - #[pygetset(name = "value")] - pub fn value(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - let zelf: &Py<Self> = instance - .downcast_ref() - .ok_or_else(|| vm.new_type_error("cannot get value of instance"))?; - let raw_value = unsafe { (*zelf.value.as_ptr()).clone() }; - - // Convert to unsigned if needed for unsigned types - match zelf._type_.as_str() { - "B" | "H" | "I" | "L" | "Q" => { - if let Ok(int_val) = raw_value.try_int(vm) { - let n = int_val.as_bigint(); - // Use platform-specific C types for correct unsigned conversion - match zelf._type_.as_str() { - "B" => { - if let Some(v) = n.to_i64() { - return Ok(vm.ctx.new_int((v as u8) as u64).into()); - } - } - "H" => { - if let Some(v) = n.to_i64() { - return Ok(vm.ctx.new_int((v as c_ushort) as u64).into()); - } - } - "I" => { - if let Some(v) = n.to_i64() { - return Ok(vm.ctx.new_int((v as c_uint) as u64).into()); - } - } - "L" => { - if let Some(v) = n.to_i128() { - return Ok(vm.ctx.new_int(v as c_ulong).into()); - } - } - "Q" => { - if let Some(v) = n.to_i128() { - return Ok(vm.ctx.new_int(v as c_ulonglong).into()); - } - } - _ => {} - }; - } - Ok(raw_value) + /// Check if a value should be stored in _objects + /// Returns true for ctypes objects and bytes (for c_char_p) + pub fn should_keep_ref(value: &PyObject) -> bool { + value.downcast_ref::<PyCData>().is_some() || value.downcast_ref::<PyBytes>().is_some() + } + + /// PyCData_set + /// Sets a field value at the given offset, handling type conversion and KeepRef + #[allow(clippy::too_many_arguments)] + pub fn set_field( + &self, + proto: &PyObject, + value: PyObjectRef, + index: usize, + size: usize, + offset: usize, + needs_swap: bool, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if this is a c_char or c_wchar array field + let is_char_array = PyCField::is_char_array(proto, vm); + let is_wchar_array = PyCField::is_wchar_array(proto, vm); + + // For c_char arrays with bytes input, copy only up to first null + if is_char_array { + if let Some(bytes_val) = value.downcast_ref::<PyBytes>() { + let src = bytes_val.as_bytes(); + let to_copy = PyCField::bytes_for_char_array(src); + let copy_len = core::cmp::min(to_copy.len(), size); + self.write_bytes_at_offset(offset, &to_copy[..copy_len]); + self.keep_ref(index, value, vm)?; + return Ok(()); + } else { + return Err(vm.new_type_error("bytes expected instead of str instance")); } - _ => Ok(raw_value), } - } - #[pygetset(name = "value", setter)] - fn set_value(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let zelf: PyRef<Self> = instance - .clone() - .downcast() - .map_err(|_| vm.new_type_error("cannot set value of instance"))?; - let content = set_primitive(zelf._type_.as_str(), &value, vm)?; + // For c_wchar arrays with str input, convert to wchar_t + if is_wchar_array { + if let Some(str_val) = value.downcast_ref::<PyStr>() { + // Convert str to wchar_t bytes (platform-dependent size) + let mut wchar_bytes = Vec::with_capacity(size); + for ch in str_val.as_str().chars().take(size / WCHAR_SIZE) { + let mut bytes = [0u8; 4]; + wchar_to_bytes(ch as u32, &mut bytes); + wchar_bytes.extend_from_slice(&bytes[..WCHAR_SIZE]); + } + // Pad with nulls to fill the array + while wchar_bytes.len() < size { + wchar_bytes.push(0); + } + self.write_bytes_at_offset(offset, &wchar_bytes); + self.keep_ref(index, value, vm)?; + return Ok(()); + } else if value.downcast_ref::<PyBytes>().is_some() { + return Err(vm.new_type_error("str expected instead of bytes instance")); + } + } - // Check if this is a swapped endian type - let swapped = instance - .class() - .as_object() - .get_attr("_swappedbytes_", vm) - .map(|v| v.is_true(vm).unwrap_or(false)) - .unwrap_or(false); - - // Update buffer when value changes - let buffer_bytes = value_to_bytes_endian(&zelf._type_, &content, swapped, vm); - zelf.cdata.write().buffer = buffer_bytes; - zelf.value.store(content); - Ok(()) - } + // Special handling for Pointer fields with Array values + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && proto_type + .class() + .fast_issubclass(super::pointer::PyCPointerType::static_type()) + && let Some(array) = value.downcast_ref::<super::array::PyCArray>() + { + let buffer_addr = { + let array_buffer = array.0.buffer.read(); + array_buffer.as_ptr() as usize + }; + let addr_bytes = buffer_addr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + self.write_bytes_at_offset(offset, &addr_bytes[..len]); + self.keep_ref(index, value, vm)?; + return Ok(()); + } - #[pyclassmethod] - fn repeat(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { - use super::_ctypes::get_size; - use super::array::create_array_type_with_stg_info; - - if n < 0 { - return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); - } - // Get element size from cls - let element_size = if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) { - if let Ok(s) = type_attr.str(vm) { - let s = s.to_string(); - if s.len() == 1 { - get_size(&s) + // For array fields with tuple/list input, instantiate the array type + // and unpack elements as positional args (Array_init expects *args) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && stg.element_type.is_some() + { + let items: Option<Vec<PyObjectRef>> = + if let Some(tuple) = value.downcast_ref::<PyTuple>() { + Some(tuple.to_vec()) } else { - std::mem::size_of::<usize>() + value + .downcast_ref::<crate::builtins::PyList>() + .map(|list| list.borrow_vec().to_vec()) + }; + if let Some(items) = items { + let array_obj = proto_type.as_object().call(items, vm).map_err(|e| { + // Wrap errors in RuntimeError with type name prefix + let type_name = proto_type.name().to_string(); + let exc_name = e.class().name().to_string(); + let exc_args = e.args(); + let exc_msg = exc_args + .first() + .and_then(|a| a.downcast_ref::<PyStr>().map(|s| s.to_string())) + .unwrap_or_default(); + vm.new_runtime_error(format!("({type_name}) {exc_name}: {exc_msg}")) + })?; + if let Some(arr) = array_obj.downcast_ref::<super::array::PyCArray>() { + let arr_buffer = arr.0.buffer.read(); + let len = core::cmp::min(arr_buffer.len(), size); + self.write_bytes_at_offset(offset, &arr_buffer[..len]); + drop(arr_buffer); + self.keep_ref(index, array_obj, vm)?; + return Ok(()); } - } else { - std::mem::size_of::<usize>() } - } else { - std::mem::size_of::<usize>() - }; - let total_size = element_size * (n as usize); - let stg_info = super::util::StgInfo::new_array( - total_size, - element_size, - n as usize, - cls.clone().into(), - element_size, - ); - create_array_type_with_stg_info(stg_info, vm) - } + } - #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { - use super::_ctypes::get_size; - // Get _type_ attribute directly - let type_attr = cls - .as_object() + // Get field type code for special handling + let field_type_code = proto .get_attr("_type_", vm) - .map_err(|_| vm.new_type_error(format!("'{}' has no _type_ attribute", cls.name())))?; - let type_str = type_attr.str(vm)?.to_string(); - let size = get_size(&type_str); + .ok() + .and_then(|attr| attr.downcast_ref::<PyStr>().map(|s| s.to_string())); - // Create instance with value read from address - let value = if address != 0 && size > 0 { - // Safety: This is inherently unsafe - reading from arbitrary memory address - unsafe { - let ptr = address as *const u8; - let bytes = std::slice::from_raw_parts(ptr, size); - // Convert bytes to appropriate Python value based on type - bytes_to_pyobject(&cls, bytes, vm)? + // c_char_p (z type) with bytes: store original in _objects, keep + // null-terminated copy alive separately for the pointer. + if field_type_code.as_deref() == Some("z") + && let Some(bytes_val) = value.downcast_ref::<PyBytes>() + { + let (kept_alive, ptr) = ensure_z_null_terminated(bytes_val, vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + if needs_swap { + result.reverse(); } + self.write_bytes_at_offset(offset, &result); + self.keep_ref(index, value, vm)?; + self.keep_alive(index, kept_alive); + return Ok(()); + } + + let (mut bytes, converted_value) = if let Some(type_code) = &field_type_code { + PyCField::value_to_bytes_for_type(type_code, &value, size, vm)? } else { - vm.ctx.none() + (PyCField::value_to_bytes(&value, size, vm)?, None) }; - // Create instance using the type's constructor - let args = FuncArgs::new(vec![value], KwArgs::default()); - PyCSimple::slot_new(cls.clone(), args, vm) + // Swap bytes for opposite endianness + if needs_swap { + bytes.reverse(); + } + + self.write_bytes_at_offset(offset, &bytes); + + // KeepRef: for z/Z types use converted value, otherwise use original + if let Some(converted) = converted_value { + self.keep_ref(index, converted, vm)?; + } else if Self::should_keep_ref(&value) { + let to_keep = Self::get_kept_objects(&value, vm); + self.keep_ref(index, to_keep, vm)?; + } + + Ok(()) } - #[pyclassmethod] - fn from_buffer( - cls: PyTypeRef, - source: PyObjectRef, - offset: OptionalArg<isize>, + /// PyCData_get + /// Gets a field value at the given offset + pub fn get_field( + &self, + proto: &PyObject, + index: usize, + size: usize, + offset: usize, + base_obj: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - use super::_ctypes::get_size; - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); + // Get buffer data at offset + let buffer = self.buffer.read(); + if offset + size > buffer.len() { + return Ok(vm.ctx.new_int(0).into()); } - let offset = offset as usize; - // Get buffer from source - let buffer = PyBuffer::try_from_object(vm, source.clone())?; + // Check if field type is an array type + if let Some(type_ref) = proto.downcast_ref::<PyType>() + && let Some(stg) = type_ref.stg_info_opt() + && stg.element_type.is_some() + { + // c_char array → return bytes + if PyCField::is_char_array(proto, vm) { + let data = &buffer[offset..offset + size]; + // Find first null terminator (or use full length) + let end = data.iter().position(|&b| b == 0).unwrap_or(data.len()); + return Ok(vm.ctx.new_bytes(data[..end].to_vec()).into()); + } - // Check if buffer is writable - if buffer.desc.readonly { - return Err(vm.new_type_error("underlying buffer is not writable".to_owned())); + // c_wchar array → return str + if PyCField::is_wchar_array(proto, vm) { + let data = &buffer[offset..offset + size]; + // wchar_t → char conversion, skip null + let chars: String = data + .chunks(WCHAR_SIZE) + .filter_map(|chunk| { + wchar_from_bytes(chunk) + .filter(|&wchar| wchar != 0) + .and_then(char::from_u32) + }) + .collect(); + return Ok(vm.ctx.new_str(chars).into()); + } + + // Other array types - create array with a copy of data from the base's buffer + // The array also keeps a reference to the base for keeping it alive and for writes + let array_data = buffer[offset..offset + size].to_vec(); + drop(buffer); + + let cdata_obj = + Self::from_base_with_data(base_obj, offset, index, stg.length, array_data); + let array_type: PyTypeRef = proto + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("expected array type"))?; + + return super::array::PyCArray(cdata_obj) + .into_ref_with_type(vm, array_type) + .map(Into::into); } - // Get _type_ attribute directly - let type_attr = cls - .as_object() - .get_attr("_type_", vm) - .map_err(|_| vm.new_type_error(format!("'{}' has no _type_ attribute", cls.name())))?; - let type_str = type_attr.str(vm)?.to_string(); - let size = get_size(&type_str); + let buffer_data = buffer[offset..offset + size].to_vec(); + drop(buffer); - // Check if buffer is large enough - let buffer_len = buffer.desc.len; - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + // Get proto as type + let proto_type: PyTypeRef = proto + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("field proto is not a type"))?; + + let proto_metaclass = proto_type.class(); + + // Simple types: return primitive value + if proto_metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) { + // Check for byte swapping + let needs_swap = base_obj + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok() + || proto_type + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok(); + + let data = if needs_swap && size > 1 { + let mut swapped = buffer_data.clone(); + swapped.reverse(); + swapped + } else { + buffer_data + }; + + return bytes_to_pyobject(&proto_type, &data, vm); } - // Read bytes from buffer at offset - let bytes = buffer.obj_bytes(); - let data = &bytes[offset..offset + size]; - let value = bytes_to_pyobject(&cls, data, vm)?; + // Complex types: create ctypes instance via PyCData_FromBaseObj + let ptr = self.buffer.read().as_ptr().wrapping_add(offset) as *mut u8; + let cdata_obj = unsafe { Self::from_base_obj(ptr, size, base_obj.clone(), index) }; + + if proto_metaclass.fast_issubclass(super::structure::PyCStructType::static_type()) + || proto_metaclass.fast_issubclass(super::union::PyCUnionType::static_type()) + || proto_metaclass.fast_issubclass(super::pointer::PyCPointerType::static_type()) + { + cdata_obj.into_ref_with_type(vm, proto_type).map(Into::into) + } else { + // Fallback + Ok(vm.ctx.new_int(0).into()) + } + } +} - // Create instance - let args = FuncArgs::new(vec![value], KwArgs::default()); - let instance = PyCSimple::slot_new(cls.clone(), args, vm)?; +#[pyclass(flags(BASETYPE))] +impl PyCData { + #[pygetset] + fn _objects(&self) -> Option<PyObjectRef> { + self.objects.read().clone() + } - // TODO: Store reference to source in _objects to keep buffer alive - Ok(instance) + #[pygetset] + fn _b_base_(&self) -> Option<PyObjectRef> { + self.base.read().clone() } + #[pygetset] + fn _b_needsfree_(&self) -> i32 { + // Borrowed (from_address) or has base object → 0 (don't free) + // Owned and no base → 1 (need to free) + if self.is_borrowed() || self.base.read().is_some() { + 0 + } else { + 1 + } + } + + // CDataType_methods - shared across all ctypes types + #[pyclassmethod] - fn from_buffer_copy( + pub(super) fn from_buffer( cls: PyTypeRef, - source: ArgBytesLike, + source: PyObjectRef, offset: OptionalArg<isize>, vm: &VirtualMachine, ) -> PyResult { - use super::_ctypes::get_size; - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); - } - let offset = offset as usize; + let cdata = Self::from_buffer_impl(&cls, source, offset.unwrap_or(0), vm)?; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } - // Get _type_ attribute directly for simple types - let type_attr = cls - .as_object() - .get_attr("_type_", vm) - .map_err(|_| vm.new_type_error(format!("'{}' has no _type_ attribute", cls.name())))?; - let type_str = type_attr.str(vm)?.to_string(); - let size = get_size(&type_str); + #[pyclassmethod] + pub(super) fn from_buffer_copy( + cls: PyTypeRef, + source: ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cdata = + Self::from_buffer_copy_impl(&cls, &source.borrow_buf(), offset.unwrap_or(0), vm)?; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } - // Borrow bytes from source - let source_bytes = source.borrow_buf(); - let buffer_len = source_bytes.len(); + #[pyclassmethod] + pub(super) fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { + let size = { + let stg_info = cls.stg_info(vm)?; + stg_info.size + }; - // Check if buffer is large enough - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + if size == 0 { + return Err(vm.new_type_error("abstract class")); } - // Copy bytes from buffer at offset - let data = &source_bytes[offset..offset + size]; - let value = bytes_to_pyobject(&cls, data, vm)?; - - // Create instance (independent copy, no reference tracking) - let args = FuncArgs::new(vec![value], KwArgs::default()); - PyCSimple::slot_new(cls.clone(), args, vm) + // PyCData_AtAddress + let cdata = unsafe { Self::at_address(address as *const u8, size) }; + cdata.into_ref_with_type(vm, cls).map(Into::into) } #[pyclassmethod] - fn in_dll(cls: PyTypeRef, dll: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - use super::_ctypes::get_size; - use libloading::Symbol; + pub(super) fn in_dll( + cls: PyTypeRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let size = { + let stg_info = cls.stg_info(vm)?; + stg_info.size + }; + + if size == 0 { + return Err(vm.new_type_error("abstract class")); + } // Get the library handle from dll object let handle = if let Ok(int_handle) = dll.try_int(vm) { - // dll is an integer handle int_handle .as_bigint() .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? + .ok_or_else(|| vm.new_value_error("Invalid library handle"))? } else { - // dll is a CDLL/PyDLL/WinDLL object with _handle attribute dll.get_attr("_handle", vm)? .try_int(vm)? .as_bigint() .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? + .ok_or_else(|| vm.new_value_error("Invalid library handle"))? }; - // Get the library from cache - let library_cache = crate::stdlib::ctypes::library::libcache().read(); + // Look up the library in the cache and use lib.get() for symbol lookup + let library_cache = super::library::libcache().read(); let library = library_cache .get_lib(handle) - .ok_or_else(|| vm.new_attribute_error("Library not found".to_owned()))?; - - // Get symbol address from library - let symbol_name = format!("{}\0", name.as_str()); + .ok_or_else(|| vm.new_value_error("Library not found"))?; let inner_lib = library.lib.lock(); - let symbol_address = if let Some(lib) = &*inner_lib { + let symbol_name_with_nul = format!("{}\0", name.as_str()); + let ptr: *const u8 = if let Some(lib) = &*inner_lib { unsafe { - // Try to get the symbol from the library - let symbol: Symbol<'_, *mut u8> = lib.get(symbol_name.as_bytes()).map_err(|e| { - vm.new_attribute_error(format!("{}: symbol '{}' not found", e, name.as_str())) - })?; - *symbol as usize + lib.get::<*const u8>(symbol_name_with_nul.as_bytes()) + .map(|sym| *sym) + .map_err(|_| { + vm.new_value_error(format!("symbol '{}' not found", name.as_str())) + })? } } else { - return Err(vm.new_attribute_error("Library is closed".to_owned())); + return Err(vm.new_value_error("Library closed")); }; - // Get _type_ attribute and size - let type_attr = cls - .as_object() - .get_attr("_type_", vm) - .map_err(|_| vm.new_type_error(format!("'{}' has no _type_ attribute", cls.name())))?; - let type_str = type_attr.str(vm)?.to_string(); - let size = get_size(&type_str); + // dlsym can return NULL for symbols that resolve to NULL (e.g., GNU IFUNC) + // Treat NULL addresses as errors + if ptr.is_null() { + return Err(vm.new_value_error(format!("symbol '{}' not found", name.as_str()))); + } - // Read value from symbol address - let value = if symbol_address != 0 && size > 0 { - // Safety: Reading from a symbol address provided by dlsym - unsafe { - let ptr = symbol_address as *const u8; - let bytes = std::slice::from_raw_parts(ptr, size); - bytes_to_pyobject(&cls, bytes, vm)? + // PyCData_AtAddress + let cdata = unsafe { Self::at_address(ptr, size) }; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } +} + +// PyCField - Field descriptor for Structure/Union types + +/// CField descriptor for Structure/Union field access +#[pyclass(name = "CField", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub struct PyCField { + /// Field name + pub(crate) name: String, + /// Byte offset of the field within the structure/union + pub(crate) offset: isize, + /// Byte size of the underlying type + pub(crate) byte_size_val: isize, + /// Index into PyCData's object array + pub(crate) index: usize, + /// The ctypes type for this field + pub(crate) proto: PyTypeRef, + /// Flag indicating if the field is anonymous (MakeAnonFields sets this) + pub(crate) anonymous: bool, + /// Bitfield size in bits (0 for non-bitfield) + pub(crate) bitfield_size: u16, + /// Bit offset within the storage unit (only meaningful for bitfields) + pub(crate) bit_offset_val: u16, +} + +impl PyCField { + /// Create a new CField descriptor (non-bitfield) + pub fn new( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + index: usize, + ) -> Self { + Self { + name, + offset, + byte_size_val: byte_size, + index, + proto, + anonymous: false, + bitfield_size: 0, + bit_offset_val: 0, + } + } + + /// Create a new CField descriptor for a bitfield + pub fn new_bitfield( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + bitfield_size: u16, + bit_offset: u16, + index: usize, + ) -> Self { + Self { + name, + offset, + byte_size_val: byte_size, + index, + proto, + anonymous: false, + bitfield_size, + bit_offset_val: bit_offset, + } + } + + /// Get the byte size of the field's underlying type + pub fn get_byte_size(&self) -> usize { + self.byte_size_val as usize + } + + /// Create a new CField from an existing field with adjusted offset and index + /// Used by MakeFields to promote anonymous fields + pub fn new_from_field(fdescr: &PyCField, index_offset: usize, offset_delta: isize) -> Self { + Self { + name: fdescr.name.clone(), + offset: fdescr.offset + offset_delta, + byte_size_val: fdescr.byte_size_val, + index: fdescr.index + index_offset, + proto: fdescr.proto.clone(), + anonymous: false, // promoted fields are not anonymous themselves + bitfield_size: fdescr.bitfield_size, + bit_offset_val: fdescr.bit_offset_val, + } + } + + /// Set anonymous flag + pub fn set_anonymous(&mut self, anonymous: bool) { + self.anonymous = anonymous; + } +} + +impl Constructor for PyCField { + type Args = crate::function::FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // PyCField_new_impl: requires _internal_use=True + let internal_use = if let Some(v) = args.kwargs.get("_internal_use") { + v.clone().try_to_bool(vm)? + } else { + false + }; + + if !internal_use { + return Err(vm.new_type_error( + "CField is not intended to be used directly; use it via Structure or Union fields" + .to_string(), + )); + } + + let name: String = args + .kwargs + .get("name") + .ok_or_else(|| vm.new_type_error("missing required argument: 'name'"))? + .try_to_value(vm)?; + + let field_type: PyTypeRef = args + .kwargs + .get("type") + .ok_or_else(|| vm.new_type_error("missing required argument: 'type'"))? + .clone() + .downcast() + .map_err(|_| vm.new_type_error("'type' must be a ctypes type"))?; + + let byte_size: isize = args + .kwargs + .get("byte_size") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_size'"))? + .try_to_value(vm)?; + + let byte_offset: isize = args + .kwargs + .get("byte_offset") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_offset'"))? + .try_to_value(vm)?; + + let index: usize = args + .kwargs + .get("index") + .ok_or_else(|| vm.new_type_error("missing required argument: 'index'"))? + .try_to_value(vm)?; + + // Validate byte_size matches the type + let type_size = super::base::get_field_size(field_type.as_object(), vm)? as isize; + if byte_size != type_size { + return Err(vm.new_value_error(format!( + "byte_size {} does not match type size {}", + byte_size, type_size + ))); + } + + let bit_size_val: Option<isize> = args + .kwargs + .get("bit_size") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + let bit_offset_val: Option<isize> = args + .kwargs + .get("bit_offset") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + if let Some(bs) = bit_size_val { + if bs < 0 { + return Err(vm.new_value_error("number of bits invalid for bit field".to_string())); + } + let bo = bit_offset_val.unwrap_or(0); + if bo < 0 { + return Err(vm.new_value_error("bit_offset must be >= 0".to_string())); } + let type_bits = byte_size * 8; + if bo + bs > type_bits { + return Err(vm.new_value_error(format!( + "bit field '{}' overflows its type ({} + {} > {})", + name, bo, bs, type_bits + ))); + } + Ok(Self::new_bitfield( + name, + field_type, + byte_offset, + byte_size, + bs as u16, + bo as u16, + index, + )) + } else { + Ok(Self::new(name, field_type, byte_offset, byte_size, index)) + } + } +} + +impl Representable for PyCField { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // Get type name from proto (which is always PyTypeRef) + let tp_name = zelf.proto.name().to_string(); + + // Bitfield: <Field type=TYPE, ofs=OFFSET:BIT_OFFSET, bits=NUM_BITS> + // Regular: <Field type=TYPE, ofs=OFFSET, size=SIZE> + if zelf.bitfield_size > 0 { + Ok(format!( + "<Field type={}, ofs={}:{}, bits={}>", + tp_name, zelf.offset, zelf.bit_offset_val, zelf.bitfield_size + )) } else { - vm.ctx.none() + Ok(format!( + "<Field type={}, ofs={}, size={}>", + tp_name, zelf.offset, zelf.byte_size_val + )) + } + } +} + +/// PyCField_get +impl GetDescriptor for PyCField { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast::<PyCField>() + .map_err(|_| vm.new_type_error("expected CField"))?; + + // If obj is None, return the descriptor itself (class attribute access) + let obj = match obj { + Some(obj) if !vm.is_none(&obj) => obj, + _ => return Ok(zelf.into()), }; - // Create instance - let args = FuncArgs::new(vec![value], KwArgs::default()); - let instance = PyCSimple::slot_new(cls.clone(), args, vm)?; + let offset = zelf.offset as usize; + let size = zelf.get_byte_size(); + + // Get PyCData from obj (works for both Structure and Union) + let cdata = PyCField::get_cdata_from_obj(&obj, vm)?; + + // PyCData_get + cdata.get_field( + zelf.proto.as_object(), + zelf.index, + size, + offset, + obj.clone(), + vm, + ) + } +} + +impl PyCField { + /// Convert a Python value to bytes + fn value_to_bytes(value: &PyObject, size: usize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + // 1. Handle bytes objects + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let src = bytes.as_bytes(); + let mut result = vec![0u8; size]; + let len = core::cmp::min(src.len(), size); + result[..len].copy_from_slice(&src[..len]); + Ok(result) + } + // 2. Handle ctypes array instances (copy their buffer) + else if let Some(cdata) = value.downcast_ref::<super::PyCData>() { + let buffer = cdata.buffer.read(); + let mut result = vec![0u8; size]; + let len = core::cmp::min(buffer.len(), size); + result[..len].copy_from_slice(&buffer[..len]); + Ok(result) + } + // 4. Handle float values (check before int, since float.try_int would truncate) + else if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + let f = float_val.to_f64(); + match size { + 4 => { + let val = f as f32; + Ok(val.to_ne_bytes().to_vec()) + } + 8 => Ok(f.to_ne_bytes().to_vec()), + _ => unreachable!("wrong payload size"), + } + } + // 4. Handle integer values + else if let Ok(int_val) = value.try_int(vm) { + let i = int_val.as_bigint(); + match size { + 1 => { + let val = i.to_i8().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 2 => { + let val = i.to_i16().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 4 => { + let val = i.to_i32().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 8 => { + let val = i.to_i64().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + _ => Ok(vec![0u8; size]), + } + } else { + Ok(vec![0u8; size]) + } + } + + /// Convert a Python value to bytes with type-specific handling for pointer types. + /// Returns (bytes, optional holder for wchar buffer). + fn value_to_bytes_for_type( + type_code: &str, + value: &PyObject, + size: usize, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, Option<PyObjectRef>)> { + match type_code { + // c_float: always convert to float first (f_set) + "f" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + let val = f as f32; + Ok((val.to_ne_bytes().to_vec(), None)) + } + // c_double: always convert to float first (d_set) + "d" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + Ok((f.to_ne_bytes().to_vec(), None)) + } + // c_longdouble: convert to float (treated as f64 in RustPython) + "g" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + Ok((f.to_ne_bytes().to_vec(), None)) + } + "z" => { + // c_char_p with bytes is handled in set_field before this call. + // This handles integer address and None cases. + // Integer address + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + "Z" => { + // c_wchar_p: store pointer to null-terminated wchar_t buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = str_to_wchar_bytes(s.as_str(), vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + return Ok((result, Some(holder))); + } + // Integer address + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + "P" => { + // c_void_p: store integer as pointer + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + _ => Ok((PyCField::value_to_bytes(value, size, vm)?, None)), + } + } - // Store base reference to keep dll alive - if let Ok(simple_ref) = instance.clone().downcast::<PyCSimple>() { - simple_ref.cdata.write().base = Some(dll); + /// Check if the field type is a c_char array (element type has _type_ == 'c') + fn is_char_array(proto: &PyObject, vm: &VirtualMachine) -> bool { + // Get element_type from StgInfo (for array types) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && let Some(element_type) = &stg.element_type + { + // Check if element type has _type_ == "c" + if let Ok(type_code) = element_type.as_object().get_attr("_type_", vm) + && let Some(s) = type_code.downcast_ref::<PyStr>() + { + return s.as_str() == "c"; + } } + false + } - Ok(instance) + /// Check if the field type is a c_wchar array (element type has _type_ == 'u') + fn is_wchar_array(proto: &PyObject, vm: &VirtualMachine) -> bool { + // Get element_type from StgInfo (for array types) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && let Some(element_type) = &stg.element_type + { + // Check if element type has _type_ == "u" + if let Ok(type_code) = element_type.as_object().get_attr("_type_", vm) + && let Some(s) = type_code.downcast_ref::<PyStr>() + { + return s.as_str() == "u"; + } + } + false + } + + /// Convert bytes for c_char array assignment (stops at first null terminator) + /// Returns (bytes_to_copy, copy_len) + fn bytes_for_char_array(src: &[u8]) -> &[u8] { + // Find first null terminator and include it + if let Some(null_pos) = src.iter().position(|&b| b == 0) { + &src[..=null_pos] + } else { + src + } } } -impl PyCSimple { - pub fn to_arg( - &self, - ty: libffi::middle::Type, +#[pyclass(flags(IMMUTABLETYPE), with(Representable, GetDescriptor, Constructor))] +impl PyCField { + /// Get PyCData from object (works for both Structure and Union) + fn get_cdata_from_obj<'a>(obj: &'a PyObjectRef, vm: &VirtualMachine) -> PyResult<&'a PyCData> { + if let Some(s) = obj.downcast_ref::<super::structure::PyCStructure>() { + Ok(&s.0) + } else if let Some(u) = obj.downcast_ref::<super::union::PyCUnion>() { + Ok(&u.0) + } else { + Err(vm.new_type_error(format!( + "descriptor works only on Structure or Union instances, got {}", + obj.class().name() + ))) + } + } + + /// PyCField_set + #[pyslot] + fn descr_set( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, vm: &VirtualMachine, - ) -> Option<libffi::middle::Arg> { - let value = unsafe { (*self.value.as_ptr()).clone() }; - if let Ok(i) = value.try_int(vm) { - let i = i.as_bigint(); - return if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { - i.to_u8().map(|r: u8| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { - i.to_i8().map(|r: i8| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { - i.to_u16().map(|r: u16| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { - i.to_i16().map(|r: i16| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { - i.to_u32().map(|r: u32| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { - i.to_i32().map(|r: i32| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { - i.to_u64().map(|r: u64| libffi::middle::Arg::new(&r)) - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { - i.to_i64().map(|r: i64| libffi::middle::Arg::new(&r)) + ) -> PyResult<()> { + let zelf = zelf + .downcast_ref::<PyCField>() + .ok_or_else(|| vm.new_type_error("expected CField"))?; + + let offset = zelf.offset as usize; + let size = zelf.get_byte_size(); + + // Get PyCData from obj (works for both Structure and Union) + let cdata = Self::get_cdata_from_obj(&obj, vm)?; + + match value { + PySetterValue::Assign(value) => { + // Check if needs byte swapping + let needs_swap = (obj + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok() + || zelf + .proto + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok()) + && size > 1; + + // PyCData_set + cdata.set_field( + zelf.proto.as_object(), + value, + zelf.index, + size, + offset, + needs_swap, + vm, + ) + } + PySetterValue::Delete => Err(vm.new_type_error("cannot delete field")), + } + } + + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "type")] + fn type_(&self) -> PyTypeRef { + self.proto.clone() + } + + #[pygetset] + fn offset(&self) -> isize { + self.offset + } + + #[pygetset] + fn byte_offset(&self) -> isize { + self.offset + } + + #[pygetset] + fn size(&self) -> isize { + // Legacy: encode as (bitfield_size << 16) | bit_offset for bitfields + if self.bitfield_size > 0 { + ((self.bitfield_size as isize) << 16) | (self.bit_offset_val as isize) + } else { + self.byte_size_val + } + } + + #[pygetset] + fn byte_size(&self) -> isize { + self.byte_size_val + } + + #[pygetset] + fn bit_offset(&self) -> isize { + self.bit_offset_val as isize + } + + #[pygetset] + fn bit_size(&self, vm: &VirtualMachine) -> PyObjectRef { + if self.bitfield_size > 0 { + vm.ctx.new_int(self.bitfield_size).into() + } else { + // Non-bitfield: bit_size = byte_size * 8 + let byte_size = self.byte_size_val as i128; + vm.ctx.new_int(byte_size * 8).into() + } + } + + #[pygetset] + fn is_bitfield(&self) -> bool { + self.bitfield_size > 0 + } + + #[pygetset] + fn is_anonymous(&self) -> bool { + self.anonymous + } +} + +// ParamFunc implementations (PyCArgObject creation) + +use super::_ctypes::CArgObject; + +/// Call the appropriate paramfunc based on StgInfo.paramfunc +/// info->paramfunc(st, obj) +pub(super) fn call_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + let cls = obj.class(); + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + + match stg_info.paramfunc { + ParamFunc::Simple => simple_paramfunc(obj, vm), + ParamFunc::Array => array_paramfunc(obj, vm), + ParamFunc::Pointer => pointer_paramfunc(obj, vm), + ParamFunc::Structure | ParamFunc::Union => struct_union_paramfunc(obj, &stg_info, vm), + ParamFunc::None => Err(vm.new_type_error("no paramfunc")), + } +} + +/// PyCSimpleType_paramfunc +fn simple_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::simple::PyCSimple; + + let simple = obj + .downcast_ref::<PyCSimple>() + .ok_or_else(|| vm.new_type_error("expected simple type"))?; + + // Get type code from _type_ attribute + let cls = obj.class().to_owned(); + let type_code = cls + .type_code(vm) + .ok_or_else(|| vm.new_type_error("no _type_ attribute"))?; + let tag = type_code.as_bytes().first().copied().unwrap_or(b'?'); + + // Read value from buffer: memcpy(&parg->value, self->b_ptr, self->b_size) + let buffer = simple.0.buffer.read(); + let ffi_value = buffer_to_ffi_value(&type_code, &buffer); + + Ok(CArgObject { + tag, + value: ffi_value, + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// PyCArrayType_paramfunc +fn array_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::array::PyCArray; + + let array = obj + .downcast_ref::<PyCArray>() + .ok_or_else(|| vm.new_type_error("expected array"))?; + + // p->value.p = (char *)self->b_ptr + let buffer = array.0.buffer.read(); + let ptr_val = buffer.as_ptr() as usize; + + Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// PyCPointerType_paramfunc +fn pointer_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::pointer::PyCPointer; + + let ptr = obj + .downcast_ref::<PyCPointer>() + .ok_or_else(|| vm.new_type_error("expected pointer"))?; + + // parg->value.p = *(void **)self->b_ptr + let ptr_val = ptr.get_ptr_value(); + + Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// StructUnionType_paramfunc (for both Structure and Union) +fn struct_union_paramfunc( + obj: &PyObject, + stg_info: &StgInfo, + _vm: &VirtualMachine, +) -> PyResult<CArgObject> { + // Get buffer pointer + // For large structs (> sizeof(void*)), we'd need to allocate and copy. + // For now, just point to buffer directly and keep obj reference for memory safety. + let buffer = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + cdata.buffer.read() + } else { + return Ok(CArgObject { + tag: b'V', + value: FfiArgValue::Pointer(0), + obj: obj.to_owned(), + size: stg_info.size, + offset: 0, + }); + }; + + let ptr_val = buffer.as_ptr() as usize; + let size = buffer.len(); + + Ok(CArgObject { + tag: b'V', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size, + offset: 0, + }) +} + +// FfiArgValue - Owned FFI argument value + +/// Owned FFI argument value. Keeps the value alive for the duration of the FFI call. +#[derive(Debug, Clone)] +pub enum FfiArgValue { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + Pointer(usize), + /// Pointer with owned data. The PyObjectRef keeps the pointed data alive. + OwnedPointer(usize, #[allow(dead_code)] PyObjectRef), +} + +impl FfiArgValue { + /// Create an Arg reference to this owned value + pub fn as_arg(&self) -> libffi::middle::Arg<'_> { + match self { + FfiArgValue::U8(v) => libffi::middle::Arg::new(v), + FfiArgValue::I8(v) => libffi::middle::Arg::new(v), + FfiArgValue::U16(v) => libffi::middle::Arg::new(v), + FfiArgValue::I16(v) => libffi::middle::Arg::new(v), + FfiArgValue::U32(v) => libffi::middle::Arg::new(v), + FfiArgValue::I32(v) => libffi::middle::Arg::new(v), + FfiArgValue::U64(v) => libffi::middle::Arg::new(v), + FfiArgValue::I64(v) => libffi::middle::Arg::new(v), + FfiArgValue::F32(v) => libffi::middle::Arg::new(v), + FfiArgValue::F64(v) => libffi::middle::Arg::new(v), + FfiArgValue::Pointer(v) => libffi::middle::Arg::new(v), + FfiArgValue::OwnedPointer(v, _) => libffi::middle::Arg::new(v), + } + } +} + +/// Convert buffer bytes to FfiArgValue based on type code +pub(super) fn buffer_to_ffi_value(type_code: &str, buffer: &[u8]) -> FfiArgValue { + match type_code { + "c" | "b" => { + let v = buffer.first().map(|&b| b as i8).unwrap_or(0); + FfiArgValue::I8(v) + } + "B" => { + let v = buffer.first().copied().unwrap_or(0); + FfiArgValue::U8(v) + } + "h" => { + let v = buffer.first_chunk().copied().map_or(0, i16::from_ne_bytes); + FfiArgValue::I16(v) + } + "H" => { + let v = buffer.first_chunk().copied().map_or(0, u16::from_ne_bytes); + FfiArgValue::U16(v) + } + "i" => { + let v = buffer.first_chunk().copied().map_or(0, i32::from_ne_bytes); + FfiArgValue::I32(v) + } + "I" => { + let v = buffer.first_chunk().copied().map_or(0, u32::from_ne_bytes); + FfiArgValue::U32(v) + } + "l" | "q" => { + let v = if let Some(&bytes) = buffer.first_chunk::<8>() { + i64::from_ne_bytes(bytes) + } else if let Some(&bytes) = buffer.first_chunk::<4>() { + i32::from_ne_bytes(bytes).into() } else { - None + 0 }; + FfiArgValue::I64(v) } - if let Ok(_f) = value.try_float(vm) { - todo!(); + "L" | "Q" => { + let v = if let Some(&bytes) = buffer.first_chunk::<8>() { + u64::from_ne_bytes(bytes) + } else if let Some(&bytes) = buffer.first_chunk::<4>() { + u32::from_ne_bytes(bytes).into() + } else { + 0 + }; + FfiArgValue::U64(v) + } + "f" => { + let v = buffer + .first_chunk::<4>() + .copied() + .map_or(0.0, f32::from_ne_bytes); + FfiArgValue::F32(v) + } + "d" | "g" => { + let v = buffer + .first_chunk::<8>() + .copied() + .map_or(0.0, f64::from_ne_bytes); + FfiArgValue::F64(v) + } + "z" | "Z" | "P" | "O" => FfiArgValue::Pointer(read_ptr_from_buffer(buffer)), + "?" => { + let v = buffer.first().map(|&b| b != 0).unwrap_or(false); + FfiArgValue::U8(if v { 1 } else { 0 }) } - if let Ok(_b) = value.try_to_bool(vm) { - todo!(); + "u" => { + // wchar_t - 4 bytes on most platforms + let v = buffer.first_chunk().copied().map_or(0, u32::from_ne_bytes); + FfiArgValue::U32(v) } - None + _ => FfiArgValue::Pointer(0), } } -static SIMPLE_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - rustpython_common::lock::PyMappedRwLockReadGuard::map( - rustpython_common::lock::PyRwLockReadGuard::map( - buffer.obj_as::<PyCSimple>().cdata.read(), - |x: &CDataObject| x, - ), - |x: &CDataObject| x.buffer.as_slice(), - ) - .into() - }, - obj_bytes_mut: |buffer| { - rustpython_common::lock::PyMappedRwLockWriteGuard::map( - rustpython_common::lock::PyRwLockWriteGuard::map( - buffer.obj_as::<PyCSimple>().cdata.write(), - |x: &mut CDataObject| x, - ), - |x: &mut CDataObject| x.buffer.as_mut_slice(), - ) - .into() - }, - release: |_| {}, - retain: |_| {}, -}; +/// Convert bytes to appropriate Python object based on ctypes type +pub(super) fn bytes_to_pyobject( + cls: &Py<PyType>, + bytes: &[u8], + vm: &VirtualMachine, +) -> PyResult<PyObjectRef> { + // Try to get _type_ attribute + if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(s) = type_attr.str(vm) + { + let ty = s.to_string(); + return match ty.as_str() { + "c" => Ok(vm.ctx.new_bytes(bytes.to_vec()).into()), + "b" => { + let val = if !bytes.is_empty() { bytes[0] as i8 } else { 0 }; + Ok(vm.ctx.new_int(val).into()) + } + "B" => { + let val = if !bytes.is_empty() { bytes[0] } else { 0 }; + Ok(vm.ctx.new_int(val).into()) + } + "h" => { + const SIZE: usize = mem::size_of::<c_short>(); + let val = if bytes.len() >= SIZE { + c_short::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "H" => { + const SIZE: usize = mem::size_of::<c_ushort>(); + let val = if bytes.len() >= SIZE { + c_ushort::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "i" => { + const SIZE: usize = mem::size_of::<c_int>(); + let val = if bytes.len() >= SIZE { + c_int::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "I" => { + const SIZE: usize = mem::size_of::<c_uint>(); + let val = if bytes.len() >= SIZE { + c_uint::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "l" => { + const SIZE: usize = mem::size_of::<c_long>(); + let val = if bytes.len() >= SIZE { + c_long::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "L" => { + const SIZE: usize = mem::size_of::<c_ulong>(); + let val = if bytes.len() >= SIZE { + c_ulong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "q" => { + const SIZE: usize = mem::size_of::<c_longlong>(); + let val = if bytes.len() >= SIZE { + c_longlong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "Q" => { + const SIZE: usize = mem::size_of::<c_ulonglong>(); + let val = if bytes.len() >= SIZE { + c_ulonglong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "f" => { + const SIZE: usize = mem::size_of::<c_float>(); + let val = if bytes.len() >= SIZE { + c_float::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val as f64).into()) + } + "d" => { + const SIZE: usize = mem::size_of::<c_double>(); + let val = if bytes.len() >= SIZE { + c_double::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val).into()) + } + "g" => { + // long double - read as f64 for now since Rust doesn't have native long double + // This may lose precision on platforms where long double > 64 bits + const SIZE: usize = mem::size_of::<c_double>(); + let val = if bytes.len() >= SIZE { + c_double::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val).into()) + } + "?" => { + let val = !bytes.is_empty() && bytes[0] != 0; + Ok(vm.ctx.new_bool(val).into()) + } + "v" => { + // VARIANT_BOOL: non-zero = True, zero = False + const SIZE: usize = mem::size_of::<c_short>(); + let val = if bytes.len() >= SIZE { + c_short::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_bool(val != 0).into()) + } + "z" => { + // c_char_p: read NULL-terminated string from pointer + let ptr = read_ptr_from_buffer(bytes); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + let c_str = unsafe { core::ffi::CStr::from_ptr(ptr as _) }; + Ok(vm.ctx.new_bytes(c_str.to_bytes().to_vec()).into()) + } + "Z" => { + // c_wchar_p: read NULL-terminated wide string from pointer + let ptr = read_ptr_from_buffer(bytes); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + let len = unsafe { libc::wcslen(ptr as *const libc::wchar_t) }; + let wchars = + unsafe { core::slice::from_raw_parts(ptr as *const libc::wchar_t, len) }; + // wchar_t is i32 on some platforms and u32 on others + #[allow( + clippy::unnecessary_cast, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| char::from_u32(c as u32)) + .collect(); + Ok(vm.ctx.new_str(s).into()) + } + "P" => { + // c_void_p: return pointer value as integer + let val = read_ptr_from_buffer(bytes); + if val == 0 { + return Ok(vm.ctx.none()); + } + Ok(vm.ctx.new_int(val).into()) + } + "u" => { + let val = if bytes.len() >= mem::size_of::<WideChar>() { + let wc = if mem::size_of::<WideChar>() == 2 { + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }; + char::from_u32(wc).unwrap_or('\0') + } else { + '\0' + }; + Ok(vm.ctx.new_str(val).into()) + } + _ => Ok(vm.ctx.none()), + }; + } + // Default: return bytes as-is + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) +} + +// Shared functions for Structure and Union types + +/// Parse a non-negative integer attribute, returning default if not present +pub(super) fn get_usize_attr( + obj: &PyObject, + attr: &str, + default: usize, + vm: &VirtualMachine, +) -> PyResult<usize> { + let Ok(attr_val) = obj.get_attr(vm.ctx.intern_str(attr), vm) else { + return Ok(default); + }; + let n = attr_val + .try_int(vm) + .map_err(|_| vm.new_value_error(format!("{attr} must be a non-negative integer")))?; + let val = n.as_bigint(); + if val.is_negative() { + return Err(vm.new_value_error(format!("{attr} must be a non-negative integer"))); + } + Ok(val.to_usize().unwrap_or(default)) +} + +/// Read a pointer value from buffer +#[inline] +pub(super) fn read_ptr_from_buffer(buffer: &[u8]) -> usize { + const PTR_SIZE: usize = core::mem::size_of::<usize>(); + buffer + .first_chunk::<PTR_SIZE>() + .copied() + .map_or(0, usize::from_ne_bytes) +} + +/// Check if a type is a "simple instance" (direct subclass of a simple type) +/// Returns TRUE for c_int, c_void_p, etc. (simple types with _type_ attribute) +/// Returns FALSE for Structure, Array, POINTER(T), etc. +pub(super) fn is_simple_instance(typ: &Py<PyType>) -> bool { + // _ctypes_simple_instance + // Check if the type's metaclass is PyCSimpleType + let metaclass = typ.class(); + metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) +} + +/// Set or initialize StgInfo on a type +pub(super) fn set_or_init_stginfo(type_ref: &PyType, stg_info: StgInfo) { + if type_ref.init_type_data(stg_info.clone()).is_err() + && let Some(mut existing) = type_ref.get_type_data_mut::<StgInfo>() + { + // Preserve pointer_type cache across StgInfo replacement + let old_pointer_type = existing.pointer_type.take(); + *existing = stg_info; + if existing.pointer_type.is_none() { + existing.pointer_type = old_pointer_type; + } + } +} + +/// Check if a field type supports byte order swapping +pub(super) fn check_other_endian_support( + field_type: &PyObject, + vm: &VirtualMachine, +) -> PyResult<()> { + let other_endian_attr = if cfg!(target_endian = "little") { + "__ctype_be__" + } else { + "__ctype_le__" + }; + + if field_type.get_attr(other_endian_attr, vm).is_ok() { + return Ok(()); + } + + // Array type: recursively check element type + if let Ok(elem_type) = field_type.get_attr("_type_", vm) + && field_type.get_attr("_length_", vm).is_ok() + { + return check_other_endian_support(&elem_type, vm); + } + + // Structure/Union: has StgInfo but no _type_ attribute + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && type_obj.stg_info_opt().is_some() + && field_type.get_attr("_type_", vm).is_err() + { + return Ok(()); + } + + Err(vm.new_type_error(format!( + "This type does not support other endian: {}", + field_type.class().name() + ))) +} + +/// Get the size of a ctypes field type +pub(super) fn get_field_size(field_type: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + { + return Ok(stg_info.size); + } + + if let Some(size) = field_type + .get_attr("_type_", vm) + .ok() + .and_then(|type_attr| type_attr.str(vm).ok()) + .and_then(|type_str| { + let s = type_str.to_string(); + (s.len() == 1).then(|| super::get_size(&s)) + }) + { + return Ok(size); + } -impl AsBuffer for PyCSimple { - fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { - let buffer_len = zelf.cdata.read().buffer.len(); - let buf = PyBuffer::new( - zelf.to_owned().into(), - BufferDescriptor::simple(buffer_len, false), // readonly=false for ctypes - &SIMPLE_BUFFER_METHODS, - ); - Ok(buf) + if let Some(s) = field_type + .get_attr("size_of_instances", vm) + .ok() + .and_then(|size_method| size_method.call((), vm).ok()) + .and_then(|size| size.try_int(vm).ok()) + .and_then(|n| n.as_bigint().to_usize()) + { + return Ok(s); + } + + Ok(core::mem::size_of::<usize>()) +} + +/// Get the alignment of a ctypes field type +pub(super) fn get_field_align(field_type: &PyObject, vm: &VirtualMachine) -> usize { + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + && stg_info.align > 0 + { + return stg_info.align; } + + if let Some(align) = field_type + .get_attr("_type_", vm) + .ok() + .and_then(|type_attr| type_attr.str(vm).ok()) + .and_then(|type_str| { + let s = type_str.to_string(); + (s.len() == 1).then(|| super::get_size(&s)) + }) + { + return align; + } + + 1 +} + +/// Promote fields from anonymous struct/union to parent type +fn make_fields( + cls: &Py<PyType>, + descr: &super::PyCField, + index: usize, + offset: isize, + vm: &VirtualMachine, +) -> PyResult<()> { + use crate::builtins::{PyList, PyTuple}; + use crate::convert::ToPyObject; + + let fields = descr.proto.as_object().get_attr("_fields_", vm)?; + let fieldlist: Vec<PyObjectRef> = if let Some(list) = fields.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = fields.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_fields_ must be a sequence")); + }; + + for pair in fieldlist.iter() { + let field_tuple = pair + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; + + if field_tuple.len() < 2 { + continue; + } + + let fname = field_tuple + .first() + .expect("len checked") + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("field name must be a string"))?; + + let fdescr_obj = descr + .proto + .as_object() + .get_attr(vm.ctx.intern_str(fname.as_str()), vm)?; + let fdescr = fdescr_obj + .downcast_ref::<super::PyCField>() + .ok_or_else(|| vm.new_type_error("unexpected type"))?; + + if fdescr.anonymous { + make_fields( + cls, + fdescr, + index + fdescr.index, + offset + fdescr.offset, + vm, + )?; + continue; + } + + let new_descr = super::PyCField::new_from_field(fdescr, index, offset); + cls.set_attr(vm.ctx.intern_str(fname.as_str()), new_descr.to_pyobject(vm)); + } + + Ok(()) +} + +/// Process _anonymous_ attribute for struct/union +pub(super) fn make_anon_fields(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<()> { + use crate::builtins::{PyList, PyTuple}; + use crate::convert::ToPyObject; + + let anon = match cls.as_object().get_attr("_anonymous_", vm) { + Ok(anon) => anon, + Err(_) => return Ok(()), + }; + + let anon_names: Vec<PyObjectRef> = if let Some(list) = anon.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = anon.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_anonymous_ must be a sequence")); + }; + + for fname_obj in anon_names.iter() { + let fname = fname_obj + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("_anonymous_ items must be strings"))?; + + let descr_obj = cls + .as_object() + .get_attr(vm.ctx.intern_str(fname.as_str()), vm)?; + + let descr = descr_obj.downcast_ref::<super::PyCField>().ok_or_else(|| { + vm.new_attribute_error(format!( + "'{}' is specified in _anonymous_ but not in _fields_", + fname.as_str() + )) + })?; + + let mut new_descr = super::PyCField::new_from_field(descr, 0, 0); + new_descr.set_anonymous(true); + cls.set_attr(vm.ctx.intern_str(fname.as_str()), new_descr.to_pyobject(vm)); + + make_fields(cls, descr, descr.index, descr.offset, vm)?; + } + + Ok(()) } diff --git a/crates/vm/src/stdlib/ctypes/field.rs b/crates/vm/src/stdlib/ctypes/field.rs deleted file mode 100644 index e760f07d035..00000000000 --- a/crates/vm/src/stdlib/ctypes/field.rs +++ /dev/null @@ -1,308 +0,0 @@ -use crate::builtins::PyType; -use crate::function::PySetterValue; -use crate::types::{GetDescriptor, Representable, Unconstructible}; -use crate::{AsObject, Py, PyObjectRef, PyResult, VirtualMachine}; -use num_traits::ToPrimitive; - -use super::structure::PyCStructure; -use super::union::PyCUnion; - -#[pyclass(name = "PyCFieldType", base = PyType, module = "_ctypes")] -#[derive(Debug)] -pub struct PyCFieldType { - pub _base: PyType, - #[allow(dead_code)] - pub(super) inner: PyCField, -} - -#[pyclass] -impl PyCFieldType {} - -#[pyclass(name = "CField", module = "_ctypes")] -#[derive(Debug, PyPayload)] -pub struct PyCField { - pub(super) byte_offset: usize, - pub(super) byte_size: usize, - #[allow(unused)] - pub(super) index: usize, - /// The ctypes type for this field (can be any ctypes type including arrays) - pub(super) proto: PyObjectRef, - pub(super) anonymous: bool, - pub(super) bitfield_size: bool, - pub(super) bit_offset: u8, - pub(super) name: String, -} - -impl PyCField { - pub fn new( - name: String, - proto: PyObjectRef, - byte_offset: usize, - byte_size: usize, - index: usize, - ) -> Self { - Self { - name, - proto, - byte_offset, - byte_size, - index, - anonymous: false, - bitfield_size: false, - bit_offset: 0, - } - } -} - -impl Representable for PyCField { - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - // Get type name from the proto object - let tp_name = if let Some(name_attr) = vm - .ctx - .interned_str("__name__") - .and_then(|s| zelf.proto.get_attr(s, vm).ok()) - { - name_attr.str(vm)?.to_string() - } else { - zelf.proto.class().name().to_string() - }; - - if zelf.bitfield_size { - Ok(format!( - "<{} type={}, ofs={byte_offset}, bit_size={bitfield_size}, bit_offset={bit_offset}", - zelf.name, - tp_name, - byte_offset = zelf.byte_offset, - bitfield_size = zelf.bitfield_size, - bit_offset = zelf.bit_offset - )) - } else { - Ok(format!( - "<{} type={tp_name}, ofs={}, size={}", - zelf.name, zelf.byte_offset, zelf.byte_size - )) - } - } -} - -impl Unconstructible for PyCField {} - -impl GetDescriptor for PyCField { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - _cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let zelf = zelf - .downcast::<PyCField>() - .map_err(|_| vm.new_type_error("expected CField".to_owned()))?; - - // If obj is None, return the descriptor itself (class attribute access) - let obj = match obj { - Some(obj) if !vm.is_none(&obj) => obj, - _ => return Ok(zelf.into()), - }; - - // Instance attribute access - read value from the structure/union's buffer - if let Some(structure) = obj.downcast_ref::<PyCStructure>() { - let cdata = structure.cdata.read(); - let offset = zelf.byte_offset; - let size = zelf.byte_size; - - if offset + size <= cdata.buffer.len() { - let bytes = &cdata.buffer[offset..offset + size]; - return PyCField::bytes_to_value(bytes, size, vm); - } - } else if let Some(union) = obj.downcast_ref::<PyCUnion>() { - let cdata = union.cdata.read(); - let offset = zelf.byte_offset; - let size = zelf.byte_size; - - if offset + size <= cdata.buffer.len() { - let bytes = &cdata.buffer[offset..offset + size]; - return PyCField::bytes_to_value(bytes, size, vm); - } - } - - // Fallback: return 0 for uninitialized or unsupported types - Ok(vm.ctx.new_int(0).into()) - } -} - -impl PyCField { - /// Convert bytes to a Python value based on size - fn bytes_to_value(bytes: &[u8], size: usize, vm: &VirtualMachine) -> PyResult { - match size { - 1 => Ok(vm.ctx.new_int(bytes[0] as i8).into()), - 2 => { - let val = i16::from_ne_bytes([bytes[0], bytes[1]]); - Ok(vm.ctx.new_int(val).into()) - } - 4 => { - let val = i32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); - Ok(vm.ctx.new_int(val).into()) - } - 8 => { - let val = i64::from_ne_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ]); - Ok(vm.ctx.new_int(val).into()) - } - _ => Ok(vm.ctx.new_int(0).into()), - } - } - - /// Convert a Python value to bytes - fn value_to_bytes(value: &PyObjectRef, size: usize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - if let Ok(int_val) = value.try_int(vm) { - let i = int_val.as_bigint(); - match size { - 1 => { - let val = i.to_i8().unwrap_or(0); - Ok(val.to_ne_bytes().to_vec()) - } - 2 => { - let val = i.to_i16().unwrap_or(0); - Ok(val.to_ne_bytes().to_vec()) - } - 4 => { - let val = i.to_i32().unwrap_or(0); - Ok(val.to_ne_bytes().to_vec()) - } - 8 => { - let val = i.to_i64().unwrap_or(0); - Ok(val.to_ne_bytes().to_vec()) - } - _ => Ok(vec![0u8; size]), - } - } else { - Ok(vec![0u8; size]) - } - } -} - -#[pyclass( - flags(DISALLOW_INSTANTIATION, IMMUTABLETYPE), - with(Unconstructible, Representable, GetDescriptor) -)] -impl PyCField { - #[pyslot] - fn descr_set( - zelf: &crate::PyObject, - obj: PyObjectRef, - value: PySetterValue<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let zelf = zelf - .downcast_ref::<PyCField>() - .ok_or_else(|| vm.new_type_error("expected CField".to_owned()))?; - - // Get the structure/union instance - use downcast_ref() to access the struct data - if let Some(structure) = obj.downcast_ref::<PyCStructure>() { - match value { - PySetterValue::Assign(value) => { - let offset = zelf.byte_offset; - let size = zelf.byte_size; - let bytes = PyCField::value_to_bytes(&value, size, vm)?; - - let mut cdata = structure.cdata.write(); - if offset + size <= cdata.buffer.len() { - cdata.buffer[offset..offset + size].copy_from_slice(&bytes); - } - Ok(()) - } - PySetterValue::Delete => { - Err(vm.new_type_error("cannot delete structure field".to_owned())) - } - } - } else if let Some(union) = obj.downcast_ref::<PyCUnion>() { - match value { - PySetterValue::Assign(value) => { - let offset = zelf.byte_offset; - let size = zelf.byte_size; - let bytes = PyCField::value_to_bytes(&value, size, vm)?; - - let mut cdata = union.cdata.write(); - if offset + size <= cdata.buffer.len() { - cdata.buffer[offset..offset + size].copy_from_slice(&bytes); - } - Ok(()) - } - PySetterValue::Delete => { - Err(vm.new_type_error("cannot delete union field".to_owned())) - } - } - } else { - Err(vm.new_type_error(format!( - "descriptor works only on Structure or Union instances, got {}", - obj.class().name() - ))) - } - } - - #[pymethod] - fn __set__( - zelf: PyObjectRef, - obj: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Assign(value), vm) - } - - #[pymethod] - fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Delete, vm) - } - - #[pygetset] - fn size(&self) -> usize { - self.byte_size - } - - #[pygetset] - fn bit_size(&self) -> bool { - self.bitfield_size - } - - #[pygetset] - fn is_bitfield(&self) -> bool { - self.bitfield_size - } - - #[pygetset] - fn is_anonymous(&self) -> bool { - self.anonymous - } - - #[pygetset] - fn name(&self) -> String { - self.name.clone() - } - - #[pygetset(name = "type")] - fn type_(&self) -> PyObjectRef { - self.proto.clone() - } - - #[pygetset] - fn offset(&self) -> usize { - self.byte_offset - } - - #[pygetset] - fn byte_offset(&self) -> usize { - self.byte_offset - } - - #[pygetset] - fn byte_size(&self) -> usize { - self.byte_size - } - - #[pygetset] - fn bit_offset(&self) -> u8 { - self.bit_offset - } -} diff --git a/crates/vm/src/stdlib/ctypes/function.rs b/crates/vm/src/stdlib/ctypes/function.rs index b4e600f77ba..d24102ed635 100644 --- a/crates/vm/src/stdlib/ctypes/function.rs +++ b/crates/vm/src/stdlib/ctypes/function.rs @@ -1,207 +1,922 @@ // spell-checker:disable -use crate::builtins::{PyNone, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef}; -use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::stdlib::ctypes::PyCData; -use crate::stdlib::ctypes::base::{CDataObject, PyCSimple, ffi_type_from_str}; -use crate::stdlib::ctypes::thunk::PyCThunk; -use crate::types::Representable; -use crate::types::{Callable, Constructor}; -use crate::{AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; -use crossbeam_utils::atomic::AtomicCell; -use libffi::middle::{Arg, Cif, CodePtr, Type}; +use super::{ + _ctypes::CArgObject, + PyCArray, PyCData, PyCPointer, PyCStructure, StgInfo, + base::{CDATA_BUFFER_METHODS, FfiArgValue, ParamFunc, StgInfoFlags}, + simple::PyCSimple, + type_info, +}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyDict, PyNone, PyStr, PyTuple, PyType, PyTypeRef}, + class::StaticType, + function::FuncArgs, + protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}, + types::{AsBuffer, AsNumber, Callable, Constructor, Initializer, Representable}, + vm::thread::with_current_vm, +}; +use alloc::borrow::Cow; +use core::ffi::c_void; +use core::fmt::Debug; +use libffi::{ + low, + middle::{Arg, Cif, Closure, CodePtr, Type}, +}; use libloading::Symbol; -use num_traits::ToPrimitive; +use num_traits::{Signed, ToPrimitive}; use rustpython_common::lock::PyRwLock; -use std::ffi::{self, c_void}; -use std::fmt::Debug; -// See also: https://github.com/python/cpython/blob/4f8bb3947cfbc20f970ff9d9531e1132a9e95396/Modules/_ctypes/callproc.c#L15 +// Internal function addresses for special ctypes functions +pub(super) const INTERNAL_CAST_ADDR: usize = 1; +pub(super) const INTERNAL_STRING_AT_ADDR: usize = 2; +pub(super) const INTERNAL_WSTRING_AT_ADDR: usize = 3; +pub(super) const INTERNAL_MEMORYVIEW_AT_ADDR: usize = 4; + +// Thread-local errno storage for ctypes +std::thread_local! { + /// Thread-local storage for ctypes errno + /// This is separate from the system errno - ctypes swaps them during FFI calls + /// when use_errno=True is specified. + static CTYPES_LOCAL_ERRNO: core::cell::Cell<i32> = const { core::cell::Cell::new(0) }; +} + +/// Get ctypes thread-local errno value +pub(super) fn get_errno_value() -> i32 { + CTYPES_LOCAL_ERRNO.with(|e| e.get()) +} + +/// Set ctypes thread-local errno value, returns old value +pub(super) fn set_errno_value(value: i32) -> i32 { + CTYPES_LOCAL_ERRNO.with(|e| { + let old = e.get(); + e.set(value); + old + }) +} + +/// Save and restore errno around FFI call (called when use_errno=True) +/// Before: restore thread-local errno to system +/// After: save system errno to thread-local +#[cfg(not(windows))] +fn swap_errno<F, R>(f: F) -> R +where + F: FnOnce() -> R, +{ + // Before call: restore thread-local errno to system + let saved = CTYPES_LOCAL_ERRNO.with(|e| e.get()); + errno::set_errno(errno::Errno(saved)); + + // Call the function + let result = f(); + + // After call: save system errno to thread-local + let new_error = errno::errno().0; + CTYPES_LOCAL_ERRNO.with(|e| e.set(new_error)); + + result +} + +#[cfg(windows)] +std::thread_local! { + /// Thread-local storage for ctypes last_error (Windows only) + static CTYPES_LOCAL_LAST_ERROR: core::cell::Cell<u32> = const { core::cell::Cell::new(0) }; +} + +#[cfg(windows)] +pub(super) fn get_last_error_value() -> u32 { + CTYPES_LOCAL_LAST_ERROR.with(|e| e.get()) +} + +#[cfg(windows)] +pub(super) fn set_last_error_value(value: u32) -> u32 { + CTYPES_LOCAL_LAST_ERROR.with(|e| { + let old = e.get(); + e.set(value); + old + }) +} + +/// Save and restore last_error around FFI call (called when use_last_error=True) +#[cfg(windows)] +fn save_and_restore_last_error<F, R>(f: F) -> R +where + F: FnOnce() -> R, +{ + // Before call: restore thread-local last_error to Windows + let saved = CTYPES_LOCAL_LAST_ERROR.with(|e| e.get()); + unsafe { windows_sys::Win32::Foundation::SetLastError(saved) }; + + // Call the function + let result = f(); + + // After call: save Windows last_error to thread-local + let new_error = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + CTYPES_LOCAL_LAST_ERROR.with(|e| e.set(new_error)); + + result +} type FP = unsafe extern "C" fn(); -pub trait ArgumentType { +/// Get FFI type for a ctypes type code +fn get_ffi_type(ty: &str) -> Option<libffi::middle::Type> { + type_info(ty).map(|t| (t.ffi_type_fn)()) +} + +// PyCFuncPtr - Function pointer implementation + +/// Get FFI type from CArgObject tag character +fn ffi_type_from_tag(tag: u8) -> Type { + match tag { + b'c' | b'b' => Type::i8(), + b'B' => Type::u8(), + b'h' => Type::i16(), + b'H' => Type::u16(), + b'i' => Type::i32(), + b'I' => Type::u32(), + b'l' => { + if core::mem::size_of::<libc::c_long>() == 8 { + Type::i64() + } else { + Type::i32() + } + } + b'L' => { + if core::mem::size_of::<libc::c_ulong>() == 8 { + Type::u64() + } else { + Type::u32() + } + } + b'q' => Type::i64(), + b'Q' => Type::u64(), + b'f' => Type::f32(), + b'd' | b'g' => Type::f64(), + b'?' => Type::u8(), + b'u' => { + if core::mem::size_of::<super::WideChar>() == 2 { + Type::u16() + } else { + Type::u32() + } + } + _ => Type::pointer(), // 'P', 'V', 'z', 'Z', 'O', etc. + } +} + +/// Convert any object to a pointer value for c_void_p arguments +/// Follows ConvParam logic for pointer types +fn convert_to_pointer(value: &PyObject, vm: &VirtualMachine) -> PyResult<FfiArgValue> { + // 0. CArgObject (from byref()) -> buffer address + offset + if let Some(carg) = value.downcast_ref::<CArgObject>() { + // Get buffer address from the underlying object + let base_addr = if let Some(cdata) = carg.obj.downcast_ref::<PyCData>() { + cdata.buffer.read().as_ptr() as usize + } else { + return Err(vm.new_type_error(format!( + "byref() argument must be a ctypes instance, not '{}'", + carg.obj.class().name() + ))); + }; + let addr = (base_addr as isize + carg.offset) as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 1. None -> NULL + if value.is(&vm.ctx.none) { + return Ok(FfiArgValue::Pointer(0)); + } + + // 2. PyCArray -> buffer address (PyCArrayType_paramfunc) + if let Some(array) = value.downcast_ref::<PyCArray>() { + let addr = array.0.buffer.read().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 3. PyCPointer -> stored pointer value + if let Some(ptr) = value.downcast_ref::<PyCPointer>() { + return Ok(FfiArgValue::Pointer(ptr.get_ptr_value())); + } + + // 4. PyCStructure -> buffer address + if let Some(struct_obj) = value.downcast_ref::<PyCStructure>() { + let addr = struct_obj.0.buffer.read().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 5. PyCSimple (c_void_p, c_char_p, etc.) -> value from buffer + if let Some(simple) = value.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + let addr = super::base::read_ptr_from_buffer(&buffer); + return Ok(FfiArgValue::Pointer(addr)); + } + } + + // 6. bytes -> buffer address (PyBytes_AsString) + if let Some(bytes) = value.downcast_ref::<crate::builtins::PyBytes>() { + let addr = bytes.as_bytes().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 7. Integer -> direct value (PyLong_AsVoidPtr behavior) + if let Ok(int_val) = value.try_int(vm) { + let bigint = int_val.as_bigint(); + // Negative values: use signed conversion (allows -1 as 0xFFFF...) + if bigint.is_negative() { + if let Some(signed_val) = bigint.to_isize() { + return Ok(FfiArgValue::Pointer(signed_val as usize)); + } + } else if let Some(unsigned_val) = bigint.to_usize() { + return Ok(FfiArgValue::Pointer(unsigned_val)); + } + // Value out of range - raise OverflowError + return Err(vm.new_overflow_error("int too large to convert to pointer".to_string())); + } + + // 8. Check _as_parameter_ attribute ( recursive ConvParam) + if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { + return convert_to_pointer(&as_param, vm); + } + + Err(vm.new_type_error(format!( + "cannot convert '{}' to c_void_p", + value.class().name() + ))) +} + +/// ConvParam-like conversion for when argtypes is None +/// Returns an Argument with FFI type, value, and optional keep object +fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult<Argument> { + // 1. CArgObject (from byref() or paramfunc) -> use stored type and value + if let Some(carg) = value.downcast_ref::<CArgObject>() { + let ffi_type = ffi_type_from_tag(carg.tag); + return Ok(Argument { + ffi_type, + keep: None, + value: carg.value.clone(), + }); + } + + // 2. None -> NULL pointer + if value.is(&vm.ctx.none) { + return Ok(Argument { + ffi_type: Type::pointer(), + keep: None, + value: FfiArgValue::Pointer(0), + }); + } + + // 3. ctypes objects -> use paramfunc + if let Ok(carg) = super::base::call_paramfunc(value, vm) { + let ffi_type = ffi_type_from_tag(carg.tag); + return Ok(Argument { + ffi_type, + keep: None, + value: carg.value.clone(), + }); + } + + // 4. Python str -> wide string pointer (like PyUnicode_AsWideCharString) + if let Some(s) = value.downcast_ref::<PyStr>() { + // Convert to null-terminated UTF-16 (wide string) + let wide: Vec<u16> = s + .as_str() + .encode_utf16() + .chain(core::iter::once(0)) + .collect(); + let wide_bytes: Vec<u8> = wide.iter().flat_map(|&x| x.to_ne_bytes()).collect(); + let keep = vm.ctx.new_bytes(wide_bytes); + let addr = keep.as_bytes().as_ptr() as usize; + return Ok(Argument { + ffi_type: Type::pointer(), + keep: Some(keep.into()), + value: FfiArgValue::Pointer(addr), + }); + } + + // 9. Python bytes -> null-terminated buffer pointer + // Need to ensure null termination like c_char_p + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let mut buffer = bytes.as_bytes().to_vec(); + buffer.push(0); // Add null terminator + let keep = vm.ctx.new_bytes(buffer); + let addr = keep.as_bytes().as_ptr() as usize; + return Ok(Argument { + ffi_type: Type::pointer(), + keep: Some(keep.into()), + value: FfiArgValue::Pointer(addr), + }); + } + + // 10. Python int -> i32 (default integer type) + if let Ok(int_val) = value.try_int(vm) { + let val = int_val.as_bigint().to_i32().unwrap_or(0); + return Ok(Argument { + ffi_type: Type::i32(), + keep: None, + value: FfiArgValue::I32(val), + }); + } + + // 11. Python float -> f64 + if let Ok(float_val) = value.try_float(vm) { + return Ok(Argument { + ffi_type: Type::f64(), + keep: None, + value: FfiArgValue::F64(float_val.to_f64()), + }); + } + + // 12. Check _as_parameter_ attribute + if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { + return conv_param(&as_param, vm); + } + + Err(vm.new_type_error(format!( + "Don't know how to convert parameter {}", + value.class().name() + ))) +} + +trait ArgumentType { fn to_ffi_type(&self, vm: &VirtualMachine) -> PyResult<Type>; - fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<Arg>; + fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<FfiArgValue>; } impl ArgumentType for PyTypeRef { fn to_ffi_type(&self, vm: &VirtualMachine) -> PyResult<Type> { + use super::pointer::PyCPointer; + use super::structure::PyCStructure; + + // CArgObject (from byref()) should be treated as pointer + if self.fast_issubclass(CArgObject::static_type()) { + return Ok(Type::pointer()); + } + + // Pointer types (POINTER(T)) are always pointer FFI type + // Check if type is a subclass of _Pointer (PyCPointer) + if self.fast_issubclass(PyCPointer::static_type()) { + return Ok(Type::pointer()); + } + + // Structure types are passed as pointers + if self.fast_issubclass(PyCStructure::static_type()) { + return Ok(Type::pointer()); + } + + // Use get_attr to traverse MRO (for subclasses like MyInt(c_int)) let typ = self - .get_class_attr(vm.ctx.intern_str("_type_")) - .ok_or(vm.new_type_error("Unsupported argument type".to_string()))?; + .as_object() + .get_attr(vm.ctx.intern_str("_type_"), vm) + .ok() + .ok_or(vm.new_type_error("Unsupported argument type"))?; let typ = typ .downcast_ref::<PyStr>() - .ok_or(vm.new_type_error("Unsupported argument type".to_string()))?; + .ok_or(vm.new_type_error("Unsupported argument type"))?; let typ = typ.to_string(); let typ = typ.as_str(); - let converted_typ = ffi_type_from_str(typ); - if let Some(typ) = converted_typ { - Ok(typ) - } else { - Err(vm.new_type_error(format!("Unsupported argument type: {}", typ))) - } + get_ffi_type(typ) + .ok_or_else(|| vm.new_type_error(format!("Unsupported argument type: {}", typ))) } - fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<Arg> { - // if self.fast_isinstance::<PyCArray>(vm) { - // let array = value.downcast::<PyCArray>()?; - // return Ok(Arg::from(array.as_ptr())); - // } - if let Ok(simple) = value.downcast::<PyCSimple>() { + fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<FfiArgValue> { + // Call from_param first to convert the value + // converter = PyTuple_GET_ITEM(argtypes, i); + // v = PyObject_CallOneArg(converter, arg); + let from_param = self + .as_object() + .get_attr(vm.ctx.intern_str("from_param"), vm)?; + let converted = from_param.call((value.clone(),), vm)?; + + // Then pass the converted value to ConvParam logic + // CArgObject (from from_param) -> use stored value directly + if let Some(carg) = converted.downcast_ref::<CArgObject>() { + return Ok(carg.value.clone()); + } + + // None -> NULL pointer + if vm.is_none(&converted) { + return Ok(FfiArgValue::Pointer(0)); + } + + // For pointer types (POINTER(T)), we need to pass the pointer VALUE stored in buffer + if self.fast_issubclass(PyCPointer::static_type()) { + if let Some(pointer) = converted.downcast_ref::<PyCPointer>() { + return Ok(FfiArgValue::Pointer(pointer.get_ptr_value())); + } + return convert_to_pointer(&converted, vm); + } + + // For structure types, convert to pointer to structure + if self.fast_issubclass(PyCStructure::static_type()) { + return convert_to_pointer(&converted, vm); + } + + // Get the type code for this argument type + let type_code = self + .as_object() + .get_attr(vm.ctx.intern_str("_type_"), vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + // For pointer types (c_void_p, c_char_p, c_wchar_p), handle as pointer + if matches!(type_code.as_deref(), Some("P") | Some("z") | Some("Z")) { + return convert_to_pointer(&converted, vm); + } + + // PyCSimple (already a ctypes instance from from_param) + if let Ok(simple) = converted.clone().downcast::<PyCSimple>() { let typ = ArgumentType::to_ffi_type(self, vm)?; - let arg = simple - .to_arg(typ, vm) - .ok_or(vm.new_type_error("Unsupported argument type".to_string()))?; - return Ok(arg); + let ffi_value = simple + .to_ffi_value(typ, vm) + .ok_or(vm.new_type_error("Unsupported argument type"))?; + return Ok(ffi_value); } - Err(vm.new_type_error("Unsupported argument type".to_string())) + + Err(vm.new_type_error("Unsupported argument type")) } } -pub trait ReturnType { - fn to_ffi_type(&self) -> Option<Type>; - #[allow(clippy::wrong_self_convention)] - fn from_ffi_type( - &self, - value: *mut ffi::c_void, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>>; +trait ReturnType { + fn to_ffi_type(&self, vm: &VirtualMachine) -> Option<Type>; } impl ReturnType for PyTypeRef { - fn to_ffi_type(&self) -> Option<Type> { - ffi_type_from_str(self.name().to_string().as_str()) - } + fn to_ffi_type(&self, vm: &VirtualMachine) -> Option<Type> { + // Try to get _type_ attribute first (for ctypes types like c_void_p) + if let Ok(type_attr) = self.as_object().get_attr(vm.ctx.intern_str("_type_"), vm) + && let Some(s) = type_attr.downcast_ref::<PyStr>() + && let Some(ffi_type) = get_ffi_type(s.as_str()) + { + return Some(ffi_type); + } - fn from_ffi_type( - &self, - value: *mut ffi::c_void, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - // Get the type code from _type_ attribute - let type_code = self - .get_class_attr(vm.ctx.intern_str("_type_")) - .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + // Check for Structure/Array types (have StgInfo but no _type_) + // _ctypes_get_ffi_type: returns appropriately sized type for struct returns + if let Some(stg_info) = self.stg_info_opt() { + let size = stg_info.size; + // Small structs can be returned in registers + // Match can_return_struct_as_int/can_return_struct_as_sint64 + return Some(if size <= 4 { + Type::i32() + } else if size <= 8 { + Type::i64() + } else { + // Large structs: use pointer-sized return + // (ABI typically returns via hidden pointer parameter) + Type::pointer() + }); + } - let result = match type_code.as_deref() { - Some("b") => vm - .ctx - .new_int(unsafe { *(value as *const i8) } as i32) - .into(), - Some("B") => vm - .ctx - .new_int(unsafe { *(value as *const u8) } as i32) - .into(), - Some("c") => vm - .ctx - .new_bytes(vec![unsafe { *(value as *const u8) }]) - .into(), - Some("h") => vm - .ctx - .new_int(unsafe { *(value as *const i16) } as i32) - .into(), - Some("H") => vm - .ctx - .new_int(unsafe { *(value as *const u16) } as i32) - .into(), - Some("i") => vm.ctx.new_int(unsafe { *(value as *const i32) }).into(), - Some("I") => vm.ctx.new_int(unsafe { *(value as *const u32) }).into(), - Some("l") => vm - .ctx - .new_int(unsafe { *(value as *const libc::c_long) }) - .into(), - Some("L") => vm - .ctx - .new_int(unsafe { *(value as *const libc::c_ulong) }) - .into(), - Some("q") => vm - .ctx - .new_int(unsafe { *(value as *const libc::c_longlong) }) - .into(), - Some("Q") => vm - .ctx - .new_int(unsafe { *(value as *const libc::c_ulonglong) }) - .into(), - Some("f") => vm - .ctx - .new_float(unsafe { *(value as *const f32) } as f64) - .into(), - Some("d") => vm.ctx.new_float(unsafe { *(value as *const f64) }).into(), - Some("P") | Some("z") | Some("Z") => vm.ctx.new_int(value as usize).into(), - Some("?") => vm - .ctx - .new_bool(unsafe { *(value as *const u8) } != 0) - .into(), - None => { - // No _type_ attribute, try to create an instance of the type - // This handles cases like Structure or Array return types - return Ok(Some( - vm.ctx.new_int(unsafe { *(value as *const i32) }).into(), - )); - } - _ => return Err(vm.new_type_error("Unsupported return type".to_string())), - }; - Ok(Some(result)) + // Fallback to class name + get_ffi_type(self.name().to_string().as_str()) } } impl ReturnType for PyNone { - fn to_ffi_type(&self) -> Option<Type> { - ffi_type_from_str("void") + fn to_ffi_type(&self, _vm: &VirtualMachine) -> Option<Type> { + get_ffi_type("void") } +} + +// PyCFuncPtrType - Metaclass for function pointer types +// PyCFuncPtrType_init + +#[pyclass(name = "PyCFuncPtrType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCFuncPtrType(PyType); + +impl Initializer for PyCFuncPtrType { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + new_type.check_not_initialized(vm)?; + + let ptr_size = core::mem::size_of::<usize>(); + let mut stg_info = StgInfo::new(ptr_size, ptr_size); + stg_info.format = Some("X{}".to_string()); + stg_info.length = 1; + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + stg_info.paramfunc = ParamFunc::Pointer; // CFuncPtr is passed as a pointer - fn from_ffi_type( - &self, - _value: *mut ffi::c_void, - _vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - Ok(None) + let _ = new_type.init_type_data(stg_info); + Ok(()) } } -#[pyclass(module = "_ctypes", name = "CFuncPtr", base = PyCData)] -pub struct PyCFuncPtr { - _base: PyCData, - pub name: PyRwLock<Option<String>>, - pub ptr: PyRwLock<Option<CodePtr>>, - #[allow(dead_code)] - pub needs_free: AtomicCell<bool>, - pub arg_types: PyRwLock<Option<Vec<PyTypeRef>>>, - pub res_type: PyRwLock<Option<PyObjectRef>>, - pub _flags_: AtomicCell<i32>, - #[allow(dead_code)] - pub handler: PyObjectRef, +#[pyclass(flags(IMMUTABLETYPE), with(Initializer))] +impl PyCFuncPtrType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } +} + +/// PyCFuncPtr - Function pointer instance +/// Saved in _base.buffer +#[pyclass( + module = "_ctypes", + name = "CFuncPtr", + base = PyCData, + metaclass = "PyCFuncPtrType" +)] +#[repr(C)] +pub(super) struct PyCFuncPtr { + pub _base: PyCData, + /// Thunk for callbacks (keeps thunk alive) + pub thunk: PyRwLock<Option<PyRef<PyCThunk>>>, + /// Original Python callable (for callbacks) + pub callable: PyRwLock<Option<PyObjectRef>>, + /// Converters cache + pub converters: PyRwLock<Option<PyObjectRef>>, + /// Instance-level argtypes override + pub argtypes: PyRwLock<Option<PyObjectRef>>, + /// Instance-level restype override + pub restype: PyRwLock<Option<PyObjectRef>>, + /// Checker function + pub checker: PyRwLock<Option<PyObjectRef>>, + /// Error checking function + pub errcheck: PyRwLock<Option<PyObjectRef>>, + /// COM method vtable index + /// When set, the function reads the function pointer from the vtable at call time + #[cfg(windows)] + pub index: PyRwLock<Option<usize>>, + /// COM method IID (interface ID) for error handling + #[cfg(windows)] + pub iid: PyRwLock<Option<PyObjectRef>>, + /// Parameter flags for COM methods (direction: IN=1, OUT=2, IN|OUT=4) + /// Each element is (direction, name, default) tuple + pub paramflags: PyRwLock<Option<PyObjectRef>>, } impl Debug for PyCFuncPtr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCFuncPtr") - .field("flags", &self._flags_) + .field("func_ptr", &self.get_func_ptr()) .finish() } } +/// Extract pointer value from a ctypes argument (c_void_p conversion) +fn extract_ptr_from_arg(arg: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + // Try CArgObject first - extract the wrapped pointer value, applying offset + if let Some(carg) = arg.downcast_ref::<super::_ctypes::CArgObject>() { + if carg.offset != 0 + && let Some(cdata) = carg.obj.downcast_ref::<PyCData>() + { + let base = cdata.buffer.read().as_ptr() as isize; + return Ok((base + carg.offset) as usize); + } + return extract_ptr_from_arg(&carg.obj, vm); + } + // Try to get pointer value from various ctypes types + if let Some(ptr) = arg.downcast_ref::<PyCPointer>() { + return Ok(ptr.get_ptr_value()); + } + if let Some(simple) = arg.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if let Some(&bytes) = buffer.first_chunk::<{ size_of::<usize>() }>() { + return Ok(usize::from_ne_bytes(bytes)); + } + } + if let Some(cdata) = arg.downcast_ref::<PyCData>() { + // For arrays/structures, return address of buffer + return Ok(cdata.buffer.read().as_ptr() as usize); + } + // PyStr: return internal buffer address + if let Some(s) = arg.downcast_ref::<PyStr>() { + return Ok(s.as_str().as_ptr() as usize); + } + // PyBytes: return internal buffer address + if let Some(bytes) = arg.downcast_ref::<PyBytes>() { + return Ok(bytes.as_bytes().as_ptr() as usize); + } + // Try as integer + if let Ok(int_val) = arg.try_int(vm) { + return Ok(int_val.as_bigint().to_usize().unwrap_or(0)); + } + Err(vm.new_type_error(format!( + "cannot convert '{}' to pointer", + arg.class().name() + ))) +} + +/// string_at implementation - read bytes from memory at ptr +fn string_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let ptr = ptr as *const u8; + let len = if size < 0 { + // size == -1 means use strlen + unsafe { libc::strlen(ptr as _) } + } else { + // Overflow check for huge size values + let size_usize = size as usize; + if size_usize > isize::MAX as usize / 2 { + return Err(vm.new_overflow_error("string too long")); + } + size_usize + }; + let bytes = unsafe { core::slice::from_raw_parts(ptr, len) }; + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) +} + +/// wstring_at implementation - read wide string from memory at ptr +fn wstring_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let w_ptr = ptr as *const libc::wchar_t; + let len = if size < 0 { + unsafe { libc::wcslen(w_ptr) } + } else { + // Overflow check for huge size values + let size_usize = size as usize; + if size_usize > isize::MAX as usize / core::mem::size_of::<libc::wchar_t>() { + return Err(vm.new_overflow_error("string too long")); + } + size_usize + }; + let wchars = unsafe { core::slice::from_raw_parts(w_ptr, len) }; + + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide + // macOS/Linux: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = wchars.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + Ok(vm.ctx.new_str(wtf8).into()) + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + Ok(vm.ctx.new_str(s).into()) + } +} + +/// A buffer wrapping raw memory at a given pointer, for zero-copy memoryview. +#[pyclass(name = "_RawMemoryBuffer", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub(super) struct RawMemoryBuffer { + ptr: *const u8, + size: usize, + readonly: bool, +} + +// SAFETY: The caller ensures the pointer remains valid +unsafe impl Send for RawMemoryBuffer {} +unsafe impl Sync for RawMemoryBuffer {} + +static RAW_MEMORY_BUFFER_METHODS: crate::protocol::BufferMethods = crate::protocol::BufferMethods { + obj_bytes: |buffer| { + let raw = buffer.obj_as::<RawMemoryBuffer>(); + let slice = unsafe { core::slice::from_raw_parts(raw.ptr, raw.size) }; + rustpython_common::borrow::BorrowedValue::Ref(slice) + }, + obj_bytes_mut: |buffer| { + let raw = buffer.obj_as::<RawMemoryBuffer>(); + let slice = unsafe { core::slice::from_raw_parts_mut(raw.ptr as *mut u8, raw.size) }; + rustpython_common::borrow::BorrowedValueMut::RefMut(slice) + }, + release: |_| {}, + retain: |_| {}, +}; + +#[pyclass(with(AsBuffer))] +impl RawMemoryBuffer {} + +impl AsBuffer for RawMemoryBuffer { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + Ok(PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::simple(zelf.size, zelf.readonly), + &RAW_MEMORY_BUFFER_METHODS, + )) + } +} + +/// memoryview_at implementation - create a memoryview from memory at ptr +fn memoryview_at_impl(ptr: usize, size: isize, readonly: bool, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyMemoryView; + + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + if size < 0 { + return Err(vm.new_value_error("negative size")); + } + let len = size as usize; + let raw_buf = RawMemoryBuffer { + ptr: ptr as *const u8, + size: len, + readonly, + } + .into_pyobject(vm); + let mv = PyMemoryView::from_object(&raw_buf, vm)?; + Ok(mv.into_pyobject(vm)) +} + +// cast_check_pointertype +fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { + use super::pointer::PyCPointerType; + + // PyCPointerTypeObject_Check + if ctype.class().fast_issubclass(PyCPointerType::static_type()) { + return true; + } + + // PyCFuncPtrTypeObject_Check - TODO + + // simple pointer types via StgInfo.proto (c_void_p, c_char_p, etc.) + if let Ok(type_attr) = ctype.get_attr("_type_", vm) + && let Some(s) = type_attr.downcast_ref::<PyStr>() + { + let c = s.as_str(); + if c.len() == 1 && "sPzUZXO".contains(c) { + return true; + } + } + + false +} + +/// cast implementation +/// _ctypes.c cast() +pub(super) fn cast_impl( + obj: PyObjectRef, + src: PyObjectRef, + ctype: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + // 1. cast_check_pointertype + if !cast_check_pointertype(&ctype, vm) { + return Err(vm.new_type_error(format!( + "cast() argument 2 must be a pointer type, not {}", + ctype.class().name() + ))); + } + + // 2. Extract pointer value - matches c_void_p_from_param_impl order + let ptr_value: usize = if vm.is_none(&obj) { + // None → NULL pointer + 0 + } else if let Ok(int_val) = obj.try_int(vm) { + // int/long → direct pointer value + int_val.as_bigint().to_usize().unwrap_or(0) + } else if let Some(bytes) = obj.downcast_ref::<PyBytes>() { + // bytes → buffer address (c_void_p_from_param: PyBytes_Check) + bytes.as_bytes().as_ptr() as usize + } else if let Some(s) = obj.downcast_ref::<PyStr>() { + // unicode/str → buffer address (c_void_p_from_param: PyUnicode_Check) + s.as_str().as_ptr() as usize + } else if let Some(ptr) = obj.downcast_ref::<PyCPointer>() { + // Pointer instance → contained pointer value + ptr.get_ptr_value() + } else if let Some(simple) = obj.downcast_ref::<PyCSimple>() { + // Simple type (c_void_p, c_char_p, etc.) → value from buffer + let buffer = simple.0.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } else if let Some(cdata) = obj.downcast_ref::<PyCData>() { + // Array, Structure, Union → buffer address (b_ptr) + cdata.buffer.read().as_ptr() as usize + } else { + return Err(vm.new_type_error(format!( + "cast() argument 1 must be a ctypes instance, not {}", + obj.class().name() + ))); + }; + + // 3. Create result instance + let result = ctype.call((), vm)?; + + // 4. _objects reference tracking + // Share _objects dict between source and result, add id(src): src + if src.class().fast_issubclass(PyCData::static_type()) { + // Get the source's _objects, create dict if needed + let shared_objects: PyObjectRef = if let Some(src_cdata) = src.downcast_ref::<PyCData>() { + let mut src_objects = src_cdata.objects.write(); + if src_objects.is_none() { + // Create new dict + let dict = vm.ctx.new_dict(); + *src_objects = Some(dict.clone().into()); + dict.into() + } else if let Some(obj) = src_objects.as_ref() { + if obj.downcast_ref::<PyDict>().is_none() { + // Convert to dict (keep existing reference) + let dict = vm.ctx.new_dict(); + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + let _ = dict.set_item(&*id_key, obj.clone(), vm); + *src_objects = Some(dict.clone().into()); + dict.into() + } else { + obj.clone() + } + } else { + vm.ctx.new_dict().into() + } + } else { + vm.ctx.new_dict().into() + }; + + // Add id(src): src to the shared dict + if let Some(dict) = shared_objects.downcast_ref::<PyDict>() { + let id_key: PyObjectRef = vm.ctx.new_int(src.get_id() as i64).into(); + let _ = dict.set_item(&*id_key, src.clone(), vm); + } + + // Set result's _objects to the shared dict + if let Some(result_cdata) = result.downcast_ref::<PyCData>() { + *result_cdata.objects.write() = Some(shared_objects); + } + } + + // 5. Store pointer value + if let Some(ptr) = result.downcast_ref::<PyCPointer>() { + ptr.set_ptr_value(ptr_value); + } else if let Some(cdata) = result.downcast_ref::<PyCData>() { + let bytes = ptr_value.to_ne_bytes(); + let mut buffer = cdata.buffer.write(); + let buf = buffer.to_mut(); + if buf.len() >= bytes.len() { + buf[..bytes.len()].copy_from_slice(&bytes); + } + } + + Ok(result) +} + +impl PyCFuncPtr { + /// Get function pointer address from buffer + fn get_func_ptr(&self) -> usize { + let buffer = self._base.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } + + /// Get CodePtr from buffer for FFI calls + fn get_code_ptr(&self) -> Option<CodePtr> { + let addr = self.get_func_ptr(); + if addr != 0 { + Some(CodePtr(addr as *mut _)) + } else { + None + } + } + + /// Create buffer with function pointer address + fn make_ptr_buffer(addr: usize) -> Vec<u8> { + addr.to_ne_bytes().to_vec() + } +} + impl Constructor for PyCFuncPtr { type Args = FuncArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // Handle different argument forms like CPython: - // 1. Empty args: create uninitialized + // Handle different argument forms: + // 1. Empty args: create uninitialized (NULL pointer) // 2. One integer argument: function address // 3. Tuple argument: (name, dll) form + // 4. Callable: callback creation + + let ptr_size = core::mem::size_of::<usize>(); if args.args.is_empty() { return PyCFuncPtr { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - ptr: PyRwLock::new(None), - needs_free: AtomicCell::new(false), - arg_types: PyRwLock::new(None), - _flags_: AtomicCell::new(0), - res_type: PyRwLock::new(None), - name: PyRwLock::new(None), - handler: vm.ctx.none(), + _base: PyCData::from_bytes(vec![0u8; ptr_size], None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), } .into_ref_with_type(vm, cls) .map(Into::into); @@ -209,18 +924,68 @@ impl Constructor for PyCFuncPtr { let first_arg = &args.args[0]; + // Check for COM method form: (index, name, [paramflags], [iid]) + // First arg is integer (vtable index), second arg is string (method name) + if args.args.len() >= 2 + && first_arg.try_int(vm).is_ok() + && args.args[1].downcast_ref::<PyStr>().is_some() + { + #[cfg(windows)] + let index = first_arg.try_int(vm)?.as_bigint().to_usize().unwrap_or(0); + + // args[3] is iid (GUID struct, optional) + // Also check if args[2] is a GUID (has Data1 attribute) when args[3] is not present + #[cfg(windows)] + let iid = args.args.get(3).cloned().or_else(|| { + args.args.get(2).and_then(|arg| { + // If it's a GUID struct (has Data1 attribute), use it as IID + if arg.get_attr("Data1", vm).is_ok() { + Some(arg.clone()) + } else { + None + } + }) + }); + + // args[2] is paramflags (tuple or None) + let paramflags = args.args.get(2).filter(|arg| !vm.is_none(arg)).cloned(); + + return PyCFuncPtr { + _base: PyCData::from_bytes(vec![0u8; ptr_size], None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(Some(index)), + #[cfg(windows)] + iid: PyRwLock::new(iid), + paramflags: PyRwLock::new(paramflags), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + // Check if first argument is an integer (function address) if let Ok(addr) = first_arg.try_int(vm) { let ptr_val = addr.as_bigint().to_usize().unwrap_or(0); return PyCFuncPtr { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - ptr: PyRwLock::new(Some(CodePtr(ptr_val as *mut _))), - needs_free: AtomicCell::new(false), - arg_types: PyRwLock::new(None), - _flags_: AtomicCell::new(0), - res_type: PyRwLock::new(None), - name: PyRwLock::new(Some(format!("CFuncPtr@{:#x}", ptr_val))), - handler: vm.ctx.new_int(ptr_val).into(), + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), } .into_ref_with_type(vm, cls) .map(Into::into); @@ -234,53 +999,64 @@ impl Constructor for PyCFuncPtr { .downcast_ref::<PyStr>() .ok_or(vm.new_type_error("Expected a string"))? .to_string(); - let handler = tuple + let dll = tuple .iter() .nth(1) .ok_or(vm.new_type_error("Expected a tuple with at least 2 elements"))? .clone(); // Get library handle and load function - let handle = handler.try_int(vm); + let handle = dll.try_int(vm); let handle = match handle { Ok(handle) => handle.as_bigint().clone(), - Err(_) => handler + Err(_) => dll .get_attr("_handle", vm)? .try_int(vm)? .as_bigint() .clone(), }; - let library_cache = crate::stdlib::ctypes::library::libcache().read(); + let library_cache = super::library::libcache().read(); let library = library_cache .get_lib( handle .to_usize() - .ok_or(vm.new_value_error("Invalid handle".to_string()))?, + .ok_or(vm.new_value_error("Invalid handle"))?, ) - .ok_or_else(|| vm.new_value_error("Library not found".to_string()))?; + .ok_or_else(|| vm.new_value_error("Library not found"))?; let inner_lib = library.lib.lock(); let terminated = format!("{}\0", &name); - let code_ptr = if let Some(lib) = &*inner_lib { + let ptr_val = if let Some(lib) = &*inner_lib { let pointer: Symbol<'_, FP> = unsafe { lib.get(terminated.as_bytes()) .map_err(|err| err.to_string()) .map_err(|err| vm.new_attribute_error(err))? }; - Some(CodePtr(*pointer as *mut _)) + let addr = *pointer as usize; + // dlsym can return NULL for symbols that resolve to NULL (e.g., GNU IFUNC) + // Treat NULL addresses as errors + if addr == 0 { + return Err(vm.new_attribute_error(format!("function '{}' not found", name))); + } + addr } else { - None + 0 }; return PyCFuncPtr { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - ptr: PyRwLock::new(code_ptr), - needs_free: AtomicCell::new(false), - arg_types: PyRwLock::new(None), - _flags_: AtomicCell::new(0), - res_type: PyRwLock::new(None), - name: PyRwLock::new(Some(name)), - handler, + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), } .into_ref_with_type(vm, cls) .map(Into::into); @@ -289,42 +1065,41 @@ impl Constructor for PyCFuncPtr { // Check if first argument is a Python callable (callback creation) if first_arg.is_callable() { // Get argument types and result type from the class - let argtypes = cls.get_attr(vm.ctx.intern_str("_argtypes_")); - let restype = cls.get_attr(vm.ctx.intern_str("_restype_")); + let class_argtypes = cls.get_attr(vm.ctx.intern_str("_argtypes_")); + let class_restype = cls.get_attr(vm.ctx.intern_str("_restype_")); + let class_flags = cls + .get_attr(vm.ctx.intern_str("_flags_")) + .and_then(|f| f.try_to_value::<u32>(vm).ok()) + .unwrap_or(0); // Create the thunk (C-callable wrapper for the Python function) - let thunk = PyCThunk::new(first_arg.clone(), argtypes.clone(), restype.clone(), vm)?; + let thunk = PyCThunk::new( + first_arg.clone(), + class_argtypes.clone(), + class_restype.clone(), + class_flags, + vm, + )?; let code_ptr = thunk.code_ptr(); - - // Parse argument types for storage - let arg_type_vec: Option<Vec<PyTypeRef>> = if let Some(ref args) = argtypes { - if vm.is_none(args) { - None - } else { - let mut types = Vec::new(); - for item in args.try_to_value::<Vec<PyObjectRef>>(vm)? { - types.push(item.downcast::<PyType>().map_err(|_| { - vm.new_type_error("_argtypes_ must be a sequence of types".to_string()) - })?); - } - Some(types) - } - } else { - None - }; + let ptr_val = code_ptr.0 as usize; // Store the thunk as a Python object to keep it alive let thunk_ref: PyRef<PyCThunk> = thunk.into_ref(&vm.ctx); return PyCFuncPtr { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - ptr: PyRwLock::new(Some(code_ptr)), - needs_free: AtomicCell::new(true), - arg_types: PyRwLock::new(arg_type_vec), - _flags_: AtomicCell::new(0), - res_type: PyRwLock::new(restype), - name: PyRwLock::new(Some("<callback>".to_string())), - handler: thunk_ref.into(), + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(Some(thunk_ref)), + callable: PyRwLock::new(Some(first_arg.clone())), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(class_argtypes), + restype: PyRwLock::new(class_restype), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), } .into_ref_with_type(vm, cls) .map(Into::into); @@ -338,142 +1113,1213 @@ impl Constructor for PyCFuncPtr { } } -impl Callable for PyCFuncPtr { - type Args = FuncArgs; - fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { - // This is completely seperate from the C python implementation - - // Cif init - let arg_types: Vec<_> = match zelf.arg_types.read().clone() { - Some(tys) => tys, - None => args - .args - .clone() - .into_iter() - .map(|a| a.class().as_object().to_pyobject(vm).downcast().unwrap()) - .collect(), - }; - let ffi_arg_types = arg_types - .clone() - .iter() - .map(|t| ArgumentType::to_ffi_type(t, vm)) - .collect::<PyResult<Vec<_>>>()?; - let return_type = zelf.res_type.read(); - let ffi_return_type = return_type - .as_ref() - .and_then(|t| t.clone().downcast::<PyType>().ok()) - .and_then(|t| ReturnType::to_ffi_type(&t)) - .unwrap_or_else(Type::i32); - let cif = Cif::new(ffi_arg_types, ffi_return_type); - - // Call the function - let ffi_args = args - .args - .into_iter() - .enumerate() - .map(|(n, arg)| { - let arg_type = arg_types - .get(n) - .ok_or_else(|| vm.new_type_error("argument amount mismatch".to_string()))?; - arg_type.convert_object(arg, vm) - }) - .collect::<Result<Vec<_>, _>>()?; - let pointer = zelf.ptr.read(); - let code_ptr = pointer - .as_ref() - .ok_or_else(|| vm.new_type_error("Function pointer not set".to_string()))?; - let mut output: c_void = unsafe { cif.call(*code_ptr, &ffi_args) }; - let return_type = return_type - .as_ref() - .and_then(|f| f.clone().downcast::<PyType>().ok()) - .map(|f| f.from_ffi_type(&mut output, vm).ok().flatten()) - .unwrap_or_else(|| Some(vm.ctx.new_int(output as i32).as_object().to_pyobject(vm))); - if let Some(return_type) = return_type { - Ok(return_type) - } else { - Ok(vm.ctx.none()) - } - } -} +// PyCFuncPtr call helpers (similar to callproc.c flow) -impl Representable for PyCFuncPtr { - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - let index = zelf.ptr.read(); - let index = index.map(|ptr| ptr.0 as usize).unwrap_or(0); - let type_name = zelf.class().name(); - if cfg!(windows) { - let index = index - 0x1000; - Ok(format!("<COM method offset {index:#x} {type_name}>")) - } else { - Ok(format!("<{type_name} object at {index:#x}>")) - } +/// Handle internal function addresses (PYFUNCTYPE special cases) +/// Returns Some(result) if handled, None if should continue with normal call +fn handle_internal_func(addr: usize, args: &FuncArgs, vm: &VirtualMachine) -> Option<PyResult> { + if addr == INTERNAL_CAST_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, PyObjectRef)> = args.clone().bind(vm); + return Some(result.and_then(|(obj, src, ctype)| cast_impl(obj, src, ctype, vm))); } -} -// TODO: fix -unsafe impl Send for PyCFuncPtr {} -unsafe impl Sync for PyCFuncPtr {} - -#[pyclass(flags(BASETYPE), with(Callable, Constructor, Representable))] -impl PyCFuncPtr { - #[pygetset(name = "_restype_")] - fn restype(&self) -> Option<PyObjectRef> { - self.res_type.read().as_ref().cloned() + if addr == INTERNAL_STRING_AT_ADDR { + let result: PyResult<(PyObjectRef, Option<PyObjectRef>)> = args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size = size_arg + .and_then(|s| s.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_isize()) + .unwrap_or(-1); + string_at_impl(ptr, size, vm) + })); } - #[pygetset(name = "_restype_", setter)] - fn set_restype(&self, restype: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // has to be type, callable, or none - // TODO: Callable support - if vm.is_none(&restype) || restype.downcast_ref::<PyType>().is_some() { - *self.res_type.write() = Some(restype); - Ok(()) - } else { - Err(vm.new_type_error("restype must be a type, a callable, or None".to_string())) - } + if addr == INTERNAL_WSTRING_AT_ADDR { + let result: PyResult<(PyObjectRef, Option<PyObjectRef>)> = args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size = size_arg + .and_then(|s| s.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_isize()) + .unwrap_or(-1); + wstring_at_impl(ptr, size, vm) + })); } - #[pygetset(name = "argtypes")] - fn argtypes(&self, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref( - self.arg_types - .read() - .clone() - .unwrap_or_default() - .into_iter() - .map(|t| t.to_pyobject(vm)) - .collect(), - &vm.ctx, - ) + if addr == INTERNAL_MEMORYVIEW_AT_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, Option<PyObjectRef>)> = + args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg, readonly_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size_int = size_arg.try_int(vm)?; + let size = size_int + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("size too large"))?; + let readonly = readonly_arg + .and_then(|r| r.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_i32()) + .unwrap_or(0) + != 0; + memoryview_at_impl(ptr, size, readonly, vm) + })); + } + + None +} + +/// Call information extracted from PyCFuncPtr (argtypes, restype, etc.) +struct CallInfo { + explicit_arg_types: Option<Vec<PyTypeRef>>, + restype_obj: Option<PyObjectRef>, + restype_is_none: bool, + ffi_return_type: Type, + is_pointer_return: bool, +} + +/// Extract call information (argtypes, restype) from PyCFuncPtr +fn extract_call_info(zelf: &Py<PyCFuncPtr>, vm: &VirtualMachine) -> PyResult<CallInfo> { + // Get argtypes - first from instance, then from type's _argtypes_ + let explicit_arg_types: Option<Vec<PyTypeRef>> = + if let Some(argtypes_obj) = zelf.argtypes.read().as_ref() { + if !vm.is_none(argtypes_obj) { + Some( + argtypes_obj + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .filter_map(|obj| obj.downcast::<PyType>().ok()) + .collect(), + ) + } else { + None // argtypes is None -> use ConvParam + } + } else if let Some(class_argtypes) = zelf + .as_object() + .class() + .get_attr(vm.ctx.intern_str("_argtypes_")) + && !vm.is_none(&class_argtypes) + { + Some( + class_argtypes + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .filter_map(|obj| obj.downcast::<PyType>().ok()) + .collect(), + ) + } else { + None // No argtypes -> use ConvParam + }; + + // Get restype - first from instance, then from class's _restype_ + let restype_obj = zelf.restype.read().clone().or_else(|| { + zelf.as_object() + .class() + .get_attr(vm.ctx.intern_str("_restype_")) + }); + + // Check if restype is explicitly None (return void) + let restype_is_none = restype_obj.as_ref().is_some_and(|t| vm.is_none(t)); + let ffi_return_type = if restype_is_none { + Type::void() + } else { + restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .and_then(|t| ReturnType::to_ffi_type(&t, vm)) + .unwrap_or_else(Type::i32) + }; + + // Check if return type is a pointer type via TYPEFLAG_ISPOINTER + // This handles c_void_p, c_char_p, c_wchar_p, and POINTER(T) types + let is_pointer_return = restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .and_then(|t| { + t.stg_info_opt() + .map(|info| info.flags.contains(StgInfoFlags::TYPEFLAG_ISPOINTER)) + }) + .unwrap_or(false); + + Ok(CallInfo { + explicit_arg_types, + restype_obj, + restype_is_none, + ffi_return_type, + is_pointer_return, + }) +} + +/// Parsed paramflags: (direction, name, default) tuples +/// direction: 1=IN, 2=OUT, 4=IN|OUT (or 1|2=3) +type ParsedParamFlags = Vec<(u32, Option<String>, Option<PyObjectRef>)>; + +/// Parse paramflags from PyCFuncPtr +fn parse_paramflags( + zelf: &Py<PyCFuncPtr>, + vm: &VirtualMachine, +) -> PyResult<Option<ParsedParamFlags>> { + let Some(pf) = zelf.paramflags.read().as_ref().cloned() else { + return Ok(None); + }; + + let pf_vec = pf.try_to_value::<Vec<PyObjectRef>>(vm)?; + let parsed = pf_vec + .into_iter() + .map(|item| { + let Some(tuple) = item.downcast_ref::<PyTuple>() else { + // Single value means just the direction + let direction = item + .try_int(vm) + .ok() + .and_then(|i| i.as_bigint().to_u32()) + .unwrap_or(1); + return (direction, None, None); + }; + let direction = tuple + .first() + .and_then(|d| d.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_u32()) + .unwrap_or(1); + let name = tuple + .get(1) + .and_then(|n| n.downcast_ref::<PyStr>().map(|s| s.to_string())); + let default = tuple.get(2).cloned(); + (direction, name, default) + }) + .collect(); + Ok(Some(parsed)) +} + +/// Resolve COM method pointer from vtable (Windows only) +/// Returns (Some(CodePtr), true) if this is a COM method call, (None, false) otherwise +#[cfg(windows)] +fn resolve_com_method( + zelf: &Py<PyCFuncPtr>, + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult<(Option<CodePtr>, bool)> { + let com_index = zelf.index.read(); + let Some(idx) = *com_index else { + return Ok((None, false)); + }; + + // First arg must be the COM object pointer + if args.args.is_empty() { + return Err( + vm.new_type_error("COM method requires at least one argument (self)".to_string()) + ); + } + + // Extract COM pointer value from first argument + let self_arg = &args.args[0]; + let com_ptr = if let Some(simple) = self_arg.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + super::base::read_ptr_from_buffer(&buffer) + } else { + 0 + } + } else if let Ok(int_val) = self_arg.try_int(vm) { + int_val.as_bigint().to_usize().unwrap_or(0) + } else { + return Err( + vm.new_type_error("COM method first argument must be a COM pointer".to_string()) + ); + }; + + if com_ptr == 0 { + return Err(vm.new_value_error("NULL COM pointer access")); + } + + // Read vtable pointer from COM object: vtable = *(void**)com_ptr + let vtable_ptr = unsafe { *(com_ptr as *const usize) }; + if vtable_ptr == 0 { + return Err(vm.new_value_error("NULL vtable pointer")); + } + + // Read function pointer from vtable: func = vtable[index] + let fptr = unsafe { + let vtable = vtable_ptr as *const usize; + *vtable.add(idx) + }; + + if fptr == 0 { + return Err(vm.new_value_error("NULL function pointer in vtable")); + } + + Ok((Some(CodePtr(fptr as *mut _)), true)) +} + +/// Single argument for FFI call +// struct argument +struct Argument { + ffi_type: Type, + value: FfiArgValue, + #[allow(dead_code)] + keep: Option<PyObjectRef>, // Object to keep alive during call +} + +/// Out buffers for paramflags OUT parameters +type OutBuffers = Vec<(usize, PyObjectRef)>; + +/// Get buffer address from a ctypes object +fn get_buffer_addr(obj: &PyObjectRef) -> Option<usize> { + obj.downcast_ref::<PyCSimple>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + .or_else(|| { + obj.downcast_ref::<super::structure::PyCStructure>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + }) + .or_else(|| { + obj.downcast_ref::<PyCPointer>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + }) +} + +/// Create OUT buffer for a parameter type +fn create_out_buffer(arg_type: &PyTypeRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // For POINTER(T) types, create T instance (the pointed-to type) + if arg_type.fast_issubclass(PyCPointer::static_type()) + && let Some(stg_info) = arg_type.stg_info_opt() + && let Some(ref proto) = stg_info.proto + { + return proto.as_object().call((), vm); + } + // Not a pointer type or no proto, create instance directly + arg_type.as_object().call((), vm) +} + +/// Build callargs when no argtypes specified (use ConvParam) +fn build_callargs_no_argtypes( + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let arguments: Vec<Argument> = args + .args + .iter() + .map(|arg| conv_param(arg, vm)) + .collect::<PyResult<Vec<_>>>()?; + Ok((arguments, Vec::new())) +} + +/// Build callargs for regular function with argtypes (no paramflags) +fn build_callargs_simple( + args: &FuncArgs, + arg_types: &[PyTypeRef], + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let arguments: Vec<Argument> = args + .args + .iter() + .enumerate() + .map(|(n, arg)| { + let arg_type = arg_types + .get(n) + .ok_or_else(|| vm.new_type_error("argument amount mismatch"))?; + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + let value = arg_type.convert_object(arg.clone(), vm)?; + Ok(Argument { + ffi_type, + keep: None, + value, + }) + }) + .collect::<PyResult<Vec<_>>>()?; + Ok((arguments, Vec::new())) +} + +/// Build callargs with paramflags (handles IN/OUT parameters) +fn build_callargs_with_paramflags( + args: &FuncArgs, + arg_types: &[PyTypeRef], + paramflags: &ParsedParamFlags, + skip_first_arg: bool, // true for COM methods + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let mut arguments = Vec::new(); + let mut out_buffers = Vec::new(); + + // For COM methods, first arg is self (pointer) + let mut caller_arg_idx = if skip_first_arg { + if !args.args.is_empty() { + let arg = conv_param(&args.args[0], vm)?; + arguments.push(arg); + } + 1usize + } else { + 0usize + }; + + // Process parameters based on paramflags + for (param_idx, (direction, _name, default)) in paramflags.iter().enumerate() { + let arg_type = arg_types + .get(param_idx) + .ok_or_else(|| vm.new_type_error("paramflags/argtypes mismatch"))?; + + let is_out = (*direction & 2) != 0; // OUT flag + let is_in = (*direction & 1) != 0 || *direction == 0; // IN flag or default + + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + + if is_out && !is_in { + // Pure OUT parameter: create buffer, don't consume caller arg + let buffer = create_out_buffer(arg_type, vm)?; + let addr = get_buffer_addr(&buffer).ok_or_else(|| { + vm.new_type_error("Cannot create OUT buffer for this type".to_string()) + })?; + arguments.push(Argument { + ffi_type, + keep: None, + value: FfiArgValue::Pointer(addr), + }); + out_buffers.push((param_idx, buffer)); + } else { + // IN or IN|OUT: get from caller args or default + let arg = if caller_arg_idx < args.args.len() { + caller_arg_idx += 1; + args.args[caller_arg_idx - 1].clone() + } else if let Some(def) = default { + def.clone() + } else { + return Err(vm.new_type_error(format!("required argument {} missing", param_idx))); + }; + + if is_out { + // IN|OUT: track for return + out_buffers.push((param_idx, arg.clone())); + } + let value = arg_type.convert_object(arg, vm)?; + arguments.push(Argument { + ffi_type, + keep: None, + value, + }); + } + } + + Ok((arguments, out_buffers)) +} + +/// Build call arguments (main dispatcher) +fn build_callargs( + args: &FuncArgs, + call_info: &CallInfo, + paramflags: Option<&ParsedParamFlags>, + is_com_method: bool, + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let Some(ref arg_types) = call_info.explicit_arg_types else { + // No argtypes: use ConvParam + return build_callargs_no_argtypes(args, vm); + }; + + if let Some(pflags) = paramflags { + // Has paramflags: handle IN/OUT + build_callargs_with_paramflags(args, arg_types, pflags, is_com_method, vm) + } else if is_com_method { + // COM method without paramflags + let mut arguments = Vec::new(); + if !args.args.is_empty() { + arguments.push(conv_param(&args.args[0], vm)?); + } + for (n, arg) in args.args.iter().skip(1).enumerate() { + let arg_type = arg_types + .get(n) + .ok_or_else(|| vm.new_type_error("argument amount mismatch"))?; + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + let value = arg_type.convert_object(arg.clone(), vm)?; + arguments.push(Argument { + ffi_type, + keep: None, + value, + }); + } + Ok((arguments, Vec::new())) + } else { + // Regular function + build_callargs_simple(args, arg_types, vm) + } +} + +/// Raw result from FFI call +enum RawResult { + Void, + Pointer(usize), + Value(libffi::low::ffi_arg), +} + +/// Execute FFI call +fn ctypes_callproc(code_ptr: CodePtr, arguments: &[Argument], call_info: &CallInfo) -> RawResult { + let ffi_arg_types: Vec<Type> = arguments.iter().map(|a| a.ffi_type.clone()).collect(); + let cif = Cif::new(ffi_arg_types, call_info.ffi_return_type.clone()); + let ffi_args: Vec<Arg<'_>> = arguments.iter().map(|a| a.value.as_arg()).collect(); + + if call_info.restype_is_none { + unsafe { cif.call::<()>(code_ptr, &ffi_args) }; + RawResult::Void + } else if call_info.is_pointer_return { + let result = unsafe { cif.call::<usize>(code_ptr, &ffi_args) }; + RawResult::Pointer(result) + } else { + let result = unsafe { cif.call::<libffi::low::ffi_arg>(code_ptr, &ffi_args) }; + RawResult::Value(result) + } +} + +/// Check and handle HRESULT errors (Windows) +#[cfg(windows)] +fn check_hresult(hresult: i32, zelf: &Py<PyCFuncPtr>, vm: &VirtualMachine) -> PyResult<()> { + if hresult >= 0 { + return Ok(()); + } + + if zelf.iid.read().is_some() { + // Raise COMError + let ctypes_module = vm.import("_ctypes", 0)?; + let com_error_type = ctypes_module.get_attr("COMError", vm)?; + let com_error_type = com_error_type + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("COMError is not a type"))?; + let hresult_obj: PyObjectRef = vm.ctx.new_int(hresult).into(); + let text: PyObjectRef = vm + .ctx + .new_str(format!("HRESULT: 0x{:08X}", hresult as u32)) + .into(); + let details: PyObjectRef = vm.ctx.none(); + let exc = vm.invoke_exception( + com_error_type.to_owned(), + vec![text.clone(), details.clone()], + )?; + let _ = exc.as_object().set_attr("hresult", hresult_obj, vm); + let _ = exc.as_object().set_attr("text", text, vm); + let _ = exc.as_object().set_attr("details", details, vm); + Err(exc) + } else { + // Raise OSError + let exc = vm.new_os_error(format!("HRESULT: 0x{:08X}", hresult as u32)); + let _ = exc + .as_object() + .set_attr("winerror", vm.ctx.new_int(hresult), vm); + Err(exc) + } +} + +/// Convert raw FFI result to Python object +// = GetResult +fn convert_raw_result( + raw_result: &mut RawResult, + call_info: &CallInfo, + vm: &VirtualMachine, +) -> Option<PyObjectRef> { + // Get result as bytes for type conversion + let (result_bytes, result_size) = match raw_result { + RawResult::Void => return None, + RawResult::Pointer(ptr) => { + let bytes = ptr.to_ne_bytes(); + (bytes.to_vec(), core::mem::size_of::<usize>()) + } + RawResult::Value(val) => { + let bytes = val.to_ne_bytes(); + (bytes.to_vec(), core::mem::size_of::<i64>()) + } + }; + + // 1. No restype → return as int + let restype = match &call_info.restype_obj { + None => { + // Default: return as int + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return Some(vm.ctx.new_int(val).into()); + } + Some(r) => r, + }; + + // 2. restype is None → return None + if restype.is(&vm.ctx.none()) { + return None; + } + + // 3. Get restype as PyType + let restype_type = match restype.clone().downcast::<PyType>() { + Ok(t) => t, + Err(_) => { + // Not a type, call it with int result + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return restype.call((val,), vm).ok(); + } + }; + + // 4. Get StgInfo + let stg_info = restype_type.stg_info_opt(); + + // No StgInfo → call restype with int + if stg_info.is_none() { + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return restype_type.as_object().call((val,), vm).ok(); + } + + let info = stg_info.unwrap(); + + // 5. Simple type with getfunc → use bytes_to_pyobject (info->getfunc) + // is_simple_instance returns TRUE for c_int, c_void_p, etc. + if super::base::is_simple_instance(&restype_type) { + return super::base::bytes_to_pyobject(&restype_type, &result_bytes, vm).ok(); + } + + // 6. Complex type → create ctypes instance (PyCData_FromBaseObj) + // This handles POINTER(T), Structure, Array, etc. + + // Special handling for POINTER(T) types - set pointer value directly + if info.flags.contains(StgInfoFlags::TYPEFLAG_ISPOINTER) + && info.proto.is_some() + && let RawResult::Pointer(ptr) = raw_result + && let Ok(instance) = restype_type.as_object().call((), vm) + { + if let Some(pointer) = instance.downcast_ref::<PyCPointer>() { + pointer.set_ptr_value(*ptr); + } + return Some(instance); + } + + // Create instance and copy result data + pycdata_from_ffi_result(&restype_type, &result_bytes, result_size, vm).ok() +} + +/// Create a ctypes instance from FFI result (PyCData_FromBaseObj equivalent) +fn pycdata_from_ffi_result( + typ: &PyTypeRef, + result_bytes: &[u8], + size: usize, + vm: &VirtualMachine, +) -> PyResult { + // Create instance + let instance = PyType::call(typ, ().into(), vm)?; + + // Copy result data into instance buffer + if let Some(cdata) = instance.downcast_ref::<PyCData>() { + let mut buffer = cdata.buffer.write(); + let copy_size = size.min(buffer.len()).min(result_bytes.len()); + if copy_size > 0 { + buffer.to_mut()[..copy_size].copy_from_slice(&result_bytes[..copy_size]); + } + } + + Ok(instance) +} + +/// Extract values from OUT buffers +fn extract_out_values( + out_buffers: Vec<(usize, PyObjectRef)>, + vm: &VirtualMachine, +) -> Vec<PyObjectRef> { + out_buffers + .into_iter() + .map(|(_, buffer)| buffer.get_attr("value", vm).unwrap_or(buffer)) + .collect() +} + +/// Build final result (main function) +fn build_result( + mut raw_result: RawResult, + call_info: &CallInfo, + out_buffers: OutBuffers, + zelf: &Py<PyCFuncPtr>, + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult { + // Check HRESULT on Windows + #[cfg(windows)] + if let RawResult::Value(val) = raw_result { + let is_hresult = call_info + .restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .is_some_and(|t| t.name().to_string() == "HRESULT"); + if is_hresult { + check_hresult(val as i32, zelf, vm)?; + } + } + + // Convert raw result to Python object + let mut result = convert_raw_result(&mut raw_result, call_info, vm); + + // Apply errcheck if set + if let Some(errcheck) = zelf.errcheck.read().as_ref() { + let args_tuple = PyTuple::new_ref(args.args.clone(), &vm.ctx); + let func_obj = zelf.as_object().to_owned(); + let result_obj = result.clone().unwrap_or_else(|| vm.ctx.none()); + result = Some(errcheck.call((result_obj, func_obj, args_tuple), vm)?); + } + + // Handle OUT parameter return values + if out_buffers.is_empty() { + return result.map(Ok).unwrap_or_else(|| Ok(vm.ctx.none())); + } + + let out_values = extract_out_values(out_buffers, vm); + Ok(match <[PyObjectRef; 1]>::try_from(out_values) { + Ok([single]) => single, + Err(v) => PyTuple::new_ref(v, &vm.ctx).into(), + }) +} + +impl Callable for PyCFuncPtr { + type Args = FuncArgs; + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { + // 1. Check for internal PYFUNCTYPE addresses + if let Some(result) = handle_internal_func(zelf.get_func_ptr(), &args, vm) { + return result; + } + + // 2. Resolve function pointer (COM or direct) + #[cfg(windows)] + let (func_ptr, is_com_method) = resolve_com_method(zelf, &args, vm)?; + #[cfg(not(windows))] + let (func_ptr, is_com_method) = (None::<CodePtr>, false); + + // 3. Extract call info (argtypes, restype) + let call_info = extract_call_info(zelf, vm)?; + + // 4. Parse paramflags + let paramflags = parse_paramflags(zelf, vm)?; + + // 5. Build call arguments + let (arguments, out_buffers) = + build_callargs(&args, &call_info, paramflags.as_ref(), is_com_method, vm)?; + + // 6. Get code pointer + let code_ptr = match func_ptr.or_else(|| zelf.get_code_ptr()) { + Some(cp) => cp, + None => { + debug_assert!(false, "NULL function pointer"); + // In release mode, this will crash + CodePtr(core::ptr::null_mut()) + } + }; + + // 7. Get flags to check for use_last_error/use_errno + let flags = PyCFuncPtr::_flags_(zelf, vm); + + // 8. Call the function (with use_last_error/use_errno handling) + #[cfg(not(windows))] + let raw_result = { + if flags & super::base::StgInfoFlags::FUNCFLAG_USE_ERRNO.bits() != 0 { + swap_errno(|| ctypes_callproc(code_ptr, &arguments, &call_info)) + } else { + ctypes_callproc(code_ptr, &arguments, &call_info) + } + }; + + #[cfg(windows)] + let raw_result = { + if flags & super::base::StgInfoFlags::FUNCFLAG_USE_LASTERROR.bits() != 0 { + save_and_restore_last_error(|| ctypes_callproc(code_ptr, &arguments, &call_info)) + } else { + ctypes_callproc(code_ptr, &arguments, &call_info) + } + }; + + // 9. Build result + build_result(raw_result, &call_info, out_buffers, zelf, &args, vm) + } +} + +// PyCFuncPtr_repr +impl Representable for PyCFuncPtr { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().name(); + // Use object id, not function pointer address + let addr = zelf.get_id(); + Ok(format!("<{} object at {:#x}>", type_name, addr)) + } +} + +// PyCData_NewGetBuffer +impl AsBuffer for PyCFuncPtr { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + // CFuncPtr types may not have StgInfo if PyCFuncPtrType metaclass is not used + // Use default values for function pointers: format="X{}", size=sizeof(pointer) + let (format, itemsize) = if let Some(stg_info) = zelf.class().stg_info_opt() { + ( + stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("X{}")), + stg_info.size, + ) + } else { + (Cow::Borrowed("X{}"), core::mem::size_of::<usize>()) + }; + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} + +#[pyclass( + flags(BASETYPE), + with(Callable, Constructor, AsNumber, Representable, AsBuffer) +)] +impl PyCFuncPtr { + // restype getter/setter + #[pygetset] + fn restype(&self) -> Option<PyObjectRef> { + self.restype.read().clone() + } + + #[pygetset(setter)] + fn set_restype(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Must be type, callable, or None + if vm.is_none(&value) { + *self.restype.write() = None; + } else if value.downcast_ref::<PyType>().is_some() || value.is_callable() { + *self.restype.write() = Some(value); + } else { + return Err(vm.new_type_error("restype must be a type, a callable, or None")); + } + Ok(()) + } + + // argtypes getter/setter + #[pygetset] + fn argtypes(&self, vm: &VirtualMachine) -> PyObjectRef { + self.argtypes + .read() + .clone() + .unwrap_or_else(|| vm.ctx.empty_tuple.clone().into()) } #[pygetset(name = "argtypes", setter)] - fn set_argtypes(&self, argtypes: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let none = vm.is_none(&argtypes); - if none { - *self.arg_types.write() = None; - Ok(()) + fn set_argtypes(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if vm.is_none(&value) { + *self.argtypes.write() = None; } else { - let tuple = argtypes.downcast::<PyTuple>().unwrap(); - *self.arg_types.write() = Some( - tuple - .iter() - .map(|obj| obj.clone().downcast::<PyType>().unwrap()) - .collect::<Vec<_>>(), - ); - Ok(()) + // Store the argtypes object directly as it is + *self.argtypes.write() = Some(value); } + Ok(()) } + // errcheck getter/setter #[pygetset] - fn __name__(&self) -> Option<String> { - self.name.read().clone() + fn errcheck(&self) -> Option<PyObjectRef> { + self.errcheck.read().clone() } #[pygetset(setter)] - fn set___name__(&self, name: String) -> PyResult<()> { - *self.name.write() = Some(name); - // TODO: update handle and stuff + fn set_errcheck(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if vm.is_none(&value) { + *self.errcheck.write() = None; + } else if value.is_callable() { + *self.errcheck.write() = Some(value); + } else { + return Err(vm.new_type_error("errcheck must be a callable or None")); + } Ok(()) } + + // _flags_ getter (read-only, from type's class attribute or StgInfo) + #[pygetset] + fn _flags_(zelf: &Py<Self>, vm: &VirtualMachine) -> u32 { + // First try to get _flags_ from type's class attribute (for dynamically created types) + // This is how CDLL sets use_errno: class _FuncPtr(_CFuncPtr): _flags_ = flags + if let Ok(flags_attr) = zelf.class().as_object().get_attr("_flags_", vm) + && let Ok(flags_int) = flags_attr.try_to_value::<u32>(vm) + { + return flags_int; + } + + // Fallback to StgInfo for native types + zelf.class() + .stg_info_opt() + .map(|stg| stg.flags.bits()) + .unwrap_or(StgInfoFlags::empty().bits()) + } +} + +impl AsNumber for PyCFuncPtr { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyCFuncPtr>().unwrap(); + Ok(zelf.get_func_ptr() != 0) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +// CThunkObject - FFI callback (thunk) implementation + +/// Userdata passed to the libffi callback. +struct ThunkUserData { + /// The Python callable to invoke + callable: PyObjectRef, + /// Argument types for conversion + arg_types: Vec<PyTypeRef>, + /// Result type for conversion (None means void) + pub res_type: Option<PyTypeRef>, + /// Function flags (FUNCFLAG_USE_ERRNO, etc.) + pub flags: u32, +} + +/// Check if ty is a subclass of a simple type (like MyInt(c_int)). +fn is_simple_subclass(ty: &Py<PyType>, vm: &VirtualMachine) -> bool { + let Ok(base) = ty.as_object().get_attr(vm.ctx.intern_str("__base__"), vm) else { + return false; + }; + base.get_attr(vm.ctx.intern_str("_type_"), vm).is_ok() +} + +/// Convert a C value to a Python object based on the type code. +fn ffi_to_python(ty: &Py<PyType>, ptr: *const c_void, vm: &VirtualMachine) -> PyObjectRef { + let type_code = ty.type_code(vm); + let raw_value: PyObjectRef = unsafe { + match type_code.as_deref() { + Some("b") => vm.ctx.new_int(*(ptr as *const i8) as i32).into(), + Some("B") => vm.ctx.new_int(*(ptr as *const u8) as i32).into(), + Some("c") => vm.ctx.new_bytes(vec![*(ptr as *const u8)]).into(), + Some("h") => vm.ctx.new_int(*(ptr as *const i16) as i32).into(), + Some("H") => vm.ctx.new_int(*(ptr as *const u16) as i32).into(), + Some("i") => vm.ctx.new_int(*(ptr as *const i32)).into(), + Some("I") => vm.ctx.new_int(*(ptr as *const u32)).into(), + Some("l") => vm.ctx.new_int(*(ptr as *const libc::c_long)).into(), + Some("L") => vm.ctx.new_int(*(ptr as *const libc::c_ulong)).into(), + Some("q") => vm.ctx.new_int(*(ptr as *const libc::c_longlong)).into(), + Some("Q") => vm.ctx.new_int(*(ptr as *const libc::c_ulonglong)).into(), + Some("f") => vm.ctx.new_float(*(ptr as *const f32) as f64).into(), + Some("d") => vm.ctx.new_float(*(ptr as *const f64)).into(), + Some("z") => { + // c_char_p: C string pointer → Python bytes + let cstr_ptr = *(ptr as *const *const libc::c_char); + if cstr_ptr.is_null() { + vm.ctx.none() + } else { + let cstr = core::ffi::CStr::from_ptr(cstr_ptr); + vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into() + } + } + Some("Z") => { + // c_wchar_p: wchar_t* → Python str + let wstr_ptr = *(ptr as *const *const libc::wchar_t); + if wstr_ptr.is_null() { + vm.ctx.none() + } else { + let mut len = 0; + while *wstr_ptr.add(len) != 0 { + len += 1; + } + let slice = core::slice::from_raw_parts(wstr_ptr, len); + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide + // Unix: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = slice.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + vm.ctx.new_str(wtf8).into() + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = slice + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + vm.ctx.new_str(s).into() + } + } + } + Some("P") => vm.ctx.new_int(*(ptr as *const usize)).into(), + Some("?") => vm.ctx.new_bool(*(ptr as *const u8) != 0).into(), + _ => return vm.ctx.none(), + } + }; + + if !is_simple_subclass(ty, vm) { + return raw_value; + } + ty.as_object() + .call((raw_value.clone(),), vm) + .unwrap_or(raw_value) +} + +/// Convert a Python object to a C value and store it at the result pointer +fn python_to_ffi(obj: PyResult, ty: &Py<PyType>, result: *mut c_void, vm: &VirtualMachine) { + let Ok(obj) = obj else { return }; + + let type_code = ty.type_code(vm); + unsafe { + match type_code.as_deref() { + Some("b") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i8) = i.as_bigint().to_i8().unwrap_or(0); + } + } + Some("B") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); + } + } + Some("c") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); + } + } + Some("h") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i16) = i.as_bigint().to_i16().unwrap_or(0); + } + } + Some("H") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u16) = i.as_bigint().to_u16().unwrap_or(0); + } + } + Some("i") => { + if let Ok(i) = obj.try_int(vm) { + let val = i.as_bigint().to_i32().unwrap_or(0); + *(result as *mut libffi::low::ffi_arg) = val as libffi::low::ffi_arg; + } + } + Some("I") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u32) = i.as_bigint().to_u32().unwrap_or(0); + } + } + Some("l") | Some("q") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i64) = i.as_bigint().to_i64().unwrap_or(0); + } + } + Some("L") | Some("Q") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u64) = i.as_bigint().to_u64().unwrap_or(0); + } + } + Some("f") => { + if let Ok(f) = obj.try_float(vm) { + *(result as *mut f32) = f.to_f64() as f32; + } + } + Some("d") => { + if let Ok(f) = obj.try_float(vm) { + *(result as *mut f64) = f.to_f64(); + } + } + Some("P") | Some("z") | Some("Z") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut usize) = i.as_bigint().to_usize().unwrap_or(0); + } + } + Some("?") => { + if let Ok(b) = obj.is_true(vm) { + *(result as *mut u8) = u8::from(b); + } + } + _ => {} + } + } +} + +/// The callback function that libffi calls when the closure is invoked. +unsafe extern "C" fn thunk_callback( + _cif: &low::ffi_cif, + result: &mut c_void, + args: *const *const c_void, + userdata: &ThunkUserData, +) { + with_current_vm(|vm| { + // Swap errno before call if FUNCFLAG_USE_ERRNO is set + let use_errno = userdata.flags & StgInfoFlags::FUNCFLAG_USE_ERRNO.bits() != 0; + let saved_errno = if use_errno { + let current = rustpython_common::os::get_errno(); + // TODO: swap with ctypes stored errno (thread-local) + Some(current) + } else { + None + }; + + let py_args: Vec<PyObjectRef> = userdata + .arg_types + .iter() + .enumerate() + .map(|(i, ty)| { + let arg_ptr = unsafe { *args.add(i) }; + ffi_to_python(ty, arg_ptr, vm) + }) + .collect(); + + let py_result = userdata.callable.call(py_args, vm); + + // Swap errno back after call + if use_errno { + let _current = rustpython_common::os::get_errno(); + // TODO: store current errno to ctypes storage + if let Some(saved) = saved_errno { + rustpython_common::os::set_errno(saved); + } + } + + // Call unraisable hook if exception occurred + if let Err(exc) = &py_result { + let repr = userdata + .callable + .repr(vm) + .map(|s| s.to_string()) + .unwrap_or_else(|_| "<unknown>".to_string()); + let msg = format!( + "Exception ignored while calling ctypes callback function {}", + repr + ); + vm.run_unraisable(exc.clone(), Some(msg), vm.ctx.none()); + } + + if let Some(ref res_type) = userdata.res_type { + python_to_ffi(py_result, res_type, result as *mut c_void, vm); + } + }); +} + +/// Holds the closure and userdata together to ensure proper lifetime. +struct ThunkData { + #[allow(dead_code)] + closure: Closure<'static>, + userdata_ptr: *mut ThunkUserData, +} + +impl Drop for ThunkData { + fn drop(&mut self) { + unsafe { + drop(Box::from_raw(self.userdata_ptr)); + } + } +} + +/// CThunkObject wraps a Python callable to make it callable from C code. +#[pyclass(name = "CThunkObject", module = "_ctypes")] +#[derive(PyPayload)] +pub(super) struct PyCThunk { + callable: PyObjectRef, + #[allow(dead_code)] + thunk_data: PyRwLock<Option<ThunkData>>, + code_ptr: CodePtr, +} + +impl Debug for PyCThunk { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCThunk") + .field("callable", &self.callable) + .finish() + } +} + +impl PyCThunk { + pub fn new( + callable: PyObjectRef, + arg_types: Option<PyObjectRef>, + res_type: Option<PyObjectRef>, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let arg_type_vec: Vec<PyTypeRef> = match arg_types { + Some(args) if !vm.is_none(&args) => args + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .map(|item| { + item.downcast::<PyType>() + .map_err(|_| vm.new_type_error("_argtypes_ must be a sequence of types")) + }) + .collect::<PyResult<Vec<_>>>()?, + _ => Vec::new(), + }; + + let res_type_ref: Option<PyTypeRef> = match res_type { + Some(ref rt) if !vm.is_none(rt) => Some( + rt.clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("restype must be a ctypes type"))?, + ), + _ => None, + }; + + let ffi_arg_types: Vec<Type> = arg_type_vec + .iter() + .map(|ty| { + ty.type_code(vm) + .and_then(|code| get_ffi_type(&code)) + .unwrap_or(Type::pointer()) + }) + .collect(); + + let ffi_res_type = res_type_ref + .as_ref() + .and_then(|ty| ty.type_code(vm)) + .and_then(|code| get_ffi_type(&code)) + .unwrap_or(Type::void()); + + let cif = Cif::new(ffi_arg_types, ffi_res_type); + + let userdata = Box::new(ThunkUserData { + callable: callable.clone(), + arg_types: arg_type_vec, + res_type: res_type_ref, + flags, + }); + let userdata_ptr = Box::into_raw(userdata); + let userdata_ref: &'static ThunkUserData = unsafe { &*userdata_ptr }; + + let closure = Closure::new(cif, thunk_callback, userdata_ref); + let code_ptr = CodePtr(*closure.code_ptr() as *mut _); + + let thunk_data = ThunkData { + closure, + userdata_ptr, + }; + + Ok(Self { + callable, + thunk_data: PyRwLock::new(Some(thunk_data)), + code_ptr, + }) + } + + pub fn code_ptr(&self) -> CodePtr { + self.code_ptr + } +} + +unsafe impl Send for PyCThunk {} +unsafe impl Sync for PyCThunk {} + +#[pyclass] +impl PyCThunk { + #[pygetset] + fn callable(&self) -> PyObjectRef { + self.callable.clone() + } } diff --git a/crates/vm/src/stdlib/ctypes/library.rs b/crates/vm/src/stdlib/ctypes/library.rs index e918470b6c8..35ccb433845 100644 --- a/crates/vm/src/stdlib/ctypes/library.rs +++ b/crates/vm/src/stdlib/ctypes/library.rs @@ -1,10 +1,12 @@ use crate::VirtualMachine; +use alloc::fmt; use libloading::Library; use rustpython_common::lock::{PyMutex, PyRwLock}; use std::collections::HashMap; -use std::ffi::c_void; -use std::fmt; -use std::ptr::null; +use std::ffi::OsStr; + +#[cfg(unix)] +use libloading::os::unix::Library as UnixLibrary; pub struct SharedLibrary { pub(crate) lib: PyMutex<Option<Library>>, @@ -17,73 +19,122 @@ impl fmt::Debug for SharedLibrary { } impl SharedLibrary { - pub fn new(name: &str) -> Result<SharedLibrary, libloading::Error> { + #[cfg(windows)] + pub fn new(name: impl AsRef<OsStr>) -> Result<SharedLibrary, libloading::Error> { + Ok(SharedLibrary { + lib: PyMutex::new(unsafe { Some(Library::new(name.as_ref())?) }), + }) + } + + #[cfg(unix)] + pub fn new_with_mode( + name: impl AsRef<OsStr>, + mode: i32, + ) -> Result<SharedLibrary, libloading::Error> { Ok(SharedLibrary { - lib: PyMutex::new(unsafe { Some(Library::new(name)?) }), + lib: PyMutex::new(Some(unsafe { + UnixLibrary::open(Some(name.as_ref()), mode)?.into() + })), }) } + /// Create a SharedLibrary from a raw dlopen handle (for pythonapi / dlopen(NULL)) + #[cfg(unix)] + pub fn from_raw_handle(handle: *mut libc::c_void) -> SharedLibrary { + SharedLibrary { + lib: PyMutex::new(Some(unsafe { UnixLibrary::from_raw(handle).into() })), + } + } + + /// Get the underlying OS handle (HMODULE on Windows, dlopen handle on Unix) pub fn get_pointer(&self) -> usize { let lib_lock = self.lib.lock(); if let Some(l) = &*lib_lock { - l as *const Library as usize + // libloading::Library internally stores the OS handle directly + // On Windows: HMODULE (*mut c_void) + // On Unix: *mut c_void from dlopen + // We use transmute_copy to read the handle without consuming the Library + unsafe { core::mem::transmute_copy::<Library, usize>(l) } } else { - null::<c_void>() as usize + 0 } } - pub fn is_closed(&self) -> bool { + fn is_closed(&self) -> bool { let lib_lock = self.lib.lock(); lib_lock.is_none() } - - pub fn close(&self) { - *self.lib.lock() = None; - } -} - -impl Drop for SharedLibrary { - fn drop(&mut self) { - self.close(); - } } -pub struct ExternalLibs { +pub(super) struct ExternalLibs { libraries: HashMap<usize, SharedLibrary>, } impl ExternalLibs { - pub fn new() -> Self { + fn new() -> Self { Self { libraries: HashMap::new(), } } - #[allow(dead_code)] pub fn get_lib(&self, key: usize) -> Option<&SharedLibrary> { self.libraries.get(&key) } + #[cfg(windows)] pub fn get_or_insert_lib( &mut self, - library_path: &str, + library_path: impl AsRef<OsStr>, _vm: &VirtualMachine, ) -> Result<(usize, &SharedLibrary), libloading::Error> { let new_lib = SharedLibrary::new(library_path)?; let key = new_lib.get_pointer(); - match self.libraries.get(&key) { - Some(l) => { - if l.is_closed() { - self.libraries.insert(key, new_lib); - } - } - _ => { - self.libraries.insert(key, new_lib); - } - }; - - Ok((key, self.libraries.get(&key).unwrap())) + // Check if library already exists and is not closed + let should_use_cached = self.libraries.get(&key).is_some_and(|l| !l.is_closed()); + + if should_use_cached { + // new_lib will be dropped, calling FreeLibrary (decrements refcount) + // But library stays loaded because cached version maintains refcount + drop(new_lib); + return Ok((key, self.libraries.get(&key).expect("just checked"))); + } + + self.libraries.insert(key, new_lib); + Ok((key, self.libraries.get(&key).expect("just inserted"))) + } + + #[cfg(unix)] + pub fn get_or_insert_lib_with_mode( + &mut self, + library_path: impl AsRef<OsStr>, + mode: i32, + _vm: &VirtualMachine, + ) -> Result<(usize, &SharedLibrary), libloading::Error> { + let new_lib = SharedLibrary::new_with_mode(library_path, mode)?; + let key = new_lib.get_pointer(); + + // Check if library already exists and is not closed + let should_use_cached = self.libraries.get(&key).is_some_and(|l| !l.is_closed()); + + if should_use_cached { + // new_lib will be dropped, calling dlclose (decrements refcount) + // But library stays loaded because cached version maintains refcount + drop(new_lib); + return Ok((key, self.libraries.get(&key).expect("just checked"))); + } + + self.libraries.insert(key, new_lib); + Ok((key, self.libraries.get(&key).expect("just inserted"))) + } + + /// Insert a raw dlopen handle into the cache (for pythonapi / dlopen(NULL)) + #[cfg(unix)] + pub fn insert_raw_handle(&mut self, handle: *mut libc::c_void) -> usize { + let shared_lib = SharedLibrary::from_raw_handle(handle); + let key = handle as usize; + self.libraries.insert(key, shared_lib); + key } pub fn drop_lib(&mut self, key: usize) { @@ -91,10 +142,9 @@ impl ExternalLibs { } } -rustpython_common::static_cell! { - static LIBCACHE: PyRwLock<ExternalLibs>; -} - -pub fn libcache() -> &'static PyRwLock<ExternalLibs> { +pub(super) fn libcache() -> &'static PyRwLock<ExternalLibs> { + rustpython_common::static_cell! { + static LIBCACHE: PyRwLock<ExternalLibs>; + } LIBCACHE.get_or_init(|| PyRwLock::new(ExternalLibs::new())) } diff --git a/crates/vm/src/stdlib/ctypes/pointer.rs b/crates/vm/src/stdlib/ctypes/pointer.rs index 735034e7936..6000bb57a37 100644 --- a/crates/vm/src/stdlib/ctypes/pointer.rs +++ b/crates/vm/src/stdlib/ctypes/pointer.rs @@ -1,37 +1,222 @@ +use super::base::CDATA_BUFFER_METHODS; +use super::{PyCArray, PyCData, PyCSimple, PyCStructure, StgInfo, StgInfoFlags}; +use crate::atomic_func; +use crate::protocol::{BufferDescriptor, PyBuffer, PyMappingMethods, PyNumberMethods}; +use crate::types::{AsBuffer, AsMapping, AsNumber, Constructor, Initializer}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyInt, PyList, PySlice, PyStr, PyType, PyTypeRef}, + class::StaticType, + function::{FuncArgs, OptionalArg}, +}; +use alloc::borrow::Cow; use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; - -use crate::builtins::{PyType, PyTypeRef}; -use crate::function::FuncArgs; -use crate::protocol::PyNumberMethods; -use crate::stdlib::ctypes::{CDataObject, PyCData}; -use crate::types::{AsNumber, Constructor}; -use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; #[pyclass(name = "PyCPointerType", base = PyType, module = "_ctypes")] #[derive(Debug)] #[repr(transparent)] -pub struct PyCPointerType(PyType); +pub(super) struct PyCPointerType(PyType); + +impl Initializer for PyCPointerType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + if new_type.is_initialized() { + return Ok(()); + } + + // Get the _type_ attribute (element type) + let proto = new_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|obj| obj.downcast::<PyType>().ok()); + + // Validate that _type_ has storage info (is a ctypes type) + if let Some(ref proto_type) = proto + && proto_type.stg_info_opt().is_none() + { + return Err(vm.new_type_error(format!("{} must have storage info", proto_type.name()))); + } + + // Initialize StgInfo for pointer type + let pointer_size = core::mem::size_of::<usize>(); + let mut stg_info = StgInfo::new(pointer_size, pointer_size); + stg_info.proto = proto; + stg_info.paramfunc = super::base::ParamFunc::Pointer; + stg_info.length = 1; + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + + // Set format string: "&<element_format>" or "&(shape)<element_format>" for arrays + if let Some(ref proto) = stg_info.proto + && let Some(item_info) = proto.stg_info_opt() + { + let current_format = item_info.format.as_deref().unwrap_or("B"); + // Include shape for array types in the pointer format + let shape_str = if !item_info.shape.is_empty() { + let dims: Vec<String> = item_info.shape.iter().map(|d| d.to_string()).collect(); + format!("({})", dims.join(",")) + } else { + String::new() + }; + stg_info.format = Some(format!("&{}{}", shape_str, current_format)); + } -#[pyclass(flags(IMMUTABLETYPE), with(AsNumber))] + let _ = new_type.init_type_data(stg_info); + + // Cache: set target_type.__pointer_type__ = self (via StgInfo, not as inheritable attr) + if let Ok(type_attr) = new_type.as_object().get_attr("_type_", vm) + && let Ok(target_type) = type_attr.downcast::<PyType>() + && let Some(mut target_info) = target_type.get_type_data_mut::<StgInfo>() + { + let zelf_obj: PyObjectRef = zelf.into(); + target_info.pointer_type = Some(zelf_obj); + } + + Ok(()) + } +} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(AsNumber, Initializer))] impl PyCPointerType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the pointer type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. None is allowed for pointer types + if vm.is_none(&value) { + return Ok(value); + } + + // 1.5 CArgObject (from byref()) - check if underlying obj is instance of _type_ + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() + && let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(type_ref) = type_attr.downcast::<PyType>() + && carg.obj.is_instance(type_ref.as_object(), vm)? + { + return Ok(value); + } + + // 2. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 3. If value is an instance of _type_ (the pointed-to type), wrap with byref + if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(type_ref) = type_attr.downcast::<PyType>() + && value.is_instance(type_ref.as_object(), vm)? + { + // Return byref(value) + return super::_ctypes::byref(value, crate::function::OptionalArg::Missing, vm); + } + + // 4. Array/Pointer instances with compatible proto + // "Array instances are also pointers when the item types are the same." + let is_pointer_or_array = value.downcast_ref::<PyCPointer>().is_some() + || value.downcast_ref::<super::array::PyCArray>().is_some(); + + if is_pointer_or_array { + let is_compatible = { + if let Some(value_stginfo) = value.class().stg_info_opt() + && let Some(ref value_proto) = value_stginfo.proto + && let Some(cls_stginfo) = cls.stg_info_opt() + && let Some(ref cls_proto) = cls_stginfo.proto + { + // Check if value's proto is a subclass of target's proto + value_proto.fast_issubclass(cls_proto) + } else { + false + } + }; + if is_compatible { + return Ok(value); + } + } + + // 5. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCPointerType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } + fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { - use super::array::create_array_type_with_stg_info; + use super::array::array_type_from_ctype; + if n < 0 { return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); } - // Pointer size - let element_size = std::mem::size_of::<usize>(); - let total_size = element_size * (n as usize); - let stg_info = super::util::StgInfo::new_array( - total_size, - element_size, - n as usize, - cls.as_object().to_owned(), - element_size, - ); - create_array_type_with_stg_info(stg_info, vm) + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) + } + + // PyCPointerType_set_type: Complete an incomplete pointer type + #[pymethod] + fn set_type(zelf: PyTypeRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::AsObject; + + // 1. Validate that typ is a type + let typ_type = typ + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_type_ must be a type"))?; + + // 2. Validate that typ has storage info + if typ_type.stg_info_opt().is_none() { + return Err(vm.new_type_error("_type_ must have storage info")); + } + + // 3. Update StgInfo.proto and format using mutable access + if let Some(mut stg_info) = zelf.get_type_data_mut::<StgInfo>() { + stg_info.proto = Some(typ_type.clone()); + + // Update format string: "&<element_format>" or "&(shape)<element_format>" for arrays + let item_info = typ_type.stg_info_opt().expect("proto has StgInfo"); + let current_format = item_info.format.as_deref().unwrap_or("B"); + // Include shape for array types in the pointer format + let shape_str = if !item_info.shape.is_empty() { + let dims: Vec<String> = item_info.shape.iter().map(|d| d.to_string()).collect(); + format!("({})", dims.join(",")) + } else { + String::new() + }; + stg_info.format = Some(format!("&{}{}", shape_str, current_format)); + } + + // 4. Set _type_ attribute on the pointer type + zelf.as_object().set_attr("_type_", typ_type.clone(), vm)?; + + // 5. Cache: set target_type.__pointer_type__ = self (via StgInfo) + if let Some(mut target_info) = typ_type.get_type_data_mut::<StgInfo>() { + target_info.pointer_type = Some(zelf.into()); + } + + Ok(()) } } @@ -41,12 +226,12 @@ impl AsNumber for PyCPointerType { multiply: Some(|a, b, vm| { let cls = a .downcast_ref::<PyType>() - .ok_or_else(|| vm.new_type_error("expected type".to_owned()))?; + .ok_or_else(|| vm.new_type_error("expected type"))?; let n = b .try_index(vm)? .as_bigint() .to_isize() - .ok_or_else(|| vm.new_overflow_error("array size too large".to_owned()))?; + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; PyCPointerType::__mul__(cls.to_owned(), n, vm) }), ..PyNumberMethods::NOT_IMPLEMENTED @@ -55,6 +240,8 @@ impl AsNumber for PyCPointerType { } } +/// PyCPointer - Pointer instance +/// `contents` is a computed property, not a stored field. #[pyclass( name = "_Pointer", base = PyCData, @@ -62,26 +249,27 @@ impl AsNumber for PyCPointerType { module = "_ctypes" )] #[derive(Debug)] -pub struct PyCPointer { - _base: PyCData, - contents: PyRwLock<PyObjectRef>, -} +#[repr(transparent)] +pub struct PyCPointer(pub PyCData); impl Constructor for PyCPointer { - type Args = (crate::function::OptionalArg<PyObjectRef>,); - - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: Self::Args = args.bind(vm)?; - // Get the initial contents value if provided - let initial_contents = args.0.into_option().unwrap_or_else(|| vm.ctx.none()); + type Args = FuncArgs; - // Create a new PyCPointer instance with the provided value - PyCPointer { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - contents: PyRwLock::new(initial_contents), + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Pointer_new: Check if _type_ is defined + let has_type = cls.stg_info_opt().is_some_and(|info| info.proto.is_some()); + if !has_type { + return Err(vm.new_type_error("Cannot create instance: has no _type_")); } - .into_ref_with_type(vm, cls) - .map(Into::into) + + // Create a new PyCPointer instance with NULL pointer (all zeros) + // Initial contents is set via __init__ if provided + let cdata = PyCData::from_bytes(vec![0u8; core::mem::size_of::<usize>()], None); + // pointer instance has b_length set to 2 (for index 0 and 1) + cdata.length.store(2); + PyCPointer(cdata) + .into_ref_with_type(vm, cls) + .map(Into::into) } fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { @@ -89,186 +277,592 @@ impl Constructor for PyCPointer { } } -#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor))] +impl Initializer for PyCPointer { + type Args = (OptionalArg<PyObjectRef>,); + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let (value,) = args; + if let OptionalArg::Present(val) = value + && !vm.is_none(&val) + { + Self::set_contents(&zelf, val, vm)?; + } + Ok(()) + } +} + +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsNumber, AsBuffer, AsMapping) +)] impl PyCPointer { - // TODO: not correct + /// Get the pointer value stored in buffer as usize + pub fn get_ptr_value(&self) -> usize { + let buffer = self.0.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } + + /// Set the pointer value in buffer + pub fn set_ptr_value(&self, value: usize) { + let mut buffer = self.0.buffer.write(); + let bytes = value.to_ne_bytes(); + if buffer.len() >= bytes.len() { + buffer.to_mut()[..bytes.len()].copy_from_slice(&bytes); + } + } + + /// contents getter - reads address from b_ptr and creates an instance of the pointed-to type #[pygetset] - fn contents(&self) -> PyResult<PyObjectRef> { - let contents = self.contents.read().clone(); - Ok(contents) + fn contents(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Pointer_get_contents + let ptr_val = zelf.get_ptr_value(); + if ptr_val == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + + // Get element type from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); + + // Create instance that references the memory directly + // PyCData.into_ref_with_type works for all ctypes (simple, structure, union, array, pointer) + let cdata = unsafe { super::base::PyCData::at_address(ptr_val as *const u8, element_size) }; + cdata + .into_ref_with_type(vm, proto_type.to_owned()) + .map(Into::into) } + + /// contents setter - stores address in b_ptr and keeps reference + /// Pointer_set_contents #[pygetset(setter)] - fn set_contents(&self, contents: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { - // Validate that the contents is a CData instance if we have a _type_ - // For now, just store it - *self.contents.write() = contents; - Ok(()) - } + fn set_contents(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Get stginfo and proto for type validation + let stg_info = zelf.class().stg_info(vm)?; + let proto = stg_info.proto(); - #[pymethod] - fn __init__( - &self, - value: crate::function::OptionalArg<PyObjectRef>, - _vm: &VirtualMachine, - ) -> PyResult<()> { - // Pointer can be initialized with 0 or 1 argument - // If 1 argument is provided, it should be a CData instance - if let crate::function::OptionalArg::Present(val) = value { - *self.contents.write() = val; + // Check if value is CData, or isinstance(value, proto) + let cdata = if let Some(c) = value.downcast_ref::<PyCData>() { + c + } else if value.is_instance(proto.as_object(), vm)? { + value + .downcast_ref::<PyCData>() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))? + } else { + return Err(vm.new_type_error(format!( + "expected {} instead of {}", + proto.name(), + value.class().name() + ))); + }; + + // Set pointer value + { + let buffer = cdata.buffer.read(); + let addr = buffer.as_ptr() as usize; + drop(buffer); + zelf.set_ptr_value(addr); + } + + // KeepRef: store the object directly with index 1 + zelf.0.keep_ref(1, value.clone(), vm)?; + + // KeepRef: store GetKeepedObjects(dst) at index 0 + if let Some(kept) = cdata.objects.read().clone() { + zelf.0.keep_ref(0, kept, vm)?; } Ok(()) } - #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { - if address == 0 { - return Err(vm.new_value_error("NULL pointer access".to_owned())); + // Pointer_subscript + fn __getitem__(zelf: &Py<Self>, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + // Note: Pointer does NOT adjust negative indices (no length) + Self::getitem_by_index(zelf, i, vm) } - // Pointer just stores the address value - Ok(PyCPointer { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - contents: PyRwLock::new(vm.ctx.new_int(address).into()), + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::getitem_by_slice(zelf, slice, vm) + } else { + Err(vm.new_type_error("Pointer indices must be integer")) } - .into_ref_with_type(vm, cls)? - .into()) } - #[pyclassmethod] - fn from_buffer( - cls: PyTypeRef, - source: PyObjectRef, - offset: crate::function::OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult { - use crate::TryFromObject; - use crate::protocol::PyBuffer; + // Pointer_item + fn getitem_by_index(zelf: &Py<Self>, index: isize, vm: &VirtualMachine) -> PyResult { + // if (*(void **)self->b_ptr == NULL) { PyErr_SetString(...); } + let ptr_value = zelf.get_ptr_value(); + if ptr_value == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + + // Get element type and size from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); + // offset = index * iteminfo->size + let offset = index * element_size as isize; + let addr = (ptr_value as isize + offset) as usize; + + // Check if it's a simple type (has _type_ attribute) + if let Ok(type_attr) = proto_type.as_object().get_attr("_type_", vm) + && let Ok(type_str) = type_attr.str(vm) + { + let type_code = type_str.to_string(); + return Self::read_value_at_address(addr, element_size, Some(&type_code), vm); } - let offset = offset as usize; - let size = std::mem::size_of::<usize>(); - let buffer = PyBuffer::try_from_object(vm, source.clone())?; + // Complex type: create instance that references the memory directly (not a copy) + // This allows p[i].val = x to modify the original memory + // PyCData.into_ref_with_type works for all ctypes (array, structure, union, pointer) + let cdata = unsafe { super::base::PyCData::at_address(addr as *const u8, element_size) }; + cdata + .into_ref_with_type(vm, proto_type.to_owned()) + .map(Into::into) + } + + // Pointer_subscript slice handling (manual parsing, not PySlice_Unpack) + fn getitem_by_slice(zelf: &Py<Self>, slice: &PySlice, vm: &VirtualMachine) -> PyResult { + // Since pointers have no length, we have to dissect the slice ourselves + + // step: defaults to 1, step == 0 is error + let step: isize = if let Some(ref step_obj) = slice.step + && !vm.is_none(step_obj) + { + let s = step_obj + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice step too large"))?; + if s == 0 { + return Err(vm.new_value_error("slice step cannot be zero")); + } + s + } else { + 1 + }; + + // start: defaults to 0, but required if step < 0 + let start: isize = if let Some(ref start_obj) = slice.start + && !vm.is_none(start_obj) + { + start_obj + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice start too large"))? + } else { + if step < 0 { + return Err(vm.new_value_error("slice start is required for step < 0")); + } + 0 + }; - if buffer.desc.readonly { - return Err(vm.new_type_error("underlying buffer is not writable".to_owned())); + // stop: ALWAYS required for pointers + if vm.is_none(&slice.stop) { + return Err(vm.new_value_error("slice stop is required")); } + let stop: isize = slice + .stop + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice stop too large"))?; - let buffer_len = buffer.desc.len; - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + // calculate length + let len: usize = if (step > 0 && start > stop) || (step < 0 && start < stop) { + 0 + } else if step > 0 { + ((stop - start - 1) / step + 1) as usize + } else { + ((stop - start + 1) / step + 1) as usize + }; + + // Get element info + let stg_info = zelf.class().stg_info(vm)?; + let element_size = if let Some(ref proto_type) = stg_info.proto { + proto_type.stg_info_opt().expect("proto has StgInfo").size + } else { + core::mem::size_of::<usize>() + }; + let type_code = stg_info + .proto + .as_ref() + .and_then(|p| p.as_object().get_attr("_type_", vm).ok()) + .and_then(|t| t.str(vm).ok()) + .map(|s| s.to_string()); + + let ptr_value = zelf.get_ptr_value(); + + // c_char → bytes + if type_code.as_deref() == Some("c") { + if len == 0 { + return Ok(vm.ctx.new_bytes(vec![]).into()); + } + let mut result = Vec::with_capacity(len); + if step == 1 { + // Optimized contiguous copy + let start_addr = (ptr_value as isize + start * element_size as isize) as *const u8; + unsafe { + result.extend_from_slice(core::slice::from_raw_parts(start_addr, len)); + } + } else { + let mut cur = start; + for _ in 0..len { + let addr = (ptr_value as isize + cur * element_size as isize) as *const u8; + unsafe { + result.push(*addr); + } + cur += step; + } + } + return Ok(vm.ctx.new_bytes(result).into()); } - // Read pointer value from buffer - let bytes = buffer.obj_bytes(); - let ptr_bytes = &bytes[offset..offset + size]; - let ptr_val = usize::from_ne_bytes(ptr_bytes.try_into().expect("size is checked above")); + // c_wchar → str + if type_code.as_deref() == Some("u") { + if len == 0 { + return Ok(vm.ctx.new_str("").into()); + } + let mut result = String::with_capacity(len); + let wchar_size = core::mem::size_of::<libc::wchar_t>(); + let mut cur = start; + for _ in 0..len { + let addr = (ptr_value as isize + cur * wchar_size as isize) as *const libc::wchar_t; + unsafe { + #[allow( + clippy::unnecessary_cast, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + if let Some(c) = char::from_u32(*addr as u32) { + result.push(c); + } + } + cur += step; + } + return Ok(vm.ctx.new_str(result).into()); + } - Ok(PyCPointer { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - contents: PyRwLock::new(vm.ctx.new_int(ptr_val).into()), + // other types → list with Pointer_item for each + let mut items = Vec::with_capacity(len); + let mut cur = start; + for _ in 0..len { + items.push(Self::getitem_by_index(zelf, cur, vm)?); + cur += step; } - .into_ref_with_type(vm, cls)? - .into()) + Ok(PyList::from(items).into_ref(&vm.ctx).into()) } - #[pyclassmethod] - fn from_buffer_copy( - cls: PyTypeRef, - source: crate::function::ArgBytesLike, - offset: crate::function::OptionalArg<isize>, + // Pointer_ass_item + fn __setitem__( + zelf: &Py<Self>, + item: PyObjectRef, + value: PyObjectRef, vm: &VirtualMachine, - ) -> PyResult { - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); + ) -> PyResult<()> { + // Pointer does not support item deletion (value always provided) + // only integer indices supported for setitem + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + Self::setitem_by_index(zelf, i, value, vm) + } else { + Err(vm.new_type_error("Pointer indices must be integer")) } - let offset = offset as usize; - let size = std::mem::size_of::<usize>(); - - let source_bytes = source.borrow_buf(); - let buffer_len = source_bytes.len(); + } - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + fn setitem_by_index( + zelf: &Py<Self>, + index: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ptr_value = zelf.get_ptr_value(); + if ptr_value == 0 { + return Err(vm.new_value_error("NULL pointer access")); } - // Read pointer value from buffer - let ptr_bytes = &source_bytes[offset..offset + size]; - let ptr_val = usize::from_ne_bytes(ptr_bytes.try_into().expect("size is checked above")); + // Get element type, size and type_code from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + + // Get type code from proto's _type_ attribute + let type_code: Option<String> = proto_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); - Ok(PyCPointer { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - contents: PyRwLock::new(vm.ctx.new_int(ptr_val).into()), + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); + + // Calculate address + let offset = index * element_size as isize; + let addr = (ptr_value as isize + offset) as usize; + + // Write value at address + // Handle Structure/Array types by copying their buffer + if let Some(cdata) = value.downcast_ref::<super::PyCData>() + && (cdata.fast_isinstance(PyCStructure::static_type()) + || cdata.fast_isinstance(PyCArray::static_type()) + || cdata.fast_isinstance(PyCSimple::static_type())) + { + let src_buffer = cdata.buffer.read(); + let copy_len = src_buffer.len().min(element_size); + unsafe { + let dest_ptr = addr as *mut u8; + core::ptr::copy_nonoverlapping(src_buffer.as_ptr(), dest_ptr, copy_len); + } + } else { + // Handle z/Z specially to store converted value + if type_code.as_deref() == Some("z") + && let Some(bytes) = value.downcast_ref::<PyBytes>() + { + let (kept_alive, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); + unsafe { + *(addr as *mut usize) = ptr_val; + } + zelf.0.keep_alive(index as usize, kept_alive); + return zelf.0.keep_ref(index as usize, value.clone(), vm); + } else if type_code.as_deref() == Some("Z") + && let Some(s) = value.downcast_ref::<PyStr>() + { + let (holder, ptr_val) = super::base::str_to_wchar_bytes(s.as_str(), vm); + unsafe { + *(addr as *mut usize) = ptr_val; + } + return zelf.0.keep_ref(index as usize, holder, vm); + } else { + Self::write_value_at_address(addr, element_size, &value, type_code.as_deref(), vm)?; + } } - .into_ref_with_type(vm, cls)? - .into()) + + // KeepRef: store reference to keep value alive using actual index + zelf.0.keep_ref(index as usize, value, vm) } - #[pyclassmethod] - fn in_dll( - cls: PyTypeRef, - dll: PyObjectRef, - name: crate::builtins::PyStrRef, + /// Read a value from memory address + fn read_value_at_address( + addr: usize, + size: usize, + type_code: Option<&str>, vm: &VirtualMachine, ) -> PyResult { - use libloading::Symbol; + unsafe { + let ptr = addr as *const u8; + match type_code { + // Single-byte types don't need read_unaligned + Some("c") => Ok(vm.ctx.new_bytes(vec![*ptr]).into()), + Some("b") => Ok(vm.ctx.new_int(*ptr as i8 as i32).into()), + Some("B") => Ok(vm.ctx.new_int(*ptr as i32).into()), + // Multi-byte types need read_unaligned for safety on strict-alignment architectures + Some("h") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i16) as i32) + .into()), + Some("H") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u16) as i32) + .into()), + Some("i") | Some("l") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i32)) + .into()), + Some("I") | Some("L") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u32)) + .into()), + Some("q") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i64)) + .into()), + Some("Q") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u64)) + .into()), + Some("f") => Ok(vm + .ctx + .new_float(core::ptr::read_unaligned(ptr as *const f32) as f64) + .into()), + Some("d") | Some("g") => Ok(vm + .ctx + .new_float(core::ptr::read_unaligned(ptr as *const f64)) + .into()), + Some("P") | Some("z") | Some("Z") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const usize)) + .into()), + _ => { + // Default: read as bytes + let bytes = core::slice::from_raw_parts(ptr, size).to_vec(); + Ok(vm.ctx.new_bytes(bytes).into()) + } + } + } + } - // Get the library handle from dll object - let handle = if let Ok(int_handle) = dll.try_int(vm) { - // dll is an integer handle - int_handle - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - } else { - // dll is a CDLL/PyDLL/WinDLL object with _handle attribute - dll.get_attr("_handle", vm)? - .try_int(vm)? - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("Invalid library handle".to_owned()))? - }; + /// Write a value to memory address + fn write_value_at_address( + addr: usize, + size: usize, + value: &PyObject, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult<()> { + unsafe { + let ptr = addr as *mut u8; - // Get the library from cache - let library_cache = crate::stdlib::ctypes::library::libcache().read(); - let library = library_cache - .get_lib(handle) - .ok_or_else(|| vm.new_attribute_error("Library not found".to_owned()))?; + // Handle c_char_p (z) and c_wchar_p (Z) - store pointer address + // Note: PyBytes/PyStr cases are handled by caller (setitem_by_index) + match type_code { + Some("z") | Some("Z") => { + let ptr_val = if vm.is_none(value) { + 0usize + } else if let Ok(int_val) = value.try_index(vm) { + int_val.as_bigint().to_usize().unwrap_or(0) + } else { + return Err(vm.new_type_error( + "bytes/string or integer address expected".to_owned(), + )); + }; + core::ptr::write_unaligned(ptr as *mut usize, ptr_val); + return Ok(()); + } + _ => {} + } - // Get symbol address from library - let symbol_name = format!("{}\0", name.as_str()); - let inner_lib = library.lib.lock(); + // Try to get value as integer + // Use write_unaligned for safety on strict-alignment architectures + if let Ok(int_val) = value.try_int(vm) { + let i = int_val.as_bigint(); + match size { + 1 => { + *ptr = i.to_u8().expect("int too large"); + } + 2 => { + core::ptr::write_unaligned( + ptr as *mut i16, + i.to_i16().expect("int too large"), + ); + } + 4 => { + core::ptr::write_unaligned( + ptr as *mut i32, + i.to_i32().expect("int too large"), + ); + } + 8 => { + core::ptr::write_unaligned( + ptr as *mut i64, + i.to_i64().expect("int too large"), + ); + } + _ => { + let bytes = i.to_signed_bytes_le(); + let copy_len = bytes.len().min(size); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + } + } + return Ok(()); + } - let symbol_address = if let Some(lib) = &*inner_lib { - unsafe { - // Try to get the symbol from the library - let symbol: Symbol<'_, *mut u8> = lib.get(symbol_name.as_bytes()).map_err(|e| { - vm.new_attribute_error(format!("{}: symbol '{}' not found", e, name.as_str())) - })?; - *symbol as usize + // Try to get value as float + if let Ok(float_val) = value.try_float(vm) { + let f = float_val.to_f64(); + match size { + 4 => { + core::ptr::write_unaligned(ptr as *mut f32, f as f32); + } + 8 => { + core::ptr::write_unaligned(ptr as *mut f64, f); + } + _ => {} + } + return Ok(()); + } + + // Try bytes + if let Ok(bytes) = value.try_bytes_like(vm, |b| b.to_vec()) { + let copy_len = bytes.len().min(size); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + return Ok(()); } - } else { - return Err(vm.new_attribute_error("Library is closed".to_owned())); - }; - // For pointer types, we return a pointer to the symbol address - Ok(PyCPointer { - _base: PyCData::new(CDataObject::from_bytes(vec![], None)), - contents: PyRwLock::new(vm.ctx.new_int(symbol_address).into()), + Err(vm.new_type_error(format!( + "cannot convert {} to ctypes data", + value.class().name() + ))) } - .into_ref_with_type(vm, cls)? - .into()) + } +} + +impl AsNumber for PyCPointer { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyCPointer>().unwrap(); + Ok(zelf.get_ptr_value() != 0) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl AsMapping for PyCPointer { + fn as_mapping() -> &'static PyMappingMethods { + use crate::common::lock::LazyLock; + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyCPointer::mapping_downcast(mapping); + PyCPointer::__getitem__(zelf, needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyCPointer::mapping_downcast(mapping); + match value { + Some(value) => PyCPointer::__setitem__(zelf, needle.to_owned(), value, vm), + None => Err(vm.new_type_error("Pointer does not support item deletion")), + } + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsBuffer for PyCPointer { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCPointer type must have StgInfo"); + let format = stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("&B")); + let itemsize = stg_info.size; + // Pointer types are scalars with ndim=0, shape=() + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) } } diff --git a/crates/vm/src/stdlib/ctypes/simple.rs b/crates/vm/src/stdlib/ctypes/simple.rs new file mode 100644 index 00000000000..67c07dcb73b --- /dev/null +++ b/crates/vm/src/stdlib/ctypes/simple.rs @@ -0,0 +1,1463 @@ +use super::_ctypes::CArgObject; +use super::array::{PyCArray, WCHAR_SIZE, wchar_to_bytes}; +use super::base::{ + CDATA_BUFFER_METHODS, FfiArgValue, PyCData, StgInfo, StgInfoFlags, buffer_to_ffi_value, + bytes_to_pyobject, +}; +use super::function::PyCFuncPtr; +use super::get_size; +use super::pointer::PyCPointer; +use crate::builtins::{PyByteArray, PyBytes, PyInt, PyNone, PyStr, PyType, PyTypeRef}; +use crate::convert::ToPyObject; +use crate::function::{Either, FuncArgs, OptionalArg}; +use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, Representable}; +use crate::{AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use core::fmt::Debug; +use num_traits::ToPrimitive; + +/// Valid type codes for ctypes simple types +#[cfg(windows)] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPXOv?g"; +#[cfg(not(windows))] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPOv?g"; + +/// Convert ctypes type code to PEP 3118 format code. +/// Some ctypes codes need to be mapped to standard-size codes based on platform. +/// _ctypes_alloc_format_string_for_type +fn ctypes_code_to_pep3118(code: char) -> char { + match code { + // c_int: map based on sizeof(int) + 'i' if core::mem::size_of::<core::ffi::c_int>() == 2 => 'h', + 'i' if core::mem::size_of::<core::ffi::c_int>() == 4 => 'i', + 'i' if core::mem::size_of::<core::ffi::c_int>() == 8 => 'q', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 2 => 'H', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 4 => 'I', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 8 => 'Q', + // c_long: map based on sizeof(long) + 'l' if core::mem::size_of::<core::ffi::c_long>() == 4 => 'l', + 'l' if core::mem::size_of::<core::ffi::c_long>() == 8 => 'q', + 'L' if core::mem::size_of::<core::ffi::c_long>() == 4 => 'L', + 'L' if core::mem::size_of::<core::ffi::c_long>() == 8 => 'Q', + // c_bool: map based on sizeof(bool) - typically 1 byte on all platforms + '?' if core::mem::size_of::<bool>() == 1 => '?', + '?' if core::mem::size_of::<bool>() == 2 => 'H', + '?' if core::mem::size_of::<bool>() == 4 => 'L', + '?' if core::mem::size_of::<bool>() == 8 => 'Q', + // Default: use the same code + _ => code, + } +} + +/// _ctypes_alloc_format_string_for_type +fn alloc_format_string_for_type(code: char, big_endian: bool) -> String { + let prefix = if big_endian { ">" } else { "<" }; + let pep_code = ctypes_code_to_pep3118(code); + format!("{}{}", prefix, pep_code) +} + +/// Create a new simple type instance from a class +fn new_simple_type( + cls: Either<&PyObject, &Py<PyType>>, + vm: &VirtualMachine, +) -> PyResult<PyCSimple> { + let cls = match cls { + Either::A(obj) => obj, + Either::B(typ) => typ.as_object(), + }; + + let _type_ = cls + .get_attr("_type_", vm) + .map_err(|_| vm.new_attribute_error("class must define a '_type_' attribute"))?; + + if !_type_.is_instance((&vm.ctx.types.str_type).as_ref(), vm)? { + return Err(vm.new_type_error("class must define a '_type_' string attribute")); + } + + let tp_str = _type_.str(vm)?.to_string(); + + if tp_str.len() != 1 { + return Err(vm.new_value_error(format!( + "class must define a '_type_' attribute which must be a string of length 1, str: {tp_str}" + ))); + } + + if !SIMPLE_TYPE_CHARS.contains(tp_str.as_str()) { + return Err(vm.new_attribute_error(format!( + "class must define a '_type_' attribute which must be\n a single character string containing one of {SIMPLE_TYPE_CHARS}, currently it is {tp_str}." + ))); + } + + let size = get_size(&tp_str); + Ok(PyCSimple(PyCData::from_bytes(vec![0u8; size], None))) +} + +fn set_primitive(_type_: &str, value: &PyObject, vm: &VirtualMachine) -> PyResult { + match _type_ { + "c" => { + // c_set: accepts bytes(len=1), bytearray(len=1), or int(0-255) + if value + .downcast_ref_if_exact::<PyBytes>(vm) + .is_some_and(|v| v.len() == 1) + || value + .downcast_ref_if_exact::<PyByteArray>(vm) + .is_some_and(|v| v.borrow_buf().len() == 1) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some_and(|v| { + v.as_bigint() + .to_i64() + .is_some_and(|n| (0..=255).contains(&n)) + }) + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("one character bytes, bytearray or integer expected")) + } + } + "u" => { + if let Ok(b) = value.str(vm).map(|v| v.to_string().chars().count() == 1) { + if b { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("one character unicode string expected")) + } + } else { + Err(vm.new_type_error(format!( + "unicode string expected instead of {} instance", + value.class().name() + ))) + } + } + "b" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + // Support __index__ protocol + if value.try_index(vm).is_ok() { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "an integer is required (got type {})", + value.class().name() + ))) + } + } + "f" | "d" | "g" => { + // Handle int specially to check overflow + if let Some(int_obj) = value.downcast_ref_if_exact::<PyInt>(vm) { + // Check if int can fit in f64 + if let Some(f) = int_obj.as_bigint().to_f64() + && f.is_finite() + { + return Ok(value.to_owned()); + } + return Err(vm.new_overflow_error("int too large to convert to float")); + } + // __float__ protocol + if value.try_float(vm).is_ok() { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!("must be real number, not {}", value.class().name()))) + } + } + "?" => Ok(PyObjectRef::from( + vm.ctx.new_bool(value.to_owned().try_to_bool(vm)?), + )), + "v" => { + // VARIANT_BOOL: any truthy → True + Ok(PyObjectRef::from( + vm.ctx.new_bool(value.to_owned().try_to_bool(vm)?), + )) + } + "B" => { + // Support __index__ protocol + if value.try_index(vm).is_ok() { + // Store as-is, conversion to unsigned happens in the getter + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!("int expected instead of {}", value.class().name()))) + } + } + "z" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyBytes>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "bytes or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + "Z" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + // O_set: py_object accepts any Python object + "O" => Ok(value.to_owned()), + // X_set: BSTR - same as Z (c_wchar_p), accepts None, int, or str + "X" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + _ => { + // "P" + if value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyNone>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("cannot be converted to pointer")) + } + } + } +} + +#[pyclass(module = "_ctypes", name = "PyCSimpleType", base = PyType)] +#[derive(Debug)] +#[repr(transparent)] +pub struct PyCSimpleType(PyType); + +#[pyclass(flags(BASETYPE), with(AsNumber, Initializer))] +impl PyCSimpleType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[allow(clippy::new_ret_no_self)] + #[pymethod] + fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(PyObjectRef::from( + new_simple_type(Either::B(&cls), vm)? + .into_ref_with_type(vm, cls)? + .clone(), + )) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the class (e.g., c_int) that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If the value is already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Get the type code to determine conversion rules + let type_code = cls.type_code(vm); + + // 3. Handle None for pointer types (c_char_p, c_wchar_p, c_void_p) + if vm.is_none(&value) && matches!(type_code.as_deref(), Some("z") | Some("Z") | Some("P")) { + return Ok(value); + } + + // Helper to create CArgObject wrapping a simple instance + let create_simple_with_value = |type_str: &str, val: &PyObject| -> PyResult { + let simple = new_simple_type(Either::B(&cls), vm)?; + let buffer_bytes = value_to_bytes_endian(type_str, val, false, vm); + *simple.0.buffer.write() = alloc::borrow::Cow::Owned(buffer_bytes.clone()); + let simple_obj: PyObjectRef = simple.into_ref_with_type(vm, cls.clone())?.into(); + // from_param returns CArgObject, not the simple type itself + let tag = type_str.as_bytes().first().copied().unwrap_or(b'?'); + let ffi_value = buffer_to_ffi_value(type_str, &buffer_bytes); + Ok(CArgObject { + tag, + value: ffi_value, + obj: simple_obj, + size: 0, + offset: 0, + } + .to_pyobject(vm)) + }; + + // 4. Try to convert value based on type code + match type_code.as_deref() { + // Integer types: accept integers + Some(tc @ ("b" | "B" | "h" | "H" | "i" | "I" | "l" | "L" | "q" | "Q")) => { + if value.try_int(vm).is_ok() { + return create_simple_with_value(tc, &value); + } + } + // Float types: accept numbers + Some(tc @ ("f" | "d" | "g")) => { + if value.try_float(vm).is_ok() || value.try_int(vm).is_ok() { + return create_simple_with_value(tc, &value); + } + } + // c_char: 1 byte character + Some("c") => { + if let Some(bytes) = value.downcast_ref::<PyBytes>() + && bytes.len() == 1 + { + return create_simple_with_value("c", &value); + } + if let Ok(int_val) = value.try_int(vm) + && int_val.as_bigint().to_u8().is_some() + { + return create_simple_with_value("c", &value); + } + return Err(vm.new_type_error( + "one character bytes, bytearray or integer expected".to_string(), + )); + } + // c_wchar: 1 unicode character + Some("u") => { + if let Some(s) = value.downcast_ref::<PyStr>() + && s.as_str().chars().count() == 1 + { + return create_simple_with_value("u", &value); + } + return Err(vm.new_type_error("one character unicode string expected")); + } + // c_char_p: bytes pointer + Some("z") => { + // 1. bytes → create CArgObject with null-terminated buffer + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + return Ok(CArgObject { + tag: b'z', + value: FfiArgValue::OwnedPointer(ptr, kept_alive), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 2. Array/Pointer with c_char element type + if is_cchar_array_or_pointer(&value, vm) { + return Ok(value); + } + // 3. CArgObject (byref(c_char(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'c' + { + return Ok(value.clone()); + } + } + // c_wchar_p: unicode pointer + Some("Z") => { + // 1. str → create CArgObject with null-terminated wchar buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::OwnedPointer(ptr, holder), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 2. Array/Pointer with c_wchar element type + if is_cwchar_array_or_pointer(&value, vm)? { + return Ok(value); + } + // 3. CArgObject (byref(c_wchar(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'u' + { + return Ok(value.clone()); + } + } + // c_void_p: most flexible - accepts int, bytes, str, any array/pointer, funcptr + Some("P") => { + // 1. int → create c_void_p with that address + if value.try_int(vm).is_ok() { + return create_simple_with_value("P", &value); + } + // 2. bytes → create CArgObject with null-terminated buffer + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + return Ok(CArgObject { + tag: b'z', + value: FfiArgValue::OwnedPointer(ptr, kept_alive), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 3. str → create CArgObject with null-terminated wchar buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::OwnedPointer(ptr, holder), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 4. Any Array or Pointer → accept directly + if value.downcast_ref::<PyCArray>().is_some() + || value.downcast_ref::<PyCPointer>().is_some() + { + return Ok(value); + } + // 5. CArgObject with 'P' tag (byref(c_void_p(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'P' + { + return Ok(value.clone()); + } + // 6. PyCFuncPtr → extract function pointer address + if let Some(funcptr) = value.downcast_ref::<PyCFuncPtr>() { + let ptr_val = { + let buffer = funcptr._base.buffer.read(); + buffer + .first_chunk::<{ size_of::<usize>() }>() + .copied() + .map_or(0, usize::from_ne_bytes) + }; + return Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 7. c_char_p or c_wchar_p instance → extract pointer value + if let Some(simple) = value.downcast_ref::<PyCSimple>() { + let value_type_code = value.class().type_code(vm); + if matches!(value_type_code.as_deref(), Some("z") | Some("Z")) { + let ptr_val = { + let buffer = simple.0.buffer.read(); + buffer + .first_chunk::<{ size_of::<usize>() }>() + .copied() + .map_or(0, usize::from_ne_bytes) + }; + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::Pointer(ptr_val), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + } + } + // c_bool + Some("?") => { + let bool_val = value.is_true(vm)?; + let bool_obj: PyObjectRef = vm.ctx.new_bool(bool_val).into(); + return create_simple_with_value("?", &bool_obj); + } + _ => {} + } + + // 5. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCSimpleType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + // 6. Type-specific error messages + match type_code.as_deref() { + Some("z") => Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as ctypes.c_char_p", + value.class().name() + ))), + Some("Z") => Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as ctypes.c_wchar_p", + value.class().name() + ))), + _ => Err(vm.new_type_error("wrong type")), + } + } + + fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + PyCSimple::repeat(cls, n, vm) + } +} + +impl AsNumber for PyCSimpleType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + multiply: Some(|a, b, vm| { + // a is a PyCSimpleType instance (type object like c_char) + // b is int (array size) + let cls = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("expected type"))?; + let n = b + .try_index(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + PyCSimple::repeat(cls.to_owned(), n, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Initializer for PyCSimpleType { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // type_init requires exactly 3 positional arguments: name, bases, dict + if args.args.len() != 3 { + return Err(vm.new_type_error(format!( + "type.__init__() takes 3 positional arguments but {} were given", + args.args.len() + ))); + } + + // Get the type from the metatype instance + let type_ref: PyTypeRef = zelf + .as_object() + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + type_ref.check_not_initialized(vm)?; + + // Get _type_ attribute + let type_attr = match type_ref.as_object().get_attr("_type_", vm) { + Ok(attr) => attr, + Err(_) => { + return Err(vm.new_attribute_error("class must define a '_type_' attribute")); + } + }; + + // Validate _type_ is a string + let type_str = type_attr.str(vm)?.to_string(); + + // Validate _type_ is a single character + if type_str.len() != 1 { + return Err(vm.new_value_error( + "class must define a '_type_' attribute which must be a string of length 1" + .to_owned(), + )); + } + + // Validate _type_ is a valid type character + if !SIMPLE_TYPE_CHARS.contains(type_str.as_str()) { + return Err(vm.new_attribute_error(format!( + "class must define a '_type_' attribute which must be a single character string containing one of '{}', currently it is '{}'.", + SIMPLE_TYPE_CHARS, type_str + ))); + } + + // Initialize StgInfo + let size = super::get_size(&type_str); + let align = super::get_align(&type_str); + let mut stg_info = StgInfo::new(size, align); + + // Set format for PEP 3118 buffer protocol + stg_info.format = Some(alloc_format_string_for_type( + type_str.chars().next().unwrap_or('?'), + cfg!(target_endian = "big"), + )); + stg_info.paramfunc = super::base::ParamFunc::Simple; + + // Set TYPEFLAG_ISPOINTER for pointer types: z (c_char_p), Z (c_wchar_p), + // P (c_void_p), s (char array), X (BSTR), O (py_object) + if matches!(type_str.as_str(), "z" | "Z" | "P" | "s" | "X" | "O") { + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + } + + super::base::set_or_init_stginfo(&type_ref, stg_info); + + // Create __ctype_le__ and __ctype_be__ swapped types + create_swapped_types(&type_ref, &type_str, vm)?; + + Ok(()) + } +} + +/// Create __ctype_le__ and __ctype_be__ swapped byte order types +/// On little-endian systems: __ctype_le__ = self, __ctype_be__ = swapped type +/// On big-endian systems: __ctype_be__ = self, __ctype_le__ = swapped type +/// +/// - Single-byte types (c, b, B): __ctype_le__ = __ctype_be__ = self +/// - Pointer/unsupported types (z, Z, P, u, O): NO __ctype_le__/__ctype_be__ attributes +/// - Multi-byte numeric types (h, H, i, I, l, L, q, Q, f, d, g, ?): create swapped types +fn create_swapped_types( + type_ref: &Py<PyType>, + type_str: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + use crate::builtins::PyDict; + + // Avoid infinite recursion - if __ctype_le__ already exists, skip + if type_ref.as_object().get_attr("__ctype_le__", vm).is_ok() { + return Ok(()); + } + + // Types that don't support byte order swapping - no __ctype_le__/__ctype_be__ + // c_void_p (P), c_char_p (z), c_wchar_p (Z), c_wchar (u), py_object (O) + let unsupported_types = ["P", "z", "Z", "u", "O"]; + if unsupported_types.contains(&type_str) { + return Ok(()); + } + + // Single-byte types - __ctype_le__ = __ctype_be__ = self (no swapping needed) + // c_char (c), c_byte (b), c_ubyte (B) + let single_byte_types = ["c", "b", "B"]; + if single_byte_types.contains(&type_str) { + type_ref + .as_object() + .set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + return Ok(()); + } + + // Multi-byte types - create swapped type + // Check system byte order at compile time + let is_little_endian = cfg!(target_endian = "little"); + + // Create dict for the swapped (non-native) type + let swapped_dict: crate::PyRef<crate::builtins::PyDict> = PyDict::default().into_ref(&vm.ctx); + swapped_dict.set_item("_type_", vm.ctx.new_str(type_str).into(), vm)?; + + // Create the swapped type using the same metaclass + let metaclass = type_ref.class(); + let bases = vm.ctx.new_tuple(vec![type_ref.as_object().to_owned()]); + + // Set placeholder first to prevent recursion + type_ref + .as_object() + .set_attr("__ctype_le__", vm.ctx.none(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", vm.ctx.none(), vm)?; + + // Create only the non-native endian type + let suffix = if is_little_endian { "_be" } else { "_le" }; + let swapped_type = metaclass.as_object().call( + ( + vm.ctx.new_str(format!("{}{}", type_ref.name(), suffix)), + bases, + swapped_dict.as_object().to_owned(), + ), + vm, + )?; + + // Set _swappedbytes_ on the swapped type to indicate byte swapping is needed + swapped_type.set_attr("_swappedbytes_", vm.ctx.none(), vm)?; + + // Update swapped type's StgInfo format to use opposite endian prefix + if let Ok(swapped_type_ref) = swapped_type.clone().downcast::<PyType>() + && let Some(mut sw_stg) = swapped_type_ref.get_type_data_mut::<StgInfo>() + { + // Swapped: little-endian system uses big-endian prefix and vice versa + sw_stg.format = Some(alloc_format_string_for_type( + type_str.chars().next().unwrap_or('?'), + is_little_endian, + )); + } + + // Set attributes based on system byte order + // Native endian attribute points to self, non-native points to swapped type + if is_little_endian { + // Little-endian system: __ctype_le__ = self, __ctype_be__ = swapped + type_ref + .as_object() + .set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", swapped_type.clone(), vm)?; + swapped_type.set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + swapped_type.set_attr("__ctype_be__", swapped_type.clone(), vm)?; + } else { + // Big-endian system: __ctype_be__ = self, __ctype_le__ = swapped + type_ref + .as_object() + .set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_le__", swapped_type.clone(), vm)?; + swapped_type.set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + swapped_type.set_attr("__ctype_le__", swapped_type.clone(), vm)?; + } + + Ok(()) +} + +#[pyclass( + module = "_ctypes", + name = "_SimpleCData", + base = PyCData, + metaclass = "PyCSimpleType" +)] +#[repr(transparent)] +pub struct PyCSimple(pub PyCData); + +impl Debug for PyCSimple { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCSimple") + .field("size", &self.0.buffer.read().len()) + .finish() + } +} + +fn value_to_bytes_endian( + _type_: &str, + value: &PyObject, + swapped: bool, + vm: &VirtualMachine, +) -> Vec<u8> { + // Helper macro for endian conversion + macro_rules! to_bytes { + ($val:expr) => { + if swapped { + // Use opposite endianness + #[cfg(target_endian = "little")] + { + $val.to_be_bytes().to_vec() + } + #[cfg(target_endian = "big")] + { + $val.to_le_bytes().to_vec() + } + } else { + $val.to_ne_bytes().to_vec() + } + }; + } + + match _type_ { + "c" => { + // c_char - single byte (bytes, bytearray, or int 0-255) + if let Some(bytes) = value.downcast_ref::<PyBytes>() + && !bytes.is_empty() + { + return vec![bytes.as_bytes()[0]]; + } + if let Some(bytearray) = value.downcast_ref::<PyByteArray>() { + let buf = bytearray.borrow_buf(); + if !buf.is_empty() { + return vec![buf[0]]; + } + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_u8() + { + return vec![v]; + } + vec![0] + } + "u" => { + // c_wchar - platform-dependent size (2 on Windows, 4 on Unix) + if let Ok(s) = value.str(vm) + && let Some(c) = s.as_str().chars().next() + { + let mut buffer = vec![0u8; WCHAR_SIZE]; + wchar_to_bytes(c as u32, &mut buffer); + if swapped { + buffer.reverse(); + } + return buffer; + } + vec![0; WCHAR_SIZE] + } + "b" => { + // c_byte - signed char (1 byte) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i8; + return vec![v as u8]; + } + vec![0] + } + "B" => { + // c_ubyte - unsigned char (1 byte) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u8; + return vec![v]; + } + vec![0] + } + "h" => { + // c_short (2 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i16; + return to_bytes!(v); + } + vec![0; 2] + } + "H" => { + // c_ushort (2 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u16; + return to_bytes!(v); + } + vec![0; 2] + } + "i" => { + // c_int (4 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i32; + return to_bytes!(v); + } + vec![0; 4] + } + "I" => { + // c_uint (4 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u32; + return to_bytes!(v); + } + vec![0; 4] + } + "l" => { + // c_long (platform dependent) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_long; + return to_bytes!(v); + } + const SIZE: usize = core::mem::size_of::<libc::c_long>(); + vec![0; SIZE] + } + "L" => { + // c_ulong (platform dependent) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_ulong; + return to_bytes!(v); + } + const SIZE: usize = core::mem::size_of::<libc::c_ulong>(); + vec![0; SIZE] + } + "q" => { + // c_longlong (8 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i64; + return to_bytes!(v); + } + vec![0; 8] + } + "Q" => { + // c_ulonglong (8 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u64; + return to_bytes!(v); + } + vec![0; 8] + } + "f" => { + // c_float (4 bytes) - also accepts int + if let Ok(float_val) = value.try_float(vm) { + return to_bytes!(float_val.to_f64() as f32); + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_f64() + { + return to_bytes!(v as f32); + } + vec![0; 4] + } + "d" => { + // c_double (8 bytes) - also accepts int + if let Ok(float_val) = value.try_float(vm) { + return to_bytes!(float_val.to_f64()); + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_f64() + { + return to_bytes!(v); + } + vec![0; 8] + } + "g" => { + // long double - platform dependent size + // Store as f64, zero-pad to platform long double size + // Note: This may lose precision on platforms where long double > 64 bits + let f64_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) + } else { + 0.0 + }; + let f64_bytes = if swapped { + #[cfg(target_endian = "little")] + { + f64_val.to_be_bytes().to_vec() + } + #[cfg(target_endian = "big")] + { + f64_val.to_le_bytes().to_vec() + } + } else { + f64_val.to_ne_bytes().to_vec() + }; + // Pad to long double size + let long_double_size = super::get_size("g"); + let mut result = f64_bytes; + result.resize(long_double_size, 0); + result + } + "?" => { + // c_bool (1 byte) + if let Ok(b) = value.to_owned().try_to_bool(vm) { + return vec![if b { 1 } else { 0 }]; + } + vec![0] + } + "v" => { + // VARIANT_BOOL: True = 0xFFFF (-1 as i16), False = 0x0000 + if let Ok(b) = value.to_owned().try_to_bool(vm) { + let val: i16 = if b { -1 } else { 0 }; + return to_bytes!(val); + } + vec![0; 2] + } + "P" => { + // c_void_p - pointer type (platform pointer size) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "z" => { + // c_char_p - pointer to char (stores pointer value from int) + // PyBytes case is handled in slot_new/set_value with make_z_buffer() + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "Z" => { + // c_wchar_p - pointer to wchar_t (stores pointer value from int) + // PyStr case is handled in slot_new/set_value with make_wchar_buffer() + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "O" => { + // py_object - store object id as non-zero marker + // The actual object is stored in _objects + // Use object's id as a non-zero placeholder (indicates non-NULL) + let id = value.get_id(); + to_bytes!(id) + } + _ => vec![0], + } +} + +/// Check if value is a c_char array or pointer(c_char) +fn is_cchar_array_or_pointer(value: &PyObject, vm: &VirtualMachine) -> bool { + // Check Array with c_char element type + if let Some(arr) = value.downcast_ref::<PyCArray>() + && let Some(info) = arr.class().stg_info_opt() + && let Some(ref elem_type) = info.element_type + && let Some(elem_code) = elem_type.type_code(vm) + { + return elem_code == "c"; + } + // Check Pointer to c_char + if let Some(ptr) = value.downcast_ref::<PyCPointer>() + && let Some(info) = ptr.class().stg_info_opt() + && let Some(ref proto) = info.proto + && let Some(proto_code) = proto.type_code(vm) + { + return proto_code == "c"; + } + false +} + +/// Check if value is a c_wchar array or pointer(c_wchar) +fn is_cwchar_array_or_pointer(value: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + // Check Array with c_wchar element type + if let Some(arr) = value.downcast_ref::<PyCArray>() { + let info = arr.class().stg_info(vm)?; + let elem_type = info.element_type.as_ref().expect("array has element_type"); + if let Some(elem_code) = elem_type.type_code(vm) { + return Ok(elem_code == "u"); + } + } + // Check Pointer to c_wchar + if let Some(ptr) = value.downcast_ref::<PyCPointer>() { + let info = ptr.class().stg_info(vm)?; + if let Some(ref proto) = info.proto + && let Some(proto_code) = proto.type_code(vm) + { + return Ok(proto_code == "u"); + } + } + Ok(false) +} + +impl Constructor for PyCSimple { + type Args = (OptionalArg,); + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let _type_ = cls + .type_code(vm) + .ok_or_else(|| vm.new_type_error("abstract class"))?; + // Save the initial argument for c_char_p/c_wchar_p _objects + let init_arg = args.0.into_option(); + + // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak + if let Some(ref v) = init_arg { + if _type_ == "z" { + if let Some(bytes) = v.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let buffer = ptr.to_ne_bytes().to_vec(); + let cdata = PyCData::from_bytes(buffer, Some(v.clone())); + *cdata.base.write() = Some(kept_alive); + return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); + } + } else if _type_ == "Z" + && let Some(s) = v.downcast_ref::<PyStr>() + { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); + let buffer = ptr.to_ne_bytes().to_vec(); + let cdata = PyCData::from_bytes(buffer, Some(holder)); + return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); + } + } + + let value = if let Some(ref v) = init_arg { + set_primitive(_type_.as_str(), v, vm)? + } else { + match _type_.as_str() { + "c" | "u" => PyObjectRef::from(vm.ctx.new_bytes(vec![0])), + "b" | "B" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + PyObjectRef::from(vm.ctx.new_int(0)) + } + "f" | "d" | "g" => PyObjectRef::from(vm.ctx.new_float(0.0)), + "?" => PyObjectRef::from(vm.ctx.new_bool(false)), + _ => vm.ctx.none(), // "z" | "Z" | "P" + } + }; + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + let buffer = value_to_bytes_endian(&_type_, &value, swapped, vm); + + // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), + // store the initial value in _objects + let objects = if (_type_ == "z" || _type_ == "Z" || _type_ == "O") && init_arg.is_some() { + init_arg + } else { + None + }; + + PyCSimple(PyCData::from_bytes(buffer, objects)) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCSimple { + type Args = (OptionalArg,); + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // If an argument is provided, update the value + if let Some(value) = args.0.into_option() { + PyCSimple::set_value(zelf.into(), value, vm)?; + } + Ok(()) + } +} + +// Simple_repr +impl Representable for PyCSimple { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let cls = zelf.class(); + let type_name = cls.name(); + + // Check if base is _SimpleCData (direct simple type like c_int, c_char) + // vs subclass of simple type (like class X(c_int): pass) + let bases = cls.bases.read(); + let is_direct_simple = bases + .iter() + .any(|base| base.name().to_string() == "_SimpleCData"); + + if is_direct_simple { + // Direct SimpleCData: "typename(repr(value))" + let value = PyCSimple::value(zelf.to_owned().into(), vm)?; + let value_repr = value.repr(vm)?.to_string(); + Ok(format!("{}({})", type_name, value_repr)) + } else { + // Subclass: "<typename object at addr>" + let addr = zelf.get_id(); + Ok(format!("<{} object at {:#x}>", type_name, addr)) + } + } +} + +#[pyclass( + flags(BASETYPE), + with(Constructor, Initializer, AsBuffer, AsNumber, Representable) +)] +impl PyCSimple { + #[pygetset] + fn _b0_(&self) -> Option<PyObjectRef> { + self.0.base.read().clone() + } + + #[pygetset] + pub fn value(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let zelf: &Py<Self> = instance + .downcast_ref() + .ok_or_else(|| vm.new_type_error("cannot get value of instance"))?; + + // Get _type_ from class + let cls = zelf.class(); + let type_attr = cls + .as_object() + .get_attr("_type_", vm) + .map_err(|_| vm.new_type_error("no _type_ attribute"))?; + let type_code = type_attr.str(vm)?.to_string(); + + // Special handling for c_char_p (z) and c_wchar_p (Z) + // z_get, Z_get - dereference pointer to get string + if type_code == "z" { + // c_char_p: read pointer from buffer, dereference to get bytes string + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated string at the address + unsafe { + let cstr = core::ffi::CStr::from_ptr(ptr as _); + return Ok(vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into()); + } + } + if type_code == "Z" { + // c_wchar_p: read pointer from buffer, dereference to get wide string + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated wide string at the address + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide for surrogate pairs + // Unix: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + unsafe { + let w_ptr = ptr as *const libc::wchar_t; + let len = libc::wcslen(w_ptr); + let wchars = core::slice::from_raw_parts(w_ptr, len); + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = wchars.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + return Ok(vm.ctx.new_str(wtf8).into()); + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + return Ok(vm.ctx.new_str(s).into()); + } + } + } + + // O_get: py_object - read PyObject pointer from buffer + if type_code == "O" { + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Err(vm.new_value_error("PyObject is NULL")); + } + // Non-NULL: return stored object from _objects if available + if let Some(obj) = zelf.0.objects.read().as_ref() { + return Ok(obj.clone()); + } + return Err(vm.new_value_error("PyObject is NULL")); + } + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + // Read value from buffer, swap bytes if needed + let buffer = zelf.0.buffer.read(); + let buffer_data: alloc::borrow::Cow<'_, [u8]> = if swapped { + // Reverse bytes for swapped endian types + let mut swapped_bytes = buffer.to_vec(); + swapped_bytes.reverse(); + alloc::borrow::Cow::Owned(swapped_bytes) + } else { + alloc::borrow::Cow::Borrowed(&*buffer) + }; + + let cls_ref = cls.to_owned(); + bytes_to_pyobject(&cls_ref, &buffer_data, vm).or_else(|_| { + // Fallback: return bytes as integer based on type + match type_code.as_str() { + "c" => { + if !buffer.is_empty() { + Ok(vm.ctx.new_bytes(vec![buffer[0]]).into()) + } else { + Ok(vm.ctx.new_bytes(vec![0]).into()) + } + } + "?" => { + let val = buffer.first().copied().unwrap_or(0); + Ok(vm.ctx.new_bool(val != 0).into()) + } + _ => Ok(vm.ctx.new_int(0).into()), + } + }) + } + + #[pygetset(setter)] + fn set_value(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf: PyRef<Self> = instance + .clone() + .downcast() + .map_err(|_| vm.new_type_error("cannot set value of instance"))?; + + // Get _type_ from class + let cls = zelf.class(); + let type_attr = cls + .as_object() + .get_attr("_type_", vm) + .map_err(|_| vm.new_type_error("no _type_ attribute"))?; + let type_code = type_attr.str(vm)?.to_string(); + + // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak + if type_code == "z" { + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.objects.write() = Some(value); + *zelf.0.base.write() = Some(kept_alive); + return Ok(()); + } + } else if type_code == "Z" + && let Some(s) = value.downcast_ref::<PyStr>() + { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.objects.write() = Some(holder); + return Ok(()); + } + + let content = set_primitive(&type_code, &value, vm)?; + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = instance + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok(); + + // Update buffer when value changes + let buffer_bytes = value_to_bytes_endian(&type_code, &content, swapped, vm); + + // If the buffer is borrowed (from shared memory), write in-place + // Otherwise replace with new owned buffer + let mut buffer = zelf.0.buffer.write(); + match &mut *buffer { + Cow::Borrowed(slice) => { + // SAFETY: For from_buffer, the slice points to writable shared memory. + // Python's from_buffer requires writable buffer, so this is safe. + let ptr = slice.as_ptr() as *mut u8; + let len = slice.len().min(buffer_bytes.len()); + unsafe { + core::ptr::copy_nonoverlapping(buffer_bytes.as_ptr(), ptr, len); + } + } + Cow::Owned(vec) => { + vec.copy_from_slice(&buffer_bytes); + } + } + + // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), + // keep the reference in _objects + if type_code == "z" || type_code == "Z" || type_code == "O" { + *zelf.0.objects.write() = Some(value); + } + Ok(()) + } + + #[pyclassmethod] + fn repeat(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + use super::array::array_type_from_ctype; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) + } + + /// Simple_from_outparm - convert output parameter back to Python value + /// For direct subclasses of _SimpleCData (e.g., c_int), returns the value. + /// For subclasses of those (e.g., class MyInt(c_int)), returns self. + #[pymethod] + fn __ctypes_from_outparam__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // _ctypes_simple_instance: returns true if NOT a direct subclass of Simple_Type + // i.e., c_int (direct) -> false, MyInt(c_int) (subclass) -> true + let is_subclass_of_simple = { + let cls = zelf.class(); + let bases = cls.bases.read(); + // If base is NOT _SimpleCData, then it's a subclass of a subclass + !bases + .iter() + .any(|base| base.name().to_string() == "_SimpleCData") + }; + + if is_subclass_of_simple { + // Subclass of simple type (e.g., MyInt(c_int)): return self + Ok(zelf.into()) + } else { + // Direct simple type (e.g., c_int): return value + PyCSimple::value(zelf.into(), vm) + } + } +} + +impl PyCSimple { + /// Extract the value from this ctypes object as an owned FfiArgValue. + /// The value must be kept alive until after the FFI call completes. + pub fn to_ffi_value( + &self, + ty: libffi::middle::Type, + _vm: &VirtualMachine, + ) -> Option<FfiArgValue> { + let buffer = self.0.buffer.read(); + let bytes: &[u8] = &buffer; + + let ret = if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { + let byte = *bytes.first()?; + FfiArgValue::U8(byte) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { + let byte = *bytes.first()?; + FfiArgValue::I8(byte as i8) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<2>()?; + FfiArgValue::U16(u16::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<2>()?; + FfiArgValue::I16(i16::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::U32(u32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::I32(i32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::U64(u64::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::I64(i64::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::F32(f32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::F64(f64::from_ne_bytes(bytes)) + } else if core::ptr::eq( + ty.as_raw_ptr(), + libffi::middle::Type::pointer().as_raw_ptr(), + ) { + let bytes = *buffer.first_chunk::<{ size_of::<usize>() }>()?; + let val = usize::from_ne_bytes(bytes); + FfiArgValue::Pointer(val) + } else { + return None; + }; + Some(ret) + } +} + +impl AsBuffer for PyCSimple { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCSimple type must have StgInfo"); + let format = stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("B")); + let itemsize = stg_info.size; + // Simple types are scalars with ndim=0, shape=() + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} + +/// Simple_bool: return non-zero if any byte in buffer is non-zero +impl AsNumber for PyCSimple { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|obj, _vm| { + let zelf = obj + .downcast_ref::<PyCSimple>() + .expect("PyCSimple::as_number called on non-PyCSimple"); + let buffer = zelf.0.buffer.read(); + // Simple_bool: memcmp(self->b_ptr, zeros, self->b_size) + // Returns true if any byte is non-zero + Ok(buffer.iter().any(|&b| b != 0)) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} diff --git a/crates/vm/src/stdlib/ctypes/structure.rs b/crates/vm/src/stdlib/ctypes/structure.rs index f32d6865cb6..c0116d9d76c 100644 --- a/crates/vm/src/stdlib/ctypes/structure.rs +++ b/crates/vm/src/stdlib/ctypes/structure.rs @@ -1,42 +1,60 @@ -use super::base::{CDataObject, PyCData}; -use super::field::PyCField; -use super::util::StgInfo; +use super::base::{CDATA_BUFFER_METHODS, PyCData, PyCField, StgInfo, StgInfoFlags}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::protocol::{BufferDescriptor, BufferMethods, PyBuffer, PyNumberMethods}; -use crate::stdlib::ctypes::_ctypes::get_size; -use crate::types::{AsBuffer, AsNumber, Constructor}; +use crate::function::{FuncArgs, OptionalArg, PySetterValue}; +use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::stdlib::warnings; +use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; -use indexmap::IndexMap; +use alloc::borrow::Cow; +use core::fmt::Debug; use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; -use std::fmt::Debug; + +/// Calculate Structure type size from _fields_ (sum of field sizes) +pub(super) fn calculate_struct_size(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<usize> { + if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + let mut total_size = 0usize; + + for field in fields.iter() { + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(field_type) = tuple.get(1) + { + total_size += super::_ctypes::sizeof(field_type.clone(), vm)?; + } + } + return Ok(total_size); + } + Ok(0) +} /// PyCStructType - metaclass for Structure #[pyclass(name = "PyCStructType", base = PyType, module = "_ctypes")] #[derive(Debug)] #[repr(transparent)] -pub struct PyCStructType(PyType); +pub(super) struct PyCStructType(PyType); impl Constructor for PyCStructType { type Args = FuncArgs; fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // 1. Create the new class using PyType::py_new + // 1. Create the new class using PyType::slot_new let new_class = crate::builtins::type_::PyType::slot_new(metatype, args, vm)?; - // 2. Process _fields_ if defined on the new class + // 2. Get the new type let new_type = new_class .clone() .downcast::<PyType>() .map_err(|_| vm.new_type_error("expected type"))?; - // Only process _fields_ if defined directly on this class (not inherited) - if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { - Self::process_fields(&new_type, fields_attr, vm)?; - } + // 3. Mark base classes as finalized (subclassing finalizes the parent) + new_type.mark_bases_final(); + + // 4. Initialize StgInfo for the new type (initialized=false, to be set in init) + let stg_info = StgInfo::default(); + let _ = new_type.init_type_data(stg_info); + // Note: _fields_ processing moved to Initializer::init() Ok(new_class) } @@ -45,11 +63,162 @@ impl Constructor for PyCStructType { } } -#[pyclass(flags(BASETYPE), with(AsNumber, Constructor))] +impl Initializer for PyCStructType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef by converting PyRef<Self> -> PyObjectRef -> PyRef<PyType> + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + // Backward compatibility: skip initialization for abstract types + if new_type + .get_direct_attr(vm.ctx.intern_str("_abstract_")) + .is_some() + { + return Ok(()); + } + + new_type.check_not_initialized(vm)?; + + // Process _fields_ if defined directly on this class (not inherited) + if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { + Self::process_fields(&new_type, fields_attr, vm)?; + } else { + // No _fields_ defined - try to copy from base class (PyCStgInfo_clone) + let (has_base_info, base_clone) = { + let bases = new_type.bases.read(); + if let Some(base) = bases.first() { + (base.stg_info_opt().is_some(), Some(base.clone())) + } else { + (false, None) + } + }; + + if has_base_info && let Some(ref base) = base_clone { + // Clone base StgInfo (release guard before getting mutable reference) + let stg_info_opt = base.stg_info_opt().map(|baseinfo| { + let mut stg_info = baseinfo.clone(); + stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL in subclass + stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable + stg_info + }); + + if let Some(stg_info) = stg_info_opt { + // Mark base as FINAL (now guard is released) + if let Some(mut base_stg) = base.get_type_data_mut::<StgInfo>() { + base_stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + super::base::set_or_init_stginfo(&new_type, stg_info); + return Ok(()); + } + } + + // No base StgInfo - create default + let mut stg_info = StgInfo::new(0, 1); + stg_info.paramfunc = super::base::ParamFunc::Structure; + stg_info.format = Some("B".to_string()); + super::base::set_or_init_stginfo(&new_type, stg_info); + } + + Ok(()) + } +} + +#[pyclass(flags(BASETYPE), with(AsNumber, Constructor, Initializer, SetAttr))] impl PyCStructType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the structure type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCStructType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } + + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: crate::function::ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + /// Called when a new Structure subclass is created #[pyclassmethod] fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + cls.mark_bases_final(); + // Check if _fields_ is defined if let Some(fields_attr) = cls.get_direct_attr(vm.ctx.intern_str("_fields_")) { Self::process_fields(&cls, fields_attr, vm)?; @@ -59,24 +228,86 @@ impl PyCStructType { /// Process _fields_ and create CField descriptors fn process_fields( - cls: &PyTypeRef, + cls: &Py<PyType>, fields_attr: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { + // Check if this is a swapped byte order structure + let is_swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + // Try to downcast to list or tuple let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() { list.borrow_vec().to_vec() } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { tuple.to_vec() } else { - return Err(vm.new_type_error("_fields_ must be a list or tuple".to_string())); + return Err(vm.new_type_error("_fields_ must be a list or tuple")); }; - let mut offset = 0usize; + let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let base_type_name = "Structure"; + let msg = format!( + "Due to '_pack_', the '{}' {} will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + base_type_name, + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + + let forced_alignment = + super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); + + // Determine byte order for format string + let big_endian = super::base::is_big_endian(is_swapped); + + // Initialize offset, alignment, type flags, and ffi_field_types from base class + let ( + mut offset, + mut max_align, + mut has_pointer, + mut has_union, + mut has_bitfield, + mut ffi_field_types, + ) = { + let bases = cls.bases.read(); + if let Some(base) = bases.first() + && let Some(baseinfo) = base.stg_info_opt() + { + ( + baseinfo.size, + core::cmp::max(baseinfo.align, forced_alignment), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASUNION), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), + baseinfo.ffi_field_types.clone(), + ) + } else { + (0, forced_alignment, false, false, false, Vec::new()) + } + }; + + // Initialize PEP3118 format string + let mut format = String::from("T{"); + let mut last_end = 0usize; // Track end of last field for padding calculation + + // Bitfield layout tracking + let mut bitfield_bit_offset: u16 = 0; // Current bit position within bitfield group + let mut last_field_bit_size: u16 = 0; // For MSVC: bit size of previous storage unit + let use_msvc_bitfields = pack > 0; // MSVC layout when _pack_ is set + for (index, field) in fields.iter().enumerate() { let field_tuple = field .downcast_ref::<PyTuple>() - .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples".to_string()))?; + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; if field_tuple.len() < 2 { return Err(vm.new_type_error( @@ -86,99 +317,262 @@ impl PyCStructType { let name = field_tuple .first() - .unwrap() + .expect("len checked") .downcast_ref::<PyStr>() - .ok_or_else(|| vm.new_type_error("field name must be a string".to_string()))? + .ok_or_else(|| vm.new_type_error("field name must be a string"))? .to_string(); - let field_type = field_tuple.get(1).unwrap().clone(); + let field_type = field_tuple.get(1).expect("len checked").clone(); + + // For swapped byte order structures, validate field type supports byte swapping + if is_swapped { + super::base::check_other_endian_support(&field_type, vm)?; + } + + // Get size and alignment of the field type + let size = super::base::get_field_size(&field_type, vm)?; + let field_align = super::base::get_field_align(&field_type, vm); - // Get size of the field type - let size = Self::get_field_size(&field_type, vm)?; + // Calculate effective alignment (PyCField_FromDesc) + let effective_align = if pack > 0 { + core::cmp::min(pack, field_align) + } else { + field_align + }; + + // Apply padding to align offset (cfield.c NO_BITFIELD case) + if effective_align > 0 && offset % effective_align != 0 { + let delta = effective_align - (offset % effective_align); + offset += delta; + } - // Create CField descriptor (accepts any ctypes type including arrays) - let c_field = PyCField::new(name.clone(), field_type, offset, size, index); + max_align = max_align.max(effective_align); + + // Propagate type flags from field type (HASPOINTER, HASUNION, HASBITFIELD) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + { + // HASPOINTER: propagate if field is pointer or contains pointer + if field_stg.flags.intersects( + StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER, + ) { + has_pointer = true; + } + // HASUNION, HASBITFIELD: propagate directly + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASUNION) { + has_union = true; + } + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD) { + has_bitfield = true; + } + // Collect FFI type for this field + ffi_field_types.push(field_stg.to_ffi_type()); + } + + // Mark field type as finalized (using type as field finalizes it) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() { + if let Some(mut stg_info) = type_obj.get_type_data_mut::<StgInfo>() { + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + // Create StgInfo with FINAL flag if it doesn't exist + let mut stg_info = StgInfo::new(size, field_align); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = type_obj.init_type_data(stg_info); + } + } + + // Build format string: add padding before field + let padding = offset - last_end; + if padding > 0 { + if padding != 1 { + format.push_str(&padding.to_string()); + } + format.push('x'); + } + + // Get field format and add to format string + let field_format = super::base::get_field_format(&field_type, big_endian, vm); + + // Handle arrays: prepend shape + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + && !field_stg.shape.is_empty() + { + let shape_str = field_stg + .shape + .iter() + .map(|d| d.to_string()) + .collect::<Vec<_>>() + .join(","); + format.push_str(&std::format!("({}){}", shape_str, field_format)); + } else { + format.push_str(&field_format); + } + + // Add field name + format.push(':'); + format.push_str(&name); + format.push(':'); + + // Create CField descriptor with padding-adjusted offset + let field_type_ref = field_type + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; + + // Check for bitfield size (optional 3rd element in tuple) + let (c_field, field_advances_offset) = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + let type_bits = (size * 8) as u16; + let (advances, bit_offset); + + if use_msvc_bitfields { + // MSVC layout: different types start new storage unit + if bitfield_bit_offset + bit_size > type_bits + || type_bits != last_field_bit_size + { + // Close previous bitfield, start new allocation unit + bitfield_bit_offset = 0; + advances = true; + } else { + advances = false; + } + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + last_field_bit_size = type_bits; + } else { + // GCC System V layout: pack within same type + let fits_in_current = bitfield_bit_offset + bit_size <= type_bits; + advances = if fits_in_current && bitfield_bit_offset > 0 { + false + } else if !fits_in_current { + bitfield_bit_offset = 0; + true + } else { + true + }; + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + } + + // For packed bitfields that share offset, use the same offset as previous + let field_offset = if !advances { + offset - size // Reuse the previous field's offset + } else { + offset + }; + + ( + PyCField::new_bitfield( + name.clone(), + field_type_ref, + field_offset as isize, + size as isize, + bit_size, + bit_offset, + index, + ), + advances, + ) + } else { + bitfield_bit_offset = 0; // Reset on non-bitfield + last_field_bit_size = 0; + ( + PyCField::new( + name.clone(), + field_type_ref, + offset as isize, + size as isize, + index, + ), + true, + ) + }; // Set the CField as a class attribute - cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); + cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); - offset += size; + // Update tracking - don't advance offset for packed bitfields + if field_advances_offset { + last_end = offset + size; + offset += size; + } } - Ok(()) - } + // Calculate total_align = max(max_align, forced_alignment) + let total_align = core::cmp::max(max_align, forced_alignment); - /// Get the size of a ctypes type - fn get_field_size(field_type: &PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - // Try to get _type_ attribute for simple types - if let Some(size) = field_type - .get_attr("_type_", vm) - .ok() - .and_then(|type_attr| type_attr.str(vm).ok()) - .and_then(|type_str| { - let s = type_str.to_string(); - (s.len() == 1).then(|| get_size(&s)) - }) - { - return Ok(size); + // Calculate aligned_size (PyCStructUnionType_update_stginfo) + let aligned_size = if total_align > 0 { + offset.div_ceil(total_align) * total_align + } else { + offset + }; + + // Complete format string: add final padding and close + let final_padding = aligned_size - last_end; + if final_padding > 0 { + if final_padding != 1 { + format.push_str(&final_padding.to_string()); + } + format.push('x'); } + format.push('}'); - // Try sizeof for other types - if let Some(s) = field_type - .get_attr("size_of_instances", vm) - .ok() - .and_then(|size_method| size_method.call((), vm).ok()) - .and_then(|size| size.try_int(vm).ok()) - .and_then(|n| n.as_bigint().to_usize()) + // Check for circular self-reference: if a field of the same type as this + // structure was encountered, it would have marked this type's stginfo as FINAL. + if let Some(stg_info) = cls.get_type_data::<StgInfo>() + && stg_info.is_final() { - return Ok(s); + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); } - // Default to pointer size for unknown types - Ok(std::mem::size_of::<usize>()) - } - - /// Get the alignment of a ctypes type - fn get_field_align(field_type: &PyObjectRef, vm: &VirtualMachine) -> usize { - // Try to get _type_ attribute for simple types - if let Some(align) = field_type - .get_attr("_type_", vm) - .ok() - .and_then(|type_attr| type_attr.str(vm).ok()) - .and_then(|type_str| { - let s = type_str.to_string(); - (s.len() == 1).then(|| get_size(&s)) // alignment == size for simple types - }) - { - return align; + // Store StgInfo with aligned size and total alignment + let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); + stg_info.format = Some(format); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; // Mark as finalized + if has_pointer { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASPOINTER; } - // Default alignment - 1 + if has_union { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASUNION; + } + if has_bitfield { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASBITFIELD; + } + stg_info.paramfunc = super::base::ParamFunc::Structure; + // Set byte order: swap if _swappedbytes_ is defined + stg_info.big_endian = super::base::is_big_endian(is_swapped); + // Store FFI field types for structure passing + stg_info.ffi_field_types = ffi_field_types; + super::base::set_or_init_stginfo(cls, stg_info); + + // Process _anonymous_ fields + super::base::make_anon_fields(cls, vm)?; + + Ok(()) } - #[pymethod] fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { - use super::array::create_array_type_with_stg_info; - use crate::stdlib::ctypes::_ctypes::size_of; + use super::array::array_type_from_ctype; if n < 0 { return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); } - - // Calculate element size from the Structure type - let element_size = size_of(cls.clone().into(), vm)?; - - let total_size = element_size - .checked_mul(n as usize) - .ok_or_else(|| vm.new_overflow_error("array size too large".to_owned()))?; - let stg_info = super::util::StgInfo::new_array( - total_size, - element_size, - n as usize, - cls.clone().into(), - element_size, - ); - create_array_type_with_stg_info(stg_info, vm) + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) } } @@ -188,12 +582,12 @@ impl AsNumber for PyCStructType { multiply: Some(|a, b, vm| { let cls = a .downcast_ref::<PyType>() - .ok_or_else(|| vm.new_type_error("expected type".to_owned()))?; + .ok_or_else(|| vm.new_type_error("expected type"))?; let n = b .try_index(vm)? .as_bigint() .to_isize() - .ok_or_else(|| vm.new_overflow_error("array size too large".to_owned()))?; + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; PyCStructType::__mul__(cls.to_owned(), n, vm) }), ..PyNumberMethods::NOT_IMPLEMENTED @@ -202,14 +596,70 @@ impl AsNumber for PyCStructType { } } -/// Structure field info stored in instance -#[allow(dead_code)] -#[derive(Debug, Clone)] -pub struct FieldInfo { - pub name: String, - pub offset: usize, - pub size: usize, - pub type_ref: PyTypeRef, +impl SetAttr for PyCStructType { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if _fields_ is being set + if attr_name.as_str() == "_fields_" { + let pytype: &Py<PyType> = zelf.to_base(); + + // Check finalization in separate scope to release read lock before process_fields + // This prevents deadlock: process_fields needs write lock on the same RwLock + let is_final = { + let Some(stg_info) = pytype.get_type_data::<StgInfo>() else { + return Err(vm.new_type_error("ctypes state is not initialized")); + }; + stg_info.is_final() + }; // Read lock released here + + if is_final { + return Err(vm.new_attribute_error("_fields_ is final")); + } + + // Process _fields_ and set attribute + let PySetterValue::Assign(fields_value) = value else { + return Err(vm.new_attribute_error("cannot delete _fields_")); + }; + // Process fields (this will also set DICTFLAG_FINAL) + PyCStructType::process_fields(pytype, fields_value.clone(), vm)?; + // Set the _fields_ attribute on the type + pytype + .attributes + .write() + .insert(vm.ctx.intern_str("_fields_"), fields_value); + return Ok(()); + } + // Delegate to PyType's setattro logic for type attributes + let attr_name_interned = vm.ctx.intern_str(attr_name.as_str()); + let pytype: &Py<PyType> = zelf.to_base(); + + // Check for data descriptor first + if let Some(attr) = pytype.get_class_attr(attr_name_interned) { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + return descriptor(&attr, pytype.to_owned().into(), value, vm); + } + } + + // Store in type's attributes dict + if let PySetterValue::Assign(value) = value { + pytype.attributes.write().insert(attr_name_interned, value); + } else { + let prev = pytype.attributes.write().shift_remove(attr_name_interned); + if prev.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + pytype.name(), + attr_name.as_str(), + ))); + } + } + Ok(()) + } } /// PyCStructure - base class for Structure instances @@ -219,19 +669,13 @@ pub struct FieldInfo { base = PyCData, metaclass = "PyCStructType" )] -pub struct PyCStructure { - _base: PyCData, - /// Common CDataObject for memory buffer - pub(super) cdata: PyRwLock<CDataObject>, - /// Field information (name -> FieldInfo) - #[allow(dead_code)] - pub(super) fields: PyRwLock<IndexMap<String, FieldInfo>>, -} +#[repr(transparent)] +pub struct PyCStructure(pub PyCData); impl Debug for PyCStructure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCStructure") - .field("size", &self.cdata.read().size()) + .field("size", &self.0.size()) .finish() } } @@ -239,94 +683,24 @@ impl Debug for PyCStructure { impl Constructor for PyCStructure { type Args = FuncArgs; - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // Get _fields_ from the class using get_attr to properly search MRO - let fields_attr = cls.as_object().get_attr("_fields_", vm).ok(); - - let mut fields_map = IndexMap::new(); - let mut total_size = 0usize; - let mut max_align = 1usize; - - if let Some(fields_attr) = fields_attr { - let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() - { - list.borrow_vec().to_vec() - } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { - tuple.to_vec() - } else { - vec![] - }; - - let mut offset = 0usize; - for field in fields.iter() { - let Some(field_tuple) = field.downcast_ref::<PyTuple>() else { - continue; - }; - if field_tuple.len() < 2 { - continue; - } - let Some(name) = field_tuple.first().unwrap().downcast_ref::<PyStr>() else { - continue; - }; - let name = name.to_string(); - let field_type = field_tuple.get(1).unwrap().clone(); - let size = PyCStructType::get_field_size(&field_type, vm)?; - let field_align = PyCStructType::get_field_align(&field_type, vm); - max_align = max_align.max(field_align); - - let type_ref = field_type - .downcast::<PyType>() - .unwrap_or_else(|_| vm.ctx.types.object_type.to_owned()); - - fields_map.insert( - name.clone(), - FieldInfo { - name, - offset, - size, - type_ref, - }, - ); - - offset += size; - } - total_size = offset; - } - - // Initialize buffer with zeros - let mut stg_info = StgInfo::new(total_size, max_align); - stg_info.length = fields_map.len(); - let cdata = CDataObject::from_stg_info(&stg_info); - let instance = PyCStructure { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), - fields: PyRwLock::new(fields_map.clone()), + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Check for abstract class and extract values in a block to drop the borrow + let (total_size, total_align, length) = { + let stg_info = cls.stg_info(vm)?; + (stg_info.size, stg_info.align, stg_info.length) }; - // Handle keyword arguments for field initialization - let py_instance = instance.into_ref_with_type(vm, cls.clone())?; - let py_obj: PyObjectRef = py_instance.clone().into(); - - // Set field values from kwargs using standard attribute setting - for (key, value) in args.kwargs.iter() { - if fields_map.contains_key(key.as_str()) { - py_obj.set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; - } + // Mark the class as finalized (instance creation finalizes the type) + if let Some(mut stg_info_mut) = cls.get_type_data_mut::<StgInfo>() { + stg_info_mut.flags |= StgInfoFlags::DICTFLAG_FINAL; } - // Set field values from positional args - let field_names: Vec<String> = fields_map.keys().cloned().collect(); - for (i, value) in args.args.iter().enumerate() { - if i < field_names.len() { - py_obj.set_attr( - vm.ctx.intern_str(field_names[i].as_str()), - value.clone(), - vm, - )?; - } - } - - Ok(py_instance.into()) + // Initialize buffer with zeros using computed size + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; + PyCStructure(PyCData::from_stg_info(&new_stg_info)) + .into_ref_with_type(vm, cls) + .map(Into::into) } fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { @@ -334,180 +708,132 @@ impl Constructor for PyCStructure { } } -// Note: GetAttr and SetAttr are not implemented here. -// Field access is handled by CField descriptors registered on the class. - -#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor))] impl PyCStructure { - #[pygetset] - fn _objects(&self) -> Option<PyObjectRef> { - self.cdata.read().objects.clone() - } - - #[pygetset] - fn _fields_(&self, vm: &VirtualMachine) -> PyObjectRef { - // Return the _fields_ from the class, not instance - vm.ctx.none() - } + /// Recursively initialize positional arguments through inheritance chain + /// Returns the number of arguments consumed + fn init_pos_args( + self_obj: &Py<Self>, + type_obj: &Py<PyType>, + args: &[PyObjectRef], + kwargs: &indexmap::IndexMap<String, PyObjectRef>, + index: usize, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let mut current_index = index; + + // 1. First process base class fields recursively + let base_clone = { + let bases = type_obj.bases.read(); + if let Some(base) = bases.first() + && base.stg_info_opt().is_some() + { + Some(base.clone()) + } else { + None + } + }; - #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; + if let Some(ref base) = base_clone { + current_index = Self::init_pos_args(self_obj, base, args, kwargs, current_index, vm)?; + } - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; + // 2. Process this class's _fields_ + if let Some(fields_attr) = type_obj.get_direct_attr(vm.ctx.intern_str("_fields_")) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; - // Read data from address - if address == 0 || size == 0 { - return Err(vm.new_value_error("NULL pointer access".to_owned())); + for field in fields.iter() { + if current_index >= args.len() { + break; + } + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(name) = tuple.first() + && let Some(name_str) = name.downcast_ref::<PyStr>() + { + let field_name = name_str.as_str().to_owned(); + // Check for duplicate in kwargs + if kwargs.contains_key(&field_name) { + return Err(vm.new_type_error(format!( + "duplicate values for field {:?}", + field_name + ))); + } + self_obj.as_object().set_attr( + vm.ctx.intern_str(field_name), + args[current_index].clone(), + vm, + )?; + current_index += 1; + } + } } - let data = unsafe { - let ptr = address as *const u8; - std::slice::from_raw_parts(ptr, size).to_vec() - }; - // Create instance - let cdata = CDataObject::from_bytes(data, None); - Ok(PyCStructure { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), - fields: PyRwLock::new(IndexMap::new()), - } - .into_ref_with_type(vm, cls)? - .into()) + Ok(current_index) } +} - #[pyclassmethod] - fn from_buffer( - cls: PyTypeRef, - source: PyObjectRef, - offset: crate::function::OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult { - use crate::TryFromObject; - use crate::protocol::PyBuffer; - use crate::stdlib::ctypes::_ctypes::size_of; +impl Initializer for PyCStructure { + type Args = FuncArgs; - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); - } - let offset = offset as usize; + fn init(zelf: crate::PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Struct_init: handle positional and keyword arguments + let cls = zelf.class().to_owned(); - // Get buffer from source - let buffer = PyBuffer::try_from_object(vm, source.clone())?; + // 1. Process positional arguments recursively through inheritance chain + if !args.args.is_empty() { + let consumed = + PyCStructure::init_pos_args(&zelf, &cls, &args.args, &args.kwargs, 0, vm)?; - // Check if buffer is writable - if buffer.desc.readonly { - return Err(vm.new_type_error("underlying buffer is not writable".to_owned())); + if consumed < args.args.len() { + return Err(vm.new_type_error("too many initializers")); + } } - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; - - // Check if buffer is large enough - let buffer_len = buffer.desc.len; - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + // 2. Process keyword arguments + for (key, value) in args.kwargs.iter() { + zelf.as_object() + .set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; } - // Read bytes from buffer at offset - let bytes = buffer.obj_bytes(); - let data = bytes[offset..offset + size].to_vec(); - - // Create instance - let cdata = CDataObject::from_bytes(data, Some(source)); - Ok(PyCStructure { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), - fields: PyRwLock::new(IndexMap::new()), - } - .into_ref_with_type(vm, cls)? - .into()) + Ok(()) } +} - #[pyclassmethod] - fn from_buffer_copy( - cls: PyTypeRef, - source: crate::function::ArgBytesLike, - offset: crate::function::OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; - - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); - } - let offset = offset as usize; - - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; - - // Borrow bytes from source - let source_bytes = source.borrow_buf(); - let buffer_len = source_bytes.len(); - - // Check if buffer is large enough - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); - } - - // Copy bytes from buffer at offset - let data = source_bytes[offset..offset + size].to_vec(); +// Note: GetAttr and SetAttr are not implemented here. +// Field access is handled by CField descriptors registered on the class. - // Create instance - let cdata = CDataObject::from_bytes(data, None); - Ok(PyCStructure { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), - fields: PyRwLock::new(IndexMap::new()), - } - .into_ref_with_type(vm, cls)? - .into()) +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsBuffer) +)] +impl PyCStructure { + #[pygetset] + fn _b0_(&self) -> Option<PyObjectRef> { + self.0.base.read().clone() } } -static STRUCTURE_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - rustpython_common::lock::PyMappedRwLockReadGuard::map( - rustpython_common::lock::PyRwLockReadGuard::map( - buffer.obj_as::<PyCStructure>().cdata.read(), - |x: &CDataObject| x, - ), - |x: &CDataObject| x.buffer.as_slice(), - ) - .into() - }, - obj_bytes_mut: |buffer| { - rustpython_common::lock::PyMappedRwLockWriteGuard::map( - rustpython_common::lock::PyRwLockWriteGuard::map( - buffer.obj_as::<PyCStructure>().cdata.write(), - |x: &mut CDataObject| x, - ), - |x: &mut CDataObject| x.buffer.as_mut_slice(), - ) - .into() - }, - release: |_| {}, - retain: |_| {}, -}; - impl AsBuffer for PyCStructure { fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { - let buffer_len = zelf.cdata.read().buffer.len(); + let buffer_len = zelf.0.buffer.read().len(); + + // PyCData_NewGetBuffer: use info->format if available, otherwise "B" + let format = zelf + .class() + .stg_info_opt() + .and_then(|info| info.format.clone()) + .unwrap_or_else(|| "B".to_string()); + + // Structure: ndim=0, shape=(), itemsize=struct_size let buf = PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(buffer_len, false), // readonly=false for ctypes - &STRUCTURE_BUFFER_METHODS, + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize: buffer_len, + format: Cow::Owned(format), + dim_desc: vec![], // ndim=0 means empty dim_desc + }, + &CDATA_BUFFER_METHODS, ); Ok(buf) } diff --git a/crates/vm/src/stdlib/ctypes/thunk.rs b/crates/vm/src/stdlib/ctypes/thunk.rs deleted file mode 100644 index 2de2308e1a3..00000000000 --- a/crates/vm/src/stdlib/ctypes/thunk.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! FFI callback (thunk) implementation for ctypes. -//! -//! This module implements CThunkObject which wraps Python callables -//! to be callable from C code via libffi closures. - -use crate::builtins::{PyStr, PyType, PyTypeRef}; -use crate::vm::thread::with_current_vm; -use crate::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; -use libffi::low; -use libffi::middle::{Cif, Closure, CodePtr, Type}; -use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; -use std::ffi::c_void; -use std::fmt::Debug; - -use super::base::ffi_type_from_str; -/// Userdata passed to the libffi callback. -/// This contains everything needed to invoke the Python callable. -pub struct ThunkUserData { - /// The Python callable to invoke - pub callable: PyObjectRef, - /// Argument types for conversion - pub arg_types: Vec<PyTypeRef>, - /// Result type for conversion (None means void) - pub res_type: Option<PyTypeRef>, -} - -/// Get the type code string from a ctypes type -fn get_type_code(ty: &PyTypeRef, vm: &VirtualMachine) -> Option<String> { - ty.get_attr(vm.ctx.intern_str("_type_")) - .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())) -} - -/// Convert a C value to a Python object based on the type code -fn ffi_to_python(ty: &PyTypeRef, ptr: *const c_void, vm: &VirtualMachine) -> PyObjectRef { - let type_code = get_type_code(ty, vm); - // SAFETY: ptr is guaranteed to be valid by libffi calling convention - unsafe { - match type_code.as_deref() { - Some("b") => vm.ctx.new_int(*(ptr as *const i8) as i32).into(), - Some("B") => vm.ctx.new_int(*(ptr as *const u8) as i32).into(), - Some("c") => vm.ctx.new_bytes(vec![*(ptr as *const u8)]).into(), - Some("h") => vm.ctx.new_int(*(ptr as *const i16) as i32).into(), - Some("H") => vm.ctx.new_int(*(ptr as *const u16) as i32).into(), - Some("i") => vm.ctx.new_int(*(ptr as *const i32)).into(), - Some("I") => vm.ctx.new_int(*(ptr as *const u32)).into(), - Some("l") => vm.ctx.new_int(*(ptr as *const libc::c_long)).into(), - Some("L") => vm.ctx.new_int(*(ptr as *const libc::c_ulong)).into(), - Some("q") => vm.ctx.new_int(*(ptr as *const libc::c_longlong)).into(), - Some("Q") => vm.ctx.new_int(*(ptr as *const libc::c_ulonglong)).into(), - Some("f") => vm.ctx.new_float(*(ptr as *const f32) as f64).into(), - Some("d") => vm.ctx.new_float(*(ptr as *const f64)).into(), - Some("P") | Some("z") | Some("Z") => vm.ctx.new_int(ptr as usize).into(), - _ => vm.ctx.none(), - } - } -} - -/// Convert a Python object to a C value and store it at the result pointer -fn python_to_ffi(obj: PyResult, ty: &PyTypeRef, result: *mut c_void, vm: &VirtualMachine) { - let obj = match obj { - Ok(o) => o, - Err(_) => return, // Exception occurred, leave result as-is - }; - - let type_code = get_type_code(ty, vm); - // SAFETY: result is guaranteed to be valid by libffi calling convention - unsafe { - match type_code.as_deref() { - Some("b") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut i8) = i.as_bigint().to_i8().unwrap_or(0); - } - } - Some("B") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); - } - } - Some("c") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); - } - } - Some("h") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut i16) = i.as_bigint().to_i16().unwrap_or(0); - } - } - Some("H") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut u16) = i.as_bigint().to_u16().unwrap_or(0); - } - } - Some("i") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut i32) = i.as_bigint().to_i32().unwrap_or(0); - } - } - Some("I") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut u32) = i.as_bigint().to_u32().unwrap_or(0); - } - } - Some("l") | Some("q") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut i64) = i.as_bigint().to_i64().unwrap_or(0); - } - } - Some("L") | Some("Q") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut u64) = i.as_bigint().to_u64().unwrap_or(0); - } - } - Some("f") => { - if let Ok(f) = obj.try_float(vm) { - *(result as *mut f32) = f.to_f64() as f32; - } - } - Some("d") => { - if let Ok(f) = obj.try_float(vm) { - *(result as *mut f64) = f.to_f64(); - } - } - Some("P") | Some("z") | Some("Z") => { - if let Ok(i) = obj.try_int(vm) { - *(result as *mut usize) = i.as_bigint().to_usize().unwrap_or(0); - } - } - _ => {} - } - } -} - -/// The callback function that libffi calls when the closure is invoked. -/// This function converts C arguments to Python objects, calls the Python -/// callable, and converts the result back to C. -unsafe extern "C" fn thunk_callback( - _cif: &low::ffi_cif, - result: &mut c_void, - args: *const *const c_void, - userdata: &ThunkUserData, -) { - with_current_vm(|vm| { - // Convert C arguments to Python objects - let py_args: Vec<PyObjectRef> = userdata - .arg_types - .iter() - .enumerate() - .map(|(i, ty)| { - let arg_ptr = unsafe { *args.add(i) }; - ffi_to_python(ty, arg_ptr, vm) - }) - .collect(); - - // Call the Python callable - let py_result = userdata.callable.call(py_args, vm); - - // Convert result back to C type - if let Some(ref res_type) = userdata.res_type { - python_to_ffi(py_result, res_type, result as *mut c_void, vm); - } - }); -} - -/// Holds the closure and userdata together to ensure proper lifetime. -/// The userdata is leaked to create a 'static reference that the closure can use. -struct ThunkData { - #[allow(dead_code)] - closure: Closure<'static>, - /// Raw pointer to the leaked userdata, for cleanup - userdata_ptr: *mut ThunkUserData, -} - -impl Drop for ThunkData { - fn drop(&mut self) { - // SAFETY: We created this with Box::into_raw, so we can reclaim it - unsafe { - drop(Box::from_raw(self.userdata_ptr)); - } - } -} - -/// CThunkObject wraps a Python callable to make it callable from C code. -#[pyclass(name = "CThunkObject", module = "_ctypes")] -#[derive(PyPayload)] -pub struct PyCThunk { - /// The Python callable - callable: PyObjectRef, - /// The libffi closure (must be kept alive) - #[allow(dead_code)] - thunk_data: PyRwLock<Option<ThunkData>>, - /// The code pointer for the closure - code_ptr: CodePtr, -} - -impl Debug for PyCThunk { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyCThunk") - .field("callable", &self.callable) - .finish() - } -} - -impl PyCThunk { - /// Create a new thunk wrapping a Python callable. - /// - /// # Arguments - /// * `callable` - The Python callable to wrap - /// * `arg_types` - Optional sequence of argument types - /// * `res_type` - Optional result type - /// * `vm` - The virtual machine - pub fn new( - callable: PyObjectRef, - arg_types: Option<PyObjectRef>, - res_type: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - // Parse argument types - let arg_type_vec: Vec<PyTypeRef> = if let Some(args) = arg_types { - if vm.is_none(&args) { - Vec::new() - } else { - let mut types = Vec::new(); - for item in args.try_to_value::<Vec<PyObjectRef>>(vm)? { - types.push(item.downcast::<PyType>().map_err(|_| { - vm.new_type_error("_argtypes_ must be a sequence of types".to_string()) - })?); - } - types - } - } else { - Vec::new() - }; - - // Parse result type - let res_type_ref: Option<PyTypeRef> = - if let Some(ref rt) = res_type { - if vm.is_none(rt) { - None - } else { - Some(rt.clone().downcast::<PyType>().map_err(|_| { - vm.new_type_error("restype must be a ctypes type".to_string()) - })?) - } - } else { - None - }; - - // Build FFI types - let ffi_arg_types: Vec<Type> = arg_type_vec - .iter() - .map(|ty| { - get_type_code(ty, vm) - .and_then(|code| ffi_type_from_str(&code)) - .unwrap_or(Type::pointer()) - }) - .collect(); - - let ffi_res_type = res_type_ref - .as_ref() - .and_then(|ty| get_type_code(ty, vm)) - .and_then(|code| ffi_type_from_str(&code)) - .unwrap_or(Type::void()); - - // Create the CIF - let cif = Cif::new(ffi_arg_types, ffi_res_type); - - // Create userdata and leak it to get a 'static reference - let userdata = Box::new(ThunkUserData { - callable: callable.clone(), - arg_types: arg_type_vec, - res_type: res_type_ref, - }); - let userdata_ptr = Box::into_raw(userdata); - - // SAFETY: We maintain the userdata lifetime by storing it in ThunkData - // and cleaning it up in Drop - let userdata_ref: &'static ThunkUserData = unsafe { &*userdata_ptr }; - - // Create the closure - let closure = Closure::new(cif, thunk_callback, userdata_ref); - - // Get the code pointer - let code_ptr = CodePtr(*closure.code_ptr() as *mut _); - - // Store closure and userdata together - let thunk_data = ThunkData { - closure, - userdata_ptr, - }; - - Ok(Self { - callable, - thunk_data: PyRwLock::new(Some(thunk_data)), - code_ptr, - }) - } - - /// Get the code pointer for this thunk - pub fn code_ptr(&self) -> CodePtr { - self.code_ptr - } -} - -// SAFETY: PyCThunk is safe to send/sync because: -// - callable is a PyObjectRef which is Send+Sync -// - thunk_data contains the libffi closure which is heap-allocated -// - code_ptr is just a pointer to executable memory -unsafe impl Send for PyCThunk {} -unsafe impl Sync for PyCThunk {} - -#[pyclass] -impl PyCThunk { - #[pygetset] - fn callable(&self) -> PyObjectRef { - self.callable.clone() - } -} diff --git a/crates/vm/src/stdlib/ctypes/union.rs b/crates/vm/src/stdlib/ctypes/union.rs index e6873e87506..fba9a75e955 100644 --- a/crates/vm/src/stdlib/ctypes/union.rs +++ b/crates/vm/src/stdlib/ctypes/union.rs @@ -1,40 +1,61 @@ -use super::base::{CDataObject, PyCData}; -use super::field::PyCField; -use super::util::StgInfo; +use super::base::{CDATA_BUFFER_METHODS, StgInfoFlags}; +use super::{PyCData, PyCField, StgInfo}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::protocol::{BufferDescriptor, BufferMethods, PyBuffer as ProtocolPyBuffer}; -use crate::stdlib::ctypes::_ctypes::get_size; -use crate::types::{AsBuffer, Constructor}; +use crate::function::{ArgBytesLike, FuncArgs, OptionalArg, PySetterValue}; +use crate::protocol::{BufferDescriptor, PyBuffer}; +use crate::stdlib::warnings; +use crate::types::{AsBuffer, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use alloc::borrow::Cow; use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; + +/// Calculate Union type size from _fields_ (max field size) +pub(super) fn calculate_union_size(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<usize> { + if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + let mut max_size = 0usize; + + for field in fields.iter() { + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(field_type) = tuple.get(1) + { + let field_size = super::_ctypes::sizeof(field_type.clone(), vm)?; + max_size = max_size.max(field_size); + } + } + return Ok(max_size); + } + Ok(0) +} /// PyCUnionType - metaclass for Union #[pyclass(name = "UnionType", base = PyType, module = "_ctypes")] #[derive(Debug)] #[repr(transparent)] -pub struct PyCUnionType(PyType); +pub(super) struct PyCUnionType(PyType); impl Constructor for PyCUnionType { type Args = FuncArgs; fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // 1. Create the new class using PyType::py_new - let new_class = crate::builtins::type_::PyType::slot_new(metatype, args, vm)?; + // 1. Create the new class using PyType::slot_new + let new_class = crate::builtins::PyType::slot_new(metatype, args, vm)?; - // 2. Process _fields_ if defined on the new class + // 2. Get the new type let new_type = new_class .clone() .downcast::<PyType>() .map_err(|_| vm.new_type_error("expected type"))?; - // Only process _fields_ if defined directly on this class (not inherited) - if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { - Self::process_fields(&new_type, fields_attr, vm)?; - } + // 3. Mark base classes as finalized (subclassing finalizes the parent) + new_type.mark_bases_final(); + + // 4. Initialize StgInfo for the new type (initialized=false, to be set in init) + let stg_info = StgInfo::default(); + let _ = new_type.init_type_data(stg_info); + // Note: _fields_ processing moved to Initializer::init() Ok(new_class) } @@ -43,26 +64,149 @@ impl Constructor for PyCUnionType { } } +impl Initializer for PyCUnionType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef by converting PyRef<Self> -> PyObjectRef -> PyRef<PyType> + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + // Check for _abstract_ attribute - skip initialization if present + if new_type + .get_direct_attr(vm.ctx.intern_str("_abstract_")) + .is_some() + { + return Ok(()); + } + + new_type.check_not_initialized(vm)?; + + // Process _fields_ if defined directly on this class (not inherited) + // Use set_attr to trigger setattro + if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { + new_type + .as_object() + .set_attr(vm.ctx.intern_str("_fields_"), fields_attr, vm)?; + } else { + // No _fields_ defined - try to copy from base class + let (has_base_info, base_clone) = { + let bases = new_type.bases.read(); + if let Some(base) = bases.first() { + (base.stg_info_opt().is_some(), Some(base.clone())) + } else { + (false, None) + } + }; + + if has_base_info && let Some(ref base) = base_clone { + // Clone base StgInfo (release guard before getting mutable reference) + let stg_info_opt = base.stg_info_opt().map(|baseinfo| { + let mut stg_info = baseinfo.clone(); + stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL flag in subclass + stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable + stg_info + }); + + if let Some(stg_info) = stg_info_opt { + // Mark base as FINAL (now guard is released) + if let Some(mut base_stg) = base.get_type_data_mut::<StgInfo>() { + base_stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + super::base::set_or_init_stginfo(&new_type, stg_info); + return Ok(()); + } + } + + // No base StgInfo - create default + let mut stg_info = StgInfo::new(0, 1); + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASUNION; + stg_info.paramfunc = super::base::ParamFunc::Union; + // PEP 3118 doesn't support union. Use 'B' for bytes. + stg_info.format = Some("B".to_string()); + super::base::set_or_init_stginfo(&new_type, stg_info); + } + + Ok(()) + } +} + impl PyCUnionType { /// Process _fields_ and create CField descriptors /// For Union, all fields start at offset 0 fn process_fields( - cls: &PyTypeRef, + cls: &Py<PyType>, fields_attr: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { + // Check if already finalized + { + let Some(stg_info) = cls.get_type_data::<StgInfo>() else { + return Err(vm.new_type_error("ctypes state is not initialized")); + }; + if stg_info.is_final() { + return Err(vm.new_attribute_error("_fields_ is final")); + } + } // Read lock released here + + // Check if this is a swapped byte order union + let is_swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() { list.borrow_vec().to_vec() } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { tuple.to_vec() } else { - return Err(vm.new_type_error("_fields_ must be a list or tuple".to_string())); + return Err(vm.new_type_error("_fields_ must be a list or tuple")); + }; + + let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let msg = format!( + "Due to '_pack_', the '{}' Union will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + + let forced_alignment = + super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); + + // Initialize size, alignment, type flags, and ffi_field_types from base class + // Note: Union fields always start at offset 0, but we inherit base size/align + let (mut max_size, mut max_align, mut has_pointer, mut has_bitfield, mut ffi_field_types) = { + let bases = cls.bases.read(); + if let Some(base) = bases.first() + && let Some(baseinfo) = base.stg_info_opt() + { + ( + baseinfo.size, + core::cmp::max(baseinfo.align, forced_alignment), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), + baseinfo.ffi_field_types.clone(), + ) + } else { + (0, forced_alignment, false, false, Vec::new()) + } }; for (index, field) in fields.iter().enumerate() { let field_tuple = field .downcast_ref::<PyTuple>() - .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples".to_string()))?; + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; if field_tuple.len() < 2 { return Err(vm.new_type_error( @@ -72,263 +216,486 @@ impl PyCUnionType { let name = field_tuple .first() - .unwrap() + .expect("len checked") .downcast_ref::<PyStr>() - .ok_or_else(|| vm.new_type_error("field name must be a string".to_string()))? + .ok_or_else(|| vm.new_type_error("field name must be a string"))? .to_string(); - let field_type = field_tuple.get(1).unwrap().clone(); - let size = Self::get_field_size(&field_type, vm)?; + let field_type = field_tuple.get(1).expect("len checked").clone(); + + // For swapped byte order unions, validate field type supports byte swapping + if is_swapped { + super::base::check_other_endian_support(&field_type, vm)?; + } + + let size = super::base::get_field_size(&field_type, vm)?; + let field_align = super::base::get_field_align(&field_type, vm); + + // Calculate effective alignment + let effective_align = if pack > 0 { + core::cmp::min(pack, field_align) + } else { + field_align + }; + + max_size = max_size.max(size); + max_align = max_align.max(effective_align); + + // Propagate type flags from field type (HASPOINTER, HASBITFIELD) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + { + // HASPOINTER: propagate if field is pointer or contains pointer + if field_stg.flags.intersects( + StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER, + ) { + has_pointer = true; + } + // HASBITFIELD: propagate directly + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD) { + has_bitfield = true; + } + // Collect FFI type for this field + ffi_field_types.push(field_stg.to_ffi_type()); + } + + // Mark field type as finalized (using type as field finalizes it) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() { + if let Some(mut stg_info) = type_obj.get_type_data_mut::<StgInfo>() { + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + // Create StgInfo with FINAL flag if it doesn't exist + let mut stg_info = StgInfo::new(size, field_align); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = type_obj.init_type_data(stg_info); + } + } // For Union, all fields start at offset 0 - // Create CField descriptor (accepts any ctypes type including arrays) - let c_field = PyCField::new(name.clone(), field_type, 0, size, index); + let field_type_ref = field_type + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; + + // Check for bitfield size (optional 3rd element in tuple) + // For unions, each field starts fresh (CPython: _layout.py) + let c_field = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + // Union fields all start at offset 0, so bit_offset = 0 + let mut bit_offset: u16 = 0; + let type_bits = (size * 8) as u16; + + // Big-endian: bit_offset = type_bits - bit_size + let big_endian = is_swapped != cfg!(target_endian = "big"); + if big_endian && type_bits >= bit_size { + bit_offset = type_bits - bit_size; + } + + PyCField::new_bitfield( + name.clone(), + field_type_ref, + 0, // Union fields always at offset 0 + size as isize, + bit_size, + bit_offset, + index, + ) + } else { + PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) + }; cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); } - Ok(()) - } + // Calculate total_align and aligned_size + let total_align = core::cmp::max(max_align, forced_alignment); + let aligned_size = if total_align > 0 { + max_size.div_ceil(total_align) * total_align + } else { + max_size + }; - fn get_field_size(field_type: &PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - if let Some(size) = field_type - .get_attr("_type_", vm) - .ok() - .and_then(|type_attr| type_attr.str(vm).ok()) - .and_then(|type_str| { - let s = type_str.to_string(); - (s.len() == 1).then(|| get_size(&s)) - }) + // Check for circular self-reference + if let Some(stg_info) = cls.get_type_data::<StgInfo>() + && stg_info.is_final() { - return Ok(size); + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); } - if let Some(s) = field_type - .get_attr("size_of_instances", vm) - .ok() - .and_then(|size_method| size_method.call((), vm).ok()) - .and_then(|size| size.try_int(vm).ok()) - .and_then(|n| n.as_bigint().to_usize()) - { - return Ok(s); + // Store StgInfo with aligned size + let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL | StgInfoFlags::TYPEFLAG_HASUNION; + // PEP 3118 doesn't support union. Use 'B' for bytes. + stg_info.format = Some("B".to_string()); + if has_pointer { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASPOINTER; } + if has_bitfield { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASBITFIELD; + } + stg_info.paramfunc = super::base::ParamFunc::Union; + // Set byte order: swap if _swappedbytes_ is defined + stg_info.big_endian = super::base::is_big_endian(is_swapped); + // Store FFI field types for union passing + stg_info.ffi_field_types = ffi_field_types; + super::base::set_or_init_stginfo(cls, stg_info); - Ok(std::mem::size_of::<usize>()) - } -} - -#[pyclass(flags(BASETYPE), with(Constructor))] -impl PyCUnionType {} + // Process _anonymous_ fields + super::base::make_anon_fields(cls, vm)?; -/// PyCUnion - base class for Union -#[pyclass(module = "_ctypes", name = "Union", base = PyCData, metaclass = "PyCUnionType")] -pub struct PyCUnion { - _base: PyCData, - /// Common CDataObject for memory buffer - pub(super) cdata: PyRwLock<CDataObject>, + Ok(()) + } } -impl std::fmt::Debug for PyCUnion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyCUnion") - .field("size", &self.cdata.read().size()) - .finish() +#[pyclass(flags(BASETYPE), with(Constructor, Initializer, SetAttr))] +impl PyCUnionType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) } -} -impl Constructor for PyCUnion { - type Args = FuncArgs; + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // Get _fields_ from the class - let fields_attr = cls.as_object().get_attr("_fields_", vm).ok(); + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the union type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; - // Calculate union size (max of all field sizes) and alignment - let mut max_size = 0usize; - let mut max_align = 1usize; + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } - if let Some(fields_attr) = fields_attr { - let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() + // 2. Check for CArgObject (PyCArg_CheckExact) + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() { + // Check against proto (for pointer types) + if let Some(stg_info) = cls.stg_info_opt() + && let Some(ref proto) = stg_info.proto + && carg.obj.is_instance(proto.as_object(), vm)? { - list.borrow_vec().to_vec() - } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { - tuple.to_vec() - } else { - vec![] - }; - - for field in fields.iter() { - let Some(field_tuple) = field.downcast_ref::<PyTuple>() else { - continue; - }; - if field_tuple.len() < 2 { - continue; - } - let field_type = field_tuple.get(1).unwrap().clone(); - let size = PyCUnionType::get_field_size(&field_type, vm)?; - max_size = max_size.max(size); - // For simple types, alignment == size - max_align = max_align.max(size); + return Ok(value); + } + // Fallback: check if the wrapped object is an instance of the requested type + if carg.obj.is_instance(cls.as_object(), vm)? { + return Ok(value); // Return the CArgObject as-is } + // CArgObject but wrong type + return Err(vm.new_type_error(format!( + "expected {} instance instead of pointer to {}", + cls.name(), + carg.obj.class().name() + ))); } - // Initialize buffer with zeros - let stg_info = StgInfo::new(max_size, max_align); - let cdata = CDataObject::from_stg_info(&stg_info); - PyCUnion { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), + // 3. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCUnionType::from_param(cls.as_object().to_owned(), as_parameter, vm); } - .into_ref_with_type(vm, cls) - .map(Into::into) + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) } - fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("use slot_new") + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) } -} -#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor, AsBuffer))] -impl PyCUnion { - #[pygetset] - fn _objects(&self) -> Option<PyObjectRef> { - self.cdata.read().objects.clone() + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) } - #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } - // Get size from cls - let size = size_of(cls.clone().into(), vm)?; + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } - // Create instance with data from address - if address == 0 || size == 0 { - return Err(vm.new_value_error("NULL pointer access".to_owned())); - } - let stg_info = StgInfo::new(size, 1); - let cdata = CDataObject::from_stg_info(&stg_info); - Ok(PyCUnion { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), + /// Called when a new Union subclass is created + #[pyclassmethod] + fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + cls.mark_bases_final(); + + // Check if _fields_ is defined + if let Some(fields_attr) = cls.get_direct_attr(vm.ctx.intern_str("_fields_")) { + Self::process_fields(&cls, fields_attr, vm)?; } - .into_ref_with_type(vm, cls)? - .into()) + Ok(()) } +} - #[pyclassmethod] - fn from_buffer( - cls: PyTypeRef, - source: PyObjectRef, - offset: crate::function::OptionalArg<isize>, +impl SetAttr for PyCUnionType { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, vm: &VirtualMachine, - ) -> PyResult { - use crate::TryFromObject; - use crate::protocol::PyBuffer; - use crate::stdlib::ctypes::_ctypes::size_of; - - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); + ) -> PyResult<()> { + let pytype: &Py<PyType> = zelf.to_base(); + let attr_name_interned = vm.ctx.intern_str(attr_name.as_str()); + + // 1. First, do PyType's setattro (PyType_Type.tp_setattro first) + // Check for data descriptor first + if let Some(attr) = pytype.get_class_attr(attr_name_interned) { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + descriptor(&attr, pytype.to_owned().into(), value.clone(), vm)?; + // After successful setattro, check if _fields_ and call process_fields + if attr_name.as_str() == "_fields_" + && let PySetterValue::Assign(fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value, vm)?; + } + return Ok(()); + } } - let offset = offset as usize; - let buffer = PyBuffer::try_from_object(vm, source.clone())?; + // 2. If _fields_, call process_fields (which checks FINAL internally) + // Check BEFORE writing to dict to avoid storing _fields_ when FINAL + if attr_name.as_str() == "_fields_" + && let PySetterValue::Assign(ref fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value.clone(), vm)?; + } - if buffer.desc.readonly { - return Err(vm.new_type_error("underlying buffer is not writable".to_owned())); + // Store in type's attributes dict + match &value { + PySetterValue::Assign(v) => { + pytype + .attributes + .write() + .insert(attr_name_interned, v.clone()); + } + PySetterValue::Delete => { + let prev = pytype.attributes.write().shift_remove(attr_name_interned); + if prev.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + pytype.name(), + attr_name.as_str(), + ))); + } + } } - let size = size_of(cls.clone().into(), vm)?; - let buffer_len = buffer.desc.len; + Ok(()) + } +} + +/// PyCUnion - base class for Union +#[pyclass(module = "_ctypes", name = "Union", base = PyCData, metaclass = "PyCUnionType")] +#[repr(transparent)] +pub struct PyCUnion(pub PyCData); - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); - } +impl core::fmt::Debug for PyCUnion { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCUnion") + .field("size", &self.0.size()) + .finish() + } +} - // Copy data from source buffer - let bytes = buffer.obj_bytes(); - let data = bytes[offset..offset + size].to_vec(); +impl Constructor for PyCUnion { + type Args = FuncArgs; - let cdata = CDataObject::from_bytes(data, None); - Ok(PyCUnion { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Check for abstract class and extract values in a block to drop the borrow + let (total_size, total_align, length) = { + let stg_info = cls.stg_info(vm)?; + (stg_info.size, stg_info.align, stg_info.length) + }; + + // Mark the class as finalized (instance creation finalizes the type) + if let Some(mut stg_info_mut) = cls.get_type_data_mut::<StgInfo>() { + stg_info_mut.flags |= StgInfoFlags::DICTFLAG_FINAL; } - .into_ref_with_type(vm, cls)? - .into()) + + // Initialize buffer with zeros using computed size + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; + PyCUnion(PyCData::from_stg_info(&new_stg_info)) + .into_ref_with_type(vm, cls) + .map(Into::into) } - #[pyclassmethod] - fn from_buffer_copy( - cls: PyTypeRef, - source: crate::function::ArgBytesLike, - offset: crate::function::OptionalArg<isize>, + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl PyCUnion { + /// Recursively initialize positional arguments through inheritance chain + /// Returns the number of arguments consumed + fn init_pos_args( + self_obj: &Py<Self>, + type_obj: &Py<PyType>, + args: &[PyObjectRef], + kwargs: &indexmap::IndexMap<String, PyObjectRef>, + index: usize, vm: &VirtualMachine, - ) -> PyResult { - use crate::stdlib::ctypes::_ctypes::size_of; + ) -> PyResult<usize> { + let mut current_index = index; + + // 1. First process base class fields recursively + // Recurse if base has StgInfo + let base_clone = { + let bases = type_obj.bases.read(); + if let Some(base) = bases.first() && + // Check if base has StgInfo + base.stg_info_opt().is_some() + { + Some(base.clone()) + } else { + None + } + }; - let offset = offset.unwrap_or(0); - if offset < 0 { - return Err(vm.new_value_error("offset cannot be negative".to_owned())); + if let Some(ref base) = base_clone { + current_index = Self::init_pos_args(self_obj, base, args, kwargs, current_index, vm)?; } - let offset = offset as usize; - let size = size_of(cls.clone().into(), vm)?; - let source_bytes = source.borrow_buf(); - let buffer_len = source_bytes.len(); + // 2. Process this class's _fields_ + if let Some(fields_attr) = type_obj.get_direct_attr(vm.ctx.intern_str("_fields_")) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; - if offset + size > buffer_len { - return Err(vm.new_value_error(format!( - "Buffer size too small ({} instead of at least {} bytes)", - buffer_len, - offset + size - ))); + for field in fields.iter() { + if current_index >= args.len() { + break; + } + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(name) = tuple.first() + && let Some(name_str) = name.downcast_ref::<PyStr>() + { + let field_name = name_str.as_str().to_owned(); + // Check for duplicate in kwargs + if kwargs.contains_key(&field_name) { + return Err(vm.new_type_error(format!( + "duplicate values for field {:?}", + field_name + ))); + } + self_obj.as_object().set_attr( + vm.ctx.intern_str(field_name), + args[current_index].clone(), + vm, + )?; + current_index += 1; + } + } } - // Copy data from source - let data = source_bytes[offset..offset + size].to_vec(); + Ok(current_index) + } +} + +impl Initializer for PyCUnion { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Struct_init: handle positional and keyword arguments + let cls = zelf.class().to_owned(); + + // 1. Process positional arguments recursively through inheritance chain + if !args.args.is_empty() { + let consumed = PyCUnion::init_pos_args(&zelf, &cls, &args.args, &args.kwargs, 0, vm)?; - let cdata = CDataObject::from_bytes(data, None); - Ok(PyCUnion { - _base: PyCData::new(cdata.clone()), - cdata: PyRwLock::new(cdata), + if consumed < args.args.len() { + return Err(vm.new_type_error("too many initializers")); + } + } + + // 2. Process keyword arguments + for (key, value) in args.kwargs.iter() { + zelf.as_object() + .set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; } - .into_ref_with_type(vm, cls)? - .into()) + + Ok(()) } } -static UNION_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - rustpython_common::lock::PyRwLockReadGuard::map( - buffer.obj_as::<PyCUnion>().cdata.read(), - |x: &CDataObject| x.buffer.as_slice(), - ) - .into() - }, - obj_bytes_mut: |buffer| { - rustpython_common::lock::PyRwLockWriteGuard::map( - buffer.obj_as::<PyCUnion>().cdata.write(), - |x: &mut CDataObject| x.buffer.as_mut_slice(), - ) - .into() - }, - release: |_| {}, - retain: |_| {}, -}; +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsBuffer) +)] +impl PyCUnion {} impl AsBuffer for PyCUnion { - fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<ProtocolPyBuffer> { - let buffer_len = zelf.cdata.read().buffer.len(); - let buf = ProtocolPyBuffer::new( + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let buffer_len = zelf.0.buffer.read().len(); + + // PyCData_NewGetBuffer: use info->format if available, otherwise "B" + let format = zelf + .class() + .stg_info_opt() + .and_then(|info| info.format.clone()) + .unwrap_or_else(|| "B".to_string()); + + // Union: ndim=0, shape=(), itemsize=union_size + let buf = PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(buffer_len, false), // readonly=false for ctypes - &UNION_BUFFER_METHODS, + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize: buffer_len, + format: Cow::Owned(format), + dim_desc: vec![], // ndim=0 means empty dim_desc + }, + &CDATA_BUFFER_METHODS, ); Ok(buf) } diff --git a/crates/vm/src/stdlib/ctypes/util.rs b/crates/vm/src/stdlib/ctypes/util.rs deleted file mode 100644 index b8c6def63ca..00000000000 --- a/crates/vm/src/stdlib/ctypes/util.rs +++ /dev/null @@ -1,88 +0,0 @@ -use crate::PyObjectRef; - -/// Storage information for ctypes types -/// Stored in TypeDataSlot of heap types (PyType::init_type_data/get_type_data) -#[derive(Clone)] -pub struct StgInfo { - pub initialized: bool, - pub size: usize, // number of bytes - pub align: usize, // alignment requirements - pub length: usize, // number of fields (for arrays/structures) - pub proto: Option<PyObjectRef>, // Only for Pointer/ArrayObject - pub flags: i32, // calling convention and such - - // Array-specific fields (moved from PyCArrayType) - pub element_type: Option<PyObjectRef>, // _type_ for arrays - pub element_size: usize, // size of each element -} - -// StgInfo is stored in type_data which requires Send + Sync. -// The PyObjectRef in proto/element_type fields is protected by the type system's locking mechanism. -// CPython: ctypes objects are not thread-safe by design; users must synchronize access. -unsafe impl Send for StgInfo {} -unsafe impl Sync for StgInfo {} - -impl std::fmt::Debug for StgInfo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("StgInfo") - .field("initialized", &self.initialized) - .field("size", &self.size) - .field("align", &self.align) - .field("length", &self.length) - .field("proto", &self.proto) - .field("flags", &self.flags) - .field("element_type", &self.element_type) - .field("element_size", &self.element_size) - .finish() - } -} - -impl Default for StgInfo { - fn default() -> Self { - StgInfo { - initialized: false, - size: 0, - align: 1, - length: 0, - proto: None, - flags: 0, - element_type: None, - element_size: 0, - } - } -} - -impl StgInfo { - pub fn new(size: usize, align: usize) -> Self { - StgInfo { - initialized: true, - size, - align, - length: 0, - proto: None, - flags: 0, - element_type: None, - element_size: 0, - } - } - - /// Create StgInfo for an array type - pub fn new_array( - size: usize, - align: usize, - length: usize, - element_type: PyObjectRef, - element_size: usize, - ) -> Self { - StgInfo { - initialized: true, - size, - align, - length, - proto: None, - flags: 0, - element_type: Some(element_type), - element_size, - } - } -} diff --git a/crates/vm/src/stdlib/errno.rs b/crates/vm/src/stdlib/errno.rs index 7a78ceaea83..8e4efbaafe9 100644 --- a/crates/vm/src/stdlib/errno.rs +++ b/crates/vm/src/stdlib/errno.rs @@ -1,25 +1,26 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use errno_mod::module_def; -#[pymodule] -mod errno {} +#[pymodule(name = "errno")] +mod errno_mod { + use crate::{Py, PyResult, VirtualMachine, builtins::PyModule}; -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = errno::make_module(vm); - let errorcode = vm.ctx.new_dict(); - extend_module!(vm, &module, { - "errorcode" => errorcode.clone(), - }); - for (name, code) in ERROR_CODES { - let name = vm.ctx.intern_str(*name); - let code = vm.new_pyobj(*code); - errorcode - .set_item(&*code, name.to_owned().into(), vm) - .unwrap(); - module.set_attr(name, code, vm).unwrap(); + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + let errorcode = vm.ctx.new_dict(); + extend_module!(vm, module, { + "errorcode" => errorcode.clone(), + }); + for (name, code) in super::ERROR_CODES { + let name = vm.ctx.intern_str(*name); + let code = vm.new_pyobj(*code); + errorcode.set_item(&*code, name.to_owned().into(), vm)?; + module.set_attr(name, code, vm)?; + } + Ok(()) } - module } #[cfg(any(unix, windows, target_os = "wasi"))] diff --git a/crates/vm/src/stdlib/functools.rs b/crates/vm/src/stdlib/functools.rs index d5a42739e96..6c5c8f2e4c5 100644 --- a/crates/vm/src/stdlib/functools.rs +++ b/crates/vm/src/stdlib/functools.rs @@ -1,31 +1,45 @@ -pub(crate) use _functools::make_module; +pub(crate) use _functools::module_def; #[pymodule] mod _functools { use crate::{ Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyDict, PyGenericAlias, PyTuple, PyTypeRef}, + builtins::{PyBoundMethod, PyDict, PyGenericAlias, PyTuple, PyType, PyTypeRef}, common::lock::PyRwLock, - function::{FuncArgs, KwArgs, OptionalArg}, + function::{FuncArgs, KwArgs, OptionalOption}, object::AsObject, protocol::PyIter, pyclass, recursion::ReprGuard, - types::{Callable, Constructor, Representable}, + types::{Callable, Constructor, GetDescriptor, Representable}, }; use indexmap::IndexMap; - #[pyfunction] - fn reduce( + #[derive(FromArgs)] + struct ReduceArgs { function: PyObjectRef, iterator: PyIter, - start_value: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { + #[pyarg(any, optional, name = "initial")] + initial: OptionalOption<PyObjectRef>, + } + + #[pyfunction] + fn reduce(args: ReduceArgs, vm: &VirtualMachine) -> PyResult { + let ReduceArgs { + function, + iterator, + initial, + } = args; let mut iter = iterator.iter_without_hint(vm)?; - let start_value = if let OptionalArg::Present(val) = start_value { - val + // OptionalOption distinguishes between: + // - Missing: no argument provided → use first element from iterator + // - Present(None): explicitly passed None → use None as initial value + // - Present(Some(v)): passed a value → use that value + let start_value = if let Some(val) = initial.into_option() { + // initial was provided (could be None or Some value) + val.unwrap_or_else(|| vm.ctx.none()) } else { + // initial was not provided at all iter.next().transpose()?.ok_or_else(|| { let exc_type = vm.ctx.exceptions.type_error.to_owned(); vm.new_exception_msg( @@ -42,8 +56,74 @@ mod _functools { Ok(accumulator) } + // Placeholder singleton for partial arguments + // The singleton is stored as _instance on the type class + #[pyattr] + #[allow(non_snake_case)] + fn Placeholder(vm: &VirtualMachine) -> PyObjectRef { + let placeholder = PyPlaceholderType.into_pyobject(vm); + // Store the singleton on the type class for slot_new to find + let typ = placeholder.class(); + typ.set_attr(vm.ctx.intern_str("_instance"), placeholder.clone()); + placeholder + } + #[pyattr] - #[pyclass(name = "partial", module = "_functools")] + #[pyclass(name = "_PlaceholderType", module = "functools")] + #[derive(Debug, PyPayload)] + pub struct PyPlaceholderType; + + impl Constructor for PyPlaceholderType { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("_PlaceholderType takes no arguments".to_owned())); + } + // Return the singleton stored on the type class + if let Some(instance) = cls.get_attr(vm.ctx.intern_str("_instance")) { + return Ok(instance); + } + // Fallback: create a new instance (shouldn't happen for base type after module init) + Ok(PyPlaceholderType.into_pyobject(vm)) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + // This is never called because we override slot_new + Ok(PyPlaceholderType) + } + } + + #[pyclass(with(Constructor, Representable))] + impl PyPlaceholderType { + #[pymethod] + fn __reduce__(&self) -> &'static str { + "Placeholder" + } + + #[pymethod] + fn __init_subclass__(_cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("cannot subclass '_PlaceholderType'".to_owned())) + } + } + + impl Representable for PyPlaceholderType { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("Placeholder".to_owned()) + } + } + + fn is_placeholder(obj: &PyObjectRef) -> bool { + &*obj.class().name() == "_PlaceholderType" + } + + fn count_placeholders(args: &[PyObjectRef]) -> usize { + args.iter().filter(|a| is_placeholder(a)).count() + } + + #[pyattr] + #[pyclass(name = "partial", module = "functools")] #[derive(Debug, PyPayload)] pub struct PyPartial { inner: PyRwLock<PyPartialInner>, @@ -54,9 +134,13 @@ mod _functools { func: PyObjectRef, args: PyRef<PyTuple>, keywords: PyRef<PyDict>, + phcount: usize, } - #[pyclass(with(Constructor, Callable, Representable), flags(BASETYPE, HAS_DICT))] + #[pyclass( + with(Constructor, Callable, GetDescriptor, Representable), + flags(BASETYPE, HAS_DICT) + )] impl PyPartial { #[pygetset] fn func(&self) -> PyObjectRef { @@ -73,8 +157,8 @@ mod _functools { self.inner.read().keywords.clone() } - #[pymethod(name = "__reduce__")] - fn reduce(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn __reduce__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { let inner = zelf.inner.read(); let partial_type = zelf.class(); @@ -157,6 +241,13 @@ mod _functools { } }; + // Validate no trailing placeholders + let args_slice = args_tuple.as_slice(); + if !args_slice.is_empty() && is_placeholder(args_slice.last().unwrap()) { + return Err(vm.new_type_error("trailing Placeholders are not allowed".to_owned())); + } + let phcount = count_placeholders(args_slice); + // Actually update the state let mut inner = zelf.inner.write(); inner.func = func.clone(); @@ -165,6 +256,7 @@ mod _functools { // Handle keywords - keep the original type inner.keywords = keywords_dict; + inner.phcount = phcount; // Update __dict__ if provided let Some(instance_dict) = zelf.as_object().dict() else { @@ -218,17 +310,54 @@ mod _functools { return Err(vm.new_type_error("the first argument must be callable")); } + // Check for placeholders in kwargs + for (key, value) in &args.kwargs { + if is_placeholder(value) { + return Err(vm.new_type_error(format!( + "Placeholder cannot be passed as a keyword argument to partial(). \ + Did you mean partial(..., {}=Placeholder, ...)(value)?", + key + ))); + } + } + // Handle nested partial objects let (final_func, final_args, final_keywords) = if let Some(partial) = func.downcast_ref::<Self>() { let inner = partial.inner.read(); - let mut combined_args = inner.args.as_slice().to_vec(); - combined_args.extend_from_slice(args_slice); - (inner.func.clone(), combined_args, inner.keywords.clone()) + let stored_args = inner.args.as_slice(); + + // Merge placeholders: replace placeholders in stored_args with new args + let mut merged_args = Vec::with_capacity(stored_args.len() + args_slice.len()); + let mut new_args_iter = args_slice.iter(); + + for stored_arg in stored_args { + if is_placeholder(stored_arg) { + // Replace placeholder with next new arg, or keep placeholder + if let Some(new_arg) = new_args_iter.next() { + merged_args.push(new_arg.clone()); + } else { + merged_args.push(stored_arg.clone()); + } + } else { + merged_args.push(stored_arg.clone()); + } + } + // Append remaining new args + merged_args.extend(new_args_iter.cloned()); + + (inner.func.clone(), merged_args, inner.keywords.clone()) } else { (func.clone(), args_slice.to_vec(), vm.ctx.new_dict()) }; + // Trailing placeholders are not allowed + if !final_args.is_empty() && is_placeholder(final_args.last().unwrap()) { + return Err(vm.new_type_error("trailing Placeholders are not allowed".to_owned())); + } + + let phcount = count_placeholders(&final_args); + // Add new keywords for (key, value) in args.kwargs { final_keywords.set_item(vm.ctx.intern_str(key.as_str()), value, vm)?; @@ -239,6 +368,7 @@ mod _functools { func: final_func, args: vm.ctx.new_tuple(final_args), keywords: final_keywords, + phcount, }), }) } @@ -248,15 +378,51 @@ mod _functools { type Args = FuncArgs; fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let inner = zelf.inner.read(); - let mut combined_args = inner.args.as_slice().to_vec(); - combined_args.extend_from_slice(&args.args); + // Clone and release lock before calling Python code to prevent deadlock + let (func, stored_args, keywords, phcount) = { + let inner = zelf.inner.read(); + ( + inner.func.clone(), + inner.args.clone(), + inner.keywords.clone(), + inner.phcount, + ) + }; + + // Check if we have enough args to fill placeholders + if phcount > 0 && args.args.len() < phcount { + return Err(vm.new_type_error(format!( + "missing positional arguments in 'partial' call; expected at least {}, got {}", + phcount, + args.args.len() + ))); + } + + // Build combined args, replacing placeholders + let mut combined_args = Vec::with_capacity(stored_args.len() + args.args.len()); + let mut new_args_iter = args.args.iter(); + + for stored_arg in stored_args.as_slice() { + if is_placeholder(stored_arg) { + // Replace placeholder with next new arg + if let Some(new_arg) = new_args_iter.next() { + combined_args.push(new_arg.clone()); + } else { + // This shouldn't happen if phcount check passed + combined_args.push(stored_arg.clone()); + } + } else { + combined_args.push(stored_arg.clone()); + } + } + // Append remaining new args + combined_args.extend(new_args_iter.cloned()); // Merge keywords from self.keywords and args.kwargs let mut final_kwargs = IndexMap::new(); // Add keywords from self.keywords - for (key, value) in &*inner.keywords { + for (key, value) in &*keywords { let key_str = key .downcast::<crate::builtins::PyStr>() .map_err(|_| vm.new_type_error("keywords must be strings"))?; @@ -268,9 +434,22 @@ mod _functools { final_kwargs.insert(key, value); } - inner - .func - .call(FuncArgs::new(combined_args, KwArgs::new(final_kwargs)), vm) + func.call(FuncArgs::new(combined_args, KwArgs::new(final_kwargs)), vm) + } + } + + impl GetDescriptor for PyPartial { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let obj = match obj { + Some(obj) if !vm.is_none(&obj) => obj, + _ => return Ok(zelf), + }; + Ok(PyBoundMethod::new(obj, zelf).into_ref(&vm.ctx).into()) } } @@ -280,15 +459,24 @@ mod _functools { // Check for recursive repr let obj = zelf.as_object(); if let Some(_guard) = ReprGuard::enter(vm, obj) { - let inner = zelf.inner.read(); - let func_repr = inner.func.repr(vm)?; + // Clone and release lock before calling Python code to prevent deadlock + let (func, args, keywords) = { + let inner = zelf.inner.read(); + ( + inner.func.clone(), + inner.args.clone(), + inner.keywords.clone(), + ) + }; + + let func_repr = func.repr(vm)?; let mut parts = vec![func_repr.as_str().to_owned()]; - for arg in inner.args.as_slice() { + for arg in args.as_slice() { parts.push(arg.repr(vm)?.as_str().to_owned()); } - for (key, value) in inner.keywords.clone() { + for (key, value) in &*keywords { // For string keys, use them directly without quotes let key_part = if let Ok(s) = key.clone().downcast::<crate::builtins::PyStr>() { s.as_str().to_owned() @@ -303,28 +491,22 @@ mod _functools { )); } - let class_name = zelf.class().name(); + let qualname = zelf.class().__qualname__(vm); + let qualname_str = qualname + .downcast::<crate::builtins::PyStr>() + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| zelf.class().name().to_owned()); let module = zelf.class().__module__(vm); - let qualified_name = if zelf.class().is(Self::class(&vm.ctx)) { - // For the base partial class, always use functools.partial - "functools.partial".to_owned() - } else { - // For subclasses, check if they're defined in __main__ or test modules - match module.downcast::<crate::builtins::PyStr>() { - Ok(module_str) => { - let module_name = module_str.as_str(); - match module_name { - "builtins" | "" | "__main__" => class_name.to_owned(), - name if name.starts_with("test.") || name == "test" => { - // For test modules, just use the class name without module prefix - class_name.to_owned() - } - _ => format!("{module_name}.{class_name}"), - } + let qualified_name = match module.downcast::<crate::builtins::PyStr>() { + Ok(module_str) => { + let module_name = module_str.as_str(); + match module_name { + "builtins" | "" => qualname_str, + _ => format!("{module_name}.{qualname_str}"), } - Err(_) => class_name.to_owned(), } + Err(_) => qualname_str, }; Ok(format!( diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs new file mode 100644 index 00000000000..82b0c68bd9e --- /dev/null +++ b/crates/vm/src/stdlib/gc.rs @@ -0,0 +1,265 @@ +pub(crate) use gc::module_def; + +#[pymodule] +mod gc { + use crate::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::PyListRef, + function::{FuncArgs, OptionalArg}, + gc_state, + }; + + // Debug flag constants + #[pyattr] + const DEBUG_STATS: u32 = gc_state::GcDebugFlags::STATS.bits(); + #[pyattr] + const DEBUG_COLLECTABLE: u32 = gc_state::GcDebugFlags::COLLECTABLE.bits(); + #[pyattr] + const DEBUG_UNCOLLECTABLE: u32 = gc_state::GcDebugFlags::UNCOLLECTABLE.bits(); + #[pyattr] + const DEBUG_SAVEALL: u32 = gc_state::GcDebugFlags::SAVEALL.bits(); + #[pyattr] + const DEBUG_LEAK: u32 = gc_state::GcDebugFlags::LEAK.bits(); + + /// Enable automatic garbage collection. + #[pyfunction] + fn enable() { + gc_state::gc_state().enable(); + } + + /// Disable automatic garbage collection. + #[pyfunction] + fn disable() { + gc_state::gc_state().disable(); + } + + /// Return true if automatic gc is enabled. + #[pyfunction] + fn isenabled() -> bool { + gc_state::gc_state().is_enabled() + } + + /// Run a garbage collection. Returns the number of unreachable objects found. + #[derive(FromArgs)] + struct CollectArgs { + #[pyarg(any, optional)] + generation: OptionalArg<i32>, + } + + #[pyfunction] + fn collect(args: CollectArgs, vm: &VirtualMachine) -> PyResult<i32> { + let generation = args.generation; + let generation_num = generation.unwrap_or(2); + if !(0..=2).contains(&generation_num) { + return Err(vm.new_value_error("invalid generation".to_owned())); + } + + // Invoke callbacks with "start" phase + invoke_callbacks(vm, "start", generation_num as usize, 0, 0); + + // Manual gc.collect() should run even if GC is disabled + let gc = gc_state::gc_state(); + let (collected, uncollectable) = gc.collect_force(generation_num as usize); + + // Move objects from gc_state.garbage to vm.ctx.gc_garbage (for DEBUG_SAVEALL) + { + let mut state_garbage = gc.garbage.lock(); + if !state_garbage.is_empty() { + let py_garbage = &vm.ctx.gc_garbage; + let mut garbage_vec = py_garbage.borrow_vec_mut(); + for obj in state_garbage.drain(..) { + garbage_vec.push(obj); + } + } + } + + // Invoke callbacks with "stop" phase + invoke_callbacks( + vm, + "stop", + generation_num as usize, + collected, + uncollectable, + ); + + Ok(collected as i32) + } + + /// Return the current collection thresholds as a tuple. + #[pyfunction] + fn get_threshold(vm: &VirtualMachine) -> PyObjectRef { + let (t0, t1, t2) = gc_state::gc_state().get_threshold(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(t0).into(), + vm.ctx.new_int(t1).into(), + vm.ctx.new_int(t2).into(), + ]) + .into() + } + + /// Set the collection thresholds. + #[pyfunction] + fn set_threshold(threshold0: u32, threshold1: OptionalArg<u32>, threshold2: OptionalArg<u32>) { + gc_state::gc_state().set_threshold( + threshold0, + threshold1.into_option(), + threshold2.into_option(), + ); + } + + /// Return the current collection counts as a tuple. + #[pyfunction] + fn get_count(vm: &VirtualMachine) -> PyObjectRef { + let (c0, c1, c2) = gc_state::gc_state().get_count(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(c0).into(), + vm.ctx.new_int(c1).into(), + vm.ctx.new_int(c2).into(), + ]) + .into() + } + + /// Return the current debugging flags. + #[pyfunction] + fn get_debug() -> u32 { + gc_state::gc_state().get_debug().bits() + } + + /// Set the debugging flags. + #[pyfunction] + fn set_debug(flags: u32) { + gc_state::gc_state().set_debug(gc_state::GcDebugFlags::from_bits_truncate(flags)); + } + + /// Return a list of per-generation gc stats. + #[pyfunction] + fn get_stats(vm: &VirtualMachine) -> PyResult<PyListRef> { + let stats = gc_state::gc_state().get_stats(); + let mut result = Vec::with_capacity(3); + + for stat in stats.iter() { + let dict = vm.ctx.new_dict(); + dict.set_item("collections", vm.ctx.new_int(stat.collections).into(), vm)?; + dict.set_item("collected", vm.ctx.new_int(stat.collected).into(), vm)?; + dict.set_item( + "uncollectable", + vm.ctx.new_int(stat.uncollectable).into(), + vm, + )?; + result.push(dict.into()); + } + + Ok(vm.ctx.new_list(result)) + } + + /// Return the list of objects tracked by the collector. + #[derive(FromArgs)] + struct GetObjectsArgs { + #[pyarg(any, optional)] + generation: OptionalArg<Option<i32>>, + } + + #[pyfunction] + fn get_objects(args: GetObjectsArgs, vm: &VirtualMachine) -> PyResult<PyListRef> { + let generation_opt = args.generation.flatten(); + if let Some(g) = generation_opt + && !(0..=2).contains(&g) + { + return Err(vm.new_value_error(format!("generation must be in range(0, 3), not {}", g))); + } + let objects = gc_state::gc_state().get_objects(generation_opt); + Ok(vm.ctx.new_list(objects)) + } + + /// Return the list of objects directly referred to by any of the arguments. + #[pyfunction] + fn get_referents(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + let mut result = Vec::new(); + + for obj in args.args { + // Use the gc_get_referents method to get references + result.extend(obj.gc_get_referents()); + } + + vm.ctx.new_list(result) + } + + /// Return the list of objects that directly refer to any of the arguments. + #[pyfunction] + fn get_referrers(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + // This is expensive: we need to scan all tracked objects + // For now, return an empty list (would need full object tracking to implement) + let _ = args; + vm.ctx.new_list(vec![]) + } + + /// Return True if the object is tracked by the garbage collector. + #[pyfunction] + fn is_tracked(obj: PyObjectRef) -> bool { + // An object is tracked if it has IS_TRACE = true (has a trace function) + obj.is_gc_tracked() + } + + /// Return True if the object has been finalized by the garbage collector. + #[pyfunction] + fn is_finalized(obj: PyObjectRef) -> bool { + obj.gc_finalized() + } + + /// Freeze all objects tracked by gc. + #[pyfunction] + fn freeze() { + gc_state::gc_state().freeze(); + } + + /// Unfreeze all objects in the permanent generation. + #[pyfunction] + fn unfreeze() { + gc_state::gc_state().unfreeze(); + } + + /// Return the number of objects in the permanent generation. + #[pyfunction] + fn get_freeze_count() -> usize { + gc_state::gc_state().get_freeze_count() + } + + /// gc.garbage - list of uncollectable objects + #[pyattr] + fn garbage(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_garbage.clone() + } + + /// gc.callbacks - list of callbacks to be invoked + #[pyattr] + fn callbacks(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_callbacks.clone() + } + + /// Helper function to invoke GC callbacks + fn invoke_callbacks( + vm: &VirtualMachine, + phase: &str, + generation: usize, + collected: usize, + uncollectable: usize, + ) { + let callbacks_list = &vm.ctx.gc_callbacks; + let callbacks: Vec<PyObjectRef> = callbacks_list.borrow_vec().to_vec(); + if callbacks.is_empty() { + return; + } + + let phase_str: PyObjectRef = vm.ctx.new_str(phase).into(); + let info = vm.ctx.new_dict(); + let _ = info.set_item("generation", vm.ctx.new_int(generation).into(), vm); + let _ = info.set_item("collected", vm.ctx.new_int(collected).into(), vm); + let _ = info.set_item("uncollectable", vm.ctx.new_int(uncollectable).into(), vm); + + for callback in callbacks { + let _ = callback.call((phase_str.clone(), info.clone()), vm); + } + } +} diff --git a/crates/vm/src/stdlib/imp.rs b/crates/vm/src/stdlib/imp.rs index 596847776ff..cf9aba02265 100644 --- a/crates/vm/src/stdlib/imp.rs +++ b/crates/vm/src/stdlib/imp.rs @@ -1,6 +1,11 @@ +use crate::builtins::{PyCode, PyStrInterned}; use crate::frozen::FrozenModule; use crate::{VirtualMachine, builtins::PyBaseExceptionRef}; -pub(crate) use _imp::make_module; +use core::borrow::Borrow; + +pub(crate) use _imp::module_def; + +pub use crate::vm::resolve_frozen_alias; #[cfg(feature = "threading")] #[pymodule(sub)] @@ -70,30 +75,52 @@ impl FrozenError { } } -// find_frozen in frozen.c +// look_up_frozen + use_frozen in import.c fn find_frozen(name: &str, vm: &VirtualMachine) -> Result<FrozenModule, FrozenError> { - vm.state + let frozen = vm + .state .frozen .get(name) .copied() - .ok_or(FrozenError::NotFound) + .ok_or(FrozenError::NotFound)?; + + // Bootstrap modules are always available regardless of override flag + if matches!( + name, + "_frozen_importlib" | "_frozen_importlib_external" | "zipimport" + ) { + return Ok(frozen); + } + + // use_frozen(): override > 0 → true, override < 0 → false, 0 → default (true) + // When disabled, non-bootstrap modules are simply not found (same as look_up_frozen) + let override_val = vm.state.override_frozen_modules.load(); + if override_val < 0 { + return Err(FrozenError::NotFound); + } + + Ok(frozen) } #[pymodule(with(lock))] mod _imp { use crate::{ - PyObjectRef, PyRef, PyResult, VirtualMachine, + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef}, + convert::TryFromBorrowedObject, function::OptionalArg, - import, + import, version, }; #[pyattr] fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { vm.ctx - .new_str(vm.state.settings.check_hash_pycs_mode.to_string()) + .new_str(vm.state.config.settings.check_hash_pycs_mode.to_string()) } + #[pyattr(name = "pyc_magic_number_token")] + use version::PYC_MAGIC_NUMBER_TOKEN; + #[pyfunction] const fn extension_suffixes() -> PyResult<Vec<PyObjectRef>> { Ok(Vec::new()) @@ -101,12 +128,12 @@ mod _imp { #[pyfunction] fn is_builtin(name: PyStrRef, vm: &VirtualMachine) -> bool { - vm.state.module_inits.contains_key(name.as_str()) + vm.state.module_defs.contains_key(name.as_str()) } #[pyfunction] fn is_frozen(name: PyStrRef, vm: &VirtualMachine) -> bool { - vm.state.frozen.contains_key(name.as_str()) + super::find_frozen(name.as_str(), vm).is_ok() } #[pyfunction] @@ -114,24 +141,72 @@ mod _imp { let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap(); let name: PyStrRef = spec.get_attr("name", vm)?.try_into_value(vm)?; - let module = if let Ok(module) = sys_modules.get_item(&*name, vm) { - module - } else if let Some(make_module_func) = vm.state.module_inits.get(name.as_str()) { - make_module_func(vm).into() - } else { - vm.ctx.none() - }; - Ok(module) + // Check sys.modules first + if let Ok(module) = sys_modules.get_item(&*name, vm) { + return Ok(module); + } + + // Try multi-phase init modules first (they need special handling) + if let Some(&def) = vm.state.module_defs.get(name.as_str()) { + // Phase 1: Create module (use create slot if provided, else default creation) + let module = if let Some(create) = def.slots.create { + // Custom module creation + create(vm, &spec, def)? + } else { + // Default module creation + PyModule::from_def(def).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + // Corresponds to PyModule_FromDefAndSpec: md_def, _add_methods_to_object, PyModule_SetDocString + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(&*name, module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + if let Some(exec) = def.slots.exec { + exec(vm, &module)?; + } + + return Ok(module.into()); + } + + Ok(vm.ctx.none()) } #[pyfunction] fn exec_builtin(_mod: PyRef<PyModule>) -> i32 { - // TODO: Should we do something here? + // For multi-phase init modules, exec is already called in create_builtin 0 } #[pyfunction] - fn get_frozen_object(name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyCode>> { + fn get_frozen_object( + name: PyStrRef, + data: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyCode>> { + if let OptionalArg::Present(data) = data + && !vm.is_none(&data) + { + let buf = crate::protocol::PyBuffer::try_from_borrowed_object(vm, &data)?; + let contiguous = buf.as_contiguous().ok_or_else(|| { + vm.new_buffer_error("get_frozen_object() requires a contiguous buffer") + })?; + let invalid_err = || { + vm.new_import_error( + format!("Frozen object named '{}' is invalid", name.as_str()), + name.clone(), + ) + }; + let bag = crate::builtins::code::PyObjBag(&vm.ctx); + let code = + rustpython_compiler_core::marshal::deserialize_code(&mut &contiguous[..], bag) + .map_err(|_| invalid_err())?; + return Ok(vm.ctx.new_code(code)); + } import::make_frozen(vm, name.as_str()) } @@ -153,8 +228,10 @@ mod _imp { } #[pyfunction] - fn _fix_co_filename(_code: PyObjectRef, _path: PyStrRef) { - // TODO: + fn _fix_co_filename(code: PyRef<PyCode>, path: PyStrRef, vm: &VirtualMachine) { + let old_name = code.source_path(); + let new_name = vm.ctx.intern_str(path.as_str()); + super::update_code_filenames(&code, old_name, new_name); } #[pyfunction] @@ -174,7 +251,7 @@ mod _imp { name: PyStrRef, withdata: OptionalArg<bool>, vm: &VirtualMachine, - ) -> PyResult<Option<(Option<PyRef<PyMemoryView>>, bool, PyStrRef)>> { + ) -> PyResult<Option<(Option<PyRef<PyMemoryView>>, bool, Option<PyStrRef>)>> { use super::FrozenError::*; if withdata.into_option().is_some() { @@ -188,7 +265,14 @@ mod _imp { Err(e) => return Err(e.to_pyexception(name.as_str(), vm)), }; - let origname = name; // FIXME: origname != name + // When origname is empty (e.g. __hello_only__), return None. + // Otherwise return the resolved alias name. + let origname_str = super::resolve_frozen_alias(name.as_str()); + let origname = if origname_str.is_empty() { + None + } else { + Some(vm.ctx.new_str(origname_str)) + }; Ok(Some((None, info.package, origname))) } @@ -198,3 +282,21 @@ mod _imp { hash.to_le_bytes().to_vec() } } + +fn update_code_filenames( + code: &PyCode, + old_name: &'static PyStrInterned, + new_name: &'static PyStrInterned, +) { + let current = code.source_path(); + if !core::ptr::eq(current, old_name) && current.as_str() != old_name.as_str() { + return; + } + code.set_source_path(new_name); + for constant in code.code.constants.iter() { + let obj: &crate::PyObject = constant.borrow(); + if let Some(inner_code) = obj.downcast_ref::<PyCode>() { + update_code_filenames(inner_code, old_name, new_name); + } + } +} diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 3d67591d567..768df4cb7bf 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -1,6 +1,8 @@ /* * I/O core tools. */ +pub(crate) use _io::module_def; + cfg_if::cfg_if! { if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { use crate::common::crt_fd::Offset; @@ -9,93 +11,54 @@ cfg_if::cfg_if! { } } +// EAGAIN constant for BlockingIOError +cfg_if::cfg_if! { + if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { + const EAGAIN: i32 = libc::EAGAIN; + } else { + const EAGAIN: i32 = 11; // Standard POSIX value + } +} + use crate::{ - PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyModule}, - common::os::ErrorExt, - convert::{IntoPyException, ToPyException}, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyModule, }; pub use _io::{OpenArgs, io_open as open}; -impl ToPyException for std::io::Error { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - let errno = self.posix_errno(); - #[cfg(windows)] - let msg = 'msg: { - // On Windows, use C runtime's strerror for POSIX errno values - // For Windows-specific error codes, fall back to FormatMessage - - // UCRT's strerror returns "Unknown error" for invalid errno values - // Windows UCRT defines errno values 1-42 plus some more up to ~127 - const MAX_POSIX_ERRNO: i32 = 127; - if errno > 0 && errno <= MAX_POSIX_ERRNO { - let ptr = unsafe { libc::strerror(errno) }; - if !ptr.is_null() { - let s = unsafe { std::ffi::CStr::from_ptr(ptr) }.to_string_lossy(); - if !s.starts_with("Unknown error") { - break 'msg s.into_owned(); - } - } - } - self.to_string() - }; - #[cfg(unix)] - let msg = { - let ptr = unsafe { libc::strerror(errno) }; - if !ptr.is_null() { - unsafe { std::ffi::CStr::from_ptr(ptr) } - .to_string_lossy() - .into_owned() - } else { - self.to_string() - } - }; - #[cfg(not(any(windows, unix)))] - let msg = self.to_string(); +fn file_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + file.get_attr("closed", vm)?.try_to_bool(vm) +} - #[allow(clippy::let_and_return)] - let exc = vm.new_errno_error(errno, msg); - #[cfg(windows)] - { - use crate::object::AsObject; - let winerror = if let Some(winerror) = self.raw_os_error() { - vm.new_pyobj(winerror) - } else { - vm.ctx.none() - }; +const DEFAULT_BUFFER_SIZE: usize = 128 * 1024; - // FIXME: manual setup winerror due to lack of OSError.__init__ support - exc.as_object() - .set_attr("winerror", vm.new_pyobj(winerror), vm) - .unwrap(); +/// iobase_finalize in Modules/_io/iobase.c +fn iobase_finalize(zelf: &PyObject, vm: &VirtualMachine) { + // If `closed` doesn't exist or can't be evaluated as bool, then the + // object is probably in an unusable state, so ignore. + let closed = match vm.get_attribute_opt(zelf.to_owned(), "closed") { + Ok(Some(val)) => match val.try_to_bool(vm) { + Ok(b) => b, + Err(_) => return, + }, + _ => return, + }; + if !closed { + // Signal close() that it was called as part of the object + // finalization process. + let _ = zelf.set_attr("_finalizing", vm.ctx.true_value.clone(), vm); + if let Err(e) = vm.call_method(zelf, "close", ()) { + // BrokenPipeError during GC finalization is expected when pipe + // buffer objects are collected after the subprocess dies. The + // underlying fd is still properly closed by raw.close(). + // Popen.__del__ catches BrokenPipeError, but our tracing GC may + // finalize pipe buffers before Popen.__del__ runs. + if !e.fast_isinstance(vm.ctx.exceptions.broken_pipe_error) { + vm.run_unraisable(e, None, zelf.to_owned()); + } } - exc.upcast() } } -impl IntoPyException for std::io::Error { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - self.to_pyexception(vm) - } -} - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let ctx = &vm.ctx; - - let module = _io::make_module(vm); - - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - fileio::extend_module(vm, &module).unwrap(); - - let unsupported_operation = _io::unsupported_operation().to_owned(); - extend_module!(vm, &module, { - "UnsupportedOperation" => unsupported_operation, - "BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(), - }); - - module -} - // not used on all platforms #[derive(Copy, Clone)] #[repr(transparent)] @@ -148,8 +111,8 @@ mod _io { AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, builtins::{ - PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyBytesRef, PyMemoryView, PyStr, - PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef, + PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyBytesRef, PyDict, PyMemoryView, + PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8Str, PyUtf8StrRef, }, class::StaticType, common::lock::{ @@ -158,6 +121,7 @@ mod _io { }, common::wtf8::{Wtf8, Wtf8Buf}, convert::ToPyObject, + exceptions::cstring_error, function::{ ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FuncArgs, IntoFuncArgs, OptionalArg, OptionalOption, PySetterValue, @@ -168,18 +132,20 @@ mod _io { recursion::ReprGuard, types::{ Callable, Constructor, DefaultConstructor, Destructor, Initializer, IterNext, Iterable, + Representable, }, vm::VirtualMachine, }; + use alloc::borrow::Cow; use bstr::ByteSlice; + use core::{ + ops::Range, + sync::atomic::{AtomicBool, Ordering}, + }; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::BigInt; use num_traits::ToPrimitive; - use std::{ - borrow::Cow, - io::{self, Cursor, SeekFrom, prelude::*}, - ops::Range, - }; + use std::io::{self, Cursor, SeekFrom, prelude::*}; #[allow(clippy::let_and_return)] fn validate_whence(whence: i32) -> bool { @@ -201,6 +167,35 @@ mod _io { } } + /// Check if an error is an OSError with errno == EINTR. + /// If so, call check_signals() and return Ok(None) to indicate retry. + /// Otherwise, return Ok(Some(val)) for success or Err for other errors. + /// This mirrors CPythons _PyIO_trap_eintr() pattern. + #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + fn trap_eintr<T>(result: PyResult<T>, vm: &VirtualMachine) -> PyResult<Option<T>> { + match result { + Ok(val) => Ok(Some(val)), + Err(exc) => { + // Check if its an OSError with errno == EINTR + if exc.fast_isinstance(vm.ctx.exceptions.os_error) + && let Ok(errno_attr) = exc.as_object().get_attr("errno", vm) + && let Ok(errno_val) = i32::try_from_object(vm, errno_attr) + && errno_val == libc::EINTR + { + vm.check_signals()?; + return Ok(None); + } + Err(exc) + } + } + } + + /// WASM version: no EINTR handling needed + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + fn trap_eintr<T>(result: PyResult<T>, _vm: &VirtualMachine) -> PyResult<Option<T>> { + result.map(Some) + } + pub fn new_unsupported_operation(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { vm.new_os_subtype_error(unsupported_operation().to_owned(), None, msg) .upcast() @@ -242,15 +237,8 @@ mod _io { } fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - { - use crate::convert::ToPyException; - err.to_pyexception(vm) - } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - { - vm.new_os_error(err.to_string()) - } + use crate::convert::ToPyException; + err.to_pyexception(vm) } pub(super) fn io_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -258,7 +246,7 @@ mod _io { } #[pyattr] - const DEFAULT_BUFFER_SIZE: usize = 8 * 1024; + const DEFAULT_BUFFER_SIZE: usize = super::DEFAULT_BUFFER_SIZE; pub(super) fn seekfrom( vm: &VirtualMachine, @@ -312,7 +300,7 @@ mod _io { // if we don't specify the number of bytes, or it's too big, give the whole rest of the slice let n = bytes.map_or_else( || avail_slice.len(), - |n| std::cmp::min(n, avail_slice.len()), + |n| core::cmp::min(n, avail_slice.len()), ); let b = avail_slice[..n].to_vec(); self.cursor.set_position((pos + n) as u64); @@ -368,10 +356,6 @@ mod _io { } } - fn file_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - file.get_attr("closed", vm)?.try_to_bool(vm) - } - fn check_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { if file_closed(file, vm)? { Err(io_closed_error(vm)) @@ -591,6 +575,11 @@ mod _io { impl Destructor for _IOBase { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + // C-level IO types (FileIO, Buffered*, TextIOWrapper) have their own + // slot_del that calls iobase_finalize with proper _finalizing flag + // and _dealloc_warn chain. This base fallback is only reached by + // Python-level subclasses, where we silently discard close() errors + // to avoid surfacing unraisable from partially initialized objects. let _ = vm.call_method(zelf, "close", ()); Ok(()) } @@ -649,17 +638,26 @@ mod _io { if let Some(size) = size.to_usize() { // FIXME: unnecessary zero-init let b = PyByteArray::from(vec![0; size]).into_ref(&vm.ctx); - let n = <Option<usize>>::try_from_object( + let n = <Option<isize>>::try_from_object( vm, vm.call_method(&instance, "readinto", (b.clone(),))?, )?; - Ok(n.map(|n| { - let mut bytes = b.borrow_buf_mut(); - bytes.truncate(n); - // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer - bytes.clone() + Ok(match n { + None => vm.ctx.none(), + Some(n) => { + // Validate the return value is within bounds + if n < 0 || (n as usize) > size { + return Err(vm.new_value_error(format!( + "readinto returned {n} outside buffer size {size}" + ))); + } + let n = n as usize; + let mut bytes = b.borrow_buf_mut(); + bytes.truncate(n); + // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer + bytes.clone().to_pyobject(vm) + } }) - .to_pyobject(vm)) } else { vm.call_method(&instance, "readall", ()) } @@ -670,7 +668,14 @@ mod _io { let mut chunks = Vec::new(); let mut total_len = 0; loop { - let data = vm.call_method(&instance, "read", (DEFAULT_BUFFER_SIZE,))?; + // Loop with EINTR handling (PEP 475) + let data = loop { + let res = vm.call_method(&instance, "read", (DEFAULT_BUFFER_SIZE,)); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + }; let data = <Option<PyBytesRef>>::try_from_object(vm, data)?; match data { None => { @@ -878,18 +883,26 @@ mod _io { let rewind = self.raw_offset() + (self.pos - self.write_pos); if rewind != 0 { self.raw_seek(-rewind, 1, vm)?; - self.raw_pos = -rewind; + self.raw_pos -= rewind; } while self.write_pos < self.write_end { let n = self.raw_write(None, self.write_pos as usize..self.write_end as usize, vm)?; - let n = n.ok_or_else(|| { - vm.new_exception_msg( - vm.ctx.exceptions.blocking_io_error.to_owned(), - "write could not complete without blocking".to_owned(), - ) - })?; + let n = match n { + Some(n) => n, + None => { + // BlockingIOError(errno, msg, characters_written=0) + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(0), + ], + )?); + } + }; self.write_pos += n as Offset; self.raw_pos = self.write_pos; vm.check_signals()?; @@ -934,7 +947,10 @@ mod _io { }; if offset >= -self.pos && offset <= available { self.pos += offset; - return Ok(current - available + offset); + // GH-95782: character devices may report raw position 0 + // even after reading, which would make this negative + let result = current - available + offset; + return Ok(if result < 0 { 0 } else { result }); } } } @@ -990,7 +1006,7 @@ mod _io { // TODO: loop if write() raises an interrupt vm.call_method(self.raw.as_ref().unwrap(), "write", (mem_obj,))? } else { - let v = std::mem::take(&mut self.buffer); + let v = core::mem::take(&mut self.buffer); let write_buf = VecBuffer::from(v).into_ref(&vm.ctx); let mem_obj = PyMemoryView::from_buffer_range( write_buf.clone().into_pybuffer(true), @@ -1046,9 +1062,51 @@ mod _io { } } - // TODO: something something check if error is BlockingIOError? - let _ = self.flush(vm); + // if BlockingIOError, shift buffer + // and try to buffer the new data; otherwise propagate the error + match self.flush(vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.blocking_io_error) => { + if self.readable() { + self.reset_read(); + } + // Shift buffer and adjust positions + let shift = self.write_pos; + if shift > 0 { + self.buffer + .copy_within(shift as usize..self.write_end as usize, 0); + self.write_end -= shift; + self.raw_pos -= shift; + self.pos -= shift; + self.write_pos = 0; + } + let avail = self.buffer.len() - self.write_end as usize; + if buf_len <= avail { + // Everything can be buffered + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..buf_len].copy_from_slice(&buf); + self.write_end += buf_len as Offset; + self.pos += buf_len as Offset; + return Ok(buf_len); + } + // Buffer as much as possible and return BlockingIOError + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]); + self.write_end += avail as Offset; + self.pos += avail as Offset; + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(avail), + ], + )?); + } + Err(e) => return Err(e), + } + // Only reach here if flush succeeded let offset = self.raw_offset(); if offset != 0 { self.raw_seek(-offset, 1, vm)?; @@ -1081,12 +1139,16 @@ mod _io { let buffer_size = self.buffer.len() as _; self.adjust_position(buffer_size); self.write_end = buffer_size; - // TODO: BlockingIOError(errno, msg, written) - // written += self.buffer.len(); - return Err(vm.new_exception_msg( + // BlockingIOError(errno, msg, characters_written) + let chars_written = written + buffer_len; + return Err(vm.invoke_exception( vm.ctx.exceptions.blocking_io_error.to_owned(), - "write could not complete without blocking".to_owned(), - )); + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(chars_written), + ], + )?); } else { break; } @@ -1152,7 +1214,7 @@ mod _io { } }; } - while remaining > 0 { + while remaining > 0 && !self.buffer.is_empty() { // MINUS_LAST_BLOCK() in CPython let r = self.buffer.len() * (remaining / self.buffer.len()); if r == 0 { @@ -1215,7 +1277,7 @@ mod _io { let res = match v { Either::A(v) => { let v = v.unwrap_or(&mut self.buffer); - let read_buf = VecBuffer::from(std::mem::take(v)).into_ref(&vm.ctx); + let read_buf = VecBuffer::from(core::mem::take(v)).into_ref(&vm.ctx); let mem_obj = PyMemoryView::from_buffer_range( read_buf.clone().into_pybuffer(false), buf_range, @@ -1223,30 +1285,60 @@ mod _io { )? .into_ref(&vm.ctx); - // TODO: loop if readinto() raises an interrupt - let res = - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (mem_obj.clone(),)); + // Loop if readinto() raises EINTR (PEP 475) + let res = loop { + let res = vm.call_method( + self.raw.as_ref().unwrap(), + "readinto", + (mem_obj.clone(),), + ); + match trap_eintr(res, vm) { + Ok(Some(val)) => break Ok(val), + Ok(None) => continue, // EINTR, retry + Err(e) => break Err(e), + } + }; mem_obj.release(); + // Always restore the buffer, even if an error occurred *v = read_buf.take(); res? } Either::B(buf) => { - let mem_obj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?; - // TODO: loop if readinto() raises an interrupt - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (mem_obj,))? + let mem_obj = + PyMemoryView::from_buffer_range(buf, buf_range, vm)?.into_ref(&vm.ctx); + // Loop if readinto() raises EINTR (PEP 475) + loop { + let res = vm.call_method( + self.raw.as_ref().unwrap(), + "readinto", + (mem_obj.clone(),), + ); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + } } }; if vm.is_none(&res) { return Ok(None); } - let n = isize::try_from_object(vm, res)?; + // Try to convert to int; if it fails, treat as -1 and chain the TypeError + let (n, type_error) = match isize::try_from_object(vm, res.clone()) { + Ok(n) => (n, None), + Err(e) => (-1, Some(e)), + }; if n < 0 || n as usize > len { - return Err(vm.new_os_error(format!( + let os_error = vm.new_os_error(format!( "raw readinto() returned invalid length {n} (should have been between 0 and {len})" - ))); + )); + if let Some(cause) = type_error { + os_error.set___cause__(Some(cause)); + } + return Err(os_error); } if n > 0 && self.abs_pos != -1 { self.abs_pos += n as Offset @@ -1289,7 +1381,14 @@ mod _io { let mut read_size = 0; loop { - let read_data = vm.call_method(self.raw.as_ref().unwrap(), "read", ())?; + // Loop with EINTR handling (PEP 475) + let read_data = loop { + let res = vm.call_method(self.raw.as_ref().unwrap(), "read", ()); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + }; let read_data = <Option<PyBytesRef>>::try_from_object(vm, read_data)?; match read_data { @@ -1375,7 +1474,7 @@ mod _io { } else if !(readinto1 && written != 0) { let n = self.fill_buffer(vm)?; if let Some(n) = n.filter(|&n| n > 0) { - let n = std::cmp::min(n, remaining); + let n = core::cmp::min(n, remaining); buf.as_contiguous_mut().unwrap()[written..][..n] .copy_from_slice(&self.buffer[self.pos as usize..][..n]); self.pos += n as Offset; @@ -1443,13 +1542,15 @@ mod _io { } #[pyclass] - trait BufferedMixin: PyPayload { + trait BufferedMixin: PyPayload + StaticType { const CLASS_NAME: &'static str; const READABLE: bool; const WRITABLE: bool; const SEEKABLE: bool = false; fn data(&self) -> &PyThreadMutex<BufferedData>; + fn closing(&self) -> &AtomicBool; + fn finalizing(&self) -> &AtomicBool; fn lock(&self, vm: &VirtualMachine) -> PyResult<PyThreadMutexGuard<'_, BufferedData>> { self.data() @@ -1460,17 +1561,16 @@ mod _io { #[pyslot] fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let zelf: PyRef<Self> = zelf.try_into_value(vm)?; - zelf.__init__(args, vm) - } - - #[pymethod] - fn __init__(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let (raw, BufferSize { buffer_size }): (PyObjectRef, _) = args.bind(vm).map_err(|e| { - let msg = format!("{}() {}", Self::CLASS_NAME, *e.__str__(vm)); + let str_repr = e + .__str__(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| "<error getting exception str>".to_owned()); + let msg = format!("{}() {}", Self::CLASS_NAME, str_repr); vm.new_exception_msg(e.class().to_owned(), msg) })?; - self.init(raw, BufferSize { buffer_size }, vm) + zelf.init(raw, BufferSize { buffer_size }, vm) } fn init( @@ -1561,9 +1661,10 @@ mod _io { let pos = pos.flatten().to_pyobject(vm); let mut data = zelf.lock(vm)?; data.check_init(vm)?; - if data.writable() { - data.flush_rewind(vm)?; + if !data.writable() { + return Err(new_unsupported_operation(vm, "truncate".to_owned())); } + data.flush_rewind(vm)?; let res = vm.call_method(data.raw.as_ref().unwrap(), "truncate", (pos,))?; let _ = data.raw_tell(vm); Ok(res) @@ -1588,19 +1689,25 @@ mod _io { Ok(self.lock(vm)?.raw.clone()) } + /// Get raw stream without holding the lock (for calling Python code safely) + fn get_raw_unlocked(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let data = self.lock(vm)?; + Ok(data.check_init(vm)?.to_owned()) + } + #[pygetset] fn closed(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("closed", vm) + self.get_raw_unlocked(vm)?.get_attr("closed", vm) } #[pygetset] fn name(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("name", vm) + self.get_raw_unlocked(vm)?.get_attr("name", vm) } #[pygetset] fn mode(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("mode", vm) + self.get_raw_unlocked(vm)?.get_attr("mode", vm) } #[pymethod] @@ -1644,17 +1751,23 @@ mod _io { #[pymethod] fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - { + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { let data = zelf.lock(vm)?; let raw = data.check_init(vm)?; if file_closed(raw, vm)? { return Ok(vm.ctx.none()); } + raw.to_owned() + }; + if zelf.finalizing().load(Ordering::Relaxed) { + // _dealloc_warn: delegate to raw._dealloc_warn(source) + let _ = vm.call_method(&raw, "_dealloc_warn", (zelf.as_object().to_owned(),)); } + // Set closing flag so that concurrent write() calls will fail + zelf.closing().store(true, Ordering::Release); let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); - let data = zelf.lock(vm)?; - let raw = data.raw.as_ref().unwrap(); - let close_res = vm.call_method(raw, "close", ()); + let close_res = vm.call_method(&raw, "close", ()); exception_chain(flush_res, close_res) } @@ -1668,10 +1781,37 @@ mod _io { Self::WRITABLE } - // TODO: this should be the default for an equivalent of _PyObject_GetState #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + #[pymethod] + fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + if zelf.class().is(Self::static_type()) { + return Err( + vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) + ); + } + let _ = proto; + reduce_ex_for_subclass(zelf, vm) + } + + #[pymethod] + fn _dealloc_warn( + zelf: PyRef<Self>, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get raw reference and release lock before calling downstream + let raw = { + let data = zelf.lock(vm)?; + data.raw.clone() + }; + if let Some(raw) = raw { + let _ = vm.call_method(&raw, "_dealloc_warn", (source,)); + } + Ok(()) } } @@ -1721,9 +1861,13 @@ mod _io { } let have = data.readahead(); if have > 0 { - let n = std::cmp::min(have as usize, n); + let n = core::cmp::min(have as usize, n); return Ok(data.read_fast(n).unwrap()); } + // Flush write buffer before reading + if data.writable() { + data.flush_rewind(vm)?; + } let mut v = vec![0; n]; data.reset_read(); let r = data @@ -1749,6 +1893,19 @@ mod _io { ensure_unclosed(raw, "readinto of closed file", vm)?; data.readinto_generic(buf.into(), true, vm) } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + // For read-only buffers, flush just calls raw.flush() + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { + let data = self.reader().lock(vm)?; + data.check_init(vm)?.to_owned() + }; + ensure_unclosed(&raw, "flush of closed file", vm)?; + vm.call_method(&raw, "flush", ())?; + Ok(()) + } } fn exception_chain<T>(e1: PyResult<()>, e2: PyResult<T>) -> PyResult<T> { @@ -1768,6 +1925,8 @@ mod _io { struct BufferedReader { _base: _BufferedIOBase, data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, } impl BufferedMixin for BufferedReader { @@ -1778,6 +1937,14 @@ mod _io { fn data(&self) -> &PyThreadMutex<BufferedData> { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } } impl BufferedReadable for BufferedReader { @@ -1796,7 +1963,10 @@ mod _io { impl Destructor for BufferedReader { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + if let Some(buf) = zelf.downcast_ref::<BufferedReader>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); Ok(()) } @@ -1816,6 +1986,27 @@ mod _io { #[pymethod] fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + // Check if close() is in progress (Issue #31976) + // If closing, wait for close() to complete by spinning until raw is closed. + // Note: This spin-wait has no timeout because close() is expected to always + // complete (flush + fd close). + if self.writer().closing().load(Ordering::Acquire) { + loop { + let raw = { + let data = self.writer().lock(vm)?; + match &data.raw { + Some(raw) => raw.to_owned(), + None => break, // detached + } + }; + if file_closed(&raw, vm)? { + break; + } + // Yield to other threads + std::thread::yield_now(); + } + return Err(vm.new_value_error("write to closed file".to_owned())); + } let mut data = self.writer().lock(vm)?; let raw = data.check_init(vm)?; ensure_unclosed(raw, "write to closed file", vm)?; @@ -1838,6 +2029,8 @@ mod _io { struct BufferedWriter { _base: _BufferedIOBase, data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, } impl BufferedMixin for BufferedWriter { @@ -1848,6 +2041,14 @@ mod _io { fn data(&self) -> &PyThreadMutex<BufferedData> { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } } impl BufferedWritable for BufferedWriter { @@ -1866,7 +2067,10 @@ mod _io { impl Destructor for BufferedWriter { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + if let Some(buf) = zelf.downcast_ref::<BufferedWriter>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); Ok(()) } @@ -1884,6 +2088,8 @@ mod _io { struct BufferedRandom { _base: _BufferedIOBase, data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, } impl BufferedMixin for BufferedRandom { @@ -1895,6 +2101,14 @@ mod _io { fn data(&self) -> &PyThreadMutex<BufferedData> { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } } impl BufferedReadable for BufferedRandom { @@ -1927,7 +2141,10 @@ mod _io { impl Destructor for BufferedRandom { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + if let Some(buf) = zelf.downcast_ref::<BufferedRandom>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); Ok(()) } @@ -2031,7 +2248,7 @@ mod _io { impl Destructor for BufferedRWPair { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + iobase_finalize(zelf, vm); Ok(()) } @@ -2048,14 +2265,14 @@ mod _io { #[pyarg(any, default)] errors: Option<PyStrRef>, #[pyarg(any, default)] - newline: Option<Newlines>, + newline: OptionalOption<Newlines>, #[pyarg(any, default)] - line_buffering: Option<bool>, + line_buffering: OptionalOption<PyObjectRef>, #[pyarg(any, default)] - write_through: Option<bool>, + write_through: OptionalOption<PyObjectRef>, } - #[derive(Debug, Copy, Clone, Default)] + #[derive(Debug, Copy, Clone, Default, PartialEq)] enum Newlines { #[default] Universal, @@ -2086,7 +2303,7 @@ mod _io { }) .ok_or(len) } - Self::Cr => s.find("\n".as_ref()).map(|p| p + 1).ok_or(len), + Self::Cr => s.find("\r".as_ref()).map(|p| p + 1).ok_or(len), Self::Crlf => { // s[searched..] == remaining let mut searched = 0; @@ -2125,7 +2342,13 @@ mod _io { obj.class().name() )) })?; - match s.as_str() { + let wtf8 = s.as_wtf8(); + if !wtf8.is_utf8() { + let repr = s.repr(vm)?.as_str().to_owned(); + return Err(vm.new_value_error(format!("illegal newline value: {repr}"))); + } + let s_str = wtf8.as_str().expect("checked utf8"); + match s_str { "" => Self::Passthrough, "\n" => Self::Lf, "\r" => Self::Cr, @@ -2137,6 +2360,22 @@ mod _io { } } + fn reduce_ex_for_subclass(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let cls = zelf.class(); + let new = vm + .get_attribute_opt(cls.to_owned().into(), "__new__")? + .ok_or_else(|| vm.new_attribute_error("type has no attribute '__new__'"))?; + let args = vm.ctx.new_tuple(vec![cls.to_owned().into()]); + let state = if let Some(getstate) = vm.get_attribute_opt(zelf.clone(), "__getstate__")? { + getstate.call((), vm)? + } else if let Ok(dict) = zelf.get_attr("__dict__", vm) { + dict + } else { + vm.ctx.none() + }; + Ok(vm.ctx.new_tuple(vec![new, args.into(), state]).into()) + } + /// A length of or index into a UTF-8 string, measured in both chars and bytes #[derive(Debug, Default, Copy, Clone)] struct Utf8size { @@ -2160,7 +2399,7 @@ mod _io { } } - impl std::ops::Add for Utf8size { + impl core::ops::Add for Utf8size { type Output = Self; #[inline] @@ -2170,7 +2409,7 @@ mod _io { } } - impl std::ops::AddAssign for Utf8size { + impl core::ops::AddAssign for Utf8size { #[inline] fn add_assign(&mut self, rhs: Self) { self.bytes += rhs.bytes; @@ -2178,7 +2417,7 @@ mod _io { } } - impl std::ops::Sub for Utf8size { + impl core::ops::Sub for Utf8size { type Output = Self; #[inline] @@ -2188,7 +2427,7 @@ mod _io { } } - impl std::ops::SubAssign for Utf8size { + impl core::ops::SubAssign for Utf8size { #[inline] fn sub_assign(&mut self, rhs: Self) { self.bytes -= rhs.bytes; @@ -2257,7 +2496,7 @@ mod _io { impl PendingWrites { fn push(&mut self, write: PendingWrite) { self.num_bytes += write.as_bytes().len(); - self.data = match std::mem::take(&mut self.data) { + self.data = match core::mem::take(&mut self.data) { PendingWritesData::None => PendingWritesData::One(write), PendingWritesData::One(write1) => PendingWritesData::Many(vec![write1, write]), PendingWritesData::Many(mut v) => { @@ -2267,13 +2506,13 @@ mod _io { } } fn take(&mut self, vm: &VirtualMachine) -> PyBytesRef { - let Self { num_bytes, data } = std::mem::take(self); + let Self { num_bytes, data } = core::mem::take(self); if let PendingWritesData::One(PendingWrite::Bytes(b)) = data { return b; } let writes_iter = match data { PendingWritesData::None => itertools::Either::Left(vec![].into_iter()), - PendingWritesData::One(write) => itertools::Either::Right(std::iter::once(write)), + PendingWritesData::One(write) => itertools::Either::Right(core::iter::once(write)), PendingWritesData::Many(writes) => itertools::Either::Left(writes.into_iter()), }; let mut buf = Vec::with_capacity(num_bytes); @@ -2295,7 +2534,7 @@ mod _io { impl TextIOCookie { const START_POS_OFF: usize = 0; - const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + std::mem::size_of::<Offset>(); + const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + core::mem::size_of::<Offset>(); const BYTES_TO_FEED_OFF: usize = Self::DEC_FLAGS_OFF + 4; const CHARS_TO_SKIP_OFF: usize = Self::BYTES_TO_FEED_OFF + 4; const NEED_EOF_OFF: usize = Self::CHARS_TO_SKIP_OFF + 4; @@ -2308,15 +2547,11 @@ mod _io { return None; } buf.resize(Self::BYTE_LEN, 0); - let buf: &[u8; Self::BYTE_LEN] = buf.as_slice().try_into().unwrap(); + let buf: &[u8; Self::BYTE_LEN] = buf.as_array()?; macro_rules! get_field { - ($t:ty, $off:ident) => {{ - <$t>::from_ne_bytes( - buf[Self::$off..][..std::mem::size_of::<$t>()] - .try_into() - .unwrap(), - ) - }}; + ($t:ty, $off:ident) => { + <$t>::from_ne_bytes(*buf[Self::$off..].first_chunk().unwrap()) + }; } Some(Self { start_pos: get_field!(Offset, START_POS_OFF), @@ -2333,7 +2568,7 @@ mod _io { macro_rules! set_field { ($field:expr, $off:ident) => {{ let field = $field; - buf[Self::$off..][..std::mem::size_of_val(&field)] + buf[Self::$off..][..core::mem::size_of_val(&field)] .copy_from_slice(&field.to_ne_bytes()) }}; } @@ -2378,6 +2613,7 @@ mod _io { struct TextIOWrapper { _base: _TextIOBase, data: PyThreadMutex<Option<TextIOData>>, + finalizing: AtomicBool, } impl DefaultConstructor for TextIOWrapper {} @@ -2393,28 +2629,37 @@ mod _io { let mut data = zelf.lock_opt(vm)?; *data = None; - let encoding = match args.encoding { - None if vm.state.settings.utf8_mode > 0 => identifier_utf8!(vm, utf_8).to_owned(), - Some(enc) if enc.as_str() != "locale" => enc, - _ => { - // None without utf8_mode or "locale" encoding - vm.import("locale", 0)? - .get_attr("getencoding", vm)? - .call((), vm)? - .try_into_value(vm)? - } - }; + let encoding = Self::resolve_encoding(args.encoding, vm)?; let errors = args .errors .unwrap_or_else(|| identifier!(vm, strict).to_owned()); + Self::validate_errors(&errors, vm)?; let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some(); let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?; - let newline = args.newline.unwrap_or_default(); + let newline = match args.newline { + OptionalArg::Missing => Newlines::default(), + OptionalArg::Present(None) => Newlines::default(), + OptionalArg::Present(Some(newline)) => newline, + }; let (encoder, decoder) = Self::find_coder(&buffer, encoding.as_str(), &errors, newline, vm)?; + if let Some((encoder, _)) = &encoder { + Self::adjust_encoder_state_for_bom(encoder, encoding.as_str(), &buffer, vm)?; + } + + let line_buffering = match args.line_buffering { + OptionalArg::Missing => false, + OptionalArg::Present(None) => false, + OptionalArg::Present(Some(value)) => value.try_to_bool(vm)?, + }; + let write_through = match args.write_through { + OptionalArg::Missing => false, + OptionalArg::Present(None) => false, + OptionalArg::Present(Some(value)) => value.try_to_bool(vm)?, + }; *data = Some(TextIOData { buffer, @@ -2423,8 +2668,8 @@ mod _io { encoding, errors, newline, - line_buffering: args.line_buffering.unwrap_or_default(), - write_through: args.write_through.unwrap_or_default(), + line_buffering, + write_through, chunk_size: 8192, seekable, has_read1, @@ -2439,6 +2684,16 @@ mod _io { Ok(()) } + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let zelf_ref: PyRef<Self> = zelf.try_into_value(vm)?; + { + let mut data = zelf_ref.lock_opt(vm)?; + *data = None; + } + let (buffer, text_args): (PyObjectRef, TextIOWrapperArgs) = args.bind(vm)?; + Self::init(zelf_ref, (buffer, text_args), vm) + } } impl TextIOWrapper { @@ -2457,6 +2712,104 @@ mod _io { .map_err(|_| vm.new_value_error("I/O operation on uninitialized object")) } + fn validate_errors(errors: &PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + if errors.as_wtf8().as_bytes().contains(&0) { + return Err(cstring_error(vm)); + } + if !errors.as_wtf8().is_utf8() { + return Err(vm.new_unicode_encode_error( + "'utf-8' codec can't encode character: surrogates not allowed".to_owned(), + )); + } + vm.state + .codec_registry + .lookup_error(errors.as_str(), vm) + .map(drop) + } + + fn bool_from_index(value: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + let int = value.try_index(vm)?; + let value: i32 = int.try_to_primitive(vm)?; + Ok(value != 0) + } + + fn resolve_encoding( + encoding: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyUtf8StrRef> { + // Note: Do not issue EncodingWarning here. The warning should only + // be issued by io.text_encoding(), the public API. This function + // is used internally (e.g., for stdin/stdout/stderr initialization) + // where no warning should be emitted. + let encoding = match encoding { + None if vm.state.config.settings.utf8_mode > 0 => { + identifier_utf8!(vm, utf_8).to_owned() + } + Some(enc) if enc.as_str() == "locale" => match vm.import("locale", 0) { + Ok(locale) => locale + .get_attr("getencoding", vm)? + .call((), vm)? + .try_into_value(vm)?, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.import_error) + || err.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => + { + identifier_utf8!(vm, utf_8).to_owned() + } + Err(err) => return Err(err), + }, + Some(enc) => { + if enc.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + enc + } + _ => match vm.import("locale", 0) { + Ok(locale) => locale + .get_attr("getencoding", vm)? + .call((), vm)? + .try_into_value(vm)?, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.import_error) + || err.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => + { + identifier_utf8!(vm, utf_8).to_owned() + } + Err(err) => return Err(err), + }, + }; + if encoding.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + Ok(encoding) + } + + fn adjust_encoder_state_for_bom( + encoder: &PyObjectRef, + encoding: &str, + buffer: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let needs_bom = matches!(encoding, "utf-8-sig" | "utf-16" | "utf-32"); + if !needs_bom { + return Ok(()); + } + let seekable = vm.call_method(buffer, "seekable", ())?.try_to_bool(vm)?; + if !seekable { + return Ok(()); + } + let pos = vm.call_method(buffer, "tell", ())?; + if vm.bool_eq(&pos, vm.ctx.new_int(0).as_ref())? { + return Ok(()); + } + if let Err(err) = vm.call_method(encoder, "setstate", (0,)) + && !err.fast_isinstance(vm.ctx.exceptions.attribute_error) + { + return Err(err); + } + Ok(()) + } + #[allow(clippy::type_complexity)] fn find_coder( buffer: &PyObject, @@ -2469,6 +2822,11 @@ mod _io { Option<PyObjectRef>, )> { let codec = vm.state.codec_registry.lookup(encoding, vm)?; + if !codec.is_text_codec(vm)? { + return Err(vm.new_lookup_error(format!( + "'{encoding}' is not a text encoding; use codecs.open() to handle arbitrary codecs" + ))); + } let encoder = if vm.call_method(buffer, "writable", ())?.try_to_bool(vm)? { let incremental_encoder = @@ -2514,39 +2872,115 @@ mod _io { } #[pyclass( - with(Constructor, Initializer, Destructor, Iterable, IterNext), + with( + Constructor, + Initializer, + Destructor, + Iterable, + IterNext, + Representable + ), flags(BASETYPE) )] impl TextIOWrapper { #[pymethod] fn reconfigure(&self, args: TextIOWrapperArgs, vm: &VirtualMachine) -> PyResult<()> { - let mut data = self.data.lock().unwrap(); - if let Some(data) = data.as_mut() { - if let Some(encoding) = args.encoding { - let (encoder, decoder) = Self::find_coder( - &data.buffer, - encoding.as_str(), - &data.errors, - data.newline, - vm, - )?; - data.encoding = encoding; - data.encoder = encoder; - data.decoder = decoder; - } - if let Some(errors) = args.errors { - data.errors = errors; + let mut data = self.lock(vm)?; + data.check_closed(vm)?; + + let mut encoding = data.encoding.clone(); + let mut errors = data.errors.clone(); + let mut newline = data.newline; + let mut encoding_changed = false; + let mut errors_changed = false; + let mut newline_changed = false; + let mut line_buffering = None; + let mut write_through = None; + let mut flush_on_reconfigure = false; + + if let Some(enc) = args.encoding { + if enc.as_str().contains('\0') && enc.as_str().starts_with("locale") { + return Err(vm.new_lookup_error(format!("unknown encoding: {enc}"))); } - if let Some(newline) = args.newline { - data.newline = newline; + let resolved = Self::resolve_encoding(Some(enc), vm)?; + encoding_changed = resolved.as_str() != encoding.as_str(); + encoding = resolved; + } + + if let Some(errs) = args.errors { + Self::validate_errors(&errs, vm)?; + errors_changed = errs.as_str() != errors.as_str(); + errors = errs; + } else if encoding_changed { + errors = identifier!(vm, strict).to_owned(); + errors_changed = true; + } + + if let OptionalArg::Present(nl) = args.newline { + let nl = nl.unwrap_or_default(); + newline_changed = nl != newline; + newline = nl; + } + + if let OptionalArg::Present(Some(value)) = args.line_buffering { + flush_on_reconfigure = true; + line_buffering = Some(Self::bool_from_index(value, vm)?); + } + if let OptionalArg::Present(Some(value)) = args.write_through { + flush_on_reconfigure = true; + write_through = Some(Self::bool_from_index(value, vm)?); + } + + if (encoding_changed || newline_changed) + && data.decoder.is_some() + && (data.decoded_chars.is_some() + || data.snapshot.is_some() + || data.decoded_chars_used.chars != 0) + { + return Err(new_unsupported_operation( + vm, + "cannot reconfigure encoding or newline after reading from the stream" + .to_owned(), + )); + } + + if flush_on_reconfigure { + if data.pending.num_bytes > 0 { + data.write_pending(vm)?; } - if let Some(line_buffering) = args.line_buffering { - data.line_buffering = line_buffering; + vm.call_method(&data.buffer, "flush", ())?; + } + + if encoding_changed || errors_changed || newline_changed { + if data.pending.num_bytes > 0 { + data.write_pending(vm)?; } - if let Some(write_through) = args.write_through { - data.write_through = write_through; + let (encoder, decoder) = + Self::find_coder(&data.buffer, encoding.as_str(), &errors, newline, vm)?; + data.encoding = encoding; + data.errors = errors; + data.newline = newline; + data.encoder = encoder; + data.decoder = decoder; + data.set_decoded_chars(None); + data.snapshot = None; + data.decoded_chars_used = Utf8size::default(); + if let Some((encoder, _)) = &data.encoder { + Self::adjust_encoder_state_for_bom( + encoder, + data.encoding.as_str(), + &data.buffer, + vm, + )?; } } + + if let Some(line_buffering) = line_buffering { + data.line_buffering = line_buffering; + } + if let Some(write_through) = write_through { + data.write_through = write_through; + } Ok(()) } @@ -2983,12 +3417,34 @@ mod _io { } })? }; - if textio.pending.num_bytes + chunk.as_bytes().len() > textio.chunk_size { - textio.write_pending(vm)?; + if textio.pending.num_bytes > 0 + && textio.pending.num_bytes + chunk.as_bytes().len() > textio.chunk_size + { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; + if textio.pending.num_bytes > 0 { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; + } } textio.pending.push(chunk); - if flush || textio.write_through || textio.pending.num_bytes >= textio.chunk_size { - textio.write_pending(vm)?; + if textio.pending.num_bytes > 0 + && (flush || textio.write_through || textio.pending.num_bytes >= textio.chunk_size) + { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; } if flush { let _ = vm.call_method(&textio.buffer, "flush", ()); @@ -3204,6 +3660,10 @@ mod _io { if file_closed(&buffer, vm)? { return Ok(()); } + if zelf.finalizing.load(Ordering::Relaxed) { + // _dealloc_warn: delegate to buffer._dealloc_warn(source) + let _ = vm.call_method(&buffer, "_dealloc_warn", (zelf.as_object().to_owned(),)); + } let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); let close_res = vm.call_method(&buffer, "close", ()).map(drop); exception_chain(flush_res, close_res) @@ -3221,8 +3681,19 @@ mod _io { } #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + #[pymethod] + fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + if zelf.class().is(TextIOWrapper::static_type()) { + return Err( + vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) + ); + } + let _ = proto; + reduce_ex_for_subclass(zelf, vm) } } @@ -3276,7 +3747,7 @@ mod _io { } else { size_hint }; - let chunk_size = std::cmp::max(self.chunk_size, size_hint); + let chunk_size = core::cmp::max(self.chunk_size, size_hint); let input_chunk = vm.call_method(&self.buffer, method, (chunk_size,))?; let buf = ArgBytesLike::try_from_borrowed_object(vm, &input_chunk).map_err(|_| { @@ -3358,8 +3829,8 @@ mod _io { vm: &VirtualMachine, ) -> PyStrRef { let empty_str = || vm.ctx.empty_str.to_owned(); - let chars_pos = std::mem::take(&mut self.decoded_chars_used).bytes; - let decoded_chars = match std::mem::take(&mut self.decoded_chars) { + let chars_pos = core::mem::take(&mut self.decoded_chars_used).bytes; + let decoded_chars = match core::mem::take(&mut self.decoded_chars) { None => return append.unwrap_or_else(empty_str), Some(s) if s.is_empty() => return append.unwrap_or_else(empty_str), Some(s) => s, @@ -3381,7 +3852,10 @@ mod _io { impl Destructor for TextIOWrapper { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + if let Some(wrapper) = zelf.downcast_ref::<TextIOWrapper>() { + wrapper.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); Ok(()) } @@ -3391,9 +3865,59 @@ mod _io { } } - impl Iterable for TextIOWrapper { - fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - check_closed(&zelf, vm)?; + impl Representable for TextIOWrapper { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().slot_name(); + let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) else { + return Err( + vm.new_runtime_error(format!("reentrant call inside {type_name}.__repr__")) + ); + }; + let Some(data) = zelf.data.lock() else { + // Reentrant call + return Ok(format!("<{type_name}>")); + }; + let Some(data) = data.as_ref() else { + return Err(vm.new_value_error("I/O operation on uninitialized object".to_owned())); + }; + + let mut result = format!("<{type_name}"); + + // Add name if present + if let Ok(Some(name)) = vm.get_attribute_opt(data.buffer.clone(), "name") { + let name_repr = name.repr(vm)?; + result.push_str(" name="); + result.push_str(name_repr.as_str()); + } + + // Add mode if present (prefer the wrapper's attribute) + let mode_obj = match vm.get_attribute_opt(zelf.as_object().to_owned(), "mode") { + Ok(Some(mode)) => Some(mode), + Ok(None) | Err(_) => match vm.get_attribute_opt(data.buffer.clone(), "mode") { + Ok(Some(mode)) => Some(mode), + _ => None, + }, + }; + if let Some(mode) = mode_obj { + let mode_repr = mode.repr(vm)?; + result.push_str(" mode="); + result.push_str(mode_repr.as_str()); + } + + // Add encoding + result.push_str(" encoding='"); + result.push_str(data.encoding.as_str()); + result.push('\''); + + result.push('>'); + Ok(result) + } + } + + impl Iterable for TextIOWrapper { + fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + check_closed(&zelf, vm)?; Ok(zelf) } @@ -3674,23 +4198,31 @@ mod _io { } impl Constructor for StringIO { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + _base: Default::default(), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), + closed: AtomicCell::new(false), + }) + } + } + + impl Initializer for StringIO { type Args = StringIONewArgs; #[allow(unused_variables)] - fn py_new( - _cls: &Py<PyType>, + fn init( + zelf: PyRef<Self>, Self::Args { object, newline }: Self::Args, _vm: &VirtualMachine, - ) -> PyResult<Self> { + ) -> PyResult<()> { let raw_bytes = object .flatten() .map_or_else(Vec::new, |v| v.as_bytes().to_vec()); - - Ok(Self { - _base: Default::default(), - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), - closed: AtomicCell::new(false), - }) + *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); + Ok(()) } } @@ -3704,7 +4236,7 @@ mod _io { } } - #[pyclass(flags(BASETYPE, HAS_DICT), with(Constructor))] + #[pyclass(flags(BASETYPE, HAS_DICT), with(Constructor, Initializer))] impl StringIO { #[pymethod] const fn readable(&self) -> bool { @@ -3796,6 +4328,77 @@ mod _io { const fn line_buffering(&self) -> bool { false } + + #[pymethod] + fn __getstate__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let buffer = zelf.buffer(vm)?; + let content = Wtf8Buf::from_bytes(buffer.getvalue()) + .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; + let pos = buffer.tell(); + drop(buffer); + + // Get __dict__ if it exists and is non-empty + let dict_obj: PyObjectRef = match zelf.as_object().dict() { + Some(d) if !d.is_empty() => d.into(), + _ => vm.ctx.none(), + }; + + // Return (content, newline, position, dict) + // TODO: store actual newline setting when it's implemented + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_str(content).into(), + vm.ctx.new_str("\n").into(), + vm.ctx.new_int(pos).into(), + dict_obj, + ])) + } + + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + // Check closed state first (like CHECK_CLOSED) + if zelf.closed.load() { + return Err(vm.new_value_error("__setstate__ on closed file")); + } + if state.len() != 4 { + return Err(vm.new_type_error(format!( + "__setstate__ argument should be 4-tuple, got {}", + state.len() + ))); + } + + let content: PyStrRef = state[0].clone().try_into_value(vm)?; + // state[1] is newline - TODO: use when newline handling is implemented + let pos: u64 = state[2].clone().try_into_value(vm)?; + let dict = &state[3]; + + // Set content and position + let raw_bytes = content.as_bytes().to_vec(); + let mut buffer = zelf.buffer.write(); + *buffer = BufferedIO::new(Cursor::new(raw_bytes)); + buffer + .seek(SeekFrom::Start(pos)) + .map_err(|err| os_err(vm, err))?; + drop(buffer); + + // Set __dict__ if provided + if !vm.is_none(dict) { + let dict_ref: PyRef<PyDict> = dict.clone().try_into_value(vm)?; + if let Some(obj_dict) = zelf.as_object().dict() { + obj_dict.clear(); + for (key, value) in dict_ref.into_iter() { + obj_dict.set_item(&*key, value, vm)?; + } + } + } + + Ok(()) + } + } + + #[derive(FromArgs)] + struct BytesIOArgs { + #[pyarg(any, optional)] + initial_bytes: OptionalArg<Option<ArgBytesLike>>, } #[pyattr] @@ -3809,22 +4412,37 @@ mod _io { } impl Constructor for BytesIO { - type Args = OptionalArg<Option<PyBytesRef>>; - - fn py_new(_cls: &Py<PyType>, object: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - let raw_bytes = object - .flatten() - .map_or_else(Vec::new, |input| input.as_bytes().to_vec()); + type Args = FuncArgs; + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { Ok(Self { _base: Default::default(), - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), closed: AtomicCell::new(false), exports: AtomicCell::new(0), }) } } + impl Initializer for BytesIO { + type Args = BytesIOArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + if zelf.exports.load() > 0 { + return Err(vm.new_buffer_error( + "Existing exports of data: object cannot be re-sized".to_owned(), + )); + } + + let raw_bytes = args + .initial_bytes + .flatten() + .map_or_else(Vec::new, |input| input.borrow_buf().to_vec()); + *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); + Ok(()) + } + } + impl BytesIO { fn buffer(&self, vm: &VirtualMachine) -> PyResult<PyRwLockWriteGuard<'_, BufferedIO>> { if !self.closed.load() { @@ -3835,7 +4453,7 @@ mod _io { } } - #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor))] + #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor, Initializer))] impl BytesIO { #[pymethod] const fn readable(&self) -> bool { @@ -3895,9 +4513,20 @@ mod _io { how: OptionalArg<i32>, vm: &VirtualMachine, ) -> PyResult<u64> { - self.buffer(vm)? - .seek(seekfrom(vm, offset, how)?) - .map_err(|err| os_err(vm, err)) + let seek_from = seekfrom(vm, offset, how)?; + let mut buffer = self.buffer(vm)?; + + // Handle negative positions by clamping to 0 + match seek_from { + SeekFrom::Current(offset) if offset < 0 => { + let current = buffer.tell(); + let new_pos = current.saturating_add_signed(offset); + buffer + .seek(SeekFrom::Start(new_pos)) + .map_err(|err| os_err(vm, err)) + } + _ => buffer.seek(seek_from).map_err(|err| os_err(vm, err)), + } } #[pymethod] @@ -3927,16 +4556,82 @@ mod _io { #[pymethod] fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - drop(self.try_resizable(vm)?); + if self.exports.load() > 0 { + return Err(vm.new_buffer_error( + "Existing exports of data: object cannot be closed".to_owned(), + )); + } self.closed.store(true); Ok(()) } + + #[pymethod] + fn __getstate__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let buffer = zelf.buffer(vm)?; + let content = buffer.getvalue(); + let pos = buffer.tell(); + drop(buffer); + + // Get __dict__ if it exists and is non-empty + let dict_obj: PyObjectRef = match zelf.as_object().dict() { + Some(d) if !d.is_empty() => d.into(), + _ => vm.ctx.none(), + }; + + // Return (content, position, dict) + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytes(content).into(), + vm.ctx.new_int(pos).into(), + dict_obj, + ])) + } + + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + if zelf.closed.load() { + return Err(vm.new_value_error("__setstate__ on closed file")); + } + if state.len() != 3 { + return Err(vm.new_type_error(format!( + "__setstate__ argument should be 3-tuple, got {}", + state.len() + ))); + } + + let content: PyBytesRef = state[0].clone().try_into_value(vm)?; + let pos: u64 = state[1].clone().try_into_value(vm)?; + let dict = &state[2]; + + // Check exports and set content (like CHECK_EXPORTS) + let mut buffer = zelf.try_resizable(vm)?; + *buffer = BufferedIO::new(Cursor::new(content.as_bytes().to_vec())); + buffer + .seek(SeekFrom::Start(pos)) + .map_err(|err| os_err(vm, err))?; + drop(buffer); + + // Set __dict__ if provided + if !vm.is_none(dict) { + let dict_ref: PyRef<PyDict> = dict.clone().try_into_value(vm)?; + if let Some(obj_dict) = zelf.as_object().dict() { + obj_dict.clear(); + for (key, value) in dict_ref.into_iter() { + obj_dict.set_item(&*key, value, vm)?; + } + } + } + + Ok(()) + } } #[pyclass] impl PyRef<BytesIO> { #[pymethod] fn getbuffer(self, vm: &VirtualMachine) -> PyResult<PyMemoryView> { + if self.closed.load() { + return Err(vm.new_value_error("I/O operation on closed file.".to_owned())); + } let len = self.buffer.read().cursor.get_ref().len(); let buffer = PyBuffer::new( self.into(), @@ -4000,7 +4695,7 @@ mod _io { plus: bool, } - impl std::str::FromStr for Mode { + impl core::str::FromStr for Mode { type Err = ParseModeError; fn from_str(s: &str) -> Result<Self, Self::Err> { @@ -4162,16 +4857,28 @@ mod _io { } // check file descriptor validity - #[cfg(unix)] + #[cfg(all(unix, feature = "host_env"))] if let Ok(crate::ospath::OsPathOrFd::Fd(fd)) = file.clone().try_into_value(vm) { nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD).map_err(|_| vm.new_last_errno_error())?; } - // Construct a FileIO (subclass of RawIOBase) + // Construct a RawIO (subclass of RawIOBase) + // On Windows, use _WindowsConsoleIO for console handles. // This is subsequently consumed by a Buffered Class. + #[cfg(all(feature = "host_env", windows))] + let is_console = super::winconsoleio::pyio_get_console_type(&file, vm) != '\0'; + #[cfg(not(all(feature = "host_env", windows)))] + let is_console = false; + let file_io_class: &Py<PyType> = { cfg_if::cfg_if! { - if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { + if #[cfg(all(feature = "host_env", windows))] { + if is_console { + Some(super::winconsoleio::WindowsConsoleIO::static_type()) + } else { + Some(super::fileio::FileIO::static_type()) + } + } else if #[cfg(feature = "host_env")] { Some(super::fileio::FileIO::static_type()) } else { None @@ -4195,6 +4902,16 @@ mod _io { bool::try_from_object(vm, atty)? }; + // Warn if line buffering is requested in binary mode + if opts.buffering == 1 && matches!(mode.encode, EncodeMode::Bytes) { + crate::stdlib::warnings::warn( + vm.ctx.exceptions.runtime_warning, + "line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used".to_owned(), + 1, + vm, + )?; + } + let line_buffering = opts.buffering == 1 || isatty; let buffering = if opts.buffering < 0 || opts.buffering == 1 { @@ -4205,7 +4922,10 @@ mod _io { if buffering == 0 { let ret = match mode.encode { - EncodeMode::Text => Err(vm.new_value_error("can't have unbuffered text I/O")), + EncodeMode::Text => { + let _ = vm.call_method(&raw, "close", ()); + Err(vm.new_value_error("can't have unbuffered text I/O")) + } EncodeMode::Bytes => Ok(raw), }; return ret; @@ -4222,19 +4942,35 @@ mod _io { match mode.encode { EncodeMode::Text => { + let encoding = if is_console && opts.encoding.is_none() { + // Console IO always uses utf-8 + Some(PyUtf8Str::from("utf-8").into_ref(&vm.ctx)) + } else { + match opts.encoding { + Some(enc) => Some(enc), + None => { + let encoding = + text_encoding(vm.ctx.none(), OptionalArg::Present(2), vm)?; + Some(PyUtf8StrRef::try_from_object(vm, encoding.into())?) + } + } + }; let tio = TextIOWrapper::static_type(); let wrapper = PyType::call( tio, ( - buffered, - opts.encoding, + buffered.clone(), + encoding, opts.errors, opts.newline, line_buffering, ) .into_args(vm), vm, - )?; + ) + .inspect_err(|_err| { + let _ = vm.call_method(&buffered, "close", ()); + })?; wrapper.set_attr("mode", vm.new_pyobj(mode_string), vm)?; Ok(wrapper) } @@ -4243,14 +4979,19 @@ mod _io { } fn create_unsupported_operation(ctx: &Context) -> PyTypeRef { + use crate::builtins::type_::PyAttributes; use crate::types::PyTypeSlots; + + let mut attrs = PyAttributes::default(); + attrs.insert(identifier!(ctx, __module__), ctx.new_str("io").into()); + PyType::new_heap( "UnsupportedOperation", vec![ ctx.exceptions.os_error.to_owned(), ctx.exceptions.value_error.to_owned(), ], - Default::default(), + attrs, PyTypeSlots::heap_default(), ctx.types.type_type.to_owned(), ctx, @@ -4268,12 +5009,35 @@ mod _io { #[pyfunction] fn text_encoding( encoding: PyObjectRef, - _stacklevel: OptionalArg<i32>, + stacklevel: OptionalArg<i32>, vm: &VirtualMachine, ) -> PyResult<PyStrRef> { if vm.is_none(&encoding) { - // TODO: This is `locale` encoding - but we don't have locale encoding yet - return Ok(vm.ctx.new_str("utf-8")); + let encoding = if vm.state.config.settings.utf8_mode > 0 { + "utf-8" + } else { + "locale" + }; + if vm.state.config.settings.warn_default_encoding { + let mut stacklevel = stacklevel.unwrap_or(2); + if stacklevel > 1 + && let Some(frame) = vm.current_frame() + && let Some(stdlib_dir) = vm.state.config.paths.stdlib_dir.as_deref() + { + let path = frame.code.source_path().as_str(); + if !path.starts_with(stdlib_dir) { + stacklevel = stacklevel.saturating_sub(1); + } + } + let stacklevel = usize::try_from(stacklevel).unwrap_or(0); + crate::stdlib::warnings::warn( + vm.ctx.exceptions.encoding_warning, + "'encoding' argument not specified".to_owned(), + stacklevel, + vm, + )?; + } + return Ok(vm.ctx.new_str(encoding)); } encoding.try_into_value(vm) } @@ -4315,21 +5079,41 @@ mod _io { assert_eq!(buffered.getvalue(), data); } } -} -// disable FileIO on WASM -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // Call auto-generated initialization first + __module_exec(vm, module); + + // Initialize FileIO types (requires host_env for filesystem access) + #[cfg(feature = "host_env")] + super::fileio::module_exec(vm, module)?; + + // Initialize WindowsConsoleIO type (Windows only) + #[cfg(all(feature = "host_env", windows))] + super::winconsoleio::module_exec(vm, module)?; + + let unsupported_operation = unsupported_operation().to_owned(); + extend_module!(vm, module, { + "UnsupportedOperation" => unsupported_operation, + "BlockingIOError" => vm.ctx.exceptions.blocking_io_error.to_owned(), + }); + Ok(()) + } +} +// FileIO requires host environment for filesystem access +#[cfg(feature = "host_env")] #[pymodule] mod fileio { - use super::{_io::*, Offset}; + use super::{_io::*, Offset, iobase_finalize}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyUtf8Str, PyUtf8StrRef}, common::crt_fd, convert::{IntoPyException, ToPyException}, + exceptions::OSErrorBuilder, function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg, OptionalOption}, - ospath::{IOErrorBuilder, OsPath, OsPathOrFd}, + ospath::{OsPath, OsPathOrFd}, stdlib::os, types::{Constructor, DefaultConstructor, Destructor, Initializer, Representable}, }; @@ -4438,7 +5222,7 @@ mod fileio { } #[pyattr] - #[pyclass(module = "io", name, base = _RawIOBase)] + #[pyclass(module = "_io", name, base = _RawIOBase)] #[derive(Debug)] pub(super) struct FileIO { _base: _RawIOBase, @@ -4446,6 +5230,8 @@ mod fileio { closefd: AtomicCell<bool>, mode: AtomicCell<Mode>, seekable: AtomicCell<Option<bool>>, + blksize: AtomicCell<i64>, + finalizing: AtomicCell<bool>, } #[derive(FromArgs)] @@ -4468,6 +5254,8 @@ mod fileio { closefd: AtomicCell::new(true), mode: AtomicCell::new(Mode::empty()), seekable: AtomicCell::new(None), + blksize: AtomicCell::new(super::DEFAULT_BUFFER_SIZE as _), + finalizing: AtomicCell::new(false), } } } @@ -4480,6 +5268,15 @@ mod fileio { fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { // TODO: let atomic_flag_works let name = args.name; + // Check if bool is used as file descriptor + if name.class().is(vm.ctx.types.bool_type) { + crate::stdlib::warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } let arg_fd = if let Some(i) = name.downcast_ref::<crate::builtins::PyInt>() { let fd = i.try_to_primitive(vm)?; if fd < 0 { @@ -4526,10 +5323,13 @@ mod fileio { let filename = OsPathOrFd::Path(path); match fd { Ok(fd) => (fd.into_raw(), Some(filename)), - Err(e) => return Err(IOErrorBuilder::with_filename(&e, filename, vm)), + Err(e) => { + return Err(OSErrorBuilder::with_filename_from_errno(&e, filename, vm)); + } } } }; + let fd_is_own = arg_fd.is_none(); zelf.fd.store(fd); let fd = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; let filename = filename.unwrap_or(OsPathOrFd::Fd(fd)); @@ -4541,7 +5341,13 @@ mod fileio { #[cfg(windows)] { if let Err(err) = fd_fstat { - return Err(IOErrorBuilder::with_filename(&err, filename, vm)); + // If the fd is invalid, prevent destructor from trying to close it + if err.raw_os_error() + == Some(windows_sys::Win32::Foundation::ERROR_INVALID_HANDLE as i32) + { + zelf.fd.store(-1); + } + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); } } #[cfg(any(unix, target_os = "wasi"))] @@ -4549,13 +5355,27 @@ mod fileio { match fd_fstat { Ok(status) => { if (status.st_mode & libc::S_IFMT) == libc::S_IFDIR { + // If fd was passed by user, don't close it on error + if !fd_is_own { + zelf.fd.store(-1); + } let err = std::io::Error::from_raw_os_error(libc::EISDIR); - return Err(IOErrorBuilder::with_filename(&err, filename, vm)); + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); + } + // Store st_blksize for _blksize property + if status.st_blksize > 1 { + #[allow( + clippy::useless_conversion, + reason = "needed for 32-bit platforms" + )] + zelf.blksize.store(i64::from(status.st_blksize)); } } Err(err) => { if err.raw_os_error() == Some(libc::EBADF) { - return Err(IOErrorBuilder::with_filename(&err, filename, vm)); + // fd is invalid, prevent destructor from trying to close it + zelf.fd.store(-1); + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); } } } @@ -4563,7 +5383,13 @@ mod fileio { #[cfg(windows)] crate::stdlib::msvcrt::setmode_binary(fd); - zelf.as_object().set_attr("name", name, vm)?; + if let Err(e) = zelf.as_object().set_attr("name", name, vm) { + // If fd was passed by user, don't close it on error + if !fd_is_own { + zelf.fd.store(-1); + } + return Err(e); + } if mode.contains(Mode::APPENDING) { let _ = os::lseek(fd, 0, libc::SEEK_END, vm); @@ -4576,17 +5402,18 @@ mod fileio { impl Representable for FileIO { #[inline] fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().slot_name(); let fd = zelf.fd.load(); if fd < 0 { - return Ok("<_io.FileIO [closed]>".to_owned()); + return Ok(format!("<{type_name} [closed]>")); } let name_repr = repr_file_obj_name(zelf.as_object(), vm)?; let mode = zelf.mode(); let closefd = if zelf.closefd.load() { "True" } else { "False" }; let repr = if let Some(name_repr) = name_repr { - format!("<_io.FileIO name={name_repr} mode='{mode}' closefd={closefd}>") + format!("<{type_name} name={name_repr} mode='{mode}' closefd={closefd}>") } else { - format!("<_io.FileIO fd={fd} mode='{mode}' closefd={closefd}>") + format!("<{type_name} fd={fd} mode='{mode}' closefd={closefd}>") }; Ok(repr) } @@ -4621,6 +5448,11 @@ mod fileio { self.closefd.load() } + #[pygetset(name = "_blksize")] + fn blksize(&self) -> i64 { + self.blksize.load() + } + #[pymethod] fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { let fd = self.fd.load(); @@ -4637,13 +5469,19 @@ mod fileio { } #[pymethod] - fn readable(&self) -> bool { - self.mode.load().contains(Mode::READABLE) + fn readable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.mode.load().contains(Mode::READABLE)) } #[pymethod] - fn writable(&self) -> bool { - self.mode.load().contains(Mode::WRITABLE) + fn writable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.mode.load().contains(Mode::WRITABLE)) } #[pygetset] @@ -4677,7 +5515,7 @@ mod fileio { zelf: &Py<Self>, read_byte: OptionalSize, vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { + ) -> PyResult<Option<Vec<u8>>> { if !zelf.mode.load().contains(Mode::READABLE) { return Err(new_unsupported_operation( vm, @@ -4687,24 +5525,55 @@ mod fileio { let mut handle = zelf.get_fd(vm)?; let bytes = if let Some(read_byte) = read_byte.to_usize() { let mut bytes = vec![0; read_byte]; - let n = handle - .read(&mut bytes) - .map_err(|err| Self::io_error(zelf, err, vm))?; + // Loop on EINTR (PEP 475) + let n = loop { + match handle.read(&mut bytes) { + Ok(n) => break n, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + }; bytes.truncate(n); bytes } else { let mut bytes = vec![]; - handle - .read_to_end(&mut bytes) - .map_err(|err| Self::io_error(zelf, err, vm))?; + // Loop on EINTR (PEP 475) + loop { + match handle.read_to_end(&mut bytes) { + Ok(_) => break, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN (only if no data read yet) + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + if bytes.is_empty() { + return Ok(None); + } + break; + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + } bytes }; - Ok(bytes) + Ok(Some(bytes)) } #[pymethod] - fn readinto(zelf: &Py<Self>, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { + fn readinto( + zelf: &Py<Self>, + obj: ArgMemoryBuffer, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { if !zelf.mode.load().contains(Mode::READABLE) { return Err(new_unsupported_operation( vm, @@ -4716,15 +5585,31 @@ mod fileio { let mut buf = obj.borrow_buf_mut(); let mut f = handle.take(buf.len() as _); - let ret = f - .read(&mut buf) - .map_err(|err| Self::io_error(zelf, err, vm))?; + // Loop on EINTR (PEP 475) + let ret = loop { + match f.read(&mut buf) { + Ok(n) => break n, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + }; - Ok(ret) + Ok(Some(ret)) } #[pymethod] - fn write(zelf: &Py<Self>, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + fn write( + zelf: &Py<Self>, + obj: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { if !zelf.mode.load().contains(Mode::WRITABLE) { return Err(new_unsupported_operation( vm, @@ -4734,12 +5619,15 @@ mod fileio { let mut handle = zelf.get_fd(vm)?; - let len = obj - .with_ref(|b| handle.write(b)) - .map_err(|err| Self::io_error(zelf, err, vm))?; + let len = match obj.with_ref(|b| handle.write(b)) { + Ok(n) => n, + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => return Ok(None), + Err(e) => return Err(Self::io_error(zelf, e, vm)), + }; //return number of bytes written - Ok(len) + Ok(Some(len)) } #[pymethod] @@ -4749,12 +5637,26 @@ mod fileio { zelf.fd.store(-1); return res; } + let flush_exc = res.err(); + if zelf.finalizing.load() { + Self::dealloc_warn(zelf, zelf.as_object().to_owned(), vm); + } let fd = zelf.fd.swap(-1); - if fd >= 0 { + let close_err = if fd >= 0 { crt_fd::close(unsafe { crt_fd::Owned::from_raw(fd) }) - .map_err(|err| Self::io_error(zelf, err, vm))?; + .map_err(|err| Self::io_error(zelf, err, vm)) + .err() + } else { + None + }; + match (flush_exc, close_err) { + (Some(fe), Some(ce)) => { + ce.set___context__(Some(fe)); + Err(ce) + } + (Some(e), None) | (None, Some(e)) => Err(e), + (None, None) => Ok(()), } - res } #[pymethod] @@ -4805,14 +5707,1096 @@ mod fileio { } #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + /// fileio_dealloc_warn in Modules/_io/fileio.c + #[pymethod(name = "_dealloc_warn")] + fn _dealloc_warn_method( + zelf: &Py<Self>, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::dealloc_warn(zelf, source, vm); + Ok(()) + } + } + + impl FileIO { + /// Issue ResourceWarning if fd is still open and closefd is true. + fn dealloc_warn(zelf: &Py<Self>, source: PyObjectRef, vm: &VirtualMachine) { + if zelf.fd.load() >= 0 && zelf.closefd.load() { + let repr = source + .repr(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| "<file>".to_owned()); + if let Err(e) = crate::stdlib::warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed file {repr}"), + 1, + vm, + ) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + } } } impl Destructor for FileIO { fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); + if let Some(fileio) = zelf.downcast_ref::<FileIO>() { + fileio.finalizing.store(true); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } +} + +// WindowsConsoleIO requires host environment and Windows +#[cfg(all(feature = "host_env", windows))] +#[pymodule] +mod winconsoleio { + use super::{_io::*, iobase_finalize}; + use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyUtf8StrRef}, + common::lock::PyMutex, + convert::{IntoPyException, ToPyException}, + function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg}, + types::{Constructor, DefaultConstructor, Destructor, Initializer, Representable}, + }; + use crossbeam_utils::atomic::AtomicCell; + use windows_sys::Win32::{ + Foundation::{self, GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, + Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, + Storage::FileSystem::{ + CreateFileW, FILE_SHARE_READ, FILE_SHARE_WRITE, GetFullPathNameW, OPEN_EXISTING, + }, + System::Console::{ + GetConsoleMode, GetNumberOfConsoleInputEvents, ReadConsoleW, WriteConsoleW, + }, + }; + + type HANDLE = Foundation::HANDLE; + + const SMALLBUF: usize = 4; + const BUFMAX: usize = 32 * 1024 * 1024; + + fn handle_from_fd(fd: i32) -> HANDLE { + unsafe { rustpython_common::suppress_iph!(libc::get_osfhandle(fd)) as HANDLE } + } + + fn is_invalid_handle(handle: HANDLE) -> bool { + handle == INVALID_HANDLE_VALUE || handle.is_null() + } + + /// Check if a HANDLE is a console and what type ('r', 'w', or '\0'). + fn get_console_type(handle: HANDLE) -> char { + if is_invalid_handle(handle) { + return '\0'; + } + let mut mode: u32 = 0; + if unsafe { GetConsoleMode(handle, &mut mode) } == 0 { + return '\0'; + } + let mut peek_count: u32 = 0; + if unsafe { GetNumberOfConsoleInputEvents(handle, &mut peek_count) } != 0 { + 'r' + } else { + 'w' + } + } + + /// Check if a Python object (fd or path string) refers to a console. + /// Returns 'r' (input), 'w' (output), 'x' (generic CON), or '\0' (not a console). + pub(super) fn pyio_get_console_type(path_or_fd: &PyObject, vm: &VirtualMachine) -> char { + // Try as integer fd first + if let Ok(fd) = i32::try_from_object(vm, path_or_fd.to_owned()) { + if fd >= 0 { + let handle = handle_from_fd(fd); + return get_console_type(handle); + } + return '\0'; + } + + // Try as string path + let Ok(name) = path_or_fd.str(vm) else { + return '\0'; + }; + let Some(name_str) = name.to_str() else { + // Surrogate strings can't be console device names + return '\0'; + }; + + if name_str.eq_ignore_ascii_case("CONIN$") { + return 'r'; + } + if name_str.eq_ignore_ascii_case("CONOUT$") { + return 'w'; + } + if name_str.eq_ignore_ascii_case("CON") { + return 'x'; + } + + // Resolve full path and check for console device names + let wide: Vec<u16> = name_str.encode_utf16().chain(core::iter::once(0)).collect(); + let mut buf = [0u16; 260]; // MAX_PATH + let length = unsafe { + GetFullPathNameW( + wide.as_ptr(), + buf.len() as u32, + buf.as_mut_ptr(), + core::ptr::null_mut(), + ) + }; + if length == 0 || length as usize > buf.len() { + return '\0'; + } + let full_path = &buf[..length as usize]; + // Skip \\?\ or \\.\ prefix + let path_part = if full_path.len() >= 4 + && full_path[0] == b'\\' as u16 + && full_path[1] == b'\\' as u16 + && (full_path[2] == b'.' as u16 || full_path[2] == b'?' as u16) + && full_path[3] == b'\\' as u16 + { + &full_path[4..] + } else { + full_path + }; + + let path_str = String::from_utf16_lossy(path_part); + if path_str.eq_ignore_ascii_case("CONIN$") { + 'r' + } else if path_str.eq_ignore_ascii_case("CONOUT$") { + 'w' + } else if path_str.eq_ignore_ascii_case("CON") { + 'x' + } else { + '\0' + } + } + + /// Find the last valid UTF-8 boundary in a byte slice. + fn find_last_utf8_boundary(buf: &[u8], len: usize) -> usize { + let len = len.min(buf.len()); + for count in 1..=4.min(len) { + let c = buf[len - count]; + if c < 0x80 { + return len; + } + if c >= 0xc0 { + let expected = if c < 0xe0 { + 2 + } else if c < 0xf0 { + 3 + } else { + 4 + }; + if count < expected { + // Incomplete multibyte sequence + return len - count; + } + return len; + } + } + len + } + + #[pyattr] + #[pyclass(module = "_io", name = "_WindowsConsoleIO", base = _RawIOBase)] + #[derive(Debug)] + pub(super) struct WindowsConsoleIO { + _base: _RawIOBase, + fd: AtomicCell<i32>, + readable: AtomicCell<bool>, + writable: AtomicCell<bool>, + closefd: AtomicCell<bool>, + finalizing: AtomicCell<bool>, + blksize: AtomicCell<i64>, + buf: PyMutex<[u8; SMALLBUF]>, + } + + impl Default for WindowsConsoleIO { + fn default() -> Self { + Self { + _base: Default::default(), + fd: AtomicCell::new(-1), + readable: AtomicCell::new(false), + writable: AtomicCell::new(false), + closefd: AtomicCell::new(false), + finalizing: AtomicCell::new(false), + blksize: AtomicCell::new(super::DEFAULT_BUFFER_SIZE as _), + buf: PyMutex::new([0u8; SMALLBUF]), + } + } + } + + impl DefaultConstructor for WindowsConsoleIO {} + + #[derive(FromArgs)] + pub struct WindowsConsoleIOArgs { + #[pyarg(positional)] + name: PyObjectRef, + #[pyarg(any, default)] + mode: Option<PyUtf8StrRef>, + #[pyarg(any, default = true)] + closefd: bool, + #[allow(dead_code)] + #[pyarg(any, default)] + opener: Option<PyObjectRef>, + } + + impl Initializer for WindowsConsoleIO { + type Args = WindowsConsoleIOArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let nameobj = args.name; + + if zelf.fd.load() >= 0 { + if zelf.closefd.load() { + internal_close(&zelf); + } else { + zelf.fd.store(-1); + } + } + + // Warn if bool is used as file descriptor + if nameobj.class().is(vm.ctx.types.bool_type) { + crate::stdlib::warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + + // Try to get fd from integer + let mut fd: i32 = -1; + if let Some(i) = nameobj.downcast_ref::<crate::builtins::PyInt>() { + fd = i.try_to_primitive::<i32>(vm).unwrap_or(-1); + if fd < 0 { + return Err(vm.new_value_error("negative file descriptor")); + } + } + + // Parse mode + let mode_str: &str = args + .mode + .as_ref() + .map(|s: &PyUtf8StrRef| s.as_str()) + .unwrap_or("r"); + + let mut rwa = false; + let mut readable = false; + let mut writable = false; + let mut console_type = '\0'; + for c in mode_str.bytes() { + match c { + b'+' | b'a' | b'b' | b'x' => {} + b'r' => { + if rwa { + return Err( + vm.new_value_error("Must have exactly one of read or write mode") + ); + } + rwa = true; + readable = true; + } + b'w' => { + if rwa { + return Err( + vm.new_value_error("Must have exactly one of read or write mode") + ); + } + rwa = true; + writable = true; + } + _ => { + return Err(vm.new_value_error(format!("invalid mode: {mode_str}"))); + } + } + } + if !rwa { + return Err(vm.new_value_error("Must have exactly one of read or write mode")); + } + + zelf.readable.store(readable); + zelf.writable.store(writable); + + let mut _name_wide: Option<Vec<u16>> = None; + + if fd < 0 { + // Get console type from name + console_type = pyio_get_console_type(&nameobj, vm); + if console_type == 'x' { + if writable { + console_type = 'w'; + } else { + console_type = 'r'; + } + } + + // Opening by name + zelf.closefd.store(true); + if !args.closefd { + return Err(vm.new_value_error("Cannot use closefd=False with file name")); + } + + let name_str = nameobj.str(vm)?; + let wide: Vec<u16> = name_str + .as_str() + .encode_utf16() + .chain(core::iter::once(0)) + .collect(); + + let access = if writable { + GENERIC_WRITE + } else { + GENERIC_READ + }; + + // Try read/write first, fall back to specific access + let mut handle: HANDLE = unsafe { + CreateFileW( + wide.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + 0, + core::ptr::null_mut(), + ) + }; + if is_invalid_handle(handle) { + handle = unsafe { + CreateFileW( + wide.as_ptr(), + access, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + 0, + core::ptr::null_mut(), + ) + }; + } + + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let osf_flags = if writable { + libc::O_WRONLY | libc::O_BINARY | 0x80 /* O_NOINHERIT */ + } else { + libc::O_RDONLY | libc::O_BINARY | 0x80 /* O_NOINHERIT */ + }; + + fd = unsafe { libc::open_osfhandle(handle as isize, osf_flags) }; + if fd < 0 { + unsafe { + Foundation::CloseHandle(handle); + } + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + _name_wide = Some(wide); + } else { + // When opened by fd, never close the fd (user owns it) + zelf.closefd.store(false); + } + + zelf.fd.store(fd); + + // Validate console type + if console_type == '\0' { + let handle = handle_from_fd(fd); + console_type = get_console_type(handle); + } + + if console_type == '\0' { + // Not a console at all + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open non-console file")); + } + + if writable && console_type != 'w' { + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open console input buffer for writing")); + } + if readable && console_type != 'r' { + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open console output buffer for reading")); + } + + zelf.blksize.store(super::DEFAULT_BUFFER_SIZE as _); + *zelf.buf.lock() = [0u8; SMALLBUF]; + + zelf.as_object().set_attr("name", nameobj, vm)?; + + Ok(()) + } + } + + fn internal_close(zelf: &WindowsConsoleIO) { + let fd = zelf.fd.swap(-1); + if fd >= 0 && zelf.closefd.load() { + unsafe { + libc::close(fd); + } + } + } + + impl Representable for WindowsConsoleIO { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().slot_name(); + let fd = zelf.fd.load(); + if fd < 0 { + return Ok(format!("<{type_name} [closed]>")); + } + let mode = if zelf.readable.load() { "rb" } else { "wb" }; + let closefd = if zelf.closefd.load() { "True" } else { "False" }; + Ok(format!("<{type_name} mode='{mode}' closefd={closefd}>")) + } + } + + #[pyclass( + with(Constructor, Initializer, Representable, Destructor), + flags(BASETYPE, HAS_DICT) + )] + impl WindowsConsoleIO { + #[allow(dead_code)] + fn io_error( + zelf: &Py<Self>, + error: std::io::Error, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let exc = error.to_pyexception(vm); + if let Ok(name) = zelf.as_object().get_attr("name", vm) { + exc.as_object() + .set_attr("filename", name, vm) + .expect("OSError.filename set must succeed"); + } + exc + } + + #[pygetset] + fn closed(&self) -> bool { + self.fd.load() < 0 + } + + #[pygetset] + fn closefd(&self) -> bool { + self.closefd.load() + } + + #[pygetset(name = "_blksize")] + fn blksize(&self) -> i64 { + self.blksize.load() + } + + #[pygetset] + fn mode(&self) -> &'static str { + if self.readable.load() { "rb" } else { "wb" } + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { + let fd = self.fd.load(); + if fd >= 0 { + Ok(fd) + } else { + Err(io_closed_error(vm)) + } + } + + fn get_fd(&self, vm: &VirtualMachine) -> PyResult<i32> { + self.fileno(vm) + } + + #[pymethod] + fn readable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.readable.load()) + } + + #[pymethod] + fn writable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.writable.load()) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(true) + } + + #[pymethod] + fn close(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let res = iobase_close(zelf.as_object(), vm); + if !zelf.closefd.load() { + zelf.fd.store(-1); + return res; + } + let flush_exc = res.err(); + if zelf.finalizing.load() { + Self::dealloc_warn(zelf, zelf.as_object().to_owned(), vm); + } + let fd = zelf.fd.swap(-1); + let close_err: Option<PyBaseExceptionRef> = if fd >= 0 { + let result = unsafe { libc::close(fd) }; + if result < 0 { + Some(std::io::Error::last_os_error().into_pyexception(vm)) + } else { + None + } + } else { + None + }; + match (flush_exc, close_err) { + (Some(fe), Some(ce)) => { + ce.set___context__(Some(fe)); + Err(ce) + } + (Some(e), None) | (None, Some(e)) => Err(e), + (None, None) => Ok(()), + } + } + + fn dealloc_warn(zelf: &Py<Self>, source: PyObjectRef, vm: &VirtualMachine) { + if zelf.fd.load() >= 0 && zelf.closefd.load() { + let repr = source + .repr(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|_| "<file>".to_owned()); + if let Err(e) = crate::stdlib::warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed file {repr}"), + 1, + vm, + ) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + } + } + + fn copy_from_buf(buf: &mut [u8; SMALLBUF], dest: &mut [u8]) -> usize { + let mut n = 0; + while buf[0] != 0 && n < dest.len() { + dest[n] = buf[0]; + n += 1; + for i in 1..SMALLBUF { + buf[i - 1] = buf[i]; + } + buf[SMALLBUF - 1] = 0; + } + n + } + + #[pymethod] + fn readinto(&self, buffer: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { + let fd = self.get_fd(vm)?; + if !self.readable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support reading".to_owned(), + )); + } + let mut buf_ref = buffer.borrow_buf_mut(); + let len = buf_ref.len(); + if len == 0 { + return Ok(0); + } + if len > BUFMAX { + return Err(vm.new_value_error(format!("cannot read more than {BUFMAX} bytes"))); + } + + let handle = handle_from_fd(fd); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + // Each character may take up to 4 bytes in UTF-8. + let mut wlen = (len / 4) as u32; + if wlen == 0 { + wlen = 1; + } + + let dest = &mut *buf_ref; + + // Copy from internal buffer first + let mut read_len = { + let mut buf = self.buf.lock(); + Self::copy_from_buf(&mut buf, dest) + }; + if read_len > 0 { + wlen = wlen.saturating_sub(1); + } + if read_len >= len || wlen == 0 { + return Ok(read_len); + } + + // Read from console + let mut wbuf = vec![0u16; wlen as usize]; + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wlen, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 { + return Ok(read_len); + } + + // Check for Ctrl+Z (EOF) + if nread > 0 && wbuf[0] == 0x1A { + return Ok(read_len); + } + + // Convert wchar to UTF-8 + let remaining = len - read_len; + let u8n; + if remaining < 4 { + // Buffer the result in the internal small buffer + let mut buf = self.buf.lock(); + let converted = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + buf.as_mut_ptr() as _, + SMALLBUF as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if converted > 0 { + u8n = Self::copy_from_buf(&mut buf, &mut dest[read_len..]) as i32; + } else { + u8n = 0; + } + } else { + u8n = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + dest[read_len..].as_mut_ptr() as _, + remaining as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + } + + if u8n > 0 { + read_len += u8n as usize; + } else { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(122) { + // ERROR_INSUFFICIENT_BUFFER + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed > 0 { + return Err(vm.new_system_error(format!( + "Buffer had room for {remaining} bytes but {needed} bytes required", + ))); + } + } + return Err(err.into_pyexception(vm)); + } + + Ok(read_len) + } + + #[pymethod] + fn readall(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let mut result = Vec::new(); + + // Copy any buffered bytes first + { + let mut buf = self.buf.lock(); + let mut tmp = [0u8; SMALLBUF]; + let n = Self::copy_from_buf(&mut buf, &mut tmp); + result.extend_from_slice(&tmp[..n]); + } + + let mut wbuf = vec![0u16; 8192]; + loop { + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wbuf.len() as u32, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 { + break; + } + // Ctrl+Z at start -> EOF + if wbuf[0] == 0x1A { + break; + } + // Convert to UTF-8 + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + let offset = result.len(); + result.resize(offset + needed as usize, 0); + let written = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + result[offset..].as_mut_ptr() as _, + needed, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if written == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + // If we didn't fill the buffer, no more data + if nread < wbuf.len() as u32 { + break; + } + } + + Ok(vm.ctx.new_bytes(result).into()) + } + + #[pymethod] + fn read(&self, size: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + if !self.readable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support reading".to_owned(), + )); + } + let size = size.unwrap_or(-1); + if size < 0 { + return self.readall(vm); + } + if size as usize > BUFMAX { + return Err(vm.new_value_error(format!("cannot read more than {BUFMAX} bytes"))); + } + let mut buf = vec![0u8; size as usize]; + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let len = size as usize; + + let mut wlen = (len / 4) as u32; + if wlen == 0 { + wlen = 1; + } + + let mut read_len = { + let mut ibuf = self.buf.lock(); + Self::copy_from_buf(&mut ibuf, &mut buf) + }; + if read_len > 0 { + wlen = wlen.saturating_sub(1); + } + if read_len >= len || wlen == 0 { + buf.truncate(read_len); + return Ok(vm.ctx.new_bytes(buf).into()); + } + + let mut wbuf = vec![0u16; wlen as usize]; + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wlen, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 || wbuf[0] == 0x1A { + buf.truncate(read_len); + return Ok(vm.ctx.new_bytes(buf).into()); + } + + let remaining = len - read_len; + let u8n; + if remaining < 4 { + let mut ibuf = self.buf.lock(); + let converted = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + ibuf.as_mut_ptr() as _, + SMALLBUF as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if converted > 0 { + u8n = Self::copy_from_buf(&mut ibuf, &mut buf[read_len..]) as i32; + } else { + u8n = 0; + } + } else { + u8n = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + buf[read_len..].as_mut_ptr() as _, + remaining as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + } + + if u8n > 0 { + read_len += u8n as usize; + } else { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(122) { + // ERROR_INSUFFICIENT_BUFFER + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed > 0 { + return Err(vm.new_system_error(format!( + "Buffer had room for {remaining} bytes but {needed} bytes required", + ))); + } + } + return Err(err.into_pyexception(vm)); + } + + buf.truncate(read_len); + Ok(vm.ctx.new_bytes(buf).into()) + } + + #[pymethod] + fn write(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + if !self.writable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support writing".to_owned(), + )); + } + + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let data = b.borrow_buf(); + let data = &*data; + if data.is_empty() { + return Ok(0); + } + + let mut len = data.len().min(BUFMAX); + + // Cap at 32766/2 wchars * 3 bytes (UTF-8 to wchar ratio is at most 3:1) + let max_wlen: u32 = 32766 / 2; + len = len.min(max_wlen as usize * 3); + + // Reduce len until wlen fits within max_wlen + let wlen; + loop { + len = find_last_utf8_boundary(data, len); + let w = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data.as_ptr(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + if w as u32 <= max_wlen { + wlen = w; + break; + } + len /= 2; + } + if wlen == 0 { + return Ok(0); + } + + let mut wbuf = vec![0u16; wlen as usize]; + let wlen = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data.as_ptr(), + len as i32, + wbuf.as_mut_ptr(), + wlen, + ) + }; + if wlen == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + let mut n_written: u32 = 0; + let res = unsafe { + WriteConsoleW( + handle, + wbuf.as_ptr() as _, + wlen as u32, + &mut n_written, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + // If we wrote fewer wchars than expected, recalculate bytes consumed + if n_written < wlen as u32 { + // Binary search to find how many input bytes correspond to n_written wchars + len = wchar_to_utf8_count(data, len, n_written); + } + + Ok(len) + } + + #[pymethod(name = "__reduce__")] + fn reduce(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle '_WindowsConsoleIO' instances".to_owned())) + } + } + + /// Find how many UTF-8 bytes correspond to n wide chars. + fn wchar_to_utf8_count(data: &[u8], mut len: usize, mut n: u32) -> usize { + let mut start: usize = 0; + loop { + let mut mid = 0; + for i in (len / 2)..=len { + mid = find_last_utf8_boundary(data, i); + if mid != 0 { + break; + } + } + if mid == len { + return start + len; + } + if mid == 0 { + mid = if len > 1 { len - 1 } else { 1 }; + } + let wlen = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data[start..].as_ptr(), + mid as i32, + core::ptr::null_mut(), + 0, + ) + } as u32; + if wlen <= n { + start += mid; + len -= mid; + n -= wlen; + } else { + len = mid; + } + } + } + + impl Destructor for WindowsConsoleIO { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(cio) = zelf.downcast_ref::<WindowsConsoleIO>() { + cio.finalizing.store(true); + } + iobase_finalize(zelf, vm); Ok(()) } diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 3fedd17f12b..d1c188b8892 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -1,41 +1,28 @@ -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[pymodule(name = "itertools")] mod decl { - use crate::stdlib::itertools::decl::int::get_value; use crate::{ - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, - VirtualMachine, - builtins::{ - PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, int, - tuple::IntoPyTuple, - }, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, VirtualMachine, + builtins::{PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyType, PyTypeRef, int}, common::{ lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, rc::PyRc, }, convert::ToPyObject, - function::{ArgCallable, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, - stdlib::{sys, warnings}, + stdlib::sys, types::{Constructor, IterNext, Iterable, Representable, SelfIter}, }; + use core::sync::atomic::{AtomicBool, Ordering}; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::BigInt; use num_traits::One; + use alloc::fmt; use num_traits::{Signed, ToPrimitive}; - use std::fmt; - - fn pickle_deprecation(vm: &VirtualMachine) -> PyResult<()> { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - "Itertool pickle/copy/deepcopy support will be removed in a Python 3.14.".to_owned(), - 1, - vm, - ) - } #[pyattr] #[pyclass(name = "chain")] @@ -79,55 +66,6 @@ mod decl { ) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) } - - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - pickle_deprecation(vm)?; - let source = zelf.source.read().clone(); - let active = zelf.active.read().clone(); - let cls = zelf.class().to_owned(); - let empty_tuple = vm.ctx.empty_tuple.clone(); - let reduced = match source { - Some(source) => match active { - Some(active) => vm.new_tuple((cls, empty_tuple, (source, active))), - None => vm.new_tuple((cls, empty_tuple, (source,))), - }, - None => vm.new_tuple((cls, empty_tuple)), - }; - Ok(reduced) - } - - #[pymethod] - fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.is_empty() { - return Err(vm.new_type_error("function takes at least 1 arguments (0 given)")); - } - if args.len() > 2 { - return Err(vm.new_type_error(format!( - "function takes at most 2 arguments ({} given)", - args.len() - ))); - } - let source = &args[0]; - if args.len() == 1 { - if !PyIter::check(source.as_ref()) { - return Err(vm.new_type_error("Arguments must be iterators.")); - } - *zelf.source.write() = source.to_owned().try_into_value(vm)?; - return Ok(()); - } - let active = &args[1]; - - if !PyIter::check(source.as_ref()) || !PyIter::check(active.as_ref()) { - return Err(vm.new_type_error("Arguments must be iterators.")); - } - let mut source_lock = zelf.source.write(); - let mut active_lock = zelf.active.write(); - *source_lock = source.to_owned().try_into_value(vm)?; - *active_lock = active.to_owned().try_into_value(vm)?; - Ok(()) - } } impl SelfIter for PyItertoolsChain {} @@ -209,16 +147,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsCompress { - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyIter, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.data.clone(), zelf.selectors.clone()), - ) - } - } + impl PyItertoolsCompress {} impl SelfIter for PyItertoolsCompress {} @@ -275,16 +204,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor, Representable))] - impl PyItertoolsCount { - // TODO: Implement this - // if (lz->cnt == PY_SSIZE_T_MAX) - // return Py_BuildValue("0(00)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyObjectRef,)) { - let _ = pickle_deprecation(vm); - (zelf.class().to_owned(), (zelf.cur.read().clone(),)) - } - } + impl PyItertoolsCount {} impl SelfIter for PyItertoolsCount {} @@ -406,16 +326,6 @@ mod decl { .ok_or_else(|| vm.new_type_error("length of unsized object."))?; Ok(*times.read()) } - - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - pickle_deprecation(vm)?; - let cls = zelf.class().to_owned(); - Ok(match zelf.times { - Some(ref times) => vm.new_tuple((cls, (zelf.object.clone(), *times.read()))), - None => vm.new_tuple((cls, (zelf.object.clone(),))), - }) - } } impl SelfIter for PyItertoolsRepeat {} @@ -474,19 +384,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsStarmap { - #[pymethod] - fn __reduce__( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.function.clone(), zelf.iterable.clone()), - ) - } - } + impl PyItertoolsStarmap {} impl SelfIter for PyItertoolsStarmap {} @@ -541,31 +439,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsTakewhile { - #[pymethod] - fn __reduce__( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - zelf.stop_flag.load() as _, - ) - } - #[pymethod] - fn __setstate__( - zelf: PyRef<Self>, - state: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.stop_flag.store(*obj); - } - Ok(()) - } - } + impl PyItertoolsTakewhile {} impl SelfIter for PyItertoolsTakewhile {} @@ -627,32 +501,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsDropwhile { - #[pymethod] - fn __reduce__( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone().into(), zelf.iterable.clone()), - (zelf.start_flag.load() as _), - ) - } - - #[pymethod] - fn __setstate__( - zelf: PyRef<Self>, - state: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.start_flag.store(*obj); - } - Ok(()) - } - } + impl PyItertoolsDropwhile {} impl SelfIter for PyItertoolsDropwhile {} @@ -942,38 +791,6 @@ mod decl { .into_ref_with_type(vm, cls) .map(Into::into) } - - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - pickle_deprecation(vm)?; - let cls = zelf.class().to_owned(); - let itr = zelf.iterable.clone(); - let cur = zelf.cur.take(); - let next = zelf.next.take(); - let step = zelf.step; - match zelf.stop { - Some(stop) => Ok(vm.new_tuple((cls, (itr, next, stop, step), (cur,)))), - _ => Ok(vm.new_tuple((cls, (itr, next, vm.new_pyobj(()), step), (cur,)))), - } - } - - #[pymethod] - fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != 1 { - return Err(vm.new_type_error(format!( - "function takes exactly 1 argument ({} given)", - args.len() - ))); - } - let cur = &args[0]; - if let Ok(cur) = cur.try_to_value(vm) { - zelf.cur.store(cur); - } else { - return Err(vm.new_type_error("Argument must be usize.")); - } - Ok(()) - } } impl SelfIter for PyItertoolsIslice {} @@ -1037,19 +854,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsFilterFalse { - #[pymethod] - fn __reduce__( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyObjectRef, PyIter)) { - let _ = pickle_deprecation(vm); - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - ) - } - } + impl PyItertoolsFilterFalse {} impl SelfIter for PyItertoolsFilterFalse {} @@ -1106,59 +911,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsAccumulate { - #[pymethod] - fn __setstate__( - zelf: PyRef<Self>, - state: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<()> { - *zelf.acc_value.write() = Some(state); - Ok(()) - } - - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let class = zelf.class().to_owned(); - let bin_op = zelf.bin_op.clone(); - let it = zelf.iterable.clone(); - let acc_value = zelf.acc_value.read().clone(); - if let Some(initial) = &zelf.initial { - let chain_args = PyList::from(vec![initial.clone(), it.to_pyobject(vm)]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some(chain_args.to_pyobject(vm).get_iter(vm).unwrap())), - active: PyRwLock::new(None), - }; - let tup = vm.new_tuple((chain, bin_op)); - return vm.new_tuple((class, tup, acc_value)); - } - match acc_value { - Some(obj) if obj.is(&vm.ctx.none) => { - let chain_args = PyList::from(vec![]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some( - chain_args.to_pyobject(vm).get_iter(vm).unwrap(), - )), - active: PyRwLock::new(None), - } - .into_pyobject(vm); - let acc = Self { - iterable: PyIter::new(chain), - bin_op, - initial: None, - acc_value: PyRwLock::new(None), - }; - let tup = vm.new_tuple((acc, 1, None::<PyObjectRef>)); - let islice_cls = PyItertoolsIslice::class(&vm.ctx).to_owned(); - return vm.new_tuple((islice_cls, tup)); - } - _ => {} - } - let tup = vm.new_tuple((it, bin_op)); - vm.new_tuple((class, tup, acc_value)) - } - } + impl PyItertoolsAccumulate {} impl SelfIter for PyItertoolsAccumulate {} @@ -1191,6 +944,7 @@ mod decl { struct PyItertoolsTeeData { iterable: PyIter, values: PyMutex<Vec<PyObjectRef>>, + running: AtomicBool, } impl PyItertoolsTeeData { @@ -1198,19 +952,33 @@ mod decl { Ok(PyRc::new(Self { iterable, values: PyMutex::new(vec![]), + running: AtomicBool::new(false), })) } fn get_item(&self, vm: &VirtualMachine, index: usize) -> PyResult<PyIterReturn> { + // Return cached value if available + { + let Some(values) = self.values.try_lock() else { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + }; + if index < values.len() { + return Ok(PyIterReturn::Return(values[index].clone())); + } + } + // Prevent concurrent/reentrant calls to iterable.next() + if self.running.swap(true, Ordering::Acquire) { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + } + let result = self.iterable.next(vm); + self.running.store(false, Ordering::Release); + let obj = raise_if_stop!(result?); let Some(mut values) = self.values.try_lock() else { return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); }; - if values.len() == index { - let obj = raise_if_stop!(self.iterable.next(vm)?); values.push(obj); } - Ok(PyIterReturn::Return(values[index].clone())) } } @@ -1320,7 +1088,7 @@ mod decl { for arg in iterables.iter() { pools.push(arg.try_to_value(vm)?); } - let pools = std::iter::repeat_n(pools, repeat) + let pools = core::iter::repeat_n(pools, repeat) .flatten() .collect::<Vec<Vec<PyObjectRef>>>(); @@ -1359,58 +1127,6 @@ mod decl { self.cur.store(idxs.len() - 1); } } - - #[pymethod] - fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != zelf.pools.len() { - return Err(vm.new_type_error("Invalid number of arguments")); - } - let mut idxs: PyRwLockWriteGuard<'_, Vec<usize>> = zelf.idxs.write(); - idxs.clear(); - for s in 0..args.len() { - let index = get_value(state.get(s).unwrap()).to_usize().unwrap(); - let pool_size = zelf.pools.get(s).unwrap().len(); - if pool_size == 0 { - zelf.stop.store(true); - return Ok(()); - } - if index >= pool_size { - idxs.push(pool_size - 1); - } else { - idxs.push(index); - } - } - zelf.stop.store(false); - Ok(()) - } - - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let class = zelf.class().to_owned(); - - if zelf.stop.load() { - return vm.new_tuple((class, (vm.ctx.empty_tuple.clone(),))); - } - - let mut pools: Vec<PyObjectRef> = Vec::new(); - for element in &zelf.pools { - pools.push(element.clone().into_pytuple(vm).into()); - } - - let mut indices: Vec<PyObjectRef> = Vec::new(); - - for item in &zelf.idxs.read()[..] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple(( - class, - pools.clone().into_pytuple(vm), - indices.into_pytuple(vm), - )) - } } impl SelfIter for PyItertoolsProduct {} @@ -1492,36 +1208,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsCombinations { - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let _ = pickle_deprecation(vm); - let r = zelf.r.load(); - - let class = zelf.class().to_owned(); - - if zelf.exhausted.load() { - return vm.new_tuple(( - class, - vm.new_tuple((vm.ctx.empty_tuple.clone(), vm.ctx.new_int(r))), - )); - } - - let tup = vm.new_tuple((zelf.pool.clone().into_pytuple(vm), vm.ctx.new_int(r))); - - if zelf.result.read().is_none() { - vm.new_tuple((class, tup)) - } else { - let mut indices: Vec<PyObjectRef> = Vec::new(); - - for item in &zelf.indices.read()[..r] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple((class, tup, indices.into_pytuple(vm))) - } - } - } + impl PyItertoolsCombinations {} impl SelfIter for PyItertoolsCombinations {} impl IterNext for PyItertoolsCombinations { @@ -1730,16 +1417,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsPermutations { - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<PyTuple> { - let _ = pickle_deprecation(vm); - vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple((zelf.pool.clone(), vm.ctx.new_int(zelf.r.load()))), - )) - } - } + impl PyItertoolsPermutations {} impl SelfIter for PyItertoolsPermutations {} @@ -1846,32 +1524,7 @@ mod decl { } #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsZipLongest { - #[pymethod] - fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - pickle_deprecation(vm)?; - let args: Vec<PyObjectRef> = zelf - .iterators - .iter() - .map(|i| i.clone().to_pyobject(vm)) - .collect(); - Ok(vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple(args), - zelf.fillvalue.read().to_owned(), - ))) - } - - #[pymethod] - fn __setstate__( - zelf: PyRef<Self>, - state: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<()> { - *zelf.fillvalue.write() = state; - Ok(()) - } - } + impl PyItertoolsZipLongest {} impl SelfIter for PyItertoolsZipLongest {} diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index aced9e48773..cf7abe65194 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -1,5 +1,5 @@ // spell-checker:ignore pyfrozen pycomplex -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[pymodule(name = "marshal")] mod decl { diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index 9fae516fe04..e3bf42a4f77 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -1,3 +1,5 @@ +mod _abc; +mod _types; #[cfg(feature = "ast")] pub(crate) mod ast; pub mod atexit; @@ -6,6 +8,7 @@ mod codecs; mod collections; pub mod errno; mod functools; +mod gc; mod imp; pub mod io; mod itertools; @@ -28,126 +31,111 @@ pub mod typing; pub mod warnings; mod weakref; -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] +#[cfg(feature = "host_env")] #[macro_use] pub mod os; -#[cfg(windows)] +#[cfg(all(feature = "host_env", windows))] pub mod nt; -#[cfg(unix)] +#[cfg(all(feature = "host_env", unix))] pub mod posix; -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] -#[cfg(not(any(unix, windows)))] +#[cfg(all(feature = "host_env", not(any(unix, windows))))] #[path = "posix_compat.rs"] pub mod posix; #[cfg(all( + feature = "host_env", any(target_os = "linux", target_os = "macos", target_os = "windows"), not(any(target_env = "musl", target_env = "sgx")) ))] mod ctypes; -#[cfg(windows)] +#[cfg(all(feature = "host_env", windows))] pub(crate) mod msvcrt; #[cfg(all( + feature = "host_env", unix, not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) ))] mod pwd; +#[cfg(all(feature = "host_env", windows))] +mod _wmi; +#[cfg(feature = "host_env")] pub(crate) mod signal; pub mod sys; -#[cfg(windows)] +#[cfg(all(feature = "host_env", windows))] mod winapi; -#[cfg(windows)] +#[cfg(all(feature = "host_env", windows))] mod winreg; +#[cfg(all(feature = "host_env", windows))] +mod winsound; -use crate::{PyRef, VirtualMachine, builtins::PyModule}; -use std::{borrow::Cow, collections::HashMap}; +use crate::{Context, builtins::PyModuleDef}; -pub type StdlibInitFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine) -> PyRef<PyModule>)>; -pub type StdlibMap = HashMap<Cow<'static, str>, StdlibInitFunc, ahash::RandomState>; - -pub fn get_module_inits() -> StdlibMap { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - let modules = [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ]; - modules.into_iter().collect() - }}; - } - modules! { - #[cfg(all())] - { - "atexit" => atexit::make_module, - "_codecs" => codecs::make_module, - "_collections" => collections::make_module, - "errno" => errno::make_module, - "_functools" => functools::make_module, - "itertools" => itertools::make_module, - "_io" => io::make_module, - "marshal" => marshal::make_module, - "_operator" => operator::make_module, - "_signal" => signal::make_module, - "_sre" => sre::make_module, - "_stat" => stat::make_module, - "_sysconfig" => sysconfig::make_module, - "_string" => string::make_module, - "time" => time::make_module, - "_typing" => typing::make_module, - "_weakref" => weakref::make_module, - "_imp" => imp::make_module, - "_warnings" => warnings::make_module, - sys::sysconfigdata_name() => sysconfigdata::make_module, - } - // parser related modules: +/// Returns module definitions for multi-phase init modules. +/// +/// These modules use multi-phase initialization pattern: +/// 1. Create module from def and add to sys.modules +/// 2. Call exec slot (can safely import other modules without circular import issues) +pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> { + vec![ + _abc::module_def(ctx), + _types::module_def(ctx), #[cfg(feature = "ast")] - { - "_ast" => ast::make_module, - } - // compiler related modules: - #[cfg(feature = "compiler")] - { - "_symtable" => symtable::make_module, - } - #[cfg(any(unix, target_os = "wasi"))] - { - "posix" => posix::make_module, - // "fcntl" => fcntl::make_module, - } - #[cfg(feature = "threading")] - { - "_thread" => thread::make_module, - } - // Unix-only - #[cfg(all( - unix, - not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) - ))] - { - "pwd" => pwd::make_module, - } - // Windows-only - #[cfg(windows)] - { - "nt" => nt::make_module, - "msvcrt" => msvcrt::make_module, - "_winapi" => winapi::make_module, - "winreg" => winreg::make_module, - } + ast::module_def(ctx), + atexit::module_def(ctx), + codecs::module_def(ctx), + collections::module_def(ctx), #[cfg(all( + feature = "host_env", any(target_os = "linux", target_os = "macos", target_os = "windows"), not(any(target_env = "musl", target_env = "sgx")) ))] - { - "_ctypes" => ctypes::make_module, - } - } + ctypes::module_def(ctx), + errno::module_def(ctx), + functools::module_def(ctx), + gc::module_def(ctx), + imp::module_def(ctx), + io::module_def(ctx), + itertools::module_def(ctx), + marshal::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + msvcrt::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + nt::module_def(ctx), + operator::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] + posix::module_def(ctx), + #[cfg(all(feature = "host_env", not(any(unix, windows, target_os = "wasi"))))] + posix::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) + ))] + pwd::module_def(ctx), + #[cfg(feature = "host_env")] + signal::module_def(ctx), + sre::module_def(ctx), + stat::module_def(ctx), + string::module_def(ctx), + #[cfg(feature = "compiler")] + symtable::module_def(ctx), + sysconfigdata::module_def(ctx), + sysconfig::module_def(ctx), + #[cfg(feature = "threading")] + thread::module_def(ctx), + time::module_def(ctx), + typing::module_def(ctx), + warnings::module_def(ctx), + weakref::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winapi::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winreg::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winsound::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + _wmi::module_def(ctx), + ] } diff --git a/crates/vm/src/stdlib/msvcrt.rs b/crates/vm/src/stdlib/msvcrt.rs index cf0dac2c9db..cf194a3f7ba 100644 --- a/crates/vm/src/stdlib/msvcrt.rs +++ b/crates/vm/src/stdlib/msvcrt.rs @@ -31,8 +31,25 @@ mod msvcrt { fn _getwche() -> u32; fn _putch(c: u32) -> i32; fn _putwch(c: u16) -> u32; + fn _ungetch(c: i32) -> i32; + fn _ungetwch(c: u32) -> u32; + fn _locking(fd: i32, mode: i32, nbytes: i64) -> i32; + fn _heapmin() -> i32; + fn _kbhit() -> i32; } + // Locking mode constants + #[pyattr] + const LK_UNLCK: i32 = 0; // Unlock + #[pyattr] + const LK_LOCK: i32 = 1; // Lock (blocking) + #[pyattr] + const LK_NBLCK: i32 = 2; // Non-blocking lock + #[pyattr] + const LK_RLCK: i32 = 3; // Lock for reading (same as LK_LOCK) + #[pyattr] + const LK_NBRLCK: i32 = 4; // Non-blocking lock for reading (same as LK_NBLCK) + #[pyfunction] fn getch() -> Vec<u8> { let c = unsafe { _getch() }; @@ -41,7 +58,7 @@ mod msvcrt { #[pyfunction] fn getwch() -> String { let c = unsafe { _getwch() }; - std::char::from_u32(c).unwrap().to_string() + char::from_u32(c).unwrap().to_string() } #[pyfunction] fn getche() -> Vec<u8> { @@ -51,7 +68,7 @@ mod msvcrt { #[pyfunction] fn getwche() -> String { let c = unsafe { _getwche() }; - std::char::from_u32(c).unwrap().to_string() + char::from_u32(c).unwrap().to_string() } #[pyfunction] fn putch(b: PyRef<PyBytes>, vm: &VirtualMachine) -> PyResult<()> { @@ -73,6 +90,60 @@ mod msvcrt { Ok(()) } + #[pyfunction] + fn ungetch(b: PyRef<PyBytes>, vm: &VirtualMachine) -> PyResult<()> { + let &c = b.as_bytes().iter().exactly_one().map_err(|_| { + vm.new_type_error("ungetch() argument must be a byte string of length 1") + })?; + let ret = unsafe { suppress_iph!(_ungetch(c as i32)) }; + if ret == -1 { + // EOF returned means the buffer is full + Err(vm.new_os_error(libc::ENOSPC)) + } else { + Ok(()) + } + } + + #[pyfunction] + fn ungetwch(s: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let c = + s.as_str().chars().exactly_one().map_err(|_| { + vm.new_type_error("ungetwch() argument must be a string of length 1") + })?; + let ret = unsafe { suppress_iph!(_ungetwch(c as u32)) }; + if ret == 0xFFFF { + // WEOF returned means the buffer is full + Err(vm.new_os_error(libc::ENOSPC)) + } else { + Ok(()) + } + } + + #[pyfunction] + fn kbhit() -> i32 { + unsafe { _kbhit() } + } + + #[pyfunction] + fn locking(fd: i32, mode: i32, nbytes: i64, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { suppress_iph!(_locking(fd, mode, nbytes)) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn heapmin(vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { suppress_iph!(_heapmin()) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + unsafe extern "C" { fn _setmode(fd: crt_fd::Borrowed<'_>, flags: i32) -> i32; } diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index ada939b1549..5b2cf3b92f5 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -1,30 +1,25 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - +pub(crate) use module::module_def; pub use module::raw_set_handle_inheritable; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - #[pymodule(name = "nt", with(super::os::_os))] pub(crate) mod module { use crate::{ - PyResult, TryFromObject, VirtualMachine, + Py, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStrRef, PyTupleRef}, common::{crt_fd, suppress_iph, windows::ToWideString}, convert::ToPyException, - function::{Either, OptionalArg}, + exceptions::OSErrorBuilder, + function::{ArgMapping, Either, OptionalArg}, ospath::{OsPath, OsPathOrFd}, stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory}, }; + use core::mem::MaybeUninit; use libc::intptr_t; use std::os::windows::io::AsRawHandle; - use std::{env, fs, io, mem::MaybeUninit, os::windows::ffi::OsStringExt}; + use std::{env, io, os::windows::ffi::OsStringExt}; use windows_sys::Win32::{ Foundation::{self, INVALID_HANDLE_VALUE}, Storage::FileSystem, @@ -76,10 +71,66 @@ pub(crate) mod module { || attr & FileSystem::FILE_ATTRIBUTE_DIRECTORY != 0)) } + #[pyfunction] + #[pyfunction(name = "unlink")] + pub(super) fn remove( + path: OsPath, + dir_fd: DirFd<'static, 0>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // On Windows, use DeleteFileW directly. + // Rust's std::fs::remove_file may have different behavior for read-only files. + // See Py_DeleteFileW. + use windows_sys::Win32::Storage::FileSystem::{ + DeleteFileW, FindClose, FindFirstFileW, RemoveDirectoryW, WIN32_FIND_DATAW, + }; + use windows_sys::Win32::System::SystemServices::{ + IO_REPARSE_TAG_MOUNT_POINT, IO_REPARSE_TAG_SYMLINK, + }; + + let [] = dir_fd.0; + let wide_path = path.to_wide_cstring(vm)?; + let attrs = unsafe { FileSystem::GetFileAttributesW(wide_path.as_ptr()) }; + + let mut is_directory = false; + let mut is_link = false; + + if attrs != FileSystem::INVALID_FILE_ATTRIBUTES { + is_directory = (attrs & FileSystem::FILE_ATTRIBUTE_DIRECTORY) != 0; + + // Check if it's a symlink or junction point + if is_directory && (attrs & FileSystem::FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; + let handle = unsafe { FindFirstFileW(wide_path.as_ptr(), &mut find_data) }; + if handle != INVALID_HANDLE_VALUE { + is_link = find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK + || find_data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT; + unsafe { FindClose(handle) }; + } + } + } + + let result = if is_directory && is_link { + unsafe { RemoveDirectoryW(wide_path.as_ptr()) } + } else { + unsafe { DeleteFileW(wide_path.as_ptr()) } + }; + + if result == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + Ok(()) + } + #[pyfunction] pub(super) fn _supports_virtual_terminal() -> PyResult<bool> { - // TODO: implement this - Ok(true) + let mut mode = 0; + let handle = unsafe { Console::GetStdHandle(Console::STD_ERROR_HANDLE) }; + if unsafe { Console::GetConsoleMode(handle, &mut mode) } == 0 { + return Ok(false); + } + Ok(mode & Console::ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0) } #[derive(FromArgs)] @@ -94,20 +145,78 @@ pub(crate) mod module { #[pyfunction] pub(super) fn symlink(args: SymlinkArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { - use std::os::windows::fs as win_fs; - let dir = args.target_is_directory.target_is_directory - || args - .dst - .as_path() - .parent() - .and_then(|dst_parent| dst_parent.join(&args.src).symlink_metadata().ok()) - .is_some_and(|meta| meta.is_dir()); - let res = if dir { - win_fs::symlink_dir(args.src.path, args.dst.path) - } else { - win_fs::symlink_file(args.src.path, args.dst.path) + use crate::exceptions::ToOSErrorBuilder; + use core::sync::atomic::{AtomicBool, Ordering}; + use windows_sys::Win32::Storage::FileSystem::WIN32_FILE_ATTRIBUTE_DATA; + use windows_sys::Win32::Storage::FileSystem::{ + CreateSymbolicLinkW, FILE_ATTRIBUTE_DIRECTORY, GetFileAttributesExW, + SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE, SYMBOLIC_LINK_FLAG_DIRECTORY, }; - res.map_err(|err| err.to_pyexception(vm)) + + static HAS_UNPRIVILEGED_FLAG: AtomicBool = AtomicBool::new(true); + + fn check_dir(src: &OsPath, dst: &OsPath) -> bool { + use windows_sys::Win32::Storage::FileSystem::GetFileExInfoStandard; + + let dst_parent = dst.as_path().parent(); + let Some(dst_parent) = dst_parent else { + return false; + }; + let resolved = if src.as_path().is_absolute() { + src.as_path().to_path_buf() + } else { + dst_parent.join(src.as_path()) + }; + let wide = match widestring::WideCString::from_os_str(&resolved) { + Ok(wide) => wide, + Err(_) => return false, + }; + let mut info: WIN32_FILE_ATTRIBUTE_DATA = unsafe { core::mem::zeroed() }; + let ok = unsafe { + GetFileAttributesExW( + wide.as_ptr(), + GetFileExInfoStandard, + &mut info as *mut _ as *mut _, + ) + }; + ok != 0 && (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0 + } + + let mut flags = 0u32; + if HAS_UNPRIVILEGED_FLAG.load(Ordering::Relaxed) { + flags |= SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + } + if args.target_is_directory.target_is_directory || check_dir(&args.src, &args.dst) { + flags |= SYMBOLIC_LINK_FLAG_DIRECTORY; + } + + let src = args.src.to_wide_cstring(vm)?; + let dst = args.dst.to_wide_cstring(vm)?; + + let mut result = unsafe { CreateSymbolicLinkW(dst.as_ptr(), src.as_ptr(), flags) }; + if !result + && HAS_UNPRIVILEGED_FLAG.load(Ordering::Relaxed) + && unsafe { Foundation::GetLastError() } == Foundation::ERROR_INVALID_PARAMETER + { + let flags = flags & !SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + result = unsafe { CreateSymbolicLinkW(dst.as_ptr(), src.as_ptr(), flags) }; + if result + || unsafe { Foundation::GetLastError() } != Foundation::ERROR_INVALID_PARAMETER + { + HAS_UNPRIVILEGED_FLAG.store(false, Ordering::Relaxed); + } + } + + if !result { + let err = io::Error::last_os_error(); + let builder = err.to_os_error_builder(vm); + let builder = builder + .filename(args.src.filename(vm)) + .filename2(args.dst.filename(vm)); + return Err(builder.build(vm).upcast()); + } + + Ok(()) } #[pyfunction] @@ -127,8 +236,19 @@ pub(crate) mod module { for (key, value) in env::vars() { // Skip hidden Windows environment variables (e.g., =C:, =D:, =ExitCode) // These are internal cmd.exe bookkeeping variables that store per-drive - // current directories. They cannot be modified via _wputenv() and should - // not be exposed to Python code. + // current directories and cannot be reliably modified via _wputenv(). + if key.starts_with('=') { + continue; + } + environ.set_item(&key, vm.new_pyobj(value), vm).unwrap(); + } + environ + } + + #[pyfunction] + fn _create_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { if key.starts_with('=') { continue; } @@ -138,9 +258,9 @@ pub(crate) mod module { } #[derive(FromArgs)] - struct ChmodArgs { + struct ChmodArgs<'a> { #[pyarg(any)] - path: OsPath, + path: OsPathOrFd<'a>, #[pyarg(any)] mode: u32, #[pyarg(flatten)] @@ -149,54 +269,140 @@ pub(crate) mod module { follow_symlinks: OptionalArg<bool>, } + const S_IWRITE: u32 = 128; + + fn win32_hchmod(handle: Foundation::HANDLE, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{ + FILE_BASIC_INFO, FileBasicInfo, GetFileInformationByHandleEx, + SetFileInformationByHandle, + }; + + // Get current file info + let mut info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let ret = unsafe { + GetFileInformationByHandleEx( + handle, + FileBasicInfo, + &mut info as *mut _ as *mut _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + // Modify readonly attribute based on S_IWRITE bit + if mode & S_IWRITE != 0 { + info.FileAttributes &= !FileSystem::FILE_ATTRIBUTE_READONLY; + } else { + info.FileAttributes |= FileSystem::FILE_ATTRIBUTE_READONLY; + } + + // Set the new attributes + let ret = unsafe { + SetFileInformationByHandle( + handle, + FileBasicInfo, + &info as *const _ as *const _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + fn fchmod_impl(fd: i32, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + // Get Windows HANDLE from fd + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; + let handle = crt_fd::as_handle(borrowed).map_err(|e| e.to_pyexception(vm))?; + let hfile = handle.as_raw_handle() as Foundation::HANDLE; + win32_hchmod(hfile, mode, vm) + } + + fn win32_lchmod(path: &OsPath, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{GetFileAttributesW, SetFileAttributesW}; + + let wide = path.to_wide_cstring(vm)?; + let attr = unsafe { GetFileAttributesW(wide.as_ptr()) }; + if attr == FileSystem::INVALID_FILE_ATTRIBUTES { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + let new_attr = if mode & S_IWRITE != 0 { + attr & !FileSystem::FILE_ATTRIBUTE_READONLY + } else { + attr | FileSystem::FILE_ATTRIBUTE_READONLY + }; + let ret = unsafe { SetFileAttributesW(wide.as_ptr(), new_attr) }; + if ret == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + Ok(()) + } + + #[pyfunction] + fn fchmod(fd: i32, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + fchmod_impl(fd, mode, vm) + } + #[pyfunction] - fn chmod(args: ChmodArgs, vm: &VirtualMachine) -> PyResult<()> { + fn chmod(args: ChmodArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { let ChmodArgs { path, mode, dir_fd, follow_symlinks, } = args; - const S_IWRITE: u32 = 128; let [] = dir_fd.0; - // On Windows, os.chmod behavior differs based on whether follow_symlinks is explicitly provided: - // - Not provided (default): use SetFileAttributesW on the path directly (doesn't follow symlinks) - // - Explicitly True: resolve symlink first, then apply permissions to target - // - Explicitly False: raise NotImplementedError (Windows can't change symlink permissions) - let actual_path: std::borrow::Cow<'_, std::path::Path> = match follow_symlinks.into_option() - { - None => { - // Default behavior: don't resolve symlinks, operate on path directly - std::borrow::Cow::Borrowed(path.as_ref()) - } - Some(true) => { - // Explicitly follow symlinks: resolve the path first - match fs::canonicalize(&path) { - Ok(p) => std::borrow::Cow::Owned(p), - Err(_) => std::borrow::Cow::Borrowed(path.as_ref()), - } - } - Some(false) => { - // follow_symlinks=False on Windows - not supported for symlinks - // Check if path is a symlink - if let Ok(meta) = fs::symlink_metadata(&path) - && meta.file_type().is_symlink() - { - return Err(vm.new_not_implemented_error( - "chmod: follow_symlinks=False is not supported on Windows for symlinks" - .to_owned(), - )); - } - std::borrow::Cow::Borrowed(path.as_ref()) + // If path is a file descriptor, use fchmod + if let OsPathOrFd::Fd(fd) = path { + if follow_symlinks.into_option().is_some() { + return Err(vm.new_value_error( + "chmod: follow_symlinks is not supported with fd argument".to_owned(), + )); } + return fchmod_impl(fd.as_raw(), mode, vm); + } + + let OsPathOrFd::Path(path) = path else { + unreachable!() }; - // Use symlink_metadata to avoid following dangling symlinks - let meta = fs::symlink_metadata(&actual_path).map_err(|err| err.to_pyexception(vm))?; - let mut permissions = meta.permissions(); - permissions.set_readonly(mode & S_IWRITE == 0); - fs::set_permissions(&*actual_path, permissions).map_err(|err| err.to_pyexception(vm)) + let follow_symlinks = follow_symlinks.into_option().unwrap_or(false); + + if follow_symlinks { + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_ATTRIBUTES, OPEN_EXISTING, + }; + + let wide = path.to_wide_cstring(vm)?; + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + let result = win32_hchmod(handle, mode, vm); + unsafe { Foundation::CloseHandle(handle) }; + result + } else { + win32_lchmod(&path, mode, vm) + } } /// Get the real file name (with correct case) without accessing the file. @@ -210,14 +416,12 @@ pub(crate) mod module { }; let wide_path = path.as_ref().to_wide_with_nul(); - let mut find_data: WIN32_FIND_DATAW = unsafe { std::mem::zeroed() }; + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; let handle = unsafe { FindFirstFileW(wide_path.as_ptr(), &mut find_data) }; if handle == INVALID_HANDLE_VALUE { - return Err(vm.new_os_error(format!( - "FindFirstFileW failed for path: {}", - path.as_ref().display() - ))); + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); } unsafe { FindClose(handle) }; @@ -254,6 +458,8 @@ pub(crate) mod module { const PY_IFDIR: u32 = 2; // Directory const PY_IFLNK: u32 = 4; // Symlink const PY_IFMNT: u32 = 8; // Mount point (junction) + const PY_IFLRP: u32 = 16; // Link Reparse Point (name-surrogate, symlink, junction) + const PY_IFRRP: u32 = 32; // Regular Reparse Point /// _testInfo - determine file type based on attributes and reparse tag fn _test_info(attributes: u32, reparse_tag: u32, disk_device: bool, tested_type: u32) -> bool { @@ -278,10 +484,38 @@ pub(crate) mod module { (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 && reparse_tag == IO_REPARSE_TAG_MOUNT_POINT } + PY_IFLRP => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && is_reparse_tag_name_surrogate(reparse_tag) + } + PY_IFRRP => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && reparse_tag != 0 + && !is_reparse_tag_name_surrogate(reparse_tag) + } _ => false, } } + fn is_reparse_tag_name_surrogate(tag: u32) -> bool { + (tag & 0x20000000) != 0 + } + + fn file_info_error_is_trustworthy(error: u32) -> bool { + use windows_sys::Win32::Foundation; + matches!( + error, + Foundation::ERROR_FILE_NOT_FOUND + | Foundation::ERROR_PATH_NOT_FOUND + | Foundation::ERROR_NOT_READY + | Foundation::ERROR_BAD_NET_NAME + | Foundation::ERROR_BAD_NETPATH + | Foundation::ERROR_BAD_PATHNAME + | Foundation::ERROR_INVALID_NAME + | Foundation::ERROR_FILENAME_EXCED_RANGE + ) + } + /// _testFileTypeByHandle - test file type using an open handle fn _test_file_type_by_handle( handle: windows_sys::Win32::Foundation::HANDLE, @@ -301,13 +535,13 @@ pub(crate) mod module { if tested_type != PY_IFREG && tested_type != PY_IFDIR { // For symlinks/junctions, need FileAttributeTagInfo to get reparse tag - let mut info: FILE_ATTRIBUTE_TAG_INFO = unsafe { std::mem::zeroed() }; + let mut info: FILE_ATTRIBUTE_TAG_INFO = unsafe { core::mem::zeroed() }; let ret = unsafe { GetFileInformationByHandleEx( handle, FileAttributeTagInfoClass, &mut info as *mut _ as *mut _, - std::mem::size_of::<FILE_ATTRIBUTE_TAG_INFO>() as u32, + core::mem::size_of::<FILE_ATTRIBUTE_TAG_INFO>() as u32, ) }; if ret == 0 { @@ -321,13 +555,13 @@ pub(crate) mod module { ) } else { // For regular files/directories, FileBasicInfo is sufficient - let mut info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; + let mut info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; let ret = unsafe { GetFileInformationByHandleEx( handle, FileBasicInfo, &mut info as *mut _ as *mut _, - std::mem::size_of::<FILE_BASIC_INFO>() as u32, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, ) }; if ret == 0 { @@ -339,140 +573,183 @@ pub(crate) mod module { /// _testFileTypeByName - test file type by path name fn _test_file_type_by_name(path: &std::path::Path, tested_type: u32) -> bool { + use crate::common::fileutils::windows::{ + FILE_INFO_BY_NAME_CLASS, get_file_information_by_name, + }; use crate::common::windows::ToWideString; use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; use windows_sys::Win32::Storage::FileSystem::{ - CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, - FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, - OPEN_EXISTING, + CreateFileW, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, OPEN_EXISTING, }; - - // For islink/isjunction, use symlink_metadata to check reparse points - if (tested_type == PY_IFLNK || tested_type == PY_IFMNT) - && let Ok(meta) = path.symlink_metadata() - { - use std::os::windows::fs::MetadataExt; - let attrs = meta.file_attributes(); - use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT; - if (attrs & FILE_ATTRIBUTE_REPARSE_POINT) == 0 { - return false; + use windows_sys::Win32::Storage::FileSystem::{FILE_DEVICE_CD_ROM, FILE_DEVICE_DISK}; + use windows_sys::Win32::System::Ioctl::FILE_DEVICE_VIRTUAL_DISK; + + match get_file_information_by_name( + path.as_os_str(), + FILE_INFO_BY_NAME_CLASS::FileStatBasicByNameInfo, + ) { + Ok(info) => { + let disk_device = matches!( + info.DeviceType, + FILE_DEVICE_DISK | FILE_DEVICE_VIRTUAL_DISK | FILE_DEVICE_CD_ROM + ); + let result = _test_info( + info.FileAttributes, + info.ReparseTag, + disk_device, + tested_type, + ); + if !result + || (tested_type != PY_IFREG && tested_type != PY_IFDIR) + || (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0 + { + return result; + } + } + Err(err) => { + if let Some(code) = err.raw_os_error() + && file_info_error_is_trustworthy(code as u32) + { + return false; + } } - // Need to check reparse tag, fall through to CreateFileW } let wide_path = path.to_wide_with_nul(); - // For symlinks/junctions, add FILE_FLAG_OPEN_REPARSE_POINT to not follow let mut flags = FILE_FLAG_BACKUP_SEMANTICS; if tested_type != PY_IFREG && tested_type != PY_IFDIR { flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - - // Use sharing flags to avoid access denied errors let handle = unsafe { CreateFileW( wide_path.as_ptr(), FILE_READ_ATTRIBUTES, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + 0, + core::ptr::null(), OPEN_EXISTING, flags, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; - if handle == INVALID_HANDLE_VALUE { - // Fallback: try using Rust's metadata for isdir/isfile - if tested_type == PY_IFDIR { - return path.metadata().is_ok_and(|m| m.is_dir()); - } else if tested_type == PY_IFREG { - return path.metadata().is_ok_and(|m| m.is_file()); - } - // For symlinks/junctions, try without FILE_FLAG_BACKUP_SEMANTICS - let handle = unsafe { - CreateFileW( - wide_path.as_ptr(), - FILE_READ_ATTRIBUTES, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT, - std::ptr::null_mut(), - ) - }; - if handle == INVALID_HANDLE_VALUE { - return false; - } - let result = _test_file_type_by_handle(handle, tested_type, true); + if handle != INVALID_HANDLE_VALUE { + let result = _test_file_type_by_handle(handle, tested_type, false); unsafe { CloseHandle(handle) }; return result; } - let result = _test_file_type_by_handle(handle, tested_type, true); - unsafe { CloseHandle(handle) }; - result + match unsafe { windows_sys::Win32::Foundation::GetLastError() } { + windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + | windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION + | windows_sys::Win32::Foundation::ERROR_CANT_ACCESS_FILE + | windows_sys::Win32::Foundation::ERROR_INVALID_PARAMETER => { + let stat = if tested_type == PY_IFREG || tested_type == PY_IFDIR { + crate::windows::win32_xstat(path.as_os_str(), true) + } else { + crate::windows::win32_xstat(path.as_os_str(), false) + }; + if let Ok(st) = stat { + let disk_device = (st.st_mode & libc::S_IFREG as u16) != 0; + return _test_info( + st.st_file_attributes, + st.st_reparse_tag, + disk_device, + tested_type, + ); + } + } + _ => {} + } + + false } /// _testFileExistsByName - test if path exists fn _test_file_exists_by_name(path: &std::path::Path, follow_links: bool) -> bool { + use crate::common::fileutils::windows::{ + FILE_INFO_BY_NAME_CLASS, get_file_information_by_name, + }; use crate::common::windows::ToWideString; - use windows_sys::Win32::Foundation::{CloseHandle, GENERIC_READ, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; use windows_sys::Win32::Storage::FileSystem::{ - CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, - FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, - OPEN_EXISTING, + CreateFileW, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, OPEN_EXISTING, }; - // First try standard Rust exists/symlink_metadata (handles \\?\ paths well) - if follow_links { - if path.exists() { - return true; + match get_file_information_by_name( + path.as_os_str(), + FILE_INFO_BY_NAME_CLASS::FileStatBasicByNameInfo, + ) { + Ok(info) => { + if (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0 + || (!follow_links && is_reparse_tag_name_surrogate(info.ReparseTag)) + { + return true; + } + } + Err(err) => { + if let Some(code) = err.raw_os_error() + && file_info_error_is_trustworthy(code as u32) + { + return false; + } } - } else if path.symlink_metadata().is_ok() { - return true; } let wide_path = path.to_wide_with_nul(); - let mut flags = FILE_FLAG_BACKUP_SEMANTICS; if !follow_links { flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - - // Fallback: try with FILE_READ_ATTRIBUTES and sharing flags let handle = unsafe { CreateFileW( wide_path.as_ptr(), FILE_READ_ATTRIBUTES, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + 0, + core::ptr::null(), OPEN_EXISTING, flags, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; - if handle != INVALID_HANDLE_VALUE { + if follow_links { + unsafe { CloseHandle(handle) }; + return true; + } + let is_regular_reparse_point = _test_file_type_by_handle(handle, PY_IFRRP, false); unsafe { CloseHandle(handle) }; - return true; + if !is_regular_reparse_point { + return true; + } + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + FILE_READ_ATTRIBUTES, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle != INVALID_HANDLE_VALUE { + unsafe { CloseHandle(handle) }; + return true; + } } - // Fallback for console devices like \\.\CON - let handle = unsafe { - CreateFileW( - wide_path.as_ptr(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - std::ptr::null(), - OPEN_EXISTING, - 0, - std::ptr::null_mut(), - ) - }; - - if handle != INVALID_HANDLE_VALUE { - unsafe { CloseHandle(handle) }; - return true; + match unsafe { windows_sys::Win32::Foundation::GetLastError() } { + windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + | windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION + | windows_sys::Win32::Foundation::ERROR_CANT_ACCESS_FILE + | windows_sys::Win32::Foundation::ERROR_INVALID_PARAMETER => { + let stat = crate::windows::win32_xstat(path.as_os_str(), follow_links); + return stat.is_ok(); + } + _ => {} } false @@ -564,6 +841,97 @@ pub(crate) mod module { .is_some_and(|p| _test_file_exists(&p, false)) } + /// Check if a path is on a Windows Dev Drive. + #[pyfunction] + fn _path_isdevdrive(path: OsPath, vm: &VirtualMachine) -> PyResult<bool> { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, + FILE_SHARE_WRITE, GetDriveTypeW, GetVolumePathNameW, OPEN_EXISTING, + }; + use windows_sys::Win32::System::IO::DeviceIoControl; + use windows_sys::Win32::System::Ioctl::FSCTL_QUERY_PERSISTENT_VOLUME_STATE; + use windows_sys::Win32::System::WindowsProgramming::DRIVE_FIXED; + + // PERSISTENT_VOLUME_STATE_DEV_VOLUME flag - not yet in windows-sys + const PERSISTENT_VOLUME_STATE_DEV_VOLUME: u32 = 0x00002000; + + // FILE_FS_PERSISTENT_VOLUME_INFORMATION structure + #[repr(C)] + struct FileFsPersistentVolumeInformation { + volume_flags: u32, + flag_mask: u32, + version: u32, + reserved: u32, + } + + let wide_path = path.to_wide_cstring(vm)?; + let mut volume = [0u16; Foundation::MAX_PATH as usize]; + + // Get volume path + let ret = unsafe { + GetVolumePathNameW(wide_path.as_ptr(), volume.as_mut_ptr(), volume.len() as _) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + // Check if it's a fixed drive + if unsafe { GetDriveTypeW(volume.as_ptr()) } != DRIVE_FIXED { + return Ok(false); + } + + // Open the volume + let handle = unsafe { + CreateFileW( + volume.as_ptr(), + FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + // Query persistent volume state + let mut volume_state = FileFsPersistentVolumeInformation { + volume_flags: 0, + flag_mask: PERSISTENT_VOLUME_STATE_DEV_VOLUME, + version: 1, + reserved: 0, + }; + + let ret = unsafe { + DeviceIoControl( + handle, + FSCTL_QUERY_PERSISTENT_VOLUME_STATE, + &volume_state as *const _ as *const core::ffi::c_void, + core::mem::size_of::<FileFsPersistentVolumeInformation>() as u32, + &mut volume_state as *mut _ as *mut core::ffi::c_void, + core::mem::size_of::<FileFsPersistentVolumeInformation>() as u32, + core::ptr::null_mut(), + core::ptr::null_mut(), + ) + }; + + unsafe { CloseHandle(handle) }; + + if ret == 0 { + let err = io::Error::last_os_error(); + // ERROR_INVALID_PARAMETER means not supported on this platform + if err.raw_os_error() == Some(Foundation::ERROR_INVALID_PARAMETER as i32) { + return Ok(false); + } + return Err(err.to_pyexception(vm)); + } + + Ok((volume_state.volume_flags & PERSISTENT_VOLUME_STATE_DEV_VOLUME) != 0) + } + // cwait is available on MSVC only #[cfg(target_env = "msvc")] unsafe extern "C" { @@ -648,10 +1016,10 @@ pub(crate) mod module { conout.as_ptr(), Foundation::GENERIC_READ | Foundation::GENERIC_WRITE, FileSystem::FILE_SHARE_READ | FileSystem::FILE_SHARE_WRITE, - std::ptr::null(), + core::ptr::null(), FileSystem::OPEN_EXISTING, 0, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if console_handle == INVALID_HANDLE_VALUE { @@ -696,16 +1064,14 @@ pub(crate) mod module { argv: Either<PyListRef, PyTupleRef>, vm: &VirtualMachine, ) -> PyResult<intptr_t> { - use std::iter::once; - - let make_widestring = - |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); + use crate::function::FsPath; + use core::iter::once; let path = path.to_wide_cstring(vm)?; let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - let arg = PyStrRef::try_from_object(vm, obj)?; - make_widestring(arg.as_str()) + let fspath = FsPath::try_from_path_like(obj, true, vm)?; + fspath.to_wide_cstring(vm) })?; let first = argv @@ -719,7 +1085,7 @@ pub(crate) mod module { let argv_spawn: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); let result = unsafe { suppress_iph!(_wspawnv(mode, path.as_ptr(), argv_spawn.as_ptr())) }; @@ -739,16 +1105,14 @@ pub(crate) mod module { env: PyDictRef, vm: &VirtualMachine, ) -> PyResult<intptr_t> { - use std::iter::once; - - let make_widestring = - |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); + use crate::function::FsPath; + use core::iter::once; let path = path.to_wide_cstring(vm)?; let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - let arg = PyStrRef::try_from_object(vm, obj)?; - make_widestring(arg.as_str()) + let fspath = FsPath::try_from_path_like(obj, true, vm)?; + fspath.to_wide_cstring(vm) })?; let first = argv @@ -762,35 +1126,35 @@ pub(crate) mod module { let argv_spawn: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); // Build environment strings as "KEY=VALUE\0" wide strings let mut env_strings: Vec<widestring::WideCString> = Vec::new(); for (key, value) in env.into_iter() { - let key = PyStrRef::try_from_object(vm, key)?; - let value = PyStrRef::try_from_object(vm, value)?; - let key_str = key.as_str(); - let value_str = value.as_str(); - - // Validate: no null characters in key or value - if key_str.contains('\0') || value_str.contains('\0') { - return Err(vm.new_value_error("embedded null character")); - } - // Validate: no '=' in key (search from index 1 because on Windows - // starting '=' is allowed for defining hidden environment variables) - if key_str.get(1..).is_some_and(|s| s.contains('=')) { + let key = FsPath::try_from_path_like(key, true, vm)?; + let value = FsPath::try_from_path_like(value, true, vm)?; + let key_str = key.to_string_lossy(); + let value_str = value.to_string_lossy(); + + // Validate: empty key or '=' in key after position 0 + // (search from index 1 because on Windows starting '=' is allowed + // for defining hidden environment variables) + if key_str.is_empty() || key_str.get(1..).is_some_and(|s| s.contains('=')) { return Err(vm.new_value_error("illegal environment variable name")); } let env_str = format!("{}={}", key_str, value_str); - env_strings.push(make_widestring(&env_str)?); + env_strings.push( + widestring::WideCString::from_os_str(&*std::ffi::OsString::from(env_str)) + .map_err(|err| err.to_pyexception(vm))?, + ); } let envp: Vec<*const u16> = env_strings .iter() .map(|s| s.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); let result = unsafe { @@ -815,7 +1179,7 @@ pub(crate) mod module { argv: Either<PyListRef, PyTupleRef>, vm: &VirtualMachine, ) -> PyResult<()> { - use std::iter::once; + use core::iter::once; let make_widestring = |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); @@ -838,7 +1202,7 @@ pub(crate) mod module { let argv_execv: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); if (unsafe { suppress_iph!(_wexecv(path.as_ptr(), argv_execv.as_ptr())) } == -1) { @@ -853,10 +1217,10 @@ pub(crate) mod module { fn execve( path: OsPath, argv: Either<PyListRef, PyTupleRef>, - env: PyDictRef, + env: ArgMapping, vm: &VirtualMachine, ) -> PyResult<()> { - use std::iter::once; + use core::iter::once; let make_widestring = |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); @@ -879,9 +1243,10 @@ pub(crate) mod module { let argv_execve: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); + let env = crate::stdlib::os::envobj_to_dict(env, vm)?; // Build environment strings as "KEY=VALUE\0" wide strings let mut env_strings: Vec<widestring::WideCString> = Vec::new(); for (key, value) in env.into_iter() { @@ -894,9 +1259,10 @@ pub(crate) mod module { if key_str.contains('\0') || value_str.contains('\0') { return Err(vm.new_value_error("embedded null character")); } - // Validate: no '=' in key (search from index 1 because on Windows - // starting '=' is allowed for defining hidden environment variables) - if key_str.get(1..).is_some_and(|s| s.contains('=')) { + // Validate: empty key or '=' in key after position 0 + // (search from index 1 because on Windows starting '=' is allowed + // for defining hidden environment variables) + if key_str.is_empty() || key_str.get(1..).is_some_and(|s| s.contains('=')) { return Err(vm.new_value_error("illegal environment variable name")); } @@ -907,7 +1273,7 @@ pub(crate) mod module { let envp: Vec<*const u16> = env_strings .iter() .map(|s| s.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); if (unsafe { suppress_iph!(_wexecve(path.as_ptr(), argv_execve.as_ptr(), envp.as_ptr())) } @@ -921,11 +1287,52 @@ pub(crate) mod module { #[pyfunction] fn _getfinalpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let real = path - .as_ref() - .canonicalize() - .map_err(|e| e.to_pyexception(vm))?; - Ok(path.mode.process_path(real, vm)) + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, GetFinalPathNameByHandleW, OPEN_EXISTING, + VOLUME_NAME_DOS, + }; + + let wide = path.to_wide_cstring(vm)?; + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + + let mut buffer: Vec<u16> = vec![0; Foundation::MAX_PATH as usize]; + let result = loop { + let ret = unsafe { + GetFinalPathNameByHandleW( + handle, + buffer.as_mut_ptr(), + buffer.len() as u32, + VOLUME_NAME_DOS, + ) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + let _ = unsafe { Foundation::CloseHandle(handle) }; + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + if (ret as usize) < buffer.len() { + let final_path = std::ffi::OsString::from_wide(&buffer[..ret as usize]); + break Ok(path.mode().process_path(final_path, vm)); + } + buffer.resize(ret as usize, 0); + }; + + unsafe { Foundation::CloseHandle(handle) }; + result } #[pyfunction] @@ -937,11 +1344,12 @@ pub(crate) mod module { wpath.as_ptr(), buffer.len() as _, buffer.as_mut_ptr(), - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if ret == 0 { - return Err(vm.new_last_os_error()); + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); } if ret as usize > buffer.len() { buffer.resize(ret as usize, 0); @@ -950,30 +1358,35 @@ pub(crate) mod module { wpath.as_ptr(), buffer.len() as _, buffer.as_mut_ptr(), - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if ret == 0 { - return Err(vm.new_last_os_error()); + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); } } let buffer = widestring::WideCString::from_vec_truncate(buffer); - Ok(path.mode.process_path(buffer.to_os_string(), vm)) + Ok(path.mode().process_path(buffer.to_os_string(), vm)) } #[pyfunction] fn _getvolumepathname(path: OsPath, vm: &VirtualMachine) -> PyResult { let wide = path.to_wide_cstring(vm)?; - let buflen = std::cmp::max(wide.len(), Foundation::MAX_PATH as usize); + let buflen = core::cmp::max(wide.len(), Foundation::MAX_PATH as usize); + if buflen > u32::MAX as usize { + return Err(vm.new_overflow_error("path too long".to_owned())); + } let mut buffer = vec![0u16; buflen]; let ret = unsafe { FileSystem::GetVolumePathNameW(wide.as_ptr(), buffer.as_mut_ptr(), buflen as _) }; if ret == 0 { - return Err(vm.new_last_os_error()); + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); } let buffer = widestring::WideCString::from_vec_truncate(buffer); - Ok(path.mode.process_path(buffer.to_os_string(), vm)) + Ok(path.mode().process_path(buffer.to_os_string(), vm)) } /// Implements _Py_skiproot logic for Windows paths @@ -1053,7 +1466,7 @@ pub(crate) mod module { use crate::builtins::{PyBytes, PyStr}; use rustpython_common::wtf8::Wtf8Buf; - // Handle path-like objects via os.fspath, but without null check (nonstrict=True) + // Handle path-like objects via os.fspath, but without null check (non_strict=True) let path = if let Some(fspath) = vm.get_method(path.clone(), identifier!(vm, __fspath__)) { fspath?.call((), vm)? } else { @@ -1067,7 +1480,7 @@ pub(crate) mod module { (wide, false) } else if let Some(b) = path.downcast_ref::<PyBytes>() { // On Windows, bytes must be valid UTF-8 - this raises UnicodeDecodeError if not - let s = std::str::from_utf8(b.as_bytes()).map_err(|e| { + let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { vm.new_exception_msg( vm.ctx.exceptions.unicode_decode_error.to_owned(), format!( @@ -1139,15 +1552,14 @@ pub(crate) mod module { .iter() .copied() .map(|c| if c == b'/' as u16 { b'\\' as u16 } else { c }) - .chain(std::iter::once(0)) // null-terminated + .chain(core::iter::once(0)) // null-terminated .collect(); - let mut end: *const u16 = std::ptr::null(); + let mut end: *const u16 = core::ptr::null(); let hr = unsafe { windows_sys::Win32::UI::Shell::PathCchSkipRoot(backslashed.as_ptr(), &mut end) }; - if hr == 0 { - // S_OK + if hr >= 0 { assert!(!end.is_null()); let len: usize = unsafe { end.offset_from(backslashed.as_ptr()) } .try_into() @@ -1159,15 +1571,186 @@ pub(crate) mod module { len, backslashed.len() ); - ( - Wtf8Buf::from_wide(&orig[..len]), - Wtf8Buf::from_wide(&orig[len..]), - ) + if len != 0 { + ( + Wtf8Buf::from_wide(&orig[..len]), + Wtf8Buf::from_wide(&orig[len..]), + ) + } else { + (Wtf8Buf::from_wide(&orig), Wtf8Buf::new()) + } } else { (Wtf8Buf::new(), Wtf8Buf::from_wide(&orig)) } } + /// Normalize a wide-char path (faithful port of _Py_normpath_and_size). + /// Uses lastC tracking like the C implementation. + fn normpath_wide(path: &[u16]) -> Vec<u16> { + if path.is_empty() { + return vec![b'.' as u16]; + } + + const SEP: u16 = b'\\' as u16; + const ALTSEP: u16 = b'/' as u16; + const DOT: u16 = b'.' as u16; + + let is_sep = |c: u16| c == SEP || c == ALTSEP; + let sep_or_end = |input: &[u16], idx: usize| idx >= input.len() || is_sep(input[idx]); + + // Work on a mutable copy with normalized separators + let mut buf: Vec<u16> = path + .iter() + .map(|&c| if c == ALTSEP { SEP } else { c }) + .collect(); + + let (drv_size, root_size) = skiproot(&buf); + let prefix_len = drv_size + root_size; + + // p1 = read cursor, p2 = write cursor + let mut p1 = prefix_len; + let mut p2 = prefix_len; + let mut min_p2 = if prefix_len > 0 { prefix_len } else { 0 }; + let mut last_c: u16 = if prefix_len > 0 { + min_p2 = prefix_len - 1; + let c = buf[min_p2]; + // On Windows, if last char of prefix is not SEP, advance min_p2 + if c != SEP { + min_p2 = prefix_len; + } + c + } else { + 0 + }; + + // Skip leading ".\" after prefix + if p1 < buf.len() && buf[p1] == DOT && sep_or_end(&buf, p1 + 1) { + p1 += 1; + last_c = SEP; // treat as if we consumed a separator + while p1 < buf.len() && buf[p1] == SEP { + p1 += 1; + } + } + + while p1 < buf.len() { + let c = buf[p1]; + + if last_c == SEP { + if c == DOT { + let sep_at_1 = sep_or_end(&buf, p1 + 1); + let sep_at_2 = !sep_at_1 && sep_or_end(&buf, p1 + 2); + if sep_at_2 && buf[p1 + 1] == DOT { + // ".." component + let mut p3 = p2; + while p3 != min_p2 && buf[p3 - 1] == SEP { + p3 -= 1; + } + while p3 != min_p2 && buf[p3 - 1] != SEP { + p3 -= 1; + } + if p2 == min_p2 + || (buf[p3] == DOT + && p3 + 1 < buf.len() + && buf[p3 + 1] == DOT + && (p3 + 2 >= buf.len() || buf[p3 + 2] == SEP)) + { + // Previous segment is also ../ or at minimum + buf[p2] = DOT; + p2 += 1; + buf[p2] = DOT; + p2 += 1; + last_c = DOT; + } else if buf[p3] == SEP { + // Absolute path - absorb segment + p2 = p3 + 1; + // last_c stays SEP + } else { + p2 = p3; + // last_c stays SEP + } + p1 += 1; // skip second dot (first dot is current p1) + } else if sep_at_1 { + // "." component - skip + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + } else if c == SEP { + // Collapse multiple separators - skip + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + + p1 += 1; + } + + // Null-terminate style: trim trailing separators + if p2 != min_p2 { + while p2 > min_p2 + 1 && buf[p2 - 1] == SEP { + p2 -= 1; + } + } + + buf.truncate(p2); + + if buf.is_empty() { vec![DOT] } else { buf } + } + + #[pyfunction] + fn _path_normpath(path: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::{PyBytes, PyStr}; + use rustpython_common::wtf8::Wtf8Buf; + + // Handle path-like objects via os.fspath + let path = if let Some(fspath) = vm.get_method(path.clone(), identifier!(vm, __fspath__)) { + fspath?.call((), vm)? + } else { + path + }; + + let (wide, is_bytes): (Vec<u16>, bool) = if let Some(s) = path.downcast_ref::<PyStr>() { + let wide: Vec<u16> = s.as_wtf8().encode_wide().collect(); + (wide, false) + } else if let Some(b) = path.downcast_ref::<PyBytes>() { + let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { + vm.new_exception_msg( + vm.ctx.exceptions.unicode_decode_error.to_owned(), + format!( + "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", + b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), + e.valid_up_to() + ), + ) + })?; + let wide: Vec<u16> = s.encode_utf16().collect(); + (wide, true) + } else { + return Err(vm.new_type_error(format!( + "expected str or bytes, not {}", + path.class().name() + ))); + }; + + let normalized = normpath_wide(&wide); + + if is_bytes { + let s = String::from_utf16(&normalized) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + Ok(vm.ctx.new_bytes(s.into_bytes()).into()) + } else { + let s = Wtf8Buf::from_wide(&normalized); + Ok(vm.ctx.new_str(s).into()) + } + } + #[pyfunction] fn _getdiskusage(path: OsPath, vm: &VirtualMachine) -> PyResult<(u64, u64)> { use FileSystem::GetDiskFreeSpaceExW; @@ -1387,8 +1970,8 @@ pub(crate) mod module { // special case: mode 0o700 sets a protected ACL let res = if args.mode == 0o700 { let mut sec_attr = SECURITY_ATTRIBUTES { - nLength: std::mem::size_of::<SECURITY_ATTRIBUTES>() as u32, - lpSecurityDescriptor: std::ptr::null_mut(), + nLength: core::mem::size_of::<SECURITY_ATTRIBUTES>() as u32, + lpSecurityDescriptor: core::ptr::null_mut(), bInheritHandle: 0, }; // Set a discretionary ACL (D) that is protected (P) and includes @@ -1402,7 +1985,7 @@ pub(crate) mod module { sddl.as_ptr(), SDDL_REVISION_1, &mut sec_attr.lpSecurityDescriptor, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if convert_result == 0 { @@ -1413,7 +1996,7 @@ pub(crate) mod module { unsafe { LocalFree(sec_attr.lpSecurityDescriptor) }; res } else { - unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), std::ptr::null_mut()) } + unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), core::ptr::null_mut()) } }; if res == 0 { @@ -1445,15 +2028,22 @@ pub(crate) mod module { #[pyfunction] fn pipe(vm: &VirtualMachine) -> PyResult<(i32, i32)> { + use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; use windows_sys::Win32::System::Pipes::CreatePipe; + let mut attr = SECURITY_ATTRIBUTES { + nLength: core::mem::size_of::<SECURITY_ATTRIBUTES>() as u32, + lpSecurityDescriptor: core::ptr::null_mut(), + bInheritHandle: 0, + }; + let (read_handle, write_handle) = unsafe { let mut read = MaybeUninit::<isize>::uninit(); let mut write = MaybeUninit::<isize>::uninit(); let res = CreatePipe( read.as_mut_ptr() as *mut _, write.as_mut_ptr() as *mut _, - std::ptr::null(), + &mut attr as *mut _, 0, ); if res == 0 { @@ -1486,7 +2076,7 @@ pub(crate) mod module { type NtQueryInformationProcessFn = unsafe extern "system" fn( process_handle: isize, process_information_class: u32, - process_information: *mut std::ffi::c_void, + process_information: *mut core::ffi::c_void, process_information_length: u32, return_length: *mut u32, ) -> i32; @@ -1509,17 +2099,17 @@ pub(crate) mod module { let Some(func) = func else { return 0; }; - let nt_query: NtQueryInformationProcessFn = unsafe { std::mem::transmute(func) }; + let nt_query: NtQueryInformationProcessFn = unsafe { core::mem::transmute(func) }; - let mut info: PROCESS_BASIC_INFORMATION = unsafe { std::mem::zeroed() }; + let mut info: PROCESS_BASIC_INFORMATION = unsafe { core::mem::zeroed() }; let status = unsafe { nt_query( GetCurrentProcess() as isize, 0, // ProcessBasicInformation - &mut info as *mut _ as *mut std::ffi::c_void, - std::mem::size_of::<PROCESS_BASIC_INFORMATION>() as u32, - std::ptr::null_mut(), + &mut info as *mut _ as *mut core::ffi::c_void, + core::mem::size_of::<PROCESS_BASIC_INFORMATION>() as u32, + core::ptr::null_mut(), ) }; @@ -1585,7 +2175,7 @@ pub(crate) mod module { use windows_sys::Win32::System::IO::DeviceIoControl; use windows_sys::Win32::System::Ioctl::FSCTL_GET_REPARSE_POINT; - let mode = path.mode; + let mode = path.mode(); let wide_path = path.as_ref().to_wide_with_nul(); // Open the file/directory with reparse point flag @@ -1594,15 +2184,19 @@ pub(crate) mod module { wide_path.as_ptr(), 0, // No access needed, just reading reparse data FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if handle == INVALID_HANDLE_VALUE { - return Err(io::Error::last_os_error().to_pyexception(vm)); + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path.clone(), + vm, + )); } // Buffer for reparse data - MAXIMUM_REPARSE_DATA_BUFFER_SIZE is 16384 @@ -1614,19 +2208,23 @@ pub(crate) mod module { DeviceIoControl( handle, FSCTL_GET_REPARSE_POINT, - std::ptr::null(), + core::ptr::null(), 0, buffer.as_mut_ptr() as *mut _, BUFFER_SIZE as u32, &mut bytes_returned, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; unsafe { CloseHandle(handle) }; if result == 0 { - return Err(io::Error::last_os_error().to_pyexception(vm)); + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path.clone(), + vm, + )); } // Parse the reparse data buffer @@ -1663,10 +2261,7 @@ pub(crate) mod module { // PathBuffer starts at offset 16 (sub_offset, sub_length, 16usize) } else { - // Unknown reparse tag - fall back to std::fs::read_link - let link_path = fs::read_link(path.as_ref()) - .map_err(|e| crate::convert::ToPyException::to_pyexception(&e, vm))?; - return Ok(mode.process_path(link_path, vm)); + return Err(vm.new_value_error("not a symbolic link".to_owned())); }; // Extract the substitute name @@ -1684,21 +2279,33 @@ pub(crate) mod module { .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) .collect(); - let mut result_path = std::ffi::OsString::from_wide(&wide_chars); - + let mut wide_chars = wide_chars; // For mount points (junctions), the substitute name typically starts with \??\ // Convert this to \\?\ - let result_str = result_path.to_string_lossy(); - if let Some(stripped) = result_str.strip_prefix(r"\??\") { - // Replace \??\ with \\?\ - let new_path = format!(r"\\?\{}", stripped); - result_path = std::ffi::OsString::from(new_path); + if wide_chars.len() > 4 + && wide_chars[0] == b'\\' as u16 + && wide_chars[1] == b'?' as u16 + && wide_chars[2] == b'?' as u16 + && wide_chars[3] == b'\\' as u16 + { + wide_chars[1] = b'\\' as u16; } + let result_path = std::ffi::OsString::from_wide(&wide_chars); + Ok(mode.process_path(std::path::PathBuf::from(result_path), vm)) } pub(crate) fn support_funcs() -> Vec<SupportFunc> { Vec::new() } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } } diff --git a/crates/vm/src/stdlib/operator.rs b/crates/vm/src/stdlib/operator.rs index 0c048ea2a3f..fb0d652361e 100644 --- a/crates/vm/src/stdlib/operator.rs +++ b/crates/vm/src/stdlib/operator.rs @@ -1,4 +1,4 @@ -pub(crate) use _operator::make_module; +pub(crate) use _operator::module_def; #[pymodule] mod _operator { @@ -323,7 +323,7 @@ mod _operator { ) -> PyResult<bool> { let res = match (a, b) { (Either::A(a), Either::A(b)) => { - if !a.is_ascii() || !b.is_ascii() { + if !a.isascii() || !b.isascii() { return Err(vm.new_type_error( "comparing strings with non-ASCII characters is not supported", )); @@ -340,7 +340,8 @@ mod _operator { Ok(res) } - /// attrgetter(attr, ...) --> attrgetter object + /// attrgetter(attr, /, *attrs) + /// -- /// /// Return a callable object that fetches the given attribute(s) from its operand. /// After f = attrgetter('name'), the call f(r) returns r.name. @@ -356,6 +357,11 @@ mod _operator { #[pyclass(with(Callable, Constructor, Representable))] impl PyAttrGetter { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<(PyTypeRef, PyTupleRef)> { let attrs = vm @@ -440,7 +446,8 @@ mod _operator { } } - /// itemgetter(item, ...) --> itemgetter object + /// itemgetter(item, /, *items) + /// -- /// /// Return a callable object that fetches the given item(s) from its operand. /// After f = itemgetter(2), the call f(r) returns r[2]. @@ -454,6 +461,11 @@ mod _operator { #[pyclass(with(Callable, Constructor, Representable))] impl PyItemGetter { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { let items = vm.ctx.new_tuple(zelf.items.to_vec()); @@ -507,7 +519,8 @@ mod _operator { } } - /// methodcaller(name, ...) --> methodcaller object + /// methodcaller(name, /, *args, **kwargs) + /// -- /// /// Return a callable object that calls the given method on its operand. /// After f = methodcaller('name'), the call f(r) returns r.name(). @@ -523,6 +536,11 @@ mod _operator { #[pyclass(with(Callable, Constructor, Representable))] impl PyMethodCaller { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + #[pymethod] fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { // With no kwargs, return (type(obj), (name, *args)) tuple. diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index f6ffd66759d..2fb71d9ec01 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -2,10 +2,10 @@ use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, - builtins::{PyModule, PySet}, + builtins::{PyDictRef, PyModule, PySet}, common::crt_fd, convert::{IntoPyException, ToPyException, ToPyObject}, - function::{ArgumentError, FromArgs, FuncArgs}, + function::{ArgMapping, ArgumentError, FromArgs, FuncArgs}, }; use std::{fs, io, path::Path}; @@ -20,20 +20,6 @@ pub(crate) fn fs_metadata<P: AsRef<Path>>( } } -#[cfg(unix)] -impl crate::convert::IntoPyException for nix::Error { - fn into_pyexception(self, vm: &VirtualMachine) -> crate::builtins::PyBaseExceptionRef { - io::Error::from(self).into_pyexception(vm) - } -} - -#[cfg(unix)] -impl crate::convert::IntoPyException for rustix::io::Errno { - fn into_pyexception(self, vm: &VirtualMachine) -> crate::builtins::PyBaseExceptionRef { - io::Error::from(self).into_pyexception(vm) - } -} - #[allow(dead_code)] #[derive(FromArgs, Default)] pub struct TargetIsDirectory { @@ -87,6 +73,7 @@ impl<const AVAILABLE: usize> FromArgs for DirFd<'_, AVAILABLE> { Some(o) if vm.is_none(&o) => Ok(DEFAULT_DIR_FD), None => Ok(DEFAULT_DIR_FD), Some(o) => { + warn_if_bool_fd(&o, vm).map_err(Into::<ArgumentError>::into)?; let fd = o.try_index_opt(vm).unwrap_or_else(|| { Err(vm.new_type_error(format!( "argument should be integer or None, not {}", @@ -118,8 +105,25 @@ fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ff .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) } +pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::class::StaticType; + if obj + .class() + .is(crate::builtins::bool_::PyBool::static_type()) + { + crate::stdlib::warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + Ok(()) +} + impl TryFromObject for crt_fd::Owned { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + warn_if_bool_fd(&obj, vm)?; let fd = crt_fd::Raw::try_from_object(vm, obj)?; unsafe { crt_fd::Owned::try_from_raw(fd) }.map_err(|e| e.into_pyexception(vm)) } @@ -127,6 +131,7 @@ impl TryFromObject for crt_fd::Owned { impl TryFromObject for crt_fd::Borrowed<'_> { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + warn_if_bool_fd(&obj, vm)?; let fd = crt_fd::Raw::try_from_object(vm, obj)?; unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }.map_err(|e| e.into_pyexception(vm)) } @@ -149,6 +154,8 @@ pub(super) mod _os { use super::{DirFd, FollowSymlinks, SupportFunc}; #[cfg(windows)] use crate::common::windows::ToWideString; + #[cfg(any(unix, windows))] + use crate::utils::ToCString; use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, builtins::{ @@ -161,23 +168,18 @@ pub(super) mod _os { suppress_iph, }, convert::{IntoPyException, ToPyObject}, - function::{ArgBytesLike, FsPath, FuncArgs, OptionalArg}, - ospath::{IOErrorBuilder, OsPath, OsPathOrFd, OutputMode}, + exceptions::{OSErrorBuilder, ToOSErrorBuilder}, + function::{ArgBytesLike, ArgMemoryBuffer, FsPath, FuncArgs, OptionalArg}, + ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter}, protocol::PyIterReturn, recursion::ReprGuard, - types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter, Unconstructible}, - utils::ToCString, + types::{Destructor, IterNext, Iterable, PyStructSequence, Representable, SelfIter}, vm::VirtualMachine, }; + use core::time::Duration; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; - use std::{ - env, fs, - fs::OpenOptions, - io, - path::PathBuf, - time::{Duration, SystemTime}, - }; + use std::{env, fs, fs::OpenOptions, io, path::PathBuf, time::SystemTime}; const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); @@ -186,10 +188,7 @@ pub(super) mod _os { pub(crate) const SYMLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); #[pyattr] - use libc::{ - O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY, SEEK_CUR, SEEK_END, - SEEK_SET, - }; + use libc::{O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; #[pyattr] pub(crate) const F_OK: u8 = 0; @@ -200,6 +199,15 @@ pub(super) mod _os { #[pyattr] pub(crate) const X_OK: u8 = 1 << 0; + // ST_RDONLY and ST_NOSUID flags for statvfs + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + const ST_RDONLY: libc::c_ulong = libc::ST_RDONLY; + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + const ST_NOSUID: libc::c_ulong = libc::ST_NOSUID; + #[pyfunction] fn close(fileno: crt_fd::Owned) -> io::Result<()> { crt_fd::close(fileno) @@ -263,7 +271,7 @@ pub(super) mod _os { crt_fd::open(&name, flags, mode) } }; - fd.map_err(|err| IOErrorBuilder::with_filename(&err, name, vm)) + fd.map_err(|err| OSErrorBuilder::with_filename_from_errno(&err, name, vm)) } #[pyfunction] @@ -272,51 +280,46 @@ pub(super) mod _os { } #[pyfunction] - fn read(fd: crt_fd::Borrowed<'_>, n: usize, vm: &VirtualMachine) -> io::Result<PyBytesRef> { + fn read(fd: crt_fd::Borrowed<'_>, n: usize, vm: &VirtualMachine) -> PyResult<PyBytesRef> { let mut buffer = vec![0u8; n]; - let n = crt_fd::read(fd, &mut buffer)?; - buffer.truncate(n); - - Ok(vm.ctx.new_bytes(buffer)) + loop { + match crt_fd::read(fd, &mut buffer) { + Ok(n) => { + buffer.truncate(n); + return Ok(vm.ctx.new_bytes(buffer)); + } + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + Err(e) => return Err(e.into_pyexception(vm)), + } + } } #[pyfunction] - fn write(fd: crt_fd::Borrowed<'_>, data: ArgBytesLike) -> io::Result<usize> { - data.with_ref(|b| crt_fd::write(fd, b)) + fn readinto( + fd: crt_fd::Borrowed<'_>, + buffer: ArgMemoryBuffer, + vm: &VirtualMachine, + ) -> PyResult<usize> { + buffer.with_ref(|buf| { + loop { + match crt_fd::read(fd, buf) { + Ok(n) => return Ok(n), + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + Err(e) => return Err(e.into_pyexception(vm)), + } + } + }) } #[pyfunction] - #[pyfunction(name = "unlink")] - fn remove(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { - let [] = dir_fd.0; - #[cfg(windows)] - let is_dir_link = { - // On Windows, we need to check if it's a directory symlink/junction - // using GetFileAttributesW, which doesn't follow symlinks. - // This is similar to CPython's Py_DeleteFileW. - use windows_sys::Win32::Storage::FileSystem::{ - FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, GetFileAttributesW, - INVALID_FILE_ATTRIBUTES, - }; - let wide_path: Vec<u16> = path.path.as_os_str().to_wide_with_nul(); - let attrs = unsafe { GetFileAttributesW(wide_path.as_ptr()) }; - if attrs != INVALID_FILE_ATTRIBUTES { - let is_dir = (attrs & FILE_ATTRIBUTE_DIRECTORY) != 0; - let is_reparse = (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; - is_dir && is_reparse - } else { - false - } - }; - #[cfg(not(windows))] - let is_dir_link = false; - - let res = if is_dir_link { - fs::remove_dir(&path) - } else { - fs::remove_file(&path) - }; - res.map_err(|err| IOErrorBuilder::with_filename(&err, path, vm)) + fn write(fd: crt_fd::Borrowed<'_>, data: ArgBytesLike) -> io::Result<usize> { + data.with_ref(|b| crt_fd::write(fd, b)) } #[cfg(not(windows))] @@ -334,7 +337,7 @@ pub(super) mod _os { let res = unsafe { libc::mkdirat(fd, c_path.as_ptr(), mode as _) }; return if res < 0 { let err = crate::common::os::errno_io_error(); - Err(IOErrorBuilder::with_filename(&err, path, vm)) + Err(OSErrorBuilder::with_filename(&err, path, vm)) } else { Ok(()) }; @@ -344,7 +347,7 @@ pub(super) mod _os { let res = unsafe { libc::mkdir(c_path.as_ptr(), mode as _) }; if res < 0 { let err = crate::common::os::errno_io_error(); - return Err(IOErrorBuilder::with_filename(&err, path, vm)); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); } Ok(()) } @@ -357,29 +360,32 @@ pub(super) mod _os { #[pyfunction] fn rmdir(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { let [] = dir_fd.0; - fs::remove_dir(&path).map_err(|err| IOErrorBuilder::with_filename(&err, path, vm)) + fs::remove_dir(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) } const LISTDIR_FD: bool = cfg!(all(unix, not(target_os = "redox"))); #[pyfunction] fn listdir( - path: OptionalArg<OsPathOrFd<'_>>, + path: OptionalArg<Option<OsPathOrFd<'_>>>, vm: &VirtualMachine, ) -> PyResult<Vec<PyObjectRef>> { - let path = path.unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str("."))); + let path = path + .flatten() + .unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str("."))); let list = match path { OsPathOrFd::Path(path) => { let dir_iter = match fs::read_dir(&path) { Ok(iter) => iter, Err(err) => { - return Err(IOErrorBuilder::with_filename(&err, path, vm)); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); } }; + let mode = path.mode(); dir_iter .map(|entry| match entry { - Ok(entry_path) => Ok(path.mode.process_path(entry_path.file_name(), vm)), - Err(err) => Err(IOErrorBuilder::with_filename(&err, path.clone(), vm)), + Ok(entry_path) => Ok(mode.process_path(entry_path.file_name(), vm)), + Err(err) => Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)), }) .collect::<PyResult<_>>()? } @@ -529,7 +535,7 @@ pub(super) mod _os { 22, format!( "Invalid argument: {}", - std::str::from_utf8(key).unwrap_or("<bytes encoding failure>") + core::str::from_utf8(key).unwrap_or("<bytes encoding failure>") ), ); @@ -543,10 +549,10 @@ pub(super) mod _os { #[pyfunction] fn readlink(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult { - let mode = path.mode; + let mode = path.mode(); let [] = dir_fd.0; let path = - fs::read_link(&path).map_err(|err| IOErrorBuilder::with_filename(&err, path, vm))?; + fs::read_link(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))?; Ok(mode.process_path(path, vm)) } @@ -568,7 +574,7 @@ pub(super) mod _os { ino: AtomicCell<Option<u64>>, } - #[pyclass(with(Representable, Unconstructible))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(Representable))] impl DirEntry { #[pygetset] fn name(&self, vm: &VirtualMachine) -> PyResult { @@ -582,15 +588,16 @@ pub(super) mod _os { #[pymethod] fn is_dir(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { + // Use cached file_type first to avoid stat() calls that may fail + if let Ok(file_type) = &self.file_type + && (!follow_symlinks.0 || !file_type.is_symlink()) + { + return Ok(file_type.is_dir()); + } match super::fs_metadata(&self.pathval, follow_symlinks.0) { Ok(meta) => Ok(meta.is_dir()), Err(e) => { if e.kind() == io::ErrorKind::NotFound { - // On Windows, use cached file_type when file is removed - #[cfg(windows)] - if let Ok(file_type) = &self.file_type { - return Ok(file_type.is_dir()); - } Ok(false) } else { Err(e.into_pyexception(vm)) @@ -601,15 +608,16 @@ pub(super) mod _os { #[pymethod] fn is_file(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { + // Use cached file_type first to avoid stat() calls that may fail + if let Ok(file_type) = &self.file_type + && (!follow_symlinks.0 || !file_type.is_symlink()) + { + return Ok(file_type.is_file()); + } match super::fs_metadata(&self.pathval, follow_symlinks.0) { Ok(meta) => Ok(meta.is_file()), Err(e) => { if e.kind() == io::ErrorKind::NotFound { - // On Windows, use cached file_type when file is removed - #[cfg(windows)] - if let Ok(file_type) = &self.file_type { - return Ok(file_type.is_file()); - } Ok(false) } else { Err(e.into_pyexception(vm)) @@ -638,7 +646,7 @@ pub(super) mod _os { stat( OsPath { path: self.pathval.as_os_str().to_owned(), - mode: OutputMode::String, + origin: None, } .into(), dir_fd, @@ -646,16 +654,28 @@ pub(super) mod _os { vm, ) }; - let lstat = || self.lstat.get_or_try_init(|| do_stat(false)); + let lstat = || match self.lstat.get() { + Some(val) => Ok(val), + None => { + let val = do_stat(false)?; + let _ = self.lstat.set(val); + Ok(self.lstat.get().unwrap()) + } + }; let stat = if follow_symlinks.0 { // if follow_symlinks == true and we aren't a symlink, cache both stat and lstat - self.stat.get_or_try_init(|| { - if self.is_symlink(vm)? { - do_stat(true) - } else { - lstat().cloned() + match self.stat.get() { + Some(val) => val, + None => { + let val = if self.is_symlink(vm)? { + do_stat(true)? + } else { + lstat()?.clone() + }; + let _ = self.stat.set(val); + self.stat.get().unwrap() } - })? + } } else { lstat()? }; @@ -669,11 +689,7 @@ pub(super) mod _os { Some(ino) => Ok(ino), None => { let stat = stat_inner( - OsPath { - path: self.pathval.as_os_str().to_owned(), - mode: OutputMode::String, - } - .into(), + OsPath::new_str(self.pathval.as_os_str()).into(), DirFd::default(), FollowSymlinks(false), ) @@ -728,6 +744,11 @@ pub(super) mod _os { ) -> PyGenericAlias { PyGenericAlias::from_args(cls, args, vm) } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'DirEntry' object".to_owned())) + } } impl Representable for DirEntry { @@ -758,8 +779,6 @@ pub(super) mod _os { } } } - impl Unconstructible for DirEntry {} - #[pyattr] #[pyclass(name = "ScandirIter")] #[derive(Debug, PyPayload)] @@ -768,7 +787,7 @@ pub(super) mod _os { mode: OutputMode, } - #[pyclass(with(IterNext, Iterable, Unconstructible))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(Destructor, IterNext, Iterable))] impl ScandirIterator { #[pymethod] fn close(&self) { @@ -785,8 +804,27 @@ pub(super) mod _os { fn __exit__(zelf: PyRef<Self>, _args: FuncArgs) { zelf.close() } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'ScandirIterator' object".to_owned())) + } + } + impl Destructor for ScandirIterator { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Emit ResourceWarning if the iterator is not yet exhausted/closed + if zelf.entries.read().is_some() { + let _ = crate::stdlib::warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed scandir iterator {:?}", zelf.as_object()), + 1, + vm, + ); + zelf.close(); + } + Ok(()) + } } - impl Unconstructible for ScandirIterator {} impl SelfIter for ScandirIterator {} impl IterNext for ScandirIterator { fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { @@ -859,10 +897,10 @@ pub(super) mod _os { fn scandir(path: OptionalArg<OsPath>, vm: &VirtualMachine) -> PyResult { let path = path.unwrap_or_else(|| OsPath::new_str(".")); let entries = fs::read_dir(&path.path) - .map_err(|err| IOErrorBuilder::with_filename(&err, path.clone(), vm))?; + .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; Ok(ScandirIterator { entries: PyRwLock::new(Some(entries)), - mode: path.mode, + mode: path.mode(), } .into_ref(&vm.ctx) .into()) @@ -908,9 +946,20 @@ pub(super) mod _os { #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_ctime_ns: i128, + // Unix-specific attributes + #[cfg(not(windows))] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_blksize: i64, + #[cfg(not(windows))] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_blocks: i64, + #[cfg(windows)] #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_reparse_tag: u32, + #[cfg(windows)] #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_file_attributes: u32, @@ -920,7 +969,7 @@ pub(super) mod _os { fn from_stat(stat: &StatStruct, vm: &VirtualMachine) -> Self { let (atime, mtime, ctime); #[cfg(any(unix, windows))] - #[cfg(not(target_os = "netbsd"))] + #[cfg(not(any(target_os = "netbsd", target_os = "wasi")))] { atime = (stat.st_atime, stat.st_atime_nsec); mtime = (stat.st_mtime, stat.st_mtime_nsec); @@ -945,13 +994,8 @@ pub(super) mod _os { #[cfg(windows)] let st_reparse_tag = stat.st_reparse_tag; - #[cfg(not(windows))] - let st_reparse_tag = 0; - #[cfg(windows)] let st_file_attributes = stat.st_file_attributes; - #[cfg(not(windows))] - let st_file_attributes = 0; // On Windows, combine st_ino and st_ino_high into a 128-bit value // like _pystat_l128_from_l64_l64 @@ -960,6 +1004,13 @@ pub(super) mod _os { #[cfg(not(windows))] let st_ino = stat.st_ino; + #[cfg(not(windows))] + #[allow(clippy::useless_conversion, reason = "needed for 32-bit platforms")] + let st_blksize = i64::from(stat.st_blksize); + #[cfg(not(windows))] + #[allow(clippy::useless_conversion, reason = "needed for 32-bit platforms")] + let st_blocks = i64::from(stat.st_blocks); + Self { st_mode: vm.ctx.new_pyref(stat.st_mode), st_ino: vm.ctx.new_pyref(st_ino), @@ -977,7 +1028,13 @@ pub(super) mod _os { st_atime_ns: to_ns(atime), st_mtime_ns: to_ns(mtime), st_ctime_ns: to_ns(ctime), + #[cfg(not(windows))] + st_blksize, + #[cfg(not(windows))] + st_blocks, + #[cfg(windows)] st_reparse_tag, + #[cfg(windows)] st_file_attributes, } } @@ -1037,12 +1094,12 @@ pub(super) mod _os { dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, follow_symlinks: FollowSymlinks, ) -> io::Result<Option<StatStruct>> { - let mut stat = std::mem::MaybeUninit::uninit(); + let mut stat = core::mem::MaybeUninit::uninit(); let ret = match file { OsPathOrFd::Path(path) => { use rustpython_common::os::ffi::OsStrExt; let path = path.as_ref().as_os_str().as_bytes(); - let path = match std::ffi::CString::new(path) { + let path = match alloc::ffi::CString::new(path) { Ok(x) => x, Err(_) => return Ok(None), }; @@ -1084,7 +1141,7 @@ pub(super) mod _os { vm: &VirtualMachine, ) -> PyResult { let stat = stat_inner(file.clone(), dir_fd, follow_symlinks) - .map_err(|err| IOErrorBuilder::with_filename(&err, file, vm))? + .map_err(|err| OSErrorBuilder::with_filename(&err, file, vm))? .ok_or_else(|| crate::exceptions::cstring_error(vm))?; Ok(StatResultData::from_stat(&stat, vm).to_pyobject(vm)) } @@ -1115,7 +1172,40 @@ pub(super) mod _os { #[pyfunction] fn chdir(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { env::set_current_dir(&path.path) - .map_err(|err| IOErrorBuilder::with_filename(&err, path, vm)) + .map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))?; + + #[cfg(windows)] + { + // win32_wchdir() + + // On Windows, set the per-drive CWD environment variable (=X:) + // This is required for GetFullPathNameW to work correctly with drive-relative paths + + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::System::Environment::SetEnvironmentVariableW; + + if let Ok(cwd) = env::current_dir() { + let cwd_str = cwd.as_os_str(); + let mut cwd_wide: Vec<u16> = cwd_str.encode_wide().collect(); + + // Check for UNC-like paths (\\server\share or //server/share) + // wcsncmp(new_path, L"\\\\", 2) == 0 || wcsncmp(new_path, L"//", 2) == 0 + let is_unc_like_path = cwd_wide.len() >= 2 + && ((cwd_wide[0] == b'\\' as u16 && cwd_wide[1] == b'\\' as u16) + || (cwd_wide[0] == b'/' as u16 && cwd_wide[1] == b'/' as u16)); + + if !is_unc_like_path { + // Create env var name "=X:" where X is the drive letter + let env_name: [u16; 4] = [b'=' as u16, cwd_wide[0], b':' as u16, 0]; + cwd_wide.push(0); // null-terminate the path + unsafe { + SetEnvironmentVariableW(env_name.as_ptr(), cwd_wide.as_ptr()); + } + } + } + } + + Ok(()) } #[pyfunction] @@ -1125,12 +1215,21 @@ pub(super) mod _os { #[pyfunction] #[pyfunction(name = "replace")] - fn rename(src: OsPath, dst: OsPath, vm: &VirtualMachine) -> PyResult<()> { + fn rename(src: PyObjectRef, dst: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let src = PathConverter::new() + .function("rename") + .argument("src") + .try_path(src, vm)?; + let dst = PathConverter::new() + .function("rename") + .argument("dst") + .try_path(dst, vm)?; + fs::rename(&src.path, &dst.path).map_err(|err| { - IOErrorBuilder::new(&err) - .filename(src) - .filename2(dst) - .into_pyexception(vm) + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + builder.build(vm).upcast() }) } @@ -1195,7 +1294,7 @@ pub(super) mod _os { use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Storage::FileSystem; let handle = crt_fd::as_handle(fd).map_err(|e| e.into_pyexception(vm))?; - let mut distance_to_move: [i32; 2] = std::mem::transmute(position); + let mut distance_to_move: [i32; 2] = core::mem::transmute(position); let ret = FileSystem::SetFilePointer( handle.as_raw_handle(), distance_to_move[0], @@ -1206,7 +1305,7 @@ pub(super) mod _os { -1 } else { distance_to_move[0] = ret as _; - std::mem::transmute::<[i32; 2], i64>(distance_to_move) + core::mem::transmute::<[i32; 2], i64>(distance_to_move) } }; if res < 0 { @@ -1216,14 +1315,76 @@ pub(super) mod _os { } } + #[derive(FromArgs)] + struct LinkArgs { + #[pyarg(any)] + src: OsPath, + #[pyarg(any)] + dst: OsPath, + #[pyarg(named, name = "follow_symlinks", optional)] + follow_symlinks: OptionalArg<bool>, + } + #[pyfunction] - fn link(src: OsPath, dst: OsPath, vm: &VirtualMachine) -> PyResult<()> { - fs::hard_link(&src.path, &dst.path).map_err(|err| { - IOErrorBuilder::new(&err) - .filename(src) - .filename2(dst) - .into_pyexception(vm) - }) + fn link(args: LinkArgs, vm: &VirtualMachine) -> PyResult<()> { + let LinkArgs { + src, + dst, + follow_symlinks, + } = args; + + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + let src_cstr = alloc::ffi::CString::new(src.path.as_os_str().as_bytes()) + .map_err(|_| vm.new_value_error("embedded null byte"))?; + let dst_cstr = alloc::ffi::CString::new(dst.path.as_os_str().as_bytes()) + .map_err(|_| vm.new_value_error("embedded null byte"))?; + + let follow = follow_symlinks.into_option().unwrap_or(true); + let flags = if follow { libc::AT_SYMLINK_FOLLOW } else { 0 }; + + let ret = unsafe { + libc::linkat( + libc::AT_FDCWD, + src_cstr.as_ptr(), + libc::AT_FDCWD, + dst_cstr.as_ptr(), + flags, + ) + }; + + if ret != 0 { + let err = std::io::Error::last_os_error(); + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + return Err(builder.build(vm).upcast()); + } + + Ok(()) + } + + #[cfg(not(unix))] + { + let src_path = match follow_symlinks.into_option() { + Some(true) => { + // Explicit follow_symlinks=True: resolve symlinks + fs::canonicalize(&src.path).unwrap_or_else(|_| PathBuf::from(src.path.clone())) + } + Some(false) | None => { + // Default or explicit no-follow: native hard_link behavior + PathBuf::from(src.path.clone()) + } + }; + + fs::hard_link(&src_path, &dst.path).map_err(|err| { + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + builder.build(vm).upcast() + }) + } } #[cfg(any(unix, windows))] @@ -1334,7 +1495,7 @@ pub(super) mod _os { ) }; if ret < 0 { - Err(IOErrorBuilder::with_filename( + Err(OSErrorBuilder::with_filename( &io::Error::last_os_error(), path_for_err, vm, @@ -1385,14 +1546,14 @@ pub(super) mod _os { .write(true) .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS) .open(&path) - .map_err(|err| IOErrorBuilder::with_filename(&err, path.clone(), vm))?; + .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; let ret = unsafe { - FileSystem::SetFileTime(f.as_raw_handle() as _, std::ptr::null(), &acc, &modif) + FileSystem::SetFileTime(f.as_raw_handle() as _, core::ptr::null(), &acc, &modif) }; if ret == 0 { - Err(IOErrorBuilder::with_filename( + Err(OSErrorBuilder::with_filename( &io::Error::last_os_error(), path, vm, @@ -1428,7 +1589,7 @@ pub(super) mod _os { fn times(vm: &VirtualMachine) -> PyResult { #[cfg(windows)] { - use std::mem::MaybeUninit; + use core::mem::MaybeUninit; use windows_sys::Win32::{Foundation::FILETIME, System::Threading}; let mut _create = MaybeUninit::<FILETIME>::uninit(); @@ -1509,9 +1670,9 @@ pub(super) mod _os { #[pyfunction] fn copy_file_range(args: CopyFileRangeArgs<'_>, vm: &VirtualMachine) -> PyResult<usize> { #[allow(clippy::unnecessary_option_map_or_else)] - let p_offset_src = args.offset_src.as_ref().map_or_else(std::ptr::null, |x| x); + let p_offset_src = args.offset_src.as_ref().map_or_else(core::ptr::null, |x| x); #[allow(clippy::unnecessary_option_map_or_else)] - let p_offset_dst = args.offset_dst.as_ref().map_or_else(std::ptr::null, |x| x); + let p_offset_dst = args.offset_dst.as_ref().map_or_else(core::ptr::null, |x| x); let count: usize = args .count .try_into() @@ -1543,7 +1704,7 @@ pub(super) mod _os { #[pyfunction] fn strerror(e: i32) -> String { - unsafe { std::ffi::CStr::from_ptr(libc::strerror(e)) } + unsafe { core::ffi::CStr::from_ptr(libc::strerror(e)) } .to_string_lossy() .into_owned() } @@ -1555,8 +1716,10 @@ pub(super) mod _os { #[pyfunction] fn truncate(path: PyObjectRef, length: crt_fd::Offset, vm: &VirtualMachine) -> PyResult<()> { - if let Ok(fd) = path.clone().try_into_value(vm) { - return ftruncate(fd, length).map_err(|e| e.into_pyexception(vm)); + match path.clone().try_into_value::<crt_fd::Borrowed<'_>>(vm) { + Ok(fd) => return ftruncate(fd, length).map_err(|e| e.into_pyexception(vm)), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.warning) => return Err(e), + Err(_) => {} } #[cold] @@ -1565,7 +1728,7 @@ pub(super) mod _os { error: std::io::Error, path: OsPath, ) -> crate::builtins::PyBaseExceptionRef { - IOErrorBuilder::with_filename(&error, path, vm) + OSErrorBuilder::with_filename(&error, path, vm) } let path = OsPath::try_from_object(vm, path)?; @@ -1644,10 +1807,10 @@ pub(super) mod _os { } else { let encoding = unsafe { let encoding = libc::nl_langinfo(libc::CODESET); - if encoding.is_null() || encoding.read() == '\0' as libc::c_char { + if encoding.is_null() || encoding.read() == b'\0' as libc::c_char { "UTF-8".to_owned() } else { - std::ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() + core::ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() } }; @@ -1687,6 +1850,103 @@ pub(super) mod _os { #[pyclass(with(PyStructSequence))] impl PyUnameResult {} + // statvfs_result: Result from statvfs or fstatvfs. + // = statvfs_result_fields + #[cfg(all(unix, not(target_os = "redox")))] + #[derive(Debug)] + #[pystruct_sequence_data] + pub(crate) struct StatvfsResultData { + pub f_bsize: libc::c_ulong, // filesystem block size + pub f_frsize: libc::c_ulong, // fragment size + pub f_blocks: libc::fsblkcnt_t, // size of fs in f_frsize units + pub f_bfree: libc::fsblkcnt_t, // free blocks + pub f_bavail: libc::fsblkcnt_t, // free blocks for unprivileged users + pub f_files: libc::fsfilcnt_t, // inodes + pub f_ffree: libc::fsfilcnt_t, // free inodes + pub f_favail: libc::fsfilcnt_t, // free inodes for unprivileged users + pub f_flag: libc::c_ulong, // mount flags + pub f_namemax: libc::c_ulong, // maximum filename length + #[pystruct_sequence(skip)] + pub f_fsid: libc::c_ulong, // filesystem ID (not in tuple but accessible as attribute) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + #[pystruct_sequence(name = "statvfs_result", module = "os", data = "StatvfsResultData")] + pub(crate) struct PyStatvfsResult; + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyclass(with(PyStructSequence))] + impl PyStatvfsResult { + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let seq: PyObjectRef = args.bind(vm)?; + crate::types::struct_sequence_new(cls, seq, vm) + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl StatvfsResultData { + fn from_statvfs(st: libc::statvfs) -> Self { + // f_fsid is a struct on some platforms (e.g., Linux fsid_t) and a scalar on others. + // We extract raw bytes and interpret as a native-endian integer. + // Note: The value may differ across architectures due to endianness. + let f_fsid = { + let ptr = core::ptr::addr_of!(st.f_fsid) as *const u8; + let size = core::mem::size_of_val(&st.f_fsid); + if size >= 8 { + let bytes = unsafe { core::slice::from_raw_parts(ptr, 8) }; + u64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], + bytes[7], + ]) as libc::c_ulong + } else if size >= 4 { + let bytes = unsafe { core::slice::from_raw_parts(ptr, 4) }; + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as libc::c_ulong + } else { + 0 + } + }; + + Self { + f_bsize: st.f_bsize, + f_frsize: st.f_frsize, + f_blocks: st.f_blocks, + f_bfree: st.f_bfree, + f_bavail: st.f_bavail, + f_files: st.f_files, + f_ffree: st.f_ffree, + f_favail: st.f_favail, + f_flag: st.f_flag, + f_namemax: st.f_namemax, + f_fsid, + } + } + } + + /// Perform a statvfs system call on the given path. + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction] + #[pyfunction(name = "fstatvfs")] + fn statvfs(path: OsPathOrFd<'_>, vm: &VirtualMachine) -> PyResult { + let mut st: libc::statvfs = unsafe { core::mem::zeroed() }; + let ret = match &path { + OsPathOrFd::Path(p) => { + let cpath = p.clone().into_cstring(vm)?; + unsafe { libc::statvfs(cpath.as_ptr(), &mut st) } + } + OsPathOrFd::Fd(fd) => unsafe { libc::fstatvfs(fd.as_raw(), &mut st) }, + }; + if ret != 0 { + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path, + vm, + )); + } + Ok(StatvfsResultData::from_statvfs(st).to_pyobject(vm)) + } + pub(super) fn support_funcs() -> Vec<SupportFunc> { let mut supports = super::platform::module::support_funcs(); supports.extend(vec![ @@ -1694,6 +1954,7 @@ pub(super) mod _os { SupportFunc::new("access", Some(false), Some(false), None), SupportFunc::new("chdir", None, Some(false), Some(false)), // chflags Some, None Some + SupportFunc::new("link", Some(false), Some(false), Some(cfg!(unix))), SupportFunc::new("listdir", Some(LISTDIR_FD), Some(false), Some(false)), SupportFunc::new("mkdir", Some(false), Some(MKDIR_DIR_FD), Some(false)), // mkfifo Some Some None @@ -1709,6 +1970,8 @@ pub(super) mod _os { SupportFunc::new("fstat", Some(true), Some(STAT_DIR_FD), Some(true)), SupportFunc::new("symlink", Some(false), Some(SYMLINK_DIR_FD), Some(false)), SupportFunc::new("truncate", Some(true), Some(false), Some(false)), + SupportFunc::new("ftruncate", Some(true), Some(false), Some(false)), + SupportFunc::new("fsync", Some(true), Some(false), Some(false)), SupportFunc::new( "utime", Some(false), @@ -1747,21 +2010,21 @@ impl SupportFunc { } } -pub fn extend_module(vm: &VirtualMachine, module: &Py<PyModule>) { +pub fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { let support_funcs = _os::support_funcs(); let supports_fd = PySet::default().into_ref(&vm.ctx); let supports_dir_fd = PySet::default().into_ref(&vm.ctx); let supports_follow_symlinks = PySet::default().into_ref(&vm.ctx); for support in support_funcs { - let func_obj = module.get_attr(support.name, vm).unwrap(); + let func_obj = module.get_attr(support.name, vm)?; if support.fd.unwrap_or(false) { - supports_fd.clone().add(func_obj.clone(), vm).unwrap(); + supports_fd.clone().add(func_obj.clone(), vm)?; } if support.dir_fd.unwrap_or(false) { - supports_dir_fd.clone().add(func_obj.clone(), vm).unwrap(); + supports_dir_fd.clone().add(func_obj.clone(), vm)?; } if support.follow_symlinks.unwrap_or(false) { - supports_follow_symlinks.clone().add(func_obj, vm).unwrap(); + supports_follow_symlinks.clone().add(func_obj, vm)?; } } @@ -1771,6 +2034,34 @@ pub fn extend_module(vm: &VirtualMachine, module: &Py<PyModule>) { "supports_follow_symlinks" => supports_follow_symlinks, "error" => vm.ctx.exceptions.os_error.to_owned(), }); + + Ok(()) +} + +/// Convert a mapping (e.g. os._Environ) to a plain dict for use by execve/posix_spawn. +/// +/// For `os._Environ`, accesses the internal `_data` dict directly at the Rust level. +/// This avoids Python-level method calls that can deadlock after fork() when +/// parking_lot locks are held by threads that no longer exist. +pub(crate) fn envobj_to_dict(env: ArgMapping, vm: &VirtualMachine) -> PyResult<PyDictRef> { + let obj = env.obj(); + if let Some(dict) = obj.downcast_ref_if_exact::<crate::builtins::PyDict>(vm) { + return Ok(dict.to_owned()); + } + if let Some(inst_dict) = obj.dict() + && let Ok(Some(data)) = inst_dict.get_item_opt("_data", vm) + && let Some(dict) = data.downcast_ref_if_exact::<crate::builtins::PyDict>(vm) + { + return Ok(dict.to_owned()); + } + let keys = vm.call_method(obj, "keys", ())?; + let dict = vm.ctx.new_dict(); + for key in keys.get_iter(vm)?.into_iter::<PyObjectRef>(vm)? { + let key = key?; + let val = obj.get_item(&*key, vm)?; + dict.set_item(&*key, val, vm)?; + } + Ok(dict) } #[cfg(not(windows))] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 680e9914a03..ce70412df76 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1,8 +1,9 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; use std::os::fd::BorrowedFd; +pub(crate) use module::module_def; + pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> { use nix::fcntl; let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); @@ -14,37 +15,53 @@ pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> Ok(()) } -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - -#[pymodule(name = "posix", with(super::os::_os))] +#[pymodule(name = "posix", with( + super::os::_os, + #[cfg(any( + target_os = "linux", + target_os = "netbsd", + target_os = "freebsd", + target_os = "android" + ))] + posix_sched +))] pub mod module { use crate::{ - AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyType, PyUtf8StrRef}, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyInt, PyListRef, PyStr, PyTupleRef}, convert::{IntoPyException, ToPyObject, TryFromObject}, - function::{Either, KwArgs, OptionalArg}, - ospath::{IOErrorBuilder, OsPath, OsPathOrFd}, - stdlib::os::{_os, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, fs_metadata}, - types::{Constructor, Representable}, - utils::ToCString, + exceptions::OSErrorBuilder, + function::{ArgMapping, Either, KwArgs, OptionalArg}, + ospath::{OsPath, OsPathOrFd}, + stdlib::os::{ + _os, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, fs_metadata, + warn_if_bool_fd, + }, }; + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + use crate::{builtins::PyStrRef, utils::ToCString}; + use alloc::ffi::CString; use bitflags::bitflags; + use core::ffi::CStr; use nix::{ fcntl, unistd::{self, Gid, Pid, Uid}, }; use std::{ - env, - ffi::{CStr, CString}, - fs, io, + env, fs, io, os::fd::{AsFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd}, }; use strum_macros::{EnumIter, EnumString}; + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use libc::{SCHED_DEADLINE, SCHED_NORMAL}; + #[cfg(target_os = "freebsd")] #[pyattr] use libc::{MFD_HUGE_MASK, SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; @@ -278,6 +295,7 @@ pub mod module { impl TryFromObject for BorrowedFd<'_> { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + crate::stdlib::os::warn_if_bool_fd(&obj, vm)?; let fd = i32::try_from_object(vm, obj)?; if fd == -1 { return Err(io::Error::from_raw_os_error(libc::EBADF).into_pyexception(vm)); @@ -359,9 +377,9 @@ pub mod module { #[cfg(any(target_os = "macos", target_os = "ios"))] fn getgroups_impl() -> nix::Result<Vec<Gid>> { + use core::ptr; use libc::{c_int, gid_t}; use nix::errno::Errno; - use std::ptr; let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; let mut groups = Vec::<Gid>::with_capacity(Errno::result(ret)? as usize); let ret = unsafe { @@ -404,16 +422,17 @@ pub mod module { ) })?; - let metadata = fs::metadata(&path.path); + let metadata = match fs::metadata(&path.path) { + Ok(m) => m, + // If the file doesn't exist, return False for any access check + Err(_) => return Ok(false), + }; // if it's only checking for F_OK if flags == AccessFlags::F_OK { - return Ok(metadata.is_ok()); + return Ok(true); // File exists } - let metadata = - metadata.map_err(|err| IOErrorBuilder::with_filename(&err, path.clone(), vm))?; - let user_id = metadata.uid(); let group_id = metadata.gid(); let mode = metadata.mode(); @@ -442,6 +461,19 @@ pub mod module { environ } + #[pyfunction] + fn _create_environ(vm: &VirtualMachine) -> PyDictRef { + use rustpython_common::os::ffi::OsStringExt; + + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars_os() { + let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); + let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); + environ.set_item(&*key, value, vm).unwrap(); + } + environ + } + #[derive(FromArgs)] pub(super) struct SymlinkArgs<'fd> { src: OsPath, @@ -473,21 +505,35 @@ pub mod module { } } + #[pyfunction] + #[pyfunction(name = "unlink")] + fn remove(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { + let [] = dir_fd.0; + fs::remove_file(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) + } + #[cfg(not(target_os = "redox"))] #[pyfunction] - fn fchdir(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<()> { - nix::unistd::fchdir(fd).map_err(|err| err.into_pyexception(vm)) + fn fchdir(fd: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + warn_if_bool_fd(&fd, vm)?; + let fd = i32::try_from_object(vm, fd)?; + let ret = unsafe { libc::fchdir(fd) }; + if ret == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error().into_pyexception(vm)) + } } #[cfg(not(target_os = "redox"))] #[pyfunction] fn chroot(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { - use crate::ospath::IOErrorBuilder; + use crate::exceptions::OSErrorBuilder; nix::unistd::chroot(&*path.path).map_err(|err| { // Use `From<nix::Error> for io::Error` when it is available - let err = io::Error::from_raw_os_error(err as i32); - IOErrorBuilder::with_filename(&err, path, vm) + let io_err: io::Error = err.into(); + OSErrorBuilder::with_filename(&io_err, path, vm) }) } @@ -533,7 +579,7 @@ pub mod module { .map_err(|err| { // Use `From<nix::Error> for io::Error` when it is available let err = io::Error::from_raw_os_error(err as i32); - IOErrorBuilder::with_filename(&err, path, vm) + OSErrorBuilder::with_filename(&err, path, vm) }) } @@ -652,7 +698,33 @@ pub mod module { } fn py_os_after_fork_child(vm: &VirtualMachine) { - let after_forkers_child: Vec<PyObjectRef> = vm.state.after_forkers_child.lock().clone(); + // Reset low-level state before any Python code runs in the child. + // Signal triggers from the parent must not fire in the child. + crate::signal::clear_after_fork(); + crate::stdlib::signal::_signal::clear_wakeup_fd_after_fork(); + + // Reset weakref stripe locks that may have been held during fork. + #[cfg(feature = "threading")] + crate::object::reset_weakref_locks_after_fork(); + + // Mark all other threads as done before running Python callbacks + #[cfg(feature = "threading")] + crate::stdlib::thread::after_fork_child(vm); + + // Initialize signal handlers for the child's main thread. + // When forked from a worker thread, the OnceCell is empty. + vm.signal_handlers + .get_or_init(crate::signal::new_signal_handlers); + + let after_forkers_child = match vm.state.after_forkers_child.try_lock() { + Some(guard) => guard.clone(), + None => { + // SAFETY: After fork in child process, only the current thread + // exists. The lock holder no longer exists. + unsafe { vm.state.after_forkers_child.force_unlock() }; + vm.state.after_forkers_child.lock().clone() + } + }; run_at_forkers(after_forkers_child, false, vm); } @@ -661,8 +733,64 @@ pub mod module { run_at_forkers(after_forkers_parent, false, vm); } + /// Warn if forking from a multi-threaded process + fn warn_if_multi_threaded(name: &str, vm: &VirtualMachine) { + // Only check threading if it was already imported + // Avoid vm.import() which can execute arbitrary Python code in the fork path + let threading = match vm + .sys_module + .get_attr("modules", vm) + .and_then(|m| m.get_item("threading", vm)) + { + Ok(m) => m, + Err(_) => return, + }; + let active = threading.get_attr("_active", vm).ok(); + let limbo = threading.get_attr("_limbo", vm).ok(); + + let count_dict = |obj: Option<crate::PyObjectRef>| -> usize { + obj.and_then(|o| o.length_opt(vm)) + .and_then(|r| r.ok()) + .unwrap_or(0) + }; + + let num_threads = count_dict(active) + count_dict(limbo); + if num_threads > 1 { + // Use Python warnings module to ensure filters are applied correctly + let Ok(warnings) = vm.import("warnings", 0) else { + return; + }; + let Ok(warn_fn) = warnings.get_attr("warn", vm) else { + return; + }; + + let pid = unsafe { libc::getpid() }; + let msg = format!( + "This process (pid={}) is multi-threaded, use of {}() may lead to deadlocks in the child.", + pid, name + ); + + // Call warnings.warn(message, DeprecationWarning, stacklevel=2) + // stacklevel=2 to point to the caller of fork() + let args = crate::function::FuncArgs::new( + vec![ + vm.ctx.new_str(msg).into(), + vm.ctx.exceptions.deprecation_warning.as_object().to_owned(), + ], + crate::function::KwArgs::new( + [("stacklevel".to_owned(), vm.ctx.new_int(2).into())] + .into_iter() + .collect(), + ), + ); + let _ = warn_fn.call(args, vm); + } + } + #[pyfunction] fn fork(vm: &VirtualMachine) -> i32 { + warn_if_multi_threaded("fork", vm); + let pid: i32; py_os_before_fork(vm); unsafe { @@ -782,175 +910,6 @@ pub mod module { nix::sched::sched_yield().map_err(|e| e.into_pyexception(vm)) } - #[pyattr] - #[pyclass(name = "sched_param")] - #[derive(Debug, PyPayload)] - struct SchedParam { - sched_priority: PyObjectRef, - } - - impl TryFromObject for SchedParam { - fn try_from_object(_vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(Self { - sched_priority: obj, - }) - } - } - - #[pyclass(with(Constructor, Representable))] - impl SchedParam { - #[pygetset] - fn sched_priority(&self, vm: &VirtualMachine) -> PyObjectRef { - self.sched_priority.clone().to_pyobject(vm) - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - fn try_to_libc(&self, vm: &VirtualMachine) -> PyResult<libc::sched_param> { - use crate::AsObject; - let priority_class = self.sched_priority.class(); - let priority_type = priority_class.name(); - let priority = self.sched_priority.clone(); - let value = priority.downcast::<PyInt>().map_err(|_| { - vm.new_type_error(format!("an integer is required (got type {priority_type})")) - })?; - let sched_priority = value.try_to_primitive(vm)?; - Ok(libc::sched_param { sched_priority }) - } - } - - #[derive(FromArgs)] - pub struct SchedParamArg { - sched_priority: PyObjectRef, - } - - impl Constructor for SchedParam { - type Args = SchedParamArg; - - fn py_new(_cls: &Py<PyType>, arg: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - Ok(Self { - sched_priority: arg.sched_priority, - }) - } - } - - impl Representable for SchedParam { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let sched_priority_repr = zelf.sched_priority.repr(vm)?; - Ok(format!( - "posix.sched_param(sched_priority = {})", - sched_priority_repr.as_str() - )) - } - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[pyfunction] - fn sched_getscheduler(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<i32> { - let policy = unsafe { libc::sched_getscheduler(pid) }; - if policy == -1 { - Err(vm.new_last_errno_error()) - } else { - Ok(policy) - } - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[derive(FromArgs)] - struct SchedSetschedulerArgs { - #[pyarg(positional)] - pid: i32, - #[pyarg(positional)] - policy: i32, - #[pyarg(positional)] - sched_param_obj: crate::PyRef<SchedParam>, - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - #[pyfunction] - fn sched_setscheduler(args: SchedSetschedulerArgs, vm: &VirtualMachine) -> PyResult<i32> { - let libc_sched_param = args.sched_param_obj.try_to_libc(vm)?; - let policy = unsafe { libc::sched_setscheduler(args.pid, args.policy, &libc_sched_param) }; - if policy == -1 { - Err(vm.new_last_errno_error()) - } else { - Ok(policy) - } - } - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[pyfunction] - fn sched_getparam(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<SchedParam> { - let param = unsafe { - let mut param = std::mem::MaybeUninit::uninit(); - if -1 == libc::sched_getparam(pid, param.as_mut_ptr()) { - return Err(vm.new_last_errno_error()); - } - param.assume_init() - }; - Ok(SchedParam { - sched_priority: param.sched_priority.to_pyobject(vm), - }) - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[derive(FromArgs)] - struct SchedSetParamArgs { - #[pyarg(positional)] - pid: i32, - #[pyarg(positional)] - sched_param_obj: crate::PyRef<SchedParam>, - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - #[pyfunction] - fn sched_setparam(args: SchedSetParamArgs, vm: &VirtualMachine) -> PyResult<i32> { - let libc_sched_param = args.sched_param_obj.try_to_libc(vm)?; - let ret = unsafe { libc::sched_setparam(args.pid, &libc_sched_param) }; - if ret == -1 { - Err(vm.new_last_errno_error()) - } else { - Ok(ret) - } - } - #[pyfunction] fn get_inheritable(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<bool> { let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD); @@ -1031,7 +990,7 @@ pub mod module { permissions.set_mode(mode); fs::set_permissions(&path, permissions) }; - body().map_err(|err| IOErrorBuilder::with_filename(&err, err_path, vm)) + body().map_err(|err| OSErrorBuilder::with_filename(&err, err_path, vm)) } #[cfg(not(target_os = "redox"))] @@ -1093,7 +1052,7 @@ pub mod module { Ok(()) } else { let err = std::io::Error::last_os_error(); - Err(IOErrorBuilder::with_filename(&err, path, vm)) + Err(OSErrorBuilder::with_filename(&err, path, vm)) } } @@ -1106,7 +1065,7 @@ pub mod module { let path = path.into_cstring(vm)?; let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - PyStrRef::try_from_object(vm, obj)?.to_cstring(vm) + OsPath::try_from_object(vm, obj)?.into_cstring(vm) })?; let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); @@ -1126,13 +1085,13 @@ pub mod module { fn execve( path: OsPath, argv: Either<PyListRef, PyTupleRef>, - env: PyDictRef, + env: ArgMapping, vm: &VirtualMachine, ) -> PyResult<()> { let path = path.into_cstring(vm)?; let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - PyStrRef::try_from_object(vm, obj)?.to_cstring(vm) + OsPath::try_from_object(vm, obj)?.into_cstring(vm) })?; let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); @@ -1144,6 +1103,7 @@ pub mod module { return Err(vm.new_value_error("execve() arg 2 first element cannot be empty")); } + let env = crate::stdlib::os::envobj_to_dict(env, vm)?; let env = env .into_iter() .map(|(k, v)| -> PyResult<_> { @@ -1152,7 +1112,7 @@ pub mod module { OsPath::try_from_object(vm, v)?.into_bytes(), ); - if memchr::memchr(b'=', &key).is_some() { + if key.is_empty() || memchr::memchr(b'=', &key).is_some() { return Err(vm.new_value_error("illegal environment variable name")); } @@ -1238,6 +1198,12 @@ pub mod module { .map_err(|err| err.into_pyexception(vm)) } + #[pyfunction] + fn setpgrp(vm: &VirtualMachine) -> PyResult<()> { + // setpgrp() is equivalent to setpgid(0, 0) + unistd::setpgid(Pid::from_raw(0), Pid::from_raw(0)).map_err(|err| err.into_pyexception(vm)) + } + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] #[pyfunction] fn setsid(vm: &VirtualMachine) -> PyResult<()> { @@ -1246,8 +1212,26 @@ pub mod module { .map_err(|err| err.into_pyexception(vm)) } + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] + #[pyfunction] + fn tcgetpgrp(fd: i32, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + use std::os::fd::BorrowedFd; + let fd = unsafe { BorrowedFd::borrow_raw(fd) }; + unistd::tcgetpgrp(fd) + .map(|pid| pid.as_raw()) + .map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] + #[pyfunction] + fn tcsetpgrp(fd: i32, pgid: libc::pid_t, vm: &VirtualMachine) -> PyResult<()> { + use std::os::fd::BorrowedFd; + let fd = unsafe { BorrowedFd::borrow_raw(fd) }; + unistd::tcsetpgrp(fd, Pid::from_raw(pgid)).map_err(|err| err.into_pyexception(vm)) + } + fn try_from_id(vm: &VirtualMachine, obj: PyObjectRef, typ_name: &str) -> PyResult<u32> { - use std::cmp::Ordering; + use core::cmp::Ordering; let i = obj .try_to_ref::<PyInt>(vm) .map_err(|_| { @@ -1482,7 +1466,7 @@ pub mod module { #[pyarg(positional)] args: crate::function::ArgIterable<OsPath>, #[pyarg(positional)] - env: crate::function::ArgMapping, + env: Option<crate::function::ArgMapping>, #[pyarg(named, default)] file_actions: Option<crate::function::ArgIterable<PyTupleRef>>, #[pyarg(named, default)] @@ -1554,7 +1538,7 @@ pub mod module { }; if let Err(err) = ret { let err = err.into(); - return Err(IOErrorBuilder::with_filename(&err, self.path, vm)); + return Err(OSErrorBuilder::with_filename(&err, self.path, vm)); } } } @@ -1645,7 +1629,22 @@ pub mod module { .map_err(|_| vm.new_value_error("path should not have nul bytes")) }) .collect::<Result<_, _>>()?; - let env = envp_from_dict(self.env, vm)?; + let env = if let Some(env_dict) = self.env { + envp_from_dict(env_dict, vm)? + } else { + // env=None means use the current environment + use rustpython_common::os::ffi::OsStringExt; + env::vars_os() + .map(|(k, v)| { + let mut entry = k.into_vec(); + entry.push(b'='); + entry.extend(v.into_vec()); + CString::new(entry).map_err(|_| { + vm.new_value_error("environment string contains null byte") + }) + }) + .collect::<PyResult<Vec<_>>>()? + }; let ret = if spawnp { nix::spawn::posix_spawnp(&path, &file_actions, &attrp, &args, &env) @@ -1653,7 +1652,7 @@ pub mod module { nix::spawn::posix_spawn(&*path, &file_actions, &attrp, &args, &env) }; ret.map(Into::into) - .map_err(|err| IOErrorBuilder::with_filename(&err.into(), self.path, vm)) + .map_err(|err| OSErrorBuilder::with_filename(&err.into(), self.path, vm)) } } @@ -1709,12 +1708,37 @@ pub mod module { libc::WTERMSIG(status) } + #[cfg(target_os = "linux")] + #[pyfunction] + fn pidfd_open( + pid: libc::pid_t, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<OwnedFd> { + let flags = flags.unwrap_or(0); + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, flags) as libc::c_long }; + if fd == -1 { + Err(vm.new_last_errno_error()) + } else { + // Safety: syscall returns a new owned file descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(fd as libc::c_int) }) + } + } + #[pyfunction] fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { let mut status = 0; - let pid = unsafe { libc::waitpid(pid, &mut status, opt) }; - let pid = nix::Error::result(pid).map_err(|err| err.into_pyexception(vm))?; - Ok((pid, status)) + loop { + let res = unsafe { libc::waitpid(pid, &mut status, opt) }; + if res == -1 { + if nix::Error::last_raw() == libc::EINTR { + vm.check_signals()?; + continue; + } + return Err(nix::Error::last().into_pyexception(vm)); + } + return Ok((res, status)); + } } #[pyfunction] @@ -1769,7 +1793,7 @@ pub mod module { #[cfg(target_os = "macos")] #[pyfunction] fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; + let ret = unsafe { fcopyfile(in_fd, out_fd, core::ptr::null_mut(), flags as u32) }; if ret < 0 { Err(vm.new_last_errno_error()) } else { @@ -1797,9 +1821,9 @@ pub mod module { #[pyfunction] fn dup2(args: Dup2Args<'_>, vm: &VirtualMachine) -> PyResult<OwnedFd> { - let mut fd2 = std::mem::ManuallyDrop::new(args.fd2); + let mut fd2 = core::mem::ManuallyDrop::new(args.fd2); nix::unistd::dup2(args.fd, &mut fd2).map_err(|e| e.into_pyexception(vm))?; - let fd2 = std::mem::ManuallyDrop::into_inner(fd2); + let fd2 = core::mem::ManuallyDrop::into_inner(fd2); if !args.inheritable { super::set_inheritable(fd2.as_fd(), false).map_err(|e| e.into_pyexception(vm))? } @@ -1831,6 +1855,8 @@ pub mod module { SupportFunc::new("umask", Some(false), Some(false), Some(false)), SupportFunc::new("execv", None, None, None), SupportFunc::new("pathconf", Some(true), None, None), + SupportFunc::new("fpathconf", Some(true), None, None), + SupportFunc::new("fchdir", Some(true), None, None), ] } @@ -1925,7 +1951,11 @@ pub mod module { let i = match obj.downcast::<PyInt>() { Ok(int) => int.try_to_primitive(vm)?, Err(obj) => { - let s = PyStrRef::try_from_object(vm, obj)?; + let s = obj.downcast::<PyStr>().map_err(|_| { + vm.new_type_error( + "configuration names must be strings or integers".to_owned(), + ) + })?; s.as_str() .parse::<PathconfVar>() .map_err(|_| vm.new_value_error("unrecognized configuration name"))? @@ -2126,7 +2156,7 @@ pub mod module { if Errno::last_raw() == 0 { Ok(None) } else { - Err(IOErrorBuilder::with_filename( + Err(OSErrorBuilder::with_filename( &io::Error::from(Errno::last()), path, vm, @@ -2319,7 +2349,11 @@ pub mod module { let i = match obj.downcast::<PyInt>() { Ok(int) => int.try_to_primitive(vm)?, Err(obj) => { - let s = PyUtf8StrRef::try_from_object(vm, obj)?; + let s = obj.downcast::<PyStr>().map_err(|_| { + vm.new_type_error( + "configuration names must be strings or integers".to_owned(), + ) + })?; s.as_str().parse::<SysconfVar>().or_else(|_| { if s.as_str() == "SC_PAGESIZE" { Ok(SysconfVar::SC_PAGESIZE) @@ -2443,7 +2477,14 @@ pub mod module { headers, trailers, ); - res.map_err(|err| err.into_pyexception(vm))?; + // On macOS, sendfile can return EAGAIN even when some bytes were written. + // In that case, we should return the number of bytes written rather than + // raising an exception. Only raise an error if no bytes were written. + if let Err(err) = res + && written == 0 + { + return Err(err.into_pyexception(vm)); + } Ok(vm.ctx.new_int(written as u64).into()) } @@ -2470,4 +2511,169 @@ pub mod module { } Ok(buf) } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } +} + +#[cfg(any( + target_os = "linux", + target_os = "netbsd", + target_os = "freebsd", + target_os = "android" +))] +#[pymodule(sub)] +mod posix_sched { + use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyInt, PyTupleRef}, + convert::ToPyObject, + function::FuncArgs, + types::PyStructSequence, + }; + + #[derive(FromArgs)] + struct SchedParamArgs { + #[pyarg(any)] + sched_priority: PyObjectRef, + } + + #[pystruct_sequence_data] + struct SchedParamData { + pub sched_priority: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "sched_param", module = "posix", data = "SchedParamData")] + struct PySchedParam; + + #[pyclass(with(PyStructSequence))] + impl PySchedParam { + #[pyslot] + fn slot_new( + cls: crate::builtins::PyTypeRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult { + use crate::PyPayload; + let SchedParamArgs { sched_priority } = args.bind(vm)?; + let items = vec![sched_priority]; + crate::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + #[extend_class] + fn extend_pyclass(ctx: &crate::vm::Context, class: &'static Py<crate::builtins::PyType>) { + // Override __reduce__ to return (type, (sched_priority,)) + // instead of the generic structseq (type, ((sched_priority,),)). + // The trait's extend_class checks contains_key before setting default. + const SCHED_PARAM_REDUCE: crate::function::PyMethodDef = + crate::function::PyMethodDef::new_const( + "__reduce__", + |zelf: crate::PyRef<crate::builtins::PyTuple>, + vm: &VirtualMachine| + -> PyTupleRef { + vm.new_tuple((zelf.class().to_owned(), (zelf[0].clone(),))) + }, + crate::function::PyMethodFlags::METHOD, + None, + ); + class.set_attr( + ctx.intern_str("__reduce__"), + SCHED_PARAM_REDUCE.to_proper_method(class, ctx), + ); + } + } + + #[cfg(not(target_env = "musl"))] + fn convert_sched_param(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<libc::sched_param> { + use crate::{builtins::PyTuple, class::StaticType}; + if !obj.fast_isinstance(PySchedParam::static_type()) { + return Err(vm.new_type_error("must have a sched_param object".to_owned())); + } + let tuple = obj.downcast_ref::<PyTuple>().unwrap(); + let priority = tuple[0].clone(); + let priority_type = priority.class().name().to_string(); + let value = priority.downcast::<PyInt>().map_err(|_| { + vm.new_type_error(format!("an integer is required (got type {priority_type})")) + })?; + let sched_priority = value.try_to_primitive(vm)?; + Ok(libc::sched_param { sched_priority }) + } + + #[pyfunction] + fn sched_getscheduler(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<i32> { + let policy = unsafe { libc::sched_getscheduler(pid) }; + if policy == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(policy) + } + } + + #[derive(FromArgs)] + struct SchedSetschedulerArgs { + #[pyarg(positional)] + pid: i32, + #[pyarg(positional)] + policy: i32, + #[pyarg(positional)] + sched_param: PyObjectRef, + } + + #[cfg(not(target_env = "musl"))] + #[pyfunction] + fn sched_setscheduler(args: SchedSetschedulerArgs, vm: &VirtualMachine) -> PyResult<i32> { + let libc_sched_param = convert_sched_param(&args.sched_param, vm)?; + let policy = unsafe { libc::sched_setscheduler(args.pid, args.policy, &libc_sched_param) }; + if policy == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(policy) + } + } + + #[pyfunction] + fn sched_getparam(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let param = unsafe { + let mut param = core::mem::MaybeUninit::uninit(); + if -1 == libc::sched_getparam(pid, param.as_mut_ptr()) { + return Err(vm.new_last_errno_error()); + } + param.assume_init() + }; + Ok(PySchedParam::from_data( + SchedParamData { + sched_priority: param.sched_priority.to_pyobject(vm), + }, + vm, + )) + } + + #[derive(FromArgs)] + struct SchedSetParamArgs { + #[pyarg(positional)] + pid: i32, + #[pyarg(positional)] + sched_param: PyObjectRef, + } + + #[cfg(not(target_env = "musl"))] + #[pyfunction] + fn sched_setparam(args: SchedSetParamArgs, vm: &VirtualMachine) -> PyResult<i32> { + let libc_sched_param = convert_sched_param(&args.sched_param, vm)?; + let ret = unsafe { libc::sched_setparam(args.pid, &libc_sched_param) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(ret) + } + } } diff --git a/crates/vm/src/stdlib/posix_compat.rs b/crates/vm/src/stdlib/posix_compat.rs index b2149b43192..89d3d94d7b2 100644 --- a/crates/vm/src/stdlib/posix_compat.rs +++ b/crates/vm/src/stdlib/posix_compat.rs @@ -1,29 +1,32 @@ // spell-checker:disable //! `posix` compatible module for `not(any(unix, windows))` -use crate::{PyRef, VirtualMachine, builtins::PyModule}; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} +pub(crate) use module::module_def; #[pymodule(name = "posix", with(super::os::_os))] pub(crate) mod module { use crate::{ - PyObjectRef, PyResult, VirtualMachine, + Py, PyObjectRef, PyResult, VirtualMachine, builtins::PyStrRef, + convert::IntoPyException, ospath::OsPath, stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory}, }; - use std::env; + use std::{env, fs}; #[pyfunction] pub(super) fn access(_path: PyStrRef, _mode: u8, vm: &VirtualMachine) -> PyResult<bool> { os_unimpl("os.access", vm) } + #[pyfunction] + #[pyfunction(name = "unlink")] + fn remove(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { + let [] = dir_fd.0; + fs::remove_file(&path).map_err(|err| err.into_pyexception(vm)) + } + #[derive(FromArgs)] #[allow(unused)] pub(super) struct SymlinkArgs<'a> { @@ -63,4 +66,13 @@ pub(crate) mod module { pub(crate) fn support_funcs() -> Vec<SupportFunc> { Vec::new() } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } } diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs index e4d7075dbc8..e8aee608cc7 100644 --- a/crates/vm/src/stdlib/pwd.rs +++ b/crates/vm/src/stdlib/pwd.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use pwd::make_module; +pub(crate) use pwd::module_def; #[pymodule] mod pwd { @@ -37,7 +37,7 @@ mod pwd { impl From<User> for PasswdData { fn from(user: User) -> Self { // this is just a pain... - let cstr_lossy = |s: std::ffi::CString| { + let cstr_lossy = |s: alloc::ffi::CString| { s.into_string() .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) }; @@ -105,7 +105,7 @@ mod pwd { let mut list = Vec::new(); unsafe { libc::setpwent() }; - while let Some(ptr) = std::ptr::NonNull::new(unsafe { libc::getpwent() }) { + while let Some(ptr) = core::ptr::NonNull::new(unsafe { libc::getpwent() }) { let user = User::from(unsafe { ptr.as_ref() }); let passwd = PasswdData::from(user).to_pyobject(vm); list.push(passwd); diff --git a/crates/vm/src/stdlib/signal.rs b/crates/vm/src/stdlib/signal.rs index 4eacb10154c..9e51cd3b425 100644 --- a/crates/vm/src/stdlib/signal.rs +++ b/crates/vm/src/stdlib/signal.rs @@ -1,25 +1,18 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _signal::make_module(vm); - - #[cfg(any(unix, windows))] - _signal::init_signal_handlers(&module, vm); - - module -} +pub(crate) use _signal::module_def; #[pymodule] pub(crate) mod _signal { #[cfg(any(unix, windows))] + use crate::convert::{IntoPyException, TryFromBorrowedObject}; + use crate::{Py, PyObjectRef, PyResult, VirtualMachine, signal}; + #[cfg(unix)] use crate::{ - Py, - convert::{IntoPyException, TryFromBorrowedObject}, + builtins::PyTypeRef, + function::{ArgIntoFloat, OptionalArg}, }; - use crate::{PyObjectRef, PyResult, VirtualMachine, signal}; - use std::sync::atomic::{self, Ordering}; + use core::sync::atomic::{self, Ordering}; #[cfg(any(unix, windows))] use libc::sighandler_t; @@ -35,7 +28,7 @@ pub(crate) mod _signal { static WAKEUP: atomic::AtomicUsize = atomic::AtomicUsize::new(INVALID_WAKEUP); // windows doesn't use the same fds for files and sockets like windows does, so we need // this to know whether to send() or write() - static WAKEUP_IS_SOCKET: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + static WAKEUP_IS_SOCKET: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); impl<'a> TryFromBorrowedObject<'a> for WakeupFd { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a crate::PyObject) -> PyResult<Self> { @@ -69,6 +62,11 @@ pub(crate) mod _signal { #[pyattr] pub use libc::{SIG_DFL, SIG_IGN}; + // pthread_sigmask 'how' constants + #[cfg(unix)] + #[pyattr] + use libc::{SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK}; + #[cfg(not(unix))] #[pyattr] pub const SIG_DFL: sighandler_t = 0; @@ -84,6 +82,18 @@ pub(crate) mod _signal { fn siginterrupt(sig: i32, flag: i32) -> i32; } + #[cfg(any(target_os = "linux", target_os = "android"))] + mod ffi { + unsafe extern "C" { + pub fn getitimer(which: libc::c_int, curr_value: *mut libc::itimerval) -> libc::c_int; + pub fn setitimer( + which: libc::c_int, + new_value: *const libc::itimerval, + old_value: *mut libc::itimerval, + ) -> libc::c_int; + } + } + #[pyattr] use crate::signal::NSIG; @@ -91,6 +101,18 @@ pub(crate) mod _signal { #[pyattr] pub use libc::{SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, SIGTERM}; + #[cfg(windows)] + #[pyattr] + const SIGBREAK: i32 = 21; // _SIGBREAK + + // Windows-specific control events for GenerateConsoleCtrlEvent + #[cfg(windows)] + #[pyattr] + const CTRL_C_EVENT: u32 = 0; + #[cfg(windows)] + #[pyattr] + const CTRL_BREAK_EVENT: u32 = 1; + #[cfg(unix)] #[pyattr] use libc::{ @@ -109,12 +131,37 @@ pub(crate) mod _signal { #[pyattr] use libc::{SIGPWR, SIGSTKFLT}; + // Interval timer constants + #[cfg(all(unix, not(target_os = "android")))] + #[pyattr] + use libc::{ITIMER_PROF, ITIMER_REAL, ITIMER_VIRTUAL}; + + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_REAL: libc::c_int = 0; + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_VIRTUAL: libc::c_int = 1; + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_PROF: libc::c_int = 2; + + #[cfg(unix)] + #[pyattr(name = "ItimerError", once)] + fn itimer_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "signal", + "ItimerError", + Some(vec![vm.ctx.exceptions.os_error.to_owned()]), + ) + } + #[cfg(any(unix, windows))] pub(super) fn init_signal_handlers( module: &Py<crate::builtins::PyModule>, vm: &VirtualMachine, ) { - if vm.state.settings.install_signal_handlers { + if vm.state.config.settings.install_signal_handlers { let sig_dfl = vm.new_pyobj(SIG_DFL as u8); let sig_ign = vm.new_pyobj(SIG_IGN as u8); @@ -130,7 +177,9 @@ pub(crate) mod _signal { } else { None }; - vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler; + vm.signal_handlers + .get_or_init(signal::new_signal_handlers) + .borrow_mut()[signum] = py_handler; } let int_handler = module @@ -158,16 +207,30 @@ pub(crate) mod _signal { vm: &VirtualMachine, ) -> PyResult<Option<PyObjectRef>> { signal::assert_in_range(signalnum, vm)?; - let signal_handlers = vm - .signal_handlers - .as_deref() - .ok_or_else(|| vm.new_value_error("signal only works in main thread"))?; + #[cfg(windows)] + { + const VALID_SIGNALS: &[i32] = &[ + libc::SIGINT, + libc::SIGILL, + libc::SIGFPE, + libc::SIGSEGV, + libc::SIGTERM, + SIGBREAK, + libc::SIGABRT, + ]; + if !VALID_SIGNALS.contains(&signalnum) { + return Err(vm.new_value_error(format!("signal number {} out of range", signalnum))); + } + } + if !vm.is_main_thread() { + return Err(vm.new_value_error("signal only works in main thread")); + } let sig_handler = match usize::try_from_borrowed_object(vm, &handler).ok() { Some(SIG_DFL) => SIG_DFL, Some(SIG_IGN) => SIG_IGN, - None if handler.is_callable() => run_signal as sighandler_t, + None if handler.is_callable() => run_signal as *const () as sighandler_t, _ => return Err(vm.new_type_error( "signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object", )), @@ -183,6 +246,7 @@ pub(crate) mod _signal { siginterrupt(signalnum, 1); } + let signal_handlers = vm.signal_handlers.get_or_init(signal::new_signal_handlers); let old_handler = signal_handlers.borrow_mut()[signalnum as usize].replace(handler); Ok(old_handler) } @@ -190,10 +254,7 @@ pub(crate) mod _signal { #[pyfunction] fn getsignal(signalnum: i32, vm: &VirtualMachine) -> PyResult { signal::assert_in_range(signalnum, vm)?; - let signal_handlers = vm - .signal_handlers - .as_deref() - .ok_or_else(|| vm.new_value_error("getsignal only works in main thread"))?; + let signal_handlers = vm.signal_handlers.get_or_init(signal::new_signal_handlers); let handler = signal_handlers.borrow()[signalnum as usize] .clone() .unwrap_or_else(|| vm.ctx.none()); @@ -211,6 +272,80 @@ pub(crate) mod _signal { prev_time.unwrap_or(0) } + #[cfg(unix)] + #[pyfunction] + fn pause(vm: &VirtualMachine) -> PyResult<()> { + unsafe { libc::pause() }; + signal::check_signals(vm)?; + Ok(()) + } + + #[cfg(unix)] + fn timeval_to_double(tv: &libc::timeval) -> f64 { + tv.tv_sec as f64 + (tv.tv_usec as f64 / 1_000_000.0) + } + + #[cfg(unix)] + fn double_to_timeval(val: f64) -> libc::timeval { + libc::timeval { + tv_sec: val.trunc() as _, + tv_usec: ((val.fract()) * 1_000_000.0) as _, + } + } + + #[cfg(unix)] + fn itimerval_to_tuple(it: &libc::itimerval) -> (f64, f64) { + ( + timeval_to_double(&it.it_value), + timeval_to_double(&it.it_interval), + ) + } + + #[cfg(unix)] + #[pyfunction] + fn setitimer( + which: i32, + seconds: ArgIntoFloat, + interval: OptionalArg<ArgIntoFloat>, + vm: &VirtualMachine, + ) -> PyResult<(f64, f64)> { + let seconds: f64 = seconds.into(); + let interval: f64 = interval.map(|v| v.into()).unwrap_or(0.0); + let new = libc::itimerval { + it_value: double_to_timeval(seconds), + it_interval: double_to_timeval(interval), + }; + let mut old = core::mem::MaybeUninit::<libc::itimerval>::uninit(); + #[cfg(any(target_os = "linux", target_os = "android"))] + let ret = unsafe { ffi::setitimer(which, &new, old.as_mut_ptr()) }; + #[cfg(not(any(target_os = "linux", target_os = "android")))] + let ret = unsafe { libc::setitimer(which, &new, old.as_mut_ptr()) }; + if ret != 0 { + let err = std::io::Error::last_os_error(); + let itimer_error = itimer_error(vm); + return Err(vm.new_exception_msg(itimer_error, err.to_string())); + } + let old = unsafe { old.assume_init() }; + Ok(itimerval_to_tuple(&old)) + } + + #[cfg(unix)] + #[pyfunction] + fn getitimer(which: i32, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + let mut old = core::mem::MaybeUninit::<libc::itimerval>::uninit(); + #[cfg(any(target_os = "linux", target_os = "android"))] + let ret = unsafe { ffi::getitimer(which, old.as_mut_ptr()) }; + #[cfg(not(any(target_os = "linux", target_os = "android")))] + let ret = unsafe { libc::getitimer(which, old.as_mut_ptr()) }; + if ret != 0 { + let err = std::io::Error::last_os_error(); + let itimer_error = itimer_error(vm); + return Err(vm.new_exception_msg(itimer_error, err.to_string())); + } + let old = unsafe { old.assume_init() }; + Ok(itimerval_to_tuple(&old)) + } + #[pyfunction] fn default_int_handler( _signum: PyObjectRef, @@ -236,8 +371,8 @@ pub(crate) mod _signal { #[cfg(not(windows))] let fd = args.fd; - if vm.signal_handlers.is_none() { - return Err(vm.new_value_error("signal only works in main thread")); + if !vm.is_main_thread() { + return Err(vm.new_value_error("set_wakeup_fd only works in main thread")); } #[cfg(windows)] @@ -246,7 +381,7 @@ pub(crate) mod _signal { crate::windows::init_winsock(); let mut res = 0i32; - let mut res_size = std::mem::size_of::<i32>() as i32; + let mut res_size = core::mem::size_of::<i32>() as i32; let res = unsafe { WinSock::getsockopt( fd, @@ -310,6 +445,40 @@ pub(crate) mod _signal { } } + #[cfg(target_os = "linux")] + #[pyfunction] + fn pidfd_send_signal( + pidfd: i32, + sig: i32, + siginfo: OptionalArg<PyObjectRef>, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<()> { + signal::assert_in_range(sig, vm)?; + if let OptionalArg::Present(obj) = siginfo + && !vm.is_none(&obj) + { + return Err(vm.new_type_error("siginfo must be None".to_owned())); + } + + let flags = flags.unwrap_or(0); + let ret = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd, + sig, + core::ptr::null::<libc::siginfo_t>(), + flags, + ) as libc::c_long + }; + + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + #[cfg(all(unix, not(target_os = "redox")))] #[pyfunction(name = "siginterrupt")] fn py_siginterrupt(signum: i32, flag: i32, vm: &VirtualMachine) -> PyResult<()> { @@ -331,18 +500,20 @@ pub(crate) mod _signal { // On Windows, only certain signals are supported #[cfg(windows)] { - use crate::convert::IntoPyException; - // Windows supports: SIGINT(2), SIGILL(4), SIGFPE(8), SIGSEGV(11), SIGTERM(15), SIGABRT(22) + // Windows supports: SIGINT(2), SIGILL(4), SIGFPE(8), SIGSEGV(11), SIGTERM(15), SIGBREAK(21), SIGABRT(22) const VALID_SIGNALS: &[i32] = &[ libc::SIGINT, libc::SIGILL, libc::SIGFPE, libc::SIGSEGV, libc::SIGTERM, + SIGBREAK, libc::SIGABRT, ]; if !VALID_SIGNALS.contains(&signalnum) { - return Err(std::io::Error::from_raw_os_error(libc::EINVAL).into_pyexception(vm)); + return Err(vm + .new_errno_error(libc::EINVAL, "Invalid argument") + .upcast()); } } @@ -368,7 +539,7 @@ pub(crate) mod _signal { if s.is_null() { Ok(None) } else { - let cstr = unsafe { std::ffi::CStr::from_ptr(s) }; + let cstr = unsafe { core::ffi::CStr::from_ptr(s) }; Ok(Some(cstr.to_string_lossy().into_owned())) } } @@ -386,6 +557,7 @@ pub(crate) mod _signal { libc::SIGFPE => "Floating-point exception", libc::SIGSEGV => "Segmentation fault", libc::SIGTERM => "Terminated", + SIGBREAK => "Break", libc::SIGABRT => "Aborted", _ => return Ok(None), }; @@ -400,14 +572,17 @@ pub(crate) mod _signal { let set = PySet::default().into_ref(&vm.ctx); #[cfg(unix)] { - // On Unix, most signals 1..NSIG are valid + // Use sigfillset to get all valid signals + let mut mask: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: mask is a valid pointer + if unsafe { libc::sigfillset(&mut mask) } != 0 { + return Err(vm.new_os_error("sigfillset failed".to_owned())); + } + // Convert the filled mask to a Python set for signum in 1..signal::NSIG { - // Skip signals that cannot be caught - #[cfg(not(target_os = "wasi"))] - if signum == libc::SIGKILL as usize || signum == libc::SIGSTOP as usize { - continue; + if unsafe { libc::sigismember(&mask, signum as i32) } == 1 { + set.add(vm.ctx.new_int(signum as i32).into(), vm)?; } - set.add(vm.ctx.new_int(signum as i32).into(), vm)?; } } #[cfg(windows)] @@ -419,6 +594,7 @@ pub(crate) mod _signal { libc::SIGFPE, libc::SIGSEGV, libc::SIGTERM, + SIGBREAK, libc::SIGABRT, ] { set.add(vm.ctx.new_int(signum).into(), vm)?; @@ -432,10 +608,87 @@ pub(crate) mod _signal { Ok(set.into()) } + #[cfg(unix)] + fn sigset_to_pyset(mask: &libc::sigset_t, vm: &VirtualMachine) -> PyResult { + use crate::PyPayload; + use crate::builtins::PySet; + let set = PySet::default().into_ref(&vm.ctx); + for signum in 1..signal::NSIG { + // SAFETY: mask is a valid sigset_t + if unsafe { libc::sigismember(mask, signum as i32) } == 1 { + set.add(vm.ctx.new_int(signum as i32).into(), vm)?; + } + } + Ok(set.into()) + } + + #[cfg(unix)] + #[pyfunction] + fn pthread_sigmask( + how: i32, + mask: crate::function::ArgIterable, + vm: &VirtualMachine, + ) -> PyResult { + use crate::convert::IntoPyException; + + // Initialize sigset + let mut sigset: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: sigset is a valid pointer + if unsafe { libc::sigemptyset(&mut sigset) } != 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + // Add signals to the set + for sig in mask.iter(vm)? { + let sig = sig?; + // Convert to i32, handling overflow by returning ValueError + let signum: i32 = sig.try_to_value(vm).map_err(|_| { + vm.new_value_error(format!( + "signal number out of range [1, {}]", + signal::NSIG - 1 + )) + })?; + // Validate signal number is in range [1, NSIG) + if signum < 1 || signum >= signal::NSIG as i32 { + return Err(vm.new_value_error(format!( + "signal number {} out of range [1, {}]", + signum, + signal::NSIG - 1 + ))); + } + // SAFETY: sigset is a valid pointer and signum is validated + if unsafe { libc::sigaddset(&mut sigset, signum) } != 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + } + + // Call pthread_sigmask + let mut old_mask: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: all pointers are valid + let err = unsafe { libc::pthread_sigmask(how, &sigset, &mut old_mask) }; + if err != 0 { + return Err(std::io::Error::from_raw_os_error(err).into_pyexception(vm)); + } + + // Check for pending signals + signal::check_signals(vm)?; + + // Convert old mask to Python set + sigset_to_pyset(&old_mask, vm) + } + #[cfg(any(unix, windows))] pub extern "C" fn run_signal(signum: i32) { signal::TRIGGERS[signum as usize].store(true, Ordering::Relaxed); signal::set_triggered(); + #[cfg(windows)] + if signum == libc::SIGINT + && let Some(handle) = signal::get_sigint_event() + { + unsafe { + windows_sys::Win32::System::Threading::SetEvent(handle as _); + } + } let wakeup_fd = WAKEUP.load(Ordering::Relaxed); if wakeup_fd != INVALID_WAKEUP { let sigbyte = signum as u8; @@ -455,4 +708,23 @@ pub(crate) mod _signal { // TODO: handle _res < 1, support warn_on_full_buffer } } + + /// Reset wakeup fd after fork in child process. + /// The child must not write to the parent's wakeup fd. + #[cfg(unix)] + pub(crate) fn clear_wakeup_fd_after_fork() { + WAKEUP.store(INVALID_WAKEUP, Ordering::Relaxed); + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + #[cfg(any(unix, windows))] + init_signal_handlers(module, vm); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/sre.rs b/crates/vm/src/stdlib/sre.rs index b950db9e1ff..eb0cb05eb7d 100644 --- a/crates/vm/src/stdlib/sre.rs +++ b/crates/vm/src/stdlib/sre.rs @@ -1,4 +1,4 @@ -pub(crate) use _sre::make_module; +pub(crate) use _sre::module_def; #[pymodule] mod _sre { @@ -190,7 +190,7 @@ mod _sre { } #[pyattr] - #[pyclass(name = "Pattern")] + #[pyclass(module = "re", name = "Pattern")] #[derive(Debug, PyPayload)] pub(crate) struct Pattern { pub pattern: PyObjectRef, @@ -554,7 +554,6 @@ mod _sre { #[inline] fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { let flag_names = [ - ("re.TEMPLATE", SreFlag::TEMPLATE), ("re.IGNORECASE", SreFlag::IGNORECASE), ("re.LOCALE", SreFlag::LOCALE), ("re.MULTILINE", SreFlag::MULTILINE), @@ -598,7 +597,7 @@ mod _sre { } #[pyattr] - #[pyclass(name = "Match")] + #[pyclass(module = "re", name = "Match")] #[derive(Debug, PyPayload)] pub(crate) struct Match { string: PyObjectRef, @@ -737,7 +736,6 @@ mod _sre { }) } - #[pymethod] fn __getitem__( &self, group: PyObjectRef, @@ -838,8 +836,8 @@ mod _sre { impl AsMapping for Match { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: std::sync::LazyLock<PyMappingMethods> = - std::sync::LazyLock::new(|| PyMappingMethods { + static AS_MAPPING: crate::common::lock::LazyLock<PyMappingMethods> = + crate::common::lock::LazyLock::new(|| PyMappingMethods { subscript: atomic_func!(|mapping, needle, vm| { Match::mapping_downcast(mapping) .__getitem__(needle.to_owned(), vm) diff --git a/crates/vm/src/stdlib/stat.rs b/crates/vm/src/stdlib/stat.rs index 90f045fcaf0..44b55628d6f 100644 --- a/crates/vm/src/stdlib/stat.rs +++ b/crates/vm/src/stdlib/stat.rs @@ -1,7 +1,7 @@ -use crate::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _stat::module_def; #[pymodule] -mod stat { +mod _stat { // Use libc::mode_t for Mode to match the system's definition #[cfg(unix)] type Mode = libc::mode_t; @@ -522,7 +522,3 @@ mod stat { result } } - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - stat::make_module(vm) -} diff --git a/crates/vm/src/stdlib/string.rs b/crates/vm/src/stdlib/string.rs index 576cae62775..c620f2a40e2 100644 --- a/crates/vm/src/stdlib/string.rs +++ b/crates/vm/src/stdlib/string.rs @@ -1,7 +1,7 @@ /* String builtin module */ -pub(crate) use _string::make_module; +pub(crate) use _string::module_def; #[pymodule] mod _string { @@ -16,7 +16,7 @@ mod _string { convert::ToPyException, convert::ToPyObject, }; - use std::mem; + use core::mem; fn create_format_part( literal: Wtf8Buf, diff --git a/crates/vm/src/stdlib/symtable.rs b/crates/vm/src/stdlib/symtable.rs index 8a142857787..570a48a8e24 100644 --- a/crates/vm/src/stdlib/symtable.rs +++ b/crates/vm/src/stdlib/symtable.rs @@ -1,16 +1,17 @@ -pub(crate) use symtable::make_module; +pub(crate) use _symtable::module_def; #[pymodule] -mod symtable { +mod _symtable { use crate::{ - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStrRef}, compiler, + types::Representable, }; + use alloc::fmt; use rustpython_codegen::symboltable::{ CompilerScope, Symbol, SymbolFlags, SymbolScope, SymbolTable, }; - use std::fmt; // Consts as defined at // https://github.com/python/cpython/blob/6cb20a219a860eaf687b2d968b41c480c7461909/Include/internal/pycore_symtable.h#L156 @@ -132,7 +133,7 @@ mod symtable { } #[pyattr] - #[pyclass(name = "SymbolTable")] + #[pyclass(name = "symtable entry")] #[derive(PyPayload)] struct PySymbolTable { symtable: SymbolTable, @@ -144,7 +145,7 @@ mod symtable { } } - #[pyclass] + #[pyclass(with(Representable))] impl PySymbolTable { #[pygetset] fn name(&self) -> String { @@ -180,7 +181,7 @@ mod symtable { #[pygetset] fn id(&self) -> usize { - self as *const Self as *const std::ffi::c_void as usize + self as *const Self as *const core::ffi::c_void as usize } #[pygetset] @@ -210,6 +211,19 @@ mod symtable { } } + impl Representable for PySymbolTable { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} {}({}), line {}>", + Self::class(&vm.ctx).name(), + zelf.symtable.name, + zelf.id(), + zelf.symtable.line_number + )) + } + } + #[pyattr] #[pyclass(name = "Symbol")] #[derive(PyPayload)] diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 45b1d566058..3ee6caf2eae 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1,32 +1,58 @@ -use crate::{Py, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; +use crate::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; -pub(crate) use sys::{__module_def, DOC, MAXSIZE, MULTIARCH, UnraisableHookArgsData}; +#[cfg(all(not(feature = "host_env"), feature = "stdio"))] +pub(crate) use sys::SandboxStdio; +pub(crate) use sys::{DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, module_def, multiarch}; + +#[pymodule(name = "_jit")] +mod sys_jit { + /// Return True if the current Python executable supports JIT compilation, + /// and False otherwise. + #[pyfunction] + const fn is_available() -> bool { + false // RustPython has no JIT + } + + /// Return True if JIT compilation is enabled for the current Python process, + /// and False otherwise. + #[pyfunction] + const fn is_enabled() -> bool { + false // RustPython has no JIT + } + + /// Return True if the topmost Python frame is currently executing JIT code, + /// and False otherwise. + #[pyfunction] + const fn is_active() -> bool { + false // RustPython has no JIT + } +} #[pymodule] mod sys { use crate::{ - AsObject, PyObject, PyObjectRef, PyRef, PyRefExact, PyResult, + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, builtins::{ - PyBaseExceptionRef, PyDictRef, PyNamespace, PyStr, PyStrRef, PyTupleRef, PyTypeRef, + PyBaseExceptionRef, PyDictRef, PyFrozenSet, PyNamespace, PyStr, PyStrRef, PyTupleRef, + PyTypeRef, }, common::{ ascii, hash::{PyHash, PyUHash}, }, convert::ToPyObject, - frame::FrameRef, - function::{FuncArgs, OptionalArg, PosArgs}, + frame::{Frame, FrameRef}, + function::{FuncArgs, KwArgs, OptionalArg, PosArgs}, stdlib::{builtins, warnings::warn}, types::PyStructSequence, version, vm::{Settings, VirtualMachine}, }; + use core::sync::atomic::Ordering; use num_traits::ToPrimitive; use std::{ env::{self, VarError}, - io::Read, - path, - sync::atomic::Ordering, + io::{IsTerminal, Read, Write}, }; #[cfg(windows)] @@ -38,16 +64,166 @@ mod sys { System::LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, }; - // not the same as CPython (e.g. rust's x86_x64-unknown-linux-gnu is just x86_64-linux-gnu) - // but hopefully that's just an implementation detail? TODO: copy CPython's multiarch exactly, - // https://github.com/python/cpython/blob/3.8/configure.ac#L725 - pub(crate) const MULTIARCH: &str = env!("RUSTPYTHON_TARGET_TRIPLE"); + // Rust target triple (e.g., "x86_64-unknown-linux-gnu") + pub(crate) const RUST_MULTIARCH: &str = env!("RUSTPYTHON_TARGET_TRIPLE"); + + /// Convert Rust target triple to CPython-style multiarch + /// e.g., "x86_64-unknown-linux-gnu" -> "x86_64-linux-gnu" + pub(crate) fn multiarch() -> String { + RUST_MULTIARCH.replace("-unknown", "") + } + + #[pyclass(no_attr, name = "_BootstrapStderr", module = "sys")] + #[derive(Debug, PyPayload)] + pub(super) struct BootstrapStderr; + + #[pyclass] + impl BootstrapStderr { + #[pymethod] + fn write(&self, s: PyStrRef) -> PyResult<usize> { + let bytes = s.as_bytes(); + let _ = std::io::stderr().write_all(bytes); + Ok(bytes.len()) + } + + #[pymethod] + fn flush(&self) -> PyResult<()> { + let _ = std::io::stderr().flush(); + Ok(()) + } + } + + /// Lightweight stdio wrapper for sandbox mode (no host_env). + /// Directly uses Rust's std::io for stdin/stdout/stderr without FileIO. + #[pyclass(no_attr, name = "_SandboxStdio", module = "sys")] + #[derive(Debug, PyPayload)] + pub struct SandboxStdio { + pub fd: i32, + pub name: String, + pub mode: String, + } + + #[pyclass] + impl SandboxStdio { + #[pymethod] + fn write(&self, s: PyStrRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.fd == 0 { + return Err(vm.new_os_error("not writable".to_owned())); + } + let bytes = s.as_bytes(); + if self.fd == 2 { + std::io::stderr() + .write_all(bytes) + .map_err(|e| vm.new_os_error(e.to_string()))?; + } else { + std::io::stdout() + .write_all(bytes) + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + Ok(bytes.len()) + } + + #[pymethod] + fn readline(&self, size: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<String> { + if self.fd != 0 { + return Err(vm.new_os_error("not readable".to_owned())); + } + let size = size.unwrap_or(-1); + if size == 0 { + return Ok(String::new()); + } + let mut line = String::new(); + std::io::stdin() + .read_line(&mut line) + .map_err(|e| vm.new_os_error(e.to_string()))?; + if size > 0 { + line.truncate(size as usize); + } + Ok(line) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + match self.fd { + 1 => { + std::io::stdout() + .flush() + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + 2 => { + std::io::stderr() + .flush() + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + _ => {} + } + Ok(()) + } + + #[pymethod] + fn fileno(&self) -> i32 { + self.fd + } + + #[pymethod] + fn isatty(&self) -> bool { + match self.fd { + 0 => std::io::stdin().is_terminal(), + 1 => std::io::stdout().is_terminal(), + 2 => std::io::stderr().is_terminal(), + _ => false, + } + } + + #[pymethod] + fn readable(&self) -> bool { + self.fd == 0 + } + + #[pymethod] + fn writable(&self) -> bool { + self.fd == 1 || self.fd == 2 + } + + #[pygetset] + fn closed(&self) -> bool { + false + } + + #[pygetset] + fn encoding(&self) -> String { + "utf-8".to_owned() + } + + #[pygetset] + fn errors(&self) -> String { + if self.fd == 2 { + "backslashreplace" + } else { + "strict" + } + .to_owned() + } + + #[pygetset(name = "name")] + fn name_prop(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "mode")] + fn mode_prop(&self) -> String { + self.mode.clone() + } + } #[pyattr(name = "_rustpython_debugbuild")] const RUSTPYTHON_DEBUGBUILD: bool = cfg!(debug_assertions); + #[cfg(not(windows))] #[pyattr(name = "abiflags")] - pub(crate) const ABIFLAGS: &str = ""; + const ABIFLAGS_ATTR: &str = "t"; // 't' for free-threaded (no GIL) + // Internal constant used for sysconfigdata_name + pub const ABIFLAGS: &str = "t"; #[pyattr(name = "api_version")] const API_VERSION: u32 = 0x0; // what C api? #[pyattr(name = "copyright")] @@ -61,9 +237,9 @@ mod sys { #[pyattr(name = "maxsize")] pub(crate) const MAXSIZE: isize = isize::MAX; #[pyattr(name = "maxunicode")] - const MAXUNICODE: u32 = std::char::MAX as u32; + const MAXUNICODE: u32 = core::char::MAX as u32; #[pyattr(name = "platform")] - pub(crate) const PLATFORM: &str = { + pub const PLATFORM: &str = { cfg_if::cfg_if! { if #[cfg(target_os = "linux")] { "linux" @@ -96,36 +272,36 @@ mod sys { const DLLHANDLE: usize = 0; #[pyattr] - const fn default_prefix(_vm: &VirtualMachine) -> &'static str { - // TODO: the windows one doesn't really make sense - if cfg!(windows) { "C:" } else { "/usr/local" } + fn prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.prefix.clone() } #[pyattr] - fn prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_PREFIX").unwrap_or_else(|| default_prefix(vm)) + fn base_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_prefix.clone() } #[pyattr] - fn base_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) + fn exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.exec_prefix.clone() } #[pyattr] - fn exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) - } - #[pyattr] - fn base_exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| exec_prefix(vm)) + fn base_exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_exec_prefix.clone() } #[pyattr] fn platlibdir(_vm: &VirtualMachine) -> &'static str { option_env!("RUSTPYTHON_PLATLIBDIR").unwrap_or("lib") } + #[pyattr] + fn _stdlib_dir(vm: &VirtualMachine) -> PyObjectRef { + vm.state.config.paths.stdlib_dir.clone().to_pyobject(vm) + } // alphabetical order with segments of pyattr and others #[pyattr] fn argv(vm: &VirtualMachine) -> Vec<PyObjectRef> { vm.state + .config .settings .argv .iter() @@ -135,9 +311,11 @@ mod sys { #[pyattr] fn builtin_module_names(vm: &VirtualMachine) -> PyTupleRef { - let mut module_names: Vec<_> = vm.state.module_inits.keys().cloned().collect(); - module_names.push("sys".into()); - module_names.push("builtins".into()); + let mut module_names: Vec<String> = + vm.state.module_defs.keys().map(|&s| s.to_owned()).collect(); + module_names.push("sys".to_owned()); + module_names.push("builtins".to_owned()); + module_names.sort(); vm.ctx.new_tuple( module_names @@ -147,6 +325,310 @@ mod sys { ) } + // List from cpython/Python/stdlib_module_names.h + const STDLIB_MODULE_NAMES: &[&str] = &[ + "__future__", + "_abc", + "_aix_support", + "_android_support", + "_apple_support", + "_ast", + "_asyncio", + "_bisect", + "_blake2", + "_bz2", + "_codecs", + "_codecs_cn", + "_codecs_hk", + "_codecs_iso2022", + "_codecs_jp", + "_codecs_kr", + "_codecs_tw", + "_collections", + "_collections_abc", + "_colorize", + "_compat_pickle", + "_compression", + "_contextvars", + "_csv", + "_ctypes", + "_curses", + "_curses_panel", + "_datetime", + "_dbm", + "_decimal", + "_elementtree", + "_frozen_importlib", + "_frozen_importlib_external", + "_functools", + "_gdbm", + "_hashlib", + "_heapq", + "_imp", + "_interpchannels", + "_interpqueues", + "_interpreters", + "_io", + "_ios_support", + "_json", + "_locale", + "_lsprof", + "_lzma", + "_markupbase", + "_md5", + "_multibytecodec", + "_multiprocessing", + "_opcode", + "_opcode_metadata", + "_operator", + "_osx_support", + "_overlapped", + "_pickle", + "_posixshmem", + "_posixsubprocess", + "_py_abc", + "_pydatetime", + "_pydecimal", + "_pyio", + "_pylong", + "_pyrepl", + "_queue", + "_random", + "_scproxy", + "_sha1", + "_sha2", + "_sha3", + "_signal", + "_sitebuiltins", + "_socket", + "_sqlite3", + "_sre", + "_ssl", + "_stat", + "_statistics", + "_string", + "_strptime", + "_struct", + "_suggestions", + "_symtable", + "_sysconfig", + "_thread", + "_threading_local", + "_tkinter", + "_tokenize", + "_tracemalloc", + "_typing", + "_uuid", + "_warnings", + "_weakref", + "_weakrefset", + "_winapi", + "_wmi", + "_zoneinfo", + "abc", + "antigravity", + "argparse", + "array", + "ast", + "asyncio", + "atexit", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "genericpath", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msvcrt", + "multiprocessing", + "netrc", + "nt", + "ntpath", + "nturl2path", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "pydoc_data", + "pyexpat", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtplib", + "socket", + "socketserver", + "sqlite3", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "tempfile", + "termios", + "textwrap", + "this", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", + ]; + + #[pyattr(once)] + fn stdlib_module_names(vm: &VirtualMachine) -> PyObjectRef { + let names = STDLIB_MODULE_NAMES + .iter() + .map(|&n| vm.ctx.new_str(n).into()); + PyFrozenSet::from_iter(vm, names) + .expect("Creating stdlib_module_names frozen set must succeed") + .to_pyobject(vm) + } + #[pyattr] fn byteorder(vm: &VirtualMachine) -> PyStrRef { // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian @@ -162,58 +644,18 @@ mod sys { } #[pyattr] - fn _base_executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - if let Ok(var) = env::var("__PYVENV_LAUNCHER__") { - ctx.new_str(var).into() - } else { - executable(vm) - } + fn _base_executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_executable.clone() } #[pyattr] fn dont_write_bytecode(vm: &VirtualMachine) -> bool { - !vm.state.settings.write_bytecode + !vm.state.config.settings.write_bytecode } #[pyattr] - fn executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - #[cfg(not(target_arch = "wasm32"))] - { - if let Some(exec_path) = env::args_os().next() - && let Ok(path) = which::which(exec_path) - { - return ctx - .new_str( - path.into_os_string() - .into_string() - .unwrap_or_else(|p| p.to_string_lossy().into_owned()), - ) - .into(); - } - } - if let Some(exec_path) = env::args().next() { - let path = path::Path::new(&exec_path); - if !path.exists() { - return ctx.new_str(ascii!("")).into(); - } - if path.is_absolute() { - return ctx.new_str(exec_path).into(); - } - if let Ok(dir) = env::current_dir() - && let Ok(dir) = dir.into_os_string().into_string() - { - return ctx - .new_str(format!( - "{}/{}", - dir, - exec_path.strip_prefix("./").unwrap_or(&exec_path) - )) - .into(); - } - } - ctx.none() + fn executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.executable.clone() } #[pyattr] @@ -234,9 +676,10 @@ mod sys { py_namespace!(vm, { "name" => ctx.new_str(NAME), "cache_tag" => ctx.new_str(cache_tag), - "_multiarch" => ctx.new_str(MULTIARCH.to_owned()), + "_multiarch" => ctx.new_str(multiarch()), "version" => version_info(vm), "hexversion" => ctx.new_int(version::VERSION_HEX), + "supports_isolated_interpreters" => ctx.new_bool(false), }) } @@ -253,8 +696,9 @@ mod sys { #[pyattr] fn path(vm: &VirtualMachine) -> Vec<PyObjectRef> { vm.state - .settings - .path_list + .config + .paths + .module_search_paths .iter() .map(|path| vm.ctx.new_str(path.clone()).into()) .collect() @@ -291,7 +735,7 @@ mod sys { fn _xoptions(vm: &VirtualMachine) -> PyDictRef { let ctx = &vm.ctx; let xopts = ctx.new_dict(); - for (key, value) in &vm.state.settings.xoptions { + for (key, value) in &vm.state.config.settings.xoptions { let value = value.as_ref().map_or_else( || ctx.new_bool(true).into(), |s| ctx.new_str(s.clone()).into(), @@ -304,6 +748,7 @@ mod sys { #[pyattr] fn warnoptions(vm: &VirtualMachine) -> Vec<PyObjectRef> { vm.state + .config .settings .warnoptions .iter() @@ -331,6 +776,17 @@ mod sys { // TODO: sys.audit implementation } + #[pyfunction] + const fn _is_gil_enabled() -> bool { + false // RustPython has no GIL (like free-threaded Python) + } + + /// Return True if remote debugging is enabled, False otherwise. + #[pyfunction] + const fn is_remote_debug_enabled() -> bool { + false // RustPython does not support remote debugging + } + #[pyfunction] fn exit(code: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult { let code = code.unwrap_or_none(vm); @@ -367,14 +823,22 @@ mod sys { vm: &VirtualMachine, ) -> PyResult<()> { let stderr = super::get_stderr(vm)?; - - // Try to normalize the exception. If it fails, print error to stderr like CPython match vm.normalize_exception(exc_type.clone(), exc_val.clone(), exc_tb) { - Ok(exc) => vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc), + Ok(exc) => { + // PyErr_Display: try traceback._print_exception_bltin first + if let Ok(tb_mod) = vm.import("traceback", 0) + && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) + && print_exc_builtin + .call((exc.as_object().to_owned(),), vm) + .is_ok() + { + return Ok(()); + } + // Fallback to Rust-level exception printing + vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc) + } Err(_) => { - // CPython prints error message to stderr instead of raising exception let type_name = exc_val.class().name(); - // TODO: fix error message let msg = format!( "TypeError: print_exception(): Exception expected for value, {type_name} found\n" ); @@ -448,7 +912,7 @@ mod sys { #[pyattr] fn flags(vm: &VirtualMachine) -> PyTupleRef { - PyFlags::from_data(FlagsData::from_settings(&vm.state.settings), vm) + PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) } #[pyattr] @@ -483,7 +947,7 @@ mod sys { let sizeof = || -> PyResult<usize> { let res = vm.call_special_method(&args.obj, identifier!(vm, __sizeof__), ())?; let res = res.try_index(vm)?.try_to_primitive::<usize>(vm)?; - Ok(res + std::mem::size_of::<PyObject>()) + Ok(res + core::mem::size_of::<PyObject>()) }; sizeof() .map(|x| vm.ctx.new_int(x).into()) @@ -508,12 +972,14 @@ mod sys { #[pyfunction] fn _getframe(offset: OptionalArg<usize>, vm: &VirtualMachine) -> PyResult<FrameRef> { let offset = offset.into_option().unwrap_or(0); - if offset > vm.frames.borrow().len() - 1 { + let frames = vm.frames.borrow(); + if offset >= frames.len() { return Err(vm.new_value_error("call stack is not deep enough")); } - let idx = vm.frames.borrow().len() - offset - 1; - let frame = &vm.frames.borrow()[idx]; - Ok(frame.clone()) + let idx = frames.len() - offset - 1; + // SAFETY: the FrameRef is alive on the call stack while it's in the Vec + let py: &crate::Py<Frame> = unsafe { frames[idx].as_ref() }; + Ok(py.to_owned()) } #[pyfunction] @@ -521,15 +987,19 @@ mod sys { let depth = depth.into_option().unwrap_or(0); // Get the frame at the specified depth - if depth > vm.frames.borrow().len() - 1 { - return Ok(vm.ctx.none()); - } - - let idx = vm.frames.borrow().len() - depth - 1; - let frame = &vm.frames.borrow()[idx]; + let func_obj = { + let frames = vm.frames.borrow(); + if depth >= frames.len() { + return Ok(vm.ctx.none()); + } + let idx = frames.len() - depth - 1; + // SAFETY: the FrameRef is alive on the call stack while it's in the Vec + let frame: &crate::Py<Frame> = unsafe { frames[idx].as_ref() }; + frame.func_obj.clone() + }; // If the frame has a function object, return its __module__ attribute - if let Some(func_obj) = &frame.func_obj { + if let Some(func_obj) = func_obj { match func_obj.get_attr(identifier!(vm, __module__), vm) { Ok(module) => Ok(module), Err(_) => { @@ -542,6 +1012,32 @@ mod sys { } } + /// Return a dictionary mapping each thread's identifier to the topmost stack frame + /// currently active in that thread at the time the function is called. + #[cfg(feature = "threading")] + #[pyfunction] + fn _current_frames(vm: &VirtualMachine) -> PyResult<PyDictRef> { + use crate::AsObject; + use crate::stdlib::thread::get_all_current_frames; + + let frames = get_all_current_frames(vm); + let dict = vm.ctx.new_dict(); + + for (thread_id, frame) in frames { + let key = vm.ctx.new_int(thread_id); + dict.set_item(key.as_object(), frame.into(), vm)?; + } + + Ok(dict) + } + + /// Stub for non-threading builds - returns empty dict + #[cfg(not(feature = "threading"))] + #[pyfunction] + fn _current_frames(vm: &VirtualMachine) -> PyResult<PyDictRef> { + Ok(vm.ctx.new_dict()) + } + #[pyfunction] fn gettrace(vm: &VirtualMachine) -> PyObjectRef { vm.trace_func.borrow().clone() @@ -571,7 +1067,7 @@ mod sys { // Get the size of the version information block let ver_block_size = - GetFileVersionInfoSizeW(kernel32_path.as_ptr(), std::ptr::null_mut()); + GetFileVersionInfoSizeW(kernel32_path.as_ptr(), core::ptr::null_mut()); if ver_block_size == 0 { return Err(std::io::Error::last_os_error()); } @@ -591,7 +1087,7 @@ mod sys { // Prepare an empty sub-block string (L"") as required by VerQueryValueW let sub_block: Vec<u16> = std::ffi::OsStr::new("").to_wide_with_nul(); - let mut ffi_ptr: *mut VS_FIXEDFILEINFO = std::ptr::null_mut(); + let mut ffi_ptr: *mut VS_FIXEDFILEINFO = core::ptr::null_mut(); let mut ffi_len: u32 = 0; if VerQueryValueW( ver_block.as_ptr() as *const _, @@ -623,8 +1119,8 @@ mod sys { GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW, }; - let mut version: OSVERSIONINFOEXW = unsafe { std::mem::zeroed() }; - version.dwOSVersionInfoSize = std::mem::size_of::<OSVERSIONINFOEXW>() as u32; + let mut version: OSVERSIONINFOEXW = unsafe { core::mem::zeroed() }; + version.dwOSVersionInfoSize = core::mem::size_of::<OSVERSIONINFOEXW>() as u32; let result = unsafe { let os_vi = &mut version as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; // SAFETY: GetVersionExW accepts a pointer of OSVERSIONINFOW, but windows-sys crate's type currently doesn't allow to do so. @@ -688,13 +1184,21 @@ mod sys { writeln!(stderr, "{}:", unraisable.err_msg.str(vm)?); } - // TODO: print received unraisable.exc_traceback - let tb_module = vm.import("traceback", 0)?; - let print_stack = tb_module.get_attr("print_stack", vm)?; - print_stack.call((), vm)?; + // Print traceback (using actual exc_traceback, not current stack) + if !vm.is_none(&unraisable.exc_traceback) { + let tb_module = vm.import("traceback", 0)?; + let print_tb = tb_module.get_attr("print_tb", vm)?; + let stderr_obj = super::get_stderr(vm)?; + let kwargs: KwArgs = [("file".to_string(), stderr_obj)].into_iter().collect(); + let _ = print_tb.call( + FuncArgs::new(vec![unraisable.exc_traceback.clone()], kwargs), + vm, + ); + } + // Check exc_type if vm.is_none(unraisable.exc_type.as_object()) { - // TODO: early return, but with what error? + return Ok(()); } assert!( unraisable @@ -702,10 +1206,28 @@ mod sys { .fast_issubclass(vm.ctx.exceptions.base_exception_type) ); - // TODO: print module name and qualname + // Print module name (if not builtins or __main__) + let module_name = unraisable.exc_type.__module__(vm); + if let Ok(module_str) = module_name.downcast::<PyStr>() { + let module = module_str.as_str(); + if module != "builtins" && module != "__main__" { + write!(stderr, "{}.", module); + } + } else { + write!(stderr, "<unknown>."); + } + // Print qualname + let qualname = unraisable.exc_type.__qualname__(vm); + if let Ok(qualname_str) = qualname.downcast::<PyStr>() { + write!(stderr, "{}", qualname_str.as_str()); + } else { + write!(stderr, "{}", unraisable.exc_type.name()); + } + + // Print exception value if !vm.is_none(&unraisable.exc_value) { - write!(stderr, "{}: ", unraisable.exc_type); + write!(stderr, ": "); if let Ok(str) = unraisable.exc_value.str(vm) { write!(stderr, "{}", str.to_str().unwrap_or("<str with surrogate>")); } else { @@ -713,7 +1235,13 @@ mod sys { } } writeln!(stderr); - // TODO: call file.flush() + + // Flush stderr + if let Ok(stderr_obj) = super::get_stderr(vm) + && let Ok(flush) = stderr_obj.get_attr("flush", vm) + { + let _ = flush.call((), vm); + } Ok(()) } @@ -802,6 +1330,22 @@ mod sys { update_use_tracing(vm); } + #[pyfunction] + fn _settraceallthreads(tracefunc: PyObjectRef, vm: &VirtualMachine) { + let func = (!vm.is_none(&tracefunc)).then(|| tracefunc.clone()); + *vm.state.global_trace_func.lock() = func; + vm.trace_func.replace(tracefunc); + update_use_tracing(vm); + } + + #[pyfunction] + fn _setprofileallthreads(profilefunc: PyObjectRef, vm: &VirtualMachine) { + let func = (!vm.is_none(&profilefunc)).then(|| profilefunc.clone()); + *vm.state.global_profile_func.lock() = func; + vm.profile_func.replace(profilefunc); + update_use_tracing(vm); + } + #[cfg(feature = "threading")] #[pyattr] fn thread_info(vm: &VirtualMachine) -> PyTupleRef { @@ -834,6 +1378,32 @@ mod sys { crate::vm::thread::COROUTINE_ORIGIN_TRACKING_DEPTH.get() as i32 } + #[pyfunction] + fn _clear_type_descriptors(type_obj: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::types::PyTypeFlags; + + // Check if type is immutable + if type_obj.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error("argument is immutable".to_owned())); + } + + let mut attributes = type_obj.attributes.write(); + + // Remove __dict__ descriptor if present + attributes.swap_remove(identifier!(vm, __dict__)); + + // Remove __weakref__ descriptor if present + attributes.swap_remove(identifier!(vm, __weakref__)); + + drop(attributes); + + // Update slots to notify subclasses and recalculate cached values + type_obj.update_slot::<true>(identifier!(vm, __dict__), &vm.ctx); + type_obj.update_slot::<true>(identifier!(vm, __weakref__), &vm.ctx); + + Ok(()) + } + #[pyfunction] fn getswitchinterval(vm: &VirtualMachine) -> f64 { // Return the stored switch interval @@ -882,10 +1452,10 @@ mod sys { } if let Some(finalizer) = args.finalizer.into_option() { - crate::vm::thread::ASYNC_GEN_FINALIZER.set(finalizer); + *vm.async_gen_finalizer.borrow_mut() = finalizer; } if let Some(firstiter) = args.firstiter.into_option() { - crate::vm::thread::ASYNC_GEN_FIRSTITER.set(firstiter); + *vm.async_gen_firstiter.borrow_mut() = firstiter; } Ok(()) @@ -907,12 +1477,8 @@ mod sys { #[pyfunction] fn get_asyncgen_hooks(vm: &VirtualMachine) -> AsyncgenHooksData { AsyncgenHooksData { - firstiter: crate::vm::thread::ASYNC_GEN_FIRSTITER - .with_borrow(Clone::clone) - .to_pyobject(vm), - finalizer: crate::vm::thread::ASYNC_GEN_FINALIZER - .with_borrow(Clone::clone) - .to_pyobject(vm), + firstiter: vm.async_gen_firstiter.borrow().clone().to_pyobject(vm), + finalizer: vm.async_gen_finalizer.borrow().clone().to_pyobject(vm), } } @@ -958,6 +1524,10 @@ mod sys { safe_path: bool, /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING warn_default_encoding: u8, + /// -X thread_inherit_context, whether new threads inherit context from parent + thread_inherit_context: bool, + /// -X context_aware_warnings, whether warnings are context aware + context_aware_warnings: bool, } impl FlagsData { @@ -981,6 +1551,8 @@ mod sys { int_max_str_digits: settings.int_max_str_digits, safe_path: settings.safe_path, warn_default_encoding: settings.warn_default_encoding as u8, + thread_inherit_context: settings.thread_inherit_context, + context_aware_warnings: settings.context_aware_warnings, } } } @@ -1077,7 +1649,7 @@ mod sys { const INFO: Self = { use rustpython_common::hash::*; Self { - width: std::mem::size_of::<PyHash>() * 8, + width: core::mem::size_of::<PyHash>() * 8, modulus: MODULUS, inf: INF, nan: NAN, @@ -1107,7 +1679,7 @@ mod sys { impl IntInfoData { const INFO: Self = Self { bits_per_digit: 30, //? - sizeof_digit: std::mem::size_of::<u32>(), + sizeof_digit: core::mem::size_of::<u32>(), default_max_str_digits: 4300, str_digits_check_threshold: 640, }; @@ -1196,7 +1768,8 @@ mod sys { } pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: &Py<PyModule>) { - sys::extend_module(vm, module).unwrap(); + module.__init_methods(vm).unwrap(); + sys::module_exec(vm, module).unwrap(); let modules = vm.ctx.new_dict(); modules @@ -1205,12 +1778,26 @@ pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: modules .set_item("builtins", builtins.to_owned().into(), vm) .unwrap(); + + // Create sys._jit submodule + let jit_def = sys_jit::module_def(&vm.ctx); + let jit_module = jit_def.create_module(vm).unwrap(); + extend_module!(vm, module, { "__doc__" => sys::DOC.to_owned().to_pyobject(vm), "modules" => modules, + "_jit" => jit_module, }); } +pub(crate) fn set_bootstrap_stderr(vm: &VirtualMachine) -> PyResult<()> { + let stderr = sys::BootstrapStderr.into_ref(&vm.ctx); + let stderr_obj: crate::PyObjectRef = stderr.into(); + vm.sys_module.set_attr("stderr", stderr_obj.clone(), vm)?; + vm.sys_module.set_attr("__stderr__", stderr_obj, vm)?; + Ok(()) +} + /// Similar to PySys_WriteStderr in CPython. /// /// # Usage @@ -1225,7 +1812,7 @@ pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: pub struct PyStderr<'vm>(pub &'vm VirtualMachine); impl PyStderr<'_> { - pub fn write_fmt(&self, args: std::fmt::Arguments<'_>) { + pub fn write_fmt(&self, args: core::fmt::Arguments<'_>) { use crate::py_io::Write; let vm = self.0; @@ -1260,6 +1847,6 @@ pub(crate) fn sysconfigdata_name() -> String { "_sysconfigdata_{}_{}_{}", sys::ABIFLAGS, sys::PLATFORM, - sys::MULTIARCH + sys::multiarch() ) } diff --git a/crates/vm/src/stdlib/sysconfig.rs b/crates/vm/src/stdlib/sysconfig.rs index df5b7100a90..724e1bcf979 100644 --- a/crates/vm/src/stdlib/sysconfig.rs +++ b/crates/vm/src/stdlib/sysconfig.rs @@ -1,7 +1,7 @@ -pub(crate) use sysconfig::make_module; +pub(crate) use _sysconfig::module_def; -#[pymodule(name = "_sysconfig")] -pub(crate) mod sysconfig { +#[pymodule] +pub(crate) mod _sysconfig { use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; #[pyfunction] diff --git a/crates/vm/src/stdlib/sysconfigdata.rs b/crates/vm/src/stdlib/sysconfigdata.rs index 90e46b83b97..a10745a8cad 100644 --- a/crates/vm/src/stdlib/sysconfigdata.rs +++ b/crates/vm/src/stdlib/sysconfigdata.rs @@ -1,23 +1,63 @@ -pub(crate) use _sysconfigdata::make_module; +// spell-checker: words LDSHARED ARFLAGS CPPFLAGS CCSHARED BASECFLAGS BLDSHARED + +pub(crate) use _sysconfigdata::module_def; #[pymodule] -pub(crate) mod _sysconfigdata { - use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject, stdlib::sys::MULTIARCH}; +mod _sysconfigdata { + use crate::stdlib::sys::{RUST_MULTIARCH, multiarch, sysconfigdata_name}; + use crate::{ + Py, PyResult, VirtualMachine, + builtins::{PyDictRef, PyModule}, + convert::ToPyObject, + }; + + fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // Set build_time_vars attribute + let build_time_vars = build_time_vars(vm); + module.set_attr("build_time_vars", build_time_vars, vm)?; + + // Ensure the module is registered under the platform-specific name + // (import_builtin() already handles this, but double-check for safety) + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let sysconfigdata_name = sysconfigdata_name(); + sys_modules.set_item(sysconfigdata_name.as_str(), module.to_owned().into(), vm)?; + + Ok(()) + } #[pyattr] fn build_time_vars(vm: &VirtualMachine) -> PyDictRef { let vars = vm.ctx.new_dict(); + let multiarch = multiarch(); macro_rules! sysvars { ($($key:literal => $value:expr),*$(,)?) => {{ $(vars.set_item($key, $value.to_pyobject(vm), vm).unwrap();)* }}; } sysvars! { - // fake shared module extension - "EXT_SUFFIX" => format!(".rustpython-{MULTIARCH}"), - "MULTIARCH" => MULTIARCH, + // Extension module suffix in CPython-compatible format + "EXT_SUFFIX" => format!(".rustpython313-{multiarch}.so"), + "MULTIARCH" => multiarch.clone(), + "RUST_MULTIARCH" => RUST_MULTIARCH, // enough for tests to stop expecting urandom() to fail after restricting file resources "HAVE_GETRANDOM" => 1, + // RustPython has no GIL (like free-threaded Python) + "Py_GIL_DISABLED" => 1, + // Compiler configuration for native extension builds + "CC" => "cc", + "CXX" => "c++", + "CFLAGS" => "", + "CPPFLAGS" => "", + "LDFLAGS" => "", + "LDSHARED" => "cc -shared", + "CCSHARED" => "", + "SHLIB_SUFFIX" => ".so", + "SO" => ".so", + "AR" => "ar", + "ARFLAGS" => "rcs", + "OPT" => "", + "BASECFLAGS" => "", + "BLDSHARED" => "cc -shared", } include!(concat!(env!("OUT_DIR"), "/env_vars.rs")); vars diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index 36252279397..bf495ecc382 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -1,23 +1,33 @@ //! Implementation of the _thread module +#[cfg(unix)] +pub(crate) use _thread::after_fork_child; +pub use _thread::get_ident; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] -pub(crate) use _thread::{RawRMutex, make_module}; +pub(crate) use _thread::{ + CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, get_all_current_frames, + init_main_thread_ident, module_def, +}; #[pymodule] pub(crate) mod _thread { use crate::{ AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyDictRef, PyStr, PyTupleRef, PyType, PyTypeRef}, - convert::ToPyException, + builtins::{PyDictRef, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef}, + frame::FrameRef, function::{ArgCallable, Either, FuncArgs, KwArgs, OptionalArg, PySetterValue}, types::{Constructor, GetAttr, Representable, SetAttr}, }; + use alloc::{ + fmt, + sync::{Arc, Weak}, + }; + use core::{cell::RefCell, time::Duration}; use crossbeam_utils::atomic::AtomicCell; use parking_lot::{ RawMutex, RawThreadId, lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, }; - use std::{cell::RefCell, fmt, thread, time::Duration}; - use thread_local::ThreadLocal; + use std::thread; // PYTHREAD_NAME: show current thread name pub const PYTHREAD_NAME: Option<&str> = { @@ -108,7 +118,8 @@ pub(crate) mod _thread { } #[pyattr(name = "LockType")] - #[pyclass(module = "thread", name = "lock")] + #[pyattr(name = "lock")] + #[pyclass(module = "_thread", name = "lock")] #[derive(PyPayload)] struct Lock { mu: RawMutex, @@ -151,7 +162,7 @@ pub(crate) mod _thread { let new_mut = RawMutex::INIT; unsafe { - let old_mutex: &AtomicCell<RawMutex> = std::mem::transmute(&self.mu); + let old_mutex: &AtomicCell<RawMutex> = core::mem::transmute(&self.mu); old_mutex.swap(new_mut); } @@ -170,10 +181,10 @@ pub(crate) mod _thread { } impl Constructor for Lock { - type Args = FuncArgs; + type Args = (); - fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { - Err(vm.new_type_error("cannot create '_thread.lock' instances")) + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { mu: RawMutex::INIT }) } } @@ -186,10 +197,11 @@ pub(crate) mod _thread { pub type RawRMutex = RawReentrantMutex<RawMutex, RawThreadId>; #[pyattr] - #[pyclass(module = "thread", name = "RLock")] + #[pyclass(module = "_thread", name = "RLock")] #[derive(PyPayload)] struct RLock { mu: RawRMutex, + count: core::sync::atomic::AtomicUsize, } impl fmt::Debug for RLock { @@ -198,12 +210,13 @@ pub(crate) mod _thread { } } - #[pyclass(with(Representable))] + #[pyclass(with(Representable), flags(BASETYPE))] impl RLock { #[pyslot] fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { Self { mu: RawRMutex::INIT, + count: core::sync::atomic::AtomicUsize::new(0), } .into_ref_with_type(vm, cls) .map(Into::into) @@ -213,7 +226,12 @@ pub(crate) mod _thread { #[pymethod(name = "acquire_lock")] #[pymethod(name = "__enter__")] fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult<bool> { - acquire_lock_impl!(&self.mu, args, vm) + let result = acquire_lock_impl!(&self.mu, args, vm)?; + if result { + self.count + .fetch_add(1, core::sync::atomic::Ordering::Relaxed); + } + Ok(result) } #[pymethod] #[pymethod(name = "release_lock")] @@ -221,6 +239,12 @@ pub(crate) mod _thread { if !self.mu.is_locked() { return Err(vm.new_runtime_error("release unlocked lock")); } + debug_assert!( + self.count.load(core::sync::atomic::Ordering::Relaxed) > 0, + "RLock count underflow" + ); + self.count + .fetch_sub(1, core::sync::atomic::Ordering::Relaxed); unsafe { self.mu.unlock() }; Ok(()) } @@ -232,19 +256,35 @@ pub(crate) mod _thread { self.mu.unlock(); }; } + self.count.store(0, core::sync::atomic::Ordering::Relaxed); let new_mut = RawRMutex::INIT; - - let old_mutex: AtomicCell<&RawRMutex> = AtomicCell::new(&self.mu); - old_mutex.swap(&new_mut); + unsafe { + let old_mutex: &AtomicCell<RawRMutex> = core::mem::transmute(&self.mu); + old_mutex.swap(new_mut); + } Ok(()) } + #[pymethod] + fn locked(&self) -> bool { + self.mu.is_locked() + } + #[pymethod] fn _is_owned(&self) -> bool { self.mu.is_owned_by_current_thread() } + #[pymethod] + fn _recursion_count(&self) -> usize { + if self.mu.is_owned_by_current_thread() { + self.count.load(core::sync::atomic::Ordering::Relaxed) + } else { + 0 + } + } + #[pymethod] fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { self.release(vm) @@ -254,17 +294,86 @@ pub(crate) mod _thread { impl Representable for RLock { #[inline] fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - repr_lock_impl!(zelf) + let count = zelf.count.load(core::sync::atomic::Ordering::Relaxed); + let status = if zelf.mu.is_locked() { + "locked" + } else { + "unlocked" + }; + Ok(format!( + "<{} {} object count={} at {:#x}>", + status, + zelf.class().name(), + count, + zelf.get_id() + )) } } + /// Get thread identity - uses pthread_self() on Unix for fork compatibility #[pyfunction] - fn get_ident() -> u64 { - thread_to_id(&thread::current()) + pub fn get_ident() -> u64 { + current_thread_id() } - fn thread_to_id(t: &thread::Thread) -> u64 { - use std::hash::{Hash, Hasher}; + /// Set the name of the current thread + #[pyfunction] + fn set_name(name: PyStrRef) { + #[cfg(target_os = "linux")] + { + use alloc::ffi::CString; + if let Ok(c_name) = CString::new(name.as_str()) { + // pthread_setname_np on Linux has a 16-byte limit including null terminator + // TODO: Potential UTF-8 boundary issue when truncating thread name on Linux. + // https://github.com/RustPython/RustPython/pull/6726/changes#r2689379171 + let truncated = if c_name.as_bytes().len() > 15 { + CString::new(&c_name.as_bytes()[..15]).unwrap_or(c_name) + } else { + c_name + }; + unsafe { + libc::pthread_setname_np(libc::pthread_self(), truncated.as_ptr()); + } + } + } + #[cfg(target_os = "macos")] + { + use alloc::ffi::CString; + if let Ok(c_name) = CString::new(name.as_str()) { + unsafe { + libc::pthread_setname_np(c_name.as_ptr()); + } + } + } + #[cfg(windows)] + { + // Windows doesn't have a simple pthread_setname_np equivalent + // SetThreadDescription requires Windows 10+ + let _ = name; + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = name; + } + } + + /// Get OS-level thread ID (pthread_self on Unix) + /// This is important for fork compatibility - the ID must remain stable after fork + #[cfg(unix)] + fn current_thread_id() -> u64 { + // pthread_self() like CPython for fork compatibility + unsafe { libc::pthread_self() as u64 } + } + + #[cfg(not(unix))] + fn current_thread_id() -> u64 { + thread_to_rust_id(&thread::current()) + } + + /// Convert Rust thread to ID (used for non-unix platforms) + #[cfg(not(unix))] + fn thread_to_rust_id(t: &thread::Thread) -> u64 { + use core::hash::{Hash, Hasher}; struct U64Hash { v: Option<u64>, } @@ -279,13 +388,25 @@ pub(crate) mod _thread { self.v.expect("should have written a u64") } } - // TODO: use id.as_u64() once it's stable, until then, ThreadId is just a wrapper - // around NonZeroU64, so this should work (?) let mut h = U64Hash { v: None }; t.id().hash(&mut h); h.finish() } + /// Get thread ID for a given thread handle (used by start_new_thread) + fn thread_to_id(handle: &thread::JoinHandle<()>) -> u64 { + #[cfg(unix)] + { + // On Unix, use pthread ID from the handle + use std::os::unix::thread::JoinHandleExt; + handle.as_pthread_t() as u64 + } + #[cfg(not(unix))] + { + thread_to_rust_id(handle.thread()) + } + } + #[pyfunction] const fn allocate_lock() -> Lock { Lock { mu: RawMutex::INIT } @@ -316,14 +437,14 @@ pub(crate) mod _thread { vm.new_thread() .make_spawn_func(move |vm| run_thread(func, args, vm)), ) - .map(|handle| { - vm.state.thread_count.fetch_add(1); - thread_to_id(handle.thread()) - }) - .map_err(|err| err.to_pyexception(vm)) + .map(|handle| thread_to_id(&handle)) + .map_err(|err| vm.new_runtime_error(format!("can't start new thread: {err}"))) } fn run_thread(func: ArgCallable, args: FuncArgs, vm: &VirtualMachine) { + // Increment thread count when thread actually starts executing + vm.state.thread_count.fetch_add(1); + match func.invoke(args, vm) { Ok(_obj) => {} Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} @@ -340,10 +461,25 @@ pub(crate) mod _thread { unsafe { lock.mu.unlock() }; } } + // Clean up thread-local storage while VM context is still active + // This ensures __del__ methods are called properly + cleanup_thread_local_data(); + // Clean up frame tracking + crate::vm::thread::cleanup_current_thread_frames(vm); vm.state.thread_count.fetch_sub(1); } - #[cfg(not(target_arch = "wasm32"))] + /// Clean up thread-local data for the current thread. + /// This triggers __del__ on objects stored in thread-local variables. + fn cleanup_thread_local_data() { + // Take all guards - this will trigger LocalGuard::drop for each, + // which removes the thread's dict from each Local instance + LOCAL_GUARDS.with(|guards| { + guards.borrow_mut().clear(); + }); + } + + #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] #[pyfunction] fn interrupt_main(signum: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<()> { crate::signal::set_interrupt_ex(signum.unwrap_or(libc::SIGINT), vm) @@ -375,23 +511,322 @@ pub(crate) mod _thread { vm.state.thread_count.load() } + #[pyfunction] + fn daemon_threads_allowed() -> bool { + // RustPython always allows daemon threads + true + } + + // Registry for non-daemon threads that need to be joined at shutdown + pub type ShutdownEntry = ( + Weak<parking_lot::Mutex<ThreadHandleInner>>, + Weak<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ); + + #[pyfunction] + fn _shutdown(vm: &VirtualMachine) { + // Wait for all non-daemon threads to finish + let current_ident = get_ident(); + + loop { + // Find a thread that's not finished and not the current thread + let handle_to_join = { + let mut handles = vm.state.shutdown_handles.lock(); + // Clean up finished entries + handles.retain(|(inner_weak, _): &ShutdownEntry| { + inner_weak.upgrade().is_some_and(|inner| { + let guard = inner.lock(); + guard.state != ThreadHandleState::Done && guard.ident != current_ident + }) + }); + + // Find first unfinished handle + handles + .iter() + .find_map(|(inner_weak, done_event_weak): &ShutdownEntry| { + let inner = inner_weak.upgrade()?; + let done_event = done_event_weak.upgrade()?; + let guard = inner.lock(); + if guard.state != ThreadHandleState::Done && guard.ident != current_ident { + Some((inner.clone(), done_event.clone())) + } else { + None + } + }) + }; + + match handle_to_join { + Some((_, done_event)) => { + // Wait for this thread to finish (infinite timeout) + // Only check done flag to avoid lock ordering issues + // (done_event lock vs inner lock) + let (lock, cvar) = &*done_event; + let mut done = lock.lock(); + while !*done { + cvar.wait(&mut done); + } + } + None => break, // No more threads to wait on + } + } + } + + /// Add a non-daemon thread handle to the shutdown registry + fn add_to_shutdown_handles( + vm: &VirtualMachine, + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + done_event: &Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ) { + let mut handles = vm.state.shutdown_handles.lock(); + handles.push((Arc::downgrade(inner), Arc::downgrade(done_event))); + } + + #[pyfunction] + fn _make_thread_handle(ident: u64, vm: &VirtualMachine) -> PyRef<ThreadHandle> { + let handle = ThreadHandle::new(vm); + { + let mut inner = handle.inner.lock(); + inner.ident = ident; + inner.state = ThreadHandleState::Running; + } + handle.into_ref(&vm.ctx) + } + + #[pyfunction] + fn _get_main_thread_ident(vm: &VirtualMachine) -> u64 { + vm.state.main_thread_ident.load() + } + + #[pyfunction] + fn _is_main_interpreter() -> bool { + // RustPython only has one interpreter + true + } + + /// Initialize the main thread ident. Should be called once at interpreter startup. + pub fn init_main_thread_ident(vm: &VirtualMachine) { + let ident = get_ident(); + vm.state.main_thread_ident.store(ident); + } + + /// ExceptHookArgs - simple class to hold exception hook arguments + /// This allows threading.py to import _excepthook and _ExceptHookArgs from _thread #[pyattr] - #[pyclass(module = "thread", name = "_local")] + #[pyclass(module = "_thread", name = "_ExceptHookArgs")] + #[derive(Debug, PyPayload)] + struct ExceptHookArgs { + exc_type: crate::PyObjectRef, + exc_value: crate::PyObjectRef, + exc_traceback: crate::PyObjectRef, + thread: crate::PyObjectRef, + } + + #[pyclass(with(Constructor))] + impl ExceptHookArgs { + #[pygetset] + fn exc_type(&self) -> crate::PyObjectRef { + self.exc_type.clone() + } + + #[pygetset] + fn exc_value(&self) -> crate::PyObjectRef { + self.exc_value.clone() + } + + #[pygetset] + fn exc_traceback(&self) -> crate::PyObjectRef { + self.exc_traceback.clone() + } + + #[pygetset] + fn thread(&self) -> crate::PyObjectRef { + self.thread.clone() + } + } + + impl Constructor for ExceptHookArgs { + // Takes a single iterable argument like namedtuple + type Args = (crate::PyObjectRef,); + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // Convert the argument to a list/tuple and extract elements + let seq: Vec<crate::PyObjectRef> = args.0.try_to_value(vm)?; + if seq.len() != 4 { + return Err(vm.new_type_error(format!( + "_ExceptHookArgs expected 4 arguments, got {}", + seq.len() + ))); + } + Ok(Self { + exc_type: seq[0].clone(), + exc_value: seq[1].clone(), + exc_traceback: seq[2].clone(), + thread: seq[3].clone(), + }) + } + } + + /// Handle uncaught exception in Thread.run() + #[pyfunction] + fn _excepthook(args: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Type check: args must be _ExceptHookArgs + let args = args.downcast::<ExceptHookArgs>().map_err(|_| { + vm.new_type_error( + "_thread._excepthook argument type must be _ExceptHookArgs".to_owned(), + ) + })?; + + let exc_type = args.exc_type.clone(); + let exc_value = args.exc_value.clone(); + let exc_traceback = args.exc_traceback.clone(); + let thread = args.thread.clone(); + + // Silently ignore SystemExit (identity check) + if exc_type.is(vm.ctx.exceptions.system_exit.as_ref()) { + return Ok(()); + } + + // Get stderr - fall back to thread._stderr if sys.stderr is None + let file = match vm.sys_module.get_attr("stderr", vm) { + Ok(stderr) if !vm.is_none(&stderr) => stderr, + _ => { + if vm.is_none(&thread) { + // do nothing if sys.stderr is None and thread is None + return Ok(()); + } + let thread_stderr = thread.get_attr("_stderr", vm)?; + if vm.is_none(&thread_stderr) { + // do nothing if sys.stderr is None and sys.stderr was None + // when the thread was created + return Ok(()); + } + thread_stderr + } + }; + + // Print "Exception in thread {thread.name}:" + let thread_name = if !vm.is_none(&thread) { + thread + .get_attr("name", vm) + .ok() + .and_then(|n| n.str(vm).ok()) + .map(|s| s.as_str().to_owned()) + } else { + None + }; + let name = thread_name.unwrap_or_else(|| format!("{}", get_ident())); + + let _ = vm.call_method( + &file, + "write", + (format!("Exception in thread {}:\n", name),), + ); + + // Display the traceback + if let Ok(traceback_mod) = vm.import("traceback", 0) + && let Ok(print_exc) = traceback_mod.get_attr("print_exception", vm) + { + use crate::function::KwArgs; + let kwargs: KwArgs = vec![("file".to_owned(), file.clone())] + .into_iter() + .collect(); + let _ = print_exc.call_with_args( + crate::function::FuncArgs::new(vec![exc_type, exc_value, exc_traceback], kwargs), + vm, + ); + } + + // Flush file + let _ = vm.call_method(&file, "flush", ()); + Ok(()) + } + + // Thread-local storage for cleanup guards + // When a thread terminates, the guard is dropped, which triggers cleanup + thread_local! { + static LOCAL_GUARDS: RefCell<Vec<LocalGuard>> = const { RefCell::new(Vec::new()) }; + } + + // Guard that removes thread-local data when dropped + struct LocalGuard { + local: Weak<LocalData>, + thread_id: std::thread::ThreadId, + } + + impl Drop for LocalGuard { + fn drop(&mut self) { + if let Some(local_data) = self.local.upgrade() { + // Remove from map while holding the lock, but drop the value + // outside the lock to prevent deadlock if __del__ accesses _local + let removed = local_data.data.lock().remove(&self.thread_id); + drop(removed); + } + } + } + + // Shared data structure for Local + struct LocalData { + data: parking_lot::Mutex<std::collections::HashMap<std::thread::ThreadId, PyDictRef>>, + } + + impl fmt::Debug for LocalData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalData").finish_non_exhaustive() + } + } + + #[pyattr] + #[pyclass(module = "_thread", name = "_local")] #[derive(Debug, PyPayload)] struct Local { - data: ThreadLocal<PyDictRef>, + inner: Arc<LocalData>, } #[pyclass(with(GetAttr, SetAttr), flags(BASETYPE))] impl Local { fn l_dict(&self, vm: &VirtualMachine) -> PyDictRef { - self.data.get_or(|| vm.ctx.new_dict()).clone() + let thread_id = std::thread::current().id(); + + // Fast path: check if dict exists under lock + if let Some(dict) = self.inner.data.lock().get(&thread_id).cloned() { + return dict; + } + + // Slow path: allocate dict outside lock to reduce lock hold time + let new_dict = vm.ctx.new_dict(); + + // Insert with double-check to handle races + let mut data = self.inner.data.lock(); + use std::collections::hash_map::Entry; + let (dict, need_guard) = match data.entry(thread_id) { + Entry::Occupied(e) => (e.get().clone(), false), + Entry::Vacant(e) => { + e.insert(new_dict.clone()); + (new_dict, true) + } + }; + drop(data); // Release lock before TLS access + + // Register cleanup guard only if we inserted a new entry + if need_guard { + let guard = LocalGuard { + local: Arc::downgrade(&self.inner), + thread_id, + }; + LOCAL_GUARDS.with(|guards| { + guards.borrow_mut().push(guard); + }); + } + + dict } #[pyslot] fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { Self { - data: ThreadLocal::new(), + inner: Arc::new(LocalData { + data: parking_lot::Mutex::new(std::collections::HashMap::new()), + }), } .into_ref_with_type(vm, cls) .map(Into::into) @@ -440,4 +875,408 @@ pub(crate) mod _thread { } } } + + // Registry of all ThreadHandles for fork cleanup + // Stores weak references so handles can be garbage collected normally + pub type HandleEntry = ( + Weak<parking_lot::Mutex<ThreadHandleInner>>, + Weak<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ); + + // Re-export type from vm::thread for PyGlobalState + pub use crate::vm::thread::CurrentFrameSlot; + + /// Get all threads' current (top) frames. Used by sys._current_frames(). + pub fn get_all_current_frames(vm: &VirtualMachine) -> Vec<(u64, FrameRef)> { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so the FramePtr is valid for the duration of the lock. + frames + .last() + .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) + }) + .collect() + } + + /// Called after fork() in child process to mark all other threads as done. + /// This prevents join() from hanging on threads that don't exist in the child. + #[cfg(unix)] + pub fn after_fork_child(vm: &VirtualMachine) { + let current_ident = get_ident(); + + // Update main thread ident - after fork, the current thread becomes the main thread + vm.state.main_thread_ident.store(current_ident); + + // Reinitialize frame slot for current thread + crate::vm::thread::reinit_frame_slot_after_fork(vm); + + // Clean up thread handles if we can acquire the lock. + // Use try_lock because the mutex might have been held during fork. + // If we can't acquire it, just skip - the child process will work + // correctly with new handles it creates. + if let Some(mut handles) = vm.state.thread_handles.try_lock() { + // Clean up dead weak refs and mark non-current threads as done + handles.retain(|(inner_weak, done_event_weak): &HandleEntry| { + let Some(inner) = inner_weak.upgrade() else { + return false; // Remove dead entries + }; + let Some(done_event) = done_event_weak.upgrade() else { + return false; + }; + + // Try to lock the inner state - skip if we can't + let Some(mut inner_guard) = inner.try_lock() else { + return false; + }; + + // Skip current thread and not-started threads + if inner_guard.ident == current_ident { + return true; + } + if inner_guard.state == ThreadHandleState::NotStarted { + return true; + } + + // Mark as done and notify waiters + inner_guard.state = ThreadHandleState::Done; + inner_guard.join_handle = None; // Can't join OS thread from child + drop(inner_guard); + + // Try to notify waiters - skip if we can't acquire the lock + let (lock, cvar) = &*done_event; + if let Some(mut done) = lock.try_lock() { + *done = true; + cvar.notify_all(); + } + + true + }); + } + + // Clean up shutdown_handles as well. + // This is critical to prevent _shutdown() from waiting on threads + // that don't exist in the child process after fork. + if let Some(mut handles) = vm.state.shutdown_handles.try_lock() { + // Mark all non-current threads as done in shutdown_handles + handles.retain(|(inner_weak, done_event_weak): &ShutdownEntry| { + let Some(inner) = inner_weak.upgrade() else { + return false; // Remove dead entries + }; + let Some(done_event) = done_event_weak.upgrade() else { + return false; + }; + + // Try to lock the inner state - skip if we can't + let Some(mut inner_guard) = inner.try_lock() else { + return false; + }; + + // Skip current thread + if inner_guard.ident == current_ident { + return true; + } + + // Keep handles for threads that have not been started yet. + // They are safe to start in the child process. + if inner_guard.state == ThreadHandleState::NotStarted { + return true; + } + + // Mark as done so _shutdown() won't wait on it + inner_guard.state = ThreadHandleState::Done; + drop(inner_guard); + + // Notify waiters + let (lock, cvar) = &*done_event; + if let Some(mut done) = lock.try_lock() { + *done = true; + cvar.notify_all(); + } + + false // Remove from shutdown_handles - these threads don't exist in child + }); + } + } + + // Thread handle state enum + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum ThreadHandleState { + NotStarted, + Starting, + Running, + Done, + } + + // Internal shared state for thread handle + pub struct ThreadHandleInner { + pub state: ThreadHandleState, + pub ident: u64, + pub join_handle: Option<thread::JoinHandle<()>>, + pub joining: bool, // True if a thread is currently joining + pub joined: bool, // Track if join has completed + } + + impl fmt::Debug for ThreadHandleInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ThreadHandleInner") + .field("state", &self.state) + .field("ident", &self.ident) + .field("join_handle", &self.join_handle.is_some()) + .field("joining", &self.joining) + .field("joined", &self.joined) + .finish() + } + } + + /// _ThreadHandle - handle for joinable threads + #[pyattr] + #[pyclass(module = "_thread", name = "_ThreadHandle")] + #[derive(Debug, PyPayload)] + struct ThreadHandle { + inner: Arc<parking_lot::Mutex<ThreadHandleInner>>, + // Event to signal thread completion (for timed join support) + done_event: Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + } + + #[pyclass] + impl ThreadHandle { + fn new(vm: &VirtualMachine) -> Self { + let inner = Arc::new(parking_lot::Mutex::new(ThreadHandleInner { + state: ThreadHandleState::NotStarted, + ident: 0, + join_handle: None, + joining: false, + joined: false, + })); + let done_event = + Arc::new((parking_lot::Mutex::new(false), parking_lot::Condvar::new())); + + // Register in global registry for fork cleanup + vm.state + .thread_handles + .lock() + .push((Arc::downgrade(&inner), Arc::downgrade(&done_event))); + + Self { inner, done_event } + } + + #[pygetset] + fn ident(&self) -> u64 { + self.inner.lock().ident + } + + #[pymethod] + fn is_done(&self) -> bool { + self.inner.lock().state == ThreadHandleState::Done + } + + #[pymethod] + fn _set_done(&self) { + self.inner.lock().state = ThreadHandleState::Done; + // Signal waiting threads that this thread is done + let (lock, cvar) = &*self.done_event; + *lock.lock() = true; + cvar.notify_all(); + } + + #[pymethod] + fn join( + &self, + timeout: OptionalArg<Option<Either<f64, i64>>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Convert timeout to Duration (None or negative = infinite wait) + let timeout_duration = match timeout.flatten() { + Some(Either::A(t)) if t >= 0.0 => Some(Duration::from_secs_f64(t)), + Some(Either::B(t)) if t >= 0 => Some(Duration::from_secs(t as u64)), + _ => None, + }; + + // Check for self-join first + { + let inner = self.inner.lock(); + let current_ident = get_ident(); + if inner.ident == current_ident && inner.state == ThreadHandleState::Running { + return Err(vm.new_runtime_error("cannot join current thread".to_owned())); + } + } + + // Wait for thread completion using Condvar (supports timeout) + // Loop to handle spurious wakeups + let (lock, cvar) = &*self.done_event; + let mut done = lock.lock(); + + while !*done { + if let Some(timeout) = timeout_duration { + let result = cvar.wait_for(&mut done, timeout); + if result.timed_out() && !*done { + // Timeout occurred and done is still false + return Ok(()); + } + } else { + // Infinite wait + cvar.wait(&mut done); + } + } + drop(done); + + // Thread is done, now perform cleanup + let join_handle = { + let mut inner = self.inner.lock(); + + // If already joined, return immediately (idempotent) + if inner.joined { + return Ok(()); + } + + // If another thread is already joining, wait for them to finish + if inner.joining { + drop(inner); + // Wait on done_event + let (lock, cvar) = &*self.done_event; + let mut done = lock.lock(); + while !*done { + cvar.wait(&mut done); + } + return Ok(()); + } + + // Mark that we're joining + inner.joining = true; + + // Take the join handle if available + inner.join_handle.take() + }; + + // Perform the actual join outside the lock + if let Some(handle) = join_handle { + // Ignore the result - panics in spawned threads are already handled + let _ = handle.join(); + } + + // Mark as joined and clear joining flag + { + let mut inner = self.inner.lock(); + inner.joined = true; + inner.joining = false; + } + + Ok(()) + } + + #[pyslot] + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + ThreadHandle::new(vm) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + } + + #[derive(FromArgs)] + struct StartJoinableThreadArgs { + #[pyarg(positional)] + function: ArgCallable, + #[pyarg(any, optional)] + handle: OptionalArg<PyRef<ThreadHandle>>, + #[pyarg(any, default = true)] + daemon: bool, + } + + #[pyfunction] + fn start_joinable_thread( + args: StartJoinableThreadArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<ThreadHandle>> { + let handle = match args.handle { + OptionalArg::Present(h) => h, + OptionalArg::Missing => ThreadHandle::new(vm).into_ref(&vm.ctx), + }; + + // Mark as starting + handle.inner.lock().state = ThreadHandleState::Starting; + + // Add non-daemon threads to shutdown registry so _shutdown() will wait for them + if !args.daemon { + add_to_shutdown_handles(vm, &handle.inner, &handle.done_event); + } + + let func = args.function; + let handle_clone = handle.clone(); + let inner_clone = handle.inner.clone(); + let done_event_clone = handle.done_event.clone(); + + let mut thread_builder = thread::Builder::new(); + let stacksize = vm.state.stacksize.load(); + if stacksize != 0 { + thread_builder = thread_builder.stack_size(stacksize); + } + + let join_handle = thread_builder + .spawn(vm.new_thread().make_spawn_func(move |vm| { + // Set ident and mark as running + { + let mut inner = inner_clone.lock(); + inner.ident = get_ident(); + inner.state = ThreadHandleState::Running; + } + + // Ensure cleanup happens even if the function panics + let inner_for_cleanup = inner_clone.clone(); + let done_event_for_cleanup = done_event_clone.clone(); + let vm_state = vm.state.clone(); + scopeguard::defer! { + // Mark as done + inner_for_cleanup.lock().state = ThreadHandleState::Done; + + // Handle sentinels + for lock in SENTINELS.take() { + if lock.mu.is_locked() { + unsafe { lock.mu.unlock() }; + } + } + + // Clean up thread-local data while VM context is still active + cleanup_thread_local_data(); + + // Clean up frame tracking + crate::vm::thread::cleanup_current_thread_frames(vm); + + vm_state.thread_count.fetch_sub(1); + + // Signal waiting threads that this thread is done + // This must be LAST to ensure all cleanup is complete before join() returns + { + let (lock, cvar) = &*done_event_for_cleanup; + *lock.lock() = true; + cvar.notify_all(); + } + } + + // Increment thread count when thread actually starts executing + vm_state.thread_count.fetch_add(1); + + // Run the function + match func.invoke((), vm) { + Ok(_) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} + Err(exc) => { + vm.run_unraisable( + exc, + Some("Exception ignored in thread started by".to_owned()), + func.into(), + ); + } + } + })) + .map_err(|err| vm.new_runtime_error(format!("can't start new thread: {err}")))?; + + // Store the join handle + handle.inner.lock().join_handle = Some(join_handle); + + Ok(handle_clone) + } } diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index b9b53cdc5c5..05790fe332b 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -3,51 +3,58 @@ // See also: // https://docs.python.org/3/library/time.html -use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub use decl::time; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - #[cfg(not(target_env = "msvc"))] - #[cfg(not(target_arch = "wasm32"))] - unsafe { - c_tzset() - }; - decl::make_module(vm) -} +pub(crate) use decl::module_def; #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] unsafe extern "C" { #[cfg(not(target_os = "freebsd"))] #[link_name = "daylight"] - static c_daylight: std::ffi::c_int; + static c_daylight: core::ffi::c_int; // pub static dstbias: std::ffi::c_int; #[link_name = "timezone"] - static c_timezone: std::ffi::c_long; + static c_timezone: core::ffi::c_long; #[link_name = "tzname"] - static c_tzname: [*const std::ffi::c_char; 2]; + static c_tzname: [*const core::ffi::c_char; 2]; #[link_name = "tzset"] fn c_tzset(); } -#[pymodule(name = "time", with(platform))] +#[pymodule(name = "time", with(#[cfg(any(unix, windows))] platform))] mod decl { use crate::{ - AsObject, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyStrRef, PyTypeRef, PyUtf8StrRef}, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStrRef, PyTypeRef}, function::{Either, FuncArgs, OptionalArg}, types::{PyStructSequence, struct_sequence_new}, }; + #[cfg(any(unix, windows))] + use crate::{common::wtf8::Wtf8Buf, convert::ToPyObject}; + #[cfg(unix)] + use alloc::ffi::CString; + #[cfg(not(any(unix, windows)))] use chrono::{ DateTime, Datelike, TimeZone, Timelike, naive::{NaiveDate, NaiveDateTime, NaiveTime}, }; - use std::time::Duration; + use core::time::Duration; #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] use windows_sys::Win32::System::Time::{GetTimeZoneInformation, TIME_ZONE_INFORMATION}; + #[cfg(windows)] + unsafe extern "C" { + fn wcsftime( + s: *mut libc::wchar_t, + max: libc::size_t, + format: *const libc::wchar_t, + tm: *const libc::tm, + ) -> libc::size_t; + } + #[allow(dead_code)] pub(super) const SEC_TO_MS: i64 = 1000; #[allow(dead_code)] @@ -90,6 +97,7 @@ mod decl { #[pyfunction] fn sleep(seconds: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let seconds_type_name = seconds.clone().class().name().to_owned(); let dur = seconds.try_into_value::<Duration>(vm).map_err(|e| { if e.class().is(vm.ctx.exceptions.value_error) && let Some(s) = e.args().first().and_then(|arg| arg.str(vm).ok()) @@ -97,6 +105,11 @@ mod decl { { return vm.new_value_error("sleep length must be non-negative"); } + if e.class().is(vm.ctx.exceptions.type_error) { + return vm.new_type_error(format!( + "'{seconds_type_name}' object cannot be interpreted as an integer or float" + )); + } e })?; @@ -104,7 +117,7 @@ mod decl { { // this is basically std::thread::sleep, but that catches interrupts and we don't want to; let ts = nix::sys::time::TimeSpec::from(dur); - let res = unsafe { libc::nanosleep(ts.as_ref(), std::ptr::null_mut()) }; + let res = unsafe { libc::nanosleep(ts.as_ref(), core::ptr::null_mut()) }; let interrupted = res == -1 && nix::Error::last_raw() == libc::EINTR; if interrupted { @@ -186,8 +199,8 @@ mod decl { #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] - fn get_tz_info() -> TIME_ZONE_INFORMATION { - let mut info: TIME_ZONE_INFORMATION = unsafe { std::mem::zeroed() }; + pub(super) fn get_tz_info() -> TIME_ZONE_INFORMATION { + let mut info: TIME_ZONE_INFORMATION = unsafe { core::mem::zeroed() }; unsafe { GetTimeZoneInformation(&mut info) }; info } @@ -200,7 +213,24 @@ mod decl { #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn timezone(_vm: &VirtualMachine) -> std::ffi::c_long { + fn altzone(_vm: &VirtualMachine) -> core::ffi::c_long { + // TODO: RUSTPYTHON; Add support for using the C altzone + unsafe { super::c_timezone - 3600 } + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn altzone(_vm: &VirtualMachine) -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.Bias + info.StandardBias) * 60 - 3600 + } + + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn timezone(_vm: &VirtualMachine) -> core::ffi::c_long { unsafe { super::c_timezone } } @@ -217,7 +247,7 @@ mod decl { #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn daylight(_vm: &VirtualMachine) -> std::ffi::c_int { + fn daylight(_vm: &VirtualMachine) -> core::ffi::c_int { unsafe { super::c_daylight } } @@ -236,8 +266,8 @@ mod decl { fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { use crate::builtins::tuple::IntoPyTuple; - unsafe fn to_str(s: *const std::ffi::c_char) -> String { - unsafe { std::ffi::CStr::from_ptr(s) } + unsafe fn to_str(s: *const core::ffi::c_char) -> String { + unsafe { core::ffi::CStr::from_ptr(s) } .to_string_lossy() .into_owned() } @@ -251,49 +281,211 @@ mod decl { use crate::builtins::tuple::IntoPyTuple; let info = get_tz_info(); let standard = widestring::decode_utf16_lossy(info.StandardName) - .filter(|&c| c != '\0') + .take_while(|&c| c != '\0') .collect::<String>(); let daylight = widestring::decode_utf16_lossy(info.DaylightName) - .filter(|&c| c != '\0') + .take_while(|&c| c != '\0') .collect::<String>(); let tz_name = (&*standard, &*daylight); tz_name.into_pytuple(vm) } + #[cfg(not(any(unix, windows)))] fn pyobj_to_date_time( value: Either<f64, i64>, vm: &VirtualMachine, ) -> PyResult<DateTime<chrono::offset::Utc>> { - let timestamp = match value { + let secs = match value { Either::A(float) => { - let secs = float.trunc() as i64; - let nano_secs = (float.fract() * 1e9) as u32; - DateTime::<chrono::offset::Utc>::from_timestamp(secs, nano_secs) + if !float.is_finite() { + return Err(vm.new_value_error("Invalid value for timestamp")); + } + float.floor() as i64 } - Either::B(int) => DateTime::<chrono::offset::Utc>::from_timestamp(int, 0), + Either::B(int) => int, }; - timestamp.ok_or_else(|| vm.new_overflow_error("timestamp out of range for platform time_t")) + DateTime::<chrono::offset::Utc>::from_timestamp(secs, 0) + .ok_or_else(|| vm.new_overflow_error("timestamp out of range for platform time_t")) } - impl OptionalArg<Either<f64, i64>> { + #[cfg(not(any(unix, windows)))] + impl OptionalArg<Option<Either<f64, i64>>> { /// Construct a localtime from the optional seconds, or get the current local time. fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { Ok(match self { - Self::Present(secs) => pyobj_to_date_time(secs, vm)? + Self::Present(Some(secs)) => pyobj_to_date_time(secs, vm)? .with_timezone(&chrono::Local) .naive_local(), - Self::Missing => chrono::offset::Local::now().naive_local(), + Self::Present(None) | Self::Missing => chrono::offset::Local::now().naive_local(), }) } + } - fn naive_or_utc(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { - Ok(match self { - Self::Present(secs) => pyobj_to_date_time(secs, vm)?.naive_utc(), - Self::Missing => chrono::offset::Utc::now().naive_utc(), - }) + #[cfg(any(unix, windows))] + struct CheckedTm { + tm: libc::tm, + #[cfg(unix)] + zone: Option<CString>, + } + + #[cfg(any(unix, windows))] + fn checked_tm_from_struct_time( + t: &StructTimeData, + vm: &VirtualMachine, + func_name: &'static str, + ) -> PyResult<CheckedTm> { + let invalid_tuple = + || vm.new_type_error(format!("{func_name}(): illegal time tuple argument")); + + let year: i64 = t.tm_year.clone().try_into_value(vm).map_err(|e| { + if e.class().is(vm.ctx.exceptions.overflow_error) { + vm.new_overflow_error("year out of range") + } else { + invalid_tuple() + } + })?; + if year < i64::from(i32::MIN) + 1900 || year > i64::from(i32::MAX) { + return Err(vm.new_overflow_error("year out of range")); + } + let year = year as i32; + let tm_mon = t + .tm_mon + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + let tm_mday = t + .tm_mday + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_hour = t + .tm_hour + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_min = t + .tm_min + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_sec = t + .tm_sec + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_wday = (t + .tm_wday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + + 1) + % 7; + let tm_yday = t + .tm_yday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + let tm_isdst = t + .tm_isdst + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + + let mut tm: libc::tm = unsafe { core::mem::zeroed() }; + tm.tm_year = year - 1900; + tm.tm_mon = tm_mon; + tm.tm_mday = tm_mday; + tm.tm_hour = tm_hour; + tm.tm_min = tm_min; + tm.tm_sec = tm_sec; + tm.tm_wday = tm_wday; + tm.tm_yday = tm_yday; + tm.tm_isdst = tm_isdst; + + if tm.tm_mon == -1 { + tm.tm_mon = 0; + } else if tm.tm_mon < 0 || tm.tm_mon > 11 { + return Err(vm.new_value_error("month out of range")); + } + if tm.tm_mday == 0 { + tm.tm_mday = 1; + } else if tm.tm_mday < 0 || tm.tm_mday > 31 { + return Err(vm.new_value_error("day of month out of range")); + } + if tm.tm_hour < 0 || tm.tm_hour > 23 { + return Err(vm.new_value_error("hour out of range")); + } + if tm.tm_min < 0 || tm.tm_min > 59 { + return Err(vm.new_value_error("minute out of range")); + } + if tm.tm_sec < 0 || tm.tm_sec > 61 { + return Err(vm.new_value_error("seconds out of range")); + } + if tm.tm_wday < 0 { + return Err(vm.new_value_error("day of week out of range")); + } + if tm.tm_yday == -1 { + tm.tm_yday = 0; + } else if tm.tm_yday < 0 || tm.tm_yday > 365 { + return Err(vm.new_value_error("day of year out of range")); + } + + #[cfg(unix)] + { + let zone = if t.tm_zone.is(&vm.ctx.none) { + None + } else { + let zone: PyStrRef = t + .tm_zone + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + Some( + CString::new(zone.as_str()) + .map_err(|_| vm.new_value_error("embedded null character"))?, + ) + }; + if let Some(zone) = &zone { + tm.tm_zone = zone.as_ptr().cast_mut(); + } + if !t.tm_gmtoff.is(&vm.ctx.none) { + let gmtoff: i64 = t + .tm_gmtoff + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_gmtoff = gmtoff as _; + } + + Ok(CheckedTm { tm, zone }) + } + #[cfg(windows)] + { + Ok(CheckedTm { tm }) } } + #[cfg(any(unix, windows))] + fn asctime_from_tm(tm: &libc::tm) -> String { + const WDAY_NAME: [&str; 7] = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + const MON_NAME: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + format!( + "{} {}{:>3} {:02}:{:02}:{:02} {}", + WDAY_NAME[tm.tm_wday as usize], + MON_NAME[tm.tm_mon as usize], + tm.tm_mday, + tm.tm_hour, + tm.tm_min, + tm.tm_sec, + tm.tm_year + 1900 + ) + } + + #[cfg(not(any(unix, windows)))] impl OptionalArg<StructTimeData> { fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { Ok(match self { @@ -306,78 +498,258 @@ mod decl { /// https://docs.python.org/3/library/time.html?highlight=gmtime#time.gmtime #[pyfunction] fn gmtime( - secs: OptionalArg<Either<f64, i64>>, + secs: OptionalArg<Option<Either<f64, i64>>>, vm: &VirtualMachine, ) -> PyResult<StructTimeData> { - let instant = secs.naive_or_utc(vm)?; - Ok(StructTimeData::new_utc(vm, instant)) + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + gmtime_from_timestamp(ts, vm) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = match secs { + OptionalArg::Present(Some(secs)) => pyobj_to_date_time(secs, vm)?.naive_utc(), + OptionalArg::Present(None) | OptionalArg::Missing => { + chrono::offset::Utc::now().naive_utc() + } + }; + Ok(StructTimeData::new_utc(vm, instant)) + } } #[pyfunction] fn localtime( - secs: OptionalArg<Either<f64, i64>>, + secs: OptionalArg<Option<Either<f64, i64>>>, vm: &VirtualMachine, ) -> PyResult<StructTimeData> { + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + localtime_from_timestamp(ts, vm) + } + + #[cfg(not(any(unix, windows)))] let instant = secs.naive_or_local(vm)?; - // TODO: isdst flag must be valid value here - // https://docs.python.org/3/library/time.html#time.localtime - Ok(StructTimeData::new_local(vm, instant, -1)) + #[cfg(not(any(unix, windows)))] + { + Ok(StructTimeData::new_local(vm, instant, 0)) + } } #[pyfunction] fn mktime(t: StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { - let datetime = t.to_date_time(vm)?; - // mktime interprets struct_time as local time - let local_dt = chrono::Local - .from_local_datetime(&datetime) - .single() - .ok_or_else(|| vm.new_overflow_error("mktime argument out of range"))?; - let seconds_since_epoch = local_dt.timestamp() as f64; - Ok(seconds_since_epoch) + #[cfg(unix)] + { + unix_mktime(&t, vm) + } + + #[cfg(windows)] + { + win_mktime(&t, vm) + } + + #[cfg(not(any(unix, windows)))] + { + let datetime = t.to_date_time(vm)?; + // mktime interprets struct_time as local time + let local_dt = chrono::Local + .from_local_datetime(&datetime) + .single() + .ok_or_else(|| vm.new_overflow_error("mktime argument out of range"))?; + let seconds_since_epoch = local_dt.timestamp() as f64; + Ok(seconds_since_epoch) + } } + #[cfg(not(any(unix, windows)))] const CFMT: &str = "%a %b %e %H:%M:%S %Y"; #[pyfunction] fn asctime(t: OptionalArg<StructTimeData>, vm: &VirtualMachine) -> PyResult { - let instant = t.naive_or_local(vm)?; - let formatted_time = instant.format(CFMT).to_string(); - Ok(vm.ctx.new_str(formatted_time).into()) + #[cfg(any(unix, windows))] + { + let tm = match t { + OptionalArg::Present(value) => { + checked_tm_from_struct_time(&value, vm, "asctime")?.tm + } + OptionalArg::Missing => { + let now = current_time_t(); + let local = localtime_from_timestamp(now, vm)?; + checked_tm_from_struct_time(&local, vm, "asctime")?.tm + } + }; + Ok(vm.ctx.new_str(asctime_from_tm(&tm)).into()) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = t.naive_or_local(vm)?; + let formatted_time = instant.format(CFMT).to_string(); + Ok(vm.ctx.new_str(formatted_time).into()) + } } #[pyfunction] - fn ctime(secs: OptionalArg<Either<f64, i64>>, vm: &VirtualMachine) -> PyResult<String> { - let instant = secs.naive_or_local(vm)?; - Ok(instant.format(CFMT).to_string()) + fn ctime(secs: OptionalArg<Option<Either<f64, i64>>>, vm: &VirtualMachine) -> PyResult<String> { + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + let local = localtime_from_timestamp(ts, vm)?; + let tm = checked_tm_from_struct_time(&local, vm, "asctime")?.tm; + Ok(asctime_from_tm(&tm)) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = secs.naive_or_local(vm)?; + Ok(instant.format(CFMT).to_string()) + } } - #[pyfunction] - fn strftime( - format: PyUtf8StrRef, - t: OptionalArg<StructTimeData>, - vm: &VirtualMachine, - ) -> PyResult { - use std::fmt::Write; + #[cfg(any(unix, windows))] + fn strftime_crt(format: &PyStrRef, checked_tm: CheckedTm, vm: &VirtualMachine) -> PyResult { + #[cfg(unix)] + let _keep_zone_alive = &checked_tm.zone; + let mut tm = checked_tm.tm; + tm.tm_isdst = tm.tm_isdst.clamp(-1, 1); + + // MSVC strftime requires year in [1; 9999] + #[cfg(windows)] + { + let year = tm.tm_year + 1900; + if !(1..=9999).contains(&year) { + return Err(vm.new_value_error("strftime() requires year in [1; 9999]".to_owned())); + } + } + + #[cfg(unix)] + fn strftime_ascii(fmt: &str, tm: &libc::tm, vm: &VirtualMachine) -> PyResult<String> { + let fmt_c = + CString::new(fmt).map_err(|_| vm.new_value_error("embedded null character"))?; + let mut size = 1024usize; + let max_scale = 256usize.saturating_mul(fmt.len().max(1)); + loop { + let mut out = vec![0u8; size]; + let written = unsafe { + libc::strftime( + out.as_mut_ptr().cast(), + out.len(), + fmt_c.as_ptr(), + tm as *const libc::tm, + ) + }; + if written > 0 || size >= max_scale { + return Ok(String::from_utf8_lossy(&out[..written]).into_owned()); + } + size = size.saturating_mul(2); + } + } + + #[cfg(windows)] + fn strftime_ascii(fmt: &str, tm: &libc::tm, vm: &VirtualMachine) -> PyResult<String> { + if fmt.contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + // Use wcsftime for proper Unicode output (e.g. %Z timezone names) + let fmt_wide: Vec<u16> = fmt.encode_utf16().chain(core::iter::once(0)).collect(); + let mut size = 1024usize; + let max_scale = 256usize.saturating_mul(fmt.len().max(1)); + loop { + let mut out = vec![0u16; size]; + let written = unsafe { + rustpython_common::suppress_iph!(wcsftime( + out.as_mut_ptr(), + out.len(), + fmt_wide.as_ptr(), + tm as *const libc::tm, + )) + }; + if written > 0 || size >= max_scale { + return Ok(String::from_utf16_lossy(&out[..written])); + } + size = size.saturating_mul(2); + } + } - let instant = t.naive_or_local(vm)?; + let mut out = Wtf8Buf::new(); + let mut ascii = String::new(); + + for codepoint in format.as_wtf8().code_points() { + if codepoint.to_u32() == 0 { + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); + ascii.clear(); + } + out.push(codepoint); + continue; + } + if let Some(ch) = codepoint.to_char() + && ch.is_ascii() + { + ascii.push(ch); + continue; + } - // On Windows/AIX/Solaris, %y format with year < 1900 is not supported - #[cfg(any(windows, target_os = "aix", target_os = "solaris"))] - if instant.year() < 1900 && format.as_str().contains("%y") { - let msg = "format %y requires year >= 1900 on Windows"; - return Err(vm.new_value_error(msg.to_owned())); + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); + ascii.clear(); + } + out.push(codepoint); + } + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); } + Ok(out.to_pyobject(vm)) + } - let mut formatted_time = String::new(); + #[pyfunction] + fn strftime(format: PyStrRef, t: OptionalArg<StructTimeData>, vm: &VirtualMachine) -> PyResult { + #[cfg(any(unix, windows))] + { + let checked_tm = match t { + OptionalArg::Present(value) => checked_tm_from_struct_time(&value, vm, "strftime")?, + OptionalArg::Missing => { + let now = current_time_t(); + let local = localtime_from_timestamp(now, vm)?; + checked_tm_from_struct_time(&local, vm, "strftime")? + } + }; + strftime_crt(&format, checked_tm, vm) + } - /* - * chrono doesn't support all formats and it - * raises an error if unsupported format is supplied. - * If error happens, we set result as input arg. - */ - write!(&mut formatted_time, "{}", instant.format(format.as_str())) - .unwrap_or_else(|_| formatted_time = format.to_string()); - Ok(vm.ctx.new_str(formatted_time).into()) + #[cfg(not(any(unix, windows)))] + { + use core::fmt::Write; + + let fmt_lossy = format.to_string_lossy(); + + // If the struct_time can't be represented as NaiveDateTime + // (e.g. month=0), return the format string as-is, matching + // the fallback behavior for unsupported chrono formats. + let instant = match t.naive_or_local(vm) { + Ok(dt) => dt, + Err(_) => return Ok(vm.ctx.new_str(fmt_lossy.into_owned()).into()), + }; + + let mut formatted_time = String::new(); + write!(&mut formatted_time, "{}", instant.format(&fmt_lossy)) + .unwrap_or_else(|_| formatted_time = format.to_string()); + Ok(vm.ctx.new_str(formatted_time).into()) + } } #[pyfunction] @@ -428,7 +800,7 @@ mod decl { #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))] fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { let t: libc::tms = unsafe { - let mut t = std::mem::MaybeUninit::uninit(); + let mut t = core::mem::MaybeUninit::uninit(); if libc::times(t.as_mut_ptr()) == -1 { return Err(vm.new_os_error("Failed to get clock time".to_owned())); } @@ -441,19 +813,6 @@ mod decl { )) } - // same as the get_process_time impl for most unixes - #[cfg(all(target_arch = "wasm32", target_os = "wasi"))] - pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - let time: libc::timespec = unsafe { - let mut time = std::mem::MaybeUninit::uninit(); - if libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, time.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - time.assume_init() - }; - Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) - } - #[cfg(not(any( windows, target_os = "macos", @@ -466,7 +825,7 @@ mod decl { target_os = "solaris", target_os = "openbsd", target_os = "redox", - all(target_arch = "wasm32", not(target_os = "unknown")) + all(target_arch = "wasm32", target_os = "emscripten") )))] fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { Err(vm.new_not_implemented_error("process time unsupported in this system")) @@ -495,18 +854,19 @@ mod decl { pub tm_yday: PyObjectRef, pub tm_isdst: PyObjectRef, #[pystruct_sequence(skip)] - pub tm_gmtoff: PyObjectRef, - #[pystruct_sequence(skip)] pub tm_zone: PyObjectRef, + #[pystruct_sequence(skip)] + pub tm_gmtoff: PyObjectRef, } - impl std::fmt::Debug for StructTimeData { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for StructTimeData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "struct_time()") } } impl StructTimeData { + #[cfg(not(any(unix, windows)))] fn new_inner( vm: &VirtualMachine, tm: NaiveDateTime, @@ -524,25 +884,27 @@ mod decl { tm_wday: vm.ctx.new_int(tm.weekday().num_days_from_monday()).into(), tm_yday: vm.ctx.new_int(tm.ordinal()).into(), tm_isdst: vm.ctx.new_int(isdst).into(), - tm_gmtoff: vm.ctx.new_int(gmtoff).into(), tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(gmtoff).into(), } } /// Create struct_time for UTC (gmtime) + #[cfg(not(any(unix, windows)))] fn new_utc(vm: &VirtualMachine, tm: NaiveDateTime) -> Self { Self::new_inner(vm, tm, 0, 0, "UTC") } /// Create struct_time for local timezone (localtime) + #[cfg(not(any(unix, windows)))] fn new_local(vm: &VirtualMachine, tm: NaiveDateTime, isdst: i32) -> Self { let local_time = chrono::Local.from_local_datetime(&tm).unwrap(); - let offset_seconds = - local_time.offset().local_minus_utc() + if isdst == 1 { 3600 } else { 0 }; + let offset_seconds = local_time.offset().local_minus_utc(); let tz_abbr = local_time.format("%Z").to_string(); Self::new_inner(vm, tm, isdst, offset_seconds, &tz_abbr) } + #[cfg(not(any(unix, windows)))] fn to_date_time(&self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { let invalid_overflow = || vm.new_overflow_error("mktime argument out of range"); let invalid_value = || vm.new_value_error("invalid struct_time parameter"); @@ -570,28 +932,127 @@ mod decl { impl PyStructTime { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let seq: PyObjectRef = args.bind(vm)?; + let (seq, _dict): (PyObjectRef, OptionalArg<PyObjectRef>) = args.bind(vm)?; struct_sequence_new(cls, seq, vm) } } + /// Extract fields from StructTimeData into a libc::tm for mktime. + #[cfg(any(unix, windows))] + pub(super) fn tm_from_struct_time( + t: &StructTimeData, + vm: &VirtualMachine, + ) -> PyResult<libc::tm> { + let invalid_tuple = || vm.new_type_error("mktime(): illegal time tuple argument"); + let year: i32 = t + .tm_year + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + if year < i32::MIN + 1900 { + return Err(vm.new_overflow_error("year out of range")); + } + + let mut tm: libc::tm = unsafe { core::mem::zeroed() }; + tm.tm_sec = t + .tm_sec + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_min = t + .tm_min + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_hour = t + .tm_hour + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_mday = t + .tm_mday + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_mon = t + .tm_mon + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + tm.tm_year = year - 1900; + tm.tm_wday = -1; + tm.tm_yday = t + .tm_yday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + tm.tm_isdst = t + .tm_isdst + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + Ok(tm) + } + + #[cfg(any(unix, windows))] + fn pyobj_to_time_t(value: Either<f64, i64>, vm: &VirtualMachine) -> PyResult<libc::time_t> { + match value { + Either::A(float) => { + if !float.is_finite() { + return Err(vm.new_value_error("Invalid value for timestamp")); + } + let secs = float.floor(); + if secs < libc::time_t::MIN as f64 || secs > libc::time_t::MAX as f64 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(secs as libc::time_t) + } + Either::B(int) => { + // try_into is needed on 32-bit platforms where time_t != i64 + #[allow(clippy::useless_conversion)] + let ts: libc::time_t = int.try_into().map_err(|_| { + vm.new_overflow_error("timestamp out of range for platform time_t") + })?; + Ok(ts) + } + } + } + + #[cfg(any(unix, windows))] #[allow(unused_imports)] use super::platform::*; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + unsafe { + super::c_tzset() + }; + + __module_exec(vm, module); + Ok(()) + } } #[cfg(unix)] #[pymodule(sub)] mod platform { #[allow(unused_imports)] - use super::decl::{SEC_TO_NS, US_TO_NS}; + use super::decl::{SEC_TO_NS, StructTimeData, US_TO_NS}; #[cfg_attr(target_os = "macos", allow(unused_imports))] use crate::{ PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::{PyNamespace, PyStrRef}, convert::IntoPyException, }; + use core::time::Duration; + use libc::time_t; use nix::{sys::time::TimeSpec, time::ClockId}; - use std::time::Duration; #[cfg(target_os = "solaris")] #[pyattr] @@ -601,6 +1062,7 @@ mod platform { target_os = "netbsd", target_os = "solaris", target_os = "openbsd", + target_os = "wasi", )))] #[pyattr] use libc::CLOCK_PROCESS_CPUTIME_ID; @@ -628,6 +1090,68 @@ mod platform { } } + fn struct_time_from_tm(vm: &VirtualMachine, tm: libc::tm) -> StructTimeData { + let zone = unsafe { + if tm.tm_zone.is_null() { + String::new() + } else { + core::ffi::CStr::from_ptr(tm.tm_zone) + .to_string_lossy() + .into_owned() + } + }; + StructTimeData { + tm_year: vm.ctx.new_int(tm.tm_year + 1900).into(), + tm_mon: vm.ctx.new_int(tm.tm_mon + 1).into(), + tm_mday: vm.ctx.new_int(tm.tm_mday).into(), + tm_hour: vm.ctx.new_int(tm.tm_hour).into(), + tm_min: vm.ctx.new_int(tm.tm_min).into(), + tm_sec: vm.ctx.new_int(tm.tm_sec).into(), + tm_wday: vm.ctx.new_int((tm.tm_wday + 6) % 7).into(), + tm_yday: vm.ctx.new_int(tm.tm_yday + 1).into(), + tm_isdst: vm.ctx.new_int(tm.tm_isdst).into(), + tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(tm.tm_gmtoff).into(), + } + } + + pub(super) fn current_time_t() -> time_t { + unsafe { libc::time(core::ptr::null_mut()) } + } + + pub(super) fn gmtime_from_timestamp( + when: time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let ret = unsafe { libc::gmtime_r(&when, out.as_mut_ptr()) }; + if ret.is_null() { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm(vm, unsafe { out.assume_init() })) + } + + pub(super) fn localtime_from_timestamp( + when: time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let ret = unsafe { libc::localtime_r(&when, out.as_mut_ptr()) }; + if ret.is_null() { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm(vm, unsafe { out.assume_init() })) + } + + pub(super) fn unix_mktime(t: &StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { + let mut tm = super::decl::tm_from_struct_time(t, vm)?; + let timestamp = unsafe { libc::mktime(&mut tm) }; + if timestamp == -1 && tm.tm_wday == -1 { + return Err(vm.new_overflow_error("mktime argument out of range")); + } + Ok(timestamp as f64) + } + fn get_clock_time(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<Duration> { let ts = nix::time::clock_gettime(clk_id).map_err(|e| e.into_pyexception(vm))?; Ok(ts.into()) @@ -801,14 +1325,14 @@ mod platform { } #[cfg(windows)] -#[pymodule] +#[pymodule(sub)] mod platform { - use super::decl::{MS_TO_NS, SEC_TO_NS, time_muldiv}; + use super::decl::{MS_TO_NS, SEC_TO_NS, StructTimeData, get_tz_info, time_muldiv}; use crate::{ PyRef, PyResult, VirtualMachine, builtins::{PyNamespace, PyStrRef}, }; - use std::time::Duration; + use core::time::Duration; use windows_sys::Win32::{ Foundation::FILETIME, System::Performance::{QueryPerformanceCounter, QueryPerformanceFrequency}, @@ -816,14 +1340,98 @@ mod platform { System::Threading::{GetCurrentProcess, GetCurrentThread, GetProcessTimes, GetThreadTimes}, }; + unsafe extern "C" { + fn _gmtime64_s(tm: *mut libc::tm, time: *const libc::time_t) -> libc::c_int; + fn _localtime64_s(tm: *mut libc::tm, time: *const libc::time_t) -> libc::c_int; + #[link_name = "_mktime64"] + fn c_mktime(tm: *mut libc::tm) -> libc::time_t; + } + + fn struct_time_from_tm( + vm: &VirtualMachine, + tm: libc::tm, + zone: &str, + gmtoff: i32, + ) -> StructTimeData { + StructTimeData { + tm_year: vm.ctx.new_int(tm.tm_year + 1900).into(), + tm_mon: vm.ctx.new_int(tm.tm_mon + 1).into(), + tm_mday: vm.ctx.new_int(tm.tm_mday).into(), + tm_hour: vm.ctx.new_int(tm.tm_hour).into(), + tm_min: vm.ctx.new_int(tm.tm_min).into(), + tm_sec: vm.ctx.new_int(tm.tm_sec).into(), + tm_wday: vm.ctx.new_int((tm.tm_wday + 6) % 7).into(), + tm_yday: vm.ctx.new_int(tm.tm_yday + 1).into(), + tm_isdst: vm.ctx.new_int(tm.tm_isdst).into(), + tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(gmtoff).into(), + } + } + + pub(super) fn current_time_t() -> libc::time_t { + unsafe { libc::time(core::ptr::null_mut()) } + } + + pub(super) fn gmtime_from_timestamp( + when: libc::time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let err = unsafe { _gmtime64_s(out.as_mut_ptr(), &when) }; + if err != 0 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm( + vm, + unsafe { out.assume_init() }, + "UTC", + 0, + )) + } + + pub(super) fn localtime_from_timestamp( + when: libc::time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let err = unsafe { _localtime64_s(out.as_mut_ptr(), &when) }; + if err != 0 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + let tm = unsafe { out.assume_init() }; + + // Get timezone info from Windows API + let info = get_tz_info(); + let (bias, name) = if tm.tm_isdst > 0 { + (info.DaylightBias, &info.DaylightName) + } else { + (info.StandardBias, &info.StandardName) + }; + let zone = widestring::decode_utf16_lossy(name.iter().copied()) + .take_while(|&c| c != '\0') + .collect::<String>(); + let gmtoff = -((info.Bias + bias) as i32) * 60; + + Ok(struct_time_from_tm(vm, tm, &zone, gmtoff)) + } + + pub(super) fn win_mktime(t: &StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { + let mut tm = super::decl::tm_from_struct_time(t, vm)?; + let timestamp = unsafe { rustpython_common::suppress_iph!(c_mktime(&mut tm)) }; + if timestamp == -1 && tm.tm_wday == -1 { + return Err(vm.new_overflow_error("mktime argument out of range")); + } + Ok(timestamp as f64) + } + fn u64_from_filetime(time: FILETIME) -> u64 { let large: [u32; 2] = [time.dwLowDateTime, time.dwHighDateTime]; - unsafe { std::mem::transmute(large) } + unsafe { core::mem::transmute(large) } } fn win_perf_counter_frequency(vm: &VirtualMachine) -> PyResult<i64> { let frequency = unsafe { - let mut freq = std::mem::MaybeUninit::uninit(); + let mut freq = core::mem::MaybeUninit::uninit(); if QueryPerformanceFrequency(freq.as_mut_ptr()) == 0 { return Err(vm.new_last_os_error()); } @@ -850,7 +1458,7 @@ mod platform { pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { let ticks = unsafe { - let mut performance_count = std::mem::MaybeUninit::uninit(); + let mut performance_count = core::mem::MaybeUninit::uninit(); QueryPerformanceCounter(performance_count.as_mut_ptr()); performance_count.assume_init() }; @@ -863,9 +1471,9 @@ mod platform { } fn get_system_time_adjustment(vm: &VirtualMachine) -> PyResult<u32> { - let mut _time_adjustment = std::mem::MaybeUninit::uninit(); - let mut time_increment = std::mem::MaybeUninit::uninit(); - let mut _is_time_adjustment_disabled = std::mem::MaybeUninit::uninit(); + let mut _time_adjustment = core::mem::MaybeUninit::uninit(); + let mut time_increment = core::mem::MaybeUninit::uninit(); + let mut _is_time_adjustment_disabled = core::mem::MaybeUninit::uninit(); let time_increment = unsafe { if GetSystemTimeAdjustment( _time_adjustment.as_mut_ptr(), @@ -927,10 +1535,10 @@ mod platform { pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { let (kernel_time, user_time) = unsafe { - let mut _creation_time = std::mem::MaybeUninit::uninit(); - let mut _exit_time = std::mem::MaybeUninit::uninit(); - let mut kernel_time = std::mem::MaybeUninit::uninit(); - let mut user_time = std::mem::MaybeUninit::uninit(); + let mut _creation_time = core::mem::MaybeUninit::uninit(); + let mut _exit_time = core::mem::MaybeUninit::uninit(); + let mut kernel_time = core::mem::MaybeUninit::uninit(); + let mut user_time = core::mem::MaybeUninit::uninit(); let thread = GetCurrentThread(); if GetThreadTimes( @@ -952,10 +1560,10 @@ mod platform { pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { let (kernel_time, user_time) = unsafe { - let mut _creation_time = std::mem::MaybeUninit::uninit(); - let mut _exit_time = std::mem::MaybeUninit::uninit(); - let mut kernel_time = std::mem::MaybeUninit::uninit(); - let mut user_time = std::mem::MaybeUninit::uninit(); + let mut _creation_time = core::mem::MaybeUninit::uninit(); + let mut _exit_time = core::mem::MaybeUninit::uninit(); + let mut kernel_time = core::mem::MaybeUninit::uninit(); + let mut user_time = core::mem::MaybeUninit::uninit(); let process = GetCurrentProcess(); if GetProcessTimes( @@ -975,8 +1583,3 @@ mod platform { Ok(Duration::from_nanos((k_time + u_time) * 100)) } } - -// mostly for wasm32 -#[cfg(not(any(unix, windows)))] -#[pymodule(sub)] -mod platform {} diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs index e8cc407da15..1b3b5fecdd4 100644 --- a/crates/vm/src/stdlib/typevar.rs +++ b/crates/vm/src/stdlib/typevar.rs @@ -1,1038 +1,1098 @@ // spell-checker:ignore typevarobject funcobj -use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr}, - common::lock::PyMutex, - function::{FuncArgs, IntoFuncArgs, PyComparisonValue}, - protocol::PyNumberMethods, - types::{AsNumber, Comparable, Constructor, Iterable, PyComparisonOp, Representable}, -}; - -pub(crate) fn _call_typing_func_object<'a>( - vm: &VirtualMachine, - func_name: impl AsPyStr<'a>, - args: impl IntoFuncArgs, -) -> PyResult { - let module = vm.import("typing", 0)?; - let func = module.get_attr(func_name.as_pystr(&vm.ctx), vm)?; - func.call(args, vm) -} -fn type_check(arg: PyObjectRef, msg: &str, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - // Calling typing.py here leads to bootstrapping problems - if vm.is_none(&arg) { - return Ok(arg.class().to_owned().into()); - } - let message_str: PyObjectRef = vm.ctx.new_str(msg).into(); - _call_typing_func_object(vm, "_type_check", (arg, message_str)) -} - -/// Get the module of the caller frame, similar to CPython's caller() function. -/// Returns the module name or None if not found. -/// -/// Note: CPython's implementation (in typevarobject.c) gets the module from the -/// frame's function object using PyFunction_GetModule(f->f_funcobj). However, -/// RustPython's Frame doesn't store a reference to the function object, so we -/// get the module name from the frame's globals dictionary instead. -fn caller(vm: &VirtualMachine) -> Option<PyObjectRef> { - let frame = vm.current_frame()?; - - // In RustPython, we get the module name from frame's globals - // This is similar to CPython's sys._getframe().f_globals.get('__name__') - frame.globals.get_item("__name__", vm).ok() -} +pub use typevar::*; + +#[pymodule(sub)] +pub(crate) mod typevar { + use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyTuple, PyTupleRef, PyType, PyTypeRef, make_union}, + common::lock::PyMutex, + function::{FuncArgs, PyComparisonValue}, + protocol::PyNumberMethods, + stdlib::typing::{call_typing_func_object, decl::const_evaluator_alloc}, + types::{AsNumber, Comparable, Constructor, Iterable, PyComparisonOp, Representable}, + }; -/// Set __module__ attribute for an object based on the caller's module. -/// This follows CPython's behavior for TypeVar and similar objects. -fn set_module_from_caller(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // Note: CPython gets module from frame->f_funcobj, but RustPython's Frame - // architecture is different - we use globals['__name__'] instead - if let Some(module_name) = caller(vm) { - // Special handling for certain module names - if let Ok(name_str) = module_name.str(vm) { - let name = name_str.as_str(); - // CPython sets __module__ to None for builtins and <...> modules - // Also set to None for exec contexts (no __name__ in globals means exec) - if name == "builtins" || name.starts_with('<') { - // Don't set __module__ attribute at all (CPython behavior) - // This allows the typing module to handle it - return Ok(()); + fn type_check(arg: PyObjectRef, msg: &str, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Calling typing.py here leads to bootstrapping problems + if vm.is_none(&arg) { + return Ok(arg.class().to_owned().into()); + } + let message_str: PyObjectRef = vm.ctx.new_str(msg).into(); + call_typing_func_object(vm, "_type_check", (arg, message_str)) + } + + fn variance_repr( + name: &str, + infer_variance: bool, + covariant: bool, + contravariant: bool, + ) -> String { + if infer_variance { + return name.to_string(); + } + let prefix = if covariant { + '+' + } else if contravariant { + '-' + } else { + '~' + }; + format!("{prefix}{name}") + } + + /// Get the module of the caller frame, similar to CPython's caller() function. + /// Returns the module name or None if not found. + /// + /// Note: CPython's implementation (in typevarobject.c) gets the module from the + /// frame's function object using PyFunction_GetModule(f->f_funcobj). However, + /// RustPython's Frame doesn't store a reference to the function object, so we + /// get the module name from the frame's globals dictionary instead. + fn caller(vm: &VirtualMachine) -> Option<PyObjectRef> { + let frame = vm.current_frame()?; + + // In RustPython, we get the module name from frame's globals + // This is similar to CPython's sys._getframe().f_globals.get('__name__') + frame.globals.get_item("__name__", vm).ok() + } + + /// Set __module__ attribute for an object based on the caller's module. + /// This follows CPython's behavior for TypeVar and similar objects. + fn set_module_from_caller(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + // Note: CPython gets module from frame->f_funcobj, but RustPython's Frame + // architecture is different - we use globals['__name__'] instead + if let Some(module_name) = caller(vm) { + // Special handling for certain module names + if let Ok(name_str) = module_name.str(vm) { + let name = name_str.as_str(); + // CPython sets __module__ to None for builtins and <...> modules + // Also set to None for exec contexts (no __name__ in globals means exec) + if name == "builtins" || name.starts_with('<') { + // Don't set __module__ attribute at all (CPython behavior) + // This allows the typing module to handle it + return Ok(()); + } } + obj.set_attr("__module__", module_name, vm)?; + } else { + // If no module name is found (e.g., in exec context), set __module__ to None + obj.set_attr("__module__", vm.ctx.none(), vm)?; } - obj.set_attr("__module__", module_name, vm)?; - } else { - // If no module name is found (e.g., in exec context), set __module__ to None - obj.set_attr("__module__", vm.ctx.none(), vm)?; + Ok(()) } - Ok(()) -} -#[pyclass(name = "TypeVar", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct TypeVar { - name: PyObjectRef, // TODO PyStrRef? - bound: parking_lot::Mutex<PyObjectRef>, - evaluate_bound: PyObjectRef, - constraints: parking_lot::Mutex<PyObjectRef>, - evaluate_constraints: PyObjectRef, - default_value: parking_lot::Mutex<PyObjectRef>, - evaluate_default: PyMutex<PyObjectRef>, - covariant: bool, - contravariant: bool, - infer_variance: bool, -} -#[pyclass(flags(HAS_DICT), with(AsNumber, Constructor, Representable))] -impl TypeVar { - #[pymethod] - fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot subclass an instance of TypeVar")) - } + #[pyattr] + #[pyclass(name = "TypeVar", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct TypeVar { + name: PyObjectRef, // TODO PyStrRef? + bound: PyMutex<PyObjectRef>, + evaluate_bound: PyObjectRef, + constraints: PyMutex<PyObjectRef>, + evaluate_constraints: PyObjectRef, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + covariant: bool, + contravariant: bool, + infer_variance: bool, + } + #[pyclass(flags(HAS_DICT), with(AsNumber, Constructor, Representable))] + impl TypeVar { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of TypeVar")) + } - #[pygetset] - fn __name__(&self) -> PyObjectRef { - self.name.clone() - } + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() + } - #[pygetset] - fn __constraints__(&self, vm: &VirtualMachine) -> PyResult { - let mut constraints = self.constraints.lock(); - if !vm.is_none(&constraints) { - return Ok(constraints.clone()); + #[pygetset] + fn __constraints__(&self, vm: &VirtualMachine) -> PyResult { + let mut constraints = self.constraints.lock(); + if !vm.is_none(&constraints) { + return Ok(constraints.clone()); + } + let r = if !vm.is_none(&self.evaluate_constraints) { + *constraints = self.evaluate_constraints.call((1i32,), vm)?; + constraints.clone() + } else { + vm.ctx.empty_tuple.clone().into() + }; + Ok(r) } - let r = if !vm.is_none(&self.evaluate_constraints) { - *constraints = self.evaluate_constraints.call((), vm)?; - constraints.clone() - } else { - vm.ctx.empty_tuple.clone().into() - }; - Ok(r) - } - #[pygetset] - fn __bound__(&self, vm: &VirtualMachine) -> PyResult { - let mut bound = self.bound.lock(); - if !vm.is_none(&bound) { - return Ok(bound.clone()); + #[pygetset] + fn __bound__(&self, vm: &VirtualMachine) -> PyResult { + let mut bound = self.bound.lock(); + if !vm.is_none(&bound) { + return Ok(bound.clone()); + } + let r = if !vm.is_none(&self.evaluate_bound) { + *bound = self.evaluate_bound.call((1i32,), vm)?; + bound.clone() + } else { + vm.ctx.none() + }; + Ok(r) } - let r = if !vm.is_none(&self.evaluate_bound) { - *bound = self.evaluate_bound.call((), vm)?; - bound.clone() - } else { - vm.ctx.none() - }; - Ok(r) - } - #[pygetset] - const fn __covariant__(&self) -> bool { - self.covariant - } + #[pygetset] + const fn __covariant__(&self) -> bool { + self.covariant + } - #[pygetset] - const fn __contravariant__(&self) -> bool { - self.contravariant - } + #[pygetset] + const fn __contravariant__(&self) -> bool { + self.contravariant + } - #[pygetset] - const fn __infer_variance__(&self) -> bool { - self.infer_variance - } + #[pygetset] + const fn __infer_variance__(&self) -> bool { + self.infer_variance + } - #[pygetset] - fn __default__(&self, vm: &VirtualMachine) -> PyResult { - let mut default_value = self.default_value.lock(); - // Check if default_value is NoDefault (not just None) - if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(default_value.clone()); - } - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - *default_value = evaluate_default.call((), vm)?; - Ok(default_value.clone()) - } else { - // Return NoDefault singleton - Ok(vm.ctx.typing_no_default.clone().into()) + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } } - } - #[pymethod] - fn __typing_subst__( - zelf: crate::PyRef<Self>, - arg: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - _call_typing_func_object(vm, "_typevar_subst", (self_obj, arg)) - } + #[pygetset] + fn evaluate_bound(&self, vm: &VirtualMachine) -> PyResult { + if !vm.is_none(&self.evaluate_bound) { + return Ok(self.evaluate_bound.clone()); + } + let bound = self.bound.lock(); + if !vm.is_none(&bound) { + return Ok(const_evaluator_alloc(bound.clone(), vm)); + } + Ok(vm.ctx.none()) + } - #[pymethod] - fn __reduce__(&self) -> PyObjectRef { - self.name.clone() - } + #[pygetset] + fn evaluate_constraints(&self, vm: &VirtualMachine) -> PyResult { + if !vm.is_none(&self.evaluate_constraints) { + return Ok(self.evaluate_constraints.clone()); + } + let constraints = self.constraints.lock(); + if !vm.is_none(&constraints) { + return Ok(const_evaluator_alloc(constraints.clone(), vm)); + } + Ok(vm.ctx.none()) + } - #[pymethod] - fn has_default(&self, vm: &VirtualMachine) -> bool { - if !vm.is_none(&self.evaluate_default.lock()) { - return true; + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) } - let default_value = self.default_value.lock(); - // Check if default_value is not NoDefault - !default_value.is(&vm.ctx.typing_no_default) - } - #[pymethod] - fn __typing_prepare_subst__( - zelf: crate::PyRef<Self>, - alias: PyObjectRef, - args: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - // Convert args to tuple if needed - let args_tuple = if let Ok(tuple) = args.try_to_ref::<rustpython_vm::builtins::PyTuple>(vm) - { - tuple - } else { - return Ok(args); - }; + #[pymethod] + fn __typing_subst__( + zelf: crate::PyRef<Self>, + arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_typevar_subst", (self_obj, arg)) + } - // Get alias.__parameters__ - let parameters = alias.get_attr(identifier!(vm, __parameters__), vm)?; - let params_tuple: PyTupleRef = parameters.try_into_value(vm)?; - - // Find our index in parameters - let self_obj: PyObjectRef = zelf.to_owned().into(); - let param_index = params_tuple.iter().position(|p| p.is(&self_obj)); - - if let Some(index) = param_index { - // Check if we have enough arguments - if args_tuple.len() <= index && zelf.has_default(vm) { - // Need to add default value - let mut new_args: Vec<PyObjectRef> = args_tuple.iter().cloned().collect(); - - // Add default value at the correct position - while new_args.len() <= index { - // For the current parameter, add its default - if new_args.len() == index { - let default_val = zelf.__default__(vm)?; - new_args.push(default_val); - } else { - // This shouldn't happen in well-formed code - break; - } - } + #[pymethod] + fn __reduce__(&self) -> PyObjectRef { + self.name.clone() + } - return Ok(rustpython_vm::builtins::PyTuple::new_ref(new_args, &vm.ctx).into()); + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; } + let default_value = self.default_value.lock(); + // Check if default_value is not NoDefault + !default_value.is(&vm.ctx.typing_no_default) } - // No changes needed - Ok(args) + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Convert args to tuple if needed + let args_tuple = + if let Ok(tuple) = args.try_to_ref::<rustpython_vm::builtins::PyTuple>(vm) { + tuple + } else { + return Ok(args); + }; + + // Get alias.__parameters__ + let parameters = alias.get_attr(identifier!(vm, __parameters__), vm)?; + let params_tuple: PyTupleRef = parameters.try_into_value(vm)?; + + // Find our index in parameters + let self_obj: PyObjectRef = zelf.to_owned().into(); + let param_index = params_tuple.iter().position(|p| p.is(&self_obj)); + + if let Some(index) = param_index { + // Check if we have enough arguments + if args_tuple.len() <= index && zelf.has_default(vm) { + // Need to add default value + let mut new_args: Vec<PyObjectRef> = args_tuple.iter().cloned().collect(); + + // Add default value at the correct position + while new_args.len() <= index { + // For the current parameter, add its default + if new_args.len() == index { + let default_val = zelf.__default__(vm)?; + new_args.push(default_val); + } else { + // This shouldn't happen in well-formed code + break; + } + } + + return Ok(rustpython_vm::builtins::PyTuple::new_ref(new_args, &vm.ctx).into()); + } + } + + // No changes needed + Ok(args) + } } -} -impl Representable for TypeVar { - #[inline(always)] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let name = zelf.name.str(vm)?; - let repr = if zelf.covariant { - format!("+{name}") - } else if zelf.contravariant { - format!("-{name}") - } else { - format!("~{name}") - }; - Ok(repr) + impl Representable for TypeVar { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.name.str(vm)?; + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) + } } -} -impl AsNumber for TypeVar { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| { - _call_typing_func_object(vm, "_make_union", (a.to_owned(), b.to_owned())) - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER + impl AsNumber for TypeVar { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + let args = PyTuple::new_ref(vec![a.to_owned(), b.to_owned()], &vm.ctx); + make_union(&args, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } } -} -impl Constructor for TypeVar { - type Args = FuncArgs; + impl Constructor for TypeVar { + type Args = FuncArgs; - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let typevar = <Self as Constructor>::py_new(&cls, args, vm)?; - let obj = typevar.into_ref_with_type(vm, cls)?; - let obj_ref: PyObjectRef = obj.into(); - set_module_from_caller(&obj_ref, vm)?; - Ok(obj_ref) - } + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let typevar = <Self as Constructor>::py_new(&cls, args, vm)?; + let obj = typevar.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) + } - fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { - let mut kwargs = args.kwargs; - // Parse arguments manually - let (name, constraints) = if args.args.is_empty() { - // Check if name is provided as keyword argument - if let Some(name) = kwargs.swap_remove("name") { - (name, vec![]) + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut kwargs = args.kwargs; + // Parse arguments manually + let (name, constraints) = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + (name, vec![]) + } else { + return Err( + vm.new_type_error("TypeVar() missing required argument: 'name' (pos 1)") + ); + } + } else if args.args.len() == 1 { + (args.args[0].clone(), vec![]) } else { - return Err( - vm.new_type_error("TypeVar() missing required argument: 'name' (pos 1)") - ); + let name = args.args[0].clone(); + let constraints = args.args[1..].to_vec(); + (name, constraints) + }; + + let bound = kwargs.swap_remove("bound"); + let covariant = kwargs + .swap_remove("covariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let contravariant = kwargs + .swap_remove("contravariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let infer_variance = kwargs + .swap_remove("infer_variance") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "TypeVar() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); } - } else if args.args.len() == 1 { - (args.args[0].clone(), vec![]) - } else { - let name = args.args[0].clone(); - let constraints = args.args[1..].to_vec(); - (name, constraints) - }; - let bound = kwargs.swap_remove("bound"); - let covariant = kwargs - .swap_remove("covariant") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let contravariant = kwargs - .swap_remove("contravariant") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let infer_variance = kwargs - .swap_remove("infer_variance") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let default = kwargs.swap_remove("default"); - - // Check for unexpected keyword arguments - if !kwargs.is_empty() { - let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); - return Err(vm.new_type_error(format!( - "TypeVar() got unexpected keyword argument(s): {}", - unexpected_keys.join(", ") - ))); - } - - // Check for invalid combinations - if covariant && contravariant { - return Err(vm.new_value_error("Bivariant type variables are not supported.")); - } - - if infer_variance && (covariant || contravariant) { - return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); - } - - // Handle constraints and bound - let (constraints_obj, evaluate_constraints) = if !constraints.is_empty() { - // Check for single constraint - if constraints.len() == 1 { - return Err(vm.new_type_error("A single constraint is not allowed")); + // Check for invalid combinations + if covariant && contravariant { + return Err(vm.new_value_error("Bivariant type variables are not supported.")); } - if bound.is_some() { - return Err(vm.new_type_error("Constraints cannot be used with bound")); + + if infer_variance && (covariant || contravariant) { + return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); } - let constraints_tuple = vm.ctx.new_tuple(constraints); - (constraints_tuple.clone().into(), constraints_tuple.into()) - } else { - (vm.ctx.none(), vm.ctx.none()) - }; - // Handle bound - let (bound_obj, evaluate_bound) = if let Some(bound) = bound { - if vm.is_none(&bound) { - (vm.ctx.none(), vm.ctx.none()) + // Handle constraints and bound + let (constraints_obj, evaluate_constraints) = if !constraints.is_empty() { + // Check for single constraint + if constraints.len() == 1 { + return Err(vm.new_type_error("A single constraint is not allowed")); + } + if bound.is_some() { + return Err(vm.new_type_error("Constraints cannot be used with bound")); + } + let constraints_tuple = vm.ctx.new_tuple(constraints); + (constraints_tuple.into(), vm.ctx.none()) } else { - // Type check the bound - let bound = type_check(bound, "Bound must be a type.", vm)?; - (bound, vm.ctx.none()) - } - } else { - (vm.ctx.none(), vm.ctx.none()) - }; + (vm.ctx.none(), vm.ctx.none()) + }; - // Handle default value - let (default_value, evaluate_default) = if let Some(default) = default { - (default, vm.ctx.none()) - } else { - // If no default provided, use NoDefault singleton - (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) - }; + // Handle bound + let (bound_obj, evaluate_bound) = if let Some(bound) = bound { + if vm.is_none(&bound) { + (vm.ctx.none(), vm.ctx.none()) + } else { + // Type check the bound + let bound = type_check(bound, "Bound must be a type.", vm)?; + (bound, vm.ctx.none()) + } + } else { + (vm.ctx.none(), vm.ctx.none()) + }; - Ok(Self { - name, - bound: parking_lot::Mutex::new(bound_obj), - evaluate_bound, - constraints: parking_lot::Mutex::new(constraints_obj), - evaluate_constraints, - default_value: parking_lot::Mutex::new(default_value), - evaluate_default: PyMutex::new(evaluate_default), - covariant, - contravariant, - infer_variance, - }) + // Handle default value + let (default_value, evaluate_default) = if let Some(default) = default { + (default, vm.ctx.none()) + } else { + // If no default provided, use NoDefault singleton + (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) + }; + + Ok(Self { + name, + bound: PyMutex::new(bound_obj), + evaluate_bound, + constraints: PyMutex::new(constraints_obj), + evaluate_constraints, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(evaluate_default), + covariant, + contravariant, + infer_variance, + }) + } } -} -impl TypeVar { - pub fn new( - vm: &VirtualMachine, - name: PyObjectRef, - evaluate_bound: PyObjectRef, - evaluate_constraints: PyObjectRef, - ) -> Self { - Self { - name, - bound: parking_lot::Mutex::new(vm.ctx.none()), - evaluate_bound, - constraints: parking_lot::Mutex::new(vm.ctx.none()), - evaluate_constraints, - default_value: parking_lot::Mutex::new(vm.ctx.typing_no_default.clone().into()), - evaluate_default: PyMutex::new(vm.ctx.none()), - covariant: false, - contravariant: false, - infer_variance: false, + impl TypeVar { + pub fn new( + vm: &VirtualMachine, + name: PyObjectRef, + evaluate_bound: PyObjectRef, + evaluate_constraints: PyObjectRef, + ) -> Self { + Self { + name, + bound: PyMutex::new(vm.ctx.none()), + evaluate_bound, + constraints: PyMutex::new(vm.ctx.none()), + evaluate_constraints, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant: false, + contravariant: false, + infer_variance: true, + } } } -} -#[pyclass(name = "ParamSpec", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct ParamSpec { - name: PyObjectRef, - bound: Option<PyObjectRef>, - default_value: PyObjectRef, - evaluate_default: PyMutex<PyObjectRef>, - covariant: bool, - contravariant: bool, - infer_variance: bool, -} + #[pyattr] + #[pyclass(name = "ParamSpec", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpec { + name: PyObjectRef, + bound: Option<PyObjectRef>, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + covariant: bool, + contravariant: bool, + infer_variance: bool, + } + + #[pyclass(flags(HAS_DICT), with(AsNumber, Constructor, Representable))] + impl ParamSpec { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpec")) + } -#[pyclass(flags(HAS_DICT), with(AsNumber, Constructor, Representable))] -impl ParamSpec { - #[pymethod] - fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot subclass an instance of ParamSpec")) - } + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() + } - #[pygetset] - fn __name__(&self) -> PyObjectRef { - self.name.clone() - } + #[pygetset] + fn args(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + let psa = ParamSpecArgs { + __origin__: self_obj, + }; + Ok(psa.into_ref(&vm.ctx).into()) + } - #[pygetset] - fn args(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - let psa = ParamSpecArgs { - __origin__: self_obj, - }; - Ok(psa.into_ref(&vm.ctx).into()) - } + #[pygetset] + fn kwargs(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + let psk = ParamSpecKwargs { + __origin__: self_obj, + }; + Ok(psk.into_ref(&vm.ctx).into()) + } - #[pygetset] - fn kwargs(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - let psk = ParamSpecKwargs { - __origin__: self_obj, - }; - Ok(psk.into_ref(&vm.ctx).into()) - } + #[pygetset] + fn __bound__(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(bound) = self.bound.clone() { + return bound; + } + vm.ctx.none() + } - #[pygetset] - fn __bound__(&self, vm: &VirtualMachine) -> PyObjectRef { - if let Some(bound) = self.bound.clone() { - return bound; + #[pygetset] + const fn __covariant__(&self) -> bool { + self.covariant } - vm.ctx.none() - } - #[pygetset] - const fn __covariant__(&self) -> bool { - self.covariant - } + #[pygetset] + const fn __contravariant__(&self) -> bool { + self.contravariant + } - #[pygetset] - const fn __contravariant__(&self) -> bool { - self.contravariant - } + #[pygetset] + const fn __infer_variance__(&self) -> bool { + self.infer_variance + } - #[pygetset] - const fn __infer_variance__(&self) -> bool { - self.infer_variance - } + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } + } - #[pygetset] - fn __default__(&self, vm: &VirtualMachine) -> PyResult { - // Check if default_value is NoDefault (not just None) - if !self.default_value.is(&vm.ctx.typing_no_default) { - return Ok(self.default_value.clone()); + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) } - // handle evaluate_default - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - let default_value = evaluate_default.call((), vm)?; - return Ok(default_value); + + #[pymethod] + fn __reduce__(&self) -> PyResult { + Ok(self.name.clone()) } - // Return NoDefault singleton - Ok(vm.ctx.typing_no_default.clone().into()) - } - #[pygetset] - fn evaluate_default(&self, _vm: &VirtualMachine) -> PyObjectRef { - self.evaluate_default.lock().clone() - } + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; + } + !self.default_value.lock().is(&vm.ctx.typing_no_default) + } - #[pymethod] - fn __reduce__(&self) -> PyResult { - Ok(self.name.clone()) - } + #[pymethod] + fn __typing_subst__( + zelf: crate::PyRef<Self>, + arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_paramspec_subst", (self_obj, arg)) + } - #[pymethod] - fn has_default(&self, vm: &VirtualMachine) -> bool { - if !vm.is_none(&self.evaluate_default.lock()) { - return true; + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_paramspec_prepare_subst", (self_obj, alias, args)) } - // Check if default_value is not NoDefault - !self.default_value.is(&vm.ctx.typing_no_default) } - #[pymethod] - fn __typing_subst__( - zelf: crate::PyRef<Self>, - arg: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - _call_typing_func_object(vm, "_paramspec_subst", (self_obj, arg)) + impl AsNumber for ParamSpec { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + let args = PyTuple::new_ref(vec![a.to_owned(), b.to_owned()], &vm.ctx); + make_union(&args, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } } - #[pymethod] - fn __typing_prepare_subst__( - zelf: crate::PyRef<Self>, - alias: PyObjectRef, - args: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - _call_typing_func_object(vm, "_paramspec_prepare_subst", (self_obj, alias, args)) - } -} + impl Constructor for ParamSpec { + type Args = FuncArgs; -impl AsNumber for ParamSpec { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| { - _call_typing_func_object(vm, "_make_union", (a.to_owned(), b.to_owned())) - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} + fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let mut kwargs = args.kwargs; + // Parse arguments manually + let name = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + name + } else { + return Err( + vm.new_type_error("ParamSpec() missing required argument: 'name' (pos 1)") + ); + } + } else if args.args.len() == 1 { + args.args[0].clone() + } else { + return Err(vm.new_type_error("ParamSpec() takes at most 1 positional argument")); + }; + + let bound = kwargs + .swap_remove("bound") + .map(|b| type_check(b, "Bound must be a type.", vm)) + .transpose()?; + let covariant = kwargs + .swap_remove("covariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let contravariant = kwargs + .swap_remove("contravariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let infer_variance = kwargs + .swap_remove("infer_variance") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "ParamSpec() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); + } -impl Constructor for ParamSpec { - type Args = FuncArgs; + // Check for invalid combinations + if covariant && contravariant { + return Err(vm.new_value_error("Bivariant type variables are not supported.")); + } - fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let mut kwargs = args.kwargs; - // Parse arguments manually - let name = if args.args.is_empty() { - // Check if name is provided as keyword argument - if let Some(name) = kwargs.swap_remove("name") { - name - } else { - return Err( - vm.new_type_error("ParamSpec() missing required argument: 'name' (pos 1)") - ); + if infer_variance && (covariant || contravariant) { + return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); } - } else if args.args.len() == 1 { - args.args[0].clone() - } else { - return Err(vm.new_type_error("ParamSpec() takes at most 1 positional argument")); - }; - let bound = kwargs.swap_remove("bound"); - let covariant = kwargs - .swap_remove("covariant") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let contravariant = kwargs - .swap_remove("contravariant") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let infer_variance = kwargs - .swap_remove("infer_variance") - .map(|v| v.try_to_bool(vm)) - .transpose()? - .unwrap_or(false); - let default = kwargs.swap_remove("default"); - - // Check for unexpected keyword arguments - if !kwargs.is_empty() { - let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); - return Err(vm.new_type_error(format!( - "ParamSpec() got unexpected keyword argument(s): {}", - unexpected_keys.join(", ") - ))); - } - - // Check for invalid combinations - if covariant && contravariant { - return Err(vm.new_value_error("Bivariant type variables are not supported.")); - } - - if infer_variance && (covariant || contravariant) { - return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); - } - - // Handle default value - let default_value = default.unwrap_or_else(|| vm.ctx.typing_no_default.clone().into()); - - let paramspec = Self { - name, - bound, - default_value, - evaluate_default: PyMutex::new(vm.ctx.none()), - covariant, - contravariant, - infer_variance, - }; + // Handle default value + let default_value = default.unwrap_or_else(|| vm.ctx.typing_no_default.clone().into()); + + let paramspec = Self { + name, + bound, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant, + contravariant, + infer_variance, + }; + + let obj = paramspec.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) + } - let obj = paramspec.into_ref_with_type(vm, cls)?; - let obj_ref: PyObjectRef = obj.into(); - set_module_from_caller(&obj_ref, vm)?; - Ok(obj_ref) + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } } - fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("use slot_new") + impl Representable for ParamSpec { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.__name__().str(vm)?; + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) + } } -} -impl Representable for ParamSpec { - #[inline(always)] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let name = zelf.__name__().str(vm)?; - Ok(format!("~{name}")) + impl ParamSpec { + pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { + Self { + name, + bound: None, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant: false, + contravariant: false, + infer_variance: true, + } + } } -} -impl ParamSpec { - pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { - Self { - name, - bound: None, - default_value: vm.ctx.typing_no_default.clone().into(), - evaluate_default: PyMutex::new(vm.ctx.none()), - covariant: false, - contravariant: false, - infer_variance: false, + #[pyattr] + #[pyclass(name = "TypeVarTuple", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct TypeVarTuple { + name: PyObjectRef, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + } + #[pyclass(flags(HAS_DICT), with(Constructor, Representable, Iterable))] + impl TypeVarTuple { + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() } - } -} -#[pyclass(name = "TypeVarTuple", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct TypeVarTuple { - name: PyObjectRef, - default_value: parking_lot::Mutex<PyObjectRef>, - evaluate_default: PyMutex<PyObjectRef>, -} -#[pyclass(flags(HAS_DICT), with(Constructor, Representable, Iterable))] -impl TypeVarTuple { - #[pygetset] - fn __name__(&self) -> PyObjectRef { - self.name.clone() - } + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } + } - #[pygetset] - fn __default__(&self, vm: &VirtualMachine) -> PyResult { - let mut default_value = self.default_value.lock(); - // Check if default_value is NoDefault (not just None) - if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(default_value.clone()); - } - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - *default_value = evaluate_default.call((), vm)?; - Ok(default_value.clone()) - } else { - // Return NoDefault singleton - Ok(vm.ctx.typing_no_default.clone().into()) + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) } - } - #[pymethod] - fn has_default(&self, vm: &VirtualMachine) -> bool { - if !vm.is_none(&self.evaluate_default.lock()) { - return true; + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; + } + let default_value = self.default_value.lock(); + !default_value.is(&vm.ctx.typing_no_default) } - let default_value = self.default_value.lock(); - // Check if default_value is not NoDefault - !default_value.is(&vm.ctx.typing_no_default) - } - #[pymethod] - fn __reduce__(&self) -> PyObjectRef { - self.name.clone() - } + #[pymethod] + fn __reduce__(&self) -> PyObjectRef { + self.name.clone() + } - #[pymethod] - fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot subclass an instance of TypeVarTuple")) - } + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of TypeVarTuple")) + } - #[pymethod] - fn __typing_subst__(&self, _arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Substitution of bare TypeVarTuple is not supported")) - } + #[pymethod] + fn __typing_subst__(&self, _arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Substitution of bare TypeVarTuple is not supported")) + } - #[pymethod] - fn __typing_prepare_subst__( - zelf: crate::PyRef<Self>, - alias: PyObjectRef, - args: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let self_obj: PyObjectRef = zelf.into(); - _call_typing_func_object(vm, "_typevartuple_prepare_subst", (self_obj, alias, args)) + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_typevartuple_prepare_subst", (self_obj, alias, args)) + } } -} -impl Iterable for TypeVarTuple { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - // When unpacking TypeVarTuple with *, return [Unpack[self]] - // This is how CPython handles Generic[*Ts] - let typing = vm.import("typing", 0)?; - let unpack = typing.get_attr("Unpack", vm)?; - let zelf_obj: PyObjectRef = zelf.into(); - let unpacked = vm.call_method(&unpack, "__getitem__", (zelf_obj,))?; - let list = vm.ctx.new_list(vec![unpacked]); - let list_obj: PyObjectRef = list.into(); - vm.call_method(&list_obj, "__iter__", ()) + impl Iterable for TypeVarTuple { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + // When unpacking TypeVarTuple with *, return [Unpack[self]] + // This is how CPython handles Generic[*Ts] + let typing = vm.import("typing", 0)?; + let unpack = typing.get_attr("Unpack", vm)?; + let zelf_obj: PyObjectRef = zelf.into(); + let unpacked = vm.call_method(&unpack, "__getitem__", (zelf_obj,))?; + let list = vm.ctx.new_list(vec![unpacked]); + let list_obj: PyObjectRef = list.into(); + vm.call_method(&list_obj, "__iter__", ()) + } } -} -impl Constructor for TypeVarTuple { - type Args = FuncArgs; + impl Constructor for TypeVarTuple { + type Args = FuncArgs; - fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let mut kwargs = args.kwargs; - // Parse arguments manually - let name = if args.args.is_empty() { - // Check if name is provided as keyword argument - if let Some(name) = kwargs.swap_remove("name") { - name + fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let mut kwargs = args.kwargs; + // Parse arguments manually + let name = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + name + } else { + return Err(vm.new_type_error( + "TypeVarTuple() missing required argument: 'name' (pos 1)", + )); + } + } else if args.args.len() == 1 { + args.args[0].clone() } else { - return Err( - vm.new_type_error("TypeVarTuple() missing required argument: 'name' (pos 1)") - ); + return Err(vm.new_type_error("TypeVarTuple() takes at most 1 positional argument")); + }; + + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "TypeVarTuple() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); } - } else if args.args.len() == 1 { - args.args[0].clone() - } else { - return Err(vm.new_type_error("TypeVarTuple() takes at most 1 positional argument")); - }; - let default = kwargs.swap_remove("default"); - - // Check for unexpected keyword arguments - if !kwargs.is_empty() { - let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); - return Err(vm.new_type_error(format!( - "TypeVarTuple() got unexpected keyword argument(s): {}", - unexpected_keys.join(", ") - ))); + // Handle default value + let (default_value, evaluate_default) = if let Some(default) = default { + (default, vm.ctx.none()) + } else { + // If no default provided, use NoDefault singleton + (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) + }; + + let typevartuple = Self { + name, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(evaluate_default), + }; + + let obj = typevartuple.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) } - // Handle default value - let (default_value, evaluate_default) = if let Some(default) = default { - (default, vm.ctx.none()) - } else { - // If no default provided, use NoDefault singleton - (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) - }; - - let typevartuple = Self { - name, - default_value: parking_lot::Mutex::new(default_value), - evaluate_default: PyMutex::new(evaluate_default), - }; - - let obj = typevartuple.into_ref_with_type(vm, cls)?; - let obj_ref: PyObjectRef = obj.into(); - set_module_from_caller(&obj_ref, vm)?; - Ok(obj_ref) - } - - fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("use slot_new") + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } } -} -impl Representable for TypeVarTuple { - #[inline(always)] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let name = zelf.name.str(vm)?; - Ok(name.to_string()) + impl Representable for TypeVarTuple { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.name.str(vm)?; + Ok(name.to_string()) + } } -} -impl TypeVarTuple { - pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { - Self { - name, - default_value: parking_lot::Mutex::new(vm.ctx.typing_no_default.clone().into()), - evaluate_default: PyMutex::new(vm.ctx.none()), + impl TypeVarTuple { + pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { + Self { + name, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + } } } -} -#[pyclass(name = "ParamSpecArgs", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct ParamSpecArgs { - __origin__: PyObjectRef, -} -#[pyclass(with(Constructor, Representable, Comparable))] -impl ParamSpecArgs { - #[pymethod] - fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot subclass an instance of ParamSpecArgs")) - } + #[pyattr] + #[pyclass(name = "ParamSpecArgs", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpecArgs { + __origin__: PyObjectRef, + } + #[pyclass(with(Constructor, Representable, Comparable))] + impl ParamSpecArgs { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpecArgs")) + } - #[pygetset] - fn __origin__(&self) -> PyObjectRef { - self.__origin__.clone() + #[pygetset] + fn __origin__(&self) -> PyObjectRef { + self.__origin__.clone() + } } -} - -impl Constructor for ParamSpecArgs { - type Args = (PyObjectRef,); - fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - let origin = args.0; - Ok(Self { __origin__: origin }) - } -} + impl Constructor for ParamSpecArgs { + type Args = (PyObjectRef,); -impl Representable for ParamSpecArgs { - #[inline(always)] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - // Check if origin is a ParamSpec - if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { - return Ok(format!("{name}.args", name = name.str(vm)?)); + fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + let origin = args.0; + Ok(Self { __origin__: origin }) } - Ok(format!("{:?}.args", zelf.__origin__)) } -} -impl Comparable for ParamSpecArgs { - fn cmp( - zelf: &crate::Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - fn eq( - zelf: &crate::Py<ParamSpecArgs>, - other: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<bool> { - // First check if other is also ParamSpecArgs - if let Ok(other_args) = other.downcast::<ParamSpecArgs>() { - // Check if they have the same origin - return Ok(zelf.__origin__.is(&other_args.__origin__)); + impl Representable for ParamSpecArgs { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + // Check if origin is a ParamSpec + if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { + return Ok(format!("{name}.args", name = name.str(vm)?)); } - Ok(false) + Ok(format!("{:?}.args", zelf.__origin__)) } - match op { - PyComparisonOp::Eq => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok(result.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - } - PyComparisonOp::Ne => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok((!result).into()) - } else { - Ok(PyComparisonValue::NotImplemented) + } + + impl Comparable for ParamSpecArgs { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_args) = other.downcast_ref::<ParamSpecArgs>() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_args.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); } - } - _ => Ok(PyComparisonValue::NotImplemented), + Ok(PyComparisonValue::NotImplemented) + }) } } -} -#[pyclass(name = "ParamSpecKwargs", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct ParamSpecKwargs { - __origin__: PyObjectRef, -} -#[pyclass(with(Constructor, Representable, Comparable))] -impl ParamSpecKwargs { - #[pymethod] - fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot subclass an instance of ParamSpecKwargs")) - } + #[pyattr] + #[pyclass(name = "ParamSpecKwargs", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpecKwargs { + __origin__: PyObjectRef, + } + #[pyclass(with(Constructor, Representable, Comparable))] + impl ParamSpecKwargs { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpecKwargs")) + } - #[pygetset] - fn __origin__(&self) -> PyObjectRef { - self.__origin__.clone() + #[pygetset] + fn __origin__(&self) -> PyObjectRef { + self.__origin__.clone() + } } -} -impl Constructor for ParamSpecKwargs { - type Args = (PyObjectRef,); + impl Constructor for ParamSpecKwargs { + type Args = (PyObjectRef,); - fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { - let origin = args.0; - Ok(Self { __origin__: origin }) - } -} - -impl Representable for ParamSpecKwargs { - #[inline(always)] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - // Check if origin is a ParamSpec - if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { - return Ok(format!("{name}.kwargs", name = name.str(vm)?)); + fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + let origin = args.0; + Ok(Self { __origin__: origin }) } - Ok(format!("{:?}.kwargs", zelf.__origin__)) } -} -impl Comparable for ParamSpecKwargs { - fn cmp( - zelf: &crate::Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - fn eq( - zelf: &crate::Py<ParamSpecKwargs>, - other: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult<bool> { - // First check if other is also ParamSpecKwargs - if let Ok(other_kwargs) = other.downcast::<ParamSpecKwargs>() { - // Check if they have the same origin - return Ok(zelf.__origin__.is(&other_kwargs.__origin__)); + impl Representable for ParamSpecKwargs { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + // Check if origin is a ParamSpec + if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { + return Ok(format!("{name}.kwargs", name = name.str(vm)?)); } - Ok(false) + Ok(format!("{:?}.kwargs", zelf.__origin__)) } - match op { - PyComparisonOp::Eq => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok(result.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - } - PyComparisonOp::Ne => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok((!result).into()) - } else { - Ok(PyComparisonValue::NotImplemented) + } + + impl Comparable for ParamSpecKwargs { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_kwargs) = other.downcast_ref::<ParamSpecKwargs>() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_kwargs.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); } - } - _ => Ok(PyComparisonValue::NotImplemented), + Ok(PyComparisonValue::NotImplemented) + }) } } -} -/// Helper function to call typing module functions with cls as first argument -/// Similar to CPython's call_typing_args_kwargs -fn call_typing_args_kwargs( - name: &'static str, - cls: PyTypeRef, - args: FuncArgs, - vm: &VirtualMachine, -) -> PyResult { - let typing = vm.import("typing", 0)?; - let func = typing.get_attr(name, vm)?; - - // Prepare arguments: (cls, *args) - let mut call_args = vec![cls.into()]; - call_args.extend(args.args); - - // Call with prepared args and original kwargs - let func_args = FuncArgs { - args: call_args, - kwargs: args.kwargs, - }; + /// Helper function to call typing module functions with cls as first argument + /// Similar to CPython's call_typing_args_kwargs + fn call_typing_args_kwargs( + name: &'static str, + cls: PyTypeRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult { + let typing = vm.import("typing", 0)?; + let func = typing.get_attr(name, vm)?; - func.call(func_args, vm) -} + // Prepare arguments: (cls, *args) + let mut call_args = vec![cls.into()]; + call_args.extend(args.args); -#[pyclass(name = "Generic", module = "typing")] -#[derive(Debug, PyPayload)] -#[allow(dead_code)] -pub struct Generic {} + // Call with prepared args and original kwargs + let func_args = FuncArgs { + args: call_args, + kwargs: args.kwargs, + }; -#[pyclass(flags(BASETYPE))] -impl Generic { - #[pyattr] - fn __slots__(ctx: &Context) -> PyTupleRef { - ctx.empty_tuple.clone() + func.call(func_args, vm) } - #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - call_typing_args_kwargs("_generic_class_getitem", cls, args, vm) - } + #[pyattr] + #[pyclass(name = "Generic", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct Generic; + + #[pyclass(flags(BASETYPE, HEAPTYPE))] + impl Generic { + #[pyattr] + fn __slots__(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } - #[pyclassmethod] - fn __init_subclass__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - call_typing_args_kwargs("_generic_init_subclass", cls, args, vm) - } -} + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + call_typing_args_kwargs("_generic_class_getitem", cls, args, vm) + } -/// Sets the default value for a type parameter, equivalent to CPython's _Py_set_typeparam_default -/// This is used by the CALL_INTRINSIC_2 SetTypeparamDefault instruction -pub fn set_typeparam_default( - type_param: PyObjectRef, - evaluate_default: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - // Inner function to handle common pattern of setting evaluate_default - fn try_set_default<T>( - obj: &PyObjectRef, - evaluate_default: &PyObjectRef, - get_field: impl FnOnce(&T) -> &PyMutex<PyObjectRef>, - ) -> bool - where - T: PyPayload, - { - if let Some(typed_obj) = obj.downcast_ref::<T>() { - *get_field(typed_obj).lock() = evaluate_default.clone(); - true - } else { - false + #[pyclassmethod] + fn __init_subclass__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + call_typing_args_kwargs("_generic_init_subclass", cls, args, vm) } } - // Try each type parameter type - if try_set_default::<TypeVar>(&type_param, &evaluate_default, |tv| &tv.evaluate_default) - || try_set_default::<ParamSpec>(&type_param, &evaluate_default, |ps| &ps.evaluate_default) - || try_set_default::<TypeVarTuple>(&type_param, &evaluate_default, |tvt| { - &tvt.evaluate_default - }) - { - Ok(type_param) - } else { - Err(vm.new_type_error(format!( - "Expected a type param, got {}", - type_param.class().name() - ))) + /// Sets the default value for a type parameter, equivalent to CPython's _Py_set_typeparam_default + /// This is used by the CALL_INTRINSIC_2 SetTypeparamDefault instruction + pub fn set_typeparam_default( + type_param: PyObjectRef, + evaluate_default: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Inner function to handle common pattern of setting evaluate_default + fn try_set_default<T>( + obj: &PyObject, + evaluate_default: &PyObject, + get_field: impl FnOnce(&T) -> &PyMutex<PyObjectRef>, + ) -> bool + where + T: PyPayload, + { + if let Some(typed_obj) = obj.downcast_ref::<T>() { + *get_field(typed_obj).lock() = evaluate_default.to_owned(); + true + } else { + false + } + } + + // Try each type parameter type + if try_set_default::<TypeVar>(&type_param, &evaluate_default, |tv| &tv.evaluate_default) + || try_set_default::<ParamSpec>(&type_param, &evaluate_default, |ps| { + &ps.evaluate_default + }) + || try_set_default::<TypeVarTuple>(&type_param, &evaluate_default, |tvt| { + &tvt.evaluate_default + }) + { + Ok(type_param) + } else { + Err(vm.new_type_error(format!( + "Expected a type param, got {}", + type_param.class().name() + ))) + } } } diff --git a/crates/vm/src/stdlib/typing.rs b/crates/vm/src/stdlib/typing.rs index afa8bd6eb90..bafc02e5764 100644 --- a/crates/vm/src/stdlib/typing.rs +++ b/crates/vm/src/stdlib/typing.rs @@ -1,51 +1,42 @@ -// spell-checker:ignore typevarobject funcobj -use crate::{PyPayload, PyRef, VirtualMachine, class::PyClassImpl, stdlib::PyModule}; +// spell-checker:ignore typevarobject funcobj typevartuples +use crate::{ + Context, PyResult, VirtualMachine, builtins::pystr::AsPyStr, class::PyClassImpl, + function::IntoFuncArgs, +}; pub use crate::stdlib::typevar::{ Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple, set_typeparam_default, }; +pub(crate) use decl::module_def; pub use decl::*; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = decl::make_module(vm); - TypeVar::make_class(&vm.ctx); - ParamSpec::make_class(&vm.ctx); - TypeVarTuple::make_class(&vm.ctx); - ParamSpecArgs::make_class(&vm.ctx); - ParamSpecKwargs::make_class(&vm.ctx); - Generic::make_class(&vm.ctx); - extend_module!(vm, &module, { - "NoDefault" => vm.ctx.typing_no_default.clone(), - "TypeVar" => TypeVar::class(&vm.ctx).to_owned(), - "ParamSpec" => ParamSpec::class(&vm.ctx).to_owned(), - "TypeVarTuple" => TypeVarTuple::class(&vm.ctx).to_owned(), - "ParamSpecArgs" => ParamSpecArgs::class(&vm.ctx).to_owned(), - "ParamSpecKwargs" => ParamSpecKwargs::class(&vm.ctx).to_owned(), - "Generic" => Generic::class(&vm.ctx).to_owned(), - }); - module +/// Initialize typing types (call extend_class) +pub fn init(ctx: &Context) { + NoDefault::extend_class(ctx, ctx.types.typing_no_default_type); +} + +pub fn call_typing_func_object<'a>( + vm: &VirtualMachine, + func_name: impl AsPyStr<'a>, + args: impl IntoFuncArgs, +) -> PyResult { + let module = vm.import("typing", 0)?; + let func = module.get_attr(func_name.as_pystr(&vm.ctx), vm)?; + func.call(args, vm) } -#[pymodule(name = "_typing")] +#[pymodule(name = "_typing", with(super::typevar::typevar))] pub(crate) mod decl { + use crate::common::lock::LazyLock; use crate::{ - Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyStrRef, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr}, - function::{FuncArgs, IntoFuncArgs}, - types::{Constructor, Representable}, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, + builtins::{PyGenericAlias, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, type_}, + function::FuncArgs, + protocol::{PyMappingMethods, PyNumberMethods}, + types::{AsMapping, AsNumber, Callable, Constructor, Iterable, Representable}, }; - pub(crate) fn _call_typing_func_object<'a>( - vm: &VirtualMachine, - func_name: impl AsPyStr<'a>, - args: impl IntoFuncArgs, - ) -> PyResult { - let module = vm.import("typing", 0)?; - let func = module.get_attr(func_name.as_pystr(&vm.ctx), vm)?; - func.call(args, vm) - } - #[pyfunction] pub(crate) fn _idfunc(args: FuncArgs, _vm: &VirtualMachine) -> PyObjectRef { args.args[0].clone() @@ -65,7 +56,7 @@ pub(crate) mod decl { #[derive(Debug, PyPayload)] pub struct NoDefault; - #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + #[pyclass(with(Constructor, Representable), flags(IMMUTABLETYPE))] impl NoDefault { #[pymethod] fn __reduce__(&self, _vm: &VirtualMachine) -> String { @@ -94,23 +85,147 @@ pub(crate) mod decl { } #[pyattr] - #[pyclass(name)] + #[pyclass(name = "_ConstEvaluator", module = "_typing")] + #[derive(Debug, PyPayload)] + pub(crate) struct ConstEvaluator { + value: PyObjectRef, + } + + #[pyclass(with(Constructor, Callable, Representable), flags(IMMUTABLETYPE))] + impl ConstEvaluator {} + + impl Constructor for ConstEvaluator { + type Args = FuncArgs; + + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_typing._ConstEvaluator' instances".to_owned())) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("ConstEvaluator cannot be instantiated from Python") + } + } + + /// annotationlib.Format.STRING = 4 + const ANNOTATE_FORMAT_STRING: i32 = 4; + + impl Callable for ConstEvaluator { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (format,): (i32,) = args.bind(vm)?; + let value = &zelf.value; + if format == ANNOTATE_FORMAT_STRING { + return typing_type_repr_value(value, vm); + } + Ok(value.clone()) + } + } + + /// String representation of a type for annotation purposes. + /// Equivalent of _Py_typing_type_repr. + fn typing_type_repr(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + // Ellipsis + if obj.is(&vm.ctx.ellipsis) { + return Ok("...".to_owned()); + } + // NoneType -> "None" + if obj.is(&vm.ctx.types.none_type.as_object()) { + return Ok("None".to_owned()); + } + // Generic aliases (has __origin__ and __args__) -> repr + let has_origin = obj.get_attr("__origin__", vm).is_ok(); + let has_args = obj.get_attr("__args__", vm).is_ok(); + if has_origin && has_args { + return Ok(obj.repr(vm)?.to_string()); + } + // Has __qualname__ and __module__ + if let Ok(qualname) = obj.get_attr("__qualname__", vm) + && let Ok(module) = obj.get_attr("__module__", vm) + && !vm.is_none(&module) + && let Some(module_str) = module.downcast_ref::<crate::builtins::PyStr>() + { + if module_str.as_str() == "builtins" { + return Ok(qualname.str(vm)?.to_string()); + } + return Ok(format!("{}.{}", module_str.as_str(), qualname.str(vm)?)); + } + // Fallback to repr + Ok(obj.repr(vm)?.to_string()) + } + + /// Format a value as a string for ANNOTATE_FORMAT_STRING. + /// Handles tuples specially by wrapping in parentheses. + fn typing_type_repr_value(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(tuple) = value.try_to_ref::<PyTuple>(vm) { + let mut parts = Vec::with_capacity(tuple.len()); + for item in tuple.iter() { + parts.push(typing_type_repr(item, vm)?); + } + let inner = if parts.len() == 1 { + format!("{},", parts[0]) + } else { + parts.join(", ") + }; + Ok(vm.ctx.new_str(format!("({})", inner)).into()) + } else { + Ok(vm.ctx.new_str(typing_type_repr(value, vm)?).into()) + } + } + + impl Representable for ConstEvaluator { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let value_repr = zelf.value.repr(vm)?; + Ok(format!("<constevaluator {}>", value_repr)) + } + } + + pub(crate) fn const_evaluator_alloc(value: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + ConstEvaluator { value }.into_ref(&vm.ctx).into() + } + + #[pyattr] + #[pyclass(name, module = "typing")] #[derive(Debug, PyPayload)] - #[allow(dead_code)] pub(crate) struct TypeAliasType { name: PyStrRef, type_params: PyTupleRef, - value: PyObjectRef, - // compute_value: PyObjectRef, - // module: PyObjectRef, + compute_value: PyObjectRef, + cached_value: crate::common::lock::PyMutex<Option<PyObjectRef>>, + module: Option<PyObjectRef>, + is_lazy: bool, } - #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + #[pyclass( + with(Constructor, Representable, AsMapping, AsNumber, Iterable), + flags(IMMUTABLETYPE) + )] impl TypeAliasType { - pub const fn new(name: PyStrRef, type_params: PyTupleRef, value: PyObjectRef) -> Self { + /// Create from intrinsic: compute_value is a callable that returns the value + pub fn new(name: PyStrRef, type_params: PyTupleRef, compute_value: PyObjectRef) -> Self { + Self { + name, + type_params, + compute_value, + cached_value: crate::common::lock::PyMutex::new(None), + module: None, + is_lazy: true, + } + } + + /// Create with an eagerly evaluated value (used by constructor) + fn new_eager( + name: PyStrRef, + type_params: PyTupleRef, + value: PyObjectRef, + module: Option<PyObjectRef>, + ) -> Self { Self { name, type_params, - value, + compute_value: value.clone(), + cached_value: crate::common::lock::PyMutex::new(Some(value)), + module, + is_lazy: false, } } @@ -120,55 +235,193 @@ pub(crate) mod decl { } #[pygetset] - fn __value__(&self) -> PyObjectRef { - self.value.clone() + fn __value__(&self, vm: &VirtualMachine) -> PyResult { + let cached = self.cached_value.lock().clone(); + if let Some(value) = cached { + return Ok(value); + } + // Call evaluator with format=1 (FORMAT_VALUE) + let value = self.compute_value.call((1i32,), vm)?; + *self.cached_value.lock() = Some(value.clone()); + Ok(value) } #[pygetset] fn __type_params__(&self) -> PyTupleRef { self.type_params.clone() } + + #[pygetset] + fn __parameters__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // TypeVarTuples must be unpacked in __parameters__ + unpack_typevartuples(&self.type_params, vm).map(|t| t.into()) + } + + #[pygetset] + fn __module__(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(ref module) = self.module { + return module.clone(); + } + // Fall back to compute_value's __module__ (like PyFunction_GetModule) + if let Ok(module) = self.compute_value.get_attr("__module__", vm) { + return module; + } + vm.ctx.none() + } + + fn __getitem__(zelf: PyRef<Self>, args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.type_params.is_empty() { + return Err( + vm.new_type_error("Only generic type aliases are subscriptable".to_owned()) + ); + } + let args_tuple = if let Ok(tuple) = args.try_to_ref::<PyTuple>(vm) { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![args], &vm.ctx) + }; + let origin: PyObjectRef = zelf.as_object().to_owned(); + Ok(PyGenericAlias::new(origin, args_tuple, false, vm).into_pyobject(vm)) + } + + #[pymethod] + fn __reduce__(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyObjectRef { + zelf.name.clone().into() + } + + #[pymethod] + fn __typing_unpacked_tuple_args__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + + #[pygetset] + fn evaluate_value(&self, vm: &VirtualMachine) -> PyResult { + if self.is_lazy { + return Ok(self.compute_value.clone()); + } + Ok(const_evaluator_alloc(self.compute_value.clone(), vm)) + } + + /// Check type_params ordering: non-default params must precede default params. + /// Uses __default__ attribute to check if a type param has a default value, + /// comparing against typing.NoDefault sentinel (like get_type_param_default). + fn check_type_params( + type_params: &PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyTupleRef>> { + if type_params.is_empty() { + return Ok(None); + } + let no_default = &vm.ctx.typing_no_default; + let mut default_seen = false; + for param in type_params.iter() { + let dflt = param.get_attr("__default__", vm).map_err(|_| { + vm.new_type_error(format!( + "Expected a type param, got {}", + param + .repr(vm) + .map(|s| s.to_string()) + .unwrap_or_else(|_| "?".to_owned()) + )) + })?; + let is_no_default = dflt.is(no_default); + if is_no_default { + if default_seen { + return Err(vm.new_type_error(format!( + "non-default type parameter '{}' follows default type parameter", + param.repr(vm)? + ))); + } + } else { + default_seen = true; + } + } + Ok(Some(type_params.clone())) + } } impl Constructor for TypeAliasType { type Args = FuncArgs; fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { - // TypeAliasType(name, value, *, type_params=None) - if args.args.len() < 2 { - return Err(vm.new_type_error(format!( - "TypeAliasType() missing {} required positional argument{}: {}", - 2 - args.args.len(), - if 2 - args.args.len() == 1 { "" } else { "s" }, - if args.args.is_empty() { - "'name' and 'value'" - } else { - "'value'" - } - ))); + // typealias(name, value, *, type_params=()) + // name and value are positional-or-keyword; type_params is keyword-only. + + // Reject unexpected keyword arguments + for key in args.kwargs.keys() { + if key != "name" && key != "value" && key != "type_params" { + return Err(vm.new_type_error(format!( + "typealias() got an unexpected keyword argument '{key}'" + ))); + } } + + // Reject too many positional arguments if args.args.len() > 2 { return Err(vm.new_type_error(format!( - "TypeAliasType() takes 2 positional arguments but {} were given", + "typealias() takes exactly 2 positional arguments ({} given)", args.args.len() ))); } - let name = args.args[0] - .clone() - .downcast::<crate::builtins::PyStr>() - .map_err(|_| vm.new_type_error("TypeAliasType name must be a string".to_owned()))?; - let value = args.args[1].clone(); + // Resolve name: positional[0] or kwarg + let name = if !args.args.is_empty() { + if args.kwargs.contains_key("name") { + return Err(vm.new_type_error( + "argument for typealias() given by name ('name') and position (1)" + .to_owned(), + )); + } + args.args[0].clone() + } else { + args.kwargs.get("name").cloned().ok_or_else(|| { + vm.new_type_error( + "typealias() missing required argument 'name' (pos 1)".to_owned(), + ) + })? + }; + + // Resolve value: positional[1] or kwarg + let value = if args.args.len() >= 2 { + if args.kwargs.contains_key("value") { + return Err(vm.new_type_error( + "argument for typealias() given by name ('value') and position (2)" + .to_owned(), + )); + } + args.args[1].clone() + } else { + args.kwargs.get("value").cloned().ok_or_else(|| { + vm.new_type_error( + "typealias() missing required argument 'value' (pos 2)".to_owned(), + ) + })? + }; + + let name = name.downcast::<crate::builtins::PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "typealias() argument 'name' must be str, not {}", + obj.class().name() + )) + })?; let type_params = if let Some(tp) = args.kwargs.get("type_params") { - tp.clone() + let tp = tp + .clone() .downcast::<crate::builtins::PyTuple>() - .map_err(|_| vm.new_type_error("type_params must be a tuple".to_owned()))? + .map_err(|_| vm.new_type_error("type_params must be a tuple".to_owned()))?; + Self::check_type_params(&tp, vm)?; + tp } else { vm.ctx.empty_tuple.clone() }; - Ok(Self::new(name, type_params, value)) + // Get caller's module name from frame globals, like typevar.rs caller() + let module = vm + .current_frame() + .and_then(|f| f.globals.get_item("__name__", vm).ok()); + + Ok(Self::new_eager(name, type_params, value, module)) } } @@ -178,15 +431,78 @@ pub(crate) mod decl { } } - // impl AsMapping for Generic { - // fn as_mapping() -> &'static PyMappingMethods { - // static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - // subscript: atomic_func!(|mapping, needle, vm| { - // call_typing_func_object(vm, "_GenericAlias", (mapping.obj, needle)) - // }), - // ..PyMappingMethods::NOT_IMPLEMENTED - // }); - // &AS_MAPPING - // } - // } + impl AsMapping for TypeAliasType { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = TypeAliasType::mapping_downcast(mapping); + TypeAliasType::__getitem__(zelf.to_owned(), needle.to_owned(), vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } + } + + impl AsNumber for TypeAliasType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| type_::or_(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl Iterable for TypeAliasType { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + // Import typing.Unpack and return iter((Unpack[self],)) + let typing = vm.import("typing", 0)?; + let unpack = typing.get_attr("Unpack", vm)?; + let zelf_obj: PyObjectRef = zelf.into(); + let unpacked = vm.call_method(&unpack, "__getitem__", (zelf_obj,))?; + let tuple = PyTuple::new_ref(vec![unpacked], &vm.ctx); + Ok(tuple.as_object().get_iter(vm)?.into()) + } + } + + /// Wrap TypeVarTuples in Unpack[], matching unpack_typevartuples() + pub(crate) fn unpack_typevartuples( + type_params: &PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + let has_tvt = type_params + .iter() + .any(|p| p.downcastable::<crate::stdlib::typevar::TypeVarTuple>()); + if !has_tvt { + return Ok(type_params.clone()); + } + let typing = vm.import("typing", 0)?; + let unpack_cls = typing.get_attr("Unpack", vm)?; + let new_params: Vec<PyObjectRef> = type_params + .iter() + .map(|p| { + if p.downcastable::<crate::stdlib::typevar::TypeVarTuple>() { + vm.call_method(&unpack_cls, "__getitem__", (p.clone(),)) + } else { + Ok(p.clone()) + } + }) + .collect::<PyResult<_>>()?; + Ok(PyTuple::new_ref(new_params, &vm.ctx)) + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + extend_module!(vm, module, { + "NoDefault" => vm.ctx.typing_no_default.clone(), + "Union" => vm.ctx.types.union_type.to_owned(), + }); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/warnings.rs b/crates/vm/src/stdlib/warnings.rs index 2d61c3b571f..9b846725347 100644 --- a/crates/vm/src/stdlib/warnings.rs +++ b/crates/vm/src/stdlib/warnings.rs @@ -1,4 +1,4 @@ -pub(crate) use _warnings::make_module; +pub(crate) use _warnings::module_def; use crate::{Py, PyResult, VirtualMachine, builtins::PyType}; @@ -8,41 +8,207 @@ pub fn warn( stack_level: usize, vm: &VirtualMachine, ) -> PyResult<()> { - // TODO: use rust warnings module - if let Ok(module) = vm.import("warnings", 0) - && let Ok(func) = module.get_attr("warn", vm) - { - let _ = func.call((message, category.to_owned(), stack_level), vm); - } - Ok(()) + crate::warn::warn( + vm.new_pyobj(message), + Some(category.to_owned()), + isize::try_from(stack_level).unwrap_or(isize::MAX), + None, + vm, + ) } #[pymodule] mod _warnings { use crate::{ - PyResult, VirtualMachine, - builtins::{PyStrRef, PyTypeRef}, + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, + convert::TryFromObject, function::OptionalArg, }; + #[pyattr] + fn filters(vm: &VirtualMachine) -> PyListRef { + vm.state.warnings.filters.clone() + } + + #[pyattr] + fn _defaultaction(vm: &VirtualMachine) -> PyStrRef { + vm.state.warnings.default_action.clone() + } + + #[pyattr] + fn _onceregistry(vm: &VirtualMachine) -> PyDictRef { + vm.state.warnings.once_registry.clone() + } + + #[pyattr] + fn _warnings_context(vm: &VirtualMachine) -> PyObjectRef { + vm.state + .warnings + .context_var + .get_or_init(|| { + // Try to create a real ContextVar if _contextvars is available. + // During early startup it may not be importable yet, in which + // case we fall back to None. This is safe because + // context_aware_warnings defaults to False. + if let Ok(contextvars) = vm.import("_contextvars", 0) + && let Ok(cv_cls) = contextvars.get_attr("ContextVar", vm) + && let Ok(cv) = cv_cls.call(("_warnings_context",), vm) + { + cv + } else { + vm.ctx.none() + } + }) + .clone() + } + + #[pyfunction] + fn _acquire_lock(vm: &VirtualMachine) { + vm.state.warnings.acquire_lock(); + } + + #[pyfunction] + fn _release_lock(vm: &VirtualMachine) -> PyResult<()> { + if !vm.state.warnings.release_lock() { + return Err(vm.new_runtime_error("cannot release un-acquired lock".to_owned())); + } + Ok(()) + } + + #[pyfunction] + fn _filters_mutated_lock_held(vm: &VirtualMachine) { + vm.state.warnings.filters_mutated(); + } + #[derive(FromArgs)] struct WarnArgs { #[pyarg(positional)] - message: PyStrRef, + message: PyObjectRef, #[pyarg(any, optional)] - category: OptionalArg<PyTypeRef>, + category: OptionalArg<PyObjectRef>, #[pyarg(any, optional)] - stacklevel: OptionalArg<u32>, + stacklevel: OptionalArg<i32>, + #[pyarg(named, optional)] + source: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + skip_file_prefixes: OptionalArg<PyTupleRef>, + } + + /// Validate and resolve the category argument, matching get_category() in C. + fn get_category( + message: &PyObjectRef, + category: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyTypeRef>> { + let cat_obj = match category { + Some(c) if !vm.is_none(&c) => c, + _ => { + if message.fast_isinstance(vm.ctx.exceptions.warning) { + return Ok(Some(message.class().to_owned())); + } else { + return Ok(None); // will default to UserWarning in warn_explicit + } + } + }; + + let cat = PyTypeRef::try_from_object(vm, cat_obj.clone()).map_err(|_| { + vm.new_type_error(format!( + "category must be a Warning subclass, not '{}'", + cat_obj.class().name() + )) + })?; + + if !cat.fast_issubclass(vm.ctx.exceptions.warning) { + return Err(vm.new_type_error(format!( + "category must be a Warning subclass, not '{}'", + cat.class().name() + ))); + } + + Ok(Some(cat)) } #[pyfunction] fn warn(args: WarnArgs, vm: &VirtualMachine) -> PyResult<()> { - let level = args.stacklevel.unwrap_or(1); - crate::warn::warn( + let level = args.stacklevel.unwrap_or(1) as isize; + + let category = get_category(&args.message, args.category.into_option(), vm)?; + + // Validate skip_file_prefixes: each element must be a str + let skip_prefixes = args.skip_file_prefixes.into_option(); + if let Some(ref prefixes) = skip_prefixes { + for item in prefixes.iter() { + if !item.class().is(vm.ctx.types.str_type) { + return Err( + vm.new_type_error("skip_file_prefixes must be a tuple of strs".to_owned()) + ); + } + } + } + + crate::warn::warn_with_skip( + args.message, + category, + level, + args.source.into_option(), + skip_prefixes, + vm, + ) + } + + #[derive(FromArgs)] + struct WarnExplicitArgs { + #[pyarg(positional)] + message: PyObjectRef, + #[pyarg(positional)] + category: PyObjectRef, + #[pyarg(positional)] + filename: PyStrRef, + #[pyarg(positional)] + lineno: usize, + #[pyarg(any, optional)] + module: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + registry: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + module_globals: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + source: OptionalArg<PyObjectRef>, + } + + #[pyfunction] + fn warn_explicit(args: WarnExplicitArgs, vm: &VirtualMachine) -> PyResult<()> { + let registry = args.registry.into_option().unwrap_or_else(|| vm.ctx.none()); + + let module = args.module.into_option(); + + // Validate module_globals: must be None or a dict + if let Some(ref mg) = args.module_globals.into_option() + && !vm.is_none(mg) + && !mg.class().is(vm.ctx.types.dict_type) + { + return Err(vm.new_type_error("module_globals must be a dict".to_owned())); + } + + let category = + if vm.is_none(&args.category) { + None + } else { + Some(PyTypeRef::try_from_object(vm, args.category).map_err(|_| { + vm.new_type_error("category must be a Warning subclass".to_owned()) + })?) + }; + + crate::warn::warn_explicit( + category, args.message, - args.category.into_option(), - level as isize, - None, + args.filename, + args.lineno, + module, + registry, + None, // source_line + args.source.into_option(), vm, ) } diff --git a/crates/vm/src/stdlib/weakref.rs b/crates/vm/src/stdlib/weakref.rs index bedfad9abbd..e7e030b2b01 100644 --- a/crates/vm/src/stdlib/weakref.rs +++ b/crates/vm/src/stdlib/weakref.rs @@ -4,7 +4,7 @@ //! - [python weakref module](https://docs.python.org/3/library/weakref.html) //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -pub(crate) use _weakref::make_module; +pub(crate) use _weakref::module_def; #[pymodule] mod _weakref { diff --git a/crates/vm/src/stdlib/winapi.rs b/crates/vm/src/stdlib/winapi.rs index 1df3ff0e42d..c176568cfc5 100644 --- a/crates/vm/src/stdlib/winapi.rs +++ b/crates/vm/src/stdlib/winapi.rs @@ -1,20 +1,21 @@ // spell-checker:disable #![allow(non_snake_case)] -pub(crate) use _winapi::make_module; +pub(crate) use _winapi::module_def; #[pymodule] mod _winapi { use crate::{ - PyObjectRef, PyResult, TryFromObject, VirtualMachine, + Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::PyStrRef, - common::windows::ToWideString, + common::{lock::PyMutex, windows::ToWideString}, convert::{ToPyException, ToPyResult}, function::{ArgMapping, ArgSequence, OptionalArg}, + types::Constructor, windows::{WinHandle, WindowsSysResult}, }; - use std::ptr::{null, null_mut}; - use windows_sys::Win32::Foundation::{INVALID_HANDLE_VALUE, MAX_PATH}; + use core::ptr::{null, null_mut}; + use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE, MAX_PATH}; #[pyattr] use windows_sys::Win32::{ @@ -31,19 +32,57 @@ mod _winapi { LCMAP_TRADITIONAL_CHINESE, LCMAP_UPPERCASE, }, Storage::FileSystem::{ - COPY_FILE_ALLOW_DECRYPTED_DESTINATION, COPY_FILE_COPY_SYMLINK, - COPY_FILE_FAIL_IF_EXISTS, COPY_FILE_NO_BUFFERING, COPY_FILE_NO_OFFLOAD, - COPY_FILE_OPEN_SOURCE_FOR_WRITE, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, - COPY_FILE_REQUEST_SECURITY_PRIVILEGES, COPY_FILE_RESTARTABLE, - COPY_FILE_RESUME_FROM_PAUSE, COPYFILE2_CALLBACK_CHUNK_FINISHED, - COPYFILE2_CALLBACK_CHUNK_STARTED, COPYFILE2_CALLBACK_ERROR, - COPYFILE2_CALLBACK_POLL_CONTINUE, COPYFILE2_CALLBACK_STREAM_FINISHED, - COPYFILE2_CALLBACK_STREAM_STARTED, COPYFILE2_PROGRESS_CANCEL, - COPYFILE2_PROGRESS_CONTINUE, COPYFILE2_PROGRESS_PAUSE, COPYFILE2_PROGRESS_QUIET, - COPYFILE2_PROGRESS_STOP, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, - FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, - FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, - PIPE_ACCESS_INBOUND, SYNCHRONIZE, + COPY_FILE_ALLOW_DECRYPTED_DESTINATION, + COPY_FILE_COPY_SYMLINK, + COPY_FILE_FAIL_IF_EXISTS, + COPY_FILE_NO_BUFFERING, + COPY_FILE_NO_OFFLOAD, + COPY_FILE_OPEN_SOURCE_FOR_WRITE, + COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, + COPY_FILE_REQUEST_SECURITY_PRIVILEGES, + COPY_FILE_RESTARTABLE, + COPY_FILE_RESUME_FROM_PAUSE, + COPYFILE2_CALLBACK_CHUNK_FINISHED, + COPYFILE2_CALLBACK_CHUNK_STARTED, + COPYFILE2_CALLBACK_ERROR, + COPYFILE2_CALLBACK_POLL_CONTINUE, + COPYFILE2_CALLBACK_STREAM_FINISHED, + COPYFILE2_CALLBACK_STREAM_STARTED, + COPYFILE2_PROGRESS_CANCEL, + COPYFILE2_PROGRESS_CONTINUE, + COPYFILE2_PROGRESS_PAUSE, + COPYFILE2_PROGRESS_QUIET, + COPYFILE2_PROGRESS_STOP, + CREATE_ALWAYS, + // CreateFile constants + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_DELETE_ON_CLOSE, + FILE_FLAG_FIRST_PIPE_INSTANCE, + FILE_FLAG_NO_BUFFERING, + FILE_FLAG_OPEN_REPARSE_POINT, + FILE_FLAG_OVERLAPPED, + FILE_FLAG_POSIX_SEMANTICS, + FILE_FLAG_RANDOM_ACCESS, + FILE_FLAG_SEQUENTIAL_SCAN, + FILE_FLAG_WRITE_THROUGH, + FILE_GENERIC_READ, + FILE_GENERIC_WRITE, + FILE_SHARE_DELETE, + FILE_SHARE_READ, + FILE_SHARE_WRITE, + FILE_TYPE_CHAR, + FILE_TYPE_DISK, + FILE_TYPE_PIPE, + FILE_TYPE_REMOTE, + FILE_TYPE_UNKNOWN, + OPEN_ALWAYS, + OPEN_EXISTING, + PIPE_ACCESS_DUPLEX, + PIPE_ACCESS_INBOUND, + SYNCHRONIZE, + TRUNCATE_EXISTING, }, System::{ Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, @@ -55,6 +94,7 @@ mod _winapi { SEC_LARGE_PAGES, SEC_NOCACHE, SEC_RESERVE, SEC_WRITECOMBINE, }, Pipes::{ + NMPWAIT_NOWAIT, NMPWAIT_USE_DEFAULT_WAIT, NMPWAIT_WAIT_FOREVER, PIPE_READMODE_MESSAGE, PIPE_TYPE_MESSAGE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, }, SystemServices::LOCALE_NAME_MAX_LENGTH, @@ -63,7 +103,8 @@ mod _winapi { CREATE_BREAKAWAY_FROM_JOB, CREATE_DEFAULT_ERROR_MODE, CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, DETACHED_PROCESS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - REALTIME_PRIORITY_CLASS, STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, + REALTIME_PRIORITY_CLASS, STARTF_FORCEOFFFEEDBACK, STARTF_FORCEONFEEDBACK, + STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, }, }, UI::WindowsAndMessaging::SW_HIDE, @@ -77,6 +118,45 @@ mod _winapi { WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) } + /// CreateFile - Create or open a file or I/O device. + #[pyfunction] + #[allow( + clippy::too_many_arguments, + reason = "matches Win32 CreateFile parameter structure" + )] + fn CreateFile( + file_name: PyStrRef, + desired_access: u32, + share_mode: u32, + _security_attributes: PyObjectRef, // Always NULL (0) + creation_disposition: u32, + flags_and_attributes: u32, + _template_file: PyObjectRef, // Always NULL (0) + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::Storage::FileSystem::CreateFileW; + + let file_name_wide = file_name.as_wtf8().to_wide_with_nul(); + + let handle = unsafe { + CreateFileW( + file_name_wide.as_ptr(), + desired_access, + share_mode, + null(), + creation_disposition, + flags_and_attributes, + null_mut(), + ) + }; + + if handle == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + Ok(WinHandle(handle)) + } + #[pyfunction] fn GetStdHandle( std_handle: windows_sys::Win32::System::Console::STD_HANDLE, @@ -102,12 +182,12 @@ mod _winapi { ) -> PyResult<(WinHandle, WinHandle)> { use windows_sys::Win32::Foundation::HANDLE; let (read, write) = unsafe { - let mut read = std::mem::MaybeUninit::<HANDLE>::uninit(); - let mut write = std::mem::MaybeUninit::<HANDLE>::uninit(); + let mut read = core::mem::MaybeUninit::<HANDLE>::uninit(); + let mut write = core::mem::MaybeUninit::<HANDLE>::uninit(); WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( read.as_mut_ptr(), write.as_mut_ptr(), - std::ptr::null(), + core::ptr::null(), size, )) .to_pyresult(vm)?; @@ -128,7 +208,7 @@ mod _winapi { ) -> PyResult<WinHandle> { use windows_sys::Win32::Foundation::HANDLE; let target = unsafe { - let mut target = std::mem::MaybeUninit::<HANDLE>::uninit(); + let mut target = core::mem::MaybeUninit::<HANDLE>::uninit(); WindowsSysResult(windows_sys::Win32::Foundation::DuplicateHandle( src_process.0, src.0, @@ -205,8 +285,8 @@ mod _winapi { vm: &VirtualMachine, ) -> PyResult<(WinHandle, WinHandle, u32, u32)> { let mut si: windows_sys::Win32::System::Threading::STARTUPINFOEXW = - unsafe { std::mem::zeroed() }; - si.StartupInfo.cb = std::mem::size_of_val(&si) as _; + unsafe { core::mem::zeroed() }; + si.StartupInfo.cb = core::mem::size_of_val(&si) as _; macro_rules! si_attr { ($attr:ident, $t:ty) => {{ @@ -274,12 +354,12 @@ mod _winapi { .map_or_else(null_mut, |w| w.as_mut_ptr()); let procinfo = unsafe { - let mut procinfo = std::mem::MaybeUninit::uninit(); + let mut procinfo = core::mem::MaybeUninit::uninit(); WindowsSysResult(windows_sys::Win32::System::Threading::CreateProcessW( app_name, command_line, - std::ptr::null(), - std::ptr::null(), + core::ptr::null(), + core::ptr::null(), args.inherit_handles, args.creation_flags | windows_sys::Win32::System::Threading::EXTENDED_STARTUPINFO_PRESENT @@ -360,7 +440,9 @@ mod _winapi { return Err(vm.new_runtime_error("environment changed size during iteration")); } - let mut out = widestring::WideString::new(); + // Deduplicate case-insensitive keys, keeping the last value + use std::collections::HashMap; + let mut last_entry: HashMap<String, widestring::WideString> = HashMap::new(); for (k, v) in keys.into_iter().zip(values.into_iter()) { let k = PyStrRef::try_from_object(vm, k)?; let k = k.as_str(); @@ -372,10 +454,22 @@ mod _winapi { if k.is_empty() || k[1..].contains('=') { return Err(vm.new_value_error("illegal environment variable name")); } - out.push_str(k); - out.push_str("="); - out.push_str(v); - out.push_str("\0"); + let key_upper = k.to_uppercase(); + let mut entry = widestring::WideString::new(); + entry.push_str(k); + entry.push_str("="); + entry.push_str(v); + entry.push_str("\0"); + last_entry.insert(key_upper, entry); + } + + // Sort by uppercase key for case-insensitive ordering + let mut entries: Vec<(String, widestring::WideString)> = last_entry.into_iter().collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut out = widestring::WideString::new(); + for (_, entry) in entries { + out.push(entry); } out.push_str("\0"); Ok(out.into_vec()) @@ -414,10 +508,10 @@ mod _winapi { let attr_count = handlelist.is_some() as u32; let (result, mut size) = unsafe { - let mut size = std::mem::MaybeUninit::uninit(); + let mut size = core::mem::MaybeUninit::uninit(); let result = WindowsSysResult( windows_sys::Win32::System::Threading::InitializeProcThreadAttributeList( - std::ptr::null_mut(), + core::ptr::null_mut(), attr_count, 0, size.as_mut_ptr(), @@ -452,9 +546,9 @@ mod _winapi { 0, (2 & 0xffff) | 0x20000, // PROC_THREAD_ATTRIBUTE_HANDLE_LIST handlelist.as_mut_ptr() as _, - (handlelist.len() * std::mem::size_of::<usize>()) as _, - std::ptr::null_mut(), - std::ptr::null(), + (handlelist.len() * core::mem::size_of::<usize>()) as _, + core::ptr::null_mut(), + core::ptr::null(), ) }) .into_pyresult(vm)?; @@ -465,7 +559,15 @@ mod _winapi { } #[pyfunction] - fn WaitForSingleObject(h: WinHandle, ms: u32, vm: &VirtualMachine) -> PyResult<u32> { + fn WaitForSingleObject(h: WinHandle, ms: i64, vm: &VirtualMachine) -> PyResult<u32> { + // Negative values (e.g., -1) map to INFINITE (0xFFFFFFFF) + let ms = if ms < 0 { + windows_sys::Win32::System::Threading::INFINITE + } else if ms > u32::MAX as i64 { + return Err(vm.new_overflow_error("timeout value is too large".to_owned())); + } else { + ms as u32 + }; let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0, ms) }; if ret == windows_sys::Win32::Foundation::WAIT_FAILED { Err(vm.new_last_os_error()) @@ -474,10 +576,52 @@ mod _winapi { } } + #[pyfunction] + fn WaitForMultipleObjects( + handle_seq: ArgSequence<isize>, + wait_all: bool, + milliseconds: u32, + vm: &VirtualMachine, + ) -> PyResult<u32> { + use windows_sys::Win32::Foundation::WAIT_FAILED; + use windows_sys::Win32::System::Threading::WaitForMultipleObjects as WinWaitForMultipleObjects; + + let handles: Vec<HANDLE> = handle_seq + .into_vec() + .into_iter() + .map(|h| h as HANDLE) + .collect(); + + if handles.is_empty() { + return Err(vm.new_value_error("handle_seq must not be empty".to_owned())); + } + + if handles.len() > 64 { + return Err( + vm.new_value_error("WaitForMultipleObjects supports at most 64 handles".to_owned()) + ); + } + + let ret = unsafe { + WinWaitForMultipleObjects( + handles.len() as u32, + handles.as_ptr(), + if wait_all { 1 } else { 0 }, + milliseconds, + ) + }; + + if ret == WAIT_FAILED { + Err(vm.new_last_os_error()) + } else { + Ok(ret) + } + } + #[pyfunction] fn GetExitCodeProcess(h: WinHandle, vm: &VirtualMachine) -> PyResult<u32> { unsafe { - let mut ec = std::mem::MaybeUninit::uninit(); + let mut ec = core::mem::MaybeUninit::uninit(); WindowsSysResult(windows_sys::Win32::System::Threading::GetExitCodeProcess( h.0, ec.as_mut_ptr(), @@ -572,7 +716,7 @@ mod _winapi { LCMapStringEx as WinLCMapStringEx, }; - // Reject unsupported flags (same as CPython) + // Reject unsupported flags if flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | LCMAP_SORTKEY) != 0 { return Err(vm.new_value_error("unsupported flags")); } @@ -677,4 +821,825 @@ mod _winapi { Ok(WinHandle(handle)) } + + // ==================== Overlapped class ==================== + // Used for asynchronous I/O operations (ConnectNamedPipe, ReadFile, WriteFile) + + #[pyattr] + #[pyclass(name = "Overlapped", module = "_winapi")] + #[derive(Debug, PyPayload)] + struct Overlapped { + inner: PyMutex<OverlappedInner>, + } + + struct OverlappedInner { + overlapped: windows_sys::Win32::System::IO::OVERLAPPED, + handle: HANDLE, + pending: bool, + completed: bool, + read_buffer: Option<Vec<u8>>, + write_buffer: Option<Vec<u8>>, + } + + impl core::fmt::Debug for OverlappedInner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedInner") + .field("handle", &self.handle) + .field("pending", &self.pending) + .field("completed", &self.completed) + .finish() + } + } + + unsafe impl Sync for OverlappedInner {} + unsafe impl Send for OverlappedInner {} + + #[pyclass(with(Constructor))] + impl Overlapped { + fn new_with_handle(handle: HANDLE) -> Self { + use windows_sys::Win32::System::Threading::CreateEventW; + + let event = unsafe { CreateEventW(null(), 1, 0, null()) }; + let mut overlapped: windows_sys::Win32::System::IO::OVERLAPPED = + unsafe { core::mem::zeroed() }; + overlapped.hEvent = event; + + Overlapped { + inner: PyMutex::new(OverlappedInner { + overlapped, + handle, + pending: false, + completed: false, + read_buffer: None, + write_buffer: None, + }), + } + } + + #[pymethod] + fn GetOverlappedResult(&self, wait: bool, vm: &VirtualMachine) -> PyResult<(u32, u32)> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_INCOMPLETE, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED, ERROR_SUCCESS, + GetLastError, + }; + use windows_sys::Win32::System::IO::GetOverlappedResult; + + let mut inner = self.inner.lock(); + + let mut transferred: u32 = 0; + + let ret = unsafe { + GetOverlappedResult( + inner.handle, + &inner.overlapped, + &mut transferred, + if wait { 1 } else { 0 }, + ) + }; + + let err = if ret == 0 { + unsafe { GetLastError() } + } else { + ERROR_SUCCESS + }; + + match err { + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_OPERATION_ABORTED => { + inner.completed = true; + inner.pending = false; + } + ERROR_IO_INCOMPLETE => {} + _ => { + inner.pending = false; + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + if inner.completed + && let Some(read_buffer) = &mut inner.read_buffer + && transferred != read_buffer.len() as u32 + { + read_buffer.truncate(transferred as usize); + } + + Ok((transferred, err)) + } + + #[pymethod] + fn getbuffer(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + let inner = self.inner.lock(); + if !inner.completed { + return Err(vm.new_value_error( + "can't get read buffer before GetOverlappedResult() signals the operation completed" + .to_owned(), + )); + } + Ok(inner + .read_buffer + .as_ref() + .map(|buf| vm.ctx.new_bytes(buf.clone()).into())) + } + + #[pymethod] + fn cancel(&self, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::IO::CancelIoEx; + + let mut inner = self.inner.lock(); + let ret = if inner.pending { + unsafe { CancelIoEx(inner.handle, &inner.overlapped) } + } else { + 1 + }; + if ret == 0 { + let err = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + if err != windows_sys::Win32::Foundation::ERROR_NOT_FOUND { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + inner.pending = false; + Ok(()) + } + + #[pygetset] + fn event(&self) -> isize { + let inner = self.inner.lock(); + inner.overlapped.hEvent as isize + } + } + + impl Constructor for Overlapped { + type Args = (); + + fn py_new( + _cls: &Py<crate::builtins::PyType>, + _args: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Overlapped::new_with_handle(null_mut())) + } + } + + impl Drop for OverlappedInner { + fn drop(&mut self) { + use windows_sys::Win32::Foundation::CloseHandle; + if !self.overlapped.hEvent.is_null() { + unsafe { CloseHandle(self.overlapped.hEvent) }; + } + } + } + + /// ConnectNamedPipe - Wait for a client to connect to a named pipe + #[derive(FromArgs)] + struct ConnectNamedPipeArgs { + #[pyarg(positional)] + handle: WinHandle, + #[pyarg(named, optional)] + overlapped: OptionalArg<bool>, + } + + #[pyfunction] + fn ConnectNamedPipe(args: ConnectNamedPipeArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_PENDING, ERROR_PIPE_CONNECTED, GetLastError, + }; + + let handle = args.handle; + let use_overlapped = args.overlapped.unwrap_or(false); + + if use_overlapped { + // Overlapped (async) mode + let ov = Overlapped::new_with_handle(handle.0); + + let _ret = { + let mut inner = ov.inner.lock(); + unsafe { + windows_sys::Win32::System::Pipes::ConnectNamedPipe( + handle.0, + &mut inner.overlapped, + ) + } + }; + + let err = unsafe { GetLastError() }; + match err { + ERROR_IO_PENDING => { + let mut inner = ov.inner.lock(); + inner.pending = true; + } + ERROR_PIPE_CONNECTED => { + let inner = ov.inner.lock(); + unsafe { + windows_sys::Win32::System::Threading::SetEvent(inner.overlapped.hEvent); + } + } + _ => { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + Ok(ov.into_pyobject(vm)) + } else { + // Synchronous mode + let ret = unsafe { + windows_sys::Win32::System::Pipes::ConnectNamedPipe(handle.0, null_mut()) + }; + + if ret == 0 { + let err = unsafe { GetLastError() }; + if err != ERROR_PIPE_CONNECTED { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + Ok(vm.ctx.none()) + } + } + + /// Helper for GetShortPathName and GetLongPathName + fn get_path_name_impl( + path: &PyStrRef, + api_fn: unsafe extern "system" fn(*const u16, *mut u16, u32) -> u32, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + use rustpython_common::wtf8::Wtf8Buf; + + let path_wide = path.as_wtf8().to_wide_with_nul(); + + // First call to get required buffer size + let size = unsafe { api_fn(path_wide.as_ptr(), null_mut(), 0) }; + + if size == 0 { + return Err(vm.new_last_os_error()); + } + + // Second call to get the actual path + let mut buffer: Vec<u16> = vec![0; size as usize]; + let result = + unsafe { api_fn(path_wide.as_ptr(), buffer.as_mut_ptr(), buffer.len() as u32) }; + + if result == 0 { + return Err(vm.new_last_os_error()); + } + + // Truncate to actual length (excluding null terminator) + buffer.truncate(result as usize); + + // Convert UTF-16 back to WTF-8 (handles surrogates properly) + let result_str = Wtf8Buf::from_wide(&buffer); + Ok(vm.ctx.new_str(result_str)) + } + + /// GetShortPathName - Return the short version of the provided path. + #[pyfunction] + fn GetShortPathName(path: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use windows_sys::Win32::Storage::FileSystem::GetShortPathNameW; + get_path_name_impl(&path, GetShortPathNameW, vm) + } + + /// GetLongPathName - Return the long version of the provided path. + #[pyfunction] + fn GetLongPathName(path: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use windows_sys::Win32::Storage::FileSystem::GetLongPathNameW; + get_path_name_impl(&path, GetLongPathNameW, vm) + } + + /// WaitNamedPipe - Wait for an instance of a named pipe to become available. + #[pyfunction] + fn WaitNamedPipe(name: PyStrRef, timeout: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Pipes::WaitNamedPipeW; + + let name_wide = name.as_wtf8().to_wide_with_nul(); + + let success = unsafe { WaitNamedPipeW(name_wide.as_ptr(), timeout) }; + + if success == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + /// PeekNamedPipe - Peek at data in a named pipe without removing it. + #[pyfunction] + fn PeekNamedPipe( + handle: WinHandle, + size: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use windows_sys::Win32::System::Pipes::PeekNamedPipe as WinPeekNamedPipe; + + let size = size.unwrap_or(0); + + if size < 0 { + return Err(vm.new_value_error("negative size".to_string())); + } + + let mut navail: u32 = 0; + let mut nleft: u32 = 0; + + if size > 0 { + let mut buf = vec![0u8; size as usize]; + let mut nread: u32 = 0; + + let ret = unsafe { + WinPeekNamedPipe( + handle.0, + buf.as_mut_ptr() as *mut _, + size as u32, + &mut nread, + &mut navail, + &mut nleft, + ) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + buf.truncate(nread as usize); + let bytes: PyObjectRef = vm.ctx.new_bytes(buf).into(); + Ok(vm + .ctx + .new_tuple(vec![ + bytes, + vm.ctx.new_int(navail).into(), + vm.ctx.new_int(nleft).into(), + ]) + .into()) + } else { + let ret = unsafe { + WinPeekNamedPipe(handle.0, null_mut(), 0, null_mut(), &mut navail, &mut nleft) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_int(navail).into(), + vm.ctx.new_int(nleft).into(), + ]) + .into()) + } + } + + /// CreateEventW - Create or open a named or unnamed event object. + #[pyfunction] + fn CreateEventW( + security_attributes: isize, // Always NULL (0) + manual_reset: bool, + initial_state: bool, + name: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<WinHandle>> { + use windows_sys::Win32::System::Threading::CreateEventW as WinCreateEventW; + + let _ = security_attributes; // Ignored, always NULL + + let name_wide = name.map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { + WinCreateEventW( + null(), + i32::from(manual_reset), + i32::from(initial_state), + name_ptr, + ) + }; + + if handle == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + if handle.is_null() { + return Ok(None); + } + + Ok(Some(WinHandle(handle))) + } + + /// SetEvent - Set the specified event object to the signaled state. + #[pyfunction] + fn SetEvent(event: WinHandle, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::SetEvent as WinSetEvent; + + let ret = unsafe { WinSetEvent(event.0) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + /// WriteFile - Write data to a file or I/O device. + #[pyfunction] + fn WriteFile( + handle: WinHandle, + buffer: crate::function::ArgBytesLike, + use_overlapped: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use windows_sys::Win32::Storage::FileSystem::WriteFile as WinWriteFile; + + let use_overlapped = use_overlapped.unwrap_or(false); + let buf = buffer.borrow_buf(); + let len = core::cmp::min(buf.len(), u32::MAX as usize) as u32; + + if use_overlapped { + use windows_sys::Win32::Foundation::ERROR_IO_PENDING; + + let ov = Overlapped::new_with_handle(handle.0); + let err = { + let mut inner = ov.inner.lock(); + inner.write_buffer = Some(buf.to_vec()); + let write_buf = inner.write_buffer.as_ref().unwrap(); + let mut written: u32 = 0; + let ret = unsafe { + WinWriteFile( + handle.0, + write_buf.as_ptr() as *const _, + len, + &mut written, + &mut inner.overlapped, + ) + }; + + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + + if ret == 0 && err != ERROR_IO_PENDING { + return Err(vm.new_last_os_error()); + } + if ret == 0 && err == ERROR_IO_PENDING { + inner.pending = true; + } + + err + }; + let result = vm + .ctx + .new_tuple(vec![ov.into_pyobject(vm), vm.ctx.new_int(err).into()]); + return Ok(result.into()); + } + + let mut written: u32 = 0; + let ret = unsafe { + WinWriteFile( + handle.0, + buf.as_ptr() as *const _, + len, + &mut written, + null_mut(), + ) + }; + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_int(written).into(), + vm.ctx.new_int(err).into(), + ]) + .into()) + } + + const MAXIMUM_WAIT_OBJECTS: usize = 64; + + /// BatchedWaitForMultipleObjects - Wait for multiple handles, supporting more than 64. + #[pyfunction] + fn BatchedWaitForMultipleObjects( + handle_seq: PyObjectRef, + wait_all: bool, + milliseconds: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicU32, Ordering}; + use windows_sys::Win32::Foundation::{CloseHandle, WAIT_FAILED, WAIT_OBJECT_0}; + use windows_sys::Win32::System::SystemInformation::GetTickCount64; + use windows_sys::Win32::System::Threading::{ + CreateEventW as WinCreateEventW, CreateThread, GetExitCodeThread, + INFINITE as WIN_INFINITE, ResumeThread, SetEvent as WinSetEvent, TerminateThread, + WaitForMultipleObjects, + }; + + let milliseconds = milliseconds.unwrap_or(WIN_INFINITE); + + // Get handles from sequence + let seq = ArgSequence::<isize>::try_from_object(vm, handle_seq)?; + let handles: Vec<isize> = seq.into_vec(); + let nhandles = handles.len(); + + if nhandles == 0 { + return if wait_all { + Ok(vm.ctx.none()) + } else { + Ok(vm.ctx.new_list(vec![]).into()) + }; + } + + let max_total_objects = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1); + if nhandles > max_total_objects { + return Err(vm.new_value_error(format!( + "need at most {} handles, got a sequence of length {}", + max_total_objects, nhandles + ))); + } + + // Create batches of handles + let batch_size = MAXIMUM_WAIT_OBJECTS - 1; // Leave room for cancel_event + let mut batches: Vec<Vec<isize>> = Vec::new(); + let mut i = 0; + while i < nhandles { + let end = core::cmp::min(i + batch_size, nhandles); + batches.push(handles[i..end].to_vec()); + i = end; + } + + #[cfg(feature = "threading")] + let sigint_event = { + let is_main = crate::stdlib::thread::get_ident() == vm.state.main_thread_ident.load(); + if is_main { + let handle = crate::signal::get_sigint_event().unwrap_or_else(|| { + let handle = unsafe { WinCreateEventW(null(), 1, 0, null()) }; + if !handle.is_null() { + crate::signal::set_sigint_event(handle as isize); + } + handle as isize + }); + if handle == 0 { None } else { Some(handle) } + } else { + None + } + }; + #[cfg(not(feature = "threading"))] + let sigint_event: Option<isize> = None; + + if wait_all { + // For wait_all, we wait sequentially for each batch + let mut err: Option<u32> = None; + let deadline = if milliseconds != WIN_INFINITE { + Some(unsafe { GetTickCount64() } + milliseconds as u64) + } else { + None + }; + + for batch in &batches { + let timeout = if let Some(deadline) = deadline { + let now = unsafe { GetTickCount64() }; + if now >= deadline { + err = Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT); + break; + } + (deadline - now) as u32 + } else { + WIN_INFINITE + }; + + let batch_handles: Vec<_> = batch.iter().map(|&h| h as _).collect(); + let result = unsafe { + WaitForMultipleObjects( + batch_handles.len() as u32, + batch_handles.as_ptr(), + 1, // wait_all = TRUE + timeout, + ) + }; + + if result == WAIT_FAILED { + err = Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + break; + } + if result == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + err = Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT); + break; + } + + if let Some(sigint_event) = sigint_event { + let sig_result = unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + sigint_event as _, + 0, + ) + }; + if sig_result == WAIT_OBJECT_0 { + err = Some(windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT); + break; + } + if sig_result == WAIT_FAILED { + err = Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + break; + } + } + } + + if let Some(err) = err { + if err == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + return Err(vm.new_exception_empty(vm.ctx.exceptions.timeout_error.to_owned())); + } + if err == windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT { + return Err(vm + .new_errno_error(libc::EINTR, "Interrupted system call") + .upcast()); + } + return Err(vm.new_os_error(err as i32)); + } + + Ok(vm.ctx.none()) + } else { + // For wait_any, we use threads to wait on each batch in parallel + let cancel_event = unsafe { WinCreateEventW(null(), 1, 0, null()) }; // Manual reset, not signaled + if cancel_event.is_null() { + return Err(vm.new_last_os_error()); + } + + struct BatchData { + handles: Vec<isize>, + cancel_event: isize, + handle_base: usize, + result: AtomicU32, + thread: core::cell::UnsafeCell<isize>, + } + + unsafe impl Send for BatchData {} + unsafe impl Sync for BatchData {} + + let batch_data: Vec<Arc<BatchData>> = batches + .iter() + .enumerate() + .map(|(idx, batch)| { + let base = idx * batch_size; + let mut handles_with_cancel = batch.clone(); + handles_with_cancel.push(cancel_event as isize); + Arc::new(BatchData { + handles: handles_with_cancel, + cancel_event: cancel_event as isize, + handle_base: base, + result: AtomicU32::new(WAIT_FAILED), + thread: core::cell::UnsafeCell::new(0), + }) + }) + .collect(); + + // Thread function + extern "system" fn batch_wait_thread(param: *mut core::ffi::c_void) -> u32 { + let data = unsafe { &*(param as *const BatchData) }; + let handles: Vec<_> = data.handles.iter().map(|&h| h as _).collect(); + let result = unsafe { + WaitForMultipleObjects( + handles.len() as u32, + handles.as_ptr(), + 0, // wait_any + WIN_INFINITE, + ) + }; + data.result.store(result, Ordering::SeqCst); + + if result == WAIT_FAILED { + let err = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + unsafe { WinSetEvent(data.cancel_event as _) }; + return err; + } else if result >= windows_sys::Win32::Foundation::WAIT_ABANDONED_0 + && result + < windows_sys::Win32::Foundation::WAIT_ABANDONED_0 + + MAXIMUM_WAIT_OBJECTS as u32 + { + data.result.store(WAIT_FAILED, Ordering::SeqCst); + unsafe { WinSetEvent(data.cancel_event as _) }; + return windows_sys::Win32::Foundation::ERROR_ABANDONED_WAIT_0; + } + 0 + } + + // Create threads + let mut thread_handles: Vec<isize> = Vec::new(); + for data in &batch_data { + let thread = unsafe { + CreateThread( + null(), + 1, // Smallest stack + Some(batch_wait_thread), + Arc::as_ptr(data) as *const _ as *mut _, + 4, // CREATE_SUSPENDED + null_mut(), + ) + }; + if thread.is_null() { + // Cleanup on error + for h in &thread_handles { + unsafe { TerminateThread(*h as _, 0) }; + unsafe { CloseHandle(*h as _) }; + } + unsafe { CloseHandle(cancel_event) }; + return Err(vm.new_last_os_error()); + } + unsafe { *data.thread.get() = thread as isize }; + thread_handles.push(thread as isize); + } + + // Resume all threads + for &thread in &thread_handles { + unsafe { ResumeThread(thread as _) }; + } + + // Wait for any thread to complete + let mut thread_handles_raw: Vec<_> = thread_handles.iter().map(|&h| h as _).collect(); + if let Some(sigint_event) = sigint_event { + thread_handles_raw.push(sigint_event as _); + } + let result = unsafe { + WaitForMultipleObjects( + thread_handles_raw.len() as u32, + thread_handles_raw.as_ptr(), + 0, // wait_any + milliseconds, + ) + }; + + let err = if result == WAIT_FAILED { + Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }) + } else if result == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT) + } else if sigint_event.is_some() + && result == WAIT_OBJECT_0 + thread_handles_raw.len() as u32 + { + Some(windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT) + } else { + None + }; + + // Signal cancel event to stop other threads + unsafe { WinSetEvent(cancel_event) }; + + // Wait for all threads to finish + let thread_handles_only: Vec<_> = thread_handles.iter().map(|&h| h as _).collect(); + unsafe { + WaitForMultipleObjects( + thread_handles_only.len() as u32, + thread_handles_only.as_ptr(), + 1, // wait_all + WIN_INFINITE, + ) + }; + + // Check for errors from threads + let mut thread_err = err; + for data in &batch_data { + if thread_err.is_none() && data.result.load(Ordering::SeqCst) == WAIT_FAILED { + let mut exit_code: u32 = 0; + let thread = unsafe { *data.thread.get() }; + if unsafe { GetExitCodeThread(thread as _, &mut exit_code) } == 0 { + thread_err = + Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + } else if exit_code != 0 { + thread_err = Some(exit_code); + } + } + let thread = unsafe { *data.thread.get() }; + unsafe { CloseHandle(thread as _) }; + } + + unsafe { CloseHandle(cancel_event) }; + + // Return result + if let Some(e) = thread_err { + if e == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + return Err(vm.new_exception_empty(vm.ctx.exceptions.timeout_error.to_owned())); + } + if e == windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT { + return Err(vm + .new_errno_error(libc::EINTR, "Interrupted system call") + .upcast()); + } + return Err(vm.new_os_error(e as i32)); + } + + // Collect triggered indices + let mut triggered_indices: Vec<PyObjectRef> = Vec::new(); + for data in &batch_data { + let result = data.result.load(Ordering::SeqCst); + let triggered = result as i32 - WAIT_OBJECT_0 as i32; + // Check if it's a valid handle index (not the cancel_event which is last) + if triggered >= 0 && (triggered as usize) < data.handles.len() - 1 { + let index = data.handle_base + triggered as usize; + triggered_indices.push(vm.ctx.new_int(index).into()); + } + } + + Ok(vm.ctx.new_list(triggered_indices).into()) + } + } } diff --git a/crates/vm/src/stdlib/winreg.rs b/crates/vm/src/stdlib/winreg.rs index a7619025866..1b165f911d0 100644 --- a/crates/vm/src/stdlib/winreg.rs +++ b/crates/vm/src/stdlib/winreg.rs @@ -1,15 +1,11 @@ // spell-checker:disable #![allow(non_snake_case)] -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - winreg::make_module(vm) -} +pub(crate) use winreg::module_def; #[pymodule] mod winreg { - use crate::builtins::{PyInt, PyTuple, PyTypeRef}; + use crate::builtins::{PyInt, PyStr, PyTuple, PyTypeRef}; use crate::common::hash::PyHash; use crate::common::windows::ToWideString; use crate::convert::TryFromObject; @@ -18,10 +14,10 @@ mod winreg { use crate::protocol::PyNumberMethods; use crate::types::{AsNumber, Hashable}; use crate::{Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; + use core::ptr; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::Sign; use num_traits::ToPrimitive; - use std::ptr; use windows_sys::Win32::Foundation::{self, ERROR_MORE_DATA}; use windows_sys::Win32::System::Registry; @@ -41,20 +37,12 @@ mod winreg { u16_slice } - // TODO: check if errno.rs can be reused here or not fn os_error_from_windows_code( vm: &VirtualMachine, code: i32, - func_name: &str, ) -> crate::PyRef<crate::builtins::PyBaseException> { - use windows_sys::Win32::Foundation::{ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}; - let msg = format!("[WinError {}] {}", code, func_name); - let exc_type = match code as u32 { - ERROR_FILE_NOT_FOUND => vm.ctx.exceptions.file_not_found_error.to_owned(), - ERROR_ACCESS_DENIED => vm.ctx.exceptions.permission_error.to_owned(), - _ => vm.ctx.exceptions.os_error.to_owned(), - }; - vm.new_exception_msg(exc_type, msg) + use crate::convert::ToPyException; + std::io::Error::from_raw_os_error(code).to_pyexception(vm) } /// Wrapper type for HKEY that can be created from PyHkey or int @@ -192,7 +180,7 @@ mod winreg { #[pymethod] fn Close(&self, vm: &VirtualMachine) -> PyResult<()> { // Atomically swap the handle with null and get the old value - let old_hkey = self.hkey.swap(std::ptr::null_mut()); + let old_hkey = self.hkey.swap(core::ptr::null_mut()); // Already closed - silently succeed if old_hkey.is_null() { return Ok(()); @@ -208,15 +196,10 @@ mod winreg { #[pymethod] fn Detach(&self) -> PyResult<usize> { // Atomically swap the handle with null and return the old value - let old_hkey = self.hkey.swap(std::ptr::null_mut()); + let old_hkey = self.hkey.swap(core::ptr::null_mut()); Ok(old_hkey as usize) } - #[pymethod] - fn __bool__(&self) -> bool { - !self.hkey.load().is_null() - } - #[pymethod] fn __enter__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Ok(zelf) @@ -227,20 +210,19 @@ mod winreg { zelf.Close(vm) } - #[pymethod] fn __int__(&self) -> usize { self.hkey.load() as usize } #[pymethod] - fn __str__(&self) -> String { - format!("<PyHkey:{:p}>", self.hkey.load()) + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + Ok(vm.ctx.new_str(format!("<PyHkey:{:p}>", zelf.hkey.load()))) } } impl Drop for PyHkey { fn drop(&mut self) { - let hkey = self.hkey.swap(std::ptr::null_mut()); + let hkey = self.hkey.swap(core::ptr::null_mut()); if !hkey.is_null() { unsafe { Registry::RegCloseKey(hkey) }; } @@ -268,13 +250,9 @@ mod winreg { negative: Some(|_a, vm| PyHkey::unary_fail(vm)), positive: Some(|_a, vm| PyHkey::unary_fail(vm)), absolute: Some(|_a, vm| PyHkey::unary_fail(vm)), - boolean: Some(|a, vm| { - if let Some(a) = a.downcast_ref::<PyHkey>() { - Ok(a.__bool__()) - } else { - PyHkey::unary_fail(vm)?; - unreachable!() - } + boolean: Some(|a, _vm| { + let zelf = a.obj.downcast_ref::<PyHkey>().unwrap(); + Ok(!zelf.hkey.load().is_null()) }), invert: Some(|_a, vm| PyHkey::unary_fail(vm)), lshift: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), @@ -304,7 +282,7 @@ mod winreg { vm: &VirtualMachine, ) -> PyResult<PyHkey> { if let Some(computer_name) = computer_name { - let mut ret_key = std::ptr::null_mut(); + let mut ret_key = core::ptr::null_mut(); let wide_computer_name = computer_name.to_wide_with_nul(); let res = unsafe { Registry::RegConnectRegistryW( @@ -319,9 +297,9 @@ mod winreg { Err(vm.new_os_error(format!("error code: {res}"))) } } else { - let mut ret_key = std::ptr::null_mut(); + let mut ret_key = core::ptr::null_mut(); let res = unsafe { - Registry::RegConnectRegistryW(std::ptr::null_mut(), key.hkey.load(), &mut ret_key) + Registry::RegConnectRegistryW(core::ptr::null_mut(), key.hkey.load(), &mut ret_key) }; if res == 0 { Ok(PyHkey::new(ret_key)) @@ -334,7 +312,7 @@ mod winreg { #[pyfunction] fn CreateKey(key: PyRef<PyHkey>, sub_key: String, vm: &VirtualMachine) -> PyResult<PyHkey> { let wide_sub_key = sub_key.to_wide_with_nul(); - let mut out_key = std::ptr::null_mut(); + let mut out_key = core::ptr::null_mut(); let res = unsafe { Registry::RegCreateKeyW(key.hkey.load(), wide_sub_key.as_ptr(), &mut out_key) }; @@ -367,12 +345,12 @@ mod winreg { key, wide_sub_key.as_ptr(), args.reserved, - std::ptr::null(), + core::ptr::null(), Registry::REG_OPTION_NON_VOLATILE, args.access, - std::ptr::null(), + core::ptr::null(), &mut res, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if err == 0 { @@ -404,7 +382,9 @@ mod winreg { #[pyfunction] fn DeleteValue(key: PyRef<PyHkey>, value: Option<String>, vm: &VirtualMachine) -> PyResult<()> { let wide_value = value.map(|v| v.to_wide_with_nul()); - let value_ptr = wide_value.as_ref().map_or(std::ptr::null(), |v| v.as_ptr()); + let value_ptr = wide_value + .as_ref() + .map_or(core::ptr::null(), |v| v.as_ptr()); let res = unsafe { Registry::RegDeleteValueW(key.hkey.load(), value_ptr) }; if res == 0 { Ok(()) @@ -459,14 +439,14 @@ mod winreg { index as u32, tmpbuf.as_mut_ptr(), &mut len, - std::ptr::null_mut(), - std::ptr::null_mut(), - std::ptr::null_mut(), - std::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), ) }; if res != 0 { - return Err(vm.new_os_error(format!("error code: {res}"))); + return Err(os_error_from_windows_code(vm, res as i32)); } String::from_utf16(&tmpbuf[..len as usize]) .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}"))) @@ -608,7 +588,7 @@ mod winreg { #[pyfunction(name = "OpenKeyEx")] fn OpenKey(args: OpenKeyArgs, vm: &VirtualMachine) -> PyResult<PyHkey> { let wide_sub_key = args.sub_key.to_wide_with_nul(); - let mut res: Registry::HKEY = std::ptr::null_mut(); + let mut res: Registry::HKEY = core::ptr::null_mut(); let err = unsafe { let key = args.key.hkey.load(); Registry::RegOpenKeyExW( @@ -625,7 +605,7 @@ mod winreg { hkey: AtomicHKEY::new(res), }) } else { - Err(os_error_from_windows_code(vm, err as i32, "RegOpenKeyEx")) + Err(os_error_from_windows_code(vm, err as i32)) } } @@ -634,20 +614,20 @@ mod winreg { let key = key.0; let mut lpcsubkeys: u32 = 0; let mut lpcvalues: u32 = 0; - let mut lpftlastwritetime: Foundation::FILETIME = unsafe { std::mem::zeroed() }; + let mut lpftlastwritetime: Foundation::FILETIME = unsafe { core::mem::zeroed() }; let err = unsafe { Registry::RegQueryInfoKeyW( key, - std::ptr::null_mut(), - std::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), 0 as _, &mut lpcsubkeys, - std::ptr::null_mut(), - std::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), &mut lpcvalues, - std::ptr::null_mut(), - std::ptr::null_mut(), - std::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), &mut lpftlastwritetime, ) }; @@ -673,7 +653,6 @@ mod winreg { return Err(os_error_from_windows_code( vm, Foundation::ERROR_INVALID_HANDLE as i32, - "RegQueryValue", )); } @@ -681,7 +660,7 @@ mod winreg { let child_key = if let Some(ref sk) = sub_key { if !sk.is_empty() { let wide_sub_key = sk.to_wide_with_nul(); - let mut out_key = std::ptr::null_mut(); + let mut out_key = core::ptr::null_mut(); let res = unsafe { Registry::RegOpenKeyExW( hkey, @@ -692,7 +671,7 @@ mod winreg { ) }; if res != 0 { - return Err(os_error_from_windows_code(vm, res as i32, "RegOpenKeyEx")); + return Err(os_error_from_windows_code(vm, res as i32)); } Some(out_key) } else { @@ -713,8 +692,8 @@ mod winreg { let res = unsafe { Registry::RegQueryValueExW( target_key, - std::ptr::null(), // NULL value name for default value - std::ptr::null_mut(), + core::ptr::null(), // NULL value name for default value + core::ptr::null_mut(), &mut reg_type, buffer.as_mut_ptr(), &mut size, @@ -730,17 +709,12 @@ mod winreg { break Ok(String::new()); } if res != 0 { - break Err(os_error_from_windows_code( - vm, - res as i32, - "RegQueryValueEx", - )); + break Err(os_error_from_windows_code(vm, res as i32)); } if reg_type != Registry::REG_SZ { break Err(os_error_from_windows_code( vm, Foundation::ERROR_INVALID_DATA as i32, - "RegQueryValue", )); } @@ -771,9 +745,9 @@ mod winreg { Registry::RegQueryValueExW( hkey, wide_name.as_ptr(), - std::ptr::null_mut(), - std::ptr::null_mut(), - std::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), &mut buf_size, ) }; @@ -781,11 +755,7 @@ mod winreg { if res == ERROR_MORE_DATA || buf_size == 0 { buf_size = 256; } else if res != 0 { - return Err(os_error_from_windows_code( - vm, - res as i32, - "RegQueryValueEx", - )); + return Err(os_error_from_windows_code(vm, res as i32)); } let mut ret_buf = vec![0u8; buf_size as usize]; @@ -799,7 +769,7 @@ mod winreg { Registry::RegQueryValueExW( hkey, wide_name.as_ptr(), - std::ptr::null_mut(), + core::ptr::null_mut(), &mut typ, ret_buf.as_mut_ptr(), &mut ret_size, @@ -808,11 +778,7 @@ mod winreg { if res != ERROR_MORE_DATA { if res != 0 { - return Err(os_error_from_windows_code( - vm, - res as i32, - "RegQueryValueEx", - )); + return Err(os_error_from_windows_code(vm, res as i32)); } break; } @@ -832,7 +798,7 @@ mod winreg { fn SaveKey(key: PyRef<PyHkey>, file_name: String, vm: &VirtualMachine) -> PyResult<()> { let file_name = file_name.to_wide_with_nul(); let res = unsafe { - Registry::RegSaveKeyW(key.hkey.load(), file_name.as_ptr(), std::ptr::null_mut()) + Registry::RegSaveKeyW(key.hkey.load(), file_name.as_ptr(), core::ptr::null_mut()) }; if res == 0 { Ok(()) @@ -858,29 +824,28 @@ mod winreg { return Err(os_error_from_windows_code( vm, Foundation::ERROR_INVALID_HANDLE as i32, - "RegSetValue", )); } // Create subkey if sub_key is non-empty let child_key = if !sub_key.is_empty() { let wide_sub_key = sub_key.to_wide_with_nul(); - let mut out_key = std::ptr::null_mut(); + let mut out_key = core::ptr::null_mut(); let res = unsafe { Registry::RegCreateKeyExW( hkey, wide_sub_key.as_ptr(), 0, - std::ptr::null(), + core::ptr::null(), 0, Registry::KEY_SET_VALUE, - std::ptr::null(), + core::ptr::null(), &mut out_key, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if res != 0 { - return Err(os_error_from_windows_code(vm, res as i32, "RegCreateKeyEx")); + return Err(os_error_from_windows_code(vm, res as i32)); } Some(out_key) } else { @@ -893,7 +858,7 @@ mod winreg { let res = unsafe { Registry::RegSetValueExW( target_key, - std::ptr::null(), // value name is NULL + core::ptr::null(), // value name is NULL 0, typ, wide_value.as_ptr() as *const u8, @@ -909,7 +874,7 @@ mod winreg { if res == 0 { Ok(()) } else { - Err(os_error_from_windows_code(vm, res as i32, "RegSetValueEx")) + Err(os_error_from_windows_code(vm, res as i32)) } } @@ -917,20 +882,18 @@ mod winreg { match typ { REG_DWORD => { // If there isn’t enough data, return 0. - if ret_data.len() < std::mem::size_of::<u32>() { - Ok(vm.ctx.new_int(0).into()) - } else { - let val = u32::from_ne_bytes(ret_data[..4].try_into().unwrap()); - Ok(vm.ctx.new_int(val).into()) - } + let val = ret_data + .first_chunk::<4>() + .copied() + .map_or(0, u32::from_ne_bytes); + Ok(vm.ctx.new_int(val).into()) } REG_QWORD => { - if ret_data.len() < std::mem::size_of::<u64>() { - Ok(vm.ctx.new_int(0).into()) - } else { - let val = u64::from_ne_bytes(ret_data[..8].try_into().unwrap()); - Ok(vm.ctx.new_int(val).into()) - } + let val = ret_data + .first_chunk::<8>() + .copied() + .map_or(0, u64::from_ne_bytes); + Ok(vm.ctx.new_int(val).into()) } REG_SZ | REG_EXPAND_SZ => { let u16_slice = bytes_as_wide_slice(ret_data); @@ -1029,7 +992,7 @@ mod winreg { return Ok(Some(vec![0u8, 0u8])); } let s = value - .downcast::<crate::builtins::PyStr>() + .downcast::<PyStr>() .map_err(|_| vm.new_type_error("value must be a string".to_string()))?; let wide = s.as_str().to_wide_with_nul(); // Convert Vec<u16> to Vec<u8> @@ -1047,11 +1010,9 @@ mod winreg { let mut bytes: Vec<u8> = Vec::new(); for item in list.borrow_vec().iter() { - let s = item - .downcast_ref::<crate::builtins::PyStr>() - .ok_or_else(|| { - vm.new_type_error("list items must be strings".to_string()) - })?; + let s = item.downcast_ref::<PyStr>().ok_or_else(|| { + vm.new_type_error("list items must be strings".to_string()) + })?; let wide = s.as_str().to_wide_with_nul(); bytes.extend(wide.iter().flat_map(|&c| c.to_le_bytes())); } @@ -1079,50 +1040,25 @@ mod winreg { #[pyfunction] fn SetValueEx( key: PyRef<PyHkey>, - value_name: String, + value_name: Option<String>, _reserved: PyObjectRef, typ: u32, value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - match py2reg(value, typ, vm) { - Ok(Some(v)) => { - let len = v.len() as u32; - let ptr = v.as_ptr(); - let wide_value_name = value_name.to_wide_with_nul(); - let res = unsafe { - Registry::RegSetValueExW( - key.hkey.load(), - wide_value_name.as_ptr(), - 0, - typ, - ptr, - len, - ) - }; - if res != 0 { - return Err(vm.new_os_error(format!("error code: {res}"))); - } - } - Ok(None) => { - let len = 0; - let ptr = std::ptr::null(); - let wide_value_name = value_name.to_wide_with_nul(); - let res = unsafe { - Registry::RegSetValueExW( - key.hkey.load(), - wide_value_name.as_ptr(), - 0, - typ, - ptr, - len, - ) - }; - if res != 0 { - return Err(vm.new_os_error(format!("error code: {res}"))); - } - } - Err(e) => return Err(e), + let wide_value_name = value_name.as_deref().map(|s| s.to_wide_with_nul()); + let value_name_ptr = wide_value_name + .as_deref() + .map_or(core::ptr::null(), |s| s.as_ptr()); + let reg_value = py2reg(value, typ, vm)?; + let (ptr, len) = match &reg_value { + Some(v) => (v.as_ptr(), v.len() as u32), + None => (core::ptr::null(), 0), + }; + let res = + unsafe { Registry::RegSetValueExW(key.hkey.load(), value_name_ptr, 0, typ, ptr, len) }; + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); } Ok(()) } @@ -1166,7 +1102,7 @@ mod winreg { let required_size = unsafe { windows_sys::Win32::System::Environment::ExpandEnvironmentStringsW( wide_input.as_ptr(), - std::ptr::null_mut(), + core::ptr::null_mut(), 0, ) }; diff --git a/crates/vm/src/stdlib/winsound.rs b/crates/vm/src/stdlib/winsound.rs new file mode 100644 index 00000000000..3f65abbb890 --- /dev/null +++ b/crates/vm/src/stdlib/winsound.rs @@ -0,0 +1,206 @@ +// spell-checker:ignore pszSound fdwSound +#![allow(non_snake_case)] + +pub(crate) use winsound::module_def; + +mod win32 { + #[link(name = "winmm")] + unsafe extern "system" { + pub fn PlaySoundW(pszSound: *const u16, hmod: isize, fdwSound: u32) -> i32; + } + + unsafe extern "system" { + pub fn Beep(dwFreq: u32, dwDuration: u32) -> i32; + pub fn MessageBeep(uType: u32) -> i32; + } +} + +#[pymodule] +mod winsound { + use crate::builtins::{PyBytes, PyStr}; + use crate::common::windows::ToWideString; + use crate::convert::{IntoPyException, TryFromBorrowedObject}; + use crate::protocol::PyBuffer; + use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine}; + + // PlaySound flags + #[pyattr] + const SND_SYNC: u32 = 0x0000; + #[pyattr] + const SND_ASYNC: u32 = 0x0001; + #[pyattr] + const SND_NODEFAULT: u32 = 0x0002; + #[pyattr] + const SND_MEMORY: u32 = 0x0004; + #[pyattr] + const SND_LOOP: u32 = 0x0008; + #[pyattr] + const SND_NOSTOP: u32 = 0x0010; + #[pyattr] + const SND_PURGE: u32 = 0x0040; + #[pyattr] + const SND_APPLICATION: u32 = 0x0080; + #[pyattr] + const SND_NOWAIT: u32 = 0x00002000; + #[pyattr] + const SND_ALIAS: u32 = 0x00010000; + #[pyattr] + const SND_FILENAME: u32 = 0x00020000; + #[pyattr] + const SND_SENTRY: u32 = 0x00080000; + #[pyattr] + const SND_SYSTEM: u32 = 0x00200000; + + // MessageBeep types + #[pyattr] + const MB_OK: u32 = 0x00000000; + #[pyattr] + const MB_ICONHAND: u32 = 0x00000010; + #[pyattr] + const MB_ICONQUESTION: u32 = 0x00000020; + #[pyattr] + const MB_ICONEXCLAMATION: u32 = 0x00000030; + #[pyattr] + const MB_ICONASTERISK: u32 = 0x00000040; + #[pyattr] + const MB_ICONERROR: u32 = MB_ICONHAND; + #[pyattr] + const MB_ICONSTOP: u32 = MB_ICONHAND; + #[pyattr] + const MB_ICONINFORMATION: u32 = MB_ICONASTERISK; + #[pyattr] + const MB_ICONWARNING: u32 = MB_ICONEXCLAMATION; + + #[derive(FromArgs)] + struct PlaySoundArgs { + #[pyarg(any)] + sound: PyObjectRef, + #[pyarg(any)] + flags: i32, + } + + #[pyfunction] + fn PlaySound(args: PlaySoundArgs, vm: &VirtualMachine) -> PyResult<()> { + let sound = args.sound; + let flags = args.flags as u32; + + if vm.is_none(&sound) { + let ok = unsafe { super::win32::PlaySoundW(core::ptr::null(), 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound".to_owned())); + } + return Ok(()); + } + + if flags & SND_MEMORY != 0 { + if flags & SND_ASYNC != 0 { + return Err( + vm.new_runtime_error("Cannot play asynchronously from memory".to_owned()) + ); + } + let buffer = PyBuffer::try_from_borrowed_object(vm, &sound)?; + let buf = buffer.as_contiguous().ok_or_else(|| { + vm.new_type_error("a bytes-like object is required, not 'str'".to_owned()) + })?; + let ok = unsafe { super::win32::PlaySoundW(buf.as_ptr() as *const u16, 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound".to_owned())); + } + return Ok(()); + } + + if sound.downcastable::<PyBytes>() { + let type_name = sound.class().name().to_string(); + return Err(vm.new_type_error(format!( + "'sound' must be str, os.PathLike, or None, not {type_name}" + ))); + } + + // os.fspath(sound) + let path = match sound.downcast_ref::<PyStr>() { + Some(s) => s.as_str().to_owned(), + None => { + let fspath = vm.get_method_or_type_error( + sound.clone(), + identifier!(vm, __fspath__), + || { + let type_name = sound.class().name().to_string(); + format!("'sound' must be str, os.PathLike, or None, not {type_name}") + }, + )?; + + if vm.is_none(&fspath) { + return Err(vm.new_type_error(format!( + "'sound' must be str, os.PathLike, or None, not {}", + sound.class().name() + ))); + } + let result = fspath.call((), vm)?; + + if result.downcastable::<PyBytes>() { + return Err( + vm.new_type_error("'sound' must resolve to str, not bytes".to_owned()) + ); + } + + let s: &PyStr = result.downcast_ref().ok_or_else(|| { + vm.new_type_error(format!( + "expected {}.__fspath__() to return str or bytes, not {}", + sound.class().name(), + result.class().name() + )) + })?; + + s.as_str().to_owned() + } + }; + + // Check for embedded null characters + if path.contains('\0') { + return Err(vm.new_value_error("embedded null character".to_owned())); + } + + let wide = path.to_wide_with_nul(); + let ok = unsafe { super::win32::PlaySoundW(wide.as_ptr(), 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound".to_owned())); + } + Ok(()) + } + + #[derive(FromArgs)] + struct BeepArgs { + #[pyarg(any)] + frequency: i32, + #[pyarg(any)] + duration: i32, + } + + #[pyfunction] + fn Beep(args: BeepArgs, vm: &VirtualMachine) -> PyResult<()> { + if !(37..=32767).contains(&args.frequency) { + return Err(vm.new_value_error("frequency must be in 37 thru 32767".to_owned())); + } + + let ok = unsafe { super::win32::Beep(args.frequency as u32, args.duration as u32) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to beep".to_owned())); + } + Ok(()) + } + + #[derive(FromArgs)] + struct MessageBeepArgs { + #[pyarg(any, default = 0)] + r#type: u32, + } + + #[pyfunction] + fn MessageBeep(args: MessageBeepArgs, vm: &VirtualMachine) -> PyResult<()> { + let ok = unsafe { super::win32::MessageBeep(args.r#type) }; + if ok == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + Ok(()) + } +} diff --git a/crates/vm/src/suggestion.rs b/crates/vm/src/suggestion.rs index 2cc935873c2..2d732160f07 100644 --- a/crates/vm/src/suggestion.rs +++ b/crates/vm/src/suggestion.rs @@ -2,19 +2,19 @@ //! This is used during tracebacks. use crate::{ - AsObject, Py, PyObjectRef, VirtualMachine, + AsObject, Py, PyObject, PyObjectRef, VirtualMachine, builtins::{PyStr, PyStrRef}, - exceptions::types::PyBaseExceptionRef, + exceptions::types::PyBaseException, sliceable::SliceableSequenceOp, }; +use core::iter::ExactSizeIterator; use rustpython_common::str::levenshtein::{MOVE_COST, levenshtein_distance}; -use std::iter::ExactSizeIterator; const MAX_CANDIDATE_ITEMS: usize = 750; pub fn calculate_suggestions<'a>( dir_iter: impl ExactSizeIterator<Item = &'a PyObjectRef>, - name: &PyObjectRef, + name: &PyObject, ) -> Option<PyStrRef> { if dir_iter.len() >= MAX_CANDIDATE_ITEMS { return None; @@ -47,14 +47,26 @@ pub fn calculate_suggestions<'a>( suggestion.map(|r| r.to_owned()) } -pub fn offer_suggestions(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> Option<PyStrRef> { - if exc.class().is(vm.ctx.exceptions.attribute_error) { - let name = exc.as_object().get_attr("name", vm).unwrap(); - let obj = exc.as_object().get_attr("obj", vm).unwrap(); +pub fn offer_suggestions(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> Option<PyStrRef> { + if exc + .class() + .fast_issubclass(vm.ctx.exceptions.attribute_error) + { + let name = exc.as_object().get_attr("name", vm).ok()?; + if vm.is_none(&name) { + return None; + } + let obj = exc.as_object().get_attr("obj", vm).ok()?; + if vm.is_none(&obj) { + return None; + } calculate_suggestions(vm.dir(Some(obj)).ok()?.borrow_vec().iter(), &name) - } else if exc.class().is(vm.ctx.exceptions.name_error) { - let name = exc.as_object().get_attr("name", vm).unwrap(); + } else if exc.class().fast_issubclass(vm.ctx.exceptions.name_error) { + let name = exc.as_object().get_attr("name", vm).ok()?; + if vm.is_none(&name) { + return None; + } let tb = exc.__traceback__()?; let tb = tb.iter().last().unwrap_or(tb); @@ -68,8 +80,18 @@ pub fn offer_suggestions(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> Optio return Some(suggestions); }; - let builtins: Vec<_> = tb.frame.builtins.as_object().try_to_value(vm).ok()?; + let builtins: Vec<_> = tb.frame.builtins.try_to_value(vm).ok()?; calculate_suggestions(builtins.iter(), &name) + } else if exc.class().fast_issubclass(vm.ctx.exceptions.import_error) { + let mod_name = exc.as_object().get_attr("name", vm).ok()?; + let wrong_name = exc.as_object().get_attr("name_from", vm).ok()?; + let mod_name_str = mod_name.downcast_ref::<PyStr>()?; + + // Look up the module in sys.modules + let sys_modules = vm.sys_module.get_attr("modules", vm).ok()?; + let module = sys_modules.get_item(mod_name_str.as_str(), vm).ok()?; + + calculate_suggestions(vm.dir(Some(module)).ok()?.borrow_vec().iter(), &wrong_name) } else { None } diff --git a/crates/vm/src/types/mod.rs b/crates/vm/src/types/mod.rs index f19328cdd2d..b17a737545f 100644 --- a/crates/vm/src/types/mod.rs +++ b/crates/vm/src/types/mod.rs @@ -1,7 +1,9 @@ mod slot; +pub mod slot_defs; mod structseq; mod zoo; pub use slot::*; +pub use slot_defs::{SLOT_DEFS, SlotAccessor, SlotDef}; pub use structseq::{PyStructSequence, PyStructSequenceData, struct_sequence_new}; pub(crate) use zoo::TypeZoo; diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index f52e7296a7b..ae00158aeb4 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -3,23 +3,23 @@ use crate::common::lock::{ }; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyInt, PyStr, PyStrInterned, PyStrRef, PyType, PyTypeRef, type_::PointerSlot}, + builtins::{PyInt, PyStr, PyStrInterned, PyStrRef, PyType, PyTypeRef}, bytecode::ComparisonOperator, - common::hash::PyHash, - convert::{ToPyObject, ToPyResult}, + common::hash::{PyHash, fix_sentinel, hash_bigint}, + convert::ToPyObject, function::{ Either, FromArgs, FuncArgs, OptionalArg, PyComparisonValue, PyMethodDef, PySetterValue, }, protocol::{ - PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyNumber, PyNumberMethods, - PyNumberSlots, PySequence, PySequenceMethods, + PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyMappingSlots, PyNumber, + PyNumberMethods, PyNumberSlots, PySequence, PySequenceMethods, PySequenceSlots, }, + types::slot_defs::{SlotAccessor, find_slot_defs_by_name}, vm::Context, }; +use core::{any::Any, any::TypeId, borrow::Borrow, cmp::Ordering, ops::Deref}; use crossbeam_utils::atomic::AtomicCell; -use malachite_bigint::BigInt; use num_traits::{Signed, ToPrimitive}; -use std::{any::Any, any::TypeId, borrow::Borrow, cmp::Ordering, ops::Deref}; /// Type-erased storage for extension module data attached to heap types. pub struct TypeDataSlot { @@ -71,7 +71,7 @@ impl<'a, T: Any + 'static> TypeDataRef<'a, T> { } } -impl<T: Any + 'static> std::ops::Deref for TypeDataRef<'_, T> { +impl<T: Any + 'static> core::ops::Deref for TypeDataRef<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -96,7 +96,7 @@ impl<'a, T: Any + 'static> TypeDataRefMut<'a, T> { } } -impl<T: Any + 'static> std::ops::Deref for TypeDataRefMut<'_, T> { +impl<T: Any + 'static> core::ops::Deref for TypeDataRefMut<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -104,7 +104,7 @@ impl<T: Any + 'static> std::ops::Deref for TypeDataRefMut<'_, T> { } } -impl<T: Any + 'static> std::ops::DerefMut for TypeDataRefMut<'_, T> { +impl<T: Any + 'static> core::ops::DerefMut for TypeDataRefMut<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.guard } @@ -113,7 +113,7 @@ impl<T: Any + 'static> std::ops::DerefMut for TypeDataRefMut<'_, T> { #[macro_export] macro_rules! atomic_func { ($x:expr) => { - crossbeam_utils::atomic::AtomicCell::new(Some($x)) + Some($x) }; } @@ -128,19 +128,19 @@ pub struct PyTypeSlots { pub(crate) name: &'static str, // tp_name with <module>.<class> for print, not class name pub basicsize: usize, - // tp_itemsize + pub itemsize: usize, // tp_itemsize // Methods to implement standard operations // Method suites for standard classes pub as_number: PyNumberSlots, - pub as_sequence: AtomicCell<Option<PointerSlot<PySequenceMethods>>>, - pub as_mapping: AtomicCell<Option<PointerSlot<PyMappingMethods>>>, + pub as_sequence: PySequenceSlots, + pub as_mapping: PyMappingSlots, // More standard operations (here for binary compatibility) pub hash: AtomicCell<Option<HashFunc>>, pub call: AtomicCell<Option<GenericMethod>>, - // tp_str + pub str: AtomicCell<Option<StringifyFunc>>, pub repr: AtomicCell<Option<StringifyFunc>>, pub getattro: AtomicCell<Option<GetattroFunc>>, pub setattro: AtomicCell<Option<SetattroFunc>>, @@ -203,8 +203,8 @@ impl PyTypeSlots { } } -impl std::fmt::Debug for PyTypeSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyTypeSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PyTypeSlots") } } @@ -288,6 +288,21 @@ pub(crate) type NewFunc = fn(PyTypeRef, FuncArgs, &VirtualMachine) -> PyResult; pub(crate) type InitFunc = fn(PyObjectRef, FuncArgs, &VirtualMachine) -> PyResult<()>; pub(crate) type DelFunc = fn(&PyObject, &VirtualMachine) -> PyResult<()>; +// Sequence sub-slot function types +pub(crate) type SeqLenFunc = fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>; +pub(crate) type SeqConcatFunc = fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult; +pub(crate) type SeqRepeatFunc = fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult; +pub(crate) type SeqItemFunc = fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult; +pub(crate) type SeqAssItemFunc = + fn(PySequence<'_>, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>; +pub(crate) type SeqContainsFunc = fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult<bool>; + +// Mapping sub-slot function types +pub(crate) type MapLenFunc = fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>; +pub(crate) type MapSubscriptFunc = fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult; +pub(crate) type MapAssSubscriptFunc = + fn(PyMapping<'_>, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>; + // slot_sq_length pub(crate) fn len_wrapper(obj: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { let ret = vm.call_special_method(obj, identifier!(vm, __len__), ())?; @@ -331,6 +346,28 @@ macro_rules! number_binary_right_op_wrapper { |a, b, vm| vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) }; } +macro_rules! number_ternary_op_wrapper { + ($name:ident) => { + |a, b, c, vm: &VirtualMachine| { + if vm.is_none(c) { + vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(),)) + } else { + vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(), c.to_owned())) + } + } + }; +} +macro_rules! number_ternary_right_op_wrapper { + ($name:ident) => { + |a, b, c, vm: &VirtualMachine| { + if vm.is_none(c) { + vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) + } else { + vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(), c.to_owned())) + } + } + }; +} fn getitem_wrapper<K: ToPyObject>(obj: &PyObject, needle: K, vm: &VirtualMachine) -> PyResult { vm.call_special_method(obj, identifier!(vm, __getitem__), (needle,)) } @@ -348,6 +385,69 @@ fn setitem_wrapper<K: ToPyObject>( .map(drop) } +#[inline(never)] +fn mapping_setitem_wrapper( + mapping: PyMapping<'_>, + key: &PyObject, + value: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + setitem_wrapper(mapping.obj, key, value, vm) +} + +#[inline(never)] +fn mapping_getitem_wrapper( + mapping: PyMapping<'_>, + key: &PyObject, + vm: &VirtualMachine, +) -> PyResult { + getitem_wrapper(mapping.obj, key, vm) +} + +#[inline(never)] +fn mapping_len_wrapper(mapping: PyMapping<'_>, vm: &VirtualMachine) -> PyResult<usize> { + len_wrapper(mapping.obj, vm) +} + +#[inline(never)] +fn sequence_len_wrapper(seq: PySequence<'_>, vm: &VirtualMachine) -> PyResult<usize> { + len_wrapper(seq.obj, vm) +} + +#[inline(never)] +fn sequence_getitem_wrapper(seq: PySequence<'_>, i: isize, vm: &VirtualMachine) -> PyResult { + getitem_wrapper(seq.obj, i, vm) +} + +#[inline(never)] +fn sequence_setitem_wrapper( + seq: PySequence<'_>, + i: isize, + value: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + setitem_wrapper(seq.obj, i, value, vm) +} + +#[inline(never)] +fn sequence_contains_wrapper( + seq: PySequence<'_>, + needle: &PyObject, + vm: &VirtualMachine, +) -> PyResult<bool> { + contains_wrapper(seq.obj, needle, vm) +} + +#[inline(never)] +fn sequence_repeat_wrapper(seq: PySequence<'_>, n: isize, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(seq.obj, identifier!(vm, __mul__), (n,)) +} + +#[inline(never)] +fn sequence_inplace_repeat_wrapper(seq: PySequence<'_>, n: isize, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(seq.obj, identifier!(vm, __imul__), (n,)) +} + fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { let ret = vm.call_special_method(zelf, identifier!(vm, __repr__), ())?; ret.downcast::<PyStr>().map_err(|obj| { @@ -358,21 +458,32 @@ fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> }) } +fn str_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let ret = vm.call_special_method(zelf, identifier!(vm, __str__), ())?; + ret.downcast::<PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "__str__ returned non-string (type {})", + obj.class() + )) + }) +} + fn hash_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { let hash_obj = vm.call_special_method(zelf, identifier!(vm, __hash__), ())?; let py_int = hash_obj .downcast_ref::<PyInt>() .ok_or_else(|| vm.new_type_error("__hash__ method should return an integer"))?; let big_int = py_int.as_bigint(); - let hash: PyHash = big_int + let hash = big_int .to_i64() - .unwrap_or_else(|| (big_int % BigInt::from(u64::MAX)).to_i64().unwrap()); + .map(fix_sentinel) + .unwrap_or_else(|| hash_bigint(big_int)); Ok(hash) } /// Marks a type as unhashable. Similar to PyObject_HashNotImplemented in CPython pub fn hash_not_implemented(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { - Err(vm.new_type_error(format!("unhashable type: {}", zelf.class().name()))) + Err(vm.new_type_error(format!("unhashable type: '{}'", zelf.class().name()))) } fn call_wrapper(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult { @@ -423,7 +534,27 @@ pub(crate) fn richcompare_wrapper( } fn iter_wrapper(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __iter__), ()) + // slot_tp_iter: if __iter__ is None, the type is explicitly not iterable + let cls = zelf.class(); + let iter_attr = cls.get_attr(identifier!(vm, __iter__)); + match iter_attr { + Some(attr) if vm.is_none(&attr) => { + Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))) + } + _ => vm.call_special_method(&zelf, identifier!(vm, __iter__), ()), + } +} + +fn bool_wrapper(num: PyNumber<'_>, vm: &VirtualMachine) -> PyResult<bool> { + let result = vm.call_special_method(num.obj, identifier!(vm, __bool__), ())?; + // __bool__ must return exactly bool, not int subclass + if !result.class().is(vm.ctx.types.bool_type) { + return Err(vm.new_type_error(format!( + "__bool__ should return bool, returned {}", + result.class().name() + ))); + } + Ok(crate::builtins::bool_::get_value(&result)) } // PyObject_SelfIter in CPython @@ -465,7 +596,10 @@ fn descr_set_wrapper( fn init_wrapper(obj: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let res = vm.call_special_method(&obj, identifier!(vm, __init__), args)?; if !vm.is_none(&res) { - return Err(vm.new_type_error("__init__ must return None")); + return Err(vm.new_type_error(format!( + "__init__ should return None, not '{:.200}'", + res.class().name() + ))); } Ok(()) } @@ -482,392 +616,853 @@ fn del_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { } impl PyType { + /// Update slots based on dunder method changes + /// + /// Iterates SLOT_DEFS to find all slots matching the given name and updates them. + /// Also recursively updates subclasses that don't have their own definition. pub(crate) fn update_slot<const ADD: bool>(&self, name: &'static PyStrInterned, ctx: &Context) { debug_assert!(name.as_str().starts_with("__")); debug_assert!(name.as_str().ends_with("__")); - macro_rules! toggle_slot { - ($name:ident, $func:expr) => {{ - self.slots.$name.store(if ADD { Some($func) } else { None }); - }}; + // Find all slot_defs matching this name and update each + // NOTE: Collect into Vec first to avoid issues during iteration + let defs: Vec<_> = find_slot_defs_by_name(name.as_str()).collect(); + for def in defs { + self.update_one_slot::<ADD>(&def.accessor, name, ctx); } - macro_rules! toggle_sub_slot { - ($group:ident, $name:ident, $func:expr) => { - self.slots - .$group - .$name - .store(if ADD { Some($func) } else { None }); + // Recursively update subclasses that don't have their own definition + self.update_subclasses::<ADD>(name, ctx); + } + + /// Recursively update subclasses' slots + /// recurse_down_subclasses + fn update_subclasses<const ADD: bool>(&self, name: &'static PyStrInterned, ctx: &Context) { + let subclasses = self.subclasses.read(); + for weak_ref in subclasses.iter() { + let Some(subclass) = weak_ref.upgrade() else { + continue; + }; + let Some(subclass) = subclass.downcast_ref::<PyType>() else { + continue; }; - } - macro_rules! update_slot { - ($name:ident, $func:expr) => {{ - self.slots.$name.store(Some($func)); - }}; + // Skip if subclass has its own definition for this attribute + if subclass.attributes.read().contains_key(name) { + continue; + } + + // Update subclass's slots + for def in find_slot_defs_by_name(name.as_str()) { + subclass.update_one_slot::<ADD>(&def.accessor, name, ctx); + } + + // Recurse into subclass's subclasses + subclass.update_subclasses::<ADD>(name, ctx); } + } - macro_rules! update_pointer_slot { - ($name:ident, $pointed:ident) => {{ - self.slots - .$name - .store(unsafe { PointerSlot::from_heaptype(self, |ext| &ext.$pointed) }); + /// Update a single slot + fn update_one_slot<const ADD: bool>( + &self, + accessor: &SlotAccessor, + name: &'static PyStrInterned, + ctx: &Context, + ) { + use crate::builtins::descriptor::SlotFunc; + + // Helper macro for main slots + macro_rules! update_main_slot { + ($slot:ident, $wrapper:expr, $variant:ident) => {{ + if ADD { + if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::$variant(f) = sf { + Some(*f) + } else { + None + } + }) { + self.slots.$slot.store(Some(func)); + } else { + self.slots.$slot.store(Some($wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } }}; } - macro_rules! toggle_ext_func { - ($n1:ident, $n2:ident, $func:expr) => {{ - self.heaptype_ext.as_ref().unwrap().$n1.$n2.store(if ADD { - Some($func) + // Helper macro for number/sequence/mapping sub-slots + macro_rules! update_sub_slot { + ($group:ident, $slot:ident, $wrapper:expr, $variant:ident) => {{ + if ADD { + // Check if this type defines any method that maps to this slot. + // Some slots like SqAssItem/MpAssSubscript are shared by multiple + // methods (__setitem__ and __delitem__). If any of those methods + // is defined, we must use the wrapper to ensure Python method calls. + let has_own = { + let guard = self.attributes.read(); + // Check the current method name + let mut result = guard.contains_key(name); + // For ass_item/ass_subscript slots, also check the paired method + // (__setitem__ and __delitem__ share the same slot) + if !result + && (stringify!($slot) == "ass_item" + || stringify!($slot) == "ass_subscript") + { + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + result = guard.contains_key(setitem) || guard.contains_key(delitem); + } + result + }; + if has_own { + self.slots.$group.$slot.store(Some($wrapper)); + } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::$variant(f) = sf { + Some(*f) + } else { + None + } + }) { + self.slots.$group.$slot.store(Some(func)); + } else { + self.slots.$group.$slot.store(Some($wrapper)); + } } else { - None - }); + accessor.inherit_from_mro(self); + } }}; } - match name { - _ if name == identifier!(ctx, __len__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, length, |seq, vm| len_wrapper(seq.obj, vm)); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, length, |mapping, vm| len_wrapper( - mapping.obj, - vm - )); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __getitem__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, item, |seq, i, vm| getitem_wrapper( - seq.obj, i, vm - )); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, subscript, |mapping, key, vm| { - getitem_wrapper(mapping.obj, key, vm) - }); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __setitem__) || name == identifier!(ctx, __delitem__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, ass_item, |seq, i, value, vm| { - setitem_wrapper(seq.obj, i, value, vm) - }); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, ass_subscript, |mapping, key, value, vm| { - setitem_wrapper(mapping.obj, key, value, vm) - }); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __contains__) => { - toggle_ext_func!(sequence_methods, contains, |seq, needle, vm| { - contains_wrapper(seq.obj, needle, vm) - }); - update_pointer_slot!(as_sequence, sequence_methods); - } - _ if name == identifier!(ctx, __repr__) => { - update_slot!(repr, repr_wrapper); - } - _ if name == identifier!(ctx, __hash__) => { - let is_unhashable = self - .attributes - .read() - .get(identifier!(ctx, __hash__)) - .is_some_and(|a| a.is(&ctx.none)); - let wrapper = if is_unhashable { - hash_not_implemented + match accessor { + // === Main slots === + SlotAccessor::TpRepr => update_main_slot!(repr, repr_wrapper, Repr), + SlotAccessor::TpStr => update_main_slot!(str, str_wrapper, Str), + SlotAccessor::TpHash => { + // Special handling for __hash__ = None + if ADD { + let method = self.attributes.read().get(name).cloned().or_else(|| { + self.mro + .read() + .iter() + .find_map(|cls| cls.attributes.read().get(name).cloned()) + }); + + if method.as_ref().is_some_and(|m| m.is(&ctx.none)) { + self.slots.hash.store(Some(hash_not_implemented)); + } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::Hash(f) = sf { + Some(*f) + } else { + None + } + }) { + self.slots.hash.store(Some(func)); + } else { + self.slots.hash.store(Some(hash_wrapper)); + } } else { - hash_wrapper - }; - toggle_slot!(hash, wrapper); - } - _ if name == identifier!(ctx, __call__) => { - toggle_slot!(call, call_wrapper); - } - _ if name == identifier!(ctx, __getattr__) - || name == identifier!(ctx, __getattribute__) => - { - update_slot!(getattro, getattro_wrapper); - } - _ if name == identifier!(ctx, __setattr__) || name == identifier!(ctx, __delattr__) => { - update_slot!(setattro, setattro_wrapper); + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __eq__) - || name == identifier!(ctx, __ne__) - || name == identifier!(ctx, __le__) - || name == identifier!(ctx, __lt__) - || name == identifier!(ctx, __ge__) - || name == identifier!(ctx, __gt__) => - { - update_slot!(richcompare, richcompare_wrapper); + SlotAccessor::TpCall => update_main_slot!(call, call_wrapper, Call), + SlotAccessor::TpIter => update_main_slot!(iter, iter_wrapper, Iter), + SlotAccessor::TpIternext => update_main_slot!(iternext, iternext_wrapper, IterNext), + SlotAccessor::TpInit => update_main_slot!(init, init_wrapper, Init), + SlotAccessor::TpNew => { + // __new__ is not wrapped via PyWrapper + if ADD { + self.slots.new.store(Some(new_wrapper)); + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __iter__) => { - toggle_slot!(iter, iter_wrapper); + SlotAccessor::TpDel => update_main_slot!(del, del_wrapper, Del), + SlotAccessor::TpGetattro => { + // __getattribute__ and __getattr__ both map to TpGetattro. + // If __getattr__ is defined anywhere in MRO, we must use the wrapper + // because the native slot won't call __getattr__. + let __getattr__ = identifier!(ctx, __getattr__); + let has_getattr = { + let attrs = self.attributes.read(); + let in_self = attrs.contains_key(__getattr__); + drop(attrs); + // mro[0] is self, so skip it + in_self + || self + .mro + .read() + .iter() + .skip(1) + .any(|cls| cls.attributes.read().contains_key(__getattr__)) + }; + + if has_getattr { + // Must use wrapper to handle __getattr__ + self.slots.getattro.store(Some(getattro_wrapper)); + } else if ADD { + if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::GetAttro(f) = sf { + Some(*f) + } else { + None + } + }) { + self.slots.getattro.store(Some(func)); + } else { + self.slots.getattro.store(Some(getattro_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __next__) => { - toggle_slot!(iternext, iternext_wrapper); + SlotAccessor::TpSetattro => { + // __setattr__ and __delattr__ share the same slot + if ADD { + if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), + _ => None, + }) { + self.slots.setattro.store(Some(func)); + } else { + self.slots.setattro.store(Some(setattro_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __get__) => { - toggle_slot!(descr_get, descr_get_wrapper); + SlotAccessor::TpDescrGet => update_main_slot!(descr_get, descr_get_wrapper, DescrGet), + SlotAccessor::TpDescrSet => { + // __set__ and __delete__ share the same slot + if ADD { + if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), + _ => None, + }) { + self.slots.descr_set.store(Some(func)); + } else { + self.slots.descr_set.store(Some(descr_set_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __set__) || name == identifier!(ctx, __delete__) => { - update_slot!(descr_set, descr_set_wrapper); + + // === Rich compare (__lt__, __le__, __eq__, __ne__, __gt__, __ge__) === + SlotAccessor::TpRichcompare => { + if ADD { + // Check if self or any class in MRO has a Python-defined comparison method + // All comparison ops share the same slot, so if any is overridden anywhere + // in the hierarchy with a Python function, we need to use the wrapper + let cmp_names = [ + identifier!(ctx, __eq__), + identifier!(ctx, __ne__), + identifier!(ctx, __lt__), + identifier!(ctx, __le__), + identifier!(ctx, __gt__), + identifier!(ctx, __ge__), + ]; + + let has_python_cmp = { + // Check self first + let attrs = self.attributes.read(); + let in_self = cmp_names.iter().any(|n| attrs.contains_key(*n)); + drop(attrs); + + // mro[0] is self, so skip it since we already checked self above + in_self + || self.mro.read()[1..].iter().any(|cls| { + let attrs = cls.attributes.read(); + cmp_names.iter().any(|n| { + if let Some(attr) = attrs.get(*n) { + // Check if it's a Python function (not wrapper_descriptor) + !attr.class().is(ctx.types.wrapper_descriptor_type) + } else { + false + } + }) + }) + }; + + if has_python_cmp { + // Use wrapper to call the Python method + self.slots.richcompare.store(Some(richcompare_wrapper)); + } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::RichCompare(f, _) = sf { + Some(*f) + } else { + None + } + }) { + self.slots.richcompare.store(Some(func)); + } else { + self.slots.richcompare.store(Some(richcompare_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __init__) => { - toggle_slot!(init, init_wrapper); + + // === Number binary operations === + SlotAccessor::NbAdd => { + if name.as_str() == "__radd__" { + update_sub_slot!( + as_number, + right_add, + number_binary_right_op_wrapper!(__radd__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + add, + number_binary_op_wrapper!(__add__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __new__) => { - toggle_slot!(new, new_wrapper); + SlotAccessor::NbInplaceAdd => { + update_sub_slot!( + as_number, + inplace_add, + number_binary_op_wrapper!(__iadd__), + NumBinary + ) } - _ if name == identifier!(ctx, __del__) => { - toggle_slot!(del, del_wrapper); + SlotAccessor::NbSubtract => { + if name.as_str() == "__rsub__" { + update_sub_slot!( + as_number, + right_subtract, + number_binary_right_op_wrapper!(__rsub__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + subtract, + number_binary_op_wrapper!(__sub__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __int__) => { - toggle_sub_slot!(as_number, int, number_unary_op_wrapper!(__int__)); + SlotAccessor::NbInplaceSubtract => { + update_sub_slot!( + as_number, + inplace_subtract, + number_binary_op_wrapper!(__isub__), + NumBinary + ) } - _ if name == identifier!(ctx, __index__) => { - toggle_sub_slot!(as_number, index, number_unary_op_wrapper!(__index__)); + SlotAccessor::NbMultiply => { + if name.as_str() == "__rmul__" { + update_sub_slot!( + as_number, + right_multiply, + number_binary_right_op_wrapper!(__rmul__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + multiply, + number_binary_op_wrapper!(__mul__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __float__) => { - toggle_sub_slot!(as_number, float, number_unary_op_wrapper!(__float__)); + SlotAccessor::NbInplaceMultiply => { + update_sub_slot!( + as_number, + inplace_multiply, + number_binary_op_wrapper!(__imul__), + NumBinary + ) } - _ if name == identifier!(ctx, __add__) => { - toggle_sub_slot!(as_number, add, number_binary_op_wrapper!(__add__)); + SlotAccessor::NbRemainder => { + if name.as_str() == "__rmod__" { + update_sub_slot!( + as_number, + right_remainder, + number_binary_right_op_wrapper!(__rmod__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + remainder, + number_binary_op_wrapper!(__mod__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __radd__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceRemainder => { + update_sub_slot!( as_number, - right_add, - number_binary_right_op_wrapper!(__radd__) - ); + inplace_remainder, + number_binary_op_wrapper!(__imod__), + NumBinary + ) } - _ if name == identifier!(ctx, __iadd__) => { - toggle_sub_slot!(as_number, inplace_add, number_binary_op_wrapper!(__iadd__)); + SlotAccessor::NbDivmod => { + if name.as_str() == "__rdivmod__" { + update_sub_slot!( + as_number, + right_divmod, + number_binary_right_op_wrapper!(__rdivmod__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + divmod, + number_binary_op_wrapper!(__divmod__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __sub__) => { - toggle_sub_slot!(as_number, subtract, number_binary_op_wrapper!(__sub__)); + SlotAccessor::NbPower => { + if name.as_str() == "__rpow__" { + update_sub_slot!( + as_number, + right_power, + number_ternary_right_op_wrapper!(__rpow__), + NumTernary + ) + } else { + update_sub_slot!( + as_number, + power, + number_ternary_op_wrapper!(__pow__), + NumTernary + ) + } } - _ if name == identifier!(ctx, __rsub__) => { - toggle_sub_slot!( + SlotAccessor::NbInplacePower => { + update_sub_slot!( as_number, - right_subtract, - number_binary_right_op_wrapper!(__rsub__) - ); + inplace_power, + number_ternary_op_wrapper!(__ipow__), + NumTernary + ) + } + SlotAccessor::NbFloorDivide => { + if name.as_str() == "__rfloordiv__" { + update_sub_slot!( + as_number, + right_floor_divide, + number_binary_right_op_wrapper!(__rfloordiv__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + floor_divide, + number_binary_op_wrapper!(__floordiv__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __isub__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceFloorDivide => { + update_sub_slot!( as_number, - inplace_subtract, - number_binary_op_wrapper!(__isub__) - ); + inplace_floor_divide, + number_binary_op_wrapper!(__ifloordiv__), + NumBinary + ) } - _ if name == identifier!(ctx, __mul__) => { - toggle_sub_slot!(as_number, multiply, number_binary_op_wrapper!(__mul__)); + SlotAccessor::NbTrueDivide => { + if name.as_str() == "__rtruediv__" { + update_sub_slot!( + as_number, + right_true_divide, + number_binary_right_op_wrapper!(__rtruediv__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + true_divide, + number_binary_op_wrapper!(__truediv__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __rmul__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceTrueDivide => { + update_sub_slot!( as_number, - right_multiply, - number_binary_right_op_wrapper!(__rmul__) - ); + inplace_true_divide, + number_binary_op_wrapper!(__itruediv__), + NumBinary + ) + } + SlotAccessor::NbMatrixMultiply => { + if name.as_str() == "__rmatmul__" { + update_sub_slot!( + as_number, + right_matrix_multiply, + number_binary_right_op_wrapper!(__rmatmul__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + matrix_multiply, + number_binary_op_wrapper!(__matmul__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __imul__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceMatrixMultiply => { + update_sub_slot!( as_number, - inplace_multiply, - number_binary_op_wrapper!(__imul__) - ); + inplace_matrix_multiply, + number_binary_op_wrapper!(__imatmul__), + NumBinary + ) } - _ if name == identifier!(ctx, __mod__) => { - toggle_sub_slot!(as_number, remainder, number_binary_op_wrapper!(__mod__)); + + // === Number bitwise operations === + SlotAccessor::NbLshift => { + if name.as_str() == "__rlshift__" { + update_sub_slot!( + as_number, + right_lshift, + number_binary_right_op_wrapper!(__rlshift__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + lshift, + number_binary_op_wrapper!(__lshift__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __rmod__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceLshift => { + update_sub_slot!( as_number, - right_remainder, - number_binary_right_op_wrapper!(__rmod__) - ); + inplace_lshift, + number_binary_op_wrapper!(__ilshift__), + NumBinary + ) } - _ if name == identifier!(ctx, __imod__) => { - toggle_sub_slot!( + SlotAccessor::NbRshift => { + if name.as_str() == "__rrshift__" { + update_sub_slot!( + as_number, + right_rshift, + number_binary_right_op_wrapper!(__rrshift__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + rshift, + number_binary_op_wrapper!(__rshift__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceRshift => { + update_sub_slot!( as_number, - inplace_remainder, - number_binary_op_wrapper!(__imod__) - ); + inplace_rshift, + number_binary_op_wrapper!(__irshift__), + NumBinary + ) } - _ if name == identifier!(ctx, __divmod__) => { - toggle_sub_slot!(as_number, divmod, number_binary_op_wrapper!(__divmod__)); + SlotAccessor::NbAnd => { + if name.as_str() == "__rand__" { + update_sub_slot!( + as_number, + right_and, + number_binary_right_op_wrapper!(__rand__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + and, + number_binary_op_wrapper!(__and__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __rdivmod__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceAnd => { + update_sub_slot!( as_number, - right_divmod, - number_binary_right_op_wrapper!(__rdivmod__) - ); + inplace_and, + number_binary_op_wrapper!(__iand__), + NumBinary + ) } - _ if name == identifier!(ctx, __pow__) => { - toggle_sub_slot!(as_number, power, |a, b, c, vm| { - let args = if vm.is_none(c) { - vec![b.to_owned()] - } else { - vec![b.to_owned(), c.to_owned()] - }; - vm.call_special_method(a, identifier!(vm, __pow__), args) - }); - } - _ if name == identifier!(ctx, __rpow__) => { - toggle_sub_slot!(as_number, right_power, |a, b, c, vm| { - let args = if vm.is_none(c) { - vec![a.to_owned()] - } else { - vec![a.to_owned(), c.to_owned()] - }; - vm.call_special_method(b, identifier!(vm, __rpow__), args) - }); + SlotAccessor::NbXor => { + if name.as_str() == "__rxor__" { + update_sub_slot!( + as_number, + right_xor, + number_binary_right_op_wrapper!(__rxor__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + xor, + number_binary_op_wrapper!(__xor__), + NumBinary + ) + } } - _ if name == identifier!(ctx, __ipow__) => { - toggle_sub_slot!(as_number, inplace_power, |a, b, _, vm| { - vm.call_special_method(a, identifier!(vm, __ipow__), (b.to_owned(),)) - }); + SlotAccessor::NbInplaceXor => { + update_sub_slot!( + as_number, + inplace_xor, + number_binary_op_wrapper!(__ixor__), + NumBinary + ) } - _ if name == identifier!(ctx, __lshift__) => { - toggle_sub_slot!(as_number, lshift, number_binary_op_wrapper!(__lshift__)); + SlotAccessor::NbOr => { + if name.as_str() == "__ror__" { + update_sub_slot!( + as_number, + right_or, + number_binary_right_op_wrapper!(__ror__), + NumBinary + ) + } else { + update_sub_slot!(as_number, or, number_binary_op_wrapper!(__or__), NumBinary) + } } - _ if name == identifier!(ctx, __rlshift__) => { - toggle_sub_slot!( + SlotAccessor::NbInplaceOr => { + update_sub_slot!( as_number, - right_lshift, - number_binary_right_op_wrapper!(__rlshift__) - ); + inplace_or, + number_binary_op_wrapper!(__ior__), + NumBinary + ) } - _ if name == identifier!(ctx, __ilshift__) => { - toggle_sub_slot!( + + // === Number unary operations === + SlotAccessor::NbNegative => { + update_sub_slot!( as_number, - inplace_lshift, - number_binary_op_wrapper!(__ilshift__) - ); - } - _ if name == identifier!(ctx, __rshift__) => { - toggle_sub_slot!(as_number, rshift, number_binary_op_wrapper!(__rshift__)); + negative, + number_unary_op_wrapper!(__neg__), + NumUnary + ) } - _ if name == identifier!(ctx, __rrshift__) => { - toggle_sub_slot!( + SlotAccessor::NbPositive => { + update_sub_slot!( as_number, - right_rshift, - number_binary_right_op_wrapper!(__rrshift__) - ); + positive, + number_unary_op_wrapper!(__pos__), + NumUnary + ) } - _ if name == identifier!(ctx, __irshift__) => { - toggle_sub_slot!( + SlotAccessor::NbAbsolute => { + update_sub_slot!( as_number, - inplace_rshift, - number_binary_op_wrapper!(__irshift__) - ); - } - _ if name == identifier!(ctx, __and__) => { - toggle_sub_slot!(as_number, and, number_binary_op_wrapper!(__and__)); + absolute, + number_unary_op_wrapper!(__abs__), + NumUnary + ) } - _ if name == identifier!(ctx, __rand__) => { - toggle_sub_slot!( + SlotAccessor::NbInvert => { + update_sub_slot!( as_number, - right_and, - number_binary_right_op_wrapper!(__rand__) - ); + invert, + number_unary_op_wrapper!(__invert__), + NumUnary + ) } - _ if name == identifier!(ctx, __iand__) => { - toggle_sub_slot!(as_number, inplace_and, number_binary_op_wrapper!(__iand__)); + SlotAccessor::NbBool => { + update_sub_slot!(as_number, boolean, bool_wrapper, NumBoolean) } - _ if name == identifier!(ctx, __xor__) => { - toggle_sub_slot!(as_number, xor, number_binary_op_wrapper!(__xor__)); + SlotAccessor::NbInt => { + update_sub_slot!(as_number, int, number_unary_op_wrapper!(__int__), NumUnary) } - _ if name == identifier!(ctx, __rxor__) => { - toggle_sub_slot!( + SlotAccessor::NbFloat => { + update_sub_slot!( as_number, - right_xor, - number_binary_right_op_wrapper!(__rxor__) - ); - } - _ if name == identifier!(ctx, __ixor__) => { - toggle_sub_slot!(as_number, inplace_xor, number_binary_op_wrapper!(__ixor__)); - } - _ if name == identifier!(ctx, __or__) => { - toggle_sub_slot!(as_number, or, number_binary_op_wrapper!(__or__)); + float, + number_unary_op_wrapper!(__float__), + NumUnary + ) } - _ if name == identifier!(ctx, __ror__) => { - toggle_sub_slot!( + SlotAccessor::NbIndex => { + update_sub_slot!( as_number, - right_or, - number_binary_right_op_wrapper!(__ror__) - ); + index, + number_unary_op_wrapper!(__index__), + NumUnary + ) } - _ if name == identifier!(ctx, __ior__) => { - toggle_sub_slot!(as_number, inplace_or, number_binary_op_wrapper!(__ior__)); + + // === Sequence slots === + SlotAccessor::SqLength => { + update_sub_slot!(as_sequence, length, sequence_len_wrapper, SeqLength) } - _ if name == identifier!(ctx, __floordiv__) => { - toggle_sub_slot!( - as_number, - floor_divide, - number_binary_op_wrapper!(__floordiv__) - ); + SlotAccessor::SqConcat | SlotAccessor::SqInplaceConcat => { + // Sequence concat uses sq_concat slot - no generic wrapper needed + // (handled by number protocol fallback) + if !ADD { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __rfloordiv__) => { - toggle_sub_slot!( - as_number, - right_floor_divide, - number_binary_right_op_wrapper!(__rfloordiv__) - ); + SlotAccessor::SqRepeat => { + update_sub_slot!(as_sequence, repeat, sequence_repeat_wrapper, SeqRepeat) } - _ if name == identifier!(ctx, __ifloordiv__) => { - toggle_sub_slot!( - as_number, - inplace_floor_divide, - number_binary_op_wrapper!(__ifloordiv__) - ); + SlotAccessor::SqInplaceRepeat => { + update_sub_slot!( + as_sequence, + inplace_repeat, + sequence_inplace_repeat_wrapper, + SeqRepeat + ) } - _ if name == identifier!(ctx, __truediv__) => { - toggle_sub_slot!( - as_number, - true_divide, - number_binary_op_wrapper!(__truediv__) - ); + SlotAccessor::SqItem => { + update_sub_slot!(as_sequence, item, sequence_getitem_wrapper, SeqItem) } - _ if name == identifier!(ctx, __rtruediv__) => { - toggle_sub_slot!( - as_number, - right_true_divide, - number_binary_right_op_wrapper!(__rtruediv__) - ); + SlotAccessor::SqAssItem => { + // SqAssItem is shared by __setitem__ (SeqSetItem) and __delitem__ (SeqDelItem) + if ADD { + let has_own = { + let guard = self.attributes.read(); + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + guard.contains_key(setitem) || guard.contains_key(delitem) + }; + if has_own { + self.slots + .as_sequence + .ass_item + .store(Some(sequence_setitem_wrapper)); + } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::SeqSetItem(f) | SlotFunc::SeqDelItem(f) => Some(*f), + _ => None, + }) { + self.slots.as_sequence.ass_item.store(Some(func)); + } else { + self.slots + .as_sequence + .ass_item + .store(Some(sequence_setitem_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } - _ if name == identifier!(ctx, __itruediv__) => { - toggle_sub_slot!( - as_number, - inplace_true_divide, - number_binary_op_wrapper!(__itruediv__) - ); + SlotAccessor::SqContains => { + update_sub_slot!( + as_sequence, + contains, + sequence_contains_wrapper, + SeqContains + ) } - _ if name == identifier!(ctx, __matmul__) => { - toggle_sub_slot!( - as_number, - matrix_multiply, - number_binary_op_wrapper!(__matmul__) - ); + + // === Mapping slots === + SlotAccessor::MpLength => { + update_sub_slot!(as_mapping, length, mapping_len_wrapper, MapLength) } - _ if name == identifier!(ctx, __rmatmul__) => { - toggle_sub_slot!( - as_number, - right_matrix_multiply, - number_binary_right_op_wrapper!(__rmatmul__) - ); + SlotAccessor::MpSubscript => { + update_sub_slot!(as_mapping, subscript, mapping_getitem_wrapper, MapSubscript) } - _ if name == identifier!(ctx, __imatmul__) => { - toggle_sub_slot!( - as_number, - inplace_matrix_multiply, - number_binary_op_wrapper!(__imatmul__) - ); + SlotAccessor::MpAssSubscript => { + // MpAssSubscript is shared by __setitem__ (MapSetSubscript) and __delitem__ (MapDelSubscript) + if ADD { + let has_own = { + let guard = self.attributes.read(); + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + guard.contains_key(setitem) || guard.contains_key(delitem) + }; + if has_own { + self.slots + .as_mapping + .ass_subscript + .store(Some(mapping_setitem_wrapper)); + } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::MapSetSubscript(f) | SlotFunc::MapDelSubscript(f) => Some(*f), + _ => None, + }) { + self.slots.as_mapping.ass_subscript.store(Some(func)); + } else { + self.slots + .as_mapping + .ass_subscript + .store(Some(mapping_setitem_wrapper)); + } + } else { + accessor.inherit_from_mro(self); + } } + + // Reserved slots - no-op _ => {} } } + + /// Look up a method in MRO and extract the slot function if it's a slot wrapper. + /// Returns Some(slot_func) if a matching slot wrapper is found, None if a real method + /// is found or no method exists. + fn lookup_slot_in_mro<T: Copy>( + &self, + name: &'static PyStrInterned, + ctx: &Context, + extract: impl Fn(&crate::builtins::descriptor::SlotFunc) -> Option<T>, + ) -> Option<T> { + use crate::builtins::descriptor::PyWrapper; + + // Helper to check if a class is a subclass of another by checking MRO + let is_subclass_of = |subclass_mro: &[PyRef<PyType>], superclass: &Py<PyType>| -> bool { + subclass_mro.iter().any(|c| c.is(superclass)) + }; + + // Helper to extract slot from an attribute if it's a wrapper descriptor + // and the wrapper's type is compatible with the given class. + // bpo-37619: wrapper descriptor from wrong class should not be used directly. + let try_extract = |attr: &PyObjectRef, for_class_mro: &[PyRef<PyType>]| -> Option<T> { + if attr.class().is(ctx.types.wrapper_descriptor_type) { + attr.downcast_ref::<PyWrapper>().and_then(|wrapper| { + // Only extract slot if for_class is a subclass of wrapper.typ + if is_subclass_of(for_class_mro, wrapper.typ) { + extract(&wrapper.wrapped) + } else { + None + } + }) + } else { + None + } + }; + + let mro = self.mro.read(); + + // Look up in self's dict first + if let Some(attr) = self.attributes.read().get(name).cloned() { + if let Some(func) = try_extract(&attr, &mro) { + return Some(func); + } + return None; + } + + // Look up in MRO (mro[0] is self, so skip it) + for (i, cls) in mro[1..].iter().enumerate() { + if let Some(attr) = cls.attributes.read().get(name).cloned() { + // Use the slice starting from this class in MRO + if let Some(func) = try_extract(&attr, &mro[i + 1..]) { + return Some(func); + } + return None; + } + } + // No method found in MRO + None + } } /// Trait for types that can be constructed via Python's `__new__` method. @@ -893,7 +1488,7 @@ impl PyType { /// - Special class type handling (e.g., `PyType` and its metaclasses) /// - Post-creation mutations that require `PyRef` #[pyclass] -pub trait Constructor: PyPayload + std::fmt::Debug { +pub trait Constructor: PyPayload + core::fmt::Debug { type Args: FromArgs; /// The type slot for `__new__`. Override this only when you need special @@ -911,7 +1506,7 @@ pub trait Constructor: PyPayload + std::fmt::Debug { fn py_new(cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self>; } -pub trait DefaultConstructor: PyPayload + Default + std::fmt::Debug { +pub trait DefaultConstructor: PyPayload + Default + core::fmt::Debug { fn construct_and_init(args: Self::Args, vm: &VirtualMachine) -> PyResult<PyRef<Self>> where Self: Initializer, @@ -922,15 +1517,6 @@ pub trait DefaultConstructor: PyPayload + Default + std::fmt::Debug { } } -/// For types that cannot be instantiated through Python code. -#[pyclass] -pub trait Unconstructible: PyPayload { - #[pyslot] - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot create '{}' instances", cls.slot_name()))) - } -} - impl<T> Constructor for T where T: DefaultConstructor, @@ -952,7 +1538,6 @@ pub trait Initializer: PyPayload { #[inline] #[pyslot] - #[pymethod(name = "__init__")] fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { #[cfg(debug_assertions)] let class_name_for_debug = zelf.class().name().to_string(); @@ -1105,12 +1690,6 @@ pub trait Hashable: PyPayload { Self::hash(zelf, vm) } - #[inline] - #[pymethod] - fn __hash__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { - Self::slot_hash(&zelf, vm) - } - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash>; } @@ -1125,12 +1704,6 @@ pub trait Representable: PyPayload { Self::repr(zelf, vm) } - #[inline] - #[pymethod] - fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { - Self::slot_repr(&zelf, vm) - } - #[inline] fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { let repr = Self::repr_str(zelf, vm)?; @@ -1254,13 +1827,13 @@ impl PyComparisonOp { } } - pub const fn eval_ord(self, ord: Ordering) -> bool { + pub fn eval_ord(self, ord: Ordering) -> bool { let bit = match ord { Ordering::Less => Self::Lt, Ordering::Equal => Self::Eq, Ordering::Greater => Self::Gt, }; - self.0 as u8 & bit.0 as u8 != 0 + u8::from(self.0) & u8::from(bit.0) != 0 } pub const fn swapped(self) -> Self { @@ -1397,24 +1970,30 @@ pub trait AsBuffer: PyPayload { #[pyclass] pub trait AsMapping: PyPayload { - #[pyslot] fn as_mapping() -> &'static PyMappingMethods; #[inline] fn mapping_downcast(mapping: PyMapping<'_>) -> &Py<Self> { unsafe { mapping.obj.downcast_unchecked_ref() } } + + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_mapping.copy_from(Self::as_mapping()); + } } #[pyclass] pub trait AsSequence: PyPayload { - #[pyslot] fn as_sequence() -> &'static PySequenceMethods; #[inline] fn sequence_downcast(seq: PySequence<'_>) -> &Py<Self> { unsafe { seq.obj.downcast_unchecked_ref() } } + + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_sequence.copy_from(Self::as_sequence()); + } } #[pyclass] @@ -1422,6 +2001,10 @@ pub trait AsNumber: PyPayload { #[pyslot] fn as_number() -> &'static PyNumberMethods; + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_number.copy_from(Self::as_number()); + } + fn clone_exact(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyRef<Self> { // not all AsNumber requires this implementation. unimplemented!() @@ -1429,7 +2012,7 @@ pub trait AsNumber: PyPayload { #[inline] fn number_downcast(num: PyNumber<'_>) -> &Py<Self> { - unsafe { num.obj().downcast_unchecked_ref() } + unsafe { num.obj.downcast_unchecked_ref() } } #[inline] @@ -1452,17 +2035,12 @@ pub trait Iterable: PyPayload { Self::iter(zelf, vm) } - #[pymethod] - fn __iter__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Self::slot_iter(zelf, vm) - } - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult; fn extend_slots(_slots: &mut PyTypeSlots) {} } -// `Iterator` fits better, but to avoid confusion with rust std::iter::Iterator +// `Iterator` fits better, but to avoid confusion with rust core::iter::Iterator #[pyclass(with(Iterable))] pub trait IterNext: PyPayload + Iterable { #[pyslot] @@ -1474,12 +2052,6 @@ pub trait IterNext: PyPayload + Iterable { } fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn>; - - #[inline] - #[pymethod] - fn __next__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Self::slot_iternext(&zelf, vm).to_pyresult(vm) - } } pub trait SelfIter: PyPayload {} @@ -1494,10 +2066,6 @@ where unreachable!("slot must be overridden for {}", repr.as_str()); } - fn __iter__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self_iter(zelf, vm) - } - #[cold] fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { unreachable!("slot_iter is implemented"); diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs new file mode 100644 index 00000000000..024776f7893 --- /dev/null +++ b/crates/vm/src/types/slot_defs.rs @@ -0,0 +1,1497 @@ +//! Slot definitions array +//! +//! This module provides a centralized array of all slot definitions, + +use super::{PyComparisonOp, PyTypeSlots}; +use crate::builtins::descriptor::SlotFunc; + +/// Slot operation type +/// +/// Used to distinguish between different operations that share the same slot: +/// - RichCompare: Lt, Le, Eq, Ne, Gt, Ge +/// - Binary ops: Left (__add__) vs Right (__radd__) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SlotOp { + // RichCompare operations + Lt, + Le, + Eq, + Ne, + Gt, + Ge, + // Binary operation direction + Left, + Right, + // Setter vs Deleter + Delete, +} + +impl SlotOp { + /// Convert to PyComparisonOp if this is a comparison operation + pub fn as_compare_op(&self) -> Option<PyComparisonOp> { + match self { + Self::Lt => Some(PyComparisonOp::Lt), + Self::Le => Some(PyComparisonOp::Le), + Self::Eq => Some(PyComparisonOp::Eq), + Self::Ne => Some(PyComparisonOp::Ne), + Self::Gt => Some(PyComparisonOp::Gt), + Self::Ge => Some(PyComparisonOp::Ge), + _ => None, + } + } + + /// Check if this is a right operation (__radd__, __rsub__, etc.) + pub fn is_right(&self) -> bool { + matches!(self, Self::Right) + } +} + +/// Slot definition entry +#[derive(Clone, Copy)] +pub struct SlotDef { + /// Method name ("__init__", "__add__", etc.) + pub name: &'static str, + + /// Slot accessor (which slot field to access) + pub accessor: SlotAccessor, + + /// Operation type (for shared slots like RichCompare, binary ops) + pub op: Option<SlotOp>, + + /// Documentation string + pub doc: &'static str, +} + +/// Slot accessor +/// +/// Values match CPython's Py_* slot IDs from typeslots.h. +/// Unused slots are included for value reservation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum SlotAccessor { + // Buffer protocol (1-2) - Reserved, not used in RustPython + BfGetBuffer = 1, + BfReleaseBuffer = 2, + + // Mapping protocol (3-5) + MpAssSubscript = 3, + MpLength = 4, + MpSubscript = 5, + + // Number protocol (6-38) + NbAbsolute = 6, + NbAdd = 7, + NbAnd = 8, + NbBool = 9, + NbDivmod = 10, + NbFloat = 11, + NbFloorDivide = 12, + NbIndex = 13, + NbInplaceAdd = 14, + NbInplaceAnd = 15, + NbInplaceFloorDivide = 16, + NbInplaceLshift = 17, + NbInplaceMultiply = 18, + NbInplaceOr = 19, + NbInplacePower = 20, + NbInplaceRemainder = 21, + NbInplaceRshift = 22, + NbInplaceSubtract = 23, + NbInplaceTrueDivide = 24, + NbInplaceXor = 25, + NbInt = 26, + NbInvert = 27, + NbLshift = 28, + NbMultiply = 29, + NbNegative = 30, + NbOr = 31, + NbPositive = 32, + NbPower = 33, + NbRemainder = 34, + NbRshift = 35, + NbSubtract = 36, + NbTrueDivide = 37, + NbXor = 38, + + // Sequence protocol (39-46) + SqAssItem = 39, + SqConcat = 40, + SqContains = 41, + SqInplaceConcat = 42, + SqInplaceRepeat = 43, + SqItem = 44, + SqLength = 45, + SqRepeat = 46, + + // Type slots (47-74) + TpAlloc = 47, // Reserved + TpBase = 48, // Reserved + TpBases = 49, // Reserved + TpCall = 50, + TpClear = 51, // Reserved + TpDealloc = 52, // Reserved + TpDel = 53, + TpDescrGet = 54, + TpDescrSet = 55, + TpDoc = 56, // Reserved + TpGetattr = 57, // Reserved (use TpGetattro) + TpGetattro = 58, + TpHash = 59, + TpInit = 60, + TpIsGc = 61, // Reserved + TpIter = 62, + TpIternext = 63, + TpMethods = 64, // Reserved + TpNew = 65, + TpRepr = 66, + TpRichcompare = 67, + TpSetattr = 68, // Reserved (use TpSetattro) + TpSetattro = 69, + TpStr = 70, + TpTraverse = 71, // Reserved + TpMembers = 72, // Reserved + TpGetset = 73, // Reserved + TpFree = 74, // Reserved + + // Number protocol additions (75-76) + NbMatrixMultiply = 75, + NbInplaceMatrixMultiply = 76, + + // Async protocol (77-81) - Reserved for future + AmAwait = 77, + AmAiter = 78, + AmAnext = 79, + TpFinalize = 80, + AmSend = 81, +} + +impl SlotAccessor { + /// Check if this accessor is for a reserved/unused slot + pub fn is_reserved(&self) -> bool { + matches!( + self, + Self::BfGetBuffer + | Self::BfReleaseBuffer + | Self::TpAlloc + | Self::TpBase + | Self::TpBases + | Self::TpClear + | Self::TpDealloc + | Self::TpDoc + | Self::TpGetattr + | Self::TpIsGc + | Self::TpMethods + | Self::TpSetattr + | Self::TpTraverse + | Self::TpMembers + | Self::TpGetset + | Self::TpFree + | Self::TpFinalize + | Self::AmAwait + | Self::AmAiter + | Self::AmAnext + | Self::AmSend + ) + } + + /// Check if this is a number binary operation slot + pub fn is_number_binary(&self) -> bool { + matches!( + self, + Self::NbAdd + | Self::NbSubtract + | Self::NbMultiply + | Self::NbRemainder + | Self::NbDivmod + | Self::NbPower + | Self::NbLshift + | Self::NbRshift + | Self::NbAnd + | Self::NbXor + | Self::NbOr + | Self::NbFloorDivide + | Self::NbTrueDivide + | Self::NbMatrixMultiply + ) + } + + /// Check if this accessor refers to a shared slot + /// + /// Shared slots are used by multiple dunder methods: + /// - TpSetattro: __setattr__ and __delattr__ + /// - TpRichcompare: __lt__, __le__, __eq__, __ne__, __gt__, __ge__ + /// - TpDescrSet: __set__ and __delete__ + /// - SqAssItem/MpAssSubscript: __setitem__ and __delitem__ + /// - Number binaries: __add__ and __radd__, etc. + pub fn is_shared_slot(&self) -> bool { + matches!( + self, + Self::TpSetattro + | Self::TpRichcompare + | Self::TpDescrSet + | Self::SqAssItem + | Self::MpAssSubscript + ) || self.is_number_binary() + } + + /// Get underlying slot field name for debugging + pub fn slot_name(&self) -> &'static str { + match self { + Self::BfGetBuffer => "bf_getbuffer", + Self::BfReleaseBuffer => "bf_releasebuffer", + Self::MpAssSubscript => "mp_ass_subscript", + Self::MpLength => "mp_length", + Self::MpSubscript => "mp_subscript", + Self::NbAbsolute => "nb_absolute", + Self::NbAdd => "nb_add", + Self::NbAnd => "nb_and", + Self::NbBool => "nb_bool", + Self::NbDivmod => "nb_divmod", + Self::NbFloat => "nb_float", + Self::NbFloorDivide => "nb_floor_divide", + Self::NbIndex => "nb_index", + Self::NbInplaceAdd => "nb_inplace_add", + Self::NbInplaceAnd => "nb_inplace_and", + Self::NbInplaceFloorDivide => "nb_inplace_floor_divide", + Self::NbInplaceLshift => "nb_inplace_lshift", + Self::NbInplaceMultiply => "nb_inplace_multiply", + Self::NbInplaceOr => "nb_inplace_or", + Self::NbInplacePower => "nb_inplace_power", + Self::NbInplaceRemainder => "nb_inplace_remainder", + Self::NbInplaceRshift => "nb_inplace_rshift", + Self::NbInplaceSubtract => "nb_inplace_subtract", + Self::NbInplaceTrueDivide => "nb_inplace_true_divide", + Self::NbInplaceXor => "nb_inplace_xor", + Self::NbInt => "nb_int", + Self::NbInvert => "nb_invert", + Self::NbLshift => "nb_lshift", + Self::NbMultiply => "nb_multiply", + Self::NbNegative => "nb_negative", + Self::NbOr => "nb_or", + Self::NbPositive => "nb_positive", + Self::NbPower => "nb_power", + Self::NbRemainder => "nb_remainder", + Self::NbRshift => "nb_rshift", + Self::NbSubtract => "nb_subtract", + Self::NbTrueDivide => "nb_true_divide", + Self::NbXor => "nb_xor", + Self::SqAssItem => "sq_ass_item", + Self::SqConcat => "sq_concat", + Self::SqContains => "sq_contains", + Self::SqInplaceConcat => "sq_inplace_concat", + Self::SqInplaceRepeat => "sq_inplace_repeat", + Self::SqItem => "sq_item", + Self::SqLength => "sq_length", + Self::SqRepeat => "sq_repeat", + Self::TpAlloc => "tp_alloc", + Self::TpBase => "tp_base", + Self::TpBases => "tp_bases", + Self::TpCall => "tp_call", + Self::TpClear => "tp_clear", + Self::TpDealloc => "tp_dealloc", + Self::TpDel => "tp_del", + Self::TpDescrGet => "tp_descr_get", + Self::TpDescrSet => "tp_descr_set", + Self::TpDoc => "tp_doc", + Self::TpGetattr => "tp_getattr", + Self::TpGetattro => "tp_getattro", + Self::TpHash => "tp_hash", + Self::TpInit => "tp_init", + Self::TpIsGc => "tp_is_gc", + Self::TpIter => "tp_iter", + Self::TpIternext => "tp_iternext", + Self::TpMethods => "tp_methods", + Self::TpNew => "tp_new", + Self::TpRepr => "tp_repr", + Self::TpRichcompare => "tp_richcompare", + Self::TpSetattr => "tp_setattr", + Self::TpSetattro => "tp_setattro", + Self::TpStr => "tp_str", + Self::TpTraverse => "tp_traverse", + Self::TpMembers => "tp_members", + Self::TpGetset => "tp_getset", + Self::TpFree => "tp_free", + Self::NbMatrixMultiply => "nb_matrix_multiply", + Self::NbInplaceMatrixMultiply => "nb_inplace_matrix_multiply", + Self::AmAwait => "am_await", + Self::AmAiter => "am_aiter", + Self::AmAnext => "am_anext", + Self::TpFinalize => "tp_finalize", + Self::AmSend => "am_send", + } + } + + /// Extract the raw function pointer from a SlotFunc if it matches this accessor's type + pub fn extract_from_slot_func(&self, slot_func: &SlotFunc) -> bool { + match self { + // Type slots + Self::TpHash => matches!(slot_func, SlotFunc::Hash(_)), + Self::TpRepr => matches!(slot_func, SlotFunc::Repr(_)), + Self::TpStr => matches!(slot_func, SlotFunc::Str(_)), + Self::TpCall => matches!(slot_func, SlotFunc::Call(_)), + Self::TpIter => matches!(slot_func, SlotFunc::Iter(_)), + Self::TpIternext => matches!(slot_func, SlotFunc::IterNext(_)), + Self::TpInit => matches!(slot_func, SlotFunc::Init(_)), + Self::TpDel => matches!(slot_func, SlotFunc::Del(_)), + Self::TpGetattro => matches!(slot_func, SlotFunc::GetAttro(_)), + Self::TpSetattro => { + matches!(slot_func, SlotFunc::SetAttro(_) | SlotFunc::DelAttro(_)) + } + Self::TpDescrGet => matches!(slot_func, SlotFunc::DescrGet(_)), + Self::TpDescrSet => { + matches!(slot_func, SlotFunc::DescrSet(_) | SlotFunc::DescrDel(_)) + } + Self::TpRichcompare => matches!(slot_func, SlotFunc::RichCompare(_, _)), + + // Number - Power (ternary) + Self::NbPower | Self::NbInplacePower => { + matches!(slot_func, SlotFunc::NumTernary(_)) + } + // Number - Boolean + Self::NbBool => matches!(slot_func, SlotFunc::NumBoolean(_)), + // Number - Unary + Self::NbNegative + | Self::NbPositive + | Self::NbAbsolute + | Self::NbInvert + | Self::NbInt + | Self::NbFloat + | Self::NbIndex => matches!(slot_func, SlotFunc::NumUnary(_)), + // Number - Binary + Self::NbAdd + | Self::NbSubtract + | Self::NbMultiply + | Self::NbRemainder + | Self::NbDivmod + | Self::NbLshift + | Self::NbRshift + | Self::NbAnd + | Self::NbXor + | Self::NbOr + | Self::NbFloorDivide + | Self::NbTrueDivide + | Self::NbMatrixMultiply + | Self::NbInplaceAdd + | Self::NbInplaceSubtract + | Self::NbInplaceMultiply + | Self::NbInplaceRemainder + | Self::NbInplaceLshift + | Self::NbInplaceRshift + | Self::NbInplaceAnd + | Self::NbInplaceXor + | Self::NbInplaceOr + | Self::NbInplaceFloorDivide + | Self::NbInplaceTrueDivide + | Self::NbInplaceMatrixMultiply => matches!(slot_func, SlotFunc::NumBinary(_)), + + // Sequence + Self::SqLength => matches!(slot_func, SlotFunc::SeqLength(_)), + Self::SqConcat | Self::SqInplaceConcat => matches!(slot_func, SlotFunc::SeqConcat(_)), + Self::SqRepeat | Self::SqInplaceRepeat => matches!(slot_func, SlotFunc::SeqRepeat(_)), + Self::SqItem => matches!(slot_func, SlotFunc::SeqItem(_)), + Self::SqAssItem => { + matches!(slot_func, SlotFunc::SeqSetItem(_) | SlotFunc::SeqDelItem(_)) + } + Self::SqContains => matches!(slot_func, SlotFunc::SeqContains(_)), + + // Mapping + Self::MpLength => matches!(slot_func, SlotFunc::MapLength(_)), + Self::MpSubscript => matches!(slot_func, SlotFunc::MapSubscript(_)), + Self::MpAssSubscript => { + matches!( + slot_func, + SlotFunc::MapSetSubscript(_) | SlotFunc::MapDelSubscript(_) + ) + } + + // New and reserved slots + Self::TpNew => false, + _ => false, // Reserved slots + } + } + + /// Inherit slot value from MRO + pub fn inherit_from_mro(&self, typ: &crate::builtins::PyType) { + // mro[0] is self, so skip it + let mro_guard = typ.mro.read(); + let mro = &mro_guard[1..]; + + macro_rules! inherit_main { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.$slot.load()); + typ.slots.$slot.store(inherited); + }}; + } + + macro_rules! inherit_number { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.as_number.$slot.load()); + typ.slots.as_number.$slot.store(inherited); + }}; + } + + macro_rules! inherit_sequence { + ($slot:ident) => {{ + let inherited = mro + .iter() + .find_map(|cls| cls.slots.as_sequence.$slot.load()); + typ.slots.as_sequence.$slot.store(inherited); + }}; + } + + macro_rules! inherit_mapping { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.as_mapping.$slot.load()); + typ.slots.as_mapping.$slot.store(inherited); + }}; + } + + match self { + // Type slots + Self::TpHash => inherit_main!(hash), + Self::TpRepr => inherit_main!(repr), + Self::TpStr => inherit_main!(str), + Self::TpCall => inherit_main!(call), + Self::TpIter => inherit_main!(iter), + Self::TpIternext => inherit_main!(iternext), + Self::TpInit => inherit_main!(init), + Self::TpNew => inherit_main!(new), + Self::TpDel => inherit_main!(del), + Self::TpGetattro => inherit_main!(getattro), + Self::TpSetattro => inherit_main!(setattro), + Self::TpDescrGet => inherit_main!(descr_get), + Self::TpDescrSet => inherit_main!(descr_set), + Self::TpRichcompare => inherit_main!(richcompare), + + // Number slots + Self::NbAdd => inherit_number!(add), + Self::NbSubtract => inherit_number!(subtract), + Self::NbMultiply => inherit_number!(multiply), + Self::NbRemainder => inherit_number!(remainder), + Self::NbDivmod => inherit_number!(divmod), + Self::NbPower => inherit_number!(power), + Self::NbLshift => inherit_number!(lshift), + Self::NbRshift => inherit_number!(rshift), + Self::NbAnd => inherit_number!(and), + Self::NbXor => inherit_number!(xor), + Self::NbOr => inherit_number!(or), + Self::NbFloorDivide => inherit_number!(floor_divide), + Self::NbTrueDivide => inherit_number!(true_divide), + Self::NbMatrixMultiply => inherit_number!(matrix_multiply), + Self::NbInplaceAdd => inherit_number!(inplace_add), + Self::NbInplaceSubtract => inherit_number!(inplace_subtract), + Self::NbInplaceMultiply => inherit_number!(inplace_multiply), + Self::NbInplaceRemainder => inherit_number!(inplace_remainder), + Self::NbInplacePower => inherit_number!(inplace_power), + Self::NbInplaceLshift => inherit_number!(inplace_lshift), + Self::NbInplaceRshift => inherit_number!(inplace_rshift), + Self::NbInplaceAnd => inherit_number!(inplace_and), + Self::NbInplaceXor => inherit_number!(inplace_xor), + Self::NbInplaceOr => inherit_number!(inplace_or), + Self::NbInplaceFloorDivide => inherit_number!(inplace_floor_divide), + Self::NbInplaceTrueDivide => inherit_number!(inplace_true_divide), + Self::NbInplaceMatrixMultiply => inherit_number!(inplace_matrix_multiply), + // Number unary + Self::NbNegative => inherit_number!(negative), + Self::NbPositive => inherit_number!(positive), + Self::NbAbsolute => inherit_number!(absolute), + Self::NbInvert => inherit_number!(invert), + Self::NbBool => inherit_number!(boolean), + Self::NbInt => inherit_number!(int), + Self::NbFloat => inherit_number!(float), + Self::NbIndex => inherit_number!(index), + + // Sequence slots + Self::SqLength => inherit_sequence!(length), + Self::SqConcat => inherit_sequence!(concat), + Self::SqRepeat => inherit_sequence!(repeat), + Self::SqItem => inherit_sequence!(item), + Self::SqAssItem => inherit_sequence!(ass_item), + Self::SqContains => inherit_sequence!(contains), + Self::SqInplaceConcat => inherit_sequence!(inplace_concat), + Self::SqInplaceRepeat => inherit_sequence!(inplace_repeat), + + // Mapping slots + Self::MpLength => inherit_mapping!(length), + Self::MpSubscript => inherit_mapping!(subscript), + Self::MpAssSubscript => inherit_mapping!(ass_subscript), + + // Reserved slots - no-op + _ => {} + } + } + + /// Copy slot from base type if self's slot is None + pub fn copyslot_if_none(&self, typ: &crate::builtins::PyType, base: &crate::builtins::PyType) { + macro_rules! copy_main { + ($slot:ident) => {{ + if typ.slots.$slot.load().is_none() { + if let Some(base_val) = base.slots.$slot.load() { + typ.slots.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_number { + ($slot:ident) => {{ + if typ.slots.as_number.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_number.$slot.load() { + typ.slots.as_number.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_sequence { + ($slot:ident) => {{ + if typ.slots.as_sequence.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_sequence.$slot.load() { + typ.slots.as_sequence.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_mapping { + ($slot:ident) => {{ + if typ.slots.as_mapping.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_mapping.$slot.load() { + typ.slots.as_mapping.$slot.store(Some(base_val)); + } + } + }}; + } + + match self { + // Type slots + Self::TpHash => copy_main!(hash), + Self::TpRepr => copy_main!(repr), + Self::TpStr => copy_main!(str), + Self::TpCall => copy_main!(call), + Self::TpIter => copy_main!(iter), + Self::TpIternext => copy_main!(iternext), + Self::TpInit => { + // SLOTDEFINED check for multiple inheritance support + if typ.slots.init.load().is_none() + && let Some(base_val) = base.slots.init.load() + { + let slot_defined = base.base.as_ref().is_none_or(|bb| { + bb.slots.init.load().map(|v| v as usize) != Some(base_val as usize) + }); + if slot_defined { + typ.slots.init.store(Some(base_val)); + } + } + } + Self::TpNew => {} // handled by set_new() + Self::TpDel => copy_main!(del), + Self::TpGetattro => copy_main!(getattro), + Self::TpSetattro => copy_main!(setattro), + Self::TpDescrGet => copy_main!(descr_get), + Self::TpDescrSet => copy_main!(descr_set), + Self::TpRichcompare => copy_main!(richcompare), + + // Number slots + Self::NbAdd => copy_number!(add), + Self::NbSubtract => copy_number!(subtract), + Self::NbMultiply => copy_number!(multiply), + Self::NbRemainder => copy_number!(remainder), + Self::NbDivmod => copy_number!(divmod), + Self::NbPower => copy_number!(power), + Self::NbLshift => copy_number!(lshift), + Self::NbRshift => copy_number!(rshift), + Self::NbAnd => copy_number!(and), + Self::NbXor => copy_number!(xor), + Self::NbOr => copy_number!(or), + Self::NbFloorDivide => copy_number!(floor_divide), + Self::NbTrueDivide => copy_number!(true_divide), + Self::NbMatrixMultiply => copy_number!(matrix_multiply), + Self::NbInplaceAdd => copy_number!(inplace_add), + Self::NbInplaceSubtract => copy_number!(inplace_subtract), + Self::NbInplaceMultiply => copy_number!(inplace_multiply), + Self::NbInplaceRemainder => copy_number!(inplace_remainder), + Self::NbInplacePower => copy_number!(inplace_power), + Self::NbInplaceLshift => copy_number!(inplace_lshift), + Self::NbInplaceRshift => copy_number!(inplace_rshift), + Self::NbInplaceAnd => copy_number!(inplace_and), + Self::NbInplaceXor => copy_number!(inplace_xor), + Self::NbInplaceOr => copy_number!(inplace_or), + Self::NbInplaceFloorDivide => copy_number!(inplace_floor_divide), + Self::NbInplaceTrueDivide => copy_number!(inplace_true_divide), + Self::NbInplaceMatrixMultiply => copy_number!(inplace_matrix_multiply), + // Number unary + Self::NbNegative => copy_number!(negative), + Self::NbPositive => copy_number!(positive), + Self::NbAbsolute => copy_number!(absolute), + Self::NbInvert => copy_number!(invert), + Self::NbBool => copy_number!(boolean), + Self::NbInt => copy_number!(int), + Self::NbFloat => copy_number!(float), + Self::NbIndex => copy_number!(index), + + // Sequence slots + Self::SqLength => copy_sequence!(length), + Self::SqConcat => copy_sequence!(concat), + Self::SqRepeat => copy_sequence!(repeat), + Self::SqItem => copy_sequence!(item), + Self::SqAssItem => copy_sequence!(ass_item), + Self::SqContains => copy_sequence!(contains), + Self::SqInplaceConcat => copy_sequence!(inplace_concat), + Self::SqInplaceRepeat => copy_sequence!(inplace_repeat), + + // Mapping slots + Self::MpLength => copy_mapping!(length), + Self::MpSubscript => copy_mapping!(subscript), + Self::MpAssSubscript => copy_mapping!(ass_subscript), + + // Reserved slots - no-op + _ => {} + } + } + + /// Get the SlotFunc from type slots for this accessor + pub fn get_slot_func(&self, slots: &PyTypeSlots) -> Option<SlotFunc> { + match self { + // Type slots + Self::TpHash => slots.hash.load().map(SlotFunc::Hash), + Self::TpRepr => slots.repr.load().map(SlotFunc::Repr), + Self::TpStr => slots.str.load().map(SlotFunc::Str), + Self::TpCall => slots.call.load().map(SlotFunc::Call), + Self::TpIter => slots.iter.load().map(SlotFunc::Iter), + Self::TpIternext => slots.iternext.load().map(SlotFunc::IterNext), + Self::TpInit => slots.init.load().map(SlotFunc::Init), + Self::TpNew => None, // __new__ handled separately + Self::TpDel => slots.del.load().map(SlotFunc::Del), + Self::TpGetattro => slots.getattro.load().map(SlotFunc::GetAttro), + Self::TpSetattro => slots.setattro.load().map(SlotFunc::SetAttro), + Self::TpDescrGet => slots.descr_get.load().map(SlotFunc::DescrGet), + Self::TpDescrSet => slots.descr_set.load().map(SlotFunc::DescrSet), + Self::TpRichcompare => slots + .richcompare + .load() + .map(|f| SlotFunc::RichCompare(f, PyComparisonOp::Eq)), + + // Number binary slots + Self::NbAdd => slots.as_number.add.load().map(SlotFunc::NumBinary), + Self::NbSubtract => slots.as_number.subtract.load().map(SlotFunc::NumBinary), + Self::NbMultiply => slots.as_number.multiply.load().map(SlotFunc::NumBinary), + Self::NbRemainder => slots.as_number.remainder.load().map(SlotFunc::NumBinary), + Self::NbDivmod => slots.as_number.divmod.load().map(SlotFunc::NumBinary), + Self::NbPower => slots.as_number.power.load().map(SlotFunc::NumTernary), + Self::NbLshift => slots.as_number.lshift.load().map(SlotFunc::NumBinary), + Self::NbRshift => slots.as_number.rshift.load().map(SlotFunc::NumBinary), + Self::NbAnd => slots.as_number.and.load().map(SlotFunc::NumBinary), + Self::NbXor => slots.as_number.xor.load().map(SlotFunc::NumBinary), + Self::NbOr => slots.as_number.or.load().map(SlotFunc::NumBinary), + Self::NbFloorDivide => slots.as_number.floor_divide.load().map(SlotFunc::NumBinary), + Self::NbTrueDivide => slots.as_number.true_divide.load().map(SlotFunc::NumBinary), + Self::NbMatrixMultiply => slots + .as_number + .matrix_multiply + .load() + .map(SlotFunc::NumBinary), + + // Number inplace slots + Self::NbInplaceAdd => slots.as_number.inplace_add.load().map(SlotFunc::NumBinary), + Self::NbInplaceSubtract => slots + .as_number + .inplace_subtract + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceMultiply => slots + .as_number + .inplace_multiply + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceRemainder => slots + .as_number + .inplace_remainder + .load() + .map(SlotFunc::NumBinary), + Self::NbInplacePower => slots + .as_number + .inplace_power + .load() + .map(SlotFunc::NumTernary), + Self::NbInplaceLshift => slots + .as_number + .inplace_lshift + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceRshift => slots + .as_number + .inplace_rshift + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceAnd => slots.as_number.inplace_and.load().map(SlotFunc::NumBinary), + Self::NbInplaceXor => slots.as_number.inplace_xor.load().map(SlotFunc::NumBinary), + Self::NbInplaceOr => slots.as_number.inplace_or.load().map(SlotFunc::NumBinary), + Self::NbInplaceFloorDivide => slots + .as_number + .inplace_floor_divide + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceTrueDivide => slots + .as_number + .inplace_true_divide + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceMatrixMultiply => slots + .as_number + .inplace_matrix_multiply + .load() + .map(SlotFunc::NumBinary), + + // Number unary slots + Self::NbNegative => slots.as_number.negative.load().map(SlotFunc::NumUnary), + Self::NbPositive => slots.as_number.positive.load().map(SlotFunc::NumUnary), + Self::NbAbsolute => slots.as_number.absolute.load().map(SlotFunc::NumUnary), + Self::NbInvert => slots.as_number.invert.load().map(SlotFunc::NumUnary), + Self::NbBool => slots.as_number.boolean.load().map(SlotFunc::NumBoolean), + Self::NbInt => slots.as_number.int.load().map(SlotFunc::NumUnary), + Self::NbFloat => slots.as_number.float.load().map(SlotFunc::NumUnary), + Self::NbIndex => slots.as_number.index.load().map(SlotFunc::NumUnary), + + // Sequence slots + Self::SqLength => slots.as_sequence.length.load().map(SlotFunc::SeqLength), + Self::SqConcat => slots.as_sequence.concat.load().map(SlotFunc::SeqConcat), + Self::SqRepeat => slots.as_sequence.repeat.load().map(SlotFunc::SeqRepeat), + Self::SqItem => slots.as_sequence.item.load().map(SlotFunc::SeqItem), + Self::SqAssItem => slots.as_sequence.ass_item.load().map(SlotFunc::SeqSetItem), + Self::SqContains => slots.as_sequence.contains.load().map(SlotFunc::SeqContains), + Self::SqInplaceConcat => slots + .as_sequence + .inplace_concat + .load() + .map(SlotFunc::SeqConcat), + Self::SqInplaceRepeat => slots + .as_sequence + .inplace_repeat + .load() + .map(SlotFunc::SeqRepeat), + + // Mapping slots + Self::MpLength => slots.as_mapping.length.load().map(SlotFunc::MapLength), + Self::MpSubscript => slots + .as_mapping + .subscript + .load() + .map(SlotFunc::MapSubscript), + Self::MpAssSubscript => slots + .as_mapping + .ass_subscript + .load() + .map(SlotFunc::MapSetSubscript), + + // Reserved slots + _ => None, + } + } + + /// Get slot function considering SlotOp for right-hand and delete operations + pub fn get_slot_func_with_op( + &self, + slots: &PyTypeSlots, + op: Option<SlotOp>, + ) -> Option<SlotFunc> { + // For Delete operations, return the delete variant + if op == Some(SlotOp::Delete) { + match self { + Self::TpSetattro => return slots.setattro.load().map(SlotFunc::DelAttro), + Self::TpDescrSet => return slots.descr_set.load().map(SlotFunc::DescrDel), + Self::SqAssItem => { + return slots.as_sequence.ass_item.load().map(SlotFunc::SeqDelItem); + } + Self::MpAssSubscript => { + return slots + .as_mapping + .ass_subscript + .load() + .map(SlotFunc::MapDelSubscript); + } + _ => {} + } + } + // For Right operations on binary number slots, use right_* fields with swapped args + if op == Some(SlotOp::Right) { + match self { + Self::NbAdd => { + return slots + .as_number + .right_add + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbSubtract => { + return slots + .as_number + .right_subtract + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbMultiply => { + return slots + .as_number + .right_multiply + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbRemainder => { + return slots + .as_number + .right_remainder + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbDivmod => { + return slots + .as_number + .right_divmod + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbPower => { + return slots + .as_number + .right_power + .load() + .map(SlotFunc::NumTernaryRight); + } + Self::NbLshift => { + return slots + .as_number + .right_lshift + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbRshift => { + return slots + .as_number + .right_rshift + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbAnd => { + return slots + .as_number + .right_and + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbXor => { + return slots + .as_number + .right_xor + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbOr => { + return slots + .as_number + .right_or + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbFloorDivide => { + return slots + .as_number + .right_floor_divide + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbTrueDivide => { + return slots + .as_number + .right_true_divide + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbMatrixMultiply => { + return slots + .as_number + .right_matrix_multiply + .load() + .map(SlotFunc::NumBinaryRight); + } + _ => {} + } + } + // For comparison operations, use the appropriate PyComparisonOp + if let Self::TpRichcompare = self + && let Some(cmp_op) = op.and_then(|o| o.as_compare_op()) + { + return slots + .richcompare + .load() + .map(|f| SlotFunc::RichCompare(f, cmp_op)); + } + // Fall back to existing get_slot_func for left/other operations + self.get_slot_func(slots) + } +} + +/// Find all slot definitions with a given name +pub fn find_slot_defs_by_name(name: &str) -> impl Iterator<Item = &'static SlotDef> { + SLOT_DEFS.iter().filter(move |def| def.name == name) +} + +/// Total number of slot definitions +pub const SLOT_DEFS_COUNT: usize = SLOT_DEFS.len(); + +/// All slot definitions +pub static SLOT_DEFS: &[SlotDef] = &[ + // Type slots (tp_*) + SlotDef { + name: "__init__", + accessor: SlotAccessor::TpInit, + op: None, + doc: "Initialize self. See help(type(self)) for accurate signature.", + }, + SlotDef { + name: "__new__", + accessor: SlotAccessor::TpNew, + op: None, + doc: "Create and return a new object. See help(type) for accurate signature.", + }, + SlotDef { + name: "__del__", + accessor: SlotAccessor::TpDel, + op: None, + doc: "Called when the instance is about to be destroyed.", + }, + SlotDef { + name: "__repr__", + accessor: SlotAccessor::TpRepr, + op: None, + doc: "Return repr(self).", + }, + SlotDef { + name: "__str__", + accessor: SlotAccessor::TpStr, + op: None, + doc: "Return str(self).", + }, + SlotDef { + name: "__hash__", + accessor: SlotAccessor::TpHash, + op: None, + doc: "Return hash(self).", + }, + SlotDef { + name: "__call__", + accessor: SlotAccessor::TpCall, + op: None, + doc: "Call self as a function.", + }, + SlotDef { + name: "__iter__", + accessor: SlotAccessor::TpIter, + op: None, + doc: "Implement iter(self).", + }, + SlotDef { + name: "__next__", + accessor: SlotAccessor::TpIternext, + op: None, + doc: "Implement next(self).", + }, + // Attribute access + SlotDef { + name: "__getattribute__", + accessor: SlotAccessor::TpGetattro, + op: None, + doc: "Return getattr(self, name).", + }, + SlotDef { + name: "__getattr__", + accessor: SlotAccessor::TpGetattro, + op: None, + doc: "Implement getattr(self, name).", + }, + SlotDef { + name: "__setattr__", + accessor: SlotAccessor::TpSetattro, + op: None, + doc: "Implement setattr(self, name, value).", + }, + SlotDef { + name: "__delattr__", + accessor: SlotAccessor::TpSetattro, + op: Some(SlotOp::Delete), + doc: "Implement delattr(self, name).", + }, + // Rich comparison - all map to TpRichcompare with different op + SlotDef { + name: "__eq__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Eq), + doc: "Return self==value.", + }, + SlotDef { + name: "__ne__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Ne), + doc: "Return self!=value.", + }, + SlotDef { + name: "__lt__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Lt), + doc: "Return self<value.", + }, + SlotDef { + name: "__le__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Le), + doc: "Return self<=value.", + }, + SlotDef { + name: "__gt__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Gt), + doc: "Return self>value.", + }, + SlotDef { + name: "__ge__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Ge), + doc: "Return self>=value.", + }, + // Descriptor protocol + SlotDef { + name: "__get__", + accessor: SlotAccessor::TpDescrGet, + op: None, + doc: "Return an attribute of instance, which is of type owner.", + }, + SlotDef { + name: "__set__", + accessor: SlotAccessor::TpDescrSet, + op: None, + doc: "Set an attribute of instance to value.", + }, + SlotDef { + name: "__delete__", + accessor: SlotAccessor::TpDescrSet, + op: Some(SlotOp::Delete), + doc: "Delete an attribute of instance.", + }, + // Mapping protocol (mp_*) - must come before Sequence protocol + // so that mp_subscript wins over sq_item for __getitem__ + // (see CPython typeobject.c:10995-11006) + SlotDef { + name: "__len__", + accessor: SlotAccessor::MpLength, + op: None, + doc: "Return len(self).", + }, + SlotDef { + name: "__getitem__", + accessor: SlotAccessor::MpSubscript, + op: None, + doc: "Return self[key].", + }, + SlotDef { + name: "__setitem__", + accessor: SlotAccessor::MpAssSubscript, + op: None, + doc: "Set self[key] to value.", + }, + SlotDef { + name: "__delitem__", + accessor: SlotAccessor::MpAssSubscript, + op: Some(SlotOp::Delete), + doc: "Delete self[key].", + }, + // Sequence protocol (sq_*) + SlotDef { + name: "__len__", + accessor: SlotAccessor::SqLength, + op: None, + doc: "Return len(self).", + }, + SlotDef { + name: "__getitem__", + accessor: SlotAccessor::SqItem, + op: None, + doc: "Return self[key].", + }, + SlotDef { + name: "__setitem__", + accessor: SlotAccessor::SqAssItem, + op: None, + doc: "Set self[key] to value.", + }, + SlotDef { + name: "__delitem__", + accessor: SlotAccessor::SqAssItem, + op: Some(SlotOp::Delete), + doc: "Delete self[key].", + }, + SlotDef { + name: "__contains__", + accessor: SlotAccessor::SqContains, + op: None, + doc: "Return key in self.", + }, + // Number protocol - binary ops with left/right variants + SlotDef { + name: "__add__", + accessor: SlotAccessor::NbAdd, + op: Some(SlotOp::Left), + doc: "Return self+value.", + }, + SlotDef { + name: "__radd__", + accessor: SlotAccessor::NbAdd, + op: Some(SlotOp::Right), + doc: "Return value+self.", + }, + SlotDef { + name: "__iadd__", + accessor: SlotAccessor::NbInplaceAdd, + op: None, + doc: "Implement self+=value.", + }, + SlotDef { + name: "__sub__", + accessor: SlotAccessor::NbSubtract, + op: Some(SlotOp::Left), + doc: "Return self-value.", + }, + SlotDef { + name: "__rsub__", + accessor: SlotAccessor::NbSubtract, + op: Some(SlotOp::Right), + doc: "Return value-self.", + }, + SlotDef { + name: "__isub__", + accessor: SlotAccessor::NbInplaceSubtract, + op: None, + doc: "Implement self-=value.", + }, + SlotDef { + name: "__mul__", + accessor: SlotAccessor::NbMultiply, + op: Some(SlotOp::Left), + doc: "Return self*value.", + }, + SlotDef { + name: "__rmul__", + accessor: SlotAccessor::NbMultiply, + op: Some(SlotOp::Right), + doc: "Return value*self.", + }, + SlotDef { + name: "__imul__", + accessor: SlotAccessor::NbInplaceMultiply, + op: None, + doc: "Implement self*=value.", + }, + SlotDef { + name: "__mod__", + accessor: SlotAccessor::NbRemainder, + op: Some(SlotOp::Left), + doc: "Return self%value.", + }, + SlotDef { + name: "__rmod__", + accessor: SlotAccessor::NbRemainder, + op: Some(SlotOp::Right), + doc: "Return value%self.", + }, + SlotDef { + name: "__imod__", + accessor: SlotAccessor::NbInplaceRemainder, + op: None, + doc: "Implement self%=value.", + }, + SlotDef { + name: "__divmod__", + accessor: SlotAccessor::NbDivmod, + op: Some(SlotOp::Left), + doc: "Return divmod(self, value).", + }, + SlotDef { + name: "__rdivmod__", + accessor: SlotAccessor::NbDivmod, + op: Some(SlotOp::Right), + doc: "Return divmod(value, self).", + }, + SlotDef { + name: "__pow__", + accessor: SlotAccessor::NbPower, + op: Some(SlotOp::Left), + doc: "Return pow(self, value, mod).", + }, + SlotDef { + name: "__rpow__", + accessor: SlotAccessor::NbPower, + op: Some(SlotOp::Right), + doc: "Return pow(value, self, mod).", + }, + SlotDef { + name: "__ipow__", + accessor: SlotAccessor::NbInplacePower, + op: None, + doc: "Implement self**=value.", + }, + SlotDef { + name: "__lshift__", + accessor: SlotAccessor::NbLshift, + op: Some(SlotOp::Left), + doc: "Return self<<value.", + }, + SlotDef { + name: "__rlshift__", + accessor: SlotAccessor::NbLshift, + op: Some(SlotOp::Right), + doc: "Return value<<self.", + }, + SlotDef { + name: "__ilshift__", + accessor: SlotAccessor::NbInplaceLshift, + op: None, + doc: "Implement self<<=value.", + }, + SlotDef { + name: "__rshift__", + accessor: SlotAccessor::NbRshift, + op: Some(SlotOp::Left), + doc: "Return self>>value.", + }, + SlotDef { + name: "__rrshift__", + accessor: SlotAccessor::NbRshift, + op: Some(SlotOp::Right), + doc: "Return value>>self.", + }, + SlotDef { + name: "__irshift__", + accessor: SlotAccessor::NbInplaceRshift, + op: None, + doc: "Implement self>>=value.", + }, + SlotDef { + name: "__and__", + accessor: SlotAccessor::NbAnd, + op: Some(SlotOp::Left), + doc: "Return self&value.", + }, + SlotDef { + name: "__rand__", + accessor: SlotAccessor::NbAnd, + op: Some(SlotOp::Right), + doc: "Return value&self.", + }, + SlotDef { + name: "__iand__", + accessor: SlotAccessor::NbInplaceAnd, + op: None, + doc: "Implement self&=value.", + }, + SlotDef { + name: "__xor__", + accessor: SlotAccessor::NbXor, + op: Some(SlotOp::Left), + doc: "Return self^value.", + }, + SlotDef { + name: "__rxor__", + accessor: SlotAccessor::NbXor, + op: Some(SlotOp::Right), + doc: "Return value^self.", + }, + SlotDef { + name: "__ixor__", + accessor: SlotAccessor::NbInplaceXor, + op: None, + doc: "Implement self^=value.", + }, + SlotDef { + name: "__or__", + accessor: SlotAccessor::NbOr, + op: Some(SlotOp::Left), + doc: "Return self|value.", + }, + SlotDef { + name: "__ror__", + accessor: SlotAccessor::NbOr, + op: Some(SlotOp::Right), + doc: "Return value|self.", + }, + SlotDef { + name: "__ior__", + accessor: SlotAccessor::NbInplaceOr, + op: None, + doc: "Implement self|=value.", + }, + SlotDef { + name: "__floordiv__", + accessor: SlotAccessor::NbFloorDivide, + op: Some(SlotOp::Left), + doc: "Return self//value.", + }, + SlotDef { + name: "__rfloordiv__", + accessor: SlotAccessor::NbFloorDivide, + op: Some(SlotOp::Right), + doc: "Return value//self.", + }, + SlotDef { + name: "__ifloordiv__", + accessor: SlotAccessor::NbInplaceFloorDivide, + op: None, + doc: "Implement self//=value.", + }, + SlotDef { + name: "__truediv__", + accessor: SlotAccessor::NbTrueDivide, + op: Some(SlotOp::Left), + doc: "Return self/value.", + }, + SlotDef { + name: "__rtruediv__", + accessor: SlotAccessor::NbTrueDivide, + op: Some(SlotOp::Right), + doc: "Return value/self.", + }, + SlotDef { + name: "__itruediv__", + accessor: SlotAccessor::NbInplaceTrueDivide, + op: None, + doc: "Implement self/=value.", + }, + SlotDef { + name: "__matmul__", + accessor: SlotAccessor::NbMatrixMultiply, + op: Some(SlotOp::Left), + doc: "Return self@value.", + }, + SlotDef { + name: "__rmatmul__", + accessor: SlotAccessor::NbMatrixMultiply, + op: Some(SlotOp::Right), + doc: "Return value@self.", + }, + SlotDef { + name: "__imatmul__", + accessor: SlotAccessor::NbInplaceMatrixMultiply, + op: None, + doc: "Implement self@=value.", + }, + // Number unary operations + SlotDef { + name: "__neg__", + accessor: SlotAccessor::NbNegative, + op: None, + doc: "Return -self.", + }, + SlotDef { + name: "__pos__", + accessor: SlotAccessor::NbPositive, + op: None, + doc: "Return +self.", + }, + SlotDef { + name: "__abs__", + accessor: SlotAccessor::NbAbsolute, + op: None, + doc: "Return abs(self).", + }, + SlotDef { + name: "__invert__", + accessor: SlotAccessor::NbInvert, + op: None, + doc: "Return ~self.", + }, + SlotDef { + name: "__bool__", + accessor: SlotAccessor::NbBool, + op: None, + doc: "Return self != 0.", + }, + SlotDef { + name: "__int__", + accessor: SlotAccessor::NbInt, + op: None, + doc: "Return int(self).", + }, + SlotDef { + name: "__float__", + accessor: SlotAccessor::NbFloat, + op: None, + doc: "Return float(self).", + }, + SlotDef { + name: "__index__", + accessor: SlotAccessor::NbIndex, + op: None, + doc: "Return self converted to an integer, if self is suitable for use as an index into a list.", + }, + // Sequence inplace operations (also map to number slots for some types) + SlotDef { + name: "__add__", + accessor: SlotAccessor::SqConcat, + op: None, + doc: "Return self+value.", + }, + SlotDef { + name: "__mul__", + accessor: SlotAccessor::SqRepeat, + op: None, + doc: "Return self*value.", + }, + SlotDef { + name: "__rmul__", + accessor: SlotAccessor::SqRepeat, + op: None, + doc: "Return value*self.", + }, + SlotDef { + name: "__iadd__", + accessor: SlotAccessor::SqInplaceConcat, + op: None, + doc: "Implement self+=value.", + }, + SlotDef { + name: "__imul__", + accessor: SlotAccessor::SqInplaceRepeat, + op: None, + doc: "Implement self*=value.", + }, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_by_name() { + // __len__ appears in both sequence and mapping + let len_defs: Vec<_> = find_slot_defs_by_name("__len__").collect(); + assert_eq!(len_defs.len(), 2); + + // __init__ appears once + let init_defs: Vec<_> = find_slot_defs_by_name("__init__").collect(); + assert_eq!(init_defs.len(), 1); + + // __add__ appears in number (left/right) and sequence + let add_defs: Vec<_> = find_slot_defs_by_name("__add__").collect(); + assert_eq!(add_defs.len(), 2); // NbAdd(Left) and SqConcat + } + + #[test] + fn test_slot_op() { + // Test comparison ops + assert_eq!(SlotOp::Lt.as_compare_op(), Some(PyComparisonOp::Lt)); + assert_eq!(SlotOp::Eq.as_compare_op(), Some(PyComparisonOp::Eq)); + assert_eq!(SlotOp::Left.as_compare_op(), None); + + // Test right check + assert!(SlotOp::Right.is_right()); + assert!(!SlotOp::Left.is_right()); + } +} diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs index be0a1c9a70c..af35f92f656 100644 --- a/crates/vm/src/types/structseq.rs +++ b/crates/vm/src/types/structseq.rs @@ -1,17 +1,24 @@ +use crate::common::lock::LazyLock; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, - builtins::{ - PyBaseExceptionRef, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, type_::PointerSlot, - }, + builtins::{PyBaseExceptionRef, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}, class::{PyClassImpl, StaticType}, - function::{Either, PyComparisonValue}, + function::{Either, FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags}, iter::PyExactSizeIterator, protocol::{PyMappingMethods, PySequenceMethods}, sliceable::{SequenceIndex, SliceableSequenceOp}, types::PyComparisonOp, vm::Context, }; -use std::sync::LazyLock; + +const DEFAULT_STRUCTSEQ_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyRef<PyTuple>, vm: &VirtualMachine| -> PyTupleRef { + vm.new_tuple((zelf.class().to_owned(), (vm.ctx.new_tuple(zelf.to_vec()),))) + }, + PyMethodFlags::METHOD, + None, +); /// Create a new struct sequence instance from a sequence. /// @@ -87,7 +94,10 @@ static STRUCT_SEQUENCE_AS_SEQUENCE: LazyLock<PySequenceMethods> = let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); // Use tuple's concat implementation - visible_tuple.as_object().to_sequence().concat(other, vm) + visible_tuple + .as_object() + .sequence_unchecked() + .concat(other, vm) }), repeat: atomic_func!(|seq, n, vm| { // Convert to visible-only tuple, then use regular tuple repeat @@ -96,7 +106,7 @@ static STRUCT_SEQUENCE_AS_SEQUENCE: LazyLock<PySequenceMethods> = let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); // Use tuple's repeat implementation - visible_tuple.as_object().to_sequence().repeat(n, vm) + visible_tuple.as_object().sequence_unchecked().repeat(n, vm) }), item: atomic_func!(|seq, i, vm| { let n_seq = get_visible_len(seq.obj, vm)?; @@ -199,24 +209,19 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { .ok_or_else(|| vm.new_type_error("unexpected payload for __repr__"))?; let field_names = Self::Data::REQUIRED_FIELD_NAMES; - let format_field = |(value, name): (&PyObjectRef, _)| { + let format_field = |(value, name): (&PyObject, _)| { let s = value.repr(vm)?; Ok(format!("{name}={s}")) }; let (body, suffix) = if let Some(_guard) = rustpython_vm::recursion::ReprGuard::enter(vm, zelf.as_ref()) { - if field_names.len() == 1 { - let value = zelf.first().unwrap(); - let formatted = format_field((value, field_names[0]))?; - (formatted, ",") - } else { - let fields: PyResult<Vec<_>> = zelf - .iter() - .zip(field_names.iter().copied()) - .map(format_field) - .collect(); - (fields?.join(", "), "") - } + let fields: PyResult<Vec<_>> = zelf + .iter() + .map(|value| value.as_ref()) + .zip(field_names.iter().copied()) + .map(format_field) + .collect(); + (fields?.join(", "), "") } else { (String::new(), "...") }; @@ -230,8 +235,45 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { } #[pymethod] - fn __reduce__(zelf: PyRef<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - vm.new_tuple((zelf.class().to_owned(), (vm.ctx.new_tuple(zelf.to_vec()),))) + fn __replace__(zelf: PyRef<PyTuple>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments".to_owned())); + } + + if Self::Data::UNNAMED_FIELDS_LEN > 0 { + return Err(vm.new_type_error(format!( + "__replace__() is not supported for {} because it has unnamed field(s)", + zelf.class().slot_name() + ))); + } + + let n_fields = + Self::Data::REQUIRED_FIELD_NAMES.len() + Self::Data::OPTIONAL_FIELD_NAMES.len(); + let mut items: Vec<PyObjectRef> = zelf.as_slice()[..n_fields].to_vec(); + + let mut kwargs = args.kwargs.clone(); + + // Replace fields from kwargs + let all_field_names: Vec<&str> = Self::Data::REQUIRED_FIELD_NAMES + .iter() + .chain(Self::Data::OPTIONAL_FIELD_NAMES.iter()) + .copied() + .collect(); + for (i, &name) in all_field_names.iter().enumerate() { + if let Some(val) = kwargs.shift_remove(name) { + items[i] = val; + } + } + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let names: Vec<&str> = kwargs.keys().map(|k| k.as_str()).collect(); + return Err(vm.new_type_error(format!("Got unexpected field name(s): {:?}", names))); + } + + PyTuple::new_unchecked(items.into_boxed_slice()) + .into_ref_with_type(vm, zelf.class().to_owned()) + .map(Into::into) } #[pymethod] @@ -305,12 +347,14 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { ); // Override as_sequence and as_mapping slots to use visible length - class.slots.as_sequence.store(Some(PointerSlot::from( - &*STRUCT_SEQUENCE_AS_SEQUENCE as &'static PySequenceMethods, - ))); - class.slots.as_mapping.store(Some(PointerSlot::from( - &*STRUCT_SEQUENCE_AS_MAPPING as &'static PyMappingMethods, - ))); + class + .slots + .as_sequence + .copy_from(&STRUCT_SEQUENCE_AS_SEQUENCE); + class + .slots + .as_mapping + .copy_from(&STRUCT_SEQUENCE_AS_MAPPING); // Override iter slot to return only visible elements class.slots.iter.store(Some(struct_sequence_iter)); @@ -323,6 +367,20 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { .slots .richcompare .store(Some(struct_sequence_richcompare)); + + // Default __reduce__: only set if not already overridden by the impl's extend_class. + // This allows struct sequences like sched_param to provide a custom __reduce__ + // (equivalent to METH_COEXIST in structseq.c). + if !class + .attributes + .read() + .contains_key(ctx.intern_str("__reduce__")) + { + class.set_attr( + ctx.intern_str("__reduce__"), + DEFAULT_STRUCTSEQ_REDUCE.to_proper_method(class, ctx), + ); + } } } diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index d994ff60210..07754d08340 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -1,11 +1,11 @@ use crate::{ Py, builtins::{ - asyncgenerator, bool_, builtin_func, bytearray, bytes, classmethod, code, complex, + asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator, - genericalias, getset, int, iter, list, map, mappingproxy, memory, module, namespace, - object, property, pystr, range, set, singletons, slice, staticmethod, super_, traceback, - tuple, + genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module, + namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_, + template, traceback, tuple, type_::{self, PyType}, union_, weakproxy, weakref, zip, }, @@ -28,6 +28,7 @@ pub struct TypeZoo { pub bytearray_iterator_type: &'static Py<PyType>, pub bool_type: &'static Py<PyType>, pub callable_iterator: &'static Py<PyType>, + pub capsule_type: &'static Py<PyType>, pub cell_type: &'static Py<PyType>, pub classmethod_type: &'static Py<PyType>, pub code_type: &'static Py<PyType>, @@ -92,8 +93,14 @@ pub struct TypeZoo { pub typing_no_default_type: &'static Py<PyType>, pub not_implemented_type: &'static Py<PyType>, pub generic_alias_type: &'static Py<PyType>, + pub generic_alias_iterator_type: &'static Py<PyType>, pub union_type: &'static Py<PyType>, + pub interpolation_type: &'static Py<PyType>, + pub template_type: &'static Py<PyType>, + pub template_iter_type: &'static Py<PyType>, pub member_descriptor_type: &'static Py<PyType>, + pub wrapper_descriptor_type: &'static Py<PyType>, + pub method_wrapper_type: &'static Py<PyType>, // RustPython-original types pub method_def: &'static Py<PyType>, @@ -103,12 +110,20 @@ impl TypeZoo { #[cold] pub(crate) fn init() -> Self { let (type_type, object_type, weakref_type) = crate::object::init_type_hierarchy(); + // the order matters for type, object, weakref, and int - must be initialized first + let type_type = type_::PyType::init_manually(type_type); + let object_type = object::PyBaseObject::init_manually(object_type); + let weakref_type = weakref::PyWeak::init_manually(weakref_type); + let int_type = int::PyInt::init_builtin_type(); + + // builtin_function_or_method and builtin_method share the same type (CPython behavior) + let builtin_function_or_method_type = builtin_func::PyNativeFunction::init_builtin_type(); + Self { - // the order matters for type, object, weakref, and int - type_type: type_::PyType::init_manually(type_type), - object_type: object::PyBaseObject::init_manually(object_type), - weakref_type: weakref::PyWeak::init_manually(weakref_type), - int_type: int::PyInt::init_builtin_type(), + type_type, + object_type, + weakref_type, + int_type, // types exposed as builtins bool_type: bool_::PyBool::init_builtin_type(), @@ -142,11 +157,12 @@ impl TypeZoo { asyncgenerator::PyAsyncGenWrappedValue::init_builtin_type(), anext_awaitable: asyncgenerator::PyAnextAwaitable::init_builtin_type(), bound_method_type: function::PyBoundMethod::init_builtin_type(), - builtin_function_or_method_type: builtin_func::PyNativeFunction::init_builtin_type(), - builtin_method_type: builtin_func::PyNativeMethod::init_builtin_type(), + builtin_function_or_method_type, + builtin_method_type: builtin_function_or_method_type, bytearray_iterator_type: bytearray::PyByteArrayIterator::init_builtin_type(), bytes_iterator_type: bytes::PyBytesIterator::init_builtin_type(), callable_iterator: iter::PyCallableIterator::init_builtin_type(), + capsule_type: capsule::PyCapsule::init_builtin_type(), cell_type: function::PyCell::init_builtin_type(), code_type: code::PyCode::init_builtin_type(), coroutine_type: coroutine::PyCoroutine::init_builtin_type(), @@ -185,8 +201,14 @@ impl TypeZoo { typing_no_default_type: crate::stdlib::typing::NoDefault::init_builtin_type(), not_implemented_type: singletons::PyNotImplemented::init_builtin_type(), generic_alias_type: genericalias::PyGenericAlias::init_builtin_type(), + generic_alias_iterator_type: genericalias::PyGenericAliasIterator::init_builtin_type(), union_type: union_::PyUnion::init_builtin_type(), + interpolation_type: interpolation::PyInterpolation::init_builtin_type(), + template_type: template::PyTemplate::init_builtin_type(), + template_iter_type: template::PyTemplateIter::init_builtin_type(), member_descriptor_type: descriptor::PyMemberDescriptor::init_builtin_type(), + wrapper_descriptor_type: descriptor::PyWrapper::init_builtin_type(), + method_wrapper_type: descriptor::PyMethodWrapper::init_builtin_type(), method_def: crate::function::HeapMethodDef::init_builtin_type(), } @@ -195,8 +217,10 @@ impl TypeZoo { /// Fill attributes of builtin types. #[cold] pub(crate) fn extend(context: &Context) { - type_::init(context); + // object must be initialized before type to set object.slots.init, + // which type will inherit via inherit_slots() object::init(context); + type_::init(context); list::init(context); set::init(context); tuple::init(context); @@ -213,6 +237,7 @@ impl TypeZoo { complex::init(context); bytes::init(context); bytearray::init(context); + capsule::init(context); property::init(context); getset::init(context); memory::init(context); @@ -237,6 +262,9 @@ impl TypeZoo { traceback::init(context); genericalias::init(context); union_::init(context); + interpolation::init(context); + template::init(context); descriptor::init(context); + crate::stdlib::typing::init(context); } } diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index af34405c7be..db232e81949 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -14,15 +14,15 @@ pub fn hash_iter<'a, I: IntoIterator<Item = &'a PyObjectRef>>( vm.state.hash_secret.hash_iter(iter, |obj| obj.hash(vm)) } -impl ToPyObject for std::convert::Infallible { +impl ToPyObject for core::convert::Infallible { fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { match self {} } } pub trait ToCString: AsRef<Wtf8> { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) + fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) } fn ensure_no_nul(&self, vm: &VirtualMachine) -> PyResult<()> { if self.as_ref().as_bytes().contains(&b'\0') { @@ -45,7 +45,7 @@ pub(crate) fn collection_repr<'a, I>( vm: &VirtualMachine, ) -> PyResult<String> where - I: std::iter::Iterator<Item = &'a PyObjectRef>, + I: core::iter::Iterator<Item = &'a PyObjectRef>, { let mut repr = String::new(); if let Some(name) = class_name { diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index 9d472e8be0a..a75a6f47de6 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -1,11 +1,12 @@ //! Several function to retrieve version information. use chrono::{Local, prelude::DateTime}; -use std::time::{Duration, UNIX_EPOCH}; +use core::time::Duration; +use std::time::UNIX_EPOCH; -// = 3.13.0alpha +// = 3.14.0alpha pub const MAJOR: usize = 3; -pub const MINOR: usize = 13; +pub const MINOR: usize = 14; pub const MICRO: usize = 0; pub const RELEASELEVEL: &str = "alpha"; pub const RELEASELEVEL_N: usize = 0xA; @@ -15,12 +16,29 @@ pub const VERSION_HEX: usize = (MAJOR << 24) | (MINOR << 16) | (MICRO << 8) | (RELEASELEVEL_N << 4) | SERIAL; pub fn get_version() -> String { + // Windows: include MSC v. for compatibility with ctypes.util.find_library + // MSC v.1929 = VS 2019, version 14+ makes find_msvcrt() return None + #[cfg(windows)] + let msc_info = { + let arch = if cfg!(target_pointer_width = "64") { + "64 bit (AMD64)" + } else { + "32 bit (Intel)" + }; + // Include both RustPython identifier and MSC v. for compatibility + format!(" MSC v.1929 {arch}",) + }; + + #[cfg(not(windows))] + let msc_info = String::new(); + format!( - "{:.80} ({:.80}) \n[RustPython {} with {:.80}]", // \n is PyPy convention + "{:.80} ({:.80}) \n[RustPython {} with {:.80}{}]", // \n is PyPy convention get_version_number(), get_build_info(), env!("CARGO_PKG_VERSION"), COMPILER, + msc_info, ) } @@ -109,3 +127,14 @@ pub fn get_git_datetime() -> String { format!("{date} {time}") } + +// Must be aligned to Lib/importlib/_bootstrap_external.py +pub const PYC_MAGIC_NUMBER: u16 = 2996; + +// CPython format: magic_number | ('\r' << 16) | ('\n' << 24) +// This protects against text-mode file reads +pub const PYC_MAGIC_NUMBER_TOKEN: u32 = + (PYC_MAGIC_NUMBER as u32) | ((b'\r' as u32) << 16) | ((b'\n' as u32) << 24); + +/// Magic number as little-endian bytes for .pyc files +pub const PYC_MAGIC_NUMBER_BYTES: [u8; 4] = PYC_MAGIC_NUMBER_TOKEN.to_le_bytes(); diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 6f1ea734926..7294dc8f897 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -1,9 +1,11 @@ +//! Python code compilation functions. +//! +//! For code execution functions, see python_run.rs + use crate::{ - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, - builtins::{PyCode, PyDictRef}, + PyRef, VirtualMachine, + builtins::PyCode, compiler::{self, CompileError, CompileOpts}, - convert::TryFromObject, - scope::Scope, }; impl VirtualMachine { @@ -23,132 +25,337 @@ impl VirtualMachine { source_path: String, opts: CompileOpts, ) -> Result<PyRef<PyCode>, CompileError> { - compiler::compile(source, mode, &source_path, opts).map(|code| self.ctx.new_code(code)) - } - - // pymain_run_file_obj - pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> { - // when pymain_run_module? - if get_importer(path, self)?.is_some() { - self.insert_sys_path(self.new_pyobj(path))?; - let runpy = self.import("runpy", 0)?; - let run_module_as_main = runpy.get_attr("_run_module_as_main", self)?; - run_module_as_main.call((identifier!(self, __main__).to_owned(), false), self)?; - return Ok(()); + let code = + compiler::compile(source, mode, &source_path, opts).map(|code| self.ctx.new_code(code)); + #[cfg(feature = "parser")] + if code.is_ok() { + self.emit_string_escape_warnings(source, &source_path); } - - // TODO: check if this is proper place - if !self.state.settings.safe_path { - let dir = std::path::Path::new(path) - .parent() - .unwrap() - .to_str() - .unwrap(); - self.insert_sys_path(self.new_pyobj(dir))?; - } - - self.run_any_file(scope, path) + code } +} - // = _PyRun_AnyFileObject - fn run_any_file(&self, scope: Scope, path: &str) -> PyResult<()> { - let path = if path.is_empty() { "???" } else { path }; +/// Scan source for invalid escape sequences in all string literals and emit +/// SyntaxWarning. +/// +/// Corresponds to: +/// - `warn_invalid_escape_sequence()` in `Parser/string_parser.c` +/// - `_PyTokenizer_warn_invalid_escape_sequence()` in `Parser/tokenizer/helpers.c` +#[cfg(feature = "parser")] +mod escape_warnings { + use super::*; + use crate::warn; + use ruff_python_ast::{self as ast, visitor::Visitor}; + use ruff_text_size::TextRange; - self.run_simple_file(scope, path) + /// Calculate 1-indexed line number at byte offset in source. + fn line_number_at(source: &str, offset: usize) -> usize { + source[..offset.min(source.len())] + .bytes() + .filter(|&b| b == b'\n') + .count() + + 1 } - // = _PyRun_SimpleFileObject - fn run_simple_file(&self, scope: Scope, path: &str) -> PyResult<()> { - // __main__ is given by scope - let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; - let main_module = sys_modules.get_item(identifier!(self, __main__), self)?; - let module_dict = main_module.dict().expect("main module must have __dict__"); - if !module_dict.contains_key(identifier!(self, __file__), self) { - module_dict.set_item( - identifier!(self, __file__), - self.ctx.new_str(path).into(), - self, - )?; - module_dict.set_item(identifier!(self, __cached__), self.ctx.none(), self)?; + /// Get content bounds (start, end byte offsets) of a quoted string literal, + /// excluding prefix characters and quote delimiters. + fn content_bounds(source: &str, range: TextRange) -> Option<(usize, usize)> { + let s = range.start().to_usize(); + let e = range.end().to_usize(); + if s >= e || e > source.len() { + return None; } - - // Consider to use enum to distinguish `path` - // https://github.com/RustPython/RustPython/pull/6276#discussion_r2529849479 - - // TODO: check .pyc here - let pyc = false; - if pyc { - todo!("running pyc is not implemented yet"); + let bytes = &source.as_bytes()[s..e]; + // Skip prefix (u, b, r, etc.) to find the first quote character. + let qi = bytes.iter().position(|&c| c == b'\'' || c == b'"')?; + let qc = bytes[qi]; + let ql = if bytes.get(qi + 1) == Some(&qc) && bytes.get(qi + 2) == Some(&qc) { + 3 } else { - if path != "<stdin>" { - // TODO: set_main_loader(dict, filename, "SourceFileLoader"); + 1 + }; + let cs = s + qi + ql; + let ce = e.checked_sub(ql)?; + if cs <= ce { Some((cs, ce)) } else { None } + } + + /// Scan `source[start..end]` for the first invalid escape sequence. + /// Returns `Some((invalid_char, byte_offset_in_source))` for the first + /// invalid escape found, or `None` if all escapes are valid. + /// + /// When `is_bytes` is true, `\u`, `\U`, and `\N` are treated as invalid + /// (bytes literals only support byte-oriented escapes). + /// + /// Only reports the **first** invalid escape per string literal, matching + /// `_PyUnicode_DecodeUnicodeEscapeInternal2` which stores only the first + /// `first_invalid_escape_char`. + fn first_invalid_escape( + source: &str, + start: usize, + end: usize, + is_bytes: bool, + ) -> Option<(char, usize)> { + let raw = &source[start..end]; + let mut chars = raw.char_indices().peekable(); + while let Some((i, ch)) = chars.next() { + if ch != '\\' { + continue; } - // TODO: replace to something equivalent to py_run_file - match std::fs::read_to_string(path) { - Ok(source) => { - let code_obj = self - .compile(&source, compiler::Mode::Exec, path.to_owned()) - .map_err(|err| self.new_syntax_error(&err, Some(&source)))?; - // trace!("Code object: {:?}", code_obj.borrow()); - self.run_code_obj(code_obj, scope)?; + let Some((_, next)) = chars.next() else { + break; + }; + let valid = match next { + '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => true, + '\n' => true, + '\r' => { + if matches!(chars.peek(), Some(&(_, '\n'))) { + chars.next(); + } + true + } + '0'..='7' => { + for _ in 0..2 { + if matches!(chars.peek(), Some(&(_, '0'..='7'))) { + chars.next(); + } else { + break; + } + } + true } - Err(err) => { - error!("Failed reading file '{path}': {err}"); - // TODO: Need to change to ExitCode or Termination - std::process::exit(1); + 'x' | 'u' | 'U' => { + // \u and \U are only valid in string literals, not bytes + if is_bytes && next != 'x' { + false + } else { + let count = match next { + 'x' => 2, + 'u' => 4, + 'U' => 8, + _ => unreachable!(), + }; + for _ in 0..count { + if chars.peek().is_some_and(|&(_, c)| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } } + 'N' => { + // \N{name} is only valid in string literals, not bytes + if is_bytes { + false + } else { + if matches!(chars.peek(), Some(&(_, '{'))) { + chars.next(); + for (_, c) in chars.by_ref() { + if c == '}' { + break; + } + } + } + true + } + } + _ => false, + }; + if !valid { + return Some((next, start + i)); } } - Ok(()) + None } - // TODO: deprecate or reimplement using other primitive functions - pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { - let code_obj = self - .compile(source, compiler::Mode::Exec, source_path.clone()) - .map_err(|err| self.new_syntax_error(&err, Some(source)))?; - // trace!("Code object: {:?}", code_obj.borrow()); - scope.globals.set_item( - identifier!(self, __file__), - self.new_pyobj(source_path), - self, - )?; - self.run_code_obj(code_obj, scope) + /// Emit `SyntaxWarning` for an invalid escape sequence. + /// + /// `warn_invalid_escape_sequence()` in `Parser/string_parser.c` + fn warn_invalid_escape_sequence( + source: &str, + ch: char, + offset: usize, + filename: &str, + vm: &VirtualMachine, + ) { + let lineno = line_number_at(source, offset); + let message = vm.ctx.new_str(format!( + "\"\\{ch}\" is an invalid escape sequence. \ + Such sequences will not work in the future. \ + Did you mean \"\\\\{ch}\"? A raw string is also an option." + )); + let fname = vm.ctx.new_str(filename); + let _ = warn::warn_explicit( + Some(vm.ctx.exceptions.syntax_warning.to_owned()), + message.into(), + fname, + lineno, + None, + vm.ctx.none(), + None, + None, + vm, + ); } - pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult { - let code_obj = self - .compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned()) - .map_err(|err| self.new_syntax_error(&err, Some(source)))?; - // trace!("Code object: {:?}", code_obj.borrow()); - self.run_code_obj(code_obj, scope) + struct EscapeWarningVisitor<'a> { + source: &'a str, + filename: &'a str, + vm: &'a VirtualMachine, } -} -fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { - let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?; - let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?; - if let Some(importer) = path_importer_cache.get_item_opt(path, vm)? { - return Ok(Some(importer)); + impl<'a> EscapeWarningVisitor<'a> { + /// Check a quoted string/bytes literal for invalid escapes. + /// The range must include the prefix and quote delimiters. + fn check_quoted_literal(&self, range: TextRange, is_bytes: bool) { + if let Some((start, end)) = content_bounds(self.source, range) + && let Some((ch, offset)) = first_invalid_escape(self.source, start, end, is_bytes) + { + warn_invalid_escape_sequence(self.source, ch, offset, self.filename, self.vm); + } + } + + /// Check an f-string literal element for invalid escapes. + /// The range covers content only (no prefix/quotes). + /// + /// Also handles `\{` / `\}` at the literal–interpolation boundary, + /// equivalent to `_PyTokenizer_warn_invalid_escape_sequence` handling + /// `FSTRING_MIDDLE` / `FSTRING_END` tokens. + fn check_fstring_literal(&self, range: TextRange) { + let start = range.start().to_usize(); + let end = range.end().to_usize(); + if start >= end || end > self.source.len() { + return; + } + if let Some((ch, offset)) = first_invalid_escape(self.source, start, end, false) { + warn_invalid_escape_sequence(self.source, ch, offset, self.filename, self.vm); + return; + } + // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles + // `\{` and `\}` for FSTRING_MIDDLE/FSTRING_END tokens. Ruff + // splits the literal element before the interpolation delimiter, + // so the `\` sits at the end of the literal range and the `{`/`}` + // sits just after it. Only warn when the number of trailing + // backslashes is odd (an even count means they are all escaped). + let trailing_bs = self.source.as_bytes()[start..end] + .iter() + .rev() + .take_while(|&&b| b == b'\\') + .count(); + if trailing_bs % 2 == 1 + && let Some(&after) = self.source.as_bytes().get(end) + && (after == b'{' || after == b'}') + { + warn_invalid_escape_sequence( + self.source, + after as char, + end - 1, + self.filename, + self.vm, + ); + } + } + + /// Visit f-string elements, checking literals and recursing into + /// interpolation expressions and format specs. + fn visit_fstring_elements(&mut self, elements: &'a ast::InterpolatedStringElements) { + for element in elements { + match element { + ast::InterpolatedStringElement::Literal(lit) => { + self.check_fstring_literal(lit.range); + } + ast::InterpolatedStringElement::Interpolation(interp) => { + self.visit_expr(&interp.expression); + if let Some(spec) = &interp.format_spec { + self.visit_fstring_elements(&spec.elements); + } + } + } + } + } } - let path = vm.ctx.new_str(path); - let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; - let mut importer = None; - let path_hooks: Vec<PyObjectRef> = path_hooks.try_into_value(vm)?; - for path_hook in path_hooks { - match path_hook.call((path.clone(),), vm) { - Ok(imp) => { - importer = Some(imp); - break; + + impl<'a> Visitor<'a> for EscapeWarningVisitor<'a> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + match expr { + // Regular string literals — decode_unicode_with_escapes path + ast::Expr::StringLiteral(string) => { + for part in string.value.as_slice() { + if !matches!( + part.flags.prefix(), + ast::str_prefix::StringLiteralPrefix::Raw { .. } + ) { + self.check_quoted_literal(part.range, false); + } + } + } + // Byte string literals — decode_bytes_with_escapes path + ast::Expr::BytesLiteral(bytes) => { + for part in bytes.value.as_slice() { + if !matches!( + part.flags.prefix(), + ast::str_prefix::ByteStringPrefix::Raw { .. } + ) { + self.check_quoted_literal(part.range, true); + } + } + } + // F-string literals — tokenizer + string_parser paths + ast::Expr::FString(fstring_expr) => { + for part in fstring_expr.value.as_slice() { + match part { + ast::FStringPart::Literal(string_lit) => { + // Plain string part in f-string concatenation + if !matches!( + string_lit.flags.prefix(), + ast::str_prefix::StringLiteralPrefix::Raw { .. } + ) { + self.check_quoted_literal(string_lit.range, false); + } + } + ast::FStringPart::FString(fstring) => { + if matches!( + fstring.flags.prefix(), + ast::str_prefix::FStringPrefix::Raw { .. } + ) { + continue; + } + self.visit_fstring_elements(&fstring.elements); + } + } + } + } + _ => ast::visitor::walk_expr(self, expr), + } + } + } + + impl VirtualMachine { + /// Walk all string literals in `source` and emit `SyntaxWarning` for + /// each that contains an invalid escape sequence. + pub(super) fn emit_string_escape_warnings(&self, source: &str, filename: &str) { + let Ok(parsed) = + ruff_python_parser::parse(source, ruff_python_parser::Mode::Module.into()) + else { + return; + }; + let ast = parsed.into_syntax(); + let mut visitor = EscapeWarningVisitor { + source, + filename, + vm: self, + }; + match ast { + ast::Mod::Module(module) => { + for stmt in &module.body { + visitor.visit_stmt(stmt); + } + } + ast::Mod::Expression(expr) => { + visitor.visit_expr(&expr.body); + } } - Err(e) if e.fast_isinstance(vm.ctx.exceptions.import_error) => continue, - Err(e) => return Err(e), } } - Ok(if let Some(imp) = importer { - let imp = path_importer_cache.get_or_insert(vm, path.into(), || imp.clone())?; - Some(imp) - } else { - None - }) } diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index fbda71dc1f6..db5406f045f 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -51,6 +51,10 @@ pub struct Context { pub(crate) string_pool: StringPool, pub(crate) slot_new_wrapper: PyMethodDef, pub names: ConstName, + + // GC module state (callbacks and garbage lists) + pub gc_callbacks: PyListRef, + pub gc_garbage: PyListRef, } macro_rules! declare_const_name { @@ -62,9 +66,9 @@ macro_rules! declare_const_name { } impl ConstName { - unsafe fn new(pool: &StringPool, typ: &PyTypeRef) -> Self { + unsafe fn new(pool: &StringPool, typ: &Py<PyType>) -> Self { Self { - $($name: unsafe { pool.intern(declare_const_name!(@string $name $($s)?), typ.clone()) },)* + $($name: unsafe { pool.intern(declare_const_name!(@string $name $($s)?), typ.to_owned()) },)* } } } @@ -91,7 +95,10 @@ declare_const_name! { __all__, __and__, __anext__, + __annotate__, + __annotate_func__, __annotations__, + __annotations_cache__, __args__, __await__, __bases__, @@ -135,6 +142,7 @@ declare_const_name! { __getformat__, __getitem__, __getnewargs__, + __getnewargs_ex__, __getstate__, __gt__, __hash__, @@ -230,12 +238,18 @@ declare_const_name! { __typing_is_unpacked_typevartuple__, __typing_prepare_subst__, __typing_unpacked_tuple_args__, + __weakref__, __xor__, // common names _attributes, _fields, + _defaultaction, + _onceregistry, _showwarnmsg, + defaultaction, + onceregistry, + filters, backslashreplace, close, copy, @@ -264,7 +278,7 @@ declare_const_name! { // Basic objects: impl Context { - pub const INT_CACHE_POOL_RANGE: std::ops::RangeInclusive<i32> = (-5)..=256; + pub const INT_CACHE_POOL_RANGE: core::ops::RangeInclusive<i32> = (-5)..=256; const INT_CACHE_POOL_MIN: i32 = *Self::INT_CACHE_POOL_RANGE.start(); pub fn genesis() -> &'static PyRc<Self> { @@ -317,7 +331,7 @@ impl Context { ); let string_pool = StringPool::default(); - let names = unsafe { ConstName::new(&string_pool, &types.str_type.to_owned()) }; + let names = unsafe { ConstName::new(&string_pool, types.str_type) }; let slot_new_wrapper = PyMethodDef::new_const( names.__new__.as_str(), @@ -328,6 +342,11 @@ impl Context { let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); + + // GC callbacks and garbage lists + let gc_callbacks = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + let gc_garbage = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + Self { true_value, false_value, @@ -347,6 +366,9 @@ impl Context { string_pool, slot_new_wrapper, names, + + gc_callbacks, + gc_garbage, } } @@ -374,14 +396,14 @@ impl Context { #[inline] pub fn empty_tuple_typed<T>(&self) -> &Py<PyTuple<T>> { let py: &Py<PyTuple> = &self.empty_tuple; - unsafe { std::mem::transmute(py) } + unsafe { core::mem::transmute(py) } } // universal pyref constructor pub fn new_pyref<T, P>(&self, value: T) -> PyRef<P> where T: Into<P>, - P: PyPayload + std::fmt::Debug, + P: PyPayload + core::fmt::Debug, { value.into().into_ref(self) } @@ -438,7 +460,11 @@ impl Context { #[inline] pub fn new_bytes(&self, data: Vec<u8>) -> PyRef<PyBytes> { - PyBytes::from(data).into_ref(self) + if data.is_empty() { + self.empty_bytes.clone() + } else { + PyBytes::from(data).into_ref(self) + } } #[inline] @@ -632,7 +658,7 @@ impl Context { pub fn new_code(&self, code: impl code::IntoCodeObject) -> PyRef<PyCode> { let code = code.into_code_object(self); - PyRef::new_ref(PyCode { code }, self.types.code_type.to_owned(), None) + PyRef::new_ref(PyCode::new(code), self.types.code_type.to_owned(), None) } } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 05613d43384..a71a9c08157 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,6 +1,274 @@ -use super::{Context, VirtualMachine, setting::Settings, thread}; -use crate::{PyResult, stdlib::atexit, vm::PyBaseExceptionRef}; -use std::sync::atomic::Ordering; +use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; +use crate::{ + PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit, + vm::PyBaseExceptionRef, +}; +use alloc::collections::BTreeMap; +use core::sync::atomic::Ordering; + +type InitFunc = Box<dyn FnOnce(&mut VirtualMachine)>; + +/// Configuration builder for constructing an Interpreter. +/// +/// This is the preferred way to configure and create an interpreter with custom modules. +/// Modules must be registered before the interpreter is built, +/// similar to CPython's `PyImport_AppendInittab` which must be called before `Py_Initialize`. +/// +/// # Example +/// ``` +/// use rustpython_vm::Interpreter; +/// +/// let builder = Interpreter::builder(Default::default()); +/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) +/// let interp = builder.build(); +/// ``` +pub struct InterpreterBuilder { + settings: Settings, + pub ctx: PyRc<Context>, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec<InitFunc>, +} + +/// Private helper to initialize a VM with settings, context, and custom initialization. +fn initialize_main_vm<F>( + settings: Settings, + ctx: PyRc<Context>, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec<InitFunc>, + init: F, +) -> (VirtualMachine, PyRc<PyGlobalState>) +where + F: FnOnce(&mut VirtualMachine), +{ + use crate::codecs::CodecsRegistry; + use crate::common::hash::HashSecret; + use crate::common::lock::PyMutex; + use crate::warn::WarningsState; + use core::sync::atomic::AtomicBool; + use crossbeam_utils::atomic::AtomicCell; + + let paths = getpath::init_path_config(&settings); + let config = PyConfig::new(settings, paths); + + crate::types::TypeZoo::extend(&ctx); + crate::exceptions::ExceptionZoo::extend(&ctx); + + // Build module_defs map from builtin modules + additional modules + let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> = + crate::stdlib::builtin_module_defs(&ctx) + .into_iter() + .chain(module_defs) + .map(|def| (def.name.as_str(), def)) + .collect(); + + // Register sysconfigdata under platform-specific name as well + if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") { + use std::sync::OnceLock; + static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new(); + let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| { + let name = crate::stdlib::sys::sysconfigdata_name(); + Box::leak(name.into_boxed_str()) + }); + all_module_defs.insert(leaked_name, sysconfigdata_def); + } + + // Create hash secret + let seed = match config.settings.hash_seed { + Some(seed) => seed, + None => super::process_hash_secret_seed(), + }; + let hash_secret = HashSecret::new(seed); + + // Create codec registry and warnings state + let codec_registry = CodecsRegistry::new(&ctx); + let warnings = WarningsState::init_state(&ctx); + + // Create int_max_str_digits + let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { + -1 => 4300, + other => other, + } as usize); + + // Initialize frozen modules (core + user-provided) + let mut frozen: std::collections::HashMap<&'static str, FrozenModule, ahash::RandomState> = + core_frozen_inits().collect(); + frozen.extend(frozen_modules); + + // Create PyGlobalState + let global_state = PyRc::new(PyGlobalState { + config, + module_defs: all_module_defs, + frozen, + stacksize: AtomicCell::new(0), + thread_count: AtomicCell::new(0), + hash_secret, + atexit_funcs: PyMutex::default(), + codec_registry, + finalizing: AtomicBool::new(false), + warnings, + override_frozen_modules: AtomicCell::new(0), + before_forkers: PyMutex::default(), + after_forkers_child: PyMutex::default(), + after_forkers_parent: PyMutex::default(), + int_max_str_digits, + switch_interval: AtomicCell::new(0.005), + global_trace_func: PyMutex::default(), + global_profile_func: PyMutex::default(), + #[cfg(feature = "threading")] + main_thread_ident: AtomicCell::new(0), + #[cfg(feature = "threading")] + thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()), + #[cfg(feature = "threading")] + thread_handles: parking_lot::Mutex::new(Vec::new()), + #[cfg(feature = "threading")] + shutdown_handles: parking_lot::Mutex::new(Vec::new()), + }); + + // Create VM with the global state + // Note: Don't clone here - init_hooks need exclusive access to mutate state + let mut vm = VirtualMachine::new(ctx, global_state); + + // Execute initialization hooks (can mutate vm.state) + for hook in init_hooks { + hook(&mut vm); + } + + // Call custom init function (can mutate vm.state) + init(&mut vm); + + vm.initialize(); + + // Clone global_state for Interpreter after all initialization is done + let global_state = vm.state.clone(); + (vm, global_state) +} + +impl InterpreterBuilder { + /// Create a new interpreter configuration with default settings. + pub fn new() -> Self { + Self { + settings: Settings::default(), + ctx: Context::genesis().clone(), + module_defs: Vec::new(), + frozen_modules: Vec::new(), + init_hooks: Vec::new(), + } + } + + /// Set custom settings for the interpreter. + /// + /// If called multiple times, only the last settings will be used. + pub fn settings(mut self, settings: Settings) -> Self { + self.settings = settings; + self + } + + /// Add a single native module definition. + /// + /// # Example + /// ``` + /// use rustpython_vm::{Interpreter, builtins::PyModuleDef}; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // Note: In practice, use module_def from your #[pymodule] + /// // let def = mymodule::module_def(&builder.ctx); + /// // let interp = builder.add_native_module(def).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self { + self.add_native_modules(&[def]) + } + + /// Add multiple native module definitions. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, use module_defs from rustpython_stdlib: + /// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + /// // let interp = builder.add_native_modules(&defs).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self { + self.module_defs.extend_from_slice(defs); + self + } + + /// Add a custom initialization hook. + /// + /// Hooks are executed in the order they are added during interpreter creation. + /// This function will be called after modules are registered but before + /// the VM is initialized, allowing for additional customization. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// .init_hook(|vm| { + /// // Custom initialization + /// }) + /// .build(); + /// ``` + pub fn init_hook<F>(mut self, init: F) -> Self + where + F: FnOnce(&mut VirtualMachine) + 'static, + { + self.init_hooks.push(Box::new(init)); + self + } + + /// Add frozen modules to the interpreter. + /// + /// Frozen modules are Python modules compiled into the binary. + /// This method accepts any iterator of (name, FrozenModule) pairs. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// // In practice: .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + /// .build(); + /// ``` + pub fn add_frozen_modules<I>(mut self, frozen: I) -> Self + where + I: IntoIterator<Item = (&'static str, FrozenModule)>, + { + self.frozen_modules.extend(frozen); + self + } + + /// Build the interpreter. + /// + /// This consumes the configuration and returns a fully initialized Interpreter. + pub fn build(self) -> Interpreter { + let (vm, global_state) = initialize_main_vm( + self.settings, + self.ctx, + self.module_defs, + self.frozen_modules, + self.init_hooks, + |_| {}, // No additional init needed + ); + Interpreter { global_state, vm } + } + + /// Alias for `build()` for compatibility with the `interpreter()` pattern. + pub fn interpreter(self) -> Interpreter { + self.build() + } +} + +impl Default for InterpreterBuilder { + fn default() -> Self { + Self::new() + } +} /// The general interface for the VM /// @@ -21,39 +289,49 @@ use std::sync::atomic::Ordering; /// }); /// ``` pub struct Interpreter { + pub global_state: PyRc<PyGlobalState>, vm: VirtualMachine, } impl Interpreter { + /// Create a new interpreter configuration builder. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) + /// let interp = builder.build(); + /// ``` + pub fn builder(settings: Settings) -> InterpreterBuilder { + InterpreterBuilder::new().settings(settings) + } + /// This is a bare unit to build up an interpreter without the standard library. - /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterConfig`. + /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterBuilder`. /// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`, - /// try to build one from the source code of `InterpreterConfig`. It will not be a one-liner but it also will not be too hard. + /// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard. pub fn without_stdlib(settings: Settings) -> Self { Self::with_init(settings, |_| {}) } /// Create with initialize function taking mutable vm reference. - /// ``` - /// use rustpython_vm::Interpreter; - /// Interpreter::with_init(Default::default(), |vm| { - /// // put this line to add stdlib to the vm - /// // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - /// }).enter(|vm| { - /// vm.run_code_string(vm.new_scope_with_builtins(), "print(1)", "<...>".to_owned()); - /// }); - /// ``` + /// + /// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead. pub fn with_init<F>(settings: Settings, init: F) -> Self where F: FnOnce(&mut VirtualMachine), { - let ctx = Context::genesis(); - crate::types::TypeZoo::extend(ctx); - crate::exceptions::ExceptionZoo::extend(ctx); - let mut vm = VirtualMachine::new(settings, ctx.clone()); - init(&mut vm); - vm.initialize(); - Self { vm } + let (vm, global_state) = initialize_main_vm( + settings, + Context::genesis().clone(), + Vec::new(), // No module_defs + Vec::new(), // No frozen_modules + Vec::new(), // No init_hooks + init, + ); + Self { global_state, vm } } /// Run a function with the main virtual machine and return a PyResult of the result. @@ -106,10 +384,13 @@ impl Interpreter { /// Finalize vm and turns an exception to exit code. /// - /// Finalization steps including 4 steps: + /// Finalization steps (matching Py_FinalizeEx): /// 1. Flush stdout and stderr. /// 1. Handle exit exception and turn it to exit code. + /// 1. Wait for thread shutdown (call threading._shutdown). + /// 1. Mark vm as finalizing. /// 1. Run atexit exit functions. + /// 1. Finalize modules (clear module dicts in reverse import order). /// 1. Mark vm as finalized. /// /// Note that calling `finalize` is not necessary by purpose though. @@ -124,10 +405,29 @@ impl Interpreter { 0 }; - atexit::_run_exitfuncs(vm); + // Wait for thread shutdown - call threading._shutdown() if available. + // This waits for all non-daemon threads to complete. + // threading module may not be imported, so ignore import errors. + if let Ok(threading) = vm.import("threading", 0) + && let Ok(shutdown) = threading.get_attr("_shutdown", vm) + && let Err(e) = shutdown.call((), vm) + { + vm.run_unraisable( + e, + Some("Exception ignored in threading shutdown".to_owned()), + threading, + ); + } + // Mark as finalizing AFTER thread shutdown vm.state.finalizing.store(true, Ordering::Release); + // Run atexit exit functions + atexit::_run_exitfuncs(vm); + + // Finalize modules: clear module dicts in reverse import order + vm.finalize_modules(); + vm.flush_std(); exit_code @@ -135,6 +435,99 @@ impl Interpreter { } } +fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> { + let iter = core::iter::empty(); + macro_rules! ext_modules { + ($iter:ident, $($t:tt)*) => { + let $iter = $iter.chain(py_freeze!($($t)*)); + }; + } + + // Python modules that the vm calls into, but are not actually part of the stdlib. They could + // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. + // Includes _importlib_bootstrap and _importlib_bootstrap_external + ext_modules!( + iter, + dir = "../../Lib/python_builtins", + crate_name = "rustpython_compiler_core" + ); + + // core stdlib Python modules that the vm calls into, but are still used in Python + // application code, e.g. copyreg + // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. + // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterBuilder::new().init_stdlib().interpreter()` + // #[cfg(not(feature = "freeze-stdlib"))] + ext_modules!( + iter, + dir = "../../Lib/core_modules", + crate_name = "rustpython_compiler_core" + ); + + // Collect and add frozen module aliases for test modules + let mut entries: Vec<_> = iter.collect(); + if let Some(hello_code) = entries + .iter() + .find(|(n, _)| *n == "__hello__") + .map(|(_, m)| m.code) + { + entries.push(( + "__hello_alias__", + FrozenModule { + code: hello_code, + package: false, + }, + )); + entries.push(( + "__phello_alias__", + FrozenModule { + code: hello_code, + package: true, + }, + )); + entries.push(( + "__phello_alias__.spam", + FrozenModule { + code: hello_code, + package: false, + }, + )); + entries.push(( + "__hello_only__", + FrozenModule { + code: hello_code, + package: false, + }, + )); + } + if let Some(code) = entries + .iter() + .find(|(n, _)| *n == "__phello__") + .map(|(_, m)| m.code) + { + entries.push(( + "__phello__.__init__", + FrozenModule { + code, + package: false, + }, + )); + } + if let Some(code) = entries + .iter() + .find(|(n, _)| *n == "__phello__.ham") + .map(|(_, m)| m.code) + { + entries.push(( + "__phello__.ham.__init__", + FrozenModule { + code, + package: false, + }, + )); + } + entries.into_iter() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/vm/src/vm/method.rs b/crates/vm/src/vm/method.rs index 5df01c556ea..5f47c8b8c5b 100644 --- a/crates/vm/src/vm/method.rs +++ b/crates/vm/src/vm/method.rs @@ -3,8 +3,8 @@ use super::VirtualMachine; use crate::{ - builtins::{PyBaseObject, PyStr, PyStrInterned}, - function::IntoFuncArgs, + builtins::{PyBaseObject, PyStr, PyStrInterned, descriptor::PyMethodDescriptor}, + function::{IntoFuncArgs, PyMethodFlags}, object::{AsObject, Py, PyObject, PyObjectRef, PyResult}, types::PyTypeFlags, }; @@ -21,8 +21,8 @@ pub enum PyMethod { impl PyMethod { pub fn get(obj: PyObjectRef, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult<Self> { let cls = obj.class(); - let getattro = cls.mro_find_map(|cls| cls.slots.getattro.load()).unwrap(); - if getattro as usize != PyBaseObject::getattro as usize { + let getattro = cls.slots.getattro.load().unwrap(); + if getattro as usize != PyBaseObject::getattro as *const () as usize { return obj.get_attr(name, vm).map(Self::Attribute); } @@ -38,14 +38,19 @@ impl PyMethod { .flags .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) { - is_method = true; - None + // For classmethods, we need descr_get to convert instance to class + if let Some(method_descr) = descr.downcast_ref::<PyMethodDescriptor>() + && method_descr.method.flags.contains(PyMethodFlags::CLASS) + { + descr_cls.slots.descr_get.load() + } else { + is_method = true; + None + } } else { - let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); + let descr_get = descr_cls.slots.descr_get.load(); if let Some(descr_get) = descr_get - && descr_cls - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some() + && descr_cls.slots.descr_set.load().is_some() { let cls = cls.to_owned().into(); return descr_get(descr, Some(obj), Some(cls), vm).map(Self::Attribute); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index fd37b2494dd..07395b80460 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -8,6 +8,8 @@ mod compile; mod context; mod interpreter; mod method; +#[cfg(feature = "rustpython-compiler")] +mod python_run; mod setting; pub mod thread; mod vm_new; @@ -17,12 +19,17 @@ mod vm_ops; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, - PyTypeRef, code::PyCode, pystr::AsPyStr, tuple::PyTuple, + self, PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, + PyStrRef, PyTypeRef, PyWeak, + code::PyCode, + dict::{PyDictItems, PyDictKeys, PyDictValues}, + pystr::AsPyStr, + tuple::PyTuple, }, codecs::CodecsRegistry, common::{hash::HashSecret, lock::PyMutex, rc::PyRc}, convert::ToPyObject, + exceptions::types::PyBaseException, frame::{ExecutionResult, Frame, FrameRef}, frozen::FrozenModule, function::{ArgMapping, FuncArgs, PySetterValue}, @@ -32,6 +39,12 @@ use crate::{ signal, stdlib, warn::WarningsState, }; +use alloc::{borrow::Cow, collections::BTreeMap}; +use core::{ + cell::{Cell, OnceCell, RefCell}, + ptr::NonNull, + sync::atomic::{AtomicBool, Ordering}, +}; use crossbeam_utils::atomic::AtomicCell; #[cfg(unix)] use nix::{ @@ -39,17 +52,14 @@ use nix::{ unistd::getpid, }; use std::{ - borrow::Cow, - cell::{Cell, Ref, RefCell}, collections::{HashMap, HashSet}, ffi::{OsStr, OsString}, - sync::atomic::AtomicBool, }; pub use context::Context; -pub use interpreter::Interpreter; +pub use interpreter::{Interpreter, InterpreterBuilder}; pub(crate) use method::PyMethod; -pub use setting::{CheckHashPycsMode, Settings}; +pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings}; pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; @@ -63,31 +73,58 @@ pub struct VirtualMachine { pub builtins: PyRef<PyModule>, pub sys_module: PyRef<PyModule>, pub ctx: PyRc<Context>, - pub frames: RefCell<Vec<FrameRef>>, + pub frames: RefCell<Vec<FramePtr>>, pub wasm_id: Option<String>, exceptions: RefCell<ExceptionStack>, pub import_func: PyObjectRef, + pub(crate) importlib: PyObjectRef, pub profile_func: RefCell<PyObjectRef>, pub trace_func: RefCell<PyObjectRef>, pub use_tracing: Cell<bool>, pub recursion_limit: Cell<usize>, - pub(crate) signal_handlers: Option<Box<RefCell<[Option<PyObjectRef>; signal::NSIG]>>>, + pub(crate) signal_handlers: OnceCell<Box<RefCell<[Option<PyObjectRef>; signal::NSIG]>>>, pub(crate) signal_rx: Option<signal::UserSignalReceiver>, pub repr_guards: RefCell<HashSet<usize>>, pub state: PyRc<PyGlobalState>, pub initialized: bool, recursion_depth: Cell<usize>, + /// C stack soft limit for detecting stack overflow (like c_stack_soft_limit) + c_stack_soft_limit: Cell<usize>, + /// Async generator firstiter hook (per-thread, set via sys.set_asyncgen_hooks) + pub async_gen_firstiter: RefCell<Option<PyObjectRef>>, + /// Async generator finalizer hook (per-thread, set via sys.set_asyncgen_hooks) + pub async_gen_finalizer: RefCell<Option<PyObjectRef>>, + /// Current running asyncio event loop for this thread + pub asyncio_running_loop: RefCell<Option<PyObjectRef>>, + /// Current running asyncio task for this thread + pub asyncio_running_task: RefCell<Option<PyObjectRef>>, } +/// Non-owning frame pointer for the frames stack. +/// The pointed-to frame is kept alive by the caller of with_frame_exc/resume_gen_frame. +#[derive(Copy, Clone)] +pub struct FramePtr(NonNull<Py<Frame>>); + +impl FramePtr { + /// # Safety + /// The pointed-to frame must still be alive. + pub unsafe fn as_ref(&self) -> &Py<Frame> { + unsafe { self.0.as_ref() } + } +} + +// SAFETY: FramePtr is only stored in the VM's frames Vec while the corresponding +// FrameRef is alive on the call stack. The Vec is always empty when the VM moves between threads. +unsafe impl Send for FramePtr {} + #[derive(Debug, Default)] struct ExceptionStack { - exc: Option<PyBaseExceptionRef>, - prev: Option<Box<ExceptionStack>>, + stack: Vec<Option<PyBaseExceptionRef>>, } pub struct PyGlobalState { - pub settings: Settings, - pub module_inits: stdlib::StdlibMap, + pub config: PyConfig, + pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>, pub stacksize: AtomicCell<usize>, pub thread_count: AtomicCell<usize>, @@ -102,6 +139,22 @@ pub struct PyGlobalState { pub after_forkers_parent: PyMutex<Vec<PyObjectRef>>, pub int_max_str_digits: AtomicCell<usize>, pub switch_interval: AtomicCell<f64>, + /// Global trace function for all threads (set by sys._settraceallthreads) + pub global_trace_func: PyMutex<Option<PyObjectRef>>, + /// Global profile function for all threads (set by sys._setprofileallthreads) + pub global_profile_func: PyMutex<Option<PyObjectRef>>, + /// Main thread identifier (pthread_self on Unix) + #[cfg(feature = "threading")] + pub main_thread_ident: AtomicCell<u64>, + /// Registry of all threads' slots for sys._current_frames() and sys._current_exceptions() + #[cfg(feature = "threading")] + pub thread_frames: parking_lot::Mutex<HashMap<u64, stdlib::thread::CurrentFrameSlot>>, + /// Registry of all ThreadHandles for fork cleanup + #[cfg(feature = "threading")] + pub thread_handles: parking_lot::Mutex<Vec<stdlib::thread::HandleEntry>>, + /// Registry for non-daemon threads that need to be joined at shutdown + #[cfg(feature = "threading")] + pub shutdown_handles: parking_lot::Mutex<Vec<stdlib::thread::ShutdownEntry>>, } pub fn process_hash_secret_seed() -> u32 { @@ -112,8 +165,22 @@ pub fn process_hash_secret_seed() -> u32 { } impl VirtualMachine { + /// Check whether the current thread is the main thread. + /// Mirrors `_Py_ThreadCanHandleSignals`. + #[allow(dead_code)] + pub(crate) fn is_main_thread(&self) -> bool { + #[cfg(feature = "threading")] + { + crate::stdlib::thread::get_ident() == self.state.main_thread_ident.load() + } + #[cfg(not(feature = "threading"))] + { + true + } + } + /// Create a new `VirtualMachine` structure. - fn new(settings: Settings, ctx: PyRc<Context>) -> Self { + pub(crate) fn new(ctx: PyRc<Context>, state: PyRc<PyGlobalState>) -> Self { flame_guard!("new VirtualMachine"); // make a new module without access to the vm; doesn't @@ -127,34 +194,16 @@ impl VirtualMachine { }; // Hard-core modules: - let builtins = new_module(stdlib::builtins::__module_def(&ctx)); - let sys_module = new_module(stdlib::sys::__module_def(&ctx)); + let builtins = new_module(stdlib::builtins::module_def(&ctx)); + let sys_module = new_module(stdlib::sys::module_def(&ctx)); let import_func = ctx.none(); + let importlib = ctx.none(); let profile_func = RefCell::new(ctx.none()); let trace_func = RefCell::new(ctx.none()); - let signal_handlers = Some(Box::new( - // putting it in a const optimizes better, prevents linear initialization of the array - const { RefCell::new([const { None }; signal::NSIG]) }, - )); - - let module_inits = stdlib::get_module_inits(); - - let seed = match settings.hash_seed { - Some(seed) => seed, - None => process_hash_secret_seed(), - }; - let hash_secret = HashSecret::new(seed); - - let codec_registry = CodecsRegistry::new(&ctx); + let signal_handlers = OnceCell::from(signal::new_signal_handlers()); - let warnings = WarningsState::init_state(&ctx); - - let int_max_str_digits = AtomicCell::new(match settings.int_max_str_digits { - -1 => 4300, - other => other, - } as usize); - let mut vm = Self { + let vm = Self { builtins, sys_module, ctx, @@ -162,6 +211,7 @@ impl VirtualMachine { wasm_id: None, exceptions: RefCell::default(), import_func, + importlib, profile_func, trace_func, use_tracing: Cell::new(false), @@ -169,26 +219,14 @@ impl VirtualMachine { signal_handlers, signal_rx: None, repr_guards: RefCell::default(), - state: PyRc::new(PyGlobalState { - settings, - module_inits, - frozen: HashMap::default(), - stacksize: AtomicCell::new(0), - thread_count: AtomicCell::new(0), - hash_secret, - atexit_funcs: PyMutex::default(), - codec_registry, - finalizing: AtomicBool::new(false), - warnings, - override_frozen_modules: AtomicCell::new(0), - before_forkers: PyMutex::default(), - after_forkers_child: PyMutex::default(), - after_forkers_parent: PyMutex::default(), - int_max_str_digits, - switch_interval: AtomicCell::new(0.005), - }), + state, initialized: false, recursion_depth: Cell::new(0), + c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()), + async_gen_firstiter: RefCell::new(None), + async_gen_finalizer: RefCell::new(None), + asyncio_running_loop: RefCell::new(None), + asyncio_running_task: RefCell::new(None), }; if vm.state.hash_secret.hash_str("") @@ -201,9 +239,6 @@ impl VirtualMachine { panic!("Interpreters in same process must share the hash seed"); } - let frozen = core_frozen_inits().collect(); - PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen; - vm.builtins.init_dict( vm.ctx.intern_str("builtins"), Some(vm.ctx.intern_str(stdlib::builtins::DOC.unwrap()).to_owned()), @@ -226,18 +261,18 @@ impl VirtualMachine { let rustpythonpath_env = std::env::var("RUSTPYTHONPATH").ok(); let pythonpath_env = std::env::var("PYTHONPATH").ok(); let env_set = rustpythonpath_env.as_ref().is_some() || pythonpath_env.as_ref().is_some(); - let path_contains_env = self.state.settings.path_list.iter().any(|s| { + let path_contains_env = self.state.config.paths.module_search_paths.iter().any(|s| { Some(s.as_str()) == rustpythonpath_env.as_deref() || Some(s.as_str()) == pythonpath_env.as_deref() }); let guide_message = if cfg!(feature = "freeze-stdlib") { - "`rustpython_pylib` maybe not set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterConfig::init_stdlib` or manually call `vm.add_frozen(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::with_init`." + "`rustpython_pylib` may not be set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterBuilder::init_stdlib` or manually call `builder.add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::builder()`." } else if !env_set { "Neither RUSTPYTHONPATH nor PYTHONPATH is set. Try setting one of them to the stdlib directory." } else if path_contains_env { "RUSTPYTHONPATH or PYTHONPATH is set, but it doesn't contain the encodings library. If you are customizing the RustPython vm/interpreter, try adding the stdlib directory to the path. If you are developing the RustPython interpreter, it might be a bug during development." } else { - "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `Settings::path_list`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." + "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `PyConfig::paths::module_search_paths`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." }; let mut msg = format!( @@ -258,21 +293,44 @@ impl VirtualMachine { Ok(()) } - fn import_utf8_encodings(&mut self) -> PyResult<()> { - import::import_frozen(self, "codecs")?; - // FIXME: See corresponding part of `core_frozen_inits` - // let encoding_module_name = if cfg!(feature = "freeze-stdlib") { - // "encodings.utf_8" - // } else { - // "encodings_utf_8" - // }; - let encoding_module_name = "encodings_utf_8"; - let encoding_module = import::import_frozen(self, encoding_module_name)?; - let getregentry = encoding_module.get_attr("getregentry", self)?; + fn import_ascii_utf8_encodings(&mut self) -> PyResult<()> { + // Use the Python import machinery (FrozenImporter) so modules get + // proper __spec__ and __loader__ attributes. + self.import("codecs", 0)?; + + // Use dotted names when freeze-stdlib is enabled (modules come from Lib/encodings/), + // otherwise use underscored names (modules come from core_modules/). + let (ascii_module_name, utf8_module_name) = if cfg!(feature = "freeze-stdlib") { + ("encodings.ascii", "encodings.utf_8") + } else { + ("encodings_ascii", "encodings_utf_8") + }; + + // Register ascii encoding + // __import__("encodings.ascii") returns top-level "encodings", so + // look up the actual submodule in sys.modules. + self.import(ascii_module_name, 0)?; + let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + let ascii_module = sys_modules.get_item(ascii_module_name, self)?; + let getregentry = ascii_module.get_attr("getregentry", self)?; + let codec_info = getregentry.call((), self)?; + self.state + .codec_registry + .register_manual("ascii", codec_info.try_into_value(self)?)?; + + // Register utf-8 encoding (also as "utf8" alias since normalize_encoding_name + // maps "utf-8" → "utf_8" but leaves "utf8" as-is) + self.import(utf8_module_name, 0)?; + let utf8_module = sys_modules.get_item(utf8_module_name, self)?; + let getregentry = utf8_module.get_attr("getregentry", self)?; let codec_info = getregentry.call((), self)?; + let utf8_codec: crate::codecs::PyCodec = codec_info.try_into_value(self)?; + self.state + .codec_registry + .register_manual("utf-8", utf8_codec.clone())?; self.state .codec_registry - .register_manual("utf-8", codec_info.try_into_value(self)?)?; + .register_manual("utf8", utf8_codec)?; Ok(()) } @@ -283,32 +341,42 @@ impl VirtualMachine { panic!("Double Initialize Error"); } + // Initialize main thread ident before any threading operations + #[cfg(feature = "threading")] + stdlib::thread::init_main_thread_ident(self); + stdlib::builtins::init_module(self, &self.builtins); stdlib::sys::init_module(self, &self.sys_module, &self.builtins); + self.expect_pyresult( + stdlib::sys::set_bootstrap_stderr(self), + "failed to initialize bootstrap stderr", + ); let mut essential_init = || -> PyResult { import::import_builtin(self, "_typing")?; - #[cfg(not(target_arch = "wasm32"))] + #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] import::import_builtin(self, "_signal")?; #[cfg(any(feature = "parser", feature = "compiler"))] import::import_builtin(self, "_ast")?; #[cfg(not(feature = "threading"))] import::import_frozen(self, "_thread")?; let importlib = import::init_importlib_base(self)?; - self.import_utf8_encodings()?; + self.import_ascii_utf8_encodings()?; - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { let io = import::import_builtin(self, "_io")?; - #[cfg(feature = "stdio")] - let make_stdio = |name, fd, write| { - let buffered_stdio = self.state.settings.buffered_stdio; + + // Full stdio: FileIO → BufferedWriter → TextIOWrapper + #[cfg(feature = "host_env")] + let make_stdio = |name: &str, fd: i32, write: bool| { + let buffered_stdio = self.state.config.settings.buffered_stdio; let unbuffered = write && !buffered_stdio; let buf = crate::stdlib::io::open( self.ctx.new_int(fd).into(), Some(if write { "wb" } else { "rb" }), crate::stdlib::io::OpenArgs { buffering: if unbuffered { 0 } else { -1 }, + closefd: false, ..Default::default() }, self, @@ -324,22 +392,52 @@ impl VirtualMachine { let line_buffering = buffered_stdio && (isatty || fd == 2); let newline = if cfg!(windows) { None } else { Some("\n") }; + let encoding = self.state.config.settings.stdio_encoding.as_deref(); + // stderr always uses backslashreplace (ignores stdio_errors) + let errors = if fd == 2 { + Some("backslashreplace") + } else { + self.state.config.settings.stdio_errors.as_deref() + }; let stdio = self.call_method( &io, "TextIOWrapper", - (buf, (), (), newline, line_buffering, write_through), + ( + buf, + encoding, + errors, + newline, + line_buffering, + write_through, + ), )?; let mode = if write { "w" } else { "r" }; stdio.set_attr("mode", self.ctx.new_str(mode), self)?; - Ok(stdio) + Ok::<_, self::PyBaseExceptionRef>(stdio) + }; + + // Sandbox stdio: lightweight wrapper using Rust's std::io directly + #[cfg(all(not(feature = "host_env"), feature = "stdio"))] + let make_stdio = |name: &str, fd: i32, write: bool| { + let mode = if write { "w" } else { "r" }; + let stdio = stdlib::sys::SandboxStdio { + fd, + name: format!("<{name}>"), + mode: mode.to_owned(), + } + .into_ref(&self.ctx); + Ok(stdio.into()) }; + + // No stdio: set to None (embedding use case) #[cfg(not(feature = "stdio"))] - let make_stdio = - |_name, _fd, _write| Ok(crate::builtins::PyNone.into_pyobject(self)); + let make_stdio = |_name: &str, _fd: i32, _write: bool| { + Ok(crate::builtins::PyNone.into_pyobject(self)) + }; let set_stdio = |name, fd, write| { - let stdio = make_stdio(name, fd, write)?; + let stdio: PyObjectRef = make_stdio(name, fd, write)?; let dunder_name = self.ctx.intern_str(format!("__{name}__")); self.sys_module.set_attr( dunder_name, // e.g. __stdin__ @@ -363,7 +461,8 @@ impl VirtualMachine { let res = essential_init(); let importlib = self.expect_pyresult(res, "essential initialization failed"); - if self.state.settings.allow_external_library + #[cfg(feature = "host_env")] + if self.state.config.settings.allow_external_library && cfg!(feature = "rustpython-compiler") && let Err(e) = import::init_importlib_package(self, importlib) { @@ -373,8 +472,11 @@ impl VirtualMachine { self.print_exception(e); } - let _expect_stdlib = - cfg!(feature = "freeze-stdlib") || !self.state.settings.path_list.is_empty(); + #[cfg(not(feature = "host_env"))] + let _ = importlib; + + let _expect_stdlib = cfg!(feature = "freeze-stdlib") + || !self.state.config.paths.module_search_paths.is_empty(); #[cfg(feature = "encodings")] if _expect_stdlib { @@ -388,7 +490,7 @@ impl VirtualMachine { // Here may not be the best place to give general `path_list` advice, // but bare rustpython_vm::VirtualMachine users skipped proper settings must hit here while properly setup vm never enters here. eprintln!( - "feature `encodings` is enabled but `settings.path_list` is empty. \ + "feature `encodings` is enabled but `paths.module_search_paths` is empty. \ Please add the library path to `settings.path_list`. If you intended to disable the entire standard library (including the `encodings` feature), please also make sure to disable the `encodings` feature.\n\ Tip: You may also want to add `\"\"` to `settings.path_list` in order to enable importing from the current working directory." ); @@ -397,53 +499,84 @@ impl VirtualMachine { self.initialized = true; } - fn state_mut(&mut self) -> &mut PyGlobalState { - PyRc::get_mut(&mut self.state) - .expect("there should not be multiple threads while a user has a mut ref to a vm") - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_native_module<S>(&mut self, name: S, module: stdlib::StdlibInitFunc) - where - S: Into<Cow<'static, str>>, - { - self.state_mut().module_inits.insert(name.into(), module); - } - - pub fn add_native_modules<I>(&mut self, iter: I) - where - I: IntoIterator<Item = (Cow<'static, str>, stdlib::StdlibInitFunc)>, - { - self.state_mut().module_inits.extend(iter); - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_frozen<I>(&mut self, frozen: I) - where - I: IntoIterator<Item = (&'static str, FrozenModule)>, - { - self.state_mut().frozen.extend(frozen); - } - /// Set the custom signal channel for the interpreter pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { self.signal_rx = Some(signal_rx); } + /// Execute Python bytecode (`.pyc`) from an in-memory buffer. + /// + /// When the RustPython CLI is available, `.pyc` files are normally executed by + /// invoking `rustpython <input>.pyc`. This method provides an alternative for + /// environments where the binary is unavailable or file I/O is restricted + /// (e.g. WASM). + /// + /// ## Preparing a `.pyc` file + /// + /// First, compile a Python source file into bytecode: + /// + /// ```sh + /// # Generate a .pyc file + /// $ rustpython -m py_compile <input>.py + /// ``` + /// + /// ## Running the bytecode + /// + /// Load the resulting `.pyc` file into memory and execute it using the VM: + /// + /// ```no_run + /// use rustpython_vm::Interpreter; + /// Interpreter::without_stdlib(Default::default()).enter(|vm| { + /// let bytes = std::fs::read("__pycache__/<input>.rustpython-314.pyc").unwrap(); + /// let main_scope = vm.new_scope_with_main().unwrap(); + /// vm.run_pyc_bytes(&bytes, main_scope); + /// }); + /// ``` + pub fn run_pyc_bytes(&self, pyc_bytes: &[u8], scope: Scope) -> PyResult<()> { + let code = PyCode::from_pyc(pyc_bytes, Some("<pyc_bytes>"), None, None, self)?; + self.with_simple_run("<source>", |_module_dict| { + self.run_code_obj(code, scope)?; + Ok(()) + }) + } + pub fn run_code_obj(&self, code: PyRef<PyCode>, scope: Scope) -> PyResult { - use crate::builtins::PyFunction; + use crate::builtins::{PyFunction, PyModule}; // Create a function object for module code, similar to CPython's PyEval_EvalCode let func = PyFunction::new(code.clone(), scope.globals.clone(), self)?; let func_obj = func.into_ref(&self.ctx).into(); - let frame = Frame::new(code, scope, self.builtins.dict(), &[], Some(func_obj), self) - .into_ref(&self.ctx); + // Extract builtins from globals["__builtins__"], like PyEval_EvalCode + let builtins = match scope + .globals + .get_item_opt(identifier!(self, __builtins__), self)? + { + Some(b) => { + if let Some(module) = b.downcast_ref::<PyModule>() { + module.dict().into() + } else { + b + } + } + None => self.builtins.dict().into(), + }; + + let frame = + Frame::new(code, scope, builtins, &[], Some(func_obj), self).into_ref(&self.ctx); self.run_frame(frame) } #[cold] pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option<String>, object: PyObjectRef) { + // During interpreter finalization, sys.unraisablehook may not be available, + // but we still need to report exceptions (especially from atexit callbacks). + // Write directly to stderr like PyErr_FormatUnraisable. + if self.state.finalizing.load(Ordering::Acquire) { + self.write_unraisable_to_stderr(&e, msg.as_deref(), &object); + return; + } + let sys_module = self.import("sys", 0).unwrap(); let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); @@ -462,6 +595,57 @@ impl VirtualMachine { } } + /// Write unraisable exception to stderr during finalization. + /// Similar to _PyErr_WriteUnraisableDefaultHook in CPython. + fn write_unraisable_to_stderr( + &self, + e: &PyBaseExceptionRef, + msg: Option<&str>, + object: &PyObjectRef, + ) { + // Get stderr once and reuse it + let stderr = crate::stdlib::sys::get_stderr(self).ok(); + + let write_to_stderr = |s: &str, stderr: &Option<PyObjectRef>, vm: &VirtualMachine| { + if let Some(stderr) = stderr { + let _ = vm.call_method(stderr, "write", (s.to_owned(),)); + } else { + eprint!("{}", s); + } + }; + + // Format: "Exception ignored {msg} {object_repr}\n" + if let Some(msg) = msg { + write_to_stderr(&format!("Exception ignored {}", msg), &stderr, self); + } else { + write_to_stderr("Exception ignored in: ", &stderr, self); + } + + if let Ok(repr) = object.repr(self) { + write_to_stderr(&format!("{}\n", repr.as_str()), &stderr, self); + } else { + write_to_stderr("<object repr failed>\n", &stderr, self); + } + + // Write exception type and message + let exc_type_name = e.class().name(); + if let Ok(exc_str) = e.as_object().str(self) { + let exc_str = exc_str.as_str(); + if exc_str.is_empty() { + write_to_stderr(&format!("{}\n", exc_type_name), &stderr, self); + } else { + write_to_stderr(&format!("{}: {}\n", exc_type_name, exc_str), &stderr, self); + } + } else { + write_to_stderr(&format!("{}\n", exc_type_name), &stderr, self); + } + + // Flush stderr to ensure output is visible + if let Some(ref stderr) = stderr { + let _ = self.call_method(stderr, "flush", ()); + } + } + #[inline(always)] pub fn run_frame(&self, frame: FrameRef) -> PyResult { match self.with_frame(frame, |f| f.run(self))? { @@ -470,41 +654,502 @@ impl VirtualMachine { } } + /// Run `run` with main scope. + fn with_simple_run( + &self, + path: &str, + run: impl FnOnce(&Py<PyDict>) -> PyResult<()>, + ) -> PyResult<()> { + let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + let main_module = sys_modules.get_item(identifier!(self, __main__), self)?; + let module_dict = main_module.dict().expect("main module must have __dict__"); + + // Track whether we set __file__ (for cleanup) + let set_file_name = !module_dict.contains_key(identifier!(self, __file__), self); + if set_file_name { + module_dict.set_item( + identifier!(self, __file__), + self.ctx.new_str(path).into(), + self, + )?; + module_dict.set_item(identifier!(self, __cached__), self.ctx.none(), self)?; + } + + let result = run(&module_dict); + + self.flush_io(); + + // Cleanup __file__ and __cached__ after execution + if set_file_name { + let _ = module_dict.del_item(identifier!(self, __file__), self); + let _ = module_dict.del_item(identifier!(self, __cached__), self); + } + + result + } + + /// flush_io + /// + /// Flush stdout and stderr. Errors are silently ignored. + fn flush_io(&self) { + if let Ok(stdout) = self.sys_module.get_attr("stdout", self) { + let _ = self.call_method(&stdout, identifier!(self, flush).as_str(), ()); + } + if let Ok(stderr) = self.sys_module.get_attr("stderr", self) { + let _ = self.call_method(&stderr, identifier!(self, flush).as_str(), ()); + } + } + + /// Clear module references during shutdown. + /// Follows the same phased algorithm as pylifecycle.c finalize_modules(): + /// no hardcoded module names, reverse import order, only builtins/sys last. + pub fn finalize_modules(&self) { + // Phase 1: Set special sys/builtins attributes to None, restore stdio + self.finalize_modules_delete_special(); + + // Phase 2: Remove all modules from sys.modules (set values to None), + // and collect weakrefs to modules preserving import order. + // Also keeps strong refs (module_refs) to prevent premature deallocation. + // CPython uses _PyGC_CollectNoFail() here to collect __globals__ cycles; + // since RustPython has no working GC, we keep modules alive through + // Phase 4 so their dicts can be explicitly cleared. + let (module_weakrefs, module_refs) = self.finalize_remove_modules(); + + // Phase 3: Clear sys.modules dict + self.finalize_clear_modules_dict(); + + // Phase 4: Clear module dicts in reverse import order using 2-pass algorithm. + // All modules are still alive (held by module_refs), so all weakrefs are valid. + // This breaks __globals__ cycles: dict entries set to None → functions freed → + // __globals__ refs dropped → dict refcount decreases. + self.finalize_clear_module_dicts(&module_weakrefs); + + // Drop strong refs → modules freed with already-cleared dicts. + // No __globals__ cycles remain (broken by Phase 4). + drop(module_refs); + + // Phase 5: Clear sys and builtins dicts last + self.finalize_clear_sys_builtins_dict(); + } + + /// Phase 1: Set special sys attributes to None and restore stdio. + fn finalize_modules_delete_special(&self) { + let none = self.ctx.none(); + let sys_dict = self.sys_module.dict(); + + // Set special sys attributes to None + for attr in &[ + "path", + "argv", + "ps1", + "ps2", + "last_exc", + "last_type", + "last_value", + "last_traceback", + "path_importer_cache", + "meta_path", + "path_hooks", + ] { + let _ = sys_dict.set_item(*attr, none.clone(), self); + } + + // Restore stdin/stdout/stderr from __stdin__/__stdout__/__stderr__ + for (std_name, dunder_name) in &[ + ("stdin", "__stdin__"), + ("stdout", "__stdout__"), + ("stderr", "__stderr__"), + ] { + let restored = sys_dict + .get_item_opt(*dunder_name, self) + .ok() + .flatten() + .unwrap_or_else(|| none.clone()); + let _ = sys_dict.set_item(*std_name, restored, self); + } + + // builtins._ = None + let _ = self.builtins.dict().set_item("_", none, self); + } + + /// Phase 2: Set all sys.modules values to None and collect weakrefs to modules. + /// Returns (weakrefs for Phase 4, strong refs to keep modules alive). + fn finalize_remove_modules(&self) -> (Vec<(String, PyRef<PyWeak>)>, Vec<PyObjectRef>) { + let mut module_weakrefs = Vec::new(); + let mut module_refs = Vec::new(); + + let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) else { + return (module_weakrefs, module_refs); + }; + let Some(modules_dict) = modules.downcast_ref::<PyDict>() else { + return (module_weakrefs, module_refs); + }; + + let none = self.ctx.none(); + let items: Vec<_> = modules_dict.into_iter().collect(); + + for (key, value) in items { + let name = key + .downcast_ref::<PyStr>() + .map(|s| s.as_str().to_owned()) + .unwrap_or_default(); + + // Save weakref and strong ref to module for later clearing + if value.downcast_ref::<PyModule>().is_some() { + if let Ok(weak) = value.downgrade(None, self) { + module_weakrefs.push((name, weak)); + } + module_refs.push(value.clone()); + } + + // Set the value to None in sys.modules + let _ = modules_dict.set_item(&*key, none.clone(), self); + } + + (module_weakrefs, module_refs) + } + + /// Phase 3: Clear sys.modules dict. + fn finalize_clear_modules_dict(&self) { + if let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) + && let Some(modules_dict) = modules.downcast_ref::<PyDict>() + { + modules_dict.clear(); + } + } + + /// Phase 4: Clear module dicts in reverse import order using 2-pass algorithm. + /// Without GC, only clear __main__ — other modules' __del__ handlers + /// need their globals intact. CPython can clear ALL module dicts because + /// _PyGC_CollectNoFail() finalizes cycle-participating objects beforehand. + fn finalize_clear_module_dicts(&self, module_weakrefs: &[(String, PyRef<PyWeak>)]) { + for (name, weakref) in module_weakrefs.iter().rev() { + // Only clear __main__ — user objects with __del__ get finalized + // while other modules' globals remain intact for their __del__ handlers. + if name != "__main__" { + continue; + } + + let Some(module_obj) = weakref.upgrade() else { + continue; + }; + let Some(module) = module_obj.downcast_ref::<PyModule>() else { + continue; + }; + + Self::module_clear_dict(&module.dict(), self); + } + } + + /// 2-pass module dict clearing (_PyModule_ClearDict algorithm). + /// Pass 1: Set names starting with '_' (except __builtins__) to None. + /// Pass 2: Set all remaining names (except __builtins__) to None. + pub(crate) fn module_clear_dict(dict: &Py<PyDict>, vm: &VirtualMachine) { + let none = vm.ctx.none(); + + // Pass 1: names starting with '_' (except __builtins__) + for (key, value) in dict.into_iter().collect::<Vec<_>>() { + if vm.is_none(&value) { + continue; + } + if let Some(key_str) = key.downcast_ref::<PyStr>() { + let name = key_str.as_str(); + if name.starts_with('_') && name != "__builtins__" && name != "__spec__" { + let _ = dict.set_item(name, none.clone(), vm); + } + } + } + + // Pass 2: all remaining (except __builtins__) + for (key, value) in dict.into_iter().collect::<Vec<_>>() { + if vm.is_none(&value) { + continue; + } + if let Some(key_str) = key.downcast_ref::<PyStr>() + && key_str.as_str() != "__builtins__" + && key_str.as_str() != "__spec__" + { + let _ = dict.set_item(key_str.as_str(), none.clone(), vm); + } + } + } + + /// Phase 5: Clear sys and builtins dicts last. + fn finalize_clear_sys_builtins_dict(&self) { + Self::module_clear_dict(&self.sys_module.dict(), self); + Self::module_clear_dict(&self.builtins.dict(), self); + } + pub fn current_recursion_depth(&self) -> usize { self.recursion_depth.get() } + /// Stack margin bytes (like _PyOS_STACK_MARGIN_BYTES). + /// 2048 * sizeof(void*) = 16KB for 64-bit. + const STACK_MARGIN_BYTES: usize = 2048 * core::mem::size_of::<usize>(); + + /// Get the stack boundaries using platform-specific APIs. + /// Returns (base, top) where base is the lowest address and top is the highest. + #[cfg(all(not(miri), windows))] + fn get_stack_bounds() -> (usize, usize) { + use windows_sys::Win32::System::Threading::{ + GetCurrentThreadStackLimits, SetThreadStackGuarantee, + }; + let mut low: usize = 0; + let mut high: usize = 0; + unsafe { + GetCurrentThreadStackLimits(&mut low as *mut usize, &mut high as *mut usize); + // Add the guaranteed stack space (reserved for exception handling) + let mut guarantee: u32 = 0; + SetThreadStackGuarantee(&mut guarantee); + low += guarantee as usize; + } + (low, high) + } + + /// Get stack boundaries on non-Windows platforms. + /// Falls back to estimating based on current stack pointer. + #[cfg(all(not(miri), not(windows)))] + fn get_stack_bounds() -> (usize, usize) { + // Use pthread_attr_getstack on platforms that support it + #[cfg(any(target_os = "linux", target_os = "android"))] + { + use libc::{ + pthread_attr_destroy, pthread_attr_getstack, pthread_attr_t, pthread_getattr_np, + pthread_self, + }; + let mut attr: pthread_attr_t = unsafe { core::mem::zeroed() }; + unsafe { + if pthread_getattr_np(pthread_self(), &mut attr) == 0 { + let mut stack_addr: *mut libc::c_void = core::ptr::null_mut(); + let mut stack_size: libc::size_t = 0; + if pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size) == 0 { + pthread_attr_destroy(&mut attr); + let base = stack_addr as usize; + let top = base + stack_size; + return (base, top); + } + pthread_attr_destroy(&mut attr); + } + } + } + + #[cfg(target_os = "macos")] + { + use libc::{pthread_get_stackaddr_np, pthread_get_stacksize_np, pthread_self}; + unsafe { + let thread = pthread_self(); + let stack_top = pthread_get_stackaddr_np(thread) as usize; + let stack_size = pthread_get_stacksize_np(thread); + let stack_base = stack_top - stack_size; + return (stack_base, stack_top); + } + } + + // Fallback: estimate based on current SP and a default stack size + #[allow(unreachable_code)] + { + let current_sp = psm::stack_pointer() as usize; + // Assume 8MB stack, estimate base + let estimated_size = 8 * 1024 * 1024; + let base = current_sp.saturating_sub(estimated_size); + let top = current_sp + 1024 * 1024; // Assume we're not at the very top + (base, top) + } + } + + /// Calculate the C stack soft limit based on actual stack boundaries. + /// soft_limit = base + 2 * margin (for downward-growing stacks) + #[cfg(not(miri))] + fn calculate_c_stack_soft_limit() -> usize { + let (base, _top) = Self::get_stack_bounds(); + // Soft limit is 2 margins above the base + base + Self::STACK_MARGIN_BYTES * 2 + } + + /// Miri doesn't support inline assembly, so disable C stack checking. + #[cfg(miri)] + fn calculate_c_stack_soft_limit() -> usize { + 0 + } + + /// Check if we're near the C stack limit (like _Py_MakeRecCheck). + /// Returns true only when stack pointer is in the "danger zone" between + /// soft_limit and hard_limit (soft_limit - 2*margin). + #[cfg(not(miri))] + #[inline(always)] + fn check_c_stack_overflow(&self) -> bool { + let current_sp = psm::stack_pointer() as usize; + let soft_limit = self.c_stack_soft_limit.get(); + // Stack grows downward: check if we're below soft limit but above hard limit + // This matches CPython's _Py_MakeRecCheck behavior + current_sp < soft_limit + && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2) + } + + /// Miri doesn't support inline assembly, so always return false. + #[cfg(miri)] + #[inline(always)] + fn check_c_stack_overflow(&self) -> bool { + false + } + /// Used to run the body of a (possibly) recursive function. It will raise a /// RecursionError if recursive functions are nested far too many times, /// preventing a stack overflow. pub fn with_recursion<R, F: FnOnce() -> PyResult<R>>(&self, _where: &str, f: F) -> PyResult<R> { self.check_recursive_call(_where)?; - self.recursion_depth.set(self.recursion_depth.get() + 1); - let result = f(); - self.recursion_depth.set(self.recursion_depth.get() - 1); - result + + // Native stack guard: check C stack like _Py_MakeRecCheck + if self.check_c_stack_overflow() { + return Err(self.new_recursion_error(_where.to_string())); + } + + self.recursion_depth.update(|d| d + 1); + scopeguard::defer! { self.recursion_depth.update(|d| d - 1) } + f() } pub fn with_frame<R, F: FnOnce(FrameRef) -> PyResult<R>>( &self, frame: FrameRef, f: F, + ) -> PyResult<R> { + self.with_frame_exc(frame, None, f) + } + + /// Like `with_frame` but allows specifying the initial exception state. + pub fn with_frame_exc<R, F: FnOnce(FrameRef) -> PyResult<R>>( + &self, + frame: FrameRef, + exc: Option<PyBaseExceptionRef>, + f: F, ) -> PyResult<R> { self.with_recursion("", || { - self.frames.borrow_mut().push(frame.clone()); - let result = f(frame); - // defer dec frame - let _popped = self.frames.borrow_mut().pop(); - result + // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, + // keeping the FramePtr valid. We pass a clone to `f` so that `f` + // consuming its FrameRef doesn't invalidate our pointer. + let fp = FramePtr(NonNull::from(&*frame)); + self.frames.borrow_mut().push(fp); + // Update the shared frame stack for sys._current_frames() and faulthandler + #[cfg(feature = "threading")] + crate::vm::thread::push_thread_frame(fp); + // Link frame into the signal-safe frame chain (previous pointer) + let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); + frame.previous.store( + old_frame as *mut Frame, + core::sync::atomic::Ordering::Relaxed, + ); + // Push exception context for frame isolation. + // For normal calls: None (clean slate). + // For generators: the saved exception from last yield. + self.push_exception(exc); + let old_owner = frame.owner.swap( + crate::frame::FrameOwner::Thread as i8, + core::sync::atomic::Ordering::AcqRel, + ); + + // Ensure cleanup on panic: restore owner, pop exception, frame chain, and frames Vec. + scopeguard::defer! { + frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); + self.pop_exception(); + crate::vm::thread::set_current_frame(old_frame); + self.frames.borrow_mut().pop(); + #[cfg(feature = "threading")] + crate::vm::thread::pop_thread_frame(); + } + + use crate::protocol::TraceEvent; + // Fire 'call' trace event after pushing frame + // (current_frame() now returns the callee's frame) + match self.trace_event(TraceEvent::Call, None) { + Ok(()) => { + // Set per-frame trace function so line events fire for this frame. + // Frames entered before sys.settrace() keep trace=None and skip line events. + if self.use_tracing.get() { + let trace_func = self.trace_func.borrow().clone(); + if !self.is_none(&trace_func) { + *frame.trace.lock() = trace_func; + } + } + let result = f(frame.clone()); + // Fire 'return' trace event on success + if result.is_ok() { + let _ = self.trace_event(TraceEvent::Return, None); + } + result + } + Err(e) => Err(e), + } }) } + /// Lightweight frame execution for generator/coroutine resume. + /// Pushes to the thread frame stack and fires trace/profile events, + /// but skips the thread exception update for performance. + pub fn resume_gen_frame<R, F: FnOnce(&Py<Frame>) -> PyResult<R>>( + &self, + frame: &FrameRef, + exc: Option<PyBaseExceptionRef>, + f: F, + ) -> PyResult<R> { + self.check_recursive_call("")?; + if self.check_c_stack_overflow() { + return Err(self.new_recursion_error(String::new())); + } + self.recursion_depth.update(|d| d + 1); + + // SAFETY: frame (&FrameRef) stays alive for the duration, so NonNull is valid until pop. + let fp = FramePtr(NonNull::from(&**frame)); + self.frames.borrow_mut().push(fp); + #[cfg(feature = "threading")] + crate::vm::thread::push_thread_frame(fp); + let old_frame = crate::vm::thread::set_current_frame((&***frame) as *const Frame); + frame.previous.store( + old_frame as *mut Frame, + core::sync::atomic::Ordering::Relaxed, + ); + // Inline exception push without thread exception update + self.exceptions.borrow_mut().stack.push(exc); + let old_owner = frame.owner.swap( + crate::frame::FrameOwner::Thread as i8, + core::sync::atomic::Ordering::AcqRel, + ); + + // Ensure cleanup on panic: restore owner, pop exception, frame chain, frames Vec, + // and recursion depth. + scopeguard::defer! { + frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); + self.exceptions.borrow_mut().stack + .pop() + .expect("pop_exception() without nested exc stack"); + crate::vm::thread::set_current_frame(old_frame); + self.frames.borrow_mut().pop(); + #[cfg(feature = "threading")] + crate::vm::thread::pop_thread_frame(); + self.recursion_depth.update(|d| d - 1); + } + + use crate::protocol::TraceEvent; + match self.trace_event(TraceEvent::Call, None) { + Ok(()) => { + let result = f(frame); + if result.is_ok() { + let _ = self.trace_event(TraceEvent::Return, None); + } + result + } + Err(e) => Err(e), + } + } + /// Returns a basic CompileOpts instance with options accurate to the vm. Used /// as the CompileOpts for `vm.compile()`. #[cfg(feature = "rustpython-codegen")] pub fn compile_opts(&self) -> crate::compiler::CompileOpts { crate::compiler::CompileOpts { - optimize: self.state.settings.optimize, + optimize: self.state.config.settings.optimize, + debug_ranges: self.state.config.settings.code_debug_ranges, } } @@ -517,15 +1162,11 @@ impl VirtualMachine { } } - pub fn current_frame(&self) -> Option<Ref<'_, FrameRef>> { - let frames = self.frames.borrow(); - if frames.is_empty() { - None - } else { - Some(Ref::map(self.frames.borrow(), |frames| { - frames.last().unwrap() - })) - } + pub fn current_frame(&self) -> Option<FrameRef> { + self.frames.borrow().last().map(|fp| { + // SAFETY: the caller keeps the FrameRef alive while it's in the Vec + unsafe { fp.as_ref() }.to_owned() + }) } pub fn current_locals(&self) -> PyResult<ArgMapping> { @@ -534,11 +1175,11 @@ impl VirtualMachine { .locals(self) } - pub fn current_globals(&self) -> Ref<'_, PyDictRef> { - let frame = self - .current_frame() - .expect("called current_globals but no frames on the stack"); - Ref::map(frame, |f| &f.globals) + pub fn current_globals(&self) -> PyDictRef { + self.current_frame() + .expect("called current_globals but no frames on the stack") + .globals + .clone() } pub fn try_class(&self, module: &'static str, class: &'static str) -> PyResult<PyTypeRef> { @@ -592,47 +1233,20 @@ impl VirtualMachine { from_list: &Py<PyTuple<PyStrRef>>, level: usize, ) -> PyResult { - // if the import inputs seem weird, e.g a package import or something, rather than just - // a straight `import ident` - let weird = module.as_str().contains('.') || level != 0 || !from_list.is_empty(); + let import_func = self + .builtins + .get_attr(identifier!(self, __import__), self) + .map_err(|_| self.new_import_error("__import__ not found", module.to_owned()))?; - let cached_module = if weird { - None + let (locals, globals) = if let Some(frame) = self.current_frame() { + (Some(frame.locals.clone()), Some(frame.globals.clone())) } else { - let sys_modules = self.sys_module.get_attr("modules", self)?; - sys_modules.get_item(module, self).ok() + (None, None) }; - - match cached_module { - Some(cached_module) => { - if self.is_none(&cached_module) { - Err(self.new_import_error( - format!("import of {module} halted; None in sys.modules"), - module.to_owned(), - )) - } else { - Ok(cached_module) - } - } - None => { - let import_func = self - .builtins - .get_attr(identifier!(self, __import__), self) - .map_err(|_| { - self.new_import_error("__import__ not found", module.to_owned()) - })?; - - let (locals, globals) = if let Some(frame) = self.current_frame() { - (Some(frame.locals.clone()), Some(frame.globals.clone())) - } else { - (None, None) - }; - let from_list: PyObjectRef = from_list.to_owned().into(); - import_func - .call((module.to_owned(), globals, locals, from_list, level), self) - .inspect_err(|exc| import::remove_importlib_frames(self, exc)) - } - } + let from_list: PyObjectRef = from_list.to_owned().into(); + import_func + .call((module.to_owned(), globals, locals, from_list, level), self) + .inspect_err(|exc| import::remove_importlib_frames(self, exc)) } pub fn extract_elements_with<T, F>(&self, value: &PyObject, func: F) -> PyResult<Vec<T>> @@ -647,6 +1261,29 @@ impl VirtualMachine { } else if cls.is(self.ctx.types.list_type) { list_borrow = value.downcast_ref::<PyList>().unwrap().borrow_vec(); &list_borrow + } else if cls.is(self.ctx.types.dict_keys_type) { + // Atomic snapshot of dict keys - prevents race condition during iteration + let keys = value.downcast_ref::<PyDictKeys>().unwrap().dict.keys_vec(); + return keys.into_iter().map(func).collect(); + } else if cls.is(self.ctx.types.dict_values_type) { + // Atomic snapshot of dict values - prevents race condition during iteration + let values = value + .downcast_ref::<PyDictValues>() + .unwrap() + .dict + .values_vec(); + return values.into_iter().map(func).collect(); + } else if cls.is(self.ctx.types.dict_items_type) { + // Atomic snapshot of dict items - prevents race condition during iteration + let items = value + .downcast_ref::<PyDictItems>() + .unwrap() + .dict + .items_vec(); + return items + .into_iter() + .map(|(k, v)| func(self.ctx.new_tuple(vec![k, v]).into())) + .collect(); } else { return self.map_py_iter(value, func); }; @@ -728,12 +1365,20 @@ impl VirtualMachine { pub fn set_attribute_error_context( &self, - exc: &PyBaseExceptionRef, + exc: &Py<PyBaseException>, obj: PyObjectRef, name: PyStrRef, ) { if exc.class().is(self.ctx.exceptions.attribute_error) { let exc = exc.as_object(); + // Check if this exception was already augmented + let already_set = exc + .get_attr("name", self) + .ok() + .is_some_and(|v| !self.is_none(&v)); + if already_set { + return; + } exc.set_attr("name", name, self).unwrap(); exc.set_attr("obj", obj, self).unwrap(); } @@ -787,44 +1432,69 @@ impl VirtualMachine { } pub(crate) fn push_exception(&self, exc: Option<PyBaseExceptionRef>) { - let mut excs = self.exceptions.borrow_mut(); - let prev = std::mem::take(&mut *excs); - excs.prev = Some(Box::new(prev)); - excs.exc = exc + self.exceptions.borrow_mut().stack.push(exc); + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); } pub(crate) fn pop_exception(&self) -> Option<PyBaseExceptionRef> { - let mut excs = self.exceptions.borrow_mut(); - let cur = std::mem::take(&mut *excs); - *excs = *cur.prev.expect("pop_exception() without nested exc stack"); - cur.exc - } - - pub(crate) fn take_exception(&self) -> Option<PyBaseExceptionRef> { - self.exceptions.borrow_mut().exc.take() + let exc = self + .exceptions + .borrow_mut() + .stack + .pop() + .expect("pop_exception() without nested exc stack"); + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); + exc } pub(crate) fn current_exception(&self) -> Option<PyBaseExceptionRef> { - self.exceptions.borrow().exc.clone() + self.exceptions.borrow().stack.last().cloned().flatten() } pub(crate) fn set_exception(&self, exc: Option<PyBaseExceptionRef>) { // don't be holding the RefCell guard while __del__ is called - let prev = std::mem::replace(&mut self.exceptions.borrow_mut().exc, exc); - drop(prev); + let mut excs = self.exceptions.borrow_mut(); + debug_assert!( + !excs.stack.is_empty(), + "set_exception called with empty exception stack" + ); + if let Some(top) = excs.stack.last_mut() { + let prev = core::mem::replace(top, exc); + drop(excs); + drop(prev); + } else { + excs.stack.push(exc); + drop(excs); + } + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); } - pub(crate) fn contextualize_exception(&self, exception: &PyBaseExceptionRef) { + pub(crate) fn contextualize_exception(&self, exception: &Py<PyBaseException>) { if let Some(context_exc) = self.topmost_exception() && !context_exc.is(exception) { + // Traverse the context chain to find `exception` and break cycles + // Uses Floyd's cycle detection: o moves every step, slow_o every other step let mut o = context_exc.clone(); + let mut slow_o = context_exc.clone(); + let mut slow_update_toggle = false; while let Some(context) = o.__context__() { if context.is(exception) { o.set___context__(None); break; } o = context; + if o.is(&slow_o) { + // Pre-existing cycle detected - all exceptions on the path were visited + break; + } + if slow_update_toggle && let Some(slow_context) = slow_o.__context__() { + slow_o = slow_context; + } + slow_update_toggle = !slow_update_toggle; } exception.set___context__(Some(context_exc)) } @@ -832,13 +1502,7 @@ impl VirtualMachine { pub(crate) fn topmost_exception(&self) -> Option<PyBaseExceptionRef> { let excs = self.exceptions.borrow(); - let mut cur = &*excs; - loop { - if let Some(exc) = &cur.exc { - return Some(exc.clone()); - } - cur = cur.prev.as_deref()?; - } + excs.stack.iter().rev().find_map(|e| e.clone()) } pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u32 { @@ -849,7 +1513,13 @@ impl VirtualMachine { [arg] => match_class!(match arg { ref i @ PyInt => { use num_traits::cast::ToPrimitive; - return i.as_bigint().to_u32().unwrap_or(0); + // Try u32 first, then i32 (for negative values), else -1 for overflow + let code = i + .as_bigint() + .to_u32() + .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) + .unwrap_or(-1i32 as u32); + return code; } arg => { if self.is_none(arg) { @@ -862,8 +1532,11 @@ impl VirtualMachine { _ => args.as_object().repr(self).ok(), }; if let Some(msg) = msg { - let stderr = stdlib::sys::PyStderr(self); - writeln!(stderr, "{msg}"); + // Write using Python's write() to use stderr's error handler (backslashreplace) + if let Ok(stderr) = stdlib::sys::get_stderr(self) { + let _ = self.call_method(&stderr, "write", (msg,)); + let _ = self.call_method(&stderr, "write", ("\n",)); + } } 1 } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { @@ -982,66 +1655,68 @@ impl AsRef<Context> for VirtualMachine { } } -fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> { - let iter = std::iter::empty(); - macro_rules! ext_modules { - ($iter:ident, $($t:tt)*) => { - let $iter = $iter.chain(py_freeze!($($t)*)); - }; +/// Resolve frozen module alias to its original name. +/// Returns the original module name if an alias exists, otherwise returns the input name. +pub fn resolve_frozen_alias(name: &str) -> &str { + match name { + "_frozen_importlib" => "importlib._bootstrap", + "_frozen_importlib_external" => "importlib._bootstrap_external", + "encodings_ascii" => "encodings.ascii", + "encodings_utf_8" => "encodings.utf_8", + "__hello_alias__" | "__phello_alias__" | "__phello_alias__.spam" => "__hello__", + "__phello__.__init__" => "<__phello__", + "__phello__.ham.__init__" => "<__phello__.ham", + "__hello_only__" => "", + _ => name, } - - // keep as example but use file one now - // ext_modules!( - // iter, - // source = "initialized = True; print(\"Hello world!\")\n", - // module_name = "__hello__", - // ); - - // Python modules that the vm calls into, but are not actually part of the stdlib. They could - // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. - // Includes _importlib_bootstrap and _importlib_bootstrap_external - ext_modules!( - iter, - dir = "./Lib/python_builtins", - crate_name = "rustpython_compiler_core" - ); - - // core stdlib Python modules that the vm calls into, but are still used in Python - // application code, e.g. copyreg - // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. - // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterConfig::new().init_stdlib().interpreter()` - // #[cfg(not(feature = "freeze-stdlib"))] - ext_modules!( - iter, - dir = "./Lib/core_modules", - crate_name = "rustpython_compiler_core" - ); - - iter } #[test] fn test_nested_frozen() { use rustpython_vm as vm; - vm::Interpreter::with_init(Default::default(), |vm| { - // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - vm.add_frozen(rustpython_vm::py_freeze!( - dir = "../../extra_tests/snippets" - )); - }) - .enter(|vm| { - let scope = vm.new_scope_with_builtins(); - - let source = "from dir_module.dir_module_inner import value2"; - let code_obj = vm - .compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned()) - .map_err(|err| vm.new_syntax_error(&err, Some(source))) - .unwrap(); - - if let Err(e) = vm.run_code_obj(code_obj, scope) { - vm.print_exception(e); - panic!(); - } - }) + vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_vm::py_freeze!( + dir = "../../../../extra_tests/snippets" + )) + .build() + .enter(|vm| { + let scope = vm.new_scope_with_builtins(); + + let source = "from dir_module.dir_module_inner import value2"; + let code_obj = vm + .compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source))) + .unwrap(); + + if let Err(e) = vm.run_code_obj(code_obj, scope) { + vm.print_exception(e); + panic!(); + } + }) +} + +#[test] +fn frozen_origname_matches() { + use rustpython_vm as vm; + + vm::Interpreter::builder(Default::default()) + .build() + .enter(|vm| { + let check = |name, expected| { + let module = import::import_frozen(vm, name).unwrap(); + let origname: PyStrRef = module + .get_attr("__origname__", vm) + .unwrap() + .try_into_value(vm) + .unwrap(); + assert_eq!(origname.as_str(), expected); + }; + + check("_frozen_importlib", "importlib._bootstrap"); + check( + "_frozen_importlib_external", + "importlib._bootstrap_external", + ); + }); } diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs new file mode 100644 index 00000000000..70d845b03f5 --- /dev/null +++ b/crates/vm/src/vm/python_run.rs @@ -0,0 +1,176 @@ +//! Python code execution functions. + +use crate::{ + AsObject, PyRef, PyResult, VirtualMachine, + builtins::PyCode, + compiler::{self}, + scope::Scope, +}; + +impl VirtualMachine { + /// PyRun_SimpleString + /// + /// Execute a string of Python code in a new scope with builtins. + pub fn run_simple_string(&self, source: &str) -> PyResult { + let scope = self.new_scope_with_builtins(); + self.run_string(scope, source, "<string>".to_owned()) + } + + /// PyRun_String + /// + /// Execute a string of Python code with explicit scope and source path. + pub fn run_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { + let code_obj = self + .compile(source, compiler::Mode::Exec, source_path) + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; + // linecache._register_code(code, source, filename) + let _ = self.register_code_in_linecache(&code_obj, source); + self.run_code_obj(code_obj, scope) + } + + /// Register a code object's source in linecache._interactive_cache + /// so that traceback can display source lines and caret indicators. + fn register_code_in_linecache(&self, code: &PyRef<PyCode>, source: &str) -> PyResult<()> { + let linecache = self.import("linecache", 0)?; + let register = linecache.get_attr("_register_code", self)?; + let source_str = self.ctx.new_str(source); + let filename = self.ctx.new_str(code.source_path().as_str()); + register.call((code.as_object().to_owned(), source_str, filename), self)?; + Ok(()) + } + + #[deprecated(note = "use run_string instead")] + pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { + self.run_string(scope, source, source_path) + } + + pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult { + let code_obj = self + .compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned()) + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; + self.run_code_obj(code_obj, scope) + } +} + +#[cfg(feature = "host_env")] +mod file_run { + use crate::{ + Py, PyResult, VirtualMachine, + builtins::{PyCode, PyDict}, + compiler::{self}, + scope::Scope, + }; + + impl VirtualMachine { + /// _PyRun_AnyFileObject (internal) + /// + /// Execute a Python file. Currently always delegates to run_simple_file + /// (interactive mode is handled separately in shell.rs). + /// + /// Note: This is an internal function. Use `run_file` for the public interface. + #[doc(hidden)] + pub fn run_any_file(&self, scope: Scope, path: &str) -> PyResult<()> { + let path = if path.is_empty() { "???" } else { path }; + self.run_simple_file(scope, path) + } + + /// _PyRun_SimpleFileObject + /// + /// Execute a Python file with __main__ module setup. + /// Sets __file__ and __cached__ before execution, removes them after. + fn run_simple_file(&self, scope: Scope, path: &str) -> PyResult<()> { + self.with_simple_run(path, |module_dict| { + self.run_simple_file_inner(module_dict, scope, path) + }) + } + + fn run_simple_file_inner( + &self, + module_dict: &Py<PyDict>, + scope: Scope, + path: &str, + ) -> PyResult<()> { + let pyc = maybe_pyc_file(path); + if pyc { + // pyc file execution + set_main_loader(module_dict, path, "SourcelessFileLoader", self)?; + let loader = module_dict.get_item("__loader__", self)?; + let get_code = loader.get_attr("get_code", self)?; + let code_obj = get_code.call((identifier!(self, __main__).to_owned(),), self)?; + let code = code_obj.downcast::<PyCode>().map_err(|_| { + self.new_runtime_error("Bad code object in .pyc file".to_owned()) + })?; + self.run_code_obj(code, scope)?; + } else { + if path != "<stdin>" { + set_main_loader(module_dict, path, "SourceFileLoader", self)?; + } + match std::fs::read_to_string(path) { + Ok(source) => { + let code_obj = self + .compile(&source, compiler::Mode::Exec, path.to_owned()) + .map_err(|err| self.new_syntax_error(&err, Some(&source)))?; + self.run_code_obj(code_obj, scope)?; + } + Err(err) => { + return Err(self.new_os_error(err.to_string())); + } + } + } + Ok(()) + } + + // #[deprecated(note = "use rustpython::run_file instead; if this changes causes problems, please report an issue.")] + pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> { + self.run_any_file(scope, path) + } + } + + fn set_main_loader( + module_dict: &Py<PyDict>, + filename: &str, + loader_name: &str, + vm: &VirtualMachine, + ) -> PyResult<()> { + vm.import("importlib.machinery", 0)?; + let sys_modules = vm.sys_module.get_attr(identifier!(vm, modules), vm)?; + let machinery = sys_modules.get_item("importlib.machinery", vm)?; + let loader_name = vm.ctx.new_str(loader_name); + let loader_class = machinery.get_attr(&loader_name, vm)?; + let loader = loader_class.call((identifier!(vm, __main__).to_owned(), filename), vm)?; + module_dict.set_item("__loader__", loader, vm)?; + Ok(()) + } + + /// Check whether a file is maybe a pyc file. + /// + /// Detection is performed by: + /// 1. Checking if the filename ends with ".pyc" + /// 2. If not, reading the first 2 bytes and comparing with the magic number + fn maybe_pyc_file(path: &str) -> bool { + if path.ends_with(".pyc") { + return true; + } + maybe_pyc_file_with_magic(path).unwrap_or(false) + } + + fn maybe_pyc_file_with_magic(path: &str) -> std::io::Result<bool> { + let path_obj = std::path::Path::new(path); + if !path_obj.is_file() { + return Ok(false); + } + + let mut file = std::fs::File::open(path)?; + let mut buf = [0u8; 2]; + + use std::io::Read; + if file.read(&mut buf)? != 2 { + return Ok(false); + } + + // Read only two bytes of the magic. If the file was opened in + // text mode, the bytes 3 and 4 of the magic (\r\n) might not + // be read as they are on disk. + Ok(crate::import::check_pyc_magic_number_bytes(&buf)) + } +} diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index deaca705c47..1a5ef9efa8f 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -1,8 +1,42 @@ #[cfg(feature = "flame-it")] use std::ffi::OsString; -/// Struct containing all kind of settings for the python vm. -/// Mostly `PyConfig` in CPython. +/// Path configuration computed at runtime (like PyConfig path outputs) +#[derive(Debug, Clone, Default)] +pub struct Paths { + /// sys.executable + pub executable: String, + /// sys._base_executable (original interpreter in venv) + pub base_executable: String, + /// sys.prefix + pub prefix: String, + /// sys.base_prefix + pub base_prefix: String, + /// sys.exec_prefix + pub exec_prefix: String, + /// sys.base_exec_prefix + pub base_exec_prefix: String, + /// sys._stdlib_dir + pub stdlib_dir: Option<String>, + /// Computed module_search_paths (complete sys.path) + pub module_search_paths: Vec<String>, +} + +/// Combined configuration: user settings + computed paths +/// CPython directly exposes every fields under both of them. +/// We separate them to maintain better ownership discipline. +pub struct PyConfig { + pub settings: Settings, + pub paths: Paths, +} + +impl PyConfig { + pub fn new(settings: Settings, paths: Paths) -> Self { + Self { settings, paths } + } +} + +/// User-configurable settings for the python vm. #[non_exhaustive] pub struct Settings { /// -I @@ -25,7 +59,8 @@ pub struct Settings { // int tracemalloc; // int perf_profiling; // int import_time; - // int code_debug_ranges; + /// -X no_debug_ranges: disable column info in bytecode + pub code_debug_ranges: bool, // int show_ref_count; // int dump_refs; // wchar_t *dump_refs_file; @@ -55,6 +90,12 @@ pub struct Settings { /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING pub warn_default_encoding: bool, + /// -X thread_inherit_context, whether new threads inherit context from parent + pub thread_inherit_context: bool, + + /// -X context_aware_warnings, whether warnings are context aware + pub context_aware_warnings: bool, + /// -i pub inspect: bool, @@ -79,9 +120,11 @@ pub struct Settings { /// -u, PYTHONUNBUFFERED=x pub buffered_stdio: bool, - // wchar_t *stdio_encoding; + /// PYTHONIOENCODING - stdio encoding + pub stdio_encoding: Option<String>, + /// PYTHONIOENCODING - stdio error handler + pub stdio_errors: Option<String>, pub utf8_mode: u8, - // wchar_t *stdio_errors; /// --check-hash-based-pycs pub check_hash_pycs_mode: CheckHashPycsMode, @@ -155,14 +198,19 @@ impl Default for Settings { isolated: false, dev_mode: false, warn_default_encoding: false, + thread_inherit_context: false, + context_aware_warnings: false, warnoptions: vec![], path_list: vec![], argv: vec![], hash_seed: None, faulthandler: false, + code_debug_ranges: true, buffered_stdio: true, check_hash_pycs_mode: CheckHashPycsMode::Default, allow_external_library: cfg!(feature = "importlib"), + stdio_encoding: None, + stdio_errors: None, utf8_mode: 1, int_max_str_digits: 4300, #[cfg(feature = "flame-it")] diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 2e687d99820..575910f7900 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -1,17 +1,48 @@ -use crate::{AsObject, PyObject, PyObjectRef, VirtualMachine}; -use itertools::Itertools; -use std::{ +#[cfg(feature = "threading")] +use super::FramePtr; +#[cfg(feature = "threading")] +use crate::builtins::PyBaseExceptionRef; +use crate::frame::Frame; +use crate::{AsObject, PyObject, VirtualMachine}; +#[cfg(feature = "threading")] +use alloc::sync::Arc; +use core::{ cell::{Cell, RefCell}, ptr::NonNull, - thread_local, + sync::atomic::{AtomicPtr, Ordering}, }; +use itertools::Itertools; +use std::thread_local; + +/// Per-thread shared state for sys._current_frames() and sys._current_exceptions(). +/// The exception field uses atomic operations for lock-free cross-thread reads. +#[cfg(feature = "threading")] +pub struct ThreadSlot { + /// Raw frame pointers, valid while the owning thread's call stack is active. + /// Readers must hold the Mutex and convert to FrameRef inside the lock. + pub frames: parking_lot::Mutex<Vec<FramePtr>>, + pub exception: crate::PyAtomicRef<Option<crate::exceptions::types::PyBaseException>>, +} + +#[cfg(feature = "threading")] +pub type CurrentFrameSlot = Arc<ThreadSlot>; thread_local! { pub(super) static VM_STACK: RefCell<Vec<NonNull<VirtualMachine>>> = Vec::with_capacity(1).into(); pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell<u32> = const { Cell::new(0) }; - pub(crate) static ASYNC_GEN_FINALIZER: RefCell<Option<PyObjectRef>> = const { RefCell::new(None) }; - pub(crate) static ASYNC_GEN_FIRSTITER: RefCell<Option<PyObjectRef>> = const { RefCell::new(None) }; + + /// Current thread's slot for sys._current_frames() and sys._current_exceptions() + #[cfg(feature = "threading")] + static CURRENT_THREAD_SLOT: RefCell<Option<CurrentFrameSlot>> = const { RefCell::new(None) }; + + /// Current top frame for signal-safe traceback walking. + /// Mirrors `PyThreadState.current_frame`. Read by faulthandler's signal + /// handler to dump tracebacks without accessing RefCell or locks. + /// Uses AtomicPtr for async-signal-safety (signal handlers may read this + /// while the owning thread is writing). + pub(crate) static CURRENT_FRAME: AtomicPtr<Frame> = + const { AtomicPtr::new(core::ptr::null_mut()) }; } scoped_tls::scoped_thread_local!(static VM_CURRENT: VirtualMachine); @@ -26,11 +57,146 @@ pub fn with_current_vm<R>(f: impl FnOnce(&VirtualMachine) -> R) -> R { pub fn enter_vm<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { VM_STACK.with(|vms| { vms.borrow_mut().push(vm.into()); + + // Initialize thread slot for this thread if not already done + #[cfg(feature = "threading")] + init_thread_slot_if_needed(vm); + scopeguard::defer! { vms.borrow_mut().pop(); } VM_CURRENT.set(vm, f) }) } +/// Initialize thread slot for current thread if not already initialized. +/// Called automatically by enter_vm(). +#[cfg(feature = "threading")] +fn init_thread_slot_if_needed(vm: &VirtualMachine) { + CURRENT_THREAD_SLOT.with(|slot| { + if slot.borrow().is_none() { + let thread_id = crate::stdlib::thread::get_ident(); + let new_slot = Arc::new(ThreadSlot { + frames: parking_lot::Mutex::new(Vec::new()), + exception: crate::PyAtomicRef::from(None::<PyBaseExceptionRef>), + }); + vm.state + .thread_frames + .lock() + .insert(thread_id, new_slot.clone()); + *slot.borrow_mut() = Some(new_slot); + } + }); +} + +/// Push a frame pointer onto the current thread's shared frame stack. +/// The pointed-to frame must remain alive until the matching pop. +#[cfg(feature = "threading")] +pub fn push_thread_frame(fp: FramePtr) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.frames.lock().push(fp); + } else { + debug_assert!( + false, + "push_thread_frame called without initialized thread slot" + ); + } + }); +} + +/// Pop a frame from the current thread's shared frame stack. +/// Called when a frame is exited. +#[cfg(feature = "threading")] +pub fn pop_thread_frame() { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.frames.lock().pop(); + } else { + debug_assert!( + false, + "pop_thread_frame called without initialized thread slot" + ); + } + }); +} + +/// Set the current thread's top frame pointer for signal-safe traceback walking. +/// Returns the previous frame pointer so it can be restored on pop. +pub fn set_current_frame(frame: *const Frame) -> *const Frame { + CURRENT_FRAME.with(|c| c.swap(frame as *mut Frame, Ordering::Relaxed) as *const Frame) +} + +/// Get the current thread's top frame pointer. +/// Used by faulthandler's signal handler to start traceback walking. +pub fn get_current_frame() -> *const Frame { + CURRENT_FRAME.with(|c| c.load(Ordering::Relaxed) as *const Frame) +} + +/// Update the current thread's exception slot atomically (no locks). +/// Called from push_exception/pop_exception/set_exception. +#[cfg(feature = "threading")] +pub fn update_thread_exception(exc: Option<PyBaseExceptionRef>) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + // SAFETY: Called only from the owning thread. The old ref is dropped + // here on the owning thread, which is safe. + let _old = unsafe { s.exception.swap(exc) }; + } + }); +} + +/// Collect all threads' current exceptions for sys._current_exceptions(). +/// Acquires the global registry lock briefly, then reads each slot's exception atomically. +#[cfg(feature = "threading")] +pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option<PyBaseExceptionRef>)> { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .map(|(id, slot)| (*id, slot.exception.to_owned())) + .collect() +} + +/// Cleanup thread slot for the current thread. Called at thread exit. +#[cfg(feature = "threading")] +pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { + let thread_id = crate::stdlib::thread::get_ident(); + vm.state.thread_frames.lock().remove(&thread_id); + CURRENT_THREAD_SLOT.with(|s| { + *s.borrow_mut() = None; + }); +} + +/// Reinitialize thread slot after fork. Called in child process. +/// Creates a fresh slot and registers it for the current thread, +/// preserving the current thread's frames from `vm.frames`. +#[cfg(feature = "threading")] +pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { + let current_ident = crate::stdlib::thread::get_ident(); + let current_frames: Vec<FramePtr> = vm.frames.borrow().clone(); + let new_slot = Arc::new(ThreadSlot { + frames: parking_lot::Mutex::new(current_frames), + exception: crate::PyAtomicRef::from(vm.topmost_exception()), + }); + + // After fork, only the current thread exists. If the lock was held by + // another thread during fork, force unlock it. + let mut registry = match vm.state.thread_frames.try_lock() { + Some(guard) => guard, + None => { + // SAFETY: After fork in child process, only the current thread + // exists. The lock holder no longer exists. + unsafe { vm.state.thread_frames.force_unlock() }; + vm.state.thread_frames.lock() + } + }; + registry.clear(); + registry.insert(current_ident, new_slot.clone()); + drop(registry); + + CURRENT_THREAD_SLOT.with(|s| { + *s.borrow_mut() = Some(new_slot); + }); +} + pub fn with_vm<F, R>(obj: &PyObject, f: F) -> Option<R> where F: Fn(&VirtualMachine) -> R, @@ -139,6 +305,10 @@ impl VirtualMachine { /// specific guaranteed behavior. #[cfg(feature = "threading")] pub fn new_thread(&self) -> ThreadedVirtualMachine { + let global_trace = self.state.global_trace_func.lock().clone(); + let global_profile = self.state.global_profile_func.lock().clone(); + let use_tracing = global_trace.is_some() || global_profile.is_some(); + let vm = Self { builtins: self.builtins.clone(), sys_module: self.sys_module.clone(), @@ -147,16 +317,22 @@ impl VirtualMachine { wasm_id: self.wasm_id.clone(), exceptions: RefCell::default(), import_func: self.import_func.clone(), - profile_func: RefCell::new(self.ctx.none()), - trace_func: RefCell::new(self.ctx.none()), - use_tracing: Cell::new(false), + importlib: self.importlib.clone(), + profile_func: RefCell::new(global_profile.unwrap_or_else(|| self.ctx.none())), + trace_func: RefCell::new(global_trace.unwrap_or_else(|| self.ctx.none())), + use_tracing: Cell::new(use_tracing), recursion_limit: self.recursion_limit.clone(), - signal_handlers: None, + signal_handlers: core::cell::OnceCell::new(), signal_rx: None, repr_guards: RefCell::default(), state: self.state.clone(), initialized: self.initialized, recursion_depth: Cell::new(0), + c_stack_soft_limit: Cell::new(VirtualMachine::calculate_c_stack_soft_limit()), + async_gen_firstiter: RefCell::new(None), + async_gen_finalizer: RefCell::new(None), + asyncio_running_loop: RefCell::new(None), + asyncio_running_task: RefCell::new(None), }; ThreadedVirtualMachine { vm } } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 36481e5dbe3..9e33f430945 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -1,5 +1,5 @@ use crate::{ - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, + AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, builtins::{ PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, PyType, PyTypeRef, @@ -8,9 +8,9 @@ use crate::{ tuple::{IntoPyTuple, PyTupleRef}, }, convert::{ToPyException, ToPyObject}, + exceptions::OSErrorBuilder, function::{IntoPyNativeFn, PyMethodFlags}, scope::Scope, - types::Constructor, vm::VirtualMachine, }; use rustpython_compiler_core::SourceLocation; @@ -62,6 +62,19 @@ impl VirtualMachine { Scope::with_builtins(None, self.ctx.new_dict(), self) } + pub fn new_scope_with_main(&self) -> PyResult<Scope> { + let scope = self.new_scope_with_builtins(); + let main_module = self.new_module("__main__", scope.globals.clone(), None); + + self.sys_module.get_attr("modules", self)?.set_item( + "__main__", + main_module.into(), + self, + )?; + + Ok(scope) + } + pub fn new_function<F, FKind>(&self, name: &'static str, f: F) -> PyRef<PyNativeFunction> where F: IntoPyNativeFn<FKind>, @@ -95,7 +108,7 @@ impl VirtualMachine { pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec<PyObjectRef>) -> PyBaseExceptionRef { debug_assert_eq!( exc_type.slots.basicsize, - std::mem::size_of::<PyBaseException>(), + core::mem::size_of::<PyBaseException>(), "vm.new_exception() is only for exception types without additional payload. The given type '{}' is not allowed.", exc_type.class().name() ); @@ -118,27 +131,9 @@ impl VirtualMachine { errno: Option<i32>, msg: impl ToPyObject, ) -> PyRef<PyOSError> { - debug_assert_eq!(exc_type.slots.basicsize, std::mem::size_of::<PyOSError>()); - let msg = msg.to_pyobject(self); - - fn new_os_subtype_error_impl( - vm: &VirtualMachine, - exc_type: PyTypeRef, - errno: Option<i32>, - msg: PyObjectRef, - ) -> PyRef<PyOSError> { - let args = match errno { - Some(e) => vec![vm.new_pyobj(e), msg], - None => vec![msg], - }; - let payload = - PyOSError::py_new(&exc_type, args.into(), vm).expect("new_os_error usage error"); - payload - .into_ref_with_type(vm, exc_type) - .expect("new_os_error usage error") - } + debug_assert_eq!(exc_type.slots.basicsize, core::mem::size_of::<PyOSError>()); - new_os_subtype_error_impl(self, exc_type, errno, msg) + OSErrorBuilder::with_subtype(exc_type, errno, msg, self).build(self) } /// Instantiate an exception with no arguments. @@ -216,7 +211,7 @@ impl VirtualMachine { op: &str, ) -> PyBaseExceptionRef { self.new_type_error(format!( - "'{}' not supported between instances of '{}' and '{}'", + "unsupported operand type(s) for {}: '{}' and '{}'", op, a.class().name(), b.class().name() @@ -337,16 +332,54 @@ impl VirtualMachine { source: Option<&str>, allow_incomplete: bool, ) -> PyBaseExceptionRef { + let incomplete_or_syntax = |allow| -> &'static Py<crate::builtins::PyType> { + if allow { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.syntax_error + } + }; + let syntax_error_type = match &error { #[cfg(feature = "parser")] - // FIXME: this condition will cause TabError even when the matching actual error is IndentationError crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::IndentationError, ), .. - }) => self.ctx.exceptions.tab_error, + }) => { + // Detect tab/space mixing to raise TabError instead of IndentationError. + // This checks both within a single line and across different lines. + let is_tab_error = source.is_some_and(|source| { + let mut has_space_indent = false; + let mut has_tab_indent = false; + for line in source.lines() { + let indent: Vec<u8> = line + .bytes() + .take_while(|&b| b == b' ' || b == b'\t') + .collect(); + if indent.is_empty() { + continue; + } + if indent.contains(&b' ') && indent.contains(&b'\t') { + return true; + } + if indent.contains(&b' ') { + has_space_indent = true; + } + if indent.contains(&b'\t') { + has_tab_indent = true; + } + } + has_space_indent && has_tab_indent + }); + if is_tab_error { + self.ctx.exceptions.tab_error + } else { + self.ctx.exceptions.indentation_error + } + } #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::UnexpectedIndentation, @@ -359,13 +392,13 @@ impl VirtualMachine { ruff_python_parser::LexicalErrorType::Eof, ), .. - }) => { - if allow_incomplete { - self.ctx.exceptions.incomplete_input_error - } else { - self.ctx.exceptions.syntax_error - } - } + }) => incomplete_or_syntax(allow_incomplete), + // Unclosed bracket errors (converted from Eof by from_ruff_parse_error) + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + is_unclosed_bracket: true, + .. + }) => incomplete_or_syntax(allow_incomplete), #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: @@ -375,13 +408,7 @@ impl VirtualMachine { ), ), .. - }) => { - if allow_incomplete { - self.ctx.exceptions.incomplete_input_error - } else { - self.ctx.exceptions.syntax_error - } - } + }) => incomplete_or_syntax(allow_incomplete), #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: @@ -405,11 +432,7 @@ impl VirtualMachine { } } - if is_incomplete { - self.ctx.exceptions.incomplete_input_error - } else { - self.ctx.exceptions.syntax_error - } + incomplete_or_syntax(is_incomplete) } else { self.ctx.exceptions.syntax_error } @@ -445,7 +468,7 @@ impl VirtualMachine { if is_incomplete { self.ctx.exceptions.incomplete_input_error } else { - self.ctx.exceptions.indentation_error + self.ctx.exceptions.indentation_error // not syntax_error } } else { self.ctx.exceptions.indentation_error @@ -463,6 +486,7 @@ impl VirtualMachine { let line = source .split('\n') .nth(loc?.line.to_zero_indexed())? + .trim_end_matches('\r') .to_owned(); Some(line + "\n") } @@ -477,16 +501,124 @@ impl VirtualMachine { if let Some(msg) = msg.get_mut(..1) { msg.make_ascii_lowercase(); } + let mut narrow_caret = false; match error { + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedString, + ) + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedString, + ), + ), + .. + }) => { + msg = "unterminated f-string literal".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ) + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ), + ), + .. + }) => { + msg = "unterminated triple-quoted f-string literal".to_owned(); + } #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::FStringError(_) - | ruff_python_parser::ParseErrorType::UnexpectedExpressionToken, + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError(_), + ), + .. + }) => { + // Replace backticks with single quotes to match CPython's error messages + msg = msg.replace('`', "'"); + msg.insert_str(0, "invalid syntax: "); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::UnexpectedExpressionToken, .. }) => msg.insert_str(0, "invalid syntax: "), + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnrecognizedToken { .. }, + ) + | ruff_python_parser::ParseErrorType::SimpleStatementsOnSameLine + | ruff_python_parser::ParseErrorType::SimpleAndCompoundStatementOnSameLine + | ruff_python_parser::ParseErrorType::ExpectedToken { .. } + | ruff_python_parser::ParseErrorType::ExpectedExpression, + .. + }) => { + msg = "invalid syntax".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::InvalidStarredExpressionUsage, + .. + }) => { + msg = "invalid syntax".to_owned(); + narrow_caret = true; + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::InvalidDeleteTarget, + .. + }) => { + msg = "invalid syntax".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::LineContinuationError, + ), + .. + }) => { + msg = "unexpected character after line continuation".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnclosedStringError, + ), + .. + }) => { + msg = "unterminated string".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + .. + }) if s.eq_ignore_ascii_case("bytes literal cannot be mixed with non-bytes literals") => { + msg = "cannot mix bytes and nonbytes literals".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + .. + }) if s.starts_with("Expected an identifier, but found a keyword") => { + msg = "invalid syntax".to_owned(); + } _ => {} } + if syntax_error_type.is(self.ctx.exceptions.tab_error) { + msg = "inconsistent use of tabs and spaces in indentation".to_owned(); + } let syntax_error = self.new_exception_msg(syntax_error_type, msg); let (lineno, offset) = error.python_location(); let lineno = self.ctx.new_int(lineno); @@ -500,6 +632,26 @@ impl VirtualMachine { .set_attr("offset", offset, self) .unwrap(); + // Set end_lineno and end_offset if available + if let Some((end_lineno, end_offset)) = error.python_end_location() { + let (end_lineno, end_offset) = if narrow_caret { + let (l, o) = error.python_location(); + (l, o + 1) + } else { + (end_lineno, end_offset) + }; + let end_lineno = self.ctx.new_int(end_lineno); + let end_offset = self.ctx.new_int(end_offset); + syntax_error + .as_object() + .set_attr("end_lineno", end_lineno, self) + .unwrap(); + syntax_error + .as_object() + .set_attr("end_offset", end_offset, self) + .unwrap(); + } + syntax_error .as_object() .set_attr("text", statement.to_pyobject(self), self) @@ -508,6 +660,23 @@ impl VirtualMachine { .as_object() .set_attr("filename", self.ctx.new_str(error.source_path()), self) .unwrap(); + + // Set _metadata for keyword typo suggestions in traceback module. + // Format: (start_line, col_offset, source_code) + // start_line=0 means "include all lines from beginning" which provides + // full context needed by _find_keyword_typos to compile-check suggestions. + if let Some(source) = source { + let metadata = self.ctx.new_tuple(vec![ + self.ctx.new_int(0).into(), + self.ctx.new_int(0).into(), + self.ctx.new_str(source).into(), + ]); + syntax_error + .as_object() + .set_attr("_metadata", metadata, self) + .unwrap(); + } + syntax_error } @@ -618,5 +787,6 @@ impl VirtualMachine { define_exception_fn!(fn new_zero_division_error, zero_division_error, ZeroDivisionError); define_exception_fn!(fn new_overflow_error, overflow_error, OverflowError); define_exception_fn!(fn new_runtime_error, runtime_error, RuntimeError); + define_exception_fn!(fn new_python_finalization_error, python_finalization_error, PythonFinalizationError); define_exception_fn!(fn new_memory_error, memory_error, MemoryError); } diff --git a/crates/vm/src/vm/vm_object.rs b/crates/vm/src/vm/vm_object.rs index e69301820d6..0d5b286148c 100644 --- a/crates/vm/src/vm/vm_object.rs +++ b/crates/vm/src/vm/vm_object.rs @@ -96,9 +96,7 @@ impl VirtualMachine { obj: Option<PyObjectRef>, cls: Option<PyObjectRef>, ) -> Option<PyResult> { - let descr_get = descr - .class() - .mro_find_map(|cls| cls.slots.descr_get.load())?; + let descr_get = descr.class().slots.descr_get.load()?; Some(descr_get(descr.to_owned(), obj, cls, self)) } diff --git a/crates/vm/src/vm/vm_ops.rs b/crates/vm/src/vm/vm_ops.rs index e30e19981a9..1a362d67bed 100644 --- a/crates/vm/src/vm/vm_ops.rs +++ b/crates/vm/src/vm/vm_ops.rs @@ -4,7 +4,7 @@ use crate::{ PyRef, builtins::{PyInt, PyStr, PyStrRef, PyUtf8Str}, object::{AsObject, PyObject, PyObjectRef, PyResult}, - protocol::{PyNumberBinaryOp, PyNumberTernaryOp, PySequence}, + protocol::{PyNumberBinaryOp, PyNumberTernaryOp}, types::PyComparisonOp, }; use num_traits::ToPrimitive; @@ -160,12 +160,12 @@ impl VirtualMachine { let class_a = a.class(); let class_b = b.class(); - // Look up number slots across MRO for inheritance - let slot_a = class_a.mro_find_map(|x| x.slots.as_number.left_binary_op(op_slot)); + // Number slots are inherited, direct access is O(1) + let slot_a = class_a.slots.as_number.left_binary_op(op_slot); let mut slot_b = None; if !class_a.is(class_b) { - let slot_bb = class_b.mro_find_map(|x| x.slots.as_number.right_binary_op(op_slot)); + let slot_bb = class_b.slots.as_number.right_binary_op(op_slot); if slot_bb.map(|x| x as usize) != slot_a.map(|x| x as usize) { slot_b = slot_bb; } @@ -231,10 +231,7 @@ impl VirtualMachine { iop_slot: PyNumberBinaryOp, op_slot: PyNumberBinaryOp, ) -> PyResult { - if let Some(slot) = a - .class() - .mro_find_map(|x| x.slots.as_number.left_binary_op(iop_slot)) - { + if let Some(slot) = a.class().slots.as_number.left_binary_op(iop_slot) { let x = slot(a, b, self)?; if !x.is(&self.ctx.not_implemented) { return Ok(x); @@ -270,12 +267,12 @@ impl VirtualMachine { let class_b = b.class(); let class_c = c.class(); - // Look up number slots across MRO for inheritance - let slot_a = class_a.mro_find_map(|x| x.slots.as_number.left_ternary_op(op_slot)); + // Number slots are inherited, direct access is O(1) + let slot_a = class_a.slots.as_number.left_ternary_op(op_slot); let mut slot_b = None; if !class_a.is(class_b) { - let slot_bb = class_b.mro_find_map(|x| x.slots.as_number.right_ternary_op(op_slot)); + let slot_bb = class_b.slots.as_number.right_ternary_op(op_slot); if slot_bb.map(|x| x as usize) != slot_a.map(|x| x as usize) { slot_b = slot_bb; } @@ -304,9 +301,9 @@ impl VirtualMachine { } } - if let Some(slot_c) = class_c.mro_find_map(|x| x.slots.as_number.left_ternary_op(op_slot)) - && slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) - && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) + if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) + && slot_a.is_some_and(|slot_a| !core::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !core::ptr::fn_addr_eq(slot_b, slot_c)) { let ret = slot_c(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { @@ -343,10 +340,7 @@ impl VirtualMachine { op_slot: PyNumberTernaryOp, op_str: &str, ) -> PyResult { - if let Some(slot) = a - .class() - .mro_find_map(|x| x.slots.as_number.left_ternary_op(iop_slot)) - { + if let Some(slot) = a.class().slots.as_number.left_ternary_op(iop_slot) { let x = slot(a, b, c, self)?; if !x.is(&self.ctx.not_implemented) { return Ok(x); @@ -386,8 +380,10 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let result = seq_a.concat(b, self)?; + // Check if concat slot is available directly, matching PyNumber_Add behavior + let seq = a.sequence_unchecked(); + if let Some(f) = seq.slots().concat.load() { + let result = f(seq, b, self)?; if !result.is(&self.ctx.not_implemented) { return Ok(result); } @@ -400,8 +396,11 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let result = seq_a.inplace_concat(b, self)?; + // Check inplace_concat or concat slot directly, matching PyNumber_InPlaceAdd behavior + let seq = a.sequence_unchecked(); + let slots = seq.slots(); + if let Some(f) = slots.inplace_concat.load().or_else(|| slots.concat.load()) { + let result = f(seq, b, self)?; if !result.is(&self.ctx.not_implemented) { return Ok(result); } @@ -414,14 +413,14 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { + if let Ok(seq_a) = a.try_sequence(self) { let n = b .try_index(self)? .as_bigint() .to_isize() .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; return seq_a.repeat(n, self); - } else if let Ok(seq_b) = PySequence::try_protocol(b, self) { + } else if let Ok(seq_b) = b.try_sequence(self) { let n = a .try_index(self)? .as_bigint() @@ -442,14 +441,14 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { + if let Ok(seq_a) = a.try_sequence(self) { let n = b .try_index(self)? .as_bigint() .to_isize() .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; return seq_a.inplace_repeat(n, self); - } else if let Ok(seq_b) = PySequence::try_protocol(b, self) { + } else if let Ok(seq_b) = b.try_sequence(self) { let n = a .try_index(self)? .as_bigint() @@ -530,7 +529,7 @@ impl VirtualMachine { } pub fn _contains(&self, haystack: &PyObject, needle: &PyObject) -> PyResult<bool> { - let seq = haystack.to_sequence(); + let seq = haystack.sequence_unchecked(); seq.contains(needle, self) } } diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 6480f778433..684630e6af0 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -1,27 +1,67 @@ use crate::{ - AsObject, Context, Py, PyObjectRef, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyResult, VirtualMachine, builtins::{ - PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, + PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, + PyTupleRef, PyTypeRef, }, - convert::{IntoObject, TryFromObject}, - types::PyComparisonOp, + convert::TryFromObject, }; +use core::sync::atomic::{AtomicUsize, Ordering}; +use rustpython_common::lock::OnceCell; pub struct WarningsState { - filters: PyListRef, - _once_registry: PyDictRef, - default_action: PyStrRef, - filters_version: usize, + pub filters: PyListRef, + pub once_registry: PyDictRef, + pub default_action: PyStrRef, + pub filters_version: AtomicUsize, + pub context_var: OnceCell<PyObjectRef>, + lock_count: AtomicUsize, } impl WarningsState { - fn create_filter(ctx: &Context) -> PyListRef { + fn create_default_filters(ctx: &Context) -> PyListRef { + // init_filters(): non-debug default filter set. ctx.new_list(vec![ ctx.new_tuple(vec![ + ctx.new_str("default").into(), + ctx.none(), + ctx.exceptions.deprecation_warning.as_object().to_owned(), ctx.new_str("__main__").into(), - ctx.types.none_type.as_object().to_owned(), - ctx.exceptions.warning.as_object().to_owned(), - ctx.new_str("ACTION").into(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.deprecation_warning.as_object().to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions + .pending_deprecation_warning + .as_object() + .to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.import_warning.as_object().to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.resource_warning.as_object().to_owned(), + ctx.none(), ctx.new_int(0).into(), ]) .into(), @@ -30,25 +70,44 @@ impl WarningsState { pub fn init_state(ctx: &Context) -> Self { Self { - filters: Self::create_filter(ctx), - _once_registry: ctx.new_dict(), + filters: Self::create_default_filters(ctx), + once_registry: ctx.new_dict(), default_action: ctx.new_str("default"), - filters_version: 0, + filters_version: AtomicUsize::new(0), + context_var: OnceCell::new(), + lock_count: AtomicUsize::new(0), } } -} -fn check_matched(obj: &PyObjectRef, arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - if obj.class().is(vm.ctx.types.none_type) { - return Ok(true); + pub fn acquire_lock(&self) { + self.lock_count.fetch_add(1, Ordering::SeqCst); } - if obj.rich_compare_bool(arg, PyComparisonOp::Eq, vm)? { - return Ok(false); + pub fn release_lock(&self) -> bool { + let prev = self.lock_count.load(Ordering::SeqCst); + if prev == 0 { + return false; + } + self.lock_count.fetch_sub(1, Ordering::SeqCst); + true } - let result = obj.call((arg.to_owned(),), vm); - Ok(result.is_ok()) + pub fn filters_mutated(&self) { + self.filters_version.fetch_add(1, Ordering::SeqCst); + } +} + +/// None matches everything; plain strings do exact comparison; +/// regex objects use .match(). +fn check_matched(obj: &PyObject, arg: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + if vm.is_none(obj) { + return Ok(true); + } + if obj.class().is(vm.ctx.types.str_type) { + return obj.rich_compare_bool(arg, crate::types::PyComparisonOp::Eq, vm); + } + let result = vm.call_method(obj, "match", (arg.to_owned(),))?; + result.is_true(vm) } fn get_warnings_attr( @@ -60,159 +119,222 @@ fn get_warnings_attr( && !vm .state .finalizing - .load(std::sync::atomic::Ordering::SeqCst) + .load(core::sync::atomic::Ordering::SeqCst) { match vm.import("warnings", 0) { Ok(module) => module, Err(_) => return Ok(None), } } else { - // TODO: finalizing support - return Ok(None); + match vm.sys_module.get_attr(identifier!(vm, modules), vm) { + Ok(modules) => match modules.get_item(vm.ctx.intern_str("warnings"), vm) { + Ok(module) => module, + Err(_) => return Ok(None), + }, + Err(_) => return Ok(None), + } }; - Ok(Some(module.get_attr(attr_name, vm)?)) + match module.get_attr(attr_name, vm) { + Ok(attr) => Ok(Some(attr)), + Err(_) => Ok(None), + } } -pub fn warn( - message: PyStrRef, - category: Option<PyTypeRef>, - stack_level: isize, - source: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - let (filename, lineno, module, registry) = setup_context(stack_level, vm)?; - warn_explicit( - category, message, filename, lineno, module, registry, None, source, vm, - ) +/// Get the warnings filters list from sys.modules['warnings'].filters, +/// falling back to vm.state.warnings.filters. +fn get_warnings_filters(vm: &VirtualMachine) -> PyResult<PyListRef> { + if let Some(filters_obj) = get_warnings_attr(vm, identifier!(&vm.ctx, filters), false)? + && let Ok(filters) = filters_obj.try_into_value::<PyListRef>(vm) + { + return Ok(filters); + } + Ok(vm.state.warnings.filters.clone()) } +/// Get the default action from sys.modules['warnings']._defaultaction, +/// falling back to vm.state.warnings.default_action. fn get_default_action(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if let Some(action) = get_warnings_attr(vm, identifier!(&vm.ctx, defaultaction), false)? { + if !action.class().is(vm.ctx.types.str_type) { + return Err(vm.new_type_error(format!( + "_warnings.defaultaction must be a string, not '{}'", + action.class().name() + ))); + } + return Ok(action); + } Ok(vm.state.warnings.default_action.clone().into()) - // .map_err(|_| { - // vm.new_value_error(format!( - // "_warnings.defaultaction must be a string, not '{}'", - // vm.state.warnings.default_action - // )) - // }) } -fn get_filter( - category: PyObjectRef, - text: PyObjectRef, - lineno: usize, - module: PyObjectRef, - mut _item: PyTupleRef, - vm: &VirtualMachine, -) -> PyResult { - let filters = vm.state.warnings.filters.as_object().to_owned(); - - let filters: PyListRef = filters - .try_into_value(vm) - .map_err(|_| vm.new_value_error("_warnings.filters must be a list"))?; - - /* WarningsState.filters could change while we are iterating over it. */ - for i in 0..filters.borrow_vec().len() { - let tmp_item = if let Some(tmp_item) = filters.borrow_vec().get(i).cloned() { - let tmp_item = PyTupleRef::try_from_object(vm, tmp_item)?; - (tmp_item.len() == 5).then_some(tmp_item) - } else { - None - } - .ok_or_else(|| vm.new_value_error(format!("_warnings.filters item {i} isn't a 5-tuple")))?; - - /* Python code: action, msg, cat, mod, ln = item */ - let action = if let Some(action) = tmp_item.first() { - action.str_utf8(vm).map(|action| action.into_object()) - } else { - Err(vm.new_type_error("action must be a string")) - }; - - let good_msg = if let Some(msg) = tmp_item.get(1) { - check_matched(msg, &text, vm)? - } else { - false - }; - - let is_subclass = if let Some(cat) = tmp_item.get(2) { - category.fast_isinstance(cat.class()) - } else { - false - }; - - let good_mod = if let Some(item_mod) = tmp_item.get(3) { - check_matched(item_mod, &module, vm)? - } else { - false - }; - - let ln = tmp_item.get(4).map_or(0, |ln_obj| { - ln_obj.try_int(vm).map_or(0, |ln| ln.as_u32_mask() as _) - }); - - if good_msg && good_mod && is_subclass && (ln == 0 || lineno == ln) { - _item = tmp_item; - return action; +/// Get the once registry from sys.modules['warnings']._onceregistry, +/// falling back to vm.state.warnings.once_registry. +fn get_once_registry(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if let Some(registry) = get_warnings_attr(vm, identifier!(&vm.ctx, onceregistry), false)? { + if !registry.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "_warnings.onceregistry must be a dict, not '{}'", + registry.class().name() + ))); } + return Ok(registry); } - - get_default_action(vm) + Ok(vm.state.warnings.once_registry.clone().into()) } fn already_warned( - registry: PyObjectRef, + registry: &PyObject, key: PyObjectRef, should_set: bool, vm: &VirtualMachine, ) -> PyResult<bool> { + if vm.is_none(registry) { + return Ok(false); + } + + let current_version = vm.state.warnings.filters_version.load(Ordering::SeqCst); let version_obj = registry.get_item(identifier!(&vm.ctx, version), vm).ok(); - let filters_version = vm.ctx.new_int(vm.state.warnings.filters_version).into(); - match version_obj { - Some(version_obj) - if version_obj.try_int(vm).is_ok() || version_obj.is(&filters_version) => + let version_matches = version_obj.as_ref().is_some_and(|v| { + v.try_int(vm) + .map(|i| i.as_u32_mask() as usize == current_version) + .unwrap_or(false) + }); + + if version_matches { + if let Ok(val) = registry.get_item(key.as_ref(), vm) + && val.is_true(vm)? { - let already_warned = registry.get_item(key.as_ref(), vm)?; - if already_warned.is_true(vm)? { - return Ok(true); - } - } - _ => { - let registry = registry.dict(); - if let Some(registry) = registry.as_ref() { - registry.clear(); - let r = registry.set_item("version", filters_version, vm); - if r.is_err() { - return Ok(false); - } - } + return Ok(true); } + } else if let Ok(dict) = PyDictRef::try_from_object(vm, registry.to_owned()) { + dict.clear(); + dict.set_item( + identifier!(&vm.ctx, version), + vm.ctx.new_int(current_version).into(), + vm, + )?; } - /* This warning wasn't found in the registry, set it. */ - if !should_set { - return Ok(false); + if should_set { + registry.set_item(key.as_ref(), vm.ctx.true_value.clone().into(), vm)?; } + Ok(false) +} - let item = vm.ctx.true_value.clone().into(); - let _ = registry.set_item(key.as_ref(), item, vm); // ignore set error - Ok(true) +/// Create a `(text, category)` or `(text, category, 0)` key and record +/// it in the registry via `already_warned`. +fn update_registry( + registry: &PyObject, + text: &PyObject, + category: &PyObject, + add_zero: bool, + vm: &VirtualMachine, +) -> PyResult<bool> { + let altkey: PyObjectRef = if add_zero { + PyTuple::new_ref( + vec![ + text.to_owned(), + category.to_owned(), + vm.ctx.new_int(0).into(), + ], + &vm.ctx, + ) + .into() + } else { + PyTuple::new_ref(vec![text.to_owned(), category.to_owned()], &vm.ctx).into() + }; + already_warned(registry, altkey, true, vm) } -fn normalize_module(filename: &Py<PyStr>, vm: &VirtualMachine) -> Option<PyObjectRef> { - let obj = match filename.char_len() { +fn normalize_module(filename: &Py<PyStr>, vm: &VirtualMachine) -> PyObjectRef { + match filename.byte_len() { 0 => vm.new_pyobj("<unknown>"), len if len >= 3 && filename.as_bytes().ends_with(b".py") => { vm.new_pyobj(&filename.as_wtf8()[..len - 3]) } _ => filename.as_object().to_owned(), - }; - Some(obj) + } +} + +/// Search the global filters list for a matching action. +// TODO: split into filter_search() + get_filter() and support +// context-aware filters (get_warnings_context_filters). +fn get_filter( + category: PyObjectRef, + text: PyObjectRef, + lineno: usize, + module: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let filters = get_warnings_filters(vm)?; + + // filters could change while we are iterating over it. + // Re-check list length each iteration (matches C behavior). + let mut i = 0; + while i < filters.borrow_vec().len() { + let Some(tmp_item) = filters.borrow_vec().get(i).cloned() else { + break; + }; + let tmp_item = PyTupleRef::try_from_object(vm, tmp_item) + .ok() + .filter(|t| t.len() == 5) + .ok_or_else(|| { + vm.new_value_error(format!("_warnings.filters item {i} isn't a 5-tuple")) + })?; + + /* action, msg, cat, mod, ln = item */ + let action = &tmp_item[0]; + let good_msg = check_matched(&tmp_item[1], &text, vm)?; + let is_subclass = category.is_subclass(&tmp_item[2], vm)?; + let good_mod = check_matched(&tmp_item[3], &module, vm)?; + let ln: usize = tmp_item[4].try_int(vm).map_or(0, |v| v.as_u32_mask() as _); + + if good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln) { + return Ok(action.to_owned()); + } + i += 1; + } + + get_default_action(vm) } +pub fn warn( + message: PyObjectRef, + category: Option<PyTypeRef>, + stack_level: isize, + source: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + warn_with_skip(message, category, stack_level, source, None, vm) +} + +/// do_warn: resolve context via setup_context, then call warn_explicit. +pub fn warn_with_skip( + message: PyObjectRef, + category: Option<PyTypeRef>, + mut stack_level: isize, + source: Option<PyObjectRef>, + skip_file_prefixes: Option<PyTupleRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(ref prefixes) = skip_file_prefixes + && !prefixes.is_empty() + && stack_level < 2 + { + stack_level = 2; + } + let (filename, lineno, module, registry) = + setup_context(stack_level, skip_file_prefixes.as_ref(), vm)?; + warn_explicit( + category, message, filename, lineno, module, registry, None, source, vm, + ) +} + +/// Core warning logic matching `warn_explicit()` in `_warnings.c`. #[allow(clippy::too_many_arguments)] -fn warn_explicit( +pub(crate) fn warn_explicit( category: Option<PyTypeRef>, - message: PyStrRef, + message: PyObjectRef, filename: PyStrRef, lineno: usize, module: Option<PyObjectRef>, @@ -221,88 +343,122 @@ fn warn_explicit( source: Option<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<()> { - let registry: PyObjectRef = registry - .try_into_value(vm) - .map_err(|_| vm.new_type_error("'registry' must be a dict or None"))?; - - // Normalize module. - let module = match module.or_else(|| normalize_module(&filename, vm)) { - Some(module) => module, - None => return Ok(()), - }; + // Normalize module. None → silent return (late-shutdown safety). + let module = module.unwrap_or_else(|| normalize_module(&filename, vm)); + if vm.is_none(&module) { + return Ok(()); + } // Normalize message. - let text = message.as_wtf8(); - - let category = if let Some(category) = category { - if !category.fast_issubclass(vm.ctx.exceptions.warning) { - return Err(vm.new_type_error(format!( - "category must be a Warning subclass, not '{}'", - category.class().name() - ))); - } - category + let is_warning = message.fast_isinstance(vm.ctx.exceptions.warning); + let (text, category, message) = if is_warning { + let text = message.str(vm)?; + let cat = message.class().to_owned(); + (text, cat, message) } else { - vm.ctx.exceptions.user_warning.to_owned() + // For non-Warning messages, convert to string via str() + let text = message.str(vm)?; + let cat = category.unwrap_or_else(|| vm.ctx.exceptions.user_warning.to_owned()); + let instance = cat.as_object().call((text.clone(),), vm)?; + (text, cat, instance) }; - let category = if message.fast_isinstance(vm.ctx.exceptions.warning) { - message.class().to_owned() - } else { - category - }; + let lineno_obj: PyObjectRef = vm.ctx.new_int(lineno).into(); - // Create key. - let key = PyTuple::new_ref( + // key = (text, category, lineno) + let key: PyObjectRef = PyTuple::new_ref( vec![ - vm.ctx.new_int(3).into(), - vm.ctx.new_str(text).into(), + text.clone().into(), category.as_object().to_owned(), - vm.ctx.new_int(lineno).into(), + lineno_obj.clone(), ], &vm.ctx, - ); + ) + .into(); - if !vm.is_none(registry.as_object()) && already_warned(registry, key.into_object(), false, vm)? - { + // Check if already warned + if !vm.is_none(&registry) && already_warned(&registry, key.clone(), false, vm)? { return Ok(()); } - let item = vm.ctx.new_tuple(vec![]); + // Get filter action let action = get_filter( category.as_object().to_owned(), - vm.ctx.new_str(text).into(), + text.clone().into(), lineno, module, - item, vm, )?; + let action_str = PyStrRef::try_from_object(vm, action) + .map_err(|_| vm.new_type_error("action must be a string".to_owned()))?; - if action.str_utf8(vm)?.as_str().eq("error") { - return Err(vm.new_type_error(message.to_string())); + if action_str.as_str() == "error" { + let exc = PyBaseExceptionRef::try_from_object(vm, message)?; + return Err(exc); + } + if action_str.as_str() == "ignore" { + return Ok(()); } - if action.str_utf8(vm)?.as_str().eq("ignore") { + // For everything except "always"/"all", record in registry then + // check per-action registries. + let already = if action_str.as_str() != "always" && action_str.as_str() != "all" { + if !vm.is_none(&registry) { + registry.set_item(&*key, vm.ctx.true_value.clone().into(), vm)?; + } + + match action_str.as_str() { + "once" => { + let reg = if vm.is_none(&registry) { + get_once_registry(vm)? + } else { + registry.clone() + }; + update_registry(&reg, text.as_ref(), category.as_object(), false, vm)? + } + "module" => { + if !vm.is_none(&registry) { + update_registry(&registry, text.as_ref(), category.as_object(), false, vm)? + } else { + false + } + } + "default" => false, + other => { + return Err(vm.new_runtime_error(format!( + "Unrecognized action ({other}) in warnings.filters:\n {other}" + ))); + } + } + } else { + false + }; + + if already { return Ok(()); } call_show_warning( - // t_state, category, + text, message, filename, - lineno, // lineno_obj, + lineno, + lineno_obj, source_line, source, vm, ) } +#[allow(clippy::too_many_arguments)] fn call_show_warning( category: PyTypeRef, - message: PyStrRef, + text: PyStrRef, + message: PyObjectRef, filename: PyStrRef, lineno: usize, + lineno_obj: PyObjectRef, source_line: Option<PyObjectRef>, source: Option<PyObjectRef>, vm: &VirtualMachine, @@ -310,22 +466,24 @@ fn call_show_warning( let Some(show_fn) = get_warnings_attr(vm, identifier!(&vm.ctx, _showwarnmsg), source.is_some())? else { - return show_warning(filename, lineno, message, category, source_line, vm); + return show_warning(filename, lineno, text, category, source_line, vm); }; if !show_fn.is_callable() { - return Err(vm.new_type_error("warnings._showwarnmsg() must be set to a callable")); + return Err( + vm.new_type_error("warnings._showwarnmsg() must be set to a callable".to_owned()) + ); } let Some(warnmsg_cls) = get_warnings_attr(vm, identifier!(&vm.ctx, WarningMessage), false)? else { - return Err(vm.new_type_error("unable to get warnings.WarningMessage")); + return Err(vm.new_runtime_error("unable to get warnings.WarningMessage".to_owned())); }; let msg = warnmsg_cls.call( vec![ - message.into(), + message, category.into(), filename.into(), - vm.new_pyobj(lineno), + lineno_obj, vm.ctx.none(), vm.ctx.none(), vm.unwrap_or_none(source), @@ -337,77 +495,118 @@ fn call_show_warning( } fn show_warning( - _filename: PyStrRef, - _lineno: usize, + filename: PyStrRef, + lineno: usize, text: PyStrRef, category: PyTypeRef, _source_line: Option<PyObjectRef>, vm: &VirtualMachine, ) -> PyResult<()> { let stderr = crate::stdlib::sys::PyStderr(vm); - writeln!(stderr, "{}: {}", category.name(), text); + writeln!( + stderr, + "{}:{}: {}: {}", + filename, + lineno, + category.name(), + text + ); Ok(()) } +/// Check if a frame's filename starts with any of the given prefixes. +fn is_filename_to_skip(frame: &crate::frame::Frame, prefixes: &PyTupleRef) -> bool { + let filename = frame.f_code().co_filename(); + let filename_s = filename.as_str(); + prefixes.iter().any(|prefix| { + prefix + .downcast_ref::<PyStr>() + .is_some_and(|s| filename_s.starts_with(s.as_str())) + }) +} + +/// Like Frame::next_external_frame but also skips frames matching prefixes. +fn next_external_frame_with_skip( + frame: &crate::frame::FrameRef, + skip_file_prefixes: Option<&PyTupleRef>, + vm: &VirtualMachine, +) -> Option<crate::frame::FrameRef> { + let mut f = frame.f_back(vm); + loop { + let current: crate::frame::FrameRef = f.take()?; + if current.is_internal_frame() + || skip_file_prefixes.is_some_and(|p| is_filename_to_skip(&current, p)) + { + f = current.f_back(vm); + } else { + return Some(current); + } + } +} + /// filename, module, and registry are new refs, globals is borrowed /// Returns `Ok` on success, or `Err` on error (no new refs) fn setup_context( mut stack_level: isize, + skip_file_prefixes: Option<&PyTupleRef>, vm: &VirtualMachine, -) -> PyResult< - // filename, lineno, module, registry - (PyStrRef, usize, Option<PyObjectRef>, PyObjectRef), -> { - let __warningregistry__ = "__warningregistry__"; - let __name__ = "__name__"; - - let mut f = vm.current_frame().as_deref().cloned(); +) -> PyResult<(PyStrRef, usize, Option<PyObjectRef>, PyObjectRef)> { + let mut f = vm.current_frame(); // Stack level comparisons to Python code is off by one as there is no // warnings-related stack level to avoid. if stack_level <= 0 || f.as_ref().is_some_and(|frame| frame.is_internal_frame()) { - loop { + while { stack_level -= 1; - if stack_level <= 0 { - break; - } - if let Some(tmp) = f { - f = tmp.f_back(vm); - } else { - break; + stack_level > 0 + } { + match f { + Some(tmp) => f = tmp.f_back(vm), + None => break, } } } else { - loop { + while { stack_level -= 1; - if stack_level <= 0 { - break; - } - if let Some(tmp) = f { - f = tmp.next_external_frame(vm); - } else { - break; + stack_level > 0 + } { + match f { + Some(tmp) => f = next_external_frame_with_skip(&tmp, skip_file_prefixes, vm), + None => break, } } } let (globals, filename, lineno) = if let Some(f) = f { - (f.globals.clone(), f.code.source_path, f.f_lineno()) + (f.globals.clone(), f.code.source_path(), f.f_lineno()) + } else if let Some(frame) = vm.current_frame() { + // We have a frame but it wasn't found during stack walking + (frame.globals.clone(), vm.ctx.intern_str("<sys>"), 1) } else { - (vm.current_globals().clone(), vm.ctx.intern_str("sys"), 1) + // No frames on the stack - use sys.__dict__ (interp->sysdict) + let globals = vm + .sys_module + .as_object() + .get_attr(identifier!(vm, __dict__), vm) + .and_then(|d| { + d.downcast::<crate::builtins::PyDict>() + .map_err(|_| vm.new_type_error("sys.__dict__ is not a dictionary".to_owned())) + })?; + (globals, vm.ctx.intern_str("<sys>"), 0) }; - let registry = if let Ok(registry) = globals.get_item(__warningregistry__, vm) { - registry - } else { - let registry = vm.ctx.new_dict(); - globals.set_item(__warningregistry__, registry.clone().into(), vm)?; - registry.into() + let registry = match globals.get_item("__warningregistry__", vm) { + Ok(r) => r, + Err(_) => { + let r = vm.ctx.new_dict(); + globals.set_item("__warningregistry__", r.clone().into(), vm)?; + r.into() + } }; // Setup module. let module = globals - .get_item(__name__, vm) + .get_item("__name__", vm) .unwrap_or_else(|_| vm.new_pyobj("<string>")); Ok((filename.to_owned(), lineno, Some(module), registry)) } diff --git a/crates/vm/src/windows.rs b/crates/vm/src/windows.rs index ccf940811b8..017384e2f5c 100644 --- a/crates/vm/src/windows.rs +++ b/crates/vm/src/windows.rs @@ -79,7 +79,7 @@ impl ToPyObject for WinHandle { pub fn init_winsock() { static WSA_INIT: parking_lot::Once = parking_lot::Once::new(); WSA_INIT.call_once(|| unsafe { - let mut wsa_data = std::mem::MaybeUninit::uninit(); + let mut wsa_data = core::mem::MaybeUninit::uninit(); let _ = windows_sys::Win32::Networking::WinSock::WSAStartup(0x0101, wsa_data.as_mut_ptr()); }) } @@ -244,7 +244,7 @@ fn attributes_from_dir( }; let wide: Vec<u16> = path.to_wide_with_nul(); - let mut find_data: WIN32_FIND_DATAW = unsafe { std::mem::zeroed() }; + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; let handle = unsafe { FindFirstFileW(wide.as_ptr(), &mut find_data) }; if handle == INVALID_HANDLE_VALUE { @@ -252,7 +252,7 @@ fn attributes_from_dir( } unsafe { FindClose(handle) }; - let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { core::mem::zeroed() }; info.dwFileAttributes = find_data.dwFileAttributes; info.ftCreationTime = find_data.ftCreationTime; info.ftLastAccessTime = find_data.ftLastAccessTime; @@ -301,14 +301,14 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatSt wide.as_ptr(), access, 0, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, flags, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; - let mut file_info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + let mut file_info: BY_HANDLE_FILE_INFORMATION = unsafe { core::mem::zeroed() }; let mut tag_info = FileAttributeTagInfo::default(); let mut is_unhandled_tag = false; @@ -337,10 +337,10 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatSt wide.as_ptr(), access | GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, flags, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if h_file == INVALID_HANDLE_VALUE { @@ -355,10 +355,10 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatSt wide.as_ptr(), access, 0, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, flags | FILE_FLAG_OPEN_REPARSE_POINT, - std::ptr::null_mut(), + core::ptr::null_mut(), ) }; if h_file == INVALID_HANDLE_VALUE { @@ -400,13 +400,13 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatSt // Query the reparse tag if !traverse || is_unhandled_tag { - let mut local_tag_info: FileAttributeTagInfo = unsafe { std::mem::zeroed() }; + let mut local_tag_info: FileAttributeTagInfo = unsafe { core::mem::zeroed() }; let ret = unsafe { GetFileInformationByHandleEx( h_file, FileAttributeTagInfo, &mut local_tag_info as *mut _ as *mut _, - std::mem::size_of::<FileAttributeTagInfo>() as u32, + core::mem::size_of::<FileAttributeTagInfo>() as u32, ) }; if ret == 0 { @@ -453,24 +453,24 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatSt } // Get FILE_BASIC_INFO - let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; + let mut basic_info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; let has_basic_info = unsafe { GetFileInformationByHandleEx( h_file, FileBasicInfo, &mut basic_info as *mut _ as *mut _, - std::mem::size_of::<FILE_BASIC_INFO>() as u32, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, ) } != 0; // Get FILE_ID_INFO (optional) - let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; + let mut id_info: FILE_ID_INFO = unsafe { core::mem::zeroed() }; let has_id_info = unsafe { GetFileInformationByHandleEx( h_file, FileIdInfo, &mut id_info as *mut _ as *mut _, - std::mem::size_of::<FILE_ID_INFO>() as u32, + core::mem::size_of::<FILE_ID_INFO>() as u32, ) } != 0; diff --git a/crates/wasm/src/browser_module.rs b/crates/wasm/src/browser_module.rs index 0b978a4b563..d1eecce28a9 100644 --- a/crates/wasm/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -1,6 +1,4 @@ -use rustpython_vm::VirtualMachine; - -pub(crate) use _browser::make_module; +pub(crate) use _browser::module_def; #[pymodule] mod _browser { @@ -117,7 +115,8 @@ mod _browser { #[pyfunction] fn request_animation_frame(func: ArgCallable, vm: &VirtualMachine) -> PyResult { - use std::{cell::RefCell, rc::Rc}; + use alloc::rc::Rc; + use core::cell::RefCell; // this basic setup for request_animation_frame taken from: // https://rustwasm.github.io/wasm-bindgen/examples/request-animation-frame.html @@ -256,8 +255,3 @@ mod _browser { Ok(PyPromise::from_future(future).into_pyobject(vm)) } } - -pub fn setup_browser_module(vm: &mut VirtualMachine) { - vm.add_native_module("_browser".to_owned(), Box::new(make_module)); - vm.add_frozen(py_freeze!(dir = "Lib")); -} diff --git a/crates/wasm/src/convert.rs b/crates/wasm/src/convert.rs index d1821f2e733..f84b0d46239 100644 --- a/crates/wasm/src/convert.rs +++ b/crates/wasm/src/convert.rs @@ -4,8 +4,8 @@ use crate::js_module; use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm}; use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; use rustpython_vm::{ - AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, - builtins::PyBaseExceptionRef, + AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, + builtins::{PyBaseException, PyBaseExceptionRef}, compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType}, exceptions, function::{ArgBytesLike, FuncArgs}, @@ -32,7 +32,7 @@ extern "C" { fn new(info: JsValue) -> PyError; } -pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyBaseExceptionRef) -> JsValue { +pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &Py<PyBaseException>) -> JsValue { let js_err = vm.try_class("_js", "JSError").ok(); let js_arg = if js_err.is_some_and(|js_err| py_err.fast_isinstance(&js_err)) { py_err.get_arg(0) diff --git a/crates/wasm/src/js_module.rs b/crates/wasm/src/js_module.rs index 1d8ca0961ca..750e85994a1 100644 --- a/crates/wasm/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -1,5 +1,4 @@ pub(crate) use _js::{PyJsValue, PyPromise}; -use rustpython_vm::VirtualMachine; #[pymodule] mod _js { @@ -8,6 +7,7 @@ mod _js { vm_class::{WASMVirtualMachine, stored_vm_from_wasm}, weak_vm, }; + use core::{cell, fmt, future}; use js_sys::{Array, Object, Promise, Reflect}; use rustpython_vm::{ Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, @@ -17,7 +17,6 @@ mod _js { protocol::PyIterReturn, types::{IterNext, Representable, SelfIter}, }; - use std::{cell, fmt, future}; use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; use wasm_bindgen_futures::{JsFuture, future_to_promise}; @@ -612,8 +611,7 @@ mod _js { fn js_error(vm: &VirtualMachine) -> PyTypeRef { let ctx = &vm.ctx; let js_error = PyRef::leak( - PyType::new_simple_heap("JSError", &vm.ctx.exceptions.exception_type.to_owned(), ctx) - .unwrap(), + PyType::new_simple_heap("JSError", vm.ctx.exceptions.exception_type, ctx).unwrap(), ); extend_class!(ctx, js_error, { "value" => ctx.new_readonly_getset("value", js_error, |exc: PyBaseExceptionRef| exc.get_arg(0)), @@ -622,8 +620,4 @@ mod _js { } } -pub(crate) use _js::make_module; - -pub fn setup_js_module(vm: &mut VirtualMachine) { - vm.add_native_module("_js".to_owned(), Box::new(make_module)); -} +pub(crate) use _js::module_def; diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs index 8d1d19ddae1..99668df2855 100644 --- a/crates/wasm/src/lib.rs +++ b/crates/wasm/src/lib.rs @@ -1,3 +1,5 @@ +extern crate alloc; + pub mod browser_module; pub mod convert; pub mod js_module; @@ -9,7 +11,6 @@ extern crate rustpython_vm; use js_sys::{Reflect, WebAssembly::RuntimeError}; use std::panic; -pub use vm_class::add_init_func; pub(crate) use vm_class::weak_vm; use wasm_bindgen::prelude::*; diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index ad5df8dab96..4ad347a19e1 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -1,20 +1,16 @@ use crate::{ - browser_module::setup_browser_module, + browser_module, convert::{self, PyResultExt}, js_module, wasm_builtins, }; +use alloc::rc::{Rc, Weak}; +use core::cell::RefCell; use js_sys::{Object, TypeError}; use rustpython_vm::{ - Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, - builtins::{PyModule, PyWeak}, - compiler::Mode, - scope::Scope, -}; -use std::{ - cell::RefCell, - collections::HashMap, - rc::{Rc, Weak}, + Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, builtins::PyWeak, + compiler::Mode, scope::Scope, }; +use std::collections::HashMap; use wasm_bindgen::prelude::*; pub(crate) struct StoredVirtualMachine { @@ -26,69 +22,67 @@ pub(crate) struct StoredVirtualMachine { } #[pymodule] -mod _window {} - -fn init_window_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _window::make_module(vm); - - extend_module!(vm, &module, { - "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), - }); - - module +mod _window { + use super::{js_module, wasm_builtins}; + use rustpython_vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + extend_module!(vm, module, { + "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), + }); + Ok(()) + } } impl StoredVirtualMachine { fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { - let mut scope = None; let mut settings = Settings::default(); settings.allow_external_library = false; - let interp = Interpreter::with_init(settings, |vm| { - #[cfg(feature = "freeze-stdlib")] - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - #[cfg(feature = "freeze-stdlib")] - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); + let mut builder = Interpreter::builder(settings); - vm.wasm_id = Some(id); + #[cfg(feature = "freeze-stdlib")] + { + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder = builder + .add_native_modules(&defs) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB); + } - js_module::setup_js_module(vm); - if inject_browser_module { - vm.add_native_module("_window".to_owned(), Box::new(init_window_module)); - setup_browser_module(vm); - } + // Add wasm-specific modules + let js_def = js_module::module_def(&builder.ctx); + builder = builder.add_native_module(js_def); - VM_INIT_FUNCS.with_borrow(|funcs| { - for f in funcs { - f(vm) - } - }); + if inject_browser_module { + let window_def = _window::module_def(&builder.ctx); + let browser_def = browser_module::module_def(&builder.ctx); + builder = builder + .add_native_modules(&[window_def, browser_def]) + .add_frozen_modules(rustpython_vm::py_freeze!(dir = "../Lib")); + } - scope = Some(vm.new_scope_with_builtins()); - }); + let interp = builder + .init_hook(move |vm| { + vm.wasm_id = Some(id); + }) + .build(); + + let scope = interp.enter(|vm| vm.new_scope_with_builtins()); StoredVirtualMachine { interp, - scope: scope.unwrap(), + scope, held_objects: RefCell::new(Vec::new()), } } } -/// Add a hook to add builtins or frozen modules to the RustPython VirtualMachine while it's -/// initializing. -pub fn add_init_func(f: fn(&mut VirtualMachine)) { - VM_INIT_FUNCS.with_borrow_mut(|funcs| funcs.push(f)) -} - // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! // probably gets compiled down to a normal-ish static variable, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { static STORED_VMS: RefCell<HashMap<String, Rc<StoredVirtualMachine>>> = RefCell::default(); - static VM_INIT_FUNCS: RefCell<Vec<fn(&mut VirtualMachine)>> = const { - RefCell::new(Vec::new()) - }; } pub fn get_vm_id(vm: &VirtualMachine) -> &str { @@ -108,6 +102,7 @@ pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak<StoredVirtualMachine> { STORED_VMS.with_borrow(|vms| Rc::downgrade(vms.get(id).expect("VirtualMachine is not valid"))) } +#[derive(Clone, Copy)] #[wasm_bindgen(js_name = vmStore)] pub struct VMStore; diff --git a/crates/wtf8/src/core_str_count.rs b/crates/wtf8/src/core_str_count.rs index f02f0a5708d..8f9d5585bc7 100644 --- a/crates/wtf8/src/core_str_count.rs +++ b/crates/wtf8/src/core_str_count.rs @@ -60,7 +60,7 @@ fn do_count_chars(s: &Wtf8) -> usize { // a subset of the sum of this chunk, like a `[u8; size_of::<usize>()]`. let mut counts = 0; - let (unrolled_chunks, remainder) = slice_as_chunks::<_, UNROLL_INNER>(chunk); + let (unrolled_chunks, remainder) = chunk.as_chunks::<UNROLL_INNER>(); for unrolled in unrolled_chunks { for &word in unrolled { // Because `CHUNK_SIZE` is < 256, this addition can't cause the @@ -137,26 +137,6 @@ const fn usize_repeat_u16(x: u16) -> usize { } r } - -fn slice_as_chunks<T, const N: usize>(slice: &[T]) -> (&[[T; N]], &[T]) { - assert!(N != 0, "chunk size must be non-zero"); - let len_rounded_down = slice.len() / N * N; - // SAFETY: The rounded-down value is always the same or smaller than the - // original length, and thus must be in-bounds of the slice. - let (multiple_of_n, remainder) = unsafe { slice.split_at_unchecked(len_rounded_down) }; - // SAFETY: We already panicked for zero, and ensured by construction - // that the length of the subslice is a multiple of N. - let array_slice = unsafe { slice_as_chunks_unchecked(multiple_of_n) }; - (array_slice, remainder) -} - -unsafe fn slice_as_chunks_unchecked<T, const N: usize>(slice: &[T]) -> &[[T; N]] { - let new_len = slice.len() / N; - // SAFETY: We cast a slice of `new_len * N` elements into - // a slice of `new_len` many `N` elements chunks. - unsafe { std::slice::from_raw_parts(slice.as_ptr().cast(), new_len) } -} - const fn unlikely(x: bool) -> bool { x } diff --git a/crates/wtf8/src/lib.rs b/crates/wtf8/src/lib.rs index 01deb61f19b..6614ca83572 100644 --- a/crates/wtf8/src/lib.rs +++ b/crates/wtf8/src/lib.rs @@ -33,8 +33,17 @@ //! [WTF-8]: https://simonsapin.github.io/wtf-8 //! [`OsStr`]: std::ffi::OsStr +#![no_std] #![allow(clippy::precedence, clippy::match_overlapping_arm)] +extern crate alloc; + +use alloc::borrow::{Cow, ToOwned}; +use alloc::boxed::Box; +use alloc::collections::TryReserveError; +use alloc::string::String; +use alloc::vec::Vec; +use core::borrow::Borrow; use core::fmt; use core::hash::{Hash, Hasher}; use core::iter::FusedIterator; @@ -46,10 +55,6 @@ use core_char::MAX_LEN_UTF8; use core_char::{MAX_LEN_UTF16, encode_utf8_raw, encode_utf16_raw, len_utf8}; use core_str::{next_code_point, next_code_point_reverse}; use itertools::{Either, Itertools}; -use std::borrow::{Borrow, Cow}; -use std::collections::TryReserveError; -use std::string::String; -use std::vec::Vec; use bstr::{ByteSlice, ByteVec}; @@ -665,7 +670,7 @@ impl PartialEq<str> for Wtf8 { impl fmt::Debug for Wtf8 { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { fn write_str_escaped(f: &mut fmt::Formatter<'_>, s: &str) -> fmt::Result { - use std::fmt::Write; + use core::fmt::Write; for c in s.chars().flat_map(|c| c.escape_debug()) { f.write_char(c)? } @@ -765,7 +770,7 @@ impl Wtf8 { #[inline] pub fn from_bytes(b: &[u8]) -> Option<&Self> { let mut rest = b; - while let Err(e) = std::str::from_utf8(rest) { + while let Err(e) = core::str::from_utf8(rest) { rest = &rest[e.valid_up_to()..]; let _ = Self::decode_surrogate(rest)?; rest = &rest[3..]; @@ -899,7 +904,7 @@ impl Wtf8 { { self.chunks().flat_map(move |chunk| match chunk { Wtf8Chunk::Utf8(s) => Either::Left(f(s).map_into()), - Wtf8Chunk::Surrogate(c) => Either::Right(std::iter::once(c)), + Wtf8Chunk::Surrogate(c) => Either::Right(core::iter::once(c)), }) } @@ -1469,7 +1474,7 @@ impl<'a> Iterator for Wtf8Chunks<'a> { } None => { let s = - unsafe { str::from_utf8_unchecked(std::mem::take(&mut self.wtf8).as_bytes()) }; + unsafe { str::from_utf8_unchecked(core::mem::take(&mut self.wtf8).as_bytes()) }; (!s.is_empty()).then_some(Wtf8Chunk::Utf8(s)) } } diff --git a/demo_closures.py b/demo_closures.py deleted file mode 100644 index 0ed673a94fd..00000000000 --- a/demo_closures.py +++ /dev/null @@ -1,12 +0,0 @@ -def foo(x): - def bar(z): - return z + x - - return bar - - -f = foo(9) -g = foo(10) - -print(f(2)) -print(g(2)) diff --git a/example_projects/frozen_stdlib/src/main.rs b/example_projects/frozen_stdlib/src/main.rs index 2688d2164a2..8ff316faed8 100644 --- a/example_projects/frozen_stdlib/src/main.rs +++ b/example_projects/frozen_stdlib/src/main.rs @@ -1,9 +1,10 @@ // spell-checker:ignore aheui -/// Setting up a project with a frozen stdlib can be done *either* by using `rustpython::InterpreterConfig` or `rustpython_vm::Interpreter::with_init`. -/// See each function for example. -/// -/// See also: `aheui-rust.md` for freezing your own package. +//! Setting up a project with a frozen stdlib can be done *either* by using `rustpython::InterpreterBuilder` or `rustpython_vm::Interpreter::builder`. +//! See each function for example. +//! +//! See also: `aheui-rust.md` for freezing your own package. +use rustpython::InterpreterBuilderExt; use rustpython_vm::{PyResult, VirtualMachine}; fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { @@ -17,21 +18,19 @@ fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { } fn interpreter_with_config() { - let interpreter = rustpython::InterpreterConfig::new() + let interpreter = rustpython::InterpreterBuilder::new() .init_stdlib() .interpreter(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython::InterpreterConfig", vm)); + interpreter.run(|vm| run("rustpython::InterpreterBuilder", vm)); } fn interpreter_with_vm() { - let interpreter = rustpython_vm::Interpreter::with_init(Default::default(), |vm| { - // This is unintuitive, but the stdlib is out of the vm crate. - // Any suggestion to improve this is welcome. - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); - }); + let interpreter = rustpython_vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .build(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython_vm::Interpreter::with_init", vm)); + interpreter.run(|vm| run("rustpython_vm::Interpreter::builder", vm)); } fn main() { diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index dee17058475..3a7c0ce610e 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -1,17 +1,12 @@ +use rustpython::InterpreterBuilderExt; use rustpython::vm::{ PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, pyclass, pymodule, }; pub fn main() { - let interp = rustpython::InterpreterConfig::new() - .init_stdlib() - .init_hook(Box::new(|vm| { - vm.add_native_module( - "rust_py_module".to_owned(), - Box::new(rust_py_module::make_module), - ); - })) - .interpreter(); + let builder = rustpython::Interpreter::builder(Default::default()); + let def = rust_py_module::module_def(&builder.ctx); + let interp = builder.init_stdlib().add_native_module(def).build(); interp.enter(|vm| { vm.insert_sys_path(vm.new_pyobj("examples")) diff --git a/examples/dis.rs b/examples/dis.rs index 504b734ca59..0b6190dde3c 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -9,9 +9,9 @@ #[macro_use] extern crate log; +use core::error::Error; use lexopt::ValueExt; use rustpython_compiler as compiler; -use std::error::Error; use std::fs; use std::path::{Path, PathBuf}; @@ -53,11 +53,14 @@ fn main() -> Result<(), lexopt::Error> { return Err("expected at least one argument".into()); } - let opts = compiler::CompileOpts { optimize }; + let opts = compiler::CompileOpts { + optimize, + debug_ranges: true, + }; for script in &scripts { if script.exists() && script.is_file() { - let res = display_script(script, mode, opts.clone(), expand_code_objects); + let res = display_script(script, mode, opts, expand_code_objects); if let Err(e) = res { error!("Error while compiling {script:?}: {e}"); } diff --git a/examples/freeze/main.rs b/examples/freeze/main.rs index 48991129073..ab26a2bc808 100644 --- a/examples/freeze/main.rs +++ b/examples/freeze/main.rs @@ -7,9 +7,8 @@ fn main() -> vm::PyResult<()> { fn run(vm: &vm::VirtualMachine) -> vm::PyResult<()> { let scope = vm.new_scope_with_builtins(); - // the file parameter is relative to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR: - // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates - let module = vm::py_compile!(file = "examples/freeze/freeze.py"); + // the file parameter is relative to the current file. + let module = vm::py_compile!(file = "freeze.py"); let res = vm.run_code_obj(vm.ctx.new_code(module), scope); diff --git a/examples/generator.rs b/examples/generator.rs index 27733a1913d..55841c767a1 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -42,9 +42,9 @@ gen() } fn main() -> ExitCode { - let interp = vm::Interpreter::with_init(Default::default(), |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(Default::default()); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); vm::common::os::exit_code(interp.run(|_vm| result)) } diff --git a/examples/package_embed.rs b/examples/package_embed.rs index e82e71f5ceb..bb2f29e3f5f 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -19,9 +19,9 @@ fn main() -> ExitCode { // Add standard library path let mut settings = vm::Settings::default(); settings.path_list.push("Lib".to_owned()); - let interp = vm::Interpreter::with_init(settings, |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); let result = result.map(|result| { println!("name: {result}"); diff --git a/extra_tests/custom_text_test_runner.py b/extra_tests/custom_text_test_runner.py index 018121f0da4..afec493a66c 100644 --- a/extra_tests/custom_text_test_runner.py +++ b/extra_tests/custom_text_test_runner.py @@ -112,7 +112,7 @@ def __call__(self, data_list, totals=None): def get_function_args(func_ref): try: - return [p for p in inspect.getargspec(func_ref).args if p != "self"] + return [p for p in inspect.getfullargspec(func_ref).args if p != "self"] except: return None diff --git a/extra_tests/jsontests.py b/extra_tests/jsontests.py index c1f92509fe7..f3213ac09d1 100644 --- a/extra_tests/jsontests.py +++ b/extra_tests/jsontests.py @@ -2,7 +2,7 @@ import unittest from custom_text_test_runner import CustomTextTestRunner as Runner -from test.libregrtest.runtest import findtests +from test.libregrtest.findtests import findtests testnames = findtests() # idk why this fixes the hanging, if it does diff --git a/extra_tests/snippets/builtin_bool.py b/extra_tests/snippets/builtin_bool.py index 6b6b4e0e08b..902ed0cced0 100644 --- a/extra_tests/snippets/builtin_bool.py +++ b/extra_tests/snippets/builtin_bool.py @@ -18,7 +18,9 @@ assert bool(1) is True assert bool({}) is False -assert bool(NotImplemented) is True +# NotImplemented cannot be used in a boolean context (Python 3.14+) +with assert_raises(TypeError): + bool(NotImplemented) assert bool(...) is True if not 1: diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index 0b7e419390e..ee11e913ff2 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -153,16 +153,41 @@ class B(bytearray): # # hex from hex assert bytearray([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytearray.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytearray.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytearray.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytearray.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytearray.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytearray.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytearray.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytearray.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytearray.fromhex(b"hhjjk") # center assert [bytearray(b"koki").center(i, b"|") for i in range(3, 10)] == [ diff --git a/extra_tests/snippets/builtin_bytes.py b/extra_tests/snippets/builtin_bytes.py index 9347fbc8fab..2cb4c317f49 100644 --- a/extra_tests/snippets/builtin_bytes.py +++ b/extra_tests/snippets/builtin_bytes.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises, skip_if_unsupported # new @@ -135,16 +137,41 @@ # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytes.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytes.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytes.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytes.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytes.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytes.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytes.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytes.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytes.fromhex(b"hhjjk") # center assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ @@ -611,6 +638,9 @@ assert b"\xc2\xae\x75\x73\x74".decode() == "®ust" assert b"\xe4\xb8\xad\xe6\x96\x87\xe5\xad\x97".decode("utf-8") == "中文字" +# gh-2391 +assert b"-\xff".decode(sys.getfilesystemencoding(), "surrogateescape") == "-\udcff" + # mod assert b"rust%bpython%b" % (b" ", b"!") == b"rust python!" assert b"x=%i y=%f" % (1, 2.5) == b"x=1 y=2.500000" diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 490f831f522..8879e130bc2 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -85,6 +85,13 @@ def __init__(self, value): assert exc.offset is None assert exc.text is None +err = SyntaxError("bad bad", ("bad.py", 1, 2, "abcdefg")) +err.msg = "changed" +assert err.msg == "changed" +assert str(err) == "changed (bad.py, line 1)" +del err.msg +assert err.msg is None + # Regression to: # https://github.com/RustPython/RustPython/issues/2779 @@ -232,12 +239,10 @@ class SubError(MyError): raise e except MyError as exc: # It was a segmentation fault before, will print info to stdout: - if platform.python_implementation() == "RustPython": - # For some reason `CPython` hangs on this code: - sys.excepthook(type(exc), exc, exc.__traceback__) - assert isinstance(exc, MyError) - assert exc.__cause__ is None - assert exc.__context__ is e + sys.excepthook(type(exc), exc, exc.__traceback__) + assert isinstance(exc, MyError) + assert exc.__cause__ is None + assert exc.__context__ is e # Regression to @@ -248,26 +253,42 @@ class SubError(MyError): assert BaseException.__init__.__qualname__ == "BaseException.__init__" assert BaseException().__dict__ == {} +# Exception inherits __init__ from BaseException assert Exception.__new__.__qualname__ == "Exception.__new__", ( Exception.__new__.__qualname__ ) -assert Exception.__init__.__qualname__ == "Exception.__init__", ( +assert Exception.__init__.__qualname__ == "BaseException.__init__", ( Exception.__init__.__qualname__ ) assert Exception().__dict__ == {} -# Extends `BaseException`, simple: +# Extends `BaseException`, simple - inherits __init__ from BaseException: assert KeyboardInterrupt.__new__.__qualname__ == "KeyboardInterrupt.__new__", ( KeyboardInterrupt.__new__.__qualname__ ) -assert KeyboardInterrupt.__init__.__qualname__ == "KeyboardInterrupt.__init__" +assert KeyboardInterrupt.__init__.__qualname__ == "BaseException.__init__" assert KeyboardInterrupt().__dict__ == {} -# Extends `Exception`, simple: +# Extends `BaseException`, complex - has its own __init__: +# SystemExit_init sets self.code based on args length +assert SystemExit.__init__.__qualname__ == "SystemExit.__init__" +assert SystemExit.__dict__.get("__init__") is not None, ( + "SystemExit must have its own __init__" +) +assert SystemExit.__init__ is not BaseException.__init__ +assert SystemExit().__dict__ == {} +# SystemExit.code behavior: +assert SystemExit().code is None +assert SystemExit(1).code == 1 +assert SystemExit(1, 2).code == (1, 2) +assert SystemExit(1, 2, 3).code == (1, 2, 3) + + +# Extends `Exception`, simple - inherits __init__ from BaseException: assert TypeError.__new__.__qualname__ == "TypeError.__new__" -assert TypeError.__init__.__qualname__ == "TypeError.__init__" +assert TypeError.__init__.__qualname__ == "BaseException.__init__" assert TypeError().__dict__ == {} @@ -349,7 +370,8 @@ class SubError(MyError): # Custom `__new__` and `__init__`: assert ImportError.__init__.__qualname__ == "ImportError.__init__" assert ImportError(name="a").name == "a" -assert ModuleNotFoundError.__init__.__qualname__ == "ModuleNotFoundError.__init__" +# ModuleNotFoundError inherits __init__ from ImportError via MRO (MiddlingExtendsException) +assert ModuleNotFoundError.__init__.__qualname__ == "ImportError.__init__" assert ModuleNotFoundError(name="a").name == "a" @@ -359,3 +381,15 @@ class SubError(MyError): vars(builtins).values(), ): assert isinstance(exc.__doc__, str) + + +# except* handling should normalize non-group exceptions +try: + raise ValueError("x") +except* ValueError as err: + assert isinstance(err, ExceptionGroup) + assert len(err.exceptions) == 1 + assert isinstance(err.exceptions[0], ValueError) + assert err.exceptions[0].args == ("x",) +else: + assert False, "except* handler did not run" diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index 96ccc46ba80..9b2c8388790 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -12,6 +12,14 @@ class A: assert type(hash(1.1)) is int assert type(hash("")) is int + +class Evil: + def __hash__(self): + return 1 << 63 + + +assert hash(Evil()) == 4 + with assert_raises(TypeError): hash({}) diff --git a/extra_tests/snippets/builtin_int.py b/extra_tests/snippets/builtin_int.py index bc3cd5fd996..aab24cbb4cc 100644 --- a/extra_tests/snippets/builtin_int.py +++ b/extra_tests/snippets/builtin_int.py @@ -318,8 +318,9 @@ def __int__(self): assert isinstance((1).__round__(0), int) assert (0).__round__(0) == 0 assert (1).__round__(0) == 1 -assert_raises(TypeError, lambda: (0).__round__(None)) -assert_raises(TypeError, lambda: (1).__round__(None)) +# Python 3.14+: __round__(None) is now allowed, same as __round__() +assert (0).__round__(None) == 0 +assert (1).__round__(None) == 1 assert_raises(TypeError, lambda: (0).__round__(0.0)) assert_raises(TypeError, lambda: (1).__round__(0.0)) diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index 3e6bb8fc943..d4afbffa1cb 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -270,6 +270,55 @@ def __gt__(self, other): lst.sort(key=C) assert lst == [1, 2, 3, 4, 5] + +# Test that sorted() uses __lt__ (not __gt__) for comparisons. +# Track which comparison method is actually called during sort. +class TrackComparison: + lt_calls = 0 + gt_calls = 0 + + def __init__(self, value): + self.value = value + + def __lt__(self, other): + TrackComparison.lt_calls += 1 + return self.value < other.value + + def __gt__(self, other): + TrackComparison.gt_calls += 1 + return self.value > other.value + + +# Reset and test sorted() +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +sorted(items) +assert TrackComparison.lt_calls > 0, "sorted() should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"sorted() should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + +# Reset and test list.sort() +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +items.sort() +assert TrackComparison.lt_calls > 0, "list.sort() should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"list.sort() should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + +# Reset and test sorted(reverse=True) - should still use __lt__, not __gt__ +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +sorted(items, reverse=True) +assert TrackComparison.lt_calls > 0, "sorted(reverse=True) should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"sorted(reverse=True) should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + lst = [5, 1, 2, 3, 4] diff --git a/extra_tests/snippets/builtin_posixshmem.py b/extra_tests/snippets/builtin_posixshmem.py new file mode 100644 index 00000000000..38ace68d584 --- /dev/null +++ b/extra_tests/snippets/builtin_posixshmem.py @@ -0,0 +1,12 @@ +import os +import sys + +if os.name != "posix": + sys.exit(0) + +import _posixshmem + +name = f"/rp_posixshmem_{os.getpid()}" +fd = _posixshmem.shm_open(name, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) +os.close(fd) +_posixshmem.shm_unlink(name) diff --git a/extra_tests/snippets/builtin_set.py b/extra_tests/snippets/builtin_set.py index 1b2f6ff0968..950875ea09a 100644 --- a/extra_tests/snippets/builtin_set.py +++ b/extra_tests/snippets/builtin_set.py @@ -200,6 +200,18 @@ class S(set): with assert_raises(TypeError): a &= [1, 2, 3] +a = set([1, 2, 3]) +a &= a +assert a == set([1, 2, 3]) + +a = set([1, 2, 3]) +a -= a +assert a == set() + +a = set([1, 2, 3]) +a ^= a +assert a == set() + a = set([1, 2, 3]) a.difference_update([3, 4, 5]) assert a == set([1, 2]) diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index abb68f812be..8cb0a09a215 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -72,6 +72,15 @@ assert object.__qualname__ == "object" assert int.__qualname__ == "int" +with assert_raises(TypeError): + type.__module__ = "nope" + +with assert_raises(TypeError): + object.__module__ = "nope" + +with assert_raises(TypeError): + map.__module__ = "nope" + class A(type): pass @@ -231,6 +240,56 @@ class C(B, BB): assert C.mro() == [C, B, A, BB, AA, object] +class TypeA: + def __init__(self): + self.a = 1 + + +class TypeB: + __slots__ = "b" + + def __init__(self): + self.b = 2 + + +obj = TypeA() +with assert_raises(TypeError) as cm: + obj.__class__ = TypeB +assert "__class__ assignment: 'TypeB' object layout differs from 'TypeA'" in str( + cm.exception +) + + +# Test: same slot count but different slot names should fail +class SlotX: + __slots__ = ("x",) + + +class SlotY: + __slots__ = ("y",) + + +slot_obj = SlotX() +with assert_raises(TypeError) as cm: + slot_obj.__class__ = SlotY +assert "__class__ assignment: 'SlotY' object layout differs from 'SlotX'" in str( + cm.exception +) + + +# Test: same slots should succeed +class SlotA: + __slots__ = ("a",) + + +class SlotA2: + __slots__ = ("a",) + + +slot_a = SlotA() +slot_a.__class__ = SlotA2 # Should work + + assert type(Exception.args).__name__ == "getset_descriptor" assert type(None).__bool__(None) is False @@ -525,6 +584,7 @@ def __new__(cls, *args, **kwargs): assert ClassWithNew().__new__.__qualname__ == "ClassWithNew.__new__" assert ClassWithNew.__new__.__name__ == "__new__" assert ClassWithNew().__new__.__name__ == "__new__" +assert isinstance(ClassWithNew.__dict__.get("__new__"), staticmethod) assert ClassWithNew.N.__new__.__qualname__ == "ClassWithNew.N.__new__" assert ClassWithNew().N.__new__.__qualname__ == "ClassWithNew.N.__new__" @@ -534,6 +594,7 @@ def __new__(cls, *args, **kwargs): assert ClassWithNew().N().__new__.__qualname__ == "ClassWithNew.N.__new__" assert ClassWithNew.N().__new__.__name__ == "__new__" assert ClassWithNew().N().__new__.__name__ == "__new__" +assert isinstance(ClassWithNew.N.__dict__.get("__new__"), staticmethod) # Regression to: diff --git a/extra_tests/snippets/builtins_module.py b/extra_tests/snippets/builtins_module.py index 6dea94d8d77..bf762425c89 100644 --- a/extra_tests/snippets/builtins_module.py +++ b/extra_tests/snippets/builtins_module.py @@ -22,6 +22,17 @@ exec("", namespace) assert namespace["__builtins__"] == __builtins__.__dict__ + +# function.__builtins__ should be a dict, not a module +# See: https://docs.python.org/3/reference/datamodel.html +def test_func(): + pass + + +assert isinstance(test_func.__builtins__, dict), ( + f"function.__builtins__ should be dict, got {type(test_func.__builtins__)}" +) + # with assert_raises(NameError): # exec('print(__builtins__)', {'__builtins__': {}}) diff --git a/extra_tests/snippets/code_co_consts.py b/extra_tests/snippets/code_co_consts.py index 58355652682..13f76a0d13e 100644 --- a/extra_tests/snippets/code_co_consts.py +++ b/extra_tests/snippets/code_co_consts.py @@ -1,39 +1,112 @@ +""" +Test co_consts behavior for Python 3.14+ + +In Python 3.14+: +- Functions with docstrings have the docstring as co_consts[0] +- CO_HAS_DOCSTRING flag (0x4000000) indicates docstring presence +- Functions without docstrings do NOT have None added as placeholder for docstring + +Note: Other constants (small integers, code objects, etc.) may still appear in co_consts +depending on optimization level. This test focuses on docstring behavior. +""" + + +# Test function with docstring - docstring should be co_consts[0] +def with_doc(): + """This is a docstring""" + return 1 + + +assert with_doc.__code__.co_consts[0] == "This is a docstring", ( + with_doc.__code__.co_consts +) +assert with_doc.__doc__ == "This is a docstring" +# Check CO_HAS_DOCSTRING flag (0x4000000) +assert with_doc.__code__.co_flags & 0x4000000, hex(with_doc.__code__.co_flags) + + +# Test function without docstring - should NOT have HAS_DOCSTRING flag +def no_doc(): + return 1 + + +assert not (no_doc.__code__.co_flags & 0x4000000), hex(no_doc.__code__.co_flags) +assert no_doc.__doc__ is None + + +# Test async function with docstring from asyncio import sleep -def f(): - def g(): - return 1 +async def async_with_doc(): + """Async docstring""" + await sleep(1) + return 1 - assert g.__code__.co_consts[0] == None - return 2 +assert async_with_doc.__code__.co_consts[0] == "Async docstring", ( + async_with_doc.__code__.co_consts +) +assert async_with_doc.__doc__ == "Async docstring" +assert async_with_doc.__code__.co_flags & 0x4000000 -assert f.__code__.co_consts[0] == None +# Test async function without docstring +async def async_no_doc(): + await sleep(1) + return 1 + + +assert not (async_no_doc.__code__.co_flags & 0x4000000) +assert async_no_doc.__doc__ is None -def generator(): + +# Test generator with docstring +def gen_with_doc(): + """Generator docstring""" yield 1 yield 2 -assert generator().gi_code.co_consts[0] == None +assert gen_with_doc.__code__.co_consts[0] == "Generator docstring" +assert gen_with_doc.__doc__ == "Generator docstring" +assert gen_with_doc.__code__.co_flags & 0x4000000 -async def async_f(): - await sleep(1) - return 1 +# Test generator without docstring +def gen_no_doc(): + yield 1 + yield 2 + +assert not (gen_no_doc.__code__.co_flags & 0x4000000) +assert gen_no_doc.__doc__ is None -assert async_f.__code__.co_consts[0] == None +# Test lambda - cannot have docstring lambda_f = lambda: 0 -assert lambda_f.__code__.co_consts[0] == None +assert not (lambda_f.__code__.co_flags & 0x4000000) +assert lambda_f.__doc__ is None + + +# Test class method with docstring +class cls_with_doc: + def method(): + """Method docstring""" + return 1 + +assert cls_with_doc.method.__code__.co_consts[0] == "Method docstring" +assert cls_with_doc.method.__doc__ == "Method docstring" -class cls: - def f(): + +# Test class method without docstring +class cls_no_doc: + def method(): return 1 -assert cls().f.__code__.co_consts[0] == None +assert not (cls_no_doc.method.__code__.co_flags & 0x4000000) +assert cls_no_doc.method.__doc__ is None + +print("All co_consts tests passed!") diff --git a/extra_tests/snippets/example_interactive.py b/extra_tests/snippets/example_interactive.py index f9484f15dcf..5958dd11707 100644 --- a/extra_tests/snippets/example_interactive.py +++ b/extra_tests/snippets/example_interactive.py @@ -4,7 +4,7 @@ def f(x, y, *args, power=1, **kwargs): - print("Constant String", 2, None, (2, 4)) + print("Constant String", 256, None, (2, 4)) assert code_class == type(c1) z = x * y return z**power @@ -19,7 +19,7 @@ def f(x, y, *args, power=1, **kwargs): # assert isinstance(c2.co_code, bytes) assert "Constant String" in c2.co_consts, c2.co_consts print(c2.co_consts) -assert 2 in c2.co_consts, c2.co_consts +assert 256 in c2.co_consts, c2.co_consts assert "example_interactive.py" in c2.co_filename assert c2.co_firstlineno == 6, str(c2.co_firstlineno) # assert isinstance(c2.co_flags, int) # 'OPTIMIZED, NEWLOCALS, NOFREE' diff --git a/extra_tests/snippets/sandbox_smoke.py b/extra_tests/snippets/sandbox_smoke.py new file mode 100644 index 00000000000..13fa0722742 --- /dev/null +++ b/extra_tests/snippets/sandbox_smoke.py @@ -0,0 +1,55 @@ +"""Sandbox mode smoke test. + +Verifies basic functionality that works in both sandbox and normal mode: +- stdio (print, sys.stdout/stdin/stderr) +- builtin modules (math, json) +- in-memory IO (BytesIO, StringIO) +- open() is properly blocked when FileIO is unavailable (sandbox) +""" + +import _io +import json +import math +import sys + +SANDBOX = not hasattr(_io, "FileIO") + +# stdio +print("1. print works") +assert sys.stdout.writable() +assert sys.stderr.writable() +assert sys.stdin.readable() +assert sys.stdout.fileno() == 1 + +# math +assert math.pi > 3.14 +print("2. math works:", math.pi) + +# json +d = json.loads('{"a": 1}') +assert d == {"a": 1} +print("3. json works:", d) + +# BytesIO / StringIO +buf = _io.BytesIO(b"hello") +assert buf.read() == b"hello" +sio = _io.StringIO("world") +assert sio.read() == "world" +print("4. BytesIO/StringIO work") + +# open() behavior depends on mode +if SANDBOX: + try: + open("/tmp/x", "w") + assert False, "should have raised" + except _io.UnsupportedOperation: + print("5. open() properly blocked (sandbox)") +else: + print("5. open() available (host_env)") + +# builtins +assert list(range(5)) == [0, 1, 2, 3, 4] +assert sorted([3, 1, 2]) == [1, 2, 3] +print("6. builtins work") + +print("All smoke tests passed!", "(sandbox)" if SANDBOX else "(host_env)") diff --git a/extra_tests/snippets/stdlib_array.py b/extra_tests/snippets/stdlib_array.py index 34eac949168..ed2a8f22369 100644 --- a/extra_tests/snippets/stdlib_array.py +++ b/extra_tests/snippets/stdlib_array.py @@ -126,3 +126,20 @@ def test_array_frombytes(): a = array("B", [0]) assert a.__contains__(0) assert not a.__contains__(1) + + +class _ReenteringWriter: + def __init__(self, arr): + self.arr = arr + self.reentered = False + + def write(self, chunk): + if not self.reentered: + self.reentered = True + self.arr.append(0) + return len(chunk) + + +arr = array("b", range(128)) +arr.tofile(_ReenteringWriter(arr)) +assert len(arr) == 129 diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 722886d34ee..7c473908295 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -10,15 +10,15 @@ result = bb.read() -assert len(result) <= 8 * 1024 +assert len(result) <= 16 * 1024 assert len(result) >= 0 assert isinstance(result, bytes) with FileIO("README.md") as fio: res = fio.read() - assert len(result) <= 8 * 1024 - assert len(result) >= 0 - assert isinstance(result, bytes) + assert len(res) <= 16 * 1024 + assert len(res) >= 0 + assert isinstance(res, bytes) fd = os.open("README.md", os.O_RDONLY) diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index 4d2e9f6e1f7..ce7a494713a 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -1,5 +1,4 @@ import itertools -import pickle from testutils import assert_raises @@ -181,10 +180,6 @@ # itertools.takewhile tests -def underten(x): - return x < 10 - - from itertools import takewhile as tw t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) @@ -226,70 +221,6 @@ def underten(x): with assert_raises(StopIteration): next(t) -it = tw(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert ( - pickle.dumps(it, 0) - == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -next(it) -next(it) -next(it) -try: - next(it) -except StopIteration: - pass -assert ( - pickle.dumps(it, 0) - == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - try: - next(pickle.loads(pickle.dumps(it, proto))) - assert False - except StopIteration: - pass - - # itertools.islice tests @@ -297,40 +228,28 @@ def assert_matches_seq(it, seq): assert list(it) == list(seq) -def test_islice_pickle(it): - for p in range(pickle.HIGHEST_PROTOCOL + 1): - it == pickle.loads(pickle.dumps(it, p)) - - i = itertools.islice it = i([1, 2, 3, 4, 5], 3) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2], None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, None) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], 1, None, None) assert_matches_seq(it, [2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], None, 2, None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, 3) assert_matches_seq(it, [1]) -test_islice_pickle(it) # itertools.filterfalse it = itertools.filterfalse(lambda x: x % 2, range(10)) @@ -359,59 +278,6 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(it) -it = itertools.dropwhile(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert ( - pickle.dumps(it, 0) - == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." -) -next(it) -assert ( - pickle.dumps(it, 0) - == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." -) -assert ( - pickle.dumps(it, 1) - == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 2) - == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 3) - == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." -) -assert ( - pickle.dumps(it, 4) - == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -assert ( - pickle.dumps(it, 5) - == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." -) -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - assert next(pickle.loads(pickle.dumps(it, proto))) == 2 - # itertools.accumulate it = itertools.accumulate([6, 3, 7, 1, 0, 9, 8, 8]) diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index a538365f707..d00924e10f2 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -518,3 +518,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): if option in ["PC_MAX_CANON", "PC_MAX_INPUT", "PC_VDISABLE"]: continue assert os.pathconf("/", index) == os.pathconf("/", option) + +# os.access - test with empty path and nonexistent files +assert os.access("", os.F_OK) is False +assert os.access("", os.R_OK) is False +assert os.access("", os.W_OK) is False +assert os.access("", os.X_OK) is False +assert os.access("nonexistent_file_12345", os.F_OK) is False +assert os.access("nonexistent_file_12345", os.W_OK) is False +assert os.access("README.md", os.F_OK) is True +assert os.access("README.md", os.R_OK) is True diff --git a/extra_tests/snippets/stdlib_socket.py b/extra_tests/snippets/stdlib_socket.py index b49fdcf08c2..3f56d2b926e 100644 --- a/extra_tests/snippets/stdlib_socket.py +++ b/extra_tests/snippets/stdlib_socket.py @@ -131,8 +131,9 @@ with assert_raises(OSError): socket.inet_aton("test") -with assert_raises(OverflowError): - socket.htonl(-1) +# TODO: RUSTPYTHON +# with assert_raises(ValueError): +# socket.htonl(-1) assert socket.htonl(0) == 0 assert socket.htonl(10) == 167772160 diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index 3a3872d2f4e..cdecf12dd2b 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -1,3 +1,5 @@ +import _ast +import platform import types from testutils import assert_raises @@ -8,3 +10,27 @@ assert ns.b == "Rust" with assert_raises(AttributeError): _ = ns.c + + +def _run_missing_type_params_regression(): + args = _ast.arguments( + posonlyargs=[], + args=[], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ) + pass_stmt = _ast.Pass(lineno=1, col_offset=4, end_lineno=1, end_col_offset=8) + fn = _ast.FunctionDef("f", args, [pass_stmt], [], None, None) + fn.lineno = 1 + fn.col_offset = 0 + fn.end_lineno = 1 + fn.end_col_offset = 8 + mod = _ast.Module([fn], []) + compiled = compile(mod, "<stdlib_types_missing_type_params>", "exec") + exec(compiled, {}) + + +_run_missing_type_params_regression() diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index ddc30b68460..07348945842 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -8,3 +8,30 @@ def abort_signal_handler( fn: Callable[[], Awaitable[T]], on_abort: Callable[[], None] | None = None ) -> T: pass + + +# Ensure PEP 604 unions work with typing.Callable aliases. +TracebackFilter = bool | Callable[[int], int] + + +# Test that Union/Optional in function parameter annotations work correctly. +# This tests that annotation scopes can access global implicit symbols (like Union) +# that are imported at module level but not explicitly bound in the function scope. +# Regression test for: rich +from typing import Optional, Union + + +def function_with_union_param(x: Optional[Union[int, str]] = None) -> None: + pass + + +class ClassWithUnionParams: + def __init__( + self, + color: Optional[Union[str, int]] = None, + bold: Optional[bool] = None, + ) -> None: + pass + + def method(self, value: Union[int, float]) -> Union[str, bytes]: + return str(value) diff --git a/extra_tests/snippets/syntax_assignment.py b/extra_tests/snippets/syntax_assignment.py index 8635dc5d795..851558a9db0 100644 --- a/extra_tests/snippets/syntax_assignment.py +++ b/extra_tests/snippets/syntax_assignment.py @@ -59,7 +59,18 @@ def g(): assert a == 1337 assert b == False -assert __annotations__['a'] == bool +# PEP 649: In Python 3.14, __annotations__ is not automatically defined at module level +# Accessing it raises NameError +from testutils import assert_raises + +with assert_raises(NameError): + __annotations__ + +# Use __annotate__ to get annotations (PEP 649) +assert callable(__annotate__) +annotations = __annotate__(1) # 1 = FORMAT_VALUE +assert annotations['a'] == bool +assert annotations['b'] == bool n = 0 diff --git a/extra_tests/snippets/syntax_forbidden_name.py b/extra_tests/snippets/syntax_forbidden_name.py index 2e114fe8800..3bd8148436e 100644 --- a/extra_tests/snippets/syntax_forbidden_name.py +++ b/extra_tests/snippets/syntax_forbidden_name.py @@ -21,6 +21,12 @@ def raisesSyntaxError(parse_stmt, exec_stmt=None): raisesSyntaxError("", "del __debug__") raisesSyntaxError("", "(a, __debug__, c) = (1, 2, 3)") raisesSyntaxError("", "(a, *__debug__, c) = (1, 2, 3)") +raisesSyntaxError("", "__debug__ : int") +raisesSyntaxError("", "__debug__ : int = 1") -# TODO: -# raisesSyntaxError("", "__debug__ : int") +# Import statements +raisesSyntaxError("import sys as __debug__") +raisesSyntaxError("from sys import path as __debug__") + +# Comprehension iteration targets +raisesSyntaxError("[x for __debug__ in range(5)]") diff --git a/extra_tests/snippets/syntax_function2.py b/extra_tests/snippets/syntax_function2.py index d0901af6a14..4a04acd51c1 100644 --- a/extra_tests/snippets/syntax_function2.py +++ b/extra_tests/snippets/syntax_function2.py @@ -80,6 +80,7 @@ def nested(): def f7(): + # PEP 649: annotations are deferred, so void is not evaluated at definition time try: def t() -> void: # noqa: F821 pass @@ -87,7 +88,7 @@ def t() -> void: # noqa: F821 return True return False -assert f7() +assert not f7() # PEP 649: no NameError because annotation is deferred def f8() -> int: diff --git a/extra_tests/snippets/syntax_short_circuit_bool.py b/extra_tests/snippets/syntax_short_circuit_bool.py index 76d89352cbb..6cbae190cae 100644 --- a/extra_tests/snippets/syntax_short_circuit_bool.py +++ b/extra_tests/snippets/syntax_short_circuit_bool.py @@ -31,3 +31,6 @@ def __bool__(self): # if ExplodingBool(False) and False and True and False: # pass + +# Issue #3567: nested BoolOps should not call __bool__ redundantly +assert (ExplodingBool(False) and False or False) == False diff --git a/scripts/check_redundant_patches.py b/scripts/check_redundant_patches.py new file mode 100644 index 00000000000..25cd2e1229e --- /dev/null +++ b/scripts/check_redundant_patches.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import ast +import pathlib +import sys + +ROOT = pathlib.Path(__file__).parents[1] +TEST_DIR = ROOT / "Lib" / "test" + + +def main(): + exit_status = 0 + for file in TEST_DIR.rglob("**/*.py"): + try: + contents = file.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + + try: + tree = ast.parse(contents) + except SyntaxError: + continue + + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + name = node.name + if not name.startswith("test"): + continue + + if node.decorator_list: + continue + + func_code = ast.unparse(node.body) + if func_code in ( + f"await super().{name}()", + f"return await super().{name}()", + f"return super().{name}()", + f"super().{name}()", + ): + exit_status += 1 + rel = file.relative_to(ROOT) + lineno = node.lineno + print( + f"{rel}:{name}:{lineno} is a test patch that can be safely removed", + file=sys.stderr, + ) + return exit_status + + +if __name__ == "__main__": + exit(main()) diff --git a/crawl_sourcecode.py b/scripts/crawl_sourcecode.py similarity index 100% rename from crawl_sourcecode.py rename to scripts/crawl_sourcecode.py diff --git a/scripts/fix_test.py b/scripts/fix_test.py deleted file mode 100644 index a5663e3eee3..00000000000 --- a/scripts/fix_test.py +++ /dev/null @@ -1,161 +0,0 @@ -""" -An automated script to mark failures in python test suite. -It adds @unittest.expectedFailure to the test functions that are failing in RustPython, but not in CPython. -As well as marking the test with a TODO comment. - -How to use: -1. Copy a specific test from the CPython repository to the RustPython repository. -2. Remove all unexpected failures from the test and skip the tests that hang -3. Run python ./scripts/fix_test.py --test test_venv --path ./Lib/test/test_venv.py or equivalent for the test from the project root. -4. Ensure that there are no unexpected successes in the test. -5. Actually fix the test. -""" - -import argparse -import ast -import itertools -import platform -from pathlib import Path - - -def parse_args(): - parser = argparse.ArgumentParser(description="Fix test.") - parser.add_argument("--path", type=Path, help="Path to test file") - parser.add_argument("--force", action="store_true", help="Force modification") - parser.add_argument( - "--platform", action="store_true", help="Platform specific failure" - ) - - args = parser.parse_args() - return args - - -class Test: - name: str = "" - path: str = "" - result: str = "" - - def __str__(self): - return f"Test(name={self.name}, path={self.path}, result={self.result})" - - -class TestResult: - tests_result: str = "" - tests = [] - stdout = "" - - def __str__(self): - return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})" - - -def parse_results(result): - lines = result.stdout.splitlines() - test_results = TestResult() - test_results.stdout = result.stdout - in_test_results = False - for line in lines: - if line == "Run tests sequentially": - in_test_results = True - elif line.startswith("-----------"): - in_test_results = False - if ( - in_test_results - and not line.startswith("tests") - and not line.startswith("[") - ): - line = line.split(" ") - if line != [] and len(line) > 3: - test = Test() - test.name = line[0] - test.path = line[1].strip("(").strip(")") - test.result = " ".join(line[3:]).lower() - test_results.tests.append(test) - else: - if "== Tests result: " in line: - res = line.split("== Tests result: ")[1] - res = res.split(" ")[0] - test_results.tests_result = res - return test_results - - -def path_to_test(path) -> list[str]: - return path.split(".")[2:] - - -def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: - a = ast.parse(file) - lines = file.splitlines() - fixture = "@unittest.expectedFailure" - for node in ast.walk(a): - if isinstance(node, ast.FunctionDef): - if node.name == test[-1]: - assert not for_platform - indent = " " * node.col_offset - lines.insert(node.lineno - 1, indent + fixture) - lines.insert(node.lineno - 1, indent + "# TODO: RUSTPYTHON") - break - return "\n".join(lines) - - -def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str: - a = ast.parse(file) - lines = file.splitlines() - fixture = "@unittest.expectedFailure" - for key, node in ast.iter_fields(a): - if key == "body": - for i, n in enumerate(node): - match n: - case ast.ClassDef(): - if len(test) == 2 and test[0] == n.name: - # look through body for function def - for i, fn in enumerate(n.body): - match fn: - case ast.FunctionDef(): - if fn.name == test[-1]: - assert not for_platform - indent = " " * fn.col_offset - lines.insert( - fn.lineno - 1, indent + fixture - ) - lines.insert( - fn.lineno - 1, - indent + "# TODO: RUSTPYTHON", - ) - break - case ast.FunctionDef(): - if n.name == test[0] and len(test) == 1: - assert not for_platform - indent = " " * n.col_offset - lines.insert(n.lineno - 1, indent + fixture) - lines.insert(n.lineno - 1, indent + "# TODO: RUSTPYTHON") - break - if i > 500: - exit() - return "\n".join(lines) - - -def run_test(test_name): - print(f"Running test: {test_name}") - rustpython_location = "./target/release/rustpython" - import subprocess - - result = subprocess.run( - [rustpython_location, "-m", "test", "-v", test_name], - capture_output=True, - text=True, - ) - return parse_results(result) - - -if __name__ == "__main__": - args = parse_args() - test_name = args.path.stem - tests = run_test(test_name) - f = open(args.path).read() - for test in tests.tests: - if test.result == "fail" or test.result == "error": - print("Modifying test:", test.name) - f = modify_test_v2(f, path_to_test(test.path), args.platform) - with open(args.path, "w") as file: - # TODO: Find validation method, and make --force override it - file.write(f) diff --git a/scripts/generate_opcode_metadata.py b/scripts/generate_opcode_metadata.py new file mode 100644 index 00000000000..42fb55a7c01 --- /dev/null +++ b/scripts/generate_opcode_metadata.py @@ -0,0 +1,81 @@ +""" +Generate Lib/_opcode_metadata.py for RustPython bytecode. + +This file generates opcode metadata that is compatible with CPython 3.13. +""" + +import itertools +import pathlib +import re +import typing + +ROOT = pathlib.Path(__file__).parents[1] +BYTECODE_FILE = ( + ROOT / "crates" / "compiler-core" / "src" / "bytecode" / "instruction.rs" +) +OPCODE_METADATA_FILE = ROOT / "Lib" / "_opcode_metadata.py" + + +class Opcode(typing.NamedTuple): + rust_name: str + id: int + + @property + def cpython_name(self) -> str: + name = re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", self.rust_name) + return re.sub(r"(\D)(\d+)$", r"\1_\2", name).upper() + + @classmethod + def from_str(cls, body: str): + raw_variants = re.split(r"(\d+),", body.strip()) + raw_variants.remove("") + for raw_name, raw_id in itertools.batched(raw_variants, 2): + name = re.findall(r"\b[A-Z][A-Za-z]*\d*\b(?=\s*[\({=])", raw_name)[0] + yield cls(rust_name=name.strip(), id=int(raw_id)) + + def __lt__(self, other: typing.Self) -> bool: + return self.id < other.id + + +def extract_enum_body(contents: str, enum_name: str) -> str: + res = re.search(f"pub enum {enum_name} " + r"\{(.+?)\n\}", contents, re.DOTALL) + if not res: + raise ValueError(f"Could not find {enum_name} enum") + + return "\n".join( + line.split("//")[0].strip() # Remove any comment. i.e. "foo // some comment" + for line in res.group(1).splitlines() + if not line.strip().startswith("//") # Ignore comment lines + ) + + +contents = BYTECODE_FILE.read_text(encoding="utf-8") +enum_body = "\n".join( + extract_enum_body(contents, enum_name) + for enum_name in ("Instruction", "PseudoInstruction") +) +opcodes = list(Opcode.from_str(enum_body)) + +# Generate the output file +output = """# This file is generated by scripts/generate_opcode_metadata.py +# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). +# Do not edit! + +_specializations = {} + +_specialized_opmap = {} + +opmap = { +""" + +for opcode in sorted(opcodes): + output += f" '{opcode.cpython_name}': {opcode.id},\n" + +output += """} + +# CPython 3.13 compatible: opcodes < 44 have no argument +HAVE_ARGUMENT = 44 +MIN_INSTRUMENTED_OPCODE = 236 +""" + +OPCODE_METADATA_FILE.write_text(output, encoding="utf-8") diff --git a/scripts/generate_sre_constants.py b/scripts/generate_sre_constants.py new file mode 100644 index 00000000000..8e4091d2eb9 --- /dev/null +++ b/scripts/generate_sre_constants.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python3 +# This script generates crates/sre_engine/src/constants.rs from Lib/re/_constants.py. + +SCRIPT_NAME = "scripts/generate_sre_constants.py" + + +def update_file(file, content): + try: + with open(file, "r") as fobj: + if fobj.read() == content: + return False + except (OSError, ValueError): + pass + with open(file, "w") as fobj: + fobj.write(content) + return True + + +sre_constants_header = f"""\ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * Auto-generated by {SCRIPT_NAME} from + * Lib/re/_constants.py. + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the sre.c file for information on usage and redistribution. + */ + +""" + + +def dump_enum(d, enum_name, derives, strip_prefix=""): + """Generate Rust enum definitions from a Python dictionary. + + Args: + d (list): The list containing the enum variants. + enum_name (str): The name of the enum to generate. + derives (str): The derive attributes to include. + strip_prefix (str, optional): A prefix to strip from the variant names. Defaults to "". + + Returns: + list: A list of strings representing the enum definition. + """ + items = sorted(d) + print(f"items is {items}") + content = [f"{derives}\n"] + content.append("#[repr(u32)]\n") + content.append("#[allow(non_camel_case_types, clippy::upper_case_acronyms)]\n") + content.append(f"pub enum {enum_name} {{\n") + for i, item in enumerate(items): + name = str(item).removeprefix(strip_prefix) + content.append(f" {name} = {i},\n") + content.append("}\n\n") + return content + + +def dump_bitflags(d, prefix, derives, struct_name, int_t): + """Generate Rust bitflags definitions from a Python dictionary. + + Args: + d (dict): The dictionary containing the bitflag variants. + prefix (str): The prefix to strip from the variant names. + derives (str): The derive attributes to include. + struct_name (str): The name of the struct to generate. + int_t (str): The integer type to use for the bitflags. + + Returns: + list: A list of strings representing the bitflags definition. + """ + items = [(value, name) for name, value in d.items() if name.startswith(prefix)] + content = ["bitflags! {\n"] + content.append(f"{derives}\n") if derives else None + content.append(f" pub struct {struct_name}: {int_t} {{\n") + for value, name in sorted(items): + name = str(name).removeprefix(prefix) + content.append(f" const {name} = {value};\n") + content.append(" }\n") + content.append("}\n\n") + return content + + +def main( + infile="Lib/re/_constants.py", + outfile_constants="crates/sre_engine/src/constants.rs", +): + ns = {} + with open(infile) as fp: + code = fp.read() + exec(code, ns) + + content = [sre_constants_header] + content.append("use bitflags::bitflags;\n\n") + content.append(f"pub const SRE_MAGIC: usize = {ns['MAGIC']};\n") + content.extend( + dump_enum( + ns["OPCODES"], + "SreOpcode", + "#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]", + ) + ) + content.extend( + dump_enum( + ns["ATCODES"], + "SreAtCode", + "#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]", + "AT_", + ) + ) + content.extend( + dump_enum( + ns["CHCODES"], + "SreCatCode", + "#[derive(num_enum::TryFromPrimitive, Debug)]", + "CATEGORY_", + ) + ) + + content.extend( + dump_bitflags( + ns, + "SRE_FLAG_", + "#[derive(Debug, PartialEq, Eq, Clone, Copy)]", + "SreFlag", + "u16", + ) + ) + content.extend(dump_bitflags(ns, "SRE_INFO_", "", "SreInfo", "u32")) + + update_file(outfile_constants, "".join(content)) + + +if __name__ == "__main__": + import sys + + main(*sys.argv[1:]) diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index 73f082a0658..be375aebe8f 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -13,7 +13,7 @@ ) # TODO: Exclude matches if they have `(` after (those are functions) -LIBC_VERSION = "0.2.177" +LIBC_VERSION = "0.2.180" EXCLUDE = frozenset( { @@ -96,7 +96,7 @@ def format_groups(groups: dict) -> "Iterator[tuple[str, str]]": def main(): wanted_consts = get_consts( - "https://docs.python.org/3.13/library/os.html", # Should we read from https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Modules/posixmodule.c#L17023 instead? + "https://docs.python.org/3.14/library/os.html", # Should we read from https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Modules/posixmodule.c#L17023 instead? pattern=OS_CONSTS_PAT, ) available = { diff --git a/scripts/update_lib/.gitignore b/scripts/update_lib/.gitignore new file mode 100644 index 00000000000..ceddaa37f12 --- /dev/null +++ b/scripts/update_lib/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/scripts/update_lib/__init__.py b/scripts/update_lib/__init__.py new file mode 100644 index 00000000000..ccb2628d6a4 --- /dev/null +++ b/scripts/update_lib/__init__.py @@ -0,0 +1,37 @@ +""" +Library for updating Python test files with RustPython-specific patches. +""" + +from .patch_spec import ( + COMMENT, + DEFAULT_INDENT, + UT, + PatchEntry, + Patches, + PatchSpec, + UtMethod, + apply_patches, + build_patch_dict, + extract_patches, + iter_patches, + iter_tests, + patches_from_json, + patches_to_json, +) + +__all__ = [ + "COMMENT", + "DEFAULT_INDENT", + "UT", + "Patches", + "PatchEntry", + "PatchSpec", + "UtMethod", + "apply_patches", + "build_patch_dict", + "extract_patches", + "iter_patches", + "iter_tests", + "patches_from_json", + "patches_to_json", +] diff --git a/scripts/update_lib/__main__.py b/scripts/update_lib/__main__.py new file mode 100644 index 00000000000..49399db6f43 --- /dev/null +++ b/scripts/update_lib/__main__.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +""" +Update library tools for RustPython. + +Usage: + python scripts/update_lib quick cpython/Lib/test/test_foo.py + python scripts/update_lib copy-lib cpython/Lib/dataclasses.py + python scripts/update_lib migrate cpython/Lib/test/test_foo.py + python scripts/update_lib patches --from Lib/test/foo.py --to cpython/Lib/test/foo.py + python scripts/update_lib auto-mark Lib/test/test_foo.py +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Update library tools for RustPython", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser( + "quick", + help="Quick update: patch + auto-mark (recommended)", + add_help=False, + ) + subparsers.add_parser( + "migrate", + help="Migrate test file(s) from CPython, preserving RustPython markers", + add_help=False, + ) + subparsers.add_parser( + "patches", + help="Patch management (extract/apply patches between files)", + add_help=False, + ) + subparsers.add_parser( + "auto-mark", + help="Run tests and auto-mark failures with @expectedFailure", + add_help=False, + ) + subparsers.add_parser( + "copy-lib", + help="Copy library file/directory from CPython (delete existing first)", + add_help=False, + ) + subparsers.add_parser( + "deps", + help="Show dependency information for a module", + add_help=False, + ) + subparsers.add_parser( + "todo", + help="Show prioritized list of modules to update", + add_help=False, + ) + + args, remaining = parser.parse_known_args(argv) + + if args.command == "quick": + from update_lib.cmd_quick import main as quick_main + + return quick_main(remaining) + + if args.command == "copy-lib": + from update_lib.cmd_copy_lib import main as copy_lib_main + + return copy_lib_main(remaining) + + if args.command == "migrate": + from update_lib.cmd_migrate import main as migrate_main + + return migrate_main(remaining) + + if args.command == "patches": + from update_lib.cmd_patches import main as patches_main + + return patches_main(remaining) + + if args.command == "auto-mark": + from update_lib.cmd_auto_mark import main as cmd_auto_mark_main + + return cmd_auto_mark_main(remaining) + + if args.command == "deps": + from update_lib.cmd_deps import main as cmd_deps_main + + return cmd_deps_main(remaining) + + if args.command == "todo": + from update_lib.cmd_todo import main as cmd_todo_main + + return cmd_todo_main(remaining) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_auto_mark.py b/scripts/update_lib/cmd_auto_mark.py new file mode 100644 index 00000000000..c77cbf300f1 --- /dev/null +++ b/scripts/update_lib/cmd_auto_mark.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env python +""" +Auto-mark test failures in Python test suite. + +This module provides functions to: +- Run tests with RustPython and parse results +- Extract test names from test file paths +- Mark failing tests with @unittest.expectedFailure +- Remove expectedFailure from tests that now pass +""" + +import ast +import pathlib +import re +import subprocess +import sys +from dataclasses import dataclass, field + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib import COMMENT, PatchSpec, UtMethod, apply_patches +from update_lib.file_utils import get_test_module_name + + +class TestRunError(Exception): + """Raised when test run fails entirely (e.g., import error, crash).""" + + pass + + +@dataclass +class Test: + name: str = "" + path: str = "" + result: str = "" + error_message: str = "" + + +@dataclass +class TestResult: + tests_result: str = "" + tests: list[Test] = field(default_factory=list) + unexpected_successes: list[Test] = field(default_factory=list) + stdout: str = "" + + +def run_test(test_name: str, skip_build: bool = False) -> TestResult: + """ + Run a test with RustPython and return parsed results. + + Args: + test_name: Test module name (e.g., "test_foo" or "test_ctypes.test_bar") + skip_build: If True, use pre-built binary instead of cargo run + + Returns: + TestResult with parsed test results + """ + if skip_build: + cmd = ["./target/release/rustpython"] + if sys.platform == "win32": + cmd = ["./target/release/rustpython.exe"] + else: + cmd = ["cargo", "run", "--release", "--"] + + result = subprocess.run( + cmd + ["-m", "test", "-v", "-u", "all", "--slowest", test_name], + stdout=subprocess.PIPE, # Capture stdout for parsing + stderr=None, # Let stderr pass through to terminal + text=True, + ) + return parse_results(result) + + +def _try_parse_test_info(test_info: str) -> tuple[str, str] | None: + """Try to extract (name, path) from 'test_name (path)' or 'test_name (path) [subtest]'.""" + first_space = test_info.find(" ") + if first_space > 0: + name = test_info[:first_space] + rest = test_info[first_space:].strip() + if rest.startswith("("): + end_paren = rest.find(")") + if end_paren > 0: + return name, rest[1:end_paren] + return None + + +def parse_results(result: subprocess.CompletedProcess) -> TestResult: + """Parse subprocess result into TestResult.""" + lines = result.stdout.splitlines() + test_results = TestResult() + test_results.stdout = result.stdout + in_test_results = False + # For multiline format: "test_name (path)\ndocstring ... RESULT" + pending_test_info = None + + for line in lines: + if re.search(r"Run \d+ tests? sequentially", line): + in_test_results = True + elif "== Tests result: " in line: + in_test_results = False + + if in_test_results and " ... " in line: + stripped = line.strip() + # Skip lines that don't look like test results + if stripped.startswith("tests") or stripped.startswith("["): + pending_test_info = None + continue + # Parse: "test_name (path) [subtest] ... RESULT" + parts = stripped.split(" ... ") + if len(parts) >= 2: + test_info = parts[0] + result_str = parts[-1].lower() + # Only process FAIL or ERROR + if result_str not in ("fail", "error"): + pending_test_info = None + continue + # Try parsing from this line (single-line format) + parsed = _try_parse_test_info(test_info) + if not parsed and pending_test_info: + # Multiline format: previous line had test_name (path) + parsed = _try_parse_test_info(pending_test_info) + if parsed: + test = Test() + test.name, test.path = parsed + test.result = result_str + test_results.tests.append(test) + pending_test_info = None + + elif in_test_results: + # Track test info for multiline format: + # test_name (path) + # docstring ... RESULT + stripped = line.strip() + if ( + stripped + and "(" in stripped + and stripped.endswith(")") + and ":" not in stripped.split("(")[0] + ): + pending_test_info = stripped + else: + pending_test_info = None + + # Also check for Tests result on non-" ... " lines + if "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + + elif "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + + # Parse: "UNEXPECTED SUCCESS: test_name (path)" + if line.startswith("UNEXPECTED SUCCESS: "): + rest = line[len("UNEXPECTED SUCCESS: ") :] + # Format: "test_name (path)" + first_space = rest.find(" ") + if first_space > 0: + test = Test() + test.name = rest[:first_space] + path_part = rest[first_space:].strip() + if path_part.startswith("(") and path_part.endswith(")"): + test.path = path_part[1:-1] + test.result = "unexpected_success" + test_results.unexpected_successes.append(test) + + # Parse error details to extract error messages + _parse_error_details(test_results, lines) + + return test_results + + +def _parse_error_details(test_results: TestResult, lines: list[str]) -> None: + """Parse error details section to extract error messages for each test.""" + # Build a lookup dict for tests by (name, path) + test_lookup: dict[tuple[str, str], Test] = {} + for test in test_results.tests: + test_lookup[(test.name, test.path)] = test + + # Parse error detail blocks + # Format: + # ====================================================================== + # FAIL: test_name (path) + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # ... + # AssertionError: message + # + # ====================================================================== + i = 0 + while i < len(lines): + line = lines[i] + # Look for FAIL: or ERROR: header + if line.startswith(("FAIL: ", "ERROR: ")): + # Parse: "FAIL: test_name (path)" or "ERROR: test_name (path)" + header = line.split(": ", 1)[1] if ": " in line else "" + first_space = header.find(" ") + if first_space > 0: + test_name = header[:first_space] + path_part = header[first_space:].strip() + if path_part.startswith("(") and path_part.endswith(")"): + test_path = path_part[1:-1] + + # Find the last non-empty line before the next separator or end + error_lines = [] + i += 1 + # Skip the separator line + if i < len(lines) and lines[i].startswith("-----"): + i += 1 + + # Collect lines until the next separator or end + while i < len(lines): + current = lines[i] + if current.startswith("=====") or current.startswith("-----"): + break + error_lines.append(current) + i += 1 + + # Find the last non-empty line (the error message) + error_message = "" + for err_line in reversed(error_lines): + stripped = err_line.strip() + if stripped: + error_message = stripped + break + + # Update the test with the error message + if (test_name, test_path) in test_lookup: + test_lookup[ + (test_name, test_path) + ].error_message = error_message + + continue + i += 1 + + +def path_to_test_parts(path: str) -> list[str]: + """ + Extract [ClassName, method_name] from test path. + + Args: + path: Test path like "test.module_name.ClassName.test_method" + + Returns: + [ClassName, method_name] - last 2 elements + """ + parts = path.split(".") + return parts[-2:] + + +def _expand_stripped_to_children( + contents: str, + stripped_tests: set[tuple[str, str]], + all_failing_tests: set[tuple[str, str]], +) -> set[tuple[str, str]]: + """Find child-class failures that correspond to stripped parent-class markers. + + When ``strip_reasonless_expected_failures`` removes a marker from a parent + (mixin) class, test failures are reported against the concrete subclasses, + not the parent itself. This function maps those child failures back so + they get re-marked (and later consolidated to the parent by + ``_consolidate_to_parent``). + + Returns the set of ``(class, method)`` pairs from *all_failing_tests* that + should be re-marked. + """ + # Direct matches (stripped test itself is a concrete TestCase) + result = stripped_tests & all_failing_tests + + unmatched = stripped_tests - all_failing_tests + if not unmatched: + return result + + tree = ast.parse(contents) + class_bases, class_methods = _build_inheritance_info(tree) + + for parent_cls, method_name in unmatched: + if method_name not in class_methods.get(parent_cls, set()): + continue + for cls in _find_all_inheritors( + parent_cls, method_name, class_bases, class_methods + ): + if (cls, method_name) in all_failing_tests: + result.add((cls, method_name)) + + return result + + +def _consolidate_to_parent( + contents: str, + failing_tests: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> tuple[set[tuple[str, str]], dict[tuple[str, str], str] | None]: + """Move failures to the parent class when ALL inheritors fail. + + If every concrete subclass that inherits a method from a parent class + appears in *failing_tests*, replace those per-subclass entries with a + single entry on the parent. This avoids creating redundant super-call + overrides in every child. + + Returns: + (consolidated_failing_tests, consolidated_error_messages) + """ + tree = ast.parse(contents) + class_bases, class_methods = _build_inheritance_info(tree) + + # Group by (defining_parent, method) → set of failing children + from collections import defaultdict + + groups: dict[tuple[str, str], set[str]] = defaultdict(set) + for class_name, method_name in failing_tests: + defining = _find_method_definition( + class_name, method_name, class_bases, class_methods + ) + if defining and defining != class_name: + groups[(defining, method_name)].add(class_name) + + if not groups: + return failing_tests, error_messages + + result = set(failing_tests) + new_error_messages = dict(error_messages) if error_messages else {} + + for (parent, method_name), failing_children in groups.items(): + all_inheritors = _find_all_inheritors( + parent, method_name, class_bases, class_methods + ) + + if all_inheritors and failing_children >= all_inheritors: + # All inheritors fail → mark on parent instead + children_keys = {(child, method_name) for child in failing_children} + result -= children_keys + result.add((parent, method_name)) + # Pick any child's error message for the parent + if new_error_messages: + for child in failing_children: + msg = new_error_messages.pop((child, method_name), "") + if msg: + new_error_messages[(parent, method_name)] = msg + + return result, new_error_messages or error_messages + + +def build_patches( + test_parts_set: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> dict: + """Convert failing tests to patch format.""" + patches = {} + error_messages = error_messages or {} + for class_name, method_name in sorted(test_parts_set): + if class_name not in patches: + patches[class_name] = {} + reason = error_messages.get((class_name, method_name), "") + patches[class_name][method_name] = [ + PatchSpec(UtMethod.ExpectedFailure, None, reason) + ] + return patches + + +def _is_super_call_only(func_node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + """Check if the method body is just 'return super().method_name()' or 'return await super().method_name()'.""" + if len(func_node.body) != 1: + return False + stmt = func_node.body[0] + if not isinstance(stmt, ast.Return) or stmt.value is None: + return False + call = stmt.value + # Unwrap await for async methods + if isinstance(call, ast.Await): + call = call.value + if not isinstance(call, ast.Call): + return False + if not isinstance(call.func, ast.Attribute): + return False + # Verify the method name matches + if call.func.attr != func_node.name: + return False + super_call = call.func.value + if not isinstance(super_call, ast.Call): + return False + if not isinstance(super_call.func, ast.Name) or super_call.func.id != "super": + return False + return True + + +def _method_removal_range( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, lines: list[str] +) -> range: + """Line range covering an entire method including decorators and a preceding COMMENT line.""" + first = ( + func_node.decorator_list[0].lineno - 1 + if func_node.decorator_list + else func_node.lineno - 1 + ) + if ( + first > 0 + and lines[first - 1].strip().startswith("#") + and COMMENT in lines[first - 1] + ): + first -= 1 + # Also remove a preceding blank line to avoid double-blanks after removal + if first > 0 and not lines[first - 1].strip(): + first -= 1 + return range(first, func_node.end_lineno) + + +def _build_inheritance_info(tree: ast.Module) -> tuple[dict, dict]: + """ + Build inheritance information from AST. + + Returns: + class_bases: dict[str, list[str]] - parent classes for each class + class_methods: dict[str, set[str]] - methods directly defined in each class + """ + all_classes = { + node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef) + } + class_bases = {} + class_methods = {} + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + bases = [ + base.id + for base in node.bases + if isinstance(base, ast.Name) and base.id in all_classes + ] + class_bases[node.name] = bases + methods = { + item.name + for item in node.body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + class_methods[node.name] = methods + + return class_bases, class_methods + + +def _find_method_definition( + class_name: str, method_name: str, class_bases: dict, class_methods: dict +) -> str | None: + """Find the class where a method is actually defined (BFS).""" + if method_name in class_methods.get(class_name, set()): + return class_name + + visited = set() + queue = list(class_bases.get(class_name, [])) + + while queue: + current = queue.pop(0) + if current in visited: + continue + visited.add(current) + + if method_name in class_methods.get(current, set()): + return current + queue.extend(class_bases.get(current, [])) + + return None + + +def _find_all_inheritors( + parent: str, method_name: str, class_bases: dict, class_methods: dict +) -> set[str]: + """Find all classes that inherit *method_name* from *parent* (not overriding it).""" + return { + cls + for cls in class_bases + if cls != parent + and method_name not in class_methods.get(cls, set()) + and _find_method_definition(cls, method_name, class_bases, class_methods) + == parent + } + + +def remove_expected_failures( + contents: str, tests_to_remove: set[tuple[str, str]] +) -> str: + """Remove @unittest.expectedFailure decorators from tests that now pass.""" + if not tests_to_remove: + return contents + + tree = ast.parse(contents) + lines = contents.splitlines() + lines_to_remove = set() + + class_bases, class_methods = _build_inheritance_info(tree) + + resolved_tests = set() + for class_name, method_name in tests_to_remove: + defining_class = _find_method_definition( + class_name, method_name, class_bases, class_methods + ) + if defining_class: + resolved_tests.add((defining_class, method_name)) + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + class_name = node.name + for item in node.body: + if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + method_name = item.name + if (class_name, method_name) not in resolved_tests: + continue + + remove_entire_method = _is_super_call_only(item) + + if remove_entire_method: + lines_to_remove.update(_method_removal_range(item, lines)) + else: + for dec in item.decorator_list: + dec_line = dec.lineno - 1 + line_content = lines[dec_line] + + if "expectedFailure" not in line_content: + continue + + has_comment_on_line = COMMENT in line_content + has_comment_before = ( + dec_line > 0 + and lines[dec_line - 1].strip().startswith("#") + and COMMENT in lines[dec_line - 1] + ) + has_comment_after = ( + dec_line + 1 < len(lines) + and lines[dec_line + 1].strip().startswith("#") + and COMMENT not in lines[dec_line + 1] + ) + + if has_comment_on_line or has_comment_before: + lines_to_remove.add(dec_line) + if has_comment_before: + lines_to_remove.add(dec_line - 1) + if has_comment_after and has_comment_on_line: + lines_to_remove.add(dec_line + 1) + + for line_idx in sorted(lines_to_remove, reverse=True): + del lines[line_idx] + + return "\n".join(lines) + "\n" if lines else "" + + +def collect_test_changes( + results: TestResult, + module_prefix: str | None = None, +) -> tuple[set[tuple[str, str]], set[tuple[str, str]], dict[tuple[str, str], str]]: + """ + Collect failing tests and unexpected successes from test results. + + Args: + results: TestResult from run_test() + module_prefix: If set, only collect tests whose path starts with this prefix + + Returns: + (failing_tests, unexpected_successes, error_messages) + - failing_tests: set of (class_name, method_name) tuples + - unexpected_successes: set of (class_name, method_name) tuples + - error_messages: dict mapping (class_name, method_name) to error message + """ + failing_tests = set() + error_messages: dict[tuple[str, str], str] = {} + for test in results.tests: + if test.result in ("fail", "error"): + if module_prefix and not test.path.startswith(module_prefix): + continue + test_parts = path_to_test_parts(test.path) + if len(test_parts) == 2: + key = tuple(test_parts) + failing_tests.add(key) + if test.error_message: + error_messages[key] = test.error_message + + unexpected_successes = set() + for test in results.unexpected_successes: + if module_prefix and not test.path.startswith(module_prefix): + continue + test_parts = path_to_test_parts(test.path) + if len(test_parts) == 2: + unexpected_successes.add(tuple(test_parts)) + + return failing_tests, unexpected_successes, error_messages + + +def apply_test_changes( + contents: str, + failing_tests: set[tuple[str, str]], + unexpected_successes: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> str: + """ + Apply test changes to content. + + Args: + contents: File content + failing_tests: Set of (class_name, method_name) to mark as expectedFailure + unexpected_successes: Set of (class_name, method_name) to remove expectedFailure + error_messages: Dict mapping (class_name, method_name) to error message + + Returns: + Modified content + """ + if unexpected_successes: + contents = remove_expected_failures(contents, unexpected_successes) + + if failing_tests: + failing_tests, error_messages = _consolidate_to_parent( + contents, failing_tests, error_messages + ) + patches = build_patches(failing_tests, error_messages) + contents = apply_patches(contents, patches) + + return contents + + +def strip_reasonless_expected_failures( + contents: str, +) -> tuple[str, set[tuple[str, str]]]: + """Strip @expectedFailure decorators that have no failure reason. + + Markers like ``@unittest.expectedFailure # TODO: RUSTPYTHON`` (without a + reason after the semicolon) are removed so the tests fail normally during + the next test run and error messages can be captured. + + Returns: + (modified_contents, stripped_tests) where stripped_tests is a set of + (class_name, method_name) tuples whose markers were removed. + """ + tree = ast.parse(contents) + lines = contents.splitlines() + stripped_tests: set[tuple[str, str]] = set() + lines_to_remove: set[int] = set() + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for item in node.body: + if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for dec in item.decorator_list: + dec_line = dec.lineno - 1 + line_content = lines[dec_line] + + if "expectedFailure" not in line_content: + continue + + has_comment_on_line = COMMENT in line_content + has_comment_before = ( + dec_line > 0 + and lines[dec_line - 1].strip().startswith("#") + and COMMENT in lines[dec_line - 1] + ) + + if not has_comment_on_line and not has_comment_before: + continue # not our marker + + # Check if there's a reason (on either the decorator or before) + for check_line in ( + line_content, + lines[dec_line - 1] if has_comment_before else "", + ): + match = re.search(rf"{COMMENT}(.*)", check_line) + if match and match.group(1).strip(";:, "): + break # has a reason, keep it + else: + # No reason found — strip this decorator + stripped_tests.add((node.name, item.name)) + + if _is_super_call_only(item): + # Remove entire super-call override (the method + # exists only to apply the decorator; without it + # the override is pointless and blocks parent + # consolidation) + lines_to_remove.update(_method_removal_range(item, lines)) + else: + lines_to_remove.add(dec_line) + + if has_comment_before: + lines_to_remove.add(dec_line - 1) + + # Also remove a reason-comment on the line after (old format) + if ( + has_comment_on_line + and dec_line + 1 < len(lines) + and lines[dec_line + 1].strip().startswith("#") + and COMMENT not in lines[dec_line + 1] + ): + lines_to_remove.add(dec_line + 1) + + if not lines_to_remove: + return contents, stripped_tests + + for idx in sorted(lines_to_remove, reverse=True): + del lines[idx] + + return "\n".join(lines) + "\n" if lines else "", stripped_tests + + +def extract_test_methods(contents: str) -> set[tuple[str, str]]: + """ + Extract all test method names from file contents. + + Returns: + Set of (class_name, method_name) tuples + """ + from update_lib.file_utils import safe_parse_ast + from update_lib.patch_spec import iter_tests + + tree = safe_parse_ast(contents) + if tree is None: + return set() + + return {(cls_node.name, fn_node.name) for cls_node, fn_node in iter_tests(tree)} + + +def auto_mark_file( + test_path: pathlib.Path, + mark_failure: bool = False, + verbose: bool = True, + original_methods: set[tuple[str, str]] | None = None, + skip_build: bool = False, +) -> tuple[int, int, int]: + """ + Run tests and auto-mark failures in a test file. + + Args: + test_path: Path to the test file + mark_failure: If True, add @expectedFailure to ALL failing tests + verbose: Print progress messages + original_methods: If provided, only auto-mark failures for NEW methods + (methods not in original_methods) even without mark_failure. + Failures in existing methods are treated as regressions. + + Returns: + (num_failures_added, num_successes_removed, num_regressions) + """ + test_path = pathlib.Path(test_path).resolve() + if not test_path.exists(): + raise FileNotFoundError(f"File not found: {test_path}") + + # Strip reason-less markers so those tests fail normally and we capture + # their error messages during the test run. + contents = test_path.read_text(encoding="utf-8") + original_contents = contents + contents, stripped_tests = strip_reasonless_expected_failures(contents) + if stripped_tests: + test_path.write_text(contents, encoding="utf-8") + + test_name = get_test_module_name(test_path) + if verbose: + print(f"Running test: {test_name}") + + results = run_test(test_name, skip_build=skip_build) + + # Check if test run failed entirely (e.g., import error, crash) + if ( + not results.tests_result + and not results.tests + and not results.unexpected_successes + ): + # Restore original contents before raising + if stripped_tests: + test_path.write_text(original_contents, encoding="utf-8") + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + + # If the run crashed (incomplete), restore original file so that markers + # for tests that never ran are preserved. Only observed results will be + # re-applied below. + if not results.tests_result and stripped_tests: + test_path.write_text(original_contents, encoding="utf-8") + stripped_tests = set() + + contents = test_path.read_text(encoding="utf-8") + + all_failing_tests, unexpected_successes, error_messages = collect_test_changes( + results + ) + + # Determine which failures to mark + if mark_failure: + failing_tests = all_failing_tests + elif original_methods is not None: + # Smart mode: only mark NEW test failures (not regressions) + current_methods = extract_test_methods(contents) + new_methods = current_methods - original_methods + failing_tests = {t for t in all_failing_tests if t in new_methods} + else: + failing_tests = set() + + # Re-mark stripped tests that still fail (to restore markers with reasons). + # Uses inheritance expansion: if a parent marker was stripped, child + # failures are included so _consolidate_to_parent can re-mark the parent. + failing_tests |= _expand_stripped_to_children( + contents, stripped_tests, all_failing_tests + ) + + regressions = all_failing_tests - failing_tests + + if verbose: + for class_name, method_name in failing_tests: + label = "(new test)" if original_methods is not None else "" + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print( + f"Marking as failing {label}: {class_name}.{method_name}{err_hint}".replace( + " ", " " + ) + ) + for class_name, method_name in unexpected_successes: + print(f"Removing expectedFailure: {class_name}.{method_name}") + + contents = apply_test_changes( + contents, failing_tests, unexpected_successes, error_messages + ) + + if failing_tests or unexpected_successes: + test_path.write_text(contents, encoding="utf-8") + + # Show hints about unmarked failures + if verbose: + unmarked_failures = all_failing_tests - failing_tests + if unmarked_failures: + print( + f"Hint: {len(unmarked_failures)} failing tests can be marked with --mark-failure; " + "but review first and do not blindly mark them all" + ) + for class_name, method_name in sorted(unmarked_failures): + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print(f" {class_name}.{method_name}{err_hint}") + + return len(failing_tests), len(unexpected_successes), len(regressions) + + +def auto_mark_directory( + test_dir: pathlib.Path, + mark_failure: bool = False, + verbose: bool = True, + original_methods_per_file: dict[pathlib.Path, set[tuple[str, str]]] | None = None, + skip_build: bool = False, +) -> tuple[int, int, int]: + """ + Run tests and auto-mark failures in a test directory. + + Runs the test once for the whole directory, then applies results to each file. + + Args: + test_dir: Path to the test directory + mark_failure: If True, add @expectedFailure to ALL failing tests + verbose: Print progress messages + original_methods_per_file: If provided, only auto-mark failures for NEW methods + even without mark_failure. Dict maps file path to + set of (class_name, method_name) tuples. + + Returns: + (num_failures_added, num_successes_removed, num_regressions) + """ + test_dir = pathlib.Path(test_dir).resolve() + if not test_dir.exists(): + raise FileNotFoundError(f"Directory not found: {test_dir}") + if not test_dir.is_dir(): + raise ValueError(f"Not a directory: {test_dir}") + + # Get all .py files in directory + test_files = sorted(test_dir.glob("**/*.py")) + + # Strip reason-less markers from ALL files before running tests so those + # tests fail normally and we capture their error messages. + stripped_per_file: dict[pathlib.Path, set[tuple[str, str]]] = {} + original_per_file: dict[pathlib.Path, str] = {} + for test_file in test_files: + contents = test_file.read_text(encoding="utf-8") + stripped_contents, stripped = strip_reasonless_expected_failures(contents) + if stripped: + original_per_file[test_file] = contents + test_file.write_text(stripped_contents, encoding="utf-8") + stripped_per_file[test_file] = stripped + + test_name = get_test_module_name(test_dir) + if verbose: + print(f"Running test: {test_name}") + + results = run_test(test_name, skip_build=skip_build) + + # Check if test run failed entirely (e.g., import error, crash) + if ( + not results.tests_result + and not results.tests + and not results.unexpected_successes + ): + # Restore original contents before raising + for fpath, original in original_per_file.items(): + fpath.write_text(original, encoding="utf-8") + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + + # If the run crashed (incomplete), restore original files so that markers + # for tests that never ran are preserved. + if not results.tests_result and original_per_file: + for fpath, original in original_per_file.items(): + fpath.write_text(original, encoding="utf-8") + stripped_per_file.clear() + + total_added = 0 + total_removed = 0 + total_regressions = 0 + all_regressions: list[tuple[str, str, str, str]] = [] + + for test_file in test_files: + # Get module prefix for this file (e.g., "test_inspect.test_inspect") + module_prefix = get_test_module_name(test_file) + # For __init__.py, the test path doesn't include "__init__" + if module_prefix.endswith(".__init__"): + module_prefix = module_prefix[:-9] # Remove ".__init__" + + all_failing_tests, unexpected_successes, error_messages = collect_test_changes( + results, module_prefix="test." + module_prefix + "." + ) + + # Determine which failures to mark + if mark_failure: + failing_tests = all_failing_tests + elif original_methods_per_file is not None: + # Smart mode: only mark NEW test failures + contents = test_file.read_text(encoding="utf-8") + current_methods = extract_test_methods(contents) + original_methods = original_methods_per_file.get(test_file, set()) + new_methods = current_methods - original_methods + failing_tests = {t for t in all_failing_tests if t in new_methods} + else: + failing_tests = set() + + # Re-mark stripped tests that still fail (restore markers with reasons). + # Uses inheritance expansion for parent→child mapping. + stripped = stripped_per_file.get(test_file, set()) + if stripped: + file_contents = test_file.read_text(encoding="utf-8") + failing_tests |= _expand_stripped_to_children( + file_contents, stripped, all_failing_tests + ) + + regressions = all_failing_tests - failing_tests + + if failing_tests or unexpected_successes: + if verbose: + for class_name, method_name in failing_tests: + label = ( + "(new test)" if original_methods_per_file is not None else "" + ) + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print( + f" {test_file.name}: Marking as failing {label}: {class_name}.{method_name}{err_hint}".replace( + " :", ":" + ) + ) + for class_name, method_name in unexpected_successes: + print( + f" {test_file.name}: Removing expectedFailure: {class_name}.{method_name}" + ) + + contents = test_file.read_text(encoding="utf-8") + contents = apply_test_changes( + contents, failing_tests, unexpected_successes, error_messages + ) + test_file.write_text(contents, encoding="utf-8") + + # Collect regressions with error messages for later reporting + for class_name, method_name in regressions: + err_msg = error_messages.get((class_name, method_name), "") + all_regressions.append((test_file.name, class_name, method_name, err_msg)) + + total_added += len(failing_tests) + total_removed += len(unexpected_successes) + total_regressions += len(regressions) + + # Show hints about unmarked failures + if verbose and total_regressions > 0: + print( + f"Hint: {total_regressions} failing tests can be marked with --mark-failure; " + "but review first and do not blindly mark them all" + ) + for file_name, class_name, method_name, err_msg in sorted(all_regressions): + err_hint = f" - {err_msg}" if err_msg else "" + print(f" {file_name}: {class_name}.{method_name}{err_hint}") + + return total_added, total_removed, total_regressions + + +def main(argv: list[str] | None = None) -> int: + import argparse + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Path to test file or directory (e.g., Lib/test/test_foo.py or Lib/test/test_foo/)", + ) + parser.add_argument( + "--mark-failure", + action="store_true", + help="Also add @expectedFailure to failing tests (default: only remove unexpected successes)", + ) + parser.add_argument( + "--build", + action=argparse.BooleanOptionalAction, + default=True, + help="Build with cargo (default: enabled)", + ) + + args = parser.parse_args(argv) + + try: + if args.path.is_dir(): + num_added, num_removed, _ = auto_mark_directory( + args.path, mark_failure=args.mark_failure, skip_build=not args.build + ) + else: + num_added, num_removed, _ = auto_mark_file( + args.path, mark_failure=args.mark_failure, skip_build=not args.build + ) + if args.mark_failure: + print(f"Added expectedFailure to {num_added} tests") + print(f"Removed expectedFailure from {num_removed} tests") + return 0 + except (FileNotFoundError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_copy_lib.py b/scripts/update_lib/cmd_copy_lib.py new file mode 100644 index 00000000000..1b16497fc83 --- /dev/null +++ b/scripts/update_lib/cmd_copy_lib.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +""" +Copy library files from CPython. + +Usage: + # Single file + python scripts/update_lib copy-lib cpython/Lib/dataclasses.py + + # Directory + python scripts/update_lib copy-lib cpython/Lib/json +""" + +import argparse +import pathlib +import shutil +import sys + + +def _copy_single( + src_path: pathlib.Path, + lib_path: pathlib.Path, + verbose: bool = True, +) -> None: + """Copy a single file or directory.""" + # Remove existing file/directory + if lib_path.exists(): + if lib_path.is_dir(): + if verbose: + print(f"Removing directory: {lib_path}") + shutil.rmtree(lib_path) + else: + if verbose: + print(f"Removing file: {lib_path}") + lib_path.unlink() + + # Copy + if src_path.is_dir(): + if verbose: + print(f"Copying directory: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src_path, lib_path) + else: + if verbose: + print(f"Copying file: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, lib_path) + + +def copy_lib( + src_path: pathlib.Path, + verbose: bool = True, +) -> None: + """ + Copy library file or directory from CPython. + + Also copies additional files if defined in DEPENDENCIES table. + + Args: + src_path: Source path (e.g., cpython/Lib/dataclasses.py or cpython/Lib/json) + verbose: Print progress messages + """ + from update_lib.deps import get_lib_paths + from update_lib.file_utils import parse_lib_path + + # Extract module name and cpython prefix from path + path_str = str(src_path).replace("\\", "/") + if "/Lib/" not in path_str: + raise ValueError(f"Path must contain '/Lib/' (got: {src_path})") + + cpython_prefix, after_lib = path_str.split("/Lib/", 1) + # Get module name (first component, without .py) + name = after_lib.split("/")[0] + if name.endswith(".py"): + name = name[:-3] + + # Get all paths to copy from DEPENDENCIES table + all_src_paths = get_lib_paths(name, cpython_prefix) + + # Copy each file + for src in all_src_paths: + if src.exists(): + lib_path = parse_lib_path(src) + _copy_single(src, lib_path, verbose) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path containing /Lib/ (e.g., cpython/Lib/dataclasses.py)", + ) + + args = parser.parse_args(argv) + + try: + copy_lib(args.path) + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_deps.py b/scripts/update_lib/cmd_deps.py new file mode 100644 index 00000000000..affb4b3609c --- /dev/null +++ b/scripts/update_lib/cmd_deps.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python +""" +Show dependency information for a module. + +Usage: + python scripts/update_lib deps dis + python scripts/update_lib deps dataclasses + python scripts/update_lib deps dis --depth 2 + python scripts/update_lib deps all # Show all modules' dependencies +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + + +def get_all_modules(cpython_prefix: str) -> list[str]: + """Get all top-level module names from cpython/Lib/. + + Includes private modules (_*) that are not hard_deps of other modules. + + Returns: + Sorted list of module names (without .py extension) + """ + from update_lib.deps import resolve_hard_dep_parent + + lib_dir = pathlib.Path(cpython_prefix) / "Lib" + if not lib_dir.exists(): + return [] + + modules = set() + for entry in lib_dir.iterdir(): + # Skip hidden files + if entry.name.startswith("."): + continue + # Skip test directory + if entry.name == "test": + continue + + if entry.is_file() and entry.suffix == ".py": + name = entry.stem + elif entry.is_dir() and (entry / "__init__.py").exists(): + name = entry.name + else: + continue + + # Skip modules that are hard_deps of other modules + # e.g., _pydatetime is a hard_dep of datetime, pydoc_data is a hard_dep of pydoc + if resolve_hard_dep_parent(name, cpython_prefix) is not None: + continue + + modules.add(name) + + return sorted(modules) + + +def format_deps_tree( + cpython_prefix: str, + lib_prefix: str, + max_depth: int, + *, + name: str | None = None, + soft_deps: set[str] | None = None, + hard_deps: set[str] | None = None, + _depth: int = 0, + _visited: set[str] | None = None, + _indent: str = "", +) -> list[str]: + """Format soft dependencies as a tree with up-to-date status. + + Args: + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + max_depth: Maximum recursion depth + name: Module name (used to compute deps if soft_deps not provided) + soft_deps: Pre-computed soft dependencies (optional) + hard_deps: Hard dependencies to show under the module (root level only) + _depth: Current depth (internal) + _visited: Already visited modules (internal) + _indent: Current indentation (internal) + + Returns: + List of formatted lines + """ + from update_lib.deps import ( + get_lib_paths, + get_rust_deps, + get_soft_deps, + is_up_to_date, + ) + + lines = [] + + if _visited is None: + _visited = set() + + # Compute deps from name if not provided + if soft_deps is None: + soft_deps = get_soft_deps(name, cpython_prefix) if name else set() + + soft_deps = sorted(soft_deps) + + if not soft_deps and not hard_deps: + return lines + + # Separate up-to-date and outdated modules + up_to_date_deps = [] + outdated_deps = [] + dup_deps = [] + + for dep in soft_deps: + # Skip if library doesn't exist in cpython + lib_paths = get_lib_paths(dep, cpython_prefix) + if not any(p.exists() for p in lib_paths): + continue + + up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix) + if up_to_date: + # Up-to-date modules collected compactly, no dup tracking needed + up_to_date_deps.append(dep) + elif dep in _visited: + # Only track dup for outdated modules + dup_deps.append(dep) + else: + outdated_deps.append(dep) + + # Show outdated modules with expansion + for dep in outdated_deps: + dep_native = get_rust_deps(dep, cpython_prefix) + native_suffix = ( + f" (native: {', '.join(sorted(dep_native))})" if dep_native else "" + ) + lines.append(f"{_indent}- [ ] {dep}{native_suffix}") + _visited.add(dep) + + # Show hard_deps under this module (only at root level, i.e., when hard_deps is provided) + if hard_deps and dep in soft_deps: + for hd in sorted(hard_deps): + hd_up_to_date = is_up_to_date(hd, cpython_prefix, lib_prefix) + hd_marker = "[x]" if hd_up_to_date else "[ ]" + lines.append(f"{_indent} - {hd_marker} {hd}") + hard_deps = None # Only show once + + # Recurse if within depth limit + if _depth < max_depth - 1: + lines.extend( + format_deps_tree( + cpython_prefix, + lib_prefix, + max_depth, + name=dep, + _depth=_depth + 1, + _visited=_visited, + _indent=_indent + " ", + ) + ) + + # Show duplicates compactly (only for outdated) + if dup_deps: + lines.append(f"{_indent}- [ ] {', '.join(dup_deps)}") + + # Show up-to-date modules compactly on one line + if up_to_date_deps: + lines.append(f"{_indent}- [x] {', '.join(up_to_date_deps)}") + + return lines + + +def format_deps( + name: str, + cpython_prefix: str, + lib_prefix: str, + max_depth: int = 10, + _visited: set[str] | None = None, +) -> list[str]: + """Format all dependency information for a module. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + max_depth: Maximum recursion depth + _visited: Shared visited set for deduplication across modules + + Returns: + List of formatted lines + """ + from update_lib.deps import ( + DEPENDENCIES, + count_test_todos, + find_dependent_tests_tree, + get_lib_paths, + get_test_paths, + is_path_synced, + is_test_up_to_date, + resolve_hard_dep_parent, + ) + + if _visited is None: + _visited = set() + + lines = [] + + # Resolve test_ prefix to module (e.g., test_pydoc -> pydoc) + if name.startswith("test_"): + module_name = name[5:] # strip "test_" + lines.append(f"(redirecting {name} -> {module_name})") + name = module_name + + # Resolve hard_dep to parent module (e.g., pydoc_data -> pydoc) + parent = resolve_hard_dep_parent(name, cpython_prefix) + if parent: + lines.append(f"(redirecting {name} -> {parent})") + name = parent + + # lib paths (only show existing) + lib_paths = get_lib_paths(name, cpython_prefix) + existing_lib_paths = [p for p in lib_paths if p.exists()] + for p in existing_lib_paths: + synced = is_path_synced(p, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + lines.append(f"{marker} lib: {p}") + + # test paths (only show existing) + test_paths = get_test_paths(name, cpython_prefix) + existing_test_paths = [p for p in test_paths if p.exists()] + for p in existing_test_paths: + test_name = p.stem if p.is_file() else p.name + synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + todo_count = count_test_todos(test_name, lib_prefix) + todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else "" + lines.append(f"{marker} test: {p}{todo_suffix}") + + # If no lib or test paths exist, module doesn't exist + if not existing_lib_paths and not existing_test_paths: + lines.append(f"(module '{name}' not found)") + return lines + + # Collect all hard_deps (explicit from DEPENDENCIES + implicit from lib_paths) + dep_info = DEPENDENCIES.get(name, {}) + explicit_hard_deps = dep_info.get("hard_deps", []) + + # Get implicit hard_deps from lib_paths (e.g., _pydecimal.py for decimal) + all_hard_deps = set() + for hd in explicit_hard_deps: + # Remove .py extension if present + all_hard_deps.add(hd[:-3] if hd.endswith(".py") else hd) + + for p in existing_lib_paths: + dep_name = p.stem if p.is_file() else p.name + if dep_name != name: # Skip the main module itself + all_hard_deps.add(dep_name) + + lines.append("\ndependencies:") + lines.extend( + format_deps_tree( + cpython_prefix, + lib_prefix, + max_depth, + soft_deps={name}, + _visited=_visited, + hard_deps=all_hard_deps, + ) + ) + + # Show dependent tests as tree (depth 2: module + direct importers + their importers) + tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2) + lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix)) + + return lines + + +def _format_dependent_tests_tree( + tree: dict, + cpython_prefix: str, + lib_prefix: str, + indent: str = "", +) -> list[str]: + """Format dependent tests tree for display.""" + from update_lib.deps import is_up_to_date + + lines = [] + module = tree["module"] + tests = tree["tests"] + children = tree["children"] + + if indent == "": + # Root level + # Count total tests in tree + def count_tests(t: dict) -> int: + total = len(t.get("tests", [])) + for c in t.get("children", []): + total += count_tests(c) + return total + + total = count_tests(tree) + if total == 0 and not children: + lines.append(f"\ndependent tests: (no tests depend on {module})") + return lines + lines.append(f"\ndependent tests: ({total} tests)") + + # Check if module is up-to-date + synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + + # Format this node + if tests: + test_str = " ".join(tests) + if indent == "": + lines.append(f"- {marker} {module}: {test_str}") + else: + lines.append(f"{indent}- {marker} {module}: {test_str}") + elif indent != "" and children: + # Has children but no direct tests + lines.append(f"{indent}- {marker} {module}:") + + # Format children + child_indent = indent + " " if indent else " " + for child in children: + lines.extend( + _format_dependent_tests_tree( + child, cpython_prefix, lib_prefix, child_indent + ) + ) + + return lines + + +def _resolve_module_name( + name: str, + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Resolve module name through redirects. + + Returns a list of module names (usually 1, but test support files may expand to multiple). + """ + import pathlib + + from update_lib.deps import ( + _build_test_import_graph, + get_lib_paths, + get_test_paths, + resolve_hard_dep_parent, + resolve_test_to_lib, + ) + + # Resolve test to library group (e.g., test_urllib2 -> urllib) + if name.startswith("test_"): + lib_group = resolve_test_to_lib(name) + if lib_group: + return [lib_group] + name = name[5:] + + # Resolve hard_dep to parent + parent = resolve_hard_dep_parent(name, cpython_prefix) + if parent: + return [parent] + + # Check if it's a valid module + lib_paths = get_lib_paths(name, cpython_prefix) + test_paths = get_test_paths(name, cpython_prefix) + if any(p.exists() for p in lib_paths) or any(p.exists() for p in test_paths): + return [name] + + # Check for test support files (e.g., string_tests -> bytes, str, userstring) + test_support_path = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{name}.py" + if test_support_path.exists(): + test_dir = pathlib.Path(lib_prefix) / "test" + if test_dir.exists(): + import_graph, _ = _build_test_import_graph(test_dir) + importing_tests = [] + for file_key, imports in import_graph.items(): + if name in imports and file_key.startswith("test_"): + importing_tests.append(file_key) + if importing_tests: + # Resolve test names to module names (test_bytes -> bytes) + return sorted(set(t[5:] for t in importing_tests)) + + return [name] + + +def show_deps( + names: list[str], + cpython_prefix: str, + lib_prefix: str, + max_depth: int = 10, +) -> None: + """Show all dependency information for modules.""" + # Expand "all" to all module names + expanded_names = [] + for name in names: + if name == "all": + expanded_names.extend(get_all_modules(cpython_prefix)) + else: + expanded_names.append(name) + + # Resolve and deduplicate names (preserving order) + seen: set[str] = set() + resolved_names: list[str] = [] + for name in expanded_names: + for resolved in _resolve_module_name(name, cpython_prefix, lib_prefix): + if resolved not in seen: + seen.add(resolved) + resolved_names.append(resolved) + + # Shared visited set across all modules + visited: set[str] = set() + + for i, name in enumerate(resolved_names): + if i > 0: + print() # blank line between modules + for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): + print(line) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "names", + nargs="+", + help="Module names (e.g., dis, dataclasses) or 'all' for all modules", + ) + parser.add_argument( + "--cpython", + default="cpython", + help="CPython directory prefix (default: cpython)", + ) + parser.add_argument( + "--lib", + default="Lib", + help="Local Lib directory prefix (default: Lib)", + ) + parser.add_argument( + "--depth", + type=int, + default=10, + help="Maximum recursion depth for soft_deps tree (default: 10)", + ) + + args = parser.parse_args(argv) + + try: + show_deps(args.names, args.cpython, args.lib, args.depth) + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py new file mode 100644 index 00000000000..97cdf7b141b --- /dev/null +++ b/scripts/update_lib/cmd_migrate.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +""" +Migrate test file(s) from CPython, preserving RustPython markers. + +Usage: + python scripts/update_lib migrate cpython/Lib/test/test_foo.py + +This will: + 1. Extract patches from Lib/test/test_foo.py (if exists) + 2. Apply them to cpython/Lib/test/test_foo.py + 3. Write result to Lib/test/test_foo.py +""" + +import argparse +import pathlib +import shutil +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.file_utils import parse_lib_path + + +def patch_single_content( + src_path: pathlib.Path, + lib_path: pathlib.Path, +) -> str: + """ + Patch content without writing to disk. + + Args: + src_path: Source file path (e.g., cpython/Lib/test/foo.py) + lib_path: Lib path to extract patches from (e.g., Lib/test/foo.py) + + Returns: + The patched content. + """ + from update_lib import apply_patches, extract_patches + + # Extract patches from existing file (if exists) + if lib_path.exists(): + patches = extract_patches(lib_path.read_text(encoding="utf-8")) + else: + patches = {} + + # Apply patches to source content + src_content = src_path.read_text(encoding="utf-8") + return apply_patches(src_content, patches) + + +def patch_file( + src_path: pathlib.Path, + lib_path: pathlib.Path | None = None, + verbose: bool = True, +) -> None: + """ + Patch a single file from source to lib. + + Args: + src_path: Source file path (e.g., cpython/Lib/test/foo.py) + lib_path: Target lib path. If None, derived from src_path. + verbose: Print progress messages + """ + if lib_path is None: + lib_path = parse_lib_path(src_path) + + if lib_path.exists(): + if verbose: + print(f"Patching: {src_path} -> {lib_path}") + content = patch_single_content(src_path, lib_path) + else: + if verbose: + print(f"Copying: {src_path} -> {lib_path}") + content = src_path.read_text(encoding="utf-8") + + lib_path.parent.mkdir(parents=True, exist_ok=True) + lib_path.write_text(content, encoding="utf-8") + + +def patch_directory( + src_dir: pathlib.Path, + lib_dir: pathlib.Path | None = None, + verbose: bool = True, +) -> None: + """ + Patch all files in a directory from source to lib. + + Args: + src_dir: Source directory path (e.g., cpython/Lib/test/test_foo/) + lib_dir: Target lib directory. If None, derived from src_dir. + verbose: Print progress messages + """ + if lib_dir is None: + lib_dir = parse_lib_path(src_dir) + + src_files = sorted(f for f in src_dir.glob("**/*") if f.is_file()) + + for src_file in src_files: + rel_path = src_file.relative_to(src_dir) + lib_file = lib_dir / rel_path + + if src_file.suffix == ".py": + if lib_file.exists(): + if verbose: + print(f"Patching: {src_file} -> {lib_file}") + content = patch_single_content(src_file, lib_file) + else: + if verbose: + print(f"Copying: {src_file} -> {lib_file}") + content = src_file.read_text(encoding="utf-8") + + lib_file.parent.mkdir(parents=True, exist_ok=True) + lib_file.write_text(content, encoding="utf-8") + else: + if verbose: + print(f"Copying: {src_file} -> {lib_file}") + lib_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_file, lib_file) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path containing /Lib/ (file or directory)", + ) + + args = parser.parse_args(argv) + + try: + if args.path.is_dir(): + patch_directory(args.path) + else: + patch_file(args.path) + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_patches.py b/scripts/update_lib/cmd_patches.py new file mode 100644 index 00000000000..67ebf1822b7 --- /dev/null +++ b/scripts/update_lib/cmd_patches.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +""" +Patch management for test files. + +Usage: + # Extract patches from one file and apply to another + python scripts/update_lib patches --from Lib/test/foo.py --to cpython/Lib/test/foo.py + + # Show patches as JSON + python scripts/update_lib patches --from Lib/test/foo.py --show-patches + + # Apply patches from JSON file + python scripts/update_lib patches -p patches.json --to Lib/test/foo.py +""" + +import argparse +import json +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + + +def write_output(data: str, dest: str) -> None: + if dest == "-": + print(data, end="") + return + + with open(dest, "w") as fd: + fd.write(data) + + +def main(argv: list[str] | None = None) -> int: + from update_lib import ( + apply_patches, + extract_patches, + patches_from_json, + patches_to_json, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + patches_group = parser.add_mutually_exclusive_group(required=True) + patches_group.add_argument( + "-p", + "--patches", + type=pathlib.Path, + help="File path to file containing patches in a JSON format", + ) + patches_group.add_argument( + "--from", + dest="gather_from", + type=pathlib.Path, + help="File to gather patches from", + ) + + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + "--to", + type=pathlib.Path, + help="File to apply patches to", + ) + group.add_argument( + "--show-patches", + action="store_true", + help="Show the patches and exit", + ) + + parser.add_argument( + "-o", + "--output", + default="-", + help="Output file. Set to '-' for stdout", + ) + + args = parser.parse_args(argv) + + # Validate required arguments + if args.to is None and not args.show_patches: + parser.error("--to or --show-patches is required") + + try: + if args.patches: + patches = patches_from_json(json.loads(args.patches.read_text())) + else: + patches = extract_patches(args.gather_from.read_text()) + + if args.show_patches: + output = json.dumps(patches_to_json(patches), indent=4) + "\n" + write_output(output, args.output) + return 0 + + patched = apply_patches(args.to.read_text(), patches) + write_output(patched, args.output) + return 0 + + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_quick.py b/scripts/update_lib/cmd_quick.py new file mode 100644 index 00000000000..c43e0761518 --- /dev/null +++ b/scripts/update_lib/cmd_quick.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python +""" +Quick update for test files from CPython. + +Usage: + # Library + test: copy lib, then patch + auto-mark test + commit + python scripts/update_lib quick cpython/Lib/dataclasses.py + + # Shortcut: just the module name + python scripts/update_lib quick dataclasses + + # Test file: patch + auto-mark + python scripts/update_lib quick cpython/Lib/test/test_foo.py + + # Test file: migrate only + python scripts/update_lib quick cpython/Lib/test/test_foo.py --no-auto-mark + + # Test file: auto-mark only (Lib/ path implies --no-migrate) + python scripts/update_lib quick Lib/test/test_foo.py + + # Directory: patch all + auto-mark all + python scripts/update_lib quick cpython/Lib/test/test_dataclasses/ + + # Skip git commit + python scripts/update_lib quick dataclasses --no-commit +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.deps import DEPENDENCIES, get_test_paths +from update_lib.file_utils import ( + construct_lib_path, + get_cpython_dir, + get_module_name, + get_test_files, + is_lib_path, + is_test_path, + lib_to_test_path, + parse_lib_path, + resolve_module_path, + safe_read_text, +) + + +def collect_original_methods( + lib_path: pathlib.Path, +) -> set[tuple[str, str]] | dict[pathlib.Path, set[tuple[str, str]]] | None: + """ + Collect original test methods from lib path before patching. + + Returns: + - For file: set of (class_name, method_name) or None if file doesn't exist + - For directory: dict mapping file path to set of methods, or None if dir doesn't exist + """ + from update_lib.cmd_auto_mark import extract_test_methods + + if not lib_path.exists(): + return None + + if lib_path.is_file(): + content = safe_read_text(lib_path) + return extract_test_methods(content) if content else set() + else: + result = {} + for lib_file in get_test_files(lib_path): + content = safe_read_text(lib_file) + if content: + result[lib_file.resolve()] = extract_test_methods(content) + return result + + +def quick( + src_path: pathlib.Path, + no_migrate: bool = False, + no_auto_mark: bool = False, + mark_failure: bool = False, + verbose: bool = True, + skip_build: bool = False, +) -> list[pathlib.Path]: + """ + Process a file or directory: migrate + auto-mark. + + Args: + src_path: Source path (file or directory) + no_migrate: Skip migration step + no_auto_mark: Skip auto-mark step + mark_failure: Add @expectedFailure to ALL failing tests + verbose: Print progress messages + skip_build: Skip cargo build, use pre-built binary + + Returns: + List of extra paths (data dirs, hard deps) that were copied/migrated. + """ + from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file + from update_lib.cmd_migrate import patch_directory, patch_file + + extra_paths: list[pathlib.Path] = [] + + # Determine lib_path and whether to migrate + if is_lib_path(src_path): + no_migrate = True + lib_path = src_path + else: + lib_path = parse_lib_path(src_path) + + is_dir = src_path.is_dir() + + # Capture original test methods before migration (for smart auto-mark) + original_methods = collect_original_methods(lib_path) + + # Step 1: Migrate + if not no_migrate: + if is_dir: + patch_directory(src_path, lib_path, verbose=verbose) + else: + patch_file(src_path, lib_path, verbose=verbose) + + # Step 1.5: Handle test dependencies + from update_lib.deps import get_test_dependencies + + test_deps = get_test_dependencies(src_path) + + # Migrate dependency files + for dep_src in test_deps["hard_deps"]: + dep_lib = parse_lib_path(dep_src) + if verbose: + print(f"Migrating dependency: {dep_src.name}") + if dep_src.is_dir(): + patch_directory(dep_src, dep_lib, verbose=False) + else: + patch_file(dep_src, dep_lib, verbose=False) + extra_paths.append(dep_lib) + + # Copy data directories (no migration) + import shutil + + for data_src in test_deps["data"]: + data_lib = parse_lib_path(data_src) + if verbose: + print(f"Copying data: {data_src.name}") + if data_lib.exists(): + if data_lib.is_dir(): + shutil.rmtree(data_lib) + else: + data_lib.unlink() + if data_src.is_dir(): + shutil.copytree(data_src, data_lib) + else: + data_lib.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(data_src, data_lib) + extra_paths.append(data_lib) + + # Step 2: Auto-mark + if not no_auto_mark: + if not lib_path.exists(): + raise FileNotFoundError(f"Path not found: {lib_path}") + + if is_dir: + num_added, num_removed, _ = auto_mark_directory( + lib_path, + mark_failure=mark_failure, + verbose=verbose, + original_methods_per_file=original_methods, + skip_build=skip_build, + ) + else: + num_added, num_removed, _ = auto_mark_file( + lib_path, + mark_failure=mark_failure, + verbose=verbose, + original_methods=original_methods, + skip_build=skip_build, + ) + + if verbose: + if num_added: + print(f"Added expectedFailure to {num_added} tests") + print(f"Removed expectedFailure from {num_removed} tests") + + return extra_paths + + +def get_cpython_version(cpython_dir: pathlib.Path) -> str: + """Get CPython version from git tag.""" + import subprocess + + result = subprocess.run( + ["git", "describe", "--tags"], + cwd=cpython_dir, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def git_commit( + name: str, + lib_path: pathlib.Path | None, + test_paths: list[pathlib.Path] | pathlib.Path | None, + cpython_dir: pathlib.Path, + hard_deps: list[pathlib.Path] | None = None, + verbose: bool = True, +) -> bool: + """Commit changes with CPython author. + + Args: + name: Module name (e.g., "dataclasses") + lib_path: Path to library file/directory (or None) + test_paths: Path(s) to test file/directory (or None) + cpython_dir: Path to cpython directory + hard_deps: Path(s) to hard dependency files (or None) + verbose: Print progress messages + + Returns: + True if commit was created, False otherwise + """ + import subprocess + + # Normalize test_paths to list + if test_paths is None: + test_paths = [] + elif isinstance(test_paths, pathlib.Path): + test_paths = [test_paths] + + # Normalize hard_deps to list + if hard_deps is None: + hard_deps = [] + + # Stage changes + paths_to_add = [] + if lib_path and lib_path.exists(): + paths_to_add.append(str(lib_path)) + for test_path in test_paths: + if test_path and test_path.exists(): + paths_to_add.append(str(test_path)) + for dep_path in hard_deps: + if dep_path and dep_path.exists(): + paths_to_add.append(str(dep_path)) + + if not paths_to_add: + return False + + version = get_cpython_version(cpython_dir) + subprocess.run(["git", "add"] + paths_to_add, check=True) + + # Check if there are staged changes + result = subprocess.run( + ["git", "diff", "--cached", "--quiet"], + capture_output=True, + ) + if result.returncode == 0: + if verbose: + print("No changes to commit") + return False + + # Commit with CPython author + message = f"Update {name} from {version}" + subprocess.run( + [ + "git", + "commit", + "--author", + "CPython Developers <>", + "-m", + message, + ], + check=True, + ) + if verbose: + print(f"Committed: {message}") + return True + + +def _expand_shortcut(path: pathlib.Path) -> pathlib.Path: + """Expand simple name to cpython/Lib path if it exists. + + Examples: + dataclasses -> cpython/Lib/dataclasses.py (if exists) + json -> cpython/Lib/json/ (if exists) + test_types -> cpython/Lib/test/test_types.py (if exists) + regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES) + """ + # Only expand if it's a simple name (no path separators) and doesn't exist + if "/" in str(path) or path.exists(): + return path + + name = str(path) + + # Check DEPENDENCIES table for path overrides (e.g., regrtest) + from update_lib.deps import DEPENDENCIES + + if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]: + lib_paths = DEPENDENCIES[name]["lib"] + if lib_paths: + override_path = construct_lib_path("cpython", lib_paths[0]) + if override_path.exists(): + return override_path + + # Test shortcut: test_foo -> cpython/Lib/test/test_foo + if name.startswith("test_"): + resolved = resolve_module_path(f"test/{name}", "cpython", prefer="dir") + if resolved.exists(): + return resolved + + # Library shortcut: foo -> cpython/Lib/foo + resolved = resolve_module_path(name, "cpython", prefer="file") + if resolved.exists(): + return resolved + + # Extension module shortcut: winreg -> cpython/Lib/test/test_winreg + # For C/Rust extension modules that have no Python source but have tests + resolved = resolve_module_path(f"test/test_{name}", "cpython", prefer="dir") + if resolved.exists(): + return resolved + + # Return original (will likely fail later with a clear error) + return path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path (file or directory)", + ) + parser.add_argument( + "--copy", + action=argparse.BooleanOptionalAction, + default=True, + help="Copy library file (default: enabled, implied disabled if test path)", + ) + parser.add_argument( + "--migrate", + action=argparse.BooleanOptionalAction, + default=True, + help="Migrate test file (default: enabled, implied disabled if Lib/ path)", + ) + parser.add_argument( + "--auto-mark", + action=argparse.BooleanOptionalAction, + default=True, + help="Auto-mark test failures (default: enabled)", + ) + parser.add_argument( + "--mark-failure", + action="store_true", + help="Add @expectedFailure to failing tests", + ) + parser.add_argument( + "--commit", + action=argparse.BooleanOptionalAction, + default=True, + help="Create git commit (default: enabled)", + ) + parser.add_argument( + "--build", + action=argparse.BooleanOptionalAction, + default=True, + help="Build with cargo (default: enabled)", + ) + + args = parser.parse_args(argv) + + try: + src_path = args.path + + # Shortcut: expand simple name to cpython/Lib path + src_path = _expand_shortcut(src_path) + original_src = src_path # Keep for commit + + # Track library path for commit + lib_file_path = None + test_path = None + hard_deps_for_commit = [] + + # If it's a library path (not test path), do copy_lib first + if not is_test_path(src_path): + # Get library destination path for commit + lib_file_path = parse_lib_path(src_path) + + if args.copy: + from update_lib.cmd_copy_lib import copy_lib + + copy_lib(src_path) + + # Get all test paths from DEPENDENCIES (or fall back to default) + module_name = get_module_name(original_src) + cpython_dir = get_cpython_dir(original_src) + test_src_paths = get_test_paths(module_name, str(cpython_dir)) + + # Fall back to default test path if DEPENDENCIES has no entry + if not test_src_paths: + default_test = lib_to_test_path(original_src) + if default_test.exists(): + test_src_paths = (default_test,) + + # Collect hard dependencies for commit + lib_deps = DEPENDENCIES.get(module_name, {}) + for dep_name in lib_deps.get("hard_deps", []): + dep_lib_path = pathlib.Path("Lib") / dep_name + if dep_lib_path.exists(): + hard_deps_for_commit.append(dep_lib_path) + + # Process all test paths + test_paths_for_commit = [] + for test_src in test_src_paths: + if not test_src.exists(): + print(f"Warning: Test path does not exist: {test_src}") + continue + + test_lib_path = parse_lib_path(test_src) + test_paths_for_commit.append(test_lib_path) + + extra = quick( + test_src, + no_migrate=not args.migrate, + no_auto_mark=not args.auto_mark, + mark_failure=args.mark_failure, + skip_build=not args.build, + ) + hard_deps_for_commit.extend(extra) + + test_paths = test_paths_for_commit + else: + # It's a test path - process single test + test_path = ( + parse_lib_path(src_path) if not is_lib_path(src_path) else src_path + ) + + extra = quick( + src_path, + no_migrate=not args.migrate, + no_auto_mark=not args.auto_mark, + mark_failure=args.mark_failure, + skip_build=not args.build, + ) + hard_deps_for_commit.extend(extra) + test_paths = [test_path] + + # Step 3: Git commit + if args.commit: + cpython_dir = get_cpython_dir(original_src) + git_commit( + get_module_name(original_src), + lib_file_path, + test_paths, + cpython_dir, + hard_deps=hard_deps_for_commit, + ) + + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except Exception as e: + # Handle TestRunError with a clean message + from update_lib.cmd_auto_mark import TestRunError + + if isinstance(e, TestRunError): + print(f"Error: {e}", file=sys.stderr) + return 1 + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_todo.py b/scripts/update_lib/cmd_todo.py new file mode 100644 index 00000000000..23aec52d7dc --- /dev/null +++ b/scripts/update_lib/cmd_todo.py @@ -0,0 +1,693 @@ +#!/usr/bin/env python +""" +Show prioritized list of modules to update. + +Usage: + python scripts/update_lib todo + python scripts/update_lib todo --limit 20 +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.deps import ( + count_test_todos, + get_module_diff_stat, + get_module_last_updated, + get_test_last_updated, + is_test_tracked, + is_test_up_to_date, +) + + +def compute_todo_list( + cpython_prefix: str, + lib_prefix: str, + include_done: bool = False, +) -> list[dict]: + """Compute prioritized list of modules to update. + + Scoring: + - Modules with no pylib dependencies: score = -1 + - Modules with pylib dependencies: score = count of NOT up-to-date deps + + Sorting (ascending by score): + 1. More reverse dependencies (modules depending on this) = higher priority + 2. Fewer native dependencies = higher priority + + Returns: + List of dicts with module info, sorted by priority + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import ( + get_all_hard_deps, + get_rust_deps, + get_soft_deps, + is_up_to_date, + ) + + all_modules = get_all_modules(cpython_prefix) + + # Build dependency data for all modules + module_data = {} + for name in all_modules: + soft_deps = get_soft_deps(name, cpython_prefix) + native_deps = get_rust_deps(name, cpython_prefix) + up_to_date = is_up_to_date(name, cpython_prefix, lib_prefix) + + # Get hard_deps and check their status + hard_deps = get_all_hard_deps(name, cpython_prefix) + hard_deps_status = { + hd: is_up_to_date(hd, cpython_prefix, lib_prefix) for hd in hard_deps + } + + module_data[name] = { + "name": name, + "soft_deps": soft_deps, + "native_deps": native_deps, + "up_to_date": up_to_date, + "hard_deps_status": hard_deps_status, + } + + # Build reverse dependency map: who depends on this module + reverse_deps: dict[str, set[str]] = {name: set() for name in all_modules} + for name, data in module_data.items(): + for dep in data["soft_deps"]: + if dep in reverse_deps: + reverse_deps[dep].add(name) + + # Compute scores and filter + result = [] + for name, data in module_data.items(): + hard_deps_status = data["hard_deps_status"] + has_outdated_hard_deps = any(not ok for ok in hard_deps_status.values()) + + # Include if: not up-to-date, or has outdated hard_deps, or --done + if data["up_to_date"] and not has_outdated_hard_deps and not include_done: + continue + + soft_deps = data["soft_deps"] + if not soft_deps: + # No pylib dependencies + score = -1 + total_deps = 0 + else: + # Count NOT up-to-date dependencies + score = sum( + 1 + for dep in soft_deps + if dep in module_data and not module_data[dep]["up_to_date"] + ) + total_deps = len(soft_deps) + + result.append( + { + "name": name, + "score": score, + "total_deps": total_deps, + "reverse_deps": reverse_deps[name], + "reverse_deps_count": len(reverse_deps[name]), + "native_deps_count": len(data["native_deps"]), + "native_deps": data["native_deps"], + "soft_deps": soft_deps, + "up_to_date": data["up_to_date"], + "hard_deps_status": hard_deps_status, + } + ) + + # Sort by: + # 1. score (ascending) - fewer outstanding deps first + # 2. reverse_deps_count (descending) - more dependents first + # 3. native_deps_count (ascending) - fewer native deps first + result.sort( + key=lambda x: ( + x["score"], + -x["reverse_deps_count"], + x["native_deps_count"], + ) + ) + + return result + + +def get_all_tests(cpython_prefix: str) -> list[str]: + """Get all test module names from cpython/Lib/test/. + + Returns: + Sorted list of test names (e.g., ["test_abc", "test_dis", ...]) + """ + test_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" + if not test_dir.exists(): + return [] + + tests = set() + for entry in test_dir.iterdir(): + # Skip private/internal and special directories + if entry.name.startswith(("_", ".")): + continue + # Skip non-test items + if not entry.name.startswith("test_"): + continue + + if entry.is_file() and entry.suffix == ".py": + tests.add(entry.stem) + elif entry.is_dir() and (entry / "__init__.py").exists(): + tests.add(entry.name) + + return sorted(tests) + + +def get_untracked_files( + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Get files that exist in cpython/Lib but not in our Lib. + + Excludes files that belong to tracked modules (shown in library todo) + and hard_deps of those modules. + Includes all file types (.py, .txt, .pem, .json, etc.) + + Returns: + Sorted list of relative paths (e.g., ["foo.py", "data/file.txt"]) + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import resolve_hard_dep_parent + + cpython_lib = pathlib.Path(cpython_prefix) / "Lib" + local_lib = pathlib.Path(lib_prefix) + + if not cpython_lib.exists(): + return [] + + # Get tracked modules (shown in library todo) + tracked_modules = set(get_all_modules(cpython_prefix)) + + untracked = [] + + for cpython_file in cpython_lib.rglob("*"): + # Skip directories + if cpython_file.is_dir(): + continue + + # Get relative path from Lib/ + rel_path = cpython_file.relative_to(cpython_lib) + + # Skip test/ directory (handled separately by test todo) + if rel_path.parts and rel_path.parts[0] == "test": + continue + + # Check if file belongs to a tracked module + # e.g., idlelib/Icons/idle.gif -> module "idlelib" + # e.g., foo.py -> module "foo" + first_part = rel_path.parts[0] + if first_part.endswith(".py"): + module_name = first_part[:-3] # Remove .py + else: + module_name = first_part + + if module_name in tracked_modules: + continue + + # Check if this is a hard_dep of a tracked module + if resolve_hard_dep_parent(module_name, cpython_prefix) is not None: + continue + + # Check if exists in local lib + local_file = local_lib / rel_path + if not local_file.exists(): + untracked.append(str(rel_path)) + + return sorted(untracked) + + +def get_original_files( + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Get top-level files/modules that exist in our Lib but not in cpython/Lib. + + These are RustPython-original files that don't come from CPython. + Modules that exist in cpython are handled by the library todo (even if + they have additional local files), so they are excluded here. + Excludes test/ directory (handled separately). + + Returns: + Sorted list of top-level names (e.g., ["_dummy_thread.py"]) + """ + cpython_lib = pathlib.Path(cpython_prefix) / "Lib" + local_lib = pathlib.Path(lib_prefix) + + if not local_lib.exists(): + return [] + + original = [] + + # Only check top-level entries + for entry in local_lib.iterdir(): + name = entry.name + + # Skip hidden files and __pycache__ + if name.startswith(".") or name == "__pycache__": + continue + + # Skip test/ directory (handled separately) + if name == "test": + continue + + # Skip site-packages (not a module) + if name == "site-packages": + continue + + # Only include if it doesn't exist in cpython at all + cpython_entry = cpython_lib / name + if not cpython_entry.exists(): + original.append(name) + + return sorted(original) + + +def _build_test_to_lib_map( + cpython_prefix: str, +) -> tuple[dict[str, str], dict[str, list[str]]]: + """Build reverse mapping from test name to library name using DEPENDENCIES. + + Returns: + Tuple of: + - Dict mapping test_name -> lib_name (e.g., "test_htmlparser" -> "html") + - Dict mapping lib_name -> ordered list of test_names + """ + import pathlib + + from update_lib.deps import DEPENDENCIES + + test_to_lib = {} + lib_test_order: dict[str, list[str]] = {} + for lib_name, dep_info in DEPENDENCIES.items(): + if "test" not in dep_info: + continue + lib_test_order[lib_name] = [] + for test_path in dep_info["test"]: + # test_path is like "test_htmlparser.py" or "test_multiprocessing_fork" + path = pathlib.Path(test_path) + if path.suffix == ".py": + test_name = path.stem + else: + test_name = path.name + test_to_lib[test_name] = lib_name + lib_test_order[lib_name].append(test_name) + + return test_to_lib, lib_test_order + + +def compute_test_todo_list( + cpython_prefix: str, + lib_prefix: str, + include_done: bool = False, + lib_status: dict[str, bool] | None = None, +) -> list[dict]: + """Compute prioritized list of tests to update. + + Scoring: + - If corresponding lib is up-to-date: score = 0 (ready) + - If no corresponding lib: score = 1 (independent) + - If corresponding lib is NOT up-to-date: score = 2 (wait for lib) + + Returns: + List of dicts with test info, sorted by priority + """ + all_tests = get_all_tests(cpython_prefix) + test_to_lib, lib_test_order = _build_test_to_lib_map(cpython_prefix) + + result = [] + for test_name in all_tests: + up_to_date = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) + + if up_to_date and not include_done: + continue + + tracked = is_test_tracked(test_name, cpython_prefix, lib_prefix) + + # Check DEPENDENCIES mapping first, then fall back to simple extraction + if test_name in test_to_lib: + lib_name = test_to_lib[test_name] + # Get order from DEPENDENCIES + test_order = lib_test_order[lib_name].index(test_name) + else: + # Extract lib name from test name (test_foo -> foo) + lib_name = test_name.removeprefix("test_") + test_order = 0 # Default order for tests not in DEPENDENCIES + + # Check if corresponding lib is up-to-date + # Scoring: 0 = lib ready (highest priority), 1 = no lib, 2 = lib pending + if lib_status and lib_name in lib_status: + lib_up_to_date = lib_status[lib_name] + if lib_up_to_date: + score = 0 # Lib is ready, can update test + else: + score = 2 # Wait for lib first + else: + score = 1 # No corresponding lib (independent test) + + todo_count = count_test_todos(test_name, lib_prefix) if tracked else 0 + + result.append( + { + "name": test_name, + "lib_name": lib_name, + "score": score, + "up_to_date": up_to_date, + "tracked": tracked, + "todo_count": todo_count, + "test_order": test_order, + } + ) + + # Sort by score (ascending) + result.sort(key=lambda x: x["score"]) + + return result + + +def _format_meta_suffix(item: dict) -> str: + """Format metadata suffix (last updated date and diff count).""" + parts = [] + last_updated = item.get("last_updated") + diff_lines = item.get("diff_lines", 0) + if last_updated: + parts.append(last_updated) + if diff_lines > 0: + parts.append(f"Δ{diff_lines}") + return f" | {' '.join(parts)}" if parts else "" + + +def _format_test_suffix(item: dict) -> str: + """Format suffix for test item (TODO count or untracked).""" + tracked = item.get("tracked", True) + if not tracked: + return " (untracked)" + todo_count = item.get("todo_count", 0) + if todo_count > 0: + return f" ({todo_count} TODO)" + return "" + + +def format_test_todo_list( + todo_list: list[dict], + limit: int | None = None, +) -> list[str]: + """Format test todo list for display. + + Groups tests by lib_name. If multiple tests share the same lib_name, + the first test is shown as the primary and others are indented below it. + """ + lines = [] + + if limit: + todo_list = todo_list[:limit] + + # Group by lib_name + grouped: dict[str, list[dict]] = {} + for item in todo_list: + lib_name = item.get("lib_name", item["name"]) + if lib_name not in grouped: + grouped[lib_name] = [] + grouped[lib_name].append(item) + + # Sort each group by test_order (from DEPENDENCIES) + for tests in grouped.values(): + tests.sort(key=lambda x: x.get("test_order", 0)) + + for lib_name, tests in grouped.items(): + # First test is the primary + primary = tests[0] + done_mark = "[x]" if primary["up_to_date"] else "[ ]" + suffix = _format_test_suffix(primary) + meta = _format_meta_suffix(primary) + lines.append(f"- {done_mark} {primary['name']}{suffix}{meta}") + + # Rest are indented + for item in tests[1:]: + done_mark = "[x]" if item["up_to_date"] else "[ ]" + suffix = _format_test_suffix(item) + meta = _format_meta_suffix(item) + lines.append(f" - {done_mark} {item['name']}{suffix}{meta}") + + return lines + + +def format_todo_list( + todo_list: list[dict], + test_by_lib: dict[str, list[dict]] | None = None, + limit: int | None = None, + verbose: bool = False, +) -> list[str]: + """Format todo list for display. + + Args: + todo_list: List from compute_todo_list() + test_by_lib: Dict mapping lib_name -> list of test infos (optional) + limit: Maximum number of items to show + verbose: Show detailed dependency information + + Returns: + List of formatted lines + """ + lines = [] + + if limit: + todo_list = todo_list[:limit] + + for item in todo_list: + name = item["name"] + score = item["score"] + total_deps = item["total_deps"] + rev_count = item["reverse_deps_count"] + + done_mark = "[x]" if item["up_to_date"] else "[ ]" + + if score == -1: + score_str = "no deps" + else: + score_str = f"{score}/{total_deps} deps" + + rev_str = f"{rev_count} dependents" if rev_count else "" + + parts = ["-", done_mark, f"[{score_str}]", f"`{name}`"] + if rev_str: + parts.append(f"({rev_str})") + + line = " ".join(parts) + _format_meta_suffix(item) + lines.append(line) + + # Show hard_deps: + # - Normal mode: only show if lib is up-to-date but hard_deps are not + # - Verbose mode: always show all hard_deps with their status + hard_deps_status = item.get("hard_deps_status", {}) + if verbose and hard_deps_status: + for hd in sorted(hard_deps_status.keys()): + hd_mark = "[x]" if hard_deps_status[hd] else "[ ]" + lines.append(f" - {hd_mark} {hd} (hard_dep)") + elif item["up_to_date"]: + for hd, ok in sorted(hard_deps_status.items()): + if not ok: + lines.append(f" - [ ] {hd} (hard_dep)") + + # Show corresponding tests if exist + if test_by_lib and name in test_by_lib: + for test_info in test_by_lib[name]: + test_done_mark = "[x]" if test_info["up_to_date"] else "[ ]" + suffix = _format_test_suffix(test_info) + meta = _format_meta_suffix(test_info) + lines.append(f" - {test_done_mark} {test_info['name']}{suffix}{meta}") + + # Verbose mode: show detailed dependency info + if verbose: + if item["reverse_deps"]: + lines.append(f" dependents: {', '.join(sorted(item['reverse_deps']))}") + if item["soft_deps"]: + lines.append(f" python: {', '.join(sorted(item['soft_deps']))}") + if item["native_deps"]: + lines.append(f" native: {', '.join(sorted(item['native_deps']))}") + + return lines + + +def format_all_todo( + cpython_prefix: str, + lib_prefix: str, + limit: int | None = None, + include_done: bool = False, + verbose: bool = False, +) -> list[str]: + """Format prioritized list of modules and tests to update. + + Returns: + List of formatted lines + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import is_up_to_date + + lines = [] + + # Build lib status map for test scoring + lib_status = {} + for name in get_all_modules(cpython_prefix): + lib_status[name] = is_up_to_date(name, cpython_prefix, lib_prefix) + + # Compute test todo (always include all to find libs with pending tests) + test_todo = compute_test_todo_list( + cpython_prefix, lib_prefix, include_done=True, lib_status=lib_status + ) + + # Build test_by_lib map (only for tests with corresponding lib) + test_by_lib: dict[str, list[dict]] = {} + no_lib_tests = [] + # Set of libs that have pending tests + libs_with_pending_tests = set() + for test in test_todo: + if test["score"] == 1: # no lib + if not test["up_to_date"] or include_done: + no_lib_tests.append(test) + else: + lib_name = test["lib_name"] + if lib_name not in test_by_lib: + test_by_lib[lib_name] = [] + test_by_lib[lib_name].append(test) + if not test["up_to_date"]: + libs_with_pending_tests.add(lib_name) + + # Sort each lib's tests by test_order (from DEPENDENCIES) + for tests in test_by_lib.values(): + tests.sort(key=lambda x: x.get("test_order", 0)) + + # Compute lib todo - include libs with pending tests even if lib is done + lib_todo_base = compute_todo_list(cpython_prefix, lib_prefix, include_done=True) + + # Filter lib todo: include if lib is not done OR has pending test + lib_todo = [] + for item in lib_todo_base: + lib_not_done = not item["up_to_date"] + has_pending_test = item["name"] in libs_with_pending_tests + + if include_done or lib_not_done or has_pending_test: + lib_todo.append(item) + + # Add metadata (last updated date and diff stat) to lib items + for item in lib_todo: + item["last_updated"] = get_module_last_updated( + item["name"], cpython_prefix, lib_prefix + ) + item["diff_lines"] = ( + 0 + if item["up_to_date"] + else get_module_diff_stat(item["name"], cpython_prefix, lib_prefix) + ) + + # Add last_updated to displayed test items (verbose only - slow) + if verbose: + for tests in test_by_lib.values(): + for test in tests: + test["last_updated"] = get_test_last_updated( + test["name"], cpython_prefix, lib_prefix + ) + for test in no_lib_tests: + test["last_updated"] = get_test_last_updated( + test["name"], cpython_prefix, lib_prefix + ) + + # Format lib todo with embedded tests + lines.extend(format_todo_list(lib_todo, test_by_lib, limit, verbose)) + + # Format "no lib" tests separately if any + if no_lib_tests: + lines.append("") + lines.append("## Standalone Tests") + lines.extend(format_test_todo_list(no_lib_tests, limit)) + + # Format untracked files (in cpython but not in our Lib) + untracked = get_untracked_files(cpython_prefix, lib_prefix) + if untracked: + lines.append("") + lines.append("## Untracked Files") + display_untracked = untracked[:limit] if limit else untracked + for path in display_untracked: + lines.append(f"- {path}") + if limit and len(untracked) > limit: + lines.append(f" ... and {len(untracked) - limit} more") + + # Format original files (in our Lib but not in cpython) + original = get_original_files(cpython_prefix, lib_prefix) + if original: + lines.append("") + lines.append("## Original Files") + display_original = original[:limit] if limit else original + for path in display_original: + lines.append(f"- {path}") + if limit and len(original) > limit: + lines.append(f" ... and {len(original) - limit} more") + + return lines + + +def show_todo( + cpython_prefix: str, + lib_prefix: str, + limit: int | None = None, + include_done: bool = False, + verbose: bool = False, +) -> None: + """Show prioritized list of modules and tests to update.""" + for line in format_all_todo( + cpython_prefix, lib_prefix, limit, include_done, verbose + ): + print(line) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--cpython", + default="cpython", + help="CPython directory prefix (default: cpython)", + ) + parser.add_argument( + "--lib", + default="Lib", + help="Local Lib directory prefix (default: Lib)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Maximum number of items to show", + ) + parser.add_argument( + "--done", + action="store_true", + help="Include already up-to-date modules", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Show detailed dependency information", + ) + + args = parser.parse_args(argv) + + try: + show_todo(args.cpython, args.lib, args.limit, args.done, args.verbose) + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py new file mode 100644 index 00000000000..58b259c8a14 --- /dev/null +++ b/scripts/update_lib/deps.py @@ -0,0 +1,1598 @@ +""" +Dependency resolution for library updates. + +Handles: +- Irregular library paths (e.g., libregrtest at Lib/test/libregrtest/) +- Library dependencies (e.g., datetime requires _pydatetime) +- Test dependencies (auto-detected from 'from test import ...') +""" + +import ast +import difflib +import functools +import pathlib +import re +import shelve +import subprocess + +from update_lib.file_utils import ( + _dircmp_is_same, + compare_dir_contents, + compare_file_contents, + compare_paths, + construct_lib_path, + cpython_to_local_path, + read_python_files, + resolve_module_path, + resolve_test_path, + safe_parse_ast, + safe_read_text, +) + +# === Import parsing utilities === + + +def _extract_top_level_code(content: str) -> str: + """Extract only top-level code from Python content for faster parsing.""" + def_idx = content.find("\ndef ") + class_idx = content.find("\nclass ") + + indices = [i for i in (def_idx, class_idx) if i != -1] + if indices: + content = content[: min(indices)] + return content.rstrip("\n") + + +_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE) +_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE) +_IMPORT_TEST_DOT_RE = re.compile(r"^import test\.(\w+)", re.MULTILINE) + + +def parse_test_imports(content: str) -> set[str]: + """Parse test file content and extract test package dependencies.""" + content = _extract_top_level_code(content) + imports = set() + + for match in _FROM_TEST_IMPORT_RE.finditer(content): + import_list = match.group(1) + for part in import_list.split(","): + name = part.split()[0].strip() + if name and name not in ("support", "__init__"): + imports.add(name) + + for match in _FROM_TEST_DOT_RE.finditer(content): + dep = match.group(1) + if dep not in ("support", "__init__"): + imports.add(dep) + + for match in _IMPORT_TEST_DOT_RE.finditer(content): + dep = match.group(1) + if dep not in ("support", "__init__"): + imports.add(dep) + + return imports + + +_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE) +_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE) + + +def parse_lib_imports(content: str) -> set[str]: + """Parse library file and extract all imported module names.""" + imports = set() + + for match in _IMPORT_RE.finditer(content): + imports.add(match.group(1)) + + for match in _FROM_IMPORT_RE.finditer(content): + imports.add(match.group(1)) + + return imports + + +# === TODO marker utilities === + +TODO_MARKER = "TODO: RUSTPYTHON" + + +def filter_rustpython_todo(content: str) -> str: + """Remove lines containing RustPython TODO markers.""" + lines = content.splitlines(keepends=True) + filtered = [line for line in lines if TODO_MARKER not in line] + return "".join(filtered) + + +def count_rustpython_todo(content: str) -> int: + """Count lines containing RustPython TODO markers.""" + return sum(1 for line in content.splitlines() if TODO_MARKER in line) + + +def count_todo_in_path(path: pathlib.Path) -> int: + """Count RustPython TODO markers in a file or directory of .py files.""" + if path.is_file(): + content = safe_read_text(path) + return count_rustpython_todo(content) if content else 0 + + total = 0 + for _, content in read_python_files(path): + total += count_rustpython_todo(content) + return total + + +# === Test utilities === + + +def _get_cpython_test_path(test_name: str, cpython_prefix: str) -> pathlib.Path | None: + """Return the CPython test path for a test name, or None if missing.""" + cpython_path = resolve_test_path(test_name, cpython_prefix, prefer="dir") + return cpython_path if cpython_path.exists() else None + + +def _get_local_test_path( + cpython_test_path: pathlib.Path, lib_prefix: str +) -> pathlib.Path: + """Return the local Lib/test path matching a CPython test path.""" + return pathlib.Path(lib_prefix) / "test" / cpython_test_path.name + + +def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test exists in the local Lib/test.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + local_path = _get_local_test_path(cpython_path, lib_prefix) + return local_path.exists() + + +def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test is up-to-date, ignoring RustPython TODO markers.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + + local_path = _get_local_test_path(cpython_path, lib_prefix) + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return compare_file_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + return compare_dir_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + +def count_test_todos(test_name: str, lib_prefix: str) -> int: + """Count RustPython TODO markers in a test file/directory.""" + local_dir = pathlib.Path(lib_prefix) / "test" / test_name + local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py" + + if local_dir.exists(): + return count_todo_in_path(local_dir) + if local_file.exists(): + return count_todo_in_path(local_file) + return 0 + + +# === Cross-process cache using shelve === + + +def _get_cpython_version(cpython_prefix: str) -> str: + """Get CPython version from git tag for cache namespace.""" + try: + result = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0"], + cwd=cpython_prefix, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + return "unknown" + + +def _get_cache_path() -> str: + """Get cache file path (without extension - shelve adds its own).""" + cache_dir = pathlib.Path(__file__).parent / ".cache" + cache_dir.mkdir(parents=True, exist_ok=True) + return str(cache_dir / "import_graph_cache") + + +def clear_import_graph_caches() -> None: + """Clear in-process import graph caches (for testing).""" + if "_test_import_graph_cache" in globals(): + globals()["_test_import_graph_cache"].clear() + if "_lib_import_graph_cache" in globals(): + globals()["_lib_import_graph_cache"].clear() + + +# Manual dependency table for irregular cases +# Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]} +# - lib: override default path (default: name.py or name/) +# - hard_deps: additional files to copy alongside the main module +DEPENDENCIES = { + # regrtest is in Lib/test/libregrtest/, not Lib/libregrtest/ + "regrtest": { + "lib": ["test/libregrtest"], + "test": ["test_regrtest"], + "data": ["test/regrtestdata"], + }, + # Rust-implemented modules (no lib file, only test) + "int": { + "lib": [], + "hard_deps": ["_pylong.py"], + "test": [ + "test_int.py", + "test_long.py", + "test_int_literal.py", + ], + }, + "exception": { + "lib": [], + "test": [ + "test_exceptions.py", + "test_baseexception.py", + "test_except_star.py", + "test_exception_group.py", + "test_exception_hierarchy.py", + "test_exception_variations.py", + ], + }, + "dict": { + "lib": [], + "test": [ + "test_dict.py", + "test_dictcomps.py", + "test_dictviews.py", + "test_userdict.py", + ], + }, + "list": { + "lib": [], + "test": [ + "test_list.py", + "test_listcomps.py", + "test_userlist.py", + ], + }, + "__future__": { + "test": [ + "test___future__.py", + "test_future_stmt.py", + ], + }, + "site": { + "hard_deps": ["_sitebuiltins.py"], + }, + "opcode": { + "hard_deps": ["_opcode_metadata.py"], + "test": [ + "test_opcode.py", + "test__opcode.py", + "test_opcodes.py", + ], + }, + "pickle": { + "hard_deps": ["_compat_pickle.py"], + "test": [ + "test_pickle.py", + "test_picklebuffer.py", + "test_pickletools.py", + ], + }, + "re": { + "hard_deps": ["sre_compile.py", "sre_constants.py", "sre_parse.py"], + }, + "weakref": { + "hard_deps": ["_weakrefset.py"], + "test": [ + "test_weakref.py", + "test_weakset.py", + ], + }, + "codecs": { + "test": [ + "test_codecs.py", + "test_codeccallbacks.py", + "test_codecencodings_cn.py", + "test_codecencodings_hk.py", + "test_codecencodings_iso2022.py", + "test_codecencodings_jp.py", + "test_codecencodings_kr.py", + "test_codecencodings_tw.py", + "test_codecmaps_cn.py", + "test_codecmaps_hk.py", + "test_codecmaps_jp.py", + "test_codecmaps_kr.py", + "test_codecmaps_tw.py", + "test_charmapcodec.py", + "test_multibytecodec.py", + ], + }, + # Non-pattern hard_deps (can't be auto-detected) + "ast": { + "hard_deps": ["_ast_unparse.py"], + "test": [ + "test_ast.py", + "test_unparse.py", + "test_type_comments.py", + ], + }, + # Data directories + "pydoc": { + "hard_deps": ["pydoc_data"], + }, + "turtle": { + "hard_deps": ["turtledemo"], + }, + "sysconfig": { + "hard_deps": ["_aix_support.py", "_osx_support.py"], + "test": [ + "test_sysconfig.py", + "test__osx_support.py", + ], + }, + "tkinter": { + "test": [ + "test_tkinter", + "test_ttk", + "test_ttk_textonly.py", + "test_tcl.py", + "test_idle", + ], + }, + # Test support library (like regrtest) + "support": { + "lib": ["test/support"], + "data": ["test/wheeldata"], + "test": [ + "test_support.py", + "test_script_helper.py", + ], + }, + # test_htmlparser tests html.parser + "html": { + "hard_deps": ["_markupbase.py"], + "test": ["test_html.py", "test_htmlparser.py"], + }, + "xml": { + "test": [ + "test_xml_etree.py", + "test_xml_etree_c.py", + "test_minidom.py", + "test_pulldom.py", + "test_pyexpat.py", + "test_sax.py", + "test_xml_dom_minicompat.py", + "test_xml_dom_xmlbuilder.py", + ], + }, + "multiprocessing": { + "test": [ + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", + "test_multiprocessing_main_handling.py", + ], + }, + "urllib": { + "test": [ + "test_urllib.py", + "test_urllib2.py", + "test_urllib2_localnet.py", + "test_urllib2net.py", + "test_urllibnet.py", + "test_urlparse.py", + "test_urllib_response.py", + "test_robotparser.py", + ], + }, + "collections": { + "hard_deps": ["_collections_abc.py"], + "test": [ + "test_collections.py", + "test_deque.py", + "test_defaultdict.py", + "test_ordered_dict.py", + ], + }, + "http": { + "test": [ + "test_httplib.py", + "test_http_cookiejar.py", + "test_http_cookies.py", + "test_httpservers.py", + ], + }, + "unicode": { + "lib": [], + "test": [ + "test_unicodedata.py", + "test_unicode_file.py", + "test_unicode_file_functions.py", + "test_unicode_identifiers.py", + "test_ucn.py", + ], + }, + "typing": { + "test": [ + "test_typing.py", + "test_type_aliases.py", + "test_type_annotations.py", + "test_type_params.py", + "test_genericalias.py", + ], + }, + "unpack": { + "lib": [], + "test": [ + "test_unpack.py", + "test_unpack_ex.py", + ], + }, + "zipimport": { + "test": [ + "test_zipimport.py", + "test_zipimport_support.py", + ], + }, + "time": { + "lib": [], + "test": [ + "test_time.py", + "test_strftime.py", + ], + }, + "sys": { + "lib": [], + "test": [ + "test_sys.py", + "test_syslog.py", + "test_sys_setprofile.py", + "test_sys_settrace.py", + ], + }, + "str": { + "lib": [], + "test": [ + "test_str.py", + "test_fstring.py", + "test_string_literals.py", + ], + }, + "thread": { + "lib": [], + "test": [ + "test_thread.py", + "test_thread_local_bytecode.py", + "test_threadsignals.py", + ], + }, + "threading": { + "hard_deps": ["_threading_local.py"], + "test": [ + "test_threading.py", + "test_threadedtempfile.py", + "test_threading_local.py", + ], + }, + "class": { + "lib": [], + "test": [ + "test_class.py", + "test_genericclass.py", + "test_subclassinit.py", + ], + }, + "generator": { + "lib": [], + "test": [ + "test_generators.py", + "test_genexps.py", + "test_generator_stop.py", + "test_yield_from.py", + ], + }, + "descr": { + "lib": [], + "test": [ + "test_descr.py", + "test_descrtut.py", + ], + }, + "code": { + "test": [ + "test_code_module.py", + ], + }, + "contextlib": { + "test": [ + "test_contextlib.py", + "test_contextlib_async.py", + ], + }, + "io": { + "hard_deps": ["_pyio.py"], + "test": [ + "test_io.py", + "test_bufio.py", + "test_fileio.py", + "test_memoryio.py", + ], + }, + "dbm": { + "test": [ + "test_dbm.py", + "test_dbm_dumb.py", + "test_dbm_gnu.py", + "test_dbm_ndbm.py", + "test_dbm_sqlite3.py", + ], + }, + "datetime": { + "hard_deps": ["_strptime.py"], + "test": [ + "test_datetime.py", + "test_strptime.py", + ], + }, + "locale": { + "test": [ + "test_locale.py", + "test__locale.py", + ], + }, + "numbers": { + "test": [ + "test_numbers.py", + "test_abstract_numbers.py", + ], + }, + "file": { + "lib": [], + "test": [ + "test_file.py", + "test_largefile.py", + ], + }, + "fcntl": { + "lib": [], + "test": [ + "test_fcntl.py", + "test_ioctl.py", + ], + }, + "select": { + "lib": [], + "test": [ + "test_select.py", + "test_poll.py", + ], + }, + "xmlrpc": { + "test": [ + "test_xmlrpc.py", + "test_docxmlrpc.py", + ], + }, + "ctypes": { + "test": [ + "test_ctypes", + "test_stable_abi_ctypes.py", + ], + }, + # Grouped tests for modules without custom lib paths + "compile": { + "lib": [], + "test": [ + "test_compile.py", + "test_compiler_assemble.py", + "test_compiler_codegen.py", + "test_peepholer.py", + ], + }, + "math": { + "lib": [], + "test": [ + "test_math.py", + "test_math_property.py", + ], + }, + "float": { + "lib": [], + "test": [ + "test_float.py", + "test_strtod.py", + ], + }, + "zipfile": { + "test": [ + "test_zipfile.py", + "test_zipfile64.py", + ], + }, + "smtplib": { + "test": [ + "test_smtplib.py", + "test_smtpnet.py", + ], + }, + "profile": { + "test": [ + "test_profile.py", + "test_cprofile.py", + ], + }, + "string": { + "test": [ + "test_string.py", + "test_userstring.py", + ], + }, + "os": { + "test": [ + "test_os.py", + "test_popen.py", + ], + }, + "pyrepl": { + "test": [ + "test_pyrepl", + "test_repl.py", + ], + }, + "concurrent": { + "test": [ + "test_concurrent_futures", + "test_interpreters", + "test__interpreters.py", + "test__interpchannels.py", + "test_crossinterp.py", + ], + }, +} + + +def resolve_hard_dep_parent(name: str, cpython_prefix: str) -> str | None: + """Resolve a hard_dep name to its parent module. + + Only returns a parent if the file is actually tracked: + - Explicitly listed in DEPENDENCIES as a hard_dep + - Or auto-detected _py{module}.py pattern where the parent module exists + + Args: + name: Module or file name (with or without .py extension) + cpython_prefix: CPython directory prefix + + Returns: + Parent module name if found and tracked, None otherwise + """ + # Normalize: remove .py extension if present + if name.endswith(".py"): + name = name[:-3] + + # Check DEPENDENCIES table first (explicit hard_deps) + for module_name, dep_info in DEPENDENCIES.items(): + hard_deps = dep_info.get("hard_deps", []) + for dep in hard_deps: + # Normalize dep: remove .py extension + dep_normalized = dep[:-3] if dep.endswith(".py") else dep + if dep_normalized == name: + return module_name + + # Auto-detect _py{module} or _py_{module} patterns + # Only if the parent module actually exists + if name.startswith("_py"): + if name.startswith("_py_"): + # _py_abc -> abc + parent = name[4:] + else: + # _pydatetime -> datetime + parent = name[3:] + + # Verify the parent module exists + lib_dir = pathlib.Path(cpython_prefix) / "Lib" + parent_file = lib_dir / f"{parent}.py" + parent_dir = lib_dir / parent + if parent_file.exists() or ( + parent_dir.exists() and (parent_dir / "__init__.py").exists() + ): + return parent + + return None + + +def resolve_test_to_lib(test_name: str) -> str | None: + """Resolve a test name to its library group from DEPENDENCIES. + + Args: + test_name: Test name with or without test_ prefix (e.g., "test_urllib2" or "urllib2") + + Returns: + Library name if test belongs to a group, None otherwise + """ + # Normalize: add test_ prefix if not present + if not test_name.startswith("test_"): + test_name = f"test_{test_name}" + + for lib_name, dep_info in DEPENDENCIES.items(): + tests = dep_info.get("test", []) + for test_path in tests: + # test_path is like "test_urllib2.py" or "test_multiprocessing_fork" + path_stem = test_path[:-3] if test_path.endswith(".py") else test_path + if path_stem == test_name: + return lib_name + + return None + + +# Test-specific dependencies (only when auto-detection isn't enough) +# - hard_deps: files to migrate (tightly coupled, must be migrated together) +# - data: directories to copy without migration +TEST_DEPENDENCIES = { + # Audio tests + "test_winsound": { + "data": ["audiodata"], + }, + "test_wave": { + "data": ["audiodata"], + }, + "audiotests": { + "data": ["audiodata"], + }, + # Archive tests + "test_tarfile": { + "data": ["archivetestdata"], + }, + "test_zipfile": { + "data": ["archivetestdata"], + }, + # Config tests + "test_configparser": { + "data": ["configdata"], + }, + "test_config": { + "data": ["configdata"], + }, + # Other data directories + "test_decimal": { + "data": ["decimaltestdata"], + }, + "test_dtrace": { + "data": ["dtracedata"], + }, + "test_math": { + "data": ["mathdata"], + }, + "test_ssl": { + "data": ["certdata"], + }, + "test_subprocess": { + "data": ["subprocessdata"], + }, + "test_tkinter": { + "data": ["tkinterdata"], + }, + "test_tokenize": { + "data": ["tokenizedata"], + }, + "test_type_annotations": { + "data": ["typinganndata"], + }, + "test_zipimport": { + "data": ["zipimport_data"], + }, + # XML tests share xmltestdata + "test_xml_etree": { + "data": ["xmltestdata"], + }, + "test_pulldom": { + "data": ["xmltestdata"], + }, + "test_sax": { + "data": ["xmltestdata"], + }, + "test_minidom": { + "data": ["xmltestdata"], + }, + # Multibytecodec support needs cjkencodings + "multibytecodec_support": { + "data": ["cjkencodings"], + }, + # i18n + "i18n_helper": { + "data": ["translationdata"], + }, + # wheeldata is used by test_makefile and support + "test_makefile": { + "data": ["wheeldata"], + }, +} + + +@functools.cache +def get_lib_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]: + """Get all library paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + Tuple of paths to copy + """ + dep_info = DEPENDENCIES.get(name, {}) + + # Get main lib path (override or default) + if "lib" in dep_info: + paths = [construct_lib_path(cpython_prefix, p) for p in dep_info["lib"]] + else: + # Default: try file first, then directory + paths = [resolve_module_path(name, cpython_prefix, prefer="file")] + + # Add hard_deps from DEPENDENCIES + for dep in dep_info.get("hard_deps", []): + paths.append(construct_lib_path(cpython_prefix, dep)) + + # Auto-detect _py{module}.py or _py_{module}.py patterns + for pattern in [f"_py{name}.py", f"_py_{name}.py"]: + auto_path = construct_lib_path(cpython_prefix, pattern) + if auto_path.exists() and auto_path not in paths: + paths.append(auto_path) + + return tuple(paths) + + +def get_all_hard_deps(name: str, cpython_prefix: str) -> list[str]: + """Get all hard_deps for a module (explicit + auto-detected). + + Args: + name: Module name (e.g., "decimal", "datetime") + cpython_prefix: CPython directory prefix + + Returns: + List of hard_dep names (without .py extension) + """ + dep_info = DEPENDENCIES.get(name, {}) + hard_deps = set() + + # Explicit hard_deps from DEPENDENCIES + for hd in dep_info.get("hard_deps", []): + # Remove .py extension if present + hard_deps.add(hd[:-3] if hd.endswith(".py") else hd) + + # Auto-detect _py{module}.py or _py_{module}.py patterns + for pattern in [f"_py{name}.py", f"_py_{name}.py"]: + auto_path = construct_lib_path(cpython_prefix, pattern) + if auto_path.exists(): + hard_deps.add(auto_path.stem) + + return sorted(hard_deps) + + +@functools.cache +def get_test_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]: + """Get all test paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + Tuple of test paths + """ + if name in DEPENDENCIES and "test" in DEPENDENCIES[name]: + return tuple( + construct_lib_path(cpython_prefix, f"test/{p}") + for p in DEPENDENCIES[name]["test"] + ) + + # Default: try directory first, then file + return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),) + + +@functools.cache +def get_all_imports(name: str, cpython_prefix: str) -> frozenset[str]: + """Get all imports from a library file. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of all imported module names + """ + all_imports = set() + for lib_path in get_lib_paths(name, cpython_prefix): + if lib_path.exists(): + for _, content in read_python_files(lib_path): + all_imports.update(parse_lib_imports(content)) + + # Remove self + all_imports.discard(name) + return frozenset(all_imports) + + +@functools.cache +def get_soft_deps(name: str, cpython_prefix: str) -> frozenset[str]: + """Get soft dependencies by parsing imports from library file. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of imported stdlib module names (those that exist in cpython/Lib/) + """ + all_imports = get_all_imports(name, cpython_prefix) + + # Filter: only include modules that exist in cpython/Lib/ + stdlib_deps = set() + for imp in all_imports: + module_path = resolve_module_path(imp, cpython_prefix) + if module_path.exists(): + stdlib_deps.add(imp) + + return frozenset(stdlib_deps) + + +@functools.cache +def get_rust_deps(name: str, cpython_prefix: str) -> frozenset[str]: + """Get Rust/C dependencies (imports that don't exist in cpython/Lib/). + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of imported module names that are built-in or C extensions + """ + all_imports = get_all_imports(name, cpython_prefix) + soft_deps = get_soft_deps(name, cpython_prefix) + return frozenset(all_imports - soft_deps) + + +def is_path_synced( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> bool: + """Check if a CPython path is synced with local. + + Args: + cpython_path: Path in CPython directory + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + True if synced, False otherwise + """ + local_path = cpython_to_local_path(cpython_path, cpython_prefix, lib_prefix) + if local_path is None: + return False + return compare_paths(cpython_path, local_path) + + +@functools.cache +def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a module is up-to-date by comparing files. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + True if all files match, False otherwise + """ + lib_paths = get_lib_paths(name, cpython_prefix) + + found_any = False + for cpython_path in lib_paths: + if not cpython_path.exists(): + continue + + found_any = True + + # Convert cpython path to local path + # cpython/Lib/foo.py -> Lib/foo.py + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + + if not compare_paths(cpython_path, local_path): + return False + + if not found_any: + dep_info = DEPENDENCIES.get(name, {}) + if dep_info.get("lib") == []: + return True + return found_any + + +def _count_file_diff(file_a: pathlib.Path, file_b: pathlib.Path) -> int: + """Count changed lines between two text files using difflib.""" + a_content = safe_read_text(file_a) + b_content = safe_read_text(file_b) + if a_content is None or b_content is None: + return 0 + if a_content == b_content: + return 0 + a_lines = a_content.splitlines() + b_lines = b_content.splitlines() + count = 0 + for line in difflib.unified_diff(a_lines, b_lines, lineterm=""): + if (line.startswith("+") and not line.startswith("+++")) or ( + line.startswith("-") and not line.startswith("---") + ): + count += 1 + return count + + +def _count_path_diff(path_a: pathlib.Path, path_b: pathlib.Path) -> int: + """Count changed lines between two paths (file or directory, *.py only).""" + if path_a.is_file() and path_b.is_file(): + return _count_file_diff(path_a, path_b) + if path_a.is_dir() and path_b.is_dir(): + total = 0 + a_files = {f.relative_to(path_a) for f in path_a.rglob("*.py")} + b_files = {f.relative_to(path_b) for f in path_b.rglob("*.py")} + for rel in a_files & b_files: + total += _count_file_diff(path_a / rel, path_b / rel) + for rel in a_files - b_files: + content = safe_read_text(path_a / rel) + if content: + total += len(content.splitlines()) + for rel in b_files - a_files: + content = safe_read_text(path_b / rel) + if content: + total += len(content.splitlines()) + return total + return 0 + + +def get_module_last_updated( + name: str, cpython_prefix: str, lib_prefix: str +) -> str | None: + """Get the last git commit date for a module's Lib files.""" + local_paths = [] + for cpython_path in get_lib_paths(name, cpython_prefix): + if not cpython_path.exists(): + continue + try: + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + if local_path.exists(): + local_paths.append(str(local_path)) + except ValueError: + continue + if not local_paths: + return None + try: + result = subprocess.run( + ["git", "log", "-1", "--format=%cd", "--date=short", "--"] + local_paths, + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip() + except Exception: + pass + return None + + +def get_module_diff_stat(name: str, cpython_prefix: str, lib_prefix: str) -> int: + """Count differing lines between cpython and local Lib for a module.""" + total = 0 + for cpython_path in get_lib_paths(name, cpython_prefix): + if not cpython_path.exists(): + continue + try: + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + continue + if not local_path.exists(): + continue + total += _count_path_diff(cpython_path, local_path) + return total + + +def get_test_last_updated( + test_name: str, cpython_prefix: str, lib_prefix: str +) -> str | None: + """Get the last git commit date for a test's files.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return None + local_path = _get_local_test_path(cpython_path, lib_prefix) + if not local_path.exists(): + return None + try: + result = subprocess.run( + ["git", "log", "-1", "--format=%cd", "--date=short", "--", str(local_path)], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip() + except Exception: + pass + return None + + +def get_test_dependencies( + test_path: pathlib.Path, +) -> dict[str, list[pathlib.Path]]: + """Get test dependencies by parsing imports. + + Args: + test_path: Path to test file or directory + + Returns: + Dict with "hard_deps" (files to migrate) and "data" (dirs to copy) + """ + result = {"hard_deps": [], "data": []} + + if not test_path.exists(): + return result + + # Parse all files for imports (auto-detect deps) + all_imports = set() + for _, content in read_python_files(test_path): + all_imports.update(parse_test_imports(content)) + + # Also add manual dependencies from TEST_DEPENDENCIES + test_name = test_path.stem if test_path.is_file() else test_path.name + manual_deps = TEST_DEPENDENCIES.get(test_name, {}) + if "hard_deps" in manual_deps: + all_imports.update(manual_deps["hard_deps"]) + + # Convert imports to paths (deps) + for imp in all_imports: + # Skip other test modules (test_*) - they are independently managed + # via their own update_lib entry. Only support/helper modules + # (e.g., string_tests, mapping_tests) should be treated as hard deps. + if imp.startswith("test_"): + continue + + dep_path = test_path.parent / f"{imp}.py" + if not dep_path.exists(): + dep_path = test_path.parent / imp + + if dep_path.exists() and dep_path not in result["hard_deps"]: + result["hard_deps"].append(dep_path) + + # Add data paths from manual table (for the test file itself) + if "data" in manual_deps: + for data_name in manual_deps["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + # Also add data from auto-detected deps' TEST_DEPENDENCIES + # e.g., test_codecencodings_kr -> multibytecodec_support -> cjkencodings + for imp in all_imports: + dep_info = TEST_DEPENDENCIES.get(imp, {}) + if "data" in dep_info: + for data_name in dep_info["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + return result + + +def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: + """Parse 'from test.X import Y' to get submodule imports. + + Args: + content: Python file content + + Returns: + Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) + """ + tree = safe_parse_ast(content) + if tree is None: + return {} + + result: dict[str, set[str]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module and node.module.startswith("test."): + # from test.test_bar import helper -> test_bar: {helper} + parts = node.module.split(".") + if len(parts) >= 2: + submodule = parts[1] + if submodule not in ("support", "__init__"): + if submodule not in result: + result[submodule] = set() + for alias in node.names: + result[submodule].add(alias.name) + + return result + + +_test_import_graph_cache: dict[ + str, tuple[dict[str, set[str]], dict[str, set[str]]] +] = {} + + +def _is_standard_lib_path(path: str) -> bool: + """Check if path is the standard Lib directory (not a temp dir).""" + if "/tmp" in path.lower() or "/var/folders" in path.lower(): + return False + return ( + path == "Lib/test" + or path.endswith("/Lib/test") + or path == "Lib" + or path.endswith("/Lib") + ) + + +def _build_test_import_graph( + test_dir: pathlib.Path, +) -> tuple[dict[str, set[str]], dict[str, set[str]]]: + """Build import graphs for files within test directory (recursive). + + Uses cross-process shelve cache based on CPython version. + + Args: + test_dir: Path to Lib/test/ directory + + Returns: + Tuple of: + - Dict mapping relative path (without .py) -> set of test modules it imports + - Dict mapping relative path (without .py) -> set of all lib imports + """ + # In-process cache + cache_key = str(test_dir) + if cache_key in _test_import_graph_cache: + return _test_import_graph_cache[cache_key] + + # Cross-process cache (only for standard Lib/test directory) + use_file_cache = _is_standard_lib_path(cache_key) + if use_file_cache: + version = _get_cpython_version("cpython") + shelve_key = f"test_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph, lib_imports_graph = db[shelve_key] + _test_import_graph_cache[cache_key] = ( + import_graph, + lib_imports_graph, + ) + return import_graph, lib_imports_graph + except Exception: + pass + + # Build from scratch + import_graph: dict[str, set[str]] = {} + lib_imports_graph: dict[str, set[str]] = {} + + for py_file in test_dir.glob("**/*.py"): + content = safe_read_text(py_file) + if content is None: + continue + + imports = set() + imports.update(parse_test_imports(content)) + all_imports = parse_lib_imports(content) + + for imp in all_imports: + if (py_file.parent / f"{imp}.py").exists(): + imports.add(imp) + if (test_dir / f"{imp}.py").exists(): + imports.add(imp) + + submodule_imports = _parse_test_submodule_imports(content) + for submodule, imported_names in submodule_imports.items(): + submodule_dir = test_dir / submodule + if submodule_dir.is_dir(): + for name in imported_names: + if (submodule_dir / f"{name}.py").exists(): + imports.add(name) + + rel_path = py_file.relative_to(test_dir) + key = str(rel_path.with_suffix("")) + import_graph[key] = imports + lib_imports_graph[key] = all_imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = (import_graph, lib_imports_graph) + except Exception: + pass + _test_import_graph_cache[cache_key] = (import_graph, lib_imports_graph) + + return import_graph, lib_imports_graph + + +_lib_import_graph_cache: dict[str, dict[str, set[str]]] = {} + + +def _build_lib_import_graph(lib_prefix: str) -> dict[str, set[str]]: + """Build import graph for Lib modules (full module paths like urllib.request). + + Uses cross-process shelve cache based on CPython version. + + Args: + lib_prefix: RustPython Lib directory + + Returns: + Dict mapping full_module_path -> set of modules it imports + """ + # In-process cache + if lib_prefix in _lib_import_graph_cache: + return _lib_import_graph_cache[lib_prefix] + + # Cross-process cache (only for standard Lib directory) + use_file_cache = _is_standard_lib_path(lib_prefix) + if use_file_cache: + version = _get_cpython_version("cpython") + shelve_key = f"lib_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph = db[shelve_key] + _lib_import_graph_cache[lib_prefix] = import_graph + return import_graph + except Exception: + pass + + # Build from scratch + lib_dir = pathlib.Path(lib_prefix) + if not lib_dir.exists(): + return {} + + import_graph: dict[str, set[str]] = {} + + for entry in lib_dir.iterdir(): + if entry.name.startswith(("_", ".")): + continue + if entry.name == "test": + continue + + if entry.is_file() and entry.suffix == ".py": + content = safe_read_text(entry) + if content: + imports = parse_lib_imports(content) + imports.discard(entry.stem) + import_graph[entry.stem] = imports + elif entry.is_dir() and (entry / "__init__.py").exists(): + for py_file in entry.glob("**/*.py"): + content = safe_read_text(py_file) + if content: + imports = parse_lib_imports(content) + rel_path = py_file.relative_to(lib_dir) + if rel_path.name == "__init__.py": + full_name = str(rel_path.parent).replace("/", ".") + else: + full_name = str(rel_path.with_suffix("")).replace("/", ".") + imports.discard(full_name.split(".")[0]) + import_graph[full_name] = imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = import_graph + except Exception: + pass + _lib_import_graph_cache[lib_prefix] = import_graph + + return import_graph + + +def _get_lib_modules_importing( + module_name: str, lib_import_graph: dict[str, set[str]] +) -> set[str]: + """Find Lib modules (full paths) that import module_name or any of its submodules.""" + importers: set[str] = set() + target_top = module_name.split(".")[0] + + for full_path, imports in lib_import_graph.items(): + if full_path.split(".")[0] == target_top: + continue # Skip same package + # Match if module imports target OR any submodule of target + # e.g., for "xml": match imports of "xml", "xml.parsers", "xml.etree.ElementTree" + matches = any( + imp == module_name or imp.startswith(module_name + ".") for imp in imports + ) + if matches: + importers.add(full_path) + + return importers + + +def _consolidate_submodules( + modules: set[str], threshold: int = 3 +) -> dict[str, set[str]]: + """Consolidate submodules if count exceeds threshold. + + Args: + modules: Set of full module paths (e.g., {"urllib.request", "urllib.parse", "xml.dom", "xml.sax"}) + threshold: If submodules > threshold, consolidate to parent + + Returns: + Dict mapping display_name -> set of original module paths + e.g., {"urllib.request": {"urllib.request"}, "xml": {"xml.dom", "xml.sax", "xml.etree", "xml.parsers"}} + """ + # Group by top-level package + by_package: dict[str, set[str]] = {} + for mod in modules: + parts = mod.split(".") + top = parts[0] + if top not in by_package: + by_package[top] = set() + by_package[top].add(mod) + + result: dict[str, set[str]] = {} + for top, submods in by_package.items(): + if len(submods) > threshold: + # Consolidate to top-level + result[top] = submods + else: + # Keep individual + for mod in submods: + result[mod] = {mod} + + return result + + +# Modules that are used everywhere - show but don't expand their dependents +_BLOCKLIST_MODULES = frozenset( + { + "unittest", + "test.support", + "support", + "doctest", + "typing", + "abc", + "collections.abc", + "functools", + "itertools", + "operator", + "contextlib", + "warnings", + "types", + "enum", + "re", + "io", + "os", + "sys", + } +) + + +def find_dependent_tests_tree( + module_name: str, + lib_prefix: str, + max_depth: int = 1, + _depth: int = 0, + _visited_tests: set[str] | None = None, + _visited_modules: set[str] | None = None, +) -> dict: + """Find dependent tests in a tree structure. + + Args: + module_name: Module to search for (e.g., "ftplib") + lib_prefix: RustPython Lib directory + max_depth: Maximum depth to recurse (default 1 = show direct + 1 level of Lib deps) + + Returns: + Dict with structure: + { + "module": "ftplib", + "tests": ["test_ftplib", "test_urllib2"], # Direct importers + "children": [ + {"module": "urllib.request", "tests": [...], "children": []}, + ... + ] + } + """ + lib_dir = pathlib.Path(lib_prefix) + test_dir = lib_dir / "test" + + if _visited_tests is None: + _visited_tests = set() + if _visited_modules is None: + _visited_modules = set() + + # Build graphs + test_import_graph, test_lib_imports = _build_test_import_graph(test_dir) + lib_import_graph = _build_lib_import_graph(lib_prefix) + + # Find tests that directly import this module + target_top = module_name.split(".")[0] + direct_tests: set[str] = set() + for file_key, imports in test_lib_imports.items(): + if file_key in _visited_tests: + continue + # Match exact module OR any child submodule + # e.g., "xml" matches imports of "xml", "xml.parsers", "xml.etree.ElementTree" + # but "collections._defaultdict" only matches "collections._defaultdict" (no children) + matches = any( + imp == module_name or imp.startswith(module_name + ".") for imp in imports + ) + if matches: + # Check if it's a test file + if pathlib.Path(file_key).name.startswith("test_"): + direct_tests.add(file_key) + _visited_tests.add(file_key) + + # Consolidate test names (test_sqlite3/test_dbapi -> test_sqlite3) + consolidated_tests = {_consolidate_file_key(t) for t in direct_tests} + + # Mark this module as visited (cycle detection) + _visited_modules.add(module_name) + _visited_modules.add(target_top) + + children = [] + # Check blocklist and depth limit + should_expand = ( + _depth < max_depth + and module_name not in _BLOCKLIST_MODULES + and target_top not in _BLOCKLIST_MODULES + ) + + if should_expand: + # Find Lib modules that import this module + lib_importers = _get_lib_modules_importing(module_name, lib_import_graph) + + # Skip already visited modules (cycle detection) and blocklisted modules + lib_importers = { + m + for m in lib_importers + if m not in _visited_modules + and m.split(".")[0] not in _visited_modules + and m not in _BLOCKLIST_MODULES + and m.split(".")[0] not in _BLOCKLIST_MODULES + } + + # Consolidate submodules (xml.dom, xml.sax, xml.etree -> xml if > 3) + consolidated_libs = _consolidate_submodules(lib_importers, threshold=3) + + # Build children + for display_name, original_mods in sorted(consolidated_libs.items()): + child = find_dependent_tests_tree( + display_name, + lib_prefix, + max_depth, + _depth + 1, + _visited_tests, + _visited_modules, + ) + if child["tests"] or child["children"]: + children.append(child) + + return { + "module": module_name, + "tests": sorted(consolidated_tests), + "children": children, + } + + +def _consolidate_file_key(file_key: str) -> str: + """Consolidate file_key to test name. + + Args: + file_key: Relative path without .py (e.g., "test_foo", "test_bar/test_sub") + + Returns: + Consolidated test name: + - "test_foo" for "test_foo" + - "test_sqlite3" for "test_sqlite3/test_dbapi" + """ + parts = pathlib.Path(file_key).parts + if len(parts) == 1: + return parts[0] + return parts[0] diff --git a/scripts/update_lib/file_utils.py b/scripts/update_lib/file_utils.py new file mode 100644 index 00000000000..cb86ee2e664 --- /dev/null +++ b/scripts/update_lib/file_utils.py @@ -0,0 +1,289 @@ +""" +File utilities for update_lib. + +This module provides functions for: +- Safe file reading with error handling +- Safe AST parsing with error handling +- Iterating over Python files +- Parsing and converting library paths +- Detecting test paths vs library paths +- Comparing files or directories for equality +""" + +from __future__ import annotations + +import ast +import filecmp +import pathlib +from collections.abc import Callable, Iterator + +# === I/O utilities === + + +def safe_read_text(path: pathlib.Path) -> str | None: + """Read file content with UTF-8 encoding, returning None on error.""" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + + +def safe_parse_ast(content: str) -> ast.Module | None: + """Parse Python content into AST, returning None on syntax error.""" + try: + return ast.parse(content) + except SyntaxError: + return None + + +def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]: + """Yield Python files from a file or directory.""" + if path.is_file(): + yield path + else: + yield from path.glob("**/*.py") + + +def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]: + """Read all Python files from a path, yielding (path, content) pairs.""" + for py_file in iter_python_files(path): + content = safe_read_text(py_file) + if content is not None: + yield py_file, content + + +# === Path utilities === + + +def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path: + """ + Extract the Lib/... portion from a path containing /Lib/. + + Example: + parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py") + """ + path_str = str(path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker not in path_str: + raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})") + + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[idx + 1 :]) + + +def is_lib_path(path: pathlib.Path) -> bool: + """Check if path starts with Lib/""" + path_str = str(path).replace("\\", "/") + return path_str.startswith("Lib/") or path_str.startswith("./Lib/") + + +def is_test_path(path: pathlib.Path) -> bool: + """Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)""" + path_str = str(path).replace("\\", "/") + return "/Lib/test/" in path_str or path_str.startswith("Lib/test/") + + +def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: + """ + Convert library path to test path. + + Examples: + cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ + cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/ + """ + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker in path_str: + lib_path = parse_lib_path(src_path) + lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name + if lib_name == "__init__": + lib_name = lib_path.parent.name + prefix = path_str[: path_str.index(lib_marker)] + dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + else: + lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name + if lib_name == "__init__": + lib_name = src_path.parent.name + dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + + +def get_test_files(path: pathlib.Path) -> list[pathlib.Path]: + """Get all .py test files in a path (file or directory).""" + if path.is_file(): + return [path] + return sorted(path.glob("**/*.py")) + + +def get_test_module_name(test_path: pathlib.Path) -> str: + """ + Extract test module name from a test file path. + + Examples: + Lib/test/test_foo.py -> test_foo + Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar + """ + test_path = pathlib.Path(test_path) + if test_path.parent.name.startswith("test_"): + return f"{test_path.parent.name}.{test_path.stem}" + return test_path.stem + + +def resolve_module_path( + name: str, prefix: str = "cpython", prefer: str = "file" +) -> pathlib.Path: + """ + Resolve module path, trying file or directory. + + Args: + name: Module name (e.g., "dataclasses", "json") + prefix: CPython directory prefix + prefer: "file" to try .py first, "dir" to try directory first + """ + file_path = pathlib.Path(f"{prefix}/Lib/{name}.py") + dir_path = pathlib.Path(f"{prefix}/Lib/{name}") + + if prefer == "file": + if file_path.exists(): + return file_path + if dir_path.exists(): + return dir_path + return file_path + else: + if dir_path.exists(): + return dir_path + if file_path.exists(): + return file_path + return dir_path + + +def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path: + """Build a path under prefix/Lib/.""" + return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts) + + +def resolve_test_path( + test_name: str, prefix: str = "cpython", prefer: str = "dir" +) -> pathlib.Path: + """Resolve a test module path under Lib/test/.""" + return resolve_module_path(f"test/{test_name}", prefix, prefer=prefer) + + +def cpython_to_local_path( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> pathlib.Path | None: + """Convert CPython path to local Lib path.""" + try: + rel_path = cpython_path.relative_to(cpython_prefix) + return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + return None + + +def get_module_name(path: pathlib.Path) -> str: + """Extract module name from path, handling __init__.py.""" + if path.suffix == ".py": + name = path.stem + if name == "__init__": + return path.parent.name + return name + return path.name + + +def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path: + """Extract CPython directory from a path containing /Lib/.""" + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + if lib_marker in path_str: + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[:idx]) + return pathlib.Path("cpython") + + +# === Comparison utilities === + + +def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool: + """Recursively check if two directories are identical.""" + if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + return False + + for subdir in dcmp.subdirs.values(): + if not _dircmp_is_same(subdir): + return False + + return True + + +def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: + """Compare a CPython path with a local path (file or directory).""" + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return filecmp.cmp(cpython_path, local_path, shallow=False) + + dcmp = filecmp.dircmp(cpython_path, local_path) + return _dircmp_is_same(dcmp) + + +def compare_file_contents( + cpython_path: pathlib.Path, + local_path: pathlib.Path, + *, + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare two files as text, optionally filtering local content.""" + try: + cpython_content = cpython_path.read_text(encoding=encoding) + local_content = local_path.read_text(encoding=encoding) + except (OSError, UnicodeDecodeError): + return False + + if local_filter is not None: + local_content = local_filter(local_content) + + return cpython_content == local_content + + +def compare_dir_contents( + cpython_dir: pathlib.Path, + local_dir: pathlib.Path, + *, + pattern: str = "*.py", + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare directory contents for matching files and text.""" + cpython_files = {f.relative_to(cpython_dir) for f in cpython_dir.rglob(pattern)} + local_files = {f.relative_to(local_dir) for f in local_dir.rglob(pattern)} + + if cpython_files != local_files: + return False + + for rel_path in cpython_files: + if not compare_file_contents( + cpython_dir / rel_path, + local_dir / rel_path, + local_filter=local_filter, + encoding=encoding, + ): + return False + + return True diff --git a/scripts/lib_updater.py b/scripts/update_lib/patch_spec.py old mode 100755 new mode 100644 similarity index 54% rename from scripts/lib_updater.py rename to scripts/update_lib/patch_spec.py index 8573705dd15..d27d2e22fa7 --- a/scripts/lib_updater.py +++ b/scripts/update_lib/patch_spec.py @@ -1,37 +1,14 @@ -#!/usr/bin/env python -__doc__ = """ -This tool helps with updating test files from CPython. +""" +Low-level module for converting between test files and JSON patches. -Examples --------- -To move the patches found in `Lib/test/foo.py` to ` ~/cpython/Lib/test/foo.py` then write the contents back to `Lib/test/foo.py` +This module handles: +- Extracting patches from test files (file -> JSON) +- Applying patches to test files (JSON -> file) +""" ->>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py -o Lib/test/foo.py - -You can run the same command without `-o` to override the `--from` path: - ->>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py - -To get a baseline of patches, you can alter the patches file with your favorite tool/script/etc and then reapply it with: - ->>> ./{fname} --from Lib/test/foo.py --show-patches -o my_patches.json - -(By default the output is set to print to stdout). - -When you want to apply your own patches: - ->>> ./{fname} -p my_patches.json --to Lib/test/foo.py -""".format(fname=__import__("os").path.basename(__file__)) - - -import argparse import ast import collections import enum -import json -import pathlib -import re -import sys import textwrap import typing @@ -105,13 +82,43 @@ def as_ast_node(self) -> ast.Attribute | ast.Call: def as_decorator(self) -> str: unparsed = ast.unparse(self.as_ast_node()) + # ast.unparse uses single quotes; convert to double quotes for ruff compatibility + unparsed = _single_to_double_quotes(unparsed) if not self.ut_method.has_args(): - unparsed = f"{unparsed} # {self._reason}" + unparsed = f"{unparsed} # {self._reason}" return f"@{unparsed}" +def _single_to_double_quotes(s: str) -> str: + """Convert single-quoted strings to double-quoted strings. + + Falls back to original if conversion breaks the AST equivalence. + """ + import re + + def replace_string(match: re.Match) -> str: + content = match.group(1) + # Unescape single quotes and escape double quotes + content = content.replace("\\'", "'").replace('"', '\\"') + return f'"{content}"' + + # Match single-quoted strings (handles escaped single quotes inside) + converted = re.sub(r"'((?:[^'\\]|\\.)*)'", replace_string, s) + + # Verify: parse converted and unparse should equal original + try: + converted_ast = ast.parse(converted, mode="eval") + if ast.unparse(converted_ast) == s: + return converted + except SyntaxError: + pass + + # Fall back to original if conversion failed + return s + + class PatchEntry(typing.NamedTuple): """ Stores patch metadata. @@ -131,9 +138,12 @@ class PatchEntry(typing.NamedTuple): spec: PatchSpec @classmethod - def iter_patch_entires( + def iter_patch_entries( cls, tree: ast.Module, lines: list[str] ) -> "Iterator[typing.Self]": + import re + import sys + for cls_node, fn_node in iter_tests(tree): parent_class = cls_node.name for dec_node in fn_node.decorator_list: @@ -214,7 +224,7 @@ def iter_tests( def iter_patches(contents: str) -> "Iterator[PatchEntry]": lines = contents.splitlines() tree = ast.parse(contents) - yield from PatchEntry.iter_patch_entires(tree, lines) + yield from PatchEntry.iter_patch_entries(tree, lines) def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches: @@ -225,12 +235,40 @@ def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches: return {k: dict(v) for k, v in patches.items()} -def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, str]]": - cache = {} # Used in phase 2. Stores the end line location of a class name. +def extract_patches(contents: str) -> Patches: + """Extract patches from file contents and return as dict.""" + return build_patch_dict(iter_patches(contents)) + + +def _iter_patch_lines( + tree: ast.Module, patches: Patches +) -> "Iterator[tuple[int, str]]": + import sys + + # Build cache of all classes (for Phase 2 to find classes without methods) + cache = {} + # Build per-class set of async method names (for Phase 2 to generate correct override) + async_methods: dict[str, set[str]] = {} + # Track class bases for inherited async method lookup + class_bases: dict[str, list[str]] = {} + all_classes = {node.name for node in tree.body if isinstance(node, ast.ClassDef)} + for node in tree.body: + if isinstance(node, ast.ClassDef): + cache[node.name] = node.end_lineno + class_bases[node.name] = [ + base.id + for base in node.bases + if isinstance(base, ast.Name) and base.id in all_classes + ] + cls_async: set[str] = set() + for item in node.body: + if isinstance(item, ast.AsyncFunctionDef): + cls_async.add(item.name) + if cls_async: + async_methods[node.name] = cls_async # Phase 1: Iterate and mark existing tests for cls_node, fn_node in iter_tests(tree): - cache[cls_node.name] = cls_node.end_lineno specs = patches.get(cls_node.name, {}).pop(fn_node.name, None) if not specs: continue @@ -243,16 +281,36 @@ def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, patch_lines = "\n".join(spec.as_decorator() for spec in specs) yield (lineno - 1, textwrap.indent(patch_lines, indent)) - # Phase 2: Iterate and mark inhereted tests - for cls_name, tests in patches.items(): + # Phase 2: Iterate and mark inherited tests + for cls_name, tests in sorted(patches.items()): lineno = cache.get(cls_name) if not lineno: print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr) continue - for test_name, specs in tests.items(): + for test_name, specs in sorted(tests.items()): decorators = "\n".join(spec.as_decorator() for spec in specs) - patch_lines = f""" + # Check current class and ancestors for async method + is_async = False + queue = [cls_name] + visited: set[str] = set() + while queue: + cur = queue.pop(0) + if cur in visited: + continue + visited.add(cur) + if test_name in async_methods.get(cur, set()): + is_async = True + break + queue.extend(class_bases.get(cur, [])) + if is_async: + patch_lines = f""" +{decorators} +async def {test_name}(self): +{DEFAULT_INDENT}return await super().{test_name}() +""".rstrip() + else: + patch_lines = f""" {decorators} def {test_name}(self): {DEFAULT_INDENT}return super().{test_name}() @@ -260,12 +318,53 @@ def {test_name}(self): yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT)) +def _has_unittest_import(tree: ast.Module) -> bool: + """Check if 'import unittest' is already present in the file.""" + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == UT and alias.asname is None: + return True + return False + + +def _find_import_insert_line(tree: ast.Module) -> int: + """Find the line number after the last import statement.""" + last_import_line = None + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = node.end_lineno or node.lineno + if last_import_line is not None: + return last_import_line + # No imports found - insert after module docstring if present, else at top + if ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + return tree.body[0].end_lineno or tree.body[0].lineno + return 0 + + def apply_patches(contents: str, patches: Patches) -> str: + """Apply patches to file contents and return modified contents.""" tree = ast.parse(contents) lines = contents.splitlines() - modifications = list(iter_patch_lines(tree, patches)) - # Going in reverse to not distrupt the line offset + modifications = list(_iter_patch_lines(tree, patches)) + + # If we have modifications and unittest is not imported, add it + if modifications and not _has_unittest_import(tree): + import_line = _find_import_insert_line(tree) + modifications.append( + ( + import_line, + "\nimport unittest # XXX: RUSTPYTHON; importing to be able to skip tests", + ) + ) + + # Going in reverse to not disrupt the line offset for lineno, patch in sorted(modifications, reverse=True): lines.insert(lineno, patch) @@ -273,80 +372,26 @@ def apply_patches(contents: str, patches: Patches) -> str: return f"{joined}\n" -def write_output(data: str, dest: str) -> None: - if dest == "-": - print(data, end="") - return - - with open(dest, "w") as fd: - fd.write(data) - - -def build_argparse() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - patches_group = parser.add_mutually_exclusive_group(required=True) - patches_group.add_argument( - "-p", - "--patches", - help="File path to file containing patches in a JSON format", - type=pathlib.Path, - ) - patches_group.add_argument( - "--from", - help="File to gather patches from", - dest="gather_from", - type=pathlib.Path, - ) - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--to", - help="File to apply patches to", - type=pathlib.Path, - ) - group.add_argument( - "--show-patches", action="store_true", help="Show the patches and exit" - ) - - parser.add_argument( - "-o", "--output", default="-", help="Output file. Set to '-' for stdout" - ) - - return parser - - -if __name__ == "__main__": - parser = build_argparse() - args = parser.parse_args() - - if args.patches: - patches = { - cls_name: { - test_name: [ - PatchSpec(**spec)._replace(ut_method=UtMethod(spec["ut_method"])) - for spec in specs - ] - for test_name, specs in tests.items() - } - for cls_name, tests in json.loads(args.patches.read_text()).items() +def patches_to_json(patches: Patches) -> dict: + """Convert patches to JSON-serializable dict.""" + return { + cls_name: { + test_name: [spec._asdict() for spec in specs] + for test_name, specs in tests.items() } - else: - patches = build_patch_dict(iter_patches(args.gather_from.read_text())) - - if args.show_patches: - patches = { - cls_name: { - test_name: [spec._asdict() for spec in specs] - for test_name, specs in tests.items() - } - for cls_name, tests in patches.items() + for cls_name, tests in patches.items() + } + + +def patches_from_json(data: dict) -> Patches: + """Convert JSON dict back to Patches.""" + return { + cls_name: { + test_name: [ + PatchSpec(**spec)._replace(ut_method=UtMethod(spec["ut_method"])) + for spec in specs + ] + for test_name, specs in tests.items() } - output = json.dumps(patches, indent=4) + "\n" - write_output(output, args.output) - sys.exit(0) - - patched = apply_patches(args.to.read_text(), patches) - write_output(patched, args.output) + for cls_name, tests in data.items() + } diff --git a/scripts/update_lib/tests/__init__.py b/scripts/update_lib/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/update_lib/tests/test_auto_mark.py b/scripts/update_lib/tests/test_auto_mark.py new file mode 100644 index 00000000000..ce89b0f9918 --- /dev/null +++ b/scripts/update_lib/tests/test_auto_mark.py @@ -0,0 +1,1085 @@ +"""Tests for auto_mark.py - test result parsing and auto-marking.""" + +import ast +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +from update_lib.cmd_auto_mark import ( + Test, + TestResult, + TestRunError, + _expand_stripped_to_children, + _is_super_call_only, + apply_test_changes, + auto_mark_directory, + auto_mark_file, + collect_test_changes, + extract_test_methods, + parse_results, + path_to_test_parts, + remove_expected_failures, + strip_reasonless_expected_failures, +) +from update_lib.patch_spec import COMMENT + + +def _make_result(stdout: str) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess( + args=["test"], returncode=0, stdout=stdout, stderr="" + ) + + +# -- fixtures shared across inheritance-aware tests -- + +BASE_TWO_CHILDREN = """import unittest + +class Base: + def test_foo(self): + pass + +class ChildA(Base, unittest.TestCase): + pass + +class ChildB(Base, unittest.TestCase): + pass +""" + +BASE_TWO_CHILDREN_ONE_OVERRIDE = """import unittest + +class Base: + def test_foo(self): + pass + +class ChildA(Base, unittest.TestCase): + pass + +class ChildB(Base, unittest.TestCase): + def test_foo(self): + # own implementation + pass +""" + + +class TestParseResults(unittest.TestCase): + """Tests for parse_results function.""" + + def test_parse_fail_and_error(self): + """FAIL and ERROR are collected; ok is ignored.""" + stdout = """\ +Run 3 tests sequentially +test_one (test.test_example.TestA.test_one) ... FAIL +test_two (test.test_example.TestA.test_two) ... ok +test_three (test.test_example.TestB.test_three) ... ERROR +----------- +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + by_name = {t.name: t for t in result.tests} + self.assertEqual(by_name["test_one"].path, "test.test_example.TestA.test_one") + self.assertEqual(by_name["test_one"].result, "fail") + self.assertEqual(by_name["test_three"].result, "error") + + def test_parse_unexpected_success(self): + stdout = """\ +Run 1 tests sequentially +test_foo (test.test_example.TestClass.test_foo) ... unexpected success +----------- +UNEXPECTED SUCCESS: test_foo (test.test_example.TestClass.test_foo) +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.unexpected_successes), 1) + self.assertEqual(result.unexpected_successes[0].name, "test_foo") + self.assertEqual( + result.unexpected_successes[0].path, "test.test_example.TestClass.test_foo" + ) + + def test_parse_tests_result(self): + result = parse_results(_make_result("== Tests result: FAILURE ==\n")) + self.assertEqual(result.tests_result, "FAILURE") + + def test_parse_crashed_run_no_tests_result(self): + """Test results are still parsed when the runner crashes (no Tests result line).""" + stdout = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_ast +test_foo (test.test_ast.test_ast.TestA.test_foo) ... FAIL +test_bar (test.test_ast.test_ast.TestA.test_bar) ... ok +test_baz (test.test_ast.test_ast.TestB.test_baz) ... ERROR +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(result.tests_result, "") + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_foo", names) + self.assertIn("test_baz", names) + + def test_parse_crashed_run_has_unexpected_success(self): + """Unexpected successes are parsed even without Tests result line.""" + stdout = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_ast +test_foo (test.test_ast.test_ast.TestA.test_foo) ... unexpected success +UNEXPECTED SUCCESS: test_foo (test.test_ast.test_ast.TestA.test_foo) +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(result.tests_result, "") + self.assertEqual(len(result.unexpected_successes), 1) + + def test_parse_error_messages(self): + """Single and multiple error messages are parsed from tracebacks.""" + stdout = """\ +Run 2 tests sequentially +test_foo (test.test_example.TestClass.test_foo) ... FAIL +test_bar (test.test_example.TestClass.test_bar) ... ERROR +----------- +====================================================================== +FAIL: test_foo (test.test_example.TestClass.test_foo) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 10, in test_foo + self.assertEqual(1, 2) +AssertionError: 1 != 2 + +====================================================================== +ERROR: test_bar (test.test_example.TestClass.test_bar) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 20, in test_bar + raise ValueError("oops") +ValueError: oops + +====================================================================== +""" + result = parse_results(_make_result(stdout)) + by_name = {t.name: t for t in result.tests} + self.assertEqual(by_name["test_foo"].error_message, "AssertionError: 1 != 2") + self.assertEqual(by_name["test_bar"].error_message, "ValueError: oops") + + def test_parse_directory_test_multiple_submodules(self): + """Failures across submodule boundaries are all detected.""" + stdout = """\ +Run 3 tests sequentially +0:00:00 [ 1/3] test_asyncio.test_buffered_proto +test_ok (test.test_asyncio.test_buffered_proto.TestProto.test_ok) ... ok + +---------------------------------------------------------------------- +Ran 1 tests in 0.1s + +OK + +0:00:01 [ 2/3] test_asyncio.test_events +test_create (test.test_asyncio.test_events.TestEvents.test_create) ... FAIL + +---------------------------------------------------------------------- +Ran 1 tests in 0.2s + +FAILED (failures=1) + +0:00:02 [ 3/3] test_asyncio.test_tasks +test_gather (test.test_asyncio.test_tasks.TestTasks.test_gather) ... ERROR + +---------------------------------------------------------------------- +Ran 1 tests in 0.3s + +FAILED (errors=1) + +== Tests result: FAILURE == +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_create", names) + self.assertIn("test_gather", names) + self.assertEqual(result.tests_result, "FAILURE") + + def test_parse_multiline_test_with_docstring(self): + """Two-line output (test_name + docstring ... RESULT) is handled.""" + stdout = """\ +Run 3 tests sequentially +test_ok (test.test_example.TestClass.test_ok) ... ok +test_with_doc (test.test_example.TestClass.test_with_doc) +Test that something works ... ERROR +test_normal_fail (test.test_example.TestClass.test_normal_fail) ... FAIL +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_with_doc", names) + self.assertIn("test_normal_fail", names) + test_doc = next(t for t in result.tests if t.name == "test_with_doc") + self.assertEqual(test_doc.path, "test.test_example.TestClass.test_with_doc") + self.assertEqual(test_doc.result, "error") + + +class TestPathToTestParts(unittest.TestCase): + def test_simple_path(self): + self.assertEqual( + path_to_test_parts("test.test_foo.TestClass.test_method"), + ["TestClass", "test_method"], + ) + + def test_nested_path(self): + self.assertEqual( + path_to_test_parts("test.test_foo.test_bar.TestClass.test_method"), + ["TestClass", "test_method"], + ) + + +class TestCollectTestChanges(unittest.TestCase): + def test_collect_failures_and_error_messages(self): + """Failures and error messages are collected; empty messages are omitted.""" + results = TestResult() + results.tests = [ + Test( + name="test_foo", + path="test.test_example.TestClass.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + Test( + name="test_bar", + path="test.test_example.TestClass.test_bar", + result="error", + error_message="", + ), + ] + failing, successes, error_messages = collect_test_changes(results) + + self.assertEqual( + failing, {("TestClass", "test_foo"), ("TestClass", "test_bar")} + ) + self.assertEqual(successes, set()) + self.assertEqual(len(error_messages), 1) + self.assertEqual( + error_messages[("TestClass", "test_foo")], "AssertionError: 1 != 2" + ) + + def test_collect_unexpected_successes(self): + results = TestResult() + results.unexpected_successes = [ + Test( + name="test_foo", + path="test.test_example.TestClass.test_foo", + result="unexpected_success", + ), + ] + _, successes, _ = collect_test_changes(results) + self.assertEqual(successes, {("TestClass", "test_foo")}) + + def test_module_prefix_filtering(self): + """Prefix filters with both short and 'test.' prefix formats.""" + results = TestResult() + results.tests = [ + Test(name="test_foo", path="test_a.TestClass.test_foo", result="fail"), + Test( + name="test_bar", + path="test.test_dataclasses.TestCase.test_bar", + result="fail", + ), + Test( + name="test_baz", + path="test.test_other.TestOther.test_baz", + result="fail", + ), + ] + failing_a, _, _ = collect_test_changes(results, module_prefix="test_a.") + self.assertEqual(failing_a, {("TestClass", "test_foo")}) + + failing_dc, _, _ = collect_test_changes( + results, module_prefix="test.test_dataclasses." + ) + self.assertEqual(failing_dc, {("TestCase", "test_bar")}) + + def test_collect_init_module_matching(self): + """__init__.py tests match after stripping .__init__ from the prefix.""" + results = TestResult() + results.tests = [ + Test( + name="test_field_repr", + path="test.test_dataclasses.TestCase.test_field_repr", + result="fail", + ), + ] + module_prefix = "test_dataclasses.__init__" + if module_prefix.endswith(".__init__"): + module_prefix = module_prefix[:-9] + module_prefix = "test." + module_prefix + "." + + failing, _, _ = collect_test_changes(results, module_prefix=module_prefix) + self.assertEqual(failing, {("TestCase", "test_field_repr")}) + + +class TestExtractTestMethods(unittest.TestCase): + def test_extract_methods(self): + """Extracts from single and multiple classes.""" + code = """ +class TestA(unittest.TestCase): + def test_a(self): + pass + +class TestB(unittest.TestCase): + def test_b(self): + pass +""" + methods = extract_test_methods(code) + self.assertEqual(methods, {("TestA", "test_a"), ("TestB", "test_b")}) + + def test_extract_syntax_error_returns_empty(self): + self.assertEqual(extract_test_methods("this is not valid python {"), set()) + + +class TestRemoveExpectedFailures(unittest.TestCase): + def test_remove_comment_before(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + + def test_remove_inline_comment(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + + def test_remove_super_call_method(self): + """Super-call-only override is removed entirely (sync).""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + return super().test_one() +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("def test_one", result) + + def test_remove_async_super_call_override(self): + """Super-call-only override is removed entirely (async).""" + code = f"""import unittest + +class BaseTest: + async def test_async_one(self): + pass + +class TestChild(BaseTest, unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + async def test_async_one(self): + return await super().test_async_one() +""" + result = remove_expected_failures(code, {("TestChild", "test_async_one")}) + self.assertNotIn("return await super().test_async_one()", result) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("class TestChild", result) + self.assertIn("async def test_async_one(self):", result) + + def test_remove_with_comment_after(self): + """Reason comment on the line after the decorator is also removed.""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + # RuntimeError: something went wrong + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertNotIn("RuntimeError: something went wrong", result) + self.assertIn("def test_one(self):", result) + + def test_no_removal_without_comment(self): + """Decorators without our COMMENT marker are left untouched.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertIn("@unittest.expectedFailure", result) + + +class TestStripReasonlessExpectedFailures(unittest.TestCase): + def test_strip_reason_formats(self): + """Strips both inline-comment and comment-before formats when no reason.""" + for label, code in [ + ( + "inline", + f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""", + ), + ( + "comment-before", + f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""", + ), + ]: + with self.subTest(label): + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + self.assertEqual(stripped, {("TestFoo", "test_one")}) + + def test_keep_with_reason(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT}; AssertionError: 1 != 2 + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, set()) + + def test_strip_with_comment_after(self): + """Old-format reason comment on the next line is also removed.""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + # RuntimeError: something went wrong + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("RuntimeError", result) + self.assertIn("def test_one(self):", result) + self.assertEqual(stripped, {("TestFoo", "test_one")}) + + def test_strip_super_call_override(self): + """Super-call overrides are removed entirely (both comment formats).""" + for label, code in [ + ( + "comment-before", + f"""import unittest + +class _BaseTests: + def test_foo(self): + pass + +class TestChild(_BaseTests, unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_foo(self): + return super().test_foo() +""", + ), + ( + "inline", + f"""import unittest + +class _BaseTests: + def test_foo(self): + pass + +class TestChild(_BaseTests, unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + return super().test_foo() +""", + ), + ]: + with self.subTest(label): + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("return super().test_foo()", result) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, {("TestChild", "test_foo")}) + self.assertIn("class _BaseTests:", result) + + def test_no_strip_without_comment(self): + """Markers without our COMMENT are NOT stripped.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, set()) + + def test_mixed_with_and_without_reason(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_no_reason(self): + pass + + @unittest.expectedFailure # {COMMENT}; has a reason + def test_has_reason(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertEqual(stripped, {("TestFoo", "test_no_reason")}) + self.assertIn("has a reason", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + +class TestExpandStrippedToChildren(unittest.TestCase): + def test_parent_to_children(self): + """Parent stripped → all/partial failing children returned.""" + stripped = {("Base", "test_foo")} + all_children = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + + # All children fail + result = _expand_stripped_to_children(BASE_TWO_CHILDREN, stripped, all_children) + self.assertEqual(result, all_children) + + # Only one child fails + partial = {("ChildA", "test_foo")} + result = _expand_stripped_to_children(BASE_TWO_CHILDREN, stripped, partial) + self.assertEqual(result, partial) + + def test_direct_match(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + s = {("TestFoo", "test_one")} + self.assertEqual(_expand_stripped_to_children(code, s, s), s) + + def test_child_with_own_override_excluded(self): + stripped = {("Base", "test_foo")} + all_failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + result = _expand_stripped_to_children( + BASE_TWO_CHILDREN_ONE_OVERRIDE, stripped, all_failing + ) + # ChildA inherits → included; ChildB has own method → excluded + self.assertEqual(result, {("ChildA", "test_foo")}) + + +class TestApplyTestChanges(unittest.TestCase): + def test_apply_failing_tests(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_test_changes(code, {("TestFoo", "test_one")}, set()) + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + def test_apply_removes_unexpected_success(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + result = apply_test_changes(code, set(), {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + + def test_apply_both_changes(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + # {COMMENT} + @unittest.expectedFailure + def test_two(self): + pass +""" + result = apply_test_changes( + code, {("TestFoo", "test_one")}, {("TestFoo", "test_two")} + ) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + def test_apply_with_error_message(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_test_changes( + code, + {("TestFoo", "test_one")}, + set(), + {("TestFoo", "test_one"): "AssertionError: 1 != 2"}, + ) + self.assertIn("AssertionError: 1 != 2", result) + self.assertIn(COMMENT, result) + + +class TestConsolidateToParent(unittest.TestCase): + def test_all_children_fail_marks_parent_with_message(self): + """All subclasses fail → marks parent; error message is transferred.""" + failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + error_messages = {("ChildA", "test_foo"): "RuntimeError: boom"} + result = apply_test_changes(BASE_TWO_CHILDREN, failing, set(), error_messages) + + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + self.assertNotIn("return super()", result) + self.assertIn("RuntimeError: boom", result) + + def test_partial_children_fail_marks_children(self): + result = apply_test_changes(BASE_TWO_CHILDREN, {("ChildA", "test_foo")}, set()) + self.assertIn("return super().test_foo()", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + def test_child_with_own_override_not_consolidated(self): + failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + result = apply_test_changes(BASE_TWO_CHILDREN_ONE_OVERRIDE, failing, set()) + self.assertEqual(result.count("@unittest.expectedFailure"), 2) + + def test_strip_then_consolidate_restores_parent_marker(self): + """End-to-end: strip parent marker → child failures → re-mark on parent.""" + code = f"""import unittest + +class _BaseTests: + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + +class ChildA(_BaseTests, unittest.TestCase): + pass + +class ChildB(_BaseTests, unittest.TestCase): + pass +""" + stripped_code, stripped_tests = strip_reasonless_expected_failures(code) + self.assertEqual(stripped_tests, {("_BaseTests", "test_foo")}) + + all_failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + error_messages = {("ChildA", "test_foo"): "RuntimeError: boom"} + + to_remark = _expand_stripped_to_children( + stripped_code, stripped_tests, all_failing + ) + self.assertEqual(to_remark, all_failing) + + result = apply_test_changes(stripped_code, to_remark, set(), error_messages) + self.assertIn("RuntimeError: boom", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + self.assertNotIn("return super()", result) + + +class TestSmartAutoMarkFiltering(unittest.TestCase): + """Tests for smart auto-mark filtering (new tests vs regressions).""" + + @staticmethod + def _filter(all_failing, original, current): + new = current - original + to_mark = {t for t in all_failing if t in new} + return to_mark, all_failing - to_mark + + def test_new_vs_regression(self): + """New failures are marked; existing (regression) failures are not.""" + original = {("TestFoo", "test_old1"), ("TestFoo", "test_old2")} + current = original | {("TestFoo", "test_new1"), ("TestFoo", "test_new2")} + all_failing = {("TestFoo", "test_old1"), ("TestFoo", "test_new1")} + + to_mark, regressions = self._filter(all_failing, original, current) + self.assertEqual(to_mark, {("TestFoo", "test_new1")}) + self.assertEqual(regressions, {("TestFoo", "test_old1")}) + + # Edge: all new → all marked + to_mark, regressions = self._filter(all_failing, set(), current) + self.assertEqual(to_mark, all_failing) + self.assertEqual(regressions, set()) + + # Edge: all old → nothing marked + to_mark, regressions = self._filter(all_failing, current, current) + self.assertEqual(to_mark, set()) + self.assertEqual(regressions, all_failing) + + def test_filters_across_classes(self): + original = {("TestA", "test_a"), ("TestB", "test_b")} + current = original | {("TestA", "test_new_a"), ("TestC", "test_c")} + all_failing = { + ("TestA", "test_a"), # regression + ("TestA", "test_new_a"), # new + ("TestC", "test_c"), # new (new class) + } + to_mark, regressions = self._filter(all_failing, original, current) + self.assertEqual(to_mark, {("TestA", "test_new_a"), ("TestC", "test_c")}) + self.assertEqual(regressions, {("TestA", "test_a")}) + + +class TestIsSuperCallOnly(unittest.TestCase): + @staticmethod + def _parse_method(code): + tree = ast.parse(code) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return node + return None + + def test_sync(self): + cases = [ + ("return super().test_one()", True), + ("return super().test_two()", False), # mismatched name + ("pass", False), # regular body + ("x = 1\n return super().test_one()", False), # multiple stmts + ] + for body, expected in cases: + with self.subTest(body=body): + code = f""" +class Foo: + def test_one(self): + {body} +""" + self.assertEqual( + _is_super_call_only(self._parse_method(code)), expected + ) + + def test_async(self): + cases = [ + ("return await super().test_one()", True), + ("return await super().test_two()", False), + ("return super().test_one()", True), # sync call in async method + ] + for body, expected in cases: + with self.subTest(body=body): + code = f""" +class Foo: + async def test_one(self): + {body} +""" + self.assertEqual( + _is_super_call_only(self._parse_method(code)), expected + ) + + +class TestAutoMarkFileWithCrashedRun(unittest.TestCase): + """auto_mark_file should process partial results when test runner crashes.""" + + CRASHED_STDOUT = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_example +test_foo (test.test_example.TestA.test_foo) ... FAIL +test_bar (test.test_example.TestA.test_bar) ... ok +====================================================================== +FAIL: test_foo (test.test_example.TestA.test_foo) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 10, in test_foo + self.assertEqual(1, 2) +AssertionError: 1 != 2 +""" + + def test_auto_mark_file_crashed_run(self): + """auto_mark_file processes results even when tests_result is empty (crash).""" + test_code = f"""import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass + + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + added, removed, regressions = auto_mark_file( + test_file, mark_failure=True, verbose=False + ) + + self.assertEqual(added, 1) + contents = test_file.read_text() + self.assertIn("expectedFailure", contents) + + def test_auto_mark_file_no_results_at_all_raises(self): + """auto_mark_file raises TestRunError when there are zero parsed results.""" + test_code = """import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [] + mock_result.stdout = "some crash output" + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + with self.assertRaises(TestRunError): + auto_mark_file(test_file, verbose=False) + + +class TestAutoMarkDirectoryWithCrashedRun(unittest.TestCase): + """auto_mark_directory should process partial results when test runner crashes.""" + + def test_auto_mark_directory_crashed_run(self): + """auto_mark_directory processes results even when tests_result is empty.""" + test_code = f"""import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.test_sub.TestA.test_foo", + result="fail", + error_message="AssertionError: oops", + ), + ] + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + side_effect=lambda p: ( + "test_example" if p == test_dir else "test_example.test_sub" + ), + ), + ): + added, removed, regressions = auto_mark_directory( + test_dir, mark_failure=True, verbose=False + ) + + self.assertEqual(added, 1) + contents = test_file.read_text() + self.assertIn("expectedFailure", contents) + + def test_auto_mark_directory_no_results_raises(self): + """auto_mark_directory raises TestRunError when zero results.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text("import unittest\n") + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [] + mock_result.stdout = "crash" + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + return_value="test_example", + ), + ): + with self.assertRaises(TestRunError): + auto_mark_directory(test_dir, verbose=False) + + +class TestAutoMarkFileRestoresOnCrash(unittest.TestCase): + """Stripped markers must be restored when the test runner crashes.""" + + def test_stripped_markers_restored_when_crash(self): + """Markers stripped before run must be restored for unobserved tests on crash.""" + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_baz(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + # Simulate a crashed run that only observed test_foo (failed) + # test_bar and test_baz never ran due to crash + mock_result = TestResult() + mock_result.tests_result = "" # no Tests result line (crash) + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + auto_mark_file(test_file, verbose=False) + + contents = test_file.read_text() + # test_bar and test_baz were not observed — their markers must be restored + self.assertIn("def test_bar", contents) + self.assertIn("def test_baz", contents) + # Count expectedFailure markers: all 3 should be present + self.assertEqual(contents.count("expectedFailure"), 3, contents) + + def test_stripped_markers_removed_when_complete_run(self): + """Markers are properly removed when the run completes normally.""" + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + # Simulate a complete run where test_foo fails but test_bar passes + mock_result = TestResult() + mock_result.tests_result = "FAILURE" # normal completion + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError", + ), + ] + # test_bar passes → shows as unexpected success + mock_result.unexpected_successes = [ + Test( + name="test_bar", + path="test.test_example.TestA.test_bar", + result="unexpected success", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + auto_mark_file(test_file, verbose=False) + + contents = test_file.read_text() + # test_foo should still have marker (re-added) + self.assertEqual(contents.count("expectedFailure"), 1, contents) + self.assertIn("def test_foo", contents) + + +class TestAutoMarkDirectoryRestoresOnCrash(unittest.TestCase): + """Stripped markers must be restored for directory runs that crash.""" + + def test_stripped_markers_restored_when_crash(self): + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" # crash + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.test_sub.TestA.test_foo", + result="fail", + ), + ] + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + side_effect=lambda p: ( + "test_example" if p == test_dir else "test_example.test_sub" + ), + ), + ): + auto_mark_directory(test_dir, verbose=False) + + contents = test_file.read_text() + # Both markers must be present (unobserved test_bar restored) + self.assertEqual(contents.count("expectedFailure"), 2, contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py new file mode 100644 index 00000000000..aca00cb18f3 --- /dev/null +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -0,0 +1,75 @@ +"""Tests for copy_lib.py - library copying with dependencies.""" + +import pathlib +import tempfile +import unittest + + +class TestCopySingle(unittest.TestCase): + """Tests for _copy_single helper function.""" + + def test_copies_file(self): + """Test copying a single file.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("content") + dst = tmpdir / "dest.py" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertEqual(dst.read_text(), "content") + + def test_copies_directory(self): + """Test copying a directory.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source_dir" + src.mkdir() + (src / "file.py").write_text("content") + dst = tmpdir / "dest_dir" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertTrue((dst / "file.py").exists()) + + def test_removes_existing_before_copy(self): + """Test that existing destination is removed before copy.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("new content") + dst = tmpdir / "dest.py" + dst.write_text("old content") + + _copy_single(src, dst, verbose=False) + + self.assertEqual(dst.read_text(), "new content") + + +class TestCopyLib(unittest.TestCase): + """Tests for copy_lib function.""" + + def test_raises_on_path_without_lib(self): + """Test that copy_lib raises ValueError when path doesn't contain /Lib/.""" + from update_lib.cmd_copy_lib import copy_lib + + with self.assertRaises(ValueError) as ctx: + copy_lib(pathlib.Path("some/path/without/lib.py")) + + self.assertIn("/Lib/", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py new file mode 100644 index 00000000000..d97af2867aa --- /dev/null +++ b/scripts/update_lib/tests/test_deps.py @@ -0,0 +1,394 @@ +"""Tests for deps.py - dependency resolution.""" + +import pathlib +import tempfile +import unittest + +from update_lib.deps import ( + get_lib_paths, + get_soft_deps, + get_test_dependencies, + get_test_paths, + parse_lib_imports, + parse_test_imports, +) + + +class TestParseTestImports(unittest.TestCase): + """Tests for parse_test_imports function.""" + + def test_from_test_import(self): + """Test parsing 'from test import foo'.""" + code = """ +from test import string_tests +from test import lock_tests, other_tests +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests", "lock_tests", "other_tests"}) + + def test_from_test_dot_module(self): + """Test parsing 'from test.foo import bar'.""" + code = """ +from test.string_tests import CommonTest +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests"}) # support is excluded + + def test_excludes_support(self): + """Test that 'support' is excluded.""" + code = """ +from test import support +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_regular_imports_ignored(self): + """Test that regular imports are ignored.""" + code = """ +import os +from collections import defaultdict +from . import helper +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_syntax_error_returns_empty(self): + """Test that syntax errors return empty set.""" + code = "this is not valid python {" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + +class TestGetLibPaths(unittest.TestCase): + """Tests for get_lib_paths function.""" + + def test_auto_detect_py_module(self): + """Test auto-detection of _py{module}.py pattern.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "mymodule.py").write_text("# mymodule") + (lib_dir / "_pymymodule.py").write_text("# _pymymodule") + + paths = get_lib_paths("mymodule", str(tmpdir)) + self.assertEqual(len(paths), 2) + self.assertIn(tmpdir / "Lib" / "mymodule.py", paths) + self.assertIn(tmpdir / "Lib" / "_pymymodule.py", paths) + + def test_default_file(self): + """Test default to .py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# foo") + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "foo.py",)) + + def test_default_directory(self): + """Test default to directory when file doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo").mkdir() + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "foo",)) + + +class TestGetTestPaths(unittest.TestCase): + """Tests for get_test_paths function.""" + + def test_known_dependency(self): + """Test test with known path override.""" + paths = get_test_paths("regrtest", "cpython") + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/test_regrtest")) + + def test_default_directory(self): + """Test default to test_name/ directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo").mkdir() + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "test" / "test_foo",)) + + def test_default_file(self): + """Test fallback to test_name.py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo.py").write_text("# test") + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "test" / "test_foo.py",)) + + +class TestGetTestDependencies(unittest.TestCase): + """Tests for get_test_dependencies function.""" + + def test_parse_file_imports(self): + """Test parsing imports from test file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test file with import + test_file = test_dir / "test_foo.py" + test_file.write_text(""" +from test import string_tests + +class TestFoo: + pass +""") + # Create the dependency file + (test_dir / "string_tests.py").write_text("# string tests") + + result = get_test_dependencies(test_file) + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual(result["hard_deps"][0], test_dir / "string_tests.py") + self.assertEqual(result["data"], []) + + def test_nonexistent_path(self): + """Test nonexistent path returns empty.""" + result = get_test_dependencies(pathlib.Path("/nonexistent/path")) + self.assertEqual(result, {"hard_deps": [], "data": []}) + + def test_transitive_data_dependency(self): + """Test that data deps are resolved transitively. + + Chain: test_codecencodings_kr -> multibytecodec_support -> cjkencodings + """ + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test_codecencodings_kr.py that imports multibytecodec_support + test_file = test_dir / "test_codecencodings_kr.py" + test_file.write_text(""" +from test import multibytecodec_support + +class TestKR: + pass +""") + # Create multibytecodec_support.py (the intermediate dependency) + (test_dir / "multibytecodec_support.py").write_text("# support module") + + # Create cjkencodings directory (the data dependency of multibytecodec_support) + (test_dir / "cjkencodings").mkdir() + + result = get_test_dependencies(test_file) + + # Should find multibytecodec_support.py as a hard_dep + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual( + result["hard_deps"][0], test_dir / "multibytecodec_support.py" + ) + + # Should find cjkencodings as data (from multibytecodec_support's TEST_DEPENDENCIES) + self.assertEqual(len(result["data"]), 1) + self.assertEqual(result["data"][0], test_dir / "cjkencodings") + + +class TestParseLibImports(unittest.TestCase): + """Tests for parse_lib_imports function.""" + + def test_import_statement(self): + """Test parsing 'import foo'.""" + code = """ +import os +import sys +import collections.abc +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"os", "sys", "collections.abc"}) + + def test_from_import(self): + """Test parsing 'from foo import bar'.""" + code = """ +from os import path +from collections.abc import Mapping +from typing import Optional +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"os", "collections.abc", "typing"}) + + def test_mixed_imports(self): + """Test mixed import styles.""" + code = """ +import sys +from os import path +from collections import defaultdict +import functools +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"sys", "os", "collections", "functools"}) + + def test_syntax_error_returns_empty(self): + """Test that syntax errors return empty set.""" + code = "this is not valid python {" + imports = parse_lib_imports(code) + self.assertEqual(imports, set()) + + def test_relative_import_skipped(self): + """Test that relative imports (no module) are skipped.""" + code = """ +from . import foo +from .. import bar +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, set()) + + +class TestGetSoftDeps(unittest.TestCase): + """Tests for get_soft_deps function.""" + + def test_with_temp_files(self): + """Test soft deps detection with temp files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports another module + (lib_dir / "foo.py").write_text(""" +import bar +from baz import something +""") + # Create the imported modules + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "baz.py").write_text("# baz module") + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertEqual(soft_deps, {"bar", "baz"}) + + def test_skips_self(self): + """Test that module doesn't include itself in soft_deps.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports itself (circular) + (lib_dir / "foo.py").write_text(""" +import foo +import bar +""") + (lib_dir / "bar.py").write_text("# bar module") + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertNotIn("foo", soft_deps) + self.assertIn("bar", soft_deps) + + def test_filters_nonexistent(self): + """Test that nonexistent modules are filtered out.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports nonexistent module + (lib_dir / "foo.py").write_text(""" +import bar +import nonexistent +""") + (lib_dir / "bar.py").write_text("# bar module") + # nonexistent.py is NOT created + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertEqual(soft_deps, {"bar"}) + + +class TestDircmpIsSame(unittest.TestCase): + """Tests for _dircmp_is_same function.""" + + def test_identical_directories(self): + """Test that identical directories return True.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + dir1.mkdir() + dir2.mkdir() + + (dir1 / "file.py").write_text("content") + (dir2 / "file.py").write_text("content") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertTrue(_dircmp_is_same(dcmp)) + + def test_different_files(self): + """Test that directories with different files return False.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + dir1.mkdir() + dir2.mkdir() + + (dir1 / "file.py").write_text("content1") + (dir2 / "file.py").write_text("content2") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertFalse(_dircmp_is_same(dcmp)) + + def test_nested_identical(self): + """Test that nested identical directories return True.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + (dir1 / "sub").mkdir(parents=True) + (dir2 / "sub").mkdir(parents=True) + + (dir1 / "sub" / "file.py").write_text("content") + (dir2 / "sub" / "file.py").write_text("content") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertTrue(_dircmp_is_same(dcmp)) + + def test_nested_different(self): + """Test that nested directories with differences return False.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + (dir1 / "sub").mkdir(parents=True) + (dir2 / "sub").mkdir(parents=True) + + (dir1 / "sub" / "file.py").write_text("content1") + (dir2 / "sub" / "file.py").write_text("content2") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertFalse(_dircmp_is_same(dcmp)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py new file mode 100644 index 00000000000..0cc247ba841 --- /dev/null +++ b/scripts/update_lib/tests/test_migrate.py @@ -0,0 +1,196 @@ +"""Tests for migrate.py - file migration operations.""" + +import pathlib +import tempfile +import unittest + +from update_lib.cmd_migrate import ( + patch_directory, + patch_file, + patch_single_content, +) +from update_lib.patch_spec import COMMENT + + +class TestPatchSingleContent(unittest.TestCase): + """Tests for patch_single_content function.""" + + def test_patch_with_no_existing_file(self): + """Test patching when lib file doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Non-existent lib path + lib_path = tmpdir / "lib.py" + + result = patch_single_content(src_path, lib_path) + + # Should return source content unchanged + self.assertIn("def test_one(self):", result) + self.assertNotIn(COMMENT, result) + + def test_patch_with_existing_patches(self): + """Test patching preserves existing patches.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file (new version) + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + def test_two(self): + pass +""") + + # Create lib file with existing patch + lib_path = tmpdir / "lib.py" + lib_path.write_text(f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + result = patch_single_content(src_path, lib_path) + + # Should have patch on test_one + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + # Should have test_two from source + self.assertIn("def test_two(self):", result) + + +class TestPatchFile(unittest.TestCase): + """Tests for patch_file function.""" + + def test_patch_file_creates_output(self): + """Test that patch_file writes output file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Output path + lib_path = tmpdir / "Lib" / "test.py" + + patch_file(src_path, lib_path, verbose=False) + + # File should exist + self.assertTrue(lib_path.exists()) + content = lib_path.read_text() + self.assertIn("def test_one(self):", content) + + def test_patch_file_preserves_patches(self): + """Test that patch_file preserves existing patches.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Create existing lib file with patch + lib_path = tmpdir / "lib.py" + lib_path.write_text(f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + patch_file(src_path, lib_path, verbose=False) + + content = lib_path.read_text() + self.assertIn("@unittest.expectedFailure", content) + + +class TestPatchDirectory(unittest.TestCase): + """Tests for patch_directory function.""" + + def test_patch_directory_all_files(self): + """Test that patch_directory processes all .py files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source directory with files + src_dir = tmpdir / "src" + src_dir.mkdir() + (src_dir / "test_a.py").write_text("# test_a") + (src_dir / "test_b.py").write_text("# test_b") + (src_dir / "subdir").mkdir() + (src_dir / "subdir" / "test_c.py").write_text("# test_c") + + # Output directory + lib_dir = tmpdir / "lib" + + patch_directory(src_dir, lib_dir, verbose=False) + + # All files should exist + self.assertTrue((lib_dir / "test_a.py").exists()) + self.assertTrue((lib_dir / "test_b.py").exists()) + self.assertTrue((lib_dir / "subdir" / "test_c.py").exists()) + + def test_patch_directory_preserves_patches(self): + """Test that patch_directory preserves patches in existing files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source directory + src_dir = tmpdir / "src" + src_dir.mkdir() + (src_dir / "test_a.py").write_text("""import unittest + +class TestA(unittest.TestCase): + def test_one(self): + pass +""") + + # Create lib directory with patched file + lib_dir = tmpdir / "lib" + lib_dir.mkdir() + (lib_dir / "test_a.py").write_text(f"""import unittest + +class TestA(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + patch_directory(src_dir, lib_dir, verbose=False) + + content = (lib_dir / "test_a.py").read_text() + self.assertIn("@unittest.expectedFailure", content) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_patch_spec.py b/scripts/update_lib/tests/test_patch_spec.py new file mode 100644 index 00000000000..798bd851b3c --- /dev/null +++ b/scripts/update_lib/tests/test_patch_spec.py @@ -0,0 +1,362 @@ +"""Tests for patch_spec.py - core patch extraction and application.""" + +import ast +import unittest + +from update_lib.patch_spec import ( + COMMENT, + PatchSpec, + UtMethod, + _find_import_insert_line, + apply_patches, + extract_patches, + iter_tests, +) + + +class TestIterTests(unittest.TestCase): + """Tests for iter_tests function.""" + + def test_iter_tests_simple(self): + """Test iterating over test methods in a class.""" + code = """ +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + def test_two(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 2) + self.assertEqual(results[0][0].name, "TestFoo") + self.assertEqual(results[0][1].name, "test_one") + self.assertEqual(results[1][1].name, "test_two") + + def test_iter_tests_multiple_classes(self): + """Test iterating over multiple test classes.""" + code = """ +class TestFoo(unittest.TestCase): + def test_foo(self): + pass + +class TestBar(unittest.TestCase): + def test_bar(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 2) + self.assertEqual(results[0][0].name, "TestFoo") + self.assertEqual(results[1][0].name, "TestBar") + + def test_iter_tests_async(self): + """Test iterating over async test methods.""" + code = """ +class TestAsync(unittest.TestCase): + async def test_async(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0][1].name, "test_async") + + +class TestExtractPatches(unittest.TestCase): + """Tests for extract_patches function.""" + + def test_extract_expected_failure(self): + """Test extracting @unittest.expectedFailure decorator.""" + code = f""" +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + self.assertIn("test_one", patches["TestFoo"]) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(len(specs), 1) + self.assertEqual(specs[0].ut_method, UtMethod.ExpectedFailure) + + def test_extract_expected_failure_inline_comment(self): + """Test extracting expectedFailure with inline comment.""" + code = f""" +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + self.assertIn("test_one", patches["TestFoo"]) + + def test_extract_skip_with_reason(self): + """Test extracting @unittest.skip with reason.""" + code = f''' +class TestFoo(unittest.TestCase): + @unittest.skip("{COMMENT}; not implemented") + def test_one(self): + pass +''' + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(specs[0].ut_method, UtMethod.Skip) + self.assertIn("not implemented", specs[0].reason) + + def test_extract_skip_if(self): + """Test extracting @unittest.skipIf decorator.""" + code = f''' +class TestFoo(unittest.TestCase): + @unittest.skipIf(sys.platform == "win32", "{COMMENT}; windows issue") + def test_one(self): + pass +''' + patches = extract_patches(code) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(specs[0].ut_method, UtMethod.SkipIf) + # ast.unparse normalizes quotes to single quotes + self.assertIn("sys.platform", specs[0].cond) + self.assertIn("win32", specs[0].cond) + + def test_no_patches_without_comment(self): + """Test that decorators without COMMENT are not extracted.""" + code = """ +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertEqual(patches, {}) + + def test_multiple_patches_same_method(self): + """Test extracting multiple decorators on same method.""" + code = f''' +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + @unittest.skip("{COMMENT}; reason") + def test_one(self): + pass +''' + patches = extract_patches(code) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(len(specs), 2) + + +class TestApplyPatches(unittest.TestCase): + """Tests for apply_patches function.""" + + def test_apply_expected_failure(self): + """Test applying @unittest.expectedFailure.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + def test_apply_skip_with_reason(self): + """Test applying @unittest.skip with reason.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.Skip, None, "not ready")]} + } + result = apply_patches(code, patches) + self.assertIn("@unittest.skip", result) + self.assertIn("not ready", result) + + def test_apply_skip_if(self): + """Test applying @unittest.skipIf.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": { + "test_one": [ + PatchSpec(UtMethod.SkipIf, "sys.platform == 'win32'", "windows") + ] + } + } + result = apply_patches(code, patches) + self.assertIn("@unittest.skipIf", result) + self.assertIn('sys.platform == "win32"', result) + + def test_apply_preserves_existing_decorators(self): + """Test that existing decorators are preserved.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @some_decorator + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + self.assertIn("@some_decorator", result) + self.assertIn("@unittest.expectedFailure", result) + + def test_apply_inherited_method(self): + """Test applying patch to inherited method (creates override).""" + code = """import unittest + +class TestFoo(unittest.TestCase): + pass +""" + patches = { + "TestFoo": { + "test_inherited": [PatchSpec(UtMethod.ExpectedFailure, None, "")] + } + } + result = apply_patches(code, patches) + self.assertIn("def test_inherited(self):", result) + self.assertIn("return super().test_inherited()", result) + + def test_apply_adds_unittest_import(self): + """Test that unittest import is added if missing.""" + code = """import sys + +class TestFoo: + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + # Should add unittest import after existing imports + self.assertIn("import unittest", result) + + def test_apply_no_duplicate_import(self): + """Test that unittest import is not duplicated.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + # Count occurrences of 'import unittest' + count = result.count("import unittest") + self.assertEqual(count, 1) + + +class TestPatchSpec(unittest.TestCase): + """Tests for PatchSpec class.""" + + def test_as_decorator_expected_failure(self): + """Test generating expectedFailure decorator string.""" + spec = PatchSpec(UtMethod.ExpectedFailure, None, "reason") + decorator = spec.as_decorator() + self.assertIn("@unittest.expectedFailure", decorator) + self.assertIn(COMMENT, decorator) + self.assertIn("reason", decorator) + + def test_as_decorator_skip(self): + """Test generating skip decorator string.""" + spec = PatchSpec(UtMethod.Skip, None, "not ready") + decorator = spec.as_decorator() + self.assertIn("@unittest.skip", decorator) + self.assertIn("not ready", decorator) + + def test_as_decorator_skip_if(self): + """Test generating skipIf decorator string.""" + spec = PatchSpec(UtMethod.SkipIf, "condition", "reason") + decorator = spec.as_decorator() + self.assertIn("@unittest.skipIf", decorator) + self.assertIn("condition", decorator) + + +class TestRoundTrip(unittest.TestCase): + """Tests for extract -> apply round trip.""" + + def test_round_trip_expected_failure(self): + """Test that extracted patches can be re-applied.""" + original = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + # Extract patches + patches = extract_patches(original) + + # Apply to clean code + clean = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_patches(clean, patches) + + # Should have the decorator + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + +class TestFindImportInsertLine(unittest.TestCase): + """Tests for _find_import_insert_line function.""" + + def test_with_imports(self): + """Test finding line after imports.""" + code = """import os +import sys + +class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 2) + + def test_no_imports_with_docstring(self): + """Test fallback to after docstring when no imports.""" + code = '''"""Module docstring.""" + +class Foo: + pass +''' + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 1) + + def test_no_imports_no_docstring(self): + """Test fallback to line 0 when no imports and no docstring.""" + code = """class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_path.py b/scripts/update_lib/tests/test_path.py new file mode 100644 index 00000000000..f2dcdcf8f05 --- /dev/null +++ b/scripts/update_lib/tests/test_path.py @@ -0,0 +1,224 @@ +"""Tests for path.py - path utilities.""" + +import pathlib +import tempfile +import unittest + +from update_lib.file_utils import ( + get_test_files, + get_test_module_name, + is_lib_path, + is_test_path, + lib_to_test_path, + parse_lib_path, +) + + +class TestParseLibPath(unittest.TestCase): + """Tests for parse_lib_path function.""" + + def test_parse_cpython_path(self): + """Test parsing cpython/Lib/... path.""" + result = parse_lib_path("cpython/Lib/test/test_foo.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo.py")) + + def test_parse_nested_path(self): + """Test parsing deeply nested path.""" + result = parse_lib_path("/home/user/cpython/Lib/test/test_foo/test_bar.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo/test_bar.py")) + + def test_parse_windows_path(self): + """Test parsing Windows-style path.""" + result = parse_lib_path("C:\\cpython\\Lib\\test\\test_foo.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo.py")) + + def test_parse_directory(self): + """Test parsing directory path.""" + result = parse_lib_path("cpython/Lib/test/test_json/") + self.assertEqual(result, pathlib.Path("Lib/test/test_json/")) + + def test_parse_no_lib_raises(self): + """Test that path without /Lib/ raises ValueError.""" + with self.assertRaises(ValueError) as ctx: + parse_lib_path("some/random/path.py") + self.assertIn("/Lib/", str(ctx.exception)) + + +class TestIsLibPath(unittest.TestCase): + """Tests for is_lib_path function.""" + + def test_lib_path(self): + """Test detecting Lib/ path.""" + self.assertTrue(is_lib_path(pathlib.Path("Lib/test/test_foo.py"))) + self.assertTrue(is_lib_path(pathlib.Path("./Lib/test/test_foo.py"))) + + def test_cpython_path_not_lib(self): + """Test that cpython/Lib/ is not detected as lib path.""" + self.assertFalse(is_lib_path(pathlib.Path("cpython/Lib/test/test_foo.py"))) + + def test_random_path_not_lib(self): + """Test that random path is not lib path.""" + self.assertFalse(is_lib_path(pathlib.Path("some/other/path.py"))) + + +class TestIsTestPath(unittest.TestCase): + """Tests for is_test_path function.""" + + def test_cpython_test_path(self): + """Test detecting cpython test path.""" + self.assertTrue(is_test_path(pathlib.Path("cpython/Lib/test/test_foo.py"))) + + def test_lib_test_path(self): + """Test detecting Lib/test path.""" + self.assertTrue(is_test_path(pathlib.Path("Lib/test/test_foo.py"))) + + def test_library_path_not_test(self): + """Test that library path (not test) is not test path.""" + self.assertFalse(is_test_path(pathlib.Path("cpython/Lib/dataclasses.py"))) + self.assertFalse(is_test_path(pathlib.Path("Lib/dataclasses.py"))) + + +class TestLibToTestPath(unittest.TestCase): + """Tests for lib_to_test_path function.""" + + def test_prefers_directory_over_file(self): + """Test that directory is preferred when both exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/foo.py, tmpdir/Lib/test/test_foo/, tmpdir/Lib/test/test_foo.py + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + (test_dir / "test_foo").mkdir() + (test_dir / "test_foo.py").write_text("# test file") + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should prefer directory + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo/") + + def test_falls_back_to_file(self): + """Test that file is used when directory doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/foo.py, tmpdir/Lib/test/test_foo.py (no directory) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + (test_dir / "test_foo.py").write_text("# test file") + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should fall back to file + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo.py") + + def test_defaults_to_directory_when_neither_exists(self): + """Test that directory path is returned when neither exists.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + # Neither test_foo/ nor test_foo.py exists + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should default to directory + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo/") + + def test_lib_path_prefers_directory(self): + """Test Lib/ path prefers directory when it exists.""" + # This test uses actual Lib/ paths, checking current behavior + # When neither exists, defaults to directory + result = lib_to_test_path(pathlib.Path("Lib/nonexistent_module.py")) + self.assertEqual(result, pathlib.Path("Lib/test/test_nonexistent_module/")) + + def test_init_py_uses_parent_name(self): + """Test __init__.py uses parent directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/json/__init__.py + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + json_dir = lib_dir / "json" + json_dir.mkdir() + (json_dir / "__init__.py").write_text("# json init") + test_dir = lib_dir / "test" + test_dir.mkdir() + + result = lib_to_test_path(tmpdir / "Lib" / "json" / "__init__.py") + # Should use "json" not "__init__" + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_json/") + + def test_init_py_lib_path_uses_parent_name(self): + """Test __init__.py with Lib/ path uses parent directory name.""" + result = lib_to_test_path(pathlib.Path("Lib/json/__init__.py")) + # Should use "json" not "__init__" + self.assertEqual(result, pathlib.Path("Lib/test/test_json/")) + + +class TestGetTestFiles(unittest.TestCase): + """Tests for get_test_files function.""" + + def test_single_file(self): + """Test getting single file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_file = tmpdir / "test.py" + test_file.write_text("# test") + + files = get_test_files(test_file) + self.assertEqual(len(files), 1) + self.assertEqual(files[0], test_file) + + def test_directory(self): + """Test getting all .py files from directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text("# a") + (tmpdir / "test_b.py").write_text("# b") + (tmpdir / "not_python.txt").write_text("# not python") + + files = get_test_files(tmpdir) + self.assertEqual(len(files), 2) + names = [f.name for f in files] + self.assertIn("test_a.py", names) + self.assertIn("test_b.py", names) + + def test_nested_directory(self): + """Test getting .py files from nested directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text("# a") + subdir = tmpdir / "subdir" + subdir.mkdir() + (subdir / "test_b.py").write_text("# b") + + files = get_test_files(tmpdir) + self.assertEqual(len(files), 2) + + +class TestTestNameFromPath(unittest.TestCase): + """Tests for get_test_module_name function.""" + + def test_simple_test_file(self): + """Test extracting name from simple test file.""" + path = pathlib.Path("Lib/test/test_foo.py") + self.assertEqual(get_test_module_name(path), "test_foo") + + def test_nested_test_file(self): + """Test extracting name from nested test directory.""" + path = pathlib.Path("Lib/test/test_ctypes/test_bar.py") + self.assertEqual(get_test_module_name(path), "test_ctypes.test_bar") + + def test_test_directory(self): + """Test extracting name from test directory.""" + path = pathlib.Path("Lib/test/test_json") + self.assertEqual(get_test_module_name(path), "test_json") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_quick.py b/scripts/update_lib/tests/test_quick.py new file mode 100644 index 00000000000..f0262eebd04 --- /dev/null +++ b/scripts/update_lib/tests/test_quick.py @@ -0,0 +1,287 @@ +"""Tests for quick.py - quick update functionality.""" + +import pathlib +import tempfile +import unittest +from unittest.mock import patch + +from update_lib.cmd_quick import ( + _expand_shortcut, + collect_original_methods, + get_cpython_dir, + git_commit, +) +from update_lib.file_utils import lib_to_test_path + + +class TestGetCpythonDir(unittest.TestCase): + """Tests for get_cpython_dir function.""" + + def test_extract_from_full_path(self): + """Test extracting cpython dir from full path.""" + path = pathlib.Path("cpython/Lib/dataclasses.py") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("cpython")) + + def test_extract_from_absolute_path(self): + """Test extracting cpython dir from absolute path.""" + path = pathlib.Path("/some/path/cpython/Lib/test/test_foo.py") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("/some/path/cpython")) + + def test_shortcut_defaults_to_cpython(self): + """Test that shortcut (no /Lib/) defaults to 'cpython'.""" + path = pathlib.Path("dataclasses") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("cpython")) + + +class TestExpandShortcut(unittest.TestCase): + """Tests for _expand_shortcut function.""" + + def test_expand_shortcut_to_test_path_integration(self): + """Test that expanded shortcut works with lib_to_test_path. + + This tests the fix for the bug where args.path was used instead of + the expanded src_path when calling lib_to_test_path. + """ + # Simulate the flow in main(): + # 1. User provides "dataclasses" + # 2. _expand_shortcut converts to "cpython/Lib/dataclasses.py" + # 3. lib_to_test_path should receive the expanded path, not original + + original_path = pathlib.Path("dataclasses") + expanded_path = _expand_shortcut(original_path) + + # If cpython/Lib/dataclasses.py exists, it should be expanded + if expanded_path != original_path: + # The expanded path should work with lib_to_test_path + test_path = lib_to_test_path(expanded_path) + # Should return a valid test path, not raise an error + self.assertTrue(str(test_path).startswith("cpython/Lib/test/")) + + # The original unexpanded path would fail or give wrong result + # This is what the bug was - using args.path instead of src_path + + def test_expand_shortcut_file(self): + """Test expanding a simple name to file path.""" + # This test checks the shortcut works when file exists + path = pathlib.Path("dataclasses") + result = _expand_shortcut(path) + + expected_file = pathlib.Path("cpython/Lib/dataclasses.py") + expected_dir = pathlib.Path("cpython/Lib/dataclasses") + + if expected_file.exists(): + self.assertEqual(result, expected_file) + elif expected_dir.exists(): + self.assertEqual(result, expected_dir) + else: + # If neither exists, should return original + self.assertEqual(result, path) + + def test_expand_shortcut_already_full_path(self): + """Test that full paths are not modified.""" + path = pathlib.Path("cpython/Lib/dataclasses.py") + result = _expand_shortcut(path) + self.assertEqual(result, path) + + def test_expand_shortcut_nonexistent(self): + """Test that nonexistent names are returned as-is.""" + path = pathlib.Path("nonexistent_module_xyz") + result = _expand_shortcut(path) + self.assertEqual(result, path) + + def test_expand_shortcut_uses_dependencies_table(self): + """Test that _expand_shortcut uses DEPENDENCIES table for overrides.""" + from update_lib.deps import DEPENDENCIES + + # regrtest has lib override in DEPENDENCIES + self.assertIn("regrtest", DEPENDENCIES) + self.assertIn("lib", DEPENDENCIES["regrtest"]) + + # _expand_shortcut should use this override when path exists + path = pathlib.Path("regrtest") + expected = pathlib.Path("cpython/Lib/test/libregrtest") + + # Only test expansion if cpython checkout exists + if expected.exists(): + result = _expand_shortcut(path) + self.assertEqual( + result, expected, "_expand_shortcut should expand 'regrtest'" + ) + + +class TestCollectOriginalMethods(unittest.TestCase): + """Tests for collect_original_methods function.""" + + def test_collect_from_file(self): + """Test collecting methods from single file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_file = tmpdir / "test.py" + test_file.write_text(""" +class TestFoo: + def test_one(self): + pass + + def test_two(self): + pass +""") + + methods = collect_original_methods(test_file) + self.assertIsInstance(methods, set) + self.assertEqual(len(methods), 2) + self.assertIn(("TestFoo", "test_one"), methods) + self.assertIn(("TestFoo", "test_two"), methods) + + def test_collect_from_directory(self): + """Test collecting methods from directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text(""" +class TestA: + def test_a(self): + pass +""") + (tmpdir / "test_b.py").write_text(""" +class TestB: + def test_b(self): + pass +""") + + methods = collect_original_methods(tmpdir) + self.assertIsInstance(methods, dict) + self.assertEqual(len(methods), 2) + + +class TestGitCommit(unittest.TestCase): + """Tests for git_commit function.""" + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_none_lib_path_not_added(self, mock_version, mock_run): + """Test that None lib_path doesn't add '.' to git.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test.py" + test_file.write_text("# test") + + git_commit("test", None, test_file, pathlib.Path("cpython"), verbose=False) + + # Check git add was called with only test_file, not "." + add_call = mock_run.call_args_list[0] + self.assertIn(str(test_file), add_call[0][0]) + self.assertNotIn(".", add_call[0][0][2:]) # Skip "git" and "add" + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_none_test_path_not_added(self, mock_version, mock_run): + """Test that None test_path doesn't add '.' to git.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + + git_commit("lib", lib_file, None, pathlib.Path("cpython"), verbose=False) + + add_call = mock_run.call_args_list[0] + self.assertIn(str(lib_file), add_call[0][0]) + self.assertNotIn(".", add_call[0][0][2:]) + + def test_both_none_returns_false(self): + """Test that both paths None returns False without git operations.""" + # No mocking needed - should return early before any subprocess calls + result = git_commit("test", None, None, pathlib.Path("cpython"), verbose=False) + self.assertFalse(result) + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_hard_deps_are_added(self, mock_version, mock_run): + """Test that hard_deps are included in git commit.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + test_file = pathlib.Path(tmpdir) / "test.py" + test_file.write_text("# test") + dep_file = pathlib.Path(tmpdir) / "_dep.py" + dep_file.write_text("# dep") + + git_commit( + "test", + lib_file, + test_file, + pathlib.Path("cpython"), + hard_deps=[dep_file], + verbose=False, + ) + + # Check git add was called with all three files + add_call = mock_run.call_args_list[0] + add_args = add_call[0][0] + self.assertIn(str(lib_file), add_args) + self.assertIn(str(test_file), add_args) + self.assertIn(str(dep_file), add_args) + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_nonexistent_hard_deps_not_added(self, mock_version, mock_run): + """Test that nonexistent hard_deps don't cause errors.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + nonexistent_dep = pathlib.Path(tmpdir) / "nonexistent.py" + + git_commit( + "test", + lib_file, + None, + pathlib.Path("cpython"), + hard_deps=[nonexistent_dep], + verbose=False, + ) + + # Check git add was called with only lib_file + add_call = mock_run.call_args_list[0] + add_args = add_call[0][0] + self.assertIn(str(lib_file), add_args) + self.assertNotIn(str(nonexistent_dep), add_args) + + +class TestQuickTestRunFailure(unittest.TestCase): + """Tests for quick() behavior when test run fails.""" + + @patch("update_lib.cmd_auto_mark.run_test") + def test_auto_mark_raises_on_test_run_failure(self, mock_run_test): + """Test that auto_mark_file raises when test run fails entirely.""" + from update_lib.cmd_auto_mark import TestResult, TestRunError, auto_mark_file + + # Simulate test runner crash (empty tests_result) + mock_run_test.return_value = TestResult( + tests_result="", tests=[], stdout="crash" + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create a fake test file with Lib/test structure + lib_test_dir = pathlib.Path(tmpdir) / "Lib" / "test" + lib_test_dir.mkdir(parents=True) + test_file = lib_test_dir / "test_foo.py" + test_file.write_text("import unittest\nclass Test(unittest.TestCase): pass") + + # auto_mark_file should raise TestRunError + with self.assertRaises(TestRunError): + auto_mark_file(test_file) + + +if __name__ == "__main__": + unittest.main() diff --git a/whats_left.py b/scripts/whats_left.py similarity index 91% rename from whats_left.py rename to scripts/whats_left.py index 91e46bef7ef..00db9a0ac5c 100755 --- a/whats_left.py +++ b/scripts/whats_left.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S python3 -I # /// script -# requires-python = ">=3.13" +# requires-python = ">=3.14" # /// # This script generates Lib/snippets/whats_left_data.py with these variables defined: @@ -29,7 +29,7 @@ if not sys.flags.isolated: print("running without -I option.") - print("python -I whats_left.py") + print("python -I scripts/whats_left.py") exit(1) GENERATED_FILE = "extra_tests/not_impl.py" @@ -37,9 +37,9 @@ implementation = platform.python_implementation() if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") -if sys.version_info[:2] < (3, 13): +if sys.version_info[:2] < (3, 14): sys.exit( - f"whats_left.py must be run under CPython 3.13 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I whats_left.py` to select a proper Python interpreter easier." + f"whats_left.py must be run under CPython 3.14 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I scripts/whats_left.py` to select a proper Python interpreter easier." ) @@ -60,6 +60,11 @@ def parse_args(): action="store_true", help="print output as JSON (instead of line by line)", ) + parser.add_argument( + "--no-default-features", + action="store_true", + help="disable default features when building RustPython", + ) parser.add_argument( "--features", action="store", @@ -195,6 +200,9 @@ def gen_methods(): typ = eval(typ_code) attrs = [] for attr in dir(typ): + # Skip attributes in dir() but not actually accessible (e.g., descriptor that raises) + if not hasattr(typ, attr): + continue if attr_is_not_inherited(typ, attr): attrs.append((attr, extra_info(getattr(typ, attr)))) methods[typ.__name__] = (typ_code, extra_info(typ), attrs) @@ -361,7 +369,9 @@ def method_incompatibility_reason(typ, method_name, real_method_value): if platform.python_implementation() == "CPython": if not_implementeds: - sys.exit("ERROR: CPython should have all the methods") + sys.exit( + f"ERROR: CPython should have all the methods but missing: {not_implementeds}" + ) mod_names = [ name.decode() @@ -436,25 +446,30 @@ def remove_one_indent(s): f.write(output + "\n") -subprocess.run( - ["cargo", "build", "--release", f"--features={args.features}"], check=True -) +cargo_build_command = ["cargo", "build", "--release"] +if args.no_default_features: + cargo_build_command.append("--no-default-features") +if args.features: + cargo_build_command.extend(["--features", args.features]) + +subprocess.run(cargo_build_command, check=True) + +cargo_run_command = ["cargo", "run", "--release"] +if args.no_default_features: + cargo_run_command.append("--no-default-features") +if args.features: + cargo_run_command.extend(["--features", args.features]) +cargo_run_command.extend(["-q", "--", GENERATED_FILE]) + result = subprocess.run( - [ - "cargo", - "run", - "--release", - f"--features={args.features}", - "-q", - "--", - GENERATED_FILE, - ], + cargo_run_command, env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"}, text=True, capture_output=True, ) # The last line should be json output, the rest of the lines can contain noise # because importing certain modules can print stuff to stdout/stderr +print(result.stderr, file=sys.stderr) result = json.loads(result.stdout.splitlines()[-1]) if args.json: diff --git a/src/interpreter.rs b/src/interpreter.rs index b79a1a0ffb4..b9ee2dbbc44 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,126 +1,76 @@ -use rustpython_vm::{Interpreter, PyRef, Settings, VirtualMachine, builtins::PyModule}; +use rustpython_vm::InterpreterBuilder; -pub type InitHook = Box<dyn FnOnce(&mut VirtualMachine)>; - -/// The convenient way to create [rustpython_vm::Interpreter] with stdlib and other stuffs. -/// -/// Basic usage: -/// ``` -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .interpreter(); -/// ``` -/// -/// To override [rustpython_vm::Settings]: -/// ``` -/// use rustpython_vm::Settings; -/// // Override your settings here. -/// let mut settings = Settings::default(); -/// settings.debug = 1; -/// // You may want to add paths to `rustpython_vm::Settings::path_list` to allow import python libraries. -/// settings.path_list.push("Lib".to_owned()); // add standard library directory -/// settings.path_list.push("".to_owned()); // add current working directory -/// let interpreter = rustpython::InterpreterConfig::new() -/// .settings(settings) -/// .interpreter(); -/// ``` -/// -/// To add native modules: -/// ```compile_fail -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .init_hook(Box::new(|vm| { -/// vm.add_native_module( -/// "your_module_name".to_owned(), -/// Box::new(your_module::make_module), -/// ); -/// })) -/// .interpreter(); -/// ``` -#[derive(Default)] -pub struct InterpreterConfig { - settings: Option<Settings>, - init_hooks: Vec<InitHook>, +/// Extension trait for InterpreterBuilder to add rustpython-specific functionality. +pub trait InterpreterBuilderExt { + /// Initialize the Python standard library. + /// + /// Requires the `stdlib` feature to be enabled. + #[cfg(feature = "stdlib")] + fn init_stdlib(self) -> Self; } -impl InterpreterConfig { - pub fn new() -> Self { - Self::default() - } - pub fn interpreter(self) -> Interpreter { - let settings = self.settings.unwrap_or_default(); - Interpreter::with_init(settings, |vm| { - for hook in self.init_hooks { - hook(vm); - } - }) - } - - pub fn settings(mut self, settings: Settings) -> Self { - self.settings = Some(settings); - self - } - pub fn init_hook(mut self, hook: InitHook) -> Self { - self.init_hooks.push(hook); - self - } - pub fn add_native_module( - self, - name: String, - make_module: fn(&VirtualMachine) -> PyRef<PyModule>, - ) -> Self { - self.init_hook(Box::new(move |vm| { - vm.add_native_module(name, Box::new(make_module)) - })) - } +impl InterpreterBuilderExt for InterpreterBuilder { #[cfg(feature = "stdlib")] - pub fn init_stdlib(self) -> Self { - self.init_hook(Box::new(init_stdlib)) + fn init_stdlib(self) -> Self { + let defs = rustpython_stdlib::stdlib_module_defs(&self.ctx); + let builder = self.add_native_modules(&defs); + + #[cfg(feature = "freeze-stdlib")] + let builder = builder + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .init_hook(set_frozen_stdlib_dir); + + #[cfg(not(feature = "freeze-stdlib"))] + let builder = builder.init_hook(setup_dynamic_stdlib); + + builder } } -#[cfg(feature = "stdlib")] -pub fn init_stdlib(vm: &mut VirtualMachine) { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); +/// Set stdlib_dir for frozen standard library +#[cfg(all(feature = "stdlib", feature = "freeze-stdlib"))] +fn set_frozen_stdlib_dir(vm: &mut crate::VirtualMachine) { + use rustpython_vm::common::rc::PyRc; - // if we're on freeze-stdlib, the core stdlib modules will be included anyway - #[cfg(feature = "freeze-stdlib")] - { - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); + let state = PyRc::get_mut(&mut vm.state).unwrap(); + state.config.paths.stdlib_dir = Some(rustpython_pylib::LIB_PATH.to_owned()); +} - // FIXME: Remove this hack once sys._stdlib_dir is properly implemented or _frozen_importlib doesn't depend on it anymore. - assert!(vm.sys_module.get_attr("_stdlib_dir", vm).is_err()); - vm.sys_module - .set_attr( - "_stdlib_dir", - vm.new_pyobj(rustpython_pylib::LIB_PATH.to_owned()), - vm, - ) - .unwrap(); - } +/// Setup dynamic standard library loading from filesystem +#[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] +fn setup_dynamic_stdlib(vm: &mut crate::VirtualMachine) { + use rustpython_vm::common::rc::PyRc; - #[cfg(not(feature = "freeze-stdlib"))] - { - use rustpython_vm::common::rc::PyRc; + let state = PyRc::get_mut(&mut vm.state).unwrap(); + let paths = collect_stdlib_paths(); - let state = PyRc::get_mut(&mut vm.state).unwrap(); - let settings = &mut state.settings; + // Set stdlib_dir to the first stdlib path if available + if let Some(first_path) = paths.first() { + state.config.paths.stdlib_dir = Some(first_path.clone()); + } - let path_list = std::mem::take(&mut settings.path_list); + // Insert at the beginning so stdlib comes before user paths + for path in paths.into_iter().rev() { + state.config.paths.module_search_paths.insert(0, path); + } +} - // BUILDTIME_RUSTPYTHONPATH should be set when distributing - if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { - settings.path_list.extend( - crate::settings::split_paths(paths) - .map(|path| path.into_os_string().into_string().unwrap()), - ) - } else { - #[cfg(feature = "rustpython-pylib")] - settings - .path_list - .push(rustpython_pylib::LIB_PATH.to_owned()) +/// Collect standard library paths from build-time configuration +#[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] +fn collect_stdlib_paths() -> Vec<String> { + // BUILDTIME_RUSTPYTHONPATH should be set when distributing + if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { + crate::settings::split_paths(paths) + .map(|path| path.into_os_string().into_string().unwrap()) + .collect() + } else { + #[cfg(feature = "rustpython-pylib")] + { + vec![rustpython_pylib::LIB_PATH.to_owned()] + } + #[cfg(not(feature = "rustpython-pylib"))] + { + vec![] } - - settings.path_list.extend(path_list); } } diff --git a/src/lib.rs b/src/lib.rs index 84a774ab029..60b66d83b3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,23 +1,29 @@ //! This is the `rustpython` binary. If you're looking to embed RustPython into your application, //! you're likely looking for the [`rustpython_vm`] crate. //! -//! You can install `rustpython` with `cargo install rustpython`, or if you'd like to inject your -//! own native modules you can make a binary crate that depends on the `rustpython` crate (and +//! You can install `rustpython` with `cargo install rustpython`. If you'd like to inject your +//! own native modules, you can make a binary crate that depends on the `rustpython` crate (and //! probably [`rustpython_vm`], too), and make a `main.rs` that looks like: //! //! ```no_run +//! use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; //! use rustpython_vm::{pymodule, py_freeze}; -//! fn main() { -//! rustpython::run(|vm| { -//! vm.add_native_module("my_mod".to_owned(), Box::new(my_mod::make_module)); -//! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); -//! }); +//! +//! fn main() -> std::process::ExitCode { +//! let builder = InterpreterBuilder::new().init_stdlib(); +//! // Add a native module using builder.ctx +//! let my_mod_def = my_mod::module_def(&builder.ctx); +//! let builder = builder +//! .add_native_module(my_mod_def) +//! // Add a frozen module +//! .add_frozen_modules(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); +//! +//! rustpython::run(builder) //! } //! //! #[pymodule] //! mod my_mod { //! use rustpython_vm::builtins::PyStrRef; -//TODO: use rustpython_vm::prelude::*; //! //! #[pyfunction] //! fn do_thing(x: i32) -> i32 { @@ -35,8 +41,9 @@ //! //! The binary will have all the standard arguments of a python interpreter (including a REPL!) but //! it will have your modules loaded into the vm. +//! +//! See [`rustpython_derive`](../rustpython_derive/index.html) crate for documentation on macros used in the example above. -#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] #![allow(clippy::needless_doctest_main)] #[macro_use] @@ -49,13 +56,13 @@ mod interpreter; mod settings; mod shell; -use rustpython_vm::{PyResult, VirtualMachine, scope::Scope}; +use rustpython_vm::{AsObject, PyObjectRef, PyResult, VirtualMachine, scope::Scope}; use std::env; use std::io::IsTerminal; use std::process::ExitCode; -pub use interpreter::InterpreterConfig; -pub use rustpython_vm as vm; +pub use interpreter::InterpreterBuilderExt; +pub use rustpython_vm::{self as vm, Interpreter, InterpreterBuilder}; pub use settings::{InstallPipMode, RunMode, parse_opts}; pub use shell::run_shell; @@ -69,7 +76,11 @@ compile_error!( /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` /// based on the return code of the python code ran through the cli. -pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { +/// +/// **Note**: This function provides no way to further initialize the VM after the builder is applied. +/// All VM initialization (adding native modules, init hooks, etc.) must be done through the +/// [`InterpreterBuilder`] parameter before calling this function. +pub fn run(mut builder: InterpreterBuilder) -> ExitCode { env_logger::init(); // NOTE: This is not a WASI convention. But it will be convenient since POSIX shell always defines it. @@ -101,34 +112,14 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { } } - let mut config = InterpreterConfig::new().settings(settings); - #[cfg(feature = "stdlib")] - { - config = config.init_stdlib(); - } - config = config.init_hook(Box::new(init)); + builder = builder.settings(settings); - let interp = config.interpreter(); + let interp = builder.interpreter(); let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode)); rustpython_vm::common::os::exit_code(exitcode) } -fn setup_main_module(vm: &VirtualMachine) -> PyResult<Scope> { - let scope = vm.new_scope_with_builtins(); - let main_module = vm.new_module("__main__", scope.globals.clone(), None); - main_module - .dict() - .set_item("__annotations__", vm.ctx.new_dict().into(), vm) - .expect("Failed to initialize __main__.__annotations__"); - - vm.sys_module - .get_attr("modules", vm)? - .set_item("__main__", main_module.into(), vm)?; - - Ok(scope) -} - fn get_pip(scope: Scope, vm: &VirtualMachine) -> PyResult<()> { let get_getpip = rustpython_vm::py_compile!( source = r#"\ @@ -144,7 +135,7 @@ __import__("io").TextIOWrapper( .downcast() .expect("TextIOWrapper.read() should return str"); eprintln!("running get-pip.py..."); - vm.run_code_string(scope, getpip_code.as_str(), "get-pip.py".to_owned())?; + vm.run_string(scope, getpip_code.as_str(), "get-pip.py".to_owned())?; Ok(()) } @@ -162,23 +153,89 @@ fn install_pip(installer: InstallPipMode, scope: Scope, vm: &VirtualMachine) -> } } +// pymain_run_file_obj in Modules/main.c +fn run_file(vm: &VirtualMachine, scope: Scope, path: &str) -> PyResult<()> { + // Check if path is a package/directory with __main__.py + if let Some(_importer) = get_importer(path, vm)? { + vm.insert_sys_path(vm.new_pyobj(path))?; + let runpy = vm.import("runpy", 0)?; + let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?; + run_module_as_main.call((vm::identifier!(vm, __main__).to_owned(), false), vm)?; + return Ok(()); + } + + // Add script directory to sys.path[0] + if !vm.state.config.settings.safe_path { + let dir = std::path::Path::new(path) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or(""); + vm.insert_sys_path(vm.new_pyobj(dir))?; + } + + #[cfg(feature = "host_env")] + { + vm.run_any_file(scope, path) + } + #[cfg(not(feature = "host_env"))] + { + // In sandbox mode, the binary reads the file and feeds source to the VM. + // The VM itself has no filesystem access. + let path = if path.is_empty() { "???" } else { path }; + match std::fs::read_to_string(path) { + Ok(source) => vm.run_string(scope, &source, path.to_owned()).map(drop), + Err(err) => Err(vm.new_os_error(err.to_string())), + } + } +} + +fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + use rustpython_vm::builtins::PyDictRef; + use rustpython_vm::convert::TryFromObject; + + let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?; + let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?; + if let Some(importer) = path_importer_cache.get_item_opt(path, vm)? { + return Ok(Some(importer)); + } + let path_obj = vm.ctx.new_str(path); + let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; + let mut importer = None; + let path_hooks: Vec<PyObjectRef> = path_hooks.try_into_value(vm)?; + for path_hook in path_hooks { + match path_hook.call((path_obj.clone(),), vm) { + Ok(imp) => { + importer = Some(imp); + break; + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.import_error) => continue, + Err(e) => return Err(e), + } + } + Ok(if let Some(imp) = importer { + let imp = path_importer_cache.get_or_insert(vm, path_obj.into(), || imp.clone())?; + Some(imp) + } else { + None + }) +} + // pymain_run_python fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); - let scope = setup_main_module(vm)?; + let scope = vm.new_scope_with_main()?; - if !vm.state.settings.safe_path { - // TODO: The prepending path depends on running mode - // See https://docs.python.org/3/using/cmdline.html#cmdoption-P - vm.run_code_string( - vm.new_scope_with_builtins(), - "import sys; sys.path.insert(0, '')", - "<embedded>".to_owned(), - )?; + // Initialize warnings module to process sys.warnoptions + // _PyWarnings_Init() + if vm.import("warnings", 0).is_err() { + warn!("Failed to import warnings module"); } + // Import site first, before setting sys.path[0] + // This matches CPython's behavior where site.removeduppaths() runs + // before sys.path[0] is set, preventing '' from being converted to cwd let site_result = vm.import("site", 0); if site_result.is_err() { warn!( @@ -187,19 +244,31 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { ); } + // _PyPathConfig_ComputeSysPath0 - set sys.path[0] after site import + if !vm.state.config.settings.safe_path { + let path0: Option<String> = match &run_mode { + RunMode::Command(_) => Some(String::new()), + RunMode::Module(_) => env::current_dir() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_owned())), + RunMode::Script(_) | RunMode::InstallPip(_) => None, // handled by run_script + RunMode::Repl => Some(String::new()), + }; + + if let Some(path) = path0 { + vm.insert_sys_path(vm.new_pyobj(path))?; + } + } + // Enable faulthandler if -X faulthandler, PYTHONFAULTHANDLER or -X dev is set // _PyFaulthandler_Init() - if vm.state.settings.faulthandler { - let _ = vm.run_code_string( - vm.new_scope_with_builtins(), - "import faulthandler; faulthandler.enable()", - "<faulthandler>".to_owned(), - ); + if vm.state.config.settings.faulthandler { + let _ = vm.run_simple_string("import faulthandler; faulthandler.enable()"); } let is_repl = matches!(run_mode, RunMode::Repl); - if !vm.state.settings.quiet - && (vm.state.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) + if !vm.state.config.settings.quiet + && (vm.state.config.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) { eprintln!( "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", @@ -217,7 +286,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { let res = match run_mode { RunMode::Command(command) => { debug!("Running command {command}"); - vm.run_code_string(scope.clone(), &command, "<string>".to_owned()) + vm.run_string(scope.clone(), &command, "<string>".to_owned()) .map(drop) } RunMode::Module(module) => { @@ -226,30 +295,31 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { } RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm), RunMode::Script(script_path) => { - // pymain_run_file + // pymain_run_file_obj debug!("Running script {}", &script_path); - vm.run_script(scope.clone(), &script_path) + run_file(vm, scope.clone(), &script_path) } RunMode::Repl => Ok(()), }; - if is_repl || vm.state.settings.inspect { - shell::run_shell(vm, scope)?; + let result = if is_repl || vm.state.config.settings.inspect { + shell::run_shell(vm, scope) } else { - res?; - } + res + }; #[cfg(feature = "flame-it")] { main_guard.end(); - if let Err(e) = write_profile(&vm.state.as_ref().settings) { + if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { error!("Error writing profile information: {}", e); } } - Ok(()) + + result } #[cfg(feature = "flame-it")] -fn write_profile(settings: &Settings) -> Result<(), Box<dyn std::error::Error>> { +fn write_profile(settings: &Settings) -> Result<(), Box<dyn core::error::Error>> { use std::{fs, io}; enum ProfileFormat { @@ -300,20 +370,23 @@ mod tests { use rustpython_vm::Interpreter; fn interpreter() -> Interpreter { - InterpreterConfig::new().init_stdlib().interpreter() + InterpreterBuilder::new().init_stdlib().interpreter() } #[test] fn test_run_script() { interpreter().enter(|vm| { vm.unwrap_pyresult((|| { - let scope = setup_main_module(vm)?; + let scope = vm.new_scope_with_main()?; // test file run - vm.run_script(scope, "extra_tests/snippets/dir_main/__main__.py")?; - - let scope = setup_main_module(vm)?; - // test module run - vm.run_script(scope, "extra_tests/snippets/dir_main")?; + run_file(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?; + + #[cfg(feature = "host_env")] + { + let scope = vm.new_scope_with_main()?; + // test module run (directory with __main__.py) + run_file(vm, scope, "extra_tests/snippets/dir_main")?; + } Ok(()) })()); diff --git a/src/main.rs b/src/main.rs index e88ea40f3df..3953b9dacfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,10 @@ +use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; + pub fn main() -> std::process::ExitCode { - rustpython::run(|_vm| {}) + let mut config = InterpreterBuilder::new(); + #[cfg(feature = "stdlib")] + { + config = config.init_stdlib(); + } + rustpython::run(config) } diff --git a/src/settings.rs b/src/settings.rs index f77db4d159d..79c67dafce0 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -269,7 +269,23 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { "dev" => settings.dev_mode = true, "faulthandler" => settings.faulthandler = true, "warn_default_encoding" => settings.warn_default_encoding = true, + "utf8" => { + settings.utf8_mode = match value { + None => 1, + Some("1") => 1, + Some("0") => 0, + _ => { + error!( + "Fatal Python error: config_init_utf8_mode: \ + -X utf8=n: n is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } "no_sig_int" => settings.install_signal_handlers = false, + "no_debug_ranges" => settings.code_debug_ranges = false, "int_max_str_digits" => { settings.int_max_str_digits = match value.unwrap().parse() { Ok(digits) if digits == 0 || digits >= 640 => digits, @@ -284,6 +300,20 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { } }; } + "thread_inherit_context" => { + settings.thread_inherit_context = match value { + Some("1") => true, + Some("0") => false, + _ => { + error!( + "Fatal Python error: config_init_thread_inherit_context: \ + -X thread_inherit_context=n: n is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } _ => {} } (name, value.map(str::to_owned)) @@ -293,6 +323,40 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { settings.warn_default_encoding = settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING"); settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER"); + if env_bool("PYTHONNODEBUGRANGES") { + settings.code_debug_ranges = false; + } + if let Some(val) = get_env("PYTHON_THREAD_INHERIT_CONTEXT") { + settings.thread_inherit_context = match val.to_str() { + Some("1") => true, + Some("0") => false, + _ => { + error!( + "Fatal Python error: config_init_thread_inherit_context: \ + PYTHON_THREAD_INHERIT_CONTEXT=N: N is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } + + // Parse PYTHONIOENCODING=encoding[:errors] + if let Some(val) = get_env("PYTHONIOENCODING") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() + { + if let Some((enc, err)) = val_str.split_once(':') { + if !enc.is_empty() { + settings.stdio_encoding = Some(enc.to_owned()); + } + if !err.is_empty() { + settings.stdio_errors = Some(err.to_owned()); + } + } else { + settings.stdio_encoding = Some(val_str.to_owned()); + } + } if settings.dev_mode { settings.warnoptions.push("default".to_owned()); @@ -306,6 +370,17 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { }; settings.warnoptions.push(warn.to_owned()); } + if let Some(val) = get_env("PYTHONWARNINGS") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() + { + for warning in val_str.split(',') { + let warning = warning.trim(); + if !warning.is_empty() { + settings.warnoptions.push(warning.to_owned()); + } + } + } settings.warnoptions.extend(args.warning_control); settings.hash_seed = match (!args.random_hash_seed) @@ -339,6 +414,7 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { /// Helper function to retrieve a sequence of paths from an environment variable. fn get_paths(env_variable_name: &str) -> impl Iterator<Item = String> + '_ { env::var_os(env_variable_name) + .filter(|v| !v.is_empty()) .into_iter() .flat_map(move |paths| { split_paths(&paths) @@ -357,8 +433,10 @@ pub(crate) use env::split_paths; pub(crate) fn split_paths<T: AsRef<std::ffi::OsStr> + ?Sized>( s: &T, ) -> impl Iterator<Item = std::path::PathBuf> + '_ { - use std::os::wasi::ffi::OsStrExt; - let s = s.as_ref().as_bytes(); - s.split(|b| *b == b':') - .map(|x| std::ffi::OsStr::from_bytes(x).to_owned().into()) + let s = s.as_ref().as_encoded_bytes(); + s.split(|b| *b == b':').map(|x| { + unsafe { std::ffi::OsStr::from_encoded_bytes_unchecked(x) } + .to_owned() + .into() + }) } diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 3acc105e3c3..287e0b6ffeb 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -22,7 +22,7 @@ "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", "serve": "^14.2.5", - "webpack": "^5.97.1", + "webpack": "^5.105.0", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" } @@ -364,9 +364,9 @@ } }, "node_modules/@types/estree": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "dev": true, "license": "MIT" }, @@ -775,8 +775,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@xtuc/ieee754": { "version": "1.2.0", @@ -814,9 +813,9 @@ } }, "node_modules/acorn": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", - "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", "bin": { @@ -826,13 +825,25 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-import-phases": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", + "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + }, + "peerDependencies": { + "acorn": "^8.14.0" + } + }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -998,6 +1009,16 @@ "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", + "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -1132,9 +1153,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", - "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", "dev": true, "funding": [ { @@ -1151,12 +1172,12 @@ } ], "license": "MIT", - "peer": true, "dependencies": { - "caniuse-lite": "^1.0.30001688", - "electron-to-chromium": "^1.5.73", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.1" + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" @@ -1254,9 +1275,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001701", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001701.tgz", - "integrity": "sha512-faRs/AW3jA9nTwmJBSO1PQ6L/EOgsB5HMQQq4iCu5zhPgVVgO/pZRHlmatwijZKetFw8/Pr4q6dEN8sJuq8qTw==", + "version": "1.0.30001768", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001768.tgz", + "integrity": "sha512-qY3aDRZC5nWPgHUgIB84WL+nySuo19wk0VJpp/XI9T34lrvkyhRvNVOFJOp2kxClQhiFBu+TaUSudf6oa3vkSA==", "dev": true, "funding": [ { @@ -1979,9 +2000,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.109", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.109.tgz", - "integrity": "sha512-AidaH9JETVRr9DIPGfp1kAarm/W6hRJTPuCnkF+2MqhF4KaAgRIcBc8nvjk+YMXZhwfISof/7WG29eS4iGxQLQ==", + "version": "1.5.286", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", + "integrity": "sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==", "dev": true, "license": "ISC" }, @@ -2003,14 +2024,14 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.1", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.1.tgz", - "integrity": "sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==", + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", + "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.0" }, "engines": { "node": ">=10.13.0" @@ -2060,9 +2081,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.6.0.tgz", - "integrity": "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", + "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", "dev": true, "license": "MIT" }, @@ -2205,40 +2226,40 @@ } }, "node_modules/express": { - "version": "4.21.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", - "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "dev": true, "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", - "on-finished": "2.4.1", + "on-finished": "~2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.12", + "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "6.13.0", + "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", + "send": "~0.19.0", + "serve-static": "~1.16.2", "setprototypeof": "1.2.0", - "statuses": "2.0.1", + "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -2271,6 +2292,22 @@ "dev": true, "license": "MIT" }, + "node_modules/express/node_modules/qs": { + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/express/node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3202,13 +3239,17 @@ } }, "node_modules/loader-runner": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", - "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", + "integrity": "sha512-IWqP2SCPhyVFTBtRcgMHdzlf9ul25NwaFx4wCEH/KjAXuuHY4yNjvPXsBokp8jCB936PyWRaPKUNh8NvylLp2Q==", "dev": true, "license": "MIT", "engines": { "node": ">=6.11.5" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/locate-path": { @@ -3225,9 +3266,9 @@ } }, "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", "dev": true, "license": "MIT" }, @@ -3508,9 +3549,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.19", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", - "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, @@ -3827,7 +3868,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", @@ -4239,9 +4279,9 @@ "license": "MIT" }, "node_modules/schema-utils": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz", - "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.3.tgz", + "integrity": "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==", "dev": true, "license": "MIT", "dependencies": { @@ -4951,13 +4991,17 @@ } }, "node_modules/tapable": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", - "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", "dev": true, "license": "MIT", "engines": { "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/terser": { @@ -4980,9 +5024,9 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.12", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.12.tgz", - "integrity": "sha512-jDLYqo7oF8tJIttjXO6jBY5Hk8p3A8W4ttih7cCEq64fQFWmgJ4VqAQjKr7WwIDlmXKEc6QeoRb5ecjZ+2afcg==", + "version": "5.3.16", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", + "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", "dev": true, "license": "MIT", "dependencies": { @@ -5086,8 +5130,7 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "dev": true, - "license": "0BSD", - "peer": true + "license": "0BSD" }, "node_modules/type-fest": { "version": "2.19.0", @@ -5134,9 +5177,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", - "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", "dev": true, "funding": [ { @@ -5242,9 +5285,9 @@ "license": "MIT" }, "node_modules/watchpack": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz", - "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", + "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", "dev": true, "license": "MIT", "dependencies": { @@ -5266,36 +5309,37 @@ } }, "node_modules/webpack": { - "version": "5.98.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz", - "integrity": "sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA==", + "version": "5.105.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", + "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", - "@types/estree": "^1.0.6", + "@types/estree": "^1.0.8", + "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.14.0", - "browserslist": "^4.24.0", + "acorn": "^8.15.0", + "acorn-import-phases": "^1.0.3", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.1", - "es-module-lexer": "^1.2.1", + "enhanced-resolve": "^5.19.0", + "es-module-lexer": "^2.0.0", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", + "loader-runner": "^4.3.1", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^4.3.0", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.1", - "webpack-sources": "^3.2.3" + "schema-utils": "^4.3.3", + "tapable": "^2.3.0", + "terser-webpack-plugin": "^5.3.16", + "watchpack": "^2.5.1", + "webpack-sources": "^3.3.3" }, "bin": { "webpack": "bin/webpack.js" @@ -5319,7 +5363,6 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -5494,9 +5537,9 @@ } }, "node_modules/webpack-sources": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", - "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.3.3.tgz", + "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", "dev": true, "license": "MIT", "engines": { diff --git a/wasm/demo/package.json b/wasm/demo/package.json index 2c08e5c416f..7954e8cd866 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -17,7 +17,7 @@ "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", "serve": "^14.2.5", - "webpack": "^5.97.1", + "webpack": "^5.105.0", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" }, diff --git a/wasm/example/package.json b/wasm/example/package.json index f6128263017..40b70e0b75d 100644 --- a/wasm/example/package.json +++ b/wasm/example/package.json @@ -6,7 +6,7 @@ }, "devDependencies": { "raw-loader": "1.0.0", - "webpack": "5.94.0", + "webpack": "5.104.1", "webpack-cli": "^3.1.2" }, "scripts": {